diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 89447963b8528..8d1faab13986c 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -56,7 +56,7 @@ runtime_targets_needs_reconfig="${5}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
-echo "--- cmake"
+echo "::group::cmake"
 export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
@@ -85,38 +85,49 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON \
       -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
 
-echo "--- ninja"
+echo "::endgroup::"
+echo "::group::ninja"
+
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
+echo "::endgroup::"
+
 if [[ "${runtime_targets}" != "" ]]; then
-  echo "--- ninja runtimes"
+  echo "::group::ninja runtimes"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets}
+
+  echo "::endgroup::"
 fi
 
 # Compiling runtimes with just-built Clang and running their tests
 # as an additional testing for Clang.
 if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
-  echo "--- cmake runtimes C++26"
+  echo "::group::cmake runtimes C++26"
 
   cmake \
     -D LIBCXX_TEST_PARAMS="std=c++26" \
     -D LIBCXXABI_TEST_PARAMS="std=c++26" \
     "${BUILD_DIR}"
 
-  echo "--- ninja runtimes C++26"
+  echo "::endgroup::"
+  echo "::group::ninja runtimes C++26"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
 
-  echo "--- cmake runtimes clang modules"
+  echo "::endgroup::"
+  echo "::group::cmake runtimes clang modules"
 
   cmake \
     -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
     -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
     "${BUILD_DIR}"
 
-  echo "--- ninja runtimes clang modules"
+  echo "::endgroup::"
+  echo "::group::ninja runtimes clang modules"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
+
+  echo "::endgroup::"
 fi
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index dc2913830e929..c27111bf5aa10 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -46,7 +46,7 @@ trap at-exit EXIT
 projects="${1}"
 targets="${2}"
 
-echo "--- cmake"
+echo "::group::cmake"
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
 export CC=cl
@@ -78,6 +78,10 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_PARALLEL_COMPILE_JOBS=${MAX_PARALLEL_COMPILE_JOBS} \
       -D LLVM_PARALLEL_LINK_JOBS=${MAX_PARALLEL_LINK_JOBS}
 
-echo "--- ninja"
+echo "::endgroup::"
+echo "::group::ninja"
+
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
+
+echo "::endgroup"
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
new file mode 100644
index 0000000000000..03748938700e3
--- /dev/null
+++ b/.github/copilot-instructions.md
@@ -0,0 +1,4 @@
+When performing a code review, pay close attention to code modifying a function's
+control flow. Could the change result in the corruption of performance profile
+data? Could the change result in invalid debug information, in particular for
+branches and calls?
diff --git a/.github/workflows/containers/github-action-ci-windows/Dockerfile b/.github/workflows/containers/github-action-ci-windows/Dockerfile
index eafe45fac8ea4..c49b6b25ffb68 100644
--- a/.github/workflows/containers/github-action-ci-windows/Dockerfile
+++ b/.github/workflows/containers/github-action-ci-windows/Dockerfile
@@ -85,7 +85,7 @@ RUN powershell -Command \
 RUN git config --system core.longpaths true & \
     git config --global core.autocrlf false
 
-ARG RUNNER_VERSION=2.325.0
+ARG RUNNER_VERSION=2.326.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \
diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile
index f2716f0a9c6a0..197bf083a20c0 100644
--- a/.github/workflows/containers/github-action-ci/Dockerfile
+++ b/.github/workflows/containers/github-action-ci/Dockerfile
@@ -86,7 +86,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.325.0
+ENV GITHUB_RUNNER_VERSION=2.326.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \
diff --git a/.github/workflows/hlsl-test-all.yaml b/.github/workflows/hlsl-test-all.yaml
index 93a1c6d2662d4..b6530fe11b840 100644
--- a/.github/workflows/hlsl-test-all.yaml
+++ b/.github/workflows/hlsl-test-all.yaml
@@ -43,13 +43,13 @@ jobs:
       - name: Checkout OffloadTest
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: llvm-beanz/offload-test-suite
+          repository: llvm/offload-test-suite
           ref: main
           path: OffloadTest
       - name: Checkout Golden Images
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: llvm-beanz/offload-golden-images
+          repository: llvm/offload-golden-images
           ref: main
           path: golden-images
       - name: Setup Windows
diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml
index 24d75f58d45e0..9b2d8dd579702 100644
--- a/.github/workflows/libc-fullbuild-tests.yml
+++ b/.github/workflows/libc-fullbuild-tests.yml
@@ -15,9 +15,25 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build_type: [Debug, Release, MinSizeRel]
+        # Build basic linux configuration with Debug/Release/MinSizeRel and all
+        # other configurations in Debug only.
         include:
           - os: ubuntu-24.04
+            build_type: Debug
+            ccache-variant: sccache
+            c_compiler: clang-21
+            cpp_compiler: clang++-21
+            target: x86_64-unknown-linux-llvm
+            include_scudo: ON
+          - os: ubuntu-24.04
+            build_type: Release
+            ccache-variant: sccache
+            c_compiler: clang-21
+            cpp_compiler: clang++-21
+            target: x86_64-unknown-linux-llvm
+            include_scudo: ON
+          - os: ubuntu-24.04
+            build_type: MinSizeRel
             ccache-variant: sccache
             c_compiler: clang-21
             cpp_compiler: clang++-21
@@ -25,12 +41,14 @@ jobs:
             include_scudo: ON
           # TODO: remove ccache logic when https://github.com/hendrikmuhs/ccache-action/issues/279 is resolved.
           - os: ubuntu-24.04-arm
+            build_type: Debug
             ccache-variant: ccache
             c_compiler: clang-21
             cpp_compiler: clang++-21
             target: aarch64-unknown-linux-llvm
             include_scudo: ON
           - os: ubuntu-24.04
+            build_type: Debug
             ccache-variant: ccache
             c_compiler: clang-21
             cpp_compiler: clang++-21
@@ -97,7 +115,7 @@ jobs:
         -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.ccache-variant }} \
         -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.ccache-variant }} \
         -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.build-install-dir }} \
-        -DLLVM_RUNTIMES_TARGET=${{ matrix.target }} \
+        -DLLVM_RUNTIME_TARGETS=${{ matrix.target }} \
         -DLLVM_ENABLE_RUNTIMES="$RUNTIMES" \
         -DLLVM_LIBC_FULL_BUILD=ON \
         -G Ninja \
diff --git a/.github/workflows/libc-overlay-tests.yml b/.github/workflows/libc-overlay-tests.yml
index da82d8d9fe8ab..f001daae030a3 100644
--- a/.github/workflows/libc-overlay-tests.yml
+++ b/.github/workflows/libc-overlay-tests.yml
@@ -16,7 +16,7 @@ jobs:
       # Set fail-fast to false to ensure that feedback is delivered for all matrix combinations.
       fail-fast: false
       matrix:
-        build_type: [Debug, Release, MinSizeRel]
+        os: [ubuntu-24.04, ubuntu-24.04-arm, windows-2022, windows-2025, macos-14]
         include:
           # TODO: add linux gcc when it is fixed
           - os: ubuntu-24.04
@@ -96,7 +96,7 @@ jobs:
         cmake -B ${{ steps.strings.outputs.build-output-dir }}
         -DCMAKE_CXX_COMPILER=${{ matrix.compiler.cpp_compiler }}
         -DCMAKE_C_COMPILER=${{ matrix.compiler.c_compiler }}
-        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
+        -DCMAKE_BUILD_TYPE=Debug
         -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
         -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
         -DCMAKE_POLICY_DEFAULT_CMP0141=NEW
@@ -110,7 +110,6 @@ jobs:
         cmake 
         --build ${{ steps.strings.outputs.build-output-dir }} 
         --parallel 
-        --config MinSizeRel 
         --target libc
 
     - name: Test
diff --git a/.github/workflows/libcxx-restart-preempted-jobs.yaml b/.github/workflows/libcxx-restart-preempted-jobs.yaml
index 9706f0459922e..accb84efb5c90 100644
--- a/.github/workflows/libcxx-restart-preempted-jobs.yaml
+++ b/.github/workflows/libcxx-restart-preempted-jobs.yaml
@@ -20,7 +20,7 @@ permissions:
 
 jobs:
   restart:
-    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled')
+    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure')
     name: "Restart Job"
     permissions:
       statuses: read
@@ -32,7 +32,10 @@ jobs:
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
         with:
           script: |
-            const failure_regex = /Process completed with exit code 1./
+            // The "The run was canceled by" message comes from a user manually canceling a workflow
+            // the "higher priority" message comes from github canceling a workflow because the user updated the change.
+            // And the "exit code 1" message indicates a genuine failure.
+            const failure_regex = /(Process completed with exit code 1.)/
             const preemption_regex = /(The runner has received a shutdown signal)|(The operation was canceled)/
 
             const wf_run = context.payload.workflow_run
@@ -74,7 +77,7 @@ jobs:
                 console.log('Check run was not completed. Skipping.');
                 continue;
               }
-              if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') {
+              if (check_run.conclusion != 'failure') {
                 console.log('Check run had conclusion: ' + check_run.conclusion + '. Skipping.');
                 continue;
               }
@@ -153,91 +156,3 @@ jobs:
                 run_id: context.payload.workflow_run.id
               })
             await create_check_run('success', 'Restarted workflow run due to preempted job')
-
-  restart-test:
-    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled') && github.event.actor.login == 'ldionne' # TESTING ONLY
-    name: "Restart Job (test)"
-    permissions:
-      statuses: read
-      checks: write
-      actions: write
-    runs-on: ubuntu-24.04
-    steps:
-      - name: "Restart Job (test)"
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
-        with:
-          script: |
-            const FAILURE_REGEX = /Process completed with exit code 1./
-            const PREEMPTION_REGEX = /(The runner has received a shutdown signal)|(The operation was canceled)/
-
-            function log(msg) {
-              core.notice(msg)
-            }
-
-            const wf_run = context.payload.workflow_run
-            log(`Running on "${wf_run.display_title}" by @${wf_run.actor.login} (event: ${wf_run.event})\nWorkflow run URL: ${wf_run.html_url}`)
-
-            log('Listing check runs for suite')
-            const check_suites = await github.rest.checks.listForSuite({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              check_suite_id: context.payload.workflow_run.check_suite_id,
-              per_page: 100 // FIXME: We don't have 100 check runs yet, but we should handle this better.
-            })
-
-            preemptions = [];
-            legitimate_failures = [];
-            for (check_run of check_suites.data.check_runs) {
-              log(`Checking check run: ${check_run.id}`);
-              if (check_run.status != 'completed') {
-                log('Check run was not completed. Skipping.');
-                continue;
-              }
-
-              if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') {
-                log(`Check run had conclusion: ${check_run.conclusion}. Skipping.`);
-                continue;
-              }
-
-              annotations = await github.rest.checks.listAnnotations({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                check_run_id: check_run.id
-              })
-
-              preemption_annotation = annotations.data.find(function(annotation) {
-                return annotation.annotation_level == 'failure' &&
-                       annotation.message.match(PREEMPTION_REGEX) != null;
-              });
-              if (preemption_annotation != null) {
-                log(`Found preemption message: ${preemption_annotation.message}`);
-                preemptions.push(check_run);
-                break;
-              }
-
-              failure_annotation = annotations.data.find(function(annotation) {
-                return annotation.annotation_level == 'failure' &&
-                       annotation.message.match(FAILURE_REGEX) != null;
-              });
-              if (failure_annotation != null) {
-                log(`Found legitimate failure annotation: ${failure_annotation.message}`);
-                legitimate_failures.push(check_run);
-                break;
-              }
-            }
-
-            if (preemptions) {
-              log('Found some preempted jobs');
-              if (legitimate_failures) {
-                log('Also found some legitimate failures, so not restarting the workflow.');
-              } else {
-                log('Did not find any legitimate failures. Restarting workflow.');
-                await github.rest.actions.reRunWorkflowFailedJobs({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  run_id: context.payload.workflow_run.id
-                })
-              }
-            } else {
-              log('Did not find any preempted jobs. Not restarting the workflow.');
-            }
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 64e524e2f13d7..91ecf89da618c 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -73,14 +73,15 @@ struct SegmentInfo {
   uint64_t FileSize;          /// Size in file.
   uint64_t Alignment;         /// Alignment of the segment.
   bool IsExecutable;          /// Is the executable bit set on the Segment?
+  bool IsWritable;            /// Is the segment writable.
 
   void print(raw_ostream &OS) const {
     OS << "SegmentInfo { Address: 0x" << Twine::utohexstr(Address)
        << ", Size: 0x" << Twine::utohexstr(Size) << ", FileOffset: 0x"
        << Twine::utohexstr(FileOffset) << ", FileSize: 0x"
        << Twine::utohexstr(FileSize) << ", Alignment: 0x"
-       << Twine::utohexstr(Alignment) << ", " << (IsExecutable ? "x" : " ")
-       << "}";
+       << Twine::utohexstr(Alignment) << ", " << (IsExecutable ? "x" : "")
+       << (IsWritable ? "w" : "") << " }";
   };
 };
 
@@ -333,9 +334,14 @@ class BinaryContext {
                                   std::optional<StringRef> Source,
                                   unsigned CUID, unsigned DWARFVersion);
 
+  /// Input file segment info
+  ///
   /// [start memory address] -> [segment info] mapping.
   std::map<uint64_t, SegmentInfo> SegmentMapInfo;
 
+  /// Newly created segments.
+  std::vector<SegmentInfo> NewSegments;
+
   /// Symbols that are expected to be undefined in MCContext during emission.
   std::unordered_set<MCSymbol *> UndefinedSymbols;
 
diff --git a/bolt/include/bolt/Core/DIEBuilder.h b/bolt/include/bolt/Core/DIEBuilder.h
index 32e455ad3030a..e4a4fc6b2f258 100644
--- a/bolt/include/bolt/Core/DIEBuilder.h
+++ b/bolt/include/bolt/Core/DIEBuilder.h
@@ -20,8 +20,8 @@
 #include "llvm/CodeGen/DIE.h"
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Allocator.h"
 
 #include <list>
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 804100db80793..b99cf8b40ef54 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1393,7 +1393,7 @@ class MCPlusBuilder {
       return getTargetSymbol(BinaryExpr->getLHS());
 
     auto *SymbolRefExpr = dyn_cast<const MCSymbolRefExpr>(Expr);
-    if (SymbolRefExpr && SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None)
+    if (SymbolRefExpr && SymbolRefExpr->getSpecifier() == 0)
       return &SymbolRefExpr->getSymbol();
 
     return nullptr;
diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index 98e4bba872846..c7400b0d45ca8 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -236,8 +236,7 @@ class DataAggregator : public DataReader {
 
   /// Launch a perf subprocess with given args and save output for later
   /// parsing.
-  void launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
-                         const char *ArgsString, bool Wait);
+  void launchPerfProcess(StringRef Name, PerfProcessInfo &PPI, StringRef Args);
 
   /// Delete all temporary files created to hold the output generated by spawned
   /// subprocesses during the aggregation job
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 94dd06e91d6b1..91d62a78de390 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -202,6 +202,9 @@ class RewriteInstance {
   /// Map code sections generated by BOLT.
   void mapCodeSections(BOLTLinker::SectionMapper MapSection);
 
+  /// Map code without relocating sections.
+  void mapCodeSectionsInPlace(BOLTLinker::SectionMapper MapSection);
+
   /// Map the rest of allocatable sections.
   void mapAllocatableSections(BOLTLinker::SectionMapper MapSection);
 
@@ -297,6 +300,9 @@ class RewriteInstance {
     return FUNC(ELF64BE);                                                      \
   }
 
+  /// Update loadable segment information based on new sections.
+  void updateSegmentInfo();
+
   /// Patch ELF book-keeping info.
   void patchELFPHDRTable();
 
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index b998d7160aae7..eec68ff5a5fce 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -3328,10 +3328,7 @@ void BinaryFunction::duplicateConstantIslands() {
 
         // Update instruction reference
         Operand = MCOperand::createExpr(BC.MIB->getTargetExprFor(
-            Inst,
-            MCSymbolRefExpr::create(ColdSymbol, MCSymbolRefExpr::VK_None,
-                                    *BC.Ctx),
-            *BC.Ctx, 0));
+            Inst, MCSymbolRefExpr::create(ColdSymbol, *BC.Ctx), *BC.Ctx, 0));
         ++OpNum;
       }
     }
diff --git a/bolt/lib/Core/CMakeLists.txt b/bolt/lib/Core/CMakeLists.txt
index 8c1f5d0bb37b5..fc72dc023c590 100644
--- a/bolt/lib/Core/CMakeLists.txt
+++ b/bolt/lib/Core/CMakeLists.txt
@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   Demangle
   MC
   MCDisassembler
diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp
index d36dbb3459249..b041dc5ea1cce 100644
--- a/bolt/lib/Core/DIEBuilder.cpp
+++ b/bolt/lib/Core/DIEBuilder.cpp
@@ -14,11 +14,11 @@
 #include "llvm/CodeGen/DIE.h"
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp
index aa1c8f3d42d4b..a9d98a6ba879b 100644
--- a/bolt/lib/Core/DebugNames.cpp
+++ b/bolt/lib/Core/DebugNames.cpp
@@ -8,8 +8,8 @@
 
 #include "bolt/Core/DebugNames.h"
 #include "bolt/Core/BinaryContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/LEB128.h"
 #include <cstdint>
diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp
index fbf889279f1c0..c2f876f0dff9e 100644
--- a/bolt/lib/Passes/Instrumentation.cpp
+++ b/bolt/lib/Passes/Instrumentation.cpp
@@ -666,8 +666,7 @@ Error Instrumentation::runOnFunctions(BinaryContext &BC) {
       auto IsLEA = [&BC](const MCInst &Inst) { return BC.MIB->isLEA64r(Inst); };
       const auto LEA = std::find_if(
           std::next(llvm::find_if(reverse(BB), IsLEA)), BB.rend(), IsLEA);
-      LEA->getOperand(4).setExpr(
-          MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *BC.Ctx));
+      LEA->getOperand(4).setExpr(MCSymbolRefExpr::create(Target, *BC.Ctx));
     } else {
       BC.errs() << "BOLT-WARNING: ___GLOBAL_init_65535 not found\n";
     }
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 88229bb31a2ad..c60591c02b6c0 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -197,34 +197,27 @@ void DataAggregator::start() {
 
   if (opts::BasicAggregation) {
     launchPerfProcess("events without LBR", MainEventsPPI,
-                      "script -F pid,event,ip",
-                      /*Wait = */ false);
+                      "script -F pid,event,ip");
   } else if (!opts::ITraceAggregation.empty()) {
     // Disable parsing memory profile from trace data, unless requested by user.
     if (!opts::ParseMemProfile.getNumOccurrences())
       opts::ParseMemProfile = false;
-
-    std::string ItracePerfScriptArgs = llvm::formatv(
-        "script -F pid,brstack --itrace={0}", opts::ITraceAggregation);
     launchPerfProcess("branch events with itrace", MainEventsPPI,
-                      ItracePerfScriptArgs.c_str(),
-                      /*Wait = */ false);
+                      "script -F pid,brstack --itrace=" +
+                          opts::ITraceAggregation);
   } else {
-    launchPerfProcess("branch events", MainEventsPPI, "script -F pid,brstack",
-                      /*Wait = */ false);
+    launchPerfProcess("branch events", MainEventsPPI, "script -F pid,brstack");
   }
 
   if (opts::ParseMemProfile)
-    launchPerfProcess("mem events", MemEventsPPI, "script -F pid,event,addr,ip",
-                      /*Wait = */ false);
+    launchPerfProcess("mem events", MemEventsPPI,
+                      "script -F pid,event,addr,ip");
 
   launchPerfProcess("process events", MMapEventsPPI,
-                    "script --show-mmap-events --no-itrace",
-                    /*Wait = */ false);
+                    "script --show-mmap-events --no-itrace");
 
   launchPerfProcess("task events", TaskEventsPPI,
-                    "script --show-task-events --no-itrace",
-                    /*Wait = */ false);
+                    "script --show-task-events --no-itrace");
 }
 
 void DataAggregator::abort() {
@@ -246,13 +239,13 @@ void DataAggregator::abort() {
 }
 
 void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
-                                       const char *ArgsString, bool Wait) {
+                                       StringRef Args) {
   SmallVector<StringRef, 4> Argv;
 
   outs() << "PERF2BOLT: spawning perf job to read " << Name << '\n';
   Argv.push_back(PerfPath.data());
 
-  StringRef(ArgsString).split(Argv, ' ');
+  Args.split(Argv, ' ');
   Argv.push_back("-f");
   Argv.push_back("-i");
   Argv.push_back(Filename.c_str());
@@ -286,64 +279,45 @@ void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
            << "\n";
   });
 
-  if (Wait)
-    PPI.PI.ReturnCode = sys::ExecuteAndWait(PerfPath.data(), Argv,
-                                            /*envp*/ std::nullopt, Redirects);
-  else
-    PPI.PI = sys::ExecuteNoWait(PerfPath.data(), Argv, /*envp*/ std::nullopt,
-                                Redirects);
+  PPI.PI = sys::ExecuteNoWait(PerfPath.data(), Argv, /*envp*/ std::nullopt,
+                              Redirects);
 }
 
 void DataAggregator::processFileBuildID(StringRef FileBuildID) {
+  auto WarningCallback = [](int ReturnCode, StringRef ErrBuf) {
+    errs() << "PERF-ERROR: return code " << ReturnCode << "\n" << ErrBuf;
+  };
+
   PerfProcessInfo BuildIDProcessInfo;
-  launchPerfProcess("buildid list",
-                    BuildIDProcessInfo,
-                    "buildid-list",
-                    /*Wait = */true);
-
-  if (BuildIDProcessInfo.PI.ReturnCode != 0) {
-    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
-        MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StderrPath.data());
-    StringRef ErrBuf = (*MB)->getBuffer();
-
-    errs() << "PERF-ERROR: return code " << BuildIDProcessInfo.PI.ReturnCode
-           << '\n';
-    errs() << ErrBuf;
+  launchPerfProcess("buildid list", BuildIDProcessInfo, "buildid-list");
+  if (prepareToParse("buildid", BuildIDProcessInfo, WarningCallback))
     return;
-  }
 
-  ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
-      MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StdoutPath.data());
-  if (std::error_code EC = MB.getError()) {
-    errs() << "Cannot open " << BuildIDProcessInfo.StdoutPath.data() << ": "
-           << EC.message() << "\n";
+  std::optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);
+  if (FileName && *FileName == sys::path::filename(BC->getFilename())) {
+    outs() << "PERF2BOLT: matched build-id and file name\n";
     return;
   }
 
-  FileBuf = std::move(*MB);
-  ParsingBuf = FileBuf->getBuffer();
-
-  std::optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);
-  if (!FileName) {
-    if (hasAllBuildIDs()) {
-      errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "
-                "This indicates the input binary supplied for data aggregation "
-                "is not the same recorded by perf when collecting profiling "
-                "data, or there were no samples recorded for the binary. "
-                "Use -ignore-build-id option to override.\n";
-      if (!opts::IgnoreBuildID)
-        abort();
-    } else {
-      errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
-                "data was recorded without it\n";
-      return;
-    }
-  } else if (*FileName != llvm::sys::path::filename(BC->getFilename())) {
+  if (FileName) {
     errs() << "PERF2BOLT-WARNING: build-id matched a different file name\n";
     BuildIDBinaryName = std::string(*FileName);
-  } else {
-    outs() << "PERF2BOLT: matched build-id and file name\n";
+    return;
   }
+
+  if (!hasAllBuildIDs()) {
+    errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
+              "data was recorded without it\n";
+    return;
+  }
+
+  errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "
+            "This indicates the input binary supplied for data aggregation "
+            "is not the same recorded by perf when collecting profiling "
+            "data, or there were no samples recorded for the binary. "
+            "Use -ignore-build-id option to override.\n";
+  if (!opts::IgnoreBuildID)
+    abort();
 }
 
 bool DataAggregator::checkPerfDataMagic(StringRef FileName) {
@@ -432,7 +406,8 @@ int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,
   std::string Error;
   outs() << "PERF2BOLT: waiting for perf " << Name
          << " collection to finish...\n";
-  sys::ProcessInfo PI = sys::Wait(Process.PI, std::nullopt, &Error);
+  std::optional<sys::ProcessStatistics> PS;
+  sys::ProcessInfo PI = sys::Wait(Process.PI, std::nullopt, &Error, &PS);
 
   if (!Error.empty()) {
     errs() << "PERF-ERROR: " << PerfPath << ": " << Error << "\n";
@@ -440,6 +415,15 @@ int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,
     exit(1);
   }
 
+  LLVM_DEBUG({
+    const float UserSec = 1.f * PS->UserTime.count() / 1e6;
+    const float TotalSec = 1.f * PS->TotalTime.count() / 1e6;
+    const float PeakGiB = 1.f * PS->PeakMemory / (1 << 20);
+    dbgs() << formatv("Finished in {0:f2}s user time, {1:f2}s total time, "
+                      "{2:f2} GiB peak RSS\n",
+                      UserSec, TotalSec, PeakGiB);
+  });
+
   if (PI.ReturnCode != 0) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorMB =
         MemoryBuffer::getFileOrSTDIN(Process.StderrPath.data());
diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt
index c83cf36982167..775036063dd5a 100644
--- a/bolt/lib/Rewrite/CMakeLists.txt
+++ b/bolt/lib/Rewrite/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   Core
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   JITLink
   MC
   Object
diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 9c9bdefe08429..0c1a1bac6c72e 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -24,10 +24,10 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCObjectWriter.h"
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 934768c244b31..96045a916232c 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -547,9 +547,14 @@ Error RewriteInstance::discoverStorage() {
       NextAvailableOffset = std::max(NextAvailableOffset,
                                      Phdr.p_offset + Phdr.p_filesz);
 
-      BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
-          Phdr.p_vaddr,  Phdr.p_memsz, Phdr.p_offset,
-          Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)};
+      BC->SegmentMapInfo[Phdr.p_vaddr] =
+          SegmentInfo{Phdr.p_vaddr,
+                      Phdr.p_memsz,
+                      Phdr.p_offset,
+                      Phdr.p_filesz,
+                      Phdr.p_align,
+                      (Phdr.p_flags & ELF::PF_X) != 0,
+                      (Phdr.p_flags & ELF::PF_W) != 0};
       if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
           Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
         BC->IsLinuxKernel = true;
@@ -3856,111 +3861,138 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() {
 }
 
 void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
-  if (BC->HasRelocations) {
-    // Map sections for functions with pre-assigned addresses.
-    for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
-      const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
-      if (!OutputAddress)
-        continue;
-
-      ErrorOr<BinarySection &> FunctionSection =
-          InjectedFunction->getCodeSection();
-      assert(FunctionSection && "function should have section");
-      FunctionSection->setOutputAddress(OutputAddress);
-      MapSection(*FunctionSection, OutputAddress);
-      InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
-      InjectedFunction->setImageSize(FunctionSection->getOutputSize());
-    }
+  if (!BC->HasRelocations) {
+    mapCodeSectionsInPlace(MapSection);
+    return;
+  }
 
-    // Populate the list of sections to be allocated.
-    std::vector<BinarySection *> CodeSections = getCodeSections();
+  // Map sections for functions with pre-assigned addresses.
+  for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
+    const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
+    if (!OutputAddress)
+      continue;
 
-    // Remove sections that were pre-allocated (patch sections).
-    llvm::erase_if(CodeSections, [](BinarySection *Section) {
-      return Section->getOutputAddress();
-    });
-    LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
-      for (const BinarySection *Section : CodeSections)
-        dbgs() << Section->getName() << '\n';
-    );
+    ErrorOr<BinarySection &> FunctionSection =
+        InjectedFunction->getCodeSection();
+    assert(FunctionSection && "function should have section");
+    FunctionSection->setOutputAddress(OutputAddress);
+    MapSection(*FunctionSection, OutputAddress);
+    InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
+    InjectedFunction->setImageSize(FunctionSection->getOutputSize());
+  }
 
-    uint64_t PaddingSize = 0; // size of padding required at the end
+  // Populate the list of sections to be allocated.
+  std::vector<BinarySection *> CodeSections = getCodeSections();
 
-    // Allocate sections starting at a given Address.
-    auto allocateAt = [&](uint64_t Address) {
-      const char *LastNonColdSectionName = BC->HasWarmSection
-                                               ? BC->getWarmCodeSectionName()
-                                               : BC->getMainCodeSectionName();
-      for (BinarySection *Section : CodeSections) {
-        Address = alignTo(Address, Section->getAlignment());
-        Section->setOutputAddress(Address);
-        Address += Section->getOutputSize();
-
-        // Hugify: Additional huge page from right side due to
-        // weird ASLR mapping addresses (4KB aligned)
-        if (opts::Hugify && !BC->HasFixedLoadAddress &&
-            Section->getName() == LastNonColdSectionName)
-          Address = alignTo(Address, Section->getAlignment());
-      }
+  // Remove sections that were pre-allocated (patch sections).
+  llvm::erase_if(CodeSections, [](BinarySection *Section) {
+    return Section->getOutputAddress();
+  });
+  LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
+             for (const BinarySection *Section : CodeSections) dbgs()
+             << Section->getName() << '\n';);
 
-      // Make sure we allocate enough space for huge pages.
-      ErrorOr<BinarySection &> TextSection =
-          BC->getUniqueSectionByName(LastNonColdSectionName);
-      if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
-        uint64_t HotTextEnd =
-            TextSection->getOutputAddress() + TextSection->getOutputSize();
-        HotTextEnd = alignTo(HotTextEnd, BC->PageAlign);
-        if (HotTextEnd > Address) {
-          PaddingSize = HotTextEnd - Address;
-          Address = HotTextEnd;
-        }
-      }
-      return Address;
-    };
+  uint64_t PaddingSize = 0; // size of padding required at the end
 
-    // Check if we can fit code in the original .text
-    bool AllocationDone = false;
-    if (opts::UseOldText) {
-      const uint64_t CodeSize =
-          allocateAt(BC->OldTextSectionAddress) - BC->OldTextSectionAddress;
+  // Allocate sections starting at a given Address.
+  auto allocateAt = [&](uint64_t Address) {
+    const char *LastNonColdSectionName = BC->HasWarmSection
+                                             ? BC->getWarmCodeSectionName()
+                                             : BC->getMainCodeSectionName();
+    for (BinarySection *Section : CodeSections) {
+      Address = alignTo(Address, Section->getAlignment());
+      Section->setOutputAddress(Address);
+      Address += Section->getOutputSize();
+
+      // Hugify: Additional huge page from right side due to
+      // weird ASLR mapping addresses (4KB aligned)
+      if (opts::Hugify && !BC->HasFixedLoadAddress &&
+          Section->getName() == LastNonColdSectionName)
+        Address = alignTo(Address, Section->getAlignment());
+    }
 
-      if (CodeSize <= BC->OldTextSectionSize) {
-        BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
-                   << Twine::utohexstr(opts::AlignText) << " alignment\n";
-        AllocationDone = true;
-      } else {
-        BC->errs()
-            << "BOLT-WARNING: original .text too small to fit the new code"
-            << " using 0x" << Twine::utohexstr(opts::AlignText)
-            << " alignment. " << CodeSize << " bytes needed, have "
-            << BC->OldTextSectionSize << " bytes available.\n";
-        opts::UseOldText = false;
+    // Make sure we allocate enough space for huge pages.
+    ErrorOr<BinarySection &> TextSection =
+        BC->getUniqueSectionByName(LastNonColdSectionName);
+    if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
+      uint64_t HotTextEnd =
+          TextSection->getOutputAddress() + TextSection->getOutputSize();
+      HotTextEnd = alignTo(HotTextEnd, BC->PageAlign);
+      if (HotTextEnd > Address) {
+        PaddingSize = HotTextEnd - Address;
+        Address = HotTextEnd;
       }
     }
+    return Address;
+  };
 
-    if (!AllocationDone)
-      NextAvailableAddress = allocateAt(NextAvailableAddress);
+  // Try to allocate sections before the \p Address and return an address for
+  // the allocation of the first section, or 0 if [0, Address) range is not
+  // big enough to fit all sections.
+  auto allocateBefore = [&](uint64_t Address) -> uint64_t {
+    for (BinarySection *Section : llvm::reverse(CodeSections)) {
+      if (Section->getOutputSize() > Address)
+        return 0;
+      Address -= Section->getOutputSize();
+      Address = alignDown(Address, Section->getAlignment());
+      Section->setOutputAddress(Address);
+    }
+    return Address;
+  };
 
-    // Do the mapping for ORC layer based on the allocation.
-    for (BinarySection *Section : CodeSections) {
-      LLVM_DEBUG(
-          dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
-                 << Twine::utohexstr(Section->getAllocAddress()) << " to 0x"
-                 << Twine::utohexstr(Section->getOutputAddress()) << '\n');
-      MapSection(*Section, Section->getOutputAddress());
-      Section->setOutputFileOffset(
-          getFileOffsetForAddress(Section->getOutputAddress()));
+  // Check if we can fit code in the original .text
+  bool AllocationDone = false;
+  if (opts::UseOldText) {
+    uint64_t StartAddress;
+    uint64_t EndAddress;
+    if (opts::HotFunctionsAtEnd) {
+      EndAddress = BC->OldTextSectionAddress + BC->OldTextSectionSize;
+      StartAddress = allocateBefore(EndAddress);
+    } else {
+      StartAddress = BC->OldTextSectionAddress;
+      EndAddress = allocateAt(BC->OldTextSectionAddress);
+    }
+
+    const uint64_t CodeSize = EndAddress - StartAddress;
+    if (CodeSize <= BC->OldTextSectionSize) {
+      BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
+                 << Twine::utohexstr(opts::AlignText) << " alignment";
+      if (StartAddress != BC->OldTextSectionAddress)
+        BC->outs() << " at 0x" << Twine::utohexstr(StartAddress);
+      BC->outs() << '\n';
+      AllocationDone = true;
+    } else {
+      BC->errs() << "BOLT-WARNING: original .text too small to fit the new code"
+                 << " using 0x" << Twine::utohexstr(opts::AlignText)
+                 << " alignment. " << CodeSize << " bytes needed, have "
+                 << BC->OldTextSectionSize << " bytes available.\n";
+      opts::UseOldText = false;
     }
+  }
 
-    // Check if we need to insert a padding section for hot text.
-    if (PaddingSize && !opts::UseOldText)
-      BC->outs() << "BOLT-INFO: padding code to 0x"
-                 << Twine::utohexstr(NextAvailableAddress)
-                 << " to accommodate hot text\n";
+  if (!AllocationDone)
+    NextAvailableAddress = allocateAt(NextAvailableAddress);
 
-    return;
+  // Do the mapping for ORC layer based on the allocation.
+  for (BinarySection *Section : CodeSections) {
+    LLVM_DEBUG(dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
+                      << Twine::utohexstr(Section->getAllocAddress())
+                      << " to 0x"
+                      << Twine::utohexstr(Section->getOutputAddress()) << '\n');
+    MapSection(*Section, Section->getOutputAddress());
+    Section->setOutputFileOffset(
+        getFileOffsetForAddress(Section->getOutputAddress()));
   }
 
+  // Check if we need to insert a padding section for hot text.
+  if (PaddingSize && !opts::UseOldText)
+    BC->outs() << "BOLT-INFO: padding code to 0x"
+               << Twine::utohexstr(NextAvailableAddress)
+               << " to accommodate hot text\n";
+}
+
+void RewriteInstance::mapCodeSectionsInPlace(
+    BOLTLinker::SectionMapper MapSection) {
   // Processing in non-relocation mode.
   uint64_t NewTextSectionStartAddress = NextAvailableAddress;
 
@@ -4155,6 +4187,74 @@ void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {
     Function->updateOutputValues(Linker);
 }
 
+void RewriteInstance::updateSegmentInfo() {
+  // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
+  // last segments size based on the NextAvailableAddress variable.
+  if (!NewWritableSegmentSize) {
+    if (NewTextSegmentAddress)
+      NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
+  } else {
+    NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
+  }
+
+  if (NewTextSegmentSize) {
+    SegmentInfo TextSegment = {NewTextSegmentAddress,
+                               NewTextSegmentSize,
+                               NewTextSegmentOffset,
+                               NewTextSegmentSize,
+                               BC->PageAlign,
+                               true,
+                               false};
+    if (!opts::Instrument) {
+      BC->NewSegments.push_back(TextSegment);
+    } else {
+      ErrorOr<BinarySection &> Sec =
+          BC->getUniqueSectionByName(".bolt.instr.counters");
+      assert(Sec && "expected one and only one `.bolt.instr.counters` section");
+      const uint64_t Addr = Sec->getOutputAddress();
+      const uint64_t Offset = Sec->getOutputFileOffset();
+      const uint64_t Size = Sec->getOutputSize();
+      assert(Addr > TextSegment.Address &&
+             Addr + Size < TextSegment.Address + TextSegment.Size &&
+             "`.bolt.instr.counters` section is expected to be included in the "
+             "new text segment");
+
+      // Set correct size for the previous header since we are breaking the
+      // new text segment into three segments.
+      uint64_t Delta = Addr - TextSegment.Address;
+      TextSegment.Size = Delta;
+      TextSegment.FileSize = Delta;
+      BC->NewSegments.push_back(TextSegment);
+
+      // Create RW segment that includes the `.bolt.instr.counters` section.
+      SegmentInfo RWSegment = {Addr,  Size, Offset, Size, BC->RegularPageSize,
+                               false, true};
+      BC->NewSegments.push_back(RWSegment);
+
+      // Create RX segment that includes all RX sections from runtime library.
+      const uint64_t AddrRX = alignTo(Addr + Size, BC->RegularPageSize);
+      const uint64_t OffsetRX = alignTo(Offset + Size, BC->RegularPageSize);
+      const uint64_t SizeRX =
+          NewTextSegmentSize - (AddrRX - TextSegment.Address);
+      SegmentInfo RXSegment = {
+          AddrRX, SizeRX, OffsetRX, SizeRX, BC->RegularPageSize, true, false};
+      BC->NewSegments.push_back(RXSegment);
+    }
+  }
+
+  if (NewWritableSegmentSize) {
+    SegmentInfo DataSegmentInfo = {
+        NewWritableSegmentAddress,
+        NewWritableSegmentSize,
+        getFileOffsetForAddress(NewWritableSegmentAddress),
+        NewWritableSegmentSize,
+        BC->RegularPageSize,
+        false,
+        true};
+    BC->NewSegments.push_back(DataSegmentInfo);
+  }
+}
+
 void RewriteInstance::patchELFPHDRTable() {
   auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);
   const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
@@ -4181,16 +4281,7 @@ void RewriteInstance::patchELFPHDRTable() {
   if (opts::Instrument)
     Phnum += 2;
 
-  // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
-  // last segments size based on the NextAvailableAddress variable.
-  if (!NewWritableSegmentSize) {
-    if (NewTextSegmentAddress)
-      NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
-  } else {
-    NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
-  }
-
-  if (!NewTextSegmentSize && !NewWritableSegmentSize) {
+  if (BC->NewSegments.empty()) {
     BC->outs() << "BOLT-INFO: not adding new segments\n";
     return;
   }
@@ -4198,90 +4289,28 @@ void RewriteInstance::patchELFPHDRTable() {
   const uint64_t SavedPos = OS.tell();
   OS.seek(PHDRTableOffset);
 
-  auto createNewPhdrs = [&]() {
-    SmallVector<ELF64LEPhdrTy, 3> NewPhdrs;
-    ELF64LEPhdrTy NewPhdr;
-    NewPhdr.p_type = ELF::PT_LOAD;
-    NewPhdr.p_offset = NewTextSegmentOffset;
-    NewPhdr.p_vaddr = NewTextSegmentAddress;
-    NewPhdr.p_paddr = NewTextSegmentAddress;
-    NewPhdr.p_filesz = NewTextSegmentSize;
-    NewPhdr.p_memsz = NewTextSegmentSize;
-    NewPhdr.p_flags = ELF::PF_X | ELF::PF_R;
-    NewPhdr.p_align = BC->PageAlign;
-
-    if (!opts::Instrument) {
-      NewPhdrs.push_back(NewPhdr);
-    } else {
-      ErrorOr<BinarySection &> Sec =
-          BC->getUniqueSectionByName(".bolt.instr.counters");
-      assert(Sec && "expected one and only one `.bolt.instr.counters` section");
-      const uint64_t Addr = Sec->getOutputAddress();
-      const uint64_t Offset = Sec->getOutputFileOffset();
-      const uint64_t Size = Sec->getOutputSize();
-      assert(Addr > NewPhdr.p_vaddr &&
-             Addr + Size < NewPhdr.p_vaddr + NewPhdr.p_memsz &&
-             "`.bolt.instr.counters` section is expected to be included in the "
-             "new text sgement");
-
-      // Set correct size for the previous header since we are breaking the
-      // new text segment into three segments.
-      uint64_t Delta = Addr - NewPhdr.p_vaddr;
-      NewPhdr.p_filesz = Delta;
-      NewPhdr.p_memsz = Delta;
-      NewPhdrs.push_back(NewPhdr);
-
-      // Create a program header for a RW segment that includes the
-      // `.bolt.instr.counters` section only.
-      ELF64LEPhdrTy NewPhdrRWSegment;
-      NewPhdrRWSegment.p_type = ELF::PT_LOAD;
-      NewPhdrRWSegment.p_offset = Offset;
-      NewPhdrRWSegment.p_vaddr = Addr;
-      NewPhdrRWSegment.p_paddr = Addr;
-      NewPhdrRWSegment.p_filesz = Size;
-      NewPhdrRWSegment.p_memsz = Size;
-      NewPhdrRWSegment.p_flags = ELF::PF_R | ELF::PF_W;
-      NewPhdrRWSegment.p_align = BC->RegularPageSize;
-      NewPhdrs.push_back(NewPhdrRWSegment);
-
-      // Create a program header for a RX segment that includes all the RX
-      // sections from runtime library.
-      ELF64LEPhdrTy NewPhdrRXSegment;
-      NewPhdrRXSegment.p_type = ELF::PT_LOAD;
-      const uint64_t AddrRX = alignTo(Addr + Size, BC->RegularPageSize);
-      const uint64_t OffsetRX = alignTo(Offset + Size, BC->RegularPageSize);
-      const uint64_t SizeRX = NewTextSegmentSize - (AddrRX - NewPhdr.p_paddr);
-      NewPhdrRXSegment.p_offset = OffsetRX;
-      NewPhdrRXSegment.p_vaddr = AddrRX;
-      NewPhdrRXSegment.p_paddr = AddrRX;
-      NewPhdrRXSegment.p_filesz = SizeRX;
-      NewPhdrRXSegment.p_memsz = SizeRX;
-      NewPhdrRXSegment.p_flags = ELF::PF_X | ELF::PF_R;
-      NewPhdrRXSegment.p_align = BC->RegularPageSize;
-      NewPhdrs.push_back(NewPhdrRXSegment);
-    }
-
-    return NewPhdrs;
+  auto createPhdr = [](const SegmentInfo &SI) {
+    ELF64LEPhdrTy Phdr;
+    Phdr.p_type = ELF::PT_LOAD;
+    Phdr.p_offset = SI.FileOffset;
+    Phdr.p_vaddr = SI.Address;
+    Phdr.p_paddr = SI.Address;
+    Phdr.p_filesz = SI.FileSize;
+    Phdr.p_memsz = SI.Size;
+    Phdr.p_flags = ELF::PF_R;
+    if (SI.IsExecutable)
+      Phdr.p_flags |= ELF::PF_X;
+    if (SI.IsWritable)
+      Phdr.p_flags |= ELF::PF_W;
+    Phdr.p_align = SI.Alignment;
+
+    return Phdr;
   };
 
   auto writeNewSegmentPhdrs = [&]() {
-    if (NewTextSegmentSize) {
-      SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
-      OS.write(reinterpret_cast<const char *>(NewPhdrs.data()),
-               sizeof(ELF64LE::Phdr) * NewPhdrs.size());
-    }
-
-    if (NewWritableSegmentSize) {
-      ELF64LEPhdrTy NewPhdr;
-      NewPhdr.p_type = ELF::PT_LOAD;
-      NewPhdr.p_offset = getFileOffsetForAddress(NewWritableSegmentAddress);
-      NewPhdr.p_vaddr = NewWritableSegmentAddress;
-      NewPhdr.p_paddr = NewWritableSegmentAddress;
-      NewPhdr.p_filesz = NewWritableSegmentSize;
-      NewPhdr.p_memsz = NewWritableSegmentSize;
-      NewPhdr.p_align = BC->RegularPageSize;
-      NewPhdr.p_flags = ELF::PF_R | ELF::PF_W;
-      OS.write(reinterpret_cast<const char *>(&NewPhdr), sizeof(NewPhdr));
+    for (const SegmentInfo &SI : BC->NewSegments) {
+      ELF64LEPhdrTy Phdr = createPhdr(SI);
+      OS.write(reinterpret_cast<const char *>(&Phdr), sizeof(Phdr));
     }
   };
 
@@ -4317,11 +4346,9 @@ void RewriteInstance::patchELFPHDRTable() {
     case ELF::PT_GNU_STACK:
       if (opts::UseGnuStack) {
         // Overwrite the header with the new segment header.
-        assert(!opts::Instrument);
-        SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
-        assert(NewPhdrs.size() == 1 &&
-               "expect exactly one program header was created");
-        NewPhdr = NewPhdrs[0];
+        assert(BC->NewSegments.size() == 1 &&
+               "Expected exactly one new segment");
+        NewPhdr = createPhdr(BC->NewSegments.front());
         ModdedGnuStack = true;
       }
       break;
@@ -5946,8 +5973,10 @@ void RewriteInstance::rewriteFile() {
     addBATSection();
 
   // Patch program header table.
-  if (!BC->IsLinuxKernel)
+  if (!BC->IsLinuxKernel) {
+    updateSegmentInfo();
     patchELFPHDRTable();
+  }
 
   // Finalize memory image of section string table.
   finalizeSectionStringTable();
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 612c1304efd60..973261765f951 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -22,7 +22,6 @@
 #include "bolt/Core/MCPlusBuilder.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegister.h"
@@ -1206,8 +1205,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       OI = Inst.begin() + 2;
     }
 
-    *OI = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    *OI = MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   /// Matches indirect branch patterns in AArch64 related to a jump table (JT),
@@ -1633,8 +1631,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                           .addImm(0));
     Code.emplace_back(MCInstBuilder(AArch64::Bcc)
                           .addImm(AArch64CC::EQ)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx)));
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     return Code;
   }
 
@@ -1656,8 +1653,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                           .addImm(0));
     Code.emplace_back(MCInstBuilder(AArch64::Bcc)
                           .addImm(AArch64CC::NE)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx)));
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     return Code;
   }
 
@@ -1957,8 +1953,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL);
     Inst.clear();
     Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-        Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        *Ctx, 0)));
+        Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));
     if (IsTailCall)
       convertJmpToTailCall(Inst);
   }
@@ -2228,9 +2223,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                           MCContext *Ctx) const override {
     Inst.setOpcode(AArch64::B);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-        Inst, MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx),
-        *Ctx, 0)));
+    Inst.addOperand(MCOperand::createExpr(
+        getTargetExprFor(Inst, MCSymbolRefExpr::create(TBB, *Ctx), *Ctx, 0)));
   }
 
   bool shouldRecordCodeRelocation(uint32_t RelType) const override {
@@ -2562,7 +2556,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     else if (Fixup.getKind() ==
              MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26))
       RelType = ELF::R_AARCH64_JUMP26;
-    else if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
+    else if (Fixup.isPCRel()) {
       switch (FKI.TargetSize) {
       default:
         return std::nullopt;
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index c7d664ab09d46..10b4913b6ab7f 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -171,8 +171,8 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     (void)Result;
     assert(Result && "unimplemented branch");
 
-    Inst.getOperand(SymOpIndex) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(SymOpIndex) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   IndirectBranchType analyzeIndirectBranch(
@@ -233,8 +233,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(RISCV::JAL);
     Inst.clear();
     Inst.addOperand(MCOperand::createReg(RISCV::X0));
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx)));
   }
 
   StringRef getTrapFillValue() const override {
@@ -246,8 +245,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(Opcode);
     Inst.clear();
     Inst.addOperand(MCOperand::createExpr(MCSpecifierExpr::create(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        ELF::R_RISCV_CALL_PLT, *Ctx)));
+        MCSymbolRefExpr::create(Target, *Ctx), ELF::R_RISCV_CALL_PLT, *Ctx)));
   }
 
   void createCall(MCInst &Inst, const MCSymbol *Target,
@@ -563,8 +561,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Insts.emplace_back(MCInstBuilder(RISCV::BEQ)
                            .addReg(RegNo)
                            .addReg(RegTmp)
-                           .addExpr(MCSymbolRefExpr::create(
-                               Target, MCSymbolRefExpr::VK_None, *Ctx)));
+                           .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     return Insts;
   }
 
@@ -663,14 +660,12 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     if (IsTailCall) {
       Inst.addOperand(MCOperand::createReg(RISCV::X0));
       Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-          Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-          *Ctx, 0)));
+          Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));
       convertJmpToTailCall(Inst);
     } else {
       Inst.addOperand(MCOperand::createReg(RISCV::X1));
       Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-          Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-          *Ctx, 0)));
+          Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));
     }
   }
 
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index b909d7fb6bf28..a60c1a6bf156e 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -18,7 +18,6 @@
 #include "bolt/Core/MCPlusBuilder.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -72,9 +71,9 @@ static InstructionListType createIncMemory(const MCSymbol *Target,
   Insts.back().addOperand(MCOperand::createImm(1));               // ScaleAmt
   Insts.back().addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
 
-  Insts.back().addOperand(MCOperand::createExpr(
-      MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None,
-                              *Ctx))); // Displacement
+  Insts.back().addOperand(
+      MCOperand::createExpr(MCSymbolRefExpr::create(Target,
+                                                    *Ctx))); // Displacement
   Insts.back().addOperand(
       MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
   return Insts;
@@ -1625,9 +1624,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.insert(Inst.begin(),
                 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
     Inst.insert(Inst.begin(),
-                MCOperand::createExpr(                  // Displacement
-                    MCSymbolRefExpr::create(TargetLocation,
-                                            MCSymbolRefExpr::VK_None, *Ctx)));
+                MCOperand::createExpr( // Displacement
+                    MCSymbolRefExpr::create(TargetLocation, *Ctx)));
     Inst.insert(Inst.begin(),
                 MCOperand::createReg(X86::NoRegister)); // IndexReg
     Inst.insert(Inst.begin(),
@@ -2420,8 +2418,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                           .addReg(RegNo)
                           .addImm(Imm));
     Code.emplace_back(MCInstBuilder(X86::JCC_1)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx))
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx))
                           .addImm(X86::COND_E));
     return Code;
   }
@@ -2432,8 +2429,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     InstructionListType Code;
     Code.emplace_back(MCInstBuilder(X86::CMP64ri8).addReg(RegNo).addImm(Imm));
     Code.emplace_back(MCInstBuilder(X86::JCC_1)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx))
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx))
                           .addImm(X86::COND_NE));
     return Code;
   }
@@ -2445,9 +2441,10 @@ class X86MCPlusBuilder : public MCPlusBuilder {
 
     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
     const uint64_t RelOffset = Fixup.getOffset();
+    auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
 
     uint32_t RelType;
-    if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
+    if (Fixup.isPCRel()) {
       switch (FKI.TargetSize) {
       default:
         return std::nullopt;
@@ -2456,6 +2453,9 @@ class X86MCPlusBuilder : public MCPlusBuilder {
       case 32: RelType = ELF::R_X86_64_PC32; break;
       case 64: RelType = ELF::R_X86_64_PC64; break;
       }
+      // Adjust PC-relative fixup offsets, which are calculated from the start
+      // of the next instruction.
+      RelAddend -= FKI.TargetSize / 8;
     } else {
       switch (FKI.TargetSize) {
       default:
@@ -2467,8 +2467,6 @@ class X86MCPlusBuilder : public MCPlusBuilder {
       }
     }
 
-    auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
-
     return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
   }
 
@@ -2738,24 +2736,23 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.clear();
     Inst.setOpcode(X86::JMP_1);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx)));
   }
 
   void createLongUncondBranch(MCInst &Inst, const MCSymbol *Target,
                               MCContext *Ctx) const override {
     Inst.setOpcode(X86::JMP_4);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
   }
 
   void createCall(MCInst &Inst, const MCSymbol *Target,
                   MCContext *Ctx) override {
     Inst.setOpcode(X86::CALL64pcrel32);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
   }
 
   void createTailCall(MCInst &Inst, const MCSymbol *Target,
@@ -2779,8 +2776,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                         MCContext *Ctx) const override {
     Inst.setOpcode(X86::JCC_1);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     Inst.addOperand(MCOperand::createImm(CC));
   }
 
@@ -2788,8 +2785,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                             MCContext *Ctx) const override {
     Inst.setOpcode(X86::JCC_4);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     Inst.addOperand(MCOperand::createImm(CC));
   }
 
@@ -2798,8 +2795,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
-    Inst.getOperand(0) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(0) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
@@ -2807,8 +2804,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     if (CC == X86::COND_INVALID)
       return false;
     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC);
-    Inst.getOperand(0) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(0) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
     return true;
   }
 
@@ -2846,8 +2843,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                            MCContext *Ctx) const override {
     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
            "Invalid instruction");
-    Inst.getOperand(0) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(0) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   MCPhysReg getX86R11() const override { return X86::R11; }
@@ -2894,8 +2891,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                         bool IsTailCall) override {
     Inst.clear();
     Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     if (IsTailCall)
       setTailCall(Inst);
   }
@@ -2905,8 +2902,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Seq.clear();
     MCInst Inst;
     Inst.setOpcode(X86::JMP_1);
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     if (IsTailCall)
       setTailCall(Inst);
     Seq.emplace_back(Inst);
@@ -3332,8 +3329,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         Target.addOperand(MCOperand::createReg(FuncAddrReg));
         if (Targets[i].first) {
           // Is this OK?
-          Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-              Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+          Target.addOperand(MCOperand::createExpr(
+              MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
         } else {
           const uint64_t Addr = Targets[i].second;
           // Immediate address is out of sign extended 32 bit range.
@@ -3409,8 +3406,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         Je.clear();
         Je.setOpcode(X86::JCC_1);
         if (Targets[i].first)
-          Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-              Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+          Je.addOperand(MCOperand::createExpr(
+              MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
         else
           Je.addOperand(MCOperand::createImm(Targets[i].second));
 
@@ -3422,8 +3419,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         // Jump to next compare if target addresses don't match.
         Jne.clear();
         Jne.setOpcode(X86::JCC_1);
-        Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-            NextTarget, MCSymbolRefExpr::VK_None, *Ctx)));
+        Jne.addOperand(
+            MCOperand::createExpr(MCSymbolRefExpr::create(NextTarget, *Ctx)));
         Jne.addOperand(MCOperand::createImm(X86::COND_NE));
 
         // Call specific target directly.
@@ -3442,8 +3439,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
           CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
 
           if (Targets[i].first)
-            CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-                Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+            CallOrJmp.addOperand(MCOperand::createExpr(
+                MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
           else
             CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second));
         }
@@ -3545,8 +3542,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
 
       // Jump to target if indices match
       JEInst.setOpcode(X86::JCC_1);
-      JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-          Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+      JEInst.addOperand(MCOperand::createExpr(
+          MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
       JEInst.addOperand(MCOperand::createImm(X86::COND_E));
     }
 
@@ -3571,9 +3568,9 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
-                                *Ctx)));                    // Displacement
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Src,
+                                                      *Ctx))); // Displacement
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
   }
 
@@ -3585,9 +3582,9 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
-                                *Ctx)));                    // Displacement
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Src,
+                                                      *Ctx))); // Displacement
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
   }
 };
diff --git a/bolt/test/code-at-high-address.c b/bolt/test/code-at-high-address.c
new file mode 100644
index 0000000000000..fa33c1eb342d6
--- /dev/null
+++ b/bolt/test/code-at-high-address.c
@@ -0,0 +1,24 @@
+// Check that llvm-bolt pushes code to higher addresses under
+// --hot-functions-at-end when rewriting code in-place.
+
+// REQUIRES: system-linux
+
+// RUN: %clang %cflags -O0 %s -o %t -no-pie -Wl,-q -falign-functions=64 \
+// RUN:   -nostartfiles -nostdlib -ffreestanding
+// RUN: llvm-bolt %t -o %t.bolt --use-old-text --align-functions=1 \
+// RUN:   --no-huge-pages --align-text=1 --use-gnu-stack --hot-functions-at-end \
+// RUN:   | FileCheck %s --check-prefix=CHECK-BOLT
+// RUN: llvm-readelf --sections %t.bolt | FileCheck %s
+
+// CHECK-BOLT: using original .text for new code with 0x1 alignment at {{.*}}
+
+// As .text is pushed higher, preceding .bolt.org.text should have non-zero
+// size.
+// CHECK: .bolt.org.text PROGBITS
+// CHECK-NOT: {{ 000000 }}
+// CHECK-SAME: AX
+// CHECK-NEXT: .text PROGBITS
+
+int foo() { return 0; }
+
+int main() { return foo(); }
diff --git a/bolt/test/program-header.test b/bolt/test/program-header.test
index 4552303ea5af2..f5490394eb6d9 100644
--- a/bolt/test/program-header.test
+++ b/bolt/test/program-header.test
@@ -2,13 +2,13 @@
 
 REQUIRES: system-linux
 
-RUN: %clang %cflags %p/Inputs/hello.c -o %t -no-pie -Wl,-q
+RUN: %clang %cflags %p/Inputs/hello.c -o %t -no-pie -Wl,-q -nostartfiles \
+RUN:   -nostdlib -ffreestanding
 RUN: llvm-bolt %t -o %t.bolt --use-old-text --align-functions=1 \
 RUN:   --no-huge-pages --align-text=1 --use-gnu-stack \
 RUN:   | FileCheck %s --check-prefix=CHECK-BOLT
 RUN: llvm-readelf -WS %t.bolt | FileCheck %s
 
-CHECK-BOLT: rewriting .eh_frame_hdr in-place
 CHECK-BOLT: not adding new segments
 
 CHECK-NOT: .bolt.org.eh_frame_hdr
diff --git a/bolt/unittests/Core/BinaryContext.cpp b/bolt/unittests/Core/BinaryContext.cpp
index 3fa0851d01e5a..d7374b323c916 100644
--- a/bolt/unittests/Core/BinaryContext.cpp
+++ b/bolt/unittests/Core/BinaryContext.cpp
@@ -199,13 +199,13 @@ TEST_P(BinaryContextTester, BaseAddress) {
   // Check that  base address calculation is correct for a binary with the
   // following segment layout:
   BC->SegmentMapInfo[0] =
-      SegmentInfo{0, 0x10e8c2b4, 0, 0x10e8c2b4, 0x1000, true};
-  BC->SegmentMapInfo[0x10e8d2b4] =
-      SegmentInfo{0x10e8d2b4, 0x3952faec, 0x10e8c2b4, 0x3952faec, 0x1000, true};
-  BC->SegmentMapInfo[0x4a3bddc0] =
-      SegmentInfo{0x4a3bddc0, 0x148e828, 0x4a3bbdc0, 0x148e828, 0x1000, true};
-  BC->SegmentMapInfo[0x4b84d5e8] =
-      SegmentInfo{0x4b84d5e8, 0x294f830, 0x4b84a5e8, 0x3d3820, 0x1000, true};
+      SegmentInfo{0, 0x10e8c2b4, 0, 0x10e8c2b4, 0x1000, true, false};
+  BC->SegmentMapInfo[0x10e8d2b4] = SegmentInfo{
+      0x10e8d2b4, 0x3952faec, 0x10e8c2b4, 0x3952faec, 0x1000, true, false};
+  BC->SegmentMapInfo[0x4a3bddc0] = SegmentInfo{
+      0x4a3bddc0, 0x148e828, 0x4a3bbdc0, 0x148e828, 0x1000, true, false};
+  BC->SegmentMapInfo[0x4b84d5e8] = SegmentInfo{
+      0x4b84d5e8, 0x294f830, 0x4b84a5e8, 0x3d3820, 0x1000, true, false};
 
   std::optional<uint64_t> BaseAddress =
       BC->getBaseAddressForMapping(0x7f13f5556000, 0x10e8c000);
@@ -220,13 +220,14 @@ TEST_P(BinaryContextTester, BaseAddress2) {
   // Check that base address calculation is correct for a binary if the
   // alignment in ELF file are different from pagesize.
   // The segment layout is as follows:
-  BC->SegmentMapInfo[0] = SegmentInfo{0, 0x2177c, 0, 0x2177c, 0x10000, true};
+  BC->SegmentMapInfo[0] =
+      SegmentInfo{0, 0x2177c, 0, 0x2177c, 0x10000, true, false};
   BC->SegmentMapInfo[0x31860] =
-      SegmentInfo{0x31860, 0x370, 0x21860, 0x370, 0x10000, true};
+      SegmentInfo{0x31860, 0x370, 0x21860, 0x370, 0x10000, true, false};
   BC->SegmentMapInfo[0x41c20] =
-      SegmentInfo{0x41c20, 0x1f8, 0x21c20, 0x1f8, 0x10000, true};
+      SegmentInfo{0x41c20, 0x1f8, 0x21c20, 0x1f8, 0x10000, true, false};
   BC->SegmentMapInfo[0x54e18] =
-      SegmentInfo{0x54e18, 0x51, 0x24e18, 0x51, 0x10000, true};
+      SegmentInfo{0x54e18, 0x51, 0x24e18, 0x51, 0x10000, true, false};
 
   std::optional<uint64_t> BaseAddress =
       BC->getBaseAddressForMapping(0xaaaaea444000, 0x21000);
@@ -242,13 +243,14 @@ TEST_P(BinaryContextTester, BaseAddressSegmentsSmallerThanAlignment) {
   // when multiple segments are close together in the ELF file (closer
   // than the required alignment in the process space).
   // See https://github.com/llvm/llvm-project/issues/109384
-  BC->SegmentMapInfo[0] = SegmentInfo{0, 0x1d1c, 0, 0x1d1c, 0x10000, false};
+  BC->SegmentMapInfo[0] =
+      SegmentInfo{0, 0x1d1c, 0, 0x1d1c, 0x10000, false, false};
   BC->SegmentMapInfo[0x11d40] =
-      SegmentInfo{0x11d40, 0x11e0, 0x1d40, 0x11e0, 0x10000, true};
+      SegmentInfo{0x11d40, 0x11e0, 0x1d40, 0x11e0, 0x10000, true, false};
   BC->SegmentMapInfo[0x22f20] =
-      SegmentInfo{0x22f20, 0x10e0, 0x2f20, 0x1f0, 0x10000, false};
+      SegmentInfo{0x22f20, 0x10e0, 0x2f20, 0x1f0, 0x10000, false, false};
   BC->SegmentMapInfo[0x33110] =
-      SegmentInfo{0x33110, 0x89, 0x3110, 0x88, 0x10000, false};
+      SegmentInfo{0x33110, 0x89, 0x3110, 0x88, 0x10000, false, false};
 
   std::optional<uint64_t> BaseAddress =
       BC->getBaseAddressForMapping(0xaaaaaaab1000, 0x1000);
diff --git a/bolt/unittests/Core/MemoryMaps.cpp b/bolt/unittests/Core/MemoryMaps.cpp
index b0cab5431bdd3..8eb8f8ae529b1 100644
--- a/bolt/unittests/Core/MemoryMaps.cpp
+++ b/bolt/unittests/Core/MemoryMaps.cpp
@@ -100,10 +100,10 @@ TEST_P(MemoryMapsTester, ParseMultipleSegments) {
       "[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
       Pid, Filename);
 
-  BC->SegmentMapInfo[0x11da000] =
-      SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
-  BC->SegmentMapInfo[0x31d0000] =
-      SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0000, 0x3000000, 0x200000, true};
+  BC->SegmentMapInfo[0x11da000] = SegmentInfo{
+      0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true, false};
+  BC->SegmentMapInfo[0x31d0000] = SegmentInfo{
+      0x31d0000, 0x51ac82c, 0x31d0000, 0x3000000, 0x200000, true, false};
 
   DataAggregator DA("");
   BC->setFilename(Filename);
@@ -131,12 +131,12 @@ TEST_P(MemoryMapsTester, MultipleSegmentsMismatchedBaseAddress) {
       "[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
       Pid, Filename);
 
-  BC->SegmentMapInfo[0x11da000] =
-      SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
+  BC->SegmentMapInfo[0x11da000] = SegmentInfo{
+      0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true, false};
   // Using '0x31d0fff' FileOffset which triggers a different base address
   // for this second text segment.
-  BC->SegmentMapInfo[0x31d0000] =
-      SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0fff, 0x3000000, 0x200000, true};
+  BC->SegmentMapInfo[0x31d0000] = SegmentInfo{
+      0x31d0000, 0x51ac82c, 0x31d0fff, 0x3000000, 0x200000, true, false};
 
   DataAggregator DA("");
   BC->setFilename(Filename);
diff --git a/bolt/utils/llvm-bolt-wrapper.py b/bolt/utils/llvm-bolt-wrapper.py
index b9d6fad825e78..b913394bce659 100755
--- a/bolt/utils/llvm-bolt-wrapper.py
+++ b/bolt/utils/llvm-bolt-wrapper.py
@@ -79,7 +79,7 @@ def get_cfg(key):
 
 
 # perf2bolt mode
-PERF2BOLT_MODE = ["-aggregate-only", "-ignore-build-id"]
+PERF2BOLT_MODE = ["-aggregate-only", "-ignore-build-id", "-show-density"]
 
 # boltdiff mode
 BOLTDIFF_MODE = ["-diff-only", "-o", "/dev/null"]
diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index 74da80c382120..f756ae6d897c8 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -94,6 +94,7 @@ static llvm::Error decodeRecord(const Record &R, InfoType &Field,
   case InfoType::IT_typedef:
   case InfoType::IT_concept:
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     Field = IT;
     return llvm::Error::success();
   }
@@ -111,6 +112,7 @@ static llvm::Error decodeRecord(const Record &R, FieldId &Field,
   case FieldId::F_child_namespace:
   case FieldId::F_child_record:
   case FieldId::F_concept:
+  case FieldId::F_friend:
   case FieldId::F_default:
     Field = F;
     return llvm::Error::success();
@@ -283,7 +285,15 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
 
 static llvm::Error parseRecord(const Record &R, unsigned ID,
                                llvm::StringRef Blob, TypeInfo *I) {
-  return llvm::Error::success();
+  switch (ID) {
+  case TYPE_IS_BUILTIN:
+    return decodeRecord(R, I->IsBuiltIn, Blob);
+  case TYPE_IS_TEMPLATE:
+    return decodeRecord(R, I->IsTemplate, Blob);
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid field for TypeInfo");
+  }
 }
 
 static llvm::Error parseRecord(const Record &R, unsigned ID,
@@ -293,6 +303,10 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
     return decodeRecord(R, I->Name, Blob);
   case FIELD_DEFAULT_VALUE:
     return decodeRecord(R, I->DefaultValue, Blob);
+  case FIELD_TYPE_IS_BUILTIN:
+    return decodeRecord(R, I->IsBuiltIn, Blob);
+  case FIELD_TYPE_IS_TEMPLATE:
+    return decodeRecord(R, I->IsTemplate, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for TypeInfo");
@@ -308,6 +322,10 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
     return decodeRecord(R, I->Access, Blob);
   case MEMBER_TYPE_IS_STATIC:
     return decodeRecord(R, I->IsStatic, Blob);
+  case MEMBER_TYPE_IS_BUILTIN:
+    return decodeRecord(R, I->IsBuiltIn, Blob);
+  case MEMBER_TYPE_IS_TEMPLATE:
+    return decodeRecord(R, I->IsTemplate, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for MemberTypeInfo");
@@ -434,6 +452,15 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
   }
 }
 
+static llvm::Error parseRecord(const Record &R, unsigned ID, StringRef Blob,
+                               FriendInfo *F) {
+  if (ID == FRIEND_IS_CLASS) {
+    return decodeRecord(R, F->IsClass, Blob);
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "invalid field for Friend");
+}
+
 template <typename T> static llvm::Expected<CommentInfo *> getCommentInfo(T I) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain CommentInfo");
@@ -509,6 +536,18 @@ template <> llvm::Error addTypeInfo(FunctionInfo *I, FieldTypeInfo &&T) {
   return llvm::Error::success();
 }
 
+template <> llvm::Error addTypeInfo(FriendInfo *I, FieldTypeInfo &&T) {
+  if (!I->Params)
+    I->Params.emplace();
+  I->Params->emplace_back(std::move(T));
+  return llvm::Error::success();
+}
+
+template <> llvm::Error addTypeInfo(FriendInfo *I, TypeInfo &&T) {
+  I->ReturnType.emplace(std::move(T));
+  return llvm::Error::success();
+}
+
 template <> llvm::Error addTypeInfo(EnumInfo *I, TypeInfo &&T) {
   I->BaseType = std::move(T);
   return llvm::Error::success();
@@ -530,6 +569,17 @@ static llvm::Error addReference(T I, Reference &&R, FieldId F) {
                                  "invalid type cannot contain Reference");
 }
 
+template <> llvm::Error addReference(VarInfo *I, Reference &&R, FieldId F) {
+  switch (F) {
+  case FieldId::F_namespace:
+    I->Namespace.emplace_back(std::move(R));
+    return llvm::Error::success();
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "VarInfo cannot contain this Reference");
+  }
+}
+
 template <> llvm::Error addReference(TypeInfo *I, Reference &&R, FieldId F) {
   switch (F) {
   case FieldId::F_type:
@@ -651,6 +701,16 @@ llvm::Error addReference(ConstraintInfo *I, Reference &&R, FieldId F) {
       "ConstraintInfo cannot contain this Reference");
 }
 
+template <>
+llvm::Error addReference(FriendInfo *Friend, Reference &&R, FieldId F) {
+  if (F == FieldId::F_friend) {
+    Friend->Ref = std::move(R);
+    return llvm::Error::success();
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "Friend cannot contain this Reference");
+}
+
 template <typename T, typename ChildInfoType>
 static void addChild(T I, ChildInfoType &&R) {
   llvm::errs() << "invalid child type for info";
@@ -684,6 +744,9 @@ template <> void addChild(RecordInfo *I, EnumInfo &&R) {
 template <> void addChild(RecordInfo *I, TypedefInfo &&R) {
   I->Children.Typedefs.emplace_back(std::move(R));
 }
+template <> void addChild(RecordInfo *I, FriendInfo &&R) {
+  I->Friends.emplace_back(std::move(R));
+}
 
 // Other types of children:
 template <> void addChild(EnumInfo *I, EnumValueInfo &&R) {
@@ -725,6 +788,9 @@ template <> void addTemplate(FunctionInfo *I, TemplateInfo &&P) {
 template <> void addTemplate(ConceptInfo *I, TemplateInfo &&P) {
   I->Template = std::move(P);
 }
+template <> void addTemplate(FriendInfo *I, TemplateInfo &&P) {
+  I->Template.emplace(std::move(P));
+}
 
 // Template specializations go only into template records.
 template <typename T>
@@ -905,6 +971,10 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
   case BI_VAR_BLOCK_ID: {
     return handleSubBlock<VarInfo>(ID, I, CreateAddFunc(addChild<T, VarInfo>));
   }
+  case BI_FRIEND_BLOCK_ID: {
+    return handleSubBlock<FriendInfo>(ID, I,
+                                      CreateAddFunc(addChild<T, FriendInfo>));
+  }
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid subblock type");
@@ -1016,6 +1086,8 @@ ClangDocBitcodeReader::readBlockToInfo(unsigned ID) {
     return createInfo<FunctionInfo>(ID);
   case BI_VAR_BLOCK_ID:
     return createInfo<VarInfo>(ID);
+  case BI_FRIEND_BLOCK_ID:
+    return createInfo<FriendInfo>(ID);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "cannot create info");
@@ -1056,6 +1128,7 @@ ClangDocBitcodeReader::readBitcode() {
     case BI_TYPEDEF_BLOCK_ID:
     case BI_CONCEPT_BLOCK_ID:
     case BI_VAR_BLOCK_ID:
+    case BI_FRIEND_BLOCK_ID:
     case BI_FUNCTION_BLOCK_ID: {
       auto InfoOrErr = readBlockToInfo(ID);
       if (!InfoOrErr)
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index c3351d1decbf5..3cc0d4ad332f0 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -131,7 +131,8 @@ static const llvm::IndexedMap<llvm::StringRef, BlockIdToIndexFunctor>
           {BI_TEMPLATE_PARAM_BLOCK_ID, "TemplateParamBlock"},
           {BI_CONSTRAINT_BLOCK_ID, "ConstraintBlock"},
           {BI_CONCEPT_BLOCK_ID, "ConceptBlock"},
-          {BI_VAR_BLOCK_ID, "VarBlock"}};
+          {BI_VAR_BLOCK_ID, "VarBlock"},
+          {BI_FRIEND_BLOCK_ID, "FriendBlock"}};
       assert(Inits.size() == BlockIdCount);
       for (const auto &Init : Inits)
         BlockIdNameMap[Init.first] = Init.second;
@@ -161,9 +162,15 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {COMMENT_ARG, {"Arg", &genStringAbbrev}},
           {FIELD_TYPE_NAME, {"Name", &genStringAbbrev}},
           {FIELD_DEFAULT_VALUE, {"DefaultValue", &genStringAbbrev}},
+          {FIELD_TYPE_IS_BUILTIN, {"IsBuiltin", &genBoolAbbrev}},
+          {FIELD_TYPE_IS_TEMPLATE, {"IsTemplate", &genBoolAbbrev}},
           {MEMBER_TYPE_NAME, {"Name", &genStringAbbrev}},
           {MEMBER_TYPE_ACCESS, {"Access", &genIntAbbrev}},
           {MEMBER_TYPE_IS_STATIC, {"IsStatic", &genBoolAbbrev}},
+          {MEMBER_TYPE_IS_BUILTIN, {"IsBuiltin", &genBoolAbbrev}},
+          {MEMBER_TYPE_IS_TEMPLATE, {"IsTemplate", &genBoolAbbrev}},
+          {TYPE_IS_BUILTIN, {"IsBuiltin", &genBoolAbbrev}},
+          {TYPE_IS_TEMPLATE, {"IsTemplate", &genBoolAbbrev}},
           {NAMESPACE_USR, {"USR", &genSymbolIdAbbrev}},
           {NAMESPACE_NAME, {"Name", &genStringAbbrev}},
           {NAMESPACE_PATH, {"Path", &genStringAbbrev}},
@@ -218,7 +225,8 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {VAR_USR, {"USR", &genSymbolIdAbbrev}},
           {VAR_NAME, {"Name", &genStringAbbrev}},
           {VAR_DEFLOCATION, {"DefLocation", &genLocationAbbrev}},
-          {VAR_IS_STATIC, {"IsStatic", &genBoolAbbrev}}};
+          {VAR_IS_STATIC, {"IsStatic", &genBoolAbbrev}},
+          {FRIEND_IS_CLASS, {"IsClass", &genBoolAbbrev}}};
 
       assert(Inits.size() == RecordIdCount);
       for (const auto &Init : Inits) {
@@ -239,12 +247,15 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
           COMMENT_PARAMNAME, COMMENT_CLOSENAME, COMMENT_SELFCLOSING,
           COMMENT_EXPLICIT, COMMENT_ATTRKEY, COMMENT_ATTRVAL, COMMENT_ARG}},
         // Type Block
-        {BI_TYPE_BLOCK_ID, {}},
+        {BI_TYPE_BLOCK_ID, {TYPE_IS_BUILTIN, TYPE_IS_TEMPLATE}},
         // FieldType Block
-        {BI_FIELD_TYPE_BLOCK_ID, {FIELD_TYPE_NAME, FIELD_DEFAULT_VALUE}},
+        {BI_FIELD_TYPE_BLOCK_ID,
+         {FIELD_TYPE_NAME, FIELD_DEFAULT_VALUE, FIELD_TYPE_IS_BUILTIN,
+          FIELD_TYPE_IS_TEMPLATE}},
         // MemberType Block
         {BI_MEMBER_TYPE_BLOCK_ID,
-         {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS, MEMBER_TYPE_IS_STATIC}},
+         {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS, MEMBER_TYPE_IS_STATIC,
+          MEMBER_TYPE_IS_BUILTIN, MEMBER_TYPE_IS_TEMPLATE}},
         // Enum Block
         {BI_ENUM_BLOCK_ID,
          {ENUM_USR, ENUM_NAME, ENUM_DEFLOCATION, ENUM_LOCATION, ENUM_SCOPED}},
@@ -284,7 +295,8 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
           CONCEPT_CONSTRAINT_EXPRESSION}},
         // Constraint Block
         {BI_CONSTRAINT_BLOCK_ID, {CONSTRAINT_EXPRESSION}},
-        {BI_VAR_BLOCK_ID, {VAR_NAME, VAR_USR, VAR_DEFLOCATION, VAR_IS_STATIC}}};
+        {BI_VAR_BLOCK_ID, {VAR_NAME, VAR_USR, VAR_DEFLOCATION, VAR_IS_STATIC}},
+        {BI_FRIEND_BLOCK_ID, {FRIEND_IS_CLASS}}};
 
 // AbbreviationMap
 
@@ -467,9 +479,24 @@ void ClangDocBitcodeWriter::emitBlock(const Reference &R, FieldId Field) {
   emitRecord((unsigned)Field, REFERENCE_FIELD);
 }
 
+void ClangDocBitcodeWriter::emitBlock(const FriendInfo &R) {
+  StreamSubBlockGuard Block(Stream, BI_FRIEND_BLOCK_ID);
+  emitBlock(R.Ref, FieldId::F_friend);
+  emitRecord(R.IsClass, FRIEND_IS_CLASS);
+  if (R.Template)
+    emitBlock(*R.Template);
+  if (R.Params)
+    for (const auto &P : *R.Params)
+      emitBlock(P);
+  if (R.ReturnType)
+    emitBlock(*R.ReturnType);
+}
+
 void ClangDocBitcodeWriter::emitBlock(const TypeInfo &T) {
   StreamSubBlockGuard Block(Stream, BI_TYPE_BLOCK_ID);
   emitBlock(T.Type, FieldId::F_type);
+  emitRecord(T.IsBuiltIn, TYPE_IS_BUILTIN);
+  emitRecord(T.IsTemplate, TYPE_IS_TEMPLATE);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const TypedefInfo &T) {
@@ -491,6 +518,8 @@ void ClangDocBitcodeWriter::emitBlock(const FieldTypeInfo &T) {
   emitBlock(T.Type, FieldId::F_type);
   emitRecord(T.Name, FIELD_TYPE_NAME);
   emitRecord(T.DefaultValue, FIELD_DEFAULT_VALUE);
+  emitRecord(T.IsBuiltIn, FIELD_TYPE_IS_BUILTIN);
+  emitRecord(T.IsTemplate, FIELD_TYPE_IS_TEMPLATE);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const MemberTypeInfo &T) {
@@ -499,6 +528,9 @@ void ClangDocBitcodeWriter::emitBlock(const MemberTypeInfo &T) {
   emitRecord(T.Name, MEMBER_TYPE_NAME);
   emitRecord(T.Access, MEMBER_TYPE_ACCESS);
   emitRecord(T.IsStatic, MEMBER_TYPE_IS_STATIC);
+  emitRecord(T.IsBuiltIn, MEMBER_TYPE_IS_BUILTIN);
+  emitRecord(T.IsTemplate, MEMBER_TYPE_IS_TEMPLATE);
+  emitRecord(T.IsTemplate, MEMBER_TYPE_IS_TEMPLATE);
   for (const auto &CI : T.Description)
     emitBlock(CI);
 }
@@ -612,6 +644,8 @@ void ClangDocBitcodeWriter::emitBlock(const RecordInfo &I) {
     emitBlock(C);
   if (I.Template)
     emitBlock(*I.Template);
+  for (const auto &C : I.Friends)
+    emitBlock(C);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const BaseRecordInfo &I) {
@@ -728,6 +762,9 @@ bool ClangDocBitcodeWriter::dispatchInfoForWrite(Info *I) {
   case InfoType::IT_variable:
     emitBlock(*static_cast<VarInfo *>(I));
     break;
+  case InfoType::IT_friend:
+    emitBlock(*static_cast<FriendInfo *>(I));
+    break;
   case InfoType::IT_default:
     llvm::errs() << "Unexpected info, unable to write.\n";
     return true;
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.h b/clang-tools-extra/clang-doc/BitcodeWriter.h
index a70e50b53a61a..d09ec4ca34006 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.h
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.h
@@ -70,6 +70,7 @@ enum BlockId {
   BI_TYPEDEF_BLOCK_ID,
   BI_CONCEPT_BLOCK_ID,
   BI_VAR_BLOCK_ID,
+  BI_FRIEND_BLOCK_ID,
   BI_LAST,
   BI_FIRST = BI_VERSION_BLOCK_ID
 };
@@ -96,11 +97,17 @@ enum RecordId {
   COMMENT_ATTRKEY,
   COMMENT_ATTRVAL,
   COMMENT_ARG,
+  TYPE_IS_BUILTIN,
+  TYPE_IS_TEMPLATE,
   FIELD_TYPE_NAME,
   FIELD_DEFAULT_VALUE,
+  FIELD_TYPE_IS_BUILTIN,
+  FIELD_TYPE_IS_TEMPLATE,
   MEMBER_TYPE_NAME,
   MEMBER_TYPE_ACCESS,
   MEMBER_TYPE_IS_STATIC,
+  MEMBER_TYPE_IS_BUILTIN,
+  MEMBER_TYPE_IS_TEMPLATE,
   NAMESPACE_USR,
   NAMESPACE_NAME,
   NAMESPACE_PATH,
@@ -147,6 +154,7 @@ enum RecordId {
   VAR_NAME,
   VAR_DEFLOCATION,
   VAR_IS_STATIC,
+  FRIEND_IS_CLASS,
   RI_LAST,
   RI_FIRST = VERSION
 };
@@ -163,7 +171,8 @@ enum class FieldId {
   F_type,
   F_child_namespace,
   F_child_record,
-  F_concept
+  F_concept,
+  F_friend
 };
 
 class ClangDocBitcodeWriter {
@@ -195,6 +204,7 @@ class ClangDocBitcodeWriter {
   void emitBlock(const ConceptInfo &T);
   void emitBlock(const ConstraintInfo &T);
   void emitBlock(const Reference &B, FieldId F);
+  void emitBlock(const FriendInfo &R);
   void emitBlock(const VarInfo &B);
 
 private:
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index c4303d287da9e..8294ff9118558 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -987,6 +987,7 @@ llvm::Error HTMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
     break;
   case InfoType::IT_concept:
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -1018,6 +1019,8 @@ static std::string getRefType(InfoType IT) {
     return "concept";
   case InfoType::IT_variable:
     return "variable";
+  case InfoType::IT_friend:
+    return "friend";
   }
   llvm_unreachable("Unknown InfoType");
 }
diff --git a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
index c611c946b3937..7aeaa1b7cf67d 100644
--- a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
@@ -588,6 +588,7 @@ Error MustacheHTMLGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
   case InfoType::IT_concept:
     break;
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return createStringError(inconvertibleErrorCode(), "unexpected InfoType");
diff --git a/clang-tools-extra/clang-doc/JSONGenerator.cpp b/clang-tools-extra/clang-doc/JSONGenerator.cpp
index 1f6167f7f9b8d..0e1a0cc347e45 100644
--- a/clang-tools-extra/clang-doc/JSONGenerator.cpp
+++ b/clang-tools-extra/clang-doc/JSONGenerator.cpp
@@ -39,8 +39,7 @@ static void serializeArray(const Container &Records, Object &Obj,
 static auto SerializeInfoLambda = [](const auto &Info, Object &Object) {
   serializeInfo(Info, Object);
 };
-static auto SerializeReferenceLambda = [](const Reference &Ref,
-                                          Object &Object) {
+static auto SerializeReferenceLambda = [](const auto &Ref, Object &Object) {
   serializeReference(Ref, Object);
 };
 
@@ -365,6 +364,22 @@ static void serializeInfo(const BaseRecordInfo &I, Object &Obj,
   Obj["IsParent"] = I.IsParent;
 }
 
+static void serializeInfo(const FriendInfo &I, Object &Obj) {
+  auto FriendRef = Object();
+  serializeReference(I.Ref, FriendRef);
+  Obj["Reference"] = std::move(FriendRef);
+  Obj["IsClass"] = I.IsClass;
+  if (I.Template)
+    serializeInfo(I.Template.value(), Obj);
+  if (I.Params)
+    serializeArray(I.Params.value(), Obj, "Params", SerializeInfoLambda);
+  if (I.ReturnType) {
+    auto ReturnTypeObj = Object();
+    serializeInfo(I.ReturnType.value(), ReturnTypeObj);
+    Obj["ReturnType"] = std::move(ReturnTypeObj);
+  }
+}
+
 static void serializeInfo(const RecordInfo &I, json::Object &Obj,
                           const std::optional<StringRef> &RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
@@ -436,6 +451,9 @@ static void serializeInfo(const RecordInfo &I, json::Object &Obj,
   if (I.Template)
     serializeInfo(I.Template.value(), Obj);
 
+  if (!I.Friends.empty())
+    serializeArray(I.Friends, Obj, "Friends", SerializeInfoLambda);
+
   serializeCommonChildren(I.Children, Obj, RepositoryUrl);
 }
 
@@ -525,6 +543,7 @@ Error JSONGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
   case InfoType::IT_function:
   case InfoType::IT_typedef:
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return createStringError(inconvertibleErrorCode(), "unexpected info type");
diff --git a/clang-tools-extra/clang-doc/MDGenerator.cpp b/clang-tools-extra/clang-doc/MDGenerator.cpp
index 608a7f6d4a9d3..6f16f5bd2f528 100644
--- a/clang-tools-extra/clang-doc/MDGenerator.cpp
+++ b/clang-tools-extra/clang-doc/MDGenerator.cpp
@@ -378,6 +378,9 @@ static llvm::Error genIndex(ClangDocContext &CDCtx) {
       case InfoType::IT_variable:
         Type = "Variable";
         break;
+      case InfoType::IT_friend:
+        Type = "Friend";
+        break;
       case InfoType::IT_default:
         Type = "Other";
       }
@@ -472,6 +475,7 @@ llvm::Error MDGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
     break;
   case InfoType::IT_concept:
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return createStringError(llvm::inconvertibleErrorCode(),
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 5b94d37d868b4..422a76d99e5b3 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -147,6 +147,8 @@ mergeInfos(std::vector<std::unique_ptr<Info>> &Values) {
     return reduce<ConceptInfo>(Values);
   case InfoType::IT_variable:
     return reduce<VarInfo>(Values);
+  case InfoType::IT_friend:
+    return reduce<FriendInfo>(Values);
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unexpected info type");
@@ -247,6 +249,15 @@ void Reference::merge(Reference &&Other) {
     Path = Other.Path;
 }
 
+bool FriendInfo::mergeable(const FriendInfo &Other) {
+  return Ref.USR == Other.Ref.USR && Ref.Name == Other.Ref.Name;
+}
+
+void FriendInfo::merge(FriendInfo &&Other) {
+  assert(mergeable(Other));
+  Ref.merge(std::move(Other.Ref));
+}
+
 void Info::mergeBase(Info &&Other) {
   assert(mergeable(Other));
   if (USR == EmptySID)
@@ -313,6 +324,8 @@ void RecordInfo::merge(RecordInfo &&Other) {
     Parents = std::move(Other.Parents);
   if (VirtualParents.empty())
     VirtualParents = std::move(Other.VirtualParents);
+  if (Friends.empty())
+    Friends = std::move(Other.Friends);
   // Reduce children if necessary.
   reduceChildren(Children.Records, std::move(Other.Children.Records));
   reduceChildren(Children.Functions, std::move(Other.Children.Functions));
@@ -422,6 +435,9 @@ llvm::SmallString<16> Info::extractName() const {
   case InfoType::IT_variable:
     return llvm::SmallString<16>("@nonymous_variable_" +
                                  toHex(llvm::toStringRef(USR)));
+  case InfoType::IT_friend:
+    return llvm::SmallString<16>("@nonymous_friend_" +
+                                 toHex(llvm::toStringRef(USR)));
   case InfoType::IT_default:
     return llvm::SmallString<16>("@nonymous_" + toHex(llvm::toStringRef(USR)));
   }
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 59874f0cfcedf..fe5cc48069d58 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -46,7 +46,8 @@ enum class InfoType {
   IT_enum,
   IT_typedef,
   IT_concept,
-  IT_variable
+  IT_variable,
+  IT_friend
 };
 
 enum class CommentKind {
@@ -379,6 +380,22 @@ struct SymbolInfo : public Info {
   bool IsStatic = false;
 };
 
+struct FriendInfo : SymbolInfo {
+  FriendInfo() : SymbolInfo(InfoType::IT_friend) {}
+  FriendInfo(SymbolID USR) : SymbolInfo(InfoType::IT_friend, USR) {}
+  FriendInfo(const InfoType IT, const SymbolID &USR,
+             const StringRef Name = StringRef())
+      : SymbolInfo(IT, USR, Name) {}
+  bool mergeable(const FriendInfo &Other);
+  void merge(FriendInfo &&Other);
+
+  Reference Ref;
+  std::optional<TemplateInfo> Template;
+  std::optional<TypeInfo> ReturnType;
+  std::optional<SmallVector<FieldTypeInfo, 4>> Params;
+  bool IsClass = false;
+};
+
 struct VarInfo : SymbolInfo {
   VarInfo() : SymbolInfo(InfoType::IT_variable) {}
   explicit VarInfo(SymbolID USR) : SymbolInfo(InfoType::IT_variable, USR) {}
@@ -454,6 +471,8 @@ struct RecordInfo : public SymbolInfo {
       Bases; // List of base/parent records; this includes inherited methods and
              // attributes
 
+  std::vector<FriendInfo> Friends;
+
   ScopeChildren Children;
 };
 
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index 7a9cb8a1eddb9..6cc372ce98a6d 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -8,8 +8,10 @@
 
 #include "Serialize.h"
 #include "BitcodeWriter.h"
+
 #include "clang/AST/Attr.h"
 #include "clang/AST/Comment.h"
+#include "clang/AST/DeclFriend.h"
 #include "clang/Index/USRGeneration.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/StringExtras.h"
@@ -403,6 +405,7 @@ std::string serialize(std::unique_ptr<Info> &I) {
     return serialize(*static_cast<ConceptInfo *>(I.get()));
   case InfoType::IT_variable:
     return serialize(*static_cast<VarInfo *>(I.get()));
+  case InfoType::IT_friend:
   case InfoType::IT_typedef:
   case InfoType::IT_default:
     return "";
@@ -556,6 +559,7 @@ static std::unique_ptr<Info> makeAndInsertIntoParent(ChildType Child) {
   case InfoType::IT_typedef:
   case InfoType::IT_concept:
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   }
   llvm_unreachable("Invalid reference type for parent namespace");
@@ -947,6 +951,55 @@ emitInfo(const NamespaceDecl *D, const FullComment *FC, Location Loc,
   return {std::move(NSI), makeAndInsertIntoParent<const NamespaceInfo &>(*NSI)};
 }
 
+static void parseFriends(RecordInfo &RI, const CXXRecordDecl *D) {
+  if (!D->hasDefinition() || !D->hasFriends())
+    return;
+
+  for (const FriendDecl *FD : D->friends()) {
+    if (FD->isUnsupportedFriend())
+      continue;
+
+    FriendInfo F(InfoType::IT_friend, getUSRForDecl(FD));
+    const auto *ActualDecl = FD->getFriendDecl();
+    if (!ActualDecl) {
+      const auto *FriendTypeInfo = FD->getFriendType();
+      if (!FriendTypeInfo)
+        continue;
+      ActualDecl = FriendTypeInfo->getType()->getAsCXXRecordDecl();
+
+      if (!ActualDecl)
+        continue;
+      F.IsClass = true;
+    }
+
+    if (const auto *ActualTD = dyn_cast_or_null<TemplateDecl>(ActualDecl)) {
+      if (isa<RecordDecl>(ActualTD->getTemplatedDecl()))
+        F.IsClass = true;
+      F.Template.emplace();
+      for (const auto *Param : ActualTD->getTemplateParameters()->asArray())
+        F.Template->Params.emplace_back(
+            getSourceCode(Param, Param->getSourceRange()));
+      ActualDecl = ActualTD->getTemplatedDecl();
+    }
+
+    if (auto *FuncDecl = dyn_cast_or_null<FunctionDecl>(ActualDecl)) {
+      FunctionInfo TempInfo;
+      parseParameters(TempInfo, FuncDecl);
+      F.Params.emplace();
+      F.Params = std::move(TempInfo.Params);
+      F.ReturnType = getTypeInfoForType(FuncDecl->getReturnType(),
+                                        FuncDecl->getLangOpts());
+    }
+
+    F.Ref =
+        Reference(getUSRForDecl(ActualDecl), ActualDecl->getNameAsString(),
+                  InfoType::IT_default, ActualDecl->getQualifiedNameAsString(),
+                  getInfoRelativePath(ActualDecl));
+
+    RI.Friends.push_back(std::move(F));
+  }
+}
+
 std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
 emitInfo(const RecordDecl *D, const FullComment *FC, Location Loc,
          bool PublicOnly) {
@@ -970,6 +1023,7 @@ emitInfo(const RecordDecl *D, const FullComment *FC, Location Loc,
     // TODO: remove first call to parseBases, that function should be deleted
     parseBases(*RI, C);
     parseBases(*RI, C, /*IsFileInRootDir=*/true, PublicOnly, /*IsParent=*/true);
+    parseFriends(*RI, C);
   }
   RI->Path = getInfoRelativePath(RI->Namespace);
 
diff --git a/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
index 3ca4d4947fa97..eeccdd804b669 100644
--- a/clang-tools-extra/clang-doc/YAMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -410,6 +410,7 @@ llvm::Error YAMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
     break;
   case InfoType::IT_concept:
   case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index f4ab93b51f4a7..808515c463b91 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -55,7 +55,8 @@ namespace clang::tidy {
 
 namespace {
 #if CLANG_TIDY_ENABLE_STATIC_ANALYZER
-static const char *AnalyzerCheckNamePrefix = "clang-analyzer-";
+static constexpr llvm::StringLiteral AnalyzerCheckNamePrefix =
+    "clang-analyzer-";
 
 class AnalyzerDiagnosticConsumer : public ento::PathDiagnosticConsumer {
 public:
@@ -351,10 +352,9 @@ ClangTidyASTConsumerFactory::ClangTidyASTConsumerFactory(
 static void
 setStaticAnalyzerCheckerOpts(const ClangTidyOptions &Opts,
                              clang::AnalyzerOptions &AnalyzerOptions) {
-  StringRef AnalyzerPrefix(AnalyzerCheckNamePrefix);
   for (const auto &Opt : Opts.CheckOptions) {
     StringRef OptName(Opt.getKey());
-    if (!OptName.consume_front(AnalyzerPrefix))
+    if (!OptName.consume_front(AnalyzerCheckNamePrefix))
       continue;
     // Analyzer options are always local options so we can ignore priority.
     AnalyzerOptions.Config[OptName] = Opt.getValue().Value;
@@ -476,7 +476,8 @@ std::vector<std::string> ClangTidyASTConsumerFactory::getCheckNames() {
 #if CLANG_TIDY_ENABLE_STATIC_ANALYZER
   for (const auto &AnalyzerCheck : getAnalyzerCheckersAndPackages(
            Context, Context.canEnableAnalyzerAlphaCheckers()))
-    CheckNames.push_back(AnalyzerCheckNamePrefix + AnalyzerCheck.first);
+    CheckNames.emplace_back(
+        (AnalyzerCheckNamePrefix + AnalyzerCheck.first).str());
 #endif // CLANG_TIDY_ENABLE_STATIC_ANALYZER
 
   llvm::sort(CheckNames);
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index a71813a103df3..f9d75978d0ea8 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -358,8 +358,27 @@ getFixIt(const tooling::Diagnostic &Diagnostic, bool AnyFix) {
 
 } // namespace clang::tidy
 
+void ClangTidyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
+                                                  const Preprocessor *PP) {
+  DiagnosticConsumer::BeginSourceFile(LangOpts, PP);
+
+  assert(!InSourceFile);
+  InSourceFile = true;
+}
+
+void ClangTidyDiagnosticConsumer::EndSourceFile() {
+  assert(InSourceFile);
+  InSourceFile = false;
+
+  DiagnosticConsumer::EndSourceFile();
+}
+
 void ClangTidyDiagnosticConsumer::HandleDiagnostic(
     DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
+  // A diagnostic should not be reported outside of a
+  // BeginSourceFile()/EndSourceFile() pair if it has a source location.
+  assert(InSourceFile || Info.getLocation().isInvalid());
+
   if (LastErrorWasIgnored && DiagLevel == DiagnosticsEngine::Note)
     return;
 
@@ -525,6 +544,9 @@ void ClangTidyDiagnosticConsumer::forwardDiagnostic(const Diagnostic &Info) {
     case clang::DiagnosticsEngine::ak_attr:
       Builder << reinterpret_cast<Attr *>(Info.getRawArg(Index));
       break;
+    case clang::DiagnosticsEngine::ak_attr_info:
+      Builder << reinterpret_cast<AttributeCommonInfo *>(Info.getRawArg(Index));
+      break;
     case clang::DiagnosticsEngine::ak_addrspace:
       Builder << static_cast<LangAS>(Info.getRawArg(Index));
       break;
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index a8851e794f24b..6e7cb7bb10e57 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -292,6 +292,11 @@ class ClangTidyDiagnosticConsumer : public DiagnosticConsumer {
   void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                         const Diagnostic &Info) override;
 
+  void BeginSourceFile(const LangOptions &LangOpts,
+                       const Preprocessor *PP = nullptr) override;
+
+  void EndSourceFile() override;
+
   // Retrieve the diagnostics that were captured.
   std::vector<ClangTidyError> take();
 
@@ -326,6 +331,11 @@ class ClangTidyDiagnosticConsumer : public DiagnosticConsumer {
   bool LastErrorRelatesToUserCode = false;
   bool LastErrorPassesLineFilter = false;
   bool LastErrorWasIgnored = false;
+  /// Tracks whether we're currently inside a
+  /// `BeginSourceFile()/EndSourceFile()` pair. Outside of a source file, we
+  /// should only receive diagnostics that have to source location, such as
+  /// command-line warnings.
+  bool InSourceFile = false;
 };
 
 } // end namespace tidy
diff --git a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
index 1eef86ddc00b9..2ae3c00f7ee3e 100644
--- a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
+++ b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
@@ -46,12 +46,12 @@ AST_POLYMORPHIC_MATCHER(
   if (PrefixPosition == StringRef::npos)
     return false;
   Path = Path.drop_front(PrefixPosition + AbslPrefix.size());
-  static const char *AbseilLibraries[] = {
+  static constexpr llvm::StringLiteral AbseilLibraries[] = {
       "algorithm", "base",     "container", "debugging", "flags",
       "hash",      "iterator", "memory",    "meta",      "numeric",
       "profiling", "random",   "status",    "strings",   "synchronization",
       "time",      "types",    "utility"};
-  return llvm::any_of(AbseilLibraries, [&](const char *Library) {
+  return llvm::any_of(AbseilLibraries, [&](llvm::StringLiteral Library) {
     return Path.starts_with(Library);
   });
 }
diff --git a/clang-tools-extra/clang-tidy/bugprone/ComparePointerToMemberVirtualFunctionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ComparePointerToMemberVirtualFunctionCheck.cpp
index f2746ba159d04..1cbf1e22a33a7 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ComparePointerToMemberVirtualFunctionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ComparePointerToMemberVirtualFunctionCheck.cpp
@@ -25,7 +25,7 @@ namespace {
 
 AST_MATCHER(CXXMethodDecl, isVirtual) { return Node.isVirtual(); }
 
-static const char *const ErrorMsg =
+static constexpr llvm::StringLiteral ErrorMsg =
     "comparing a pointer to member virtual function with other pointer is "
     "unspecified behavior, only compare it with a null-pointer constant for "
     "equality.";
diff --git a/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp
index 4aba5831e6772..23de8d971898e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp
@@ -50,6 +50,8 @@ void MisleadingSetterOfReferenceCheck::check(
     const MatchFinder::MatchResult &Result) {
   const auto *Found = Result.Nodes.getNodeAs<CXXMethodDecl>("bad-set-function");
   const auto *Member = Result.Nodes.getNodeAs<FieldDecl>("member");
+  assert(Found != nullptr);
+  assert(Member != nullptr);
 
   diag(Found->getBeginLoc(),
        "function '%0' can be mistakenly used in order to change the "
diff --git a/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp
index db0ac281ddfcf..c1ea63cda5003 100644
--- a/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/TaggedUnionMemberCountCheck.cpp
@@ -144,7 +144,8 @@ TaggedUnionMemberCountCheck::getNumberOfEnumValues(const EnumDecl *ED) {
 
   if (EnableCountingEnumHeuristic && LastEnumConstant &&
       isCountingEnumLikeName(LastEnumConstant->getName()) &&
-      (LastEnumConstant->getInitVal() == (EnumValues.size() - 1))) {
+      llvm::APSInt::isSameValue(LastEnumConstant->getInitVal(),
+                                llvm::APSInt::get(EnumValues.size() - 1))) {
     return {EnumValues.size() - 1, LastEnumConstant};
   }
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp
index cf299609e646d..268b51f76a2c3 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp
@@ -120,7 +120,7 @@ void MissingStdForwardCheck::registerMatchers(MatchFinder *Finder) {
                           equalsBoundNode("param"), equalsBoundNode("var")))))),
                 CapturedInLambda)),
       callee(unresolvedLookupExpr(hasAnyDeclaration(
-          namedDecl(hasUnderlyingDecl(hasName("::std::forward")))))),
+          namedDecl(hasUnderlyingDecl(hasName(ForwardFunction)))))),
 
       unless(anyOf(hasAncestor(typeLoc()),
                    hasAncestor(expr(hasUnevaluatedContext())))));
@@ -149,4 +149,13 @@ void MissingStdForwardCheck::check(const MatchFinder::MatchResult &Result) {
       << Param;
 }
 
+MissingStdForwardCheck::MissingStdForwardCheck(StringRef Name,
+                                               ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      ForwardFunction(Options.get("ForwardFunction", "::std::forward")) {}
+
+void MissingStdForwardCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "ForwardFunction", ForwardFunction);
+}
+
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h
index 5995ad588855e..f833b8031f8af 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h
@@ -21,8 +21,7 @@ namespace clang::tidy::cppcoreguidelines {
 /// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines/missing-std-forward.html
 class MissingStdForwardCheck : public ClangTidyCheck {
 public:
-  MissingStdForwardCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+  MissingStdForwardCheck(StringRef Name, ClangTidyContext *Context);
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
@@ -31,6 +30,10 @@ class MissingStdForwardCheck : public ClangTidyCheck {
   std::optional<TraversalKind> getCheckTraversalKind() const override {
     return TK_IgnoreUnlessSpelledInSource;
   }
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+
+private:
+  const StringRef ForwardFunction;
 };
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
index a1494a095f5b6..9ac7b9e057e35 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
@@ -15,16 +15,13 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::cppcoreguidelines {
 
 void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
-  if (!getLangOpts().CPlusPlus)
-    return;
-
   const auto AllPointerTypes =
-      anyOf(hasType(pointerType()),
+      anyOf(hasType(hasUnqualifiedDesugaredType(pointerType())),
             hasType(autoType(
                 hasDeducedType(hasUnqualifiedDesugaredType(pointerType())))),
             hasType(decltypeType(hasUnderlyingType(pointerType()))));
 
-  // Flag all operators +, -, +=, -=, ++, -- that result in a pointer
+  // Flag all operators +, -, +=, -= that result in a pointer
   Finder->addMatcher(
       binaryOperator(
           hasAnyOperatorName("+", "-", "+=", "-="), AllPointerTypes,
@@ -32,8 +29,12 @@ void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
           .bind("expr"),
       this);
 
+  // Flag all operators ++, -- that result in a pointer
   Finder->addMatcher(
-      unaryOperator(hasAnyOperatorName("++", "--"), hasType(pointerType()))
+      unaryOperator(hasAnyOperatorName("++", "--"),
+                    hasType(hasUnqualifiedDesugaredType(pointerType())),
+                    unless(hasUnaryOperand(
+                        ignoringImpCasts(declRefExpr(to(isImplicit()))))))
           .bind("expr"),
       this);
 
@@ -42,7 +43,8 @@ void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
       arraySubscriptExpr(
           hasBase(ignoringImpCasts(
               anyOf(AllPointerTypes,
-                    hasType(decayedType(hasDecayedType(pointerType())))))))
+                    hasType(decayedType(hasDecayedType(pointerType())))))),
+          hasIndex(hasType(isInteger())))
           .bind("expr"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
index 67f7d1bf9dd69..3466c72a769e9 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
@@ -23,6 +23,9 @@ class ProBoundsPointerArithmeticCheck : public ClangTidyCheck {
 public:
   ProBoundsPointerArithmeticCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
index aaaaf6b66b51a..b413b12cd37ab 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
@@ -373,8 +373,8 @@ static bool isEmpty(ASTContext &Context, const QualType &Type) {
   return isIncompleteOrZeroLengthArrayType(Context, Type);
 }
 
-static const char *getInitializer(QualType QT, bool UseAssignment) {
-  const char *DefaultInitializer = "{}";
+static llvm::StringLiteral getInitializer(QualType QT, bool UseAssignment) {
+  static constexpr llvm::StringLiteral DefaultInitializer = "{}";
   if (!UseAssignment)
     return DefaultInitializer;
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
index 8c386d5bc7945..272152644d7dd 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
@@ -42,9 +42,9 @@ void RvalueReferenceParamNotMovedCheck::registerMatchers(MatchFinder *Finder) {
   StatementMatcher MoveCallMatcher =
       callExpr(
           argumentCountIs(1),
-          anyOf(callee(functionDecl(hasName("::std::move"))),
+          anyOf(callee(functionDecl(hasName(MoveFunction))),
                 callee(unresolvedLookupExpr(hasAnyDeclaration(
-                    namedDecl(hasUnderlyingDecl(hasName("::std::move"))))))),
+                    namedDecl(hasUnderlyingDecl(hasName(MoveFunction))))))),
           hasArgument(
               0, argumentOf(
                      AllowPartialMove,
@@ -122,7 +122,8 @@ RvalueReferenceParamNotMovedCheck::RvalueReferenceParamNotMovedCheck(
       AllowPartialMove(Options.get("AllowPartialMove", false)),
       IgnoreUnnamedParams(Options.get("IgnoreUnnamedParams", false)),
       IgnoreNonDeducedTemplateTypes(
-          Options.get("IgnoreNonDeducedTemplateTypes", false)) {}
+          Options.get("IgnoreNonDeducedTemplateTypes", false)),
+      MoveFunction(Options.get("MoveFunction", "::std::move")) {}
 
 void RvalueReferenceParamNotMovedCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
@@ -130,6 +131,7 @@ void RvalueReferenceParamNotMovedCheck::storeOptions(
   Options.store(Opts, "IgnoreUnnamedParams", IgnoreUnnamedParams);
   Options.store(Opts, "IgnoreNonDeducedTemplateTypes",
                 IgnoreNonDeducedTemplateTypes);
+  Options.store(Opts, "MoveFunction", MoveFunction);
 }
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h
index d8c3d2bd4ba0e..950c0206745d7 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h
@@ -32,6 +32,7 @@ class RvalueReferenceParamNotMovedCheck : public ClangTidyCheck {
   const bool AllowPartialMove;
   const bool IgnoreUnnamedParams;
   const bool IgnoreNonDeducedTemplateTypes;
+  const StringRef MoveFunction;
 };
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
index 7639476980529..79ae5ee98182b 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
@@ -1,5 +1,4 @@
-//===--- ConfusableIdentifierCheck.cpp -
-// clang-tidy--------------------------===//
+//===--- ConfusableIdentifierCheck.cpp - clang-tidy -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,6 +8,7 @@
 
 #include "ConfusableIdentifierCheck.h"
 
+#include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ConvertUTF.h"
@@ -88,99 +88,98 @@ static llvm::SmallString<64U> skeleton(StringRef Name) {
   return Skeleton;
 }
 
-static bool mayShadowImpl(const DeclContext *DC0, const DeclContext *DC1) {
-  return DC0 && DC0 == DC1;
-}
-
-static bool mayShadowImpl(const NamedDecl *ND0, const NamedDecl *ND1) {
-  return isa<TemplateTypeParmDecl>(ND0) || isa<TemplateTypeParmDecl>(ND1);
-}
-
-static bool isMemberOf(const ConfusableIdentifierCheck::ContextInfo *DC0,
-                       const ConfusableIdentifierCheck::ContextInfo *DC1) {
-  return llvm::is_contained(DC1->Bases, DC0->PrimaryContext);
-}
-
-static bool enclosesContext(const ConfusableIdentifierCheck::ContextInfo *DC0,
-                            const ConfusableIdentifierCheck::ContextInfo *DC1) {
-  if (DC0->PrimaryContext == DC1->PrimaryContext)
-    return true;
-
-  return llvm::is_contained(DC0->PrimaryContexts, DC1->PrimaryContext) ||
-         llvm::is_contained(DC1->PrimaryContexts, DC0->PrimaryContext);
-}
-
-static bool mayShadow(const NamedDecl *ND0,
-                      const ConfusableIdentifierCheck::ContextInfo *DC0,
-                      const NamedDecl *ND1,
-                      const ConfusableIdentifierCheck::ContextInfo *DC1) {
-
-  if (!DC0->Bases.empty() && !DC1->Bases.empty()) {
-    // if any of the declaration is a non-private member of the other
-    // declaration, it's shadowed by the former
-
-    if (ND1->getAccess() != AS_private && isMemberOf(DC1, DC0))
-      return true;
+namespace {
+struct Entry {
+  const NamedDecl *ND;
+  const Decl *Parent;
+  bool FromDerivedClass;
+};
+} // namespace
 
-    if (ND0->getAccess() != AS_private && isMemberOf(DC0, DC1))
+// Map from a context to the declarations in that context with the current
+// skeleton. At most one entry per distinct identifier is tracked. The
+// context is usually a `DeclContext`, but can also be a template declaration
+// that has no corresponding context, such as an alias template or variable
+// template.
+using DeclsWithinContextMap =
+    llvm::DenseMap<const Decl *, llvm::SmallVector<Entry, 1>>;
+
+static bool addToContext(DeclsWithinContextMap &DeclsWithinContext,
+                         const Decl *Context, Entry E) {
+  auto &Decls = DeclsWithinContext[Context];
+  if (!Decls.empty() &&
+      Decls.back().ND->getIdentifier() == E.ND->getIdentifier()) {
+    // Already have a declaration with this identifier in this context. Don't
+    // track another one. This means that if an outer name is confusable with an
+    // inner name, we'll only diagnose the outer name once, pointing at the
+    // first inner declaration with that name.
+    if (Decls.back().FromDerivedClass && !E.FromDerivedClass) {
+      // Prefer the declaration that's not from the derived class, because that
+      // conflicts with more declarations.
+      Decls.back() = E;
       return true;
-  }
-
-  if (!mayShadowImpl(DC0->NonTransparentContext, DC1->NonTransparentContext) &&
-      !mayShadowImpl(ND0, ND1))
+    }
     return false;
-
-  return enclosesContext(DC0, DC1);
+  }
+  Decls.push_back(E);
+  return true;
 }
 
-const ConfusableIdentifierCheck::ContextInfo *
-ConfusableIdentifierCheck::getContextInfo(const DeclContext *DC) {
-  const DeclContext *PrimaryContext = DC->getPrimaryContext();
-  auto [It, Inserted] = ContextInfos.try_emplace(PrimaryContext);
-  if (!Inserted)
-    return &It->second;
-
-  ContextInfo &Info = It->second;
-  Info.PrimaryContext = PrimaryContext;
-  Info.NonTransparentContext = PrimaryContext;
+static void addToEnclosingContexts(DeclsWithinContextMap &DeclsWithinContext,
+                                   const Decl *Parent, const NamedDecl *ND) {
+  const Decl *Outer = Parent;
+  while (Outer) {
+    if (const auto *NS = dyn_cast<NamespaceDecl>(Outer))
+      Outer = NS->getCanonicalDecl();
+
+    if (!addToContext(DeclsWithinContext, Outer, {ND, Parent, false}))
+      return;
+
+    if (const auto *RD = dyn_cast<CXXRecordDecl>(Outer)) {
+      RD = RD->getDefinition();
+      if (RD) {
+        RD->forallBases([&](const CXXRecordDecl *Base) {
+          addToContext(DeclsWithinContext, Base, {ND, Parent, true});
+          return true;
+        });
+      }
+    }
 
-  while (Info.NonTransparentContext->isTransparentContext()) {
-    Info.NonTransparentContext = Info.NonTransparentContext->getParent();
-    if (!Info.NonTransparentContext)
+    auto *OuterDC = Outer->getDeclContext();
+    if (!OuterDC)
       break;
+    Outer = cast_or_null<Decl>(OuterDC->getNonTransparentContext());
   }
+}
+
+void ConfusableIdentifierCheck::check(
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  const auto *ND = Result.Nodes.getNodeAs<NamedDecl>("nameddecl");
+  if (!ND)
+    return;
 
-  if (Info.NonTransparentContext)
-    Info.NonTransparentContext =
-        Info.NonTransparentContext->getPrimaryContext();
+  addDeclToCheck(ND,
+                 cast<Decl>(ND->getDeclContext()->getNonTransparentContext()));
 
-  while (DC) {
-    if (!isa<LinkageSpecDecl>(DC) && !isa<ExportDecl>(DC))
-      Info.PrimaryContexts.push_back(DC->getPrimaryContext());
-    DC = DC->getParent();
+  // Associate template parameters with this declaration of this template.
+  if (const auto *TD = dyn_cast<TemplateDecl>(ND)) {
+    for (const NamedDecl *Param : *TD->getTemplateParameters())
+      addDeclToCheck(Param, TD->getTemplatedDecl());
   }
 
-  if (const auto *RD = dyn_cast<CXXRecordDecl>(PrimaryContext)) {
-    RD = RD->getDefinition();
-    if (RD) {
-      Info.Bases.push_back(RD);
-      RD->forallBases([&](const CXXRecordDecl *Base) {
-        Info.Bases.push_back(Base);
-        return false;
-      });
-    }
+  // Associate function parameters with this declaration of this function.
+  if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
+    for (const NamedDecl *Param : FD->parameters())
+      addDeclToCheck(Param, ND);
   }
-
-  return &Info;
 }
 
-void ConfusableIdentifierCheck::check(
-    const ast_matchers::MatchFinder::MatchResult &Result) {
-  const auto *ND = Result.Nodes.getNodeAs<NamedDecl>("nameddecl");
-  if (!ND)
+void ConfusableIdentifierCheck::addDeclToCheck(const NamedDecl *ND,
+                                               const Decl *Parent) {
+  if (!ND || !Parent)
     return;
 
-  IdentifierInfo *NDII = ND->getIdentifier();
+  const IdentifierInfo *NDII = ND->getIdentifier();
   if (!NDII)
     return;
 
@@ -188,34 +187,78 @@ void ConfusableIdentifierCheck::check(
   if (NDName.empty())
     return;
 
-  const ContextInfo *Info = getContextInfo(ND->getDeclContext());
+  NameToDecls[NDII].push_back({ND, Parent});
+}
+
+void ConfusableIdentifierCheck::onEndOfTranslationUnit() {
+  llvm::StringMap<llvm::SmallVector<const IdentifierInfo *, 1>> SkeletonToNames;
+  // Compute the skeleton for each identifier.
+  for (auto &[Ident, Decls] : NameToDecls) {
+    SkeletonToNames[skeleton(Ident->getName())].push_back(Ident);
+  }
 
-  llvm::SmallVector<Entry> &Mapped = Mapper[skeleton(NDName)];
-  for (const Entry &E : Mapped) {
-    if (!mayShadow(ND, Info, E.Declaration, E.Info))
+  // Visit each skeleton with more than one identifier.
+  for (auto &[Skel, Idents] : SkeletonToNames) {
+    if (Idents.size() < 2) {
       continue;
+    }
 
-    const IdentifierInfo *ONDII = E.Declaration->getIdentifier();
-    StringRef ONDName = ONDII->getName();
-    if (ONDName == NDName)
-      continue;
+    // Find the declaration contexts that transitively contain each identifier.
+    DeclsWithinContextMap DeclsWithinContext;
+    for (const IdentifierInfo *II : Idents) {
+      for (auto [ND, Parent] : NameToDecls[II]) {
+        addToEnclosingContexts(DeclsWithinContext, Parent, ND);
+      }
+    }
 
-    diag(ND->getLocation(), "%0 is confusable with %1") << ND << E.Declaration;
-    diag(E.Declaration->getLocation(), "other declaration found here",
-         DiagnosticIDs::Note);
+    // Check to see if any declaration is declared in a context that
+    // transitively contains another declaration with a different identifier but
+    // the same skeleton.
+    for (const IdentifierInfo *II : Idents) {
+      for (auto [OuterND, OuterParent] : NameToDecls[II]) {
+        for (Entry Inner : DeclsWithinContext[OuterParent]) {
+          // Don't complain if the identifiers are the same.
+          if (OuterND->getIdentifier() == Inner.ND->getIdentifier())
+            continue;
+
+          // Don't complain about a derived-class name shadowing a base class
+          // private member.
+          if (OuterND->getAccess() == AS_private && Inner.FromDerivedClass)
+            continue;
+
+          // If the declarations are in the same context, only diagnose the
+          // later one.
+          if (OuterParent == Inner.Parent &&
+              Inner.ND->getASTContext()
+                  .getSourceManager()
+                  .isBeforeInTranslationUnit(Inner.ND->getLocation(),
+                                             OuterND->getLocation()))
+            continue;
+
+          diag(Inner.ND->getLocation(), "%0 is confusable with %1")
+              << Inner.ND << OuterND;
+          diag(OuterND->getLocation(), "other declaration found here",
+               DiagnosticIDs::Note);
+        }
+      }
+    }
   }
 
-  Mapped.push_back({ND, Info});
-}
-
-void ConfusableIdentifierCheck::onEndOfTranslationUnit() {
-  Mapper.clear();
-  ContextInfos.clear();
+  NameToDecls.clear();
 }
 
 void ConfusableIdentifierCheck::registerMatchers(
     ast_matchers::MatchFinder *Finder) {
-  Finder->addMatcher(ast_matchers::namedDecl().bind("nameddecl"), this);
+  // Parameter declarations sometimes use the translation unit or some outer
+  // enclosing context as their `DeclContext`, instead of their parent, so
+  // we handle them specially in `check`.
+  auto AnyParamDecl = ast_matchers::anyOf(
+      ast_matchers::parmVarDecl(), ast_matchers::templateTypeParmDecl(),
+      ast_matchers::nonTypeTemplateParmDecl(),
+      ast_matchers::templateTemplateParmDecl());
+  Finder->addMatcher(ast_matchers::namedDecl(ast_matchers::unless(AnyParamDecl))
+                         .bind("nameddecl"),
+                     this);
 }
 
 } // namespace clang::tidy::misc
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h
index f3b0c8ed00306..9cce6cce67682 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h
@@ -1,5 +1,4 @@
-//===--- ConfusableIdentifierCheck.h - clang-tidy
-//-------------------------------*- C++ -*-===//
+//===--- ConfusableIdentifierCheck.h - clang-tidy ---------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,7 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_CONFUSABLE_IDENTIFIER_CHECK_H
 
 #include "../ClangTidyCheck.h"
-#include <unordered_map>
+#include "llvm/ADT/DenseMap.h"
 
 namespace clang::tidy::misc {
 
@@ -31,23 +30,13 @@ class ConfusableIdentifierCheck : public ClangTidyCheck {
     return TK_IgnoreUnlessSpelledInSource;
   }
 
-  struct ContextInfo {
-    const DeclContext *PrimaryContext;
-    const DeclContext *NonTransparentContext;
-    llvm::SmallVector<const DeclContext *> PrimaryContexts;
-    llvm::SmallVector<const CXXRecordDecl *> Bases;
-  };
-
 private:
-  struct Entry {
-    const NamedDecl *Declaration;
-    const ContextInfo *Info;
-  };
-
-  const ContextInfo *getContextInfo(const DeclContext *DC);
+  void addDeclToCheck(const NamedDecl *ND, const Decl *Parent);
 
-  llvm::StringMap<llvm::SmallVector<Entry>> Mapper;
-  std::unordered_map<const DeclContext *, ContextInfo> ContextInfos;
+  llvm::DenseMap<
+      const IdentifierInfo *,
+      llvm::SmallVector<std::pair<const NamedDecl *, const Decl *>, 1>>
+      NameToDecls;
 };
 
 } // namespace clang::tidy::misc
diff --git a/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
index afc4897eeb2ae..3fdaf9239f6af 100644
--- a/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
@@ -66,8 +66,11 @@ void UnconventionalAssignOperatorCheck::registerMatchers(
                                 hasArgument(0, cxxThisExpr())),
             cxxOperatorCallExpr(
                 hasOverloadedOperatorName("="),
-                hasArgument(
-                    0, unaryOperator(hasOperatorName("*"),
+                hasArgument(0, unaryOperator(hasOperatorName("*"),
+                                             hasUnaryOperand(cxxThisExpr())))),
+            binaryOperator(
+                hasOperatorName("="),
+                hasLHS(unaryOperator(hasOperatorName("*"),
                                      hasUnaryOperand(cxxThisExpr())))))))));
   const auto IsGoodAssign = cxxMethodDecl(IsAssign, HasGoodReturnType);
 
diff --git a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
index bab1167fb15ff..619a27b2f9bb6 100644
--- a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
@@ -42,6 +42,7 @@ add_clang_library(clangTidyModernizeModule STATIC
   UseNullptrCheck.cpp
   UseOverrideCheck.cpp
   UseRangesCheck.cpp
+  UseScopedLockCheck.cpp
   UseStartsEndsWithCheck.cpp
   UseStdFormatCheck.cpp
   UseStdNumbersCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp b/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
index 0cf59b6e0216a..fdf38bc4b6308 100644
--- a/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
@@ -43,6 +43,7 @@
 #include "UseNullptrCheck.h"
 #include "UseOverrideCheck.h"
 #include "UseRangesCheck.h"
+#include "UseScopedLockCheck.h"
 #include "UseStartsEndsWithCheck.h"
 #include "UseStdFormatCheck.h"
 #include "UseStdNumbersCheck.h"
@@ -80,6 +81,8 @@ class ModernizeModule : public ClangTidyModule {
     CheckFactories.registerCheck<UseIntegerSignComparisonCheck>(
         "modernize-use-integer-sign-comparison");
     CheckFactories.registerCheck<UseRangesCheck>("modernize-use-ranges");
+    CheckFactories.registerCheck<UseScopedLockCheck>(
+        "modernize-use-scoped-lock");
     CheckFactories.registerCheck<UseStartsEndsWithCheck>(
         "modernize-use-starts-ends-with");
     CheckFactories.registerCheck<UseStdFormatCheck>("modernize-use-std-format");
diff --git a/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
new file mode 100644
index 0000000000000..9c2fc9e06fb45
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
@@ -0,0 +1,311 @@
+//===--- UseScopedLockCheck.cpp - clang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UseScopedLockCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Type.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/Lexer.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::modernize {
+
+static bool isLockGuardDecl(const NamedDecl *Decl) {
+  return Decl->getDeclName().isIdentifier() &&
+         Decl->getName() == "lock_guard" && Decl->isInStdNamespace();
+}
+
+static bool isLockGuard(const QualType &Type) {
+  if (const auto *Record = Type->getAs<RecordType>())
+    if (const RecordDecl *Decl = Record->getDecl())
+      return isLockGuardDecl(Decl);
+
+  if (const auto *TemplateSpecType = Type->getAs<TemplateSpecializationType>())
+    if (const TemplateDecl *Decl =
+            TemplateSpecType->getTemplateName().getAsTemplateDecl())
+      return isLockGuardDecl(Decl);
+
+  return false;
+}
+
+static llvm::SmallVector<const VarDecl *>
+getLockGuardsFromDecl(const DeclStmt *DS) {
+  llvm::SmallVector<const VarDecl *> LockGuards;
+
+  for (const Decl *Decl : DS->decls()) {
+    if (const auto *VD = dyn_cast<VarDecl>(Decl)) {
+      const QualType Type =
+          VD->getType().getCanonicalType().getUnqualifiedType();
+      if (isLockGuard(Type))
+        LockGuards.push_back(VD);
+    }
+  }
+
+  return LockGuards;
+}
+
+// Scans through the statements in a block and groups consecutive
+// 'std::lock_guard' variable declarations together.
+static llvm::SmallVector<llvm::SmallVector<const VarDecl *>>
+findLocksInCompoundStmt(const CompoundStmt *Block,
+                        const ast_matchers::MatchFinder::MatchResult &Result) {
+  // store groups of consecutive 'std::lock_guard' declarations
+  llvm::SmallVector<llvm::SmallVector<const VarDecl *>> LockGuardGroups;
+  llvm::SmallVector<const VarDecl *> CurrentLockGuardGroup;
+
+  auto AddAndClearCurrentGroup = [&]() {
+    if (!CurrentLockGuardGroup.empty()) {
+      LockGuardGroups.push_back(CurrentLockGuardGroup);
+      CurrentLockGuardGroup.clear();
+    }
+  };
+
+  for (const Stmt *Stmt : Block->body()) {
+    if (const auto *DS = dyn_cast<DeclStmt>(Stmt)) {
+      llvm::SmallVector<const VarDecl *> LockGuards = getLockGuardsFromDecl(DS);
+
+      if (!LockGuards.empty()) {
+        CurrentLockGuardGroup.append(LockGuards);
+        continue;
+      }
+    }
+    AddAndClearCurrentGroup();
+  }
+
+  AddAndClearCurrentGroup();
+
+  return LockGuardGroups;
+}
+
+static TemplateSpecializationTypeLoc
+getTemplateLockGuardTypeLoc(const TypeSourceInfo *SourceInfo) {
+  const TypeLoc Loc = SourceInfo->getTypeLoc();
+
+  const auto ElaboratedLoc = Loc.getAs<ElaboratedTypeLoc>();
+  if (!ElaboratedLoc)
+    return {};
+
+  return ElaboratedLoc.getNamedTypeLoc().getAs<TemplateSpecializationTypeLoc>();
+}
+
+// Find the exact source range of the 'lock_guard' token
+static SourceRange getLockGuardRange(const TypeSourceInfo *SourceInfo) {
+  const TypeLoc LockGuardTypeLoc = SourceInfo->getTypeLoc();
+
+  return SourceRange(LockGuardTypeLoc.getBeginLoc(),
+                     LockGuardTypeLoc.getEndLoc());
+}
+
+// Find the exact source range of the 'lock_guard' name token
+static SourceRange getLockGuardNameRange(const TypeSourceInfo *SourceInfo) {
+  const TemplateSpecializationTypeLoc TemplateLoc =
+      getTemplateLockGuardTypeLoc(SourceInfo);
+  if (!TemplateLoc)
+    return {};
+
+  return SourceRange(TemplateLoc.getTemplateNameLoc(),
+                     TemplateLoc.getLAngleLoc().getLocWithOffset(-1));
+}
+
+const static StringRef UseScopedLockMessage =
+    "use 'std::scoped_lock' instead of 'std::lock_guard'";
+
+UseScopedLockCheck::UseScopedLockCheck(StringRef Name,
+                                       ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      WarnOnSingleLocks(Options.get("WarnOnSingleLocks", true)),
+      WarnOnUsingAndTypedef(Options.get("WarnOnUsingAndTypedef", true)) {}
+
+void UseScopedLockCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "WarnOnSingleLocks", WarnOnSingleLocks);
+  Options.store(Opts, "WarnOnUsingAndTypedef", WarnOnUsingAndTypedef);
+}
+
+void UseScopedLockCheck::registerMatchers(MatchFinder *Finder) {
+  const auto LockGuardClassDecl =
+      namedDecl(hasName("lock_guard"), isInStdNamespace());
+
+  const auto LockGuardType = qualType(anyOf(
+      hasUnqualifiedDesugaredType(
+          recordType(hasDeclaration(LockGuardClassDecl))),
+      elaboratedType(namesType(hasUnqualifiedDesugaredType(
+          templateSpecializationType(hasDeclaration(LockGuardClassDecl)))))));
+
+  const auto LockVarDecl = varDecl(hasType(LockGuardType));
+
+  if (WarnOnSingleLocks) {
+    Finder->addMatcher(
+        compoundStmt(
+            unless(isExpansionInSystemHeader()),
+            has(declStmt(has(LockVarDecl)).bind("lock-decl-single")),
+            unless(has(declStmt(unless(equalsBoundNode("lock-decl-single")),
+                                has(LockVarDecl))))),
+        this);
+  }
+
+  Finder->addMatcher(
+      compoundStmt(unless(isExpansionInSystemHeader()),
+                   has(declStmt(has(LockVarDecl)).bind("lock-decl-multiple")),
+                   has(declStmt(unless(equalsBoundNode("lock-decl-multiple")),
+                                has(LockVarDecl))))
+          .bind("block-multiple"),
+      this);
+
+  if (WarnOnUsingAndTypedef) {
+    // Match 'typedef std::lock_guard<std::mutex> Lock'
+    Finder->addMatcher(typedefDecl(unless(isExpansionInSystemHeader()),
+                                   hasUnderlyingType(LockGuardType))
+                           .bind("lock-guard-typedef"),
+                       this);
+
+    // Match 'using Lock = std::lock_guard<std::mutex>'
+    Finder->addMatcher(
+        typeAliasDecl(
+            unless(isExpansionInSystemHeader()),
+            hasType(elaboratedType(namesType(templateSpecializationType(
+                hasDeclaration(LockGuardClassDecl))))))
+            .bind("lock-guard-using-alias"),
+        this);
+
+    // Match 'using std::lock_guard'
+    Finder->addMatcher(
+        usingDecl(unless(isExpansionInSystemHeader()),
+                  hasAnyUsingShadowDecl(hasTargetDecl(LockGuardClassDecl)))
+            .bind("lock-guard-using-decl"),
+        this);
+  }
+}
+
+void UseScopedLockCheck::check(const MatchFinder::MatchResult &Result) {
+  if (const auto *DS = Result.Nodes.getNodeAs<DeclStmt>("lock-decl-single")) {
+    llvm::SmallVector<const VarDecl *> Decls = getLockGuardsFromDecl(DS);
+    diagOnMultipleLocks({Decls}, Result);
+    return;
+  }
+
+  if (const auto *Compound =
+          Result.Nodes.getNodeAs<CompoundStmt>("block-multiple")) {
+    diagOnMultipleLocks(findLocksInCompoundStmt(Compound, Result), Result);
+    return;
+  }
+
+  if (const auto *Typedef =
+          Result.Nodes.getNodeAs<TypedefDecl>("lock-guard-typedef")) {
+    diagOnSourceInfo(Typedef->getTypeSourceInfo(), Result);
+    return;
+  }
+
+  if (const auto *UsingAlias =
+          Result.Nodes.getNodeAs<TypeAliasDecl>("lock-guard-using-alias")) {
+    diagOnSourceInfo(UsingAlias->getTypeSourceInfo(), Result);
+    return;
+  }
+
+  if (const auto *Using =
+          Result.Nodes.getNodeAs<UsingDecl>("lock-guard-using-decl")) {
+    diagOnUsingDecl(Using, Result);
+  }
+}
+
+void UseScopedLockCheck::diagOnSingleLock(
+    const VarDecl *LockGuard, const MatchFinder::MatchResult &Result) {
+  auto Diag = diag(LockGuard->getBeginLoc(), UseScopedLockMessage);
+
+  const SourceRange LockGuardTypeRange =
+      getLockGuardRange(LockGuard->getTypeSourceInfo());
+
+  if (LockGuardTypeRange.isInvalid())
+    return;
+
+  // Create Fix-its only if we can find the constructor call to properly handle
+  // 'std::lock_guard l(m, std::adopt_lock)' case.
+  const auto *CtorCall = dyn_cast<CXXConstructExpr>(LockGuard->getInit());
+  if (!CtorCall)
+    return;
+
+  if (CtorCall->getNumArgs() == 1) {
+    Diag << FixItHint::CreateReplacement(LockGuardTypeRange,
+                                         "std::scoped_lock");
+    return;
+  }
+
+  if (CtorCall->getNumArgs() == 2) {
+    const Expr *const *CtorArgs = CtorCall->getArgs();
+
+    const Expr *MutexArg = CtorArgs[0];
+    const Expr *AdoptLockArg = CtorArgs[1];
+
+    const StringRef MutexSourceText = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(MutexArg->getSourceRange()),
+        *Result.SourceManager, Result.Context->getLangOpts());
+    const StringRef AdoptLockSourceText = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(AdoptLockArg->getSourceRange()),
+        *Result.SourceManager, Result.Context->getLangOpts());
+
+    Diag << FixItHint::CreateReplacement(LockGuardTypeRange, "std::scoped_lock")
+         << FixItHint::CreateReplacement(
+                SourceRange(MutexArg->getBeginLoc(), AdoptLockArg->getEndLoc()),
+                (llvm::Twine(AdoptLockSourceText) + ", " + MutexSourceText)
+                    .str());
+    return;
+  }
+
+  llvm_unreachable("Invalid argument number of std::lock_guard constructor");
+}
+
+void UseScopedLockCheck::diagOnMultipleLocks(
+    const llvm::SmallVector<llvm::SmallVector<const VarDecl *>> &LockGroups,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  for (const llvm::SmallVector<const VarDecl *> &Group : LockGroups) {
+    if (Group.size() == 1) {
+      if (WarnOnSingleLocks)
+        diagOnSingleLock(Group[0], Result);
+    } else {
+      diag(Group[0]->getBeginLoc(),
+           "use single 'std::scoped_lock' instead of multiple "
+           "'std::lock_guard'");
+
+      for (const VarDecl *Lock : llvm::drop_begin(Group))
+        diag(Lock->getLocation(), "additional 'std::lock_guard' declared here",
+             DiagnosticIDs::Note);
+    }
+  }
+}
+
+void UseScopedLockCheck::diagOnSourceInfo(
+    const TypeSourceInfo *LockGuardSourceInfo,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  const TypeLoc TL = LockGuardSourceInfo->getTypeLoc();
+
+  if (const auto ElaboratedTL = TL.getAs<ElaboratedTypeLoc>()) {
+    auto Diag = diag(ElaboratedTL.getBeginLoc(), UseScopedLockMessage);
+
+    const SourceRange LockGuardRange =
+        getLockGuardNameRange(LockGuardSourceInfo);
+    if (LockGuardRange.isInvalid())
+      return;
+
+    Diag << FixItHint::CreateReplacement(LockGuardRange, "scoped_lock");
+  }
+}
+
+void UseScopedLockCheck::diagOnUsingDecl(
+    const UsingDecl *UsingDecl,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  diag(UsingDecl->getLocation(), UseScopedLockMessage)
+      << FixItHint::CreateReplacement(UsingDecl->getLocation(), "scoped_lock");
+}
+
+} // namespace clang::tidy::modernize
diff --git a/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h
new file mode 100644
index 0000000000000..a5697805c15ca
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h
@@ -0,0 +1,54 @@
+//===--- UseScopedLockCheck.h - clang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESCOPEDLOCKCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESCOPEDLOCKCHECK_H
+
+#include "../ClangTidyCheck.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/Stmt.h"
+#include <optional>
+
+namespace clang::tidy::modernize {
+
+/// Finds uses of ``std::lock_guard`` and suggests replacing them with C++17's
+/// alternative ``std::scoped_lock``.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/modernize/use-scoped-lock.html
+class UseScopedLockCheck : public ClangTidyCheck {
+public:
+  UseScopedLockCheck(StringRef Name, ClangTidyContext *Context);
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus17;
+  }
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+
+private:
+  void diagOnSingleLock(const VarDecl *LockGuard,
+                        const ast_matchers::MatchFinder::MatchResult &Result);
+  void diagOnMultipleLocks(
+      const llvm::SmallVector<llvm::SmallVector<const VarDecl *>> &LockGroups,
+      const ast_matchers::MatchFinder::MatchResult &Result);
+  void diagOnSourceInfo(const TypeSourceInfo *LockGuardSourceInfo,
+                        const ast_matchers::MatchFinder::MatchResult &Result);
+  void diagOnUsingDecl(const UsingDecl *UsingDecl,
+                       const ast_matchers::MatchFinder::MatchResult &Result);
+
+  const bool WarnOnSingleLocks;
+  const bool WarnOnUsingAndTypedef;
+};
+
+} // namespace clang::tidy::modernize
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESCOPEDLOCKCHECK_H
diff --git a/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
index 30fcba367db67..936a906651f16 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
@@ -119,7 +119,8 @@ void UseUsingCheck::check(const MatchFinder::MatchResult &Result) {
   if (StartLoc.isMacroID() && IgnoreMacros)
     return;
 
-  static const char *UseUsingWarning = "use 'using' instead of 'typedef'";
+  static constexpr llvm::StringLiteral UseUsingWarning =
+      "use 'using' instead of 'typedef'";
 
   // Warn at StartLoc but do not fix if there is macro or array.
   if (MatchedDecl->getUnderlyingType()->isArrayType() || StartLoc.isMacroID()) {
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
index 969bf2d1add6b..7f1882c775c59 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
@@ -72,11 +72,13 @@ static SourceLocation getInlineTokenLocation(SourceRange RangeLocation,
 }
 
 void RedundantInlineSpecifierCheck::registerMatchers(MatchFinder *Finder) {
+  const auto IsPartOfRecordDecl = hasAncestor(recordDecl());
   Finder->addMatcher(
       functionDecl(isInlineSpecified(),
-                   anyOf(isConstexpr(), isDeleted(), isDefaulted(),
+                   anyOf(isConstexpr(), isDeleted(),
+                         allOf(isDefaulted(), IsPartOfRecordDecl),
                          isInternalLinkage(StrictMode),
-                         allOf(isDefinition(), hasAncestor(recordDecl()))))
+                         allOf(isDefinition(), IsPartOfRecordDecl)))
           .bind("fun_decl"),
       this);
 
@@ -88,7 +90,6 @@ void RedundantInlineSpecifierCheck::registerMatchers(MatchFinder *Finder) {
         this);
 
   if (getLangOpts().CPlusPlus17) {
-    const auto IsPartOfRecordDecl = hasAncestor(recordDecl());
     Finder->addMatcher(
         varDecl(
             isInlineSpecified(),
diff --git a/clang-tools-extra/clangd/InlayHints.cpp b/clang-tools-extra/clangd/InlayHints.cpp
index 20a238612a7e4..197c62c40dcf0 100644
--- a/clang-tools-extra/clangd/InlayHints.cpp
+++ b/clang-tools-extra/clangd/InlayHints.cpp
@@ -33,7 +33,6 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/ADT/identity.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -339,53 +338,6 @@ QualType maybeDesugar(ASTContext &AST, QualType QT) {
   return QT;
 }
 
-// Given a callee expression `Fn`, if the call is through a function pointer,
-// try to find the declaration of the corresponding function pointer type,
-// so that we can recover argument names from it.
-// FIXME: This function is mostly duplicated in SemaCodeComplete.cpp; unify.
-static FunctionProtoTypeLoc getPrototypeLoc(Expr *Fn) {
-  TypeLoc Target;
-  Expr *NakedFn = Fn->IgnoreParenCasts();
-  if (const auto *T = NakedFn->getType().getTypePtr()->getAs<TypedefType>()) {
-    Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc();
-  } else if (const auto *DR = dyn_cast<DeclRefExpr>(NakedFn)) {
-    const auto *D = DR->getDecl();
-    if (const auto *const VD = dyn_cast<VarDecl>(D)) {
-      Target = VD->getTypeSourceInfo()->getTypeLoc();
-    }
-  }
-
-  if (!Target)
-    return {};
-
-  // Unwrap types that may be wrapping the function type
-  while (true) {
-    if (auto P = Target.getAs<PointerTypeLoc>()) {
-      Target = P.getPointeeLoc();
-      continue;
-    }
-    if (auto A = Target.getAs<AttributedTypeLoc>()) {
-      Target = A.getModifiedLoc();
-      continue;
-    }
-    if (auto P = Target.getAs<ParenTypeLoc>()) {
-      Target = P.getInnerLoc();
-      continue;
-    }
-    break;
-  }
-
-  if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
-    // In some edge cases the AST can contain a "trivial" FunctionProtoTypeLoc
-    // which has null parameters. Avoid these as they don't contain useful
-    // information.
-    if (llvm::all_of(F.getParams(), llvm::identity<ParmVarDecl *>()))
-      return F;
-  }
-
-  return {};
-}
-
 ArrayRef<const ParmVarDecl *>
 maybeDropCxxExplicitObjectParameters(ArrayRef<const ParmVarDecl *> Params) {
   if (!Params.empty() && Params.front()->isExplicitObjectParameter())
@@ -514,7 +466,8 @@ class InlayHintVisitor : public RecursiveASTVisitor<InlayHintVisitor> {
       Callee.Decl = FD;
     else if (const auto *FTD = dyn_cast<FunctionTemplateDecl>(CalleeDecls[0]))
       Callee.Decl = FTD->getTemplatedDecl();
-    else if (FunctionProtoTypeLoc Loc = getPrototypeLoc(E->getCallee()))
+    else if (FunctionProtoTypeLoc Loc =
+                 Resolver->getFunctionProtoTypeLoc(E->getCallee()))
       Callee.Loc = Loc;
     else
       return true;
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp b/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
index d6556bba14725..c9704492bf1cd 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
@@ -341,7 +341,7 @@ const FunctionDecl *findTarget(const FunctionDecl *FD) {
 
 // Returns the beginning location for a FunctionDecl. Returns location of
 // template keyword for templated functions.
-const SourceLocation getBeginLoc(const FunctionDecl *FD) {
+SourceLocation getBeginLoc(const FunctionDecl *FD) {
   // Include template parameter list.
   if (auto *FTD = FD->getDescribedFunctionTemplate())
     return FTD->getBeginLoc();
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
index 90dac3b76c648..c74250ccbe9ea 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
@@ -359,9 +359,9 @@ struct ParsedBinaryOperator {
 //    +   c <- End
 //   / \
 //  a   b <- Start
-const SourceRange getBinaryOperatorRange(const SelectionTree::Node &N,
-                                         const SourceManager &SM,
-                                         const LangOptions &LangOpts) {
+SourceRange getBinaryOperatorRange(const SelectionTree::Node &N,
+                                   const SourceManager &SM,
+                                   const LangOptions &LangOpts) {
   // If N is not a suitable binary operator, bail out.
   ParsedBinaryOperator Op;
   if (!Op.parse(N.ignoreImplicit()) || !Op.associative() ||
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index b11d194da04db..f287439f10cab 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -668,7 +668,6 @@ class FlagsConfigProvider : public config::Provider {
     std::optional<Config::ExternalIndexSpec> IndexSpec;
     std::optional<Config::BackgroundPolicy> BGPolicy;
     std::optional<Config::ArgumentListsPolicy> ArgumentLists;
-    std::optional<Config::HeaderInsertionPolicy> HeaderInsertionPolicy;
 
     // If --compile-commands-dir arg was invoked, check value and override
     // default path.
@@ -713,11 +712,6 @@ class FlagsConfigProvider : public config::Provider {
       BGPolicy = Config::BackgroundPolicy::Skip;
     }
 
-    // If CLI has set never, use that regardless of what the config files have
-    if (HeaderInsertion == Config::HeaderInsertionPolicy::NeverInsert) {
-      HeaderInsertionPolicy = Config::HeaderInsertionPolicy::NeverInsert;
-    }
-
     if (std::optional<bool> Enable = shouldEnableFunctionArgSnippets()) {
       ArgumentLists = *Enable ? Config::ArgumentListsPolicy::FullPlaceholders
                               : Config::ArgumentListsPolicy::Delimiters;
@@ -732,8 +726,8 @@ class FlagsConfigProvider : public config::Provider {
         C.Index.Background = *BGPolicy;
       if (ArgumentLists)
         C.Completion.ArgumentLists = *ArgumentLists;
-      if (HeaderInsertionPolicy)
-        C.Completion.HeaderInsertion = *HeaderInsertionPolicy;
+      if (HeaderInsertion.getNumOccurrences())
+        C.Completion.HeaderInsertion = HeaderInsertion;
       if (AllScopesCompletion.getNumOccurrences())
         C.Completion.AllScopes = AllScopesCompletion;
 
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 311f0d98904ad..b7c64c7a06745 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -4363,6 +4363,36 @@ TEST(CompletionTest, PreambleFromDifferentTarget) {
   EXPECT_THAT(Result.Completions, Not(testing::IsEmpty()));
   EXPECT_THAT(Signatures.signatures, Not(testing::IsEmpty()));
 }
+
+TEST(CompletionTest, SkipExplicitObjectParameter) {
+  Annotations Code(R"cpp(
+    struct A {
+      void foo(this auto&& self, int arg); 
+    };
+
+    int main() {
+      A a {};
+      a.^
+    }
+  )cpp");
+
+  auto TU = TestTU::withCode(Code.code());
+  TU.ExtraArgs = {"-std=c++23"};
+
+  auto Preamble = TU.preamble();
+  ASSERT_TRUE(Preamble);
+
+  CodeCompleteOptions Opts{};
+
+  MockFS FS;
+  auto Inputs = TU.inputs(FS);
+  auto Result = codeComplete(testPath(TU.Filename), Code.point(),
+                             Preamble.get(), Inputs, Opts);
+
+  EXPECT_THAT(Result.Completions,
+              ElementsAre(AllOf(named("foo"), signature("(int arg)"),
+                                snippetSuffix("(${1:int arg})"))));
+}
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 934d52086b3b9..e021d6350694e 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -142,6 +142,12 @@ New checks
   Finds unscoped (non-class) ``enum`` declarations and suggests using
   ``enum class`` instead.
 
+- New :doc:`modernize-use-scoped-lock
+  <clang-tidy/checks/modernize/use-scoped-lock>` check.
+
+  Finds uses of ``std::lock_guard`` and suggests replacing them with C++17's
+  alternative ``std::scoped_lock``.
+
 - New :doc:`portability-avoid-pragma-once
   <clang-tidy/checks/portability/avoid-pragma-once>` check.
 
@@ -206,6 +212,21 @@ Changes in existing checks
   <clang-tidy/checks/cppcoreguidelines/avoid-goto>` check by adding the option
   `IgnoreMacros` to ignore ``goto`` labels defined in macros.
 
+- Improved :doc:`cppcoreguidelines-missing-std-forward
+  <clang-tidy/checks/cppcoreguidelines/missing-std-forward>` check by adding a
+  flag to specify the function used for forwarding instead of ``std::forward``.
+   
+- Improved :doc:`cppcoreguidelines-pro-bounds-pointer-arithmetic
+  <clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic>` check by
+  fixing false positives when calling indexing operators that do not perform
+  pointer arithmetic in template, for example ``std::map::operator[]`` and
+  when pointer arithmetic was used through type aliases.
+
+- Improved :doc:`cppcoreguidelines-rvalue-reference-param-not-moved
+  <clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved>` check
+  by adding a flag to specify the function used for moving instead of
+  ``std::move``.
+
 - Improved :doc:`cppcoreguidelines-special-member-functions
   <clang-tidy/checks/cppcoreguidelines/special-member-functions>` check by
   adding the option `IgnoreMacros` to ignore classes defined in macros.
@@ -245,6 +266,11 @@ Changes in existing checks
   <clang-tidy/checks/misc/redundant-expression>` check by providing additional
   examples and fixing some macro related false positives.
 
+- Improved :doc:`misc-unconventional-assign-operator
+  <clang-tidy/checks/misc/unconventional-assign-operator>` check by fixing
+  false positives when copy assignment operator function in a template class
+  returns the result of another assignment to ``*this`` (``return *this=...``).
+
 - Improved :doc:`misc-unused-using-decls
   <clang-tidy/checks/misc/unused-using-decls>` check by fixing false positives
   on ``operator""`` with template parameters.
@@ -313,6 +339,10 @@ Changes in existing checks
   <clang-tidy/checks/readability/qualified-auto>` check by adding the option
   `AllowedTypes`, that excludes specified types from adding qualifiers.
 
+- Improved :doc:`readability-redundant-inline-specifier
+  <clang-tidy/checks/readability/redundant-inline-specifier>` check by fixing
+  false positives on out-of-line explicitly defaulted functions.
+
 - Improved :doc:`readability-redundant-smartptr-get
   <clang-tidy/checks/readability/redundant-smartptr-get>` check by fixing
   some false positives involving smart pointers to arrays.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst
index 0c311b59a5d5a..62e38fcd3b9dc 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst
@@ -35,6 +35,13 @@ Example:
     f(1, 2); // Incorrect - may not invoke the desired qualified function operator
   }
 
+Options
+-------
+
+.. option:: ForwardFunction
+
+   Specify the function used for forwarding. Default is `::std::forward`.
+
 This check implements `F.19
 <http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-forward>`_
 from the C++ Core Guidelines.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
index ffa3a9d61e48e..2fea9f16b3bb0 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
@@ -79,6 +79,10 @@ Options
       T other = std::forward<T>(t);
     }
 
+.. option:: MoveFunction
+
+   Specify the function used for moving. Default is `::std::move`.
+
 This check implements `F.18
 <http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#f18-for-will-move-from-parameters-pass-by-x-and-stdmove-the-parameter>`_
 from the C++ Core Guidelines.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index ccb78ee45e9c4..5098582d0c42b 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -312,6 +312,7 @@ Clang-Tidy Checks
    :doc:`modernize-use-nullptr <modernize/use-nullptr>`, "Yes"
    :doc:`modernize-use-override <modernize/use-override>`, "Yes"
    :doc:`modernize-use-ranges <modernize/use-ranges>`, "Yes"
+   :doc:`modernize-use-scoped-lock <modernize/use-scoped-lock>`, "Yes"
    :doc:`modernize-use-starts-ends-with <modernize/use-starts-ends-with>`, "Yes"
    :doc:`modernize-use-std-format <modernize/use-std-format>`, "Yes"
    :doc:`modernize-use-std-numbers <modernize/use-std-numbers>`, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst
new file mode 100644
index 0000000000000..d184f1aefd806
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst
@@ -0,0 +1,101 @@
+.. title:: clang-tidy - modernize-use-scoped-lock
+
+modernize-use-scoped-lock
+=========================
+
+Finds uses of ``std::lock_guard`` and suggests replacing them with C++17's
+alternative ``std::scoped_lock``.
+
+Fix-its are provided for single declarations of ``std::lock_guard`` and warning
+is emitted for multiple declarations of ``std::lock_guard`` that can be
+replaced with a single declaration of ``std::scoped_lock``.
+
+Examples
+--------
+
+Single ``std::lock_guard`` declaration:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::lock_guard<std::mutex> L(M);
+
+
+Transforms to:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::scoped_lock L(M);
+
+Single ``std::lock_guard`` declaration with ``std::adopt_lock``:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::lock(M);
+  std::lock_guard<std::mutex> L(M, std::adopt_lock);
+
+
+Transforms to:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::lock(M);
+  std::scoped_lock L(std::adopt_lock, M);
+
+Multiple ``std::lock_guard`` declarations only emit warnings:
+
+.. code-block:: c++
+
+  std::mutex M1, M2;
+  std::lock(M1, M2);
+  std::lock_guard Lock1(M, std::adopt_lock); // warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  std::lock_guard Lock2(M, std::adopt_lock); // note: additional 'std::lock_guard' declared here
+
+
+Limitations
+-----------
+
+The check will not emit warnings if ``std::lock_guard`` is used implicitly via
+``template`` parameter:
+
+.. code-block:: c++
+
+  template <template <typename> typename Lock>
+  void TemplatedLock() {
+    std::mutex M;
+    Lock<std::mutex> L(M); // no warning
+  }
+
+  void instantiate() {
+    TemplatedLock<std::lock_guard>();
+  }
+
+
+Options
+-------
+
+.. option:: WarnOnSingleLocks
+
+  When `true`, the check will warn on single ``std::lock_guard`` declarations.
+  Set this option to `false` if you want to get warnings only on multiple
+  ``std::lock_guard`` declarations that can be replaced with a single
+  ``std::scoped_lock``. Default is `true`.
+
+.. option:: WarnOnUsingAndTypedef
+
+  When `true`, the check will emit warnings if ``std::lock_guard`` is used
+  in ``using`` or ``typedef`` context. Default is `true`.
+
+  .. code-block:: c++
+
+    template <typename T>
+    using Lock = std::lock_guard<T>; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    
+    using LockMutex = std::lock_guard<std::mutex>; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    
+    typedef std::lock_guard<std::mutex> LockDef; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+
+    using std::lock_guard; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
\ No newline at end of file
diff --git a/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst b/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst
index aacf3675f9798..520c1b50733ae 100644
--- a/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst
+++ b/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst
@@ -1,3 +1,3 @@
 :orphan:
 :template: clangd_redirect.html
-:redirect_target: https://clangd.llvm.org/design.html
+:redirect_target: https://clangd.llvm.org/design/
diff --git a/clang-tools-extra/modularize/Modularize.cpp b/clang-tools-extra/modularize/Modularize.cpp
index 2a90c5e3f6782..376ad0c7875bf 100644
--- a/clang-tools-extra/modularize/Modularize.cpp
+++ b/clang-tools-extra/modularize/Modularize.cpp
@@ -395,7 +395,7 @@ struct Location {
     Column = SM.getColumnNumber(Decomposed.first, Decomposed.second);
   }
 
-  operator bool() const { return File != nullptr; }
+  explicit operator bool() const { return File != nullptr; }
 
   friend bool operator==(const Location &X, const Location &Y) {
     return X.File == Y.File && X.Line == Y.Line && X.Column == Y.Column;
@@ -459,7 +459,7 @@ struct HeaderEntry {
     return !(X == Y);
   }
   friend bool operator<(const HeaderEntry &X, const HeaderEntry &Y) {
-    return X.Loc < Y.Loc || (X.Loc == Y.Loc && X.Name < Y.Name);
+    return std::tie(X.Loc, X.Name) < std::tie(Y.Loc, Y.Name);
   }
   friend bool operator>(const HeaderEntry &X, const HeaderEntry &Y) {
     return Y < X;
diff --git a/clang-tools-extra/test/clang-doc/json/class-template.cpp b/clang-tools-extra/test/clang-doc/json/class-template.cpp
index e3ca086d1d9a4..fb9c4c2f21c2e 100644
--- a/clang-tools-extra/test/clang-doc/json/class-template.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class-template.cpp
@@ -17,7 +17,7 @@ template<typename T> struct MyClass {
 // CHECK-NEXT:    ], 
 // CHECK-NEXT:    "ReturnType": {
 // CHECK-NEXT:      "IsBuiltIn": false,
-// CHECK-NEXT:      "IsTemplate": false,
+// CHECK-NEXT:      "IsTemplate": true,
 // CHECK-NEXT:      "Name": "T",
 // CHECK-NEXT:      "QualName": "T"
 // CHECK-NEXT:      "USR": "0000000000000000000000000000000000000000"
diff --git a/clang-tools-extra/test/clang-doc/json/class.cpp b/clang-tools-extra/test/clang-doc/json/class.cpp
index 9ae1f14e2d2a6..ae47da75edccb 100644
--- a/clang-tools-extra/test/clang-doc/json/class.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class.cpp
@@ -23,6 +23,9 @@ struct MyClass {
   typedef int MyTypedef;
   
   class NestedClass;
+  
+  friend struct Foo;
+  template<typename T> friend void friendFunction(int);
 protected:
   int protectedMethod();
 
@@ -86,6 +89,44 @@ struct MyClass {
 // CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
 // CHECK-NEXT:      }
 // CHECK-NEXT:    ],
+// CHECK-NEXT:    "Friends": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "IsClass": false,
+// CHECK-NEXT:        "Params": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "",
+// CHECK-NEXT:            "Type": "int"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "Reference": {
+// CHECK-NEXT:          "Name": "friendFunction",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "friendFunction",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "ReturnType": {
+// CHECK-NEXT:          "IsBuiltIn": true,
+// CHECK-NEXT:          "IsTemplate": false,
+// CHECK-NEXT:          "Name": "void",
+// CHECK-NEXT:          "QualName": "void",
+// CHECK-NEXT:          "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "Template": {
+// CHECK-NEXT:          "Parameters": [
+// CHECK-NEXT:            "typename T"
+// CHECK-NEXT:          ]
+// CHECK-NEXT:        }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "IsClass": true,
+// CHECK-NEXT:        "Reference": {
+// CHECK-NEXT:          "Name": "Foo",
+// CHECK-NEXT:          "Path": "GlobalNamespace",
+// CHECK-NEXT:          "QualName": "Foo",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
 // COM:           FIXME: FullName is not emitted correctly.
 // CHECK-NEXT:    "FullName": "",
 // CHECK-NEXT:    "IsTypedef": false,
@@ -107,7 +148,7 @@ struct MyClass {
 // CHECK-NEXT:         "GlobalNamespace"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "ReturnType": {
-// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsBuiltIn": true,
 // CHECK-NEXT:         "IsTemplate": false,
 // CHECK-NEXT:         "Name": "int",
 // CHECK-NEXT:         "QualName": "int",
@@ -137,7 +178,7 @@ struct MyClass {
 // CHECK-NEXT:          }
 // CHECK-NEXT:        ],
 // CHECK-NEXT:        "ReturnType": {
-// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsBuiltIn": true,
 // CHECK-NEXT:          "IsTemplate": false,
 // CHECK-NEXT:          "Name": "int",
 // CHECK-NEXT:          "QualName": "int",
@@ -184,7 +225,7 @@ struct MyClass {
 // CHECK-NEXT:        "TypeDeclaration": "",
 // CHECK-NEXT:        "USR": "{{[0-9A-F]*}}",
 // CHECK-NEXT:        "Underlying": {
-// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsBuiltIn": true,
 // CHECK-NEXT:          "IsTemplate": false,
 // CHECK-NEXT:          "Name": "int",
 // CHECK-NEXT:          "QualName": "int",
diff --git a/clang-tools-extra/test/clang-doc/json/function-requires.cpp b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
index 99eb2bdb898f3..4e8432e088c4f 100644
--- a/clang-tools-extra/test/clang-doc/json/function-requires.cpp
+++ b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
@@ -23,7 +23,7 @@ template<Incrementable T> Incrementable auto incrementTwo(T t);
 // CHECK-NEXT:        }
 // CHECK-NEXT:      ],
 // CHECK-NEXT:      "ReturnType": {
-// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsBuiltIn": true,
 // CHECK-NEXT:        "IsTemplate": false,
 // CHECK-NEXT:        "Name": "void",
 // CHECK-NEXT:        "QualName": "void",
diff --git a/clang-tools-extra/test/clang-doc/json/method-template.cpp b/clang-tools-extra/test/clang-doc/json/method-template.cpp
index c51a2706d1c22..ea9110d6c2d1c 100644
--- a/clang-tools-extra/test/clang-doc/json/method-template.cpp
+++ b/clang-tools-extra/test/clang-doc/json/method-template.cpp
@@ -27,7 +27,7 @@ struct MyClass {
 // CHECK-NEXT:          ],
 // CHECK-NEXT:          "ReturnType": {
 // CHECK-NEXT:            "IsBuiltIn": false,
-// CHECK-NEXT:            "IsTemplate": false,
+// CHECK-NEXT:            "IsTemplate": true,
 // CHECK-NEXT:            "Name": "T",
 // CHECK-NEXT:            "QualName": "T",
 // CHECK-NEXT:            "USR": "0000000000000000000000000000000000000000"
diff --git a/clang-tools-extra/test/clang-doc/json/namespace.cpp b/clang-tools-extra/test/clang-doc/json/namespace.cpp
index f1e094f6cab17..6e4fc6938d856 100644
--- a/clang-tools-extra/test/clang-doc/json/namespace.cpp
+++ b/clang-tools-extra/test/clang-doc/json/namespace.cpp
@@ -56,7 +56,7 @@ typedef int MyTypedef;
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "ReturnType": {
-// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsBuiltIn": true,
 // CHECK-NEXT:         "IsTemplate": false,
 // CHECK-NEXT:         "Name": "void",
 // CHECK-NEXT:         "QualName": "void",
@@ -93,7 +93,7 @@ typedef int MyTypedef;
 // CHECK-NEXT:      "TypeDeclaration": "",
 // CHECK-NEXT:      "USR": "{{[0-9A-F]*}}",
 // CHECK-NEXT:      "Underlying": {
-// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsBuiltIn": true,
 // CHECK-NEXT:        "IsTemplate": false,
 // CHECK-NEXT:        "Name": "int",
 // CHECK-NEXT:        "QualName": "int",
@@ -111,8 +111,7 @@ typedef int MyTypedef;
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "Name": "Global",
 // CHECK-NEXT:       "Type": {
-// COM:                FIXME: IsBuiltIn emits as its default value
-// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsBuiltIn": true,
 // CHECK-NEXT:         "IsTemplate": false,
 // CHECK-NEXT:         "Name": "int",
 // CHECK-NEXT:         "QualName": "int",
diff --git a/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp b/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
new file mode 100644
index 0000000000000..9b176feb67a00
--- /dev/null
+++ b/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
@@ -0,0 +1,36 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/nested/index.json --check-prefix=NESTED
+// RUN: FileCheck %s < %t/nested/inner/index.json --check-prefix=INNER
+
+namespace nested {
+  int Global;
+  namespace inner {
+    int InnerGlobal;
+  } // namespace inner
+} // namespace nested
+
+// NESTED:       "Variables": [
+// NESTED-NEXT:    {
+// NESTED-NEXT:      "IsStatic": false,
+// NESTED-NEXT:      "Location": {
+// NESTED-NEXT:        "Filename": "{{.*}}nested-namespace.cpp",
+// NESTED-NEXT:        "LineNumber": 7
+// NESTED-NEXT:      },
+// NESTED-NEXT:      "Name": "Global",
+// NESTED-NEXT:      "Namespace": [
+// NESTED-NEXT:        "nested"
+// NESTED-NEXT:      ],
+
+// INNER:       "Variables": [
+// INNER-NEXT:    {
+// INNER-NEXT:      "IsStatic": false,
+// INNER-NEXT:      "Location": {
+// INNER-NEXT:        "Filename": "{{.*}}nested-namespace.cpp",
+// INNER-NEXT:        "LineNumber": 9
+// INNER-NEXT:      },
+// INNER-NEXT:      "Name": "InnerGlobal",
+// INNER-NEXT:      "Namespace": [
+// INNER-NEXT:        "inner",
+// INNER-NEXT:        "nested"
+// INNER-NEXT:      ],
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex
new file mode 100644
index 0000000000000..a0030a2b9da0a
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex
@@ -0,0 +1,33 @@
+#ifndef _MUTEX_
+#define _MUTEX_
+
+namespace std {
+
+struct mutex {
+  void lock() {}
+  void unlock() {}
+};
+
+template<class Lockable1, class Lockable2, class... LockableN >
+void lock(Lockable1& lock1, Lockable2& lock2, LockableN&... lockn );
+
+struct adopt_lock_t { };
+std::adopt_lock_t adopt_lock {};
+
+template <typename Mutex>
+struct lock_guard {
+  lock_guard(Mutex &m) { }
+  lock_guard(Mutex &m, std::adopt_lock_t t) {}
+  lock_guard(const lock_guard&) = delete;
+};
+
+template <typename... MutexTypes>
+struct scoped_lock {
+  scoped_lock(MutexTypes&... m) {}
+  scoped_lock(std::adopt_lock_t t, MutexTypes&... m) {}
+  scoped_lock(const scoped_lock&) = delete;
+};
+
+} // namespace std
+
+#endif // _MUTEX_
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp
new file mode 100644
index 0000000000000..7ccac1a10015f
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp
@@ -0,0 +1,42 @@
+// RUN: %check_clang_tidy -std=c++14 %s cppcoreguidelines-missing-std-forward %t -- \
+// RUN: -config="{CheckOptions: {cppcoreguidelines-missing-std-forward.ForwardFunction: custom_forward}}" -- -fno-delayed-template-parsing
+
+// NOLINTBEGIN
+namespace std {
+
+template <typename T> struct remove_reference      { using type = T; };
+template <typename T> struct remove_reference<T&>  { using type = T; };
+template <typename T> struct remove_reference<T&&> { using type = T; };
+
+template <typename T> using remove_reference_t = typename remove_reference<T>::type;
+
+template <typename T> constexpr T &&forward(remove_reference_t<T> &t) noexcept;
+template <typename T> constexpr T &&forward(remove_reference_t<T> &&t) noexcept;
+template <typename T> constexpr remove_reference_t<T> &&move(T &&x);
+
+} // namespace std
+// NOLINTEND
+
+template<class T>
+constexpr decltype(auto) custom_forward(std::remove_reference_t<T>& tmp) noexcept
+{
+  return static_cast<T&&>(tmp);
+}
+
+template<class T>
+constexpr decltype(auto) custom_forward(std::remove_reference_t<T>&& tmp) noexcept
+{
+  return static_cast<T&&>(tmp);
+}
+
+template<class T>
+void forward_with_std(T&& t) {
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: forwarding reference parameter 't' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward]
+
+  T other{std::forward<T>(t)};
+}
+
+template<class T>
+void move_with_custom(T&& t) {
+  T other{custom_forward<T>(t)};
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp
index faab2a1ccf0e1..b3d3fab9c5409 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp
@@ -1,11 +1,14 @@
 // RUN: %check_clang_tidy -std=c++14-or-later %s cppcoreguidelines-pro-bounds-pointer-arithmetic %t
 
 // Fix PR36489 and detect auto-deduced value correctly.
+typedef char* charPtr;
+
 char *getPtr();
 auto getPtrAuto() { return getPtr(); }
 decltype(getPtr()) getPtrDeclType();
 decltype(auto) getPtrDeclTypeAuto() { return getPtr(); }
 auto getPtrWithTrailingReturnType() -> char *;
+charPtr getCharPtr() { return getPtr(); }
 
 void auto_deduction_binary() {
   auto p1 = getPtr() + 1;
@@ -28,6 +31,10 @@ void auto_deduction_binary() {
   // CHECK-MESSAGES: :[[@LINE-1]]:31: warning: do not use pointer arithmetic
   auto *p9 = getPtrDeclTypeAuto() + 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:35: warning: do not use pointer arithmetic
+  auto p10 = getCharPtr() + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not use pointer 
+  auto* p11 = getCharPtr() + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not use pointer arithmetic
 }
 
 void auto_deduction_subscript() {
@@ -50,4 +57,6 @@ void auto_deduction_subscript() {
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: do not use pointer arithmetic
   auto p8 = getPtrDeclTypeAuto()[9];
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: do not use pointer arithmetic
+  auto p9 = getCharPtr()[10];
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: do not use pointer arithmetic
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
index 7cbc6ddf96ab6..aed6080471e1f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
@@ -4,10 +4,13 @@ enum E {
   ENUM_LITERAL = 1
 };
 
+typedef int* IntPtr;
+
 int i = 4;
 int j = 1;
 int *p = 0;
 int *q = 0;
+IntPtr ip = 0;
 
 void fail() {
   q = p + 4;
@@ -50,6 +53,32 @@ void fail() {
 
   i = p[1];
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
+
+  p = ip + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: do not use pointer arithmetic
+  ip++;
+  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use pointer arithmetic
+  i = ip[1];
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
+}
+
+template <typename T>
+void template_fail() {
+  T* p;
+  T* q;
+
+  p = q + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: do not use pointer arithmetic
+  q = p - 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: do not use pointer arithmetic
+  p++;
+  // CHECK-MESSAGES: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
+  i = p[1];
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
+}
+
+void instantiate() {
+  template_fail<int>();
 }
 
 struct S {
@@ -87,3 +116,25 @@ void okay() {
 
   for(int ii : a) ; // OK, pointer arithmetic generated by compiler
 }
+
+namespace gh126424 {
+
+namespace std {
+template <typename, typename>
+class pair {};
+
+template <typename Key, typename Value>
+class map {
+  public:
+   using value_type = pair<Key, Value>;
+   value_type& operator[](const Key& key);
+   value_type& operator[](Key&& key);
+ };
+}
+
+template <typename R>
+int f(std::map<R*, int>& map, R* r) {
+  return map[r]; // OK
+}
+
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp
new file mode 100644
index 0000000000000..4ed29824496fc
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp
@@ -0,0 +1,47 @@
+// RUN: %check_clang_tidy -std=c++11 %s cppcoreguidelines-rvalue-reference-param-not-moved %t -- \
+// RUN: -config="{CheckOptions: {cppcoreguidelines-rvalue-reference-param-not-moved.AllowPartialMove: true, cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams: true, cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreNonDeducedTemplateTypes: true, cppcoreguidelines-rvalue-reference-param-not-moved.MoveFunction: custom_move}}" -- -fno-delayed-template-parsing
+
+// NOLINTBEGIN
+namespace std {
+template <typename>
+struct remove_reference;
+
+template <typename _Tp> struct remove_reference { typedef _Tp type; };
+template <typename _Tp> struct remove_reference<_Tp&> { typedef _Tp type; };
+template <typename _Tp> struct remove_reference<_Tp&&> { typedef _Tp type; };
+
+template <typename _Tp>
+constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) noexcept;
+
+template <typename _Tp>
+constexpr _Tp &&
+forward(typename remove_reference<_Tp>::type &__t) noexcept;
+
+}
+// NOLINTEND
+
+
+struct Obj {
+  Obj();
+  Obj(const Obj&);
+  Obj& operator=(const Obj&);
+  Obj(Obj&&);
+  Obj& operator=(Obj&&);
+  void member() const;
+};
+
+template<class T>
+constexpr typename std::remove_reference<T>::type&& custom_move(T&& x) noexcept
+{
+    return static_cast<typename std::remove_reference<T>::type&&>(x);
+}
+
+void move_with_std(Obj&& o) {
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: rvalue reference parameter 'o' is never moved from inside the function body [cppcoreguidelines-rvalue-reference-param-not-moved]
+  Obj other{std::move(o)};
+}
+
+void move_with_custom(Obj&& o) {
+  Obj other{custom_move(o)};
+}
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp
index cdfed7edb431d..fb7eeed7e715f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp
@@ -74,6 +74,47 @@ template <typename t1, typename tl>
 // CHECK-MESSAGES: :[[#@LINE-2]]:20: note: other declaration found here
 void f9();
 
+namespace different_contexts {
+  // No warning for names in unrelated contexts.
+  template <typename u1> void different_templates_1();
+  template <typename ul> void different_templates_2();
+  namespace inner {
+    int ul;
+  }
+}
+
+namespace same_template {
+  template <typename u1, typename ul> using T = int;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:35: warning: 'ul' is confusable with 'u1' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-2]]:22: note: other declaration found here
+
+  template <typename v1, typename vl> int n;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:35: warning: 'vl' is confusable with 'v1' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-2]]:22: note: other declaration found here
+
+  template <typename w1> int wl;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:22: warning: 'w1' is confusable with 'wl' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-2]]:30: note: other declaration found here
+
+  int xl;
+  template <typename x1> int x;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:22: warning: 'x1' is confusable with 'xl' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-3]]:7: note: other declaration found here
+}
+
+namespace f10 {
+int il;
+namespace inner {
+  int i1;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:7: warning: 'i1' is confusable with 'il' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-4]]:5: note: other declaration found here
+  int j1;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:7: warning: 'j1' is confusable with 'jl' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE+2]]:5: note: other declaration found here
+}
+int jl;
+}
+
 struct Base0 {
   virtual void mO0();
 
@@ -103,3 +144,21 @@ struct Derived1 : Base1 {
 
   long mI1(); // no warning: mII is private
 };
+
+struct Base2 {
+  long nO0;
+
+private:
+  long nII;
+};
+
+struct Mid2 : Base0, Base1, Base2 {
+};
+
+struct Derived2 : Mid2 {
+  long nOO;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:8: warning: 'nOO' is confusable with 'nO0' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-12]]:8: note: other declaration found here
+
+  long nI1(); // no warning: mII is private
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
index 74a22a7c083f4..28b53ae4af63d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
@@ -163,3 +163,16 @@ struct TemplateTypeAlias {
   Alias3<TypeAlias::Alias> &operator=(double) { return *this; }
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: operator=() should return 'TemplateTypeAlias&' [misc-unconventional-assign-operator]
 };
+
+namespace gh143237 {
+template<typename T>
+struct TemplateAssignment {
+  explicit TemplateAssignment(int) {
+  }
+
+  TemplateAssignment& operator=(int n) {
+    // No warning
+    return *this = TemplateAssignment(n);
+  }
+};
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp
new file mode 100644
index 0000000000000..77df845756530
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp
@@ -0,0 +1,31 @@
+// RUN: %check_clang_tidy -std=c++17-or-later %s modernize-use-scoped-lock %t -- \
+// RUN:   -config="{CheckOptions: {modernize-use-scoped-lock.WarnOnUsingAndTypedef: false}}" \
+// RUN:   -- -isystem %clang_tidy_headers -fno-delayed-template-parsing
+
+#include <mutex>
+
+template <typename T>
+using Lock = std::lock_guard<T>;
+
+using LockM = std::lock_guard<std::mutex>;
+
+typedef std::lock_guard<std::mutex> LockDef;
+
+void PositiveUsingDecl() {
+  using std::lock_guard;
+
+  using LockMFun = std::lock_guard<std::mutex>;
+  
+  typedef std::lock_guard<std::mutex> LockDefFun;
+}
+
+template <typename T>
+void PositiveUsingDeclTemplate() {
+  using std::lock_guard;
+
+  using LockFunT = std::lock_guard<T>;
+
+  using LockMFunT = std::lock_guard<std::mutex>;
+
+  typedef std::lock_guard<std::mutex> LockDefFunT;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp
new file mode 100644
index 0000000000000..7b30cdd75c801
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp
@@ -0,0 +1,102 @@
+// RUN: %check_clang_tidy -std=c++17-or-later %s modernize-use-scoped-lock %t -- \
+// RUN:   -config="{CheckOptions: {modernize-use-scoped-lock.WarnOnSingleLocks: false}}" \
+// RUN:   -- -isystem %clang_tidy_headers -fno-delayed-template-parsing
+
+#include <mutex>
+
+void Positive() {
+  std::mutex m;
+
+  {
+    std::lock_guard<std::mutex> l1(m);
+    std::lock_guard<std::mutex> l2(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m), l2(m), l3(m);
+    std::lock_guard<std::mutex> l4(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:40: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:47: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:33: note: additional 'std::lock_guard' declared here
+  }
+  
+  { 
+    std::lock(m, m);
+    std::lock_guard<std::mutex> l1(m, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  } 
+}
+
+void Negative() {
+  std::mutex m;
+  {
+    std::lock_guard<std::mutex> l(m);
+  }
+
+  {
+    std::lock_guard<std::mutex> l(m, std::adopt_lock);
+  }
+
+  { 
+    std::lock_guard<std::mutex> l3(m);
+    int a = 0;
+    std::lock_guard<std::mutex> l4(m, std::adopt_lock);
+  }
+}
+
+void PositiveInsideArg(std::mutex &m1, std::mutex &m2, std::mutex &m3) {
+  std::lock_guard<std::mutex> l1(m1);
+  std::lock_guard<std::mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here
+}
+
+
+void NegativeInsideArg(std::mutex &m1, std::mutex &m2, std::mutex &m3) {
+  std::lock_guard<std::mutex> l3(m3);
+}
+
+template <typename T>
+void PositiveTemplated() {
+  std::mutex m1, m2;
+  
+  std::lock_guard<std::mutex> l1(m1);
+  std::lock_guard<std::mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here  
+}
+
+template <typename T>
+void NegativeTemplated() {
+  std::mutex m1, m2, m3;
+  std::lock_guard<std::mutex> l(m1);
+}
+
+template <typename Mutex>
+void PositiveTemplatedMutex() {
+  Mutex m1, m2;
+
+  std::lock_guard<Mutex> l1(m1);
+  std::lock_guard<Mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:26: note: additional 'std::lock_guard' declared here
+}
+
+template <typename Mutex>
+void NegativeTemplatedMutex() {
+  Mutex m1;
+  std::lock_guard<Mutex> l(m1);
+}
+
+struct NegativeClass {
+  void Negative() {
+    std::lock_guard<std::mutex> l(m1);
+  }
+  
+  std::mutex m1;
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp
new file mode 100644
index 0000000000000..45eabdbece68c
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp
@@ -0,0 +1,471 @@
+// RUN: %check_clang_tidy -std=c++17-or-later %s modernize-use-scoped-lock %t -- -- -isystem %clang_tidy_headers -fno-delayed-template-parsing
+
+#include <mutex>
+
+void Positive() {
+  std::mutex m;
+  {
+    std::lock_guard<std::mutex> l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    std::lock_guard<std::mutex> l(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(std::adopt_lock, m);
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m);
+    std::lock_guard<std::mutex> l2(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m), l2(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:40: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m), l2(m), l3(m);
+    std::lock_guard<std::mutex> l4(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:40: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:47: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:33: note: additional 'std::lock_guard' declared here
+  }
+  
+  { 
+    std::lock(m, m);
+    std::lock_guard<std::mutex> l1(m, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<std::mutex> l3(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l3(m);
+    int b = 0;
+    std::lock_guard<std::mutex> l4(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l4(std::adopt_lock, m);
+  } 
+}
+
+
+std::mutex p_m1;
+void PositiveShortFunction() {
+  std::lock_guard<std::mutex> l(p_m1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l(p_m1);
+}
+
+
+void PositiveNested() {
+  std::mutex m1;
+  if (true) {
+    std::lock_guard<std::mutex> l(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m1);
+    {
+      std::lock_guard<std::mutex> l2(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l2(m1);
+      {
+        std::lock_guard<std::mutex> l3(m1);
+        std::lock_guard<std::mutex> l4(m1);
+        // CHECK-MESSAGES: :[[@LINE-2]]:9: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+        // CHECK-MESSAGES: :[[@LINE-2]]:37: note: additional 'std::lock_guard' declared here
+      }
+      {
+        std::lock_guard<std::mutex> l2(m1);
+        // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+        // CHECK-FIXES: std::scoped_lock l2(m1);
+      }
+    }
+  }
+  std::lock_guard<std::mutex> l(m1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l(m1);
+}
+
+
+void PositiveInsideArg(std::mutex &m1, std::mutex &m2, std::mutex &m3) {
+  std::lock_guard<std::mutex> l1(m1);
+  std::lock_guard<std::mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here
+  int a = 0;
+  std::lock_guard<std::mutex> l3(m3);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l3(m3);
+}
+
+
+void PositiveInsideConditional() {
+  std::mutex m1;
+  if (true) {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  } else {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  }
+
+  while (true) {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  }
+
+  for (int i = 0; i < 10; ++i) {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  }
+}
+
+void PositiveLambda() {
+  std::mutex m;
+  std::lock_guard<std::mutex> l1(m);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l1(m);
+  auto lambda1 = [&]() {
+    std::lock_guard<std::mutex> l1(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m);
+  };
+
+  std::lock_guard<std::mutex> l3(m);
+  std::lock_guard<std::mutex> l4(m);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here
+  auto lamda2 = [&]() {
+    std::lock_guard<std::mutex> l3(m);
+    std::lock_guard<std::mutex> l4(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  };
+
+  auto lamda3 = [&]() {
+    std::lock(m, m);
+    std::lock_guard<std::mutex> l1(m, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<std::mutex> l3(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l3(m);
+    int b = 0;
+    std::lock_guard<std::mutex> l4(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l4(std::adopt_lock, m);
+  };
+
+  auto lamda4 = [&]() {
+    std::lock_guard<std::mutex> l1(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m);
+    int a = 0;
+    std::lock_guard<std::mutex> l2(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l2(m);
+  };
+}
+
+template <typename T>
+void PositiveTemplated() {
+  std::mutex m1, m2, m3;
+  {
+    std::lock_guard<std::mutex> l(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m1);
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m1);
+    std::lock_guard<std::mutex> l2(m2);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock(m1, m2);
+    std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<std::mutex> l3(m3);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l3(m3);
+  }
+}
+
+
+template <typename Mutex>
+void PositiveTemplatedMutex() {
+  Mutex m1, m2, m3;
+  {
+    std::lock_guard<Mutex> l(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+
+  {
+    std::lock_guard<Mutex> l1(m1);
+    std::lock_guard<Mutex> l2(m2);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:28: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock(m1, m2);
+    std::lock_guard<Mutex> l1(m1, std::adopt_lock);
+    std::lock_guard<Mutex> l2(m2, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:28: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<Mutex> l3(m3);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+}
+
+
+template <template <typename> typename Lock>
+void NegativeTemplate() {
+  std::mutex m1, m2;
+  {
+    Lock<std::mutex> l(m1);
+  }
+
+  {
+    Lock<std::mutex> l1(m1);
+    Lock<std::mutex> l2(m2);
+  }
+}
+
+void instantiate() {
+  NegativeTemplate<std::lock_guard>();
+}
+
+
+struct PositiveClass {
+  void Positive() {
+    {
+      std::lock_guard<std::mutex> l(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l(m1);
+    }
+
+    {
+      std::lock_guard<std::mutex> l1(m1);
+      std::lock_guard<std::mutex> l2(m2);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+    }
+
+    {
+      std::lock(m1, m2);  
+      std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+      std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+      int a = 0;
+      std::lock_guard<std::mutex> l3(m3);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l3(m3);
+    }
+  }
+  
+  std::mutex m1;
+  std::mutex m2;
+  std::mutex m3;
+};
+
+
+template <typename T>
+struct PositiveTemplatedClass {
+  void Positive() {
+    {
+      std::lock_guard<std::mutex> l(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l(m1);
+    }
+
+    {
+      std::lock(m1, m2);  
+      std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+      std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+      int a = 0;
+      std::lock_guard<std::mutex> l3(m3);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l3(m3);
+    }
+  }
+
+  template <typename... Ts>
+  void TemplatedPositive() {
+    {
+      std::lock_guard<std::mutex> l(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l(m1);
+    }
+
+    {
+      std::lock(m1, m2);  
+      std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+      std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+      int a = 0;
+      std::lock_guard<std::mutex> l3(m3);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l3(m3);
+    }
+  }
+  
+  std::mutex m1;
+  std::mutex m2;
+  std::mutex m3;
+};
+
+
+template <typename T>
+using Lock = std::lock_guard<T>;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+// CHECK-FIXES: using Lock = std::scoped_lock<T>;
+
+using LockM = std::lock_guard<std::mutex>;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+// CHECK-FIXES: using LockM = std::scoped_lock<std::mutex>;
+
+typedef std::lock_guard<std::mutex> LockDef;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+// CHECK-FIXES: typedef std::scoped_lock<std::mutex> LockDef;
+
+
+void PositiveUsingDecl() {
+  using std::lock_guard;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using std::scoped_lock;
+
+  using LockMFun = std::lock_guard<std::mutex>;
+  // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using LockMFun = std::scoped_lock<std::mutex>;
+
+  typedef std::lock_guard<std::mutex> LockDefFun;
+  // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: typedef std::scoped_lock<std::mutex> LockDefFun;
+}
+
+template <typename T>
+void PositiveUsingDeclTemplate() {
+  using std::lock_guard;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using std::scoped_lock;
+
+  std::mutex m;
+  lock_guard<std::mutex> l(m);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l(m);
+
+  using LockFunT = std::lock_guard<T>;
+  // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using LockFunT = std::scoped_lock<T>;
+
+  using LockMFunT = std::lock_guard<std::mutex>;
+  // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using LockMFunT = std::scoped_lock<std::mutex>;
+
+  typedef std::lock_guard<std::mutex> LockDefFunT;
+  // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: typedef std::scoped_lock<std::mutex> LockDefFunT;
+}
+
+void PositiveInUsingTypedefs() {
+  std::mutex m;
+
+  {
+    Lock<std::mutex> l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    LockM l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    LockDef l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    std::lock(m, m);
+    Lock<std::mutex> l1(m, std::adopt_lock);
+    LockM l2(m, std::adopt_lock);
+    LockDef l3(m), l4(m);
+    // CHECK-MESSAGES: :[[@LINE-3]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:11: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-3]]:13: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:20: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    LockDef l5(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l5(m);
+  }
+}
+
+template <typename Mutex>
+void PositiveInUsingTypedefsTemplated() {
+  Mutex m;
+
+  {
+    Lock<Mutex> l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+
+  {
+    std::lock(m, m);
+    Lock<Mutex> l1(m, std::adopt_lock);
+    LockM l2(m, std::adopt_lock);
+    LockDef l3(m), l4(m);
+    // CHECK-MESSAGES: :[[@LINE-3]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:11: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-3]]:13: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:20: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    LockDef l5(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+}
+
+// Non-STD lock_guard.
+template <typename Mutex>
+struct lock_guard {
+  lock_guard(Mutex &m) { }
+  lock_guard(const lock_guard& ) = delete;
+};
+
+void NegativeNonStdLockGuard() {
+  std::mutex m;
+  {
+    lock_guard<std::mutex> l(m);
+  }
+
+  {
+    lock_guard<std::mutex> l1(m);
+    lock_guard<std::mutex> l2(m);
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp
index 14f9e88f7e721..882ce640b13c1 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp
@@ -149,3 +149,13 @@ class A
   // CHECK-FIXES-STRICT: static float test4;
 };
 }
+
+namespace ns {
+class B
+{
+public:
+  ~B();
+};
+
+inline B::~B() = default;
+}
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp
index 7efa7d070f69f..30bcdc3f87743 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp
@@ -12,8 +12,9 @@
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER %s
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER %s
 
-// Check that `-header-filter` operates on the same file paths as paths in
-// diagnostics printed by ClangTidy.
+// `-header-filter` operates on the actual file path that the user provided in
+// the #include directive; however, Clang's path name simplification causes the
+// path to be printed in canonicalised form here.
 #include "dir1/dir2/../header_alias.h"
-// CHECK_HEADER_ALIAS: dir1/dir2/../header_alias.h:1:11: warning: single-argument constructors
+// CHECK_HEADER_ALIAS: dir1/header.h:1:11: warning: single-argument constructors
 // CHECK_HEADER-NOT: warning:
diff --git a/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp b/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
index 323431c4fc9e0..d88b0c91bb9a9 100644
--- a/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
@@ -41,6 +41,8 @@ static std::string writeInfo(Info *I) {
     return writeInfo(*static_cast<ConceptInfo *>(I));
   case InfoType::IT_variable:
     return writeInfo(*static_cast<VarInfo *>(I));
+  case InfoType::IT_friend:
+    return writeInfo(*static_cast<FriendInfo *>(I));
   case InfoType::IT_default:
     return "";
   }
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 94607a8e8473c..1bb73599970c1 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -150,6 +150,12 @@ if(CLANG_BUILT_STANDALONE)
     endif()
 
     umbrella_lit_testsuite_begin(check-all)
+
+    # If LLVM was built with Z3, check if we still have the headers around.
+    # They are used by a few tests.
+    if (LLVM_WITH_Z3)
+      find_package(Z3 4.8.9)
+    endif()
   endif() # LLVM_INCLUDE_TESTS
 endif() # standalone
 
diff --git a/clang/docs/DebuggingCoroutines.rst b/clang/docs/DebuggingCoroutines.rst
index 80df321340724..8702152747c65 100644
--- a/clang/docs/DebuggingCoroutines.rst
+++ b/clang/docs/DebuggingCoroutines.rst
@@ -8,288 +8,735 @@ Debugging C++ Coroutines
 Introduction
 ============
 
-For performance and other architectural reasons, the C++ Coroutines feature in
-the Clang compiler is implemented in two parts of the compiler.  Semantic
-analysis is performed in Clang, and Coroutine construction and optimization
-takes place in the LLVM middle-end.
+Coroutines in C++ were introduced in C++20, and the user experience for
+debugging them can still be challenging. This document guides you how to most
+efficiently debug coroutines and how to navigate existing shortcomings in
+debuggers and compilers.
+
+Coroutines are generally used either as generators or for asynchronous
+programming. In this document, we will discuss both use cases. Even if you are
+using coroutines for asynchronous programming, you should still read the
+generators section, as it will introduce foundational debugging techniques also
+applicable to the debugging of asynchronous programs.
+
+Both compilers (clang, gcc, ...) and debuggers (lldb, gdb, ...) are
+still improving their support for coroutines. As such, we recommend using the
+latest available version of your toolchain.
+
+This document focuses on clang and lldb. The screenshots show
+`lldb-dap <https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.lldb-dap>`_
+in combination with VS Code. The same techniques can also be used in other
+IDEs.
+
+Debugging clang-compiled binaries with gdb is possible, but requires more
+scripting. This guide comes with a basic GDB script for coroutine debugging.
+
+This guide will first showcase the more polished, bleeding-edge experience, but
+will also show you how to debug coroutines with older toolchains. In general,
+the older your toolchain, the deeper you will have to dive into the
+implementation details of coroutines (such as their ABI). The further down in
+this document you go, the more low-level, technical the content will become. If
+you are on an up-to-date toolchain, you will hopefully be able to stop reading
+earlier.
+
+Debugging generators
+====================
+
+One of the two major use cases for coroutines in C++ are generators, i.e.,
+functions which can produce values via ``co_yield``. Values are produced
+lazily, on-demand. For this purpose, every time a new value is requested, the
+coroutine gets resumed. As soon as it reaches a ``co_yield`` and thereby
+returns the requested value, the coroutine is suspended again.
+
+This logic is encapsulated in a ``generator`` type similar to this one:
 
-However, this design forces us to generate insufficient debugging information.
-Typically, the compiler generates debug information in the Clang frontend, as
-debug information is highly language specific. However, this is not possible
-for Coroutine frames because the frames are constructed in the LLVM middle-end.
-
-To mitigate this problem, the LLVM middle end attempts to generate some debug
-information, which is unfortunately incomplete, since much of the language
-specific information is missing in the middle end.
+.. code-block:: c++
 
-This document describes how to use this debug information to better debug
-coroutines.
+  // generator.hpp
+  #include <coroutine>
 
-Terminology
-===========
+  // `generator` is a stripped down, minimal generator type.
+  template<typename T>
+  struct generator {
+    struct promise_type {
+      T current_value{};
 
-Due to the recent nature of C++20 Coroutines, the terminology used to describe
-the concepts of Coroutines is not settled.  This section defines a common,
-understandable terminology to be used consistently throughout this document.
+      auto get_return_object() {
+        return std::coroutine_handle<promise_type>::from_promise(*this);
+      }
+      auto initial_suspend() { return std::suspend_always(); }
+      auto final_suspend() noexcept { return std::suspend_always(); }
+      auto return_void() { return std::suspend_always(); }
+      void unhandled_exception() { __builtin_unreachable(); }
+      auto yield_value(T v) {
+        current_value = v;
+        return std::suspend_always();
+      }
+    };
 
-coroutine type
---------------
+    generator(std::coroutine_handle<promise_type> h) : hdl(h) { hdl.resume(); }
+    ~generator() { hdl.destroy(); }
 
-A `coroutine function` is any function that contains any of the Coroutine
-Keywords `co_await`, `co_yield`, or `co_return`.  A `coroutine type` is a
-possible return type of one of these `coroutine functions`.  `Task` and
-`Generator` are commonly referred to coroutine types.
+    generator<T>& operator++() { hdl.resume(); return *this; } // resume the coroutine
+    T operator*() const { return hdl.promise().current_value; }
 
-coroutine
----------
+    private:
+    std::coroutine_handle<promise_type> hdl;
+  };
 
-By technical definition, a `coroutine` is a suspendable function. However,
-programmers typically use `coroutine` to refer to an individual instance.
-For example:
+We can then use this ``generator`` class to print the Fibonacci sequence:
 
 .. code-block:: c++
 
-  std::vector<Task> Coros; // Task is a coroutine type.
-  for (int i = 0; i < 3; i++)
-    Coros.push_back(CoroTask()); // CoroTask is a coroutine function, which
-                                 // would return a coroutine type 'Task'.
+  #include "generator.hpp"
+  #include <iostream>
 
-In practice, we typically say "`Coros` contains 3 coroutines" in the above
-example, though this is not strictly correct.  More technically, this should
-say "`Coros` contains 3 coroutine instances" or "Coros contains 3 coroutine
-objects."
+  generator<int> fibonacci() {
+    co_yield 0;
+    int prev = 0;
+    co_yield 1;
+    int current = 1;
+    while (true) {
+      int next = current + prev;
+      co_yield next;
+      prev = current;
+      current = next;
+    }
+  }
 
-In this document, we follow the common practice of using `coroutine` to refer
-to an individual `coroutine instance`, since the terms `coroutine instance` and
-`coroutine object` aren't sufficiently defined in this case.
+  template<typename T>
+  void print10Elements(generator<T>& gen) {
+    for (unsigned i = 0; i < 10; ++i) {
+      std::cerr << *gen << "\n";
+      ++gen;
+    }
+  }
 
-coroutine frame
----------------
+  int main() {
+    std::cerr << "Fibonacci sequence - here we go\n";
+    generator<int> fib = fibonacci();
+    for (unsigned i = 0; i < 5; ++i) {
+      ++fib;
+    }
+    print10Elements(fib);
+  }
 
-The C++ Standard uses `coroutine state` to describe the allocated storage. In
-the compiler, we use `coroutine frame` to describe the generated data structure
-that contains the necessary information.
+To compile this code, use ``clang++ --std=c++23 generator-example.cpp -g``.
 
-The structure of coroutine frames
-=================================
+Breakpoints inside the generators
+---------------------------------
 
-The structure of coroutine frames is defined as:
+We can set breakpoints inside coroutines just as we set them in regular
+functions. For VS Code, that means clicking next the line number in the editor.
+In the ``lldb`` CLI or in ``gdb``, you can use ``b`` to set a breakpoint.
 
-.. code-block:: c++
+Inspecting variables in a coroutine
+-----------------------------------
 
-  struct {
-    void (*__r)(); // function pointer to the `resume` function
-    void (*__d)(); // function pointer to the `destroy` function
-    promise_type; // the corresponding `promise_type`
-    ... // Any other needed information
-  }
+If you hit a breakpoint inside the ``fibonacci`` function, you should be able
+to inspect all local variables (``prev``, ``current``, ``next``) just like in
+a regular function.
 
-In the debugger, the function's name is obtainable from the address of the
-function. And the name of `resume` function is equal to the name of the
-coroutine function. So the name of the coroutine is obtainable once the
-address of the coroutine is known.
+.. image:: ./coro-generator-variables.png
 
-Print promise_type
-==================
+Note the two additional variables ``__promise`` and ``__coro_frame``. Those
+show the internal state of the coroutine. They are not relevant for our
+generator example, but will be relevant for asynchronous programming described
+in the next section.
 
-Every coroutine has a `promise_type`, which defines the behavior
-for the corresponding coroutine. In other words, if two coroutines have the
-same `promise_type`, they should behave in the same way.
-To print a `promise_type` in a debugger when stopped at a breakpoint inside a
-coroutine, printing the `promise_type` can be done by:
+Stepping out of a coroutine
+---------------------------
 
-.. parsed-literal::
+When single-stepping, you will notice that the debugger will leave the
+``fibonacci`` function as soon as you hit a ``co_yield`` statement. You might
+find yourself inside some standard library code. After stepping out of the
+library code, you will be back in the ``main`` function.
 
-  print __promise
+Stepping into a coroutine
+-------------------------
 
-It is also possible to print the `promise_type` of a coroutine from the address
-of the coroutine frame. For example, if the address of a coroutine frame is
-0x416eb0, and the type of the `promise_type` is `task::promise_type`, printing
-the `promise_type` can be done by:
+If you stop at ``++fib`` and try to step into the generator, you will first
+find yourself inside ``operator++``. Stepping into the ``handle.resume()`` will
+not work by default.
 
-.. parsed-literal::
+This is because lldb does not step into functions from the standard library by
+default. To make this work, you first need to run ``settings set
+target.process.thread.step-avoid-regexp ""``. You can do so from the "Debug
+Console" towards the bottom of the screen. With that setting change, you can
+step through ``coroutine_handle::resume`` and into your generator.
 
-  print (task::promise_type)*(0x416eb0+0x10)
+You might find yourself at the top of the coroutine at first, instead of at
+your previous suspension point. In that case, single-step and you will arrive
+at the previously suspended ``co_yield`` statement.
 
-This is possible because the `promise_type` is guaranteed by the ABI to be at a
-16 bit offset from the coroutine frame.
 
-Note that there is also an ABI independent method:
+Inspecting a suspended coroutine
+--------------------------------
 
-.. parsed-literal::
+The ``print10Elements`` function receives an opaque ``generator`` type. Let's
+assume we are suspended at the ``++gen;`` line, and want to inspect the
+generator and its internal state.
 
-  print std::coroutine_handle<task::promise_type>::from_address((void*)0x416eb0).promise()
+To do so, we can simply look into the ``gen.hdl`` variable. LLDB comes with a
+pretty printer for ``std::coroutine_handle`` which will show us the internal
+state of the coroutine. For GDB, you will have to use the ``show-coro-frame``
+command provided by the :ref:`gdb-script`.
 
-The functions `from_address(void*)` and `promise()` are often small enough to
-be removed during optimization, so this method may not be possible.
+.. image:: ./coro-generator-suspended.png
 
-Print coroutine frames
-======================
+We can see two function pointers ``resume`` and ``destroy``. These pointers
+point to the resume / destroy functions. By inspecting those function pointers,
+we can see that our ``generator`` is actually backed by our ``fibonacci``
+coroutine. When using VS Code + lldb-dap, you can Cmd+Click on the function
+address (``0x555...`` in the screenshot) to directly jump to the function
+definition backing your coroutine handle.
 
-LLVM generates the debug information for the coroutine frame in the LLVM middle
-end, which permits printing of the coroutine frame in the debugger. Much like
-the `promise_type`, when stopped at a breakpoint inside a coroutine we can
-print the coroutine frame by:
+Next, we see the ``promise``. In our case, this reveals the current value of
+our generator.
 
-.. parsed-literal::
+The ``coro_frame`` member represents the internal state of the coroutine. It
+contains our internal coroutine state ``prev``, ``current``, ``next``.
+Furthermore, it contains many internal, compiler-specific members, which are
+named based on their type. These represent temporary values which the compiler
+decided to spill across suspension points, but which were not declared in our
+original source code and hence have no proper user-provided name.
 
-  print __coro_frame
+Tracking the exact suspension point
+-----------------------------------
 
+Among the compiler-generated members, the ``__coro_index`` is particularly
+important. This member identifies the suspension point at which the coroutine
+is currently suspended.
 
-Just as printing the `promise_type` is possible from the coroutine address,
-printing the details of the coroutine frame from an address is also possible:
+However, it is non-trivial to map this number back to a source code location.
+The compiler emits debug info labels for the suspension points. This allows us
+to map the suspension point index back to a source code location. In gdb, we
+can use the ``info line`` command to get the source code location of the
+suspension point.
 
 ::
 
-  (gdb) # Get the address of coroutine frame
-  (gdb) print/x *0x418eb0
-  $1 = 0x4019e0
-  (gdb) # Get the linkage name for the coroutine
-  (gdb) x 0x4019e0
-  0x4019e0 <_ZL9coro_taski>:  0xe5894855
-  (gdb) # Turn off the demangler temporarily to avoid the debugger misunderstanding the name.
-  (gdb) set demangle-style none
-  (gdb) # The coroutine frame type is 'linkage_name.coro_frame_ty'
-  (gdb) print  ('_ZL9coro_taski.coro_frame_ty')*(0x418eb0)
-  $2 = {__resume_fn = 0x4019e0 <coro_task(int)>, __destroy_fn = 0x402000 <coro_task(int)>, __promise = {...}, ...}
+  (gdb) info line -function coro_task -label __coro_resume_2
+  Line 45 of "llvm-example.cpp" starts at address 0x1b1b <_ZL9coro_taski.resume+555> and ends at 0x1b46 <_ZL9coro_taski.resume+598>.
+  Line 45 of "llvm-example.cpp" starts at address 0x201b <_ZL9coro_taski.destroy+555> and ends at 0x2046 <_ZL9coro_taski.destroy+598>.
+  Line 45 of "llvm-example.cpp" starts at address 0x253b <_ZL9coro_taski.cleanup+555> and ends at 0x2566 <_ZL9coro_taski.cleanup+598>.
+
+LLDB does not support looking up labels. Furthermore, those labels are only emitted
+starting with clang 21.0.
+
+For simple cases, you might still be able to guess the suspension point correctly.
+Alternatively, you might also want to modify your coroutine library to store
+the line number of the current suspension point in the promise:
+
+.. code-block:: c++
 
-The above is possible because:
+  // For all promise_types we need a new `_coro_return_address` variable:
+  class promise_type {
+    ...
+    void* _coro_return_address = nullptr;
+  };
 
-(1) The name of the debug type of the coroutine frame is the `linkage_name`,
-plus the `.coro_frame_ty` suffix because each coroutine function shares the
-same coroutine type.
+  // For all the awaiter types we need:
+  class awaiter {
+    ...
+    template <typename Promise>
+    __attribute__((noinline)) auto await_suspend(std::coroutine_handle<Promise> handle) {
+          ...
+          handle.promise()._coro_return_address = __builtin_return_address(0);
+    }
+  };
 
-(2) The coroutine function name is accessible from the address of the coroutine
-frame.
+This stores the return address of ``await_suspend`` within the promise.
+Thereby, we can read it back from the promise of a suspended coroutine, and map
+it to an exact source code location. For a complete example, see the ``task``
+type used below for asynchronous programming.
 
-The above commands can be simplified by placing them in debug scripts.
+Alternatively, we can modify the C++ code to store the line number in the
+promise type. We can use a ``std::source_location`` to get the line number of
+the await and store it inside the ``promise_type``. In the debugger, we can
+then read the line number from the promise of the suspended coroutine.
 
-Examples to print coroutine frames
-----------------------------------
+.. code-block:: c++
+
+  // For all the awaiter types we need:
+  class awaiter {
+    ...
+    template <typename Promise>
+    void await_suspend(std::coroutine_handle<Promise> handle,
+                       std::source_location sl = std::source_location::current()) {
+          ...
+          handle.promise().line_number = sl.line();
+    }
+  };
+
+The downside of both approaches is that they come at the price of additional
+runtime cost. In particular the second approach increases binary size, since it
+requires additional ``std::source_location`` objects, and those source
+locations are not stripped by split-dwarf. Whether the first approach is worth
+the additional runtime cost is a trade-off you need to make yourself.
 
-The print examples below use the following definition:
+Async stack traces
+==================
+
+Besides generators, the second common use case for coroutines in C++ is
+asynchronous programming, usually involving libraries such as stdexec, folly,
+cppcoro, boost::asio, or similar libraries. Some of those libraries already
+provide custom debugging support, so in addition to this guide, you might want
+to check out their documentation.
+
+When using coroutines for asynchronous programming, your library usually
+provides you some ``task`` type. This type usually looks similar to this:
 
 .. code-block:: c++
 
+  // async-task-library.hpp
   #include <coroutine>
-  #include <iostream>
+  #include <utility>
 
-  struct task{
+  struct task {
     struct promise_type {
       task get_return_object() { return std::coroutine_handle<promise_type>::from_promise(*this); }
-      std::suspend_always initial_suspend() { return {}; }
-      std::suspend_always final_suspend() noexcept { return {}; }
-      void return_void() noexcept {}
+      auto initial_suspend() { return std::suspend_always{}; }
+
       void unhandled_exception() noexcept {}
 
-      int count = 0;
-    };
+      auto final_suspend() noexcept {
+        struct FinalSuspend {
+          std::coroutine_handle<> continuation;
+          auto await_ready() noexcept { return false; }
+          auto await_suspend(std::coroutine_handle<> handle) noexcept {
+            return continuation;
+          }
+          void await_resume() noexcept {}
+        };
+        return FinalSuspend{continuation};
+      }
 
-    void resume() noexcept {
-      handle.resume();
-    }
+      void return_value(int res) { result = res; }
+
+      std::coroutine_handle<> continuation = std::noop_coroutine();
+      int result = 0;
+      #ifndef NDEBUG
+      void* _coro_suspension_point_addr = nullptr;
+      #endif
+    };
 
-    task(std::coroutine_handle<promise_type> hdl) : handle(hdl) {}
+    task(std::coroutine_handle<promise_type> handle) : handle(handle) {}
     ~task() {
       if (handle)
         handle.destroy();
     }
 
-    std::coroutine_handle<> handle;
-  };
-
-  class await_counter : public std::suspend_always {
-    public:
-      template<class PromiseType>
-      void await_suspend(std::coroutine_handle<PromiseType> handle) noexcept {
-          handle.promise().count++;
+    struct Awaiter {
+      std::coroutine_handle<promise_type> handle;
+      auto await_ready() { return false; }
+
+      template <typename P>
+      #ifndef NDEBUG
+      __attribute__((noinline))
+      #endif
+      auto await_suspend(std::coroutine_handle<P> continuation) {
+        handle.promise().continuation = continuation;
+        #ifndef NDEBUG
+        continuation.promise()._coro_suspension_point_addr = __builtin_return_address(0);
+        #endif
+        return handle;
       }
+      int await_resume() {
+        return handle.promise().result;
+      }
+    };
+
+    auto operator co_await() {
+      return Awaiter{handle};
+    }
+
+    int syncStart() {
+      handle.resume();
+      return handle.promise().result;
+    }
+
+  private:
+    std::coroutine_handle<promise_type> handle;
   };
 
-  static task coro_task(int v) {
-    int a = v;
-    co_await await_counter{};
-    a++;
-    std::cout << a << "\n";
-    a++;
-    std::cout << a << "\n";
-    a++;
-    std::cout << a << "\n";
-    co_await await_counter{};
-    a++;
-    std::cout << a << "\n";
-    a++;
-    std::cout << a << "\n";
+Note how the ``task::promise_type`` has a member variable
+``std::coroutine_handle<> continuation``. This is the handle of the coroutine
+that will be resumed when the current coroutine is finished executing (see
+``final_suspend``). In a sense, this is the "return address" of the coroutine.
+It is set inside ``operator co_await`` when another coroutine calls our
+generator and awaits for the next value to be produced.
+
+The result value is returned via the ``int result`` member. It is written in
+``return_value`` and read by ``Awaiter::await_resume``. Usually, the result
+type of a task is a template argument. For simplicity's sake, we hard-coded the
+``int`` type in this example.
+
+Stack traces of in-flight coroutines
+------------------------------------
+
+Let's assume you have the following program and set a breakpoint inside the
+``write_output`` function. There are multiple call paths through which this
+function could have been reached. How can we find out said call path?
+
+.. code-block:: c++
+
+  #include <iostream>
+  #include <string_view>
+  #include "async-task-library.hpp"
+
+  static task write_output(std::string_view contents) {
+    std::cout << contents << "\n";
+    co_return contents.size();
+  }
+
+  static task greet() {
+    int bytes_written = 0;
+    bytes_written += co_await write_output("Hello");
+    bytes_written += co_await write_output("World");
+    co_return bytes_written;
   }
 
   int main() {
-    task t = coro_task(43);
-    t.resume();
-    t.resume();
-    t.resume();
+    int bytes_written = greet().syncStart();
+    std::cout << "Bytes written: " << bytes_written << "\n";
     return 0;
   }
 
-In debug mode (`O0` + `g`), the printing result would be:
+To do so, let's break inside ``write_output``. We can understand our call-stack
+by looking into the special ``__promise`` variable. This artificial variable is
+generated by the compiler and points to the ``promise_type`` instance
+corresponding to the currently in-flight coroutine. In this case, the
+``__promise`` variable contains the ``continuation`` which points to our
+caller. That caller again contains a ``promise`` with a ``continuation`` which
+points to our caller's caller.
 
-.. parsed-literal::
+.. image:: ./coro-async-task-continuations.png
 
-  {__resume_fn = 0x4019e0 <coro_task(int)>, __destroy_fn = 0x402000 <coro_task(int)>, __promise = {count = 1}, v = 43, a = 45, __coro_index = 1 '\001', struct_std__suspend_always_0 = {__int_8 = 0 '\000'},
-    class_await_counter_1 = {__int_8 = 0 '\000'}, class_await_counter_2 = {__int_8 = 0 '\000'}, struct_std__suspend_always_3 = {__int_8 = 0 '\000'}}
+We can figure out the involved coroutine functions and their current suspension
+points as discussed above in the "Inspecting a suspended coroutine" section.
 
-In the above, the values of `v` and `a` are clearly expressed, as are the
-temporary values for `await_counter` (`class_await_counter_1` and
-`class_await_counter_2`) and `std::suspend_always` (
-`struct_std__suspend_always_0` and `struct_std__suspend_always_3`). The index
-of the current suspension point of the coroutine is emitted as `__coro_index`.
-In the above example, the `__coro_index` value of `1` means the coroutine
-stopped at the second suspend point (Note that `__coro_index` is zero indexed)
-which is the first `co_await await_counter{};` in `coro_task`. Note that the
-first initial suspend point is the compiler generated
-`co_await promise_type::initial_suspend()`.
+When using LLDB's CLI, the command ``p --ptr-depth 4 __promise`` might also be
+useful to automatically dereference all the pointers up to the given depth.
 
-However, when optimizations are enabled, the printed result changes drastically:
+To get a flat representation of that call stack, we can use a debugger script,
+such as the one shown in the :ref:`lldb-script` section. With that
+script, we can run ``coro bt`` to get the following stack trace:
 
-.. parsed-literal::
+.. code-block::
 
-  {__resume_fn = 0x401280 <coro_task(int)>, __destroy_fn = 0x401390 <coro_task(int)>, __promise = {count = 1}, __int_32_0 = 43, __coro_index = 1 '\001'}
+  (lldb) coro bt
+  frame #0: write_output(std::basic_string_view<char, std::char_traits<char>>) at /home/avogelsgesang/Documents/corotest/async-task-example.cpp:6:16
+  [async] frame #1: greet() at /home/avogelsgesang/Documents/corotest/async-task-example.cpp:12:20
+  [async] frame #2: std::__n4861::coroutine_handle<std::__n4861::noop_coroutine_promise>::__frame::__dummy_resume_destroy() at /usr/include/c++/14/coroutine:298, suspension point unknown
+  frame #3: std::__n4861::coroutine_handle<task::promise_type>::resume() const at /usr/include/c++/14/coroutine:242:29
+  frame #4: task::syncStart() at /home/avogelsgesang/Documents/corotest/async-task-library.hpp:78:14
+  frame #5: main at /home/avogelsgesang/Documents/corotest/async-task-example.cpp:18:11
+  frame #6: __libc_start_call_main at sysdeps/nptl/libc_start_call_main.h:58:16
+  frame #7: __libc_start_main_impl at csu/libc-start.c:360:3
+  frame #8: _start at :4294967295
 
-Unused values are optimized out, as well as the name of the local variable `a`.
-The only information remained is the value of a 32-bit integer. In this simple
-case, it seems to be pretty clear that `__int_32_0` represents `a`. However, it
-is not true.
+Note how the frames #1 and #2 are async frames.
 
-An important note with optimization is that the value of a variable may not
-properly express the intended value in the source code.  For example:
+The ``coro bt`` command already includes logic to identify the exact suspension
+point of each frame based on the ``_coro_suspension_point_addr`` stored inside
+the promise.
+
+Stack traces of suspended coroutines
+------------------------------------
+
+Usually, while a coroutine is waiting for, e.g., an in-flight network request,
+the suspended ``coroutine_handle`` is stored within the work queues inside the
+IO scheduler. As soon as we get hold of the coroutine handle, we can backtrace
+it by using ``coro bt <coro_handle>`` where ``<coro_handle>`` is an expression
+evaluating to the coroutine handle of the suspended coroutine.
+
+Keeping track of all existing coroutines
+----------------------------------------
+
+Usually, we should be able to get hold of all currently suspended coroutines by
+inspecting the worker queues of the IO scheduler. In cases where this is not
+possible, we can use the following approach to keep track of all currently
+suspended coroutines.
+
+One such solution is to store the list of in-flight coroutines in a collection:
+
+.. code-block:: c++
+
+  inline std::unordered_set<std::coroutine_handle<void>> inflight_coroutines;
+  inline std::mutex inflight_coroutines_mutex;
+
+  class promise_type {
+  public:
+      promise_type() {
+          std::unique_lock<std::mutex> lock(inflight_coroutines_mutex);
+          inflight_coroutines.insert(std::coroutine_handle<promise_type>::from_promise(*this));
+      }
+      ~promise_type() {
+          std::unique_lock<std::mutex> lock(inflight_coroutines_mutex);
+          inflight_coroutines.erase(std::coroutine_handle<promise_type>::from_promise(*this));
+      }
+  };
+
+With this in place, it is possible to inspect ``inflight_coroutines`` from the
+debugger, and rely on LLDB's ``std::coroutine_handle`` pretty-printer to
+inspect the coroutines.
+
+This technique will track *all* coroutines, also the ones which are currently
+awaiting another coroutine, though. To identify just the "roots" of our
+in-flight coroutines, we can use the ``coro in-flight inflight_coroutines``
+command provided by the :ref:`lldb-script`.
+
+Please note that the above is expensive from a runtime performance perspective,
+and requires locking to prevent data races. As such, it is not recommended to
+use this approach in production code.
+
+Known issues & workarounds for older LLDB versions
+==================================================
+
+LLDB before 21.0 did not yet show the ``__coro_frame`` inside
+``coroutine_handle``. To inspect the coroutine frame, you had to use the
+approach described in the :ref:`devirtualization` section.
+
+LLDB before 18.0 was hiding the ``__promise`` and ``__coro_frame``
+variable by default. The variables are still present, but they need to be
+explicitly added to the "watch" pane in VS Code or requested via
+``print __promise`` and ``print __coro_frame`` from the debugger console.
+
+LLDB before 16.0 did not yet provide a pretty-printer for
+``std::coroutine_handle``. To inspect the coroutine handle, you had to manually
+use the approach described in the :ref:`devirtualization`
+section.
+
+Toolchain Implementation Details
+================================
+
+This section covers the ABI, as well as additional compiler-specific behavior.
+The ABI is followed by all compilers, on all major systems, including Windows,
+Linux and macOS. Different compilers emit different debug information, though.
+
+Ramp, resume and destroy functions
+----------------------------------
+
+Every coroutine is split into three parts:
+
+* The ramp function allocates the coroutine frame and initializes it, usually
+  copying over all variables into the coroutine frame
+* The resume function continues the coroutine from its previous suspension point
+* The destroy function destroys and deallocates the coroutine frame
+* The cleanup function destroys the coroutine frame but does not deallocate it.
+  It is used when the coroutine's allocation was elided thanks to
+  `Heap Allocation Elision (HALO) <https://www.open-std.org/JTC1/SC22/WG21/docs/papers/2018/p0981r0.html>`_
+
+The ramp function is called by the coroutine's caller, and available under the
+original function name used in the C++ source code. The resume function is
+called via ``std::coroutine_handle::resume``. The destroy function is called
+via ``std::coroutine_handle::destroy``.
+
+Information between the three functions is passed via the coroutine frame, a
+compiler-synthesized struct that contains all necessary internal state. The
+resume function knows where to resume execution by reading the suspension point
+index from the coroutine frame. Similarly, the destroy function relies on the
+suspension point index to know which variables are currently in scope and need
+to be destructed.
+
+Usually, the destroy function calls all destructors and deallocates the
+coroutine frame. When a coroutine frame was elided thanks to HALO, only the
+destructors need to be called, but the coroutine frame must not be deallocated.
+In those cases, the cleanup function is used instead of the destroy function.
+
+For coroutines allocated with ``[[clang::coro_await_elidable]]``, clang also
+generates a ``.noalloc`` variant of the ramp function, which does not allocate
+the coroutine frame by itself, but instead expects the caller to allocate the
+coroutine frame and pass it to the ramp function.
+
+When trying to intercept all creations of new coroutines in the debugger, you
+hence might have to set breakpoints in the ramp function and its ``.noalloc``
+variant.
+
+Artificial ``__promise`` and ``__coro_frame`` variables
+-------------------------------------------------------
+
+Inside all coroutine functions, clang / LLVM synthesize a ``__promise`` and
+``__coro_frame`` variable. These variables are used to store the coroutine's
+state. When inside the coroutine function, those can be used to directly
+inspect the promise and the coroutine frame of the own function.
+
+The ABI of a coroutine
+----------------------
+
+A ``std::coroutine_handle`` essentially only holds a pointer to a coroutine
+frame. It resembles the following struct:
+
+.. code-block:: c++
+
+  template<typename promise_type>
+  struct coroutine_handle {
+    void* __coroutine_frame = nullptr;
+  };
+
+The structure of coroutine frames is defined as
+
+.. code-block:: c++
+
+  struct my_coroutine_frame {
+    void (*__resume)(coroutine_frame*); // function pointer to the `resume` function
+    void (*__destroy)(coroutine_frame*); // function pointer to the `destroy` function
+    promise_type promise; // the corresponding `promise_type`
+    ... // Internal coroutine state
+  }
+
+For each coroutine, the compiler synthesizes a different coroutine type,
+storing all necessary internal state. The actual coroutine type is type-erased
+behind the ``std::coroutine_handle``.
+
+However, all coroutine frames always contain the ``resume`` and ``destroy``
+functions as their first two members. As such, we can read the function
+pointers from the coroutine frame and then obtain the function's name from its
+address.
+
+The promise is guaranteed to be at a 16 byte offset from the coroutine frame.
+If we have a coroutine handle at address 0x416eb0, we can hence reinterpret-cast
+the promise as follows:
+
+.. code-block:: text
+
+  print (task::promise_type)*(0x416eb0+16)
+
+Implementation in clang / LLVM
+------------------------------
+
+The C++ Coroutines feature in the Clang compiler is implemented in two parts of
+the compiler. Semantic analysis is performed in Clang, and Coroutine
+construction and optimization takes place in the LLVM middle-end.
+
+For each coroutine function, the frontend generates a single corresponding
+LLVM-IR function. This function uses special ``llvm.coro.suspend`` intrinsics
+to mark the suspension points of the coroutine. The middle end first optimizes
+this function and applies, e.g., constant propagation across the whole,
+non-split coroutine.
+
+CoroSplit then splits the function into ramp, resume and destroy functions.
+This pass also moves stack-local variables which are alive across suspension
+points into the coroutine frame. Most of the heavy lifting to preserve debugging
+information is done in this pass. This pass needs to rewrite all variable
+locations to point into the coroutine frame.
+
+Afterwards, a couple of additional optimizations are applied, before code
+gets emitted, but none of them are really interesting regarding debugging
+information.
+
+For more details on the IR representation of coroutines and the relevant
+optimization passes, see `Coroutines in LLVM <https://llvm.org/docs/Coroutines.html>`_.
+
+Emitting debug information inside ``CoroSplit`` forces us to generate
+insufficient debugging information. Usually, the compiler generates debug
+information in the frontend, as debug information is highly language specific.
+However, this is not possible for coroutine frames because the frames are
+constructed in the LLVM middle-end.
+
+To mitigate this problem, the LLVM middle end attempts to generate some debug
+information, which is unfortunately incomplete, since much of the language
+specific information is missing in the middle end.
+
+.. _devirtualization:
+
+Devirtualization of coroutine handles
+-------------------------------------
+
+Figuring out the promise type and the coroutine frame type of a coroutine
+handle requires inspecting the ``resume`` and ``destroy`` function pointers.
+There are two possible approaches to do so:
+
+1. clang always names the type by appending ``.coro_frame_ty`` to the
+   linkage name of the ramp function.
+2. Both clang and GCC add the function-local ``__promise`` and
+   ``__coro_frame`` variables to the resume and destroy functions.
+   We can lookup their types and thereby get the types of promise
+   and coroutine frame.
+
+In gdb, one can use the following approach to devirtualize coroutine type,
+assuming we have a ``std::coroutine_handle`` is at address 0x418eb0:
+
+::
+
+  (gdb) # Get the address of coroutine frame
+  (gdb) print/x *0x418eb0
+  $1 = 0x4019e0
+  (gdb) # Get the linkage name for the coroutine
+  (gdb) x 0x4019e0
+  0x4019e0 <_ZL9coro_taski>:  0xe5894855
+  (gdb) # Turn off the demangler temporarily to avoid the debugger misunderstanding the name.
+  (gdb) set demangle-style none
+  (gdb) # The coroutine frame type is 'linkage_name.coro_frame_ty'
+  (gdb) print  ('_ZL9coro_taski.coro_frame_ty')*(0x418eb0)
+  $2 = {__resume_fn = 0x4019e0 <coro_task(int)>, __destroy_fn = 0x402000 <coro_task(int)>, __promise = {...}, ...}
+
+In practice, one would use the ``show-coro-frame`` command provided by the
+:ref:`gdb-script`.
+
+LLDB comes with devirtualization support out of the box, as part of the
+pretty-printer for ``std::coroutine_handle``. Internally, this pretty-printer
+uses the second approach. We look up the types in the destroy function and not
+the resume function because the resume function pointer will be set to a
+nullptr as soon as a coroutine reaches its final suspension point. If we used
+the resume function, devirtualization would hence fail for all coroutines that
+have reached their final suspension point.
+
+Interpreting the coroutine frame in optimized builds
+----------------------------------------------------
+
+The ``__coro_frame`` variable usually refers to the coroutine frame of an
+*in-flight* coroutine. This means, the coroutine is currently executing.
+However, the compiler only guarantees the coroutine frame to be in a consistent
+state while the coroutine is suspended. As such, the variables inside the
+``__coro_frame`` variable might be outdated, in particular when optimizations
+are enabled.
+
+Furthermore, when optimizations are enabled, the compiler will layout the
+coroutine frame more aggressively. Unused values are optimized out, and the
+state will usually contain only the minimal information required to reconstruct
+the coroutine's state.
+
+clang / LLVM usually use variables like ``__int_32_0`` to represent this
+optimized storage. Those values usually do not directly correspond to variables
+in the source code.
+
+When compiling the program
 
 .. code-block:: c++
 
   static task coro_task(int v) {
     int a = v;
-    co_await await_counter{};
+    co_await some_other_task();
     a++; // __int_32_0 is 43 here
     std::cout << a << "\n";
     a++; // __int_32_0 is still 43 here
     std::cout << a << "\n";
     a++; // __int_32_0 is still 43 here!
     std::cout << a << "\n";
-    co_await await_counter{};
+    co_await some_other_task();
     a++; // __int_32_0 is still 43 here!!
     std::cout << a << "\n";
     a++; // Why is __int_32_0 still 43 here?
     std::cout << a << "\n";
   }
 
-When debugging step-by-step, the value of `__int_32_0` seemingly does not
-change, despite being frequently incremented, and instead is always `43`.
+clang creates a single entry ``__int_32_0`` in the coroutine state.
+
+Intuitively, one might assume that ``__int_32_0`` represents the value of the
+local variable ``a``. However, inspecting ``__int_32_0`` in the debugger while
+single-stepping will reveal that the value of ``__int_32_0`` stays constant,
+despite ``a`` being frequently incremented.
+
 While this might be surprising, this is a result of the optimizer recognizing
-that it can eliminate most of the load/store operations. The above code gets
-optimized to the equivalent of:
+that it can eliminate most of the load/store operations.
+The above code gets optimized to the equivalent of:
 
 .. code-block:: c++
 
   static task coro_task(int v) {
-    store v to __int_32_0 in the frame
+    store v into __int_32_0 in the frame
     co_await await_counter{};
     a = load __int_32_0
     std::cout << a+1 << "\n";
@@ -301,173 +748,280 @@ optimized to the equivalent of:
     std::cout << a+5 << "\n";
   }
 
-It should now be obvious why the value of `__int_32_0` remains unchanged
-throughout the function. It is important to recognize that `__int_32_0`
-does not directly correspond to `a`, but is instead a variable generated
-to assist the compiler in code generation. The variables in an optimized
-coroutine frame should not be thought of as directly representing the
-variables in the C++ source.
+It should now be obvious why the value of ``__int_32_0`` remains unchanged
+throughout the function. It is important to recognize that ``__int_32_0`` does
+not directly correspond to ``a``, but is instead a variable generated to assist
+the compiler in code generation. The variables in an optimized coroutine frame
+should not be thought of as directly representing the variables in the C++
+source.
 
-Get the suspended points
-========================
-
-An important requirement for debugging coroutines is to understand suspended
-points, which are where the coroutine is currently suspended and awaiting.
-
-For simple cases like the above, inspecting the value of the `__coro_index`
-variable in the coroutine frame works well.
-
-However, it is not quite so simple in really complex situations. In these
-cases, it is necessary to use the coroutine libraries to insert the
-line-number.
-
-For example:
 
-.. code-block:: c++
+Resources
+=========
 
-  // For all the promise_type we want:
-  class promise_type {
-    ...
-  +  unsigned line_number = 0xffffffff;
-  };
+.. _lldb-script:
 
-  #include <source_location>
+LLDB Debugger Script
+--------------------
 
-  // For all the awaiter types we need:
-  class awaiter {
-    ...
-    template <typename Promise>
-    void await_suspend(std::coroutine_handle<Promise> handle,
-                       std::source_location sl = std::source_location::current()) {
-          ...
-          handle.promise().line_number = sl.line();
-    }
-  };
+The following script provides the ``coro bt`` and ``coro in-flight`` commands
+discussed above. It can be loaded into LLDB using ``command script import
+lldb_coro_debugging.py``. To load this by default, add this command to your
+``~/.lldbinit`` file.
 
-In this case, we use `std::source_location` to store the line number of the
-await inside the `promise_type`.  Since we can locate the coroutine function
-from the address of the coroutine, we can identify suspended points this way
-as well.
+Note that this script requires LLDB 21.0 or newer.
 
-The downside here is that this comes at the price of additional runtime cost.
-This is consistent with the C++ philosophy of "Pay for what you use".
-
-Get the asynchronous stack
-==========================
-
-Another important requirement to debug a coroutine is to print the asynchronous
-stack to identify the asynchronous caller of the coroutine.  As many
-implementations of coroutine types store `std::coroutine_handle<> continuation`
-in the promise type, identifying the caller should be trivial.  The
-`continuation` is typically the awaiting coroutine for the current coroutine.
-That is, the asynchronous parent.
-
-Since the `promise_type` is obtainable from the address of a coroutine and
-contains the corresponding continuation (which itself is a coroutine with a
-`promise_type`), it should be trivial to print the entire asynchronous stack.
-
-This logic should be quite easily captured in a debugger script.
-
-Examples to print asynchronous stack
-------------------------------------
-
-Here is an example to print the asynchronous stack for the normal task implementation.
+.. code-block:: python
 
-.. code-block:: c++
+  # lldb_coro_debugging.py
+  import lldb
+  from lldb.plugins.parsed_cmd import ParsedCommand
+
+  def _get_first_var_path(v, paths):
+      """
+      Tries multiple variable paths via `GetValueForExpressionPath`
+      and returns the first one that succeeds, or None if none succeed.
+      """
+      for path in paths:
+          var = v.GetValueForExpressionPath(path)
+          if var.error.Success():
+              return var
+      return None
+
+
+  def _print_async_bt(coro_hdl, result, *, curr_idx, start, limit, continuation_paths, prefix=""):
+      """
+      Prints a backtrace for an async coroutine stack starting from `coro_hdl`,
+      using the given `continuation_paths` to get the next coroutine from the promise.
+      """
+      target = coro_hdl.GetTarget()
+      while curr_idx < limit and coro_hdl is not None and coro_hdl.error.Success():
+          # Print the stack frame, if in range
+          if curr_idx >= start:
+              # Figure out the function name
+              destroy_func_var = coro_hdl.GetValueForExpressionPath(".destroy")
+              destroy_addr = target.ResolveLoadAddress(destroy_func_var.GetValueAsAddress())
+              func_name = destroy_addr.function.name
+              # Figure out the line entry to show
+              suspension_addr_var = coro_hdl.GetValueForExpressionPath(".promise._coro_suspension_point_addr")
+              if suspension_addr_var.error.Success():
+                  line_entry = target.ResolveLoadAddress(suspension_addr_var.GetValueAsAddress()).line_entry
+                  print(f"{prefix} frame #{curr_idx}: {func_name} at {line_entry}", file=result)
+              else:
+                  # We don't know the exact line, print the suspension point ID, so we at least show
+                  # the id of the current suspension point
+                  suspension_point_var = coro_hdl.GetValueForExpressionPath(".coro_frame.__coro_index")
+                  if suspension_point_var.error.Success():
+                      suspension_point = suspension_point_var.GetValueAsUnsigned()
+                  else:
+                      suspension_point = "unknown"
+                  line_entry = destroy_addr.line_entry
+                  print(f"{prefix} frame #{curr_idx}: {func_name} at {line_entry}, suspension point {suspension_point}", file=result)
+          # Move to the next stack frame
+          curr_idx += 1
+          promise_var = coro_hdl.GetChildMemberWithName("promise")
+          coro_hdl = _get_first_var_path(promise_var, continuation_paths)
+      return curr_idx
+
+  def _print_combined_bt(frame, result, *, unfiltered, curr_idx, start, limit, continuation_paths):
+      """
+      Prints a backtrace starting from `frame`, interleaving async coroutine frames
+      with regular frames.
+      """
+      while curr_idx < limit and frame.IsValid():
+          if curr_idx >= start and (unfiltered or not frame.IsHidden()):
+              print(f"frame #{curr_idx}: {frame.name} at {frame.line_entry}", file=result)
+          curr_idx += 1
+          coro_var = _get_first_var_path(frame.GetValueForVariablePath("__promise"), continuation_paths)
+          if coro_var:
+              curr_idx = _print_async_bt(coro_var, result,
+                  curr_idx=curr_idx, start=start, limit=limit,
+                  continuation_paths=continuation_paths, prefix="[async]")
+          frame = frame.parent
+
+
+  class CoroBacktraceCommand(ParsedCommand):
+      def get_short_help(self):
+          return "Create a backtrace for C++-20 coroutines"
+
+      def get_flags(self):
+          return lldb.eCommandRequiresFrame | lldb.eCommandProcessMustBePaused
+
+      def setup_command_definition(self):
+          ov_parser = self.get_parser()
+          ov_parser.add_option(
+              "e",
+              "continuation-expr",
+              help = (
+                  "Semi-colon-separated list of expressions evaluated against the promise object"
+                  "to get the next coroutine (e.g. `.continuation;.coro_parent`)"
+              ),
+              value_type = lldb.eArgTypeNone,
+              dest = "continuation_expr_arg",
+              default = ".continuation",
+          )
+          ov_parser.add_option(
+              "c",
+              "count",
+              help = "How many frames to display (0 for all)",
+              value_type = lldb.eArgTypeCount,
+              dest = "count_arg",
+              default = 20,
+          )
+          ov_parser.add_option(
+              "s",
+              "start",
+              help = "Frame in which to start the backtrace",
+              value_type = lldb.eArgTypeIndex,
+              dest = "frame_index_arg",
+              default = 0,
+          )
+          ov_parser.add_option(
+              "u",
+              "unfiltered",
+              help = "Do not filter out frames according to installed frame recognizers",
+              value_type = lldb.eArgTypeBoolean,
+              dest = "unfiltered_arg",
+              default = False,
+          )
+          ov_parser.add_argument_set([
+              ov_parser.make_argument_element(
+                  lldb.eArgTypeExpression,
+                  repeat="optional"
+              )
+          ])
+
+      def __call__(self, debugger, args_array, exe_ctx, result):
+          ov_parser = self.get_parser()
+          continuation_paths = ov_parser.continuation_expr_arg.split(";")
+          count = ov_parser.count_arg
+          if count == 0:
+              count = 99999
+          frame_index = ov_parser.frame_index_arg
+          unfiltered = ov_parser.unfiltered_arg
+
+          frame = exe_ctx.GetFrame()
+          if not frame.IsValid():
+              result.SetError("invalid frame")
+              return
 
-  // debugging-example.cpp
-  #include <coroutine>
-  #include <iostream>
-  #include <utility>
+          if len(args_array) > 1:
+              result.SetError("At most one expression expected")
+              return
+          elif len(args_array) == 1:
+              expr = args_array.GetItemAtIndex(0).GetStringValue(9999)
+              coro_hdl = frame.EvaluateExpression(expr)
+              if not coro_hdl.error.Success():
+                  result.AppendMessage(
+                      f'error: expression failed {expr} => {coro_hdl.error}'
+                  )
+                  result.SetError(f"Expression `{expr}` failed to evaluate")
+                  return
+              _print_async_bt(coro_hdl, result,
+                  curr_idx = 0, start = frame_index, limit = frame_index + count,
+                  continuation_paths = continuation_paths)
+          else:
+              _print_combined_bt(frame, result, unfiltered=unfiltered,
+                  curr_idx = 0, start = frame_index, limit = frame_index + count,
+                  continuation_paths = continuation_paths)
+
+
+  class CoroInflightCommand(ParsedCommand):
+      def get_short_help(self):
+          return "Identify all in-flight coroutines"
+
+      def get_flags(self):
+          return lldb.eCommandRequiresTarget | lldb.eCommandProcessMustBePaused
+
+      def setup_command_definition(self):
+          ov_parser = self.get_parser()
+          ov_parser.add_option(
+              "e",
+              "continuation-expr",
+              help = (
+                  "Semi-colon-separated list of expressions evaluated against the promise object"
+                  "to get the next coroutine (e.g. `.continuation;.coro_parent`)"
+              ),
+              value_type = lldb.eArgTypeNone,
+              dest = "continuation_expr_arg",
+              default = ".continuation",
+          )
+          ov_parser.add_option(
+              "c",
+              "count",
+              help = "How many frames to display (0 for all)",
+              value_type = lldb.eArgTypeCount,
+              dest = "count_arg",
+              default = 5,
+          )
+          ov_parser.add_argument_set([
+              ov_parser.make_argument_element(
+                  lldb.eArgTypeExpression,
+                  repeat="plus"
+              )
+          ])
+
+      def __call__(self, debugger, args_array, exe_ctx, result):
+          ov_parser = self.get_parser()
+          continuation_paths = ov_parser.continuation_expr_arg.split(";")
+          count = ov_parser.count_arg
+
+          # Collect all coroutine_handles from the provided containers
+          all_coros = []
+          for entry in args_array:
+              expr = entry.GetStringValue(9999)
+              if exe_ctx.frame.IsValid():
+                  coro_container = exe_ctx.frame.EvaluateExpression(expr)
+              else:
+                  coro_container = exe_ctx.target.EvaluateExpression(expr)
+              if not coro_container.error.Success():
+                  result.AppendMessage(
+                      f'error: expression failed {expr} => {coro_container.error}'
+                  )
+                  result.SetError(f"Expression `{expr}` failed to evaluate")
+                  return
+              for entry in coro_container.children:
+                  if "coroutine_handle" not in entry.GetType().name:
+                      result.SetError(f"Found entry of type {entry.GetType().name} in {expr},"
+                                      "  expected a coroutine handle")
+                      return
+                  all_coros.append(entry)
 
-  struct task {
-    struct promise_type {
-      task get_return_object();
-      std::suspend_always initial_suspend() { return {}; }
+          # Remove all coroutines that are currently waiting for other coroutines to finish
+          coro_roots = {c.GetChildMemberWithName("coro_frame").GetValueAsAddress(): c for c in all_coros}
+          for coro_hdl in all_coros:
+              parent_coro = _get_first_var_path(coro_hdl.GetChildMemberWithName("promise"), continuation_paths)
+              parent_addr = parent_coro.GetChildMemberWithName("coro_frame").GetValueAsAddress()
+              if parent_addr in coro_roots:
+                  del coro_roots[parent_addr]
 
-      void unhandled_exception() noexcept {}
+          # Print all remaining coroutines
+          for addr, root_hdl in coro_roots.items():
+              print(f"coroutine root 0x{addr:x}", file=result)
+              _print_async_bt(root_hdl, result,
+                              curr_idx=0, start=0, limit=count,
+                              continuation_paths=continuation_paths, prefix="    ")
 
-      struct FinalSuspend {
-        std::coroutine_handle<> continuation;
-        auto await_ready() noexcept { return false; }
-        auto await_suspend(std::coroutine_handle<> handle) noexcept {
-          return continuation;
-        }
-        void await_resume() noexcept {}
-      };
-      FinalSuspend final_suspend() noexcept { return {continuation}; }
 
-      void return_value(int res) { result = res; }
+  def __lldb_init_module(debugger, internal_dict):
+      debugger.HandleCommand("command container add -h 'Debugging utilities for C++20 coroutines' coro")
+      debugger.HandleCommand(f"command script add -o -p -c {__name__}.CoroBacktraceCommand coro bt")
+      debugger.HandleCommand(f"command script add -o -p -c {__name__}.CoroInflightCommand coro in-flight")
+      print("Coro debugging utilities installed. Use `help coro` to see available commands.")
 
-      std::coroutine_handle<> continuation = std::noop_coroutine();
-      int result = 0;
-    };
+  if __name__ == '__main__':
+      print("This script should be loaded from LLDB using `command script import <filename>`")
 
-    task(std::coroutine_handle<promise_type> handle) : handle(handle) {}
-    ~task() {
-      if (handle)
-        handle.destroy();
-    }
+.. _gdb-script:
 
-    auto operator co_await() {
-      struct Awaiter {
-        std::coroutine_handle<promise_type> handle;
-        auto await_ready() { return false; }
-        auto await_suspend(std::coroutine_handle<> continuation) {
-          handle.promise().continuation = continuation;
-          return handle;
-        }
-        int await_resume() {
-          int ret = handle.promise().result;
-          handle.destroy();
-          return ret;
-        }
-      };
-      return Awaiter{std::exchange(handle, nullptr)};
-    }
+GDB Debugger Script
+-------------------
 
-    int syncStart() {
-      handle.resume();
-      return handle.promise().result;
-    }
+For GDB, the following script provides a couple of useful commands:
 
-  private:
-    std::coroutine_handle<promise_type> handle;
-  };
-
-  task task::promise_type::get_return_object() {
-    return std::coroutine_handle<promise_type>::from_promise(*this);
-  }
-
-  namespace detail {
-  template <int N>
-  task chain_fn() {
-    co_return N + co_await chain_fn<N - 1>();
-  }
-
-  template <>
-  task chain_fn<0>() {
-    // This is the default breakpoint.
-    __builtin_debugtrap();
-    co_return 0;
-  }
-  }  // namespace detail
-
-  task chain() {
-    co_return co_await detail::chain_fn<30>();
-  }
-
-  int main() {
-    std::cout << chain().syncStart() << "\n";
-    return 0;
-  }
-
-In the example, the ``task`` coroutine holds a ``continuation`` field,
-which would be resumed once the ``task`` completes.
-In another word, the ``continuation`` is the asynchronous caller for the ``task``.
-Just like the normal function returns to its caller when the function completes.
-
-So we can use the ``continuation`` field to construct the asynchronous stack:
+* ``async-bt`` to print the stack trace of a coroutine
+* ``show-coro-frame`` to print the coroutine frame, similar to
+  LLDB's builtin pretty-printer for coroutine frames
 
 .. code-block:: python
 
@@ -599,7 +1153,7 @@ So we can use the ``continuation`` field to construct the asynchronous stack:
           addr = int(argv[0], 16)
           block = gdb.block_for_pc(long(cast_addr2long_pointer(addr).dereference()))
           if block is None:
-              print "block " + str(addr) + "  is none."
+              print "block " + str(addr) + " is None."
               return
 
           # Disable demangling since gdb will treat names starting with `_Z`(The marker for Itanium ABI) specially.
@@ -614,128 +1168,17 @@ So we can use the ``continuation`` field to construct the asynchronous stack:
 
   ShowCoroFrame()
 
-Then let's run:
-
-.. code-block:: text
-
-  $ clang++ -std=c++20 -g debugging-example.cpp -o debugging-example
-  $ gdb ./debugging-example
-  (gdb) # We've already set the breakpoint.
-  (gdb) r
-  Program received signal SIGTRAP, Trace/breakpoint trap.
-  detail::chain_fn<0> () at debugging-example2.cpp:73
-  73	  co_return 0;
-  (gdb) # Executes the debugging scripts
-  (gdb) source debugging-helper.py
-  (gdb) # Print the asynchronous stack
-  (gdb) async-bt
-  #0 0x401c40 in detail::chain_fn<0>() ['frame_addr = 0x441860', 'promise_addr = 0x441870', 'continuation_addr = 0x441870'] at debugging-example.cpp:71
-  #1 0x4022d0 in detail::chain_fn<1>() ['frame_addr = 0x441810', 'promise_addr = 0x441820', 'continuation_addr = 0x441820'] at debugging-example.cpp:66
-  #2 0x403060 in detail::chain_fn<2>() ['frame_addr = 0x4417c0', 'promise_addr = 0x4417d0', 'continuation_addr = 0x4417d0'] at debugging-example.cpp:66
-  #3 0x403df0 in detail::chain_fn<3>() ['frame_addr = 0x441770', 'promise_addr = 0x441780', 'continuation_addr = 0x441780'] at debugging-example.cpp:66
-  #4 0x404b80 in detail::chain_fn<4>() ['frame_addr = 0x441720', 'promise_addr = 0x441730', 'continuation_addr = 0x441730'] at debugging-example.cpp:66
-  #5 0x405910 in detail::chain_fn<5>() ['frame_addr = 0x4416d0', 'promise_addr = 0x4416e0', 'continuation_addr = 0x4416e0'] at debugging-example.cpp:66
-  #6 0x4066a0 in detail::chain_fn<6>() ['frame_addr = 0x441680', 'promise_addr = 0x441690', 'continuation_addr = 0x441690'] at debugging-example.cpp:66
-  #7 0x407430 in detail::chain_fn<7>() ['frame_addr = 0x441630', 'promise_addr = 0x441640', 'continuation_addr = 0x441640'] at debugging-example.cpp:66
-  #8 0x4081c0 in detail::chain_fn<8>() ['frame_addr = 0x4415e0', 'promise_addr = 0x4415f0', 'continuation_addr = 0x4415f0'] at debugging-example.cpp:66
-  #9 0x408f50 in detail::chain_fn<9>() ['frame_addr = 0x441590', 'promise_addr = 0x4415a0', 'continuation_addr = 0x4415a0'] at debugging-example.cpp:66
-  #10 0x409ce0 in detail::chain_fn<10>() ['frame_addr = 0x441540', 'promise_addr = 0x441550', 'continuation_addr = 0x441550'] at debugging-example.cpp:66
-  #11 0x40aa70 in detail::chain_fn<11>() ['frame_addr = 0x4414f0', 'promise_addr = 0x441500', 'continuation_addr = 0x441500'] at debugging-example.cpp:66
-  #12 0x40b800 in detail::chain_fn<12>() ['frame_addr = 0x4414a0', 'promise_addr = 0x4414b0', 'continuation_addr = 0x4414b0'] at debugging-example.cpp:66
-  #13 0x40c590 in detail::chain_fn<13>() ['frame_addr = 0x441450', 'promise_addr = 0x441460', 'continuation_addr = 0x441460'] at debugging-example.cpp:66
-  #14 0x40d320 in detail::chain_fn<14>() ['frame_addr = 0x441400', 'promise_addr = 0x441410', 'continuation_addr = 0x441410'] at debugging-example.cpp:66
-  #15 0x40e0b0 in detail::chain_fn<15>() ['frame_addr = 0x4413b0', 'promise_addr = 0x4413c0', 'continuation_addr = 0x4413c0'] at debugging-example.cpp:66
-  #16 0x40ee40 in detail::chain_fn<16>() ['frame_addr = 0x441360', 'promise_addr = 0x441370', 'continuation_addr = 0x441370'] at debugging-example.cpp:66
-  #17 0x40fbd0 in detail::chain_fn<17>() ['frame_addr = 0x441310', 'promise_addr = 0x441320', 'continuation_addr = 0x441320'] at debugging-example.cpp:66
-  #18 0x410960 in detail::chain_fn<18>() ['frame_addr = 0x4412c0', 'promise_addr = 0x4412d0', 'continuation_addr = 0x4412d0'] at debugging-example.cpp:66
-  #19 0x4116f0 in detail::chain_fn<19>() ['frame_addr = 0x441270', 'promise_addr = 0x441280', 'continuation_addr = 0x441280'] at debugging-example.cpp:66
-  #20 0x412480 in detail::chain_fn<20>() ['frame_addr = 0x441220', 'promise_addr = 0x441230', 'continuation_addr = 0x441230'] at debugging-example.cpp:66
-  #21 0x413210 in detail::chain_fn<21>() ['frame_addr = 0x4411d0', 'promise_addr = 0x4411e0', 'continuation_addr = 0x4411e0'] at debugging-example.cpp:66
-  #22 0x413fa0 in detail::chain_fn<22>() ['frame_addr = 0x441180', 'promise_addr = 0x441190', 'continuation_addr = 0x441190'] at debugging-example.cpp:66
-  #23 0x414d30 in detail::chain_fn<23>() ['frame_addr = 0x441130', 'promise_addr = 0x441140', 'continuation_addr = 0x441140'] at debugging-example.cpp:66
-  #24 0x415ac0 in detail::chain_fn<24>() ['frame_addr = 0x4410e0', 'promise_addr = 0x4410f0', 'continuation_addr = 0x4410f0'] at debugging-example.cpp:66
-  #25 0x416850 in detail::chain_fn<25>() ['frame_addr = 0x441090', 'promise_addr = 0x4410a0', 'continuation_addr = 0x4410a0'] at debugging-example.cpp:66
-  #26 0x4175e0 in detail::chain_fn<26>() ['frame_addr = 0x441040', 'promise_addr = 0x441050', 'continuation_addr = 0x441050'] at debugging-example.cpp:66
-  #27 0x418370 in detail::chain_fn<27>() ['frame_addr = 0x440ff0', 'promise_addr = 0x441000', 'continuation_addr = 0x441000'] at debugging-example.cpp:66
-  #28 0x419100 in detail::chain_fn<28>() ['frame_addr = 0x440fa0', 'promise_addr = 0x440fb0', 'continuation_addr = 0x440fb0'] at debugging-example.cpp:66
-  #29 0x419e90 in detail::chain_fn<29>() ['frame_addr = 0x440f50', 'promise_addr = 0x440f60', 'continuation_addr = 0x440f60'] at debugging-example.cpp:66
-  #30 0x41ac20 in detail::chain_fn<30>() ['frame_addr = 0x440f00', 'promise_addr = 0x440f10', 'continuation_addr = 0x440f10'] at debugging-example.cpp:66
-  #31 0x41b9b0 in chain() ['frame_addr = 0x440eb0', 'promise_addr = 0x440ec0', 'continuation_addr = 0x440ec0'] at debugging-example.cpp:77
-
-Now we get the complete asynchronous stack!
-It is also possible to print other asynchronous stack which doesn't live in the top of the stack.
-We can make it by passing the address of the corresponding coroutine frame to ``async-bt`` command.
-
-By the debugging scripts, we can print any coroutine frame too as long as we know the address.
-For example, we can print the coroutine frame for ``detail::chain_fn<18>()`` in the above example.
-From the log record, we know the address of the coroutine frame is ``0x4412c0`` in the run. Then we can:
-
-.. code-block:: text
-
-  (gdb) show-coro-frame 0x4412c0
-  {
-    __resume_fn = 0x410960 <detail::chain_fn<18>()>,
-    __destroy_fn = 0x410d60 <detail::chain_fn<18>()>,
-    __promise = {
-      continuation = {
-        _M_fr_ptr = 0x441270
-      },
-      result = 0
-    },
-    struct_Awaiter_0 = {
-      struct_std____n4861__coroutine_handle_0 = {
-        struct_std____n4861__coroutine_handle = {
-          PointerType = 0x441310
-        }
-      }
-    },
-    struct_task_1 = {
-      struct_std____n4861__coroutine_handle_0 = {
-        struct_std____n4861__coroutine_handle = {
-          PointerType = 0x0
-        }
-      }
-    },
-    struct_task__promise_type__FinalSuspend_2 = {
-      struct_std____n4861__coroutine_handle = {
-        PointerType = 0x0
-      }
-    },
-    __coro_index = 1 '\001',
-    struct_std____n4861__suspend_always_3 = {
-      __int_8 = 0 '\000'
-    }
-
-
-Get the living coroutines
-=========================
-
-Another useful task when debugging coroutines is to enumerate the list of
-living coroutines, which is often done with threads.  While technically
-possible, this task is not recommended in production code as it is costly at
-runtime. One such solution is to store the list of currently running coroutines
-in a collection:
-
-.. code-block:: c++
+Further Reading
+---------------
 
-  inline std::unordered_set<void*> lived_coroutines;
-  // For all promise_type we want to record
-  class promise_type {
-  public:
-      promise_type() {
-          // Note to avoid data races
-          lived_coroutines.insert(std::coroutine_handle<promise_type>::from_promise(*this).address());
-      }
-      ~promise_type() {
-          // Note to avoid data races
-          lived_coroutines.erase(std::coroutine_handle<promise_type>::from_promise(*this).address());
-      }
-  };
+The authors of the Folly libraries wrote a blog post series on how they debug coroutines:
 
-In the above code snippet, we save the address of every lived coroutine in the
-`lived_coroutines` `unordered_set`. As before, once we know the address of the
-coroutine we can derive the function, `promise_type`, and other members of the
-frame. Thus, we could print the list of lived coroutines from that collection.
+* `Async stack traces in folly: Introduction <https://developers.facebook.com/blog/post/2021/09/16/async-stack-traces-folly-Introduction/>`_
+* `Async stack traces in folly: Synchronous and asynchronous stack traces <https://developers.facebook.com/blog/post/2021/09/23/async-stack-traces-folly-synchronous-asynchronous-stack-traces/>`_
+* `Async stack traces in folly: Forming an async stack from individual frames <https://developers.facebook.com/blog/post/2021/09/30/async-stack-traces-folly-forming-async-stack-individual-frames/>`_
+* `Async Stack Traces for C++ Coroutines in Folly: Walking the async stack <https://developers.facebook.com/blog/post/2021/10/14/async-stack-traces-c-plus-plus-coroutines-folly-walking-async-stack/>`_
+* `Async stack traces in folly: Improving debugging in the developer lifecycle <https://developers.facebook.com/blog/post/2021/10/21/async-stack-traces-folly-improving-debugging-developer-lifecycle/>`_
 
-Please note that the above is expensive from a storage perspective, and requires
-some level of locking (not pictured) on the collection to prevent data races.
+Besides some topics also covered here (stack traces from the debugger), Folly's blog post series also covers
+more additional topics, such as capturing async stack traces in performance profiles via eBPF filters
+and printing async stack traces on crashes.
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 0ba301f438d04..a42a546555716 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -3798,6 +3798,17 @@ Trivially relocates ``count`` objects of relocatable, complete type ``T``
 from ``src`` to ``dest`` and returns ``dest``.
 This builtin is used to implement ``std::trivially_relocate``.
 
+``__builtin_invoke``
+--------------------
+
+**Syntax**:
+
+.. code-block:: c++
+
+  template <class Callee, class... Args>
+  decltype(auto) __builtin_invoke(Callee&& callee, Args&&... args);
+
+``__builtin_invoke`` is equivalent to ``std::invoke``.
 
 ``__builtin_preserve_access_index``
 -----------------------------------
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index d9847fadc21e5..8b4f9229c4463 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -71,6 +71,11 @@ C++ Specific Potentially Breaking Changes
   if it's out of range. The Boost numeric_conversion library is impacted by
   this; it was fixed in Boost 1.81. (#GH143034)
 
+- Fully implemented `CWG400 Using-declarations and the `
+  `"struct hack" <https://wg21.link/CWG400>`_. Invalid member using-declaration
+  whose nested-name-specifier doesn't refer to a base class such as
+  ``using CurrentClass::Foo;`` is now rejected in C++98 mode.
+
 ABI Changes in This Version
 ---------------------------
 
@@ -130,6 +135,8 @@ C++2c Feature Support
 
 - Implemented `P2719R4 Type-aware allocation and deallocation functions <https://wg21.link/P2719>`_.
 
+- Implemented `P3618R0 Allow attaching main to the global module <https://wg21.link/P3618>`_.
+
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -242,6 +249,8 @@ C Language Changes
 - The ``[[clang::assume()]]`` attribute is now correctly recognized in C. The
   ``__attribute__((assume()))`` form has always been supported, so the fix is
   specific to the attribute syntax used.
+- The ``clang-cl`` driver now recognizes ``/std:clatest`` and sets the language
+  mode to C23. (#GH147233)
 
 C2y Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -299,6 +308,8 @@ C23 Feature Support
   which clarified how Clang is handling underspecified object declarations.
 - Clang now accepts single variadic parameter in type-name. It's a part of
   `WG14 N2975 <https://open-std.org/JTC1/SC22/WG14/www/docs/n2975.pdf>`_
+- Fixed a bug with handling the type operand form of ``typeof`` when it is used
+  to specify a fixed underlying type for an enumeration. #GH146351
 
 C11 Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -327,6 +338,7 @@ Non-comprehensive list of changes in this release
   different than before.
 - Fixed a crash when a VLA with an invalid size expression was used within a
   ``sizeof`` or ``typeof`` expression. (#GH138444)
+- ``__builtin_invoke`` has been added to improve the compile time of ``std::invoke``.
 - Deprecation warning is emitted for the deprecated ``__reference_binds_to_temporary`` intrinsic.
   ``__reference_constructs_from_temporary`` should be used instead. (#GH44056)
 - Added `__builtin_get_vtable_pointer` to directly load the primary vtable pointer from a
@@ -649,6 +661,13 @@ Improvements to Clang's diagnostics
   #GH69470, #GH59391, #GH58172, #GH46215, #GH45915, #GH45891, #GH44490,
   #GH36703, #GH32903, #GH23312, #GH69874.
 
+- Fixed false positives in ``-Wformat-truncation`` and ``-Wformat-overflow``
+  diagnostics when floating-point numbers had both width field and plus or space
+  prefix specified. (#GH143951)
+
+- A warning is now emitted when ``main`` is attached to a named module,
+  which can be turned off with ``-Wno-main-attached-to-named-module``. (#GH146247)
+
 - Clang now avoids issuing `-Wreturn-type` warnings in some cases where
   the final statement of a non-void function is a `throw` expression, or
   a call to a function that is trivially known to always throw (i.e., its
@@ -656,7 +675,15 @@ Improvements to Clang's diagnostics
   false positives in exception-heavy code, though only simple patterns
   are currently recognized.
 
-  
+- Clang now accepts ``@tparam`` comments on variable template partial
+  specializations. (#GH144775)
+
+- Fixed a bug that caused diagnostic line wrapping to not function correctly on
+  some systems. (#GH139499)
+
+- Clang now tries to avoid printing file paths that contain ``..``, instead preferring
+  the canonical file path if it ends up being shorter.
+
 Improvements to Clang's time-trace
 ----------------------------------
 
@@ -734,7 +761,12 @@ Bug Fixes in This Version
 - Fixed incorrect token location when emitting diagnostics for tokens expanded from macros. (#GH143216)
 - Fixed an infinite recursion when checking constexpr destructors. (#GH141789)
 - Fixed a crash when a malformed using declaration appears in a ``constexpr`` function. (#GH144264)
-- Fixed a bug when use unicode character name in macro concatenation. (#GH145240) 
+- Fixed a bug when use unicode character name in macro concatenation. (#GH145240)
+- Clang doesn't erroneously inject a ``static_assert`` macro in ms-compatibility and
+  -std=c99 mode. This resulted in deletion of ``-W/Wno-microsoft-static-assert``
+  flag and diagnostic because the macro injection was used to emit this warning.
+  Unfortunately there is no other good way to diagnose usage of ``static_assert``
+  macro without inclusion of ``<assert.h>``.
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -872,6 +904,7 @@ Bug Fixes to C++ Support
 - Clang now correctly parses arbitrary order of ``[[]]``, ``__attribute__`` and ``alignas`` attributes for declarations (#GH133107)
 - Fixed a crash when forming an invalid function type in a dependent context. (#GH138657) (#GH115725) (#GH68852)
 - Fixed a function declaration mismatch that caused inconsistencies between concepts and variable template declarations. (#GH139476)
+- Fixed an out-of-line declaration mismatch involving nested template parameters. (#GH145521)
 - Clang no longer segfaults when there is a configuration mismatch between modules and their users (http://crbug.com/400353616).
 - Fix an incorrect deduction when calling an explicit object member function template through an overload set address.
 - Fixed bug in constant evaluation that would allow using the value of a
@@ -884,6 +917,12 @@ Bug Fixes to C++ Support
 - Fixed a crash when constant evaluating some explicit object member assignment operators. (#GH142835)
 - Fixed an access checking bug when substituting into concepts (#GH115838)
 - Fix a bug where private access specifier of overloaded function not respected. (#GH107629)
+- Correctly handle allocations in the condition of a ``if constexpr``.(#GH120197) (#GH134820)
+- Fixed a crash when handling invalid member using-declaration in C++20+ mode. (#GH63254)
+- Fix a crash when trying to instantiate an ambiguous specialization. (#GH51866)
+- Improved handling of variables with ``consteval`` constructors, to
+  consistently treat the initializer as manifestly constant-evaluated.
+  (#GH135281)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -892,7 +931,9 @@ Bug Fixes to AST Handling
 - Fixed a malformed printout of ``CXXParenListInitExpr`` in certain contexts.
 - Fixed a malformed printout of certain calling convention function attributes. (#GH143160)
 - Fixed dependency calculation for TypedefTypes (#GH89774)
+- The ODR checker now correctly hashes the names of conversion operators. (#GH143152)
 - Fixed the right parenthesis source location of ``CXXTemporaryObjectExpr``. (#GH143711)
+- Fixed a crash when performing an ``IgnoreUnlessSpelledInSource`` traversal of ASTs containing ``catch(...)`` statements. (#GH146103)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -955,6 +996,8 @@ Arm and AArch64 Support
 
 - For AArch64, added support for generating executable-only code sections by using the
   ``-mexecute-only`` or ``-mpure-code`` compiler flags. (#GH125688)
+- Added ``-msve-streaming-vector-bits=`` flag, which allows specifying the
+  SVE vector width in streaming mode.
 
 Android Support
 ^^^^^^^^^^^^^^^
@@ -973,6 +1016,23 @@ Windows Support
 LoongArch Support
 ^^^^^^^^^^^^^^^^^
 
+- Add support for OHOS on loongarch64.
+
+- Add target attribute support for function. Supported formats include:
+  * `arch=<arch>` strings - specifies architecture features for a function (equivalent to `-march=<arch>`).
+  * `tune=<cpu>` strings - specifies the tune CPU for a function (equivalent to `-mtune`).
+  * `<feature>`/`no-<feature>` - enables/disables specific features.
+
+- Add support for the `_Float16` type. And fix incorrect ABI lowering of `_Float16`
+  in the case of structs containing fp16 that are eligible for passing via `GPR+FPR`
+  or `FPR+FPR`. Also fix `int16` -> `__fp16` conversion code gen, which uses generic LLVM
+  IR rather than `llvm.convert.to.fp16` intrinsics.
+
+- Add support for the `__bf16` type.
+
+- Fix incorrect _BitInt(N>64) alignment. Now consistently uses 16-byte alignment for all
+  `_BitInt(N)` where N > 64.
+
 RISC-V Support
 ^^^^^^^^^^^^^^
 
diff --git a/clang/docs/Toolchain.rst b/clang/docs/Toolchain.rst
index 958199eb7a2e2..d56b21d74c7e3 100644
--- a/clang/docs/Toolchain.rst
+++ b/clang/docs/Toolchain.rst
@@ -347,3 +347,8 @@ workarounds for issues discovered in libstdc++, and these are removed
 as fixed libstdc++ becomes sufficiently old.
 
 You can instruct Clang to use libstdc++ with the ``-stdlib=libstdc++`` flag.
+
+GCC Installation
+=================
+Users can point to their GCC installation by using the ``-gcc-toolchain`` or by
+using ``-gcc-install-dir`` flag.
diff --git a/clang/docs/coro-async-task-continuations.png b/clang/docs/coro-async-task-continuations.png
new file mode 100644
index 0000000000000..0a8894d8fdb1d
Binary files /dev/null and b/clang/docs/coro-async-task-continuations.png differ
diff --git a/clang/docs/coro-generator-suspended.png b/clang/docs/coro-generator-suspended.png
new file mode 100644
index 0000000000000..278929f3e145d
Binary files /dev/null and b/clang/docs/coro-generator-suspended.png differ
diff --git a/clang/docs/coro-generator-variables.png b/clang/docs/coro-generator-variables.png
new file mode 100644
index 0000000000000..53a8c9f4d509b
Binary files /dev/null and b/clang/docs/coro-generator-variables.png differ
diff --git a/clang/include/clang-c/FatalErrorHandler.h b/clang/include/clang-c/FatalErrorHandler.h
index 22f34fa815ccf..4f18980dea240 100644
--- a/clang/include/clang-c/FatalErrorHandler.h
+++ b/clang/include/clang-c/FatalErrorHandler.h
@@ -11,6 +11,7 @@
 #define LLVM_CLANG_C_FATAL_ERROR_HANDLER_H
 
 #include "clang-c/ExternC.h"
+#include "clang-c/Platform.h"
 
 LLVM_CLANG_C_EXTERN_C_BEGIN
 
@@ -18,14 +19,14 @@ LLVM_CLANG_C_EXTERN_C_BEGIN
  * Installs error handler that prints error message to stderr and calls abort().
  * Replaces currently installed error handler (if any).
  */
-void clang_install_aborting_llvm_fatal_error_handler(void);
+CINDEX_LINKAGE void clang_install_aborting_llvm_fatal_error_handler(void);
 
 /**
  * Removes currently installed error handler (if any).
  * If no error handler is intalled, the default strategy is to print error
  * message to stderr and call exit(1).
  */
-void clang_uninstall_llvm_fatal_error_handler(void);
+CINDEX_LINKAGE void clang_uninstall_llvm_fatal_error_handler(void);
 
 LLVM_CLANG_C_EXTERN_C_END
 
diff --git a/clang/include/clang-c/Platform.h b/clang/include/clang-c/Platform.h
index 67c1fff8ff783..8d341ddd6f8eb 100644
--- a/clang/include/clang-c/Platform.h
+++ b/clang/include/clang-c/Platform.h
@@ -22,7 +22,7 @@ LLVM_CLANG_C_EXTERN_C_BEGIN
 #ifndef CINDEX_NO_EXPORTS
   #define CINDEX_EXPORTS
 #endif
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__CYGWIN__)
   #ifdef CINDEX_EXPORTS
     #ifdef _CINDEX_LIB_
       #define CINDEX_LINKAGE __declspec(dllexport)
diff --git a/clang/include/clang/AST/ASTNodeTraverser.h b/clang/include/clang/AST/ASTNodeTraverser.h
index 8d02a50e2e8a5..8ebabb2bde10d 100644
--- a/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/clang/include/clang/AST/ASTNodeTraverser.h
@@ -99,7 +99,7 @@ class ASTNodeTraverser
   TraversalKind GetTraversalKind() const { return Traversal; }
 
   void Visit(const Decl *D, bool VisitLocs = false) {
-    if (Traversal == TK_IgnoreUnlessSpelledInSource && D->isImplicit())
+    if (Traversal == TK_IgnoreUnlessSpelledInSource && D && D->isImplicit())
       return;
 
     getNodeDelegate().AddChild([=] {
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index c4202f1f3d07e..f70a039bf3517 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -888,13 +888,17 @@ struct EvaluatedStmt {
   bool HasICEInit : 1;
   bool CheckedForICEInit : 1;
 
+  bool HasSideEffects : 1;
+  bool CheckedForSideEffects : 1;
+
   LazyDeclStmtPtr Value;
   APValue Evaluated;
 
   EvaluatedStmt()
       : WasEvaluated(false), IsEvaluating(false),
         HasConstantInitialization(false), HasConstantDestruction(false),
-        HasICEInit(false), CheckedForICEInit(false) {}
+        HasICEInit(false), CheckedForICEInit(false), HasSideEffects(false),
+        CheckedForSideEffects(false) {}
 };
 
 /// Represents a variable declaration or definition.
@@ -1353,9 +1357,11 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
     return const_cast<VarDecl *>(this)->getInitializingDeclaration();
   }
 
-  /// Checks whether this declaration has an initializer with side effects,
-  /// without triggering deserialization if the initializer is not yet
-  /// deserialized.
+  /// Checks whether this declaration has an initializer with side effects.
+  /// The result is cached. If the result hasn't been computed this can trigger
+  /// deserialization and constant evaluation. By running this during
+  /// serialization and serializing the result all clients can safely call this
+  /// without triggering further deserialization.
   bool hasInitWithSideEffects() const;
 
   /// Determine whether this variable's value might be usable in a
diff --git a/clang/include/clang/AST/DeclarationName.h b/clang/include/clang/AST/DeclarationName.h
index 9bf740b3bf7ce..284228dc0ee47 100644
--- a/clang/include/clang/AST/DeclarationName.h
+++ b/clang/include/clang/AST/DeclarationName.h
@@ -698,13 +698,13 @@ class DeclarationNameLoc {
 
   // The location (if any) of the operator keyword is stored elsewhere.
   struct CXXOpName {
-    SourceLocation::UIntTy BeginOpNameLoc;
-    SourceLocation::UIntTy EndOpNameLoc;
+    SourceLocation BeginOpNameLoc;
+    SourceLocation EndOpNameLoc;
   };
 
   // The location (if any) of the operator keyword is stored elsewhere.
   struct CXXLitOpName {
-    SourceLocation::UIntTy OpNameLoc;
+    SourceLocation OpNameLoc;
   };
 
   // struct {} CXXUsingDirective;
@@ -720,12 +720,12 @@ class DeclarationNameLoc {
   void setNamedTypeLoc(TypeSourceInfo *TInfo) { NamedType.TInfo = TInfo; }
 
   void setCXXOperatorNameRange(SourceRange Range) {
-    CXXOperatorName.BeginOpNameLoc = Range.getBegin().getRawEncoding();
-    CXXOperatorName.EndOpNameLoc = Range.getEnd().getRawEncoding();
+    CXXOperatorName.BeginOpNameLoc = Range.getBegin();
+    CXXOperatorName.EndOpNameLoc = Range.getEnd();
   }
 
   void setCXXLiteralOperatorNameLoc(SourceLocation Loc) {
-    CXXLiteralOperatorName.OpNameLoc = Loc.getRawEncoding();
+    CXXLiteralOperatorName.OpNameLoc = Loc;
   }
 
 public:
@@ -739,12 +739,12 @@ class DeclarationNameLoc {
 
   /// Return the beginning location of the getCXXOperatorNameRange() range.
   SourceLocation getCXXOperatorNameBeginLoc() const {
-    return SourceLocation::getFromRawEncoding(CXXOperatorName.BeginOpNameLoc);
+    return CXXOperatorName.BeginOpNameLoc;
   }
 
   /// Return the end location of the getCXXOperatorNameRange() range.
   SourceLocation getCXXOperatorNameEndLoc() const {
-    return SourceLocation::getFromRawEncoding(CXXOperatorName.EndOpNameLoc);
+    return CXXOperatorName.EndOpNameLoc;
   }
 
   /// Return the range of the operator name (without the operator keyword).
@@ -759,7 +759,7 @@ class DeclarationNameLoc {
   /// keyword). Assumes that the object stores location information of a literal
   /// operator.
   SourceLocation getCXXLiteralOperatorNameLoc() const {
-    return SourceLocation::getFromRawEncoding(CXXLiteralOperatorName.OpNameLoc);
+    return CXXLiteralOperatorName.OpNameLoc;
   }
 
   /// Construct location information for a constructor, destructor or conversion
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index d95396fd59b95..523c0326d47ef 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -3534,6 +3534,10 @@ class CompoundLiteralExpr : public Expr {
   /// The int part of the pair stores whether this expr is file scope.
   llvm::PointerIntPair<TypeSourceInfo *, 1, bool> TInfoAndScope;
   Stmt *Init;
+
+  /// Value of constant literals with static storage duration.
+  mutable APValue *StaticValue = nullptr;
+
 public:
   CompoundLiteralExpr(SourceLocation lparenloc, TypeSourceInfo *tinfo,
                       QualType T, ExprValueKind VK, Expr *init, bool fileScope)
@@ -3563,6 +3567,10 @@ class CompoundLiteralExpr : public Expr {
     TInfoAndScope.setPointer(tinfo);
   }
 
+  bool hasStaticStorage() const { return isFileScope() && isGLValue(); }
+  APValue &getOrCreateStaticValue(ASTContext &Ctx) const;
+  APValue &getStaticValue() const;
+
   SourceLocation getBeginLoc() const LLVM_READONLY {
     // FIXME: Init should never be null.
     if (!Init)
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 477373f07f25d..a22c32241ac61 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -84,7 +84,7 @@ class CXXOperatorCallExpr final : public CallExpr {
   friend class ASTStmtReader;
   friend class ASTStmtWriter;
 
-  SourceRange Range;
+  SourceLocation BeginLoc;
 
   // CXXOperatorCallExpr has some trailing objects belonging
   // to CallExpr. See CallExpr for the details.
@@ -158,9 +158,9 @@ class CXXOperatorCallExpr final : public CallExpr {
                : getOperatorLoc();
   }
 
-  SourceLocation getBeginLoc() const { return Range.getBegin(); }
-  SourceLocation getEndLoc() const { return Range.getEnd(); }
-  SourceRange getSourceRange() const { return Range; }
+  SourceLocation getBeginLoc() const { return BeginLoc; }
+  SourceLocation getEndLoc() const { return getSourceRangeImpl().getEnd(); }
+  SourceRange getSourceRange() const { return getSourceRangeImpl(); }
 
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == CXXOperatorCallExprClass;
diff --git a/clang/include/clang/AST/ExternalASTSource.h b/clang/include/clang/AST/ExternalASTSource.h
index e91d5132da10f..8fc490b1d8471 100644
--- a/clang/include/clang/AST/ExternalASTSource.h
+++ b/clang/include/clang/AST/ExternalASTSource.h
@@ -196,10 +196,6 @@ class ExternalASTSource : public RefCountedBase<ExternalASTSource> {
   /// module.
   virtual bool wasThisDeclarationADefinition(const FunctionDecl *FD);
 
-  virtual bool hasInitializerWithSideEffects(const VarDecl *VD) const {
-    return false;
-  }
-
   /// Finds all declarations lexically contained within the given
   /// DeclContext, after applying an optional filter predicate.
   ///
@@ -434,17 +430,6 @@ struct LazyOffsetPtr {
     return GetPtr();
   }
 
-  /// Retrieve the pointer to the AST node that this lazy pointer points to,
-  /// if it can be done without triggering deserialization.
-  ///
-  /// \returns a pointer to the AST node, or null if not yet deserialized.
-  T *getWithoutDeserializing() const {
-    if (isOffset()) {
-      return nullptr;
-    }
-    return GetPtr();
-  }
-
   /// Retrieve the address of the AST node pointer. Deserializes the pointee if
   /// necessary.
   T **getAddressOfPointer(ExternalASTSource *Source) const {
diff --git a/clang/include/clang/AST/ODRHash.h b/clang/include/clang/AST/ODRHash.h
index a923901b32dc0..11f917a1a4634 100644
--- a/clang/include/clang/AST/ODRHash.h
+++ b/clang/include/clang/AST/ODRHash.h
@@ -96,7 +96,13 @@ class ODRHash {
   void AddNestedNameSpecifier(const NestedNameSpecifier *NNS);
   void AddDependentTemplateName(const DependentTemplateStorage &Name);
   void AddTemplateName(TemplateName Name);
-  void AddDeclarationName(DeclarationName Name, bool TreatAsDecl = false);
+  void AddDeclarationNameInfo(DeclarationNameInfo NameInfo,
+                              bool TreatAsDecl = false);
+  void AddDeclarationName(DeclarationName Name, bool TreatAsDecl = false) {
+    AddDeclarationNameInfo(DeclarationNameInfo(Name, SourceLocation()),
+                           TreatAsDecl);
+  }
+
   void AddTemplateArgument(TemplateArgument TA);
   void AddTemplateParameterList(const TemplateParameterList *TPL);
 
@@ -108,7 +114,7 @@ class ODRHash {
   static bool isSubDeclToBeProcessed(const Decl *D, const DeclContext *Parent);
 
 private:
-  void AddDeclarationNameImpl(DeclarationName Name);
+  void AddDeclarationNameInfoImpl(DeclarationNameInfo NameInfo);
 };
 
 }  // end namespace clang
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index c6f99fb21a0f0..5b2206af75bee 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -295,7 +295,8 @@ template <class T> class OMPVarListClause : public OMPClause {
 
   /// Fetches list of variables associated with this clause.
   MutableArrayRef<Expr *> getVarRefs() {
-    return static_cast<T *>(this)->template getTrailingObjects<Expr *>(NumVars);
+    return static_cast<T *>(this)->template getTrailingObjectsNonStrict<Expr *>(
+        NumVars);
   }
 
   /// Sets the list of variables for this clause.
@@ -334,8 +335,8 @@ template <class T> class OMPVarListClause : public OMPClause {
 
   /// Fetches list of all variables in the clause.
   ArrayRef<const Expr *> getVarRefs() const {
-    return static_cast<const T *>(this)->template getTrailingObjects<Expr *>(
-        NumVars);
+    return static_cast<const T *>(this)
+        ->template getTrailingObjectsNonStrict<Expr *>(NumVars);
   }
 };
 
@@ -380,7 +381,7 @@ template <class T> class OMPDirectiveListClause : public OMPClause {
 
   MutableArrayRef<OpenMPDirectiveKind> getDirectiveKinds() {
     return static_cast<T *>(this)
-        ->template getTrailingObjects<OpenMPDirectiveKind>(NumKinds);
+        ->template getTrailingObjectsNonStrict<OpenMPDirectiveKind>(NumKinds);
   }
 
   void setDirectiveKinds(ArrayRef<OpenMPDirectiveKind> DK) {
@@ -5921,15 +5922,17 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// Get the unique declarations that are in the trailing objects of the
   /// class.
   MutableArrayRef<ValueDecl *> getUniqueDeclsRef() {
-    return static_cast<T *>(this)->template getTrailingObjects<ValueDecl *>(
-        NumUniqueDeclarations);
+    return static_cast<T *>(this)
+        ->template getTrailingObjectsNonStrict<ValueDecl *>(
+            NumUniqueDeclarations);
   }
 
   /// Get the unique declarations that are in the trailing objects of the
   /// class.
   ArrayRef<ValueDecl *> getUniqueDeclsRef() const {
     return static_cast<const T *>(this)
-        ->template getTrailingObjects<ValueDecl *>(NumUniqueDeclarations);
+        ->template getTrailingObjectsNonStrict<ValueDecl *>(
+            NumUniqueDeclarations);
   }
 
   /// Set the unique declarations that are in the trailing objects of the
@@ -5943,15 +5946,15 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// Get the number of lists per declaration that are in the trailing
   /// objects of the class.
   MutableArrayRef<unsigned> getDeclNumListsRef() {
-    return static_cast<T *>(this)->template getTrailingObjects<unsigned>(
-        NumUniqueDeclarations);
+    return static_cast<T *>(this)
+        ->template getTrailingObjectsNonStrict<unsigned>(NumUniqueDeclarations);
   }
 
   /// Get the number of lists per declaration that are in the trailing
   /// objects of the class.
   ArrayRef<unsigned> getDeclNumListsRef() const {
-    return static_cast<const T *>(this)->template getTrailingObjects<unsigned>(
-        NumUniqueDeclarations);
+    return static_cast<const T *>(this)
+        ->template getTrailingObjectsNonStrict<unsigned>(NumUniqueDeclarations);
   }
 
   /// Set the number of lists per declaration that are in the trailing
@@ -5966,7 +5969,8 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// objects of the class. They are appended after the number of lists.
   MutableArrayRef<unsigned> getComponentListSizesRef() {
     return MutableArrayRef<unsigned>(
-        static_cast<T *>(this)->template getTrailingObjects<unsigned>() +
+        static_cast<T *>(this)
+                ->template getTrailingObjectsNonStrict<unsigned>() +
             NumUniqueDeclarations,
         NumComponentLists);
   }
@@ -5975,7 +5979,8 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// objects of the class. They are appended after the number of lists.
   ArrayRef<unsigned> getComponentListSizesRef() const {
     return ArrayRef<unsigned>(
-        static_cast<const T *>(this)->template getTrailingObjects<unsigned>() +
+        static_cast<const T *>(this)
+                ->template getTrailingObjectsNonStrict<unsigned>() +
             NumUniqueDeclarations,
         NumComponentLists);
   }
@@ -5991,13 +5996,15 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// Get the components that are in the trailing objects of the class.
   MutableArrayRef<MappableComponent> getComponentsRef() {
     return static_cast<T *>(this)
-        ->template getTrailingObjects<MappableComponent>(NumComponents);
+        ->template getTrailingObjectsNonStrict<MappableComponent>(
+            NumComponents);
   }
 
   /// Get the components that are in the trailing objects of the class.
   ArrayRef<MappableComponent> getComponentsRef() const {
     return static_cast<const T *>(this)
-        ->template getTrailingObjects<MappableComponent>(NumComponents);
+        ->template getTrailingObjectsNonStrict<MappableComponent>(
+            NumComponents);
   }
 
   /// Set the components that are in the trailing objects of the class.
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 1a87608e5be34..21b97102db95a 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -6404,7 +6404,7 @@ class SpirvOperand {
   SpirvOperand() : Kind(Invalid), ResultType(), Value() {}
 
   SpirvOperand(SpirvOperandKind Kind, QualType ResultType, llvm::APInt Value)
-      : Kind(Kind), ResultType(ResultType), Value(Value) {}
+      : Kind(Kind), ResultType(ResultType), Value(std::move(Value)) {}
 
   SpirvOperand(const SpirvOperand &Other) { *this = Other; }
   ~SpirvOperand() {}
@@ -6438,11 +6438,11 @@ class SpirvOperand {
   }
 
   static SpirvOperand createConstant(QualType ResultType, llvm::APInt Val) {
-    return SpirvOperand(ConstantId, ResultType, Val);
+    return SpirvOperand(ConstantId, ResultType, std::move(Val));
   }
 
   static SpirvOperand createLiteral(llvm::APInt Val) {
-    return SpirvOperand(Literal, QualType(), Val);
+    return SpirvOperand(Literal, QualType(), std::move(Val));
   }
 
   static SpirvOperand createType(QualType T) {
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index e4d605d165324..bae004896c11f 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6519,11 +6519,11 @@ AST_POLYMORPHIC_MATCHER(
 /// Given
 /// \code
 ///   void a(int);
-///   void b(long);
+///   void b(unsigned long);
 ///   void c(double);
 /// \endcode
 /// functionDecl(hasAnyParameter(hasType(isInteger())))
-/// matches "a(int)", "b(long)", but not "c(double)".
+/// matches "a(int)", "b(unsigned long)", but not "c(double)".
 AST_MATCHER(QualType, isInteger) {
     return Node->isIntegerType();
 }
diff --git a/clang/include/clang/Analysis/CFG.h b/clang/include/clang/Analysis/CFG.h
index e70c70335e597..1b1ff5e558ec5 100644
--- a/clang/include/clang/Analysis/CFG.h
+++ b/clang/include/clang/Analysis/CFG.h
@@ -1394,10 +1394,9 @@ class CFG {
   //===--------------------------------------------------------------------===//
 
   template <typename Callback> void VisitBlockStmts(Callback &O) const {
-    for (const_iterator I = begin(), E = end(); I != E; ++I)
-      for (CFGBlock::const_iterator BI = (*I)->begin(), BE = (*I)->end();
-           BI != BE; ++BI) {
-        if (std::optional<CFGStmt> stmt = BI->getAs<CFGStmt>())
+    for (CFGBlock *BB : *this)
+      for (const CFGElement &Elem : *BB) {
+        if (std::optional<CFGStmt> stmt = Elem.getAs<CFGStmt>())
           O(const_cast<Stmt *>(stmt->getStmt()));
       }
   }
diff --git a/clang/include/clang/Basic/AttributeCommonInfo.h b/clang/include/clang/Basic/AttributeCommonInfo.h
index 21a7a88a3fb98..77b5eb8a1a7cc 100644
--- a/clang/include/clang/Basic/AttributeCommonInfo.h
+++ b/clang/include/clang/Basic/AttributeCommonInfo.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_BASIC_ATTRIBUTECOMMONINFO_H
 
 #include "clang/Basic/AttributeScopeInfo.h"
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TokenKinds.h"
 
@@ -175,6 +176,10 @@ class AttributeCommonInfo {
       : AttributeCommonInfo(nullptr, AttributeScopeInfo(), AttrRange, K,
                             FormUsed) {}
 
+  AttributeCommonInfo(SourceRange AttrRange, AttributeScopeInfo AttrScope,
+                      Kind K, Form FormUsed)
+      : AttributeCommonInfo(nullptr, AttrScope, AttrRange, K, FormUsed) {}
+
   AttributeCommonInfo(AttributeCommonInfo &&) = default;
   AttributeCommonInfo(const AttributeCommonInfo &) = default;
 
@@ -292,6 +297,18 @@ inline bool doesKeywordAttributeTakeArgs(tok::TokenKind Kind) {
   }
 }
 
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
+                                             const AttributeCommonInfo *CI) {
+  DB.AddTaggedVal(reinterpret_cast<uint64_t>(CI),
+                  DiagnosticsEngine::ak_attr_info);
+  return DB;
+}
+
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
+                                             const AttributeCommonInfo &CI) {
+  return DB << &CI;
+}
+
 } // namespace clang
 
 #endif // LLVM_CLANG_BASIC_ATTRIBUTECOMMONINFO_H
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 24810292d1d55..5ebb82180521d 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4314,6 +4314,12 @@ def MoveIfNsoexcept : CxxLibBuiltin<"utility"> {
   let Namespace = "std";
 }
 
+def Invoke : Builtin {
+  let Spellings = ["__builtin_invoke"];
+  let Attributes = [CustomTypeChecking, Constexpr];
+  let Prototype = "void(...)";
+}
+
 def Annotation : Builtin {
   let Spellings = ["__builtin_annotation"];
   let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 909e35792b461..b2aa36acd4506 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -274,6 +274,7 @@ TARGET_HEADER_BUILTIN(_ReadWriteBarrier, "v", "nh", INTRIN_H, ALL_MS_LANGUAGES,
 TARGET_HEADER_BUILTIN(__getReg, "ULLii", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_ReadStatusReg,  "LLii",  "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_WriteStatusReg, "viLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sys, "UiiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_AddressOfReturnAddress, "v*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(__mulh,  "SLLiSLLiSLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 1d1f5a4ee3f9f..a5ee8013adff6 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -429,6 +429,7 @@ TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_32x32x32_fp8_fp8, "V16fV2iV4iV16fiIiI
 
 TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_bf8, "fiIi", "nc", "fp8-conversion-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_fp8, "fiIi", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_fp8_e5m3, "fiIi", "nc", "fp8e5m3-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f32_bf8, "V2fiIb", "nc", "fp8-conversion-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f32_fp8, "V2fiIb", "nc", "fp8-conversion-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_bf8_f32, "iffiIb", "nc", "fp8-conversion-insts")
@@ -640,8 +641,32 @@ TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_f16_f32, "V2hV2hfUiIb", "nc", "f32-to-f16
 // GFX1250+ only builtins.
 //===----------------------------------------------------------------------===//
 
+TARGET_BUILTIN(__builtin_amdgcn_tensor_load_to_lds, "vV4iV8iV4iV4iIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_tensor_load_to_lds_d2, "vV4iV8iIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_tensor_store_from_lds, "vV4iV8iV4iV4iIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_tensor_store_from_lds_d2, "vV4iV8iIi", "nc", "gfx1250-insts")
+
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr4_b64_v2i32, "V2iV2i*1", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr8_b64_v2i32, "V2iV2i*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr6_b96_v3i32, "V3iV3i*1", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr16_b128_v8i16, "V8sV8s*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr16_b128_v8f16, "V8hV8h*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr16_b128_v8bf16, "V8yV8y*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr4_b64_v2i32, "V2iV2i*3", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr8_b64_v2i32, "V2iV2i*3", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr6_b96_v3i32, "V3iV3i*3", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr16_b128_v8i16, "V8sV8s*3", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr16_b128_v8f16, "V8hV8h*3", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr16_b128_v8bf16, "V8yV8y*3", "nc", "gfx1250-insts,wavefrontsize32")
+
+TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_async_barrier_arrive_b64, "vLi*3", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64, "LiLi*3Li", "nc", "gfx1250-insts")
+
 TARGET_BUILTIN(__builtin_amdgcn_s_setprio_inc_wg, "vIs", "n", "setprio-inc-wg-inst")
+TARGET_BUILTIN(__builtin_amdgcn_s_monitor_sleep,  "vIs", "n", "gfx1250-insts")
 
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f16_fp8, "hiIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f16_bf8, "hiIi", "nc", "gfx1250-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f16_fp8, "V2hs", "nc", "gfx1250-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f16_bf8, "V2hs", "nc", "gfx1250-insts")
 
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index e5566a540dc65..41031026c99f6 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -12,481 +12,473 @@
 // that have enumeration type and VALUE_CODEGENOPT is a code
 // generation option that describes a value rather than a flag.
 //
-// AFFECTING_VALUE_CODEGENOPT is used for code generation options that can
-// affect the AST.
-//
 //===----------------------------------------------------------------------===//
 #ifndef CODEGENOPT
-#  error Define the CODEGENOPT macro to handle language options
+#  error Define the CODEGENOPT macro to handle codegen options
 #endif
 
 #ifndef VALUE_CODEGENOPT
-#  define VALUE_CODEGENOPT(Name, Bits, Default) \
-CODEGENOPT(Name, Bits, Default)
+#  define VALUE_CODEGENOPT(Name, Bits, Default, Compatibility) \
+CODEGENOPT(Name, Bits, Default, Compatibility)
 #endif
 
 #ifndef ENUM_CODEGENOPT
-#  define ENUM_CODEGENOPT(Name, Type, Bits, Default) \
-CODEGENOPT(Name, Bits, Default)
-#endif
-
-#ifndef AFFECTING_VALUE_CODEGENOPT
-#  define AFFECTING_VALUE_CODEGENOPT(Name, Bits, Default) \
-VALUE_CODEGENOPT(Name, Bits, Default)
+#  define ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility) \
+CODEGENOPT(Name, Bits, Default, Compatibility)
 #endif
 
-CODEGENOPT(DisableIntegratedAS, 1, 0) ///< -no-integrated-as
-CODEGENOPT(Crel, 1, 0) ///< -Wa,--crel
-CODEGENOPT(ImplicitMapSyms, 1, 0) ///< -Wa,-mmapsyms=implicit
-CODEGENOPT(AsmVerbose        , 1, 0) ///< -dA, -fverbose-asm.
-CODEGENOPT(PreserveAsmComments, 1, 1) ///< -dA, -fno-preserve-as-comments.
-CODEGENOPT(AssumeSaneOperatorNew , 1, 1) ///< implicit __attribute__((malloc)) operator new
-CODEGENOPT(AssumeUniqueVTables , 1, 1) ///< Assume a class has only one vtable.
-CODEGENOPT(Autolink          , 1, 1) ///< -fno-autolink
-CODEGENOPT(AutoImport        , 1, 1) ///< -fno-auto-import
-CODEGENOPT(ObjCAutoRefCountExceptions , 1, 0) ///< Whether ARC should be EH-safe.
-CODEGENOPT(Backchain         , 1, 0) ///< -mbackchain
-CODEGENOPT(ControlFlowGuardNoChecks  , 1, 0) ///< -cfguard-no-checks
-CODEGENOPT(ControlFlowGuard  , 1, 0) ///< -cfguard
-CODEGENOPT(EHContGuard       , 1, 0) ///< -ehcontguard
-CODEGENOPT(CXAAtExit         , 1, 1) ///< Use __cxa_atexit for calling destructors.
-CODEGENOPT(RegisterGlobalDtorsWithAtExit, 1, 1) ///< Use atexit or __cxa_atexit to register global destructors.
-CODEGENOPT(CXXCtorDtorAliases, 1, 0) ///< Emit complete ctors/dtors as linker
+CODEGENOPT(DisableIntegratedAS, 1, 0, Benign) ///< -no-integrated-as
+CODEGENOPT(Crel, 1, 0, Benign) ///< -Wa,--crel
+CODEGENOPT(ImplicitMapSyms, 1, 0, Benign) ///< -Wa,-mmapsyms=implicit
+CODEGENOPT(AsmVerbose        , 1, 0, Benign) ///< -dA, -fverbose-asm.
+CODEGENOPT(PreserveAsmComments, 1, 1, Benign) ///< -dA, -fno-preserve-as-comments.
+CODEGENOPT(AssumeSaneOperatorNew , 1, 1, Benign) ///< implicit __attribute__((malloc)) operator new
+CODEGENOPT(AssumeUniqueVTables , 1, 1, Benign) ///< Assume a class has only one vtable.
+CODEGENOPT(Autolink          , 1, 1, Benign) ///< -fno-autolink
+CODEGENOPT(AutoImport        , 1, 1, Benign) ///< -fno-auto-import
+CODEGENOPT(ObjCAutoRefCountExceptions , 1, 0, Benign) ///< Whether ARC should be EH-safe.
+CODEGENOPT(Backchain         , 1, 0, Benign) ///< -mbackchain
+CODEGENOPT(ControlFlowGuardNoChecks  , 1, 0, Benign) ///< -cfguard-no-checks
+CODEGENOPT(ControlFlowGuard  , 1, 0, Benign) ///< -cfguard
+CODEGENOPT(EHContGuard       , 1, 0, Benign) ///< -ehcontguard
+CODEGENOPT(CXAAtExit         , 1, 1, Benign) ///< Use __cxa_atexit for calling destructors.
+CODEGENOPT(RegisterGlobalDtorsWithAtExit, 1, 1, Benign) ///< Use atexit or __cxa_atexit to register global destructors.
+CODEGENOPT(CXXCtorDtorAliases, 1, 0, Benign) ///< Emit complete ctors/dtors as linker
                                      ///< aliases to base ctors when possible.
-CODEGENOPT(DataSections      , 1, 0) ///< Set when -fdata-sections is enabled.
-CODEGENOPT(UniqueSectionNames, 1, 1) ///< Set for -funique-section-names.
-CODEGENOPT(UniqueBasicBlockSectionNames, 1, 1) ///< Set for -funique-basic-block-section-names,
+CODEGENOPT(DataSections      , 1, 0, Benign) ///< Set when -fdata-sections is enabled.
+CODEGENOPT(UniqueSectionNames, 1, 1, Benign) ///< Set for -funique-section-names.
+CODEGENOPT(UniqueBasicBlockSectionNames, 1, 1, Benign) ///< Set for -funique-basic-block-section-names,
                                                ///< Produce unique section names with
                                                ///< basic block sections.
-CODEGENOPT(SeparateNamedSections, 1, 0) ///< Set for -fseparate-named-sections.
-CODEGENOPT(EnableAIXExtendedAltivecABI, 1, 0) ///< Set for -mabi=vec-extabi. Enables the extended Altivec ABI on AIX.
-CODEGENOPT(XCOFFReadOnlyPointers, 1, 0) ///< Set for -mxcoff-roptr.
-CODEGENOPT(AllTocData, 1, 0) ///< AIX -mtocdata
-ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None) /// frame-pointer: all,non-leaf,reserved,none
-
-CODEGENOPT(ClearASTBeforeBackend , 1, 0) ///< Free the AST before running backend code generation. Only works with -disable-free.
-CODEGENOPT(DisableFree       , 1, 0) ///< Don't free memory.
-CODEGENOPT(DiscardValueNames , 1, 0) ///< Discard Value Names from the IR (LLVMContext flag)
-CODEGENOPT(DisableLLVMPasses , 1, 0) ///< Don't run any LLVM IR passes to get
-                                     ///< the pristine IR generated by the
-                                     ///< frontend.
-CODEGENOPT(DisableLifetimeMarkers, 1, 0) ///< Don't emit any lifetime markers
-CODEGENOPT(DisableO0ImplyOptNone , 1, 0) ///< Don't annonate function with optnone at O0
-CODEGENOPT(ExperimentalStrictFloatingPoint, 1, 0) ///< Enables the new, experimental
-                                                  ///< strict floating point.
-CODEGENOPT(EnableNoundefAttrs, 1, 0) ///< Enable emitting `noundef` attributes on IR call arguments and return values
-CODEGENOPT(DebugPassManager, 1, 0) ///< Prints debug information for the new
-                                   ///< pass manager.
-CODEGENOPT(DisableRedZone    , 1, 0) ///< Set when -mno-red-zone is enabled.
-CODEGENOPT(EmitCallSiteInfo, 1, 0) ///< Emit call site info only in the case of
-                                   ///< '-g' + 'O>0' level.
-CODEGENOPT(IndirectTlsSegRefs, 1, 0) ///< Set when -mno-tls-direct-seg-refs
-                                     ///< is specified.
-CODEGENOPT(DisableTailCalls  , 1, 0) ///< Do not emit tail calls.
-CODEGENOPT(NoEscapingBlockTailCalls, 1, 0) ///< Do not emit tail calls from
-                                           ///< escaping blocks.
-CODEGENOPT(EmitDeclMetadata  , 1, 0) ///< Emit special metadata indicating what
-                                     ///< Decl* various IR entities came from.
-                                     ///< Only useful when running CodeGen as a
-                                     ///< subroutine.
-CODEGENOPT(EmitVersionIdentMetadata , 1, 1) ///< Emit compiler version metadata.
-CODEGENOPT(EmitOpenCLArgMetadata , 1, 0) ///< Emit OpenCL kernel arg metadata.
-CODEGENOPT(EmulatedTLS       , 1, 0) ///< Set by default or -f[no-]emulated-tls.
+CODEGENOPT(SeparateNamedSections, 1, 0, Benign) ///< Set for -fseparate-named-sections.
+CODEGENOPT(EnableAIXExtendedAltivecABI, 1, 0, Benign) ///< Set for -mabi=vec-extabi. Enables the extended Altivec ABI on AIX.
+CODEGENOPT(XCOFFReadOnlyPointers, 1, 0, Benign) ///< Set for -mxcoff-roptr.
+CODEGENOPT(AllTocData, 1, 0, Benign) ///< AIX -mtocdata
+ENUM_CODEGENOPT(FramePointer, FramePointerKind, 2, FramePointerKind::None, Benign) /// frame-pointer: all,non-leaf,reserved,none
+
+CODEGENOPT(ClearASTBeforeBackend , 1, 0, Benign) ///< Free the AST before running backend code generation. Only works with -disable-free.
+CODEGENOPT(DisableFree       , 1, 0, Benign) ///< Don't free memory.
+CODEGENOPT(DiscardValueNames , 1, 0, Benign) ///< Discard Value Names from the IR (LLVMContext flag)
+CODEGENOPT(DisableLLVMPasses , 1, 0, Benign) ///< Don't run any LLVM IR passes to get
+                                             ///< the pristine IR generated by the
+                                             ///< frontend.
+CODEGENOPT(DisableLifetimeMarkers, 1, 0, Benign) ///< Don't emit any lifetime markers
+CODEGENOPT(DisableO0ImplyOptNone , 1, 0, Benign) ///< Don't annonate function with optnone at O0
+CODEGENOPT(ExperimentalStrictFloatingPoint, 1, 0, Benign) ///< Enables the new, experimental
+                                                          ///< strict floating point.
+CODEGENOPT(EnableNoundefAttrs, 1, 0, Benign) ///< Enable emitting `noundef` attributes on IR call arguments and return values
+CODEGENOPT(DebugPassManager, 1, 0, Benign) ///< Prints debug information for the new
+                                           ///< pass manager.
+CODEGENOPT(DisableRedZone    , 1, 0, Benign) ///< Set when -mno-red-zone is enabled.
+CODEGENOPT(EmitCallSiteInfo, 1, 0, Benign) ///< Emit call site info only in the case of
+                                           ///< '-g' + 'O>0' level.
+CODEGENOPT(IndirectTlsSegRefs, 1, 0, Benign) ///< Set when -mno-tls-direct-seg-refs
+                                             ///< is specified.
+CODEGENOPT(DisableTailCalls  , 1, 0, Benign) ///< Do not emit tail calls.
+CODEGENOPT(NoEscapingBlockTailCalls, 1, 0, Benign) ///< Do not emit tail calls from
+                                                   ///< escaping blocks.
+CODEGENOPT(EmitDeclMetadata  , 1, 0, Benign) ///< Emit special metadata indicating what
+                                             ///< Decl* various IR entities came from.
+                                             ///< Only useful when running CodeGen as a
+                                             ///< subroutine.
+CODEGENOPT(EmitVersionIdentMetadata , 1, 1, Benign) ///< Emit compiler version metadata.
+CODEGENOPT(EmitOpenCLArgMetadata , 1, 0, Benign) ///< Emit OpenCL kernel arg metadata.
+CODEGENOPT(EmulatedTLS       , 1, 0, Benign) ///< Set by default or -f[no-]emulated-tls.
 /// Embed Bitcode mode (off/all/bitcode/marker).
-ENUM_CODEGENOPT(EmbedBitcode, EmbedBitcodeKind, 2, Embed_Off)
+ENUM_CODEGENOPT(EmbedBitcode, EmbedBitcodeKind, 2, Embed_Off, Benign)
 /// Inline asm dialect, -masm=(att|intel)
-ENUM_CODEGENOPT(InlineAsmDialect, InlineAsmDialectKind, 1, IAD_ATT)
-CODEGENOPT(OutputAsmVariant, 2, 3) ///< Set the asm variant for output (3: unspecified).
-CODEGENOPT(ForbidGuardVariables , 1, 0) ///< Issue errors if C++ guard variables
-                                        ///< are required.
-CODEGENOPT(FunctionSections  , 1, 0) ///< Set when -ffunction-sections is enabled.
-CODEGENOPT(BBAddrMap  , 1, 0) ///< Set when -fbasic-block-address-map is enabled.
-CODEGENOPT(InstrumentFunctions , 1, 0) ///< Set when -finstrument-functions is
-                                       ///< enabled.
-CODEGENOPT(InstrumentFunctionsAfterInlining , 1, 0) ///< Set when
+ENUM_CODEGENOPT(InlineAsmDialect, InlineAsmDialectKind, 1, IAD_ATT, Benign)
+CODEGENOPT(OutputAsmVariant, 2, 3, Benign) ///< Set the asm variant for output (3: unspecified).
+CODEGENOPT(ForbidGuardVariables , 1, 0, Benign) ///< Issue errors if C++ guard variables
+                                                ///< are required.
+CODEGENOPT(FunctionSections  , 1, 0, Benign) ///< Set when -ffunction-sections is enabled.
+CODEGENOPT(BBAddrMap  , 1, 0, Benign) ///< Set when -fbasic-block-address-map is enabled.
+CODEGENOPT(InstrumentFunctions , 1, 0, Benign) ///< Set when -finstrument-functions is
+                                               ///< enabled.
+CODEGENOPT(InstrumentFunctionsAfterInlining , 1, 0, Benign) ///< Set when
                           ///< -finstrument-functions-after-inlining is enabled.
-CODEGENOPT(InstrumentFunctionEntryBare , 1, 0) ///< Set when
+CODEGENOPT(InstrumentFunctionEntryBare , 1, 0, Benign) ///< Set when
                                ///< -finstrument-function-entry-bare is enabled.
-CODEGENOPT(CFProtectionReturn , 1, 0) ///< if -fcf-protection is
+CODEGENOPT(CFProtectionReturn , 1, 0, Benign) ///< if -fcf-protection is
                                       ///< set to full or return.
-CODEGENOPT(CFProtectionBranch , 1, 0) ///< if -fcf-protection is
-                                      ///< set to full or branch.
+CODEGENOPT(CFProtectionBranch , 1, 0, Benign) ///< if -fcf-protection is
+                                              ///< set to full or branch.
 ENUM_CODEGENOPT(CFBranchLabelScheme, CFBranchLabelSchemeKind, 2,
-    CFBranchLabelSchemeKind::Default) ///< if -mcf-branch-label-scheme is set.
-CODEGENOPT(FunctionReturnThunks, 1, 0) ///< -mfunction-return={keep|thunk-extern}
-CODEGENOPT(IndirectBranchCSPrefix, 1, 0) ///< if -mindirect-branch-cs-prefix
-                                         ///< is set.
+    CFBranchLabelSchemeKind::Default, Benign) ///< if -mcf-branch-label-scheme is set.
+CODEGENOPT(FunctionReturnThunks, 1, 0, Benign) ///< -mfunction-return={keep|thunk-extern}
+CODEGENOPT(IndirectBranchCSPrefix, 1, 0, Benign) ///< if -mindirect-branch-cs-prefix
+                                                 ///< is set.
 
-CODEGENOPT(XRayInstrumentFunctions , 1, 0) ///< Set when -fxray-instrument is
-                                           ///< enabled.
-CODEGENOPT(StackSizeSection  , 1, 0) ///< Set when -fstack-size-section is enabled.
+CODEGENOPT(XRayInstrumentFunctions , 1, 0, Benign) ///< Set when -fxray-instrument is
+                                                   ///< enabled.
+CODEGENOPT(StackSizeSection  , 1, 0, Benign) ///< Set when -fstack-size-section is enabled.
 
 ///< Set when -femit-compact-unwind-non-canonical is enabled.
-CODEGENOPT(EmitCompactUnwindNonCanonical, 1, 0)
+CODEGENOPT(EmitCompactUnwindNonCanonical, 1, 0, Benign)
 
 ///< Set when -fxray-always-emit-customevents is enabled.
-CODEGENOPT(XRayAlwaysEmitCustomEvents , 1, 0)
+CODEGENOPT(XRayAlwaysEmitCustomEvents , 1, 0, Benign)
 
 ///< Set when -fxray-always-emit-typedevents is enabled.
-CODEGENOPT(XRayAlwaysEmitTypedEvents , 1, 0)
+CODEGENOPT(XRayAlwaysEmitTypedEvents , 1, 0, Benign)
 
 ///< Set when -fxray-ignore-loops is enabled.
-CODEGENOPT(XRayIgnoreLoops , 1, 0)
+CODEGENOPT(XRayIgnoreLoops , 1, 0, Benign)
 
 ///< Emit the XRay function index section.
-CODEGENOPT(XRayFunctionIndex , 1, 1)
+CODEGENOPT(XRayFunctionIndex , 1, 1, Benign)
 
 ///< Set when -fxray-shared is enabled
-CODEGENOPT(XRayShared , 1, 0)
+CODEGENOPT(XRayShared , 1, 0, Benign)
 
 ///< Set the minimum number of instructions in a function to determine selective
 ///< XRay instrumentation.
-VALUE_CODEGENOPT(XRayInstructionThreshold , 32, 200)
+VALUE_CODEGENOPT(XRayInstructionThreshold , 32, 200, Benign)
 
 ///< Only instrument 1 in N functions, by dividing functions into N total groups and
 ///< instrumenting only the specified group at a time. Group numbers start at 0
 ///< and end at N-1.
-VALUE_CODEGENOPT(XRayTotalFunctionGroups, 32, 1)
-VALUE_CODEGENOPT(XRaySelectedFunctionGroup, 32, 0)
-
-VALUE_CODEGENOPT(PatchableFunctionEntryCount , 32, 0) ///< Number of NOPs at function entry
-VALUE_CODEGENOPT(PatchableFunctionEntryOffset , 32, 0)
-
-CODEGENOPT(HotPatch, 1, 0) ///< Supports the Microsoft /HOTPATCH flag and
-                           ///< generates a 'patchable-function' attribute.
-
-CODEGENOPT(TlsGuards , 1, 1) ///< Controls emission of tls guards via -fms-tls-guards
-CODEGENOPT(JMCInstrument, 1, 0) ///< Set when -fjmc is enabled.
-CODEGENOPT(InstrumentForProfiling , 1, 0) ///< Set when -pg is enabled.
-CODEGENOPT(CallFEntry , 1, 0) ///< Set when -mfentry is enabled.
-CODEGENOPT(MNopMCount , 1, 0) ///< Set when -mnop-mcount is enabled.
-CODEGENOPT(RecordMCount , 1, 0) ///< Set when -mrecord-mcount is enabled.
-CODEGENOPT(PackedStack , 1, 0) ///< Set when -mpacked-stack is enabled.
-CODEGENOPT(LessPreciseFPMAD  , 1, 0) ///< Enable less precise MAD instructions to
-                                     ///< be generated.
-CODEGENOPT(PrepareForLTO     , 1, 0) ///< Set when -flto is enabled on the
-                                     ///< compile step.
-CODEGENOPT(PrepareForThinLTO , 1, 0) ///< Set when -flto=thin is enabled on the
-                                     ///< compile step.
-CODEGENOPT(LTOUnit, 1, 0) ///< Emit IR to support LTO unit features (CFI, whole
-                          ///< program vtable opt).
-CODEGENOPT(FatLTO, 1, 0) ///< Set when -ffat-lto-objects is enabled.
-CODEGENOPT(EnableSplitLTOUnit, 1, 0) ///< Enable LTO unit splitting to support
-				     /// CFI and traditional whole program
-				     /// devirtualization that require whole
-				     /// program IR support.
-CODEGENOPT(UnifiedLTO, 1, 0) ///< Use the unified LTO pipeline.
-CODEGENOPT(IncrementalLinkerCompatible, 1, 0) ///< Emit an object file which can
-                                              ///< be used with an incremental
-                                              ///< linker.
-CODEGENOPT(MergeAllConstants , 1, 1) ///< Merge identical constants.
-CODEGENOPT(MergeFunctions    , 1, 0) ///< Set when -fmerge-functions is enabled.
-CODEGENOPT(NoCommon          , 1, 0) ///< Set when -fno-common or C++ is enabled.
-CODEGENOPT(NoExecStack       , 1, 0) ///< Set when -Wa,--noexecstack is enabled.
-CODEGENOPT(MipsMsa           , 1, 0) ///< Set when -Wa,-mmsa is enabled.
-CODEGENOPT(FatalWarnings     , 1, 0) ///< Set when -Wa,--fatal-warnings is
-                                     ///< enabled.
-CODEGENOPT(NoWarn            , 1, 0) ///< Set when -Wa,--no-warn is enabled.
-CODEGENOPT(NoTypeCheck       , 1, 0) ///< Set when -Wa,--no-type-check is enabled.
-CODEGENOPT(MisExpect         , 1, 0) ///< Set when -Wmisexpect is enabled
-CODEGENOPT(EnableSegmentedStacks , 1, 0) ///< Set when -fsplit-stack is enabled.
-CODEGENOPT(StackClashProtector, 1, 0) ///< Set when -fstack-clash-protection is enabled.
-CODEGENOPT(NoImplicitFloat   , 1, 0) ///< Set when -mno-implicit-float is enabled.
-CODEGENOPT(NullPointerIsValid , 1, 0) ///< Assume Null pointer deference is defined.
-CODEGENOPT(OpenCLCorrectlyRoundedDivSqrt, 1, 0) ///< -cl-fp32-correctly-rounded-divide-sqrt
-CODEGENOPT(HIPCorrectlyRoundedDivSqrt, 1, 1) ///< -fno-hip-fp32-correctly-rounded-divide-sqrt
-CODEGENOPT(DisableBlockSignatureString, 1, 0) ///< Set when -fdisable-block-signature-string is enabled.
-CODEGENOPT(HIPSaveKernelArgName, 1, 0) ///< Set when -fhip-kernel-arg-name is enabled.
-CODEGENOPT(UniqueInternalLinkageNames, 1, 0) ///< Internal Linkage symbols get unique names.
-CODEGENOPT(SplitMachineFunctions, 1, 0) ///< Split machine functions using profile information.
-CODEGENOPT(PPCUseFullRegisterNames, 1, 0) ///< Print full register names in assembly
-CODEGENOPT(X86RelaxRelocations, 1, 1) ///< -Wa,-mrelax-relocations={yes,no}
-CODEGENOPT(X86Sse2Avx        , 1, 0)   ///< -Wa,-msse2avx
+VALUE_CODEGENOPT(XRayTotalFunctionGroups, 32, 1, Benign)
+VALUE_CODEGENOPT(XRaySelectedFunctionGroup, 32, 0, Benign)
+
+VALUE_CODEGENOPT(PatchableFunctionEntryCount , 32, 0, Benign) ///< Number of NOPs at function entry
+VALUE_CODEGENOPT(PatchableFunctionEntryOffset , 32, 0, Benign)
+
+CODEGENOPT(HotPatch, 1, 0, Benign) ///< Supports the Microsoft /HOTPATCH flag and
+                                   ///< generates a 'patchable-function' attribute.
+
+CODEGENOPT(TlsGuards , 1, 1, Benign) ///< Controls emission of tls guards via -fms-tls-guards
+CODEGENOPT(JMCInstrument, 1, 0, Benign) ///< Set when -fjmc is enabled.
+CODEGENOPT(InstrumentForProfiling , 1, 0, Benign) ///< Set when -pg is enabled.
+CODEGENOPT(CallFEntry , 1, 0, Benign) ///< Set when -mfentry is enabled.
+CODEGENOPT(MNopMCount , 1, 0, Benign) ///< Set when -mnop-mcount is enabled.
+CODEGENOPT(RecordMCount , 1, 0, Benign) ///< Set when -mrecord-mcount is enabled.
+CODEGENOPT(PackedStack , 1, 0, Benign) ///< Set when -mpacked-stack is enabled.
+CODEGENOPT(LessPreciseFPMAD  , 1, 0, Benign) ///< Enable less precise MAD instructions to
+                                             ///< be generated.
+CODEGENOPT(PrepareForLTO     , 1, 0, Benign) ///< Set when -flto is enabled on the
+                                             ///< compile step.
+CODEGENOPT(PrepareForThinLTO , 1, 0, Benign) ///< Set when -flto=thin is enabled on the
+                                             ///< compile step.
+CODEGENOPT(LTOUnit, 1, 0, Benign) ///< Emit IR to support LTO unit features (CFI, whole
+                                 ///< program vtable opt).
+CODEGENOPT(FatLTO, 1, 0, Benign) ///< Set when -ffat-lto-objects is enabled.
+CODEGENOPT(EnableSplitLTOUnit, 1, 0, Benign) ///< Enable LTO unit splitting to support
+                                             /// CFI and traditional whole program
+                                             /// devirtualization that require whole
+                                             /// program IR support.
+CODEGENOPT(UnifiedLTO, 1, 0, Benign) ///< Use the unified LTO pipeline.
+CODEGENOPT(IncrementalLinkerCompatible, 1, 0, Benign) ///< Emit an object file which can
+                                                      ///< be used with an incremental
+                                                      ///< linker.
+CODEGENOPT(MergeAllConstants , 1, 1, Benign) ///< Merge identical constants.
+CODEGENOPT(MergeFunctions    , 1, 0, Benign) ///< Set when -fmerge-functions is enabled.
+CODEGENOPT(NoCommon          , 1, 0, Benign) ///< Set when -fno-common or C++ is enabled.
+CODEGENOPT(NoExecStack       , 1, 0, Benign) ///< Set when -Wa,--noexecstack is enabled.
+CODEGENOPT(MipsMsa           , 1, 0, Benign) ///< Set when -Wa,-mmsa is enabled.
+CODEGENOPT(FatalWarnings     , 1, 0, Benign) ///< Set when -Wa,--fatal-warnings is
+                                             ///< enabled.
+CODEGENOPT(NoWarn            , 1, 0, Benign) ///< Set when -Wa,--no-warn is enabled.
+CODEGENOPT(NoTypeCheck       , 1, 0, Benign) ///< Set when -Wa,--no-type-check is enabled.
+CODEGENOPT(MisExpect         , 1, 0, Benign) ///< Set when -Wmisexpect is enabled
+CODEGENOPT(EnableSegmentedStacks , 1, 0, Benign) ///< Set when -fsplit-stack is enabled.
+CODEGENOPT(StackClashProtector, 1, 0, Benign) ///< Set when -fstack-clash-protection is enabled.
+CODEGENOPT(NoImplicitFloat   , 1, 0, Benign) ///< Set when -mno-implicit-float is enabled.
+CODEGENOPT(NullPointerIsValid , 1, 0, Benign) ///< Assume Null pointer deference is defined.
+CODEGENOPT(OpenCLCorrectlyRoundedDivSqrt, 1, 0, Benign) ///< -cl-fp32-correctly-rounded-divide-sqrt
+CODEGENOPT(HIPCorrectlyRoundedDivSqrt, 1, 1, Benign) ///< -fno-hip-fp32-correctly-rounded-divide-sqrt
+CODEGENOPT(DisableBlockSignatureString, 1, 0, Benign) ///< Set when -fdisable-block-signature-string is enabled.
+CODEGENOPT(HIPSaveKernelArgName, 1, 0, Benign) ///< Set when -fhip-kernel-arg-name is enabled.
+CODEGENOPT(UniqueInternalLinkageNames, 1, 0, Benign) ///< Internal Linkage symbols get unique names.
+CODEGENOPT(SplitMachineFunctions, 1, 0, Benign) ///< Split machine functions using profile information.
+CODEGENOPT(PPCUseFullRegisterNames, 1, 0, Benign) ///< Print full register names in assembly
+CODEGENOPT(X86RelaxRelocations, 1, 1, Benign) ///< -Wa,-mrelax-relocations={yes,no}
+CODEGENOPT(X86Sse2Avx        , 1, 0, Benign)   ///< -Wa,-msse2avx
 
 /// When false, this attempts to generate code as if the result of an
 /// overflowing conversion matches the overflowing behavior of a target's native
 /// float-to-int conversion instructions.
-CODEGENOPT(StrictFloatCastOverflow, 1, 1)
+CODEGENOPT(StrictFloatCastOverflow, 1, 1, Benign)
 
-CODEGENOPT(NoZeroInitializedInBSS , 1, 0) ///< -fno-zero-initialized-in-bss.
+CODEGENOPT(NoZeroInitializedInBSS , 1, 0, Benign) ///< -fno-zero-initialized-in-bss.
 /// Method of Objective-C dispatch to use.
-ENUM_CODEGENOPT(ObjCDispatchMethod, ObjCDispatchMethodKind, 2, Legacy)
+ENUM_CODEGENOPT(ObjCDispatchMethod, ObjCDispatchMethodKind, 2, Legacy, Benign)
 /// Replace certain message sends with calls to ObjC runtime entrypoints
-CODEGENOPT(ObjCConvertMessagesToRuntimeCalls , 1, 1)
-CODEGENOPT(ObjCAvoidHeapifyLocalBlocks, 1, 0)
+CODEGENOPT(ObjCConvertMessagesToRuntimeCalls , 1, 1, Benign)
+CODEGENOPT(ObjCAvoidHeapifyLocalBlocks, 1, 0, Benign)
 
 
-// The optimization options affect frontend options, whicn in turn do affect the AST.
-AFFECTING_VALUE_CODEGENOPT(OptimizationLevel, 2, 0) ///< The -O[0-3] option specified.
-AFFECTING_VALUE_CODEGENOPT(OptimizeSize, 2, 0) ///< If -Os (==1) or -Oz (==2) is specified.
+// The optimization options affect frontend options, which in turn do affect the AST.
+VALUE_CODEGENOPT(OptimizationLevel, 2, 0, Affecting) ///< The -O[0-3] option specified.
+VALUE_CODEGENOPT(OptimizeSize, 2, 0, Affecting) ///< If -Os (==1, Benign) or -Oz (==2, Benign) is specified.
 
-CODEGENOPT(AtomicProfileUpdate , 1, 0) ///< Set -fprofile-update=atomic
-CODEGENOPT(ContinuousProfileSync, 1, 0) ///< Enable continuous instrumentation profiling
+CODEGENOPT(AtomicProfileUpdate , 1, 0, Benign) ///< Set -fprofile-update=atomic
+CODEGENOPT(ContinuousProfileSync, 1, 0, Benign) ///< Enable continuous instrumentation profiling
 /// Choose profile instrumenation kind or no instrumentation.
 
-ENUM_CODEGENOPT(ProfileInstr, llvm::driver::ProfileInstrKind, 4, llvm::driver::ProfileInstrKind::ProfileNone)
+ENUM_CODEGENOPT(ProfileInstr, llvm::driver::ProfileInstrKind, 4, llvm::driver::ProfileInstrKind::ProfileNone, Benign)
 
 /// Choose profile kind for PGO use compilation.
-ENUM_CODEGENOPT(ProfileUse, llvm::driver::ProfileInstrKind, 2, llvm::driver::ProfileInstrKind::ProfileNone)
+ENUM_CODEGENOPT(ProfileUse, llvm::driver::ProfileInstrKind, 2, llvm::driver::ProfileInstrKind::ProfileNone, Benign)
 /// Partition functions into N groups and select only functions in group i to be
 /// instrumented. Selected group numbers can be 0 to N-1 inclusive.
-VALUE_CODEGENOPT(ProfileTotalFunctionGroups, 32, 1)
-VALUE_CODEGENOPT(ProfileSelectedFunctionGroup, 32, 0)
-CODEGENOPT(CoverageMapping , 1, 0) ///< Generate coverage mapping regions to
-                                   ///< enable code coverage analysis.
-CODEGENOPT(DumpCoverageMapping , 1, 0) ///< Dump the generated coverage mapping
-                                       ///< regions.
-CODEGENOPT(MCDCCoverage , 1, 0) ///< Enable MC/DC code coverage criteria.
-VALUE_CODEGENOPT(MCDCMaxConds, 16, 32767) ///< MC/DC Maximum conditions.
-VALUE_CODEGENOPT(MCDCMaxTVs, 32, 0x7FFFFFFE) ///< MC/DC Maximum test vectors.
+VALUE_CODEGENOPT(ProfileTotalFunctionGroups, 32, 1, Benign)
+VALUE_CODEGENOPT(ProfileSelectedFunctionGroup, 32, 0, Benign)
+CODEGENOPT(CoverageMapping , 1, 0, Benign) ///< Generate coverage mapping regions to
+                                           ///< enable code coverage analysis.
+CODEGENOPT(DumpCoverageMapping , 1, 0, Benign) ///< Dump the generated coverage mapping
+                                               ///< regions.
+CODEGENOPT(MCDCCoverage , 1, 0, Benign) ///< Enable MC/DC code coverage criteria.
+VALUE_CODEGENOPT(MCDCMaxConds, 16, 32767, Benign) ///< MC/DC Maximum conditions.
+VALUE_CODEGENOPT(MCDCMaxTVs, 32, 0x7FFFFFFE, Benign) ///< MC/DC Maximum test vectors.
 
   /// If -fpcc-struct-return or -freg-struct-return is specified.
-ENUM_CODEGENOPT(StructReturnConvention, StructReturnConventionKind, 2, SRCK_Default)
-
-CODEGENOPT(RelaxAll          , 1, 0) ///< Relax all machine code instructions.
-CODEGENOPT(RelaxedAliasing   , 1, 0) ///< Set when -fno-strict-aliasing is enabled.
-CODEGENOPT(PointerTBAA       , 1, 1) ///< Whether or not to use distinct TBAA tags for pointers.
-CODEGENOPT(StructPathTBAA    , 1, 0) ///< Whether or not to use struct-path TBAA.
-CODEGENOPT(NewStructPathTBAA , 1, 0) ///< Whether or not to use enhanced struct-path TBAA.
-CODEGENOPT(SaveTempLabels    , 1, 0) ///< Save temporary labels.
-CODEGENOPT(SanitizeAddressUseAfterScope , 1, 0) ///< Enable use-after-scope detection
-                                                ///< in AddressSanitizer
+ENUM_CODEGENOPT(StructReturnConvention, StructReturnConventionKind, 2, SRCK_Default, Benign)
+
+CODEGENOPT(RelaxAll          , 1, 0, Benign) ///< Relax all machine code instructions.
+CODEGENOPT(RelaxedAliasing   , 1, 0, Benign) ///< Set when -fno-strict-aliasing is enabled.
+CODEGENOPT(PointerTBAA       , 1, 1, Benign) ///< Whether or not to use distinct TBAA tags for pointers.
+CODEGENOPT(StructPathTBAA    , 1, 0, Benign) ///< Whether or not to use struct-path TBAA.
+CODEGENOPT(NewStructPathTBAA , 1, 0, Benign) ///< Whether or not to use enhanced struct-path TBAA.
+CODEGENOPT(SaveTempLabels    , 1, 0, Benign) ///< Save temporary labels.
+CODEGENOPT(SanitizeAddressUseAfterScope , 1, 0, Benign) ///< Enable use-after-scope detection
+                                                        ///< in AddressSanitizer
 ENUM_CODEGENOPT(SanitizeAddressUseAfterReturn,
                 llvm::AsanDetectStackUseAfterReturnMode, 2,
-                llvm::AsanDetectStackUseAfterReturnMode::Runtime
+                llvm::AsanDetectStackUseAfterReturnMode::Runtime,
+                Benign
                 ) ///< Set detection mode for stack-use-after-return.
-CODEGENOPT(SanitizeAddressPoisonCustomArrayCookie, 1,
-           0) ///< Enable poisoning operator new[] which is not a replaceable
-              ///< global allocation function in AddressSanitizer
-CODEGENOPT(SanitizeAddressGlobalsDeadStripping, 1, 0) ///< Enable linker dead stripping
-                                                      ///< of globals in AddressSanitizer
-CODEGENOPT(SanitizeAddressUseOdrIndicator, 1, 0) ///< Enable ODR indicator globals
-CODEGENOPT(SanitizeMemoryTrackOrigins, 2, 0) ///< Enable tracking origins in
-                                             ///< MemorySanitizer
+CODEGENOPT(SanitizeAddressPoisonCustomArrayCookie, 1, 0, Benign) ///< Enable poisoning operator new[] which is not a replaceable
+                                                                 ///< global allocation function in AddressSanitizer
+CODEGENOPT(SanitizeAddressGlobalsDeadStripping, 1, 0, Benign) ///< Enable linker dead stripping
+                                                              ///< of globals in AddressSanitizer
+CODEGENOPT(SanitizeAddressUseOdrIndicator, 1, 0, Benign) ///< Enable ODR indicator globals
+CODEGENOPT(SanitizeMemoryTrackOrigins, 2, 0, Benign) ///< Enable tracking origins in
+                                                     ///< MemorySanitizer
 ENUM_CODEGENOPT(SanitizeAddressDtor, llvm::AsanDtorKind, 2,
-                llvm::AsanDtorKind::Global)  ///< Set how ASan global
-                                             ///< destructors are emitted.
-CODEGENOPT(SanitizeMemoryParamRetval, 1, 0) ///< Enable detection of uninitialized
-                                            ///< parameters and return values
-                                            ///< in MemorySanitizer
-CODEGENOPT(SanitizeMemoryUseAfterDtor, 1, 0) ///< Enable use-after-delete detection
-                                             ///< in MemorySanitizer
-CODEGENOPT(SanitizeCfiCrossDso, 1, 0) ///< Enable cross-dso support in CFI.
-CODEGENOPT(SanitizeMinimalRuntime, 1, 0) ///< Use "_minimal" sanitizer runtime for
-                                         ///< diagnostics.
-CODEGENOPT(SanitizeCfiICallGeneralizePointers, 1, 0) ///< Generalize pointer types in
-                                                     ///< CFI icall function signatures
-CODEGENOPT(SanitizeCfiICallNormalizeIntegers, 1, 0) ///< Normalize integer types in
-                                                    ///< CFI icall function signatures
-CODEGENOPT(SanitizeCfiCanonicalJumpTables, 1, 0) ///< Make jump table symbols canonical
-                                                 ///< instead of creating a local jump table.
-CODEGENOPT(SanitizeKcfiArity, 1, 0) ///< Embed arity in KCFI patchable function prefix
-CODEGENOPT(SanitizeCoverageType, 2, 0) ///< Type of sanitizer coverage
-                                       ///< instrumentation.
-CODEGENOPT(SanitizeCoverageIndirectCalls, 1, 0) ///< Enable sanitizer coverage
-                                                ///< for indirect calls.
-CODEGENOPT(SanitizeCoverageTraceBB, 1, 0) ///< Enable basic block tracing in
-                                          ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverageTraceCmp, 1, 0) ///< Enable cmp instruction tracing
-                                           ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverageTraceDiv, 1, 0) ///< Enable div instruction tracing
-                                           ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverageTraceGep, 1, 0) ///< Enable GEP instruction tracing
-                                           ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverage8bitCounters, 1, 0) ///< Use 8-bit frequency counters
-                                               ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverageTracePC, 1, 0) ///< Enable PC tracing
-                                          ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverageTracePCGuard, 1, 0) ///< Enable PC tracing with guard
-                                               ///< in sanitizer coverage.
-CODEGENOPT(SanitizeCoverageInline8bitCounters, 1, 0) ///< Use inline 8bit counters.
-CODEGENOPT(SanitizeCoverageInlineBoolFlag, 1, 0) ///< Use inline bool flag.
-CODEGENOPT(SanitizeCoveragePCTable, 1, 0) ///< Create a PC Table.
-CODEGENOPT(SanitizeCoverageControlFlow, 1, 0) ///< Collect control flow
-CODEGENOPT(SanitizeCoverageNoPrune, 1, 0) ///< Disable coverage pruning.
-CODEGENOPT(SanitizeCoverageStackDepth, 1, 0) ///< Enable max stack depth tracing
-VALUE_CODEGENOPT(SanitizeCoverageStackDepthCallbackMin , 32, 0) ///< Enable stack depth tracing callbacks.
-CODEGENOPT(SanitizeCoverageTraceLoads, 1, 0) ///< Enable tracing of loads.
-CODEGENOPT(SanitizeCoverageTraceStores, 1, 0) ///< Enable tracing of stores.
-CODEGENOPT(SanitizeBinaryMetadataCovered, 1, 0) ///< Emit PCs for covered functions.
-CODEGENOPT(SanitizeBinaryMetadataAtomics, 1, 0) ///< Emit PCs for atomic operations.
-CODEGENOPT(SanitizeBinaryMetadataUAR, 1, 0) ///< Emit PCs for start of functions
-                                            ///< that are subject for use-after-return checking.
-CODEGENOPT(SanitizeStats     , 1, 0) ///< Collect statistics for sanitizers.
-CODEGENOPT(SimplifyLibCalls  , 1, 1) ///< Set when -fbuiltin is enabled.
-CODEGENOPT(SoftFloat         , 1, 0) ///< -soft-float.
-CODEGENOPT(SpeculativeLoadHardening, 1, 0) ///< Enable speculative load hardening.
-CODEGENOPT(FineGrainedBitfieldAccesses, 1, 0) ///< Enable fine-grained bitfield accesses.
-CODEGENOPT(StrictEnums       , 1, 0) ///< Optimize based on strict enum definition.
-CODEGENOPT(StrictVTablePointers, 1, 0) ///< Optimize based on the strict vtable pointers
-CODEGENOPT(TimePasses        , 1, 0) ///< Set when -ftime-report or -ftime-report= or -ftime-report-json is enabled.
-CODEGENOPT(TimePassesPerRun  , 1, 0) ///< Set when -ftime-report=per-pass-run is enabled.
-CODEGENOPT(TimePassesJson    , 1, 0) ///< Set when -ftime-report-json is enabled.
-CODEGENOPT(TimeTrace         , 1, 0) ///< Set when -ftime-trace is enabled.
-VALUE_CODEGENOPT(TimeTraceGranularity, 32, 500) ///< Minimum time granularity (in microseconds),
-                                               ///< traced by time profiler
-CODEGENOPT(InterchangeLoops  , 1, 0) ///< Run loop-interchange.
-CODEGENOPT(UnrollLoops       , 1, 0) ///< Control whether loops are unrolled.
-CODEGENOPT(RerollLoops       , 1, 0) ///< Control whether loops are rerolled.
-CODEGENOPT(NoUseJumpTables   , 1, 0) ///< Set when -fno-jump-tables is enabled.
-VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind tables (1) or asynchronous unwind tables (2)
-CODEGENOPT(LinkBitcodePostopt, 1, 0) ///< Link builtin bitcodes after optimization pipeline.
-CODEGENOPT(VectorizeLoop     , 1, 0) ///< Run loop vectorizer.
-CODEGENOPT(VectorizeSLP      , 1, 0) ///< Run SLP vectorizer.
-CODEGENOPT(ProfileSampleAccurate, 1, 0) ///< Sample profile is accurate.
+                llvm::AsanDtorKind::Global, Benign)  ///< Set how ASan global
+                                                     ///< destructors are emitted.
+CODEGENOPT(SanitizeMemoryParamRetval, 1, 0, Benign) ///< Enable detection of uninitialized
+                                                    ///< parameters and return values
+                                                    ///< in MemorySanitizer
+CODEGENOPT(SanitizeMemoryUseAfterDtor, 1, 0, Benign) ///< Enable use-after-delete detection
+                                                     ///< in MemorySanitizer
+CODEGENOPT(SanitizeCfiCrossDso, 1, 0, Benign) ///< Enable cross-dso support in CFI.
+CODEGENOPT(SanitizeMinimalRuntime, 1, 0, Benign) ///< Use "_minimal" sanitizer runtime for
+                                                 ///< diagnostics.
+CODEGENOPT(SanitizeCfiICallGeneralizePointers, 1, 0, Benign) ///< Generalize pointer types in
+                                                             ///< CFI icall function signatures
+CODEGENOPT(SanitizeCfiICallNormalizeIntegers, 1, 0, Benign) ///< Normalize integer types in
+                                                            ///< CFI icall function signatures
+CODEGENOPT(SanitizeCfiCanonicalJumpTables, 1, 0, Benign) ///< Make jump table symbols canonical
+                                                         ///< instead of creating a local jump table.
+CODEGENOPT(SanitizeKcfiArity, 1, 0, Benign) ///< Embed arity in KCFI patchable function prefix
+CODEGENOPT(SanitizeCoverageType, 2, 0, Benign) ///< Type of sanitizer coverage
+                                               ///< instrumentation.
+CODEGENOPT(SanitizeCoverageIndirectCalls, 1, 0, Benign) ///< Enable sanitizer coverage
+                                                        ///< for indirect calls.
+CODEGENOPT(SanitizeCoverageTraceBB, 1, 0, Benign) ///< Enable basic block tracing in
+                                                  ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverageTraceCmp, 1, 0, Benign) ///< Enable cmp instruction tracing
+                                                   ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverageTraceDiv, 1, 0, Benign) ///< Enable div instruction tracing
+                                                   ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverageTraceGep, 1, 0, Benign) ///< Enable GEP instruction tracing
+                                                   ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverage8bitCounters, 1, 0, Benign) ///< Use 8-bit frequency counters
+                                                       ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverageTracePC, 1, 0, Benign) ///< Enable PC tracing
+                                                  ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverageTracePCGuard, 1, 0, Benign) ///< Enable PC tracing with guard
+                                                       ///< in sanitizer coverage.
+CODEGENOPT(SanitizeCoverageInline8bitCounters, 1, 0, Benign) ///< Use inline 8bit counters.
+CODEGENOPT(SanitizeCoverageInlineBoolFlag, 1, 0, Benign) ///< Use inline bool flag.
+CODEGENOPT(SanitizeCoveragePCTable, 1, 0, Benign) ///< Create a PC Table.
+CODEGENOPT(SanitizeCoverageControlFlow, 1, 0, Benign) ///< Collect control flow
+CODEGENOPT(SanitizeCoverageNoPrune, 1, 0, Benign) ///< Disable coverage pruning.
+CODEGENOPT(SanitizeCoverageStackDepth, 1, 0, Benign) ///< Enable max stack depth tracing
+VALUE_CODEGENOPT(SanitizeCoverageStackDepthCallbackMin , 32, 0, Benign) ///< Enable stack depth tracing callbacks.
+CODEGENOPT(SanitizeCoverageTraceLoads, 1, 0, Benign) ///< Enable tracing of loads.
+CODEGENOPT(SanitizeCoverageTraceStores, 1, 0, Benign) ///< Enable tracing of stores.
+CODEGENOPT(SanitizeBinaryMetadataCovered, 1, 0, Benign) ///< Emit PCs for covered functions.
+CODEGENOPT(SanitizeBinaryMetadataAtomics, 1, 0, Benign) ///< Emit PCs for atomic operations.
+CODEGENOPT(SanitizeBinaryMetadataUAR, 1, 0, Benign) ///< Emit PCs for start of functions
+                                                    ///< that are subject for use-after-return checking.
+CODEGENOPT(SanitizeStats     , 1, 0, Benign) ///< Collect statistics for sanitizers.
+CODEGENOPT(SimplifyLibCalls  , 1, 1, Benign) ///< Set when -fbuiltin is enabled.
+CODEGENOPT(SoftFloat         , 1, 0, Benign) ///< -soft-float.
+CODEGENOPT(SpeculativeLoadHardening, 1, 0, Benign) ///< Enable speculative load hardening.
+CODEGENOPT(FineGrainedBitfieldAccesses, 1, 0, Benign) ///< Enable fine-grained bitfield accesses.
+CODEGENOPT(StrictEnums       , 1, 0, Benign) ///< Optimize based on strict enum definition.
+CODEGENOPT(StrictVTablePointers, 1, 0, Benign) ///< Optimize based on the strict vtable pointers
+CODEGENOPT(TimePasses        , 1, 0, Benign) ///< Set when -ftime-report or -ftime-report= or -ftime-report-json is enabled.
+CODEGENOPT(TimePassesPerRun  , 1, 0, Benign) ///< Set when -ftime-report=per-pass-run is enabled.
+CODEGENOPT(TimePassesJson    , 1, 0, Benign) ///< Set when -ftime-report-json is enabled.
+CODEGENOPT(TimeTrace         , 1, 0, Benign) ///< Set when -ftime-trace is enabled.
+VALUE_CODEGENOPT(TimeTraceGranularity, 32, 500, Benign) ///< Minimum time granularity (in microseconds),
+                                                        ///< traced by time profiler
+CODEGENOPT(InterchangeLoops  , 1, 0, Benign) ///< Run loop-interchange.
+CODEGENOPT(UnrollLoops       , 1, 0, Benign) ///< Control whether loops are unrolled.
+CODEGENOPT(RerollLoops       , 1, 0, Benign) ///< Control whether loops are rerolled.
+CODEGENOPT(NoUseJumpTables   , 1, 0, Benign) ///< Set when -fno-jump-tables is enabled.
+VALUE_CODEGENOPT(UnwindTables, 2, 0, Benign) ///< Unwind tables (1, Benign) or asynchronous unwind tables (2, Benign)
+CODEGENOPT(LinkBitcodePostopt, 1, 0, Benign) ///< Link builtin bitcodes after optimization pipeline.
+CODEGENOPT(VectorizeLoop     , 1, 0, Benign) ///< Run loop vectorizer.
+CODEGENOPT(VectorizeSLP      , 1, 0, Benign) ///< Run SLP vectorizer.
+CODEGENOPT(ProfileSampleAccurate, 1, 0, Benign) ///< Sample profile is accurate.
 
 /// Treat loops as finite: language, always, never.
-ENUM_CODEGENOPT(FiniteLoops, FiniteLoopsKind, 2, FiniteLoopsKind::Language)
+ENUM_CODEGENOPT(FiniteLoops, FiniteLoopsKind, 2, FiniteLoopsKind::Language, Benign)
 
   /// Attempt to use register sized accesses to bit-fields in structures, when
   /// possible.
-CODEGENOPT(UseRegisterSizedBitfieldAccess , 1, 0)
-
-CODEGENOPT(VerifyModule      , 1, 1) ///< Control whether the module should be run
-                                     ///< through the LLVM Verifier.
-CODEGENOPT(VerifyEach        , 1, 1) ///< Control whether the LLVM verifier
-                                     ///< should run after every pass.
-
-CODEGENOPT(StackRealignment  , 1, 0) ///< Control whether to force stack
-                                     ///< realignment.
-CODEGENOPT(UseInitArray      , 1, 0) ///< Control whether to use .init_array or
-                                     ///< .ctors.
-VALUE_CODEGENOPT(LoopAlignment     , 32, 0) ///< Overrides default loop
-                                            ///< alignment, if not 0.
-VALUE_CODEGENOPT(StackAlignment    , 32, 0) ///< Overrides default stack
-                                            ///< alignment, if not 0.
-VALUE_CODEGENOPT(StackProbeSize    , 32, 4096) ///< Overrides default stack
-                                               ///< probe size, even if 0.
-VALUE_CODEGENOPT(WarnStackSize     , 32, UINT_MAX) ///< Set via -fwarn-stack-size.
-CODEGENOPT(NoStackArgProbe, 1, 0) ///< Set when -mno-stack-arg-probe is used
-CODEGENOPT(EmitLLVMUseLists, 1, 0) ///< Control whether to serialize use-lists.
-
-CODEGENOPT(WholeProgramVTables, 1, 0) ///< Whether to apply whole-program
-                                      ///  vtable optimization.
-
-CODEGENOPT(VirtualFunctionElimination, 1, 0) ///< Whether to apply the dead
-                                             /// virtual function elimination
-                                             /// optimization.
+CODEGENOPT(UseRegisterSizedBitfieldAccess , 1, 0, Benign)
+
+CODEGENOPT(VerifyModule      , 1, 1, Benign) ///< Control whether the module should be run
+                                             ///< through the LLVM Verifier.
+CODEGENOPT(VerifyEach        , 1, 1, Benign) ///< Control whether the LLVM verifier
+                                             ///< should run after every pass.
+
+CODEGENOPT(StackRealignment  , 1, 0, Benign) ///< Control whether to force stack
+                                             ///< realignment.
+CODEGENOPT(UseInitArray      , 1, 0, Benign) ///< Control whether to use .init_array or
+                                             ///< .ctors.
+VALUE_CODEGENOPT(LoopAlignment     , 32, 0, Benign) ///< Overrides default loop
+                                                    ///< alignment, if not 0.
+VALUE_CODEGENOPT(StackAlignment    , 32, 0, Benign) ///< Overrides default stack
+                                                    ///< alignment, if not 0.
+VALUE_CODEGENOPT(StackProbeSize    , 32, 4096, Benign) ///< Overrides default stack
+                                                       ///< probe size, even if 0.
+VALUE_CODEGENOPT(WarnStackSize     , 32, UINT_MAX, Benign) ///< Set via -fwarn-stack-size.
+CODEGENOPT(NoStackArgProbe, 1, 0, Benign) ///< Set when -mno-stack-arg-probe is used
+CODEGENOPT(EmitLLVMUseLists, 1, 0, Benign) ///< Control whether to serialize use-lists.
+
+CODEGENOPT(WholeProgramVTables, 1, 0, Benign) ///< Whether to apply whole-program
+                                              ///  vtable optimization.
+
+CODEGENOPT(VirtualFunctionElimination, 1, 0, Benign) ///< Whether to apply the dead
+                                                     /// virtual function elimination
+                                                     /// optimization.
 
 /// Whether to use public LTO visibility for entities in std and stdext
 /// namespaces. This is enabled by clang-cl's /MT and /MTd flags.
-CODEGENOPT(LTOVisibilityPublicStd, 1, 0)
+CODEGENOPT(LTOVisibilityPublicStd, 1, 0, Benign)
 
 /// The user specified number of registers to be used for integral arguments,
 /// or 0 if unspecified.
-VALUE_CODEGENOPT(NumRegisterParameters, 32, 0)
+VALUE_CODEGENOPT(NumRegisterParameters, 32, 0, Benign)
 
 /// The threshold to put data into small data section.
-VALUE_CODEGENOPT(SmallDataLimit, 32, 0)
+VALUE_CODEGENOPT(SmallDataLimit, 32, 0, Benign)
 
 /// The lower bound for a buffer to be considered for stack protection.
-VALUE_CODEGENOPT(SSPBufferSize, 32, 0)
+VALUE_CODEGENOPT(SSPBufferSize, 32, 0, Benign)
 
 /// The kind of inlining to perform.
-ENUM_CODEGENOPT(Inlining, InliningMethod, 2, NormalInlining)
+ENUM_CODEGENOPT(Inlining, InliningMethod, 2, NormalInlining, Benign)
 
 /// The maximum stack size a function can have to be considered for inlining.
-VALUE_CODEGENOPT(InlineMaxStackSize, 32, UINT_MAX)
+VALUE_CODEGENOPT(InlineMaxStackSize, 32, UINT_MAX, Benign)
 
 // Vector functions library to use.
-ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary)
+ENUM_CODEGENOPT(VecLib, llvm::driver::VectorLibrary, 4, llvm::driver::VectorLibrary::NoLibrary, Benign)
 
 /// The default TLS model to use.
-ENUM_CODEGENOPT(DefaultTLSModel, TLSModel, 2, GeneralDynamicTLSModel)
+ENUM_CODEGENOPT(DefaultTLSModel, TLSModel, 2, GeneralDynamicTLSModel, Benign)
 
 /// Whether to enable TLSDESC. AArch64 enables TLSDESC regardless of this value.
-CODEGENOPT(EnableTLSDESC, 1, 0)
+CODEGENOPT(EnableTLSDESC, 1, 0, Benign)
 
 /// Bit size of immediate TLS offsets (0 == use the default).
-VALUE_CODEGENOPT(TLSSize, 8, 0)
+VALUE_CODEGENOPT(TLSSize, 8, 0, Benign)
 
 /// The types of variables that we will extend the live ranges of.
-ENUM_CODEGENOPT(ExtendVariableLiveness, ExtendVariableLivenessKind, 2, ExtendVariableLivenessKind::None)
+ENUM_CODEGENOPT(ExtendVariableLiveness, ExtendVariableLivenessKind, 2, ExtendVariableLivenessKind::None, Benign)
 
 /// The default stack protector guard offset to use.
-VALUE_CODEGENOPT(StackProtectorGuardOffset, 32, INT_MAX)
+VALUE_CODEGENOPT(StackProtectorGuardOffset, 32, INT_MAX, Benign)
 
 /// Number of path components to strip when emitting checks. (0 == full
 /// filename)
-VALUE_CODEGENOPT(EmitCheckPathComponentsToStrip, 32, 0)
+VALUE_CODEGENOPT(EmitCheckPathComponentsToStrip, 32, 0, Benign)
 
 /// Whether to report the hotness of the code region for optimization remarks.
-CODEGENOPT(DiagnosticsWithHotness, 1, 0)
+CODEGENOPT(DiagnosticsWithHotness, 1, 0, Benign)
 
 /// Whether to use direct access relocations (instead of GOT) to reference external data symbols.
-CODEGENOPT(DirectAccessExternalData, 1, 0)
+CODEGENOPT(DirectAccessExternalData, 1, 0, Benign)
 
 /// Whether we should use the undefined behaviour optimization for control flow
 /// paths that reach the end of a function without executing a required return.
-CODEGENOPT(StrictReturn, 1, 1)
+CODEGENOPT(StrictReturn, 1, 1, Benign)
 
 /// Whether emit pseudo probes for sample pgo profile collection.
-CODEGENOPT(PseudoProbeForProfiling, 1, 0)
+CODEGENOPT(PseudoProbeForProfiling, 1, 0, Benign)
 
-CODEGENOPT(NoPLT, 1, 0)
+CODEGENOPT(NoPLT, 1, 0, Benign)
 
 /// Whether to emit all vtables
-CODEGENOPT(ForceEmitVTables, 1, 0)
+CODEGENOPT(ForceEmitVTables, 1, 0, Benign)
 
 /// Whether to emit an address-significance table into the object file.
-CODEGENOPT(Addrsig, 1, 0)
+CODEGENOPT(Addrsig, 1, 0, Benign)
 
 /// Whether to emit unused static constants.
-CODEGENOPT(KeepStaticConsts, 1, 0)
+CODEGENOPT(KeepStaticConsts, 1, 0, Benign)
 
 /// Whether to emit all variables that have a persistent storage duration,
 /// including global, static and thread local variables.
-CODEGENOPT(KeepPersistentStorageVariables, 1, 0)
+CODEGENOPT(KeepPersistentStorageVariables, 1, 0, Benign)
 
 /// Whether to follow the AAPCS enforcing at least one read before storing to a volatile bitfield
-CODEGENOPT(ForceAAPCSBitfieldLoad, 1, 0)
+CODEGENOPT(ForceAAPCSBitfieldLoad, 1, 0, Benign)
 
 /// Assume that by-value parameters do not alias any other values.
-CODEGENOPT(PassByValueIsNoAlias, 1, 0)
+CODEGENOPT(PassByValueIsNoAlias, 1, 0, Benign)
 
 /// Whether to store register parameters to stack.
-CODEGENOPT(SaveRegParams, 1, 0)
+CODEGENOPT(SaveRegParams, 1, 0, Benign)
 
 /// Whether to not follow the AAPCS that enforces volatile bit-field access width to be
 /// according to the field declaring type width.
-CODEGENOPT(AAPCSBitfieldWidth, 1, 1)
+CODEGENOPT(AAPCSBitfieldWidth, 1, 1, Benign)
 
 /// Sets the IEEE bit in the expected default floating point mode register.
 /// Floating point opcodes that support exception flag gathering quiet and
 /// propagate signaling NaN inputs per IEEE 754-2008 (AMDGPU Only)
-CODEGENOPT(EmitIEEENaNCompliantInsts, 1, 1)
+CODEGENOPT(EmitIEEENaNCompliantInsts, 1, 1, Benign)
 
 // Whether to emit Swift Async function extended frame information: auto,
 // never, always.
 ENUM_CODEGENOPT(SwiftAsyncFramePointer, SwiftAsyncFramePointerKind, 2,
-                SwiftAsyncFramePointerKind::Always)
+                SwiftAsyncFramePointerKind::Always, Benign)
 
 /// Whether to skip RAX setup when passing variable arguments (x86 only).
-CODEGENOPT(SkipRaxSetup, 1, 0)
+CODEGENOPT(SkipRaxSetup, 1, 0, Benign)
 
 /// Whether to zero out caller-used registers before returning.
 ENUM_CODEGENOPT(ZeroCallUsedRegs, llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind,
-                5, llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip)
+                5, llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip, Benign)
 
 /// Modify C++ ABI to returning `this` pointer from constructors and
 /// non-deleting destructors. (No effect on Microsoft ABI.)
-CODEGENOPT(CtorDtorReturnThis, 1, 0)
+CODEGENOPT(CtorDtorReturnThis, 1, 0, Benign)
 
 /// Enables emitting Import Call sections on supported targets that can be used
 /// by the Windows kernel to enable import call optimization.
-CODEGENOPT(ImportCallOptimization, 1, 0)
+CODEGENOPT(ImportCallOptimization, 1, 0, Benign)
 
 /// Controls whether we generate code for static linking of libclosure
 /// (BlocksRuntime) on Windows.
-CODEGENOPT(StaticClosure, 1, 0)
+CODEGENOPT(StaticClosure, 1, 0, Benign)
 
 /// Assume that UAVs/SRVs may alias
-CODEGENOPT(ResMayAlias, 1, 0)
+CODEGENOPT(ResMayAlias, 1, 0, Benign)
 
 /// Controls how unwind v2 (epilog) information should be generated for x64
 /// Windows.
 ENUM_CODEGENOPT(WinX64EHUnwindV2, llvm::WinX64EHUnwindV2Mode,
-                2, llvm::WinX64EHUnwindV2Mode::Disabled)
+                2, llvm::WinX64EHUnwindV2Mode::Disabled, Benign)
 
 /// FIXME: Make DebugOptions its own top-level .def file.
 #include "DebugOptions.def"
@@ -494,4 +486,3 @@ ENUM_CODEGENOPT(WinX64EHUnwindV2, llvm::WinX64EHUnwindV2Mode,
 #undef CODEGENOPT
 #undef ENUM_CODEGENOPT
 #undef VALUE_CODEGENOPT
-#undef AFFECTING_VALUE_CODEGENOPT
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 77a0c559f7689..fe634d221b424 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -41,13 +41,26 @@ class CodeGenOptionsBase {
   friend class CompilerInvocationBase;
 
 public:
-#define CODEGENOPT(Name, Bits, Default) unsigned Name : Bits;
-#define ENUM_CODEGENOPT(Name, Type, Bits, Default)
+  /// For ASTs produced with different option value, signifies their level of
+  /// compatibility.
+  enum class CompatibilityKind {
+    /// Does affect the construction of the AST in a way that does prevent
+    /// module interoperability.
+    Affecting,
+    /// Does not affect the construction of the AST in any way (that is, the
+    /// value can be different between an implicit module and the user of that
+    /// module).
+    Benign,
+  };
+
+#define CODEGENOPT(Name, Bits, Default, Compatibility) unsigned Name : Bits;
+#define ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility)
 #include "clang/Basic/CodeGenOptions.def"
 
 protected:
-#define CODEGENOPT(Name, Bits, Default)
-#define ENUM_CODEGENOPT(Name, Type, Bits, Default) unsigned Name : Bits;
+#define CODEGENOPT(Name, Bits, Default, Compatibility)
+#define ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility)              \
+  unsigned Name : Bits;
 #include "clang/Basic/CodeGenOptions.def"
 };
 
@@ -399,6 +412,8 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// with extra debug info.
   SanitizerSet SanitizeAnnotateDebugInfo;
 
+  std::optional<double> AllowRuntimeCheckSkipHotCutoff;
+
   /// List of backend command-line options for -fembed-bitcode.
   std::vector<uint8_t> CmdArgs;
 
@@ -505,9 +520,9 @@ class CodeGenOptions : public CodeGenOptionsBase {
 
 public:
   // Define accessors/mutators for code generation options of enumeration type.
-#define CODEGENOPT(Name, Bits, Default)
-#define ENUM_CODEGENOPT(Name, Type, Bits, Default) \
-  Type get##Name() const { return static_cast<Type>(Name); } \
+#define CODEGENOPT(Name, Bits, Default, Compatibility)
+#define ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility)              \
+  Type get##Name() const { return static_cast<Type>(Name); }                   \
   void set##Name(Type Value) { Name = static_cast<unsigned>(Value); }
 #include "clang/Basic/CodeGenOptions.def"
 
diff --git a/clang/include/clang/Basic/DebugOptions.def b/clang/include/clang/Basic/DebugOptions.def
index 7a9d2e838c1ca..e2ab06184b96a 100644
--- a/clang/include/clang/Basic/DebugOptions.def
+++ b/clang/include/clang/Basic/DebugOptions.def
@@ -12,143 +12,122 @@
 // options that have enumeration type), and VALUE_DEBUGOPT (is a debug option
 // that describes a value rather than a flag).
 //
-// BENIGN_ variants of the macros are used to describe options that do not
-// affect the generated PCM.
-//
 //===----------------------------------------------------------------------===//
 #ifndef DEBUGOPT
-#define DEBUGOPT(Name, Bits, Default) \
-CODEGENOPT(Name, Bits, Default)
+#define DEBUGOPT(Name, Bits, Default, Compatibility) \
+CODEGENOPT(Name, Bits, Default, Compatibility)
 #endif
 
 #ifndef VALUE_DEBUGOPT
-#  define VALUE_DEBUGOPT(Name, Bits, Default) \
-VALUE_CODEGENOPT(Name, Bits, Default)
+#  define VALUE_DEBUGOPT(Name, Bits, Default, Compatibility) \
+VALUE_CODEGENOPT(Name, Bits, Default, Compatibility)
 #endif
 
 #ifndef ENUM_DEBUGOPT
-#  define ENUM_DEBUGOPT(Name, Type, Bits, Default) \
-ENUM_CODEGENOPT(Name, Type, Bits, Default)
-#endif
-
-#ifndef BENIGN_DEBUGOPT
-#define BENIGN_DEBUGOPT(Name, Bits, Default) \
-DEBUGOPT(Name, Bits, Default)
-#endif
-
-#ifndef BENIGN_VALUE_DEBUGOPT
-#  define BENIGN_VALUE_DEBUGOPT(Name, Bits, Default) \
-VALUE_DEBUGOPT(Name, Bits, Default)
-#endif
-
-#ifndef BENIGN_ENUM_DEBUGOPT
-#  define BENIGN_ENUM_DEBUGOPT(Name, Type, Bits, Default) \
-ENUM_DEBUGOPT(Name, Type, Bits, Default)
+#  define ENUM_DEBUGOPT(Name, Type, Bits, Default, Compatibility) \
+ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility)
 #endif
 
-BENIGN_ENUM_DEBUGOPT(CompressDebugSections, llvm::DebugCompressionType, 2,
-                     llvm::DebugCompressionType::None)
-DEBUGOPT(Dwarf64, 1, 0) ///< -gdwarf64.
-BENIGN_DEBUGOPT(EnableDIPreservationVerify, 1, 0) ///< Enable di preservation
-                                                  ///< verify each (it means
-                                                  ///< check the original debug
-                                                  ///< info metadata
-                                                  ///< preservation).
-BENIGN_DEBUGOPT(ForceDwarfFrameSection , 1, 0) ///< Set when -fforce-dwarf-frame
-                                               ///< is enabled.
+ENUM_DEBUGOPT(CompressDebugSections, llvm::DebugCompressionType, 2,
+                     llvm::DebugCompressionType::None, Benign)
+DEBUGOPT(Dwarf64, 1, 0, Affecting) ///< -gdwarf64.
+DEBUGOPT(EnableDIPreservationVerify, 1, 0, Benign) ///< Enable di preservation
+                                                   ///< verify each (it means
+                                                   ///< check the original debug
+                                                   ///< info metadata
+                                                   ///< preservation).
+DEBUGOPT(ForceDwarfFrameSection , 1, 0, Benign) ///< Set when -fforce-dwarf-frame
+                                                ///< is enabled.
 
 ///< Set when -femit-dwarf-unwind is passed.
-BENIGN_ENUM_DEBUGOPT(EmitDwarfUnwind, llvm::EmitDwarfUnwindType, 2,
-                     llvm::EmitDwarfUnwindType::Default)
+ENUM_DEBUGOPT(EmitDwarfUnwind, llvm::EmitDwarfUnwindType, 2,
+              llvm::EmitDwarfUnwindType::Default, Benign)
 
-BENIGN_DEBUGOPT(NoDwarfDirectoryAsm , 1, 0) ///< Set when -fno-dwarf-directory-asm
-                                            ///< is enabled.
+DEBUGOPT(NoDwarfDirectoryAsm , 1, 0, Benign) ///< Set when -fno-dwarf-directory-asm
+                                             ///< is enabled.
 
-BENIGN_DEBUGOPT(NoInlineLineTables, 1, 0) ///< Whether debug info should contain
-                                          ///< inline line tables.
+DEBUGOPT(NoInlineLineTables, 1, 0, Benign) ///< Whether debug info should contain
+                                           ///< inline line tables.
 
-DEBUGOPT(DebugStrictDwarf, 1, 1) ///< Whether or not to use strict DWARF info.
-DEBUGOPT(DebugOmitUnreferencedMethods, 1, 0) ///< Omit unreferenced member
-					     ///< functions in type debug info.
+DEBUGOPT(DebugStrictDwarf, 1, 1, Affecting) ///< Whether or not to use strict DWARF info.
+DEBUGOPT(DebugOmitUnreferencedMethods, 1, 0, Affecting) ///< Omit unreferenced member
+                                                        ///< functions in type debug info.
 
 /// Control the Assignment Tracking debug info feature.
-BENIGN_ENUM_DEBUGOPT(AssignmentTrackingMode, AssignmentTrackingOpts, 2,
-                     AssignmentTrackingOpts::Disabled)
+ENUM_DEBUGOPT(AssignmentTrackingMode, AssignmentTrackingOpts, 2,
+              AssignmentTrackingOpts::Disabled, Benign)
 
 /// Whether or not to use Key Instructions to determine breakpoint locations.
-BENIGN_DEBUGOPT(DebugKeyInstructions, 1, 0)
+DEBUGOPT(DebugKeyInstructions, 1, 0, Benign)
 
-DEBUGOPT(DebugColumnInfo, 1, 0) ///< Whether or not to use column information
-                                ///< in debug info.
+DEBUGOPT(DebugColumnInfo, 1, 0, Affecting) ///< Whether or not to use column information
+                                           ///< in debug info.
 
-DEBUGOPT(DebugTypeExtRefs, 1, 0) ///< Whether or not debug info should contain
-                                 ///< external references to a PCH or module.
+DEBUGOPT(DebugTypeExtRefs, 1, 0, Affecting) ///< Whether or not debug info should contain
+                                            ///< external references to a PCH or module.
 
-DEBUGOPT(DebugExplicitImport, 1, 0)  ///< Whether or not debug info should
-                                     ///< contain explicit imports for
-                                     ///< anonymous namespaces
+DEBUGOPT(DebugExplicitImport, 1, 0, Affecting)  ///< Whether or not debug info should
+                                                ///< contain explicit imports for
+                                                ///< anonymous namespaces
 
 /// Set debug info source file hashing algorithm.
-ENUM_DEBUGOPT(DebugSrcHash, DebugSrcHashKind, 2, DSH_MD5)
-
-DEBUGOPT(SplitDwarfInlining, 1, 1) ///< Whether to include inlining info in the
-                                   ///< skeleton CU to allow for symbolication
-                                   ///< of inline stack frames without .dwo files.
-DEBUGOPT(DebugFwdTemplateParams, 1, 0) ///< Whether to emit complete
-                                       ///< template parameter descriptions in
-                                       ///< forward declarations (versus just
-                                       ///< including them in the name).
+ENUM_DEBUGOPT(DebugSrcHash, DebugSrcHashKind, 2, DSH_MD5, Affecting)
+
+DEBUGOPT(SplitDwarfInlining, 1, 1, Affecting) ///< Whether to include inlining info in the
+                                              ///< skeleton CU to allow for symbolication
+                                              ///< of inline stack frames without .dwo files.
+DEBUGOPT(DebugFwdTemplateParams, 1, 0, Affecting) ///< Whether to emit complete
+                                                  ///< template parameter descriptions in
+                                                  ///< forward declarations (versus just
+                                                  ///< including them in the name).
 ENUM_DEBUGOPT(DebugSimpleTemplateNames,
               llvm::codegenoptions::DebugTemplateNamesKind, 2,
-              llvm::codegenoptions::DebugTemplateNamesKind::Full)
+              llvm::codegenoptions::DebugTemplateNamesKind::Full, Affecting)
               ///< Whether to emit template parameters in the textual names of
               ///< template specializations.
-			  ///< Implies DebugFwdTemplateNames to allow decorated names to be
-			  ///< reconstructed when needed.
+              ///< Implies DebugFwdTemplateNames to allow decorated names to be
+              ///< reconstructed when needed.
 
 /// The kind of generated debug info.
 ENUM_DEBUGOPT(DebugInfo, llvm::codegenoptions::DebugInfoKind, 4,
-              llvm::codegenoptions::NoDebugInfo)
+              llvm::codegenoptions::NoDebugInfo, Affecting)
 
 /// Whether to generate macro debug info.
-DEBUGOPT(MacroDebugInfo, 1, 0)
+DEBUGOPT(MacroDebugInfo, 1, 0, Affecting)
 
 /// Tune the debug info for this debugger.
 ENUM_DEBUGOPT(DebuggerTuning, llvm::DebuggerKind, 3,
-                llvm::DebuggerKind::Default)
+              llvm::DebuggerKind::Default, Affecting)
 
 /// Dwarf version. Version zero indicates to LLVM that no DWARF should be
 /// emitted.
-VALUE_DEBUGOPT(DwarfVersion, 3, 0)
+VALUE_DEBUGOPT(DwarfVersion, 3, 0, Affecting)
 
 /// Whether we should emit CodeView debug information. It's possible to emit
 /// CodeView and DWARF into the same object.
-DEBUGOPT(EmitCodeView, 1, 0)
+DEBUGOPT(EmitCodeView, 1, 0, Affecting)
 
 /// Whether to emit the .debug$H section containing hashes of CodeView types.
-DEBUGOPT(CodeViewGHash, 1, 0)
+DEBUGOPT(CodeViewGHash, 1, 0, Affecting)
 
 /// Whether to emit the compiler path and command line into the CodeView debug information.
-DEBUGOPT(CodeViewCommandLine, 1, 0)
+DEBUGOPT(CodeViewCommandLine, 1, 0, Affecting)
 
 /// Whether emit extra debug info for sample pgo profile collection.
-DEBUGOPT(DebugInfoForProfiling, 1, 0)
+DEBUGOPT(DebugInfoForProfiling, 1, 0, Affecting)
 
 /// Whether to emit DW_TAG_template_alias for template aliases.
-DEBUGOPT(DebugTemplateAlias, 1, 0)
+DEBUGOPT(DebugTemplateAlias, 1, 0, Affecting)
 
 /// Whether to emit .debug_gnu_pubnames section instead of .debug_pubnames.
-DEBUGOPT(DebugNameTable, 2, 0)
+DEBUGOPT(DebugNameTable, 2, 0, Affecting)
 
 /// Whether to use DWARF base address specifiers in .debug_ranges.
-DEBUGOPT(DebugRangesBaseAddress, 1, 0)
+DEBUGOPT(DebugRangesBaseAddress, 1, 0, Affecting)
 
 /// Whether to embed source in DWARF debug line section.
-DEBUGOPT(EmbedSource, 1, 0)
+DEBUGOPT(EmbedSource, 1, 0, Affecting)
 
 #undef DEBUGOPT
 #undef ENUM_DEBUGOPT
 #undef VALUE_DEBUGOPT
-#undef BENIGN_DEBUGOPT
-#undef BENIGN_ENUM_DEBUGOPT
-#undef BENIGN_VALUE_DEBUGOPT
diff --git a/clang/include/clang/Basic/Diagnostic.h b/clang/include/clang/Basic/Diagnostic.h
index 7ae4ef7df138c..c7a627600f3cc 100644
--- a/clang/include/clang/Basic/Diagnostic.h
+++ b/clang/include/clang/Basic/Diagnostic.h
@@ -289,6 +289,9 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
 
     /// Expr *
     ak_expr,
+
+    /// AttributeCommonInfo *
+    ak_attr_info,
   };
 
   /// Represents on argument value, which is a union discriminated
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index d2cd86d05d55a..a67b9995d3b54 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -240,6 +240,8 @@ def note_constexpr_access_static_temporary : Note<
   "outside the expression that created the temporary">;
 def note_constexpr_access_unreadable_object : Note<
   "%sub{access_kind}0 object '%1' whose value is not known">;
+def note_constexpr_access_unknown_variable : Note<
+  "%sub{access_kind}0 variable %1 whose value is not known">;
 def note_constexpr_access_deleted_object : Note<
   "%sub{access_kind}0 heap allocated object that has been deleted">;
 def note_constexpr_modify_global : Note<
@@ -999,6 +1001,8 @@ def note_module_odr_violation_mismatch_decl_unknown : Note<
   "different friend declaration|different function template|different method|"
   "different instance variable|different property|another unexpected decl}2">;
 
+def err_struct_too_large : Error<
+  "structure '%0' is too large, which exceeds maximum allowed size of %1 bytes">;
 
 def remark_sanitize_address_insert_extra_padding_accepted : Remark<
     "-fsanitize-address-field-padding applied to %0">, ShowInSystemHeader,
diff --git a/clang/include/clang/Basic/DiagnosticCommentKinds.td b/clang/include/clang/Basic/DiagnosticCommentKinds.td
index 1122ace3027d8..0ce53b395ecee 100644
--- a/clang/include/clang/Basic/DiagnosticCommentKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommentKinds.td
@@ -90,8 +90,10 @@ def warn_doc_api_container_decl_mismatch : Warning<
   InGroup<Documentation>, DefaultIgnore;
 
 def warn_doc_container_decl_mismatch : Warning<
-  "'%select{\\|@}0%select{classdesign|coclass|dependency|helper"
-  "|helperclass|helps|instancesize|ownership|performance|security|superclass}1' "
+  "'%select{\\|@}0%enum_select<DocCommandKind>{%ClassDesign{classdesign}|"
+  "%CoClass{coclass}|%Dependency{dependency}|%Helper{helper}|%HelperClass{helperclass}|"
+  "%Helps{helps}|%InstanceSize{instancesize}|%Ownership{ownership}|"
+  "%Performance{performance}|%Security{security}|%Superclass{superclass}}1' "
   "command should not be used in a comment attached to a non-container declaration">,
   InGroup<Documentation>, DefaultIgnore;
 
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 68f87ebb1b39f..34b6c0d7a8acd 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -847,6 +847,9 @@ def note_drv_available_multilibs : Note<
   "available multilibs are:%0">;
 def err_drv_multilib_custom_error : Error<
   "multilib configuration error: %0">;
+def warn_drv_multilib_not_available_for_target: Warning<
+  "no multilib structure encoded for Arm, Aarch64 and PPC targets">,
+  InGroup<DiagGroup<"multilib-not-found">>;
 
 def err_drv_experimental_crel : Error<
   "-Wa,--allow-experimental-crel must be specified to use -Wa,--crel. "
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 36fa3227fd6a6..f54a830b0103e 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1450,7 +1450,6 @@ def MicrosoftAnonTag : DiagGroup<"microsoft-anon-tag">;
 def MicrosoftCommentPaste : DiagGroup<"microsoft-comment-paste">;
 def MicrosoftEndOfFile : DiagGroup<"microsoft-end-of-file">;
 def MicrosoftInaccessibleBase : DiagGroup<"microsoft-inaccessible-base">;
-def MicrosoftStaticAssert : DiagGroup<"microsoft-static-assert">;
 def MicrosoftInitFromPredefined : DiagGroup<"microsoft-init-from-predefined">;
 def MicrosoftStringLiteralFromPredefined : DiagGroup<
     "microsoft-string-literal-from-predefined">;
@@ -1472,7 +1471,7 @@ def Microsoft : DiagGroup<"microsoft",
      MicrosoftRedeclareStatic, MicrosoftEnumForwardReference, MicrosoftGoto,
      MicrosoftFlexibleArray, MicrosoftExtraQualification, MicrosoftCast,
      MicrosoftConstInit, MicrosoftVoidPseudoDtor, MicrosoftAnonTag,
-     MicrosoftCommentPaste, MicrosoftEndOfFile, MicrosoftStaticAssert,
+     MicrosoftCommentPaste, MicrosoftEndOfFile,
      MicrosoftInitFromPredefined, MicrosoftStringLiteralFromPredefined,
      MicrosoftInconsistentDllImport, MicrosoftInlineOnNonFunction]>;
 
@@ -1530,9 +1529,11 @@ def OpenMPPre51Compat : DiagGroup<"pre-openmp-51-compat">;
 def OpenMP51Ext : DiagGroup<"openmp-51-extensions">;
 def OpenMPExtensions : DiagGroup<"openmp-extensions">;
 def OpenMPTargetException : DiagGroup<"openmp-target-exception">;
+def OpenMPFuture : DiagGroup<"openmp-future">;
 def OpenMP : DiagGroup<"openmp", [
     SourceUsesOpenMP, OpenMPClauses, OpenMPLoopForm, OpenMPTarget,
-    OpenMPMapping, OpenMP51Ext, OpenMPExtensions, OpenMPTargetException
+    OpenMPMapping, OpenMP51Ext, OpenMPExtensions, OpenMPTargetException,
+    OpenMPFuture
   ]>;
 
 // OpenACC warnings.
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 6c30da376dafb..4636eaf2291ab 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -474,9 +474,6 @@ def err_bool_redeclaration : Error<
 def warn_cxx98_compat_static_assert : Warning<
   "'static_assert' declarations are incompatible with C++98">,
   InGroup<CXX98Compat>, DefaultIgnore;
-def ext_ms_static_assert : ExtWarn<
-  "use of 'static_assert' without inclusion of <assert.h> is a Microsoft "
-  "extension">, InGroup<MicrosoftStaticAssert>;
 def ext_cxx_static_assert_no_message : ExtWarn<
   "'static_assert' with no message is a C++17 extension">, InGroup<CXX17>;
 def ext_c_static_assert_no_message : ExtWarn<
@@ -1488,6 +1485,9 @@ def err_omp_multiple_step_or_linear_modifier : Error<
   "multiple %select{'step size'|'linear modifier'}0 found in linear clause">;
 def err_omp_deprecate_old_syntax: Error<
   "old syntax '%0' on '%1' clause was deprecated, use new syntax '%2'">;
+def warn_omp_future_directive_spelling: Warning<
+  "directive spelling '%0' is introduced in a later OpenMP version">,
+  InGroup<OpenMPFuture>;
 def warn_pragma_expected_colon_r_paren : Warning<
   "missing ':' or ')' after %0 - ignoring">, InGroup<IgnoredPragmas>;
 def err_omp_unknown_directive : Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 5062505cf3c01..0b2553d82153c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1062,6 +1062,9 @@ def err_constexpr_main : Error<
   "'main' is not allowed to be declared %select{constexpr|consteval}0">;
 def err_deleted_main : Error<"'main' is not allowed to be deleted">;
 def err_mainlike_template_decl : Error<"%0 cannot be a template">;
+def warn_main_in_named_module
+    : ExtWarn<"'main' never has module linkage">,
+      InGroup<DiagGroup<"main-attached-to-named-module">>;
 def err_main_returns_nonint : Error<"'main' must return 'int'">;
 def ext_main_returns_nonint : ExtWarn<"return type of 'main' is not 'int'">,
     InGroup<MainReturnType>;
@@ -1084,9 +1087,9 @@ def warn_main_redefined : Warning<"variable named 'main' with external linkage "
     "has undefined behavior">, InGroup<Main>;
 def ext_main_used : Extension<
     "referring to 'main' within an expression is a Clang extension">, InGroup<Main>;
-def ext_main_invalid_linkage_specification : ExtWarn<
-  "'main' should not be "
-  "'extern \"%select{C|C++}0\"'">, InGroup<Main>;
+def ext_main_invalid_linkage_specification : ExtWarn<"'main' should not be "
+                                                     "'extern \"C\"'">,
+                                             InGroup<Main>;
 
 /// parser diagnostics
 def ext_no_declarators : ExtWarn<"declaration does not declare anything">,
@@ -1809,7 +1812,10 @@ def note_unsatisfied_trait_reason
            "%DeletedAssign{has a deleted %select{copy|move}1 "
            "assignment operator}|"
            "%UnionWithUserDeclaredSMF{is a union with a user-declared "
-           "%sub{select_special_member_kind}1}"
+           "%sub{select_special_member_kind}1}|"
+           "%FunctionType{is a function type}|"
+           "%CVVoidType{is a cv void type}|"
+           "%IncompleteArrayType{is an incomplete array type}"
            "}0">;
 
 def warn_consteval_if_always_true : Warning<
@@ -3287,10 +3293,11 @@ def err_attribute_wrong_number_arguments : Error<
 def err_attribute_wrong_number_arguments_for : Error <
   "%0 attribute references function %1, which %plural{0:takes no arguments|1:takes one argument|"
   ":takes exactly %2 arguments}2">;
-def err_callback_attribute_wrong_arg_count : Error<
-  "'callback' attribute references function of type %0 which expects %1 "
-  "%plural{1:argument|:arguments}1 but attribute specifies %2 parameter index "
-  "%plural{1:argument|:arguments}2">;
+def err_attribute_wrong_arg_count_for_func
+    : Error<"%0 attribute references function of type %1 which expects %2 "
+            "%plural{1:argument|:arguments}2 but attribute specifies %3 "
+            "parameter index "
+            "%plural{1:argument|:arguments}3">;
 def err_attribute_bounds_for_function : Error<
   "%0 attribute references parameter %1, but the function %2 has only %3 parameters">;
 def err_attribute_no_member_function : Error<
@@ -3329,6 +3336,9 @@ def err_sve_vector_in_non_sve_target : Error<
   "SVE vector type %0 cannot be used in a target without sve">;
 def err_sve_vector_in_non_streaming_function : Error<
   "SVE vector type %0 cannot be used in a non-streaming function">;
+def err_sve_fixed_vector_in_streaming_function
+    : Error<"fixed width SVE vector type %0 cannot be used in a "
+            "%select{streaming|streaming-compatible}1 function">;
 def err_attribute_riscv_rvv_bits_unsupported : Error<
   "%0 is only supported when '-mrvv-vector-bits=<bits>' is specified with a "
   "value of \"zvl\" or a power 2 in the range [64,65536]">;
@@ -4706,9 +4716,9 @@ def note_protocol_decl : Note<
   "protocol is declared here">;
 def note_protocol_decl_undefined : Note<
   "protocol %0 has no definition">;
-def err_attribute_preferred_name_arg_invalid : Error<
-  "argument %0 to 'preferred_name' attribute is not a typedef for "
-  "a specialization of %1">;
+def err_attribute_not_typedef_for_specialization
+    : Error<"argument %0 to %1 attribute is not a typedef for "
+            "a specialization of %2">;
 def err_attribute_builtin_alias : Error<
   "%0 attribute can only be applied to a ARM, HLSL, SPIR-V or RISC-V builtin">;
 
@@ -13214,8 +13224,12 @@ def err_acc_not_a_var_ref_use_device_declare
     : Error<"OpenACC variable %select{in 'use_device' clause|on 'declare' "
             "construct}0 is not a valid variable name or array name">;
 def err_acc_not_a_var_ref_cache
-    : Error<"OpenACC variable in cache directive is not a valid sub-array or "
+    : Error<"OpenACC variable in 'cache' directive is not a valid sub-array or "
             "array element">;
+def warn_acc_cache_var_not_outside_loop
+    : Warning<"OpenACC variable in 'cache' directive was not declared outside "
+              "of the associated 'loop' directive; directive has no effect">,
+      InGroup<DiagGroup<"openacc-cache-var-inside-loop">>;
 def err_acc_typecheck_subarray_value
     : Error<"OpenACC sub-array subscripted value is not an array or pointer">;
 def err_acc_subarray_function_type
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 789761c1f3647..72321c204ce96 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -9,31 +9,16 @@
 // This file defines the language options. Users of this file must
 // define the LANGOPT macro to make use of this information. The arguments to
 // the macro are:
-//   LANGOPT(Name, Bits, DefaultValue, Description)
+//   LANGOPT(Name, Bits, DefaultValue, Compatibility, Description)
 // Note that the DefaultValue must be a constant value (literal or enumeration);
 // it cannot depend on the value of another language option.
 //
 // Optionally, the user may also define:
 //
-// BENIGN_LANGOPT: for options that don't affect the construction of the AST in
-//     any way (that is, the value can be different between an implicit module
-//     and the user of that module).
-//
-// COMPATIBLE_LANGOPT: for options that affect the construction of the AST in
-//     a way that doesn't prevent interoperability (that is, the value can be
-//     different between an explicit module and the user of that module).
-//
 // ENUM_LANGOPT: for options that have enumeration, rather than unsigned, type.
 //
 // VALUE_LANGOPT: for options that describe a value rather than a flag.
 //
-// BENIGN_ENUM_LANGOPT, COMPATIBLE_ENUM_LANGOPT,
-// BENIGN_VALUE_LANGOPT, COMPATIBLE_VALUE_LANGOPT: combinations of the above.
-//
-// FIXME: Clients should be able to more easily select whether they want
-// different levels of compatibility versus how to handle different kinds
-// of option.
-//
 // The Description field should be a noun phrase, for instance "frobbing all
 // widgets" or "C's implicit blintz feature".
 //===----------------------------------------------------------------------===//
@@ -42,504 +27,471 @@
 #  error Define the LANGOPT macro to handle language options
 #endif
 
-#ifndef COMPATIBLE_LANGOPT
-#  define COMPATIBLE_LANGOPT(Name, Bits, Default, Description) \
-     LANGOPT(Name, Bits, Default, Description)
-#endif
-
-#ifndef BENIGN_LANGOPT
-#  define BENIGN_LANGOPT(Name, Bits, Default, Description) \
-     COMPATIBLE_LANGOPT(Name, Bits, Default, Description)
-#endif
-
 #ifndef ENUM_LANGOPT
-#  define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-     LANGOPT(Name, Bits, Default, Description)
-#endif
-
-#ifndef COMPATIBLE_ENUM_LANGOPT
-#  define COMPATIBLE_ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-     ENUM_LANGOPT(Name, Type, Bits, Default, Description)
-#endif
-
-#ifndef BENIGN_ENUM_LANGOPT
-#  define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-     COMPATIBLE_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+#  define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description) \
+     LANGOPT(Name, Bits, Default, Compatibility, Description)
 #endif
 
 #ifndef VALUE_LANGOPT
-#  define VALUE_LANGOPT(Name, Bits, Default, Description) \
-     LANGOPT(Name, Bits, Default, Description)
-#endif
-
-#ifndef COMPATIBLE_VALUE_LANGOPT
-#  define COMPATIBLE_VALUE_LANGOPT(Name, Bits, Default, Description) \
-     VALUE_LANGOPT(Name, Bits, Default, Description)
-#endif
-
-#ifndef BENIGN_VALUE_LANGOPT
-#  define BENIGN_VALUE_LANGOPT(Name, Bits, Default, Description) \
-     COMPATIBLE_VALUE_LANGOPT(Name, Bits, Default, Description)
+#  define VALUE_LANGOPT(Name, Bits, Default, Compatibility, Description) \
+     LANGOPT(Name, Bits, Default, Compatibility, Description)
 #endif
 
-// FIXME: A lot of the BENIGN_ options should be COMPATIBLE_ instead.
-LANGOPT(C99               , 1, 0, "C99")
-LANGOPT(C11               , 1, 0, "C11")
-LANGOPT(C17               , 1, 0, "C17")
-LANGOPT(C23               , 1, 0, "C23")
-LANGOPT(C2y               , 1, 0, "C2y")
-LANGOPT(MSVCCompat        , 1, 0, "Microsoft Visual C++ full compatibility mode")
-LANGOPT(Kernel            , 1, 0, "Kernel mode")
-LANGOPT(MicrosoftExt      , 1, 0, "Microsoft C++ extensions")
-LANGOPT(ZOSExt            , 1, 0, "z/OS extensions")
-LANGOPT(AsmBlocks         , 1, 0, "Microsoft inline asm blocks")
-LANGOPT(Borland           , 1, 0, "Borland extensions")
-LANGOPT(CPlusPlus         , 1, 0, "C++")
-LANGOPT(CPlusPlus11       , 1, 0, "C++11")
-LANGOPT(CPlusPlus14       , 1, 0, "C++14")
-LANGOPT(CPlusPlus17       , 1, 0, "C++17")
-LANGOPT(CPlusPlus20       , 1, 0, "C++20")
-LANGOPT(CPlusPlus23       , 1, 0, "C++23")
-LANGOPT(CPlusPlus26       , 1, 0, "C++26")
-LANGOPT(ObjC              , 1, 0, "Objective-C")
-BENIGN_LANGOPT(ObjCDefaultSynthProperties , 1, 0,
-               "Objective-C auto-synthesized properties")
-BENIGN_LANGOPT(EncodeExtendedBlockSig , 1, 0,
-               "Encoding extended block type signature")
-BENIGN_LANGOPT(EncodeCXXClassTemplateSpec , 1, 0,
-               "Fully encode c++ class template specialization")
-BENIGN_LANGOPT(ObjCInferRelatedResultType , 1, 1,
-               "Objective-C related result type inference")
-LANGOPT(AppExt , 1, 0, "Objective-C App Extension")
-LANGOPT(Trigraphs         , 1, 0,"trigraphs")
-LANGOPT(LineComment       , 1, 0, "'//' comments")
-LANGOPT(Bool              , 1, 0, "bool, true, and false keywords")
-LANGOPT(Half              , 1, 0, "half keyword")
-LANGOPT(WChar             , 1, 0, "wchar_t keyword")
-LANGOPT(Char8             , 1, 0, "char8_t keyword")
-LANGOPT(IEEE128           , 1, 0, "__ieee128 keyword")
-LANGOPT(DeclSpecKeyword   , 1, 0, "__declspec keyword")
-BENIGN_LANGOPT(DollarIdents   , 1, 1, "'$' in identifiers")
-BENIGN_LANGOPT(AsmPreprocessor, 1, 0, "preprocessor in asm mode")
-LANGOPT(GNUMode           , 1, 1, "GNU extensions")
-LANGOPT(GNUKeywords       , 1, 1, "GNU keywords")
-VALUE_LANGOPT(GNUCVersion , 32, 0, "GNU C compatibility version")
-LANGOPT(DisableKNRFunctions, 1, 0, "require function types to have a prototype")
-LANGOPT(Digraphs          , 1, 0, "digraphs")
-BENIGN_LANGOPT(HexFloats  , 1, 0, "C99 hexadecimal float constants")
-LANGOPT(CXXOperatorNames  , 1, 0, "C++ operator name keywords")
-LANGOPT(AppleKext         , 1, 0, "Apple kext support")
-BENIGN_LANGOPT(PascalStrings, 1, 0, "Pascal string support")
-LANGOPT(WritableStrings   , 1, 0, "writable string support")
-LANGOPT(ConstStrings      , 1, 0, "const-qualified string support")
+// FIXME: A lot of the Benign options should be Compatible instead.
+LANGOPT(C99               , 1, 0, NotCompatible, "C99")
+LANGOPT(C11               , 1, 0, NotCompatible, "C11")
+LANGOPT(C17               , 1, 0, NotCompatible, "C17")
+LANGOPT(C23               , 1, 0, NotCompatible, "C23")
+LANGOPT(C2y               , 1, 0, NotCompatible, "C2y")
+LANGOPT(MSVCCompat        , 1, 0, NotCompatible, "Microsoft Visual C++ full compatibility mode")
+LANGOPT(Kernel            , 1, 0, NotCompatible, "Kernel mode")
+LANGOPT(MicrosoftExt      , 1, 0, NotCompatible, "Microsoft C++ extensions")
+LANGOPT(ZOSExt            , 1, 0, NotCompatible, "z/OS extensions")
+LANGOPT(AsmBlocks         , 1, 0, NotCompatible, "Microsoft inline asm blocks")
+LANGOPT(Borland           , 1, 0, NotCompatible, "Borland extensions")
+LANGOPT(CPlusPlus         , 1, 0, NotCompatible, "C++")
+LANGOPT(CPlusPlus11       , 1, 0, NotCompatible, "C++11")
+LANGOPT(CPlusPlus14       , 1, 0, NotCompatible, "C++14")
+LANGOPT(CPlusPlus17       , 1, 0, NotCompatible, "C++17")
+LANGOPT(CPlusPlus20       , 1, 0, NotCompatible, "C++20")
+LANGOPT(CPlusPlus23       , 1, 0, NotCompatible, "C++23")
+LANGOPT(CPlusPlus26       , 1, 0, NotCompatible, "C++26")
+LANGOPT(ObjC              , 1, 0, NotCompatible, "Objective-C")
+LANGOPT(ObjCDefaultSynthProperties , 1, 0, Benign,
+        "Objective-C auto-synthesized properties")
+LANGOPT(EncodeExtendedBlockSig , 1, 0, Benign,
+        "Encoding extended block type signature")
+LANGOPT(EncodeCXXClassTemplateSpec , 1, 0, Benign,
+        "Fully encode c++ class template specialization")
+LANGOPT(ObjCInferRelatedResultType , 1, 1, Benign,
+        "Objective-C related result type inference")
+LANGOPT(AppExt            , 1, 0, NotCompatible, "Objective-C App Extension")
+LANGOPT(Trigraphs         , 1, 0, NotCompatible, "trigraphs")
+LANGOPT(LineComment       , 1, 0, NotCompatible, "'//' comments")
+LANGOPT(Bool              , 1, 0, NotCompatible, "bool, true, and false keywords")
+LANGOPT(Half              , 1, 0, NotCompatible, "half keyword")
+LANGOPT(WChar             , 1, 0, NotCompatible, "wchar_t keyword")
+LANGOPT(Char8             , 1, 0, NotCompatible, "char8_t keyword")
+LANGOPT(IEEE128           , 1, 0, NotCompatible, "__ieee128 keyword")
+LANGOPT(DeclSpecKeyword   , 1, 0, NotCompatible, "__declspec keyword")
+LANGOPT(DollarIdents      , 1, 1, Benign, "'$' in identifiers")
+LANGOPT(AsmPreprocessor   , 1, 0, Benign, "preprocessor in asm mode")
+LANGOPT(GNUMode           , 1, 1, NotCompatible, "GNU extensions")
+LANGOPT(GNUKeywords       , 1, 1, NotCompatible, "GNU keywords")
+VALUE_LANGOPT(GNUCVersion , 32, 0, NotCompatible, "GNU C compatibility version")
+LANGOPT(DisableKNRFunctions, 1, 0, NotCompatible, "require function types to have a prototype")
+LANGOPT(Digraphs          , 1, 0, NotCompatible, "digraphs")
+LANGOPT(HexFloats         , 1, 0, Benign, "C99 hexadecimal float constants")
+LANGOPT(CXXOperatorNames  , 1, 0, NotCompatible, "C++ operator name keywords")
+LANGOPT(AppleKext         , 1, 0, NotCompatible, "Apple kext support")
+LANGOPT(PascalStrings     , 1, 0, Benign, "Pascal string support")
+LANGOPT(WritableStrings   , 1, 0, NotCompatible, "writable string support")
+LANGOPT(ConstStrings      , 1, 0, NotCompatible, "const-qualified string support")
 ENUM_LANGOPT(LaxVectorConversions, LaxVectorConversionKind, 2,
-             LaxVectorConversionKind::All, "lax vector conversions")
+             LaxVectorConversionKind::All, NotCompatible, "lax vector conversions")
 ENUM_LANGOPT(AltivecSrcCompat, AltivecSrcCompatKind, 2,
-             AltivecSrcCompatKind::Default, "Altivec source compatibility")
-LANGOPT(ConvergentFunctions, 1, 1, "Assume convergent functions")
-LANGOPT(AltiVec           , 1, 0, "AltiVec-style vector initializers")
-LANGOPT(ZVector           , 1, 0, "System z vector extensions")
-LANGOPT(Exceptions        , 1, 0, "exception handling")
-LANGOPT(ObjCExceptions    , 1, 0, "Objective-C exceptions")
-LANGOPT(CXXExceptions     , 1, 0, "C++ exceptions")
-LANGOPT(EHAsynch          , 1, 0, "C/C++ EH Asynch exceptions")
+             AltivecSrcCompatKind::Default, NotCompatible, "Altivec source compatibility")
+LANGOPT(ConvergentFunctions, 1, 1, NotCompatible, "Assume convergent functions")
+LANGOPT(AltiVec           , 1, 0, NotCompatible, "AltiVec-style vector initializers")
+LANGOPT(ZVector           , 1, 0, NotCompatible, "System z vector extensions")
+LANGOPT(Exceptions        , 1, 0, NotCompatible, "exception handling")
+LANGOPT(ObjCExceptions    , 1, 0, NotCompatible, "Objective-C exceptions")
+LANGOPT(CXXExceptions     , 1, 0, NotCompatible, "C++ exceptions")
+LANGOPT(EHAsynch          , 1, 0, NotCompatible, "C/C++ EH Asynch exceptions")
 ENUM_LANGOPT(ExceptionHandling, ExceptionHandlingKind, 3,
-             ExceptionHandlingKind::None, "exception handling")
-LANGOPT(IgnoreExceptions  , 1, 0, "ignore exceptions")
-LANGOPT(ExternCNoUnwind   , 1, 0, "Assume extern C functions don't unwind")
-LANGOPT(AssumeNothrowExceptionDtor , 1, 0, "Assume exception object's destructor is nothrow")
-LANGOPT(TraditionalCPP    , 1, 0, "traditional CPP emulation")
-LANGOPT(RTTI              , 1, 1, "run-time type information")
-LANGOPT(RTTIData          , 1, 1, "emit run-time type information data")
-LANGOPT(MSBitfields       , 1, 0, "Microsoft-compatible structure layout")
-LANGOPT(MSVolatile        , 1, 0, "Microsoft-compatible volatile loads and stores")
-LANGOPT(Freestanding, 1, 0, "freestanding implementation")
-LANGOPT(NoBuiltin         , 1, 0, "disable builtin functions")
-LANGOPT(NoMathBuiltin     , 1, 0, "disable math builtin functions")
-LANGOPT(GNUAsm            , 1, 1, "GNU-style inline assembly")
-LANGOPT(Coroutines        , 1, 0, "C++20 coroutines")
-LANGOPT(CoroAlignedAllocation, 1, 0, "prefer Aligned Allocation according to P2014 Option 2")
-LANGOPT(DllExportInlines  , 1, 1, "dllexported classes dllexport inline methods")
-LANGOPT(ExperimentalLibrary, 1, 0, "enable unstable and experimental library features")
-
-LANGOPT(PointerAuthIntrinsics, 1, 0, "pointer authentication intrinsics")
-LANGOPT(PointerAuthCalls  , 1, 0, "function pointer authentication")
-LANGOPT(PointerAuthReturns, 1, 0, "return pointer authentication")
-LANGOPT(PointerAuthIndirectGotos, 1, 0, "indirect gotos pointer authentication")
-LANGOPT(PointerAuthAuthTraps, 1, 0, "pointer authentication failure traps")
-LANGOPT(PointerAuthVTPtrAddressDiscrimination, 1, 0, "incorporate address discrimination in authenticated vtable pointers")
-LANGOPT(PointerAuthVTPtrTypeDiscrimination, 1, 0, "incorporate type discrimination in authenticated vtable pointers")
-LANGOPT(PointerAuthTypeInfoVTPtrDiscrimination, 1, 0, "incorporate type and address discrimination in authenticated vtable pointers for std::type_info")
-BENIGN_LANGOPT(PointerAuthFunctionTypeDiscrimination, 1, 0,
-               "Use type discrimination when signing function pointers")
-LANGOPT(PointerAuthInitFini, 1, 0, "sign function pointers in init/fini arrays")
-LANGOPT(PointerAuthInitFiniAddressDiscrimination, 1, 0,
+             ExceptionHandlingKind::None,  NotCompatible, "exception handling")
+LANGOPT(IgnoreExceptions  , 1, 0, NotCompatible, "ignore exceptions")
+LANGOPT(ExternCNoUnwind   , 1, 0, NotCompatible, "Assume extern C functions don't unwind")
+LANGOPT(AssumeNothrowExceptionDtor , 1, 0, NotCompatible, "Assume exception object's destructor is nothrow")
+LANGOPT(TraditionalCPP    , 1, 0, NotCompatible, "traditional CPP emulation")
+LANGOPT(RTTI              , 1, 1, NotCompatible, "run-time type information")
+LANGOPT(RTTIData          , 1, 1, NotCompatible, "emit run-time type information data")
+LANGOPT(MSBitfields       , 1, 0, NotCompatible, "Microsoft-compatible structure layout")
+LANGOPT(MSVolatile        , 1, 0, NotCompatible, "Microsoft-compatible volatile loads and stores")
+LANGOPT(Freestanding      , 1, 0, NotCompatible, "freestanding implementation")
+LANGOPT(NoBuiltin         , 1, 0, NotCompatible, "disable builtin functions")
+LANGOPT(NoMathBuiltin     , 1, 0, NotCompatible, "disable math builtin functions")
+LANGOPT(GNUAsm            , 1, 1, NotCompatible, "GNU-style inline assembly")
+LANGOPT(Coroutines        , 1, 0, NotCompatible, "C++20 coroutines")
+LANGOPT(CoroAlignedAllocation, 1, 0, NotCompatible, "prefer Aligned Allocation according to P2014 Option 2")
+LANGOPT(DllExportInlines  , 1, 1, NotCompatible, "dllexported classes dllexport inline methods")
+LANGOPT(ExperimentalLibrary, 1, 0, NotCompatible, "enable unstable and experimental library features")
+
+LANGOPT(PointerAuthIntrinsics, 1, 0, NotCompatible, "pointer authentication intrinsics")
+LANGOPT(PointerAuthCalls  , 1, 0, NotCompatible, "function pointer authentication")
+LANGOPT(PointerAuthReturns, 1, 0, NotCompatible, "return pointer authentication")
+LANGOPT(PointerAuthIndirectGotos, 1, 0, NotCompatible, "indirect gotos pointer authentication")
+LANGOPT(PointerAuthAuthTraps, 1, 0, NotCompatible, "pointer authentication failure traps")
+LANGOPT(PointerAuthVTPtrAddressDiscrimination, 1, 0, NotCompatible, "incorporate address discrimination in authenticated vtable pointers")
+LANGOPT(PointerAuthVTPtrTypeDiscrimination, 1, 0, NotCompatible, "incorporate type discrimination in authenticated vtable pointers")
+LANGOPT(PointerAuthTypeInfoVTPtrDiscrimination, 1, 0, NotCompatible, "incorporate type and address discrimination in authenticated vtable pointers for std::type_info")
+LANGOPT(PointerAuthFunctionTypeDiscrimination, 1, 0, Benign,
+        "Use type discrimination when signing function pointers")
+LANGOPT(PointerAuthInitFini, 1, 0, NotCompatible, "sign function pointers in init/fini arrays")
+LANGOPT(PointerAuthInitFiniAddressDiscrimination, 1, 0, NotCompatible,
         "incorporate address discrimination in authenticated function pointers in init/fini arrays")
-LANGOPT(PointerAuthELFGOT, 1, 0, "authenticate pointers from GOT")
-LANGOPT(AArch64JumpTableHardening, 1, 0, "use hardened lowering for jump-table dispatch")
-
-LANGOPT(DoubleSquareBracketAttributes, 1, 0, "'[[]]' attributes extension for all language standard modes")
-LANGOPT(ExperimentalLateParseAttributes, 1, 0, "experimental late parsing of attributes")
-
-COMPATIBLE_LANGOPT(RecoveryAST, 1, 1, "Preserve expressions in AST when encountering errors")
-COMPATIBLE_LANGOPT(RecoveryASTType, 1, 1, "Preserve the type in recovery expressions")
-
-BENIGN_LANGOPT(ThreadsafeStatics , 1, 1, "thread-safe static initializers")
-LANGOPT(POSIXThreads      , 1, 0, "POSIX thread support")
-LANGOPT(Blocks            , 1, 0, "blocks extension to C")
-BENIGN_LANGOPT(EmitAllDecls      , 1, 0, "emitting all declarations")
-LANGOPT(MathErrno         , 1, 1, "errno in math functions")
-LANGOPT(Modules           , 1, 0, "modules semantics")
-COMPATIBLE_LANGOPT(CPlusPlusModules, 1, 0, "C++ modules syntax")
-LANGOPT(SkipODRCheckInGMF, 1, 0, "Skip ODR checks for decls in the global module fragment")
-LANGOPT(BuiltinHeadersInSystemModules, 1, 0, "builtin headers belong to system modules, and _Builtin_ modules are ignored for cstdlib headers")
-BENIGN_ENUM_LANGOPT(CompilingModule, CompilingModuleKind, 3, CMK_None,
-                    "compiling a module interface")
-BENIGN_LANGOPT(CompilingPCH, 1, 0, "building a pch")
-BENIGN_LANGOPT(BuildingPCHWithObjectFile, 1, 0, "building a pch which has a corresponding object file")
-BENIGN_LANGOPT(CacheGeneratedPCH, 1, 0, "cache generated PCH files in memory")
-BENIGN_LANGOPT(PCHInstantiateTemplates, 1, 0, "instantiate templates while building a PCH")
-COMPATIBLE_LANGOPT(ModulesDeclUse    , 1, 0, "require declaration of module uses")
-BENIGN_LANGOPT(ModulesSearchAll  , 1, 1, "searching even non-imported modules to find unresolved references")
-COMPATIBLE_LANGOPT(ModulesStrictDeclUse, 1, 0, "requiring declaration of module uses and all headers to be in modules")
-COMPATIBLE_LANGOPT(ModulesValidateTextualHeaderIncludes, 1, 1, "validation of textual header includes")
-BENIGN_LANGOPT(ModulesErrorRecovery, 1, 1, "automatically importing modules as needed when performing error recovery")
-BENIGN_LANGOPT(ImplicitModules, 1, 1, "building modules that are not specified via -fmodule-file")
-COMPATIBLE_LANGOPT(ModulesLocalVisibility, 1, 0, "local submodule visibility")
-COMPATIBLE_LANGOPT(Optimize          , 1, 0, "__OPTIMIZE__ predefined macro")
-COMPATIBLE_LANGOPT(OptimizeSize      , 1, 0, "__OPTIMIZE_SIZE__ predefined macro")
-COMPATIBLE_LANGOPT(Static            , 1, 0, "__STATIC__ predefined macro (as opposed to __DYNAMIC__)")
-VALUE_LANGOPT(PackStruct  , 32, 0,
+LANGOPT(PointerAuthELFGOT, 1, 0, NotCompatible, "authenticate pointers from GOT")
+LANGOPT(AArch64JumpTableHardening, 1, 0, NotCompatible, "use hardened lowering for jump-table dispatch")
+
+LANGOPT(DoubleSquareBracketAttributes, 1, 0, NotCompatible, "'[[]]' attributes extension for all language standard modes")
+LANGOPT(ExperimentalLateParseAttributes, 1, 0, NotCompatible, "experimental late parsing of attributes")
+
+LANGOPT(RecoveryAST, 1, 1, Compatible, "Preserve expressions in AST when encountering errors")
+LANGOPT(RecoveryASTType, 1, 1, Compatible, "Preserve the type in recovery expressions")
+
+LANGOPT(ThreadsafeStatics , 1, 1, Benign, "thread-safe static initializers")
+LANGOPT(POSIXThreads      , 1, 0, NotCompatible, "POSIX thread support")
+LANGOPT(Blocks            , 1, 0, NotCompatible, "blocks extension to C")
+LANGOPT(EmitAllDecls      , 1, 0, Benign, "emitting all declarations")
+LANGOPT(MathErrno         , 1, 1, NotCompatible, "errno in math functions")
+LANGOPT(Modules           , 1, 0, NotCompatible, "modules semantics")
+LANGOPT(CPlusPlusModules  , 1, 0, Compatible, "C++ modules syntax")
+LANGOPT(SkipODRCheckInGMF , 1, 0, NotCompatible, "Skip ODR checks for decls in the global module fragment")
+LANGOPT(BuiltinHeadersInSystemModules, 1, 0, NotCompatible, "builtin headers belong to system modules, and _Builtin_ modules are ignored for cstdlib headers")
+ENUM_LANGOPT(CompilingModule, CompilingModuleKind, 3, CMK_None, Benign,
+             "compiling a module interface")
+LANGOPT(CompilingPCH, 1, 0, Benign, "building a pch")
+LANGOPT(BuildingPCHWithObjectFile, 1, 0, Benign, "building a pch which has a corresponding object file")
+LANGOPT(CacheGeneratedPCH, 1, 0, Benign, "cache generated PCH files in memory")
+LANGOPT(PCHInstantiateTemplates, 1, 0, Benign, "instantiate templates while building a PCH")
+LANGOPT(ModulesDeclUse    , 1, 0, Compatible, "require declaration of module uses")
+LANGOPT(ModulesSearchAll  , 1, 1, Benign, "searching even non-imported modules to find unresolved references")
+LANGOPT(ModulesStrictDeclUse, 1, 0, Compatible, "requiring declaration of module uses and all headers to be in modules")
+LANGOPT(ModulesValidateTextualHeaderIncludes, 1, 1, Compatible, "validation of textual header includes")
+LANGOPT(ModulesErrorRecovery, 1, 1, Benign, "automatically importing modules as needed when performing error recovery")
+LANGOPT(ImplicitModules, 1, 1, Benign, "building modules that are not specified via -fmodule-file")
+LANGOPT(ModulesLocalVisibility, 1, 0, Compatible, "local submodule visibility")
+LANGOPT(Optimize          , 1, 0, Compatible, "__OPTIMIZE__ predefined macro")
+LANGOPT(OptimizeSize      , 1, 0, Compatible, "__OPTIMIZE_SIZE__ predefined macro")
+LANGOPT(Static            , 1, 0, Compatible, "__STATIC__ predefined macro (as opposed to __DYNAMIC__)")
+VALUE_LANGOPT(PackStruct  , 32, 0, NotCompatible,
               "default struct packing maximum alignment")
-VALUE_LANGOPT(MaxTypeAlign  , 32, 0,
+VALUE_LANGOPT(MaxTypeAlign  , 32, 0, NotCompatible,
               "default maximum alignment for types")
-VALUE_LANGOPT(AlignDouble            , 1, 0, "Controls if doubles should be aligned to 8 bytes (x86 only)")
-VALUE_LANGOPT(DoubleSize            , 32, 0, "width of double")
-VALUE_LANGOPT(LongDoubleSize        , 32, 0, "width of long double")
-LANGOPT(PPCIEEELongDouble            , 1, 0, "use IEEE 754 quadruple-precision for long double")
-LANGOPT(EnableAIXExtendedAltivecABI  , 1, 0, "__EXTABI__  predefined macro")
-LANGOPT(EnableAIXQuadwordAtomicsABI  , 1, 0, "Use 16-byte atomic lock free semantics")
-COMPATIBLE_VALUE_LANGOPT(PICLevel    , 2, 0, "__PIC__ level")
-COMPATIBLE_VALUE_LANGOPT(PIE         , 1, 0, "is pie")
-LANGOPT(ROPI                         , 1, 0, "Read-only position independence")
-LANGOPT(RWPI                         , 1, 0, "Read-write position independence")
-COMPATIBLE_LANGOPT(GNUInline         , 1, 0, "GNU inline semantics")
-COMPATIBLE_LANGOPT(NoInlineDefine    , 1, 0, "__NO_INLINE__ predefined macro")
-COMPATIBLE_LANGOPT(Deprecated        , 1, 0, "__DEPRECATED predefined macro")
-COMPATIBLE_LANGOPT(FastMath          , 1, 0, "fast FP math optimizations, and __FAST_MATH__ predefined macro")
-COMPATIBLE_LANGOPT(UnsafeFPMath      , 1, 0, "Unsafe Floating Point Math")
-COMPATIBLE_LANGOPT(ProtectParens     , 1, 0, "optimizer honors parentheses "
-                   "when floating-point expressions are evaluated")
-BENIGN_LANGOPT(AllowFPReassoc    , 1, 0, "Permit Floating Point reassociation")
-BENIGN_LANGOPT(NoHonorNaNs       , 1, 0, "Permit Floating Point optimization without regard to NaN")
-BENIGN_LANGOPT(NoHonorInfs       , 1, 0, "Permit Floating Point optimization without regard to infinities")
-BENIGN_LANGOPT(NoSignedZero      , 1, 0, "Permit Floating Point optimization without regard to signed zeros")
-BENIGN_LANGOPT(AllowRecip        , 1, 0, "Permit Floating Point reciprocal")
-BENIGN_LANGOPT(ApproxFunc        , 1, 0, "Permit Floating Point approximation")
-
-ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 3, CX_None, "Enable use of range reduction for complex arithmetics.")
-
-BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars")
-
-BENIGN_LANGOPT(AccessControl     , 1, 1, "C++ access control")
-LANGOPT(CharIsSigned      , 1, 1, "signed char")
-LANGOPT(WCharSize         , 4, 0, "width of wchar_t")
-LANGOPT(WCharIsSigned        , 1, 0, "signed or unsigned wchar_t")
-ENUM_LANGOPT(MSPointerToMemberRepresentationMethod, PragmaMSPointersToMembersKind, 2, PPTMK_BestCase, "member-pointer representation method")
-ENUM_LANGOPT(DefaultCallingConv, DefaultCallingConvention, 3, DCC_None, "default calling convention")
-
-LANGOPT(ShortEnums        , 1, 0, "short enum types")
-
-LANGOPT(OpenCL            , 1, 0, "OpenCL")
-LANGOPT(OpenCLVersion     , 32, 0, "OpenCL C version")
-LANGOPT(OpenCLCPlusPlus   , 1, 0, "C++ for OpenCL")
-LANGOPT(OpenCLCPlusPlusVersion     , 32, 0, "C++ for OpenCL version")
-LANGOPT(OpenCLGenericAddressSpace, 1, 0, "OpenCL generic keyword")
-LANGOPT(OpenCLPipes              , 1, 0, "OpenCL pipes language constructs and built-ins")
-LANGOPT(NativeHalfType    , 1, 0, "Native half type support")
-LANGOPT(NativeHalfArgsAndReturns, 1, 0, "Native half args and returns")
-LANGOPT(CUDA              , 1, 0, "CUDA")
-LANGOPT(HIP               , 1, 0, "HIP")
-LANGOPT(OpenMP            , 32, 0, "OpenMP support and version of OpenMP (31, 40 or 45)")
-LANGOPT(OpenMPExtensions  , 1, 1, "Enable all Clang extensions for OpenMP directives and clauses")
-LANGOPT(OpenMPSimd        , 1, 0, "Use SIMD only OpenMP support.")
-LANGOPT(OpenMPUseTLS      , 1, 0, "Use TLS for threadprivates or runtime calls")
-LANGOPT(OpenMPIsTargetDevice    , 1, 0, "Generate code only for OpenMP target device")
-LANGOPT(OpenMPCUDAMode    , 1, 0, "Generate code for OpenMP pragmas in SIMT/SPMD mode")
-LANGOPT(OpenMPIRBuilder   , 1, 0, "Use the experimental OpenMP-IR-Builder codegen path.")
-LANGOPT(OpenMPCUDANumSMs  , 32, 0, "Number of SMs for CUDA devices.")
-LANGOPT(OpenMPCUDABlocksPerSM  , 32, 0, "Number of blocks per SM for CUDA devices.")
-LANGOPT(OpenMPCUDAReductionBufNum , 32, 1024, "Number of the reduction records in the intermediate reduction buffer used for the teams reductions.")
-LANGOPT(OpenMPTargetDebug , 32, 0, "Enable debugging in the OpenMP offloading device RTL")
-LANGOPT(OpenMPOptimisticCollapse  , 1, 0, "Use at most 32 bits to represent the collapsed loop nest counter.")
-LANGOPT(OpenMPThreadSubscription  , 1, 0, "Assume work-shared loops do not have more iterations than participating threads.")
-LANGOPT(OpenMPTeamSubscription  , 1, 0, "Assume distributed loops do not have more iterations than participating teams.")
-LANGOPT(OpenMPNoThreadState  , 1, 0, "Assume that no thread in a parallel region will modify an ICV.")
-LANGOPT(OpenMPNoNestedParallelism  , 1, 0, "Assume that no thread in a parallel region will encounter a parallel region")
-LANGOPT(OpenMPOffloadMandatory  , 1, 0, "Assert that offloading is mandatory and do not create a host fallback.")
-LANGOPT(OpenMPForceUSM     , 1, 0, "Enable OpenMP unified shared memory mode via compiler.")
-LANGOPT(NoGPULib  , 1, 0, "Indicate a build without the standard GPU libraries.")
-
-LANGOPT(HLSL, 1, 0, "HLSL")
-ENUM_LANGOPT(HLSLVersion, HLSLLangStd, 16, HLSL_Unset, "HLSL Version")
-LANGOPT(HLSLStrictAvailability, 1, 0,
+VALUE_LANGOPT(AlignDouble            , 1, 0, NotCompatible, "Controls if doubles should be aligned to 8 bytes (x86 only)")
+VALUE_LANGOPT(DoubleSize            , 32, 0, NotCompatible, "width of double")
+VALUE_LANGOPT(LongDoubleSize        , 32, 0, NotCompatible, "width of long double")
+LANGOPT(PPCIEEELongDouble            , 1, 0, NotCompatible, "use IEEE 754 quadruple-precision for long double")
+LANGOPT(EnableAIXExtendedAltivecABI  , 1, 0, NotCompatible, "__EXTABI__  predefined macro")
+LANGOPT(EnableAIXQuadwordAtomicsABI  , 1, 0, NotCompatible, "Use 16-byte atomic lock free semantics")
+VALUE_LANGOPT(PICLevel               , 2, 0, Compatible, "__PIC__ level")
+VALUE_LANGOPT(PIE                    , 1, 0, Compatible, "is pie")
+LANGOPT(ROPI                         , 1, 0, NotCompatible, "Read-only position independence")
+LANGOPT(RWPI                         , 1, 0, NotCompatible, "Read-write position independence")
+LANGOPT(GNUInline                    , 1, 0, Compatible, "GNU inline semantics")
+LANGOPT(NoInlineDefine               , 1, 0, Compatible, "__NO_INLINE__ predefined macro")
+LANGOPT(Deprecated                   , 1, 0, Compatible, "__DEPRECATED predefined macro")
+LANGOPT(FastMath                     , 1, 0, Compatible, "fast FP math optimizations, and __FAST_MATH__ predefined macro")
+LANGOPT(UnsafeFPMath                 , 1, 0, Compatible, "Unsafe Floating Point Math")
+LANGOPT(ProtectParens                , 1, 0, Compatible, "optimizer honors parentheses "
+        "when floating-point expressions are evaluated")
+LANGOPT(AllowFPReassoc    , 1, 0, Benign, "Permit Floating Point reassociation")
+LANGOPT(NoHonorNaNs       , 1, 0, Benign, "Permit Floating Point optimization without regard to NaN")
+LANGOPT(NoHonorInfs       , 1, 0, Benign, "Permit Floating Point optimization without regard to infinities")
+LANGOPT(NoSignedZero      , 1, 0, Benign, "Permit Floating Point optimization without regard to signed zeros")
+LANGOPT(AllowRecip        , 1, 0, Benign, "Permit Floating Point reciprocal")
+LANGOPT(ApproxFunc        , 1, 0, Benign, "Permit Floating Point approximation")
+
+ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 3, CX_None, NotCompatible, "Enable use of range reduction for complex arithmetics.")
+
+LANGOPT(ObjCGCBitmapPrint , 1, 0, Benign, "printing of GC's bitmap layout for __weak/__strong ivars")
+
+LANGOPT(AccessControl     , 1, 1, Benign, "C++ access control")
+LANGOPT(CharIsSigned      , 1, 1, NotCompatible, "signed char")
+LANGOPT(WCharSize         , 4, 0, NotCompatible, "width of wchar_t")
+LANGOPT(WCharIsSigned        , 1, 0, NotCompatible, "signed or unsigned wchar_t")
+ENUM_LANGOPT(MSPointerToMemberRepresentationMethod, PragmaMSPointersToMembersKind, 2, PPTMK_BestCase, NotCompatible, "member-pointer representation method")
+ENUM_LANGOPT(DefaultCallingConv, DefaultCallingConvention, 3, DCC_None, NotCompatible, "default calling convention")
+
+LANGOPT(ShortEnums        , 1, 0, NotCompatible, "short enum types")
+
+LANGOPT(OpenCL            , 1, 0, NotCompatible, "OpenCL")
+LANGOPT(OpenCLVersion     , 32, 0, NotCompatible, "OpenCL C version")
+LANGOPT(OpenCLCPlusPlus   , 1, 0, NotCompatible, "C++ for OpenCL")
+LANGOPT(OpenCLCPlusPlusVersion     , 32, 0, NotCompatible, "C++ for OpenCL version")
+LANGOPT(OpenCLGenericAddressSpace, 1, 0, NotCompatible, "OpenCL generic keyword")
+LANGOPT(OpenCLPipes              , 1, 0, NotCompatible, "OpenCL pipes language constructs and built-ins")
+LANGOPT(NativeHalfType    , 1, 0, NotCompatible, "Native half type support")
+LANGOPT(NativeHalfArgsAndReturns, 1, 0, NotCompatible, "Native half args and returns")
+LANGOPT(CUDA              , 1, 0, NotCompatible, "CUDA")
+LANGOPT(HIP               , 1, 0, NotCompatible, "HIP")
+LANGOPT(OpenMP            , 32, 0, NotCompatible, "OpenMP support and version of OpenMP (31, 40 or 45)")
+LANGOPT(OpenMPExtensions  , 1, 1, NotCompatible, "Enable all Clang extensions for OpenMP directives and clauses")
+LANGOPT(OpenMPSimd        , 1, 0, NotCompatible, "Use SIMD only OpenMP support.")
+LANGOPT(OpenMPUseTLS      , 1, 0, NotCompatible, "Use TLS for threadprivates or runtime calls")
+LANGOPT(OpenMPIsTargetDevice    , 1, 0, NotCompatible, "Generate code only for OpenMP target device")
+LANGOPT(OpenMPCUDAMode    , 1, 0, NotCompatible, "Generate code for OpenMP pragmas in SIMT/SPMD mode")
+LANGOPT(OpenMPIRBuilder   , 1, 0, NotCompatible, "Use the experimental OpenMP-IR-Builder codegen path.")
+LANGOPT(OpenMPCUDANumSMs  , 32, 0, NotCompatible, "Number of SMs for CUDA devices.")
+LANGOPT(OpenMPCUDABlocksPerSM  , 32, 0, NotCompatible, "Number of blocks per SM for CUDA devices.")
+LANGOPT(OpenMPCUDAReductionBufNum , 32, 1024, NotCompatible, "Number of the reduction records in the intermediate reduction buffer used for the teams reductions.")
+LANGOPT(OpenMPTargetDebug , 32, 0, NotCompatible, "Enable debugging in the OpenMP offloading device RTL")
+LANGOPT(OpenMPOptimisticCollapse  , 1, 0, NotCompatible, "Use at most 32 bits to represent the collapsed loop nest counter.")
+LANGOPT(OpenMPThreadSubscription  , 1, 0, NotCompatible, "Assume work-shared loops do not have more iterations than participating threads.")
+LANGOPT(OpenMPTeamSubscription  , 1, 0, NotCompatible, "Assume distributed loops do not have more iterations than participating teams.")
+LANGOPT(OpenMPNoThreadState  , 1, 0, NotCompatible, "Assume that no thread in a parallel region will modify an ICV.")
+LANGOPT(OpenMPNoNestedParallelism  , 1, 0, NotCompatible, "Assume that no thread in a parallel region will encounter a parallel region")
+LANGOPT(OpenMPOffloadMandatory  , 1, 0, NotCompatible, "Assert that offloading is mandatory and do not create a host fallback.")
+LANGOPT(OpenMPForceUSM     , 1, 0, NotCompatible, "Enable OpenMP unified shared memory mode via compiler.")
+LANGOPT(NoGPULib  , 1, 0, NotCompatible, "Indicate a build without the standard GPU libraries.")
+
+LANGOPT(HLSL, 1, 0, NotCompatible, "HLSL")
+ENUM_LANGOPT(HLSLVersion, HLSLLangStd, 16, HLSL_Unset, NotCompatible, "HLSL Version")
+LANGOPT(HLSLStrictAvailability, 1, 0, NotCompatible,
         "Strict availability diagnostic mode for HLSL built-in functions.")
 
-LANGOPT(CUDAIsDevice      , 1, 0, "compiling for CUDA device")
-LANGOPT(CUDAAllowVariadicFunctions, 1, 0, "allowing variadic functions in CUDA device code")
-LANGOPT(CUDAHostDeviceConstexpr, 1, 1, "treating unattributed constexpr functions as __host__ __device__")
-LANGOPT(GPUDeviceApproxTranscendentals, 1, 0, "using approximate transcendental functions")
-LANGOPT(GPURelocatableDeviceCode, 1, 0, "generate relocatable device code")
-LANGOPT(OffloadImplicitHostDeviceTemplates, 1, 0, "assume template functions to be implicitly host device by default for CUDA/HIP")
-LANGOPT(GPUAllowDeviceInit, 1, 0, "allowing device side global init functions for HIP")
-LANGOPT(GPUMaxThreadsPerBlock, 32, 1024, "default max threads per block for kernel launch bounds for HIP")
-LANGOPT(GPUDeferDiag, 1, 0, "defer host/device related diagnostic messages for CUDA/HIP")
-LANGOPT(GPUExcludeWrongSideOverloads, 1, 0, "always exclude wrong side overloads in overloading resolution for CUDA/HIP")
-LANGOPT(OffloadingNewDriver, 1, 0, "use the new driver for generating offloading code.")
-LANGOPT(OffloadViaLLVM, 1, 0, "target LLVM/Offload as portable offloading runtime.")
-
-LANGOPT(SYCLIsDevice      , 1, 0, "Generate code for SYCL device")
-LANGOPT(SYCLIsHost        , 1, 0, "SYCL host compilation")
-ENUM_LANGOPT(SYCLVersion  , SYCLMajorVersion, 2, SYCL_None, "Version of the SYCL standard used")
-
-LANGOPT(HIPUseNewLaunchAPI, 1, 0, "Use new kernel launching API for HIP")
-LANGOPT(OffloadUniformBlock, 1, 0, "Assume that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)")
-LANGOPT(HIPStdPar, 1, 0, "Enable Standard Parallel Algorithm Acceleration for HIP (experimental)")
-LANGOPT(HIPStdParInterposeAlloc, 1, 0, "Replace allocations / deallocations with HIP RT calls when Standard Parallel Algorithm Acceleration for HIP is enabled (Experimental)")
-
-LANGOPT(OpenACC           , 1, 0, "OpenACC Enabled")
-
-LANGOPT(MSVCEnableStdcMacro , 1, 0, "Define __STDC__ with '-fms-compatibility'")
-LANGOPT(SizedDeallocation , 1, 0, "sized deallocation")
-LANGOPT(AlignedAllocation , 1, 0, "aligned allocation")
-LANGOPT(AlignedAllocationUnavailable, 1, 0, "aligned allocation functions are unavailable")
-LANGOPT(NewAlignOverride  , 32, 0, "maximum alignment guaranteed by '::operator new(size_t)'")
-BENIGN_LANGOPT(ModulesCodegen , 1, 0, "Modules code generation")
-BENIGN_LANGOPT(ModulesDebugInfo , 1, 0, "Modules debug info")
-BENIGN_LANGOPT(ElideConstructors , 1, 1, "C++ copy constructor elision")
-BENIGN_LANGOPT(DumpRecordLayouts , 1, 0, "dumping the layout of IRgen'd records")
-BENIGN_LANGOPT(DumpRecordLayoutsSimple , 1, 0, "dumping the layout of IRgen'd records in a simple form")
-BENIGN_LANGOPT(DumpRecordLayoutsCanonical , 1, 0, "dumping the AST layout of records using canonical field types")
-BENIGN_LANGOPT(DumpRecordLayoutsComplete , 1, 0, "dumping the AST layout of all complete records")
-BENIGN_LANGOPT(DumpVTableLayouts , 1, 0, "dumping the layouts of emitted vtables")
-LANGOPT(NoConstantCFStrings , 1, 0, "no constant CoreFoundation strings")
-BENIGN_LANGOPT(InlineVisibilityHidden , 1, 0, "hidden visibility for inline C++ methods")
-BENIGN_ENUM_LANGOPT(DefaultVisibilityExportMapping, DefaultVisiblityExportMapping, 2, DefaultVisiblityExportMapping::None, "controls mapping of default visibility to dllexport")
-BENIGN_LANGOPT(IgnoreXCOFFVisibility, 1, 0, "All the visibility attributes that are specified in the source code are ignored in aix XCOFF.")
-BENIGN_LANGOPT(VisibilityInlinesHiddenStaticLocalVar, 1, 0,
-               "hidden visibility for static local variables in inline C++ "
-               "methods when -fvisibility-inlines hidden is enabled")
-ENUM_LANGOPT(GlobalAllocationFunctionVisibility, VisibilityForcedKinds, 3, VisibilityForcedKinds::ForceDefault,
+LANGOPT(CUDAIsDevice      , 1, 0, NotCompatible, "compiling for CUDA device")
+LANGOPT(CUDAAllowVariadicFunctions, 1, 0, NotCompatible, "allowing variadic functions in CUDA device code")
+LANGOPT(CUDAHostDeviceConstexpr, 1, 1, NotCompatible, "treating unattributed constexpr functions as __host__ __device__")
+LANGOPT(GPUDeviceApproxTranscendentals, 1, 0, NotCompatible, "using approximate transcendental functions")
+LANGOPT(GPURelocatableDeviceCode, 1, 0, NotCompatible, "generate relocatable device code")
+LANGOPT(OffloadImplicitHostDeviceTemplates, 1, 0, NotCompatible, "assume template functions to be implicitly host device by default for CUDA/HIP")
+LANGOPT(GPUAllowDeviceInit, 1, 0, NotCompatible, "allowing device side global init functions for HIP")
+LANGOPT(GPUMaxThreadsPerBlock, 32, 1024, NotCompatible, "default max threads per block for kernel launch bounds for HIP")
+LANGOPT(GPUDeferDiag, 1, 0, NotCompatible, "defer host/device related diagnostic messages for CUDA/HIP")
+LANGOPT(GPUExcludeWrongSideOverloads, 1, 0, NotCompatible, "always exclude wrong side overloads in overloading resolution for CUDA/HIP")
+LANGOPT(OffloadingNewDriver, 1, 0, NotCompatible, "use the new driver for generating offloading code.")
+LANGOPT(OffloadViaLLVM, 1, 0, NotCompatible, "target LLVM/Offload as portable offloading runtime.")
+
+LANGOPT(SYCLIsDevice      , 1, 0, NotCompatible, "Generate code for SYCL device")
+LANGOPT(SYCLIsHost        , 1, 0, NotCompatible, "SYCL host compilation")
+ENUM_LANGOPT(SYCLVersion  , SYCLMajorVersion, 2, SYCL_None, NotCompatible, "Version of the SYCL standard used")
+
+LANGOPT(HIPUseNewLaunchAPI, 1, 0, NotCompatible, "Use new kernel launching API for HIP")
+LANGOPT(OffloadUniformBlock, 1, 0, NotCompatible, "Assume that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)")
+LANGOPT(HIPStdPar, 1, 0, NotCompatible, "Enable Standard Parallel Algorithm Acceleration for HIP (experimental)")
+LANGOPT(HIPStdParInterposeAlloc, 1, 0, NotCompatible, "Replace allocations / deallocations with HIP RT calls when Standard Parallel Algorithm Acceleration for HIP is enabled (Experimental)")
+
+LANGOPT(OpenACC           , 1, 0, NotCompatible, "OpenACC Enabled")
+
+LANGOPT(MSVCEnableStdcMacro , 1, 0, NotCompatible, "Define __STDC__ with '-fms-compatibility'")
+LANGOPT(SizedDeallocation , 1, 0, NotCompatible, "sized deallocation")
+LANGOPT(AlignedAllocation , 1, 0, NotCompatible, "aligned allocation")
+LANGOPT(AlignedAllocationUnavailable, 1, 0, NotCompatible, "aligned allocation functions are unavailable")
+LANGOPT(NewAlignOverride  , 32, 0, NotCompatible, "maximum alignment guaranteed by '::operator new(size_t)'")
+LANGOPT(ModulesCodegen , 1, 0, Benign, "Modules code generation")
+LANGOPT(ModulesDebugInfo , 1, 0, Benign, "Modules debug info")
+LANGOPT(ElideConstructors , 1, 1, Benign, "C++ copy constructor elision")
+LANGOPT(DumpRecordLayouts , 1, 0, Benign, "dumping the layout of IRgen'd records")
+LANGOPT(DumpRecordLayoutsSimple , 1, 0, Benign, "dumping the layout of IRgen'd records in a simple form")
+LANGOPT(DumpRecordLayoutsCanonical , 1, 0, Benign, "dumping the AST layout of records using canonical field types")
+LANGOPT(DumpRecordLayoutsComplete , 1, 0, Benign, "dumping the AST layout of all complete records")
+LANGOPT(DumpVTableLayouts , 1, 0, Benign, "dumping the layouts of emitted vtables")
+LANGOPT(NoConstantCFStrings , 1, 0, NotCompatible, "no constant CoreFoundation strings")
+LANGOPT(InlineVisibilityHidden , 1, 0, Benign, "hidden visibility for inline C++ methods")
+ENUM_LANGOPT(DefaultVisibilityExportMapping, DefaultVisiblityExportMapping, 2, DefaultVisiblityExportMapping::None, Benign, "controls mapping of default visibility to dllexport")
+LANGOPT(IgnoreXCOFFVisibility, 1, 0, Benign, "All the visibility attributes that are specified in the source code are ignored in aix XCOFF.")
+LANGOPT(VisibilityInlinesHiddenStaticLocalVar, 1, 0, Benign,
+        "hidden visibility for static local variables in inline C++ "
+        "methods when -fvisibility-inlines hidden is enabled")
+ENUM_LANGOPT(GlobalAllocationFunctionVisibility, VisibilityForcedKinds, 3, VisibilityForcedKinds::ForceDefault, NotCompatible,
              "How to apply visibility to global operator new and delete declarations")
-LANGOPT(NewInfallible , 1, 0, "Treats throwing global C++ operator new as always returning valid memory (annotates with __attribute__((returns_nonnull)) and throw()). This is detectable in source.")
-BENIGN_LANGOPT(ParseUnknownAnytype, 1, 0, "__unknown_anytype")
-BENIGN_LANGOPT(DebuggerSupport , 1, 0, "debugger support")
-BENIGN_LANGOPT(DebuggerCastResultToId, 1, 0, "for 'po' in the debugger, cast the result to id if it is of unknown type")
-BENIGN_LANGOPT(DebuggerObjCLiteral , 1, 0, "debugger Objective-C literals and subscripting support")
-
-BENIGN_LANGOPT(SpellChecking , 1, 1, "spell-checking")
-LANGOPT(SinglePrecisionConstants , 1, 0, "treating double-precision floating point constants as single precision constants")
-LANGOPT(FastRelaxedMath , 1, 0, "OpenCL fast relaxed math")
-BENIGN_LANGOPT(CLNoSignedZero , 1, 0, "Permit Floating Point optimization without regard to signed zeros")
-COMPATIBLE_LANGOPT(CLUnsafeMath , 1, 0, "Unsafe Floating Point Math")
+LANGOPT(NewInfallible , 1, 0, NotCompatible, "Treats throwing global C++ operator new as always returning valid memory (annotates with __attribute__((returns_nonnull)) and throw()). This is detectable in source.")
+LANGOPT(ParseUnknownAnytype, 1, 0, Benign, "__unknown_anytype")
+LANGOPT(DebuggerSupport , 1, 0, Benign, "debugger support")
+LANGOPT(DebuggerCastResultToId, 1, 0, Benign, "for 'po' in the debugger, cast the result to id if it is of unknown type")
+LANGOPT(DebuggerObjCLiteral , 1, 0, Benign, "debugger Objective-C literals and subscripting support")
+
+LANGOPT(SpellChecking , 1, 1, Benign, "spell-checking")
+LANGOPT(SinglePrecisionConstants , 1, 0, NotCompatible, "treating double-precision floating point constants as single precision constants")
+LANGOPT(FastRelaxedMath , 1, 0, NotCompatible, "OpenCL fast relaxed math")
+LANGOPT(CLNoSignedZero , 1, 0, Benign, "Permit Floating Point optimization without regard to signed zeros")
+LANGOPT(CLUnsafeMath , 1, 0, Compatible, "Unsafe Floating Point Math")
 /// FP_CONTRACT mode (on/off/fast).
-BENIGN_ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, "FP contraction type")
-COMPATIBLE_LANGOPT(ExpStrictFP, 1, false, "Enable experimental strict floating point")
-BENIGN_LANGOPT(RoundingMath, 1, false, "Do not assume default floating-point rounding behavior")
-BENIGN_ENUM_LANGOPT(FPExceptionMode, FPExceptionModeKind, 2, FPE_Default, "FP Exception Behavior Mode type")
-BENIGN_ENUM_LANGOPT(FPEvalMethod, FPEvalMethodKind, 2, FEM_UnsetOnCommandLine, "FP type used for floating point arithmetic")
-ENUM_LANGOPT(Float16ExcessPrecision, ExcessPrecisionKind, 2, FPP_Standard, "Intermediate truncation behavior for Float16 arithmetic")
-ENUM_LANGOPT(BFloat16ExcessPrecision, ExcessPrecisionKind, 2, FPP_Standard, "Intermediate truncation behavior for BFloat16 arithmetic")
-LANGOPT(NoBitFieldTypeAlign , 1, 0, "bit-field type alignment")
-LANGOPT(HexagonQdsp6Compat , 1, 0, "hexagon-qdsp6 backward compatibility")
-LANGOPT(ObjCAutoRefCount , 1, 0, "Objective-C automated reference counting")
-LANGOPT(ObjCWeakRuntime     , 1, 0, "__weak support in the ARC runtime")
-LANGOPT(ObjCWeak            , 1, 0, "Objective-C __weak in ARC and MRC files")
-LANGOPT(ObjCSubscriptingLegacyRuntime         , 1, 0, "Subscripting support in legacy ObjectiveC runtime")
-BENIGN_LANGOPT(CompatibilityQualifiedIdBlockParamTypeChecking, 1, 0,
-               "compatibility mode for type checking block parameters "
-               "involving qualified id types")
-LANGOPT(ObjCDisableDirectMethodsForTesting, 1, 0,
+ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, Benign, "FP contraction type")
+LANGOPT(ExpStrictFP, 1, false, Compatible, "Enable experimental strict floating point")
+LANGOPT(RoundingMath, 1, false, Benign, "Do not assume default floating-point rounding behavior")
+ENUM_LANGOPT(FPExceptionMode, FPExceptionModeKind, 2, FPE_Default, Benign, "FP Exception Behavior Mode type")
+ENUM_LANGOPT(FPEvalMethod, FPEvalMethodKind, 2, FEM_UnsetOnCommandLine, Benign, "FP type used for floating point arithmetic")
+ENUM_LANGOPT(Float16ExcessPrecision, ExcessPrecisionKind, 2, FPP_Standard, NotCompatible, "Intermediate truncation behavior for Float16 arithmetic")
+ENUM_LANGOPT(BFloat16ExcessPrecision, ExcessPrecisionKind, 2, FPP_Standard, NotCompatible, "Intermediate truncation behavior for BFloat16 arithmetic")
+LANGOPT(NoBitFieldTypeAlign , 1, 0, NotCompatible, "bit-field type alignment")
+LANGOPT(HexagonQdsp6Compat , 1, 0, NotCompatible, "hexagon-qdsp6 backward compatibility")
+LANGOPT(ObjCAutoRefCount , 1, 0, NotCompatible, "Objective-C automated reference counting")
+LANGOPT(ObjCWeakRuntime     , 1, 0, NotCompatible, "__weak support in the ARC runtime")
+LANGOPT(ObjCWeak            , 1, 0, NotCompatible, "Objective-C __weak in ARC and MRC files")
+LANGOPT(ObjCSubscriptingLegacyRuntime         , 1, 0, NotCompatible, "Subscripting support in legacy ObjectiveC runtime")
+LANGOPT(CompatibilityQualifiedIdBlockParamTypeChecking, 1, 0, Benign,
+        "compatibility mode for type checking block parameters "
+        "involving qualified id types")
+LANGOPT(ObjCDisableDirectMethodsForTesting, 1, 0, NotCompatible,
         "Disable recognition of objc_direct methods")
-LANGOPT(CFProtectionBranch , 1, 0, "Control-Flow Branch Protection enabled")
-ENUM_LANGOPT(CFBranchLabelScheme, CFBranchLabelSchemeKind, 2, CFBranchLabelSchemeKind::Default,
+LANGOPT(CFProtectionBranch , 1, 0, NotCompatible, "Control-Flow Branch Protection enabled")
+ENUM_LANGOPT(CFBranchLabelScheme, CFBranchLabelSchemeKind, 2, CFBranchLabelSchemeKind::Default, NotCompatible,
              "Control-Flow Branch Protection Label Scheme")
-LANGOPT(CFProtectionReturn, 1, 0, "Control-Flow Return Protection enabled")
-LANGOPT(FakeAddressSpaceMap , 1, 0, "OpenCL fake address space map")
-ENUM_LANGOPT(AddressSpaceMapMangling , AddrSpaceMapMangling, 2, ASMM_Target, "OpenCL address space map mangling mode")
-LANGOPT(IncludeDefaultHeader, 1, 0, "Include default header file for OpenCL")
-LANGOPT(DeclareOpenCLBuiltins, 1, 0, "Declare OpenCL builtin functions")
-BENIGN_LANGOPT(DelayedTemplateParsing , 1, 0, "delayed template parsing")
-LANGOPT(BlocksRuntimeOptional , 1, 0, "optional blocks runtime")
+LANGOPT(CFProtectionReturn, 1, 0, NotCompatible, "Control-Flow Return Protection enabled")
+LANGOPT(FakeAddressSpaceMap , 1, 0, NotCompatible, "OpenCL fake address space map")
+ENUM_LANGOPT(AddressSpaceMapMangling , AddrSpaceMapMangling, 2, ASMM_Target, NotCompatible, "OpenCL address space map mangling mode")
+LANGOPT(IncludeDefaultHeader, 1, 0, NotCompatible, "Include default header file for OpenCL")
+LANGOPT(DeclareOpenCLBuiltins, 1, 0, NotCompatible, "Declare OpenCL builtin functions")
+LANGOPT(DelayedTemplateParsing , 1, 0, Benign, "delayed template parsing")
+LANGOPT(BlocksRuntimeOptional , 1, 0, NotCompatible, "optional blocks runtime")
 LANGOPT(
-    CompleteMemberPointers, 1, 0,
+    CompleteMemberPointers, 1, 0, NotCompatible,
     "Require member pointer base types to be complete at the point where the "
     "type's inheritance model would be determined under the Microsoft ABI")
 
-ENUM_LANGOPT(GC, GCMode, 2, NonGC, "Objective-C Garbage Collection mode")
-BENIGN_ENUM_LANGOPT(ValueVisibilityMode, Visibility, 3, DefaultVisibility,
+ENUM_LANGOPT(GC, GCMode, 2, NonGC, NotCompatible, "Objective-C Garbage Collection mode")
+ENUM_LANGOPT(ValueVisibilityMode, Visibility, 3, DefaultVisibility, Benign,
              "default visibility for functions and variables [-fvisibility]")
-BENIGN_ENUM_LANGOPT(TypeVisibilityMode, Visibility, 3, DefaultVisibility,
+ENUM_LANGOPT(TypeVisibilityMode, Visibility, 3, DefaultVisibility, Benign,
              "default visibility for types [-ftype-visibility]")
-LANGOPT(SetVisibilityForExternDecls, 1, 0,
+LANGOPT(SetVisibilityForExternDecls, 1, 0, NotCompatible,
         "apply global symbol visibility to external declarations without an explicit visibility")
-BENIGN_LANGOPT(VisibilityFromDLLStorageClass, 1, 0,
-               "override the visibility of globals based on their final DLL storage class [-fvisibility-from-dllstorageclass]")
-BENIGN_ENUM_LANGOPT(DLLExportVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Default,
+LANGOPT(VisibilityFromDLLStorageClass, 1, 0, Benign,
+        "override the visibility of globals based on their final DLL storage class [-fvisibility-from-dllstorageclass]")
+ENUM_LANGOPT(DLLExportVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Default, Benign,
              "how to adjust the visibility for functions and variables with dllexport annotations [-fvisibility-dllexport]")
-BENIGN_ENUM_LANGOPT(NoDLLStorageClassVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Hidden,
+ENUM_LANGOPT(NoDLLStorageClassVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Hidden, Benign,
              "how to adjust the visibility for functions and variables without an explicit DLL storage class [-fvisibility-nodllstorageclass]")
-BENIGN_ENUM_LANGOPT(ExternDeclDLLImportVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Default,
+ENUM_LANGOPT(ExternDeclDLLImportVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Default, Benign,
              "how to adjust the visibility for external declarations with dllimport annotations [-fvisibility-externs-dllimport]")
-BENIGN_ENUM_LANGOPT(ExternDeclNoDLLStorageClassVisibility, VisibilityFromDLLStorageClassKinds, 3,  VisibilityFromDLLStorageClassKinds::Hidden,
+ENUM_LANGOPT(ExternDeclNoDLLStorageClassVisibility, VisibilityFromDLLStorageClassKinds, 3,  VisibilityFromDLLStorageClassKinds::Hidden, Benign,
              "how to adjust the visibility for external declarations without an explicit DLL storage class [-fvisibility-externs-nodllstorageclass]")
-BENIGN_LANGOPT(SemanticInterposition        , 1, 0, "semantic interposition")
-BENIGN_LANGOPT(HalfNoSemanticInterposition, 1, 0,
-               "Like -fno-semantic-interposition but don't use local aliases")
-ENUM_LANGOPT(StackProtector, StackProtectorMode, 2, SSPOff,
+LANGOPT(SemanticInterposition        , 1, 0, Benign, "semantic interposition")
+LANGOPT(HalfNoSemanticInterposition, 1, 0, Benign,
+        "Like -fno-semantic-interposition but don't use local aliases")
+ENUM_LANGOPT(StackProtector, StackProtectorMode, 2, SSPOff, NotCompatible,
              "stack protector mode")
-BENIGN_ENUM_LANGOPT(TrivialAutoVarInit, TrivialAutoVarInitKind, 2, TrivialAutoVarInitKind::Uninitialized,
+ENUM_LANGOPT(TrivialAutoVarInit, TrivialAutoVarInitKind, 2, TrivialAutoVarInitKind::Uninitialized, Benign,
              "trivial automatic variable initialization")
-BENIGN_VALUE_LANGOPT(TrivialAutoVarInitStopAfter, 32, 0,
-             "stop trivial automatic variable initialization after the specified number of instances. Must be greater than 0.")
-BENIGN_VALUE_LANGOPT(TrivialAutoVarInitMaxSize, 32, 0,
-             "stop trivial automatic variable initialization if var size exceeds the specified size (in bytes). Must be greater than 0.")
-ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined,
+VALUE_LANGOPT(TrivialAutoVarInitStopAfter, 32, 0, Benign,
+              "stop trivial automatic variable initialization after the specified number of instances. Must be greater than 0.")
+VALUE_LANGOPT(TrivialAutoVarInitMaxSize, 32, 0, Benign,
+              "stop trivial automatic variable initialization if var size exceeds the specified size (in bytes). Must be greater than 0.")
+ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined, NotCompatible,
              "signed integer overflow handling")
-LANGOPT(PointerOverflowDefined, 1, 0, "make pointer overflow defined")
-ENUM_LANGOPT(ThreadModel  , ThreadModelKind, 2, ThreadModelKind::POSIX, "Thread Model")
-
-BENIGN_LANGOPT(ArrowDepth, 32, 256,
-               "maximum number of operator->s to follow")
-BENIGN_LANGOPT(InstantiationDepth, 32, 1024,
-               "maximum template instantiation depth")
-BENIGN_LANGOPT(ConstexprCallDepth, 32, 512,
-               "maximum constexpr call depth")
-BENIGN_LANGOPT(ConstexprStepLimit, 32, 1048576,
-               "maximum constexpr evaluation steps")
-BENIGN_LANGOPT(EnableNewConstInterp, 1, 0,
-               "enable the experimental new constant interpreter")
-BENIGN_LANGOPT(BracketDepth, 32, 256,
-               "maximum bracket nesting depth")
-BENIGN_LANGOPT(NumLargeByValueCopy, 32, 0,
+LANGOPT(PointerOverflowDefined, 1, 0, NotCompatible, "make pointer overflow defined")
+ENUM_LANGOPT(ThreadModel  , ThreadModelKind, 2, ThreadModelKind::POSIX, NotCompatible, "Thread Model")
+
+LANGOPT(ArrowDepth, 32, 256, Benign,
+        "maximum number of operator->s to follow")
+LANGOPT(InstantiationDepth, 32, 1024, Benign,
+        "maximum template instantiation depth")
+LANGOPT(ConstexprCallDepth, 32, 512, Benign,
+        "maximum constexpr call depth")
+LANGOPT(ConstexprStepLimit, 32, 1048576, Benign,
+        "maximum constexpr evaluation steps")
+LANGOPT(EnableNewConstInterp, 1, 0, Benign,
+        "enable the experimental new constant interpreter")
+LANGOPT(BracketDepth, 32, 256, Benign,
+        "maximum bracket nesting depth")
+LANGOPT(NumLargeByValueCopy, 32, 0, Benign,
         "if non-zero, warn about parameter or return Warn if parameter/return value is larger in bytes than this setting. 0 is no check.")
-VALUE_LANGOPT(MSCompatibilityVersion, 32, 0, "Microsoft Visual C/C++ Version")
-ENUM_LANGOPT(VtorDispMode, MSVtorDispMode, 2, MSVtorDispMode::ForVBaseOverride,
+VALUE_LANGOPT(MSCompatibilityVersion, 32, 0, NotCompatible, "Microsoft Visual C/C++ Version")
+ENUM_LANGOPT(VtorDispMode, MSVtorDispMode, 2, MSVtorDispMode::ForVBaseOverride, NotCompatible,
              "How many vtordisps to insert")
 
-LANGOPT(ApplePragmaPack, 1, 0, "Apple gcc-compatible #pragma pack handling")
+LANGOPT(ApplePragmaPack, 1, 0, NotCompatible, "Apple gcc-compatible #pragma pack handling")
 
-LANGOPT(XLPragmaPack, 1, 0, "IBM XL #pragma pack handling")
+LANGOPT(XLPragmaPack, 1, 0, NotCompatible, "IBM XL #pragma pack handling")
 
-COMPATIBLE_LANGOPT(RetainCommentsFromSystemHeaders, 1, 0, "retain documentation comments from system headers in the AST")
+LANGOPT(RetainCommentsFromSystemHeaders, 1, 0, Compatible, "retain documentation comments from system headers in the AST")
 
-LANGOPT(APINotes, 1, 0, "use external API notes")
-LANGOPT(APINotesModules, 1, 0, "use module-based external API notes")
+LANGOPT(APINotes, 1, 0, NotCompatible, "use external API notes")
+LANGOPT(APINotesModules, 1, 0, NotCompatible, "use module-based external API notes")
 
-LANGOPT(SanitizeAddressFieldPadding, 2, 0, "controls how aggressive is ASan "
-                                           "field padding (0: none, 1:least "
-                                           "aggressive, 2: more aggressive)")
+LANGOPT(SanitizeAddressFieldPadding, 2, 0, NotCompatible, "controls how aggressive is ASan "
+                                                      "field padding (0: none, 1:least "
+                                                      "aggressive, 2: more aggressive)")
 
-LANGOPT(Cmse, 1, 0, "ARM Security extensions support")
+LANGOPT(Cmse, 1, 0, NotCompatible, "ARM Security extensions support")
 
-LANGOPT(XRayInstrument, 1, 0, "controls whether to do XRay instrumentation")
-LANGOPT(XRayAlwaysEmitCustomEvents, 1, 0,
+LANGOPT(XRayInstrument, 1, 0, NotCompatible, "controls whether to do XRay instrumentation")
+LANGOPT(XRayAlwaysEmitCustomEvents, 1, 0, NotCompatible,
         "controls whether to always emit intrinsic calls to "
         "__xray_customevent(...) builtin.")
-LANGOPT(XRayAlwaysEmitTypedEvents, 1, 0,
+LANGOPT(XRayAlwaysEmitTypedEvents, 1, 0, NotCompatible,
         "controls whether to always emit intrinsic calls to "
         "__xray_typedevent(...) builtin.")
 
-LANGOPT(ForceEmitVTables, 1, 0, "whether to emit all vtables")
+LANGOPT(ForceEmitVTables, 1, 0, NotCompatible, "whether to emit all vtables")
 
-BENIGN_LANGOPT(AllowEditorPlaceholders, 1, 0,
-               "allow editor placeholders in source")
+LANGOPT(AllowEditorPlaceholders, 1, 0, Benign,
+        "allow editor placeholders in source")
 
-ENUM_LANGOPT(ClangABICompat, ClangABI, 4, ClangABI::Latest,
+ENUM_LANGOPT(ClangABICompat, ClangABI, 4, ClangABI::Latest, NotCompatible,
              "version of Clang that we should attempt to be ABI-compatible "
              "with")
 
-COMPATIBLE_VALUE_LANGOPT(FunctionAlignment, 5, 0, "Default alignment for functions")
-COMPATIBLE_VALUE_LANGOPT(LoopAlignment, 32, 0, "Default alignment for loops")
+VALUE_LANGOPT(FunctionAlignment, 5, 0, Compatible, "Default alignment for functions")
+VALUE_LANGOPT(LoopAlignment, 32, 0, Compatible, "Default alignment for loops")
 
-LANGOPT(FixedPoint, 1, 0, "fixed point types")
-LANGOPT(PaddingOnUnsignedFixedPoint, 1, 0,
+LANGOPT(FixedPoint, 1, 0, NotCompatible, "fixed point types")
+LANGOPT(PaddingOnUnsignedFixedPoint, 1, 0, NotCompatible,
         "unsigned fixed point types having one extra padding bit")
 
 ENUM_LANGOPT(RegisterStaticDestructors, RegisterStaticDestructorsKind, 2,
-             RegisterStaticDestructorsKind::All,
+             RegisterStaticDestructorsKind::All, NotCompatible,
              "Register C++ static destructors")
 
-LANGOPT(RegCall4, 1, 0, "Set __regcall4 as a default calling convention to respect __regcall ABI v.4")
+LANGOPT(RegCall4, 1, 0, NotCompatible, "Set __regcall4 as a default calling convention to respect __regcall ABI v.4")
 
-LANGOPT(MatrixTypes, 1, 0, "Enable or disable the builtin matrix type")
+LANGOPT(MatrixTypes, 1, 0, NotCompatible, "Enable or disable the builtin matrix type")
 
-LANGOPT(CXXAssumptions, 1, 1, "Enable or disable codegen and compile-time checks for C++23's [[assume]] attribute")
+LANGOPT(CXXAssumptions, 1, 1, NotCompatible, "Enable or disable codegen and compile-time checks for C++23's [[assume]] attribute")
 
-LANGOPT(RawStringLiterals, 1, 1, "Enable or disable raw string literals")
+LANGOPT(RawStringLiterals, 1, 1, NotCompatible, "Enable or disable raw string literals")
 
 ENUM_LANGOPT(StrictFlexArraysLevel, StrictFlexArraysLevelKind, 2,
-             StrictFlexArraysLevelKind::Default,
+             StrictFlexArraysLevelKind::Default, NotCompatible,
              "Rely on strict definition of flexible arrays")
 
-COMPATIBLE_VALUE_LANGOPT(MaxTokens, 32, 0, "Max number of tokens per TU or 0")
+VALUE_LANGOPT(MaxTokens, 32, 0, Compatible, "Max number of tokens per TU or 0")
 
-ENUM_LANGOPT(SignReturnAddressScope, SignReturnAddressScopeKind, 2, SignReturnAddressScopeKind::None,
+ENUM_LANGOPT(SignReturnAddressScope, SignReturnAddressScopeKind, 2, SignReturnAddressScopeKind::None, NotCompatible,
              "Scope of return address signing")
-ENUM_LANGOPT(SignReturnAddressKey, SignReturnAddressKeyKind, 1, SignReturnAddressKeyKind::AKey,
+ENUM_LANGOPT(SignReturnAddressKey, SignReturnAddressKeyKind, 1, SignReturnAddressKeyKind::AKey, NotCompatible,
              "Key used for return address signing")
-LANGOPT(BranchTargetEnforcement, 1, 0, "Branch-target enforcement enabled")
-LANGOPT(BranchProtectionPAuthLR, 1, 0, "Use PC as a diversifier using PAuthLR NOP instructions.")
-LANGOPT(GuardedControlStack, 1, 0, "Guarded control stack enabled")
+LANGOPT(BranchTargetEnforcement, 1, 0, NotCompatible, "Branch-target enforcement enabled")
+LANGOPT(BranchProtectionPAuthLR, 1, 0, NotCompatible, "Use PC as a diversifier using PAuthLR NOP instructions.")
+LANGOPT(GuardedControlStack, 1, 0, NotCompatible, "Guarded control stack enabled")
 
-LANGOPT(SpeculativeLoadHardening, 1, 0, "Speculative load hardening enabled")
+LANGOPT(SpeculativeLoadHardening, 1, 0, NotCompatible, "Speculative load hardening enabled")
 
-LANGOPT(RelativeCXXABIVTables, 1, 0,
+LANGOPT(RelativeCXXABIVTables, 1, 0, NotCompatible,
         "Use an ABI-incompatible v-table layout that uses relative references")
 
-LANGOPT(OmitVTableRTTI, 1, 0,
+LANGOPT(OmitVTableRTTI, 1, 0, NotCompatible,
         "Use an ABI-incompatible v-table layout that omits the RTTI component")
 
-LANGOPT(VScaleMin, 32, 0, "Minimum vscale value")
-LANGOPT(VScaleMax, 32, 0, "Maximum vscale value")
+LANGOPT(VScaleMin, 32, 0, NotCompatible, "Minimum vscale value")
+LANGOPT(VScaleMax, 32, 0, NotCompatible, "Maximum vscale value")
+
+LANGOPT(VScaleStreamingMin, 32, 0, NotCompatible, "Minimum streaming vscale value")
+LANGOPT(VScaleStreamingMax, 32, 0, NotCompatible, "Maximum streaming vscale value")
 
-ENUM_LANGOPT(ExtendIntArgs, ExtendArgsKind, 1, ExtendArgsKind::ExtendTo32,
+ENUM_LANGOPT(ExtendIntArgs, ExtendArgsKind, 1, ExtendArgsKind::ExtendTo32, NotCompatible,
              "Controls how scalar integer arguments are extended in calls "
              "to unprototyped and varargs functions")
 
-VALUE_LANGOPT(FuchsiaAPILevel, 32, 0, "Fuchsia API level")
+VALUE_LANGOPT(FuchsiaAPILevel, 32, 0, NotCompatible, "Fuchsia API level")
 
 // This option will be removed in the future once the backend
 // supports all operations (like division or float-to-integer conversion)
 // on large _BitInts.
-BENIGN_VALUE_LANGOPT(MaxBitIntWidth, 32, 128, "Maximum width of a _BitInt")
+VALUE_LANGOPT(MaxBitIntWidth, 32, 128, Benign, "Maximum width of a _BitInt")
 
-COMPATIBLE_LANGOPT(IncrementalExtensions, 1, 0, "True if we want to process statements "
+LANGOPT(IncrementalExtensions, 1, 0, Compatible,  "True if we want to process statements "
         "on the global scope, ignore EOF token and continue later on (thus "
         "avoid tearing the Lexer and etc. down). Controlled by "
         "-fincremental-extensions.")
 
-BENIGN_LANGOPT(CheckNew, 1, 0, "Do not assume C++ operator new may not return NULL")
+LANGOPT(CheckNew, 1, 0, Benign, "Do not assume C++ operator new may not return NULL")
 
 // FIXME: It would be better for us to find a way to encode the state of this
 // diagnostic in tablegen so that we can specify a particular diagnostic option
 // is disabled or enabled based on other language options or made it easier to
 // do this from the compiler invocation without hitting option round-tripping
 // issues.
-BENIGN_LANGOPT(CheckConstexprFunctionBodies, 1, 1,
-               "Emit diagnostics for a constexpr function body that can never "
-               "be used in a constant expression.")
+LANGOPT(CheckConstexprFunctionBodies, 1, 1, Benign,
+        "Emit diagnostics for a constexpr function body that can never "
+        "be used in a constant expression.")
 
-LANGOPT(BoundsSafety, 1, 0, "Bounds safety extension for C")
+LANGOPT(BoundsSafety, 1, 0, NotCompatible, "Bounds safety extension for C")
 
-LANGOPT(PreserveVec3Type, 1, 0, "Preserve 3-component vector type")
+LANGOPT(PreserveVec3Type, 1, 0, NotCompatible, "Preserve 3-component vector type")
 
 #undef LANGOPT
-#undef COMPATIBLE_LANGOPT
-#undef BENIGN_LANGOPT
 #undef ENUM_LANGOPT
-#undef COMPATIBLE_ENUM_LANGOPT
-#undef BENIGN_ENUM_LANGOPT
 #undef VALUE_LANGOPT
-#undef COMPATIBLE_VALUE_LANGOPT
-#undef BENIGN_VALUE_LANGOPT
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index b7c5ed994c992..4c642c9e10c91 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -77,6 +77,22 @@ class LangOptionsBase {
   using RoundingMode = llvm::RoundingMode;
   using CFBranchLabelSchemeKind = clang::CFBranchLabelSchemeKind;
 
+  /// For ASTs produced with different option value, signifies their level of
+  /// compatibility.
+  enum class CompatibilityKind {
+    /// Does affect the construction of the AST in a way that does prevent
+    /// module interoperability.
+    NotCompatible,
+    /// Does affect the construction of the AST in a way that doesn't prevent
+    /// interoperability (that is, the value can be different between an
+    /// explicit module and the user of that module).
+    Compatible,
+    /// Does not affect the construction of the AST in any way (that is, the
+    /// value can be different between an implicit module and the user of that
+    /// module).
+    Benign,
+  };
+
   enum GCMode { NonGC, GCOnly, HybridGC };
   enum StackProtectorMode { SSPOff, SSPOn, SSPStrong, SSPReq };
 
@@ -486,16 +502,17 @@ class LangOptionsBase {
   };
 
   // Define simple language options (with no accessors).
-#define LANGOPT(Name, Bits, Default, Description) unsigned Name : Bits;
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
+  unsigned Name : Bits;
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)
 #include "clang/Basic/LangOptions.def"
 
 protected:
   // Define language options of enumeration type. These are private, and will
   // have accessors (below).
-#define LANGOPT(Name, Bits, Default, Description)
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  LLVM_PREFERRED_TYPE(Type) \
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
+  LLVM_PREFERRED_TYPE(Type)                                                    \
   unsigned Name : Bits;
 #include "clang/Basic/LangOptions.def"
 };
@@ -655,8 +672,8 @@ class LangOptions : public LangOptionsBase {
                   LangStandard::Kind LangStd = LangStandard::lang_unspecified);
 
   // Define accessors/mutators for language options of enumeration type.
-#define LANGOPT(Name, Bits, Default, Description)
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)                   \
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
   Type get##Name() const { return static_cast<Type>(Name); }                   \
   void set##Name(Type Value) {                                                 \
     assert(static_cast<unsigned>(Value) < (1u << Bits));                       \
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index eefd4885534c8..00da67fba669c 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -719,6 +719,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// Positive FileIDs are indexes into this table. Entry 0 indicates an invalid
   /// expansion.
   SmallVector<SrcMgr::SLocEntry, 0> LocalSLocEntryTable;
+  /// An in-parallel offset table, merely used for speeding up FileID lookup.
+  SmallVector<SourceLocation::UIntTy> LocalLocOffsetTable;
 
   /// The table of SLocEntries that are loaded from other modules.
   ///
@@ -767,6 +769,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// LastFileIDLookup records the last FileID looked up or created, because it
   /// is very common to look up many tokens from the same file.
   mutable FileID LastFileIDLookup;
+  mutable SourceLocation::UIntTy LastLookupStartOffset;
+  mutable SourceLocation::UIntTy LastLookupEndOffset; // exclude
 
   /// Holds information for \#line directives.
   ///
@@ -820,6 +824,12 @@ class SourceManager : public RefCountedBase<SourceManager> {
 
   mutable std::unique_ptr<SrcMgr::SLocEntry> FakeSLocEntryForRecovery;
 
+  /// Cache for filenames used in diagnostics. See 'getNameForDiagnostic()'.
+  mutable llvm::StringMap<StringRef> DiagNames;
+
+  /// Allocator for absolute/short names.
+  mutable llvm::BumpPtrAllocator DiagNameAlloc;
+
   /// Lazily computed map of macro argument chunks to their expanded
   /// source location.
   using MacroArgsMap = std::map<unsigned, SourceLocation>;
@@ -1844,6 +1854,16 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// \return Location of the top-level macro caller.
   SourceLocation getTopMacroCallerLoc(SourceLocation Loc) const;
 
+  /// Retrieve the name of a file suitable for diagnostics.
+  // FIXME: Passing in the DiagnosticOptions here is a workaround for the
+  // fact that installapi does some weird things with DiagnosticsEngines,
+  // which causes the 'Diag' member of SourceManager (or at least the
+  // DiagnosticsOptions member thereof) to be a dangling reference
+  // sometimes. We should probably fix that or decouple the two classes
+  // to avoid this issue entirely.
+  StringRef getNameForDiagnostic(StringRef Filename,
+                                 const DiagnosticOptions &Opts) const;
+
 private:
   friend class ASTReader;
   friend class ASTWriter;
@@ -1901,9 +1921,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
 
   FileID getFileID(SourceLocation::UIntTy SLocOffset) const {
     // If our one-entry cache covers this offset, just return it.
-    if (isOffsetInFileID(LastFileIDLookup, SLocOffset))
+    if (SLocOffset >= LastLookupStartOffset && SLocOffset < LastLookupEndOffset)
       return LastFileIDLookup;
-
     return getFileIDSlow(SLocOffset);
   }
 
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index c3bce6e807f34..c2c8cc3804e7f 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1034,9 +1034,15 @@ class TargetInfo : public TransferrableTargetInfo,
   /// set of primary and secondary targets.
   virtual llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const = 0;
 
+  enum class ArmStreamingKind {
+    NotStreaming,
+    StreamingCompatible,
+    Streaming,
+  };
+
   /// Returns target-specific min and max values VScale_Range.
   virtual std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts, bool IsArmStreamingFunction,
+  getVScaleRange(const LangOptions &LangOpts, ArmStreamingKind Mode,
                  llvm::StringMap<bool> *FeatureMap = nullptr) const {
     return std::nullopt;
   }
@@ -1851,7 +1857,7 @@ class TargetInfo : public TransferrableTargetInfo,
 
   /// Returns the version of the darwin target variant SDK which was used during
   /// the compilation if one was specified, or an empty version otherwise.
-  const std::optional<VersionTuple> getDarwinTargetVariantSDKVersion() const {
+  std::optional<VersionTuple> getDarwinTargetVariantSDKVersion() const {
     return !getTargetOpts().DarwinTargetVariantSDKVersion.empty()
                ? getTargetOpts().DarwinTargetVariantSDKVersion
                : std::optional<VersionTuple>();
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 7251cc2d1759a..0daef4ad1701f 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -767,47 +767,47 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)" in {
 
 ////////////////////////////////////////////////////////////////////////////////
 // Load/Store
-def LD1 : WInst<"vld1", ".(c*!)", "dQdPlQPl">;
-def LD2 : WInst<"vld2", "2(c*!)", "QUlQldQdPlQPl">;
-def LD3 : WInst<"vld3", "3(c*!)", "QUlQldQdPlQPl">;
-def LD4 : WInst<"vld4", "4(c*!)", "QUlQldQdPlQPl">;
-def ST1 : WInst<"vst1", "v*(.!)", "dQdPlQPl">;
-def ST2 : WInst<"vst2", "v*(2!)", "QUlQldQdPlQPl">;
-def ST3 : WInst<"vst3", "v*(3!)", "QUlQldQdPlQPl">;
-def ST4 : WInst<"vst4", "v*(4!)", "QUlQldQdPlQPl">;
+def LD1 : WInst<"vld1", ".(c*!)", "dQdPlQPlmQm">;
+def LD2 : WInst<"vld2", "2(c*!)", "QUlQldQdPlQPlmQm">;
+def LD3 : WInst<"vld3", "3(c*!)", "QUlQldQdPlQPlmQm">;
+def LD4 : WInst<"vld4", "4(c*!)", "QUlQldQdPlQPlmQm">;
+def ST1 : WInst<"vst1", "v*(.!)", "dQdPlQPlmQm">;
+def ST2 : WInst<"vst2", "v*(2!)", "QUlQldQdPlQPlmQm">;
+def ST3 : WInst<"vst3", "v*(3!)", "QUlQldQdPlQPlmQm">;
+def ST4 : WInst<"vst4", "v*(4!)", "QUlQldQdPlQPlmQm">;
 
 def LD1_X2 : WInst<"vld1_x2", "2(c*!)",
-                   "dQdPlQPl">;
+                   "dQdPlQPlmQm">;
 def LD1_X3 : WInst<"vld1_x3", "3(c*!)",
-                   "dQdPlQPl">;
+                   "dQdPlQPlmQm">;
 def LD1_X4 : WInst<"vld1_x4", "4(c*!)",
-                   "dQdPlQPl">;
+                   "dQdPlQPlmQm">;
 
-def ST1_X2 : WInst<"vst1_x2", "v*(2!)", "dQdPlQPl">;
-def ST1_X3 : WInst<"vst1_x3", "v*(3!)", "dQdPlQPl">;
-def ST1_X4 : WInst<"vst1_x4", "v*(4!)", "dQdPlQPl">;
+def ST1_X2 : WInst<"vst1_x2", "v*(2!)", "dQdPlQPlmQm">;
+def ST1_X3 : WInst<"vst1_x3", "v*(3!)", "dQdPlQPlmQm">;
+def ST1_X4 : WInst<"vst1_x4", "v*(4!)", "dQdPlQPlmQm">;
 
-def LD1_LANE : WInst<"vld1_lane", ".(c*!).I", "dQdPlQPl",
+def LD1_LANE : WInst<"vld1_lane", ".(c*!).I", "dQdPlQPlmQm",
                     [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
-def LD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def LD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<4, ImmCheckLaneIndex, 1>]>;
-def LD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def LD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<5, ImmCheckLaneIndex, 1>]>;
-def LD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def LD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<6, ImmCheckLaneIndex, 1>]>;
-def ST1_LANE : WInst<"vst1_lane", "v*(.!)I", "dQdPlQPl",
+def ST1_LANE : WInst<"vst1_lane", "v*(.!)I", "dQdPlQPlmQm",
                     [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
-def ST2_LANE : WInst<"vst2_lane", "v*(2!)I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def ST2_LANE : WInst<"vst2_lane", "v*(2!)I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<3, ImmCheckLaneIndex, 1>]>;
-def ST3_LANE : WInst<"vst3_lane", "v*(3!)I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def ST3_LANE : WInst<"vst3_lane", "v*(3!)I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<4, ImmCheckLaneIndex, 1>]>;
-def ST4_LANE : WInst<"vst4_lane", "v*(4!)I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def ST4_LANE : WInst<"vst4_lane", "v*(4!)I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<5, ImmCheckLaneIndex, 1>]>;
 
-def LD1_DUP  : WInst<"vld1_dup", ".(c*!)", "dQdPlQPl">;
-def LD2_DUP  : WInst<"vld2_dup", "2(c*!)", "dQdPlQPl">;
-def LD3_DUP  : WInst<"vld3_dup", "3(c*!)", "dQdPlQPl">;
-def LD4_DUP  : WInst<"vld4_dup", "4(c*!)", "dQdPlQPl">;
+def LD1_DUP  : WInst<"vld1_dup", ".(c*!)", "dQdPlQPlmQm">;
+def LD2_DUP  : WInst<"vld2_dup", "2(c*!)", "dQdPlQPlmQm">;
+def LD3_DUP  : WInst<"vld3_dup", "3(c*!)", "dQdPlQPlmQm">;
+def LD4_DUP  : WInst<"vld4_dup", "4(c*!)", "dQdPlQPlmQm">;
 
 def VLDRQ : WInst<"vldrq", "1(c*!)", "Pk">;
 def VSTRQ : WInst<"vstrq", "v*(1!)", "Pk">;
@@ -2194,4 +2194,4 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
   // fscale
   def FSCALE_V128 : WInst<"vscale", "..(.S)", "QdQfQh">;
   def FSCALE_V64 : WInst<"vscale", "(.q)(.q)(.qS)", "fh">;
-}
\ No newline at end of file
+}
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 1b3131be78452..7c0e5e7310527 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -121,7 +121,7 @@ def SVLD1UW_GATHER_INDEX_S   : MInst<"svld1uw_gather[_{2}base]_index_{d}", "dPul
 
 
 // First-faulting load one vector (scalar base)
-def SVLDFF1   : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
+def SVLDFF1   : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
 def SVLDFF1SB : MInst<"svldff1sb_{d}", "dPS", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1UB : MInst<"svldff1ub_{d}", "dPW", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1SH : MInst<"svldff1sh_{d}", "dPT", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldff1">;
@@ -130,7 +130,7 @@ def SVLDFF1SW : MInst<"svldff1sw_{d}", "dPU", "lUl",             [IsLoad],
 def SVLDFF1UW : MInst<"svldff1uw_{d}", "dPY", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldff1">;
 
 // First-faulting load one vector (scalar base, VL displacement)
-def SVLDFF1_VNUM   : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
+def SVLDFF1_VNUM   : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
 def SVLDFF1SB_VNUM : MInst<"svldff1sb_vnum_{d}", "dPSl", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1UB_VNUM : MInst<"svldff1ub_vnum_{d}", "dPWl", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1SH_VNUM : MInst<"svldff1sh_vnum_{d}", "dPTl", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldff1">;
@@ -223,7 +223,7 @@ def SVLDFF1SW_GATHER_INDEX_S : MInst<"svldff1sw_gather[_{2}base]_index_{d}", "dP
 def SVLDFF1UW_GATHER_INDEX_S : MInst<"svldff1uw_gather[_{2}base]_index_{d}", "dPul", "lUl",      [IsGatherLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldff1_gather_scalar_offset">;
 
 // Non-faulting load one vector (scalar base)
-def SVLDNF1   : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
+def SVLDNF1   : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
 def SVLDNF1SB : MInst<"svldnf1sb_{d}", "dPS", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1UB : MInst<"svldnf1ub_{d}", "dPW", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1SH : MInst<"svldnf1sh_{d}", "dPT", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldnf1">;
@@ -232,7 +232,7 @@ def SVLDNF1SW : MInst<"svldnf1sw_{d}", "dPU", "lUl",             [IsLoad],
 def SVLDNF1UW : MInst<"svldnf1uw_{d}", "dPY", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldnf1">;
 
 // Non-faulting load one vector (scalar base, VL displacement)
-def SVLDNF1_VNUM   : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
+def SVLDNF1_VNUM   : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
 def SVLDNF1SB_VNUM : MInst<"svldnf1sb_vnum_{d}", "dPSl", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1UB_VNUM : MInst<"svldnf1ub_vnum_{d}", "dPWl", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1SH_VNUM : MInst<"svldnf1sh_vnum_{d}", "dPTl", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldnf1">;
@@ -258,32 +258,32 @@ let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
 }
 
 // Load one quadword and replicate (scalar base)
-def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>;
+def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfdm", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>;
 
 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
   def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc",  "b", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>;
 }
 
-multiclass StructLoad<string name, string proto, string i> {
-  def : SInst<name, proto, "csilUcUsUiUlhfdm", MergeNone, i, [IsStructLoad, VerifyRuntimeMode]>;
+multiclass StructLoad<string name, string proto, string i, list<FlagType> f = []> {
+  def : SInst<name, proto, "csilUcUsUiUlhfdm", MergeNone, i, !listconcat(f, [IsStructLoad])>;
   let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
-    def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad, VerifyRuntimeMode]>;
+    def: SInst<name, proto, "b", MergeNone, i, !listconcat(f, [IsStructLoad])>;
   }
 }
 
 // Load N-element structure into N vectors (scalar base)
-defm SVLD2 : StructLoad<"svld2[_{2}]", "2Pc", "aarch64_sve_ld2_sret">;
-defm SVLD3 : StructLoad<"svld3[_{2}]", "3Pc", "aarch64_sve_ld3_sret">;
-defm SVLD4 : StructLoad<"svld4[_{2}]", "4Pc", "aarch64_sve_ld4_sret">;
+defm SVLD2 : StructLoad<"svld2[_{2}]", "2Pc", "aarch64_sve_ld2_sret", [VerifyRuntimeMode]>;
+defm SVLD3 : StructLoad<"svld3[_{2}]", "3Pc", "aarch64_sve_ld3_sret", [VerifyRuntimeMode]>;
+defm SVLD4 : StructLoad<"svld4[_{2}]", "4Pc", "aarch64_sve_ld4_sret", [VerifyRuntimeMode]>;
 
 // Load N-element structure into N vectors (scalar base, VL displacement)
-defm SVLD2_VNUM : StructLoad<"svld2_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2_sret">;
-defm SVLD3_VNUM : StructLoad<"svld3_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3_sret">;
-defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4_sret">;
+defm SVLD2_VNUM : StructLoad<"svld2_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2_sret", [VerifyRuntimeMode]>;
+defm SVLD3_VNUM : StructLoad<"svld3_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3_sret", [VerifyRuntimeMode]>;
+defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4_sret", [VerifyRuntimeMode]>;
 
 // Load one octoword and replicate (scalar base)
 let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in {
-  def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1ro">;
+  def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfdm", MergeNone, "aarch64_sve_ld1ro">;
 }
 let SVETargetGuard = "sve,f64mm,bf16", SMETargetGuard = InvalidMode in {
   def SVLD1RO_BF16 : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">;
@@ -434,21 +434,21 @@ def SVST1H_SCATTER_INDEX_S    : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "v
 def SVST1W_SCATTER_INDEX_S    : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl",      [IsScatterStore], MemEltTyInt32,   "aarch64_sve_st1_scatter_scalar_offset">;
 } // let SVETargetGuard = "sve"
 
-multiclass StructStore<string name, string proto, string i> {
-  def : SInst<name, proto, "csilUcUsUiUlhfdm", MergeNone, i, [IsStructStore, VerifyRuntimeMode]>;
+multiclass StructStore<string name, string proto, string i, list<FlagType> f = []> {
+  def : SInst<name, proto, "csilUcUsUiUlhfdm", MergeNone, i, !listconcat(f, [IsStructStore])>;
   let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
-    def: SInst<name, proto, "b", MergeNone, i, [IsStructStore, VerifyRuntimeMode]>;
+    def: SInst<name, proto, "b", MergeNone, i, !listconcat(f, [IsStructStore])>;
   }
 }
 // Store N vectors into N-element structure (scalar base)
-defm SVST2 : StructStore<"svst2[_{d}]", "vPp2", "aarch64_sve_st2">;
-defm SVST3 : StructStore<"svst3[_{d}]", "vPp3", "aarch64_sve_st3">;
-defm SVST4 : StructStore<"svst4[_{d}]", "vPp4", "aarch64_sve_st4">;
+defm SVST2 : StructStore<"svst2[_{d}]", "vPp2", "aarch64_sve_st2", [VerifyRuntimeMode]>;
+defm SVST3 : StructStore<"svst3[_{d}]", "vPp3", "aarch64_sve_st3", [VerifyRuntimeMode]>;
+defm SVST4 : StructStore<"svst4[_{d}]", "vPp4", "aarch64_sve_st4", [VerifyRuntimeMode]>;
 
 // Store N vectors into N-element structure (scalar base, VL displacement)
-defm SVST2_VNUM : StructStore<"svst2_vnum[_{d}]", "vPpl2", "aarch64_sve_st2">;
-defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">;
-defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">;
+defm SVST2_VNUM : StructStore<"svst2_vnum[_{d}]", "vPpl2", "aarch64_sve_st2", [VerifyRuntimeMode]>;
+defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3", [VerifyRuntimeMode]>;
+defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4", [VerifyRuntimeMode]>;
 
 // Store one vector, with no truncation, non-temporal (scalar base)
 def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfdm", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">;
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index 5faf81272ec53..d800028cdcee5 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -2313,7 +2313,7 @@ defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfdy",
                                  [["vv", "v", "vvUv"]]>;
 defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfdy",
                                  [["vx", "v", "vvz"]]>;
-defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd",
+defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfdy",
                                      [["vv", "v", "vv(Log2EEW:4)Uv"]]>;
 // unsigned type
 defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil",
diff --git a/clang/include/clang/Basic/riscv_vector_common.td b/clang/include/clang/Basic/riscv_vector_common.td
index e3d589699645b..767bcee7b1596 100644
--- a/clang/include/clang/Basic/riscv_vector_common.td
+++ b/clang/include/clang/Basic/riscv_vector_common.td
@@ -591,11 +591,8 @@ class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> {
 let UnMaskedPolicyScheme = HasPolicyOperand,
     HasMaskedOffOperand = false in {
   multiclass RVVSlideUpBuiltinSet {
-    defm "" : RVVOutBuiltinSet<NAME, "csilfd",
+    defm "" : RVVOutBuiltinSet<NAME, "csilxfdy",
                                [["vx","v", "vvvz"]]>;
-    let RequiredFeatures = ["zvfhmin"] in
-      defm "" : RVVOutBuiltinSet<NAME, "x",
-                                 [["vx","v", "vvvz"]]>;
     defm "" : RVVOutBuiltinSet<NAME, "csil",
                                [["vx","Uv", "UvUvUvz"]]>;
   }
@@ -616,11 +613,8 @@ let UnMaskedPolicyScheme = HasPassthruOperand,
       IntrinsicTypes = {ResultType, Ops.back()->getType()};
     }] in {
   multiclass RVVSlideDownBuiltinSet {
-    defm "" : RVVOutBuiltinSet<NAME, "csilfd",
+    defm "" : RVVOutBuiltinSet<NAME, "csilxfdy",
                                [["vx","v", "vvz"]]>;
-    let RequiredFeatures = ["zvfhmin"] in
-      defm "" : RVVOutBuiltinSet<NAME, "x",
-                                 [["vx","v", "vvz"]]>;
     defm "" : RVVOutBuiltinSet<NAME, "csil",
                                [["vx","Uv", "UvUvz"]]>;
   }
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 3e052c564112e..277c278fd38b7 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -63,7 +63,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value getConstAPInt(mlir::Location loc, mlir::Type typ,
                             const llvm::APInt &val) {
-    return create<cir::ConstantOp>(loc, getAttr<cir::IntAttr>(typ, val));
+    return create<cir::ConstantOp>(loc, cir::IntAttr::get(typ, val));
   }
 
   cir::ConstantOp getConstant(mlir::Location loc, mlir::TypedAttr attr) {
@@ -228,25 +228,28 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   cir::CallOp createCallOp(mlir::Location loc, mlir::SymbolRefAttr callee,
                            mlir::Type returnType, mlir::ValueRange operands,
-                           cir::SideEffect sideEffect = cir::SideEffect::All) {
-    return create<cir::CallOp>(loc, callee, returnType, operands, sideEffect);
+                           llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
+    auto op = create<cir::CallOp>(loc, callee, returnType, operands);
+    op->setAttrs(attrs);
+    return op;
   }
 
   cir::CallOp createCallOp(mlir::Location loc, cir::FuncOp callee,
                            mlir::ValueRange operands,
-                           cir::SideEffect sideEffect = cir::SideEffect::All) {
+                           llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
     return createCallOp(loc, mlir::SymbolRefAttr::get(callee),
                         callee.getFunctionType().getReturnType(), operands,
-                        sideEffect);
+                        attrs);
   }
 
-  cir::CallOp createIndirectCallOp(mlir::Location loc,
-                                   mlir::Value indirectTarget,
-                                   cir::FuncType funcType,
-                                   mlir::ValueRange operands,
-                                   cir::SideEffect sideEffect) {
-    return create<cir::CallOp>(loc, indirectTarget, funcType.getReturnType(),
-                               operands, sideEffect);
+  cir::CallOp
+  createIndirectCallOp(mlir::Location loc, mlir::Value indirectTarget,
+                       cir::FuncType funcType, mlir::ValueRange operands,
+                       llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
+    llvm::SmallVector<mlir::Value> resOperands{indirectTarget};
+    resOperands.append(operands.begin(), operands.end());
+    return createCallOp(loc, mlir::SymbolRefAttr(), funcType.getReturnType(),
+                        resOperands, attrs);
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h
index 64556bcb92baa..925a9a87e267f 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h
@@ -10,16 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_H
-#define LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_H
-
-#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
-#include "clang/CIR/Dialect/IR/CIRTypes.h"
+#ifndef CLANG_CIR_DIALECT_IR_CIRATTRS_H
+#define CLANG_CIR_DIALECT_IR_CIRATTRS_H
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 
-#include "llvm/ADT/SmallVector.h"
+#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
+
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 
 //===----------------------------------------------------------------------===//
 // CIR Dialect Attrs
@@ -27,15 +26,22 @@
 
 namespace clang {
 class FunctionDecl;
-class VarDecl;
 class RecordDecl;
+class VarDecl;
 } // namespace clang
 
 namespace cir {
 class ArrayType;
+class BoolType;
+class ComplexType;
+class IntType;
+class MethodType;
+class PointerType;
+class RecordType;
+class VectorType;
 } // namespace cir
 
 #define GET_ATTRDEF_CLASSES
 #include "clang/CIR/Dialect/IR/CIROpsAttributes.h.inc"
 
-#endif // LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_H
+#endif // CLANG_CIR_DIALECT_IR_CIRATTRS_H
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index 9a6560519eec4..75650beec0c6d 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -65,6 +65,54 @@ class CIRUnitAttr<string name, string attrMnemonic, list<Trait> traits = []>
   let isOptional = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// OptInfoAttr
+//===----------------------------------------------------------------------===//
+
+def CIR_OptInfoAttr : CIR_Attr<"OptInfo", "opt_info"> {
+  let summary =
+    "A module-level attribute that holds the optimization information";
+  let description = [{
+    The `#cir.opt_info` attribute holds optimization related information. For
+    now this attribute is a module-level attribute that gets attached to the
+    module operation during CIRGen.
+
+    The `level` parameter gives the optimization level. It must be an integer
+    between 0 and 3, inclusive. It corresponds to the `OptimizationLevel` field
+    within the `clang::CodeGenOptions` structure.
+
+    The `size` parameter gives the code size optimization level. It must be an
+    integer between 0 and 2, inclusive. It corresponds to the `OptimizeSize`
+    field within the `clang::CodeGenOptions` structure.
+
+    The `level` and `size` parameters correspond to the optimization level
+    command line options passed to clang driver. The table below lists the
+    current correspondance relationship:
+
+    | Flag             | `level` | `size` |
+    |------------------|---------|--------|
+    | `-O0` or nothing | 0       | 0      |
+    | `-O1`            | 1       | 0      |
+    | `-O2`            | 2       | 0      |
+    | `-O3`            | 3       | 0      |
+    | `-Os`            | 2       | 1      |
+    | `-Oz`            | 2       | 2      |
+
+    Examples:
+
+    ```mlir
+    #cir.opt_info<level = 2, size = 0>  // -O2
+    ```
+  }];
+
+  let parameters = (ins "unsigned":$level, "unsigned":$size);
+
+  let assemblyFormat = [{
+    `<` struct(params) `>`
+  }];
+  let genVerifyDecl = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // BoolAttr
 //===----------------------------------------------------------------------===//
@@ -117,70 +165,101 @@ def UndefAttr : CIR_TypedAttr<"Undef", "undef"> {
 // IntegerAttr
 //===----------------------------------------------------------------------===//
 
-def IntAttr : CIR_Attr<"Int", "int", [TypedAttrInterface]> {
+def CIR_IntAttr : CIR_Attr<"Int", "int", [TypedAttrInterface]> {
   let summary = "An attribute containing an integer value";
   let description = [{
     An integer attribute is a literal attribute that represents an integral
     value of the specified integer type.
   }];
-  let parameters = (ins AttributeSelfTypeParameter<"">:$type,
-                        APIntParameter<"">:$value);
+
+  let parameters = (ins
+    AttributeSelfTypeParameter<"", "cir::IntTypeInterface">:$type,
+    APIntParameter<"">:$value
+  );
+
   let builders = [
     AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
                                         "const llvm::APInt &":$value), [{
-      return $_get(type.getContext(), type, value);
+      auto intType = mlir::cast<cir::IntTypeInterface>(type);
+      return $_get(type.getContext(), intType, value);
     }]>,
     AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
                                         "int64_t":$value), [{
-      IntType intType = mlir::cast<IntType>(type);
+      auto intType = mlir::cast<cir::IntTypeInterface>(type);
       mlir::APInt apValue(intType.getWidth(), value, intType.isSigned());
       return $_get(intType.getContext(), intType, apValue);
     }]>,
   ];
+
   let extraClassDeclaration = [{
-    int64_t getSInt() const { return getValue().getSExtValue(); }
-    uint64_t getUInt() const { return getValue().getZExtValue(); }
-    bool isNullValue() const { return getValue() == 0; }
-    uint64_t getBitWidth() const {
-      return mlir::cast<IntType>(getType()).getWidth();
+    int64_t getSInt() const;
+    uint64_t getUInt() const;
+    bool isNullValue() const;
+    bool isSigned() const;
+    bool isUnsigned() const;
+    uint64_t getBitWidth() const;
+  }];
+
+ let extraClassDefinition = [{
+    int64_t $cppClass::getSInt() const {
+      return getValue().getSExtValue();
+    }
+    uint64_t $cppClass::getUInt() const {
+      return getValue().getZExtValue();
+    }
+    bool $cppClass::isNullValue() const {
+      return getValue() == 0;
     }
+    bool $cppClass::isSigned() const {
+      return mlir::cast<IntTypeInterface>(getType()).isSigned();
+    }
+    bool $cppClass::isUnsigned() const {
+      return mlir::cast<IntTypeInterface>(getType()).isUnsigned();
+    }
+    uint64_t $cppClass::getBitWidth() const {
+      return mlir::cast<IntTypeInterface>(getType()).getWidth();
+    }
+  }];
+
+  let assemblyFormat = [{
+    `<` custom<IntLiteral>($value, ref($type)) `>`
   }];
+
   let genVerifyDecl = 1;
-  let hasCustomAssemblyFormat = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // FPAttr
 //===----------------------------------------------------------------------===//
 
-def FPAttr : CIR_Attr<"FP", "fp", [TypedAttrInterface]> {
+def CIR_FPAttr : CIR_Attr<"FP", "fp", [TypedAttrInterface]> {
   let summary = "An attribute containing a floating-point value";
   let description = [{
     An fp attribute is a literal attribute that represents a floating-point
     value of the specified floating-point type. Supporting only CIR FP types.
   }];
+
   let parameters = (ins
     AttributeSelfTypeParameter<"", "::cir::FPTypeInterface">:$type,
     APFloatParameter<"">:$value
   );
+
   let builders = [
     AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
                                         "const llvm::APFloat &":$value), [{
       return $_get(type.getContext(), mlir::cast<FPTypeInterface>(type), value);
-    }]>,
-    AttrBuilder<(ins "mlir::Type":$type,
-                     "const llvm::APFloat &":$value), [{
-      return $_get($_ctxt, mlir::cast<FPTypeInterface>(type), value);
-    }]>,
+    }]>
   ];
+
   let extraClassDeclaration = [{
     static FPAttr getZero(mlir::Type type);
   }];
-  let genVerifyDecl = 1;
 
   let assemblyFormat = [{
     `<` custom<FloatLiteral>($value, ref($type)) `>`
   }];
+
+  let genVerifyDecl = 1;
 }
 
 
@@ -374,4 +453,67 @@ def CIR_VisibilityAttr : CIR_EnumAttr<CIR_VisibilityKind, "visibility"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// BitfieldInfoAttr
+//===----------------------------------------------------------------------===//
+
+def BitfieldInfoAttr : CIR_Attr<"BitfieldInfo", "bitfield_info"> {
+  let summary = "Represents info for a bit-field member";
+  let description = [{
+    Holds the following information about bitfields: name, storage type, size
+    and position in the storage, and signedness.
+    Example:
+      Given the following struct with bitfields:
+        ```c++
+        typedef struct {
+          int a : 4;
+          int b : 27;
+          int c : 17;
+          int d : 2;
+          int e : 15;
+        } S;
+        ```
+
+      The CIR representation of the struct `S` might look like:
+      ```mlir
+        !rec_S = !cir.record<struct "S" packed padded {!u64i, !u16i,
+                                                       !cir.array<!u8i x 2>}>
+      ```
+      And the bitfield info attribute for member `a` would be:
+      ```mlir
+        #bfi_a = #cir.bitfield_info<name = "a", storage_type = !u64i,
+                                    size = 4, offset = 0, is_signed = true>
+      ```
+
+      This metadata describes that field `a` is stored in a 64-bit integer,
+      is 4 bits wide, starts at offset 0, and is signed.
+  }];
+  let parameters = (ins "mlir::StringAttr":$name,
+                        "mlir::Type":$storageType,
+                        "uint64_t":$size,
+                        "uint64_t":$offset,
+                        "bool":$isSigned);
+
+  let assemblyFormat = [{`<` struct($name,
+                                    $storageType,
+                                    $size,
+                                    $offset,
+                                    $isSigned)
+                         `>`
+                        }];
+
+  let builders = [
+    AttrBuilder<(ins "llvm::StringRef":$name,
+                     "mlir::Type":$storageType,
+                     "uint64_t":$size,
+                     "uint64_t":$offset,
+                     "bool":$isSigned
+                     ), [{
+      return $_get($_ctxt, mlir::StringAttr::get($_ctxt, name), storageType,
+                   size, offset, isSigned);
+    }]>
+  ];
+}
+
+
 #endif // LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_TD
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDialect.h b/clang/include/clang/CIR/Dialect/IR/CIRDialect.h
index 5de1722cf5bc2..fdd56ac1f218f 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRDialect.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDialect.h
@@ -29,6 +29,7 @@
 #include "clang/CIR/Dialect/IR/CIRAttrs.h"
 #include "clang/CIR/Dialect/IR/CIROpsDialect.h.inc"
 #include "clang/CIR/Dialect/IR/CIROpsEnums.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
 #include "clang/CIR/Interfaces/CIRLoopOpInterface.h"
 #include "clang/CIR/Interfaces/CIROpInterfaces.h"
 #include "clang/CIR/MissingFeatures.h"
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDialect.td b/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
index 52e32eedf774d..fa73e5bf41f23 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
@@ -36,6 +36,10 @@ def CIR_Dialect : Dialect {
 
   let extraClassDeclaration = [{
     static llvm::StringRef getTripleAttrName() { return "cir.triple"; }
+    static llvm::StringRef getOptInfoAttrName() { return "cir.opt_info"; }
+    static llvm::StringRef getCalleeAttrName() { return "callee"; }
+    static llvm::StringRef getNoThrowAttrName() { return "nothrow"; }
+    static llvm::StringRef getSideEffectAttrName() { return "side_effect"; }
 
     void registerAttributes();
     void registerTypes();
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 4daff74cbae5a..6529f1386599c 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -489,7 +489,7 @@ def ReturnOp : CIR_Op<"return", [ParentOneOf<["FuncOp", "ScopeOp", "IfOp",
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
@@ -1669,6 +1669,85 @@ def GetGlobalOp : CIR_Op<"get_global",
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// GetBitfieldOp
+//===----------------------------------------------------------------------===//
+
+def GetBitfieldOp : CIR_Op<"get_bitfield"> {
+  let summary = "Get the information for a bitfield member";
+  let description = [{
+    The `cir.get_bitfield` operation provides a load-like access to
+    a bit field of a record.
+
+    It expects a name if a bit field, a pointer to a storage in the
+    base record, a type of the storage, a name of the bitfield,
+    a size the bit field, an offset of the bit field and a sign.
+
+    A unit attribute `volatile` can be used to indicate a volatile load of the
+    bitfield.
+
+    Example:
+    Suppose we have a struct with multiple bitfields stored in
+    different members. The `cir.get_bitfield` operation gets the value
+    of the bitfield.
+    ```C++
+    typedef struct {
+      int a : 4;
+      int b : 27;
+      int c : 17;
+      int d : 2;
+      int e : 15;
+    } S;
+
+    int load_bitfield(S& s) {
+      return s.e;
+    }
+    ```
+
+    ```mlir
+    // 'e' is in the storage with the index 1
+    !cir.record<struct "S" packed padded {!u64i, !u16i, !cir.array<!u8i x 2>}>
+    #bfi_e = #cir.bitfield_info<name = "e", storage_type = !u16i, size = 15,
+                                offset = 0, is_signed = true>
+
+    %2 = cir.load %0 : !cir.ptr<!cir.ptr<!record_type>>, !cir.ptr<!record_type>
+    %3 = cir.get_member %2[1] {name = "e"} : !cir.ptr<!record_type>
+                                                             -> !cir.ptr<!u16i>
+    %4 = cir.get_bitfield(#bfi_e, %3 : !cir.ptr<!u16i>) -> !s32i
+    ```
+    }];
+
+  let arguments = (ins
+    Arg<CIR_PointerType, "the address to load from", [MemRead]>:$addr,
+    BitfieldInfoAttr:$bitfield_info,
+    UnitAttr:$is_volatile
+    );
+
+  let results = (outs CIR_IntType:$result);
+
+  let assemblyFormat = [{ `(`$bitfield_info `,` $addr attr-dict `:`
+   qualified(type($addr)) `)` `->` type($result) }];
+
+  let builders = [
+    OpBuilder<(ins "mlir::Type":$type,
+                   "mlir::Value":$addr,
+                   "mlir::Type":$storage_type,
+                   "llvm::StringRef":$name,
+                   "unsigned":$size,
+                   "unsigned":$offset,
+                   "bool":$is_signed,
+                   "bool":$is_volatile
+                   ),
+   [{
+      BitfieldInfoAttr info =
+        BitfieldInfoAttr::get($_builder.getContext(),
+                              name, storage_type,
+                              size, offset, is_signed);
+      build($_builder, $_state, type, addr, info, is_volatile);
+    }]>
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // GetMemberOp
 //===----------------------------------------------------------------------===//
@@ -1912,6 +1991,7 @@ class CIR_CallOpBase<string mnemonic, list<Trait> extra_traits = []>
 
   dag commonArgs = (ins OptionalAttr<FlatSymbolRefAttr>:$callee,
       Variadic<CIR_AnyType>:$args,
+      UnitAttr:$nothrow,
       DefaultValuedAttr<CIR_SideEffect, "SideEffect::All">:$side_effect);
 }
 
@@ -1941,29 +2021,14 @@ def CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
   let arguments = commonArgs;
 
   let builders = [
-    // Build a call op for a direct call
     OpBuilder<(ins "mlir::SymbolRefAttr":$callee, "mlir::Type":$resType,
-                   "mlir::ValueRange":$operands,
-                   CArg<"SideEffect", "SideEffect::All">:$sideEffect), [{
-      assert(callee && "callee attribute is required for direct call");
+                   "mlir::ValueRange":$operands), [{
       $_state.addOperands(operands);
-      $_state.addAttribute("callee", callee);
-      $_state.addAttribute("side_effect",
-        SideEffectAttr::get($_builder.getContext(), sideEffect));
+      if (callee)
+        $_state.addAttribute("callee", callee);
       if (resType && !isa<VoidType>(resType))
         $_state.addTypes(resType);
-    }]>,
-    // Build a call op for an indirect call
-    OpBuilder<(ins "mlir::Value":$calleePtr, "mlir::Type":$resType,
-                   "mlir::ValueRange":$operands,
-                   CArg<"SideEffect", "SideEffect::All">:$sideEffect), [{
-      $_state.addOperands(calleePtr);
-      $_state.addOperands(operands);
-      if (resType && !isa<VoidType>(resType))
-        $_state.addTypes(resType);
-      $_state.addAttribute("side_effect",
-        SideEffectAttr::get($_builder.getContext(), sideEffect));
-    }]>,
+    }]>
   ];
 }
 
@@ -2457,27 +2522,142 @@ def ComplexImagOp : CIR_Op<"complex.imag", [Pure]> {
 }
 
 //===----------------------------------------------------------------------===//
-// ComplexEqualOp
+// Bit Manipulation Operations
 //===----------------------------------------------------------------------===//
 
-def ComplexEqualOp : CIR_Op<"complex.eq", [Pure, SameTypeOperands]> {
+class CIR_BitOpBase<string mnemonic, TypeConstraint operandTy>
+  : CIR_Op<mnemonic, [Pure, SameOperandsAndResultType]> {
+  let arguments = (ins operandTy:$input);
+  let results = (outs operandTy:$result);
 
-  let summary = "Computes whether two complex values are equal";
+  let assemblyFormat = [{
+    `(` $input `:` type($input) `)` `:` type($result) attr-dict
+  }];
+}
+
+class CIR_BitZeroCountOpBase<string mnemonic, TypeConstraint operandTy>
+  : CIR_BitOpBase<mnemonic, operandTy> {
+  let arguments = (ins operandTy:$input, UnitAttr:$poison_zero);
+
+  let assemblyFormat = [{
+    `(` $input `:` type($input) `)` (`poison_zero` $poison_zero^)?
+    `:` type($result) attr-dict
+  }];
+}
+
+def BitClrsbOp : CIR_BitOpBase<"bit.clrsb", CIR_SIntOfWidths<[32, 64]>> {
+  let summary = "Get the number of leading redundant sign bits in the input";
   let description = [{
-   The `complex.equal` op takes two complex numbers and returns whether
-   they are equal.
+    Compute the number of leading redundant sign bits in the input integer.
+
+    The input integer must be a signed integer. The most significant bit of the
+    input integer is the sign bit. The `cir.bit.clrsb` operation returns the
+    number of consecutive bits following the sign bit that are identical to the
+    sign bit.
+
+    The bit width of the input integer must be either 32 or 64.
+
+    Examples:
 
     ```mlir
-    %r = cir.complex.eq %a, %b : !cir.complex<!cir.float>
+    // %0 = 0b1101_1110_1010_1101_1011_1110_1110_1111
+    %0 = cir.const #cir.int<3735928559> : !s32i
+    // %1 will be 1 because there is 1 bit following the most significant bit
+    // that is identical to it.
+    %1 = cir.bit.clrsb(%0 : !s32i) : !s32i
+
+    // %2 = 1, 0b0000_0000_0000_0000_0000_0000_0000_0001
+    %2 = cir.const #cir.int<1> : !s32i
+    // %3 will be 30 because there are 30 consecutive bits following the sign
+    // bit that are identical to the sign bit.
+    %3 = cir.bit.clrsb(%2 : !s32i) : !s32i
     ```
   }];
+}
 
-  let results = (outs CIR_BoolType:$result);
-  let arguments = (ins CIR_ComplexType:$lhs, CIR_ComplexType:$rhs);
+def BitClzOp : CIR_BitZeroCountOpBase<"bit.clz",
+                                      CIR_UIntOfWidths<[16, 32, 64]>> {
+  let summary = "Get the number of leading 0-bits in the input";
+  let description = [{
+    Compute the number of leading 0-bits in the input.
 
-  let assemblyFormat = [{
-    $lhs `,` $rhs
-    `:` qualified(type($lhs)) attr-dict
+    The input integer must be an unsigned integer. The `cir.bit.clz` operation
+    returns the number of consecutive 0-bits at the most significant bit
+    position in the input.
+
+    If the `poison_zero` attribute is present, this operation will have
+    undefined behavior if the input value is 0.
+
+    Example:
+
+    ```mlir
+    // %0 = 0b0000_0000_0000_0000_0000_0000_0000_1000
+    %0 = cir.const #cir.int<8> : !u32i
+    // %1 will be 28
+    %1 = cir.bit.clz(%0 : !u32i) poison_zero : !u32i
+    ```
+  }];
+}
+
+def BitCtzOp : CIR_BitZeroCountOpBase<"bit.ctz",
+                                      CIR_UIntOfWidths<[16, 32, 64]>> {
+  let summary = "Get the number of trailing 0-bits in the input";
+  let description = [{
+    Compute the number of trailing 0-bits in the input.
+
+    The input integer must be an unsigned integer. The `cir.bit.ctz` operation
+    counts the number of consecutive 0-bits starting from the least significant
+    bit.
+
+    If the `poison_zero` attribute is present, this operation will have
+    undefined behavior if the input value is 0.
+
+    Example:
+
+    ```mlir
+    // %0 = 0b1000
+    %0 = cir.const #cir.int<8> : !u32i
+    // %1 will be 3
+    %1 = cir.bit.ctz(%0 : !u32i) poison_zero : !u32i
+    ```
+  }];
+}
+
+def BitParityOp : CIR_BitOpBase<"bit.parity", CIR_UIntOfWidths<[32, 64]>> {
+  let summary = "Get the parity of input";
+  let description = [{
+    Compute the parity of the input. The parity of an integer is the number of
+    1-bits in it modulo 2.
+
+    The input must be an unsigned integer.
+
+    Example:
+
+    ```mlir
+    // %0 = 0x0110_1000
+    %0 = cir.const #cir.int<104> : !u32i
+    // %1 will be 1 since there are three 1-bits in %0
+    %1 = cir.bit.parity(%0 : !u32i) : !u32i
+    ```
+  }];
+}
+
+def BitPopcountOp : CIR_BitOpBase<"bit.popcnt",
+                                  CIR_UIntOfWidths<[16, 32, 64]>> {
+  let summary = "Get the number of 1-bits in input";
+  let description = [{
+    Compute the number of 1-bits in the input.
+
+    The input must be an unsigned integer.
+
+    Example:
+
+    ```mlir
+    // %0 = 0x0110_1000
+    %0 = cir.const #cir.int<104> : !u32i
+    // %1 will be 3 since there are 3 1-bits in %0
+    %1 = cir.bit.popcnt(%0 : !u32i) : !u32i
+    ```
   }];
 }
 
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index bfe42562abad7..37c32d9889e8b 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -35,7 +35,8 @@ class CIR_Type<string name, string typeMnemonic, list<Trait> traits = [],
 
 def CIR_IntType : CIR_Type<"Int", "int", [
     DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
-    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_IntTypeInterface>,
 ]> {
   let summary = "Integer type with arbitrary precision up to a fixed limit";
   let description = [{
@@ -508,9 +509,12 @@ def CIR_RecordType : CIR_Type<"Record", "record", [
     A few examples:
 
     ```mlir
-        !complete = !cir.record<struct "complete" {!cir.int<u, 8>}>
-        !incomplete = !cir.record<struct "incomplete" incomplete>
-        !anonymous = !cir.record<struct {!cir.int<u, 8>}>
+        !rec_complete = !cir.record<struct "complete" {!u8i}>
+        !rec_incomplete = !cir.record<struct "incomplete" incomplete>
+        !anonymous_struct = !cir.record<struct {!u8i}>
+        !rec_p1 = !cir.record<struct "p1" packed {!u8i, !u8i}>
+        !rec_p2 = !cir.record<struct "p2" padded {!u8i, !u8i}>
+        !rec_p3 = !cir.record<struct "p3" packed padded {!s32i, !u8i, !u8i}>
     ```
 
     Incomplete records are mutable, meaning they can be later completed with a
@@ -551,7 +555,7 @@ def CIR_RecordType : CIR_Type<"Record", "record", [
       "bool":$padded,
       "RecordKind":$kind
     ), [{
-      return $_get($_ctxt, members, name, /*complete=*/true, packed, padded,
+      return $_get($_ctxt, members, name, /*incomplete=*/false, packed, padded,
                    kind);
     }]>,
 
@@ -563,6 +567,17 @@ def CIR_RecordType : CIR_Type<"Record", "record", [
       return $_get($_ctxt, /*members=*/llvm::ArrayRef<Type>{}, name,
                          /*incomplete=*/true, /*packed=*/false,
                          /*padded=*/false, kind);
+    }]>,
+
+    // Create an anonymous record type (always complete).
+    TypeBuilder<(ins
+      "llvm::ArrayRef<mlir::Type>":$members,
+      "bool":$packed,
+      "bool":$padded,
+      "RecordKind":$kind
+    ), [{
+      return $_get($_ctxt, members, mlir::StringAttr{}, /*incomplete=*/false,
+                      packed, padded, kind);
     }]>];
 
   let extraClassDeclaration = [{
diff --git a/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td b/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
index 5817b91b49e31..4702b4ef08acc 100644
--- a/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
+++ b/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
@@ -34,6 +34,8 @@ let cppNamespace = "::cir" in {
           "Return the number of operands, accounts for indirect call or "
           "exception info",
           "unsigned", "getNumArgOperands", (ins)>,
+      InterfaceMethod<"Return whether the callee is nothrow",
+                      "bool", "getNothrow", (ins)>,
       InterfaceMethod<"Return the side effects of the call operation",
                       "cir::SideEffect", "getSideEffect", (ins)>,
     ];
diff --git a/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td b/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td
index 1b1acf749e773..cf6c8571ddcd9 100644
--- a/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td
+++ b/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td
@@ -15,6 +15,48 @@
 
 include "mlir/IR/OpBase.td"
 
+def CIR_IntTypeInterface : TypeInterface<"IntTypeInterface"> {
+  let description = [{
+    Contains helper functions to query properties about an integer type.
+  }];
+  let cppNamespace = "::cir";
+  let methods = [
+    InterfaceMethod<[{
+        Returns true if this is a signed integer type.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isSigned",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_type.isSigned();
+      }]
+    >,
+    InterfaceMethod<[{
+        Returns true if this is an unsigned integer type.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isUnsigned",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_type.isUnsigned();
+      }]
+    >,
+    InterfaceMethod<[{
+        Returns the bit width of this integer type.
+      }],
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getWidth",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_type.getWidth();
+      }]
+    >
+  ];
+}
+
 def CIR_FPTypeInterface : TypeInterface<"FPTypeInterface"> {
   let description = [{
     Contains helper functions to query properties about a floating-point type.
diff --git a/clang/include/clang/CIR/LoweringHelpers.h b/clang/include/clang/CIR/LoweringHelpers.h
index 3077010ee5ffe..66e99c7e84416 100644
--- a/clang/include/clang/CIR/LoweringHelpers.h
+++ b/clang/include/clang/CIR/LoweringHelpers.h
@@ -37,4 +37,18 @@ std::optional<mlir::Attribute>
 lowerConstArrayAttr(cir::ConstArrayAttr constArr,
                     const mlir::TypeConverter *converter);
 
+mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc,
+                          mlir::Type typ, const llvm::APInt &val);
+
+mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ,
+                     unsigned val);
+
+mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs);
+
+mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs);
+
+mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs,
+                      const llvm::APInt &rhs);
+
+mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs);
 #endif
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 06dc1ab714dd7..c76737549a0fc 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -172,6 +172,7 @@ struct MissingFeatures {
   static bool alignCXXRecordDecl() { return false; }
   static bool armComputeVolatileBitfields() { return false; }
   static bool asmLabelAttr() { return false; }
+  static bool assignMemcpyizer() { return false; }
   static bool astVarDeclInterface() { return false; }
   static bool attributeBuiltin() { return false; }
   static bool attributeNoBuiltin() { return false; }
@@ -179,6 +180,7 @@ struct MissingFeatures {
   static bool builtinCall() { return false; }
   static bool builtinCallF128() { return false; }
   static bool builtinCallMathErrno() { return false; }
+  static bool builtinCheckKind() { return false; }
   static bool cgFPOptionsRAII() { return false; }
   static bool cirgenABIInfo() { return false; }
   static bool cleanupAfterErrorDiags() { return false; }
diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index ddb21c1e8a8b8..26aa3ccf84786 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -31,6 +31,8 @@ void AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
                      const llvm::opt::ArgList &Args,
                      llvm::opt::ArgStringList &CmdArgs, const JobAction &JA);
 
+const char *getLDMOption(const llvm::Triple &T, const llvm::opt::ArgList &Args);
+
 void addLinkerCompressDebugSectionsOption(const ToolChain &TC,
                                           const llvm::opt::ArgList &Args,
                                           llvm::opt::ArgStringList &CmdArgs);
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 0822df2640d23..95bea6cf1d76f 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1138,7 +1138,11 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
-def offload_arch_EQ : Joined<["--"], "offload-arch=">,
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
+def offload_arch_EQ : CommaJoined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
            "If 'native' is used the compiler will detect locally installed architectures. "
@@ -1169,10 +1173,12 @@ def offload_host_device : Flag<["--"], "offload-host-device">,
 def gpu_use_aux_triple_only : Flag<["--"], "gpu-use-aux-triple-only">,
   InternalDriverOpt, HelpText<"Prepare '-aux-triple' only without populating "
                               "'-aux-target-cpu' and '-aux-target-feature'.">;
+def offload_arch_tool_EQ : Joined<["--"], "offload-arch-tool=">,
+  HelpText<"Tool used for detecting offloading architectures in the system.">;
 def amdgpu_arch_tool_EQ : Joined<["--"], "amdgpu-arch-tool=">,
-  HelpText<"Tool used for detecting AMD GPU arch in the system.">;
+  Alias<offload_arch_tool_EQ>;
 def nvptx_arch_tool_EQ : Joined<["--"], "nvptx-arch-tool=">,
-  HelpText<"Tool used for detecting NVIDIA GPU arch in the system.">;
+  Alias<offload_arch_tool_EQ>;
 
 defm gpu_rdc : BoolFOption<"gpu-rdc",
   LangOpts<"GPURelocatableDeviceCode">, DefaultFalse,
@@ -1698,6 +1704,8 @@ def fsample_profile_use_profi : Flag<["-"], "fsample-profile-use-profi">,
                basic block counts to branch probabilities to fix them by extended
                and re-engineered classic MCMF (min-cost max-flow) approach.}]>;
 def fno_profile_sample_accurate : Flag<["-"], "fno-profile-sample-accurate">, Group<f_Group>;
+def fno_sample_profile_use_profi : Flag<["-"], "fno-sample-profile-use-profi">,
+                                   Group<f_Group>;
 def fno_auto_profile : Flag<["-"], "fno-auto-profile">, Group<f_Group>,
     Alias<fno_profile_sample_use>;
 def fauto_profile_EQ : Joined<["-"], "fauto-profile=">,
@@ -2668,6 +2676,15 @@ def fsanitize_skip_hot_cutoff_EQ
 
 } // end -f[no-]sanitize* flags
 
+def fallow_runtime_check_skip_hot_cutoff_EQ
+    : Joined<["-"], "fallow-runtime-check-skip-hot-cutoff=">,
+      Group<f_clang_Group>,
+      Visibility<[ClangOption, CC1Option]>,
+      HelpText<"Exclude __builtin_allow_runtime_check for the top hottest "
+               "code responsible for the given fraction of PGO counters "
+               "(0.0 [default] = skip none; 1.0 = skip all). "
+               "Argument format: <value>">;
+
 def funsafe_math_optimizations : Flag<["-"], "funsafe-math-optimizations">,
   Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Allow unsafe floating-point math optimizations which may decrease precision">,
@@ -3478,7 +3495,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
     HelpTextForVariants<[ClangOption, CC1Option],
       "Use the given vector functions library.\n"
       "  Note: -fveclib={ArmPL,SLEEF,libmvec} implies -fno-math-errno.\n"
@@ -3669,7 +3687,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
@@ -5206,6 +5224,14 @@ def msve_vector_bits_EQ : Joined<["-"], "msve-vector-bits=">, Group<m_aarch64_Fe
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify the size in bits of an SVE vector register. Defaults to the"
            " vector length agnostic value of \"scalable\". (AArch64 only)">;
+def msve_streaming_vector_bits_EQ
+    : Joined<["-"], "msve-streaming-vector-bits=">,
+      Group<m_aarch64_Features_Group>,
+      Visibility<[ClangOption, FlangOption]>,
+      HelpText<
+          "Specify the size in bits of an SVE vector register in streaming "
+          "mode. Defaults to the vector length agnostic value of "
+          "\"scalable\". (AArch64 only)">;
 } // let Flags = [TargetSpecific]
 
 def mvscale_min_EQ : Joined<["-"], "mvscale-min=">,
@@ -5217,6 +5243,17 @@ def mvscale_max_EQ : Joined<["-"], "mvscale-max=">,
   HelpText<"Specify the vscale maximum. Defaults to the"
            " vector length agnostic value of \"0\". (AArch64/RISC-V only)">,
   MarshallingInfoInt<LangOpts<"VScaleMax">>;
+def mvscale_streaming_min_EQ
+    : Joined<["-"], "mvscale-streaming-min=">,
+      Visibility<[CC1Option, FC1Option]>,
+      HelpText<"Specify the vscale minimum. Defaults to \"1\". (AArch64 only)">,
+      MarshallingInfoInt<LangOpts<"VScaleStreamingMin">>;
+def mvscale_streaming_max_EQ
+    : Joined<["-"], "mvscale-streaming-max=">,
+      Visibility<[CC1Option, FC1Option]>,
+      HelpText<"Specify the vscale maximum. Defaults to the"
+               " vector length agnostic value of \"0\". (AArch64 only)">,
+      MarshallingInfoInt<LangOpts<"VScaleStreamingMax">>;
 
 def msign_return_address_EQ : Joined<["-"], "msign-return-address=">,
   Visibility<[ClangOption, CC1Option]>,
@@ -9344,6 +9381,8 @@ def dxc_validator_path_EQ : Joined<["--"], "dxv-path=">, Group<dxc_Group>,
   HelpText<"DXIL validator installation path">;
 def dxc_disable_validation : DXCFlag<"Vd">,
   HelpText<"Disable validation">;
+def dxc_gis : DXCFlag<"Gis">,
+  HelpText<"Enable IEEE strict mode (equivalent to -ffp-model=strict)">;
 def : Option<["/", "-"], "Qembed_debug", KIND_FLAG>, Group<dxc_Group>,
   Flags<[Ignored]>, Visibility<[DXCOption]>,
   HelpText<"Embed PDB in shader container (ignored)">;
diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h
index f8663e3193a18..78dff1165dcf5 100644
--- a/clang/include/clang/Interpreter/Interpreter.h
+++ b/clang/include/clang/Interpreter/Interpreter.h
@@ -20,6 +20,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/Support/Error.h"
 #include <memory>
@@ -136,10 +137,14 @@ class Interpreter {
 public:
   virtual ~Interpreter();
   static llvm::Expected<std::unique_ptr<Interpreter>>
-  create(std::unique_ptr<CompilerInstance> CI);
+  create(std::unique_ptr<CompilerInstance> CI,
+         std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder = nullptr);
   static llvm::Expected<std::unique_ptr<Interpreter>>
   createWithCUDA(std::unique_ptr<CompilerInstance> CI,
                  std::unique_ptr<CompilerInstance> DCI);
+  static llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+  createLLJITBuilder(std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC,
+                     llvm::StringRef OrcRuntimePath);
   const ASTContext &getASTContext() const;
   ASTContext &getASTContext();
   const CompilerInstance *getCompilerInstance() const;
diff --git a/clang/include/clang/Interpreter/RemoteJITUtils.h b/clang/include/clang/Interpreter/RemoteJITUtils.h
new file mode 100644
index 0000000000000..8705a3b1f669d
--- /dev/null
+++ b/clang/include/clang/Interpreter/RemoteJITUtils.h
@@ -0,0 +1,38 @@
+//===-- RemoteJITUtils.h - Utilities for remote-JITing ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for ExecutorProcessControl-based remote JITing with Orc and
+// JITLink.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H
+#define LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h"
+#include "llvm/Support/Error.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
+launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory,
+               llvm::StringRef SlabAllocateSizeString);
+
+/// Create a JITLinkExecutor that connects to the given network address
+/// through a TCP socket. A valid NetworkAddress provides hostname and port,
+/// e.g. localhost:20000.
+llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
+connectTCPSocket(llvm::StringRef NetworkAddress, bool UseSharedMemory,
+                 llvm::StringRef SlabAllocateSizeString);
+
+#endif // LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H
diff --git a/clang/include/clang/Lex/LexHLSLRootSignature.h b/clang/include/clang/Lex/LexHLSLRootSignature.h
index 9901485b44d38..9bfefc96335b8 100644
--- a/clang/include/clang/Lex/LexHLSLRootSignature.h
+++ b/clang/include/clang/Lex/LexHLSLRootSignature.h
@@ -31,16 +31,17 @@ struct RootSignatureToken {
 
   Kind TokKind = Kind::invalid;
 
-  // Retain the SouceLocation of the token for diagnostics
-  clang::SourceLocation TokLoc;
+  // Retain the location offset of the token in the Signature
+  // string
+  uint32_t LocOffset;
 
   // Retain spelling of an numeric constant to be parsed later
   StringRef NumSpelling;
 
   // Constructors
-  RootSignatureToken(clang::SourceLocation TokLoc) : TokLoc(TokLoc) {}
-  RootSignatureToken(Kind TokKind, clang::SourceLocation TokLoc)
-      : TokKind(TokKind), TokLoc(TokLoc) {}
+  RootSignatureToken(uint32_t LocOffset) : LocOffset(LocOffset) {}
+  RootSignatureToken(Kind TokKind, uint32_t LocOffset)
+      : TokKind(TokKind), LocOffset(LocOffset) {}
 };
 
 inline const DiagnosticBuilder &
@@ -61,8 +62,7 @@ operator<<(const DiagnosticBuilder &DB, const RootSignatureToken::Kind Kind) {
 
 class RootSignatureLexer {
 public:
-  RootSignatureLexer(StringRef Signature, clang::SourceLocation SourceLoc)
-      : Buffer(Signature), SourceLoc(SourceLoc) {}
+  RootSignatureLexer(StringRef Signature) : Buffer(Signature) {}
 
   /// Consumes and returns the next token.
   RootSignatureToken consumeToken();
@@ -76,15 +76,13 @@ class RootSignatureLexer {
   }
 
 private:
-  // Internal buffer to iterate over
+  // Internal buffer state
   StringRef Buffer;
+  uint32_t LocOffset = 0;
 
   // Current peek state
   std::optional<RootSignatureToken> NextToken = std::nullopt;
 
-  // Passed down parameters from Sema
-  clang::SourceLocation SourceLoc;
-
   /// Consumes the buffer and returns the lexed token.
   RootSignatureToken lexToken();
 
@@ -92,7 +90,7 @@ class RootSignatureLexer {
   /// Updates the SourceLocation appropriately.
   void advanceBuffer(unsigned NumCharacters = 1) {
     Buffer = Buffer.drop_front(NumCharacters);
-    SourceLoc = SourceLoc.getLocWithOffset(NumCharacters);
+    LocOffset += NumCharacters;
   }
 };
 
diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h
index afa2c4d8cfe50..007c3f7ba1e9c 100644
--- a/clang/include/clang/Parse/ParseHLSLRootSignature.h
+++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
 #define LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
 
+#include "clang/AST/Expr.h"
 #include "clang/Basic/DiagnosticParse.h"
 #include "clang/Lex/LexHLSLRootSignature.h"
 #include "clang/Lex/Preprocessor.h"
@@ -27,8 +28,9 @@ namespace hlsl {
 
 class RootSignatureParser {
 public:
-  RootSignatureParser(SmallVector<llvm::hlsl::rootsig::RootElement> &Elements,
-                      RootSignatureLexer &Lexer, clang::Preprocessor &PP);
+  RootSignatureParser(llvm::dxbc::RootSignatureVersion Version,
+                      SmallVector<llvm::hlsl::rootsig::RootElement> &Elements,
+                      StringLiteral *Signature, Preprocessor &PP);
 
   /// Consumes tokens from the Lexer and constructs the in-memory
   /// representations of the RootElements. Tokens are consumed until an
@@ -71,7 +73,7 @@ class RootSignatureParser {
   // expected, or, there is a lexing error
 
   /// Root Element parse methods:
-  std::optional<llvm::hlsl::rootsig::RootFlags> parseRootFlags();
+  std::optional<llvm::dxbc::RootFlags> parseRootFlags();
   std::optional<llvm::hlsl::rootsig::RootConstants> parseRootConstants();
   std::optional<llvm::hlsl::rootsig::RootDescriptor> parseRootDescriptor();
   std::optional<llvm::hlsl::rootsig::DescriptorTable> parseDescriptorTable();
@@ -86,15 +88,15 @@ class RootSignatureParser {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
     std::optional<uint32_t> Num32BitConstants;
     std::optional<uint32_t> Space;
-    std::optional<llvm::hlsl::rootsig::ShaderVisibility> Visibility;
+    std::optional<llvm::dxbc::ShaderVisibility> Visibility;
   };
   std::optional<ParsedConstantParams> parseRootConstantParams();
 
   struct ParsedRootDescriptorParams {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
     std::optional<uint32_t> Space;
-    std::optional<llvm::hlsl::rootsig::ShaderVisibility> Visibility;
-    std::optional<llvm::hlsl::rootsig::RootDescriptorFlags> Flags;
+    std::optional<llvm::dxbc::ShaderVisibility> Visibility;
+    std::optional<llvm::dxbc::RootDescriptorFlags> Flags;
   };
   std::optional<ParsedRootDescriptorParams>
   parseRootDescriptorParams(RootSignatureToken::Kind RegType);
@@ -104,25 +106,25 @@ class RootSignatureParser {
     std::optional<uint32_t> NumDescriptors;
     std::optional<uint32_t> Space;
     std::optional<uint32_t> Offset;
-    std::optional<llvm::hlsl::rootsig::DescriptorRangeFlags> Flags;
+    std::optional<llvm::dxbc::DescriptorRangeFlags> Flags;
   };
   std::optional<ParsedClauseParams>
   parseDescriptorTableClauseParams(RootSignatureToken::Kind RegType);
 
   struct ParsedStaticSamplerParams {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
-    std::optional<llvm::hlsl::rootsig::SamplerFilter> Filter;
-    std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressU;
-    std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressV;
-    std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressW;
+    std::optional<llvm::dxbc::SamplerFilter> Filter;
+    std::optional<llvm::dxbc::TextureAddressMode> AddressU;
+    std::optional<llvm::dxbc::TextureAddressMode> AddressV;
+    std::optional<llvm::dxbc::TextureAddressMode> AddressW;
     std::optional<float> MipLODBias;
     std::optional<uint32_t> MaxAnisotropy;
-    std::optional<llvm::hlsl::rootsig::ComparisonFunc> CompFunc;
-    std::optional<llvm::hlsl::rootsig::StaticBorderColor> BorderColor;
+    std::optional<llvm::dxbc::ComparisonFunc> CompFunc;
+    std::optional<llvm::dxbc::StaticBorderColor> BorderColor;
     std::optional<float> MinLOD;
     std::optional<float> MaxLOD;
     std::optional<uint32_t> Space;
-    std::optional<llvm::hlsl::rootsig::ShaderVisibility> Visibility;
+    std::optional<llvm::dxbc::ShaderVisibility> Visibility;
   };
   std::optional<ParsedStaticSamplerParams> parseStaticSamplerParams();
 
@@ -132,17 +134,13 @@ class RootSignatureParser {
   std::optional<float> parseFloatParam();
 
   /// Parsing methods of various enums
-  std::optional<llvm::hlsl::rootsig::ShaderVisibility> parseShaderVisibility();
-  std::optional<llvm::hlsl::rootsig::SamplerFilter> parseSamplerFilter();
-  std::optional<llvm::hlsl::rootsig::TextureAddressMode>
-  parseTextureAddressMode();
-  std::optional<llvm::hlsl::rootsig::ComparisonFunc> parseComparisonFunc();
-  std::optional<llvm::hlsl::rootsig::StaticBorderColor>
-  parseStaticBorderColor();
-  std::optional<llvm::hlsl::rootsig::RootDescriptorFlags>
-  parseRootDescriptorFlags();
-  std::optional<llvm::hlsl::rootsig::DescriptorRangeFlags>
-  parseDescriptorRangeFlags();
+  std::optional<llvm::dxbc::ShaderVisibility> parseShaderVisibility();
+  std::optional<llvm::dxbc::SamplerFilter> parseSamplerFilter();
+  std::optional<llvm::dxbc::TextureAddressMode> parseTextureAddressMode();
+  std::optional<llvm::dxbc::ComparisonFunc> parseComparisonFunc();
+  std::optional<llvm::dxbc::StaticBorderColor> parseStaticBorderColor();
+  std::optional<llvm::dxbc::RootDescriptorFlags> parseRootDescriptorFlags();
+  std::optional<llvm::dxbc::DescriptorRangeFlags> parseDescriptorRangeFlags();
 
   /// Use NumericLiteralParser to convert CurToken.NumSpelling into a unsigned
   /// 32-bit integer
@@ -190,11 +188,23 @@ class RootSignatureParser {
   bool tryConsumeExpectedToken(RootSignatureToken::Kind Expected);
   bool tryConsumeExpectedToken(ArrayRef<RootSignatureToken::Kind> Expected);
 
+  /// Convert the token's offset in the signature string to its SourceLocation
+  ///
+  /// This allows to currently retrieve the location for multi-token
+  /// StringLiterals
+  SourceLocation getTokenLocation(RootSignatureToken Tok);
+
+  /// Construct a diagnostics at the location of the current token
+  DiagnosticBuilder reportDiag(unsigned DiagID) {
+    return getDiags().Report(getTokenLocation(CurToken), DiagID);
+  }
+
 private:
+  llvm::dxbc::RootSignatureVersion Version;
   SmallVector<llvm::hlsl::rootsig::RootElement> &Elements;
-  RootSignatureLexer &Lexer;
-
-  clang::Preprocessor &PP;
+  StringLiteral *Signature;
+  RootSignatureLexer Lexer;
+  Preprocessor &PP;
 
   RootSignatureToken CurToken;
 };
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index a47e23ffbd357..683934321a449 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -101,8 +101,13 @@ enum class ObjCTypeQual {
   NumQuals
 };
 
-/// TypeCastState - State whether an expression is or may be a type cast.
-enum class TypeCastState { NotTypeCast = 0, MaybeTypeCast, IsTypeCast };
+/// If a typo should be encountered, should typo correction suggest type names,
+/// non type names, or both?
+enum class TypoCorrectionTypeBehavior {
+  AllowNonTypes,
+  AllowTypes,
+  AllowBoth,
+};
 
 /// Control what ParseCastExpression will parse.
 enum class CastParseKind { AnyCastExpr = 0, UnaryExprOnly, PrimaryExprOnly };
@@ -116,6 +121,15 @@ enum class ParenParseOption {
   CastExpr         // Also allow '(' type-name ')' <anything>
 };
 
+/// In a call to ParseParenExpression, are the initial parentheses part of an
+/// operator that requires the parens be there (like typeof(int)) or could they
+/// be something else, such as part of a compound literal or a sizeof
+/// expression, etc.
+enum class ParenExprKind {
+  PartOfOperator, // typeof(int)
+  Unknown,        // sizeof(int) or sizeof (int)1.0f, or compound literal, etc
+};
+
 /// Describes the behavior that should be taken for an __if_exists
 /// block.
 enum class IfExistsBehavior {
@@ -3709,11 +3723,12 @@ class Parser : public CodeCompletionHandler {
   ///         assignment-expression ...[opt]
   ///         expression ',' assignment-expression ...[opt]
   /// \endverbatim
-  ExprResult
-  ParseExpression(TypeCastState isTypeCast = TypeCastState::NotTypeCast);
+  ExprResult ParseExpression(TypoCorrectionTypeBehavior CorrectionBehavior =
+                                 TypoCorrectionTypeBehavior::AllowNonTypes);
 
   ExprResult ParseConstantExpressionInExprEvalContext(
-      TypeCastState isTypeCast = TypeCastState::NotTypeCast);
+      TypoCorrectionTypeBehavior CorrectionBehavior =
+          TypoCorrectionTypeBehavior::AllowNonTypes);
   ExprResult ParseConstantExpression();
   ExprResult ParseArrayBoundExpression();
   ExprResult ParseCaseExpression(SourceLocation CaseLoc);
@@ -3750,8 +3765,9 @@ class Parser : public CodeCompletionHandler {
   ExprResult ParseConstraintLogicalOrExpression(bool IsTrailingRequiresClause);
 
   /// Parse an expr that doesn't include (top-level) commas.
-  ExprResult ParseAssignmentExpression(
-      TypeCastState isTypeCast = TypeCastState::NotTypeCast);
+  ExprResult
+  ParseAssignmentExpression(TypoCorrectionTypeBehavior CorrectionBehavior =
+                                TypoCorrectionTypeBehavior::AllowNonTypes);
 
   ExprResult ParseConditionalExpression();
 
@@ -4017,14 +4033,15 @@ class Parser : public CodeCompletionHandler {
   ///
   ExprResult ParseCastExpression(CastParseKind ParseKind,
                                  bool isAddressOfOperand, bool &NotCastExpr,
-                                 TypeCastState isTypeCast,
+                                 TypoCorrectionTypeBehavior CorrectionBehavior,
+                                 bool isVectorLiteral = false,
+                                 bool *NotPrimaryExpression = nullptr);
+  ExprResult ParseCastExpression(CastParseKind ParseKind,
+                                 bool isAddressOfOperand = false,
+                                 TypoCorrectionTypeBehavior CorrectionBehavior =
+                                     TypoCorrectionTypeBehavior::AllowNonTypes,
                                  bool isVectorLiteral = false,
                                  bool *NotPrimaryExpression = nullptr);
-  ExprResult
-  ParseCastExpression(CastParseKind ParseKind, bool isAddressOfOperand = false,
-                      TypeCastState isTypeCast = TypeCastState::NotTypeCast,
-                      bool isVectorLiteral = false,
-                      bool *NotPrimaryExpression = nullptr);
 
   /// Returns true if the next token cannot start an expression.
   bool isNotExpressionStart();
@@ -4181,10 +4198,15 @@ class Parser : public CodeCompletionHandler {
   /// \endverbatim
   bool ParseSimpleExpressionList(SmallVectorImpl<Expr *> &Exprs);
 
-  /// ParseParenExpression - This parses the unit that starts with a '(' token,
-  /// based on what is allowed by ExprType.  The actual thing parsed is returned
-  /// in ExprType. If stopIfCastExpr is true, it will only return the parsed
-  /// type, not the parsed cast-expression.
+  /// This parses the unit that starts with a '(' token, based on what is
+  /// allowed by ExprType. The actual thing parsed is returned in ExprType. If
+  /// StopIfCastExpr is true, it will only return the parsed type, not the
+  /// parsed cast-expression. If ParenBehavior is ParenExprKind::PartOfOperator,
+  /// the initial open paren and its matching close paren are known to be part
+  /// of another grammar production and not part of the operand. e.g., the
+  /// typeof and typeof_unqual operators in C. Otherwise, the function has to
+  /// parse the parens to determine whether they're part of a cast or compound
+  /// literal expression rather than a parenthesized type.
   ///
   /// \verbatim
   ///       primary-expression: [C99 6.5.1]
@@ -4209,7 +4231,9 @@ class Parser : public CodeCompletionHandler {
   ///       '(' '[' expression ']' { '[' expression ']' } cast-expression
   /// \endverbatim
   ExprResult ParseParenExpression(ParenParseOption &ExprType,
-                                  bool stopIfCastExpr, bool isTypeCast,
+                                  bool StopIfCastExpr,
+                                  ParenExprKind ParenBehavior,
+                                  TypoCorrectionTypeBehavior CorrectionBehavior,
                                   ParsedType &CastTy,
                                   SourceLocation &RParenLoc);
 
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 6c4a32c4ac2f0..567ad2d5934d4 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -1821,8 +1821,8 @@ class DecompositionDeclarator {
     if (DeleteBindings)
       delete[] Bindings;
     else
-      llvm::for_each(llvm::MutableArrayRef(Bindings, NumBindings),
-                     [](Binding &B) { B.Attrs.reset(); });
+      for (Binding &B : llvm::MutableArrayRef(Bindings, NumBindings))
+        B.Attrs.reset();
     Bindings = nullptr;
     NumBindings = 0;
     DeleteBindings = false;
diff --git a/clang/include/clang/Sema/HeuristicResolver.h b/clang/include/clang/Sema/HeuristicResolver.h
index df60d3359c6a6..e193c0bc14cd9 100644
--- a/clang/include/clang/Sema/HeuristicResolver.h
+++ b/clang/include/clang/Sema/HeuristicResolver.h
@@ -20,6 +20,7 @@ class CXXBasePath;
 class CXXDependentScopeMemberExpr;
 class DeclarationName;
 class DependentScopeDeclRefExpr;
+class FunctionProtoTypeLoc;
 class NamedDecl;
 class Type;
 class UnresolvedUsingValueDecl;
@@ -93,6 +94,12 @@ class HeuristicResolver {
   // during simplification, and the operation fails if no pointer type is found.
   QualType simplifyType(QualType Type, const Expr *E, bool UnwrapPointer);
 
+  // Given an expression `Fn` representing the callee in a function call,
+  // if the call is through a function pointer, try to find the declaration of
+  // the corresponding function pointer type, so that we can recover argument
+  // names from it.
+  FunctionProtoTypeLoc getFunctionProtoTypeLoc(const Expr *Fn) const;
+
 private:
   ASTContext &Ctx;
 };
diff --git a/clang/include/clang/Sema/MultiplexExternalSemaSource.h b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
index 7c66c26a17a13..391c2177d75ec 100644
--- a/clang/include/clang/Sema/MultiplexExternalSemaSource.h
+++ b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
@@ -94,8 +94,6 @@ class MultiplexExternalSemaSource : public ExternalSemaSource {
 
   bool wasThisDeclarationADefinition(const FunctionDecl *FD) override;
 
-  bool hasInitializerWithSideEffects(const VarDecl *VD) const override;
-
   /// Find all declarations with the given name in the
   /// given context.
   bool FindExternalVisibleDeclsByName(const DeclContext *DC,
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index 6b3c5a173417a..18d52782db2c5 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -1094,45 +1094,6 @@ enum AttributeDeclKind {
   ExpectedTypedef,
 };
 
-inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
-                                             const ParsedAttr &At) {
-  DB.AddTaggedVal(reinterpret_cast<uint64_t>(At.getAttrName()),
-                  DiagnosticsEngine::ak_identifierinfo);
-  return DB;
-}
-
-inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
-                                             const ParsedAttr *At) {
-  DB.AddTaggedVal(reinterpret_cast<uint64_t>(At->getAttrName()),
-                  DiagnosticsEngine::ak_identifierinfo);
-  return DB;
-}
-
-/// AttributeCommonInfo has a non-explicit constructor which takes an
-/// SourceRange as its only argument, this constructor has many uses so making
-/// it explicit is hard. This constructor causes ambiguity with
-/// DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB, SourceRange R).
-/// We use SFINAE to disable any conversion and remove any ambiguity.
-template <
-    typename ACI,
-    std::enable_if_t<std::is_same<ACI, AttributeCommonInfo>::value, int> = 0>
-inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
-                                             const ACI &CI) {
-  DB.AddTaggedVal(reinterpret_cast<uint64_t>(CI.getAttrName()),
-                  DiagnosticsEngine::ak_identifierinfo);
-  return DB;
-}
-
-template <
-    typename ACI,
-    std::enable_if_t<std::is_same<ACI, AttributeCommonInfo>::value, int> = 0>
-inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
-                                             const ACI *CI) {
-  DB.AddTaggedVal(reinterpret_cast<uint64_t>(CI->getAttrName()),
-                  DiagnosticsEngine::ak_identifierinfo);
-  return DB;
-}
-
 } // namespace clang
 
 #endif // LLVM_CLANG_SEMA_PARSEDATTR_H
diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h
index 07b9e1bc10f5a..757f3dcc3fe8d 100644
--- a/clang/include/clang/Sema/Scope.h
+++ b/clang/include/clang/Sema/Scope.h
@@ -158,11 +158,15 @@ class Scope {
     /// constructs, since they have the same behavior.
     OpenACCComputeConstructScope = 0x10000000,
 
+    /// This is the scope of an OpenACC Loop/Combined construct, which is used
+    /// to determine whether a 'cache' construct variable reference is legal.
+    OpenACCLoopConstructScope = 0x20000000,
+
     /// This is a scope of type alias declaration.
-    TypeAliasScope = 0x20000000,
+    TypeAliasScope = 0x40000000,
 
     /// This is a scope of friend declaration.
-    FriendScope = 0x40000000,
+    FriendScope = 0x80000000,
   };
 
 private:
@@ -294,7 +298,7 @@ class Scope {
   // is disallowed despite being a continue scope.
   void setIsConditionVarScope(bool InConditionVarScope) {
     Flags = (Flags & ~ConditionVarScope) |
-            (InConditionVarScope ? ConditionVarScope : 0);
+            (InConditionVarScope ? ConditionVarScope : NoScope);
   }
 
   bool isConditionVarScope() const {
@@ -549,6 +553,10 @@ class Scope {
     return getFlags() & Scope::OpenACCComputeConstructScope;
   }
 
+  bool isOpenACCLoopConstructScope() const {
+    return getFlags() & Scope::OpenACCLoopConstructScope;
+  }
+
   /// Determine if this scope (or its parents) are a compute construct. If the
   /// argument is provided, the search will stop at any of the specified scopes.
   /// Otherwise, it will stop only at the normal 'no longer search' scopes.
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 9afe3c7710476..a21836dffb0e6 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -6761,6 +6761,7 @@ class Sema final : public SemaBase {
       EK_Decltype,
       EK_TemplateArgument,
       EK_AttrArgument,
+      EK_VariableInit,
       EK_Other
     } ExprContext;
 
@@ -7723,12 +7724,12 @@ class Sema final : public SemaBase {
 
   class ConditionResult {
     Decl *ConditionVar;
-    FullExprArg Condition;
+    ExprResult Condition;
     bool Invalid;
     std::optional<bool> KnownValue;
 
     friend class Sema;
-    ConditionResult(Sema &S, Decl *ConditionVar, FullExprArg Condition,
+    ConditionResult(Sema &S, Decl *ConditionVar, ExprResult Condition,
                     bool IsConstexpr)
         : ConditionVar(ConditionVar), Condition(Condition), Invalid(false) {
       if (IsConstexpr && Condition.get()) {
@@ -7739,7 +7740,7 @@ class Sema final : public SemaBase {
       }
     }
     explicit ConditionResult(bool Invalid)
-        : ConditionVar(nullptr), Condition(nullptr), Invalid(Invalid),
+        : ConditionVar(nullptr), Condition(Invalid), Invalid(Invalid),
           KnownValue(std::nullopt) {}
 
   public:
@@ -15192,11 +15193,18 @@ class Sema final : public SemaBase {
                                SourceLocation Loc);
   QualType BuiltinRemoveReference(QualType BaseType, UTTKind UKind,
                                   SourceLocation Loc);
+
+  QualType BuiltinRemoveCVRef(QualType BaseType, SourceLocation Loc) {
+    return BuiltinRemoveReference(BaseType, UTTKind::RemoveCVRef, Loc);
+  }
+
   QualType BuiltinChangeCVRQualifiers(QualType BaseType, UTTKind UKind,
                                       SourceLocation Loc);
   QualType BuiltinChangeSignedness(QualType BaseType, UTTKind UKind,
                                    SourceLocation Loc);
 
+  bool BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT, QualType RhsT);
+
   /// Ensure that the type T is a literal type.
   ///
   /// This routine checks whether the type @p T is a literal type. If @p T is an
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index c4a986a3eb6f9..b7e7f5d97bcef 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -117,6 +117,15 @@ class SemaOpenACC : public SemaBase {
     OpenACCDirectiveKind DirectiveKind = OpenACCDirectiveKind::Invalid;
   } TileInfo;
 
+  /// The 'cache' var-list requires some additional work to track variable
+  /// references to make sure they are on the 'other' side of a `loop`. This
+  /// structure is used during parse time to track vardecl use while parsing a
+  /// cache var list.
+  struct CacheParseInfo {
+    bool ParsingCacheVarList = false;
+    bool IsInvalidCacheRef = false;
+  } CacheInfo;
+
   /// A list of the active reduction clauses, which allows us to check that all
   /// vars on nested constructs for the same reduction var have the same
   /// reduction operator. Currently this is enforced against all constructs
@@ -861,6 +870,12 @@ class SemaOpenACC : public SemaBase {
   ExprResult ActOnIntExpr(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
                           SourceLocation Loc, Expr *IntExpr);
 
+  /// Called right before a 'var' is parsed, so we can set the state for parsing
+  /// a 'cache' var.
+  void ActOnStartParseVar(OpenACCDirectiveKind DK, OpenACCClauseKind CK);
+  /// Called only if the parse of a 'var' was invalid, else 'ActOnVar' should be
+  /// called.
+  void ActOnInvalidParseVar();
   /// Called when encountering a 'var' for OpenACC, ensures it is actually a
   /// declaration reference to a variable of the correct type.
   ExprResult ActOnVar(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
@@ -913,6 +928,10 @@ class SemaOpenACC : public SemaBase {
                            OpenACCDirectiveKind DK, OpenACCGangKind GK,
                            Expr *E);
 
+  // Called when a declaration is referenced, so that we can make sure certain
+  // clauses don't do the 'wrong' thing/have incorrect references.
+  void CheckDeclReference(SourceLocation Loc, Expr *E, Decl *D);
+
   // Does the checking for a 'gang' clause that needs to be done in dependent
   // and not dependent cases.
   OpenACCClause *
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 7d4b4467eb97d..3d96f8e65180d 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -525,7 +525,7 @@ class ASTReader
   ContinuousRangeMap<unsigned, ModuleFile*, 64> GlobalSLocEntryMap;
 
   using GlobalSLocOffsetMapType =
-      ContinuousRangeMap<unsigned, ModuleFile *, 64>;
+      ContinuousRangeMap<SourceLocation::UIntTy, ModuleFile *, 64>;
 
   /// A map of reversed (SourceManager::MaxLoadedOffset - SLocOffset)
   /// SourceLocation offsets to the modules containing them.
@@ -1453,12 +1453,6 @@ class ASTReader
     const StringRef &operator*() && = delete;
   };
 
-  /// VarDecls with initializers containing side effects must be emitted,
-  /// but DeclMustBeEmitted is not allowed to deserialize the intializer.
-  /// FIXME: Lower memory usage by removing VarDecls once the initializer
-  /// is deserialized.
-  llvm::SmallPtrSet<Decl *, 16> InitSideEffectVars;
-
 public:
   /// Get the buffer for resolving paths.
   SmallString<0> &getPathBuf() { return PathBuf; }
@@ -2410,8 +2404,6 @@ class ASTReader
 
   bool wasThisDeclarationADefinition(const FunctionDecl *FD) override;
 
-  bool hasInitializerWithSideEffects(const VarDecl *VD) const override;
-
   /// Retrieve a selector from the given module with its local ID
   /// number.
   Selector getLocalSelector(ModuleFile &M, unsigned LocalID);
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
index e995151927c96..bd11e0d40cfa9 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
@@ -448,14 +448,15 @@ class ExplodedNodeSet {
 
 public:
   ExplodedNodeSet(ExplodedNode *N) {
-    assert(N && !static_cast<ExplodedNode*>(N)->isSink());
+    assert(N && !N->isSink());
     Impl.insert(N);
   }
 
   ExplodedNodeSet() = default;
 
   void Add(ExplodedNode *N) {
-    if (N && !static_cast<ExplodedNode*>(N)->isSink()) Impl.insert(N);
+    if (N && !N->isSink())
+      Impl.insert(N);
   }
 
   using iterator = ImplTy::iterator;
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
index 70a7953918ace..a6cb6c0f12a8c 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
@@ -598,6 +598,9 @@ class SMTConv {
     if (APSIntBitwidth == 1 && Ty.isNull())
       return {Int.extend(Ctx.getTypeSize(Ctx.BoolTy)),
               getAPSIntType(Ctx, NewInt)};
+    else if (APSIntBitwidth == 1 && !Ty.isNull())
+      return {Int.extend(Ctx.getTypeSize(getAPSIntType(Ctx, Int))),
+              getAPSIntType(Ctx, NewInt)};
     if (llvm::isPowerOf2_32(APSIntBitwidth) || Ty.isNull())
       return {Int, Ty};
     return {Int.extend(Ctx.getTypeSize(Ty)), Ty};
diff --git a/clang/include/clang/Tooling/Transformer/Transformer.h b/clang/include/clang/Tooling/Transformer/Transformer.h
index 71b1fe81b9518..ecf7c0912d2d8 100644
--- a/clang/include/clang/Tooling/Transformer/Transformer.h
+++ b/clang/include/clang/Tooling/Transformer/Transformer.h
@@ -42,11 +42,6 @@ class TransformerImpl {
   virtual void
   onMatchImpl(const ast_matchers::MatchFinder::MatchResult &Result) = 0;
 };
-
-// FIXME: Use std::type_identity or backport when available.
-template <class T> struct type_identity {
-  using type = T;
-};
 } // namespace detail
 
 template <typename T> struct TransformerResult {
@@ -95,8 +90,8 @@ class Transformer : public ast_matchers::MatchFinder::MatchCallback {
   template <typename MetadataT>
   explicit Transformer(
       transformer::RewriteRuleWith<MetadataT> Rule,
-      std::function<void(llvm::Expected<TransformerResult<
-                             typename detail::type_identity<MetadataT>::type>>)>
+      std::function<void(
+          llvm::Expected<TransformerResult<llvm::type_identity_t<MetadataT>>>)>
           Consumer);
 
   /// N.B. Passes `this` pointer to `MatchFinder`.  So, this object should not
@@ -200,8 +195,8 @@ template <typename T> class WithMetadataImpl final : public TransformerImpl {
 template <typename MetadataT>
 Transformer::Transformer(
     transformer::RewriteRuleWith<MetadataT> Rule,
-    std::function<void(llvm::Expected<TransformerResult<
-                           typename detail::type_identity<MetadataT>::type>>)>
+    std::function<void(
+        llvm::Expected<TransformerResult<llvm::type_identity_t<MetadataT>>>)>
         Consumer)
     : Impl(std::make_unique<detail::WithMetadataImpl<MetadataT>>(
           std::move(Rule), std::move(Consumer))) {}
diff --git a/clang/lib/AST/APValue.cpp b/clang/lib/AST/APValue.cpp
index c641ff6b99bab..ee3dc84479fd9 100644
--- a/clang/lib/AST/APValue.cpp
+++ b/clang/lib/AST/APValue.cpp
@@ -821,7 +821,7 @@ void APValue::printPretty(raw_ostream &Out, const PrintingPolicy &Policy,
     else if (isLValueOnePastTheEnd())
       Out << "*(&";
 
-    QualType ElemTy = Base.getType();
+    QualType ElemTy = Base.getType().getNonReferenceType();
     if (const ValueDecl *VD = Base.dyn_cast<const ValueDecl*>()) {
       Out << *VD;
     } else if (TypeInfoLValue TI = Base.dyn_cast<TypeInfoLValue>()) {
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index b13bdd5642977..679812adcdf12 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -10513,8 +10513,8 @@ bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
 /// getRVVTypeSize - Return RVV vector register size.
 static uint64_t getRVVTypeSize(ASTContext &Context, const BuiltinType *Ty) {
   assert(Ty->isRVVVLSBuiltinType() && "Invalid RVV Type");
-  auto VScale =
-      Context.getTargetInfo().getVScaleRange(Context.getLangOpts(), false);
+  auto VScale = Context.getTargetInfo().getVScaleRange(
+      Context.getLangOpts(), TargetInfo::ArmStreamingKind::NotStreaming);
   if (!VScale)
     return 0;
 
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index 20a4c3a43d2ca..0fa9234159f22 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -506,7 +506,15 @@ void clang::FormatASTNodeDiagnosticArgument(
     case DiagnosticsEngine::ak_attr: {
       const Attr *At = reinterpret_cast<Attr *>(Val);
       assert(At && "Received null Attr object!");
-      OS << '\'' << At->getSpelling() << '\'';
+
+      OS << '\'';
+      if (At->hasScope()) {
+        OS << At->getNormalizedFullName(At->getScopeName()->getName(),
+                                        At->getSpelling());
+      } else {
+        OS << At->getSpelling();
+      }
+      OS << '\'';
       NeedQuotes = false;
       break;
     }
@@ -516,6 +524,20 @@ void clang::FormatASTNodeDiagnosticArgument(
       E->printPretty(OS, /*Helper=*/nullptr, Context.getPrintingPolicy());
       break;
     }
+    case DiagnosticsEngine::ak_attr_info: {
+      AttributeCommonInfo *AT = reinterpret_cast<AttributeCommonInfo *>(Val);
+      assert(AT && "Received null AttributeCommonInfo object!");
+
+      OS << '\'';
+      if (AT->isStandardAttributeSyntax()) {
+        OS << AT->getNormalizedFullName();
+      } else {
+        OS << AT->getAttrName()->getName();
+      }
+      OS << '\'';
+      NeedQuotes = false;
+      break;
+    }
   }
 
   if (NeedQuotes) {
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 81da16e797bce..ee9c3890794af 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -2886,7 +2886,20 @@ bool Compiler<Emitter>::VisitMaterializeTemporaryExpr(
 template <class Emitter>
 bool Compiler<Emitter>::VisitCXXBindTemporaryExpr(
     const CXXBindTemporaryExpr *E) {
-  return this->delegate(E->getSubExpr());
+  const Expr *SubExpr = E->getSubExpr();
+
+  if (Initializing)
+    return this->delegate(SubExpr);
+
+  // Make sure we create a temporary even if we're discarding, since that will
+  // make sure we will also call the destructor.
+
+  if (!this->visit(SubExpr))
+    return false;
+
+  if (DiscardResult)
+    return this->emitPopPtr(E);
+  return true;
 }
 
 template <class Emitter>
@@ -4574,22 +4587,23 @@ VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD,
 template <class Emitter>
 bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
                                            bool ConstantContext) {
-  std::optional<PrimType> VarT = classify(VD->getType());
 
   // We only create variables if we're evaluating in a constant context.
   // Otherwise, just evaluate the initializer and return it.
   if (!ConstantContext) {
     DeclScope<Emitter> LS(this, VD);
-    if (!this->visit(VD->getAnyInitializer()))
+    const Expr *Init = VD->getInit();
+    if (!this->visit(Init))
       return false;
-    return this->emitRet(VarT.value_or(PT_Ptr), VD) && LS.destroyLocals() &&
-           this->emitCheckAllocations(VD);
+    return this->emitRet(classify(Init).value_or(PT_Ptr), VD) &&
+           LS.destroyLocals() && this->emitCheckAllocations(VD);
   }
 
   LocalScope<Emitter> VDScope(this, VD);
   if (!this->visitVarDecl(VD, /*Toplevel=*/true))
     return false;
 
+  std::optional<PrimType> VarT = classify(VD->getType());
   if (Context::shouldBeGloballyIndexed(VD)) {
     auto GlobalIndex = P.getGlobal(VD);
     assert(GlobalIndex); // visitVarDecl() didn't return false.
@@ -5274,7 +5288,7 @@ bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
 template <class Emitter>
 bool Compiler<Emitter>::maybeEmitDeferredVarInit(const VarDecl *VD) {
   if (auto *DD = dyn_cast_if_present<DecompositionDecl>(VD)) {
-    for (auto *BD : DD->bindings())
+    for (auto *BD : DD->flat_bindings())
       if (auto *KD = BD->getHoldingVar(); KD && !this->visitVarDecl(KD))
         return false;
   }
@@ -5544,7 +5558,6 @@ bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
   const Stmt *BeginStmt = S->getBeginStmt();
   const Stmt *RangeStmt = S->getRangeStmt();
   const Stmt *EndStmt = S->getEndStmt();
-  const VarDecl *LoopVar = S->getLoopVariable();
 
   LabelTy EndLabel = this->getLabel();
   LabelTy CondLabel = this->getLabel();
@@ -5569,7 +5582,7 @@ bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
   if (!this->jumpFalse(EndLabel))
     return false;
 
-  if (!this->visitVarDecl(LoopVar))
+  if (!this->visitDeclStmt(S->getLoopVarStmt(), /*EvaluateConditionDecl=*/true))
     return false;
 
   // Body.
@@ -5616,6 +5629,9 @@ bool Compiler<Emitter>::visitContinueStmt(const ContinueStmt *S) {
 template <class Emitter>
 bool Compiler<Emitter>::visitSwitchStmt(const SwitchStmt *S) {
   const Expr *Cond = S->getCond();
+  if (Cond->containsErrors())
+    return false;
+
   PrimType CondT = this->classifyPrim(Cond->getType());
   LocalScope<Emitter> LS(this);
 
@@ -5835,6 +5851,8 @@ bool Compiler<Emitter>::compileConstructor(const CXXConstructorDecl *Ctor) {
     return false;
 
   if (R->isUnion() && Ctor->isCopyOrMoveConstructor()) {
+    if (R->getNumFields() == 0)
+      return this->emitRetVoid(Ctor);
     // union copy and move ctors are special.
     assert(cast<CompoundStmt>(Ctor->getBody())->body_empty());
     if (!this->emitThis(Ctor))
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index a1d068cc7e0ae..1a5fd86785872 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -574,17 +574,17 @@ template <class Emitter> class LocalScope : public VariableScope<Emitter> {
     // Emit destructor calls for local variables of record
     // type with a destructor.
     for (Scope::Local &Local : llvm::reverse(this->Ctx->Descriptors[*Idx])) {
-      if (!Local.Desc->isPrimitive() && !Local.Desc->isPrimitiveArray()) {
-        if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
-          return false;
+      if (Local.Desc->hasTrivialDtor())
+        continue;
+      if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
+        return false;
 
-        if (!this->Ctx->emitDestruction(Local.Desc, Local.Desc->getLoc()))
-          return false;
+      if (!this->Ctx->emitDestruction(Local.Desc, Local.Desc->getLoc()))
+        return false;
 
-        if (!this->Ctx->emitPopPtr(E))
-          return false;
-        removeIfStoredOpaqueValue(Local);
-      }
+      if (!this->Ctx->emitPopPtr(E))
+        return false;
+      removeIfStoredOpaqueValue(Local);
     }
     return true;
   }
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index 971eb7fd58876..a629ff9569428 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -67,7 +67,8 @@ bool Context::evaluateAsRValue(State &Parent, const Expr *E, APValue &Result) {
   }
 
   if (!Recursing) {
-    assert(Stk.empty());
+    // We *can* actually get here with a non-empty stack, since
+    // things like InterpState::noteSideEffect() exist.
     C.cleanup();
 #ifndef NDEBUG
     // Make sure we don't rely on some value being still alive in
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 46e4d0d940b3e..c89eca9bef440 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -502,6 +502,21 @@ SourceInfo Descriptor::getLoc() const {
   llvm_unreachable("Invalid descriptor type");
 }
 
+bool Descriptor::hasTrivialDtor() const {
+  if (isPrimitive() || isPrimitiveArray() || isDummy())
+    return true;
+
+  if (isRecord()) {
+    assert(ElemRecord);
+    const CXXDestructorDecl *Dtor = ElemRecord->getDestructor();
+    return !Dtor || Dtor->isTrivial();
+  }
+
+  // Composite arrays.
+  assert(ElemDesc);
+  return ElemDesc->hasTrivialDtor();
+}
+
 bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); }
 
 InitMap::InitMap(unsigned N)
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index f25ad8f4c758c..4591eabb69bb4 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -281,6 +281,9 @@ struct Descriptor final {
   /// Checks if this is a dummy descriptor.
   bool isDummy() const { return IsDummy; }
 
+  /// Whether variables of this descriptor need their destructor called or not.
+  bool hasTrivialDtor() const;
+
   void dump() const;
   void dump(llvm::raw_ostream &OS) const;
   void dumpFull(unsigned Offset = 0, unsigned Indent = 0) const;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index a3eecd06369b1..f64501f4a31e8 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -63,7 +63,7 @@ template <> inline std::string printArg<Floating>(Program &P, CodePtr &OpPC) {
 
   std::string S;
   llvm::raw_string_ostream SS(S);
-  SS << Result;
+  SS << std::move(Result);
   return S;
 }
 
@@ -81,7 +81,7 @@ inline std::string printArg<IntegralAP<false>>(Program &P, CodePtr &OpPC) {
 
   std::string Str;
   llvm::raw_string_ostream SS(Str);
-  SS << Result;
+  SS << std::move(Result);
   return Str;
 }
 
@@ -99,7 +99,7 @@ inline std::string printArg<IntegralAP<true>>(Program &P, CodePtr &OpPC) {
 
   std::string Str;
   llvm::raw_string_ostream SS(Str);
-  SS << Result;
+  SS << std::move(Result);
   return Str;
 }
 
@@ -109,7 +109,7 @@ template <> inline std::string printArg<FixedPoint>(Program &P, CodePtr &OpPC) {
 
   std::string Result;
   llvm::raw_string_ostream SS(Result);
-  SS << F;
+  SS << std::move(F);
   return Result;
 }
 
diff --git a/clang/lib/AST/ByteCode/FunctionPointer.cpp b/clang/lib/AST/ByteCode/FunctionPointer.cpp
index 4ab7af170efe4..2488626697284 100644
--- a/clang/lib/AST/ByteCode/FunctionPointer.cpp
+++ b/clang/lib/AST/ByteCode/FunctionPointer.cpp
@@ -27,8 +27,6 @@ void FunctionPointer::print(llvm::raw_ostream &OS) const {
   OS << "FnPtr(";
   if (Func)
     OS << Func->getName();
-  else if (Func)
-    OS << reinterpret_cast<uintptr_t>(Func);
   else
     OS << "nullptr";
   OS << ")";
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index dcc4587751974..85a07abb5ed01 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -552,11 +552,6 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    // Result.atIndex(0).deref<Floating>() = Floating(ResR);
-    // Result.atIndex(0).initialize();
-    // Result.atIndex(1).deref<Floating>() = Floating(ResI);
-    // Result.atIndex(1).initialize();
-
     Floating RA = S.allocFloat(A.getSemantics());
     RA.copy(ResR);
     Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
@@ -565,6 +560,7 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     Floating RI = S.allocFloat(A.getSemantics());
     RI.copy(ResI);
     Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
+    Result.atIndex(1).initialize();
 
     Result.initialize();
   } else {
@@ -1143,30 +1139,12 @@ inline bool CmpHelperEQ<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   }
 
   if (Pointer::hasSameBase(LHS, RHS)) {
-    if (LHS.inUnion() && RHS.inUnion()) {
-      // If the pointers point into a union, things are a little more
-      // complicated since the offset we save in interp::Pointer can't be used
-      // to compare the pointers directly.
-      size_t A = LHS.computeOffsetForComparison();
-      size_t B = RHS.computeOffsetForComparison();
-      S.Stk.push<BoolT>(BoolT::from(Fn(Compare(A, B))));
-      return true;
-    }
-
-    unsigned VL = LHS.getByteOffset();
-    unsigned VR = RHS.getByteOffset();
-    // In our Pointer class, a pointer to an array and a pointer to the first
-    // element in the same array are NOT equal. They have the same Base value,
-    // but a different Offset. This is a pretty rare case, so we fix this here
-    // by comparing pointers to the first elements.
-    if (!LHS.isZero() && LHS.isArrayRoot())
-      VL = LHS.atIndex(0).getByteOffset();
-    if (!RHS.isZero() && RHS.isArrayRoot())
-      VR = RHS.atIndex(0).getByteOffset();
-
-    S.Stk.push<BoolT>(BoolT::from(Fn(Compare(VL, VR))));
+    size_t A = LHS.computeOffsetForComparison();
+    size_t B = RHS.computeOffsetForComparison();
+    S.Stk.push<BoolT>(BoolT::from(Fn(Compare(A, B))));
     return true;
   }
+
   // Otherwise we need to do a bunch of extra checks before returning Unordered.
   if (LHS.isOnePastEnd() && !RHS.isOnePastEnd() && !RHS.isZero() &&
       RHS.getOffset() == 0) {
@@ -1332,20 +1310,6 @@ bool GE(InterpState &S, CodePtr OpPC) {
   });
 }
 
-//===----------------------------------------------------------------------===//
-// InRange
-//===----------------------------------------------------------------------===//
-
-template <PrimType Name, class T = typename PrimConv<Name>::T>
-bool InRange(InterpState &S, CodePtr OpPC) {
-  const T RHS = S.Stk.pop<T>();
-  const T LHS = S.Stk.pop<T>();
-  const T Value = S.Stk.pop<T>();
-
-  S.Stk.push<bool>(LHS <= Value && Value <= RHS);
-  return true;
-}
-
 //===----------------------------------------------------------------------===//
 // Dup, Pop, Test
 //===----------------------------------------------------------------------===//
@@ -1966,8 +1930,10 @@ bool StoreBitField(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckStore(S, OpPC, Ptr))
     return false;
-  if (Ptr.canBeInitialized())
+  if (Ptr.canBeInitialized()) {
     Ptr.initialize();
+    Ptr.activate();
+  }
   if (const auto *FD = Ptr.getField())
     Ptr.deref<T>() = Value.truncate(FD->getBitWidthValue());
   else
@@ -1981,8 +1947,10 @@ bool StoreBitFieldPop(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
   if (!CheckStore(S, OpPC, Ptr))
     return false;
-  if (Ptr.canBeInitialized())
+  if (Ptr.canBeInitialized()) {
     Ptr.initialize();
+    Ptr.activate();
+  }
   if (const auto *FD = Ptr.getField())
     Ptr.deref<T>() = Value.truncate(FD->getBitWidthValue());
   else
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 93002172949a5..1fe6d146b511d 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -555,8 +555,8 @@ static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
   APSInt FPClassArg = popToAPSInt(S.Stk, FPClassArgT);
   const Floating &F = S.Stk.pop<Floating>();
 
-  int32_t Result =
-      static_cast<int32_t>((F.classify() & FPClassArg).getZExtValue());
+  int32_t Result = static_cast<int32_t>(
+      (F.classify() & std::move(FPClassArg)).getZExtValue());
   pushInteger(S, Result, Call->getType());
 
   return true;
@@ -856,12 +856,14 @@ static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
 
     if (!APSInt::isSameValue(Temp, Result))
       Overflow = true;
-    Result = Temp;
+    Result = std::move(Temp);
   }
 
   // Write Result to ResultPtr and put Overflow on the stack.
   assignInteger(S, ResultPtr, ResultT, Result);
-  ResultPtr.initialize();
+  if (ResultPtr.canBeInitialized())
+    ResultPtr.initialize();
+
   assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
   S.Stk.push<Boolean>(Overflow);
   return true;
@@ -1135,17 +1137,17 @@ static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
 
   if (isIntegralType(FirstArgT)) {
     const APSInt &Src = popToAPSInt(S.Stk, FirstArgT);
-    APSInt Align = Alignment.extOrTrunc(Src.getBitWidth());
+    APInt AlignMinusOne = Alignment.extOrTrunc(Src.getBitWidth()) - 1;
     if (BuiltinOp == Builtin::BI__builtin_align_up) {
       APSInt AlignedVal =
-          APSInt((Src + (Align - 1)) & ~(Align - 1), Src.isUnsigned());
+          APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
       pushInteger(S, AlignedVal, Call->getType());
     } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
-      APSInt AlignedVal = APSInt(Src & ~(Align - 1), Src.isUnsigned());
+      APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
       pushInteger(S, AlignedVal, Call->getType());
     } else {
       assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
-      S.Stk.push<Boolean>((Src & (Align - 1)) == 0);
+      S.Stk.push<Boolean>((Src & AlignMinusOne) == 0);
     }
     return true;
   }
@@ -1425,7 +1427,7 @@ static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
 
   QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
   PrimType CarryOutT = *S.getContext().classify(CarryOutType);
-  assignInteger(S, CarryOutPtr, CarryOutT, APSInt(Result, true));
+  assignInteger(S, CarryOutPtr, CarryOutT, APSInt(std::move(Result), true));
 
   pushInteger(S, CarryOut, Call->getType());
 
@@ -1531,34 +1533,21 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
   std::optional<PrimType> ElemT = S.getContext().classify(ElemType);
   DynamicAllocator &Allocator = S.getAllocator();
   if (ElemT) {
-    if (IsArray) {
-      Block *B = Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
-                                    S.Ctx.getEvalID(),
-                                    DynamicAllocator::Form::Operator);
-      assert(B);
-      S.Stk.push<Pointer>(Pointer(B).atIndex(0));
-      return true;
-    }
-
-    const Descriptor *Desc = S.P.createDescriptor(
-        NewCall, *ElemT, ElemType.getTypePtr(), Descriptor::InlineDescMD,
-        /*IsConst=*/false, /*IsTemporary=*/false,
-        /*IsMutable=*/false);
-    Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
-                                  DynamicAllocator::Form::Operator);
+    Block *B =
+        Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
+                           S.Ctx.getEvalID(), DynamicAllocator::Form::Operator);
     assert(B);
-
-    S.Stk.push<Pointer>(B);
+    S.Stk.push<Pointer>(Pointer(B).atIndex(0));
     return true;
   }
 
   assert(!ElemT);
-  // Structs etc.
-  const Descriptor *Desc =
-      S.P.createDescriptor(NewCall, ElemType.getTypePtr(),
-                           IsArray ? std::nullopt : Descriptor::InlineDescMD);
 
+  // Composite arrays
   if (IsArray) {
+    const Descriptor *Desc =
+        S.P.createDescriptor(NewCall, ElemType.getTypePtr(),
+                             IsArray ? std::nullopt : Descriptor::InlineDescMD);
     Block *B =
         Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),
                            DynamicAllocator::Form::Operator);
@@ -1567,10 +1556,17 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
     return true;
   }
 
+  // Records. Still allocate them as single-element arrays.
+  QualType AllocType = S.getASTContext().getConstantArrayType(
+      ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);
+
+  const Descriptor *Desc =
+      S.P.createDescriptor(NewCall, AllocType.getTypePtr(),
+                           IsArray ? std::nullopt : Descriptor::InlineDescMD);
   Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
                                 DynamicAllocator::Form::Operator);
   assert(B);
-  S.Stk.push<Pointer>(B);
+  S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
   return true;
 }
 
@@ -2212,7 +2208,7 @@ static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
   if (Ptr.isOnePastEnd())
     return Error(1);
 
-  bool Result = true;
+  bool Result = Ptr.getLifetime() != Lifetime::Ended;
   if (!Ptr.isActive()) {
     Result = false;
   } else {
@@ -2220,7 +2216,19 @@ static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
       return false;
     if (!CheckMutable(S, OpPC, Ptr))
       return false;
+    if (!CheckDummy(S, OpPC, Ptr, AK_Read))
+      return false;
+  }
+
+  // Check if we're currently running an initializer.
+  for (InterpFrame *Frame = S.Current; Frame; Frame = Frame->Caller) {
+    if (const Function *F = Frame->getFunction();
+        F && F->isConstructor() && Frame->getThis().block() == Ptr.block()) {
+      return Error(2);
+    }
   }
+  if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)
+    return Error(2);
 
   pushInteger(S, Result, Call->getType());
   return true;
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index 2569cac018b31..feac97d4b1a69 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -335,7 +335,8 @@ bool clang::interp::DoBitCast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
   BitcastBuffer Buffer(FullBitWidth);
   size_t BuffSize = FullBitWidth.roundToBytes();
-  if (!CheckBitcastType(S, OpPC, Ptr.getType(), /*IsToType=*/false))
+  QualType DataType = Ptr.getFieldDesc()->getDataType(S.getASTContext());
+  if (!CheckBitcastType(S, OpPC, DataType, /*IsToType=*/false))
     return false;
 
   bool Success = readPointerToBuffer(S.getContext(), Ptr, Buffer,
@@ -370,8 +371,8 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
   assert(FromPtr.isBlockPointer());
   assert(ToPtr.isBlockPointer());
 
-  QualType FromType = FromPtr.getType();
-  QualType ToType = ToPtr.getType();
+  QualType FromType = FromPtr.getFieldDesc()->getDataType(S.getASTContext());
+  QualType ToType = ToPtr.getFieldDesc()->getDataType(S.getASTContext());
 
   if (!CheckBitcastType(S, OpPC, ToType, /*IsToType=*/true))
     return false;
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index a849ac71a7263..a5a4bd25fe712 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -8,7 +8,6 @@
 
 #include "InterpFrame.h"
 #include "Boolean.h"
-#include "Floating.h"
 #include "Function.h"
 #include "InterpStack.h"
 #include "InterpState.h"
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index f0b0384f32ac8..6769e51a8ad00 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -176,19 +176,8 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   if (const auto *VD = Desc->asValueDecl())
     Base = VD;
   else if (const auto *E = Desc->asExpr()) {
-    // Create a DynamicAlloc base of the right type.
-    if (const auto *NewExpr = dyn_cast<CXXNewExpr>(E)) {
-      QualType AllocatedType;
-      if (NewExpr->isArray()) {
-        assert(Desc->isArray());
-        APInt ArraySize(64, static_cast<uint64_t>(Desc->getNumElems()),
-                        /*IsSigned=*/false);
-        AllocatedType =
-            ASTCtx.getConstantArrayType(NewExpr->getAllocatedType(), ArraySize,
-                                        nullptr, ArraySizeModifier::Normal, 0);
-      } else {
-        AllocatedType = NewExpr->getAllocatedType();
-      }
+    if (block()->isDynamic()) {
+      QualType AllocatedType = getDeclPtr().getFieldDesc()->getDataType(ASTCtx);
       // FIXME: Suboptimal counting of dynamic allocations. Move this to Context
       // or InterpState?
       static int ReportedDynamicAllocs = 0;
@@ -218,7 +207,7 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
 
   bool UsePath = true;
   if (const ValueDecl *VD = getDeclDesc()->asValueDecl();
-      VD && VD->getType()->isLValueReferenceType())
+      VD && VD->getType()->isReferenceType())
     UsePath = false;
 
   // Build the path into the object.
@@ -349,16 +338,28 @@ void Pointer::print(llvm::raw_ostream &OS) const {
   }
 }
 
-/// Compute an integer that can be used to compare this pointer to
-/// another one.
 size_t Pointer::computeOffsetForComparison() const {
+  if (isIntegralPointer())
+    return asIntPointer().Value + Offset;
+  if (isTypeidPointer())
+    return reinterpret_cast<uintptr_t>(asTypeidPointer().TypePtr) + Offset;
+
   if (!isBlockPointer())
     return Offset;
 
   size_t Result = 0;
   Pointer P = *this;
-  while (!P.isRoot()) {
-    if (P.isArrayRoot()) {
+  while (true) {
+
+    if (P.isVirtualBaseClass()) {
+      Result += getInlineDesc()->Offset;
+      P = P.getBase();
+      continue;
+    }
+
+    if (P.isBaseClass()) {
+      if (P.getRecord()->getNumVirtualBases() > 0)
+        Result += P.getInlineDesc()->Offset;
       P = P.getBase();
       continue;
     }
@@ -369,14 +370,26 @@ size_t Pointer::computeOffsetForComparison() const {
       continue;
     }
 
+    if (P.isRoot()) {
+      if (P.isOnePastEnd())
+        ++Result;
+      break;
+    }
+
     if (const Record *R = P.getBase().getRecord(); R && R->isUnion()) {
       // Direct child of a union - all have offset 0.
       P = P.getBase();
       continue;
     }
 
+    // Fields, etc.
     Result += P.getInlineDesc()->Offset;
+    if (P.isOnePastEnd())
+      ++Result;
+
     P = P.getBase();
+    if (P.isRoot())
+      break;
   }
 
   return Result;
@@ -492,33 +505,36 @@ void Pointer::activate() const {
   auto activate = [](Pointer &P) -> void {
     P.getInlineDesc()->IsActive = true;
   };
-  auto deactivate = [](Pointer &P) -> void {
-    P.getInlineDesc()->IsActive = false;
-  };
 
-  // Unions might be nested etc., so find the topmost Pointer that's
-  // not in a union anymore.
-  Pointer UnionPtr = getBase();
-  while (!UnionPtr.isRoot() && UnionPtr.inUnion())
-    UnionPtr = UnionPtr.getBase();
-
-  assert(UnionPtr.getFieldDesc()->isUnion());
+  std::function<void(Pointer &)> deactivate;
+  deactivate = [&deactivate](Pointer &P) -> void {
+    P.getInlineDesc()->IsActive = false;
 
-  const Record *UnionRecord = UnionPtr.getRecord();
-  for (const Record::Field &F : UnionRecord->fields()) {
-    Pointer FieldPtr = UnionPtr.atField(F.Offset);
-    if (FieldPtr == *this) {
-    } else {
-      deactivate(FieldPtr);
-      // FIXME: Recurse.
+    if (const Record *R = P.getRecord()) {
+      for (const Record::Field &F : R->fields()) {
+        Pointer FieldPtr = P.atField(F.Offset);
+        if (FieldPtr.getInlineDesc()->IsActive)
+          deactivate(FieldPtr);
+      }
+      // FIXME: Bases?
     }
-  }
+  };
 
   Pointer B = *this;
-  while (B != UnionPtr) {
+  while (!B.isRoot() && B.inUnion()) {
     activate(B);
-    // FIXME: Need to de-activate other fields of parent records.
+
+    // When walking up the pointer chain, deactivate
+    // all union child pointers that aren't on our path.
+    Pointer Cur = B;
     B = B.getBase();
+    if (const Record *BR = B.getRecord(); BR && BR->isUnion()) {
+      for (const Record::Field &F : BR->fields()) {
+        Pointer FieldPtr = B.atField(F.Offset);
+        if (FieldPtr != Cur)
+          deactivate(FieldPtr);
+      }
+    }
   }
 }
 
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 0234ab02ab8f6..d525f84fd6605 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -761,6 +761,9 @@ class Pointer {
   /// Prints the pointer.
   void print(llvm::raw_ostream &OS) const;
 
+  /// Compute an integer that can be used to compare this pointer to
+  /// another one. This is usually NOT the same as the pointer offset
+  /// regarding the AST record layout.
   size_t computeOffsetForComparison() const;
 
 private:
diff --git a/clang/lib/AST/Comment.cpp b/clang/lib/AST/Comment.cpp
index d022c5c845ae6..cd73d2709dc65 100644
--- a/clang/lib/AST/Comment.cpp
+++ b/clang/lib/AST/Comment.cpp
@@ -291,6 +291,13 @@ void DeclInfo::fill() {
     TemplateParameters = CTPSD->getTemplateParameters();
     break;
   }
+  case Decl::VarTemplatePartialSpecialization: {
+    const auto *VTPSD = cast<VarTemplatePartialSpecializationDecl>(CommentDecl);
+    Kind = VariableKind;
+    TemplateKind = TemplatePartialSpecialization;
+    TemplateParameters = VTPSD->getTemplateParameters();
+    break;
+  }
   case Decl::ClassTemplateSpecialization:
     Kind = ClassKind;
     TemplateKind = TemplateSpecialization;
diff --git a/clang/lib/AST/CommentSema.cpp b/clang/lib/AST/CommentSema.cpp
index 36a9ebe14d7da..95f6bf79d118c 100644
--- a/clang/lib/AST/CommentSema.cpp
+++ b/clang/lib/AST/CommentSema.cpp
@@ -171,50 +171,49 @@ void Sema::checkContainerDecl(const BlockCommandComment *Comment) {
   const CommandInfo *Info = Traits.getCommandInfo(Comment->getCommandID());
   if (!Info->IsRecordLikeDetailCommand || isRecordLikeDecl())
     return;
-  unsigned DiagSelect;
+  std::optional<unsigned> DiagSelect;
   switch (Comment->getCommandID()) {
     case CommandTraits::KCI_classdesign:
-      DiagSelect = 1;
+      DiagSelect = diag::DocCommandKind::ClassDesign;
       break;
     case CommandTraits::KCI_coclass:
-      DiagSelect = 2;
+      DiagSelect = diag::DocCommandKind::CoClass;
       break;
     case CommandTraits::KCI_dependency:
-      DiagSelect = 3;
+      DiagSelect = diag::DocCommandKind::Dependency;
       break;
     case CommandTraits::KCI_helper:
-      DiagSelect = 4;
+      DiagSelect = diag::DocCommandKind::Helper;
       break;
     case CommandTraits::KCI_helperclass:
-      DiagSelect = 5;
+      DiagSelect = diag::DocCommandKind::HelperClass;
       break;
     case CommandTraits::KCI_helps:
-      DiagSelect = 6;
+      DiagSelect = diag::DocCommandKind::Helps;
       break;
     case CommandTraits::KCI_instancesize:
-      DiagSelect = 7;
+      DiagSelect = diag::DocCommandKind::InstanceSize;
       break;
     case CommandTraits::KCI_ownership:
-      DiagSelect = 8;
+      DiagSelect = diag::DocCommandKind::Ownership;
       break;
     case CommandTraits::KCI_performance:
-      DiagSelect = 9;
+      DiagSelect = diag::DocCommandKind::Performance;
       break;
     case CommandTraits::KCI_security:
-      DiagSelect = 10;
+      DiagSelect = diag::DocCommandKind::Security;
       break;
     case CommandTraits::KCI_superclass:
-      DiagSelect = 11;
+      DiagSelect = diag::DocCommandKind::Superclass;
       break;
     default:
-      DiagSelect = 0;
+      DiagSelect = std::nullopt;
       break;
   }
   if (DiagSelect)
     Diag(Comment->getLocation(), diag::warn_doc_container_decl_mismatch)
-    << Comment->getCommandMarker()
-    << (DiagSelect-1)
-    << Comment->getSourceRange();
+        << Comment->getCommandMarker() << (*DiagSelect)
+        << Comment->getSourceRange();
 }
 
 /// Turn a string into the corresponding PassDirection or -1 if it's not
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index bc88045c17f73..8855d0107daca 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -66,7 +66,6 @@
 #include <cassert>
 #include <cstddef>
 #include <cstring>
-#include <memory>
 #include <optional>
 #include <string>
 #include <tuple>
@@ -2445,24 +2444,17 @@ bool VarDecl::hasInitWithSideEffects() const {
   if (!hasInit())
     return false;
 
-  // Check if we can get the initializer without deserializing
-  const Expr *E = nullptr;
-  if (auto *S = dyn_cast<Stmt *>(Init)) {
-    E = cast<Expr>(S);
-  } else {
-    E = cast_or_null<Expr>(getEvaluatedStmt()->Value.getWithoutDeserializing());
-  }
-
-  if (E)
-    return E->HasSideEffects(getASTContext()) &&
-           // We can get a value-dependent initializer during error recovery.
-           (E->isValueDependent() || !evaluateValue());
-
-  assert(getEvaluatedStmt()->Value.isOffset());
-  // ASTReader tracks this without having to deserialize the initializer
-  if (auto Source = getASTContext().getExternalSource())
-    return Source->hasInitializerWithSideEffects(this);
-  return false;
+  EvaluatedStmt *ES = ensureEvaluatedStmt();
+  if (!ES->CheckedForSideEffects) {
+    const Expr *E = getInit();
+    ES->HasSideEffects =
+        E->HasSideEffects(getASTContext()) &&
+        // We can get a value-dependent initializer during error recovery.
+        (E->isValueDependent() || getType()->isDependentType() ||
+         !evaluateValue());
+    ES->CheckedForSideEffects = true;
+  }
+  return ES->HasSideEffects;
 }
 
 bool VarDecl::isOutOfLine() const {
diff --git a/clang/lib/AST/DeclObjC.cpp b/clang/lib/AST/DeclObjC.cpp
index 5cf0e9a7b2599..a66eb72981084 100644
--- a/clang/lib/AST/DeclObjC.cpp
+++ b/clang/lib/AST/DeclObjC.cpp
@@ -27,7 +27,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <cstring>
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 149b274f36b63..36fd5ee271e03 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -2020,7 +2020,8 @@ CXXBaseSpecifier **CastExpr::path_buffer() {
 #define ABSTRACT_STMT(x)
 #define CASTEXPR(Type, Base)                                                   \
   case Stmt::Type##Class:                                                      \
-    return static_cast<Type *>(this)->getTrailingObjects<CXXBaseSpecifier *>();
+    return static_cast<Type *>(this)                                           \
+        ->getTrailingObjectsNonStrict<CXXBaseSpecifier *>();
 #define STMT(Type, Base)
 #include "clang/AST/StmtNodes.inc"
   default:
@@ -5445,3 +5446,17 @@ ConvertVectorExpr *ConvertVectorExpr::Create(
   return new (Mem) ConvertVectorExpr(SrcExpr, TI, DstType, VK, OK, BuiltinLoc,
                                      RParenLoc, FPFeatures);
 }
+
+APValue &CompoundLiteralExpr::getOrCreateStaticValue(ASTContext &Ctx) const {
+  assert(hasStaticStorage());
+  if (!StaticValue) {
+    StaticValue = new (Ctx) APValue;
+    Ctx.addDestruction(StaticValue);
+  }
+  return *StaticValue;
+}
+
+APValue &CompoundLiteralExpr::getStaticValue() const {
+  assert(StaticValue);
+  return *StaticValue;
+}
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 764e20a54dc89..5833a6405125d 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -600,7 +600,7 @@ CXXOperatorCallExpr::CXXOperatorCallExpr(OverloadedOperatorKind OpKind,
   assert(
       (CXXOperatorCallExprBits.OperatorKind == static_cast<unsigned>(OpKind)) &&
       "OperatorKind overflow!");
-  Range = getSourceRangeImpl();
+  BeginLoc = getSourceRangeImpl().getBegin();
 }
 
 CXXOperatorCallExpr::CXXOperatorCallExpr(unsigned NumArgs, bool HasFPFeatures,
diff --git a/clang/lib/AST/ExprConcepts.cpp b/clang/lib/AST/ExprConcepts.cpp
index a2cf431a312af..ac0e566fe6e72 100644
--- a/clang/lib/AST/ExprConcepts.cpp
+++ b/clang/lib/AST/ExprConcepts.cpp
@@ -23,7 +23,6 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
-#include <algorithm>
 
 using namespace clang;
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index bf9208763b1ab..60c658a8d8f99 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -48,6 +48,7 @@
 #include "clang/AST/OptionalDiagnostic.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtVisitor.h"
+#include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticSema.h"
@@ -202,7 +203,9 @@ namespace {
     assert(!isBaseAnAllocSizeCall(Base) &&
            "Unsized arrays shouldn't appear here");
     unsigned MostDerivedLength = 0;
-    Type = getType(Base);
+    // The type of Base is a reference type if the base is a constexpr-unknown
+    // variable. In that case, look through the reference type.
+    Type = getType(Base).getNonReferenceType();
 
     for (unsigned I = 0, N = Path.size(); I != N; ++I) {
       if (Type->isArrayType()) {
@@ -289,7 +292,7 @@ namespace {
         : Invalid(false), IsOnePastTheEnd(false),
           FirstEntryIsAnUnsizedArray(false), MostDerivedIsArrayElement(false),
           MostDerivedPathLength(0), MostDerivedArraySize(0),
-          MostDerivedType(T) {}
+          MostDerivedType(T.isNull() ? QualType() : T.getNonReferenceType()) {}
 
     SubobjectDesignator(ASTContext &Ctx, const APValue &V)
         : Invalid(!V.isLValue() || !V.hasLValuePath()), IsOnePastTheEnd(false),
@@ -571,7 +574,6 @@ namespace {
     typedef std::map<MapKeyTy, APValue> MapTy;
     /// Temporaries - Temporary lvalues materialized within this stack frame.
     MapTy Temporaries;
-    MapTy ConstexprUnknownAPValues;
 
     /// CallRange - The source range of the call expression for this call.
     SourceRange CallRange;
@@ -646,9 +648,6 @@ namespace {
     APValue &createTemporary(const KeyT *Key, QualType T,
                              ScopeKind Scope, LValue &LV);
 
-    APValue &createConstexprUnknownAPValues(const VarDecl *Key,
-                                            APValue::LValueBase Base);
-
     /// Allocate storage for a parameter of a function call made in this frame.
     APValue &createParam(CallRef Args, const ParmVarDecl *PVD, LValue &LV);
 
@@ -1756,7 +1755,8 @@ namespace {
         return;
       }
       if (checkSubobject(Info, E, CSK_ArrayToPointer)) {
-        assert(getType(Base)->isPointerType() || getType(Base)->isArrayType());
+        assert(getType(Base).getNonReferenceType()->isPointerType() ||
+               getType(Base).getNonReferenceType()->isArrayType());
         Designator.FirstEntryIsAnUnsizedArray = true;
         Designator.addUnsizedArrayUnchecked(ElemTy);
       }
@@ -1955,15 +1955,6 @@ APValue &CallStackFrame::createTemporary(const KeyT *Key, QualType T,
   return createLocal(Base, Key, T, Scope);
 }
 
-APValue &
-CallStackFrame::createConstexprUnknownAPValues(const VarDecl *Key,
-                                               APValue::LValueBase Base) {
-  APValue &Result = ConstexprUnknownAPValues[MapKeyTy(Key, Base.getVersion())];
-  Result = APValue(Base, CharUnits::Zero(), APValue::ConstexprUnknown{});
-
-  return Result;
-}
-
 /// Allocate storage for a parameter of a function call made in this frame.
 APValue &CallStackFrame::createParam(CallRef Args, const ParmVarDecl *PVD,
                                      LValue &LV) {
@@ -3493,7 +3484,7 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
   APValue::LValueBase Base(VD, Frame ? Frame->Index : 0, Version);
 
   auto CheckUninitReference = [&](bool IsLocalVariable) {
-    if (!Result->hasValue() && VD->getType()->isReferenceType()) {
+    if (!Result || (!Result->hasValue() && VD->getType()->isReferenceType())) {
       // C++23 [expr.const]p8
       // ... For such an object that is not usable in constant expressions, the
       // dynamic type of the object is constexpr-unknown. For such a reference
@@ -3509,7 +3500,7 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
           Info.FFDiag(E, diag::note_constexpr_use_uninit_reference);
         return false;
       }
-      Result = &Info.CurrentCall->createConstexprUnknownAPValues(VD, Base);
+      Result = nullptr;
     }
     return true;
   };
@@ -3552,7 +3543,7 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
   // ... its lifetime began within the evaluation of E;
   if (isa<ParmVarDecl>(VD)) {
     if (AllowConstexprUnknown) {
-      Result = &Info.CurrentCall->createConstexprUnknownAPValues(VD, Base);
+      Result = nullptr;
       return true;
     }
 
@@ -3659,12 +3650,8 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
 
   Result = VD->getEvaluatedValue();
 
-  if (!Result) {
-    if (AllowConstexprUnknown)
-      Result = &Info.CurrentCall->createConstexprUnknownAPValues(VD, Base);
-    else
-      return false;
-  }
+  if (!Result && !AllowConstexprUnknown)
+    return false;
 
   return CheckUninitReference(/*IsLocalVariable=*/false);
 }
@@ -3947,11 +3934,6 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
   const FieldDecl *LastField = nullptr;
   const FieldDecl *VolatileField = nullptr;
 
-  // C++23 [expr.const]p8 If we have an unknown reference or pointers and it
-  // does not have a value then bail out.
-  if (O->allowConstexprUnknown() && !O->hasValue())
-    return false;
-
   // Walk the designator's path to find the subobject.
   for (unsigned I = 0, N = Sub.Entries.size(); /**/; ++I) {
     // Reading an indeterminate value is undefined, but assigning over one is OK.
@@ -4491,6 +4473,15 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
 
     if (!evaluateVarDeclInit(Info, E, VD, Frame, LVal.getLValueVersion(), BaseVal))
       return CompleteObject();
+    // If evaluateVarDeclInit sees a constexpr-unknown variable, it returns
+    // a null BaseVal. Any constexpr-unknown variable seen here is an error:
+    // we can't access a constexpr-unknown object.
+    if (!BaseVal) {
+      Info.FFDiag(E, diag::note_constexpr_access_unknown_variable, 1)
+          << AK << VD;
+      Info.Note(VD->getLocation(), diag::note_declared_at);
+      return CompleteObject();
+    }
   } else if (DynamicAllocLValue DA = LVal.Base.dyn_cast<DynamicAllocLValue>()) {
     std::optional<DynAlloc *> Alloc = Info.lookupDynamicAlloc(DA);
     if (!Alloc) {
@@ -4544,6 +4535,30 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
 
         BaseVal = MTE->getOrCreateValue(false);
         assert(BaseVal && "got reference to unevaluated temporary");
+      } else if (const CompoundLiteralExpr *CLE =
+                     dyn_cast_or_null<CompoundLiteralExpr>(Base)) {
+        // According to GCC info page:
+        //
+        // 6.28 Compound Literals
+        //
+        // As an optimization, G++ sometimes gives array compound literals
+        // longer lifetimes: when the array either appears outside a function or
+        // has a const-qualified type. If foo and its initializer had elements
+        // of type char *const rather than char *, or if foo were a global
+        // variable, the array would have static storage duration. But it is
+        // probably safest just to avoid the use of array compound literals in
+        // C++ code.
+        //
+        // Obey that rule by checking constness for converted array types.
+        if (QualType CLETy = CLE->getType(); CLETy->isArrayType() &&
+                                             !LValType->isArrayType() &&
+                                             !CLETy.isConstant(Info.Ctx)) {
+          Info.FFDiag(E);
+          Info.Note(CLE->getExprLoc(), diag::note_declared_at);
+          return CompleteObject();
+        }
+
+        BaseVal = &CLE->getStaticValue();
       } else {
         if (!IsAccess)
           return CompleteObject(LVal.getLValueBase(), nullptr, BaseType);
@@ -4609,44 +4624,7 @@ handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv, QualType Type,
       WantObjectRepresentation ? AK_ReadObjectRepresentation : AK_Read;
 
   if (Base && !LVal.getLValueCallIndex() && !Type.isVolatileQualified()) {
-    if (const CompoundLiteralExpr *CLE = dyn_cast<CompoundLiteralExpr>(Base)) {
-      // In C99, a CompoundLiteralExpr is an lvalue, and we defer evaluating the
-      // initializer until now for such expressions. Such an expression can't be
-      // an ICE in C, so this only matters for fold.
-      if (Type.isVolatileQualified()) {
-        Info.FFDiag(Conv);
-        return false;
-      }
-
-      APValue Lit;
-      if (!Evaluate(Lit, Info, CLE->getInitializer()))
-        return false;
-
-      // According to GCC info page:
-      //
-      // 6.28 Compound Literals
-      //
-      // As an optimization, G++ sometimes gives array compound literals longer
-      // lifetimes: when the array either appears outside a function or has a
-      // const-qualified type. If foo and its initializer had elements of type
-      // char *const rather than char *, or if foo were a global variable, the
-      // array would have static storage duration. But it is probably safest
-      // just to avoid the use of array compound literals in C++ code.
-      //
-      // Obey that rule by checking constness for converted array types.
-
-      QualType CLETy = CLE->getType();
-      if (CLETy->isArrayType() && !Type->isArrayType()) {
-        if (!CLETy.isConstant(Info.Ctx)) {
-          Info.FFDiag(Conv);
-          Info.Note(CLE->getExprLoc(), diag::note_declared_at);
-          return false;
-        }
-      }
-
-      CompleteObject LitObj(LVal.Base, &Lit, Base->getType());
-      return extractSubobject(Info, Conv, LitObj, LVal.Designator, RVal, AK);
-    } else if (isa<StringLiteral>(Base) || isa<PredefinedExpr>(Base)) {
+    if (isa<StringLiteral>(Base) || isa<PredefinedExpr>(Base)) {
       // Special-case character extraction so we don't have to construct an
       // APValue for the whole string.
       assert(LVal.Designator.Entries.size() <= 1 &&
@@ -6057,15 +6035,6 @@ struct CheckDynamicTypeHandler {
 /// dynamic type.
 static bool checkDynamicType(EvalInfo &Info, const Expr *E, const LValue &This,
                              AccessKinds AK, bool Polymorphic) {
-  // We are not allowed to invoke a virtual function whose dynamic type
-  // is constexpr-unknown, so stop early and let this fail later on if we
-  // attempt to do so.
-  // C++23 [expr.const]p5.6
-  // an invocation of a virtual function ([class.virtual]) for an object whose
-  // dynamic type is constexpr-unknown;
-  if (This.allowConstexprUnknown())
-    return true;
-
   if (This.Designator.Invalid)
     return false;
 
@@ -6139,9 +6108,7 @@ static std::optional<DynamicType> ComputeDynamicType(EvalInfo &Info,
   // meaningful dynamic type. (We consider objects of non-class type to have no
   // dynamic type.)
   if (!checkDynamicType(Info, E, This, AK,
-                        (AK == AK_TypeId
-                             ? (E->getType()->isReferenceType() ? true : false)
-                             : true)))
+                        AK != AK_TypeId || This.AllowConstexprUnknown))
     return std::nullopt;
 
   if (This.Designator.Invalid)
@@ -9063,6 +9030,12 @@ bool LValueExprEvaluator::VisitVarDecl(const Expr *E, const VarDecl *VD) {
   if (!evaluateVarDeclInit(Info, E, VD, Frame, Version, V))
     return false;
 
+  if (!V) {
+    Result.set(VD);
+    Result.AllowConstexprUnknown = true;
+    return true;
+  }
+
   return Success(*V, E);
 }
 
@@ -9159,9 +9132,29 @@ bool
 LValueExprEvaluator::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
   assert((!Info.getLangOpts().CPlusPlus || E->isFileScope()) &&
          "lvalue compound literal in c++?");
-  // Defer visiting the literal until the lvalue-to-rvalue conversion. We can
-  // only see this when folding in C, so there's no standard to follow here.
-  return Success(E);
+  APValue *Lit;
+  // If CompountLiteral has static storage, its value can be used outside
+  // this expression. So evaluate it once and store it in ASTContext.
+  if (E->hasStaticStorage()) {
+    Lit = &E->getOrCreateStaticValue(Info.Ctx);
+    Result.set(E);
+    // Reset any previously evaluated state, otherwise evaluation below might
+    // fail.
+    // FIXME: Should we just re-use the previously evaluated value instead?
+    *Lit = APValue();
+  } else {
+    assert(!Info.getLangOpts().CPlusPlus);
+    Lit = &Info.CurrentCall->createTemporary(E, E->getInitializer()->getType(),
+                                             ScopeKind::Block, Result);
+  }
+  // FIXME: Evaluating in place isn't always right. We should figure out how to
+  // use appropriate evaluation context here, see
+  // clang/test/AST/static-compound-literals-reeval.cpp for a failure.
+  if (!EvaluateInPlace(*Lit, Info, Result, E->getInitializer())) {
+    *Lit = APValue();
+    return false;
+  }
+  return true;
 }
 
 bool LValueExprEvaluator::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
diff --git a/clang/lib/AST/ExprObjC.cpp b/clang/lib/AST/ExprObjC.cpp
index 3df9c404b5ca8..50d3a4474c4d7 100644
--- a/clang/lib/AST/ExprObjC.cpp
+++ b/clang/lib/AST/ExprObjC.cpp
@@ -17,7 +17,6 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "llvm/Support/ErrorHandling.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 84936b72bb4fa..8a1d4e8313e28 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -4273,7 +4273,8 @@ void CXXNameMangler::mangleRISCVFixedRVVVectorType(const VectorType *T) {
 
   // Apend the LMUL suffix.
   auto VScale = getASTContext().getTargetInfo().getVScaleRange(
-      getASTContext().getLangOpts(), false);
+      getASTContext().getLangOpts(),
+      TargetInfo::ArmStreamingKind::NotStreaming);
   unsigned VLen = VScale->first * llvm::RISCV::RVVBitsPerBlock;
 
   if (T->getVectorKind() == VectorKind::RVVFixedLengthData) {
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index f8446dfbc6859..7fdfcfa3014f3 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -30,19 +30,21 @@ void ODRHash::AddIdentifierInfo(const IdentifierInfo *II) {
   ID.AddString(II->getName());
 }
 
-void ODRHash::AddDeclarationName(DeclarationName Name, bool TreatAsDecl) {
+void ODRHash::AddDeclarationNameInfo(DeclarationNameInfo NameInfo,
+                                     bool TreatAsDecl) {
   if (TreatAsDecl)
     // Matches the NamedDecl check in AddDecl
     AddBoolean(true);
 
-  AddDeclarationNameImpl(Name);
+  AddDeclarationNameInfoImpl(NameInfo);
 
   if (TreatAsDecl)
     // Matches the ClassTemplateSpecializationDecl check in AddDecl
     AddBoolean(false);
 }
 
-void ODRHash::AddDeclarationNameImpl(DeclarationName Name) {
+void ODRHash::AddDeclarationNameInfoImpl(DeclarationNameInfo NameInfo) {
+  DeclarationName Name = NameInfo.getName();
   // Index all DeclarationName and use index numbers to refer to them.
   auto Result = DeclNameMap.insert(std::make_pair(Name, DeclNameMap.size()));
   ID.AddInteger(Result.first->second);
@@ -85,7 +87,11 @@ void ODRHash::AddDeclarationNameImpl(DeclarationName Name) {
   }
   case DeclarationName::CXXConstructorName:
   case DeclarationName::CXXDestructorName:
-    AddQualType(Name.getCXXNameType());
+  case DeclarationName::CXXConversionFunctionName:
+    if (auto *TSI = NameInfo.getNamedTypeInfo())
+      AddQualType(TSI->getType());
+    else
+      AddQualType(Name.getCXXNameType());
     break;
   case DeclarationName::CXXOperatorName:
     ID.AddInteger(Name.getCXXOverloadedOperator());
@@ -93,9 +99,6 @@ void ODRHash::AddDeclarationNameImpl(DeclarationName Name) {
   case DeclarationName::CXXLiteralOperatorName:
     AddIdentifierInfo(Name.getCXXLiteralIdentifier());
     break;
-  case DeclarationName::CXXConversionFunctionName:
-    AddQualType(Name.getCXXNameType());
-    break;
   case DeclarationName::CXXUsingDirective:
     break;
   case DeclarationName::CXXDeductionGuideName: {
@@ -314,7 +317,10 @@ class ODRDeclVisitor : public ConstDeclVisitor<ODRDeclVisitor> {
   }
 
   void VisitNamedDecl(const NamedDecl *D) {
-    Hash.AddDeclarationName(D->getDeclName());
+    if (const auto *FD = dyn_cast<FunctionDecl>(D))
+      Hash.AddDeclarationNameInfo(FD->getNameInfo());
+    else
+      Hash.AddDeclarationName(D->getDeclName());
     Inherited::VisitNamedDecl(D);
   }
 
@@ -828,7 +834,10 @@ void ODRHash::AddDecl(const Decl *D) {
     return;
   }
 
-  AddDeclarationName(ND->getDeclName());
+  if (auto *FD = dyn_cast<FunctionDecl>(D))
+    AddDeclarationNameInfo(FD->getNameInfo());
+  else
+    AddDeclarationName(ND->getDeclName());
 
   // If this was a specialization we should take into account its template
   // arguments. This helps to reduce collisions coming when visiting template
@@ -1017,7 +1026,7 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
   }
 
   void VisitDecltypeType(const DecltypeType *T) {
-    AddStmt(T->getUnderlyingExpr());
+    Hash.AddStmt(T->getUnderlyingExpr());
     VisitType(T);
   }
 
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index aacc079f2521d..6d819031cbef4 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -3463,6 +3463,13 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
 
   ASTRecordLayouts[D] = NewEntry;
 
+  constexpr uint64_t MaxStructSizeInBytes = 1ULL << 60;
+  CharUnits StructSize = NewEntry->getSize();
+  if (static_cast<uint64_t>(StructSize.getQuantity()) >= MaxStructSizeInBytes) {
+    getDiagnostics().Report(D->getLocation(), diag::err_struct_too_large)
+        << D->getName() << MaxStructSizeInBytes;
+  }
+
   if (getLangOpts().DumpRecordLayouts) {
     llvm::outs() << "\n*** Dumping AST Record Layout\n";
     DumpRecordLayout(D, llvm::outs(), getLangOpts().DumpRecordLayoutsSimple);
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index f8ec3f65b7eb4..be02bdde38a3d 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -47,7 +47,6 @@
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index bb860a8f76742..9d7c2757d6ee4 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -24,7 +24,7 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TypeTraits.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 
 #include <algorithm>
 #include <utility>
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 2c1158e8f9b9a..e5a1ab2ff8906 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -4870,15 +4870,6 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
                                       ->getCanonicalTypeInternal());
   case Type::HLSLInlineSpirv:
     return LinkageInfo::external();
-    {
-      const auto *ST = cast<HLSLInlineSpirvType>(T);
-      LinkageInfo LV = LinkageInfo::external();
-      for (auto &Operand : ST->getOperands()) {
-        if (Operand.isConstant() || Operand.isType())
-          LV.merge(computeTypeLinkageInfo(Operand.getResultType()));
-      }
-      return LV;
-    }
   }
 
   llvm_unreachable("unhandled type class");
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 1bdb86ad445a4..5c45c596538f8 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include <algorithm>
@@ -652,9 +653,9 @@ static ConceptReference *createTrivialConceptReference(ASTContext &Context,
       DeclarationNameInfo(AT->getTypeConstraintConcept()->getDeclName(), Loc,
                           AT->getTypeConstraintConcept()->getDeclName());
   unsigned size = AT->getTypeConstraintArguments().size();
-  TemplateArgumentLocInfo *TALI = new TemplateArgumentLocInfo[size];
+  llvm::SmallVector<TemplateArgumentLocInfo, 8> TALI(size);
   TemplateSpecializationTypeLoc::initializeArgLocs(
-      Context, AT->getTypeConstraintArguments(), TALI, Loc);
+      Context, AT->getTypeConstraintArguments(), TALI.data(), Loc);
   TemplateArgumentListInfo TAListI;
   for (unsigned i = 0; i < size; ++i) {
     TAListI.addArgument(
@@ -666,7 +667,6 @@ static ConceptReference *createTrivialConceptReference(ASTContext &Context,
       Context, NestedNameSpecifierLoc{}, Loc, DNI, nullptr,
       AT->getTypeConstraintConcept(),
       ASTTemplateArgumentListInfo::Create(Context, TAListI));
-  delete[] TALI;
   return ConceptRef;
 }
 
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index cf7595952be27..d960d5130332b 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -898,8 +898,7 @@ class CFGBuilder {
       return;
     }
 
-    B->appendStmt(const_cast<ObjCMessageExpr *>(ME),
-                  cfg->getBumpVectorContext());
+    B->appendStmt(ME, cfg->getBumpVectorContext());
   }
 
   void appendTemporaryDtor(CFGBlock *B, CXXBindTemporaryExpr *E) {
@@ -1754,10 +1753,9 @@ std::unique_ptr<CFG> CFGBuilder::buildCFG(const Decl *D, Stmt *Statement) {
 
   // Add successors to the Indirect Goto Dispatch block (if we have one).
   if (CFGBlock *B = cfg->getIndirectGotoBlock())
-    for (LabelSetTy::iterator I = AddressTakenLabels.begin(),
-                              E = AddressTakenLabels.end(); I != E; ++I ) {
+    for (LabelDecl *LD : AddressTakenLabels) {
       // Lookup the target block.
-      LabelMapTy::iterator LI = LabelMap.find(*I);
+      LabelMapTy::iterator LI = LabelMap.find(LD);
 
       // If there is no target block that contains label, then we are looking
       // at an incomplete AST.  Handle this by not registering a successor.
diff --git a/clang/lib/Analysis/CFGStmtMap.cpp b/clang/lib/Analysis/CFGStmtMap.cpp
index c3a4581e1fb10..028e62ba89b79 100644
--- a/clang/lib/Analysis/CFGStmtMap.cpp
+++ b/clang/lib/Analysis/CFGStmtMap.cpp
@@ -83,8 +83,8 @@ CFGStmtMap *CFGStmtMap::Build(CFG *C, ParentMap *PM) {
 
   // Walk all blocks, accumulating the block-level expressions, labels,
   // and terminators.
-  for (CFG::iterator I = C->begin(), E = C->end(); I != E; ++I)
-    Accumulate(*SM, *I);
+  for (CFGBlock *BB : *C)
+    Accumulate(*SM, BB);
 
   return new CFGStmtMap(PM, SM);
 }
diff --git a/clang/lib/Analysis/CallGraph.cpp b/clang/lib/Analysis/CallGraph.cpp
index 6a2b0392ccea6..26646cd6b6502 100644
--- a/clang/lib/Analysis/CallGraph.cpp
+++ b/clang/lib/Analysis/CallGraph.cpp
@@ -223,10 +223,7 @@ void CallGraph::print(raw_ostream &OS) const {
   // We are going to print the graph in reverse post order, partially, to make
   // sure the output is deterministic.
   llvm::ReversePostOrderTraversal<const CallGraph *> RPOT(this);
-  for (llvm::ReversePostOrderTraversal<const CallGraph *>::rpo_iterator
-         I = RPOT.begin(), E = RPOT.end(); I != E; ++I) {
-    const CallGraphNode *N = *I;
-
+  for (const CallGraphNode *N : RPOT) {
     OS << "  Function: ";
     if (N == Root)
       OS << "< root >";
diff --git a/clang/lib/Analysis/LiveVariables.cpp b/clang/lib/Analysis/LiveVariables.cpp
index c1b2ff41edc0a..375fdb3695e2f 100644
--- a/clang/lib/Analysis/LiveVariables.cpp
+++ b/clang/lib/Analysis/LiveVariables.cpp
@@ -546,8 +546,8 @@ LiveVariablesImpl::runOnBlock(const CFGBlock *block,
 
 void LiveVariables::runOnAllBlocks(LiveVariables::Observer &obs) {
   const CFG *cfg = getImpl(impl).analysisContext.getCFG();
-  for (CFG::const_iterator it = cfg->begin(), ei = cfg->end(); it != ei; ++it)
-    getImpl(impl).runOnBlock(*it, getImpl(impl).blocksEndToLiveness[*it], &obs);
+  for (CFGBlock *B : *cfg)
+    getImpl(impl).runOnBlock(B, getImpl(impl).blocksEndToLiveness[B], &obs);
 }
 
 LiveVariables::LiveVariables(void *im) : impl(im) {}
@@ -618,10 +618,8 @@ void LiveVariables::dumpBlockLiveness(const SourceManager &M) {
 
 void LiveVariablesImpl::dumpBlockLiveness(const SourceManager &M) {
   std::vector<const CFGBlock *> vec;
-  for (llvm::DenseMap<const CFGBlock *, LiveVariables::LivenessValues>::iterator
-       it = blocksEndToLiveness.begin(), ei = blocksEndToLiveness.end();
-       it != ei; ++it) {
-    vec.push_back(it->first);
+  for (const auto &KV : blocksEndToLiveness) {
+    vec.push_back(KV.first);
   }
   llvm::sort(vec, [](const CFGBlock *A, const CFGBlock *B) {
     return A->getBlockID() < B->getBlockID();
diff --git a/clang/lib/Analysis/PathDiagnostic.cpp b/clang/lib/Analysis/PathDiagnostic.cpp
index b1fbc3c9eff94..ef24efd3c4bd0 100644
--- a/clang/lib/Analysis/PathDiagnostic.cpp
+++ b/clang/lib/Analysis/PathDiagnostic.cpp
@@ -1146,9 +1146,9 @@ void PathDiagnostic::FullProfile(llvm::FoldingSetNodeID &ID) const {
 
 LLVM_DUMP_METHOD void PathPieces::dump() const {
   unsigned index = 0;
-  for (PathPieces::const_iterator I = begin(), E = end(); I != E; ++I) {
+  for (const PathDiagnosticPieceRef &Piece : *this) {
     llvm::errs() << "[" << index++ << "]  ";
-    (*I)->dump();
+    Piece->dump();
     llvm::errs() << "\n";
   }
 }
diff --git a/clang/lib/Analysis/ReachableCode.cpp b/clang/lib/Analysis/ReachableCode.cpp
index 739c47b12e8c4..4a9ab5d9f0f73 100644
--- a/clang/lib/Analysis/ReachableCode.cpp
+++ b/clang/lib/Analysis/ReachableCode.cpp
@@ -460,9 +460,8 @@ static bool isInCoroutineStmt(const Stmt *DeadStmt, const CFGBlock *Block) {
   const Stmt *CoroStmt = nullptr;
   // Find the first coroutine statement after the DeadStmt in the block.
   bool AfterDeadStmt = false;
-  for (CFGBlock::const_iterator I = Block->begin(), E = Block->end(); I != E;
-       ++I)
-    if (std::optional<CFGStmt> CS = I->getAs<CFGStmt>()) {
+  for (const CFGElement &Elem : *Block)
+    if (std::optional<CFGStmt> CS = Elem.getAs<CFGStmt>()) {
       const Stmt *S = CS->getStmt();
       if (S == DeadStmt)
         AfterDeadStmt = true;
diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp
index 8d279d969b613..987f894a2918f 100644
--- a/clang/lib/Analysis/RetainSummaryManager.cpp
+++ b/clang/lib/Analysis/RetainSummaryManager.cpp
@@ -12,13 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include "clang/Analysis/RetainSummaryManager.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/AST/ParentMap.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include <optional>
 
 using namespace clang;
diff --git a/clang/lib/Basic/CodeGenOptions.cpp b/clang/lib/Basic/CodeGenOptions.cpp
index 1f899dee48dbd..9a2aa9beb9e0d 100644
--- a/clang/lib/Basic/CodeGenOptions.cpp
+++ b/clang/lib/Basic/CodeGenOptions.cpp
@@ -11,43 +11,39 @@
 namespace clang {
 
 CodeGenOptions::CodeGenOptions() {
-#define CODEGENOPT(Name, Bits, Default) Name = Default;
-#define ENUM_CODEGENOPT(Name, Type, Bits, Default) set##Name(Default);
+#define CODEGENOPT(Name, Bits, Default, Compatibility) Name = Default;
+#define ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility)              \
+  set##Name(Default);
 #include "clang/Basic/CodeGenOptions.def"
 
   RelocationModel = llvm::Reloc::PIC_;
 }
 
 void CodeGenOptions::resetNonModularOptions(StringRef ModuleFormat) {
-  // First reset all CodeGen options only. The Debug options are handled later.
-#define DEBUGOPT(Name, Bits, Default)
-#define VALUE_DEBUGOPT(Name, Bits, Default)
-#define ENUM_DEBUGOPT(Name, Type, Bits, Default)
-#define CODEGENOPT(Name, Bits, Default) Name = Default;
-#define ENUM_CODEGENOPT(Name, Type, Bits, Default) set##Name(Default);
-// Do not reset AST affecting code generation options.
-#define AFFECTING_VALUE_CODEGENOPT(Name, Bits, Default)
+  // FIXME: Replace with C++20 `using enum CodeGenOptions::CompatibilityKind`.
+  using CK = CompatibilityKind;
+
+  // First reset benign codegen and debug options.
+#define CODEGENOPT(Name, Bits, Default, Compatibility)                         \
+  if constexpr (CK::Compatibility == CK::Benign)                               \
+    Name = Default;
+#define ENUM_CODEGENOPT(Name, Type, Bits, Default, Compatibility)              \
+  if constexpr (CK::Compatibility == CK::Benign)                               \
+    set##Name(Default);
 #include "clang/Basic/CodeGenOptions.def"
 
-  // Next reset all debug options that can always be reset, because they never
-  // affect the PCM.
-#define DEBUGOPT(Name, Bits, Default)
-#define VALUE_DEBUGOPT(Name, Bits, Default)
-#define ENUM_DEBUGOPT(Name, Type, Bits, Default)
-#define BENIGN_DEBUGOPT(Name, Bits, Default) Name = Default;
-#define BENIGN_VALUE_DEBUGOPT(Name, Bits, Default) Name = Default;
-#define BENIGN_ENUM_DEBUGOPT(Name, Type, Bits, Default) set##Name(Default);
-#include "clang/Basic/DebugOptions.def"
-
   // Conditionally reset debug options that only matter when the debug info is
   // emitted into the PCM (-gmodules).
   if (ModuleFormat == "raw" && !DebugTypeExtRefs) {
-#define DEBUGOPT(Name, Bits, Default) Name = Default;
-#define VALUE_DEBUGOPT(Name, Bits, Default) Name = Default;
-#define ENUM_DEBUGOPT(Name, Type, Bits, Default) set##Name(Default);
-#define BENIGN_DEBUGOPT(Name, Bits, Default)
-#define BENIGN_VALUE_DEBUGOPT(Name, Bits, Default)
-#define BENIGN_ENUM_DEBUGOPT(Name, Type, Bits, Default)
+#define DEBUGOPT(Name, Bits, Default, Compatibility)                           \
+  if constexpr (CK::Compatibility == CK::Affecting)                            \
+    Name = Default;
+#define VALUE_DEBUGOPT(Name, Bits, Default, Compatibility)                     \
+  if constexpr (CK::Compatibility == CK::Affecting)                            \
+    Name = Default;
+#define ENUM_DEBUGOPT(Name, Type, Bits, Default, Compatibility)                \
+  if constexpr (CK::Compatibility == CK::Affecting)                            \
+    set##Name(Default);
 #include "clang/Basic/DebugOptions.def"
   }
 
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index ab0525e96f3ba..e33e843db6a44 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -1348,6 +1348,7 @@ void Diagnostic::FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
     case DiagnosticsEngine::ak_declcontext:
     case DiagnosticsEngine::ak_attr:
     case DiagnosticsEngine::ak_expr:
+    case DiagnosticsEngine::ak_attr_info:
       getDiags()->ConvertArgToString(Kind, getRawArg(ArgNo),
                                      StringRef(Modifier, ModifierLen),
                                      StringRef(Argument, ArgumentLen),
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index dcf0c6cb54282..73f24a82d4c75 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -832,8 +832,12 @@ bool DiagnosticIDs::isUnrecoverable(unsigned DiagID) const {
       DiagID == diag::err_unavailable_message)
     return false;
 
-  // Currently we consider all ARC errors as recoverable.
-  if (isARCDiagnostic(DiagID))
+  // All ARC errors are currently considered recoverable, with the exception of
+  // err_arc_may_not_respond. This specific error is treated as unrecoverable
+  // because sending a message with an unknown selector could lead to crashes
+  // within CodeGen if the resulting expression is used to initialize a C++
+  // auto variable, where type deduction is required.
+  if (isARCDiagnostic(DiagID) && DiagID != diag::err_arc_may_not_respond)
     return false;
 
   if (isCodegenABICheckDiagnostic(DiagID))
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 3bd7e0ca5bcd2..fc4ec785d3764 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -141,7 +141,7 @@ FileManager::getDirectoryRef(StringRef DirName, bool CacheFailure) {
   if (DirName.size() > 1 &&
       DirName != llvm::sys::path::root_path(DirName) &&
       llvm::sys::path::is_separator(DirName.back()))
-    DirName = DirName.substr(0, DirName.size()-1);
+    DirName = DirName.drop_back();
   std::optional<std::string> DirNameStr;
   if (is_style_windows(llvm::sys::path::Style::native)) {
     // Fixing a problem with "clang C:test.c" on Windows.
diff --git a/clang/lib/Basic/LangOptions.cpp b/clang/lib/Basic/LangOptions.cpp
index 912b890569cf5..9c14a25699f89 100644
--- a/clang/lib/Basic/LangOptions.cpp
+++ b/clang/lib/Basic/LangOptions.cpp
@@ -16,16 +16,19 @@
 using namespace clang;
 
 LangOptions::LangOptions() : LangStd(LangStandard::lang_unspecified) {
-#define LANGOPT(Name, Bits, Default, Description) Name = Default;
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) set##Name(Default);
+#define LANGOPT(Name, Bits, Default, Compatibility, Description) Name = Default;
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
+  set##Name(Default);
 #include "clang/Basic/LangOptions.def"
 }
 
 void LangOptions::resetNonModularOptions() {
-#define LANGOPT(Name, Bits, Default, Description)
-#define BENIGN_LANGOPT(Name, Bits, Default, Description) Name = Default;
-#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  Name = static_cast<unsigned>(Default);
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
+  if constexpr (CompatibilityKind::Compatibility == CompatibilityKind::Benign) \
+    Name = Default;
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
+  if constexpr (CompatibilityKind::Compatibility == CompatibilityKind::Benign) \
+    Name = static_cast<unsigned>(Default);
 #include "clang/Basic/LangOptions.def"
 
   // Reset "benign" options with implied values (Options.td ImpliedBy relations)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index a05d6c16caa32..e6b61302cc294 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -329,12 +329,14 @@ SourceManager::~SourceManager() {
 void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
+  LocalLocOffsetTable.clear();
   LoadedSLocEntryTable.clear();
   SLocEntryLoaded.clear();
   SLocEntryOffsetLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
+  LastLookupStartOffset = LastLookupEndOffset = 0;
 
   IncludedLocMap.clear();
   if (LineTable)
@@ -587,17 +589,6 @@ SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
 					  FileCharacter);
 }
 
-/// Helper function to determine if an input file requires conversion
-bool needConversion(StringRef Filename) {
-#ifdef __MVS__
-  llvm::ErrorOr<bool> NeedConversion =
-      llvm::needzOSConversion(Filename.str().c_str());
-  return NeedConversion && *NeedConversion;
-#else
-  return false;
-#endif
-}
-
 /// createFileID - Create a new FileID for the specified ContentCache and
 /// include position.  This works regardless of whether the ContentCache
 /// corresponds to a file or some other input source.
@@ -617,8 +608,9 @@ FileID SourceManager::createFileIDImpl(ContentCache &File, StringRef Filename,
     return FileID::get(LoadedID);
   }
   unsigned FileSize = File.getSize();
-  bool NeedConversion = needConversion(Filename);
-  if (NeedConversion) {
+  llvm::ErrorOr<bool> NeedConversion =
+      llvm::needConversion(Filename.str().c_str());
+  if (NeedConversion && *NeedConversion) {
     // Buffer size may increase due to potential z/OS EBCDIC to UTF-8
     // conversion.
     if (std::optional<llvm::MemoryBufferRef> Buffer =
@@ -637,12 +629,16 @@ FileID SourceManager::createFileIDImpl(ContentCache &File, StringRef Filename,
     noteSLocAddressSpaceUsage(Diag);
     return FileID();
   }
+  assert(LocalSLocEntryTable.size() == LocalLocOffsetTable.size());
   LocalSLocEntryTable.push_back(
       SLocEntry::get(NextLocalOffset,
                      FileInfo::get(IncludePos, File, FileCharacter, Filename)));
+  LocalLocOffsetTable.push_back(NextLocalOffset);
+  LastLookupStartOffset = NextLocalOffset;
   // We do a +1 here because we want a SourceLocation that means "the end of the
   // file", e.g. for the "no newline at the end of the file" diagnostic.
   NextLocalOffset += FileSize + 1;
+  LastLookupEndOffset = NextLocalOffset;
   updateSlocUsageStats();
 
   // Set LastFileIDLookup to the newly created file.  The next getFileID call is
@@ -691,7 +687,9 @@ SourceManager::createExpansionLocImpl(const ExpansionInfo &Info,
     SLocEntryLoaded[Index] = SLocEntryOffsetLoaded[Index] = true;
     return SourceLocation::getMacroLoc(LoadedOffset);
   }
+  assert(LocalSLocEntryTable.size() == LocalLocOffsetTable.size());
   LocalSLocEntryTable.push_back(SLocEntry::get(NextLocalOffset, Info));
+  LocalLocOffsetTable.push_back(NextLocalOffset);
   if (NextLocalOffset + Length + 1 <= NextLocalOffset ||
       NextLocalOffset + Length + 1 > CurrentLoadedOffset) {
     Diag.Report(diag::err_sloc_space_too_large);
@@ -812,6 +810,10 @@ FileID SourceManager::getFileIDSlow(SourceLocation::UIntTy SLocOffset) const {
 /// loaded one.
 FileID SourceManager::getFileIDLocal(SourceLocation::UIntTy SLocOffset) const {
   assert(SLocOffset < NextLocalOffset && "Bad function choice");
+  assert(SLocOffset >= LocalSLocEntryTable[0].getOffset() && SLocOffset > 0 &&
+         "Invalid SLocOffset");
+  assert(LocalSLocEntryTable.size() == LocalLocOffsetTable.size());
+  assert(LastFileIDLookup.ID >= 0 && "Only cache local file sloc entry");
 
   // After the first and second level caches, I see two common sorts of
   // behavior: 1) a lot of searched FileID's are "near" the cached file
@@ -830,60 +832,50 @@ FileID SourceManager::getFileIDLocal(SourceLocation::UIntTy SLocOffset) const {
   // SLocOffset.
   unsigned LessIndex = 0;
   // upper bound of the search range.
-  unsigned GreaterIndex = LocalSLocEntryTable.size();
-  if (LastFileIDLookup.ID >= 0) {
-    // Use the LastFileIDLookup to prune the search space.
-    if (LocalSLocEntryTable[LastFileIDLookup.ID].getOffset() < SLocOffset)
-      LessIndex = LastFileIDLookup.ID;
-    else
-      GreaterIndex = LastFileIDLookup.ID;
-  }
+  unsigned GreaterIndex = LocalLocOffsetTable.size();
+  // Use the LastFileIDLookup to prune the search space.
+  if (LastLookupStartOffset < SLocOffset)
+    LessIndex = LastFileIDLookup.ID;
+  else
+    GreaterIndex = LastFileIDLookup.ID;
 
   // Find the FileID that contains this.
   unsigned NumProbes = 0;
   while (true) {
     --GreaterIndex;
-    assert(GreaterIndex < LocalSLocEntryTable.size());
-    if (LocalSLocEntryTable[GreaterIndex].getOffset() <= SLocOffset) {
+    assert(GreaterIndex < LocalLocOffsetTable.size());
+    if (LocalLocOffsetTable[GreaterIndex] <= SLocOffset) {
       FileID Res = FileID::get(int(GreaterIndex));
       // Remember it.  We have good locality across FileID lookups.
       LastFileIDLookup = Res;
-      NumLinearScans += NumProbes+1;
+      LastLookupStartOffset = LocalLocOffsetTable[GreaterIndex];
+      LastLookupEndOffset =
+          GreaterIndex + 1 >= LocalLocOffsetTable.size()
+              ? NextLocalOffset
+              : LocalLocOffsetTable[GreaterIndex + 1];
+      NumLinearScans += NumProbes + 1;
       return Res;
     }
     if (++NumProbes == 8)
       break;
   }
 
-  NumProbes = 0;
-  while (true) {
-    unsigned MiddleIndex = (GreaterIndex-LessIndex)/2+LessIndex;
-    SourceLocation::UIntTy MidOffset =
-        getLocalSLocEntry(MiddleIndex).getOffset();
-
-    ++NumProbes;
+  while (LessIndex < GreaterIndex) {
+    ++NumBinaryProbes;
 
-    // If the offset of the midpoint is too large, chop the high side of the
-    // range to the midpoint.
-    if (MidOffset > SLocOffset) {
+    unsigned MiddleIndex = LessIndex + (GreaterIndex - LessIndex) / 2;
+    if (LocalLocOffsetTable[MiddleIndex] <= SLocOffset)
+      LessIndex = MiddleIndex + 1;
+    else
       GreaterIndex = MiddleIndex;
-      continue;
-    }
-
-    // If the middle index contains the value, succeed and return.
-    if (MiddleIndex + 1 == LocalSLocEntryTable.size() ||
-        SLocOffset < getLocalSLocEntry(MiddleIndex + 1).getOffset()) {
-      FileID Res = FileID::get(MiddleIndex);
-
-      // Remember it.  We have good locality across FileID lookups.
-      LastFileIDLookup = Res;
-      NumBinaryProbes += NumProbes;
-      return Res;
-    }
-
-    // Otherwise, move the low-side up to the middle index.
-    LessIndex = MiddleIndex;
   }
+
+  // At this point, LessIndex is the index of the *first element greater than*
+  // SLocOffset. The element we are actually looking for is the one immediately
+  // before it.
+  LastLookupStartOffset = LocalLocOffsetTable[LessIndex - 1];
+  LastLookupEndOffset = LocalLocOffsetTable[LessIndex];
+  return LastFileIDLookup = FileID::get(LessIndex - 1);
 }
 
 /// Return the FileID for a SourceLocation with a high offset.
@@ -1803,9 +1795,8 @@ void SourceManager::associateFileChunkWithMacroArgExp(
     // spelling range and if one is itself a macro argument expansion, recurse
     // and associate the file chunk that it represents.
 
-    FileID SpellFID; // Current FileID in the spelling range.
-    unsigned SpellRelativeOffs;
-    std::tie(SpellFID, SpellRelativeOffs) = getDecomposedLoc(SpellLoc);
+    // Current FileID in the spelling range.
+    auto [SpellFID, SpellRelativeOffs] = getDecomposedLoc(SpellLoc);
     while (true) {
       const SLocEntry &Entry = getSLocEntry(SpellFID);
       SourceLocation::UIntTy SpellFIDBeginOffs = Entry.getOffset();
@@ -1887,9 +1878,7 @@ SourceManager::getMacroArgExpandedLocation(SourceLocation Loc) const {
   if (Loc.isInvalid() || !Loc.isFileID())
     return Loc;
 
-  FileID FID;
-  unsigned Offset;
-  std::tie(FID, Offset) = getDecomposedLoc(Loc);
+  auto [FID, Offset] = getDecomposedLoc(Loc);
   if (FID.isInvalid())
     return Loc;
 
@@ -2401,3 +2390,75 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
   assert(ID.isValid());
   SourceMgr->setMainFileID(ID);
 }
+
+StringRef
+SourceManager::getNameForDiagnostic(StringRef Filename,
+                                    const DiagnosticOptions &Opts) const {
+  OptionalFileEntryRef File = getFileManager().getOptionalFileRef(Filename);
+  if (!File)
+    return Filename;
+
+  bool SimplifyPath = [&] {
+    if (Opts.AbsolutePath)
+      return true;
+
+    // Try to simplify paths that contain '..' in any case since paths to
+    // standard library headers especially tend to get quite long otherwise.
+    // Only do that for local filesystems though to avoid slowing down
+    // compilation too much.
+    if (!File->getName().contains(".."))
+      return false;
+
+    // If we're not on Windows, check if we're on a network file system and
+    // avoid simplifying the path in that case since that can be slow. On
+    // Windows, the check for a local filesystem is already slow, so skip it.
+#ifndef _WIN32
+    if (!llvm::sys::fs::is_local(File->getName()))
+      return false;
+#endif
+
+    return true;
+  }();
+
+  if (!SimplifyPath)
+    return Filename;
+
+  // This may involve computing canonical names, so cache the result.
+  StringRef &CacheEntry = DiagNames[Filename];
+  if (!CacheEntry.empty())
+    return CacheEntry;
+
+  // We want to print a simplified absolute path, i. e. without "dots".
+  //
+  // The hardest part here are the paths like "<part1>/<link>/../<part2>".
+  // On Unix-like systems, we cannot just collapse "<link>/..", because
+  // paths are resolved sequentially, and, thereby, the path
+  // "<part1>/<part2>" may point to a different location. That is why
+  // we use FileManager::getCanonicalName(), which expands all indirections
+  // with llvm::sys::fs::real_path() and caches the result.
+  //
+  // On the other hand, it would be better to preserve as much of the
+  // original path as possible, because that helps a user to recognize it.
+  // real_path() expands all links, which sometimes too much. Luckily,
+  // on Windows we can just use llvm::sys::path::remove_dots(), because,
+  // on that system, both aforementioned paths point to the same place.
+  SmallString<256> TempBuf;
+#ifdef _WIN32
+  TempBuf = File->getName();
+  llvm::sys::fs::make_absolute(TempBuf);
+  llvm::sys::path::native(TempBuf);
+  llvm::sys::path::remove_dots(TempBuf, /* remove_dot_dot */ true);
+#else
+  TempBuf = getFileManager().getCanonicalName(*File);
+#endif
+
+  // In some cases, the resolved path may actually end up being longer (e.g.
+  // if it was originally a relative path), so just retain whichever one
+  // ends up being shorter.
+  if (!Opts.AbsolutePath && TempBuf.size() > Filename.size())
+    CacheEntry = Filename;
+  else
+    CacheEntry = TempBuf.str().copy(DiagNameAlloc);
+
+  return CacheEntry;
+}
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 124b340b62d9f..72d2e5fcf4619 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -125,79 +125,6 @@ static constexpr auto BuiltinAArch64Infos =
 #include "clang/Basic/BuiltinsAArch64.def"
     });
 
-void AArch64TargetInfo::setArchFeatures() {
-  if (*ArchInfo == llvm::AArch64::ARMV8R) {
-    HasDotProd = true;
-    HasDIT = true;
-    HasFlagM = true;
-    HasRCPC = true;
-    FPU |= NeonMode;
-    HasCCPP = true;
-    HasCRC = true;
-    HasLSE = true;
-    HasRDM = true;
-  } else if (ArchInfo->Version.getMajor() == 8) {
-    if (ArchInfo->Version.getMinor() >= 7u) {
-      HasWFxT = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 6u) {
-      HasBFloat16 = true;
-      HasMatMul = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 5u) {
-      HasAlternativeNZCV = true;
-      HasFRInt3264 = true;
-      HasSSBS = true;
-      HasSB = true;
-      HasPredRes = true;
-      HasBTI = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 4u) {
-      HasDotProd = true;
-      HasDIT = true;
-      HasFlagM = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 3u) {
-      HasRCPC = true;
-      FPU |= NeonMode;
-    }
-    if (ArchInfo->Version.getMinor() >= 2u) {
-      HasCCPP = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 1u) {
-      HasCRC = true;
-      HasLSE = true;
-      HasRDM = true;
-    }
-  } else if (ArchInfo->Version.getMajor() == 9) {
-    if (ArchInfo->Version.getMinor() >= 2u) {
-      HasWFxT = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 1u) {
-      HasBFloat16 = true;
-      HasMatMul = true;
-    }
-    FPU |= SveMode;
-    HasSVE2 = true;
-    HasFullFP16 = true;
-    HasAlternativeNZCV = true;
-    HasFRInt3264 = true;
-    HasSSBS = true;
-    HasSB = true;
-    HasPredRes = true;
-    HasBTI = true;
-    HasDotProd = true;
-    HasDIT = true;
-    HasFlagM = true;
-    HasRCPC = true;
-    FPU |= NeonMode;
-    HasCCPP = true;
-    HasCRC = true;
-    HasLSE = true;
-    HasRDM = true;
-  }
-}
-
 AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
                                      const TargetOptions &Opts)
     : TargetInfo(Triple), ABI("aapcs") {
@@ -823,16 +750,36 @@ AArch64TargetInfo::getTargetBuiltins() const {
 
 std::optional<std::pair<unsigned, unsigned>>
 AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts,
-                                  bool IsArmStreamingFunction,
+                                  ArmStreamingKind Mode,
                                   llvm::StringMap<bool> *FeatureMap) const {
-  if (LangOpts.VScaleMin || LangOpts.VScaleMax)
+  if (Mode == ArmStreamingKind::NotStreaming &&
+      (LangOpts.VScaleMin || LangOpts.VScaleMax))
+    return std::pair<unsigned, unsigned>(
+        LangOpts.VScaleMin ? LangOpts.VScaleMin : 1,
+        LangOpts.VScaleMax ? LangOpts.VScaleMax : 16);
+
+  if (Mode == ArmStreamingKind::Streaming &&
+      (LangOpts.VScaleStreamingMin || LangOpts.VScaleStreamingMax))
     return std::pair<unsigned, unsigned>(
-        LangOpts.VScaleMin ? LangOpts.VScaleMin : 1, LangOpts.VScaleMax);
+        LangOpts.VScaleStreamingMin ? LangOpts.VScaleStreamingMin : 1,
+        LangOpts.VScaleStreamingMax ? LangOpts.VScaleStreamingMax : 16);
+
+  if (Mode == ArmStreamingKind::StreamingCompatible &&
+      ((LangOpts.VScaleMin && LangOpts.VScaleStreamingMin) ||
+       (LangOpts.VScaleMax && LangOpts.VScaleStreamingMax))) {
+    unsigned Min =
+        std::min(LangOpts.VScaleMin ? LangOpts.VScaleMin : 1,
+                 LangOpts.VScaleStreamingMin ? LangOpts.VScaleStreamingMin : 1);
+    unsigned Max = std::max(
+        LangOpts.VScaleMax ? LangOpts.VScaleMax : 16,
+        LangOpts.VScaleStreamingMax ? LangOpts.VScaleStreamingMax : 16);
+    return std::pair(Min, Max);
+  }
 
   if (hasFeature("sve") || (FeatureMap && (FeatureMap->lookup("sve"))))
     return std::pair<unsigned, unsigned>(1, 16);
 
-  if (IsArmStreamingFunction &&
+  if (Mode == ArmStreamingKind::Streaming &&
       (hasFeature("sme") || (FeatureMap && (FeatureMap->lookup("sme")))))
     return std::pair<unsigned, unsigned>(1, 16);
 
@@ -1266,7 +1213,6 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   }
 
   setDataLayout();
-  setArchFeatures();
 
   if (HasNoFP) {
     FPU &= ~FPUMode;
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 1951e0679d2ec..f4277e95b19be 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -158,8 +158,6 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
     return false;
   }
 
-  void setArchFeatures();
-
   void getTargetDefinesARMV81A(const LangOptions &Opts,
                                MacroBuilder &Builder) const;
   void getTargetDefinesARMV82A(const LangOptions &Opts,
@@ -198,7 +196,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts, bool IsArmStreamingFunction,
+  getVScaleRange(const LangOptions &LangOpts, ArmStreamingKind Mode,
                  llvm::StringMap<bool> *FeatureMap = nullptr) const override;
   bool doesFeatureAffectCodeGen(StringRef Name) const override;
   bool validateCpuSupports(StringRef FeatureStr) const override;
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index 509128f3cf070..1358abb70f984 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -439,6 +439,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
   // pre-defined macros.
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override {
+    HasFullBFloat16 = true;
     auto TargetIDFeatures =
         getAllPossibleTargetIDFeatures(getTriple(), getArchNameAMDGCN(GPUKind));
     for (const auto &F : Features) {
diff --git a/clang/lib/Basic/Targets/LoongArch.cpp b/clang/lib/Basic/Targets/LoongArch.cpp
index 5312767498d96..f6915df1520b7 100644
--- a/clang/lib/Basic/Targets/LoongArch.cpp
+++ b/clang/lib/Basic/Targets/LoongArch.cpp
@@ -81,38 +81,46 @@ LoongArchTargetInfo::getGCCRegAliases() const {
       {{"s6", "$s6", "r29"}, "$r29"},
       {{"s7", "$s7", "r30"}, "$r30"},
       {{"s8", "$s8", "r31"}, "$r31"},
-      {{"$fa0"}, "$f0"},
-      {{"$fa1"}, "$f1"},
-      {{"$fa2"}, "$f2"},
-      {{"$fa3"}, "$f3"},
-      {{"$fa4"}, "$f4"},
-      {{"$fa5"}, "$f5"},
-      {{"$fa6"}, "$f6"},
-      {{"$fa7"}, "$f7"},
-      {{"$ft0"}, "$f8"},
-      {{"$ft1"}, "$f9"},
-      {{"$ft2"}, "$f10"},
-      {{"$ft3"}, "$f11"},
-      {{"$ft4"}, "$f12"},
-      {{"$ft5"}, "$f13"},
-      {{"$ft6"}, "$f14"},
-      {{"$ft7"}, "$f15"},
-      {{"$ft8"}, "$f16"},
-      {{"$ft9"}, "$f17"},
-      {{"$ft10"}, "$f18"},
-      {{"$ft11"}, "$f19"},
-      {{"$ft12"}, "$f20"},
-      {{"$ft13"}, "$f21"},
-      {{"$ft14"}, "$f22"},
-      {{"$ft15"}, "$f23"},
-      {{"$fs0"}, "$f24"},
-      {{"$fs1"}, "$f25"},
-      {{"$fs2"}, "$f26"},
-      {{"$fs3"}, "$f27"},
-      {{"$fs4"}, "$f28"},
-      {{"$fs5"}, "$f29"},
-      {{"$fs6"}, "$f30"},
-      {{"$fs7"}, "$f31"},
+      {{"fa0", "$fa0", "f0"}, "$f0"},
+      {{"fa1", "$fa1", "f1"}, "$f1"},
+      {{"fa2", "$fa2", "f2"}, "$f2"},
+      {{"fa3", "$fa3", "f3"}, "$f3"},
+      {{"fa4", "$fa4", "f4"}, "$f4"},
+      {{"fa5", "$fa5", "f5"}, "$f5"},
+      {{"fa6", "$fa6", "f6"}, "$f6"},
+      {{"fa7", "$fa7", "f7"}, "$f7"},
+      {{"ft0", "$ft0", "f8"}, "$f8"},
+      {{"ft1", "$ft1", "f9"}, "$f9"},
+      {{"ft2", "$ft2", "f10"}, "$f10"},
+      {{"ft3", "$ft3", "f11"}, "$f11"},
+      {{"ft4", "$ft4", "f12"}, "$f12"},
+      {{"ft5", "$ft5", "f13"}, "$f13"},
+      {{"ft6", "$ft6", "f14"}, "$f14"},
+      {{"ft7", "$ft7", "f15"}, "$f15"},
+      {{"ft8", "$ft8", "f16"}, "$f16"},
+      {{"ft9", "$ft9", "f17"}, "$f17"},
+      {{"ft10", "$ft10", "f18"}, "$f18"},
+      {{"ft11", "$ft11", "f19"}, "$f19"},
+      {{"ft12", "$ft12", "f20"}, "$f20"},
+      {{"ft13", "$ft13", "f21"}, "$f21"},
+      {{"ft14", "$ft14", "f22"}, "$f22"},
+      {{"ft15", "$ft15", "f23"}, "$f23"},
+      {{"fs0", "$fs0", "f24"}, "$f24"},
+      {{"fs1", "$fs1", "f25"}, "$f25"},
+      {{"fs2", "$fs2", "f26"}, "$f26"},
+      {{"fs3", "$fs3", "f27"}, "$f27"},
+      {{"fs4", "$fs4", "f28"}, "$f28"},
+      {{"fs5", "$fs5", "f29"}, "$f29"},
+      {{"fs6", "$fs6", "f30"}, "$f30"},
+      {{"fs7", "$fs7", "f31"}, "$f31"},
+      {{"fcc0"}, "$fcc0"},
+      {{"fcc1"}, "$fcc1"},
+      {{"fcc2"}, "$fcc2"},
+      {{"fcc3"}, "$fcc3"},
+      {{"fcc4"}, "$fcc4"},
+      {{"fcc5"}, "$fcc5"},
+      {{"fcc6"}, "$fcc6"},
+      {{"fcc7"}, "$fcc7"},
   };
   return llvm::ArrayRef(GCCRegAliases);
 }
diff --git a/clang/lib/Basic/Targets/LoongArch.h b/clang/lib/Basic/Targets/LoongArch.h
index 3ad5abca927b7..88dc433924d6c 100644
--- a/clang/lib/Basic/Targets/LoongArch.h
+++ b/clang/lib/Basic/Targets/LoongArch.h
@@ -60,6 +60,7 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
     SuitableAlign = 128;
     WCharType = SignedInt;
     WIntType = UnsignedInt;
+    BitIntMaxAlign = 128;
   }
 
   bool setCPU(const std::string &Name) override {
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index f0c77f027858f..3988cb5294560 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -56,6 +56,7 @@ M68kTargetInfo::M68kTargetInfo(const llvm::Triple &Triple,
   SizeType = UnsignedInt;
   PtrDiffType = SignedInt;
   IntPtrType = SignedInt;
+  IntAlign = LongAlign = PointerAlign = 16;
 }
 
 bool M68kTargetInfo::setCPU(const std::string &Name) {
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 5dac699c2bb45..30d861a7ca605 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -1002,6 +1002,7 @@ class LLVM_LIBRARY_VISIBILITY EmscriptenTargetInfo
     // Emscripten's ABI is unstable and we may change this back to 128 to match
     // the WebAssembly default in the future.
     this->LongDoubleAlign = 64;
+    this->Float128Align = 64;
   }
 };
 
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 2098449dd83a3..8a28c0788aad7 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -222,7 +222,7 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   // Currently we support the v1.0 RISC-V V intrinsics.
   Builder.defineMacro("__riscv_v_intrinsic", Twine(getVersionValue(1, 0)));
 
-  auto VScale = getVScaleRange(Opts, false);
+  auto VScale = getVScaleRange(Opts, ArmStreamingKind::NotStreaming);
   if (VScale && VScale->first && VScale->first == VScale->second)
     Builder.defineMacro("__riscv_v_fixed_vlen",
                         Twine(VScale->first * llvm::RISCV::RVVBitsPerBlock));
@@ -367,7 +367,7 @@ bool RISCVTargetInfo::initFeatureMap(
 
 std::optional<std::pair<unsigned, unsigned>>
 RISCVTargetInfo::getVScaleRange(const LangOptions &LangOpts,
-                                bool IsArmStreamingFunction,
+                                ArmStreamingKind IsArmStreamingFunction,
                                 llvm::StringMap<bool> *FeatureMap) const {
   // RISCV::RVVBitsPerBlock is 64.
   unsigned VScaleMin = ISAInfo->getMinVLen() / llvm::RISCV::RVVBitsPerBlock;
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index 0b36c9d5d9cc8..8d629abab7bec 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -99,7 +99,7 @@ class RISCVTargetInfo : public TargetInfo {
                  const std::vector<std::string> &FeaturesVec) const override;
 
   std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts, bool IsArmStreamingFunction,
+  getVScaleRange(const LangOptions &LangOpts, ArmStreamingKind Mode,
                  llvm::StringMap<bool> *FeatureMap = nullptr) const override;
 
   bool hasFeature(StringRef Feature) const override;
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
index b94963c71f9c1..4a5a1dd53a05a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
@@ -63,7 +63,7 @@ cir::ConstantOp
 clang::CIRGen::CIRGenBuilderTy::getConstFP(mlir::Location loc, mlir::Type t,
                                            llvm::APFloat fpVal) {
   assert(mlir::isa<cir::FPTypeInterface>(t) && "expected floating point type");
-  return create<cir::ConstantOp>(loc, getAttr<cir::FPAttr>(t, fpVal));
+  return create<cir::ConstantOp>(loc, cir::FPAttr::get(t, fpVal));
 }
 
 // This can't be defined in Address.h because that file is included by
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index 17b931a0693aa..0b33f6c7d03b7 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
 
 #include "Address.h"
+#include "CIRGenRecordLayout.h"
 #include "CIRGenTypeCache.h"
 #include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 #include "clang/CIR/MissingFeatures.h"
@@ -392,6 +393,15 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
 
     return createGlobal(module, loc, uniqueName, type, linkage);
   }
+
+  mlir::Value createGetBitfield(mlir::Location loc, mlir::Type resultType,
+                                mlir::Value addr, mlir::Type storageType,
+                                const CIRGenBitFieldInfo &info,
+                                bool isLvalueVolatile, bool useVolatile) {
+    return create<cir::GetBitfieldOp>(loc, resultType, addr, storageType,
+                                      info.name, info.size, info.offset,
+                                      info.isSigned, isLvalueVolatile);
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 0943b5788b73a..fb046533a91b8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -34,6 +34,29 @@ static RValue emitLibraryCall(CIRGenFunction &cgf, const FunctionDecl *fd,
   return cgf.emitCall(e->getCallee()->getType(), callee, e, ReturnValueSlot());
 }
 
+template <typename Op>
+static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
+                               bool poisonZero = false) {
+  assert(!cir::MissingFeatures::builtinCheckKind());
+
+  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
+  CIRGenBuilderTy &builder = cgf.getBuilder();
+
+  Op op;
+  if constexpr (std::is_same_v<Op, cir::BitClzOp> ||
+                std::is_same_v<Op, cir::BitCtzOp>)
+    op = builder.create<Op>(cgf.getLoc(e->getSourceRange()), arg, poisonZero);
+  else
+    op = builder.create<Op>(cgf.getLoc(e->getSourceRange()), arg);
+
+  mlir::Value result = op.getResult();
+  mlir::Type exprTy = cgf.convertType(e->getType());
+  if (exprTy != result.getType())
+    result = builder.createIntCast(result, exprTy);
+
+  return RValue::get(result);
+}
+
 RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
                                        const CallExpr *e,
                                        ReturnValueSlot returnValue) {
@@ -101,6 +124,47 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
     return RValue::get(complex);
   }
 
+  case Builtin::BI__builtin_clrsb:
+  case Builtin::BI__builtin_clrsbl:
+  case Builtin::BI__builtin_clrsbll:
+    return emitBuiltinBitOp<cir::BitClrsbOp>(*this, e);
+
+  case Builtin::BI__builtin_ctzs:
+  case Builtin::BI__builtin_ctz:
+  case Builtin::BI__builtin_ctzl:
+  case Builtin::BI__builtin_ctzll:
+  case Builtin::BI__builtin_ctzg:
+    assert(!cir::MissingFeatures::builtinCheckKind());
+    return emitBuiltinBitOp<cir::BitCtzOp>(*this, e, /*poisonZero=*/true);
+
+  case Builtin::BI__builtin_clzs:
+  case Builtin::BI__builtin_clz:
+  case Builtin::BI__builtin_clzl:
+  case Builtin::BI__builtin_clzll:
+  case Builtin::BI__builtin_clzg:
+    assert(!cir::MissingFeatures::builtinCheckKind());
+    return emitBuiltinBitOp<cir::BitClzOp>(*this, e, /*poisonZero=*/true);
+
+  case Builtin::BI__builtin_parity:
+  case Builtin::BI__builtin_parityl:
+  case Builtin::BI__builtin_parityll:
+    return emitBuiltinBitOp<cir::BitParityOp>(*this, e);
+
+  case Builtin::BI__lzcnt16:
+  case Builtin::BI__lzcnt:
+  case Builtin::BI__lzcnt64:
+    assert(!cir::MissingFeatures::builtinCheckKind());
+    return emitBuiltinBitOp<cir::BitClzOp>(*this, e, /*poisonZero=*/false);
+
+  case Builtin::BI__popcnt16:
+  case Builtin::BI__popcnt:
+  case Builtin::BI__popcnt64:
+  case Builtin::BI__builtin_popcount:
+  case Builtin::BI__builtin_popcountl:
+  case Builtin::BI__builtin_popcountll:
+  case Builtin::BI__builtin_popcountg:
+    return emitBuiltinBitOp<cir::BitPopcountOp>(*this, e);
+
   case Builtin::BI__builtin_expect:
   case Builtin::BI__builtin_expect_with_probability: {
     mlir::Value argValue = emitScalarExpr(e->getArg(0));
diff --git a/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
index e0d30ad22d271..8da832d9118e3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
@@ -85,13 +85,13 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
     return RValue::get(nullptr);
   }
 
-  bool trivialForCodegen =
-      md->isTrivial() || (md->isDefaulted() && md->getParent()->isUnion());
-  bool trivialAssignment =
-      trivialForCodegen &&
-      (md->isCopyAssignmentOperator() || md->isMoveAssignmentOperator()) &&
-      !md->getParent()->mayInsertExtraPadding();
-  (void)trivialAssignment;
+  // Note on trivial assignment
+  // --------------------------
+  // Classic codegen avoids generating the trivial copy/move assignment operator
+  // when it isn't necessary, choosing instead to just produce IR with an
+  // equivalent effect. We have chosen not to do that in CIR, instead emitting
+  // trivial copy/move assignment operators and allowing later transformations
+  // to optimize them away if appropriate.
 
   // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment
   // operator before the LHS.
@@ -99,9 +99,10 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
   CallArgList *rtlArgs = nullptr;
   if (auto *oce = dyn_cast<CXXOperatorCallExpr>(ce)) {
     if (oce->isAssignmentOp()) {
-      cgm.errorNYI(
-          oce->getSourceRange(),
-          "emitCXXMemberOrOperatorMemberCallExpr: assignment operator");
+      rtlArgs = &rtlArgStorage;
+      emitCallArgs(*rtlArgs, md->getType()->castAs<FunctionProtoType>(),
+                   drop_begin(ce->arguments(), 1), ce->getDirectCallee(),
+                   /*ParamsToSkip*/ 0);
     }
   }
 
@@ -121,19 +122,9 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
     return RValue::get(nullptr);
   }
 
-  if (trivialForCodegen) {
-    if (isa<CXXDestructorDecl>(md))
-      return RValue::get(nullptr);
-
-    if (trivialAssignment) {
-      cgm.errorNYI(ce->getSourceRange(),
-                   "emitCXXMemberOrOperatorMemberCallExpr: trivial assignment");
-      return RValue::get(nullptr);
-    }
-
-    assert(md->getParent()->mayInsertExtraPadding() &&
-           "unknown trivial member function");
-  }
+  if ((md->isTrivial() || (md->isDefaulted() && md->getParent()->isUnion())) &&
+      isa<CXXDestructorDecl>(md))
+    return RValue::get(nullptr);
 
   // Compute the function type we're calling
   const CXXMethodDecl *calleeDecl = md;
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
index 9c9c96604c168..938d1436a76ea 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
@@ -77,17 +77,40 @@ void CIRGenFunction::emitAggregateStore(mlir::Value value, Address dest) {
   builder.createStore(*currSrcLoc, value, dest);
 }
 
+static void addAttributesFromFunctionProtoType(CIRGenBuilderTy &builder,
+                                               mlir::NamedAttrList &attrs,
+                                               const FunctionProtoType *fpt) {
+  if (!fpt)
+    return;
+
+  if (!isUnresolvedExceptionSpec(fpt->getExceptionSpecType()) &&
+      fpt->isNothrow())
+    attrs.set(cir::CIRDialect::getNoThrowAttrName(),
+              mlir::UnitAttr::get(builder.getContext()));
+}
+
 /// Construct the CIR attribute list of a function or call.
 void CIRGenModule::constructAttributeList(CIRGenCalleeInfo calleeInfo,
-                                          cir::SideEffect &sideEffect) {
+                                          mlir::NamedAttrList &attrs) {
   assert(!cir::MissingFeatures::opCallCallConv());
-  sideEffect = cir::SideEffect::All;
+  auto sideEffect = cir::SideEffect::All;
 
-  assert(!cir::MissingFeatures::opCallAttrs());
+  addAttributesFromFunctionProtoType(getBuilder(), attrs,
+                                     calleeInfo.getCalleeFunctionProtoType());
 
   const Decl *targetDecl = calleeInfo.getCalleeDecl().getDecl();
 
   if (targetDecl) {
+    if (targetDecl->hasAttr<NoThrowAttr>())
+      attrs.set(cir::CIRDialect::getNoThrowAttrName(),
+                mlir::UnitAttr::get(&getMLIRContext()));
+
+    if (const FunctionDecl *func = dyn_cast<FunctionDecl>(targetDecl)) {
+      addAttributesFromFunctionProtoType(
+          getBuilder(), attrs, func->getType()->getAs<FunctionProtoType>());
+      assert(!cir::MissingFeatures::opCallAttrs());
+    }
+
     assert(!cir::MissingFeatures::opCallAttrs());
 
     // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
@@ -104,6 +127,9 @@ void CIRGenModule::constructAttributeList(CIRGenCalleeInfo calleeInfo,
   }
 
   assert(!cir::MissingFeatures::opCallAttrs());
+
+  attrs.set(cir::CIRDialect::getSideEffectAttrName(),
+            cir::SideEffectAttr::get(&getMLIRContext(), sideEffect));
 }
 
 /// Returns the canonical formal type of the given C++ method.
@@ -404,7 +430,7 @@ CIRGenTypes::arrangeFunctionDeclaration(const FunctionDecl *fd) {
           funcTy.getAs<FunctionNoProtoType>()) {
     assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
     assert(!cir::MissingFeatures::opCallFnInfoOpts());
-    return arrangeCIRFunctionInfo(noProto->getReturnType(), std::nullopt,
+    return arrangeCIRFunctionInfo(noProto->getReturnType(), {},
                                   RequiredArgs::All);
   }
 
@@ -416,7 +442,7 @@ emitCallLikeOp(CIRGenFunction &cgf, mlir::Location callLoc,
                cir::FuncType indirectFuncTy, mlir::Value indirectFuncVal,
                cir::FuncOp directFuncOp,
                const SmallVectorImpl<mlir::Value> &cirCallArgs,
-               cir::SideEffect sideEffect) {
+               const mlir::NamedAttrList &attrs) {
   CIRGenBuilderTy &builder = cgf.getBuilder();
 
   assert(!cir::MissingFeatures::opCallSurroundingTry());
@@ -424,14 +450,17 @@ emitCallLikeOp(CIRGenFunction &cgf, mlir::Location callLoc,
 
   assert(builder.getInsertionBlock() && "expected valid basic block");
 
+  cir::CallOp op;
   if (indirectFuncTy) {
     // TODO(cir): Set calling convention for indirect calls.
     assert(!cir::MissingFeatures::opCallCallConv());
-    return builder.createIndirectCallOp(
-        callLoc, indirectFuncVal, indirectFuncTy, cirCallArgs, sideEffect);
+    op = builder.createIndirectCallOp(callLoc, indirectFuncVal, indirectFuncTy,
+                                      cirCallArgs, attrs);
+  } else {
+    op = builder.createCallOp(callLoc, directFuncOp, cirCallArgs, attrs);
   }
 
-  return builder.createCallOp(callLoc, directFuncOp, cirCallArgs, sideEffect);
+  return op;
 }
 
 const CIRGenFunctionInfo &
@@ -544,8 +573,7 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
 
   assert(!cir::MissingFeatures::opCallCallConv());
   assert(!cir::MissingFeatures::opCallAttrs());
-  cir::SideEffect sideEffect;
-  cgm.constructAttributeList(callee.getAbstractInfo(), sideEffect);
+  cgm.constructAttributeList(callee.getAbstractInfo(), attrs);
 
   assert(!cir::MissingFeatures::invokeOp());
 
@@ -566,12 +594,10 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
     indirectFuncVal = calleePtr->getResult(0);
   }
 
-  assert(!cir::MissingFeatures::opCallAttrs());
-
   mlir::Location callLoc = loc;
   cir::CIRCallOpInterface theCall =
       emitCallLikeOp(*this, loc, indirectFuncTy, indirectFuncVal, directFuncOp,
-                     cirCallArgs, sideEffect);
+                     cirCallArgs, attrs);
 
   if (callOp)
     *callOp = theCall;
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.h b/clang/lib/CIR/CodeGen/CIRGenCall.h
index 56c76c51a46d8..bd113293fdafd 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -33,7 +33,8 @@ class CIRGenCalleeInfo {
   CIRGenCalleeInfo(const clang::FunctionProtoType *calleeProtoTy,
                    clang::GlobalDecl calleeDecl)
       : calleeProtoTy(calleeProtoTy), calleeDecl(calleeDecl) {}
-  CIRGenCalleeInfo(clang::GlobalDecl calleeDecl) : calleeDecl(calleeDecl) {}
+  CIRGenCalleeInfo(clang::GlobalDecl calleeDecl)
+      : calleeProtoTy(nullptr), calleeDecl(calleeDecl) {}
 
   const clang::FunctionProtoType *getCalleeFunctionProtoType() const {
     return calleeProtoTy;
diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
index 278cc8931f308..da8166a596d42 100644
--- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -258,6 +258,31 @@ void CIRGenFunction::emitDelegateCXXConstructorCall(
                          /*Delegating=*/true, thisAddr, delegateArgs, loc);
 }
 
+void CIRGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &args) {
+  const auto *assignOp = cast<CXXMethodDecl>(curGD.getDecl());
+  assert(assignOp->isCopyAssignmentOperator() ||
+         assignOp->isMoveAssignmentOperator());
+  const Stmt *rootS = assignOp->getBody();
+  assert(isa<CompoundStmt>(rootS) &&
+         "Body of an implicit assignment operator should be compound stmt.");
+  const auto *rootCS = cast<CompoundStmt>(rootS);
+
+  assert(!cir::MissingFeatures::incrementProfileCounter());
+  assert(!cir::MissingFeatures::runCleanupsScope());
+
+  // Classic codegen uses a special class to attempt to replace member
+  // initializers with memcpy. We could possibly defer that to the
+  // lowering or optimization phases to keep the memory accesses more
+  // explicit. For now, we don't insert memcpy at all, though in some
+  // cases the AST contains a call to memcpy.
+  assert(!cir::MissingFeatures::assignMemcpyizer());
+  for (Stmt *s : rootCS->body())
+    if (emitStmt(s, /*useCurrentScope=*/true).failed())
+      cgm.errorNYI(s->getSourceRange(),
+                   std::string("emitImplicitAssignmentOperatorBody: ") +
+                       s->getStmtClassName());
+}
+
 void CIRGenFunction::emitDelegatingCXXConstructorCall(
     const CXXConstructorDecl *ctor, const FunctionArgList &args) {
   assert(ctor->isDelegatingConstructor());
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index 5c6604d784156..68d7f1f5bca48 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -326,13 +326,62 @@ mlir::Value CIRGenFunction::emitStoreThroughBitfieldLValue(RValue src,
   return {};
 }
 
+RValue CIRGenFunction::emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc) {
+  const CIRGenBitFieldInfo &info = lv.getBitFieldInfo();
+
+  // Get the output type.
+  mlir::Type resLTy = convertType(lv.getType());
+  Address ptr = lv.getBitFieldAddress();
+
+  assert(!cir::MissingFeatures::armComputeVolatileBitfields());
+
+  mlir::Value field = builder.createGetBitfield(
+      getLoc(loc), resLTy, ptr.getPointer(), ptr.getElementType(), info,
+      lv.isVolatile(), false);
+  assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck() && "NYI");
+  return RValue::get(field);
+}
+
+Address CIRGenFunction::getAddrOfBitFieldStorage(LValue base,
+                                                 const FieldDecl *field,
+                                                 mlir::Type fieldType,
+                                                 unsigned index) {
+  mlir::Location loc = getLoc(field->getLocation());
+  cir::PointerType fieldPtr = cir::PointerType::get(fieldType);
+  cir::GetMemberOp sea = getBuilder().createGetMember(
+      loc, fieldPtr, base.getPointer(), field->getName(), index);
+  return Address(sea, CharUnits::One());
+}
+
+LValue CIRGenFunction::emitLValueForBitField(LValue base,
+                                             const FieldDecl *field) {
+  LValueBaseInfo baseInfo = base.getBaseInfo();
+  const CIRGenRecordLayout &layout =
+      cgm.getTypes().getCIRGenRecordLayout(field->getParent());
+  const CIRGenBitFieldInfo &info = layout.getBitFieldInfo(field);
+  assert(!cir::MissingFeatures::armComputeVolatileBitfields());
+  assert(!cir::MissingFeatures::preservedAccessIndexRegion());
+  unsigned idx = layout.getCIRFieldNo(field);
+
+  Address addr = getAddrOfBitFieldStorage(base, field, info.storageType, idx);
+
+  mlir::Location loc = getLoc(field->getLocation());
+  if (addr.getElementType() != info.storageType)
+    addr = builder.createElementBitCast(loc, addr, info.storageType);
+
+  QualType fieldType =
+      field->getType().withCVRQualifiers(base.getVRQualifiers());
+  // TODO(cir): Support TBAA for bit fields.
+  assert(!cir::MissingFeatures::opTBAA());
+  LValueBaseInfo fieldBaseInfo(baseInfo.getAlignmentSource());
+  return LValue::makeBitfield(addr, info, fieldType, fieldBaseInfo);
+}
+
 LValue CIRGenFunction::emitLValueForField(LValue base, const FieldDecl *field) {
   LValueBaseInfo baseInfo = base.getBaseInfo();
 
-  if (field->isBitField()) {
-    cgm.errorNYI(field->getSourceRange(), "emitLValueForField: bitfield");
-    return LValue();
-  }
+  if (field->isBitField())
+    return emitLValueForBitField(base, field);
 
   QualType fieldType = field->getType();
   const RecordDecl *rec = field->getParent();
@@ -460,6 +509,9 @@ RValue CIRGenFunction::emitLoadOfLValue(LValue lv, SourceLocation loc) {
   assert(!lv.getType()->isFunctionType());
   assert(!(lv.getType()->isConstantMatrixType()) && "not implemented");
 
+  if (lv.isBitField())
+    return emitLoadOfBitfieldLValue(lv, loc);
+
   if (lv.isSimple())
     return RValue::get(emitLoadOfScalar(lv, loc));
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 43f2e47eac152..84fad959ebf49 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -39,13 +39,24 @@ class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
   void emitStoreOfComplex(mlir::Location loc, mlir::Value val, LValue lv,
                           bool isInit);
 
+  mlir::Value
+  VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
+  mlir::Value VisitArraySubscriptExpr(Expr *e);
   mlir::Value VisitBinAssign(const BinaryOperator *e);
+  mlir::Value VisitBinComma(const BinaryOperator *e);
   mlir::Value VisitCallExpr(const CallExpr *e);
+  mlir::Value VisitCastExpr(CastExpr *e);
   mlir::Value VisitChooseExpr(ChooseExpr *e);
+  mlir::Value VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *e);
   mlir::Value VisitDeclRefExpr(DeclRefExpr *e);
+  mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *e);
   mlir::Value VisitImplicitCastExpr(ImplicitCastExpr *e);
   mlir::Value VisitInitListExpr(const InitListExpr *e);
   mlir::Value VisitImaginaryLiteral(const ImaginaryLiteral *il);
+  mlir::Value VisitParenExpr(ParenExpr *e);
+  mlir::Value
+  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e);
+  mlir::Value VisitUnaryDeref(const Expr *e);
 };
 } // namespace
 
@@ -76,12 +87,13 @@ LValue ComplexExprEmitter::emitBinAssignLValue(const BinaryOperator *e,
 mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
                                          QualType destTy) {
   switch (ck) {
+  case CK_NoOp:
   case CK_LValueToRValue:
     return Visit(op);
   default:
-    cgf.cgm.errorNYI("ComplexType Cast");
     break;
   }
+  cgf.cgm.errorNYI("ComplexType Cast");
   return {};
 }
 
@@ -118,6 +130,31 @@ void ComplexExprEmitter::emitStoreOfComplex(mlir::Location loc, mlir::Value val,
   builder.createStore(loc, val, destAddr);
 }
 
+mlir::Value ComplexExprEmitter::VisitAbstractConditionalOperator(
+    const AbstractConditionalOperator *e) {
+  mlir::Value condValue = Visit(e->getCond());
+  mlir::Location loc = cgf.getLoc(e->getSourceRange());
+
+  return builder
+      .create<cir::TernaryOp>(
+          loc, condValue,
+          /*thenBuilder=*/
+          [&](mlir::OpBuilder &b, mlir::Location loc) {
+            mlir::Value trueValue = Visit(e->getTrueExpr());
+            b.create<cir::YieldOp>(loc, trueValue);
+          },
+          /*elseBuilder=*/
+          [&](mlir::OpBuilder &b, mlir::Location loc) {
+            mlir::Value falseValue = Visit(e->getFalseExpr());
+            b.create<cir::YieldOp>(loc, falseValue);
+          })
+      .getResult();
+}
+
+mlir::Value ComplexExprEmitter::VisitArraySubscriptExpr(Expr *e) {
+  return emitLoadOfLValue(e);
+}
+
 mlir::Value ComplexExprEmitter::VisitBinAssign(const BinaryOperator *e) {
   mlir::Value value;
   LValue lv = emitBinAssignLValue(e, value);
@@ -134,6 +171,11 @@ mlir::Value ComplexExprEmitter::VisitBinAssign(const BinaryOperator *e) {
   return emitLoadOfLValue(lv, e->getExprLoc());
 }
 
+mlir::Value ComplexExprEmitter::VisitBinComma(const BinaryOperator *e) {
+  cgf.emitIgnoredExpr(e->getLHS());
+  return Visit(e->getRHS());
+}
+
 mlir::Value ComplexExprEmitter::VisitCallExpr(const CallExpr *e) {
   if (e->getCallReturnType(cgf.getContext())->isReferenceType())
     return emitLoadOfLValue(e);
@@ -141,16 +183,43 @@ mlir::Value ComplexExprEmitter::VisitCallExpr(const CallExpr *e) {
   return cgf.emitCallExpr(e).getValue();
 }
 
+mlir::Value ComplexExprEmitter::VisitCastExpr(CastExpr *e) {
+  if (const auto *ece = dyn_cast<ExplicitCastExpr>(e)) {
+    // Bind VLAs in the cast type.
+    if (ece->getType()->isVariablyModifiedType()) {
+      cgf.cgm.errorNYI("VisitCastExpr Bind VLAs in the cast type");
+      return {};
+    }
+  }
+
+  if (e->changesVolatileQualification())
+    return emitLoadOfLValue(e);
+
+  return emitCast(e->getCastKind(), e->getSubExpr(), e->getType());
+}
+
 mlir::Value ComplexExprEmitter::VisitChooseExpr(ChooseExpr *e) {
   return Visit(e->getChosenSubExpr());
 }
 
+mlir::Value
+ComplexExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *e) {
+  mlir::Location loc = cgf.getLoc(e->getExprLoc());
+  mlir::Type complexTy = cgf.convertType(e->getType());
+  return builder.getNullValue(complexTy, loc);
+}
+
 mlir::Value ComplexExprEmitter::VisitDeclRefExpr(DeclRefExpr *e) {
   if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(e))
     return emitConstant(constant, e);
   return emitLoadOfLValue(e);
 }
 
+mlir::Value
+ComplexExprEmitter::VisitGenericSelectionExpr(GenericSelectionExpr *e) {
+  return Visit(e->getResultExpr());
+}
+
 mlir::Value ComplexExprEmitter::VisitImplicitCastExpr(ImplicitCastExpr *e) {
   // Unlike for scalars, we don't have to worry about function->ptr demotion
   // here.
@@ -209,6 +278,19 @@ ComplexExprEmitter::VisitImaginaryLiteral(const ImaginaryLiteral *il) {
   return builder.create<cir::ConstantOp>(loc, complexAttr);
 }
 
+mlir::Value ComplexExprEmitter::VisitParenExpr(ParenExpr *e) {
+  return Visit(e->getSubExpr());
+}
+
+mlir::Value ComplexExprEmitter::VisitSubstNonTypeTemplateParmExpr(
+    SubstNonTypeTemplateParmExpr *e) {
+  return Visit(e->getReplacement());
+}
+
+mlir::Value ComplexExprEmitter::VisitUnaryDeref(const Expr *e) {
+  return emitLoadOfLValue(e);
+}
+
 LValue CIRGenFunction::emitComplexAssignmentLValue(const BinaryOperator *e) {
   assert(e->getOpcode() == BO_Assign && "Expected assign op");
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index e25819cdb11bf..5b3bf85cefbb0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -684,7 +684,7 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
     if (mlir::isa<cir::BoolType>(ty))
       return builder.getCIRBoolAttr(value.getInt().getZExtValue());
     assert(mlir::isa<cir::IntType>(ty) && "expected integral type");
-    return cgm.getBuilder().getAttr<cir::IntAttr>(ty, value.getInt());
+    return cir::IntAttr::get(ty, value.getInt());
   }
   case APValue::Float: {
     const llvm::APFloat &init = value.getFloat();
@@ -698,7 +698,7 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
     mlir::Type ty = cgm.convertType(destType);
     assert(mlir::isa<cir::FPTypeInterface>(ty) &&
            "expected floating-point type");
-    return cgm.getBuilder().getAttr<cir::FPAttr>(ty, init);
+    return cir::FPAttr::get(ty, init);
   }
   case APValue::Array: {
     const ArrayType *arrayTy = cgm.getASTContext().getAsArrayType(destType);
@@ -789,8 +789,8 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
       llvm::APSInt real = value.getComplexIntReal();
       llvm::APSInt imag = value.getComplexIntImag();
       return builder.getAttr<cir::ConstComplexAttr>(
-          complexType, builder.getAttr<cir::IntAttr>(complexElemTy, real),
-          builder.getAttr<cir::IntAttr>(complexElemTy, imag));
+          complexType, cir::IntAttr::get(complexElemTy, real),
+          cir::IntAttr::get(complexElemTy, imag));
     }
 
     assert(isa<cir::FPTypeInterface>(complexElemTy) &&
@@ -798,8 +798,8 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
     llvm::APFloat real = value.getComplexFloatReal();
     llvm::APFloat imag = value.getComplexFloatImag();
     return builder.getAttr<cir::ConstComplexAttr>(
-        complexType, builder.getAttr<cir::FPAttr>(complexElemTy, real),
-        builder.getAttr<cir::FPAttr>(complexElemTy, imag));
+        complexType, cir::FPAttr::get(complexElemTy, real),
+        cir::FPAttr::get(complexElemTy, imag));
   }
   case APValue::FixedPoint:
   case APValue::AddrLabelDiff:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 955bb5ffc4395..10c485c85e22a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -125,8 +125,16 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     return {};
   }
 
+  mlir::Value VisitPackIndexingExpr(PackIndexingExpr *e) {
+    return Visit(e->getSelectedExpr());
+  }
+
   mlir::Value VisitParenExpr(ParenExpr *pe) { return Visit(pe->getSubExpr()); }
 
+  mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
+    return Visit(ge->getResultExpr());
+  }
+
   /// Emits the address of the l-value, then loads and returns the result.
   mlir::Value emitLoadOfLValue(const Expr *e) {
     LValue lv = cgf.emitLValue(e);
@@ -149,8 +157,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
   mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) {
     mlir::Type type = cgf.convertType(e->getType());
     return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()),
-        builder.getAttr<cir::IntAttr>(type, e->getValue()));
+        cgf.getLoc(e->getExprLoc()), cir::IntAttr::get(type, e->getValue()));
   }
 
   mlir::Value VisitFloatingLiteral(const FloatingLiteral *e) {
@@ -158,8 +165,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     assert(mlir::isa<cir::FPTypeInterface>(type) &&
            "expect floating-point type");
     return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()),
-        builder.getAttr<cir::FPAttr>(type, e->getValue()));
+        cgf.getLoc(e->getExprLoc()), cir::FPAttr::get(type, e->getValue()));
   }
 
   mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) {
@@ -365,6 +371,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     return builder.create<cir::CastOp>(src.getLoc(), fullDstTy, *castKind, src);
   }
 
+  mlir::Value
+  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
+    return Visit(e->getReplacement());
+  }
+
   mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
   mlir::Value
   VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
@@ -901,14 +912,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
       assert(e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE);
 
       BinOpInfo boInfo = emitBinOps(e);
-      if (e->getOpcode() == BO_EQ) {
-        result =
-            builder.create<cir::ComplexEqualOp>(loc, boInfo.lhs, boInfo.rhs);
-      } else {
-        assert(!cir::MissingFeatures::complexType());
-        cgf.cgm.errorNYI(loc, "complex not equal");
-        result = builder.getBool(false, loc);
-      }
+      result = builder.create<cir::CmpOp>(loc, kind, boInfo.lhs, boInfo.rhs);
     }
 
     return emitScalarConversion(result, cgf.getContext().BoolTy, e->getType(),
@@ -1962,21 +1966,21 @@ mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
                                      "sizeof operator for VariableArrayType",
                                      e->getStmtClassName());
       return builder.getConstant(
-          loc, builder.getAttr<cir::IntAttr>(
-                   cgf.cgm.UInt64Ty, llvm::APSInt(llvm::APInt(64, 1), true)));
+          loc, cir::IntAttr::get(cgf.cgm.UInt64Ty,
+                                 llvm::APSInt(llvm::APInt(64, 1), true)));
     }
   } else if (e->getKind() == UETT_OpenMPRequiredSimdAlign) {
     cgf.getCIRGenModule().errorNYI(
         e->getSourceRange(), "sizeof operator for OpenMpRequiredSimdAlign",
         e->getStmtClassName());
     return builder.getConstant(
-        loc, builder.getAttr<cir::IntAttr>(
-                 cgf.cgm.UInt64Ty, llvm::APSInt(llvm::APInt(64, 1), true)));
+        loc, cir::IntAttr::get(cgf.cgm.UInt64Ty,
+                               llvm::APSInt(llvm::APInt(64, 1), true)));
   }
 
   return builder.getConstant(
-      loc, builder.getAttr<cir::IntAttr>(
-               cgf.cgm.UInt64Ty, e->EvaluateKnownConstInt(cgf.getContext())));
+      loc, cir::IntAttr::get(cgf.cgm.UInt64Ty,
+                             e->EvaluateKnownConstInt(cgf.getContext())));
 }
 
 /// Return true if the specified expression is cheap enough and side-effect-free
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index c029853929a58..c4efabd6b12ab 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -462,21 +462,23 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
 
     startFunction(gd, retTy, fn, funcType, args, loc, bodyRange.getBegin());
 
-    if (isa<CXXDestructorDecl>(funcDecl))
+    if (isa<CXXDestructorDecl>(funcDecl)) {
       getCIRGenModule().errorNYI(bodyRange, "C++ destructor definition");
-    else if (isa<CXXConstructorDecl>(funcDecl))
+    } else if (isa<CXXConstructorDecl>(funcDecl)) {
       emitConstructorBody(args);
-    else if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice &&
-             funcDecl->hasAttr<CUDAGlobalAttr>())
+    } else if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice &&
+               funcDecl->hasAttr<CUDAGlobalAttr>()) {
       getCIRGenModule().errorNYI(bodyRange, "CUDA kernel");
-    else if (isa<CXXMethodDecl>(funcDecl) &&
-             cast<CXXMethodDecl>(funcDecl)->isLambdaStaticInvoker())
+    } else if (isa<CXXMethodDecl>(funcDecl) &&
+               cast<CXXMethodDecl>(funcDecl)->isLambdaStaticInvoker()) {
       getCIRGenModule().errorNYI(bodyRange, "Lambda static invoker");
-    else if (funcDecl->isDefaulted() && isa<CXXMethodDecl>(funcDecl) &&
-             (cast<CXXMethodDecl>(funcDecl)->isCopyAssignmentOperator() ||
-              cast<CXXMethodDecl>(funcDecl)->isMoveAssignmentOperator()))
-      getCIRGenModule().errorNYI(bodyRange, "Default assignment operator");
-    else if (body) {
+    } else if (funcDecl->isDefaulted() && isa<CXXMethodDecl>(funcDecl) &&
+               (cast<CXXMethodDecl>(funcDecl)->isCopyAssignmentOperator() ||
+                cast<CXXMethodDecl>(funcDecl)->isMoveAssignmentOperator())) {
+      // Implicit copy-assignment gets the same special treatment as implicit
+      // copy-constructors.
+      emitImplicitAssignmentOperatorBody(args);
+    } else if (body) {
       if (mlir::failed(emitFunctionBody(body))) {
         fn.erase();
         return nullptr;
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 7e6fdf130cca1..76353bae68e21 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -550,6 +550,9 @@ class CIRGenFunction : public CIRGenTypeCache {
     return it->second;
   }
 
+  Address getAddrOfBitFieldStorage(LValue base, const clang::FieldDecl *field,
+                                   mlir::Type fieldType, unsigned index);
+
   /// Load the value for 'this'. This function is only valid while generating
   /// code for an C++ member function.
   /// FIXME(cir): this should return a mlir::Value!
@@ -878,6 +881,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitFunctionBody(const clang::Stmt *body);
 
+  void emitImplicitAssignmentOperatorBody(FunctionArgList &args);
+
   void emitInitializerForField(clang::FieldDecl *field, LValue lhs,
                                clang::Expr *init);
 
@@ -894,9 +899,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   // Build CIR for a statement. useCurrentScope should be true if no
   // new scopes need be created when finding a compound statement.
-  mlir::LogicalResult
-  emitStmt(const clang::Stmt *s, bool useCurrentScope,
-           llvm::ArrayRef<const Attr *> attrs = std::nullopt);
+  mlir::LogicalResult emitStmt(const clang::Stmt *s, bool useCurrentScope,
+                               llvm::ArrayRef<const Attr *> attrs = {});
 
   mlir::LogicalResult emitSimpleStmt(const clang::Stmt *s,
                                      bool useCurrentScope);
@@ -955,6 +959,8 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// ignoring the result.
   void emitIgnoredExpr(const clang::Expr *e);
 
+  RValue emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc);
+
   /// Given an expression that represents a value lvalue, this method emits
   /// the address of the lvalue, then loads the result as an rvalue,
   /// returning the rvalue.
@@ -975,6 +981,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// of the expression.
   /// FIXME: document this function better.
   LValue emitLValue(const clang::Expr *e);
+  LValue emitLValueForBitField(LValue base, const FieldDecl *field);
   LValue emitLValueForField(LValue base, const clang::FieldDecl *field);
 
   /// Like emitLValueForField, excpet that if the Field is a reference, this
@@ -1197,7 +1204,41 @@ class CIRGenFunction : public CIRGenTypeCache {
   void updateLoopOpParallelism(mlir::acc::LoopOp &op, bool isOrphan,
                                OpenACCDirectiveKind dk);
 
+  // The OpenACC 'cache' construct actually applies to the 'loop' if present. So
+  // keep track of the 'loop' so that we can add the cache vars to it correctly.
+  mlir::acc::LoopOp *activeLoopOp = nullptr;
+
+  struct ActiveOpenACCLoopRAII {
+    CIRGenFunction &cgf;
+    mlir::acc::LoopOp *oldLoopOp;
+
+    ActiveOpenACCLoopRAII(CIRGenFunction &cgf, mlir::acc::LoopOp *newOp)
+        : cgf(cgf), oldLoopOp(cgf.activeLoopOp) {
+      cgf.activeLoopOp = newOp;
+    }
+    ~ActiveOpenACCLoopRAII() { cgf.activeLoopOp = oldLoopOp; }
+  };
+
 public:
+  // Helper type used to store the list of important information for a 'data'
+  // clause variable, or a 'cache' variable reference.
+  struct OpenACCDataOperandInfo {
+    mlir::Location beginLoc;
+    mlir::Value varValue;
+    std::string name;
+    llvm::SmallVector<mlir::Value> bounds;
+  };
+  // Gets the collection of info required to lower and OpenACC clause or cache
+  // construct variable reference.
+  OpenACCDataOperandInfo getOpenACCDataOperandInfo(const Expr *e);
+  // Helper function to emit the integer expressions as required by an OpenACC
+  // clause/construct.
+  mlir::Value emitOpenACCIntExpr(const Expr *intExpr);
+  // Helper function to emit an integer constant as an mlir int type, used for
+  // constants in OpenACC constructs/clauses.
+  mlir::Value createOpenACCConstantInt(mlir::Location loc, unsigned width,
+                                       int64_t value);
+
   mlir::LogicalResult
   emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s);
   mlir::LogicalResult emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 0f38d9aee5e72..c1434ee697f4c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -104,6 +104,12 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
 
   theModule->setAttr(cir::CIRDialect::getTripleAttrName(),
                      builder.getStringAttr(getTriple().str()));
+
+  if (cgo.OptimizationLevel > 0 || cgo.OptimizeSize > 0)
+    theModule->setAttr(cir::CIRDialect::getOptInfoAttrName(),
+                       cir::OptInfoAttr::get(&mlirContext,
+                                             cgo.OptimizationLevel,
+                                             cgo.OptimizeSize));
 }
 
 CIRGenModule::~CIRGenModule() = default;
@@ -393,12 +399,6 @@ void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
 void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
                                                 mlir::Operation *op) {
   auto const *funcDecl = cast<FunctionDecl>(gd.getDecl());
-  if (funcDecl->getIdentifier() == nullptr) {
-    errorNYI(funcDecl->getSourceRange().getBegin(),
-             "function definition with a non-identifier for a name");
-    return;
-  }
-
   const CIRGenFunctionInfo &fi = getTypes().arrangeGlobalDeclaration(gd);
   cir::FuncType funcType = getTypes().getFunctionType(fi);
   cir::FuncOp funcOp = dyn_cast_if_present<cir::FuncOp>(op);
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 5a1eb9dea35c6..16922b115027e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -166,7 +166,7 @@ class CIRGenModule : public CIRGenTypeCache {
   /// constructed for. If valid, the attributes applied to this decl may
   /// contribute to the function attributes and calling convention.
   void constructAttributeList(CIRGenCalleeInfo calleeInfo,
-                              cir::SideEffect &sideEffect);
+                              mlir::NamedAttrList &attrs);
 
   /// Return a constant array for the given string.
   mlir::Attribute getConstantArrayFromStringLiteral(const StringLiteral *e);
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp
new file mode 100644
index 0000000000000..49ff1249827a4
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp
@@ -0,0 +1,131 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic OpenACC lowering functions not Stmt, Decl, or clause specific.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
+#include "clang/AST/ExprCXX.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+mlir::Value createBound(CIRGenFunction &cgf, CIRGen::CIRGenBuilderTy &builder,
+                        mlir::Location boundLoc, mlir::Value lowerBound,
+                        mlir::Value upperBound, mlir::Value extent) {
+  // Arrays always have a start-idx of 0.
+  mlir::Value startIdx = cgf.createOpenACCConstantInt(boundLoc, 64, 0);
+  // Stride is always 1 in C/C++.
+  mlir::Value stride = cgf.createOpenACCConstantInt(boundLoc, 64, 1);
+
+  auto bound =
+      builder.create<mlir::acc::DataBoundsOp>(boundLoc, lowerBound, upperBound);
+  bound.getStartIdxMutable().assign(startIdx);
+  if (extent)
+    bound.getExtentMutable().assign(extent);
+  bound.getStrideMutable().assign(stride);
+
+  return bound;
+}
+} // namespace
+
+mlir::Value CIRGenFunction::emitOpenACCIntExpr(const Expr *intExpr) {
+  mlir::Value expr = emitScalarExpr(intExpr);
+  mlir::Location exprLoc = cgm.getLoc(intExpr->getBeginLoc());
+
+  mlir::IntegerType targetType = mlir::IntegerType::get(
+      &getMLIRContext(), getContext().getIntWidth(intExpr->getType()),
+      intExpr->getType()->isSignedIntegerOrEnumerationType()
+          ? mlir::IntegerType::SignednessSemantics::Signed
+          : mlir::IntegerType::SignednessSemantics::Unsigned);
+
+  auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
+      exprLoc, targetType, expr);
+  return conversionOp.getResult(0);
+}
+
+mlir::Value CIRGenFunction::createOpenACCConstantInt(mlir::Location loc,
+                                                     unsigned width,
+                                                     int64_t value) {
+  mlir::IntegerType ty =
+      mlir::IntegerType::get(&getMLIRContext(), width,
+                             mlir::IntegerType::SignednessSemantics::Signless);
+  auto constOp = builder.create<mlir::arith::ConstantOp>(
+      loc, builder.getIntegerAttr(ty, value));
+
+  return constOp.getResult();
+}
+
+CIRGenFunction::OpenACCDataOperandInfo
+CIRGenFunction::getOpenACCDataOperandInfo(const Expr *e) {
+  const Expr *curVarExpr = e->IgnoreParenImpCasts();
+
+  mlir::Location exprLoc = cgm.getLoc(curVarExpr->getBeginLoc());
+  llvm::SmallVector<mlir::Value> bounds;
+
+  std::string exprString;
+  llvm::raw_string_ostream os(exprString);
+  e->printPretty(os, nullptr, getContext().getPrintingPolicy());
+
+  while (isa<ArraySectionExpr, ArraySubscriptExpr>(curVarExpr)) {
+    mlir::Location boundLoc = cgm.getLoc(curVarExpr->getBeginLoc());
+    mlir::Value lowerBound;
+    mlir::Value upperBound;
+    mlir::Value extent;
+
+    if (const auto *section = dyn_cast<ArraySectionExpr>(curVarExpr)) {
+      if (const Expr *lb = section->getLowerBound())
+        lowerBound = emitOpenACCIntExpr(lb);
+      else
+        lowerBound = createOpenACCConstantInt(boundLoc, 64, 0);
+
+      if (const Expr *len = section->getLength()) {
+        extent = emitOpenACCIntExpr(len);
+      } else {
+        QualType baseTy = ArraySectionExpr::getBaseOriginalType(
+            section->getBase()->IgnoreParenImpCasts());
+        // We know this is the case as implicit lengths are only allowed for
+        // array types with a constant size, or a dependent size.  AND since
+        // we are codegen we know we're not dependent.
+        auto *arrayTy = getContext().getAsConstantArrayType(baseTy);
+        // Rather than trying to calculate the extent based on the
+        // lower-bound, we can just emit this as an upper bound.
+        upperBound = createOpenACCConstantInt(boundLoc, 64,
+                                              arrayTy->getLimitedSize() - 1);
+      }
+
+      curVarExpr = section->getBase()->IgnoreParenImpCasts();
+    } else {
+      const auto *subscript = cast<ArraySubscriptExpr>(curVarExpr);
+
+      lowerBound = emitOpenACCIntExpr(subscript->getIdx());
+      // Length of an array index is always 1.
+      extent = createOpenACCConstantInt(boundLoc, 64, 1);
+      curVarExpr = subscript->getBase()->IgnoreParenImpCasts();
+    }
+
+    bounds.push_back(createBound(*this, this->builder, boundLoc, lowerBound,
+                                 upperBound, extent));
+  }
+
+  if (const auto *memExpr = dyn_cast<MemberExpr>(curVarExpr))
+    return {exprLoc, emitMemberExpr(memExpr).getPointer(), exprString,
+            std::move(bounds)};
+
+  // Sema has made sure that only 4 types of things can get here, array
+  // subscript, array section, member expr, or DRE to a var decl (or the
+  // former 3 wrapping a var-decl), so we should be able to assume this is
+  // right.
+  const auto *dre = cast<DeclRefExpr>(curVarExpr);
+  return {exprLoc, emitDeclRefLValue(dre).getPointer(), exprString,
+          std::move(bounds)};
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
index b17994ee8771e..e45d3b8f4aa82 100644
--- a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
@@ -75,26 +75,12 @@ class OpenACCClauseCIREmitter final
   void setLastDeviceTypeClause(const OpenACCDeviceTypeClause &clause) {
     lastDeviceTypeValues.clear();
 
-    llvm::for_each(clause.getArchitectures(),
-                   [this](const DeviceTypeArgument &arg) {
-                     lastDeviceTypeValues.push_back(
-                         decodeDeviceType(arg.getIdentifierInfo()));
-                   });
+    for (const DeviceTypeArgument &arg : clause.getArchitectures())
+      lastDeviceTypeValues.push_back(decodeDeviceType(arg.getIdentifierInfo()));
   }
 
   mlir::Value emitIntExpr(const Expr *intExpr) {
-    mlir::Value expr = cgf.emitScalarExpr(intExpr);
-    mlir::Location exprLoc = cgf.cgm.getLoc(intExpr->getBeginLoc());
-
-    mlir::IntegerType targetType = mlir::IntegerType::get(
-        &cgf.getMLIRContext(), cgf.getContext().getIntWidth(intExpr->getType()),
-        intExpr->getType()->isSignedIntegerOrEnumerationType()
-            ? mlir::IntegerType::SignednessSemantics::Signed
-            : mlir::IntegerType::SignednessSemantics::Unsigned);
-
-    auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
-        exprLoc, targetType, expr);
-    return conversionOp.getResult(0);
+    return cgf.emitOpenACCIntExpr(intExpr);
   }
 
   // 'condition' as an OpenACC grammar production is used for 'if' and (some
@@ -114,6 +100,7 @@ class OpenACCClauseCIREmitter final
 
   mlir::Value createConstantInt(mlir::Location loc, unsigned width,
                                 int64_t value) {
+    return cgf.createOpenACCConstantInt(loc, width, value);
     mlir::IntegerType ty = mlir::IntegerType::get(
         &cgf.getMLIRContext(), width,
         mlir::IntegerType::SignednessSemantics::Signless);
@@ -187,105 +174,6 @@ class OpenACCClauseCIREmitter final
     dataOperands.append(computeEmitter.dataOperands);
   }
 
-  struct DataOperandInfo {
-    mlir::Location beginLoc;
-    mlir::Value varValue;
-    std::string name;
-    llvm::SmallVector<mlir::Value> bounds;
-  };
-
-  mlir::Value createBound(mlir::Location boundLoc, mlir::Value lowerBound,
-                          mlir::Value upperBound, mlir::Value extent) {
-    // Arrays always have a start-idx of 0.
-    mlir::Value startIdx = createConstantInt(boundLoc, 64, 0);
-    // Stride is always 1 in C/C++.
-    mlir::Value stride = createConstantInt(boundLoc, 64, 1);
-
-    auto bound = builder.create<mlir::acc::DataBoundsOp>(boundLoc, lowerBound,
-                                                         upperBound);
-    bound.getStartIdxMutable().assign(startIdx);
-    if (extent)
-      bound.getExtentMutable().assign(extent);
-    bound.getStrideMutable().assign(stride);
-
-    return bound;
-  }
-
-  // A helper function that gets the information from an operand to a data
-  // clause, so that it can be used to emit the data operations.
-  DataOperandInfo getDataOperandInfo(OpenACCDirectiveKind dk, const Expr *e) {
-    // TODO: OpenACC: Cache was different enough as to need a separate
-    // `ActOnCacheVar`, so we are going to need to do some investigations here
-    // when it comes to implement this for cache.
-    if (dk == OpenACCDirectiveKind::Cache) {
-      cgf.cgm.errorNYI(e->getSourceRange(),
-                       "OpenACC data operand for 'cache' directive");
-      return {cgf.cgm.getLoc(e->getBeginLoc()), {}, {}, {}};
-    }
-
-    const Expr *curVarExpr = e->IgnoreParenImpCasts();
-
-    mlir::Location exprLoc = cgf.cgm.getLoc(curVarExpr->getBeginLoc());
-    llvm::SmallVector<mlir::Value> bounds;
-
-    std::string exprString;
-    llvm::raw_string_ostream os(exprString);
-    e->printPretty(os, nullptr, cgf.getContext().getPrintingPolicy());
-
-    // Assemble the list of bounds.
-    while (isa<ArraySectionExpr, ArraySubscriptExpr>(curVarExpr)) {
-      mlir::Location boundLoc = cgf.cgm.getLoc(curVarExpr->getBeginLoc());
-      mlir::Value lowerBound;
-      mlir::Value upperBound;
-      mlir::Value extent;
-
-      if (const auto *section = dyn_cast<ArraySectionExpr>(curVarExpr)) {
-        if (const Expr *lb = section->getLowerBound())
-          lowerBound = emitIntExpr(lb);
-        else
-          lowerBound = createConstantInt(boundLoc, 64, 0);
-
-        if (const Expr *len = section->getLength()) {
-          extent = emitIntExpr(len);
-        } else {
-          QualType baseTy = ArraySectionExpr::getBaseOriginalType(
-              section->getBase()->IgnoreParenImpCasts());
-          // We know this is the case as implicit lengths are only allowed for
-          // array types with a constant size, or a dependent size.  AND since
-          // we are codegen we know we're not dependent.
-          auto *arrayTy = cgf.getContext().getAsConstantArrayType(baseTy);
-          // Rather than trying to calculate the extent based on the
-          // lower-bound, we can just emit this as an upper bound.
-          upperBound =
-              createConstantInt(boundLoc, 64, arrayTy->getLimitedSize() - 1);
-        }
-
-        curVarExpr = section->getBase()->IgnoreParenImpCasts();
-      } else {
-        const auto *subscript = cast<ArraySubscriptExpr>(curVarExpr);
-
-        lowerBound = emitIntExpr(subscript->getIdx());
-        // Length of an array index is always 1.
-        extent = createConstantInt(boundLoc, 64, 1);
-        curVarExpr = subscript->getBase()->IgnoreParenImpCasts();
-      }
-
-      bounds.push_back(createBound(boundLoc, lowerBound, upperBound, extent));
-    }
-
-    if (const auto *memExpr = dyn_cast<MemberExpr>(curVarExpr))
-      return {exprLoc, cgf.emitMemberExpr(memExpr).getPointer(), exprString,
-              std::move(bounds)};
-
-    // Sema has made sure that only 4 types of things can get here, array
-    // subscript, array section, member expr, or DRE to a var decl (or the
-    // former 3 wrapping a var-decl), so we should be able to assume this is
-    // right.
-    const auto *dre = cast<DeclRefExpr>(curVarExpr);
-    return {exprLoc, cgf.emitDeclRefLValue(dre).getPointer(), exprString,
-            std::move(bounds)};
-  }
-
   mlir::acc::DataClauseModifier
   convertModifiers(OpenACCModifierKind modifiers) {
     using namespace mlir::acc;
@@ -317,7 +205,8 @@ class OpenACCClauseCIREmitter final
   void addDataOperand(const Expr *varOperand, mlir::acc::DataClause dataClause,
                       OpenACCModifierKind modifiers, bool structured,
                       bool implicit) {
-    DataOperandInfo opInfo = getDataOperandInfo(dirKind, varOperand);
+    CIRGenFunction::OpenACCDataOperandInfo opInfo =
+        cgf.getOpenACCDataOperandInfo(varOperand);
 
     auto beforeOp =
         builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured,
@@ -358,7 +247,8 @@ class OpenACCClauseCIREmitter final
   void addDataOperand(const Expr *varOperand, mlir::acc::DataClause dataClause,
                       OpenACCModifierKind modifiers, bool structured,
                       bool implicit) {
-    DataOperandInfo opInfo = getDataOperandInfo(dirKind, varOperand);
+    CIRGenFunction::OpenACCDataOperandInfo opInfo =
+        cgf.getOpenACCDataOperandInfo(varOperand);
     auto beforeOp =
         builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured,
                                    implicit, opInfo.name, opInfo.bounds);
@@ -376,10 +266,21 @@ class OpenACCClauseCIREmitter final
   // on all operation types.
   mlir::ArrayAttr getAsyncOnlyAttr() {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>)
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::UpdateOp>) {
       return operation.getAsyncOnlyAttr();
-    else if constexpr (isCombinedType<OpTy>)
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp,
+                                      mlir::acc::ExitDataOp>) {
+      if (!operation.getAsyncAttr())
+        return mlir::ArrayAttr{};
+
+      llvm::SmallVector<mlir::Attribute> devTysTemp;
+      devTysTemp.push_back(mlir::acc::DeviceTypeAttr::get(
+          builder.getContext(), mlir::acc::DeviceType::None));
+      return mlir::ArrayAttr::get(builder.getContext(), devTysTemp);
+    } else if constexpr (isCombinedType<OpTy>) {
       return operation.computeOp.getAsyncOnlyAttr();
+    }
 
     // Note: 'wait' has async as well, but it cannot have data clauses, so we
     // don't have to handle them here.
@@ -391,10 +292,21 @@ class OpenACCClauseCIREmitter final
   // on all operation types.
   mlir::ArrayAttr getAsyncOperandsDeviceTypeAttr() {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>)
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::UpdateOp>) {
       return operation.getAsyncOperandsDeviceTypeAttr();
-    else if constexpr (isCombinedType<OpTy>)
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp,
+                                      mlir::acc::ExitDataOp>) {
+      if (!operation.getAsyncOperand())
+        return mlir::ArrayAttr{};
+
+      llvm::SmallVector<mlir::Attribute> devTysTemp;
+      devTysTemp.push_back(mlir::acc::DeviceTypeAttr::get(
+          builder.getContext(), mlir::acc::DeviceType::None));
+      return mlir::ArrayAttr::get(builder.getContext(), devTysTemp);
+    } else if constexpr (isCombinedType<OpTy>) {
       return operation.computeOp.getAsyncOperandsDeviceTypeAttr();
+    }
 
     // Note: 'wait' has async as well, but it cannot have data clauses, so we
     // don't have to handle them here.
@@ -407,8 +319,12 @@ class OpenACCClauseCIREmitter final
   // on all operation types.
   mlir::OperandRange getAsyncOperands() {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>)
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::UpdateOp>)
       return operation.getAsyncOperands();
+    else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp,
+                                    mlir::acc::ExitDataOp>)
+      return operation.getAsyncOperandMutable();
     else if constexpr (isCombinedType<OpTy>)
       return operation.computeOp.getAsyncOperands();
 
@@ -485,11 +401,9 @@ class OpenACCClauseCIREmitter final
 
     if constexpr (isOneOfTypes<OpTy, mlir::acc::InitOp,
                                mlir::acc::ShutdownOp>) {
-      llvm::for_each(
-          clause.getArchitectures(), [this](const DeviceTypeArgument &arg) {
-            operation.addDeviceType(builder.getContext(),
-                                    decodeDeviceType(arg.getIdentifierInfo()));
-          });
+      for (const DeviceTypeArgument &arg : clause.getArchitectures())
+        operation.addDeviceType(builder.getContext(),
+                                decodeDeviceType(arg.getIdentifierInfo()));
     } else if constexpr (isOneOfTypes<OpTy, mlir::acc::SetOp>) {
       assert(!operation.getDeviceTypeAttr() && "already have device-type?");
       assert(clause.getArchitectures().size() <= 1);
@@ -499,7 +413,8 @@ class OpenACCClauseCIREmitter final
             decodeDeviceType(clause.getArchitectures()[0].getIdentifierInfo()));
     } else if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp,
                                       mlir::acc::SerialOp, mlir::acc::KernelsOp,
-                                      mlir::acc::DataOp, mlir::acc::LoopOp>) {
+                                      mlir::acc::DataOp, mlir::acc::LoopOp,
+                                      mlir::acc::UpdateOp>) {
       // Nothing to do here, these constructs don't have any IR for these, as
       // they just modify the other clauses IR.  So setting of
       // `lastDeviceTypeValues` (done above) is all we need.
@@ -508,7 +423,7 @@ class OpenACCClauseCIREmitter final
       // 'lastDeviceTypeValues' to set the value for the child visitor.
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. update, data, routine constructs remain.
+      // unreachable. routine construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -542,10 +457,12 @@ class OpenACCClauseCIREmitter final
   void VisitAsyncClause(const OpenACCAsyncClause &clause) {
     hasAsyncClause = true;
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
-      if (!clause.hasIntExpr())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::EnterDataOp, mlir::acc::ExitDataOp,
+                               mlir::acc::UpdateOp>) {
+      if (!clause.hasIntExpr()) {
         operation.addAsyncOnly(builder.getContext(), lastDeviceTypeValues);
-      else {
+      } else {
 
         mlir::Value intExpr;
         {
@@ -572,8 +489,7 @@ class OpenACCClauseCIREmitter final
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. Combined constructs remain. Data, enter data, exit data,
-      // update constructs remain.
+      // unreachable. Combined constructs remain. update construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -590,12 +506,39 @@ class OpenACCClauseCIREmitter final
       } else {
         llvm_unreachable("var-list version of self shouldn't get here");
       }
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      assert(!clause.isEmptySelfClause() && !clause.isConditionExprClause() &&
+             "var-list version of self required for update");
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::UpdateHostOp>(
+            var, mlir::acc::DataClause::acc_update_self, {},
+            /*structured=*/false, /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. update construct remains.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitSelfClause");
+    }
+  }
+
+  void VisitHostClause(const OpenACCHostClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::UpdateHostOp>(
+            var, mlir::acc::DataClause::acc_update_host, {},
+            /*structured=*/false, /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitHostClause");
+    }
+  }
+
+  void VisitDeviceClause(const OpenACCDeviceClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::UpdateDeviceOp>(
+            var, mlir::acc::DataClause::acc_update_device, {},
+            /*structured=*/false, /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitDeviceClause");
     }
   }
 
@@ -604,28 +547,21 @@ class OpenACCClauseCIREmitter final
                                mlir::acc::KernelsOp, mlir::acc::InitOp,
                                mlir::acc::ShutdownOp, mlir::acc::SetOp,
                                mlir::acc::DataOp, mlir::acc::WaitOp,
-                               mlir::acc::HostDataOp>) {
+                               mlir::acc::HostDataOp, mlir::acc::EnterDataOp,
+                               mlir::acc::ExitDataOp, mlir::acc::UpdateOp>) {
       operation.getIfCondMutable().append(
           createCondition(clause.getConditionExpr()));
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // 'if' applies to most of the constructs, but hold off on lowering them
-      // until we can write tests/know what we're doing with codegen to make
-      // sure we get it right.
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. Enter data, exit data, host_data, update constructs
-      // remain.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitIfClause");
     }
   }
 
   void VisitIfPresentClause(const OpenACCIfPresentClause &clause) {
-    if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp>) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp,
+                               mlir::acc::UpdateOp>) {
       operation.setIfPresent(true);
-    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
-      // Last unimplemented one here, so just put it in this way instead.
-      return clauseNotImplemented(clause);
     } else {
       llvm_unreachable("unknown construct kind in VisitIfPresentClause");
     }
@@ -659,7 +595,9 @@ class OpenACCClauseCIREmitter final
 
   void VisitWaitClause(const OpenACCWaitClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::EnterDataOp, mlir::acc::ExitDataOp,
+                               mlir::acc::UpdateOp>) {
       if (!clause.hasExprs()) {
         operation.addWaitOnly(builder.getContext(), lastDeviceTypeValues);
       } else {
@@ -675,7 +613,7 @@ class OpenACCClauseCIREmitter final
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. Enter data, exit data, update constructs remain.
+      // unreachable. update construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -866,11 +804,16 @@ class OpenACCClauseCIREmitter final
             var, mlir::acc::DataClause::acc_copyin, clause.getModifierList(),
             /*structured=*/true,
             /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CopyinOp>(
+            var, mlir::acc::DataClause::acc_copyin, clause.getModifierList(),
+            /*structured=*/false, /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. enter-data, declare constructs remain.
+      // unreachable. declare construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -883,11 +826,17 @@ class OpenACCClauseCIREmitter final
             var, mlir::acc::DataClause::acc_copyout, clause.getModifierList(),
             /*structured=*/true,
             /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::CopyoutOp>(
+            var, mlir::acc::DataClause::acc_copyout, clause.getModifierList(),
+            /*structured=*/false,
+            /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. exit data, declare constructs remain.
+      // unreachable. declare construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -900,15 +849,52 @@ class OpenACCClauseCIREmitter final
             var, mlir::acc::DataClause::acc_create, clause.getModifierList(),
             /*structured=*/true,
             /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CreateOp>(
+            var, mlir::acc::DataClause::acc_create, clause.getModifierList(),
+            /*structured=*/false, /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. enter-data, declare constructs remain.
+      // unreachable. declare construct remains.
       return clauseNotImplemented(clause);
     }
   }
 
+  void VisitDeleteClause(const OpenACCDeleteClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_delete, {},
+            /*structured=*/false,
+            /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitDeleteClause");
+    }
+  }
+
+  void VisitDetachClause(const OpenACCDetachClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::DetachOp>(
+            var, mlir::acc::DataClause::acc_detach, {},
+            /*structured=*/false,
+            /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitDetachClause");
+    }
+  }
+
+  void VisitFinalizeClause(const OpenACCFinalizeClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      operation.setFinalize(true);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitFinalizeClause");
+    }
+  }
+
   void VisitUseDeviceClause(const OpenACCUseDeviceClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp>) {
       for (const Expr *var : clause.getVarList())
@@ -974,12 +960,15 @@ class OpenACCClauseCIREmitter final
         addDataOperand<mlir::acc::AttachOp, mlir::acc::DetachOp>(
             var, mlir::acc::DataClause::acc_attach, {}, /*structured=*/true,
             /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::AttachOp>(
+            var, mlir::acc::DataClause::acc_attach, {},
+            /*structured=*/false, /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. enter data remains.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitAttachClause");
     }
   }
 };
@@ -1018,6 +1007,9 @@ EXPL_SPEC(mlir::acc::ShutdownOp)
 EXPL_SPEC(mlir::acc::SetOp)
 EXPL_SPEC(mlir::acc::WaitOp)
 EXPL_SPEC(mlir::acc::HostDataOp)
+EXPL_SPEC(mlir::acc::EnterDataOp)
+EXPL_SPEC(mlir::acc::ExitDataOp)
+EXPL_SPEC(mlir::acc::UpdateOp)
 #undef EXPL_SPEC
 
 template <typename ComputeOp, typename LoopOp>
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index 1feefa55eb270..e89393c92db33 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -95,6 +95,8 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct(
       builder.setInsertionPointToEnd(&innerBlock);
 
       LexicalScope ls{*this, start, builder.getInsertionBlock()};
+      ActiveOpenACCLoopRAII activeLoop{*this, &loopOp};
+
       res = emitStmt(loopStmt, /*useCurrentScope=*/true);
 
       builder.create<mlir::acc::YieldOp>(end);
@@ -250,26 +252,60 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct(
 
 mlir::LogicalResult CIRGenFunction::emitOpenACCEnterDataConstruct(
     const OpenACCEnterDataConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC EnterData Construct");
-  return mlir::failure();
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  emitOpenACCOp<EnterDataOp>(start, s.getDirectiveKind(), s.getDirectiveLoc(),
+                             s.clauses());
+  return mlir::success();
 }
+
 mlir::LogicalResult CIRGenFunction::emitOpenACCExitDataConstruct(
     const OpenACCExitDataConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC ExitData Construct");
-  return mlir::failure();
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  emitOpenACCOp<ExitDataOp>(start, s.getDirectiveKind(), s.getDirectiveLoc(),
+                            s.clauses());
+  return mlir::success();
 }
+
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC Update Construct");
-  return mlir::failure();
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  emitOpenACCOp<UpdateOp>(start, s.getDirectiveKind(), s.getDirectiveLoc(),
+                          s.clauses());
+  return mlir::success();
 }
+
 mlir::LogicalResult
-CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC Atomic Construct");
-  return mlir::failure();
+CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) {
+  // The 'cache' directive 'may' be at the top of a loop by standard, but
+  // doesn't have to be. Additionally, there is nothing that requires this be a
+  // loop affected by an OpenACC pragma. Sema doesn't do any level of
+  // enforcement here, since it isn't particularly valuable to do so thanks to
+  // that. Instead, we treat cache as a 'noop' if there is no acc.loop to apply
+  // it to.
+  if (!activeLoopOp)
+    return mlir::success();
+
+  mlir::acc::LoopOp loopOp = *activeLoopOp;
+
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPoint(loopOp);
+
+  for (const Expr *var : s.getVarList()) {
+    CIRGenFunction::OpenACCDataOperandInfo opInfo =
+        getOpenACCDataOperandInfo(var);
+
+    auto cacheOp = builder.create<CacheOp>(
+        opInfo.beginLoc, opInfo.varValue,
+        /*structured=*/false, /*implicit=*/false, opInfo.name, opInfo.bounds);
+
+    loopOp.getCacheOperandsMutable().append(cacheOp.getResult());
+  }
+
+  return mlir::success();
 }
+
 mlir::LogicalResult
-CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC Cache Construct");
+CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) {
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Atomic Construct");
   return mlir::failure();
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
index 71f3ccb8e040e..f3911ae8b6772 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
@@ -130,6 +130,7 @@ CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) {
     mlir::OpBuilder::InsertionGuard guardCase(builder);
     builder.setInsertionPointToEnd(&block);
     LexicalScope ls{*this, start, builder.getInsertionBlock()};
+    ActiveOpenACCLoopRAII activeLoop{*this, &op};
 
     stmtRes = emitStmt(s.getLoop(), /*useCurrentScope=*/true);
     builder.create<mlir::acc::YieldOp>(end);
diff --git a/clang/lib/CIR/CodeGen/CIRGenValue.h b/clang/lib/CIR/CodeGen/CIRGenValue.h
index a5a457ddafa9c..e1b0f805a7b21 100644
--- a/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -19,6 +19,7 @@
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Type.h"
 
+#include "CIRGenRecordLayout.h"
 #include "mlir/IR/Value.h"
 
 #include "clang/CIR/MissingFeatures.h"
@@ -162,6 +163,7 @@ class LValue {
   mlir::Value vectorIdx; // Index for vector subscript
   mlir::Type elementType;
   LValueBaseInfo baseInfo;
+  const CIRGenBitFieldInfo *bitFieldInfo{nullptr};
 
   void initialize(clang::QualType type, clang::Qualifiers quals,
                   clang::CharUnits alignment, LValueBaseInfo baseInfo) {
@@ -245,6 +247,38 @@ class LValue {
     r.initialize(t, t.getQualifiers(), vecAddress.getAlignment(), baseInfo);
     return r;
   }
+
+  // bitfield lvalue
+  Address getBitFieldAddress() const {
+    return Address(getBitFieldPointer(), elementType, getAlignment());
+  }
+
+  mlir::Value getBitFieldPointer() const {
+    assert(isBitField());
+    return v;
+  }
+
+  const CIRGenBitFieldInfo &getBitFieldInfo() const {
+    assert(isBitField());
+    return *bitFieldInfo;
+  }
+
+  /// Create a new object to represent a bit-field access.
+  ///
+  /// \param Addr - The base address of the bit-field sequence this
+  /// bit-field refers to.
+  /// \param Info - The information describing how to perform the bit-field
+  /// access.
+  static LValue makeBitfield(Address addr, const CIRGenBitFieldInfo &info,
+                             clang::QualType type, LValueBaseInfo baseInfo) {
+    LValue r;
+    r.lvType = BitField;
+    r.v = addr.getPointer();
+    r.elementType = addr.getElementType();
+    r.bitFieldInfo = &info;
+    r.initialize(type, type.getQualifiers(), addr.getAlignment(), baseInfo);
+    return r;
+  }
 };
 
 /// An aggregate value slot.
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 385bea066c61c..03ea60c76c87d 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -25,6 +25,7 @@ add_clang_library(clangCIR
   CIRGenFunction.cpp
   CIRGenItaniumCXXABI.cpp
   CIRGenModule.cpp
+  CIRGenOpenACC.cpp
   CIRGenOpenACCClause.cpp
   CIRGenRecordLayoutBuilder.cpp
   CIRGenStmt.cpp
diff --git a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
index 29a9a815c31a1..c039bdcd7f6a1 100644
--- a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
@@ -15,6 +15,19 @@
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
+//===-----------------------------------------------------------------===//
+// IntLiteral
+//===-----------------------------------------------------------------===//
+
+static void printIntLiteral(mlir::AsmPrinter &p, llvm::APInt value,
+                            cir::IntTypeInterface ty);
+static mlir::ParseResult parseIntLiteral(mlir::AsmParser &parser,
+                                         llvm::APInt &value,
+                                         cir::IntTypeInterface ty);
+//===-----------------------------------------------------------------===//
+// FloatLiteral
+//===-----------------------------------------------------------------===//
+
 static void printFloatLiteral(mlir::AsmPrinter &p, llvm::APFloat value,
                               mlir::Type ty);
 static mlir::ParseResult
@@ -55,6 +68,21 @@ void CIRDialect::printAttribute(Attribute attr, DialectAsmPrinter &os) const {
     llvm_unreachable("unexpected CIR type kind");
 }
 
+//===----------------------------------------------------------------------===//
+// OptInfoAttr definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult OptInfoAttr::verify(function_ref<InFlightDiagnostic()> emitError,
+                                  unsigned level, unsigned size) {
+  if (level > 3)
+    return emitError()
+           << "optimization level must be between 0 and 3 inclusive";
+  if (size > 2)
+    return emitError()
+           << "size optimization level must be between 0 and 2 inclusive";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // ConstPtrAttr definitions
 //===----------------------------------------------------------------------===//
@@ -82,69 +110,52 @@ static void printConstPtr(AsmPrinter &p, mlir::IntegerAttr value) {
 // IntAttr definitions
 //===----------------------------------------------------------------------===//
 
-Attribute IntAttr::parse(AsmParser &parser, Type odsType) {
-  mlir::APInt apValue;
-
-  if (!mlir::isa<IntType>(odsType))
-    return {};
-  auto type = mlir::cast<IntType>(odsType);
-
-  // Consume the '<' symbol.
-  if (parser.parseLess())
-    return {};
-
-  // Fetch arbitrary precision integer value.
-  if (type.isSigned()) {
-    int64_t value = 0;
-    if (parser.parseInteger(value)) {
-      parser.emitError(parser.getCurrentLocation(), "expected integer value");
-    } else {
-      apValue = mlir::APInt(type.getWidth(), value, type.isSigned(),
-                            /*implicitTrunc=*/true);
-      if (apValue.getSExtValue() != value)
-        parser.emitError(parser.getCurrentLocation(),
-                         "integer value too large for the given type");
-    }
+template <typename IntT>
+static bool isTooLargeForType(const mlir::APInt &value, IntT expectedValue) {
+  if constexpr (std::is_signed_v<IntT>) {
+    return value.getSExtValue() != expectedValue;
   } else {
-    uint64_t value = 0;
-    if (parser.parseInteger(value)) {
-      parser.emitError(parser.getCurrentLocation(), "expected integer value");
-    } else {
-      apValue = mlir::APInt(type.getWidth(), value, type.isSigned(),
-                            /*implicitTrunc=*/true);
-      if (apValue.getZExtValue() != value)
-        parser.emitError(parser.getCurrentLocation(),
-                         "integer value too large for the given type");
-    }
+    return value.getZExtValue() != expectedValue;
   }
+}
 
-  // Consume the '>' symbol.
-  if (parser.parseGreater())
-    return {};
+template <typename IntT>
+static mlir::ParseResult parseIntLiteralImpl(mlir::AsmParser &p,
+                                             llvm::APInt &value,
+                                             cir::IntTypeInterface ty) {
+  IntT ivalue;
+  const bool isSigned = ty.isSigned();
+  if (p.parseInteger(ivalue))
+    return p.emitError(p.getCurrentLocation(), "expected integer value");
+
+  value = mlir::APInt(ty.getWidth(), ivalue, isSigned, /*implicitTrunc=*/true);
+  if (isTooLargeForType(value, ivalue))
+    return p.emitError(p.getCurrentLocation(),
+                       "integer value too large for the given type");
 
-  return IntAttr::get(type, apValue);
+  return success();
 }
 
-void IntAttr::print(AsmPrinter &printer) const {
-  auto type = mlir::cast<IntType>(getType());
-  printer << '<';
-  if (type.isSigned())
-    printer << getSInt();
+mlir::ParseResult parseIntLiteral(mlir::AsmParser &parser, llvm::APInt &value,
+                                  cir::IntTypeInterface ty) {
+  if (ty.isSigned())
+    return parseIntLiteralImpl<int64_t>(parser, value, ty);
+  return parseIntLiteralImpl<uint64_t>(parser, value, ty);
+}
+
+void printIntLiteral(mlir::AsmPrinter &p, llvm::APInt value,
+                     cir::IntTypeInterface ty) {
+  if (ty.isSigned())
+    p << value.getSExtValue();
   else
-    printer << getUInt();
-  printer << '>';
+    p << value.getZExtValue();
 }
 
 LogicalResult IntAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, APInt value) {
-  if (!mlir::isa<IntType>(type))
-    return emitError() << "expected 'simple.int' type";
-
-  auto intType = mlir::cast<IntType>(type);
-  if (value.getBitWidth() != intType.getWidth())
+                              cir::IntTypeInterface type, llvm::APInt value) {
+  if (value.getBitWidth() != type.getWidth())
     return emitError() << "type and value bitwidth mismatch: "
-                       << intType.getWidth() << " != " << value.getBitWidth();
-
+                       << type.getWidth() << " != " << value.getBitWidth();
   return success();
 }
 
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 2d5dcf69c489f..8512b229c2663 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -65,6 +65,10 @@ struct CIROpAsmDialectInterface : public OpAsmDialectInterface {
       os << (boolAttr.getValue() ? "true" : "false");
       return AliasResult::FinalAlias;
     }
+    if (auto bitfield = mlir::dyn_cast<cir::BitfieldInfoAttr>(attr)) {
+      os << "bfi_" << bitfield.getName().str();
+      return AliasResult::FinalAlias;
+    }
     return AliasResult::NoAlias;
   }
 };
@@ -593,7 +597,9 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
   llvm::ArrayRef<mlir::Type> allResultTypes;
 
   // If we cannot parse a string callee, it means this is an indirect call.
-  if (!parser.parseOptionalAttribute(calleeAttr, "callee", result.attributes)
+  if (!parser
+           .parseOptionalAttribute(calleeAttr, CIRDialect::getCalleeAttrName(),
+                                   result.attributes)
            .has_value()) {
     OpAsmParser::UnresolvedOperand indirectVal;
     // Do not resolve right now, since we need to figure out the type
@@ -611,6 +617,10 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
   if (parser.parseRParen())
     return mlir::failure();
 
+  if (parser.parseOptionalKeyword("nothrow").succeeded())
+    result.addAttribute(CIRDialect::getNoThrowAttrName(),
+                        mlir::UnitAttr::get(parser.getContext()));
+
   if (parser.parseOptionalKeyword("side_effect").succeeded()) {
     if (parser.parseLParen().failed())
       return failure();
@@ -620,7 +630,7 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
     if (parser.parseRParen().failed())
       return failure();
     auto attr = cir::SideEffectAttr::get(parser.getContext(), sideEffect);
-    result.addAttribute("side_effect", attr);
+    result.addAttribute(CIRDialect::getSideEffectAttrName(), attr);
   }
 
   if (parser.parseOptionalAttrDict(result.attributes))
@@ -645,7 +655,7 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
 static void printCallCommon(mlir::Operation *op,
                             mlir::FlatSymbolRefAttr calleeSym,
                             mlir::Value indirectCallee,
-                            mlir::OpAsmPrinter &printer,
+                            mlir::OpAsmPrinter &printer, bool isNothrow,
                             cir::SideEffect sideEffect) {
   printer << ' ';
 
@@ -662,13 +672,19 @@ static void printCallCommon(mlir::Operation *op,
   }
   printer << "(" << ops << ")";
 
+  if (isNothrow)
+    printer << " nothrow";
+
   if (sideEffect != cir::SideEffect::All) {
     printer << " side_effect(";
     printer << stringifySideEffect(sideEffect);
     printer << ")";
   }
 
-  printer.printOptionalAttrDict(op->getAttrs(), {"callee", "side_effect"});
+  printer.printOptionalAttrDict(op->getAttrs(),
+                                {CIRDialect::getCalleeAttrName(),
+                                 CIRDialect::getNoThrowAttrName(),
+                                 CIRDialect::getSideEffectAttrName()});
 
   printer << " : ";
   printer.printFunctionalType(op->getOperands().getTypes(),
@@ -683,13 +699,15 @@ mlir::ParseResult cir::CallOp::parse(mlir::OpAsmParser &parser,
 void cir::CallOp::print(mlir::OpAsmPrinter &p) {
   mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr;
   cir::SideEffect sideEffect = getSideEffect();
-  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, sideEffect);
+  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, getNothrow(),
+                  sideEffect);
 }
 
 static LogicalResult
 verifyCallCommInSymbolUses(mlir::Operation *op,
                            SymbolTableCollection &symbolTable) {
-  auto fnAttr = op->getAttrOfType<FlatSymbolRefAttr>("callee");
+  auto fnAttr =
+      op->getAttrOfType<FlatSymbolRefAttr>(CIRDialect::getCalleeAttrName());
   if (!fnAttr) {
     // This is an indirect call, thus we don't have to check the symbol uses.
     return mlir::success();
diff --git a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
index c6760cf1618cb..40da5e60a93f9 100644
--- a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
@@ -96,6 +96,8 @@ Type RecordType::parse(mlir::AsmParser &parser) {
   FailureOr<AsmParser::CyclicParseReset> cyclicParseGuard;
   const llvm::SMLoc loc = parser.getCurrentLocation();
   const mlir::Location eLoc = parser.getEncodedSourceLoc(loc);
+  bool packed = false;
+  bool padded = false;
   RecordKind kind;
   mlir::MLIRContext *context = parser.getContext();
 
@@ -138,6 +140,12 @@ Type RecordType::parse(mlir::AsmParser &parser) {
     }
   }
 
+  if (parser.parseOptionalKeyword("packed").succeeded())
+    packed = true;
+
+  if (parser.parseOptionalKeyword("padded").succeeded())
+    padded = true;
+
   // Parse record members or lack thereof.
   bool incomplete = true;
   llvm::SmallVector<mlir::Type> members;
@@ -159,8 +167,15 @@ Type RecordType::parse(mlir::AsmParser &parser) {
   mlir::Type type = {};
   if (name && incomplete) { // Identified & incomplete
     type = getChecked(eLoc, context, name, kind);
-  } else if (!incomplete) { // complete
-    parser.emitError(loc, "complete records are not yet supported");
+  } else if (!name && !incomplete) { // Anonymous & complete
+    type = getChecked(eLoc, context, membersRef, packed, padded, kind);
+  } else if (!incomplete) { // Identified & complete
+    type = getChecked(eLoc, context, membersRef, name, packed, padded, kind);
+    // If the record has a self-reference, its type already exists in a
+    // incomplete state. In this case, we must complete it.
+    if (mlir::cast<RecordType>(type).isIncomplete())
+      mlir::cast<RecordType>(type).complete(membersRef, packed, padded);
+    assert(!cir::MissingFeatures::astRecordDeclAttr());
   } else { // anonymous & incomplete
     parser.emitError(loc, "anonymous records must be complete");
     return {};
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 1034b8780c03c..af307f6ad673d 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -232,7 +232,7 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,
   return value;
 }
 
-void convertSideEffectForCall(mlir::Operation *callOp,
+void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
                               cir::SideEffect sideEffect,
                               mlir::LLVM::MemoryEffectsAttr &memoryEffect,
                               bool &noUnwind, bool &willReturn) {
@@ -241,7 +241,7 @@ void convertSideEffectForCall(mlir::Operation *callOp,
   switch (sideEffect) {
   case cir::SideEffect::All:
     memoryEffect = {};
-    noUnwind = false;
+    noUnwind = isNothrow;
     willReturn = false;
     break;
 
@@ -460,6 +460,81 @@ mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
+    cir::BitClrsbOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto zero = rewriter.create<mlir::LLVM::ConstantOp>(
+      op.getLoc(), adaptor.getInput().getType(), 0);
+  auto isNeg = rewriter.create<mlir::LLVM::ICmpOp>(
+      op.getLoc(),
+      mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
+                                         mlir::LLVM::ICmpPredicate::slt),
+      adaptor.getInput(), zero);
+
+  auto negOne = rewriter.create<mlir::LLVM::ConstantOp>(
+      op.getLoc(), adaptor.getInput().getType(), -1);
+  auto flipped = rewriter.create<mlir::LLVM::XOrOp>(op.getLoc(),
+                                                    adaptor.getInput(), negOne);
+
+  auto select = rewriter.create<mlir::LLVM::SelectOp>(
+      op.getLoc(), isNeg, flipped, adaptor.getInput());
+
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto clz = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
+      op.getLoc(), resTy, select, /*is_zero_poison=*/false);
+
+  auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
+  auto res = rewriter.create<mlir::LLVM::SubOp>(op.getLoc(), clz, one);
+  rewriter.replaceOp(op, res);
+
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite(
+    cir::BitClzOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto llvmOp = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
+      op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
+  rewriter.replaceOp(op, llvmOp);
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite(
+    cir::BitCtzOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto llvmOp = rewriter.create<mlir::LLVM::CountTrailingZerosOp>(
+      op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
+  rewriter.replaceOp(op, llvmOp);
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(
+    cir::BitParityOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto popcnt = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
+                                                     adaptor.getInput());
+
+  auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
+  auto popcntMod2 =
+      rewriter.create<mlir::LLVM::AndOp>(op.getLoc(), popcnt, one);
+  rewriter.replaceOp(op, popcntMod2);
+
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite(
+    cir::BitPopcountOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto llvmOp = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
+                                                     adaptor.getInput());
+  rewriter.replaceOp(op, llvmOp);
+  return mlir::LogicalResult::success();
+}
+
 mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
     cir::BrCondOp brOp, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -800,8 +875,8 @@ rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
   mlir::LLVM::MemoryEffectsAttr memoryEffects;
   bool noUnwind = false;
   bool willReturn = false;
-  convertSideEffectForCall(op, call.getSideEffect(), memoryEffects, noUnwind,
-                           willReturn);
+  convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(),
+                           memoryEffects, noUnwind, willReturn);
 
   mlir::LLVM::LLVMFunctionType llvmFnTy;
   if (calleeAttr) { // direct call
@@ -969,10 +1044,19 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
                                                getTypeConverter()));
     return mlir::success();
   } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {
-    auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
     mlir::Type complexElemTy = complexTy.getElementType();
     mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);
 
+    if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) {
+      mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy);
+      mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr});
+      rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
+          op, getTypeConverter()->convertType(op.getType()), array);
+      return mlir::success();
+    }
+
+    auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
+
     mlir::Attribute components[2];
     if (mlir::isa<cir::IntType>(complexElemTy)) {
       components[0] = rewriter.getIntegerAttr(
@@ -1586,7 +1670,6 @@ mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
   assert(!cir::MissingFeatures::dataMemberType());
   assert(!cir::MissingFeatures::methodType());
 
-  // Lower to LLVM comparison op.
   if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {
     bool isSigned = mlir::isa<cir::IntType>(type)
                         ? mlir::cast<cir::IntType>(type).isSigned()
@@ -1595,22 +1678,82 @@ mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
         convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
-  } else if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
+    return mlir::success();
+  }
+
+  if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
     mlir::LLVM::ICmpPredicate kind =
         convertCmpKindToICmpPredicate(cmpOp.getKind(),
                                       /* isSigned=*/false);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
-  } else if (mlir::isa<cir::FPTypeInterface>(type)) {
+    return mlir::success();
+  }
+
+  if (mlir::isa<cir::FPTypeInterface>(type)) {
     mlir::LLVM::FCmpPredicate kind =
         convertCmpKindToFCmpPredicate(cmpOp.getKind());
     rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
         cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
-  } else {
-    return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
+    return mlir::success();
   }
 
-  return mlir::success();
+  if (mlir::isa<cir::ComplexType>(type)) {
+    mlir::Value lhs = adaptor.getLhs();
+    mlir::Value rhs = adaptor.getRhs();
+    mlir::Location loc = cmpOp.getLoc();
+
+    auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());
+    mlir::Type complexElemTy =
+        getTypeConverter()->convertType(complexType.getElementType());
+
+    auto lhsReal =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
+    auto lhsImag =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
+    auto rhsReal =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
+    auto rhsImag =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
+
+    if (cmpOp.getKind() == cir::CmpOpKind::eq) {
+      if (complexElemTy.isInteger()) {
+        auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
+        auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
+        rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
+        return mlir::success();
+      }
+
+      auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
+      auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
+      rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
+      return mlir::success();
+    }
+
+    if (cmpOp.getKind() == cir::CmpOpKind::ne) {
+      if (complexElemTy.isInteger()) {
+        auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);
+        auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);
+        rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
+        return mlir::success();
+      }
+
+      auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);
+      auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);
+      rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
+      return mlir::success();
+    }
+  }
+
+  return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
 }
 
 mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(
@@ -1896,17 +2039,22 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMAssumeOpLowering,
                CIRToLLVMBaseClassAddrOpLowering,
                CIRToLLVMBinOpLowering,
+               CIRToLLVMBitClrsbOpLowering,
+               CIRToLLVMBitClzOpLowering,
+               CIRToLLVMBitCtzOpLowering,
+               CIRToLLVMBitParityOpLowering,
+               CIRToLLVMBitPopcountOpLowering,
                CIRToLLVMBrCondOpLowering,
                CIRToLLVMBrOpLowering,
                CIRToLLVMCallOpLowering,
                CIRToLLVMCmpOpLowering,
                CIRToLLVMComplexCreateOpLowering,
-               CIRToLLVMComplexEqualOpLowering,
                CIRToLLVMComplexImagOpLowering,
                CIRToLLVMComplexRealOpLowering,
                CIRToLLVMConstantOpLowering,
                CIRToLLVMExpectOpLowering,
                CIRToLLVMFuncOpLowering,
+               CIRToLLVMGetBitfieldOpLowering,
                CIRToLLVMGetGlobalOpLowering,
                CIRToLLVMGetMemberOpLowering,
                CIRToLLVMSelectOpLowering,
@@ -2245,40 +2393,56 @@ mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
-mlir::LogicalResult CIRToLLVMComplexEqualOpLowering::matchAndRewrite(
-    cir::ComplexEqualOp op, OpAdaptor adaptor,
+mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
+    cir::GetBitfieldOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
-  mlir::Value lhs = adaptor.getLhs();
-  mlir::Value rhs = adaptor.getRhs();
 
-  auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
-  mlir::Type complexElemTy =
-      getTypeConverter()->convertType(complexType.getElementType());
+  mlir::OpBuilder::InsertionGuard guard(rewriter);
+  rewriter.setInsertionPoint(op);
 
-  mlir::Location loc = op.getLoc();
-  auto lhsReal =
-      rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
-  auto lhsImag =
-      rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
-  auto rhsReal =
-      rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
-  auto rhsImag =
-      rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
-
-  if (complexElemTy.isInteger()) {
-    auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
-        loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
-    auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
-        loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
-    rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, realCmp, imagCmp);
-    return mlir::success();
+  cir::BitfieldInfoAttr info = op.getBitfieldInfo();
+  uint64_t size = info.getSize();
+  uint64_t offset = info.getOffset();
+  mlir::Type storageType = info.getStorageType();
+  mlir::MLIRContext *context = storageType.getContext();
+  unsigned storageSize = 0;
+
+  mlir::IntegerType intType =
+      TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
+          .Case<cir::ArrayType>([&](cir::ArrayType atTy) {
+            storageSize = atTy.getSize() * 8;
+            return mlir::IntegerType::get(context, storageSize);
+          })
+          .Case<cir::IntType>([&](cir::IntType intTy) {
+            storageSize = intTy.getWidth();
+            return mlir::IntegerType::get(context, storageSize);
+          })
+          .Default([](mlir::Type) -> mlir::IntegerType {
+            llvm_unreachable(
+                "Either ArrayType or IntType expected for bitfields storage");
+          });
+
+  mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>(
+      op.getLoc(), intType, adaptor.getAddr(), 0, op.getIsVolatile());
+  val = rewriter.create<mlir::LLVM::BitcastOp>(op.getLoc(), intType, val);
+
+  if (info.getIsSigned()) {
+    assert(static_cast<unsigned>(offset + size) <= storageSize);
+    unsigned highBits = storageSize - offset - size;
+    val = createShL(rewriter, val, highBits);
+    val = createAShR(rewriter, val, offset + highBits);
+  } else {
+    val = createLShR(rewriter, val, offset);
+
+    if (static_cast<unsigned>(offset) + size < storageSize)
+      val = createAnd(rewriter, val,
+                      llvm::APInt::getLowBitsSet(storageSize, size));
   }
 
-  auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
-      loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
-  auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
-      loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
-  rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, realCmp, imagCmp);
+  mlir::Type resTy = getTypeConverter()->convertType(op.getType());
+  mlir::Value newOp = createIntCast(
+      rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());
+  rewriter.replaceOp(op, newOp);
   return mlir::success();
 }
 
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index 25cf218cf8b6c..d9fb91066317b 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -29,7 +29,7 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, mlir::Attribute attr,
 
 mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage);
 
-void convertSideEffectForCall(mlir::Operation *callOp,
+void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
                               cir::SideEffect sideEffect,
                               mlir::LLVM::MemoryEffectsAttr &memoryEffect,
                               bool &noUnwind, bool &willReturn);
@@ -44,6 +44,56 @@ class CIRToLLVMAssumeOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMBitClrsbOpLowering
+    : public mlir::OpConversionPattern<cir::BitClrsbOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitClrsbOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitClrsbOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitClzOpLowering
+    : public mlir::OpConversionPattern<cir::BitClzOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitClzOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitClzOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitCtzOpLowering
+    : public mlir::OpConversionPattern<cir::BitCtzOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitCtzOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitCtzOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitParityOpLowering
+    : public mlir::OpConversionPattern<cir::BitParityOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitParityOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitParityOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitPopcountOpLowering
+    : public mlir::OpConversionPattern<cir::BitPopcountOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitPopcountOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitPopcountOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMBrCondOpLowering
     : public mlir::OpConversionPattern<cir::BrCondOp> {
 public:
@@ -463,13 +513,13 @@ class CIRToLLVMComplexImagOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
-class CIRToLLVMComplexEqualOpLowering
-    : public mlir::OpConversionPattern<cir::ComplexEqualOp> {
+class CIRToLLVMGetBitfieldOpLowering
+    : public mlir::OpConversionPattern<cir::GetBitfieldOp> {
 public:
-  using mlir::OpConversionPattern<cir::ComplexEqualOp>::OpConversionPattern;
+  using mlir::OpConversionPattern<cir::GetBitfieldOp>::OpConversionPattern;
 
   mlir::LogicalResult
-  matchAndRewrite(cir::ComplexEqualOp op, OpAdaptor,
+  matchAndRewrite(cir::GetBitfieldOp op, OpAdaptor,
                   mlir::ConversionPatternRewriter &) const override;
 };
 
diff --git a/clang/lib/CIR/Lowering/LoweringHelpers.cpp b/clang/lib/CIR/Lowering/LoweringHelpers.cpp
index da3fe4412ec2a..d5f132431aba2 100644
--- a/clang/lib/CIR/Lowering/LoweringHelpers.cpp
+++ b/clang/lib/CIR/Lowering/LoweringHelpers.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/CIR/LoweringHelpers.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "clang/CIR/MissingFeatures.h"
 
 mlir::DenseElementsAttr
@@ -144,3 +145,40 @@ lowerConstArrayAttr(cir::ConstArrayAttr constArr,
 
   return std::nullopt;
 }
+
+mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc,
+                          mlir::Type typ, const llvm::APInt &val) {
+  return bld.create<mlir::LLVM::ConstantOp>(loc, typ, val);
+}
+
+mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ,
+                     unsigned val) {
+  return bld.create<mlir::LLVM::ConstantOp>(loc, typ, val);
+}
+
+mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
+  if (!rhs)
+    return lhs;
+  mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::ShlOp>(lhs.getLoc(), lhs, rhsVal);
+}
+
+mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
+  if (!rhs)
+    return lhs;
+  mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::AShrOp>(lhs.getLoc(), lhs, rhsVal);
+}
+
+mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs,
+                      const llvm::APInt &rhs) {
+  mlir::Value rhsVal = getConstAPInt(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::AndOp>(lhs.getLoc(), lhs, rhsVal);
+}
+
+mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
+  if (!rhs)
+    return lhs;
+  mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::LShrOp>(lhs.getLoc(), lhs, rhsVal);
+}
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 97bc063ad34e5..2f6d4c414e737 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Frontend/Driver/CodeGenOptions.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
@@ -805,17 +806,21 @@ static void addSanitizers(const Triple &TargetTriple,
   // SanitizeSkipHotCutoffs: doubles with range [0, 1]
   // Opts.cutoffs: unsigned ints with range [0, 1000000]
   auto ScaledCutoffs = CodeGenOpts.SanitizeSkipHotCutoffs.getAllScaled(1000000);
-
+  uint64_t AllowRuntimeCheckSkipHotCutoff =
+      CodeGenOpts.AllowRuntimeCheckSkipHotCutoff.value_or(0.0) * 1000000;
   // TODO: remove IsRequested()
-  if (LowerAllowCheckPass::IsRequested() || ScaledCutoffs.has_value()) {
+  if (LowerAllowCheckPass::IsRequested() || ScaledCutoffs.has_value() ||
+      CodeGenOpts.AllowRuntimeCheckSkipHotCutoff.has_value()) {
     // We want to call it after inline, which is about OptimizerEarlyEPCallback.
     PB.registerOptimizerEarlyEPCallback(
-        [ScaledCutoffs](ModulePassManager &MPM, OptimizationLevel Level,
-                        ThinOrFullLTOPhase Phase) {
+        [ScaledCutoffs, AllowRuntimeCheckSkipHotCutoff](
+            ModulePassManager &MPM, OptimizationLevel Level,
+            ThinOrFullLTOPhase Phase) {
           LowerAllowCheckPass::Options Opts;
           // TODO: after removing IsRequested(), make this unconditional
           if (ScaledCutoffs.has_value())
             Opts.cutoffs = ScaledCutoffs.value();
+          Opts.runtime_check = AllowRuntimeCheckSkipHotCutoff;
           MPM.addPass(
               createModuleToFunctionPassAdaptor(LowerAllowCheckPass(Opts)));
         });
@@ -957,7 +962,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
           CodeGenOpts.DIBugsReportFilePath);
     Debugify.registerCallbacks(PIC, MAM);
 
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
     // If we're using debug location coverage tracking, mark all the
     // instructions coming out of the frontend without a DebugLoc as being
     // compiler-generated, to prevent both those instructions and new
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 13792c1042046..4a465e6526da0 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -627,6 +627,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
                                   CXXCtorInitializer *MemberInit,
                                   const CXXConstructorDecl *Constructor,
                                   FunctionArgList &Args) {
+  ApplyAtomGroup Grp(CGF.getDebugInfo());
   ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
   assert(MemberInit->isAnyMemberInitializer() &&
          "Must have member initializer!");
@@ -1000,7 +1001,8 @@ namespace {
     void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
       DestPtr = DestPtr.withElementType(CGF.Int8Ty);
       SrcPtr = SrcPtr.withElementType(CGF.Int8Ty);
-      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
+      auto *I = CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
+      CGF.addInstToCurrentSourceAtom(I, nullptr);
     }
 
     void addInitialField(FieldDecl *F) {
@@ -1113,6 +1115,7 @@ namespace {
       }
 
       pushEHDestructors();
+      ApplyAtomGroup Grp(CGF.getDebugInfo());
       emitMemcpy();
       AggregatedInits.clear();
     }
@@ -1248,6 +1251,7 @@ namespace {
         reset();
       }
 
+      ApplyAtomGroup Grp(CGF.getDebugInfo());
       emitMemcpy();
       AggregatedStmts.clear();
     }
@@ -1338,9 +1342,9 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
     assert(!Member->isBaseInitializer());
     assert(Member->isAnyMemberInitializer() &&
            "Delegating initializer on non-delegating constructor");
-    ApplyAtomGroup Grp(getDebugInfo());
     CM.addMemberInitializer(Member);
   }
+
   CM.finish();
 }
 
@@ -1563,6 +1567,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args)
   AssignmentMemcpyizer AM(*this, AssignOp, Args);
   for (auto *I : RootCS->body())
     AM.emitAssignment(I);
+
   AM.finish();
 }
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 7ab0e2fdaa731..8bf7d24d8551d 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1904,7 +1904,9 @@ CGDebugInfo::createInlinedSubprogram(StringRef FuncName,
         /*ScopeLine=*/0,
         /*Flags=*/llvm::DINode::FlagArtificial,
         /*SPFlags=*/llvm::DISubprogram::SPFlagDefinition,
-        /*TParams=*/nullptr, /*ThrownTypes=*/nullptr, /*Annotations=*/nullptr);
+        /*TParams=*/nullptr, /*Decl=*/nullptr, /*ThrownTypes=*/nullptr,
+        /*Annotations=*/nullptr, /*TargetFuncName=*/StringRef(),
+        /*UseKeyInstructions=*/CGM.getCodeGenOpts().DebugKeyInstructions);
   }
 
   return SP;
@@ -2106,8 +2108,17 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
   return getOrCreateInstanceMethodType(ThisType, Func, Unit);
 }
 
-llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
-    QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile *Unit) {
+llvm::DISubroutineType *CGDebugInfo::getOrCreateMethodTypeForDestructor(
+    const CXXMethodDecl *Method, llvm::DIFile *Unit, QualType FNType) {
+  const FunctionProtoType *Func = FNType->getAs<FunctionProtoType>();
+  // skip the first param since it is also this
+  return getOrCreateInstanceMethodType(Method->getThisType(), Func, Unit, true);
+}
+
+llvm::DISubroutineType *
+CGDebugInfo::getOrCreateInstanceMethodType(QualType ThisPtr,
+                                           const FunctionProtoType *Func,
+                                           llvm::DIFile *Unit, bool SkipFirst) {
   FunctionProtoType::ExtProtoInfo EPI = Func->getExtProtoInfo();
   Qualifiers &Qc = EPI.TypeQuals;
   Qc.removeConst();
@@ -2147,7 +2158,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
   }
 
   // Copy rest of the arguments.
-  for (unsigned i = 1, e = Args.size(); i != e; ++i)
+  for (unsigned i = (SkipFirst ? 2 : 1), e = Args.size(); i < e; ++i)
     Elts.push_back(Args[i]);
 
   // Attach FlagObjectPointer to the explicit "this" parameter.
@@ -2284,7 +2295,8 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction(
   llvm::DISubprogram *SP = DBuilder.createMethod(
       RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine,
       MethodTy, VIndex, ThisAdjustment, ContainingType, Flags, SPFlags,
-      TParamsArray.get());
+      TParamsArray.get(), /*ThrownTypes*/ nullptr,
+      CGM.getCodeGenOpts().DebugKeyInstructions);
 
   SPCache[Method->getCanonicalDecl()].reset(SP);
 
@@ -4350,7 +4362,9 @@ llvm::DISubprogram *CGDebugInfo::getFunctionFwdDeclOrStub(GlobalDecl GD,
     return DBuilder.createFunction(
         DContext, Name, LinkageName, Unit, Line,
         getOrCreateFunctionType(GD.getDecl(), FnType, Unit), 0, Flags, SPFlags,
-        TParamsArray.get(), getFunctionDeclaration(FD));
+        TParamsArray.get(), getFunctionDeclaration(FD), /*ThrownTypes*/ nullptr,
+        /*Annotations*/ nullptr, /*TargetFuncName*/ "",
+        CGM.getCodeGenOpts().DebugKeyInstructions);
   }
 
   llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl(
@@ -4517,6 +4531,12 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D,
     // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields.
     return DBuilder.createSubroutineType(DBuilder.getOrCreateTypeArray({}));
 
+  if (const auto *Method = dyn_cast<CXXDestructorDecl>(D)) {
+    // Read method type from 'FnType' because 'D.getType()' does not cover
+    // implicit arguments for destructors.
+    return getOrCreateMethodTypeForDestructor(Method, F, FnType);
+  }
+
   if (const auto *Method = dyn_cast<CXXMethodDecl>(D))
     return getOrCreateMethodType(Method, F);
 
@@ -4686,8 +4706,9 @@ void CGDebugInfo::emitFunctionStart(GlobalDecl GD, SourceLocation Loc,
   llvm::DISubprogram *SP = DBuilder.createFunction(
       FDContext, Name, LinkageName, Unit, LineNo, DIFnType, ScopeLine,
       FlagsForDef, SPFlagsForDef, TParamsArray.get(), Decl, nullptr,
-      Annotations);
+      Annotations, "", CGM.getCodeGenOpts().DebugKeyInstructions);
   Fn->setSubprogram(SP);
+
   // We might get here with a VarDecl in the case we're generating
   // code for the initialization of globals. Do not record these decls
   // as they will overwrite the actual VarDecl Decl in the cache.
@@ -4746,9 +4767,12 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc,
 
   llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D);
   llvm::DISubroutineType *STy = getOrCreateFunctionType(D, FnType, Unit);
+  // Key Instructions: Don't set flag on declarations.
+  assert(~SPFlags & llvm::DISubprogram::SPFlagDefinition);
   llvm::DISubprogram *SP = DBuilder.createFunction(
       FDContext, Name, LinkageName, Unit, LineNo, STy, ScopeLine, Flags,
-      SPFlags, TParamsArray.get(), nullptr, nullptr, Annotations);
+      SPFlags, TParamsArray.get(), nullptr, nullptr, Annotations,
+      /*TargetFunctionName*/ "", /*UseKeyInstructions*/ false);
 
   // Preserve btf_decl_tag attributes for parameters of extern functions
   // for BPF target. The parameters created in this loop are attached as
@@ -5292,8 +5316,9 @@ void CGDebugInfo::EmitLabel(const LabelDecl *D, CGBuilderTy &Builder) {
   StringRef Name = D->getName();
 
   // Create the descriptor for the label.
-  auto *L =
-      DBuilder.createLabel(Scope, Name, Unit, Line, CGM.getLangOpts().Optimize);
+  auto *L = DBuilder.createLabel(
+      Scope, Name, Unit, Line, Column, /*IsArtificial=*/false,
+      /*CoroSuspendIdx=*/std::nullopt, CGM.getLangOpts().Optimize);
 
   // Insert an llvm.dbg.label into the current block.
   DBuilder.insertLabel(L,
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 16759f25123cd..411b2e718da30 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -263,9 +263,14 @@ class CGDebugInfo {
   /// to get a method type which includes \c this pointer.
   llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method,
                                                 llvm::DIFile *F);
+
+  llvm::DISubroutineType *
+  getOrCreateMethodTypeForDestructor(const CXXMethodDecl *Method,
+                                     llvm::DIFile *F, QualType FNType);
+
   llvm::DISubroutineType *
   getOrCreateInstanceMethodType(QualType ThisPtr, const FunctionProtoType *Func,
-                                llvm::DIFile *Unit);
+                                llvm::DIFile *Unit, bool SkipFirst = false);
   llvm::DISubroutineType *
   getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F);
   /// \return debug info descriptor for vtable.
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 193710bef2d16..44931d0481e26 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2114,8 +2114,10 @@ static int getAsInt32(llvm::ConstantInt *C, llvm::Type *I32Ty) {
 Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
   bool Ignore = TestAndClearIgnoreResultAssign();
   (void)Ignore;
-  assert (Ignore == false && "init list ignored");
   unsigned NumInitElements = E->getNumInits();
+  assert(Ignore == false ||
+         (NumInitElements == 0 && E->getType()->isVoidType()) &&
+             "init list ignored");
 
   // HLSL initialization lists in the AST are an expansion which can contain
   // side-effecting expressions wrapped in opaque value expressions. To properly
@@ -2440,7 +2442,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_BitCast: {
-    Value *Src = Visit(const_cast<Expr*>(E));
+    Value *Src = Visit(E);
     llvm::Type *SrcTy = Src->getType();
     llvm::Type *DstTy = ConvertType(DestTy);
 
@@ -2606,11 +2608,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_AtomicToNonAtomic:
   case CK_NonAtomicToAtomic:
   case CK_UserDefinedConversion:
-    return Visit(const_cast<Expr*>(E));
+    return Visit(E);
 
   case CK_NoOp: {
-    return CE->changesVolatileQualification() ? EmitLoadOfLValue(CE)
-                                              : Visit(const_cast<Expr *>(E));
+    return CE->changesVolatileQualification() ? EmitLoadOfLValue(CE) : Visit(E);
   }
 
   case CK_BaseToDerived: {
@@ -2712,10 +2713,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_LValueToRValue:
     assert(CGF.getContext().hasSameUnqualifiedType(E->getType(), DestTy));
     assert(E->isGLValue() && "lvalue-to-rvalue applied to r-value!");
-    return Visit(const_cast<Expr*>(E));
+    return Visit(E);
 
   case CK_IntegralToPointer: {
-    Value *Src = Visit(const_cast<Expr*>(E));
+    Value *Src = Visit(E);
 
     // First, convert to the correct width so that we control the kind of
     // extension.
@@ -2768,7 +2769,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_HLSLAggregateSplatCast:
   case CK_VectorSplat: {
     llvm::Type *DstTy = ConvertType(DestTy);
-    Value *Elt = Visit(const_cast<Expr *>(E));
+    Value *Elt = Visit(E);
     // Splat the element across to all elements
     llvm::ElementCount NumElements =
         cast<llvm::VectorType>(DstTy)->getElementCount();
@@ -2906,7 +2907,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_HLSLVectorTruncation: {
     assert((DestTy->isVectorType() || DestTy->isBuiltinType()) &&
            "Destination type must be a vector or builtin type.");
-    Value *Vec = Visit(const_cast<Expr *>(E));
+    Value *Vec = Visit(E);
     if (auto *VecTy = DestTy->getAs<VectorType>()) {
       SmallVector<int> Mask;
       unsigned NumElts = VecTy->getNumElements();
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index f954003bf5092..c1fd6f22f693b 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -23,7 +23,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/TargetOptions.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalVariable.h"
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 8ccc37ef98a74..a5f2f0efa2c3b 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -6801,6 +6801,11 @@ class MappableExprsHandler {
       llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
   using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
   using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
+  using MapData =
+      std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
+                 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
+                 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
+  using MapDataArrayTy = SmallVector<MapData, 4>;
 
   /// This structure contains combined information generated for mappable
   /// clauses, including base pointers, pointers, sizes, map types, user-defined
@@ -8496,6 +8501,7 @@ class MappableExprsHandler {
                          const StructRangeInfoTy &PartialStruct, bool IsMapThis,
                          llvm::OpenMPIRBuilder &OMPBuilder,
                          const ValueDecl *VD = nullptr,
+                         unsigned OffsetForMemberOfFlag = 0,
                          bool NotTargetParams = true) const {
     if (CurTypes.size() == 1 &&
         ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
@@ -8583,8 +8589,8 @@ class MappableExprsHandler {
     // All other current entries will be MEMBER_OF the combined entry
     // (except for PTR_AND_OBJ entries which do not have a placeholder value
     // 0xFFFF in the MEMBER_OF field).
-    OpenMPOffloadMappingFlags MemberOfFlag =
-        OMPBuilder.getMemberOfFlag(CombinedInfo.BasePointers.size() - 1);
+    OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
+        OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
     for (auto &M : CurTypes)
       OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
   }
@@ -8727,11 +8733,13 @@ class MappableExprsHandler {
     }
   }
 
-  /// Generate the base pointers, section pointers, sizes, map types, and
-  /// mappers associated to a given capture (all included in \a CombinedInfo).
-  void generateInfoForCapture(const CapturedStmt::Capture *Cap,
-                              llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
-                              StructRangeInfoTy &PartialStruct) const {
+  /// For a capture that has an associated clause, generate the base pointers,
+  /// section pointers, sizes, map types, and mappers (all included in
+  /// \a CurCaptureVarInfo).
+  void generateInfoForCaptureFromClauseInfo(
+      const CapturedStmt::Capture *Cap, llvm::Value *Arg,
+      MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
+      unsigned OffsetForMemberOfFlag) const {
     assert(!Cap->capturesVariableArrayType() &&
            "Not expecting to generate map info for a variable array type!");
 
@@ -8749,26 +8757,22 @@ class MappableExprsHandler {
     // pass the pointer by value. If it is a reference to a declaration, we just
     // pass its value.
     if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
-      CombinedInfo.Exprs.push_back(VD);
-      CombinedInfo.BasePointers.emplace_back(Arg);
-      CombinedInfo.DevicePtrDecls.emplace_back(VD);
-      CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
-      CombinedInfo.Pointers.push_back(Arg);
-      CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
+      CurCaptureVarInfo.Exprs.push_back(VD);
+      CurCaptureVarInfo.BasePointers.emplace_back(Arg);
+      CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD);
+      CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
+      CurCaptureVarInfo.Pointers.push_back(Arg);
+      CurCaptureVarInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
           CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty,
           /*isSigned=*/true));
-      CombinedInfo.Types.push_back(
+      CurCaptureVarInfo.Types.push_back(
           OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
           OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
-      CombinedInfo.Mappers.push_back(nullptr);
+      CurCaptureVarInfo.Mappers.push_back(nullptr);
       return;
     }
 
-    using MapData =
-        std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
-                   OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>, bool,
-                   const ValueDecl *, const Expr *>;
-    SmallVector<MapData, 4> DeclComponentLists;
+    MapDataArrayTy DeclComponentLists;
     // For member fields list in is_device_ptr, store it in
     // DeclComponentLists for generating components info.
     static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
@@ -8826,6 +8830,51 @@ class MappableExprsHandler {
       return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
     });
 
+    auto GenerateInfoForComponentLists =
+        [&](ArrayRef<MapData> DeclComponentLists,
+            bool IsEligibleForTargetParamFlag) {
+          MapCombinedInfoTy CurInfoForComponentLists;
+          StructRangeInfoTy PartialStruct;
+
+          if (DeclComponentLists.empty())
+            return;
+
+          generateInfoForCaptureFromComponentLists(
+              VD, DeclComponentLists, CurInfoForComponentLists, PartialStruct,
+              IsEligibleForTargetParamFlag,
+              /*AreBothBasePtrAndPteeMapped=*/HasMapBasePtr && HasMapArraySec);
+
+          // If there is an entry in PartialStruct it means we have a
+          // struct with individual members mapped. Emit an extra combined
+          // entry.
+          if (PartialStruct.Base.isValid()) {
+            CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);
+            emitCombinedEntry(
+                CurCaptureVarInfo, CurInfoForComponentLists.Types,
+                PartialStruct, Cap->capturesThis(), OMPBuilder, nullptr,
+                OffsetForMemberOfFlag,
+                /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
+          }
+
+          // Return if we didn't add any entries.
+          if (CurInfoForComponentLists.BasePointers.empty())
+            return;
+
+          CurCaptureVarInfo.append(CurInfoForComponentLists);
+        };
+
+    GenerateInfoForComponentLists(DeclComponentLists,
+                                  /*IsEligibleForTargetParamFlag=*/true);
+  }
+
+  /// Generate the base pointers, section pointers, sizes, map types, and
+  /// mappers associated to \a DeclComponentLists for a given capture
+  /// \a VD (all included in \a CurComponentListInfo).
+  void generateInfoForCaptureFromComponentLists(
+      const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
+      MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
+      bool IsListEligibleForTargetParamFlag,
+      bool AreBothBasePtrAndPteeMapped = false) const {
     // Find overlapping elements (including the offset from the base element).
     llvm::SmallDenseMap<
         const MapData *,
@@ -8949,7 +8998,7 @@ class MappableExprsHandler {
 
     // Associated with a capture, because the mapping flags depend on it.
     // Go through all of the elements with the overlapped elements.
-    bool IsFirstComponentList = true;
+    bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
     MapCombinedInfoTy StructBaseCombinedInfo;
     for (const auto &Pair : OverlappedData) {
       const MapData &L = *Pair.getFirst();
@@ -8964,11 +9013,11 @@ class MappableExprsHandler {
       ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
           OverlappedComponents = Pair.getSecond();
       generateInfoForComponentList(
-          MapType, MapModifiers, {}, Components, CombinedInfo,
-          StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
-          IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
+          MapType, MapModifiers, {}, Components, CurComponentListInfo,
+          StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag, IsImplicit,
+          /*GenerateAllInfoForClauses*/ false, Mapper,
           /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents);
-      IsFirstComponentList = false;
+      AddTargetParamFlag = false;
     }
     // Go through other elements without overlapped elements.
     for (const MapData &L : DeclComponentLists) {
@@ -8983,12 +9032,12 @@ class MappableExprsHandler {
       auto It = OverlappedData.find(&L);
       if (It == OverlappedData.end())
         generateInfoForComponentList(
-            MapType, MapModifiers, {}, Components, CombinedInfo,
-            StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
+            MapType, MapModifiers, {}, Components, CurComponentListInfo,
+            StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag,
             IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
             /*ForDeviceAddr=*/false, VD, VarRef,
-            /*OverlappedElements*/ {}, HasMapBasePtr && HasMapArraySec);
-      IsFirstComponentList = false;
+            /*OverlappedElements*/ {}, AreBothBasePtrAndPteeMapped);
+      AddTargetParamFlag = false;
     }
   }
 
@@ -9467,7 +9516,6 @@ static void genMapInfoForCaptures(
                                             CE = CS.capture_end();
        CI != CE; ++CI, ++RI, ++CV) {
     MappableExprsHandler::MapCombinedInfoTy CurInfo;
-    MappableExprsHandler::StructRangeInfoTy PartialStruct;
 
     // VLA sizes are passed to the outlined region by copy and do not have map
     // information associated.
@@ -9488,13 +9536,18 @@ static void genMapInfoForCaptures(
     } else {
       // If we have any information in the map clause, we use it, otherwise we
       // just do a default mapping.
-      MEHandler.generateInfoForCapture(CI, *CV, CurInfo, PartialStruct);
+      MEHandler.generateInfoForCaptureFromClauseInfo(
+          CI, *CV, CurInfo, OMPBuilder,
+          /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
+
       if (!CI->capturesThis())
         MappedVarSet.insert(CI->getCapturedVar());
       else
         MappedVarSet.insert(nullptr);
-      if (CurInfo.BasePointers.empty() && !PartialStruct.Base.isValid())
+
+      if (CurInfo.BasePointers.empty())
         MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
+
       // Generate correct mapping for variables captured by reference in
       // lambdas.
       if (CI->capturesVariable())
@@ -9502,7 +9555,7 @@ static void genMapInfoForCaptures(
                                                 CurInfo, LambdaPointers);
     }
     // We expect to have at least an element of information for this capture.
-    assert((!CurInfo.BasePointers.empty() || PartialStruct.Base.isValid()) &&
+    assert(!CurInfo.BasePointers.empty() &&
            "Non-existing map pointer for capture!");
     assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
            CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
@@ -9510,15 +9563,6 @@ static void genMapInfoForCaptures(
            CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
            "Inconsistent map information sizes!");
 
-    // If there is an entry in PartialStruct it means we have a struct with
-    // individual members mapped. Emit an extra combined entry.
-    if (PartialStruct.Base.isValid()) {
-      CombinedInfo.append(PartialStruct.PreliminaryMapData);
-      MEHandler.emitCombinedEntry(CombinedInfo, CurInfo.Types, PartialStruct,
-                                  CI->capturesThis(), OMPBuilder, nullptr,
-                                  /*NotTargetParams*/ false);
-    }
-
     // We need to append the results of this capture to what we already have.
     CombinedInfo.append(CurInfo);
   }
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index d9195d749e056..5822e0f6db89a 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -713,7 +713,7 @@ CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
     VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
   PGO->assignRegionCounters(GlobalDecl(CD), F);
   CapturedStmtInfo->EmitBody(*this, CD->getBody());
-  (void)LocalScope.ForceCleanup();
+  LocalScope.ForceCleanup();
   FinishFunction(CD->getBodyRBrace());
   if (!NeedWrapperFunction)
     return F;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 70a09795d02fe..776a646ceb32f 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1109,10 +1109,16 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
 
   // Add vscale_range attribute if appropriate.
   llvm::StringMap<bool> FeatureMap;
-  bool IsArmStreaming = false;
+  auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
   if (FD) {
     getContext().getFunctionFeatureMap(FeatureMap, FD);
-    IsArmStreaming = IsArmStreamingFunction(FD, true);
+    if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
+      if (T->getAArch64SMEAttributes() &
+          FunctionType::SME_PStateSMCompatibleMask)
+        IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
+
+    if (IsArmStreamingFunction(FD, true))
+      IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
   }
   std::optional<std::pair<unsigned, unsigned>> VScaleRange =
       getContext().getTargetInfo().getVScaleRange(getLangOpts(), IsArmStreaming,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 96fdab212beb1..c8866f15745c2 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -332,6 +332,71 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   return *TheTargetCodeGenInfo;
 }
 
+static void checkDataLayoutConsistency(const TargetInfo &Target,
+                                       llvm::LLVMContext &Context,
+                                       const LangOptions &Opts) {
+#ifndef NDEBUG
+  // Don't verify non-standard ABI configurations.
+  if (Opts.AlignDouble || Opts.OpenCL || Opts.HLSL)
+    return;
+
+  llvm::Triple Triple = Target.getTriple();
+  llvm::DataLayout DL(Target.getDataLayoutString());
+  auto Check = [&](const char *Name, llvm::Type *Ty, unsigned Alignment) {
+    llvm::Align DLAlign = DL.getABITypeAlign(Ty);
+    llvm::Align ClangAlign(Alignment / 8);
+    if (DLAlign != ClangAlign) {
+      llvm::errs() << "For target " << Triple.str() << " type " << Name
+                   << " mapping to " << *Ty << " has data layout alignment "
+                   << DLAlign.value() << " while clang specifies "
+                   << ClangAlign.value() << "\n";
+      abort();
+    }
+  };
+
+  Check("bool", llvm::Type::getIntNTy(Context, Target.BoolWidth),
+        Target.BoolAlign);
+  Check("short", llvm::Type::getIntNTy(Context, Target.ShortWidth),
+        Target.ShortAlign);
+  Check("int", llvm::Type::getIntNTy(Context, Target.IntWidth),
+        Target.IntAlign);
+  Check("long", llvm::Type::getIntNTy(Context, Target.LongWidth),
+        Target.LongAlign);
+  // FIXME: M68k specifies incorrect long long alignment in both LLVM and Clang.
+  if (Triple.getArch() != llvm::Triple::m68k)
+    Check("long long", llvm::Type::getIntNTy(Context, Target.LongLongWidth),
+          Target.LongLongAlign);
+  // FIXME: There are int128 alignment mismatches on multiple targets.
+  if (Target.hasInt128Type() && !Target.getTargetOpts().ForceEnableInt128 &&
+      !Triple.isAMDGPU() && !Triple.isSPIRV() &&
+      Triple.getArch() != llvm::Triple::ve)
+    Check("__int128", llvm::Type::getIntNTy(Context, 128), Target.Int128Align);
+
+  if (Target.hasFloat16Type())
+    Check("half", llvm::Type::getFloatingPointTy(Context, *Target.HalfFormat),
+          Target.HalfAlign);
+  if (Target.hasBFloat16Type())
+    Check("bfloat", llvm::Type::getBFloatTy(Context), Target.BFloat16Align);
+  Check("float", llvm::Type::getFloatingPointTy(Context, *Target.FloatFormat),
+        Target.FloatAlign);
+  // FIXME: AIX specifies wrong double alignment in DataLayout
+  if (!Triple.isOSAIX()) {
+    Check("double",
+          llvm::Type::getFloatingPointTy(Context, *Target.DoubleFormat),
+          Target.DoubleAlign);
+    Check("long double",
+          llvm::Type::getFloatingPointTy(Context, *Target.LongDoubleFormat),
+          Target.LongDoubleAlign);
+  }
+  if (Target.hasFloat128Type())
+    Check("__float128", llvm::Type::getFP128Ty(Context), Target.Float128Align);
+  if (Target.hasIbm128Type())
+    Check("__ibm128", llvm::Type::getPPC_FP128Ty(Context), Target.Ibm128Align);
+
+  Check("void*", llvm::PointerType::getUnqual(Context), Target.PointerAlign);
+#endif
+}
+
 CodeGenModule::CodeGenModule(ASTContext &C,
                              IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
                              const HeaderSearchOptions &HSO,
@@ -487,6 +552,9 @@ CodeGenModule::CodeGenModule(ASTContext &C,
 
     llvm::sort(this->MSHotPatchFunctions);
   }
+
+  if (!Context.getAuxTargetInfo())
+    checkDataLayoutConsistency(Context.getTargetInfo(), LLVMContext, LangOpts);
 }
 
 CodeGenModule::~CodeGenModule() {}
diff --git a/clang/lib/CodeGen/ConstantInitBuilder.cpp b/clang/lib/CodeGen/ConstantInitBuilder.cpp
index ce1fe137c1919..7b3e7aea0f647 100644
--- a/clang/lib/CodeGen/ConstantInitBuilder.cpp
+++ b/clang/lib/CodeGen/ConstantInitBuilder.cpp
@@ -160,7 +160,7 @@ ConstantAggregateBuilderBase::getAddrOfPosition(llvm::Type *type,
                                         nullptr, "");
   Builder.SelfReferences.emplace_back(dummy);
   auto &entry = Builder.SelfReferences.back();
-  (void)getGEPIndicesTo(entry.Indices, position + Begin);
+  getGEPIndicesTo(entry.Indices, position + Begin);
   return dummy;
 }
 
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 982e5cd37ffd1..f09b3b92c4ea0 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -545,6 +545,18 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
@@ -555,6 +567,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     switch (BuiltinID) {
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
       IID = Intrinsic::amdgcn_global_load_tr_b64;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
@@ -563,8 +576,31 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
       IID = Intrinsic::amdgcn_global_load_tr_b128;
       break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
+      IID = Intrinsic::amdgcn_global_load_tr4_b64;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
+      IID = Intrinsic::amdgcn_global_load_tr6_b96;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
+      IID = Intrinsic::amdgcn_ds_load_tr4_b64;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
+      IID = Intrinsic::amdgcn_ds_load_tr6_b96;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
+      IID = Intrinsic::amdgcn_ds_load_tr8_b64;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
+      IID = Intrinsic::amdgcn_ds_load_tr16_b128;
+      break;
     case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
       IID = Intrinsic::amdgcn_ds_read_tr4_b64;
       break;
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index e30a8c6133055..7e6a47fd7c103 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -838,20 +838,20 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
   NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
   NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
   NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
-  NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
-  NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
-  NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
+  NEONMAP1(vrnd_v, trunc, Add1ArgType),
+  NEONMAP1(vrnda_v, round, Add1ArgType),
+  NEONMAP1(vrndaq_v, round, Add1ArgType),
   NEONMAP0(vrndi_v),
   NEONMAP0(vrndiq_v),
-  NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
-  NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
-  NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
-  NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
-  NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
-  NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
-  NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
-  NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
-  NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
+  NEONMAP1(vrndm_v, floor, Add1ArgType),
+  NEONMAP1(vrndmq_v, floor, Add1ArgType),
+  NEONMAP1(vrndn_v, roundeven, Add1ArgType),
+  NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
+  NEONMAP1(vrndp_v, ceil, Add1ArgType),
+  NEONMAP1(vrndpq_v, ceil, Add1ArgType),
+  NEONMAP1(vrndq_v, trunc, Add1ArgType),
+  NEONMAP1(vrndx_v, rint, Add1ArgType),
+  NEONMAP1(vrndxq_v, rint, Add1ArgType),
   NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
   NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
   NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
@@ -3132,7 +3132,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case NEON::BI__builtin_neon_vrndns_f32: {
     Value *Arg = EmitScalarExpr(E->getArg(0));
     llvm::Type *Tys[] = {Arg->getType()};
-    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
+    Function *F = CGM.getIntrinsic(Intrinsic::roundeven, Tys);
     return Builder.CreateCall(F, {Arg}, "vrndn"); }
 
   case NEON::BI__builtin_neon_vset_lane_i8:
@@ -5471,19 +5471,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
   }
 
   if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
-      BuiltinID == clang::AArch64::BI_WriteStatusReg) {
+      BuiltinID == clang::AArch64::BI_WriteStatusReg ||
+      BuiltinID == clang::AArch64::BI__sys) {
     LLVMContext &Context = CGM.getLLVMContext();
 
     unsigned SysReg =
       E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
 
     std::string SysRegStr;
-    llvm::raw_string_ostream(SysRegStr) <<
-                       ((1 << 1) | ((SysReg >> 14) & 1))  << ":" <<
-                       ((SysReg >> 11) & 7)               << ":" <<
-                       ((SysReg >> 7)  & 15)              << ":" <<
-                       ((SysReg >> 3)  & 15)              << ":" <<
-                       ( SysReg        & 7);
+    unsigned SysRegOp0 = (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
+                          BuiltinID == clang::AArch64::BI_WriteStatusReg)
+                             ? ((1 << 1) | ((SysReg >> 14) & 1))
+                             : 1;
+    llvm::raw_string_ostream(SysRegStr)
+        << SysRegOp0 << ":" << ((SysReg >> 11) & 7) << ":"
+        << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
+        << (SysReg & 7);
 
     llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
     llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
@@ -5500,8 +5503,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
 
     llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
     llvm::Value *ArgValue = EmitScalarExpr(E->getArg(1));
-
-    return Builder.CreateCall(F, { Metadata, ArgValue });
+    llvm::Value *Result = Builder.CreateCall(F, {Metadata, ArgValue});
+    if (BuiltinID == clang::AArch64::BI__sys) {
+      // Return 0 for convenience, even though MSVC returns some other undefined
+      // value.
+      Result = ConstantInt::get(Builder.getInt32Ty(), 0);
+    }
+    return Result;
   }
 
   if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index 14d4cee7c61d3..cc3d487da83b5 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -544,7 +544,7 @@ ABIArgInfo RISCVABIInfo::coerceVLSVector(QualType Ty, unsigned ABIVLen) const {
   assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
 
   auto VScale = getContext().getTargetInfo().getVScaleRange(
-      getContext().getLangOpts(), false);
+      getContext().getLangOpts(), TargetInfo::ArmStreamingKind::NotStreaming);
 
   unsigned NumElts = VT->getNumElements();
   llvm::Type *EltType = llvm::Type::getInt1Ty(getVMContext());
diff --git a/clang/lib/CodeGen/Targets/SPIR.cpp b/clang/lib/CodeGen/Targets/SPIR.cpp
index be9a5e60af358..a66de79ed7cd1 100644
--- a/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -58,7 +58,7 @@ class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
               const SmallVector<int32_t> *Packoffsets = nullptr) const override;
   llvm::Type *getSPIRVImageTypeFromHLSLResource(
       const HLSLAttributedResourceType::Attributes &attributes,
-      llvm::Type *ElementType, llvm::LLVMContext &Ctx) const;
+      QualType SampledType, CodeGenModule &CGM) const;
   void
   setOCLKernelStubCallingConvention(const FunctionType *&FT) const override;
 };
@@ -421,14 +421,12 @@ static llvm::Type *getInlineSpirvType(CodeGenModule &CGM,
     case SpirvOperandKind::ConstantId: {
       llvm::Type *IntegralType =
           CGM.getTypes().ConvertType(Operand.getResultType());
-      llvm::APInt Value = Operand.getValue();
 
-      Result = getInlineSpirvConstant(CGM, IntegralType, Value);
+      Result = getInlineSpirvConstant(CGM, IntegralType, Operand.getValue());
       break;
     }
     case SpirvOperandKind::Literal: {
-      llvm::APInt Value = Operand.getValue();
-      Result = getInlineSpirvConstant(CGM, nullptr, Value);
+      Result = getInlineSpirvConstant(CGM, nullptr, Operand.getValue());
       break;
     }
     case SpirvOperandKind::TypeId: {
@@ -487,12 +485,12 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
     assert(!ResAttrs.IsROV &&
            "Rasterizer order views not implemented for SPIR-V yet");
 
-    llvm::Type *ElemType = CGM.getTypes().ConvertType(ContainedTy);
     if (!ResAttrs.RawBuffer) {
       // convert element type
-      return getSPIRVImageTypeFromHLSLResource(ResAttrs, ElemType, Ctx);
+      return getSPIRVImageTypeFromHLSLResource(ResAttrs, ContainedTy, CGM);
     }
 
+    llvm::Type *ElemType = CGM.getTypes().ConvertTypeForMem(ContainedTy);
     llvm::ArrayType *RuntimeArrayType = llvm::ArrayType::get(ElemType, 0);
     uint32_t StorageClass = /* StorageBuffer storage class */ 12;
     bool IsWritable = ResAttrs.ResourceClass == llvm::dxil::ResourceClass::UAV;
@@ -520,13 +518,18 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
 }
 
 llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
-    const HLSLAttributedResourceType::Attributes &attributes,
-    llvm::Type *ElementType, llvm::LLVMContext &Ctx) const {
+    const HLSLAttributedResourceType::Attributes &attributes, QualType Ty,
+    CodeGenModule &CGM) const {
+  llvm::LLVMContext &Ctx = CGM.getLLVMContext();
+
+  Ty = Ty->getCanonicalTypeUnqualified();
+  if (const VectorType *V = dyn_cast<VectorType>(Ty))
+    Ty = V->getElementType();
+  assert(!Ty->isVectorType() && "We still have a vector type.");
 
-  if (ElementType->isVectorTy())
-    ElementType = ElementType->getScalarType();
+  llvm::Type *SampledType = CGM.getTypes().ConvertTypeForMem(Ty);
 
-  assert((ElementType->isIntegerTy() || ElementType->isFloatingPointTy()) &&
+  assert((SampledType->isIntegerTy() || SampledType->isFloatingPointTy()) &&
          "The element type for a SPIR-V resource must be a scalar integer or "
          "floating point type.");
 
@@ -535,6 +538,9 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
   // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage.
   SmallVector<unsigned, 6> IntParams(6, 0);
 
+  const char *Name =
+      Ty->isSignedIntegerType() ? "spirv.SignedImage" : "spirv.Image";
+
   // Dim
   // For now we assume everything is a buffer.
   IntParams[0] = 5;
@@ -557,7 +563,9 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
   // Setting to unknown for now.
   IntParams[5] = 0;
 
-  return llvm::TargetExtType::get(Ctx, "spirv.Image", {ElementType}, IntParams);
+  llvm::TargetExtType *ImageType =
+      llvm::TargetExtType::get(Ctx, Name, {SampledType}, IntParams);
+  return ImageType;
 }
 
 std::unique_ptr<TargetCodeGenInfo>
diff --git a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
index 2ffbc1a226958..440b5633fb1b6 100644
--- a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
+++ b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
@@ -9,19 +9,14 @@
 #include "DirectoryScanner.h"
 #include "clang/DirectoryWatcher/DirectoryWatcher.h"
 
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/Path.h"
-#include <atomic>
 #include <condition_variable>
 #include <mutex>
 #include <queue>
 #include <string>
 #include <thread>
-#include <vector>
 
 #include <fcntl.h>
 #include <limits.h>
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 3cfd671e9d8f2..0268ec49eaf11 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -76,7 +76,6 @@ add_clang_library(clangDriver
   ToolChains/OHOS.cpp
   ToolChains/OpenBSD.cpp
   ToolChains/PS4CPU.cpp
-  ToolChains/RISCVToolchain.cpp
   ToolChains/Solaris.cpp
   ToolChains/SPIRV.cpp
   ToolChains/SPIRVOpenMP.cpp
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 0c5503a66b8f0..55748c0bce918 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -43,7 +43,6 @@
 #include "ToolChains/PPCFreeBSD.h"
 #include "ToolChains/PPCLinux.h"
 #include "ToolChains/PS4CPU.h"
-#include "ToolChains/RISCVToolchain.h"
 #include "ToolChains/SPIRV.h"
 #include "ToolChains/SPIRVOpenMP.h"
 #include "ToolChains/SYCL.h"
@@ -193,7 +192,12 @@ std::string Driver::GetResourcesPath(StringRef BinaryPath) {
 
   StringRef ConfiguredResourceDir(CLANG_RESOURCE_DIR);
   if (!ConfiguredResourceDir.empty()) {
-    llvm::sys::path::append(P, ConfiguredResourceDir);
+    // FIXME: We should fix the behavior of llvm::sys::path::append so we don't
+    // need to check for absolute paths here.
+    if (llvm::sys::path::is_absolute(ConfiguredResourceDir))
+      P = ConfiguredResourceDir;
+    else
+      llvm::sys::path::append(P, ConfiguredResourceDir);
   } else {
     // On Windows, libclang.dll is in bin/.
     // On non-Windows, libclang.so/.dylib is in lib/.
@@ -992,7 +996,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                                         /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1026,7 +1030,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1042,7 +1046,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
@@ -1086,19 +1090,16 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       llvm::Triple AMDTriple("amdgcn-amd-amdhsa");
       llvm::Triple NVPTXTriple("nvptx64-nvidia-cuda");
 
-      for (StringRef A :
+      for (StringRef Arch :
            C.getInputArgs().getAllArgValues(options::OPT_offload_arch_EQ)) {
-        for (StringRef Arch : llvm::split(A, ",")) {
-          bool IsNVPTX = IsNVIDIAOffloadArch(
-              StringToOffloadArch(getProcessorFromTargetID(NVPTXTriple, Arch)));
-          bool IsAMDGPU = IsAMDOffloadArch(
-              StringToOffloadArch(getProcessorFromTargetID(AMDTriple, Arch)));
-          if (!IsNVPTX && !IsAMDGPU && !Arch.empty() &&
-              !Arch.equals_insensitive("native")) {
-            Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
-                << Arch;
-            return;
-          }
+        bool IsNVPTX = IsNVIDIAOffloadArch(
+            StringToOffloadArch(getProcessorFromTargetID(NVPTXTriple, Arch)));
+        bool IsAMDGPU = IsAMDOffloadArch(
+            StringToOffloadArch(getProcessorFromTargetID(AMDTriple, Arch)));
+        if (!IsNVPTX && !IsAMDGPU && !Arch.empty() &&
+            !Arch.equals_insensitive("native")) {
+          Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch) << Arch;
+          return;
         }
       }
 
@@ -1123,7 +1124,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
@@ -4788,7 +4789,7 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
     // Add or remove the seen architectures in order of appearance. If an
     // invalid architecture is given we simply exit.
     if (Arg->getOption().matches(options::OPT_offload_arch_EQ)) {
-      for (StringRef Arch : llvm::split(Arg->getValue(), ",")) {
+      for (StringRef Arch : Arg->getValues()) {
         if (Arch == "native" || Arch.empty()) {
           auto GPUsOrErr = TC->getSystemGPUArchs(Args);
           if (!GPUsOrErr) {
@@ -6955,16 +6956,11 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
         TC = std::make_unique<toolchains::AVRToolChain>(*this, Target, Args);
         break;
       case llvm::Triple::msp430:
-        TC =
-            std::make_unique<toolchains::MSP430ToolChain>(*this, Target, Args);
+        TC = std::make_unique<toolchains::MSP430ToolChain>(*this, Target, Args);
         break;
       case llvm::Triple::riscv32:
       case llvm::Triple::riscv64:
-        if (toolchains::RISCVToolChain::hasGCCToolchain(*this, Args))
-          TC =
-              std::make_unique<toolchains::RISCVToolChain>(*this, Target, Args);
-        else
-          TC = std::make_unique<toolchains::BareMetal>(*this, Target, Args);
+        TC = std::make_unique<toolchains::BareMetal>(*this, Target, Args);
         break;
       case llvm::Triple::ve:
         TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 571ac53f4571a..3f9b808b2722e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1731,7 +1731,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index b7564a0495da8..7fc34f451f183 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -836,7 +836,7 @@ Expected<SmallVector<std::string>>
 AMDGPUToolChain::getSystemGPUArchs(const ArgList &Args) const {
   // Detect AMD GPUs availible on the system.
   std::string Program;
-  if (Arg *A = Args.getLastArg(options::OPT_amdgpu_arch_tool_EQ))
+  if (Arg *A = Args.getLastArg(options::OPT_offload_arch_tool_EQ))
     Program = A->getValue();
   else
     Program = GetProgramPath("amdgpu-arch");
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index ea906d2e2acfc..6bd710ec6b8b5 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -156,39 +156,18 @@ static bool
 getAArch64MicroArchFeaturesFromMtune(const Driver &D, StringRef Mtune,
                                      const ArgList &Args,
                                      std::vector<StringRef> &Features) {
-  std::string MtuneLowerCase = Mtune.lower();
   // Check CPU name is valid, but ignore any extensions on it.
+  std::string MtuneLowerCase = Mtune.lower();
   llvm::AArch64::ExtensionSet Extensions;
   StringRef Tune;
-  if (!DecodeAArch64Mcpu(D, MtuneLowerCase, Tune, Extensions))
-    return false;
-
-  // Handle CPU name is 'native'.
-  if (MtuneLowerCase == "native")
-    MtuneLowerCase = std::string(llvm::sys::getHostCPUName());
-
-  // 'cyclone' and later have zero-cycle register moves and zeroing.
-  if (MtuneLowerCase == "cyclone" ||
-      StringRef(MtuneLowerCase).starts_with("apple")) {
-    Features.push_back("+zcm");
-    Features.push_back("+zcz");
-  }
-
-  return true;
+  return DecodeAArch64Mcpu(D, MtuneLowerCase, Tune, Extensions);
 }
 
 static bool
 getAArch64MicroArchFeaturesFromMcpu(const Driver &D, StringRef Mcpu,
                                     const ArgList &Args,
                                     std::vector<StringRef> &Features) {
-  StringRef CPU;
-  // Check CPU name is valid, but ignore any extensions on it.
-  llvm::AArch64::ExtensionSet DecodedFeature;
-  std::string McpuLowerCase = Mcpu.lower();
-  if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, DecodedFeature))
-    return false;
-
-  return getAArch64MicroArchFeaturesFromMtune(D, CPU, Args, Features);
+  return getAArch64MicroArchFeaturesFromMtune(D, Mcpu, Args, Features);
 }
 
 void aarch64::getAArch64TargetFeatures(const Driver &D,
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index d8168ed15febd..e670696cd59ae 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -31,6 +31,40 @@ using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 
+/// Is the triple {aarch64.aarch64_be}-none-elf?
+static bool isAArch64BareMetal(const llvm::Triple &Triple) {
+  if (Triple.getArch() != llvm::Triple::aarch64 &&
+      Triple.getArch() != llvm::Triple::aarch64_be)
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
+}
+
+static bool isRISCVBareMetal(const llvm::Triple &Triple) {
+  if (!Triple.isRISCV())
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
+}
+
+/// Is the triple powerpc[64][le]-*-none-eabi?
+static bool isPPCBareMetal(const llvm::Triple &Triple) {
+  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
+         Triple.getEnvironment() == llvm::Triple::EABI;
+}
+
 static bool findRISCVMultilibs(const Driver &D,
                                const llvm::Triple &TargetTriple,
                                const ArgList &Args, DetectedMultilibs &Result) {
@@ -95,7 +129,8 @@ static bool findRISCVMultilibs(const Driver &D,
   return false;
 }
 
-static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
+static std::string computeClangRuntimesSysRoot(const Driver &D,
+                                               bool IncludeTriple) {
   if (!D.SysRoot.empty())
     return D.SysRoot;
 
@@ -108,56 +143,133 @@ static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
   return std::string(SysRootDir);
 }
 
-BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
-                     const ArgList &Args)
-    : ToolChain(D, Triple, Args),
-      SysRoot(computeBaseSysRoot(D, /*IncludeTriple=*/true)) {
-  getProgramPaths().push_back(getDriver().Dir);
-
-  findMultilibs(D, Triple, Args);
-  SmallString<128> SysRoot(computeSysRoot());
-  if (!SysRoot.empty()) {
-    for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRoot);
-      llvm::sys::path::append(Dir, M.osSuffix(), "lib");
-      getFilePaths().push_back(std::string(Dir));
-      getLibraryPaths().push_back(std::string(Dir));
-    }
+// Only consider the GCC toolchain based on the values provided through the
+// `--gcc-toolchain` and `--gcc-install-dir` flags. The function below returns
+// whether the GCC toolchain was initialized successfully.
+bool BareMetal::initGCCInstallation(const llvm::Triple &Triple,
+                                    const llvm::opt::ArgList &Args) {
+  if (Args.getLastArg(options::OPT_gcc_toolchain) ||
+      Args.getLastArg(clang::driver::options::OPT_gcc_install_dir_EQ)) {
+    GCCInstallation.init(Triple, Args);
+    return GCCInstallation.isValid();
   }
+  return false;
 }
 
-/// Is the triple {aarch64.aarch64_be}-none-elf?
-static bool isAArch64BareMetal(const llvm::Triple &Triple) {
-  if (Triple.getArch() != llvm::Triple::aarch64 &&
-      Triple.getArch() != llvm::Triple::aarch64_be)
-    return false;
+// This logic is adapted from RISCVToolChain.cpp as part of the ongoing effort
+// to merge RISCVToolChain into the Baremetal toolchain. It infers the presence
+// of a valid GCC toolchain by checking whether the `crt0.o` file exists in the
+// `bin/../<target-triple>/lib` directory.
+static bool detectGCCToolchainAdjacent(const Driver &D) {
+  SmallString<128> GCCDir;
+  llvm::sys::path::append(GCCDir, D.Dir, "..", D.getTargetTriple(),
+                          "lib/crt0.o");
+  return llvm::sys::fs::exists(GCCDir);
+}
 
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
+// If no sysroot is provided the driver will first attempt to infer it from the
+// values of `--gcc-install-dir` or `--gcc-toolchain`, which specify the
+// location of a GCC toolchain.
+// If neither flag is used, the sysroot defaults to either:
+//    - `bin/../<target-triple>`
+//    - `bin/../lib/clang-runtimes/<target-triple>`
+//
+// To use the `clang-runtimes` path, ensure that `../<target-triple>/lib/crt0.o`
+// does not exist relative to the driver.
+std::string BareMetal::computeSysRoot() const {
+  // Use Baremetal::sysroot if it has already been set.
+  if (!SysRoot.empty())
+    return SysRoot;
+
+  // Use the sysroot specified via the `--sysroot` command-line flag, if
+  // provided.
+  const Driver &D = getDriver();
+  if (!D.SysRoot.empty())
+    return D.SysRoot;
 
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
+  // Attempt to infer sysroot from a valid GCC installation.
+  // If no valid GCC installation, check for a GCC toolchain alongside Clang.
+  SmallString<128> inferredSysRoot;
+  if (IsGCCInstallationValid) {
+    llvm::sys::path::append(inferredSysRoot, GCCInstallation.getParentLibPath(),
+                            "..", GCCInstallation.getTriple().str());
+  } else if (detectGCCToolchainAdjacent(D)) {
+    // Use the triple as provided to the driver. Unlike the parsed triple
+    // this has not been normalized to always contain every field.
+    llvm::sys::path::append(inferredSysRoot, D.Dir, "..", D.getTargetTriple());
+  }
+  // If a valid sysroot was inferred and exists, use it
+  if (!inferredSysRoot.empty() && llvm::sys::fs::exists(inferredSysRoot))
+    return std::string(inferredSysRoot);
 
-  return Triple.getEnvironmentName() == "elf";
+  // Use the clang-runtimes path.
+  return computeClangRuntimesSysRoot(D, /*IncludeTriple*/ true);
 }
 
-static bool isRISCVBareMetal(const llvm::Triple &Triple) {
-  if (!Triple.isRISCV())
-    return false;
-
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
-
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
+std::string BareMetal::getCompilerRTPath() const {
+  const Driver &D = getDriver();
+  if (IsGCCInstallationValid || detectGCCToolchainAdjacent(getDriver())) {
+    SmallString<128> Path(D.ResourceDir);
+    llvm::sys::path::append(Path, "lib");
+    return std::string(Path.str());
+  }
+  return ToolChain::getCompilerRTPath();
+}
 
-  return Triple.getEnvironmentName() == "elf";
+static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
+                                  const Multilib &Multilib,
+                                  StringRef InstallPath,
+                                  ToolChain::path_list &Paths) {
+  if (const auto &PathsCallback = Multilibs.filePathsCallback())
+    for (const auto &Path : PathsCallback(Multilib))
+      addPathIfExists(D, InstallPath + Path, Paths);
 }
 
-/// Is the triple powerpc[64][le]-*-none-eabi?
-static bool isPPCBareMetal(const llvm::Triple &Triple) {
-  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
-         Triple.getEnvironment() == llvm::Triple::EABI;
+// GCC mutltilibs will only work for those targets that have their multlib
+// structure encoded into GCCInstallation. Baremetal toolchain supports ARM,
+// AArch64, RISCV and PPC and of these only RISCV have GCC multilibs hardcoded
+// in GCCInstallation.
+BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
+                     const ArgList &Args)
+    : Generic_ELF(D, Triple, Args) {
+  IsGCCInstallationValid = initGCCInstallation(Triple, Args);
+  std::string ComputedSysRoot = computeSysRoot();
+  if (IsGCCInstallationValid) {
+    if (!isRISCVBareMetal(Triple))
+      D.Diag(clang::diag::warn_drv_multilib_not_available_for_target);
+
+    Multilibs = GCCInstallation.getMultilibs();
+    SelectedMultilibs.assign({GCCInstallation.getMultilib()});
+
+    path_list &Paths = getFilePaths();
+    // Add toolchain/multilib specific file paths.
+    addMultilibsFilePaths(D, Multilibs, SelectedMultilibs.back(),
+                          GCCInstallation.getInstallPath(), Paths);
+    // Adding filepath for locating crt{begin,end}.o files.
+    Paths.push_back(GCCInstallation.getInstallPath().str());
+    // Adding filepath for locating crt0.o file.
+    Paths.push_back(ComputedSysRoot + "/lib");
+
+    ToolChain::path_list &PPaths = getProgramPaths();
+    // Multilib cross-compiler GCC installations put ld in a triple-prefixed
+    // directory off of the parent of the GCC installation.
+    PPaths.push_back(Twine(GCCInstallation.getParentLibPath() + "/../" +
+                           GCCInstallation.getTriple().str() + "/bin")
+                         .str());
+    PPaths.push_back((GCCInstallation.getParentLibPath() + "/../bin").str());
+  } else {
+    getProgramPaths().push_back(getDriver().Dir);
+    findMultilibs(D, Triple, Args);
+    const SmallString<128> SysRootDir(computeSysRoot());
+    if (!SysRootDir.empty()) {
+      for (const Multilib &M : getOrderedMultilibs()) {
+        SmallString<128> Dir(SysRootDir);
+        llvm::sys::path::append(Dir, M.osSuffix(), "lib");
+        getFilePaths().push_back(std::string(Dir));
+        getLibraryPaths().push_back(std::string(Dir));
+      }
+    }
+  }
 }
 
 static void
@@ -216,7 +328,7 @@ getMultilibConfigPath(const Driver &D, const llvm::Triple &Triple,
       return {};
     }
   } else {
-    MultilibPath = computeBaseSysRoot(D, /*IncludeTriple=*/false);
+    MultilibPath = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
     llvm::sys::path::append(MultilibPath, MultilibFilename);
   }
   return MultilibPath;
@@ -234,7 +346,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
   if (D.getVFS().exists(*MultilibPath)) {
     // If multilib.yaml is found, update sysroot so it doesn't use a target
     // specific suffix
-    SysRoot = computeBaseSysRoot(D, /*IncludeTriple=*/false);
+    SysRoot = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
     SmallVector<StringRef> CustomFlagMacroDefines;
     findMultilibsFromYAML(*this, D, *MultilibPath, Args, Result,
                           CustomFlagMacroDefines);
@@ -242,7 +354,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
     Multilibs = Result.Multilibs;
     MultilibMacroDefines.append(CustomFlagMacroDefines.begin(),
                                 CustomFlagMacroDefines.end());
-  } else if (isRISCVBareMetal(Triple)) {
+  } else if (isRISCVBareMetal(Triple) && !detectGCCToolchainAdjacent(D)) {
     if (findRISCVMultilibs(D, Triple, Args, Result)) {
       SelectedMultilibs = Result.SelectedMultilibs;
       Multilibs = Result.Multilibs;
@@ -263,8 +375,6 @@ Tool *BareMetal::buildStaticLibTool() const {
   return new tools::baremetal::StaticLibTool(*this);
 }
 
-std::string BareMetal::computeSysRoot() const { return SysRoot; }
-
 BareMetal::OrderedMultilibs BareMetal::getOrderedMultilibs() const {
   // Get multilibs in reverse order because they're ordered most-specific last.
   if (!SelectedMultilibs.empty())
@@ -275,6 +385,28 @@ BareMetal::OrderedMultilibs BareMetal::getOrderedMultilibs() const {
   return llvm::reverse(Default);
 }
 
+ToolChain::CXXStdlibType BareMetal::GetDefaultCXXStdlibType() const {
+  if (getTriple().isRISCV() && IsGCCInstallationValid)
+    return ToolChain::CST_Libstdcxx;
+  return ToolChain::CST_Libcxx;
+}
+
+ToolChain::RuntimeLibType BareMetal::GetDefaultRuntimeLibType() const {
+  if (getTriple().isRISCV() && IsGCCInstallationValid)
+    return ToolChain::RLT_Libgcc;
+  return ToolChain::RLT_CompilerRT;
+}
+
+// TODO: Add a validity check for GCCInstallation.
+//       If valid, use `UNW_Libgcc`; otherwise, use `UNW_None`.
+ToolChain::UnwindLibType
+BareMetal::GetUnwindLibType(const llvm::opt::ArgList &Args) const {
+  if (getTriple().isRISCV())
+    return ToolChain::UNW_None;
+
+  return ToolChain::GetUnwindLibType(Args);
+}
+
 void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                           ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc))
@@ -292,10 +424,10 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   if (std::optional<std::string> Path = getStdlibIncludePath())
     addSystemInclude(DriverArgs, CC1Args, *Path);
 
-  const SmallString<128> SysRoot(computeSysRoot());
-  if (!SysRoot.empty()) {
+  const SmallString<128> SysRootDir(computeSysRoot());
+  if (!SysRootDir.empty()) {
     for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRoot);
+      SmallString<128> Dir(SysRootDir);
       llvm::sys::path::append(Dir, M.includeSuffix());
       llvm::sys::path::append(Dir, "include");
       addSystemInclude(DriverArgs, CC1Args, Dir.str());
@@ -309,6 +441,19 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs,
   CC1Args.push_back("-nostdsysteminc");
 }
 
+void BareMetal::addLibStdCxxIncludePaths(
+    const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args) const {
+  if (!IsGCCInstallationValid)
+    return;
+  const GCCVersion &Version = GCCInstallation.getVersion();
+  StringRef TripleStr = GCCInstallation.getTriple().str();
+  const Multilib &Multilib = GCCInstallation.getMultilib();
+  addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
+                           TripleStr, Multilib.includeSuffix(), DriverArgs,
+                           CC1Args);
+}
+
 void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc,
@@ -339,23 +484,23 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
   };
 
   switch (GetCXXStdlibType(DriverArgs)) {
-    case ToolChain::CST_Libcxx: {
-      SmallString<128> P(D.Dir);
-      llvm::sys::path::append(P, "..", "include");
-      AddCXXIncludePath(P);
-      break;
-    }
-    case ToolChain::CST_Libstdcxx:
-      // We only support libc++ toolchain installation.
-      break;
+  case ToolChain::CST_Libcxx: {
+    SmallString<128> P(D.Dir);
+    llvm::sys::path::append(P, "..", "include");
+    AddCXXIncludePath(P);
+    break;
+  }
+  case ToolChain::CST_Libstdcxx:
+    addLibStdCxxIncludePaths(DriverArgs, CC1Args);
+    break;
   }
 
-  std::string SysRoot(computeSysRoot());
-  if (SysRoot.empty())
+  std::string SysRootDir(computeSysRoot());
+  if (SysRootDir.empty())
     return;
 
   for (const Multilib &M : getOrderedMultilibs()) {
-    SmallString<128> Dir(SysRoot);
+    SmallString<128> Dir(SysRootDir);
     llvm::sys::path::append(Dir, M.gccSuffix());
     switch (GetCXXStdlibType(DriverArgs)) {
     case ToolChain::CST_Libcxx: {
@@ -455,12 +600,24 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   const llvm::Triple::ArchType Arch = TC.getArch();
   const llvm::Triple &Triple = getToolChain().getEffectiveTriple();
 
-  AddLinkerInputs(TC, Inputs, Args, CmdArgs, JA);
+  if (!D.SysRoot.empty())
+    CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
   CmdArgs.push_back("-Bstatic");
 
-  if (TC.getTriple().isRISCV() && Args.hasArg(options::OPT_mno_relax))
-    CmdArgs.push_back("--no-relax");
+  if (const char *LDMOption = getLDMOption(TC.getTriple(), Args)) {
+    CmdArgs.push_back("-m");
+    CmdArgs.push_back(LDMOption);
+  } else {
+    D.Diag(diag::err_target_unknown_triple) << Triple.str();
+    return;
+  }
+
+  if (Triple.isRISCV()) {
+    CmdArgs.push_back("-X");
+    if (Args.hasArg(options::OPT_mno_relax))
+      CmdArgs.push_back("--no-relax");
+  }
 
   if (Triple.isARM() || Triple.isThumb()) {
     bool IsBigEndian = arm::isARMBigEndian(Triple, Args);
@@ -471,19 +628,48 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Arch == llvm::Triple::aarch64_be ? "-EB" : "-EL");
   }
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
-                   options::OPT_r)) {
-    CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath("crt0.o")));
+  bool NeedCRTs =
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles);
+
+  const char *CRTBegin, *CRTEnd;
+  if (NeedCRTs) {
+    if (!Args.hasArg(options::OPT_r))
+      CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath("crt0.o")));
+    if (TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D)) {
+      auto RuntimeLib = TC.GetRuntimeLibType(Args);
+      switch (RuntimeLib) {
+      case (ToolChain::RLT_Libgcc): {
+        CRTBegin = "crtbegin.o";
+        CRTEnd = "crtend.o";
+        break;
+      }
+      case (ToolChain::RLT_CompilerRT): {
+        CRTBegin =
+            TC.getCompilerRTArgString(Args, "crtbegin", ToolChain::FT_Object);
+        CRTEnd =
+            TC.getCompilerRTArgString(Args, "crtend", ToolChain::FT_Object);
+        break;
+      }
+      }
+      CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(CRTBegin)));
+    }
   }
 
-  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
-                            options::OPT_s, options::OPT_t, options::OPT_r});
+  Args.addAllArgs(CmdArgs,
+                  {options::OPT_L, options::OPT_u, options::OPT_T_Group,
+                   options::OPT_s, options::OPT_t, options::OPT_r});
 
   TC.AddFilePathLibArgs(Args, CmdArgs);
 
   for (const auto &LibPath : TC.getLibraryPaths())
     CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-L", LibPath)));
 
+  if (D.isUsingLTO())
+    addLTOOptions(TC, Args, CmdArgs, Output, Inputs,
+                  D.getLTOMode() == LTOK_Thin);
+
+  AddLinkerInputs(TC, Inputs, Args, CmdArgs, JA);
+
   if (TC.ShouldLinkCXXStdlib(Args)) {
     bool OnlyLibstdcxxStatic = Args.hasArg(options::OPT_static_libstdcxx) &&
                                !Args.hasArg(options::OPT_static);
@@ -496,14 +682,17 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
+    CmdArgs.push_back("--start-group");
     AddRunTimeLibs(TC, D, CmdArgs, Args);
-
     CmdArgs.push_back("-lc");
+    if (TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D))
+      CmdArgs.push_back("-lgloss");
+    CmdArgs.push_back("--end-group");
   }
 
-  if (D.isUsingLTO())
-    addLTOOptions(TC, Args, CmdArgs, Output, Inputs,
-                  D.getLTOMode() == LTOK_Thin);
+  if ((TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D)) &&
+      NeedCRTs)
+    CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(CRTEnd)));
 
   if (TC.getTriple().isRISCV())
     CmdArgs.push_back("-X");
diff --git a/clang/lib/Driver/ToolChains/BareMetal.h b/clang/lib/Driver/ToolChains/BareMetal.h
index f6295bda0a6a2..d3d415b337a0b 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.h
+++ b/clang/lib/Driver/ToolChains/BareMetal.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 
+#include "ToolChains/Gnu.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 
@@ -19,7 +20,7 @@ namespace driver {
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
+class LLVM_LIBRARY_VISIBILITY BareMetal : public Generic_ELF {
 public:
   BareMetal(const Driver &D, const llvm::Triple &Triple,
             const llvm::opt::ArgList &Args);
@@ -35,7 +36,9 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
   Tool *buildStaticLibTool() const override;
 
 public:
-  bool useIntegratedAs() const override { return true; }
+  bool initGCCInstallation(const llvm::Triple &Triple,
+                           const llvm::opt::ArgList &Args);
+  bool hasValidGCCInstallation() const { return IsGCCInstallationValid; }
   bool isBareMetal() const override { return true; }
   bool isCrossCompiling() const override { return true; }
   bool HasNativeLLVMSupport() const override { return true; }
@@ -48,14 +51,16 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
 
   StringRef getOSLibName() const override { return "baremetal"; }
 
-  RuntimeLibType GetDefaultRuntimeLibType() const override {
-    return ToolChain::RLT_CompilerRT;
-  }
-  CXXStdlibType GetDefaultCXXStdlibType() const override {
-    return ToolChain::CST_Libcxx;
+  UnwindTableLevel
+  getDefaultUnwindTableLevel(const llvm::opt::ArgList &Args) const override {
+    return UnwindTableLevel::None;
   }
 
-  const char *getDefaultLinker() const override { return "ld.lld"; }
+  CXXStdlibType GetDefaultCXXStdlibType() const override;
+
+  RuntimeLibType GetDefaultRuntimeLibType() const override;
+
+  UnwindLibType GetUnwindLibType(const llvm::opt::ArgList &Args) const override;
 
   void
   AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
@@ -67,7 +72,11 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
   void AddClangCXXStdlibIncludeArgs(
       const llvm::opt::ArgList &DriverArgs,
       llvm::opt::ArgStringList &CC1Args) const override;
+  void
+  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
+                           llvm::opt::ArgStringList &CC1Args) const override;
   std::string computeSysRoot() const override;
+  std::string getCompilerRTPath() const override;
   SanitizerMask getSupportedSanitizers() const override;
 
   SmallVector<std::string>
@@ -80,6 +89,8 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
 
   std::string SysRoot;
 
+  bool IsGCCInstallationValid;
+
   SmallVector<std::string> MultilibMacroDefines;
 };
 
@@ -104,7 +115,7 @@ class LLVM_LIBRARY_VISIBILITY StaticLibTool : public Tool {
 
 class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
 public:
-  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "ld.lld", TC) {}
+  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "linker", TC) {}
   bool isLinkJob() const override { return true; }
   bool hasIntegratedCPP() const override { return false; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 8a18865b899d2..7b8120da558f2 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1668,7 +1668,8 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   }
 
   // Handle -msve_vector_bits=<bits>
-  if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
+  auto HandleVectorBits = [&](Arg *A, StringRef VScaleMin,
+                              StringRef VScaleMax) {
     StringRef Val = A->getValue();
     const Driver &D = getToolChain().getDriver();
     if (Val == "128" || Val == "256" || Val == "512" || Val == "1024" ||
@@ -1676,22 +1677,31 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
         Val == "1024+" || Val == "2048+") {
       unsigned Bits = 0;
       if (!Val.consume_back("+")) {
-        bool Invalid = Val.getAsInteger(10, Bits); (void)Invalid;
+        bool Invalid = Val.getAsInteger(10, Bits);
+        (void)Invalid;
         assert(!Invalid && "Failed to parse value");
         CmdArgs.push_back(
-            Args.MakeArgString("-mvscale-max=" + llvm::Twine(Bits / 128)));
+            Args.MakeArgString(VScaleMax + llvm::Twine(Bits / 128)));
       }
 
-      bool Invalid = Val.getAsInteger(10, Bits); (void)Invalid;
+      bool Invalid = Val.getAsInteger(10, Bits);
+      (void)Invalid;
       assert(!Invalid && "Failed to parse value");
+
       CmdArgs.push_back(
-          Args.MakeArgString("-mvscale-min=" + llvm::Twine(Bits / 128)));
-    // Silently drop requests for vector-length agnostic code as it's implied.
-    } else if (Val != "scalable")
+          Args.MakeArgString(VScaleMin + llvm::Twine(Bits / 128)));
+    } else if (Val == "scalable") {
+      // Silently drop requests for vector-length agnostic code as it's implied.
+    } else {
       // Handle the unsupported values passed to msve-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
-  }
+    }
+  };
+  if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ))
+    HandleVectorBits(A, "-mvscale-min=", "-mvscale-max=");
+  if (Arg *A = Args.getLastArg(options::OPT_msve_streaming_vector_bits_EQ))
+    HandleVectorBits(A, "-mvscale-streaming-min=", "-mvscale-streaming-max=");
 
   AddAAPCSVolatileBitfieldArgs(Args, CmdArgs);
 
@@ -4631,11 +4641,8 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T,
   }
 
   if (Args.hasFlag(options::OPT_gkey_instructions,
-                   options::OPT_gno_key_instructions, false)) {
+                   options::OPT_gno_key_instructions, false))
     CmdArgs.push_back("-gkey-instructions");
-    CmdArgs.push_back("-mllvm");
-    CmdArgs.push_back("-dwarf-use-key-instructions");
-  }
 
   if (EmitCodeView) {
     CmdArgs.push_back("-gcodeview");
@@ -6028,7 +6035,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -fasynchronous-unwind-tables and -fnon-call-exceptions interact in more
   // complicated ways.
   auto SanitizeArgs = TC.getSanitizerArgs(Args);
-
+  Args.AddLastArg(CmdArgs,
+                  options::OPT_fallow_runtime_check_skip_hot_cutoff_EQ);
   bool IsAsyncUnwindTablesDefault =
       TC.getDefaultUnwindTableLevel(Args) == ToolChain::UnwindTableLevel::Asynchronous;
   bool IsSyncUnwindTablesDefault =
@@ -6283,7 +6291,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fclang_abi_compat_EQ);
 
   if (getLastProfileSampleUseArg(Args) &&
-      Args.hasArg(options::OPT_fsample_profile_use_profi)) {
+      Args.hasFlag(options::OPT_fsample_profile_use_profi,
+                   options::OPT_fno_sample_profile_use_profi, false)) {
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-sample-profile-use-profi");
   }
@@ -7203,6 +7212,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       LanguageStandard = llvm::StringSwitch<StringRef>(StdArg->getValue())
                              .Case("c11", "-std=c11")
                              .Case("c17", "-std=c17")
+                             // TODO: add c23 when MSVC supports it.
+                             .Case("clatest", "-std=c23")
                              .Default("");
       if (LanguageStandard.empty())
         D.Diag(clang::diag::warn_drv_unused_argument)
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 672b73432847d..9ae76fc307a56 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -174,7 +174,13 @@ static bool mustUseNonLeafFramePointerForTarget(const llvm::Triple &Triple) {
 // even if new frame records are not created.
 static bool mustMaintainValidFrameChain(const llvm::opt::ArgList &Args,
                                         const llvm::Triple &Triple) {
-  if (Triple.isARM() || Triple.isThumb()) {
+  switch (Triple.getArch()) {
+  default:
+    return false;
+  case llvm::Triple::arm:
+  case llvm::Triple::armeb:
+  case llvm::Triple::thumb:
+  case llvm::Triple::thumbeb:
     // For 32-bit Arm, the -mframe-chain=aapcs and -mframe-chain=aapcs+leaf
     // options require the frame pointer register to be reserved (or point to a
     // new AAPCS-compilant frame record), even with	-fno-omit-frame-pointer.
@@ -183,8 +189,13 @@ static bool mustMaintainValidFrameChain(const llvm::opt::ArgList &Args,
       return V != "none";
     }
     return false;
+
+  case llvm::Triple::aarch64:
+    // Arm64 Windows requires that the frame chain is valid, as there is no
+    // way to indicate during a stack walk that a frame has used the frame
+    // pointer as a general purpose register.
+    return Triple.isOSWindows();
   }
-  return false;
 }
 
 // True if a target-specific option causes -fno-omit-frame-pointer to also
@@ -527,6 +538,78 @@ void tools::AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
   }
 }
 
+const char *tools::getLDMOption(const llvm::Triple &T, const ArgList &Args) {
+  switch (T.getArch()) {
+  case llvm::Triple::x86:
+    if (T.isOSIAMCU())
+      return "elf_iamcu";
+    return "elf_i386";
+  case llvm::Triple::aarch64:
+    if (T.isOSManagarm())
+      return "aarch64managarm";
+    return "aarch64linux";
+  case llvm::Triple::aarch64_be:
+    return "aarch64linuxb";
+  case llvm::Triple::arm:
+  case llvm::Triple::thumb:
+  case llvm::Triple::armeb:
+  case llvm::Triple::thumbeb:
+    return tools::arm::isARMBigEndian(T, Args) ? "armelfb_linux_eabi"
+                                               : "armelf_linux_eabi";
+  case llvm::Triple::m68k:
+    return "m68kelf";
+  case llvm::Triple::ppc:
+    if (T.isOSLinux())
+      return "elf32ppclinux";
+    return "elf32ppc";
+  case llvm::Triple::ppcle:
+    if (T.isOSLinux())
+      return "elf32lppclinux";
+    return "elf32lppc";
+  case llvm::Triple::ppc64:
+    return "elf64ppc";
+  case llvm::Triple::ppc64le:
+    return "elf64lppc";
+  case llvm::Triple::riscv32:
+    return "elf32lriscv";
+  case llvm::Triple::riscv64:
+    return "elf64lriscv";
+  case llvm::Triple::sparc:
+  case llvm::Triple::sparcel:
+    return "elf32_sparc";
+  case llvm::Triple::sparcv9:
+    return "elf64_sparc";
+  case llvm::Triple::loongarch32:
+    return "elf32loongarch";
+  case llvm::Triple::loongarch64:
+    return "elf64loongarch";
+  case llvm::Triple::mips:
+    return "elf32btsmip";
+  case llvm::Triple::mipsel:
+    return "elf32ltsmip";
+  case llvm::Triple::mips64:
+    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
+      return "elf32btsmipn32";
+    return "elf64btsmip";
+  case llvm::Triple::mips64el:
+    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
+      return "elf32ltsmipn32";
+    return "elf64ltsmip";
+  case llvm::Triple::systemz:
+    return "elf64_s390";
+  case llvm::Triple::x86_64:
+    if (T.isX32())
+      return "elf32_x86_64";
+    return "elf_x86_64";
+  case llvm::Triple::ve:
+    return "elf64ve";
+  case llvm::Triple::csky:
+    return "cskyelf_linux";
+  default:
+    return nullptr;
+  }
+}
+
 void tools::addLinkerCompressDebugSectionsOption(
     const ToolChain &TC, const llvm::opt::ArgList &Args,
     llvm::opt::ArgStringList &CmdArgs) {
@@ -1332,7 +1415,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index b92c18f1b60f5..2373d945ae509 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -810,7 +810,7 @@ Expected<SmallVector<std::string>>
 NVPTXToolChain::getSystemGPUArchs(const ArgList &Args) const {
   // Detect NVIDIA GPUs availible on the system.
   std::string Program;
-  if (Arg *A = Args.getLastArg(options::OPT_nvptx_arch_tool_EQ))
+  if (Arg *A = Args.getLastArg(options::OPT_offload_arch_tool_EQ))
     Program = A->getValue();
   else
     Program = GetProgramPath("nvptx-arch");
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index e4e321ba1e195..75ba2af543c7a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -484,11 +484,18 @@ void Flang::addTargetOptions(const ArgList &Args,
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
-    } else if (Name == "libmvec" || Name == "AMDLIBM") {
+    } else if (Name == "AMDLIBM") {
       if (Triple.getArch() != llvm::Triple::x86 &&
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
+    } else if (Name == "libmvec") {
+      if (Triple.getArch() != llvm::Triple::x86 &&
+          Triple.getArch() != llvm::Triple::x86_64 &&
+          Triple.getArch() != llvm::Triple::aarch64 &&
+          Triple.getArch() != llvm::Triple::aarch64_be)
+        D.Diag(diag::err_drv_unsupported_opt_for_target)
+            << Name << Triple.getArchName();
     } else if (Name == "SLEEF" || Name == "ArmPL") {
       if (Triple.getArch() != llvm::Triple::aarch64 &&
           Triple.getArch() != llvm::Triple::aarch64_be)
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index afce4fffe1d5f..f5e2655857432 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -219,78 +219,6 @@ void tools::gcc::Linker::RenderExtraToolArgs(const JobAction &JA,
   // The types are (hopefully) good enough.
 }
 
-static const char *getLDMOption(const llvm::Triple &T, const ArgList &Args) {
-  switch (T.getArch()) {
-  case llvm::Triple::x86:
-    if (T.isOSIAMCU())
-      return "elf_iamcu";
-    return "elf_i386";
-  case llvm::Triple::aarch64:
-    if (T.isOSManagarm())
-      return "aarch64managarm";
-    return "aarch64linux";
-  case llvm::Triple::aarch64_be:
-    return "aarch64linuxb";
-  case llvm::Triple::arm:
-  case llvm::Triple::thumb:
-  case llvm::Triple::armeb:
-  case llvm::Triple::thumbeb:
-    return tools::arm::isARMBigEndian(T, Args) ? "armelfb_linux_eabi"
-                                               : "armelf_linux_eabi";
-  case llvm::Triple::m68k:
-    return "m68kelf";
-  case llvm::Triple::ppc:
-    if (T.isOSLinux())
-      return "elf32ppclinux";
-    return "elf32ppc";
-  case llvm::Triple::ppcle:
-    if (T.isOSLinux())
-      return "elf32lppclinux";
-    return "elf32lppc";
-  case llvm::Triple::ppc64:
-    return "elf64ppc";
-  case llvm::Triple::ppc64le:
-    return "elf64lppc";
-  case llvm::Triple::riscv32:
-    return "elf32lriscv";
-  case llvm::Triple::riscv64:
-    return "elf64lriscv";
-  case llvm::Triple::sparc:
-  case llvm::Triple::sparcel:
-    return "elf32_sparc";
-  case llvm::Triple::sparcv9:
-    return "elf64_sparc";
-  case llvm::Triple::loongarch32:
-    return "elf32loongarch";
-  case llvm::Triple::loongarch64:
-    return "elf64loongarch";
-  case llvm::Triple::mips:
-    return "elf32btsmip";
-  case llvm::Triple::mipsel:
-    return "elf32ltsmip";
-  case llvm::Triple::mips64:
-    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
-      return "elf32btsmipn32";
-    return "elf64btsmip";
-  case llvm::Triple::mips64el:
-    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
-      return "elf32ltsmipn32";
-    return "elf64ltsmip";
-  case llvm::Triple::systemz:
-    return "elf64_s390";
-  case llvm::Triple::x86_64:
-    if (T.isX32())
-      return "elf32_x86_64";
-    return "elf_x86_64";
-  case llvm::Triple::ve:
-    return "elf64ve";
-  case llvm::Triple::csky:
-    return "cskyelf_linux";
-  default:
-    return nullptr;
-  }
-}
-
 static bool getStaticPIE(const ArgList &Args, const ToolChain &TC) {
   bool HasStaticPIE = Args.hasArg(options::OPT_static_pie);
   if (HasStaticPIE && Args.hasArg(options::OPT_no_pie)) {
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index b8f3a70ee827e..5fe0f85ef09af 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -372,19 +372,22 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
   // Maintain compatability with --hip-device-lib.
   auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
   if (!BCLibArgs.empty()) {
-    llvm::for_each(BCLibArgs, [&](StringRef BCName) {
+    for (StringRef BCName : BCLibArgs) {
       StringRef FullName;
+      bool Found = false;
       for (StringRef LibraryPath : LibraryPaths) {
         SmallString<128> Path(LibraryPath);
         llvm::sys::path::append(Path, BCName);
         FullName = Path;
         if (llvm::sys::fs::exists(FullName)) {
           BCLibs.emplace_back(FullName);
-          return;
+          Found = true;
+          break;
         }
       }
-      getDriver().Diag(diag::err_drv_no_such_file) << BCName;
-    });
+      if (!Found)
+        getDriver().Diag(diag::err_drv_no_such_file) << BCName;
+    }
   } else {
     if (!RocmInstallation->hasDeviceLibrary()) {
       getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
diff --git a/clang/lib/Driver/ToolChains/HIPSPV.cpp b/clang/lib/Driver/ToolChains/HIPSPV.cpp
index c86790f66a79e..53649ca40d99b 100644
--- a/clang/lib/Driver/ToolChains/HIPSPV.cpp
+++ b/clang/lib/Driver/ToolChains/HIPSPV.cpp
@@ -149,11 +149,9 @@ void HIPSPVToolChain::addClangTargetOptions(
     CC1Args.append(
         {"-fvisibility=hidden", "-fapply-global-visibility-to-externs"});
 
-  llvm::for_each(getDeviceLibs(DriverArgs),
-                 [&](const BitCodeLibraryInfo &BCFile) {
-                   CC1Args.append({"-mlink-builtin-bitcode",
-                                   DriverArgs.MakeArgString(BCFile.Path)});
-                 });
+  for (const BitCodeLibraryInfo &BCFile : getDeviceLibs(DriverArgs))
+    CC1Args.append(
+        {"-mlink-builtin-bitcode", DriverArgs.MakeArgString(BCFile.Path)});
 }
 
 Tool *HIPSPVToolChain::buildLinker() const {
@@ -228,7 +226,8 @@ HIPSPVToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
   // Maintain compatability with --hip-device-lib.
   auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
   if (!BCLibArgs.empty()) {
-    llvm::for_each(BCLibArgs, [&](StringRef BCName) {
+    bool Found = false;
+    for (StringRef BCName : BCLibArgs) {
       StringRef FullName;
       for (std::string LibraryPath : LibraryPaths) {
         SmallString<128> Path(LibraryPath);
@@ -236,11 +235,13 @@ HIPSPVToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
         FullName = Path;
         if (llvm::sys::fs::exists(FullName)) {
           BCLibs.emplace_back(FullName.str());
-          return;
+          Found = true;
+          break;
         }
       }
-      getDriver().Diag(diag::err_drv_no_such_file) << BCName;
-    });
+      if (!Found)
+        getDriver().Diag(diag::err_drv_no_such_file) << BCName;
+    }
   } else {
     // Search device library named as 'hipspv-<triple>.bc'.
     auto TT = getTriple().normalize();
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 2b83f408b4c07..38f4643abad98 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -339,7 +339,13 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
       A->claim();
       continue;
     }
-
+    if (A->getOption().getID() == options::OPT_dxc_gis) {
+      // Translate -Gis into -ffp_model_EQ=strict
+      DAL->AddSeparateArg(nullptr, Opts.getOption(options::OPT_ffp_model_EQ),
+                          "strict");
+      A->claim();
+      continue;
+    }
     if (A->getOption().getID() == options::OPT_fvk_use_dx_layout) {
       // This is the only implemented layout so far.
       A->claim();
diff --git a/clang/lib/Driver/ToolChains/Managarm.cpp b/clang/lib/Driver/ToolChains/Managarm.cpp
index ff455f2c6ec70..0f56f0f6aad77 100644
--- a/clang/lib/Driver/ToolChains/Managarm.cpp
+++ b/clang/lib/Driver/ToolChains/Managarm.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "Managarm.h"
-#include "Arch/ARM.h"
 #include "Arch/RISCV.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/CommonArgs.h"
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
deleted file mode 100644
index 42bbc130637be..0000000000000
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ /dev/null
@@ -1,231 +0,0 @@
-//===--- RISCVToolchain.cpp - RISC-V ToolChain Implementations --*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "RISCVToolchain.h"
-#include "clang/Driver/CommonArgs.h"
-#include "clang/Driver/Compilation.h"
-#include "clang/Driver/InputInfo.h"
-#include "clang/Driver/Options.h"
-#include "llvm/Option/ArgList.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Path.h"
-
-using namespace clang::driver;
-using namespace clang::driver::toolchains;
-using namespace clang::driver::tools;
-using namespace clang;
-using namespace llvm::opt;
-
-static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
-                                  const Multilib &Multilib,
-                                  StringRef InstallPath,
-                                  ToolChain::path_list &Paths) {
-  if (const auto &PathsCallback = Multilibs.filePathsCallback())
-    for (const auto &Path : PathsCallback(Multilib))
-      addPathIfExists(D, InstallPath + Path, Paths);
-}
-
-// This function tests whether a gcc installation is present either
-// through gcc-toolchain argument or in the same prefix where clang
-// is installed. This helps decide whether to instantiate this toolchain
-// or Baremetal toolchain.
-bool RISCVToolChain::hasGCCToolchain(const Driver &D,
-                                     const llvm::opt::ArgList &Args) {
-  if (Args.getLastArg(options::OPT_gcc_toolchain))
-    return true;
-
-  SmallString<128> GCCDir;
-  llvm::sys::path::append(GCCDir, D.Dir, "..", D.getTargetTriple(),
-                          "lib/crt0.o");
-  return llvm::sys::fs::exists(GCCDir);
-}
-
-/// RISC-V Toolchain
-RISCVToolChain::RISCVToolChain(const Driver &D, const llvm::Triple &Triple,
-                               const ArgList &Args)
-    : Generic_ELF(D, Triple, Args) {
-  GCCInstallation.init(Triple, Args);
-  if (GCCInstallation.isValid()) {
-    Multilibs = GCCInstallation.getMultilibs();
-    SelectedMultilibs.assign({GCCInstallation.getMultilib()});
-    path_list &Paths = getFilePaths();
-    // Add toolchain/multilib specific file paths.
-    addMultilibsFilePaths(D, Multilibs, SelectedMultilibs.back(),
-                          GCCInstallation.getInstallPath(), Paths);
-    getFilePaths().push_back(GCCInstallation.getInstallPath().str());
-    ToolChain::path_list &PPaths = getProgramPaths();
-    // Multilib cross-compiler GCC installations put ld in a triple-prefixed
-    // directory off of the parent of the GCC installation.
-    PPaths.push_back(Twine(GCCInstallation.getParentLibPath() + "/../" +
-                           GCCInstallation.getTriple().str() + "/bin")
-                         .str());
-    PPaths.push_back((GCCInstallation.getParentLibPath() + "/../bin").str());
-  } else {
-    getProgramPaths().push_back(D.Dir);
-  }
-  getFilePaths().push_back(computeSysRoot() + "/lib");
-}
-
-Tool *RISCVToolChain::buildLinker() const {
-  return new tools::RISCV::Linker(*this);
-}
-
-ToolChain::RuntimeLibType RISCVToolChain::GetDefaultRuntimeLibType() const {
-  return GCCInstallation.isValid() ?
-    ToolChain::RLT_Libgcc : ToolChain::RLT_CompilerRT;
-}
-
-ToolChain::UnwindLibType
-RISCVToolChain::GetUnwindLibType(const llvm::opt::ArgList &Args) const {
-  return ToolChain::UNW_None;
-}
-
-ToolChain::UnwindTableLevel RISCVToolChain::getDefaultUnwindTableLevel(
-    const llvm::opt::ArgList &Args) const {
-  return UnwindTableLevel::None;
-}
-
-void RISCVToolChain::addClangTargetOptions(
-    const llvm::opt::ArgList &DriverArgs,
-    llvm::opt::ArgStringList &CC1Args,
-    Action::OffloadKind) const {
-  CC1Args.push_back("-nostdsysteminc");
-}
-
-void RISCVToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
-                                               ArgStringList &CC1Args) const {
-  if (DriverArgs.hasArg(options::OPT_nostdinc))
-    return;
-
-  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
-    SmallString<128> Dir(getDriver().ResourceDir);
-    llvm::sys::path::append(Dir, "include");
-    addSystemInclude(DriverArgs, CC1Args, Dir.str());
-  }
-
-  if (!DriverArgs.hasArg(options::OPT_nostdlibinc)) {
-    SmallString<128> Dir(computeSysRoot());
-    llvm::sys::path::append(Dir, "include");
-    addSystemInclude(DriverArgs, CC1Args, Dir.str());
-  }
-}
-
-void RISCVToolChain::addLibStdCxxIncludePaths(
-    const llvm::opt::ArgList &DriverArgs,
-    llvm::opt::ArgStringList &CC1Args) const {
-  const GCCVersion &Version = GCCInstallation.getVersion();
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-  const Multilib &Multilib = GCCInstallation.getMultilib();
-  addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
-                           TripleStr, Multilib.includeSuffix(), DriverArgs,
-                           CC1Args);
-}
-
-std::string RISCVToolChain::computeSysRoot() const {
-  if (!getDriver().SysRoot.empty())
-    return getDriver().SysRoot;
-
-  SmallString<128> SysRootDir;
-  if (GCCInstallation.isValid()) {
-    StringRef LibDir = GCCInstallation.getParentLibPath();
-    StringRef TripleStr = GCCInstallation.getTriple().str();
-    llvm::sys::path::append(SysRootDir, LibDir, "..", TripleStr);
-  } else {
-    // Use the triple as provided to the driver. Unlike the parsed triple
-    // this has not been normalized to always contain every field.
-    llvm::sys::path::append(SysRootDir, getDriver().Dir, "..",
-                            getDriver().getTargetTriple());
-  }
-
-  if (!llvm::sys::fs::exists(SysRootDir))
-    return std::string();
-
-  return std::string(SysRootDir);
-}
-
-void RISCV::Linker::ConstructJob(Compilation &C, const JobAction &JA,
-                                 const InputInfo &Output,
-                                 const InputInfoList &Inputs,
-                                 const ArgList &Args,
-                                 const char *LinkingOutput) const {
-  const ToolChain &ToolChain = getToolChain();
-  const Driver &D = ToolChain.getDriver();
-  ArgStringList CmdArgs;
-
-  if (!D.SysRoot.empty())
-    CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
-
-  if (Args.hasArg(options::OPT_mno_relax))
-    CmdArgs.push_back("--no-relax");
-
-  bool IsRV64 = ToolChain.getArch() == llvm::Triple::riscv64;
-  CmdArgs.push_back("-m");
-  if (IsRV64) {
-    CmdArgs.push_back("elf64lriscv");
-  } else {
-    CmdArgs.push_back("elf32lriscv");
-  }
-  CmdArgs.push_back("-X");
-
-  std::string Linker = getToolChain().GetLinkerPath();
-
-  bool WantCRTs =
-      !Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles);
-
-  const char *crtbegin, *crtend;
-  auto RuntimeLib = ToolChain.GetRuntimeLibType(Args);
-  if (RuntimeLib == ToolChain::RLT_Libgcc) {
-    crtbegin = "crtbegin.o";
-    crtend = "crtend.o";
-  } else {
-    assert (RuntimeLib == ToolChain::RLT_CompilerRT);
-    crtbegin = ToolChain.getCompilerRTArgString(Args, "crtbegin",
-                                                ToolChain::FT_Object);
-    crtend = ToolChain.getCompilerRTArgString(Args, "crtend",
-                                              ToolChain::FT_Object);
-  }
-
-  if (WantCRTs) {
-    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crt0.o")));
-    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
-  }
-
-  AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
-
-  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
-
-  ToolChain.AddFilePathLibArgs(Args, CmdArgs);
-  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
-                            options::OPT_t, options::OPT_r});
-
-  // TODO: add C++ includes and libs if compiling C++.
-
-  if (!Args.hasArg(options::OPT_nostdlib) &&
-      !Args.hasArg(options::OPT_nodefaultlibs)) {
-    if (D.CCCIsCXX()) {
-      if (ToolChain.ShouldLinkCXXStdlib(Args))
-        ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
-      CmdArgs.push_back("-lm");
-    }
-    CmdArgs.push_back("--start-group");
-    CmdArgs.push_back("-lc");
-    CmdArgs.push_back("-lgloss");
-    CmdArgs.push_back("--end-group");
-    AddRunTimeLibs(ToolChain, ToolChain.getDriver(), CmdArgs, Args);
-  }
-
-  if (WantCRTs)
-    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtend)));
-
-  CmdArgs.push_back("-o");
-  CmdArgs.push_back(Output.getFilename());
-  C.addCommand(std::make_unique<Command>(
-      JA, *this, ResponseFileSupport::AtFileCurCP(), Args.MakeArgString(Linker),
-      CmdArgs, Inputs, Output));
-}
-// RISCV tools end.
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.h b/clang/lib/Driver/ToolChains/RISCVToolchain.h
deleted file mode 100644
index fa0aa265d842b..0000000000000
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.h
+++ /dev/null
@@ -1,67 +0,0 @@
-//===--- RISCVToolchain.h - RISC-V ToolChain Implementations ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_RISCVTOOLCHAIN_H
-#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_RISCVTOOLCHAIN_H
-
-#include "Gnu.h"
-#include "clang/Driver/ToolChain.h"
-
-namespace clang {
-namespace driver {
-namespace toolchains {
-
-class LLVM_LIBRARY_VISIBILITY RISCVToolChain : public Generic_ELF {
-public:
-  RISCVToolChain(const Driver &D, const llvm::Triple &Triple,
-                 const llvm::opt::ArgList &Args);
-
-  static bool hasGCCToolchain(const Driver &D, const llvm::opt::ArgList &Args);
-  void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
-                             llvm::opt::ArgStringList &CC1Args,
-                             Action::OffloadKind) const override;
-  RuntimeLibType GetDefaultRuntimeLibType() const override;
-  UnwindLibType
-  GetUnwindLibType(const llvm::opt::ArgList &Args) const override;
-  UnwindTableLevel
-  getDefaultUnwindTableLevel(const llvm::opt::ArgList &Args) const override;
-  void
-  AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
-                            llvm::opt::ArgStringList &CC1Args) const override;
-  void
-  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
-                           llvm::opt::ArgStringList &CC1Args) const override;
-
-protected:
-  Tool *buildLinker() const override;
-
-private:
-  std::string computeSysRoot() const override;
-};
-
-} // end namespace toolchains
-
-namespace tools {
-namespace RISCV {
-class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
-public:
-  Linker(const ToolChain &TC) : Tool("RISCV::Linker", "ld", TC) {}
-  bool hasIntegratedCPP() const override { return false; }
-  bool isLinkJob() const override { return true; }
-  void ConstructJob(Compilation &C, const JobAction &JA,
-                    const InputInfo &Output, const InputInfoList &Inputs,
-                    const llvm::opt::ArgList &TCArgs,
-                    const char *LinkingOutput) const override;
-};
-} // end namespace RISCV
-} // end namespace tools
-
-} // end namespace driver
-} // end namespace clang
-
-#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_RISCVTOOLCHAIN_H
diff --git a/clang/lib/Edit/Commit.cpp b/clang/lib/Edit/Commit.cpp
index b1d58a0cad1e4..415b1573d789a 100644
--- a/clang/lib/Edit/Commit.cpp
+++ b/clang/lib/Edit/Commit.cpp
@@ -16,7 +16,6 @@
 #include "clang/Lex/PPConditionalDirectiveRecord.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
-#include <utility>
 
 using namespace clang;
 using namespace edit;
diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index 348e7588690a2..791afc1a97575 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -324,10 +324,15 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
 
   // Declaration fragments of a pointer type is the declaration fragments of
   // the pointee type followed by a `*`,
-  if (T->isPointerType() && !T->isFunctionPointerType())
-    return Fragments
-        .append(getFragmentsForType(T->getPointeeType(), Context, After))
-        .append(" *", DeclarationFragments::FragmentKind::Text);
+  if (T->isPointerType() && !T->isFunctionPointerType()) {
+    QualType PointeeT = T->getPointeeType();
+    Fragments.append(getFragmentsForType(PointeeT, Context, After));
+    // If the pointee is itself a pointer, we do not want to insert a space
+    // before the `*` as the preceding character in the type name is a `*`.
+    if (!PointeeT->isAnyPointerType())
+      Fragments.appendSpace();
+    return Fragments.append("*", DeclarationFragments::FragmentKind::Text);
+  }
 
   // For Objective-C `id` and `Class` pointers
   // we do not spell out the `*`.
@@ -631,7 +636,7 @@ DeclarationFragmentsBuilder::getFragmentsForParam(const ParmVarDecl *Param) {
                 DeclarationFragments::FragmentKind::InternalParam);
   } else {
     Fragments.append(std::move(TypeFragments));
-    if (!T->isBlockPointerType())
+    if (!T->isAnyPointerType() && !T->isBlockPointerType())
       Fragments.appendSpace();
     Fragments
         .append(Param->getName(),
@@ -706,18 +711,20 @@ DeclarationFragmentsBuilder::getFragmentsForFunction(const FunctionDecl *Func) {
 
   // FIXME: Is `after` actually needed here?
   DeclarationFragments After;
+  QualType ReturnType = Func->getReturnType();
   auto ReturnValueFragment =
-      getFragmentsForType(Func->getReturnType(), Func->getASTContext(), After);
+      getFragmentsForType(ReturnType, Func->getASTContext(), After);
   if (StringRef(ReturnValueFragment.begin()->Spelling)
           .starts_with("type-parameter")) {
-    std::string ProperArgName = Func->getReturnType().getAsString();
+    std::string ProperArgName = ReturnType.getAsString();
     ReturnValueFragment.begin()->Spelling.swap(ProperArgName);
   }
 
-  Fragments.append(std::move(ReturnValueFragment))
-      .appendSpace()
-      .append(Func->getNameAsString(),
-              DeclarationFragments::FragmentKind::Identifier);
+  Fragments.append(std::move(ReturnValueFragment));
+  if (!ReturnType->isAnyPointerType())
+    Fragments.appendSpace();
+  Fragments.append(Func->getNameAsString(),
+                   DeclarationFragments::FragmentKind::Identifier);
 
   if (Func->getTemplateSpecializationInfo()) {
     Fragments.append("<", DeclarationFragments::FragmentKind::Text);
@@ -1610,10 +1617,13 @@ DeclarationFragmentsBuilder::getFunctionSignature(const ObjCMethodDecl *);
 DeclarationFragments
 DeclarationFragmentsBuilder::getSubHeading(const NamedDecl *Decl) {
   DeclarationFragments Fragments;
-  if (isa<CXXConstructorDecl>(Decl) || isa<CXXDestructorDecl>(Decl))
+  if (isa<CXXConstructorDecl>(Decl)) {
     Fragments.append(cast<CXXRecordDecl>(Decl->getDeclContext())->getName(),
                      DeclarationFragments::FragmentKind::Identifier);
-  else if (isa<CXXConversionDecl>(Decl)) {
+  } else if (isa<CXXDestructorDecl>(Decl)) {
+    Fragments.append(cast<CXXDestructorDecl>(Decl)->getNameAsString(),
+                     DeclarationFragments::FragmentKind::Identifier);
+  } else if (isa<CXXConversionDecl>(Decl)) {
     Fragments.append(
         cast<CXXConversionDecl>(Decl)->getConversionType().getAsString(),
         DeclarationFragments::FragmentKind::Identifier);
@@ -1627,9 +1637,11 @@ DeclarationFragmentsBuilder::getSubHeading(const NamedDecl *Decl) {
   } else if (Decl->getIdentifier()) {
     Fragments.append(Decl->getName(),
                      DeclarationFragments::FragmentKind::Identifier);
-  } else
+  } else {
     Fragments.append(Decl->getDeclName().getAsString(),
                      DeclarationFragments::FragmentKind::Identifier);
+  }
+
   return Fragments;
 }
 
diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 5317c05f3a460..def0d73e77539 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -158,8 +158,7 @@ getCommentSplit(StringRef Text, unsigned ContentStartColumn,
       return BreakableToken::Split(StringRef::npos, 0);
     StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim(Blanks);
     StringRef AfterCut = Text.substr(SpaceOffset);
-    // Don't trim the leading blanks if it would create a */ after the break.
-    if (!DecorationEndsWithStar || AfterCut.size() <= 1 || AfterCut[1] != '/')
+    if (!DecorationEndsWithStar)
       AfterCut = AfterCut.ltrim(Blanks);
     return BreakableToken::Split(BeforeCut.size(),
                                  AfterCut.begin() - BeforeCut.end());
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index b4745477b96ef..4010f7fbd25be 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1929,6 +1929,15 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
     return;
   }
 
+  const bool EndsInComma = [](const FormatToken *Tok) {
+    if (!Tok)
+      return false;
+    const auto *Prev = Tok->getPreviousNonComment();
+    if (!Prev)
+      return false;
+    return Prev->is(tok::comma);
+  }(Current.MatchingParen);
+
   unsigned NewIndent;
   unsigned LastSpace = CurrentState.LastSpace;
   bool AvoidBinPacking;
@@ -1948,9 +1957,6 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
       NewIndent = CurrentState.LastSpace + Style.ContinuationIndentWidth;
     }
     const FormatToken *NextNonComment = Current.getNextNonComment();
-    bool EndsInComma = Current.MatchingParen &&
-                       Current.MatchingParen->Previous &&
-                       Current.MatchingParen->Previous->is(tok::comma);
     AvoidBinPacking = EndsInComma || Current.is(TT_DictLiteral) ||
                       Style.isProto() || !Style.BinPackArguments ||
                       (NextNonComment && NextNonComment->isOneOf(
@@ -1984,11 +1990,6 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
       LastSpace = std::max(LastSpace, CurrentState.Indent);
     }
 
-    bool EndsInComma =
-        Current.MatchingParen &&
-        Current.MatchingParen->getPreviousNonComment() &&
-        Current.MatchingParen->getPreviousNonComment()->is(tok::comma);
-
     // If ObjCBinPackProtocolList is unspecified, fall back to BinPackParameters
     // for backwards compatibility.
     bool ObjCBinPackProtocolList =
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index bdaf264e9adce..f0412cddc6f19 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3582,7 +3582,7 @@ tooling::Replacements sortJavaImports(const FormatStyle &Style, StringRef Code,
       ImportsInBlock.push_back(
           {Identifier, Line, Prev, AssociatedCommentLines, IsStatic});
       AssociatedCommentLines.clear();
-    } else if (Trimmed.size() > 0 && !ImportsInBlock.empty()) {
+    } else if (!Trimmed.empty() && !ImportsInBlock.empty()) {
       // Associating comments within the imports with the nearest import below
       AssociatedCommentLines.push_back(Line);
     }
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 06f68ec8b0fc1..40b62b2a993d8 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -488,7 +488,7 @@ bool FormatTokenLexer::tryMergeCSharpKeywordVariables() {
 
 // In C# transform identifier foreach into kw_foreach
 bool FormatTokenLexer::tryTransformCSharpForEach() {
-  if (Tokens.size() < 1)
+  if (Tokens.empty())
     return false;
   auto &Identifier = *(Tokens.end() - 1);
   if (Identifier->isNot(tok::identifier))
@@ -948,7 +948,7 @@ void FormatTokenLexer::handleTableGenNumericLikeIdentifier() {
   // 4. The first non-digit character is 'x', and the next is a hex digit.
   // Note that in the case 3 and 4, if the next character does not exists in
   // this token, the token is an identifier.
-  if (Text.size() < 1 || Text[0] == '+' || Text[0] == '-')
+  if (Text.empty() || Text[0] == '+' || Text[0] == '-')
     return;
   const auto NonDigitPos = Text.find_if([](char C) { return !isdigit(C); });
   // All the characters are digits
@@ -1371,8 +1371,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
       } else if (FormatTok->TokenText == "``") {
         FormatTok->Tok.setIdentifierInfo(nullptr);
         FormatTok->Tok.setKind(tok::hashhash);
-      } else if (Tokens.size() > 0 &&
-                 Tokens.back()->is(Keywords.kw_apostrophe) &&
+      } else if (!Tokens.empty() && Tokens.back()->is(Keywords.kw_apostrophe) &&
                  NumberBase.match(FormatTok->TokenText, &Matches)) {
         // In Verilog in a based number literal like `'b10`, there may be
         // whitespace between `'b` and `10`. Therefore we handle the base and
@@ -1420,7 +1419,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
     tryParseJavaTextBlock();
   }
 
-  if (Style.isVerilog() && Tokens.size() > 0 &&
+  if (Style.isVerilog() && !Tokens.empty() &&
       Tokens.back()->is(TT_VerilogNumberBase) &&
       FormatTok->Tok.isOneOf(tok::identifier, tok::question)) {
     // Mark the number following a base like `'h?a0` as a number.
@@ -1454,9 +1453,9 @@ FormatToken *FormatTokenLexer::getNextToken() {
   if (Style.isCpp()) {
     auto *Identifier = FormatTok->Tok.getIdentifierInfo();
     auto it = Macros.find(Identifier);
-    if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() &&
-          Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() ==
-              tok::pp_define) &&
+    if ((Tokens.empty() || !Tokens.back()->Tok.getIdentifierInfo() ||
+         Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() !=
+             tok::pp_define) &&
         it != Macros.end()) {
       FormatTok->setType(it->second);
       if (it->second == TT_IfMacro) {
diff --git a/clang/lib/Format/MacroExpander.cpp b/clang/lib/Format/MacroExpander.cpp
index 37c25e6941ef2..85a53c9bb12fe 100644
--- a/clang/lib/Format/MacroExpander.cpp
+++ b/clang/lib/Format/MacroExpander.cpp
@@ -226,7 +226,7 @@ MacroExpander::expand(FormatToken *ID,
     New->MacroCtx = MacroExpansion(MR_Hidden);
     pushToken(New);
   }
-  assert(Result.size() >= 1 && Result.back()->is(tok::eof));
+  assert(!Result.empty() && Result.back()->is(tok::eof));
   if (Result.size() > 1) {
     ++Result[0]->MacroCtx->StartOfExpansion;
     ++Result[Result.size() - 2]->MacroCtx->EndOfExpansion;
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index b0dda65adfba1..441a37a4902b7 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -640,7 +640,7 @@ bool isPossibleMacro(const FormatToken *Tok) {
     return false;
 
   const auto Text = Tok->TokenText;
-  assert(Text.size() > 0);
+  assert(!Text.empty());
 
   // T,K,U,V likely could be template arguments
   if (Text.size() == 1)
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index f2ed027b2c047..0adf7ee9ed543 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -223,8 +223,6 @@ class LineJoiner {
   tryFitMultipleLinesInOne(LevelIndentTracker &IndentTracker,
                            ArrayRef<AnnotatedLine *>::const_iterator I,
                            ArrayRef<AnnotatedLine *>::const_iterator E) {
-    const unsigned Indent = IndentTracker.getIndent();
-
     // Can't join the last line with anything.
     if (I + 1 == E)
       return 0;
@@ -240,6 +238,7 @@ class LineJoiner {
       return 0;
     }
 
+    const auto Indent = IndentTracker.getIndent();
     if (Style.ColumnLimit > 0 && Indent > Style.ColumnLimit)
       return 0;
 
@@ -986,8 +985,10 @@ class LineJoiner {
   void join(AnnotatedLine &A, const AnnotatedLine &B) {
     assert(!A.Last->Next);
     assert(!B.First->Previous);
-    if (B.Affected)
+    if (B.Affected || B.LeadingEmptyLinesAffected) {
+      assert(B.Affected || A.Last->Children.empty());
       A.Affected = true;
+    }
     A.Last->Next = B.First;
     B.First->Previous = A.Last;
     B.First->CanBreakBefore = true;
@@ -1088,7 +1089,7 @@ class LineFormatter {
     const FormatToken *LBrace = State.NextToken->getPreviousNonComment();
     bool HasLBrace = LBrace && LBrace->is(tok::l_brace) && LBrace->is(BK_Block);
     FormatToken &Previous = *State.NextToken->Previous;
-    if (Previous.Children.size() == 0 || (!HasLBrace && !LBrace->MacroParent)) {
+    if (Previous.Children.empty() || (!HasLBrace && !LBrace->MacroParent)) {
       // The previous token does not open a block. Nothing to do. We don't
       // assert so that we can simply call this function for all tokens.
       return true;
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index c961ce7959750..7e8634aeec4e0 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -4601,7 +4601,7 @@ void UnwrappedLineParser::addUnwrappedLine(LineLevel AdjustLevel) {
   } else {
     // At the top level we only get here when no unexpansion is going on, or
     // when conditional formatting led to unfinished macro reconstructions.
-    assert(!Reconstruct || (CurrentLines != &Lines) || PPStack.size() > 0);
+    assert(!Reconstruct || (CurrentLines != &Lines) || !PPStack.empty());
     CurrentLines->push_back(std::move(*Line));
   }
   Line->Tokens.clear();
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index cfaadf07edfd0..cc3cc0f6906cc 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -314,7 +314,7 @@ AlignTokenSequence(const FormatStyle &Style, unsigned Start, unsigned End,
 
   for (unsigned i = Start; i != End; ++i) {
     auto &CurrentChange = Changes[i];
-    if (ScopeStack.size() != 0 &&
+    if (!ScopeStack.empty() &&
         CurrentChange.indentAndNestingLevel() <
             Changes[ScopeStack.back()].indentAndNestingLevel()) {
       ScopeStack.pop_back();
@@ -332,7 +332,7 @@ AlignTokenSequence(const FormatStyle &Style, unsigned Start, unsigned End,
       ScopeStack.push_back(i);
     }
 
-    bool InsideNestedScope = ScopeStack.size() != 0;
+    bool InsideNestedScope = !ScopeStack.empty();
     bool ContinuedStringLiteral = i > Start &&
                                   CurrentChange.Tok->is(tok::string_literal) &&
                                   Changes[i - 1].Tok->is(tok::string_literal);
@@ -1048,7 +1048,7 @@ void WhitespaceManager::alignChainedConditionals() {
           return C.Tok->is(TT_ConditionalExpr) &&
                  ((C.Tok->is(tok::question) && !C.NewlinesBefore) ||
                   (C.Tok->is(tok::colon) && C.Tok->Next &&
-                   (C.Tok->Next->FakeLParens.size() == 0 ||
+                   (C.Tok->Next->FakeLParens.empty() ||
                     C.Tok->Next->FakeLParens.back() != prec::Conditional)));
         },
         Changes, /*StartAt=*/0);
@@ -1057,7 +1057,7 @@ void WhitespaceManager::alignChainedConditionals() {
       FormatToken *Previous = C.Tok->getPreviousNonComment();
       return C.NewlinesBefore && Previous && Previous->is(TT_ConditionalExpr) &&
              (Previous->is(tok::colon) &&
-              (C.Tok->FakeLParens.size() == 0 ||
+              (C.Tok->FakeLParens.empty() ||
                C.Tok->FakeLParens.back() != prec::Conditional));
     };
     // Ensure we keep alignment of wrapped operands with non-wrapped operands
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 3beff3d6d849f..4ebec8e074330 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -2466,9 +2466,7 @@ void ASTUnit::addFileLevelDecl(Decl *D) {
 
   SourceLocation FileLoc = SM.getFileLoc(Loc);
   assert(SM.isLocalSourceLocation(FileLoc));
-  FileID FID;
-  unsigned Offset;
-  std::tie(FID, Offset) = SM.getDecomposedLoc(FileLoc);
+  auto [FID, Offset] = SM.getDecomposedLoc(FileLoc);
   if (FID.isInvalid())
     return;
 
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 9e269ab244d4c..b9f75796ecc16 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1820,6 +1820,11 @@ void CompilerInvocationBase::GenerateCodeGenArgs(const CodeGenOptions &Opts,
   for (std::string Sanitizer : Values)
     GenerateArg(Consumer, OPT_fsanitize_skip_hot_cutoff_EQ, Sanitizer);
 
+  if (Opts.AllowRuntimeCheckSkipHotCutoff) {
+    GenerateArg(Consumer, OPT_fallow_runtime_check_skip_hot_cutoff_EQ,
+                std::to_string(*Opts.AllowRuntimeCheckSkipHotCutoff));
+  }
+
   for (StringRef Sanitizer :
        serializeSanitizerKinds(Opts.SanitizeAnnotateDebugInfo))
     GenerateArg(Consumer, OPT_fsanitize_annotate_debug_info_EQ, Sanitizer);
@@ -2322,6 +2327,18 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
       Args.getAllArgValues(OPT_fsanitize_annotate_debug_info_EQ), Diags,
       Opts.SanitizeAnnotateDebugInfo);
 
+  if (StringRef V =
+          Args.getLastArgValue(OPT_fallow_runtime_check_skip_hot_cutoff_EQ);
+      !V.empty()) {
+    double A;
+    if (V.getAsDouble(A) || A < 0.0 || A > 1.0) {
+      Diags.Report(diag::err_drv_invalid_value)
+          << "-fallow-runtime-check-skip-hot-cutoff=" << V;
+    } else {
+      Opts.AllowRuntimeCheckSkipHotCutoff = A;
+    }
+  }
+
   Opts.EmitVersionIdentMetadata = Args.hasFlag(OPT_Qy, OPT_Qn, true);
 
   if (!LangOpts->CUDAIsDevice)
@@ -3662,6 +3679,23 @@ static StringRef GetInputKindName(InputKind IK) {
   llvm_unreachable("unknown input language");
 }
 
+static StringRef getExceptionHandlingName(unsigned EHK) {
+  switch (static_cast<LangOptions::ExceptionHandlingKind>(EHK)) {
+  case LangOptions::ExceptionHandlingKind::None:
+    return "none";
+  case LangOptions::ExceptionHandlingKind::DwarfCFI:
+    return "dwarf";
+  case LangOptions::ExceptionHandlingKind::SjLj:
+    return "sjlj";
+  case LangOptions::ExceptionHandlingKind::WinEH:
+    return "seh";
+  case LangOptions::ExceptionHandlingKind::Wasm:
+    return "wasm";
+  }
+
+  llvm_unreachable("covered switch");
+}
+
 void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
                                               ArgumentConsumer Consumer,
                                               const llvm::Triple &T,
@@ -3677,6 +3711,10 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
       GenerateArg(Consumer, OPT_pic_is_pie);
     for (StringRef Sanitizer : serializeSanitizerKinds(Opts.Sanitize))
       GenerateArg(Consumer, OPT_fsanitize_EQ, Sanitizer);
+    if (Opts.ExceptionHandling) {
+      GenerateArg(Consumer, OPT_exception_model,
+                  getExceptionHandlingName(Opts.ExceptionHandling));
+    }
 
     return;
   }
@@ -3853,7 +3891,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -3985,6 +4023,24 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     parseSanitizerKinds("-fsanitize=", Args.getAllArgValues(OPT_fsanitize_EQ),
                         Diags, Opts.Sanitize);
 
+    if (const Arg *A = Args.getLastArg(options::OPT_exception_model)) {
+      std::optional<LangOptions::ExceptionHandlingKind> EMValue =
+          llvm::StringSwitch<std::optional<LangOptions::ExceptionHandlingKind>>(
+              A->getValue())
+              .Case("dwarf", LangOptions::ExceptionHandlingKind::DwarfCFI)
+              .Case("sjlj", LangOptions::ExceptionHandlingKind::SjLj)
+              .Case("seh", LangOptions::ExceptionHandlingKind::WinEH)
+              .Case("wasm", LangOptions::ExceptionHandlingKind::Wasm)
+              .Case("none", LangOptions::ExceptionHandlingKind::None)
+              .Default(std::nullopt);
+      if (EMValue) {
+        Opts.ExceptionHandling = static_cast<unsigned>(*EMValue);
+      } else {
+        Diags.Report(diag::err_drv_invalid_value)
+            << A->getAsString(Args) << A->getValue();
+      }
+    }
+
     return Diags.getNumErrors() == NumErrorsBefore;
   }
 
@@ -4266,7 +4322,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
 
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
@@ -4326,7 +4382,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
@@ -4564,6 +4620,11 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     if (StringRef(A->getValue()).getAsInteger(10, VScaleMin) || VScaleMin == 0)
       Diags.Report(diag::err_cc1_unbounded_vscale_min);
   }
+  if (Arg *A = Args.getLastArg(options::OPT_mvscale_streaming_min_EQ)) {
+    unsigned VScaleMin;
+    if (StringRef(A->getValue()).getAsInteger(10, VScaleMin) || VScaleMin == 0)
+      Diags.Report(diag::err_cc1_unbounded_vscale_min);
+  }
 
   if (const Arg *A = Args.getLastArg(OPT_frandomize_layout_seed_file_EQ)) {
     std::ifstream SeedFile(A->getValue(0));
@@ -5161,11 +5222,14 @@ std::string CompilerInvocation::getModuleHash() const {
   HBuilder.add(serialization::VERSION_MAJOR, serialization::VERSION_MINOR);
 
   // Extend the signature with the language options
-#define LANGOPT(Name, Bits, Default, Description) HBuilder.add(LangOpts->Name);
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)                   \
-  HBuilder.add(static_cast<unsigned>(LangOpts->get##Name()));
-#define BENIGN_LANGOPT(Name, Bits, Default, Description)
-#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
+  // FIXME: Replace with C++20 `using enum LangOptions::CompatibilityKind`.
+  using CK = LangOptions::CompatibilityKind;
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
+  if constexpr (CK::Compatibility != CK::Benign)                               \
+    HBuilder.add(LangOpts->Name);
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
+  if constexpr (CK::Compatibility != CK::Benign)                               \
+    HBuilder.add(static_cast<unsigned>(LangOpts->get##Name()));
 #include "clang/Basic/LangOptions.def"
 
   HBuilder.addRange(getLangOpts().ModuleFeatures);
@@ -5244,13 +5308,17 @@ std::string CompilerInvocation::getModuleHash() const {
 
   // Extend the signature with the affecting debug options.
   if (getHeaderSearchOpts().ModuleFormat == "obj") {
-#define DEBUGOPT(Name, Bits, Default) HBuilder.add(CodeGenOpts->Name);
-#define VALUE_DEBUGOPT(Name, Bits, Default) HBuilder.add(CodeGenOpts->Name);
-#define ENUM_DEBUGOPT(Name, Type, Bits, Default)                               \
-  HBuilder.add(static_cast<unsigned>(CodeGenOpts->get##Name()));
-#define BENIGN_DEBUGOPT(Name, Bits, Default)
-#define BENIGN_VALUE_DEBUGOPT(Name, Bits, Default)
-#define BENIGN_ENUM_DEBUGOPT(Name, Type, Bits, Default)
+    // FIXME: Replace with C++20 `using enum CodeGenOptions::CompatibilityKind`.
+    using CK = CodeGenOptions::CompatibilityKind;
+#define DEBUGOPT(Name, Bits, Default, Compatibility)                           \
+  if constexpr (CK::Compatibility == CK::Affecting)                            \
+    HBuilder.add(CodeGenOpts->Name);
+#define VALUE_DEBUGOPT(Name, Bits, Default, Compatibility)                     \
+  if constexpr (CK::Compatibility == CK::Affecting)                            \
+    HBuilder.add(CodeGenOpts->Name);
+#define ENUM_DEBUGOPT(Name, Type, Bits, Default, Compatibility)                \
+  if constexpr (CK::Compatibility == CK::Affecting)                            \
+    HBuilder.add(static_cast<unsigned>(CodeGenOpts->get##Name()));
 #include "clang/Basic/DebugOptions.def"
   }
 
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index ef7ae27a2694a..f5996a8e1e88b 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -1243,13 +1243,15 @@ llvm::Error FrontendAction::Execute() {
 void FrontendAction::EndSourceFile() {
   CompilerInstance &CI = getCompilerInstance();
 
-  // Inform the diagnostic client we are done with this source file.
-  CI.getDiagnosticClient().EndSourceFile();
-
   // Inform the preprocessor we are done.
   if (CI.hasPreprocessor())
     CI.getPreprocessor().EndSourceFile();
 
+  // Inform the diagnostic client we are done with this source file.
+  // Do this after notifying the preprocessor, so that end-of-file preprocessor
+  // callbacks can report diagnostics.
+  CI.getDiagnosticClient().EndSourceFile();
+
   // Finalize the action.
   EndSourceFileAction();
 
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index c0ab708d8b844..89eb0b689f85c 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -643,16 +643,20 @@ namespace {
     bool ReadLanguageOptions(const LangOptions &LangOpts,
                              StringRef ModuleFilename, bool Complain,
                              bool AllowCompatibleDifferences) override {
+      // FIXME: Replace with C++20 `using enum LangOptions::CompatibilityKind`.
+      using CK = LangOptions::CompatibilityKind;
+
       Out.indent(2) << "Language options:\n";
-#define LANGOPT(Name, Bits, Default, Description) \
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
+    if constexpr (CK::Compatibility != CK::Benign)                             \
       DUMP_BOOLEAN(LangOpts.Name, Description);
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-      Out.indent(4) << Description << ": "                   \
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
+    if constexpr (CK::Compatibility != CK::Benign)                             \
+      Out.indent(4) << Description << ": "                                     \
                     << static_cast<unsigned>(LangOpts.get##Name()) << "\n";
-#define VALUE_LANGOPT(Name, Bits, Default, Description) \
+#define VALUE_LANGOPT(Name, Bits, Default, Compatibility, Description)         \
+    if constexpr (CK::Compatibility != CK::Benign)                             \
       Out.indent(4) << Description << ": " << LangOpts.Name << "\n";
-#define BENIGN_LANGOPT(Name, Bits, Default, Description)
-#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
 #include "clang/Basic/LangOptions.def"
 
       if (!LangOpts.ModuleFeatures.empty()) {
diff --git a/clang/lib/Frontend/SARIFDiagnostic.cpp b/clang/lib/Frontend/SARIFDiagnostic.cpp
index ac27d7480de3e..f9714018f7b46 100644
--- a/clang/lib/Frontend/SARIFDiagnostic.cpp
+++ b/clang/lib/Frontend/SARIFDiagnostic.cpp
@@ -163,36 +163,7 @@ SARIFDiagnostic::addDiagnosticLevelToRule(SarifRule Rule,
 
 llvm::StringRef SARIFDiagnostic::emitFilename(StringRef Filename,
                                               const SourceManager &SM) {
-  if (DiagOpts.AbsolutePath) {
-    auto File = SM.getFileManager().getOptionalFileRef(Filename);
-    if (File) {
-      // We want to print a simplified absolute path, i. e. without "dots".
-      //
-      // The hardest part here are the paths like "<part1>/<link>/../<part2>".
-      // On Unix-like systems, we cannot just collapse "<link>/..", because
-      // paths are resolved sequentially, and, thereby, the path
-      // "<part1>/<part2>" may point to a different location. That is why
-      // we use FileManager::getCanonicalName(), which expands all indirections
-      // with llvm::sys::fs::real_path() and caches the result.
-      //
-      // On the other hand, it would be better to preserve as much of the
-      // original path as possible, because that helps a user to recognize it.
-      // real_path() expands all links, which is sometimes too much. Luckily,
-      // on Windows we can just use llvm::sys::path::remove_dots(), because,
-      // on that system, both aforementioned paths point to the same place.
-#ifdef _WIN32
-      SmallString<256> TmpFilename = File->getName();
-      llvm::sys::fs::make_absolute(TmpFilename);
-      llvm::sys::path::native(TmpFilename);
-      llvm::sys::path::remove_dots(TmpFilename, /* remove_dot_dot */ true);
-      Filename = StringRef(TmpFilename.data(), TmpFilename.size());
-#else
-      Filename = SM.getFileManager().getCanonicalName(*File);
-#endif
-    }
-  }
-
-  return Filename;
+  return SM.getNameForDiagnostic(Filename, DiagOpts);
 }
 
 /// Print out the file/line/column information and include trace.
diff --git a/clang/lib/Frontend/TextDiagnostic.cpp b/clang/lib/Frontend/TextDiagnostic.cpp
index ccdd59da89bd1..e5bf86791c44a 100644
--- a/clang/lib/Frontend/TextDiagnostic.cpp
+++ b/clang/lib/Frontend/TextDiagnostic.cpp
@@ -738,39 +738,7 @@ void TextDiagnostic::printDiagnosticMessage(raw_ostream &OS,
 }
 
 void TextDiagnostic::emitFilename(StringRef Filename, const SourceManager &SM) {
-#ifdef _WIN32
-  SmallString<4096> TmpFilename;
-#endif
-  if (DiagOpts.AbsolutePath) {
-    auto File = SM.getFileManager().getOptionalFileRef(Filename);
-    if (File) {
-      // We want to print a simplified absolute path, i. e. without "dots".
-      //
-      // The hardest part here are the paths like "<part1>/<link>/../<part2>".
-      // On Unix-like systems, we cannot just collapse "<link>/..", because
-      // paths are resolved sequentially, and, thereby, the path
-      // "<part1>/<part2>" may point to a different location. That is why
-      // we use FileManager::getCanonicalName(), which expands all indirections
-      // with llvm::sys::fs::real_path() and caches the result.
-      //
-      // On the other hand, it would be better to preserve as much of the
-      // original path as possible, because that helps a user to recognize it.
-      // real_path() expands all links, which sometimes too much. Luckily,
-      // on Windows we can just use llvm::sys::path::remove_dots(), because,
-      // on that system, both aforementioned paths point to the same place.
-#ifdef _WIN32
-      TmpFilename = File->getName();
-      llvm::sys::fs::make_absolute(TmpFilename);
-      llvm::sys::path::native(TmpFilename);
-      llvm::sys::path::remove_dots(TmpFilename, /* remove_dot_dot */ true);
-      Filename = StringRef(TmpFilename.data(), TmpFilename.size());
-#else
-      Filename = SM.getFileManager().getCanonicalName(*File);
-#endif
-    }
-  }
-
-  OS << Filename;
+  OS << SM.getNameForDiagnostic(Filename, DiagOpts);
 }
 
 /// Print out the file/line/column information and include trace.
diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 6fc587f4de9de..443eb4f1a29bf 100644
--- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -26,7 +26,6 @@
 #include "clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h"
 #include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "llvm/Option/OptTable.h"
-#include "llvm/Option/Option.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index d9382440f4b42..588c283cbdfba 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -392,6 +392,7 @@ __int64 _InterlockedAdd64_nf(__int64 volatile *, __int64);
 __int64 _InterlockedAdd64_rel(__int64 volatile *, __int64);
 __int64 _ReadStatusReg(int);
 void _WriteStatusReg(int, __int64);
+unsigned int __sys(int, __int64);
 
 unsigned short __cdecl _byteswap_ushort(unsigned short val);
 unsigned long __cdecl _byteswap_ulong (unsigned long val);
diff --git a/clang/lib/Headers/opencl-c.h b/clang/lib/Headers/opencl-c.h
index 8d8ef497cec49..e1e0fdeadb0aa 100644
--- a/clang/lib/Headers/opencl-c.h
+++ b/clang/lib/Headers/opencl-c.h
@@ -17482,7 +17482,50 @@ double  __ovld __conv intel_sub_group_shuffle_up( double prev, double cur, uint
 double  __ovld __conv intel_sub_group_shuffle_xor( double, uint );
 #endif
 
-#endif //cl_intel_subgroups
+#if defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||   \
+    defined(cl_intel_subgroups_long)
+
+#if defined(__opencl_c_images)
+uint       __ovld __conv intel_sub_group_block_read_ui(read_only image2d_t, int2);
+uint2      __ovld __conv intel_sub_group_block_read_ui2(read_only image2d_t, int2);
+uint4      __ovld __conv intel_sub_group_block_read_ui4(read_only image2d_t, int2);
+uint8      __ovld __conv intel_sub_group_block_read_ui8(read_only image2d_t, int2);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+uint       __ovld __conv intel_sub_group_block_read_ui(read_write image2d_t, int2);
+uint2      __ovld __conv intel_sub_group_block_read_ui2(read_write image2d_t, int2);
+uint4      __ovld __conv intel_sub_group_block_read_ui4(read_write image2d_t, int2);
+uint8      __ovld __conv intel_sub_group_block_read_ui8(read_write image2d_t, int2);
+#endif // defined(__opencl_c_read_write_images)
+
+uint       __ovld __conv intel_sub_group_block_read_ui( const __global uint* p );
+uint2      __ovld __conv intel_sub_group_block_read_ui2( const __global uint* p );
+uint4      __ovld __conv intel_sub_group_block_read_ui4( const __global uint* p );
+uint8      __ovld __conv intel_sub_group_block_read_ui8( const __global uint* p );
+
+#if defined(__opencl_c_images)
+void       __ovld __conv intel_sub_group_block_write_ui(read_only image2d_t, int2, uint);
+void       __ovld __conv intel_sub_group_block_write_ui2(read_only image2d_t, int2, uint2);
+void       __ovld __conv intel_sub_group_block_write_ui4(read_only image2d_t, int2, uint4);
+void       __ovld __conv intel_sub_group_block_write_ui8(read_only image2d_t, int2, uint8);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+void       __ovld __conv intel_sub_group_block_write_ui(read_write image2d_t, int2, uint);
+void       __ovld __conv intel_sub_group_block_write_ui2(read_write image2d_t, int2, uint2);
+void       __ovld __conv intel_sub_group_block_write_ui4(read_write image2d_t, int2, uint4);
+void       __ovld __conv intel_sub_group_block_write_ui8(read_write image2d_t, int2, uint8);
+#endif // defined(__opencl_c_read_write_images)
+
+void       __ovld __conv intel_sub_group_block_write_ui( __global uint* p, uint data );
+void       __ovld __conv intel_sub_group_block_write_ui2( __global uint* p, uint2 data );
+void       __ovld __conv intel_sub_group_block_write_ui4( __global uint* p, uint4 data );
+void       __ovld __conv intel_sub_group_block_write_ui8( __global uint* p, uint8 data );
+
+#endif // defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||
+       // defined(cl_intel_subgroups_long)
+#endif // cl_intel_subgroups
 
 #if defined(cl_intel_subgroups_short)
 short       __ovld __conv intel_sub_group_broadcast( short , uint sub_group_local_id );
@@ -17574,49 +17617,12 @@ ushort      __ovld __conv intel_sub_group_scan_inclusive_min( ushort  x );
 short       __ovld __conv intel_sub_group_scan_inclusive_max( short   x );
 ushort      __ovld __conv intel_sub_group_scan_inclusive_max( ushort  x );
 
-#if defined(__opencl_c_images)
-uint       __ovld __conv intel_sub_group_block_read_ui(read_only image2d_t, int2);
-uint2      __ovld __conv intel_sub_group_block_read_ui2(read_only image2d_t, int2);
-uint4      __ovld __conv intel_sub_group_block_read_ui4(read_only image2d_t, int2);
-uint8      __ovld __conv intel_sub_group_block_read_ui8(read_only image2d_t, int2);
-#endif // defined(__opencl_c_images)
-
-#if defined(__opencl_c_read_write_images)
-uint       __ovld __conv intel_sub_group_block_read_ui(read_write image2d_t, int2);
-uint2      __ovld __conv intel_sub_group_block_read_ui2(read_write image2d_t, int2);
-uint4      __ovld __conv intel_sub_group_block_read_ui4(read_write image2d_t, int2);
-uint8      __ovld __conv intel_sub_group_block_read_ui8(read_write image2d_t, int2);
-#endif // defined(__opencl_c_read_write_images)
-
-uint       __ovld __conv intel_sub_group_block_read_ui( const __global uint* p );
-uint2      __ovld __conv intel_sub_group_block_read_ui2( const __global uint* p );
-uint4      __ovld __conv intel_sub_group_block_read_ui4( const __global uint* p );
-uint8      __ovld __conv intel_sub_group_block_read_ui8( const __global uint* p );
-
-#if defined(__opencl_c_images)
-void       __ovld __conv intel_sub_group_block_write_ui(read_only image2d_t, int2, uint);
-void       __ovld __conv intel_sub_group_block_write_ui2(read_only image2d_t, int2, uint2);
-void       __ovld __conv intel_sub_group_block_write_ui4(read_only image2d_t, int2, uint4);
-void       __ovld __conv intel_sub_group_block_write_ui8(read_only image2d_t, int2, uint8);
-#endif //defined(__opencl_c_images)
-
-#if defined(__opencl_c_read_write_images)
-void       __ovld __conv intel_sub_group_block_write_ui(read_write image2d_t, int2, uint);
-void       __ovld __conv intel_sub_group_block_write_ui2(read_write image2d_t, int2, uint2);
-void       __ovld __conv intel_sub_group_block_write_ui4(read_write image2d_t, int2, uint4);
-void       __ovld __conv intel_sub_group_block_write_ui8(read_write image2d_t, int2, uint8);
-#endif // defined(__opencl_c_read_write_images)
-
-void       __ovld __conv intel_sub_group_block_write_ui( __global uint* p, uint data );
-void       __ovld __conv intel_sub_group_block_write_ui2( __global uint* p, uint2 data );
-void       __ovld __conv intel_sub_group_block_write_ui4( __global uint* p, uint4 data );
-void       __ovld __conv intel_sub_group_block_write_ui8( __global uint* p, uint8 data );
-
 #if defined(__opencl_c_images)
 ushort      __ovld __conv intel_sub_group_block_read_us(read_only image2d_t, int2);
 ushort2     __ovld __conv intel_sub_group_block_read_us2(read_only image2d_t, int2);
 ushort4     __ovld __conv intel_sub_group_block_read_us4(read_only image2d_t, int2);
 ushort8     __ovld __conv intel_sub_group_block_read_us8(read_only image2d_t, int2);
+ushort16    __ovld __conv intel_sub_group_block_read_us16(read_only image2d_t, int2);
 #endif // defined(__opencl_c_images)
 
 #if defined(__opencl_c_read_write_images)
@@ -17624,18 +17630,21 @@ ushort      __ovld __conv intel_sub_group_block_read_us(read_write image2d_t, in
 ushort2     __ovld __conv intel_sub_group_block_read_us2(read_write image2d_t, int2);
 ushort4     __ovld __conv intel_sub_group_block_read_us4(read_write image2d_t, int2);
 ushort8     __ovld __conv intel_sub_group_block_read_us8(read_write image2d_t, int2);
+ushort16    __ovld __conv intel_sub_group_block_read_us16(read_write image2d_t, int2);
 #endif // defined(__opencl_c_read_write_images)
 
 ushort      __ovld __conv intel_sub_group_block_read_us(  const __global ushort* p );
 ushort2     __ovld __conv intel_sub_group_block_read_us2( const __global ushort* p );
 ushort4     __ovld __conv intel_sub_group_block_read_us4( const __global ushort* p );
 ushort8     __ovld __conv intel_sub_group_block_read_us8( const __global ushort* p );
+ushort16    __ovld __conv intel_sub_group_block_read_us16(const __global ushort* p);
 
 #if defined(__opencl_c_images)
 void        __ovld __conv intel_sub_group_block_write_us(write_only image2d_t, int2, ushort);
 void        __ovld __conv intel_sub_group_block_write_us2(write_only image2d_t, int2, ushort2);
 void        __ovld __conv intel_sub_group_block_write_us4(write_only image2d_t, int2, ushort4);
 void        __ovld __conv intel_sub_group_block_write_us8(write_only image2d_t, int2, ushort8);
+void        __ovld __conv intel_sub_group_block_write_us16(write_only image2d_t, int2, ushort16);
 #endif // defined(__opencl_c_images)
 
 #if defined(__opencl_c_read_write_images)
@@ -17643,14 +17652,261 @@ void        __ovld __conv intel_sub_group_block_write_us(read_write image2d_t, i
 void        __ovld __conv intel_sub_group_block_write_us2(read_write image2d_t, int2, ushort2);
 void        __ovld __conv intel_sub_group_block_write_us4(read_write image2d_t, int2, ushort4);
 void        __ovld __conv intel_sub_group_block_write_us8(read_write image2d_t, int2, ushort8);
+void        __ovld __conv intel_sub_group_block_write_us16(read_write image2d_t, int2, ushort16);
 #endif // defined(__opencl_c_read_write_images)
 
 void        __ovld __conv intel_sub_group_block_write_us(  __global ushort* p, ushort  data );
 void        __ovld __conv intel_sub_group_block_write_us2( __global ushort* p, ushort2 data );
 void        __ovld __conv intel_sub_group_block_write_us4( __global ushort* p, ushort4 data );
 void        __ovld __conv intel_sub_group_block_write_us8( __global ushort* p, ushort8 data );
+void        __ovld __conv intel_sub_group_block_write_us16( __global ushort* p, ushort16 data );
 #endif // cl_intel_subgroups_short
 
+#if defined(cl_intel_subgroups_char)
+char       __ovld __conv intel_sub_group_broadcast( char , uint sub_group_local_id );
+char2      __ovld __conv intel_sub_group_broadcast( char2, uint sub_group_local_id );
+char3      __ovld __conv intel_sub_group_broadcast( char3, uint sub_group_local_id );
+char4      __ovld __conv intel_sub_group_broadcast( char4, uint sub_group_local_id );
+char8      __ovld __conv intel_sub_group_broadcast( char8, uint sub_group_local_id );
+
+uchar      __ovld __conv intel_sub_group_broadcast( uchar , uint sub_group_local_id );
+uchar2     __ovld __conv intel_sub_group_broadcast( uchar2, uint sub_group_local_id );
+uchar3     __ovld __conv intel_sub_group_broadcast( uchar3, uint sub_group_local_id );
+uchar4     __ovld __conv intel_sub_group_broadcast( uchar4, uint sub_group_local_id );
+uchar8     __ovld __conv intel_sub_group_broadcast( uchar8, uint sub_group_local_id );
+
+char       __ovld __conv intel_sub_group_shuffle( char  , uint );
+char2      __ovld __conv intel_sub_group_shuffle( char2 , uint );
+char3      __ovld __conv intel_sub_group_shuffle( char3 , uint );
+char4      __ovld __conv intel_sub_group_shuffle( char4 , uint );
+char8      __ovld __conv intel_sub_group_shuffle( char8 , uint );
+char16     __ovld __conv intel_sub_group_shuffle( char16, uint);
+
+uchar      __ovld __conv intel_sub_group_shuffle( uchar  , uint );
+uchar2     __ovld __conv intel_sub_group_shuffle( uchar2 , uint );
+uchar3     __ovld __conv intel_sub_group_shuffle( uchar3 , uint );
+uchar4     __ovld __conv intel_sub_group_shuffle( uchar4 , uint );
+uchar8     __ovld __conv intel_sub_group_shuffle( uchar8 , uint );
+uchar16    __ovld __conv intel_sub_group_shuffle( uchar16, uint );
+
+char       __ovld __conv intel_sub_group_shuffle_down( char   cur, char   next, uint );
+char2      __ovld __conv intel_sub_group_shuffle_down( char2  cur, char2  next, uint );
+char3      __ovld __conv intel_sub_group_shuffle_down( char3  cur, char3  next, uint );
+char4      __ovld __conv intel_sub_group_shuffle_down( char4  cur, char4  next, uint );
+char8      __ovld __conv intel_sub_group_shuffle_down( char8  cur, char8  next, uint );
+char16     __ovld __conv intel_sub_group_shuffle_down( char16 cur, char16 next, uint );
+
+uchar      __ovld __conv intel_sub_group_shuffle_down( uchar   cur, uchar   next, uint );
+uchar2     __ovld __conv intel_sub_group_shuffle_down( uchar2  cur, uchar2  next, uint );
+uchar3     __ovld __conv intel_sub_group_shuffle_down( uchar3  cur, uchar3  next, uint );
+uchar4     __ovld __conv intel_sub_group_shuffle_down( uchar4  cur, uchar4  next, uint );
+uchar8     __ovld __conv intel_sub_group_shuffle_down( uchar8  cur, uchar8  next, uint );
+uchar16    __ovld __conv intel_sub_group_shuffle_down( uchar16 cur, uchar16 next, uint );
+
+char       __ovld __conv intel_sub_group_shuffle_up( char   cur, char   next, uint );
+char2      __ovld __conv intel_sub_group_shuffle_up( char2  cur, char2  next, uint );
+char3      __ovld __conv intel_sub_group_shuffle_up( char3  cur, char3  next, uint );
+char4      __ovld __conv intel_sub_group_shuffle_up( char4  cur, char4  next, uint );
+char8      __ovld __conv intel_sub_group_shuffle_up( char8  cur, char8  next, uint );
+char16     __ovld __conv intel_sub_group_shuffle_up( char16 cur, char16 next, uint );
+
+uchar      __ovld __conv intel_sub_group_shuffle_up( uchar   cur, uchar   next, uint );
+uchar2     __ovld __conv intel_sub_group_shuffle_up( uchar2  cur, uchar2  next, uint );
+uchar3     __ovld __conv intel_sub_group_shuffle_up( uchar3  cur, uchar3  next, uint );
+uchar4     __ovld __conv intel_sub_group_shuffle_up( uchar4  cur, uchar4  next, uint );
+uchar8     __ovld __conv intel_sub_group_shuffle_up( uchar8  cur, uchar8  next, uint );
+uchar16    __ovld __conv intel_sub_group_shuffle_up( uchar16 cur, uchar16 next, uint );
+
+char       __ovld __conv intel_sub_group_shuffle_xor( char  , uint );
+char2      __ovld __conv intel_sub_group_shuffle_xor( char2 , uint );
+char3      __ovld __conv intel_sub_group_shuffle_xor( char3 , uint );
+char4      __ovld __conv intel_sub_group_shuffle_xor( char4 , uint );
+char8      __ovld __conv intel_sub_group_shuffle_xor( char8 , uint );
+char16     __ovld __conv intel_sub_group_shuffle_xor( char16, uint );
+
+uchar      __ovld __conv intel_sub_group_shuffle_xor( uchar  , uint );
+uchar2     __ovld __conv intel_sub_group_shuffle_xor( uchar2 , uint );
+uchar3     __ovld __conv intel_sub_group_shuffle_xor( uchar3 , uint );
+uchar4     __ovld __conv intel_sub_group_shuffle_xor( uchar4 , uint );
+uchar8     __ovld __conv intel_sub_group_shuffle_xor( uchar8 , uint );
+uchar16    __ovld __conv intel_sub_group_shuffle_xor( uchar16, uint );
+
+char       __ovld __conv intel_sub_group_reduce_add( char   x );
+uchar      __ovld __conv intel_sub_group_reduce_add( uchar  x );
+char       __ovld __conv intel_sub_group_reduce_min( char   x );
+uchar      __ovld __conv intel_sub_group_reduce_min( uchar  x );
+char       __ovld __conv intel_sub_group_reduce_max( char   x );
+uchar      __ovld __conv intel_sub_group_reduce_max( uchar  x );
+
+char       __ovld __conv intel_sub_group_scan_exclusive_add( char   x );
+uchar      __ovld __conv intel_sub_group_scan_exclusive_add( uchar  x );
+char       __ovld __conv intel_sub_group_scan_exclusive_min( char   x );
+uchar      __ovld __conv intel_sub_group_scan_exclusive_min( uchar  x );
+char       __ovld __conv intel_sub_group_scan_exclusive_max( char   x );
+uchar      __ovld __conv intel_sub_group_scan_exclusive_max( uchar  x );
+
+char       __ovld __conv intel_sub_group_scan_inclusive_add( char   x );
+uchar      __ovld __conv intel_sub_group_scan_inclusive_add( uchar  x );
+char       __ovld __conv intel_sub_group_scan_inclusive_min( char   x );
+uchar      __ovld __conv intel_sub_group_scan_inclusive_min( uchar  x );
+char       __ovld __conv intel_sub_group_scan_inclusive_max( char   x );
+uchar      __ovld __conv intel_sub_group_scan_inclusive_max( uchar  x );
+
+#if defined(__opencl_c_images)
+uchar      __ovld __conv intel_sub_group_block_read_uc(read_only image2d_t, int2);
+uchar2     __ovld __conv intel_sub_group_block_read_uc2(read_only image2d_t, int2);
+uchar4     __ovld __conv intel_sub_group_block_read_uc4(read_only image2d_t, int2);
+uchar8     __ovld __conv intel_sub_group_block_read_uc8(read_only image2d_t, int2);
+uchar16    __ovld __conv intel_sub_group_block_read_uc16(read_only image2d_t, int2);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+uchar      __ovld __conv intel_sub_group_block_read_uc(read_write image2d_t, int2);
+uchar2     __ovld __conv intel_sub_group_block_read_uc2(read_write image2d_t, int2);
+uchar4     __ovld __conv intel_sub_group_block_read_uc4(read_write image2d_t, int2);
+uchar8     __ovld __conv intel_sub_group_block_read_uc8(read_write image2d_t, int2);
+uchar16    __ovld __conv intel_sub_group_block_read_uc16(read_write image2d_t, int2);
+#endif // defined(__opencl_c_read_write_images)
+
+uchar      __ovld __conv intel_sub_group_block_read_uc(  const __global uchar* p );
+uchar2     __ovld __conv intel_sub_group_block_read_uc2( const __global uchar* p );
+uchar4     __ovld __conv intel_sub_group_block_read_uc4( const __global uchar* p );
+uchar8     __ovld __conv intel_sub_group_block_read_uc8( const __global uchar* p );
+uchar16    __ovld __conv intel_sub_group_block_read_uc16( const __global uchar* p );
+
+#if defined(__opencl_c_images)
+void        __ovld __conv intel_sub_group_block_write_uc(write_only image2d_t, int2, uchar);
+void        __ovld __conv intel_sub_group_block_write_uc2(write_only image2d_t, int2, uchar2);
+void        __ovld __conv intel_sub_group_block_write_uc4(write_only image2d_t, int2, uchar4);
+void        __ovld __conv intel_sub_group_block_write_uc8(write_only image2d_t, int2, uchar8);
+void        __ovld __conv intel_sub_group_block_write_uc16(write_only image2d_t, int2, uchar16);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+void        __ovld __conv intel_sub_group_block_write_uc(read_write image2d_t, int2, uchar);
+void        __ovld __conv intel_sub_group_block_write_uc2(read_write image2d_t, int2, uchar2);
+void        __ovld __conv intel_sub_group_block_write_uc4(read_write image2d_t, int2, uchar4);
+void        __ovld __conv intel_sub_group_block_write_uc8(read_write image2d_t, int2, uchar8);
+void        __ovld __conv intel_sub_group_block_write_uc16(read_write image2d_t, int2, uchar16);
+#endif // defined(__opencl_c_read_write_images)
+
+void        __ovld __conv intel_sub_group_block_write_uc(  __global uchar* p, uchar  data );
+void        __ovld __conv intel_sub_group_block_write_uc2( __global uchar* p, uchar2 data );
+void        __ovld __conv intel_sub_group_block_write_uc4( __global uchar* p, uchar4 data );
+void        __ovld __conv intel_sub_group_block_write_uc8( __global uchar* p, uchar8 data );
+void        __ovld __conv intel_sub_group_block_write_uc16( __global uchar* p, uchar16 data );
+#endif // cl_intel_subgroups_char
+
+#if defined(cl_intel_subgroups_long)
+#if defined(__opencl_c_images)
+ulong      __ovld __conv intel_sub_group_block_read_ul(read_only image2d_t, int2);
+ulong2     __ovld __conv intel_sub_group_block_read_ul2(read_only image2d_t, int2);
+ulong4     __ovld __conv intel_sub_group_block_read_ul4(read_only image2d_t, int2);
+ulong8     __ovld __conv intel_sub_group_block_read_ul8(read_only image2d_t, int2);
+ulong16    __ovld __conv intel_sub_group_block_read_ul16(read_only image2d_t, int2);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+ulong      __ovld __conv intel_sub_group_block_read_ul(read_write image2d_t, int2);
+ulong2     __ovld __conv intel_sub_group_block_read_ul2(read_write image2d_t, int2);
+ulong4     __ovld __conv intel_sub_group_block_read_ul4(read_write image2d_t, int2);
+ulong8     __ovld __conv intel_sub_group_block_read_ul8(read_write image2d_t, int2);
+ulong16    __ovld __conv intel_sub_group_block_read_ul16(read_write image2d_t, int2);
+#endif // defined(__opencl_c_read_write_images)
+
+ulong      __ovld __conv intel_sub_group_block_read_ul(  const __global ulong* p );
+ulong2     __ovld __conv intel_sub_group_block_read_ul2( const __global ulong* p );
+ulong4     __ovld __conv intel_sub_group_block_read_ul4( const __global ulong* p );
+ulong8     __ovld __conv intel_sub_group_block_read_ul8( const __global ulong* p );
+
+#if defined(__opencl_c_images)
+void        __ovld __conv intel_sub_group_block_write_ul(write_only image2d_t, int2, ulong);
+void        __ovld __conv intel_sub_group_block_write_ul2(write_only image2d_t, int2, ulong2);
+void        __ovld __conv intel_sub_group_block_write_ul4(write_only image2d_t, int2, ulong4);
+void        __ovld __conv intel_sub_group_block_write_ul8(write_only image2d_t, int2, ulong8);
+void        __ovld __conv intel_sub_group_block_write_ul16(write_only image2d_t, int2, ulong16);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+void        __ovld __conv intel_sub_group_block_write_ul(read_write image2d_t, int2, ulong);
+void        __ovld __conv intel_sub_group_block_write_ul2(read_write image2d_t, int2, ulong2);
+void        __ovld __conv intel_sub_group_block_write_ul4(read_write image2d_t, int2, ulong4);
+void        __ovld __conv intel_sub_group_block_write_ul8(read_write image2d_t, int2, ulong8);
+void        __ovld __conv intel_sub_group_block_write_ul16(read_write image2d_t, int2, ulong16);
+#endif // defined(__opencl_c_read_write_images)
+
+void        __ovld __conv intel_sub_group_block_write_ul(  __global ulong* p, ulong  data );
+void        __ovld __conv intel_sub_group_block_write_ul2( __global ulong* p, ulong2 data );
+void        __ovld __conv intel_sub_group_block_write_ul4( __global ulong* p, ulong4 data );
+void        __ovld __conv intel_sub_group_block_write_ul8( __global ulong* p, ulong8 data);
+#endif // cl_intel_subgroups_long
+
+#if defined(cl_intel_subgroup_local_block_io)
+uint    __ovld __conv intel_sub_group_block_read( const __local uint* p );
+uint2   __ovld __conv intel_sub_group_block_read2( const __local uint* p );
+uint4   __ovld __conv intel_sub_group_block_read4( const __local uint* p );
+uint8   __ovld __conv intel_sub_group_block_read8( const __local uint* p );
+
+void    __ovld __conv intel_sub_group_block_write( __local uint* p, uint data );
+void    __ovld __conv intel_sub_group_block_write2( __local uint* p, uint2 data );
+void    __ovld __conv intel_sub_group_block_write4( __local uint* p, uint4 data );
+void    __ovld __conv intel_sub_group_block_write8( __local uint* p, uint8 data );
+
+#if defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||   \
+    defined(cl_intel_subgroups_long)
+uint       __ovld __conv intel_sub_group_block_read_ui( const __local uint* p );
+uint2      __ovld __conv intel_sub_group_block_read_ui2( const __local uint* p );
+uint4      __ovld __conv intel_sub_group_block_read_ui4( const __local uint* p );
+uint8      __ovld __conv intel_sub_group_block_read_ui8( const __local uint* p );
+
+void       __ovld __conv intel_sub_group_block_write_ui( __local uint* p, uint data );
+void       __ovld __conv intel_sub_group_block_write_ui2( __local uint* p, uint2 data );
+void       __ovld __conv intel_sub_group_block_write_ui4( __local uint* p, uint4 data );
+void       __ovld __conv intel_sub_group_block_write_ui8( __local uint* p, uint8 data );
+#endif // defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||
+       // defined(cl_intel_subgroups_long)
+
+#if defined(cl_intel_subgroups_char)
+uchar      __ovld __conv intel_sub_group_block_read_uc(   const __local uchar* p );
+uchar2     __ovld __conv intel_sub_group_block_read_uc2(  const __local uchar* p );
+uchar4     __ovld __conv intel_sub_group_block_read_uc4(  const __local uchar* p );
+uchar8     __ovld __conv intel_sub_group_block_read_uc8(  const __local uchar* p );
+uchar16    __ovld __conv intel_sub_group_block_read_uc16( const __local uchar* p );
+
+void        __ovld __conv intel_sub_group_block_write_uc(   __local uchar* p, uchar  data );
+void        __ovld __conv intel_sub_group_block_write_uc2(  __local uchar* p, uchar2 data );
+void        __ovld __conv intel_sub_group_block_write_uc4(  __local uchar* p, uchar4 data );
+void        __ovld __conv intel_sub_group_block_write_uc8(  __local uchar* p, uchar8 data );
+void        __ovld __conv intel_sub_group_block_write_uc16( __local uchar* p, uchar16 data );
+#endif // defined(cl_intel_subgroups_char)
+
+#if defined(cl_intel_subgroups_short)
+ushort      __ovld __conv intel_sub_group_block_read_us(  const __local ushort* p );
+ushort2     __ovld __conv intel_sub_group_block_read_us2( const __local ushort* p );
+ushort4     __ovld __conv intel_sub_group_block_read_us4( const __local ushort* p );
+ushort8     __ovld __conv intel_sub_group_block_read_us8( const __local ushort* p );
+ushort16    __ovld __conv intel_sub_group_block_read_us16( const __local ushort* p );
+
+void        __ovld __conv intel_sub_group_block_write_us(  __local ushort* p, ushort  data );
+void        __ovld __conv intel_sub_group_block_write_us2( __local ushort* p, ushort2 data );
+void        __ovld __conv intel_sub_group_block_write_us4( __local ushort* p, ushort4 data );
+void        __ovld __conv intel_sub_group_block_write_us8( __local ushort* p, ushort8 data );
+void        __ovld __conv intel_sub_group_block_write_us16( __local ushort* p, ushort16 data );
+#endif // defined(cl_intel_subgroups_short)
+
+#if defined(cl_intel_subgroups_long)
+ulong      __ovld __conv intel_sub_group_block_read_ul(  const __local ulong* p );
+ulong2     __ovld __conv intel_sub_group_block_read_ul2( const __local ulong* p );
+ulong4     __ovld __conv intel_sub_group_block_read_ul4( const __local ulong* p );
+ulong8     __ovld __conv intel_sub_group_block_read_ul8( const __local ulong* p );
+
+void        __ovld __conv intel_sub_group_block_write_ul(  __local ulong* p, ulong  data );
+void        __ovld __conv intel_sub_group_block_write_ul2( __local ulong* p, ulong2 data );
+void        __ovld __conv intel_sub_group_block_write_ul4( __local ulong* p, ulong4 data );
+void        __ovld __conv intel_sub_group_block_write_ul8( __local ulong* p, ulong8 data );
+#endif // defined(cl_intel_subgroups_long)
+#endif // cl_intel_subgroup_local_block_io
+
 #ifdef cl_intel_device_side_avc_motion_estimation
 #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : begin
 
diff --git a/clang/lib/Index/IndexBody.cpp b/clang/lib/Index/IndexBody.cpp
index 98ce6f73ec849..1979117d4695c 100644
--- a/clang/lib/Index/IndexBody.cpp
+++ b/clang/lib/Index/IndexBody.cpp
@@ -69,7 +69,7 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
     while (isa<CastExpr>(*It) || isa<ParenExpr>(*It)) {
       if (auto ICE = dyn_cast<ImplicitCastExpr>(*It)) {
         if (ICE->getCastKind() == CK_LValueToRValue)
-          Roles |= (unsigned)(unsigned)SymbolRole::Read;
+          Roles |= (unsigned)SymbolRole::Read;
       }
       if (It == StmtStack.begin())
         break;
diff --git a/clang/lib/Interpreter/CMakeLists.txt b/clang/lib/Interpreter/CMakeLists.txt
index bf70cdfbee01e..38cf139fa86a6 100644
--- a/clang/lib/Interpreter/CMakeLists.txt
+++ b/clang/lib/Interpreter/CMakeLists.txt
@@ -27,6 +27,7 @@ add_clang_library(clangInterpreter
   Interpreter.cpp
   InterpreterValuePrinter.cpp
   InterpreterUtils.cpp
+  RemoteJITUtils.cpp
   Value.cpp
   ${WASM_SRC}
   PARTIAL_SOURCES_INTENDED
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index 84feff82c63a7..98fc0a5e383a3 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -46,6 +46,7 @@
 #include "clang/Sema/Lookup.h"
 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Errc.h"
@@ -372,8 +373,11 @@ Interpreter::Interpreter(std::unique_ptr<CompilerInstance> Instance,
   auto LLVMCtx = std::make_unique<llvm::LLVMContext>();
   TSCtx = std::make_unique<llvm::orc::ThreadSafeContext>(std::move(LLVMCtx));
 
-  Act = std::make_unique<IncrementalAction>(*CI, *TSCtx->getContext(), ErrOut,
-                                            *this, std::move(Consumer));
+  Act = TSCtx->withContextDo([&](llvm::LLVMContext *Ctx) {
+    return std::make_unique<IncrementalAction>(*CI, *Ctx, ErrOut, *this,
+                                               std::move(Consumer));
+  });
+
   if (ErrOut)
     return;
   CI->ExecuteAction(*Act);
@@ -455,10 +459,11 @@ const char *const Runtimes = R"(
 )";
 
 llvm::Expected<std::unique_ptr<Interpreter>>
-Interpreter::create(std::unique_ptr<CompilerInstance> CI) {
+Interpreter::create(std::unique_ptr<CompilerInstance> CI,
+                    std::unique_ptr<llvm::orc::LLJITBuilder> JB) {
   llvm::Error Err = llvm::Error::success();
-  auto Interp =
-      std::unique_ptr<Interpreter>(new Interpreter(std::move(CI), Err));
+  auto Interp = std::unique_ptr<Interpreter>(
+      new Interpreter(std::move(CI), Err, JB ? std::move(JB) : nullptr));
   if (Err)
     return std::move(Err);
 
@@ -493,10 +498,10 @@ Interpreter::createWithCUDA(std::unique_ptr<CompilerInstance> CI,
   std::unique_ptr<Interpreter> Interp = std::move(*InterpOrErr);
 
   llvm::Error Err = llvm::Error::success();
-  llvm::LLVMContext &LLVMCtx = *Interp->TSCtx->getContext();
 
-  auto DeviceAct =
-      std::make_unique<IncrementalAction>(*DCI, LLVMCtx, Err, *Interp);
+  auto DeviceAct = Interp->TSCtx->withContextDo([&](llvm::LLVMContext *Ctx) {
+    return std::make_unique<IncrementalAction>(*DCI, *Ctx, Err, *Interp);
+  });
 
   if (Err)
     return std::move(Err);
@@ -617,6 +622,25 @@ createJITTargetMachineBuilder(const std::string &TT) {
   return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT));
 }
 
+llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+Interpreter::createLLJITBuilder(
+    std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC,
+    llvm::StringRef OrcRuntimePath) {
+  const std::string &TT = EPC->getTargetTriple().getTriple();
+  auto JTMB = createJITTargetMachineBuilder(TT);
+  if (!JTMB)
+    return JTMB.takeError();
+  auto JB = IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB));
+  if (!JB)
+    return JB.takeError();
+
+  (*JB)->setExecutorProcessControl(std::move(EPC));
+  (*JB)->setPlatformSetUp(
+      llvm::orc::ExecutorNativePlatform(OrcRuntimePath.str()));
+
+  return std::move(*JB);
+}
+
 llvm::Error Interpreter::CreateExecutor() {
   if (IncrExecutor)
     return llvm::make_error<llvm::StringError>("Operation failed. "
@@ -756,11 +780,13 @@ llvm::Error Interpreter::LoadDynamicLibrary(const char *name) {
   if (!EE)
     return EE.takeError();
 
-  auto &DL = EE->getDataLayout();
-
-  if (auto DLSG = llvm::orc::DynamicLibrarySearchGenerator::Load(
-          name, DL.getGlobalPrefix()))
-    EE->getMainJITDylib().addGenerator(std::move(*DLSG));
+  if (llvm::Expected<
+          std::unique_ptr<llvm::orc::EPCDynamicLibrarySearchGenerator>>
+          DLSG = llvm::orc::EPCDynamicLibrarySearchGenerator::Load(
+              EE->getExecutionSession(), name))
+    // FIXME: Eventually we should put each library in its own JITDylib and
+    //        turn off process symbols by default.
+    EE->getProcessSymbolsJITDylib()->addGenerator(std::move(*DLSG));
   else
     return DLSG.takeError();
 #endif
diff --git a/clang/lib/Interpreter/RemoteJITUtils.cpp b/clang/lib/Interpreter/RemoteJITUtils.cpp
new file mode 100644
index 0000000000000..c0e663b764785
--- /dev/null
+++ b/clang/lib/Interpreter/RemoteJITUtils.cpp
@@ -0,0 +1,267 @@
+//===-- RemoteJITUtils.cpp - Utilities for remote-JITing --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// FIXME: Unify this code with similar functionality in llvm-jitlink.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Interpreter/RemoteJITUtils.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
+#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h"
+#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
+#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+#ifdef LLVM_ON_UNIX
+#include <netdb.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#endif // LLVM_ON_UNIX
+
+using namespace llvm;
+using namespace llvm::orc;
+
+Expected<uint64_t> getSlabAllocSize(StringRef SizeString) {
+  SizeString = SizeString.trim();
+
+  uint64_t Units = 1024;
+
+  if (SizeString.ends_with_insensitive("kb"))
+    SizeString = SizeString.drop_back(2).rtrim();
+  else if (SizeString.ends_with_insensitive("mb")) {
+    Units = 1024 * 1024;
+    SizeString = SizeString.drop_back(2).rtrim();
+  } else if (SizeString.ends_with_insensitive("gb")) {
+    Units = 1024 * 1024 * 1024;
+    SizeString = SizeString.drop_back(2).rtrim();
+  }
+
+  uint64_t SlabSize = 0;
+  if (SizeString.getAsInteger(10, SlabSize))
+    return make_error<StringError>("Invalid numeric format for slab size",
+                                   inconvertibleErrorCode());
+
+  return SlabSize * Units;
+}
+
+Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>>
+createSharedMemoryManager(SimpleRemoteEPC &SREPC,
+                          StringRef SlabAllocateSizeString) {
+  SharedMemoryMapper::SymbolAddrs SAs;
+  if (auto Err = SREPC.getBootstrapSymbols(
+          {{SAs.Instance, rt::ExecutorSharedMemoryMapperServiceInstanceName},
+           {SAs.Reserve,
+            rt::ExecutorSharedMemoryMapperServiceReserveWrapperName},
+           {SAs.Initialize,
+            rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName},
+           {SAs.Deinitialize,
+            rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName},
+           {SAs.Release,
+            rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName}}))
+    return std::move(Err);
+
+#ifdef _WIN32
+  size_t SlabSize = 1024 * 1024;
+#else
+  size_t SlabSize = 1024 * 1024 * 1024;
+#endif
+
+  if (!SlabAllocateSizeString.empty()) {
+    if (Expected<uint64_t> S = getSlabAllocSize(SlabAllocateSizeString))
+      SlabSize = *S;
+    else
+      return S.takeError();
+  }
+
+  return MapperJITLinkMemoryManager::CreateWithMapper<SharedMemoryMapper>(
+      SlabSize, SREPC, SAs);
+}
+
+Expected<std::unique_ptr<SimpleRemoteEPC>>
+launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
+               llvm::StringRef SlabAllocateSizeString) {
+#ifndef LLVM_ON_UNIX
+  // FIXME: Add support for Windows.
+  return make_error<StringError>("-" + ExecutablePath +
+                                     " not supported on non-unix platforms",
+                                 inconvertibleErrorCode());
+#elif !LLVM_ENABLE_THREADS
+  // Out of process mode using SimpleRemoteEPC depends on threads.
+  return make_error<StringError>(
+      "-" + ExecutablePath +
+          " requires threads, but LLVM was built with "
+          "LLVM_ENABLE_THREADS=Off",
+      inconvertibleErrorCode());
+#else
+
+  if (!sys::fs::can_execute(ExecutablePath))
+    return make_error<StringError>(
+        formatv("Specified executor invalid: {0}", ExecutablePath),
+        inconvertibleErrorCode());
+
+  constexpr int ReadEnd = 0;
+  constexpr int WriteEnd = 1;
+
+  // Pipe FDs.
+  int ToExecutor[2];
+  int FromExecutor[2];
+
+  pid_t ChildPID;
+
+  // Create pipes to/from the executor..
+  if (pipe(ToExecutor) != 0 || pipe(FromExecutor) != 0)
+    return make_error<StringError>("Unable to create pipe for executor",
+                                   inconvertibleErrorCode());
+
+  ChildPID = fork();
+
+  if (ChildPID == 0) {
+    // In the child...
+
+    // Close the parent ends of the pipes
+    close(ToExecutor[WriteEnd]);
+    close(FromExecutor[ReadEnd]);
+
+    // Execute the child process.
+    std::unique_ptr<char[]> ExecutorPath, FDSpecifier;
+    {
+      ExecutorPath = std::make_unique<char[]>(ExecutablePath.size() + 1);
+      strcpy(ExecutorPath.get(), ExecutablePath.data());
+
+      std::string FDSpecifierStr("filedescs=");
+      FDSpecifierStr += utostr(ToExecutor[ReadEnd]);
+      FDSpecifierStr += ',';
+      FDSpecifierStr += utostr(FromExecutor[WriteEnd]);
+      FDSpecifier = std::make_unique<char[]>(FDSpecifierStr.size() + 1);
+      strcpy(FDSpecifier.get(), FDSpecifierStr.c_str());
+    }
+
+    char *const Args[] = {ExecutorPath.get(), FDSpecifier.get(), nullptr};
+    int RC = execvp(ExecutorPath.get(), Args);
+    if (RC != 0) {
+      errs() << "unable to launch out-of-process executor \""
+             << ExecutorPath.get() << "\"\n";
+      exit(1);
+    }
+  }
+  // else we're the parent...
+
+  // Close the child ends of the pipes
+  close(ToExecutor[ReadEnd]);
+  close(FromExecutor[WriteEnd]);
+
+  SimpleRemoteEPC::Setup S = SimpleRemoteEPC::Setup();
+  if (UseSharedMemory)
+    S.CreateMemoryManager = [SlabAllocateSizeString](SimpleRemoteEPC &EPC) {
+      return createSharedMemoryManager(EPC, SlabAllocateSizeString);
+    };
+
+  return SimpleRemoteEPC::Create<FDSimpleRemoteEPCTransport>(
+      std::make_unique<DynamicThreadPoolTaskDispatcher>(std::nullopt),
+      std::move(S), FromExecutor[ReadEnd], ToExecutor[WriteEnd]);
+#endif
+}
+
+#if LLVM_ON_UNIX && LLVM_ENABLE_THREADS
+
+static Expected<int> connectTCPSocketImpl(std::string Host,
+                                          std::string PortStr) {
+  addrinfo *AI;
+  addrinfo Hints{};
+  Hints.ai_family = AF_INET;
+  Hints.ai_socktype = SOCK_STREAM;
+  Hints.ai_flags = AI_NUMERICSERV;
+
+  if (int EC = getaddrinfo(Host.c_str(), PortStr.c_str(), &Hints, &AI))
+    return make_error<StringError>(
+        formatv("address resolution failed ({0})", gai_strerror(EC)),
+        inconvertibleErrorCode());
+  // Cycle through the returned addrinfo structures and connect to the first
+  // reachable endpoint.
+  int SockFD;
+  addrinfo *Server;
+  for (Server = AI; Server != nullptr; Server = Server->ai_next) {
+    // socket might fail, e.g. if the address family is not supported. Skip to
+    // the next addrinfo structure in such a case.
+    if ((SockFD = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0)
+      continue;
+
+    // If connect returns null, we exit the loop with a working socket.
+    if (connect(SockFD, Server->ai_addr, Server->ai_addrlen) == 0)
+      break;
+
+    close(SockFD);
+  }
+  freeaddrinfo(AI);
+
+  // If we reached the end of the loop without connecting to a valid endpoint,
+  // dump the last error that was logged in socket() or connect().
+  if (Server == nullptr)
+    return make_error<StringError>("invalid hostname",
+                                   inconvertibleErrorCode());
+
+  return SockFD;
+}
+#endif
+
+Expected<std::unique_ptr<SimpleRemoteEPC>>
+connectTCPSocket(StringRef NetworkAddress, bool UseSharedMemory,
+                 llvm::StringRef SlabAllocateSizeString) {
+#ifndef LLVM_ON_UNIX
+  // FIXME: Add TCP support for Windows.
+  return make_error<StringError>("-" + NetworkAddress +
+                                     " not supported on non-unix platforms",
+                                 inconvertibleErrorCode());
+#elif !LLVM_ENABLE_THREADS
+  // Out of process mode using SimpleRemoteEPC depends on threads.
+  return make_error<StringError>(
+      "-" + NetworkAddress +
+          " requires threads, but LLVM was built with "
+          "LLVM_ENABLE_THREADS=Off",
+      inconvertibleErrorCode());
+#else
+
+  auto CreateErr = [NetworkAddress](Twine Details) {
+    return make_error<StringError>(
+        formatv("Failed to connect TCP socket '{0}': {1}", NetworkAddress,
+                Details),
+        inconvertibleErrorCode());
+  };
+
+  StringRef Host, PortStr;
+  std::tie(Host, PortStr) = NetworkAddress.split(':');
+  if (Host.empty())
+    return CreateErr("Host name for -" + NetworkAddress + " can not be empty");
+  if (PortStr.empty())
+    return CreateErr("Port number in -" + NetworkAddress + " can not be empty");
+  int Port = 0;
+  if (PortStr.getAsInteger(10, Port))
+    return CreateErr("Port number '" + PortStr + "' is not a valid integer");
+
+  Expected<int> SockFD = connectTCPSocketImpl(Host.str(), PortStr.str());
+  if (!SockFD)
+    return SockFD.takeError();
+
+  SimpleRemoteEPC::Setup S = SimpleRemoteEPC::Setup();
+  if (UseSharedMemory)
+    S.CreateMemoryManager = [SlabAllocateSizeString](SimpleRemoteEPC &EPC) {
+      return createSharedMemoryManager(EPC, SlabAllocateSizeString);
+    };
+
+  return SimpleRemoteEPC::Create<FDSimpleRemoteEPCTransport>(
+      std::make_unique<DynamicThreadPoolTaskDispatcher>(std::nullopt),
+      std::move(S), *SockFD, *SockFD);
+#endif
+}
diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp
index a862abcc44b38..d894c265a07a2 100644
--- a/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -419,7 +419,6 @@ static bool isQuoteCppDigitSeparator(const char *const Start,
 }
 
 void Scanner::skipLine(const char *&First, const char *const End) {
-  char LastNonWhitespace = ' ';
   for (;;) {
     assert(First <= End);
     if (First == End)
@@ -430,6 +429,9 @@ void Scanner::skipLine(const char *&First, const char *const End) {
       return;
     }
     const char *Start = First;
+    // Use `LastNonWhitespace`to track if a line-continuation has ever been seen
+    // before a new-line character:
+    char LastNonWhitespace = ' ';
     while (First != End && !isVerticalWhitespace(*First)) {
       // Iterate over strings correctly to avoid comments and newlines.
       if (*First == '"' ||
diff --git a/clang/lib/Lex/LexHLSLRootSignature.cpp b/clang/lib/Lex/LexHLSLRootSignature.cpp
index e5de9ad15b07f..a89462c13c8e3 100644
--- a/clang/lib/Lex/LexHLSLRootSignature.cpp
+++ b/clang/lib/Lex/LexHLSLRootSignature.cpp
@@ -27,10 +27,10 @@ RootSignatureToken RootSignatureLexer::lexToken() {
   advanceBuffer(Buffer.take_while(isspace).size());
 
   if (isEndOfBuffer())
-    return RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
+    return RootSignatureToken(TokenKind::end_of_stream, LocOffset);
 
   // Record where this token is in the text for usage in parser diagnostics
-  RootSignatureToken Result(SourceLoc);
+  RootSignatureToken Result(LocOffset);
 
   char C = Buffer.front();
 
@@ -62,7 +62,7 @@ RootSignatureToken RootSignatureLexer::lexToken() {
 
   // All following tokens require at least one additional character
   if (Buffer.size() <= 1) {
-    Result = RootSignatureToken(TokenKind::invalid, SourceLoc);
+    Result = RootSignatureToken(TokenKind::invalid, LocOffset);
     return Result;
   }
 
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index dfdba2317ee89..78988e928a689 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -44,7 +44,6 @@
 #include <optional>
 #include <string>
 #include <tuple>
-#include <utility>
 
 #ifdef __SSE4_2__
 #include <nmmintrin.h>
@@ -930,9 +929,7 @@ static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
   }
 
   // Break down the source locations.
-  FileID FID;
-  unsigned BeginOffs;
-  std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
+  auto [FID, BeginOffs] = SM.getDecomposedLoc(Begin);
   if (FID.isInvalid())
     return {};
 
@@ -4590,6 +4587,9 @@ bool Lexer::LexDependencyDirectiveToken(Token &Result) {
 
   if (Result.is(tok::hash) && Result.isAtStartOfLine()) {
     PP->HandleDirective(Result);
+    if (PP->hadModuleLoaderFatalFailure())
+      // With a fatal failure in the module loader, we abort parsing.
+      return true;
     return false;
   }
   if (Result.is(tok::raw_identifier)) {
diff --git a/clang/lib/Lex/MacroInfo.cpp b/clang/lib/Lex/MacroInfo.cpp
index e4ad97406e995..8b296d5b22748 100644
--- a/clang/lib/Lex/MacroInfo.cpp
+++ b/clang/lib/Lex/MacroInfo.cpp
@@ -22,7 +22,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <optional>
-#include <utility>
 
 using namespace clang;
 
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index e6da19d24f1c5..3fa060f7ec1bd 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -3304,23 +3304,6 @@ void Preprocessor::HandleDefineDirective(
   // If the callbacks want to know, tell them about the macro definition.
   if (Callbacks)
     Callbacks->MacroDefined(MacroNameTok, MD);
-
-  // If we're in MS compatibility mode and the macro being defined is the
-  // assert macro, implicitly add a macro definition for static_assert to work
-  // around their broken assert.h header file in C. Only do so if there isn't
-  // already a static_assert macro defined.
-  if (!getLangOpts().CPlusPlus && getLangOpts().MSVCCompat &&
-      MacroNameTok.getIdentifierInfo()->isStr("assert") &&
-      !isMacroDefined("static_assert")) {
-    MacroInfo *MI = AllocateMacroInfo(SourceLocation());
-
-    Token Tok;
-    Tok.startToken();
-    Tok.setKind(tok::kw__Static_assert);
-    Tok.setIdentifierInfo(getIdentifierInfo("_Static_assert"));
-    MI->setTokens({Tok}, BP);
-    (void)appendDefMacroDirective(getIdentifierInfo("static_assert"), MI);
-  }
 }
 
 /// HandleUndefDirective - Implements \#undef.
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 7e739e09b15e8..c34a1fb6004a9 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2710,6 +2710,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     break;
   }
   case InitKind::Uninitialized: {
+    InitializerScopeRAII InitScope(*this, D, ThisDecl);
     Actions.ActOnUninitializedDecl(ThisDecl);
     break;
   }
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index c1493a5bfd3b3..eae8281964692 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -25,7 +25,6 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
-#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCodeCompletion.h"
@@ -938,9 +937,6 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd) {
     if (!getLangOpts().CPlusPlus) {
       if (getLangOpts().C23)
         Diag(Tok, diag::warn_c23_compat_keyword) << Tok.getName();
-      else
-        Diag(Tok, diag::ext_ms_static_assert) << FixItHint::CreateReplacement(
-            Tok.getLocation(), "_Static_assert");
     } else
       Diag(Tok, diag::warn_cxx98_compat_static_assert);
   }
@@ -4945,19 +4941,16 @@ void Parser::ParseHLSLRootSignatureAttributeArgs(ParsedAttributes &Attrs) {
   }
 
   // Construct our identifier
-  StringRef Signature = StrLiteral.value()->getString();
+  StringLiteral *Signature = StrLiteral.value();
   auto [DeclIdent, Found] =
-      Actions.HLSL().ActOnStartRootSignatureDecl(Signature);
+      Actions.HLSL().ActOnStartRootSignatureDecl(Signature->getString());
   // If we haven't found an already defined DeclIdent then parse the root
   // signature string and construct the in-memory elements
   if (!Found) {
-    // Offset location 1 to account for '"'
-    SourceLocation SignatureLoc =
-        StrLiteral.value()->getExprLoc().getLocWithOffset(1);
     // Invoke the root signature parser to construct the in-memory constructs
-    hlsl::RootSignatureLexer Lexer(Signature, SignatureLoc);
     SmallVector<llvm::hlsl::rootsig::RootElement> RootElements;
-    hlsl::RootSignatureParser Parser(RootElements, Lexer, PP);
+    hlsl::RootSignatureParser Parser(getLangOpts().HLSLRootSigVer, RootElements,
+                                     Signature, PP);
     if (Parser.parse()) {
       T.consumeClose();
       return;
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 3cf3d4ea7d705..bc238a9517a37 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -43,8 +43,9 @@
 #include <optional>
 using namespace clang;
 
-ExprResult Parser::ParseExpression(TypeCastState isTypeCast) {
-  ExprResult LHS(ParseAssignmentExpression(isTypeCast));
+ExprResult
+Parser::ParseExpression(TypoCorrectionTypeBehavior CorrectionBehavior) {
+  ExprResult LHS(ParseAssignmentExpression(CorrectionBehavior));
   return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
@@ -71,7 +72,8 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
   return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
-ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
+ExprResult Parser::ParseAssignmentExpression(
+    TypoCorrectionTypeBehavior CorrectionBehavior) {
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
     Actions.CodeCompletion().CodeCompleteExpression(
@@ -86,7 +88,7 @@ ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
 
   ExprResult LHS =
       ParseCastExpression(CastParseKind::AnyCastExpr,
-                          /*isAddressOfOperand=*/false, isTypeCast);
+                          /*isAddressOfOperand=*/false, CorrectionBehavior);
   return ParseRHSOfBinaryExpression(LHS, prec::Assignment);
 }
 
@@ -98,9 +100,9 @@ ExprResult Parser::ParseConditionalExpression() {
     return ExprError();
   }
 
-  ExprResult LHS = ParseCastExpression(CastParseKind::AnyCastExpr,
-                                       /*isAddressOfOperand=*/false,
-                                       TypeCastState::NotTypeCast);
+  ExprResult LHS = ParseCastExpression(
+      CastParseKind::AnyCastExpr,
+      /*isAddressOfOperand=*/false, TypoCorrectionTypeBehavior::AllowNonTypes);
   return ParseRHSOfBinaryExpression(LHS, prec::Conditional);
 }
 
@@ -116,14 +118,14 @@ Parser::ParseAssignmentExprWithObjCMessageExprStart(SourceLocation LBracLoc,
   return ParseRHSOfBinaryExpression(R, prec::Assignment);
 }
 
-ExprResult
-Parser::ParseConstantExpressionInExprEvalContext(TypeCastState isTypeCast) {
+ExprResult Parser::ParseConstantExpressionInExprEvalContext(
+    TypoCorrectionTypeBehavior CorrectionBehavior) {
   assert(Actions.ExprEvalContexts.back().Context ==
              Sema::ExpressionEvaluationContext::ConstantEvaluated &&
          "Call this function only if your ExpressionEvaluationContext is "
          "already ConstantEvaluated");
-  ExprResult LHS(
-      ParseCastExpression(CastParseKind::AnyCastExpr, false, isTypeCast));
+  ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr, false,
+                                     CorrectionBehavior));
   ExprResult Res(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   return Actions.ActOnConstantExpression(Res);
 }
@@ -135,7 +137,8 @@ ExprResult Parser::ParseConstantExpression() {
   // C++98 and C++11 have no such rule, but this is only a defect in C++98.
   EnterExpressionEvaluationContext ConstantEvaluated(
       Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
-  return ParseConstantExpressionInExprEvalContext(TypeCastState::NotTypeCast);
+  return ParseConstantExpressionInExprEvalContext(
+      TypoCorrectionTypeBehavior::AllowNonTypes);
 }
 
 ExprResult Parser::ParseArrayBoundExpression() {
@@ -163,7 +166,8 @@ ExprResult Parser::ParseArrayBoundExpression() {
       break;
     Iter->InConditionallyConstantEvaluateContext = true;
   }
-  return ParseConstantExpressionInExprEvalContext(TypeCastState::NotTypeCast);
+  return ParseConstantExpressionInExprEvalContext(
+      TypoCorrectionTypeBehavior::AllowNonTypes);
 }
 
 ExprResult Parser::ParseCaseExpression(SourceLocation CaseLoc) {
@@ -171,8 +175,9 @@ ExprResult Parser::ParseCaseExpression(SourceLocation CaseLoc) {
       Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
   Actions.currentEvaluationContext().IsCaseExpr = true;
 
-  ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr, false,
-                                     TypeCastState::NotTypeCast));
+  ExprResult LHS(
+      ParseCastExpression(CastParseKind::AnyCastExpr, false,
+                          TypoCorrectionTypeBehavior::AllowNonTypes));
   ExprResult Res(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   return Actions.ActOnCaseExpr(CaseLoc, Res);
 }
@@ -194,11 +199,10 @@ Parser::ParseConstraintLogicalAndExpression(bool IsTrailingRequiresClause) {
       Actions, Sema::ExpressionEvaluationContext::Unevaluated);
   bool NotPrimaryExpression = false;
   auto ParsePrimary = [&]() {
-    ExprResult E =
-        ParseCastExpression(CastParseKind::PrimaryExprOnly,
-                            /*isAddressOfOperand=*/false,
-                            /*isTypeCast=*/TypeCastState::NotTypeCast,
-                            /*isVectorLiteral=*/false, &NotPrimaryExpression);
+    ExprResult E = ParseCastExpression(
+        CastParseKind::PrimaryExprOnly,
+        /*isAddressOfOperand=*/false, TypoCorrectionTypeBehavior::AllowNonTypes,
+        /*isVectorLiteral=*/false, &NotPrimaryExpression);
     if (E.isInvalid())
       return ExprError();
     auto RecoverFromNonPrimary = [&] (ExprResult E, bool Note) {
@@ -555,18 +559,14 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
   }
 }
 
-ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
-                                       bool isAddressOfOperand,
-                                       TypeCastState isTypeCast,
-                                       bool isVectorLiteral,
-                                       bool *NotPrimaryExpression) {
+ExprResult
+Parser::ParseCastExpression(CastParseKind ParseKind, bool isAddressOfOperand,
+                            TypoCorrectionTypeBehavior CorrectionBehavior,
+                            bool isVectorLiteral, bool *NotPrimaryExpression) {
   bool NotCastExpr;
-  ExprResult Res = ParseCastExpression(ParseKind,
-                                       isAddressOfOperand,
-                                       NotCastExpr,
-                                       isTypeCast,
-                                       isVectorLiteral,
-                                       NotPrimaryExpression);
+  ExprResult Res = ParseCastExpression(ParseKind, isAddressOfOperand,
+                                       NotCastExpr, CorrectionBehavior,
+                                       isVectorLiteral, NotPrimaryExpression);
   if (NotCastExpr)
     Diag(Tok, diag::err_expected_expression);
   return Res;
@@ -574,10 +574,14 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
 namespace {
 class CastExpressionIdValidator final : public CorrectionCandidateCallback {
- public:
-  CastExpressionIdValidator(Token Next, bool AllowTypes, bool AllowNonTypes)
-      : NextToken(Next), AllowNonTypes(AllowNonTypes) {
-    WantTypeSpecifiers = WantFunctionLikeCasts = AllowTypes;
+public:
+  CastExpressionIdValidator(Token Next,
+                            TypoCorrectionTypeBehavior CorrectionBehavior)
+      : NextToken(Next) {
+    WantTypeSpecifiers = WantFunctionLikeCasts =
+        (CorrectionBehavior != TypoCorrectionTypeBehavior::AllowNonTypes);
+    AllowNonTypes =
+        (CorrectionBehavior != TypoCorrectionTypeBehavior::AllowTypes);
   }
 
   bool ValidateCandidate(const TypoCorrection &candidate) override {
@@ -713,12 +717,11 @@ ExprResult Parser::ParseBuiltinPtrauthTypeDiscriminator() {
       /*isType=*/true, Ty.get().getAsOpaquePtr(), SourceRange(Loc, EndLoc));
 }
 
-ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
-                                       bool isAddressOfOperand,
-                                       bool &NotCastExpr,
-                                       TypeCastState isTypeCast,
-                                       bool isVectorLiteral,
-                                       bool *NotPrimaryExpression) {
+ExprResult
+Parser::ParseCastExpression(CastParseKind ParseKind, bool isAddressOfOperand,
+                            bool &NotCastExpr,
+                            TypoCorrectionTypeBehavior CorrectionBehavior,
+                            bool isVectorLiteral, bool *NotPrimaryExpression) {
   ExprResult Res;
   tok::TokenKind SavedKind = Tok.getKind();
   auto SavedType = PreferredType;
@@ -757,9 +760,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     }
     ParsedType CastTy;
     SourceLocation RParenLoc;
-    Res = ParseParenExpression(ParenExprType, false /*stopIfCastExr*/,
-                               isTypeCast == TypeCastState::IsTypeCast, CastTy,
-                               RParenLoc);
+    Res = ParseParenExpression(ParenExprType, /*StopIfCastExr=*/false,
+                               ParenExprKind::Unknown, CorrectionBehavior,
+                               CastTy, RParenLoc);
 
     // FIXME: What should we do if a vector literal is followed by a
     // postfix-expression suffix? Usually postfix operators are permitted on
@@ -842,8 +845,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
   case tok::annot_embed: {
     injectEmbedTokens();
-    return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                               isVectorLiteral, NotPrimaryExpression);
+    return ParseCastExpression(ParseKind, isAddressOfOperand,
+                               CorrectionBehavior, isVectorLiteral,
+                               NotPrimaryExpression);
   }
 
   case tok::kw___super:
@@ -852,8 +856,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     if (TryAnnotateTypeOrScopeToken())
       return ExprError();
     assert(Tok.isNot(tok::kw_decltype) && Tok.isNot(tok::kw___super));
-    return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                               isVectorLiteral, NotPrimaryExpression);
+    return ParseCastExpression(ParseKind, isAddressOfOperand,
+                               CorrectionBehavior, isVectorLiteral,
+                               NotPrimaryExpression);
 
   case tok::identifier:
   ParseIdentifier: {    // primary-expression: identifier
@@ -873,8 +878,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         // indexing
         if (!TryAnnotateTypeOrScopeToken() &&
             Tok.isOneOf(tok::annot_pack_indexing_type, tok::annot_cxxscope))
-          return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                                     isVectorLiteral, NotPrimaryExpression);
+          return ParseCastExpression(ParseKind, isAddressOfOperand,
+                                     CorrectionBehavior, isVectorLiteral,
+                                     NotPrimaryExpression);
       }
 
       // If this identifier was reverted from a token ID, and the next token
@@ -886,9 +892,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         tok::TokenKind Kind;
         if (isRevertibleTypeTrait(II, &Kind)) {
           Tok.setKind(Kind);
-          return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                     NotCastExpr, isTypeCast,
-                                     isVectorLiteral, NotPrimaryExpression);
+          return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
+                                     CorrectionBehavior, isVectorLiteral,
+                                     NotPrimaryExpression);
         }
       }
 
@@ -899,9 +905,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         if (TryAnnotateTypeOrScopeToken())
           return ExprError();
         if (!Tok.is(tok::identifier))
-          return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                     NotCastExpr, isTypeCast,
-                                     isVectorLiteral,
+          return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
+                                     CorrectionBehavior, isVectorLiteral,
                                      NotPrimaryExpression);
       }
     }
@@ -998,10 +1003,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     CXXScopeSpec ScopeSpec;
     SourceLocation TemplateKWLoc;
     Token Replacement;
-    CastExpressionIdValidator Validator(
-        /*Next=*/Tok,
-        /*AllowTypes=*/isTypeCast != TypeCastState::NotTypeCast,
-        /*AllowNonTypes=*/isTypeCast != TypeCastState::IsTypeCast);
+    CastExpressionIdValidator Validator(Tok, CorrectionBehavior);
     Validator.IsAddressOfOperand = isAddressOfOperand;
     if (Tok.isOneOf(tok::periodstar, tok::arrowstar)) {
       Validator.WantExpressionKeywords = false;
@@ -1017,10 +1019,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         Tok.is(tok::r_paren) ? nullptr : &Replacement);
     if (!Res.isInvalid() && Res.isUnset()) {
       UnconsumeToken(Replacement);
-      return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                 NotCastExpr, isTypeCast,
-                                 /*isVectorLiteral=*/false,
-                                 NotPrimaryExpression);
+      return ParseCastExpression(
+          ParseKind, isAddressOfOperand, NotCastExpr, CorrectionBehavior,
+          /*isVectorLiteral=*/false, NotPrimaryExpression);
     }
     Res = tryParseCXXPackIndexingExpression(Res);
     if (!Res.isInvalid() && Tok.is(tok::less))
@@ -1102,10 +1103,11 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     // One special case is implicitly handled here: if the preceding tokens are
     // an ambiguous cast expression, such as "(T())++", then we recurse to
     // determine whether the '++' is prefix or postfix.
-    Res = ParseCastExpression(
-        getLangOpts().CPlusPlus ? CastParseKind::UnaryExprOnly
-                                : CastParseKind::AnyCastExpr,
-        /*isAddressOfOperand*/ false, NotCastExpr, TypeCastState::NotTypeCast);
+    Res = ParseCastExpression(getLangOpts().CPlusPlus
+                                  ? CastParseKind::UnaryExprOnly
+                                  : CastParseKind::AnyCastExpr,
+                              /*isAddressOfOperand*/ false, NotCastExpr,
+                              TypoCorrectionTypeBehavior::AllowNonTypes);
     if (NotCastExpr) {
       // If we return with NotCastExpr = true, we must not consume any tokens,
       // so put the token back where we found it.
@@ -1366,7 +1368,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
       return ExprError();
     if (!Tok.is(tok::annot_cxxscope))
       return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
-                                 isTypeCast, isVectorLiteral,
+                                 CorrectionBehavior, isVectorLiteral,
                                  NotPrimaryExpression);
 
     Token Next = NextToken();
@@ -1382,7 +1384,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
                                        /*EnteringContext=*/false);
         AnnotateTemplateIdTokenAsType(SS, ImplicitTypenameContext::Yes);
         return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
-                                   isTypeCast, isVectorLiteral,
+                                   CorrectionBehavior, isVectorLiteral,
                                    NotPrimaryExpression);
       }
     }
@@ -1400,8 +1402,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
       // expression.
       CXXScopeSpec SS;
       AnnotateTemplateIdTokenAsType(SS, ImplicitTypenameContext::Yes);
-      return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                 NotCastExpr, isTypeCast, isVectorLiteral,
+      return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
+                                 CorrectionBehavior, isVectorLiteral,
                                  NotPrimaryExpression);
     }
 
@@ -1419,8 +1421,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     if (TryAnnotateTypeOrScopeToken())
       return ExprError();
     if (!Tok.is(tok::coloncolon))
-      return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                                 isVectorLiteral, NotPrimaryExpression);
+      return ParseCastExpression(ParseKind, isAddressOfOperand,
+                                 CorrectionBehavior, isVectorLiteral,
+                                 NotPrimaryExpression);
 
     // ::new -> [C++] new-expression
     // ::delete -> [C++] delete-expression
@@ -2110,12 +2113,27 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok,
     // If it starts with a '(', we know that it is either a parenthesized
     // type-name, or it is a unary-expression that starts with a compound
     // literal, or starts with a primary-expression that is a parenthesized
-    // expression.
+    // expression. Most unary operators have an expression form without parens
+    // as part of the grammar for the operator, and a type form with the parens
+    // as part of the grammar for the operator. However, typeof and
+    // typeof_unqual require parens for both forms. This means that we *know*
+    // that the open and close parens cannot be part of a cast expression,
+    // which means we definitely are not parsing a compound literal expression.
+    // This disambiguates a case like enum E : typeof(int) { }; where we've
+    // parsed typeof and need to handle the (int){} tokens properly despite
+    // them looking like a compound literal, as in sizeof (int){}; where the
+    // parens could be part of a parenthesized type name or for a cast
+    // expression of some kind.
+    bool ParenKnownToBeNonCast =
+        OpTok.isOneOf(tok::kw_typeof, tok::kw_typeof_unqual);
     ParenParseOption ExprType = ParenParseOption::CastExpr;
     SourceLocation LParenLoc = Tok.getLocation(), RParenLoc;
 
-    Operand = ParseParenExpression(ExprType, true/*stopIfCastExpr*/,
-                                   false, CastTy, RParenLoc);
+    Operand = ParseParenExpression(
+        ExprType, /*StopIfCastExr=*/true,
+        ParenKnownToBeNonCast ? ParenExprKind::PartOfOperator
+                              : ParenExprKind::Unknown,
+        TypoCorrectionTypeBehavior::AllowBoth, CastTy, RParenLoc);
     CastRange = SourceRange(LParenLoc, RParenLoc);
 
     // If ParseParenExpression parsed a '(typename)' sequence only, then this is
@@ -2590,9 +2608,10 @@ bool Parser::tryParseOpenMPArrayShapingCastPart() {
 }
 
 ExprResult
-Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
-                             bool isTypeCast, ParsedType &CastTy,
-                             SourceLocation &RParenLoc) {
+Parser::ParseParenExpression(ParenParseOption &ExprType, bool StopIfCastExpr,
+                             ParenExprKind ParenBehavior,
+                             TypoCorrectionTypeBehavior CorrectionBehavior,
+                             ParsedType &CastTy, SourceLocation &RParenLoc) {
   assert(Tok.is(tok::l_paren) && "Not a paren expr!");
   ColonProtectionRAIIObject ColonProtection(*this, false);
   BalancedDelimiterTracker T(*this, tok::l_paren);
@@ -2714,7 +2733,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     // in which case we should treat it as type-id.
     // if stopIfCastExpr is false, we need to determine the context past the
     // parens, so we defer to ParseCXXAmbiguousParenExpression for that.
-    if (isAmbiguousTypeId && !stopIfCastExpr) {
+    if (isAmbiguousTypeId && !StopIfCastExpr) {
       ExprResult res = ParseCXXAmbiguousParenExpression(ExprType, CastTy, T,
                                                         ColonProtection);
       RParenLoc = T.getCloseLocation();
@@ -2747,7 +2766,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       T.consumeClose();
       ColonProtection.restore();
       RParenLoc = T.getCloseLocation();
-      if (Tok.is(tok::l_brace)) {
+      if (ParenBehavior == ParenExprKind::Unknown && Tok.is(tok::l_brace)) {
         ExprType = ParenParseOption::CompoundLiteral;
         TypeResult Ty;
         {
@@ -2757,7 +2776,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
         return ParseCompoundLiteralExpression(Ty.get(), OpenLoc, RParenLoc);
       }
 
-      if (Tok.is(tok::l_paren)) {
+      if (ParenBehavior == ParenExprKind::Unknown && Tok.is(tok::l_paren)) {
         // This could be OpenCL vector Literals
         if (getLangOpts().OpenCL)
         {
@@ -2781,7 +2800,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
             Result = ParseCastExpression(
                 /*isUnaryExpression=*/CastParseKind::AnyCastExpr,
                 /*isAddressOfOperand=*/false,
-                /*isTypeCast=*/TypeCastState::IsTypeCast,
+                TypoCorrectionTypeBehavior::AllowTypes,
                 /*isVectorLiteral=*/true);
 
             if (!Result.isInvalid()) {
@@ -2808,7 +2827,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
 
         // Note that this doesn't parse the subsequent cast-expression, it just
         // returns the parsed type to the callee.
-        if (stopIfCastExpr) {
+        if (StopIfCastExpr) {
           TypeResult Ty;
           {
             InMessageExpressionRAIIObject InMessage(*this, false);
@@ -2834,7 +2853,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
         Result = ParseCastExpression(
             /*isUnaryExpression=*/CastParseKind::AnyCastExpr,
             /*isAddressOfOperand=*/false,
-            /*isTypeCast=*/TypeCastState::IsTypeCast);
+            TypoCorrectionTypeBehavior::AllowTypes);
         if (!Result.isInvalid()) {
           Result = Actions.ActOnCastExpr(getCurScope(), OpenLoc,
                                          DeclaratorInfo, CastTy,
@@ -2850,7 +2869,8 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
              isFoldOperator(NextToken().getKind())) {
     ExprType = ParenParseOption::FoldExpr;
     return ParseFoldExpression(ExprResult(), T);
-  } else if (isTypeCast) {
+  } else if (CorrectionBehavior == TypoCorrectionTypeBehavior::AllowTypes) {
+    // FIXME: This should not be predicated on typo correction behavior.
     // Parse the expression-list.
     InMessageExpressionRAIIObject InMessage(*this, false);
     ExprVector ArgExprs;
@@ -2902,7 +2922,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
   } else {
     InMessageExpressionRAIIObject InMessage(*this, false);
 
-    Result = ParseExpression(TypeCastState::MaybeTypeCast);
+    Result = ParseExpression(TypoCorrectionTypeBehavior::AllowBoth);
     if (ExprType >= ParenParseOption::FoldExpr &&
         isFoldOperator(Tok.getKind()) && NextToken().is(tok::ellipsis)) {
       ExprType = ParenParseOption::FoldExpr;
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 1ea0cf52933f6..8dce2268082d9 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1931,15 +1931,13 @@ Parser::ParseCXXCondition(StmtResult *InitStmt, SourceLocation Loc,
       return ParseCXXCondition(nullptr, Loc, CK, MissingOK);
     }
 
-    ExprResult Expr = [&] {
-      EnterExpressionEvaluationContext Eval(
-          Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated,
-          /*LambdaContextDecl=*/nullptr,
-          /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
-          /*ShouldEnter=*/CK == Sema::ConditionKind::ConstexprIf);
-      // Parse the expression.
-      return ParseExpression(); // expression
-    }();
+    EnterExpressionEvaluationContext Eval(
+        Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated,
+        /*LambdaContextDecl=*/nullptr,
+        /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
+        /*ShouldEnter=*/CK == Sema::ConditionKind::ConstexprIf);
+
+    ExprResult Expr = ParseExpression();
 
     if (Expr.isInvalid())
       return Sema::ConditionError();
@@ -3607,7 +3605,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
       Result = ParseCastExpression(CastParseKind::AnyCastExpr,
                                    false /*isAddressofOperand*/, NotCastExpr,
                                    // type-id has priority.
-                                   TypeCastState::IsTypeCast);
+                                   TypoCorrectionTypeBehavior::AllowTypes);
     }
 
     // If we parsed a cast-expression, it's really a type-id, otherwise it's
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index e510302c3aae0..cf86c62f3b671 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -17,10 +17,12 @@ namespace hlsl {
 
 using TokenKind = RootSignatureToken::Kind;
 
-RootSignatureParser::RootSignatureParser(SmallVector<RootElement> &Elements,
-                                         RootSignatureLexer &Lexer,
-                                         Preprocessor &PP)
-    : Elements(Elements), Lexer(Lexer), PP(PP), CurToken(SourceLocation()) {}
+RootSignatureParser::RootSignatureParser(
+    llvm::dxbc::RootSignatureVersion Version,
+    SmallVector<RootElement> &Elements, StringLiteral *Signature,
+    Preprocessor &PP)
+    : Version(Version), Elements(Elements), Signature(Signature),
+      Lexer(Signature->getString()), PP(PP), CurToken(0) {}
 
 bool RootSignatureParser::parse() {
   // Iterate as many RootElements as possible
@@ -76,7 +78,7 @@ static FlagType maybeOrFlag(std::optional<FlagType> Flags, FlagType Flag) {
                                llvm::to_underlying(Flag));
 }
 
-std::optional<RootFlags> RootSignatureParser::parseRootFlags() {
+std::optional<llvm::dxbc::RootFlags> RootSignatureParser::parseRootFlags() {
   assert(CurToken.TokKind == TokenKind::kw_RootFlags &&
          "Expects to only be invoked starting at given keyword");
 
@@ -84,12 +86,12 @@ std::optional<RootFlags> RootSignatureParser::parseRootFlags() {
                            CurToken.TokKind))
     return std::nullopt;
 
-  std::optional<RootFlags> Flags = RootFlags::None;
+  std::optional<llvm::dxbc::RootFlags> Flags = llvm::dxbc::RootFlags::None;
 
   // Handle the edge-case of '0' to specify no flags set
   if (tryConsumeExpectedToken(TokenKind::int_literal)) {
     if (!verifyZeroFlag()) {
-      getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_non_zero_flag);
+      reportDiag(diag::err_hlsl_rootsig_non_zero_flag);
       return std::nullopt;
     }
   } else {
@@ -104,7 +106,8 @@ std::optional<RootFlags> RootSignatureParser::parseRootFlags() {
         switch (CurToken.TokKind) {
 #define ROOT_FLAG_ENUM(NAME, LIT)                                              \
   case TokenKind::en_##NAME:                                                   \
-    Flags = maybeOrFlag<RootFlags>(Flags, RootFlags::NAME);                    \
+    Flags = maybeOrFlag<llvm::dxbc::RootFlags>(Flags,                          \
+                                               llvm::dxbc::RootFlags::NAME);   \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
         default:
@@ -138,7 +141,7 @@ std::optional<RootConstants> RootSignatureParser::parseRootConstants() {
 
   // Check mandatory parameters where provided
   if (!Params->Num32BitConstants.has_value()) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
+    reportDiag(diag::err_hlsl_rootsig_missing_param)
         << TokenKind::kw_num32BitConstants;
     return std::nullopt;
   }
@@ -146,8 +149,7 @@ std::optional<RootConstants> RootSignatureParser::parseRootConstants() {
   Constants.Num32BitConstants = Params->Num32BitConstants.value();
 
   if (!Params->Reg.has_value()) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
-        << TokenKind::bReg;
+    reportDiag(diag::err_hlsl_rootsig_missing_param) << TokenKind::bReg;
     return std::nullopt;
   }
 
@@ -198,7 +200,7 @@ std::optional<RootDescriptor> RootSignatureParser::parseRootDescriptor() {
     ExpectedReg = TokenKind::uReg;
     break;
   }
-  Descriptor.setDefaultFlags();
+  Descriptor.setDefaultFlags(Version);
 
   auto Params = parseRootDescriptorParams(ExpectedReg);
   if (!Params.has_value())
@@ -206,8 +208,7 @@ std::optional<RootDescriptor> RootSignatureParser::parseRootDescriptor() {
 
   // Check mandatory parameters were provided
   if (!Params->Reg.has_value()) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
-        << ExpectedReg;
+    reportDiag(diag::err_hlsl_rootsig_missing_param) << ExpectedReg;
     return std::nullopt;
   }
 
@@ -240,7 +241,7 @@ std::optional<DescriptorTable> RootSignatureParser::parseDescriptorTable() {
     return std::nullopt;
 
   DescriptorTable Table;
-  std::optional<ShaderVisibility> Visibility;
+  std::optional<llvm::dxbc::ShaderVisibility> Visibility;
 
   // Iterate as many Clauses as possible
   do {
@@ -255,8 +256,7 @@ std::optional<DescriptorTable> RootSignatureParser::parseDescriptorTable() {
 
     if (tryConsumeExpectedToken(TokenKind::kw_visibility)) {
       if (Visibility.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -317,7 +317,7 @@ RootSignatureParser::parseDescriptorTableClause() {
     ExpectedReg = TokenKind::sReg;
     break;
   }
-  Clause.setDefaultFlags();
+  Clause.setDefaultFlags(Version);
 
   auto Params = parseDescriptorTableClauseParams(ExpectedReg);
   if (!Params.has_value())
@@ -325,8 +325,7 @@ RootSignatureParser::parseDescriptorTableClause() {
 
   // Check mandatory parameters were provided
   if (!Params->Reg.has_value()) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
-        << ExpectedReg;
+    reportDiag(diag::err_hlsl_rootsig_missing_param) << ExpectedReg;
     return std::nullopt;
   }
 
@@ -369,8 +368,7 @@ std::optional<StaticSampler> RootSignatureParser::parseStaticSampler() {
 
   // Check mandatory parameters were provided
   if (!Params->Reg.has_value()) {
-    getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_missing_param)
-        << TokenKind::sReg;
+    reportDiag(diag::err_hlsl_rootsig_missing_param) << TokenKind::sReg;
     return std::nullopt;
   }
 
@@ -434,8 +432,7 @@ RootSignatureParser::parseRootConstantParams() {
     // `num32BitConstants` `=` POS_INT
     if (tryConsumeExpectedToken(TokenKind::kw_num32BitConstants)) {
       if (Params.Num32BitConstants.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -451,8 +448,7 @@ RootSignatureParser::parseRootConstantParams() {
     // `b` POS_INT
     if (tryConsumeExpectedToken(TokenKind::bReg)) {
       if (Params.Reg.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
       auto Reg = parseRegister();
@@ -464,8 +460,7 @@ RootSignatureParser::parseRootConstantParams() {
     // `space` `=` POS_INT
     if (tryConsumeExpectedToken(TokenKind::kw_space)) {
       if (Params.Space.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -481,8 +476,7 @@ RootSignatureParser::parseRootConstantParams() {
     // `visibility` `=` SHADER_VISIBILITY
     if (tryConsumeExpectedToken(TokenKind::kw_visibility)) {
       if (Params.Visibility.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -509,8 +503,7 @@ RootSignatureParser::parseRootDescriptorParams(TokenKind RegType) {
     // ( `b` | `t` | `u`) POS_INT
     if (tryConsumeExpectedToken(RegType)) {
       if (Params.Reg.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
       auto Reg = parseRegister();
@@ -522,8 +515,7 @@ RootSignatureParser::parseRootDescriptorParams(TokenKind RegType) {
     // `space` `=` POS_INT
     if (tryConsumeExpectedToken(TokenKind::kw_space)) {
       if (Params.Space.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -539,8 +531,7 @@ RootSignatureParser::parseRootDescriptorParams(TokenKind RegType) {
     // `visibility` `=` SHADER_VISIBILITY
     if (tryConsumeExpectedToken(TokenKind::kw_visibility)) {
       if (Params.Visibility.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -556,8 +547,7 @@ RootSignatureParser::parseRootDescriptorParams(TokenKind RegType) {
     // `flags` `=` ROOT_DESCRIPTOR_FLAGS
     if (tryConsumeExpectedToken(TokenKind::kw_flags)) {
       if (Params.Flags.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -584,8 +574,7 @@ RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
     // ( `b` | `t` | `u` | `s`) POS_INT
     if (tryConsumeExpectedToken(RegType)) {
       if (Params.Reg.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
       auto Reg = parseRegister();
@@ -597,8 +586,7 @@ RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
     // `numDescriptors` `=` POS_INT | unbounded
     if (tryConsumeExpectedToken(TokenKind::kw_numDescriptors)) {
       if (Params.NumDescriptors.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -620,8 +608,7 @@ RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
     // `space` `=` POS_INT
     if (tryConsumeExpectedToken(TokenKind::kw_space)) {
       if (Params.Space.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -637,8 +624,7 @@ RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
     // `offset` `=` POS_INT | DESCRIPTOR_RANGE_OFFSET_APPEND
     if (tryConsumeExpectedToken(TokenKind::kw_offset)) {
       if (Params.Offset.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -660,8 +646,7 @@ RootSignatureParser::parseDescriptorTableClauseParams(TokenKind RegType) {
     // `flags` `=` DESCRIPTOR_RANGE_FLAGS
     if (tryConsumeExpectedToken(TokenKind::kw_flags)) {
       if (Params.Flags.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -689,8 +674,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `s` POS_INT
     if (tryConsumeExpectedToken(TokenKind::sReg)) {
       if (Params.Reg.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
       auto Reg = parseRegister();
@@ -702,8 +686,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `filter` `=` FILTER
     if (tryConsumeExpectedToken(TokenKind::kw_filter)) {
       if (Params.Filter.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -719,8 +702,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `addressU` `=` TEXTURE_ADDRESS
     if (tryConsumeExpectedToken(TokenKind::kw_addressU)) {
       if (Params.AddressU.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -736,8 +718,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `addressV` `=` TEXTURE_ADDRESS
     if (tryConsumeExpectedToken(TokenKind::kw_addressV)) {
       if (Params.AddressV.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -753,8 +734,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `addressW` `=` TEXTURE_ADDRESS
     if (tryConsumeExpectedToken(TokenKind::kw_addressW)) {
       if (Params.AddressW.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -770,8 +750,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `mipLODBias` `=` NUMBER
     if (tryConsumeExpectedToken(TokenKind::kw_mipLODBias)) {
       if (Params.MipLODBias.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -787,8 +766,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `maxAnisotropy` `=` POS_INT
     if (tryConsumeExpectedToken(TokenKind::kw_maxAnisotropy)) {
       if (Params.MaxAnisotropy.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -804,8 +782,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `comparisonFunc` `=` COMPARISON_FUNC
     if (tryConsumeExpectedToken(TokenKind::kw_comparisonFunc)) {
       if (Params.CompFunc.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -821,8 +798,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `borderColor` `=` STATIC_BORDER_COLOR
     if (tryConsumeExpectedToken(TokenKind::kw_borderColor)) {
       if (Params.BorderColor.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -838,8 +814,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `minLOD` `=` NUMBER
     if (tryConsumeExpectedToken(TokenKind::kw_minLOD)) {
       if (Params.MinLOD.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -855,8 +830,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `maxLOD` `=` NUMBER
     if (tryConsumeExpectedToken(TokenKind::kw_maxLOD)) {
       if (Params.MaxLOD.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -872,8 +846,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `space` `=` POS_INT
     if (tryConsumeExpectedToken(TokenKind::kw_space)) {
       if (Params.Space.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -889,8 +862,7 @@ RootSignatureParser::parseStaticSamplerParams() {
     // `visibility` `=` SHADER_VISIBILITY
     if (tryConsumeExpectedToken(TokenKind::kw_visibility)) {
       if (Params.Visibility.has_value()) {
-        getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_repeat_param)
-            << CurToken.TokKind;
+        reportDiag(diag::err_hlsl_rootsig_repeat_param) << CurToken.TokKind;
         return std::nullopt;
       }
 
@@ -983,7 +955,7 @@ std::optional<float> RootSignatureParser::parseFloatParam() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::ShaderVisibility>
+std::optional<llvm::dxbc::ShaderVisibility>
 RootSignatureParser::parseShaderVisibility() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -999,7 +971,7 @@ RootSignatureParser::parseShaderVisibility() {
   switch (CurToken.TokKind) {
 #define SHADER_VISIBILITY_ENUM(NAME, LIT)                                      \
   case TokenKind::en_##NAME:                                                   \
-    return ShaderVisibility::NAME;                                             \
+    return llvm::dxbc::ShaderVisibility::NAME;                                 \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1009,7 +981,7 @@ RootSignatureParser::parseShaderVisibility() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::SamplerFilter>
+std::optional<llvm::dxbc::SamplerFilter>
 RootSignatureParser::parseSamplerFilter() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1025,7 +997,7 @@ RootSignatureParser::parseSamplerFilter() {
   switch (CurToken.TokKind) {
 #define FILTER_ENUM(NAME, LIT)                                                 \
   case TokenKind::en_##NAME:                                                   \
-    return SamplerFilter::NAME;                                                \
+    return llvm::dxbc::SamplerFilter::NAME;                                    \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1035,7 +1007,7 @@ RootSignatureParser::parseSamplerFilter() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::TextureAddressMode>
+std::optional<llvm::dxbc::TextureAddressMode>
 RootSignatureParser::parseTextureAddressMode() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1051,7 +1023,7 @@ RootSignatureParser::parseTextureAddressMode() {
   switch (CurToken.TokKind) {
 #define TEXTURE_ADDRESS_MODE_ENUM(NAME, LIT)                                   \
   case TokenKind::en_##NAME:                                                   \
-    return TextureAddressMode::NAME;                                           \
+    return llvm::dxbc::TextureAddressMode::NAME;                               \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1061,7 +1033,7 @@ RootSignatureParser::parseTextureAddressMode() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::ComparisonFunc>
+std::optional<llvm::dxbc::ComparisonFunc>
 RootSignatureParser::parseComparisonFunc() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1077,7 +1049,7 @@ RootSignatureParser::parseComparisonFunc() {
   switch (CurToken.TokKind) {
 #define COMPARISON_FUNC_ENUM(NAME, LIT)                                        \
   case TokenKind::en_##NAME:                                                   \
-    return ComparisonFunc::NAME;                                               \
+    return llvm::dxbc::ComparisonFunc::NAME;                                   \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1087,7 +1059,7 @@ RootSignatureParser::parseComparisonFunc() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::StaticBorderColor>
+std::optional<llvm::dxbc::StaticBorderColor>
 RootSignatureParser::parseStaticBorderColor() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1103,7 +1075,7 @@ RootSignatureParser::parseStaticBorderColor() {
   switch (CurToken.TokKind) {
 #define STATIC_BORDER_COLOR_ENUM(NAME, LIT)                                    \
   case TokenKind::en_##NAME:                                                   \
-    return StaticBorderColor::NAME;                                            \
+    return llvm::dxbc::StaticBorderColor::NAME;                                \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1113,7 +1085,7 @@ RootSignatureParser::parseStaticBorderColor() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::RootDescriptorFlags>
+std::optional<llvm::dxbc::RootDescriptorFlags>
 RootSignatureParser::parseRootDescriptorFlags() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1121,10 +1093,10 @@ RootSignatureParser::parseRootDescriptorFlags() {
   // Handle the edge-case of '0' to specify no flags set
   if (tryConsumeExpectedToken(TokenKind::int_literal)) {
     if (!verifyZeroFlag()) {
-      getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_non_zero_flag);
+      reportDiag(diag::err_hlsl_rootsig_non_zero_flag);
       return std::nullopt;
     }
-    return RootDescriptorFlags::None;
+    return llvm::dxbc::RootDescriptorFlags::None;
   }
 
   TokenKind Expected[] = {
@@ -1132,15 +1104,15 @@ RootSignatureParser::parseRootDescriptorFlags() {
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   };
 
-  std::optional<RootDescriptorFlags> Flags;
+  std::optional<llvm::dxbc::RootDescriptorFlags> Flags;
 
   do {
     if (tryConsumeExpectedToken(Expected)) {
       switch (CurToken.TokKind) {
 #define ROOT_DESCRIPTOR_FLAG_ENUM(NAME, LIT)                                   \
   case TokenKind::en_##NAME:                                                   \
-    Flags =                                                                    \
-        maybeOrFlag<RootDescriptorFlags>(Flags, RootDescriptorFlags::NAME);    \
+    Flags = maybeOrFlag<llvm::dxbc::RootDescriptorFlags>(                      \
+        Flags, llvm::dxbc::RootDescriptorFlags::NAME);                         \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
       default:
@@ -1152,7 +1124,7 @@ RootSignatureParser::parseRootDescriptorFlags() {
   return Flags;
 }
 
-std::optional<llvm::hlsl::rootsig::DescriptorRangeFlags>
+std::optional<llvm::dxbc::DescriptorRangeFlags>
 RootSignatureParser::parseDescriptorRangeFlags() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1160,10 +1132,10 @@ RootSignatureParser::parseDescriptorRangeFlags() {
   // Handle the edge-case of '0' to specify no flags set
   if (tryConsumeExpectedToken(TokenKind::int_literal)) {
     if (!verifyZeroFlag()) {
-      getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_non_zero_flag);
+      reportDiag(diag::err_hlsl_rootsig_non_zero_flag);
       return std::nullopt;
     }
-    return DescriptorRangeFlags::None;
+    return llvm::dxbc::DescriptorRangeFlags::None;
   }
 
   TokenKind Expected[] = {
@@ -1171,15 +1143,15 @@ RootSignatureParser::parseDescriptorRangeFlags() {
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   };
 
-  std::optional<DescriptorRangeFlags> Flags;
+  std::optional<llvm::dxbc::DescriptorRangeFlags> Flags;
 
   do {
     if (tryConsumeExpectedToken(Expected)) {
       switch (CurToken.TokKind) {
 #define DESCRIPTOR_RANGE_FLAG_ENUM(NAME, LIT, ON)                              \
   case TokenKind::en_##NAME:                                                   \
-    Flags =                                                                    \
-        maybeOrFlag<DescriptorRangeFlags>(Flags, DescriptorRangeFlags::NAME);  \
+    Flags = maybeOrFlag<llvm::dxbc::DescriptorRangeFlags>(                     \
+        Flags, llvm::dxbc::DescriptorRangeFlags::NAME);                        \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
       default:
@@ -1193,9 +1165,9 @@ RootSignatureParser::parseDescriptorRangeFlags() {
 
 std::optional<uint32_t> RootSignatureParser::handleUIntLiteral() {
   // Parse the numeric value and do semantic checks on its specification
-  clang::NumericLiteralParser Literal(CurToken.NumSpelling, CurToken.TokLoc,
-                                      PP.getSourceManager(), PP.getLangOpts(),
-                                      PP.getTargetInfo(), PP.getDiagnostics());
+  clang::NumericLiteralParser Literal(
+      CurToken.NumSpelling, getTokenLocation(CurToken), PP.getSourceManager(),
+      PP.getLangOpts(), PP.getTargetInfo(), PP.getDiagnostics());
   if (Literal.hadError)
     return std::nullopt; // Error has already been reported so just return
 
@@ -1205,8 +1177,7 @@ std::optional<uint32_t> RootSignatureParser::handleUIntLiteral() {
   llvm::APSInt Val(32, /*IsUnsigned=*/true);
   if (Literal.GetIntegerValue(Val)) {
     // Report that the value has overflowed
-    PP.getDiagnostics().Report(CurToken.TokLoc,
-                               diag::err_hlsl_number_literal_overflow)
+    reportDiag(diag::err_hlsl_number_literal_overflow)
         << /*integer type*/ 0 << /*is signed*/ 0;
     return std::nullopt;
   }
@@ -1216,9 +1187,9 @@ std::optional<uint32_t> RootSignatureParser::handleUIntLiteral() {
 
 std::optional<int32_t> RootSignatureParser::handleIntLiteral(bool Negated) {
   // Parse the numeric value and do semantic checks on its specification
-  clang::NumericLiteralParser Literal(CurToken.NumSpelling, CurToken.TokLoc,
-                                      PP.getSourceManager(), PP.getLangOpts(),
-                                      PP.getTargetInfo(), PP.getDiagnostics());
+  clang::NumericLiteralParser Literal(
+      CurToken.NumSpelling, getTokenLocation(CurToken), PP.getSourceManager(),
+      PP.getLangOpts(), PP.getTargetInfo(), PP.getDiagnostics());
   if (Literal.hadError)
     return std::nullopt; // Error has already been reported so just return
 
@@ -1239,8 +1210,7 @@ std::optional<int32_t> RootSignatureParser::handleIntLiteral(bool Negated) {
 
   if (Overflowed) {
     // Report that the value has overflowed
-    PP.getDiagnostics().Report(CurToken.TokLoc,
-                               diag::err_hlsl_number_literal_overflow)
+    reportDiag(diag::err_hlsl_number_literal_overflow)
         << /*integer type*/ 0 << /*is signed*/ 1;
     return std::nullopt;
   }
@@ -1253,9 +1223,9 @@ std::optional<int32_t> RootSignatureParser::handleIntLiteral(bool Negated) {
 
 std::optional<float> RootSignatureParser::handleFloatLiteral(bool Negated) {
   // Parse the numeric value and do semantic checks on its specification
-  clang::NumericLiteralParser Literal(CurToken.NumSpelling, CurToken.TokLoc,
-                                      PP.getSourceManager(), PP.getLangOpts(),
-                                      PP.getTargetInfo(), PP.getDiagnostics());
+  clang::NumericLiteralParser Literal(
+      CurToken.NumSpelling, getTokenLocation(CurToken), PP.getSourceManager(),
+      PP.getLangOpts(), PP.getTargetInfo(), PP.getDiagnostics());
   if (Literal.hadError)
     return std::nullopt; // Error has already been reported so just return
 
@@ -1283,16 +1253,13 @@ std::optional<float> RootSignatureParser::handleFloatLiteral(bool Negated) {
 
   if (Status & llvm::APFloat::opStatus::opUnderflow) {
     // Report that the value has underflowed
-    PP.getDiagnostics().Report(CurToken.TokLoc,
-                               diag::err_hlsl_number_literal_underflow);
+    reportDiag(diag::err_hlsl_number_literal_underflow);
     return std::nullopt;
   }
 
   if (Status & llvm::APFloat::opStatus::opOverflow) {
     // Report that the value has overflowed
-    PP.getDiagnostics().Report(CurToken.TokLoc,
-                               diag::err_hlsl_number_literal_overflow)
-        << /*float type*/ 1;
+    reportDiag(diag::err_hlsl_number_literal_overflow) << /*float type*/ 1;
     return std::nullopt;
   }
 
@@ -1303,9 +1270,7 @@ std::optional<float> RootSignatureParser::handleFloatLiteral(bool Negated) {
   double FloatMax = double(std::numeric_limits<float>::max());
   if (FloatMax < DoubleVal || DoubleVal < -FloatMax) {
     // Report that the value has overflowed
-    PP.getDiagnostics().Report(CurToken.TokLoc,
-                               diag::err_hlsl_number_literal_overflow)
-        << /*float type*/ 1;
+    reportDiag(diag::err_hlsl_number_literal_overflow) << /*float type*/ 1;
     return std::nullopt;
   }
 
@@ -1334,7 +1299,7 @@ bool RootSignatureParser::consumeExpectedToken(TokenKind Expected,
     return false;
 
   // Report unexpected token kind error
-  DiagnosticBuilder DB = getDiags().Report(CurToken.TokLoc, DiagID);
+  DiagnosticBuilder DB = reportDiag(DiagID);
   switch (DiagID) {
   case diag::err_expected:
     DB << Expected;
@@ -1363,5 +1328,10 @@ bool RootSignatureParser::tryConsumeExpectedToken(
   return true;
 }
 
+SourceLocation RootSignatureParser::getTokenLocation(RootSignatureToken Tok) {
+  return Signature->getLocationOfByte(Tok.LocOffset, PP.getSourceManager(),
+                                      PP.getLangOpts(), PP.getTargetInfo());
+}
+
 } // namespace hlsl
 } // namespace clang
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index f2849c4eac7cc..b79a956d51505 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -606,13 +606,20 @@ unsigned getOpenACCScopeFlags(OpenACCDirectiveKind DirKind) {
   case OpenACCDirectiveKind::Parallel:
   case OpenACCDirectiveKind::Serial:
   case OpenACCDirectiveKind::Kernels:
+    // Mark this as a BreakScope/ContinueScope as well as a compute construct
+    // so that we can diagnose trying to 'break'/'continue' inside of one.
+    return Scope::BreakScope | Scope::ContinueScope |
+           Scope::OpenACCComputeConstructScope;
   case OpenACCDirectiveKind::ParallelLoop:
   case OpenACCDirectiveKind::SerialLoop:
   case OpenACCDirectiveKind::KernelsLoop:
     // Mark this as a BreakScope/ContinueScope as well as a compute construct
     // so that we can diagnose trying to 'break'/'continue' inside of one.
     return Scope::BreakScope | Scope::ContinueScope |
-           Scope::OpenACCComputeConstructScope;
+           Scope::OpenACCComputeConstructScope |
+           Scope::OpenACCLoopConstructScope;
+  case OpenACCDirectiveKind::Loop:
+    return Scope::OpenACCLoopConstructScope;
   case OpenACCDirectiveKind::Data:
   case OpenACCDirectiveKind::EnterData:
   case OpenACCDirectiveKind::ExitData:
@@ -621,7 +628,6 @@ unsigned getOpenACCScopeFlags(OpenACCDirectiveKind DirKind) {
   case OpenACCDirectiveKind::Init:
   case OpenACCDirectiveKind::Shutdown:
   case OpenACCDirectiveKind::Cache:
-  case OpenACCDirectiveKind::Loop:
   case OpenACCDirectiveKind::Atomic:
   case OpenACCDirectiveKind::Declare:
   case OpenACCDirectiveKind::Routine:
@@ -1416,12 +1422,15 @@ Parser::OpenACCVarParseResult Parser::ParseOpenACCVar(OpenACCDirectiveKind DK,
                                                       OpenACCClauseKind CK) {
   OpenACCArraySectionRAII ArraySections(*this);
 
+  getActions().OpenACC().ActOnStartParseVar(DK, CK);
   ExprResult Res = ParseAssignmentExpression();
-  if (!Res.isUsable())
+
+  if (!Res.isUsable()) {
+    getActions().OpenACC().ActOnInvalidParseVar();
     return {Res, OpenACCParseCanContinue::Cannot};
+  }
 
   Res = getActions().OpenACC().ActOnVar(DK, CK, Res.get());
-
   return {Res, OpenACCParseCanContinue::Can};
 }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index f694ae1d0d112..4c8a1f97d4b1f 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -25,6 +25,7 @@
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
 #include "llvm/Frontend/OpenMP/OMPAssume.h"
 #include "llvm/Frontend/OpenMP/OMPContext.h"
 #include <optional>
@@ -37,48 +38,6 @@ using namespace llvm::omp;
 //===----------------------------------------------------------------------===//
 
 namespace {
-enum OpenMPDirectiveKindEx {
-  OMPD_cancellation = llvm::omp::Directive_enumSize + 1,
-  OMPD_data,
-  OMPD_declare,
-  OMPD_end,
-  OMPD_end_declare,
-  OMPD_enter,
-  OMPD_exit,
-  OMPD_point,
-  OMPD_reduction,
-  OMPD_target_enter,
-  OMPD_target_exit,
-  OMPD_update,
-  OMPD_distribute_parallel,
-  OMPD_teams_distribute_parallel,
-  OMPD_target_teams_distribute_parallel,
-  OMPD_mapper,
-  OMPD_variant,
-  OMPD_begin,
-  OMPD_begin_declare,
-};
-
-// Helper to unify the enum class OpenMPDirectiveKind with its extension
-// the OpenMPDirectiveKindEx enum which allows to use them together as if they
-// are unsigned values.
-struct OpenMPDirectiveKindExWrapper {
-  OpenMPDirectiveKindExWrapper(unsigned Value) : Value(Value) {}
-  OpenMPDirectiveKindExWrapper(OpenMPDirectiveKind DK) : Value(unsigned(DK)) {}
-  bool operator==(OpenMPDirectiveKindExWrapper V) const {
-    return Value == V.Value;
-  }
-  bool operator!=(OpenMPDirectiveKindExWrapper V) const {
-    return Value != V.Value;
-  }
-  bool operator==(OpenMPDirectiveKind V) const { return Value == unsigned(V); }
-  bool operator!=(OpenMPDirectiveKind V) const { return Value != unsigned(V); }
-  bool operator<(OpenMPDirectiveKind V) const { return Value < unsigned(V); }
-  operator unsigned() const { return Value; }
-  operator OpenMPDirectiveKind() const { return OpenMPDirectiveKind(Value); }
-  unsigned Value;
-};
-
 class DeclDirectiveListParserHelper final {
   SmallVector<Expr *, 4> Identifiers;
   Parser *P;
@@ -97,130 +56,52 @@ class DeclDirectiveListParserHelper final {
 };
 } // namespace
 
-// Map token string to extended OMP token kind that are
-// OpenMPDirectiveKind + OpenMPDirectiveKindEx.
-static unsigned getOpenMPDirectiveKindEx(StringRef S) {
-  OpenMPDirectiveKindExWrapper DKind = getOpenMPDirectiveKind(S);
-  if (DKind != OMPD_unknown)
-    return DKind;
-
-  return llvm::StringSwitch<OpenMPDirectiveKindExWrapper>(S)
-      .Case("cancellation", OMPD_cancellation)
-      .Case("data", OMPD_data)
-      .Case("declare", OMPD_declare)
-      .Case("end", OMPD_end)
-      .Case("enter", OMPD_enter)
-      .Case("exit", OMPD_exit)
-      .Case("point", OMPD_point)
-      .Case("reduction", OMPD_reduction)
-      .Case("update", OMPD_update)
-      .Case("mapper", OMPD_mapper)
-      .Case("variant", OMPD_variant)
-      .Case("begin", OMPD_begin)
-      .Default(OMPD_unknown);
+static OpenMPDirectiveKind checkOpenMPDirectiveName(Parser &P,
+                                                    SourceLocation Loc,
+                                                    OpenMPDirectiveKind Kind,
+                                                    StringRef Name) {
+  unsigned Version = P.getLangOpts().OpenMP;
+  auto [D, VR] = getOpenMPDirectiveKindAndVersions(Name);
+  assert(D == Kind && "Directive kind mismatch");
+  // Ignore the case Version > VR.Max: In OpenMP 6.0 all prior spellings
+  // are explicitly allowed.
+  if (static_cast<int>(Version) < VR.Min)
+    P.Diag(Loc, diag::warn_omp_future_directive_spelling) << Name;
+
+  return Kind;
 }
 
-static OpenMPDirectiveKindExWrapper parseOpenMPDirectiveKind(Parser &P) {
-  // Array of foldings: F[i][0] F[i][1] ===> F[i][2].
-  // E.g.: OMPD_for OMPD_simd ===> OMPD_for_simd
-  // TODO: add other combined directives in topological order.
-  static const OpenMPDirectiveKindExWrapper F[][3] = {
-      {OMPD_begin, OMPD_declare, OMPD_begin_declare},
-      {OMPD_begin, OMPD_assumes, OMPD_begin_assumes},
-      {OMPD_end, OMPD_declare, OMPD_end_declare},
-      {OMPD_end, OMPD_assumes, OMPD_end_assumes},
-      {OMPD_cancellation, OMPD_point, OMPD_cancellation_point},
-      {OMPD_declare, OMPD_reduction, OMPD_declare_reduction},
-      {OMPD_declare, OMPD_mapper, OMPD_declare_mapper},
-      {OMPD_declare, OMPD_simd, OMPD_declare_simd},
-      {OMPD_declare, OMPD_target, OMPD_declare_target},
-      {OMPD_declare, OMPD_variant, OMPD_declare_variant},
-      {OMPD_begin_declare, OMPD_target, OMPD_begin_declare_target},
-      {OMPD_begin_declare, OMPD_variant, OMPD_begin_declare_variant},
-      {OMPD_end_declare, OMPD_variant, OMPD_end_declare_variant},
-      {OMPD_distribute, OMPD_parallel, OMPD_distribute_parallel},
-      {OMPD_distribute_parallel, OMPD_for, OMPD_distribute_parallel_for},
-      {OMPD_distribute_parallel_for, OMPD_simd,
-       OMPD_distribute_parallel_for_simd},
-      {OMPD_distribute, OMPD_simd, OMPD_distribute_simd},
-      {OMPD_end_declare, OMPD_target, OMPD_end_declare_target},
-      {OMPD_target, OMPD_data, OMPD_target_data},
-      {OMPD_target, OMPD_enter, OMPD_target_enter},
-      {OMPD_target, OMPD_exit, OMPD_target_exit},
-      {OMPD_target, OMPD_update, OMPD_target_update},
-      {OMPD_target_enter, OMPD_data, OMPD_target_enter_data},
-      {OMPD_target_exit, OMPD_data, OMPD_target_exit_data},
-      {OMPD_for, OMPD_simd, OMPD_for_simd},
-      {OMPD_parallel, OMPD_for, OMPD_parallel_for},
-      {OMPD_parallel_for, OMPD_simd, OMPD_parallel_for_simd},
-      {OMPD_parallel, OMPD_loop, OMPD_parallel_loop},
-      {OMPD_parallel, OMPD_sections, OMPD_parallel_sections},
-      {OMPD_taskloop, OMPD_simd, OMPD_taskloop_simd},
-      {OMPD_target, OMPD_parallel, OMPD_target_parallel},
-      {OMPD_target, OMPD_simd, OMPD_target_simd},
-      {OMPD_target_parallel, OMPD_loop, OMPD_target_parallel_loop},
-      {OMPD_target_parallel, OMPD_for, OMPD_target_parallel_for},
-      {OMPD_target_parallel_for, OMPD_simd, OMPD_target_parallel_for_simd},
-      {OMPD_teams, OMPD_distribute, OMPD_teams_distribute},
-      {OMPD_teams_distribute, OMPD_simd, OMPD_teams_distribute_simd},
-      {OMPD_teams_distribute, OMPD_parallel, OMPD_teams_distribute_parallel},
-      {OMPD_teams_distribute_parallel, OMPD_for,
-       OMPD_teams_distribute_parallel_for},
-      {OMPD_teams_distribute_parallel_for, OMPD_simd,
-       OMPD_teams_distribute_parallel_for_simd},
-      {OMPD_teams, OMPD_loop, OMPD_teams_loop},
-      {OMPD_target, OMPD_teams, OMPD_target_teams},
-      {OMPD_target_teams, OMPD_distribute, OMPD_target_teams_distribute},
-      {OMPD_target_teams, OMPD_loop, OMPD_target_teams_loop},
-      {OMPD_target_teams_distribute, OMPD_parallel,
-       OMPD_target_teams_distribute_parallel},
-      {OMPD_target_teams_distribute, OMPD_simd,
-       OMPD_target_teams_distribute_simd},
-      {OMPD_target_teams_distribute_parallel, OMPD_for,
-       OMPD_target_teams_distribute_parallel_for},
-      {OMPD_target_teams_distribute_parallel_for, OMPD_simd,
-       OMPD_target_teams_distribute_parallel_for_simd},
-      {OMPD_master, OMPD_taskloop, OMPD_master_taskloop},
-      {OMPD_masked, OMPD_taskloop, OMPD_masked_taskloop},
-      {OMPD_master_taskloop, OMPD_simd, OMPD_master_taskloop_simd},
-      {OMPD_masked_taskloop, OMPD_simd, OMPD_masked_taskloop_simd},
-      {OMPD_parallel, OMPD_master, OMPD_parallel_master},
-      {OMPD_parallel, OMPD_masked, OMPD_parallel_masked},
-      {OMPD_parallel_master, OMPD_taskloop, OMPD_parallel_master_taskloop},
-      {OMPD_parallel_masked, OMPD_taskloop, OMPD_parallel_masked_taskloop},
-      {OMPD_parallel_master_taskloop, OMPD_simd,
-       OMPD_parallel_master_taskloop_simd},
-      {OMPD_parallel_masked_taskloop, OMPD_simd,
-       OMPD_parallel_masked_taskloop_simd}};
-  enum { CancellationPoint = 0, DeclareReduction = 1, TargetData = 2 };
+static OpenMPDirectiveKind parseOpenMPDirectiveKind(Parser &P) {
+  static const DirectiveNameParser DirParser;
+
+  const DirectiveNameParser::State *S = DirParser.initial();
+
   Token Tok = P.getCurToken();
-  OpenMPDirectiveKindExWrapper DKind =
-      Tok.isAnnotation()
-          ? static_cast<unsigned>(OMPD_unknown)
-          : getOpenMPDirectiveKindEx(P.getPreprocessor().getSpelling(Tok));
-  if (DKind == OMPD_unknown)
+  if (Tok.isAnnotation())
     return OMPD_unknown;
 
-  for (const auto &I : F) {
-    if (DKind != I[0])
-      continue;
+  std::string Concat = P.getPreprocessor().getSpelling(Tok);
+  SourceLocation Loc = Tok.getLocation();
 
-    Tok = P.getPreprocessor().LookAhead(0);
-    OpenMPDirectiveKindExWrapper SDKind =
-        Tok.isAnnotation()
-            ? static_cast<unsigned>(OMPD_unknown)
-            : getOpenMPDirectiveKindEx(P.getPreprocessor().getSpelling(Tok));
-    if (SDKind == OMPD_unknown)
-      continue;
+  S = DirParser.consume(S, Concat);
+  if (S == nullptr)
+    return OMPD_unknown;
 
-    if (SDKind == I[1]) {
+  while (!Tok.isAnnotation()) {
+    OpenMPDirectiveKind DKind = S->Value;
+    Tok = P.getPreprocessor().LookAhead(0);
+    if (!Tok.isAnnotation()) {
+      std::string TS = P.getPreprocessor().getSpelling(Tok);
+      S = DirParser.consume(S, TS);
+      if (S == nullptr)
+        return checkOpenMPDirectiveName(P, Loc, DKind, Concat);
+      Concat += ' ' + TS;
       P.ConsumeToken();
-      DKind = I[2];
     }
   }
-  return unsigned(DKind) < llvm::omp::Directive_enumSize
-             ? static_cast<OpenMPDirectiveKind>(DKind)
-             : OMPD_unknown;
+
+  assert(S && "Should have exited early");
+  return checkOpenMPDirectiveName(P, Loc, S->Value, Concat);
 }
 
 static DeclarationName parseOpenMPReductionId(Parser &P) {
@@ -2629,10 +2510,6 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     Diag(Tok, diag::err_omp_unknown_directive);
     return StmtError();
   }
-  if (!(getDirectiveLanguages(DKind) & SourceLanguage::C)) {
-    // Treat directives that are not allowed in C/C++ as unknown.
-    DKind = OMPD_unknown;
-  }
 
   StmtResult Directive = StmtError();
 
@@ -3503,9 +3380,9 @@ ExprResult Parser::ParseOpenMPParensExpr(StringRef ClauseName,
     return ExprError();
 
   SourceLocation ELoc = Tok.getLocation();
-  ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr,
-                                     IsAddressOfOperand,
-                                     TypeCastState::NotTypeCast));
+  ExprResult LHS(
+      ParseCastExpression(CastParseKind::AnyCastExpr, IsAddressOfOperand,
+                          TypoCorrectionTypeBehavior::AllowNonTypes));
   ExprResult Val(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   Val = Actions.ActOnFinishFullExpr(Val.get(), ELoc, /*DiscardedValue*/ false);
 
@@ -4014,7 +3891,7 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
     KLoc.push_back(Tok.getLocation());
     TentativeParsingAction TPA(*this);
     auto DK = parseOpenMPDirectiveKind(*this);
-    Arg.push_back(DK);
+    Arg.push_back(static_cast<unsigned>(DK));
     if (DK != OMPD_unknown) {
       ConsumeToken();
       if (Tok.is(tok::colon) && getLangOpts().OpenMP > 40) {
@@ -4036,8 +3913,9 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
                           Kind == OMPC_num_threads;
   if (NeedAnExpression) {
     SourceLocation ELoc = Tok.getLocation();
-    ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr, false,
-                                       TypeCastState::NotTypeCast));
+    ExprResult LHS(
+        ParseCastExpression(CastParseKind::AnyCastExpr, false,
+                            TypoCorrectionTypeBehavior::AllowNonTypes));
     Val = ParseRHSOfBinaryExpression(LHS, prec::Conditional);
     Val =
         Actions.ActOnFinishFullExpr(Val.get(), ELoc, /*DiscardedValue*/ false);
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index a16dbe95b788d..9a04bf298c0ac 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -1318,8 +1318,8 @@ ParsedTemplateArgument Parser::ParseTemplateArgument() {
   if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace))
     ExprArg = ParseBraceInitializer();
   else
-    ExprArg =
-        ParseConstantExpressionInExprEvalContext(TypeCastState::MaybeTypeCast);
+    ExprArg = ParseConstantExpressionInExprEvalContext(
+        TypoCorrectionTypeBehavior::AllowBoth);
   if (ExprArg.isInvalid() || !ExprArg.get()) {
     return ParsedTemplateArgument();
   }
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 18f399aca59e8..8834bf80c4016 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1606,8 +1606,9 @@ ExprResult Parser::ParseAsmStringLiteral(bool ForAsmLabel) {
 
     EnterExpressionEvaluationContext ConstantEvaluated(
         Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
-    AsmString = ParseParenExpression(ExprType, true /*stopIfCastExpr*/, false,
-                                     CastTy, RParenLoc);
+    AsmString = ParseParenExpression(
+        ExprType, /*StopIfCastExr=*/true, ParenExprKind::Unknown,
+        TypoCorrectionTypeBehavior::AllowBoth, CastTy, RParenLoc);
     if (!AsmString.isInvalid())
       AsmString = Actions.ActOnConstantExpression(AsmString);
 
diff --git a/clang/lib/Sema/HeuristicResolver.cpp b/clang/lib/Sema/HeuristicResolver.cpp
index 0c67f1f2a3878..6874d30516f8f 100644
--- a/clang/lib/Sema/HeuristicResolver.cpp
+++ b/clang/lib/Sema/HeuristicResolver.cpp
@@ -13,6 +13,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
+#include "llvm/ADT/identity.h"
 
 namespace clang {
 
@@ -50,6 +51,7 @@ class HeuristicResolverImpl {
                       llvm::function_ref<bool(const NamedDecl *ND)> Filter);
   TagDecl *resolveTypeToTagDecl(QualType T);
   QualType simplifyType(QualType Type, const Expr *E, bool UnwrapPointer);
+  FunctionProtoTypeLoc getFunctionProtoTypeLoc(const Expr *Fn);
 
 private:
   ASTContext &Ctx;
@@ -506,6 +508,56 @@ std::vector<const NamedDecl *> HeuristicResolverImpl::resolveDependentMember(
   }
   return {};
 }
+
+FunctionProtoTypeLoc
+HeuristicResolverImpl::getFunctionProtoTypeLoc(const Expr *Fn) {
+  TypeLoc Target;
+  const Expr *NakedFn = Fn->IgnoreParenCasts();
+  if (const auto *T = NakedFn->getType().getTypePtr()->getAs<TypedefType>()) {
+    Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc();
+  } else if (const auto *DR = dyn_cast<DeclRefExpr>(NakedFn)) {
+    const auto *D = DR->getDecl();
+    if (const auto *const VD = dyn_cast<VarDecl>(D)) {
+      Target = VD->getTypeSourceInfo()->getTypeLoc();
+    }
+  } else if (const auto *ME = dyn_cast<MemberExpr>(NakedFn)) {
+    const auto *MD = ME->getMemberDecl();
+    if (const auto *FD = dyn_cast<FieldDecl>(MD)) {
+      Target = FD->getTypeSourceInfo()->getTypeLoc();
+    }
+  }
+
+  if (!Target)
+    return {};
+
+  // Unwrap types that may be wrapping the function type
+  while (true) {
+    if (auto P = Target.getAs<PointerTypeLoc>()) {
+      Target = P.getPointeeLoc();
+      continue;
+    }
+    if (auto A = Target.getAs<AttributedTypeLoc>()) {
+      Target = A.getModifiedLoc();
+      continue;
+    }
+    if (auto P = Target.getAs<ParenTypeLoc>()) {
+      Target = P.getInnerLoc();
+      continue;
+    }
+    break;
+  }
+
+  if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
+    // In some edge cases the AST can contain a "trivial" FunctionProtoTypeLoc
+    // which has null parameters. Avoid these as they don't contain useful
+    // information.
+    if (llvm::all_of(F.getParams(), llvm::identity<ParmVarDecl *>()))
+      return F;
+  }
+
+  return {};
+}
+
 } // namespace
 
 std::vector<const NamedDecl *> HeuristicResolver::resolveMemberExpr(
@@ -557,4 +609,9 @@ QualType HeuristicResolver::simplifyType(QualType Type, const Expr *E,
   return HeuristicResolverImpl(Ctx).simplifyType(Type, E, UnwrapPointer);
 }
 
+FunctionProtoTypeLoc
+HeuristicResolver::getFunctionProtoTypeLoc(const Expr *Fn) const {
+  return HeuristicResolverImpl(Ctx).getFunctionProtoTypeLoc(Fn);
+}
+
 } // namespace clang
diff --git a/clang/lib/Sema/MultiplexExternalSemaSource.cpp b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
index 9f19f13592e86..fbfb242598c24 100644
--- a/clang/lib/Sema/MultiplexExternalSemaSource.cpp
+++ b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
@@ -115,14 +115,6 @@ bool MultiplexExternalSemaSource::wasThisDeclarationADefinition(
   return false;
 }
 
-bool MultiplexExternalSemaSource::hasInitializerWithSideEffects(
-    const VarDecl *VD) const {
-  for (const auto &S : Sources)
-    if (S->hasInitializerWithSideEffects(VD))
-      return true;
-  return false;
-}
-
 bool MultiplexExternalSemaSource::FindExternalVisibleDeclsByName(
     const DeclContext *DC, DeclarationName Name,
     const DeclContext *OriginalDC) {
diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td
index 528b700a275e0..fe86d42dac4db 100644
--- a/clang/lib/Sema/OpenCLBuiltins.td
+++ b/clang/lib/Sema/OpenCLBuiltins.td
@@ -139,6 +139,10 @@ def ArmIntegerDotProductAccumulateInt8         : FunctionExtension<"cl_arm_integ
 def ArmIntegerDotProductAccumulateInt16        : FunctionExtension<"cl_arm_integer_dot_product_accumulate_int16">;
 def ArmIntegerDotProductAccumulateSaturateInt8 : FunctionExtension<"cl_arm_integer_dot_product_accumulate_saturate_int8">;
 
+// AMD extensions.
+def AMDMediaOps : FunctionExtension<"cl_amd_media_ops">;
+def AMDMediaOps2 : FunctionExtension<"cl_amd_media_ops2">;
+
 // Qualified Type.  These map to ASTContext::QualType.
 class QualType<string _TypeExpr, bit _IsAbstract=0> {
   // Expression to obtain the QualType inside OCL2Qual.
@@ -1904,3 +1908,31 @@ let Extension = ArmIntegerDotProductAccumulateSaturateInt8 in {
     def : Builtin<name, [Int, VectorType<Char, 4>, VectorType<Char, 4>, Int]>;
   }
 }
+
+//--------------------------------------------------------------------
+// AMD extensions.
+let Extension = AMDMediaOps in {
+  foreach name = ["amd_bitalign", "amd_bytealign", "amd_lerp", "amd_sadhi", "amd_sad"] in {
+    def : Builtin<name, [GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  }
+  foreach name = ["amd_unpack0", "amd_unpack1", "amd_unpack2", "amd_unpack3"] in {
+    def : Builtin<name, [GenTypeFloatVecAndScalar, GenTypeUIntVecAndScalar]>;
+  }
+  def : Builtin<"amd_pack", [UInt, VectorType<Float, 4>]>;
+  def : Builtin<"amd_sad4", [UInt, VectorType<UInt, 4>, VectorType<UInt, 4>, UInt]>;
+}
+
+let Extension = AMDMediaOps2 in {
+  def : Builtin<"amd_bfe", [GenTypeIntVecAndScalar, GenTypeIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  foreach name = ["amd_bfe", "amd_msad", "amd_sadd", "amd_sadw"] in {
+    def : Builtin<name, [GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  }
+  def : Builtin<"amd_bfm", [GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  foreach name = ["amd_max3", "amd_median3", "amd_min3"] in {
+    foreach gentype = [GenTypeFloatVecAndScalar, GenTypeIntVecAndScalar, GenTypeUIntVecAndScalar] in {
+      def : Builtin<name, [gentype, gentype, gentype, gentype]>;
+    }
+  }
+  def : Builtin<"amd_mqsad", [GenTypeULongVecAndScalar, GenTypeULongVecAndScalar, GenTypeUIntVecAndScalar, GenTypeULongVecAndScalar]>;
+  def : Builtin<"amd_qsad", [GenTypeULongVecAndScalar, GenTypeULongVecAndScalar, GenTypeUIntVecAndScalar, GenTypeULongVecAndScalar]>;
+}
diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp
index 5bc7e79a68186..ab04fe554be82 100644
--- a/clang/lib/Sema/Scope.cpp
+++ b/clang/lib/Sema/Scope.cpp
@@ -234,7 +234,8 @@ void Scope::dumpImpl(raw_ostream &OS) const {
       {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"},
       {TypeAliasScope, "TypeAliasScope"},
       {FriendScope, "FriendScope"},
-  };
+      {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"},
+      {OpenACCLoopConstructScope, "OpenACCLoopConstructScope"}};
 
   for (auto Info : FlagInfo) {
     if (Flags & Info.first) {
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index dfc5a2767f579..56608e990fd50 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -2261,6 +2261,24 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) {
         }
       }
     }
+
+    if (auto *VT = Ty->getAs<VectorType>();
+        VT && FD &&
+        (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
+         VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) &&
+        (LangOpts.VScaleMin != LangOpts.VScaleStreamingMin ||
+         LangOpts.VScaleMax != LangOpts.VScaleStreamingMax)) {
+      if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true)) {
+        Diag(Loc, diag::err_sve_fixed_vector_in_streaming_function)
+            << Ty << /*Streaming*/ 0;
+      } else if (const auto *FTy = FD->getType()->getAs<FunctionProtoType>()) {
+        if (FTy->getAArch64SMEAttributes() &
+            FunctionType::SME_PStateSMCompatibleMask) {
+          Diag(Loc, diag::err_sve_fixed_vector_in_streaming_function)
+              << Ty << /*StreamingCompatible*/ 1;
+        }
+      }
+    }
   };
 
   CheckType(Ty);
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index f7965ff889bba..bd603a925d15e 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -569,34 +569,39 @@ static bool checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall,
   // * When compiling for SVE only, the caller must be in non-streaming mode.
   // * When compiling for both SVE and SME, the caller can be in either mode.
   if (BuiltinType == SemaARM::VerifyRuntimeMode) {
-    llvm::StringMap<bool> CallerFeatureMapWithoutSVE;
-    S.Context.getFunctionFeatureMap(CallerFeatureMapWithoutSVE, FD);
-    CallerFeatureMapWithoutSVE["sve"] = false;
+    llvm::StringMap<bool> CallerFeatures;
+    S.Context.getFunctionFeatureMap(CallerFeatures, FD);
 
     // Avoid emitting diagnostics for a function that can never compile.
-    if (FnType == SemaARM::ArmStreaming && !CallerFeatureMapWithoutSVE["sme"])
+    if (FnType == SemaARM::ArmStreaming && !CallerFeatures["sme"])
       return false;
 
-    llvm::StringMap<bool> CallerFeatureMapWithoutSME;
-    S.Context.getFunctionFeatureMap(CallerFeatureMapWithoutSME, FD);
-    CallerFeatureMapWithoutSME["sme"] = false;
+    const auto FindTopLevelPipe = [](const char *S) {
+      unsigned Depth = 0;
+      unsigned I = 0, E = strlen(S);
+      for (; I < E; ++I) {
+        if (S[I] == '|' && Depth == 0)
+          break;
+        if (S[I] == '(')
+          ++Depth;
+        else if (S[I] == ')')
+          --Depth;
+      }
+      return I;
+    };
+
+    const char *RequiredFeatures =
+        S.Context.BuiltinInfo.getRequiredFeatures(BuiltinID);
+    unsigned PipeIdx = FindTopLevelPipe(RequiredFeatures);
+    assert(PipeIdx != 0 && PipeIdx != strlen(RequiredFeatures) &&
+           "Expected feature string of the form 'SVE-EXPR|SME-EXPR'");
+    StringRef NonStreamingBuiltinGuard = StringRef(RequiredFeatures, PipeIdx);
+    StringRef StreamingBuiltinGuard = StringRef(RequiredFeatures + PipeIdx + 1);
 
-    // We know the builtin requires either some combination of SVE flags, or
-    // some combination of SME flags, but we need to figure out which part
-    // of the required features is satisfied by the target features.
-    //
-    // For a builtin with target guard 'sve2p1|sme2', if we compile with
-    // '+sve2p1,+sme', then we know that it satisfies the 'sve2p1' part if we
-    // evaluate the features for '+sve2p1,+sme,+nosme'.
-    //
-    // Similarly, if we compile with '+sve2,+sme2', then we know it satisfies
-    // the 'sme2' part if we evaluate the features for '+sve2,+sme2,+nosve'.
-    StringRef BuiltinTargetGuards(
-        S.Context.BuiltinInfo.getRequiredFeatures(BuiltinID));
     bool SatisfiesSVE = Builtin::evaluateRequiredTargetFeatures(
-        BuiltinTargetGuards, CallerFeatureMapWithoutSME);
+        NonStreamingBuiltinGuard, CallerFeatures);
     bool SatisfiesSME = Builtin::evaluateRequiredTargetFeatures(
-        BuiltinTargetGuards, CallerFeatureMapWithoutSVE);
+        StreamingBuiltinGuard, CallerFeatures);
 
     if ((SatisfiesSVE && SatisfiesSME) ||
         (SatisfiesSVE && FnType == SemaARM::ArmStreamingCompatible))
@@ -1084,7 +1089,7 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
   // converted to a register of the form S1_2_C3_C4_5. Let the hardware throw
   // an exception for incorrect registers. This matches MSVC behavior.
   if (BuiltinID == AArch64::BI_ReadStatusReg ||
-      BuiltinID == AArch64::BI_WriteStatusReg)
+      BuiltinID == AArch64::BI_WriteStatusReg || BuiltinID == AArch64::BI__sys)
     return SemaRef.BuiltinConstantArgRange(TheCall, 0, 0, 0x7fff);
 
   if (BuiltinID == AArch64::BI__getReg)
@@ -1410,21 +1415,41 @@ void SemaARM::CheckSMEFunctionDefAttributes(const FunctionDecl *FD) {
 }
 
 /// getSVETypeSize - Return SVE vector or predicate register size.
-static uint64_t getSVETypeSize(ASTContext &Context, const BuiltinType *Ty) {
+static uint64_t getSVETypeSize(ASTContext &Context, const BuiltinType *Ty,
+                               bool IsStreaming) {
   assert(Ty->isSveVLSBuiltinType() && "Invalid SVE Type");
+  uint64_t VScale = IsStreaming ? Context.getLangOpts().VScaleStreamingMin
+                                : Context.getLangOpts().VScaleMin;
   if (Ty->getKind() == BuiltinType::SveBool ||
       Ty->getKind() == BuiltinType::SveCount)
-    return (Context.getLangOpts().VScaleMin * 128) / Context.getCharWidth();
-  return Context.getLangOpts().VScaleMin * 128;
+    return (VScale * 128) / Context.getCharWidth();
+  return VScale * 128;
 }
 
 bool SemaARM::areCompatibleSveTypes(QualType FirstType, QualType SecondType) {
-  auto IsValidCast = [this](QualType FirstType, QualType SecondType) {
+  bool IsStreaming = false;
+  if (getLangOpts().VScaleMin != getLangOpts().VScaleStreamingMin ||
+      getLangOpts().VScaleMax != getLangOpts().VScaleStreamingMax) {
+    if (const FunctionDecl *FD =
+            SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
+      // For streaming-compatible functions, we don't know vector length.
+      if (const auto *T = FD->getType()->getAs<FunctionProtoType>()) {
+        if (T->getAArch64SMEAttributes() &
+            FunctionType::SME_PStateSMCompatibleMask)
+          return false;
+      }
+
+      if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true))
+        IsStreaming = true;
+    }
+  }
+
+  auto IsValidCast = [&](QualType FirstType, QualType SecondType) {
     if (const auto *BT = FirstType->getAs<BuiltinType>()) {
       if (const auto *VT = SecondType->getAs<VectorType>()) {
-        ASTContext &Context = getASTContext();
         // Predicates have the same representation as uint8 so we also have to
         // check the kind to make these types incompatible.
+        ASTContext &Context = getASTContext();
         if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
           return BT->getKind() == BuiltinType::SveBool;
         else if (VT->getVectorKind() == VectorKind::SveFixedLengthData)
@@ -1432,7 +1457,7 @@ bool SemaARM::areCompatibleSveTypes(QualType FirstType, QualType SecondType) {
                  FirstType->getSveEltType(Context);
         else if (VT->getVectorKind() == VectorKind::Generic)
           return Context.getTypeSize(SecondType) ==
-                     getSVETypeSize(Context, BT) &&
+                     getSVETypeSize(Context, BT, IsStreaming) &&
                  Context.hasSameType(
                      VT->getElementType(),
                      Context.getBuiltinVectorTypeInfo(BT).ElementType);
@@ -1447,7 +1472,23 @@ bool SemaARM::areCompatibleSveTypes(QualType FirstType, QualType SecondType) {
 
 bool SemaARM::areLaxCompatibleSveTypes(QualType FirstType,
                                        QualType SecondType) {
-  auto IsLaxCompatible = [this](QualType FirstType, QualType SecondType) {
+  bool IsStreaming = false;
+  if (getLangOpts().VScaleMin != getLangOpts().VScaleStreamingMin ||
+      getLangOpts().VScaleMax != getLangOpts().VScaleStreamingMax) {
+    if (const FunctionDecl *FD =
+            SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
+      // For streaming-compatible functions, we don't know vector length.
+      if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
+        if (T->getAArch64SMEAttributes() &
+            FunctionType::SME_PStateSMCompatibleMask)
+          return false;
+
+      if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true))
+        IsStreaming = true;
+    }
+  }
+
+  auto IsLaxCompatible = [&](QualType FirstType, QualType SecondType) {
     const auto *BT = FirstType->getAs<BuiltinType>();
     if (!BT)
       return false;
@@ -1471,7 +1512,8 @@ bool SemaARM::areLaxCompatibleSveTypes(QualType FirstType,
       // ACLE Spec Version 00bet6, 3.7.3.2. Behavior common to vectors and
       // predicates.
       if (VecTy->getVectorKind() == VectorKind::Generic &&
-          Context.getTypeSize(SecondType) != getSVETypeSize(Context, BT))
+          Context.getTypeSize(SecondType) !=
+              getSVETypeSize(Context, BT, IsStreaming))
         return false;
 
       // If -flax-vector-conversions=all is specified, the types are
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index c1675a6c67f14..3eed6ad7fe6b3 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -436,7 +436,7 @@ void Sema::ActOnPragmaClangSection(SourceLocation PragmaLoc,
 }
 
 void Sema::ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action,
-                           StringRef SlotLabel, Expr *alignment) {
+                           StringRef SlotLabel, Expr *Alignment) {
   bool IsXLPragma = getLangOpts().XLPragmaPack;
   // XL pragma pack does not support identifier syntax.
   if (IsXLPragma && !SlotLabel.empty()) {
@@ -445,7 +445,6 @@ void Sema::ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action,
   }
 
   const AlignPackInfo CurVal = AlignPackStack.CurrentValue;
-  Expr *Alignment = static_cast<Expr *>(alignment);
 
   // If specified then alignment must be a "small" power of two.
   unsigned AlignmentVal = 0;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 926adebf7de0f..dd5b710d7e1d4 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1012,7 +1012,10 @@ class EstimateSizeFormatHandler
       break;
     }
 
-    Size += FS.hasPlusPrefix() || FS.hasSpacePrefix();
+    // If field width is specified, the sign/space is already accounted for
+    // within the field width, so no additional size is needed.
+    if ((FS.hasPlusPrefix() || FS.hasSpacePrefix()) && FieldWidth == 0)
+      Size += 1;
 
     if (FS.hasAlternativeForm()) {
       switch (FS.getConversionSpecifier().getKind()) {
@@ -2261,6 +2264,99 @@ static bool BuiltinCountZeroBitsGeneric(Sema &S, CallExpr *TheCall) {
   return false;
 }
 
+static ExprResult BuiltinInvoke(Sema &S, CallExpr *TheCall) {
+  SourceLocation Loc = TheCall->getBeginLoc();
+  MutableArrayRef Args(TheCall->getArgs(), TheCall->getNumArgs());
+  assert(llvm::none_of(Args, [](Expr *Arg) { return Arg->isTypeDependent(); }));
+
+  if (Args.size() == 0) {
+    S.Diag(TheCall->getBeginLoc(),
+           diag::err_typecheck_call_too_few_args_at_least)
+        << /*callee_type=*/0 << /*min_arg_count=*/1 << /*actual_arg_count=*/0
+        << /*is_non_object=*/0 << TheCall->getSourceRange();
+    return ExprError();
+  }
+
+  QualType FuncT = Args[0]->getType();
+
+  if (const auto *MPT = FuncT->getAs<MemberPointerType>()) {
+    if (Args.size() < 2) {
+      S.Diag(TheCall->getBeginLoc(),
+             diag::err_typecheck_call_too_few_args_at_least)
+          << /*callee_type=*/0 << /*min_arg_count=*/2 << /*actual_arg_count=*/1
+          << /*is_non_object=*/0 << TheCall->getSourceRange();
+      return ExprError();
+    }
+
+    const Type *MemPtrClass = MPT->getQualifier()->getAsType();
+    QualType ObjectT = Args[1]->getType();
+
+    if (MPT->isMemberDataPointer() && S.checkArgCount(TheCall, 2))
+      return ExprError();
+
+    ExprResult ObjectArg = [&]() -> ExprResult {
+      // (1.1): (t1.*f)(t2, ..., tN) when f is a pointer to a member function of
+      // a class T and is_same_v<T, remove_cvref_t<decltype(t1)>> ||
+      // is_base_of_v<T, remove_cvref_t<decltype(t1)>> is true;
+      // (1.4): t1.*f when N=1 and f is a pointer to data member of a class T
+      // and is_same_v<T, remove_cvref_t<decltype(t1)>> ||
+      // is_base_of_v<T, remove_cvref_t<decltype(t1)>> is true;
+      if (S.Context.hasSameType(QualType(MemPtrClass, 0),
+                                S.BuiltinRemoveCVRef(ObjectT, Loc)) ||
+          S.BuiltinIsBaseOf(Args[1]->getBeginLoc(), QualType(MemPtrClass, 0),
+                            S.BuiltinRemoveCVRef(ObjectT, Loc))) {
+        return Args[1];
+      }
+
+      // (t1.get().*f)(t2, ..., tN) when f is a pointer to a member function of
+      // a class T and remove_cvref_t<decltype(t1)> is a specialization of
+      // reference_wrapper;
+      if (const auto *RD = ObjectT->getAsCXXRecordDecl()) {
+        if (RD->isInStdNamespace() &&
+            RD->getDeclName().getAsString() == "reference_wrapper") {
+          CXXScopeSpec SS;
+          IdentifierInfo *GetName = &S.Context.Idents.get("get");
+          UnqualifiedId GetID;
+          GetID.setIdentifier(GetName, Loc);
+
+          ExprResult MemExpr = S.ActOnMemberAccessExpr(
+              S.getCurScope(), Args[1], Loc, tok::period, SS,
+              /*TemplateKWLoc=*/SourceLocation(), GetID, nullptr);
+
+          if (MemExpr.isInvalid())
+            return ExprError();
+
+          return S.ActOnCallExpr(S.getCurScope(), MemExpr.get(), Loc, {}, Loc);
+        }
+      }
+
+      // ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a
+      // class T and t1 does not satisfy the previous two items;
+
+      return S.ActOnUnaryOp(S.getCurScope(), Loc, tok::star, Args[1]);
+    }();
+
+    if (ObjectArg.isInvalid())
+      return ExprError();
+
+    ExprResult BinOp = S.ActOnBinOp(S.getCurScope(), TheCall->getBeginLoc(),
+                                    tok::periodstar, ObjectArg.get(), Args[0]);
+    if (BinOp.isInvalid())
+      return ExprError();
+
+    if (MPT->isMemberDataPointer())
+      return BinOp;
+
+    auto *MemCall = new (S.Context)
+        ParenExpr(SourceLocation(), SourceLocation(), BinOp.get());
+
+    return S.ActOnCallExpr(S.getCurScope(), MemCall, TheCall->getBeginLoc(),
+                           Args.drop_front(2), TheCall->getRParenLoc());
+  }
+  return S.ActOnCallExpr(S.getCurScope(), Args.front(), TheCall->getBeginLoc(),
+                         Args.drop_front(), TheCall->getRParenLoc());
+}
+
 ExprResult
 Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
                                CallExpr *TheCall) {
@@ -2420,6 +2516,8 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     return BuiltinShuffleVector(TheCall);
     // TheCall will be freed by the smart pointer here, but that's fine, since
     // BuiltinShuffleVector guts it, but then doesn't release it.
+  case Builtin::BI__builtin_invoke:
+    return BuiltinInvoke(*this, TheCall);
   case Builtin::BI__builtin_prefetch:
     if (BuiltinPrefetch(TheCall))
       return ExprError();
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index f9f7c192f19d2..a43ac9eb7610d 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -26,6 +26,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/AttributeCommonInfo.h"
 #include "clang/Basic/CharInfo.h"
+#include "clang/Basic/ExceptionSpecificationType.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Lex/HeaderSearch.h"
@@ -3259,6 +3260,13 @@ static void AddFunctionParameterChunks(Preprocessor &PP,
       break;
     }
 
+    // C++23 introduces an explicit object parameter, a.k.a. "deducing this"
+    // Skip it for autocomplete and treat the next parameter as the first
+    // parameter
+    if (FirstParameter && Param->isExplicitObjectParameter()) {
+      continue;
+    }
+
     if (FirstParameter)
       FirstParameter = false;
     else
@@ -3427,6 +3435,25 @@ AddFunctionTypeQualsToCompletionString(CodeCompletionBuilder &Result,
   Result.AddInformativeChunk(Result.getAllocator().CopyString(QualsStr));
 }
 
+static void
+AddFunctionExceptSpecToCompletionString(std::string &NameAndSignature,
+                                        const FunctionDecl *Function) {
+  const auto *Proto = Function->getType()->getAs<FunctionProtoType>();
+  if (!Proto)
+    return;
+
+  auto ExceptInfo = Proto->getExceptionSpecInfo();
+  switch (ExceptInfo.Type) {
+  case EST_BasicNoexcept:
+  case EST_NoexceptTrue:
+    NameAndSignature += " noexcept";
+    break;
+
+  default:
+    break;
+  }
+}
+
 /// Add the name of the given declaration
 static void AddTypedNameChunk(ASTContext &Context, const PrintingPolicy &Policy,
                               const NamedDecl *ND,
@@ -3642,6 +3669,13 @@ CodeCompletionResult::createCodeCompletionStringForOverride(
   std::string NameAndSignature;
   // For overrides all chunks go into the result, none are informative.
   printOverrideString(*CCS, BeforeName, NameAndSignature);
+
+  // If the virtual function is declared with "noexcept", add it in the result
+  // code completion string.
+  const auto *VirtualFunc = dyn_cast<FunctionDecl>(Declaration);
+  assert(VirtualFunc && "overridden decl must be a function");
+  AddFunctionExceptSpecToCompletionString(NameAndSignature, VirtualFunc);
+
   NameAndSignature += " override";
 
   Result.AddTextChunk(Result.getAllocator().CopyString(BeforeName));
@@ -6283,54 +6317,6 @@ ProduceSignatureHelp(Sema &SemaRef, MutableArrayRef<ResultCandidate> Candidates,
   return getParamType(SemaRef, Candidates, CurrentArg);
 }
 
-// Given a callee expression `Fn`, if the call is through a function pointer,
-// try to find the declaration of the corresponding function pointer type,
-// so that we can recover argument names from it.
-static FunctionProtoTypeLoc GetPrototypeLoc(Expr *Fn) {
-  TypeLoc Target;
-
-  if (const auto *T = Fn->getType().getTypePtr()->getAs<TypedefType>()) {
-    Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc();
-
-  } else if (const auto *DR = dyn_cast<DeclRefExpr>(Fn)) {
-    const auto *D = DR->getDecl();
-    if (const auto *const VD = dyn_cast<VarDecl>(D)) {
-      Target = VD->getTypeSourceInfo()->getTypeLoc();
-    }
-  } else if (const auto *ME = dyn_cast<MemberExpr>(Fn)) {
-    const auto *MD = ME->getMemberDecl();
-    if (const auto *FD = dyn_cast<FieldDecl>(MD)) {
-      Target = FD->getTypeSourceInfo()->getTypeLoc();
-    }
-  }
-
-  if (!Target)
-    return {};
-
-  // Unwrap types that may be wrapping the function type
-  while (true) {
-    if (auto P = Target.getAs<PointerTypeLoc>()) {
-      Target = P.getPointeeLoc();
-      continue;
-    }
-    if (auto A = Target.getAs<AttributedTypeLoc>()) {
-      Target = A.getModifiedLoc();
-      continue;
-    }
-    if (auto P = Target.getAs<ParenTypeLoc>()) {
-      Target = P.getInnerLoc();
-      continue;
-    }
-    break;
-  }
-
-  if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
-    return F;
-  }
-
-  return {};
-}
-
 QualType
 SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
                                              SourceLocation OpenParLoc) {
@@ -6419,7 +6405,7 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
       // Lastly we check whether expression's type is function pointer or
       // function.
 
-      FunctionProtoTypeLoc P = GetPrototypeLoc(NakedFn);
+      FunctionProtoTypeLoc P = Resolver.getFunctionProtoTypeLoc(NakedFn);
       QualType T = NakedFn->getType();
       if (!T->getPointeeType().isNull())
         T = T->getPointeeType();
@@ -8465,13 +8451,11 @@ void SemaCodeCompletion::CodeCompleteObjCClassMessage(
 }
 
 void SemaCodeCompletion::CodeCompleteObjCInstanceMessage(
-    Scope *S, Expr *Receiver, ArrayRef<const IdentifierInfo *> SelIdents,
+    Scope *S, Expr *RecExpr, ArrayRef<const IdentifierInfo *> SelIdents,
     bool AtArgumentExpression, ObjCInterfaceDecl *Super) {
   typedef CodeCompletionResult Result;
   ASTContext &Context = getASTContext();
 
-  Expr *RecExpr = static_cast<Expr *>(Receiver);
-
   // If necessary, apply function/array conversion to the receiver.
   // C99 6.7.5.3p[7,8].
   if (RecExpr) {
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 1594b4423e4d2..4a16412a5b04b 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -898,7 +898,7 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
     Sema &S, const Sema::TemplateCompareNewDeclInfo &DeclInfo,
     const Expr *ConstrExpr) {
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
-      DeclInfo.getDecl(), DeclInfo.getLexicalDeclContext(), /*Final=*/false,
+      DeclInfo.getDecl(), DeclInfo.getDeclContext(), /*Final=*/false,
       /*Innermost=*/std::nullopt,
       /*RelativeToPrimary=*/true,
       /*Pattern=*/nullptr, /*ForConstraintInstantiation=*/true,
@@ -1138,8 +1138,8 @@ bool Sema::CheckFunctionTemplateConstraints(
   }
 
   CXXThisScopeRAII ThisScope(*this, Record, ThisQuals, Record != nullptr);
-  LambdaScopeForCallOperatorInstantiationRAII LambdaScope(
-      *this, const_cast<FunctionDecl *>(Decl), *MLTAL, Scope);
+  LambdaScopeForCallOperatorInstantiationRAII LambdaScope(*this, Decl, *MLTAL,
+                                                          Scope);
 
   return CheckConstraintSatisfaction(Template, TemplateAC, *MLTAL,
                                      PointOfInstantiation, Satisfaction);
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index a1389c6c034b1..d193a33f22393 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -783,7 +783,8 @@ static bool checkSuspensionContext(Sema &S, SourceLocation Loc,
   const auto ExprContext = S.currentEvaluationContext().ExprContext;
   const bool BadContext =
       S.isUnevaluatedContext() ||
-      ExprContext != Sema::ExpressionEvaluationContextRecord::EK_Other;
+      (ExprContext != Sema::ExpressionEvaluationContextRecord::EK_Other &&
+       ExprContext != Sema::ExpressionEvaluationContextRecord::EK_VariableInit);
   if (BadContext) {
     S.Diag(Loc, diag::err_coroutine_unevaluated_context) << Keyword;
     return false;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index a60a558155e69..11cbda412667f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -62,7 +62,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 #include "llvm/Support/SaveAndRestore.h"
 #include "llvm/TargetParser/Triple.h"
 #include <algorithm>
@@ -12401,12 +12400,9 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
 
 void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
   // [basic.start.main]p3
-  //    The main function shall not be declared with a linkage-specification.
-  if (FD->isExternCContext() ||
-      (FD->isExternCXXContext() &&
-       FD->getDeclContext()->getRedeclContext()->isTranslationUnit()))
-    Diag(FD->getLocation(), diag::ext_main_invalid_linkage_specification)
-        << FD->getLanguageLinkage();
+  //    The main function shall not be declared with C linkage-specification.
+  if (FD->isExternCContext())
+    Diag(FD->getLocation(), diag::ext_main_invalid_linkage_specification);
 
   // C++11 [basic.start.main]p3:
   //   A program that [...] declares main to be inline, static or
@@ -12490,6 +12486,15 @@ void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
                                 : FixItHint());
       FD->setInvalidDecl(true);
     }
+
+    // [basic.start.main]p3:
+    // A program that declares a function main that belongs to the global scope
+    // and is attached to a named module is ill-formed.
+    if (FD->isInNamedModule()) {
+      const SourceLocation start = FD->getTypeSpecStartLoc();
+      Diag(start, diag::warn_main_in_named_module)
+          << FixItHint::CreateInsertion(start, "extern \"C++\" ", true);
+    }
   }
 
   // Treat protoless main() as nullary.
@@ -13524,21 +13529,19 @@ bool Sema::GloballyUniqueObjectMightBeAccidentallyDuplicated(
 
   // The target is "hidden" (from the dynamic linker) if:
   // 1. On posix, it has hidden visibility, or
-  // 2. On windows, it has no import/export annotation
+  // 2. On windows, it has no import/export annotation, and neither does the
+  // class which directly contains it.
   if (Context.getTargetInfo().shouldDLLImportComdatSymbols()) {
     if (Target->hasAttr<DLLExportAttr>() || Target->hasAttr<DLLImportAttr>())
       return false;
 
     // If the variable isn't directly annotated, check to see if it's a member
     // of an annotated class.
-    const VarDecl *VD = dyn_cast<VarDecl>(Target);
+    const CXXRecordDecl *Ctx =
+        dyn_cast<CXXRecordDecl>(Target->getDeclContext());
+    if (Ctx && (Ctx->hasAttr<DLLExportAttr>() || Ctx->hasAttr<DLLImportAttr>()))
+      return false;
 
-    if (VD && VD->isStaticDataMember()) {
-      const CXXRecordDecl *Ctx = dyn_cast<CXXRecordDecl>(VD->getDeclContext());
-      if (Ctx &&
-          (Ctx->hasAttr<DLLExportAttr>() || Ctx->hasAttr<DLLImportAttr>()))
-        return false;
-    }
   } else if (Lnk.getVisibility() != HiddenVisibility) {
     // Posix case
     return false;
@@ -18633,8 +18636,7 @@ ExprResult Sema::VerifyBitField(SourceLocation FieldLoc,
         << FieldName << FieldTy << BitWidth->getSourceRange();
     return Diag(FieldLoc, diag::err_not_integral_type_anon_bitfield)
       << FieldTy << BitWidth->getSourceRange();
-  } else if (DiagnoseUnexpandedParameterPack(const_cast<Expr *>(BitWidth),
-                                             UPPC_BitFieldWidth))
+  } else if (DiagnoseUnexpandedParameterPack(BitWidth, UPPC_BitFieldWidth))
     return ExprError();
 
   // If the bit-width is type- or value-dependent, don't try to check
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 52313e6a15ff1..7ebb53318702c 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -1226,8 +1226,8 @@ static void handlePreferredName(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  S.Diag(AL.getLoc(), diag::err_attribute_preferred_name_arg_invalid)
-      << T << CTD;
+  S.Diag(AL.getLoc(), diag::err_attribute_not_typedef_for_specialization)
+      << T << AL << CTD;
   if (const auto *TT = T->getAs<TypedefType>())
     S.Diag(TT->getDecl()->getLocation(), diag::note_entity_declared_at)
         << TT->getDecl();
@@ -1979,6 +1979,10 @@ void clang::inferNoReturnAttr(Sema &S, const Decl *D) {
   if (!FD->hasAttr<NoReturnAttr>() && !FD->hasAttr<InferredNoReturnAttr>() &&
       isKnownToAlwaysThrow(FD)) {
     NonConstFD->addAttr(InferredNoReturnAttr::CreateImplicit(S.Context));
+
+    // Emit a diagnostic suggesting the function being marked [[noreturn]].
+    S.Diag(FD->getLocation(), diag::warn_suggest_noreturn_function)
+        << /*isFunction=*/0 << FD;
   }
 }
 
@@ -4190,8 +4194,9 @@ static void handleCallbackAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   if (CalleeFnProtoType->getNumParams() != EncodingIndices.size() - 1) {
-    S.Diag(AL.getLoc(), diag::err_callback_attribute_wrong_arg_count)
-        << QualType{CalleeFnProtoType, 0} << CalleeFnProtoType->getNumParams()
+    S.Diag(AL.getLoc(), diag::err_attribute_wrong_arg_count_for_func)
+        << AL << QualType{CalleeFnProtoType, 0}
+        << CalleeFnProtoType->getNumParams()
         << (unsigned)(EncodingIndices.size() - 1);
     return;
   }
@@ -8016,9 +8021,7 @@ void Sema::checkUnusedDeclAttributes(Declarator &D) {
 }
 
 void Sema::DiagnoseUnknownAttribute(const ParsedAttr &AL) {
-  std::string NormalizedFullName = '\'' + AL.getNormalizedFullName() + '\'';
   SourceRange NR = AL.getNormalizedRange();
-
   StringRef ScopeName = AL.getNormalizedScopeName();
   std::optional<StringRef> CorrectedScopeName =
       AL.tryGetCorrectedScopeName(ScopeName);
@@ -8040,7 +8043,7 @@ void Sema::DiagnoseUnknownAttribute(const ParsedAttr &AL) {
         Diag(CorrectedScopeName ? NR.getBegin() : AL.getRange().getBegin(),
              diag::warn_unknown_attribute_ignored_suggestion);
 
-    D << NormalizedFullName << CorrectedFullName;
+    D << AL << CorrectedFullName;
 
     if (AL.isExplicitScope()) {
       D << FixItHint::CreateReplacement(NR, CorrectedFullName) << NR;
@@ -8054,8 +8057,7 @@ void Sema::DiagnoseUnknownAttribute(const ParsedAttr &AL) {
       }
     }
   } else {
-    Diag(NR.getBegin(), diag::warn_unknown_attribute_ignored)
-        << NormalizedFullName << NR;
+    Diag(NR.getBegin(), diag::warn_unknown_attribute_ignored) << AL << NR;
   }
 }
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index c492e5f3f2dba..a49a8abb677c5 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -3431,7 +3431,7 @@ static bool IsUnusedPrivateField(const FieldDecl *FD) {
 NamedDecl *
 Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                                MultiTemplateParamsArg TemplateParameterLists,
-                               Expr *BW, const VirtSpecifiers &VS,
+                               Expr *BitWidth, const VirtSpecifiers &VS,
                                InClassInitStyle InitStyle) {
   const DeclSpec &DS = D.getDeclSpec();
   DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
@@ -3442,8 +3442,6 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
   if (Loc.isInvalid())
     Loc = D.getBeginLoc();
 
-  Expr *BitWidth = static_cast<Expr*>(BW);
-
   assert(isa<CXXRecordDecl>(CurContext));
   assert(!DS.isFriendSpecified());
 
@@ -13619,82 +13617,40 @@ bool Sema::CheckUsingDeclQualifier(SourceLocation UsingLoc, bool HasTypename,
       RequireCompleteDeclContext(const_cast<CXXScopeSpec&>(SS), NamedContext))
     return true;
 
-  if (getLangOpts().CPlusPlus11) {
-    // C++11 [namespace.udecl]p3:
-    //   In a using-declaration used as a member-declaration, the
-    //   nested-name-specifier shall name a base class of the class
-    //   being defined.
-
-    if (cast<CXXRecordDecl>(CurContext)->isProvablyNotDerivedFrom(
-                                 cast<CXXRecordDecl>(NamedContext))) {
+  // C++26 [namespace.udecl]p3:
+  //   In a using-declaration used as a member-declaration, each
+  //   using-declarator shall either name an enumerator or have a
+  //   nested-name-specifier naming a base class of the current class
+  //   ([expr.prim.this]). ...
+  // "have a nested-name-specifier naming a base class of the current class"
+  // was introduced by CWG400.
 
-      if (Cxx20Enumerator) {
-        Diag(NameLoc, diag::warn_cxx17_compat_using_decl_non_member_enumerator)
-            << SS.getRange();
-        return false;
-      }
+  if (cast<CXXRecordDecl>(CurContext)
+          ->isProvablyNotDerivedFrom(cast<CXXRecordDecl>(NamedContext))) {
 
-      if (CurContext == NamedContext) {
-        Diag(SS.getBeginLoc(),
-             diag::err_using_decl_nested_name_specifier_is_current_class)
-            << SS.getRange();
-        return !getLangOpts().CPlusPlus20;
-      }
+    if (Cxx20Enumerator) {
+      Diag(NameLoc, diag::warn_cxx17_compat_using_decl_non_member_enumerator)
+          << SS.getRange();
+      return false;
+    }
 
-      if (!cast<CXXRecordDecl>(NamedContext)->isInvalidDecl()) {
-        Diag(SS.getBeginLoc(),
-             diag::err_using_decl_nested_name_specifier_is_not_base_class)
-            << SS.getScopeRep() << cast<CXXRecordDecl>(CurContext)
-            << SS.getRange();
-      }
+    if (CurContext == NamedContext) {
+      Diag(SS.getBeginLoc(),
+           diag::err_using_decl_nested_name_specifier_is_current_class)
+          << SS.getRange();
       return true;
     }
 
-    return false;
-  }
-
-  // C++03 [namespace.udecl]p4:
-  //   A using-declaration used as a member-declaration shall refer
-  //   to a member of a base class of the class being defined [etc.].
-
-  // Salient point: SS doesn't have to name a base class as long as
-  // lookup only finds members from base classes.  Therefore we can
-  // diagnose here only if we can prove that can't happen,
-  // i.e. if the class hierarchies provably don't intersect.
-
-  // TODO: it would be nice if "definitely valid" results were cached
-  // in the UsingDecl and UsingShadowDecl so that these checks didn't
-  // need to be repeated.
-
-  llvm::SmallPtrSet<const CXXRecordDecl *, 4> Bases;
-  auto Collect = [&Bases](const CXXRecordDecl *Base) {
-    Bases.insert(Base);
+    if (!cast<CXXRecordDecl>(NamedContext)->isInvalidDecl()) {
+      Diag(SS.getBeginLoc(),
+           diag::err_using_decl_nested_name_specifier_is_not_base_class)
+          << SS.getScopeRep() << cast<CXXRecordDecl>(CurContext)
+          << SS.getRange();
+    }
     return true;
-  };
-
-  // Collect all bases. Return false if we find a dependent base.
-  if (!cast<CXXRecordDecl>(CurContext)->forallBases(Collect))
-    return false;
-
-  // Returns true if the base is dependent or is one of the accumulated base
-  // classes.
-  auto IsNotBase = [&Bases](const CXXRecordDecl *Base) {
-    return !Bases.count(Base);
-  };
-
-  // Return false if the class has a dependent base or if it or one
-  // of its bases is present in the base set of the current context.
-  if (Bases.count(cast<CXXRecordDecl>(NamedContext)) ||
-      !cast<CXXRecordDecl>(NamedContext)->forallBases(IsNotBase))
-    return false;
-
-  Diag(SS.getRange().getBegin(),
-       diag::err_using_decl_nested_name_specifier_is_not_base_class)
-    << SS.getScopeRep()
-    << cast<CXXRecordDecl>(CurContext)
-    << SS.getRange();
+  }
 
-  return true;
+  return false;
 }
 
 Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS,
@@ -18936,7 +18892,8 @@ void Sema::ActOnCXXEnterDeclInitializer(Scope *S, Decl *D) {
     EnterDeclaratorContext(S, D->getDeclContext());
 
   PushExpressionEvaluationContext(
-      ExpressionEvaluationContext::PotentiallyEvaluated, D);
+      ExpressionEvaluationContext::PotentiallyEvaluated, D,
+      ExpressionEvaluationContextRecord::EK_VariableInit);
 }
 
 void Sema::ActOnCXXExitDeclInitializer(Scope *S, Decl *D) {
@@ -18945,29 +18902,6 @@ void Sema::ActOnCXXExitDeclInitializer(Scope *S, Decl *D) {
   if (S && D->isOutOfLine())
     ExitDeclaratorContext(S);
 
-  if (getLangOpts().CPlusPlus23) {
-    // An expression or conversion is 'manifestly constant-evaluated' if it is:
-    // [...]
-    // - the initializer of a variable that is usable in constant expressions or
-    //   has constant initialization.
-    if (auto *VD = dyn_cast<VarDecl>(D);
-        VD && (VD->isUsableInConstantExpressions(Context) ||
-               VD->hasConstantInitialization())) {
-      // An expression or conversion is in an 'immediate function context' if it
-      // is potentially evaluated and either:
-      // [...]
-      // - it is a subexpression of a manifestly constant-evaluated expression
-      //   or conversion.
-      ExprEvalContexts.back().InImmediateFunctionContext = true;
-    }
-  }
-
-  // Unless the initializer is in an immediate function context (as determined
-  // above), this will evaluate all contained immediate function calls as
-  // constant expressions. If the initializer IS an immediate function context,
-  // the initializer has been determined to be a constant expression, and all
-  // such evaluations will be elided (i.e., as if we "knew the whole time" that
-  // it was a constant expression).
   PopExpressionEvaluationContext();
 }
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index a3f534ee6712e..01df1913339d9 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -55,7 +55,6 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaFixItUtils.h"
 #include "clang/Sema/SemaHLSL.h"
-#include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaPseudoObject.h"
@@ -17935,6 +17934,25 @@ HandleImmediateInvocations(Sema &SemaRef,
       Rec.isImmediateFunctionContext() || SemaRef.RebuildingImmediateInvocation)
     return;
 
+  // An expression or conversion is 'manifestly constant-evaluated' if it is:
+  // [...]
+  // - the initializer of a variable that is usable in constant expressions or
+  //   has constant initialization.
+  if (SemaRef.getLangOpts().CPlusPlus23 &&
+      Rec.ExprContext ==
+          Sema::ExpressionEvaluationContextRecord::EK_VariableInit) {
+    auto *VD = cast<VarDecl>(Rec.ManglingContextDecl);
+    if (VD->isUsableInConstantExpressions(SemaRef.Context) ||
+        VD->hasConstantInitialization()) {
+      // An expression or conversion is in an 'immediate function context' if it
+      // is potentially evaluated and either:
+      // [...]
+      // - it is a subexpression of a manifestly constant-evaluated expression
+      //   or conversion.
+      return;
+    }
+  }
+
   /// When we have more than 1 ImmediateInvocationCandidates or previously
   /// failed immediate invocations, we need to check for nested
   /// ImmediateInvocationCandidates in order to avoid duplicate diagnostics.
@@ -18253,6 +18271,11 @@ static bool isImplicitlyDefinableConstexprFunction(FunctionDecl *Func) {
 
   if (Func->isImplicitlyInstantiable() || !Func->isUserProvided())
     return true;
+
+  // Lambda conversion operators are never user provided.
+  if (CXXConversionDecl *Conv = dyn_cast<CXXConversionDecl>(Func))
+    return isLambdaConversionOperator(Conv);
+
   auto *CCD = dyn_cast<CXXConstructorDecl>(Func);
   return CCD && CCD->getInheritedConstructor();
 }
@@ -20170,6 +20193,9 @@ MarkExprReferenced(Sema &SemaRef, SourceLocation Loc, Decl *D, Expr *E,
   if (SemaRef.OpenMP().isInOpenMPDeclareTargetContext())
     SemaRef.OpenMP().checkDeclIsAllowedInOpenMPTarget(E, D);
 
+  if (SemaRef.getLangOpts().OpenACC)
+    SemaRef.OpenACC().CheckDeclReference(Loc, E, D);
+
   if (VarDecl *Var = dyn_cast<VarDecl>(D)) {
     DoMarkVarDeclReferenced(SemaRef, Loc, Var, E, RefsMinusAssignments);
     if (SemaRef.getLangOpts().CPlusPlus)
@@ -20629,6 +20655,7 @@ Sema::ConditionResult Sema::ActOnCondition(Scope *S, SourceLocation Loc,
     break;
 
   case ConditionKind::ConstexprIf:
+    // Note: this might produce a FullExpr
     Cond = CheckBooleanCondition(Loc, SubExpr, true);
     break;
 
@@ -20641,13 +20668,13 @@ Sema::ConditionResult Sema::ActOnCondition(Scope *S, SourceLocation Loc,
                               {SubExpr}, PreferredConditionType(CK));
     if (!Cond.get())
       return ConditionError();
-  }
-  // FIXME: FullExprArg doesn't have an invalid bit, so check nullness instead.
-  FullExprArg FullExpr = MakeFullExpr(Cond.get(), Loc);
-  if (!FullExpr.get())
+  } else if (Cond.isUsable() && !isa<FullExpr>(Cond.get()))
+    Cond = ActOnFinishFullExpr(Cond.get(), Loc, /*DiscardedValue*/ false);
+
+  if (!Cond.isUsable())
     return ConditionError();
 
-  return ConditionResult(*this, nullptr, FullExpr,
+  return ConditionResult(*this, nullptr, Cond,
                          CK == ConditionKind::ConstexprIf);
 }
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 4a86cbd0633b6..f17a338825423 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4357,7 +4357,8 @@ Sema::ConditionResult Sema::ActOnConditionVariable(Decl *ConditionVar,
       CheckConditionVariable(cast<VarDecl>(ConditionVar), StmtLoc, CK);
   if (E.isInvalid())
     return ConditionError();
-  return ConditionResult(*this, ConditionVar, MakeFullExpr(E.get(), StmtLoc),
+  E = ActOnFinishFullExpr(E.get(), /*DiscardedValue*/ false);
+  return ConditionResult(*this, ConditionVar, E,
                          CK == ConditionKind::ConstexprIf);
 }
 
@@ -4417,6 +4418,12 @@ ExprResult Sema::CheckCXXBooleanCondition(Expr *CondExpr, bool IsConstexpr) {
   if (!IsConstexpr || E.isInvalid() || E.get()->isValueDependent())
     return E;
 
+  E = ActOnFinishFullExpr(E.get(), E.get()->getExprLoc(),
+                          /*DiscardedValue*/ false,
+                          /*IsConstexpr*/ true);
+  if (E.isInvalid())
+    return E;
+
   // FIXME: Return this value to the caller so they don't need to recompute it.
   llvm::APSInt Cond;
   E = VerifyIntegerConstantExpression(
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index bad357b50929b..968b96990bd85 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -39,7 +39,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/DXILABI.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1200,10 +1200,9 @@ bool SemaHLSL::handleRootSignatureDecl(HLSLRootSignatureDecl *D,
   auto ReportOverlap = [this, Loc, &HadOverlap](const RangeInfo *Info,
                                                 const RangeInfo *OInfo) {
     HadOverlap = true;
-    auto CommonVis =
-        Info->Visibility == llvm::hlsl::rootsig::ShaderVisibility::All
-            ? OInfo->Visibility
-            : Info->Visibility;
+    auto CommonVis = Info->Visibility == llvm::dxbc::ShaderVisibility::All
+                         ? OInfo->Visibility
+                         : Info->Visibility;
     this->Diag(Loc, diag::err_hlsl_resource_range_overlap)
         << llvm::to_underlying(Info->Class) << Info->LowerBound
         << /*unbounded=*/(Info->UpperBound == RangeInfo::Unbounded)
@@ -1238,7 +1237,7 @@ bool SemaHLSL::handleRootSignatureDecl(HLSLRootSignatureDecl *D,
     // ResourceRanges in the former case and it will be an ArrayRef to just the
     // all visiblity ResourceRange in the latter case.
     ArrayRef<ResourceRange> OverlapRanges =
-        Info.Visibility == llvm::hlsl::rootsig::ShaderVisibility::All
+        Info.Visibility == llvm::dxbc::ShaderVisibility::All
             ? ArrayRef<ResourceRange>{Ranges}.drop_front()
             : ArrayRef<ResourceRange>{Ranges}.take_front();
 
@@ -1641,6 +1640,15 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
     return false;
 
   Attr *A = nullptr;
+
+  AttributeCommonInfo ACI(
+      AL.getLoc(), AttributeScopeInfo(AL.getScopeName(), AL.getScopeLoc()),
+      AttributeCommonInfo::NoSemaHandlerAttribute,
+      {
+          AttributeCommonInfo::AS_CXX11, 0, false /*IsAlignas*/,
+          false /*IsRegularKeywordAttribute*/
+      });
+
   switch (AL.getKind()) {
   case ParsedAttr::AT_HLSLResourceClass: {
     if (!AL.isArgIdent(0)) {
@@ -1660,16 +1668,16 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
           << "ResourceClass" << Identifier;
       return false;
     }
-    A = HLSLResourceClassAttr::Create(getASTContext(), RC, AL.getLoc());
+    A = HLSLResourceClassAttr::Create(getASTContext(), RC, ACI);
     break;
   }
 
   case ParsedAttr::AT_HLSLROV:
-    A = HLSLROVAttr::Create(getASTContext(), AL.getLoc());
+    A = HLSLROVAttr::Create(getASTContext(), ACI);
     break;
 
   case ParsedAttr::AT_HLSLRawBuffer:
-    A = HLSLRawBufferAttr::Create(getASTContext(), AL.getLoc());
+    A = HLSLRawBufferAttr::Create(getASTContext(), ACI);
     break;
 
   case ParsedAttr::AT_HLSLContainedType: {
@@ -1684,7 +1692,7 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
     if (SemaRef.RequireCompleteType(TSI->getTypeLoc().getBeginLoc(), QT,
                                     diag::err_incomplete_type))
       return false;
-    A = HLSLContainedTypeAttr::Create(getASTContext(), TSI, AL.getLoc());
+    A = HLSLContainedTypeAttr::Create(getASTContext(), TSI, ACI);
     break;
   }
 
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index e63d75c934f21..46aa7dd0dcc21 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -519,8 +519,31 @@ bool SemaOpenACC::CheckVarIsPointerType(OpenACCClauseKind ClauseKind,
   return false;
 }
 
+void SemaOpenACC::ActOnStartParseVar(OpenACCDirectiveKind DK,
+                                     OpenACCClauseKind CK) {
+  if (DK == OpenACCDirectiveKind::Cache) {
+    CacheInfo.ParsingCacheVarList = true;
+    CacheInfo.IsInvalidCacheRef = false;
+  }
+}
+
+void SemaOpenACC::ActOnInvalidParseVar() {
+  CacheInfo.ParsingCacheVarList = false;
+  CacheInfo.IsInvalidCacheRef = false;
+}
+
 ExprResult SemaOpenACC::ActOnCacheVar(Expr *VarExpr) {
   Expr *CurVarExpr = VarExpr->IgnoreParenImpCasts();
+  // Clear this here, so we can do the returns based on the invalid cache ref
+  // here.  Note all return statements in this function must return ExprError if
+  // IsInvalidCacheRef. However, instead of doing an 'early return' in that
+  // case, we can let the rest of the diagnostics happen, as the invalid decl
+  // ref is a warning.
+  bool WasParsingInvalidCacheRef =
+      CacheInfo.ParsingCacheVarList && CacheInfo.IsInvalidCacheRef;
+  CacheInfo.ParsingCacheVarList = false;
+  CacheInfo.IsInvalidCacheRef = false;
+
   if (!isa<ArraySectionExpr, ArraySubscriptExpr>(CurVarExpr)) {
     Diag(VarExpr->getExprLoc(), diag::err_acc_not_a_var_ref_cache);
     return ExprError();
@@ -540,19 +563,19 @@ ExprResult SemaOpenACC::ActOnCacheVar(Expr *VarExpr) {
   if (const auto *DRE = dyn_cast<DeclRefExpr>(CurVarExpr)) {
     if (isa<VarDecl, NonTypeTemplateParmDecl>(
             DRE->getFoundDecl()->getCanonicalDecl()))
-      return VarExpr;
+      return WasParsingInvalidCacheRef ? ExprEmpty() : VarExpr;
   }
 
   if (const auto *ME = dyn_cast<MemberExpr>(CurVarExpr)) {
     if (isa<FieldDecl>(ME->getMemberDecl()->getCanonicalDecl())) {
-      return VarExpr;
+      return WasParsingInvalidCacheRef ? ExprEmpty() : VarExpr;
     }
   }
 
   // Nothing really we can do here, as these are dependent.  So just return they
   // are valid.
   if (isa<DependentScopeDeclRefExpr, CXXDependentScopeMemberExpr>(CurVarExpr))
-    return VarExpr;
+    return WasParsingInvalidCacheRef ? ExprEmpty() : VarExpr;
 
   // There isn't really anything we can do in the case of a recovery expr, so
   // skip the diagnostic rather than produce a confusing diagnostic.
@@ -562,6 +585,45 @@ ExprResult SemaOpenACC::ActOnCacheVar(Expr *VarExpr) {
   Diag(VarExpr->getExprLoc(), diag::err_acc_not_a_var_ref_cache);
   return ExprError();
 }
+
+void SemaOpenACC::CheckDeclReference(SourceLocation Loc, Expr *E, Decl *D) {
+  if (!getLangOpts().OpenACC || !CacheInfo.ParsingCacheVarList || !D ||
+      D->isInvalidDecl())
+    return;
+  // A 'cache' variable reference MUST be declared before the 'acc.loop' we
+  // generate in codegen, so we have to mark it invalid here in some way.  We do
+  // so in a bit of a convoluted way as there is no good way to put this into
+  // the AST, so we store it in SemaOpenACC State.  We can check the Scope
+  // during parsing to make sure there is a 'loop' before the decl is
+  // declared(and skip during instantiation).
+  // We only diagnose this as a warning, as this isn't required by the standard
+  // (unless you take a VERY awkward reading of some awkward prose).
+
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // if we are at TU level, we are either doing some EXTRA wacky, or are in a
+  // template instantiation, so just give up.
+  if (CurScope->getDepth() == 0)
+    return;
+
+  while (CurScope) {
+    // If we run into a loop construct scope, than this is 'correct' in that the
+    // declaration is outside of the loop.
+    if (CurScope->isOpenACCLoopConstructScope())
+      return;
+
+    if (CurScope->isDeclScope(D)) {
+      Diag(Loc, diag::warn_acc_cache_var_not_outside_loop);
+
+      CacheInfo.IsInvalidCacheRef = true;
+    }
+
+    CurScope = CurScope->getParent();
+  }
+  // If we don't find the decl at all, we assume that it must be outside of the
+  // loop (or we aren't in a loop!) so skip the diagnostic.
+}
+
 ExprResult SemaOpenACC::ActOnVar(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
                                  Expr *VarExpr) {
   // This has unique enough restrictions that we should split it to a separate
diff --git a/clang/lib/Sema/SemaRISCV.cpp b/clang/lib/Sema/SemaRISCV.cpp
index 37e7468103752..cc110e1115ed5 100644
--- a/clang/lib/Sema/SemaRISCV.cpp
+++ b/clang/lib/Sema/SemaRISCV.cpp
@@ -1435,9 +1435,15 @@ void SemaRISCV::checkRVVTypeSupport(QualType Ty, SourceLocation Loc, Decl *D,
            !FeatureMap.lookup("zve64x"))
     Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve64x";
   else if (Info.ElementType->isFloat16Type() && !FeatureMap.lookup("zvfh") &&
-           !FeatureMap.lookup("zvfhmin"))
-    Diag(Loc, diag::err_riscv_type_requires_extension, D)
-        << Ty << "zvfh or zvfhmin";
+           !FeatureMap.lookup("zvfhmin") &&
+           !FeatureMap.lookup("xandesvpackfph"))
+    if (DeclareAndesVectorBuiltins) {
+      Diag(Loc, diag::err_riscv_type_requires_extension, D)
+          << Ty << "zvfh, zvfhmin or xandesvpackfph";
+    } else {
+      Diag(Loc, diag::err_riscv_type_requires_extension, D)
+          << Ty << "zvfh or zvfhmin";
+    }
   else if (Info.ElementType->isBFloat16Type() &&
            !FeatureMap.lookup("zvfbfmin") &&
            !FeatureMap.lookup("xandesvbfhcvt"))
diff --git a/clang/lib/Sema/SemaSPIRV.cpp b/clang/lib/Sema/SemaSPIRV.cpp
index c27d3fed2b990..76d3cff908b37 100644
--- a/clang/lib/Sema/SemaSPIRV.cpp
+++ b/clang/lib/Sema/SemaSPIRV.cpp
@@ -116,7 +116,7 @@ static bool checkGenericCastToPtr(Sema &SemaRef, CallExpr *Call) {
   RT = RT->getPointeeType();
   auto Qual = RT.getQualifiers();
   LangAS AddrSpace;
-  switch (static_cast<spirv::StorageClass>(StorageClass)) {
+  switch (StorageClass) {
   case spirv::StorageClass::CrossWorkgroup:
     AddrSpace =
         SemaRef.LangOpts.isSYCL() ? LangAS::sycl_global : LangAS::opencl_global;
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index a1c7143e3b874..9140f00fcd740 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -4111,7 +4111,6 @@ static ActionResult<CXXRecordDecl *> getPatternForClassTemplateSpecialization(
         if (Ambiguous) {
           // Partial ordering did not produce a clear winner. Complain.
           Inst.Clear();
-          ClassTemplateSpec->setInvalidDecl();
           S.Diag(PointOfInstantiation,
                  diag::err_partial_spec_ordering_ambiguous)
               << ClassTemplateSpec;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index a25bfd1c48dee..70a4c159f9805 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3029,7 +3029,7 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl(
   LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
 
   Sema::LambdaScopeForCallOperatorInstantiationRAII LambdaScope(
-      SemaRef, const_cast<CXXMethodDecl *>(D), TemplateArgs, Scope);
+      SemaRef, D, TemplateArgs, Scope);
 
   // Instantiate enclosing template arguments for friends.
   SmallVector<TemplateParameterList *, 4> TempParamLists;
@@ -6103,7 +6103,8 @@ void Sema::InstantiateVariableInitializer(
   ContextRAII SwitchContext(*this, Var->getDeclContext());
 
   EnterExpressionEvaluationContext Evaluated(
-      *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated, Var);
+      *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated, Var,
+      ExpressionEvaluationContextRecord::EK_VariableInit);
   currentEvaluationContext().InLifetimeExtendingContext =
       parentEvaluationContext().InLifetimeExtendingContext;
   currentEvaluationContext().RebuildDefaultArgOrDefaultInit =
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 785d7b89e778e..88aa63d3b9e83 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -8522,8 +8522,8 @@ static void HandleRISCVRVVVectorBitsTypeAttr(QualType &CurType,
     return;
   }
 
-  auto VScale =
-      S.Context.getTargetInfo().getVScaleRange(S.getLangOpts(), false);
+  auto VScale = S.Context.getTargetInfo().getVScaleRange(
+      S.getLangOpts(), TargetInfo::ArmStreamingKind::NotStreaming);
   if (!VScale || !VScale->first || VScale->first != VScale->second) {
     S.Diag(Attr.getLoc(), diag::err_attribute_riscv_rvv_bits_unsupported)
         << Attr;
diff --git a/clang/lib/Sema/SemaTypeTraits.cpp b/clang/lib/Sema/SemaTypeTraits.cpp
index cb3d9b77ee4dd..1d8687e4bf1c1 100644
--- a/clang/lib/Sema/SemaTypeTraits.cpp
+++ b/clang/lib/Sema/SemaTypeTraits.cpp
@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
+#include "clang/Basic/DiagnosticIDs.h"
 #include "clang/Basic/DiagnosticParse.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TypeTraits.h"
@@ -1579,6 +1581,58 @@ ExprResult Sema::ActOnTypeTrait(TypeTrait Kind, SourceLocation KWLoc,
   return BuildTypeTrait(Kind, KWLoc, ConvertedArgs, RParenLoc);
 }
 
+bool Sema::BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT,
+                           QualType RhsT) {
+  // C++0x [meta.rel]p2
+  // Base is a base class of Derived without regard to cv-qualifiers or
+  // Base and Derived are not unions and name the same class type without
+  // regard to cv-qualifiers.
+
+  const RecordType *lhsRecord = LhsT->getAs<RecordType>();
+  const RecordType *rhsRecord = RhsT->getAs<RecordType>();
+  if (!rhsRecord || !lhsRecord) {
+    const ObjCObjectType *LHSObjTy = LhsT->getAs<ObjCObjectType>();
+    const ObjCObjectType *RHSObjTy = RhsT->getAs<ObjCObjectType>();
+    if (!LHSObjTy || !RHSObjTy)
+      return false;
+
+    ObjCInterfaceDecl *BaseInterface = LHSObjTy->getInterface();
+    ObjCInterfaceDecl *DerivedInterface = RHSObjTy->getInterface();
+    if (!BaseInterface || !DerivedInterface)
+      return false;
+
+    if (RequireCompleteType(RhsTLoc, RhsT,
+                            diag::err_incomplete_type_used_in_type_trait_expr))
+      return false;
+
+    return BaseInterface->isSuperClassOf(DerivedInterface);
+  }
+
+  assert(Context.hasSameUnqualifiedType(LhsT, RhsT) ==
+         (lhsRecord == rhsRecord));
+
+  // Unions are never base classes, and never have base classes.
+  // It doesn't matter if they are complete or not. See PR#41843
+  if (lhsRecord && lhsRecord->getDecl()->isUnion())
+    return false;
+  if (rhsRecord && rhsRecord->getDecl()->isUnion())
+    return false;
+
+  if (lhsRecord == rhsRecord)
+    return true;
+
+  // C++0x [meta.rel]p2:
+  //   If Base and Derived are class types and are different types
+  //   (ignoring possible cv-qualifiers) then Derived shall be a
+  //   complete type.
+  if (RequireCompleteType(RhsTLoc, RhsT,
+                          diag::err_incomplete_type_used_in_type_trait_expr))
+    return false;
+
+  return cast<CXXRecordDecl>(rhsRecord->getDecl())
+      ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getDecl()));
+}
+
 static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
                                     const TypeSourceInfo *Lhs,
                                     const TypeSourceInfo *Rhs,
@@ -1590,58 +1644,9 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
          "Cannot evaluate traits of dependent types");
 
   switch (BTT) {
-  case BTT_IsBaseOf: {
-    // C++0x [meta.rel]p2
-    // Base is a base class of Derived without regard to cv-qualifiers or
-    // Base and Derived are not unions and name the same class type without
-    // regard to cv-qualifiers.
-
-    const RecordType *lhsRecord = LhsT->getAs<RecordType>();
-    const RecordType *rhsRecord = RhsT->getAs<RecordType>();
-    if (!rhsRecord || !lhsRecord) {
-      const ObjCObjectType *LHSObjTy = LhsT->getAs<ObjCObjectType>();
-      const ObjCObjectType *RHSObjTy = RhsT->getAs<ObjCObjectType>();
-      if (!LHSObjTy || !RHSObjTy)
-        return false;
-
-      ObjCInterfaceDecl *BaseInterface = LHSObjTy->getInterface();
-      ObjCInterfaceDecl *DerivedInterface = RHSObjTy->getInterface();
-      if (!BaseInterface || !DerivedInterface)
-        return false;
-
-      if (Self.RequireCompleteType(
-              Rhs->getTypeLoc().getBeginLoc(), RhsT,
-              diag::err_incomplete_type_used_in_type_trait_expr))
-        return false;
-
-      return BaseInterface->isSuperClassOf(DerivedInterface);
-    }
-
-    assert(Self.Context.hasSameUnqualifiedType(LhsT, RhsT) ==
-           (lhsRecord == rhsRecord));
-
-    // Unions are never base classes, and never have base classes.
-    // It doesn't matter if they are complete or not. See PR#41843
-    if (lhsRecord && lhsRecord->getDecl()->isUnion())
-      return false;
-    if (rhsRecord && rhsRecord->getDecl()->isUnion())
-      return false;
-
-    if (lhsRecord == rhsRecord)
-      return true;
-
-    // C++0x [meta.rel]p2:
-    //   If Base and Derived are class types and are different types
-    //   (ignoring possible cv-qualifiers) then Derived shall be a
-    //   complete type.
-    if (Self.RequireCompleteType(
-            Rhs->getTypeLoc().getBeginLoc(), RhsT,
-            diag::err_incomplete_type_used_in_type_trait_expr))
-      return false;
+  case BTT_IsBaseOf:
+    return Self.BuiltinIsBaseOf(Rhs->getTypeLoc().getBeginLoc(), LhsT, RhsT);
 
-    return cast<CXXRecordDecl>(rhsRecord->getDecl())
-        ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getDecl()));
-  }
   case BTT_IsVirtualBaseOf: {
     const RecordType *BaseRecord = LhsT->getAs<RecordType>();
     const RecordType *DerivedRecord = RhsT->getAs<RecordType>();
@@ -1960,6 +1965,7 @@ static std::optional<TypeTrait> StdNameToTypeTrait(StringRef Name) {
       .Case("is_assignable", TypeTrait::BTT_IsAssignable)
       .Case("is_empty", TypeTrait::UTT_IsEmpty)
       .Case("is_standard_layout", TypeTrait::UTT_IsStandardLayout)
+      .Case("is_constructible", TypeTrait::TT_IsConstructible)
       .Default(std::nullopt);
 }
 
@@ -1996,8 +2002,16 @@ static ExtractedTypeTraitInfo ExtractTypeTraitFromExpression(const Expr *E) {
     Trait = StdNameToTypeTrait(Name);
     if (!Trait)
       return std::nullopt;
-    for (const auto &Arg : VD->getTemplateArgs().asArray())
-      Args.push_back(Arg.getAsType());
+    for (const auto &Arg : VD->getTemplateArgs().asArray()) {
+      if (Arg.getKind() == TemplateArgument::ArgKind::Pack) {
+        for (const auto &InnerArg : Arg.pack_elements())
+          Args.push_back(InnerArg.getAsType());
+      } else if (Arg.getKind() == TemplateArgument::ArgKind::Type) {
+        Args.push_back(Arg.getAsType());
+      } else {
+        llvm_unreachable("Unexpected kind");
+      }
+    }
     return {{Trait.value(), std::move(Args)}};
   }
 
@@ -2270,6 +2284,60 @@ static void DiagnoseNonTriviallyCopyableReason(Sema &SemaRef,
   }
 }
 
+static void DiagnoseNonConstructibleReason(
+    Sema &SemaRef, SourceLocation Loc,
+    const llvm::SmallVector<clang::QualType, 1> &Ts) {
+  if (Ts.empty()) {
+    return;
+  }
+
+  bool ContainsVoid = false;
+  for (const QualType &ArgTy : Ts) {
+    ContainsVoid |= ArgTy->isVoidType();
+  }
+
+  if (ContainsVoid)
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::CVVoidType;
+
+  QualType T = Ts[0];
+  if (T->isFunctionType())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::FunctionType;
+
+  if (T->isIncompleteArrayType())
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::IncompleteArrayType;
+
+  const CXXRecordDecl *D = T->getAsCXXRecordDecl();
+  if (!D || D->isInvalidDecl() || !D->hasDefinition())
+    return;
+
+  llvm::BumpPtrAllocator OpaqueExprAllocator;
+  SmallVector<Expr *, 2> ArgExprs;
+  ArgExprs.reserve(Ts.size() - 1);
+  for (unsigned I = 1, N = Ts.size(); I != N; ++I) {
+    QualType ArgTy = Ts[I];
+    if (ArgTy->isObjectType() || ArgTy->isFunctionType())
+      ArgTy = SemaRef.Context.getRValueReferenceType(ArgTy);
+    ArgExprs.push_back(
+        new (OpaqueExprAllocator.Allocate<OpaqueValueExpr>())
+            OpaqueValueExpr(Loc, ArgTy.getNonLValueExprType(SemaRef.Context),
+                            Expr::getValueKindForType(ArgTy)));
+  }
+
+  EnterExpressionEvaluationContext Unevaluated(
+      SemaRef, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::ContextRAII TUContext(SemaRef,
+                              SemaRef.Context.getTranslationUnitDecl());
+  InitializedEntity To(InitializedEntity::InitializeTemporary(T));
+  InitializationKind InitKind(InitializationKind::CreateDirect(Loc, Loc, Loc));
+  InitializationSequence Init(SemaRef, To, InitKind, ArgExprs);
+
+  Init.Diagnose(SemaRef, To, InitKind, ArgExprs);
+  SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
+}
+
 static void DiagnoseNonTriviallyCopyableReason(Sema &SemaRef,
                                                SourceLocation Loc, QualType T) {
   SemaRef.Diag(Loc, diag::note_unsatisfied_trait)
@@ -2556,6 +2624,9 @@ void Sema::DiagnoseTypeTraitDetails(const Expr *E) {
   case UTT_IsStandardLayout:
     DiagnoseNonStandardLayoutReason(*this, E->getBeginLoc(), Args[0]);
     break;
+  case TT_IsConstructible:
+    DiagnoseNonConstructibleReason(*this, E->getBeginLoc(), Args);
+    break;
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 0d58587cb8a99..758012f894a41 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -4561,6 +4561,13 @@ bool TreeTransform<Derived>::TransformExprs(Expr *const *Inputs,
 template <typename Derived>
 Sema::ConditionResult TreeTransform<Derived>::TransformCondition(
     SourceLocation Loc, VarDecl *Var, Expr *Expr, Sema::ConditionKind Kind) {
+
+  EnterExpressionEvaluationContext Eval(
+      SemaRef, Sema::ExpressionEvaluationContext::ConstantEvaluated,
+      /*LambdaContextDecl=*/nullptr,
+      /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
+      /*ShouldEnter=*/Kind == Sema::ConditionKind::ConstexprIf);
+
   if (Var) {
     VarDecl *ConditionVar = cast_or_null<VarDecl>(
         getDerived().TransformDefinition(Var->getLocation(), Var));
@@ -15723,13 +15730,9 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
 
   // FIXME: Sema's lambda-building mechanism expects us to push an expression
   // evaluation context even if we're not transforming the function body.
-  getSema().PushExpressionEvaluationContext(
-      E->getCallOperator()->isConsteval() ?
-      Sema::ExpressionEvaluationContext::ImmediateFunctionContext :
-      Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
-  getSema().currentEvaluationContext().InImmediateEscalatingFunctionContext =
-      getSema().getLangOpts().CPlusPlus20 &&
-      E->getCallOperator()->isImmediateEscalating();
+  getSema().PushExpressionEvaluationContextForFunction(
+      Sema::ExpressionEvaluationContext::PotentiallyEvaluated,
+      E->getCallOperator());
 
   Sema::CodeSynthesisContext C;
   C.Kind = clang::Sema::CodeSynthesisContext::LambdaExpressionSubstitution;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 486971818f109..30e0973149594 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -278,51 +278,57 @@ static bool checkLanguageOptions(const LangOptions &LangOpts,
                                  StringRef ModuleFilename,
                                  DiagnosticsEngine *Diags,
                                  bool AllowCompatibleDifferences = true) {
-#define LANGOPT(Name, Bits, Default, Description)                              \
-  if (ExistingLangOpts.Name != LangOpts.Name) {                                \
-    if (Diags) {                                                               \
-      if (Bits == 1)                                                           \
-        Diags->Report(diag::err_ast_file_langopt_mismatch)                     \
-            << Description << LangOpts.Name << ExistingLangOpts.Name           \
-            << ModuleFilename;                                                 \
-      else                                                                     \
-        Diags->Report(diag::err_ast_file_langopt_value_mismatch)               \
-            << Description << ModuleFilename;                                  \
+  // FIXME: Replace with C++20 `using enum LangOptions::CompatibilityKind`.
+  using CK = LangOptions::CompatibilityKind;
+
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
+  if constexpr (CK::Compatibility != CK::Benign) {                             \
+    if ((CK::Compatibility == CK::NotCompatible) ||                            \
+        (CK::Compatibility == CK::Compatible &&                                \
+         !AllowCompatibleDifferences)) {                                       \
+      if (ExistingLangOpts.Name != LangOpts.Name) {                            \
+        if (Diags) {                                                           \
+          if (Bits == 1)                                                       \
+            Diags->Report(diag::err_ast_file_langopt_mismatch)                 \
+                << Description << LangOpts.Name << ExistingLangOpts.Name       \
+                << ModuleFilename;                                             \
+          else                                                                 \
+            Diags->Report(diag::err_ast_file_langopt_value_mismatch)           \
+                << Description << ModuleFilename;                              \
+        }                                                                      \
+        return true;                                                           \
+      }                                                                        \
     }                                                                          \
-    return true;                                                               \
   }
 
-#define VALUE_LANGOPT(Name, Bits, Default, Description)                        \
-  if (ExistingLangOpts.Name != LangOpts.Name) {                                \
-    if (Diags)                                                                 \
-      Diags->Report(diag::err_ast_file_langopt_value_mismatch)                 \
-          << Description << ModuleFilename;                                    \
-    return true;                                                               \
+#define VALUE_LANGOPT(Name, Bits, Default, Compatibility, Description)         \
+  if constexpr (CK::Compatibility != CK::Benign) {                             \
+    if ((CK::Compatibility == CK::NotCompatible) ||                            \
+        (CK::Compatibility == CK::Compatible &&                                \
+         !AllowCompatibleDifferences)) {                                       \
+      if (ExistingLangOpts.Name != LangOpts.Name) {                            \
+        if (Diags)                                                             \
+          Diags->Report(diag::err_ast_file_langopt_value_mismatch)             \
+              << Description << ModuleFilename;                                \
+        return true;                                                           \
+      }                                                                        \
+    }                                                                          \
   }
 
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)                   \
-  if (ExistingLangOpts.get##Name() != LangOpts.get##Name()) {                  \
-    if (Diags)                                                                 \
-      Diags->Report(diag::err_ast_file_langopt_value_mismatch)                 \
-          << Description << ModuleFilename;                                    \
-    return true;                                                               \
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
+  if constexpr (CK::Compatibility != CK::Benign) {                             \
+    if ((CK::Compatibility == CK::NotCompatible) ||                            \
+        (CK::Compatibility == CK::Compatible &&                                \
+         !AllowCompatibleDifferences)) {                                       \
+      if (ExistingLangOpts.get##Name() != LangOpts.get##Name()) {              \
+        if (Diags)                                                             \
+          Diags->Report(diag::err_ast_file_langopt_value_mismatch)             \
+              << Description << ModuleFilename;                                \
+        return true;                                                           \
+      }                                                                        \
+    }                                                                          \
   }
 
-#define COMPATIBLE_LANGOPT(Name, Bits, Default, Description)  \
-  if (!AllowCompatibleDifferences)                            \
-    LANGOPT(Name, Bits, Default, Description)
-
-#define COMPATIBLE_ENUM_LANGOPT(Name, Bits, Default, Description)  \
-  if (!AllowCompatibleDifferences)                                 \
-    ENUM_LANGOPT(Name, Bits, Default, Description)
-
-#define COMPATIBLE_VALUE_LANGOPT(Name, Bits, Default, Description) \
-  if (!AllowCompatibleDifferences)                                 \
-    VALUE_LANGOPT(Name, Bits, Default, Description)
-
-#define BENIGN_LANGOPT(Name, Bits, Default, Description)
-#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
-#define BENIGN_VALUE_LANGOPT(Name, Bits, Default, Description)
 #include "clang/Basic/LangOptions.def"
 
   if (ExistingLangOpts.ModuleFeatures != LangOpts.ModuleFeatures) {
@@ -6353,9 +6359,9 @@ bool ASTReader::ParseLanguageOptions(const RecordData &Record,
                                      bool AllowCompatibleDifferences) {
   LangOptions LangOpts;
   unsigned Idx = 0;
-#define LANGOPT(Name, Bits, Default, Description) \
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
   LangOpts.Name = Record[Idx++];
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
   LangOpts.set##Name(static_cast<LangOptions::Type>(Record[Idx++]));
 #include "clang/Basic/LangOptions.def"
 #define SANITIZER(NAME, ID)                                                    \
@@ -8564,14 +8570,22 @@ bool ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
     Find(It->second.Table, Name);
   }
 
-  if (auto *NamedModule =
-          OriginalDC ? cast<Decl>(OriginalDC)->getTopLevelOwningNamedModule()
-                     : nullptr) {
+  auto FindModuleLocalLookup = [&, this](Module *NamedModule) {
     if (auto It = ModuleLocalLookups.find(DC); It != ModuleLocalLookups.end()) {
       ++NumModuleLocalVisibleDeclContexts;
       Find(It->second.Table, std::make_pair(Name, NamedModule));
     }
-  }
+  };
+  if (auto *NamedModule =
+          OriginalDC ? cast<Decl>(OriginalDC)->getTopLevelOwningNamedModule()
+                     : nullptr)
+    FindModuleLocalLookup(NamedModule);
+  // See clang/test/Modules/ModulesLocalNamespace.cppm for the motiviation case.
+  // We're going to find a decl but the decl context of the lookup is
+  // unspecified. In this case, the OriginalDC may be the decl context in other
+  // module.
+  if (ContextObj && ContextObj->getCurrentNamedModule())
+    FindModuleLocalLookup(ContextObj->getCurrentNamedModule());
 
   if (auto It = TULocalLookups.find(DC); It != TULocalLookups.end()) {
     ++NumTULocalVisibleDeclContexts;
@@ -9455,8 +9469,7 @@ void ASTReader::AssignedLambdaNumbering(CXXRecordDecl *Lambda) {
 
   // Keep track of this lambda so it can be merged with another lambda that
   // is loaded later.
-  LambdaDeclarationsForMerging.insert(
-      {LambdaInfo, const_cast<CXXRecordDecl *>(Lambda)});
+  LambdaDeclarationsForMerging.insert({LambdaInfo, Lambda});
 }
 
 void ASTReader::LoadSelector(Selector Sel) {
@@ -9675,7 +9688,7 @@ ModuleFile *ASTReader::getLocalModuleFile(ModuleFile &M, unsigned ID) const {
     // It's a prefix (preamble, PCH, ...). Look it up by index.
    int IndexFromEnd = static_cast<int>(ID >> 1);
     assert(IndexFromEnd && "got reference to unknown module file");
-    return getModuleManager().pch_modules().end()[-static_cast<int>(IndexFromEnd)];
+    return getModuleManager().pch_modules().end()[-IndexFromEnd];
   }
 }
 
@@ -9725,10 +9738,6 @@ bool ASTReader::wasThisDeclarationADefinition(const FunctionDecl *FD) {
   return ThisDeclarationWasADefinitionSet.contains(FD);
 }
 
-bool ASTReader::hasInitializerWithSideEffects(const VarDecl *VD) const {
-  return InitSideEffectVars.count(VD);
-}
-
 Selector ASTReader::getLocalSelector(ModuleFile &M, unsigned LocalID) {
   return DecodeSelector(getGlobalSelectorID(M, LocalID));
 }
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0ffd78424be0d..8bde213dc4cb3 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1628,9 +1628,6 @@ RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) {
     VD->NonParmVarDeclBits.PreviousDeclInSameBlockScope =
         VarDeclBits.getNextBit();
 
-    if (VarDeclBits.getNextBit())
-      Reader.InitSideEffectVars.insert(VD);
-
     VD->NonParmVarDeclBits.EscapingByref = VarDeclBits.getNextBit();
     HasDeducedType = VarDeclBits.getNextBit();
     VD->NonParmVarDeclBits.ImplicitParamKind =
@@ -1701,6 +1698,8 @@ void ASTDeclReader::ReadVarDeclInit(VarDecl *VD) {
     Eval->HasConstantInitialization = (Val & 2) != 0;
     Eval->HasConstantDestruction = (Val & 4) != 0;
     Eval->WasEvaluated = (Val & 8) != 0;
+    Eval->HasSideEffects = (Val & 16) != 0;
+    Eval->CheckedForSideEffects = true;
     if (Eval->WasEvaluated) {
       Eval->Evaluated = Record.readAPValue();
       if (Eval->Evaluated.needsCleanup())
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 8945407cf666e..0166e493bf03f 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -1733,7 +1733,7 @@ void ASTStmtReader::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) {
 void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   E->CXXOperatorCallExprBits.OperatorKind = Record.readInt();
-  E->Range = Record.readSourceRange();
+  E->BeginLoc = Record.readSourceLocation();
 }
 
 void ASTStmtReader::VisitCXXRewrittenBinaryOperator(
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 06cd6c7305114..1a20fc9595dce 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1612,9 +1612,9 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, StringRef isysroot) {
   // Language options.
   Record.clear();
   const LangOptions &LangOpts = PP.getLangOpts();
-#define LANGOPT(Name, Bits, Default, Description) \
+#define LANGOPT(Name, Bits, Default, Compatibility, Description)               \
   Record.push_back(LangOpts.Name);
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Compatibility, Description)    \
   Record.push_back(static_cast<unsigned>(LangOpts.get##Name()));
 #include "clang/Basic/LangOptions.def"
 #define SANITIZER(NAME, ID)                                                    \
@@ -6652,7 +6652,7 @@ void ASTWriter::AddFileID(FileID FID, RecordDataImpl &Record) {
 
 SourceLocationEncoding::RawLocEncoding
 ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
-  unsigned BaseOffset = 0;
+  SourceLocation::UIntTy BaseOffset = 0;
   unsigned ModuleFileIndex = 0;
 
   // See SourceLocationEncoding.h for the encoding details.
@@ -6968,9 +6968,7 @@ void ASTWriter::associateDeclWithFile(const Decl *D, LocalDeclID ID) {
   SourceManager &SM = PP->getSourceManager();
   SourceLocation FileLoc = SM.getFileLoc(Loc);
   assert(SM.isLocalSourceLocation(FileLoc));
-  FileID FID;
-  unsigned Offset;
-  std::tie(FID, Offset) = SM.getDecomposedLoc(FileLoc);
+  auto [FID, Offset] = SM.getDecomposedLoc(FileLoc);
   if (FID.isInvalid())
     return;
   assert(SM.getSLocEntry(FID).isFile());
@@ -7320,6 +7318,10 @@ void ASTRecordWriter::AddVarDeclInit(const VarDecl *VD) {
 
   uint64_t Val = 1;
   if (EvaluatedStmt *ES = VD->getEvaluatedStmt()) {
+    // This may trigger evaluation, so run it first
+    if (VD->hasInitWithSideEffects())
+      Val |= 16;
+    assert(ES->CheckedForSideEffects);
     Val |= (ES->HasConstantInitialization ? 2 : 0);
     Val |= (ES->HasConstantDestruction ? 4 : 0);
     APValue *Evaluated = VD->getEvaluatedValue();
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 2e390dbe79ec6..b8a68afc30738 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -23,7 +23,6 @@
 #include "clang/Serialization/ASTRecordWriter.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Support/ErrorHandling.h"
-#include <optional>
 using namespace clang;
 using namespace serialization;
 
@@ -302,7 +301,7 @@ namespace clang {
     }
     MutableArrayRef<FunctionTemplateSpecializationInfo>
     getPartialSpecializations(FunctionTemplateDecl::Common *) {
-      return std::nullopt;
+      return {};
     }
 
     template<typename DeclTy>
@@ -1306,7 +1305,6 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
     VarDeclBits.addBit(D->isConstexpr());
     VarDeclBits.addBit(D->isInitCapture());
     VarDeclBits.addBit(D->isPreviousDeclInSameBlockScope());
-    VarDeclBits.addBit(D->hasInitWithSideEffects());
 
     VarDeclBits.addBit(D->isEscapingByref());
     HasDeducedType = D->getType()->getContainedDeducedType();
@@ -2935,7 +2933,6 @@ void ASTWriter::WriteDeclAbbrevs() {
   // CXXOperatorCallExpr
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Operator Kind
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location
   CXXOperatorCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv));
 
   // Abbreviation for EXPR_CXX_MEMBER_CALL
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 87536be8c8d98..be9bad9e96cc1 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -1702,7 +1702,7 @@ void ASTStmtWriter::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) {
 void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   Record.push_back(E->getOperator());
-  Record.AddSourceRange(E->Range);
+  Record.AddSourceLocation(E->BeginLoc);
 
   if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind()))
     AbbrevToUse = Writer.getCXXOperatorCallExprAbbrev();
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index 4d12fdcec1f1a..31cb150892a5d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -2223,6 +2223,44 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallEvent &Call,
         Result = lastElement;
     }
 
+    // For bounded method, amountCopied take the minimum of two values,
+    // for ConcatFnKind::strlcat:
+    // amountCopied = min (size - dstLen - 1 , srcLen)
+    // for others:
+    // amountCopied = min (srcLen, size)
+    // So even if we don't know about amountCopied, as long as one of them will
+    // not cause an out-of-bound access, the whole function's operation will not
+    // too, that will avoid invalidating the superRegion of data member in that
+    // situation.
+    bool CouldAccessOutOfBound = true;
+    if (IsBounded && amountCopied.isUnknown()) {
+      auto CouldAccessOutOfBoundForSVal =
+          [&](std::optional<NonLoc> Val) -> bool {
+        if (!Val)
+          return true;
+        return !isFirstBufInBound(C, state, C.getSVal(Dst.Expression),
+                                  Dst.Expression->getType(), *Val,
+                                  C.getASTContext().getSizeType());
+      };
+
+      CouldAccessOutOfBound = CouldAccessOutOfBoundForSVal(strLengthNL);
+
+      if (CouldAccessOutOfBound) {
+        // Get the max number of characters to copy.
+        const Expr *LenExpr = Call.getArgExpr(2);
+        SVal LenVal = state->getSVal(LenExpr, LCtx);
+
+        // Protect against misdeclared strncpy().
+        LenVal = svalBuilder.evalCast(LenVal, sizeTy, LenExpr->getType());
+
+        // Because analyzer doesn't handle expressions like `size -
+        // dstLen - 1` very well, we roughly use `size` for
+        // ConcatFnKind::strlcat here, same with other concat kinds.
+        CouldAccessOutOfBound =
+            CouldAccessOutOfBoundForSVal(LenVal.getAs<NonLoc>());
+      }
+    }
+
     // Invalidate the destination (regular invalidation without pointer-escaping
     // the address of the top-level region). This must happen before we set the
     // C string length because invalidation will clear the length.
@@ -2230,9 +2268,13 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallEvent &Call,
     // can use LazyCompoundVals to copy the source values into the destination.
     // This would probably remove any existing bindings past the end of the
     // string, but that's still an improvement over blank invalidation.
-    state = invalidateDestinationBufferBySize(
-        C, state, Dst.Expression, Call.getCFGElementRef(), *dstRegVal,
-        amountCopied, C.getASTContext().getSizeType());
+    if (CouldAccessOutOfBound)
+      state = invalidateDestinationBufferBySize(
+          C, state, Dst.Expression, Call.getCFGElementRef(), *dstRegVal,
+          amountCopied, C.getASTContext().getSizeType());
+    else
+      state = invalidateDestinationBufferNeverOverflows(
+          C, state, Call.getCFGElementRef(), *dstRegVal);
 
     // Invalidate the source (const-invalidation without const-pointer-escaping
     // the address of the top-level region).
diff --git a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 86476b32309c3..23935647a5826 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -253,6 +253,8 @@ class FindUninitializedField {
       const RecordDecl *RD = RT->getDecl()->getDefinition();
       assert(RD && "Referred record has no definition");
       for (const auto *I : RD->fields()) {
+        if (I->isUnnamedBitField())
+          continue;
         const FieldRegion *FR = MrMgr.getFieldRegion(I, R);
         FieldChain.push_back(I);
         T = I->getType();
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index a33e61fabc2c1..0a58d7cc2635a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -333,19 +333,64 @@ template <typename T> static bool isStandardNewDelete(const T &FD) {
   return isStandardDelete(FD) || isStandardNew(FD);
 }
 
+namespace {
+
 //===----------------------------------------------------------------------===//
-// Definition of the MallocChecker class.
+// Utility classes that provide access to the bug types and can model that some
+// of the bug types are shared by multiple checker frontends.
 //===----------------------------------------------------------------------===//
 
-namespace {
+#define BUGTYPE_PROVIDER(NAME, DEF)                                            \
+  struct NAME : virtual public CheckerFrontend {                               \
+    BugType NAME##Bug{this, DEF, categories::MemoryError};                     \
+  };
+
+BUGTYPE_PROVIDER(DoubleFree, "Double free")
+// TODO: Remove DoubleDelete as a separate bug type and when it would be
+// emitted, emit DoubleFree reports instead. (Note that DoubleFree is already
+// used for all allocation families, not just malloc/free.)
+BUGTYPE_PROVIDER(DoubleDelete, "Double delete")
+
+struct Leak : virtual public CheckerFrontend {
+  // Leaks should not be reported if they are post-dominated by a sink:
+  // (1) Sinks are higher importance bugs.
+  // (2) NoReturnFunctionChecker uses sink nodes to represent paths ending
+  //     with __noreturn functions such as assert() or exit(). We choose not
+  //     to report leaks on such paths.
+  BugType LeakBug{this, "Memory leak", categories::MemoryError,
+                  /*SuppressOnSink=*/true};
+};
+
+BUGTYPE_PROVIDER(UseFree, "Use-after-free")
+BUGTYPE_PROVIDER(BadFree, "Bad free")
+BUGTYPE_PROVIDER(FreeAlloca, "Free 'alloca()'")
+BUGTYPE_PROVIDER(MismatchedDealloc, "Bad deallocator")
+BUGTYPE_PROVIDER(OffsetFree, "Offset free")
+BUGTYPE_PROVIDER(UseZeroAllocated, "Use of zero allocated")
+
+#undef BUGTYPE_PROVIDER
+
+template <typename... BT_PROVIDERS>
+struct DynMemFrontend : virtual public CheckerFrontend, public BT_PROVIDERS... {
+  template <typename T> const T *getAs() const {
+    if constexpr (std::is_same_v<T, CheckerFrontend> ||
+                  (std::is_same_v<T, BT_PROVIDERS> || ...))
+      return static_cast<const T *>(this);
+    return nullptr;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Definition of the MallocChecker class.
+//===----------------------------------------------------------------------===//
 
 class MallocChecker
-    : public Checker<check::DeadSymbols, check::PointerEscape,
-                     check::ConstPointerEscape, check::PreStmt<ReturnStmt>,
-                     check::EndFunction, check::PreCall, check::PostCall,
-                     eval::Call, check::NewAllocator,
-                     check::PostStmt<BlockExpr>, check::PostObjCMessage,
-                     check::Location, eval::Assume> {
+    : public CheckerFamily<
+          check::DeadSymbols, check::PointerEscape, check::ConstPointerEscape,
+          check::PreStmt<ReturnStmt>, check::EndFunction, check::PreCall,
+          check::PostCall, eval::Call, check::NewAllocator,
+          check::PostStmt<BlockExpr>, check::PostObjCMessage, check::Location,
+          eval::Assume> {
 public:
   /// In pessimistic mode, the checker assumes that it does not know which
   /// functions might free the memory.
@@ -355,26 +400,29 @@ class MallocChecker
 
   bool ShouldRegisterNoOwnershipChangeVisitor = false;
 
-  /// Many checkers are essentially built into this one, so enabling them will
-  /// make MallocChecker perform additional modeling and reporting.
-  enum CheckKind {
-    /// When a subchecker is enabled but MallocChecker isn't, model memory
-    /// management but do not emit warnings emitted with MallocChecker only
-    /// enabled.
-    CK_MallocChecker,
-    CK_NewDeleteChecker,
-    CK_NewDeleteLeaksChecker,
-    CK_MismatchedDeallocatorChecker,
-    CK_InnerPointerChecker,
-    CK_TaintedAllocChecker,
-    CK_NumCheckKinds
-  };
+  // This checker family implements many bug types and frontends, and several
+  // bug types are shared between multiple frontends, so most of the frontends
+  // are declared with the helper class DynMemFrontend.
+  // FIXME: There is no clear reason for separating NewDelete vs NewDeleteLeaks
+  // while e.g. MallocChecker covers both non-leak and leak bugs together. It
+  // would be nice to redraw the boundaries between the frontends in a more
+  // logical way.
+  DynMemFrontend<DoubleFree, Leak, UseFree, BadFree, FreeAlloca, OffsetFree,
+                 UseZeroAllocated>
+      MallocChecker;
+  DynMemFrontend<DoubleFree, DoubleDelete, UseFree, BadFree, OffsetFree,
+                 UseZeroAllocated>
+      NewDeleteChecker;
+  DynMemFrontend<Leak> NewDeleteLeaksChecker;
+  DynMemFrontend<FreeAlloca, MismatchedDealloc> MismatchedDeallocatorChecker;
+  DynMemFrontend<UseFree> InnerPointerChecker;
+  // This last frontend is associated with a single bug type which is not used
+  // elsewhere and has a different bug category, so it's declared separately.
+  CheckerFrontendWithBugType TaintedAllocChecker{"Tainted Memory Allocation",
+                                                 categories::TaintedData};
 
   using LeakInfo = std::pair<const ExplodedNode *, const MemRegion *>;
 
-  bool ChecksEnabled[CK_NumCheckKinds] = {false};
-  CheckerNameRef CheckNames[CK_NumCheckKinds];
-
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
@@ -401,18 +449,9 @@ class MallocChecker
   void printState(raw_ostream &Out, ProgramStateRef State,
                   const char *NL, const char *Sep) const override;
 
-private:
-  mutable std::unique_ptr<BugType> BT_DoubleFree[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_DoubleDelete;
-  mutable std::unique_ptr<BugType> BT_Leak[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_UseFree[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_BadFree[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_FreeAlloca[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_MismatchedDealloc;
-  mutable std::unique_ptr<BugType> BT_OffsetFree[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_UseZerroAllocated[CK_NumCheckKinds];
-  mutable std::unique_ptr<BugType> BT_TaintedAlloc;
+  StringRef getDebugTag() const override { return "MallocChecker"; }
 
+private:
 #define CHECK_FN(NAME)                                                         \
   void NAME(ProgramStateRef State, const CallEvent &Call, CheckerContext &C)   \
       const;
@@ -439,7 +478,7 @@ class MallocChecker
                     CheckerContext &C, bool ShouldFreeOnFail) const;
 
   using CheckFn =
-      std::function<void(const MallocChecker *, ProgramStateRef State,
+      std::function<void(const class MallocChecker *, ProgramStateRef State,
                          const CallEvent &Call, CheckerContext &C)>;
 
   const CallDescriptionMap<CheckFn> PreFnMap{
@@ -773,14 +812,16 @@ class MallocChecker
   void checkEscapeOnReturn(const ReturnStmt *S, CheckerContext &C) const;
 
   ///@{
-  /// Tells if a given family/call/symbol is tracked by the current checker.
-  /// Sets CheckKind to the kind of the checker responsible for this
-  /// family/call/symbol.
-  std::optional<CheckKind> getCheckIfTracked(AllocationFamily Family,
-                                             bool IsALeakCheck = false) const;
-
-  std::optional<CheckKind> getCheckIfTracked(CheckerContext &C, SymbolRef Sym,
-                                             bool IsALeakCheck = false) const;
+  /// Returns a pointer to the checker frontend corresponding to the given
+  /// family or symbol. The template argument T may be either CheckerFamily or
+  /// a BUGTYPE_PROVIDER class; in the latter case the query is restricted to
+  /// frontends that descend from that PROVIDER class (i.e. can emit that bug
+  /// type). Note that this may return a frontend which is disabled.
+  template <class T>
+  const T *getRelevantFrontendAs(AllocationFamily Family) const;
+
+  template <class T>
+  const T *getRelevantFrontendAs(CheckerContext &C, SymbolRef Sym) const;
   ///@}
   static bool SummarizeValue(raw_ostream &os, SVal V);
   static bool SummarizeRegion(ProgramStateRef State, raw_ostream &os,
@@ -1558,7 +1599,7 @@ void MallocChecker::checkOwnershipAttr(ProgramStateRef State,
   if (!FD)
     return;
   if (ShouldIncludeOwnershipAnnotatedFunctions ||
-      ChecksEnabled[CK_MismatchedDeallocatorChecker]) {
+      MismatchedDeallocatorChecker.isEnabled()) {
     // Check all the attributes, if there are any.
     // There can be multiple of these attributes.
     if (FD->hasAttrs())
@@ -1883,11 +1924,8 @@ void MallocChecker::reportTaintBug(StringRef Msg, ProgramStateRef State,
                                    llvm::ArrayRef<SymbolRef> TaintedSyms,
                                    AllocationFamily Family) const {
   if (ExplodedNode *N = C.generateNonFatalErrorNode(State, this)) {
-    if (!BT_TaintedAlloc)
-      BT_TaintedAlloc.reset(new BugType(CheckNames[CK_TaintedAllocChecker],
-                                        "Tainted Memory Allocation",
-                                        categories::TaintedData));
-    auto R = std::make_unique<PathSensitiveBugReport>(*BT_TaintedAlloc, Msg, N);
+    auto R =
+        std::make_unique<PathSensitiveBugReport>(TaintedAllocChecker, Msg, N);
     for (const auto *TaintedSym : TaintedSyms) {
       R->markInteresting(TaintedSym);
     }
@@ -1898,7 +1936,7 @@ void MallocChecker::reportTaintBug(StringRef Msg, ProgramStateRef State,
 void MallocChecker::checkTaintedness(CheckerContext &C, const CallEvent &Call,
                                      const SVal SizeSVal, ProgramStateRef State,
                                      AllocationFamily Family) const {
-  if (!ChecksEnabled[CK_TaintedAllocChecker])
+  if (!TaintedAllocChecker.isEnabled())
     return;
   std::vector<SymbolRef> TaintedSyms =
       taint::getTaintedSymbols(State, SizeSVal);
@@ -2348,53 +2386,47 @@ MallocChecker::FreeMemAux(CheckerContext &C, const Expr *ArgExpr,
                                  RefState::getReleased(Family, ParentExpr));
 }
 
-std::optional<MallocChecker::CheckKind>
-MallocChecker::getCheckIfTracked(AllocationFamily Family,
-                                 bool IsALeakCheck) const {
+template <class T>
+const T *MallocChecker::getRelevantFrontendAs(AllocationFamily Family) const {
   switch (Family.Kind) {
   case AF_Malloc:
   case AF_Alloca:
   case AF_Custom:
-  case AF_IfNameIndex: {
-    if (ChecksEnabled[CK_MallocChecker])
-      return CK_MallocChecker;
-    return std::nullopt;
-  }
+  case AF_IfNameIndex:
+    return MallocChecker.getAs<T>();
   case AF_CXXNew:
   case AF_CXXNewArray: {
-    if (IsALeakCheck) {
-      if (ChecksEnabled[CK_NewDeleteLeaksChecker])
-        return CK_NewDeleteLeaksChecker;
-    }
-    else {
-      if (ChecksEnabled[CK_NewDeleteChecker])
-        return CK_NewDeleteChecker;
+    const T *ND = NewDeleteChecker.getAs<T>();
+    const T *NDL = NewDeleteLeaksChecker.getAs<T>();
+    // Bugs corresponding to C++ new/delete allocations are split between these
+    // two frontends.
+    if constexpr (std::is_same_v<T, CheckerFrontend>) {
+      assert(ND && NDL && "Casting to CheckerFrontend always succeeds");
+      // Prefer NewDelete unless it's disabled and NewDeleteLeaks is enabled.
+      return (!ND->isEnabled() && NDL->isEnabled()) ? NDL : ND;
     }
-    return std::nullopt;
-  }
-  case AF_InnerBuffer: {
-    if (ChecksEnabled[CK_InnerPointerChecker])
-      return CK_InnerPointerChecker;
-    return std::nullopt;
+    assert(!(ND && NDL) &&
+           "NewDelete and NewDeleteLeaks must not share a bug type");
+    return ND ? ND : NDL;
   }
-  case AF_None: {
+  case AF_InnerBuffer:
+    return InnerPointerChecker.getAs<T>();
+  case AF_None:
     assert(false && "no family");
-    return std::nullopt;
-  }
+    return nullptr;
   }
   assert(false && "unhandled family");
-  return std::nullopt;
+  return nullptr;
 }
-
-std::optional<MallocChecker::CheckKind>
-MallocChecker::getCheckIfTracked(CheckerContext &C, SymbolRef Sym,
-                                 bool IsALeakCheck) const {
+template <class T>
+const T *MallocChecker::getRelevantFrontendAs(CheckerContext &C,
+                                              SymbolRef Sym) const {
   if (C.getState()->contains<ReallocSizeZeroSymbols>(Sym))
-    return CK_MallocChecker;
+    return MallocChecker.getAs<T>();
 
   const RefState *RS = C.getState()->get<RegionState>(Sym);
   assert(RS);
-  return getCheckIfTracked(RS->getAllocationFamily(), IsALeakCheck);
+  return getRelevantFrontendAs<T>(RS->getAllocationFamily());
 }
 
 bool MallocChecker::SummarizeValue(raw_ostream &os, SVal V) {
@@ -2490,21 +2522,15 @@ void MallocChecker::HandleNonHeapDealloc(CheckerContext &C, SVal ArgVal,
                                          SourceRange Range,
                                          const Expr *DeallocExpr,
                                          AllocationFamily Family) const {
-
-  if (!ChecksEnabled[CK_MallocChecker] && !ChecksEnabled[CK_NewDeleteChecker]) {
+  const BadFree *Frontend = getRelevantFrontendAs<BadFree>(Family);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(Family);
-  if (!CheckKind)
-    return;
-
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_BadFree[*CheckKind])
-      BT_BadFree[*CheckKind].reset(new BugType(
-          CheckNames[*CheckKind], "Bad free", categories::MemoryError));
-
     SmallString<100> buf;
     llvm::raw_svector_ostream os(buf);
 
@@ -2526,7 +2552,7 @@ void MallocChecker::HandleNonHeapDealloc(CheckerContext &C, SVal ArgVal,
 
     printExpectedAllocName(os, Family);
 
-    auto R = std::make_unique<PathSensitiveBugReport>(*BT_BadFree[*CheckKind],
+    auto R = std::make_unique<PathSensitiveBugReport>(Frontend->BadFreeBug,
                                                       os.str(), N);
     R->markInteresting(MR);
     R->addRange(Range);
@@ -2536,25 +2562,20 @@ void MallocChecker::HandleNonHeapDealloc(CheckerContext &C, SVal ArgVal,
 
 void MallocChecker::HandleFreeAlloca(CheckerContext &C, SVal ArgVal,
                                      SourceRange Range) const {
+  const FreeAlloca *Frontend;
 
-  std::optional<MallocChecker::CheckKind> CheckKind;
-
-  if (ChecksEnabled[CK_MallocChecker])
-    CheckKind = CK_MallocChecker;
-  else if (ChecksEnabled[CK_MismatchedDeallocatorChecker])
-    CheckKind = CK_MismatchedDeallocatorChecker;
+  if (MallocChecker.isEnabled())
+    Frontend = &MallocChecker;
+  else if (MismatchedDeallocatorChecker.isEnabled())
+    Frontend = &MismatchedDeallocatorChecker;
   else {
     C.addSink();
     return;
   }
 
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_FreeAlloca[*CheckKind])
-      BT_FreeAlloca[*CheckKind].reset(new BugType(
-          CheckNames[*CheckKind], "Free 'alloca()'", categories::MemoryError));
-
     auto R = std::make_unique<PathSensitiveBugReport>(
-        *BT_FreeAlloca[*CheckKind],
+        Frontend->FreeAllocaBug,
         "Memory allocated by 'alloca()' should not be deallocated", N);
     R->markInteresting(ArgVal.getAsRegion());
     R->addRange(Range);
@@ -2567,18 +2588,12 @@ void MallocChecker::HandleMismatchedDealloc(CheckerContext &C,
                                             const Expr *DeallocExpr,
                                             const RefState *RS, SymbolRef Sym,
                                             bool OwnershipTransferred) const {
-
-  if (!ChecksEnabled[CK_MismatchedDeallocatorChecker]) {
+  if (!MismatchedDeallocatorChecker.isEnabled()) {
     C.addSink();
     return;
   }
 
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_MismatchedDealloc)
-      BT_MismatchedDealloc.reset(
-          new BugType(CheckNames[CK_MismatchedDeallocatorChecker],
-                      "Bad deallocator", categories::MemoryError));
-
     SmallString<100> buf;
     llvm::raw_svector_ostream os(buf);
 
@@ -2612,8 +2627,8 @@ void MallocChecker::HandleMismatchedDealloc(CheckerContext &C,
         printOwnershipTakesList(os, C, DeallocExpr);
     }
 
-    auto R = std::make_unique<PathSensitiveBugReport>(*BT_MismatchedDealloc,
-                                                      os.str(), N);
+    auto R = std::make_unique<PathSensitiveBugReport>(
+        MismatchedDeallocatorChecker.MismatchedDeallocBug, os.str(), N);
     R->markInteresting(Sym);
     R->addRange(Range);
     R->addVisitor<MallocBugVisitor>(Sym);
@@ -2625,24 +2640,18 @@ void MallocChecker::HandleOffsetFree(CheckerContext &C, SVal ArgVal,
                                      SourceRange Range, const Expr *DeallocExpr,
                                      AllocationFamily Family,
                                      const Expr *AllocExpr) const {
-
-  if (!ChecksEnabled[CK_MallocChecker] && !ChecksEnabled[CK_NewDeleteChecker]) {
+  const OffsetFree *Frontend = getRelevantFrontendAs<OffsetFree>(Family);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(Family);
-  if (!CheckKind)
-    return;
-
   ExplodedNode *N = C.generateErrorNode();
   if (!N)
     return;
 
-  if (!BT_OffsetFree[*CheckKind])
-    BT_OffsetFree[*CheckKind].reset(new BugType(
-        CheckNames[*CheckKind], "Offset free", categories::MemoryError));
-
   SmallString<100> buf;
   llvm::raw_svector_ostream os(buf);
   SmallString<20> AllocNameBuf;
@@ -2672,7 +2681,7 @@ void MallocChecker::HandleOffsetFree(CheckerContext &C, SVal ArgVal,
   else
     os << "allocated memory";
 
-  auto R = std::make_unique<PathSensitiveBugReport>(*BT_OffsetFree[*CheckKind],
+  auto R = std::make_unique<PathSensitiveBugReport>(Frontend->OffsetFreeBug,
                                                     os.str(), N);
   R->markInteresting(MR->getBaseRegion());
   R->addRange(Range);
@@ -2681,27 +2690,20 @@ void MallocChecker::HandleOffsetFree(CheckerContext &C, SVal ArgVal,
 
 void MallocChecker::HandleUseAfterFree(CheckerContext &C, SourceRange Range,
                                        SymbolRef Sym) const {
-
-  if (!ChecksEnabled[CK_MallocChecker] && !ChecksEnabled[CK_NewDeleteChecker] &&
-      !ChecksEnabled[CK_InnerPointerChecker]) {
+  const UseFree *Frontend = getRelevantFrontendAs<UseFree>(C, Sym);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(C, Sym);
-  if (!CheckKind)
-    return;
-
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_UseFree[*CheckKind])
-      BT_UseFree[*CheckKind].reset(new BugType(
-          CheckNames[*CheckKind], "Use-after-free", categories::MemoryError));
-
     AllocationFamily AF =
         C.getState()->get<RegionState>(Sym)->getAllocationFamily();
 
     auto R = std::make_unique<PathSensitiveBugReport>(
-        *BT_UseFree[*CheckKind],
+        Frontend->UseFreeBug,
         AF.Kind == AF_InnerBuffer
             ? "Inner pointer of container used after re/deallocation"
             : "Use of memory after it is freed",
@@ -2721,23 +2723,17 @@ void MallocChecker::HandleUseAfterFree(CheckerContext &C, SourceRange Range,
 void MallocChecker::HandleDoubleFree(CheckerContext &C, SourceRange Range,
                                      bool Released, SymbolRef Sym,
                                      SymbolRef PrevSym) const {
-
-  if (!ChecksEnabled[CK_MallocChecker] && !ChecksEnabled[CK_NewDeleteChecker]) {
+  const DoubleFree *Frontend = getRelevantFrontendAs<DoubleFree>(C, Sym);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(C, Sym);
-  if (!CheckKind)
-    return;
-
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_DoubleFree[*CheckKind])
-      BT_DoubleFree[*CheckKind].reset(new BugType(
-          CheckNames[*CheckKind], "Double free", categories::MemoryError));
-
     auto R = std::make_unique<PathSensitiveBugReport>(
-        *BT_DoubleFree[*CheckKind],
+        Frontend->DoubleFreeBug,
         (Released ? "Attempt to free released memory"
                   : "Attempt to free non-owned memory"),
         N);
@@ -2751,24 +2747,18 @@ void MallocChecker::HandleDoubleFree(CheckerContext &C, SourceRange Range,
 }
 
 void MallocChecker::HandleDoubleDelete(CheckerContext &C, SymbolRef Sym) const {
-
-  if (!ChecksEnabled[CK_NewDeleteChecker]) {
+  const DoubleDelete *Frontend = getRelevantFrontendAs<DoubleDelete>(C, Sym);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(C, Sym);
-  if (!CheckKind)
-    return;
-
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_DoubleDelete)
-      BT_DoubleDelete.reset(new BugType(CheckNames[CK_NewDeleteChecker],
-                                        "Double delete",
-                                        categories::MemoryError));
 
     auto R = std::make_unique<PathSensitiveBugReport>(
-        *BT_DoubleDelete, "Attempt to delete released memory", N);
+        Frontend->DoubleDeleteBug, "Attempt to delete released memory", N);
 
     R->markInteresting(Sym);
     R->addVisitor<MallocBugVisitor>(Sym);
@@ -2778,26 +2768,19 @@ void MallocChecker::HandleDoubleDelete(CheckerContext &C, SymbolRef Sym) const {
 
 void MallocChecker::HandleUseZeroAlloc(CheckerContext &C, SourceRange Range,
                                        SymbolRef Sym) const {
-
-  if (!ChecksEnabled[CK_MallocChecker] && !ChecksEnabled[CK_NewDeleteChecker]) {
+  const UseZeroAllocated *Frontend =
+      getRelevantFrontendAs<UseZeroAllocated>(C, Sym);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(C, Sym);
-
-  if (!CheckKind)
-    return;
-
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_UseZerroAllocated[*CheckKind])
-      BT_UseZerroAllocated[*CheckKind].reset(
-          new BugType(CheckNames[*CheckKind], "Use of zero allocated",
-                      categories::MemoryError));
-
     auto R = std::make_unique<PathSensitiveBugReport>(
-        *BT_UseZerroAllocated[*CheckKind],
-        "Use of memory allocated with size zero", N);
+        Frontend->UseZeroAllocatedBug, "Use of memory allocated with size zero",
+        N);
 
     R->addRange(Range);
     if (Sym) {
@@ -2812,20 +2795,15 @@ void MallocChecker::HandleFunctionPtrFree(CheckerContext &C, SVal ArgVal,
                                           SourceRange Range,
                                           const Expr *FreeExpr,
                                           AllocationFamily Family) const {
-  if (!ChecksEnabled[CK_MallocChecker]) {
+  const BadFree *Frontend = getRelevantFrontendAs<BadFree>(Family);
+  if (!Frontend)
+    return;
+  if (!Frontend->isEnabled()) {
     C.addSink();
     return;
   }
 
-  std::optional<MallocChecker::CheckKind> CheckKind = getCheckIfTracked(Family);
-  if (!CheckKind)
-    return;
-
   if (ExplodedNode *N = C.generateErrorNode()) {
-    if (!BT_BadFree[*CheckKind])
-      BT_BadFree[*CheckKind].reset(new BugType(
-          CheckNames[*CheckKind], "Bad free", categories::MemoryError));
-
     SmallString<100> Buf;
     llvm::raw_svector_ostream Os(Buf);
 
@@ -2839,7 +2817,7 @@ void MallocChecker::HandleFunctionPtrFree(CheckerContext &C, SVal ArgVal,
 
     Os << " is a function pointer";
 
-    auto R = std::make_unique<PathSensitiveBugReport>(*BT_BadFree[*CheckKind],
+    auto R = std::make_unique<PathSensitiveBugReport>(Frontend->BadFreeBug,
                                                       Os.str(), N);
     R->markInteresting(MR);
     R->addRange(Range);
@@ -3016,10 +2994,7 @@ MallocChecker::LeakInfo MallocChecker::getAllocationSite(const ExplodedNode *N,
 
 void MallocChecker::HandleLeak(SymbolRef Sym, ExplodedNode *N,
                                CheckerContext &C) const {
-
-  if (!ChecksEnabled[CK_MallocChecker] &&
-      !ChecksEnabled[CK_NewDeleteLeaksChecker])
-    return;
+  assert(N && "HandleLeak is only called with a non-null node");
 
   const RefState *RS = C.getState()->get<RegionState>(Sym);
   assert(RS && "cannot leak an untracked symbol");
@@ -3028,24 +3003,13 @@ void MallocChecker::HandleLeak(SymbolRef Sym, ExplodedNode *N,
   if (Family.Kind == AF_Alloca)
     return;
 
-  std::optional<MallocChecker::CheckKind> CheckKind =
-      getCheckIfTracked(Family, true);
-
-  if (!CheckKind)
+  const Leak *Frontend = getRelevantFrontendAs<Leak>(Family);
+  // Note that for leaks we don't add a sink when the relevant frontend is
+  // disabled because the leak is reported with a non-fatal error node, while
+  // the sink would be the "silent" alternative of a (fatal) error node.
+  if (!Frontend || !Frontend->isEnabled())
     return;
 
-  assert(N);
-  if (!BT_Leak[*CheckKind]) {
-    // Leaks should not be reported if they are post-dominated by a sink:
-    // (1) Sinks are higher importance bugs.
-    // (2) NoReturnFunctionChecker uses sink nodes to represent paths ending
-    //     with __noreturn functions such as assert() or exit(). We choose not
-    //     to report leaks on such paths.
-    BT_Leak[*CheckKind].reset(new BugType(CheckNames[*CheckKind], "Memory leak",
-                                          categories::MemoryError,
-                                          /*SuppressOnSink=*/true));
-  }
-
   // Most bug reports are cached at the location where they occurred.
   // With leaks, we want to unique them by the location where they were
   // allocated, and only report a single path.
@@ -3070,7 +3034,7 @@ void MallocChecker::HandleLeak(SymbolRef Sym, ExplodedNode *N,
   }
 
   auto R = std::make_unique<PathSensitiveBugReport>(
-      *BT_Leak[*CheckKind], os.str(), N, LocUsedForUniqueing,
+      Frontend->LeakBug, os.str(), N, LocUsedForUniqueing,
       AllocNode->getLocationContext()->getDecl());
   R->markInteresting(Sym);
   R->addVisitor<MallocBugVisitor>(Sym, true);
@@ -3150,7 +3114,11 @@ void MallocChecker::checkPreCall(const CallEvent &Call,
   if (const auto *DC = dyn_cast<CXXDeallocatorCall>(&Call)) {
     const CXXDeleteExpr *DE = DC->getOriginExpr();
 
-    if (!ChecksEnabled[CK_NewDeleteChecker])
+    // FIXME: I don't see a good reason for restricting the check against
+    // use-after-free violations to the case when NewDeleteChecker is disabled.
+    // (However, if NewDeleteChecker is enabled, perhaps it would be better to
+    // do this check a bit later?)
+    if (!NewDeleteChecker.isEnabled())
       if (SymbolRef Sym = C.getSVal(DE->getArgument()).getAsSymbol())
         checkUseAfterFree(Sym, C, DE->getArgument());
 
@@ -3181,13 +3149,19 @@ void MallocChecker::checkPreCall(const CallEvent &Call,
     return;
   }
 
-  // We will check for double free in the post visit.
+  // We will check for double free in the `evalCall` callback.
+  // FIXME: It would be more logical to emit double free and use-after-free
+  // reports via the same pathway (because double free is essentially a specia
+  // case of use-after-free).
   if (const AnyFunctionCall *FC = dyn_cast<AnyFunctionCall>(&Call)) {
     const FunctionDecl *FD = FC->getDecl();
     if (!FD)
       return;
 
-    if (ChecksEnabled[CK_MallocChecker] && isFreeingCall(Call))
+    // FIXME: I suspect we should remove `MallocChecker.isEnabled() &&` because
+    // it's fishy that the enabled/disabled state of one frontend may influence
+    // reports produced by other frontends.
+    if (MallocChecker.isEnabled() && isFreeingCall(Call))
       return;
   }
 
@@ -3902,16 +3876,15 @@ void MallocChecker::printState(raw_ostream &Out, ProgramStateRef State,
     for (auto [Sym, Data] : RS) {
       const RefState *RefS = State->get<RegionState>(Sym);
       AllocationFamily Family = RefS->getAllocationFamily();
-      std::optional<MallocChecker::CheckKind> CheckKind =
-          getCheckIfTracked(Family);
-      if (!CheckKind)
-        CheckKind = getCheckIfTracked(Family, true);
+
+      const CheckerFrontend *Frontend =
+          getRelevantFrontendAs<CheckerFrontend>(Family);
 
       Sym->dumpToStream(Out);
       Out << " : ";
       Data.dump(Out);
-      if (CheckKind)
-        Out << " (" << CheckNames[*CheckKind] << ")";
+      if (Frontend && Frontend->isEnabled())
+        Out << " (" << Frontend->getName() << ")";
       Out << NL;
     }
   }
@@ -3933,36 +3906,34 @@ markReleased(ProgramStateRef State, SymbolRef Sym, const Expr *Origin) {
 
 // Intended to be used in InnerPointerChecker to register the part of
 // MallocChecker connected to it.
-void ento::registerInnerPointerCheckerAux(CheckerManager &mgr) {
-  MallocChecker *checker = mgr.getChecker<MallocChecker>();
-  checker->ChecksEnabled[MallocChecker::CK_InnerPointerChecker] = true;
-  checker->CheckNames[MallocChecker::CK_InnerPointerChecker] =
-      mgr.getCurrentCheckerName();
+void ento::registerInnerPointerCheckerAux(CheckerManager &Mgr) {
+  Mgr.getChecker<MallocChecker>()->InnerPointerChecker.enable(Mgr);
 }
 
-void ento::registerDynamicMemoryModeling(CheckerManager &mgr) {
-  auto *checker = mgr.registerChecker<MallocChecker>();
-  checker->ShouldIncludeOwnershipAnnotatedFunctions =
-      mgr.getAnalyzerOptions().getCheckerBooleanOption(checker, "Optimistic");
-  checker->ShouldRegisterNoOwnershipChangeVisitor =
-      mgr.getAnalyzerOptions().getCheckerBooleanOption(
-          checker, "AddNoOwnershipChangeNotes");
+void ento::registerDynamicMemoryModeling(CheckerManager &Mgr) {
+  auto *Chk = Mgr.getChecker<MallocChecker>();
+  // FIXME: This is a "hidden" undocumented frontend but there are public
+  // checker options which are attached to it.
+  CheckerNameRef DMMName = Mgr.getCurrentCheckerName();
+  Chk->ShouldIncludeOwnershipAnnotatedFunctions =
+      Mgr.getAnalyzerOptions().getCheckerBooleanOption(DMMName, "Optimistic");
+  Chk->ShouldRegisterNoOwnershipChangeVisitor =
+      Mgr.getAnalyzerOptions().getCheckerBooleanOption(
+          DMMName, "AddNoOwnershipChangeNotes");
 }
 
 bool ento::shouldRegisterDynamicMemoryModeling(const CheckerManager &mgr) {
   return true;
 }
 
-#define REGISTER_CHECKER(name)                                                 \
-  void ento::register##name(CheckerManager &mgr) {                             \
-    MallocChecker *checker = mgr.getChecker<MallocChecker>();                  \
-    checker->ChecksEnabled[MallocChecker::CK_##name] = true;                   \
-    checker->CheckNames[MallocChecker::CK_##name] =                            \
-        mgr.getCurrentCheckerName();                                           \
+#define REGISTER_CHECKER(NAME)                                                 \
+  void ento::register##NAME(CheckerManager &Mgr) {                             \
+    Mgr.getChecker<MallocChecker>()->NAME.enable(Mgr);                         \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
+  bool ento::shouldRegister##NAME(const CheckerManager &) { return true; }
 
+// TODO: NewDelete and NewDeleteLeaks shouldn't be registered when not in C++.
 REGISTER_CHECKER(MallocChecker)
 REGISTER_CHECKER(NewDeleteChecker)
 REGISTER_CHECKER(NewDeleteLeaksChecker)
diff --git a/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h
index 027f1a156a7c0..44fed68b97acb 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h
+++ b/clang/lib/StaticAnalyzer/Checkers/NoOwnershipChangeVisitor.h
@@ -17,7 +17,7 @@ class NoOwnershipChangeVisitor : public NoStateChangeFuncVisitor {
 protected:
   // The symbol whose (lack of) ownership change we are interested in.
   SymbolRef Sym;
-  const CheckerBase &Checker;
+  const CheckerBackend &Checker;
 
   LLVM_DUMP_METHOD static std::string
   getFunctionName(const ExplodedNode *CallEnterN);
@@ -63,7 +63,7 @@ class NoOwnershipChangeVisitor : public NoStateChangeFuncVisitor {
   OwnerSet getOwnersAtNode(const ExplodedNode *N);
 
 public:
-  NoOwnershipChangeVisitor(SymbolRef Sym, const CheckerBase *Checker)
+  NoOwnershipChangeVisitor(SymbolRef Sym, const CheckerBackend *Checker)
       : NoStateChangeFuncVisitor(bugreporter::TrackingKind::Thorough), Sym(Sym),
         Checker(*Checker) {}
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
index 6bc39ab565041..764e2c640feb8 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
@@ -294,7 +294,7 @@ class RawPtrRefCallArgsChecker
     if (name == "adoptRef" || name == "getPtr" || name == "WeakPtr" ||
         name == "is" || name == "equal" || name == "hash" || name == "isType" ||
         // FIXME: Most/all of these should be implemented via attributes.
-        name == "equalIgnoringASCIICase" ||
+        name == "CFEqual" || name == "equalIgnoringASCIICase" ||
         name == "equalIgnoringASCIICaseCommon" ||
         name == "equalIgnoringNullity" || name == "toString")
       return true;
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 3686bd4488877..4631e0cad5546 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -2793,9 +2793,14 @@ PathDiagnosticPieceRef ConditionBRVisitor::VisitTerminator(
   // more tricky because there are more than two branches to account for.
   default:
     return nullptr;
-  case Stmt::IfStmtClass:
-    Cond = cast<IfStmt>(Term)->getCond();
+  case Stmt::IfStmtClass: {
+    const auto *IfStatement = cast<IfStmt>(Term);
+    // Handle if consteval which doesn't have a traditional condition.
+    if (IfStatement->isConsteval())
+      return nullptr;
+    Cond = IfStatement->getCond();
     break;
+  }
   case Stmt::ConditionalOperatorClass:
     Cond = cast<ConditionalOperator>(Term)->getCond();
     break;
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 3b95a8ea48596..e554ef41a8d0f 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -1738,6 +1738,26 @@ namespace WithinLifetime {
                                            // both-note {{'__builtin_is_within_lifetime' cannot be called with a one-past-the-end pointer}} \
                                            // both-warning {{expression result unused}}
   }
+
+
+  constexpr bool self = __builtin_is_within_lifetime(&self); // both-error {{must be initialized by a constant expression}} \
+                                                             // both-note {{'__builtin_is_within_lifetime' cannot be called with a pointer to an object whose lifetime has not yet begun}} \
+                                                             // ref-error {{call to consteval function '__builtin_is_within_lifetime' is not a constant expression}} \
+                                                             // ref-note {{initializer of 'self' is not a constant expression}} \
+                                                             // ref-note {{declared here}}
+
+  int nontCE(int p) { // both-note {{declared here}}
+    return __builtin_is_within_lifetime(&p); // both-error {{call to consteval function}} \
+                                             // both-note {{function parameter 'p' with unknown value cannot be used in a constant expression}}
+  }
+
+
+  struct XStd {
+    consteval XStd() {
+      __builtin_is_within_lifetime(this); // both-note {{cannot be called with a pointer to an object whose lifetime has not yet begun}}
+    }
+  } xstd; // both-error {{is not a constant expression}} \
+          // both-note {{in call to}}
 }
 
 #ifdef __SIZEOF_INT128__
@@ -1753,4 +1773,12 @@ namespace I128Mul {
 }
 #endif
 
+namespace InitParam {
+  constexpr int foo(int a) {
+      __builtin_mul_overflow(20, 10, &a);
+      return a;
+  }
+  static_assert(foo(10) == 200);
+}
+
 #endif
diff --git a/clang/test/AST/ByteCode/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
index 417d35dbca946..2856b872d44ab 100644
--- a/clang/test/AST/ByteCode/cxx23.cpp
+++ b/clang/test/AST/ByteCode/cxx23.cpp
@@ -4,6 +4,16 @@
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20,all,all20 %s -fexperimental-new-constant-interpreter
 // RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=expected23,all,all23 %s -fexperimental-new-constant-interpreter
 
+
+#define assert_active(F)   if (!__builtin_is_within_lifetime(&F)) (1/0);
+#define assert_inactive(F) if ( __builtin_is_within_lifetime(&F)) (1/0);
+
+inline constexpr void* operator new(__SIZE_TYPE__, void* p) noexcept { return p; }
+namespace std {
+template<typename T, typename... Args>
+constexpr T* construct_at(T* p, Args&&... args) { return ::new((void*)p) T(static_cast<Args&&>(args)...); }
+}
+
 constexpr int f(int n) {  // all20-error {{constexpr function never produces a constant expression}}
   static const int m = n; // all-note {{control flows through the definition of a static variable}} \
                           // all20-note {{control flows through the definition of a static variable}} \
@@ -328,3 +338,38 @@ namespace VoidCast {
   constexpr int *f = (int*)(void*)b; // all-error {{must be initialized by a constant expression}} \
                                      // all-note {{cast from 'void *' is not allowed in a constant expression}}
 }
+
+#if __cplusplus >= 202302L
+namespace NestedUnions {
+  consteval bool test_nested() {
+    union {
+      union { int i; char c; } u;
+      long l;
+    };
+    std::construct_at(&l);
+    assert_active(l);
+    assert_inactive(u);
+
+    std::construct_at(&u);
+    assert_active(u);
+    assert_inactive(l);
+    assert_active(u.i);
+    assert_inactive(u.c);
+
+    std::construct_at(&u.i);
+    assert_active(u);
+    assert_inactive(u.c);
+
+
+    std::construct_at(&u.c);
+    assert_active(u);
+    assert_inactive(u.i);
+    assert_active(u.c);
+    assert_inactive(l);
+    return true;
+  }
+  static_assert(test_nested());
+
+
+}
+#endif
diff --git a/clang/test/AST/ByteCode/functions.cpp b/clang/test/AST/ByteCode/functions.cpp
index a767d104b3c8a..b5e6f5bd7ece2 100644
--- a/clang/test/AST/ByteCode/functions.cpp
+++ b/clang/test/AST/ByteCode/functions.cpp
@@ -637,7 +637,7 @@ namespace {
 
 namespace {
   /// The InitListExpr here is of void type.
-  void bir [[clang::annotate("B", {1, 2, 3, 4})]] (); // both-error {{'annotate' attribute requires parameter 1 to be a constant expression}} \
+  void bir [[clang::annotate("B", {1, 2, 3, 4})]] (); // both-error {{'clang::annotate' attribute requires parameter 1 to be a constant expression}} \
                                                       // both-note {{subexpression not valid in a constant expression}}
 }
 
diff --git a/clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp b/clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp
new file mode 100644
index 0000000000000..0e8127d27f888
--- /dev/null
+++ b/clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp
@@ -0,0 +1,122 @@
+// RUN: %clang_cc1 -std=c++2c -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++2c -verify=ref,both      %s
+
+template <int __v> struct integral_constant {
+  static const int value = __v;
+};
+template <bool _Val> using _BoolConstant = integral_constant<_Val>;
+template <class _From, class _To>
+constexpr bool is_convertible_v = __is_convertible(_From, _To);
+template <class _Tp> _Tp __declval(int);
+template <class _Tp> decltype(__declval<_Tp>(0)) declval();
+template <class _Tp, class _Up>
+using _IsSame = _BoolConstant<__is_same(_Tp, _Up)>;
+template <class _If> struct conditional {
+  using type = _If;
+};
+template <bool, class _IfRes, class>
+using conditional_t = conditional<_IfRes>::type;
+template <class _Tp, class>
+concept __weakly_equality_comparable_with = requires(_Tp __t) { __t; };
+template <bool, class _Tp = void> using __enable_if_t = _Tp;
+template <template <class> class, class>
+integral_constant<true> __sfinae_test_impl(int);
+template <template <class> class _Templ, class... _Args>
+using _IsValidExpansion = decltype(__sfinae_test_impl<_Templ, _Args...>(0));
+template <class _Tp>
+using __test_for_primary_template =
+    __enable_if_t<_IsSame<_Tp, typename _Tp::__primary_template>::value>;
+template <class _Tp>
+using __is_primary_template =
+    _IsValidExpansion<__test_for_primary_template, _Tp>;
+template <class _Ip>
+using iter_difference_t =
+    conditional_t<__is_primary_template<_Ip>::value, _Ip, _Ip>::difference_type;
+template <int> struct _OrImpl {
+  template <class, class _First>
+  using _Result = _OrImpl<!_First::value>::template _Result<_First>;
+};
+template <> struct _OrImpl<false> {
+  template <class _Res> using _Result = _Res;
+};
+template <class... _Args>
+using _Or = _OrImpl<sizeof...(_Args)>::template _Result<_Args...>;
+struct input_iterator_tag {};
+template <class _Dp, class _Bp>
+concept derived_from = is_convertible_v<_Dp, _Bp>;
+template <class _Ip>
+concept input_or_output_iterator = requires(_Ip __i) { __i; };
+template <class _Sp, class _Ip>
+concept sentinel_for = __weakly_equality_comparable_with<_Sp, _Ip>;
+struct __iter_concept_category_test {
+  template <class> using _Apply = input_iterator_tag;
+};
+struct __test_iter_concept
+    : _IsValidExpansion<__iter_concept_category_test::_Apply, int>,
+      __iter_concept_category_test {};
+struct __iter_concept_cache {
+  using type = _Or<int, __test_iter_concept>;
+};
+template <class _Iter>
+using _ITER_CONCEPT = __iter_concept_cache::type::_Apply<_Iter>;
+template <class _Ip>
+concept input_iterator = derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>;
+template <class _T1, class _T2> struct pair {
+  _T1 first;
+  _T2 second;
+};
+struct {
+  template <class _Tp> auto operator()(_Tp __t) { return __t.begin(); }
+} begin;
+template <class _Tp> using iterator_t = decltype(begin(declval<_Tp>()));
+template <class _Tp>
+concept __member_size = requires(_Tp __t) { __t; };
+struct {
+  template <__member_size _Tp> constexpr void operator()(_Tp &&__t) {
+    __t.size(); // both-note 2{{in instantiation}}
+  }
+} size;
+template <class _Tp>
+concept range = requires(_Tp __t) { __t; };
+template <class _Tp>
+concept input_range = input_iterator<_Tp>;
+template <range _Rp>
+using range_difference_t = iter_difference_t<iterator_t<_Rp>>;
+struct {
+  template <range _Rp> constexpr range_difference_t<_Rp> operator()(_Rp &&__r) {
+    size(__r); // both-note 2{{in instantiation}}
+  } // both-warning {{does not return a value}}
+} distance;
+template <input_or_output_iterator _Iter, sentinel_for<_Iter> _Sent>
+struct subrange {
+  _Iter __begin_;
+  _Sent __end_;
+  _Iter begin();
+  constexpr _Iter size() { __end_ - __begin_; } // both-warning {{does not return a value}} \
+                                                // both-note {{in instantiation}}
+};
+struct {
+  template <input_range _Range1, input_range _Range2>
+  void operator()(_Range1 &&__range1, _Range2) {
+    (void)(distance(__range1) != 0); // both-note 3{{in instantiation}}
+  }
+} equal;
+template <class _Owner> struct __key_value_iterator {
+  using difference_type = _Owner::difference_type;
+  constexpr friend difference_type operator-(__key_value_iterator,
+                                             __key_value_iterator &) {} // both-warning {{does not return a value}}
+};
+struct flat_multimap {
+  template <bool> using __iterator = __key_value_iterator<flat_multimap>;
+  using difference_type =
+      decltype(static_cast<int *>(nullptr) - static_cast<int *>(nullptr));
+  pair<__iterator<true>, __iterator<true>> equal_range(const char *);
+} test_expected_range;
+void test() {
+  flat_multimap m;
+  auto test_found = [](auto map, auto expected_key, int) {
+    auto [first, last] = map.equal_range(expected_key);
+    equal(subrange(first, last), test_expected_range); // both-note 3{{in instantiation}}
+  };
+  test_found(m, "", {});
+}
diff --git a/clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp b/clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp
new file mode 100644
index 0000000000000..df7ef3fe5ae96
--- /dev/null
+++ b/clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp
@@ -0,0 +1,264 @@
+// RUN: %clang_cc1 -std=c++2c -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++2c -verify=ref,both      %s
+
+// both-no-diagnostics
+
+namespace std {
+typedef __SIZE_TYPE__ size_t;
+}
+extern "C++" {
+namespace std {
+template <typename> struct iterator_traits;
+}
+}
+namespace std {
+template <typename _Tp, _Tp __v> struct integral_constant {
+  static constexpr _Tp value = __v;
+};
+template <bool __v> using __bool_constant = integral_constant<bool, __v>;
+using false_type = __bool_constant<false>;
+template <bool, typename _Tp = void> struct enable_if {
+  using type = _Tp;
+};
+template <bool> struct __conditional {
+  template <typename _Tp, typename> using type = _Tp;
+};
+template <bool _Cond, typename _If, typename _Else>
+using __conditional_t =
+    typename __conditional<_Cond>::template type<_If, _Else>;
+template <typename _Tp>
+struct is_empty : public __bool_constant<__is_empty(_Tp)> {};
+template <typename _Tp, typename _Up>
+struct is_same : public __bool_constant<__is_same(_Tp, _Up)> {};
+template <typename _Tp> struct remove_cv {
+  using type = __remove_cv(_Tp);
+};
+template <typename _Tp> struct tuple_size;
+template <typename _Tp, typename _Up = typename remove_cv<_Tp>::type,
+          typename = typename enable_if<is_same<_Tp, _Up>::value>::type,
+          size_t = tuple_size<_Tp>::value>
+using __enable_if_has_tuple_size = _Tp;
+template <typename _Tp>
+struct tuple_size<const __enable_if_has_tuple_size<_Tp>>
+    : public tuple_size<_Tp> {};
+template <size_t __i, typename _Tp> struct tuple_element;
+template <size_t __i, typename _Tp>
+using __tuple_element_t = typename tuple_element<__i, _Tp>::type;
+template <size_t __i, typename _Tp> struct tuple_element<__i, const _Tp> {
+  using type = const __tuple_element_t<__i, _Tp>;
+};
+template <size_t _Np, typename... _Types> struct _Nth_type {
+  using type = __type_pack_element<_Np, _Types...>;
+};
+template <typename _Tp> struct iterator_traits<_Tp *> {
+  using reference = _Tp &;
+};
+} // namespace std
+extern "C++" {
+void *operator new(std::size_t, void *__p);
+}
+namespace std {
+template <typename _Tp, typename... _Args>
+constexpr inline void _Construct(_Tp *__p, _Args &&...__args) {
+  ::new (__p) _Tp(__args...);
+};
+} // namespace std
+namespace __gnu_cxx {
+template <typename _Iterator, typename _Container> class __normal_iterator {
+protected:
+  _Iterator _M_current;
+  typedef std::iterator_traits<_Iterator> __traits_type;
+
+public:
+  typedef _Iterator iterator_type;
+  typedef typename __traits_type::reference reference;
+  explicit constexpr __normal_iterator(const _Iterator &__i)
+      : _M_current(__i) {};
+  constexpr reference operator*() const { return *_M_current; }
+  constexpr __normal_iterator &operator++() {
+    ++_M_current;
+    return *this;
+  }
+  constexpr const _Iterator &base() const { return _M_current; }
+};
+template <typename _Iterator, typename _Container>
+constexpr bool
+operator==(const __normal_iterator<_Iterator, _Container> &__lhs,
+           const __normal_iterator<_Iterator, _Container> &__rhs) {
+  return __lhs.base() == __rhs.base();
+}
+} // namespace __gnu_cxx
+namespace std {
+template <typename _Tp> class __new_allocator {};
+template <typename _Tp> using __allocator_base = __new_allocator<_Tp>;
+template <typename> struct allocator_traits;
+template <typename _Tp> class allocator : public __allocator_base<_Tp> {
+public:
+  typedef _Tp value_type;
+  constexpr _Tp *allocate(size_t __n) {
+    __n *= sizeof(_Tp);
+    return static_cast<_Tp *>(::operator new(__n));
+  }
+  constexpr void deallocate(_Tp *__p, size_t __n) { ::operator delete(__p); }
+};
+template <typename _Tp> struct allocator_traits<allocator<_Tp>> {
+  using allocator_type = allocator<_Tp>;
+  using pointer = _Tp *;
+  using size_type = std::size_t;
+  template <typename _Up> using rebind_alloc = allocator<_Up>;
+  static constexpr pointer allocate(allocator_type &__a, size_type __n) {
+    return __a.allocate(__n);
+  }
+  static constexpr void deallocate(allocator_type &__a, pointer __p,
+                                   size_type __n) {
+    __a.deallocate(__p, __n);
+  }
+};
+} // namespace std
+namespace __gnu_cxx {
+template <typename _Alloc, typename = typename _Alloc::value_type>
+struct __alloc_traits : std::allocator_traits<_Alloc> {
+  typedef std::allocator_traits<_Alloc> _Base_type;
+  template <typename _Tp> struct rebind {
+    typedef typename _Base_type::template rebind_alloc<_Tp> other;
+  };
+};
+} // namespace __gnu_cxx
+namespace std {
+template <typename _InputIterator, typename _ForwardIterator>
+constexpr _ForwardIterator __do_uninit_copy(_InputIterator __first,
+                                            _InputIterator __last,
+                                            _ForwardIterator __result) {
+  _ForwardIterator __cur = __result;
+  for (; __first != __last; ++__first, ++__cur)
+    std::_Construct(&*__cur, *__first);
+  return __cur;
+};
+template <typename _InputIterator, typename _ForwardIterator, typename _Tp>
+constexpr inline _ForwardIterator
+__uninitialized_copy_a(_InputIterator __first, _InputIterator __last,
+                       _ForwardIterator __result, allocator<_Tp> &) {
+  return std::__do_uninit_copy(__first, __last, __result);
+}
+template <typename _Tp, typename _Alloc> struct _Vector_base {
+  typedef
+      typename __gnu_cxx::__alloc_traits<_Alloc>::template rebind<_Tp>::other
+          _Tp_alloc_type;
+  typedef typename __gnu_cxx::__alloc_traits<_Tp_alloc_type>::pointer pointer;
+  struct _Vector_impl_data {
+    pointer _M_start;
+    pointer _M_finish;
+    pointer _M_end_of_storage;
+  };
+  struct _Vector_impl : public _Tp_alloc_type, public _Vector_impl_data {};
+
+public:
+  typedef _Alloc allocator_type;
+  constexpr _Tp_alloc_type &_M_get_Tp_allocator() { return this->_M_impl; }
+  constexpr _Vector_base(const allocator_type &__a) : _M_impl(__a) {}
+  constexpr ~_Vector_base() {
+    _M_deallocate(_M_impl._M_start,
+                  _M_impl._M_end_of_storage - _M_impl._M_start);
+  }
+
+public:
+  _Vector_impl _M_impl;
+  constexpr pointer _M_allocate(size_t __n) {
+    typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr;
+    return __n != 0 ? _Tr::allocate(_M_impl, __n) : pointer();
+  }
+  constexpr void _M_deallocate(pointer __p, size_t __n) {
+    typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr;
+    if (__p)
+      _Tr::deallocate(_M_impl, __p, __n);
+  }
+
+protected:
+};
+template <typename _Tp, typename _Alloc = std::allocator<_Tp>>
+class vector : protected _Vector_base<_Tp, _Alloc> {
+  typedef _Vector_base<_Tp, _Alloc> _Base;
+
+public:
+  typedef _Tp value_type;
+  typedef typename _Base::pointer pointer;
+  typedef __gnu_cxx::__normal_iterator<pointer, vector> iterator;
+  typedef size_t size_type;
+  typedef _Alloc allocator_type;
+  using _Base::_M_get_Tp_allocator;
+
+public:
+private:
+public:
+  constexpr vector(_Tp __l, const allocator_type &__a = allocator_type()) : _Base(__a) {
+
+  }
+  constexpr iterator begin() { return iterator(this->_M_impl._M_start); }
+  constexpr iterator end() { return iterator(this->_M_impl._M_finish); }
+
+protected:
+  template <typename _Iterator>
+  constexpr void _M_range_initialize_n(_Iterator __first, _Iterator __last,
+                                       size_type __n) {
+    pointer __start = this->_M_impl._M_start = this->_M_allocate(((__n)));
+    this->_M_impl._M_end_of_storage = __start + __n;
+    this->_M_impl._M_finish = std::__uninitialized_copy_a(
+        __first, __last, __start, _M_get_Tp_allocator());
+  }
+};
+template <typename _Tp> struct __is_empty_non_tuple : is_empty<_Tp> {};
+template <typename _Tp>
+using __empty_not_final =
+    __conditional_t<__is_final(_Tp), false_type, __is_empty_non_tuple<_Tp>>;
+template <size_t _Idx, typename _Head, bool = __empty_not_final<_Head>::value>
+struct _Head_base;
+template <size_t _Idx, typename _Head> struct _Head_base<_Idx, _Head, false> {
+  static constexpr const _Head &_M_head(const _Head_base &__b) {
+    return __b._M_head_impl;
+  }
+  _Head _M_head_impl;
+};
+template <size_t _Idx, typename... _Elements> struct _Tuple_impl;
+template <size_t _Idx, typename _Head>
+struct _Tuple_impl<_Idx, _Head> : private _Head_base<_Idx, _Head> {
+  typedef _Head_base<_Idx, _Head> _Base;
+  static constexpr const _Head &_M_head(const _Tuple_impl &__t) {
+    return _Base::_M_head(__t);
+  }
+  explicit constexpr _Tuple_impl(const _Head &__head) : _Base(__head) {}
+
+protected:
+};
+template <typename... _Elements>
+class tuple : public _Tuple_impl<0, _Elements...> {
+  using _Inherited = _Tuple_impl<0, _Elements...>;
+
+public:
+  template <typename = void>
+  constexpr tuple(const _Elements &...__elements) : _Inherited(__elements...) {}
+};
+template <typename... _Elements>
+struct tuple_size<tuple<_Elements...>>
+    : public integral_constant<size_t, sizeof...(_Elements)> {};
+template <size_t __i, typename... _Types>
+struct tuple_element<__i, tuple<_Types...>> {
+  using type = typename _Nth_type<__i, _Types...>::type;
+};
+template <size_t __i, typename _Head, typename... _Tail>
+constexpr const _Head &
+__get_helper(const _Tuple_impl<__i, _Head, _Tail...> &__t) {
+  return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t);
+};
+template <size_t __i, typename... _Elements>
+constexpr const int get(const tuple<_Elements...> &&__t) {
+  return std::__get_helper<__i>(__t);
+};
+} // namespace std
+constexpr int foo() {
+  std::vector<std::tuple<int>> data_tuples = {{1}};
+  for (const auto [id] : data_tuples) {
+    int a = id + 3;
+  }
+  return 1;
+}
+static_assert(foo() == 1);
diff --git a/clang/test/AST/ByteCode/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
index 699746c0b2c4a..ddf1d2bebdbd0 100644
--- a/clang/test/AST/ByteCode/literals.cpp
+++ b/clang/test/AST/ByteCode/literals.cpp
@@ -1418,3 +1418,15 @@ constexpr int usingDirectiveDecl() {
   return FB;
 }
 static_assert(usingDirectiveDecl() == 10, "");
+
+namespace OnePastEndCmp {
+  struct S {
+   int a;
+   int b;
+  };
+
+  constexpr S s{12,13};
+  constexpr const int *p = &s.a;
+  constexpr const int *q = &s.a + 1;
+  static_assert(p != q, "");
+}
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index 9c293e5d15fc8..840736f332250 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -1022,6 +1022,42 @@ namespace OpNewNothrow {
                       // both-note {{in call to}}
 }
 
+namespace BaseCompare {
+  struct Cmp {
+    void *p;
+
+    template<typename T>
+    constexpr Cmp(T *t) : p(t) {}
+
+    constexpr friend bool operator==(Cmp a, Cmp b) {
+      return a.p == b.p;
+    }
+  };
+
+  class Base {};
+  class Derived : public Base {};
+  constexpr bool foo() {
+    Derived *D = std::allocator<Derived>{}.allocate(1);;
+    std::construct_at<Derived>(D);
+
+    Derived *d = D;
+    Base    *b = D;
+
+    Cmp ca(d);
+    Cmp cb(b);
+
+    if (ca == cb) {
+      std::allocator<Derived>{}.deallocate(D);
+      return true;
+    }
+    std::allocator<Derived>{}.deallocate(D);
+
+    return false;
+
+  }
+  static_assert(foo());
+}
+
 #else
 /// Make sure we reject this prior to C++20
 constexpr int a() { // both-error {{never produces a constant expression}}
diff --git a/clang/test/AST/ByteCode/placement-new.cpp b/clang/test/AST/ByteCode/placement-new.cpp
index f23d71510602c..670def2d5870e 100644
--- a/clang/test/AST/ByteCode/placement-new.cpp
+++ b/clang/test/AST/ByteCode/placement-new.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_cc1 -std=c++2c -fcxx-exceptions -fexperimental-new-constant-interpreter -verify=expected,both %s -DBYTECODE
 // RUN: %clang_cc1 -std=c++2c -fcxx-exceptions -verify=ref,both %s
 
+typedef __INT64_TYPE__ int64_t;
 namespace std {
   using size_t = decltype(sizeof(0));
   template<typename T> struct allocator {
@@ -465,3 +466,23 @@ namespace ArrayRoot {
 
   static_assert(foo() == 0);
 }
+
+namespace bitcast {
+  template <typename F, typename T>
+  constexpr T bit_cast(const F &f) {
+    return __builtin_bit_cast(T, f);
+  }
+  constexpr int foo() {
+    double *d = std::allocator<double>{}.allocate(2);
+    std::construct_at<double>(d, 0);
+
+    double &dd = *d;
+
+    int64_t i = bit_cast<double, int64_t>(*d);
+
+
+    std::allocator<double>{}.deallocate(d);
+    return i;
+  }
+  static_assert(foo() == 0);
+}
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 36f4b72335af3..f97990c1ff849 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -std=c++20 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -verify=ref,both %s
-// RUN: %clang_cc1 -std=c++20 -verify=ref,both %s
+// RUN: %clang_cc1            -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++20 -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1            -verify=ref,both      %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref,both      %s
 
 union U {
   int a;
@@ -600,6 +600,118 @@ namespace MoveOrAssignOp {
   }
   static_assert(foo());
 }
+
+namespace CopyEmptyUnion {
+  struct A {
+    union {}; // both-warning {{declaration does not declare anything}}
+  };
+  constexpr int foo() {
+     A a;
+     A a2 = a;
+     return 1;
+  }
+  static_assert(foo() == 1);
+}
+
+namespace BitFields {
+  constexpr bool simple() {
+    union U {
+      unsigned a : 1;
+      unsigned b : 1;
+    };
+
+    U u{1};
+    u.b = 1;
+    return u.b;
+  }
+  static_assert(simple());
+}
+
+namespace deactivateRecurses {
+
+  constexpr int foo() {
+    struct A {
+      struct {
+        int a;
+      };
+      int b;
+    };
+    struct B {
+      struct {
+        int a;
+        int b;
+      };
+    };
+
+    union U {
+      A a;
+      B b;
+    } u;
+
+    u.b.a = 10;
+    ++u.b.a;
+
+    u.a.a = 10;
+    ++u.a.a;
+
+    if (__builtin_constant_p(u.b.a))
+      return 10;
+
+    return 1;
+  }
+  static_assert(foo() == 1);
+}
+
+namespace AnonymousUnion {
+  struct Long {
+    struct {
+      unsigned is_long;
+    };
+    unsigned Size;
+  };
+
+  struct Short {
+    struct {
+      unsigned is_long;
+      unsigned Size;
+    };
+    char data;
+  };
+
+  union Rep {
+    Short S;
+    Long L;
+  };
+
+#define assert_active(F)   if (!__builtin_is_within_lifetime(&F)) (1/0);
+#define assert_inactive(F) if ( __builtin_is_within_lifetime(&F)) (1/0);
+  consteval int test() {
+    union UU {
+      struct {
+        Rep R;
+        int a;
+      };
+    } U;
+
+    U.R.S.Size = 10;
+    assert_active(U);
+    assert_active(U.R);
+    assert_active(U.R.S);
+    assert_active(U.R.S.Size);
+
+    U.a = 10;
+    assert_active(U.a);
+    assert_active(U);
+
+    assert_active(U);
+    assert_active(U.R);
+    assert_active(U.R.S);
+    assert_active(U.R.S.Size);
+
+    return 1;
+  }
+  static_assert(test() == 1);
+}
 #endif
 
 namespace AddressComparison {
diff --git a/clang/test/AST/HLSL/RootSignatures-AST.hlsl b/clang/test/AST/HLSL/RootSignatures-AST.hlsl
index 1e60b9367c145..27c40430c9d0a 100644
--- a/clang/test/AST/HLSL/RootSignatures-AST.hlsl
+++ b/clang/test/AST/HLSL/RootSignatures-AST.hlsl
@@ -1,11 +1,11 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -ast-dump \
-// RUN:  -disable-llvm-passes -o - %s | FileCheck %s
+// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1_1
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -ast-dump \
 // RUN:  -fdx-rootsignature-version=rootsig_1_0 \
-// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-V1_0
+// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1_0
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -ast-dump \
 // RUN:  -fdx-rootsignature-version=rootsig_1_1 \
-// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-V1_1
+// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1_1
 
 // This test ensures that the sample root signature is parsed without error and
 // the Attr AST Node is created succesfully. If an invalid root signature was
@@ -37,13 +37,17 @@
 // CHECK-SAME:   space = 1, visibility = All, flags = DataStatic
 // CHECK-SAME: ),
 // CHECK-SAME: RootSRV(t0,
-// CHECK-SAME:   space = 0, visibility = All, flags = DataStaticWhileSetAtExecute
+// CHECK-SAME:   space = 0, visibility = All,
+// CHECK-V1_0-SAME: flags = DataVolatile
+// CHECK-V1_1-SAME: flags = DataStaticWhileSetAtExecute
 // CHECK-SAME: ),
 // CHECK-SAME: RootUAV(
 // CHECK-SAME:   u0, space = 0, visibility = All, flags = DataVolatile
 // CHECK-SAME: ),
 // CHECK-SAME: CBV(
-// CHECK-SAME:   b1, numDescriptors = 1, space = 0, offset = DescriptorTableOffsetAppend, flags = DataStaticWhileSetAtExecute
+// CHECK-SAME:   b1, numDescriptors = 1, space = 0, offset = DescriptorTableOffsetAppend,
+// CHECK-V1_0-SAME: flags = DescriptorsVolatile | DataVolatile
+// CHECK-V1_1-SAME: flags = DataStaticWhileSetAtExecute
 // CHECK-SAME: ),
 // CHECK-SAME: SRV(
 // CHECK-SAME:   t1, numDescriptors = 8, space = 0, offset = DescriptorTableOffsetAppend, flags = DescriptorsVolatile
@@ -55,7 +59,9 @@
 // CHECK-SAME:   numClauses = 3, visibility = All
 // CHECK-SAME: ),
 // CHECK-SAME: Sampler(
-// CHECK-SAME:   s0, numDescriptors = 4, space = 1, offset = DescriptorTableOffsetAppend, flags = None
+// CHECK-SAME:   s0, numDescriptors = 4, space = 1, offset = DescriptorTableOffsetAppend,
+// CHECK-V1_0-SAME:  flags = DescriptorsVolatile
+// CHECK-V1_1-SAME:  flags = None
 // CHECK-SAME: ),
 // CHECK-SAME: DescriptorTable(
 // CHECK-SAME:   numClauses = 1, visibility = All
@@ -112,9 +118,13 @@ void same_rs_string_main() {}
 // a seperate decl and identifier to reference
 
 // CHECK: -HLSLRootSignatureDecl 0x{{.*}} {{.*}} implicit [[DIFF_RS_DECL:__hlsl_rootsig_decl_\d*]]
+// CHECK-V1_0: version: 1.0,
+// CHECK-V1_1: version: 1.1,
 // CHECK-SAME: RootElements{
-// CHECK-SAME:   Sampler(s0, numDescriptors = 4, space = 1,
-// CHECK-SAME:     offset = DescriptorTableOffsetAppend, flags = None),
+// CHECK-SAME:   Sampler(s0, numDescriptors = 4, space = 1, offset = DescriptorTableOffsetAppend,
+// CHECK-V1_0-SAME:  flags = DescriptorsVolatile
+// CHECK-V1_1-SAME:  flags = None
+// CHECK-SAME: ),
 // CHECK-SAME:   DescriptorTable(numClauses = 1, visibility = All)
 // CHECK-SAME: }
 
diff --git a/clang/test/AST/absurdly_big_struct.cpp b/clang/test/AST/absurdly_big_struct.cpp
new file mode 100644
index 0000000000000..c17274343d57a
--- /dev/null
+++ b/clang/test/AST/absurdly_big_struct.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s -triple x86_64-linux-gnu
+
+struct a { // expected-error {{structure 'a' is too large, which exceeds maximum allowed size of 1152921504606846976 bytes}}
+  char x[1ull<<60]; 
+  char x2[1ull<<60]; 
+};
+
+a z[1];
+long long x() { return sizeof(a); }
+long long x2() { return sizeof(a::x); }
+long long x3() { return sizeof(a::x2); }
+long long x4() { return sizeof(z); }
+
diff --git a/clang/test/AST/ast-print-openacc-cache-construct.cpp b/clang/test/AST/ast-print-openacc-cache-construct.cpp
index e98ff3ca67600..26dd1333ee9ed 100644
--- a/clang/test/AST/ast-print-openacc-cache-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-cache-construct.cpp
@@ -2,8 +2,11 @@
 
 void foo() {
   int Array[5];
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
   // CHECK: #pragma acc cache(readonly: Array[1], Array[1:2])
   #pragma acc cache(readonly:Array[1], Array[1:2])
   // CHECK: #pragma acc cache(Array[1], Array[1:2])
   #pragma acc cache(Array[1], Array[1:2])
+  }
 }
diff --git a/clang/test/AST/static-compound-literals-crash.cpp b/clang/test/AST/static-compound-literals-crash.cpp
new file mode 100644
index 0000000000000..1b37bfe8fad7b
--- /dev/null
+++ b/clang/test/AST/static-compound-literals-crash.cpp
@@ -0,0 +1,17 @@
+// FIXME: These test cases currently crash during codegen, despite initializers
+// for CLEs being constant.
+// RUN: not --crash %clang_cc1 -verify -std=c++20 -emit-llvm %s
+// expected-no-diagnostics
+namespace case1 {
+struct RR { int&& r; };
+struct Z { RR* x; };
+constinit Z z = { (RR[1]){1} };
+}
+
+
+namespace case2 {
+struct RR { int r; };
+struct Z { int x; const RR* y; int z; };
+inline int f() { return 0; }
+Z z2 = { 10, (const RR[1]){__builtin_constant_p(z2.x)}, z2.y->r+f() };
+}
diff --git a/clang/test/AST/static-compound-literals-reeval.cpp b/clang/test/AST/static-compound-literals-reeval.cpp
new file mode 100644
index 0000000000000..bd8f0f27af24a
--- /dev/null
+++ b/clang/test/AST/static-compound-literals-reeval.cpp
@@ -0,0 +1,9 @@
+// Test that we can successfully compile this code, especially under ASAN.
+// RUN: %clang_cc1 -emit-llvm -std=c++20 %s -o- | FileCheck %s
+struct RR { int r; };
+struct Z { int x; const RR* y; int z; };
+constinit Z z = { 10, (const RR[1]){__builtin_constant_p(z.x)}, z.y->r };
+// Check that we zero-initialize z.y->r.
+// CHECK: @.compoundliteral = internal constant [1 x %struct.RR] zeroinitializer
+// FIXME: Despite of z.y->r being 0, we evaluate z.z to 1.
+// CHECK: global %struct.Z { i32 10, ptr @.compoundliteral, i32 1 }
diff --git a/clang/test/AST/static-compound-literals.cpp b/clang/test/AST/static-compound-literals.cpp
new file mode 100644
index 0000000000000..ceb8b985ab9dc
--- /dev/null
+++ b/clang/test/AST/static-compound-literals.cpp
@@ -0,0 +1,12 @@
+// Test that we can successfully compile this code, especially under ASAN.
+// RUN: %clang_cc1 -verify -std=c++20 -fsyntax-only %s
+// expected-no-diagnostics
+struct Foo {
+  Foo* f;
+  operator bool() const { return true; }
+};
+constexpr Foo f((Foo[]){});
+int foo() {
+  if (Foo(*f.f)) return 1;
+  return 0;
+}
diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp
index 6887e0a35fb8b..2710fa1e1e112 100644
--- a/clang/test/Analysis/ArrayDelete.cpp
+++ b/clang/test/Analysis/ArrayDelete.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s
 
 struct Base {
     virtual ~Base() = default;
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm b/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm
new file mode 100644
index 0000000000000..7ccc0d61f0acf
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm
@@ -0,0 +1,12 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
+// expected-no-diagnostics
+
+#include "objc-mock-types.h"
+
+CGImageRef provideImage();
+
+Boolean cfe(CFTypeRef obj)
+{
+  return CFEqual(obj, provideImage());
+}
+
diff --git a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
index 9e4356a71f1b5..09b303961fd6a 100644
--- a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
@@ -45,6 +45,7 @@ CFDictionaryRef CFDictionaryCreateCopy(CFAllocatorRef allocator, CFDictionaryRef
 CFDictionaryRef CFDictionaryCreateMutableCopy(CFAllocatorRef allocator, CFIndex capacity, CFDictionaryRef theDict);
 CFIndex CFDictionaryGetCount(CFDictionaryRef theDict);
 Boolean CFDictionaryContainsKey(CFDictionaryRef theDict, const void *key);
+Boolean CFEqual(CFTypeRef, CFTypeRef);
 Boolean CFDictionaryContainsValue(CFDictionaryRef theDict, const void *value);
 const void *CFDictionaryGetValue(CFDictionaryRef theDict, const void *key);
 void CFDictionaryAddValue(CFMutableDictionaryRef theDict, const void *key, const void *value);
diff --git a/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp b/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp
index 2809f3758eee1..68ae1aee2e289 100644
--- a/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp
+++ b/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.DeleteWithNonVirtualDtor -std=c++11 -verify -analyzer-output=text %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.cplusplus.DeleteWithNonVirtualDtor -std=c++11 -verify -analyzer-output=text %s
 
 struct Virtual {
   virtual ~Virtual() {}
diff --git a/clang/test/Analysis/PR37855.c b/clang/test/Analysis/PR37855.c
index e08ce8f250f86..92ed9ffe69a7d 100644
--- a/clang/test/Analysis/PR37855.c
+++ b/clang/test/Analysis/PR37855.c
@@ -1,24 +1,22 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=core -w -DNO_CROSSCHECK -verify %s
-// RUN: %clang_cc1 -analyze -analyzer-checker=core -w -analyzer-config crosscheck-with-z3=true -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -w -DNO_CROSSCHECK -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -w -analyzer-config crosscheck-with-z3=true -verify %s
 // REQUIRES: z3
 
-// XFAIL: *
-
 typedef struct o p;
 struct o {
   struct {
   } s;
 };
 
-void q(*r, p2) { r < p2; }
+void q(int *r, int p2) { r < p2; }
 
-void k(l, node) {
+void k(int l, int node) {
   struct {
     p *node;
   } * n, *nodep, path[sizeof(void)];
-  path->node = l;
+  path->node = (p*) l;
   for (n = path; node != l;) {
-    q(node, n->node);
+    q((int *)&node, (int)n->node);
     nodep = n;
   }
   if (nodep) // expected-warning {{Branch condition evaluates to a garbage value}}
diff --git a/clang/test/Analysis/aggrinit-cfg-output.cpp b/clang/test/Analysis/aggrinit-cfg-output.cpp
index 2e95a7f9e24fe..682399399d21d 100644
--- a/clang/test/Analysis/aggrinit-cfg-output.cpp
+++ b/clang/test/Analysis/aggrinit-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-expand-default-aggr-inits=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCFG -analyzer-config cfg-expand-default-aggr-inits=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 static char a[] = "foobar";
diff --git a/clang/test/Analysis/call-and-message.c b/clang/test/Analysis/call-and-message.c
index b79ec8c344b6c..ade51145e2a93 100644
--- a/clang/test/Analysis/call-and-message.c
+++ b/clang/test/Analysis/call-and-message.c
@@ -1,12 +1,19 @@
 // RUN: %clang_analyze_cc1 %s -verify \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=true \
+// RUN:   -analyzer-config core.CallAndMessage:ArgInitializedness=false \
 // RUN:   -analyzer-output=plist -o %t.plist
 // RUN: cat %t.plist | FileCheck %s
 
 // RUN: %clang_analyze_cc1 %s -verify=no-pointee \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=false
+// RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=false \
+// RUN:   -analyzer-config core.CallAndMessage:ArgInitializedness=false
+
+// RUN: %clang_analyze_cc1 %s -verify=arg-init \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=false \
+// RUN:   -analyzer-config core.CallAndMessage:ArgInitializedness=true
 
 // no-pointee-no-diagnostics
 
@@ -22,3 +29,21 @@ void pointee_uninit(void) {
 // checker, as described in the CallAndMessage comments!
 // CHECK: <key>issue_hash_content_of_line_in_context</key>
 // CHECK-SAME: <string>97a74322d64dca40aa57303842c745a1</string>
+
+typedef struct {
+  int i  :2;
+  int    :30;  // unnamed bit-field
+} B;
+
+extern void consume_B(B);
+
+void bitfield_B_init(void) {
+  B b1;
+  b1.i = 1; // b1 is initialized
+  consume_B(b1);
+}
+
+void bitfield_B_uninit(void) {
+  B b2;
+  consume_B(b2); // arg-init-warning{{Passed-by-value struct argument contains uninitialized data (e.g., field: 'i') [core.CallAndMessage]}}
+}
diff --git a/clang/test/Analysis/call-and-message.cpp b/clang/test/Analysis/call-and-message.cpp
index 25ae23833478b..1e76973f49e13 100644
--- a/clang/test/Analysis/call-and-message.cpp
+++ b/clang/test/Analysis/call-and-message.cpp
@@ -169,4 +169,20 @@ void record_uninit() {
 // CHECK-SAME: <string>a46bb5c1ee44d4611ffeb13f7f499605</string>
 // CHECK: <key>issue_hash_content_of_line_in_context</key>
 // CHECK-SAME: <string>e0e0d30ea5a7b2e3a71e1931fa0768a5</string>
+
+struct B{
+  int i  :2;
+  int    :30;  // unnamed bit-field
+};
+
+void bitfield_B_init(void) {
+  B b1;
+  b1.i = 1; // b1 is initialized
+  consume(b1);
+}
+
+void bitfield_B_uninit(void) {
+  B b2;
+  consume(b2); // arg-init-warning{{Passed-by-value struct argument contains uninitialized data (e.g., field: 'i') [core.CallAndMessage]}}
+}
 } // namespace uninit_arg
diff --git a/clang/test/Analysis/compound-literals.c b/clang/test/Analysis/compound-literals.c
index c74eacc3aa683..90ef0ceb46b2e 100644
--- a/clang/test/Analysis/compound-literals.c
+++ b/clang/test/Analysis/compound-literals.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple=i386-apple-darwin10 -verify %s -analyze \
+// RUN: %clang_analyze_cc1 -triple=i386-apple-darwin10 -verify %s \
 // RUN:   -analyzer-checker=debug.ExprInspection
 
 #define NULL 0
diff --git a/clang/test/Analysis/consteval-if.cpp b/clang/test/Analysis/consteval-if.cpp
new file mode 100644
index 0000000000000..e673680453025
--- /dev/null
+++ b/clang/test/Analysis/consteval-if.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_analyze_cc1 -std=c++23 -analyzer-checker=core -verify %s
+// RUN: %clang_analyze_cc1 -std=c++26 -analyzer-checker=core -verify %s
+
+void test_consteval() {
+  if consteval {
+    int *ptr = nullptr;
+    *ptr = 42; // expected-warning{{Dereference of null pointer (loaded from variable 'ptr')}}
+  }
+}
+
+void test_not_consteval() {
+  if !consteval {
+    int *ptr = nullptr;
+    *ptr = 42; // expected-warning{{Dereference of null pointer (loaded from variable 'ptr')}}
+  }
+}
diff --git a/clang/test/Analysis/cstring-should-not-invalidate.cpp b/clang/test/Analysis/cstring-should-not-invalidate.cpp
new file mode 100644
index 0000000000000..abd4689b2f9b9
--- /dev/null
+++ b/clang/test/Analysis/cstring-should-not-invalidate.cpp
@@ -0,0 +1,132 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.cstring,alpha.unix.cstring,debug.ExprInspection -verify %s
+
+typedef decltype(sizeof(int)) size_t;
+void clang_analyzer_eval(bool);
+
+char *strncpy(char *dest, const char *src, size_t x);
+
+constexpr int initB = 100;
+struct Base {
+  int b;
+  Base(): b(initB) {}
+};
+
+// issue 143807
+struct strncpyTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strncpy(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strncpy(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strncpyTest(char *src, size_t n) {
+  strncpyTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
+
+size_t strlcpy(char *dest, const char *src, size_t size);
+
+struct strlcpyTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strlcpy(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strlcpy(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strlcpyTest(char *src, size_t n) {
+  strlcpyTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
+
+char *strncat(char *s1, const char *s2, size_t n);
+
+struct strncatTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strncat(m_buff, src, sizeof(m_buff) - 1); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strncat(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strncatTest(char *src, size_t n) {
+  strncatTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
+
+struct strncatReportOutOfBoundTestClass {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    // expected-warning@+1{{String concatenation function overflows the destination buffer}}
+    strncat(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+};
+
+void strncatReportOutOfBoundTest(char *src, size_t n) {
+  strncatReportOutOfBoundTestClass rep;
+  rep.KnownLen(src);
+}
+
+size_t strlcat(char *dst, const char *src, size_t size);
+
+struct strlcatTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strlcat(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strlcat(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strlcatTest(char *src, size_t n) {
+  strlcatTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
diff --git a/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp b/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp
index 18f03ee6b0101..995db5614527c 100644
--- a/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp
+++ b/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -emit-pch -o %t/ctudir/ctu-cxxdefaultinitexpr-import.cpp.ast %S/Inputs/ctu-cxxdefaultinitexpr-import.cpp
 // RUN: cp %S/Inputs/ctu-cxxdefaultinitexpr-import.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=%t/ctudir \
diff --git a/clang/test/Analysis/ctu-implicit.c b/clang/test/Analysis/ctu-implicit.c
index c0b28fc25bd25..63df5fb0542b1 100644
--- a/clang/test/Analysis/ctu-implicit.c
+++ b/clang/test/Analysis/ctu-implicit.c
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1  \
 // RUN:   -emit-pch -o %t/ctudir2/ctu-import.c.ast %S/Inputs/ctu-import.c
 // RUN: cp %S/Inputs/ctu-import.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
diff --git a/clang/test/Analysis/ctu-import-threshold.c b/clang/test/Analysis/ctu-import-threshold.c
index 62bbeda20d5ba..f7e1285d595ea 100644
--- a/clang/test/Analysis/ctu-import-threshold.c
+++ b/clang/test/Analysis/ctu-import-threshold.c
@@ -1,6 +1,6 @@
 // Ensure analyzer option 'ctu-import-threshold' is a recognized option.
 //
-// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-threshold=30 -verify %s
-// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-cpp-threshold=30 -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-config ctu-import-threshold=30 -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-config ctu-import-cpp-threshold=30 -verify %s
 //
 // expected-no-diagnostics
diff --git a/clang/test/Analysis/ctu-lookup-name-with-space.cpp b/clang/test/Analysis/ctu-lookup-name-with-space.cpp
index ef733d5c52e3d..0761cfe092296 100644
--- a/clang/test/Analysis/ctu-lookup-name-with-space.cpp
+++ b/clang/test/Analysis/ctu-lookup-name-with-space.cpp
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -emit-pch %/S/Inputs/ctu-lookup-name-with-space.cpp -o %t/importee.ast
 
 // RUN: cd %t
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=. \
diff --git a/clang/test/Analysis/ctu-main.c b/clang/test/Analysis/ctu-main.c
index 0193a7e6d135d..e24510e8aadc4 100644
--- a/clang/test/Analysis/ctu-main.c
+++ b/clang/test/Analysis/ctu-main.c
@@ -4,7 +4,7 @@
 // RUN:   -emit-pch -o %t/ctudir2/ctu-other.c.ast %S/Inputs/ctu-other.c
 // RUN: cp %S/Inputs/ctu-other.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c89 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c89 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
@@ -14,7 +14,7 @@
 
 // Simulate the behavior of the previous CTU implementation by inlining all
 // functions during the first phase. This way, the second phase is a noop.
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c89 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c89 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
diff --git a/clang/test/Analysis/ctu-on-demand-parsing.c b/clang/test/Analysis/ctu-on-demand-parsing.c
index 17ade150ded5e..2107b989d9cc3 100644
--- a/clang/test/Analysis/ctu-on-demand-parsing.c
+++ b/clang/test/Analysis/ctu-on-demand-parsing.c
@@ -11,7 +11,7 @@
 //
 // RUN: cd "%t" && %clang_extdef_map "%t/ctu-other.c" > externalDefMap.txt
 //
-// RUN: cd "%t" && %clang_cc1 -std=c89 -analyze \
+// RUN: cd "%t" && %clang_analyze_cc1 -std=c89 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=. \
diff --git a/clang/test/Analysis/ctu-test-import-failure.cpp b/clang/test/Analysis/ctu-test-import-failure.cpp
index 2295a66538fc9..812fa01b6c67f 100644
--- a/clang/test/Analysis/ctu-test-import-failure.cpp
+++ b/clang/test/Analysis/ctu-test-import-failure.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -emit-pch -o %t/ctudir/ctu-test-import-failure-import.cpp.ast %S/Inputs/ctu-test-import-failure-import.cpp
 // RUN: cp %S/Inputs/ctu-test-import-failure-import.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=%t/ctudir \
diff --git a/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp b/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp
index 01914a86ad4a5..76e67a8690bd9 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 struct S {};
diff --git a/clang/test/Analysis/exploded-graph-rewriter/escapes.c b/clang/test/Analysis/exploded-graph-rewriter/escapes.c
index 830e082730a84..a3a6868cf8dfa 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/escapes.c
+++ b/clang/test/Analysis/exploded-graph-rewriter/escapes.c
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 void escapes(void) {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp b/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp
index 82b7f3269f318..52f23e298ec55 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 struct A {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp b/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
index f18c4c5afe147..3798846c8873e 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
@@ -1,8 +1,7 @@
 // CAUTION: The name of this file should start with `l` for proper tests.
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 void test1() {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/macros.c b/clang/test/Analysis/exploded-graph-rewriter/macros.c
index f47342fc524b7..32b8d55d553e9 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/macros.c
+++ b/clang/test/Analysis/exploded-graph-rewriter/macros.c
@@ -6,10 +6,9 @@ void *foo(void) {
 // The code above shall go first, because check tags below are sensetive to the line numbers on which the code is placed.
 // You can change lines below in the way you need.
 
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 // CHECK: macros.c:<b>3</b>:<b>10</b>
diff --git a/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp b/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp
index f2628ee686632..145a56c1c44f9 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp
@@ -1,8 +1,7 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -std=c++14 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyze-function "test()" \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyze-function "test()" \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 struct S {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp b/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
index 5b955a0aae3ec..2c1c934d264f5 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter --verbose %t.dot 2>&1 | FileCheck %s
 // REQUIRES: system-windows
 
diff --git a/clang/test/Analysis/gmalloc.c b/clang/test/Analysis/gmalloc.c
index 279e2ae844b78..dae28ebfa910c 100644
--- a/clang/test/Analysis/gmalloc.c
+++ b/clang/test/Analysis/gmalloc.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=core,unix.Malloc -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -verify %s
 
 #include "Inputs/system-header-simulator.h"
 
diff --git a/clang/test/Analysis/handle_constructors_for_default_arguments.cpp b/clang/test/Analysis/handle_constructors_for_default_arguments.cpp
index e20ec5e57d3a4..2a934be2e8d34 100644
--- a/clang/test/Analysis/handle_constructors_for_default_arguments.cpp
+++ b/clang/test/Analysis/handle_constructors_for_default_arguments.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection %s -verify
 
 // These test cases demonstrate lack of Static Analyzer features.
diff --git a/clang/test/Analysis/handle_constructors_with_new_array.cpp b/clang/test/Analysis/handle_constructors_with_new_array.cpp
index d55d354f9c5cc..4acdaaf454903 100644
--- a/clang/test/Analysis/handle_constructors_with_new_array.cpp
+++ b/clang/test/Analysis/handle_constructors_with_new_array.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection %s -verify
 
 // These test cases demonstrate lack of Static Analyzer features.
diff --git a/clang/test/Analysis/initialization.c b/clang/test/Analysis/initialization.c
index bd003b721d91d..32bdf733bcb98 100644
--- a/clang/test/Analysis/initialization.c
+++ b/clang/test/Analysis/initialization.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple i386-apple-darwin10 -analyze -analyzer-config eagerly-assume=false  -analyzer-checker=core.uninitialized.Assign,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -analyzer-config eagerly-assume=false  -analyzer-checker=core.uninitialized.Assign,debug.ExprInspection -verify %s
 
 void clang_analyzer_eval(int);
 
diff --git a/clang/test/Analysis/initialization.cpp b/clang/test/Analysis/initialization.cpp
index e5e19daad0895..d064dd6172d63 100644
--- a/clang/test/Analysis/initialization.cpp
+++ b/clang/test/Analysis/initialization.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++14 -triple i386-apple-darwin10 -analyze -analyzer-config eagerly-assume=false -analyzer-checker=core.uninitialized.Assign,core.builtin,debug.ExprInspection,core.uninitialized.UndefReturn -verify %s
+// RUN: %clang_analyze_cc1 -std=c++14 -triple i386-apple-darwin10 -analyzer-config eagerly-assume=false -analyzer-checker=core.uninitialized.Assign,core.builtin,debug.ExprInspection,core.uninitialized.UndefReturn -verify %s
 
 template <typename T>
 void clang_analyzer_dump(T x);
diff --git a/clang/test/Analysis/lifetime-cfg-output.cpp b/clang/test/Analysis/lifetime-cfg-output.cpp
index d8b8dc4a71d4a..0a75c5bcc0bcc 100644
--- a/clang/test/Analysis/lifetime-cfg-output.cpp
+++ b/clang/test/Analysis/lifetime-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-lifetime=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -fcxx-exceptions -fexceptions -analyzer-checker=debug.DumpCFG -analyzer-config cfg-lifetime=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 // FIXME Most of the cases in this file test only with the objects of type `A`,
diff --git a/clang/test/Analysis/loopexit-cfg-output.cpp b/clang/test/Analysis/loopexit-cfg-output.cpp
index 8e53ce3066b7f..26ef33e4a6a9a 100644
--- a/clang/test/Analysis/loopexit-cfg-output.cpp
+++ b/clang/test/Analysis/loopexit-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-loopexit=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCFG -analyzer-config cfg-loopexit=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 // CHECK:       [B6 (ENTRY)]
diff --git a/clang/test/Analysis/method-arg-decay.m b/clang/test/Analysis/method-arg-decay.m
index a222346861a5d..69fd7a241b712 100644
--- a/clang/test/Analysis/method-arg-decay.m
+++ b/clang/test/Analysis/method-arg-decay.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyzer-checker=core -verify %s -Wno-incomplete-implementation
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -verify %s -Wno-incomplete-implementation
 typedef signed char BOOL;
 typedef int NSInteger;
 typedef unsigned int NSUInteger;
diff --git a/clang/test/Analysis/new.cpp b/clang/test/Analysis/new.cpp
index 3542b5c594b50..8439a4e55d812 100644
--- a/clang/test/Analysis/new.cpp
+++ b/clang/test/Analysis/new.cpp
@@ -327,39 +327,7 @@ void testArrayDestr() {
   clang_analyzer_warnIfReached(); // no-warning
 }
 
-// Invalidate Region even in case of default destructor
-class InvalidateDestTest {
-public:
-  int x;
-  int *y;
-  ~InvalidateDestTest();
-};
-
-int test_member_invalidation() {
-
-  //test invalidation of member variable
-  InvalidateDestTest *test = new InvalidateDestTest();
-  test->x = 5;
-  int *k = &(test->x);
-  clang_analyzer_eval(*k == 5); // expected-warning{{TRUE}}
-  delete test;
-  clang_analyzer_eval(*k == 5); // expected-warning{{UNKNOWN}}
-
-  //test invalidation of member pointer
-  int localVar = 5;
-  test = new InvalidateDestTest();
-  test->y = &localVar;
-  delete test;
-  clang_analyzer_eval(localVar == 5); // expected-warning{{UNKNOWN}}
-
-  // Test aray elements are invalidated.
-  int Var1 = 5;
-  int Var2 = 5;
-  InvalidateDestTest *a = new InvalidateDestTest[2];
-  a[0].y = &Var1;
-  a[1].y = &Var2;
-  delete[] a;
-  clang_analyzer_eval(Var1 == 5); // expected-warning{{UNKNOWN}}
-  clang_analyzer_eval(Var2 == 5); // expected-warning{{UNKNOWN}}
-  return 0;
-}
+// See also test-member-invalidation.cpp which validates that calling an
+// unknown destructor invalidates the members of an object. This behavior
+// cannot be tested in this file because here `MallocChecker.cpp` sinks
+// execution paths that refer to members of a deleted object.
diff --git a/clang/test/Analysis/null-deref-static.m b/clang/test/Analysis/null-deref-static.m
index 06a97644aa1d1..2bfb4af29241a 100644
--- a/clang/test/Analysis/null-deref-static.m
+++ b/clang/test/Analysis/null-deref-static.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -w -fblocks -analyze -analyzer-checker=core,deadcode,alpha.core,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -w -fblocks -analyzer-checker=core,deadcode,alpha.core,debug.ExprInspection -verify %s
 
 void *malloc(unsigned long);
 void clang_analyzer_warnIfReached(void);
diff --git a/clang/test/Analysis/objc-encode.m b/clang/test/Analysis/objc-encode.m
index 1d6515c2405a2..afa71c3a9432d 100644
--- a/clang/test/Analysis/objc-encode.m
+++ b/clang/test/Analysis/objc-encode.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.ExprInspection -verify %s
 // expected-no-diagnostics
 
 void clang_analyzer_eval(int);
diff --git a/clang/test/Analysis/openmp-unsupported.c b/clang/test/Analysis/openmp-unsupported.c
index 85dd4b66ba871..c6769bd166db0 100644
--- a/clang/test/Analysis/openmp-unsupported.c
+++ b/clang/test/Analysis/openmp-unsupported.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple i386-apple-darwin10 -analyze -analyzer-checker=core.builtin -fopenmp -verify %s
+// RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -analyzer-checker=core.builtin -fopenmp -verify %s
 // expected-no-diagnostics
 
 void openmp_parallel_crash_test(void) {
diff --git a/clang/test/Analysis/pointer-escape-on-conservative-calls.c b/clang/test/Analysis/pointer-escape-on-conservative-calls.c
index 19d86ce1866f8..5de2b1cf31082 100644
--- a/clang/test/Analysis/pointer-escape-on-conservative-calls.c
+++ b/clang/test/Analysis/pointer-escape-on-conservative-calls.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:PointerEscape=true -analyzer-config debug.AnalysisOrder:PostCall=true %s 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:PointerEscape=true -analyzer-config debug.AnalysisOrder:PostCall=true %s 2>&1 | FileCheck %s
 
 
 void f(int *);
diff --git a/clang/test/Analysis/print-ranges.cpp b/clang/test/Analysis/print-ranges.cpp
index 89a4249ebfaa5..8964f68dbfee9 100644
--- a/clang/test/Analysis/print-ranges.cpp
+++ b/clang/test/Analysis/print-ranges.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -analyzer-config eagerly-assume=false -verify %s
-// UNSUPPORTED z3
+// UNSUPPORTED: z3
 
 template <typename T>
 void clang_analyzer_value(T x);
diff --git a/clang/test/Analysis/scopes-cfg-output.cpp b/clang/test/Analysis/scopes-cfg-output.cpp
index c082bb179545f..6ed6f3638f75b 100644
--- a/clang/test/Analysis/scopes-cfg-output.cpp
+++ b/clang/test/Analysis/scopes-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-scopes=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -fcxx-exceptions -fexceptions -analyzer-checker=debug.DumpCFG -analyzer-config cfg-scopes=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 class A {
diff --git a/clang/test/Analysis/taint-diagnostic-visitor.c b/clang/test/Analysis/taint-diagnostic-visitor.c
index afe7117db7150..7bbc1faa27c4d 100644
--- a/clang/test/Analysis/taint-diagnostic-visitor.c
+++ b/clang/test/Analysis/taint-diagnostic-visitor.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=optin.taint,core,security.ArrayBound -analyzer-output=text -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=optin.taint,core,security.ArrayBound -analyzer-output=text -verify %s
 
 // This file is for testing enhanced diagnostics produced by the GenericTaintChecker
 
diff --git a/clang/test/Analysis/temporaries-callback-order.cpp b/clang/test/Analysis/temporaries-callback-order.cpp
index 120fabc10478e..00a79c44e222e 100644
--- a/clang/test/Analysis/temporaries-callback-order.cpp
+++ b/clang/test/Analysis/temporaries-callback-order.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:Bind=true -analyzer-config debug.AnalysisOrder:RegionChanges=true %s 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:Bind=true -analyzer-config debug.AnalysisOrder:RegionChanges=true %s 2>&1 | FileCheck %s
 
 struct Super {
   virtual void m();
diff --git a/clang/test/Analysis/test-member-invalidation.cpp b/clang/test/Analysis/test-member-invalidation.cpp
new file mode 100644
index 0000000000000..cbff3986325df
--- /dev/null
+++ b/clang/test/Analysis/test-member-invalidation.cpp
@@ -0,0 +1,47 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -std=c++11 -verify=expected,nosink -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc,debug.ExprInspection -std=c++11 -verify=expected,sink -analyzer-config eagerly-assume=false %s
+
+// This test validates that calling an unknown destructor invalidates the
+// members of an object. This was originally a part of the test file `new.cpp`,
+// but was split off into a separate file because the checker family
+// implemented in `MallocChecker.cpp` (which is activated via unix.Malloc in
+// `new.cpp` sinks all execution paths that refer to members of a deleted object.
+
+void clang_analyzer_eval(bool);
+
+// Invalidate Region even in case of default destructor
+class InvalidateDestTest {
+public:
+  int x;
+  int *y;
+  ~InvalidateDestTest();
+};
+
+int test_member_invalidation() {
+
+  //test invalidation of member variable
+  InvalidateDestTest *test = new InvalidateDestTest();
+  test->x = 5;
+  int *k = &(test->x);
+  clang_analyzer_eval(*k == 5); // expected-warning{{TRUE}}
+  delete test;
+  clang_analyzer_eval(*k == 5); // nosink-warning{{UNKNOWN}}
+
+  //test invalidation of member pointer
+  int localVar = 5;
+  test = new InvalidateDestTest();
+  test->y = &localVar;
+  delete test;
+  clang_analyzer_eval(localVar == 5); // nosink-warning{{UNKNOWN}}
+
+  // Test aray elements are invalidated.
+  int Var1 = 5;
+  int Var2 = 5;
+  InvalidateDestTest *a = new InvalidateDestTest[2];
+  a[0].y = &Var1;
+  a[1].y = &Var2;
+  delete[] a;
+  clang_analyzer_eval(Var1 == 5); // nosink-warning{{UNKNOWN}}
+  clang_analyzer_eval(Var2 == 5); // nosink-warning{{UNKNOWN}}
+  return 0;
+}
diff --git a/clang/test/Analysis/undef-call.c b/clang/test/Analysis/undef-call.c
index 8019028282aa0..3afdb6760d1f3 100644
--- a/clang/test/Analysis/undef-call.c
+++ b/clang/test/Analysis/undef-call.c
@@ -1,6 +1,6 @@
 // RUN: rm -rf %T/ctudir
 // RUN: mkdir %T/ctudir
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.ExprInspection -analyzer-config experimental-enable-naive-ctu-analysis=true -analyzer-config ctu-dir=%T/ctudir -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.ExprInspection -analyzer-config experimental-enable-naive-ctu-analysis=true -analyzer-config ctu-dir=%T/ctudir -verify %s
 // expected-no-diagnostics
 
 struct S {
diff --git a/clang/test/Analysis/z3-crosscheck-max-attempts.cpp b/clang/test/Analysis/z3-crosscheck-max-attempts.cpp
index 572e452fdcac2..8439236883f10 100644
--- a/clang/test/Analysis/z3-crosscheck-max-attempts.cpp
+++ b/clang/test/Analysis/z3-crosscheck-max-attempts.cpp
@@ -3,15 +3,10 @@
 // RUN: | FileCheck %s --match-full-lines
 // CHECK: crosscheck-with-z3-max-attempts-per-query = 3
 
-// RUN: rm -rf %t && mkdir %t
-// RUN: %host_cxx -shared -fPIC -I %z3_include_dir        \
-// RUN:   %S/z3/Inputs/MockZ3_solver_check.cpp            \
-// RUN:   -o %t/MockZ3_solver_check.so
-
-// DEFINE: %{mocked_clang} =                              \
-// DEFINE: LD_PRELOAD="%t/MockZ3_solver_check.so"         \
-// DEFINE: %clang_cc1 %s -analyze -setup-static-analyzer  \
-// DEFINE:   -analyzer-config crosscheck-with-z3=true     \
+// DEFINE: %{mocked_clang} =                                      \
+// DEFINE: LD_PRELOAD="%llvmshlibdir/MockZ3SolverCheck%pluginext" \
+// DEFINE: %clang_analyze_cc1 %s                                  \
+// DEFINE:   -analyzer-config crosscheck-with-z3=true             \
 // DEFINE:   -analyzer-checker=core
 
 // DEFINE: %{attempts} = -analyzer-config crosscheck-with-z3-max-attempts-per-query
@@ -32,7 +27,7 @@
 // RUN: Z3_SOLVER_RESULTS="UNDEF,UNDEF,SAT"   %{mocked_clang} %{attempts}=3 -verify=accepted
 
 
-// REQUIRES: z3, asserts, shell, system-linux
+// REQUIRES: z3, z3-mock, asserts, shell, system-linux
 
 // refuted-no-diagnostics
 
diff --git a/clang/test/Analysis/z3-crosscheck.c b/clang/test/Analysis/z3-crosscheck.c
index bd447d33f8c5d..8cbe7dd2975d0 100644
--- a/clang/test/Analysis/z3-crosscheck.c
+++ b/clang/test/Analysis/z3-crosscheck.c
@@ -1,9 +1,7 @@
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -analyze -analyzer-checker=core,unix.Malloc,debug.ExprInspection -DNO_CROSSCHECK -verify %s
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -analyze -analyzer-checker=core,unix.Malloc,debug.ExprInspection -analyzer-config crosscheck-with-z3=true -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -analyzer-checker=core,unix.Malloc,debug.ExprInspection -DNO_CROSSCHECK -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -analyzer-checker=core,unix.Malloc,debug.ExprInspection -analyzer-config crosscheck-with-z3=true -verify %s
 // REQUIRES: z3
 
-// XFAIL: *
-
 void clang_analyzer_dump(float);
 
 int foo(int x) 
diff --git a/clang/test/Analysis/z3/D83660.c b/clang/test/Analysis/z3/D83660.c
index 0a7c8bab8e345..16ea4ffb0e1be 100644
--- a/clang/test/Analysis/z3/D83660.c
+++ b/clang/test/Analysis/z3/D83660.c
@@ -1,14 +1,9 @@
-// RUN: rm -rf %t && mkdir %t
-// RUN: %host_cxx -shared -fPIC -I %z3_include_dir \
-// RUN:   %S/Inputs/MockZ3_solver_check.cpp \
-// RUN:   -o %t/MockZ3_solver_check.so
-//
 // RUN: Z3_SOLVER_RESULTS="SAT,SAT,SAT,SAT,UNDEF" \
-// RUN: LD_PRELOAD="%t/MockZ3_solver_check.so" \
-// RUN: %clang_cc1 -analyze -analyzer-constraints=z3 -setup-static-analyzer \
+// RUN: LD_PRELOAD="%llvmshlibdir/MockZ3SolverCheck%pluginext" \
+// RUN: %clang_analyze_cc1 -analyzer-constraints=z3 \
 // RUN:   -analyzer-checker=core %s -verify
 //
-// REQUIRES: z3, asserts, shell, system-linux
+// REQUIRES: z3, z3-mock, asserts, shell, system-linux
 //
 // Works only with the z3 constraint manager.
 // expected-no-diagnostics
diff --git a/clang/test/Analysis/z3/pretty-dump.c b/clang/test/Analysis/z3/pretty-dump.c
index 811da172e7490..e6cd6d83373b8 100644
--- a/clang/test/Analysis/z3/pretty-dump.c
+++ b/clang/test/Analysis/z3/pretty-dump.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-constraints=z3 -setup-static-analyzer \
+// RUN: %clang_analyze_cc1 -analyzer-constraints=z3 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection %s 2>&1 | FileCheck %s
 //
 // REQUIRES: z3
diff --git a/clang/test/C/C23/n3037.c b/clang/test/C/C23/n3037.c
index 966b583c9f376..ce6f4c4ea7acf 100644
--- a/clang/test/C/C23/n3037.c
+++ b/clang/test/C/C23/n3037.c
@@ -60,7 +60,7 @@ union purr { float y; int x; }; // c23-error {{type 'union purr' has incompatibl
 
 // The presence of an attribute makes two types not compatible.
 struct [[gnu::packed]] attr_test { // c17-note {{previous definition is here}} \
-                                      c23-note {{attribute 'packed' here}}
+                                      c23-note {{attribute 'gnu::packed' here}}
   int x;
 };
 
@@ -75,26 +75,26 @@ struct attr_test_2 { // c17-note {{previous definition is here}}
 
 struct [[gnu::packed]] attr_test_2 { // c17-error {{redefinition of 'attr_test_2'}} \
                                         c23-error {{type 'struct attr_test_2' has an attribute which currently causes the types to be treated as though they are incompatible}} \
-                                        c23-note {{attribute 'packed' here}}
+                                        c23-note {{attribute 'gnu::packed' here}}
   int x;
 };
 
 // This includes the same attribute on both types.
 struct [[gnu::packed]] attr_test_3 { // c17-note {{previous definition is here}} \
-                                       c23-note {{attribute 'packed' here}}
+                                       c23-note {{attribute 'gnu::packed' here}}
   int x;
 };
 
 struct [[gnu::packed]] attr_test_3 { // c17-error {{redefinition of 'attr_test_3'}} \
                                         c23-error {{type 'struct attr_test_3' has an attribute which currently causes the types to be treated as though they are incompatible}} \
-                                        c23-note {{attribute 'packed' here}}
+                                        c23-note {{attribute 'gnu::packed' here}}
   int x;
 };
 
 // Everything which applies to the tag itself also applies to fields.
 struct field_attr_test_1 { // c17-note {{previous definition is here}}
   int x;
-  [[gnu::packed]] int y; // c23-note {{attribute 'packed' here}}
+  [[gnu::packed]] int y; // c23-note {{attribute 'gnu::packed' here}}
 };
 
 struct field_attr_test_1 { // c17-error {{redefinition of 'field_attr_test_1'}} \
@@ -104,7 +104,7 @@ struct field_attr_test_1 { // c17-error {{redefinition of 'field_attr_test_1'}}
 };
 
 struct field_attr_test_2 { // c17-note {{previous definition is here}}
-  [[gnu::packed]] int x; // c23-note {{attribute 'packed' here}}
+  [[gnu::packed]] int x; // c23-note {{attribute 'gnu::packed' here}}
   int y;
 };
 
@@ -115,13 +115,13 @@ struct field_attr_test_2 { // c17-error {{redefinition of 'field_attr_test_2'}}
 };
 
 struct field_attr_test_3 { // c17-note {{previous definition is here}}
-  [[gnu::packed]] int x;   // c23-note {{attribute 'packed' here}}
+  [[gnu::packed]] int x;   // c23-note {{attribute 'gnu::packed' here}}
   int y;
 };
 
 struct field_attr_test_3 { // c17-error {{redefinition of 'field_attr_test_3'}} \
                               c23-error {{type 'struct field_attr_test_3' has a member with an attribute which currently causes the types to be treated as though they are incompatible}}
-  int x [[gnu::packed]];   // c23-note {{attribute 'packed' here}}
+  int x [[gnu::packed]];   // c23-note {{attribute 'gnu::packed' here}}
   int y;
 };
 
diff --git a/clang/test/CIR/CodeGen/assign-operator.cpp b/clang/test/CIR/CodeGen/assign-operator.cpp
new file mode 100644
index 0000000000000..3e509f59368b6
--- /dev/null
+++ b/clang/test/CIR/CodeGen/assign-operator.cpp
@@ -0,0 +1,144 @@
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-none-linux-android21 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-none-linux-android21 -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-none-linux-android21 -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+class x {
+public: int operator=(int);
+};
+void a() {
+  x a;
+  a = 1u;
+}
+
+// CIR: cir.func private @_ZN1xaSEi(!cir.ptr<!rec_x>, !s32i)
+// CIR: cir.func{{.*}} @_Z1av()
+// CIR:   %[[A_ADDR:.*]] = cir.alloca !rec_x, !cir.ptr<!rec_x>, ["a"]
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !u32i
+// CIR:   %[[ONE_CAST:.*]] = cir.cast(integral, %[[ONE]] : !u32i), !s32i
+// CIR:   %[[RET:.*]] = cir.call @_ZN1xaSEi(%[[A_ADDR]], %[[ONE_CAST]]) : (!cir.ptr<!rec_x>, !s32i) -> !s32i
+
+// LLVM: define{{.*}} @_Z1av()
+// OGCG: define{{.*}} @_Z1av()
+
+void f(int i, int j) {
+  (i += j) = 17;
+}
+
+// CIR: cir.func{{.*}} @_Z1fii(%arg0: !s32i {{.*}}, %arg1: !s32i {{.*}})
+// CIR:   %[[I_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init]
+// CIR:   %[[J_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["j", init]
+// CIR:   cir.store %arg0, %[[I_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store %arg1, %[[J_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[SEVENTEEN:.*]] = cir.const #cir.int<17> : !s32i
+// CIR:   %[[J_LOAD:.*]] = cir.load align(4) %[[J_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   %[[I_LOAD:.*]] = cir.load align(4) %[[I_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   %[[ADD:.*]] = cir.binop(add, %[[I_LOAD]], %[[J_LOAD]]) nsw : !s32i
+// CIR:   cir.store align(4) %[[ADD]], %[[I_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store align(4) %[[SEVENTEEN]], %[[I_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.return
+
+// Ensure that we use memcpy when we would have selected a trivial assignment
+// operator, even for a non-trivially-copyable type.
+struct A {
+  A &operator=(const A&);
+};
+struct B {
+  B(const B&);
+  B &operator=(const B&) = default;
+  int n;
+};
+struct C {
+  A a;
+  B b[16];
+};
+void copy_c(C &c1, C &c2) {
+  c1 = c2;
+}
+
+// CIR: cir.func private @_ZN1AaSERKS_(!cir.ptr<!rec_A>, !cir.ptr<!rec_A>) -> !cir.ptr<!rec_A>
+// CIR: cir.func private @memcpy(!cir.ptr<!void>, !cir.ptr<!void>, !u64i) -> !cir.ptr<!void>
+
+// Implicit assignment operator for C.
+
+// CIR: cir.func comdat linkonce_odr @_ZN1CaSERKS_(%arg0: !cir.ptr<!rec_C> {{.*}}, %arg1: !cir.ptr<!rec_C> {{.*}}) -> !cir.ptr<!rec_C>
+// CIR:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["this", init]
+// CIR:   %[[ARG1_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["", init, const]
+// CIR:   %[[RET_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["__retval"]
+// CIR:   cir.store %arg0, %[[THIS_ADDR]]
+// CIR:   cir.store %arg1, %[[ARG1_ADDR]]
+// CIR:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
+// CIR:   %[[A_MEMBER:.*]] = cir.get_member %[[THIS]][0] {name = "a"}
+// CIR:   %[[ARG1_LOAD:.*]] = cir.load{{.*}} %[[ARG1_ADDR]]
+// CIR:   %[[A_MEMBER_2:.*]] = cir.get_member %[[ARG1_LOAD]][0] {name = "a"}
+// CIR:   %[[C_A:.*]] = cir.call @_ZN1AaSERKS_(%[[A_MEMBER]], %[[A_MEMBER_2]])
+// CIR:   %[[B_MEMBER:.*]] = cir.get_member %[[THIS]][1] {name = "b"}
+// CIR:   %[[B_VOID_PTR:.*]] = cir.cast(bitcast, %[[B_MEMBER]] : !cir.ptr<!cir.array<!rec_B x 16>>), !cir.ptr<!void>
+// CIR:   %[[RET_LOAD:.*]] = cir.load %[[ARG1_ADDR]]
+// CIR:   %[[B_MEMBER_2:.*]] = cir.get_member %[[RET_LOAD]][1] {name = "b"}
+// CIR:   %[[B_VOID_PTR_2:.*]] = cir.cast(bitcast, %[[B_MEMBER_2]] : !cir.ptr<!cir.array<!rec_B x 16>>), !cir.ptr<!void>
+// CIR:   %[[SIZE:.*]] = cir.const #cir.int<64> : !u64i
+// CIR:   %[[COUNT:.*]] = cir.call @memcpy(%[[B_VOID_PTR]], %[[B_VOID_PTR_2]], %[[SIZE]])
+// CIR:   cir.store %[[THIS]], %[[RET_ADDR]]
+// CIR:   %[[RET_VAL:.*]] = cir.load{{.*}} %[[RET_ADDR]]
+// CIR:   cir.return %[[RET_VAL]]
+
+// CIR: cir.func{{.*}} @_Z6copy_cR1CS0_(%arg0: !cir.ptr<!rec_C> {{.*}}, %arg1: !cir.ptr<!rec_C> {{.*}})
+// CIR:   %[[C1_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["c1", init, const]
+// CIR:   %[[C2_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["c2", init, const]
+// CIR:   cir.store %arg0, %[[C1_ADDR]]
+// CIR:   cir.store %arg1, %[[C2_ADDR]]
+// CIR:   %[[C2_LOAD:.*]] = cir.load{{.*}} %[[C2_ADDR]]
+// CIR:   %[[C1_LOAD:.*]] = cir.load{{.*}} %[[C1_ADDR]]
+// CIR:   %[[RET:.*]] = cir.call @_ZN1CaSERKS_(%[[C1_LOAD]], %[[C2_LOAD]])
+
+struct D {
+  D &operator=(const D&);
+};
+struct E {
+  D &get_d_ref() { return d; }
+private:
+  D d;
+};
+
+void copy_ref_to_ref(E &e1, E &e2) {
+  e1.get_d_ref() = e2.get_d_ref();
+}
+
+// The call to e2.get_d_ref() must occur before the call to e1.get_d_ref().
+
+// CIR: cir.func{{.*}} @_Z15copy_ref_to_refR1ES0_(%arg0: !cir.ptr<!rec_E> {{.*}}, %arg1: !cir.ptr<!rec_E> {{.*}})
+// CIR:   %[[E1_ADDR:.*]] = cir.alloca !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>, ["e1", init, const]
+// CIR:   %[[E2_ADDR:.*]] = cir.alloca !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>, ["e2", init, const]
+// CIR:   cir.store %arg0, %[[E1_ADDR]] : !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>
+// CIR:   cir.store %arg1, %[[E2_ADDR]] : !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>
+// CIR:   %[[E2:.*]] = cir.load %[[E2_ADDR]]
+// CIR:   %[[D2_REF:.*]] = cir.call @_ZN1E9get_d_refEv(%[[E2]])
+// CIR:   %[[E1:.*]] = cir.load %[[E1_ADDR]]
+// CIR:   %[[D1_REF:.*]] = cir.call @_ZN1E9get_d_refEv(%[[E1]])
+// CIR:   %[[D1_REF_2:.*]] = cir.call @_ZN1DaSERKS_(%[[D1_REF]], %[[D2_REF]])
+// CIR:   cir.return
+
+// LLVM: define{{.*}} void @_Z15copy_ref_to_refR1ES0_(ptr %[[ARG0:.*]], ptr %[[ARG1:.*]]) {
+// LLVM:   %[[E1_ADDR:.*]] = alloca ptr
+// LLVM:   %[[E2_ADDR:.*]] = alloca ptr
+// LLVM:   store ptr %[[ARG0]], ptr %[[E1_ADDR]]
+// LLVM:   store ptr %[[ARG1]], ptr %[[E2_ADDR]]
+// LLVM:   %[[E2:.*]] = load ptr, ptr %[[E2_ADDR]]
+// LLVM:   %[[D2_REF:.*]] = call ptr @_ZN1E9get_d_refEv(ptr %[[E2]])
+// LLVM:   %[[E1:.*]] = load ptr, ptr %[[E1_ADDR]]
+// LLVM:   %[[D1_REF:.*]] = call ptr @_ZN1E9get_d_refEv(ptr %[[E1]])
+// LLVM:   %[[D1_REF_2:.*]] = call ptr @_ZN1DaSERKS_(ptr %[[D1_REF]], ptr %[[D2_REF]])
+
+// OGCG: define{{.*}} void @_Z15copy_ref_to_refR1ES0_(ptr{{.*}} %[[ARG0:.*]], ptr{{.*}} %[[ARG1:.*]])
+// OGCG:   %[[E1_ADDR:.*]] = alloca ptr
+// OGCG:   %[[E2_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[ARG0]], ptr %[[E1_ADDR]]
+// OGCG:   store ptr %[[ARG1]], ptr %[[E2_ADDR]]
+// OGCG:   %[[E2:.*]] = load ptr, ptr %[[E2_ADDR]]
+// OGCG:   %[[D2_REF:.*]] = call{{.*}} ptr @_ZN1E9get_d_refEv(ptr{{.*}} %[[E2]])
+// OGCG:   %[[E1:.*]] = load ptr, ptr %[[E1_ADDR]]
+// OGCG:   %[[D1_REF:.*]] = call{{.*}} ptr @_ZN1E9get_d_refEv(ptr{{.*}} %[[E1]])
+// OGCG:   %[[D1_REF_2:.*]] = call{{.*}} ptr @_ZN1DaSERKS_(ptr{{.*}} %[[D1_REF]], ptr{{.*}} %[[D2_REF]])
diff --git a/clang/test/CIR/CodeGen/bitfields.c b/clang/test/CIR/CodeGen/bitfields.c
index ff5c6bc1787b4..6eb753c5cc3d2 100644
--- a/clang/test/CIR/CodeGen/bitfields.c
+++ b/clang/test/CIR/CodeGen/bitfields.c
@@ -13,6 +13,7 @@ typedef struct {
 } A;
 
 // CIR-DAG:  !rec_A = !cir.record<struct "A" packed padded {!s8i, !s8i, !s8i, !u16i, !cir.array<!u8i x 3>}>
+// CIR-DAG:  #bfi_more_bits = #cir.bitfield_info<name = "more_bits", storageType = !u16i, size = 4, offset = 3, isSigned = false>
 // LLVM-DAG: %struct.A = type <{ i8, i8, i8, i16, [3 x i8] }>
 // OGCG-DAG: %struct.A = type <{ i8, i8, i8, i16, [3 x i8] }>
 
@@ -34,6 +35,7 @@ typedef struct {
   int e : 15;
   unsigned f; // type other than int above, not a bitfield
 } S;
+// CIR-DAG:  #bfi_c = #cir.bitfield_info<name = "c", storageType = !u64i, size = 17, offset = 32, isSigned = true>
 // CIR-DAG:  !rec_S = !cir.record<struct "S" {!u64i, !u16i, !u32i}>
 // LLVM-DAG: %struct.S = type { i64, i16, i32 }
 // OGCG-DAG: %struct.S = type { i64, i16, i32 }
@@ -76,3 +78,59 @@ void def() {
   T t;
   U u;
 }
+
+int load_field(S* s) {
+  return s->c;
+}
+
+// CIR: cir.func {{.*@load_field}}
+// CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["s", init]
+// CIR:   [[TMP1:%.*]] = cir.load{{.*}} [[TMP0]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+// CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][0] {name = "c"} : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+// CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_c, [[TMP2]] : !cir.ptr<!u64i>) -> !s32i
+
+// LLVM: define dso_local i32 @load_field
+// LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+// LLVM:   [[TMP1:%.*]] = alloca i32, i64 1, align 4
+// LLVM:   [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+// LLVM:   [[TMP3:%.*]] = getelementptr %struct.S, ptr [[TMP2]], i32 0, i32 0
+// LLVM:   [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+// LLVM:   [[TMP5:%.*]] = shl i64 [[TMP4]], 15
+// LLVM:   [[TMP6:%.*]] = ashr i64 [[TMP5]], 47
+// LLVM:   [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+
+// OGCG: define dso_local i32 @load_field
+// OGCG:  [[TMP0:%.*]] = alloca ptr, align 8
+// OGCG:  [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// OGCG:  [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
+// OGCG:  [[TMP3:%.*]] = shl i64 [[TMP2]], 15
+// OGCG:  [[TMP4:%.*]] = ashr i64 [[TMP3]], 47
+// OGCG:  [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+
+unsigned int load_field_unsigned(A* s) {
+  return s->more_bits;
+}
+
+//CIR: cir.func dso_local @load_field_unsigned
+//CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_A>, !cir.ptr<!cir.ptr<!rec_A>>, ["s", init] {alignment = 8 : i64}
+//CIR:   [[TMP1:%.*]] = cir.load align(8) [[TMP0]] : !cir.ptr<!cir.ptr<!rec_A>>, !cir.ptr<!rec_A>
+//CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][3] {name = "more_bits"} : !cir.ptr<!rec_A> -> !cir.ptr<!u16i>
+//CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_more_bits, [[TMP2]] : !cir.ptr<!u16i>) -> !u32i
+
+//LLVM: define dso_local i32 @load_field_unsigned
+//LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+//LLVM:   [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+//LLVM:   [[TMP2:%.*]] = getelementptr %struct.A, ptr [[TMP1]], i32 0, i32 3
+//LLVM:   [[TMP3:%.*]] = load i16, ptr [[TMP2]], align 2
+//LLVM:   [[TMP4:%.*]] = lshr i16 [[TMP3]], 3
+//LLVM:   [[TMP5:%.*]] = and i16 [[TMP4]], 15
+//LLVM:   [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
+
+//OGCG: define dso_local i32 @load_field_unsigned
+//OGCG:   [[TMP0:%.*]] = alloca ptr, align 8
+//OGCG:   [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+//OGCG:   [[TMP2:%.*]] = getelementptr inbounds nuw %struct.A, ptr [[TMP1]], i32 0, i32 3
+//OGCG:   [[TMP3:%.*]] = load i16, ptr [[TMP2]], align 1
+//OGCG:   [[TMP4:%.*]] = lshr i16 [[TMP3]], 3
+//OGCG:   [[TMP5:%.*]] = and i16 [[TMP4]], 15
+//OGCG:   [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
diff --git a/clang/test/CIR/CodeGen/bitfields.cpp b/clang/test/CIR/CodeGen/bitfields.cpp
index 762d249884741..a4d58b5cadcec 100644
--- a/clang/test/CIR/CodeGen/bitfields.cpp
+++ b/clang/test/CIR/CodeGen/bitfields.cpp
@@ -14,6 +14,7 @@ typedef struct {
   unsigned f; // type other than int above, not a bitfield
 } S;
 // CIR-DAG:  !rec_S = !cir.record<struct "S" {!u64i, !u16i, !u32i}>
+// CIR-DAG:  #bfi_c = #cir.bitfield_info<name = "c", storageType = !u64i, size = 17, offset = 32, isSigned = true>
 // LLVM-DAG: %struct.S = type { i64, i16, i32 }
 // OGCG-DAG: %struct.S = type { i64, i16, i32 }
 
@@ -30,3 +31,30 @@ void def() {
   S s;
   T t;
 }
+
+int load_field(S* s) {
+  return s->c;
+}
+// CIR: cir.func dso_local @_Z10load_field
+// CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["s", init]
+// CIR:   [[TMP1:%.*]] = cir.load{{.*}} [[TMP0]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+// CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][0] {name = "c"} : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+// CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_c, [[TMP2]] : !cir.ptr<!u64i>) -> !s32i
+
+// LLVM: define dso_local i32 @_Z10load_fieldP1S
+// LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+// LLVM:   [[TMP1:%.*]] = alloca i32, i64 1, align 4
+// LLVM:   [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+// LLVM:   [[TMP3:%.*]] = getelementptr %struct.S, ptr [[TMP2]], i32 0, i32 0
+// LLVM:   [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+// LLVM:   [[TMP5:%.*]] = shl i64 [[TMP4]], 15
+// LLVM:   [[TMP6:%.*]] = ashr i64 [[TMP5]], 47
+// LLVM:   [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+
+// OGCG: define dso_local noundef i32 @_Z10load_fieldP1S
+// OGCG:  [[TMP0:%.*]] = alloca ptr, align 8
+// OGCG:  [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// OGCG:  [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
+// OGCG:  [[TMP3:%.*]] = shl i64 [[TMP2]], 15
+// OGCG:  [[TMP4:%.*]] = ashr i64 [[TMP3]], 47
+// OGCG:  [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
diff --git a/clang/test/CIR/CodeGen/bitfields_be.c b/clang/test/CIR/CodeGen/bitfields_be.c
index 149e9c9ac33ff..e839bc2b9698d 100644
--- a/clang/test/CIR/CodeGen/bitfields_be.c
+++ b/clang/test/CIR/CodeGen/bitfields_be.c
@@ -10,8 +10,35 @@ typedef struct {
     int b : 11;
     int c : 17;
 } S;
-S s;
 
 // CIR:  !rec_S = !cir.record<struct "S" {!u32i}>
 // LLVM: %struct.S = type { i32 }
 // OGCG: %struct.S = type { i32 }
+void def() {
+  S s;
+}
+int init(S* s) {
+  return s->c;
+}
+
+//CIR: cir.func dso_local @init
+//CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["s", init] {alignment = 8 : i64}
+//CIR:   [[TMP1:%.*]] = cir.load align(8) [[TMP0]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+//CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][0] {name = "c"} : !cir.ptr<!rec_S> -> !cir.ptr<!u32i>
+//CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_c, [[TMP2]] : !cir.ptr<!u32i>) -> !s32i
+
+//LLVM: define dso_local i32 @init(ptr %0) {
+//LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+//LLVM:   [[TMP1:%.*]] = alloca i32, i64 1, align 4
+//LLVM:   [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+//LLVM:   [[TMP3:%.*]] = getelementptr %struct.S, ptr [[TMP2]], i32 0, i32 0
+//LLVM:   [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+//LLVM:   [[TMP5:%.*]] = shl i32 [[TMP4]], 15
+//LLVM:   [[TMP6:%.*]] = ashr i32 [[TMP5]], 15
+
+//OGCG: define dso_local i32 @init
+//OGCG:   [[TMP0:%.*]] = alloca ptr, align 8
+//OGCG:   [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+//OGCG:   [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
+//OGCG:   [[TMP3:%.*]] = shl i32 [[TMP2]], 15
+//OGCG:   [[TMP4:%.*]] = ashr i32 [[TMP3]], 15
diff --git a/clang/test/CIR/CodeGen/builtin_bit.cpp b/clang/test/CIR/CodeGen/builtin_bit.cpp
new file mode 100644
index 0000000000000..ba56c91ce7401
--- /dev/null
+++ b/clang/test/CIR/CodeGen/builtin_bit.cpp
@@ -0,0 +1,327 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+int test_builtin_clrsb(int x) {
+  return __builtin_clrsb(x);
+}
+
+// CIR-LABEL: _Z18test_builtin_clrsbi
+// CIR:         [[TMP:%.+]] = cir.bit.clrsb(%{{.+}} : !s32i) : !s32i
+
+// LLVM-LABEL: _Z18test_builtin_clrsbi
+// LLVM:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[X_NEG:.+]] = icmp slt i32 %[[X]], 0
+// LLVM-NEXT:    %[[X_NOT:.+]] = xor i32 %[[X]], -1
+// LLVM-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i32 %[[X_NOT]], i32 %[[X]]
+// LLVM-NEXT:    %[[LZ:.+]] = call i32 @llvm.ctlz.i32(i32 %[[P]], i1 false)
+// LLVM-NEXT:    %{{.+}} = sub i32 %[[LZ]], 1
+
+// OGCG-LABEL: _Z18test_builtin_clrsbi
+// OGCG:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// OGCG-NEXT:    %[[X_NEG:.+]] = icmp slt i32 %[[X]], 0
+// OGCG-NEXT:    %[[X_NOT:.+]] = xor i32 %[[X]], -1
+// OGCG-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i32 %[[X_NOT]], i32 %[[X]]
+// OGCG-NEXT:    %[[LZ:.+]] = call i32 @llvm.ctlz.i32(i32 %[[P]], i1 false)
+// OGCG-NEXT:    %{{.+}} = sub i32 %[[LZ]], 1
+
+int test_builtin_clrsbl(long x) {
+  return __builtin_clrsbl(x);
+}
+
+// CIR-LABEL: _Z19test_builtin_clrsbll
+// CIR:         [[TMP:%.+]] = cir.bit.clrsb(%{{.+}} : !s64i) : !s64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !s64i), !s32i
+
+// LLVM-LABEL: _Z19test_builtin_clrsbll
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// LLVM-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// LLVM-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// LLVM-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// LLVM-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+// OGCG-LABEL: _Z19test_builtin_clrsbll
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// OGCG-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// OGCG-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// OGCG-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// OGCG-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+int test_builtin_clrsbll(long long x) {
+  return __builtin_clrsbll(x);
+}
+
+// CIR-LABEL: _Z20test_builtin_clrsbllx
+// CIR:         [[TMP:%.+]] = cir.bit.clrsb(%{{.+}} : !s64i) : !s64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !s64i), !s32i
+
+// LLVM-LABEL: _Z20test_builtin_clrsbllx
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// LLVM-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// LLVM-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// LLVM-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// LLVM-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+// OGCG-LABEL: _Z20test_builtin_clrsbllx
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// OGCG-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// OGCG-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// OGCG-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// OGCG-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+int test_builtin_ctzs(unsigned short x) {
+  return __builtin_ctzs(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_ctzst
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u16i) poison_zero : !u16i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u16i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_ctzst
+// LLVM:         %{{.+}} = call i16 @llvm.cttz.i16(i16 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_ctzst
+// OGCG:         %{{.+}} = call i16 @llvm.cttz.i16(i16 %{{.+}}, i1 true)
+
+int test_builtin_ctz(unsigned x) {
+  return __builtin_ctz(x);
+}
+
+// CIR-LABEL: _Z16test_builtin_ctzj
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z16test_builtin_ctzj
+// LLVM:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z16test_builtin_ctzj
+// OGCG:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_ctzl(unsigned long x) {
+  return __builtin_ctzl(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_ctzlm
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_ctzlm
+// LLVM:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_ctzlm
+// OGCG:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_ctzll(unsigned long long x) {
+  return __builtin_ctzll(x);
+}
+
+// CIR-LABEL: _Z18test_builtin_ctzlly
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z18test_builtin_ctzlly
+// LLVM:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z18test_builtin_ctzlly
+// OGCG:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_ctzg(unsigned x) {
+  return __builtin_ctzg(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_ctzgj
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_ctzgj
+// LLVM:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_ctzgj
+// OGCG:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_clzs(unsigned short x) {
+  return __builtin_clzs(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_clzst
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u16i) poison_zero : !u16i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u16i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_clzst
+// LLVM:         %{{.+}} = call i16 @llvm.ctlz.i16(i16 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_clzst
+// OGCG:         %{{.+}} = call i16 @llvm.ctlz.i16(i16 %{{.+}}, i1 true)
+
+int test_builtin_clz(unsigned x) {
+  return __builtin_clz(x);
+}
+
+// CIR-LABEL: _Z16test_builtin_clzj
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z16test_builtin_clzj
+// LLVM:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z16test_builtin_clzj
+// OGCG:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_clzl(unsigned long x) {
+  return __builtin_clzl(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_clzlm
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_clzlm
+// LLVM:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_clzlm
+// OGCG:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_clzll(unsigned long long x) {
+  return __builtin_clzll(x);
+}
+
+// CIR-LABEL: _Z18test_builtin_clzlly
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z18test_builtin_clzlly
+// LLVM:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z18test_builtin_clzlly
+// OGCG:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_clzg(unsigned x) {
+  return __builtin_clzg(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_clzgj
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_clzgj
+// LLVM:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_clzgj
+// OGCG:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_parity(unsigned x) {
+  return __builtin_parity(x);
+}
+
+// CIR-LABEL: _Z19test_builtin_parityj
+// CIR:         [[TMP:%.+]] = cir.bit.parity(%{{.+}} : !u32i) : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z19test_builtin_parityj
+// LLVM:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[POPCNT:.+]] = call i32 @llvm.ctpop.i32(i32 %[[X]])
+// LLVM-NEXT:    %{{.+}} = and i32 %[[POPCNT]], 1
+
+// OGCG-LABEL: _Z19test_builtin_parityj
+// OGCG:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// OGCG-NEXT:    %[[POPCNT:.+]] = call i32 @llvm.ctpop.i32(i32 %[[X]])
+// OGCG-NEXT:    %{{.+}} = and i32 %[[POPCNT]], 1
+
+int test_builtin_parityl(unsigned long x) {
+  return __builtin_parityl(x);
+}
+
+// CIR-LABEL: _Z20test_builtin_paritylm
+// CIR:         [[TMP:%.+]] = cir.bit.parity(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z20test_builtin_paritylm
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// LLVM-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+// OGCG-LABEL: _Z20test_builtin_paritylm
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// OGCG-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+int test_builtin_parityll(unsigned long long x) {
+  return __builtin_parityll(x);
+}
+
+// CIR-LABEL: _Z21test_builtin_paritylly
+// CIR:         [[TMP:%.+]] = cir.bit.parity(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z21test_builtin_paritylly
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// LLVM-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+// OGCG-LABEL: _Z21test_builtin_paritylly
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// OGCG-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+int test_builtin_popcount(unsigned x) {
+  return __builtin_popcount(x);
+}
+
+// CIR-LABEL: _Z21test_builtin_popcountj
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u32i) : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z21test_builtin_popcountj
+// LLVM:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
+
+// OGCG-LABEL: _Z21test_builtin_popcountj
+// OGCG:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
+
+int test_builtin_popcountl(unsigned long x) {
+  return __builtin_popcountl(x);
+}
+
+// CIR-LABEL: _Z22test_builtin_popcountlm
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z22test_builtin_popcountlm
+// LLVM:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+// OGCG-LABEL: _Z22test_builtin_popcountlm
+// OGCG:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+int test_builtin_popcountll(unsigned long long x) {
+  return __builtin_popcountll(x);
+}
+
+// CIR-LABEL: _Z23test_builtin_popcountlly
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z23test_builtin_popcountlly
+// LLVM:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+// OGCG-LABEL: _Z23test_builtin_popcountlly
+// OGCG:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+int test_builtin_popcountg(unsigned x) {
+  return __builtin_popcountg(x);
+}
+
+// CIR-LABEL: _Z22test_builtin_popcountgj
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u32i) : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z22test_builtin_popcountgj
+// LLVM:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
+
+// OGCG-LABEL: _Z22test_builtin_popcountgj
+// OGCG:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
diff --git a/clang/test/CIR/CodeGen/builtin_call.cpp b/clang/test/CIR/CodeGen/builtin_call.cpp
index b956f2580593e..ad0e478040836 100644
--- a/clang/test/CIR/CodeGen/builtin_call.cpp
+++ b/clang/test/CIR/CodeGen/builtin_call.cpp
@@ -84,8 +84,8 @@ void library_builtins() {
 
 // CIR: cir.func{{.*}} @_Z16library_builtinsv() {
 // CIR: %[[NULL:.+]] = cir.const #cir.ptr<null> : !cir.ptr<!s8i>
-// CIR: cir.call @printf(%[[NULL]]) : (!cir.ptr<!s8i>) -> !s32i
-// CIR: cir.call @abort() : () -> ()
+// CIR: cir.call @printf(%[[NULL]]) nothrow : (!cir.ptr<!s8i>) -> !s32i
+// CIR: cir.call @abort() nothrow : () -> ()
 
 // LLVM: define{{.*}} void @_Z16library_builtinsv()
 // LLVM: call i32 (ptr, ...) @printf(ptr null)
diff --git a/clang/test/CIR/CodeGen/builtin_printf.cpp b/clang/test/CIR/CodeGen/builtin_printf.cpp
index d12f822d43ebf..43128e44b19e4 100644
--- a/clang/test/CIR/CodeGen/builtin_printf.cpp
+++ b/clang/test/CIR/CodeGen/builtin_printf.cpp
@@ -26,16 +26,16 @@ void func(char const * const str, int i) {
 // CIR:   cir.store %[[arg0]], %[[str_ptr]] : !cir.ptr<!s8i>, !cir.ptr<!cir.ptr<!s8i>>
 // CIR:   cir.store %[[arg1]], %[[i_ptr]] : !s32i, !cir.ptr<!s32i>
 // CIR:   %[[null_ptr:.+]] = cir.const #cir.ptr<null> : !cir.ptr<!s8i>
-// CIR:   %[[printf_result1:.+]] = cir.call @printf(%[[null_ptr]]) : (!cir.ptr<!s8i>) -> !s32i
+// CIR:   %[[printf_result1:.+]] = cir.call @printf(%[[null_ptr]]) nothrow : (!cir.ptr<!s8i>) -> !s32i
 // CIR:   %[[str_fmt_global:.+]] = cir.get_global @".str" : !cir.ptr<!cir.array<!s8i x 3>>
 // CIR:   %[[str_fmt_ptr:.+]] = cir.cast(array_to_ptrdecay, %[[str_fmt_global]] : !cir.ptr<!cir.array<!s8i x 3>>), !cir.ptr<!s8i>
 // CIR:   %[[str_val:.+]] = cir.load{{.*}} %[[str_ptr]] : !cir.ptr<!cir.ptr<!s8i>>, !cir.ptr<!s8i>
-// CIR:   %[[printf_result2:.+]] = cir.call @printf(%[[str_fmt_ptr]], %[[str_val]]) : (!cir.ptr<!s8i>, !cir.ptr<!s8i>) -> !s32i
+// CIR:   %[[printf_result2:.+]] = cir.call @printf(%[[str_fmt_ptr]], %[[str_val]]) nothrow : (!cir.ptr<!s8i>, !cir.ptr<!s8i>) -> !s32i
 // CIR:   %[[full_fmt_global:.+]] = cir.get_global @".str.1" : !cir.ptr<!cir.array<!s8i x 7>>
 // CIR:   %[[full_fmt_ptr:.+]] = cir.cast(array_to_ptrdecay, %[[full_fmt_global]] : !cir.ptr<!cir.array<!s8i x 7>>), !cir.ptr<!s8i>
 // CIR:   %[[str_val2:.+]] = cir.load{{.*}} %[[str_ptr]] : !cir.ptr<!cir.ptr<!s8i>>, !cir.ptr<!s8i>
 // CIR:   %[[i_val:.+]] = cir.load{{.*}} %[[i_ptr]] : !cir.ptr<!s32i>, !s32i
-// CIR:   %[[printf_result3:.+]] = cir.call @printf(%[[full_fmt_ptr]], %[[str_val2]], %[[i_val]]) : (!cir.ptr<!s8i>, !cir.ptr<!s8i>, !s32i) -> !s32i
+// CIR:   %[[printf_result3:.+]] = cir.call @printf(%[[full_fmt_ptr]], %[[str_val2]], %[[i_val]]) nothrow : (!cir.ptr<!s8i>, !cir.ptr<!s8i>, !s32i) -> !s32i
 // CIR:   cir.return
 
 // LLVM: define{{.*}} void @_Z4funcPKci(ptr %[[arg0:.+]], i32 %[[arg1:.+]])
diff --git a/clang/test/CIR/CodeGen/call.cpp b/clang/test/CIR/CodeGen/call.cpp
index f7653ed7a572b..43e5d150a949f 100644
--- a/clang/test/CIR/CodeGen/call.cpp
+++ b/clang/test/CIR/CodeGen/call.cpp
@@ -102,3 +102,18 @@ void f12() {
 // LLVM:         %[[#slot:]] = alloca %struct.S, i64 1, align 4
 // LLVM-NEXT:    %[[#ret:]] = call %struct.S @_Z3f10v()
 // LLVM-NEXT:    store %struct.S %[[#ret]], ptr %[[#slot]], align 4
+
+void f13() noexcept;
+void f14() {
+  f13();
+}
+
+// CIR-LABEL: cir.func{{.+}} @_Z3f14v()
+// CIR:         cir.call @_Z3f13v() nothrow : () -> ()
+// CIR:       }
+
+// LLVM-LABEL: define{{.+}} void @_Z3f14v()
+// LLVM:         call void @_Z3f13v() #[[LLVM_ATTR_0:.+]]
+// LLVM:       }
+
+// LLLVM: attributes #[[LLVM_ATTR_0]] = { nounwind }
diff --git a/clang/test/CIR/CodeGen/complex.cpp b/clang/test/CIR/CodeGen/complex.cpp
index 1e9ce0e29fd46..6e7e889df146f 100644
--- a/clang/test/CIR/CodeGen/complex.cpp
+++ b/clang/test/CIR/CodeGen/complex.cpp
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
 // RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
 // RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
 
 int _Complex ci;
@@ -376,7 +376,7 @@ bool foo18(int _Complex a, int _Complex b) {
 
 // CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
 // CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
-// CIR: %[[RESULT:.*]] = cir.complex.eq %[[COMPLEX_A]], %[[COMPLEX_B]] : !cir.complex<!s32i>
+// CIR: %[[RESULT:.*]] = cir.cmp(eq, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!s32i>, !cir.bool
 
 // LLVM: %[[COMPLEX_A:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
 // LLVM: %[[COMPLEX_B:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
@@ -408,7 +408,8 @@ bool foo19(double _Complex a, double _Complex b) {
 
 // CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
 // CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
-// CIR: %[[RESULT:.*]] = cir.complex.eq %[[COMPLEX_A]], %[[COMPLEX_B]] : !cir.complex<!cir.double>
+// CIR: %[[RESULT:.*]] = cir.cmp(eq, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!cir.double>, !cir.bool
+
 
 // LLVM: %[[COMPLEX_A:.*]] = load { double, double }, ptr {{.*}}, align 8
 // LLVM: %[[COMPLEX_B:.*]] = load { double, double }, ptr {{.*}}, align 8
@@ -442,3 +443,312 @@ bool foo19(double _Complex a, double _Complex b) {
 // OGCG: %[[CMP_IMAG:.*]] = fcmp oeq double %[[A_IMAG]], %[[B_IMAG]]
 // OGCG: %[[RESULT:.*]] = and i1 %[[CMP_REAL]], %[[CMP_IMAG]]
 
+
+bool foo20(int _Complex a, int _Complex b) {
+  return a != b;
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[RESULT:.*]] = cir.cmp(ne, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!s32i>, !cir.bool
+
+// LLVM: %[[COMPLEX_A:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[COMPLEX_B:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 1
+// LLVM: %[[CMP_REAL:.*]] = icmp ne i32 %[[A_REAL]], %[[B_REAL]]
+// LLVM: %[[CMP_IMAG:.*]] = icmp ne i32 %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[CMP_REAL:.*]] = icmp ne i32 %[[A_REAL]], %[[B_REAL]]
+// OGCG: %[[CMP_IMAG:.*]] = icmp ne i32 %[[A_IMAG]], %[[B_IMAG]]
+// OGCG: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+bool foo21(double _Complex a, double _Complex b) {
+  return a != b;
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[RESULT:.*]] = cir.cmp(ne, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!cir.double>, !cir.bool
+
+// LLVM: %[[COMPLEX_A:.*]] = load { double, double }, ptr {{.*}}, align 8
+// LLVM: %[[COMPLEX_B:.*]] = load { double, double }, ptr {{.*}}, align 8
+// LLVM: %[[A_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { double, double } %[[COMPLEX_B]], 1
+// LLVM: %[[CMP_REAL:.*]] = fcmp une double %[[A_REAL]], %[[B_REAL]]
+// LLVM: %[[CMP_IMAG:.*]] = fcmp une double %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { double, double }, align 8
+// OGCG: %[[COMPLEX_B:.*]] = alloca { double, double }, align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: store double {{.*}}, ptr %[[A_REAL_PTR]], align 8
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: store double {{.*}}, ptr %[[A_IMAG_PTR]], align 8
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: store double {{.*}}, ptr %[[B_REAL_PTR]], align 8
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: store double {{.*}}, ptr %[[B_IMAG_PTR]], align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load double, ptr %[[A_REAL_PTR]], align 8
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load double, ptr %[[A_IMAG_PTR]], align 8
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load double, ptr %[[B_REAL_PTR]], align 8
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load double, ptr %[[B_IMAG_PTR]], align 8
+// OGCG: %[[CMP_REAL:.*]] = fcmp une double %[[A_REAL]], %[[B_REAL]]
+// OGCG: %[[CMP_IMAG:.*]] = fcmp une double %[[A_IMAG]], %[[B_IMAG]]
+// OGCG: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+void foo22(int _Complex a, int _Complex b) {
+  int _Complex c = (a, b);
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP_B]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM: store { i32, i32 } %[[TMP_B]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG: store i32 %[[B_REAL]], ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG: store i32 %[[B_IMAG]], ptr %[[RESULT_IMAG_PTR]], align 4
+
+void foo23(int _Complex a, int _Complex b) {
+  float _Complex f;
+  int _Complex c = _Generic(a, int _Complex: b, default: f);
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[COMPLEX_F:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["f"]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_F:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM: store { i32, i32 } %[[TMP]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_F:.*]] = alloca { float, float }, align 4
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG: store i32 %[[B_REAL]], ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG: store i32 %[[B_IMAG]], ptr %[[RESULT_IMAG_PTR]], align 4
+
+void foo24() {
+  int _Complex arr[2];
+  int _Complex r = arr[1];
+}
+
+// CIR: %[[ARR:.*]] = cir.alloca !cir.array<!cir.complex<!s32i> x 2>, !cir.ptr<!cir.array<!cir.complex<!s32i> x 2>>, ["arr"]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["r", init]
+// CIR: %[[IDX:.*]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[ARR_PTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARR]] : !cir.ptr<!cir.array<!cir.complex<!s32i> x 2>>), !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[RESULT_VAL:.*]] = cir.ptr_stride(%[[ARR_PTR]] : !cir.ptr<!cir.complex<!s32i>>, %[[IDX]] : !s32i), !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[RESULT_VAL]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[ARR:.*]] = alloca [2 x { i32, i32 }], i64 1, align 16
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[ARR_PTR:.*]] = getelementptr { i32, i32 }, ptr %[[ARR]], i32 0
+// LLVM: %[[RESULT_VAL:.*]] = getelementptr { i32, i32 }, ptr %[[ARR_PTR]], i64 1
+// LLVM: %[[TMP:.*]] = load { i32, i32 }, ptr %[[RESULT_VAL]], align 8
+// LLVM: store { i32, i32 } %[[TMP]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[ARR:.*]] = alloca [2 x { i32, i32 }], align 16
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[ELEM_PTR:.*]] = getelementptr inbounds [2 x { i32, i32 }], ptr %[[ARR]], i64 0, i64 1
+// OGCG: %[[ELEM_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[ELEM_PTR]], i32 0, i32 0
+// OGCG: %[[ELEM_REAL:.*]] = load i32, ptr %[[ELEM_REAL_PTR]]
+// OGCG: %[[ELEM_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[ELEM_PTR]], i32 0, i32 1
+// OGCG: %[[ELEM_IMAG:.*]] = load i32, ptr %[[ELEM_IMAG_PTR]]
+// OGCG: %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG: store i32 %[[ELEM_REAL]], ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG: store i32 %[[ELEM_IMAG]], ptr %[[RESULT_IMAG_PTR]], align 4
+
+template <double _Complex N> void template_foo() { double _Complex C = N; }
+
+void foo25() {
+  template_foo<__builtin_complex(1.0, 2.0)>();
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["C", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.fp<1.000000e+00> : !cir.double, #cir.fp<2.000000e+00> : !cir.double> : !cir.complex<!cir.double>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>
+
+// LLVM: %[[INIT:.*]] = alloca { double, double }, i64 1, align 8
+// LLVM: store { double, double } { double 1.000000e+00, double 2.000000e+00 }, ptr %[[INIT]], align 8
+
+// OGCG: %[[INIT:.*]] = alloca { double, double }, align 8
+// OGCG: %[[INIT_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[INIT]], i32 0, i32 0
+// OGCG: %[[INIT_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[INIT]], i32 0, i32 1
+// OGCG: store double 1.000000e+00, ptr %[[INIT_REAL_PTR]], align 8
+// OGCG: store double 2.000000e+00, ptr %[[INIT_IMAG_PTR]], align 8
+
+void foo26(int _Complex* a) {
+  int _Complex b = *a;
+}
+
+// CIR: %[[COMPLEX_A_PTR:.*]] = cir.alloca !cir.ptr<!cir.complex<!s32i>>, !cir.ptr<!cir.ptr<!cir.complex<!s32i>>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[COMPLEX_A:.*]] = cir.load deref {{.*}} %[[COMPLEX_A_PTR]] : !cir.ptr<!cir.ptr<!cir.complex<!s32i>>>, !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP]], %[[COMPLEX_B]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A_PTR:.*]] = alloca ptr, i64 1, align 8
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_A:.*]] = load ptr, ptr %[[COMPLEX_A_PTR]], align 8
+// LLVM: %[[TMP:.*]] = load { i32, i32 }, ptr %[[COMPLEX_A]], align 4
+// LLVM: store { i32, i32 } %[[TMP]], ptr %[[COMPLEX_B]], align 4
+
+// OGCG: %[[COMPLEX_A_PTR:.*]] = alloca ptr, align 8
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_A:.*]] = load ptr, ptr %[[COMPLEX_A_PTR]], align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: store i32 %[[A_REAL]], ptr %[[B_REAL_PTR]], align 4
+// OGCG: store i32 %[[A_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+void foo27(bool cond, int _Complex a, int _Complex b) {
+  int _Complex c = cond ? a : b;
+}
+
+// CIR: %[[COND:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["cond", init]
+// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[TMP_COND:.*]] = cir.load{{.*}} %[[COND]] : !cir.ptr<!cir.bool>, !cir.bool
+// CIR: %[[RESULT_VAL:.*]] = cir.ternary(%[[TMP_COND]], true {
+// CIR:   %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR:   cir.yield %[[TMP_A]] : !cir.complex<!s32i>
+// CIR: }, false {
+// CIR:   %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR:   cir.yield %[[TMP_B]] : !cir.complex<!s32i>
+// CIR: }) : (!cir.bool) -> !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[RESULT_VAL]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COND:.*]] = alloca i8, i64 1, align 1
+// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_COND:.*]] = load i8, ptr %[[COND]], align 1
+// LLVM: %[[COND_VAL:.*]] = trunc i8 %[[TMP_COND]] to i1
+// LLVM: br i1 %[[COND_VAL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVM: [[TRUE_BB]]:
+// LLVM:  %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[COMPLEX_A]], align 4
+// LLVM:  br label %[[END_BB:.*]]
+// LLVM: [[FALSE_BB]]:
+// LLVM:  %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM:  br label %[[END_BB]]
+// LLVM: [[END_BB]]:
+// LLVM: %[[RESULT_VAL:.*]] = phi { i32, i32 } [ %[[TMP_B]], %[[FALSE_BB]] ], [ %[[TMP_A]], %[[TRUE_BB]] ]
+// LLVM: store { i32, i32 } %[[RESULT_VAL]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COND:.*]] = alloca i8, align 1
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[TMP_COND:.*]] = load i8, ptr %[[COND]], align 1
+// OGCG: %[[COND_VAL:.*]] = trunc i8 %[[TMP_COND]] to i1
+// OGCG: br i1 %[[COND_VAL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// OGCG: [[TRUE_BB]]:
+// OGCG:  %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG:  %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG:  %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG:  %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG:  br label %[[END_BB:.*]]
+// OGCG: [[FALSE_BB]]:
+// OGCG:  %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG:  %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG:  %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG:  %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG:  br label %[[END_BB]]
+// OGCG: [[END_BB]]:
+// OGCG:  %[[REAL:.*]] = phi i32 [ %[[A_REAL]], %[[TRUE_BB]] ], [ %[[B_REAL]], %[[FALSE_BB]] ]
+// OGCG:  %[[IMAG:.*]] = phi i32 [ %[[A_IMAG]], %[[TRUE_BB]] ], [ %[[B_IMAG]], %[[FALSE_BB]] ]
+// OGCG:  %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG:  %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG:  store i32 %[[REAL]], ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG:  store i32 %[[IMAG]], ptr %[[RESULT_IMAG_PTR]], align 4
+
+void foo28() {
+  using IntComplex = int _Complex;
+  int _Complex a = IntComplex();
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.zero : !cir.complex<!s32i>
+// CIR: cir.store align(4) %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: store { i32, i32 } zeroinitializer, ptr %[[INIT]], align 4
+
+// OGCG: %[[INIT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[INIT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[INIT]], i32 0, i32 0
+// OGCG: %[[INIT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[INIT]], i32 0, i32 1
+// OGCG: store i32 0, ptr %[[INIT_REAL_PTR]], align 4
+// OGCG: store i32 0, ptr %[[INIT_IMAG_PTR]], align 4
+
+void foo29() {
+  using IntComplex = int _Complex;
+  int _Complex a = IntComplex{};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.int<0> : !s32i, #cir.int<0> : !s32i> : !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: store { i32, i32 } zeroinitializer, ptr %[[INIT]], align 4
+
+// OGCG: %[[INIT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[INIT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[INIT]], i32 0, i32 0
+// OGCG: %[[INIT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[INIT]], i32 0, i32 1
+// OGCG: store i32 0, ptr %[[INIT_REAL_PTR]], align 4
+// OGCG: store i32 0, ptr %[[INIT_IMAG_PTR]], align 4
\ No newline at end of file
diff --git a/clang/test/CIR/CodeGen/generic-selection.c b/clang/test/CIR/CodeGen/generic-selection.c
new file mode 100644
index 0000000000000..c1659382a7059
--- /dev/null
+++ b/clang/test/CIR/CodeGen/generic-selection.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void foo() {
+  int a;
+  int r = _Generic(a, double: 1, float: 2, int: 3, default: 4);
+}
+
+// CIR: %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a"]
+// CIR: %[[RES:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["r", init]
+// CIR: %[[RES_VAL:.*]] = cir.const #cir.int<3> : !s32i
+// CIR: cir.store{{.*}} %[[RES_VAL]], %[[RES]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: %[[A:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RES:.*]] = alloca i32, i64 1, align 4
+// LLVM: store i32 3, ptr %[[RES]], align 4
+
+// OGCG: %[[A:.*]] = alloca i32, align 4
+// OGCG: %[[RES:.*]] = alloca i32, align 4
+// OGCG: store i32 3, ptr %[[RES]], align 4
diff --git a/clang/test/CIR/CodeGen/non-type-template-param.cpp b/clang/test/CIR/CodeGen/non-type-template-param.cpp
new file mode 100644
index 0000000000000..8ef1d5785c700
--- /dev/null
+++ b/clang/test/CIR/CodeGen/non-type-template-param.cpp
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+template <const int N>
+void template_foo() {
+  int a = N + 5;
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
+// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[CONST_2:.*]] = cir.const #cir.int<5> : !s32i
+// CIR: %[[ADD:.*]] = cir.binop(add, %[[CONST_1]], %[[CONST_2]]) nsw : !s32i
+// CIR: cir.store{{.*}} %[[ADD]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: %[[INIT:.*]] = alloca i32, i64 1, align 4
+// LLVM: store i32 6, ptr %[[INIT]], align 4
+
+// OGCG: %[[INIT:.*]] = alloca i32, align 4
+// OGCG: store i32 6, ptr %[[INIT]], align 4
+
+void foo() {
+  template_foo<1>();
+}
diff --git a/clang/test/CIR/CodeGen/opt-info-attr.cpp b/clang/test/CIR/CodeGen/opt-info-attr.cpp
new file mode 100644
index 0000000000000..444286b8db8a9
--- /dev/null
+++ b/clang/test/CIR/CodeGen/opt-info-attr.cpp
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O0
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -O1 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O1
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -O2 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O2
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -O3 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O3
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Os -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-Os
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Oz -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-Oz
+
+void f() {}
+
+// CHECK-O0: module attributes
+// CHECK-O0-NOT: cir.opt_info
+// CHECK-O1: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 1, size = 0>{{.+}}
+// CHECK-O2: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 2, size = 0>{{.+}}
+// CHECK-O3: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 3, size = 0>{{.+}}
+// CHECK-Os: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 2, size = 1>{{.+}}
+// CHECK-Oz: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 2, size = 2>{{.+}}
diff --git a/clang/test/CIR/CodeGen/pack-indexing.cpp b/clang/test/CIR/CodeGen/pack-indexing.cpp
new file mode 100644
index 0000000000000..8b12b505164f8
--- /dev/null
+++ b/clang/test/CIR/CodeGen/pack-indexing.cpp
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++2c -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++2c -fclangir -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++2c -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+int pack_indexing(auto... p) { return p...[0]; }
+
+// CIR: %[[P_0:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["p", init]
+// CIR: %[[P_1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["p", init]
+// CIR: %[[P_2:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["p", init]
+// CIR: %[[RET_VAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR: %[[RESULT:.*]] = cir.load{{.*}} %[[P_0]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.store %[[RESULT]], %[[RET_VAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[TMP:.*]] = cir.load %[[RET_VAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return %[[TMP]] : !s32i
+
+// LLVM: %[[P_0:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[P_1:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[P_2:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RET_VAL:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = load i32, ptr %[[P_0]], align 4
+// LLVM: store i32 %[[RESULT]], ptr %[[RET_VAL]], align 4
+// LLVM: %[[TMP:.*]] = load i32, ptr %[[RET_VAL]], align 4
+// LLVM: ret i32 %[[TMP]]
+
+// OGCG-DAG: %[[P_0:.*]] = alloca i32, align 4
+// OGCG-DAG: %[[P_1:.*]] = alloca i32, align 4
+// OGCG-DAG: %[[P_2:.*]] = alloca i32, align 4
+// OGCG-DAG: %[[RESULT:.*]] = load i32, ptr %[[P_0]], align 4
+// OGCG-DAG-NEXT: ret i32 %[[RESULT]]
+
+int foo() { return pack_indexing(1, 2, 3); }
+
+// CIR: %[[RET_VAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR: %[[RESULT:.*]] = cir.call @_Z13pack_indexingIJiiiEEiDpT_({{.*}}, {{.*}}, {{.*}}) : (!s32i, !s32i, !s32i) -> !s32i
+// CIR: cir.store %[[RESULT]], %[[RET_VAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[TMP:.*]] = cir.load %[[RET_VAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return %[[TMP]] : !s32i
+
+// LLVM: %[[RET_VAL:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = call i32 @_Z13pack_indexingIJiiiEEiDpT_(i32 1, i32 2, i32 3)
+// LLVM: store i32 %[[RESULT]], ptr %[[RET_VAL]], align 4
+// LLVM: %[[TMP:.*]] = load i32, ptr %[[RET_VAL]], align 4
+// LLVM: ret i32 %[[TMP]]
+
+// OGCG-DAG: %[[CALL:.*]] = call noundef i32 @_Z13pack_indexingIJiiiEEiDpT_(i32 noundef 1, i32 noundef 2, i32 noundef 3)
+// OGCG-DAG-NEXT: ret i32 %[[RESULT]]
diff --git a/clang/test/CIR/CodeGenOpenACC/cache.c b/clang/test/CIR/CodeGenOpenACC/cache.c
new file mode 100644
index 0000000000000..76651c132f738
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/cache.c
@@ -0,0 +1,132 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_cache() {
+  // CHECK: cir.func{{.*}} @acc_cache() {
+
+  int iArr[10];
+  // CHECK-NEXT: %[[IARR:.*]] = cir.alloca !cir.array<!s32i x 10>, !cir.ptr<!cir.array<!s32i x 10>>, ["iArr"]
+  float fArr[10];
+  // CHECK-NEXT: %[[FARR:.*]] = cir.alloca !cir.array<!cir.float x 10>, !cir.ptr<!cir.array<!cir.float x 10>>, ["fArr"]
+
+#pragma acc cache(iArr[1], fArr[1:5])
+  // This does nothing, as it is not in a loop.
+
+#pragma acc parallel
+  {
+#pragma acc cache(iArr[1], fArr[1:5])
+  // This does nothing, as it is not in a loop.
+  }
+  // CHECK-NEXT: acc.parallel {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
+    for(int j = 0; j < 5; ++j) {
+#pragma acc cache(iArr[1], fArr[1:5])
+    }
+  }
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
+#pragma acc cache(iArr[1], fArr[1:5])
+  }
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+
+#pragma acc parallel loop
+  for(int i = 0; i < 5; ++i) {
+#pragma acc cache(iArr[1], fArr[1:5])
+  }
+  // CHECK-NEXT: acc.parallel combined(loop) {
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop combined(parallel) cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop
+  for(int i = 0; i < 5; ++i) {
+    int localArr[5];
+    // The first term here isn't lowered, because it references data inside of the 'loop'.
+#pragma acc cache(localArr[i], iArr[1], fArr[1:5])
+  }
+  // CHECK-NEXT: acc.parallel combined(loop) {
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop combined(parallel) cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/enter-data.c b/clang/test/CIR/CodeGenOpenACC/enter-data.c
new file mode 100644
index 0000000000000..1785fba1a1059
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/enter-data.c
@@ -0,0 +1,125 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+void acc_data(int parmVar, int *ptrParmVar) {
+  // CHECK: cir.func{{.*}} @acc_data(%[[ARG:.*]]: !s32i{{.*}}, %[[PTRARG:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  // CHECK-NEXT: %[[PTRPARM:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptrParmVar", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[PTRARG]], %[[PTRPARM]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc enter data copyin(parmVar)
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data copyin(readonly, alwaysin: parmVar)
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data copyin(readonly, alwaysin: parmVar) async
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data async copyin(readonly, alwaysin: parmVar)
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data copyin(readonly, alwaysin: parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data async(parmVar) copyin(readonly, alwaysin: parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(parmVar)
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(zero: parmVar)
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(zero: parmVar) async
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(zero: parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data attach(ptrParmVar)
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc enter data attach(ptrParmVar) async
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc enter data attach(ptrParmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc enter data if (parmVar == 1) copyin(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data if(%[[CMP_CAST]]) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data async if (parmVar == 1) copyin(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data if(%[[CMP_CAST]]) async dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data if (parmVar == 1) async(parmVar) copyin(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data if(%[[CMP_CAST]]) async(%[[PARM_CAST]] : si32) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait create(parmVar)
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait(1) create(parmVar)
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait(%[[ONE_CAST]] : si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait(parmVar, 1, 2) create(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait(%[[PARM_CAST]], %[[ONE_CAST]], %[[TWO_CAST]] : si32, si32, si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait(devnum: parmVar: 1, 2) create(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait_devnum(%[[PARM_CAST]] : si32) wait(%[[ONE_CAST]], %[[TWO_CAST]] : si32, si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/exit-data.c b/clang/test/CIR/CodeGenOpenACC/exit-data.c
new file mode 100644
index 0000000000000..ff987d20d5b6c
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/exit-data.c
@@ -0,0 +1,134 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+void acc_data(int parmVar, int *ptrParmVar) {
+  // CHECK: cir.func{{.*}} @acc_data(%[[ARG:.*]]: !s32i{{.*}}, %[[PTRARG:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  // CHECK-NEXT: %[[PTRPARM:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptrParmVar", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[PTRARG]], %[[PTRPARM]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc exit data copyout(parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data copyout(zero, alwaysout: parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data copyout(zero, alwaysout: parmVar) async
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data async copyout(zero, alwaysout: parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data finalize copyout(zero, alwaysout: parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>) attributes {finalize}
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data async(parmVar) copyout(zero, alwaysout: parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data delete(parmVar) finalize
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!s32i>) attributes {finalize}
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data delete(parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>)  async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) {name = "parmVar", structured = false}
+
+#pragma acc exit data detach(ptrParmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_detach>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.detach accPtr(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) {name = "ptrParmVar", structured = false}
+
+#pragma acc exit data detach(ptrParmVar) async
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_detach>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async dataOperands(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.detach accPtr(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) async {name = "ptrParmVar", structured = false}
+
+#pragma acc exit data detach(ptrParmVar) async(parmVar) finalize
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_detach>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) attributes {finalize}
+  // CHECK-NEXT: acc.detach accPtr(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) async(%[[PARM_CAST]] : si32) {name = "ptrParmVar", structured = false}
+
+#pragma acc exit data if (parmVar == 1) copyout(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data if(%[[CMP_CAST]]) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data async if (parmVar == 1) copyout(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data if(%[[CMP_CAST]]) async dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data if (parmVar == 1) async(parmVar) copyout(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data if(%[[CMP_CAST]]) async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait delete(parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait(1) delete(parmVar)
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait(%[[ONE_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait(parmVar, 1, 2) delete(parmVar) finalize
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait(%[[PARM_CAST]], %[[ONE_CAST]], %[[TWO_CAST]] : si32, si32, si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>) attributes {finalize}
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait(devnum: parmVar: 1, 2) delete(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait_devnum(%[[PARM_CAST]] : si32) wait(%[[ONE_CAST]], %[[TWO_CAST]] : si32, si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/update.c b/clang/test/CIR/CodeGenOpenACC/update.c
new file mode 100644
index 0000000000000..2b29504e6ca20
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/update.c
@@ -0,0 +1,178 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_update(int parmVar, int *ptrParmVar) {
+  // CHECK: cir.func{{.*}} @acc_update(%[[ARG:.*]]: !s32i{{.*}}, %[[PTRARG:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  // CHECK-NEXT: %[[PTRPARM:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptrParmVar", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[PTRARG]], %[[PTRPARM]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc update device(parmVar)
+  // CHECK-NEXT: %[[UPD_DEV1:.*]] = acc.update_device varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[UPD_DEV1]] : !cir.ptr<!s32i>)
+
+#pragma acc update device(parmVar, ptrParmVar)
+  // CHECK-NEXT: %[[UPD_DEV1:.*]] = acc.update_device varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[UPD_DEV2:.*]] = acc.update_device varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[UPD_DEV1]], %[[UPD_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc update device(parmVar) device(ptrParmVar)
+  // CHECK-NEXT: %[[UPD_DEV1:.*]] = acc.update_device varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[UPD_DEV2:.*]] = acc.update_device varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[UPD_DEV1]], %[[UPD_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc update host(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_host>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc update host(parmVar, ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_host>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_host>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc update host(parmVar) host(ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_host>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_host>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar, ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) self(ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device(ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[UPD_DEV2:.*]] = acc.update_device varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[UPD_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) if (parmVar == 1)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: acc.update if(%[[CMP_CAST]]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+#pragma acc update self(parmVar) if (parmVar == 1) if_present
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: acc.update if(%[[CMP_CAST]]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>) attributes {ifPresent}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update wait dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait device_type(nvidia)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update wait dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update wait([#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32}) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait(parmVar) device_type(nvidia)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32}) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32} [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait(parmVar, 1, 2)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2>
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32, %[[ONE_CAST]] : si32, %[[TWO_CAST]] : si32} [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait(devnum:parmVar: 1, 2)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2>
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: acc.update wait({devnum: %[[PARM_CAST]] : si32, %[[ONE_CAST]] : si32, %[[TWO_CAST]] : si32} [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async device_type(nvidia)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) async
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async([#acc.device_type<radeon>]) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async([#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async([#acc.device_type<radeon>]) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async(parmVar) device_type(nvidia)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32 [#acc.device_type<radeon>]) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async(%[[PARM_CAST]] : si32 [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32 [#acc.device_type<radeon>]) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+}
diff --git a/clang/test/CIR/IR/invalid-opt-info.cir b/clang/test/CIR/IR/invalid-opt-info.cir
new file mode 100644
index 0000000000000..52d6afd2c6974
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-opt-info.cir
@@ -0,0 +1,13 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+module attributes {
+  // expected-error @below {{optimization level must be between 0 and 3 inclusive}}
+  cir.opt_info = #cir.opt_info<level = 4, size = 0>
+} {}
+
+// -----
+
+module attributes {
+  // expected-error @below {{size optimization level must be between 0 and 2 inclusive}}
+  cir.opt_info = #cir.opt_info<level = 0, size = 3>
+} {}
diff --git a/clang/test/CIR/IR/struct.cir b/clang/test/CIR/IR/struct.cir
index 5ff5ff7cd1f69..85f475f643ee5 100644
--- a/clang/test/CIR/IR/struct.cir
+++ b/clang/test/CIR/IR/struct.cir
@@ -1,19 +1,74 @@
 // RUN: cir-opt %s | FileCheck %s
 
+!u8i = !cir.int<u, 8>
+!u16i = !cir.int<u, 16>
+!s32i = !cir.int<s, 32>
+!u32i = !cir.int<u, 32>
+
 !rec_C = !cir.record<class "C" incomplete>
 !rec_S = !cir.record<struct "S" incomplete>
 !rec_U = !cir.record<union "U" incomplete>
 
-// CHECK: !rec_C = !cir.record<class "C" incomplete>
-// CHECK: !rec_S = !cir.record<struct "S" incomplete>
-// CHECK: !rec_U = !cir.record<union "U" incomplete>
+// CHECK-DAG: !rec_C = !cir.record<class "C" incomplete>
+// CHECK-DAG: !rec_S = !cir.record<struct "S" incomplete>
+// CHECK-DAG: !rec_U = !cir.record<union "U" incomplete>
+
+!rec_anon_struct = !cir.record<struct {!cir.array<!cir.ptr<!u8i> x 5>}>
+!rec_anon_struct1 = !cir.record<struct {!cir.ptr<!u8i>, !cir.ptr<!u8i>, !cir.ptr<!u8i>}>
+!rec_S1 = !cir.record<struct "S1" {!s32i, !s32i}>
+!rec_Sc = !cir.record<struct "Sc" {!u8i, !u16i, !u32i}>
+
+// CHECK-DAG: !cir.record<struct {!cir.array<!cir.ptr<!u8i> x 5>}>
+// CHECK-DAG: !cir.record<struct {!cir.ptr<!u8i>, !cir.ptr<!u8i>, !cir.ptr<!u8i>}>
+// CHECK-DAG: !rec_S1 = !cir.record<struct "S1" {!s32i, !s32i}>
+// CHECK-DAG: !rec_Sc = !cir.record<struct "Sc" {!u8i, !u16i, !u32i}>
+
+// Packed and padded structs
+!rec_P1 = !cir.record<struct "P1" packed {!s32i, !s32i}>
+!rec_P2 = !cir.record<struct "P2" padded {!u8i, !u16i, !u32i}>
+!rec_P3 = !cir.record<struct "P3" packed padded {!u8i, !u16i, !u32i}>
+
+// CHECK-DAG: !rec_P1 = !cir.record<struct "P1" packed {!s32i, !s32i}>
+// CHECK-DAG: !rec_P2 = !cir.record<struct "P2" padded {!u8i, !u16i, !u32i}>
+// CHECK-DAG: !rec_P3 = !cir.record<struct "P3" packed padded {!u8i, !u16i, !u32i}>
 
-module  {
-    cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
-    cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
-    cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
-}
 
+// Complete a previously incomplete record
+!rec_A = !cir.record<class "A" incomplete>
+!rec_Ac = !cir.record<class "A" {!u8i, !s32i}>
+// CHECK-DAG: !rec_A = !cir.record<class "A" {!u8i, !s32i}>
+
+// Test recursive struct parsing/printing.
+!rec_Node = !cir.record<struct "Node" {!cir.ptr<!cir.record<struct "Node">>}>
+// CHECK-DAG: !cir.record<struct "Node" {!cir.ptr<!cir.record<struct "Node">>}>
+
+module  {
+  cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
+  cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
+  cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
 // CHECK: cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
 // CHECK: cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
 // CHECK: cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
+
+  // Dummy function to use types and force them to be printed.
+  cir.func @useTypes(%arg0: !rec_Node,
+                     %arg1: !rec_anon_struct1,
+                     %arg2: !rec_anon_struct,
+                     %arg3: !rec_S1,
+                     %arg4: !rec_Ac,
+                     %arg5: !rec_P1,
+                     %arg6: !rec_P2,
+                     %arg7: !rec_P3) {
+    cir.return
+  }
+
+  cir.func @structs() {
+    %0 = cir.alloca !cir.ptr<!cir.record<struct "Sc" {!u8i, !u16i, !u32i}>>, !cir.ptr<!cir.ptr<!cir.record<struct "Sc" {!u8i, !u16i, !u32i}>>>, ["sc", init]
+    %1 = cir.alloca !cir.ptr<!cir.record<union "U" incomplete>>, !cir.ptr<!cir.ptr<!cir.record<union "U" incomplete>>>, ["u", init]
+    cir.return
+  }
+
+// CHECK: cir.func @structs() {
+// CHECK:     %0 = cir.alloca !cir.ptr<!rec_Sc>, !cir.ptr<!cir.ptr<!rec_Sc>>, ["sc", init]
+// CHECK:     %1 = cir.alloca !cir.ptr<!rec_U>, !cir.ptr<!cir.ptr<!rec_U>>, ["u", init]
+}
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index e5b4a3bb84645..416af9ab4d0aa 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -29,6 +29,26 @@ llvm_canonicalize_cmake_booleans(
   LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
   )
 
+# Run tests requiring Z3 headers only if LLVM was built with Z3
+# and the headers are available while building Clang -- the latter may
+# not be the case when building standalone against installed LLVM.
+set(TEST_WITH_Z3_MOCK 0)
+if(LLVM_WITH_Z3 AND Z3_FOUND AND CMAKE_SYSTEM_NAME MATCHES "Linux")
+  llvm_add_library(
+    MockZ3SolverCheck
+    MODULE Analysis/z3/Inputs/MockZ3_solver_check.cpp
+    DISABLE_LLVM_LINK_LLVM_DYLIB)
+
+  if(TARGET MockZ3SolverCheck)
+    list(APPEND CLANG_TEST_DEPS
+      MockZ3SolverCheck)
+    target_include_directories(
+      MockZ3SolverCheck
+      PRIVATE ${Z3_INCLUDE_DIR})
+    set(TEST_WITH_Z3_MOCK 1)
+  endif()
+endif()
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
@@ -94,6 +114,7 @@ if(CLANG_ENABLE_CIR)
   list(APPEND CLANG_TEST_DEPS
     cir-opt
     cir-translate
+    mlir-translate
     )
 endif()
 
diff --git a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
index 497a9328c1156..a8cd523b0837f 100644
--- a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
+++ b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
@@ -102,11 +102,12 @@ namespace ns {
 }
 
 #elif TEST13
+// expected-no-diagnostics
 extern "C++" {
-  int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
+  int main();
 }
 
-extern "C++" int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
+extern "C++" int main();
 
 namespace ns1 {
   extern "C++" int main(); // ok
diff --git a/clang/test/CXX/class.access/class.access.dcl/p1.cpp b/clang/test/CXX/class.access/class.access.dcl/p1.cpp
index fdb1373dd9b12..b7ee6c9a7c04b 100644
--- a/clang/test/CXX/class.access/class.access.dcl/p1.cpp
+++ b/clang/test/CXX/class.access/class.access.dcl/p1.cpp
@@ -318,26 +318,23 @@ namespace test4 {
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
 #endif
 
-    C::foo; // legal in C++03
+    C::foo;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
 #else
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
-    // expected-error@-5 {{using declaration refers to its own class}}
 #endif
+    // expected-error@-6 {{using declaration refers to its own class}}
 
-    Subclass::foo; // legal in C++03
+    Subclass::foo;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
 #else
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
-    // expected-error@-5 {{using declaration refers into 'Subclass', which is not a base class of 'C'}}
 #endif
+    // expected-error@-6 {{using declaration refers into 'Subclass', which is not a base class of 'C'}}
 
     int bar();
-#if __cplusplus <= 199711L
-    //expected-note@-2 {{target of using declaration}}
-#endif
     C::bar;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
index 657695657e0f7..bcc26bf4f4356 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
@@ -30,17 +30,10 @@ class D2 : public B {
   using B::x;
   using C::g; // expected-error{{using declaration refers into 'C', which is not a base class of 'D2'}}
 
-  // These are valid in C++98 but not in C++11.
-  using D::f2;
-  using D::E2;
-  using D::e2;
-  using D::x2;
-#if __cplusplus >= 201103L
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-#endif
+  using D::f2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  using D::E2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  using D::e2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  using D::x2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
 
   using B::EC;
   using B::EC::ec; // expected-warning {{a C++20 extension}} expected-warning 0-1 {{C++11}}
@@ -71,13 +64,7 @@ namespace test1 {
     using Base::bar; // expected-error {{no member named 'bar'}}
     using Unrelated::foo; // expected-error {{not a base class}}
 
-    // In C++98, it's hard to see that these are invalid, because indirect
-    // references to base class members are permitted.
-    using C::foo;
-    using Subclass::foo;
-#if __cplusplus >= 201103L
-    // expected-error@-3 {{refers to its own class}}
-    // expected-error@-3 {{not a base class}}
-#endif
+    using C::foo; // expected-error {{refers to its own class}}
+    using Subclass::foo; // expected-error {{not a base class}}
   };
 }
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
index 973cd77279f19..26b66350b86d9 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
@@ -206,17 +206,9 @@ namespace test4 {
     using InnerNS::foo; // expected-error {{not a class}}
     using Base::bar; // expected-error {{no member named 'bar'}}
     using Unrelated::foo; // expected-error {{not a base class}}
-    using C::foo; // legal in C++03
-    using Subclass::foo; // legal in C++03
-#if __cplusplus >= 201103L
-    // expected-error@-3 {{refers to its own class}}
-    // expected-error@-3 {{refers into 'Subclass', which is not a base class}}
-#endif
-
+    using C::foo; // expected-error {{refers to its own class}}
+    using Subclass::foo; // expected-error {{refers into 'Subclass', which is not a base class}}
     int bar();
-#if __cplusplus < 201103L
-    // expected-note@-2 {{target of using declaration}}
-#endif
     using C::bar; // expected-error {{refers to its own class}}
   };
 }
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 5b4551ba0143b..9948075852135 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -564,11 +564,12 @@ struct A {
 namespace ex2 {
 #if __cplusplus >= 201103L
 struct Bar {
-  struct Baz {
+  struct Baz { // #cwg1890-Baz
     int a = 0;
   };
   static_assert(__is_constructible(Baz), "");
   // since-cxx11-error@-1 {{static assertion failed due to requirement '__is_constructible(cwg1890::ex2::Bar::Baz)'}}
+  // since-cxx11-note@#cwg1890-Baz {{'Baz' defined here}}
 };
 #endif
 } // namespace ex2
diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
index 210f7ae71ec04..dc53ae712a0b5 100644
--- a/clang/test/CXX/drs/cwg4xx.cpp
+++ b/clang/test/CXX/drs/cwg4xx.cpp
@@ -36,6 +36,15 @@ namespace cwg400 { // cwg400: 2.7
   // expected-error@-1 {{member 'a' found in multiple base classes of different types}}
   //   expected-note@#cwg400-A {{member type 'cwg400::A::a' found by ambiguous name lookup}}
   //   expected-note@#cwg400-B {{member type 'cwg400::B::a' found by ambiguous name lookup}}
+  struct F : A {};
+  struct G : A {
+    using G::A;
+    // expected-error@-1 {{using declaration refers to its own class}}
+    using G::a;
+    // expected-error@-1 {{using declaration refers to its own class}}
+    using F::a;
+    // expected-error@-1 {{using declaration refers into 'F', which is not a base class of 'G'}}
+  };
 } // namespace cwg400
 
 namespace cwg401 { // cwg401: 2.8
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p5.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p5.cpp
index 84d84a61a8d26..fdb370f7685ff 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p5.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p5.cpp
@@ -6,7 +6,7 @@ void test_attributes() {
   auto nrl = [](int x) -> int { if (x > 0) return x; }; // expected-error{{non-void lambda does not return a value in all control paths}}
 
   // FIXME: GCC accepts the [[gnu::noreturn]] attribute here.
-  auto nrl2 = []() [[gnu::noreturn]] { return; }; // expected-warning{{attribute 'noreturn' ignored}}
+  auto nrl2 = []() [[gnu::noreturn]] { return; }; // expected-warning{{attribute 'gnu::noreturn' ignored}}
 }
 
 template<typename T>
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp
index 7a570e07c2126..062d1b5f27342 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp
@@ -3,7 +3,7 @@
 template<typename T>
 void test_attributes() {
   // FIXME: GCC accepts [[gnu::noreturn]] here.
-  auto nrl = []() [[gnu::noreturn]] {}; // expected-warning{{attribute 'noreturn' ignored}}
+  auto nrl = []() [[gnu::noreturn]] {}; // expected-warning{{attribute 'gnu::noreturn' ignored}}
 }
 
 template void test_attributes<int>();
diff --git a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
index ac21e36daf870..abb42447d3e0b 100644
--- a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
+++ b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
@@ -218,4 +218,26 @@ void regression() {
 
 }
 
+namespace GH146063 {
+template <typename>
+struct A {
+  static_assert([]() constexpr { return true; }());
+};
+
+void f1() {
+  if constexpr (false) {
+    A<int> a;
+  }
+}
+
+void f2() {
+  if constexpr (false) {
+    static_assert([]{});
+    // expected-warning@-1 {{address of lambda function pointer conversion operator will always evaluate to 'true'}}
+  }
+}
+
+}
+
+
 #endif
diff --git a/clang/test/ClangScanDeps/fatal-module-loader-error.m b/clang/test/ClangScanDeps/fatal-module-loader-error.m
new file mode 100644
index 0000000000000..ed4d8628a6471
--- /dev/null
+++ b/clang/test/ClangScanDeps/fatal-module-loader-error.m
@@ -0,0 +1,42 @@
+// This tests that after a fatal module loader error, we do not continue parsing.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: mkdir %t/ModulesCache
+// RUN: touch %t/ModulesCache/Unusable.pcm
+
+// RUN: not clang-scan-deps -format experimental-full -mode preprocess-dependency-directives -- \
+// RUN:   %clang -fmodules -fimplicit-modules -Xclang -fdisable-module-hash -fmodules-cache-path=%t/ModulesCache \
+// RUN:   -F %t/Frameworks -c %t/test.m -o %t/test.o
+// RUN: ls %t/ModulesCache | not grep AfterUnusable
+
+//--- Frameworks/Unusable.framework/Headers/Unusable.h
+// empty
+//--- Frameworks/Unusable.framework/Modules/module.modulemap
+framework module Unusable { header "Unusable.h" }
+
+//--- Frameworks/AfterUnusable.framework/Headers/AfterUnusable.h
+// empty
+//--- Frameworks/AfterUnusable.framework/Modules/module.modulemap
+framework module AfterUnusable { header "AfterUnusable.h" }
+
+//--- Frameworks/Importer.framework/Headers/Importer.h
+#import <Importer/ImportUnusable.h>
+// Parsing should have stopped and we should not handle AfterUnusable.
+#import <AfterUnusable/AfterUnusable.h>
+
+//--- Frameworks/Importer.framework/Headers/ImportUnusable.h
+// It is important that this header is a submodule.
+#import <Unusable/Unusable.h>
+// Parsing should have stopped and we should not handle AfterUnusable.
+#import <AfterUnusable/AfterUnusable.h>
+
+//--- Frameworks/Importer.framework/Modules/module.modulemap
+framework module Importer {
+  umbrella header "Importer.h"
+  module * { export * }
+  export *
+}
+
+//--- test.m
+#import <Importer/Importer.h>
diff --git a/clang/test/CodeCompletion/overrides.cpp b/clang/test/CodeCompletion/overrides.cpp
index 7f330904eede9..fcafe03021f1a 100644
--- a/clang/test/CodeCompletion/overrides.cpp
+++ b/clang/test/CodeCompletion/overrides.cpp
@@ -41,3 +41,17 @@ void func() {
 // Runs completion at empty line on line 37.
 // RUN: not %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-5):1 %s -o - | FileCheck -check-prefix=CHECK-CC4 %s
 // CHECK-CC4: COMPLETION: Pattern : void vfunc(bool param, int p) override{{$}}
+
+class NoexceptBase {
+ public:
+  virtual void method() noexcept;
+};
+
+class NoexceptDerived : public NoexceptBase {
+ public:
+  met;
+};
+
+// Runs completion at met^ in the body of NoexceptDerived.
+// RUN: not %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-4):6 %s -o - | FileCheck -check-prefix=CHECK-CC5 %s
+// CHECK-CC5: COMPLETION: Pattern : void method() noexcept override{{$}}
diff --git a/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp b/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
new file mode 100644
index 0000000000000..55c16bb126fee
--- /dev/null
+++ b/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
@@ -0,0 +1,14 @@
+struct A {
+  void foo(this A self, int arg);
+};
+
+int main() {
+  A a {};
+  a.
+}
+// RUN: %clang_cc1 -cc1 -fsyntax-only -code-completion-at=%s:%(line-2):5 -std=c++23 %s | FileCheck %s
+// CHECK: COMPLETION: A : A::
+// CHECK-NEXT: COMPLETION: foo : [#void#]foo(<#int arg#>)
+// CHECK-NEXT: COMPLETION: operator= : [#A &#]operator=(<#const A &#>)
+// CHECK-NEXT: COMPLETION: operator= : [#A &#]operator=(<#A &&#>)
+// CHECK-NEXT: COMPLETION: ~A : [#void#]~A()
diff --git a/clang/test/CodeGen/AArch64/attr-fp8-function.c b/clang/test/CodeGen/AArch64/attr-fp8-function.c
new file mode 100644
index 0000000000000..54bfd177bd809
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/attr-fp8-function.c
@@ -0,0 +1,37 @@
+// Check memory attribute for FP8 function
+
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +fp8 -target-feature +sve -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32  -target-feature +ssve-fp8fma -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_neon.h>
+#include <arm_sme.h>
+
+
+// SIMD
+mfloat8x16_t test_vcvtq_mf8_f16_fpm(float16x8_t vn, float16x8_t vm, fpm_t fpm) {
+  return vcvtq_mf8_f16_fpm(vn, vm, fpm);
+}
+
+// SVE
+svfloat16_t test_svcvtlt2_f16_mf8(svmfloat8_t zn, fpm_t fpm) __arm_streaming {
+  return svcvtlt2_f16_mf8_fpm(zn, fpm);
+}
+
+// CHECK: declare void @llvm.aarch64.set.fpmr(i64) [[ATTR3:#.*]]
+// CHECK: declare <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt2.nxv8f16(<vscale x 16 x i8>) [[ATTR4:#.*]]
+
+
+// SME
+svfloat32_t test_svmlalltt_lane_f32_mf8(svfloat32_t zda, svmfloat8_t zn, svmfloat8_t zm, fpm_t fpm) __arm_streaming {
+  return svmlalltt_lane_f32_mf8_fpm(zda, zn, zm, 7, fpm);
+}
+
+// CHECK: declare <vscale x 4 x float> @llvm.aarch64.sve.fp8.fmlalltt.lane.nxv4f32(<vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32 immarg) [[ATTR4]]
+
+// CHECK: declare <16 x i8> @llvm.aarch64.neon.fp8.fcvtn.v16i8.v8f16(<8 x half>, <8 x half>) [[ATTR4]]
+
+// CHECK: attributes [[ATTR1:#.*]] = {{{.*}}} 
+// CHECK: attributes [[ATTR2:#.*]] = {{{.*}}}
+// CHECK: attributes [[ATTR3]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+// CHECK: attributes [[ATTR4]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: read) }
diff --git a/clang/test/CodeGen/AArch64/fmv-dependencies.c b/clang/test/CodeGen/AArch64/fmv-dependencies.c
index a97c4e95cd032..8f7f23f93dfea 100644
--- a/clang/test/CodeGen/AArch64/fmv-dependencies.c
+++ b/clang/test/CodeGen/AArch64/fmv-dependencies.c
@@ -189,6 +189,6 @@ int caller() {
 // CHECK: attributes #[[sve2_aes]] = { {{.*}} "target-features"="+aes,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve-aes,+sve2,+sve2-aes,+v8a"
 // CHECK: attributes #[[sve2_bitperm]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve-bitperm,+sve2,+sve2-bitperm,+v8a"
 // CHECK: attributes #[[sve2_sha3]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sha2,+sha3,+sve,+sve-sha3,+sve2,+sve2-sha3,+v8a"
-// CHECK: attributes #[[sve2_sm4]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sm4,+sve,+sve2,+sve2-sm4,+v8a"
+// CHECK: attributes #[[sve2_sm4]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sm4,+sve,+sve-sm4,+sve2,+sve2-sm4,+v8a"
 // CHECK: attributes #[[wfxt]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a,+wfxt"
 // CHECK: attributes #[[cssc]] = { {{.*}} "target-features"="+cssc,+fp-armv8,+neon,+outline-atomics,+v8a"
diff --git a/clang/test/CodeGen/AArch64/neon-intrinsics.c b/clang/test/CodeGen/AArch64/neon-intrinsics.c
index 791f0a1a29409..6304245614342 100644
--- a/clang/test/CodeGen/AArch64/neon-intrinsics.c
+++ b/clang/test/CodeGen/AArch64/neon-intrinsics.c
@@ -13112,6 +13112,16 @@ float64x2_t test_vld1q_f64(float64_t const *a) {
   return vld1q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[A]], align 1
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vld1q_mf8(mfloat8_t const *a) {
+  return vld1q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13242,6 +13252,16 @@ float64x1_t test_vld1_f64(float64_t const *a) {
   return vld1_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <8 x i8> @test_vld1_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[A]], align 1
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vld1_mf8(mfloat8_t const *a) {
+  return vld1_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <8 x i8> @test_vld1_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13546,6 +13566,20 @@ float64x2x2_t test_vld2q_f64(float64_t const *a) {
   return vld2q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld2q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld2q_mf8(mfloat8_t const *a) {
+  return vld2q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x2_t @test_vld2q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13728,6 +13762,20 @@ float64x1x2_t test_vld2_f64(float64_t const *a) {
   return vld2_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld2_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld2_mf8(mfloat8_t const *a) {
+  return vld2_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x2_t @test_vld2_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13932,6 +13980,22 @@ float64x2x3_t test_vld3q_f64(float64_t const *a) {
   return vld3q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld3q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld3q_mf8(mfloat8_t const *a) {
+  return vld3q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x3_t @test_vld3q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14140,6 +14204,22 @@ float64x1x3_t test_vld3_f64(float64_t const *a) {
   return vld3_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld3_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld3_mf8(mfloat8_t const *a) {
+  return vld3_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x3_t @test_vld3_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14370,6 +14450,24 @@ float64x2x4_t test_vld4q_f64(float64_t const *a) {
   return vld4q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld4q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld4q_mf8(mfloat8_t const *a) {
+  return vld4q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x4_t @test_vld4q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14604,6 +14702,24 @@ float64x1x4_t test_vld4_f64(float64_t const *a) {
   return vld4_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld4_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld4_mf8(mfloat8_t const *a) {
+  return vld4_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x4_t @test_vld4_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14732,6 +14848,16 @@ void test_vst1q_s64(int64_t *a, int64x2_t b) {
   vst1q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    store <16 x i8> [[VAL]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8(mfloat8_t *a, mfloat8x16_t val) {
+  vst1q_mf8(a, val);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <8 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14885,6 +15011,16 @@ void test_vst1_s64(int64_t *a, int64x1_t b) {
   vst1_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    store <8 x i8> [[VAL]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8(mfloat8_t *a, mfloat8x8_t val) {
+  vst1_mf8(a, val);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <4 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15067,6 +15203,18 @@ void test_vst2q_s64(int64_t *a, int64x2x2_t b) {
   vst2q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2q_mf8(mfloat8_t *a, mfloat8x16x2_t b) {
+  vst2q_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15269,6 +15417,18 @@ void test_vst2_s64(int64_t *a, int64x1x2_t b) {
   vst2_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2_mf8(mfloat8_t *a, mfloat8x8x2_t b) {
+  vst2_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15493,6 +15653,19 @@ void test_vst3q_s64(int64_t *a, int64x2x3_t b) {
   vst3q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3q_mf8(mfloat8_t *a, mfloat8x16x3_t b) {
+  vst3q_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15731,6 +15904,19 @@ void test_vst3_s64(int64_t *a, int64x1x3_t b) {
   vst3_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3_mf8(mfloat8_t *a, mfloat8x8x3_t b) {
+  vst3_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15992,6 +16178,20 @@ void test_vst4q_s64(int64_t *a, int64x2x4_t b) {
   vst4q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4q_mf8(mfloat8_t *a, mfloat8x16x4_t b) {
+  vst4q_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16266,6 +16466,20 @@ void test_vst4_s64(int64_t *a, int64x1x4_t b) {
   vst4_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4_mf8(mfloat8_t *a, mfloat8x8x4_t b) {
+  vst4_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16398,6 +16612,20 @@ float64x2x2_t test_vld1q_f64_x2(float64_t const *a) {
   return vld1q_f64_x2(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld1q_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x2.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld1q_mf8_x2(mfloat8_t const *a) {
+  return vld1q_mf8_x2(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x2_t @test_vld1q_p64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16426,6 +16654,20 @@ float64x1x2_t test_vld1_f64_x2(float64_t const *a) {
   return vld1_f64_x2(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld1_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x2.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld1_mf8_x2(mfloat8_t const *a) {
+  return vld1_mf8_x2(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x2_t @test_vld1_p64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16456,6 +16698,22 @@ float64x2x3_t test_vld1q_f64_x3(float64_t const *a) {
   return vld1q_f64_x3(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld1q_mf8_x3(
+// CHECK-SAME: ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x3.v16i8.p0(ptr [[PTR]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld1q_mf8_x3(mfloat8_t const *ptr) {
+  return vld1q_mf8_x3(ptr);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x3_t @test_vld1q_p64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16488,6 +16746,23 @@ float64x1x3_t test_vld1_f64_x3(float64_t const *a) {
   return vld1_f64_x3(a);
 }
 
+
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld1_mf8_x3(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x3.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld1_mf8_x3(mfloat8_t const *a) {
+  return vld1_mf8_x3(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x3_t @test_vld1_p64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16522,6 +16797,24 @@ float64x2x4_t test_vld1q_f64_x4(float64_t const *a) {
   return vld1q_f64_x4(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld1q_mf8_x4(
+// CHECK-SAME: ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x4.v16i8.p0(ptr [[PTR]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD1XN_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld1q_mf8_x4(mfloat8_t const *ptr) {
+  return vld1q_mf8_x4(ptr);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x4_t @test_vld1q_p64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16558,6 +16851,24 @@ float64x1x4_t test_vld1_f64_x4(float64_t const *a) {
   return vld1_f64_x4(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld1_mf8_x4(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x4.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD1XN_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld1_mf8_x4(mfloat8_t const *a) {
+  return vld1_mf8_x4(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x4_t @test_vld1_p64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16576,6 +16887,18 @@ poly64x1x4_t test_vld1_p64_x4(poly64_t const *a) {
   return vld1_p64_x4(a);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x2.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8_x2(mfloat8_t *a, mfloat8x16x2_t b) {
+  vst1q_mf8_x2(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <2 x double>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16610,6 +16933,18 @@ void test_vst1q_p64_x2(poly64_t *a, poly64x2x2_t b) {
   vst1q_p64_x2(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x2.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8_x2(mfloat8_t *a, mfloat8x8x2_t b) {
+  vst1_mf8_x2(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <1 x double>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16646,6 +16981,19 @@ void test_vst1_p64_x2(poly64_t *a, poly64x1x2_t b) {
   vst1_p64_x2(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8_x3(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x3.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8_x3(mfloat8_t *a, mfloat8x16x3_t b) {
+  vst1q_mf8_x3(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <2 x double>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16687,6 +17035,19 @@ void test_vst1q_p64_x3(poly64_t *a, poly64x2x3_t b) {
   vst1q_p64_x3(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8_x3(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x3.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8_x3(mfloat8_t *a, mfloat8x8x3_t b) {
+  vst1_mf8_x3(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <1 x double>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16731,6 +17092,20 @@ void test_vst1_p64_x3(poly64_t *a, poly64x1x3_t b) {
   vst1_p64_x3(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8_x4(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x4.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8_x4(mfloat8_t *a, mfloat8x16x4_t b) {
+  vst1q_mf8_x4(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <2 x double>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16779,6 +17154,20 @@ void test_vst1q_p64_x4(poly64_t *a, poly64x2x4_t b) {
   vst1q_p64_x4(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8_x4(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x4.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8_x4(mfloat8_t *a, mfloat8x8x4_t b) {
+  vst1_mf8_x4(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <1 x double>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/neon-ldst-one.c b/clang/test/CodeGen/AArch64/neon-ldst-one.c
index 2cff007826ba6..9a87e8d2ce6f8 100644
--- a/clang/test/CodeGen/AArch64/neon-ldst-one.c
+++ b/clang/test/CodeGen/AArch64/neon-ldst-one.c
@@ -139,6 +139,18 @@ float64x2_t test_vld1q_dup_f64(float64_t  *a) {
   return vld1q_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i32 0
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP1]], <16 x i32> zeroinitializer
+// CHECK-NEXT:    ret <16 x i8> [[LANE]]
+//
+mfloat8x16_t test_vld1q_dup_mf8(mfloat8_t  *a) {
+  return vld1q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_dup_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -307,6 +319,18 @@ float64x1_t test_vld1_dup_f64(float64_t  *a) {
   return vld1_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <8 x i8> @test_vld1_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i32 0
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> [[TMP1]], <8 x i32> zeroinitializer
+// CHECK-NEXT:    ret <8 x i8> [[LANE]]
+//
+mfloat8x8_t test_vld1_dup_mf8(mfloat8_t  *a) {
+  return vld1_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <8 x i8> @test_vld1_dup_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -385,6 +409,20 @@ float64x2x2_t test_vld2q_dup_f64(float64_t  *a) {
   return vld2q_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld2q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2r.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld2q_dup_mf8(mfloat8_t  *a) {
+  return vld2q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x2_t @test_vld2q_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -413,6 +451,20 @@ float64x1x2_t test_vld2_dup_f64(float64_t  *a) {
   return vld2_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld2_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2r.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld2_dup_mf8(mfloat8_t  *a) {
+  return vld2_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x2_t @test_vld2_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -478,6 +530,22 @@ float64x2x3_t test_vld3q_dup_f64(float64_t  *a) {
   // [{{x[0-9]+|sp}}]
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld3q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3r.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld3q_dup_mf8(mfloat8_t  *a) {
+  return vld3q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x3_t @test_vld3q_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -512,6 +580,22 @@ float64x1x3_t test_vld3_dup_f64(float64_t  *a) {
   // [{{x[0-9]+|sp}}]
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld3_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3r.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld3_dup_mf8(mfloat8_t  *a) {
+  return vld3_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x3_t @test_vld3_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -583,6 +667,24 @@ float64x2x4_t test_vld4q_dup_f64(float64_t  *a) {
   return vld4q_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld4q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4r.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld4q_dup_mf8(mfloat8_t  *a) {
+  return vld4q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x4_t @test_vld4q_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -619,6 +721,24 @@ float64x1x4_t test_vld4_dup_f64(float64_t  *a) {
   return vld4_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld4_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4r.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld4_dup_mf8(mfloat8_t  *a) {
+  return vld4_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x4_t @test_vld4_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -779,6 +899,17 @@ float64x2_t test_vld1q_lane_f64(float64_t  *a, float64x2_t b) {
   return vld1q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[VLD1_LANE:%.*]] = insertelement <16 x i8> [[B]], i8 [[TMP0]], i32 15
+// CHECK-NEXT:    ret <16 x i8> [[VLD1_LANE]]
+//
+mfloat8x16_t test_vld1q_lane_mf8(mfloat8_t  *a, mfloat8x16_t b) {
+  return vld1q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -959,6 +1090,17 @@ float64x1_t test_vld1_lane_f64(float64_t  *a, float64x1_t b) {
   return vld1_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local <8 x i8> @test_vld1_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[VLD1_LANE:%.*]] = insertelement <8 x i8> [[B]], i8 [[TMP0]], i32 7
+// CHECK-NEXT:    ret <8 x i8> [[VLD1_LANE]]
+//
+mfloat8x8_t test_vld1_lane_mf8(mfloat8_t  *a, mfloat8x8_t b) {
+  return vld1_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local <8 x i8> @test_vld1_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -1268,6 +1410,22 @@ float64x2x2_t test_vld2q_lane_f64(float64_t  *a, float64x2x2_t b) {
   return vld2q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld2q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[VLD2_LANE:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2_LANE]], 0
+// CHECK-NEXT:    [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2_LANE]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld2q_lane_mf8(mfloat8_t  *a, mfloat8x16x2_t b) {
+  return vld2q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly16x8x2_t @test_vld2q_lane_p16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i16>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -1528,6 +1686,22 @@ float64x1x2_t test_vld2_lane_f64(float64_t  *a, float64x1x2_t b) {
   return vld2_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld2_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[VLD2_LANE:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2_LANE]], 0
+// CHECK-NEXT:    [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2_LANE]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld2_lane_mf8(mfloat8_t  *a, mfloat8x8x2_t b) {
+  return vld2_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x2_t @test_vld2_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -1818,6 +1992,26 @@ float64x2x3_t test_vld3q_lane_f64(float64_t  *a, float64x2x3_t b) {
   return vld3q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld3q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[VLD3_LANE:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3_LANE]], 0
+// CHECK-NEXT:    [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3_LANE]], 1
+// CHECK-NEXT:    [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3_LANE]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld3q_lane_mf8(mfloat8_t  *a, mfloat8x16x3_t b) {
+  return vld3q_lane_mf8(a, b, 15);
+}
+
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x3_t @test_vld3q_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -2162,6 +2356,25 @@ float64x1x3_t test_vld3_lane_f64(float64_t  *a, float64x1x3_t b) {
   return vld3_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld3_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[VLD3_LANE:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3_LANE]], 0
+// CHECK-NEXT:    [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3_LANE]], 1
+// CHECK-NEXT:    [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3_LANE]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld3_lane_mf8(mfloat8_t  *a, mfloat8x8x3_t b) {
+  return vld3_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x3_t @test_vld3_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -2557,6 +2770,28 @@ float64x2x4_t test_vld4q_lane_f64(float64_t  *a, float64x2x4_t b) {
   return vld4q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld4q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    [[VLD4_LANE:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 0
+// CHECK-NEXT:    [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 1
+// CHECK-NEXT:    [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 2
+// CHECK-NEXT:    [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld4q_lane_mf8(mfloat8_t  *a, mfloat8x16x4_t b) {
+  return vld4q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x4_t @test_vld4q_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -2969,6 +3204,28 @@ float64x1x4_t test_vld4_lane_f64(float64_t  *a, float64x1x4_t b) {
   return vld4_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld4_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    [[VLD4_LANE:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 0
+// CHECK-NEXT:    [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 1
+// CHECK-NEXT:    [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 2
+// CHECK-NEXT:    [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld4_lane_mf8(mfloat8_t  *a, mfloat8x8x4_t b) {
+  return vld4_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x4_t @test_vld4_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3151,6 +3408,17 @@ void test_vst1q_lane_s64(int64_t  *a, int64x2_t b) {
   vst1q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <16 x i8> [[B]], i32 15
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_lane_mf8(mfloat8_t *a, mfloat8x16_t b) {
+  vst1q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <8 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3330,6 +3598,17 @@ void test_vst1_lane_s64(int64_t  *a, int64x1_t b) {
   vst1_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i8> [[B]], i32 7
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1_lane_mf8(mfloat8_t *a, mfloat8x8_t b) {
+  vst1_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <4 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3530,6 +3809,18 @@ void test_vst2q_lane_s64(int64_t  *a, int64x2x2_t b) {
   vst2q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2q_lane_mf8(mfloat8_t *a, mfloat8x16x2_t b) {
+  vst2q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3748,6 +4039,18 @@ void test_vst2_lane_s64(int64_t  *a, int64x1x2_t b) {
   vst2_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2_lane_mf8(mfloat8_t *a, mfloat8x8x2_t b) {
+  vst2_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3988,6 +4291,19 @@ void test_vst3q_lane_s64(int64_t  *a, int64x2x3_t b) {
   vst3q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3q_lane_mf8(mfloat8_t *a, mfloat8x16x3_t b) {
+  vst3q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -4245,6 +4561,19 @@ void test_vst3_lane_s64(int64_t  *a, int64x1x3_t b) {
   vst3_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3_lane_mf8(mfloat8_t *a, mfloat8x8x3_t b) {
+  vst3_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -4525,6 +4854,20 @@ void test_vst4q_lane_s64(int64_t  *a, int64x2x4_t b) {
   vst4q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4q_lane_mf8(mfloat8_t *a, mfloat8x16x4_t b) {
+  vst4q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -4821,6 +5164,20 @@ void test_vst4_lane_s64(int64_t  *a, int64x1x4_t b) {
   vst4_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4_lane_mf8(mfloat8_t *a, mfloat8x8x4_t b) {
+  vst4_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c
index 2baba98cc5050..e7520a504b121 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c
@@ -3,6 +3,7 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve -target-feature +f64mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +f64mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +f64mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve -target-feature +f64mm -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 
 // REQUIRES: aarch64-registered-target
 
@@ -15,6 +16,20 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svld1ro_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1ro_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svld1ro_mf8(svbool_t pg, mfloat8_t const *base) {
+  return SVE_ACLE_FUNC(svld1ro, _mf8, , )(pg, base);
+}
+
 // CHECK-LABEL: @test_svld1ro_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c
index 9784e1b6b8001..0e7455d413274 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c
@@ -23,6 +23,21 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svld1rq_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1rq_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svld1rq_mf8(svbool_t pg, mfloat8_t const *base) MODE_ATTR
+{
+  return SVE_ACLE_FUNC(svld1rq,_mf8,,)(pg, base);
+}
+
 // CHECK-LABEL: @test_svld1rq_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c
index a647eb0469f9d..ba4091660bfae 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c
@@ -14,6 +14,21 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svldff1_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svldff1_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svldff1_mf8(svbool_t pg, const mfloat8_t *base)
+{
+  return SVE_ACLE_FUNC(svldff1,_mf8,,)(pg, base);
+}
+
 // CHECK-LABEL: @test_svldff1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
@@ -197,6 +212,23 @@ svfloat64_t test_svldff1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldff1,_f64,,)(pg, base);
 }
 
+// CHECK-LABEL: @test_svldff1_vnum_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_mf8u10__SVBool_tPKu6__mfp8l(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+svmfloat8_t test_svldff1_vnum_mf8(svbool_t pg, mfloat8_t const *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldff1_vnum,_mf8,,)(pg, base, vnum);
+}
+
 // CHECK-LABEL: @test_svldff1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c
index 017f960d37061..8e738d839cd85 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c
@@ -14,6 +14,21 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svldnf1_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svldnf1_mf8(svbool_t pg, mfloat8_t const *base)
+{
+  return SVE_ACLE_FUNC(svldnf1,_mf8,,)(pg, base);
+}
+
 // CHECK-LABEL: @test_svldnf1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
@@ -197,6 +212,23 @@ svfloat64_t test_svldnf1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldnf1,_f64,,)(pg, base);
 }
 
+// CHECK-LABEL: @test_svldnf1_vnum_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_mf8u10__SVBool_tPKu6__mfp8l(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+svmfloat8_t test_svldnf1_vnum_mf8(svbool_t pg, mfloat8_t const *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnf1_vnum,_mf8,,)(pg, base, vnum);
+}
+
 // CHECK-LABEL: @test_svldnf1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/AMDGPU/full-bf16.c b/clang/test/CodeGen/AMDGPU/full-bf16.c
new file mode 100644
index 0000000000000..d2ec34561cd8e
--- /dev/null
+++ b/clang/test/CodeGen/AMDGPU/full-bf16.c
@@ -0,0 +1,28 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu tahiti -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu fiji -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu gfx900 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu gfx950 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu gfx1010 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu gfx1100 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa-gnu -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck %s
+
+// CHECK-LABEL: define dso_local bfloat @div(
+// CHECK-SAME: bfloat noundef [[A:%.*]], bfloat noundef [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[B_ADDR]] to ptr
+// CHECK-NEXT:    store bfloat [[A]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    store bfloat [[B]], ptr [[B_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load bfloat, ptr [[B_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv bfloat [[TMP0]], [[TMP1]]
+// CHECK-NEXT:    ret bfloat [[DIV]]
+//
+__bf16 div(__bf16 a, __bf16 b) {
+  return a / b;
+}
diff --git a/clang/test/CodeGen/LoongArch/bitint.c b/clang/test/CodeGen/LoongArch/bitint.c
new file mode 100644
index 0000000000000..f346f569d0eb0
--- /dev/null
+++ b/clang/test/CodeGen/LoongArch/bitint.c
@@ -0,0 +1,122 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple loongarch32 -fexperimental-max-bitint-width=1024 -emit-llvm %s -o - | FileCheck %s --check-prefix=LA32
+// RUN: %clang_cc1 -triple loongarch64 -fexperimental-max-bitint-width=1024 -emit-llvm %s -o - | FileCheck %s --check-prefix=LA64
+
+/// Testing the alignment and size of _BitInt(N) for different values of N.
+
+void pass_BitInt7(_BitInt(7));
+void pass_BitInt65(_BitInt(65));
+void pass_BitInt129(_BitInt(129));
+
+// LA32-LABEL: define dso_local void @example_BitInt(
+// LA32-SAME: ) #[[ATTR0:[0-9]+]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    [[L7:%.*]] = alloca i8, align 1
+// LA32-NEXT:    [[L65:%.*]] = alloca i128, align 16
+// LA32-NEXT:    [[L129:%.*]] = alloca [32 x i8], align 16
+// LA32-NEXT:    [[BYVAL_TEMP:%.*]] = alloca i128, align 16
+// LA32-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca [32 x i8], align 16
+// LA32-NEXT:    store i8 0, ptr [[L7]], align 1
+// LA32-NEXT:    store i128 0, ptr [[L65]], align 16
+// LA32-NEXT:    store i256 0, ptr [[L129]], align 16
+// LA32-NEXT:    [[TMP0:%.*]] = load i8, ptr [[L7]], align 1
+// LA32-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP0]] to i7
+// LA32-NEXT:    call void @pass_BitInt7(i7 noundef signext [[LOADEDV]])
+// LA32-NEXT:    [[TMP1:%.*]] = load i128, ptr [[L65]], align 16
+// LA32-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
+// LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV1]] to i128
+// LA32-NEXT:    store i128 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
+// LA32-NEXT:    call void @pass_BitInt65(ptr noundef [[BYVAL_TEMP]])
+// LA32-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
+// LA32-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
+// LA32-NEXT:    [[STOREDV4:%.*]] = sext i129 [[LOADEDV2]] to i256
+// LA32-NEXT:    store i256 [[STOREDV4]], ptr [[BYVAL_TEMP3]], align 16
+// LA32-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP3]])
+// LA32-NEXT:    ret void
+//
+// LA64-LABEL: define dso_local void @example_BitInt(
+// LA64-SAME: ) #[[ATTR0:[0-9]+]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    [[L7:%.*]] = alloca i8, align 1
+// LA64-NEXT:    [[L65:%.*]] = alloca i128, align 16
+// LA64-NEXT:    [[L129:%.*]] = alloca i256, align 16
+// LA64-NEXT:    [[BYVAL_TEMP:%.*]] = alloca i256, align 16
+// LA64-NEXT:    store i8 0, ptr [[L7]], align 1
+// LA64-NEXT:    store i128 0, ptr [[L65]], align 16
+// LA64-NEXT:    store i256 0, ptr [[L129]], align 16
+// LA64-NEXT:    [[TMP0:%.*]] = load i8, ptr [[L7]], align 1
+// LA64-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP0]] to i7
+// LA64-NEXT:    call void @pass_BitInt7(i7 noundef signext [[LOADEDV]])
+// LA64-NEXT:    [[TMP1:%.*]] = load i128, ptr [[L65]], align 16
+// LA64-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
+// LA64-NEXT:    call void @pass_BitInt65(i65 noundef [[LOADEDV1]])
+// LA64-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
+// LA64-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
+// LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV2]] to i256
+// LA64-NEXT:    store i256 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
+// LA64-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP]])
+// LA64-NEXT:    ret void
+//
+void example_BitInt(void) {
+    _BitInt(7) l7 = {0};
+    _BitInt(65) l65 = {0};
+    _BitInt(129) l129 = {0};
+    pass_BitInt7(l7);
+    pass_BitInt65(l65);
+    pass_BitInt129(l129);
+}
+
+// LA32-LABEL: define dso_local signext i7 @return_large_BitInt7(
+// LA32-SAME: ) #[[ATTR0]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    ret i7 0
+//
+// LA64-LABEL: define dso_local signext i7 @return_large_BitInt7(
+// LA64-SAME: ) #[[ATTR0]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    ret i7 0
+//
+_BitInt(7) return_large_BitInt7(void) { return 0; }
+// LA32-LABEL: define dso_local void @return_large_BitInt65(
+// LA32-SAME: ptr dead_on_unwind noalias writable sret(i128) align 16 [[AGG_RESULT:%.*]]) #[[ATTR0]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 4
+// LA32-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 4
+// LA32-NEXT:    store i128 0, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[TMP0:%.*]] = load i128, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[LOADEDV:%.*]] = trunc i128 [[TMP0]] to i65
+// LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV]] to i128
+// LA32-NEXT:    store i128 [[STOREDV]], ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    ret void
+//
+// LA64-LABEL: define dso_local i65 @return_large_BitInt65(
+// LA64-SAME: ) #[[ATTR0]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    ret i65 0
+//
+_BitInt(65) return_large_BitInt65(void) { return 0; }
+// LA32-LABEL: define dso_local void @return_large_BitInt129(
+// LA32-SAME: ptr dead_on_unwind noalias writable sret([32 x i8]) align 16 [[AGG_RESULT:%.*]]) #[[ATTR0]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 4
+// LA32-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 4
+// LA32-NEXT:    store i256 0, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[TMP0:%.*]] = load i256, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
+// LA32-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
+// LA32-NEXT:    store i256 [[STOREDV]], ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    ret void
+//
+// LA64-LABEL: define dso_local void @return_large_BitInt129(
+// LA64-SAME: ptr dead_on_unwind noalias writable sret(i256) align 16 [[AGG_RESULT:%.*]]) #[[ATTR0]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 8
+// LA64-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 8
+// LA64-NEXT:    store i256 0, ptr [[AGG_RESULT]], align 16
+// LA64-NEXT:    [[TMP0:%.*]] = load i256, ptr [[AGG_RESULT]], align 16
+// LA64-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
+// LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
+// LA64-NEXT:    store i256 [[STOREDV]], ptr [[AGG_RESULT]], align 16
+// LA64-NEXT:    ret void
+//
+_BitInt(129) return_large_BitInt129(void) { return 0; }
diff --git a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c
index c5ecf0c929af8..cab6182ac61c3 100644
--- a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c
+++ b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c
@@ -8,13 +8,4 @@ void test(void) {
   register float a1 asm ("$f32");
 // CHECK: :[[#@LINE+1]]:24: error: unknown register name '$foo' in asm
   register int a2 asm ("$foo");
-
-/// Names not prefixed with '$' are invalid.
-
-// CHECK: :[[#@LINE+1]]:26: error: unknown register name 'f0' in asm
-  register float a5 asm ("f0");
-// CHECK: :[[#@LINE+1]]:26: error: unknown register name 'fa0' in asm
-  register float a6 asm ("fa0");
-// CHECK: :[[#@LINE+1]]:15: error: unknown register name 'fcc0' in asm
-  asm ("" ::: "fcc0");
 }
diff --git a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c
index e1015f6fc01d5..80623c7fc3560 100644
--- a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c
+++ b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c
@@ -4,7 +4,8 @@
 /// Check GCC register names and alias can be used in register variable definition.
 
 // CHECK-LABEL: @test_r0
-// CHECK: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r0}"(i32 undef)
 void test_r0() {
     register int a asm ("$r0");
     register int b asm ("r0");
@@ -13,7 +14,8 @@ void test_r0() {
 }
 
 // CHECK-LABEL: @test_r12
-// CHECK: call void asm sideeffect "", "{$r12}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r12}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r12}"(i32 undef)
 void test_r12() {
     register int a asm ("$r12");
     register int b asm ("r12");
@@ -22,7 +24,8 @@ void test_r12() {
 }
 
 // CHECK-LABEL: @test_r31
-// CHECK: call void asm sideeffect "", "{$r31}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r31}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r31}"(i32 undef)
 void test_r31() {
     register int a asm ("$r31");
     register int b asm ("r31");
@@ -31,7 +34,8 @@ void test_r31() {
 }
 
 // CHECK-LABEL: @test_zero
-// CHECK: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r0}"(i32 undef)
 void test_zero() {
     register int a asm ("$zero");
     register int b asm ("zero");
@@ -40,7 +44,8 @@ void test_zero() {
 }
 
 // CHECK-LABEL: @test_a0
-// CHECK: call void asm sideeffect "", "{$r4}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r4}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r4}"(i32 undef)
 void test_a0() {
     register int a asm ("$a0");
     register int b asm ("a0");
@@ -49,7 +54,8 @@ void test_a0() {
 }
 
 // CHECK-LABEL: @test_t1
-// CHECK: call void asm sideeffect "", "{$r13}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r13}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r13}"(i32 undef)
 void test_t1() {
     register int a asm ("$t1");
     register int b asm ("t1");
@@ -58,7 +64,8 @@ void test_t1() {
 }
 
 // CHECK-LABEL: @test_fp
-// CHECK: call void asm sideeffect "", "{$r22}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r22}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r22}"(i32 undef)
 void test_fp() {
     register int a asm ("$fp");
     register int b asm ("fp");
@@ -67,7 +74,8 @@ void test_fp() {
 }
 
 // CHECK-LABEL: @test_s2
-// CHECK: call void asm sideeffect "", "{$r25}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r25}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r25}"(i32 undef)
 void test_s2() {
     register int a asm ("$s2");
     register int b asm ("s2");
@@ -76,51 +84,73 @@ void test_s2() {
 }
 
 // CHECK-LABEL: @test_f0
-// CHECK: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f0}"(float undef)
 void test_f0() {
     register float a asm ("$f0");
+    register float b asm ("f0");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_f14
-// CHECK: call void asm sideeffect "", "{$f14}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f14}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f14}"(float undef)
 void test_f14() {
     register float a asm ("$f14");
+    register float b asm ("f14");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_f31
-// CHECK: call void asm sideeffect "", "{$f31}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f31}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f31}"(float undef)
 void test_f31() {
     register float a asm ("$f31");
+    register float b asm ("f31");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_fa0
-// CHECK: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f0}"(float undef)
 void test_fa0() {
     register float a asm ("$fa0");
+    register float b asm ("fa0");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_ft1
-// CHECK: call void asm sideeffect "", "{$f9}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f9}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f9}"(float undef)
 void test_ft1() {
     register float a asm ("$ft1");
+    register float b asm ("ft1");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_fs2
-// CHECK: call void asm sideeffect "", "{$f26}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f26}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f26}"(float undef)
 void test_fs2() {
     register float a asm ("$fs2");
+    register float b asm ("fs2");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_fcc
-// CHECK: call void asm sideeffect "", "~{$fcc0}"()
-// CHECK: call void asm sideeffect "", "~{$fcc7}"()
+// CHECK-COUNT-2: call void asm sideeffect "", "~{$fcc0}"()
+// CHECK-NOT: call void asm sideeffect "", "~{$fcc0}"()
+// CHECK-COUNT-2: call void asm sideeffect "", "~{$fcc7}"()
+// CHECK-NOT: call void asm sideeffect "", "~{$fcc7}"()
 void test_fcc() {
     asm ("" ::: "$fcc0");
+    asm ("" ::: "fcc0");
     asm ("" ::: "$fcc7");
+    asm ("" ::: "fcc7");
 }
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfncvtbf16s.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfncvtbf16s.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfncvtbf16s.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadb.c
index 48e3a1b0fc295..ec0ad7d23ac8b 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadt.c
index e435a31eb3314..e0627b3e0f6cc 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfwcvtsbf16.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfwcvtsbf16.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfwcvtsbf16.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfncvtbf16s.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfncvtbf16s.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfncvtbf16s.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadb.c
index 46c6648bc7a82..686b476b0d594 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadt.c
index 62fe9b7bc10e7..228d474f2c221 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfwcvtsbf16.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfwcvtsbf16.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfwcvtsbf16.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfncvtbf16s.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfncvtbf16s.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfncvtbf16s.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadb.c
index bda6a5292ae3d..3c6143dc32606 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadt.c
index bfcc1e18bd351..f8852fba2f991 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfwcvtsbf16.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfwcvtsbf16.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfwcvtsbf16.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfncvtbf16s.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfncvtbf16s.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfncvtbf16s.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadb.c
index c7e6abc7a7e19..69e025709ec46 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadt.c
index 7e351e096821e..a8270fa18cc7e 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfwcvtsbf16.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfwcvtsbf16.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfwcvtsbf16.c
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv.c b/clang/test/CodeGen/RISCV/riscv-vector-callingconv.c
index 6a71d1a9db81f..997deec17ba08 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv.c
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv.c
@@ -11,7 +11,7 @@ void __attribute__((riscv_vector_cc)) test_no_attribute(int x) { } // expected-e
 [[riscv::vector_cc]] int var2; // expected-warning {{'vector_cc' only applies to function types; type here is 'int'}}
 
 [[riscv::vector_cc]] void func2();
-[[riscv::vector_cc(1)]] void func_invalid2(); // expected-error {{'vector_cc' attribute takes no arguments}}
+[[riscv::vector_cc(1)]] void func_invalid2(); // expected-error {{'riscv::vector_cc' attribute takes no arguments}}
 
 void test_no_attribute2(int); // expected-note {{previous declaration is here}}
 [[riscv::vector_cc]] void test_no_attribute2(int x) { } // expected-error {{function declared 'riscv_vector_cc' here was previously declared without calling convention}}
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv.cpp b/clang/test/CodeGen/RISCV/riscv-vector-callingconv.cpp
index f041b0d36529c..9386c64b5c23f 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv.cpp
@@ -20,7 +20,7 @@ void test_lambda() {
 [[riscv::vector_cc]] int var2; // expected-warning {{'vector_cc' only applies to function types; type here is 'int'}}
 
 [[riscv::vector_cc]] void func2();
-[[riscv::vector_cc(1)]] void func_invalid2(); // expected-error {{'vector_cc' attribute takes no arguments}}
+[[riscv::vector_cc(1)]] void func_invalid2(); // expected-error {{'riscv::vector_cc' attribute takes no arguments}}
 
 void test_no_attribute2(int); // expected-note {{previous declaration is here}}
 [[riscv::vector_cc]] void test_no_attribute2(int x) { } // expected-error {{function declared 'riscv_vector_cc' here was previously declared without calling convention}}
diff --git a/clang/test/CodeGen/RISCV/riscv-xcvalu-c-api.c b/clang/test/CodeGen/RISCV/riscv-xcvalu-c-api.c
index a0409e346d708..04dc5a78e4cea 100644
--- a/clang/test/CodeGen/RISCV/riscv-xcvalu-c-api.c
+++ b/clang/test/CodeGen/RISCV/riscv-xcvalu-c-api.c
@@ -1,26 +1,14 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple riscv32 -target-feature +xcvalu -emit-llvm %s -o - \
-// RUN:     | FileCheck %s
+// RUN:     -disable-O0-optnone | opt -S -passes=mem2reg | FileCheck %s
 
 #include <stdint.h>
 #include <riscv_corev_alu.h>
 
 // CHECK-LABEL: @test_alu_sle(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = icmp sle i32 [[TMP2]], [[TMP3]]
-// CHECK-NEXT:    [[SLE_I:%.*]] = zext i1 [[TMP4]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[A:%.*]], [[B:%.*]]
+// CHECK-NEXT:    [[SLE_I:%.*]] = zext i1 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[SLE_I]]
 //
 int test_alu_sle(int32_t a, int32_t b) {
@@ -29,20 +17,8 @@ int test_alu_sle(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_sleu(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = icmp ule i32 [[TMP2]], [[TMP3]]
-// CHECK-NEXT:    [[SLEU_I:%.*]] = zext i1 [[TMP4]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
+// CHECK-NEXT:    [[SLEU_I:%.*]] = zext i1 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[SLEU_I]]
 //
 int test_alu_sleu(uint32_t a, uint32_t b) {
@@ -51,19 +27,7 @@ int test_alu_sleu(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_min(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[ELT_MIN_I:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP2]], i32 [[TMP3]])
+// CHECK-NEXT:    [[ELT_MIN_I:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 // CHECK-NEXT:    ret i32 [[ELT_MIN_I]]
 //
 int test_alu_min(int32_t a, int32_t b) {
@@ -72,19 +36,7 @@ int test_alu_min(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_minu(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[ELT_MIN_I:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP2]], i32 [[TMP3]])
+// CHECK-NEXT:    [[ELT_MIN_I:%.*]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 // CHECK-NEXT:    ret i32 [[ELT_MIN_I]]
 //
 int test_alu_minu(uint32_t a, uint32_t b) {
@@ -93,19 +45,7 @@ int test_alu_minu(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_max(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[ELT_MAX_I:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP2]], i32 [[TMP3]])
+// CHECK-NEXT:    [[ELT_MAX_I:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 // CHECK-NEXT:    ret i32 [[ELT_MAX_I]]
 //
 int test_alu_max(int32_t a, int32_t b) {
@@ -114,19 +54,7 @@ int test_alu_max(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_maxu(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[ELT_MAX_I:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP2]], i32 [[TMP3]])
+// CHECK-NEXT:    [[ELT_MAX_I:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 // CHECK-NEXT:    ret i32 [[ELT_MAX_I]]
 //
 int test_alu_maxu(uint32_t a, uint32_t b) {
@@ -135,14 +63,8 @@ int test_alu_maxu(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_exths(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    store i16 [[A:%.*]], ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    store i16 [[TMP0]], ptr [[A_ADDR_I]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[A_ADDR_I]], align 2
-// CHECK-NEXT:    [[CONV_I:%.*]] = sext i16 [[TMP1]] to i32
-// CHECK-NEXT:    [[EXTHS_I:%.*]] = sext i16 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV_I:%.*]] = sext i16 [[A:%.*]] to i32
+// CHECK-NEXT:    [[EXTHS_I:%.*]] = sext i16 [[A]] to i32
 // CHECK-NEXT:    ret i32 [[EXTHS_I]]
 //
 int test_alu_exths(int16_t a) {
@@ -151,14 +73,8 @@ int test_alu_exths(int16_t a) {
 
 // CHECK-LABEL: @test_alu_exthz(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    store i16 [[A:%.*]], ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    store i16 [[TMP0]], ptr [[A_ADDR_I]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[A_ADDR_I]], align 2
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i16 [[TMP1]] to i32
-// CHECK-NEXT:    [[EXTHZ_I:%.*]] = zext i16 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i16 [[A:%.*]] to i32
+// CHECK-NEXT:    [[EXTHZ_I:%.*]] = zext i16 [[A]] to i32
 // CHECK-NEXT:    ret i32 [[EXTHZ_I]]
 //
 int test_alu_exthz(uint16_t a) {
@@ -167,14 +83,8 @@ int test_alu_exthz(uint16_t a) {
 
 // CHECK-LABEL: @test_alu_extbs(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    store i8 [[A:%.*]], ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    store i8 [[TMP0]], ptr [[A_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[A_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = sext i8 [[TMP1]] to i32
-// CHECK-NEXT:    [[EXTBS_I:%.*]] = sext i8 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV_I:%.*]] = sext i8 [[A:%.*]] to i32
+// CHECK-NEXT:    [[EXTBS_I:%.*]] = sext i8 [[A]] to i32
 // CHECK-NEXT:    ret i32 [[EXTBS_I]]
 //
 int test_alu_extbs(int8_t a) {
@@ -183,14 +93,8 @@ int test_alu_extbs(int8_t a) {
 
 // CHECK-LABEL: @test_alu_extbz(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    store i8 [[A:%.*]], ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    store i8 [[TMP0]], ptr [[A_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[A_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP1]] to i32
-// CHECK-NEXT:    [[EXTBZ_I:%.*]] = zext i8 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[A:%.*]] to i32
+// CHECK-NEXT:    [[EXTBZ_I:%.*]] = zext i8 [[A]] to i32
 // CHECK-NEXT:    ret i32 [[EXTBZ_I]]
 //
 int test_alu_extbz(uint8_t a) {
@@ -199,17 +103,8 @@ int test_alu_extbz(uint8_t a) {
 
 // CHECK-LABEL: @test_alu_clip(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 0, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.riscv.cv.alu.clip(i32 [[TMP1]], i32 [[TMP2]])
-// CHECK-NEXT:    ret i32 [[TMP3]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.clip(i32 [[A:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_clip(int32_t a) {
   return __riscv_cv_alu_clip(a, 0);
@@ -217,17 +112,8 @@ int test_alu_clip(int32_t a) {
 
 // CHECK-LABEL: @test_alu_clipu(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 0, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.riscv.cv.alu.clipu(i32 [[TMP1]], i32 [[TMP2]])
-// CHECK-NEXT:    ret i32 [[TMP3]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.clipu(i32 [[A:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_clipu(uint32_t a) {
   return __riscv_cv_alu_clipu(a, 0);
@@ -235,24 +121,9 @@ int test_alu_clipu(uint32_t a) {
 
 // CHECK-LABEL: @test_alu_addN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.addN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.addN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_addN(int32_t a, int32_t b) {
   return __riscv_cv_alu_addN(a, b, 0);
@@ -260,24 +131,9 @@ int test_alu_addN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_adduN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.adduN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.adduN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_adduN(uint32_t a, uint32_t b) {
   return __riscv_cv_alu_adduN(a, b, 0);
@@ -285,24 +141,9 @@ int test_alu_adduN(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_addRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.addRN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.addRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_addRN(int32_t a, int32_t b) {
   return __riscv_cv_alu_addRN(a, b, 0);
@@ -310,24 +151,9 @@ int test_alu_addRN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_adduRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.adduRN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.adduRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_adduRN(uint32_t a, uint32_t b) {
   return __riscv_cv_alu_adduRN(a, b, 0);
@@ -335,24 +161,9 @@ int test_alu_adduRN(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_subN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.subN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subN(int32_t a, int32_t b) {
   return __riscv_cv_alu_subN(a, b, 0);
@@ -360,24 +171,9 @@ int test_alu_subN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_subuN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.subuN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subuN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subuN(uint32_t a, uint32_t b) {
   return __riscv_cv_alu_subuN(a, b, 0);
@@ -385,24 +181,9 @@ int test_alu_subuN(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_subRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.subRN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subRN(int32_t a, int32_t b) {
   return __riscv_cv_alu_subRN(a, b, 0);
@@ -410,24 +191,9 @@ int test_alu_subRN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_subuRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR_I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[SHFT_ADDR_I:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[TMP0]], ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    store i32 [[TMP1]], ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    store i8 0, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[B_ADDR_I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[SHFT_ADDR_I]], align 1
-// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 [[TMP4]] to i32
-// CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.riscv.cv.alu.subuRN(i32 [[TMP2]], i32 [[TMP3]], i32 [[CONV_I]])
-// CHECK-NEXT:    ret i32 [[TMP5]]
+// CHECK-NEXT:    [[CONV_I:%.*]] = zext i8 0 to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subuRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 [[CONV_I]])
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subuRN(uint32_t a, uint32_t b) {
   return __riscv_cv_alu_subuRN(a, b, 0);
diff --git a/clang/test/CodeGen/RISCV/riscv-xcvalu.c b/clang/test/CodeGen/RISCV/riscv-xcvalu.c
index 858f53ac49ba0..8a4fbbc3a115e 100644
--- a/clang/test/CodeGen/RISCV/riscv-xcvalu.c
+++ b/clang/test/CodeGen/RISCV/riscv-xcvalu.c
@@ -1,16 +1,13 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple riscv32 -target-feature +xcvalu -emit-llvm %s -o - \
-// RUN:     | FileCheck %s
+// RUN:     -disable-O0-optnone | opt -S -passes=mem2reg | FileCheck %s
 
 #include <stdint.h>
 
 // CHECK-LABEL: @test_abs(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.abs.i32(i32 [[TMP0]], i1 true)
-// CHECK-NEXT:    ret i32 [[TMP1]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.abs.i32(i32 [[A:%.*]], i1 true)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_abs(int a) {
   return __builtin_abs(a);
@@ -18,14 +15,8 @@ int test_abs(int a) {
 
 // CHECK-LABEL: @test_alu_sle(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[TMP0]], [[TMP1]]
-// CHECK-NEXT:    [[SLE:%.*]] = zext i1 [[TMP2]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[A:%.*]], [[B:%.*]]
+// CHECK-NEXT:    [[SLE:%.*]] = zext i1 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[SLE]]
 //
 int test_alu_sle(int32_t a, int32_t b) {
@@ -34,14 +25,8 @@ int test_alu_sle(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_sleu(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[TMP0]], [[TMP1]]
-// CHECK-NEXT:    [[SLEU:%.*]] = zext i1 [[TMP2]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
+// CHECK-NEXT:    [[SLEU:%.*]] = zext i1 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[SLEU]]
 //
 int test_alu_sleu(uint32_t a, uint32_t b) {
@@ -50,12 +35,9 @@ int test_alu_sleu(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_exths(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    store i16 [[A:%.*]], ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[CONV]] to i16
-// CHECK-NEXT:    [[EXTHS:%.*]] = sext i16 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[A:%.*]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[CONV]] to i16
+// CHECK-NEXT:    [[EXTHS:%.*]] = sext i16 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[EXTHS]]
 //
 int test_alu_exths(int16_t a) {
@@ -64,12 +46,9 @@ int test_alu_exths(int16_t a) {
 
 // CHECK-LABEL: @test_alu_exthz(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    store i16 [[A:%.*]], ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[TMP0]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[CONV]] to i16
-// CHECK-NEXT:    [[EXTHZ:%.*]] = zext i16 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[A:%.*]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[CONV]] to i16
+// CHECK-NEXT:    [[EXTHZ:%.*]] = zext i16 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[EXTHZ]]
 //
 int test_alu_exthz(uint16_t a) {
@@ -78,12 +57,9 @@ int test_alu_exthz(uint16_t a) {
 
 // CHECK-LABEL: @test_alu_extbs(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    store i8 [[A:%.*]], ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP0]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[CONV]] to i8
-// CHECK-NEXT:    [[EXTBS:%.*]] = sext i8 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[A:%.*]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[CONV]] to i8
+// CHECK-NEXT:    [[EXTBS:%.*]] = sext i8 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[EXTBS]]
 //
 int test_alu_extbs(int8_t a) {
@@ -92,12 +68,9 @@ int test_alu_extbs(int8_t a) {
 
 // CHECK-LABEL: @test_alu_extbz(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i8, align 1
-// CHECK-NEXT:    store i8 [[A:%.*]], ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_ADDR]], align 1
-// CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[CONV]] to i8
-// CHECK-NEXT:    [[EXTBZ:%.*]] = zext i8 [[TMP1]] to i32
+// CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[A:%.*]] to i32
+// CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[CONV]] to i8
+// CHECK-NEXT:    [[EXTBZ:%.*]] = zext i8 [[TMP0]] to i32
 // CHECK-NEXT:    ret i32 [[EXTBZ]]
 //
 int test_alu_extbz(uint8_t a) {
@@ -106,11 +79,8 @@ int test_alu_extbz(uint8_t a) {
 
 // CHECK-LABEL: @test_alu_clip(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.cv.alu.clip(i32 [[TMP0]], i32 15)
-// CHECK-NEXT:    ret i32 [[TMP1]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.clip(i32 [[A:%.*]], i32 15)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_clip(int32_t a) {
   return __builtin_riscv_cv_alu_clip(a, 15);
@@ -118,11 +88,8 @@ int test_alu_clip(int32_t a) {
 
 // CHECK-LABEL: @test_alu_clipu(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.cv.alu.clipu(i32 [[TMP0]], i32 15)
-// CHECK-NEXT:    ret i32 [[TMP1]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.clipu(i32 [[A:%.*]], i32 15)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_clipu(uint32_t a) {
   return __builtin_riscv_cv_alu_clipu(a, 15);
@@ -130,14 +97,8 @@ int test_alu_clipu(uint32_t a) {
 
 // CHECK-LABEL: @test_alu_addN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.addN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.addN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_addN(int32_t a, int32_t b) {
   return __builtin_riscv_cv_alu_addN(a, b, 0);
@@ -145,14 +106,8 @@ int test_alu_addN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_adduN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.adduN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.adduN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_adduN(uint32_t a, uint32_t b) {
   return __builtin_riscv_cv_alu_adduN(a, b, 0);
@@ -160,14 +115,8 @@ int test_alu_adduN(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_addRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.addRN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.addRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_addRN(int32_t a, int32_t b) {
   return __builtin_riscv_cv_alu_addRN(a, b, 0);
@@ -175,14 +124,8 @@ int test_alu_addRN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_adduRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.adduRN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.adduRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_adduRN(uint32_t a, uint32_t b) {
   return __builtin_riscv_cv_alu_adduRN(a, b, 0);
@@ -190,14 +133,8 @@ int test_alu_adduRN(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_subN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.subN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subN(int32_t a, int32_t b) {
   return __builtin_riscv_cv_alu_subN(a, b, 0);
@@ -205,14 +142,8 @@ int test_alu_subN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_subuN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.subuN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subuN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subuN(uint32_t a, uint32_t b) {
   return __builtin_riscv_cv_alu_subuN(a, b, 0);
@@ -220,14 +151,8 @@ int test_alu_subuN(uint32_t a, uint32_t b) {
 
 // CHECK-LABEL: @test_alu_subRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.subRN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subRN(int32_t a, int32_t b) {
   return __builtin_riscv_cv_alu_subRN(a, b, 0);
@@ -235,14 +160,8 @@ int test_alu_subRN(int32_t a, int32_t b) {
 
 // CHECK-LABEL: @test_alu_subuRN(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.riscv.cv.alu.subuRN(i32 [[TMP0]], i32 [[TMP1]], i32 0)
-// CHECK-NEXT:    ret i32 [[TMP2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.cv.alu.subuRN(i32 [[A:%.*]], i32 [[B:%.*]], i32 0)
+// CHECK-NEXT:    ret i32 [[TMP0]]
 //
 int test_alu_subuRN(uint32_t a, uint32_t b) {
   return __builtin_riscv_cv_alu_subuRN(a, b, 0);
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vrgatherei16.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vrgatherei16.c
new file mode 100644
index 0000000000000..2cab8f92b0dbb
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vrgatherei16.c
@@ -0,0 +1,140 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4(vbfloat16mf4_t vs2,
+                                            vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf4(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2(vbfloat16mf2_t vs2,
+                                            vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf2(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1(vbfloat16m1_t vs2, vuint16m1_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m1(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2(vbfloat16m2_t vs2, vuint16m2_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m2(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4(vbfloat16m4_t vs2, vuint16m4_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m4(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8(vbfloat16m8_t vs2, vuint16m8_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m8(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_m(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_m(vbool64_t vm, vbfloat16mf4_t vs2,
+                                              vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf4_m(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_m(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_m(vbool32_t vm, vbfloat16mf2_t vs2,
+                                              vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf2_m(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_m(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_m(vbool16_t vm, vbfloat16m1_t vs2,
+                                            vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m1_m(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_m(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_m(vbool8_t vm, vbfloat16m2_t vs2,
+                                            vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m2_m(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_m(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_m(vbool4_t vm, vbfloat16m4_t vs2,
+                                            vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m4_m(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_m(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_m(vbool2_t vm, vbfloat16m8_t vs2,
+                                            vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m8_m(vm, vs2, vs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vslidedown.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vslidedown.c
new file mode 100644
index 0000000000000..1d1a9f93065c7
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vslidedown.c
@@ -0,0 +1,140 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4(vbfloat16mf4_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf4(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2(vbfloat16mf2_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf2(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1(vbfloat16m1_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown_vx_bf16m1(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2(vbfloat16m2_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown_vx_bf16m2(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4(vbfloat16m4_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown_vx_bf16m4(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8(vbfloat16m8_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown_vx_bf16m8(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_m(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_m(vbool64_t vm, vbfloat16mf4_t vs2,
+                                            size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf4_m(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_m(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_m(vbool32_t vm, vbfloat16mf2_t vs2,
+                                            size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf2_m(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_m(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_m(vbool16_t vm, vbfloat16m1_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m1_m(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_m(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_m(vbool8_t vm, vbfloat16m2_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m2_m(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_m(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_m(vbool4_t vm, vbfloat16m4_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m4_m(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_m(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_m(vbool2_t vm, vbfloat16m8_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m8_m(vm, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vslideup.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vslideup.c
new file mode 100644
index 0000000000000..ff59ac19007d0
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/bfloat16/vslideup.c
@@ -0,0 +1,146 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4(vbfloat16mf4_t vd, vbfloat16mf4_t vs2,
+                                        size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf4(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2(vbfloat16mf2_t vd, vbfloat16mf2_t vs2,
+                                        size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf2(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1(vbfloat16m1_t vd, vbfloat16m1_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m1(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2(vbfloat16m2_t vd, vbfloat16m2_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m2(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4(vbfloat16m4_t vd, vbfloat16m4_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m4(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8(vbfloat16m8_t vd, vbfloat16m8_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m8(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_m(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_m(vbool64_t vm, vbfloat16mf4_t vd,
+                                          vbfloat16mf4_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup_vx_bf16mf4_m(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_m(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_m(vbool32_t vm, vbfloat16mf2_t vd,
+                                          vbfloat16mf2_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup_vx_bf16mf2_m(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_m(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_m(vbool16_t vm, vbfloat16m1_t vd,
+                                        vbfloat16m1_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup_vx_bf16m1_m(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_m(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_m(vbool8_t vm, vbfloat16m2_t vd,
+                                        vbfloat16m2_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup_vx_bf16m2_m(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_m(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_m(vbool4_t vm, vbfloat16m4_t vd,
+                                        vbfloat16m4_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup_vx_bf16m4_m(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_m(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_m(vbool2_t vm, vbfloat16m8_t vd,
+                                        vbfloat16m8_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup_vx_bf16m8_m(vm, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vrgatherei16.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vrgatherei16.c
new file mode 100644
index 0000000000000..96bd28edba20e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vrgatherei16.c
@@ -0,0 +1,140 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4(vbfloat16mf4_t vs2,
+                                            vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2(vbfloat16mf2_t vs2,
+                                            vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1(vbfloat16m1_t vs2, vuint16m1_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2(vbfloat16m2_t vs2, vuint16m2_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4(vbfloat16m4_t vs2, vuint16m4_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8(vbfloat16m8_t vs2, vuint16m8_t vs1,
+                                          size_t vl) {
+  return __riscv_vrgatherei16(vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_m(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_m(vbool64_t vm, vbfloat16mf4_t vs2,
+                                              vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_m(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_m(vbool32_t vm, vbfloat16mf2_t vs2,
+                                              vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_m(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_m(vbool16_t vm, vbfloat16m1_t vs2,
+                                            vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_m(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_m(vbool8_t vm, vbfloat16m2_t vs2,
+                                            vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_m(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_m(vbool4_t vm, vbfloat16m4_t vs2,
+                                            vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vm, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_m(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_m(vbool2_t vm, vbfloat16m8_t vs2,
+                                            vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16(vm, vs2, vs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vslidedown.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vslidedown.c
new file mode 100644
index 0000000000000..f121bdb4d6def
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vslidedown.c
@@ -0,0 +1,140 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4(vbfloat16mf4_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslidedown(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2(vbfloat16mf2_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslidedown(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1(vbfloat16m1_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2(vbfloat16m2_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4(vbfloat16m4_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8(vbfloat16m8_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslidedown(vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_m(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_m(vbool64_t vm, vbfloat16mf4_t vs2,
+                                            size_t rs1, size_t vl) {
+  return __riscv_vslidedown(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_m(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_m(vbool32_t vm, vbfloat16mf2_t vs2,
+                                            size_t rs1, size_t vl) {
+  return __riscv_vslidedown(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_m(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_m(vbool16_t vm, vbfloat16m1_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_m(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_m(vbool8_t vm, vbfloat16m2_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_m(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_m(vbool4_t vm, vbfloat16m4_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown(vm, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_m(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> poison, <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_m(vbool2_t vm, vbfloat16m8_t vs2,
+                                          size_t rs1, size_t vl) {
+  return __riscv_vslidedown(vm, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vslideup.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vslideup.c
new file mode 100644
index 0000000000000..e70692574f2e6
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/bfloat16/vslideup.c
@@ -0,0 +1,146 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4(vbfloat16mf4_t vd, vbfloat16mf4_t vs2,
+                                        size_t rs1, size_t vl) {
+  return __riscv_vslideup(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2(vbfloat16mf2_t vd, vbfloat16mf2_t vs2,
+                                        size_t rs1, size_t vl) {
+  return __riscv_vslideup(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1(vbfloat16m1_t vd, vbfloat16m1_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2(vbfloat16m2_t vd, vbfloat16m2_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4(vbfloat16m4_t vd, vbfloat16m4_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8(vbfloat16m8_t vd, vbfloat16m8_t vs2,
+                                      size_t rs1, size_t vl) {
+  return __riscv_vslideup(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_m(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_m(vbool64_t vm, vbfloat16mf4_t vd,
+                                          vbfloat16mf4_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_m(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_m(vbool32_t vm, vbfloat16mf2_t vd,
+                                          vbfloat16mf2_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_m(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_m(vbool16_t vm, vbfloat16m1_t vd,
+                                        vbfloat16m1_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_m(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_m(vbool8_t vm, vbfloat16m2_t vd,
+                                        vbfloat16m2_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_m(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_m(vbool4_t vm, vbfloat16m4_t vd,
+                                        vbfloat16m4_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_m(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_m(vbool2_t vm, vbfloat16m8_t vd,
+                                        vbfloat16m8_t vs2, size_t rs1,
+                                        size_t vl) {
+  return __riscv_vslideup(vm, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vrgatherei16.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vrgatherei16.c
new file mode 100644
index 0000000000000..da3c55aaa2ac8
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vrgatherei16.c
@@ -0,0 +1,248 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_tu(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_tu(vbfloat16mf4_t vd, vbfloat16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf4_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_tu(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_tu(vbfloat16mf2_t vd, vbfloat16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf2_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_tu(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_tu(vbfloat16m1_t vd, vbfloat16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m1_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_tu(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_tu(vbfloat16m2_t vd, vbfloat16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m2_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_tu(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_tu(vbfloat16m4_t vd, vbfloat16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m4_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_tu(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_tu(vbfloat16m8_t vd, vbfloat16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m8_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_tum(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_tum(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf4_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_tum(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_tum(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf2_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_tum(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_tum(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m1_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_tum(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_tum(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m2_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_tum(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_tum(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m4_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_tum(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_tum(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m8_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_tumu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_tumu(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf4_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_tumu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_tumu(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf2_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_tumu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_tumu(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m1_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_tumu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_tumu(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m2_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_tumu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_tumu(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m4_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_tumu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_tumu(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m8_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_mu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_mu(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf4_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_mu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_mu(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16mf2_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_mu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_mu(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m1_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_mu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_mu(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m2_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_mu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_mu(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m4_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_mu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_mu(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_vv_bf16m8_mu(vm, vd, vs2, vs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vslidedown.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vslidedown.c
new file mode 100644
index 0000000000000..6af481d5700ec
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vslidedown.c
@@ -0,0 +1,248 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_tu(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_tu(vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf4_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_tu(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_tu(vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf2_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_tu(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_tu(vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m1_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_tu(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_tu(vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m2_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_tu(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_tu(vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m4_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_tu(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_tu(vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m8_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_tum(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_tum(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf4_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_tum(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_tum(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf2_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_tum(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_tum(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m1_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_tum(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_tum(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m2_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_tum(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_tum(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m4_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_tum(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_tum(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m8_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_tumu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_tumu(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf4_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_tumu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_tumu(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf2_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_tumu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_tumu(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m1_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_tumu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_tumu(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m2_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_tumu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_tumu(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m4_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_tumu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_tumu(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m8_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_mu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_mu(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf4_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_mu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_mu(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16mf2_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_mu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_mu(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m1_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_mu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_mu(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m2_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_mu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_mu(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m4_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_mu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_mu(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslidedown_vx_bf16m8_mu(vm, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vslideup.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vslideup.c
new file mode 100644
index 0000000000000..bcc8f7a05a084
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/bfloat16/vslideup.c
@@ -0,0 +1,248 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_tu(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_tu(vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf4_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_tu(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_tu(vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf2_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_tu(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_tu(vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m1_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_tu(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_tu(vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m2_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_tu(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_tu(vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m4_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_tu(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_tu(vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m8_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_tum(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_tum(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf4_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_tum(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_tum(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf2_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_tum(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_tum(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m1_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_tum(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_tum(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m2_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_tum(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_tum(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m4_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_tum(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_tum(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m8_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_tumu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_tumu(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf4_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_tumu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_tumu(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf2_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_tumu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_tumu(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m1_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_tumu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_tumu(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m2_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_tumu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_tumu(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m4_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_tumu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_tumu(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m8_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_mu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_mu(vbool64_t vm, vbfloat16mf4_t vd, vbfloat16mf4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf4_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_mu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_mu(vbool32_t vm, vbfloat16mf2_t vd, vbfloat16mf2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16mf2_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_mu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_mu(vbool16_t vm, vbfloat16m1_t vd, vbfloat16m1_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m1_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_mu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_mu(vbool8_t vm, vbfloat16m2_t vd, vbfloat16m2_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m2_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_mu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_mu(vbool4_t vm, vbfloat16m4_t vd, vbfloat16m4_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m4_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_mu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_mu(vbool2_t vm, vbfloat16m8_t vd, vbfloat16m8_t vs2, size_t rs1, size_t vl) {
+  return __riscv_vslideup_vx_bf16m8_mu(vm, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vrgatherei16.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vrgatherei16.c
new file mode 100644
index 0000000000000..9e68b02d46697
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vrgatherei16.c
@@ -0,0 +1,298 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_tu(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_tu(vbfloat16mf4_t vd,
+                                               vbfloat16mf4_t vs2,
+                                               vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_tu(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_tu(vbfloat16mf2_t vd,
+                                               vbfloat16mf2_t vs2,
+                                               vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_tu(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_tu(vbfloat16m1_t vd,
+                                             vbfloat16m1_t vs2, vuint16m1_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_tu(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_tu(vbfloat16m2_t vd,
+                                             vbfloat16m2_t vs2, vuint16m2_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_tu(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_tu(vbfloat16m4_t vd,
+                                             vbfloat16m4_t vs2, vuint16m4_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_tu(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_tu(vbfloat16m8_t vd,
+                                             vbfloat16m8_t vs2, vuint16m8_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_tu(vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_tum(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_tum(vbool64_t vm, vbfloat16mf4_t vd,
+                                                vbfloat16mf4_t vs2,
+                                                vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_tum(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_tum(vbool32_t vm, vbfloat16mf2_t vd,
+                                                vbfloat16mf2_t vs2,
+                                                vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_tum(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_tum(vbool16_t vm, vbfloat16m1_t vd,
+                                              vbfloat16m1_t vs2,
+                                              vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_tum(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_tum(vbool8_t vm, vbfloat16m2_t vd,
+                                              vbfloat16m2_t vs2,
+                                              vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_tum(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_tum(vbool4_t vm, vbfloat16m4_t vd,
+                                              vbfloat16m4_t vs2,
+                                              vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_tum(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_tum(vbool2_t vm, vbfloat16m8_t vd,
+                                              vbfloat16m8_t vs2,
+                                              vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tum(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_tumu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_tumu(vbool64_t vm,
+                                                 vbfloat16mf4_t vd,
+                                                 vbfloat16mf4_t vs2,
+                                                 vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_tumu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_tumu(vbool32_t vm,
+                                                 vbfloat16mf2_t vd,
+                                                 vbfloat16mf2_t vs2,
+                                                 vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_tumu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_tumu(vbool16_t vm, vbfloat16m1_t vd,
+                                               vbfloat16m1_t vs2,
+                                               vuint16m1_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_tumu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_tumu(vbool8_t vm, vbfloat16m2_t vd,
+                                               vbfloat16m2_t vs2,
+                                               vuint16m2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_tumu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_tumu(vbool4_t vm, vbfloat16m4_t vd,
+                                               vbfloat16m4_t vs2,
+                                               vuint16m4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_tumu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_tumu(vbool2_t vm, vbfloat16m8_t vd,
+                                               vbfloat16m8_t vs2,
+                                               vuint16m8_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_tumu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vrgatherei16_vv_bf16mf4_mu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], <vscale x 1 x i16> [[VS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vrgatherei16_vv_bf16mf4_mu(vbool64_t vm, vbfloat16mf4_t vd,
+                                               vbfloat16mf4_t vs2,
+                                               vuint16mf4_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vrgatherei16_vv_bf16mf2_mu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], <vscale x 2 x i16> [[VS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vrgatherei16_vv_bf16mf2_mu(vbool32_t vm, vbfloat16mf2_t vd,
+                                               vbfloat16mf2_t vs2,
+                                               vuint16mf2_t vs1, size_t vl) {
+  return __riscv_vrgatherei16_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vrgatherei16_vv_bf16m1_mu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], <vscale x 4 x i16> [[VS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vrgatherei16_vv_bf16m1_mu(vbool16_t vm, vbfloat16m1_t vd,
+                                             vbfloat16m1_t vs2, vuint16m1_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vrgatherei16_vv_bf16m2_mu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], <vscale x 8 x i16> [[VS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vrgatherei16_vv_bf16m2_mu(vbool8_t vm, vbfloat16m2_t vd,
+                                             vbfloat16m2_t vs2, vuint16m2_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vrgatherei16_vv_bf16m4_mu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], <vscale x 16 x i16> [[VS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vrgatherei16_vv_bf16m4_mu(vbool4_t vm, vbfloat16m4_t vd,
+                                             vbfloat16m4_t vs2, vuint16m4_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_mu(vm, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vrgatherei16_vv_bf16m8_mu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], <vscale x 32 x i16> [[VS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vrgatherei16_vv_bf16m8_mu(vbool2_t vm, vbfloat16m8_t vd,
+                                             vbfloat16m8_t vs2, vuint16m8_t vs1,
+                                             size_t vl) {
+  return __riscv_vrgatherei16_mu(vm, vd, vs2, vs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vslidedown.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vslidedown.c
new file mode 100644
index 0000000000000..6ac03c2e16ea2
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vslidedown.c
@@ -0,0 +1,292 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_tu(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_tu(vbfloat16mf4_t vd,
+                                             vbfloat16mf4_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_tu(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_tu(vbfloat16mf2_t vd,
+                                             vbfloat16mf2_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_tu(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_tu(vbfloat16m1_t vd, vbfloat16m1_t vs2,
+                                           size_t rs1, size_t vl) {
+  return __riscv_vslidedown_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_tu(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_tu(vbfloat16m2_t vd, vbfloat16m2_t vs2,
+                                           size_t rs1, size_t vl) {
+  return __riscv_vslidedown_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_tu(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_tu(vbfloat16m4_t vd, vbfloat16m4_t vs2,
+                                           size_t rs1, size_t vl) {
+  return __riscv_vslidedown_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_tu(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_tu(vbfloat16m8_t vd, vbfloat16m8_t vs2,
+                                           size_t rs1, size_t vl) {
+  return __riscv_vslidedown_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_tum(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_tum(vbool64_t vm, vbfloat16mf4_t vd,
+                                              vbfloat16mf4_t vs2, size_t rs1,
+                                              size_t vl) {
+  return __riscv_vslidedown_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_tum(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_tum(vbool32_t vm, vbfloat16mf2_t vd,
+                                              vbfloat16mf2_t vs2, size_t rs1,
+                                              size_t vl) {
+  return __riscv_vslidedown_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_tum(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_tum(vbool16_t vm, vbfloat16m1_t vd,
+                                            vbfloat16m1_t vs2, size_t rs1,
+                                            size_t vl) {
+  return __riscv_vslidedown_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_tum(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_tum(vbool8_t vm, vbfloat16m2_t vd,
+                                            vbfloat16m2_t vs2, size_t rs1,
+                                            size_t vl) {
+  return __riscv_vslidedown_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_tum(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_tum(vbool4_t vm, vbfloat16m4_t vd,
+                                            vbfloat16m4_t vs2, size_t rs1,
+                                            size_t vl) {
+  return __riscv_vslidedown_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_tum(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_tum(vbool2_t vm, vbfloat16m8_t vd,
+                                            vbfloat16m8_t vs2, size_t rs1,
+                                            size_t vl) {
+  return __riscv_vslidedown_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_tumu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_tumu(vbool64_t vm, vbfloat16mf4_t vd,
+                                               vbfloat16mf4_t vs2, size_t rs1,
+                                               size_t vl) {
+  return __riscv_vslidedown_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_tumu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_tumu(vbool32_t vm, vbfloat16mf2_t vd,
+                                               vbfloat16mf2_t vs2, size_t rs1,
+                                               size_t vl) {
+  return __riscv_vslidedown_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_tumu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_tumu(vbool16_t vm, vbfloat16m1_t vd,
+                                             vbfloat16m1_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_tumu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_tumu(vbool8_t vm, vbfloat16m2_t vd,
+                                             vbfloat16m2_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_tumu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_tumu(vbool4_t vm, vbfloat16m4_t vd,
+                                             vbfloat16m4_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_tumu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_tumu(vbool2_t vm, vbfloat16m8_t vd,
+                                             vbfloat16m8_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslidedown_vx_bf16mf4_mu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslidedown_vx_bf16mf4_mu(vbool64_t vm, vbfloat16mf4_t vd,
+                                             vbfloat16mf4_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslidedown_vx_bf16mf2_mu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslidedown_vx_bf16mf2_mu(vbool32_t vm, vbfloat16mf2_t vd,
+                                             vbfloat16mf2_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslidedown_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslidedown_vx_bf16m1_mu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslidedown_vx_bf16m1_mu(vbool16_t vm, vbfloat16m1_t vd,
+                                           vbfloat16m1_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslidedown_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslidedown_vx_bf16m2_mu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslidedown_vx_bf16m2_mu(vbool8_t vm, vbfloat16m2_t vd,
+                                           vbfloat16m2_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslidedown_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslidedown_vx_bf16m4_mu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslidedown_vx_bf16m4_mu(vbool4_t vm, vbfloat16m4_t vd,
+                                           vbfloat16m4_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslidedown_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslidedown_vx_bf16m8_mu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslidedown.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslidedown_vx_bf16m8_mu(vbool2_t vm, vbfloat16m8_t vd,
+                                           vbfloat16m8_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslidedown_mu(vm, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vslideup.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vslideup.c
new file mode 100644
index 0000000000000..f3e5c47d4cb6d
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/bfloat16/vslideup.c
@@ -0,0 +1,292 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +zvfbfmin -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <riscv_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_tu(
+// CHECK-RV64-SAME: <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_tu(vbfloat16mf4_t vd,
+                                           vbfloat16mf4_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_tu(
+// CHECK-RV64-SAME: <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_tu(vbfloat16mf2_t vd,
+                                           vbfloat16mf2_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_tu(
+// CHECK-RV64-SAME: <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_tu(vbfloat16m1_t vd, vbfloat16m1_t vs2,
+                                         size_t rs1, size_t vl) {
+  return __riscv_vslideup_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_tu(
+// CHECK-RV64-SAME: <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_tu(vbfloat16m2_t vd, vbfloat16m2_t vs2,
+                                         size_t rs1, size_t vl) {
+  return __riscv_vslideup_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_tu(
+// CHECK-RV64-SAME: <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_tu(vbfloat16m4_t vd, vbfloat16m4_t vs2,
+                                         size_t rs1, size_t vl) {
+  return __riscv_vslideup_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_tu(
+// CHECK-RV64-SAME: <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_tu(vbfloat16m8_t vd, vbfloat16m8_t vs2,
+                                         size_t rs1, size_t vl) {
+  return __riscv_vslideup_tu(vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_tum(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_tum(vbool64_t vm, vbfloat16mf4_t vd,
+                                            vbfloat16mf4_t vs2, size_t rs1,
+                                            size_t vl) {
+  return __riscv_vslideup_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_tum(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_tum(vbool32_t vm, vbfloat16mf2_t vd,
+                                            vbfloat16mf2_t vs2, size_t rs1,
+                                            size_t vl) {
+  return __riscv_vslideup_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_tum(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_tum(vbool16_t vm, vbfloat16m1_t vd,
+                                          vbfloat16m1_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_tum(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_tum(vbool8_t vm, vbfloat16m2_t vd,
+                                          vbfloat16m2_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_tum(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_tum(vbool4_t vm, vbfloat16m4_t vd,
+                                          vbfloat16m4_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_tum(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_tum(vbool2_t vm, vbfloat16m8_t vd,
+                                          vbfloat16m8_t vs2, size_t rs1,
+                                          size_t vl) {
+  return __riscv_vslideup_tum(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_tumu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_tumu(vbool64_t vm, vbfloat16mf4_t vd,
+                                             vbfloat16mf4_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslideup_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_tumu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_tumu(vbool32_t vm, vbfloat16mf2_t vd,
+                                             vbfloat16mf2_t vs2, size_t rs1,
+                                             size_t vl) {
+  return __riscv_vslideup_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_tumu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_tumu(vbool16_t vm, vbfloat16m1_t vd,
+                                           vbfloat16m1_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_tumu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_tumu(vbool8_t vm, vbfloat16m2_t vd,
+                                           vbfloat16m2_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_tumu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_tumu(vbool4_t vm, vbfloat16m4_t vd,
+                                           vbfloat16m4_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_tumu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 0)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_tumu(vbool2_t vm, vbfloat16m8_t vd,
+                                           vbfloat16m8_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_tumu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_vslideup_vx_bf16mf4_mu(
+// CHECK-RV64-SAME: <vscale x 1 x i1> [[VM:%.*]], <vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 1 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_vslideup_vx_bf16mf4_mu(vbool64_t vm, vbfloat16mf4_t vd,
+                                           vbfloat16mf4_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_vslideup_vx_bf16mf2_mu(
+// CHECK-RV64-SAME: <vscale x 2 x i1> [[VM:%.*]], <vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 2 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_vslideup_vx_bf16mf2_mu(vbool32_t vm, vbfloat16mf2_t vd,
+                                           vbfloat16mf2_t vs2, size_t rs1,
+                                           size_t vl) {
+  return __riscv_vslideup_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_vslideup_vx_bf16m1_mu(
+// CHECK-RV64-SAME: <vscale x 4 x i1> [[VM:%.*]], <vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 4 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_vslideup_vx_bf16m1_mu(vbool16_t vm, vbfloat16m1_t vd,
+                                         vbfloat16m1_t vs2, size_t rs1,
+                                         size_t vl) {
+  return __riscv_vslideup_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_vslideup_vx_bf16m2_mu(
+// CHECK-RV64-SAME: <vscale x 8 x i1> [[VM:%.*]], <vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 8 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_vslideup_vx_bf16m2_mu(vbool8_t vm, vbfloat16m2_t vd,
+                                         vbfloat16m2_t vs2, size_t rs1,
+                                         size_t vl) {
+  return __riscv_vslideup_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_vslideup_vx_bf16m4_mu(
+// CHECK-RV64-SAME: <vscale x 16 x i1> [[VM:%.*]], <vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 16 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_vslideup_vx_bf16m4_mu(vbool4_t vm, vbfloat16m4_t vd,
+                                         vbfloat16m4_t vs2, size_t rs1,
+                                         size_t vl) {
+  return __riscv_vslideup_mu(vm, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 32 x bfloat> @test_vslideup_vx_bf16m8_mu(
+// CHECK-RV64-SAME: <vscale x 32 x i1> [[VM:%.*]], <vscale x 32 x bfloat> [[VD:%.*]], <vscale x 32 x bfloat> [[VS2:%.*]], i64 noundef [[RS1:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  [[ENTRY:.*:]]
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x bfloat> @llvm.riscv.vslideup.mask.nxv32bf16.i64(<vscale x 32 x bfloat> [[VD]], <vscale x 32 x bfloat> [[VS2]], i64 [[RS1]], <vscale x 32 x i1> [[VM]], i64 [[VL]], i64 1)
+// CHECK-RV64-NEXT:    ret <vscale x 32 x bfloat> [[TMP0]]
+//
+vbfloat16m8_t test_vslideup_vx_bf16m8_mu(vbool2_t vm, vbfloat16m8_t vd,
+                                         vbfloat16m8_t vs2, size_t rs1,
+                                         size_t vl) {
+  return __riscv_vslideup_mu(vm, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll b/clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll
new file mode 100644
index 0000000000000..85bfc7f74daed
--- /dev/null
+++ b/clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll
@@ -0,0 +1,17 @@
+; REQUIRES: webassembly-registered-target
+
+; Check all the options parse
+; RUN: %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=none %s | FileCheck %s
+; RUN: %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=wasm -mllvm -wasm-enable-eh %s | FileCheck %s
+
+; RUN: not %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=invalid %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=dwarf %s 2>&1 | FileCheck -check-prefix=ERR-BE %s
+; RUN: not %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=sjlj %s 2>&1 | FileCheck -check-prefix=ERR-BE %s
+
+; CHECK-LABEL: define void @test(
+
+; ERR: error: invalid value 'invalid' in '-exception-model=invalid'
+; ERR-BE: fatal error: error in backend: -exception-model should be either 'none' or 'wasm'
+define void @test() {
+  ret void
+}
diff --git a/clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll b/clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll
new file mode 100644
index 0000000000000..27fb696c92499
--- /dev/null
+++ b/clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll
@@ -0,0 +1,8 @@
+; RUN: not %clang_cc1 -triple wasm32 -exception-model=arst -S %s 2>&1 | FileCheck -check-prefix=INVALID-VALUE %s
+
+; Make sure invalid values are rejected for -exception-model when the
+; input is IR.
+
+; INVALID-VALUE: error: invalid value 'arst' in '-exception-model=arst'
+
+target triple = "wasm32"
diff --git a/clang/test/CodeGen/arm-neon-directed-rounding.c b/clang/test/CodeGen/arm-neon-directed-rounding.c
index be587ea8e697a..08667314e37ce 100644
--- a/clang/test/CodeGen/arm-neon-directed-rounding.c
+++ b/clang/test/CodeGen/arm-neon-directed-rounding.c
@@ -16,7 +16,7 @@
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDA_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDA_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrinta.v2f32(<2 x float> [[VRNDA_V_I]])
+// CHECK-A32-NEXT:    [[VRNDA_V1_I:%.*]] = call <2 x float> @llvm.round.v2f32(<2 x float> [[VRNDA_V_I]])
 // CHECK-A32-NEXT:    [[VRNDA_V2_I:%.*]] = bitcast <2 x float> [[VRNDA_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDA_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -41,7 +41,7 @@ float32x2_t test_vrnda_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDAQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrinta.v4f32(<4 x float> [[VRNDAQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[VRNDAQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDAQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDAQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDAQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -66,7 +66,7 @@ float32x4_t test_vrndaq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDM_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDM_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintm.v2f32(<2 x float> [[VRNDM_V_I]])
+// CHECK-A32-NEXT:    [[VRNDM_V1_I:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[VRNDM_V_I]])
 // CHECK-A32-NEXT:    [[VRNDM_V2_I:%.*]] = bitcast <2 x float> [[VRNDM_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDM_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -91,7 +91,7 @@ float32x2_t test_vrndm_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDMQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintm.v4f32(<4 x float> [[VRNDMQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[VRNDMQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDMQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDMQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDMQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -116,7 +116,7 @@ float32x4_t test_vrndmq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDN_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintn.v2f32(<2 x float> [[VRNDN_V_I]])
+// CHECK-A32-NEXT:    [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.roundeven.v2f32(<2 x float> [[VRNDN_V_I]])
 // CHECK-A32-NEXT:    [[VRNDN_V2_I:%.*]] = bitcast <2 x float> [[VRNDN_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDN_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -141,7 +141,7 @@ float32x2_t test_vrndn_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDNQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintn.v4f32(<4 x float> [[VRNDNQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.roundeven.v4f32(<4 x float> [[VRNDNQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDNQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDNQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDNQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -166,7 +166,7 @@ float32x4_t test_vrndnq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDP_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDP_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintp.v2f32(<2 x float> [[VRNDP_V_I]])
+// CHECK-A32-NEXT:    [[VRNDP_V1_I:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[VRNDP_V_I]])
 // CHECK-A32-NEXT:    [[VRNDP_V2_I:%.*]] = bitcast <2 x float> [[VRNDP_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDP_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -191,7 +191,7 @@ float32x2_t test_vrndp_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDPQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintp.v4f32(<4 x float> [[VRNDPQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[VRNDPQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDPQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDPQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDPQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -216,7 +216,7 @@ float32x4_t test_vrndpq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDX_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDX_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintx.v2f32(<2 x float> [[VRNDX_V_I]])
+// CHECK-A32-NEXT:    [[VRNDX_V1_I:%.*]] = call <2 x float> @llvm.rint.v2f32(<2 x float> [[VRNDX_V_I]])
 // CHECK-A32-NEXT:    [[VRNDX_V2_I:%.*]] = bitcast <2 x float> [[VRNDX_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDX_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -241,7 +241,7 @@ float32x2_t test_vrndx_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintx.v4f32(<4 x float> [[VRNDXQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[VRNDXQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDXQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDXQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDXQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -266,7 +266,7 @@ float32x4_t test_vrndxq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRND_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRND_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintz.v2f32(<2 x float> [[VRND_V_I]])
+// CHECK-A32-NEXT:    [[VRND_V1_I:%.*]] = call <2 x float> @llvm.trunc.v2f32(<2 x float> [[VRND_V_I]])
 // CHECK-A32-NEXT:    [[VRND_V2_I:%.*]] = bitcast <2 x float> [[VRND_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRND_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -291,7 +291,7 @@ float32x2_t test_vrnd_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintz.v4f32(<4 x float> [[VRNDQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDQ_V1_I:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[VRNDQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -310,24 +310,18 @@ float32x4_t test_vrndq_f32(float32x4_t a) {
   return vrndq_f32(a);
 }
 
-// CHECK-A32-LABEL: define dso_local float @test_vrndns_f32(
-// CHECK-A32-SAME: float noundef [[A:%.*]]) #[[ATTR0]] {
-// CHECK-A32-NEXT:  [[ENTRY:.*:]]
-// CHECK-A32-NEXT:    [[VRNDN_I:%.*]] = call float @llvm.arm.neon.vrintn.f32(float [[A]])
-// CHECK-A32-NEXT:    ret float [[VRNDN_I]]
-//
-// CHECK-A64-LABEL: define dso_local float @test_vrndns_f32(
-// CHECK-A64-SAME: float noundef [[A:%.*]]) #[[ATTR0]] {
-// CHECK-A64-NEXT:  [[ENTRY:.*:]]
-// CHECK-A64-NEXT:    [[VRNDN_I:%.*]] = call float @llvm.roundeven.f32(float [[A]])
-// CHECK-A64-NEXT:    ret float [[VRNDN_I]]
+// CHECK-LABEL: define dso_local float @test_vrndns_f32(
+// CHECK-SAME: float noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VRNDN_I:%.*]] = call float @llvm.roundeven.f32(float [[A]])
+// CHECK-NEXT:    ret float [[VRNDN_I]]
 //
 float32_t test_vrndns_f32(float32_t a) {
   return vrndns_f32(a);
 }
 
 // CHECK-LABEL: define dso_local <2 x float> @test_vrndi_f32(
-// CHECK-SAME: <2 x float> noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: <2 x float> noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
diff --git a/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c b/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
index bd424172a1865..98548749a04d6 100644
--- a/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
+++ b/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
@@ -1,22 +1,66 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=8 -mvscale-max=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=16 -mvscale-max=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-UNBOUNDED
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-UNBOUNDED
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-NONE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=2
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=8 -mvscale-max=8 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=8
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=16 -mvscale-max=16 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=16
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +sme -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +sme -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=2
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=2 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=2
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=8 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=8
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=16 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=16
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +sme -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-UNBOUNDED,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-UNBOUNDED,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-streaming-min=1 -mvscale-streaming-max=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-STREAMING,CHECK-NOCOMPATIBLE -D#STREAMINGVBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-streaming-min=4 -mvscale-streaming-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-STREAMING,CHECK-NOCOMPATIBLE -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-streaming-min=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-STREAMING-NOMAX,CHECK-NOCOMPATIBLE -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-min=2 -mvscale-streaming-min=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-STREAMING-NOMAX,CHECK-COMPATIBLE-NOMAX -D#VBITS=2 -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-max=2 -mvscale-streaming-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMIN,CHECK-STREAMING-NOMIN,CHECK-COMPATIBLE-NOMIN -D#VBITS=2 -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme \
+// RUN:            -mvscale-min=2 -mvscale-streaming-min=4 -mvscale-max=2 -mvscale-streaming-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-STREAMING,CHECK-COMPATIBLE -D#VBITS=2 -D#STREAMINGVBITS=4
+
 
 // CHECK-LABEL: @func() #0
-// CHECK: attributes #0 = { {{.*}} vscale_range([[#VBITS]],[[#VBITS]]) {{.*}} }
-// CHECK-NOMAX: attributes #0 = { {{.*}} vscale_range([[#VBITS]],0) {{.*}} }
-// CHECK-UNBOUNDED: attributes #0 = { {{.*}} vscale_range(1,0) {{.*}} }
+// CHECK-LABEL: @func2() #1
+// CHECK-LABEL: @func3() #2
+// CHECK-MINMAX: attributes #0 = { {{.*}} vscale_range([[#VBITS]],[[#VBITS]]) {{.*}} }
+// CHECK-NOMAX: attributes #0 = { {{.*}} vscale_range([[#VBITS]],16) {{.*}} }
+// CHECK-NOMIN: attributes #0 = { {{.*}} vscale_range(1,[[#VBITS]]) {{.*}} }
+// CHECK-UNBOUNDED: attributes #0 = { {{.*}} vscale_range(1,16) {{.*}} }
 // CHECK-NONE: attributes #0 = { {{.*}} vscale_range(1,16) {{.*}} }
+// CHECK-STREAMING: attributes #1 = { {{.*}} vscale_range([[#STREAMINGVBITS]],[[#STREAMINGVBITS]])
+// CHECK-STREAMING-NOMAX: attributes #1 = { {{.*}} vscale_range([[#STREAMINGVBITS]],16)
+// CHECK-STREAMING-NOMIN: attributes #1 = { {{.*}} vscale_range(1,[[#STREAMINGVBITS]]) {{.*}} }
+// CHECK-NOSTREAMING: attributes #1 = { {{.*}} vscale_range(1,16) {{.*}} }
+// CHECK-NOCOMPATIBLE: attributes #2 = { {{.*}} vscale_range(1,16) {{.*}} }
+// CHECK-COMPATIBLE: attributes #2 = { {{.*}} vscale_range([[#VBITS]],[[#STREAMINGVBITS]]) {{.*}} }
+// CHECK-COMPATIBLE-NOMAX: attributes #2 = { {{.*}} vscale_range([[#VBITS]],16) {{.*}} }
+// CHECK-COMPATIBLE-NOMIN: attributes #2 = { {{.*}} vscale_range(1,[[#STREAMINGVBITS]]) {{.*}} }
 void func(void) {}
+__arm_locally_streaming void func2(void) {}
+void func3(void) __arm_streaming_compatible {}
diff --git a/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c b/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
index f85deeeca757f..17d4eef1f7631 100644
--- a/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
+++ b/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
@@ -522,7 +522,7 @@ float16x8_t test_vrecpeq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRND_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRND_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintz.v4f16(<4 x half> [[VRND_V_I]])
+// CHECK-NEXT:    [[VRND_V1_I:%.*]] = call <4 x half> @llvm.trunc.v4f16(<4 x half> [[VRND_V_I]])
 // CHECK-NEXT:    [[VRND_V2_I:%.*]] = bitcast <4 x half> [[VRND_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRND_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -538,7 +538,7 @@ float16x4_t test_vrnd_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintz.v8f16(<8 x half> [[VRNDQ_V_I]])
+// CHECK-NEXT:    [[VRNDQ_V1_I:%.*]] = call <8 x half> @llvm.trunc.v8f16(<8 x half> [[VRNDQ_V_I]])
 // CHECK-NEXT:    [[VRNDQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -554,7 +554,7 @@ float16x8_t test_vrndq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDA_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDA_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrinta.v4f16(<4 x half> [[VRNDA_V_I]])
+// CHECK-NEXT:    [[VRNDA_V1_I:%.*]] = call <4 x half> @llvm.round.v4f16(<4 x half> [[VRNDA_V_I]])
 // CHECK-NEXT:    [[VRNDA_V2_I:%.*]] = bitcast <4 x half> [[VRNDA_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDA_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -570,7 +570,7 @@ float16x4_t test_vrnda_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDAQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrinta.v8f16(<8 x half> [[VRNDAQ_V_I]])
+// CHECK-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <8 x half> @llvm.round.v8f16(<8 x half> [[VRNDAQ_V_I]])
 // CHECK-NEXT:    [[VRNDAQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDAQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDAQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -586,7 +586,7 @@ float16x8_t test_vrndaq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDM_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDM_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintm.v4f16(<4 x half> [[VRNDM_V_I]])
+// CHECK-NEXT:    [[VRNDM_V1_I:%.*]] = call <4 x half> @llvm.floor.v4f16(<4 x half> [[VRNDM_V_I]])
 // CHECK-NEXT:    [[VRNDM_V2_I:%.*]] = bitcast <4 x half> [[VRNDM_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDM_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -602,7 +602,7 @@ float16x4_t test_vrndm_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDMQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintm.v8f16(<8 x half> [[VRNDMQ_V_I]])
+// CHECK-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <8 x half> @llvm.floor.v8f16(<8 x half> [[VRNDMQ_V_I]])
 // CHECK-NEXT:    [[VRNDMQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDMQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDMQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -618,7 +618,7 @@ float16x8_t test_vrndmq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDN_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDN_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintn.v4f16(<4 x half> [[VRNDN_V_I]])
+// CHECK-NEXT:    [[VRNDN_V1_I:%.*]] = call <4 x half> @llvm.roundeven.v4f16(<4 x half> [[VRNDN_V_I]])
 // CHECK-NEXT:    [[VRNDN_V2_I:%.*]] = bitcast <4 x half> [[VRNDN_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDN_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -634,7 +634,7 @@ float16x4_t test_vrndn_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDNQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintn.v8f16(<8 x half> [[VRNDNQ_V_I]])
+// CHECK-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <8 x half> @llvm.roundeven.v8f16(<8 x half> [[VRNDNQ_V_I]])
 // CHECK-NEXT:    [[VRNDNQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDNQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDNQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -650,7 +650,7 @@ float16x8_t test_vrndnq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDP_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDP_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintp.v4f16(<4 x half> [[VRNDP_V_I]])
+// CHECK-NEXT:    [[VRNDP_V1_I:%.*]] = call <4 x half> @llvm.ceil.v4f16(<4 x half> [[VRNDP_V_I]])
 // CHECK-NEXT:    [[VRNDP_V2_I:%.*]] = bitcast <4 x half> [[VRNDP_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDP_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -666,7 +666,7 @@ float16x4_t test_vrndp_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDPQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintp.v8f16(<8 x half> [[VRNDPQ_V_I]])
+// CHECK-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <8 x half> @llvm.ceil.v8f16(<8 x half> [[VRNDPQ_V_I]])
 // CHECK-NEXT:    [[VRNDPQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDPQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDPQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -682,7 +682,7 @@ float16x8_t test_vrndpq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDX_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDX_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintx.v4f16(<4 x half> [[VRNDX_V_I]])
+// CHECK-NEXT:    [[VRNDX_V1_I:%.*]] = call <4 x half> @llvm.rint.v4f16(<4 x half> [[VRNDX_V_I]])
 // CHECK-NEXT:    [[VRNDX_V2_I:%.*]] = bitcast <4 x half> [[VRNDX_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDX_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -698,7 +698,7 @@ float16x4_t test_vrndx_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintx.v8f16(<8 x half> [[VRNDXQ_V_I]])
+// CHECK-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <8 x half> @llvm.rint.v8f16(<8 x half> [[VRNDXQ_V_I]])
 // CHECK-NEXT:    [[VRNDXQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDXQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDXQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
diff --git a/clang/test/CodeGen/arm64-microsoft-sys.c b/clang/test/CodeGen/arm64-microsoft-sys.c
new file mode 100644
index 0000000000000..5913a02d933ae
--- /dev/null
+++ b/clang/test/CodeGen/arm64-microsoft-sys.c
@@ -0,0 +1,65 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple arm64-windows -fms-compatibility -S \
+// RUN: -o - %s | FileCheck %s -check-prefix CHECK-ASM
+
+// RUN: %clang_cc1 -triple arm64-windows -fms-compatibility -emit-llvm \
+// RUN: -o - %s | FileCheck %s -check-prefix CHECK-IR
+
+// RUN: %clang_cc1 -triple arm64-darwin -fms-compatibility -emit-llvm \
+// RUN: -o - %s | FileCheck %s -check-prefix CHECK-IR
+
+// From winnt.h
+// op0=1 encodings, use with __sys
+#define ARM64_SYSINSTR(op0, op1, crn, crm, op2) \
+        ( ((op1 & 7) << 11) | \
+          ((crn & 15) << 7) | \
+          ((crm & 15) << 3) | \
+          ((op2 & 7) << 0) )
+
+//
+// Sampling of instructions
+//
+#define ARM64_DC_CGDSW_EL1      ARM64_SYSINSTR(1,0, 7,10,6) // Clean of Data and Allocation Tags by Set/Way
+#define ARM64_IC_IALLU_EL1      ARM64_SYSINSTR(1,0, 7, 5,0) // Instruction Cache Invalidate All to PoU
+#define ARM64_AT_S1E2W          ARM64_SYSINSTR(1,4, 7, 8,1) // Translate Stage1, EL2, write
+#define ARM64_TLBI_VMALLE1      ARM64_SYSINSTR(1,0, 8, 7,0) // Invalidate stage 1 TLB [CP15_TLBIALL]
+#define ARM64_CFP_RCTX          ARM64_SYSINSTR(1,3, 7, 3,4) // Control Flow Prediction Restriction by Context
+
+// From intrin.h
+unsigned int __sys(int, __int64);
+
+void check__sys(__int64 v) {
+  __int64 ret;
+
+  __sys(ARM64_DC_CGDSW_EL1, v);
+// CHECK-ASM: msr     S1_0_C7_C10_6, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD2:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_IC_IALLU_EL1, v);
+// CHECK-ASM: msr     S1_0_C7_C5_0, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD3:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_AT_S1E2W, v);
+// CHECK-ASM: msr     S1_4_C7_C8_1, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD4:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_TLBI_VMALLE1, v);
+// CHECK-ASM: msr     S1_0_C8_C7_0, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD5:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_CFP_RCTX, v);
+// CHECK-ASM: msr     S1_3_C7_C3_4, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD6:.*]], i64 %[[VAR]])
+}
+
+// CHECK-IR: ![[MD2]] = !{!"1:0:7:10:6"}
+// CHECK-IR: ![[MD3]] = !{!"1:0:7:5:0"}
+// CHECK-IR: ![[MD4]] = !{!"1:4:7:8:1"}
+// CHECK-IR: ![[MD5]] = !{!"1:0:8:7:0"}
+// CHECK-IR: ![[MD6]] = !{!"1:3:7:3:4"}
diff --git a/clang/test/CodeGen/bitfield-access-pad.c b/clang/test/CodeGen/bitfield-access-pad.c
index 8608c5bd8be11..2044746118add 100644
--- a/clang/test/CodeGen/bitfield-access-pad.c
+++ b/clang/test/CodeGen/bitfield-access-pad.c
@@ -18,7 +18,7 @@
 // RUN: %clang_cc1 -triple=hexagon-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-T %s
 
 // Big endian
-// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-T %s
+// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-M68K %s
 // RUN: %clang_cc1 -triple=mips-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-T %s
 
 // And now a few with -fno-bitfield-type-align. Precisely how this behaves is
@@ -45,6 +45,7 @@ struct P1 {
 // CHECK-LABEL: LLVMType:%struct.P1 =
 // LAYOUT-T-SAME: type { i8, i8, [2 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -60,6 +61,9 @@ struct P1 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:1
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:1
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -75,6 +79,7 @@ struct P2 {
 // CHECK-LABEL: LLVMType:%struct.P2 =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -90,6 +95,9 @@ struct P2 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -105,6 +113,7 @@ struct P3 {
 // CHECK-LABEL: LLVMType:%struct.P3 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -120,6 +129,9 @@ struct P3 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -134,6 +146,7 @@ struct P4 {
 // CHECK-LABEL: LLVMType:%struct.P4 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -149,6 +162,9 @@ struct P4 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -162,6 +178,7 @@ struct P5 {
 // CHECK-LABEL: LLVMType:%struct.P5 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -177,6 +194,9 @@ struct P5 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -192,6 +212,7 @@ struct P6 {
 // CHECK-LABEL: LLVMType:%struct.P6 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -207,6 +228,9 @@ struct P6 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -220,6 +244,7 @@ struct P7 {
 // CHECK-LABEL: LLVMType:%struct.P7 =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -235,6 +260,9 @@ struct P7 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -250,6 +278,7 @@ struct __attribute__ ((aligned (2))) P7_align {
 // CHECK-LABEL: LLVMType:%struct.P7_align =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -265,6 +294,9 @@ struct __attribute__ ((aligned (2))) P7_align {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -278,6 +310,7 @@ struct P8 {
 // CHECK-LABEL: LLVMType:%struct.P8 =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i16 }
 // LAYOUT-STRICT-NT-SAME: type { i16 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -293,6 +326,9 @@ struct P8 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -306,6 +342,7 @@ struct P9 {
 // CHECK-LABEL: LLVMType:%struct.P9 =
 // LAYOUT-T-SAME: type { i8, i8, [2 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8 }
 // LAYOUT-NT-SAME: type { i16 }
 // LAYOUT-STRICT-NT-SAME: type { i16 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -321,6 +358,9 @@ struct P9 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:1
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:1
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -335,6 +375,7 @@ struct __attribute__((aligned(4))) P10 {
 // CHECK-LABEL: LLVMType:%struct.P10 =
 // LAYOUT-T-SAME: type { i32 }
 // LAYOUT-ARM64-T-SAME: type { i32 }
+// LAYOUT-M68K-SAME: type { i32 }
 // LAYOUT-NT-SAME: type { i32 }
 // LAYOUT-STRICT-NT-SAME: type { i32 }
 // LAYOUT-DWN32-SAME: type { i32 }
@@ -354,6 +395,10 @@ struct __attribute__((aligned(4))) P10 {
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
@@ -369,6 +414,7 @@ struct __attribute__((aligned(4))) P11 {
 // CHECK-LABEL: LLVMType:%struct.P11 =
 // LAYOUT-T-SAME: type { i32 }
 // LAYOUT-ARM64-T-SAME: type { i32 }
+// LAYOUT-M68K-SAME: type { i32 }
 // LAYOUT-NT-SAME: type { i32 }
 // LAYOUT-STRICT-NT-SAME: type { i32 }
 // LAYOUT-DWN32-SAME: type { i32 }
@@ -388,6 +434,10 @@ struct __attribute__((aligned(4))) P11 {
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:10 IsSigned:0 StorageSize:32 StorageOffset:0
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:10 IsSigned:0 StorageSize:32 StorageOffset:0
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
diff --git a/clang/test/CodeGen/debug-label-inline.c b/clang/test/CodeGen/debug-label-inline.c
index 972a32b5af32d..9d92ffb5ef61f 100644
--- a/clang/test/CodeGen/debug-label-inline.c
+++ b/clang/test/CodeGen/debug-label-inline.c
@@ -23,6 +23,6 @@ int f2(void) {
 
 // CHECK: distinct !DISubprogram(name: "f1", {{.*}}, retainedNodes: [[ELEMENTS:!.*]])
 // CHECK: [[ELEMENTS]] = !{{{.*}}, [[LABEL_METADATA]]}
-// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 8)
+// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 8, column: 1)
 // CHECK: [[INLINEDAT:!.*]] = distinct !DILocation(line: 18,
 // CHECK: [[LABEL_LOCATION]] = !DILocation(line: 8, {{.*}}, inlinedAt: [[INLINEDAT]])
diff --git a/clang/test/CodeGen/debug-label.c b/clang/test/CodeGen/debug-label.c
index 662b1a7dd2062..308d664ab712b 100644
--- a/clang/test/CodeGen/debug-label.c
+++ b/clang/test/CodeGen/debug-label.c
@@ -12,5 +12,5 @@ int f1(int a, int b) {
   return sum;
 }
 
-// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 9)
+// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 9, column: 1)
 // CHECK: [[LABEL_LOCATION]] = !DILocation(line: 9,
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index 14efd54e24ffb..55d4c438eedad 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -27,8 +27,8 @@
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -target-abi darwinpcs -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64DARWIN
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=ARM
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch64 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch32 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch64 -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch32 -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA32
 
 // Make sure 128 and 64 bit versions are passed like integers.
 void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
@@ -158,8 +158,8 @@ void ParamPassing4(_BitInt(129) a) {}
 // PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// LA64-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// LA32-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
 #endif
 
 _BitInt(63) ReturnPassing(void) { return 0; }
@@ -317,8 +317,8 @@ _BitInt(129) ReturnPassing5(void) { return 0; }
 // PPC32-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // AARCH64DARWIN-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // ARM-NOT: define{{.*}} arm_aapcscc void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
-// LA64-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
-// LA32-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
+// LA64: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
+// LA32: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 
 // SparcV9 is odd in that it has a return-size limit of 256, not 128 or 64
 // like other platforms, so test to make sure this behavior will still work.
diff --git a/clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c b/clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c
new file mode 100644
index 0000000000000..0f1cdf9512395
--- /dev/null
+++ b/clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c
@@ -0,0 +1,15 @@
+// RUN: %clang -fallow-runtime-check-skip-hot-cutoff=1.0 -S -emit-llvm %s -o - -O2 | FileCheck --check-prefix=ONE %s
+// RUN: %clang -fallow-runtime-check-skip-hot-cutoff=0.0 -S -emit-llvm %s -o - -O2 | FileCheck --check-prefix=ZERO %s
+// RUN: not %clang -fallow-runtime-check-skip-hot-cutoff=6.0 -S -emit-llvm %s -o - -O2 2>&1 | FileCheck --check-prefix=SIX %s
+// RUN: not %clang -fallow-runtime-check-skip-hot-cutoff=-1.0 -S -emit-llvm %s -o - -O2 2>&1 | FileCheck --check-prefix=MINUSONE %s
+// RUN: not %clang -fallow-runtime-check-skip-hot-cutoff=string -S -emit-llvm %s -o - -O2 2>&1 | FileCheck --check-prefix=STRING %s
+
+// ONE: ret i32 0
+// ZERO: ret i32 1
+// SIX: invalid value '6.0' in '-fallow-runtime-check-skip-hot-cutoff='
+// MINUSONE: invalid value '-1.0' in '-fallow-runtime-check-skip-hot-cutoff='
+// STRING: invalid value 'string' in '-fallow-runtime-check-skip-hot-cutoff='
+
+int main(int argc, char** argv) {
+    return __builtin_allow_runtime_check("foo");
+}
diff --git a/clang/test/CodeGenCXX/auto-var-init.cpp b/clang/test/CodeGenCXX/auto-var-init.cpp
index 94386e44573b5..67bc5d417bce9 100644
--- a/clang/test/CodeGenCXX/auto-var-init.cpp
+++ b/clang/test/CodeGenCXX/auto-var-init.cpp
@@ -1342,6 +1342,7 @@ TEST_UNINIT(base, base);
 // ZERO-O1-NOT: !annotation
 
 TEST_BRACES(base, base);
+// ZERO-LABEL: @test_base_braces()
 // CHECK-LABEL: @test_base_braces()
 // CHECK:       %braces = alloca %struct.base, align [[ALIGN:[0-9]*]]
 // CHECK-NEXT:  call void @llvm.memset{{.*}}(ptr align [[ALIGN]] %{{.*}}, i8 0, i64 8, i1 false)
diff --git a/clang/test/CodeGenCXX/bitfield-access-empty.cpp b/clang/test/CodeGenCXX/bitfield-access-empty.cpp
index 460fe6eef4b90..0485ae5cda4a9 100644
--- a/clang/test/CodeGenCXX/bitfield-access-empty.cpp
+++ b/clang/test/CodeGenCXX/bitfield-access-empty.cpp
@@ -35,7 +35,7 @@
 
 // Big endian
 // RUN: %clang_cc1 -triple=lanai-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s
-// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s
+// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-M68K %s
 // RUN: %clang_cc1 -triple=mips-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s
 // RUN: %clang_cc1 -triple=mips64-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s
 // RUN: %clang_cc1 -triple=sparc-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT %s
@@ -51,6 +51,7 @@ struct P1 {
 // CHECK-LABEL: LLVMType:%struct.P1 =
 // LAYOUT-SAME: type { i16, i16 }
 // LAYOUT-DWN32-SAME: type { i16, i16 }
+// LAYOUT-DWN32-M68K: type { i16, i16 }
 // CHECK-NEXT: NonVirtualBaseLLVMType:%struct.P1 =
 // CHECK: BitFields:[
 // LAYOUT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:0
@@ -58,6 +59,9 @@ struct P1 {
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:2
+
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:2
 // CHECK-NEXT: ]>
 
 struct P2 {
@@ -68,6 +72,7 @@ struct P2 {
 // CHECK-LABEL: LLVMType:%struct.P2 =
 // LAYOUT-SAME: type { i16, i16 }
 // LAYOUT-DWN32-SAME: type { i16, i16 }
+// LAYOUT-M68K-SAME: type { i16, i16 }
 // CHECK-NEXT: NonVirtualBaseLLVMType:%struct.P2 =
 // CHECK: BitFields:[
 // LAYOUT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:15 IsSigned:0 StorageSize:16 StorageOffset:0
@@ -75,6 +80,9 @@ struct P2 {
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:15 IsSigned:0 StorageSize:16 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:15 IsSigned:0 StorageSize:16 StorageOffset:2
+
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:15 IsSigned:0 StorageSize:16 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:15 IsSigned:0 StorageSize:16 StorageOffset:2
 // CHECK-NEXT: ]>
 
 struct P3 {
@@ -85,6 +93,7 @@ struct P3 {
 // CHECK-LABEL: LLVMType:%struct.P3 =
 // LAYOUT-SAME: type { i16, [2 x i8], i16, [2 x i8] }
 // LAYOUT-DWN32-SAME: type <{ i16, i8, i16 }>
+// LAYOUT-M68K-SAME: type <{ i16, i8, i16, i8 }>
 // CHECK-NEXT: NonVirtualBaseLLVMType:%struct.P3 =
 // CHECK: BitFields:[
 // LAYOUT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:0
@@ -92,6 +101,9 @@ struct P3 {
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:3
+
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:16 StorageOffset:3
 // CHECK-NEXT: ]>
 
 struct P4 {
@@ -121,6 +133,7 @@ struct P6 {
 // CHECK-LABEL: LLVMType:%struct.P6 =
 // LAYOUT-SAME: type { i32, i32 }
 // LAYOUT-DWN32-SAME: type { i32, i32 }
+// LAYOUT-M68K-SAME: type { i32, i32 }
 // CHECK-NEXT: NonVirtualBaseLLVMType:%struct.P6 =
 // CHECK: BitFields:[
 // LAYOUT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:32 StorageOffset:0
@@ -128,6 +141,9 @@ struct P6 {
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:32 StorageOffset:0
+
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:32 StorageOffset:0
 // CHECK-NEXT: ]>
 
 struct P7 {
@@ -139,6 +155,7 @@ struct P7 {
 // CHECK-LABEL: LLVMType:%struct.P7 =
 // LAYOUT-SAME: type { i32, i32 }
 // LAYOUT-DWN32-SAME: type { i32, i32 }
+// LAYOUT-M68K-SAME: type { i32, i32 }
 // CHECK-NEXT: NonVirtualBaseLLVMType:%struct.P7 =
 // CHECK: BitFields:[
 // LAYOUT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:32 StorageOffset:0
@@ -146,4 +163,7 @@ struct P7 {
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:32 StorageOffset:0
+
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:16 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:32 StorageOffset:0
 // CHECK-NEXT: ]>
diff --git a/clang/test/CodeGenCXX/builtin-invoke.cpp b/clang/test/CodeGenCXX/builtin-invoke.cpp
new file mode 100644
index 0000000000000..af66dfd4dae30
--- /dev/null
+++ b/clang/test/CodeGenCXX/builtin-invoke.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void* memcpy(void*, const void*, decltype(sizeof(int)));
+void func();
+
+namespace std {
+  template <class T>
+  class reference_wrapper {
+    T* ptr;
+
+  public:
+    T& get() { return *ptr; }
+  };
+} // namespace std
+
+struct Callable {
+  void operator()() {}
+
+  void func();
+};
+
+extern "C" void call1() {
+  __builtin_invoke(func);
+  __builtin_invoke(Callable{});
+  __builtin_invoke(memcpy, nullptr, nullptr, 0);
+
+  // CHECK:      define dso_local void @call1
+  // CHECK-NEXT: entry:
+  // CHECK-NEXT:   %ref.tmp = alloca %struct.Callable, align 1
+  // CHECK-NEXT:   call void @_Z4funcv()
+  // CHECK-NEXT:   call void @_ZN8CallableclEv(ptr noundef nonnull align 1 dereferenceable(1) %ref.tmp)
+  // CHECK-NEXT:   call void @llvm.memcpy.p0.p0.i64(ptr align 1 null, ptr align 1 null, i64 0, i1 false)
+  // CHECK-NEXT:   ret void
+}
+
+extern "C" void call_memptr(std::reference_wrapper<Callable> wrapper) {
+  __builtin_invoke(&Callable::func, wrapper);
+
+  // CHECK:      define dso_local void @call_memptr
+  // CHECK-NEXT: entry:
+  // CHECK-NEXT:   %wrapper = alloca %"class.std::reference_wrapper", align 8
+  // CHECK-NEXT:   %coerce.dive = getelementptr inbounds nuw %"class.std::reference_wrapper", ptr %wrapper, i32 0, i32 0
+  // CHECK-NEXT:   store ptr %wrapper.coerce, ptr %coerce.dive, align 8
+  // CHECK-NEXT:   %call = call noundef nonnull align 1 dereferenceable(1) ptr @_ZNSt17reference_wrapperI8CallableE3getEv(ptr noundef nonnull align 8 dereferenceable(8) %wrapper)
+  // CHECK-NEXT:   %0 = getelementptr inbounds i8, ptr %call, i64 0
+  // CHECK-NEXT:   br i1 false, label %memptr.virtual, label %memptr.nonvirtual
+  // CHECK-EMPTY:
+  // CHECK-NEXT: memptr.virtual:
+  // CHECK-NEXT:   %vtable = load ptr, ptr %0, align 8
+  // CHECK-NEXT:   %1 = getelementptr i8, ptr %vtable, i64 sub (i64 ptrtoint (ptr @_ZN8Callable4funcEv to i64), i64 1), !nosanitize !2
+  // CHECK-NEXT:   %memptr.virtualfn = load ptr, ptr %1, align 8, !nosanitize !2
+  // CHECK-NEXT:   br label %memptr.end
+  // CHECK-EMPTY:
+  // CHECK-NEXT: memptr.nonvirtual:
+  // CHECK-NEXT:   br label %memptr.end
+  // CHECK-EMPTY:
+  // CHECK-NEXT: memptr.end:
+  // CHECK-NEXT:   %2 = phi ptr [ %memptr.virtualfn, %memptr.virtual ], [ @_ZN8Callable4funcEv, %memptr.nonvirtual ]
+  // CHECK-NEXT:   call void %2(ptr noundef nonnull align 1 dereferenceable(1) %0)
+  // CHECK-NEXT:   ret void
+}
diff --git a/clang/test/CodeGenCXX/builtins-eh-wasm.cpp b/clang/test/CodeGenCXX/builtins-eh-wasm.cpp
index b0f763d3e54dc..b3a6ffc92e11b 100644
--- a/clang/test/CodeGenCXX/builtins-eh-wasm.cpp
+++ b/clang/test/CodeGenCXX/builtins-eh-wasm.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple wasm32-unknown-unknown -fexceptions -fcxx-exceptions -target-feature +reference-types -target-feature +exception-handling -target-feature +multivalue -exception-model=wasm -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: webassembly-registered-target
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -fexceptions -fcxx-exceptions -target-feature +reference-types -target-feature +exception-handling -target-feature +multivalue -mllvm -wasm-enable-eh -exception-model=wasm -emit-llvm -o - %s | FileCheck %s
 
 // Check if __builtin_wasm_throw and __builtin_wasm_rethrow are correctly
 // invoked when placed in try-catch.
diff --git a/clang/test/CodeGenCXX/cxx0x-initializer-scalars.cpp b/clang/test/CodeGenCXX/cxx0x-initializer-scalars.cpp
index 2f6a6820a7589..837d86814088b 100644
--- a/clang/test/CodeGenCXX/cxx0x-initializer-scalars.cpp
+++ b/clang/test/CodeGenCXX/cxx0x-initializer-scalars.cpp
@@ -1,7 +1,19 @@
-// RUN: %clang_cc1 -std=c++11 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-linux -emit-llvm -o - %s | FileCheck %s
 
 void f()
 {
   // CHECK: store i32 0
   int i{};
 }
+
+
+namespace GH116440 {
+void f() {
+  void{};
+  void();
+}
+
+// CHECK: define{{.*}} void @_ZN8GH1164401fEv()
+// CHECK-NEXT: entry
+// CHECK-NEXT: ret void
+}
diff --git a/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp b/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
new file mode 100644
index 0000000000000..4bb51dcc4da51
--- /dev/null
+++ b/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-none-linux-gnu -emit-llvm -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -emit-llvm -debug-info-kind=limited %s -o - | FileCheck --check-prefix MSVC %s
+
+struct B {
+    virtual ~B() {}
+};
+
+struct A : virtual B {
+};
+
+A a;
+
+
+// CHECK-DAG: !{{[0-9]+}} = !DILocalVariable(name: "vtt", arg: 2, scope: ![[destructor:[0-9]+]], type: ![[vtttype:[0-9]+]], flags: DIFlagArtificial)
+// CHECK-DAG: ![[destructor]] = distinct !DISubprogram(name: "~A", {{.*}}, type: ![[subroutinetype:[0-9]+]]
+// CHECK-DAG: ![[subroutinetype]] = !DISubroutineType(types: ![[types:[0-9]+]])
+// CHECK-DAG: [[types]] = !{null, !{{[0-9]+}}, ![[vtttype]]}
+
+// MSVC-DAG: ![[inttype:[0-9]+]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+// MSVC-DAG: ![[voidpointertype:[0-9]+]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+// MSVC-DAG: ![[destructor:[0-9]+]] = distinct !DISubprogram(name: "~A", linkageName: "??_GA@@UEAAPEAXI@Z", {{.*}}, type: ![[subroutinetype:[0-9]+]]
+// MSVC-DAG: !{{[0-9]+}} = !DILocalVariable(name: "should_call_delete", arg: 2, scope: ![[destructor]], type: ![[inttype]], flags: DIFlagArtificial)
+// MSVC-DAG: ![[subroutinetype]] = !DISubroutineType(types: ![[types:[0-9]+]])
+// MSVC-DAG: [[types]] = !{![[voidpointertype]], !{{[0-9]+}}, ![[inttype]]}
diff --git a/clang/test/CodeGenCXX/wasm-eh.cpp b/clang/test/CodeGenCXX/wasm-eh.cpp
index faff764878f5d..f243f37ecb435 100644
--- a/clang/test/CodeGenCXX/wasm-eh.cpp
+++ b/clang/test/CodeGenCXX/wasm-eh.cpp
@@ -393,9 +393,9 @@ void noexcept_throw() noexcept {
 // CHECK-NEXT:  call void @_ZSt9terminatev()
 
 
-// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -exception-model=wasm -target-feature +exception-handling -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-DEFAULT
-// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -exception-model=wasm -target-feature +exception-handling -Wwasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-ON
-// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -exception-model=wasm -target-feature +exception-handling -Wno-wasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-OFF
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -mllvm -wasm-enable-eh -exception-model=wasm -target-feature +exception-handling -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-DEFAULT
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -mllvm -wasm-enable-eh -exception-model=wasm -target-feature +exception-handling -Wwasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-ON
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -mllvm -wasm-enable-eh -exception-model=wasm -target-feature +exception-handling -Wno-wasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-OFF
 // RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fexceptions -fcxx-exceptions -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=EM-EH-WARNING
 
 // Wasm EH ignores dynamic exception specifications with types at the moment.
diff --git a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
index 55cc57fadf080..28d9a4b8e596c 100644
--- a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 struct MyStruct {
   float4 a;
@@ -21,7 +22,9 @@ struct MyStruct {
 // DXIL: %"class.hlsl::AppendStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
 // DXIL: %struct.MyStruct = type <{ <4 x float>, <2 x i32> }>
 // DXIL: %"class.hlsl::AppendStructuredBuffer.13" = type { target("dx.RawBuffer", i32, 1, 0)
+// SPV: %"class.hlsl::AppendStructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 1)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.14" = type { target("dx.RawBuffer", <4 x i32>, 1, 0)
+// SPV: %"class.hlsl::AppendStructuredBuffer.14" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 1)
 
 AppendStructuredBuffer<int16_t> BufI16;
 AppendStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
index 94fdd3cd9d0b3..70fe8670c59af 100644
--- a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 struct MyStruct {
   float4 a;
@@ -21,7 +22,9 @@ struct MyStruct {
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
 // DXIL: %struct.MyStruct = type <{ <4 x float>, <2 x i32> }>
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.13" = type { target("dx.RawBuffer", i32, 1, 0)
+// SPV: %"class.hlsl::ConsumeStructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 1)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.14" = type { target("dx.RawBuffer", <4 x i32>, 1, 0)
+// SPV: %"class.hlsl::ConsumeStructuredBuffer.14" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 1)
 
 ConsumeStructuredBuffer<int16_t> BufI16;
 ConsumeStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
index 0944ad59d5fb5..5512a657bc5f0 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
@@ -16,20 +16,20 @@
 // DXIL: %"class.hlsl::RWBuffer.11" = type { target("dx.TypedBuffer", <3 x float>, 1, 0, 0) }
 // DXIL: %"class.hlsl::RWBuffer.12" = type { target("dx.TypedBuffer", <4 x i32>, 1, 0, 1) }
 
-// SPIRV: %"class.hlsl::RWBuffer" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer" = type { target("spirv.SignedImage", i16, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.0" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.1" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.1" = type { target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.2" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.3" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.3" = type { target("spirv.SignedImage", i64, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.4" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.5" = type { target("spirv.Image", half, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.6" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.7" = type { target("spirv.Image", double, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.8" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.8" = type { target("spirv.SignedImage", i16, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.9" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.10" = type { target("spirv.Image", half, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.11" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.12" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.12" = type { target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) }
 
 RWBuffer<int16_t> BufI16;
 RWBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
index cf810ed909eb7..63e3552b680b6 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
@@ -9,18 +9,18 @@ void main(unsigned GI : SV_GroupIndex) {
   // CHECK: define void @main()
 
   // DXC: %[[INPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[INPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[INPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: %[[LOAD:.*]] = load i32, ptr {{.*}}%[[INPTR]]
   // DXC: %[[OUTPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: store i32 %[[LOAD]], ptr {{.*}}%[[OUTPTR]]
   Out[GI] = In[GI];
 
   // DXC: %[[INPTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[INPTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[INPTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: %[[LOAD:.*]] = load i32, ptr {{.*}}%[[INPTR]]
   // DXC: %[[OUTPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: store i32 %[[LOAD]], ptr {{.*}}%[[OUTPTR]]
   Out[GI] = In.Load(GI);
 }
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
index e23859ee83d15..f7cdc5d4d39f5 100644
--- a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 // CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", i16, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.0" = type { target("dx.RawBuffer", i16, 1, 0) }
@@ -14,7 +15,9 @@
 // CHECK: %"class.hlsl::RWStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.12" = type { target("dx.RawBuffer", i32, 1, 0) }
+// SPV: %"class.hlsl::RWStructuredBuffer.12" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 1)
 // CHECK: %"class.hlsl::RWStructuredBuffer.13" = type { target("dx.RawBuffer", <4 x i32>, 1, 0) }
+// SPV: %"class.hlsl::RWStructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 1)
 
 RWStructuredBuffer<int16_t> BufI16;
 RWStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index 6c69b2b34bb29..0bb08065ddc28 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 // CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", i16, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.0" = type { target("dx.RawBuffer", i16, 0, 0) }
@@ -14,7 +15,9 @@
 // CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.12" = type { target("dx.RawBuffer", i32, 0, 0) }
+// SPV: %"class.hlsl::StructuredBuffer.12" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 0)
 // CHECK: %"class.hlsl::StructuredBuffer.13" = type { target("dx.RawBuffer", <4 x i32>, 0, 0) }
+// SPV: %"class.hlsl::StructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 0)
 
 StructuredBuffer<int16_t> BufI16;
 StructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index dc7a83002b7f1..77d2414230cf2 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -108,7 +108,7 @@
 // GFX1153: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
-// GFX1250: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bitop3-insts,+ci-insts,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+transpose-load-f4f6-insts,+wavefrontsize32"
+// GFX1250: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bitop3-insts,+ci-insts,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+fp8e5m3-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+transpose-load-f4f6-insts,+wavefrontsize32"
 
 // GFX1103-W64: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize64"
 
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl
new file mode 100644
index 0000000000000..e3fe31ff7dd75
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl
@@ -0,0 +1,24 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1250
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_atomic_async_barrier_arrive_b64(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) [[ADDR:%.*]])
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_ds_atomic_async_barrier_arrive_b64(local long* addr)
+{
+  __builtin_amdgcn_ds_atomic_async_barrier_arrive_b64(addr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_atomic_barrier_arrive_rtn_b64(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) [[ADDR:%.*]], i64 [[DATA:%.*]])
+// CHECK-GFX1250-NEXT:    store i64 [[TMP0]], ptr [[OUT:%.*]], align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_ds_atomic_barrier_arrive_rtn_b64(local long* addr, long data, long *out)
+{
+  *out = __builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64(addr, data);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl
new file mode 100644
index 0000000000000..1e3a88a41f90e
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl
@@ -0,0 +1,130 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -target-feature +wavefrontsize32 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1250
+
+typedef int    v2i   __attribute__((ext_vector_type(2)));
+typedef int    v3i   __attribute__((ext_vector_type(3)));
+typedef int    v4i   __attribute__((ext_vector_type(4)));
+typedef short  v8s   __attribute__((ext_vector_type(8)));
+typedef half   v8h   __attribute__((ext_vector_type(8)));
+typedef __bf16 v8y  __attribute__((ext_vector_type(8)));
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr4_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_global_load_tr4_b64_v2i32(global v2i* inptr)
+{
+  return __builtin_amdgcn_global_load_tr4_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr8_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_global_load_tr8_b64_v2i32(global v2i* inptr)
+{
+  return __builtin_amdgcn_global_load_tr8_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr6_b96_v3i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3i test_amdgcn_global_load_tr6_b96_v3i32(global v3i* inptr)
+{
+  return __builtin_amdgcn_global_load_tr6_b96_v3i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr16_b128_v8i16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x i16> [[TMP0]]
+//
+v8s test_amdgcn_global_load_tr16_b128_v8i16(global v8s* inptr)
+{
+  return __builtin_amdgcn_global_load_tr16_b128_v8i16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr16_b128_v8f16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x half> [[TMP0]]
+//
+v8h test_amdgcn_global_load_tr16_b128_v8f16(global v8h* inptr)
+{
+  return __builtin_amdgcn_global_load_tr16_b128_v8f16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr16_b128_v8bf16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+v8y test_amdgcn_global_load_tr16_b128_v8bf16(global v8y* inptr)
+{
+  return __builtin_amdgcn_global_load_tr16_b128_v8bf16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr4_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_ds_load_tr4_b64_v2i32(local v2i* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr4_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr8_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_ds_load_tr8_b64_v2i32(local v2i* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr8_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr6_b96_v3i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3i test_amdgcn_ds_load_tr6_b96_v3i32(local v3i* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr6_b96_v3i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr16_b128_v8i16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x i16> [[TMP0]]
+//
+v8s test_amdgcn_ds_load_tr16_b128_v8i16(local v8s* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr16_b128_v8i16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr16_b128_v8f16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x half> [[TMP0]]
+//
+v8h test_amdgcn_ds_load_tr16_b128_v8f16(local v8h* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr16_b128_v8f16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr16_b128_v8bf16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+v8y test_amdgcn_ds_load_tr16_b128_v8bf16(local v8y* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr16_b128_v8bf16(inptr);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl
new file mode 100644
index 0000000000000..49ffbf4517160
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl
@@ -0,0 +1,46 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1250
+
+typedef int    v4i   __attribute__((ext_vector_type(4)));
+typedef int    v8i   __attribute__((ext_vector_type(8)));
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_load_to_lds(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], <4 x i32> [[SG2:%.*]], <4 x i32> [[SG3:%.*]], i32 0)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_load_to_lds(v4i sg0, v8i sg1, v4i sg2, v4i sg3)
+{
+  __builtin_amdgcn_tensor_load_to_lds(sg0, sg1, sg2, sg3, 0);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_load_to_lds_d2(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], i32 27)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_load_to_lds_d2(v4i sg0, v8i sg1)
+{
+  __builtin_amdgcn_tensor_load_to_lds_d2(sg0, sg1, 27);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_store_from_lds(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], <4 x i32> [[SG2:%.*]], <4 x i32> [[SG3:%.*]], i32 22)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_store_from_lds(v4i sg0, v8i sg1, v4i sg2, v4i sg3)
+{
+  __builtin_amdgcn_tensor_store_from_lds(sg0, sg1, sg2, sg3, 22);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_store_from_lds_d2(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], i32 0)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_store_from_lds_d2(v4i sg0, v8i sg1)
+{
+  __builtin_amdgcn_tensor_store_from_lds_d2(sg0, sg1, 0);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 864e301859682..421099d3876e3 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -15,6 +15,91 @@ void test_setprio_inc_wg() {
   __builtin_amdgcn_s_setprio_inc_wg(10);
 }
 
+// CHECK-LABEL: @test_s_monitor_sleep(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.monitor.sleep(i16 10)
+// CHECK-NEXT:    ret void
+//
+void test_s_monitor_sleep() {
+  __builtin_amdgcn_s_monitor_sleep(10);
+}
+
+// CHECK-LABEL: @test_cvt_f16_fp8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP0]], i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP2]], i64 0
+// CHECK-NEXT:    store half [[TMP1]], ptr addrspace(1) [[ARRAYIDX]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP3]], i32 1)
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP5]], i64 1
+// CHECK-NEXT:    store half [[TMP4]], ptr addrspace(1) [[ARRAYIDX1]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP6]], i32 2)
+// CHECK-NEXT:    [[TMP8:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP8]], i64 2
+// CHECK-NEXT:    store half [[TMP7]], ptr addrspace(1) [[ARRAYIDX2]], align 2
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP9]], i32 3)
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP11]], i64 3
+// CHECK-NEXT:    store half [[TMP10]], ptr addrspace(1) [[ARRAYIDX3]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_cvt_f16_fp8(global half* out, int a)
+{
+  out[0] = __builtin_amdgcn_cvt_f16_fp8(a, 0);
+  out[1] = __builtin_amdgcn_cvt_f16_fp8(a, 1);
+  out[2] = __builtin_amdgcn_cvt_f16_fp8(a, 2);
+  out[3] = __builtin_amdgcn_cvt_f16_fp8(a, 3);
+}
+
+// CHECK-LABEL: @test_cvt_f16_bf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP0]], i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP2]], i64 0
+// CHECK-NEXT:    store half [[TMP1]], ptr addrspace(1) [[ARRAYIDX]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP3]], i32 1)
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP5]], i64 1
+// CHECK-NEXT:    store half [[TMP4]], ptr addrspace(1) [[ARRAYIDX1]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP6]], i32 2)
+// CHECK-NEXT:    [[TMP8:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP8]], i64 2
+// CHECK-NEXT:    store half [[TMP7]], ptr addrspace(1) [[ARRAYIDX2]], align 2
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP9]], i32 3)
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP11]], i64 3
+// CHECK-NEXT:    store half [[TMP10]], ptr addrspace(1) [[ARRAYIDX3]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_cvt_f16_bf8(global half* out, int a)
+{
+  out[0] = __builtin_amdgcn_cvt_f16_bf8(a, 0);
+  out[1] = __builtin_amdgcn_cvt_f16_bf8(a, 1);
+  out[2] = __builtin_amdgcn_cvt_f16_bf8(a, 2);
+  out[3] = __builtin_amdgcn_cvt_f16_bf8(a, 3);
+}
+
 // CHECK-LABEL: @test_cvt_pk_f16_fp8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -54,3 +139,41 @@ void test_cvt_pk_f16_bf8(global half2* out, short a)
 {
   out[0] = __builtin_amdgcn_cvt_pk_f16_bf8(a);
 }
+
+// CHECK-LABEL: @test_cvt_f32_fp8_e5m3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 [[TMP0]], i32 0)
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[TMP1]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[CONV]], ptr addrspace(1) [[TMP2]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 [[TMP3]], i32 1)
+// CHECK-NEXT:    [[CONV1:%.*]] = fptosi float [[TMP4]] to i32
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[CONV1]], ptr addrspace(1) [[TMP5]], align 4
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 [[TMP6]], i32 2)
+// CHECK-NEXT:    [[CONV2:%.*]] = fptosi float [[TMP7]] to i32
+// CHECK-NEXT:    [[TMP8:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[CONV2]], ptr addrspace(1) [[TMP8]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 [[TMP9]], i32 3)
+// CHECK-NEXT:    [[CONV3:%.*]] = fptosi float [[TMP10]] to i32
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[CONV3]], ptr addrspace(1) [[TMP11]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_cvt_f32_fp8_e5m3(global int* out, int a)
+{
+  *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, 0);
+  *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, 1);
+  *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, 2);
+  *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, 3);
+}
diff --git a/clang/test/CodeGenSYCL/unique_stable_name_windows_diff.cpp b/clang/test/CodeGenSYCL/unique_stable_name_windows_diff.cpp
index 7dd08a0c89255..14366a092a1fe 100644
--- a/clang/test/CodeGenSYCL/unique_stable_name_windows_diff.cpp
+++ b/clang/test/CodeGenSYCL/unique_stable_name_windows_diff.cpp
@@ -1,19 +1,19 @@
-// RUN: %clang_cc1 -triple spir64-unknown-unknown-sycldevice -aux-triple x86_64-pc-windows-msvc -fsycl-is-device -disable-llvm-passes -fsycl-is-device -emit-llvm %s -o - | FileCheck %s '-D$ADDRSPACE=addrspace(1) '
-// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -fsycl-is-device -disable-llvm-passes -fsycl-is-device -emit-llvm %s -o - | FileCheck %s '-D$ADDRSPACE='
+// RUN: %clang_cc1 -triple spir64-unknown-unknown -aux-triple x86_64-pc-windows-msvc -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s '-D$ADDRSPACE=addrspace(1) '
+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -fsycl-is-host -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s '-D$ADDRSPACE='
 
 
 template<typename KN, typename Func>
-__attribute__((sycl_kernel)) void kernel(Func F){
+[[clang::sycl_kernel_entry_point(KN)]] void kernel(Func F){
   F();
 }
 
 template<typename KN, typename Func>
-__attribute__((sycl_kernel)) void kernel2(Func F){
+[[clang::sycl_kernel_entry_point(KN)]] void kernel2(Func F){
   F(1);
 }
 
 template<typename KN, typename Func>
-__attribute__((sycl_kernel)) void kernel3(Func F){
+[[clang::sycl_kernel_entry_point(KN)]] void kernel3(Func F){
   F(1.1);
 }
 
@@ -31,10 +31,19 @@ int main() {
 
   // Ensure the kernels are named the same between the device and host
   // invocations.
+  // Call from host.
   (void)__builtin_sycl_unique_stable_name(decltype(lambda1));
   (void)__builtin_sycl_unique_stable_name(decltype(lambda2));
   (void)__builtin_sycl_unique_stable_name(decltype(lambda3));
 
+  // Call from device.
+  auto lambda4 = [](){
+    (void)__builtin_sycl_unique_stable_name(decltype(lambda1));
+    (void)__builtin_sycl_unique_stable_name(decltype(lambda2));
+    (void)__builtin_sycl_unique_stable_name(decltype(lambda3));
+  };
+  kernel<class K4>(lambda4);
+
   // Make sure the following 3 are the same between the host and device compile.
   // Note that these are NOT the same value as each other, they differ by the
   // signature.
diff --git a/clang/test/DebugInfo/KeyInstructions/flag.cpp b/clang/test/DebugInfo/KeyInstructions/flag.cpp
index 93503dd4bdb4c..813f7e908011c 100644
--- a/clang/test/DebugInfo/KeyInstructions/flag.cpp
+++ b/clang/test/DebugInfo/KeyInstructions/flag.cpp
@@ -3,13 +3,19 @@
 //// Default: Off.
 // RUN: %clang -### -target x86_64 -c -gdwarf %s 2>&1 | FileCheck %s --check-prefixes=NO-KEY-INSTRUCTIONS
 
-//// Help hidden.
-// RUN %clang --help | FileCheck %s --check-prefix=HELP
-// HELP-NOT: key-instructions
-
 // KEY-INSTRUCTIONS: "-gkey-instructions"
-// KEY-INSTRUCTIONS: "-mllvm" "-dwarf-use-key-instructions"
-
 // NO-KEY-INSTRUCTIONS-NOT: key-instructions
 
-//// TODO: Add smoke test once some functionality has been added.
+//// Help hidden: flag should not be visible.
+// RUN: %clang --help | FileCheck %s --check-prefix=HELP
+// HELP-NOT: key-instructions
+
+// Smoke test: check for Key Instructions keywords in the IR.
+void f() {}
+// RUN: %clang_cc1 %s -triple x86_64-linux-gnu -debug-info-kind=line-tables-only -emit-llvm -o - | FileCheck %s --check-prefix=SMOKETEST-OFF
+// SMOKETEST-OFF-NOT: keyInstructions:
+// SMOKETEST-OFF-NOT: atomGroup
+
+// RUN: %clang_cc1 %s -triple x86_64-linux-gnu -gkey-instructions -debug-info-kind=line-tables-only -emit-llvm -o - | FileCheck %s --check-prefix=SMOKETEST-ON
+// SMOKETEST-ON: keyInstructions: true
+// SMOKETEST-ON: atomGroup: 1
diff --git a/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp
new file mode 100644
index 0000000000000..c94fc588bf13b
--- /dev/null
+++ b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions %s -gno-column-info -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s
+
+// g::h and i can be memcpy'd, check the assignment gets Key Instructions metadata.
+
+struct e {
+  e(e &);
+  e& operator=(const e&);
+};
+
+struct g {
+  e f;
+  int h;
+  int i;
+};
+
+// Copy assignment operator.
+// CHECK: define{{.*}}ptr @_ZN1gaSERKS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %call = call {{.*}}ptr @_ZN1eaSERKS_(ptr {{.*}}%this1, ptr {{.*}}%1)
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: call void @llvm.memcpy{{.*}}(ptr align 4 %h, ptr align 4 %h2, i64 8, i1 false), !dbg [[S1_G1R1:!.*]]
+// CHECK-NEXT: ret ptr %this1, !dbg
+
+// Copy ctor.
+// CHECK: define{{.*}}void @_ZN1gC2ERS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: call void @_ZN1eC1ERS_
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: call void @llvm.memcpy{{.*}}(ptr align 4 %h, ptr align 4 %h2, i64 8, i1 false), !dbg [[S2_G1R1:!.*]]
+// CHECK-NEXT: ret void, !dbg
+
+// CHECK: [[S1:!.*]] = distinct !DISubprogram(name: "operator=",
+// CHECK: [[S1_G1R1]] = !DILocation(line: 11, scope: [[S1]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[S2:!.*]] = distinct !DISubprogram(name: "g",
+// CHECK: [[S2_G1R1]] = !DILocation(line: 11, scope: [[S2]], atomGroup: 1, atomRank: 1)
+
+[[gnu::nodebug]]
+void fun(g *x) {
+  g y = g(*x);
+  y = *x;
+}
diff --git a/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp
new file mode 100644
index 0000000000000..cd3807735fa32
--- /dev/null
+++ b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions %s -gno-column-info -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s
+
+// g::h can be memcpy'd (in this case emitted as load/stored), check the
+// assignment gets Key Instructions metadata.
+
+struct e {
+  e(e&);
+  e& operator=(const e&);
+};
+
+struct g {
+  e f;
+  int h;
+};
+
+// Copy assignment operator.
+// CHECK: define{{.*}}ptr @_ZN1gaSERKS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %call = call {{.*}}ptr @_ZN1eaSERKS_(ptr {{.*}}%this1, ptr {{.*}}%1)
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: %3 = load i32, ptr %h, align 4, !dbg [[S1_G1R2:!.*]]
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: store i32 %3, ptr %h2, align 4, !dbg [[S1_G1R1:!.*]]
+// CHECK-NEXT: ret ptr %this1, !dbg
+
+// Copy ctor.
+// CHECK: define{{.*}}void @_ZN1gC2ERS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: call void @_ZN1eC1ERS_
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: %3 = load i32, ptr %h2, align 4, !dbg [[S2_G1R2:!.*]]
+// CHECK-NEXT: store i32 %3, ptr %h, align 4, !dbg [[S2_G1R1:!.*]]
+// CHECK-NEXT: ret void, !dbg
+
+// CHECK: [[S1:!.*]] = distinct !DISubprogram(name: "operator=",
+// CHECK: [[S1_G1R2]] = !DILocation(line: 12, scope: [[S1]], atomGroup: 1, atomRank: 2)
+// CHECK: [[S1_G1R1]] = !DILocation(line: 12, scope: [[S1]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[S2:!.*]] = distinct !DISubprogram(name: "g",
+// CHECK: [[S2_G1R2]] = !DILocation(line: 12, scope: [[S2]], atomGroup: 1, atomRank: 2)
+// CHECK: [[S2_G1R1]] = !DILocation(line: 12, scope: [[S2]], atomGroup: 1, atomRank: 1)
+
+void fun(g *x) {
+  g y = g(*x);
+  y = *x;
+}
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/a.txt b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/a.txt
rename to clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt
rename to clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
new file mode 100755
index 0000000000000..b23e55619b2ff
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
new file mode 100755
index 0000000000000..b23e55619b2ff
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
new file mode 100755
index 0000000000000..b23e55619b2ff
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
new file mode 100755
index 0000000000000..b23e55619b2ff
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/offload-arch/offload_arch_sm_70_gfx906 b/clang/test/Driver/Inputs/offload-arch/offload_arch_sm_70_gfx906
new file mode 100644
index 0000000000000..e604142a9d258
--- /dev/null
+++ b/clang/test/Driver/Inputs/offload-arch/offload_arch_sm_70_gfx906
@@ -0,0 +1,4 @@
+#!/bin/sh
+echo "sm_70"
+echo "gfx906"
+exit 0
diff --git a/clang/test/Driver/aarch64-gnutools.c b/clang/test/Driver/aarch64-gnutools.c
new file mode 100644
index 0000000000000..0214639ed3804
--- /dev/null
+++ b/clang/test/Driver/aarch64-gnutools.c
@@ -0,0 +1,4 @@
+// RUN: %clang --target=aarch64-none-elf  --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -fno-integrated-as %s -### -c \
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/aarch64-implied-sve-features.c b/clang/test/Driver/aarch64-implied-sve-features.c
index 18c39974a5c14..1bda2f529fc70 100644
--- a/clang/test/Driver/aarch64-implied-sve-features.c
+++ b/clang/test/Driver/aarch64-implied-sve-features.c
@@ -49,7 +49,7 @@
 // SVE2-SHA3-REVERT: "-target-feature" "+sve" "-target-feature" "-sve-sha3" "-target-feature" "+sve2" "-target-feature" "-sve2-sha3"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4+nosve2-sm4 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SM4-REVERT
-// SVE2-SM4-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-sm4"
+// SVE2-SM4-REVERT: "-target-feature" "+sve" "-target-feature" "-sve-sm4" "-target-feature" "+sve2" "-target-feature" "-sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sha3 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SHA3
 // SVE2-SHA3: "-target-feature" "+sve" "-target-feature" "+sve-sha3" "-target-feature" "+sve2" "-target-feature" "+sve2-sha3"
@@ -61,14 +61,14 @@
 // SVE2-AES: "-target-feature" "+sve" "-target-feature" "+sve-aes" "-target-feature" "+sve2" "-target-feature" "+sve2-aes"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SM4
-// SVE2-SM4: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-sm4"
+// SVE2-SM4: "-target-feature" "+sve" "-target-feature" "+sve-sm4" "-target-feature" "+sve2" "-target-feature" "+sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-bitperm+nosve2-aes %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SUBFEATURE-MIX
 // SVE2-SUBFEATURE-MIX: "-target-feature" "+sve" "-target-feature" "+sve-bitperm" "-target-feature" "+sve2" "-target-feature" "+sve2-bitperm"
 // SVE2-SUBFEATURE-NOT: sve2-aes
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4+nosve2 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SUBFEATURE-CONFLICT
-// SVE2-SUBFEATURE-CONFLICT: "-target-feature" "+sve" "-target-feature" "-sve2" "-target-feature" "-sve2-sm4"
+// SVE2-SUBFEATURE-CONFLICT: "-target-feature" "+sve" "-target-feature" "-sve-sm4" "-target-feature" "-sve2" "-target-feature" "-sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-aes+nosve %s -### 2>&1 | FileCheck %s --check-prefix=SVE-SUBFEATURE-CONFLICT
 // SVE-SUBFEATURE-CONFLICT-NOT: "-target-feature" "+sve2-aes"
diff --git a/clang/test/Driver/aarch64-sve-vector-bits.c b/clang/test/Driver/aarch64-sve-vector-bits.c
index 535b0f157019a..f54cd94764bfb 100644
--- a/clang/test/Driver/aarch64-sve-vector-bits.c
+++ b/clang/test/Driver/aarch64-sve-vector-bits.c
@@ -24,6 +24,8 @@
 // RUN:  -msve-vector-bits=2048+ 2>&1 | FileCheck --check-prefix=CHECK-2048P %s
 // RUN: %clang -c %s -### --target=aarch64-none-linux-gnu -march=armv8-a+sve \
 // RUN:  -msve-vector-bits=scalable 2>&1 | FileCheck --check-prefix=CHECK-SCALABLE %s
+// RUN: %clang -c %s -### --target=aarch64-none-linux-gnu -march=armv8-a+sve+sme \
+// RUN:  -msve-streaming-vector-bits=128 2>&1 | FileCheck --check-prefix=STREAMING-128 %s
 
 // CHECK-128: "-mvscale-max=1" "-mvscale-min=1"
 // CHECK-256: "-mvscale-max=2" "-mvscale-min=2"
@@ -44,6 +46,8 @@
 // CHECK-SCALABLE-NOT: "-mvscale-min=
 // CHECK-SCALABLE-NOT: "-mvscale-max=
 
+// STREAMING-128: "-mvscale-streaming-max=1" "-mvscale-streaming-min=1"
+
 // Error out if an unsupported value is passed to -msve-vector-bits.
 // -----------------------------------------------------------------------------
 // RUN: not %clang -c %s -### --target=aarch64-none-linux-gnu -march=armv8-a+sve \
diff --git a/clang/test/Driver/aarch64-toolchain-extra.c b/clang/test/Driver/aarch64-toolchain-extra.c
new file mode 100644
index 0000000000000..4945a622969c6
--- /dev/null
+++ b/clang/test/Driver/aarch64-toolchain-extra.c
@@ -0,0 +1,35 @@
+// A basic clang -cc1 command-line, and simple environment check.
+
+// The tests here are similar to those in aarch64-toolchain.c, however
+// these tests need to create symlinks to test directory trees in order to
+// set up the environment and therefore shell support is required.
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// If there is no GCC install detected then the driver searches for executables
+// and runtime starting from the directory tree above the driver itself.
+// The test below checks that the driver correctly finds the linker and
+// runtime if and only if they exist.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/aarch64-nogcc/bin
+// RUN: ln -s %clang %t/aarch64-nogcc/bin/clang
+// RUN: ln -s %S/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf %t/aarch64-nogcc/aarch64-none-elf
+// RUN: ln -s %S/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld %t/aarch64-nogcc/bin/aarch64-none-elf-ld
+// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --gcc-toolchain=%t/aarch64-nogcc/invalid \
+// RUN:    --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOGCC %s
+
+// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --sysroot=%t/aarch64-nogcc/bin/../aarch64-none-elf \
+// RUN:    --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOGCC %s
+
+// C-AARCH64-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/include"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/bin/aarch64-none-elf-ld"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/lib/crt0.o"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/{{.*}}/aarch64-none-elf/lib/crtbegin.o"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/lib"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/{{.*}}/aarch64-none-elf/lib/crtend.o"
diff --git a/clang/test/Driver/aarch64-toolchain.c b/clang/test/Driver/aarch64-toolchain.c
new file mode 100644
index 0000000000000..cfad4b8eb6829
--- /dev/null
+++ b/clang/test/Driver/aarch64-toolchain.c
@@ -0,0 +1,159 @@
+// UNSUPPORTED: system-windows
+
+// Test interaction with -fuse-ld=lld
+// RUN: %clang -### %s -fuse-ld=lld -B%S/Inputs/lld \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=LLD-AARCH64-BAREMETAL %s
+
+// LLD-AARCH64-BAREMETAL: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// LLD-AARCH64-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// LLD-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/lld/ld.lld"
+// LLD-AARCH64-BAREMETAL: "-Bstatic" "-m" "aarch64linux" "-EL"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// LLD-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// LLD-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// LLD-AARCH64-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL %s
+
+// C-AARCH64-BAREMETAL: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// C-AARCH64-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// C-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// C-AARCH64-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// C-AARCH64-BAREMETAL: "-Bstatic" "-m" "aarch64linux" "-EL"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// C-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// C-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// C-AARCH64-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOSYSROOT %s
+
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "aarch64linux" "-EL"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib/crt0.o"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL %s
+
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/backward"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT %s
+
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/backward"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-LIBCXX %s
+
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/v1"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "--sysroot={{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX %s
+
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/v1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=compiler-rt --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=AARCH64-BAREMETAL-COMPILER-RT %s
+
+// AARCH64-BAREMETAL-COMPILER-RT: "{{.*}}crt0.o"
+// AARCH64-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtbegin.o"
+// AARCH64-BAREMETAL-COMPILER-RT: "--start-group" "{{.*}}libclang_rt.builtins{{.*}}.a" "-lc" "-lgloss" "--end-group"
+// AARCH64-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=platform --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=AARCH64-BAREMETAL-UNWINDLIB %s
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=compiler-rt --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=AARCH64-BAREMETAL-UNWINDLIB %s
+
+// AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}crt0.o"
+// AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtbegin.o"
+// AARCH64-BAREMETAL-UNWINDLIB: "--start-group" "{{.*}}libclang_rt.builtins{{.*}}.a" "--as-needed" "-lunwind" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtend.o"
diff --git a/clang/test/Driver/aix-default-target-triple.c b/clang/test/Driver/aix-default-target-triple.c
new file mode 100644
index 0000000000000..afad278de78d0
--- /dev/null
+++ b/clang/test/Driver/aix-default-target-triple.c
@@ -0,0 +1,14 @@
+// Test for the Triple constructor ambiguity fix on AIX
+// This test verifies that the default target triple is correctly resolved
+// and doesn't fall back to "unknown" due to constructor ambiguity.
+
+// REQUIRES: system-aix
+// RUN: %clang -v %s -c 2>&1 | FileCheck %s --check-prefix=CHECK-TARGET
+
+// Test that the target triple contains AIX and is not "unknown"
+// The target should be something like "powerpc-ibm-aix7.3.0.0"
+// CHECK-TARGET: Target: {{.*}}aix{{.*}}
+
+int main() {
+    return 0;
+}
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57ccc..985eca1692802 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/arm-gnutools.c b/clang/test/Driver/arm-gnutools.c
new file mode 100644
index 0000000000000..6e107f19dabc5
--- /dev/null
+++ b/clang/test/Driver/arm-gnutools.c
@@ -0,0 +1,6 @@
+// check that gnu assembler is invoked with arm baremetal as well
+
+// RUN: %clang --target=armv6m-none-eabi  --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -fno-integrated-as %s -### -c \
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/arm-toolchain-extra.c b/clang/test/Driver/arm-toolchain-extra.c
new file mode 100644
index 0000000000000..43cca6112176c
--- /dev/null
+++ b/clang/test/Driver/arm-toolchain-extra.c
@@ -0,0 +1,36 @@
+// A basic clang -cc1 command-line, and simple environment check.
+
+// The tests here are similar to those in arm-toolchain.c, however
+// these tests need to create symlinks to test directory trees in order to
+// set up the environment and therefore shell support is required.
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// If there is no GCC install detected then the driver searches for executables
+// and runtime starting from the directory tree above the driver itself.
+// The test below checks that the driver correctly finds the linker and
+// runtime if and only if they exist.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/arm-nogcc/bin
+// RUN: ln -s %clang %t/arm-nogcc/bin/clang
+// RUN: ln -s %S/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi %t/arm-nogcc/armv6m-none-eabi
+// RUN: ln -s %S/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld %t/arm-nogcc/bin/armv6m-none-eabi-ld
+// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --gcc-toolchain=%t/arm-nogcc/invalid \
+// RUN:    --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --sysroot=%t/arm-nogcc/bin/../armv6m-none-eabi \
+// RUN:    --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// C-ARM-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/include"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/bin/armv6m-none-eabi-ld"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/lib/crt0.o"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/{{.*}}/armv6m-none-eabi/lib/crtbegin.o"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/lib"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/{{.*}}/armv6m-none-eabi/lib/crtend.o"
+
diff --git a/clang/test/Driver/arm-toolchain.c b/clang/test/Driver/arm-toolchain.c
new file mode 100644
index 0000000000000..c367594b0a758
--- /dev/null
+++ b/clang/test/Driver/arm-toolchain.c
@@ -0,0 +1,160 @@
+// UNSUPPORTED: system-windows
+
+// RUN: %clang -### %s -fuse-ld=lld -B%S/Inputs/lld \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=LLD-ARM-BAREMETAL %s
+
+// LLD-ARM-BAREMETAL: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// LLD-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// LLD-ARM-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/lld/ld.lld"
+// LLD-ARM-BAREMETAL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// LLD-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// LLD-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// LLD-ARM-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL %s
+
+// C-ARM-BAREMETAL: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// C-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// C-ARM-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// C-ARM-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// C-ARM-BAREMETAL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// C-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// C-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// C-ARM-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL-NOSYSROOT %s
+
+// C-ARM-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// C-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// C-ARM-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib/crt0.o"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// C-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// C-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL %s
+
+// CXX-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/backward"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT %s
+
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/backward"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-LIBCXX %s
+
+// CXX-ARM-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/v1"
+// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL-LIBCXX: "--sysroot={{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-LIBCXX: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX %s
+
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/v1"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=compiler-rt --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=ARM-BAREMETAL-COMPILER-RT %s
+
+// ARM-BAREMETAL-COMPILER-RT: "{{.*}}crt0.o"
+// ARM-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtbegin.o"
+// ARM-BAREMETAL-COMPILER-RT: "--start-group" "{{.*}}libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
+// ARM-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=platform --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=ARM-BAREMETAL-UNWINDLIB %s
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=compiler-rt --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=ARM-BAREMETAL-UNWINDLIB %s
+
+// ARM-BAREMETAL-UNWINDLIB: "{{.*}}crt0.o"
+// ARM-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtbegin.o"
+// ARM-BAREMETAL-UNWINDLIB: "--start-group" "{{.*}}libclang_rt.builtins.a" "--as-needed" "-lunwind" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// ARM-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtend.o"
diff --git a/clang/test/Driver/autocomplete.c b/clang/test/Driver/autocomplete.c
index 8cc604dbff875..4983b71496834 100644
--- a/clang/test/Driver/autocomplete.c
+++ b/clang/test/Driver/autocomplete.c
@@ -111,6 +111,7 @@
 // RUN: %clang --autocomplete=-Wma | FileCheck %s -check-prefix=WARNING
 // WARNING: -Wmacro-redefined
 // WARNING-NEXT: -Wmain
+// WARNING-NEXT: -Wmain-attached-to-named-module
 // WARNING-NEXT: -Wmain-return-type
 // WARNING-NEXT: -Wmalformed-warning-check
 // WARNING-NEXT: -Wmany-braces-around-scalar-init
diff --git a/clang/test/Driver/baremetal-multilib.yaml b/clang/test/Driver/baremetal-multilib.yaml
index 853a4e9e36e43..1a80c3b4ccfc8 100644
--- a/clang/test/Driver/baremetal-multilib.yaml
+++ b/clang/test/Driver/baremetal-multilib.yaml
@@ -8,8 +8,9 @@
 # CHECK-SAME: "-internal-isystem" "[[SYSROOT:[^"]*]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/include/c++/v1"
 # CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/include"
 # CHECK-SAME: "-x" "c++" "{{.*}}baremetal-multilib.yaml"
-# CHECK-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+# CHECK-NEXT: ld{{(.exe)?}}" "-Bstatic"
 # CHECK-SAME: "-L[[SYSROOT]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/lib"
+# CHECK-SAME: "{{.*}}.o"
 # CHECK-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 # CHECK-SAME: "-lc"
 # CHECK-SAME: "-o" "{{.*}}.tmp.out"
diff --git a/clang/test/Driver/baremetal-sysroot.cpp b/clang/test/Driver/baremetal-sysroot.cpp
index 5d5b336a01b0b..47f0616df8501 100644
--- a/clang/test/Driver/baremetal-sysroot.cpp
+++ b/clang/test/Driver/baremetal-sysroot.cpp
@@ -9,15 +9,17 @@
 // RUN: mkdir -p %T/baremetal_default_sysroot/lib/clang-runtimes/armv6m-none-eabi
 // RUN: ln -s %clang %T/baremetal_default_sysroot/bin/clang
 
-// RUN: %T/baremetal_default_sysroot/bin/clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
+// RUN: %T/baremetal_default_sysroot/bin/clang -no-canonical-prefixes %s -### -o %t.out 2>&1 \
 // RUN:     -target armv6m-none-eabi --sysroot= \
 // RUN:   | FileCheck --check-prefix=CHECK-V6M-C %s
 // CHECK-V6M-C: "{{.*}}clang{{.*}}" "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
 // CHECK-V6M-C-SAME: "-internal-isystem" "{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-C-SAME: "-internal-isystem" "{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}include"
 // CHECK-V6M-C-SAME: "-x" "c++" "{{.*}}baremetal-sysroot.cpp"
-// CHECK-V6M-C-NEXT: "{{[^"]*}}ld{{(\.(lld|bfd|gold))?}}{{(\.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-V6M-C-NEXT: "{{[^"]*}}ld{{(\.(lld|bfd|gold))?}}{{(\.exe)?}}" "-Bstatic"
+// CHECK-V6M-C-SAME: "crt0.o"
 // CHECK-V6M-C-SAME: "-L{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}lib"
+// CHECK-V6M-C-SAME: "{{.*}}.o"
 // CHECK-V6M-C-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-C-SAME: "-lc"
-// CHECK-V6M-C-SAME: "-o" "{{.*}}.o"
+// CHECK-V6M-C-SAME: "-o" "{{.*}}.tmp.out"
diff --git a/clang/test/Driver/baremetal-undefined-symbols.c b/clang/test/Driver/baremetal-undefined-symbols.c
new file mode 100644
index 0000000000000..6ed955b1b749c
--- /dev/null
+++ b/clang/test/Driver/baremetal-undefined-symbols.c
@@ -0,0 +1,10 @@
+// Check the arguments are correctly passed
+
+// Make sure -T is the last with gcc-toolchain option
+// RUN: %clang -### --target=aarch64-none-elf --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-LD %s
+// RUN: %clang -### --target=armv6m-none-eabi --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-LD %s
+// RUN: %clang -### --target=riscv32 --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-LD %s
+// CHECK-LD: {{.*}} "-T" "a.lds" "-u" "foo" {{.*}} "--defsym=FOO=10"
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index a80aa9b437117..4dc320191317e 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -15,11 +15,14 @@
 // CHECK-V6M-C-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-C-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include"
 // CHECK-V6M-C-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-V6M-C-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-C-NEXT: ld{{(.exe)?}}"
+// CHECK-V6M-C-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-C-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-C-SAME: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-V6M-C-SAME: "-T" "semihosted.lds" "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-V6M-C-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
-// CHECK-V6M-C-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-C-SAME: "{{.*}}.o"
+// CHECK-V6M-C-SAME: {{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-C-SAME: "-lc"
 // CHECK-V6M-C-SAME: "--target2=rel" "-o" "{{.*}}.tmp.out"
 
@@ -39,9 +42,11 @@
 // CHECK-V6M-TREE-SAME: {{^}} "-internal-isystem" "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-TREE-SAME: "-internal-isystem" "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}include{{[/\\]+}}armv6m-unknown-none-eabi"
 // CHECK-V6M-TREE-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-V6M-TREE-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-TREE-NEXT: ld{{(.exe)?}}"
+// CHECK-V6M-TREE-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-TREE-SAME: "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}armv6m-unknown-none-eabi{{[/\\]+}}crt0.o"
 // CHECK-V6M-TREE-SAME: "-L[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}armv6m-unknown-none-eabi"
+// CHECK-V6M-TREE-SAME "{{.*}}.o"
 // CHECK-V6M-TREE-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-TREE-SAME: "-lc"
 // CHECK-V6M-TREE-SAME: "--target2=rel" "-o" "{{.*}}.tmp.out"
@@ -53,19 +58,25 @@
 // CHECK-ARMV7M-PER-TARGET: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-ARMV7M-PER-TARGET: "-isysroot" "[[SYSROOT:[^"]*]]"
 // CHECK-ARMV7M-PER-TARGET: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-ARMV7M-PER-TARGET: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-ARMV7M-PER-TARGET: ld{{(.exe)?}}"
+// CHECK-ARMV7M-PER-TARGET: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-ARMV7M-PER-TARGET: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-ARMV7M-PER_TARGET: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-ARMV7M-PER-TARGET: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
 // CHECK-ARMV7M-PER-TARGET: "-L[[RESOURCE_DIR:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}armv7m-vendor-none-eabi
+// CHECK-ARMV7M-PER-TARGET: "{{.*}}.o"
 // CHECK-ARMV7M-PER-TARGET: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-ARMV7M-PER-TARGET: "-lc"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-DEFAULTCXX %s
 // CHECK-V6M-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-V6M-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-DEFAULTCXX: ld{{(.exe)?}}"
+// CHECK-V6M-DEFAULTCXX: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-DEFAULTCXX: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-DEFAULTCXX-SAME: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-V6M-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
+// CHECK-V6M-DEFAULTCXX-SAME: "{{.*}}.o"
 // CHECK-V6M-DEFAULTCXX-SAME: "-lc++"
 // CHECK-V6M-DEFAULTCXX-SAME: "-lm"
 // CHECK-V6M-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
@@ -77,8 +88,11 @@
 // CHECK-V6M-LIBCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-V6M-LIBCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}{{[^v].*}}"
 // CHECK-V6M-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-V6M-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-LIBCXX: ld{{(.exe)?}}"
+// CHECK-V6M-LIBCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-LIBCXX-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
+// CHECK-V6M-LIBCXX-SAME: "{{.*}}.o"
 // CHECK-V6M-LIBCXX-SAME: "-lc++"
 // CHECK-V6M-LIBCXX-SAME: "-lm"
 // CHECK-V6M-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
@@ -92,8 +106,11 @@
 // CHECK-V6M-LIBSTDCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-V6M-LIBSTDCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}6.0.0"
-// CHECK-V6M-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-LIBSTDCXX: ld{{(.exe)?}}"
+// CHECK-V6M-LIBSTDCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-LIBSTDCXX-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
+// CHECK-V6M-LIBSTDCXX-SAME: "{{.*}}.o"
 // CHECK-V6M-LIBSTDCXX-SAME: "-lstdc++" "-lm"
 // CHECK-V6M-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-LIBSTDCXX-SAME: "-lc"
@@ -104,7 +121,9 @@
 // RUN:     -nodefaultlibs \
 // RUN:   | FileCheck --check-prefix=CHECK-V6M-NDL %s
 // CHECK-V6M-NDL: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-V6M-NDL: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-NDL: ld{{(.exe)?}}"
+// CHECK-V6M-NDL: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-NDL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-NDL-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
 
 // RUN: rm -rf %T/baremetal_cxx_sysroot
@@ -119,6 +138,7 @@
 // CHECK-V6M-LIBCXX-USR-SAME: "-internal-isystem" "{{[^"]+}}baremetal_cxx_sysroot{{[/\\]+}}usr{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-LIBCXX-USR: "{{[^"]*}}-Bstatic"
 // CHECK-V6M-LIBCXX-USR-SAME: "-L{{[^"]*}}{{[/\\]+}}baremetal_cxx_sysroot{{[/\\]+}}lib"
+// CHECK-V6M-LIBCXX-USR: "{{.*}}.o"
 // CHECK-V6M-LIBCXX-USR-SAME: "-lc++" "-lm"
 // CHECK-V6M-LIBCXX-USR-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-LIBCXX-USR-SAME: "-lc"
@@ -149,7 +169,9 @@
 
 // RUN: %clang -### %s --target=armebv7-none-eabi --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-ARMV7EB %s
-// CHECK-ARMV7EB: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "--be8" "-EB"
+// CHECK-ARMV7EB: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ARMV7EB: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-ARMV7EB: "-Bstatic" "-m" "armelfb_linux_eabi" "--be8" "-EB"
 
 // RUN: %clang -### %s --target=armv7-none-eabi -mbig-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-ARMV7EB %s
@@ -159,7 +181,9 @@
 
 // RUN: %clang -### %s --target=armv7-none-eabi --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-ARMV7EL %s
-// CHECK-ARMV7EL: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-ARMV7EL: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ARMV7EL: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-ARMV7EL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-ARMV7EL-NOT: "--be8"
 
 // RUN: %clang -### %s --target=armebv7-none-eabi -mlittle-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
@@ -170,7 +194,9 @@
 
 // RUN: %clang -### %s --target=aarch64_be-none-elf --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-AARCH64BE %s
-// CHECK-AARCH64BE: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EB"
+// CHECK-AARCH64BE: "{{.*}}ld{{(.exe)?}}"
+// CHECK-AARCH64BE: sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-AARCH64BE: "-Bstatic" "-m" "aarch64linuxb" "-EB"
 // CHECK-AARCH64BE-NOT: "--be8"
 
 // RUN: %clang -### %s --target=aarch64-none-elf -mbig-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
@@ -181,7 +207,9 @@
 
 // RUN: %clang -### %s --target=aarch64-none-elf --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-AARCH64LE %s
-// CHECK-AARCH64LE: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-AARCH64LE: "{{.*}}ld{{(.exe)?}}"
+// CHECK-AARCH64LE: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-AARCH64LE: "-Bstatic" "-m" "aarch64linux" "-EL"
 // CHECK-AARCH64LE-NOT: "--be8"
 
 // RUN: %clang -### %s --target=aarch64_be-none-elf -mlittle-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
@@ -196,6 +224,22 @@
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 
+// RUN: %clang -no-canonical-prefixes %s -### --target=riscv32-unknown-elf 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-RISCV32-NO-HOST-INC %s
+// CHECK-RISCV32-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
+// CHECK-RISCV32-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
+
+// RUN: %clang -no-canonical-prefixes %s -### --target=riscv64-unknown-elf 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-RISCV64-NO-HOST-INC %s
+// CHECK-RISCV64-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
+// CHECK-RISCV64-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
+
 // RUN: %clang %s -### --target=riscv64-unknown-elf -o %t.out -L some/directory/user/asked/for \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64 %s
@@ -205,9 +249,12 @@
 // CHECK-RV64-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include"
 // CHECK-RV64-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV64-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-NEXT: ld{{(.exe)?}}"
+// CHECK-RV64-SAME: "--sysroot={{.*}}/Inputs/basic_riscv64_tree/riscv64-unknown-elf"
+// CHECK-RV64-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-SAME: "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-RV64-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
+// CHECK-RV64-SAME:"{{.*}}.o"
 // CHECK-RV64-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-SAME: "-lc"
 // CHECK-RV64-SAME: "-X" "-o" "{{.*}}.tmp.out"
@@ -216,8 +263,11 @@
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-DEFAULTCXX %s
 // CHECK-RV64-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-RV64-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-DEFAULTCXX: ld{{(.exe)?}}"
+// CHECK-RV64-DEFAULTCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-DEFAULTCXX-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV64-DEFAULTCXX-SAME:"{{.*}}.o"
 // CHECK-RV64-DEFAULTCXX-SAME: "-lc++" "-lm"
 // CHECK-RV64-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-DEFAULTCXX-SAME: "-lc"
@@ -230,8 +280,11 @@
 // CHECK-RV64-LIBCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV64-LIBCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}{{[^v].*}}"
 // CHECK-RV64-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-RV64-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-LIBCXX: ld{{(.exe)?}}"
+// CHECK-RV64-LIBCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-LIBCXX-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV64-LIBCXX-SAME:"{{.*}}.o"
 // CHECK-RV64-LIBCXX-SAME: "-lc++" "-lm"
 // CHECK-RV64-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-LIBCXX-SAME: "-lc"
@@ -244,8 +297,11 @@
 // CHECK-RV64-LIBSTDCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV64-LIBSTDCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}8.0.1"
-// CHECK-RV64-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-LIBSTDCXX: ld{{(.exe)?}}"
+// CHECK-RV64-LIBSTDCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-LIBSTDCXX-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV64-LIBSTDCXX-SAME: "{{.*}}.o"
 // CHECK-RV64-LIBSTDCXX-SAME: "-lstdc++" "-lm"
 // CHECK-RV64-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-LIBSTDCXX-SAME: "-lc"
@@ -261,9 +317,12 @@
 // CHECK-RV32-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include"
 // CHECK-RV32-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32-SAME: "--sysroot={{.*}}/Inputs/basic_riscv32_tree/riscv32-unknown-elf"
+// CHECK-RV32-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-SAME: "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-RV32-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
+// CHECK-RV32-SAME: "{{.*}}.o"
 // CHECK-RV32-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV32-SAME: "-lc"
 // CHECK-RV32-SAME: "-X" "-o" "a.out"
@@ -272,8 +331,11 @@
 // RUN:     --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf \
 // RUN:   | FileCheck --check-prefix=CHECK-RV32-DEFAULTCXX %s
 // CHECK-RV32-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-RV32-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-DEFAULTCXX: ld{{(.exe)?}}"
+// CHECK-RV32-DEFAULTCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32-DEFAULTCXX-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV32-DEFAULTCXX-SAME: "{{.*}}.o"
 // CHECK-RV32-DEFAULTCXX-SAME: "-lc++" "-lm"
 // CHECK-RV32-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV32-DEFAULTCXX-SAME: "-lc"
@@ -286,8 +348,11 @@
 // CHECK-RV32-LIBCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV32-LIBCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}{{[^v].*}}"
 // CHECK-RV32-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-RV32-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-LIBCXX: ld{{(.exe)?}}"
+// CHECK-RV32-LIBCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32-LIBCXX-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV32-LIBCXX-SAME: "{{.*}}.o"
 // CHECK-RV32-LIBCXX-SAME: "-lc++" "-lm"
 // CHECK-RV32-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV32-LIBCXX-SAME: "-X" "-o" "a.out"
@@ -299,11 +364,15 @@
 // CHECK-RV32-LIBSTDCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV32-LIBSTDCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}8.0.1"
-// CHECK-RV32-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-LIBSTDCXX: ld{{(.exe)?}}"
+// CHECK-RV32-LIBSTDCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32-LIBSTDCXX-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV32-LIBSTDCXX-SAME: "{{.*}}.o"
 // CHECK-RV32-LIBSTDCXX-SAME: "-lstdc++" "-lm"
 // CHECK-RV32-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
-// CHECK-RV32-LIBSTDCXX-SAME: "-lc" "-X" "-o" "a.out"
+// CHECK-RV32-LIBSTDCXX-SAME: "-lc"
+// CHECK-RV32-LIBSTDCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clang %s -### 2>&1 --target=riscv64-unknown-elf \
 // RUN:     -nostdlibinc -nobuiltininc \
@@ -320,7 +389,9 @@
 // RUN:     -nodefaultlibs \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-NDL %s
 // CHECK-RV64-NDL: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-RV64-NDL: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-NDL: ld{{(.exe)?}}"
+// CHECK-RV64-NDL-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-NDL-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-NDL-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv64-unknown-elf \
@@ -339,7 +410,9 @@
 // CHECK-RV64FD-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv64imafdc{{[/\\]+}}lp64d{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64FD-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv64imafdc{{[/\\]+}}lp64d{{[/\\]+}}include"
 // CHECK-RV64FD-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV64FD-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64FD-NEXT: ld{{(.exe)?}}"
+// CHECK-RV64FD-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64FD-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64FD-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv64imafdc{{[/\\]+}}lp64d{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
@@ -358,7 +431,9 @@
 // CHECK-RV32I-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32i{{[/\\]+}}ilp32{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32I-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32i{{[/\\]+}}ilp32{{[/\\]+}}include"
 // CHECK-RV32I-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32I-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32I-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32I-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32I-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32I-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32i{{[/\\]+}}ilp32{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
@@ -377,7 +452,9 @@
 // CHECK-RV32IM-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32im{{[/\\]+}}ilp32{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32IM-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32im{{[/\\]+}}ilp32{{[/\\]+}}include"
 // CHECK-RV32IM-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32IM-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32IM-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32IM-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32IM-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32IM-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32im{{[/\\]+}}ilp32{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
@@ -391,7 +468,9 @@
 // CHECK-RV32IAC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32iac{{[/\\]+}}ilp32{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32IAC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32iac{{[/\\]+}}ilp32{{[/\\]+}}include"
 // CHECK-RV32IAC-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32IAC-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32IAC-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32IAC-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32IAC-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32IAC-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32iac{{[/\\]+}}ilp32{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf -march=rv32imafc -mabi=ilp32f \
@@ -412,7 +491,9 @@
 // CHECK-RV32IMAFC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32imafc{{[/\\]+}}ilp32f{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32IMAFC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32imafc{{[/\\]+}}ilp32f{{[/\\]+}}include"
 // CHECK-RV32IMAFC-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32IMAFC-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32IMAFC-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32IMAFC-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32IMAFC-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32IMAFC-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32imafc{{[/\\]+}}ilp32f{{[/\\]+}}lib"
 
 // RUN: %clang -no-canonical-prefixes %s -### --target=powerpc-unknown-eabi 2>&1 \
@@ -423,8 +504,9 @@
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPCEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPCEABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPCEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPCEABI-SAME:"{{.*}}.o"
 // CHECK-PPCEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPCEABI-SAME: "-lc"
 // CHECK-PPCEABI-SAME: "-o" "a.out"
@@ -437,8 +519,9 @@
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPC64EABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPC64EABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPC64EABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPC64EABI-SAME:"{{.*}}.o"
 // CHECK-PPC64EABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPC64EABI-SAME: "-lc"
 // CHECK-PPC64EABI-SAME: "-o" "a.out"
@@ -451,8 +534,9 @@
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPCLEEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPCLEEABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPCLEEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPCLEEABI-SAME:"{{.*}}.o"
 // CHECK-PPCLEEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPCLEEABI-SAME: "-lc"
 // CHECK-PPCLEEABI-SAME: "-o" "a.out"
@@ -465,8 +549,9 @@
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPC64LEEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPC64LEEABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPC64LEEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPC64LEEABI-SAME:"{{.*}}.o"
 // CHECK-PPC64LEEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPC64LEEABI-SAME: "-lc"
 // CHECK-PPC64LEEABI-SAME: "-o" "a.out"
diff --git a/clang/test/Driver/check-no-multlib-warning.c b/clang/test/Driver/check-no-multlib-warning.c
new file mode 100644
index 0000000000000..9a0d7cee450a3
--- /dev/null
+++ b/clang/test/Driver/check-no-multlib-warning.c
@@ -0,0 +1,10 @@
+// UNSUPPORTED: system-windows
+
+
+// RUN: %clang --target=armv6m-none-eabi --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -### 2>&1 | FileCheck %s
+// RUN: %clang --target=aarch64-none-elf --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -### 2>&1 | FileCheck %s
+// RUN: %clang --target=riscv32-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv32_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
+// RUN: %clang --target=riscv64-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv64_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
+
+// CHECK: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
+// NOCHECK-NOT: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index eb079895a0a88..57e16e8795a28 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -362,6 +362,12 @@
 // RUN: %clang_cl -c -### /std:c11 -- %s 2>&1 | FileCheck -check-prefix CHECK-C11 %s
 // CHECK-C11: -std=c11
 
+// RUN: %clang_cl -c -### /std:c17 -- %s 2>&1 | FileCheck -check-prefix CHECK-C17 %s
+// CHECK-C17: -std=c17
+
+// RUN: %clang_cl -c -### /std:clatest -- %s 2>&1 | FileCheck -check-prefix CHECK-CLATEST %s
+// CHECK-CLATEST: -std=c23
+
 // For some warning ids, we can map from MSVC warning to Clang warning.
 // RUN: %clang_cl -wd4005 -wd4100 -wd4910 -wd4996 -wd12345678 -### -- %s 2>&1 | FileCheck -check-prefix=Wno %s
 // Wno: "-cc1"
diff --git a/clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c b/clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c
new file mode 100644
index 0000000000000..b186e6e74f162
--- /dev/null
+++ b/clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c
@@ -0,0 +1,6 @@
+// RUN: %clang -### -fallow-runtime-check-skip-hot-cutoff=1.0 %s 2>&1 | FileCheck %s
+// CHECK: -fallow-runtime-check-skip-hot-cutoff=1.0
+
+int main(int argc, char** argv) {
+    return __builtin_allow_runtime_check("foo");
+}
diff --git a/clang/test/Driver/frame-pointer-elim.c b/clang/test/Driver/frame-pointer-elim.c
index f64ff6efc7261..0dd7eb0c738db 100644
--- a/clang/test/Driver/frame-pointer-elim.c
+++ b/clang/test/Driver/frame-pointer-elim.c
@@ -4,6 +4,8 @@
 // KEEP-NON-LEAF: "-mframe-pointer=non-leaf"
 // KEEP-NONE-NOT: warning: argument unused
 // KEEP-NONE:     "-mframe-pointer=none"
+// KEEP-RESERVED-NOT: warning: argument unused
+// KEEP-RESERVED: "-mframe-pointer=reserved"
 
 // On Linux x86, omit frame pointer when optimization is enabled.
 // RUN: %clang -### --target=i386-linux -S -fomit-frame-pointer %s 2>&1 | \
@@ -215,5 +217,9 @@
 // RUN: %clang -### --target=aarch64-none-elf -S -O1 -fno-omit-frame-pointer %s 2>&1 |  \
 // RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
 
+// AArch64 Windows requires that the frame pointer be reserved
+// RUN: %clang -### --target=aarch64-pc-windows-msvc -S -fomit-frame-pointer %s 2>&1 |  \
+// RUN:   FileCheck --check-prefix=KEEP-RESERVED %s
+
 void f0() {}
 void f1() { f0(); }
diff --git a/clang/test/Driver/gcc-toolchain.cpp b/clang/test/Driver/gcc-toolchain.cpp
index 6d4ad417cfec6..a14e8d00af1ef 100644
--- a/clang/test/Driver/gcc-toolchain.cpp
+++ b/clang/test/Driver/gcc-toolchain.cpp
@@ -6,11 +6,6 @@
 // RUN:   --gcc-toolchain=%S/Inputs/ubuntu_14.04_multiarch_tree/usr -stdlib=libstdc++ --rtlib=libgcc --unwindlib=libgcc -no-pie 2>&1 | \
 // RUN:   FileCheck %s
 //
-// Additionally check that the legacy spelling of the flag works.
-// RUN: %clangxx %s -### --target=x86_64-linux-gnu --sysroot= \
-// RUN:   --gcc-toolchain=%S/Inputs/ubuntu_14.04_multiarch_tree/usr -stdlib=libstdc++ --rtlib=libgcc --unwindlib=libgcc -no-pie 2>&1 | \
-// RUN:   FileCheck %s
-//
 // Test for header search toolchain detection.
 // CHECK: "-internal-isystem"
 // CHECK: "[[TOOLCHAIN:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8"
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca3638565..4fb5571b838fb 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/clang/test/Driver/ir-exception-model.c b/clang/test/Driver/ir-exception-model.c
new file mode 100644
index 0000000000000..9e8f998de0d6b
--- /dev/null
+++ b/clang/test/Driver/ir-exception-model.c
@@ -0,0 +1,14 @@
+// RUN: %clang -### -target wasm32-unknown-unknown -fwasm-exceptions -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=wasm -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=dwarf -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=DWARF %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=sjlj -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=SJLJ %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=wineh -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=WINEH %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=arst -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=INVALID %s
+
+// Check that -fwasm-exceptions propagates -exception-model to cc1
+
+// CHECK: "-exception-model=wasm"
+// DWARF: "-exception-model=dwarf"
+// SJLJ: "-exception-model=sjlj"
+// WINEH: "-exception-model=wineh"
+// INVALID: "-exception-model=arst"
diff --git a/clang/test/Driver/openmp-system-arch.c b/clang/test/Driver/openmp-system-arch.c
index 51021e352f4d3..b18ecf3ec474b 100644
--- a/clang/test/Driver/openmp-system-arch.c
+++ b/clang/test/Driver/openmp-system-arch.c
@@ -6,6 +6,7 @@
 // RUN: cp %S/Inputs/amdgpu-arch/amdgpu_arch_gfx906 %t/
 // RUN: cp %S/Inputs/nvptx-arch/nvptx_arch_fail %t/
 // RUN: cp %S/Inputs/nvptx-arch/nvptx_arch_sm_70 %t/
+// RUN: cp %S/Inputs/offload-arch/offload_arch_sm_70_gfx906 %t/
 // RUN: echo '#!/bin/sh' > %t/amdgpu_arch_empty
 // RUN: chmod +x %t/amdgpu_arch_fail
 // RUN: chmod +x %t/amdgpu_arch_gfx906
@@ -14,6 +15,7 @@
 // RUN: chmod +x %t/nvptx_arch_fail
 // RUN: chmod +x %t/nvptx_arch_sm_70
 // RUN: chmod +x %t/nvptx_arch_empty
+// RUN: chmod +x %t/offload_arch_sm_70_gfx906
 
 // case when nvptx-arch and amdgpu-arch return nothing or fails
 // RUN:   not %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp --offload-arch=native \
@@ -41,23 +43,23 @@
 
 // case when nvptx-arch succeeds.
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp --offload-arch=native \
-// RUN:     --nvptx-arch-tool=%t/nvptx_arch_sm_70 --amdgpu-arch-tool=%t/amdgpu_arch_fail %s 2>&1 \
+// RUN:     --amdgpu-arch-tool=%t/amdgpu_arch_fail --nvptx-arch-tool=%t/nvptx_arch_sm_70 %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=ARCH-SM_70
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
-// RUN:     --nvptx-arch-tool=%t/nvptx_arch_sm_70 --amdgpu-arch-tool=%t/amdgpu_arch_fail %s 2>&1 \
+// RUN:     --amdgpu-arch-tool=%t/amdgpu_arch_fail --nvptx-arch-tool=%t/nvptx_arch_sm_70 %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=ARCH-SM_70
 // ARCH-SM_70: "-cc1" "-triple" "nvptx64-nvidia-cuda"{{.*}}"-target-cpu" "sm_70"
 
 // case when both nvptx-arch and amdgpu-arch succeed.
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp --offload-arch=native \
-// RUN:     --nvptx-arch-tool=%t/nvptx_arch_sm_70 --amdgpu-arch-tool=%t/amdgpu_arch_gfx906 %s 2>&1 \
+// RUN:     --offload-arch-tool=%t/offload_arch_sm_70_gfx906 %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=ARCH-SM_70-GFX906
 // ARCH-SM_70-GFX906: "-cc1" "-triple" "amdgcn-amd-amdhsa"{{.*}}"-target-cpu" "gfx906"
 // ARCH-SM_70-GFX906: "-cc1" "-triple" "nvptx64-nvidia-cuda"{{.*}}"-target-cpu" "sm_70"
 
 // case when both nvptx-arch and amdgpu-arch succeed with other archs.
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -nogpulib -fopenmp=libomp --offload-arch=native,sm_75,gfx1030 \
-// RUN:     --nvptx-arch-tool=%t/nvptx_arch_sm_70 --amdgpu-arch-tool=%t/amdgpu_arch_gfx906 %s 2>&1 \
+// RUN:     --offload-arch-tool=%t/offload_arch_sm_70_gfx906 %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=ARCH-MULTIPLE
 // ARCH-MULTIPLE: "-cc1" "-triple" "amdgcn-amd-amdhsa"{{.*}}"-target-cpu" "gfx1030"
 // ARCH-MULTIPLE: "-cc1" "-triple" "amdgcn-amd-amdhsa"{{.*}}"-target-cpu" "gfx906"
diff --git a/clang/test/Driver/pgo-sample-use-profi.c b/clang/test/Driver/pgo-sample-use-profi.c
index 0370d947ce590..aef00724f8687 100644
--- a/clang/test/Driver/pgo-sample-use-profi.c
+++ b/clang/test/Driver/pgo-sample-use-profi.c
@@ -1,4 +1,23 @@
-/// Test if profi flat is enabled in frontend as user-facing feature.
+/// Test if profi flag is enabled/disabled correctly based on user-specified configuration.
+/// Ensure that profi flag is disabled by default
+
+// RUN: %clang --target=x86_64 -c -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
 // RUN: %clang --target=x86_64 -c -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+// RUN: %clang --target=x86_64 -c -fno-sample-profile-use-profi -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+
+// RUN: %clang --target=AArch64 -c -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=AArch64 -c -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+// RUN: %clang --target=AArch64 -c -fno-sample-profile-use-profi -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+
+// Cases where profi flag is explicitly disabled:
+// RUN: %clang --target=x86_64 -c -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=x86_64 -c -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=x86_64 -c -fsample-profile-use-profi -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+
+// RUN: %clang --target=AArch64 -c -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=AArch64 -c -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=AArch64 -c -fsample-profile-use-profi -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+
 
 // CHECK: "-mllvm" "-sample-profile-use-profi"
+// CHECK-NO-PROFI-NOT: "-sample-profile-use-profi"
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c b/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
index 8576edff9bcbe..e0802b3866cc8 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
@@ -71,7 +71,7 @@
 // CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
 // CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
 // CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
-// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SVE SM4 instructions
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
 // CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-gb10.c b/clang/test/Driver/print-enabled-extensions/aarch64-gb10.c
new file mode 100644
index 0000000000000..f482a1bd1a31e
--- /dev/null
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-gb10.c
@@ -0,0 +1,69 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=gb10 | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s
+
+// CHECK: Extensions enabled for the given AArch64 target
+// CHECK-EMPTY:
+// CHECK-NEXT:     Architecture Feature(s)                                Description
+// CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
+// CHECK-NEXT:     FEAT_AMUv1p1                                           Enable Armv8.6-A Activity Monitors Virtualization support
+// CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
+// CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
+// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
+// CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
+// CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
+// CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
+// CHECK-NEXT:     FEAT_DPB2                                              Enable Armv8.5-A Cache Clean to Point of Deep Persistence
+// CHECK-NEXT:     FEAT_DotProd                                           Enable dot product support
+// CHECK-NEXT:     FEAT_ECV                                               Enable enhanced counter virtualization extension
+// CHECK-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// CHECK-NEXT:     FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
+// CHECK-NEXT:     FEAT_FGT                                               Enable fine grained virtualization traps extension
+// CHECK-NEXT:     FEAT_FHM                                               Enable FP16 FML instructions
+// CHECK-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// CHECK-NEXT:     FEAT_FPAC                                              Enable Armv8.3-A Pointer Authentication Faulting enhancement
+// CHECK-NEXT:     FEAT_FRINTTS                                           Enable FRInt[32|64][Z|X] instructions that round a floating-point number to an integer (in FP format) forcing it to fit into a 32- or 64-bit int
+// CHECK-NEXT:     FEAT_FlagM                                             Enable Armv8.4-A Flag Manipulation instructions
+// CHECK-NEXT:     FEAT_FlagM2                                            Enable alternative NZCV format for floating point comparisons
+// CHECK-NEXT:     FEAT_HCX                                               Enable Armv8.7-A HCRX_EL2 system register
+// CHECK-NEXT:     FEAT_I8MM                                              Enable Matrix Multiply Int8 Extension
+// CHECK-NEXT:     FEAT_JSCVT                                             Enable Armv8.3-A JavaScript FP conversion instructions
+// CHECK-NEXT:     FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
+// CHECK-NEXT:     FEAT_LRCPC                                             Enable support for RCPC extension
+// CHECK-NEXT:     FEAT_LRCPC2                                            Enable Armv8.4-A RCPC instructions with Immediate Offsets
+// CHECK-NEXT:     FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
+// CHECK-NEXT:     FEAT_LSE2                                              Enable Armv8.4-A Large System Extension 2 (LSE2) atomicity rules
+// CHECK-NEXT:     FEAT_MPAM                                              Enable Armv8.4-A Memory system Partitioning and Monitoring extension
+// CHECK-NEXT:     FEAT_MTE, FEAT_MTE2                                    Enable Memory Tagging Extension
+// CHECK-NEXT:     FEAT_NV, FEAT_NV2                                      Enable Armv8.4-A Nested Virtualization Enchancement
+// CHECK-NEXT:     FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
+// CHECK-NEXT:     FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
+// CHECK-NEXT:     FEAT_PAuth                                             Enable Armv8.3-A Pointer Authentication extension
+// CHECK-NEXT:     FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-NEXT:     FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
+// CHECK-NEXT:     FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
+// CHECK-NEXT:     FEAT_SB                                                Enable Armv8.5-A Speculation Barrier
+// CHECK-NEXT:     FEAT_SEL2                                              Enable Armv8.4-A Secure Exception Level 2 extension
+// CHECK-NEXT:     FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+// CHECK-NEXT:     FEAT_SHA3, FEAT_SHA512                                 Enable SHA512 and SHA3 support
+// CHECK-NEXT:     FEAT_SM4, FEAT_SM3                                     Enable SM3 and SM4 support
+// CHECK-NEXT:     FEAT_SPE                                               Enable Statistical Profiling extension
+// CHECK-NEXT:     FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
+// CHECK-NEXT:     FEAT_SPEv1p2                                           Enable extra register in the Statistical Profiling Extension
+// CHECK-NEXT:     FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
+// CHECK-NEXT:     FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// CHECK-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
+// CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
+// CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
+// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SVE SM4 instructions
+// CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
+// CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+// CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
+// CHECK-NEXT:     FEAT_UAO                                               Enable Armv8.2-A UAO PState
+// CHECK-NEXT:     FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
+// CHECK-NEXT:     FEAT_WFxT                                              Enable Armv8.7-A WFET and WFIT instruction
+// CHECK-NEXT:     FEAT_XS                                                Enable Armv8.7-A limited-TLB-maintenance instruction
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-grace.c b/clang/test/Driver/print-enabled-extensions/aarch64-grace.c
index bb24dfbbc0702..78d991c3ab184 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-grace.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-grace.c
@@ -55,7 +55,7 @@
 // CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
 // CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
 // CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
-// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SVE SM4 instructions
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
 // CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c b/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c
index 0cfb9e1fb9817..c50c0b130307b 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c
@@ -71,7 +71,7 @@
 // CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
 // CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
 // CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
-// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SVE SM4 instructions
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
 // CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c
index cfb4d0ed58d11..5d616c48247de 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv32 -mcpu=andes-a25 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv32 -mcpu=andes-a25 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c
index 3c3c554dffc57..b5885f8e5200d 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv32 -mcpu=andes-a45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv32 -mcpu=andes-a45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c
index 70100a0a8df13..a8b909cf9fdd8 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-ax25 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-ax25 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c
index d2b1a32e321e5..45571a1713635 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-ax45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-ax45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c
index bab35b117f573..1917d5332feef 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-ax45mpv --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-ax45mpv --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c
index 1a2c30bfc7a2e..18cd4b7bbcdce 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv32 -mcpu=andes-n45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv32 -mcpu=andes-n45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c
index 50c38da3bd034..ca6a13b3d9830 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-nx45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-nx45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c b/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c
index f76e5daec672b..e50768a62e6eb 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=sifive-p870 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=sifive-p870 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c b/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
index 3dc4d6a6b4fa3..aa038a18693e0 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=sifive-x390 -menable-experimental-extensions --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=sifive-x390 -menable-experimental-extensions --print-enabled-extensions | FileCheck %s
 
 // CHECK:     Name                 Version   Description
 // CHECK-NEXT:     i                    2.1       'I' (Base Integer Instruction Set)
diff --git a/clang/test/Driver/print-libgcc-file-name-clangrt.c b/clang/test/Driver/print-libgcc-file-name-clangrt.c
index a902eedc85209..1cb28d43748ec 100644
--- a/clang/test/Driver/print-libgcc-file-name-clangrt.c
+++ b/clang/test/Driver/print-libgcc-file-name-clangrt.c
@@ -63,3 +63,12 @@
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-ARM-BAREMETAL-PER-TARGET %s
 // CHECK-CLANGRT-ARM-BAREMETAL-PER-TARGET: libclang_rt.builtins.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=riscv32-unknown-elf \
+// RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_riscv32_tree 2>&1 \
+// RUN:    | FileCheck --check-prefix=CHECK-CLANGRT-RISCV-BAREMETAL %s
+// CHECK-CLANGRT-RISCV-BAREMETAL-NOT: baremetal{{/|\\}}libclang_rt.builtins-riscv32.a
+// CHECK-CLANGRT-RISCV-BAREMETAL: libclang_rt.builtins.a
diff --git a/clang/test/Driver/print-supported-extensions-aarch64.c b/clang/test/Driver/print-supported-extensions-aarch64.c
index af4db9944c2df..5c840a60685c6 100644
--- a/clang/test/Driver/print-supported-extensions-aarch64.c
+++ b/clang/test/Driver/print-supported-extensions-aarch64.c
@@ -93,11 +93,12 @@
 // CHECK-NEXT:     sve-bitperm         FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
 // CHECK-NEXT:     sve-f16f32mm        FEAT_SVE_F16F32MM                                      Enable Armv9.6-A FP16 to FP32 Matrix Multiply
 // CHECK-NEXT:     sve-sha3            FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
+// CHECK-NEXT:     sve-sm4             FEAT_SVE_SM4                                           Enable SVE SM4 instructions
 // CHECK-NEXT:     sve2                FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
 // CHECK-NEXT:     sve2-aes                                                                   Shorthand for +sve2+sve-aes
 // CHECK-NEXT:     sve2-bitperm                                                               Shorthand for +sve2+sve-bitperm
 // CHECK-NEXT:     sve2-sha3                                                                  Shorthand for +sve2+sve-sha3
-// CHECK-NEXT:     sve2-sm4            FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
+// CHECK-NEXT:     sve2-sm4                                                                   Shorthand for +sve2+sve-sm4
 // CHECK-NEXT:     sve2p1              FEAT_SVE2p1                                            Enable Scalable Vector Extension 2.1 instructions
 // CHECK-NEXT:     sve2p2              FEAT_SVE2p2                                            Enable Armv9.6-A Scalable Vector Extension 2.2 instructions
 // CHECK-NEXT:     the                 FEAT_THE                                               Enable Armv8.9-A Translation Hardening Extension
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 314fdad7e52e7..b87f0cb568e00 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -162,6 +162,7 @@
 // CHECK-NEXT:     xandesvbfhcvt        5.0       'XAndesVBFHCvt' (Andes Vector BFLOAT16 Conversion Extension)
 // CHECK-NEXT:     xandesvdot           5.0       'XAndesVDot' (Andes Vector Dot Product Extension)
 // CHECK-NEXT:     xandesvpackfph       5.0       'XAndesVPackFPH' (Andes Vector Packed FP16 Extension)
+// CHECK-NEXT:     xandesvsintload      5.0       'XAndesVSIntLoad' (Andes Vector INT4 Load Extension)
 // CHECK-NEXT:     xcvalu               1.0       'XCValu' (CORE-V ALU Operations)
 // CHECK-NEXT:     xcvbi                1.0       'XCVbi' (CORE-V Immediate Branching)
 // CHECK-NEXT:     xcvbitmanip          1.0       'XCVbitmanip' (CORE-V Bit Manipulation)
@@ -169,6 +170,7 @@
 // CHECK-NEXT:     xcvmac               1.0       'XCVmac' (CORE-V Multiply-Accumulate)
 // CHECK-NEXT:     xcvmem               1.0       'XCVmem' (CORE-V Post-incrementing Load & Store)
 // CHECK-NEXT:     xcvsimd              1.0       'XCVsimd' (CORE-V SIMD ALU)
+// CHECK-NEXT:     xmipscbop            1.0       'XMIPSCBOP' (MIPS Software Prefetch)
 // CHECK-NEXT:     xmipscmov            1.0       'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov))
 // CHECK-NEXT:     xmipslsp             1.0       'XMIPSLSP' (MIPS optimization for hardware load-store bonding)
 // CHECK-NEXT:     xsfcease             1.0       'XSfcease' (SiFive sf.cease Instruction)
diff --git a/clang/test/Driver/ps5-linker.c b/clang/test/Driver/ps5-linker.c
index 753085d2e1b0b..16c9967bc2b41 100644
--- a/clang/test/Driver/ps5-linker.c
+++ b/clang/test/Driver/ps5-linker.c
@@ -165,7 +165,7 @@
 // CHECK-SHARED-CRT-SAME: "-l:crtendS.o" "-l:crtn.o"
 // CHECK-STATIC-CRT-SAME: "-l:crtend.o" "-l:crtn.o"
 
-// CHECK-NO-CRT-NOT: crt{{[^"]*}}.o"
+// CHECK-NO-CRT-NOT: "-l:crt
 // CHECK-NO-LIBS-NOT: "-l{{[^"]*}}"
 
 // Test the driver's control over the -fcrash-diagnostics-dir behavior with linker flags.
diff --git a/clang/test/Driver/riscv-args.c b/clang/test/Driver/riscv-args.c
deleted file mode 100644
index cab08e5b0f811..0000000000000
--- a/clang/test/Driver/riscv-args.c
+++ /dev/null
@@ -1,6 +0,0 @@
-// Check the arguments are correctly passed
-
-// Make sure -T is the last with gcc-toolchain option
-// RUN: %clang -### --target=riscv32 --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHECK-LD %s
-// CHECK-LD: {{.*}} "--defsym=FOO=10" {{.*}} "-u" "foo" {{.*}} "-T" "a.lds"
diff --git a/clang/test/Driver/riscv-cpus.c b/clang/test/Driver/riscv-cpus.c
index 3736e76ed06dd..d32d1c1a8183f 100644
--- a/clang/test/Driver/riscv-cpus.c
+++ b/clang/test/Driver/riscv-cpus.c
@@ -186,6 +186,9 @@
 // MCPU-MIPS-P8700-SAME: "-target-feature" "+zalrsc"
 // MCPU-MIPS-P8700-SAME: "-target-feature" "+zba"
 // MCPU-MIPS-P8700-SAME: "-target-feature" "+zbb"
+// MCPU-MIPS-P8700-SAME: "-target-feature" "+xmipscbop"
+// MCPU-MIPS-P8700-SAME: "-target-feature" "+xmipscmov"
+// MCPU-MIPS-P8700-SAME: "-target-feature" "+xmipslsp"
 
 // RUN: %clang --target=riscv32 -### -c %s 2>&1 -mtune=syntacore-scr1-base | FileCheck -check-prefix=MTUNE-SYNTACORE-SCR1-BASE %s
 // MTUNE-SYNTACORE-SCR1-BASE: "-tune-cpu" "syntacore-scr1-base"
diff --git a/clang/test/Driver/riscv32-toolchain-extra.c b/clang/test/Driver/riscv32-toolchain-extra.c
index cbb3c23ebb342..420f7b5203609 100644
--- a/clang/test/Driver/riscv32-toolchain-extra.c
+++ b/clang/test/Driver/riscv32-toolchain-extra.c
@@ -18,12 +18,12 @@
 // RUN: ln -s %S/Inputs/basic_riscv32_nogcc_tree/riscv32-unknown-elf %t/riscv32-nogcc/riscv32-unknown-elf
 // RUN: %t/riscv32-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --gcc-toolchain=%t/riscv32-nogcc/invalid \
-// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV32-BAREMETAL-ILP32-NOGCC %s
 
 // RUN: %t/riscv32-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --sysroot=%t/riscv32-nogcc/bin/../riscv32-unknown-elf \
-// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV32-BAREMETAL-ILP32-NOGCC %s
 
 // C-RV32-BAREMETAL-ILP32-NOGCC: "-internal-isystem" "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/include"
@@ -31,6 +31,5 @@
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/lib/crt0.o"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/clang_rt.crtbegin.o"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/lib"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "--start-group" "-lc" "-lgloss" "--end-group"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/libclang_rt.builtins.a"
+// C-RV32-BAREMETAL-ILP32-NOGCC: "--start-group" "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/clang_rt.crtend.o"
diff --git a/clang/test/Driver/riscv32-toolchain.c b/clang/test/Driver/riscv32-toolchain.c
index 50db13f8294c1..8cf20aa592a3a 100644
--- a/clang/test/Driver/riscv32-toolchain.c
+++ b/clang/test/Driver/riscv32-toolchain.c
@@ -28,7 +28,7 @@
 // C-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // C-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // C-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/riscv32-unknown-elf/lib"
-// C-RV32-BAREMETAL-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32-BAREMETAL-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: %clang -### %s -fuse-ld= \
@@ -42,7 +42,7 @@
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/../../../../riscv32-unknown-elf/lib"
-// C-RV32-BAREMETAL-NOSYSROOT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32-BAREMETAL-NOSYSROOT-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: %clangxx -### %s -fuse-ld= \
@@ -58,7 +58,7 @@
 // CXX-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // CXX-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/riscv32-unknown-elf/lib"
-// CXX-RV32-BAREMETAL-ILP32: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV32-BAREMETAL-ILP32: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: %clangxx -### %s -fuse-ld= \
@@ -73,7 +73,7 @@
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/../../../../riscv32-unknown-elf/lib"
-// CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld -no-pie \
@@ -117,7 +117,7 @@
 // C-RV32I-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtbegin.o"
 // C-RV32I-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32I-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32I-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32I-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32I-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -132,7 +132,7 @@
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32im/ilp32/crtbegin.o"
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IM-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IM-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32im/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -147,7 +147,7 @@
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32iac/ilp32/crtbegin.o"
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32iac/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -162,7 +162,7 @@
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtbegin.o"
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IMAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IMAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -177,7 +177,7 @@
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imafc/ilp32f/crtbegin.o"
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imafc/ilp32f/crtend.o"
 
 // Check that --rtlib can be used to override the used runtime library
@@ -187,7 +187,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV32-RTLIB-LIBGCC-ILP32 %s
 // C-RV32-RTLIB-LIBGCC-ILP32: "{{.*}}crt0.o"
 // C-RV32-RTLIB-LIBGCC-ILP32: "{{.*}}crtbegin.o"
-// C-RV32-RTLIB-LIBGCC-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32-RTLIB-LIBGCC-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32-RTLIB-LIBGCC-ILP32: "{{.*}}crtend.o"
 
 // RUN: %clang -### %s \
@@ -196,7 +196,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV32-RTLIB-COMPILERRT-ILP32 %s
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}crt0.o"
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtbegin.o"
-// C-RV32-RTLIB-COMPILERRT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins.a"
+// C-RV32-RTLIB-COMPILERRT-ILP32: "--start-group" "{{.*}}libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtend.o"
 
 // RUN: %clang -### %s --target=riscv32 \
@@ -215,7 +215,7 @@
 
 // RUN: %clang --target=riscv32 %s -emit-llvm -S -o - | FileCheck %s
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is forwarded to the linker for RISC-V.
 // RUN: env "PATH=" %clang %s -### 2>&1 -mno-relax \
 // RUN:   --target=riscv32-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv32imac -mabi=lp32\
@@ -223,7 +223,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV32-NORELAX %s
 // CHECK-RV32-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is not forwarded to the linker for RISC-V.
 // RUN:env "PATH=" %clang %s -### 2>&1 \
 // RUN:   --target=riscv32-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv32imac -mabi=lp32\
diff --git a/clang/test/Driver/riscv64-toolchain-extra.c b/clang/test/Driver/riscv64-toolchain-extra.c
index 99460a3030d3a..87bcdeb651a1e 100644
--- a/clang/test/Driver/riscv64-toolchain-extra.c
+++ b/clang/test/Driver/riscv64-toolchain-extra.c
@@ -18,12 +18,12 @@
 // RUN: ln -s %S/Inputs/basic_riscv64_nogcc_tree/riscv64-unknown-elf %t/riscv64-nogcc/riscv64-unknown-elf
 // RUN: %t/riscv64-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --gcc-toolchain=%t/riscv64-nogcc/invalid \
-// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV64-BAREMETAL-LP64-NOGCC %s
 
 // RUN: %t/riscv64-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --sysroot=%t/riscv64-nogcc/bin/../riscv64-unknown-elf \
-// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV64-BAREMETAL-LP64-NOGCC %s
 
 // C-RV64-BAREMETAL-LP64-NOGCC: "-internal-isystem" "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/include"
@@ -31,6 +31,5 @@
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/lib/crt0.o"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/clang_rt.crtbegin.o"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/lib"
-// C-RV64-BAREMETAL-LP64-NOGCC: "--start-group" "-lc" "-lgloss" "--end-group"
-// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/libclang_rt.builtins.a"
+// C-RV64-BAREMETAL-LP64-NOGCC: "--start-group" "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/clang_rt.crtend.o"
diff --git a/clang/test/Driver/riscv64-toolchain.c b/clang/test/Driver/riscv64-toolchain.c
index 06d5f50f3f408..1550f46af8c9c 100644
--- a/clang/test/Driver/riscv64-toolchain.c
+++ b/clang/test/Driver/riscv64-toolchain.c
@@ -28,7 +28,7 @@
 // C-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // C-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // C-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/riscv64-unknown-elf/lib"
-// C-RV64-BAREMETAL-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64-BAREMETAL-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld= \
@@ -42,7 +42,7 @@
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/../../../../riscv64-unknown-elf/lib"
-// C-RV64-BAREMETAL-NOSYSROOT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64-BAREMETAL-NOSYSROOT-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clangxx -### %s -fuse-ld= \
@@ -58,7 +58,7 @@
 // CXX-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // CXX-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/riscv64-unknown-elf/lib"
-// CXX-RV64-BAREMETAL-LP64: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV64-BAREMETAL-LP64: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clangxx -### %s -fuse-ld= \
@@ -73,7 +73,7 @@
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/../../../../riscv64-unknown-elf/lib"
-// CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld= -no-pie \
@@ -118,7 +118,7 @@
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imac/lp64/crtbegin.o"
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV64IMAC-BAREMETAL-MULTI-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64IMAC-BAREMETAL-MULTI-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imac/lp64/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -133,7 +133,7 @@
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imafdc/lp64d/crtbegin.o"
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imafdc/lp64d/crtend.o"
 
 // Check that --rtlib can be used to override the used runtime library
@@ -143,7 +143,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV64-RTLIB-LIBGCC-LP64 %s
 // C-RV64-RTLIB-LIBGCC-LP64: "{{.*}}crt0.o"
 // C-RV64-RTLIB-LIBGCC-LP64: "{{.*}}crtbegin.o"
-// C-RV64-RTLIB-LIBGCC-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64-RTLIB-LIBGCC-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64-RTLIB-LIBGCC-LP64: "{{.*}}crtend.o"
 
 // RUN: %clang -### %s \
@@ -152,7 +152,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV64-RTLIB-COMPILERRT-LP64 %s
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}crt0.o"
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtbegin.o"
-// C-RV64-RTLIB-COMPILERRT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins.a"
+// C-RV64-RTLIB-COMPILERRT-LP64: "--start-group" "{{.*}}libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtend.o"
 
 // RUN: %clang -### %s --target=riscv64 \
@@ -171,7 +171,7 @@
 
 // RUN: %clang --target=riscv64 %s -emit-llvm -S -o - | FileCheck %s
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is forwarded to the linker for RISC-V.
 // RUN: env "PATH=" %clang %s -### 2>&1 -mno-relax \
 // RUN:   --target=riscv64-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv64imac -mabi=lp64\
@@ -179,7 +179,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-NORELAX %s
 // CHECK-RV64-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is not forwarded to the linker for RISC-V.
 // RUN:env "PATH=" %clang %s -### 2>&1 \
 // RUN:   --target=riscv64-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv64imac -mabi=lp64\
diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index befd322d027c9..d2e4877e89d78 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -1033,7 +1033,7 @@
 // RUN:     --target=riscv32-unknown-elf -fuse-ld=ld \
 // RUN:   | %{filecheck} --check-prefix=CHECK-SHADOWCALLSTACK-ELF-RISCV32
 // CHECK-SHADOWCALLSTACK-ELF-RISCV32-NOT: error:
-// CHECK-SHADOWCALLSTACK-ELF-RISCV32: "{{(.*[^-.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
+// CHECK-SHADOWCALLSTACK-ELF-RISCV32: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
 
 // RUN: %clang -fsanitize=shadow-call-stack -### %s 2>&1 \
 // RUN:     --target=riscv64-unknown-linux -fuse-ld=ld \
diff --git a/clang/test/ExtractAPI/constructor_destructor.cpp b/clang/test/ExtractAPI/constructor_destructor.cpp
index 27112c95ac45c..2f2150a6d0da0 100644
--- a/clang/test/ExtractAPI/constructor_destructor.cpp
+++ b/clang/test/ExtractAPI/constructor_destructor.cpp
@@ -213,7 +213,7 @@ class Foo {
         "subHeading": [
           {
             "kind": "identifier",
-            "spelling": "Foo"
+            "spelling": "~Foo"
           }
         ],
         "title": "~Foo"
diff --git a/clang/test/ExtractAPI/global_record.c b/clang/test/ExtractAPI/global_record.c
index a08d51d21f955..287fa24c4c64e 100644
--- a/clang/test/ExtractAPI/global_record.c
+++ b/clang/test/ExtractAPI/global_record.c
@@ -185,7 +185,7 @@ char unavailable __attribute__((unavailable));
         },
         {
           "kind": "text",
-          "spelling": " * "
+          "spelling": " *"
         },
         {
           "kind": "internalParam",
@@ -341,7 +341,7 @@ char unavailable __attribute__((unavailable));
               },
               {
                 "kind": "text",
-                "spelling": " * "
+                "spelling": " *"
               },
               {
                 "kind": "internalParam",
diff --git a/clang/test/ExtractAPI/global_record_multifile.c b/clang/test/ExtractAPI/global_record_multifile.c
index ffdfbcb7eb808..b98cd27b1601e 100644
--- a/clang/test/ExtractAPI/global_record_multifile.c
+++ b/clang/test/ExtractAPI/global_record_multifile.c
@@ -187,7 +187,7 @@ char unavailable __attribute__((unavailable));
         },
         {
           "kind": "text",
-          "spelling": " * "
+          "spelling": " *"
         },
         {
           "kind": "internalParam",
@@ -343,7 +343,7 @@ char unavailable __attribute__((unavailable));
               },
               {
                 "kind": "text",
-                "spelling": " * "
+                "spelling": " *"
               },
               {
                 "kind": "internalParam",
diff --git a/clang/test/ExtractAPI/macro_undefined.c b/clang/test/ExtractAPI/macro_undefined.c
index ec60f95d3d6c4..7bb50af380c24 100644
--- a/clang/test/ExtractAPI/macro_undefined.c
+++ b/clang/test/ExtractAPI/macro_undefined.c
@@ -148,7 +148,7 @@ FUNC_GEN(bar, const int *, unsigned);
         },
         {
           "kind": "text",
-          "spelling": " * "
+          "spelling": " *"
         },
         {
           "kind": "internalParam",
@@ -195,7 +195,7 @@ FUNC_GEN(bar, const int *, unsigned);
               },
               {
                 "kind": "text",
-                "spelling": " * "
+                "spelling": " *"
               },
               {
                 "kind": "internalParam",
diff --git a/clang/test/ExtractAPI/pointers.c b/clang/test/ExtractAPI/pointers.c
new file mode 100644
index 0000000000000..d7baf541ec03e
--- /dev/null
+++ b/clang/test/ExtractAPI/pointers.c
@@ -0,0 +1,388 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \
+// RUN: %t/reference.output.json.in >> %t/reference.output.json
+// RUN: %clang -extract-api --pretty-sgf --product-name=Pointers -target arm64-apple-macosx \
+// RUN: %t/input.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+
+//--- input.h
+void foo(int *a);
+void bar(int **a);
+void *baz();
+void **qux();
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "Pointers",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationships": [],
+  "symbols": [
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "foo"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " *"
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "a"
+        },
+        {
+          "kind": "text",
+          "spelling": ");"
+        }
+      ],
+      "functionSignature": {
+        "parameters": [
+          {
+            "declarationFragments": [
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " *"
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "a"
+              }
+            ],
+            "name": "a"
+          }
+        ],
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@foo"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 5,
+          "line": 0
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "foo"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "foo"
+          }
+        ],
+        "title": "foo"
+      },
+      "pathComponents": [
+        "foo"
+      ]
+    },
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "bar"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " **"
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "a"
+        },
+        {
+          "kind": "text",
+          "spelling": ");"
+        }
+      ],
+      "functionSignature": {
+        "parameters": [
+          {
+            "declarationFragments": [
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " **"
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "a"
+              }
+            ],
+            "name": "a"
+          }
+        ],
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@bar"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 5,
+          "line": 1
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "bar"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "bar"
+          }
+        ],
+        "title": "bar"
+      },
+      "pathComponents": [
+        "bar"
+      ]
+    },
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " *"
+        },
+        {
+          "kind": "identifier",
+          "spelling": "baz"
+        },
+        {
+          "kind": "text",
+          "spelling": "();"
+        }
+      ],
+      "functionSignature": {
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          },
+          {
+            "kind": "text",
+            "spelling": " *"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@baz"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 6,
+          "line": 2
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "baz"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "baz"
+          }
+        ],
+        "title": "baz"
+      },
+      "pathComponents": [
+        "baz"
+      ]
+    },
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " **"
+        },
+        {
+          "kind": "identifier",
+          "spelling": "qux"
+        },
+        {
+          "kind": "text",
+          "spelling": "();"
+        }
+      ],
+      "functionSignature": {
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          },
+          {
+            "kind": "text",
+            "spelling": " **"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@qux"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 7,
+          "line": 3
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "qux"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "qux"
+          }
+        ],
+        "title": "qux"
+      },
+      "pathComponents": [
+        "qux"
+      ]
+    }
+  ]
+}
diff --git a/clang/test/Frontend/absolute-paths.c b/clang/test/Frontend/absolute-paths.c
index e06cf262dd8e2..e507f0d4c34b3 100644
--- a/clang/test/Frontend/absolute-paths.c
+++ b/clang/test/Frontend/absolute-paths.c
@@ -8,9 +8,9 @@
 
 #include "absolute-paths.h"
 
-// Check whether the diagnostic from the header above includes the dummy
-// directory in the path.
-// NORMAL: SystemHeaderPrefix
+// Check that the bogus prefix we added is stripped out even if absolute paths
+// are disabled.
+// NORMAL-NOT: SystemHeaderPrefix
 // ABSOLUTE-NOT: SystemHeaderPrefix
 // CHECK: warning: non-void function does not return a value
 
diff --git a/clang/test/Frontend/noderef.c b/clang/test/Frontend/noderef.c
index a376f2d68d91b..3f0d8269ca88d 100644
--- a/clang/test/Frontend/noderef.c
+++ b/clang/test/Frontend/noderef.c
@@ -227,18 +227,18 @@ int test(void) {
 // ignored.
 // For details see https://github.com/llvm/llvm-project/issues/55790
 void test_standard_syntax() {
-  [[clang::noderef]] int i; // expected-warning {{'noderef' attribute ignored}}
+  [[clang::noderef]] int i; // expected-warning {{'clang::noderef' attribute ignored}}
 
-  [[clang::noderef]] int *p1; // expected-warning {{'noderef' attribute ignored}}
+  [[clang::noderef]] int *p1; // expected-warning {{'clang::noderef' attribute ignored}}
   *p1;
 
-  int *p2 [[clang::noderef]]; // expected-warning {{'noderef' attribute ignored}}
+  int *p2 [[clang::noderef]]; // expected-warning {{'clang::noderef' attribute ignored}}
   *p2;
 
-  int * [[clang::noderef]] p3; // expected-warning {{'noderef' attribute ignored}}
+  int * [[clang::noderef]] p3; // expected-warning {{'clang::noderef' attribute ignored}}
   *p3;
 
   typedef int* IntPtr;
-  [[clang::noderef]] IntPtr p4; // expected-warning {{'noderef' attribute ignored}}
+  [[clang::noderef]] IntPtr p4; // expected-warning {{'clang::noderef' attribute ignored}}
   *p4;
 }
diff --git a/clang/test/Frontend/simplify-paths.c b/clang/test/Frontend/simplify-paths.c
new file mode 100644
index 0000000000000..90af7c4ee3836
--- /dev/null
+++ b/clang/test/Frontend/simplify-paths.c
@@ -0,0 +1,18 @@
+// REQUIRES: shell
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/a/b/
+// RUN: echo 'void x;' > %t/test.h
+// RUN: echo 'const void x;' > %t/header_with_a_really_long_name.h
+// RUN: ln -s %t/header_with_a_really_long_name.h %t/a/shorter_name.h
+//
+// RUN: %clang_cc1 -fsyntax-only -I %t %s 2> %t/output.txt || true
+// RUN: cat %t/output.txt | FileCheck %s
+
+// Check that we strip '..' by canonicalising the path...
+#include "a/b/../../test.h"
+// CHECK: simplify-paths.c.tmp/test.h:1:6: error: variable has incomplete type 'void'
+
+// ... but only if the resulting path is actually shorter.
+#include "a/b/../shorter_name.h"
+// CHECK: simplify-paths.c.tmp/a/b/../shorter_name.h:1:12: error: variable has incomplete type 'const void'
diff --git a/clang/test/Interpreter/out-of-process.cpp b/clang/test/Interpreter/out-of-process.cpp
new file mode 100644
index 0000000000000..6922ca6e82053
--- /dev/null
+++ b/clang/test/Interpreter/out-of-process.cpp
@@ -0,0 +1,88 @@
+// REQUIRES: host-supports-jit,  host-supports-out-of-process-jit, x86_64-linux
+
+// RUN: cat %s | clang-repl -oop-executor -orc-runtime | FileCheck %s
+
+extern "C" int printf(const char *, ...);
+
+int intVar = 0;
+double doubleVar = 3.14;
+%undo
+double doubleVar = 2.71;
+
+auto r1 = printf("intVar = %d\n", intVar);
+// CHECK: intVar = 0
+auto r2 = printf("doubleVar = %.2f\n", doubleVar);
+// CHECK: doubleVar = 2.71
+
+// Test redefinition with inline and static functions.
+int add(int a, int b, int c) { return a + b + c; }
+%undo  // Revert to the initial version of add
+inline int add(int a, int b) { return a + b; }
+
+auto r3 = printf("add(1, 2) = %d\n", add(1, 2));
+// CHECK-NEXT: add(1, 2) = 3
+
+// Test inline and lambda functions with variations.
+inline int square(int x) { return x * x; }
+auto lambdaSquare = [](int x) { return x * x; };
+auto lambdaMult = [](int a, int b) { return a * b; };
+
+auto r4 = printf("square(4) = %d\n", square(4));
+// CHECK-NEXT: square(4) = 16
+auto lambda_r1 = printf("lambdaSquare(5) = %d\n", lambdaSquare(5));
+// CHECK-NEXT: lambdaSquare(5) = 25
+auto lambda_r2 = printf("lambdaMult(2, 3) = %d\n", lambdaMult(2, 3));
+// CHECK-NEXT: lambdaMult(2, 3) = 6
+
+%undo  // Undo previous lambda assignments
+auto lambda_r3 = lambdaMult(3, 4);  // Should fail or revert to the original lambda
+
+// Test weak and strong symbol linkage.
+int __attribute__((weak)) weakFunc() { return 42; }
+int strongFunc() { return 100; }
+%undo  // Revert the weak function
+
+auto r5 = printf("weakFunc() = %d\n", weakFunc());
+// CHECK: weakFunc() = 42
+auto r6 = printf("strongFunc() = %d\n", strongFunc());
+// CHECK-NEXT: strongFunc() = 100
+
+// Weak variable linkage with different types.
+int varA = 20;
+static __typeof(varA) weakVarA __attribute__((__weakref__("varA")));
+char charVar = 'c';
+static __typeof(charVar) weakCharVar __attribute__((__weakref__("charVar")));
+auto r7 = printf("weakVarA = %d\n", weakVarA);
+// CHECK: weakVarA = 20
+auto r8 = printf("weakCharVar = %c\n", weakCharVar);
+// CHECK-NEXT: weakCharVar = c
+
+// Test complex lambdas with captures.
+int captureVar = 5;
+auto captureLambda = [](int x) { return x + captureVar; };
+int result1 = captureLambda(10);
+%undo  // Undo capture lambda
+
+auto r9 = printf("captureLambda(10) = %d\n", result1);
+// CHECK: captureLambda(10) = 15
+
+// Multiline statement test with arithmetic operations.
+int sum = \
+  5 + \
+  10;
+int prod = sum * 2;
+auto r10 = printf("sum = %d, prod = %d\n", sum, prod);
+// CHECK: sum = 15, prod = 30
+
+// Test multiline functions and macro behavior.
+#define MULTIPLY(a, b) ((a) * (b))
+
+int complexFunc(int x) \
+{ \
+  return MULTIPLY(x, 2) + x; \
+}
+
+auto r11 = printf("complexFunc(5) = %d\n", complexFunc(5));
+// CHECK: complexFunc(5) = 15
+
+%quit
\ No newline at end of file
diff --git a/clang/test/Misc/target-invalid-cpu-note/aarch64.c b/clang/test/Misc/target-invalid-cpu-note/aarch64.c
index 0a5c485e896be..0346ab2bb6b13 100644
--- a/clang/test/Misc/target-invalid-cpu-note/aarch64.c
+++ b/clang/test/Misc/target-invalid-cpu-note/aarch64.c
@@ -75,6 +75,7 @@
 // CHECK-SAME: {{^}}, exynos-m5
 // CHECK-SAME: {{^}}, falkor
 // CHECK-SAME: {{^}}, fujitsu-monaka
+// CHECK-SAME: {{^}}, gb10
 // CHECK-SAME: {{^}}, generic
 // CHECK-SAME: {{^}}, grace
 // CHECK-SAME: {{^}}, kryo
diff --git a/clang/test/Modules/ModulesLocalNamespace.cppm b/clang/test/Modules/ModulesLocalNamespace.cppm
new file mode 100644
index 0000000000000..9e05597809752
--- /dev/null
+++ b/clang/test/Modules/ModulesLocalNamespace.cppm
@@ -0,0 +1,47 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/m-a.cppm -emit-module-interface -o %t/m-a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-module-interface -o %t/m-b.pcm \
+// RUN:     -fprebuilt-module-path=%t
+// RUN: %clang_cc1 -std=c++20 %t/b.cpp -fsyntax-only -verify -fprebuilt-module-path=%t
+
+//--- a.cppm
+export module a;
+namespace aa {
+
+}
+
+//--- m-a.cppm
+export module m:a;
+namespace aa {
+struct A {};
+}
+
+//--- b.cppm
+module m:b;
+import :a;
+
+namespace bb {
+struct B {
+    void func(aa::A);
+};
+}
+
+//--- b.cpp
+// expected-no-diagnostics
+module m:b.impl;
+import a;
+import :b;
+
+namespace bb {
+using namespace aa;
+
+void B::func(A) {} 
+
+}
+
+
+
diff --git a/clang/test/Modules/mingw-exceptions.cppm b/clang/test/Modules/mingw-exceptions.cppm
new file mode 100644
index 0000000000000..db7aa2ce90a94
--- /dev/null
+++ b/clang/test/Modules/mingw-exceptions.cppm
@@ -0,0 +1,5 @@
+// REQUIRES: x86-registered-target
+// RUN: %clang -target x86_64-windows-gnu -x c++-module -std=gnu++23 -c -o /dev/null -Xclang -disable-llvm-passes %s
+
+// Make sure the command succeeds and doesn't break on the -exception-model flag in cc1.
+export module empty;
diff --git a/clang/test/Modules/odr_hash.cpp b/clang/test/Modules/odr_hash.cpp
index da24b1fe8729a..8ef53e32f2e95 100644
--- a/clang/test/Modules/odr_hash.cpp
+++ b/clang/test/Modules/odr_hash.cpp
@@ -5164,6 +5164,29 @@ namespace A {
 A::X x;
 #endif
 
+namespace TemplateDecltypeOperator {
+
+#if defined(FIRST) || defined(SECOND)
+template <class T6>
+T6 func();
+#endif
+
+#if defined(SECOND)
+template <class UnrelatedT>
+using UnrelatedAlias = decltype(func<UnrelatedT>())();
+#endif
+
+#if defined(FIRST) || defined(SECOND)
+class A {
+  template <class T6>
+  operator decltype(func<T6>()) () {}
+};
+#else
+A a;
+#endif
+
+}
+
 // Keep macros contained to one file.
 #ifdef FIRST
 #undef FIRST
diff --git a/clang/test/Modules/var-init-side-effects-modulemap.cpp b/clang/test/Modules/var-init-side-effects-modulemap.cpp
new file mode 100644
index 0000000000000..750a6f1731405
--- /dev/null
+++ b/clang/test/Modules/var-init-side-effects-modulemap.cpp
@@ -0,0 +1,51 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -fsyntax-only -fmodules -fmodules-cache-path=%t -fmodule-map-file=%t/module.modulemap  %t/test.cppm -I%t
+//
+
+//--- test.cppm
+#pragma clang module import Baz
+
+//--- Foo.h
+#pragma once
+class foo {
+  char dummy = 1;
+
+public:
+  static foo var;
+
+};
+
+inline foo foo::var;
+
+//--- Bar.h
+#pragma once
+#include <Foo.h>
+
+void bar() {
+  (void) foo::var;
+}
+
+//--- Baz.h
+#pragma once
+#include <Foo.h>
+
+void baz() {
+  (void) foo::var;
+}
+
+#include <Bar.h>
+
+//--- module.modulemap
+module Foo {
+  header "Foo.h"
+}
+module Bar {
+  header "Bar.h"
+}
+module Baz {
+  header "Baz.h"
+}
+
diff --git a/clang/test/Modules/var-init-side-effects-templated.cpp b/clang/test/Modules/var-init-side-effects-templated.cpp
new file mode 100644
index 0000000000000..542ca270429b6
--- /dev/null
+++ b/clang/test/Modules/var-init-side-effects-templated.cpp
@@ -0,0 +1,20 @@
+// Tests referencing variable with initializer containing side effect across module boundary
+
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %s -o %t
+
+export module Foo;
+
+export template <class Float>
+struct Wrapper {
+  double value;
+};
+
+export constexpr Wrapper<double> Compute() {
+  return Wrapper<double>{1.0};
+}
+
+export template <typename Float>
+Wrapper<Float> ComputeInFloat() {
+  const Wrapper<Float> a = Compute();
+  return a;
+}
diff --git a/clang/test/OpenMP/attr-assume.cpp b/clang/test/OpenMP/attr-assume.cpp
index 2d807d9a00b0d..9046ebd4bd45b 100644
--- a/clang/test/OpenMP/attr-assume.cpp
+++ b/clang/test/OpenMP/attr-assume.cpp
@@ -7,8 +7,8 @@
 [[omp::assume("omp_noopenmp")]] void f6(); // expected-warning {{unknown assumption string 'omp_noopenmp' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp'?}}
 [[omp::assume("omp_no_openmp_routine")]] void f7(); // expected-warning {{unknown assumption string 'omp_no_openmp_routine' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp_routines'?}}
 [[omp::assume("omp_no_openmp1")]] void f8(); // expected-warning {{unknown assumption string 'omp_no_openmp1' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp'?}}
-[[omp::assume("omp_no_openmp", "omp_no_openmp")]] void f9(); // expected-error {{'assume' attribute takes one argument}}
+[[omp::assume("omp_no_openmp", "omp_no_openmp")]] void f9(); // expected-error {{'omp::assume' attribute takes one argument}}
 [[omp::assume("omp_no_openmp_construct")]] void f10(); // expected-warning {{unknown assumption string 'omp_no_openmp_construct' may be misspelled; attribute is potentially ignored, did you mean 'omp_no_openmp_constructs'?}}
 
 [[omp::assume(3)]] int g1; // expected-error {{expected string literal as argument of 'assume' attribute}}
-[[omp::assume("omp_no_openmp")]] int g2; // expected-warning {{'assume' attribute only applies to functions and Objective-C methods}}
+[[omp::assume("omp_no_openmp")]] int g2; // expected-warning {{'omp::assume' attribute only applies to functions and Objective-C methods}}
diff --git a/clang/test/OpenMP/ompx_attributes_messages.cpp b/clang/test/OpenMP/ompx_attributes_messages.cpp
index c59c19027d26f..3e3ade0eb6acf 100644
--- a/clang/test/OpenMP/ompx_attributes_messages.cpp
+++ b/clang/test/OpenMP/ompx_attributes_messages.cpp
@@ -20,9 +20,9 @@ void bad() {
   {}
   #pragma omp target ompx_attribute(__attribute__((amdgpu_flat_work_group_size(1, 2, 3,)))) //  expected-error {{expected expression}}
   {}
-  #pragma omp target ompx_attribute([[clang::amdgpu_waves_per_eu(1, 2, 3)]]) //  expected-error {{'amdgpu_waves_per_eu' attribute takes no more than 2 arguments}}
+  #pragma omp target ompx_attribute([[clang::amdgpu_waves_per_eu(1, 2, 3)]]) //  expected-error {{'clang::amdgpu_waves_per_eu' attribute takes no more than 2 arguments}}
   {}
-  #pragma omp target ompx_attribute([[clang::unknown]]) //  expected-warning {{'ompx_attribute' clause only allows 'amdgpu_flat_work_group_size', 'amdgpu_waves_per_eu', and 'launch_bounds'; 'unknown' is ignored}}
+  #pragma omp target ompx_attribute([[clang::unknown]]) //  expected-warning {{'ompx_attribute' clause only allows 'amdgpu_flat_work_group_size', 'amdgpu_waves_per_eu', and 'launch_bounds'; 'clang::unknown' is ignored}}
   {}
   #pragma omp target ompx_attribute(baz) //  expected-error {{expected ')'}} expected-note {{to match this '('}}
   {}
diff --git a/clang/test/OpenMP/openmp-6-future-spellings.cpp b/clang/test/OpenMP/openmp-6-future-spellings.cpp
new file mode 100644
index 0000000000000..642ed3502d475
--- /dev/null
+++ b/clang/test/OpenMP/openmp-6-future-spellings.cpp
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=52 -ferror-limit 100 -o - %s
+
+// expected-warning@+1 {{directive spelling 'begin declare_target' is introduced in a later OpenMP version}}
+#pragma omp begin declare_target
+void f0();
+// expected-warning@+1 {{directive spelling 'end declare_target' is introduced in a later OpenMP version}}
+#pragma omp end declare_target
+
+// expected-warning@+1 {{directive spelling 'begin declare_variant' is introduced in a later OpenMP version}}
+#pragma omp begin declare_variant match(user={condition(true)})
+void f1();
+// expected-warning@+1 {{directive spelling 'end declare_variant' is introduced in a later OpenMP version}}
+#pragma omp end declare_variant
+
+int x;
+// expected-warning@+1 {{directive spelling 'declare_target' is introduced in a later OpenMP version}}
+#pragma omp declare_target(x)
+
+struct A {
+  int x, y;
+};
+// expected-warning@+1 {{directive spelling 'declare_mapper' is introduced in a later OpenMP version}}
+#pragma omp declare_mapper(mymapper: A a) map(tofrom:a.x, a.y)
+A add(A, A);
+// expected-warning@+1 {{directive spelling 'declare_reduction' is introduced in a later OpenMP version}}
+#pragma omp declare_reduction(+: A: omp_out = add(omp_in, omp_out))
+
+// expected-warning@+1 {{directive spelling 'declare_simd' is introduced in a later OpenMP version}}
+#pragma omp declare_simd
+void f2();
+
+void g3();
+// expected-warning@+1 {{directive spelling 'declare_variant' is introduced in a later OpenMP version}}
+#pragma omp declare_variant(g3) match(user={condition(true)})
+void f3() {}
+
+void fred() {
+  #pragma omp parallel
+  {
+    // expected-warning@+1 {{directive spelling 'cancellation_point' is introduced in a later OpenMP version}}
+    #pragma omp cancellation_point parallel
+  }
+
+  // expected-warning@+1 {{directive spelling 'target_data' is introduced in a later OpenMP version}}
+  #pragma omp target_data map(tofrom: x)
+  {}
+
+  // expected-warning@+1 {{directive spelling 'target_enter_data' is introduced in a later OpenMP version}}
+  #pragma omp target_enter_data map(to: x)
+  // expected-warning@+1 {{directive spelling 'target_exit_data' is introduced in a later OpenMP version}}
+  #pragma omp target_exit_data map(from: x)
+  // expected-warning@+1 {{directive spelling 'target_update' is introduced in a later OpenMP version}}
+  #pragma omp target_update from(x)
+}
+
diff --git a/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp b/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp
index 87fa7fe462daa..4264e58dd2152 100644
--- a/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp
+++ b/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp
@@ -1,4 +1,4 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --filter-out-after "getelem.*kernel" --filter-out "= alloca.*" --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*" --global-hex-value-regex ".offload_maptypes.*"
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
@@ -7,168 +7,249 @@
 #ifndef HEADER
 #define HEADER
 
-extern void *malloc (int __size) throw () __attribute__ ((__malloc__));
+void f1() {
+  int *ptr;
+  // &ptr, &ptr, sizeof(ptr), TO | PARAM
+  #pragma omp target map(ptr)
+    ptr[1] = 5;
+}
 
-void foo() {
-  int *ptr = (int *) malloc(3 * sizeof(int));
+void f2() {
+  int *ptr;
+  // &ptr[0], &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM
+  #pragma omp target map(ptr[2])
+    ptr[1] = 6;
+}
 
+void f3() {
+  int *ptr;
+  // &ptr, &ptr[0], sizeof(ptr[0:2]), TO | FROM | PARAM | PTR_AND_OBJ
   #pragma omp target map(ptr, ptr[0:2])
-  {
-    ptr[1] = 6;
-  }
+    ptr[1] = 7;
+}
+
+void f4() {
+  int *ptr;
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
   #pragma omp target map(ptr, ptr[2])
-  {
     ptr[2] = 8;
-  }
-  #pragma omp target data map(ptr, ptr[2])
-  {
+}
+
+void f5() {
+  int *ptr;
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+ #pragma omp target map(ptr[2], ptr)
     ptr[2] = 9;
-  }
+}
+
+void f6() {
+  int *ptr;
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+  // TODO: PARAM should not be needed here.
+  #pragma omp target data map(ptr, ptr[2])
+    ptr[2] = 10;
+}
+
+void f7() {
+  int *ptr;
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+  // TODO: PARAM should not be needed here.
+  #pragma omp target data map(ptr[2], ptr)
+    ptr[2] = 11;
 }
 #endif
-// CHECK-LABEL: define {{[^@]+}}@_Z3foov
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 [[#0x23]]]
+// CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 [[#0x23]]]
+// CHECK: @.offload_sizes.3 = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes.4 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.5 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.6 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.7 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.8 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.9 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.10 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.11 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.12 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+//.
+// CHECK-LABEL: define {{[^@]+}}@_Z2f1v
 // CHECK-SAME: () #[[ATTR0:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[PTR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_BASEPTRS:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_PTRS:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_MAPPERS:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[KERNEL_ARGS:%.*]] = alloca [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_BASEPTRS2:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_PTRS3:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_MAPPERS4:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[KERNEL_ARGS5:%.*]] = alloca [[STRUCT___TGT_KERNEL_ARGUMENTS]], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_BASEPTRS9:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_PTRS10:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[DOTOFFLOAD_MAPPERS11:%.*]] = alloca [1 x ptr], align 8
-// CHECK-NEXT:    [[CALL:%.*]] = call noalias noundef ptr @_Z6malloci(i32 noundef signext 12) #[[ATTR3:[0-9]+]]
-// CHECK-NEXT:    store ptr [[CALL]], ptr [[PTR]], align 8
-// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR]], align 8
-// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i64 0
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
-// CHECK-NEXT:    store ptr [[PTR]], ptr [[TMP2]], align 8
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
-// CHECK-NEXT:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 0
-// CHECK-NEXT:    store ptr null, ptr [[TMP4]], align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
-// CHECK-NEXT:    store i32 3, ptr [[TMP7]], align 4
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 1
-// CHECK-NEXT:    store i32 1, ptr [[TMP8]], align 4
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 2
-// CHECK-NEXT:    store ptr [[TMP5]], ptr [[TMP9]], align 8
-// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 3
-// CHECK-NEXT:    store ptr [[TMP6]], ptr [[TMP10]], align 8
-// CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 4
-// CHECK-NEXT:    store ptr @.offload_sizes, ptr [[TMP11]], align 8
-// CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 5
-// CHECK-NEXT:    store ptr @.offload_maptypes, ptr [[TMP12]], align 8
-// CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 6
-// CHECK-NEXT:    store ptr null, ptr [[TMP13]], align 8
-// CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 7
-// CHECK-NEXT:    store ptr null, ptr [[TMP14]], align 8
-// CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 8
-// CHECK-NEXT:    store i64 0, ptr [[TMP15]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 9
-// CHECK-NEXT:    store i64 0, ptr [[TMP16]], align 8
-// CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 10
-// CHECK-NEXT:    store [3 x i32] [i32 -1, i32 0, i32 0], ptr [[TMP17]], align 4
-// CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 11
-// CHECK-NEXT:    store [3 x i32] zeroinitializer, ptr [[TMP18]], align 4
-// CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 12
-// CHECK-NEXT:    store i32 0, ptr [[TMP19]], align 4
-// CHECK-NEXT:    [[TMP20:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1:[0-9]+]], i64 -1, i32 -1, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l15.region_id, ptr [[KERNEL_ARGS]])
-// CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i32 [[TMP20]], 0
-// CHECK-NEXT:    br i1 [[TMP21]], label [[OMP_OFFLOAD_FAILED:%.*]], label [[OMP_OFFLOAD_CONT:%.*]]
-// CHECK:       omp_offload.failed:
-// CHECK-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l15(ptr [[TMP0]]) #[[ATTR3]]
-// CHECK-NEXT:    br label [[OMP_OFFLOAD_CONT]]
-// CHECK:       omp_offload.cont:
-// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[PTR]], align 8
-// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[PTR]], align 8
-// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 2
-// CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS2]], i32 0, i32 0
-// CHECK-NEXT:    store ptr [[PTR]], ptr [[TMP24]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS3]], i32 0, i32 0
-// CHECK-NEXT:    store ptr [[ARRAYIDX1]], ptr [[TMP25]], align 8
-// CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS4]], i64 0, i64 0
-// CHECK-NEXT:    store ptr null, ptr [[TMP26]], align 8
-// CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS2]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS3]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 0
-// CHECK-NEXT:    store i32 3, ptr [[TMP29]], align 4
-// CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 1
-// CHECK-NEXT:    store i32 1, ptr [[TMP30]], align 4
-// CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 2
-// CHECK-NEXT:    store ptr [[TMP27]], ptr [[TMP31]], align 8
-// CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 3
-// CHECK-NEXT:    store ptr [[TMP28]], ptr [[TMP32]], align 8
-// CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 4
-// CHECK-NEXT:    store ptr @.offload_sizes.1, ptr [[TMP33]], align 8
-// CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 5
-// CHECK-NEXT:    store ptr @.offload_maptypes.2, ptr [[TMP34]], align 8
-// CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 6
-// CHECK-NEXT:    store ptr null, ptr [[TMP35]], align 8
-// CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 7
-// CHECK-NEXT:    store ptr null, ptr [[TMP36]], align 8
-// CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 8
-// CHECK-NEXT:    store i64 0, ptr [[TMP37]], align 8
-// CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 9
-// CHECK-NEXT:    store i64 0, ptr [[TMP38]], align 8
-// CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 10
-// CHECK-NEXT:    store [3 x i32] [i32 -1, i32 0, i32 0], ptr [[TMP39]], align 4
-// CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 11
-// CHECK-NEXT:    store [3 x i32] zeroinitializer, ptr [[TMP40]], align 4
-// CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 12
-// CHECK-NEXT:    store i32 0, ptr [[TMP41]], align 4
-// CHECK-NEXT:    [[TMP42:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 -1, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l19.region_id, ptr [[KERNEL_ARGS5]])
-// CHECK-NEXT:    [[TMP43:%.*]] = icmp ne i32 [[TMP42]], 0
-// CHECK-NEXT:    br i1 [[TMP43]], label [[OMP_OFFLOAD_FAILED6:%.*]], label [[OMP_OFFLOAD_CONT7:%.*]]
-// CHECK:       omp_offload.failed6:
-// CHECK-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l19(ptr [[TMP22]]) #[[ATTR3]]
-// CHECK-NEXT:    br label [[OMP_OFFLOAD_CONT7]]
-// CHECK:       omp_offload.cont7:
-// CHECK-NEXT:    [[TMP44:%.*]] = load ptr, ptr [[PTR]], align 8
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[TMP44]], i64 2
-// CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS9]], i32 0, i32 0
-// CHECK-NEXT:    store ptr [[PTR]], ptr [[TMP45]], align 8
-// CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS10]], i32 0, i32 0
-// CHECK-NEXT:    store ptr [[ARRAYIDX8]], ptr [[TMP46]], align 8
-// CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS11]], i64 0, i64 0
-// CHECK-NEXT:    store ptr null, ptr [[TMP47]], align 8
-// CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS9]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS10]], i32 0, i32 0
-// CHECK-NEXT:    call void @__tgt_target_data_begin_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP48]], ptr [[TMP49]], ptr @.offload_sizes.3, ptr @.offload_maptypes.4, ptr null, ptr null)
-// CHECK-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[PTR]], align 8
-// CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, ptr [[TMP50]], i64 2
-// CHECK-NEXT:    store i32 9, ptr [[ARRAYIDX12]], align 4
-// CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS9]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS10]], i32 0, i32 0
-// CHECK-NEXT:    call void @__tgt_target_data_end_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP51]], ptr [[TMP52]], ptr @.offload_sizes.3, ptr @.offload_maptypes.4, ptr null, ptr null)
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l15
-// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8
-// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
-// CHECK-NEXT:    store i32 6, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l19
-// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR2]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8
-// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
-// CHECK-NEXT:    store i32 8, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    ret void
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR:%.*]], ptr [[TMP0]], align 8
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f1v_l13
+// CHECK-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8, !nonnull [[META13:![0-9]+]], !align [[META14:![0-9]+]]
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 1
+// CHECK:    store i32 5, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f2v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 2
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f2v_l20
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f3v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i64 0
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f3v_l27
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 7, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f4v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f4v_l34
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    store i32 8, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f5v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f5v_l41
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    store i32 9, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f6v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_begin_mapper(ptr @[[GLOB1:[0-9]+]], i64 -1, i32 1, ptr [[TMP4]], ptr [[TMP5]], ptr @.offload_sizes.9, ptr @.offload_maptypes.10, ptr null, ptr null)
+// CHECK:    [[TMP6:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 2
+// CHECK:    store i32 10, ptr [[ARRAYIDX1]], align 4
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_end_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP7]], ptr [[TMP8]], ptr @.offload_sizes.9, ptr @.offload_maptypes.10, ptr null, ptr null)
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f7v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_begin_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP4]], ptr [[TMP5]], ptr @.offload_sizes.11, ptr @.offload_maptypes.12, ptr null, ptr null)
+// CHECK:    [[TMP6:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 2
+// CHECK:    store i32 11, ptr [[ARRAYIDX1]], align 4
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_end_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP7]], ptr [[TMP8]], ptr @.offload_sizes.11, ptr @.offload_maptypes.12, ptr null, ptr null)
+// CHECK:    ret void
 //
diff --git a/clang/test/OpenMP/target_map_both_pointer_pointee_codegen_global.cpp b/clang/test/OpenMP/target_map_both_pointer_pointee_codegen_global.cpp
new file mode 100644
index 0000000000000..ee9cb9af4d3ca
--- /dev/null
+++ b/clang/test/OpenMP/target_map_both_pointer_pointee_codegen_global.cpp
@@ -0,0 +1,249 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --filter-out-after "getelem.*kernel" --filter-out "= alloca.*" --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*" --global-hex-value-regex ".offload_maptypes.*"
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+int *ptr;
+
+void f1() {
+  // &ptr, &ptr, sizeof(ptr), TO | PARAM
+  #pragma omp target map(ptr)
+    ptr[1] = 5;
+}
+
+void f2() {
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+  #pragma omp target map(ptr[2])
+    ptr[1] = 6;
+}
+
+void f3() {
+  // &ptr, &ptr[0], sizeof(ptr[0:2]), TO | FROM | PARAM | PTR_AND_OBJ
+  #pragma omp target map(ptr, ptr[0:2])
+    ptr[1] = 7;
+}
+
+void f4() {
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+  #pragma omp target map(ptr, ptr[2])
+    ptr[2] = 8;
+}
+
+void f5() {
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+ #pragma omp target map(ptr[2], ptr)
+    ptr[2] = 9;
+}
+
+void f6() {
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+  // TODO: PARAM should not be needed here.
+  #pragma omp target data map(ptr, ptr[2])
+    ptr[2] = 10;
+}
+
+void f7() {
+  // &ptr, &ptr[2], sizeof(ptr[2]), TO | FROM | PARAM | PTR_AND_OBJ
+  // TODO: PARAM should not be needed here.
+  #pragma omp target data map(ptr[2], ptr)
+    ptr[2] = 11;
+}
+#endif
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 [[#0x23]]]
+// CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.3 = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes.4 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.5 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.6 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.7 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.8 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.9 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.10 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.11 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.12 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+//.
+// CHECK-LABEL: define {{[^@]+}}@_Z2f1v
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP0]], align 8
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f1v_l14
+// CHECK-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8, !nonnull [[META13:![0-9]+]], !align [[META14:![0-9]+]]
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 1
+// CHECK:    store i32 5, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f2v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f2v_l20
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f3v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i64 0
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f3v_l26
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 7, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f4v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f4v_l32
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    store i32 8, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f5v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f5v_l38
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    store i32 9, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f6v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_begin_mapper(ptr @[[GLOB1:[0-9]+]], i64 -1, i32 1, ptr [[TMP4]], ptr [[TMP5]], ptr @.offload_sizes.9, ptr @.offload_maptypes.10, ptr null, ptr null)
+// CHECK:    [[TMP6:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 2
+// CHECK:    store i32 10, ptr [[ARRAYIDX1]], align 4
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_end_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP7]], ptr [[TMP8]], ptr @.offload_sizes.9, ptr @.offload_maptypes.10, ptr null, ptr null)
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f7v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[ARRAYIDX]], ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_begin_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP4]], ptr [[TMP5]], ptr @.offload_sizes.11, ptr @.offload_maptypes.12, ptr null, ptr null)
+// CHECK:    [[TMP6:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 2
+// CHECK:    store i32 11, ptr [[ARRAYIDX1]], align 4
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    call void @__tgt_target_data_end_mapper(ptr @[[GLOB1]], i64 -1, i32 1, ptr [[TMP7]], ptr [[TMP8]], ptr @.offload_sizes.11, ptr @.offload_maptypes.12, ptr null, ptr null)
+// CHECK:    ret void
+//
diff --git a/clang/test/OpenMP/target_map_ptr_and_star_global.cpp b/clang/test/OpenMP/target_map_ptr_and_star_global.cpp
new file mode 100644
index 0000000000000..84899cb8e4fad
--- /dev/null
+++ b/clang/test/OpenMP/target_map_ptr_and_star_global.cpp
@@ -0,0 +1,161 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --filter-out-after "getelem.*kernel" --filter-out "= alloca.*" --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*" --global-hex-value-regex ".offload_maptypes.*"
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+int *ptr;
+void f1() {
+  // &ptr, &ptr, sizeof(ptr), TO | FROM | PARAM
+  #pragma omp target map(ptr)
+    ptr[1] = 6;
+}
+
+void f2() {
+  // &ptr, &ptr[0], sizeof(ptr[0]), TO | FROM | PARAM | PTR_AND_OBJ
+  #pragma omp target map(*ptr)
+    ptr[1] = 6;
+}
+
+void f3() {
+  // &ptr, &ptr, sizeof(ptr), TO | FROM | PARAM
+  // &ptr, &ptr[0], sizeof(ptr[0]), TO | FROM | PTR_AND_OBJ
+  #pragma omp target map(ptr, *ptr)
+    ptr[1] = 6;
+}
+
+void f4() {
+  // &ptr, &ptr[0], sizeof(ptr[0]), TO | FROM | PTR_AND_OBJ | PARAM
+  // &ptr, &ptr, sizeof(ptr), TO | FROM | PARAM
+  #pragma omp target map(*ptr, ptr)
+    ptr[2] = 8;
+}
+
+#endif
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 [[#0x23]]]
+// CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 [[#0x33]]]
+// CHECK: @.offload_sizes.3 = private unnamed_addr constant [2 x i64] [i64 8, i64 4]
+// CHECK: @.offload_maptypes.4 = private unnamed_addr constant [2 x i64] [i64 [[#0x23]], i64 [[#0x13]]]
+// CHECK: @.offload_sizes.5 = private unnamed_addr constant [2 x i64] [i64 4, i64 8]
+// CHECK: @.offload_maptypes.6 = private unnamed_addr constant [2 x i64] [i64 [[#0x33]], i64 [[#0x3]]]
+//.
+// CHECK-LABEL: define {{[^@]+}}@_Z2f1v
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP0]], align 8
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f1v_l13
+// CHECK-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8, !nonnull [[META11:![0-9]+]], !align [[META12:![0-9]+]]
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f2v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f2v_l19
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f3v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr @ptr, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP1]], ptr [[TMP6]], align 8
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP7]], align 8
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f3v_l26
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f4v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ptr, align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ptr, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr @ptr, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr @ptr, ptr [[TMP6]], align 8
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP7]], align 8
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f4v_l33
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    store i32 8, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
diff --git a/clang/test/OpenMP/target_map_ptr_and_star_local.cpp b/clang/test/OpenMP/target_map_ptr_and_star_local.cpp
new file mode 100644
index 0000000000000..246c0c5f99a68
--- /dev/null
+++ b/clang/test/OpenMP/target_map_ptr_and_star_local.cpp
@@ -0,0 +1,167 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --filter-out-after "getelem.*kernel" --filter-out "= alloca.*" --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*" --global-hex-value-regex ".offload_maptypes.*"
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+void f1() {
+  int *ptr;
+  // &ptr, &ptr, sizeof(ptr), TO | FROM | PARAM
+  #pragma omp target map(ptr)
+    ptr[1] = 6;
+}
+
+void f2() {
+  int *ptr;
+  // &ptr[0], &ptr[0], sizeof(ptr[0]), TO | FROM | PARAM
+  #pragma omp target map(*ptr)
+    ptr[1] = 6;
+}
+
+void f3() {
+  int *ptr;
+  // &ptr, &ptr, sizeof(ptr), TO | FROM | PARAM
+  // &ptr[0], &ptr[0], sizeof(ptr[0]), TO | FROM
+  #pragma omp target map(ptr, *ptr)
+    ptr[1] = 6;
+}
+
+void f4() {
+  int *ptr;
+  // &ptr[0], &ptr[0], sizeof(ptr[0]), TO | FROM
+  // &ptr, &ptr, sizeof(ptr), TO | FROM | PARAM
+  #pragma omp target map(*ptr, ptr)
+    ptr[2] = 8;
+}
+
+#endif
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 [[#0x23]]]
+// CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 [[#0x23]]]
+// CHECK: @.offload_sizes.3 = private unnamed_addr constant [2 x i64] [i64 8, i64 4]
+// CHECK: @.offload_maptypes.4 = private unnamed_addr constant [2 x i64] [i64 [[#0x23]], i64 [[#0x3]]]
+// CHECK: @.offload_sizes.5 = private unnamed_addr constant [2 x i64] [i64 4, i64 8]
+// CHECK: @.offload_maptypes.6 = private unnamed_addr constant [2 x i64] [i64 [[#0x23]], i64 [[#0x3]]]
+//.
+// CHECK-LABEL: define {{[^@]+}}@_Z2f1v
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR:%.*]], ptr [[TMP0]], align 8
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f1v_l13
+// CHECK-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8, !nonnull [[META11:![0-9]+]], !align [[META12:![0-9]+]]
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f2v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP2]], ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f2v_l20
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f3v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PTR]], ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP1]], ptr [[TMP6]], align 8
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP2]], ptr [[TMP7]], align 8
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP8]], align 8
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f3v_l28
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_Z2f4v
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK:  entry:
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP2]], ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[PTR]], ptr [[TMP6]], align 8
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[PTR]], ptr [[TMP7]], align 8
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP8]], align 8
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f4v_l36
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR1]] {
+// CHECK:  entry:
+// CHECK:    store ptr [[PTR]], ptr [[PTR_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK:    store i32 8, ptr [[ARRAYIDX]], align 4
+// CHECK:    ret void
+//
diff --git a/clang/test/OpenMP/target_map_structptr_and_member_global.cpp b/clang/test/OpenMP/target_map_structptr_and_member_global.cpp
new file mode 100644
index 0000000000000..523f88dc8dba3
--- /dev/null
+++ b/clang/test/OpenMP/target_map_structptr_and_member_global.cpp
@@ -0,0 +1,275 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --filter-out-after "getelem.*kernel" --filter-out "= alloca.*" --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*" --global-hex-value-regex ".offload_maptypes.*" --version 5
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+struct S {
+  short x;
+  int y;
+  int *p;
+};
+
+S s, *ps;
+
+void f1() {
+  // &ps, &ps, sizeof(ps), TO | PARAM
+  #pragma omp target map(to: ps)
+    ps->y = 5;
+}
+
+void f2() {
+  // &ps[0], &ps->y, sizeof(ps->y), TO | PARAM
+  #pragma omp target map(to: ps->y)
+    ps->y = 6;
+}
+
+void f3() {
+  // &ps[0], &ps[0], sizeof(ps[0]), PARAM | ALLOC
+  // &ps, &ps, sizeof(ps), TO | MEMBER_OF(1)
+  // &ps[0], &ps->y, sizeof(ps->y), TO | MEMBER_OF(1)
+  #pragma omp target map(to: ps, ps->y)
+    ps->y = 7;
+}
+
+void f4() {
+  // &ps[0], &ps[0], sizeof(ps[0]), PARAM | ALLOC
+  // &ps[0], &ps->y, sizeof(ps->y), TO | MEMBER_OF(1)
+  // &ps, &ps, sizeof(ps), TO | MEMBER_OF(1)
+  #pragma omp target map(to: ps->y, ps)
+    ps->y = 8;
+}
+
+void f5() {
+  // &ps[0], &ps[0], sizeof(ps[0]), PARAM | ALLOC
+  // &ps[0], &ps->y, sizeof(ps->y), TO | MEMBER_OF(1)
+  // &ps, &ps, sizeof(ps), TO | MEMBER_OF(1)
+  // &ps[0], &ps->x, sizeof(ps->x), TO | MEMBER_OF(1)
+  #pragma omp target map(to: ps->y, ps, ps->x)
+    ps->y = 9;
+}
+
+#endif
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 [[#0x21]]]
+// CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 [[#0x21]]]
+// CHECK: @.offload_sizes.3 = private unnamed_addr constant [3 x i64] [i64 0, i64 8, i64 4]
+// CHECK: @.offload_maptypes.4 = private unnamed_addr constant [3 x i64] [i64 [[#0x20]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]]]
+// CHECK: @.offload_sizes.5 = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 8]
+// CHECK: @.offload_maptypes.6 = private unnamed_addr constant [3 x i64] [i64 [[#0x20]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]]]
+// CHECK: @.offload_sizes.7 = private unnamed_addr constant [4 x i64] [i64 0, i64 4, i64 8, i64 2]
+// CHECK: @.offload_maptypes.8 = private unnamed_addr constant [4 x i64] [i64 [[#0x20]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]]]
+//.
+// CHECK-LABEL: define dso_local void @_Z2f1v(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ps, ptr [[TMP0]], align 8
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr @ps, ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f1v_l20(
+// CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(8) [[PS:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8, !nonnull [[META13:![0-9]+]], !align [[META14:![0-9]+]]
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP1]], i32 0, i32 1
+// CHECK:    store i32 5, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f2v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[Y]], ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f2v_l26(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 6, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f3v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = getelementptr [[STRUCT_S]], ptr [[TMP1]], i32 1
+// CHECK:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+// CHECK:    [[TMP5:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK:    [[TMP6:%.*]] = sub i64 [[TMP4]], [[TMP5]]
+// CHECK:    [[TMP7:%.*]] = sdiv exact i64 [[TMP6]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES:%.*]], ptr align 8 @.offload_sizes.3, i64 24, i1 false)
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP8]], align 8
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP9]], align 8
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    store i64 [[TMP7]], ptr [[TMP10]], align 8
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP11]], align 8
+// CHECK:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr @ps, ptr [[TMP12]], align 8
+// CHECK:    [[TMP13:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr @ps, ptr [[TMP13]], align 8
+// CHECK:    [[TMP14:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP14]], align 8
+// CHECK:    [[TMP15:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2
+// CHECK:    store ptr [[TMP1]], ptr [[TMP15]], align 8
+// CHECK:    [[TMP16:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2
+// CHECK:    store ptr [[Y]], ptr [[TMP16]], align 8
+// CHECK:    [[TMP17:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2
+// CHECK:    store ptr null, ptr [[TMP17]], align 8
+// CHECK:    [[TMP18:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP19:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP20:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    [[TMP21:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f3v_l34(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 7, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f4v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = getelementptr [[STRUCT_S]], ptr [[TMP1]], i32 1
+// CHECK:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+// CHECK:    [[TMP5:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK:    [[TMP6:%.*]] = sub i64 [[TMP4]], [[TMP5]]
+// CHECK:    [[TMP7:%.*]] = sdiv exact i64 [[TMP6]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES:%.*]], ptr align 8 @.offload_sizes.5, i64 24, i1 false)
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP8]], align 8
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP9]], align 8
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    store i64 [[TMP7]], ptr [[TMP10]], align 8
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP11]], align 8
+// CHECK:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP1]], ptr [[TMP12]], align 8
+// CHECK:    [[TMP13:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[Y]], ptr [[TMP13]], align 8
+// CHECK:    [[TMP14:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP14]], align 8
+// CHECK:    [[TMP15:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2
+// CHECK:    store ptr @ps, ptr [[TMP15]], align 8
+// CHECK:    [[TMP16:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2
+// CHECK:    store ptr @ps, ptr [[TMP16]], align 8
+// CHECK:    [[TMP17:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2
+// CHECK:    store ptr null, ptr [[TMP17]], align 8
+// CHECK:    [[TMP18:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP19:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP20:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    [[TMP21:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f4v_l42(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 8, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f5v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[TMP4:%.*]] = load ptr, ptr @ps, align 8
+// CHECK:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP4]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr [[STRUCT_S]], ptr [[TMP1]], i32 1
+// CHECK:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK:    [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK:    [[TMP8:%.*]] = sub i64 [[TMP6]], [[TMP7]]
+// CHECK:    [[TMP9:%.*]] = sdiv exact i64 [[TMP8]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES:%.*]], ptr align 8 @.offload_sizes.7, i64 32, i1 false)
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP10]], align 8
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP11]], align 8
+// CHECK:    [[TMP12:%.*]] = getelementptr inbounds [4 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    store i64 [[TMP9]], ptr [[TMP12]], align 8
+// CHECK:    [[TMP13:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP13]], align 8
+// CHECK:    [[TMP14:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP1]], ptr [[TMP14]], align 8
+// CHECK:    [[TMP15:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[Y]], ptr [[TMP15]], align 8
+// CHECK:    [[TMP16:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP16]], align 8
+// CHECK:    [[TMP17:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2
+// CHECK:    store ptr @ps, ptr [[TMP17]], align 8
+// CHECK:    [[TMP18:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2
+// CHECK:    store ptr @ps, ptr [[TMP18]], align 8
+// CHECK:    [[TMP19:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2
+// CHECK:    store ptr null, ptr [[TMP19]], align 8
+// CHECK:    [[TMP20:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 3
+// CHECK:    store ptr [[TMP3]], ptr [[TMP20]], align 8
+// CHECK:    [[TMP21:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 3
+// CHECK:    store ptr [[X]], ptr [[TMP21]], align 8
+// CHECK:    [[TMP22:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 3
+// CHECK:    store ptr null, ptr [[TMP22]], align 8
+// CHECK:    [[TMP23:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP24:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP25:%.*]] = getelementptr inbounds [4 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    [[TMP26:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f5v_l51(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 9, ptr [[Y]], align 4
+// CHECK:    ret void
+//
diff --git a/clang/test/OpenMP/target_map_structptr_and_member_local.cpp b/clang/test/OpenMP/target_map_structptr_and_member_local.cpp
new file mode 100644
index 0000000000000..b366f331941b7
--- /dev/null
+++ b/clang/test/OpenMP/target_map_structptr_and_member_local.cpp
@@ -0,0 +1,278 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --filter-out-after "getelem.*kernel" --filter-out "= alloca.*" --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex "\.offload_.*" --global-hex-value-regex ".offload_maptypes.*" --version 5
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+struct S {
+  short x;
+  int y;
+  int *p;
+};
+
+void f1() {
+  S s, *ps;
+  // &ps, &ps, sizeof(ps), TO | PARAM
+  #pragma omp target map(to: ps)
+    ps->y = 5;
+}
+
+void f2() {
+  S s, *ps;
+  // &ps[0], &ps->y, sizeof(ps->y), TO | PARAM
+  #pragma omp target map(to: ps->y)
+    ps->y = 6;
+}
+
+void f3() {
+  S s, *ps;
+  // &ps[0], &ps[0], sizeof(ps[0]), PARAM | ALLOC
+  // &ps, &ps, sizeof(ps), TO | MEMBER_OF(1)
+  // &ps[0], &ps->y, sizeof(ps->y), TO | MEMBER_OF(1)
+  #pragma omp target map(to: ps, ps->y)
+    ps->y = 7;
+}
+
+void f4() {
+  S s, *ps;
+  // &ps[0], &ps[0], sizeof(ps[0]), PARAM | ALLOC
+  // &ps[0], &ps->y, sizeof(ps->y), TO | MEMBER_OF(1)
+  // &ps, &ps, sizeof(ps), TO | MEMBER_OF(1)
+  #pragma omp target map(to: ps->y, ps)
+    ps->y = 8;
+}
+
+void f5() {
+  S s, *ps;
+  // &ps[0], &ps[0], sizeof(ps[0]), PARAM | ALLOC
+  // &ps[0], &ps->y, sizeof(ps->y), TO | MEMBER_OF(1)
+  // &ps, &ps, sizeof(ps), TO | MEMBER_OF(1)
+  // &ps[0], &ps->x, sizeof(ps->x), TO | MEMBER_OF(1)
+  #pragma omp target map(to: ps->y, ps, ps->x)
+    ps->y = 9;
+}
+
+#endif
+//.
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 [[#0x21]]]
+// CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 4]
+// CHECK: @.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 [[#0x21]]]
+// CHECK: @.offload_sizes.3 = private unnamed_addr constant [3 x i64] [i64 0, i64 8, i64 4]
+// CHECK: @.offload_maptypes.4 = private unnamed_addr constant [3 x i64] [i64 [[#0x20]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]]]
+// CHECK: @.offload_sizes.5 = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 8]
+// CHECK: @.offload_maptypes.6 = private unnamed_addr constant [3 x i64] [i64 [[#0x20]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]]]
+// CHECK: @.offload_sizes.7 = private unnamed_addr constant [4 x i64] [i64 0, i64 4, i64 8, i64 2]
+// CHECK: @.offload_maptypes.8 = private unnamed_addr constant [4 x i64] [i64 [[#0x20]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]], i64 [[#0x1000000000001]]]
+//.
+// CHECK-LABEL: define dso_local void @_Z2f1v(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PS:%.*]], ptr [[TMP0]], align 8
+// CHECK:    [[TMP1:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[PS]], ptr [[TMP1]], align 8
+// CHECK:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP2]], align 8
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f1v_l19(
+// CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(8) [[PS:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8, !nonnull [[META13:![0-9]+]], !align [[META14:![0-9]+]]
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP1]], i32 0, i32 1
+// CHECK:    store i32 5, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f2v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP3]], align 8
+// CHECK:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[Y]], ptr [[TMP4]], align 8
+// CHECK:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP5]], align 8
+// CHECK:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP7:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f2v_l26(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 6, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f3v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = getelementptr [[STRUCT_S]], ptr [[TMP1]], i32 1
+// CHECK:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+// CHECK:    [[TMP5:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK:    [[TMP6:%.*]] = sub i64 [[TMP4]], [[TMP5]]
+// CHECK:    [[TMP7:%.*]] = sdiv exact i64 [[TMP6]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES:%.*]], ptr align 8 @.offload_sizes.3, i64 24, i1 false)
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP8]], align 8
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP9]], align 8
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    store i64 [[TMP7]], ptr [[TMP10]], align 8
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP11]], align 8
+// CHECK:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[PS]], ptr [[TMP12]], align 8
+// CHECK:    [[TMP13:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[PS]], ptr [[TMP13]], align 8
+// CHECK:    [[TMP14:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP14]], align 8
+// CHECK:    [[TMP15:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2
+// CHECK:    store ptr [[TMP1]], ptr [[TMP15]], align 8
+// CHECK:    [[TMP16:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2
+// CHECK:    store ptr [[Y]], ptr [[TMP16]], align 8
+// CHECK:    [[TMP17:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2
+// CHECK:    store ptr null, ptr [[TMP17]], align 8
+// CHECK:    [[TMP18:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP19:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP20:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    [[TMP21:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f3v_l35(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 7, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f4v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = getelementptr [[STRUCT_S]], ptr [[TMP1]], i32 1
+// CHECK:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+// CHECK:    [[TMP5:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK:    [[TMP6:%.*]] = sub i64 [[TMP4]], [[TMP5]]
+// CHECK:    [[TMP7:%.*]] = sdiv exact i64 [[TMP6]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES:%.*]], ptr align 8 @.offload_sizes.5, i64 24, i1 false)
+// CHECK:    [[TMP8:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP8]], align 8
+// CHECK:    [[TMP9:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP9]], align 8
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    store i64 [[TMP7]], ptr [[TMP10]], align 8
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP11]], align 8
+// CHECK:    [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP1]], ptr [[TMP12]], align 8
+// CHECK:    [[TMP13:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[Y]], ptr [[TMP13]], align 8
+// CHECK:    [[TMP14:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP14]], align 8
+// CHECK:    [[TMP15:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2
+// CHECK:    store ptr [[PS]], ptr [[TMP15]], align 8
+// CHECK:    [[TMP16:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2
+// CHECK:    store ptr [[PS]], ptr [[TMP16]], align 8
+// CHECK:    [[TMP17:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2
+// CHECK:    store ptr null, ptr [[TMP17]], align 8
+// CHECK:    [[TMP18:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP19:%.*]] = getelementptr inbounds [3 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP20:%.*]] = getelementptr inbounds [3 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    [[TMP21:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f4v_l44(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 8, ptr [[Y]], align 4
+// CHECK:    ret void
+//
+//
+// CHECK-LABEL: define dso_local void @_Z2f5v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS:%.*]], align 8
+// CHECK:    [[TMP1:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[TMP2:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP2]], i32 0, i32 1
+// CHECK:    [[TMP3:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[TMP4:%.*]] = load ptr, ptr [[PS]], align 8
+// CHECK:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP4]], i32 0, i32 0
+// CHECK:    [[TMP5:%.*]] = getelementptr [[STRUCT_S]], ptr [[TMP1]], i32 1
+// CHECK:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK:    [[TMP7:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK:    [[TMP8:%.*]] = sub i64 [[TMP6]], [[TMP7]]
+// CHECK:    [[TMP9:%.*]] = sdiv exact i64 [[TMP8]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+// CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTOFFLOAD_SIZES:%.*]], ptr align 8 @.offload_sizes.7, i64 32, i1 false)
+// CHECK:    [[TMP10:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP10]], align 8
+// CHECK:    [[TMP11:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS:%.*]], i32 0, i32 0
+// CHECK:    store ptr [[TMP1]], ptr [[TMP11]], align 8
+// CHECK:    [[TMP12:%.*]] = getelementptr inbounds [4 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    store i64 [[TMP9]], ptr [[TMP12]], align 8
+// CHECK:    [[TMP13:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS:%.*]], i64 0, i64 0
+// CHECK:    store ptr null, ptr [[TMP13]], align 8
+// CHECK:    [[TMP14:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 1
+// CHECK:    store ptr [[TMP1]], ptr [[TMP14]], align 8
+// CHECK:    [[TMP15:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 1
+// CHECK:    store ptr [[Y]], ptr [[TMP15]], align 8
+// CHECK:    [[TMP16:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 1
+// CHECK:    store ptr null, ptr [[TMP16]], align 8
+// CHECK:    [[TMP17:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 2
+// CHECK:    store ptr [[PS]], ptr [[TMP17]], align 8
+// CHECK:    [[TMP18:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 2
+// CHECK:    store ptr [[PS]], ptr [[TMP18]], align 8
+// CHECK:    [[TMP19:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 2
+// CHECK:    store ptr null, ptr [[TMP19]], align 8
+// CHECK:    [[TMP20:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 3
+// CHECK:    store ptr [[TMP3]], ptr [[TMP20]], align 8
+// CHECK:    [[TMP21:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 3
+// CHECK:    store ptr [[X]], ptr [[TMP21]], align 8
+// CHECK:    [[TMP22:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 3
+// CHECK:    store ptr null, ptr [[TMP22]], align 8
+// CHECK:    [[TMP23:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK:    [[TMP24:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK:    [[TMP25:%.*]] = getelementptr inbounds [4 x i64], ptr [[DOTOFFLOAD_SIZES]], i32 0, i32 0
+// CHECK:    [[TMP26:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], ptr [[KERNEL_ARGS:%.*]], i32 0, i32 0
+//
+//
+// CHECK-LABEL: define internal void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z2f5v_l54(
+// CHECK-SAME: ptr noundef [[PS:%.*]]) #[[ATTR1]] {
+// CHECK:  [[ENTRY:.*:]]
+// CHECK:    store ptr [[PS]], ptr [[PS_ADDR:%.*]], align 8
+// CHECK:    [[TMP0:%.*]] = load ptr, ptr [[PS_ADDR]], align 8
+// CHECK:    [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[TMP0]], i32 0, i32 1
+// CHECK:    store i32 9, ptr [[Y]], align 4
+// CHECK:    ret void
+//
diff --git a/clang/test/Options/Gis.hlsl b/clang/test/Options/Gis.hlsl
new file mode 100644
index 0000000000000..a124b7807eaa1
--- /dev/null
+++ b/clang/test/Options/Gis.hlsl
@@ -0,0 +1,13 @@
+// RUN: %clang_dxc -T lib_6_4 -Gis %s 2>&1 -### | FileCheck -check-prefix=Gis %s
+// RUN: %clang_dxc -T lib_6_4 %s 2>&1 -### | FileCheck -check-prefix=NO_Gis %s
+
+// Gis: "-ffp-contract=off" "-frounding-math" "-ffp-exception-behavior=strict" "-complex-range=full"
+// assert expected floating point options are present
+// NO_Gis-NOT: "-ffp-contract=off" 
+// NO_Gis-NOT: "-frounding-math" 
+// NO_Gis-NOT: "-ffp-exception-behavior=strict" 
+// NO_Gis-NOT: "-complex-range=full"
+float4 main(float4 a : A) : SV_TARGET
+{
+  return -a.yxxx;
+}
diff --git a/clang/test/Parser/asm.c b/clang/test/Parser/asm.c
index ecc0ef95a25b8..0eb89ea72c1c9 100644
--- a/clang/test/Parser/asm.c
+++ b/clang/test/Parser/asm.c
@@ -14,7 +14,7 @@ void f2(void) {
   asm("foo" : : "r" (b)); // expected-error {{use of undeclared identifier 'b'}} 
 
   [[]] asm("");
-  [[gnu::deprecated]] asm(""); // expected-warning {{'deprecated' attribute ignored}}
+  [[gnu::deprecated]] asm(""); // expected-warning {{'gnu::deprecated' attribute ignored}}
 }
 
 void a(void) __asm__(""); // expected-error {{cannot use an empty string literal in 'asm'}}
diff --git a/clang/test/Parser/asm.cpp b/clang/test/Parser/asm.cpp
index 4d590992f6b09..cf23b6f42a077 100644
--- a/clang/test/Parser/asm.cpp
+++ b/clang/test/Parser/asm.cpp
@@ -10,7 +10,7 @@ int foo6 asm ("" L"bar7"); // expected-error {{cannot use wide string literal in
 
 void f() {
   [[]] asm("");
-  [[gnu::deprecated]] asm(""); // expected-warning {{'deprecated' attribute ignored}}
+  [[gnu::deprecated]] asm(""); // expected-warning {{'gnu::deprecated' attribute ignored}}
 }
 
 
diff --git a/clang/test/Parser/atomic-options.hip b/clang/test/Parser/atomic-options.hip
index 4deb9677766c6..f4928cd02a0c5 100644
--- a/clang/test/Parser/atomic-options.hip
+++ b/clang/test/Parser/atomic-options.hip
@@ -33,13 +33,13 @@ __device__ __host__ void test_invalid_format(float *a) {
   }
 }
 
-[[clang::atomic(no_remote_memory)]] // expected-error {{'atomic' attribute cannot be applied to a declaration}}
+[[clang::atomic(no_remote_memory)]] // expected-error {{'clang::atomic' attribute cannot be applied to a declaration}}
 __device__ __host__ void test_not_compound_stmt(float *a) {
     __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
 }
 
 __device__ __host__ void test_quoted(float *a) {
-  [[clang::atomic("no_remote_memory", "remote_memory")]] { // expected-error {{'atomic' attribute requires an identifier}}
+  [[clang::atomic("no_remote_memory", "remote_memory")]] { // expected-error {{'clang::atomic' attribute requires an identifier}}
     __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
   }
 }
diff --git a/clang/test/Parser/c23-typeof.m b/clang/test/Parser/c23-typeof.m
new file mode 100644
index 0000000000000..e6a8dc918dbb1
--- /dev/null
+++ b/clang/test/Parser/c23-typeof.m
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -verify -std=c23 -fblocks -Wno-unused %s
+// expected-no-diagnostics
+
+void f() {
+  ^ typeof((void)0) {}; // Ok
+  ^ typeof(void) {};    // Ok
+}
diff --git a/clang/test/Parser/c2x-typeof.c b/clang/test/Parser/c2x-typeof.c
index 9c836dfa6d823..245c127d71e03 100644
--- a/clang/test/Parser/c2x-typeof.c
+++ b/clang/test/Parser/c2x-typeof.c
@@ -42,3 +42,13 @@ _Static_assert(__builtin_offsetof(typeof(s), i) == 0);
 _Static_assert(__builtin_offsetof(typeof_unqual(struct S), i) == 0);
 _Static_assert(__builtin_offsetof(typeof_unqual(s), i) == 0);
 
+// Show that typeof and typeof_unqual can be used in the underlying type of an
+// enumeration even when given the type form. Note, this can look like a
+// compound literal expression, which caused GH146351.
+enum E3 : typeof(int) { ThirdZero }; // (int) {}; is not a compound literal!
+enum E4 : typeof_unqual(int) { FourthZero }; // Same here
+
+// Ensure that this invalid construct is diagnosed instead of being treated
+// as typeof((int){ 0 }).
+typeof(int) { 0 } x; // expected-error {{a type specifier is required for all declarations}} \
+                        expected-error {{expected identifier or '('}}
diff --git a/clang/test/Parser/cxx0x-attributes.cpp b/clang/test/Parser/cxx0x-attributes.cpp
index 372a373a49ec5..417c103f60d97 100644
--- a/clang/test/Parser/cxx0x-attributes.cpp
+++ b/clang/test/Parser/cxx0x-attributes.cpp
@@ -292,7 +292,7 @@ template <int... Is> void variadic_nttp() {
   void bar [[noreturn...]] ();                        // expected-error {{attribute 'noreturn' cannot be used as an attribute pack}}
   void baz [[clang::no_sanitize(Is...)]] ();          // expected-error {{expected string literal as argument of 'no_sanitize' attribute}}
   void bor [[clang::annotate("A", "V" ...)]] ();      // expected-error {{pack expansion does not contain any unexpanded parameter packs}}
-  void bir [[clang::annotate("B", {1, 2, 3, 4})]] (); // expected-error {{'annotate' attribute requires parameter 1 to be a constant expression}} expected-note {{subexpression not valid in a constant expression}}
+  void bir [[clang::annotate("B", {1, 2, 3, 4})]] (); // expected-error {{'clang::annotate' attribute requires parameter 1 to be a constant expression}} expected-note {{subexpression not valid in a constant expression}}
   void boo [[unknown::foo(Is...)]] ();                // expected-warning {{unknown attribute 'unknown::foo' ignored}}
   void faz [[clang::annotate("C", (Is + ...))]] ();   // expected-warning {{pack fold expression is a C++17 extension}}
   void far [[clang::annotate("D", Is...)]] ();
@@ -306,8 +306,8 @@ void bar () {
   // FIXME: GCC accepts [[gnu::noreturn]] on a lambda, even though it appertains
   // to the operator()'s type, and GCC does not otherwise accept attributes
   // applied to types. Use that to test this.
-  [] () [[gnu::noreturn]] { return; } (); // expected-warning {{attribute 'noreturn' ignored}} FIXME-error {{should not return}}
-  [] () [[gnu::noreturn]] { throw; } (); // expected-warning {{attribute 'noreturn' ignored}}
+  [] () [[gnu::noreturn]] { return; } (); // expected-warning {{attribute 'gnu::noreturn' ignored}} FIXME-error {{should not return}}
+  [] () [[gnu::noreturn]] { throw; } (); // expected-warning {{attribute 'gnu::noreturn' ignored}}
   new int[42][[]][5][[]]{};
 }
 
@@ -350,16 +350,16 @@ namespace arguments {
 }
 
 // Forbid attributes on decl specifiers.
-unsigned [[gnu::used]] static int [[gnu::unused]] v1; // expected-error {{'unused' attribute cannot be applied to types}} \
+unsigned [[gnu::used]] static int [[gnu::unused]] v1; // expected-error {{'gnu::unused' attribute cannot be applied to types}} \
            expected-error {{an attribute list cannot appear here}}
-typedef [[gnu::used]] unsigned long [[gnu::unused]] v2; // expected-error {{'unused' attribute cannot be applied to types}} \
+typedef [[gnu::used]] unsigned long [[gnu::unused]] v2; // expected-error {{'gnu::unused' attribute cannot be applied to types}} \
           expected-error {{an attribute list cannot appear here}}
 int [[carries_dependency]] foo(int [[carries_dependency]] x); // expected-error 2{{'carries_dependency' attribute cannot be applied to types}}
 
 // Forbid [[gnu::...]] attributes on declarator chunks.
-int *[[gnu::unused]] v3; // expected-warning {{attribute 'unused' ignored}}
-int v4[2][[gnu::unused]]; // expected-warning {{attribute 'unused' ignored}}
-int v5()[[gnu::unused]]; // expected-warning {{attribute 'unused' ignored}}
+int *[[gnu::unused]] v3; // expected-warning {{attribute 'gnu::unused' ignored}}
+int v4[2][[gnu::unused]]; // expected-warning {{attribute 'gnu::unused' ignored}}
+int v5()[[gnu::unused]]; // expected-warning {{attribute 'gnu::unused' ignored}}
 
 [[attribute_declaration]]; // expected-warning {{unknown attribute 'attribute_declaration' ignored}}
 [[noreturn]]; // expected-error {{'noreturn' attribute only applies to functions}}
diff --git a/clang/test/Parser/cxx0x-decl.cpp b/clang/test/Parser/cxx0x-decl.cpp
index 6d4f16c6f533e..69a8d8a46557d 100644
--- a/clang/test/Parser/cxx0x-decl.cpp
+++ b/clang/test/Parser/cxx0x-decl.cpp
@@ -147,7 +147,7 @@ namespace UsingDeclAttrs {
   static_assert(alignof(T) == 1, "");
 
   using [[gnu::aligned(1)]] T = int; // expected-error {{an attribute list cannot appear here}}
-  using T = int [[gnu::aligned(1)]]; // expected-error {{'aligned' attribute cannot be applied to types}}
+  using T = int [[gnu::aligned(1)]]; // expected-error {{'gnu::aligned' attribute cannot be applied to types}}
 }
 
 namespace DuplicateSpecifier {
diff --git a/clang/test/Parser/pragma-attribute.cpp b/clang/test/Parser/pragma-attribute.cpp
index d5b1f848abd06..eaf1a775c1c88 100644
--- a/clang/test/Parser/pragma-attribute.cpp
+++ b/clang/test/Parser/pragma-attribute.cpp
@@ -158,7 +158,7 @@ _Pragma("clang attribute pop");
 #pragma clang attribute pop
 
 #pragma clang attribute push ([[fallthrough]], apply_to=function) // expected-error {{attribute 'fallthrough' is not supported by '#pragma clang attribute'}}
-#pragma clang attribute push ([[clang::fallthrough]], apply_to=function) // expected-error {{attribute 'fallthrough' is not supported by '#pragma clang attribute'}}
+#pragma clang attribute push ([[clang::fallthrough]], apply_to=function) // expected-error {{attribute 'clang::fallthrough' is not supported by '#pragma clang attribute'}}
 
 #pragma clang attribute push ([[]], apply_to = function) // A noop
 
@@ -182,15 +182,15 @@ _Pragma("clang attribute pop");
 
 #pragma clang attribute push([[clang::uninitialized]], apply_to = variable(is_local))
 #pragma clang attribute pop
-#pragma clang attribute push([[clang::uninitialized]], apply_to = function) // expected-error {{attribute 'uninitialized' cannot be applied to 'function'}}
+#pragma clang attribute push([[clang::uninitialized]], apply_to = function) // expected-error {{attribute 'clang::uninitialized' cannot be applied to 'function'}}
 #pragma clang attribute pop
-#pragma clang attribute push([[clang::uninitialized]], apply_to = variable) // expected-error {{attribute 'uninitialized' cannot be applied to 'variable'}}
+#pragma clang attribute push([[clang::uninitialized]], apply_to = variable) // expected-error {{attribute 'clang::uninitialized' cannot be applied to 'variable'}}
 #pragma clang attribute pop
-#pragma clang attribute push([[clang::uninitialized]], apply_to = variable(is_thread_local)) // expected-error {{attribute 'uninitialized' cannot be applied to 'variable(is_thread_local)'}}
+#pragma clang attribute push([[clang::uninitialized]], apply_to = variable(is_thread_local)) // expected-error {{attribute 'clang::uninitialized' cannot be applied to 'variable(is_thread_local)'}}
 #pragma clang attribute pop
-#pragma clang attribute push([[clang::uninitialized]], apply_to = variable(is_global)) // expected-error {{attribute 'uninitialized' cannot be applied to 'variable(is_global)'}}
+#pragma clang attribute push([[clang::uninitialized]], apply_to = variable(is_global)) // expected-error {{attribute 'clang::uninitialized' cannot be applied to 'variable(is_global)'}}
 #pragma clang attribute pop
-#pragma clang attribute push([[clang::uninitialized]], apply_to = any(variable(is_parameter), variable(unless(is_parameter)))) // expected-error {{attribute 'uninitialized' cannot be applied to 'variable(is_parameter)', and 'variable(unless(is_parameter))'}}
+#pragma clang attribute push([[clang::uninitialized]], apply_to = any(variable(is_parameter), variable(unless(is_parameter)))) // expected-error {{attribute 'clang::uninitialized' cannot be applied to 'variable(is_parameter)', and 'variable(unless(is_parameter))'}}
 #pragma clang attribute pop
 // We're allowed to apply attributes to subsets of allowed subjects.
 #pragma clang attribute push([[clang::no_destroy]], apply_to = variable)
diff --git a/clang/test/Parser/static_assert.c b/clang/test/Parser/static_assert.c
index e9d7d3f0833f3..1b9b6b676e3e6 100644
--- a/clang/test/Parser/static_assert.c
+++ b/clang/test/Parser/static_assert.c
@@ -20,7 +20,6 @@ static_assert(1, ""); // c17-warning {{'static_assert' is a keyword in C23}} \
                       // c17-error {{expected ')'}} \
                       // c17-note {{to match this '('}} \
                       // c17-error {{type specifier missing, defaults to 'int'; ISO C99 and later do not support implicit int}} \
-                      // c17-ms-warning {{use of 'static_assert' without inclusion of <assert.h> is a Microsoft extension}}
 
 #endif
 
diff --git a/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
index b2d492d95945c..5c8fe778e4abd 100644
--- a/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
@@ -2,10 +2,10 @@
 
 typedef vector<float, 4> float4;
 
-// expected-error@+1{{'contained_type' attribute cannot be applied to a declaration}}
+// expected-error@+1{{'hlsl::contained_type' attribute cannot be applied to a declaration}}
 [[hlsl::contained_type(float4)]] __hlsl_resource_t h1;
 
-// expected-error@+1{{'contained_type' attribute takes one argument}}
+// expected-error@+1{{'hlsl::contained_type' attribute takes one argument}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type()]] h3;
 
 // expected-error@+1{{expected a type}}
@@ -17,12 +17,12 @@ __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(a)]] h5;
 // expected-error@+1{{expected a type}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type("b", c)]] h6;
 
-// expected-warning@+1{{attribute 'contained_type' is already applied}}
+// expected-warning@+1{{attribute 'hlsl::contained_type' is already applied}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(float)]] h7;
 
-// expected-warning@+1{{attribute 'contained_type' is already applied with different arguments}}
+// expected-warning@+1{{attribute 'hlsl::contained_type' is already applied with different arguments}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(int)]] h8;
 
-// expected-error@+2{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-// expected-error@+1{{attribute 'contained_type' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'hlsl::contained_type' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 float [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] res5;
diff --git a/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
index 9920b65561527..f0333a5c74157 100644
--- a/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
@@ -1,20 +1,20 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -o - %s -verify
 
-// expected-error@+1{{'is_rov' attribute cannot be applied to a declaration}}
+// expected-error@+1{{'hlsl::is_rov' attribute cannot be applied to a declaration}}
 [[hlsl::is_rov]] __hlsl_resource_t res0;
 
 // expected-error@+1{{HLSL resource needs to have [[hlsl::resource_class()]] attribute}}
 __hlsl_resource_t [[hlsl::is_rov]] res1;
 
-// expected-error@+1{{'is_rov' attribute takes no arguments}}
+// expected-error@+1{{'hlsl::is_rov' attribute takes no arguments}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(3)]] res2;
 
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(gibberish)]] res3;
 
-// expected-warning@+1{{attribute 'is_rov' is already applied}}
+// expected-warning@+1{{attribute 'hlsl::is_rov' is already applied}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] [[hlsl::is_rov]] res4;
 
-// expected-error@+2{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-// expected-error@+1{{attribute 'is_rov' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'hlsl::is_rov' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 float [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] res5;
diff --git a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
index a638d1d3e156c..9f4dfbe8d2c71 100644
--- a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
@@ -1,17 +1,17 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -o - %s -verify
 
-// expected-error@+1{{'raw_buffer' attribute cannot be applied to a declaration}}
+// expected-error@+1{{'hlsl::raw_buffer' attribute cannot be applied to a declaration}}
 [[hlsl::raw_buffer]] __hlsl_resource_t res0;
 
-// expected-error@+1{{'raw_buffer' attribute takes no arguments}}
+// expected-error@+1{{'hlsl::raw_buffer' attribute takes no arguments}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(3)]] res2;
 
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(gibberish)]] res3;
 
-// expected-warning@+1{{attribute 'raw_buffer' is already applied}}
+// expected-warning@+1{{attribute 'hlsl::raw_buffer' is already applied}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] [[hlsl::raw_buffer]] res4;
 
-// expected-error@+2{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-// expected-error@+1{{attribute 'raw_buffer' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'hlsl::raw_buffer' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 float [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] res5;
diff --git a/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl
index 63e39daff949b..e4a56586c3911 100644
--- a/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl
@@ -1,22 +1,22 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -o - %s -verify
 
-// expected-error@+1{{'resource_class' attribute cannot be applied to a declaration}}
+// expected-error@+1{{'hlsl::resource_class' attribute cannot be applied to a declaration}}
 [[hlsl::resource_class(UAV)]] __hlsl_resource_t e0;
 
-// expected-error@+1{{'resource_class' attribute takes one argument}}
+// expected-error@+1{{'hlsl::resource_class' attribute takes one argument}}
 __hlsl_resource_t [[hlsl::resource_class()]] e1;
 
 // expected-warning@+1{{ResourceClass attribute argument not supported: gibberish}}
 __hlsl_resource_t [[hlsl::resource_class(gibberish)]] e2;
 
-// expected-warning@+1{{attribute 'resource_class' is already applied with different arguments}}
+// expected-warning@+1{{attribute 'hlsl::resource_class' is already applied with different arguments}}
 __hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::resource_class(UAV)]] e3;
 
-// expected-warning@+1{{attribute 'resource_class' is already applied}}
+// expected-warning@+1{{attribute 'hlsl::resource_class' is already applied}}
 __hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::resource_class(SRV)]] e4;
 
-// expected-error@+1{{'resource_class' attribute takes one argument}}
+// expected-error@+1{{'hlsl::resource_class' attribute takes one argument}}
 __hlsl_resource_t [[hlsl::resource_class(SRV, "aa")]] e5;
 
-// expected-error@+1{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 float [[hlsl::resource_class(UAV)]] e6;
diff --git a/clang/test/ParserOpenACC/parse-cache-construct.c b/clang/test/ParserOpenACC/parse-cache-construct.c
index d52a5e4ea3441..76d6b32a574e8 100644
--- a/clang/test/ParserOpenACC/parse-cache-construct.c
+++ b/clang/test/ParserOpenACC/parse-cache-construct.c
@@ -46,10 +46,11 @@ void func() {
     #pragma acc cache(invalid
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected ')'}}
     // expected-note@+2{{to match this '('}}
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(ArrayPtr
   }
 
@@ -58,11 +59,13 @@ void func() {
     #pragma acc cache(invalid)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+5{{expected expression}}
     // expected-error@+4{{expected ']'}}
@@ -72,6 +75,7 @@ void func() {
     #pragma acc cache(ArrayPtr[
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ']'}}
@@ -79,6 +83,7 @@ void func() {
     #pragma acc cache(ArrayPtr[, 5)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ']'}}
@@ -86,10 +91,12 @@ void func() {
     #pragma acc cache(Array[)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(Array[*readonly])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+5{{expected expression}}
     // expected-error@+4{{expected ']'}}
@@ -99,49 +106,59 @@ void func() {
     #pragma acc cache(Array[*readonly:
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+2{{invalid tag 'devnum' on 'cache' directive}}
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(devnum:ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+2{{invalid tag 'invalid' on 'cache' directive}}
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(invalid:ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly:ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(readonly:ArrayPtr[5:1])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(readonly:ArrayPtr[5:*readonly])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly:ArrayPtr[5:*readonly], Array)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(readonly:ArrayPtr[5:*readonly], Array[*readonly:3])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    #pragma acc cache(readonly:ArrayPtr[5 + i:*readonly], Array[*readonly + i:3])
+    #pragma acc cache(readonly:ArrayPtr[5 + 0:*readonly], Array[*readonly + 0:3])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ')'}}
@@ -149,26 +166,31 @@ void func() {
     #pragma acc cache(readonly:ArrayPtr[5:*readonly],
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{expected expression}}
     #pragma acc cache(readonly:ArrayPtr[5:*readonly],)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
     #pragma acc cache(readonly:ArrayPtr[5,6:*readonly])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
     #pragma acc cache(readonly:ArrayPtr[5:3, *readonly], ArrayPtr[0])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly:s.foo)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
     #pragma acc cache(readonly:s.Array[1,2])
diff --git a/clang/test/ParserOpenACC/parse-cache-construct.cpp b/clang/test/ParserOpenACC/parse-cache-construct.cpp
index 948f2e30f149c..fe794707bb7da 100644
--- a/clang/test/ParserOpenACC/parse-cache-construct.cpp
+++ b/clang/test/ParserOpenACC/parse-cache-construct.cpp
@@ -8,28 +8,34 @@ char *getArrayPtr();
 template<typename T, int I>
 void func() {
   char *ArrayPtr = getArrayPtr();
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
-    #pragma acc cache(ArrayPtr[T::value + I:I + 3], T::array[(i + T::value, 2): 4])
+    #pragma acc cache(ArrayPtr[T::value + I:I + 3], T::array[(T::value, 2): 2])
   }
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(NS::NSArray[NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(NS::NSArray[NS::NSInt : NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{use of undeclared identifier 'NSArray'}}
     #pragma acc cache(NSArray[NS::NSInt : NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{use of undeclared identifier 'NSInt'}}
     #pragma acc cache(NS::NSArray[NSInt : NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{use of undeclared identifier 'NSInt'}}
     #pragma acc cache(NS::NSArray[NS::NSInt : NSInt])
@@ -53,43 +59,53 @@ struct HasMembersArray {
 void use() {
 
   Members s;
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(s.array[s.value])
   }
   HasMembersArray Arrs;
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(Arrs.MemArr[3].array[4])
   }
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(Arrs.MemArr[3].array[1:4])
   }
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[2:1].array[1:4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[2:1].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+2{{expected ']'}}
     // expected-note@+1{{to match this '['}}
     #pragma acc cache(Arrs.MemArr[3:4:].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[:].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{expected unqualified-id}}
     #pragma acc cache(Arrs.MemArr[::].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ']'}}
     // expected-note@+1{{to match this '['}}
     #pragma acc cache(Arrs.MemArr[: :].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[3:].array[4])
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index fd83e4b689a2a..d7201207db572 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -316,7 +316,7 @@
 
 // ================== Check whether -mtune accepts mixed-case features.
 // RUN: %clang -target aarch64 -mtune=CYCLONE -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MTUNE-CYCLONE %s
-// CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a"
+// CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a"
 
 // RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
 // RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
@@ -342,12 +342,12 @@
 // RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
 // RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
 // RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
-// CHECK-MCPU-APPLE-A7: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A10: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lor" "-target-feature" "+neon" "-target-feature" "+pan" "-target-feature" "+perfmon" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+vh"
-// CHECK-MCPU-APPLE-A11: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"{{.*}}"-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A12: "-cc1"{{.*}} "-triple" "aarch64"{{.*}} "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A7: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A10: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lor" "-target-feature" "+neon" "-target-feature" "+pan" "-target-feature" "+perfmon" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+vh"
+// CHECK-MCPU-APPLE-A11: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"{{.*}} "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A12: "-cc1"{{.*}} "-triple" "aarch64"{{.*}} "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
 // CHECK-MCPU-A34: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A13: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sha3"
+// CHECK-MCPU-APPLE-A13: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sha3"
 // CHECK-MCPU-A35: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
 // CHECK-MCPU-A53: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
 // CHECK-MCPU-A57: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
@@ -362,10 +362,10 @@
 // CHECK-MCPU-CARMEL: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
 
 // RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
-// CHECK-ARCH-ARM64: "-target-cpu" "apple-m1" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+altnzcv" "-target-feature" "+ccdp" "-target-feature" "+ccpp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+flagm" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fptoint" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+sha2" "-target-feature" "+sha3" "-target-feature" "+specrestrict" "-target-feature" "+ssbs"
+// CHECK-ARCH-ARM64: "-target-cpu" "apple-m1" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+altnzcv" "-target-feature" "+ccdp" "-target-feature" "+ccpp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+flagm" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fptoint" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+sha2" "-target-feature" "+sha3" "-target-feature" "+specrestrict" "-target-feature" "+ssbs"
 
 // RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
-// CHECK-ARCH-ARM64_32: "-target-cpu" "apple-s4" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-ARCH-ARM64_32: "-target-cpu" "apple-s4" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
 
 // RUN: %clang -target aarch64 -march=armv8-a+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
 // RUN: %clang -target aarch64 -march=armv8-a+nofp+nosimd+nocrc+nocrypto+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
diff --git a/clang/test/Preprocessor/arm-target-features.c b/clang/test/Preprocessor/arm-target-features.c
index 0f9063a27371e..fc37bebcb8330 100644
--- a/clang/test/Preprocessor/arm-target-features.c
+++ b/clang/test/Preprocessor/arm-target-features.c
@@ -1049,3 +1049,8 @@
 // CHECK-NOSIMD-NOT: #define __ARM_NEON 1
 // CHECK-NOSIMD-NOT: #define __ARM_NEON_FP 0x6
 // CHECK-NOSIMD-NOT: #define __ARM_NEON__ 1
+
+// Check that disabling SVE, as set by armv9-a, does not define __ARM_FEATURE_SVE* macros.
+// RUN:  %clang -target aarch64 -march=armv9-a+nosve -x c -E -dM -o - %s | FileCheck -check-prefix=CHECK-NOSVE %s
+// CHECK-NOSVE-NOT: #define __ARM_FEATURE_SVE 1
+// CHECK-NOSVE-NOT: #define __ARM_FEATURE_SVE2 1
diff --git a/clang/test/Preprocessor/riscv-target-features-andes.c b/clang/test/Preprocessor/riscv-target-features-andes.c
index c66d4427b5cf2..083deb02a2679 100644
--- a/clang/test/Preprocessor/riscv-target-features-andes.c
+++ b/clang/test/Preprocessor/riscv-target-features-andes.c
@@ -4,6 +4,8 @@
 // RUN:   -o - | FileCheck %s
 
 // CHECK-NOT: __riscv_xandesperf {{.*$}}
+// CHECK-NOT: __riscv_xandesvbfhcvt {{.*$}}
+// CHECK-NOT: __riscv_xandesvsintload {{.*$}}
 // CHECK-NOT: __riscv_xandesvpackfph {{.*$}}
 // CHECK-NOT: __riscv_xandesvdot {{.*$}}
 
@@ -23,6 +25,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVBFHCVT %s
 // CHECK-XANDESVBFHCVT: __riscv_xandesvbfhcvt  5000000{{$}}
 
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32i_xandesvsintload -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVSINTLOAD %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64i_xandesvsintload -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVSINTLOAD %s
+// CHECK-XANDESVSINTLOAD: __riscv_xandesvsintload  5000000{{$}}
+
 // RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32i_xandesvpackfph -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVPACKFPH %s
diff --git a/clang/test/Preprocessor/static_assert.c b/clang/test/Preprocessor/static_assert.c
deleted file mode 100644
index 7fa9975e1d468..0000000000000
--- a/clang/test/Preprocessor/static_assert.c
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: %clang_cc1 -E -dM %s | FileCheck --strict-whitespace --check-prefix=NOMS %s
-// RUN: %clang_cc1 -fms-compatibility -E -dM %s | FileCheck --strict-whitespace --check-prefix=MS %s
-
-// If the assert macro is defined in MS compatibility mode in C, we
-// automatically inject a macro definition for static_assert. Test that the
-// macro is properly added to the preprocessed output. This allows us to
-// diagonse use of the static_assert keyword when <assert.h> has not been
-// included while still being able to compile preprocessed code.
-#define assert
-
-MS: #define static_assert _Static_assert
-NOMS-NOT: #define static_assert _Static_assert
diff --git a/clang/test/Sema/annotate-type.c b/clang/test/Sema/annotate-type.c
index bf1632d302e55..0a0a300785337 100644
--- a/clang/test/Sema/annotate-type.c
+++ b/clang/test/Sema/annotate-type.c
@@ -18,25 +18,25 @@ void foo(float *[[clang::annotate_type("foo")]] a) {
   int *__attribute__((annotate_type("bar"))) y2; // expected-warning {{unknown attribute 'annotate_type' ignored}}
 
   // Various error cases
-  [[clang::annotate_type("bar")]] int *z1; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  int *z2 [[clang::annotate_type("bar")]]; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  [[clang::annotate_type("bar")]]; // expected-error {{'annotate_type' attribute cannot be applied to a statement}}
+  [[clang::annotate_type("bar")]] int *z1; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  int *z2 [[clang::annotate_type("bar")]]; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  [[clang::annotate_type("bar")]]; // expected-error {{'clang::annotate_type' attribute cannot be applied to a statement}}
   int *[[clang::annotate_type(1)]] z3; // expected-error {{expected string literal as argument of 'annotate_type' attribute}}
-  int *[[clang::annotate_type()]] z4; // expected-error {{'annotate_type' attribute takes at least 1 argument}}
-  int *[[clang::annotate_type]] z5; // expected-error {{'annotate_type' attribute takes at least 1 argument}}
+  int *[[clang::annotate_type()]] z4; // expected-error {{'clang::annotate_type' attribute takes at least 1 argument}}
+  int *[[clang::annotate_type]] z5; // expected-error {{'clang::annotate_type' attribute takes at least 1 argument}}
   int *[[clang::annotate_type(some_function())]] z6; // expected-error {{expected string literal as argument of 'annotate_type' attribute}}
-  int *[[clang::annotate_type("bar", some_function())]] z7; // expected-error {{'annotate_type' attribute requires parameter 1 to be a constant expression}} expected-note{{subexpression not valid in a constant expression}}
-  int *[[clang::annotate_type("bar", z7)]] z8; // expected-error {{'annotate_type' attribute requires parameter 1 to be a constant expression}} expected-note{{subexpression not valid in a constant expression}}
+  int *[[clang::annotate_type("bar", some_function())]] z7; // expected-error {{'clang::annotate_type' attribute requires parameter 1 to be a constant expression}} expected-note{{subexpression not valid in a constant expression}}
+  int *[[clang::annotate_type("bar", z7)]] z8; // expected-error {{'clang::annotate_type' attribute requires parameter 1 to be a constant expression}} expected-note{{subexpression not valid in a constant expression}}
   int *[[clang::annotate_type("bar", int)]] z9; // expected-error {{expected expression}}
 }
 // More error cases: Prohibit adding the attribute to declarations.
 // Different declarations hit different code paths, so they need separate tests.
-[[clang::annotate_type("bar")]] int *global; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-[[clang::annotate_type("bar")]] void annotated_function(); // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-void g([[clang::annotate_type("bar")]] int); // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-struct [[clang::annotate_type("foo")]] S; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-struct [[clang::annotate_type("foo")]] S{ // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  [[clang::annotate_type("foo")]] int member; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+[[clang::annotate_type("bar")]] int *global; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+[[clang::annotate_type("bar")]] void annotated_function(); // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+void g([[clang::annotate_type("bar")]] int); // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+struct [[clang::annotate_type("foo")]] S; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+struct [[clang::annotate_type("foo")]] S{ // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  [[clang::annotate_type("foo")]] int member; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
   [[clang::annotate_type("foo")]] union { // expected-error {{an attribute list cannot appear here}}
     int i;
     float f;
diff --git a/clang/test/Sema/annotate.c b/clang/test/Sema/annotate.c
index f2ef08d637897..1e31952b3e007 100644
--- a/clang/test/Sema/annotate.c
+++ b/clang/test/Sema/annotate.c
@@ -16,6 +16,6 @@ void __attribute__((annotate("foo"))) foo(float *a) {
   float b = __builtin_annotation(*a, "foo"); // expected-error {{first argument to __builtin_annotation must be an integer}}
 
   __attribute__((annotate())) int c; // expected-error {{'annotate' attribute takes at least 1 argument}}
-  [[clang::annotate()]] int c2;      // expected-error {{'annotate' attribute takes at least 1 argument}}
-  [[clang::annotate()]] c2 += 1;     // expected-error {{'annotate' attribute takes at least 1 argument}}
+  [[clang::annotate()]] int c2;      // expected-error {{'clang::annotate' attribute takes at least 1 argument}}
+  [[clang::annotate()]] c2 += 1;     // expected-error {{'clang::annotate' attribute takes at least 1 argument}}
 }
diff --git a/clang/test/Sema/assume.c b/clang/test/Sema/assume.c
index 07a5490addfaf..6b922975891d7 100644
--- a/clang/test/Sema/assume.c
+++ b/clang/test/Sema/assume.c
@@ -19,7 +19,7 @@ void test(int n) {
   __attribute__((assume(n++))); // expected-warning {{assumption is ignored because it contains (potential) side-effects}}
   [[clang::assume(n++)]];       // expected-warning {{assumption is ignored because it contains (potential) side-effects}}
 
-  [[clang::assume(true)]] int x;       // expected-error {{'assume' attribute cannot be applied to a declaration}}
+  [[clang::assume(true)]] int x;       // expected-error {{'clang::assume' attribute cannot be applied to a declaration}}
   __attribute__((assume(true))) int y; // expected-error {{'assume' attribute cannot be applied to a declaration}}
 }
 
diff --git a/clang/test/Sema/attr-alwaysinline.cpp b/clang/test/Sema/attr-alwaysinline.cpp
index 6b71a0e86bf12..9d8188c5b9488 100644
--- a/clang/test/Sema/attr-alwaysinline.cpp
+++ b/clang/test/Sema/attr-alwaysinline.cpp
@@ -10,22 +10,22 @@ int bar();
 
 void foo() {
   [[clang::always_inline]] bar();
-  [[clang::always_inline(0)]] bar(); // expected-error {{'always_inline' attribute takes no arguments}}
+  [[clang::always_inline(0)]] bar(); // expected-error {{'clang::always_inline' attribute takes no arguments}}
   int x;
-  [[clang::always_inline]] int i = bar();  // expected-warning {{'always_inline' attribute only applies to functions and statements}}
-  [[clang::always_inline]] x = 0;          // expected-warning {{'always_inline' attribute is ignored because there exists no call expression inside the statement}}
-  [[clang::always_inline]] { asm("nop"); } // expected-warning {{'always_inline' attribute is ignored because there exists no call expression inside the statement}}
-  [[clang::always_inline]] label : x = 1;  // expected-warning {{'always_inline' attribute only applies to functions and statements}}
+  [[clang::always_inline]] int i = bar();  // expected-warning {{'clang::always_inline' attribute only applies to functions and statements}}
+  [[clang::always_inline]] x = 0;          // expected-warning {{'clang::always_inline' attribute is ignored because there exists no call expression inside the statement}}
+  [[clang::always_inline]] { asm("nop"); } // expected-warning {{'clang::always_inline' attribute is ignored because there exists no call expression inside the statement}}
+  [[clang::always_inline]] label : x = 1;  // expected-warning {{'clang::always_inline' attribute only applies to functions and statements}}
 
   [[clang::always_inline]] always_inline_fn();
-  [[clang::always_inline]] noinline_fn(); // expected-warning {{statement attribute 'always_inline' has higher precedence than function attribute 'noinline'}}
-  [[clang::always_inline]] flatten_fn();  // expected-warning {{statement attribute 'always_inline' has higher precedence than function attribute 'flatten'}}
+  [[clang::always_inline]] noinline_fn(); // expected-warning {{statement attribute 'clang::always_inline' has higher precedence than function attribute 'noinline'}}
+  [[clang::always_inline]] flatten_fn();  // expected-warning {{statement attribute 'clang::always_inline' has higher precedence than function attribute 'flatten'}}
 
   [[gnu::always_inline]] bar();         // expected-warning {{attribute is ignored on this statement as it only applies to functions; use '[[clang::always_inline]]' on statements}}
   __attribute__((always_inline)) bar(); // expected-warning {{attribute is ignored on this statement as it only applies to functions; use '[[clang::always_inline]]' on statements}}
 }
 
-[[clang::always_inline]] static int i = bar(); // expected-warning {{'always_inline' attribute only applies to functions and statements}}
+[[clang::always_inline]] static int i = bar(); // expected-warning {{'clang::always_inline' attribute only applies to functions and statements}}
 
 // This used to crash the compiler.
 template<int D>
@@ -41,13 +41,13 @@ int non_dependent(int x){return x;} // #NO_DEP
 
 template<int D> [[gnu::noinline]]
 int baz(int x) { // #BAZ
-  // expected-warning@+2{{statement attribute 'always_inline' has higher precedence than function attribute 'noinline'}}
+  // expected-warning@+2{{statement attribute 'clang::always_inline' has higher precedence than function attribute 'noinline'}}
   // expected-note@#NO_DEP{{conflicting attribute is here}}
   [[clang::always_inline]] non_dependent(x);
   if constexpr (D>0) {
-    // expected-warning@+6{{statement attribute 'always_inline' has higher precedence than function attribute 'noinline'}}
+    // expected-warning@+6{{statement attribute 'clang::always_inline' has higher precedence than function attribute 'noinline'}}
     // expected-note@#NO_DEP{{conflicting attribute is here}}
-    // expected-warning@+4 3{{statement attribute 'always_inline' has higher precedence than function attribute 'noinline'}}
+    // expected-warning@+4 3{{statement attribute 'clang::always_inline' has higher precedence than function attribute 'noinline'}}
     // expected-note@#BAZ 3{{conflicting attribute is here}}
     // expected-note@#BAZ_INST 3{{in instantiation}}
     // expected-note@+1 3{{in instantiation}}
@@ -61,9 +61,9 @@ template<int ... D>
 int variadic_baz(int x) {
   // Diagnoses NO_DEP 2x, once during phase 1, the second during instantiation.
   // Dianoses DEP 3x, once per variadic expansion.
-  // expected-warning@+5 2{{statement attribute 'always_inline' has higher precedence than function attribute 'noinline'}}
+  // expected-warning@+5 2{{statement attribute 'clang::always_inline' has higher precedence than function attribute 'noinline'}}
   // expected-note@#NO_DEP 2{{conflicting attribute is here}}
-  // expected-warning@+3 3{{statement attribute 'always_inline' has higher precedence than function attribute 'noinline'}}
+  // expected-warning@+3 3{{statement attribute 'clang::always_inline' has higher precedence than function attribute 'noinline'}}
   // expected-note@#DEP 3{{conflicting attribute is here}}
   // expected-note@#VARIADIC_INST{{in instantiation}}
   [[clang::always_inline]] return non_dependent(x) + (dependent<D>(x) + ...);
diff --git a/clang/test/Sema/attr-arm-sve-vector-bits.c b/clang/test/Sema/attr-arm-sve-vector-bits.c
index d34b13a74df6d..447addb4d5d33 100644
--- a/clang/test/Sema/attr-arm-sve-vector-bits.c
+++ b/clang/test/Sema/attr-arm-sve-vector-bits.c
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=1 -mvscale-max=1 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=2 -mvscale-max=2 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=4 -mvscale-max=4 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=8 -mvscale-max=8 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=16 -mvscale-max=16 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=1 -mvscale-max=1 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=2 -mvscale-max=2 -mvscale-streaming-min=2 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected -mvscale-min=4 -mvscale-max=4 -mvscale-streaming-min=4 -mvscale-streaming-max=4 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=8 -mvscale-max=8 -mvscale-streaming-min=4 -mvscale-streaming-max=8 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=16 -mvscale-max=16 %s
 
 #include <stdint.h>
 
@@ -382,3 +382,24 @@ TEST_INT_OPS(fixed_uint64_t)
 TEST_OPS(fixed_float16_t)
 TEST_OPS(fixed_float32_t)
 TEST_OPS(fixed_float64_t)
+
+// --------------------------------------------------------------------------//
+// Streaming
+__arm_locally_streaming void locally_streaming() {
+  svint8_t t1 = extern_int8; // streamingdifferent-error {{cannot be used in a streaming function}}
+  svbool_t t2 = extern_bool; // streamingdifferent-error {{cannot be used in a streaming function}}
+  void* t3 = extern_int8_ptr;
+}
+void streaming(void) __arm_streaming {
+  svint8_t t1 = extern_int8; // streamingdifferent-error {{cannot be used in a streaming function}}
+  svbool_t t2 = extern_bool; // streamingdifferent-error {{cannot be used in a streaming function}}
+  void* t3 = extern_int8_ptr;
+}
+void streaming_compatible(void) __arm_streaming_compatible {
+  svint8_t t1 = extern_int8; // streamingdifferent-error {{cannot be used in a streaming-compatible function}} \
+                             // streamingdifferent-error {{initializing}}
+  svbool_t t2 = extern_bool; // streamingdifferent-error {{cannot be used in a streaming-compatible function}} \
+                             // streamingdifferent-error {{initializing}}
+  void* t3 = extern_int8_ptr;
+}
+__arm_locally_streaming void locally_streaming_arg(fixed_int8_t x) {} // streamingdifferent-error {{cannot be used in a streaming function}}
diff --git a/clang/test/Sema/attr-enforce-tcb-errors.cpp b/clang/test/Sema/attr-enforce-tcb-errors.cpp
index b5effe2fe5180..732636eddc44d 100644
--- a/clang/test/Sema/attr-enforce-tcb-errors.cpp
+++ b/clang/test/Sema/attr-enforce-tcb-errors.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 
-[[clang::enforce_tcb("oops")]] int wrong_subject_type; // expected-warning{{'enforce_tcb' attribute only applies to functions}}
+[[clang::enforce_tcb("oops")]] int wrong_subject_type; // expected-warning{{'clang::enforce_tcb' attribute only applies to functions}}
 
 void no_arguments() __attribute__((enforce_tcb)); // expected-error{{'enforce_tcb' attribute takes one argument}}
 
@@ -8,7 +8,7 @@ void too_many_arguments() __attribute__((enforce_tcb("test", 12))); // expected-
 
 void wrong_argument_type() __attribute__((enforce_tcb(12))); // expected-error{{expected string literal as argument of 'enforce_tcb' attribute}}
 
-[[clang::enforce_tcb_leaf("oops")]] int wrong_subject_type_leaf; // expected-warning{{'enforce_tcb_leaf' attribute only applies to functions}}
+[[clang::enforce_tcb_leaf("oops")]] int wrong_subject_type_leaf; // expected-warning{{'clang::enforce_tcb_leaf' attribute only applies to functions}}
 
 void no_arguments_leaf() __attribute__((enforce_tcb_leaf)); // expected-error{{'enforce_tcb_leaf' attribute takes one argument}}
 
diff --git a/clang/test/Sema/attr-external-source-symbol.c b/clang/test/Sema/attr-external-source-symbol.c
index 8ecd790f33c8f..03c79eb10cc3a 100644
--- a/clang/test/Sema/attr-external-source-symbol.c
+++ b/clang/test/Sema/attr-external-source-symbol.c
@@ -29,5 +29,5 @@ void fiveClauses2(void);
 
 [[clang::external_source_symbol(generated_declaration)]] void oneClause2(void);
 
-[[clang::external_source_symbol]] // expected-error {{'external_source_symbol' attribute takes at least 1 argument}}
+[[clang::external_source_symbol]] // expected-error {{'clang::external_source_symbol' attribute takes at least 1 argument}}
 void noArguments2(void);
diff --git a/clang/test/Sema/attr-handles.cpp b/clang/test/Sema/attr-handles.cpp
index ff1c1f68dfec8..a82dae9f6e130 100644
--- a/clang/test/Sema/attr-handles.cpp
+++ b/clang/test/Sema/attr-handles.cpp
@@ -17,7 +17,7 @@ int (* __attribute__((acquire_handle("Fuchsia"))) fpt)(char *); // expected-warn
 auto lambdat = [](int handle __attribute__((use_handle("Fuchsia"))))
     __attribute__((acquire_handle("Fuchsia"))) -> int { return 0; };
 int __attribute((acquire_handle("Fuchsia"))) ta; // expected-warning {{'acquire_handle' attribute only applies to functions, typedefs, and parameters}}
-int open(const char *path, int flags, ...) [[clang::acquire_handle]]; // expected-error {{'acquire_handle' attribute takes one argument}}
+int open(const char *path, int flags, ...) [[clang::acquire_handle]]; // expected-error {{'clang::acquire_handle' attribute takes one argument}}
 
 // Typedefs.
 typedef int callback(char *) __attribute__((acquire_handle("Fuchsia")));
diff --git a/clang/test/Sema/attr-likelihood.c b/clang/test/Sema/attr-likelihood.c
index 955216f8fd215..6222b80b65e78 100644
--- a/clang/test/Sema/attr-likelihood.c
+++ b/clang/test/Sema/attr-likelihood.c
@@ -5,7 +5,7 @@ void g(void) {
     [[clang::likely]] {}
 }
 void m(void) {
-  [[clang::likely]] int x = 42; // expected-error {{'likely' attribute cannot be applied to a declaration}}
+  [[clang::likely]] int x = 42; // expected-error {{'clang::likely' attribute cannot be applied to a declaration}}
 
   if (x)
     [[clang::unlikely]] {}
@@ -44,7 +44,7 @@ void m(void) {
     goto lbl;
 
   // FIXME: allow the attribute on the label
-  [[clang::unlikely]] lbl : // expected-error {{'unlikely' attribute cannot be applied to a declaration}}
+  [[clang::unlikely]] lbl : // expected-error {{'clang::unlikely' attribute cannot be applied to a declaration}}
   [[clang::likely]] x = x + 1;
 
   [[clang::likely]]++ x;
diff --git a/clang/test/Sema/attr-mig.cpp b/clang/test/Sema/attr-mig.cpp
index 5dfc43bc1ec88..b5aa87a060155 100644
--- a/clang/test/Sema/attr-mig.cpp
+++ b/clang/test/Sema/attr-mig.cpp
@@ -11,7 +11,7 @@ class MyServer {
   virtual __attribute__((mig_server_routine)) void anotherMethod(); // expected-warning{{'mig_server_routine' attribute only applies to routines that return a kern_return_t}}
   virtual __attribute__((mig_server_routine)) int yetAnotherMethod(); // expected-warning{{'mig_server_routine' attribute only applies to routines that return a kern_return_t}}
   [[clang::mig_server_routine]] virtual IOReturn cppAnnotatedMethod();
-  [[clang::mig_server_routine("arg")]] virtual IOReturn cppAnnotatedMethodWithInvalidArgs(); // expected-error{{'mig_server_routine' attribute takes no arguments}}
+  [[clang::mig_server_routine("arg")]] virtual IOReturn cppAnnotatedMethodWithInvalidArgs(); // expected-error{{'clang::mig_server_routine' attribute takes no arguments}}
   [[clang::mig_server_routine]] virtual int cppInvalidAnnotatedMethod(); // expected-warning{{'mig_server_routine' attribute only applies to routines that return a kern_return_t}}
 };
 
diff --git a/clang/test/Sema/attr-nocf_check.cpp b/clang/test/Sema/attr-nocf_check.cpp
index e306c8ef79cd5..c5586b85dbef0 100644
--- a/clang/test/Sema/attr-nocf_check.cpp
+++ b/clang/test/Sema/attr-nocf_check.cpp
@@ -9,8 +9,8 @@ typedef void (*FuncPointer)(void);
 void testNoCfCheck(){}; //  expected-error {{conflicting types for 'testNoCfCheck'}}
 
 // No variable or parameter declaration
-int [[gnu::nocf_check]] i;                              // expected-error {{'nocf_check' attribute cannot be applied to types}}
-void testNoCfCheckImpl(double i [[gnu::nocf_check]]) {} // expected-warning {{'nocf_check' attribute only applies to functions and function pointers}}
+int [[gnu::nocf_check]] i;                              // expected-error {{'gnu::nocf_check' attribute cannot be applied to types}}
+void testNoCfCheckImpl(double i [[gnu::nocf_check]]) {} // expected-warning {{'gnu::nocf_check' attribute only applies to functions and function pointers}}
 
 // Allow attributed function pointers as well as casting between attributed
 // and non-attributed function pointers.
@@ -20,4 +20,4 @@ void testNoCfCheckMismatch(FuncPointer f) {
 }
 
 // 'nocf_check' Attribute has no parameters.
-[[gnu::nocf_check(1)]] int testNoCfCheckParams(); // expected-error {{'nocf_check' attribute takes no arguments}}
+[[gnu::nocf_check(1)]] int testNoCfCheckParams(); // expected-error {{'gnu::nocf_check' attribute takes no arguments}}
diff --git a/clang/test/Sema/attr-noinline.cpp b/clang/test/Sema/attr-noinline.cpp
index 6da0e873af1b6..d6d48321d6604 100644
--- a/clang/test/Sema/attr-noinline.cpp
+++ b/clang/test/Sema/attr-noinline.cpp
@@ -10,15 +10,15 @@ int bar();
 
 void foo() {
   [[clang::noinline]] bar();
-  [[clang::noinline(0)]] bar(); // expected-error {{'noinline' attribute takes no arguments}}
+  [[clang::noinline(0)]] bar(); // expected-error {{'clang::noinline' attribute takes no arguments}}
   int x;
-  [[clang::noinline]] x = 0; // expected-warning {{'noinline' attribute is ignored because there exists no call expression inside the statement}}
-  [[clang::noinline]] { asm("nop"); } // expected-warning {{'noinline' attribute is ignored because there exists no call expression inside the statement}}
-  [[clang::noinline]] label: x = 1; // expected-warning {{'noinline' attribute only applies to functions and statements}}
+  [[clang::noinline]] x = 0; // expected-warning {{'clang::noinline' attribute is ignored because there exists no call expression inside the statement}}
+  [[clang::noinline]] { asm("nop"); } // expected-warning {{'clang::noinline' attribute is ignored because there exists no call expression inside the statement}}
+  [[clang::noinline]] label: x = 1; // expected-warning {{'clang::noinline' attribute only applies to functions and statements}}
 
 
-  [[clang::noinline]] always_inline_fn(); // expected-warning {{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
-  [[clang::noinline]] flatten_fn(); // expected-warning {{statement attribute 'noinline' has higher precedence than function attribute 'flatten'}}
+  [[clang::noinline]] always_inline_fn(); // expected-warning {{statement attribute 'clang::noinline' has higher precedence than function attribute 'always_inline'}}
+  [[clang::noinline]] flatten_fn(); // expected-warning {{statement attribute 'clang::noinline' has higher precedence than function attribute 'flatten'}}
   [[clang::noinline]] noinline_fn();
 
   [[gnu::noinline]] bar(); // expected-warning {{attribute is ignored on this statement as it only applies to functions; use '[[clang::noinline]]' on statements}}
@@ -27,19 +27,19 @@ void foo() {
 
 void ms_noi_check() {
   [[msvc::noinline]] bar();
-  [[msvc::noinline(0)]] bar(); // expected-error {{'noinline' attribute takes no arguments}}
+  [[msvc::noinline(0)]] bar(); // expected-error {{'msvc::noinline' attribute takes no arguments}}
   int x;
-  [[msvc::noinline]] x = 0; // expected-warning {{'noinline' attribute is ignored because there exists no call expression inside the statement}}
-  [[msvc::noinline]] { asm("nop"); } // expected-warning {{'noinline' attribute is ignored because there exists no call expression inside the statement}}
-  [[msvc::noinline]] label: x = 1; // expected-warning {{'noinline' attribute only applies to functions and statements}}
+  [[msvc::noinline]] x = 0; // expected-warning {{'msvc::noinline' attribute is ignored because there exists no call expression inside the statement}}
+  [[msvc::noinline]] { asm("nop"); } // expected-warning {{'msvc::noinline' attribute is ignored because there exists no call expression inside the statement}}
+  [[msvc::noinline]] label: x = 1; // expected-warning {{'msvc::noinline' attribute only applies to functions and statements}}
 
-  [[msvc::noinline]] always_inline_fn(); // expected-warning {{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
-  [[msvc::noinline]] flatten_fn(); // expected-warning {{statement attribute 'noinline' has higher precedence than function attribute 'flatten'}}
+  [[msvc::noinline]] always_inline_fn(); // expected-warning {{statement attribute 'msvc::noinline' has higher precedence than function attribute 'always_inline'}}
+  [[msvc::noinline]] flatten_fn(); // expected-warning {{statement attribute 'msvc::noinline' has higher precedence than function attribute 'flatten'}}
   [[msvc::noinline]] noinline_fn();
 }
 
-[[clang::noinline]] static int i = bar(); // expected-warning {{'noinline' attribute only applies to functions and statements}}
-[[msvc::noinline]] static int j = bar(); // expected-warning {{'noinline' attribute only applies to functions and statements}}
+[[clang::noinline]] static int i = bar(); // expected-warning {{'clang::noinline' attribute only applies to functions and statements}}
+[[msvc::noinline]] static int j = bar(); // expected-warning {{'msvc::noinline' attribute only applies to functions and statements}}
 
 // This used to crash the compiler.
 template<int D>
@@ -55,13 +55,13 @@ int non_dependent(int x){return x;} // #NO_DEP
 
 template<int D> [[clang::always_inline]]
 int baz(int x) { // #BAZ
-  // expected-warning@+2{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+  // expected-warning@+2{{statement attribute 'clang::noinline' has higher precedence than function attribute 'always_inline'}}
   // expected-note@#NO_DEP{{conflicting attribute is here}}
   [[clang::noinline]] non_dependent(x);
   if constexpr (D>0) {
-    // expected-warning@+6{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+    // expected-warning@+6{{statement attribute 'clang::noinline' has higher precedence than function attribute 'always_inline'}}
     // expected-note@#NO_DEP{{conflicting attribute is here}}
-    // expected-warning@+4 3{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+    // expected-warning@+4 3{{statement attribute 'clang::noinline' has higher precedence than function attribute 'always_inline'}}
     // expected-note@#BAZ 3{{conflicting attribute is here}}
     // expected-note@#BAZ_INST 3{{in instantiation}}
     // expected-note@+1 3{{in instantiation}}
@@ -75,9 +75,9 @@ template<int ... D>
 int variadic_baz(int x) {
   // Diagnoses NO_DEP 2x, once during phase 1, the second during instantiation.
   // Dianoses DEP 3x, once per variadic expansion.
-  // expected-warning@+5 2{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+  // expected-warning@+5 2{{statement attribute 'clang::noinline' has higher precedence than function attribute 'always_inline'}}
   // expected-note@#NO_DEP 2{{conflicting attribute is here}}
-  // expected-warning@+3 3{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+  // expected-warning@+3 3{{statement attribute 'clang::noinline' has higher precedence than function attribute 'always_inline'}}
   // expected-note@#DEP 3{{conflicting attribute is here}}
   // expected-note@#VARIADIC_INST{{in instantiation}}
   [[clang::noinline]] return non_dependent(x) + (dependent<D>(x) + ...);
@@ -85,13 +85,13 @@ int variadic_baz(int x) {
 
 template<int D> [[clang::always_inline]]
 int qux(int x) { // #QUX
-  // expected-warning@+2{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+  // expected-warning@+2{{statement attribute 'msvc::noinline' has higher precedence than function attribute 'always_inline'}}
   // expected-note@#NO_DEP{{conflicting attribute is here}}
   [[msvc::noinline]] non_dependent(x);
   if constexpr (D>0) {
-    // expected-warning@+6{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+    // expected-warning@+6{{statement attribute 'msvc::noinline' has higher precedence than function attribute 'always_inline'}}
     // expected-note@#NO_DEP{{conflicting attribute is here}}
-    // expected-warning@+4 3{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+    // expected-warning@+4 3{{statement attribute 'msvc::noinline' has higher precedence than function attribute 'always_inline'}}
     // expected-note@#QUX 3{{conflicting attribute is here}}
     // expected-note@#QUX_INST 3{{in instantiation}}
     // expected-note@+1 3{{in instantiation}}
@@ -105,9 +105,9 @@ template<int ... D>
 int variadic_qux(int x) {
   // Diagnoses NO_DEP 2x, once during phase 1, the second during instantiation.
   // Dianoses DEP 3x, once per variadic expansion.
-  // expected-warning@+5 2{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+  // expected-warning@+5 2{{statement attribute 'msvc::noinline' has higher precedence than function attribute 'always_inline'}}
   // expected-note@#NO_DEP 2{{conflicting attribute is here}}
-  // expected-warning@+3 3{{statement attribute 'noinline' has higher precedence than function attribute 'always_inline'}}
+  // expected-warning@+3 3{{statement attribute 'msvc::noinline' has higher precedence than function attribute 'always_inline'}}
   // expected-note@#DEP 3{{conflicting attribute is here}}
   // expected-note@#QUX_VARIADIC_INST{{in instantiation}}
   [[msvc::noinline]] return non_dependent(x) + (dependent<D>(x) + ...);
diff --git a/clang/test/Sema/attr-nomerge.cpp b/clang/test/Sema/attr-nomerge.cpp
index 69a535bde76e2..010f8d26310ae 100644
--- a/clang/test/Sema/attr-nomerge.cpp
+++ b/clang/test/Sema/attr-nomerge.cpp
@@ -4,18 +4,18 @@ void bar();
 
 void foo() {
   [[clang::nomerge]] bar();
-  [[clang::nomerge(1, 2)]] bar(); // expected-error {{'nomerge' attribute takes no arguments}}
+  [[clang::nomerge(1, 2)]] bar(); // expected-error {{'clang::nomerge' attribute takes no arguments}}
   int x;
-  [[clang::nomerge]] x = 10; // expected-warning {{'nomerge' attribute is ignored because there exists no call expression inside the statement}}
+  [[clang::nomerge]] x = 10; // expected-warning {{'clang::nomerge' attribute is ignored because there exists no call expression inside the statement}}
 
-  [[clang::nomerge]] label: bar(); // expected-error {{'nomerge' attribute only applies to functions, statements and variables}}
+  [[clang::nomerge]] label: bar(); // expected-error {{'clang::nomerge' attribute only applies to functions, statements and variables}}
 
 }
 
 [[clang::nomerge]] int f();
 
-[[clang::nomerge]] static int i = f(); // expected-warning {{'nomerge' attribute is ignored because 'i' is not a function pointer}}
+[[clang::nomerge]] static int i = f(); // expected-warning {{'clang::nomerge' attribute is ignored because 'i' is not a function pointer}}
 
 [[clang::nomerge]] void (*j)(void);
 
-struct [[clang::nomerge]] buz {}; // expected-error {{'nomerge' attribute only applies to functions, statements and variables}}
+struct [[clang::nomerge]] buz {}; // expected-error {{'clang::nomerge' attribute only applies to functions, statements and variables}}
diff --git a/clang/test/Sema/attr-nonblocking-sema.cpp b/clang/test/Sema/attr-nonblocking-sema.cpp
index 9056f81f5296b..f13cc783dfc33 100644
--- a/clang/test/Sema/attr-nonblocking-sema.cpp
+++ b/clang/test/Sema/attr-nonblocking-sema.cpp
@@ -8,8 +8,8 @@
 // --- ATTRIBUTE SYNTAX: SUBJECTS ---
 
 int nl_var [[clang::nonblocking]]; // expected-warning {{'nonblocking' only applies to function types; type here is 'int'}}
-struct nl_struct {} [[clang::nonblocking]]; // expected-warning {{attribute 'nonblocking' is ignored, place it after "struct" to apply attribute to type declaration}}
-struct [[clang::nonblocking]] nl_struct2 {}; // expected-error {{'nonblocking' attribute cannot be applied to a declaration}}
+struct nl_struct {} [[clang::nonblocking]]; // expected-warning {{attribute 'clang::nonblocking' is ignored, place it after "struct" to apply attribute to type declaration}}
+struct [[clang::nonblocking]] nl_struct2 {}; // expected-error {{'clang::nonblocking' attribute cannot be applied to a declaration}}
 
 // Positive case
 typedef void (*fo)() [[clang::nonblocking]];
@@ -19,10 +19,10 @@ void (*read_me_and_weep(
   [[clang::nonblocking]];
 
 // --- ATTRIBUTE SYNTAX: ARGUMENT COUNT ---
-void nargs_1() [[clang::nonblocking(1, 2)]];  // expected-error {{'nonblocking' attribute takes no more than 1 argument}}
-void nargs_2() [[clang::nonallocating(1, 2)]]; // expected-error {{'nonallocating' attribute takes no more than 1 argument}}
-void nargs_3() [[clang::blocking(1)]]; // expected-error {{'blocking' attribute takes no arguments}}
-void nargs_4() [[clang::allocating(1)]]; // expected-error {{'allocating' attribute takes no arguments}}
+void nargs_1() [[clang::nonblocking(1, 2)]];  // expected-error {{'clang::nonblocking' attribute takes no more than 1 argument}}
+void nargs_2() [[clang::nonallocating(1, 2)]]; // expected-error {{'clang::nonallocating' attribute takes no more than 1 argument}}
+void nargs_3() [[clang::blocking(1)]]; // expected-error {{'clang::blocking' attribute takes no arguments}}
+void nargs_4() [[clang::allocating(1)]]; // expected-error {{'clang::allocating' attribute takes no arguments}}
 
 // --- ATTRIBUTE SYNTAX: COMBINATIONS ---
 // Check invalid combinations of nonblocking/nonallocating attributes
diff --git a/clang/test/Sema/attr-only-in-default-eval.cpp b/clang/test/Sema/attr-only-in-default-eval.cpp
index 510863cf2580a..35e690fdf3efa 100644
--- a/clang/test/Sema/attr-only-in-default-eval.cpp
+++ b/clang/test/Sema/attr-only-in-default-eval.cpp
@@ -6,20 +6,20 @@ using double_t __attribute__((available_only_in_default_eval_method)) = double;
 // expected-error@+1{{'available_only_in_default_eval_method' attribute only applies to typedefs}}
 class  __attribute__((available_only_in_default_eval_method)) C1 {
 };
-// expected-error@+1{{'available_only_in_default_eval_method' attribute only applies to typedefs}}
+// expected-error@+1{{'clang::available_only_in_default_eval_method' attribute only applies to typedefs}}
 class  [[clang::available_only_in_default_eval_method]] C2 {
 };
 
-// expected-error@+1{{'available_only_in_default_eval_method' attribute only applies to typedefs}}
+// expected-error@+1{{'clang::available_only_in_default_eval_method' attribute only applies to typedefs}}
 struct [[clang::available_only_in_default_eval_method]] S1;
 // expected-error@+1{{'available_only_in_default_eval_method' attribute only applies to typedefs}}
 struct __attribute__((available_only_in_default_eval_method)) S2;
 
 // expected-error@+1{{'available_only_in_default_eval_method' attribute only applies to typedefs}}
 void __attribute__((available_only_in_default_eval_method)) foo();
-// expected-error@+1{{'available_only_in_default_eval_method' attribute cannot be applied to types}}
+// expected-error@+1{{'clang::available_only_in_default_eval_method' attribute cannot be applied to types}}
 void [[clang::available_only_in_default_eval_method]] goo();
-// expected-error@+1{{'available_only_in_default_eval_method' attribute cannot be applied to types}}
+// expected-error@+1{{'clang::available_only_in_default_eval_method' attribute cannot be applied to types}}
 void bar() [[clang::available_only_in_default_eval_method]];
 // expected-error@+1{{'available_only_in_default_eval_method' attribute only applies to typedefs}}
 void barz() __attribute__((available_only_in_default_eval_method));
diff --git a/clang/test/Sema/attr-preferred-type.cpp b/clang/test/Sema/attr-preferred-type.cpp
index dfa7e636c21e9..cd5b5cd6e928a 100644
--- a/clang/test/Sema/attr-preferred-type.cpp
+++ b/clang/test/Sema/attr-preferred-type.cpp
@@ -9,9 +9,9 @@ struct A {
   [[clang::preferred_type(bool)]] unsigned b4 : 1;
   [[clang::preferred_type(bool)]] unsigned b5 : 2;
   [[clang::preferred_type()]] unsigned b6 : 2;
-  // expected-error@-1 {{'preferred_type' attribute takes one argument}}
+  // expected-error@-1 {{'clang::preferred_type' attribute takes one argument}}
   [[clang::preferred_type]] unsigned b7 : 2;
-  // expected-error@-1 {{'preferred_type' attribute takes one argument}}
+  // expected-error@-1 {{'clang::preferred_type' attribute takes one argument}}
   [[clang::preferred_type(E, int)]] unsigned b8 : 2;
   // expected-error@-1 {{expected ')'}}
   // expected-error@-2 {{expected ','}}
diff --git a/clang/test/Sema/attr-regparm.c b/clang/test/Sema/attr-regparm.c
index 1fd54a5b6656e..3f641c98ac43d 100644
--- a/clang/test/Sema/attr-regparm.c
+++ b/clang/test/Sema/attr-regparm.c
@@ -14,4 +14,4 @@ void __attribute__((regparm(2))) x5(int); // expected-error{{function declared w
 [[gnu::regparm(2)]] void x6(int); // expected-error{{function declared with regparm(2) attribute was previously declared with the regparm(3) attribute}}
 void x6 [[gnu::regparm(3)]] (int);
 void [[gnu::regparm(3)]] x6(int); // expected-warning{{'regparm' only applies to function types; type here is 'void'}}
-void x6(int) [[gnu::regparm(3)]]; // expected-warning{{GCC does not allow the 'regparm' attribute to be written on a type}}
+void x6(int) [[gnu::regparm(3)]]; // expected-warning{{GCC does not allow the 'gnu::regparm' attribute to be written on a type}}
diff --git a/clang/test/Sema/attr-type-safety.c b/clang/test/Sema/attr-type-safety.c
index f1027738f6083..cd528d2eaac56 100644
--- a/clang/test/Sema/attr-type-safety.c
+++ b/clang/test/Sema/attr-type-safety.c
@@ -4,7 +4,7 @@ struct A {};
 
 typedef struct A *MPI_Datatype;
 
-extern struct A datatype_wrong1 [[clang::type_tag_for_datatype]]; // expected-error {{'type_tag_for_datatype' attribute requires parameter 1 to be an identifier}}
+extern struct A datatype_wrong1 [[clang::type_tag_for_datatype]]; // expected-error {{'clang::type_tag_for_datatype' attribute requires parameter 1 to be an identifier}}
 
 extern struct A datatype_wrong2 [[clang::type_tag_for_datatype(mpi,1,2)]]; // expected-error {{expected a type}}
 
@@ -22,7 +22,7 @@ extern struct A B_tag [[clang::type_tag_for_datatype(b,int)]];
 static const int C_tag [[clang::type_tag_for_datatype(c,int)]] = 10;
 static const int D_tag [[clang::type_tag_for_datatype(d,int)]] = 20;
 
-[[clang::pointer_with_type_tag]] // expected-error {{'pointer_with_type_tag' attribute requires exactly 3 arguments}}
+[[clang::pointer_with_type_tag]] // expected-error {{'clang::pointer_with_type_tag' attribute requires exactly 3 arguments}}
 int wrong1(void *buf, MPI_Datatype datatype);
 
 [[clang::pointer_with_type_tag(mpi,0,7)]]  // expected-error {{attribute parameter 2 is out of bounds}}
diff --git a/clang/test/Sema/builtins-microsoft-arm64.c b/clang/test/Sema/builtins-microsoft-arm64.c
index 322cf7542f43a..a915a370cde14 100644
--- a/clang/test/Sema/builtins-microsoft-arm64.c
+++ b/clang/test/Sema/builtins-microsoft-arm64.c
@@ -24,3 +24,12 @@ void check_ReadWriteStatusReg(int v) {
   _ReadStatusReg(x); // expected-error {{argument to '_ReadStatusReg' must be a constant integer}}
   _WriteStatusReg(x, v); // expected-error {{argument to '_WriteStatusReg' must be a constant integer}}
 }
+
+void check__sys(int v) {
+  int x;
+  __sys(x, v); // expected-error {{argument to '__sys' must be a constant integer}}
+}
+
+unsigned int check__sys_retval() {
+  return __sys(0, 1); // builtin has superfluous return value for MSVC compatibility
+}
diff --git a/clang/test/Sema/code_align.c b/clang/test/Sema/code_align.c
index 083757b578d0b..52c5879d3a390 100644
--- a/clang/test/Sema/code_align.c
+++ b/clang/test/Sema/code_align.c
@@ -9,14 +9,14 @@ void foo() {
   for (i = 0; i < 10; ++i) {  // this is OK
     a[i] = b[i] = 0;
   }
-  // expected-error@+1{{'code_align' attribute only applies to 'for', 'while', and 'do' statements}}
+  // expected-error@+1{{'clang::code_align' attribute only applies to 'for', 'while', and 'do' statements}}
   [[clang::code_align(4)]]
   i = 7;
   for (i = 0; i < 10; ++i) {
     a[i] = b[i] = 0;
   }
 
-  // expected-error@+1{{'code_align' attribute cannot be applied to a declaration}}
+  // expected-error@+1{{'clang::code_align' attribute cannot be applied to a declaration}}
   [[clang::code_align(12)]] int n[10];
 }
 
@@ -24,11 +24,11 @@ void bar(int);
 // cpp-local-note@+1{{declared here}}
 void foo1(int A)
 {
-  // expected-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 0}}
+  // expected-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 0}}
   [[clang::code_align(0)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // expected-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -4}}
+  // expected-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -4}}
   [[clang::code_align(-4)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
@@ -37,11 +37,11 @@ void foo1(int A)
   [[clang::code_align(64.0)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // expected-error@+1{{'code_align' attribute takes one argument}}
+  // expected-error@+1{{'clang::code_align' attribute takes one argument}}
   [[clang::code_align()]]
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // expected-error@+1{{'code_align' attribute takes one argument}}
+  // expected-error@+1{{'clang::code_align' attribute takes one argument}}
   [[clang::code_align(4,8)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
@@ -59,47 +59,47 @@ void foo1(int A)
   for(int I=0; I<128; ++I) { bar(I); }
 
   [[clang::code_align(8)]]  // expected-note{{previous attribute is here}}
-  [[clang::code_align(64)]] // expected-error{{conflicting loop attribute 'code_align'}}
+  [[clang::code_align(64)]] // expected-error{{conflicting loop attribute 'clang::code_align'}}
   for(int I=0; I<128; ++I) { bar(I); }
 
   [[clang::code_align(4)]] // expected-note{{previous attribute is here}}
   [[clang::code_align(4)]] // OK
-  [[clang::code_align(8)]] // expected-error{{conflicting loop attribute 'code_align'}}
+  [[clang::code_align(8)]] // expected-error{{conflicting loop attribute 'clang::code_align'}}
   for(int I=0; I<128; ++I) { bar(I); }
 
   [[clang::code_align(4)]]  // expected-note 2{{previous attribute is here}}
   [[clang::code_align(4)]]  // OK
-  [[clang::code_align(8)]]  // expected-error{{conflicting loop attribute 'code_align'}}
-  [[clang::code_align(64)]] // expected-error{{conflicting loop attribute 'code_align'}}
+  [[clang::code_align(8)]]  // expected-error{{conflicting loop attribute 'clang::code_align'}}
+  [[clang::code_align(64)]] // expected-error{{conflicting loop attribute 'clang::code_align'}}
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // expected-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 7}}
+  // expected-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 7}}
   [[clang::code_align(7)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // expected-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 5000}}
+  // expected-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 5000}}
   [[clang::code_align(5000)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
   // expected-warning@+2{{integer literal is too large to be represented in a signed integer type, interpreting as unsigned}}
-  // expected-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -9223372036854775808}}
+  // expected-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -9223372036854775808}}
   [[clang::code_align(9223372036854775808)]]
   for(int I=0; I<256; ++I) { bar(I); }
 
 #ifdef __SIZEOF_INT128__
-  // expected-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was '(__int128_t)1311768467294899680ULL << 64'}}
+  // expected-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was '(__int128_t)1311768467294899680ULL << 64'}}
   [[clang::code_align((__int128_t)0x1234567890abcde0ULL << 64)]]
   for(int I=0; I<256; ++I) { bar(I); }
 #endif
 
-  // expected-error@+1 {{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -922337203685477}}
+  // expected-error@+1 {{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -922337203685477}}
   [[clang::code_align(-922337203685477)]]
   for(int I=0; I<256; ++I) { bar(I); }
 
 #ifdef __SIZEOF_INT128__
   // cpp-local-error@+3{{expression is not an integral constant expression}}
   // cpp-local-note@+2{{left shift of negative value -1311768467294899680}}
-  // c-local-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was '-(__int128_t)1311768467294899680ULL << 64'}}
+  // c-local-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was '-(__int128_t)1311768467294899680ULL << 64'}}
   [[clang::code_align(-(__int128_t)0x1234567890abcde0ULL << 64)]]
   for(int I=0; I<256; ++I) { bar(I); }
 #endif
@@ -143,25 +143,25 @@ void code_align_dependent() {
   for(int I=0; I<128; ++I) { bar(I); }
 
   [[clang::code_align(A)]] // cpp-local-note{{previous attribute is here}}
-  [[clang::code_align(E)]] // cpp-local-error{{conflicting loop attribute 'code_align'}}
+  [[clang::code_align(E)]] // cpp-local-error{{conflicting loop attribute 'clang::code_align'}}
   for(int I=0; I<128; ++I) { bar(I); }
 
   [[clang::code_align(A)]] // cpp-local-note{{previous attribute is here}}
   [[clang::code_align(A)]] // OK
-  [[clang::code_align(E)]] // cpp-local-error{{conflicting loop attribute 'code_align'}}
+  [[clang::code_align(E)]] // cpp-local-error{{conflicting loop attribute 'clang::code_align'}}
   for(int I=0; I<128; ++I) { bar(I); }
 
   [[clang::code_align(A)]] // cpp-local-note 2{{previous attribute is here}}
   [[clang::code_align(A)]] // OK
-  [[clang::code_align(C)]] // cpp-local-error{{conflicting loop attribute 'code_align'}}
-  [[clang::code_align(E)]] // cpp-local-error{{conflicting loop attribute 'code_align'}}
+  [[clang::code_align(C)]] // cpp-local-error{{conflicting loop attribute 'clang::code_align'}}
+  [[clang::code_align(E)]] // cpp-local-error{{conflicting loop attribute 'clang::code_align'}}
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // cpp-local-error@+1{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 23}}
+  // cpp-local-error@+1{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was 23}}
   [[clang::code_align(B)]]
   for(int I=0; I<128; ++I) { bar(I); }
 
-  // cpp-local-error@+2{{'code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -10}}
+  // cpp-local-error@+2{{'clang::code_align' attribute requires an integer argument which is a constant power of two between 1 and 4096 inclusive; provided argument was -10}}
   // cpp-local-note@#neg-instantiation{{in instantiation of function template specialization 'code_align_dependent<8, 23, 32, -10, 64>' requested here}}
   [[clang::code_align(D)]]
   for(int I=0; I<128; ++I) { bar(I); }
@@ -170,7 +170,7 @@ void code_align_dependent() {
 template<int ITMPL>
 void bar3() {
   [[clang::code_align(8)]]      // cpp-local-note{{previous attribute is here}}
-  [[clang::code_align(ITMPL)]] // cpp-local-error{{conflicting loop attribute 'code_align'}} \
+  [[clang::code_align(ITMPL)]] // cpp-local-error{{conflicting loop attribute 'clang::code_align'}} \
 	                       // cpp-local-note@#temp-instantiation{{in instantiation of function template specialization 'bar3<4>' requested here}}
   for(int I=0; I<128; ++I) { bar(I); }
 }
@@ -178,7 +178,7 @@ void bar3() {
 template<int ITMPL1>
 void bar4() {
   [[clang::code_align(ITMPL1)]] // cpp-local-note{{previous attribute is here}}
-  [[clang::code_align(32)]]    // cpp-local-error{{conflicting loop attribute 'code_align'}} \
+  [[clang::code_align(32)]]    // cpp-local-error{{conflicting loop attribute 'clang::code_align'}} \
 	                       // cpp-local-note@#temp-instantiation1{{in instantiation of function template specialization 'bar4<64>' requested here}}
   for(int I=0; I<128; ++I) { bar(I); }
 }
diff --git a/clang/test/Sema/internal_linkage.c b/clang/test/Sema/internal_linkage.c
index a0c421527e801..a1bff73fb6620 100644
--- a/clang/test/Sema/internal_linkage.c
+++ b/clang/test/Sema/internal_linkage.c
@@ -20,7 +20,7 @@ struct __attribute__((internal_linkage)) S { // expected-warning{{'internal_link
 __attribute__((internal_linkage("foo"))) int g(void) {} // expected-error{{'internal_linkage' attribute takes no arguments}}
 
 int var6 [[clang::internal_linkage]];
-int var7 [[clang::internal_linkage]] __attribute__((common)); // expected-error{{'internal_linkage' and 'common' attributes are not compatible}} \
+int var7 [[clang::internal_linkage]] __attribute__((common)); // expected-error{{'clang::internal_linkage' and 'common' attributes are not compatible}} \
                                                    // expected-note{{conflicting attribute is here}}
-__attribute__((common)) int var8 [[clang::internal_linkage]]; // expected-error{{'internal_linkage' and 'common' attributes are not compatible}} \
+__attribute__((common)) int var8 [[clang::internal_linkage]]; // expected-error{{'clang::internal_linkage' and 'common' attributes are not compatible}} \
                                                    // expected-note{{conflicting attribute is here}}
diff --git a/clang/test/Sema/matrix-type-builtins.c b/clang/test/Sema/matrix-type-builtins.c
index b5548831002e3..b92f3ce6a3e8c 100644
--- a/clang/test/Sema/matrix-type-builtins.c
+++ b/clang/test/Sema/matrix-type-builtins.c
@@ -11,15 +11,15 @@ typedef unsigned ix3x3 __attribute__((matrix_type(3, 3)));
 typedef float [[clang::matrix_type(5, 10)]] sx5x10_t; // expected-warning {{[[]] attributes are a C23 extension}}
 typedef int [[clang::matrix_type(3, 2)]] ix3x2_t; // expected-warning {{[[]] attributes are a C23 extension}}
 [[clang::matrix_type(5, 10)]] typedef float sx5x10_t; // expected-warning {{[[]] attributes are a C23 extension}}
-// expected-warning@-1 {{applying attribute 'matrix_type' to a declaration is deprecated; apply it to the type instead}}
+// expected-warning@-1 {{applying attribute 'clang::matrix_type' to a declaration is deprecated; apply it to the type instead}}
 [[clang::matrix_type(3, 2)]] typedef int ix3x2_t; // expected-warning {{[[]] attributes are a C23 extension}}
-// expected-warning@-1 {{applying attribute 'matrix_type' to a declaration is deprecated; apply it to the type instead}}
+// expected-warning@-1 {{applying attribute 'clang::matrix_type' to a declaration is deprecated; apply it to the type instead}}
 
 // Attribute may not be used outside typedefs.
 [[clang::matrix_type(3, 2)]] int ix3x2_var; // expected-warning {{[[]] attributes are a C23 extension}}
-// expected-error@-1 {{'matrix_type' attribute only applies to typedefs}}
+// expected-error@-1 {{'clang::matrix_type' attribute only applies to typedefs}}
 int [[clang::matrix_type(3, 2)]] ix3x2_var; // expected-warning {{[[]] attributes are a C23 extension}}
-// expected-error@-1 {{'matrix_type' attribute only applies to typedefs}}
+// expected-error@-1 {{'clang::matrix_type' attribute only applies to typedefs}}
 
 void transpose(sx5x10_t a, ix3x2_t b, dx3x3 c, int *d, int e) {
   a = __builtin_matrix_transpose(b);
diff --git a/clang/test/Sema/offsetof-64.c b/clang/test/Sema/offsetof-64.c
index 8ffc3af985880..692698fe39e00 100644
--- a/clang/test/Sema/offsetof-64.c
+++ b/clang/test/Sema/offsetof-64.c
@@ -2,7 +2,7 @@
 
 // PR15216
 // Don't crash when taking computing the offset of structs with large arrays.
-const unsigned long Size = (1l << 60);
+const unsigned long Size = (1l << 58);
 
 struct Chunk1 {
   char padding[Size]; // expected-warning {{folded to constant}}
@@ -10,7 +10,7 @@ struct Chunk1 {
   char data;
 };
 
-int test1 = __builtin_offsetof(struct Chunk1, data);
+unsigned long test1 = __builtin_offsetof(struct Chunk1, data);
 
 struct Chunk2 {
   char padding[Size][Size][Size];  // expected-error {{array is too large}}
diff --git a/clang/test/Sema/overloadable.c b/clang/test/Sema/overloadable.c
index 4c6fd0102c59a..f236ee7ad104d 100644
--- a/clang/test/Sema/overloadable.c
+++ b/clang/test/Sema/overloadable.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s -Wincompatible-pointer-types -Wno-strict-prototypes
 
 int var __attribute__((overloadable)); // expected-error{{'overloadable' attribute only applies to functions}}
-void bad_attr_target(int) [[clang::overloadable]]; // expected-error{{'overloadable' attribute cannot be applied to types}}
+void bad_attr_target(int) [[clang::overloadable]]; // expected-error{{'clang::overloadable' attribute cannot be applied to types}}
 void params(void) __attribute__((overloadable(12))); // expected-error {{'overloadable' attribute takes no arguments}}
 
 int *f(int) __attribute__((overloadable)); // expected-note{{previous overload of function is here}}
@@ -268,4 +268,4 @@ void can_overload_2(...) __attribute__((overloadable)); // ok
 [[clang::overloadable]] void can_overload_3(...);       // ok, was previously rejected
 void can_overload_4 [[clang::overloadable]] (...);      // ok
 void cannot_overload(...) [[clang::overloadable]];      // expected-error {{ISO C requires a named parameter before '...'}} \
-                                                        // expected-error {{'overloadable' attribute cannot be applied to types}}
+                                                        // expected-error {{'clang::overloadable' attribute cannot be applied to types}}
diff --git a/clang/test/Sema/patchable-function-entry-attr.cpp b/clang/test/Sema/patchable-function-entry-attr.cpp
index efc9fe0e41130..7498e67de302d 100644
--- a/clang/test/Sema/patchable-function-entry-attr.cpp
+++ b/clang/test/Sema/patchable-function-entry-attr.cpp
@@ -14,6 +14,6 @@
 
 // silence-no-diagnostics
 
-// AIX-error@+2 {{'patchable_function_entry' attribute is not yet supported on AIX}}
+// AIX-error@+2 {{'gnu::patchable_function_entry' attribute is not yet supported on AIX}}
 // expected-warning@+1 {{unknown attribute 'gnu::patchable_function_entry' ignored}}
 [[gnu::patchable_function_entry(0)]] void f();
diff --git a/clang/test/Sema/rvv-andes-required-features-invalid.c b/clang/test/Sema/rvv-andes-required-features-invalid.c
index dbb8b2895bedf..cd59322225e05 100644
--- a/clang/test/Sema/rvv-andes-required-features-invalid.c
+++ b/clang/test/Sema/rvv-andes-required-features-invalid.c
@@ -9,3 +9,12 @@ vfloat32mf2_t test_nds_vfwcvt_s_bf16_f32mf2(vbfloat16mf4_t vs2, size_t vl) {
   // expected-error@-1 {{RISC-V type 'vbfloat16mf4_t' (aka '__rvv_bfloat16mf4_t') requires the 'zvfbfmin or xandesvbfhcvt' extension}}
   // expected-error@-2 {{builtin requires at least one of the following extensions: xandesvbfhcvt}}
 }
+
+vfloat16mf4_t test_nds_vfpmadb_vf_f16mf4(vfloat16mf4_t op1, float op2, size_t vl) {
+  // expected-error@-1 {{RISC-V type 'vfloat16mf4_t' (aka '__rvv_float16mf4_t') requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-2 {{RISC-V type 'vfloat16mf4_t' (aka '__rvv_float16mf4_t') requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  return __riscv_nds_vfpmadb_vf_f16mf4(op1, op2, vl); // expected-error {{RISC-V type '__rvv_float16mf4_t' requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-1 {{RISC-V type '__rvv_float16mf4_t' requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-2 {{RISC-V type 'vfloat16mf4_t' (aka '__rvv_float16mf4_t') requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-3 {{builtin requires at least one of the following extensions: xandesvpackfph}}
+}
diff --git a/clang/test/Sema/static-assert.c b/clang/test/Sema/static-assert.c
index d603bc19bb824..dd83cd7684d2c 100644
--- a/clang/test/Sema/static-assert.c
+++ b/clang/test/Sema/static-assert.c
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify %s
 // RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms %s
+// RUN: %clang_cc1 -std=c99 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms %s
 // RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext %s
 // RUN: %clang_cc1 -xc++ -std=c++11 -pedantic -fsyntax-only -verify=expected,ext,cxx %s
 
@@ -13,7 +14,7 @@ _Static_assert(0, "0 is nonzero"); // expected-error {{static assertion failed:
                                    // ext-warning {{'_Static_assert' is a C11 extension}}
 
 #ifdef MS
-static_assert(1, "1 is nonzero"); // ms-warning {{use of 'static_assert' without inclusion of <assert.h> is a Microsoft extension}}
+static_assert(1, "1 is nonzero");
 #endif
 
 void foo(void) {
@@ -21,7 +22,7 @@ void foo(void) {
   _Static_assert(0, "0 is nonzero"); // expected-error {{static assertion failed: 0 is nonzero}} \
                                      // ext-warning {{'_Static_assert' is a C11 extension}}
 #ifdef MS
-  static_assert(1, "1 is nonzero"); // ms-warning {{use of 'static_assert' without}}
+  static_assert(1, "1 is nonzero");
 #endif
 }
 
@@ -38,7 +39,7 @@ struct A {
   _Static_assert(0, "0 is nonzero"); // expected-error {{static assertion failed: 0 is nonzero}} \
                                      // ext-warning {{'_Static_assert' is a C11 extension}}
 #ifdef MS
-  static_assert(1, "1 is nonzero"); // ms-warning {{use of 'static_assert' without}}
+  static_assert(1, "1 is nonzero");
 #endif
 };
 
@@ -64,16 +65,27 @@ typedef UNION(char, short) U3; // expected-error {{static assertion failed due t
 typedef UNION(float, 0.5f) U4; // expected-error {{expected a type}} \
                                // ext-warning 3 {{'_Static_assert' is a C11 extension}}
 
-// After defining the assert macro in MS-compatibility mode, we should
-// no longer warn about including <assert.h> under the assumption the
-// user already did that.
+// MSVC accepts static_assert in all language modes without including <assert.h>
+// and so do we in ms-compatibility mode. Unfortunately, there is no good way
+// to diagnose that with a pedantic warning. We'd have to track inclusion of
+// <assert.h> which is difficult when modules and PCH are involved. Adding
+// implicit definition of the macro causes unexpected results in c99 mode.
 #ifdef MS
+
+#if __STDC_VERSION__ < 201112L
+#if defined(static_assert)
+static_assert(0, "0 is nonzero"); // ok because we should not define static_assert
+                                  // macro in c99.
+#endif
+
+static_assert(0, "0 is nonzero"); // ms-error {{static assertion failed: 0 is nonzero}}
+#endif
+
 #define assert(expr)
 static_assert(1, "1 is nonzero"); // ok
 
-// But we should still warn if the user did something bonkers.
 #undef static_assert
-static_assert(1, "1 is nonzero"); // ms-warning {{use of 'static_assert' without}}
+static_assert(1, "1 is nonzero"); // yes, still ok.
 #endif
 
 _Static_assert(1 , "") // expected-error {{expected ';' after '_Static_assert'}} \
diff --git a/clang/test/Sema/vector-gcc-compat.c b/clang/test/Sema/vector-gcc-compat.c
index 7764b3bf686ad..6bb2c5168457c 100644
--- a/clang/test/Sema/vector-gcc-compat.c
+++ b/clang/test/Sema/vector-gcc-compat.c
@@ -26,7 +26,7 @@ typedef int v2i32 [[gnu::vector_size(8)]]; // expected-warning{{[[]] attributes
 // Check various positions where the [[]] spelling can or cannot be used.
 [[gnu::vector_size(16)]] typedef long long v2i64; // expected-warning{{[[]] attributes are a C23 extension}}
 typedef long long [[gnu::vector_size(16)]] v2i64_ignored;
-  // expected-warning@-1{{'vector_size' attribute ignored}}
+  // expected-warning@-1{{'gnu::vector_size' attribute ignored}}
   // expected-warning@-2{{[[]] attributes are a C23 extension}}
 // FIXME: Contrary to the error message that we emit, GCC does actually allow
 // the attribute in the following position. Somewhat surprisingly, the attribute
@@ -34,7 +34,7 @@ typedef long long [[gnu::vector_size(16)]] v2i64_ignored;
 // the same effect in GCC as the other declarations for `v2i64`.
 typedef long long *[[gnu::vector_size(16)]] v2i64_doesnt_work;
   // expected-error@-1{{invalid vector element type 'long long *'}}
-  // expected-warning@-2{{GCC does not allow the 'vector_size' attribute to be written on a type}}
+  // expected-warning@-2{{GCC does not allow the 'gnu::vector_size' attribute to be written on a type}}
   // expected-warning@-3{{[[]] attributes are a C23 extension}}
 
 // Verify that we can use the attribute outside of a typedef.
diff --git a/clang/test/Sema/warn-documentation.cpp b/clang/test/Sema/warn-documentation.cpp
index 4839bdff61795..0d1faa1b562fe 100644
--- a/clang/test/Sema/warn-documentation.cpp
+++ b/clang/test/Sema/warn-documentation.cpp
@@ -1524,3 +1524,15 @@ F &f = FF; ///< \return none
 // expected-warning@-1 {{'\return' command used in a comment that is not attached to a function or method declaration}}
 
 } // namespace PR42844
+
+#if __cplusplus >= 201402L
+namespace GH144775 {
+/// @brief primary template
+/// @tparam T type
+template <class T, class = void> constexpr auto var = T{};
+
+/// @brief variable template partial specialization
+/// @tparam T type
+template <typename T> constexpr auto var<T> = T{};
+} // namespace GH144775
+#endif
diff --git a/clang/test/Sema/warn-format-overflow-truncation.c b/clang/test/Sema/warn-format-overflow-truncation.c
index c64a1ed8aaa05..eb1feaae834ec 100644
--- a/clang/test/Sema/warn-format-overflow-truncation.c
+++ b/clang/test/Sema/warn-format-overflow-truncation.c
@@ -43,6 +43,13 @@ void call_snprintf(double d, int n, int *ptr) {
   __builtin_snprintf(node_name, sizeof(node_name), "%pOFn", ptr); // nonkprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
   __builtin_snprintf(node_name, sizeof(node_name), "12345%pOFn", ptr); // nonkprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 12}}
   __builtin_snprintf(node_name, sizeof(node_name), "123456%pOFn", ptr); // nonkprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 13}}
+  __builtin_snprintf(buf, 6, "%5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "%+5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "% 5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "%   5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "%+6.1f", 9.f); // kprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
+  __builtin_snprintf(buf, 6, "% 6.1f", 9.f); // kprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
+  __builtin_snprintf(buf, 6, "%   6.1f", 9.f); // kprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
 }
 
 void call_vsnprintf(void) {
@@ -153,4 +160,11 @@ void call_sprintf(void) {
   sprintf(buf, "%+.3f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
   sprintf(buf, "%.0e", 9.f);
   sprintf(buf, "5%.1e", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 8}}
+  sprintf(buf, "%5.1f", 9.f);
+  sprintf(buf, "%+5.1f", 9.f);
+  sprintf(buf, "% 5.1f", 9.f);
+  sprintf(buf, "%   5.1f", 9.f);
+  sprintf(buf, "%+6.1f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
+  sprintf(buf, "% 6.1f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
+  sprintf(buf, "%   6.1f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
 }
diff --git a/clang/test/Sema/xray-always-instrument-attr.cpp b/clang/test/Sema/xray-always-instrument-attr.cpp
index d6a33955cd746..06a93f479f238 100644
--- a/clang/test/Sema/xray-always-instrument-attr.cpp
+++ b/clang/test/Sema/xray-always-instrument-attr.cpp
@@ -1,10 +1,10 @@
 // RUN: %clang_cc1 %s -verify -fsyntax-only -std=c++11 -x c++
 void foo [[clang::xray_always_instrument]] ();
 
-struct [[clang::xray_always_instrument]] a { int x; }; // expected-warning {{'xray_always_instrument' attribute only applies to functions and Objective-C methods}}
+struct [[clang::xray_always_instrument]] a { int x; }; // expected-warning {{'clang::xray_always_instrument' attribute only applies to functions and Objective-C methods}}
 
 class b {
  void c [[clang::xray_always_instrument]] ();
 };
 
-void baz [[clang::xray_always_instrument("not-supported")]] (); // expected-error {{'xray_always_instrument' attribute takes no arguments}}
+void baz [[clang::xray_always_instrument("not-supported")]] (); // expected-error {{'clang::xray_always_instrument' attribute takes no arguments}}
diff --git a/clang/test/Sema/xray-log-args-class.cpp b/clang/test/Sema/xray-log-args-class.cpp
index 10da93f614fea..cad9d10de7d96 100644
--- a/clang/test/Sema/xray-log-args-class.cpp
+++ b/clang/test/Sema/xray-log-args-class.cpp
@@ -2,6 +2,6 @@
 
 class Class {
   [[clang::xray_always_instrument, clang::xray_log_args(1)]] void Method();
-  [[clang::xray_log_args(-1)]] void Invalid(); // expected-error {{'xray_log_args' attribute parameter 1 is out of bounds}}
-  [[clang::xray_log_args("invalid")]] void InvalidStringArg(); // expected-error {{'xray_log_args'}}
+  [[clang::xray_log_args(-1)]] void Invalid(); // expected-error {{'clang::xray_log_args' attribute parameter 1 is out of bounds}}
+  [[clang::xray_log_args("invalid")]] void InvalidStringArg(); // expected-error {{'clang::xray_log_args'}}
 };
diff --git a/clang/test/Sema/xray-log-args-oob.cpp b/clang/test/Sema/xray-log-args-oob.cpp
index 82f3be30c6deb..6a106f21ef050 100644
--- a/clang/test/Sema/xray-log-args-oob.cpp
+++ b/clang/test/Sema/xray-log-args-oob.cpp
@@ -1,9 +1,9 @@
 // RUN: %clang_cc1 %s -verify -fsyntax-only -std=c++11 -x c++
 void foo [[clang::xray_log_args(1)]] (int);
-struct [[clang::xray_log_args(1)]] a { int x; }; // expected-warning {{'xray_log_args' attribute only applies to functions and Objective-C methods}}
+struct [[clang::xray_log_args(1)]] a { int x; }; // expected-warning {{'clang::xray_log_args' attribute only applies to functions and Objective-C methods}}
 
-void fop [[clang::xray_log_args(1)]] (); // expected-error {{'xray_log_args' attribute parameter 1 is out of bounds}}
+void fop [[clang::xray_log_args(1)]] (); // expected-error {{'clang::xray_log_args' attribute parameter 1 is out of bounds}}
 
-void foq [[clang::xray_log_args(-1)]] (); // expected-error {{'xray_log_args' attribute parameter 1 is out of bounds}}
+void foq [[clang::xray_log_args(-1)]] (); // expected-error {{'clang::xray_log_args' attribute parameter 1 is out of bounds}}
 
-void fos [[clang::xray_log_args(0)]] (); // expected-error {{'xray_log_args' attribute parameter 1 is out of bounds}}
+void fos [[clang::xray_log_args(0)]] (); // expected-error {{'clang::xray_log_args' attribute parameter 1 is out of bounds}}
diff --git a/clang/test/SemaCUDA/attr-noconvergent.cu b/clang/test/SemaCUDA/attr-noconvergent.cu
index 0c051fdde4379..bddf40fd2544c 100644
--- a/clang/test/SemaCUDA/attr-noconvergent.cu
+++ b/clang/test/SemaCUDA/attr-noconvergent.cu
@@ -7,10 +7,10 @@ __device__ __attribute__((noconvergent)) float f1(float);
 [[clang::noconvergent]] __device__ float f2(float);
 
 __device__ [[clang::noconvergent(1)]] float f3(float);
-// expected-error@-1 {{'noconvergent' attribute takes no arguments}}
+// expected-error@-1 {{'clang::noconvergent' attribute takes no arguments}}
 
 __device__ [[clang::noconvergent]] float g0;
-// expected-warning@-1 {{'noconvergent' attribute only applies to functions and statements}}
+// expected-warning@-1 {{'clang::noconvergent' attribute only applies to functions and statements}}
 
 __device__ __attribute__((convergent)) __attribute__((noconvergent)) float f4(float);
 // expected-error@-1 {{'noconvergent' and 'convergent' attributes are not compatible}}
@@ -18,17 +18,17 @@ __device__ __attribute__((convergent)) __attribute__((noconvergent)) float f4(fl
 
 __device__ [[clang::noconvergent]] float f5(float);
 __device__ [[clang::convergent]] float f5(float);
-// expected-error@-1 {{'convergent' and 'noconvergent' attributes are not compatible}}
+// expected-error@-1 {{'clang::convergent' and 'clang::noconvergent' attributes are not compatible}}
 // expected-note@-3 {{conflicting attribute is here}}
 
 __device__ float f5(float x) {
   [[clang::noconvergent]] float y;
-// expected-warning@-1 {{'noconvergent' attribute only applies to functions and statements}}
+// expected-warning@-1 {{'clang::noconvergent' attribute only applies to functions and statements}}
 
   float z;
 
   [[clang::noconvergent]] z = 1;
-// expected-warning@-1 {{'noconvergent' attribute is ignored because there exists no call expression inside the statement}}
+// expected-warning@-1 {{'clang::noconvergent' attribute is ignored because there exists no call expression inside the statement}}
 
   [[clang::noconvergent]] z = f0(x);
 }
diff --git a/clang/test/SemaCXX/P1811.cpp b/clang/test/SemaCXX/P1811.cpp
new file mode 100644
index 0000000000000..329110a2ce4dd
--- /dev/null
+++ b/clang/test/SemaCXX/P1811.cpp
@@ -0,0 +1,41 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/mod.cpp -o %t/mod.pcm
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify -fmodule-file=M=%t/mod.pcm %t/main1.cpp
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify -fmodule-file=M=%t/mod.pcm %t/main2.cpp
+
+//--- mod.cpp
+// expected-no-diagnostics
+module;
+#include "A.h"
+export module M;
+export A f() {return A{};}
+
+//--- A.h
+// expected-no-diagnostics
+#ifndef X
+#define X
+
+struct A{};
+
+#endif
+
+//--- main1.cpp
+// expected-no-diagnostics
+#include "A.h"
+import M;
+
+extern "C++" int main() {
+  A a;
+}
+
+//--- main2.cpp
+// expected-no-diagnostics
+import M;
+#include "A.h"
+
+extern "C++" int main() {
+  A a;
+}
diff --git a/clang/test/SemaCXX/P1979.cpp b/clang/test/SemaCXX/P1979.cpp
new file mode 100644
index 0000000000000..65c4b839632c2
--- /dev/null
+++ b/clang/test/SemaCXX/P1979.cpp
@@ -0,0 +1,38 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/A.cpp -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/myV.cpp -o %t/myV.pcm
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface -fmodule-file=V=%t/myV.pcm %t/partition.cpp -o %t/partition.pcm
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify -fmodule-file=V=%t/myV.pcm -fmodule-file=A=%t/A.pcm -fmodule-file=A:partition=%t/partition.pcm %t/interface.cpp
+
+//--- A.cpp
+// expected-no-diagnostics
+export module A;
+
+//--- myV.cpp
+// expected-no-diagnostics
+export module V;
+
+export struct myV{};
+
+//--- uses_vector.h
+// expected-no-diagnostics
+#ifndef x
+#define x
+
+import V;
+#endif
+
+//--- partition.cpp
+// expected-no-diagnostics
+module;
+#include "uses_vector.h" // textually expands to import V;
+module A:partition;
+
+//--- interface.cpp
+module A;
+import :partition;
+myV V; // expected-error {{declaration of 'myV' must be imported from module 'V' before it is required}}
+       // expected-note@myV.cpp:4 {{declaration here is not visible}}
diff --git a/clang/test/SemaCXX/P2115.cpp b/clang/test/SemaCXX/P2115.cpp
new file mode 100644
index 0000000000000..4389be992440b
--- /dev/null
+++ b/clang/test/SemaCXX/P2115.cpp
@@ -0,0 +1,21 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// RUN: %clang -std=c++20 -fmodule-header %t/A.h -o %t/A.pcm
+// RUN: %clang -std=c++20 -fmodule-header %t/B.h -o %t/B.pcm
+// RUN: %clang -std=c++20 -fsyntax-only -fmodule-file=%t/A.pcm -fmodule-file=%t/B.pcm %t/main.cpp
+
+//--- A.h
+// expected-no-diagnostics
+enum { A = 0 };
+
+//--- B.h
+// expected-no-diagnostics
+enum { B = 1 };
+
+//--- main.cpp
+// expected-no-diagnostics
+import "A.h";
+import "B.h";
+int main() {}
diff --git a/clang/test/SemaCXX/P2615.cpp b/clang/test/SemaCXX/P2615.cpp
new file mode 100644
index 0000000000000..fed177c21bc79
--- /dev/null
+++ b/clang/test/SemaCXX/P2615.cpp
@@ -0,0 +1,11 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// RUN: %clang_cc1 -std=c++20 -verify -fsyntax-only %t/A.cpp
+
+//--- A.cpp
+// expected-no-diagnostics
+export module A;
+export namespace N {int x = 42;}
+export using namespace N;
diff --git a/clang/test/SemaCXX/P2788.cpp b/clang/test/SemaCXX/P2788.cpp
new file mode 100644
index 0000000000000..a9184e1521e8e
--- /dev/null
+++ b/clang/test/SemaCXX/P2788.cpp
@@ -0,0 +1,17 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/B.cpp -o %t/B.pcm
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/A.cpp -fmodule-file=A:B=%t/B.pcm -o %t/A.pcm
+
+//--- A.cpp
+// expected-no-diagnostics
+export module A;
+import :B;
+export int x = dimensions + 1;
+
+//--- B.cpp
+// expected-no-diagnostics
+export module A:B;
+const int dimensions=3;
diff --git a/clang/test/SemaCXX/PR76631.cpp b/clang/test/SemaCXX/PR76631.cpp
index 947fa3fc2635e..0348b9a1bcab5 100644
--- a/clang/test/SemaCXX/PR76631.cpp
+++ b/clang/test/SemaCXX/PR76631.cpp
@@ -5,5 +5,5 @@
 }
 
 void throw_int_wrapper() {
-  [[clang::musttail]] return throw_int(); // expected-error {{'musttail' attribute may not be used with no-return-attribute functions}}
+  [[clang::musttail]] return throw_int(); // expected-error {{'clang::musttail' attribute may not be used with no-return-attribute functions}}
 }
diff --git a/clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp b/clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp
new file mode 100644
index 0000000000000..1520e2b572ce6
--- /dev/null
+++ b/clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp
@@ -0,0 +1,69 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -mvscale-max=1 -mvscale-streaming-min=2 -mvscale-streaming-max=2 -flax-vector-conversions=integer -ffreestanding -fsyntax-only -verify %s
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+#define SVE_BITS 128
+#define SVE_FIXED_ATTR __attribute__((arm_sve_vector_bits(SVE_BITS)))
+#define GNU_FIXED_ATTR __attribute__((vector_size(SVE_BITS / 8)))
+#define GNU_BOOL_FIXED_ATTR __attribute__((vector_size(SVE_BITS / 64)))
+#define STREAMING_BITS 256
+#define GNU_FIXED_STREAMING_ATTR __attribute__((vector_size(STREAMING_BITS / 8)))
+#define GNU_BOOL_FIXED_STREAMING_ATTR __attribute__((vector_size(STREAMING_BITS / 64)))
+
+typedef svfloat32_t sve_fixed_float32_t SVE_FIXED_ATTR;
+typedef svint32_t sve_fixed_int32_t SVE_FIXED_ATTR;
+typedef svbool_t sve_fixed_bool_t SVE_FIXED_ATTR;
+typedef float gnu_fixed_float32_t GNU_FIXED_ATTR;
+typedef int gnu_fixed_int32_t GNU_FIXED_ATTR;
+typedef int8_t gnu_fixed_bool_t GNU_BOOL_FIXED_ATTR;
+
+typedef float gnu_fixed_float32_t_streaming GNU_FIXED_STREAMING_ATTR;
+typedef int gnu_fixed_int32_t_streaming GNU_FIXED_STREAMING_ATTR;
+typedef int8_t gnu_fixed_bool_t_streaming GNU_BOOL_FIXED_STREAMING_ATTR;
+
+void sve_fixed() {
+  gnu_fixed_int32_t fi;
+  gnu_fixed_float32_t_streaming fi_wrong;
+  gnu_fixed_float32_t ff;
+  gnu_fixed_float32_t_streaming ff_wrong;
+  gnu_fixed_bool_t fb;
+  gnu_fixed_bool_t_streaming fb_wrong;
+  *(volatile svint32_t*)0 = fi;
+  *(volatile svint32_t*)0 = fi_wrong; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff;
+  *(volatile svfloat32_t*)0 = ff_wrong; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb;
+  *(volatile svbool_t*)0 = fb_wrong; // expected-error {{incompatible}}
+}
+
+__arm_locally_streaming void streaming_fixed() {
+  gnu_fixed_int32_t_streaming fi;
+  gnu_fixed_float32_t fi_wrong;
+  gnu_fixed_float32_t_streaming ff;
+  gnu_fixed_float32_t ff_wrong;
+  gnu_fixed_bool_t_streaming fb;
+  gnu_fixed_bool_t fb_wrong;
+  *(volatile svint32_t*)0 = fi;
+  *(volatile svint32_t*)0 = fi_wrong; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff;
+  *(volatile svfloat32_t*)0 = ff_wrong; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb;
+  *(volatile svbool_t*)0 = fb_wrong; // expected-error {{incompatible}}
+}
+
+void streaming_compatible() __arm_streaming_compatible {
+  gnu_fixed_int32_t fi_ns;
+  gnu_fixed_float32_t_streaming fi_s;
+  gnu_fixed_float32_t ff_ns;
+  gnu_fixed_float32_t_streaming ff_s;
+  gnu_fixed_bool_t fb_ns;
+  gnu_fixed_bool_t_streaming fb_s;
+  *(volatile svint32_t*)0 = fi_ns; // expected-error {{incompatible}}
+  *(volatile svint32_t*)0 = fi_s; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff_ns; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff_s; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb_ns; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb_s; // expected-error {{incompatible}}
+}
+
diff --git a/clang/test/SemaCXX/address-space-placement.cpp b/clang/test/SemaCXX/address-space-placement.cpp
index 2d8b94041259c..90a81ac1d5569 100644
--- a/clang/test/SemaCXX/address-space-placement.cpp
+++ b/clang/test/SemaCXX/address-space-placement.cpp
@@ -4,18 +4,18 @@
 // spelling of the `address_space` attribute is applied to a declaration instead
 // of a type. Also check that the attribute can instead be applied to the type.
 
-void f([[clang::address_space(1)]] int* param) { // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
-  [[clang::address_space(1)]] int* local1; // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
-  int* local2 [[clang::address_space(1)]]; // expected-error {{automatic variable qualified with an address space}} expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+void f([[clang::address_space(1)]] int* param) { // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
+  [[clang::address_space(1)]] int* local1; // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
+  int* local2 [[clang::address_space(1)]]; // expected-error {{automatic variable qualified with an address space}} expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
   int [[clang::address_space(1)]] * local3;
   int* [[clang::address_space(1)]] local4; // expected-error {{automatic variable qualified with an address space}}
 
-  for ([[clang::address_space(1)]] int* p = nullptr; p; ++p) {} // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
-  for (; [[clang::address_space(1)]] int* p = nullptr; ) {} // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
-  while([[clang::address_space(1)]] int* p = nullptr) {} // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
-  if ([[clang::address_space(1)]] int* p = nullptr) {} // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+  for ([[clang::address_space(1)]] int* p = nullptr; p; ++p) {} // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
+  for (; [[clang::address_space(1)]] int* p = nullptr; ) {} // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
+  while([[clang::address_space(1)]] int* p = nullptr) {} // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
+  if ([[clang::address_space(1)]] int* p = nullptr) {} // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
   try {
-  } catch([[clang::address_space(1)]] int& i) { // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+  } catch([[clang::address_space(1)]] int& i) { // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
   }
 
   for (int [[clang::address_space(1)]] * p = nullptr; p; ++p) {}
@@ -27,28 +27,28 @@ void f([[clang::address_space(1)]] int* param) { // expected-warning {{applying
   }
 }
 
-[[clang::address_space(1)]] int* return_value(); // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+[[clang::address_space(1)]] int* return_value(); // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
 int [[clang::address_space(1)]] * return_value();
 
-[[clang::address_space(1)]] int global1; // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
-int global2 [[clang::address_space(1)]]; // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+[[clang::address_space(1)]] int global1; // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
+int global2 [[clang::address_space(1)]]; // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
 int [[clang::address_space(1)]] global3;
 int [[clang::address_space(1)]] global4;
 
-struct [[clang::address_space(1)]] S { // expected-error {{'address_space' attribute cannot be applied to a declaration}}
-  [[clang::address_space(1)]] int* member_function_1(); // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+struct [[clang::address_space(1)]] S { // expected-error {{'clang::address_space' attribute cannot be applied to a declaration}}
+  [[clang::address_space(1)]] int* member_function_1(); // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
   int [[clang::address_space(1)]] * member_function_2();
 };
 
 template <class T>
-[[clang::address_space(1)]] T var_template_1; // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+[[clang::address_space(1)]] T var_template_1; // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
 template <class T>
 T [[clang::address_space(1)]] var_template_2;
 
-using void_ptr [[clang::address_space(1)]] = void *; // expected-warning {{applying attribute 'address_space' to a declaration is deprecated; apply it to the type instead}}
+using void_ptr [[clang::address_space(1)]] = void *; // expected-warning {{applying attribute 'clang::address_space' to a declaration is deprecated; apply it to the type instead}}
 // Intentionally using the same alias name to check that the aliases define the
 // same type.
 using void_ptr = void [[clang::address_space(1)]] *;
 
 namespace N {}
-[[clang::address_space(1)]] using namespace N; // expected-error {{'address_space' attribute cannot be applied to a declaration}}
+[[clang::address_space(1)]] using namespace N; // expected-error {{'clang::address_space' attribute cannot be applied to a declaration}}
diff --git a/clang/test/SemaCXX/annotate-type.cpp b/clang/test/SemaCXX/annotate-type.cpp
index 545b3ccfdef8e..462ca8df580c0 100644
--- a/clang/test/SemaCXX/annotate-type.cpp
+++ b/clang/test/SemaCXX/annotate-type.cpp
@@ -2,7 +2,7 @@
 
 struct S1 {
   void f() [[clang::annotate_type("foo")]];
-  [[clang::annotate_type("foo")]] void g(); // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+  [[clang::annotate_type("foo")]] void g(); // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
 };
 
 template <typename T1, typename T2> struct is_same {
@@ -40,26 +40,26 @@ int f2() { variadic_func_template<1, 2, 3>(); }
 // [[clang::annotate_type]] inside a template argument.
 template <typename Ty> void func_template();
 void f3() {
-  func_template<int [[clang::annotate_type()]]>(); // expected-error {{'annotate_type' attribute takes at least 1 argument}}
+  func_template<int [[clang::annotate_type()]]>(); // expected-error {{'clang::annotate_type' attribute takes at least 1 argument}}
 }
 
 // More error cases: Prohibit adding the attribute to declarations.
 // Different declarations hit different code paths, so they need separate tests.
-namespace [[clang::annotate_type("foo")]] my_namespace {} // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-struct [[clang::annotate_type("foo")]] S3; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-struct [[clang::annotate_type("foo")]] S3{ // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  [[clang::annotate_type("foo")]] int member; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+namespace [[clang::annotate_type("foo")]] my_namespace {} // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+struct [[clang::annotate_type("foo")]] S3; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+struct [[clang::annotate_type("foo")]] S3{ // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  [[clang::annotate_type("foo")]] int member; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
 };
 void f4() {
-  for ([[clang::annotate_type("foo")]] int i = 0; i < 42; ++i) {} // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  for (; [[clang::annotate_type("foo")]] bool b = false;) {} // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  while ([[clang::annotate_type("foo")]] bool b = false) {} // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
-  if ([[clang::annotate_type("foo")]] bool b = false) {} // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+  for ([[clang::annotate_type("foo")]] int i = 0; i < 42; ++i) {} // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  for (; [[clang::annotate_type("foo")]] bool b = false;) {} // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  while ([[clang::annotate_type("foo")]] bool b = false) {} // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
+  if ([[clang::annotate_type("foo")]] bool b = false) {} // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
   try {
-  } catch ([[clang::annotate_type("foo")]] int i) { // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+  } catch ([[clang::annotate_type("foo")]] int i) { // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
   }
 }
 template <class T>
-[[clang::annotate_type("foo")]] T var_template; // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+[[clang::annotate_type("foo")]] T var_template; // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
 [[clang::annotate_type("foo")]] extern "C" int extern_c_func(); // expected-error {{an attribute list cannot appear here}}
-extern "C" [[clang::annotate_type("foo")]] int extern_c_func(); // expected-error {{'annotate_type' attribute cannot be applied to a declaration}}
+extern "C" [[clang::annotate_type("foo")]] int extern_c_func(); // expected-error {{'clang::annotate_type' attribute cannot be applied to a declaration}}
diff --git a/clang/test/SemaCXX/attr-annotate.cpp b/clang/test/SemaCXX/attr-annotate.cpp
index ff538fee9cb3c..6f5ba7919927f 100644
--- a/clang/test/SemaCXX/attr-annotate.cpp
+++ b/clang/test/SemaCXX/attr-annotate.cpp
@@ -68,10 +68,10 @@ struct B {
     // expected-note@-2 {{is not allowed in a constant expression}}
     [[clang::annotate("qdwqwd", cf, cb)]] void t() {}
     [[clang::annotate("qdwqwd", f, cb)]] void t1() {}
-    // expected-error@-1 {{'annotate' attribute requires parameter 1 to be a constant expression}}
+    // expected-error@-1 {{'clang::annotate' attribute requires parameter 1 to be a constant expression}}
     // expected-note@-2 {{is not allowed in a constant expression}}
     [[clang::annotate("jui", b, cf)]] void t2() {}
-    // expected-error@-1 {{'annotate' attribute requires parameter 1 to be a constant expression}}
+    // expected-error@-1 {{'clang::annotate' attribute requires parameter 1 to be a constant expression}}
     // expected-note@-2 {{is not allowed in a constant expression}}
     [[clang::annotate("jui", ((void)b, 0), cf)]] [[clang::annotate("jui", &b, cf, &foo::t2, str())]] void t3() {}
   };
@@ -88,7 +88,7 @@ template<int I>
 int f() {
   [[clang::annotate("test", I)]] int v = 0; // expected-note {{declared here}}
   [[clang::annotate("test", v)]] int v2 = 0;
-  // expected-error@-1 {{'annotate' attribute requires parameter 1 to be a constant expression}}
+  // expected-error@-1 {{'clang::annotate' attribute requires parameter 1 to be a constant expression}}
   // expected-note@-2 {{is not allowed in a constant expression}}
   [[clang::annotate("test", rtyui)]] int v3 = 0;
     // expected-error@-1 {{use of undeclared identifier 'rtyui'}}
@@ -104,7 +104,7 @@ void f() {
   int vla[n];
 
   [[clang::annotate("vlas are awful", sizeof(vla))]] int i = 0; // reject, the sizeof is not unevaluated
-  // expected-error@-1 {{'annotate' attribute requires parameter 1 to be a constant expression}}
+  // expected-error@-1 {{'clang::annotate' attribute requires parameter 1 to be a constant expression}}
   // expected-note@-2 {{subexpression not valid in a constant expression}}
   [[clang::annotate("_Generic selection expression should be fine", _Generic(n, int : 0, default : 1))]]
   int j = 0; // second arg should resolve to 0 fine
@@ -126,13 +126,13 @@ constexpr int foldable_but_invalid() {
 }
 
 [[clang::annotate("", foldable_but_invalid())]] void f1() {}
-// expected-error@-1 {{'annotate' attribute requires parameter 1 to be a constant expression}}
+// expected-error@-1 {{'clang::annotate' attribute requires parameter 1 to be a constant expression}}
 
 [[clang::annotate()]] void f2() {}
-// expected-error@-1 {{'annotate' attribute takes at least 1 argument}}
+// expected-error@-1 {{'clang::annotate' attribute takes at least 1 argument}}
 
 template <typename T> [[clang::annotate()]] void f2() {}
-// expected-error@-1 {{'annotate' attribute takes at least 1 argument}}
+// expected-error@-1 {{'clang::annotate' attribute takes at least 1 argument}}
 }
 
 namespace test5 {
diff --git a/clang/test/SemaCXX/attr-cxx0x.cpp b/clang/test/SemaCXX/attr-cxx0x.cpp
index 58180a21ca8e9..5b087be908a1f 100644
--- a/clang/test/SemaCXX/attr-cxx0x.cpp
+++ b/clang/test/SemaCXX/attr-cxx0x.cpp
@@ -61,8 +61,8 @@ void b([[deprecated, maybe_unused]] void) {}     // expected-warning {{attribute
                                                  // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}} \
                                                  // expected-warning {{use of the 'deprecated' attribute is a C++14 extension}} \
                                                  // expected-warning {{use of the 'maybe_unused' attribute is a C++17 extension}}
-void c([[clang::lifetimebound]] void) {}         // expected-warning {{attribute 'lifetimebound' cannot be applied to a 'void' parameter}}
-void d([[clang::annotate("a", "b", 1)]] void) {} // expected-warning {{attribute 'annotate' cannot be applied to a 'void' parameter}}
+void c([[clang::lifetimebound]] void) {}         // expected-warning {{attribute 'clang::lifetimebound' cannot be applied to a 'void' parameter}}
+void d([[clang::annotate("a", "b", 1)]] void) {} // expected-warning {{attribute 'clang::annotate' cannot be applied to a 'void' parameter}}
 
 struct S {
   void e([[maybe_unused]] void) {} // expected-warning {{attribute 'maybe_unused' cannot be applied to a 'void' parameter}} \
diff --git a/clang/test/SemaCXX/attr-declspec-ignored.cpp b/clang/test/SemaCXX/attr-declspec-ignored.cpp
index 98e0ffd1a1afd..85f92b89b9621 100644
--- a/clang/test/SemaCXX/attr-declspec-ignored.cpp
+++ b/clang/test/SemaCXX/attr-declspec-ignored.cpp
@@ -33,18 +33,18 @@ namespace test1 {
     // Also test [[]] attribute syntax. (On a non-nested declaration, these
     // generate a hard "misplaced attributes" error, which we test for
     // elsewhere.)
-    [[gnu::visibility("hidden")]]  [[gnu::aligned]] class E; // expected-warning{{attribute 'visibility' is ignored, place it after "class" to apply attribute to type declaration}} \
-    // expected-warning{{attribute 'aligned' is ignored, place it after "class" to apply attribute to type declaration}}
-    [[gnu::visibility("hidden")]]  [[gnu::aligned]] struct F; // expected-warning{{attribute 'visibility' is ignored, place it after "struct" to apply attribute to type declaration}} \
-    // expected-warning{{attribute 'aligned' is ignored, place it after "struct" to apply attribute to type declaration}}
-    [[gnu::visibility("hidden")]]  [[gnu::aligned]] union G; // expected-warning{{attribute 'visibility' is ignored, place it after "union" to apply attribute to type declaration}} \
-    // expected-warning{{attribute 'aligned' is ignored, place it after "union" to apply attribute to type declaration}}
-    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum H {H}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum" to apply attribute to type declaration}} \
-    // expected-warning{{attribute 'aligned' is ignored, place it after "enum" to apply attribute to type declaration}}
-    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum class I {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum class" to apply attribute to type declaration}} \
-    // expected-warning{{attribute 'aligned' is ignored, place it after "enum class" to apply attribute to type declaration}}
-    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum struct J {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum struct" to apply attribute to type declaration}} \
-    // expected-warning{{attribute 'aligned' is ignored, place it after "enum struct" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] class E; // expected-warning{{attribute 'gnu::visibility' is ignored, place it after "class" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'gnu::aligned' is ignored, place it after "class" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] struct F; // expected-warning{{attribute 'gnu::visibility' is ignored, place it after "struct" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'gnu::aligned' is ignored, place it after "struct" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] union G; // expected-warning{{attribute 'gnu::visibility' is ignored, place it after "union" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'gnu::aligned' is ignored, place it after "union" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum H {H}; // expected-warning{{attribute 'gnu::visibility' is ignored, place it after "enum" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'gnu::aligned' is ignored, place it after "enum" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum class I {}; // expected-warning{{attribute 'gnu::visibility' is ignored, place it after "enum class" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'gnu::aligned' is ignored, place it after "enum class" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum struct J {}; // expected-warning{{attribute 'gnu::visibility' is ignored, place it after "enum struct" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'gnu::aligned' is ignored, place it after "enum struct" to apply attribute to type declaration}}
   };
 }
 
diff --git a/clang/test/SemaCXX/attr-deprecated-replacement-error.cpp b/clang/test/SemaCXX/attr-deprecated-replacement-error.cpp
index 8c9d9b29c5535..48886ae4c80ac 100644
--- a/clang/test/SemaCXX/attr-deprecated-replacement-error.cpp
+++ b/clang/test/SemaCXX/attr-deprecated-replacement-error.cpp
@@ -7,7 +7,7 @@
 int a1 [[deprecated("warning", "fixit")]]; // expected-error{{'deprecated' attribute takes no more than 1 argument}}
 int a2 [[deprecated("warning", 1)]]; // expected-error{{expected string literal as argument of 'deprecated' attribute}}
 
-int b1 [[gnu::deprecated("warning", "fixit")]]; // expected-error{{'deprecated' attribute takes no more than 1 argument}}
+int b1 [[gnu::deprecated("warning", "fixit")]]; // expected-error{{'gnu::deprecated' attribute takes no more than 1 argument}}
 int b2 [[gnu::deprecated("warning", 1)]]; // expected-error{{expected string literal as argument of 'deprecated' attribute}}
 
 __declspec(deprecated("warning", "fixit")) int c1; // expected-error{{'deprecated' attribute takes no more than 1 argument}}
diff --git a/clang/test/SemaCXX/attr-flatten.cpp b/clang/test/SemaCXX/attr-flatten.cpp
index afcba72b6429e..6dce8fdd154f8 100644
--- a/clang/test/SemaCXX/attr-flatten.cpp
+++ b/clang/test/SemaCXX/attr-flatten.cpp
@@ -16,19 +16,19 @@ struct A {
   static void mf2() __attribute__((flatten));
 };
 
-int ci [[gnu::flatten]]; // expected-error {{'flatten' attribute only applies to functions}}
+int ci [[gnu::flatten]]; // expected-error {{'gnu::flatten' attribute only applies to functions}}
 
 [[gnu::flatten]] void cf1();
-[[gnu::flatten(1)]] void cf2(); // expected-error {{'flatten' attribute takes no arguments}}
+[[gnu::flatten(1)]] void cf2(); // expected-error {{'gnu::flatten' attribute takes no arguments}}
 
 template <typename T>
 [[gnu::flatten]]
 void ctf1();
 
-int cf3(int c[[gnu::flatten]], int); // expected-error{{'flatten' attribute only applies to functions}}
+int cf3(int c[[gnu::flatten]], int); // expected-error{{'gnu::flatten' attribute only applies to functions}}
 
 struct CA {
-  int f [[gnu::flatten]];  // expected-error{{'flatten' attribute only applies to functions}}
+  int f [[gnu::flatten]];  // expected-error{{'gnu::flatten' attribute only applies to functions}}
   [[gnu::flatten]] void mf1();
   [[gnu::flatten]] static void mf2();
 };
diff --git a/clang/test/SemaCXX/attr-gsl-owner-pointer.cpp b/clang/test/SemaCXX/attr-gsl-owner-pointer.cpp
index 6c1bfe405e869..4c33d5c0f5a9d 100644
--- a/clang/test/SemaCXX/attr-gsl-owner-pointer.cpp
+++ b/clang/test/SemaCXX/attr-gsl-owner-pointer.cpp
@@ -2,21 +2,21 @@
 // RUN: FileCheck --implicit-check-not OwnerAttr --implicit-check-not PointerAttr %s
 
 int [[gsl::Owner]] i;
-// expected-error@-1 {{'Owner' attribute cannot be applied to types}}
+// expected-error@-1 {{'gsl::Owner' attribute cannot be applied to types}}
 void [[gsl::Owner]] f();
-// expected-error@-1 {{'Owner' attribute cannot be applied to types}}
+// expected-error@-1 {{'gsl::Owner' attribute cannot be applied to types}}
 
 [[gsl::Owner]] void f();
-// expected-warning@-1 {{'Owner' attribute only applies to structs}}
+// expected-warning@-1 {{'gsl::Owner' attribute only applies to structs}}
 
 union [[gsl::Owner(int)]] Union{};
-// expected-warning@-1 {{'Owner' attribute only applies to structs}}
+// expected-warning@-1 {{'gsl::Owner' attribute only applies to structs}}
 
 struct S {
 };
 
 S [[gsl::Owner]] Instance;
-// expected-error@-1 {{'Owner' attribute cannot be applied to types}}
+// expected-error@-1 {{'gsl::Owner' attribute cannot be applied to types}}
 
 class [[gsl::Owner(7)]] OwnerDerefNoType{};
 // expected-error@-1 {{expected a type}}
@@ -25,7 +25,7 @@ class [[gsl::Pointer("int")]] PointerDerefNoType{};
 // expected-error@-1 {{expected a type}}
 
 class [[gsl::Owner(int)]] [[gsl::Pointer(int)]] BothOwnerPointer{};
-// expected-error@-1 {{'Pointer' and 'Owner' attributes are not compatible}}
+// expected-error@-1 {{'gsl::Pointer' and 'gsl::Owner' attributes are not compatible}}
 // expected-note@-2 {{conflicting attribute is here}}
 // CHECK: CXXRecordDecl {{.*}} BothOwnerPointer
 // CHECK: OwnerAttr {{.*}} int
@@ -41,7 +41,7 @@ class [[gsl::Pointer(int)]] AddConflictLater{};
 // CHECK: CXXRecordDecl {{.*}} AddConflictLater
 // CHECK: PointerAttr {{.*}} int
 class [[gsl::Owner(int)]] AddConflictLater;
-// expected-error@-1 {{'Owner' and 'Pointer' attributes are not compatible}}
+// expected-error@-1 {{'gsl::Owner' and 'gsl::Pointer' attributes are not compatible}}
 // expected-note@-5 {{conflicting attribute is here}}
 // CHECK: CXXRecordDecl {{.*}} AddConflictLater
 // CHECK: PointerAttr {{.*}} Inherited int
@@ -50,22 +50,22 @@ class [[gsl::Owner(int)]] AddConflictLater2{};
 // CHECK: CXXRecordDecl {{.*}} AddConflictLater2
 // CHECK: OwnerAttr {{.*}} int
 class [[gsl::Owner(float)]] AddConflictLater2;
-// expected-error@-1 {{'Owner' and 'Owner' attributes are not compatible}}
+// expected-error@-1 {{'gsl::Owner' and 'gsl::Owner' attributes are not compatible}}
 // expected-note@-5 {{conflicting attribute is here}}
 // CHECK: CXXRecordDecl {{.*}} AddConflictLater
 // CHECK: OwnerAttr {{.*}} Inherited int
 
 class [[gsl::Owner()]] [[gsl::Owner(int)]] WithAndWithoutParameter{};
-// expected-error@-1 {{'Owner' and 'Owner' attributes are not compatible}}
+// expected-error@-1 {{'gsl::Owner' and 'gsl::Owner' attributes are not compatible}}
 // expected-note@-2 {{conflicting attribute is here}}
 // CHECK: CXXRecordDecl {{.*}} WithAndWithoutParameter
 // CHECK: OwnerAttr
 
 class [[gsl::Owner(int &)]] ReferenceType{};
-// expected-error@-1 {{a reference type is an invalid argument to attribute 'Owner'}}
+// expected-error@-1 {{a reference type is an invalid argument to attribute 'gsl::Owner'}}
 
 class [[gsl::Pointer(int[])]] ArrayType{};
-// expected-error@-1 {{an array type is an invalid argument to attribute 'Pointer'}}
+// expected-error@-1 {{an array type is an invalid argument to attribute 'gsl::Pointer'}}
 
 class [[gsl::Owner]] OwnerMissingParameter{};
 // CHECK: CXXRecordDecl {{.*}} OwnerMissingParameter
diff --git a/clang/test/SemaCXX/attr-lifetime-capture-by.cpp b/clang/test/SemaCXX/attr-lifetime-capture-by.cpp
index 03ab425331720..70a5fe5a45376 100644
--- a/clang/test/SemaCXX/attr-lifetime-capture-by.cpp
+++ b/clang/test/SemaCXX/attr-lifetime-capture-by.cpp
@@ -8,7 +8,7 @@ struct S {
 ///////////////////////////
 // Test for valid usages.
 ///////////////////////////
-[[clang::lifetime_capture_by(unknown)]] // expected-error {{'lifetime_capture_by' attribute only applies to parameters and implicit object parameters}}
+[[clang::lifetime_capture_by(unknown)]] // expected-error {{'clang::lifetime_capture_by' attribute only applies to parameters and implicit object parameters}}
 void nonMember(
     const int &x1 [[clang::lifetime_capture_by(s, t)]],
     S &s,
diff --git a/clang/test/SemaCXX/attr-lifetimebound.cpp b/clang/test/SemaCXX/attr-lifetimebound.cpp
index 2bb88171bdfe4..06532550d312e 100644
--- a/clang/test/SemaCXX/attr-lifetimebound.cpp
+++ b/clang/test/SemaCXX/attr-lifetimebound.cpp
@@ -10,24 +10,24 @@ namespace usage_invalid {
     static int *static_class_member() [[clang::lifetimebound]]; // expected-error {{static member function has no implicit object parameter}}
     int *explicit_object(this A&) [[clang::lifetimebound]]; // expected-error {{explicit object member function has no implicit object parameter}}
     int attr_on_var [[clang::lifetimebound]]; // expected-error {{only applies to parameters and implicit object parameters}}
-    int [[clang::lifetimebound]] attr_on_int; // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
-    int * [[clang::lifetimebound]] attr_on_int_ptr; // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
-    int * [[clang::lifetimebound]] * attr_on_int_ptr_ptr; // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
-    int (* [[clang::lifetimebound]] attr_on_func_ptr)(); // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
+    int [[clang::lifetimebound]] attr_on_int; // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
+    int * [[clang::lifetimebound]] attr_on_int_ptr; // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
+    int * [[clang::lifetimebound]] * attr_on_int_ptr_ptr; // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
+    int (* [[clang::lifetimebound]] attr_on_func_ptr)(); // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
     void void_return_member() [[clang::lifetimebound]]; // expected-error {{'lifetimebound' attribute cannot be applied to an implicit object parameter of a function that returns void; did you mean 'lifetime_capture_by(X)'}}
   };
   int *attr_with_param(int &param [[clang::lifetimebound(42)]]); // expected-error {{takes no arguments}}
 
-  void attr_on_ptr_arg(int * [[clang::lifetimebound]] ptr); // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
-  static_assert((int [[clang::lifetimebound]]) 12); // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
-  int* attr_on_unnamed_arg(const int& [[clang::lifetimebound]]); // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  void attr_on_ptr_arg(int * [[clang::lifetimebound]] ptr); // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  static_assert((int [[clang::lifetimebound]]) 12); // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  int* attr_on_unnamed_arg(const int& [[clang::lifetimebound]]); // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
   template <typename T>
-  int* attr_on_template_ptr_arg(T * [[clang::lifetimebound]] ptr); // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  int* attr_on_template_ptr_arg(T * [[clang::lifetimebound]] ptr); // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
 
-  int (*func_ptr)(int) [[clang::lifetimebound]]; // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
-  int (*(*func_ptr_ptr)(int) [[clang::lifetimebound]])(int); // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  int (*func_ptr)(int) [[clang::lifetimebound]]; // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  int (*(*func_ptr_ptr)(int) [[clang::lifetimebound]])(int); // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
   struct X {};
-  int (X::*member_func_ptr)(int) [[clang::lifetimebound]]; // expected-error {{'lifetimebound' attribute only applies to parameters and implicit object parameters}}
+  int (X::*member_func_ptr)(int) [[clang::lifetimebound]]; // expected-error {{'clang::lifetimebound' attribute only applies to parameters and implicit object parameters}}
 }
 
 namespace usage_ok {
diff --git a/clang/test/SemaCXX/attr-lto-visibility-public.cpp b/clang/test/SemaCXX/attr-lto-visibility-public.cpp
index 42e0f79279d9b..9f1f2f4d9c367 100644
--- a/clang/test/SemaCXX/attr-lto-visibility-public.cpp
+++ b/clang/test/SemaCXX/attr-lto-visibility-public.cpp
@@ -1,14 +1,14 @@
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
 
-int i [[clang::lto_visibility_public]]; // expected-warning {{'lto_visibility_public' attribute only applies to structs, unions, and classes}}
-typedef int t [[clang::lto_visibility_public]]; // expected-warning {{'lto_visibility_public' attribute only applies to}}
-[[clang::lto_visibility_public]] void f(); // expected-warning {{'lto_visibility_public' attribute only applies to}}
-void f() [[clang::lto_visibility_public]]; // expected-error {{'lto_visibility_public' attribute cannot be applied to types}}
+int i [[clang::lto_visibility_public]]; // expected-warning {{'clang::lto_visibility_public' attribute only applies to structs, unions, and classes}}
+typedef int t [[clang::lto_visibility_public]]; // expected-warning {{'clang::lto_visibility_public' attribute only applies to}}
+[[clang::lto_visibility_public]] void f(); // expected-warning {{'clang::lto_visibility_public' attribute only applies to}}
+void f() [[clang::lto_visibility_public]]; // expected-error {{'clang::lto_visibility_public' attribute cannot be applied to types}}
 
 struct [[clang::lto_visibility_public]] s1 {
-  int i [[clang::lto_visibility_public]]; // expected-warning {{'lto_visibility_public' attribute only applies to}}
-  [[clang::lto_visibility_public]] void f(); // expected-warning {{'lto_visibility_public' attribute only applies to}}
+  int i [[clang::lto_visibility_public]]; // expected-warning {{'clang::lto_visibility_public' attribute only applies to}}
+  [[clang::lto_visibility_public]] void f(); // expected-warning {{'clang::lto_visibility_public' attribute only applies to}}
 };
 
-struct [[clang::lto_visibility_public(1)]] s2 { // expected-error {{'lto_visibility_public' attribute takes no arguments}}
+struct [[clang::lto_visibility_public(1)]] s2 { // expected-error {{'clang::lto_visibility_public' attribute takes no arguments}}
 };
diff --git a/clang/test/SemaCXX/attr-musttail.cpp b/clang/test/SemaCXX/attr-musttail.cpp
index 12cfd89c64a6f..b88f031c64574 100644
--- a/clang/test/SemaCXX/attr-musttail.cpp
+++ b/clang/test/SemaCXX/attr-musttail.cpp
@@ -2,33 +2,33 @@
 
 int ReturnsInt1();
 int Func1() {
-  [[clang::musttail]] ReturnsInt1();              // expected-error {{'musttail' attribute only applies to return statements}}
-  [[clang::musttail(1, 2)]] return ReturnsInt1(); // expected-error {{'musttail' attribute takes no arguments}}
-  [[clang::musttail]] return 5;                   // expected-error {{'musttail' attribute requires that the return value is the result of a function call}}
+  [[clang::musttail]] ReturnsInt1();              // expected-error {{'clang::musttail' attribute only applies to return statements}}
+  [[clang::musttail(1, 2)]] return ReturnsInt1(); // expected-error {{'clang::musttail' attribute takes no arguments}}
+  [[clang::musttail]] return 5;                   // expected-error {{'clang::musttail' attribute requires that the return value is the result of a function call}}
   [[clang::musttail]] return ReturnsInt1();
 }
 
 void NoFunctionCall() {
-  [[clang::musttail]] return; // expected-error {{'musttail' attribute requires that the return value is the result of a function call}}
+  [[clang::musttail]] return; // expected-error {{'clang::musttail' attribute requires that the return value is the result of a function call}}
 }
 
-[[clang::musttail]] static int int_val = ReturnsInt1(); // expected-error {{'musttail' attribute cannot be applied to a declaration}}
+[[clang::musttail]] static int int_val = ReturnsInt1(); // expected-error {{'clang::musttail' attribute cannot be applied to a declaration}}
 
 void NoParams(); // expected-note {{target function has different number of parameters (expected 1 but has 0)}}
 void TestParamArityMismatch(int x) {
-  [[clang::musttail]] // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]] // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return NoParams();  // expected-error {{cannot perform a tail call to function 'NoParams' because its signature is incompatible with the calling function}}
 }
 
 void LongParam(long x); // expected-note {{target function has type mismatch at 1st parameter (expected 'long' but has 'int')}}
 void TestParamTypeMismatch(int x) {
-  [[clang::musttail]]  // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]  // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return LongParam(x); // expected-error {{cannot perform a tail call to function 'LongParam' because its signature is incompatible with the calling function}}
 }
 
 long ReturnsLong(); // expected-note {{target function has different return type ('int' expected but has 'long')}}
 int TestReturnTypeMismatch() {
-  [[clang::musttail]]   // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]   // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return ReturnsLong(); // expected-error {{cannot perform a tail call to function 'ReturnsLong' because its signature is incompatible with the calling function}}
 }
 
@@ -37,14 +37,14 @@ struct Struct1 {
 };
 void TestNonMemberToMember() {
   Struct1 st;
-  [[clang::musttail]]         // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]         // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return st.MemberFunction(); // expected-error {{non-member function cannot perform a tail call to non-static member function 'MemberFunction'}}
 }
 
 void ReturnsVoid(); // expected-note {{'ReturnsVoid' declared here}}
 struct Struct2 {
   void TestMemberToNonMember() {
-    [[clang::musttail]]   // expected-note {{tail call required by 'musttail' attribute here}}
+    [[clang::musttail]]   // expected-note {{tail call required by 'clang::musttail' attribute here}}
     return ReturnsVoid(); // expected-error{{non-static member function cannot perform a tail call to non-member function 'ReturnsVoid'}}
   }
 };
@@ -75,7 +75,7 @@ HasNonTrivialDestructor TestReturnsNonTrivialValue() {
 }
 
 HasNonTrivialDestructor TestReturnsNonTrivialNonFunctionCall() {
-  [[clang::musttail]] return HasNonTrivialDestructor(); // expected-error {{'musttail' attribute requires that the return value is the result of a function call}}
+  [[clang::musttail]] return HasNonTrivialDestructor(); // expected-error {{'clang::musttail' attribute requires that the return value is the result of a function call}}
 }
 
 struct UsesPointerToMember {
@@ -83,7 +83,7 @@ struct UsesPointerToMember {
 };
 void TestUsesPointerToMember(UsesPointerToMember *foo) {
   // "this" pointer cannot double as first parameter.
-  [[clang::musttail]]            // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]            // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return (foo->*(foo->p_mem))(); // expected-error {{non-member function cannot perform a tail call to pointer-to-member function 'p_mem'}}
 }
 
@@ -107,7 +107,7 @@ int OkTemplateFunc(int x) {
 }
 template <class T>
 T BadTemplateFunc(T x) {
-  [[clang::musttail]]          // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]          // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return TemplateFunc<int>(x); // expected-error {{cannot perform a tail call to function 'TemplateFunc' because its signature is incompatible with the calling function}}
 }
 long TestBadTemplateFunc(long x) {
@@ -126,17 +126,17 @@ void TestNonTrivialDestructorSubArg(int x) {
 
 void VariadicFunction(int x, ...);
 void TestVariadicFunction(int x, ...) {
-  [[clang::musttail]] return VariadicFunction(x); // expected-error {{'musttail' attribute may not be used with variadic functions}}
+  [[clang::musttail]] return VariadicFunction(x); // expected-error {{'clang::musttail' attribute may not be used with variadic functions}}
 }
 
 int TakesIntParam(int x);     // expected-note {{target function has type mismatch at 1st parameter (expected 'int' but has 'short')}}
 int TakesShortParam(short x); // expected-note {{target function has type mismatch at 1st parameter (expected 'short' but has 'int')}}
 int TestIntParamMismatch(int x) {
-  [[clang::musttail]]        // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]        // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return TakesShortParam(x); // expected-error {{cannot perform a tail call to function 'TakesShortParam' because its signature is incompatible with the calling function}}
 }
 int TestIntParamMismatch2(short x) {
-  [[clang::musttail]]      // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]      // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return TakesIntParam(x); // expected-error {{cannot perform a tail call to function 'TakesIntParam' because its signature is incompatible with the calling function}}
 }
 
@@ -146,20 +146,20 @@ struct TestClassMismatch1 {
 TestClassMismatch1 *tcm1;
 struct TestClassMismatch2 {
   void FromFunction() {
-    [[clang::musttail]]        // expected-note {{tail call required by 'musttail' attribute here}}
+    [[clang::musttail]]        // expected-note {{tail call required by 'clang::musttail' attribute here}}
     return tcm1->ToFunction(); // expected-error {{cannot perform a tail call to function 'ToFunction' because its signature is incompatible with the calling function}}
   }
 };
 
 __regcall int RegCallReturnsInt(); // expected-note {{target function has calling convention regcall (expected cdecl)}}
 int TestMismatchCallingConvention() {
-  [[clang::musttail]]         // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]         // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return RegCallReturnsInt(); // expected-error {{cannot perform a tail call to function 'RegCallReturnsInt' because it uses an incompatible calling convention}}
 }
 
 int TestNonCapturingLambda() {
   auto lambda = []() { return 12; }; // expected-note {{'operator()' declared here}}
-  [[clang::musttail]]                // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]                // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return lambda();                   // expected-error {{non-member function cannot perform a tail call to non-static member function 'operator()'}}
 
   // This works.
@@ -171,7 +171,7 @@ int TestNonCapturingLambda() {
 int TestCapturingLambda() {
   int x;
   auto lambda = [x]() { return 12; }; // expected-note {{'operator()' declared here}}
-  [[clang::musttail]]                 // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]                 // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return lambda();                    // expected-error {{non-member function cannot perform a tail call to non-static member function 'operator()'}}
 }
 
@@ -182,14 +182,14 @@ int TestNonTrivialTemporary(int) {
 void ReturnsVoid();
 struct TestDestructor {
   ~TestDestructor() {
-    [[clang::musttail]]   // expected-note {{tail call required by 'musttail' attribute here}}
+    [[clang::musttail]]   // expected-note {{tail call required by 'clang::musttail' attribute here}}
     return ReturnsVoid(); // expected-error {{destructor '~TestDestructor' must not return void expression}}  // expected-error {{cannot perform a tail call from a destructor}}
   }
 };
 
 struct ClassWithDestructor { // expected-note {{target destructor is declared here}}
   void TestExplicitDestructorCall() {
-    [[clang::musttail]]                  // expected-note {{tail call required by 'musttail' attribute here}}
+    [[clang::musttail]]                  // expected-note {{tail call required by 'clang::musttail' attribute here}}
     return this->~ClassWithDestructor(); // expected-error {{cannot perform a tail call to a destructor}}
   }
 };
@@ -201,14 +201,14 @@ HasNonTrivialCopyConstructor ReturnsClassByValue();
 HasNonTrivialCopyConstructor TestNonElidableCopyConstructor() {
   // This is an elidable constructor, but when it is written explicitly
   // we decline to elide it.
-  [[clang::musttail]] return HasNonTrivialCopyConstructor(ReturnsClassByValue()); // expected-error{{'musttail' attribute requires that the return value is the result of a function call}}
+  [[clang::musttail]] return HasNonTrivialCopyConstructor(ReturnsClassByValue()); // expected-error{{'clang::musttail' attribute requires that the return value is the result of a function call}}
 }
 
 struct ClassWithConstructor {
   ClassWithConstructor() = default; // expected-note {{target constructor is declared here}}
 };
 void TestExplicitConstructorCall(ClassWithConstructor a) {
-  [[clang::musttail]]                                    // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]                                    // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return a.ClassWithConstructor::ClassWithConstructor(); // expected-error{{cannot perform a tail call to a constructor}}  expected-warning{{explicit constructor calls are a Microsoft extension}}
 }
 
@@ -230,14 +230,14 @@ void TestTryBlock() {
 using IntFunctionType = int();
 IntFunctionType *ReturnsIntFunction();
 long TestRValueFunctionPointer() {
-  [[clang::musttail]]            // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]            // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return ReturnsIntFunction()(); // expected-error{{cannot perform a tail call to function because its signature is incompatible with the calling function}}  // expected-note{{target function has different return type ('long' expected but has 'int')}}
 }
 
 void TestPseudoDestructor() {
   int n;
   using T = int;
-  [[clang::musttail]] // expected-note {{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]] // expected-note {{tail call required by 'clang::musttail' attribute here}}
   return n.~T();      // expected-error{{cannot perform a tail call to a destructor}}
 }
 
@@ -249,7 +249,7 @@ struct StructPMF {
 StructPMF *St;
 StructPMF::PMF ReturnsPMF();
 void StructPMF::TestReturnsPMF() {
-  [[clang::musttail]]           // expected-note{{tail call required by 'musttail' attribute here}}
+  [[clang::musttail]]           // expected-note{{tail call required by 'clang::musttail' attribute here}}
   return (St->*ReturnsPMF())(); // expected-error{{static member function cannot perform a tail call to pointer-to-member function}}
 }
 
diff --git a/clang/test/SemaCXX/attr-no-specializations.cpp b/clang/test/SemaCXX/attr-no-specializations.cpp
index 64f6a76416605..1fb57db14a446 100644
--- a/clang/test/SemaCXX/attr-no-specializations.cpp
+++ b/clang/test/SemaCXX/attr-no-specializations.cpp
@@ -4,15 +4,15 @@
 #  error
 #endif
 
-struct [[clang::no_specializations]] S {}; // expected-warning {{'no_specializations' attribute only applies to class templates, function templates, and variable templates}}
+struct [[clang::no_specializations]] S {}; // expected-warning {{'clang::no_specializations' attribute only applies to class templates, function templates, and variable templates}}
 
 template <class T, class U>
-struct [[clang::no_specializations]] is_same { // expected-note 2 {{marked 'no_specializations' here}}
+struct [[clang::no_specializations]] is_same { // expected-note 2 {{marked 'clang::no_specializations' here}}
   static constexpr bool value = __is_same(T, U);
 };
 
 template <class T>
-using alias [[clang::no_specializations]] = T; // expected-warning {{'no_specializations' attribute only applies to class templates, function templates, and variable templates}}
+using alias [[clang::no_specializations]] = T; // expected-warning {{'clang::no_specializations' attribute only applies to class templates, function templates, and variable templates}}
 
 template <>
 struct is_same<int, char> {}; // expected-error {{'is_same' cannot be specialized}}
@@ -26,7 +26,7 @@ struct is_same<Template<T>, Template <T>> {}; // expected-error {{'is_same' cann
 bool test_instantiation1 = is_same<int, int>::value;
 
 template <class T, class U>
-[[clang::no_specializations]] inline constexpr bool is_same_v = __is_same(T, U); // expected-note 2 {{marked 'no_specializations' here}}
+[[clang::no_specializations]] inline constexpr bool is_same_v = __is_same(T, U); // expected-note 2 {{marked 'clang::no_specializations' here}}
 
 template <>
 inline constexpr bool is_same_v<int, char> = false; // expected-error {{'is_same_v' cannot be specialized}}
@@ -37,7 +37,7 @@ inline constexpr bool is_same_v<Template <T>, Template <T>> = true; // expected-
 bool test_instantiation2 = is_same_v<int, int>;
 
 template <class T>
-struct [[clang::no_specializations("specializing type traits results in undefined behaviour")]] is_trivial { // expected-note {{marked 'no_specializations' here}}
+struct [[clang::no_specializations("specializing type traits results in undefined behaviour")]] is_trivial { // expected-note {{marked 'clang::no_specializations' here}}
   static constexpr bool value = __is_trivial(T);
 };
 
@@ -45,7 +45,7 @@ template <>
 struct is_trivial<int> {}; // expected-error {{'is_trivial' cannot be specialized: specializing type traits results in undefined behaviour}}
 
 template <class T>
-[[clang::no_specializations("specializing type traits results in undefined behaviour")]] inline constexpr bool is_trivial_v = __is_trivial(T); // expected-note {{marked 'no_specializations' here}}
+[[clang::no_specializations("specializing type traits results in undefined behaviour")]] inline constexpr bool is_trivial_v = __is_trivial(T); // expected-note {{marked 'clang::no_specializations' here}}
 
 template <>
 inline constexpr bool is_trivial_v<int> = false; // expected-error {{'is_trivial_v' cannot be specialized: specializing type traits results in undefined behaviour}}
@@ -54,9 +54,9 @@ template <class T>
 struct Partial {};
 
 template <class T>
-struct [[clang::no_specializations]] Partial<Template <T>> {}; // expected-warning {{'no_specializations' attribute only applies to class templates, function templates, and variable templates}}
+struct [[clang::no_specializations]] Partial<Template <T>> {}; // expected-warning {{'clang::no_specializations' attribute only applies to class templates, function templates, and variable templates}}
 
 template <class T>
-[[clang::no_specializations]] void func(); // expected-note {{marked 'no_specializations' here}}
+[[clang::no_specializations]] void func(); // expected-note {{marked 'clang::no_specializations' here}}
 
 template <> void func<int>(); // expected-error {{'func' cannot be specialized}}
diff --git a/clang/test/SemaCXX/attr-no-speculative-load-hardening.cpp b/clang/test/SemaCXX/attr-no-speculative-load-hardening.cpp
index eebc5278f2fc8..fd43cbbf0f977 100644
--- a/clang/test/SemaCXX/attr-no-speculative-load-hardening.cpp
+++ b/clang/test/SemaCXX/attr-no-speculative-load-hardening.cpp
@@ -16,19 +16,19 @@ struct A {
   static void mf2() __attribute__((no_speculative_load_hardening));
 };
 
-int ci [[clang::no_speculative_load_hardening]]; // expected-error {{'no_speculative_load_hardening' attribute only applies to functions}}
+int ci [[clang::no_speculative_load_hardening]]; // expected-error {{'clang::no_speculative_load_hardening' attribute only applies to functions}}
 
 [[clang::no_speculative_load_hardening]] void cf1();
-[[clang::no_speculative_load_hardening(1)]] void cf2(); // expected-error {{'no_speculative_load_hardening' attribute takes no arguments}}
+[[clang::no_speculative_load_hardening(1)]] void cf2(); // expected-error {{'clang::no_speculative_load_hardening' attribute takes no arguments}}
 
 template <typename T>
 [[clang::no_speculative_load_hardening]]
 void ctf1();
 
-int cf3(int c[[clang::no_speculative_load_hardening]], int); // expected-error {{'no_speculative_load_hardening' attribute only applies to functions}}
+int cf3(int c[[clang::no_speculative_load_hardening]], int); // expected-error {{'clang::no_speculative_load_hardening' attribute only applies to functions}}
 
 struct CA {
-  int f [[clang::no_speculative_load_hardening]];  // expected-error {{'no_speculative_load_hardening' attribute only applies to functions}}
+  int f [[clang::no_speculative_load_hardening]];  // expected-error {{'clang::no_speculative_load_hardening' attribute only applies to functions}}
   [[clang::no_speculative_load_hardening]] void mf1();
   [[clang::no_speculative_load_hardening]] static void mf2();
 };
diff --git a/clang/test/SemaCXX/attr-no-split-stack.cpp b/clang/test/SemaCXX/attr-no-split-stack.cpp
index 3575e9983f41c..e5263da56bebf 100644
--- a/clang/test/SemaCXX/attr-no-split-stack.cpp
+++ b/clang/test/SemaCXX/attr-no-split-stack.cpp
@@ -16,19 +16,19 @@ struct A {
   static void mf2() __attribute__((no_split_stack));
 };
 
-int ci [[gnu::no_split_stack]]; // expected-error {{'no_split_stack' attribute only applies to functions}}
+int ci [[gnu::no_split_stack]]; // expected-error {{'gnu::no_split_stack' attribute only applies to functions}}
 
 [[gnu::no_split_stack]] void cf1();
-[[gnu::no_split_stack(1)]] void cf2(); // expected-error {{'no_split_stack' attribute takes no arguments}}
+[[gnu::no_split_stack(1)]] void cf2(); // expected-error {{'gnu::no_split_stack' attribute takes no arguments}}
 
 template <typename T>
 [[gnu::no_split_stack]]
 void ctf1();
 
-int cf3(int c[[gnu::no_split_stack]], int); // expected-error{{'no_split_stack' attribute only applies to functions}}
+int cf3(int c[[gnu::no_split_stack]], int); // expected-error{{'gnu::no_split_stack' attribute only applies to functions}}
 
 struct CA {
-  int f [[gnu::no_split_stack]];  // expected-error{{'no_split_stack' attribute only applies to functions}}
+  int f [[gnu::no_split_stack]];  // expected-error{{'gnu::no_split_stack' attribute only applies to functions}}
   [[gnu::no_split_stack]] void mf1();
   [[gnu::no_split_stack]] static void mf2();
 };
diff --git a/clang/test/SemaCXX/attr-optnone.cpp b/clang/test/SemaCXX/attr-optnone.cpp
index c8e85eec37116..7e331baedbef3 100644
--- a/clang/test/SemaCXX/attr-optnone.cpp
+++ b/clang/test/SemaCXX/attr-optnone.cpp
@@ -58,10 +58,10 @@ int bar2() __attribute__((noinline));
 [[clang::optnone]] // expected-note {{conflicting}}
 int baz2() __attribute__((always_inline)); // expected-warning{{'always_inline' attribute ignored}}
 
-[[clang::optnone]] int globalVar2; //expected-warning{{'optnone' attribute only applies to functions}}
+[[clang::optnone]] int globalVar2; //expected-warning{{'clang::optnone' attribute only applies to functions}}
 
 struct A2 {
-  [[clang::optnone]] int aField; // expected-warning{{'optnone' attribute only applies to functions}}
+  [[clang::optnone]] int aField; // expected-warning{{'clang::optnone' attribute only applies to functions}}
 };
 
 struct B2 {
@@ -78,4 +78,4 @@ struct B2 {
 [[clang::__optnone__]] int foo5();
 [[_Clang::__optnone__]] int foo6();
 
-[[_Clang::optnone]] int foo7; // expected-warning {{'optnone' attribute only applies to functions}}
+[[_Clang::optnone]] int foo7; // expected-warning {{'clang::optnone' attribute only applies to functions}}
diff --git a/clang/test/SemaCXX/attr-reinitializes.cpp b/clang/test/SemaCXX/attr-reinitializes.cpp
index 129d359b8b742..606cb29d0da16 100644
--- a/clang/test/SemaCXX/attr-reinitializes.cpp
+++ b/clang/test/SemaCXX/attr-reinitializes.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
 
-[[clang::reinitializes]] int a; // expected-error {{'reinitializes' attribute only applies to non-static non-const member functions}}
+[[clang::reinitializes]] int a; // expected-error {{'clang::reinitializes' attribute only applies to non-static non-const member functions}}
 
 [[clang::reinitializes]] void f(); // expected-error {{only applies to}}
 
@@ -11,5 +11,5 @@ struct A {
   [[clang::reinitializes]] static void baz(); // expected-error {{only applies to}}
   [[clang::reinitializes]] int a; // expected-error {{only applies to}}
 
-  [[clang::reinitializes("arg")]] void qux(); // expected-error {{'reinitializes' attribute takes no arguments}}
+  [[clang::reinitializes("arg")]] void qux(); // expected-error {{'clang::reinitializes' attribute takes no arguments}}
 };
diff --git a/clang/test/SemaCXX/attr-speculative-load-hardening.cpp b/clang/test/SemaCXX/attr-speculative-load-hardening.cpp
index 3f24069f1e84c..3c72c080eaaec 100644
--- a/clang/test/SemaCXX/attr-speculative-load-hardening.cpp
+++ b/clang/test/SemaCXX/attr-speculative-load-hardening.cpp
@@ -26,19 +26,19 @@ void f6() __attribute__((no_speculative_load_hardening)); // expected-note {{con
 
 void f6() __attribute__((speculative_load_hardening)); // expected-error {{'speculative_load_hardening' and 'no_speculative_load_hardening' attributes are not compatible}}
 
-int ci [[clang::speculative_load_hardening]]; // expected-error {{'speculative_load_hardening' attribute only applies to functions}}
+int ci [[clang::speculative_load_hardening]]; // expected-error {{'clang::speculative_load_hardening' attribute only applies to functions}}
 
 [[clang::speculative_load_hardening]] void cf1();
-[[clang::speculative_load_hardening(1)]] void cf2(); // expected-error {{'speculative_load_hardening' attribute takes no arguments}}
+[[clang::speculative_load_hardening(1)]] void cf2(); // expected-error {{'clang::speculative_load_hardening' attribute takes no arguments}}
 
 template <typename T>
 [[clang::speculative_load_hardening]]
 void ctf1();
 
-int cf3(int c[[clang::speculative_load_hardening]], int); // expected-error {{'speculative_load_hardening' attribute only applies to functions}}
+int cf3(int c[[clang::speculative_load_hardening]], int); // expected-error {{'clang::speculative_load_hardening' attribute only applies to functions}}
 
 struct CA {
-  int f [[clang::speculative_load_hardening]];  // expected-error {{'speculative_load_hardening' attribute only applies to functions}}
+  int f [[clang::speculative_load_hardening]];  // expected-error {{'clang::speculative_load_hardening' attribute only applies to functions}}
   [[clang::speculative_load_hardening]] void mf1();
   [[clang::speculative_load_hardening]] static void mf2();
 };
@@ -53,5 +53,5 @@ struct CA {
 void cf6(); // expected-note@-1 {{conflicting attribute is here}}
 
 [[clang::no_speculative_load_hardening]]
-void cf6(); // expected-error@-1 {{'no_speculative_load_hardening' and 'speculative_load_hardening' attributes are not compatible}} \
+void cf6(); // expected-error@-1 {{'clang::no_speculative_load_hardening' and 'clang::speculative_load_hardening' attributes are not compatible}} \
 
diff --git a/clang/test/SemaCXX/attr-suppress.cpp b/clang/test/SemaCXX/attr-suppress.cpp
index e8f6d97988091..f097379e9c375 100644
--- a/clang/test/SemaCXX/attr-suppress.cpp
+++ b/clang/test/SemaCXX/attr-suppress.cpp
@@ -12,9 +12,9 @@ namespace N {
     p = reinterpret_cast<int *>(7);
   }
 
-  [[gsl::suppress]] int x;       // expected-error {{'suppress' attribute takes at least 1 argument}}
-  [[gsl::suppress()]] int y;     // expected-error {{'suppress' attribute takes at least 1 argument}}
-  int [[gsl::suppress("r")]] z;  // expected-error {{'suppress' attribute cannot be applied to types}}
+  [[gsl::suppress]] int x;       // expected-error {{'gsl::suppress' attribute takes at least 1 argument}}
+  [[gsl::suppress()]] int y;     // expected-error {{'gsl::suppress' attribute takes at least 1 argument}}
+  int [[gsl::suppress("r")]] z;  // expected-error {{'gsl::suppress' attribute cannot be applied to types}}
   [[gsl::suppress(f_)]] float f; // expected-error {{expected string literal as argument of 'suppress' attribute}}
 }
 
@@ -48,7 +48,7 @@ namespace N {
   }
 
   int [[clang::suppress("r")]] z;
-  // expected-error@-1 {{'suppress' attribute cannot be applied to types}}
+  // expected-error@-1 {{'clang::suppress' attribute cannot be applied to types}}
   [[clang::suppress(foo)]] float f;
   // expected-error@-1 {{expected string literal as argument of 'suppress' attribute}}
 }
@@ -60,5 +60,5 @@ class [[clang::suppress("type.1")]] V {
 
 // FIXME: There's no good reason why we shouldn't support this case.
 // But it doesn't look like clang generally supports such attributes yet.
-class W : [[clang::suppress]] public V { // expected-error{{'suppress' attribute cannot be applied to a base specifier}}
+class W : [[clang::suppress]] public V { // expected-error{{'clang::suppress' attribute cannot be applied to a base specifier}}
 };
diff --git a/clang/test/SemaCXX/attr-unsafe-buffer-usage.cpp b/clang/test/SemaCXX/attr-unsafe-buffer-usage.cpp
index 91ba446663383..cab67784d9075 100644
--- a/clang/test/SemaCXX/attr-unsafe-buffer-usage.cpp
+++ b/clang/test/SemaCXX/attr-unsafe-buffer-usage.cpp
@@ -3,4 +3,4 @@
 // Function annotations.
 [[clang::unsafe_buffer_usage]]
 void f(int *buf, int size);
-void g(int *buffer [[clang::unsafe_buffer_usage("buffer")]], int size); // expected-warning {{'unsafe_buffer_usage' attribute only applies to functions}}
+void g(int *buffer [[clang::unsafe_buffer_usage("buffer")]], int size); // expected-warning {{'clang::unsafe_buffer_usage' attribute only applies to functions}}
diff --git a/clang/test/SemaCXX/builtin-invoke.cpp b/clang/test/SemaCXX/builtin-invoke.cpp
new file mode 100644
index 0000000000000..d7694d619c692
--- /dev/null
+++ b/clang/test/SemaCXX/builtin-invoke.cpp
@@ -0,0 +1,240 @@
+// RUN: %clang_cc1 -verify -fsyntax-only %s -std=c++23
+
+void func() { // expected-note {{'func' declared here}}
+  __builtin_invoke(); // expected-error {{too few arguments to function call, expected at least 1, have 0}}
+}
+
+void nfunc() noexcept {}
+
+struct S {};
+void argfunc(int, S) {} // expected-note {{'argfunc' declared here}}
+
+struct Callable {
+  void operator()() {}
+
+  void func() {}
+
+  int var;
+};
+
+void* malloc(decltype(sizeof(int)));
+
+template <class T>
+struct pointer_wrapper {
+  T* v;
+
+  T& operator*() {
+    return *v;
+  }
+};
+
+namespace std {
+  template <class T>
+  class reference_wrapper {
+    T* ptr;
+
+  public:
+    constexpr reference_wrapper(T& ref) : ptr(&ref) {}
+
+    constexpr T& get() { return *ptr; }
+  };
+
+  template <class T>
+  constexpr reference_wrapper<T> ref(T& v) {
+    return reference_wrapper<T>(v);
+  }
+} // namespace std
+
+struct InvalidSpecialization1 {
+  void func() {}
+
+  int var;
+};
+
+template <>
+class std::reference_wrapper<InvalidSpecialization1> {
+public:
+  reference_wrapper(InvalidSpecialization1&) {}
+};
+
+struct InvalidSpecialization2 {
+  void func() {}
+
+  int var;
+};
+
+template <>
+class std::reference_wrapper<InvalidSpecialization2> {
+public:
+  reference_wrapper(InvalidSpecialization2&) {}
+
+private:
+  InvalidSpecialization2& get(); // expected-note 2 {{declared private here}}
+};
+
+struct ExplicitObjectParam {
+  void func(this const ExplicitObjectParam& self) {}
+};
+
+struct Incomplete; // expected-note 2 {{forward declaration}}
+struct Incomplete2;
+
+void incomplete_by_val_test(Incomplete);
+
+void incomplete_test(Incomplete& incomplete) {
+  __builtin_invoke((int (Incomplete2::*)){}, incomplete); // expected-error {{incomplete type 'Incomplete' used in type trait expression}} \
+                                                             expected-error {{indirection requires pointer operand ('Incomplete' invalid)}}
+  __builtin_invoke(incomplete_test, incomplete);
+  __builtin_invoke(incomplete_by_val_test, incomplete); // expected-error {{argument type 'Incomplete' is incomplete}}
+}
+
+void call() {
+  __builtin_invoke(func);
+  __builtin_invoke(nfunc);
+  static_assert(!noexcept(__builtin_invoke(func)));
+  static_assert(noexcept(__builtin_invoke(nfunc)));
+  __builtin_invoke(func, 1); // expected-error {{too many arguments to function call, expected 0, have 1}}
+  __builtin_invoke(argfunc, 1); // expected-error {{too few arguments to function call, expected 2, have 1}}
+  __builtin_invoke(Callable{});
+  __builtin_invoke(malloc, 0);
+  __builtin_invoke(__builtin_malloc, 0); // expected-error {{builtin functions must be directly called}}
+
+  // Variadic function
+  void variadic_func(int, ...); // expected-note {{declared here}}
+
+  __builtin_invoke(variadic_func); // expected-error {{too few arguments to function call, expected at least 1, have 0}}
+  __builtin_invoke(variadic_func, 1);
+  __builtin_invoke(variadic_func, 1, 2, 3);
+
+  // static member function
+  struct StaticMember {
+    static void func(int);
+  };
+
+  __builtin_invoke(StaticMember::func, 1);
+  StaticMember sm;
+  __builtin_invoke(sm.func, 1);
+
+  // lambda
+  __builtin_invoke([] {});
+  __builtin_invoke([](int) {}, 1);
+
+  // Member function pointer
+  __builtin_invoke(&Callable::func); // expected-error {{too few arguments to function call, expected at least 2, have 1}}
+  __builtin_invoke(&Callable::func, 1); // expected-error {{indirection requires pointer operand ('int' invalid)}}
+  __builtin_invoke(&Callable::func, Callable{});
+  __builtin_invoke(&Callable::func, Callable{}, 1); // expected-error {{too many arguments to function call, expected 0, have 1}}
+  __builtin_invoke(&ExplicitObjectParam::func, ExplicitObjectParam{});
+
+  Callable c;
+  __builtin_invoke(&Callable::func, &c);
+  __builtin_invoke(&Callable::func, std::ref(c));
+  __builtin_invoke(&Callable::func, &c);
+  __builtin_invoke(&Callable::func, &c, 2); // expected-error {{too many arguments to function call, expected 0, have 1}}
+  __builtin_invoke(&Callable::func, pointer_wrapper<Callable>{&c});
+  __builtin_invoke(&Callable::func, pointer_wrapper<Callable>{&c}, 2); // expected-error {{too many arguments to function call, expected 0, have 1}}
+
+  InvalidSpecialization1 is1;
+  InvalidSpecialization2 is2;
+  __builtin_invoke(&InvalidSpecialization1::func, std::ref(is1)); // expected-error {{no member named 'get' in 'std::reference_wrapper<InvalidSpecialization1>'}}
+  __builtin_invoke(&InvalidSpecialization2::func, std::ref(is2)); // expected-error {{'get' is a private member of 'std::reference_wrapper<InvalidSpecialization2>'}}
+
+  // Member data pointer
+  __builtin_invoke(&Callable::var); // expected-error {{too few arguments to function call, expected at least 2, have 1}}
+  __builtin_invoke(&Callable::var, 1); // expected-error {{indirection requires pointer operand ('int' invalid)}}
+  (void)__builtin_invoke(&Callable::var, Callable{});
+  __builtin_invoke(&Callable::var, Callable{}, 1); // expected-error {{too many arguments to function call, expected 2, have 3}}
+
+  (void)__builtin_invoke(&Callable::var, &c);
+  (void)__builtin_invoke(&Callable::var, std::ref(c));
+  (void)__builtin_invoke(&Callable::var, &c);
+  __builtin_invoke(&Callable::var, &c, 2); // expected-error {{too many arguments to function call, expected 2, have 3}}
+  (void)__builtin_invoke(&Callable::var, pointer_wrapper<Callable>{&c});
+  __builtin_invoke(&Callable::var, pointer_wrapper<Callable>{&c}, 2); // expected-error {{too many arguments to function call, expected 2, have 3}}
+
+  __builtin_invoke(&InvalidSpecialization1::var, std::ref(is1)); // expected-error {{no member named 'get' in 'std::reference_wrapper<InvalidSpecialization1>'}}
+  (void)__builtin_invoke(&InvalidSpecialization2::var, std::ref(is2)); // expected-error {{'get' is a private member of 'std::reference_wrapper<InvalidSpecialization2>'}}
+}
+
+[[nodiscard]] int diagnose_discard();
+int no_diagnose_discard();
+
+namespace std {
+  template <class... Args>
+  auto invoke(Args&&... args) -> decltype(__builtin_invoke(args...));
+} // namespace std
+
+template <class... Args>
+concept invocable = requires(Args... args) { __builtin_invoke(args...); };
+
+static_assert(!invocable<std::reference_wrapper<InvalidSpecialization1>>);
+static_assert(!invocable<std::reference_wrapper<InvalidSpecialization2>>);
+
+void test3() {
+  __builtin_invoke(diagnose_discard); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  __builtin_invoke(no_diagnose_discard);
+}
+
+template <class T>
+auto test(T v) {
+  return __builtin_invoke(v);
+}
+
+auto call2() {
+  test(call);
+}
+
+template <class ClassT, class FuncT>
+void func(ClassT& c, FuncT&& func) {
+  __builtin_invoke(func, c, 1, 2, 3); // expected-error {{too many arguments to function call, expected 0, have 3}}
+}
+
+struct DependentTest {
+  void func(int, int, int);
+  void bad_func();
+};
+
+void call3() {
+  DependentTest d;
+  func(d, &DependentTest::func);
+  func(d, &DependentTest::bad_func); // expected-note {{requested here}}
+}
+
+constexpr int constexpr_func() {
+  return 42;
+}
+
+struct ConstexprTestStruct {
+  int i;
+  constexpr int func() {
+    return 55;
+  }
+};
+
+// Make sure that constant evaluation works
+static_assert([]() {
+
+  ConstexprTestStruct s;
+  if (__builtin_invoke(&ConstexprTestStruct::func, s) != 55) // [func.requires]/p1.1
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::func, std::ref(s)) != 55) // [func.requires]/p1.2
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::func, &s) != 55) // [func.requires]/p1.3
+    return false;
+
+  s.i = 22;
+  if (__builtin_invoke(&ConstexprTestStruct::i, s) != 22) // [func.requires]/p1.4
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::i, std::ref(s)) != 22) // [func.requires]/p1.5
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::i, &s) != 22) // [func.requires]/p1.6
+    return false;
+
+  // [func.requires]/p1.7
+  if (__builtin_invoke(constexpr_func) != 42)
+    return false;
+  if (__builtin_invoke([] { return 34; }) != 34)
+    return false;
+
+  return true;
+}());
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index ab4e50072f654..c390fee1c38d9 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1466,7 +1466,7 @@ namespace InstantiateCaseStmt {
 
 namespace ConvertedConstantExpr {
   extern int &m;
-  extern int &n; // pre-cxx23-note 2{{declared here}}
+  extern int &n; // expected-note 2{{declared here}}
 
   constexpr int k = 4;
   int &m = const_cast<int&>(k);
@@ -1475,9 +1475,9 @@ namespace ConvertedConstantExpr {
   // useless note and instead just point to the non-constant subexpression.
   enum class E {
     em = m,
-    en = n, // expected-error {{enumerator value is not a constant expression}} cxx11_20-note {{initializer of 'n' is unknown}}
+    en = n, // expected-error {{enumerator value is not a constant expression}} cxx11_20-note {{initializer of 'n' is unknown}} cxx23-note {{read of non-constexpr variable 'n'}}
     eo = (m + // expected-error {{not a constant expression}}
-          n // cxx11_20-note {{initializer of 'n' is unknown}}
+          n // cxx11_20-note {{initializer of 'n' is unknown}} cxx23-note {{read of non-constexpr variable 'n'}}
           ),
     eq = reinterpret_cast<long>((int*)0) // expected-error {{not a constant expression}} expected-note {{reinterpret_cast}}
   };
diff --git a/clang/test/SemaCXX/constant-expression-p2280r4.cpp b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
index 50637917ba210..dffb386f530f4 100644
--- a/clang/test/SemaCXX/constant-expression-p2280r4.cpp
+++ b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
@@ -35,7 +35,7 @@ constexpr int how_many(Swim& swam) {
   return (p + 1 - 1)->phelps();
 }
 
-void splash(Swim& swam) {
+void splash(Swim& swam) {                 // nointerpreter-note {{declared here}}
   static_assert(swam.phelps() == 28);     // ok
   static_assert((&swam)->phelps() == 28); // ok
   Swim* pswam = &swam;                    // expected-note {{declared here}}
@@ -43,8 +43,10 @@ void splash(Swim& swam) {
                                           // expected-note {{read of non-constexpr variable 'pswam' is not allowed in a constant expression}}
   static_assert(how_many(swam) == 28);    // ok
   static_assert(Swim().lochte() == 12);   // ok
-  static_assert(swam.lochte() == 12);     // expected-error {{static assertion expression is not an integral constant expression}}
-  static_assert(swam.coughlin == 12);     // expected-error {{static assertion expression is not an integral constant expression}}
+  static_assert(swam.lochte() == 12);     // expected-error {{static assertion expression is not an integral constant expression}} \
+                                          // nointerpreter-note {{virtual function called on object 'swam' whose dynamic type is not constant}}
+  static_assert(swam.coughlin == 12);     // expected-error {{static assertion expression is not an integral constant expression}} \
+                                          // nointerpreter-note {{read of variable 'swam' whose value is not known}}
 }
 
 extern Swim dc;
@@ -52,6 +54,7 @@ extern Swim& trident; // interpreter-note {{declared here}}
 
 constexpr auto& sandeno   = typeid(dc);         // ok: can only be typeid(Swim)
 constexpr auto& gallagher = typeid(trident);    // expected-error {{constexpr variable 'gallagher' must be initialized by a constant expression}} \
+                                                // nointerpreter-note {{typeid applied to object 'trident' whose dynamic type is not constant}} \
                                                 // interpreter-note {{initializer of 'trident' is unknown}}
 
 namespace explicitThis {
@@ -253,12 +256,88 @@ namespace uninit_reference_used {
 
 namespace param_reference {
   constexpr int arbitrary = -12345;
-  constexpr void f(const int &x = arbitrary) { // expected-note {{declared here}}
+  constexpr void f(const int &x = arbitrary) { // nointerpreter-note 3 {{declared here}} interpreter-note {{declared here}}
     constexpr const int &v1 = x; // expected-error {{must be initialized by a constant expression}} \
     // expected-note {{reference to 'x' is not a constant expression}}
     constexpr const int &v2 = (x, arbitrary); // expected-warning {{left operand of comma operator has no effect}}
-    constexpr int v3 = x; // expected-error {{must be initialized by a constant expression}}
-    static_assert(x==arbitrary); // expected-error {{static assertion expression is not an integral constant expression}}
+    constexpr int v3 = x; // expected-error {{must be initialized by a constant expression}} \
+                          // nointerpreter-note {{read of variable 'x' whose value is not known}}
+    static_assert(x==arbitrary); // expected-error {{static assertion expression is not an integral constant expression}} \
+                                 // nointerpreter-note {{read of variable 'x' whose value is not known}}
     static_assert(&x - &x == 0);
   }
 }
+
+namespace dropped_note {
+  extern int &x; // expected-note {{declared here}}
+  constexpr int f() { return x; } // nointerpreter-note {{read of non-constexpr variable 'x'}} \
+                                  // interpreter-note {{initializer of 'x' is unknown}}
+  constexpr int y = f(); // expected-error {{constexpr variable 'y' must be initialized by a constant expression}} expected-note {{in call to 'f()'}}
+}
+
+namespace dynamic {
+  struct A {virtual ~A();};
+  struct B : A {};
+  void f(A& a) {
+    constexpr B* b = dynamic_cast<B*>(&a); // expected-error {{must be initialized by a constant expression}} \
+                                           // nointerpreter-note {{dynamic_cast applied to object 'a' whose dynamic type is not constant}}
+    constexpr void* b2 = dynamic_cast<void*>(&a); // expected-error {{must be initialized by a constant expression}} \
+                                                  // nointerpreter-note {{dynamic_cast applied to object 'a' whose dynamic type is not constant}}
+  }
+}
+
+namespace unsized_array {
+  void f(int (&a)[], int (&b)[], int (&c)[4]) {
+    constexpr int t1 = a - a;
+    constexpr int t2 = a - b; // expected-error {{constexpr variable 't2' must be initialized by a constant expression}} \
+                              // nointerpreter-note {{arithmetic involving unrelated objects '&a[0]' and '&b[0]' has unspecified value}} \
+                              // interpreter-note {{arithmetic involving unrelated objects 'a' and 'b' has unspecified value}}
+    constexpr int t3 = a - &c[2];  // expected-error {{constexpr variable 't3' must be initialized by a constant expression}} \
+                              // nointerpreter-note {{arithmetic involving unrelated objects '&a[0]' and '&c[2]' has unspecified value}} \
+                              // interpreter-note {{arithmetic involving unrelated objects 'a' and '*((char*)&c + 8)' has unspecified value}}
+  }
+}
+
+namespace casting {
+  struct A {};
+  struct B : A {};
+  struct C : A {};
+  extern A &a; // interpreter-note {{declared here}}
+  extern B &b; // expected-note {{declared here}} interpreter-note 2 {{declared here}}
+  constexpr B &t1 = (B&)a; // expected-error {{must be initialized by a constant expression}} \
+                           // nointerpreter-note {{cannot cast object of dynamic type 'A' to type 'B'}} \
+                           // interpreter-note {{initializer of 'a' is unknown}}
+  constexpr B &t2 = (B&)(A&)b; // expected-error {{must be initialized by a constant expression}} \
+                               // nointerpreter-note {{initializer of 'b' is not a constant expression}} \
+                               // interpreter-note {{initializer of 'b' is unknown}}
+  // FIXME: interpreter incorrectly rejects.
+  constexpr bool t3 = &b + 1 == &(B&)(A&)b; // interpreter-error {{must be initialized by a constant expression}} \
+                                            // interpreter-note {{initializer of 'b' is unknown}}
+  constexpr C &t4 = (C&)(A&)b; // expected-error {{must be initialized by a constant expression}} \
+                               // nointerpreter-note {{cannot cast object of dynamic type 'B' to type 'C'}} \
+                               // interpreter-note {{initializer of 'b' is unknown}}
+}
+
+namespace pointer_comparisons {
+  extern int &extern_n; // interpreter-note 2 {{declared here}}
+  extern int &extern_n2;
+  constexpr int f1(bool b, int& n) {
+    if (b) {
+      return &extern_n == &n;
+    }
+    return f1(true, n);
+  }
+  // FIXME: interpreter incorrectly rejects; both sides are the same constexpr-unknown value.
+  static_assert(f1(false, extern_n)); // interpreter-error {{static assertion expression is not an integral constant expression}} \
+                                      // interpreter-note {{initializer of 'extern_n' is unknown}}
+  // FIXME: We should diagnose this: we don't know if the references bind
+  // to the same object.
+  static_assert(&extern_n != &extern_n2); // interpreter-error {{static assertion expression is not an integral constant expression}} \
+                                          // interpreter-note {{initializer of 'extern_n' is unknown}}
+  void f2(const int &n) {
+    // FIXME: We should not diagnose this: the two objects provably have
+    // different addresses because the lifetime of "n" extends across
+    // the initialization.
+    constexpr int x = &x == &n; // nointerpreter-error {{must be initialized by a constant expression}}
+  }
+}
diff --git a/clang/test/SemaCXX/cxx11-attributes-on-using-declaration.cpp b/clang/test/SemaCXX/cxx11-attributes-on-using-declaration.cpp
index 5823ba7bfc2f2..06b24154a7990 100644
--- a/clang/test/SemaCXX/cxx11-attributes-on-using-declaration.cpp
+++ b/clang/test/SemaCXX/cxx11-attributes-on-using-declaration.cpp
@@ -12,7 +12,7 @@ using NS::x [[deprecated]];                                    // expected-warni
 using NS::x __attribute__((deprecated));                       // expected-warning {{'deprecated' currently has no effect on a using declaration}}
 using NS::x __attribute__((availability(macos,introduced=1))); // expected-warning {{'availability' currently has no effect on a using declaration}}
 
-[[clang::availability(macos,introduced=1)]] using NS::x; // expected-warning {{'availability' currently has no effect on a using declaration}} expected-warning{{ISO C++ does not allow}}
+[[clang::availability(macos,introduced=1)]] using NS::x; // expected-warning {{'clang::availability' currently has no effect on a using declaration}} expected-warning{{ISO C++ does not allow}}
 
 // expected-warning@+1 3 {{ISO C++ does not allow an attribute list to appear here}}
 [[clang::annotate("A")]] using NS::x [[clang::annotate("Y")]], NS::x [[clang::annotate("Z")]];
diff --git a/clang/test/SemaCXX/cxx11-gnu-attrs.cpp b/clang/test/SemaCXX/cxx11-gnu-attrs.cpp
index b8d914a6791a9..348519ddf8715 100644
--- a/clang/test/SemaCXX/cxx11-gnu-attrs.cpp
+++ b/clang/test/SemaCXX/cxx11-gnu-attrs.cpp
@@ -5,22 +5,22 @@
 [[gnu::this_attribute_does_not_exist]] int unknown_attr;
 // expected-warning@-1 {{unknown attribute 'gnu::this_attribute_does_not_exist' ignored}}
 int [[gnu::unused]] attr_on_type;
-// expected-error@-1 {{'unused' attribute cannot be applied to types}}
+// expected-error@-1 {{'gnu::unused' attribute cannot be applied to types}}
 int *[[gnu::unused]] attr_on_ptr;
-// expected-warning@-1 {{attribute 'unused' ignored, because it cannot be applied to a type}}
+// expected-warning@-1 {{attribute 'gnu::unused' ignored, because it cannot be applied to a type}}
 [[gnu::fastcall]] void pr17424_1();
-// expected-warning@-1 {{'fastcall' calling convention is not supported for this target}}
+// expected-warning@-1 {{'gnu::fastcall' calling convention is not supported for this target}}
 [[gnu::fastcall]] [[gnu::stdcall]] void pr17424_2();
-// expected-warning@-1 {{'fastcall' calling convention is not supported for this target}}
-// expected-warning@-2 {{'stdcall' calling convention is not supported for this target}}
+// expected-warning@-1 {{'gnu::fastcall' calling convention is not supported for this target}}
+// expected-warning@-2 {{'gnu::stdcall' calling convention is not supported for this target}}
 [[gnu::fastcall]] __stdcall void pr17424_3();
-// expected-warning@-1 {{'fastcall' calling convention is not supported for this target}}
+// expected-warning@-1 {{'gnu::fastcall' calling convention is not supported for this target}}
 // expected-warning@-2 {{'__stdcall' calling convention is not supported for this target}}
 [[gnu::fastcall]] void pr17424_4() [[gnu::stdcall]];
-// expected-warning@-1 {{'fastcall' calling convention is not supported for this target}}
-// expected-warning@-2 {{'stdcall' calling convention is not supported for this target}}
+// expected-warning@-1 {{'gnu::fastcall' calling convention is not supported for this target}}
+// expected-warning@-2 {{'gnu::stdcall' calling convention is not supported for this target}}
 void pr17424_5 [[gnu::fastcall]]();
-// expected-warning@-1 {{'fastcall' calling convention is not supported for this target}}
+// expected-warning@-1 {{'gnu::fastcall' calling convention is not supported for this target}}
 
 // Valid cases.
 
diff --git a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
index ed8cbbbfe7067..86bed5f14441e 100644
--- a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
+++ b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
@@ -3,6 +3,11 @@
 // RUN: %clang_cc1 -std=c++2a -verify=expected,cxx20 %s "-DNEW=::operator new" "-DDELETE=::operator delete"
 // RUN: %clang_cc1 -std=c++2c -verify=expected,cxx26 %s "-DNEW=::operator new" "-DDELETE=::operator delete"
 
+// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx20 %s -DNEW=__builtin_operator_new -DDELETE=__builtin_operator_delete
+// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx20 %s "-DNEW=operator new" "-DDELETE=operator delete"
+// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx20 %s "-DNEW=::operator new" "-DDELETE=::operator delete"
+// RUN: %clang_cc1 -std=c++2c -verify=expected,cxx26 %s "-DNEW=::operator new" "-DDELETE=::operator delete"
+
 constexpr bool alloc_from_user_code() {
   void *p = NEW(sizeof(int)); // expected-note {{cannot allocate untyped memory in a constant expression; use 'std::allocator<T>::allocate'}}
   DELETE(p);
@@ -242,3 +247,67 @@ void f() {
 }
 
 }
+
+namespace GH134820 {
+struct S {
+    char* c = new char;
+    constexpr ~S() {
+        delete c;
+    }
+    int i = 0;
+};
+
+int f() {
+    if constexpr((S{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(S s; (S{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(S s; (s, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(constexpr int _ = S{}.i; true) {
+        return 1;
+    }
+    return 0;
+}
+
+template <typename T>
+int f2() {
+    if constexpr((T{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(T s; (T{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(T s; (s, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(constexpr int _ = T{}.i; true) {
+        return 1;
+    }
+    return 0;
+}
+
+void test() {
+  f2<S>(); // expected-note {{in instantiation}}
+}
+
+}
+namespace GH120197{
+struct NonTrivialDtor {
+  NonTrivialDtor() = default;
+  NonTrivialDtor(const NonTrivialDtor&) = default;
+  NonTrivialDtor(NonTrivialDtor&&) = default;
+  NonTrivialDtor& operator=(const NonTrivialDtor&)  = default;
+  NonTrivialDtor& operator=(NonTrivialDtor&&) = default;
+  constexpr ~NonTrivialDtor() noexcept {}
+};
+
+static_assert(((void)NonTrivialDtor{}, true)); // passes
+
+void f() {
+  if constexpr ((void)NonTrivialDtor{}, true) {}
+}
+}
diff --git a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
index dd5063cb29c5b..c4cfd9398920a 100644
--- a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
+++ b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
@@ -576,3 +576,37 @@ int f() {
   //expected-note@-2 {{read of non-const variable 'a' is not allowed in a constant expression}}
 }
 }
+
+#if __cplusplus >= 202302L
+namespace GH135281 {
+  struct B {
+    const void* p;
+    consteval B() : p{this} {}
+  };
+  B b;
+  B b2{};
+  B &&b3{};
+  void f() {
+    static B b4;
+    B b5; // expected-error {{call to consteval function 'GH135281::B::B' is not a constant expression}} \
+          // expected-note {{pointer to temporary is not a constant expression}} \
+          // expected-note {{temporary created here}}
+  }
+  template<typename T> T temp_var_uninit;
+  template<typename T> T temp_var_brace_init{};
+  B* b6 = &temp_var_uninit<B>;
+  B* b7 = &temp_var_brace_init<B>;
+  B* b8 = &temp_var_brace_init<B&&>;
+  template<typename T> void f2() {
+    static T b9;
+    T b10; // expected-error {{call to consteval function 'GH135281::B::B' is not a constant expression}} \
+           // expected-note {{pointer to temporary is not a constant expression}} \
+           // expected-note {{temporary created here}}
+    static B b11;
+    B b12; // expected-error 2 {{call to consteval function 'GH135281::B::B' is not a constant expression}} \
+           // expected-note 2 {{pointer to temporary is not a constant expression}} \
+           // expected-note 2 {{temporary created here}}
+  }
+  void (*ff)() = f2<B>; // expected-note {{instantiation of function template specialization}}
+}
+#endif
diff --git a/clang/test/SemaCXX/cxx2c-binding-pack.cpp b/clang/test/SemaCXX/cxx2c-binding-pack.cpp
index 0f10dad3937ba..a8c1386ea5c18 100644
--- a/clang/test/SemaCXX/cxx2c-binding-pack.cpp
+++ b/clang/test/SemaCXX/cxx2c-binding-pack.cpp
@@ -1,10 +1,11 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++26 %s -verify
+// RUN: %clang_cc1 -fsyntax-only -std=c++26 %s -verify -fexperimental-new-constant-interpreter
 
 template <typename T>
 struct type_ { };
 
 template <typename ...T>
-auto sum(T... t) { return (t + ...); }
+constexpr auto sum(T... t) { return (t + ...); }
 
 struct my_struct {
   int a;
@@ -17,7 +18,7 @@ struct fake_tuple {
   int arr[4] = {1, 2, 3, 6};
 
   template <unsigned i>
-  int get() {
+  constexpr int& get() {
     return arr[i];
   }
 };
@@ -233,3 +234,28 @@ void g() {
 }
 
 }
+
+namespace constant_interpreter {
+using Arr = int[2];
+struct Triple { int x, y, z = 3; };
+
+constexpr int ref_to_same_obj(auto&& arg) {
+  auto& [...xs] = arg;
+  auto& [...ys] = arg;
+  (..., (xs += 2));
+  return sum(ys...);
+}
+static_assert(ref_to_same_obj(Arr{}) == 4);
+static_assert(ref_to_same_obj(fake_tuple{}) == 20);
+static_assert(ref_to_same_obj(Triple{}) == 9);
+
+constexpr int copy_obj(auto&& arg) {
+  auto& [...xs] = arg;
+  auto [...ys] = arg;
+  (..., (xs += 2));
+  return sum(ys...);
+}
+static_assert(copy_obj(Arr{}) == 0);
+static_assert(copy_obj(fake_tuple{}) == 12);
+static_assert(copy_obj(Triple{}) == 3);
+}
diff --git a/clang/test/SemaCXX/internal_linkage.cpp b/clang/test/SemaCXX/internal_linkage.cpp
index 81ab01f603e4e..4fcf377b83ac2 100644
--- a/clang/test/SemaCXX/internal_linkage.cpp
+++ b/clang/test/SemaCXX/internal_linkage.cpp
@@ -41,7 +41,7 @@ int A::y;
 void A::f1() {
 }
 
-void g(int a [[clang::internal_linkage]]) { // expected-warning{{'internal_linkage' attribute only applies to variables, functions and classes}}
+void g(int a [[clang::internal_linkage]]) { // expected-warning{{'clang::internal_linkage' attribute only applies to variables, functions and classes}}
   int x [[clang::internal_linkage]]; // expected-warning{{'internal_linkage' attribute on a non-static local variable is ignored}}
   static int y [[clang::internal_linkage]];
 }
diff --git a/clang/test/SemaCXX/modules.cppm b/clang/test/SemaCXX/modules.cppm
index 5d0d6da44a2ed..4ef4146df4d9d 100644
--- a/clang/test/SemaCXX/modules.cppm
+++ b/clang/test/SemaCXX/modules.cppm
@@ -41,6 +41,8 @@ struct S {
   export static int n; // expected-error {{expected member name or ';'}}
 };
 
+int main() {} // expected-warning {{'main' never has module linkage}}
+
 // FIXME: Exports of declarations without external linkage are disallowed.
 // Exports of declarations with non-external-linkage types are disallowed.
 
@@ -68,6 +70,8 @@ int n;
 //--- test3.cpp
 export module bar;
 
+extern "C++" int main() {}
+
 static int m;
 
 int n;
diff --git a/clang/test/SemaCXX/ms-constexpr-invalid.cpp b/clang/test/SemaCXX/ms-constexpr-invalid.cpp
index e5bec0c7119b0..89102c6fb954f 100644
--- a/clang/test/SemaCXX/ms-constexpr-invalid.cpp
+++ b/clang/test/SemaCXX/ms-constexpr-invalid.cpp
@@ -17,7 +17,7 @@ struct D1 : virtual B1 { // expected-note {{virtual base class declared here}}
     [[msvc::constexpr]] D1() {} // expected-error {{constexpr constructor not allowed in struct with virtual base class}}
 };
 
-struct [[msvc::constexpr]] S2{}; // expected-error {{'constexpr' attribute only applies to functions and return statements}}
+struct [[msvc::constexpr]] S2{}; // expected-error {{'msvc::constexpr' attribute only applies to functions and return statements}}
 
 // Check invalid code mixed with valid code
 
@@ -38,10 +38,10 @@ static_assert(f7()); // expected-error {{static assertion expression is not an i
                      // expected-note {{in call to 'f7()'}}
 
 constexpr bool f8() { // expected-error {{constexpr function never produces a constant expression}}
-    [[msvc::constexpr]] f4(32); // expected-error {{'constexpr' attribute only applies to functions and return statements}} \
+    [[msvc::constexpr]] f4(32); // expected-error {{'msvc::constexpr' attribute only applies to functions and return statements}} \
                                 // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} \
                                 // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}}
-    [[msvc::constexpr]] int i5 = f4(32); // expected-error {{'constexpr' attribute only applies to functions and return statements}}
+    [[msvc::constexpr]] int i5 = f4(32); // expected-error {{'msvc::constexpr' attribute only applies to functions and return statements}}
     return i5 == 5;
 }
 static_assert(f8()); // expected-error {{static assertion expression is not an integral constant expression}} \
diff --git a/clang/test/SemaCXX/ms-constexpr-new.cpp b/clang/test/SemaCXX/ms-constexpr-new.cpp
index 096014be803e7..08794565f91df 100644
--- a/clang/test/SemaCXX/ms-constexpr-new.cpp
+++ b/clang/test/SemaCXX/ms-constexpr-new.cpp
@@ -3,12 +3,12 @@
 // supported-no-diagnostics
 
 [[nodiscard]]
-[[msvc::constexpr]] // unsupported-warning {{unknown attribute 'constexpr' ignored}}
+[[msvc::constexpr]] // unsupported-warning {{unknown attribute 'msvc::constexpr' ignored}}
 inline void* operator new(decltype(sizeof(void*)), void* p) noexcept { return p; }
 
 namespace std {
   constexpr int* construct_at(int* p, int v) {
-    [[msvc::constexpr]] return ::new (p) int(v); // unsupported-warning {{unknown attribute 'constexpr' ignored}}
+    [[msvc::constexpr]] return ::new (p) int(v); // unsupported-warning {{unknown attribute 'msvc::constexpr' ignored}}
   }
 }
 
@@ -16,7 +16,7 @@ constexpr bool check_std_construct_at() { int x; return *std::construct_at(&x, 4
 static_assert(check_std_construct_at());
 
 constexpr int* construct_at(int* p, int v) { [[msvc::constexpr]] return ::new (p) int(v); } // unsupported-error {{constexpr function never produces a constant expression}} \
-                                                                                            // unsupported-warning {{unknown attribute 'constexpr' ignored}} \
+                                                                                            // unsupported-warning {{unknown attribute 'msvc::constexpr' ignored}} \
                                                                                             // unsupported-note 2{{this placement new expression is not supported in constant expressions before C++2c}}
 constexpr bool check_construct_at() { int x; return *construct_at(&x, 42) == 42; }          // unsupported-note {{in call to 'construct_at(&x, 42)'}}
 static_assert(check_construct_at());                                                        // unsupported-error {{static assertion expression is not an integral constant expression}}\
diff --git a/clang/test/SemaCXX/nested-name-spec.cpp b/clang/test/SemaCXX/nested-name-spec.cpp
index abeaba9d8dde2..fedbb3070070b 100644
--- a/clang/test/SemaCXX/nested-name-spec.cpp
+++ b/clang/test/SemaCXX/nested-name-spec.cpp
@@ -1,4 +1,11 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++98 -verify -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++98 -verify=expected,cxx98,cxx98-11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify=expected,since-cxx11,cxx98-11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++14 -verify=expected,since-cxx11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++17 -verify=expected,since-cxx11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++20 -verify=expected,since-cxx11,since-cxx20 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify=expected,since-cxx11,since-cxx20 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++26 -verify=expected,since-cxx11,since-cxx20 -fblocks %s
+
 namespace A {
   struct C {
     static int cx;
@@ -118,7 +125,7 @@ namespace E {
     };
 
     int f() {
-      return E::X; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
+      return E::X; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
     }
   }
 }
@@ -170,8 +177,9 @@ void ::global_func2(int) { } // expected-warning{{extra qualification on member
 
 void N::f() { } // okay
 
-struct Y;  // expected-note{{forward declaration of 'Y'}}
-Y::foo y; // expected-error{{incomplete type 'Y' named in nested name specifier}}
+// FIXME (GH147000): duplicate diagnostics
+struct Y;  // expected-note{{forward declaration of 'Y'}} since-cxx20-note{{forward declaration of 'Y'}}
+Y::foo y; // expected-error{{incomplete type 'Y' named in nested name specifier}} since-cxx20-error{{incomplete type 'Y' named in nested name specifier}}
 
 namespace PR25156 {
 struct Y;  // expected-note{{forward declaration of 'PR25156::Y'}}
@@ -189,7 +197,9 @@ bool (foo_S::value);
 
 
 namespace somens {
-  struct a { }; // expected-note{{candidate constructor (the implicit copy constructor)}}
+  struct a { };
+  // expected-note@-1 {{candidate constructor (the implicit copy constructor)}}
+  // since-cxx11-note@-2 {{candidate constructor (the implicit move constructor)}}
 }
 
 template <typename T>
@@ -432,20 +442,20 @@ namespace PR16951 {
     };
   }
 
-  int x1 = ns::an_enumeration::ENUMERATOR; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
+  int x1 = ns::an_enumeration::ENUMERATOR; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
 
-  int x2 = ns::an_enumeration::ENUMERATOR::vvv; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
+  int x2 = ns::an_enumeration::ENUMERATOR::vvv; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
                                                 // expected-error{{'ENUMERATOR' is not a class, namespace, or enumeration}} \
 
-  int x3 = ns::an_enumeration::X; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
+  int x3 = ns::an_enumeration::X; // cxx98-warning {{use of enumeration in a nested name specifier is a C++11 extension}} \
                                   // expected-error{{no member named 'X'}}
 
   enum enumerator_2 {
     ENUMERATOR_2
   };
 
-  int x4 = enumerator_2::ENUMERATOR_2; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
-  int x5 = enumerator_2::X2; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
+  int x4 = enumerator_2::ENUMERATOR_2; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
+  int x5 = enumerator_2::X2; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
                              // expected-error{{no member named 'X2' in 'PR16951::enumerator_2'}} \
                              // expected-error{{cannot initialize a variable of type 'int' with an lvalue of type 'int (*)()'}}
 
@@ -487,7 +497,7 @@ struct x; // expected-note {{template is declared here}}
 
 template <typename T>
 int issue55962 = x::a; // expected-error {{use of class template 'x' requires template arguments}} \
-                       // expected-warning {{variable templates are a C++14 extension}}
+                       // cxx98-11-warning {{variable templates are a C++14 extension}}
 
 namespace ForwardDeclared {
   typedef class A B;
@@ -496,3 +506,11 @@ namespace ForwardDeclared {
     void F(B::C);
   };
 }
+
+namespace GH63254 {
+template <typename...> struct V {}; // cxx98-warning {{variadic templates are a C++11 extension}}
+struct S : V<> {
+  using S::V; // expected-error {{using declaration refers to its own class}}
+  V<> v; // no crash
+};
+}
diff --git a/clang/test/SemaCXX/no_destroy.cpp b/clang/test/SemaCXX/no_destroy.cpp
index d39bcaeff860a..1ec994cd73ca5 100644
--- a/clang/test/SemaCXX/no_destroy.cpp
+++ b/clang/test/SemaCXX/no_destroy.cpp
@@ -28,14 +28,14 @@ int main() {
   [[clang::always_destroy]] static int p4;
 }
 
-[[clang::always_destroy]] [[clang::no_destroy]] int p; // expected-error{{'no_destroy' and 'always_destroy' attributes are not compatible}} // expected-note{{here}}
-[[clang::no_destroy]] [[clang::always_destroy]] int p2; // expected-error{{'always_destroy' and 'no_destroy' attributes are not compatible}} // expected-note{{here}}
+[[clang::always_destroy]] [[clang::no_destroy]] int p; // expected-error{{'clang::no_destroy' and 'clang::always_destroy' attributes are not compatible}} // expected-note{{here}}
+[[clang::no_destroy]] [[clang::always_destroy]] int p2; // expected-error{{'clang::always_destroy' and 'clang::no_destroy' attributes are not compatible}} // expected-note{{here}}
 
-[[clang::always_destroy]] void f() {} // expected-warning{{'always_destroy' attribute only applies to variables}}
-struct [[clang::no_destroy]] DoesntApply {};  // expected-warning{{'no_destroy' attribute only applies to variables}}
+[[clang::always_destroy]] void f() {} // expected-warning{{'clang::always_destroy' attribute only applies to variables}}
+struct [[clang::no_destroy]] DoesntApply {};  // expected-warning{{'clang::no_destroy' attribute only applies to variables}}
 
-[[clang::no_destroy(0)]] int no_args; // expected-error{{'no_destroy' attribute takes no arguments}}
-[[clang::always_destroy(0)]] int no_args2; // expected-error{{'always_destroy' attribute takes no arguments}}
+[[clang::no_destroy(0)]] int no_args; // expected-error{{'clang::no_destroy' attribute takes no arguments}}
+[[clang::always_destroy(0)]] int no_args2; // expected-error{{'clang::always_destroy' attribute takes no arguments}}
 
 // expected-error@+1 {{temporary of type 'SecretDestructor' has private destructor}}
 SecretDestructor arr[10];
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
index 7cb71e075d50e..46c3670848529 100644
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -80,21 +80,30 @@ struct ImplicitlyCopyable {
 static_assert(__is_constructible(ImplicitlyCopyable, const ImplicitlyCopyable&));
 
 
-struct Movable {
+struct Movable { // #Movable
   template <typename T>
   requires __is_constructible(Movable, T) // #err-self-constraint-1
-  explicit Movable(T op) noexcept; // #1
-  Movable(Movable&&) noexcept = default; // #2
+  explicit Movable(T op) noexcept; // #Movable1
+  Movable(Movable&&) noexcept = default; // #Movable2
 };
 static_assert(__is_constructible(Movable, Movable&&));
 static_assert(__is_constructible(Movable, const Movable&));
-// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}}
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, const Movable &)'}} \
+// expected-error@-1 {{call to implicitly-deleted copy constructor of 'Movable'}} \
+// expected-note@#Movable  {{'Movable' defined here}} \
+// expected-note@#Movable  {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const Movable' for 1st argument}} \
+// expected-note@#Movable2  {{copy constructor is implicitly deleted because 'Movable' has a user-declared move constructor}} \
+// expected-note@#Movable2  {{candidate constructor not viable: no known conversion from 'int' to 'Movable' for 1st argument}} \
+// expected-note@#Movable1  {{candidate template ignored: constraints not satisfied [with T = int]}}
+
 
 static_assert(__is_constructible(Movable, int));
-// expected-error@-1{{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(Movable, int)'}} \
+// expected-error@-1 {{no matching constructor for initialization of 'Movable'}} \
 // expected-note@-1 2{{}}
 // expected-error@#err-self-constraint-1{{satisfaction of constraint '__is_constructible(Movable, T)' depends on itself}}
 // expected-note@#err-self-constraint-1 4{{}}
+// expected-note@#Movable  {{'Movable' defined here}}
 
 template <typename T>
 struct Members {
diff --git a/clang/test/SemaCXX/switch-implicit-fallthrough.cpp b/clang/test/SemaCXX/switch-implicit-fallthrough.cpp
index bb39d6349a179..750663717b66a 100644
--- a/clang/test/SemaCXX/switch-implicit-fallthrough.cpp
+++ b/clang/test/SemaCXX/switch-implicit-fallthrough.cpp
@@ -326,16 +326,16 @@ int fallthrough_placement_error(int n) {
 int fallthrough_targets(int n) {
   [[clang::fallthrough]]; // expected-error{{fallthrough annotation is outside switch statement}}
 
-  [[clang::fallthrough]]  // expected-error{{'fallthrough' attribute only applies to empty statements}}
+  [[clang::fallthrough]]  // expected-error{{'clang::fallthrough' attribute only applies to empty statements}}
   switch (n) {
     case 121:
       n += 400;
       [[clang::fallthrough]]; // no warning here, correct target
     case 123:
-      [[clang::fallthrough]]  // expected-error{{'fallthrough' attribute only applies to empty statements}}
+      [[clang::fallthrough]]  // expected-error{{'clang::fallthrough' attribute only applies to empty statements}}
       n += 800;
       break;
-    [[clang::fallthrough]]    // expected-error{{'fallthrough' attribute is only allowed on empty statements}} expected-note{{did you forget ';'?}}
+    [[clang::fallthrough]]    // expected-error{{'clang::fallthrough' attribute is only allowed on empty statements}} expected-note{{did you forget ';'?}}
     case 125:
       n += 1600;
   }
diff --git a/clang/test/SemaCXX/type-attrs.cpp b/clang/test/SemaCXX/type-attrs.cpp
index ae77df614ea83..1c17fed185b0c 100644
--- a/clang/test/SemaCXX/type-attrs.cpp
+++ b/clang/test/SemaCXX/type-attrs.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-pc-win32 -Wgcc-compat -std=c++11 -verify %s
 
-void f() [[gnu::cdecl]] {}  // expected-warning {{GCC does not allow the 'cdecl' attribute to be written on a type}}
-void g() [[gnu::stdcall]] {}  // expected-warning {{GCC does not allow the 'stdcall' attribute to be written on a type}}
-void i() [[gnu::fastcall]] {}  // expected-warning {{GCC does not allow the 'fastcall' attribute to be written on a type}}
+void f() [[gnu::cdecl]] {}  // expected-warning {{GCC does not allow the 'gnu::cdecl' attribute to be written on a type}}
+void g() [[gnu::stdcall]] {}  // expected-warning {{GCC does not allow the 'gnu::stdcall' attribute to be written on a type}}
+void i() [[gnu::fastcall]] {}  // expected-warning {{GCC does not allow the 'gnu::fastcall' attribute to be written on a type}}
diff --git a/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp b/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
index cf33ac283ab42..f3ddbbfe15bdc 100644
--- a/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
+++ b/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
@@ -42,6 +42,14 @@ static constexpr bool value = __is_standard_layout(T);
 };
 template <typename T>
 constexpr bool is_standard_layout_v = __is_standard_layout(T);
+
+template <typename... Args>
+struct is_constructible {
+    static constexpr bool value = __is_constructible(Args...);
+};
+
+template <typename... Args>
+constexpr bool is_constructible_v = __is_constructible(Args...);
 #endif
 
 #ifdef STD2
@@ -97,6 +105,17 @@ template <typename T>
 using is_standard_layout = __details_is_standard_layout<T>;
 template <typename T>
 constexpr bool is_standard_layout_v = __is_standard_layout(T);
+
+template <typename... Args>
+struct __details_is_constructible{
+    static constexpr bool value = __is_constructible(Args...);
+};
+
+template <typename... Args>
+using is_constructible  = __details_is_constructible<Args...>;
+
+template <typename... Args>
+constexpr bool is_constructible_v = __is_constructible(Args...);
 #endif
 
 
@@ -149,6 +168,15 @@ template <typename T>
 using is_standard_layout = __details_is_standard_layout<T>;
 template <typename T>
 constexpr bool is_standard_layout_v = is_standard_layout<T>::value;
+
+template <typename... Args>
+struct __details_is_constructible : bool_constant<__is_constructible(Args...)> {};
+
+template <typename... Args>
+using is_constructible  = __details_is_constructible<Args...>;
+
+template <typename... Args>
+constexpr bool is_constructible_v = is_constructible<Args...>::value;
 #endif
 
 }
@@ -211,6 +239,15 @@ static_assert(std::is_assignable_v<int&, void>);
 // expected-error@-1 {{static assertion failed due to requirement 'std::is_assignable_v<int &, void>'}} \
 // expected-error@-1 {{assigning to 'int' from incompatible type 'void'}}
 
+static_assert(std::is_constructible<int, int>::value);
+
+static_assert(std::is_constructible<void>::value);
+// expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_constructible<void>::value'}} \
+// expected-note@-1 {{because it is a cv void type}}
+static_assert(std::is_constructible_v<void>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_constructible_v<void>'}} \
+// expected-note@-1 {{because it is a cv void type}}
+
 namespace test_namespace {
     using namespace std;
     static_assert(is_trivially_relocatable<int&>::value);
@@ -256,6 +293,13 @@ namespace test_namespace {
     // expected-error@-1 {{static assertion failed due to requirement 'is_empty_v<int &>'}} \
     // expected-note@-1 {{'int &' is not empty}} \
     // expected-note@-1 {{because it is a reference type}}
+
+    static_assert(is_constructible<void>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_constructible<void>::value'}} \
+    // expected-note@-1 {{because it is a cv void type}}
+    static_assert(is_constructible_v<void>);
+    // expected-error@-1 {{static assertion failed due to requirement 'is_constructible_v<void>'}} \
+    // expected-note@-1 {{because it is a cv void type}}
 }
 
 
@@ -284,6 +328,15 @@ concept C4 = std::is_assignable_v<T, U>; // #concept8
 
 template <C4<void> T> void g4();  // #cand8
 
+template <typename... Args>
+requires std::is_constructible<Args...>::value void f3();  // #cand5
+
+template <typename... Args>
+concept C3 = std::is_constructible_v<Args...>; // #concept6
+
+template <C3 T> void g3();  // #cand6
+
+
 void test() {
     f<int&>();
     // expected-error@-1 {{no matching function for call to 'f'}} \
@@ -327,6 +380,19 @@ void test() {
     // expected-note@#cand8 {{because 'C4<int &, void>' evaluated to false}} \
     // expected-note@#concept8 {{because 'std::is_assignable_v<int &, void>' evaluated to false}} \
     // expected-error@#concept8 {{assigning to 'int' from incompatible type 'void'}}
+
+    f3<void>();
+    // expected-error@-1 {{no matching function for call to 'f3'}} \
+    // expected-note@#cand5 {{candidate template ignored: constraints not satisfied [with Args = <void>]}} \
+    // expected-note-re@#cand5 {{because '{{.*}}is_constructible<void>::value' evaluated to false}} \
+    // expected-note@#cand5 {{because it is a cv void type}}
+
+    g3<void>();
+    // expected-error@-1 {{no matching function for call to 'g3'}} \
+    // expected-note@#cand6 {{candidate template ignored: constraints not satisfied [with T = void]}} \
+    // expected-note@#cand6 {{because 'void' does not satisfy 'C3'}} \
+    // expected-note@#concept6 {{because 'std::is_constructible_v<void>' evaluated to false}} \
+    // expected-note@#concept6 {{because it is a cv void type}}
 }
 }
 
diff --git a/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp b/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
index cc923d206ab35..54806a93ddf80 100644
--- a/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
+++ b/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
@@ -489,6 +489,68 @@ static_assert(__is_trivially_copyable(S12));
 // expected-note@#tc-S12 {{'S12' defined here}}
 }
 
+namespace constructible {
+
+struct S1 {  // #c-S1
+    S1(int); // #cc-S1
+};
+static_assert(__is_constructible(S1, char*));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(constructible::S1, char *)'}} \
+// expected-error@-1 {{no matching constructor for initialization of 'S1'}} \
+// expected-note@#c-S1 {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'char *' to 'const S1' for 1st argument}} \
+// expected-note@#c-S1 {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'char *' to 'S1' for 1st argument}} \
+// expected-note@#cc-S1 {{candidate constructor not viable: no known conversion from 'char *' to 'int' for 1st argument; dereference the argument with *}} \
+// expected-note@#c-S1 {{'S1' defined here}}
+
+struct S2 { // #c-S2
+    S2(int, float, double); // #cc-S2
+};
+static_assert(__is_constructible(S2, float));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(constructible::S2, float)'}} \
+// expected-note@#c-S2 {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'float' to 'const S2' for 1st argument}} \
+// expected-note@#c-S2 {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'float' to 'S2' for 1st argument}} \
+// expected-error@-1 {{no matching constructor for initialization of 'S2'}} \
+// expected-note@#cc-S2 {{candidate constructor not viable: requires 3 arguments, but 1 was provided}} \
+// expected-note@#c-S2 {{'S2' defined here}}
+
+static_assert(__is_constructible(S2, float, void));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(constructible::S2, float, void)'}} \
+// expected-note@#c-S2 {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} \
+// expected-note@#c-S2 {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} \
+// expected-note@-1{{because it is a cv void type}} \
+// expected-error@-1 {{no matching constructor for initialization of 'S2'}} \
+// expected-note@#cc-S2 {{candidate constructor not viable: requires 3 arguments, but 2 were provided}} \
+// expected-note@#c-S2 {{'S2' defined here}}
+
+static_assert(__is_constructible(int[]));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(int[])'}} \
+// expected-note@-1 {{because it is an incomplete array type}}
+
+static_assert(__is_constructible(void));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(void)'}} \
+// expected-note@-1 {{because it is a cv void type}}
+
+static_assert(__is_constructible(void, void));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(void, void)'}} \
+// expected-note@-1 {{because it is a cv void type}}
+
+static_assert(__is_constructible(const void));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(const void)'}} \
+// expected-note@-1 {{because it is a cv void type}}
+
+static_assert(__is_constructible(volatile void));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(volatile void)'}} \
+// expected-note@-1 {{because it is a cv void type}}
+
+static_assert(__is_constructible(int ()));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(int ())'}} \
+// expected-note@-1 {{because it is a function type}}
+
+static_assert(__is_constructible(void (int, float)));
+// expected-error@-1 {{static assertion failed due to requirement '__is_constructible(void (int, float))'}} \
+// expected-note@-1 {{because it is a function type}}
+}
+
 namespace assignable {
 struct S1;
 static_assert(__is_assignable(S1&, const S1&));
diff --git a/clang/test/SemaCXX/uninitialized.cpp b/clang/test/SemaCXX/uninitialized.cpp
index c7b987e2172e6..c7cef3e06fb2b 100644
--- a/clang/test/SemaCXX/uninitialized.cpp
+++ b/clang/test/SemaCXX/uninitialized.cpp
@@ -1508,7 +1508,7 @@ struct InheritWithExplicit<Special> {
 void aggregate() {
   struct NonAgg {
     NonAgg() { }
-    [[clang::require_explicit_initialization]] int na;  // expected-warning {{'require_explicit_initialization' attribute is ignored in non-aggregate type 'NonAgg'}}
+    [[clang::require_explicit_initialization]] int na;  // expected-warning {{'clang::require_explicit_initialization' attribute is ignored in non-aggregate type 'NonAgg'}}
   };
   NonAgg nonagg;  // no-warning
   (void)nonagg;
@@ -1717,7 +1717,7 @@ void aggregate() {
   InheritWithExplicit<> agg;  // expected-warning {{field in 'InheritWithExplicit<>' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_G2 {{'g2' declared here}}
   (void)agg;
 
-  InheritWithExplicit<Polymorphic> polymorphic;  // expected-warning@#FIELD_G2 {{'require_explicit_initialization' attribute is ignored in non-aggregate type 'InheritWithExplicit<Polymorphic>'}}
+  InheritWithExplicit<Polymorphic> polymorphic;  // expected-warning@#FIELD_G2 {{'clang::require_explicit_initialization' attribute is ignored in non-aggregate type 'InheritWithExplicit<Polymorphic>'}}
   (void)polymorphic;
 
   Inherit<Special> specialized_explicit;  // expected-warning {{field in 'Inherit<Special>' requires explicit initialization but is not explicitly initialized}} expected-note@#FIELD_G3 {{'g3' declared here}}
diff --git a/clang/test/SemaCXX/unique_object_duplication.h b/clang/test/SemaCXX/unique_object_duplication.h
index bd0ee6bd14d64..a5da767b79474 100644
--- a/clang/test/SemaCXX/unique_object_duplication.h
+++ b/clang/test/SemaCXX/unique_object_duplication.h
@@ -173,10 +173,6 @@ namespace GlobalTest {
 
     // Always visible
     VISIBLE static inline float allowedStaticMember2 = 0.0;
-
-    // Tests here are sparse because the AddrTest case below will define plenty
-    // more, which aren't problematic to define (because they're immutable), but
-    // may still cause problems if their address is taken.
   };
 
   inline float Test::disallowedStaticMember2 = 2.3; // hidden-warning {{'disallowedStaticMember2' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
@@ -211,3 +207,44 @@ inline int allowedTemplate2 = 0;
 template int allowedTemplate2<int>;
 
 } // namespace TemplateTest
+
+/******************************************************************************
+ * Case four: Nested classes
+ ******************************************************************************/
+
+namespace NestedClassTest {
+// Unlike visibility, declexport annotations do not propagate down to nested
+// classes. Hence on windows, we only avoid duplication of class members if
+// their immediate containing class is annotated. On posix, we get avoid
+// duplication if any containing class is annotated.
+
+class VISIBLE Outer {
+  // Visible because their class is exported
+  inline static int allowedOuterMember = 0;
+  int* allowedOuterFunction() {
+    static int allowed = 0;
+    return &allowed;
+  }
+
+  // No annotation, and visibility is only propagated on posix.
+  class HiddenOnWindows {
+    inline static int disallowedInnerMember = 0; // windows-warning {{'disallowedInnerMember' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+
+
+    int* disallowedInnerFunction() {
+      static int disallowed = 0; // windows-warning {{'disallowed' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+      return &disallowed;
+    }
+  };
+
+  class VISIBLE AlwaysVisible {
+    inline static int allowedInnerMember = 0;
+
+    int* allowedInnerFunction() {
+      static int allowed = 0;
+      return &allowed;
+    }
+  };
+};
+
+}
\ No newline at end of file
diff --git a/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp b/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp
index f228c4e6ad0c0..390305f5cb73d 100644
--- a/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp
+++ b/clang/test/SemaCXX/vtable_pointer_authentication_attribute.cpp
@@ -12,7 +12,7 @@ namespace basic {
   }
 
 TEST_AUTH(NoParams);
-// expected-error@-1{{'ptrauth_vtable_pointer' attribute takes at least 3 arguments}}
+// expected-error@-1{{'clang::ptrauth_vtable_pointer' attribute takes at least 3 arguments}}
 TEST_AUTH(NoAuth, no_authentication, default_address_discrimination, default_extra_discrimination);
 TEST_AUTH(InvalidKey, wat, default_address_discrimination, default_extra_discrimination);
 // expected-error@-1{{invalid authentication key 'wat'}}
@@ -26,7 +26,7 @@ TEST_AUTH(InvalidCustomDiscrimination, no_authentication, default_address_discri
 // expected-error@-1{{invalid custom discrimination}}
 TEST_AUTH(Default, default_key, default_address_discrimination, default_extra_discrimination);
 TEST_AUTH(InvalidDefaultExtra, default_key, default_address_discrimination, default_extra_discrimination, 1);
-// expected-error@-1{{'ptrauth_vtable_pointer' attribute takes no more than 3 arguments}}
+// expected-error@-1{{'clang::ptrauth_vtable_pointer' attribute takes no more than 3 arguments}}
 TEST_AUTH(ProcessDependentKey, process_dependent, default_address_discrimination, default_extra_discrimination);
 TEST_AUTH(ProcessIndependentKey, process_independent, default_address_discrimination, default_extra_discrimination);
 TEST_AUTH(DefaultAddressDiscrimination, process_independent, default_address_discrimination, default_extra_discrimination);
diff --git a/clang/test/SemaCXX/warn-unused-result.cpp b/clang/test/SemaCXX/warn-unused-result.cpp
index 5105f347db8b5..fa7540c116e67 100644
--- a/clang/test/SemaCXX/warn-unused-result.cpp
+++ b/clang/test/SemaCXX/warn-unused-result.cpp
@@ -108,7 +108,7 @@ void lazy() {
   (void)DoAnotherThing();
   (void)DoYetAnotherThing();
 
-  DoSomething(); // expected-warning {{ignoring return value of type 'Status' declared with 'warn_unused_result'}}
+  DoSomething(); // expected-warning {{ignoring return value of type 'Status' declared with 'clang::warn_unused_result'}}
   DoSomethingElse();
   DoAnotherThing();
   DoYetAnotherThing();
@@ -120,11 +120,11 @@ class [[clang::warn_unused_result]] StatusOr {
 StatusOr<int> doit();
 void test() {
   Foo f;
-  f.doStuff(); // expected-warning {{ignoring return value of type 'Status' declared with 'warn_unused_result'}}
-  doit(); // expected-warning {{ignoring return value of type 'StatusOr<int>' declared with 'warn_unused_result'}}
+  f.doStuff(); // expected-warning {{ignoring return value of type 'Status' declared with 'clang::warn_unused_result'}}
+  doit(); // expected-warning {{ignoring return value of type 'StatusOr<int>' declared with 'clang::warn_unused_result'}}
 
   auto func = []() { return Status(); };
-  func(); // expected-warning {{ignoring return value of type 'Status' declared with 'warn_unused_result'}}
+  func(); // expected-warning {{ignoring return value of type 'Status' declared with 'clang::warn_unused_result'}}
 }
 }
 
@@ -139,7 +139,7 @@ struct Status {};
 
 void Bar() {
   Foo f;
-  f.Bar(); // expected-warning {{ignoring return value of type 'Status' declared with 'warn_unused_result'}}
+  f.Bar(); // expected-warning {{ignoring return value of type 'Status' declared with 'clang::warn_unused_result'}}
 };
 
 }
@@ -215,18 +215,18 @@ P operator--(const P &) { return {}; };
 void f() {
   S s;
   P p;
-  s.DoThing(); // expected-warning {{ignoring return value of type 'S' declared with 'warn_unused_result'}}
-  p.DoThing(); // expected-warning {{ignoring return value of type 'P' declared with 'warn_unused_result'}}
+  s.DoThing(); // expected-warning {{ignoring return value of type 'S' declared with 'clang::warn_unused_result'}}
+  p.DoThing(); // expected-warning {{ignoring return value of type 'P' declared with 'clang::warn_unused_result'}}
   // Only postfix is expected to warn when written correctly.
-  s++; // expected-warning {{ignoring return value of type 'S' declared with 'warn_unused_result'}}
-  s--; // expected-warning {{ignoring return value of type 'S' declared with 'warn_unused_result'}}
-  p++; // expected-warning {{ignoring return value of type 'P' declared with 'warn_unused_result'}}
-  p--; // expected-warning {{ignoring return value of type 'P' declared with 'warn_unused_result'}}
+  s++; // expected-warning {{ignoring return value of type 'S' declared with 'clang::warn_unused_result'}}
+  s--; // expected-warning {{ignoring return value of type 'S' declared with 'clang::warn_unused_result'}}
+  p++; // expected-warning {{ignoring return value of type 'P' declared with 'clang::warn_unused_result'}}
+  p--; // expected-warning {{ignoring return value of type 'P' declared with 'clang::warn_unused_result'}}
   // Improperly written prefix operators should still warn.
-  ++s; // expected-warning {{ignoring return value of type 'S' declared with 'warn_unused_result'}}
-  --s; // expected-warning {{ignoring return value of type 'S' declared with 'warn_unused_result'}}
-  ++p; // expected-warning {{ignoring return value of type 'P' declared with 'warn_unused_result'}}
-  --p; // expected-warning {{ignoring return value of type 'P' declared with 'warn_unused_result'}}
+  ++s; // expected-warning {{ignoring return value of type 'S' declared with 'clang::warn_unused_result'}}
+  --s; // expected-warning {{ignoring return value of type 'S' declared with 'clang::warn_unused_result'}}
+  ++p; // expected-warning {{ignoring return value of type 'P' declared with 'clang::warn_unused_result'}}
+  --p; // expected-warning {{ignoring return value of type 'P' declared with 'clang::warn_unused_result'}}
 
   // Silencing the warning by cast to void still works.
   (void)s.DoThing();
@@ -243,7 +243,7 @@ namespace PR39837 {
 void g() {
   int a[2];
   for (int b : a)
-    f(b); // expected-warning {{ignoring return value of function declared with 'warn_unused_result'}}
+    f(b); // expected-warning {{ignoring return value of function declared with 'clang::warn_unused_result'}}
 }
 } // namespace PR39837
 
@@ -261,12 +261,12 @@ typedef a indirect;
 a af1();
 indirect indirectf1();
 void af2() {
-  af1(); // expected-warning {{ignoring return value of type 'a' declared with 'warn_unused_result'}}
+  af1(); // expected-warning {{ignoring return value of type 'a' declared with 'clang::warn_unused_result'}}
   void *(*a1)();
   a1(); // no warning
   a (*a2)();
-  a2(); // expected-warning {{ignoring return value of type 'a' declared with 'warn_unused_result'}}
-  indirectf1(); // expected-warning {{ignoring return value of type 'a' declared with 'warn_unused_result'}}
+  a2(); // expected-warning {{ignoring return value of type 'a' declared with 'clang::warn_unused_result'}}
+  indirectf1(); // expected-warning {{ignoring return value of type 'a' declared with 'clang::warn_unused_result'}}
 }
 [[nodiscard]] typedef void *b1; // expected-warning {{'[[nodiscard]]' attribute ignored when applied to a typedef; consider using '__attribute__((warn_unused_result))' or '[[clang::warn_unused_result]]' instead}}
 [[gnu::warn_unused_result]] typedef void *b2; // expected-warning {{'[[gnu::warn_unused_result]]' attribute ignored when applied to a typedef; consider using '__attribute__((warn_unused_result))' or '[[clang::warn_unused_result]]' instead}}
@@ -305,7 +305,7 @@ __attribute__((warn_unused_result)) S<T> obtain3(const T&) { return {2}; }
 void use() {
   obtain(1.0);             // no warning
   obtain(1);               // expected-warning {{ignoring return value of type 'S<int>' declared with 'nodiscard'}}
-  obtain<const double>(1); // expected-warning {{ignoring return value of type 'S<const double>' declared with 'warn_unused_result'}}
+  obtain<const double>(1); // expected-warning {{ignoring return value of type 'S<const double>' declared with 'clang::warn_unused_result'}}
 
   S<double>(2);     // no warning
   S<int>(2);        // expected-warning {{ignoring temporary of type 'S<int>' declared with 'nodiscard'}}
@@ -347,7 +347,7 @@ void use2() {
   // here, constructor/function should take precedence over type
   G{2};       // expected-warning {{ignoring temporary of type 'G' declared with 'nodiscard'}}
   G(2.0);     // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard'}}
-  G("Hello"); // expected-warning {{ignoring temporary created by a constructor declared with 'warn_unused_result'}}
+  G("Hello"); // expected-warning {{ignoring temporary created by a constructor declared with 'clang::warn_unused_result'}}
 
   // no warning for explicit cast to void
   (void)G(2);
diff --git a/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp b/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
new file mode 100644
index 0000000000000..7548ba8904a71
--- /dev/null
+++ b/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wreturn-type -Wmissing-noreturn -verify %s
+
+namespace std {
+  class string {
+  public:
+    string(const char*);
+  };
+  class runtime_error {
+  public:
+    runtime_error(const string&);
+  };
+}
+
+// This function always throws. Suggest [[noreturn]].
+void throwError(const std::string& msg) { // expected-warning {{function 'throwError' could be declared with attribute 'noreturn'}}
+  throw std::runtime_error(msg);
+}
+
+// The non-void caller should not warn about missing return.
+int ensureZero(int i) {
+  if (i == 0) return 0;
+  throwError("ERROR"); // no-warning
+}
diff --git a/clang/test/SemaHLSL/RootSignature-err.hlsl b/clang/test/SemaHLSL/RootSignature-err.hlsl
index aec8a15f8ed7f..118fc38daf3f2 100644
--- a/clang/test/SemaHLSL/RootSignature-err.hlsl
+++ b/clang/test/SemaHLSL/RootSignature-err.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - %s -verify
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -fsyntax-only %s -verify
+// RUN: not %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -fsyntax-only %s 2>&1 | FileCheck %s
 
 // Attr test
 
@@ -22,3 +23,14 @@ void bad_root_signature_4() {}
 // expected-error@+1 {{expected ')' to denote end of parameters, or, another valid parameter of RootConstants}}
 [RootSignature("RootConstants(b0, num32BitConstants = 1, invalid)")]
 void bad_root_signature_5() {}
+
+#define MultiLineRootSignature \
+ "CBV(b0)," \
+ "RootConstants(num32BitConstants = 3, b0, invalid)"
+
+// CHECK: [[@LINE-2]]:42: note: expanded from macro 'MultiLineRootSignature'
+// CHECK-NEXT: [[@LINE-3]] | "RootConstants(num32BitConstants = 3, b0, invalid)"
+// CHECK-NEXT:             |                                         ^
+// expected-error@+1 {{expected ')' to denote end of parameters, or, another valid parameter of RootConstants}}
+[RootSignature(MultiLineRootSignature)]
+void bad_root_signature_6() {}
diff --git a/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl b/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl
index e636e146cf68a..bc49fc7cabeb4 100644
--- a/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl
+++ b/clang/test/SemaHLSL/vk-ext-input-builtin.hlsl
@@ -1,24 +1,24 @@
 // RUN: %clang_cc1 -triple spirv-unkown-vulkan1.3-compute -x hlsl -hlsl-entry foo  -finclude-default-header -o - %s -verify
 
-// expected-error@+1 {{'ext_builtin_input' attribute only applies to static const globals}}
+// expected-error@+1 {{'vk::ext_builtin_input' attribute only applies to static const globals}}
 [[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
 const uint3 groupid1;
 
-// expected-error@+1 {{'ext_builtin_input' attribute only applies to static const globals}}
+// expected-error@+1 {{'vk::ext_builtin_input' attribute only applies to static const globals}}
 [[vk::ext_builtin_input(/* WorkgroupId */ 26)]]
 static uint3 groupid2;
 
-// expected-error@+1 {{'ext_builtin_input' attribute takes one argument}}
+// expected-error@+1 {{'vk::ext_builtin_input' attribute takes one argument}}
 [[vk::ext_builtin_input()]]
 // expected-error@+1 {{default initialization of an object of const type 'const hlsl_private uint3' (aka 'const hlsl_private vector<uint, 3>')}}
 static const uint3 groupid3;
 
-// expected-error@+1 {{'ext_builtin_input' attribute requires an integer constant}}
+// expected-error@+1 {{'vk::ext_builtin_input' attribute requires an integer constant}}
 [[vk::ext_builtin_input(0.4f)]]
 // expected-error@+1 {{default initialization of an object of const type 'const hlsl_private uint3' (aka 'const hlsl_private vector<uint, 3>')}}
 static const uint3 groupid4;
 
-// expected-error@+1 {{'ext_builtin_input' attribute only applies to static const globals}}
+// expected-error@+1 {{'vk::ext_builtin_input' attribute only applies to static const globals}}
 [[vk::ext_builtin_input(1)]]
 void some_function() {
 }
diff --git a/clang/test/SemaHLSL/vk.spec-constant.error.hlsl b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
index 24873d272a540..0e466cc9156cf 100644
--- a/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
+++ b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
@@ -2,7 +2,7 @@
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.8-compute -verify %s
 
 #ifndef __spirv__
-// expected-warning@+2{{'constant_id' attribute ignored}}
+// expected-warning@+2{{'vk::constant_id' attribute ignored}}
 #endif
 [[vk::constant_id(0)]]
 const bool sc0 = true;
@@ -12,7 +12,7 @@ const bool sc0 = true;
 [[vk::constant_id(1)]]
 const bool sc1 = sc0; // error
 
-// expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
+// expected-warning@+1{{'vk::constant_id' attribute only applies to external global variables}}
 [[vk::constant_id(2)]]
 static const bool sc2 = false; // error
 
@@ -30,7 +30,7 @@ const int2 sc5 = {0,0}; // error
 
 [numthreads(1,1,1)]
 void main() {
-  // expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
+  // expected-warning@+1{{'vk::constant_id' attribute only applies to external global variables}}
   [[vk::constant_id(6)]]
   const bool sc6 = false; // error
 }
diff --git a/clang/test/SemaObjC/attr-objc-gc.m b/clang/test/SemaObjC/attr-objc-gc.m
index 7bdb961833bc5..c199df725cd9f 100644
--- a/clang/test/SemaObjC/attr-objc-gc.m
+++ b/clang/test/SemaObjC/attr-objc-gc.m
@@ -19,8 +19,8 @@
 ak int i;
 
 static id [[clang::objc_gc(weak)]] j;
-[[clang::objc_gc(weak)]] static id k; // expected-warning {{applying attribute 'objc_gc' to a declaration is deprecated; apply it to the type instead}}
-static id l [[clang::objc_gc(weak)]]; // expected-warning {{applying attribute 'objc_gc' to a declaration is deprecated; apply it to the type instead}}
+[[clang::objc_gc(weak)]] static id k; // expected-warning {{applying attribute 'clang::objc_gc' to a declaration is deprecated; apply it to the type instead}}
+static id l [[clang::objc_gc(weak)]]; // expected-warning {{applying attribute 'clang::objc_gc' to a declaration is deprecated; apply it to the type instead}}
 
 void test2(id __attribute((objc_gc(strong))) *strong,
            id __attribute((objc_gc(weak))) *weak) {
diff --git a/clang/test/SemaObjCXX/arc-0x.mm b/clang/test/SemaObjCXX/arc-0x.mm
index ac788686a7374..bcaa5da6b9283 100644
--- a/clang/test/SemaObjCXX/arc-0x.mm
+++ b/clang/test/SemaObjCXX/arc-0x.mm
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fobjc-arc -fobjc-runtime-has-weak -fobjc-weak -verify -fblocks -fobjc-exceptions %s
+// RUN: %clang_cc1 -std=c++11 -fobjc-arc -fobjc-runtime-has-weak -fobjc-weak -verify -fblocks -fobjc-exceptions -emit-llvm -o - %s
 
 // "Move" semantics, trivial version.
 void move_it(__strong id &&from) {
@@ -14,6 +14,11 @@ @interface A
 // don't warn about this
 extern "C" A* MakeA();
 
+void test_nonexistent_method(A *a) {
+  // This used to crash in codegen.
+  auto a1 = [a foo]; // expected-error {{no visible @interface for 'A' declares the selector 'foo'}}
+}
+
 // Ensure that deduction works with lifetime qualifiers.
 void deduction(id obj) {
   auto a = [[A alloc] init];
diff --git a/clang/test/SemaOpenACC/cache-construct-ast.cpp b/clang/test/SemaOpenACC/cache-construct-ast.cpp
index 44c80beed77a3..fd8cd049840c2 100644
--- a/clang/test/SemaOpenACC/cache-construct-ast.cpp
+++ b/clang/test/SemaOpenACC/cache-construct-ast.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-cache-var-inside-loop -ast-dump | FileCheck %s
 
 // Test this with PCH.
 // RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
@@ -14,6 +14,21 @@ void use() {
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}Array 'int[5]'
 
+#pragma acc loop
+  // CHECK-NEXT: OpenACCLoopConstruct
+  for (int i = 0; i < 5; ++i) {
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
 #pragma acc cache(Array[1])
   // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'int' lvalue
@@ -33,6 +48,18 @@ _Pragma("acc cache(Array[1])")
   // CHECK-NEXT: DeclRefExpr{{.*}}'Array' 'int[5]'
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
+
+  // No vars here, since they are bad references.
+#pragma acc cache (Array[i])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  }
+
+  // No vars here, since they are bad references.
+#pragma acc cache(Array[1:2])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+
+  return;
+  // CHECK-NEXT: ReturnStmt
 }
 
 struct S {
@@ -42,6 +69,21 @@ struct S {
   void StructUse() {
     // CHECK: CXXMethodDecl{{.*}}StructUse 'void ()'
     // CHECK-NEXT: CompoundStmt
+#pragma acc loop
+  // CHECK-NEXT: OpenACCLoopConstruct
+  for (int i = 0; i < 5; ++i) {
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
 #pragma acc cache(Array[1])
     // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
     // CHECK-NEXT: ArraySubscriptExpr{{.*}}'int' lvalue
@@ -79,6 +121,7 @@ struct S {
     // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
     // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
   }
+  }
 };
 
 template<typename T>
@@ -89,6 +132,21 @@ void templ_use() {
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}Array 'T[5]'
 
+#pragma acc loop
+  // CHECK-NEXT: OpenACCLoopConstruct
+  for (int i = 0; i < 5; ++i) {
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
 #pragma acc cache(Array[1])
   // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'T' lvalue
@@ -101,6 +159,17 @@ void templ_use() {
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
 
+    // No vars here, since they are bad references.
+#pragma acc cache(Array[i])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  }
+  // No vars here, since they are bad references.
+#pragma acc cache(Array[1])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+
+  return;
+  // CHECK-NEXT: ReturnStmt
+
   // Instantiation:
   // CHECK: FunctionDecl{{.*}} templ_use 'void ()' implicit_instantiation
   // CHECK-NEXT: TemplateArgument type 'int'
@@ -108,6 +177,21 @@ void templ_use() {
   // CHECK-NEXT: CompoundStmt
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}Array 'int[5]'
+
+  // CHECK-NEXT: OpenACCLoopConstruct
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
+  //
   // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'int' lvalue
   // CHECK-NEXT: ImplicitCastExpr{{.*}}'int *' <ArrayToPointerDecay>
@@ -119,6 +203,11 @@ void templ_use() {
   // CHECK-NEXT: DeclRefExpr{{.*}}'Array' 'int[5]'
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
+
+  // No vars here, since they are bad references.
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  // CHECK-NEXT: ReturnStmt
 }
 
 void foo() {
diff --git a/clang/test/SemaOpenACC/cache-construct.cpp b/clang/test/SemaOpenACC/cache-construct.cpp
index 0a32754c82710..3196cd010ceae 100644
--- a/clang/test/SemaOpenACC/cache-construct.cpp
+++ b/clang/test/SemaOpenACC/cache-construct.cpp
@@ -5,13 +5,16 @@ void use() {
   int Array[5];
   int NotArray;
 
+#pragma acc loop
+  for (int i = 0; i < 5;++i) {
 #pragma acc cache(Array[1])
 #pragma acc cache(Array[1:2])
 
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(Array)
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(NotArray)
+  }
 }
 
 struct S {
@@ -20,16 +23,19 @@ struct S {
   int Array2D[5][5];
 
   void use() {
+#pragma acc loop
+  for (int i = 0; i < 5;++i) {
 #pragma acc cache(Array[1])
 #pragma acc cache(Array[1:2])
 #pragma acc cache(Array2D[1][1])
 #pragma acc cache(Array2D[1][1:2])
 
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(Array)
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(NotArray)
   }
+  }
 };
 
 template<typename T>
@@ -37,13 +43,16 @@ void templ_use() {
   T Array[5];
   T NotArray;
 
+#pragma acc loop
+  for (int i = 0; i < 5;++i) {
 #pragma acc cache(Array[1])
 #pragma acc cache(Array[1:2])
 
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(Array)
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(NotArray)
+  }
 }
 
 void foo() {
diff --git a/clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp b/clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp
new file mode 100644
index 0000000000000..bc24263c669dc
--- /dev/null
+++ b/clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+void foo() {
+  int Array[5];
+  // expected-warning@+1 2{{OpenACC variable in 'cache' directive was not declared outside of the associated 'loop' directive; directive has no effect}}
+  #pragma acc cache(readonly:Array[1], Array[1:2])
+}
+
+
+void foo2() {
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
+    int Array[5];
+    // expected-warning@+1 2{{OpenACC variable in 'cache' directive was not declared outside of the associated 'loop' directive; directive has no effect}}
+    #pragma acc cache(readonly:Array[1], Array[1:2])
+  }
+}
+
+
+template<typename T>
+void foo3() {
+  T Array[5];
+  // expected-warning@+1 2{{OpenACC variable in 'cache' directive was not declared outside of the associated 'loop' directive; directive has no effect}}
+  #pragma acc cache(readonly:Array[1], Array[1:2])
+}
+
+void Inst() {
+  foo3<int>();
+}
diff --git a/clang/test/SemaOpenCL/amd-media-ops.cl b/clang/test/SemaOpenCL/amd-media-ops.cl
new file mode 100644
index 0000000000000..9b9b9f54ecf8b
--- /dev/null
+++ b/clang/test/SemaOpenCL/amd-media-ops.cl
@@ -0,0 +1,103 @@
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops,+cl_amd_media_ops2 -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops,+cl_amd_media_ops2 -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header -fdeclare-opencl-builtins
+// expected-no-diagnostics
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops2 -verify=ops  -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops  -verify=ops2 -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops2 -verify=ops  -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header -fdeclare-opencl-builtins
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops  -verify=ops2 -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header -fdeclare-opencl-builtins
+
+#define TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    ret test_ ## builtin ## _ ## ret ## _## type (type a) { \
+        return builtin(a); \
+    }
+
+#define TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    ret test_ ## builtin ## _ ## ret ## _## type (type a, type b) { \
+        return builtin(a, b); \
+    }
+
+#define TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    ret test_ ## builtin ## _ ## ret ## _ ## type (type a, type b, type c) { \
+        return builtin(a, b, c); \
+    }
+
+#define TEST_1ARG_BUILTIN(builtin, ret, type) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 2, type ## 2) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 3, type ## 3) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 4, type ## 4) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 8, type ## 8) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 16, type ## 16)
+
+#define TEST_2ARG_BUILTIN(builtin, ret, type) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 2, type ## 2) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 3, type ## 3) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 4, type ## 4) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 8, type ## 8) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 16, type ## 16)
+
+#define TEST_3ARG_BUILTIN(builtin, ret, type) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 2, type ## 2) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 3, type ## 3) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 4, type ## 4) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 8, type ## 8) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 16, type ## 16)
+
+#define TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret, type_a, type_b, type_c) \
+    ret test_ ## builtin ## _ ## ret ## _ ## type_a ## _ ## type_b ## _ ## type_c (type_a a, type_b b, type_c c) { \
+        return builtin(a, b, c); \
+    }
+
+#define TEST_3ARG_WITH_TYPES_BUILTIN(builtin, ret, type_a, type_b, type_c) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret, type_a, type_b, type_c) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 2, type_a ## 2, type_b ## 2, type_c ## 2) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 3, type_a ## 3, type_b ## 3, type_c ## 3) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 4, type_a ## 4, type_b ## 4, type_c ## 4) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 8, type_a ## 8, type_b ## 8, type_c ## 8) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 16, type_a ## 16, type_b ## 16, type_c ## 16)
+
+TEST_3ARG_BUILTIN(amd_bitalign, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_bytealign, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_lerp, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+
+uint test_amd_pack(float4 a) {
+    return amd_pack(a); // ops-error{{use of undeclared identifier}}
+}
+
+uint test_amd_sad4(uint4 a, uint4 b, uint c) {
+    return amd_sad4(a, b, c); // ops-error{{use of undeclared identifier}}
+}
+
+TEST_3ARG_BUILTIN(amd_sadhi, uint, uint)// ops-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_sad, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+
+
+TEST_1ARG_BUILTIN(amd_unpack0, float, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_1ARG_BUILTIN(amd_unpack1, float, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_1ARG_BUILTIN(amd_unpack2, float, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_1ARG_BUILTIN(amd_unpack3, float, uint) // ops-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_WITH_TYPES_BUILTIN(amd_bfe, int, int, uint, uint) // ops2-error 6 {{use of undeclared identifier}} #amd_bfe0
+TEST_3ARG_BUILTIN(amd_bfe, uint, uint) // ops2-error 6 {{use of undeclared identifier}} #amd_bfe1
+TEST_2ARG_BUILTIN(amd_bfm, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_max3, float, float) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_max3, int, int) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_max3, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_median3, float, float) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_median3, int, int) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_median3, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_min3, float, float) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_min3, int, int) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_min3, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_WITH_TYPES_BUILTIN(amd_mqsad, ulong, ulong, uint, ulong) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_WITH_TYPES_BUILTIN(amd_qsad, ulong, ulong, uint, ulong) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_msad, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_sadd, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_sadw, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl
index b69fcb5f445bc..7494c4f984353 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl
@@ -1,6 +1,34 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx1250 -verify -S -o - %s
 
+typedef int    v4i   __attribute__((ext_vector_type(4)));
+typedef int    v8i   __attribute__((ext_vector_type(8)));
+
 void test_setprio_inc_wg(short a) {
   __builtin_amdgcn_s_setprio_inc_wg(a); // expected-error {{'__builtin_amdgcn_s_setprio_inc_wg' must be a constant integer}}
 }
+
+void test_s_monitor_sleep(short a) {
+  __builtin_amdgcn_s_monitor_sleep(a); // expected-error {{'__builtin_amdgcn_s_monitor_sleep' must be a constant integer}}
+}
+
+void test__builtin_amdgcn_cvt_f16_fp8(int a, int b) {
+  __builtin_amdgcn_cvt_f16_fp8(a, b); // expected-error {{'__builtin_amdgcn_cvt_f16_fp8' must be a constant integer}}
+}
+
+void test__builtin_amdgcn_cvt_f16_bf8(int a, int b) {
+  __builtin_amdgcn_cvt_f16_bf8(a, b); // expected-error {{'__builtin_amdgcn_cvt_f16_bf8' must be a constant integer}}
+}
+
+void test_amdgcn_tensor_load_store(v4i sg0, v8i sg1, v4i sg2, v4i sg3, int cpol)
+{
+  __builtin_amdgcn_tensor_load_to_lds(sg0, sg1, sg2, sg3, cpol); // expected-error {{'__builtin_amdgcn_tensor_load_to_lds' must be a constant integer}}
+  __builtin_amdgcn_tensor_load_to_lds_d2(sg0, sg1, cpol); // expected-error {{'__builtin_amdgcn_tensor_load_to_lds_d2' must be a constant integer}}
+  __builtin_amdgcn_tensor_store_from_lds(sg0, sg1, sg2, sg3, cpol); // expected-error {{'__builtin_amdgcn_tensor_store_from_lds' must be a constant integer}}
+  __builtin_amdgcn_tensor_store_from_lds_d2(sg0, sg1, cpol); // expected-error {{'__builtin_amdgcn_tensor_store_from_lds_d2' must be a constant integer}}
+}
+
+void test_cvt_f32_fp8_e5m3(global int* out, int a)
+{
+  *out = __builtin_amdgcn_cvt_f32_fp8_e5m3(a, a); // expected-error {{'__builtin_amdgcn_cvt_f32_fp8_e5m3' must be a constant integer}}
+}
diff --git a/clang/test/SemaSYCL/kernel-attribute-on-non-sycl.cpp b/clang/test/SemaSYCL/kernel-attribute-on-non-sycl.cpp
index 05b7d3216b884..9257c78367d7d 100644
--- a/clang/test/SemaSYCL/kernel-attribute-on-non-sycl.cpp
+++ b/clang/test/SemaSYCL/kernel-attribute-on-non-sycl.cpp
@@ -3,7 +3,7 @@
 
 #ifndef __SYCL_DEVICE_ONLY__
 // expected-warning@+7 {{'sycl_kernel' attribute ignored}}
-// expected-warning@+8 {{'sycl_kernel' attribute ignored}}
+// expected-warning@+8 {{'clang::sycl_kernel' attribute ignored}}
 #else
 // expected-no-diagnostics
 #endif
diff --git a/clang/test/SemaSYCL/kernel-attribute.cpp b/clang/test/SemaSYCL/kernel-attribute.cpp
index 84ba69fd46f9a..096b2b9191dd2 100644
--- a/clang/test/SemaSYCL/kernel-attribute.cpp
+++ b/clang/test/SemaSYCL/kernel-attribute.cpp
@@ -1,17 +1,17 @@
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -fsycl-is-device -verify %s
 
 // Only function templates
-[[clang::sycl_kernel]] int gv2 = 0; // expected-warning {{'sycl_kernel' attribute only applies to function templates}}
+[[clang::sycl_kernel]] int gv2 = 0; // expected-warning {{'clang::sycl_kernel' attribute only applies to function templates}}
 __attribute__((sycl_kernel)) int gv3 = 0; // expected-warning {{'sycl_kernel' attribute only applies to function templates}}
 
 __attribute__((sycl_kernel)) void foo(); // expected-warning {{'sycl_kernel' attribute only applies to function templates}}
-[[clang::sycl_kernel]] void foo1(); // expected-warning {{'sycl_kernel' attribute only applies to function templates}}
+[[clang::sycl_kernel]] void foo1(); // expected-warning {{'clang::sycl_kernel' attribute only applies to function templates}}
 
 // Attribute takes no arguments
 template <typename T, typename A>
 __attribute__((sycl_kernel(1))) void foo(T P); // expected-error {{'sycl_kernel' attribute takes no arguments}}
 template <typename T, typename A, int I>
-[[clang::sycl_kernel(1)]] void foo1(T P);// expected-error {{'sycl_kernel' attribute takes no arguments}}
+[[clang::sycl_kernel(1)]] void foo1(T P);// expected-error {{'clang::sycl_kernel' attribute takes no arguments}}
 
 // At least two template parameters
 template <typename T>
diff --git a/clang/test/SemaSYCL/special-class-attribute.cpp b/clang/test/SemaSYCL/special-class-attribute.cpp
index 1bfe7cc5b96c2..a8ffa88ecbee2 100644
--- a/clang/test/SemaSYCL/special-class-attribute.cpp
+++ b/clang/test/SemaSYCL/special-class-attribute.cpp
@@ -64,14 +64,14 @@ struct __attribute__((sycl_special_class)) struct7 {
 };
 
 // Only classes
-[[clang::sycl_special_class]] int var1 = 0;       // expected-warning {{'sycl_special_class' attribute only applies to classes}}
+[[clang::sycl_special_class]] int var1 = 0;       // expected-warning {{'clang::sycl_special_class' attribute only applies to classes}}
 __attribute__((sycl_special_class)) int var2 = 0; // expected-warning {{'sycl_special_class' attribute only applies to classes}}
 
-[[clang::sycl_special_class]] void foo1();       // expected-warning {{'sycl_special_class' attribute only applies to classes}}
+[[clang::sycl_special_class]] void foo1();       // expected-warning {{'clang::sycl_special_class' attribute only applies to classes}}
 __attribute__((sycl_special_class)) void foo2(); // expected-warning {{'sycl_special_class' attribute only applies to classes}}
 
 // Attribute takes no arguments
-class [[clang::sycl_special_class(1)]] class9{};         // expected-error {{'sycl_special_class' attribute takes no arguments}}
+class [[clang::sycl_special_class(1)]] class9{};         // expected-error {{'clang::sycl_special_class' attribute takes no arguments}}
 class __attribute__((sycl_special_class(1))) class10 {}; // expected-error {{'sycl_special_class' attribute takes no arguments}}
 
 // __init method must be defined inside the CXXRecordDecl.
diff --git a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp
index d06b40c5bf02d..872a44d80bf56 100644
--- a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp
+++ b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp
@@ -150,14 +150,14 @@ template<int> struct BADKN;
 
 struct B1 {
   // Non-static data member declaration.
-  // expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+  // expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
   [[clang::sycl_kernel_entry_point(BADKN<1>)]]
   int bad1;
 };
 
 struct B2 {
   // Static data member declaration.
-  // expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+  // expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
   [[clang::sycl_kernel_entry_point(BADKN<2>)]]
   static int bad2;
 };
@@ -169,42 +169,42 @@ struct B3 {
   void bad3();
 };
 
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 namespace [[clang::sycl_kernel_entry_point(BADKN<4>)]] bad4 {}
 
 #if __cplusplus >= 202002L
-// expected-error@+2 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+2 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 template<typename>
 concept bad5 [[clang::sycl_kernel_entry_point(BADKN<5>)]] = true;
 #endif
 
 // Type alias declarations.
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 typedef void bad6 [[clang::sycl_kernel_entry_point(BADKN<6>)]] ();
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 using bad7 [[clang::sycl_kernel_entry_point(BADKN<7>)]] = void();
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 using bad8 [[clang::sycl_kernel_entry_point(BADKN<8>)]] = int;
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute cannot be applied to types}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute cannot be applied to types}}
 using bad9 = int [[clang::sycl_kernel_entry_point(BADKN<9>)]];
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute cannot be applied to types}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute cannot be applied to types}}
 using bad10 = int() [[clang::sycl_kernel_entry_point(BADKN<10>)]];
 
 // Variable declaration.
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 [[clang::sycl_kernel_entry_point(BADKN<11>)]]
 int bad11;
 
 // Class declaration.
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 struct [[clang::sycl_kernel_entry_point(BADKN<12>)]] bad12;
 
 // Enumeration declaration.
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 enum [[clang::sycl_kernel_entry_point(BADKN<13>)]] bad13 {};
 
 // Enumerator.
-// expected-error@+2 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+2 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 enum {
   bad14 [[clang::sycl_kernel_entry_point(BADKN<14>)]]
 };
@@ -222,7 +222,7 @@ void bad15();
 int bad16();
 
 // Function parameters.
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute only applies to functions}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
 void bad17(void (fp [[clang::sycl_kernel_entry_point(BADKN<17>)]])());
 
 // Function template parameters.
@@ -276,7 +276,7 @@ consteval void bad25() {}
 [[clang::sycl_kernel_entry_point(BADKN<26>)]]
 [[noreturn]] void bad26();
 
-// expected-error@+3 {{attribute 'target' multiversioning cannot be combined with attribute 'sycl_kernel_entry_point'}}
+// expected-error@+3 {{attribute 'target' multiversioning cannot be combined with attribute 'clang::sycl_kernel_entry_point'}}
 __attribute__((target("avx"))) void bad27();
 [[clang::sycl_kernel_entry_point(BADKN<27>)]]
 __attribute__((target("sse4.2"))) void bad27();
diff --git a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-grammar.cpp b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-grammar.cpp
index 14b5d2746631d..8f81fa218c171 100644
--- a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-grammar.cpp
+++ b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-grammar.cpp
@@ -74,10 +74,10 @@ void test_ok13() {
 // Invalid declarations.
 ////////////////////////////////////////////////////////////////////////////////
 
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute takes one argument}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute takes one argument}}
 [[clang::sycl_kernel_entry_point]] void bad1();
 
-// expected-error@+1 {{'sycl_kernel_entry_point' attribute takes one argument}}
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute takes one argument}}
 [[clang::sycl_kernel_entry_point()]] void bad2();
 
 struct B3;
diff --git a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-ignored.cpp b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-ignored.cpp
index 30de6ae0b0e6f..4d89bbbe43fc3 100644
--- a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-ignored.cpp
+++ b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-ignored.cpp
@@ -6,11 +6,11 @@
 // A unique kernel name type is required for each declared kernel entry point.
 template<int> struct KN;
 
-// expected-warning@+1 {{'sycl_kernel_entry_point' attribute ignored}}
+// expected-warning@+1 {{'clang::sycl_kernel_entry_point' attribute ignored}}
 [[clang::sycl_kernel_entry_point(KN<1>)]]
 void ok1();
 
-// expected-warning@+2 {{'sycl_kernel_entry_point' attribute ignored}}
+// expected-warning@+2 {{'clang::sycl_kernel_entry_point' attribute ignored}}
 template<typename KNT>
 [[clang::sycl_kernel_entry_point(KNT)]]
 void ok2() {}
diff --git a/clang/test/SemaTemplate/attributes.cpp b/clang/test/SemaTemplate/attributes.cpp
index 020cfd291a502..20fe983af28f7 100644
--- a/clang/test/SemaTemplate/attributes.cpp
+++ b/clang/test/SemaTemplate/attributes.cpp
@@ -572,14 +572,14 @@ void UseRedeclaredAnnotatedFunc() {
 
 namespace preferred_name {
   int x [[clang::preferred_name("frank")]]; // expected-error {{expected a type}}
-  int y [[clang::preferred_name(int)]]; // expected-warning {{'preferred_name' attribute only applies to class templates}}
-  struct [[clang::preferred_name(int)]] A; // expected-warning {{'preferred_name' attribute only applies to class templates}}
-  template<typename T> struct [[clang::preferred_name(int)]] B; // expected-error {{argument 'int' to 'preferred_name' attribute is not a typedef for a specialization of 'B'}}
+  int y [[clang::preferred_name(int)]]; // expected-warning {{'clang::preferred_name' attribute only applies to class templates}}
+  struct [[clang::preferred_name(int)]] A; // expected-warning {{'clang::preferred_name' attribute only applies to class templates}}
+  template<typename T> struct [[clang::preferred_name(int)]] B; // expected-error {{argument 'int' to 'clang::preferred_name' attribute is not a typedef for a specialization of 'B'}}
   template<typename T> struct C;
   using X = C<int>; // expected-note {{'X' declared here}}
   typedef C<float> Y;
   using Z = const C<double>; // expected-note {{'Z' declared here}}
-  template<typename T> struct [[clang::preferred_name(C<int>)]] C; // expected-error {{argument 'C<int>' to 'preferred_name' attribute is not a typedef for a specialization of 'C'}}
+  template<typename T> struct [[clang::preferred_name(C<int>)]] C; // expected-error {{argument 'C<int>' to 'clang::preferred_name' attribute is not a typedef for a specialization of 'C'}}
   template<typename T> struct [[clang::preferred_name(X), clang::preferred_name(Y)]] C;
   template<typename T> struct [[clang::preferred_name(const X)]] C; // expected-error {{argument 'const X'}}
   template<typename T> struct [[clang::preferred_name(Z)]] C; // expected-error {{argument 'Z' (aka 'const C<double>')}}
diff --git a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
index bf505dec0ca14..9811b18f4301b 100644
--- a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
+++ b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
@@ -868,3 +868,28 @@ template <typename T> requires moo::baa<T>
 void moo::caw() {}
 
 }
+
+namespace GH145521 {
+
+template <typename X>
+concept is_valid = true;
+
+template<typename T>
+class Nesting
+{
+public:
+    template<typename Q> requires is_valid<Q>
+    class Inner;
+
+    template<typename Q> requires is_valid<Q>
+    friend class Inner2;
+};
+
+template<typename T>
+template<typename Q> requires is_valid<Q>
+class Nesting<T>::Inner {};
+
+template<typename Q> requires is_valid<Q>
+class Inner2 {};
+
+}
diff --git a/clang/test/SemaTemplate/partial-order.cpp b/clang/test/SemaTemplate/partial-order.cpp
index db2624d1766bc..2619524b147ae 100644
--- a/clang/test/SemaTemplate/partial-order.cpp
+++ b/clang/test/SemaTemplate/partial-order.cpp
@@ -47,3 +47,29 @@ namespace GH132562 {
     // expected-note@-2 {{value of type 'const J' is not implicitly convertible to 'int'}}
   } // namespace t3
 } // namespace GH132562
+
+namespace GH51866 {
+
+template <class> struct Trait;
+template <class T>
+    requires T::one
+struct Trait<T> {}; // #gh51866-one
+template <class T>
+    requires T::two
+struct Trait<T> {}; // #gh51866-two
+
+struct Y {
+    static constexpr bool one = true;
+    static constexpr bool two = true;
+};
+
+template <class T>
+concept C = sizeof(Trait<T>) != 0;
+
+static_assert(!C<Y>);
+
+Trait<Y> t;
+// expected-error@-1{{ambiguous partial specializations of 'Trait<GH51866::Y>'}}
+// expected-note@#gh51866-one{{partial specialization matches}}
+// expected-note@#gh51866-two{{partial specialization matches}}
+}
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 24bcdb5b668fc..1957bb1715eb6 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -117,6 +117,29 @@
     config.available_features.add("examples")
 
 
+def have_host_out_of_process_jit_feature_support():
+    clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
+
+    if not clang_repl_exe:
+        return False
+
+    testcode = b"\n".join([b"int i = 0;", b"%quit"])
+
+    try:
+        clang_repl_cmd = subprocess.run(
+            [clang_repl_exe, "-orc-runtime", "-oop-executor"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            input=testcode,
+        )
+    except OSError:
+        return False
+
+    if clang_repl_cmd.returncode == 0:
+        return True
+
+    return False
+
 def have_host_jit_feature_support(feature_name):
     clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
 
@@ -169,15 +192,17 @@ def have_host_clang_repl_cuda():
     if have_host_clang_repl_cuda():
         config.available_features.add('host-supports-cuda')
 
+    if have_host_out_of_process_jit_feature_support():
+        config.available_features.add("host-supports-out-of-process-jit")
+
 if config.clang_staticanalyzer:
     config.available_features.add("staticanalyzer")
     tools.append("clang-check")
 
     if config.clang_staticanalyzer_z3:
         config.available_features.add("z3")
-        config.substitutions.append(
-            ("%z3_include_dir", config.clang_staticanalyzer_z3_include_dir)
-        )
+        if config.clang_staticanalyzer_z3_mock:
+            config.available_features.add("z3-mock")
     else:
         config.available_features.add("no-z3")
 
@@ -223,9 +248,6 @@ def have_host_clang_repl_cuda():
     )
 )
 
-config.substitutions.append(("%host_cc", config.host_cc))
-config.substitutions.append(("%host_cxx", config.host_cxx))
-
 # Determine whether the test target is compatible with execution on the host.
 if "aarch64" in config.host_arch:
     config.available_features.add("aarch64-host")
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 00480d34852da..176cf644badcc 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -17,8 +17,6 @@ config.clang_tools_dir = lit_config.substitute(path(r"@CURRENT_TOOLS_DIR@"))
 config.clang_lib_dir = path(r"@CMAKE_LIBRARY_OUTPUT_DIRECTORY@")
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
-config.host_cc = "@CMAKE_C_COMPILER@"
-config.host_cxx = "@CMAKE_CXX_COMPILER@"
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.have_zstd = @LLVM_ENABLE_ZSTD@
@@ -27,7 +25,7 @@ config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@
 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
 config.clang_staticanalyzer_z3 = @LLVM_WITH_Z3@
-config.clang_staticanalyzer_z3_include_dir = "@Z3_INCLUDE_DIR@"
+config.clang_staticanalyzer_z3_mock = @TEST_WITH_Z3_MOCK@
 config.clang_enable_cir = @CLANG_ENABLE_CIR@
 config.clang_examples = @CLANG_BUILD_EXAMPLES@
 config.enable_shared = @ENABLE_SHARED@
diff --git a/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt b/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt
index 9962f9850f545..49a95a02d0447 100644
--- a/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt
+++ b/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt
@@ -24,4 +24,5 @@ add_clang_library(clangHandleLLVM
 
   DEPENDS
   intrinsics_gen
+  vt_gen
   )
diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp
index f68236abaa6c9..60e9fc4354be4 100644
--- a/clang/tools/clang-installapi/ClangInstallAPI.cpp
+++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp
@@ -102,8 +102,8 @@ static bool run(ArrayRef<const char *> Args, const char *ProgName) {
 
   if (!Opts.DriverOpts.DylibToVerify.empty()) {
     TargetList Targets;
-    llvm::for_each(Opts.DriverOpts.Targets,
-                   [&](const auto &T) { Targets.push_back(T.first); });
+    for (const auto &T : Opts.DriverOpts.Targets)
+      Targets.push_back(T.first);
     if (!Ctx.Verifier->verifyBinaryAttrs(Targets, Ctx.BA, Ctx.Reexports,
                                          Opts.LinkerOpts.AllowableClients,
                                          Opts.LinkerOpts.RPaths, Ctx.FT))
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 52b48a34d361c..64324a3f8b010 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -100,8 +100,8 @@ getArgListFromJSON(const StringRef Input, llvm::opt::OptTable *Table,
   }
 
   std::vector<const char *> CArgs(Storage.size());
-  llvm::for_each(Storage,
-                 [&CArgs](StringRef Str) { CArgs.emplace_back(Str.data()); });
+  for (StringRef Str : Storage)
+    CArgs.emplace_back(Str.data());
 
   unsigned MissingArgIndex, MissingArgCount;
   return Table->ParseArgs(CArgs, MissingArgIndex, MissingArgCount);
@@ -730,8 +730,8 @@ Options::Options(DiagnosticsEngine &Diag, FileManager *FM,
   // After all InstallAPI necessary arguments have been collected. Go back and
   // assign values that were unknown before the clang driver opt table was used.
   ArchitectureSet AllArchs;
-  llvm::for_each(DriverOpts.Targets,
-                 [&AllArchs](const auto &T) { AllArchs.set(T.first.Arch); });
+  for (const auto &T : DriverOpts.Targets)
+    AllArchs.set(T.first.Arch);
   auto assignDefaultLibAttrs = [&AllArchs](LibAttrs &Attrs) {
     for (auto &[_, Archs] : Attrs.get())
       if (Archs.empty())
@@ -794,8 +794,8 @@ std::pair<LibAttrs, ReexportedInterfaces> Options::getReexportedLibraries() {
   };
 
   PlatformSet Platforms;
-  llvm::for_each(DriverOpts.Targets,
-                 [&](const auto &T) { Platforms.insert(T.first.Platform); });
+  for (const auto &T : DriverOpts.Targets)
+    Platforms.insert(T.first.Platform);
   // Populate search paths by looking at user paths before system ones.
   PathSeq FwkSearchPaths(FEOpts.FwkPaths.begin(), FEOpts.FwkPaths.end());
   for (const PlatformType P : Platforms) {
diff --git a/clang/tools/clang-repl/CMakeLists.txt b/clang/tools/clang-repl/CMakeLists.txt
index f9a911b0ae8e2..68d86dd98cac3 100644
--- a/clang/tools/clang-repl/CMakeLists.txt
+++ b/clang/tools/clang-repl/CMakeLists.txt
@@ -4,7 +4,9 @@ set( LLVM_LINK_COMPONENTS
   LineEditor
   Option
   OrcJIT
+  OrcShared
   Support
+  TargetParser
   )
 
 add_clang_tool(clang-repl
diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp
index 7af8e4f25d99e..54b27782d3db3 100644
--- a/clang/tools/clang-repl/ClangRepl.cpp
+++ b/clang/tools/clang-repl/ClangRepl.cpp
@@ -10,7 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Interpreter/RemoteJITUtils.h"
+
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/Version.h"
+#include "clang/Config/config.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Interpreter/CodeCompletion.h"
@@ -24,8 +28,11 @@
 #include "llvm/Support/ManagedStatic.h" // llvm_shutdown
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/TargetParser/Host.h"
 #include <optional>
 
+#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h"
+
 // Disable LSan for this test.
 // FIXME: Re-enable once we can assume GCC 13.2 or higher.
 // https://llvm.org/github.com/llvm/llvm-project/issues/67586.
@@ -34,10 +41,36 @@
 LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; }
 #endif
 
+#define DEBUG_TYPE "clang-repl"
+
 static llvm::cl::opt<bool> CudaEnabled("cuda", llvm::cl::Hidden);
 static llvm::cl::opt<std::string> CudaPath("cuda-path", llvm::cl::Hidden);
 static llvm::cl::opt<std::string> OffloadArch("offload-arch", llvm::cl::Hidden);
-
+static llvm::cl::OptionCategory OOPCategory("Out-of-process Execution Options");
+static llvm::cl::opt<std::string> SlabAllocateSizeString(
+    "slab-allocate",
+    llvm::cl::desc("Allocate from a slab of the given size "
+                   "(allowable suffixes: Kb, Mb, Gb. default = "
+                   "Kb)"),
+    llvm::cl::init(""), llvm::cl::cat(OOPCategory));
+static llvm::cl::opt<std::string>
+    OOPExecutor("oop-executor",
+                llvm::cl::desc("Launch an out-of-process executor to run code"),
+                llvm::cl::init(""), llvm::cl::ValueOptional,
+                llvm::cl::cat(OOPCategory));
+static llvm::cl::opt<std::string> OOPExecutorConnect(
+    "oop-executor-connect",
+    llvm::cl::desc(
+        "Connect to an out-of-process executor through a TCP socket"),
+    llvm::cl::value_desc("<hostname>:<port>"));
+static llvm::cl::opt<std::string>
+    OrcRuntimePath("orc-runtime", llvm::cl::desc("Path to the ORC runtime"),
+                   llvm::cl::init(""), llvm::cl::ValueOptional,
+                   llvm::cl::cat(OOPCategory));
+static llvm::cl::opt<bool> UseSharedMemory(
+    "use-shared-memory",
+    llvm::cl::desc("Use shared memory to transfer generated code and data"),
+    llvm::cl::init(false), llvm::cl::cat(OOPCategory));
 static llvm::cl::list<std::string>
     ClangArgs("Xcc",
               llvm::cl::desc("Argument to pass to the CompilerInvocation"),
@@ -47,6 +80,72 @@ static llvm::cl::opt<bool> OptHostSupportsJit("host-supports-jit",
 static llvm::cl::list<std::string> OptInputs(llvm::cl::Positional,
                                              llvm::cl::desc("[code to run]"));
 
+static llvm::Error sanitizeOopArguments(const char *ArgV0) {
+  // Only one of -oop-executor and -oop-executor-connect can be used.
+  if (!!OOPExecutor.getNumOccurrences() &&
+      !!OOPExecutorConnect.getNumOccurrences())
+    return llvm::make_error<llvm::StringError>(
+        "Only one of -" + OOPExecutor.ArgStr + " and -" +
+            OOPExecutorConnect.ArgStr + " can be specified",
+        llvm::inconvertibleErrorCode());
+
+  llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
+  // TODO: Remove once out-of-process execution support is implemented for
+  // non-Unix platforms.
+  if ((!SystemTriple.isOSBinFormatELF() &&
+       !SystemTriple.isOSBinFormatMachO()) &&
+      (OOPExecutor.getNumOccurrences() ||
+       OOPExecutorConnect.getNumOccurrences()))
+    return llvm::make_error<llvm::StringError>(
+        "Out-of-process execution is only supported on Unix platforms",
+        llvm::inconvertibleErrorCode());
+
+  // If -slab-allocate is passed, check that we're not trying to use it in
+  // -oop-executor or -oop-executor-connect mode.
+  //
+  // FIXME: Remove once we enable remote slab allocation.
+  if (SlabAllocateSizeString != "") {
+    if (OOPExecutor.getNumOccurrences() ||
+        OOPExecutorConnect.getNumOccurrences())
+      return llvm::make_error<llvm::StringError>(
+          "-slab-allocate cannot be used with -oop-executor or "
+          "-oop-executor-connect",
+          llvm::inconvertibleErrorCode());
+  }
+
+  // Out-of-process executors require the ORC runtime.
+  if (OrcRuntimePath.empty() && (OOPExecutor.getNumOccurrences() ||
+                                 OOPExecutorConnect.getNumOccurrences())) {
+    llvm::SmallString<256> BasePath(llvm::sys::fs::getMainExecutable(
+        ArgV0, reinterpret_cast<void *>(&sanitizeOopArguments)));
+    llvm::sys::path::remove_filename(BasePath); // Remove clang-repl filename.
+    llvm::sys::path::remove_filename(BasePath); // Remove ./bin directory.
+    llvm::sys::path::append(BasePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang",
+                            CLANG_VERSION_MAJOR_STRING);
+    if (SystemTriple.isOSBinFormatELF())
+      OrcRuntimePath =
+          BasePath.str().str() + "/lib/x86_64-unknown-linux-gnu/liborc_rt.a";
+    else if (SystemTriple.isOSBinFormatMachO())
+      OrcRuntimePath = BasePath.str().str() + "/lib/darwin/liborc_rt_osx.a";
+    else
+      return llvm::make_error<llvm::StringError>(
+          "Out-of-process execution is not supported on non-unix platforms",
+          llvm::inconvertibleErrorCode());
+  }
+
+  // If -oop-executor was used but no value was specified then use a sensible
+  // default.
+  if (!!OOPExecutor.getNumOccurrences() && OOPExecutor.empty()) {
+    llvm::SmallString<256> OOPExecutorPath(llvm::sys::fs::getMainExecutable(
+        ArgV0, reinterpret_cast<void *>(&sanitizeOopArguments)));
+    llvm::sys::path::remove_filename(OOPExecutorPath);
+    llvm::sys::path::append(OOPExecutorPath, "llvm-jitlink-executor");
+    OOPExecutor = OOPExecutorPath.str().str();
+  }
+
+  return llvm::Error::success();
+}
+
 static void LLVMErrorHandler(void *UserData, const char *Message,
                              bool GenCrashDiag) {
   auto &Diags = *static_cast<clang::DiagnosticsEngine *>(UserData);
@@ -183,6 +282,25 @@ int main(int argc, const char **argv) {
     DeviceCI = ExitOnErr(CB.CreateCudaDevice());
   }
 
+  ExitOnErr(sanitizeOopArguments(argv[0]));
+
+  std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC;
+  if (OOPExecutor.getNumOccurrences()) {
+    // Launch an out-of-process executor locally in a child process.
+    EPC = ExitOnErr(
+        launchExecutor(OOPExecutor, UseSharedMemory, SlabAllocateSizeString));
+  } else if (OOPExecutorConnect.getNumOccurrences()) {
+    EPC = ExitOnErr(connectTCPSocket(OOPExecutorConnect, UseSharedMemory,
+                                     SlabAllocateSizeString));
+  }
+
+  std::unique_ptr<llvm::orc::LLJITBuilder> JB;
+  if (EPC) {
+    CB.SetTargetTriple(EPC->getTargetTriple().getTriple());
+    JB = ExitOnErr(
+        clang::Interpreter::createLLJITBuilder(std::move(EPC), OrcRuntimePath));
+  }
+
   // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It
   // can replace the boilerplate code for creation of the compiler instance.
   std::unique_ptr<clang::CompilerInstance> CI;
@@ -214,6 +332,9 @@ int main(int argc, const char **argv) {
       auto CudaRuntimeLibPath = CudaPath + "/lib/libcudart.so";
       ExitOnErr(Interp->LoadDynamicLibrary(CudaRuntimeLibPath.c_str()));
     }
+  } else if (JB) {
+    Interp =
+        ExitOnErr(clang::Interpreter::create(std::move(CI), std::move(JB)));
   } else
     Interp = ExitOnErr(clang::Interpreter::create(std::move(CI)));
 
diff --git a/clang/tools/libclang/Indexing.cpp b/clang/tools/libclang/Indexing.cpp
index 3cd49bf90235b..32a7147af3382 100644
--- a/clang/tools/libclang/Indexing.cpp
+++ b/clang/tools/libclang/Indexing.cpp
@@ -389,9 +389,7 @@ class IndexingFrontendAction : public ASTFrontendAction {
     if (SM.isInSystemHeader(Loc))
       return true; // always skip bodies from system headers.
 
-    FileID FID;
-    unsigned Offset;
-    std::tie(FID, Offset) = SM.getDecomposedLoc(Loc);
+    auto [FID, Offset] = SM.getDecomposedLoc(Loc);
     // Don't skip bodies from main files; this may be revisited.
     if (SM.getMainFileID() == FID)
       return false;
diff --git a/clang/unittests/AST/ASTTraverserTest.cpp b/clang/unittests/AST/ASTTraverserTest.cpp
index 8b6e3e90c0ea6..988e81d8e51de 100644
--- a/clang/unittests/AST/ASTTraverserTest.cpp
+++ b/clang/unittests/AST/ASTTraverserTest.cpp
@@ -28,6 +28,10 @@ class NodeTreePrinter : public TextTreeStructure {
       : TextTreeStructure(OS, /* showColors */ false), OS(OS) {}
 
   void Visit(const Decl *D) {
+    if (!D) {
+      OS << "<<<NULL>>>";
+      return;
+    }
     OS << D->getDeclKindName() << "Decl";
     if (auto *ND = dyn_cast<NamedDecl>(D)) {
       OS << " '" << ND->getDeclName() << "'";
@@ -1932,4 +1936,75 @@ CXXRewrittenBinaryOperator
   }
 }
 
+TEST(Traverse, CatchStatements) {
+
+  auto AST = buildASTFromCode(R"cpp(
+void test()
+{
+  try
+  {
+    int a;
+  }
+  catch (...)
+  {
+    int b;
+  }
+
+  try
+  {
+    int a;
+  }
+  catch (const int&)
+  {
+    int b;
+  }
+}
+)cpp");
+
+  auto BN =
+      ast_matchers::match(cxxCatchStmt().bind("catch"), AST->getASTContext());
+  EXPECT_EQ(BN.size(), 2u);
+  const auto *catchWithoutDecl = BN[0].getNodeAs<Stmt>("catch");
+
+  llvm::StringRef Expected = R"cpp(
+CXXCatchStmt
+|-<<<NULL>>>
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_AsIs, catchWithoutDecl), Expected);
+
+  Expected = R"cpp(
+CXXCatchStmt
+|-<<<NULL>>>
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_IgnoreUnlessSpelledInSource, catchWithoutDecl),
+            Expected);
+
+  const auto *catchWithDecl = BN[1].getNodeAs<Stmt>("catch");
+
+  Expected = R"cpp(
+CXXCatchStmt
+|-VarDecl ''
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_AsIs, catchWithDecl), Expected);
+
+  Expected = R"cpp(
+CXXCatchStmt
+|-VarDecl ''
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_IgnoreUnlessSpelledInSource, catchWithDecl),
+            Expected);
+}
+
 } // namespace clang
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index a05bf8305716b..944e7c3fb152a 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -27953,13 +27953,6 @@ TEST_F(FormatTest, AlignArrayOfStructuresLeftAlignmentNonSquare) {
                "};",
                Style);
   verifyFormat("void foo() {\n"
-               "  auto thing = test{\n"
-               "      {\n"
-               "       {13}, {something}, // A\n"
-               "      }\n"
-               "  };\n"
-               "}",
-               "void foo() {\n"
                "  auto thing = test{\n"
                "      {\n"
                "       {13},\n"
@@ -28017,13 +28010,6 @@ TEST_F(FormatTest, AlignArrayOfStructuresRightAlignmentNonSquare) {
                "};",
                Style);
   verifyFormat("void foo() {\n"
-               "  auto thing = test{\n"
-               "      {\n"
-               "       {13}, {something}, // A\n"
-               "      }\n"
-               "  };\n"
-               "}",
-               "void foo() {\n"
                "  auto thing = test{\n"
                "      {\n"
                "       {13},\n"
diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp
index 5eefd767706a3..a16fbffb76270 100644
--- a/clang/unittests/Format/FormatTestComments.cpp
+++ b/clang/unittests/Format/FormatTestComments.cpp
@@ -2486,7 +2486,7 @@ TEST_F(FormatTestComments, BlockComments) {
   EXPECT_EQ("/*\n"
             "**\n"
             "* aaaaaa\n"
-            "*aaaaaa\n"
+            "* aaaaaa\n"
             "*/",
             format("/*\n"
                    "**\n"
diff --git a/clang/unittests/Format/FormatTestSelective.cpp b/clang/unittests/Format/FormatTestSelective.cpp
index 624684c7a079b..0b7ac21fd33d3 100644
--- a/clang/unittests/Format/FormatTestSelective.cpp
+++ b/clang/unittests/Format/FormatTestSelective.cpp
@@ -41,6 +41,11 @@ TEST_F(FormatTestSelective, RemovesTrailingWhitespaceOfFormattedLine) {
   EXPECT_EQ("int a;", format("int a;         ", 0, 0));
   EXPECT_EQ("int a;\n", format("int a;  \n   \n   \n ", 0, 0));
   EXPECT_EQ("int a;\nint b;    ", format("int a;  \nint b;    ", 0, 0));
+
+  EXPECT_EQ("void f() {}", format("void f() {\n"
+                                  "  \n"
+                                  "}",
+                                  11, 0));
 }
 
 TEST_F(FormatTestSelective, FormatsCorrectRegionForLeadingWhitespace) {
diff --git a/clang/unittests/Index/IndexTests.cpp b/clang/unittests/Index/IndexTests.cpp
index 8e9a1c6bf8824..05ce448237fdf 100644
--- a/clang/unittests/Index/IndexTests.cpp
+++ b/clang/unittests/Index/IndexTests.cpp
@@ -35,9 +35,7 @@ struct Position {
 
   static Position fromSourceLocation(SourceLocation Loc,
                                      const SourceManager &SM) {
-    FileID FID;
-    unsigned Offset;
-    std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc);
+    auto [FID, Offset] = SM.getDecomposedSpellingLoc(Loc);
     Position P;
     P.Line = SM.getLineNumber(FID, Offset);
     P.Column = SM.getColumnNumber(FID, Offset);
diff --git a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
index 61f74929c1e98..d2ef27155df94 100644
--- a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
+++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
@@ -880,6 +880,37 @@ TEST(MinimizeSourceToDependencyDirectivesTest,
   EXPECT_EQ(pp_eof, Directives[22].Kind);
 }
 
+TEST(MinimizeSourceToDependencyDirectivesTest,
+     TestFixedBugThatReportUnterminatedDirectiveFalsely) {
+  SmallVector<char, 512> Out;
+  SmallVector<dependency_directives_scan::Token, 16> Tokens;
+  SmallVector<Directive, 16> Directives;
+
+  StringRef Input = "#ifndef __TEST \n"
+                    "#define __TEST \n"
+                    "#if defined(__TEST_DUMMY) \n"
+                    "#if defined(__TEST_DUMMY2) \n"
+                    "#pragma GCC warning        \\  \n"
+                    "\"hello!\"\n"
+                    "#else\n"
+                    "#pragma GCC error          \\  \n"
+                    "\"world!\" \n"
+                    "#endif // defined(__TEST_DUMMY2) \n"
+                    "#endif  // defined(__TEST_DUMMY) \n"
+                    "#endif // #ifndef __TEST \n";
+  ASSERT_FALSE( // False on no error:
+      minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives));
+  ASSERT_TRUE(Directives.size() == 8);
+  EXPECT_EQ(pp_ifndef, Directives[0].Kind);
+  EXPECT_EQ(pp_define, Directives[1].Kind);
+  EXPECT_EQ(pp_if, Directives[2].Kind);
+  EXPECT_EQ(pp_if, Directives[3].Kind);
+  EXPECT_EQ(pp_endif, Directives[4].Kind);
+  EXPECT_EQ(pp_endif, Directives[5].Kind);
+  EXPECT_EQ(pp_endif, Directives[6].Kind);
+  EXPECT_EQ(pp_eof, Directives[7].Kind);
+}
+
 TEST(MinimizeSourceToDependencyDirectivesTest, PoundWarningAndError) {
   SmallVector<char, 128> Out;
 
diff --git a/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp b/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp
index b5bd233c52557..01f8d4f97b092 100644
--- a/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Lex/LexHLSLRootSignatureTest.cpp
@@ -49,9 +49,7 @@ TEST_F(LexHLSLRootSignatureTest, ValidLexNumbersTest) {
     42.e+10f
   )cc";
 
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  hlsl::RootSignatureLexer Lexer(Source);
 
   SmallVector<hlsl::RootSignatureToken> Tokens;
   SmallVector<TokenKind> Expected = {
@@ -229,8 +227,7 @@ TEST_F(LexHLSLRootSignatureTest, ValidLexAllTokensTest) {
     STATIC_BORDER_COLOR_OPAQUE_BLACK_UINT
     STATIC_BORDER_COLOR_OPAQUE_WHITE_UINT
   )cc";
-  auto TokLoc = SourceLocation();
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  hlsl::RootSignatureLexer Lexer(Source);
 
   SmallVector<hlsl::RootSignatureToken> Tokens;
   SmallVector<TokenKind> Expected = {
@@ -251,8 +248,7 @@ TEST_F(LexHLSLRootSignatureTest, ValidCaseInsensitiveKeywordsTest) {
     SPACE visibility FLAGS
     numDescriptors OFFSET
   )cc";
-  auto TokLoc = SourceLocation();
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  hlsl::RootSignatureLexer Lexer(Source);
 
   SmallVector<hlsl::RootSignatureToken> Tokens;
   SmallVector<TokenKind> Expected = {
@@ -276,8 +272,7 @@ TEST_F(LexHLSLRootSignatureTest, ValidLexPeekTest) {
   const llvm::StringLiteral Source = R"cc(
     )1
   )cc";
-  auto TokLoc = SourceLocation();
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  hlsl::RootSignatureLexer Lexer(Source);
 
   // Test basic peek
   hlsl::RootSignatureToken Res = Lexer.peekNextToken();
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 1e46ee35d5d49..ff1697f1bbb9a 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
@@ -29,6 +31,8 @@ using namespace llvm::hlsl::rootsig;
 
 namespace {
 
+using llvm::dxbc::RootSignatureVersion;
+
 // Diagnostic helper for helper tests
 class ExpectedDiagConsumer : public DiagnosticConsumer {
   virtual void anchor() {}
@@ -91,6 +95,22 @@ class ParseHLSLRootSignatureTest : public ::testing::Test {
     return PP;
   }
 
+  std::unique_ptr<ASTContext> createMinimalASTContext() {
+    IdentifierTable Idents(LangOpts);
+    SelectorTable Selectors;
+    Builtin::Context Builtins;
+
+    return std::make_unique<ASTContext>(LangOpts, SourceMgr, Idents, Selectors,
+                                        Builtins, TU_Complete);
+  }
+
+  StringLiteral *wrapSource(std::unique_ptr<ASTContext> &Ctx,
+                            StringRef Source) {
+    SourceLocation Locs[1] = {SourceLocation()};
+    return StringLiteral::Create(*Ctx, Source, StringLiteralKind::Unevaluated,
+                                 false, Ctx->VoidTy, Locs);
+  }
+
   FileSystemOptions FileMgrOpts;
   FileManager FileMgr;
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
@@ -109,13 +129,15 @@ class ParseHLSLRootSignatureTest : public ::testing::Test {
 TEST_F(ParseHLSLRootSignatureTest, ValidParseEmptyTest) {
   const llvm::StringLiteral Source = R"cc()cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -127,6 +149,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseEmptyTest) {
 }
 
 TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
+  using llvm::dxbc::DescriptorRangeFlags;
   const llvm::StringLiteral Source = R"cc(
     DescriptorTable(
       CBV(b0),
@@ -142,13 +165,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
     DescriptorTable()
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -205,20 +230,27 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 0u);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Offset,
             DescriptorTableOffsetAppend);
+  auto ValidDescriptorRangeFlags =
+      DescriptorRangeFlags::DescriptorsVolatile |
+      DescriptorRangeFlags::DataVolatile |
+      DescriptorRangeFlags::DataStaticWhileSetAtExecute |
+      DescriptorRangeFlags::DataStatic |
+      DescriptorRangeFlags::DescriptorsStaticKeepingBufferBoundsChecks;
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
-            DescriptorRangeFlags::ValidFlags);
+            ValidDescriptorRangeFlags);
 
   Elem = Elements[4];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
   ASSERT_EQ(std::get<DescriptorTable>(Elem).NumClauses, (uint32_t)4);
   ASSERT_EQ(std::get<DescriptorTable>(Elem).Visibility,
-            ShaderVisibility::Pixel);
+            llvm::dxbc::ShaderVisibility::Pixel);
 
   // Empty Descriptor Table
   Elem = Elements[5];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
   ASSERT_EQ(std::get<DescriptorTable>(Elem).NumClauses, 0u);
-  ASSERT_EQ(std::get<DescriptorTable>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<DescriptorTable>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
@@ -238,13 +270,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -258,19 +292,25 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
   ASSERT_TRUE(std::holds_alternative<StaticSampler>(Elem));
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.ViewType, RegisterType::SReg);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.Number, 0u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).Filter, SamplerFilter::Anisotropic);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU, TextureAddressMode::Wrap);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV, TextureAddressMode::Wrap);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW, TextureAddressMode::Wrap);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Filter,
+            llvm::dxbc::SamplerFilter::Anisotropic);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU,
+            llvm::dxbc::TextureAddressMode::Wrap);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV,
+            llvm::dxbc::TextureAddressMode::Wrap);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW,
+            llvm::dxbc::TextureAddressMode::Wrap);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MipLODBias, 0.f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).MaxAnisotropy, 16u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc, ComparisonFunc::LessEqual);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc,
+            llvm::dxbc::ComparisonFunc::LessEqual);
   ASSERT_EQ(std::get<StaticSampler>(Elem).BorderColor,
-            StaticBorderColor::OpaqueWhite);
+            llvm::dxbc::StaticBorderColor::OpaqueWhite);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MinLOD, 0.f);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MaxLOD, 3.402823466e+38f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
 
   // Check values can be set as expected
   Elem = Elements[1];
@@ -278,19 +318,24 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.ViewType, RegisterType::SReg);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Filter,
-            SamplerFilter::MaximumMinPointMagLinearMipPoint);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU, TextureAddressMode::Mirror);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV, TextureAddressMode::Border);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW, TextureAddressMode::Clamp);
+            llvm::dxbc::SamplerFilter::MaximumMinPointMagLinearMipPoint);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU,
+            llvm::dxbc::TextureAddressMode::Mirror);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV,
+            llvm::dxbc::TextureAddressMode::Border);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW,
+            llvm::dxbc::TextureAddressMode::Clamp);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MipLODBias, 230.f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).MaxAnisotropy, 3u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc, ComparisonFunc::NotEqual);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc,
+            llvm::dxbc::ComparisonFunc::NotEqual);
   ASSERT_EQ(std::get<StaticSampler>(Elem).BorderColor,
-            StaticBorderColor::OpaqueBlackUint);
+            llvm::dxbc::StaticBorderColor::OpaqueBlackUint);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MinLOD, 4.2f);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MaxLOD, 9000.f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Space, 4u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility, ShaderVisibility::Domain);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::Domain);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
@@ -312,13 +357,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseFloatsTest) {
     StaticSampler(s0, mipLODBias = 2147483648),
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -387,13 +434,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidSamplerFlagsTest) {
     DescriptorTable(Sampler(s0, flags = DESCRIPTORS_VOLATILE))
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -403,8 +452,9 @@ TEST_F(ParseHLSLRootSignatureTest, ValidSamplerFlagsTest) {
   RootElement Elem = Elements[0];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
-            DescriptorRangeFlags::ValidSamplerFlags);
+  auto ValidSamplerFlags =
+      llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile;
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidSamplerFlags);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
@@ -417,13 +467,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootConsantsTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -438,7 +490,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootConsantsTest) {
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootConstants>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
 
   Elem = Elements[1];
   ASSERT_TRUE(std::holds_alternative<RootConstants>(Elem));
@@ -446,12 +499,14 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootConsantsTest) {
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.Number, 42u);
   ASSERT_EQ(std::get<RootConstants>(Elem).Space, 3u);
-  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility, ShaderVisibility::Hull);
+  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::Hull);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
 TEST_F(ParseHLSLRootSignatureTest, ValidParseRootFlagsTest) {
+  using llvm::dxbc::RootFlags;
   const llvm::StringLiteral Source = R"cc(
     RootFlags(),
     RootFlags(0),
@@ -471,13 +526,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootFlagsTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -496,12 +553,25 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootFlagsTest) {
 
   Elem = Elements[2];
   ASSERT_TRUE(std::holds_alternative<RootFlags>(Elem));
-  ASSERT_EQ(std::get<RootFlags>(Elem), RootFlags::ValidFlags);
+  auto ValidRootFlags = RootFlags::AllowInputAssemblerInputLayout |
+                        RootFlags::DenyVertexShaderRootAccess |
+                        RootFlags::DenyHullShaderRootAccess |
+                        RootFlags::DenyDomainShaderRootAccess |
+                        RootFlags::DenyGeometryShaderRootAccess |
+                        RootFlags::DenyPixelShaderRootAccess |
+                        RootFlags::AllowStreamOutput |
+                        RootFlags::LocalRootSignature |
+                        RootFlags::DenyAmplificationShaderRootAccess |
+                        RootFlags::DenyMeshShaderRootAccess |
+                        RootFlags::CBVSRVUAVHeapDirectlyIndexed |
+                        RootFlags::SamplerHeapDirectlyIndexed;
+  ASSERT_EQ(std::get<RootFlags>(Elem), ValidRootFlags);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
 TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
+  using llvm::dxbc::RootDescriptorFlags;
   const llvm::StringLiteral Source = R"cc(
     CBV(b0),
     SRV(space = 4, t42, visibility = SHADER_VISIBILITY_GEOMETRY,
@@ -511,13 +581,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
     CBV(b0, flags = 0),
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -532,7 +604,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             RootDescriptorFlags::DataStaticWhileSetAtExecute);
 
@@ -543,9 +616,12 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 42u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 4u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
-            ShaderVisibility::Geometry);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
-            RootDescriptorFlags::ValidFlags);
+            llvm::dxbc::ShaderVisibility::Geometry);
+  auto ValidRootDescriptorFlags =
+      RootDescriptorFlags::DataVolatile |
+      RootDescriptorFlags::DataStaticWhileSetAtExecute |
+      RootDescriptorFlags::DataStatic;
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, ValidRootDescriptorFlags);
 
   Elem = Elements[2];
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
@@ -553,7 +629,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::UReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 34893247u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility, ShaderVisibility::Hull);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::Hull);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             RootDescriptorFlags::DataVolatile);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
@@ -564,7 +641,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, RootDescriptorFlags::None);
 
   ASSERT_TRUE(Consumer->isSatisfied());
@@ -579,19 +657,169 @@ TEST_F(ParseHLSLRootSignatureTest, ValidTrailingCommaTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
+
+  // Test no diagnostics produced
+  Consumer->setNoDiag();
+
+  ASSERT_FALSE(Parser.parse());
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, ValidVersion10Test) {
+  // This test checks that the default values are set correctly
+  // when parsing with root signature version 1.0
+  const llvm::StringLiteral Source = R"cc(
+    CBV(b0),
+    SRV(t0),
+    UAV(u0),
+    DescriptorTable(
+      CBV(b1),
+      SRV(t1),
+      UAV(u1),
+      Sampler(s1),
+    )
+  )cc";
+
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_0, Elements,
+                                   Signature, *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
 
   ASSERT_FALSE(Parser.parse());
 
+  auto DefRootDescriptorFlag = llvm::dxbc::RootDescriptorFlags::DataVolatile;
+  RootElement Elem = Elements[0];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
+
+  Elem = Elements[1];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
+
+  Elem = Elements[2];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
+
+  auto ValidNonSamplerFlags =
+      llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile |
+      llvm::dxbc::DescriptorRangeFlags::DataVolatile;
+  Elem = Elements[3];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
+
+  Elem = Elements[4];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
+
+  Elem = Elements[5];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
+
+  Elem = Elements[6];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile);
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, ValidVersion11Test) {
+  // This test checks that the default values are set correctly
+  // when parsing with root signature version 1.1
+  const llvm::StringLiteral Source = R"cc(
+    CBV(b0),
+    SRV(t0),
+    UAV(u0),
+    DescriptorTable(
+      CBV(b1),
+      SRV(t1),
+      UAV(u1),
+      Sampler(s1),
+    )
+  )cc";
+
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
+
+  // Test no diagnostics produced
+  Consumer->setNoDiag();
+
+  ASSERT_FALSE(Parser.parse());
+
+  RootElement Elem = Elements[0];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
+            llvm::dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[1];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
+            llvm::dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[2];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
+            llvm::dxbc::RootDescriptorFlags::DataVolatile);
+
+  Elem = Elements[3];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[4];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[5];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DataVolatile);
+
+  Elem = Elements[6];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::None);
+
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
@@ -603,13 +831,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedTokenTest) {
     space
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_unexpected_end_of_params);
@@ -623,13 +853,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseInvalidTokenTest) {
     notAnIdentifier
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced - invalid token
   Consumer->setExpected(diag::err_hlsl_unexpected_end_of_params);
@@ -643,13 +875,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedEndOfStreamTest) {
     DescriptorTable
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced - end of stream
   Consumer->setExpected(diag::err_expected_after);
@@ -668,13 +902,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidMissingDTParameterTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
@@ -690,13 +926,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidMissingRDParameterTest) {
     SRV()
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
@@ -712,13 +950,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidMissingRCParameterTest) {
     RootConstants(b0)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
@@ -736,13 +976,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedMandatoryDTParameterTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -758,13 +1000,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedMandatoryRCParameterTest) {
     RootConstants(num32BitConstants = 32, num32BitConstants = 24)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -782,13 +1026,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedOptionalDTParameterTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -808,13 +1054,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedOptionalRCParameterTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -831,13 +1079,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedNumberTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -853,13 +1103,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseOverflowedNegativeNumberTest) {
     StaticSampler(s0, mipLODBias = -4294967295)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -874,13 +1126,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedFloatTest) {
     StaticSampler(s0, mipLODBias = 3.402823467e+38F)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -895,13 +1149,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexNegOverflowedFloatTest) {
     StaticSampler(s0, mipLODBias = -3.402823467e+38F)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -916,13 +1172,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedDoubleTest) {
     StaticSampler(s0, mipLODBias = 1.e+500)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -937,13 +1195,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexUnderflowFloatTest) {
     StaticSampler(s0, mipLODBias = 10e-309)
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_underflow);
@@ -961,13 +1221,15 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidNonZeroFlagsTest) {
     )
   )cc";
 
+  auto Ctx = createMinimalASTContext();
+  StringLiteral *Signature = wrapSource(Ctx, Source);
+
   TrivialModuleLoader ModLoader;
   auto PP = createPP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
 
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements,
+                                   Signature, *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_non_zero_flag);
diff --git a/clang/unittests/Sema/HeuristicResolverTest.cpp b/clang/unittests/Sema/HeuristicResolverTest.cpp
index 3ed6bba790be3..ee434f7a1d43a 100644
--- a/clang/unittests/Sema/HeuristicResolverTest.cpp
+++ b/clang/unittests/Sema/HeuristicResolverTest.cpp
@@ -766,5 +766,85 @@ TEST(HeuristicResolver, UsingValueDecl) {
                    cxxMethodDecl(hasName("waldo")).bind("output"));
 }
 
+// `arg` is a ParamVarDecl*, `Expected` is a string
+MATCHER_P(ParamNameMatcher, Expected, "paramNameMatcher") {
+  EXPECT_TRUE(arg);
+  if (IdentifierInfo *Ident = arg->getDeclName().getAsIdentifierInfo()) {
+    return Ident->getName() == Expected;
+  }
+  return false;
+}
+
+// Helper function for testing HeuristicResolver::getProtoTypeLoc.
+// Takes a matcher that selects a callee expression bound to the ID "input",
+// calls getProtoTypeLoc() on it, and checks that the call found a
+// FunctionProtoTypeLoc encoding the given parameter names.
+template <typename InputMatcher, typename... ParameterNames>
+void expectParameterNames(ASTContext &Ctx, const InputMatcher &IM,
+                          ParameterNames... ExpectedParameterNames) {
+  auto InputMatches = match(IM, Ctx);
+  ASSERT_EQ(1u, InputMatches.size());
+  const auto *Input = InputMatches[0].template getNodeAs<Expr>("input");
+  ASSERT_TRUE(Input);
+
+  HeuristicResolver H(Ctx);
+  auto Loc = H.getFunctionProtoTypeLoc(Input);
+  ASSERT_TRUE(Loc);
+  EXPECT_THAT(Loc.getParams(),
+              ElementsAre(ParamNameMatcher(ExpectedParameterNames)...));
+}
+
+TEST(HeuristicResolver, ProtoTypeLoc) {
+  std::string Code = R"cpp(
+    void (*f1)(int param1);
+    void (__stdcall *f2)(int param2);
+    using f3_t = void(*)(int param3);
+    f3_t f3;
+    using f4_t = void(__stdcall *)(int param4);
+    f4_t f4;
+    struct S {
+      void (*f5)(int param5);
+      using f6_t = void(*)(int param6);
+      f6_t f6;
+    };
+    void bar() {
+      f1(42);
+      f2(42);
+      f3(42);
+      f4(42);
+      S s;
+      s.f5(42);
+      s.f6(42);
+    }
+  )cpp";
+  auto TU = tooling::buildASTFromCodeWithArgs(Code, {"-std=c++20"});
+  auto &Ctx = TU->getASTContext();
+  auto checkFreeFunction = [&](llvm::StringRef FunctionName,
+                               llvm::StringRef ParamName) {
+    expectParameterNames(
+        Ctx,
+        callExpr(
+            callee(implicitCastExpr(hasSourceExpression(declRefExpr(
+                                        to(namedDecl(hasName(FunctionName))))))
+                       .bind("input"))),
+        ParamName);
+  };
+  checkFreeFunction("f1", "param1");
+  checkFreeFunction("f2", "param2");
+  checkFreeFunction("f3", "param3");
+  checkFreeFunction("f4", "param4");
+  auto checkMemberFunction = [&](llvm::StringRef MemberName,
+                                 llvm::StringRef ParamName) {
+    expectParameterNames(
+        Ctx,
+        callExpr(callee(implicitCastExpr(hasSourceExpression(memberExpr(
+                                             member(hasName(MemberName)))))
+                            .bind("input"))),
+        ParamName);
+  };
+  checkMemberFunction("f5", "param5");
+  checkMemberFunction("f6", "param6");
+}
+
 } // namespace
 } // namespace clang
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index dfeb6b1b1ec19..d63e79a5f5155 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -4904,9 +4904,8 @@ void EmitClangAttrParsedAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
   }
 
   OS << "static const ParsedAttrInfo *AttrInfoMap[] = {\n";
-  for (auto I = Attrs.begin(), E = Attrs.end(); I != E; ++I) {
-    OS << "&ParsedAttrInfo" << I->first << "::Instance,\n";
-  }
+  for (const auto &Attr : Attrs)
+    OS << "&ParsedAttrInfo" << Attr.first << "::Instance,\n";
   OS << "};\n\n";
 
   // Generate function for handling attributes with delayed arguments
@@ -5261,10 +5260,9 @@ GetAttributeHeadingAndSpellings(const Record &Documentation,
       Heading = Spellings.begin()->name();
     else {
       std::set<std::string> Uniques;
-      for (auto I = Spellings.begin(), E = Spellings.end();
-           I != E; ++I) {
+      for (const FlattenedSpelling &FS : Spellings) {
         std::string Spelling =
-            NormalizeNameForSpellingComparison(I->name()).str();
+            NormalizeNameForSpellingComparison(FS.name()).str();
         Uniques.insert(Spelling);
       }
       // If the semantic map has only one spelling, that is sufficient for our
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index b506546110983..e2da20efaba5a 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -1901,6 +1901,9 @@ void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
     if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
         !Def->getSMEGuard().empty())
       report_fatal_error("Missing VerifyRuntimeMode flag");
+    if (Def->isFlagSet(VerifyRuntimeMode) &&
+        (Def->getSVEGuard().empty() || Def->getSMEGuard().empty()))
+      report_fatal_error("VerifyRuntimeMode requires SVE and SME guards");
 
     if (Def->isFlagSet(IsStreamingFlag))
       StreamingMap["ArmStreaming"].insert(Def->getMangledName());
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index a70e65e35d5ee..bb7144b827c3c 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -317,7 +317,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Attaching main to the global module</td>
   <td><a href="https://wg21.link/P3618">P3618R0</a> (<a href="#dr">DR</a>)</td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 21</td>
  </tr>
  <tr>
   <td>Expansion Statements</td>
@@ -911,7 +911,7 @@ <h2 id="cxx20">C++20 implementation status</h2>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p1811r0">P1811R0</a></td>
-        <td class="none" align="center">No</td>
+        <td class="full" align="center">Clang 17</td>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p1703r1">P1703R1</a></td>
@@ -923,7 +923,7 @@ <h2 id="cxx20">C++20 implementation status</h2>
       </tr>
       <tr> <!-- from Belfast -->
         <td><a href="https://wg21.link/p1979r0">P1979R0</a></td>
-        <td class="none" align="center">No</td>
+        <td class="full" align="center">Clang 17</td>
       </tr>
       <tr> <!-- from Prague -->
         <td><a href="https://wg21.link/p1779r3">P1779R3</a></td>
@@ -935,7 +935,7 @@ <h2 id="cxx20">C++20 implementation status</h2>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p2115r0">P2115R0</a></td>
-        <td class="partial" align="center">Partial</td>
+        <td class="full" align="center">Yes</td>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p1815r2">P1815R2</a></td>
@@ -943,11 +943,11 @@ <h2 id="cxx20">C++20 implementation status</h2>
       </tr>
       <tr> <!-- from Kona 2022 -->
         <td><a href="https://wg21.link/P2615R1">P2615R1</a> (<a href="#dr">DR</a>)</td>
-        <td class="none" align="center">No</td>
+        <td class="full" align="center">Clang 17</td>
       </tr>
       <tr> <!-- from Issaquah 2023 -->
         <td><a href="https://wg21.link/P2788R0">P2788R0</a> (<a href="#dr">DR</a>)</td>
-        <td class="none" align="center">No</td>
+        <td class="full" align="center">Clang 17</td>
       </tr>
     <tr>
       <td>Coroutines</td>
diff --git a/cmake/Modules/GetClangResourceDir.cmake b/cmake/Modules/GetClangResourceDir.cmake
index def5bd60424e7..518220188f96d 100644
--- a/cmake/Modules/GetClangResourceDir.cmake
+++ b/cmake/Modules/GetClangResourceDir.cmake
@@ -10,7 +10,8 @@ function(get_clang_resource_dir out_var)
   cmake_parse_arguments(ARG "" "PREFIX;SUBDIR" "" ${ARGN})
 
   if(DEFINED CLANG_RESOURCE_DIR AND NOT CLANG_RESOURCE_DIR STREQUAL "")
-    set(ret_dir bin/${CLANG_RESOURCE_DIR})
+    set(ret_dir bin)
+    cmake_path(APPEND ret_dir ${CLANG_RESOURCE_DIR})
   else()
     if (NOT CLANG_VERSION_MAJOR)
       string(REGEX MATCH "^[0-9]+" CLANG_VERSION_MAJOR ${PACKAGE_VERSION})
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index 45a2f5c0a61fc..9a0426ff29470 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -584,7 +584,7 @@ endmacro(add_compiler_rt_script src name)
 
 
 macro(add_compiler_rt_cfg target_name file_name component arch)
-  set(src_file "${CMAKE_CURRENT_SOURCE_DIR}/${file_name}")
+  set(src_file "${CMAKE_CURRENT_SOURCE_DIR}/AIX/${file_name}")
   get_compiler_rt_output_dir(${arch} output_dir)
   set(dst_file "${output_dir}/${file_name}")
   add_custom_command(OUTPUT ${dst_file}
diff --git a/compiler-rt/lib/asan/asan.link_with_main_exec.txt b/compiler-rt/lib/asan/AIX/asan.link_with_main_exec.txt
similarity index 100%
rename from compiler-rt/lib/asan/asan.link_with_main_exec.txt
rename to compiler-rt/lib/asan/AIX/asan.link_with_main_exec.txt
diff --git a/compiler-rt/lib/asan/asan_cxx.link_with_main_exec.txt b/compiler-rt/lib/asan/AIX/asan_cxx.link_with_main_exec.txt
similarity index 100%
rename from compiler-rt/lib/asan/asan_cxx.link_with_main_exec.txt
rename to compiler-rt/lib/asan/AIX/asan_cxx.link_with_main_exec.txt
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 3a55c2af65653..d3c0288285b86 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -424,10 +424,15 @@ struct Allocator {
     PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
   }
 
-  void ReInitialize(const AllocatorOptions &options) {
+  // Apply provided AllocatorOptions to an Allocator
+  void ApplyOptions(const AllocatorOptions &options) {
     SetAllocatorMayReturnNull(options.may_return_null);
     allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms);
     SharedInitCode(options);
+  }
+
+  void ReInitialize(const AllocatorOptions &options) {
+    ApplyOptions(options);
 
     // Poison all existing allocation's redzones.
     if (CanPoisonMemory()) {
@@ -977,6 +982,11 @@ void ReInitializeAllocator(const AllocatorOptions &options) {
   instance.ReInitialize(options);
 }
 
+// Apply provided AllocatorOptions to an Allocator
+void ApplyAllocatorOptions(const AllocatorOptions &options) {
+  instance.ApplyOptions(options);
+}
+
 void GetAllocatorOptions(AllocatorOptions *options) {
   instance.GetOptions(options);
 }
diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index db8dc3bebfc62..a94ef958aa75e 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -47,6 +47,7 @@ struct AllocatorOptions {
 void InitializeAllocator(const AllocatorOptions &options);
 void ReInitializeAllocator(const AllocatorOptions &options);
 void GetAllocatorOptions(AllocatorOptions *options);
+void ApplyAllocatorOptions(const AllocatorOptions &options);
 
 class AsanChunkView {
  public:
diff --git a/compiler-rt/lib/asan/asan_flags.cpp b/compiler-rt/lib/asan/asan_flags.cpp
index 9cfb70bd00c78..190a89345dd18 100644
--- a/compiler-rt/lib/asan/asan_flags.cpp
+++ b/compiler-rt/lib/asan/asan_flags.cpp
@@ -144,6 +144,7 @@ static void InitializeDefaultFlags() {
   DisplayHelpMessages(&asan_parser);
 }
 
+// Validate flags and report incompatible configurations
 static void ProcessFlags() {
   Flags *f = flags();
 
@@ -217,11 +218,12 @@ void InitializeFlags() {
   ProcessFlags();
 
 #if SANITIZER_WINDOWS
-  // On Windows, weak symbols are emulated by having the user program
-  // register which weak functions are defined.
-  // The ASAN DLL will initialize flags prior to user module initialization,
-  // so __asan_default_options will not point to the user definition yet.
-  // We still want to ensure we capture when options are passed via
+  // On Windows, weak symbols (such as the `__asan_default_options` function)
+  // are emulated by having the user program register which weak functions are
+  // defined. The ASAN DLL will initialize flags prior to user module
+  // initialization, so __asan_default_options will not point to the user
+  // definition yet. We still want to ensure we capture when options are passed
+  // via
   // __asan_default_options, so we add a callback to be run
   // when it is registered with the runtime.
 
@@ -232,21 +234,13 @@ void InitializeFlags() {
   // __sanitizer_register_weak_function.
   AddRegisterWeakFunctionCallback(
       reinterpret_cast<uptr>(__asan_default_options), []() {
-        FlagParser asan_parser;
-
-        RegisterAsanFlags(&asan_parser, flags());
-        RegisterCommonFlags(&asan_parser);
-        asan_parser.ParseString(__asan_default_options());
-
-        DisplayHelpMessages(&asan_parser);
+        // We call `InitializeDefaultFlags` again, instead of just parsing
+        // `__asan_default_options` directly, to ensure that flags set through
+        // `ASAN_OPTS` take precedence over those set through
+        // `__asan_default_options`.
+        InitializeDefaultFlags();
         ProcessFlags();
-
-        // TODO: Update other globals and data structures that may need to change
-        // after initialization due to new flags potentially being set changing after
-        // `__asan_default_options` is registered.
-        // See GH issue 'https://github.com/llvm/llvm-project/issues/117925' for
-        // details.
-        SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+        ApplyFlags();
       });
 
 #  if CAN_SANITIZE_UB
diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index 8205e19fdab7f..35c887eaf851c 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -61,6 +61,7 @@ using __sanitizer::StackTrace;
 
 void AsanInitFromRtl();
 bool TryAsanInitFromRtl();
+void ApplyFlags();
 
 // asan_win.cpp
 void InitializePlatformExceptionHandlers();
diff --git a/compiler-rt/lib/asan/asan_new_delete.cpp b/compiler-rt/lib/asan/asan_new_delete.cpp
index b5b1ced8ac5ed..99c7c9938dfb3 100644
--- a/compiler-rt/lib/asan/asan_new_delete.cpp
+++ b/compiler-rt/lib/asan/asan_new_delete.cpp
@@ -60,18 +60,42 @@ enum class align_val_t: size_t {};
 // TODO(alekseyshl): throw std::bad_alloc instead of dying on OOM.
 // For local pool allocation, align to SHADOW_GRANULARITY to match asan
 // allocator behavior.
-#define OPERATOR_NEW_BODY(type, nothrow)            \
-  GET_STACK_TRACE_MALLOC;                           \
-  void *res = asan_memalign(0, size, &stack, type); \
-  if (!nothrow && UNLIKELY(!res))                   \
-    ReportOutOfMemory(size, &stack);                \
-  return res;
-#define OPERATOR_NEW_BODY_ALIGN(type, nothrow)                \
-  GET_STACK_TRACE_MALLOC;                                     \
-  void *res = asan_memalign((uptr)align, size, &stack, type); \
-  if (!nothrow && UNLIKELY(!res))                             \
-    ReportOutOfMemory(size, &stack);                          \
-  return res;
+#define OPERATOR_NEW_BODY                               \
+  GET_STACK_TRACE_MALLOC;                               \
+  void *res = asan_memalign(0, size, &stack, FROM_NEW); \
+  if (UNLIKELY(!res))                                   \
+    ReportOutOfMemory(size, &stack);                    \
+  return res
+#define OPERATOR_NEW_BODY_NOTHROW \
+  GET_STACK_TRACE_MALLOC;         \
+  return asan_memalign(0, size, &stack, FROM_NEW)
+#define OPERATOR_NEW_BODY_ARRAY                            \
+  GET_STACK_TRACE_MALLOC;                                  \
+  void *res = asan_memalign(0, size, &stack, FROM_NEW_BR); \
+  if (UNLIKELY(!res))                                      \
+    ReportOutOfMemory(size, &stack);                       \
+  return res
+#define OPERATOR_NEW_BODY_ARRAY_NOTHROW \
+  GET_STACK_TRACE_MALLOC;               \
+  return asan_memalign(0, size, &stack, FROM_NEW_BR)
+#define OPERATOR_NEW_BODY_ALIGN                                   \
+  GET_STACK_TRACE_MALLOC;                                         \
+  void *res = asan_memalign((uptr)align, size, &stack, FROM_NEW); \
+  if (UNLIKELY(!res))                                             \
+    ReportOutOfMemory(size, &stack);                              \
+  return res
+#define OPERATOR_NEW_BODY_ALIGN_NOTHROW \
+  GET_STACK_TRACE_MALLOC;               \
+  return asan_memalign((uptr)align, size, &stack, FROM_NEW)
+#define OPERATOR_NEW_BODY_ALIGN_ARRAY                                \
+  GET_STACK_TRACE_MALLOC;                                            \
+  void *res = asan_memalign((uptr)align, size, &stack, FROM_NEW_BR); \
+  if (UNLIKELY(!res))                                                \
+    ReportOutOfMemory(size, &stack);                                 \
+  return res
+#define OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW \
+  GET_STACK_TRACE_MALLOC;                     \
+  return asan_memalign((uptr)align, size, &stack, FROM_NEW_BR)
 
 // On OS X it's not enough to just provide our own 'operator new' and
 // 'operator delete' implementations, because they're going to be in the
@@ -82,106 +106,128 @@ enum class align_val_t: size_t {};
 // OS X we need to intercept them using their mangled names.
 #if !SANITIZER_APPLE
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size)
-{ OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); }
+void *operator new(size_t size) { OPERATOR_NEW_BODY; }
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size)
-{ OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/); }
+void *operator new[](size_t size) { OPERATOR_NEW_BODY_ARRAY; }
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/); }
+void *operator new(size_t size, std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_NOTHROW;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); }
+void *operator new[](size_t size, std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ARRAY_NOTHROW;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/); }
+void *operator new(size_t size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/); }
+void *operator new[](size_t size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/); }
+void *operator new(size_t size, std::align_val_t align,
+                   std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ALIGN_NOTHROW;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); }
+void *operator new[](size_t size, std::align_val_t align,
+                     std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW;
+}
 
 #else  // SANITIZER_APPLE
-INTERCEPTOR(void *, _Znwm, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/);
-}
-INTERCEPTOR(void *, _Znam, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/);
-}
+INTERCEPTOR(void *, _Znwm, size_t size) { OPERATOR_NEW_BODY; }
+INTERCEPTOR(void *, _Znam, size_t size) { OPERATOR_NEW_BODY_ARRAY; }
 INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/);
+  OPERATOR_NEW_BODY_NOTHROW;
 }
 INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
+  OPERATOR_NEW_BODY_ARRAY_NOTHROW;
 }
 #endif  // !SANITIZER_APPLE
 
-#define OPERATOR_DELETE_BODY(type) \
+#define OPERATOR_DELETE_BODY \
+  GET_STACK_TRACE_FREE;      \
+  asan_delete(ptr, 0, 0, &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_ARRAY \
   GET_STACK_TRACE_FREE;            \
-  asan_delete(ptr, 0, 0, &stack, type);
-
-#define OPERATOR_DELETE_BODY_SIZE(type) \
-  GET_STACK_TRACE_FREE;                 \
-  asan_delete(ptr, size, 0, &stack, type);
-
-#define OPERATOR_DELETE_BODY_ALIGN(type) \
+  asan_delete(ptr, 0, 0, &stack, FROM_NEW_BR)
+#define OPERATOR_DELETE_BODY_ALIGN \
+  GET_STACK_TRACE_FREE;            \
+  asan_delete(ptr, 0, static_cast<uptr>(align), &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_ALIGN_ARRAY \
   GET_STACK_TRACE_FREE;                  \
-  asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
-
-#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
+  asan_delete(ptr, 0, static_cast<uptr>(align), &stack, FROM_NEW_BR)
+#define OPERATOR_DELETE_BODY_SIZE \
+  GET_STACK_TRACE_FREE;           \
+  asan_delete(ptr, size, 0, &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_SIZE_ARRAY \
+  GET_STACK_TRACE_FREE;                 \
+  asan_delete(ptr, size, 0, &stack, FROM_NEW_BR)
+#define OPERATOR_DELETE_BODY_SIZE_ALIGN \
+  GET_STACK_TRACE_FREE;                 \
+  asan_delete(ptr, size, static_cast<uptr>(align), &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY \
   GET_STACK_TRACE_FREE;                       \
-  asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
+  asan_delete(ptr, size, static_cast<uptr>(align), &stack, FROM_NEW_BR)
 
 #if !SANITIZER_APPLE
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr) NOEXCEPT
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr) NOEXCEPT
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY_ARRAY; }
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
+void operator delete(void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+void operator delete[](void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, size_t size) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW); }
+void operator delete(void *ptr, size_t size) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, size_t size) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); }
+void operator delete[](void *ptr, size_t size) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); }
+void operator delete(void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); }
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_ALIGN_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); }
+void operator delete(void *ptr, std::align_val_t align,
+                     std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); }
+void operator delete[](void *ptr, std::align_val_t align,
+                       std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ALIGN_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); }
+void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); }
+void operator delete[](void *ptr, size_t size,
+                       std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY;
+}
 
 #else  // SANITIZER_APPLE
-INTERCEPTOR(void, _ZdlPv, void *ptr)
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
-INTERCEPTOR(void, _ZdaPv, void *ptr)
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
-INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
-INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+INTERCEPTOR(void, _ZdlPv, void *ptr) { OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdaPv, void *ptr) { OPERATOR_DELETE_BODY_ARRAY; }
+INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ARRAY;
+}
 #endif  // !SANITIZER_APPLE
diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index b3f6677a99cfb..b9ba250f5bcd7 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -390,6 +390,39 @@ void PrintAddressSpaceLayout() {
           kHighShadowBeg > kMidMemEnd);
 }
 
+// Apply most options specified either through the ASAN_OPTIONS
+// environment variable, or through the `__asan_default_options` user function.
+//
+// This function may be called multiple times, once per weak reference callback
+// on Windows, so it needs to be idempotent.
+//
+// Context:
+// For maximum compatibility on Windows, it is necessary for ASan options to be
+// configured/registered/applied inside this method (instead of in
+// ASanInitInternal, for example). That's because, on Windows, the user-provided
+// definition for `__asan_default_opts` may not be bound when `ASanInitInternal`
+// is invoked (it is bound later).
+//
+// To work around the late binding on windows, `ApplyOptions` will be called,
+// again, after binding to the user-provided `__asan_default_opts` function.
+// Therefore, any flags not configured here are not guaranteed to be
+// configurable through `__asan_default_opts` on Windows.
+//
+//
+// For more details on this issue, see:
+// https://github.com/llvm/llvm-project/issues/117925
+void ApplyFlags() {
+  SetCanPoisonMemory(flags()->poison_heap);
+  SetMallocContextSize(common_flags()->malloc_context_size);
+
+  __asan_option_detect_stack_use_after_return =
+      flags()->detect_stack_use_after_return;
+
+  AllocatorOptions allocator_options;
+  allocator_options.SetFrom(flags(), common_flags());
+  ApplyAllocatorOptions(allocator_options);
+}
+
 static bool AsanInitInternal() {
   if (LIKELY(AsanInited()))
     return true;
@@ -397,8 +430,9 @@ static bool AsanInitInternal() {
 
   CacheBinaryName();
 
-  // Initialize flags. This must be done early, because most of the
-  // initialization steps look at flags().
+  // Initialize flags. On Windows it also also register weak function callbacks.
+  // This must be done early, because most of the initialization steps look at
+  // flags().
   InitializeFlags();
 
   WaitForDebugger(flags()->sleep_before_init, "before init");
@@ -416,9 +450,6 @@ static bool AsanInitInternal() {
   AsanCheckDynamicRTPrereqs();
   AvoidCVE_2016_2143();
 
-  SetCanPoisonMemory(flags()->poison_heap);
-  SetMallocContextSize(common_flags()->malloc_context_size);
-
   InitializePlatformExceptionHandlers();
 
   InitializeHighMemEnd();
@@ -429,10 +460,6 @@ static bool AsanInitInternal() {
   SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
 
   __sanitizer_set_report_path(common_flags()->log_path);
-
-  __asan_option_detect_stack_use_after_return =
-      flags()->detect_stack_use_after_return;
-
   __sanitizer::InitializePlatformEarly();
 
   // Setup internal allocator callback.
@@ -472,6 +499,13 @@ static bool AsanInitInternal() {
   allocator_options.SetFrom(flags(), common_flags());
   InitializeAllocator(allocator_options);
 
+  // Apply ASan flags.
+  // NOTE: In order for options specified through `__asan_default_options` to be
+  // honored on Windows, it is necessary for those options to be configured
+  // inside the `ApplyOptions` method. See the function-level comment for
+  // `ApplyFlags` for more details.
+  ApplyFlags();
+
   if (SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL)
     MaybeStartBackgroudThread();
 
@@ -494,7 +528,6 @@ static bool AsanInitInternal() {
   AsanThread *main_thread = CreateMainThread();
   CHECK_EQ(0, main_thread->tid());
   force_interface_symbols();  // no-op.
-  SanitizerInitializeUnwinder();
 
   if (CAN_SANITIZE_LEAKS) {
     __lsan::InitCommonLsan();
diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
index ba1eab1a90f44..f485404758b54 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
@@ -80,8 +80,8 @@ static void *MallocStress(void *NumOfItrPtr) {
         case 2: size += 4096; break;
       }
       size_t alignment = 1 << (my_rand_r(&seed) % 10 + 1);
-      char *ptr = (char*)__asan::asan_memalign(alignment, size,
-                                               &stack2, __asan::FROM_MALLOC);
+      char *ptr = (char *)__asan::asan_memalign(alignment, size, &stack2,
+                                                __asan::FROM_MALLOC);
       EXPECT_EQ(size, __asan::asan_malloc_usable_size(ptr, 0, 0));
       vec.push_back(ptr);
       ptr[0] = 0;
diff --git a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp
index f0fd3726ef1b1..232eb0eb6da67 100644
--- a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp
@@ -22,23 +22,75 @@
 #if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
 
 // TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
-#  define OPERATOR_NEW_BODY(nothrow)         \
+#  define OPERATOR_NEW_BODY                  \
     GET_MALLOC_STACK_TRACE;                  \
     void *res = hwasan_malloc(size, &stack); \
-    if (!nothrow && UNLIKELY(!res))          \
+    if (UNLIKELY(!res))                      \
       ReportOutOfMemory(size, &stack);       \
     return res
-#  define OPERATOR_NEW_ALIGN_BODY(nothrow)                               \
+#  define OPERATOR_NEW_BODY_NOTHROW \
+    GET_MALLOC_STACK_TRACE;         \
+    return hwasan_malloc(size, &stack)
+#  define OPERATOR_NEW_BODY_ARRAY            \
+    GET_MALLOC_STACK_TRACE;                  \
+    void *res = hwasan_malloc(size, &stack); \
+    if (UNLIKELY(!res))                      \
+      ReportOutOfMemory(size, &stack);       \
+    return res
+#  define OPERATOR_NEW_BODY_ARRAY_NOTHROW \
+    GET_MALLOC_STACK_TRACE;               \
+    return hwasan_malloc(size, &stack)
+#  define OPERATOR_NEW_BODY_ALIGN                                        \
+    GET_MALLOC_STACK_TRACE;                                              \
+    void *res = hwasan_memalign(static_cast<uptr>(align), size, &stack); \
+    if (UNLIKELY(!res))                                                  \
+      ReportOutOfMemory(size, &stack);                                   \
+    return res
+#  define OPERATOR_NEW_BODY_ALIGN_NOTHROW \
+    GET_MALLOC_STACK_TRACE;               \
+    return hwasan_memalign(static_cast<uptr>(align), size, &stack)
+#  define OPERATOR_NEW_BODY_ALIGN_ARRAY                                  \
     GET_MALLOC_STACK_TRACE;                                              \
     void *res = hwasan_memalign(static_cast<uptr>(align), size, &stack); \
-    if (!nothrow && UNLIKELY(!res))                                      \
+    if (UNLIKELY(!res))                                                  \
       ReportOutOfMemory(size, &stack);                                   \
     return res
+#  define OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW \
+    GET_MALLOC_STACK_TRACE;                     \
+    return hwasan_memalign(static_cast<uptr>(align), size, &stack)
 
 #  define OPERATOR_DELETE_BODY \
     GET_MALLOC_STACK_TRACE;    \
     if (ptr)                   \
     hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_ARRAY \
+    GET_MALLOC_STACK_TRACE;          \
+    if (ptr)                         \
+    hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_ALIGN \
+    GET_MALLOC_STACK_TRACE;          \
+    if (ptr)                         \
+    hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_ALIGN_ARRAY \
+    GET_MALLOC_STACK_TRACE;                \
+    if (ptr)                               \
+    hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_SIZE \
+    GET_MALLOC_STACK_TRACE;         \
+    if (ptr)                        \
+    hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_SIZE_ARRAY \
+    GET_MALLOC_STACK_TRACE;               \
+    if (ptr)                              \
+    hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_SIZE_ALIGN \
+    GET_MALLOC_STACK_TRACE;               \
+    if (ptr)                              \
+    hwasan_free(ptr, &stack)
+#  define OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY \
+    GET_MALLOC_STACK_TRACE;                     \
+    if (ptr)                                    \
+    hwasan_free(ptr, &stack)
 
 #elif defined(__ANDROID__)
 
@@ -46,8 +98,14 @@
 // since we previously released a runtime that intercepted these functions,
 // removing the interceptors would break ABI. Therefore we simply forward to
 // malloc and free.
-#  define OPERATOR_NEW_BODY(nothrow) return malloc(size)
+#  define OPERATOR_NEW_BODY return malloc(size)
+#  define OPERATOR_NEW_BODY_NOTHROW return malloc(size)
+#  define OPERATOR_NEW_BODY_ARRAY return malloc(size)
+#  define OPERATOR_NEW_BODY_ARRAY_NOTHROW return malloc(size)
 #  define OPERATOR_DELETE_BODY free(ptr)
+#  define OPERATOR_DELETE_BODY_ARRAY free(ptr)
+#  define OPERATOR_DELETE_BODY_SIZE free(ptr)
+#  define OPERATOR_DELETE_BODY_SIZE_ARRAY free(ptr)
 
 #endif
 
@@ -61,19 +119,19 @@ struct nothrow_t {};
 }  // namespace std
 
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size) {
-  OPERATOR_NEW_BODY(false /*nothrow*/);
+  OPERATOR_NEW_BODY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
     size_t size) {
-  OPERATOR_NEW_BODY(false /*nothrow*/);
+  OPERATOR_NEW_BODY_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
     size_t size, std::nothrow_t const &) {
-  OPERATOR_NEW_BODY(true /*nothrow*/);
+  OPERATOR_NEW_BODY_NOTHROW;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
     size_t size, std::nothrow_t const &) {
-  OPERATOR_NEW_BODY(true /*nothrow*/);
+  OPERATOR_NEW_BODY_ARRAY_NOTHROW;
 }
 
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
@@ -82,7 +140,7 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
     void *ptr, std::nothrow_t const &) {
@@ -90,20 +148,20 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr, std::nothrow_t const &) {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
     void *ptr, size_t) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_SIZE;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr, size_t) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_SIZE_ARRAY;
 }
 
 #endif  // OPERATOR_NEW_BODY
 
-#ifdef OPERATOR_NEW_ALIGN_BODY
+#ifdef OPERATOR_NEW_BODY_ALIGN
 
 namespace std {
 enum class align_val_t : size_t {};
@@ -111,52 +169,52 @@ enum class align_val_t : size_t {};
 
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
     size_t size, std::align_val_t align) {
-  OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
+  OPERATOR_NEW_BODY_ALIGN;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
     size_t size, std::align_val_t align) {
-  OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
+  OPERATOR_NEW_BODY_ALIGN_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
     size_t size, std::align_val_t align, std::nothrow_t const &) {
-  OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
+  OPERATOR_NEW_BODY_ALIGN_NOTHROW;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
     size_t size, std::align_val_t align, std::nothrow_t const &) {
-  OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
+  OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW;
 }
 
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
     void *ptr, std::align_val_t align) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_ALIGN;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr, std::align_val_t) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_ALIGN_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
     void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_ALIGN;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_ALIGN_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
     void *ptr, size_t, std::align_val_t) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_SIZE_ALIGN;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr, size_t, std::align_val_t) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
     void *ptr, size_t, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_SIZE_ALIGN;
 }
 INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
     void *ptr, size_t, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
-  OPERATOR_DELETE_BODY;
+  OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY;
 }
 
-#endif  // OPERATOR_NEW_ALIGN_BODY
+#endif  // OPERATOR_NEW_BODY_ALIGN
diff --git a/compiler-rt/lib/memprof/memprof_rtl.cpp b/compiler-rt/lib/memprof/memprof_rtl.cpp
index ef8884a7e56f4..4fd4b5210a7ec 100644
--- a/compiler-rt/lib/memprof/memprof_rtl.cpp
+++ b/compiler-rt/lib/memprof/memprof_rtl.cpp
@@ -219,7 +219,6 @@ static void MemprofInitInternal() {
   MemprofThread *main_thread = CreateMainThread();
   CHECK_EQ(0, main_thread->tid());
   force_interface_symbols(); // no-op.
-  SanitizerInitializeUnwinder();
 
   Symbolizer::LateInitialize();
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 3f52cfcaeecaa..120c2861c1ac0 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -925,12 +925,6 @@ class ListOfModules {
 // Callback type for iterating over a set of memory ranges.
 typedef void (*RangeIteratorCallback)(uptr begin, uptr end, void *arg);
 
-enum AndroidApiLevel {
-  ANDROID_NOT_ANDROID = 0,
-  ANDROID_LOLLIPOP_MR1 = 22,
-  ANDROID_POST_LOLLIPOP = 23
-};
-
 void WriteToSyslog(const char *buffer);
 
 #if defined(SANITIZER_WINDOWS) && defined(_MSC_VER) && !defined(__clang__)
@@ -963,19 +957,8 @@ inline void AndroidLogInit() {}
 inline void SetAbortMessage(const char *) {}
 #endif
 
-#if SANITIZER_ANDROID
-void SanitizerInitializeUnwinder();
-AndroidApiLevel AndroidGetApiLevel();
-#else
-inline void AndroidLogWrite(const char *buffer_unused) {}
-inline void SanitizerInitializeUnwinder() {}
-inline AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; }
-#endif
-
 inline uptr GetPthreadDestructorIterations() {
-#if SANITIZER_ANDROID
-  return (AndroidGetApiLevel() == ANDROID_LOLLIPOP_MR1) ? 8 : 4;
-#elif SANITIZER_POSIX
+#if SANITIZER_POSIX
   return 4;
 #else
 // Unused on Windows.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 4446823c47d22..16caf699a4c24 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -1901,54 +1901,6 @@ int internal_uname(struct utsname *buf) {
 }
 #  endif
 
-#  if SANITIZER_ANDROID
-static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
-                                   void *data) {
-  // Any name starting with "lib" indicates a bug in L where library base names
-  // are returned instead of paths.
-  if (info->dlpi_name && info->dlpi_name[0] == 'l' &&
-      info->dlpi_name[1] == 'i' && info->dlpi_name[2] == 'b') {
-    *(bool *)data = true;
-    return 1;
-  }
-  return 0;
-}
-
-static atomic_uint32_t android_api_level;
-
-static AndroidApiLevel AndroidDetectApiLevelStatic() {
-#    if __ANDROID_API__ <= 22
-  return ANDROID_LOLLIPOP_MR1;
-#    else
-  return ANDROID_POST_LOLLIPOP;
-#    endif
-}
-
-static AndroidApiLevel AndroidDetectApiLevel() {
-  bool base_name_seen = false;
-  dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);
-  if (base_name_seen)
-    return ANDROID_LOLLIPOP_MR1;  // L MR1
-  return ANDROID_POST_LOLLIPOP;   // post-L
-  // Plain L (API level 21) is completely broken wrt ASan and not very
-  // interesting to detect.
-}
-
-extern "C" __attribute__((weak)) void *_DYNAMIC;
-
-AndroidApiLevel AndroidGetApiLevel() {
-  AndroidApiLevel level =
-      (AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed);
-  if (level)
-    return level;
-  level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic()
-                               : AndroidDetectApiLevel();
-  atomic_store(&android_api_level, level, memory_order_relaxed);
-  return level;
-}
-
-#  endif
-
 static HandleSignalMode GetHandleSignalModeImpl(int signum) {
   switch (signum) {
     case SIGABRT:
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 3bf9f8b316d36..f5cb85bc1bf30 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -782,42 +782,13 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
   return 0;
 }
 
-static bool requiresProcmaps() {
-#  if SANITIZER_ANDROID && __ANDROID_API__ <= 22
-  // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
-  // The runtime check allows the same library to work with
-  // both K and L (and future) Android releases.
-  return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
-#  else
-  return false;
-#  endif
-}
-
-static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
-  MemoryMappingLayout memory_mapping(/*cache_enabled*/ true);
-  memory_mapping.DumpListOfModules(modules);
-}
-
 void ListOfModules::init() {
   clearOrInit();
-  if (requiresProcmaps()) {
-    procmapsInit(&modules_);
-  } else {
-    DlIteratePhdrData data = {&modules_, true};
-    dl_iterate_phdr(dl_iterate_phdr_cb, &data);
-  }
+  DlIteratePhdrData data = {&modules_, true};
+  dl_iterate_phdr(dl_iterate_phdr_cb, &data);
 }
 
-// When a custom loader is used, dl_iterate_phdr may not contain the full
-// list of modules. Allow callers to fall back to using procmaps.
-void ListOfModules::fallbackInit() {
-  if (!requiresProcmaps()) {
-    clearOrInit();
-    procmapsInit(&modules_);
-  } else {
-    clear();
-  }
-}
+void ListOfModules::fallbackInit() { clear(); }
 
 // getrusage does not give us the current RSS, only the max RSS.
 // Still, this is better than nothing if /proc/self/statm is not available
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
index 3f6901a2e7f1b..4f1538eeb9c55 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
@@ -91,38 +91,6 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
 
 }  // namespace
 
-#if SANITIZER_ANDROID
-void SanitizerInitializeUnwinder() {
-  if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return;
-
-  // Pre-lollipop Android can not unwind through signal handler frames with
-  // libgcc unwinder, but it has a libcorkscrew.so library with the necessary
-  // workarounds.
-  void *p = dlopen("libcorkscrew.so", RTLD_LAZY);
-  if (!p) {
-    VReport(1,
-            "Failed to open libcorkscrew.so. You may see broken stack traces "
-            "in SEGV reports.");
-    return;
-  }
-  acquire_my_map_info_list =
-      (acquire_my_map_info_list_func)(uptr)dlsym(p, "acquire_my_map_info_list");
-  release_my_map_info_list =
-      (release_my_map_info_list_func)(uptr)dlsym(p, "release_my_map_info_list");
-  unwind_backtrace_signal_arch = (unwind_backtrace_signal_arch_func)(uptr)dlsym(
-      p, "unwind_backtrace_signal_arch");
-  if (!acquire_my_map_info_list || !release_my_map_info_list ||
-      !unwind_backtrace_signal_arch) {
-    VReport(1,
-            "Failed to find one of the required symbols in libcorkscrew.so. "
-            "You may see broken stack traces in SEGV reports.");
-    acquire_my_map_info_list = 0;
-    unwind_backtrace_signal_arch = 0;
-    release_my_map_info_list = 0;
-  }
-}
-#endif
-
 void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
   CHECK_GE(max_depth, 2);
   size = 0;
diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
index fe49d944d3a2f..4d435976d3a10 100755
--- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
+++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
@@ -174,6 +174,7 @@ $LINK $LIBCXX_ARCHIVE_DIR/libc++.a \
       $LLVM_BUILD/lib/libLLVMObject.a \
       $LLVM_BUILD/lib/libLLVMBinaryFormat.a \
       $LLVM_BUILD/lib/libLLVMDebugInfoDWARF.a \
+      $LLVM_BUILD/lib/libLLVMDebugInfoDWARFLowLevel.a \
       $LLVM_BUILD/lib/libLLVMDebugInfoGSYM.a \
       $LLVM_BUILD/lib/libLLVMSupport.a \
       $LLVM_BUILD/lib/libLLVMDebugInfoPDB.a \
diff --git a/compiler-rt/lib/scudo/standalone/allocator_common.h b/compiler-rt/lib/scudo/standalone/allocator_common.h
index 0169601f9c625..cfa6950d4947b 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_common.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_common.h
@@ -14,27 +14,26 @@
 
 namespace scudo {
 
-template <class SizeClassAllocator> struct TransferBatch {
+template <class SizeClassAllocator> struct Batch {
   typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
   typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
 
-  static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
   void setFromArray(CompactPtrT *Array, u16 N) {
-    DCHECK_LE(N, MaxNumCached);
+    DCHECK_LE(N, SizeClassAllocator::MaxNumBlocksInBatch);
     Count = N;
-    memcpy(Batch, Array, sizeof(Batch[0]) * Count);
+    memcpy(Blocks, Array, sizeof(Blocks[0]) * Count);
   }
   void appendFromArray(CompactPtrT *Array, u16 N) {
-    DCHECK_LE(N, MaxNumCached - Count);
-    memcpy(Batch + Count, Array, sizeof(Batch[0]) * N);
+    DCHECK_LE(N, SizeClassAllocator::MaxNumBlocksInBatch - Count);
+    memcpy(Blocks + Count, Array, sizeof(Blocks[0]) * N);
     // u16 will be promoted to int by arithmetic type conversion.
     Count = static_cast<u16>(Count + N);
   }
-  void appendFromTransferBatch(TransferBatch *B, u16 N) {
-    DCHECK_LE(N, MaxNumCached - Count);
+  void appendFromBatch(Batch *B, u16 N) {
+    DCHECK_LE(N, SizeClassAllocator::MaxNumBlocksInBatch - Count);
     DCHECK_GE(B->Count, N);
     // Append from the back of `B`.
-    memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N);
+    memcpy(Blocks + Count, B->Blocks + (B->Count - N), sizeof(Blocks[0]) * N);
     // u16 will be promoted to int by arithmetic type conversion.
     Count = static_cast<u16>(Count + N);
     B->Count = static_cast<u16>(B->Count - N);
@@ -42,30 +41,30 @@ template <class SizeClassAllocator> struct TransferBatch {
   void clear() { Count = 0; }
   bool empty() { return Count == 0; }
   void add(CompactPtrT P) {
-    DCHECK_LT(Count, MaxNumCached);
-    Batch[Count++] = P;
+    DCHECK_LT(Count, SizeClassAllocator::MaxNumBlocksInBatch);
+    Blocks[Count++] = P;
   }
   void moveToArray(CompactPtrT *Array) {
-    memcpy(Array, Batch, sizeof(Batch[0]) * Count);
+    memcpy(Array, Blocks, sizeof(Blocks[0]) * Count);
     clear();
   }
 
   void moveNToArray(CompactPtrT *Array, u16 N) {
     DCHECK_LE(N, Count);
-    memcpy(Array, Batch + Count - N, sizeof(Batch[0]) * N);
+    memcpy(Array, Blocks + Count - N, sizeof(Blocks[0]) * N);
     Count = static_cast<u16>(Count - N);
   }
   u16 getCount() const { return Count; }
   bool isEmpty() const { return Count == 0U; }
   CompactPtrT get(u16 I) const {
     DCHECK_LE(I, Count);
-    return Batch[I];
+    return Blocks[I];
   }
-  TransferBatch *Next;
+  Batch *Next;
 
 private:
-  CompactPtrT Batch[MaxNumCached];
   u16 Count;
+  CompactPtrT Blocks[];
 };
 
 // A BatchGroup is used to collect blocks. Each group has a group id to
@@ -78,9 +77,12 @@ template <class SizeClassAllocator> struct BatchGroup {
   // This is used to track how many bytes are not in-use since last time we
   // tried to release pages.
   uptr BytesInBGAtLastCheckpoint;
-  // Blocks are managed by TransferBatch in a list.
-  SinglyLinkedList<TransferBatch<SizeClassAllocator>> Batches;
+  // Blocks are managed by Batch in a list.
+  SinglyLinkedList<Batch<SizeClassAllocator>> Batches;
   // Cache value of SizeClassAllocatorLocalCache::getMaxCached()
+  // TODO(chiahungduan): Except BatchClass, every Batch stores the same number
+  // of blocks. As long as we make BatchClass follow this constraint, this
+  // field can be removed.
   u16 MaxCachedPerBatch;
 };
 
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 0932d47ad4563..756adcadd0c7b 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -34,7 +34,7 @@ namespace scudo {
 // predictable address pattern (the predictability increases with the block
 // size).
 //
-// Regions for size class 0 are special and used to hold TransferBatches, which
+// Regions for size class 0 are special and used to hold Batches, which
 // allow to transfer arrays of pointers from the global size class freelist to
 // the thread specific freelist for said class, and back.
 //
@@ -56,12 +56,21 @@ template <typename Config> class SizeClassAllocator32 {
       typename Conditional<Config::getEnableBlockCache(),
                            SizeClassAllocatorLocalCache<ThisT>,
                            SizeClassAllocatorNoCache<ThisT>>::type;
-  typedef TransferBatch<ThisT> TransferBatchT;
+  typedef Batch<ThisT> BatchT;
   typedef BatchGroup<ThisT> BatchGroupT;
+  static const u16 MaxNumBlocksInBatch = SizeClassMap::MaxNumCachedHint;
+
+  static constexpr uptr getSizeOfBatchClass() {
+    const uptr HeaderSize = sizeof(BatchT);
+    return HeaderSize + sizeof(CompactPtrT) * MaxNumBlocksInBatch;
+  }
+
+  static_assert(sizeof(BatchGroupT) <= getSizeOfBatchClass(),
+                "BatchGroupT also uses BatchClass");
 
   static uptr getSizeByClassId(uptr ClassId) {
     return (ClassId == SizeClassMap::BatchClassId)
-               ? Max(sizeof(BatchGroupT), sizeof(TransferBatchT))
+               ? getSizeOfBatchClass()
                : SizeClassMap::getSizeByClassId(ClassId);
   }
 
@@ -124,7 +133,7 @@ template <typename Config> class SizeClassAllocator32 {
 
   // When all blocks are freed, it has to be the same size as `AllocatedUser`.
   void verifyAllBlocksAreReleasedTestOnly() {
-    // `BatchGroup` and `TransferBatch` also use the blocks from BatchClass.
+    // `BatchGroup` and `Batch` also use the blocks from BatchClass.
     uptr BatchClassUsedInFreeLists = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       // We have to count BatchClassUsedInFreeLists in other regions first.
@@ -134,7 +143,7 @@ template <typename Config> class SizeClassAllocator32 {
       ScopedLock L1(Sci->Mutex);
       uptr TotalBlocks = 0;
       for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) {
-        // `BG::Batches` are `TransferBatches`. +1 for `BatchGroup`.
+        // `BG::Batches` are `Batches`. +1 for `BatchGroup`.
         BatchClassUsedInFreeLists += BG.Batches.size() + 1;
         for (const auto &It : BG.Batches)
           TotalBlocks += It.getCount();
@@ -153,7 +162,7 @@ template <typename Config> class SizeClassAllocator32 {
         for (const auto &It : BG.Batches)
           TotalBlocks += It.getCount();
       } else {
-        // `BatchGroup` with empty freelist doesn't have `TransferBatch` record
+        // `BatchGroup` with empty freelist doesn't have `Batch` record
         // itself.
         ++TotalBlocks;
       }
@@ -487,13 +496,13 @@ template <typename Config> class SizeClassAllocator32 {
       REQUIRES(Sci->Mutex) {
     DCHECK_EQ(Sci, getSizeClassInfo(SizeClassMap::BatchClassId));
 
-    // Free blocks are recorded by TransferBatch in freelist for all
-    // size-classes. In addition, TransferBatch is allocated from BatchClassId.
+    // Free blocks are recorded by Batch in freelist for all
+    // size-classes. In addition, Batch is allocated from BatchClassId.
     // In order not to use additional block to record the free blocks in
-    // BatchClassId, they are self-contained. I.e., A TransferBatch records the
+    // BatchClassId, they are self-contained. I.e., A Batch records the
     // block address of itself. See the figure below:
     //
-    // TransferBatch at 0xABCD
+    // Batch at 0xABCD
     // +----------------------------+
     // | Free blocks' addr          |
     // | +------+------+------+     |
@@ -501,25 +510,25 @@ template <typename Config> class SizeClassAllocator32 {
     // | +------+------+------+     |
     // +----------------------------+
     //
-    // When we allocate all the free blocks in the TransferBatch, the block used
-    // by TransferBatch is also free for use. We don't need to recycle the
-    // TransferBatch. Note that the correctness is maintained by the invariant,
+    // When we allocate all the free blocks in the Batch, the block used
+    // by Batch is also free for use. We don't need to recycle the
+    // Batch. Note that the correctness is maintained by the invariant,
     //
-    //   Each popBlocks() request returns the entire TransferBatch. Returning
-    //   part of the blocks in a TransferBatch is invalid.
+    //   Each popBlocks() request returns the entire Batch. Returning
+    //   part of the blocks in a Batch is invalid.
     //
-    // This ensures that TransferBatch won't leak the address itself while it's
+    // This ensures that Batch won't leak the address itself while it's
     // still holding other valid data.
     //
     // Besides, BatchGroup is also allocated from BatchClassId and has its
-    // address recorded in the TransferBatch too. To maintain the correctness,
+    // address recorded in the Batch too. To maintain the correctness,
     //
-    //   The address of BatchGroup is always recorded in the last TransferBatch
+    //   The address of BatchGroup is always recorded in the last Batch
     //   in the freelist (also imply that the freelist should only be
-    //   updated with push_front). Once the last TransferBatch is popped,
+    //   updated with push_front). Once the last Batch is popped,
     //   the block used by BatchGroup is also free for use.
     //
-    // With this approach, the blocks used by BatchGroup and TransferBatch are
+    // With this approach, the blocks used by BatchGroup and Batch are
     // reusable and don't need additional space for them.
 
     Sci->FreeListInfo.PushedBlocks += Size;
@@ -548,12 +557,12 @@ template <typename Config> class SizeClassAllocator32 {
     //   1. just allocated a new `BatchGroup`.
     //   2. Only 1 block is pushed when the freelist is empty.
     if (BG->Batches.empty()) {
-      // Construct the `TransferBatch` on the last element.
-      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
+      // Construct the `Batch` on the last element.
+      BatchT *TB = reinterpret_cast<BatchT *>(
           decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1]));
       TB->clear();
-      // As mentioned above, addresses of `TransferBatch` and `BatchGroup` are
-      // recorded in the TransferBatch.
+      // As mentioned above, addresses of `Batch` and `BatchGroup` are
+      // recorded in the Batch.
       TB->add(Array[Size - 1]);
       TB->add(
           compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(BG)));
@@ -561,14 +570,14 @@ template <typename Config> class SizeClassAllocator32 {
       BG->Batches.push_front(TB);
     }
 
-    TransferBatchT *CurBatch = BG->Batches.front();
+    BatchT *CurBatch = BG->Batches.front();
     DCHECK_NE(CurBatch, nullptr);
 
     for (u32 I = 0; I < Size;) {
       u16 UnusedSlots =
           static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
       if (UnusedSlots == 0) {
-        CurBatch = reinterpret_cast<TransferBatchT *>(
+        CurBatch = reinterpret_cast<BatchT *>(
             decompactPtr(SizeClassMap::BatchClassId, Array[I]));
         CurBatch->clear();
         // Self-contained
@@ -596,7 +605,7 @@ template <typename Config> class SizeClassAllocator32 {
   //                            TB
   //
   // Each BlockGroup(BG) will associate with unique group id and the free blocks
-  // are managed by a list of TransferBatch(TB). To reduce the time of inserting
+  // are managed by a list of Batch(TB). To reduce the time of inserting
   // blocks, BGs are sorted and the input `Array` are supposed to be sorted so
   // that we can get better performance of maintaining sorted property.
   // Use `SameGroup=true` to indicate that all blocks in the array are from the
@@ -613,21 +622,21 @@ template <typename Config> class SizeClassAllocator32 {
       BatchGroupT *BG = reinterpret_cast<BatchGroupT *>(
           SizeClassAllocator->getBatchClassBlock());
       BG->Batches.clear();
-      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
-          SizeClassAllocator->getBatchClassBlock());
+      BatchT *TB =
+          reinterpret_cast<BatchT *>(SizeClassAllocator->getBatchClassBlock());
       TB->clear();
 
       BG->CompactPtrGroupBase = CompactPtrGroupBase;
       BG->Batches.push_front(TB);
       BG->BytesInBGAtLastCheckpoint = 0;
-      BG->MaxCachedPerBatch = TransferBatchT::MaxNumCached;
+      BG->MaxCachedPerBatch = MaxNumBlocksInBatch;
 
       return BG;
     };
 
     auto InsertBlocks = [&](BatchGroupT *BG, CompactPtrT *Array, u32 Size) {
-      SinglyLinkedList<TransferBatchT> &Batches = BG->Batches;
-      TransferBatchT *CurBatch = Batches.front();
+      SinglyLinkedList<BatchT> &Batches = BG->Batches;
+      BatchT *CurBatch = Batches.front();
       DCHECK_NE(CurBatch, nullptr);
 
       for (u32 I = 0; I < Size;) {
@@ -635,7 +644,7 @@ template <typename Config> class SizeClassAllocator32 {
         u16 UnusedSlots =
             static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
         if (UnusedSlots == 0) {
-          CurBatch = reinterpret_cast<TransferBatchT *>(
+          CurBatch = reinterpret_cast<BatchT *>(
               SizeClassAllocator->getBatchClassBlock());
           CurBatch->clear();
           Batches.push_front(CurBatch);
@@ -716,7 +725,7 @@ template <typename Config> class SizeClassAllocator32 {
     if (Sci->FreeListInfo.BlockList.empty())
       return 0U;
 
-    SinglyLinkedList<TransferBatchT> &Batches =
+    SinglyLinkedList<BatchT> &Batches =
         Sci->FreeListInfo.BlockList.front()->Batches;
 
     if (Batches.empty()) {
@@ -725,8 +734,8 @@ template <typename Config> class SizeClassAllocator32 {
       Sci->FreeListInfo.BlockList.pop_front();
 
       // Block used by `BatchGroup` is from BatchClassId. Turn the block into
-      // `TransferBatch` with single block.
-      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(BG);
+      // `Batch` with single block.
+      BatchT *TB = reinterpret_cast<BatchT *>(BG);
       ToArray[0] =
           compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(TB));
       Sci->FreeListInfo.PoppedBlocks += 1;
@@ -734,18 +743,18 @@ template <typename Config> class SizeClassAllocator32 {
     }
 
     // So far, instead of always filling the blocks to `MaxBlockCount`, we only
-    // examine single `TransferBatch` to minimize the time spent on the primary
-    // allocator. Besides, the sizes of `TransferBatch` and
+    // examine single `Batch` to minimize the time spent on the primary
+    // allocator. Besides, the sizes of `Batch` and
     // `SizeClassAllocatorT::getMaxCached()` may also impact the time spent on
     // accessing the primary allocator.
     // TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
-    // blocks and/or adjust the size of `TransferBatch` according to
+    // blocks and/or adjust the size of `Batch` according to
     // `SizeClassAllocatorT::getMaxCached()`.
-    TransferBatchT *B = Batches.front();
+    BatchT *B = Batches.front();
     DCHECK_NE(B, nullptr);
     DCHECK_GT(B->getCount(), 0U);
 
-    // BachClassId should always take all blocks in the TransferBatch. Read the
+    // BachClassId should always take all blocks in the Batch. Read the
     // comment in `pushBatchClassBlocks()` for more details.
     const u16 PopCount = ClassId == SizeClassMap::BatchClassId
                              ? B->getCount()
@@ -756,7 +765,7 @@ template <typename Config> class SizeClassAllocator32 {
     // done without holding `Mutex`.
     if (B->empty()) {
       Batches.pop_front();
-      // `TransferBatch` of BatchClassId is self-contained, no need to
+      // `Batch` of BatchClassId is self-contained, no need to
       // deallocate. Read the comment in `pushBatchClassBlocks()` for more
       // details.
       if (ClassId != SizeClassMap::BatchClassId)
@@ -769,7 +778,7 @@ template <typename Config> class SizeClassAllocator32 {
         // We don't keep BatchGroup with zero blocks to avoid empty-checking
         // while allocating. Note that block used for constructing BatchGroup is
         // recorded as free blocks in the last element of BatchGroup::Batches.
-        // Which means, once we pop the last TransferBatch, the block is
+        // Which means, once we pop the last Batch, the block is
         // implicitly deallocated.
         if (ClassId != SizeClassMap::BatchClassId)
           SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, BG);
@@ -816,8 +825,7 @@ template <typename Config> class SizeClassAllocator32 {
             static_cast<u32>((RegionSize - Offset) / Size));
     DCHECK_GT(NumberOfBlocks, 0U);
 
-    constexpr u32 ShuffleArraySize =
-        MaxNumBatches * TransferBatchT::MaxNumCached;
+    constexpr u32 ShuffleArraySize = MaxNumBatches * MaxNumBlocksInBatch;
     // Fill the transfer batches and put them in the size-class freelist. We
     // need to randomize the blocks for security purposes, so we first fill a
     // local array that we then shuffle before populating the batches.
@@ -1102,7 +1110,7 @@ template <typename Config> class SizeClassAllocator32 {
       if (AllocatedGroupSize == 0)
         continue;
 
-      // TransferBatches are pushed in front of BG.Batches. The first one may
+      // Batches are pushed in front of BG.Batches. The first one may
       // not have all caches used.
       const uptr NumBlocks = (BG.Batches.size() - 1) * BG.MaxCachedPerBatch +
                              BG.Batches.front()->getCount();
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 25ee999199114..e70dbe91d7acd 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -38,7 +38,7 @@ namespace scudo {
 // they belong to. The Blocks created are shuffled to prevent predictable
 // address patterns (the predictability increases with the size of the Blocks).
 //
-// The 1st Region (for size class 0) holds the TransferBatches. This is a
+// The 1st Region (for size class 0) holds the Batches. This is a
 // structure used to transfer arrays of available pointers from the class size
 // freelist to the thread specific freelist, and back.
 //
@@ -57,19 +57,28 @@ template <typename Config> class SizeClassAllocator64 {
                 "Group size shouldn't be greater than the region size");
   static const uptr GroupScale = GroupSizeLog - CompactPtrScale;
   typedef SizeClassAllocator64<Config> ThisT;
-  typedef TransferBatch<ThisT> TransferBatchT;
+  typedef Batch<ThisT> BatchT;
   typedef BatchGroup<ThisT> BatchGroupT;
   using SizeClassAllocatorT =
       typename Conditional<Config::getEnableBlockCache(),
                            SizeClassAllocatorLocalCache<ThisT>,
                            SizeClassAllocatorNoCache<ThisT>>::type;
+  static const u16 MaxNumBlocksInBatch = SizeClassMap::MaxNumCachedHint;
+
+  static constexpr uptr getSizeOfBatchClass() {
+    const uptr HeaderSize = sizeof(BatchT);
+    return roundUp(HeaderSize + sizeof(CompactPtrT) * MaxNumBlocksInBatch,
+                   1 << CompactPtrScale);
+  }
+
+  static_assert(sizeof(BatchGroupT) <= getSizeOfBatchClass(),
+                "BatchGroupT also uses BatchClass");
 
   // BachClass is used to store internal metadata so it needs to be at least as
   // large as the largest data structure.
   static uptr getSizeByClassId(uptr ClassId) {
     return (ClassId == SizeClassMap::BatchClassId)
-               ? roundUp(Max(sizeof(TransferBatchT), sizeof(BatchGroupT)),
-                         1U << CompactPtrScale)
+               ? getSizeOfBatchClass()
                : SizeClassMap::getSizeByClassId(ClassId);
   }
 
@@ -171,7 +180,7 @@ template <typename Config> class SizeClassAllocator64 {
 
   // When all blocks are freed, it has to be the same size as `AllocatedUser`.
   void verifyAllBlocksAreReleasedTestOnly() {
-    // `BatchGroup` and `TransferBatch` also use the blocks from BatchClass.
+    // `BatchGroup` and `Batch` also use the blocks from BatchClass.
     uptr BatchClassUsedInFreeLists = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       // We have to count BatchClassUsedInFreeLists in other regions first.
@@ -183,7 +192,7 @@ template <typename Config> class SizeClassAllocator64 {
       const uptr BlockSize = getSizeByClassId(I);
       uptr TotalBlocks = 0;
       for (BatchGroupT &BG : Region->FreeListInfo.BlockList) {
-        // `BG::Batches` are `TransferBatches`. +1 for `BatchGroup`.
+        // `BG::Batches` are `Batches`. +1 for `BatchGroup`.
         BatchClassUsedInFreeLists += BG.Batches.size() + 1;
         for (const auto &It : BG.Batches)
           TotalBlocks += It.getCount();
@@ -204,7 +213,7 @@ template <typename Config> class SizeClassAllocator64 {
         for (const auto &It : BG.Batches)
           TotalBlocks += It.getCount();
       } else {
-        // `BatchGroup` with empty freelist doesn't have `TransferBatch` record
+        // `BatchGroup` with empty freelist doesn't have `Batch` record
         // itself.
         ++TotalBlocks;
       }
@@ -656,13 +665,13 @@ template <typename Config> class SizeClassAllocator64 {
       REQUIRES(Region->FLLock) {
     DCHECK_EQ(Region, getRegionInfo(SizeClassMap::BatchClassId));
 
-    // Free blocks are recorded by TransferBatch in freelist for all
-    // size-classes. In addition, TransferBatch is allocated from BatchClassId.
+    // Free blocks are recorded by Batch in freelist for all
+    // size-classes. In addition, Batch is allocated from BatchClassId.
     // In order not to use additional block to record the free blocks in
-    // BatchClassId, they are self-contained. I.e., A TransferBatch records the
+    // BatchClassId, they are self-contained. I.e., A Batch records the
     // block address of itself. See the figure below:
     //
-    // TransferBatch at 0xABCD
+    // Batch at 0xABCD
     // +----------------------------+
     // | Free blocks' addr          |
     // | +------+------+------+     |
@@ -670,25 +679,25 @@ template <typename Config> class SizeClassAllocator64 {
     // | +------+------+------+     |
     // +----------------------------+
     //
-    // When we allocate all the free blocks in the TransferBatch, the block used
-    // by TransferBatch is also free for use. We don't need to recycle the
-    // TransferBatch. Note that the correctness is maintained by the invariant,
+    // When we allocate all the free blocks in the Batch, the block used
+    // by Batch is also free for use. We don't need to recycle the
+    // Batch. Note that the correctness is maintained by the invariant,
     //
-    //   Each popBlocks() request returns the entire TransferBatch. Returning
-    //   part of the blocks in a TransferBatch is invalid.
+    //   Each popBlocks() request returns the entire Batch. Returning
+    //   part of the blocks in a Batch is invalid.
     //
-    // This ensures that TransferBatch won't leak the address itself while it's
+    // This ensures that Batch won't leak the address itself while it's
     // still holding other valid data.
     //
     // Besides, BatchGroup is also allocated from BatchClassId and has its
-    // address recorded in the TransferBatch too. To maintain the correctness,
+    // address recorded in the Batch too. To maintain the correctness,
     //
-    //   The address of BatchGroup is always recorded in the last TransferBatch
+    //   The address of BatchGroup is always recorded in the last Batch
     //   in the freelist (also imply that the freelist should only be
-    //   updated with push_front). Once the last TransferBatch is popped,
+    //   updated with push_front). Once the last Batch is popped,
     //   the block used by BatchGroup is also free for use.
     //
-    // With this approach, the blocks used by BatchGroup and TransferBatch are
+    // With this approach, the blocks used by BatchGroup and Batch are
     // reusable and don't need additional space for them.
 
     Region->FreeListInfo.PushedBlocks += Size;
@@ -717,12 +726,12 @@ template <typename Config> class SizeClassAllocator64 {
     //   1. just allocated a new `BatchGroup`.
     //   2. Only 1 block is pushed when the freelist is empty.
     if (BG->Batches.empty()) {
-      // Construct the `TransferBatch` on the last element.
-      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
+      // Construct the `Batch` on the last element.
+      BatchT *TB = reinterpret_cast<BatchT *>(
           decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1]));
       TB->clear();
-      // As mentioned above, addresses of `TransferBatch` and `BatchGroup` are
-      // recorded in the TransferBatch.
+      // As mentioned above, addresses of `Batch` and `BatchGroup` are
+      // recorded in the Batch.
       TB->add(Array[Size - 1]);
       TB->add(
           compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(BG)));
@@ -730,14 +739,14 @@ template <typename Config> class SizeClassAllocator64 {
       BG->Batches.push_front(TB);
     }
 
-    TransferBatchT *CurBatch = BG->Batches.front();
+    BatchT *CurBatch = BG->Batches.front();
     DCHECK_NE(CurBatch, nullptr);
 
     for (u32 I = 0; I < Size;) {
       u16 UnusedSlots =
           static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
       if (UnusedSlots == 0) {
-        CurBatch = reinterpret_cast<TransferBatchT *>(
+        CurBatch = reinterpret_cast<BatchT *>(
             decompactPtr(SizeClassMap::BatchClassId, Array[I]));
         CurBatch->clear();
         // Self-contained
@@ -766,7 +775,7 @@ template <typename Config> class SizeClassAllocator64 {
   //                            TB
   //
   // Each BlockGroup(BG) will associate with unique group id and the free blocks
-  // are managed by a list of TransferBatch(TB). To reduce the time of inserting
+  // are managed by a list of Batch(TB). To reduce the time of inserting
   // blocks, BGs are sorted and the input `Array` are supposed to be sorted so
   // that we can get better performance of maintaining sorted property.
   // Use `SameGroup=true` to indicate that all blocks in the array are from the
@@ -781,21 +790,21 @@ template <typename Config> class SizeClassAllocator64 {
       BatchGroupT *BG = reinterpret_cast<BatchGroupT *>(
           SizeClassAllocator->getBatchClassBlock());
       BG->Batches.clear();
-      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(
-          SizeClassAllocator->getBatchClassBlock());
+      BatchT *TB =
+          reinterpret_cast<BatchT *>(SizeClassAllocator->getBatchClassBlock());
       TB->clear();
 
       BG->CompactPtrGroupBase = CompactPtrGroupBase;
       BG->Batches.push_front(TB);
       BG->BytesInBGAtLastCheckpoint = 0;
-      BG->MaxCachedPerBatch = TransferBatchT::MaxNumCached;
+      BG->MaxCachedPerBatch = MaxNumBlocksInBatch;
 
       return BG;
     };
 
     auto InsertBlocks = [&](BatchGroupT *BG, CompactPtrT *Array, u32 Size) {
-      SinglyLinkedList<TransferBatchT> &Batches = BG->Batches;
-      TransferBatchT *CurBatch = Batches.front();
+      SinglyLinkedList<BatchT> &Batches = BG->Batches;
+      BatchT *CurBatch = Batches.front();
       DCHECK_NE(CurBatch, nullptr);
 
       for (u32 I = 0; I < Size;) {
@@ -803,7 +812,7 @@ template <typename Config> class SizeClassAllocator64 {
         u16 UnusedSlots =
             static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
         if (UnusedSlots == 0) {
-          CurBatch = reinterpret_cast<TransferBatchT *>(
+          CurBatch = reinterpret_cast<BatchT *>(
               SizeClassAllocator->getBatchClassBlock());
           CurBatch->clear();
           Batches.push_front(CurBatch);
@@ -961,7 +970,7 @@ template <typename Config> class SizeClassAllocator64 {
     if (Region->FreeListInfo.BlockList.empty())
       return 0U;
 
-    SinglyLinkedList<TransferBatchT> &Batches =
+    SinglyLinkedList<BatchT> &Batches =
         Region->FreeListInfo.BlockList.front()->Batches;
 
     if (Batches.empty()) {
@@ -970,8 +979,8 @@ template <typename Config> class SizeClassAllocator64 {
       Region->FreeListInfo.BlockList.pop_front();
 
       // Block used by `BatchGroup` is from BatchClassId. Turn the block into
-      // `TransferBatch` with single block.
-      TransferBatchT *TB = reinterpret_cast<TransferBatchT *>(BG);
+      // `Batch` with single block.
+      BatchT *TB = reinterpret_cast<BatchT *>(BG);
       ToArray[0] =
           compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(TB));
       Region->FreeListInfo.PoppedBlocks += 1;
@@ -979,18 +988,18 @@ template <typename Config> class SizeClassAllocator64 {
     }
 
     // So far, instead of always filling blocks to `MaxBlockCount`, we only
-    // examine single `TransferBatch` to minimize the time spent in the primary
-    // allocator. Besides, the sizes of `TransferBatch` and
+    // examine single `Batch` to minimize the time spent in the primary
+    // allocator. Besides, the sizes of `Batch` and
     // `SizeClassAllocatorT::getMaxCached()` may also impact the time spent on
     // accessing the primary allocator.
     // TODO(chiahungduan): Evaluate if we want to always prepare `MaxBlockCount`
-    // blocks and/or adjust the size of `TransferBatch` according to
+    // blocks and/or adjust the size of `Batch` according to
     // `SizeClassAllocatorT::getMaxCached()`.
-    TransferBatchT *B = Batches.front();
+    BatchT *B = Batches.front();
     DCHECK_NE(B, nullptr);
     DCHECK_GT(B->getCount(), 0U);
 
-    // BachClassId should always take all blocks in the TransferBatch. Read the
+    // BachClassId should always take all blocks in the Batch. Read the
     // comment in `pushBatchClassBlocks()` for more details.
     const u16 PopCount = ClassId == SizeClassMap::BatchClassId
                              ? B->getCount()
@@ -1001,7 +1010,7 @@ template <typename Config> class SizeClassAllocator64 {
     // done without holding `FLLock`.
     if (B->empty()) {
       Batches.pop_front();
-      // `TransferBatch` of BatchClassId is self-contained, no need to
+      // `Batch` of BatchClassId is self-contained, no need to
       // deallocate. Read the comment in `pushBatchClassBlocks()` for more
       // details.
       if (ClassId != SizeClassMap::BatchClassId)
@@ -1014,7 +1023,7 @@ template <typename Config> class SizeClassAllocator64 {
         // We don't keep BatchGroup with zero blocks to avoid empty-checking
         // while allocating. Note that block used for constructing BatchGroup is
         // recorded as free blocks in the last element of BatchGroup::Batches.
-        // Which means, once we pop the last TransferBatch, the block is
+        // Which means, once we pop the last Batch, the block is
         // implicitly deallocated.
         if (ClassId != SizeClassMap::BatchClassId)
           SizeClassAllocator->deallocate(SizeClassMap::BatchClassId, BG);
@@ -1082,8 +1091,7 @@ template <typename Config> class SizeClassAllocator64 {
                              Size));
     DCHECK_GT(NumberOfBlocks, 0);
 
-    constexpr u32 ShuffleArraySize =
-        MaxNumBatches * TransferBatchT::MaxNumCached;
+    constexpr u32 ShuffleArraySize = MaxNumBatches * MaxNumBlocksInBatch;
     CompactPtrT ShuffleArray[ShuffleArraySize];
     DCHECK_LE(NumberOfBlocks, ShuffleArraySize);
 
@@ -1492,7 +1500,7 @@ template <typename Config> class SizeClassAllocator64 {
       DCHECK_LE(Region->RegionBeg, BatchGroupBase);
       DCHECK_GE(AllocatedUserEnd, BatchGroupBase);
       DCHECK_EQ((Region->RegionBeg - BatchGroupBase) % GroupSize, 0U);
-      // TransferBatches are pushed in front of BG.Batches. The first one may
+      // Batches are pushed in front of BG.Batches. The first one may
       // not have all caches used.
       const uptr NumBlocks = (BG->Batches.size() - 1) * BG->MaxCachedPerBatch +
                              BG->Batches.front()->getCount();
@@ -1747,7 +1755,7 @@ template <typename Config> class SizeClassAllocator64 {
       }
 
       BatchGroupT *Cur = GroupsToRelease.front();
-      TransferBatchT *UnusedTransferBatch = nullptr;
+      BatchT *UnusedBatch = nullptr;
       GroupsToRelease.pop_front();
 
       if (BG->CompactPtrGroupBase == Cur->CompactPtrGroupBase) {
@@ -1756,36 +1764,35 @@ template <typename Config> class SizeClassAllocator64 {
         BG->BytesInBGAtLastCheckpoint = Cur->BytesInBGAtLastCheckpoint;
         const uptr MaxCachedPerBatch = BG->MaxCachedPerBatch;
 
-        // Note that the first TransferBatches in both `Batches` may not be
-        // full and only the first TransferBatch can have non-full blocks. Thus
+        // Note that the first Batches in both `Batches` may not be
+        // full and only the first Batch can have non-full blocks. Thus
         // we have to merge them before appending one to another.
         if (Cur->Batches.front()->getCount() == MaxCachedPerBatch) {
           BG->Batches.append_back(&Cur->Batches);
         } else {
-          TransferBatchT *NonFullBatch = Cur->Batches.front();
+          BatchT *NonFullBatch = Cur->Batches.front();
           Cur->Batches.pop_front();
           const u16 NonFullBatchCount = NonFullBatch->getCount();
           // The remaining Batches in `Cur` are full.
           BG->Batches.append_back(&Cur->Batches);
 
           if (BG->Batches.front()->getCount() == MaxCachedPerBatch) {
-            // Only 1 non-full TransferBatch, push it to the front.
+            // Only 1 non-full Batch, push it to the front.
             BG->Batches.push_front(NonFullBatch);
           } else {
             const u16 NumBlocksToMove = static_cast<u16>(
                 Min(static_cast<u16>(MaxCachedPerBatch -
                                      BG->Batches.front()->getCount()),
                     NonFullBatchCount));
-            BG->Batches.front()->appendFromTransferBatch(NonFullBatch,
-                                                         NumBlocksToMove);
+            BG->Batches.front()->appendFromBatch(NonFullBatch, NumBlocksToMove);
             if (NonFullBatch->isEmpty())
-              UnusedTransferBatch = NonFullBatch;
+              UnusedBatch = NonFullBatch;
             else
               BG->Batches.push_front(NonFullBatch);
           }
         }
 
-        const u32 NeededSlots = UnusedTransferBatch == nullptr ? 1U : 2U;
+        const u32 NeededSlots = UnusedBatch == nullptr ? 1U : 2U;
         if (UNLIKELY(Idx + NeededSlots > MaxUnusedSize)) {
           ScopedLock L(BatchClassRegion->FLLock);
           pushBatchClassBlocks(BatchClassRegion, Blocks, Idx);
@@ -1795,10 +1802,9 @@ template <typename Config> class SizeClassAllocator64 {
         }
         Blocks[Idx++] =
             compactPtr(SizeClassMap::BatchClassId, reinterpret_cast<uptr>(Cur));
-        if (UnusedTransferBatch) {
-          Blocks[Idx++] =
-              compactPtr(SizeClassMap::BatchClassId,
-                         reinterpret_cast<uptr>(UnusedTransferBatch));
+        if (UnusedBatch) {
+          Blocks[Idx++] = compactPtr(SizeClassMap::BatchClassId,
+                                     reinterpret_cast<uptr>(UnusedBatch));
         }
         Prev = BG;
         BG = BG->Next;
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 7dc38c2ede8ca..1f5df28fd7771 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -369,8 +369,8 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
         auto Pair = V.back();
         SizeClassAllocator.deallocate(Pair.first, Pair.second);
         V.pop_back();
-        // This increases the chance of having non-full TransferBatches and it
-        // will jump into the code path of merging TransferBatches.
+        // This increases the chance of having non-full Batches and it will jump
+        // into the code path of merging Batches.
         if (std::rand() % 8 == 0)
           SizeClassAllocator.drain();
       }
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index a4bc3d6cff91a..14b25a8995dab 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -22,6 +22,7 @@
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_posix.h"
@@ -747,6 +748,41 @@ TSAN_INTERCEPTOR(void, free, void *p) {
   user_free(thr, pc, p);
 }
 
+#  if SANITIZER_INTERCEPT_FREE_SIZED
+TSAN_INTERCEPTOR(void, free_sized, void *p, uptr size) {
+  if (UNLIKELY(!p))
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(free_sized, p, size);
+  user_free(thr, pc, p);
+}
+#    define TSAN_MAYBE_INTERCEPT_FREE_SIZED INTERCEPT_FUNCTION(free_sized)
+#  else
+#    define TSAN_MAYBE_INTERCEPT_FREE_SIZED
+#  endif
+
+#  if SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
+TSAN_INTERCEPTOR(void, free_aligned_sized, void *p, uptr alignment, uptr size) {
+  if (UNLIKELY(!p))
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(free_aligned_sized, p, alignment, size);
+  user_free(thr, pc, p);
+}
+#    define TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED \
+      INTERCEPT_FUNCTION(free_aligned_sized)
+#  else
+#    define TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
+#  endif
+
 TSAN_INTERCEPTOR(void, cfree, void *p) {
   if (UNLIKELY(!p))
     return;
@@ -763,6 +799,9 @@ TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
   SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
   return user_alloc_usable_size(p);
 }
+#else
+#  define TSAN_MAYBE_INTERCEPT_FREE_SIZED
+#  define TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
 #endif
 
 TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) {
@@ -2963,6 +3002,8 @@ void InitializeInterceptors() {
   TSAN_INTERCEPT(realloc);
   TSAN_INTERCEPT(reallocarray);
   TSAN_INTERCEPT(free);
+  TSAN_MAYBE_INTERCEPT_FREE_SIZED;
+  TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED;
   TSAN_INTERCEPT(cfree);
   TSAN_INTERCEPT(munmap);
   TSAN_MAYBE_INTERCEPT_MEMALIGN;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp
index e973be963e575..9d097806fa9b2 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp
@@ -73,15 +73,19 @@ using namespace __tsan;
   invoke_free_hook(ptr);                                     \
   SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
   user_free(thr, pc, ptr)
-#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
-#define COMMON_MALLOC_FILL_STATS(zone, stats)
-#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
-  (void)zone_name; \
-  Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
-#define COMMON_MALLOC_NAMESPACE __tsan
-#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
-#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
+#  define COMMON_MALLOC_FREE_SIZED(ptr, size) COMMON_MALLOC_FREE(ptr)
+#  define COMMON_MALLOC_FREE_ALIGNED_SIZED(ptr, alignment, size) \
+    COMMON_MALLOC_FREE(ptr)
+#  define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
+#  define COMMON_MALLOC_FILL_STATS(zone, stats)
+#  define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name)    \
+    (void)zone_name;                                                        \
+    Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", \
+           ptr);
+#  define COMMON_MALLOC_NAMESPACE __tsan
+#  define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
+#  define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
 
-#include "sanitizer_common/sanitizer_malloc_mac.inc"
+#  include "sanitizer_common/sanitizer_malloc_mac.inc"
 
 #endif
diff --git a/compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp b/compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp
new file mode 100644
index 0000000000000..c7d62f15c3c3e
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp
@@ -0,0 +1,29 @@
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-MISMATCH
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SUCCESS
+
+// RUN: %clangxx_asan -O0 %s -o %t -DUSER_FUNCTION
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-MISMATCH
+
+// It is expected that ASAN_OPTS will override the value set through the user function.
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SUCCESS
+
+#if USER_FUNCTION
+// It's important to test the `alloc_dealloc_mismatch` flag set through the user function because, on Windows,
+// flags configured through the user-defined function `__asan_default_options` are not always be honored.
+// See: https://github.com/llvm/llvm-project/issues/117925
+extern "C" __declspec(dllexport) extern const char *__asan_default_options() {
+  return "alloc_dealloc_mismatch=1";
+}
+#endif
+
+#include <cstdio>
+#include <cstdlib>
+
+// Tests the `alloc_dealloc_mismatch` flag set both via user function and through the environment variable.
+int main() {
+  // In the 'CHECK-MISMATCH' case, we simply check that the AddressSanitizer reports an error.
+  delete (new int[10]); // CHECK-MISMATCH: AddressSanitizer:
+  printf("Success");    // CHECK-SUCCESS: Success
+  return 0;
+}
\ No newline at end of file
diff --git a/compiler-rt/test/fuzzer/uncaught-exception.test b/compiler-rt/test/fuzzer/uncaught-exception.test
index d1b98cfb7c74b..264bb5dc3a914 100644
--- a/compiler-rt/test/fuzzer/uncaught-exception.test
+++ b/compiler-rt/test/fuzzer/uncaught-exception.test
@@ -6,8 +6,6 @@ RUN: %cpp_compiler %S/UncaughtException.cpp -o %t-UncaughtException
 
 # Clang will fail the test with 'deadly signal', but other compilers may fail with different error messages.
 # For example, msvc fails with 'uncaught C++ exception'. So the error we check depends on the compiler target.
-RUN: not %run %t-UncaughtException 2>&1 | FileCheck %s --check-prefixes=CHECK-CRASH,%if target={{.*-windows-msvc.*}} %{CHECK-MSVC%} %else %{CHECK-ERROR%}
+RUN: not %run %t-UncaughtException 2>&1 | FileCheck %s
 
-CHECK-ERROR: ERROR: libFuzzer: deadly signal
-CHECK-MSVC: ERROR: libFuzzer: uncaught C++ exception
-CHECK-CRASH: Test unit written to ./crash
+CHECK: ERROR: libFuzzer: {{deadly signal|uncaught C\+\+ exception}}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
index 7710c62368191..dc11e7fa5e9b3 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
index 9eac562fecb03..7a993d481bc5a 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py b/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py
index 57a01f3852b06..ddc260fa61644 100755
--- a/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py
+++ b/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py
@@ -5,13 +5,13 @@
 
 idx = 1
 for arg in sys.argv[1:]:
-  if not "=" in arg:
-    break
-  idx += 1
-  (argname, argval) = arg.split("=")
-  os.environ["SIMCTL_CHILD_" + argname] = argval
+    if not "=" in arg:
+        break
+    idx += 1
+    (argname, argval) = arg.split("=", maxsplit=1)
+    os.environ["SIMCTL_CHILD_" + argname] = argval
 
 exitcode = subprocess.call(sys.argv[idx:])
 if exitcode > 125:
-  exitcode = 126
+    exitcode = 126
 sys.exit(exitcode)
diff --git a/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py b/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py
index 5e977ea5ed908..ec394067dc7a5 100755
--- a/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py
+++ b/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import glob, os, pipes, sys, subprocess
+import glob, os, shlex, sys, subprocess
 
 
 device_id = os.environ.get("SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER")
@@ -21,8 +21,11 @@
     "ASAN_ACTIVATION_OPTIONS",
     "MallocNanoZone",
 ]:
-    if e in os.environ:
-        os.environ["SIMCTL_CHILD_" + e] = os.environ[e]
+    simctl_version = "SIMCTL_CHILD_" + e
+    # iossim_env.py might have already set these using arguments it was given
+    # (and that we can't see from inside this script). Don't overwrite them!
+    if e in os.environ and simctl_version not in os.environ:
+        os.environ[simctl_version] = os.environ[e]
 
 find_atos_cmd = "xcrun -sdk iphonesimulator -f atos"
 atos_path = (
@@ -49,8 +52,7 @@
             # Don't quote glob pattern
             rm_args.append(arg)
         else:
-            # FIXME(dliew): pipes.quote() is deprecated
-            rm_args.append(pipes.quote(arg))
+            rm_args.append(shlex.quote(arg))
     rm_cmd_line = ["/bin/rm"] + rm_args
     rm_cmd_line_str = " ".join(rm_cmd_line)
     # We use `shell=True` so that any wildcard globs get expanded by the shell.
diff --git a/compiler-rt/test/sanitizer_common/lit.common.cfg.py b/compiler-rt/test/sanitizer_common/lit.common.cfg.py
index c3c1336bacd53..88d3ea9bc5ad2 100644
--- a/compiler-rt/test/sanitizer_common/lit.common.cfg.py
+++ b/compiler-rt/test/sanitizer_common/lit.common.cfg.py
@@ -87,7 +87,7 @@ def build_invocation(compile_flags):
 config.substitutions.append(("%tool_name", config.tool_name))
 config.substitutions.append(("%tool_options", tool_options))
 config.substitutions.append(
-    ("%env_tool_opts=", "env " + tool_options + "=" + default_tool_options_str)
+    ("%env_tool_opts=", "%env " + tool_options + "=" + default_tool_options_str)
 )
 
 config.suffixes = [".c", ".cpp"]
diff --git a/compiler-rt/test/tsan/munmap_clear_shadow.c b/compiler-rt/test/tsan/munmap_clear_shadow.c
index f48a12a4cc0cb..1fbc77ec8fd14 100644
--- a/compiler-rt/test/tsan/munmap_clear_shadow.c
+++ b/compiler-rt/test/tsan/munmap_clear_shadow.c
@@ -1,5 +1,10 @@
 // RUN: %clang_tsan %s -o %t && %run %t | FileCheck %s
 
+// In these systems, the behavior of ReleaseMemoryPagesToOS is madvise(beg, end, MADV_FREE),
+// which tags the relevant pages as 'FREE' and does not release them immediately.
+// Therefore, we cannot assume that __tsan_read1 will not race with the shadow cleared.
+// UNSUPPORTED: darwin,target={{.*(freebsd|netbsd|solaris|haiku).*}}
+
 #include "test.h"
 #include <assert.h>
 #include <pthread.h>
diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt
index 7f2fee48fda77..b4b1f47626073 100644
--- a/cross-project-tests/CMakeLists.txt
+++ b/cross-project-tests/CMakeLists.txt
@@ -94,6 +94,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}
diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md
new file mode 100644
index 0000000000000..12f9aa19b0d9b
--- /dev/null
+++ b/cross-project-tests/dtlto/README.md
@@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.
\ No newline at end of file
diff --git a/cross-project-tests/dtlto/ld-dtlto.c b/cross-project-tests/dtlto/ld-dtlto.c
new file mode 100644
index 0000000000000..3ee962346bd4a
--- /dev/null
+++ b/cross-project-tests/dtlto/ld-dtlto.c
@@ -0,0 +1,41 @@
+// REQUIRES: ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | sort | FileCheck %s
+
+/// No files are expected before.
+// CHECK-NOT: {{.}}
+
+/// Linked ELF.
+// CHECK: {{^}}a.out{{$}}
+
+/// Produced by the bitcode compilation.
+// CHECK-NEXT: {{^}}dtlto.o{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-NEXT: {{^}}dtlto.s{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}}
+
+/// No files are expected after.
+// CHECK-NOT: {{.}}
+
+int _start() { return 0; }
diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg
new file mode 100644
index 0000000000000..38eed282704c6
--- /dev/null
+++ b/cross-project-tests/dtlto/lit.local.cfg
@@ -0,0 +1,5 @@
+if any(
+    f not in config.available_features
+    for f in ("clang", "x86-registered-target")
+):
+    config.unsupported = True
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index f936a4192a33c..2ca4952b0fcc9 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -598,7 +598,8 @@ RT_API_ATTRS int DerivedAssignTicket<IS_COMPONENTWISE>::Continue(
         std::size_t componentByteSize{
             this->component_->SizeInBytes(this->instance_)};
         if (IS_COMPONENTWISE && toIsContiguous_ && fromIsContiguous_) {
-          std::size_t offset{this->component_->offset()};
+          std::size_t offset{
+              static_cast<std::size_t>(this->component_->offset())};
           char *to{this->instance_.template OffsetElement<char>(offset)};
           const char *from{
               this->from_->template OffsetElement<const char>(offset)};
@@ -630,7 +631,8 @@ RT_API_ATTRS int DerivedAssignTicket<IS_COMPONENTWISE>::Continue(
       std::size_t componentByteSize{
           this->component_->SizeInBytes(this->instance_)};
       if (IS_COMPONENTWISE && toIsContiguous_ && fromIsContiguous_) {
-        std::size_t offset{this->component_->offset()};
+        std::size_t offset{
+            static_cast<std::size_t>(this->component_->offset())};
         char *to{this->instance_.template OffsetElement<char>(offset)};
         const char *from{
             this->from_->template OffsetElement<const char>(offset)};
diff --git a/flang-rt/lib/runtime/descriptor-io.cpp b/flang-rt/lib/runtime/descriptor-io.cpp
index e7b99e6fc3a2b..b208cb2c397b3 100644
--- a/flang-rt/lib/runtime/descriptor-io.cpp
+++ b/flang-rt/lib/runtime/descriptor-io.cpp
@@ -451,39 +451,42 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
     if (const typeInfo::DerivedType *type{
             addendum ? addendum->derivedType() : nullptr}) {
       // derived type unformatted I/O
-      if (table_) {
-        if (const auto *definedIo{table_->Find(*type,
-                DIR == Direction::Input
-                    ? common::DefinedIo::ReadUnformatted
-                    : common::DefinedIo::WriteUnformatted)}) {
-          if (definedIo->subroutine) {
-            typeInfo::SpecialBinding special{DIR == Direction::Input
-                    ? typeInfo::SpecialBinding::Which::ReadUnformatted
-                    : typeInfo::SpecialBinding::Which::WriteUnformatted,
-                definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                false};
-            if (DefinedUnformattedIo(io_, instance_, *type, special)) {
-              anyIoTookPlace_ = true;
-              return StatOk;
+      if (DIR == Direction::Input || !io_.get_if<InquireIOLengthState>()) {
+        if (table_) {
+          if (const auto *definedIo{table_->Find(*type,
+                  DIR == Direction::Input
+                      ? common::DefinedIo::ReadUnformatted
+                      : common::DefinedIo::WriteUnformatted)}) {
+            if (definedIo->subroutine) {
+              typeInfo::SpecialBinding special{DIR == Direction::Input
+                      ? typeInfo::SpecialBinding::Which::ReadUnformatted
+                      : typeInfo::SpecialBinding::Which::WriteUnformatted,
+                  definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
+                  false};
+              if (DefinedUnformattedIo(io_, instance_, *type, special)) {
+                anyIoTookPlace_ = true;
+                return StatOk;
+              }
+            } else {
+              int status{workQueue.BeginDerivedIo<DIR>(
+                  io_, instance_, *type, table_, anyIoTookPlace_)};
+              return status == StatContinue ? StatOk : status; // done here
             }
-          } else {
-            int status{workQueue.BeginDerivedIo<DIR>(
-                io_, instance_, *type, table_, anyIoTookPlace_)};
-            return status == StatContinue ? StatOk : status; // done here
           }
         }
-      }
-      if (const typeInfo::SpecialBinding *special{
-              type->FindSpecialBinding(DIR == Direction::Input
-                      ? typeInfo::SpecialBinding::Which::ReadUnformatted
-                      : typeInfo::SpecialBinding::Which::WriteUnformatted)}) {
-        if (!table_ || !table_->ignoreNonTbpEntries || special->IsTypeBound()) {
-          // defined derived type unformatted I/O
-          if (DefinedUnformattedIo(io_, instance_, *type, *special)) {
-            anyIoTookPlace_ = true;
-            return StatOk;
-          } else {
-            return IostatEnd;
+        if (const typeInfo::SpecialBinding *special{
+                type->FindSpecialBinding(DIR == Direction::Input
+                        ? typeInfo::SpecialBinding::Which::ReadUnformatted
+                        : typeInfo::SpecialBinding::Which::WriteUnformatted)}) {
+          if (!table_ || !table_->ignoreNonTbpEntries ||
+              special->IsTypeBound()) {
+            // defined derived type unformatted I/O
+            if (DefinedUnformattedIo(io_, instance_, *type, *special)) {
+              anyIoTookPlace_ = true;
+              return StatOk;
+            } else {
+              return IostatEnd;
+            }
           }
         }
       }
diff --git a/flang-rt/lib/runtime/type-info.cpp b/flang-rt/lib/runtime/type-info.cpp
index d023c3392d559..3e1d7c9c3c788 100644
--- a/flang-rt/lib/runtime/type-info.cpp
+++ b/flang-rt/lib/runtime/type-info.cpp
@@ -140,7 +140,7 @@ RT_API_ATTRS void Component::CreatePointerDescriptor(Descriptor &descriptor,
     const SubscriptValue *subscripts) const {
   RUNTIME_CHECK(terminator, genre_ == Genre::Data);
   EstablishDescriptor(descriptor, container, terminator);
-  std::size_t offset{offset_};
+  std::size_t offset{static_cast<std::size_t>(offset_)};
   if (subscripts) {
     offset += container.SubscriptsToByteOffset(subscripts);
   }
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 068d134671db8..0bfada476348a 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -504,9 +504,10 @@ add_subdirectory(include)
 add_subdirectory(lib)
 add_subdirectory(cmake/modules)
 
+option(FLANG_INCLUDE_TOOLS "Generate build targets for the Flang tools." ON)
 option(FLANG_BUILD_TOOLS
   "Build the Flang tools. If OFF, just generate build targets." ON)
-if (FLANG_BUILD_TOOLS)
+if (FLANG_INCLUDE_TOOLS)
   add_subdirectory(tools)
 endif()
 
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 871749934810c..37a49f12f9177 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -868,6 +868,20 @@ print *, [(j,j=1,10)]
   the elements for each component before proceeding to the next component.
   A program using defined assignment might be able to detect the difference.
 
+* The standard forbids instances of derived types with defined unformatted
+  WRITE subroutines from appearing in the I/O list of an `INQUIRE(IOLENGTH=...)`
+  statement.  It then also says that these defined I/O procedures should be
+  ignored for that statement.  So we allow them to appear (like most
+  compilers) and don't use any defined unformatted WRITE that might have been
+  defined.
+
+* Forward references to target objects are allowed to appear
+  in the initializers of data pointer declarationss.
+  Forward references to target objects are not accepted in the default
+  initializers of derived type component declarations, however,
+  since these default values need to be available to process incomplete
+  structure constructors.
+
 ## De Facto Standard Features
 
 * `EXTENDS_TYPE_OF()` returns `.TRUE.` if both of its arguments have the
diff --git a/flang/docs/OpenMP-declare-target.md b/flang/docs/OpenMP-declare-target.md
index 45062469007b6..478db2d004c87 100644
--- a/flang/docs/OpenMP-declare-target.md
+++ b/flang/docs/OpenMP-declare-target.md
@@ -7,7 +7,15 @@
 
 -->
 
-# Introduction to Declare Target
+# OpenMP Declare Target
+
+```{contents}
+---
+local:
+---
+```
+
+## Introduction to Declare Target
 
 In OpenMP `declare target` is a directive that can be applied to a function or
 variable (primarily global) to notate to the compiler that it should be 
@@ -55,7 +63,7 @@ in this case, leave off the `declare target` from the function and it
 would be implicitly marked `declare target any` (for both host and device), 
 as it's been utilised within a target region.
 
-# Declare Target as represented in the OpenMP Dialect
+## Declare Target as represented in the OpenMP Dialect
 
 In the OpenMP Dialect `declare target` is not represented by a specific 
 `operation`. Instead, it's an OpenMP dialect specific `attribute` that can be 
@@ -89,7 +97,7 @@ llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(Op.getOperation());
 declareTargetGlobal.isDeclareTarget();
 ```
 
-# Declare Target Fortran OpenMP Lowering
+## Declare Target Fortran OpenMP Lowering
 
 The initial lowering of `declare target` to MLIR for both use-cases is done
 inside of the usual OpenMP lowering in flang/lib/Lower/OpenMP.cpp. However, 
@@ -145,7 +153,7 @@ to the way `threadprivate` handles these cases, we raise and initialize the
 variable as an internal `GlobalOp` and apply the attribute. This occurs in the
 flang/lib/Lower/OpenMP.cpp function `genDeclareTargetIntGlobal`.
 
-# Declare Target Transformation Passes for Flang
+## Declare Target Transformation Passes for Flang
 
 There are currently two passes within Flang that are related to the processing 
 of `declare target`:
@@ -173,7 +181,7 @@ While this infrastructure could be generally applicable to more than just Flang,
 it is only utilised in the Flang frontend, so it resides there rather than in 
 the OpenMP dialect codebase. 
 
-# Declare Target OpenMP Dialect To LLVM-IR Lowering
+## Declare Target OpenMP Dialect To LLVM-IR Lowering
 
 The OpenMP dialect lowering of `declare target` is done through the 
 `amendOperation` flow, as it's not an `operation` but rather an 
@@ -228,7 +236,7 @@ place. It may be possible to store this information in the OMPIRBuilder and
 then perform this cleanup process on finalization, but this is open for 
 discussion and implementation still.
 
-# Current Support
+## Current Support
 
 For the moment, `declare target` should work for:
 * Marking functions/subroutines and function/subroutine interfaces for 
diff --git a/flang/docs/OpenMP-descriptor-management.md b/flang/docs/OpenMP-descriptor-management.md
index cbc27f2a0fcb8..d0764c2df4642 100644
--- a/flang/docs/OpenMP-descriptor-management.md
+++ b/flang/docs/OpenMP-descriptor-management.md
@@ -6,8 +6,15 @@
 
 -->
 
-# OpenMP dialect: Fortran descriptor type mapping for offload
+# Fortran descriptor type mapping for OpenMP offload
+```{contents}
+---
+local:
+---
+```
+
 
+## Details
 The initial method for mapping Fortran types tied to descriptors for OpenMP offloading is to treat these types 
 as a special case of OpenMP record type (C/C++ structure/class, Fortran derived type etc.) mapping as far as the 
 runtime is concerned. Where the box (descriptor information) is the holding container and the underlying 
@@ -103,7 +110,7 @@ it is a little rigid in how the descriptor mappings are handled as there is no s
 to specialise the mappings for possible edge cases without polluting the dialect or lowering with further
 knowledge of Fortran and the FIR dialect.
 
-# OpenMP dialect differences from OpenACC dialect
+## Differences from OpenACC
 
 The descriptor mapping for OpenMP currently works differently to the planned direction for OpenACC, however, 
 it is possible and would likely be ideal to align the method with OpenACC in the future. 
diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index 35da8323e0a10..b2dbbcb5630f4 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -34,6 +34,8 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and
+  Clang/AArch64) (requires GLIBC 2.40 or newer)
 
 ## Windows Support
 
diff --git a/flang/include/flang/Common/interval.h b/flang/include/flang/Common/interval.h
index c4cab0ccf1130..2f0b6d9b55fe5 100644
--- a/flang/include/flang/Common/interval.h
+++ b/flang/include/flang/Common/interval.h
@@ -60,7 +60,8 @@ template <typename A> class Interval {
     return start_ <= x && x < start_ + size_;
   }
   constexpr bool Contains(const Interval &that) const {
-    return Contains(that.start_) && Contains(that.start_ + (that.size_ - 1));
+    return Contains(that.start_) &&
+        ((that.size_ == 0) || Contains(that.start_ + (that.size_ - 1)));
   }
   constexpr bool IsDisjointWith(const Interval &that) const {
     return that.NextAfter() <= start_ || NextAfter() <= that.start_;
diff --git a/flang/include/flang/Evaluate/complex.h b/flang/include/flang/Evaluate/complex.h
index 2dcd28b59968c..720ccaf512df6 100644
--- a/flang/include/flang/Evaluate/complex.h
+++ b/flang/include/flang/Evaluate/complex.h
@@ -45,7 +45,7 @@ template <typename REAL_TYPE> class Complex {
         im_.Compare(that.im_) == Relation::Equal;
   }
 
-  constexpr bool IsZero() const { return re_.IsZero() || im_.IsZero(); }
+  constexpr bool IsZero() const { return re_.IsZero() && im_.IsZero(); }
 
   constexpr bool IsInfinite() const {
     return re_.IsInfinite() || im_.IsInfinite();
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index d26a477ddded1..18c244f6f450f 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1286,17 +1286,6 @@ bool CheckForCoindexedObject(parser::ContextualMessages &,
     const std::optional<ActualArgument> &, const std::string &procName,
     const std::string &argName);
 
-inline bool CanCUDASymbolHaveSaveAttr(const Symbol &sym) {
-  if (const auto *details =
-          sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()) {
-    if (details->cudaDataAttr() &&
-        *details->cudaDataAttr() != common::CUDADataAttr::Unified) {
-      return false;
-    }
-  }
-  return true;
-}
-
 inline bool IsCUDADeviceSymbol(const Symbol &sym) {
   if (const auto *details =
           sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()) {
@@ -1585,6 +1574,7 @@ bool IsExtensibleType(const DerivedTypeSpec *);
 bool IsSequenceOrBindCType(const DerivedTypeSpec *);
 bool IsBuiltinDerivedType(const DerivedTypeSpec *derived, const char *name);
 bool IsBuiltinCPtr(const Symbol &);
+bool IsFromBuiltinModule(const Symbol &);
 bool IsEventType(const DerivedTypeSpec *);
 bool IsLockType(const DerivedTypeSpec *);
 bool IsNotifyType(const DerivedTypeSpec *);
diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 59f8ba0cffcf0..ef956a36917fd 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -15,6 +15,7 @@
 
 #include "flang/Lower/LoweringOptions.h"
 #include "flang/Lower/PFTDefs.h"
+#include "flang/Lower/Support/Utils.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Semantics/symbol.h"
@@ -23,6 +24,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
 
 namespace mlir {
 class SymbolTable;
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index d97abf4d864b8..3263ab129d076 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -66,5 +66,9 @@ ENUM_LOWERINGOPT(RepackArraysWhole, unsigned, 1, 0)
 /// If true, CUDA Fortran runtime check is inserted.
 ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
 
+/// If true, do not generate definition for runtime type info global objects of
+/// derived types defined in other compilation units.
+ENUM_LOWERINGOPT(SkipExternalRttiDefinition, unsigned, 1, 0)
+
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Lower/SymbolMap.h b/flang/include/flang/Lower/SymbolMap.h
index c03f9afd40801..e52d1075e2b9d 100644
--- a/flang/include/flang/Lower/SymbolMap.h
+++ b/flang/include/flang/Lower/SymbolMap.h
@@ -128,7 +128,7 @@ struct SymbolBox : public fir::details::matcher<SymbolBox> {
                                        const SymbolBox &symBox);
 
   /// Dump the map. For debugging.
-  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; }
+  LLVM_DUMP_METHOD void dump() const;
 
 private:
   VT box;
@@ -293,7 +293,7 @@ class SymMap {
                                        const SymMap &symMap);
 
   /// Dump the map. For debugging.
-  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; }
+  LLVM_DUMP_METHOD void dump() const;
 
   void addVariableDefinition(semantics::SymbolRef symRef,
                              fir::FortranVariableOpInterface definingOp,
diff --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index 65b9cce1d2021..5c56dd6b695f8 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -27,7 +27,8 @@ mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
 bool isCUDADeviceContext(mlir::Operation *op);
-bool isCUDADeviceContext(mlir::Region &);
+bool isCUDADeviceContext(mlir::Region &,
+                         bool isDoConcurrentOffloadEnabled = false);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
 bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index e1eaab3346901..f2c363e09e9e6 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -900,6 +900,10 @@ uint64_t getAllocaAddressSpace(const mlir::DataLayout *dataLayout);
 llvm::SmallVector<mlir::Value> deduceOptimalExtents(mlir::ValueRange extents1,
                                                     mlir::ValueRange extents2);
 
+uint64_t getGlobalAddressSpace(mlir::DataLayout *dataLayout);
+
+uint64_t getProgramAddressSpace(mlir::DataLayout *dataLayout);
+
 /// Given array extents generate code that sets them all to zeroes,
 /// if the array is empty, e.g.:
 ///   %false = arith.constant false
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 3cf7a4920ed7d..d38c5b6d09a82 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -443,6 +443,8 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genTranspose(mlir::Type,
                                   llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genThisGrid(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genThisThreadBlock(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genThisWarp(mlir::Type, llvm::ArrayRef<mlir::Value>);
   void genThreadFence(llvm::ArrayRef<fir::ExtendedValue>);
   void genThreadFenceBlock(llvm::ArrayRef<fir::ExtendedValue>);
   void genThreadFenceSystem(llvm::ArrayRef<fir::ExtendedValue>);
@@ -751,7 +753,7 @@ static inline mlir::FunctionType genFuncType(mlir::MLIRContext *context,
   }
 
   if (TyR::ty == ParamTypeId::Void)
-    return mlir::FunctionType::get(context, argTypes, std::nullopt);
+    return mlir::FunctionType::get(context, argTypes, {});
 
   auto resType = getTypeHelper(context, builder, TyR::ty, TyR::kind);
   return mlir::FunctionType::get(context, argTypes, {resType});
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 0398d0f248e08..93f07d8d5d4d9 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -39,6 +39,9 @@ struct FIRToLLVMPassOptions {
   // that such programs would crash at runtime if the derived type descriptors
   // are required by the runtime, so this is only an option to help debugging.
   bool ignoreMissingTypeDescriptors = false;
+  // Similar to ignoreMissingTypeDescriptors, but generate external declaration
+  // for the missing type descriptor globals instead.
+  bool skipExternalRttiDefinition = false;
 
   // Generate TBAA information for FIR types and memory accessing operations.
   bool applyTBAA = false;
diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
index 7b1c14e4dfdc9..cbdcf6b501b8e 100644
--- a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -189,6 +189,9 @@ class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern {
   unsigned
   getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
 
+  unsigned
+  getGlobalAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
+
   const fir::FIRToLLVMPassOptions &options;
 
   using ConvertToLLVMPattern::matchAndRewrite;
diff --git a/flang/include/flang/Optimizer/Passes/CommandLineOpts.h b/flang/include/flang/Optimizer/Passes/CommandLineOpts.h
index 1cfaf285e75e6..76ac9d0622d2b 100644
--- a/flang/include/flang/Optimizer/Passes/CommandLineOpts.h
+++ b/flang/include/flang/Optimizer/Passes/CommandLineOpts.h
@@ -32,6 +32,19 @@ extern llvm::cl::opt<std::size_t> arrayStackAllocationThreshold;
 /// generated by the frontend.
 extern llvm::cl::opt<bool> ignoreMissingTypeDescriptors;
 
+/// Shared option in tools to only generate rtti static object definitions for
+/// derived types defined in the current compilation unit. Derived type
+/// descriptor object for types defined in other objects will only be declared
+/// as external. This also changes the linkage of rtti objects defined in the
+/// current compilation unit from linkonce_odr to external so that unused rtti
+/// objects are retained and can be accessed from other compilation units. This
+/// is an experimental option to explore compilation speed improvements and is
+/// an ABI breaking change because of the linkage change.
+/// It will also require linking against module file objects of modules defining
+/// only types (even for trivial types without type bound procedures, which
+/// differs from most compilers).
+extern llvm::cl::opt<bool> skipExternalRttiDefinition;
+
 /// Default optimization level used to create Flang pass pipeline is O0.
 extern llvm::OptimizationLevel defaultOptLevel;
 
diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index ec73af6ec72e9..83c936b7dcada 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -35,32 +35,6 @@ inline std::int64_t toInt(mlir::arith::ConstantOp cop) {
       .getSExtValue();
 }
 
-// Reconstruct binding tables for dynamic dispatch.
-using BindingTable = llvm::DenseMap<llvm::StringRef, unsigned>;
-using BindingTables = llvm::DenseMap<llvm::StringRef, BindingTable>;
-
-inline void buildBindingTables(BindingTables &bindingTables,
-                               mlir::ModuleOp mod) {
-
-  // The binding tables are defined in FIR after lowering inside fir.type_info
-  // operations. Go through each binding tables and store the procedure name and
-  // binding index for later use by the fir.dispatch conversion pattern.
-  for (auto typeInfo : mod.getOps<fir::TypeInfoOp>()) {
-    unsigned bindingIdx = 0;
-    BindingTable bindings;
-    if (typeInfo.getDispatchTable().empty()) {
-      bindingTables[typeInfo.getSymName()] = bindings;
-      continue;
-    }
-    for (auto dtEntry :
-         typeInfo.getDispatchTable().front().getOps<fir::DTEntryOp>()) {
-      bindings[dtEntry.getMethod()] = bindingIdx;
-      ++bindingIdx;
-    }
-    bindingTables[typeInfo.getSymName()] = bindings;
-  }
-}
-
 // Translate front-end KINDs for use in the IR and code gen.
 inline std::vector<fir::KindTy>
 fromDefaultKinds(const Fortran::common::IntrinsicTypeDefaultKinds &defKinds) {
diff --git a/flang/include/flang/Parser/char-block.h b/flang/include/flang/Parser/char-block.h
index 38f4f7b82e1ea..b3b1f04034d3c 100644
--- a/flang/include/flang/Parser/char-block.h
+++ b/flang/include/flang/Parser/char-block.h
@@ -150,7 +150,12 @@ class CharBlock {
 
   int Compare(const char *that) const {
     std::size_t bytes{size()};
-    if (int cmp{std::strncmp(begin(), that, bytes)}) {
+    // strncmp is undefined if either pointer is null.
+    if (!bytes) {
+      return that == nullptr ? 0 : -1;
+    } else if (!that) {
+      return 1;
+    } else if (int cmp{std::strncmp(begin(), that, bytes)}) {
       return cmp;
     }
     return that[bytes] == '\0' ? 0 : -1;
diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index e19b16c23b82b..7371a46ad36ce 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -292,7 +292,8 @@ class Message : public common::ReferenceCounted<Message> {
   std::optional<ProvenanceRange> GetProvenanceRange(
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
-      bool echoSourceLine = true) const;
+      bool echoSourceLine = true,
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
@@ -352,7 +353,8 @@ class Messages {
   void Copy(const Messages &);
   void ResolveProvenances(const AllCookedSources &);
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
-      bool echoSourceLines = true) const;
+      bool echoSourceLines = true,
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
   bool AnyFatalError() const;
 
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e5..7e752eeb4dfe4 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -267,6 +267,7 @@ struct AccEndCombinedDirective;
 struct OpenACCDeclarativeConstruct;
 struct OpenACCRoutineConstruct;
 struct OpenMPConstruct;
+struct OpenMPLoopConstruct;
 struct OpenMPDeclarativeConstruct;
 struct OmpEndLoopDirective;
 struct OmpMemoryOrderClause;
@@ -5021,11 +5022,13 @@ struct OpenMPBlockConstruct {
 };
 
 // OpenMP directives enclosing do loop
+using NestedConstruct =
+    std::variant<DoConstruct, common::Indirection<OpenMPLoopConstruct>>;
 struct OpenMPLoopConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPLoopConstruct);
   OpenMPLoopConstruct(OmpBeginLoopDirective &&a)
       : t({std::move(a), std::nullopt, std::nullopt}) {}
-  std::tuple<OmpBeginLoopDirective, std::optional<DoConstruct>,
+  std::tuple<OmpBeginLoopDirective, std::optional<NestedConstruct>,
       std::optional<OmpEndLoopDirective>>
       t;
 };
diff --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
index e90d3ae8baf1e..6c5a061d1c1a2 100644
--- a/flang/include/flang/Semantics/runtime-type-info.h
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -38,6 +38,10 @@ RuntimeDerivedTypeTables BuildRuntimeDerivedTypeTables(SemanticsContext &);
 /// to describe other derived types at runtime in flang descriptor.
 constexpr char typeInfoBuiltinModule[]{"__fortran_type_info"};
 
+/// Name of the builtin derived type in __fortran_type_inf that is used for
+/// derived type descriptors.
+constexpr char typeDescriptorTypeName[]{"derivedtype"};
+
 /// Name of the bindings descriptor component in the DerivedType type of the
 /// __Fortran_type_info module
 constexpr char bindingDescCompName[]{"binding"};
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index f3cfa9b99fb4d..fb670528f3ce4 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -248,6 +248,8 @@ inline bool NeedCUDAAlloc(const Symbol &sym) {
   return false;
 }
 
+bool CanCUDASymbolBeGlobal(const Symbol &sym);
+
 const Scope *FindCUDADeviceContext(const Scope *);
 std::optional<common::CUDADataAttr> GetCUDADataAttr(const Symbol *);
 
@@ -654,6 +656,8 @@ DirectComponentIterator::const_iterator FindAllocatableOrPointerDirectComponent(
     const DerivedTypeSpec &);
 PotentialComponentIterator::const_iterator
 FindPolymorphicAllocatablePotentialComponent(const DerivedTypeSpec &);
+UltimateComponentIterator::const_iterator
+FindCUDADeviceAllocatableUltimateComponent(const DerivedTypeSpec &);
 
 // The LabelEnforce class (given a set of labels) provides an error message if
 // there is a branch to a label which is not in the given set.
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 39356daa3606a..857de9479e4e5 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -55,7 +55,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     SavedLocalInSpecExpr, PrintNamelist, AssumedRankPassedToNonAssumedRank,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
-    InaccessibleDeferredOverride, CudaWarpMatchFunction)
+    InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
@@ -156,14 +156,13 @@ class LanguageFeatureControl {
 private:
   // Map from Cli syntax of language features and usage warnings to their enum
   // values.
-  std::unordered_map<std::string, std::variant<LanguageFeature, UsageWarning>>
-      cliOptions_;
+  std::unordered_map<std::string, LanguageFeatureOrWarning> cliOptions_;
   // These two arrays map the enum values to their cannonical Cli spellings.
   // Since each of the CanonicalSpelling is a string in the domain of the map
   // above we just use a view of the string instead of another copy.
-  std::array<std::string_view, LanguageFeature_enumSize>
+  std::array<std::string, LanguageFeature_enumSize>
       languageFeatureCliCanonicalSpelling_;
-  std::array<std::string_view, UsageWarning_enumSize>
+  std::array<std::string, UsageWarning_enumSize>
       usageWarningCliCanonicalSpelling_;
   LanguageFeatures disable_;
   LanguageFeatures warnLanguage_;
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index 0154833baaf17..52e954d1cffbf 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -2192,7 +2192,14 @@ Expr<T> FoldOperation(FoldingContext &context, Power<T> &&x) {
       }
       return Expr<T>{Constant<T>{power.power}};
     } else {
-      if (auto callable{GetHostRuntimeWrapper<T, T, T>("pow")}) {
+      if (folded->first.IsZero()) {
+        if (folded->second.IsZero()) {
+          context.messages().Say(common::UsageWarning::FoldingException,
+              "REAL/COMPLEX 0**0 is not defined"_warn_en_US);
+        } else {
+          return Expr<T>(Constant<T>{folded->first}); // 0. ** nonzero -> 0.
+        }
+      } else if (auto callable{GetHostRuntimeWrapper<T, T, T>("pow")}) {
         return Expr<T>{
             Constant<T>{(*callable)(context, folded->first, folded->second)}};
       } else if (context.languageFeatures().ShouldWarn(
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 68838564f87ba..6a57d87a30e93 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -2194,8 +2194,7 @@ bool IsSaved(const Symbol &original) {
     return false;
   } else if (scopeKind == Scope::Kind::Module ||
       (scopeKind == Scope::Kind::MainProgram &&
-          (symbol.attrs().test(Attr::TARGET) || evaluate::IsCoarray(symbol)) &&
-          Fortran::evaluate::CanCUDASymbolHaveSaveAttr(symbol))) {
+          (symbol.attrs().test(Attr::TARGET) || evaluate::IsCoarray(symbol)))) {
     // 8.5.16p4
     // In main programs, implied SAVE matters only for pointer
     // initialization targets and coarrays.
@@ -2204,8 +2203,7 @@ bool IsSaved(const Symbol &original) {
       (features.IsEnabled(common::LanguageFeature::SaveMainProgram) ||
           (features.IsEnabled(
                common::LanguageFeature::SaveBigMainProgramVariables) &&
-              symbol.size() > 32)) &&
-      Fortran::evaluate::CanCUDASymbolHaveSaveAttr(symbol)) {
+              symbol.size() > 32))) {
     // With SaveBigMainProgramVariables, keeping all unsaved main program
     // variables of 32 bytes or less on the stack allows keeping numerical and
     // logical scalars, small scalar characters or derived, small arrays, and
@@ -2223,15 +2221,15 @@ bool IsSaved(const Symbol &original) {
   } else if (symbol.test(Symbol::Flag::InDataStmt)) {
     return true;
   } else if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()};
-             object && object->init()) {
+      object && object->init()) {
     return true;
   } else if (IsProcedurePointer(symbol) && symbol.has<ProcEntityDetails>() &&
       symbol.get<ProcEntityDetails>().init()) {
     return true;
   } else if (scope.hasSAVE()) {
     return true; // bare SAVE statement
-  } else if (const Symbol * block{FindCommonBlockContaining(symbol)};
-             block && block->attrs().test(Attr::SAVE)) {
+  } else if (const Symbol *block{FindCommonBlockContaining(symbol)};
+      block && block->attrs().test(Attr::SAVE)) {
     return true; // in COMMON with SAVE
   } else {
     return false;
@@ -2334,6 +2332,11 @@ bool IsBuiltinCPtr(const Symbol &symbol) {
   return false;
 }
 
+bool IsFromBuiltinModule(const Symbol &symbol) {
+  const Scope &scope{symbol.GetUltimate().owner()};
+  return IsSameModule(&scope, scope.context().GetBuiltinsScope());
+}
+
 bool IsIsoCType(const DerivedTypeSpec *derived) {
   return IsBuiltinDerivedType(derived, "c_ptr") ||
       IsBuiltinDerivedType(derived, "c_funptr");
diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 5b5f3c2cd0cf0..99dc8b1e5c676 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -299,13 +299,18 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &,
 
 // F2023 7.5.3.2
 static bool AreSameComponent(const semantics::Symbol &x,
-    const semantics::Symbol &y, bool ignoreSequence,
+    const semantics::Symbol &y, bool ignoreSequence, bool sameModuleName,
     SetOfDerivedTypePairs &inProgress) {
   if (x.attrs() != y.attrs()) {
     return false;
   }
-  if (x.attrs().test(semantics::Attr::PRIVATE)) {
-    return false;
+  if (x.attrs().test(semantics::Attr::PRIVATE) ||
+      y.attrs().test(semantics::Attr::PRIVATE)) {
+    if (!sameModuleName ||
+        x.attrs().test(semantics::Attr::PRIVATE) !=
+            y.attrs().test(semantics::Attr::PRIVATE)) {
+      return false;
+    }
   }
   if (x.size() && y.size()) {
     if (x.offset() != y.offset() || x.size() != y.size()) {
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
+  bool sameModuleName{false};
+  const semantics::Scope &xOwner{xSymbol.owner()};
+  const semantics::Scope &yOwner{ySymbol.owner()};
+  if (xOwner.IsModule() && yOwner.IsModule()) {
+    if (auto xModuleName{xOwner.GetName()}) {
+      if (auto yModuleName{yOwner.GetName()}) {
+        if (*xModuleName == *yModuleName) {
+          sameModuleName = true;
+        }
+      }
+    }
+  }
+  if (!sameModuleName && !ignoreSequence && !xDetails.sequence() &&
+      !xSymbol.attrs().test(semantics::Attr::BIND_C)) {
     // PGI does not enforce this requirement; all other Fortran
     // compilers do with a hard error when violations are caught.
     return false;
@@ -502,9 +518,10 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const auto xLookup{xSymbol.scope()->find(*xComponentName)};
     const auto yLookup{ySymbol.scope()->find(*yComponentName)};
     if (xLookup == xSymbol.scope()->end() ||
-        yLookup == ySymbol.scope()->end() ||
-        !AreSameComponent(
-            *xLookup->second, *yLookup->second, ignoreSequence, inProgress)) {
+        yLookup == ySymbol.scope()->end()) {
+      return false;
+    } else if (!AreSameComponent(*xLookup->second, *yLookup->second,
+                   ignoreSequence, sameModuleName, inProgress)) {
       return false;
     }
   }
@@ -576,17 +593,15 @@ static bool AreCompatibleTypes(const DynamicType &x, const DynamicType &y,
     const auto yLen{y.knownLength()};
     return x.kind() == y.kind() &&
         (ignoreLengths || !xLen || !yLen || *xLen == *yLen);
-  } else if (x.category() != TypeCategory::Derived) {
-    if (x.IsTypelessIntrinsicArgument()) {
-      return y.IsTypelessIntrinsicArgument();
-    } else {
-      return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
-    }
-  } else {
+  } else if (x.category() == TypeCategory::Derived) {
     const auto *xdt{GetDerivedTypeSpec(x)};
     const auto *ydt{GetDerivedTypeSpec(y)};
     return AreCompatibleDerivedTypes(
         xdt, ydt, x.IsPolymorphic(), ignoreTypeParameterValues, false);
+  } else if (x.IsTypelessIntrinsicArgument()) {
+    return y.IsTypelessIntrinsicArgument();
+  } else {
+    return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
   }
 }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 766131fd69d85..e5c6b1d5c61ad 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -14,6 +14,7 @@
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Frontend/TargetOptions.h"
+#include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Support/Fortran-features.h"
 #include "flang/Support/OpenMP-features.h"
@@ -290,13 +291,14 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
 
   opts.AliasAnalysis = opts.OptimizationLevel > 0;
 
-  // -mframe-pointer=none/non-leaf/all option.
+  // -mframe-pointer=none/non-leaf/reserved/all option.
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_mframe_pointer_EQ)) {
     std::optional<llvm::FramePointerKind> val =
         llvm::StringSwitch<std::optional<llvm::FramePointerKind>>(a->getValue())
             .Case("none", llvm::FramePointerKind::None)
             .Case("non-leaf", llvm::FramePointerKind::NonLeaf)
+            .Case("reserved", llvm::FramePointerKind::Reserved)
             .Case("all", llvm::FramePointerKind::All)
             .Default(std::nullopt);
 
@@ -1247,7 +1249,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())
@@ -1792,6 +1794,7 @@ void CompilerInvocation::setLoweringOptions() {
   // Lower TRANSPOSE as a runtime call under -O0.
   loweringOpts.setOptimizeTranspose(codegenOpts.OptimizationLevel > 0);
   loweringOpts.setUnderscoring(codegenOpts.Underscoring);
+  loweringOpts.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
 
   const Fortran::common::LangOptions &langOptions = getLangOpts();
   loweringOpts.setIntegerWrapAround(langOptions.getSignedOverflowBehavior() ==
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index d178fd6a9395d..5d788fe5619a2 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -171,7 +171,10 @@ bool FrontendAction::runParse(bool emitMessages) {
   if (emitMessages) {
     // Report any non-fatal diagnostics from getParsing now rather than
     // combining them with messages from semantics.
-    ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
+    const common::LanguageFeatureControl &features{
+        ci.getInvocation().getFortranOpts().features};
+    ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources(),
+                                    /*echoSourceLine=*/true, &features);
   }
   return true;
 }
@@ -223,6 +226,8 @@ bool FrontendAction::generateRtTypeTables() {
 
 template <unsigned N>
 bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
+  const common::LanguageFeatureControl &features{
+      instance->getInvocation().getFortranOpts().features};
   if (!instance->getParsing().messages().empty() &&
       (instance->getInvocation().getWarnAsErr() ||
        instance->getParsing().messages().AnyFatalError())) {
@@ -230,7 +235,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(llvm::errs(),
-                                           instance->getAllCookedSources());
+                                           instance->getAllCookedSources(),
+                                           /*echoSourceLines=*/true, &features);
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -240,7 +246,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(llvm::errs(),
-                                           instance->getAllCookedSources());
+                                           instance->getAllCookedSources(),
+                                           /*echoSourceLine=*/true, &features);
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 7b640dd497af3..696473605a4e0 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -262,6 +262,7 @@ class TypeInfoConverter {
   }
 
   void createTypeInfo(Fortran::lower::AbstractConverter &converter) {
+    createTypeInfoForTypeDescriptorBuiltinType(converter);
     while (!registeredTypeInfoA.empty()) {
       currentTypeInfoStack = &registeredTypeInfoB;
       for (const TypeInfo &info : registeredTypeInfoA)
@@ -277,10 +278,22 @@ class TypeInfoConverter {
 private:
   void createTypeInfoOpAndGlobal(Fortran::lower::AbstractConverter &converter,
                                  const TypeInfo &info) {
-    Fortran::lower::createRuntimeTypeInfoGlobal(converter, info.symbol.get());
+    if (!converter.getLoweringOptions().getSkipExternalRttiDefinition())
+      Fortran::lower::createRuntimeTypeInfoGlobal(converter, info.symbol.get());
     createTypeInfoOp(converter, info);
   }
 
+  void createTypeInfoForTypeDescriptorBuiltinType(
+      Fortran::lower::AbstractConverter &converter) {
+    if (registeredTypeInfoA.empty())
+      return;
+    auto builtinTypeInfoType = llvm::cast<fir::RecordType>(
+        converter.genType(registeredTypeInfoA[0].symbol.get()));
+    converter.getFirOpBuilder().createTypeInfoOp(
+        registeredTypeInfoA[0].loc, builtinTypeInfoType,
+        /*parentType=*/fir::RecordType{});
+  }
+
   void createTypeInfoOp(Fortran::lower::AbstractConverter &converter,
                         const TypeInfo &info) {
     fir::RecordType parentType{};
@@ -4873,7 +4886,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Location loc = getCurrentLocation();
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
-    bool isInDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
+    bool isInDeviceContext = cuf::isCUDADeviceContext(
+        builder.getRegion(),
+        getFoldingContext().languageFeatures().IsEnabled(
+            Fortran::common::LanguageFeature::DoConcurrentOffload));
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 9c5db2b126510..8049cdf333173 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -23,6 +23,7 @@ add_flang_library(FortranLower
   LoweringOptions.cpp
   Mangler.cpp
   OpenACC.cpp
+  OpenMP/Atomic.cpp
   OpenMP/ClauseProcessor.cpp
   OpenMP/Clauses.cpp
   OpenMP/DataSharingProcessor.cpp
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 864499e6c3431..6ed15df0de754 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -20,6 +20,7 @@
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
+#include "flang/Optimizer/Builder/CUFCommon.h"
 #include "flang/Optimizer/Builder/Character.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
@@ -545,7 +546,8 @@ Fortran::lower::genCallOpAndResult(
       caller.getProcedureAttrs(builder.getContext());
 
   if (converter.getLoweringOptions().getCUDARuntimeCheck()) {
-    if (caller.getCallDescription().chevrons().empty()) {
+    if (caller.getCallDescription().chevrons().empty() &&
+        !cuf::isCUDADeviceContext(builder.getRegion())) {
       for (auto [oper, arg] :
            llvm::zip(operands, caller.getPassedArguments())) {
         if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(oper.getType())) {
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 49e6ea02d51a7..44f534e7d569a 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -647,13 +647,19 @@ fir::GlobalOp Fortran::lower::defineGlobal(
 
 /// Return linkage attribute for \p var.
 static mlir::StringAttr
-getLinkageAttribute(fir::FirOpBuilder &builder,
+getLinkageAttribute(Fortran::lower::AbstractConverter &converter,
                     const Fortran::lower::pft::Variable &var) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   // Runtime type info for a same derived type is identical in each compilation
   // unit. It desired to avoid having to link against module that only define a
   // type. Therefore the runtime type info is generated everywhere it is needed
-  // with `linkonce_odr` LLVM linkage.
-  if (var.isRuntimeTypeInfoData())
+  // with `linkonce_odr` LLVM linkage (unless the skipExternalRttiDefinition
+  // option is set, in which case one will need to link against objects of
+  // modules defining types). Builtin objects rtti is always generated because
+  // the builtin module is currently not compiled or part of the runtime.
+  if (var.isRuntimeTypeInfoData() &&
+      (!converter.getLoweringOptions().getSkipExternalRttiDefinition() ||
+       Fortran::semantics::IsFromBuiltinModule(var.getSymbol())))
     return builder.createLinkOnceODRLinkage();
   if (var.isModuleOrSubmoduleVariable())
     return {}; // external linkage
@@ -673,7 +679,7 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   std::string globalName = converter.mangleName(sym);
   mlir::Location loc = genLocation(converter, sym);
-  mlir::StringAttr linkage = getLinkageAttribute(builder, var);
+  mlir::StringAttr linkage = getLinkageAttribute(converter, var);
   fir::GlobalOp global;
   if (var.isModuleOrSubmoduleVariable()) {
     // A non-intrinsic module global is defined when lowering the module.
@@ -696,6 +702,29 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
   mapSymbolAttributes(converter, var, symMap, stmtCtx, cast);
 }
 
+bool needCUDAAlloc(const Fortran::semantics::Symbol &sym) {
+  if (Fortran::semantics::IsDummy(sym))
+    return false;
+  if (const auto *details{
+          sym.GetUltimate()
+              .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
+    if (details->cudaDataAttr() &&
+        (*details->cudaDataAttr() == Fortran::common::CUDADataAttr::Device ||
+         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Managed ||
+         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Unified ||
+         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Shared ||
+         *details->cudaDataAttr() == Fortran::common::CUDADataAttr::Pinned))
+      return true;
+    const Fortran::semantics::DeclTypeSpec *type{details->type()};
+    const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
+                                                            : nullptr};
+    if (derived)
+      if (FindCUDADeviceAllocatableUltimateComponent(*derived))
+        return true;
+  }
+  return false;
+}
+
 //===----------------------------------------------------------------===//
 // Local variables instantiation (not for alias)
 //===----------------------------------------------------------------===//
@@ -726,7 +755,7 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
   if (ultimateSymbol.test(Fortran::semantics::Symbol::Flag::CrayPointee))
     return builder.create<fir::ZeroOp>(loc, fir::ReferenceType::get(ty));
 
-  if (Fortran::semantics::NeedCUDAAlloc(ultimateSymbol)) {
+  if (needCUDAAlloc(ultimateSymbol)) {
     cuf::DataAttributeAttr dataAttr =
         Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
                                                         ultimateSymbol);
@@ -1081,7 +1110,7 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
     Fortran::lower::defaultInitializeAtRuntime(converter, var.getSymbol(),
                                                symMap);
   auto *builder = &converter.getFirOpBuilder();
-  if (Fortran::semantics::NeedCUDAAlloc(var.getSymbol()) &&
+  if (needCUDAAlloc(var.getSymbol()) &&
       !cuf::isCUDADeviceContext(builder->getRegion())) {
     cuf::DataAttributeAttr dataAttr =
         Fortran::lower::translateSymbolCUFDataAttribute(builder->getContext(),
@@ -1265,7 +1294,7 @@ instantiateAggregateStore(Fortran::lower::AbstractConverter &converter,
   if (var.isGlobal()) {
     fir::GlobalOp global;
     auto &aggregate = var.getAggregateStore();
-    mlir::StringAttr linkage = getLinkageAttribute(builder, var);
+    mlir::StringAttr linkage = getLinkageAttribute(converter, var);
     if (var.isModuleOrSubmoduleVariable()) {
       // A module global was or will be defined when lowering the module. Emit
       // only a declaration if the global does not exist at that point.
@@ -2470,8 +2499,7 @@ void Fortran::lower::defineModuleVariable(
     AbstractConverter &converter, const Fortran::lower::pft::Variable &var) {
   // Use empty linkage for module variables, which makes them available
   // for use in another unit.
-  mlir::StringAttr linkage =
-      getLinkageAttribute(converter.getFirOpBuilder(), var);
+  mlir::StringAttr linkage = getLinkageAttribute(converter, var);
   if (!var.isGlobal())
     fir::emitFatalError(converter.getCurrentLocation(),
                         "attempting to lower module variable as local");
@@ -2606,10 +2634,9 @@ void Fortran::lower::createIntrinsicModuleGlobal(
 void Fortran::lower::createRuntimeTypeInfoGlobal(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::semantics::Symbol &typeInfoSym) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   std::string globalName = converter.mangleName(typeInfoSym);
   auto var = Fortran::lower::pft::Variable(typeInfoSym, /*global=*/true);
-  mlir::StringAttr linkage = getLinkageAttribute(builder, var);
+  mlir::StringAttr linkage = getLinkageAttribute(converter, var);
   defineGlobal(converter, var, globalName, linkage);
 }
 
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 3ef3330cba2d6..e56d7f7ed9b6f 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4561,7 +4561,8 @@ void createOpenACCRoutineConstruct(
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes) {
 
   for (auto routineOp : mod.getOps<mlir::acc::RoutineOp>()) {
-    if (routineOp.getFuncName().str().compare(funcName) == 0) {
+    if (routineOp.getFuncName().getLeafReference().str().compare(funcName) ==
+        0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
       if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
@@ -4579,7 +4580,9 @@ void createOpenACCRoutineConstruct(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   modBuilder.create<mlir::acc::RoutineOp>(
-      loc, routineOpStr, funcName, getArrayAttrOrNull(builder, bindNames),
+      loc, routineOpStr,
+      mlir::SymbolRefAttr::get(builder.getContext(), funcName),
+      getArrayAttrOrNull(builder, bindNames),
       getArrayAttrOrNull(builder, bindNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
new file mode 100644
index 0000000000000..2ab91b239a3cc
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -0,0 +1,783 @@
+//===-- Atomic.cpp -- Lowering of atomic constructs -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Atomic.h"
+#include "Clauses.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/fold.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Lower/StatementContext.h"
+#include "flang/Lower/SymbolMap.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <optional>
+#include <string>
+#include <type_traits>
+#include <variant>
+#include <vector>
+
+static llvm::cl::opt<bool> DumpAtomicAnalysis("fdebug-dump-atomic-analysis");
+
+using namespace Fortran;
+
+// Don't import the entire Fortran::lower.
+namespace omp {
+using namespace Fortran::lower::omp;
+}
+
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      (void)Derived;
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
+  static constexpr auto Integer = common::TypeCategory::Integer;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  static constexpr auto Real = common::TypeCategory::Real;
+  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
+};
+
+template <typename T, typename U = std::remove_const_t<T>>
+U AsRvalue(T &t) {
+  U copy{t};
+  return std::move(copy);
+}
+
+template <typename T>
+T &&AsRvalue(T &&t) {
+  return std::move(t);
+}
+
+struct ArgumentReplacer
+    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
+  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
+  using Result = bool;
+
+  Result Default() const { return false; }
+
+  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
+      : Base(*this), args_(std::move(newArgs)) {}
+
+  using Base::operator();
+
+  template <typename T>
+  Result operator()(const evaluate::FunctionRef<T> &x) {
+    assert(!done_);
+    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
+    mut.arguments() = args_;
+    done_ = true;
+    return true;
+  }
+
+  Result Combine(Result &&a, Result &&b) { return a || b; }
+
+private:
+  bool done_{false};
+  evaluate::ActualArguments &&args_;
+};
+} // namespace
+
+[[maybe_unused]] static void
+dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
+  auto whatStr = [](int k) {
+    std::string txt = "?";
+    switch (k & parser::OpenMPAtomicConstruct::Analysis::Action) {
+    case parser::OpenMPAtomicConstruct::Analysis::None:
+      txt = "None";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Read:
+      txt = "Read";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Write:
+      txt = "Write";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Update:
+      txt = "Update";
+      break;
+    }
+    switch (k & parser::OpenMPAtomicConstruct::Analysis::Condition) {
+    case parser::OpenMPAtomicConstruct::Analysis::IfTrue:
+      txt += " | IfTrue";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::IfFalse:
+      txt += " | IfFalse";
+      break;
+    }
+    return txt;
+  };
+
+  auto exprStr = [&](const parser::TypedExpr &expr) {
+    if (auto *maybe = expr.get()) {
+      if (maybe->v)
+        return maybe->v->AsFortran();
+    }
+    return "<null>"s;
+  };
+  auto assignStr = [&](const parser::AssignmentStmt::TypedAssignment &assign) {
+    if (auto *maybe = assign.get(); maybe && maybe->v) {
+      std::string str;
+      llvm::raw_string_ostream os(str);
+      maybe->v->AsFortran(os);
+      return str;
+    }
+    return "<null>"s;
+  };
+
+  const semantics::SomeExpr &atom = *analysis.atom.get()->v;
+
+  llvm::errs() << "Analysis {\n";
+  llvm::errs() << "  atom: " << atom.AsFortran() << "\n";
+  llvm::errs() << "  cond: " << exprStr(analysis.cond) << "\n";
+  llvm::errs() << "  op0 {\n";
+  llvm::errs() << "    what: " << whatStr(analysis.op0.what) << "\n";
+  llvm::errs() << "    assign: " << assignStr(analysis.op0.assign) << "\n";
+  llvm::errs() << "  }\n";
+  llvm::errs() << "  op1 {\n";
+  llvm::errs() << "    what: " << whatStr(analysis.op1.what) << "\n";
+  llvm::errs() << "    assign: " << assignStr(analysis.op1.assign) << "\n";
+  llvm::errs() << "  }\n";
+  llvm::errs() << "}\n";
+}
+
+static bool isPointerAssignment(const evaluate::Assignment &assign) {
+  return common::visit(
+      common::visitors{
+          [](const evaluate::Assignment::BoundsSpec &) { return true; },
+          [](const evaluate::Assignment::BoundsRemapping &) { return true; },
+          [](const auto &) { return false; },
+      },
+      assign.u);
+}
+
+static fir::FirOpBuilder::InsertPoint
+getInsertionPointBefore(mlir::Operation *op) {
+  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
+                                        mlir::Block::iterator(op));
+}
+
+static fir::FirOpBuilder::InsertPoint
+getInsertionPointAfter(mlir::Operation *op) {
+  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
+                                        ++mlir::Block::iterator(op));
+}
+
+static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
+                                       const omp::List<omp::Clause> &clauses) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  for (const omp::Clause &clause : clauses) {
+    if (clause.id != llvm::omp::Clause::OMPC_hint)
+      continue;
+    auto &hint = std::get<omp::clause::Hint>(clause.u);
+    auto maybeVal = evaluate::ToInt64(hint.v);
+    CHECK(maybeVal);
+    return builder.getI64IntegerAttr(*maybeVal);
+  }
+  return nullptr;
+}
+
+static mlir::omp::ClauseMemoryOrderKind
+getMemoryOrderKind(common::OmpMemoryOrderType kind) {
+  switch (kind) {
+  case common::OmpMemoryOrderType::Acq_Rel:
+    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+  case common::OmpMemoryOrderType::Acquire:
+    return mlir::omp::ClauseMemoryOrderKind::Acquire;
+  case common::OmpMemoryOrderType::Relaxed:
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  case common::OmpMemoryOrderType::Release:
+    return mlir::omp::ClauseMemoryOrderKind::Release;
+  case common::OmpMemoryOrderType::Seq_Cst:
+    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+  }
+  llvm_unreachable("Unexpected kind");
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getMemoryOrderKind(llvm::omp::Clause clauseId) {
+  switch (clauseId) {
+  case llvm::omp::Clause::OMPC_acq_rel:
+    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+  case llvm::omp::Clause::OMPC_acquire:
+    return mlir::omp::ClauseMemoryOrderKind::Acquire;
+  case llvm::omp::Clause::OMPC_relaxed:
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  case llvm::omp::Clause::OMPC_release:
+    return mlir::omp::ClauseMemoryOrderKind::Release;
+  case llvm::omp::Clause::OMPC_seq_cst:
+    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+  default:
+    return std::nullopt;
+  }
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getMemoryOrderFromRequires(const semantics::Scope &scope) {
+  // The REQUIRES construct is only allowed in the main program scope
+  // and module scope, but seems like we also accept it in a subprogram
+  // scope.
+  // For safety, traverse all enclosing scopes and check if their symbol
+  // contains REQUIRES.
+  for (const auto *sc{&scope}; sc->kind() != semantics::Scope::Kind::Global;
+       sc = &sc->parent()) {
+    const semantics::Symbol *sym = sc->symbol();
+    if (!sym)
+      continue;
+
+    const common::OmpMemoryOrderType *admo = common::visit(
+        [](auto &&s) {
+          using WithOmpDeclarative = semantics::WithOmpDeclarative;
+          if constexpr (std::is_convertible_v<decltype(s),
+                                              const WithOmpDeclarative &>) {
+            return s.ompAtomicDefaultMemOrder();
+          }
+          return static_cast<const common::OmpMemoryOrderType *>(nullptr);
+        },
+        sym->details());
+    if (admo)
+      return getMemoryOrderKind(*admo);
+  }
+
+  return std::nullopt;
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getDefaultAtomicMemOrder(semantics::SemanticsContext &semaCtx) {
+  unsigned version = semaCtx.langOptions().OpenMPVersion;
+  if (version > 50)
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  return std::nullopt;
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getAtomicMemoryOrder(semantics::SemanticsContext &semaCtx,
+                     const omp::List<omp::Clause> &clauses,
+                     const semantics::Scope &scope) {
+  for (const omp::Clause &clause : clauses) {
+    if (auto maybeKind = getMemoryOrderKind(clause.id))
+      return *maybeKind;
+  }
+
+  if (auto maybeKind = getMemoryOrderFromRequires(scope))
+    return *maybeKind;
+
+  return getDefaultAtomicMemOrder(semaCtx);
+}
+
+static mlir::omp::ClauseMemoryOrderKindAttr
+makeMemOrderAttr(lower::AbstractConverter &converter,
+                 std::optional<mlir::omp::ClauseMemoryOrderKind> maybeKind) {
+  if (maybeKind) {
+    return mlir::omp::ClauseMemoryOrderKindAttr::get(
+        converter.getFirOpBuilder().getContext(), *maybeKind);
+  }
+  return nullptr;
+}
+
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atomArg.
+  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
+  // call = min/max(a_t, tmp).
+  // Return "call".
+
+  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
+  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
+  // argument to a non-optional position. This could happen if a_t is at
+  // position 0 or 1.
+  if (args.size() <= 2)
+    return orig;
+
+  evaluate::ActualArguments nonAtoms;
+
+  auto AsActual = [](const semantics::SomeExpr &x) {
+    semantics::SomeExpr copy = x;
+    return evaluate::ActualArgument(std::move(copy));
+  };
+  // Semantic checks guarantee that the "atom" shows exactly once in the
+  // argument list (with potential conversions around it).
+  // For the first two (non-optional) arguments, if "atom" is among them,
+  // replace it with another occurrence of the other non-optional argument.
+  if (atomArg == &args[0]) {
+    // (atom, x, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[1]));
+    nonAtoms.push_back(AsActual(args[1]));
+  } else if (atomArg == &args[1]) {
+    // (x, atom, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[0]));
+  } else {
+    // (x, y, z...) -> unchanged
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[1]));
+  }
+
+  // The rest of arguments are optional, so we can just skip "atom".
+  for (size_t i = 2, e = args.size(); i != e; ++i) {
+    if (atomArg != &args[i])
+      nonAtoms.push_back(AsActual(args[i]));
+  }
+
+  // The type of the intermediate min/max is the same as the type of its
+  // arguments, which may be different from the type of the original
+  // expression. The original expression may have additional coverts.
+  auto tmp =
+      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
+  semantics::SomeExpr call = orig;
+  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
+  return call;
+}
+
+static mlir::Operation * //
+genAtomicRead(lower::AbstractConverter &converter,
+              semantics::SemanticsContext &semaCtx, mlir::Location loc,
+              lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+              const semantics::SomeExpr &atom,
+              const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+              std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+              fir::FirOpBuilder::InsertPoint preAt,
+              fir::FirOpBuilder::InsertPoint atomicAt,
+              fir::FirOpBuilder::InsertPoint postAt) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  // If the atomic clause is read then the memory-order clause must
+  // not be release.
+  if (memOrder) {
+    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Release) {
+      // Reset it back to the default.
+      memOrder = getDefaultAtomicMemOrder(semaCtx);
+    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
+      // The MLIR verifier doesn't like acq_rel either.
+      memOrder = mlir::omp::ClauseMemoryOrderKind::Acquire;
+    }
+  }
+
+  mlir::Value storeAddr =
+      fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+  mlir::Type storeType = fir::unwrapRefType(storeAddr.getType());
+
+  mlir::Value toAddr = [&]() {
+    if (atomType == storeType)
+      return storeAddr;
+    return builder.createTemporary(loc, atomType, ".tmp.atomval");
+  }();
+
+  builder.restoreInsertionPoint(atomicAt);
+  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
+      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
+      makeMemOrderAttr(converter, memOrder));
+
+  if (atomType != storeType) {
+    lower::ExprToValueMap overrides;
+    // The READ operation could be a part of UPDATE CAPTURE, so make sure
+    // we don't emit extra code into the body of the atomic op.
+    builder.restoreInsertionPoint(postAt);
+    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
+    overrides.try_emplace(&atom, load);
+
+    converter.overrideExprValues(&overrides);
+    mlir::Value value =
+        fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+    converter.resetExprOverrides();
+
+    builder.create<fir::StoreOp>(loc, value, storeAddr);
+  }
+  return op;
+}
+
+static mlir::Operation * //
+genAtomicWrite(lower::AbstractConverter &converter,
+               semantics::SemanticsContext &semaCtx, mlir::Location loc,
+               lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+               const semantics::SomeExpr &atom,
+               const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+               std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+               fir::FirOpBuilder::InsertPoint preAt,
+               fir::FirOpBuilder::InsertPoint atomicAt,
+               fir::FirOpBuilder::InsertPoint postAt) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  // If the atomic clause is write then the memory-order clause must
+  // not be acquire.
+  if (memOrder) {
+    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acquire) {
+      // Reset it back to the default.
+      memOrder = getDefaultAtomicMemOrder(semaCtx);
+    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
+      // The MLIR verifier doesn't like acq_rel either.
+      memOrder = mlir::omp::ClauseMemoryOrderKind::Release;
+    }
+  }
+
+  mlir::Value value =
+      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+  mlir::Value converted = builder.createConvert(loc, atomType, value);
+
+  builder.restoreInsertionPoint(atomicAt);
+  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
+      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
+  return op;
+}
+
+static mlir::Operation *
+genAtomicUpdate(lower::AbstractConverter &converter,
+                semantics::SemanticsContext &semaCtx, mlir::Location loc,
+                lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+                const semantics::SomeExpr &atom,
+                const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+                std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+                fir::FirOpBuilder::InsertPoint preAt,
+                fir::FirOpBuilder::InsertPoint atomicAt,
+                fir::FirOpBuilder::InsertPoint postAt) {
+  lower::ExprToValueMap overrides;
+  lower::StatementContext naCtx;
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+
+  // This must exist by now.
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
+  assert(!args.empty() && "Update operation without arguments");
+
+  // Pass args as an argument to avoid capturing a structured binding.
+  const semantics::SomeExpr *atomArg = [&](auto &args) {
+    for (const semantics::SomeExpr &e : args) {
+      if (evaluate::IsSameOrConvertOf(e, atom))
+        return &e;
+    }
+    llvm_unreachable("Atomic variable not in argument list");
+  }(args);
+
+  if (opcode == evaluate::operation::Operator::Min ||
+      opcode == evaluate::operation::Operator::Max) {
+    // Min and max operations are expanded inline, so reduce them to
+    // operations with exactly two (non-optional) arguments.
+    rhs = genReducedMinMax(rhs, atomArg, args);
+    input = *evaluate::GetConvertInput(rhs);
+    std::tie(opcode, args) = evaluate::GetTopLevelOperation(input);
+    atomArg = nullptr; // No longer valid.
+  }
+  for (auto &arg : args) {
+    if (!evaluate::IsSameOrConvertOf(arg, atom)) {
+      mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
+      overrides.try_emplace(&arg, val);
+    }
+  }
+
+  builder.restoreInsertionPoint(atomicAt);
+  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
+      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+
+  mlir::Region &region = updateOp->getRegion(0);
+  mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
+  mlir::Value localAtom = fir::getBase(block->getArgument(0));
+  overrides.try_emplace(&atom, localAtom);
+
+  converter.overrideExprValues(&overrides);
+  mlir::Value updated =
+      fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
+  mlir::Value converted = builder.createConvert(loc, atomType, updated);
+  builder.create<mlir::omp::YieldOp>(loc, converted);
+  converter.resetExprOverrides();
+
+  builder.restoreInsertionPoint(postAt); // For naCtx cleanups
+  return updateOp;
+}
+
+static mlir::Operation *
+genAtomicOperation(lower::AbstractConverter &converter,
+                   semantics::SemanticsContext &semaCtx, mlir::Location loc,
+                   lower::StatementContext &stmtCtx, int action,
+                   mlir::Value atomAddr, const semantics::SomeExpr &atom,
+                   const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+                   std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+                   fir::FirOpBuilder::InsertPoint preAt,
+                   fir::FirOpBuilder::InsertPoint atomicAt,
+                   fir::FirOpBuilder::InsertPoint postAt) {
+  if (isPointerAssignment(assign)) {
+    TODO(loc, "Code generation for pointer assignment is not implemented yet");
+  }
+
+  // This function and the functions called here do not preserve the
+  // builder's insertion point, or set it to anything specific.
+  switch (action) {
+  case parser::OpenMPAtomicConstruct::Analysis::Read:
+    return genAtomicRead(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                         assign, hint, memOrder, preAt, atomicAt, postAt);
+  case parser::OpenMPAtomicConstruct::Analysis::Write:
+    return genAtomicWrite(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                          assign, hint, memOrder, preAt, atomicAt, postAt);
+  case parser::OpenMPAtomicConstruct::Analysis::Update:
+    return genAtomicUpdate(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                           assign, hint, memOrder, preAt, atomicAt, postAt);
+  default:
+    return nullptr;
+  }
+}
+
+void Fortran::lower::omp::lowerAtomic(
+    AbstractConverter &converter, SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, pft::Evaluation &eval,
+    const parser::OpenMPAtomicConstruct &construct) {
+  auto get = [](auto &&typedWrapper) -> decltype(&*typedWrapper.get()->v) {
+    if (auto *maybe = typedWrapper.get(); maybe && maybe->v) {
+      return &*maybe->v;
+    } else {
+      return nullptr;
+    }
+  };
+
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  auto &dirSpec = std::get<parser::OmpDirectiveSpecification>(construct.t);
+  omp::List<omp::Clause> clauses = makeClauses(dirSpec.Clauses(), semaCtx);
+  lower::StatementContext stmtCtx;
+
+  const parser::OpenMPAtomicConstruct::Analysis &analysis = construct.analysis;
+  if (DumpAtomicAnalysis)
+    dumpAtomicAnalysis(analysis);
+
+  const semantics::SomeExpr &atom = *get(analysis.atom);
+  mlir::Location loc = converter.genLocation(construct.source);
+  mlir::Value atomAddr =
+      fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
+  mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
+  std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder =
+      getAtomicMemoryOrder(semaCtx, clauses,
+                           semaCtx.FindScope(construct.source));
+
+  if (auto *cond = get(analysis.cond)) {
+    (void)cond;
+    TODO(loc, "OpenMP ATOMIC COMPARE");
+  } else {
+    int action0 = analysis.op0.what & analysis.Action;
+    int action1 = analysis.op1.what & analysis.Action;
+    mlir::Operation *captureOp = nullptr;
+    fir::FirOpBuilder::InsertPoint preAt = builder.saveInsertionPoint();
+    fir::FirOpBuilder::InsertPoint atomicAt, postAt;
+
+    if (construct.IsCapture()) {
+      // Capturing operation.
+      assert(action0 != analysis.None && action1 != analysis.None &&
+             "Expexcing two actions");
+      (void)action0;
+      (void)action1;
+      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
+          loc, hint, makeMemOrderAttr(converter, memOrder));
+      // Set the non-atomic insertion point to before the atomic.capture.
+      preAt = getInsertionPointBefore(captureOp);
+
+      mlir::Block *block = builder.createBlock(&captureOp->getRegion(0));
+      builder.setInsertionPointToEnd(block);
+      // Set the atomic insertion point to before the terminator inside
+      // atomic.capture.
+      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
+      atomicAt = getInsertionPointBefore(term);
+      postAt = getInsertionPointAfter(captureOp);
+      hint = nullptr;
+      memOrder = std::nullopt;
+    } else {
+      // Non-capturing operation.
+      assert(action0 != analysis.None && action1 == analysis.None &&
+             "Expexcing single action");
+      assert(!(analysis.op0.what & analysis.Condition));
+      postAt = atomicAt = preAt;
+    }
+
+    // The builder's insertion point needs to be specifically set before
+    // each call to `genAtomicOperation`.
+    mlir::Operation *firstOp = genAtomicOperation(
+        converter, semaCtx, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
+        *get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
+    assert(firstOp && "Should have created an atomic operation");
+    atomicAt = getInsertionPointAfter(firstOp);
+
+    mlir::Operation *secondOp = nullptr;
+    if (analysis.op1.what != analysis.None) {
+      secondOp = genAtomicOperation(
+          converter, semaCtx, loc, stmtCtx, analysis.op1.what, atomAddr, atom,
+          *get(analysis.op1.assign), hint, memOrder, preAt, atomicAt, postAt);
+    }
+
+    if (construct.IsCapture()) {
+      // If this is a capture operation, the first/second ops will be inside
+      // of it. Set the insertion point to past the capture op itself.
+      builder.restoreInsertionPoint(postAt);
+    } else {
+      if (secondOp) {
+        builder.setInsertionPointAfter(secondOp);
+      } else {
+        builder.setInsertionPointAfter(firstOp);
+      }
+    }
+  }
+}
diff --git a/flang/lib/Lower/OpenMP/Atomic.h b/flang/lib/Lower/OpenMP/Atomic.h
new file mode 100644
index 0000000000000..96db4d7e90c8d
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Atomic.h
@@ -0,0 +1,36 @@
+//===-- Atomic.h -- Lowering of atomic constructs -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_LOWER_OPENMP_ATOMIC_H
+#define FORTRAN_LOWER_OPENMP_ATOMIC_H
+
+namespace Fortran {
+namespace lower {
+class AbstractConverter;
+class SymMap;
+
+namespace pft {
+struct Evaluation;
+}
+} // namespace lower
+
+namespace parser {
+struct OpenMPAtomicConstruct;
+}
+
+namespace semantics {
+class SemanticsContext;
+}
+} // namespace Fortran
+
+namespace Fortran::lower::omp {
+void lowerAtomic(AbstractConverter &converter, SymMap &symTable,
+                 semantics::SemanticsContext &semaCtx, pft::Evaluation &eval,
+                 const parser::OpenMPAtomicConstruct &construct);
+}
+
+#endif // FORTRAN_LOWER_OPENMP_ATOMIC_H
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index e4c15e8ba254c..0a56e888ac44b 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Lower/OpenMP.h"
 
+#include "Atomic.h"
 #include "ClauseProcessor.h"
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
@@ -42,13 +43,10 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
-#include "llvm/Support/CommandLine.h"
 
 using namespace Fortran::lower::omp;
 using namespace Fortran::common::openmp;
 
-static llvm::cl::opt<bool> DumpAtomicAnalysis("fdebug-dump-atomic-analysis");
-
 //===----------------------------------------------------------------------===//
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
@@ -1159,16 +1157,6 @@ markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
-static bool isPointerAssignment(const evaluate::Assignment &assign) {
-  return common::visit(
-      common::visitors{
-          [](const evaluate::Assignment::BoundsSpec &) { return true; },
-          [](const evaluate::Assignment::BoundsRemapping &) { return true; },
-          [](const auto &) { return false; },
-      },
-      assign.u);
-}
-
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -2732,308 +2720,6 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
-//===----------------------------------------------------------------------===//
-// Code generation for atomic operations
-//===----------------------------------------------------------------------===//
-static fir::FirOpBuilder::InsertPoint
-getInsertionPointBefore(mlir::Operation *op) {
-  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
-                                        mlir::Block::iterator(op));
-}
-
-static fir::FirOpBuilder::InsertPoint
-getInsertionPointAfter(mlir::Operation *op) {
-  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
-                                        ++mlir::Block::iterator(op));
-}
-
-static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
-                                       const List<Clause> &clauses) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  for (const Clause &clause : clauses) {
-    if (clause.id != llvm::omp::Clause::OMPC_hint)
-      continue;
-    auto &hint = std::get<clause::Hint>(clause.u);
-    auto maybeVal = evaluate::ToInt64(hint.v);
-    CHECK(maybeVal);
-    return builder.getI64IntegerAttr(*maybeVal);
-  }
-  return nullptr;
-}
-
-static mlir::omp::ClauseMemoryOrderKind
-getMemoryOrderKind(common::OmpMemoryOrderType kind) {
-  switch (kind) {
-  case common::OmpMemoryOrderType::Acq_Rel:
-    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
-  case common::OmpMemoryOrderType::Acquire:
-    return mlir::omp::ClauseMemoryOrderKind::Acquire;
-  case common::OmpMemoryOrderType::Relaxed:
-    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-  case common::OmpMemoryOrderType::Release:
-    return mlir::omp::ClauseMemoryOrderKind::Release;
-  case common::OmpMemoryOrderType::Seq_Cst:
-    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
-  }
-  llvm_unreachable("Unexpected kind");
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getMemoryOrderKind(llvm::omp::Clause clauseId) {
-  switch (clauseId) {
-  case llvm::omp::Clause::OMPC_acq_rel:
-    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
-  case llvm::omp::Clause::OMPC_acquire:
-    return mlir::omp::ClauseMemoryOrderKind::Acquire;
-  case llvm::omp::Clause::OMPC_relaxed:
-    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-  case llvm::omp::Clause::OMPC_release:
-    return mlir::omp::ClauseMemoryOrderKind::Release;
-  case llvm::omp::Clause::OMPC_seq_cst:
-    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
-  default:
-    return std::nullopt;
-  }
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getMemoryOrderFromRequires(const semantics::Scope &scope) {
-  // The REQUIRES construct is only allowed in the main program scope
-  // and module scope, but seems like we also accept it in a subprogram
-  // scope.
-  // For safety, traverse all enclosing scopes and check if their symbol
-  // contains REQUIRES.
-  for (const auto *sc{&scope}; sc->kind() != semantics::Scope::Kind::Global;
-       sc = &sc->parent()) {
-    const semantics::Symbol *sym = sc->symbol();
-    if (!sym)
-      continue;
-
-    const common::OmpMemoryOrderType *admo = common::visit(
-        [](auto &&s) {
-          using WithOmpDeclarative = semantics::WithOmpDeclarative;
-          if constexpr (std::is_convertible_v<decltype(s),
-                                              const WithOmpDeclarative &>) {
-            return s.ompAtomicDefaultMemOrder();
-          }
-          return static_cast<const common::OmpMemoryOrderType *>(nullptr);
-        },
-        sym->details());
-    if (admo)
-      return getMemoryOrderKind(*admo);
-  }
-
-  return std::nullopt;
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getDefaultAtomicMemOrder(semantics::SemanticsContext &semaCtx) {
-  unsigned version = semaCtx.langOptions().OpenMPVersion;
-  if (version > 50)
-    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-  return std::nullopt;
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getAtomicMemoryOrder(semantics::SemanticsContext &semaCtx,
-                     const List<Clause> &clauses,
-                     const semantics::Scope &scope) {
-  for (const Clause &clause : clauses) {
-    if (auto maybeKind = getMemoryOrderKind(clause.id))
-      return *maybeKind;
-  }
-
-  if (auto maybeKind = getMemoryOrderFromRequires(scope))
-    return *maybeKind;
-
-  return getDefaultAtomicMemOrder(semaCtx);
-}
-
-static mlir::omp::ClauseMemoryOrderKindAttr
-makeMemOrderAttr(lower::AbstractConverter &converter,
-                 std::optional<mlir::omp::ClauseMemoryOrderKind> maybeKind) {
-  if (maybeKind) {
-    return mlir::omp::ClauseMemoryOrderKindAttr::get(
-        converter.getFirOpBuilder().getContext(), *maybeKind);
-  }
-  return nullptr;
-}
-
-static mlir::Operation * //
-genAtomicRead(lower::AbstractConverter &converter,
-              semantics::SemanticsContext &semaCtx, mlir::Location loc,
-              lower::StatementContext &stmtCtx, mlir::Value atomAddr,
-              const semantics::SomeExpr &atom,
-              const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-              std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-              fir::FirOpBuilder::InsertPoint preAt,
-              fir::FirOpBuilder::InsertPoint atomicAt,
-              fir::FirOpBuilder::InsertPoint postAt) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.restoreInsertionPoint(preAt);
-
-  // If the atomic clause is read then the memory-order clause must
-  // not be release.
-  if (memOrder) {
-    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Release) {
-      // Reset it back to the default.
-      memOrder = getDefaultAtomicMemOrder(semaCtx);
-    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
-      // The MLIR verifier doesn't like acq_rel either.
-      memOrder = mlir::omp::ClauseMemoryOrderKind::Acquire;
-    }
-  }
-
-  mlir::Value storeAddr =
-      fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
-  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
-  mlir::Type storeType = fir::unwrapRefType(storeAddr.getType());
-
-  mlir::Value toAddr = [&]() {
-    if (atomType == storeType)
-      return storeAddr;
-    return builder.createTemporary(loc, atomType, ".tmp.atomval");
-  }();
-
-  builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
-      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
-      makeMemOrderAttr(converter, memOrder));
-
-  if (atomType != storeType) {
-    lower::ExprToValueMap overrides;
-    // The READ operation could be a part of UPDATE CAPTURE, so make sure
-    // we don't emit extra code into the body of the atomic op.
-    builder.restoreInsertionPoint(postAt);
-    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
-    overrides.try_emplace(&atom, load);
-
-    converter.overrideExprValues(&overrides);
-    mlir::Value value =
-        fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
-    converter.resetExprOverrides();
-
-    builder.create<fir::StoreOp>(loc, value, storeAddr);
-  }
-  return op;
-}
-
-static mlir::Operation * //
-genAtomicWrite(lower::AbstractConverter &converter,
-               semantics::SemanticsContext &semaCtx, mlir::Location loc,
-               lower::StatementContext &stmtCtx, mlir::Value atomAddr,
-               const semantics::SomeExpr &atom,
-               const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-               std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-               fir::FirOpBuilder::InsertPoint preAt,
-               fir::FirOpBuilder::InsertPoint atomicAt,
-               fir::FirOpBuilder::InsertPoint postAt) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.restoreInsertionPoint(preAt);
-
-  // If the atomic clause is write then the memory-order clause must
-  // not be acquire.
-  if (memOrder) {
-    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acquire) {
-      // Reset it back to the default.
-      memOrder = getDefaultAtomicMemOrder(semaCtx);
-    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
-      // The MLIR verifier doesn't like acq_rel either.
-      memOrder = mlir::omp::ClauseMemoryOrderKind::Release;
-    }
-  }
-
-  mlir::Value value =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
-  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
-  mlir::Value converted = builder.createConvert(loc, atomType, value);
-
-  builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
-      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
-  return op;
-}
-
-static mlir::Operation *
-genAtomicUpdate(lower::AbstractConverter &converter,
-                semantics::SemanticsContext &semaCtx, mlir::Location loc,
-                lower::StatementContext &stmtCtx, mlir::Value atomAddr,
-                const semantics::SomeExpr &atom,
-                const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-                std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-                fir::FirOpBuilder::InsertPoint preAt,
-                fir::FirOpBuilder::InsertPoint atomicAt,
-                fir::FirOpBuilder::InsertPoint postAt) {
-  lower::ExprToValueMap overrides;
-  lower::StatementContext naCtx;
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.restoreInsertionPoint(preAt);
-
-  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
-
-  // This must exist by now.
-  SomeExpr input = *Fortran::evaluate::GetConvertInput(assign.rhs);
-  std::vector<SomeExpr> args{
-      Fortran::evaluate::GetTopLevelOperation(input).second};
-  assert(!args.empty() && "Update operation without arguments");
-  for (auto &arg : args) {
-    if (!Fortran::evaluate::IsSameOrConvertOf(arg, atom)) {
-      mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
-      overrides.try_emplace(&arg, val);
-    }
-  }
-
-  builder.restoreInsertionPoint(atomicAt);
-  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
-
-  mlir::Region &region = updateOp->getRegion(0);
-  mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
-  mlir::Value localAtom = fir::getBase(block->getArgument(0));
-  overrides.try_emplace(&atom, localAtom);
-
-  converter.overrideExprValues(&overrides);
-  mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
-  mlir::Value converted = builder.createConvert(loc, atomType, updated);
-  builder.create<mlir::omp::YieldOp>(loc, converted);
-  converter.resetExprOverrides();
-
-  builder.restoreInsertionPoint(postAt); // For naCtx cleanups
-  return updateOp;
-}
-
-static mlir::Operation *
-genAtomicOperation(lower::AbstractConverter &converter,
-                   semantics::SemanticsContext &semaCtx, mlir::Location loc,
-                   lower::StatementContext &stmtCtx, int action,
-                   mlir::Value atomAddr, const semantics::SomeExpr &atom,
-                   const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-                   std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-                   fir::FirOpBuilder::InsertPoint preAt,
-                   fir::FirOpBuilder::InsertPoint atomicAt,
-                   fir::FirOpBuilder::InsertPoint postAt) {
-  if (isPointerAssignment(assign)) {
-    TODO(loc, "Code generation for pointer assignment is not implemented yet");
-  }
-
-  // This function and the functions called here do not preserve the
-  // builder's insertion point, or set it to anything specific.
-  switch (action) {
-  case parser::OpenMPAtomicConstruct::Analysis::Read:
-    return genAtomicRead(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
-                         assign, hint, memOrder, preAt, atomicAt, postAt);
-  case parser::OpenMPAtomicConstruct::Analysis::Write:
-    return genAtomicWrite(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
-                          assign, hint, memOrder, preAt, atomicAt, postAt);
-  case parser::OpenMPAtomicConstruct::Analysis::Update:
-    return genAtomicUpdate(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
-                           assign, hint, memOrder, preAt, atomicAt, postAt);
-  default:
-    return nullptr;
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3938,163 +3624,11 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
 // OpenMPConstruct visitors
 //===----------------------------------------------------------------------===//
 
-[[maybe_unused]] static void
-dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
-  auto whatStr = [](int k) {
-    std::string txt = "?";
-    switch (k & parser::OpenMPAtomicConstruct::Analysis::Action) {
-    case parser::OpenMPAtomicConstruct::Analysis::None:
-      txt = "None";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::Read:
-      txt = "Read";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::Write:
-      txt = "Write";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::Update:
-      txt = "Update";
-      break;
-    }
-    switch (k & parser::OpenMPAtomicConstruct::Analysis::Condition) {
-    case parser::OpenMPAtomicConstruct::Analysis::IfTrue:
-      txt += " | IfTrue";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::IfFalse:
-      txt += " | IfFalse";
-      break;
-    }
-    return txt;
-  };
-
-  auto exprStr = [&](const parser::TypedExpr &expr) {
-    if (auto *maybe = expr.get()) {
-      if (maybe->v)
-        return maybe->v->AsFortran();
-    }
-    return "<null>"s;
-  };
-  auto assignStr = [&](const parser::AssignmentStmt::TypedAssignment &assign) {
-    if (auto *maybe = assign.get(); maybe && maybe->v) {
-      std::string str;
-      llvm::raw_string_ostream os(str);
-      maybe->v->AsFortran(os);
-      return str;
-    }
-    return "<null>"s;
-  };
-
-  const SomeExpr &atom = *analysis.atom.get()->v;
-
-  llvm::errs() << "Analysis {\n";
-  llvm::errs() << "  atom: " << atom.AsFortran() << "\n";
-  llvm::errs() << "  cond: " << exprStr(analysis.cond) << "\n";
-  llvm::errs() << "  op0 {\n";
-  llvm::errs() << "    what: " << whatStr(analysis.op0.what) << "\n";
-  llvm::errs() << "    assign: " << assignStr(analysis.op0.assign) << "\n";
-  llvm::errs() << "  }\n";
-  llvm::errs() << "  op1 {\n";
-  llvm::errs() << "    what: " << whatStr(analysis.op1.what) << "\n";
-  llvm::errs() << "    assign: " << assignStr(analysis.op1.assign) << "\n";
-  llvm::errs() << "  }\n";
-  llvm::errs() << "}\n";
-}
-
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAtomicConstruct &construct) {
-  auto get = [](auto &&typedWrapper) -> decltype(&*typedWrapper.get()->v) {
-    if (auto *maybe = typedWrapper.get(); maybe && maybe->v) {
-      return &*maybe->v;
-    } else {
-      return nullptr;
-    }
-  };
-
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  auto &dirSpec = std::get<parser::OmpDirectiveSpecification>(construct.t);
-  List<Clause> clauses = makeClauses(dirSpec.Clauses(), semaCtx);
-  lower::StatementContext stmtCtx;
-
-  const parser::OpenMPAtomicConstruct::Analysis &analysis = construct.analysis;
-  if (DumpAtomicAnalysis)
-    dumpAtomicAnalysis(analysis);
-
-  const semantics::SomeExpr &atom = *get(analysis.atom);
-  mlir::Location loc = converter.genLocation(construct.source);
-  mlir::Value atomAddr =
-      fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
-  mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
-  std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder =
-      getAtomicMemoryOrder(semaCtx, clauses,
-                           semaCtx.FindScope(construct.source));
-
-  if (auto *cond = get(analysis.cond)) {
-    (void)cond;
-    TODO(loc, "OpenMP ATOMIC COMPARE");
-  } else {
-    int action0 = analysis.op0.what & analysis.Action;
-    int action1 = analysis.op1.what & analysis.Action;
-    mlir::Operation *captureOp = nullptr;
-    fir::FirOpBuilder::InsertPoint preAt = builder.saveInsertionPoint();
-    fir::FirOpBuilder::InsertPoint atomicAt, postAt;
-
-    if (construct.IsCapture()) {
-      // Capturing operation.
-      assert(action0 != analysis.None && action1 != analysis.None &&
-             "Expexcing two actions");
-      (void)action0;
-      (void)action1;
-      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
-          loc, hint, makeMemOrderAttr(converter, memOrder));
-      // Set the non-atomic insertion point to before the atomic.capture.
-      preAt = getInsertionPointBefore(captureOp);
-
-      mlir::Block *block = builder.createBlock(&captureOp->getRegion(0));
-      builder.setInsertionPointToEnd(block);
-      // Set the atomic insertion point to before the terminator inside
-      // atomic.capture.
-      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
-      atomicAt = getInsertionPointBefore(term);
-      postAt = getInsertionPointAfter(captureOp);
-      hint = nullptr;
-      memOrder = std::nullopt;
-    } else {
-      // Non-capturing operation.
-      assert(action0 != analysis.None && action1 == analysis.None &&
-             "Expexcing single action");
-      assert(!(analysis.op0.what & analysis.Condition));
-      postAt = atomicAt = preAt;
-    }
-
-    // The builder's insertion point needs to be specifically set before
-    // each call to `genAtomicOperation`.
-    mlir::Operation *firstOp = genAtomicOperation(
-        converter, semaCtx, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
-        *get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
-    assert(firstOp && "Should have created an atomic operation");
-    atomicAt = getInsertionPointAfter(firstOp);
-
-    mlir::Operation *secondOp = nullptr;
-    if (analysis.op1.what != analysis.None) {
-      secondOp = genAtomicOperation(
-          converter, semaCtx, loc, stmtCtx, analysis.op1.what, atomAddr, atom,
-          *get(analysis.op1.assign), hint, memOrder, preAt, atomicAt, postAt);
-    }
-
-    if (construct.IsCapture()) {
-      // If this is a capture operation, the first/second ops will be inside
-      // of it. Set the insertion point to past the capture op itself.
-      builder.restoreInsertionPoint(postAt);
-    } else {
-      if (secondOp) {
-        builder.setInsertionPointAfter(secondOp);
-      } else {
-        builder.setInsertionPointAfter(firstOp);
-      }
-    }
-  }
+  lowerAtomic(converter, symTable, semaCtx, eval, construct);
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
@@ -4235,6 +3769,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Location currentLocation =
       converter.genLocation(beginLoopDirective.source);
 
+  auto &optLoopCons =
+      std::get<std::optional<parser::NestedConstruct>>(loopConstruct.t);
+  if (optLoopCons.has_value()) {
+    if (auto *ompNestedLoopCons{
+            std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
+                &*optLoopCons)}) {
+      genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+    }
+  }
+
   llvm::omp::Directive directive =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
   const parser::CharBlock &source =
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index d7f520e86e532..330cef7b54c74 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -149,13 +149,17 @@ std::string ReductionProcessor::getReductionName(
     reductionName = "multiply_reduction";
     break;
   case omp::clause::DefinedOperator::IntrinsicOperator::AND:
-    return "and_reduction";
+    reductionName = "and_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
-    return "eqv_reduction";
+    reductionName = "eqv_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::OR:
-    return "or_reduction";
+    reductionName = "or_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
-    return "neqv_reduction";
+    reductionName = "neqv_reduction";
+    break;
   default:
     reductionName = "other_reduction";
     break;
@@ -673,9 +677,6 @@ void ReductionProcessor::processReductionArguments(
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
-    mlir::Type redNameTy = redType;
-    if (mlir::isa<fir::LogicalType>(redType.getEleTy()))
-      redNameTy = builder.getI1Type();
 
     if (const auto &redDefinedOp =
             std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
@@ -697,8 +698,7 @@ void ReductionProcessor::processReductionArguments(
         break;
       }
 
-      reductionName =
-          getReductionName(intrinsicOp, kindMap, redNameTy, isByRef);
+      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
     } else if (const auto *reductionIntrinsic =
                    std::get_if<omp::clause::ProcedureDesignator>(
                        &redOperator.u)) {
@@ -709,7 +709,7 @@ void ReductionProcessor::processReductionArguments(
       redId = getReductionType(*reductionIntrinsic);
       reductionName =
           getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redNameTy, isByRef);
+                           redType, isByRef);
     } else {
       TODO(currentLocation, "Unexpected reduction type");
     }
diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp
index 68023610c3c50..a28cc019a7310 100644
--- a/flang/lib/Lower/PFTBuilder.cpp
+++ b/flang/lib/Lower/PFTBuilder.cpp
@@ -1472,8 +1472,8 @@ bool Fortran::lower::definedInCommonBlock(const semantics::Symbol &sym) {
 
 /// Is the symbol `sym` a global?
 bool Fortran::lower::symbolIsGlobal(const semantics::Symbol &sym) {
-  return semantics::IsSaved(sym) || lower::definedInCommonBlock(sym) ||
-         semantics::IsNamedConstant(sym);
+  return (semantics::IsSaved(sym) && semantics::CanCUDASymbolBeGlobal(sym)) ||
+         lower::definedInCommonBlock(sym) || semantics::IsNamedConstant(sym);
 }
 
 namespace {
diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index e878041d37c03..c3a5b6101ce00 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -502,22 +502,37 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
 
   // Allocating on the heap in case the whole reduction/privatization is nested
   // inside of a loop
-  auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
-  // if needsDealloc isn't statically false, add cleanup region. Always
-  // do this for allocatable boxes because they might have been re-allocated
-  // in the body of the loop/parallel region
-
-  std::optional<int64_t> cstNeedsDealloc = fir::getIntIfConstant(needsDealloc);
-  assert(cstNeedsDealloc.has_value() &&
-         "createTempFromMold decides this statically");
-  if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
-                        isDoConcurrent);
-  } else {
-    assert(!isAllocatableOrPointer &&
-           "Pointer-like arrays must be heap allocated");
-  }
+  auto temp = [&]() {
+    bool shouldAllocateOnStack = false;
+
+    // On the GPU, always allocate on the stack since heap allocatins are very
+    // expensive.
+    if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
+            *builder.getModule()))
+      shouldAllocateOnStack = offloadMod.getIsGPU();
+
+    if (shouldAllocateOnStack)
+      return createStackTempFromMold(loc, builder, source);
+
+    auto [temp, needsDealloc] = createTempFromMold(loc, builder, source);
+    // if needsDealloc isn't statically false, add cleanup region. Always
+    // do this for allocatable boxes because they might have been re-allocated
+    // in the body of the loop/parallel region
+
+    std::optional<int64_t> cstNeedsDealloc =
+        fir::getIntIfConstant(needsDealloc);
+    assert(cstNeedsDealloc.has_value() &&
+           "createTempFromMold decides this statically");
+    if (cstNeedsDealloc.has_value() && *cstNeedsDealloc != false) {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      createCleanupRegion(converter, loc, argType, cleanupRegion, sym,
+                          isDoConcurrent);
+    } else {
+      assert(!isAllocatableOrPointer &&
+             "Pointer-like arrays must be heap allocated");
+    }
+    return temp;
+  }();
 
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
diff --git a/flang/lib/Lower/SymbolMap.cpp b/flang/lib/Lower/SymbolMap.cpp
index ce078a7fbde65..b929dfbd5aec4 100644
--- a/flang/lib/Lower/SymbolMap.cpp
+++ b/flang/lib/Lower/SymbolMap.cpp
@@ -82,6 +82,10 @@ Fortran::lower::SymMap::lookupImpliedDo(Fortran::lower::SymMap::AcDoVar var) {
   return {};
 }
 
+void Fortran::lower::SymbolBox::dump() const { llvm::errs() << *this << '\n'; }
+
+void Fortran::lower::SymMap::dump() const { llvm::errs() << *this << '\n'; }
+
 llvm::raw_ostream &
 Fortran::lower::operator<<(llvm::raw_ostream &os,
                            const Fortran::lower::SymbolBox &symBox) {
diff --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index 5f286c04a7ca0..dcbf4991907bf 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -43,7 +43,8 @@ bool cuf::isCUDADeviceContext(mlir::Operation *op) {
 // for it.
 // If the insertion point is inside an OpenACC region op, it is considered
 // device context.
-bool cuf::isCUDADeviceContext(mlir::Region &region) {
+bool cuf::isCUDADeviceContext(mlir::Region &region,
+                              bool isDoConcurrentOffloadEnabled) {
   if (region.getParentOfType<cuf::KernelOp>())
     return true;
   if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
@@ -56,6 +57,9 @@ bool cuf::isCUDADeviceContext(mlir::Region &region) {
              cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
+  if (isDoConcurrentOffloadEnabled &&
+      region.getParentOfType<fir::DoConcurrentLoopOp>())
+    return true;
   return false;
 }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 6ac87067f6511..b5cabdb830e5c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -1867,6 +1867,20 @@ fir::factory::deduceOptimalExtents(mlir::ValueRange extents1,
   return extents;
 }
 
+uint64_t fir::factory::getGlobalAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getGlobalMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
+}
+
+uint64_t fir::factory::getProgramAddressSpace(mlir::DataLayout *dataLayout) {
+  if (dataLayout)
+    if (mlir::Attribute addrSpace = dataLayout->getProgramMemorySpace())
+      return mlir::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return 0;
+}
+
 llvm::SmallVector<mlir::Value> fir::factory::updateRuntimeExtentsForEmptyArrays(
     fir::FirOpBuilder &builder, mlir::Location loc, mlir::ValueRange extents) {
   if (extents.size() <= 1)
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index a00dc9f5d30a2..d32c1fde59f27 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -933,6 +933,8 @@ static constexpr IntrinsicHandler handlers[]{
      /*isElemental=*/false},
     {"tand", &I::genTand},
     {"this_grid", &I::genThisGrid, {}, /*isElemental=*/false},
+    {"this_thread_block", &I::genThisThreadBlock, {}, /*isElemental=*/false},
+    {"this_warp", &I::genThisWarp, {}, /*isElemental=*/false},
     {"threadfence", &I::genThreadFence, {}, /*isElemental=*/false},
     {"threadfence_block", &I::genThreadFenceBlock, {}, /*isElemental=*/false},
     {"threadfence_system", &I::genThreadFenceSystem, {}, /*isElemental=*/false},
@@ -2646,16 +2648,18 @@ mlir::Value IntrinsicLibrary::genAbs(mlir::Type resultType,
 // ACOSD
 mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
+  // maps ACOSD to ACOS * 180 / pi
   assert(args.size() == 1);
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value result =
+      getRuntimeCallGenerator("acos", ftype)(builder, loc, {args[0]});
   llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
   mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
+      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
-  return getRuntimeCallGenerator("acos", ftype)(builder, loc, {arg});
+  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
 }
 
 // ADJUSTL & ADJUSTR
@@ -2797,16 +2801,18 @@ IntrinsicLibrary::genAny(mlir::Type resultType,
 // ASIND
 mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
+  // maps ASIND to ASIN * 180 / pi
   assert(args.size() == 1);
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value result =
+      getRuntimeCallGenerator("asin", ftype)(builder, loc, {args[0]});
   llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
   mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
+      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
-  return getRuntimeCallGenerator("asin", ftype)(builder, loc, {arg});
+  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
 }
 
 // ATAND, ATAN2D
@@ -5148,9 +5154,9 @@ void IntrinsicLibrary::genIeeeGetOrSetModesOrStatus(
         isModes ? fir::runtime::genGetModesTypeSize(builder, loc)
                 : fir::runtime::genGetStatusTypeSize(builder, loc);
     byteSize = builder.createConvert(loc, builder.getIndexType(), byteSize);
-    addr =
-        builder.create<fir::AllocMemOp>(loc, extractSequenceType(heapTy),
-                                        /*typeparams=*/std::nullopt, byteSize);
+    addr = builder.create<fir::AllocMemOp>(loc, extractSequenceType(heapTy),
+                                           /*typeparams=*/mlir::ValueRange(),
+                                           byteSize);
     mlir::Value shape = builder.create<fir::ShapeOp>(loc, byteSize);
     builder.create<fir::StoreOp>(
         loc, builder.create<fir::EmboxOp>(loc, fieldTy, addr, shape), fieldRef);
@@ -8194,6 +8200,99 @@ mlir::Value IntrinsicLibrary::genThisGrid(mlir::Type resultType,
   return res;
 }
 
+// THIS_THREAD_BLOCK
+mlir::Value
+IntrinsicLibrary::genThisThreadBlock(mlir::Type resultType,
+                                     llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0);
+  auto recTy = mlir::cast<fir::RecordType>(resultType);
+  assert(recTy && "RecordType expepected");
+  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Type i32Ty = builder.getI32Type();
+
+  // this_thread_block%size = blockDim.z * blockDim.y * blockDim.x;
+  mlir::Value blockDimX = builder.create<mlir::NVVM::BlockDimXOp>(loc, i32Ty);
+  mlir::Value blockDimY = builder.create<mlir::NVVM::BlockDimYOp>(loc, i32Ty);
+  mlir::Value blockDimZ = builder.create<mlir::NVVM::BlockDimZOp>(loc, i32Ty);
+  mlir::Value size =
+      builder.create<mlir::arith::MulIOp>(loc, blockDimZ, blockDimY);
+  size = builder.create<mlir::arith::MulIOp>(loc, size, blockDimX);
+
+  // this_thread_block%rank = ((threadIdx.z * blockDim.y) * blockDim.x) +
+  //   (threadIdx.y * blockDim.x) + threadIdx.x + 1;
+  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
+  mlir::Value threadIdY = builder.create<mlir::NVVM::ThreadIdYOp>(loc, i32Ty);
+  mlir::Value threadIdZ = builder.create<mlir::NVVM::ThreadIdZOp>(loc, i32Ty);
+  mlir::Value r1 =
+      builder.create<mlir::arith::MulIOp>(loc, threadIdZ, blockDimY);
+  mlir::Value r2 = builder.create<mlir::arith::MulIOp>(loc, r1, blockDimX);
+  mlir::Value r3 =
+      builder.create<mlir::arith::MulIOp>(loc, threadIdY, blockDimX);
+  mlir::Value r2r3 = builder.create<mlir::arith::AddIOp>(loc, r2, r3);
+  mlir::Value rank = builder.create<mlir::arith::AddIOp>(loc, r2r3, threadIdX);
+  mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
+  rank = builder.create<mlir::arith::AddIOp>(loc, rank, one);
+
+  auto sizeFieldName = recTy.getTypeList()[1].first;
+  mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
+  mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
+  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, sizeFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+
+  auto rankFieldName = recTy.getTypeList()[2].first;
+  mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
+  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, rankFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  return res;
+}
+
+// THIS_WARP
+mlir::Value IntrinsicLibrary::genThisWarp(mlir::Type resultType,
+                                          llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0);
+  auto recTy = mlir::cast<fir::RecordType>(resultType);
+  assert(recTy && "RecordType expepected");
+  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Type i32Ty = builder.getI32Type();
+
+  // coalesced_group%size = 32
+  mlir::Value size = builder.createIntegerConstant(loc, i32Ty, 32);
+  auto sizeFieldName = recTy.getTypeList()[1].first;
+  mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
+  mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
+  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, sizeFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+
+  // coalesced_group%rank = threadIdx.x & 31 + 1
+  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
+  mlir::Value mask = builder.createIntegerConstant(loc, i32Ty, 31);
+  mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
+  mlir::Value masked =
+      builder.create<mlir::arith::AndIOp>(loc, threadIdX, mask);
+  mlir::Value rank = builder.create<mlir::arith::AddIOp>(loc, masked, one);
+  auto rankFieldName = recTy.getTypeList()[2].first;
+  mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
+  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, rankFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  return res;
+}
+
 // TRAILZ
 mlir::Value IntrinsicLibrary::genTrailz(mlir::Type resultType,
                                         llvm::ArrayRef<mlir::Value> args) {
diff --git a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
index fcc91752552c3..0094ce892d61b 100644
--- a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
@@ -2888,8 +2888,7 @@ void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {
     llvm_unreachable("invalid vector operation for generator");
   }
 
-  auto funcType{
-      mlir::FunctionType::get(context, {stTy, addr.getType()}, std::nullopt)};
+  auto funcType{mlir::FunctionType::get(context, {stTy, addr.getType()}, {})};
   mlir::func::FuncOp funcOp = builder.createFunction(loc, fname, funcType);
 
   llvm::SmallVector<mlir::Value, 4> biArgs;
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a3de3ae9d116a..09038f6e4d3ae 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -137,6 +137,38 @@ addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
 }
 
 namespace {
+
+mlir::Value replaceWithAddrOfOrASCast(mlir::ConversionPatternRewriter &rewriter,
+                                      mlir::Location loc,
+                                      std::uint64_t globalAS,
+                                      std::uint64_t programAS,
+                                      llvm::StringRef symName, mlir::Type type,
+                                      mlir::Operation *replaceOp = nullptr) {
+  if (mlir::isa<mlir::LLVM::LLVMPointerType>(type)) {
+    if (globalAS != programAS) {
+      auto llvmAddrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
+          loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+      if (replaceOp)
+        return rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(
+            replaceOp, ::getLlvmPtrType(rewriter.getContext(), programAS),
+            llvmAddrOp);
+      return rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+          loc, getLlvmPtrType(rewriter.getContext(), programAS), llvmAddrOp);
+    }
+
+    if (replaceOp)
+      return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
+          replaceOp, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+    return rewriter.create<mlir::LLVM::AddressOfOp>(
+        loc, getLlvmPtrType(rewriter.getContext(), globalAS), symName);
+  }
+
+  if (replaceOp)
+    return rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(replaceOp, type,
+                                                                symName);
+  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, type, symName);
+}
+
 /// Lower `fir.address_of` operation to `llvm.address_of` operation.
 struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   using FIROpConversion::FIROpConversion;
@@ -144,9 +176,15 @@ struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   llvm::LogicalResult
   matchAndRewrite(fir::AddrOfOp addr, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-    auto ty = convertType(addr.getType());
-    rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
-        addr, ty, addr.getSymbol().getRootReference().getValue());
+    auto global = addr->getParentOfType<mlir::ModuleOp>()
+                      .lookupSymbol<mlir::LLVM::GlobalOp>(addr.getSymbol());
+    replaceWithAddrOfOrASCast(
+        rewriter, addr->getLoc(),
+        global ? global.getAddrSpace() : getGlobalAddressSpace(rewriter),
+        getProgramAddressSpace(rewriter),
+        global ? global.getSymName()
+               : addr.getSymbol().getRootReference().getValue(),
+        convertType(addr.getType()), addr);
     return mlir::success();
   }
 };
@@ -1294,6 +1332,56 @@ genCUFAllocDescriptor(mlir::Location loc,
       .getResult();
 }
 
+/// Get the address of the type descriptor global variable that was created by
+/// lowering for derived type \p recType.
+template <typename ModOpTy>
+static mlir::Value
+getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
+                  mlir::Location loc, fir::RecordType recType,
+                  const fir::FIRToLLVMPassOptions &options) {
+  std::string name =
+      options.typeDescriptorsRenamedForAssembly
+          ? fir::NameUniquer::getTypeDescriptorAssemblyName(recType.getName())
+          : fir::NameUniquer::getTypeDescriptorName(recType.getName());
+  mlir::Type llvmPtrTy = ::getLlvmPtrType(mod.getContext());
+  mlir::DataLayout dataLayout(mod);
+  if (auto global = mod.template lookupSymbol<fir::GlobalOp>(name))
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, fir::factory::getGlobalAddressSpace(&dataLayout),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
+  // The global may have already been translated to LLVM.
+  if (auto global = mod.template lookupSymbol<mlir::LLVM::GlobalOp>(name))
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, global.getAddrSpace(),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
+  // Type info derived types do not have type descriptors since they are the
+  // types defining type descriptors.
+  if (options.ignoreMissingTypeDescriptors ||
+      fir::NameUniquer::belongsToModule(
+          name, Fortran::semantics::typeInfoBuiltinModule))
+    return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
+
+  if (!options.skipExternalRttiDefinition)
+    fir::emitFatalError(loc,
+                        "runtime derived type info descriptor was not "
+                        "generated and skipExternalRttiDefinition and "
+                        "ignoreMissingTypeDescriptors options are not set");
+
+  // Rtti for a derived type defined in another compilation unit and for which
+  // rtti was not defined in lowering because of the skipExternalRttiDefinition
+  // option. Generate the object declaration now.
+  auto insertPt = rewriter.saveInsertionPoint();
+  rewriter.setInsertionPoint(mod.getBody(), mod.getBody()->end());
+  mlir::LLVM::GlobalOp global = rewriter.create<mlir::LLVM::GlobalOp>(
+      loc, llvmPtrTy, /*constant=*/true, mlir::LLVM::Linkage::External, name,
+      mlir::Attribute());
+  rewriter.restoreInsertionPoint(insertPt);
+  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
+                                                  global.getSymName());
+}
+
 /// Common base class for embox to descriptor conversion.
 template <typename OP>
 struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
@@ -1406,36 +1494,6 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
                        stride);
   }
 
-  /// Get the address of the type descriptor global variable that was created by
-  /// lowering for derived type \p recType.
-  template <typename ModOpTy>
-  mlir::Value
-  getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
-                    mlir::Location loc, fir::RecordType recType) const {
-    std::string name =
-        this->options.typeDescriptorsRenamedForAssembly
-            ? fir::NameUniquer::getTypeDescriptorAssemblyName(recType.getName())
-            : fir::NameUniquer::getTypeDescriptorName(recType.getName());
-    mlir::Type llvmPtrTy = ::getLlvmPtrType(mod.getContext());
-    if (auto global = mod.template lookupSymbol<fir::GlobalOp>(name)) {
-      return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
-                                                      global.getSymName());
-    }
-    if (auto global = mod.template lookupSymbol<mlir::LLVM::GlobalOp>(name)) {
-      // The global may have already been translated to LLVM.
-      return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
-                                                      global.getSymName());
-    }
-    // Type info derived types do not have type descriptors since they are the
-    // types defining type descriptors.
-    if (!this->options.ignoreMissingTypeDescriptors &&
-        !fir::NameUniquer::belongsToModule(
-            name, Fortran::semantics::typeInfoBuiltinModule))
-      fir::emitFatalError(
-          loc, "runtime derived type info descriptor was not generated");
-    return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
-  }
-
   template <typename ModOpTy>
   mlir::Value populateDescriptor(mlir::Location loc, ModOpTy mod,
                                  fir::BaseBoxType boxTy, mlir::Type inputType,
@@ -1500,7 +1558,8 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
           mlir::Type innerType = fir::unwrapInnerType(inputType);
           if (innerType && mlir::isa<fir::RecordType>(innerType)) {
             auto recTy = mlir::dyn_cast<fir::RecordType>(innerType);
-            typeDesc = getTypeDescriptor(mod, rewriter, loc, recTy);
+            typeDesc =
+                getTypeDescriptor(mod, rewriter, loc, recTy, this->options);
           } else {
             // Unlimited polymorphic type descriptor with no record type. Set
             // type descriptor address to a clean state.
@@ -1508,8 +1567,8 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
                 loc, ::getLlvmPtrType(mod.getContext()));
           }
         } else {
-          typeDesc = getTypeDescriptor(mod, rewriter, loc,
-                                       fir::unwrapIfDerived(boxTy));
+          typeDesc = getTypeDescriptor(
+              mod, rewriter, loc, fir::unwrapIfDerived(boxTy), this->options);
         }
       }
       if (typeDesc)
@@ -3021,22 +3080,10 @@ struct TypeDescOpConversion : public fir::FIROpConversion<fir::TypeDescOp> {
     assert(mlir::isa<fir::RecordType>(inTy) && "expecting fir.type");
     auto recordType = mlir::dyn_cast<fir::RecordType>(inTy);
     auto module = typeDescOp.getOperation()->getParentOfType<mlir::ModuleOp>();
-    std::string typeDescName =
-        this->options.typeDescriptorsRenamedForAssembly
-            ? fir::NameUniquer::getTypeDescriptorAssemblyName(
-                  recordType.getName())
-            : fir::NameUniquer::getTypeDescriptorName(recordType.getName());
-    auto llvmPtrTy = ::getLlvmPtrType(typeDescOp.getContext());
-    if (auto global = module.lookupSymbol<mlir::LLVM::GlobalOp>(typeDescName)) {
-      rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
-          typeDescOp, llvmPtrTy, global.getSymName());
-      return mlir::success();
-    } else if (auto global = module.lookupSymbol<fir::GlobalOp>(typeDescName)) {
-      rewriter.replaceOpWithNewOp<mlir::LLVM::AddressOfOp>(
-          typeDescOp, llvmPtrTy, global.getSymName());
-      return mlir::success();
-    }
-    return mlir::failure();
+    mlir::Value typeDesc = getTypeDescriptor(
+        module, rewriter, typeDescOp.getLoc(), recordType, this->options);
+    rewriter.replaceOp(typeDescOp, typeDesc);
+    return mlir::success();
   }
 };
 
@@ -3126,8 +3173,8 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
     mlir::SymbolRefAttr comdat;
     llvm::ArrayRef<mlir::NamedAttribute> attrs;
     auto g = rewriter.create<mlir::LLVM::GlobalOp>(
-        loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0, 0,
-        false, false, comdat, attrs, dbgExprs);
+        loc, tyAttr, isConst, linkage, global.getSymName(), initAttr, 0,
+        getGlobalAddressSpace(rewriter), false, false, comdat, attrs, dbgExprs);
 
     if (global.getAlignment() && *global.getAlignment() > 0)
       g.setAlignment(*global.getAlignment());
diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
index 12021deb4bd97..75a54fd0ee14f 100644
--- a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/Debug.h"
 
@@ -365,4 +366,13 @@ unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace(
   return defaultAddressSpace;
 }
 
+unsigned ConvertFIRToLLVMPattern::getGlobalAddressSpace(
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
+  assert(parentOp != nullptr &&
+         "expected insertion block to have parent operation");
+  auto dataLayout = mlir::DataLayout::closest(parentOp);
+  return fir::factory::getGlobalAddressSpace(&dataLayout);
+}
+
 } // namespace fir
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 91df8672c20d9..434973f5887e8 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/HLFIR/Passes.h"
 #include "flang/Optimizer/OpenMP/Passes.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/Dominance.h"
@@ -786,13 +787,54 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   mlir::Value shape = hlfir::genShape(loc, builder, lhs);
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
-  hlfir::LoopNest loopNest =
-      hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
-                         flangomp::shouldUseWorkshareLowering(assign));
-  builder.setInsertionPointToStart(loopNest.body);
-  auto arrayElement =
-      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+
+  if (lhs.isSimplyContiguous() && extents.size() > 1) {
+    // Flatten the array to use a single assign loop, that can be better
+    // optimized.
+    mlir::Value n = extents[0];
+    for (size_t i = 1; i < extents.size(); ++i)
+      n = builder.create<mlir::arith::MulIOp>(loc, n, extents[i]);
+    llvm::SmallVector<mlir::Value> flatExtents = {n};
+
+    mlir::Type flatArrayType;
+    mlir::Value flatArray = lhs.getBase();
+    if (mlir::isa<fir::BoxType>(lhs.getType())) {
+      shape = builder.genShape(loc, flatExtents);
+      flatArrayType = fir::BoxType::get(fir::SequenceType::get(eleTy, 1));
+      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
+                                               shape, /*slice=*/mlir::Value{});
+    } else {
+      // Array references must have fixed shape, when used in assignments.
+      auto refTy = mlir::cast<fir::ReferenceType>(lhs.getType());
+      auto seqTy = mlir::cast<fir::SequenceType>(refTy.getElementType());
+      llvm::ArrayRef<int64_t> fixedShape = seqTy.getShape();
+      int64_t flatExtent = 1;
+      for (int64_t extent : fixedShape)
+        flatExtent *= extent;
+      flatArrayType =
+          fir::ReferenceType::get(fir::SequenceType::get({flatExtent}, eleTy));
+      flatArray = builder.createConvert(loc, flatArrayType, flatArray);
+    }
+
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, flatExtents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+
+    mlir::Value arrayElement =
+        builder.create<hlfir::DesignateOp>(loc, fir::ReferenceType::get(eleTy),
+                                           flatArray, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  } else {
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+    auto arrayElement =
+        hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  }
+
   rewriter.eraseOp(assign);
   return mlir::success();
 }
diff --git a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
index 673d7e86c7ba0..317a41a2129c3 100644
--- a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
@@ -306,6 +306,10 @@ static bool isArrayLike(mlir::Type type) {
 }
 
 static bool isCompositeLike(mlir::Type type) {
+  // class(*) is not a composite type since it does not have a determined type.
+  if (fir::isUnlimitedPolymorphicType(type))
+    return false;
+
   return mlir::isa<fir::RecordType, fir::ClassType, mlir::TupleType>(type);
 }
 
@@ -320,8 +324,18 @@ template <>
 mlir::acc::VariableTypeCategory
 OpenACCMappableModel<fir::BaseBoxType>::getTypeCategory(mlir::Type type,
                                                         mlir::Value var) const {
+  // Class-type does not behave like a normal box because it does not hold an
+  // element type. Thus special handle it here.
+  if (mlir::isa<fir::ClassType>(type)) {
+    // class(*) is not a composite type since it does not have a determined
+    // type.
+    if (fir::isUnlimitedPolymorphicType(type))
+      return mlir::acc::VariableTypeCategory::uncategorized;
+    return mlir::acc::VariableTypeCategory::composite;
+  }
 
   mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
+  assert(eleTy && "expect to be able to unwrap the element type");
 
   // If the type enclosed by the box is a mappable type, then have it
   // provide the type category.
@@ -346,7 +360,7 @@ OpenACCMappableModel<fir::BaseBoxType>::getTypeCategory(mlir::Type type,
   return mlir::acc::VariableTypeCategory::nonscalar;
 }
 
-static mlir::TypedValue<mlir::acc::PointerLikeType>
+static mlir::Value
 getBaseRef(mlir::TypedValue<mlir::acc::PointerLikeType> varPtr) {
   // If there is no defining op - the unwrapped reference is the base one.
   mlir::Operation *op = varPtr.getDefiningOp();
@@ -372,7 +386,7 @@ getBaseRef(mlir::TypedValue<mlir::acc::PointerLikeType> varPtr) {
           })
           .Default([&](mlir::Operation *) { return varPtr; });
 
-  return mlir::cast<mlir::TypedValue<mlir::acc::PointerLikeType>>(baseRef);
+  return baseRef;
 }
 
 static mlir::acc::VariableTypeCategory
@@ -384,10 +398,17 @@ categorizePointee(mlir::Type pointer,
   // value would both be represented as !fir.ref<f32>. We do not want to treat
   // such a reference as a scalar. Thus unwrap interior pointer calculations.
   auto baseRef = getBaseRef(varPtr);
-  mlir::Type eleTy = baseRef.getType().getElementType();
 
-  if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
-    return mappableTy.getTypeCategory(varPtr);
+  if (auto mappableTy =
+          mlir::dyn_cast<mlir::acc::MappableType>(baseRef.getType()))
+    return mappableTy.getTypeCategory(baseRef);
+
+  // It must be a pointer-like type since it is not a MappableType.
+  auto ptrLikeTy = mlir::cast<mlir::acc::PointerLikeType>(baseRef.getType());
+  mlir::Type eleTy = ptrLikeTy.getElementType();
+
+  if (auto mappableEleTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
+    return mappableEleTy.getTypeCategory(varPtr);
 
   if (isScalarLike(eleTy))
     return mlir::acc::VariableTypeCategory::scalar;
@@ -397,8 +418,12 @@ categorizePointee(mlir::Type pointer,
     return mlir::acc::VariableTypeCategory::composite;
   if (mlir::isa<fir::CharacterType, mlir::FunctionType>(eleTy))
     return mlir::acc::VariableTypeCategory::nonscalar;
+  // Assumed-type (type(*))does not have a determined type that can be
+  // categorized.
+  if (mlir::isa<mlir::NoneType>(eleTy))
+    return mlir::acc::VariableTypeCategory::uncategorized;
   // "pointers" - in the sense of raw address point-of-view, are considered
-  // scalars. However
+  // scalars.
   if (mlir::isa<fir::LLVMPointerType>(eleTy))
     return mlir::acc::VariableTypeCategory::scalar;
 
diff --git a/flang/lib/Optimizer/Passes/CommandLineOpts.cpp b/flang/lib/Optimizer/Passes/CommandLineOpts.cpp
index f95a280883cba..014237542f24d 100644
--- a/flang/lib/Optimizer/Passes/CommandLineOpts.cpp
+++ b/flang/lib/Optimizer/Passes/CommandLineOpts.cpp
@@ -39,6 +39,12 @@ cl::opt<bool> ignoreMissingTypeDescriptors(
              "translating FIR to LLVM"),
     cl::init(false), cl::Hidden);
 
+cl::opt<bool> skipExternalRttiDefinition(
+    "skip-external-rtti-definition", llvm::cl::init(false),
+    llvm::cl::desc("do not define rtti static objects for types belonging to "
+                   "other compilation units"),
+    cl::Hidden);
+
 OptimizationLevel defaultOptLevel{OptimizationLevel::O0};
 
 codegenoptions::DebugInfoKind noDebugInfo{codegenoptions::NoDebugInfo};
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 70f57bdeddd3f..dca176abe5ec5 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -108,6 +108,7 @@ void addFIRToLLVMPass(mlir::PassManager &pm,
                       const MLIRToLLVMPassPipelineConfig &config) {
   fir::FIRToLLVMPassOptions options;
   options.ignoreMissingTypeDescriptors = ignoreMissingTypeDescriptors;
+  options.skipExternalRttiDefinition = skipExternalRttiDefinition;
   options.applyTBAA = config.AliasAnalysis;
   options.forceUnifiedTBAATree = useOldAliasTags;
   options.typeDescriptorsRenamedForAssembly =
@@ -357,6 +358,8 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::NonLeaf;
   else if (config.FramePointerKind == llvm::FramePointerKind::All)
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::All;
+  else if (config.FramePointerKind == llvm::FramePointerKind::Reserved)
+    framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::Reserved;
   else
     framePointerKind = mlir::LLVM::framePointerKind::FramePointerKind::None;
 
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index 5cfbdc33285f9..4e788f75d9d77 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -73,7 +73,12 @@ class PassState {
   }
 
   void processFunctionScopes(mlir::func::FuncOp func);
-  fir::DummyScopeOp getDeclarationScope(fir::DeclareOp declareOp);
+  // For the given fir.declare returns the dominating fir.dummy_scope
+  // operation.
+  fir::DummyScopeOp getDeclarationScope(fir::DeclareOp declareOp) const;
+  // For the given fir.declare returns the outermost fir.dummy_scope
+  // in the current function.
+  fir::DummyScopeOp getOutermostScope(fir::DeclareOp declareOp) const;
 
 private:
   mlir::DominanceInfo &domInfo;
@@ -119,9 +124,8 @@ void PassState::processFunctionScopes(mlir::func::FuncOp func) {
   }
 }
 
-// For the given fir.declare returns the dominating fir.dummy_scope
-// operation.
-fir::DummyScopeOp PassState::getDeclarationScope(fir::DeclareOp declareOp) {
+fir::DummyScopeOp
+PassState::getDeclarationScope(fir::DeclareOp declareOp) const {
   auto func = declareOp->getParentOfType<mlir::func::FuncOp>();
   assert(func && "fir.declare does not have parent func.func");
   auto &scopeOps = sortedScopeOperations.at(func);
@@ -132,6 +136,15 @@ fir::DummyScopeOp PassState::getDeclarationScope(fir::DeclareOp declareOp) {
   return nullptr;
 }
 
+fir::DummyScopeOp PassState::getOutermostScope(fir::DeclareOp declareOp) const {
+  auto func = declareOp->getParentOfType<mlir::func::FuncOp>();
+  assert(func && "fir.declare does not have parent func.func");
+  auto &scopeOps = sortedScopeOperations.at(func);
+  if (!scopeOps.empty())
+    return scopeOps[0];
+  return nullptr;
+}
+
 class AddAliasTagsPass : public fir::impl::AddAliasTagsBase<AddAliasTagsPass> {
 public:
   void runOnOperation() override;
@@ -279,6 +292,15 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
     else
       unknownAllocOp = true;
 
+    if (auto declOp = source.origin.instantiationPoint) {
+      // Use the outermost scope for local allocations,
+      // because using the innermost scope may result
+      // in incorrect TBAA, when calls are inlined in MLIR.
+      auto declareOp = mlir::dyn_cast<fir::DeclareOp>(declOp);
+      assert(declareOp && "Instantiation point must be fir.declare");
+      scopeOp = state.getOutermostScope(declareOp);
+    }
+
     if (unknownAllocOp) {
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "WARN: unknown defining op for SourceKind::Allocate " << *op
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c02..6eb914e67fd54 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more difficult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create
diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index 328e2374115b0..bfb0daeacb8c3 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -113,8 +113,16 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
       return signalPassFailure();
     mlir::SymbolTable gpuSymTable(gpuMod);
     for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
-      if (cuf::isRegisteredDeviceGlobal(globalOp))
+      if (cuf::isRegisteredDeviceGlobal(globalOp)) {
         candidates.insert(globalOp);
+      } else if (globalOp.getConstant() &&
+                 mlir::isa<fir::SequenceType>(
+                     fir::unwrapRefType(globalOp.resultType()))) {
+        mlir::Attribute initAttr =
+            globalOp.getInitVal().value_or(mlir::Attribute());
+        if (initAttr && mlir::dyn_cast<mlir::DenseElementsAttr>(initAttr))
+          candidates.insert(globalOp);
+      }
     }
     for (auto globalOp : candidates) {
       auto globalName{globalOp.getSymbol().getValue()};
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index cdd30dce183dd..abad500d3f657 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -48,8 +48,7 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m,
                                        mlir::SymbolTable *symbolTable_,
                                        const mlir::DataLayout &dl)
     : module(m), symbolTable(symbolTable_), dataLayout{&dl},
-      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl),
-      derivedTypeDepth(0) {
+      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl) {
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 
   mlir::MLIRContext *context = module.getContext();
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
+
+void DerivedTypeCache::preComponentVisitUpdate() {
+  componentActiveRecursionLevels.clear();
+}
+
+void DerivedTypeCache::postComponentVisitUpdate(
+    ActiveLevels &activeRecursionLevels) {
+  if (componentActiveRecursionLevels.empty())
+    return;
+  ActiveLevels oldLevels;
+  oldLevels.swap(activeRecursionLevels);
+  std::set_union(componentActiveRecursionLevels.begin(),
+                 componentActiveRecursionLevels.end(), oldLevels.begin(),
+                 oldLevels.end(), std::back_inserter(activeRecursionLevels));
+}
+
+void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,
+                                ActiveLevels &&activeRecursionLevels) {
+  // If there is no nested recursion or if this type does not point to any type
+  // nodes above it, it is safe to cache it indefinitely (it can be used in any
+  // contexts).
+  if (activeRecursionLevels.empty() ||
+      (activeRecursionLevels[0] == derivedTypeDepth)) {
+    typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(attr, {});
+    componentActiveRecursionLevels.clear();
+    cleanUpCache(derivedTypeDepth);
+    --derivedTypeDepth;
+    return;
+  }
+  // Trim any recursion below the current type.
+  if (activeRecursionLevels.back() >= derivedTypeDepth) {
+    auto last = llvm::find_if(activeRecursionLevels, [&](std::int32_t depth) {
+      return depth >= derivedTypeDepth;
+    });
+    if (last != activeRecursionLevels.end()) {
+      activeRecursionLevels.erase(last, activeRecursionLevels.end());
+    }
+  }
+  componentActiveRecursionLevels = std::move(activeRecursionLevels);
+  typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      attr, componentActiveRecursionLevels);
+  cleanUpCache(derivedTypeDepth);
+  if (!componentActiveRecursionLevels.empty())
+    insertCacheCleanUp(ty, componentActiveRecursionLevels.back());
+  --derivedTypeDepth;
+}
+
+void DerivedTypeCache::insertCacheCleanUp(mlir::Type type, int32_t depth) {
+  auto iter = llvm::find_if(cacheCleanupList,
+                            [&](const auto &x) { return x.second >= depth; });
+  if (iter == cacheCleanupList.end()) {
+    cacheCleanupList.emplace_back(
+        std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+    return;
+  }
+  if (iter->second == depth) {
+    iter->first.push_back(type);
+    return;
+  }
+  cacheCleanupList.insert(
+      iter, std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+}
+
+void DerivedTypeCache::cleanUpCache(int32_t depth) {
+  if (cacheCleanupList.empty())
+    return;
+  // cleanups are done in the post actions when visiting a derived type
+  // tree. So if there is a clean-up for the current depth, it has to be
+  // the last one (deeper ones must have been done already).
+  if (cacheCleanupList.back().second == depth) {
+    for (mlir::Type type : cacheCleanupList.back().first)
+      typeCache[type].first = nullptr;
+    cacheCleanupList.pop_back_n(1);
+  }
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     fir::RecordType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
-  // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
 
-  bool canCacheThisType = true;
-  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
   mlir::MLIRContext *context = module.getContext();
-  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto [nameKind, sourceName] = fir::NameUniquer::deconstruct(Ty.getName());
+  if (nameKind != fir::NameUniquer::NameKind::DERIVED_TYPE)
+    return genPlaceholderType(context);
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   // Generate a place holder TypeAttr which will be used if a member
   // references the parent type.
-  auto comAttr = mlir::LLVM::DICompositeTypeAttr::get(
+  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto placeHolder = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/true, llvm::dwarf::DW_TAG_structure_type,
       mlir::StringAttr::get(context, ""), fileAttr, /*line=*/0, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = comAttr;
-
-  auto result = fir::NameUniquer::deconstruct(Ty.getName());
-  if (result.first != fir::NameUniquer::NameKind::DERIVED_TYPE)
-    return genPlaceholderType(context);
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty, placeHolder);
 
   fir::TypeInfoOp tiOp = symbolTable->lookup<fir::TypeInfoOp>(Ty.getName());
   unsigned line = (tiOp) ? getLineFromLoc(tiOp.getLoc()) : 1;
@@ -305,6 +400,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
         fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
@@ -317,7 +413,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     mlir::LLVM::DITypeAttr elemTy;
     if (lowerBounds && seqTy &&
         lowerBounds->size() == seqTy.getShape().size()) {
-      llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> arrayElements;
       for (auto [bound, dim] :
            llvm::zip_equal(*lowerBounds, seqTy.getShape())) {
         auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
@@ -325,14 +421,14 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
             context, countAttr, lowerAttr, /*upperBound=*/nullptr,
             /*stride=*/nullptr);
-        elements.push_back(subrangeTy);
+        arrayElements.push_back(subrangeTy);
       }
       elemTy = mlir::LLVM::DICompositeTypeAttr::get(
           context, llvm::dwarf::DW_TAG_array_type, /*name=*/nullptr,
           /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr,
           convertType(seqTy.getEleTy(), fileAttr, scope, declOp),
           mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0, /*alignInBits=*/0,
-          elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+          arrayElements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
           /*allocated=*/nullptr, /*associated=*/nullptr);
     } else
       elemTy = convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -344,80 +440,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
-
-    // Currently, the handling of recursive debug type in mlir has some
-    // limitations that were discussed at the end of the thread for following
-    // PR.
-    // https://github.com/llvm/llvm-project/pull/106571
-    //
-    // Problem could be explained with the following example code:
-    //  type t2
-    //   type(t1), pointer :: p1
-    // end type
-    // type t1
-    //   type(t2), pointer :: p2
-    // end type
-    // In the description below, type_self means a temporary type that is
-    // generated
-    // as a place holder while the members of that type are being processed.
-    //
-    // If we process t1 first then we will have the following structure after
-    // it has been processed.
-    // t1 -> t2 -> t1_self
-    // This is because when we started processing t2, we did not have the
-    // complete t1 but its place holder t1_self.
-    // Now if some entity requires t2, we will already have that in cache and
-    // will return it. But this t2 refers to t1_self and not to t1. In mlir
-    // handling, only those types are allowed to have _self reference which are
-    // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
-    // because the t1_self reference can be resolved by the outer t1. But
-    // standalone t2 is not because there will be no way to resolve it. Until
-    // this is fixed in mlir, we avoid caching such types. Please see
-    // DebugTranslation::translateRecursive for details on how mlir handles
-    // recursive types.
-    // The code below checks for situation where it will be unsafe to cache
-    // a type to avoid this problem. We do that in 2 situations.
-    // 1. If a member is record type, then its type would have been processed
-    // before reaching here. If it is not in the cache, it means that it was
-    // found to be unsafe to cache. So any type containing it will also not
-    // be cached
-    // 2. The type of the member is found in the cache but it is a place holder.
-    // In this case, its recID should match the recID of the type we are
-    // processing. This helps us to cache the following type.
-    // type t
-    //  type(t), allocatable :: p
-    // end type
-    mlir::Type baseTy = getDerivedType(fieldTy);
-    if (auto recTy = mlir::dyn_cast<fir::RecordType>(baseTy)) {
-      auto iter = typeCache.find(recTy);
-      if (iter == typeCache.end())
-        canCacheThisType = false;
-      else {
-        if (auto tyAttr =
-                mlir::dyn_cast<mlir::LLVM::DICompositeTypeAttr>(iter->second)) {
-          if (tyAttr.getIsRecSelf() && tyAttr.getRecId() != recId)
-            canCacheThisType = false;
-        }
-      }
-    }
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto finalAttr = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/false, llvm::dwarf::DW_TAG_structure_type,
-      mlir::StringAttr::get(context, result.second.name), fileAttr, line, scope,
+      mlir::StringAttr::get(context, sourceName.name), fileAttr, line, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
 
-  // derivedTypeDepth == 1 means that it is a top level type which is safe to
-  // cache.
-  if (canCacheThisType || derivedTypeDepth == 1) {
-    typeCache[Ty] = finalAttr;
-  } else {
-    auto iter = typeCache.find(Ty);
-    if (iter != typeCache.end())
-      typeCache.erase(iter);
-  }
+  derivedTypeCache.finalize(Ty, finalAttr, std::move(nestedRecursions));
+
   return finalAttr;
 }
 
@@ -425,15 +459,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
     mlir::TupleType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
   // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty);
 
   llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   mlir::MLIRContext *context = module.getContext();
 
   std::uint64_t offset = 0;
   for (auto fieldTy : Ty.getTypes()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     mlir::LLVM::DITypeAttr elemTy =
         convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -445,6 +482,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto typeAttr = mlir::LLVM::DICompositeTypeAttr::get(
@@ -453,7 +491,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = typeAttr;
+  derivedTypeCache.finalize(Ty, typeAttr, std::move(nestedRecursions));
   return typeAttr;
 }
 
@@ -667,27 +705,7 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertCharacterType(charTy, fileAttr, scope, declOp,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_if_present<fir::RecordType>(Ty)) {
-    // For nested derived types like shown below, the call sequence of the
-    // convertRecordType will look something like as follows:
-    // convertRecordType (t1)
-    //  convertRecordType (t2)
-    //    convertRecordType (t3)
-    // We need to recognize when we are processing the top level type like t1
-    // to make caching decision. The variable `derivedTypeDepth` is used for
-    // this purpose and maintains the current depth of derived type processing.
-    //  type t1
-    //   type(t2), pointer :: p1
-    // end type
-    // type t2
-    //   type(t3), pointer :: p2
-    // end type
-    // type t2
-    //   integer a
-    // end type
-    derivedTypeDepth++;
-    auto result = convertRecordType(recTy, fileAttr, scope, declOp);
-    derivedTypeDepth--;
-    return result;
+    return convertRecordType(recTy, fileAttr, scope, declOp);
   } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
     return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index 93b9ac2d90fdf..854e4397ca32b 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -23,6 +23,77 @@
 
 namespace fir {
 
+/// Special cache to deal with the fact that mlir::LLVM::DITypeAttr for
+/// derived types may only be valid in specific nesting contexts in presence
+/// of derived type recursion and cannot be cached for the whole compilation.
+/// It is however still desirable to cache such mlir::LLVM::DITypeAttr as
+/// long as possible to avoid catastrophic compilation slow downs in very
+/// complex derived types where an intermediate type in a derived type cycle may
+/// indirectly appear hundreds of times under the top type of the derived type
+/// cycle. More details in the comment below.
+class DerivedTypeCache {
+public:
+  // Currently, the handling of recursive debug type in mlir has some
+  // limitations that were discussed at the end of the thread for following
+  // PR.
+  // https://github.com/llvm/llvm-project/pull/106571
+  //
+  // Problem could be explained with the following example code:
+  //  type t2
+  //   type(t1), pointer :: p1
+  // end type
+  // type t1
+  //   type(t2), pointer :: p2
+  // end type
+  // In the description below, type_self means a temporary type that is
+  // generated
+  // as a place holder while the members of that type are being processed.
+  //
+  // If we process t1 first then we will have the following structure after
+  // it has been processed.
+  // t1 -> t2 -> t1_self
+  // This is because when we started processing t2, we did not have the
+  // complete t1 but its place holder t1_self.
+  // Now if some entity requires t2, we will already have that in cache and
+  // will return it. But this t2 refers to t1_self and not to t1. In mlir
+  // handling, only those types are allowed to have _self reference which are
+  // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
+  // because the t1_self reference can be resolved by the outer t1. But
+  // standalone t2 is not because there will be no way to resolve it. Until
+  // this is fixed in mlir, we avoid caching such types. Please see
+  // DebugTranslation::translateRecursive for details on how mlir handles
+  // recursive types.
+  using ActiveLevels = llvm::SmallVector<int32_t, 1>;
+  mlir::LLVM::DITypeAttr lookup(mlir::Type);
+  ActiveLevels startTranslating(mlir::Type,
+                                mlir::LLVM::DITypeAttr placeHolder = nullptr);
+  void finalize(mlir::Type, mlir::LLVM::DITypeAttr, ActiveLevels &&);
+  void preComponentVisitUpdate();
+  void postComponentVisitUpdate(ActiveLevels &);
+
+private:
+  void insertCacheCleanUp(mlir::Type type, int32_t depth);
+  void cleanUpCache(int32_t depth);
+  // Current depth inside a top level derived type being converted.
+  int32_t derivedTypeDepth = 0;
+  // Cache for already translated derived types with the minimum depth where
+  // this cache entry is valid. Zero means the translation is always valid, "i"
+  // means the type depends its derived type tree parent node at depth "i". Such
+  // types should be cleaned-up from the cache in the post visit of node "i".
+  // Note that any new metadata created for a type with a component in the cache
+  // with validity of "i" shall not be added to the cache with a validity
+  // smaller than "i".
+  llvm::DenseMap<mlir::Type, std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>>
+      typeCache;
+  // List of parent nodes that are being recursively referred to in the
+  // component type that has just been computed.
+  ActiveLevels componentActiveRecursionLevels;
+  // Helper list that maintains the list of nodes that must be deleted from the
+  // cache when going back past listed parent depths.
+  llvm::SmallVector<std::pair<llvm::SmallVector<mlir::Type>, int32_t>>
+      cacheCleanupList;
+};
+
 /// This converts FIR/mlir type to DITypeAttr.
 class DebugTypeGenerator {
 public:
@@ -91,8 +162,7 @@ class DebugTypeGenerator {
   std::uint64_t lenOffset;
   std::uint64_t rankOffset;
   std::uint64_t rankSize;
-  int32_t derivedTypeDepth;
-  llvm::DenseMap<mlir::Type, mlir::LLVM::DITypeAttr> typeCache;
+  DerivedTypeCache derivedTypeCache;
 };
 
 } // namespace fir
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index f9a4c4d0283c7..57eae1ff052a2 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -16,7 +16,6 @@
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Support/TypeCode.h"
-#include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Runtime/derived-api.h"
 #include "flang/Semantics/runtime-type-info.h"
@@ -38,6 +37,45 @@ namespace fir {
 using namespace fir;
 using namespace mlir;
 
+// Reconstruct binding tables for dynamic dispatch.
+using BindingTable = llvm::DenseMap<llvm::StringRef, unsigned>;
+using BindingTables = llvm::DenseMap<llvm::StringRef, BindingTable>;
+
+static std::string getTypeDescriptorTypeName() {
+  llvm::SmallVector<llvm::StringRef, 1> modules = {
+      Fortran::semantics::typeInfoBuiltinModule};
+  return fir::NameUniquer::doType(modules, /*proc=*/{}, /*blockId=*/0,
+                                  Fortran::semantics::typeDescriptorTypeName,
+                                  /*kinds=*/{});
+}
+
+static std::optional<mlir::Type>
+buildBindingTables(BindingTables &bindingTables, mlir::ModuleOp mod) {
+
+  std::optional<mlir::Type> typeDescriptorType;
+  std::string typeDescriptorTypeName = getTypeDescriptorTypeName();
+  // The binding tables are defined in FIR after lowering inside fir.type_info
+  // operations. Go through each binding tables and store the procedure name and
+  // binding index for later use by the fir.dispatch conversion pattern.
+  for (auto typeInfo : mod.getOps<fir::TypeInfoOp>()) {
+    if (!typeDescriptorType && typeInfo.getSymName() == typeDescriptorTypeName)
+      typeDescriptorType = typeInfo.getType();
+    unsigned bindingIdx = 0;
+    BindingTable bindings;
+    if (typeInfo.getDispatchTable().empty()) {
+      bindingTables[typeInfo.getSymName()] = bindings;
+      continue;
+    }
+    for (auto dtEntry :
+         typeInfo.getDispatchTable().front().getOps<fir::DTEntryOp>()) {
+      bindings[dtEntry.getMethod()] = bindingIdx;
+      ++bindingIdx;
+    }
+    bindingTables[typeInfo.getSymName()] = bindings;
+  }
+  return typeDescriptorType;
+}
+
 namespace {
 
 /// SelectTypeOp converted to an if-then-else chain
@@ -77,9 +115,10 @@ class SelectTypeConv : public OpConversionPattern<fir::SelectTypeOp> {
 struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
   using OpConversionPattern<fir::DispatchOp>::OpConversionPattern;
 
-  DispatchOpConv(mlir::MLIRContext *ctx, const BindingTables &bindingTables)
+  DispatchOpConv(mlir::MLIRContext *ctx, const BindingTables &bindingTables,
+                 std::optional<mlir::Type> typeDescriptorType)
       : mlir::OpConversionPattern<fir::DispatchOp>(ctx),
-        bindingTables(bindingTables) {}
+        bindingTables(bindingTables), typeDescriptorType{typeDescriptorType} {}
 
   llvm::LogicalResult
   matchAndRewrite(fir::DispatchOp dispatch, OpAdaptor adaptor,
@@ -111,13 +150,11 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
 
     mlir::Value passedObject = dispatch.getObject();
 
-    auto module = dispatch.getOperation()->getParentOfType<mlir::ModuleOp>();
-    Type typeDescTy;
-    std::string typeDescName =
-        NameUniquer::getTypeDescriptorName(recordType.getName());
-    if (auto global = module.lookupSymbol<fir::GlobalOp>(typeDescName)) {
-      typeDescTy = global.getType();
-    }
+    if (!typeDescriptorType)
+      return emitError(loc) << "cannot find " << getTypeDescriptorTypeName()
+                            << " fir.type_info that is required to get the "
+                               "related builtin type and lower fir.dispatch";
+    mlir::Type typeDescTy = *typeDescriptorType;
 
     // clang-format off
     // Before:
@@ -213,6 +250,7 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
 
 private:
   BindingTables bindingTables;
+  std::optional<mlir::Type> typeDescriptorType;
 };
 
 /// Convert FIR structured control flow ops to CFG ops.
@@ -229,10 +267,11 @@ class PolymorphicOpConversion
     mlir::RewritePatternSet patterns(context);
 
     BindingTables bindingTables;
-    buildBindingTables(bindingTables, mod);
+    std::optional<mlir::Type> typeDescriptorType =
+        buildBindingTables(bindingTables, mod);
 
     patterns.insert<SelectTypeConv>(context);
-    patterns.insert<DispatchOpConv>(context, bindingTables);
+    patterns.insert<DispatchOpConv>(context, bindingTables, typeDescriptorType);
     mlir::ConversionTarget target(*context);
     target.addLegalDialect<mlir::affine::AffineDialect,
                            mlir::cf::ControlFlowDialect, FIROpsDialect,
@@ -379,16 +418,11 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   } else if (auto a = mlir::dyn_cast<fir::SubclassAttr>(attr)) {
     // Retrieve the type descriptor from the type guard statement record type.
     assert(mlir::isa<fir::RecordType>(a.getType()) && "expect fir.record type");
-    fir::RecordType recTy = mlir::dyn_cast<fir::RecordType>(a.getType());
-    std::string typeDescName =
-        fir::NameUniquer::getTypeDescriptorName(recTy.getName());
-    auto typeDescGlobal = mod.lookupSymbol<fir::GlobalOp>(typeDescName);
-    auto typeDescAddr = rewriter.create<fir::AddrOfOp>(
-        loc, fir::ReferenceType::get(typeDescGlobal.getType()),
-        typeDescGlobal.getSymbol());
-    mlir::Type typeDescTy = ReferenceType::get(rewriter.getNoneType());
+    mlir::Value typeDescAddr =
+        rewriter.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(a.getType()));
+    mlir::Type refNoneType = ReferenceType::get(rewriter.getNoneType());
     mlir::Value typeDesc =
-        rewriter.create<ConvertOp>(loc, typeDescTy, typeDescAddr);
+        rewriter.create<ConvertOp>(loc, refNoneType, typeDescAddr);
 
     // Prepare the selector descriptor for the runtime call.
     mlir::Type descNoneTy = fir::BoxType::get(rewriter.getNoneType());
@@ -406,7 +440,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
                                mlir::UnitAttr::get(rewriter.getContext()));
       callee =
           fir::createFuncOp(rewriter.getUnknownLoc(), mod, fctName,
-                            rewriter.getFunctionType({descNoneTy, typeDescTy},
+                            rewriter.getFunctionType({descNoneTy, refNoneType},
                                                      rewriter.getI1Type()),
                             {runtimeAttr});
     }
@@ -435,20 +469,11 @@ SelectTypeConv::genTypeDescCompare(mlir::Location loc, mlir::Value selector,
                                    mlir::Type ty, mlir::ModuleOp mod,
                                    mlir::PatternRewriter &rewriter) const {
   assert(mlir::isa<fir::RecordType>(ty) && "expect fir.record type");
-  fir::RecordType recTy = mlir::dyn_cast<fir::RecordType>(ty);
-  std::string typeDescName =
-      fir::NameUniquer::getTypeDescriptorName(recTy.getName());
-  auto typeDescGlobal = mod.lookupSymbol<fir::GlobalOp>(typeDescName);
-  if (!typeDescGlobal)
-    return {};
-  auto typeDescAddr = rewriter.create<fir::AddrOfOp>(
-      loc, fir::ReferenceType::get(typeDescGlobal.getType()),
-      typeDescGlobal.getSymbol());
+  mlir::Value typeDescAddr =
+      rewriter.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(ty));
+  mlir::Value selectorTdescAddr = rewriter.create<fir::BoxTypeDescOp>(
+      loc, typeDescAddr.getType(), selector);
   auto intPtrTy = rewriter.getIndexType();
-  mlir::Type tdescType =
-      fir::TypeDescType::get(mlir::NoneType::get(rewriter.getContext()));
-  mlir::Value selectorTdescAddr =
-      rewriter.create<fir::BoxTypeDescOp>(loc, tdescType, selector);
   auto typeDescInt =
       rewriter.create<fir::ConvertOp>(loc, intPtrTy, typeDescAddr);
   auto selectorTdescInt =
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 799998c54b531..f2237da3857a2 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -273,14 +273,36 @@ static llvm::raw_ostream::Colors PrefixColor(Severity severity) {
   return llvm::raw_ostream::SAVEDCOLOR;
 }
 
+static std::string HintLanguageControlFlag(
+    const common::LanguageFeatureControl *hintFlagPtr,
+    std::optional<common::LanguageFeature> feature,
+    std::optional<common::UsageWarning> warning) {
+  if (hintFlagPtr) {
+    std::string flag;
+    if (warning) {
+      flag = hintFlagPtr->getDefaultCliSpelling(*warning);
+    } else if (feature) {
+      flag = hintFlagPtr->getDefaultCliSpelling(*feature);
+    }
+    if (!flag.empty()) {
+      return " [-W" + flag + "]";
+    }
+  }
+  return "";
+}
+
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine) const {
+    bool echoSourceLine,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
-  sources.EmitMessage(o, provenanceRange, ToString(), Prefix(severity()),
+  const std::string text{ToString()};
+  const std::string hint{
+      HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
       PrefixColor(severity()), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
@@ -430,7 +452,8 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 }
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLines) const {
+    bool echoSourceLines,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -443,7 +466,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
   }
 }
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d969503..30b5d4f9513a7 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1553,7 +1553,7 @@ static OmpMapperSpecifier ConstructOmpMapperSpecifier(
         std::get<Name>(derived->t).ToString() + llvm::omp::OmpDefaultMapperName,
         std::move(typeSpec), std::move(varName)};
   }
-  return OmpMapperSpecifier{std::string("omp.default.mapper"),
+  return OmpMapperSpecifier{std::string(llvm::omp::OmpDefaultMapperName),
       std::move(typeSpec), std::move(varName)};
 }
 
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index ef24c704db885..8ef9810463d5a 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1276,16 +1276,20 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
     case BITXOR:
       left = left ^ right;
       break;
     case BITOR:
-    case OR:
       left = left | right;
       break;
+    case AND:
+      left = left && right;
+      break;
+    case OR:
+      left = left || right;
+      break;
     case LT:
       left = -(left < right);
       break;
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 9aef0c9981e3c..ec894ab8513d2 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -593,13 +593,13 @@ bool Prescanner::SkipToNextSignificantCharacter() {
     return false;
   } else {
     auto anyContinuationLine{false};
-    bool mightNeedSpace{false};
+    bool atNewline{false};
     if (MustSkipToEndOfLine()) {
       SkipToEndOfLine();
     } else {
-      mightNeedSpace = *at_ == '\n';
+      atNewline = *at_ == '\n';
     }
-    for (; Continuation(mightNeedSpace); mightNeedSpace = false) {
+    for (; Continuation(atNewline); atNewline = false) {
       anyContinuationLine = true;
       ++continuationLines_;
       if (MustSkipToEndOfLine()) {
@@ -641,7 +641,7 @@ void Prescanner::SkipSpaces() {
   while (IsSpaceOrTab(at_)) {
     NextChar();
   }
-  insertASpace_ = false;
+  brokenToken_ = false;
 }
 
 const char *Prescanner::SkipWhiteSpace(const char *p) {
@@ -745,10 +745,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       }
     }
   }
-  if (insertASpace_) {
-    tokens.PutNextTokenChar(' ', spaceProvenance_);
-    insertASpace_ = false;
-  }
+  brokenToken_ = false;
   if (*at_ == '\n') {
     return false;
   }
@@ -808,7 +805,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     bool anyDefined{false};
     bool hadContinuation{false};
     // Subtlety: When an identifier is split across continuation lines,
-    // its parts are kept as distinct pp-tokens if that macro replacement
+    // its parts are kept as distinct pp-tokens if macro replacement
     // should operate on them independently.  This trick accommodates the
     // historic practice of using line continuation for token pasting after
     // replacement.
@@ -822,6 +819,9 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       ++at_, ++column_;
       hadContinuation = SkipToNextSignificantCharacter();
       if (hadContinuation && IsLegalIdentifierStart(*at_)) {
+        if (brokenToken_) {
+          break;
+        }
         // Continued identifier
         tokens.CloseToken();
         ++parts;
@@ -915,12 +915,21 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
     int startColumn{column_};
     TokenSequence possible;
     EmitCharAndAdvance(possible, *at_);
+    if (InFixedFormSource()) {
+      SkipSpaces();
+    }
     if (*at_ == '+' || *at_ == '-') {
       EmitCharAndAdvance(possible, *at_);
+      if (InFixedFormSource()) {
+        SkipSpaces();
+      }
     }
     if (IsDecimalDigit(*at_)) { // it's an exponent; scan it
       while (IsDecimalDigit(*at_)) {
         EmitCharAndAdvance(possible, *at_);
+        if (InFixedFormSource()) {
+          SkipSpaces();
+        }
       }
       possible.CloseToken();
       tokens.AppendRange(possible, 0); // appends to current token
@@ -940,13 +949,22 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
   TokenSequence withUnderscore, separate;
   EmitChar(withUnderscore, '_');
   EmitCharAndAdvance(separate, '_');
+  if (InFixedFormSource()) {
+    SkipSpaces();
+  }
   if (IsLegalInIdentifier(*at_)) {
     separate.CloseToken();
     EmitChar(withUnderscore, *at_);
     EmitCharAndAdvance(separate, *at_);
+    if (InFixedFormSource()) {
+      SkipSpaces();
+    }
     while (IsLegalInIdentifier(*at_)) {
       EmitChar(withUnderscore, *at_);
       EmitCharAndAdvance(separate, *at_);
+      if (InFixedFormSource()) {
+        SkipSpaces();
+      }
     }
   }
   withUnderscore.CloseToken();
@@ -1330,7 +1348,7 @@ bool Prescanner::SkipCommentLine(bool afterAmpersand) {
   return false;
 }
 
-const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
+const char *Prescanner::FixedFormContinuationLine(bool atNewline) {
   if (IsAtEnd()) {
     return nullptr;
   }
@@ -1363,8 +1381,8 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
       }
       const char *col6{nextLine_ + 5};
       if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
-        if (mightNeedSpace && !IsSpace(nextLine_ + 6)) {
-          insertASpace_ = true;
+        if (atNewline && !IsSpace(nextLine_ + 6)) {
+          brokenToken_ = true;
         }
         return nextLine_ + 6;
       }
@@ -1377,7 +1395,9 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
         nextLine_[4] == ' ' && IsCompilerDirectiveSentinel(&nextLine_[1], 1)) {
       if (const char *col6{nextLine_ + 5};
           *col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
-        insertASpace_ |= mightNeedSpace && !IsSpace(nextLine_ + 6);
+        if (atNewline && !IsSpace(nextLine_ + 6)) {
+          brokenToken_ = true;
+        }
         return nextLine_ + 6;
       } else {
         return nullptr;
@@ -1446,7 +1466,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
     p = SkipWhiteSpace(p);
     if (*p == '&') {
       if (!ampersand) {
-        insertASpace_ = true;
+        brokenToken_ = true;
       }
       return p + 1;
     } else if (ampersand) {
@@ -1476,7 +1496,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
     } else if (p > lineStart && IsSpaceOrTab(p - 1)) {
       --p;
     } else {
-      insertASpace_ = true;
+      brokenToken_ = true;
     }
     return p;
   } else {
@@ -1484,14 +1504,14 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
   }
 }
 
-bool Prescanner::FixedFormContinuation(bool mightNeedSpace) {
+bool Prescanner::FixedFormContinuation(bool atNewline) {
   // N.B. We accept '&' as a continuation indicator in fixed form, too,
   // but not in a character literal.
   if (*at_ == '&' && inCharLiteral_) {
     return false;
   }
   do {
-    if (const char *cont{FixedFormContinuationLine(mightNeedSpace)}) {
+    if (const char *cont{FixedFormContinuationLine(atNewline)}) {
       BeginSourceLine(cont);
       column_ = 7;
       NextLine();
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index ec4c53cf3e0f2..f650d548e6eff 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -203,10 +203,10 @@ class Prescanner {
   std::optional<std::size_t> IsIncludeLine(const char *) const;
   void FortranInclude(const char *quote);
   const char *IsPreprocessorDirectiveLine(const char *) const;
-  const char *FixedFormContinuationLine(bool mightNeedSpace);
+  const char *FixedFormContinuationLine(bool atNewline);
   const char *FreeFormContinuationLine(bool ampersand);
   bool IsImplicitContinuation() const;
-  bool FixedFormContinuation(bool mightNeedSpace);
+  bool FixedFormContinuation(bool atNewline);
   bool FreeFormContinuation();
   bool Continuation(bool mightNeedFixedFormSpace);
   std::optional<LineClassification> IsFixedFormCompilerDirectiveLine(
@@ -256,10 +256,15 @@ class Prescanner {
   bool continuationInCharLiteral_{false};
   bool inPreprocessorDirective_{false};
 
-  // In some edge cases of compiler directive continuation lines, it
-  // is necessary to treat the line break as a space character by
-  // setting this flag, which is cleared by EmitChar().
-  bool insertASpace_{false};
+  // True after processing a continuation that can't be allowed
+  // to appear in the middle of an identifier token, but is fixed form,
+  // or is free form and doesn't have a space character handy to use as
+  // a separator when:
+  // a) (standard) doesn't begin with a leading '&' on the continuation
+  //     line, but has a non-blank in column 1, or
+  // b) (extension) does have a leading '&', but didn't have one
+  //    on the continued line.
+  bool brokenToken_{false};
 
   // When a free form continuation marker (&) appears at the end of a line
   // before a INCLUDE or #include, we delete it and omit the newline, so
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index 40a074eaf0a47..9deb513e4f64f 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -30,7 +30,8 @@ void TokenSequence::clear() {
 
 void TokenSequence::pop_back() {
   CHECK(!start_.empty());
-  CHECK(nextStart_ > start_.back());
+  // If the last token is empty then `nextStart_ == start_.back()`.
+  CHECK(nextStart_ >= start_.back());
   std::size_t bytes{nextStart_ - start_.back()};
   nextStart_ = start_.back();
   start_.pop_back();
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b98..dcd1ac165adc1 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2094,7 +2094,7 @@ class UnparseVisitor {
   }
   void Unparse(const OmpMapperSpecifier &x) {
     const auto &mapperName{std::get<std::string>(x.t)};
-    if (mapperName.find("omp.default.mapper") == std::string::npos) {
+    if (mapperName.find(llvm::omp::OmpDefaultMapperName) == std::string::npos) {
       Walk(mapperName);
       Put(":");
     }
@@ -2740,7 +2740,7 @@ class UnparseVisitor {
     Word("!$OMP DECLARE MAPPER (");
     const auto &spec{std::get<OmpMapperSpecifier>(z.t)};
     const auto &mapperName{std::get<std::string>(spec.t)};
-    if (mapperName.find("omp.default.mapper") == std::string::npos) {
+    if (mapperName.find(llvm::omp::OmpDefaultMapperName) == std::string::npos) {
       Walk(mapperName);
       Put(":");
     }
@@ -2926,7 +2926,8 @@ class UnparseVisitor {
     Walk(std::get<OmpBeginLoopDirective>(x.t));
     Put("\n");
     EndOpenMP();
-    Walk(std::get<std::optional<DoConstruct>>(x.t));
+    Walk(std::get<std::optional<std::variant<DoConstruct,
+            common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t));
     Walk(std::get<std::optional<OmpEndLoopDirective>>(x.t));
   }
   void Unparse(const BasedPointer &x) {
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index c0fda3631c01f..109bc2dbb8569 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -20,6 +20,9 @@ add_flang_library(FortranSemantics
   check-io.cpp
   check-namelist.cpp
   check-nullify.cpp
+  check-omp-atomic.cpp
+  check-omp-loop.cpp
+  check-omp-metadirective.cpp
   check-omp-structure.cpp
   check-purity.cpp
   check-return.cpp
@@ -34,12 +37,13 @@ add_flang_library(FortranSemantics
   mod-file.cpp
   openmp-dsa.cpp
   openmp-modifiers.cpp
+  openmp-utils.cpp
   pointer-assignment.cpp
   program-tree.cpp
-  resolve-labels.cpp
   resolve-directives.cpp
-  resolve-names-utils.cpp
+  resolve-labels.cpp
   resolve-names.cpp
+  resolve-names-utils.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp
diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 43e23a9d8f60b..88e08887160d9 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -42,7 +42,6 @@ class AssignmentContext {
   void Analyze(const parser::AssignmentStmt &);
   void Analyze(const parser::PointerAssignmentStmt &);
   void Analyze(const parser::ConcurrentControl &);
-  int deviceConstructDepth_{0};
   SemanticsContext &context() { return context_; }
 
 private:
@@ -97,21 +96,6 @@ void AssignmentContext::Analyze(const parser::AssignmentStmt &stmt) {
     if (whereDepth_ > 0) {
       CheckShape(lhsLoc, &lhs);
     }
-    if (context_.foldingContext().languageFeatures().IsEnabled(
-            common::LanguageFeature::CUDA)) {
-      const auto &scope{context_.FindScope(lhsLoc)};
-      const Scope &progUnit{GetProgramUnitContaining(scope)};
-      if (!IsCUDADeviceContext(&progUnit) && deviceConstructDepth_ == 0) {
-        if (Fortran::evaluate::HasCUDADeviceAttrs(lhs) &&
-            Fortran::evaluate::HasCUDAImplicitTransfer(rhs)) {
-          if (GetNbOfCUDAManagedOrUnifiedSymbols(lhs) == 1 &&
-              GetNbOfCUDAManagedOrUnifiedSymbols(rhs) == 1 &&
-              GetNbOfCUDADeviceSymbols(rhs) == 1)
-            return; // This is a special case handled on the host.
-          context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
-        }
-      }
-    }
   }
 }
 
@@ -254,46 +238,6 @@ void AssignmentChecker::Enter(const parser::MaskedElsewhereStmt &x) {
 void AssignmentChecker::Leave(const parser::MaskedElsewhereStmt &) {
   context_.value().PopWhereContext();
 }
-void AssignmentChecker::Enter(const parser::CUFKernelDoConstruct &x) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
-  const auto &beginBlockDirective =
-      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
-  const auto &blockDirective =
-      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
-  if (blockDirective.v == llvm::acc::ACCD_parallel ||
-      blockDirective.v == llvm::acc::ACCD_serial ||
-      blockDirective.v == llvm::acc::ACCD_kernels) {
-    return true;
-  }
-  return false;
-}
-void AssignmentChecker::Enter(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    ++context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Leave(const parser::OpenACCBlockConstruct &x) {
-  if (IsOpenACCComputeConstruct(x)) {
-    --context_.value().deviceConstructDepth_;
-  }
-}
-void AssignmentChecker::Enter(const parser::OpenACCCombinedConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCCombinedConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Enter(const parser::OpenACCLoopConstruct &) {
-  ++context_.value().deviceConstructDepth_;
-}
-void AssignmentChecker::Leave(const parser::OpenACCLoopConstruct &) {
-  --context_.value().deviceConstructDepth_;
-}
 
 } // namespace Fortran::semantics
 template class Fortran::common::Indirection<
diff --git a/flang/lib/Semantics/assignment.h b/flang/lib/Semantics/assignment.h
index 4a1bb92037119..ba537744bfaaa 100644
--- a/flang/lib/Semantics/assignment.h
+++ b/flang/lib/Semantics/assignment.h
@@ -46,14 +46,6 @@ class AssignmentChecker : public virtual BaseChecker {
   void Leave(const parser::EndWhereStmt &);
   void Enter(const parser::MaskedElsewhereStmt &);
   void Leave(const parser::MaskedElsewhereStmt &);
-  void Enter(const parser::CUFKernelDoConstruct &);
-  void Leave(const parser::CUFKernelDoConstruct &);
-  void Enter(const parser::OpenACCBlockConstruct &);
-  void Leave(const parser::OpenACCBlockConstruct &);
-  void Enter(const parser::OpenACCCombinedConstruct &);
-  void Leave(const parser::OpenACCCombinedConstruct &);
-  void Enter(const parser::OpenACCLoopConstruct &);
-  void Leave(const parser::OpenACCLoopConstruct &);
 
   SemanticsContext &context();
 
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 5164f1dc6faab..cf05d8463277f 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -8,6 +8,7 @@
 
 #include "canonicalize-omp.h"
 #include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
 
 // After Loop Canonicalization, rewrite OpenMP parse tree to make OpenMP
 // Constructs more structured which provide explicit scopes for later
@@ -125,6 +126,16 @@ class CanonicalizationOfOmp {
     parser::Block::iterator nextIt;
     auto &beginDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
     auto &dir{std::get<parser::OmpLoopDirective>(beginDir.t)};
+    auto missingDoConstruct = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
+    auto tileUnrollError = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "If a loop construct has been fully unrolled, it cannot then be tiled"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
 
     nextIt = it;
     while (++nextIt != block.end()) {
@@ -135,7 +146,8 @@ class CanonicalizationOfOmp {
       if (auto *doCons{GetConstructIf<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
-          std::get<std::optional<parser::DoConstruct>>(x.t) =
+          std::get<std::optional<std::variant<parser::DoConstruct,
+              common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
@@ -144,7 +156,7 @@ class CanonicalizationOfOmp {
                     GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
               std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
                   std::move(*endDir);
-              block.erase(nextIt);
+              nextIt = block.erase(nextIt);
             }
           }
         } else {
@@ -152,14 +164,79 @@ class CanonicalizationOfOmp {
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+      } else if (auto *ompLoopCons{
+                     GetOmpIf<parser::OpenMPLoopConstruct>(*nextIt)}) {
+        // We should allow UNROLL and TILE constructs to be inserted between an
+        // OpenMP Loop Construct and the DO loop itself
+        auto &nestedBeginDirective =
+            std::get<parser::OmpBeginLoopDirective>(ompLoopCons->t);
+        auto &nestedBeginLoopDirective =
+            std::get<parser::OmpLoopDirective>(nestedBeginDirective.t);
+        if ((nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll ||
+                nestedBeginLoopDirective.v ==
+                    llvm::omp::Directive::OMPD_tile) &&
+            !(nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll &&
+                dir.v == llvm::omp::Directive::OMPD_tile)) {
+          // iterate through the remaining block items to find the end directive
+          // for the unroll/tile directive.
+          parser::Block::iterator endIt;
+          endIt = nextIt;
+          while (endIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*endIt)}) {
+              auto &endLoopDirective =
+                  std::get<parser::OmpLoopDirective>(endDir->t);
+              if (endLoopDirective.v == dir.v) {
+                std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                    std::move(*endDir);
+                endIt = block.erase(endIt);
+                continue;
+              }
+            }
+            ++endIt;
+          }
+          RewriteOpenMPLoopConstruct(*ompLoopCons, block, nextIt);
+          auto &ompLoop = std::get<std::optional<parser::NestedConstruct>>(x.t);
+          ompLoop =
+              std::optional<parser::NestedConstruct>{parser::NestedConstruct{
+                  common::Indirection{std::move(*ompLoopCons)}}};
+          nextIt = block.erase(nextIt);
+        } else if (nestedBeginLoopDirective.v ==
+                llvm::omp::Directive::OMPD_unroll &&
+            dir.v == llvm::omp::Directive::OMPD_tile) {
+          // if a loop has been unrolled, the user can not then tile that loop
+          // as it has been unrolled
+          parser::OmpClauseList &unrollClauseList{
+              std::get<parser::OmpClauseList>(nestedBeginDirective.t)};
+          if (unrollClauseList.v.empty()) {
+            // if the clause list is empty for an unroll construct, we assume
+            // the loop is being fully unrolled
+            tileUnrollError(dir, messages_);
+          } else {
+            // parse the clauses for the unroll directive to find the full
+            // clause
+            for (auto clause{unrollClauseList.v.begin()};
+                clause != unrollClauseList.v.end(); ++clause) {
+              if (clause->Id() == llvm::omp::OMPC_full) {
+                tileUnrollError(dir, messages_);
+              }
+            }
+          }
+        } else {
+          messages_.Say(nestedBeginLoopDirective.source,
+              "Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs"_err_en_US,
+              parser::ToUpperCaseLetters(
+                  nestedBeginLoopDirective.source.ToString()));
+        }
       } else {
-        messages_.Say(dir.source,
-            "A DO loop must follow the %s directive"_err_en_US,
-            parser::ToUpperCaseLetters(dir.source.ToString()));
+        missingDoConstruct(dir, messages_);
       }
       // If we get here, we either found a loop, or issued an error message.
       return;
     }
+    if (nextIt == block.end()) {
+      missingDoConstruct(dir, messages_);
+    }
   }
 
   void RewriteOmpAllocations(parser::ExecutionPart &body) {
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 8decfb0149829..b01147606a99c 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -685,18 +685,67 @@ void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
       std::get<std::list<parser::CUFReduction>>(directive.t)) {
     CheckReduce(context_, reduce);
   }
-  inCUFKernelDoConstruct_ = true;
+  ++deviceConstructDepth_;
+}
+
+static bool IsOpenACCComputeConstruct(const parser::OpenACCBlockConstruct &x) {
+  const auto &beginBlockDirective =
+      std::get<Fortran::parser::AccBeginBlockDirective>(x.t);
+  const auto &blockDirective =
+      std::get<Fortran::parser::AccBlockDirective>(beginBlockDirective.t);
+  if (blockDirective.v == llvm::acc::ACCD_parallel ||
+      blockDirective.v == llvm::acc::ACCD_serial ||
+      blockDirective.v == llvm::acc::ACCD_kernels) {
+    return true;
+  }
+  return false;
 }
 
 void CUDAChecker::Leave(const parser::CUFKernelDoConstruct &) {
-  inCUFKernelDoConstruct_ = false;
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::OpenACCBlockConstruct &x) {
+  if (IsOpenACCComputeConstruct(x)) {
+    --deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Enter(const parser::OpenACCCombinedConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCCombinedConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::OpenACCLoopConstruct &) {
+  ++deviceConstructDepth_;
+}
+void CUDAChecker::Leave(const parser::OpenACCLoopConstruct &) {
+  --deviceConstructDepth_;
+}
+void CUDAChecker::Enter(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::DoConcurrentOffload)) {
+    ++deviceConstructDepth_;
+  }
+}
+void CUDAChecker::Leave(const parser::DoConstruct &x) {
+  if (x.IsDoConcurrent() &&
+      context_.foldingContext().languageFeatures().IsEnabled(
+          common::LanguageFeature::DoConcurrentOffload)) {
+    --deviceConstructDepth_;
+  }
 }
 
 void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
   auto lhsLoc{std::get<parser::Variable>(x.t).GetSource()};
   const auto &scope{context_.FindScope(lhsLoc)};
   const Scope &progUnit{GetProgramUnitContaining(scope)};
-  if (IsCUDADeviceContext(&progUnit) || inCUFKernelDoConstruct_) {
+  if (IsCUDADeviceContext(&progUnit) || deviceConstructDepth_ > 0) {
     return; // Data transfer with assignment is only perform on host.
   }
 
@@ -714,6 +763,16 @@ void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
     context_.Say(lhsLoc,
         "More than one reference to a CUDA object on the right hand side of the assigment"_err_en_US);
   }
+
+  if (Fortran::evaluate::HasCUDADeviceAttrs(assign->lhs) &&
+      Fortran::evaluate::HasCUDAImplicitTransfer(assign->rhs)) {
+    if (GetNbOfCUDAManagedOrUnifiedSymbols(assign->lhs) == 1 &&
+        GetNbOfCUDAManagedOrUnifiedSymbols(assign->rhs) == 1 &&
+        GetNbOfCUDADeviceSymbols(assign->rhs) == 1) {
+      return; // This is a special case handled on the host.
+    }
+    context_.Say(lhsLoc, "Unsupported CUDA data transfer"_err_en_US);
+  }
 }
 
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-cuda.h b/flang/lib/Semantics/check-cuda.h
index 222a2ee04b57c..10000253ffe5a 100644
--- a/flang/lib/Semantics/check-cuda.h
+++ b/flang/lib/Semantics/check-cuda.h
@@ -41,10 +41,18 @@ class CUDAChecker : public virtual BaseChecker {
   void Enter(const parser::CUFKernelDoConstruct &);
   void Leave(const parser::CUFKernelDoConstruct &);
   void Enter(const parser::AssignmentStmt &);
+  void Enter(const parser::OpenACCBlockConstruct &);
+  void Leave(const parser::OpenACCBlockConstruct &);
+  void Enter(const parser::OpenACCCombinedConstruct &);
+  void Leave(const parser::OpenACCCombinedConstruct &);
+  void Enter(const parser::OpenACCLoopConstruct &);
+  void Leave(const parser::OpenACCLoopConstruct &);
+  void Enter(const parser::DoConstruct &);
+  void Leave(const parser::DoConstruct &);
 
 private:
   SemanticsContext &context_;
-  bool inCUFKernelDoConstruct_ = false;
+  int deviceConstructDepth_{0};
 };
 
 bool CanonicalizeCUDA(parser::Program &);
diff --git a/flang/lib/Semantics/check-io.cpp b/flang/lib/Semantics/check-io.cpp
index 10b32d9af0f88..a1ff4b922268b 100644
--- a/flang/lib/Semantics/check-io.cpp
+++ b/flang/lib/Semantics/check-io.cpp
@@ -478,6 +478,8 @@ void IoChecker::Enter(const parser::InquireSpec::LogVar &spec) {
     specKind = IoSpecKind::Pending;
     break;
   }
+  CheckForDefinableVariable(std::get<parser::ScalarLogicalVariable>(spec.t),
+      parser::ToUpperCaseLetters(common::EnumToString(specKind)));
   SetSpecifier(specKind);
 }
 
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
new file mode 100644
index 0000000000000..047c604693460
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -0,0 +1,1295 @@
+//===-- lib/Semantics/check-omp-atomic.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks related to the ATOMIC construct.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <cassert>
+#include <list>
+#include <optional>
+#include <string_view>
+#include <tuple>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+namespace operation = Fortran::evaluate::operation;
+
+template <typename T, typename U>
+static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
+  return !(e == f);
+}
+
+// There is no consistent way to get the source of a given ActionStmt, so
+// extract the source information from Statement<ActionStmt> when we can,
+// and keep it around for error reporting in further analyses.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+struct AnalyzedCondStmt {
+  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
+  parser::CharBlock source;
+  SourcedActionStmt ift, iff;
+};
+
+static SourcedActionStmt GetActionStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+static SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
+// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
+// is that the ActionStmt will be either an assignment or a pointer-assignment,
+// otherwise return std::nullopt.
+// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
+// the "typedAssignment" is unset. This can happen if there are semantic errors
+// in the purported assignment.
+static std::optional<evaluate::Assignment> GetEvaluateAssignment(
+    const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
+
+  return common::visit(
+      [](auto &&s) -> std::optional<evaluate::Assignment> {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>) {
+          const TypedAssignment &typed{s.value().typedAssignment};
+          // ForwardOwningPointer                 typedAssignment
+          // `- GenericAssignmentWrapper          ^.get()
+          //    `- std::optional<Assignment>      ^->v
+          return typed.get()->v;
+        } else {
+          return std::nullopt;
+        }
+      },
+      x->u);
+}
+
+static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  // Extract the evaluate::Expr from ScalarLogicalExpr.
+  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
+    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
+    const parser::Expr &expr{logical.thing.thing.value()};
+    return GetEvaluateExpr(expr);
+  }};
+
+  // Recognize either
+  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
+  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
+
+  if (auto &&action{GetActionStmt(x)}) {
+    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
+            &action.stmt->u)}) {
+      const parser::IfStmt &s{ifs->value()};
+      auto &&maybeCond{
+          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
+      auto &thenStmt{
+          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
+      if (maybeCond) {
+        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
+            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
+            SourcedActionStmt{}};
+      }
+    }
+    return std::nullopt;
+  }
+
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    if (auto *ifc{
+            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
+      using ElseBlock = parser::IfConstruct::ElseBlock;
+      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
+      const parser::IfConstruct &s{ifc->value()};
+
+      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
+        // Not expecting any else-if statements.
+        return std::nullopt;
+      }
+      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
+      auto &&maybeCond{getFromLogical(
+          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
+      if (!maybeCond) {
+        return std::nullopt;
+      }
+
+      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)),
+            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
+        if (result.ift.stmt && result.iff.stmt) {
+          return result;
+        }
+      } else {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
+        if (result.ift.stmt) {
+          return result;
+        }
+      }
+    }
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
+    parser::CharBlock source) {
+  // Find => in the range, if not found, find = that is not a part of
+  // <=, >=, ==, or /=.
+  auto trim{[](std::string_view v) {
+    const char *begin{v.data()};
+    const char *end{begin + v.size()};
+    while (*begin == ' ' && begin != end) {
+      ++begin;
+    }
+    while (begin != end && end[-1] == ' ') {
+      --end;
+    }
+    assert(begin != end && "Source should not be empty");
+    return parser::CharBlock(begin, end - begin);
+  }};
+
+  std::string_view sv(source.begin(), source.size());
+
+  if (auto where{sv.find("=>")}; where != sv.npos) {
+    std::string_view lhs(sv.data(), where);
+    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
+    return std::make_pair(trim(lhs), trim(rhs));
+  }
+
+  // Go backwards, since all the exclusions above end with a '='.
+  for (size_t next{source.size()}; next > 1; --next) {
+    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
+      std::string_view lhs(sv.data(), next - 1);
+      std::string_view rhs(sv.data() + next, sv.size() - next);
+      return std::make_pair(trim(lhs), trim(rhs));
+    }
+  }
+  llvm_unreachable("Could not find assignment operator");
+}
+
+static bool IsCheckForAssociated(const SomeExpr &cond) {
+  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
+}
+
+static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
+  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
+  // for example.
+  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
+}
+
+static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
+  if (value) {
+    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
+        evaluate::GenericExprWrapper::Deleter);
+  }
+}
+
+static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
+    std::optional<evaluate::Assignment> value) {
+  if (value) {
+    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
+        evaluate::GenericAssignmentWrapper::Deleter);
+  }
+}
+
+static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
+    int what,
+    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+  parser::OpenMPAtomicConstruct::Analysis::Op operation;
+  operation.what = what;
+  SetAssignment(operation.assign, maybeAssign);
+  return operation;
+}
+
+static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
+    const SomeExpr &atom, const MaybeExpr &cond,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
+  // Defined in flang/include/flang/Parser/parse-tree.h
+  //
+  // struct Analysis {
+  //   struct Kind {
+  //     static constexpr int None = 0;
+  //     static constexpr int Read = 1;
+  //     static constexpr int Write = 2;
+  //     static constexpr int Update = Read | Write;
+  //     static constexpr int Action = 3; // Bits containing N, R, W, U
+  //     static constexpr int IfTrue = 4;
+  //     static constexpr int IfFalse = 8;
+  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
+  //   };
+  //   struct Op {
+  //     int what;
+  //     TypedAssignment assign;
+  //   };
+  //   TypedExpr atom, cond;
+  //   Op op0, op1;
+  // };
+
+  parser::OpenMPAtomicConstruct::Analysis an;
+  SetExpr(an.atom, atom);
+  SetExpr(an.cond, cond);
+  an.op0 = std::move(op0);
+  an.op1 = std::move(op1);
+  return an;
+}
+
+/// Check if `expr` satisfies the following conditions for x and v:
+///
+/// [6.0:189:10-12]
+/// - x and v (as applicable) are either scalar variables or
+///   function references with scalar data pointer result of non-character
+///   intrinsic type or variables that are non-polymorphic scalar pointers
+///   and any length type parameter must be constant.
+void OmpStructureChecker::CheckAtomicType(
+    SymbolRef sym, parser::CharBlock source, std::string_view name) {
+  const DeclTypeSpec *typeSpec{sym->GetType()};
+  if (!typeSpec) {
+    return;
+  }
+
+  if (!IsPointer(sym)) {
+    using Category = DeclTypeSpec::Category;
+    Category cat{typeSpec->category()};
+    if (cat == Category::Character) {
+      context_.Say(source,
+          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
+    } else if (cat != Category::Numeric && cat != Category::Logical) {
+      context_.Say(source,
+          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
+    }
+    return;
+  }
+
+  // Variable is a pointer.
+  if (typeSpec->IsPolymorphic()) {
+    context_.Say(source,
+        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
+        name);
+    return;
+  }
+
+  // Go over all length parameters, if any, and check if they are
+  // explicit.
+  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
+    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
+          // "entry" is a map entry
+          return entry.second.isLen() && !entry.second.isExplicit();
+        })) {
+      context_.Say(source,
+          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
+          name);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckAtomicVariable(
+    const SomeExpr &atom, parser::CharBlock source) {
+  if (atom.Rank() != 0) {
+    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
+        atom.AsFortran());
+  }
+
+  std::vector<SomeExpr> dsgs{GetAllDesignators(atom)};
+  assert(dsgs.size() == 1 && "Should have a single top-level designator");
+  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
+
+  CheckAtomicType(syms.back(), source, atom.AsFortran());
+
+  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
+    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
+        atom.AsFortran());
+  }
+}
+
+void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
+    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
+    parser::CharBlock source) {
+  if (auto *expr{HasStorageOverlap(base, exprs)}) {
+    context_.Say(source,
+        "Within atomic operation %s and %s access the same storage"_warn_en_US,
+        base.AsFortran(), expr->AsFortran());
+  }
+}
+
+void OmpStructureChecker::ErrorShouldBeVariable(
+    const MaybeExpr &expr, parser::CharBlock source) {
+  if (expr) {
+    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
+        expr->AsFortran());
+  } else {
+    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
+  }
+}
+
+std::pair<const parser::ExecutionPartConstruct *,
+    const parser::ExecutionPartConstruct *>
+OmpStructureChecker::CheckUpdateCapture(
+    const parser::ExecutionPartConstruct *ec1,
+    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
+  // Decide which statement is the atomic update and which is the capture.
+  //
+  // The two allowed cases are:
+  //   x = ...      atomic-var = ...
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  // or
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  //   x = ...      atomic-var = ...
+  //
+  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
+  // (which makes more sense, as it captures the original value of the atomic
+  // variable).
+  //
+  // If the two statements don't fit these criteria, return a pair of default-
+  // constructed values.
+  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>;
+
+  SourcedActionStmt act1{GetActionStmt(ec1)};
+  SourcedActionStmt act2{GetActionStmt(ec2)};
+  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
+  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
+  if (!maybeAssign1 || !maybeAssign2) {
+    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
+      context_.Say(source,
+          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
+    }
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  auto as1{*maybeAssign1}, as2{*maybeAssign2};
+
+  auto isUpdateCapture{
+      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
+        return IsSameOrConvertOf(c.rhs, u.lhs);
+      }};
+
+  // Do some checks that narrow down the possible choices for the update
+  // and the capture statements. This will help to emit better diagnostics.
+  // 1. An assignment could be an update (cbu) if the left-hand side is a
+  //    subexpression of the right-hand side.
+  // 2. An assignment could be a capture (cbc) if the right-hand side is
+  //    a variable (or a function ref), with potential type conversions.
+  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
+  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
+  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
+  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
+
+  // We want to diagnose cases where both assignments cannot be an update,
+  // or both cannot be a capture, as well as cases where either assignment
+  // cannot be any of these two.
+  //
+  // If we organize these boolean values into a matrix
+  //   |cbu1 cbu2|
+  //   |cbc1 cbc2|
+  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
+  // row or column, including the case where everything is zero. All these
+  // cases correspond to the determinant of the matrix being 0, which suggests
+  // that checking the det may be a convenient diagnostic check. There is only
+  // one additional case where the det is 0, which is when the matrix is all 1
+  // ("true"). The "all true" case represents the situation where both
+  // assignments could be an update as well as a capture. On the other hand,
+  // whenever det != 0, the roles of the update and the capture can be
+  // unambiguously assigned to as1 and as2 [1].
+  //
+  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
+  // det = 0, leaving 6 cases where det != 0:
+  //   0 1   0 1   1 0   1 0   1 1   1 1
+  //   1 0   1 1   0 1   1 1   0 1   1 0
+  // In each case the classification is unambiguous.
+
+  //     |cbu1 cbu2|
+  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
+  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
+
+  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
+                                  const std::string &expr) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
+        expr);
+  }};
+
+  auto errorNeitherWorks{[&]() {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
+  }};
+
+  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
+    // If det != 0, then the checks unambiguously suggest a specific
+    // categorization.
+    // If det == 0, then this function should be called only if the
+    // checks haven't ruled out any possibility, i.e. when both assigments
+    // could still be either updates or captures.
+    if (det > 0) {
+      // as1 is update, as2 is capture
+      if (isUpdateCapture(as1, as2)) {
+        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
+      } else {
+        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else if (det < 0) {
+      // as2 is update, as1 is capture
+      if (isUpdateCapture(as2, as1)) {
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      } else {
+        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else {
+      bool updateFirst{isUpdateCapture(as1, as2)};
+      bool captureFirst{isUpdateCapture(as2, as1)};
+      if (updateFirst && captureFirst) {
+        // If both assignment could be the update and both could be the
+        // capture, emit a warning about the ambiguity.
+        context_.Say(act1.source,
+            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      }
+      if (updateFirst != captureFirst) {
+        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
+        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
+        return std::make_pair(upd, cap);
+      }
+      assert(!updateFirst && !captureFirst);
+      errorNeitherWorks();
+      return std::make_pair(nullptr, nullptr);
+    }
+  }};
+
+  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
+    return makeSelectionFromDet(det);
+  }
+  assert(det == 0 && "Prior checks should have covered det != 0");
+
+  // If neither of the statements is an RMW update, it could still be a
+  // "write" update. Pretty much any assignment can be a write update, so
+  // recompute det with cbu1 = cbu2 = true.
+  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
+    return makeSelectionFromDet(writeDet);
+  }
+
+  // It's only errors from here on.
+
+  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
+    errorNeitherWorks();
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // The remaining cases are that
+  // - no candidate for update, or for capture,
+  // - one of the assigments cannot be anything.
+
+  if (!cbu1 && !cbu2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  } else if (!cbc1 && !cbc2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
+    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
+    context_.Say(src,
+        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // All cases should have been covered.
+  llvm_unreachable("Unchecked condition");
+}
+
+void OmpStructureChecker::CheckAtomicCaptureAssignment(
+    const evaluate::Assignment &capture, const SomeExpr &atom,
+    parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &cap{capture.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, rsrc);
+    // This part should have been checked prior to calling this function.
+    assert(*GetConvertInput(capture.rhs) == atom &&
+        "This cannot be a capture assignment");
+    CheckStorageOverlap(atom, {cap}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicReadAssignment(
+    const evaluate::Assignment &read, parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+
+  if (auto maybe{GetConvertInput(read.rhs)}) {
+    const SomeExpr &atom{*maybe};
+
+    if (!IsVarOrFunctionRef(atom)) {
+      ErrorShouldBeVariable(atom, rsrc);
+    } else {
+      CheckAtomicVariable(atom, rsrc);
+      CheckStorageOverlap(atom, {read.lhs}, source);
+    }
+  } else {
+    ErrorShouldBeVariable(read.rhs, rsrc);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWriteAssignment(
+    const evaluate::Assignment &write, parser::CharBlock source) {
+  // [6.0:190:13-15]
+  // A write structured block is write-statement, a write statement that has
+  // one of the following forms:
+  //   x = expr
+  //   x => expr
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{write.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, lsrc);
+    CheckStorageOverlap(atom, {write.rhs}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateAssignment(
+    const evaluate::Assignment &update, parser::CharBlock source) {
+  // [6.0:191:1-7]
+  // An update structured block is update-statement, an update statement
+  // that has one of the following forms:
+  //   x = x operator expr
+  //   x = expr operator x
+  //   x = intrinsic-procedure-name (x)
+  //   x = intrinsic-procedure-name (x, expr-list)
+  //   x = intrinsic-procedure-name (expr-list, x)
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{update.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, lsrc);
+
+  std::pair<operation::Operator, std::vector<SomeExpr>> top{
+      operation::Operator::Unknown, {}};
+  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+    top = GetTopLevelOperation(*maybeInput);
+  }
+  switch (top.first) {
+  case operation::Operator::Add:
+  case operation::Operator::Sub:
+  case operation::Operator::Mul:
+  case operation::Operator::Div:
+  case operation::Operator::And:
+  case operation::Operator::Or:
+  case operation::Operator::Eqv:
+  case operation::Operator::Neqv:
+  case operation::Operator::Min:
+  case operation::Operator::Max:
+  case operation::Operator::Identity:
+    break;
+  case operation::Operator::Call:
+    context_.Say(source,
+        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Convert:
+    context_.Say(source,
+        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Intrinsic:
+    context_.Say(source,
+        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Constant:
+  case operation::Operator::Unknown:
+    context_.Say(
+        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(source,
+        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+        operation::ToString(top.first));
+    return;
+  }
+  // Check how many times `atom` occurs as an argument, if it's a subexpression
+  // of an argument, and collect the non-atom arguments.
+  std::vector<SomeExpr> nonAtom;
+  MaybeExpr subExpr;
+  auto atomCount{[&]() {
+    int count{0};
+    for (const SomeExpr &arg : top.second) {
+      if (IsSameOrConvertOf(arg, atom)) {
+        ++count;
+      } else {
+        if (!subExpr && IsSubexpressionOf(atom, arg)) {
+          subExpr = arg;
+        }
+        nonAtom.push_back(arg);
+      }
+    }
+    return count;
+  }()};
+
+  bool hasError{false};
+  if (subExpr) {
+    context_.Say(rsrc,
+        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+        atom.AsFortran(), subExpr->AsFortran());
+    hasError = true;
+  }
+  if (top.first == operation::Operator::Identity) {
+    // This is "x = y".
+    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+          atom.AsFortran());
+      hasError = true;
+    }
+  } else {
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    } else if (atomCount > 1) {
+      context_.Say(rsrc,
+          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    }
+  }
+
+  if (!hasError) {
+    CheckStorageOverlap(atom, nonAtom, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
+    const SomeExpr &cond, parser::CharBlock condSource,
+    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
+  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
+  const SomeExpr &atom{assign.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, arsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, alsrc);
+
+  auto top{GetTopLevelOperation(cond)};
+  // Missing arguments to operations would have been diagnosed by now.
+
+  switch (top.first) {
+  case operation::Operator::Associated:
+    if (atom != top.second.front()) {
+      context_.Say(assignSource,
+          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
+    }
+    break;
+  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
+  case operation::Operator::Eq:
+  case operation::Operator::Eqv:
+  // x ordop expr | expr ordop x
+  case operation::Operator::Lt:
+  case operation::Operator::Gt: {
+    const SomeExpr &arg0{top.second[0]};
+    const SomeExpr &arg1{top.second[1]};
+    if (IsSameOrConvertOf(arg0, atom)) {
+      CheckStorageOverlap(atom, {arg1}, condSource);
+    } else if (IsSameOrConvertOf(arg1, atom)) {
+      CheckStorageOverlap(atom, {arg0}, condSource);
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(assignSource,
+          "An argument of the %s operator should be the target of the assignment"_err_en_US,
+          operation::ToString(top.first));
+    }
+    break;
+  }
+  case operation::Operator::Identity:
+  case operation::Operator::True:
+  case operation::Operator::False:
+    break;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(condSource,
+        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
+        operation::ToString(top.first));
+    break;
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
+    const AnalyzedCondStmt &update, parser::CharBlock source) {
+  // The condition/statements must be:
+  // - cond: x equalop e      ift: x =  d     iff: -
+  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
+  // - cond: associated(x)    ift: x => expr  iff: -
+  // - cond: associated(x, e) ift: x => expr  iff: -
+
+  // The if-true statement must be present, and must be an assignment.
+  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
+  if (!maybeAssign) {
+    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
+    } else {
+      context_.Say(
+          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
+    }
+    return;
+  }
+  const evaluate::Assignment assign{*maybeAssign};
+  const SomeExpr &atom{assign.lhs};
+
+  CheckAtomicConditionalUpdateAssignment(
+      update.cond, update.source, assign, update.ift.source);
+
+  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
+
+  if (update.iff) {
+    context_.Say(update.iff.source,
+        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateOnly(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeUpdate->lhs};
+      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdate(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // Allowable forms are (single-statement):
+  // - if ...
+  // - x = (... ? ... : x)
+  // and two-statement:
+  // - r = cond ; if (r) ...
+
+  const parser::ExecutionPartConstruct *ust{nullptr}; // update
+  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
+
+  if (body.size() == 1) {
+    ust = &body.front();
+  } else if (body.size() == 2) {
+    cst = &body.front();
+    ust = &body.back();
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
+    return;
+  }
+
+  // Flang doesn't support conditional-expr yet, so all update statements
+  // are if-statements.
+
+  // IfStmt:        if (...) ...
+  // IfConstruct:   if (...) then ... endif
+  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
+  if (!maybeUpdate) {
+    context_.Say(source,
+        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
+    return;
+  }
+
+  AnalyzedCondStmt &update{*maybeUpdate};
+
+  if (SourcedActionStmt action{GetActionStmt(cst)}) {
+    // The "condition" statement must be `r = cond`.
+    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
+      if (maybeCond->lhs != update.cond) {
+        context_.Say(update.source,
+            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
+            maybeCond->lhs.AsFortran());
+      } else {
+        // If it's "r = ...; if (r) ..." then put the original condition
+        // in `update`.
+        update.cond = maybeCond->rhs;
+      }
+    } else {
+      context_.Say(action.source,
+          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
+    }
+  }
+
+  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
+
+  CheckAtomicConditionalUpdateStmt(update, source);
+  if (IsCheckForAssociated(update.cond)) {
+    if (!IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
+    }
+  } else {
+    if (IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
+    }
+  }
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
+      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
+      MakeAtomicAnalysisOp(Analysis::None));
+}
+
+void OmpStructureChecker::CheckAtomicUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() != 2) {
+    context_.Say(source,
+        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
+    return;
+  }
+
+  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
+  if (!uec || !cec) {
+    // Diagnostics already emitted.
+    return;
+  }
+  SourcedActionStmt uact{GetActionStmt(uec)};
+  SourcedActionStmt cact{GetActionStmt(cec)};
+  // The "dereferences" of std::optional are guaranteed to be valid after
+  // CheckUpdateCapture.
+  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
+  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
+
+  const SomeExpr &atom{update.lhs};
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  int action;
+
+  if (IsMaybeAtomicWrite(update)) {
+    action = Analysis::Write;
+    CheckAtomicWriteAssignment(update, uact.source);
+  } else {
+    action = Analysis::Update;
+    CheckAtomicUpdateAssignment(update, uact.source);
+  }
+  CheckAtomicCaptureAssignment(capture, atom, cact.source);
+
+  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+    context_.Say(cact.source,
+        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+    return;
+  }
+
+  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(action, update),
+        MakeAtomicAnalysisOp(Analysis::Read, capture));
+  } else {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(Analysis::Read, capture),
+        MakeAtomicAnalysisOp(action, update));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // There are two different variants of this:
+  // (1) conditional-update and capture separately:
+  //     This form only allows single-statement updates, i.e. the update
+  //     form "r = cond; if (r) ..." is not allowed.
+  // (2) conditional-update combined with capture in a single statement:
+  //     This form does allow the condition to be calculated separately,
+  //     i.e. "r = cond; if (r) ...".
+  // Regardless of what form it is, the actual update assignment is a
+  // proper write, i.e. "x = d", where d does not depend on x.
+
+  AnalyzedCondStmt update;
+  SourcedActionStmt capture;
+  bool captureAlways{true}, captureFirst{true};
+
+  auto extractCapture{[&]() {
+    capture = update.iff;
+    captureAlways = false;
+    update.iff = SourcedActionStmt{};
+  }};
+
+  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
+    // The non-update statement is either "r = cond" or the capture.
+    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
+      if (update.cond == maybeAssign->lhs) {
+        // If this is "r = cond; if (r) ...", then update the condition.
+        update.cond = maybeAssign->rhs;
+        update.source = action.source;
+        // In this form, the update and the capture are combined into
+        // an IF-THEN-ELSE statement.
+        extractCapture();
+      } else {
+        // Assume this is the capture-statement.
+        capture = action;
+      }
+    }
+  }};
+
+  if (body.size() == 2) {
+    // This could be
+    // - capture; conditional-update (in any order), or
+    // - r = cond; if (r) capture-update
+    const parser::ExecutionPartConstruct *st1{&body.front()};
+    const parser::ExecutionPartConstruct *st2{&body.back()};
+    // In either case, the conditional statement can be analyzed by
+    // AnalyzeConditionalStmt, whereas the other statement cannot.
+    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
+      update = *maybeUpdate1;
+      classifyNonUpdate(GetActionStmt(st2));
+      captureFirst = false;
+    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
+      update = *maybeUpdate2;
+      classifyNonUpdate(GetActionStmt(st1));
+    } else {
+      // None of the statements are conditional, this rules out the
+      // "r = cond; if (r) ..." and the "capture + conditional-update"
+      // variants. This could still be capture + write (which is classified
+      // as conditional-update-capture in the spec).
+      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
+      if (!uec || !cec) {
+        // Diagnostics already emitted.
+        return;
+      }
+      SourcedActionStmt uact{GetActionStmt(uec)};
+      SourcedActionStmt cact{GetActionStmt(cec)};
+      update.ift = uact;
+      capture = cact;
+      if (uec == st1) {
+        captureFirst = false;
+      }
+    }
+  } else if (body.size() == 1) {
+    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
+      update = *maybeUpdate;
+      // This is the form with update and capture combined into an IF-THEN-ELSE
+      // statement. The capture-statement is always the ELSE branch.
+      extractCapture();
+    } else {
+      goto invalid;
+    }
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
+    return;
+  invalid:
+    context_.Say(source,
+        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
+    return;
+  }
+
+  // The update must have a form `x = d` or `x => d`.
+  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
+    const SomeExpr &atom{maybeWrite->lhs};
+    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
+    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
+      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
+
+      if (IsPointerAssignment(*maybeWrite) !=
+          IsPointerAssignment(*maybeCapture)) {
+        context_.Say(capture.source,
+            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+        return;
+      }
+    } else {
+      if (!IsAssignment(capture.stmt)) {
+        context_.Say(capture.source,
+            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
+      }
+      return;
+    }
+  } else {
+    if (!IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
+    }
+    return;
+  }
+
+  // update.iff should be empty here, the capture statement should be
+  // stored in "capture".
+
+  // Fill out the analysis in the AST node.
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  bool condUnused{std::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
+          return true;
+        } else {
+          return false;
+        }
+      },
+      update.cond.u)};
+
+  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
+  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
+
+  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
+  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+
+  if (captureFirst) {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+  } else {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicRead(
+    const parser::OpenMPAtomicConstruct &x) {
+  // [6.0:190:5-7]
+  // A read structured block is read-statement, a read statement that has one
+  // of the following forms:
+  //   v = x
+  //   v => x
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Read cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
+      CheckAtomicReadAssignment(*maybeRead, action.source);
+
+      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
+        const SomeExpr &atom{*maybe};
+        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
+            MakeAtomicAnalysisOp(Analysis::None));
+      }
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC READ operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWrite(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Write cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeWrite->lhs};
+      CheckAtomicWriteAssignment(*maybeWrite, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC WRITE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdate(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &block{std::get<parser::Block>(x.t)};
+
+  bool isConditional{x.IsCompare()};
+  bool isCapture{x.IsCapture()};
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (isConditional && isCapture) {
+    CheckAtomicConditionalUpdateCapture(x, body, x.source);
+  } else if (isConditional) {
+    CheckAtomicConditionalUpdate(x, body, x.source);
+  } else if (isCapture) {
+    CheckAtomicUpdateCapture(x, body, x.source);
+  } else { // update-only
+    CheckAtomicUpdateOnly(x, body, x.source);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  if (visitedAtomicSource_.empty())
+    visitedAtomicSource_ = x.source;
+
+  // All of the following groups have the "exclusive" property, i.e. at
+  // most one clause from each group is allowed.
+  // The exclusivity-checking code should eventually be unified for all
+  // clauses, with clause groups defined in OMP.td.
+  std::array atomic{llvm::omp::Clause::OMPC_read,
+      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
+  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
+      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
+      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
+
+  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
+                          std::string_view name,
+                          const parser::OmpClauseList &clauses) {
+    const parser::OmpClause *present{nullptr};
+    for (const parser::OmpClause &clause : clauses.v) {
+      llvm::omp::Clause id{clause.Id()};
+      if (!llvm::is_contained(group, id)) {
+        continue;
+      }
+      if (present == nullptr) {
+        present = &clause;
+        continue;
+      } else if (id == present->Id()) {
+        // Ignore repetitions of the same clause, those will be diagnosed
+        // separately.
+        continue;
+      }
+      parser::MessageFormattedText txt(
+          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
+          name.data());
+      parser::Message message(clause.source, txt);
+      message.Attach(present->source,
+          "Previous clause from this group provided here"_en_US);
+      context_.Say(std::move(message));
+      return;
+    }
+  }};
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
+  llvm::omp::Clause kind{x.GetKind()};
+
+  checkExclusive(atomic, "atomic", dirSpec.Clauses());
+  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
+
+  switch (kind) {
+  case llvm::omp::Clause::OMPC_read:
+    CheckAtomicRead(x);
+    break;
+  case llvm::omp::Clause::OMPC_write:
+    CheckAtomicWrite(x);
+    break;
+  case llvm::omp::Clause::OMPC_update:
+    CheckAtomicUpdate(x);
+    break;
+  default:
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
new file mode 100644
index 0000000000000..b82e2f7342d85
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -0,0 +1,671 @@
+//===-- lib/Semantics/check-omp-loop.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks for constructs and clauses related to loops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <variant>
+
+namespace {
+using namespace Fortran;
+
+class AssociatedLoopChecker {
+public:
+  AssociatedLoopChecker(
+      semantics::SemanticsContext &context, std::int64_t level)
+      : context_{context}, level_{level} {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::DoConstruct &dc) {
+    level_--;
+    const auto &doStmt{
+        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
+    const auto &constructName{
+        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
+    if (constructName) {
+      constructNamesAndLevels_.emplace(
+          constructName.value().ToString(), level_);
+    }
+    if (level_ >= 0) {
+      if (dc.IsDoWhile()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
+      }
+      if (!dc.GetLoopControl()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
+      }
+    }
+    return true;
+  }
+
+  void Post(const parser::DoConstruct &dc) { level_++; }
+
+  bool Pre(const parser::CycleStmt &cyclestmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (cyclestmt.v) {
+      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second > 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ > 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::ExitStmt &exitStmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (exitStmt.v) {
+      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ >= 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "EXIT statement terminates associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
+    source_ = &actionstmt.source;
+    return true;
+  }
+
+private:
+  semantics::SemanticsContext &context_;
+  const parser::CharBlock *source_;
+  std::int64_t level_;
+  std::map<std::string, std::int64_t> constructNamesAndLevels_;
+};
+} // namespace
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::HasInvalidDistributeNesting(
+    const parser::OpenMPLoopConstruct &x) {
+  bool violation{false};
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
+    // `distribute` region has to be nested
+    if (!CurrentDirectiveIsNested()) {
+      violation = true;
+    } else {
+      // `distribute` region has to be strictly nested inside `teams`
+      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+        violation = true;
+      }
+    }
+  }
+  if (violation) {
+    context_.Say(beginDir.source,
+        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
+        "region."_err_en_US);
+  }
+}
+void OmpStructureChecker::HasInvalidLoopBinding(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
+    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+    for (const auto &clause : clauseList.v) {
+      if (const auto *bindClause{
+              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
+        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
+          context_.Say(beginDir.source, msg);
+        }
+      }
+    }
+  };
+
+  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
+      CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
+        "strictly nested inside a `TEAMS` region."_err_en_US);
+  }
+
+  if (OmpDirectiveSet{
+          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
+          .test(beginDir.v)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
+        "combined with a `TEAMS` construct."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
+  // Check the following:
+  //  The only OpenMP constructs that can be encountered during execution of
+  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
+  // construct and the `ordered` construct with the `simd` clause.
+
+  // Check if the parent context has the SIMD clause
+  // Please note that we use GetContext() instead of GetContextParent()
+  // because PushContextAndClauseSets() has not been called on the
+  // current context yet.
+  // TODO: Check for declare simd regions.
+  bool eligibleSIMD{false};
+  common::visit(
+      common::visitors{
+          // Allow `!$OMP ORDERED SIMD`
+          [&](const parser::OpenMPBlockConstruct &c) {
+            const auto &beginBlockDir{
+                std::get<parser::OmpBeginBlockDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
+            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
+              const auto &clauses{
+                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
+              for (const auto &clause : clauses.v) {
+                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
+                  eligibleSIMD = true;
+                  break;
+                }
+              }
+            }
+          },
+          [&](const parser::OpenMPStandaloneConstruct &c) {
+            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
+                    &c.u)}) {
+              llvm::omp::Directive dirId{ssc->v.DirId()};
+              if (dirId == llvm::omp::Directive::OMPD_ordered) {
+                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
+                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
+                    eligibleSIMD = true;
+                    break;
+                  }
+                }
+              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
+                eligibleSIMD = true;
+              }
+            }
+          },
+          // Allowing SIMD and loop construct
+          [&](const parser::OpenMPLoopConstruct &c) {
+            const auto &beginLoopDir{
+                std::get<parser::OmpBeginLoopDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
+              eligibleSIMD = true;
+            }
+          },
+          [&](const parser::OpenMPAtomicConstruct &c) {
+            // Allow `!$OMP ATOMIC`
+            eligibleSIMD = true;
+          },
+          [&](const auto &c) {},
+      },
+      c.u);
+  if (!eligibleSIMD) {
+    context_.Say(parser::FindSourceLocation(c),
+        "The only OpenMP constructs that can be encountered during execution "
+        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
+        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
+        "construct with the `SIMD` clause."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
+  loopStack_.push_back(&x);
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
+
+  // check matching, End directive is optional
+  if (const auto &endLoopDir{
+          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
+    const auto &endDir{
+        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
+
+    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
+
+    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
+  }
+
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    EnterDirectiveNest(SIMDNest);
+  }
+
+  // Combined target loop constructs are target device constructs. Keep track of
+  // whether any such construct has been visited to later check that REQUIRES
+  // directives for target-related options don't appear after them.
+  if (llvm::omp::allTargetSet.test(beginDir.v)) {
+    deviceConstructFound_ = true;
+  }
+
+  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
+    // 2.7.1 do-clause -> private-clause |
+    //                    firstprivate-clause |
+    //                    lastprivate-clause |
+    //                    linear-clause |
+    //                    reduction-clause |
+    //                    schedule-clause |
+    //                    collapse-clause |
+    //                    ordered-clause
+
+    // nesting check
+    HasInvalidWorksharingNesting(
+        beginDir.source, llvm::omp::nestedWorkshareErrSet);
+  }
+  SetLoopInfo(x);
+
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &doConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+    }
+  }
+  CheckLoopItrVariableIsInt(x);
+  CheckAssociatedLoopConstraints(x);
+  HasInvalidDistributeNesting(x);
+  HasInvalidLoopBinding(x);
+  if (CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
+  }
+  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
+      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
+    CheckDistLinear(x);
+  }
+}
+
+const parser::Name OmpStructureChecker::GetLoopIndex(
+    const parser::DoConstruct *x) {
+  using Bounds = parser::LoopControl::Bounds;
+  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
+}
+
+void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const parser::DoConstruct *loop{&*loopConstruct};
+      if (loop && loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        SetLoopIv(itrVal.symbol);
+      }
+    }
+  }
+}
+
+void OmpStructureChecker::CheckLoopItrVariableIsInt(
+    const parser::OpenMPLoopConstruct &x) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            const auto *type{itrVal.symbol->GetType()};
+            if (!type->IsNumeric(TypeCategory::Integer)) {
+              context_.Say(itrVal.source,
+                  "The DO loop iteration"
+                  " variable must be of the type integer."_err_en_US,
+                  itrVal.ToString());
+            }
+          }
+        }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
+      }
+    }
+  }
+}
+
+std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+  std::int64_t orderedCollapseLevel{1};
+  std::int64_t orderedLevel{1};
+  std::int64_t collapseLevel{1};
+
+  for (const auto &clause : clauseList.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
+      if (const auto v{GetIntValue(collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+    if (const auto *orderedClause{
+            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
+      if (const auto v{GetIntValue(orderedClause->v)}) {
+        orderedLevel = *v;
+      }
+    }
+  }
+  if (orderedLevel >= collapseLevel) {
+    orderedCollapseLevel = orderedLevel;
+  } else {
+    orderedCollapseLevel = collapseLevel;
+  }
+  return orderedCollapseLevel;
+}
+
+void OmpStructureChecker::CheckAssociatedLoopConstraints(
+    const parser::OpenMPLoopConstruct &x) {
+  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
+  AssociatedLoopChecker checker{context_, ordCollapseLevel};
+  parser::Walk(x, checker);
+}
+
+void OmpStructureChecker::CheckDistLinear(
+    const parser::OpenMPLoopConstruct &x) {
+
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  SymbolSourceMap indexVars;
+
+  // Collect symbols of all the variables from linear clauses
+  for (auto &clause : clauses.v) {
+    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
+      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
+      GetSymbolsInObjectList(objects, indexVars);
+    }
+  }
+
+  if (!indexVars.empty()) {
+    // Get collapse level, if given, to find which loops are "associated."
+    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
+    // Include the top loop if no collapse is specified
+    if (collapseVal == 0) {
+      collapseVal = 1;
+    }
+
+    // Match the loop index variables with the collected symbols from linear
+    // clauses.
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &loopConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+          if (loop->IsDoNormal()) {
+            const parser::Name &itrVal{GetLoopIndex(loop)};
+            if (itrVal.symbol) {
+              // Remove the symbol from the collected set
+              indexVars.erase(&itrVal.symbol->GetUltimate());
+            }
+            collapseVal--;
+            if (collapseVal == 0) {
+              break;
+            }
+          }
+          // Get the next DoConstruct if block is not empty.
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                   : nullptr;
+        }
+      }
+    }
+
+    // Show error for the remaining variables
+    for (auto &[symbol, source] : indexVars) {
+      const Symbol &root{GetAssociationRoot(*symbol)};
+      context_.Say(source,
+          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
+          root.name());
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  // A few semantic checks for InScan reduction are performed below as SCAN
+  // constructs inside LOOP may add the relevant information. Scan reduction is
+  // supported only in loop constructs, so same checks are not applicable to
+  // other directives.
+  using ReductionModifier = parser::OmpReductionModifier;
+  for (const auto &clause : clauseList.v) {
+    if (const auto *reductionClause{
+            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
+      auto &modifiers{OmpGetModifiers(reductionClause->v)};
+      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
+      if (maybeModifier &&
+          maybeModifier->v == ReductionModifier::Value::Inscan) {
+        const auto &objectList{
+            std::get<parser::OmpObjectList>(reductionClause->v.t)};
+        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
+          if (auto &symbol = name->symbol) {
+            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
+                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
+              context_.Say(name->source,
+                  "List item %s must appear in EXCLUSIVE or "
+                  "INCLUSIVE clause of an "
+                  "enclosed SCAN directive"_err_en_US,
+                  name->ToString());
+            }
+          }
+        };
+        for (const auto &ompObj : objectList.v) {
+          common::visit(
+              common::visitors{
+                  [&](const parser::Designator &designator) {
+                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
+                            designator)}) {
+                      checkReductionSymbolInScan(name);
+                    }
+                  },
+                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
+              },
+              ompObj.u);
+        }
+      }
+    }
+  }
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    ExitDirectiveNest(SIMDNest);
+  }
+  dirContext_.pop_back();
+
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top{loopStack_.back()};
+  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
+  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+}
+
+void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
+  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  // 2.7.1 end-do -> END DO [nowait-clause]
+  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
+  case llvm::omp::Directive::OMPD_do:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
+    break;
+  default:
+    // no clauses are allowed
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
+    dirContext_.pop_back();
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
+  unsigned version{context_.langOptions().OpenMPVersion};
+  llvm::omp::Directive dir{GetContext().directive};
+  parser::CharBlock clauseSource{GetContext().clauseSource};
+  const parser::OmpLinearModifier *linearMod{nullptr};
+
+  SymbolSourceMap symbols;
+  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckCrayPointee(objects, "LINEAR", false);
+  GetSymbolsInObjectList(objects, symbols);
+
+  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
+    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      context_.Say(source,
+          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
+          symbol->name(), desc.name.str());
+    }
+  }};
+
+  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
+    if (linearMod) {
+      // 2.7 Loop Construct Restriction
+      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
+        context_.Say(clauseSource,
+            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
+            ContextDirectiveAsFortran());
+        return;
+      }
+
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      for (auto &[symbol, source] : symbols) {
+        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
+          CheckIntegerNoRef(symbol, source);
+        } else {
+          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
+              !IsPolymorphic(*symbol)) {
+            context_.Say(source,
+                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
+                symbol->name(), desc.name.str());
+          }
+        }
+        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
+            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
+          if (!IsDummy(*symbol) || IsValue(*symbol)) {
+            context_.Say(source,
+                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
+                desc.name.str(), symbol->name());
+          }
+        }
+      } // for (symbol, source)
+
+      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
+        context_.Say(OmpGetModifierSource(modifiers, linearMod),
+            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
+            ThisVersion(version));
+      }
+    }
+  }
+
+  // OpenMP 5.2: Ordered clause restriction
+  if (const auto *clause{
+          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
+    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
+    if (orderedClause.v) {
+      return;
+    }
+  }
+
+  // OpenMP 5.2: Linear clause Restrictions
+  for (auto &[symbol, source] : symbols) {
+    if (!linearMod) {
+      // Already checked this with the modifier present.
+      CheckIntegerNoRef(symbol, source);
+    }
+    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
+      context_.Say(source,
+          "The list item `%s` must be a dummy argument"_err_en_US,
+          symbol->name());
+    }
+    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
+      context_.Say(source,
+          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
+          symbol->name());
+    }
+    if (FindCommonBlockContaining(*symbol)) {
+      context_.Say(source,
+          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
+          symbol->name());
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
+  Base::Enter(x);
+  loopStack_.push_back(&x);
+}
+
+void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top = loopStack_.back();
+  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
+  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+  Base::Leave(x);
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
new file mode 100644
index 0000000000000..03487da64f1bf
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -0,0 +1,548 @@
+//===-- lib/Semantics/check-omp-metadirective.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks for METADIRECTIVE and related constructs/clauses.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
+}
+
+void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
+  EnterDirectiveNest(ContextSelectorNest);
+
+  using SetName = parser::OmpTraitSetSelectorName;
+  std::map<SetName::Value, const SetName *> visited;
+
+  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
+    auto &name{std::get<SetName>(traitSet.t)};
+    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait set name %s in a context specifier"_err_en_US,
+          showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait set %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+    CheckTraitSetSelector(traitSet);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
+  ExitDirectiveNest(ContextSelectorNest);
+}
+
+const std::list<parser::OmpTraitProperty> &
+OmpStructureChecker::GetTraitPropertyList(
+    const parser::OmpTraitSelector &trait) {
+  static const std::list<parser::OmpTraitProperty> empty{};
+  auto &[_, maybeProps]{trait.t};
+  if (maybeProps) {
+    using PropertyList = std::list<parser::OmpTraitProperty>;
+    return std::get<PropertyList>(maybeProps->t);
+  } else {
+    return empty;
+  }
+}
+
+std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
+    const parser::OmpTraitProperty &property) {
+  using MaybeClause = std::optional<llvm::omp::Clause>;
+
+  // The parser for OmpClause will only succeed if the clause was
+  // given with all required arguments.
+  // If this is a string or complex extension with a clause name,
+  // treat it as a clause and let the trait checker deal with it.
+
+  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
+    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
+    if (id != llvm::omp::Clause::OMPC_unknown) {
+      return id;
+    } else {
+      return std::nullopt;
+    }
+  }};
+
+  return common::visit( //
+      common::visitors{
+          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
+            return getClauseFromString(x.v);
+          },
+          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
+            return x.value().Id();
+          },
+          [&](const parser::ScalarExpr &x) -> MaybeClause {
+            return std::nullopt;
+          },
+          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
+            using ExtProperty = parser::OmpTraitPropertyExtension;
+            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
+              return getClauseFromString(name->v);
+            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
+              return getClauseFromString(
+                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
+            }
+            return std::nullopt;
+          },
+      },
+      property.u);
+}
+
+void OmpStructureChecker::CheckTraitSelectorList(
+    const std::list<parser::OmpTraitSelector> &traits) {
+  // [6.0:322:20]
+  // Each trait-selector-name may only be specified once in a trait selector
+  // set.
+
+  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
+  using TraitName = parser::OmpTraitSelectorName;
+  using BareName = decltype(TraitName::u);
+  std::map<BareName, const TraitName *> visited;
+
+  for (const parser::OmpTraitSelector &trait : traits) {
+    auto &name{std::get<TraitName>(trait.t)};
+
+    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait name %s in a trait set"_err_en_US, showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSetSelector(
+    const parser::OmpTraitSetSelector &traitSet) {
+
+  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
+  //
+  // Construct      |     Simd, directive-name |      Yes |       No |    No |
+  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
+  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
+  //                |      Extension, Requires |          |          |       |
+  //                |                   Vendor |          |          |       |
+  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
+  //                |                Kind, Uid |          |          |       |
+  // User           |                Condition |       No |       No |   Yes |
+
+  struct TraitSetConfig {
+    std::set<parser::OmpTraitSelectorName::Value> allowed;
+    bool allowsDirectiveTraits;
+    bool allowsExtensionTraits;
+    bool allowsScore;
+  };
+
+  using SName = parser::OmpTraitSetSelectorName::Value;
+  using TName = parser::OmpTraitSelectorName::Value;
+
+  static const std::map<SName, TraitSetConfig> configs{
+      {SName::Construct, //
+          {{TName::Simd}, true, false, false}},
+      {SName::Device, //
+          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
+      {SName::Implementation, //
+          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
+               TName::Vendor},
+              false, true, true}},
+      {SName::Target_Device, //
+          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
+               TName::Uid},
+              false, true, false}},
+      {SName::User, //
+          {{TName::Condition}, false, false, true}},
+  };
+
+  auto checkTraitSet{[&](const TraitSetConfig &config) {
+    auto &[setName, traits]{traitSet.t};
+    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
+
+    // Check if there are any duplicate traits.
+    CheckTraitSelectorList(traits);
+
+    for (const parser::OmpTraitSelector &trait : traits) {
+      // Don't use structured bindings here, because they cannot be captured
+      // before C++20.
+      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
+      auto &maybeProps =
+          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
+              trait.t);
+
+      // Check allowed traits
+      common::visit( //
+          common::visitors{
+              [&](parser::OmpTraitSelectorName::Value v) {
+                if (!config.allowed.count(v)) {
+                  context_.Say(traitName.source,
+                      "%s is not a valid trait for %s trait set"_err_en_US,
+                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
+                }
+              },
+              [&](llvm::omp::Directive) {
+                if (!config.allowsDirectiveTraits) {
+                  context_.Say(traitName.source,
+                      "Directive name is not a valid trait for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+              [&](const std::string &) {
+                if (!config.allowsExtensionTraits) {
+                  context_.Say(traitName.source,
+                      "Extension traits are not valid for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+          },
+          traitName.u);
+
+      // Check score
+      if (maybeProps) {
+        auto &[maybeScore, _]{maybeProps->t};
+        if (maybeScore) {
+          CheckTraitScore(*maybeScore);
+        }
+      }
+
+      // Check the properties of the individual traits
+      CheckTraitSelector(traitSet, trait);
+    }
+  }};
+
+  checkTraitSet(
+      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
+}
+
+void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
+  // [6.0:322:23]
+  // A score-expression must be a non-negative constant integer expression.
+  if (auto value{GetIntValue(score)}; !value || value < 0) {
+    context_.Say(score.source,
+        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
+  }
+}
+
+bool OmpStructureChecker::VerifyTraitPropertyLists(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  using PropertyList = std::list<parser::OmpTraitProperty>;
+  auto &[traitName, maybeProps]{trait.t};
+
+  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
+                             const std::string &message) {
+    bool foundInvalid{false};
+    for (const parser::OmpTraitProperty &prop : properties) {
+      if (!isValid(prop)) {
+        if (foundInvalid) {
+          context_.Say(
+              prop.source, "More invalid properties are present"_err_en_US);
+          break;
+        }
+        context_.Say(prop.source, "%s"_err_en_US, message);
+        foundInvalid = true;
+      }
+    }
+    return !foundInvalid;
+  }};
+
+  bool invalid{false};
+
+  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
+    // Directive-name traits don't have properties.
+    if (maybeProps) {
+      context_.Say(trait.source,
+          "Directive-name traits cannot have properties"_err_en_US);
+      invalid = true;
+    }
+  }
+  // Ignore properties on extension traits.
+
+  // See `TraitSelectorParser` in openmp-parser.cpp
+  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
+    switch (*v) {
+    // name-list properties
+    case parser::OmpTraitSelectorName::Value::Arch:
+    case parser::OmpTraitSelectorName::Value::Extension:
+    case parser::OmpTraitSelectorName::Value::Isa:
+    case parser::OmpTraitSelectorName::Value::Kind:
+    case parser::OmpTraitSelectorName::Value::Uid:
+    case parser::OmpTraitSelectorName::Value::Vendor:
+      if (maybeProps) {
+        auto isName{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isName, "Trait property should be a name");
+      }
+      break;
+    // clause-list
+    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+    case parser::OmpTraitSelectorName::Value::Requires:
+    case parser::OmpTraitSelectorName::Value::Simd:
+      if (maybeProps) {
+        auto isClause{[&](const parser::OmpTraitProperty &prop) {
+          return GetClauseFromProperty(prop).has_value();
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isClause, "Trait property should be a clause");
+      }
+      break;
+    // expr-list
+    case parser::OmpTraitSelectorName::Value::Condition:
+    case parser::OmpTraitSelectorName::Value::Device_Num:
+      if (maybeProps) {
+        auto isExpr{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::ScalarExpr>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isExpr, "Trait property should be a scalar expression");
+      }
+      break;
+    } // switch
+  }
+
+  return !invalid;
+}
+
+void OmpStructureChecker::CheckTraitSelector(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  auto &[traitName, maybeProps]{trait.t};
+
+  // Only do the detailed checks if the property lists are valid.
+  if (VerifyTraitPropertyLists(traitSet, trait)) {
+    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
+        std::holds_alternative<std::string>(traitName.u)) {
+      // No properties here: directives don't have properties, and
+      // we don't implement any extension traits now.
+      return;
+    }
+
+    // Specific traits we want to check.
+    // Limitations:
+    // (1) The properties for these traits are defined in "Additional
+    // Definitions for the OpenMP API Specification". It's not clear how
+    // to define them in a portable way, and how to verify their validity,
+    // especially if they get replaced by their integer values (in case
+    // they are defined as enums).
+    // (2) These are entirely implementation-defined, and at the moment
+    // there is no known schema to validate these values.
+    auto v{std::get<TraitName::Value>(traitName.u)};
+    switch (v) {
+    case TraitName::Value::Arch:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Atomic_Default_Mem_Order:
+      CheckTraitADMO(traitSet, trait);
+      break;
+    case TraitName::Value::Condition:
+      CheckTraitCondition(traitSet, trait);
+      break;
+    case TraitName::Value::Device_Num:
+      CheckTraitDeviceNum(traitSet, trait);
+      break;
+    case TraitName::Value::Extension:
+      // Ignore
+      break;
+    case TraitName::Value::Isa:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Kind:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Requires:
+      CheckTraitRequires(traitSet, trait);
+      break;
+    case TraitName::Value::Simd:
+      CheckTraitSimd(traitSet, trait);
+      break;
+    case TraitName::Value::Uid:
+      // Unchecked, TBD(2)
+      break;
+    case TraitName::Value::Vendor:
+      // Unchecked, TBD(1)
+      break;
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitADMO(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single clause property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto clauseId{*GetClauseFromProperty(property)};
+    // Check that the clause belongs to the memory-order clause-set.
+    // Clause sets will hopefully be autogenerated at some point.
+    switch (clauseId) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+    case llvm::omp::Clause::OMPC_acquire:
+    case llvm::omp::Clause::OMPC_relaxed:
+    case llvm::omp::Clause::OMPC_release:
+    case llvm::omp::Clause::OMPC_seq_cst:
+      break;
+    default:
+      context_.Say(property.source,
+          "%s trait requires a clause from the memory-order clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitCondition(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
+
+    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
+    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
+      context_.Say(property.source,
+          "%s trait requires a single LOGICAL expression"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitDeviceNum(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  }
+  // No other checks at the moment.
+}
+
+void OmpStructureChecker::CheckTraitRequires(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_requires, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause from the requirement clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSimd(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_declare_simd, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()),
+          parser::ToUpperCaseLetters(
+              getDirectiveName(llvm::omp::OMPD_declare_simd)));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  EnterDirectiveNest(MetadirectiveNest);
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
+}
+
+void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  ExitDirectiveNest(MetadirectiveNest);
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 3abb5a304b00c..aa4cdb9d27019 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -7,27 +7,56 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
 #include "definable.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
-#include "flang/Evaluate/check-expression.h"
-#include "flang/Evaluate/expression.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran-features.h"
+
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
 #include <variant>
 
 namespace Fortran::semantics {
 
-template <typename T, typename U>
-static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
-  return !(e == f);
-}
+using namespace Fortran::semantics::omp;
 
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
@@ -53,66 +82,6 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
     CheckAllowedClause(llvm::omp::Y); \
   }
 
-std::string ThisVersion(unsigned version) {
-  std::string tv{
-      std::to_string(version / 10) + "." + std::to_string(version % 10)};
-  return "OpenMP v" + tv;
-}
-
-std::string TryVersion(unsigned version) {
-  return "try -fopenmp-version=" + std::to_string(version);
-}
-
-static const parser::Designator *GetDesignatorFromObj(
-    const parser::OmpObject &object) {
-  return std::get_if<parser::Designator>(&object.u);
-}
-
-static const parser::DataRef *GetDataRefFromObj(
-    const parser::OmpObject &object) {
-  if (auto *desg{GetDesignatorFromObj(object)}) {
-    return std::get_if<parser::DataRef>(&desg->u);
-  }
-  return nullptr;
-}
-
-static const parser::ArrayElement *GetArrayElementFromObj(
-    const parser::OmpObject &object) {
-  if (auto *dataRef{GetDataRefFromObj(object)}) {
-    using ElementIndirection = common::Indirection<parser::ArrayElement>;
-    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
-      return &ind->value();
-    }
-  }
-  return nullptr;
-}
-
-static bool IsVarOrFunctionRef(const MaybeExpr &expr) {
-  if (expr) {
-    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
-        evaluate::IsVariable(*expr);
-  } else {
-    return false;
-  }
-}
-
-static std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
-  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
-  // ForwardOwningPointer           typedExpr
-  // `- GenericExprWrapper          ^.get()
-  //    `- std::optional<Expr>      ^->v
-  return typedExpr.get()->v;
-}
-
-static std::optional<evaluate::DynamicType> GetDynamicType(
-    const parser::Expr &parserExpr) {
-  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
-    return maybeExpr->GetType();
-  } else {
-    return std::nullopt;
-  }
-}
-
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -172,85 +141,6 @@ class OmpWorkshareBlockChecker {
   parser::CharBlock source_;
 };
 
-class AssociatedLoopChecker {
-public:
-  AssociatedLoopChecker(SemanticsContext &context, std::int64_t level)
-      : context_{context}, level_{level} {}
-
-  template <typename T> bool Pre(const T &) { return true; }
-  template <typename T> void Post(const T &) {}
-
-  bool Pre(const parser::DoConstruct &dc) {
-    level_--;
-    const auto &doStmt{
-        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
-    const auto &constructName{
-        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
-    if (constructName) {
-      constructNamesAndLevels_.emplace(
-          constructName.value().ToString(), level_);
-    }
-    if (level_ >= 0) {
-      if (dc.IsDoWhile()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
-      }
-      if (!dc.GetLoopControl()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
-      }
-    }
-    return true;
-  }
-
-  void Post(const parser::DoConstruct &dc) { level_++; }
-
-  bool Pre(const parser::CycleStmt &cyclestmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (cyclestmt.v) {
-      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second > 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ > 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::ExitStmt &exitStmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (exitStmt.v) {
-      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ >= 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "EXIT statement terminates associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
-    source_ = &actionstmt.source;
-    return true;
-  }
-
-private:
-  SemanticsContext &context_;
-  const parser::CharBlock *source_;
-  std::int64_t level_;
-  std::map<std::string, std::int64_t> constructNamesAndLevels_;
-};
-
 // `OmpUnitedTaskDesignatorChecker` is used to check if the designator
 // can appear within the TASK construct
 class OmpUnitedTaskDesignatorChecker {
@@ -318,18 +208,6 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
   return CheckAllowed(clause);
 }
 
-bool OmpStructureChecker::IsCommonBlock(const Symbol &sym) {
-  return sym.detailsIf<CommonBlockDetails>() != nullptr;
-}
-
-bool OmpStructureChecker::IsVariableListItem(const Symbol &sym) {
-  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
-}
-
-bool OmpStructureChecker::IsExtendedListItem(const Symbol &sym) {
-  return IsVariableListItem(sym) || sym.IsSubprogram();
-}
-
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   // Definition of close nesting:
   //
@@ -371,60 +249,6 @@ bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   return false;
 }
 
-namespace {
-struct ContiguousHelper {
-  ContiguousHelper(SemanticsContext &context)
-      : fctx_(context.foldingContext()) {}
-
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
-    return Visit(x.value());
-  }
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Reference<Contained> &x) {
-    return Visit(x.get());
-  }
-  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
-    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
-  }
-  template <typename T>
-  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
-    return common::visit(
-        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
-  }
-  template <typename T> std::optional<bool> Visit(const T &) {
-    // Everything else.
-    return std::nullopt;
-  }
-
-private:
-  evaluate::FoldingContext &fctx_;
-};
-} // namespace
-
-// Return values:
-// - std::optional<bool>{true} if the object is known to be contiguous
-// - std::optional<bool>{false} if the object is known not to be contiguous
-// - std::nullopt if the object contiguity cannot be determined
-std::optional<bool> OmpStructureChecker::IsContiguous(
-    const parser::OmpObject &object) {
-  return common::visit( //
-      common::visitors{
-          [&](const parser::Name &x) {
-            // Any member of a common block must be contiguous.
-            return std::optional<bool>{true};
-          },
-          [&](const parser::Designator &x) {
-            evaluate::ExpressionAnalyzer ea{context_};
-            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
-              return ContiguousHelper{context_}.Visit(*maybeExpr);
-            }
-            return std::optional<bool>{};
-          },
-      },
-      object.u);
-}
-
 void OmpStructureChecker::CheckVariableListItem(
     const SymbolSourceMap &symbols) {
   for (auto &[symbol, source] : symbols) {
@@ -522,62 +346,6 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
   return false;
 }
 
-void OmpStructureChecker::HasInvalidDistributeNesting(
-    const parser::OpenMPLoopConstruct &x) {
-  bool violation{false};
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
-    // `distribute` region has to be nested
-    if (!CurrentDirectiveIsNested()) {
-      violation = true;
-    } else {
-      // `distribute` region has to be strictly nested inside `teams`
-      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-        violation = true;
-      }
-    }
-  }
-  if (violation) {
-    context_.Say(beginDir.source,
-        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
-        "region."_err_en_US);
-  }
-}
-void OmpStructureChecker::HasInvalidLoopBinding(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
-    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-    for (const auto &clause : clauseList.v) {
-      if (const auto *bindClause{
-              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
-        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
-          context_.Say(beginDir.source, msg);
-        }
-      }
-    }
-  };
-
-  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
-      CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
-        "strictly nested inside a `TEAMS` region."_err_en_US);
-  }
-
-  if (OmpDirectiveSet{
-          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
-          .test(beginDir.v)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
-        "combined with a `TEAMS` construct."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::HasInvalidTeamsNesting(
     const llvm::omp::Directive &dir, const parser::CharBlock &source) {
   if (!llvm::omp::nestedTeamsAllowedSet.test(dir)) {
@@ -668,16 +436,6 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
-  EnterDirectiveNest(MetadirectiveNest);
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
-}
-
-void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
-  ExitDirectiveNest(MetadirectiveNest);
-  dirContext_.pop_back();
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
@@ -717,85 +475,6 @@ void OmpStructureChecker::AddEndDirectiveClauses(
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
-  loopStack_.push_back(&x);
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  PushContextAndClauseSets(beginDir.source, beginDir.v);
-
-  // check matching, End directive is optional
-  if (const auto &endLoopDir{
-          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
-    const auto &endDir{
-        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
-
-    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
-
-    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
-  }
-
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    EnterDirectiveNest(SIMDNest);
-  }
-
-  // Combined target loop constructs are target device constructs. Keep track of
-  // whether any such construct has been visited to later check that REQUIRES
-  // directives for target-related options don't appear after them.
-  if (llvm::omp::allTargetSet.test(beginDir.v)) {
-    deviceConstructFound_ = true;
-  }
-
-  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    // 2.7.1 do-clause -> private-clause |
-    //                    firstprivate-clause |
-    //                    lastprivate-clause |
-    //                    linear-clause |
-    //                    reduction-clause |
-    //                    schedule-clause |
-    //                    collapse-clause |
-    //                    ordered-clause
-
-    // nesting check
-    HasInvalidWorksharingNesting(
-        beginDir.source, llvm::omp::nestedWorkshareErrSet);
-  }
-  SetLoopInfo(x);
-
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
-  }
-  CheckLoopItrVariableIsInt(x);
-  CheckAssociatedLoopConstraints(x);
-  HasInvalidDistributeNesting(x);
-  HasInvalidLoopBinding(x);
-  if (CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
-  }
-  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
-      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
-    CheckDistLinear(x);
-  }
-}
-const parser::Name OmpStructureChecker::GetLoopIndex(
-    const parser::DoConstruct *x) {
-  using Bounds = parser::LoopControl::Bounds;
-  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
-}
-void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
-    }
-  }
-}
-
 void OmpStructureChecker::CheckIteratorRange(
     const parser::OmpIteratorSpecifier &x) {
   // Check:
@@ -855,108 +534,6 @@ void OmpStructureChecker::CheckIteratorModifier(const parser::OmpIterator &x) {
   }
 }
 
-void OmpStructureChecker::CheckLoopItrVariableIsInt(
-    const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
-          }
-        }
-      }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
-  // Check the following:
-  //  The only OpenMP constructs that can be encountered during execution of
-  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
-  // construct and the `ordered` construct with the `simd` clause.
-
-  // Check if the parent context has the SIMD clause
-  // Please note that we use GetContext() instead of GetContextParent()
-  // because PushContextAndClauseSets() has not been called on the
-  // current context yet.
-  // TODO: Check for declare simd regions.
-  bool eligibleSIMD{false};
-  common::visit(
-      common::visitors{
-          // Allow `!$OMP ORDERED SIMD`
-          [&](const parser::OpenMPBlockConstruct &c) {
-            const auto &beginBlockDir{
-                std::get<parser::OmpBeginBlockDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
-            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
-              const auto &clauses{
-                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
-              for (const auto &clause : clauses.v) {
-                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
-                  eligibleSIMD = true;
-                  break;
-                }
-              }
-            }
-          },
-          [&](const parser::OpenMPStandaloneConstruct &c) {
-            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
-                    &c.u)}) {
-              llvm::omp::Directive dirId{ssc->v.DirId()};
-              if (dirId == llvm::omp::Directive::OMPD_ordered) {
-                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
-                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
-                    eligibleSIMD = true;
-                    break;
-                  }
-                }
-              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
-                eligibleSIMD = true;
-              }
-            }
-          },
-          // Allowing SIMD and loop construct
-          [&](const parser::OpenMPLoopConstruct &c) {
-            const auto &beginLoopDir{
-                std::get<parser::OmpBeginLoopDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
-              eligibleSIMD = true;
-            }
-          },
-          [&](const parser::OpenMPAtomicConstruct &c) {
-            // Allow `!$OMP ATOMIC`
-            eligibleSIMD = true;
-          },
-          [&](const auto &c) {},
-      },
-      c.u);
-  if (!eligibleSIMD) {
-    context_.Say(parser::FindSourceLocation(c),
-        "The only OpenMP constructs that can be encountered during execution "
-        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
-        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
-        "construct with the `SIMD` clause."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   // 2.12.5 Target Construct Restriction
   bool eligibleTarget{true};
@@ -1014,187 +591,6 @@ void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   }
 }
 
-std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-  std::int64_t orderedCollapseLevel{1};
-  std::int64_t orderedLevel{1};
-  std::int64_t collapseLevel{1};
-
-  for (const auto &clause : clauseList.v) {
-    if (const auto *collapseClause{
-            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
-      if (const auto v{GetIntValue(collapseClause->v)}) {
-        collapseLevel = *v;
-      }
-    }
-    if (const auto *orderedClause{
-            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
-      if (const auto v{GetIntValue(orderedClause->v)}) {
-        orderedLevel = *v;
-      }
-    }
-  }
-  if (orderedLevel >= collapseLevel) {
-    orderedCollapseLevel = orderedLevel;
-  } else {
-    orderedCollapseLevel = collapseLevel;
-  }
-  return orderedCollapseLevel;
-}
-
-void OmpStructureChecker::CheckAssociatedLoopConstraints(
-    const parser::OpenMPLoopConstruct &x) {
-  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
-  AssociatedLoopChecker checker{context_, ordCollapseLevel};
-  parser::Walk(x, checker);
-}
-
-void OmpStructureChecker::CheckDistLinear(
-    const parser::OpenMPLoopConstruct &x) {
-
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  SymbolSourceMap indexVars;
-
-  // Collect symbols of all the variables from linear clauses
-  for (auto &clause : clauses.v) {
-    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
-      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
-      GetSymbolsInObjectList(objects, indexVars);
-    }
-  }
-
-  if (!indexVars.empty()) {
-    // Get collapse level, if given, to find which loops are "associated."
-    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
-    // Include the top loop if no collapse is specified
-    if (collapseVal == 0) {
-      collapseVal = 1;
-    }
-
-    // Match the loop index variables with the collected symbols from linear
-    // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
-          }
-        }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
-      }
-    }
-
-    // Show error for the remaining variables
-    for (auto &[symbol, source] : indexVars) {
-      const Symbol &root{GetAssociationRoot(*symbol)};
-      context_.Say(source,
-          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
-          root.name());
-    }
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  // A few semantic checks for InScan reduction are performed below as SCAN
-  // constructs inside LOOP may add the relevant information. Scan reduction is
-  // supported only in loop constructs, so same checks are not applicable to
-  // other directives.
-  using ReductionModifier = parser::OmpReductionModifier;
-  for (const auto &clause : clauseList.v) {
-    if (const auto *reductionClause{
-            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
-      auto &modifiers{OmpGetModifiers(reductionClause->v)};
-      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
-      if (maybeModifier &&
-          maybeModifier->v == ReductionModifier::Value::Inscan) {
-        const auto &objectList{
-            std::get<parser::OmpObjectList>(reductionClause->v.t)};
-        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
-          if (auto &symbol = name->symbol) {
-            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
-                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
-              context_.Say(name->source,
-                  "List item %s must appear in EXCLUSIVE or "
-                  "INCLUSIVE clause of an "
-                  "enclosed SCAN directive"_err_en_US,
-                  name->ToString());
-            }
-          }
-        };
-        for (const auto &ompObj : objectList.v) {
-          common::visit(
-              common::visitors{
-                  [&](const parser::Designator &designator) {
-                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
-                            designator)}) {
-                      checkReductionSymbolInScan(name);
-                    }
-                  },
-                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
-              },
-              ompObj.u);
-        }
-      }
-    }
-  }
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    ExitDirectiveNest(SIMDNest);
-  }
-  dirContext_.pop_back();
-
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top{loopStack_.back()};
-  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
-  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
-  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  // 2.7.1 end-do -> END DO [nowait-clause]
-  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
-  case llvm::omp::Directive::OMPD_do:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
-    break;
-  case llvm::omp::Directive::OMPD_do_simd:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
-    dirContext_.pop_back();
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &endBlockDir{std::get<parser::OmpEndBlockDirective>(x.t)};
@@ -2348,7 +1744,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPFlushConstruct &x) {
 void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &x) {
   auto &flushList{std::get<std::optional<parser::OmpArgumentList>>(x.v.t)};
 
-  auto isVariableListItemOrCommonBlock{[this](const Symbol &sym) {
+  auto isVariableListItemOrCommonBlock{[](const Symbol &sym) {
     return IsVariableListItem(sym) ||
         sym.detailsIf<semantics::CommonBlockDetails>();
   }};
@@ -2659,1488 +2055,46 @@ void OmpStructureChecker::CheckCancellationNest(
           parser::ToUpperCaseLetters(typeName.str()));
       break;
     default:
-      // This is diagnosed later.
-      return;
-    }
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
-  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  case llvm::omp::Directive::OMPD_scope:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
-    break;
-  // 2.7.3 end-single-clause -> copyprivate-clause |
-  //                            nowait-clause
-  case llvm::omp::Directive::OMPD_single:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
-    break;
-  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
-  case llvm::omp::Directive::OMPD_workshare:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_workshare);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-// TODO: Verify the popping of dirContext requirement after nowait
-// implementation, as there is an implicit barrier at the end of the worksharing
-// constructs unless a nowait clause is specified. Only OMPD_end_single and
-// end_workshareare popped as they are pushed while entering the
-// EndBlockDirective.
-void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
-    dirContext_.pop_back();
-  }
-}
-
-/// parser::Block is a list of executable constructs, parser::BlockConstruct
-/// is Fortran's BLOCK/ENDBLOCK construct.
-/// Strip the outermost BlockConstructs, return the reference to the Block
-/// in the executable part of the innermost of the stripped constructs.
-/// Specifically, if the given `block` has a single entry (it's a list), and
-/// the entry is a BlockConstruct, get the Block contained within. Repeat
-/// this step as many times as possible.
-static const parser::Block &GetInnermostExecPart(const parser::Block &block) {
-  const parser::Block *iter{&block};
-  while (iter->size() == 1) {
-    const parser::ExecutionPartConstruct &ep{iter->front()};
-    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
-      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
-      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
-        iter = &std::get<parser::Block>(bc->value().t);
-        continue;
-      }
-    }
-    break;
-  }
-  return *iter;
-}
-
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
-struct AnalyzedCondStmt {
-  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
-  parser::CharBlock source;
-  SourcedActionStmt ift, iff;
-};
-
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
-// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
-// is that the ActionStmt will be either an assignment or a pointer-assignment,
-// otherwise return std::nullopt.
-// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
-// the "typedAssignment" is unset. This can happen if there are semantic errors
-// in the purported assignment.
-static std::optional<evaluate::Assignment> GetEvaluateAssignment(
-    const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
-
-  return common::visit(
-      [](auto &&s) -> std::optional<evaluate::Assignment> {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>) {
-          const TypedAssignment &typed{s.value().typedAssignment};
-          // ForwardOwningPointer                 typedAssignment
-          // `- GenericAssignmentWrapper          ^.get()
-          //    `- std::optional<Assignment>      ^->v
-          return typed.get()->v;
-        } else {
-          return std::nullopt;
-        }
-      },
-      x->u);
-}
-
-// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
-// to separate cases where the source has something that looks like an
-// assignment, but is semantically wrong (diagnosed by general semantic
-// checks), and where the source has some other statement (which we want
-// to report as "should be an assignment").
-static bool IsAssignment(const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return false;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-
-  return common::visit(
-      [](auto &&s) -> bool {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>;
-      },
-      x->u);
-}
-
-static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  // Extract the evaluate::Expr from ScalarLogicalExpr.
-  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
-    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
-    const parser::Expr &expr{logical.thing.thing.value()};
-    return GetEvaluateExpr(expr);
-  }};
-
-  // Recognize either
-  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
-  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
-
-  if (auto &&action{GetActionStmt(x)}) {
-    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
-            &action.stmt->u)}) {
-      const parser::IfStmt &s{ifs->value()};
-      auto &&maybeCond{
-          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
-      auto &thenStmt{
-          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
-      if (maybeCond) {
-        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
-            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
-            SourcedActionStmt{}};
-      }
-    }
-    return std::nullopt;
-  }
-
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    if (auto *ifc{
-            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
-      using ElseBlock = parser::IfConstruct::ElseBlock;
-      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
-      const parser::IfConstruct &s{ifc->value()};
-
-      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
-        // Not expecting any else-if statements.
-        return std::nullopt;
-      }
-      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
-      auto &&maybeCond{getFromLogical(
-          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
-      if (!maybeCond) {
-        return std::nullopt;
-      }
-
-      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)),
-            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
-        if (result.ift.stmt && result.iff.stmt) {
-          return result;
-        }
-      } else {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
-        if (result.ift.stmt) {
-          return result;
-        }
-      }
-    }
-    return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
-    parser::CharBlock source) {
-  // Find => in the range, if not found, find = that is not a part of
-  // <=, >=, ==, or /=.
-  auto trim{[](std::string_view v) {
-    const char *begin{v.data()};
-    const char *end{begin + v.size()};
-    while (*begin == ' ' && begin != end) {
-      ++begin;
-    }
-    while (begin != end && end[-1] == ' ') {
-      --end;
-    }
-    assert(begin != end && "Source should not be empty");
-    return parser::CharBlock(begin, end - begin);
-  }};
-
-  std::string_view sv(source.begin(), source.size());
-
-  if (auto where{sv.find("=>")}; where != sv.npos) {
-    std::string_view lhs(sv.data(), where);
-    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
-    return std::make_pair(trim(lhs), trim(rhs));
-  }
-
-  // Go backwards, since all the exclusions above end with a '='.
-  for (size_t next{source.size()}; next > 1; --next) {
-    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
-      std::string_view lhs(sv.data(), next - 1);
-      std::string_view rhs(sv.data() + next, sv.size() - next);
-      return std::make_pair(trim(lhs), trim(rhs));
-    }
-  }
-  llvm_unreachable("Could not find assignment operator");
-}
-
-namespace atomic {
-
-struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
-                                 std::vector<SomeExpr>, false> {
-  using Result = std::vector<SomeExpr>;
-  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
-  DesignatorCollector() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    // Once in a designator, don't traverse it any further (i.e. only
-    // collect top-level designators).
-    auto copy{x};
-    return Result{AsGenericExpr(std::move(copy))};
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    Result v(std::move(result));
-    auto moveAppend{[](auto &accum, auto &&other) {
-      for (auto &&s : other) {
-        accum.push_back(std::move(s));
-      }
-    }};
-    (moveAppend(v, std::move(results)), ...);
-    return v;
-  }
-};
-
-struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
-  using Base = evaluate::AnyTraverse<VariableFinder>;
-  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
-
-  using Base::operator();
-
-  template <typename T>
-  bool operator()(const evaluate::Designator<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-  template <typename T>
-  bool operator()(const evaluate::FunctionRef<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-private:
-  const SomeExpr &var;
-};
-} // namespace atomic
-
-static bool IsPointerAssignment(const evaluate::Assignment &x) {
-  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
-      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
-}
-
-namespace operation = Fortran::evaluate::operation;
-
-static bool IsCheckForAssociated(const SomeExpr &cond) {
-  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
-}
-
-static bool HasCommonDesignatorSymbols(
-    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
-  // Compare the designators used in "other" with the designators whose
-  // symbols are given in baseSyms.
-  // This is a part of the check if these two expressions can access the same
-  // storage: if the designators used in them are different enough, then they
-  // will be assumed not to access the same memory.
-  //
-  // Consider an (array element) expression x%y(w%z), the corresponding symbol
-  // vector will be {x, y, w, z} (i.e. the symbols for these names).
-  // Check whether this exact sequence appears anywhere in any the symbol
-  // vector for "other". This will be true for x(y) and x(y+1), so this is
-  // not a sufficient condition, but can be used to eliminate candidates
-  // before doing more exhaustive checks.
-  //
-  // If any of the symbols in this sequence are function names, assume that
-  // there is no storage overlap, mostly because it would be impossible in
-  // general to determine what storage the function will access.
-  // Note: if f is pure, then two calls to f will access the same storage
-  // when called with the same arguments. This check is not done yet.
-
-  if (llvm::any_of(
-          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
-    // If there is a function symbol in the chain then we can't infer much
-    // about the accessed storage.
-    return false;
-  }
-
-  auto isSubsequence{// Is u a subsequence of v.
-      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
-        size_t us{u.size()}, vs{v.size()};
-        if (us > vs) {
-          return false;
-        }
-        for (size_t off{0}; off != vs - us + 1; ++off) {
-          bool same{true};
-          for (size_t i{0}; i != us; ++i) {
-            if (u[i] != v[off + i]) {
-              same = false;
-              break;
-            }
-          }
-          if (same) {
-            return true;
-          }
-        }
-        return false;
-      }};
-
-  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
-  return isSubsequence(baseSyms, otherSyms);
-}
-
-static bool HasCommonTopLevelDesignators(
-    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
-  // Compare designators directly as expressions. This will ensure
-  // that x(y) and x(y+1) are not flagged as overlapping, whereas
-  // the symbol vectors for both of these would be identical.
-  std::vector<SomeExpr> otherDsgs{atomic::DesignatorCollector{}(other)};
-
-  for (auto &s : baseDsgs) {
-    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
-      return true;
-    }
-  }
-  return false;
-}
-
-static const SomeExpr *HasStorageOverlap(
-    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
-  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
-  std::vector<SomeExpr> baseDsgs{atomic::DesignatorCollector{}(base)};
-
-  for (const SomeExpr &expr : exprs) {
-    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
-      continue;
-    }
-    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
-      return &expr;
-    }
-  }
-  return nullptr;
-}
-
-static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
-  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
-  // for example.
-  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
-}
-
-static bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
-  return atomic::VariableFinder{sub}(super);
-}
-
-static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
-  if (value) {
-    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
-        evaluate::GenericExprWrapper::Deleter);
-  }
-}
-
-static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
-    std::optional<evaluate::Assignment> value) {
-  if (value) {
-    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
-        evaluate::GenericAssignmentWrapper::Deleter);
-  }
-}
-
-static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
-    int what,
-    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
-  parser::OpenMPAtomicConstruct::Analysis::Op operation;
-  operation.what = what;
-  SetAssignment(operation.assign, maybeAssign);
-  return operation;
-}
-
-static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
-    const SomeExpr &atom, const MaybeExpr &cond,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
-  // Defined in flang/include/flang/Parser/parse-tree.h
-  //
-  // struct Analysis {
-  //   struct Kind {
-  //     static constexpr int None = 0;
-  //     static constexpr int Read = 1;
-  //     static constexpr int Write = 2;
-  //     static constexpr int Update = Read | Write;
-  //     static constexpr int Action = 3; // Bits containing N, R, W, U
-  //     static constexpr int IfTrue = 4;
-  //     static constexpr int IfFalse = 8;
-  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
-  //   };
-  //   struct Op {
-  //     int what;
-  //     TypedAssignment assign;
-  //   };
-  //   TypedExpr atom, cond;
-  //   Op op0, op1;
-  // };
-
-  parser::OpenMPAtomicConstruct::Analysis an;
-  SetExpr(an.atom, atom);
-  SetExpr(an.cond, cond);
-  an.op0 = std::move(op0);
-  an.op1 = std::move(op1);
-  return an;
-}
-
-void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
-    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
-    parser::CharBlock source) {
-  if (auto *expr{HasStorageOverlap(base, exprs)}) {
-    context_.Say(source,
-        "Within atomic operation %s and %s access the same storage"_warn_en_US,
-        base.AsFortran(), expr->AsFortran());
-  }
-}
-
-void OmpStructureChecker::ErrorShouldBeVariable(
-    const MaybeExpr &expr, parser::CharBlock source) {
-  if (expr) {
-    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
-        expr->AsFortran());
-  } else {
-    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
-  }
-}
-
-/// Check if `expr` satisfies the following conditions for x and v:
-///
-/// [6.0:189:10-12]
-/// - x and v (as applicable) are either scalar variables or
-///   function references with scalar data pointer result of non-character
-///   intrinsic type or variables that are non-polymorphic scalar pointers
-///   and any length type parameter must be constant.
-void OmpStructureChecker::CheckAtomicType(
-    SymbolRef sym, parser::CharBlock source, std::string_view name) {
-  const DeclTypeSpec *typeSpec{sym->GetType()};
-  if (!typeSpec) {
-    return;
-  }
-
-  if (!IsPointer(sym)) {
-    using Category = DeclTypeSpec::Category;
-    Category cat{typeSpec->category()};
-    if (cat == Category::Character) {
-      context_.Say(source,
-          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
-    } else if (cat != Category::Numeric && cat != Category::Logical) {
-      context_.Say(source,
-          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
-    }
-    return;
-  }
-
-  // Variable is a pointer.
-  if (typeSpec->IsPolymorphic()) {
-    context_.Say(source,
-        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
-        name);
-    return;
-  }
-
-  // Go over all length parameters, if any, and check if they are
-  // explicit.
-  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
-    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
-          // "entry" is a map entry
-          return entry.second.isLen() && !entry.second.isExplicit();
-        })) {
-      context_.Say(source,
-          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
-          name);
-    }
-  }
-}
-
-void OmpStructureChecker::CheckAtomicVariable(
-    const SomeExpr &atom, parser::CharBlock source) {
-  if (atom.Rank() != 0) {
-    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
-        atom.AsFortran());
-  }
-
-  std::vector<SomeExpr> dsgs{atomic::DesignatorCollector{}(atom)};
-  assert(dsgs.size() == 1 && "Should have a single top-level designator");
-  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
-
-  CheckAtomicType(syms.back(), source, atom.AsFortran());
-
-  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
-    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
-        atom.AsFortran());
-  }
-}
-
-std::pair<const parser::ExecutionPartConstruct *,
-    const parser::ExecutionPartConstruct *>
-OmpStructureChecker::CheckUpdateCapture(
-    const parser::ExecutionPartConstruct *ec1,
-    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
-  // Decide which statement is the atomic update and which is the capture.
-  //
-  // The two allowed cases are:
-  //   x = ...      atomic-var = ...
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  // or
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  //   x = ...      atomic-var = ...
-  //
-  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
-  // (which makes more sense, as it captures the original value of the atomic
-  // variable).
-  //
-  // If the two statements don't fit these criteria, return a pair of default-
-  // constructed values.
-  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
-      const parser::ExecutionPartConstruct *>;
-
-  SourcedActionStmt act1{GetActionStmt(ec1)};
-  SourcedActionStmt act2{GetActionStmt(ec2)};
-  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
-  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
-  if (!maybeAssign1 || !maybeAssign2) {
-    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
-      context_.Say(source,
-          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
-    }
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  auto as1{*maybeAssign1}, as2{*maybeAssign2};
-
-  auto isUpdateCapture{
-      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
-        return IsSameOrConvertOf(c.rhs, u.lhs);
-      }};
-
-  // Do some checks that narrow down the possible choices for the update
-  // and the capture statements. This will help to emit better diagnostics.
-  // 1. An assignment could be an update (cbu) if the left-hand side is a
-  //    subexpression of the right-hand side.
-  // 2. An assignment could be a capture (cbc) if the right-hand side is
-  //    a variable (or a function ref), with potential type conversions.
-  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
-  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
-  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
-  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
-
-  // We want to diagnose cases where both assignments cannot be an update,
-  // or both cannot be a capture, as well as cases where either assignment
-  // cannot be any of these two.
-  //
-  // If we organize these boolean values into a matrix
-  //   |cbu1 cbu2|
-  //   |cbc1 cbc2|
-  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
-  // row or column, including the case where everything is zero. All these
-  // cases correspond to the determinant of the matrix being 0, which suggests
-  // that checking the det may be a convenient diagnostic check. There is only
-  // one additional case where the det is 0, which is when the matrix is all 1
-  // ("true"). The "all true" case represents the situation where both
-  // assignments could be an update as well as a capture. On the other hand,
-  // whenever det != 0, the roles of the update and the capture can be
-  // unambiguously assigned to as1 and as2 [1].
-  //
-  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
-  // det = 0, leaving 6 cases where det != 0:
-  //   0 1   0 1   1 0   1 0   1 1   1 1
-  //   1 0   1 1   0 1   1 1   0 1   1 0
-  // In each case the classification is unambiguous.
-
-  //     |cbu1 cbu2|
-  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
-  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
-
-  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
-                                  const std::string &expr) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
-        expr);
-  }};
-
-  auto errorNeitherWorks{[&]() {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
-  }};
-
-  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
-    // If det != 0, then the checks unambiguously suggest a specific
-    // categorization.
-    // If det == 0, then this function should be called only if the
-    // checks haven't ruled out any possibility, i.e. when both assigments
-    // could still be either updates or captures.
-    if (det > 0) {
-      // as1 is update, as2 is capture
-      if (isUpdateCapture(as1, as2)) {
-        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
-      } else {
-        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else if (det < 0) {
-      // as2 is update, as1 is capture
-      if (isUpdateCapture(as2, as1)) {
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      } else {
-        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else {
-      bool updateFirst{isUpdateCapture(as1, as2)};
-      bool captureFirst{isUpdateCapture(as2, as1)};
-      if (updateFirst && captureFirst) {
-        // If both assignment could be the update and both could be the
-        // capture, emit a warning about the ambiguity.
-        context_.Say(act1.source,
-            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      }
-      if (updateFirst != captureFirst) {
-        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
-        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
-        return std::make_pair(upd, cap);
-      }
-      assert(!updateFirst && !captureFirst);
-      errorNeitherWorks();
-      return std::make_pair(nullptr, nullptr);
-    }
-  }};
-
-  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
-    return makeSelectionFromDet(det);
-  }
-  assert(det == 0 && "Prior checks should have covered det != 0");
-
-  // If neither of the statements is an RMW update, it could still be a
-  // "write" update. Pretty much any assignment can be a write update, so
-  // recompute det with cbu1 = cbu2 = true.
-  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
-    return makeSelectionFromDet(writeDet);
-  }
-
-  // It's only errors from here on.
-
-  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
-    errorNeitherWorks();
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // The remaining cases are that
-  // - no candidate for update, or for capture,
-  // - one of the assigments cannot be anything.
-
-  if (!cbu1 && !cbu2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  } else if (!cbc1 && !cbc2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
-    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
-    context_.Say(src,
-        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // All cases should have been covered.
-  llvm_unreachable("Unchecked condition");
-}
-
-void OmpStructureChecker::CheckAtomicCaptureAssignment(
-    const evaluate::Assignment &capture, const SomeExpr &atom,
-    parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &cap{capture.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, rsrc);
-    // This part should have been checked prior to calling this function.
-    assert(*GetConvertInput(capture.rhs) == atom &&
-        "This cannot be a capture assignment");
-    CheckStorageOverlap(atom, {cap}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicReadAssignment(
-    const evaluate::Assignment &read, parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-
-  if (auto maybe{GetConvertInput(read.rhs)}) {
-    const SomeExpr &atom{*maybe};
-
-    if (!IsVarOrFunctionRef(atom)) {
-      ErrorShouldBeVariable(atom, rsrc);
-    } else {
-      CheckAtomicVariable(atom, rsrc);
-      CheckStorageOverlap(atom, {read.lhs}, source);
-    }
-  } else {
-    ErrorShouldBeVariable(read.rhs, rsrc);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicWriteAssignment(
-    const evaluate::Assignment &write, parser::CharBlock source) {
-  // [6.0:190:13-15]
-  // A write structured block is write-statement, a write statement that has
-  // one of the following forms:
-  //   x = expr
-  //   x => expr
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{write.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, lsrc);
-    CheckStorageOverlap(atom, {write.rhs}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateAssignment(
-    const evaluate::Assignment &update, parser::CharBlock source) {
-  // [6.0:191:1-7]
-  // An update structured block is update-statement, an update statement
-  // that has one of the following forms:
-  //   x = x operator expr
-  //   x = expr operator x
-  //   x = intrinsic-procedure-name (x)
-  //   x = intrinsic-procedure-name (x, expr-list)
-  //   x = intrinsic-procedure-name (expr-list, x)
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{update.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, lsrc);
-
-  std::pair<operation::Operator, std::vector<SomeExpr>> top{
-      operation::Operator::Unknown, {}};
-  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
-    top = GetTopLevelOperation(*maybeInput);
-  }
-  switch (top.first) {
-  case operation::Operator::Add:
-  case operation::Operator::Sub:
-  case operation::Operator::Mul:
-  case operation::Operator::Div:
-  case operation::Operator::And:
-  case operation::Operator::Or:
-  case operation::Operator::Eqv:
-  case operation::Operator::Neqv:
-  case operation::Operator::Min:
-  case operation::Operator::Max:
-  case operation::Operator::Identity:
-    break;
-  case operation::Operator::Call:
-    context_.Say(source,
-        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Convert:
-    context_.Say(source,
-        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Intrinsic:
-    context_.Say(source,
-        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Constant:
-  case operation::Operator::Unknown:
-    context_.Say(
-        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(source,
-        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
-        operation::ToString(top.first));
-    return;
-  }
-  // Check how many times `atom` occurs as an argument, if it's a subexpression
-  // of an argument, and collect the non-atom arguments.
-  std::vector<SomeExpr> nonAtom;
-  MaybeExpr subExpr;
-  auto atomCount{[&]() {
-    int count{0};
-    for (const SomeExpr &arg : top.second) {
-      if (IsSameOrConvertOf(arg, atom)) {
-        ++count;
-      } else {
-        if (!subExpr && IsSubexpressionOf(atom, arg)) {
-          subExpr = arg;
-        }
-        nonAtom.push_back(arg);
-      }
-    }
-    return count;
-  }()};
-
-  bool hasError{false};
-  if (subExpr) {
-    context_.Say(rsrc,
-        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
-        atom.AsFortran(), subExpr->AsFortran());
-    hasError = true;
-  }
-  if (top.first == operation::Operator::Identity) {
-    // This is "x = y".
-    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
-          atom.AsFortran());
-      hasError = true;
-    }
-  } else {
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    } else if (atomCount > 1) {
-      context_.Say(rsrc,
-          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    }
-  }
-
-  if (!hasError) {
-    CheckStorageOverlap(atom, nonAtom, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
-    const SomeExpr &cond, parser::CharBlock condSource,
-    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
-  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
-  const SomeExpr &atom{assign.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, arsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, alsrc);
-
-  auto top{GetTopLevelOperation(cond)};
-  // Missing arguments to operations would have been diagnosed by now.
-
-  switch (top.first) {
-  case operation::Operator::Associated:
-    if (atom != top.second.front()) {
-      context_.Say(assignSource,
-          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
-    }
-    break;
-  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
-  case operation::Operator::Eq:
-  case operation::Operator::Eqv:
-  // x ordop expr | expr ordop x
-  case operation::Operator::Lt:
-  case operation::Operator::Gt: {
-    const SomeExpr &arg0{top.second[0]};
-    const SomeExpr &arg1{top.second[1]};
-    if (IsSameOrConvertOf(arg0, atom)) {
-      CheckStorageOverlap(atom, {arg1}, condSource);
-    } else if (IsSameOrConvertOf(arg1, atom)) {
-      CheckStorageOverlap(atom, {arg0}, condSource);
-    } else {
-      assert(top.first != operation::Operator::Identity &&
-          "Handle this separately");
-      context_.Say(assignSource,
-          "An argument of the %s operator should be the target of the assignment"_err_en_US,
-          operation::ToString(top.first));
-    }
-    break;
-  }
-  case operation::Operator::Identity:
-  case operation::Operator::True:
-  case operation::Operator::False:
-    break;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(condSource,
-        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
-        operation::ToString(top.first));
-    break;
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
-    const AnalyzedCondStmt &update, parser::CharBlock source) {
-  // The condition/statements must be:
-  // - cond: x equalop e      ift: x =  d     iff: -
-  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
-  // - cond: associated(x)    ift: x => expr  iff: -
-  // - cond: associated(x, e) ift: x => expr  iff: -
-
-  // The if-true statement must be present, and must be an assignment.
-  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
-  if (!maybeAssign) {
-    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
-    } else {
-      context_.Say(
-          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
-    }
-    return;
-  }
-  const evaluate::Assignment assign{*maybeAssign};
-  const SomeExpr &atom{assign.lhs};
-
-  CheckAtomicConditionalUpdateAssignment(
-      update.cond, update.source, assign, update.ift.source);
-
-  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
-
-  if (update.iff) {
-    context_.Say(update.iff.source,
-        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateOnly(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeUpdate->lhs};
-      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdate(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // Allowable forms are (single-statement):
-  // - if ...
-  // - x = (... ? ... : x)
-  // and two-statement:
-  // - r = cond ; if (r) ...
-
-  const parser::ExecutionPartConstruct *ust{nullptr}; // update
-  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
-
-  if (body.size() == 1) {
-    ust = &body.front();
-  } else if (body.size() == 2) {
-    cst = &body.front();
-    ust = &body.back();
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
-    return;
-  }
-
-  // Flang doesn't support conditional-expr yet, so all update statements
-  // are if-statements.
-
-  // IfStmt:        if (...) ...
-  // IfConstruct:   if (...) then ... endif
-  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
-  if (!maybeUpdate) {
-    context_.Say(source,
-        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
-    return;
-  }
-
-  AnalyzedCondStmt &update{*maybeUpdate};
-
-  if (SourcedActionStmt action{GetActionStmt(cst)}) {
-    // The "condition" statement must be `r = cond`.
-    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
-      if (maybeCond->lhs != update.cond) {
-        context_.Say(update.source,
-            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
-            maybeCond->lhs.AsFortran());
-      } else {
-        // If it's "r = ...; if (r) ..." then put the original condition
-        // in `update`.
-        update.cond = maybeCond->rhs;
-      }
-    } else {
-      context_.Say(action.source,
-          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
-    }
-  }
-
-  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
-
-  CheckAtomicConditionalUpdateStmt(update, source);
-  if (IsCheckForAssociated(update.cond)) {
-    if (!IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
-    }
-  } else {
-    if (IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
-    }
-  }
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
-      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
-      MakeAtomicAnalysisOp(Analysis::None));
-}
-
-void OmpStructureChecker::CheckAtomicUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() != 2) {
-    context_.Say(source,
-        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
-    return;
-  }
-
-  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
-  if (!uec || !cec) {
-    // Diagnostics already emitted.
-    return;
-  }
-  SourcedActionStmt uact{GetActionStmt(uec)};
-  SourcedActionStmt cact{GetActionStmt(cec)};
-  // The "dereferences" of std::optional are guaranteed to be valid after
-  // CheckUpdateCapture.
-  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
-  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
-
-  const SomeExpr &atom{update.lhs};
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  int action;
-
-  if (IsMaybeAtomicWrite(update)) {
-    action = Analysis::Write;
-    CheckAtomicWriteAssignment(update, uact.source);
-  } else {
-    action = Analysis::Update;
-    CheckAtomicUpdateAssignment(update, uact.source);
-  }
-  CheckAtomicCaptureAssignment(capture, atom, cact.source);
-
-  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
-    context_.Say(cact.source,
-        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-    return;
-  }
-
-  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(action, update),
-        MakeAtomicAnalysisOp(Analysis::Read, capture));
-  } else {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(Analysis::Read, capture),
-        MakeAtomicAnalysisOp(action, update));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // There are two different variants of this:
-  // (1) conditional-update and capture separately:
-  //     This form only allows single-statement updates, i.e. the update
-  //     form "r = cond; if (r) ..." is not allowed.
-  // (2) conditional-update combined with capture in a single statement:
-  //     This form does allow the condition to be calculated separately,
-  //     i.e. "r = cond; if (r) ...".
-  // Regardless of what form it is, the actual update assignment is a
-  // proper write, i.e. "x = d", where d does not depend on x.
-
-  AnalyzedCondStmt update;
-  SourcedActionStmt capture;
-  bool captureAlways{true}, captureFirst{true};
-
-  auto extractCapture{[&]() {
-    capture = update.iff;
-    captureAlways = false;
-    update.iff = SourcedActionStmt{};
-  }};
-
-  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
-    // The non-update statement is either "r = cond" or the capture.
-    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
-      if (update.cond == maybeAssign->lhs) {
-        // If this is "r = cond; if (r) ...", then update the condition.
-        update.cond = maybeAssign->rhs;
-        update.source = action.source;
-        // In this form, the update and the capture are combined into
-        // an IF-THEN-ELSE statement.
-        extractCapture();
-      } else {
-        // Assume this is the capture-statement.
-        capture = action;
-      }
-    }
-  }};
-
-  if (body.size() == 2) {
-    // This could be
-    // - capture; conditional-update (in any order), or
-    // - r = cond; if (r) capture-update
-    const parser::ExecutionPartConstruct *st1{&body.front()};
-    const parser::ExecutionPartConstruct *st2{&body.back()};
-    // In either case, the conditional statement can be analyzed by
-    // AnalyzeConditionalStmt, whereas the other statement cannot.
-    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
-      update = *maybeUpdate1;
-      classifyNonUpdate(GetActionStmt(st2));
-      captureFirst = false;
-    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
-      update = *maybeUpdate2;
-      classifyNonUpdate(GetActionStmt(st1));
-    } else {
-      // None of the statements are conditional, this rules out the
-      // "r = cond; if (r) ..." and the "capture + conditional-update"
-      // variants. This could still be capture + write (which is classified
-      // as conditional-update-capture in the spec).
-      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
-      if (!uec || !cec) {
-        // Diagnostics already emitted.
-        return;
-      }
-      SourcedActionStmt uact{GetActionStmt(uec)};
-      SourcedActionStmt cact{GetActionStmt(cec)};
-      update.ift = uact;
-      capture = cact;
-      if (uec == st1) {
-        captureFirst = false;
-      }
-    }
-  } else if (body.size() == 1) {
-    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
-      update = *maybeUpdate;
-      // This is the form with update and capture combined into an IF-THEN-ELSE
-      // statement. The capture-statement is always the ELSE branch.
-      extractCapture();
-    } else {
-      goto invalid;
-    }
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
-    return;
-  invalid:
-    context_.Say(source,
-        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
-    return;
-  }
-
-  // The update must have a form `x = d` or `x => d`.
-  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
-    const SomeExpr &atom{maybeWrite->lhs};
-    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
-    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
-      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
-
-      if (IsPointerAssignment(*maybeWrite) !=
-          IsPointerAssignment(*maybeCapture)) {
-        context_.Say(capture.source,
-            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-        return;
-      }
-    } else {
-      if (!IsAssignment(capture.stmt)) {
-        context_.Say(capture.source,
-            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
-      }
-      return;
-    }
-  } else {
-    if (!IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
-    }
-    return;
-  }
-
-  // update.iff should be empty here, the capture statement should be
-  // stored in "capture".
-
-  // Fill out the analysis in the AST node.
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  bool condUnused{std::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
-          return true;
-        } else {
-          return false;
-        }
-      },
-      update.cond.u)};
-
-  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
-  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
-
-  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
-  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
-
-  if (captureFirst) {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
-  } else {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicRead(
-    const parser::OpenMPAtomicConstruct &x) {
-  // [6.0:190:5-7]
-  // A read structured block is read-statement, a read statement that has one
-  // of the following forms:
-  //   v = x
-  //   v => x
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Read cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
-      CheckAtomicReadAssignment(*maybeRead, action.source);
-
-      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
-        const SomeExpr &atom{*maybe};
-        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
-            MakeAtomicAnalysisOp(Analysis::None));
-      }
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+      // This is diagnosed later.
+      return;
     }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC READ operation should have a single statement"_err_en_US);
   }
 }
 
-void OmpStructureChecker::CheckAtomicWrite(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Write cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeWrite->lhs};
-      CheckAtomicWriteAssignment(*maybeWrite, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC WRITE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdate(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &block{std::get<parser::Block>(x.t)};
-
-  bool isConditional{x.IsCompare()};
-  bool isCapture{x.IsCapture()};
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (isConditional && isCapture) {
-    CheckAtomicConditionalUpdateCapture(x, body, x.source);
-  } else if (isConditional) {
-    CheckAtomicConditionalUpdate(x, body, x.source);
-  } else if (isCapture) {
-    CheckAtomicUpdateCapture(x, body, x.source);
-  } else { // update-only
-    CheckAtomicUpdateOnly(x, body, x.source);
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
-  if (visitedAtomicSource_.empty())
-    visitedAtomicSource_ = x.source;
-
-  // All of the following groups have the "exclusive" property, i.e. at
-  // most one clause from each group is allowed.
-  // The exclusivity-checking code should eventually be unified for all
-  // clauses, with clause groups defined in OMP.td.
-  std::array atomic{llvm::omp::Clause::OMPC_read,
-      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
-  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
-      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
-      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
-
-  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
-                          std::string_view name,
-                          const parser::OmpClauseList &clauses) {
-    const parser::OmpClause *present{nullptr};
-    for (const parser::OmpClause &clause : clauses.v) {
-      llvm::omp::Clause id{clause.Id()};
-      if (!llvm::is_contained(group, id)) {
-        continue;
-      }
-      if (present == nullptr) {
-        present = &clause;
-        continue;
-      } else if (id == present->Id()) {
-        // Ignore repetitions of the same clause, those will be diagnosed
-        // separately.
-        continue;
-      }
-      parser::MessageFormattedText txt(
-          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
-          name.data());
-      parser::Message message(clause.source, txt);
-      message.Attach(present->source,
-          "Previous clause from this group provided here"_en_US);
-      context_.Say(std::move(message));
-      return;
-    }
-  }};
-
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
-  llvm::omp::Clause kind{x.GetKind()};
-
-  checkExclusive(atomic, "atomic", dirSpec.Clauses());
-  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
-
-  switch (kind) {
-  case llvm::omp::Clause::OMPC_read:
-    CheckAtomicRead(x);
+void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
+  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  case llvm::omp::Directive::OMPD_scope:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
     break;
-  case llvm::omp::Clause::OMPC_write:
-    CheckAtomicWrite(x);
+  // 2.7.3 end-single-clause -> copyprivate-clause |
+  //                            nowait-clause
+  case llvm::omp::Directive::OMPD_single:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
     break;
-  case llvm::omp::Clause::OMPC_update:
-    CheckAtomicUpdate(x);
+  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
+  case llvm::omp::Directive::OMPD_workshare:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_workshare);
     break;
   default:
+    // no clauses are allowed
     break;
   }
 }
 
-void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
-  dirContext_.pop_back();
+// TODO: Verify the popping of dirContext requirement after nowait
+// implementation, as there is an implicit barrier at the end of the worksharing
+// constructs unless a nowait clause is specified. Only OMPD_end_single and
+// end_workshareare popped as they are pushed while entering the
+// EndBlockDirective.
+void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
+    dirContext_.pop_back();
+  }
 }
 
 // Clauses
@@ -5208,102 +3162,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
-  unsigned version{context_.langOptions().OpenMPVersion};
-  llvm::omp::Directive dir{GetContext().directive};
-  parser::CharBlock clauseSource{GetContext().clauseSource};
-  const parser::OmpLinearModifier *linearMod{nullptr};
-
-  SymbolSourceMap symbols;
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
-  CheckCrayPointee(objects, "LINEAR", false);
-  GetSymbolsInObjectList(objects, symbols);
-
-  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
-    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      context_.Say(source,
-          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
-          symbol->name(), desc.name.str());
-    }
-  }};
-
-  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
-    auto &modifiers{OmpGetModifiers(x.v)};
-    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
-    if (linearMod) {
-      // 2.7 Loop Construct Restriction
-      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
-        context_.Say(clauseSource,
-            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
-            ContextDirectiveAsFortran());
-        return;
-      }
-
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      for (auto &[symbol, source] : symbols) {
-        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
-          CheckIntegerNoRef(symbol, source);
-        } else {
-          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
-              !IsPolymorphic(*symbol)) {
-            context_.Say(source,
-                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
-                symbol->name(), desc.name.str());
-          }
-        }
-        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
-            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
-          if (!IsDummy(*symbol) || IsValue(*symbol)) {
-            context_.Say(source,
-                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
-                desc.name.str(), symbol->name());
-          }
-        }
-      } // for (symbol, source)
-
-      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
-        context_.Say(OmpGetModifierSource(modifiers, linearMod),
-            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
-            ThisVersion(version));
-      }
-    }
-  }
-
-  // OpenMP 5.2: Ordered clause restriction
-  if (const auto *clause{
-          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
-    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
-    if (orderedClause.v) {
-      return;
-    }
-  }
-
-  // OpenMP 5.2: Linear clause Restrictions
-  for (auto &[symbol, source] : symbols) {
-    if (!linearMod) {
-      // Already checked this with the modifier present.
-      CheckIntegerNoRef(symbol, source);
-    }
-    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
-      context_.Say(source,
-          "The list item `%s` must be a dummy argument"_err_en_US,
-          symbol->name());
-    }
-    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
-      context_.Say(source,
-          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
-          symbol->name());
-    }
-    if (FindCommonBlockContaining(*symbol)) {
-      context_.Say(source,
-          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
-          symbol->name());
-    }
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   unsigned version{context_.langOptions().OpenMPVersion};
   if (version >= 52) {
@@ -6012,503 +3870,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::OmpxBare &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
-  OmpVerifyModifiers(
-      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
-}
-
-void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
-  EnterDirectiveNest(ContextSelectorNest);
-
-  using SetName = parser::OmpTraitSetSelectorName;
-  std::map<SetName::Value, const SetName *> visited;
-
-  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
-    auto &name{std::get<SetName>(traitSet.t)};
-    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait set name %s in a context specifier"_err_en_US,
-          showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait set %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-    CheckTraitSetSelector(traitSet);
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
-  ExitDirectiveNest(ContextSelectorNest);
-}
-
-const std::list<parser::OmpTraitProperty> &
-OmpStructureChecker::GetTraitPropertyList(
-    const parser::OmpTraitSelector &trait) {
-  static const std::list<parser::OmpTraitProperty> empty{};
-  auto &[_, maybeProps]{trait.t};
-  if (maybeProps) {
-    using PropertyList = std::list<parser::OmpTraitProperty>;
-    return std::get<PropertyList>(maybeProps->t);
-  } else {
-    return empty;
-  }
-}
-
-std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
-    const parser::OmpTraitProperty &property) {
-  using MaybeClause = std::optional<llvm::omp::Clause>;
-
-  // The parser for OmpClause will only succeed if the clause was
-  // given with all required arguments.
-  // If this is a string or complex extension with a clause name,
-  // treat it as a clause and let the trait checker deal with it.
-
-  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
-    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
-    if (id != llvm::omp::Clause::OMPC_unknown) {
-      return id;
-    } else {
-      return std::nullopt;
-    }
-  }};
-
-  return common::visit( //
-      common::visitors{
-          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
-            return getClauseFromString(x.v);
-          },
-          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
-            return x.value().Id();
-          },
-          [&](const parser::ScalarExpr &x) -> MaybeClause {
-            return std::nullopt;
-          },
-          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
-            using ExtProperty = parser::OmpTraitPropertyExtension;
-            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
-              return getClauseFromString(name->v);
-            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
-              return getClauseFromString(
-                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
-            }
-            return std::nullopt;
-          },
-      },
-      property.u);
-}
-
-void OmpStructureChecker::CheckTraitSelectorList(
-    const std::list<parser::OmpTraitSelector> &traits) {
-  // [6.0:322:20]
-  // Each trait-selector-name may only be specified once in a trait selector
-  // set.
-
-  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
-  using TraitName = parser::OmpTraitSelectorName;
-  using BareName = decltype(TraitName::u);
-  std::map<BareName, const TraitName *> visited;
-
-  for (const parser::OmpTraitSelector &trait : traits) {
-    auto &name{std::get<TraitName>(trait.t)};
-
-    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait name %s in a trait set"_err_en_US, showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSetSelector(
-    const parser::OmpTraitSetSelector &traitSet) {
-
-  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
-  //
-  // Construct      |     Simd, directive-name |      Yes |       No |    No |
-  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
-  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
-  //                |      Extension, Requires |          |          |       |
-  //                |                   Vendor |          |          |       |
-  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
-  //                |                Kind, Uid |          |          |       |
-  // User           |                Condition |       No |       No |   Yes |
-
-  struct TraitSetConfig {
-    std::set<parser::OmpTraitSelectorName::Value> allowed;
-    bool allowsDirectiveTraits;
-    bool allowsExtensionTraits;
-    bool allowsScore;
-  };
-
-  using SName = parser::OmpTraitSetSelectorName::Value;
-  using TName = parser::OmpTraitSelectorName::Value;
-
-  static const std::map<SName, TraitSetConfig> configs{
-      {SName::Construct, //
-          {{TName::Simd}, true, false, false}},
-      {SName::Device, //
-          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
-      {SName::Implementation, //
-          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
-               TName::Vendor},
-              false, true, true}},
-      {SName::Target_Device, //
-          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
-               TName::Uid},
-              false, true, false}},
-      {SName::User, //
-          {{TName::Condition}, false, false, true}},
-  };
-
-  auto checkTraitSet{[&](const TraitSetConfig &config) {
-    auto &[setName, traits]{traitSet.t};
-    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
-
-    // Check if there are any duplicate traits.
-    CheckTraitSelectorList(traits);
-
-    for (const parser::OmpTraitSelector &trait : traits) {
-      // Don't use structured bindings here, because they cannot be captured
-      // before C++20.
-      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
-      auto &maybeProps =
-          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
-              trait.t);
-
-      // Check allowed traits
-      common::visit( //
-          common::visitors{
-              [&](parser::OmpTraitSelectorName::Value v) {
-                if (!config.allowed.count(v)) {
-                  context_.Say(traitName.source,
-                      "%s is not a valid trait for %s trait set"_err_en_US,
-                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
-                }
-              },
-              [&](llvm::omp::Directive) {
-                if (!config.allowsDirectiveTraits) {
-                  context_.Say(traitName.source,
-                      "Directive name is not a valid trait for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-              [&](const std::string &) {
-                if (!config.allowsExtensionTraits) {
-                  context_.Say(traitName.source,
-                      "Extension traits are not valid for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-          },
-          traitName.u);
-
-      // Check score
-      if (maybeProps) {
-        auto &[maybeScore, _]{maybeProps->t};
-        if (maybeScore) {
-          CheckTraitScore(*maybeScore);
-        }
-      }
-
-      // Check the properties of the individual traits
-      CheckTraitSelector(traitSet, trait);
-    }
-  }};
-
-  checkTraitSet(
-      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
-}
-
-void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
-  // [6.0:322:23]
-  // A score-expression must be a non-negative constant integer expression.
-  if (auto value{GetIntValue(score)}; !value || value < 0) {
-    context_.Say(score.source,
-        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
-  }
-}
-
-bool OmpStructureChecker::VerifyTraitPropertyLists(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  using PropertyList = std::list<parser::OmpTraitProperty>;
-  auto &[traitName, maybeProps]{trait.t};
-
-  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
-                             const std::string &message) {
-    bool foundInvalid{false};
-    for (const parser::OmpTraitProperty &prop : properties) {
-      if (!isValid(prop)) {
-        if (foundInvalid) {
-          context_.Say(
-              prop.source, "More invalid properties are present"_err_en_US);
-          break;
-        }
-        context_.Say(prop.source, "%s"_err_en_US, message);
-        foundInvalid = true;
-      }
-    }
-    return !foundInvalid;
-  }};
-
-  bool invalid{false};
-
-  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
-    // Directive-name traits don't have properties.
-    if (maybeProps) {
-      context_.Say(trait.source,
-          "Directive-name traits cannot have properties"_err_en_US);
-      invalid = true;
-    }
-  }
-  // Ignore properties on extension traits.
-
-  // See `TraitSelectorParser` in openmp-parser.cpp
-  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
-    switch (*v) {
-    // name-list properties
-    case parser::OmpTraitSelectorName::Value::Arch:
-    case parser::OmpTraitSelectorName::Value::Extension:
-    case parser::OmpTraitSelectorName::Value::Isa:
-    case parser::OmpTraitSelectorName::Value::Kind:
-    case parser::OmpTraitSelectorName::Value::Uid:
-    case parser::OmpTraitSelectorName::Value::Vendor:
-      if (maybeProps) {
-        auto isName{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isName, "Trait property should be a name");
-      }
-      break;
-    // clause-list
-    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
-    case parser::OmpTraitSelectorName::Value::Requires:
-    case parser::OmpTraitSelectorName::Value::Simd:
-      if (maybeProps) {
-        auto isClause{[&](const parser::OmpTraitProperty &prop) {
-          return GetClauseFromProperty(prop).has_value();
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isClause, "Trait property should be a clause");
-      }
-      break;
-    // expr-list
-    case parser::OmpTraitSelectorName::Value::Condition:
-    case parser::OmpTraitSelectorName::Value::Device_Num:
-      if (maybeProps) {
-        auto isExpr{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::ScalarExpr>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isExpr, "Trait property should be a scalar expression");
-      }
-      break;
-    } // switch
-  }
-
-  return !invalid;
-}
-
-void OmpStructureChecker::CheckTraitSelector(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  auto &[traitName, maybeProps]{trait.t};
-
-  // Only do the detailed checks if the property lists are valid.
-  if (VerifyTraitPropertyLists(traitSet, trait)) {
-    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
-        std::holds_alternative<std::string>(traitName.u)) {
-      // No properties here: directives don't have properties, and
-      // we don't implement any extension traits now.
-      return;
-    }
-
-    // Specific traits we want to check.
-    // Limitations:
-    // (1) The properties for these traits are defined in "Additional
-    // Definitions for the OpenMP API Specification". It's not clear how
-    // to define them in a portable way, and how to verify their validity,
-    // especially if they get replaced by their integer values (in case
-    // they are defined as enums).
-    // (2) These are entirely implementation-defined, and at the moment
-    // there is no known schema to validate these values.
-    auto v{std::get<TraitName::Value>(traitName.u)};
-    switch (v) {
-    case TraitName::Value::Arch:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Atomic_Default_Mem_Order:
-      CheckTraitADMO(traitSet, trait);
-      break;
-    case TraitName::Value::Condition:
-      CheckTraitCondition(traitSet, trait);
-      break;
-    case TraitName::Value::Device_Num:
-      CheckTraitDeviceNum(traitSet, trait);
-      break;
-    case TraitName::Value::Extension:
-      // Ignore
-      break;
-    case TraitName::Value::Isa:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Kind:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Requires:
-      CheckTraitRequires(traitSet, trait);
-      break;
-    case TraitName::Value::Simd:
-      CheckTraitSimd(traitSet, trait);
-      break;
-    case TraitName::Value::Uid:
-      // Unchecked, TBD(2)
-      break;
-    case TraitName::Value::Vendor:
-      // Unchecked, TBD(1)
-      break;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitADMO(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single clause property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto clauseId{*GetClauseFromProperty(property)};
-    // Check that the clause belongs to the memory-order clause-set.
-    // Clause sets will hopefully be autogenerated at some point.
-    switch (clauseId) {
-    case llvm::omp::Clause::OMPC_acq_rel:
-    case llvm::omp::Clause::OMPC_acquire:
-    case llvm::omp::Clause::OMPC_relaxed:
-    case llvm::omp::Clause::OMPC_release:
-    case llvm::omp::Clause::OMPC_seq_cst:
-      break;
-    default:
-      context_.Say(property.source,
-          "%s trait requires a clause from the memory-order clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitCondition(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
-
-    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
-    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
-      context_.Say(property.source,
-          "%s trait requires a single LOGICAL expression"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitDeviceNum(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  }
-  // No other checks at the moment.
-}
-
-void OmpStructureChecker::CheckTraitRequires(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_requires, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause from the requirement clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSimd(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_declare_simd, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()),
-          parser::ToUpperCaseLetters(
-              getDirectiveName(llvm::omp::OMPD_declare_simd)));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
   return llvm::omp::getOpenMPClauseName(clause);
 }
@@ -6519,39 +3880,6 @@ llvm::StringRef OmpStructureChecker::getDirectiveName(
   return llvm::omp::getOpenMPDirectiveName(directive, version);
 }
 
-const Symbol *OmpStructureChecker::GetObjectSymbol(
-    const parser::OmpObject &object) {
-  // Some symbols may be missing if the resolution failed, e.g. when an
-  // undeclared name is used with implicit none.
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    auto &last{GetLastName(*desg)};
-    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
-  }
-  return nullptr;
-}
-
-const Symbol *OmpStructureChecker::GetArgumentSymbol(
-    const parser::OmpArgument &argument) {
-  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
-    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
-      return GetObjectSymbol(*object);
-    }
-  }
-  return nullptr;
-}
-
-std::optional<parser::CharBlock> OmpStructureChecker::GetObjectSource(
-    const parser::OmpObject &object) {
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->source;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    return GetLastName(*desg).source;
-  }
-  return std::nullopt;
-}
-
 void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
   common::visit(
       common::visitors{
@@ -6861,7 +4189,7 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
 }
 
 void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
-  if (auto contig{IsContiguous(object)}; contig && !*contig) {
+  if (auto contig{IsContiguous(context_, object)}; contig && !*contig) {
     const parser::Name *name{GetObjectName(object)};
     assert(name && "Expecting name component");
     context_.Say(name->source,
@@ -6964,22 +4292,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::SelfMaps &x) {
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_self_maps);
 }
 
-void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
-  Base::Enter(x);
-  loopStack_.push_back(&x);
-}
-
-void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top = loopStack_.back();
-  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
-  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-  Base::Leave(x);
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPInteropConstruct &x) {
   bool isDependClauseOccured{false};
   int targetCount{0}, targetSyncCount{0};
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index beb6e0528e814..2a3853335fd1c 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -162,10 +162,6 @@ class OmpStructureChecker
 
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
-  bool IsVariableListItem(const Symbol &sym);
-  bool IsExtendedListItem(const Symbol &sym);
-  bool IsCommonBlock(const Symbol &sym);
-  std::optional<bool> IsContiguous(const parser::OmpObject &object);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -215,10 +211,6 @@ class OmpStructureChecker
       typename IterTy = decltype(std::declval<RangeTy>().begin())>
   std::optional<IterTy> FindDuplicate(RangeTy &&);
 
-  const Symbol *GetObjectSymbol(const parser::OmpObject &object);
-  const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
-  std::optional<parser::CharBlock> GetObjectSource(
-      const parser::OmpObject &object);
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
new file mode 100644
index 0000000000000..fd9596a09cd52
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -0,0 +1,393 @@
+//===-- lib/Semantics/openmp-utils.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
+
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Common/reference.h"
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Evaluate/variable.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/expression.h"
+#include "flang/Semantics/semantics.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <optional>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics::omp {
+
+std::string ThisVersion(unsigned version) {
+  std::string tv{
+      std::to_string(version / 10) + "." + std::to_string(version % 10)};
+  return "OpenMP v" + tv;
+}
+
+std::string TryVersion(unsigned version) {
+  return "try -fopenmp-version=" + std::to_string(version);
+}
+
+const parser::Designator *GetDesignatorFromObj(
+    const parser::OmpObject &object) {
+  return std::get_if<parser::Designator>(&object.u);
+}
+
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object) {
+  if (auto *desg{GetDesignatorFromObj(object)}) {
+    return std::get_if<parser::DataRef>(&desg->u);
+  }
+  return nullptr;
+}
+
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object) {
+  if (auto *dataRef{GetDataRefFromObj(object)}) {
+    using ElementIndirection = common::Indirection<parser::ArrayElement>;
+    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
+      return &ind->value();
+    }
+  }
+  return nullptr;
+}
+
+const Symbol *GetObjectSymbol(const parser::OmpObject &object) {
+  // Some symbols may be missing if the resolution failed, e.g. when an
+  // undeclared name is used with implicit none.
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    auto &last{GetLastName(*desg)};
+    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
+  }
+  return nullptr;
+}
+
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument) {
+  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
+    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
+      return GetObjectSymbol(*object);
+    }
+  }
+  return nullptr;
+}
+
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->source;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return GetLastName(*desg).source;
+  }
+  return std::nullopt;
+}
+
+bool IsCommonBlock(const Symbol &sym) {
+  return sym.detailsIf<CommonBlockDetails>() != nullptr;
+}
+
+bool IsVariableListItem(const Symbol &sym) {
+  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
+}
+
+bool IsExtendedListItem(const Symbol &sym) {
+  return IsVariableListItem(sym) || sym.IsSubprogram();
+}
+
+bool IsVarOrFunctionRef(const MaybeExpr &expr) {
+  if (expr) {
+    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
+        evaluate::IsVariable(*expr);
+  } else {
+    return false;
+  }
+}
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
+  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  return typedExpr.get()->v;
+}
+
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr) {
+  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
+namespace {
+struct ContiguousHelper {
+  ContiguousHelper(SemanticsContext &context)
+      : fctx_(context.foldingContext()) {}
+
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
+    return Visit(x.value());
+  }
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Reference<Contained> &x) {
+    return Visit(x.get());
+  }
+  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
+    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
+  }
+  template <typename T>
+  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
+    return common::visit(
+        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
+  }
+  template <typename T> std::optional<bool> Visit(const T &) {
+    // Everything else.
+    return std::nullopt;
+  }
+
+private:
+  evaluate::FoldingContext &fctx_;
+};
+} // namespace
+
+// Return values:
+// - std::optional<bool>{true} if the object is known to be contiguous
+// - std::optional<bool>{false} if the object is known not to be contiguous
+// - std::nullopt if the object contiguity cannot be determined
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object) {
+  return common::visit( //
+      common::visitors{
+          [&](const parser::Name &x) {
+            // Any member of a common block must be contiguous.
+            return std::optional<bool>{true};
+          },
+          [&](const parser::Designator &x) {
+            evaluate::ExpressionAnalyzer ea{semaCtx};
+            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
+              return ContiguousHelper{semaCtx}.Visit(*maybeExpr);
+            }
+            return std::optional<bool>{};
+          },
+      },
+      object.u);
+}
+
+struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
+                                 std::vector<SomeExpr>, false> {
+  using Result = std::vector<SomeExpr>;
+  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
+  DesignatorCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    // Once in a designator, don't traverse it any further (i.e. only
+    // collect top-level designators).
+    auto copy{x};
+    return Result{AsGenericExpr(std::move(copy))};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (moveAppend(v, std::move(results)), ...);
+    return v;
+  }
+};
+
+struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
+  using Base = evaluate::AnyTraverse<VariableFinder>;
+  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
+
+  using Base::operator();
+
+  template <typename T>
+  bool operator()(const evaluate::Designator<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+  template <typename T>
+  bool operator()(const evaluate::FunctionRef<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+private:
+  const SomeExpr &var;
+};
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr) {
+  return DesignatorCollector{}(expr);
+}
+
+static bool HasCommonDesignatorSymbols(
+    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
+  // Compare the designators used in "other" with the designators whose
+  // symbols are given in baseSyms.
+  // This is a part of the check if these two expressions can access the same
+  // storage: if the designators used in them are different enough, then they
+  // will be assumed not to access the same memory.
+  //
+  // Consider an (array element) expression x%y(w%z), the corresponding symbol
+  // vector will be {x, y, w, z} (i.e. the symbols for these names).
+  // Check whether this exact sequence appears anywhere in any the symbol
+  // vector for "other". This will be true for x(y) and x(y+1), so this is
+  // not a sufficient condition, but can be used to eliminate candidates
+  // before doing more exhaustive checks.
+  //
+  // If any of the symbols in this sequence are function names, assume that
+  // there is no storage overlap, mostly because it would be impossible in
+  // general to determine what storage the function will access.
+  // Note: if f is pure, then two calls to f will access the same storage
+  // when called with the same arguments. This check is not done yet.
+
+  if (llvm::any_of(
+          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
+    // If there is a function symbol in the chain then we can't infer much
+    // about the accessed storage.
+    return false;
+  }
+
+  auto isSubsequence{// Is u a subsequence of v.
+      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
+        size_t us{u.size()}, vs{v.size()};
+        if (us > vs) {
+          return false;
+        }
+        for (size_t off{0}; off != vs - us + 1; ++off) {
+          bool same{true};
+          for (size_t i{0}; i != us; ++i) {
+            if (u[i] != v[off + i]) {
+              same = false;
+              break;
+            }
+          }
+          if (same) {
+            return true;
+          }
+        }
+        return false;
+      }};
+
+  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
+  return isSubsequence(baseSyms, otherSyms);
+}
+
+static bool HasCommonTopLevelDesignators(
+    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
+  // Compare designators directly as expressions. This will ensure
+  // that x(y) and x(y+1) are not flagged as overlapping, whereas
+  // the symbol vectors for both of these would be identical.
+  std::vector<SomeExpr> otherDsgs{GetAllDesignators(other)};
+
+  for (auto &s : baseDsgs) {
+    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
+  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
+  std::vector<SomeExpr> baseDsgs{GetAllDesignators(base)};
+
+  for (const SomeExpr &expr : exprs) {
+    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
+      continue;
+    }
+    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
+      return &expr;
+    }
+  }
+  return nullptr;
+}
+
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
+  return VariableFinder{sub}(super);
+}
+
+// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
+// to separate cases where the source has something that looks like an
+// assignment, but is semantically wrong (diagnosed by general semantic
+// checks), and where the source has some other statement (which we want
+// to report as "should be an assignment").
+bool IsAssignment(const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return false;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+
+  return common::visit(
+      [](auto &&s) -> bool {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>;
+      },
+      x->u);
+}
+
+bool IsPointerAssignment(const evaluate::Assignment &x) {
+  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
+      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
+}
+
+/// parser::Block is a list of executable constructs, parser::BlockConstruct
+/// is Fortran's BLOCK/ENDBLOCK construct.
+/// Strip the outermost BlockConstructs, return the reference to the Block
+/// in the executable part of the innermost of the stripped constructs.
+/// Specifically, if the given `block` has a single entry (it's a list), and
+/// the entry is a BlockConstruct, get the Block contained within. Repeat
+/// this step as many times as possible.
+const parser::Block &GetInnermostExecPart(const parser::Block &block) {
+  const parser::Block *iter{&block};
+  while (iter->size() == 1) {
+    const parser::ExecutionPartConstruct &ep{iter->front()};
+    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
+      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
+      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
+        iter = &std::get<parser::Block>(bc->value().t);
+        continue;
+      }
+    }
+    break;
+  }
+  return *iter;
+}
+
+} // namespace Fortran::semantics::omp
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
new file mode 100644
index 0000000000000..dbb0565215357
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -0,0 +1,66 @@
+//===-- lib/Semantics/openmp-utils.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
+#define FORTRAN_SEMANTICS_OPENMP_UTILS_H
+
+#include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <optional>
+#include <string>
+
+namespace Fortran::semantics {
+class SemanticsContext;
+class Symbol;
+
+// Add this namespace to avoid potential conflicts
+namespace omp {
+std::string ThisVersion(unsigned version);
+std::string TryVersion(unsigned version);
+
+const parser::Designator *GetDesignatorFromObj(const parser::OmpObject &object);
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object);
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object);
+const Symbol *GetObjectSymbol(const parser::OmpObject &object);
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object);
+
+bool IsCommonBlock(const Symbol &sym);
+bool IsExtendedListItem(const Symbol &sym);
+bool IsVariableListItem(const Symbol &sym);
+bool IsVarOrFunctionRef(const MaybeExpr &expr);
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr);
+
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object);
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr);
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs);
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super);
+bool IsAssignment(const parser::ActionStmt *x);
+bool IsPointerAssignment(const evaluate::Assignment &x);
+const parser::Block &GetInnermostExecPart(const parser::Block &block);
+} // namespace omp
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_OPENMP_UTILS_H
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74b..299bb6ff876e7 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -495,6 +495,12 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
+    return true;
+  }
+  void Post(const parser::OmpDeclareVariantDirective &) { PopContext(); };
+
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,
     // except common blocks (the ResolveNamesVisitor does not visit
@@ -1796,10 +1802,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
 
   if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    if (const auto &doConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      if (doConstruct.value().IsDoWhile()) {
-        return true;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &doConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        if (doConstruct->IsDoWhile()) {
+          return true;
+        }
       }
     }
   }
@@ -1962,48 +1971,69 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
   bool hasCollapseClause{
       clause ? (clause->Id() == llvm::omp::OMPC_collapse) : false};
 
-  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  if (outer.has_value()) {
-    for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-      if (loop->IsDoConcurrent()) {
-        // DO CONCURRENT is explicitly allowed for the LOOP construct so long as
-        // there isn't a COLLAPSE clause
-        if (isLoopConstruct) {
-          if (hasCollapseClause) {
-            // hasCollapseClause implies clause != nullptr
-            context_.Say(clause->source,
-                "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &outer{std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      for (const parser::DoConstruct *loop{&*outer}; loop && level > 0;
+          --level) {
+        if (loop->IsDoConcurrent()) {
+          // DO CONCURRENT is explicitly allowed for the LOOP construct so long
+          // as there isn't a COLLAPSE clause
+          if (isLoopConstruct) {
+            if (hasCollapseClause) {
+              // hasCollapseClause implies clause != nullptr
+              context_.Say(clause->source,
+                  "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+            }
+          } else {
+            auto &stmt =
+                std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
+            context_.Say(stmt.source,
+                "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
           }
-        } else {
-          auto &stmt =
-              std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
-          context_.Say(stmt.source,
-              "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
-        }
-      }
-      // go through all the nested do-loops and resolve index variables
-      const parser::Name *iv{GetLoopIndex(*loop)};
-      if (iv) {
-        if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-          SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
-          iv->symbol = symbol; // adjust the symbol within region
-          AddToContextObjectWithDSA(*symbol, ivDSA);
         }
+        // go through all the nested do-loops and resolve index variables
+        const parser::Name *iv{GetLoopIndex(*loop)};
+        if (iv) {
+          if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
+            SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
+            iv->symbol = symbol; // adjust the symbol within region
+            AddToContextObjectWithDSA(*symbol, ivDSA);
+          }
 
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+        }
+      }
+      CheckAssocLoopLevel(level, GetAssociatedClause());
+    } else if (const auto &loop{std::get_if<
+                   common::Indirection<parser::OpenMPLoopConstruct>>(
+                   &*optLoopCons)}) {
+      auto &beginDirective =
+          std::get<parser::OmpBeginLoopDirective>(loop->value().t);
+      auto &beginLoopDirective =
+          std::get<parser::OmpLoopDirective>(beginDirective.t);
+      if (beginLoopDirective.v != llvm::omp::Directive::OMPD_unroll &&
+          beginLoopDirective.v != llvm::omp::Directive::OMPD_tile) {
+        context_.Say(GetContext().directiveSource,
+            "Only UNROLL or TILE constructs are allowed between an OpenMP Loop Construct and a DO construct"_err_en_US,
+            parser::ToUpperCaseLetters(llvm::omp::getOpenMPDirectiveName(
+                GetContext().directive, version)
+                    .str()));
+      } else {
+        PrivatizeAssociatedLoopIndexAndCheckLoopLevel(loop->value());
       }
+    } else {
+      context_.Say(GetContext().directiveSource,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
+                  .str()));
     }
-    CheckAssocLoopLevel(level, GetAssociatedClause());
-  } else {
-    context_.Say(GetContext().directiveSource,
-        "A DO loop must follow the %s directive"_err_en_US,
-        parser::ToUpperCaseLetters(
-            llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
-                .str()));
   }
 }
+
 void OmpAttributeVisitor::CheckAssocLoopLevel(
     std::int64_t level, const parser::OmpClause *clause) {
   if (clause && level != 0) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 9e465f8ff3e1e..d0336c9cb661d 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5383,7 +5383,7 @@ void DeclarationVisitor::Post(const parser::EntityDecl &x) {
     ConvertToObjectEntity(symbol) || ConvertToProcEntity(symbol);
     symbol.set(
         Symbol::Flag::EntryDummyArgument, false); // forestall excessive errors
-    Initialization(name, *init, false);
+    Initialization(name, *init, /*inComponentDecl=*/false);
   } else if (attrs.test(Attr::PARAMETER)) { // C882, C883
     Say(name, "Missing initialization for parameter '%s'"_err_en_US);
   }
@@ -6398,7 +6398,7 @@ void DeclarationVisitor::Post(const parser::ComponentDecl &x) {
     SetCUDADataAttr(name.source, symbol, cudaDataAttr());
     if (symbol.has<ObjectEntityDetails>()) {
       if (auto &init{std::get<std::optional<parser::Initialization>>(x.t)}) {
-        Initialization(name, *init, true);
+        Initialization(name, *init, /*inComponentDecl=*/true);
       }
     }
     currScope().symbol()->get<DerivedTypeDetails>().add_component(symbol);
@@ -7278,6 +7278,14 @@ bool DeclarationVisitor::PassesLocalityChecks(
           specName);
       return false;
     }
+    if (const DerivedTypeSpec *derived{type->AsDerived()}) { // F'2023 C1130
+      if (auto bad{FindAllocatableUltimateComponent(*derived)}) {
+        SayWithDecl(name, symbol,
+            "Derived type variable '%s' with ultimate ALLOCATABLE component '%s' not allowed in a %s locality-spec"_err_en_US,
+            bad.BuildResultDesignatorName(), specName);
+        return false;
+      }
+    }
   }
   if (symbol.attrs().test(Attr::ASYNCHRONOUS) && isReduce) { // F'2023 C1131
     SayWithDecl(name, symbol,
@@ -8933,9 +8941,13 @@ void DeclarationVisitor::Initialization(const parser::Name &name,
             // Defer analysis to the end of the specification part
             // so that forward references and attribute checks like SAVE
             // work better.
-            auto restorer{common::ScopedSet(deferImplicitTyping_, true)};
-            Walk(target);
-            ultimate.set(Symbol::Flag::InDataStmt);
+            if (inComponentDecl) {
+              PointerInitialization(name, target);
+            } else {
+              auto restorer{common::ScopedSet(deferImplicitTyping_, true)};
+              Walk(target);
+              ultimate.set(Symbol::Flag::InDataStmt);
+            }
           },
           [&](const std::list<Indirection<parser::DataStmtValue>> &values) {
             // Handled later in data-to-inits conversion
@@ -10355,11 +10367,6 @@ class DeferredCheckVisitor {
         std::get<std::optional<parser::Initialization>>(decl.t));
     return false;
   }
-  bool Pre(const parser::ComponentDecl &decl) {
-    Init(std::get<parser::Name>(decl.t),
-        std::get<std::optional<parser::Initialization>>(decl.t));
-    return false;
-  }
   bool Pre(const parser::ProcDecl &decl) {
     if (const auto &init{
             std::get<std::optional<parser::ProcPointerInit>>(decl.t)}) {
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index e07054f8ec564..ed41c6fb16892 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -655,8 +655,10 @@ bool Semantics::Perform() {
 void Semantics::EmitMessages(llvm::raw_ostream &os) {
   // Resolve the CharBlock locations of the Messages to ProvenanceRanges
   // so messages from parsing and semantics are intermixed in source order.
+  const common::LanguageFeatureControl &features{context_.languageFeatures()};
   context_.messages().ResolveProvenances(context_.allCookedSources());
-  context_.messages().Emit(os, context_.allCookedSources());
+  context_.messages().Emit(
+      os, context_.allCookedSources(), /*echoSourceLine=*/true, &features);
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index d053179448c00..d27d250b3f11e 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -1081,10 +1081,67 @@ const Scope *FindCUDADeviceContext(const Scope *scope) {
   });
 }
 
+bool IsDeviceAllocatable(const Symbol &symbol) {
+  if (IsAllocatable(symbol)) {
+    if (const auto *details{
+            symbol.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()}) {
+      if (details->cudaDataAttr() &&
+          *details->cudaDataAttr() != common::CUDADataAttr::Pinned) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+UltimateComponentIterator::const_iterator
+FindCUDADeviceAllocatableUltimateComponent(const DerivedTypeSpec &derived) {
+  UltimateComponentIterator ultimates{derived};
+  return std::find_if(ultimates.begin(), ultimates.end(), IsDeviceAllocatable);
+}
+
+bool CanCUDASymbolBeGlobal(const Symbol &sym) {
+  const Symbol &symbol{GetAssociationRoot(sym)};
+  const Scope &scope{symbol.owner()};
+  auto scopeKind{scope.kind()};
+  const common::LanguageFeatureControl &features{
+      scope.context().languageFeatures()};
+  if (features.IsEnabled(common::LanguageFeature::CUDA) &&
+      scopeKind == Scope::Kind::MainProgram) {
+    if (const auto *details{
+            sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()}) {
+      const Fortran::semantics::DeclTypeSpec *type{details->type()};
+      const Fortran::semantics::DerivedTypeSpec *derived{
+          type ? type->AsDerived() : nullptr};
+      if (derived) {
+        if (FindCUDADeviceAllocatableUltimateComponent(*derived)) {
+          return false;
+        }
+      }
+      if (details->cudaDataAttr() &&
+          *details->cudaDataAttr() != common::CUDADataAttr::Unified) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 std::optional<common::CUDADataAttr> GetCUDADataAttr(const Symbol *symbol) {
-  const auto *object{
+  const auto *details{
       symbol ? symbol->detailsIf<ObjectEntityDetails>() : nullptr};
-  return object ? object->cudaDataAttr() : std::nullopt;
+  if (details) {
+    const Fortran::semantics::DeclTypeSpec *type{details->type()};
+    const Fortran::semantics::DerivedTypeSpec *derived{
+        type ? type->AsDerived() : nullptr};
+    if (derived) {
+      if (FindCUDADeviceAllocatableUltimateComponent(*derived)) {
+        return common::CUDADataAttr::Managed;
+      }
+    }
+    return details->cudaDataAttr();
+  }
+  return std::nullopt;
 }
 
 bool IsAccessible(const Symbol &original, const Scope &scope) {
@@ -1788,4 +1845,4 @@ bool HadUseError(
   }
 }
 
-} // namespace Fortran::semantics
\ No newline at end of file
+} // namespace Fortran::semantics
diff --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index 17b5f8368916d..fc69fc638eda1 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -59,7 +59,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
     std::string cliOption{details::CamelCaseToLowerCaseHyphenated(name)};
     cliOptions_.insert({cliOption, {feature}});
     languageFeatureCliCanonicalSpelling_[EnumToInt(feature)] =
-        std::string_view{cliOption};
+        std::move(cliOption);
   });
 
   ForEachUsageWarning([&](auto warning) {
@@ -67,7 +67,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
     std::string cliOption{details::CamelCaseToLowerCaseHyphenated(name)};
     cliOptions_.insert({cliOption, {warning}});
     usageWarningCliCanonicalSpelling_[EnumToInt(warning)] =
-        std::string_view{cliOption};
+        std::move(cliOption);
   });
 
   // These features must be explicitly enabled by command line options.
@@ -174,18 +174,16 @@ bool LanguageFeatureControl::EnableWarning(std::string_view input) {
 
 void LanguageFeatureControl::ReplaceCliCanonicalSpelling(
     LanguageFeature f, std::string input) {
-  std::string_view &old{languageFeatureCliCanonicalSpelling_[EnumToInt(f)]};
-  cliOptions_.erase(std::string{old});
-  languageFeatureCliCanonicalSpelling_[EnumToInt(f)] = input;
+  cliOptions_.erase(languageFeatureCliCanonicalSpelling_[EnumToInt(f)]);
   cliOptions_.insert({input, {f}});
+  languageFeatureCliCanonicalSpelling_[EnumToInt(f)] = std::move(input);
 }
 
 void LanguageFeatureControl::ReplaceCliCanonicalSpelling(
     UsageWarning w, std::string input) {
-  std::string_view &old{usageWarningCliCanonicalSpelling_[EnumToInt(w)]};
-  cliOptions_.erase(std::string{old});
-  usageWarningCliCanonicalSpelling_[EnumToInt(w)] = input;
+  cliOptions_.erase(usageWarningCliCanonicalSpelling_[EnumToInt(w)]);
   cliOptions_.insert({input, {w}});
+  usageWarningCliCanonicalSpelling_[EnumToInt(w)] = std::move(input);
 }
 
 std::vector<const char *> LanguageFeatureControl::GetNames(
diff --git a/flang/module/cooperative_groups.f90 b/flang/module/cooperative_groups.f90
index 935e41fc56c1a..b8875f72f8079 100644
--- a/flang/module/cooperative_groups.f90
+++ b/flang/module/cooperative_groups.f90
@@ -20,6 +20,18 @@ module cooperative_groups
   integer(4) :: rank
 end type grid_group
 
+type :: coalesced_group
+  type(c_devptr), private :: handle
+  integer(4) :: size
+  integer(4) :: rank
+end type coalesced_group
+
+type :: thread_group
+  type(c_devptr), private :: handle
+  integer(4) :: size
+  integer(4) :: rank
+end type thread_group
+
 interface
   attributes(device) function this_grid()
     import
@@ -27,4 +39,18 @@ attributes(device) function this_grid()
   end function
 end interface
 
+interface
+  attributes(device) function this_thread_block()
+    import
+    type(thread_group) :: this_thread_block
+  end function
+end interface
+
+interface this_warp
+  attributes(device) function this_warp()
+    import
+    type(coalesced_group) :: this_warp
+  end function
+end interface
+
 end module
diff --git a/flang/test/Driver/func-attr.f90 b/flang/test/Driver/func-attr.f90
index 7442ac54dcaa5..e8cab4fca2226 100644
--- a/flang/test/Driver/func-attr.f90
+++ b/flang/test/Driver/func-attr.f90
@@ -4,6 +4,7 @@
 
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=none -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONEFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=non-leaf -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-NONLEAFFP
+! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=reserved -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-RESERVEDFP
 ! RUN: %flang_fc1 -triple aarch64-none-none -mframe-pointer=all -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-ALLFP
 ! RUN: not %flang_fc1 -triple aarch64-none-none -mframe-pointer=wrongval -emit-llvm -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-WRONGVALUEFP
 
@@ -20,6 +21,7 @@ end subroutine func
 
 ! CHECK-NONEFP-NOT: attributes #0 = { "frame-pointer"="{{.*}}" }
 ! CHECK-NONLEAFFP: attributes #0 = { "frame-pointer"="non-leaf" }
+! CHECK-RESERVEDFP: attributes #0 = { "frame-pointer"="reserved" }
 ! CHECK-ALLFP: attributes #0 = { "frame-pointer"="all" }
 
 ! CHECK-WRONGVALUEFP:error: invalid value 'wrongval' in '-mframe-pointer=wrongval'
diff --git a/flang/test/Driver/fveclib-codegen.f90 b/flang/test/Driver/fveclib-codegen.f90
index 4cbb1e284f18e..88b31da40167e 100644
--- a/flang/test/Driver/fveclib-codegen.f90
+++ b/flang/test/Driver/fveclib-codegen.f90
@@ -1,6 +1,8 @@
 ! test that -fveclib= is passed to the backend
 ! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=SLEEF -o - %s | FileCheck %s --check-prefix=SLEEF %}
 ! RUN: %if x86-registered-target %{ %flang -S -Ofast -target x86_64-unknown-linux-gnu -fveclib=libmvec -o - %s | FileCheck %s %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+nosve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-NEON %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+sve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-SVE %}
 ! RUN: %if x86-registered-target %{ %flang -S -O3 -ffast-math -target x86_64-unknown-linux-gnu -fveclib=AMDLIBM -o - %s | FileCheck %s --check-prefix=AMDLIBM %}
 ! RUN: %flang -S -Ofast -fveclib=NoLibrary -o - %s | FileCheck %s --check-prefix=NOLIB
 
@@ -11,6 +13,8 @@ subroutine sb(a, b)
 ! check that we used a vectorized call to powf()
 ! CHECK: _ZGVbN4vv_powf
 ! SLEEF: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-NEON: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-SVE: _ZGVsMxvv_powf
 ! AMDLIBM: amd_vrs4_powf
 ! NOLIB: powf
     a(i) = a(i) ** b(i)
diff --git a/flang/test/Driver/fveclib.f90 b/flang/test/Driver/fveclib.f90
index 431a4bfc02522..d21e85e486f8d 100644
--- a/flang/test/Driver/fveclib.f90
+++ b/flang/test/Driver/fveclib.f90
@@ -1,6 +1,7 @@
 ! RUN: %flang -### -c -fveclib=none %s 2>&1 | FileCheck -check-prefix CHECK-NOLIB %s
 ! RUN: %flang -### -c -fveclib=Accelerate %s 2>&1 | FileCheck -check-prefix CHECK-ACCELERATE %s
 ! RUN: %flang -### -c --target=x86_64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
+! RUN: %flang -### -c --target=aarch64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
 ! RUN: %flang -### -c -fveclib=MASSV %s 2>&1 | FileCheck -check-prefix CHECK-MASSV %s
 ! RUN: %flang -### -c -fveclib=Darwin_libsystem_m %s 2>&1 | FileCheck -check-prefix CHECK-DARWIN_LIBSYSTEM_M %s
 ! RUN: %flang -### -c --target=aarch64-none-none -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-SLEEF %s
@@ -23,7 +24,7 @@
 
 ! RUN: not %flang --target=x86-none-none -c -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=x86-none-none -c -fveclib=ArmPL %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
-! RUN: not %flang --target=aarch64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
+! RUN: not %flang --target=riscv64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=SVML %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=AMDLIBM %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! CHECK-ERROR: unsupported option {{.*}} for target
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 335bfad4b1884..f4f0c21dc1f5e 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -225,8 +225,8 @@
 ! RUN: | FileCheck %s --check-prefix=OFFLOAD-TARGETS
 
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
 ! OFFLOAD-TARGETS-NEXT: "{{[^"]*}}flang" "-fc1" "-triple" "amdgcn-amd-amdhsa"
-! OFFLOAD-TARGETS-NOT: -fopenmp-targets
+! OFFLOAD-TARGETS-NOT: --offload-targets
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
diff --git a/flang/test/Driver/tco-emit-final-mlir.fir b/flang/test/Driver/tco-emit-final-mlir.fir
new file mode 100644
index 0000000000000..75f8f153127af
--- /dev/null
+++ b/flang/test/Driver/tco-emit-final-mlir.fir
@@ -0,0 +1,19 @@
+// Test tco's -emit-final-mlir option.
+
+// RUN: tco -emit-final-mlir %s 2>&1 | FileCheck %s
+
+// Check that the FIR file is translated into the LLVM IR
+// MLIR dialect, but not into LLVM IR itself.
+
+// CHECK-NOT: func.func
+// CHECK-LABEL: llvm.func @_QPfoo()
+// CHECK-NOT: fir.alloca
+// CHECK: %[[VAL_0:.*]] = llvm.mlir.constant(1 : i64) : i64
+// CHECK: %[[VAL_1:.*]] = llvm.alloca %[[VAL_0]] x i32 : (i64) -> !llvm.ptr
+// CHECK: llvm.return
+// CHECK-NOT: func.func
+
+func.func @_QPfoo() {
+  %1 = fir.alloca i32
+  return
+}
diff --git a/flang/test/Evaluate/fold-dim.f90 b/flang/test/Evaluate/fold-dim.f90
index 40163c3f5a5ee..8e1b6ff0d1d8d 100644
--- a/flang/test/Evaluate/fold-dim.f90
+++ b/flang/test/Evaluate/fold-dim.f90
@@ -11,7 +11,7 @@ module m
   logical, parameter :: test_a3 = dim(2., 1.) == 1.
   logical, parameter :: test_a4 = dim(2., -1.) == 3.
   logical, parameter :: test_a5 = dim(-1., 2.) == 0.
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real, parameter :: nan = 0./0.
   logical, parameter :: test_a6 = dim(nan, 1.) /= dim(nan, 1.)
 end module
diff --git a/flang/test/Evaluate/fold-nearest.f90 b/flang/test/Evaluate/fold-nearest.f90
index 48b9ef37840e3..af5750462f6b0 100644
--- a/flang/test/Evaluate/fold-nearest.f90
+++ b/flang/test/Evaluate/fold-nearest.f90
@@ -19,16 +19,16 @@ module m1
   real, parameter :: negZero = sign(0., -1.)
   logical, parameter :: test_12 = nearest(negZero, 1.) == minSubnormal
   logical, parameter :: test_13 = nearest(negZero, -1.) == -minSubnormal
-  !WARN: warning: NEAREST: S argument is zero
+  !WARN: warning: NEAREST: S argument is zero [-Wfolding-value-checks]
   logical, parameter :: test_14 = nearest(0., negZero) == -minSubnormal
-  !WARN: warning: NEAREST: S argument is zero
+  !WARN: warning: NEAREST: S argument is zero [-Wfolding-value-checks]
   logical, parameter :: test_15 = nearest(negZero, 0.) == minSubnormal
   logical, parameter :: test_16 = nearest(tiny(1.),-1.) == 1.1754942E-38
   logical, parameter :: test_17 = nearest(tiny(1.),1.) == 1.1754945E-38
  contains
   subroutine subr(a)
     real, intent(in) :: a
-    !WARN: warning: NEAREST: S argument is zero
+    !WARN: warning: NEAREST: S argument is zero [-Wfolding-value-checks]
     print *, nearest(a, 0.)
   end
 end module
@@ -42,7 +42,7 @@ module m2
   logical, parameter :: test_2 = ieee_next_after(minSubnormal, -1.) == 0
   logical, parameter :: test_3 = ieee_next_after(1., 2.) == 1.0000001
   logical, parameter :: test_4 = ieee_next_after(1.0000001, -1.) == 1
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real, parameter :: inf = 1. / 0.
   logical, parameter :: test_5 = ieee_next_after(inf, inf) == inf
   logical, parameter :: test_6 = ieee_next_after(inf, -inf) == h
@@ -54,12 +54,12 @@ module m2
   logical, parameter :: test_11 = ieee_next_after(1.9999999999999999999_10, 3.) == 2._10
 #endif
   logical, parameter :: test_12 = ieee_next_after(1., 1.) == 1.
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real, parameter :: nan = 0. / 0.
-  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered
+  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered [-Wfolding-value-checks]
   real, parameter :: x13 = ieee_next_after(nan, nan)
   logical, parameter :: test_13 = .not. (x13 == x13)
-  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered
+  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered [-Wfolding-value-checks]
   real, parameter :: x14 = ieee_next_after(nan, 0.)
   logical, parameter :: test_14 = .not. (x14 == x14)
 end module
@@ -72,7 +72,7 @@ module m3
   logical, parameter :: test_2 = ieee_next_down(0.d0) == -minSubnormal
   logical, parameter :: test_3 = ieee_next_up(1.d0) == 1.0000000000000002d0
   logical, parameter :: test_4 = ieee_next_down(1.0000000000000002d0) == 1.d0
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(kind(0.d0)), parameter :: inf = 1.d0 / 0.d0
   logical, parameter :: test_5 = ieee_next_up(huge(0.d0)) == inf
   logical, parameter :: test_6 = ieee_next_down(-huge(0.d0)) == -inf
@@ -82,12 +82,12 @@ module m3
   logical, parameter :: test_10 = ieee_next_down(-inf) == -inf
   logical, parameter :: test_11 = ieee_next_up(1.9999999999999997d0) == 2.d0
   logical, parameter :: test_12 = ieee_next_down(2.d0) == 1.9999999999999997d0
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(kind(0.d0)), parameter :: nan = 0.d0 / 0.d0
-  !WARN: warning: IEEE_NEXT_UP intrinsic folding: argument is NaN
+  !WARN: warning: IEEE_NEXT_UP intrinsic folding: argument is NaN [-Wfolding-exception]
   real(kind(0.d0)), parameter :: x13 = ieee_next_up(nan)
   logical, parameter :: test_13 = .not. (x13 == x13)
-  !WARN: warning: IEEE_NEXT_DOWN intrinsic folding: argument is NaN
+  !WARN: warning: IEEE_NEXT_DOWN intrinsic folding: argument is NaN [-Wfolding-exception]
   real(kind(0.d0)), parameter :: x14 = ieee_next_down(nan)
   logical, parameter :: test_14 = .not. (x14 == x14)
 end module
diff --git a/flang/test/Evaluate/fold-out_of_range.f90 b/flang/test/Evaluate/fold-out_of_range.f90
index 6360eee5322bb..a2f453acb4cb2 100644
--- a/flang/test/Evaluate/fold-out_of_range.f90
+++ b/flang/test/Evaluate/fold-out_of_range.f90
@@ -9,23 +9,23 @@ module m
   integer(4),  parameter :: i4v(*)  = [ -huge(1_4)  - 1_4,  huge(1_4) ]
   integer(8),  parameter :: i8v(*)  = [ -huge(1_8)  - 1_8,  huge(1_8) ]
   integer(16), parameter :: i16v(*) = [ -huge(1_16) - 1_16, huge(1_16) ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(2),  parameter :: r2v(*)  = [ -huge(1._2),  huge(1._2),  1._2/0._2,   0._2/0._2 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(3),  parameter :: r3v(*)  = [ -huge(1._3),  huge(1._3),  1._3/0._3,   0._3/0._3 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4),  parameter :: r4v(*)  = [ -huge(1._4),  huge(1._4),  1._4/0._4,   0._4/0._4 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(8),  parameter :: r8v(*)  = [ -huge(1._8),  huge(1._8),  1._8/0._8,   0._8/0._8 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(10), parameter :: r10v(*) = [ -huge(1._10), huge(1._10), 1._10/0._10, 0._10/0._10 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(16), parameter :: r16v(*) = [ -huge(1._16), huge(1._16), 1._16/0._16, 0._16/0._16 ]
   logical, parameter :: finites(*) = [ .true., .true., .false., .false. ]
 
@@ -93,7 +93,7 @@ module m
   logical, parameter :: test_r2r10  = .not. any(out_of_range(r2v,  1._10))
   logical, parameter :: test_r2r16  = .not. any(out_of_range(r2v,  1._16))
   logical, parameter :: test_r3r2   =       all(out_of_range(r3v,  1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(3) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(3) conversion [-Wfolding-exception]
   logical, parameter :: test_r3r2b  = .not. any(out_of_range(real(r2v, 3),  1._2))
   logical, parameter :: test_r3r3   = .not. any(out_of_range(r3v,  1._3))
   logical, parameter :: test_r3r4   = .not. any(out_of_range(r3v,  1._4))
@@ -101,55 +101,55 @@ module m
   logical, parameter :: test_r3r10  = .not. any(out_of_range(r3v,  1._10))
   logical, parameter :: test_r3r16  = .not. any(out_of_range(r3v,  1._16))
   logical, parameter :: test_r4r2   =       all(out_of_range(r4v,  1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(4) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(4) conversion [-Wfolding-exception]
   logical, parameter :: test_r4r2b  = .not. any(out_of_range(real(r2v, 4),  1._2))
   logical, parameter :: test_r4r3   =       all(out_of_range(r4v,  1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(4) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(4) conversion [-Wfolding-exception]
   logical, parameter :: test_r4r3b  = .not. any(out_of_range(real(r3v, 4),  1._3))
   logical, parameter :: test_r4r4   = .not. any(out_of_range(r4v,  1._4))
   logical, parameter :: test_r4r8   = .not. any(out_of_range(r4v,  1._8))
   logical, parameter :: test_r4r10  = .not. any(out_of_range(r4v,  1._10))
   logical, parameter :: test_r4r16  = .not. any(out_of_range(r4v,  1._16))
   logical, parameter :: test_r8r2   =       all(out_of_range(r8v,  1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(8) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(8) conversion [-Wfolding-exception]
   logical, parameter :: test_r8r2b  = .not. any(out_of_range(real(r2v, 8),  1._2))
   logical, parameter :: test_r8r3   =       all(out_of_range(r8v,  1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(8) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(8) conversion [-Wfolding-exception]
   logical, parameter :: test_r8r3b  = .not. any(out_of_range(real(r3v, 8),  1._3))
   logical, parameter :: test_r8r4   =       all(out_of_range(r8v,  1._4) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(4) to REAL(8) conversion
+  !WARN: warning: invalid argument on REAL(4) to REAL(8) conversion [-Wfolding-exception]
   logical, parameter :: test_r8r4b  = .not. any(out_of_range(real(r4v, 8),  1._4))
   logical, parameter :: test_r8r8   = .not. any(out_of_range(r8v,  1._8))
   logical, parameter :: test_r8r10  = .not. any(out_of_range(r8v,  1._10))
   logical, parameter :: test_r8r16  = .not. any(out_of_range(r8v,  1._16))
   logical, parameter :: test_r10r2  =       all(out_of_range(r10v, 1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r2b = .not. any(out_of_range(real(r2v, 10),  1._2))
   logical, parameter :: test_r10r3  =       all(out_of_range(r10v, 1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r3b = .not. any(out_of_range(real(r3v, 10),  1._3))
   logical, parameter :: test_r10r4  =       all(out_of_range(r10v, 1._4) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(4) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(4) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r4b = .not. any(out_of_range(real(r4v, 10),  1._4))
   logical, parameter :: test_r10r8  =       all(out_of_range(r10v, 1._8) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(8) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(8) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r8b = .not. any(out_of_range(real(r8v, 10),  1._8))
   logical, parameter :: test_r10r10 = .not. any(out_of_range(r10v, 1._10))
   logical, parameter :: test_r10r16 = .not. any(out_of_range(r10v, 1._16))
   logical, parameter :: test_r16r2  =       all(out_of_range(r16v, 1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r2b = .not. any(out_of_range(real(r2v, 16), 1._2))
   logical, parameter :: test_r16r3  =       all(out_of_range(r16v, 1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r3b = .not. any(out_of_range(real(r3v, 16), 1._3))
   logical, parameter :: test_r16r4  =       all(out_of_range(r16v, 1._4) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(4) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(4) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r4b = .not. any(out_of_range(real(r4v, 16), 1._4))
   logical, parameter :: test_r16r8  =       all(out_of_range(r16v, 1._8) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(8) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(8) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r8b = .not. any(out_of_range(real(r8v, 16), 1._8))
   logical, parameter :: test_r16r10 =       all(out_of_range(r16v, 1._10) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(10) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(10) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r10b= .not. any(out_of_range(real(r10v, 16), 1._10))
   logical, parameter :: test_r16r16 = .not. any(out_of_range(r16v, 1._16))
 
@@ -223,29 +223,29 @@ module m
   logical, parameter :: test_r2i2ur   = all(out_of_range(real(i2v,  kind=2)+.5_2, 1_2,  .true.)  .eqv. [.false., .true.])
   logical, parameter :: test_r2i2d    = all(out_of_range(real(i2v,  kind=2)-.5_2, 1_2,  .false.) .eqv. [.false., .true.])
   logical, parameter :: test_r2i2dr   = all(out_of_range(real(i2v,  kind=2)-.5_2, 1_2,  .true.)  .eqv. [.false., .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4u    = all(out_of_range(real(i4v,  kind=2)+.5_2, 1_4,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4ur   = all(out_of_range(real(i4v,  kind=2)+.5_2, 1_4,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4d    = all(out_of_range(real(i4v,  kind=2)-.5_2, 1_4,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4dr   = all(out_of_range(real(i4v,  kind=2)-.5_2, 1_4,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8u    = all(out_of_range(real(i8v,  kind=2)+.5_2, 1_8,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8ur   = all(out_of_range(real(i8v,  kind=2)+.5_2, 1_8,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8d    = all(out_of_range(real(i8v,  kind=2)-.5_2, 1_8,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8dr   = all(out_of_range(real(i8v,  kind=2)-.5_2, 1_8,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16u   = all(out_of_range(real(i16v, kind=2)+.5_2, 1_16, .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16ur  = all(out_of_range(real(i16v, kind=2)+.5_2, 1_16, .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16d   = all(out_of_range(real(i16v, kind=2)-.5_2, 1_16, .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16dr  = all(out_of_range(real(i16v, kind=2)-.5_2, 1_16, .true.)  .eqv. [.true.,  .true.])
 
   logical, parameter :: test_r3i1u    = all(out_of_range(real(i1v,  kind=3)+.5_3, 1_1,  .false.) .eqv. [.false., .false.])
@@ -357,7 +357,7 @@ module m
   subroutine s(x, r)
     real(8), intent(in) :: x
     logical, intent(in), optional :: r
-    !WARN: warning: ROUND= argument to OUT_OF_RANGE() is an optional dummy argument that must be present at execution
+    !WARN: warning: ROUND= argument to OUT_OF_RANGE() is an optional dummy argument that must be present at execution [-Woptional-must-be-present]
     print *, out_of_range(x, 1, round=r)
   end
 end
diff --git a/flang/test/Evaluate/fold-unsigned.f90 b/flang/test/Evaluate/fold-unsigned.f90
index 719bdcc1a40b9..d53b7c7be0d5b 100644
--- a/flang/test/Evaluate/fold-unsigned.f90
+++ b/flang/test/Evaluate/fold-unsigned.f90
@@ -24,7 +24,7 @@ module m
 
   logical, parameter :: test_cus0    = int(0u,1) == 0
   logical, parameter :: test_cus0_k  = kind(int(0u,1)) == 1
-  !WARN: warning: conversion of 255_U1 to INTEGER(1) overflowed; result is -1
+  !WARN: warning: conversion of 255_U1 to INTEGER(1) overflowed; result is -1 [-Wfolding-exception]
   logical, parameter :: test_cus255  = int(255u_1,1) == -1
   logical, parameter :: test_cur255  = real(255u) == 255.
 
diff --git a/flang/test/Evaluate/folding03.f90 b/flang/test/Evaluate/folding03.f90
index 827bde86757c8..5b7ddd3c6c230 100644
--- a/flang/test/Evaluate/folding03.f90
+++ b/flang/test/Evaluate/folding03.f90
@@ -15,32 +15,32 @@ module integer_tests
   ! Integer division by zero are not tested here because they are handled as fatal
   ! errors in constants.
 
-  !WARN: warning: INTEGER(4) negation overflowed
+  !WARN: warning: INTEGER(4) negation overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_unary_minus1 = (-i4_nmax).EQ.i4_nmax
   logical, parameter :: test_no_overflow_unary_minus1 = (-i4_pmax).EQ.(i4_nmax+1_4)
   logical, parameter :: test_no_overflow_unary_plus1 = (+i4_pmax).EQ.i4_pmax
   logical, parameter :: test_no_overflow_unary_plus2 = (+i4_nmax).EQ.i4_nmax
 
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add1 = (i4_pmax+1_4).EQ.i4_nmax
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add2 = (i4_nmax + (-1_4)).EQ.i4_pmax
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add3 = (i4_pmax + i4_pmax).EQ.(-2_4)
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add4 = (i4_nmax + i4_nmax).EQ.(0_4)
   logical, parameter :: test_no_overflow_add1 = (i4_pmax + 0_4).EQ.i4_pmax
   logical, parameter :: test_no_overflow_add2 = (i4_nmax + (-0_4)).EQ.i4_nmax
   logical, parameter :: test_no_overflow_add3 = (i4_pmax + i4_nmax).EQ.(-1_4)
   logical, parameter :: test_no_overflow_add4 = (i4_nmax + i4_pmax).EQ.(-1_4)
 
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub1 = (i4_nmax - 1_4).EQ.i4_pmax
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub2 = (i4_pmax - (-1_4)).EQ.i4_nmax
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub3 = (i4_nmax - i4_pmax).EQ.(1_4)
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub4 = (i4_pmax - i4_nmax).EQ.(-1_4)
   logical, parameter :: test_no_overflow_sub1 = (i4_nmax - 0_4).EQ.i4_nmax
   logical, parameter :: test_no_overflow_sub2 = (i4_pmax - (-0_4)).EQ.i4_pmax
@@ -48,23 +48,23 @@ module integer_tests
   logical, parameter :: test_no_overflow_sub4 = (i4_pmax - i4_pmax).EQ.0_4
 
 
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult1 = (i4_pmax*2_4).EQ.(-2_4)
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult2 = (i4_nmax*2_4).EQ.(0_4)
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult3 = (i4_nmax*i4_nmax).EQ.(0_4)
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult4 = (i4_pmax*i4_pmax).EQ.(1_4)
 
-  !WARN: warning: INTEGER(4) division overflowed
+  !WARN: warning: INTEGER(4) division overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_div1 = (i4_nmax/(-1_4)).EQ.(i4_nmax)
   logical, parameter :: test_no_overflow_div1 = (i4_nmax/(-2_4)).EQ.(1_4 + i4_pmax/2_4)
   logical, parameter :: test_no_overflow_div2 = (i4_nmax/i4_nmax).EQ.(1_4)
 
-  !WARN: warning: INTEGER(4) power overflowed
+  !WARN: warning: INTEGER(4) power overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_pow1 = (i4_pmax**2_4).EQ.(1_4)
-  !WARN: warning: INTEGER(4) power overflowed
+  !WARN: warning: INTEGER(4) power overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_pow3 = (i4_nmax**2_4).EQ.(0_4)
   logical, parameter :: test_no_overflow_pow1 = ((-1_4)**i4_nmax).EQ.(1_4)
   logical, parameter :: test_no_overflow_pow2 = ((-1_4)**i4_pmax).EQ.(-1_4)
@@ -76,12 +76,12 @@ module real_tests
 
   real(4), parameter :: r4_pmax = 3.4028235E38
   real(4), parameter :: r4_nmax = -3.4028235E38
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan = 0._4/0._4
   TEST_ISNAN(r4_nan)
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_pinf = 1._4/0._4
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_ninf = -1._4/0._4
 
   logical, parameter :: test_r4_nan_parentheses1 = .NOT.(((r4_nan)).EQ.r4_nan)
@@ -106,13 +106,13 @@ module real_tests
   real(4), parameter :: r4_nan_plus = (+r4_nan)
   TEST_ISNAN(r4_nan_plus)
 
-  !WARN: warning: overflow on addition
+  !WARN: warning: overflow on addition [-Wfolding-exception]
   logical, parameter :: test_inf_r4_add9 = (r4_pmax + r4_pmax).eq.(r4_pinf)
-  !WARN: warning: overflow on addition
+  !WARN: warning: overflow on addition [-Wfolding-exception]
   logical, parameter :: test_inf_r4_add10 = (r4_nmax + r4_nmax).eq.(r4_ninf)
-  !WARN: warning: overflow on subtraction
+  !WARN: warning: overflow on subtraction [-Wfolding-exception]
   logical, parameter :: test_inf_r4_sub9 = (r4_pmax - r4_nmax).eq.(r4_pinf)
-  !WARN: warning: overflow on subtraction
+  !WARN: warning: overflow on subtraction [-Wfolding-exception]
   logical, parameter :: test_inf_r4_sub10 = (r4_nmax - r4_pmax).eq.(r4_ninf)
 
   ! No warnings expected below (inf propagation).
@@ -125,16 +125,16 @@ module real_tests
   logical, parameter :: test_inf_r4_add7 = (r4_ninf + 0._4).EQ.(r4_ninf)
   logical, parameter :: test_inf_r4_add8 = (r4_pinf + 0._4).EQ.(r4_pinf)
 
-  !WARN: warning: invalid argument on subtraction
+  !WARN: warning: invalid argument on subtraction [-Wfolding-exception]
   real(4), parameter :: r4_nan_sub1 = r4_pinf - r4_pinf
   TEST_ISNAN(r4_nan_sub1)
-  !WARN: warning: invalid argument on subtraction
+  !WARN: warning: invalid argument on subtraction [-Wfolding-exception]
   real(4), parameter :: r4_nan_sub2 = r4_ninf - r4_ninf
   TEST_ISNAN(r4_nan_sub2)
-  !WARN: warning: invalid argument on addition
+  !WARN: warning: invalid argument on addition [-Wfolding-exception]
   real(4), parameter :: r4_nan_add1 = r4_ninf + r4_pinf
   TEST_ISNAN(r4_nan_add1)
-  !WARN: warning: invalid argument on addition
+  !WARN: warning: invalid argument on addition [-Wfolding-exception]
   real(4), parameter :: r4_nan_add2 = r4_pinf + r4_ninf
   TEST_ISNAN(r4_nan_add2)
 
@@ -156,13 +156,13 @@ module real_tests
   real(4), parameter :: r4_nan_add6 = r4_nan + r4_nan
   TEST_ISNAN(r4_nan_add6)
 
-  !WARN: warning: overflow on multiplication
+  !WARN: warning: overflow on multiplication [-Wfolding-exception]
   logical, parameter :: test_inf_r4_mult1 = (1.5_4*r4_pmax).eq.(r4_pinf)
-  !WARN: warning: overflow on multiplication
+  !WARN: warning: overflow on multiplication [-Wfolding-exception]
   logical, parameter :: test_inf_r4_mult2 = (1.5_4*r4_nmax).eq.(r4_ninf)
-  !WARN: warning: overflow on division
+  !WARN: warning: overflow on division [-Wfolding-exception]
   logical, parameter :: test_inf_r4_div1 = (r4_nmax/(-0.5_4)).eq.(r4_pinf)
-  !WARN: warning: overflow on division
+  !WARN: warning: overflow on division [-Wfolding-exception]
   logical, parameter :: test_inf_r4_div2 = (r4_pmax/(-0.5_4)).eq.(r4_ninf)
 
   ! No warnings expected below (inf propagation).
@@ -179,25 +179,25 @@ module real_tests
   logical, parameter :: test_inf_r4_div9 = (r4_nmax/r4_pinf).EQ.(0.)
   logical, parameter :: test_inf_r4_div10 = (r4_nmax/r4_ninf).EQ.(0.)
 
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div1 = 0._4/0._4
   TEST_ISNAN(r4_nan_div1)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div2 = r4_ninf/r4_ninf
   TEST_ISNAN(r4_nan_div2)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div3 = r4_ninf/r4_pinf
   TEST_ISNAN(r4_nan_div3)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div4 = r4_pinf/r4_ninf
   TEST_ISNAN(r4_nan_div4)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div5 = r4_pinf/r4_pinf
   TEST_ISNAN(r4_nan_div5)
-  !WARN: warning: invalid argument on multiplication
+  !WARN: warning: invalid argument on multiplication [-Wfolding-exception]
   real(4), parameter :: r4_nan_mult1 = r4_pinf*0._4
   TEST_ISNAN(r4_nan_mult1)
-  !WARN: warning: invalid argument on multiplication
+  !WARN: warning: invalid argument on multiplication [-Wfolding-exception]
   real(4), parameter :: r4_nan_mult2 = 0._4*r4_ninf
   TEST_ISNAN(r4_nan_mult2)
 
diff --git a/flang/test/Evaluate/folding04.f90 b/flang/test/Evaluate/folding04.f90
index 08c5c6eee9b12..027db20f608b2 100644
--- a/flang/test/Evaluate/folding04.f90
+++ b/flang/test/Evaluate/folding04.f90
@@ -10,11 +10,11 @@ module real_tests
 
   real(4), parameter :: r4_pmax = 3.4028235E38
   real(4), parameter :: r4_nmax = -3.4028235E38
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan = 0._4/0._4
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_pinf = 1._4/0._4
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_ninf = -1._4/0._4
 
   !WARN: warning: argument is out of range [-1., 1.]
@@ -37,7 +37,7 @@ module real_tests
   TEST_ISNAN(nan_r8_dasin1)
   !WARN: warning: complex argument must be different from zero
   complex(4), parameter :: c4_clog1 = clog((0., 0.))
-  !WARN: warning: MOD: P argument is zero
+  !WARN: warning: MOD: P argument is zero [-Wfolding-avoids-runtime-crash]
   real(4), parameter :: nan_r4_mod = mod(3.5, 0.)
   TEST_ISNAN(nan_r4_mod)
   real(4), parameter :: ok_r4_gamma = gamma(-1.1)
@@ -53,17 +53,17 @@ module real_tests
   !WARN: warning: 'x' and 'y' arguments must not be both zero
   real(4), parameter :: r4_atan2 = atan2(0., 0.)
 
-  !WARN: warning: overflow on evaluation of intrinsic function or operation
+  !WARN: warning: overflow on evaluation of intrinsic function or operation [-Wfolding-exception]
   logical, parameter :: test_exp_overflow = exp(256._4).EQ.r4_pinf
  contains
   subroutine s1(a,j)
-    !WARN: warning: MOD: P argument is zero
+    !WARN: warning: MOD: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, mod(a, 0.)
-    !WARN: warning: MODULO: P argument is zero
+    !WARN: warning: MODULO: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, modulo(a, 0.)
-    !WARN: warning: MOD: P argument is zero
+    !WARN: warning: MOD: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, mod(j, 0.)
-    !WARN: warning: MODULO: P argument is zero
+    !WARN: warning: MODULO: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, modulo(j, 0.)
   end
 end module
@@ -86,13 +86,13 @@ module specific_extremums
   ! specified for f18 (converting the result).
   integer(8), parameter :: max_i32_8 = 2_8**31-1
   integer, parameter :: expected_min0 = int(min(max_i32_8, 2_8*max_i32_8), 4)
-  !WARN: portability: Argument types do not match specific intrinsic 'min0' requirements; using 'min' generic instead and converting the result to INTEGER(4) if needed
+  !WARN: portability: Argument types do not match specific intrinsic 'min0' requirements; using 'min' generic instead and converting the result to INTEGER(4) if needed [-Wuse-generic-intrinsic-when-specific-doesnt-match]
   integer, parameter :: result_min0 =  min0(max_i32_8, 2_8*max_i32_8)
   ! result_min0 would be -2  if arguments were converted to default integer.
   logical, parameter :: test_min0 = expected_min0 .EQ. result_min0
 
   real, parameter :: expected_amax0 = real(max(max_i32_8, 2_8*max_i32_8), 4)
-  !WARN: portability: Argument types do not match specific intrinsic 'amax0' requirements; using 'max' generic instead and converting the result to REAL(4) if needed
+  !WARN: portability: Argument types do not match specific intrinsic 'amax0' requirements; using 'max' generic instead and converting the result to REAL(4) if needed [-Wuse-generic-intrinsic-when-specific-doesnt-match]
   real, parameter :: result_amax0 = amax0(max_i32_8, 2_8*max_i32_8)
   ! result_amax0 would be 2.1474836E+09 if arguments were converted to default integer first.
   logical, parameter :: test_amax0 = expected_amax0 .EQ. result_amax0
diff --git a/flang/test/Evaluate/folding05.f90 b/flang/test/Evaluate/folding05.f90
index eb762775e6416..aed1906e6a640 100644
Binary files a/flang/test/Evaluate/folding05.f90 and b/flang/test/Evaluate/folding05.f90 differ
diff --git a/flang/test/Evaluate/folding06.f90 b/flang/test/Evaluate/folding06.f90
index da434fb6d4869..d4f11b0c30f1f 100644
--- a/flang/test/Evaluate/folding06.f90
+++ b/flang/test/Evaluate/folding06.f90
@@ -7,12 +7,12 @@ module m
   integer, pointer :: int_pointer
   integer, allocatable :: int_allocatable
   logical, parameter :: test_Assoc1 = .not.(associated(null()))
-  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
   !WARN: because: 'NULL()' is a null pointer
   logical, parameter :: test_Assoc2 = .not.(associated(null(), null()))
   logical, parameter :: test_Assoc3 = .not.(associated(null(int_pointer)))
   logical, parameter :: test_Alloc1 = .not.(allocated(null(int_allocatable)))
-  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
   !WARN: because: 'NULL()' is a null pointer
   logical, parameter :: test_Assoc5 = .not. associated(null(), null(int_pointer))
 
diff --git a/flang/test/Evaluate/folding14.f90 b/flang/test/Evaluate/folding14.f90
index 2cf1c32f6cb3f..4ff9e107d1610 100644
--- a/flang/test/Evaluate/folding14.f90
+++ b/flang/test/Evaluate/folding14.f90
@@ -4,9 +4,9 @@ module m1
   logical, parameter :: results(*) = isnan([ &
     0., &
     -0., &
-!WARN: warning: division by zero
+!WARN: warning: division by zero [-Wfolding-exception]
     1./0., &
-!WARN: warning: invalid argument on division
+!WARN: warning: invalid argument on division [-Wfolding-exception]
     0./0., &
     real(z'7ff80001',kind=4), &
     real(z'fff80001',kind=4), &
diff --git a/flang/test/Evaluate/folding28.f90 b/flang/test/Evaluate/folding28.f90
index 642919de7414a..3a795540812ab 100644
--- a/flang/test/Evaluate/folding28.f90
+++ b/flang/test/Evaluate/folding28.f90
@@ -15,7 +15,7 @@ module m
   ! -0 (sqrt is -0)
   real(8), parameter :: n08 = z'8000000000000000'
   real(8), parameter :: sqrt_n08 = sqrt(n08)
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(8), parameter :: inf_n08 = 1.0_8 / sqrt_n08, inf_n08z = z'fff0000000000000'
   logical, parameter :: test_n08 = inf_n08 == inf_n08z
   ! min normal
diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
index 8cac643b27c34..4c634513745fd 100644
--- a/flang/test/Fir/CUDA/cuda-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -11,3 +11,16 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.conta
 
 // CHECK: gpu.module @cuda_device_mo
 // CHECK-NEXT: fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda<device>} : !fir.array<5xi32>
+
+// -----
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} {
+  fir.global @_QMm1ECb(dense<[90, 100, 110]> : tensor<3xi32>) constant : !fir.array<3xi32>
+  fir.global @_QMm2ECc(dense<[100, 200, 300]> : tensor<3xi32>) constant : !fir.array<3xi32>
+}
+
+// CHECK: fir.global @_QMm1ECb
+// CHECK: fir.global @_QMm2ECc
+// CHECK: gpu.module @cuda_device_mod
+// CHECK-DAG: fir.global @_QMm2ECc
+// CHECK-DAG: fir.global @_QMm1ECb
diff --git a/flang/test/Fir/OpenACC/openacc-type-categories-class.f90 b/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
new file mode 100644
index 0000000000000..58025bfa556a5
--- /dev/null
+++ b/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
@@ -0,0 +1,46 @@
+! RUN: bbc -fopenacc -emit-hlfir %s -o - | fir-opt -pass-pipeline='builtin.module(test-fir-openacc-interfaces)' --mlir-disable-threading 2>&1 | FileCheck %s
+
+module mm
+  type, public :: polyty
+    real :: field
+  end type
+contains
+  subroutine init(this)
+    class(polyty), intent(inout) :: this
+    !$acc enter data copyin(this, this%field)
+  end subroutine
+  subroutine init_assumed_type(var)
+    type(*), intent(inout) :: var
+    !$acc enter data copyin(var)
+  end subroutine
+  subroutine init_unlimited(this)
+    class(*), intent(inout) :: this
+    !$acc enter data copyin(this)
+    select type(this)
+    type is(real)
+      !$acc enter data copyin(this)
+    class is(polyty)
+      !$acc enter data copyin(this, this%field)
+    end select
+  end subroutine
+end module
+
+! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "this", structured = false}
+! CHECK: Mappable: !fir.class<!fir.type<_QMmmTpolyty{field:f32}>>
+! CHECK: Type category: composite
+! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "this%field", structured = false}
+! CHECK: Pointer-like: !fir.ref<f32>
+! CHECK: Type category: composite
+
+! For unlimited polymorphic entities and assumed types - they effectively have
+! no declared type. Thus the type categorizer cannot categorize it.
+! CHECK: Visiting: {{.*}} = acc.copyin {{.*}} {name = "var", structured = false}
+! CHECK: Pointer-like: !fir.ref<none>
+! CHECK: Type category: uncategorized
+! CHECK: Visiting: {{.*}} = acc.copyin {{.*}} {name = "this", structured = false}
+! CHECK: Mappable: !fir.class<none>
+! CHECK: Type category: uncategorized
+
+! TODO: After using select type - the appropriate type category should be
+! possible. Add the rest of the test once OpenACC lowering correctly handles
+! unlimited polymorhic.
diff --git a/flang/test/Fir/convert-nontemporal-to-llvm.fir b/flang/test/Fir/convert-nontemporal-to-llvm.fir
index 6200ef1c621d7..e0a836ab0dad1 100644
--- a/flang/test/Fir/convert-nontemporal-to-llvm.fir
+++ b/flang/test/Fir/convert-nontemporal-to-llvm.fir
@@ -14,7 +14,7 @@
 // CHECK:    %[[VAL5:.*]] = llvm.alloca %[[CONST_VAL4]] x i32 {bindc_name = "a"} : (i64) -> !llvm.ptr
 // CHECK:    %[[CONST_VAL5:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[VAL6:.*]] = llvm.load %[[VAL1]] : !llvm.ptr -> i32
-// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) private(@_QFtestEi_private_i32 %[[VAL2]] -> %arg0 : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) {
 // CHECK:      omp.loop_nest (%{{.*}}) : i32 = (%[[CONST_VAL5]]) to (%[[VAL6]]) inclusive step (%[[CONST_VAL5]]) {
 // CHECK:        llvm.store %{{.*}}, %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[VAL8:.*]] = llvm.load %[[VAL5]] {nontemporal} : !llvm.ptr -> i32
@@ -33,9 +33,9 @@
     %3 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtestEi"}
     %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFtestEn"}
     %5 = fir.load %4 : !fir.ref<i32>
-    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFtestEi_private_i32 %3 -> %arg0 : !fir.ref<i32>) {
+    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%5) inclusive step (%c1_i32) {
-        fir.store %arg1 to %arg0 : !fir.ref<i32>
+        fir.store %arg1 to %3 : !fir.ref<i32>
         %6 = fir.load %0 {nontemporal}: !fir.ref<i32>
         %7 = fir.load %1 : !fir.ref<i32>
         %8 = arith.addi %6, %7 : i32
@@ -52,12 +52,12 @@
 // CHECK:    %[[IDX_VAL:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[CONST_VAL1:.*]] = llvm.mlir.constant(0 : index) : i64
 // CHECK:    %[[END_IDX:.*]] = llvm.mlir.constant(100 : i32) : i32
-// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %[[ALLOCA2]] -> %[[ARG2:.*]] : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) {
 // CHECK:      omp.loop_nest (%[[ARG3:.*]]) : i32 = (%[[IDX_VAL]]) to (%[[END_IDX]]) inclusive step (%[[IDX_VAL]]) {
-// CHECK:        llvm.store %[[ARG3]], %[[ARG2]] : i32, !llvm.ptr
+// CHECK:        llvm.store %[[ARG3]], %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[CONST_VAL2:.*]] = llvm.mlir.constant(48 : i32) : i32
 // CHECK:        "llvm.intr.memcpy"(%[[ALLOCA1:.*]], %[[ARG0]], %[[CONST_VAL2]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
-// CHECK:        %[[VAL1:.*]] = llvm.load %[[ARG2]] : !llvm.ptr -> i32
+// CHECK:        %[[VAL1:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
 // CHECK:        %[[VAL2:.*]] = llvm.sext %[[VAL1]] : i32 to i64
 // CHECK:        %[[VAL3:.*]] = llvm.getelementptr %[[ALLOCA1]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
 // CHECK:        %[[VAL4:.*]] = llvm.load %[[VAL3]] : !llvm.ptr -> !llvm.ptr
@@ -91,11 +91,11 @@
     %1 = fir.allocmem !fir.array<?xi32>, %c100 {fir.must_be_heap = true, uniq_name = "_QFsimd_nontemporal_allocatableEx.alloc"}
     %2 = fircg.ext_embox %1(%c100) : (!fir.heap<!fir.array<?xi32>>, index) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %2 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %0 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        fir.store %arg3 to %arg2 : !fir.ref<i32>
+        fir.store %arg3 to %0 : !fir.ref<i32>
         %7 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-        %8 = fir.load %arg2 : !fir.ref<i32>
+        %8 = fir.load %0 : !fir.ref<i32>
         %9 = fir.convert %8 : (i32) -> i64
         %10 = fir.box_addr %7 : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
         %11:3 = fir.box_dims %7, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
diff --git a/flang/test/Fir/simd-nontemporal.fir b/flang/test/Fir/simd-nontemporal.fir
index 31051ff52f9bd..a58bf01a60980 100644
--- a/flang/test/Fir/simd-nontemporal.fir
+++ b/flang/test/Fir/simd-nontemporal.fir
@@ -17,11 +17,11 @@ func.func @_QPsimd_with_nontemporal_clause(%arg0: !fir.ref<i32> {fir.bindc_name
     %8 = fir.declare %7 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
     %9 = fir.declare %arg0 dummy_scope %0 {uniq_name = "_QFsimd_with_nontemporal_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
     %10 = fir.load %9 : !fir.ref<i32>
-    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) {
     // CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%10) inclusive step (%c1_i32) {
-        %11 = fir.declare %arg1 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %11 = fir.declare %8 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg2 to %11 : !fir.ref<i32>
         // CHECK:  %[[LOAD:.*]] = fir.load %[[A_DECL]] {nontemporal} : !fir.ref<i32>
         %12 = fir.load %2 : !fir.ref<i32>
@@ -56,11 +56,11 @@ func.func @_QPsimd_nontemporal_allocatable(%arg0: !fir.ref<!fir.box<!fir.heap<!f
     %9 = fir.shape %7 : (index) -> !fir.shape<1>
     %10 = fir.embox %8(%9) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %10 to %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
     // CHECK:   omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        %16 = fir.declare %arg2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %16 = fir.declare %2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg3 to %16 : !fir.ref<i32>
         // CHECK:  %[[VAL1:.*]] = fir.load %[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
         %17 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 02ab02945b042..468a5dbf988d7 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -12,18 +12,18 @@ func.func @_QPtest1() {
   return
 }
 // CHECK-LABEL:   func.func @_QPtest1() {
-// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 11 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 13 : index
-// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
-// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_5]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
-// CHECK:           fir.do_loop %[[VAL_7:.*]] = %[[VAL_0]] to %[[VAL_3]] step %[[VAL_0]] unordered {
-// CHECK:             fir.do_loop %[[VAL_8:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_0]] unordered {
-// CHECK:               %[[VAL_9:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_8]], %[[VAL_7]])  : (!fir.ref<!fir.array<11x13xf32>>, index, index) -> !fir.ref<f32>
-// CHECK:               hlfir.assign %[[VAL_1]] to %[[VAL_9]] : f32, !fir.ref<f32>
-// CHECK:             }
+// CHECK:           %[[VAL_0:.*]] = arith.constant 143 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 11 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 13 : index
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.array<11x13xf32>>) -> !fir.ref<!fir.array<143xf32>>
+// CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_1]] to %[[VAL_0]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_9]]) : (!fir.ref<!fir.array<143xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_2]] to %[[VAL_10]] : f32, !fir.ref<f32>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
@@ -129,3 +129,58 @@ func.func @_QPtest5(%arg0: !fir.ref<!fir.array<77xcomplex<f32>>> {fir.bindc_name
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+  %c0_i32 = arith.constant 0 : i32
+  %0:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+  hlfir.assign %c0_i32 to %0#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest6(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_1]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = arith.muli %[[VAL_6]]#1, %[[VAL_7]]#1 : index
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_5]](%[[VAL_9]]) : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           fir.do_loop %[[VAL_11:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_11]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_3]] to %[[VAL_12]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+func.func @_QQmain() {
+  return
+}
+
+func.func private @_QFPtest7(%arg0: !fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>>> {fir.host_assoc}) attributes {fir.host_symbol = @_QQmain, llvm.linkage = #llvm.linkage<internal>} {
+  %0 = fir.dummy_scope : !fir.dscope
+  %c0_i32 = arith.constant 0 : i32
+  %1 = fir.coordinate_of %arg0, %c0_i32 : (!fir.ref<tuple<!fir.box<!fir.array<2x2xi32>>>>, i32) -> !fir.ref<!fir.box<!fir.array<2x2xi32>>>
+  %2 = fir.load %1 : !fir.ref<!fir.box<!fir.array<2x2xi32>>>
+  %3 = fir.box_addr %2 : (!fir.box<!fir.array<2x2xi32>>) -> !fir.ref<!fir.array<2x2xi32>>
+  %c0 = arith.constant 0 : index
+  %4:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.array<2x2xi32>>, index) -> (index, index, index)
+  %c1 = arith.constant 1 : index
+  %5:3 = fir.box_dims %2, %c1 : (!fir.box<!fir.array<2x2xi32>>, index) -> (index, index, index)
+  %6 = fir.shape %4#1, %5#1 : (index, index) -> !fir.shape<2>
+  %7:2 = hlfir.declare %3(%6) {fortran_attrs = #fir.var_attrs<host_assoc>, uniq_name = "_QFEa"} : (!fir.ref<!fir.array<2x2xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<2x2xi32>>, !fir.ref<!fir.array<2x2xi32>>)
+  %c0_i32_0 = arith.constant 0 : i32
+  hlfir.assign %c0_i32_0 to %7#0 : i32, !fir.ref<!fir.array<2x2xi32>>
+  return
+}
+
+// CHECK-LABEL:   func.func private @_QFPtest7({{.*}}) {{.*}} {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : i32
+// CHECK:           fir.do_loop %[[VAL_1:.*]] = %{{.*}} to %{{.*}} step %{{.*}} unordered {
+// CHECK:             %[[VAL_2:.*]] = hlfir.designate %{{.*}} (%[[VAL_1]]) : (!fir.ref<!fir.array<4xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_0]] to %[[VAL_2]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:         }
diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 35eec823e30de..aa68c13558a7b 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -6,7 +6,7 @@
 ! added to this directory and sub-directories.
 !===----------------------------------------------------------------------===!
 
-!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa %s -o - | FileCheck %s
 
 !===============================================================================
 ! Check MapTypes for target implicit captures
@@ -494,15 +494,18 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_single_member_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-!CHECK: store ptr %[[MEMBER_ACCESS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_nested_members_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 0
-!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS_0:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 0
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 1
 !CHECK: %[[ARR_END_OFF:.*]] = getelementptr float, ptr %[[MEMBER_ACCESS_2]], i64 1
 !CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
 !CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_1]] to i64
@@ -525,8 +528,9 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_member_with_bounds_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 2
-!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 2
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_1]], i64 0, i64 1
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -574,19 +578,27 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
 !CHECK: store i64 %[[DTYPE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
+
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_alloca_derived_type_{{.*}}
 !CHECK: %{{.*}} = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_3:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1, align 8
-!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
-!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
-!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
 !CHECK: %[[ACCESS_DESC_MEMBER_UB:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[LOAD_DESC_MEMBER_UB:.*]] = load i64, ptr %[[ACCESS_DESC_MEMBER_UB]], align 8
 !CHECK: %[[OFFSET_MEMBER_UB:.*]] = sub i64 %[[LOAD_DESC_MEMBER_UB]], 1
+!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
+!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
+!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
 !CHECK: %[[DTYPE_ALLOCA_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ALLOCA_MEMBER_ACCESS]], i32 0, i32 0
+
 !CHECK: %[[DTYPE_BASE_ADDR_ACCESS_2:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA]], i32 0, i32 0
 !CHECK: %[[DTYPE_BASE_ADDR_LOAD_2:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS_2]], align 8
 !CHECK: %[[DTYPE_NONALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD_2]], i32 0, i32 5
@@ -715,11 +727,11 @@ end subroutine mapType_common_block_members
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_nested_derived_type_alloca{{.*}}
 !CHECK: %[[ALLOCATABLE_MEMBER_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_nested_derived_type_allocaTtop_layer, i64 1, align 8
+!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
+!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ALLOCATABLE_MEMBER_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[ALLOCATABLE_MEMBER_ADDR_LOAD:.*]] = load i64, ptr %[[ALLOCATABLE_MEMBER_BASE_ADDR]], align 8
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_1:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_ADDR_LOAD]], 1
-!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
-!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[NESTED_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_2:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_1]], 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_3:.*]] = add i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_2]], 1
diff --git a/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90 b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
new file mode 100644
index 0000000000000..249d403057b6c
--- /dev/null
+++ b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
@@ -0,0 +1,18 @@
+!REQUIRES: amdgpu-registered-target
+
+!RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -target-cpu gfx908 %s -o - | FileCheck %s
+
+subroutine maintest
+  implicit none
+
+  type r1_t
+  end type r1_t
+
+  type(r1_t), pointer :: A
+end subroutine
+
+! CHECK: @[[TYPE_DESC:.*XdtXr1_t]] = linkonce_odr addrspace(1) constant %_QM__fortran_type_infoTderivedtype
+
+! CHECK: define void @maintest_() {{.*}} {
+! CHECK:   store { {{.*}} } { {{.*}}, ptr addrspacecast (ptr addrspace(1) @[[TYPE_DESC]] to ptr), {{.*}} }, {{.*}}
+! CHECK: }
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
new file mode 100644
index 0000000000000..72b7a78bef510
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
@@ -0,0 +1,116 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Test that debug metadata for derived types is not duplicated more than needed
+! when emitting debug info with non trivial cycles.
+
+
+! In the type graph below, G has a back edge to B, and F to D.
+! This causes C to be in the middle of B cycle, and E to be
+! both in B and D cycles.
+! C and E are used in several contexts, under B, under D, and outside
+! of it to test how metadata is generated for them.
+!
+! Without "local caching" of C and E when generating mlir::LLVM::DITypeAttr
+! for such derived types, many duplicate llvm metadata for the derived types
+! would be emitted, while with the right duplication of mlir::LLVM::DITypeAttr,
+! a lot more duplicate llvm IR metadata ends up emitted (19 DICompositeType
+! vs 71 before the patch that added this test).
+!
+!
+!  A -> B -> C -> D -> E -> F -> G -> B
+!  |    |         |         |
+!  |    |         |         | -> D
+!  |    |         |
+!  |    |         | -> H -> E
+!  |    |
+!  |    | -> I -> E
+!  |         | -> C
+!  |
+!  | -> C
+!  | -> E
+
+subroutine type_cycles_caching()
+  type g
+    type(b), pointer :: c_b
+  end type
+  type f
+    type(g) :: c_b
+    type(d), pointer :: c_d
+  end type
+  type e
+    type(f) :: c_f
+  end type
+    type h
+      ! Can reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+    end type
+  type d
+    type(e) :: c_e
+    type(h) :: c_h
+  end type
+  type c
+    type(d) :: c_d
+  end type
+    type i
+      ! Cannot reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+      ! Can reuse metadata of type(c) under 'b'.
+      type(c) :: c_c
+    end type
+  type b
+    type(c) :: c_c
+    type(i) :: c_i
+  end type
+  type a
+    type(b) :: c_b
+    ! Cannot reuse metadata of type(c) under 'b'
+    type(c) :: c_c
+    ! Cannot reuse metadata of type(e) under 'd', nor the one under 'b'
+    type(e) :: c_e
+  end type
+  type(a) :: xa
+  ! Can reuse metadata of type(c) for xa%c_c
+  type(c) :: xc
+  ! Can reuse metadata of type(c) for xa%c_e
+  type(e) :: xe
+  call bar(xa, xc, xe)
+end subroutine
+
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "a",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "b",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "i",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
new file mode 100644
index 0000000000000..c1f49d2f4a74a
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
@@ -0,0 +1,39 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Simple test that checks that metadata for `t0` is only duplicated once.
+
+! The difficulty is that at the mlir::LLVM::DITypeAttr, because of the
+! lack of MLIR attribute true recursion, the mlir::LLVM::DITypeAttr for
+! `t0` inside `t1` is special because it is only valid when found
+! in an mlir::LLVM::DITypeAttr tree under `base` (it will point to `base`
+! via an integer id that is only meanigfu;l when a node with such id has
+! been traversed).
+! A different node has to be created for `t0` usage in `x0` (will
+! point to the actual mlir::LLVM::DITypeAttr for `base` instead of
+! an integer id since the cycle is already "taken care of" in `base`
+! definition).
+! However, the same special `t0` node can be used for both `x1_1` and
+! `x1_2` components because they are both under `base` in the
+! mlir::LLVM::DITypeAttr tree definition. This used to not be the case,
+! leading to a lot of duplicate mlir::LLVM::DITypeAttr and actual LLVM IR
+! metadata, causing noticeable compilation slowdowns in apps with non trivial
+! derived types.
+
+subroutine duplicate_cycle_branch()
+  type t0
+    type(base), pointer :: x
+  end type
+  type t1
+    type(t0) :: x1_1
+    type(t0) :: x1_2
+  end type
+  type base
+    type(t1) :: x_base
+  end type
+  type(base) :: x
+  type(t0) :: x0
+  call bar(x, x0)
+end subroutine
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK-NOT: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
diff --git a/flang/test/Integration/skip-external-rtti-definition.F90 b/flang/test/Integration/skip-external-rtti-definition.F90
new file mode 100644
index 0000000000000..35a5d559efa5e
--- /dev/null
+++ b/flang/test/Integration/skip-external-rtti-definition.F90
@@ -0,0 +1,47 @@
+! Test -skip-external-rtti-definition option
+
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -fsyntax-only -DSTEP=1 -J%t %s
+!RUN: %flang_fc1 -emit-llvm -J%t  %s -o - | FileCheck %s -check-prefix=LINKONCE
+!RUN: %flang_fc1 -emit-llvm -J%t -mllvm -skip-external-rtti-definition %s -o - | FileCheck %s -check-prefix=EXTERNAL
+
+#if STEP == 1
+module module_external_type_definition
+ type t1
+ end type
+end module
+#else
+
+module module_same_unit_type_definition
+ type t2
+ end type
+end module
+
+subroutine test
+  use module_external_type_definition
+  use module_same_unit_type_definition
+  interface
+  subroutine needs_descriptor(x)
+    class(*) :: x
+  end subroutine
+  end interface
+  type(t1) :: x1
+  type(t2) :: x2
+  call needs_descriptor(x1)
+  call needs_descriptor(x2)
+end subroutine
+
+#endif
+
+! LINKONCE-DAG: @_QMmodule_external_type_definitionEXnXt1 = linkonce_odr constant [2 x i8] c"t1"
+! LINKONCE-DAG: @_QMmodule_external_type_definitionEXdtXt1 = linkonce_odr constant {{.*}} {
+! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXnXt2 = linkonce_odr constant [2 x i8] c"t2"
+! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXdtXt2 = linkonce_odr constant {{.*}} {
+
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
+! EXTERNAL: @_QMmodule_same_unit_type_definitionEXnXt2 = constant [2 x i8] c"t2"
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
+! EXTERNAL: @_QMmodule_same_unit_type_definitionEXdtXt2 = constant {{.*}} {
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
+! EXTERNAL: @_QMmodule_external_type_definitionEXdtXt1 =  external constant ptr
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
diff --git a/flang/test/Lower/CUDA/cuda-cooperative.cuf b/flang/test/Lower/CUDA/cuda-cooperative.cuf
index 54523b18b20db..657a87c0b5a05 100644
--- a/flang/test/Lower/CUDA/cuda-cooperative.cuf
+++ b/flang/test/Lower/CUDA/cuda-cooperative.cuf
@@ -50,3 +50,50 @@ end subroutine
 ! CHECK: fir.store %[[SIZE]] to %[[COORD_SIZE]] : !fir.ref<i32>
 ! CHECK: %[[COORD_RANK:.*]] = fir.coordinate_of %[[RES]], rank : (!fir.ref<!fir.type<_QMcooperative_groupsTgrid_group{_QMcooperative_groupsTgrid_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
 ! CHECK: fir.store %[[RANK]] to %[[COORD_RANK]] : !fir.ref<i32>
+
+attributes(grid_global) subroutine w1()
+  use cooperative_groups
+  type(coalesced_group) :: gg
+  gg = this_warp()
+end subroutine
+
+! CHECK: %[[WARPSIZE:.*]] = fir.alloca i32 {bindc_name = "__builtin_warpsize", uniq_name = "_QM__fortran_builtinsEC__builtin_warpsize"}
+! CHECK: %[[WARPSIZE_DECL:.*]]:2 = hlfir.declare %[[WARPSIZE]] {uniq_name = "_QM__fortran_builtinsEC__builtin_warpsize"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[COALESCED_GROUP:.*]] = fir.alloca !fir.type<_QMcooperative_groupsTcoalesced_group{_QMcooperative_groupsTcoalesced_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[C32:.*]] = arith.constant 32 : i32
+! CHECK: %[[SIZE_COORD:.*]] = fir.coordinate_of %[[COALESCED_GROUP]], size : (!fir.ref<!fir.type<_QMcooperative_groupsTcoalesced_group{_QMcooperative_groupsTcoalesced_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[C32]] to %[[SIZE_COORD]] : !fir.ref<i32>
+
+! CHECK: %[[THREAD_ID:.*]] = nvvm.read.ptx.sreg.tid.x : i32
+! CHECK: %[[C31:.*]] = arith.constant 31 : i32
+! CHECK: %[[C1:.*]] = arith.constant 1 : i32
+! CHECK: %[[AND:.*]] = arith.andi %[[THREAD_ID]], %[[C31]] : i32
+! CHECK: %[[RANK:.*]] = arith.addi %[[AND]], %[[C1]] : i32
+! CHECK: %[[RANK_COORD:.*]] = fir.coordinate_of %{{.*}}, rank : (!fir.ref<!fir.type<_QMcooperative_groupsTcoalesced_group{_QMcooperative_groupsTcoalesced_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+
+attributes(grid_global) subroutine t1()
+  use cooperative_groups
+  type(thread_group) :: gg
+  gg = this_thread_block()
+end subroutine
+! CHECK: %{{.*}} = fir.alloca !fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[THREAD_GROUP:.*]] = fir.alloca !fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[NTID_X:.*]] = nvvm.read.ptx.sreg.ntid.x : i32
+! CHECK: %[[NTID_Y:.*]] = nvvm.read.ptx.sreg.ntid.y : i32
+! CHECK: %[[NTID_Z:.*]] = nvvm.read.ptx.sreg.ntid.z : i32
+! CHECK: %[[SIZE_ZY:.*]] = arith.muli %[[NTID_Z]], %[[NTID_Y]] : i32
+! CHECK: %[[SIZE:.*]] = arith.muli %[[SIZE_ZY]], %[[NTID_X]] : i32
+! CHECK: %[[TID_X:.*]] = nvvm.read.ptx.sreg.tid.x : i32
+! CHECK: %[[TID_Y:.*]] = nvvm.read.ptx.sreg.tid.y : i32
+! CHECK: %[[TID_Z:.*]] = nvvm.read.ptx.sreg.tid.z : i32
+! CHECK: %[[RANK_ZY:.*]] = arith.muli %[[TID_Z]], %[[NTID_Y]] : i32
+! CHECK: %[[RANK_ZYX:.*]] = arith.muli %[[RANK_ZY]], %[[NTID_X]] : i32
+! CHECK: %[[RANK_YX:.*]] = arith.muli %[[TID_Y]], %[[NTID_X]] : i32
+! CHECK: %[[RANK_SUM1:.*]] = arith.addi %[[RANK_ZYX]], %[[RANK_YX]] : i32
+! CHECK: %[[RANK_SUM2:.*]] = arith.addi %[[RANK_SUM1]], %[[TID_X]] : i32
+! CHECK: %[[C1:.*]] = arith.constant 1 : i32
+! CHECK: %[[RANK:.*]] = arith.addi %[[RANK_SUM2]], %[[C1]] : i32
+! CHECK: %[[SIZE_COORD:.*]] = fir.coordinate_of %[[THREAD_GROUP]], size : (!fir.ref<!fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[SIZE]] to %[[SIZE_COORD]] : !fir.ref<i32>
+! CHECK: %[[RANK_COORD:.*]] = fir.coordinate_of %[[THREAD_GROUP]], rank : (!fir.ref<!fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[RANK]] to %[[RANK_COORD]] : !fir.ref<i32>
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index d0032af415604..68a0202f951fe 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -403,3 +403,19 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPsub20()
 ! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
+
+subroutine sub21()
+  real, allocatable,device:: a(:,:), b(:,:)
+  real:: s
+  integer:: i,j,N=16
+  allocate(a(N,N),b(N,N))
+  do concurrent(i=1:N, j=1:N) reduce(+:s)
+    b(i,j)=a(i,j)**2
+    s=s+b(i,j)
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub21()
+! CHECK: fir.do_concurrent.loop
+! CHECK-NOT: cuf.data_transfer
+! CHECK: hlfir.assign
diff --git a/flang/test/Lower/CUDA/cuda-derived.cuf b/flang/test/Lower/CUDA/cuda-derived.cuf
new file mode 100644
index 0000000000000..96250d88d81c4
--- /dev/null
+++ b/flang/test/Lower/CUDA/cuda-derived.cuf
@@ -0,0 +1,30 @@
+! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
+
+module m1
+  type ty_device
+    integer, device, allocatable, dimension(:) :: x
+  end type
+
+  type t1; real, device, allocatable :: a(:); end type
+  type t2; type(t1) :: b; end type
+contains
+  subroutine sub1()
+    type(ty_device) :: a
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub1()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} -> !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub1Ea"} : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
+! CHECK: cuf.free %[[DECL]]#0 : !fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}>> {data_attr = #cuf.cuda<managed>}
+
+end module
+
+program main
+  use m1
+  type(ty_device) :: a
+  type(t2) :: b
+end
+
+! CHECK-LABEL: func.func @_QQmain() attributes {fir.bindc_name = "main"}
+! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "a", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEa"}
+! CHECK: %{{.*}} = cuf.alloc !fir.type<_QMm1Tt2{b:!fir.type<_QMm1Tt1{a:!fir.box<!fir.heap<!fir.array<?xf32>>>}>}> {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QFEb"}
diff --git a/flang/test/Lower/CUDA/cuda-runtime-check.cuf b/flang/test/Lower/CUDA/cuda-runtime-check.cuf
index f26d372769cab..1aa95ec0ff405 100644
--- a/flang/test/Lower/CUDA/cuda-runtime-check.cuf
+++ b/flang/test/Lower/CUDA/cuda-runtime-check.cuf
@@ -1,22 +1,36 @@
 ! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
 
-! Check insertion of runtime checks
+! Check insertion of runtime checks in host side only
+
+module section_test
+
+contains
 
-interface
   subroutine foo(a)
     real, device, dimension(:,:) :: a
   end subroutine
-end interface
 
-  real, device, allocatable, dimension(:,:) :: a
-  allocate(a(10,10))
-  call foo(a(1:10,1:10:2))
-end
+  subroutine test_host()
+    real, device, allocatable, dimension(:,:) :: a
+    allocate(a(10,10))
+    call foo(a(1:10,1:10:2))
+  end subroutine
+
+  attributes(device) subroutine zoo(a)
+    real, device, dimension(:,:) :: a
+  end subroutine
 
-subroutine foo(a)
-  real, device, dimension(:,:) :: a
-end subroutine
+  attributes(global) subroutine test_device()
+    real, device, allocatable, dimension(:,:) :: a
+    allocate(a(10,10))
+    call zoo(a(1:10,1:10:2))
+  end subroutine
+end module
 
-! CHECK-LABEL: func.func @_QQmain()
+! CHECK-LABEL: func.func @_QMsection_testPtest_host()
 ! CHECK: fir.call @_FortranACUFDescriptorCheckSection
-! CHECK: fir.call @_QPfoo
+! CHECK: fir.call @_QMsection_testPfoo
+
+! CHECK-LABEL: func.func @_QMsection_testPtest_device()
+! CHECK-NOT: fir.call @_FortranACUFDescriptorCheckSection
+! CHECK: fir.call @_QMsection_testPzoo
diff --git a/flang/test/Lower/Intrinsics/acosd.f90 b/flang/test/Lower/Intrinsics/acosd.f90
index 3f558d66c35a1..7dfa28fd6494e 100644
--- a/flang/test/Lower/Intrinsics/acosd.f90
+++ b/flang/test/Lower/Intrinsics/acosd.f90
@@ -1,5 +1,3 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
-! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -8,10 +6,10 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 0.017453292519943295 : f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f32
 ! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f32
-! CHECK-PRECISE: %{{.*}} = fir.call @acosf(%[[arg]]) fastmath<contract> : (f32) -> f32
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
   real(8) :: x, test_real8
@@ -19,6 +17,6 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[factor:.*]] = arith.constant 0.017453292519943295 : f64
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f64
-! CHECK-PRECISE: %{{.*}} = fir.call @acos(%[[arg]]) fastmath<contract> : (f64) -> f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64
diff --git a/flang/test/Lower/Intrinsics/asind.f90 b/flang/test/Lower/Intrinsics/asind.f90
index e03f2e5598555..564fa955fa969 100644
--- a/flang/test/Lower/Intrinsics/asind.f90
+++ b/flang/test/Lower/Intrinsics/asind.f90
@@ -1,5 +1,3 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
-! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -8,10 +6,10 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 0.017453292519943295 : f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f32
 ! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f32
-! CHECK-PRECISE: %{{.*}} = fir.call @asinf(%[[arg]]) fastmath<contract> : (f32) -> f32
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
   real(8) :: x, test_real8
@@ -19,6 +17,6 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[factor:.*]] = arith.constant 0.017453292519943295 : f64
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f64
-! CHECK-PRECISE: %{{.*}} = fir.call @asin(%[[arg]]) fastmath<contract> : (f64) -> f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64
diff --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index aae8071b6a8b5..8d20ff8f3178f 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,5 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain
diff --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index f5f7cff106cee..df7c1e4ce7eaa 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPproduct_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32
diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b1263887..3167617b60457 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 3f840acefa6e8..f88bbea6fca85 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -107,8 +107,6 @@ program OmpAtomicUpdate
 !CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_Y_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK:  {{.*}} = arith.cmpi sgt, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.select {{.*}}, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.cmpi sgt, {{.*}}
 !CHECK:  %[[TEMP:.*]] = arith.select {{.*}} : i32
 !CHECK:  omp.yield(%[[TEMP]] : i32)
 !CHECK: }
@@ -177,13 +175,9 @@ program OmpAtomicUpdate
 !CHECK:   %[[VAL_Z_LOADED:.*]] = fir.load %[[VAL_Z_DECLARE]]#0 : !fir.ref<i32>
 !CHECK:   omp.atomic.update %[[VAL_W_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG_W:.*]]: i32):
-!CHECK:     %[[WX_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WX_MIN:.*]] = arith.select %[[WX_CMP]], %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WXY_CMP:.*]] = arith.cmpi sgt, %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXY_MIN:.*]] = arith.select %[[WXY_CMP]], %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXYZ_CMP:.*]] = arith.cmpi sgt, %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     %[[WXYZ_MIN:.*]] = arith.select %[[WXYZ_CMP]], %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     omp.yield(%[[WXYZ_MIN]] : i32)
+!CHECK:     %[[W_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], {{.*}} : i32
+!CHECK:     %[[WXYZ_MAX:.*]] = arith.select %[[W_CMP]], %[[ARG_W]], {{.*}} : i32
+!CHECK:     omp.yield(%[[WXYZ_MAX]] : i32)
 !CHECK:   }
   !$omp atomic update
     w = max(w,x,y,z)
diff --git a/flang/test/Lower/OpenMP/minmax-optional-parameters.f90 b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
new file mode 100644
index 0000000000000..418a3cad8cdaf
--- /dev/null
+++ b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
@@ -0,0 +1,68 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<f32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpf ogt, %[[V4]], %[[V5]] fastmath<contract> : f32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : f32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (f32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<f32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpf ogt, %[[V8]], %[[V10]] fastmath<contract> : f32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : f32
+!CHECK:   fir.result %[[V12]] : f32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : f32
+!CHECK: }
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
+end
+
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<i32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpi slt, %[[V4]], %[[V5]] : i32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : i32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (i32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<i32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpi slt, %[[V8]], %[[V10]] : i32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : i32
+!CHECK:   fir.result %[[V12]] : i32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : i32
+!CHECK: }
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32
+!CHECK:   omp.yield(%[[V11]] : i32)
+!CHECK: }
+
+subroutine f01(a, x, y)
+  integer :: a
+  integer, optional :: x, y
+  !$omp atomic update
+  a = min(x, a, y)
+end
+
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..a76e7e52100db
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for tile OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end do
+end subroutine
+
+!CHECK: not yet implemented: Unhandled loop directive (tile)
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..33b7c5a917619
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for unroll OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+program loop_transformation_construct
+  implicit none
+  integer, parameter :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do x = 1, I
+    y(x) = y(x) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end program loop_transformation_construct
+
+!CHECK: not yet implemented: Unhandled loop directive (unroll)
diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 017b2a3f2edd2..2bb1036c4a7fa 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,8 +1,8 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! REQUIRES: shell
-! RUN: bbc -fopenmp -emit-hlfir %s -o - \
+! RUN: bbc -target x86_64-unknown-linux-gnu -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 !CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
@@ -29,7 +29,7 @@
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -57,8 +57,8 @@ subroutine firstprivate_complex(arg1, arg2)
 !CHECK:    %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:    %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1,
-!%[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
+!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0,
+!%[[ARG5_PVT_DECL]]#0, %[[ARG6_PVT_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
 !CHECK:    omp.terminator
 !CHECK:  }
 
@@ -88,7 +88,7 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
-!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
+!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0, %[[ARG5_PVT_DECL]]#0) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -116,7 +116,7 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
@@ -136,7 +136,7 @@ subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK-KIND10:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
 !CHECK-KIND10:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND10:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f80>) -> ()
+!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f80>) -> ()
 !CHECK-KIND10:     omp.terminator
 !CHECK-KIND10:   }
 subroutine firstprivate_real10(arg1)
@@ -152,7 +152,7 @@ subroutine firstprivate_real10(arg1)
 !CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f128>) -> ()
 !CHECK-KIND16:     omp.terminator
 !CHECK-KIND16:   }
 subroutine firstprivate_real16(arg1)
@@ -171,7 +171,7 @@ subroutine firstprivate_real16(arg1)
 !CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[A_PRIV_ADDR:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[B_PRIV_ADDR:.*]] : {{.*}}) {
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#1, %[[B_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#0, %[[B_PRIV_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 !CHECK:             omp.terminator
 !CHECK:           }
 !CHECK:           return
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 8e3de498f59c1..d410714b1573e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -1,5 +1,9 @@
-! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s --check-prefix=CPU
+
+! RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device -fopenmp-is-gpu -o - %s 2>&1 | FileCheck %s --check-prefix=GPU
+! RUN: %if amdgpu-registered-target %{ %flang_fc1 -triple amdgcn-amd-amdhsa -emit-hlfir -fopenmp -fopenmp-is-target-device -o - %s 2>&1 | \
+! RUN: FileCheck %s --check-prefix=GPU %}
 
 program reduce
 integer, dimension(3) :: i = 0
@@ -13,81 +17,88 @@ program reduce
 print *,i
 end program
 
-! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
-! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK-LABEL:   } init {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
-! CHECK:           %[[TRUE:.*]]  = arith.constant true
+! CPU-LABEL:   omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref<!fir.box<!fir.array<3xi32>>> alloc {
+! CPU:           %[[VAL_8:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           omp.yield(%[[VAL_8]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU-LABEL:   } init {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[ALLOC:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = arith.constant 0 : i32
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_1:.*]] = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp", uniq_name = ""}
+! CPU:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3xi32>>,
+! CPU:           %[[TRUE:.*]]  = arith.constant true
 !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
-! CHECK:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         } combiner {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
-! CHECK:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
-! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
-! CHECK:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
-! CHECK:           }
-! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:         }  cleanup {
-! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
-! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
-! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
-! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:           fir.if %[[VAL_5]] {
-! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
-! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
-! CHECK:           }
-! CHECK:           omp.yield
-! CHECK:         }
+! CPU:           %[[C0:.*]] = arith.constant 0 : index
+! CPU:           %[[DIMS:.*]]:3 = fir.box_dims %[[VAL_3]], %[[C0]] : (!fir.box<!fir.array<3xi32>>, index) -> (index, index, index)
+! CPU:           %[[SHIFT:.*]] = fir.shape_shift %[[DIMS]]#0, %[[DIMS]]#1 : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[SHIFT]]) : (!fir.heap<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           hlfir.assign %[[VAL_2]] to %[[VAL_7]] : i32, !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_7]] to %[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.yield(%[[ALLOC]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         } combiner {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[C1:.*]] = arith.constant 1 : index
+! CPU:           %[[C3:.*]] = arith.constant 3 : index
+! CPU:           %[[SHAPE_SHIFT:.*]] = fir.shape_shift %[[C1]], %[[C3]] : (index, index) -> !fir.shapeshift<1>
+! CPU:           %[[C1_0:.*]] = arith.constant 1 : index
+! CPU:           fir.do_loop %[[VAL_8:.*]] = %[[C1_0]] to %[[C3]] step %[[C1_0]] unordered {
+! CPU:             %[[VAL_9:.*]] = fir.array_coor %[[VAL_2]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_10:.*]] = fir.array_coor %[[VAL_3]](%[[SHAPE_SHIFT]]) %[[VAL_8]] : (!fir.box<!fir.array<3xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
+! CPU:             %[[VAL_11:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = fir.load %[[VAL_10]] : !fir.ref<i32>
+! CPU:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CPU:             fir.store %[[VAL_13]] to %[[VAL_9]] : !fir.ref<i32>
+! CPU:           }
+! CPU:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:         }  cleanup {
+! CPU:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3xi32>>>):
+! CPU:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3xi32>>) -> !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> i64
+! CPU:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CPU:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CPU:           fir.if %[[VAL_5]] {
+! CPU:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3xi32>>) -> !fir.heap<!fir.array<3xi32>>
+! CPU:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3xi32>>
+! CPU:           }
+! CPU:           omp.yield
+! CPU:         }
+
+! CPU-LABEL:   func.func @_QQmain()
+! CPU:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
+! CPU:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CPU:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! CPU:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CPU:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
+! CPU:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
+! CPU:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CPU:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
+! CPU:             %[[VAL_8:.*]] = arith.constant 1 : i32
+! CPU:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_10:.*]] = arith.constant 1 : index
+! CPU:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_12:.*]] = arith.constant 2 : i32
+! CPU:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_14:.*]] = arith.constant 2 : index
+! CPU:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
+! CPU:             %[[VAL_16:.*]] = arith.constant 3 : i32
+! CPU:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
+! CPU:             %[[VAL_18:.*]] = arith.constant 3 : index
+! CPU:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+! CPU:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
+! CPU:             omp.terminator
+! CPU:           }
 
-! CHECK-LABEL:   func.func @_QQmain()
-! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEi) : !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
-! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
-! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
-! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : index
-! CHECK:             %[[VAL_11:.*]] = hlfir.designate %[[VAL_9]] (%[[VAL_10]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_11]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_12:.*]] = arith.constant 2 : i32
-! CHECK:             %[[VAL_13:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_14:.*]] = arith.constant 2 : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_13]] (%[[VAL_14]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_12]] to %[[VAL_15]] : i32, !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = arith.constant 3 : i32
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:             %[[VAL_18:.*]] = arith.constant 3 : index
-! CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_18]])  : (!fir.box<!fir.array<3xi32>>, index) -> !fir.ref<i32>
-! CHECK:             hlfir.assign %[[VAL_16]] to %[[VAL_19]] : i32, !fir.ref<i32>
-! CHECK:             omp.terminator
-! CHECK:           }
+! GPU: omp.declare_reduction {{.*}} alloc {
+! GPU: } init {
+! GPU-NOT: fir.allocmem {{.*}} {bindc_name = ".tmp", {{.*}}}
+! GPU:     fir.alloca {{.*}} {bindc_name = ".tmp"}
+! GPU: } combiner {
+! GPU: }
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 18e252a787d78..cf5c1b127c34d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -147,7 +147,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 7f3e3922ab29b..eff97aa5d0db0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -139,7 +139,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index e7654edffec8b..bbe4bf04b6827 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -89,7 +89,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -146,7 +146,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index dffd6545b95bf..304c7eefd58b0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
new file mode 100644
index 0000000000000..f7fd2a52060e6
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
@@ -0,0 +1,24 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Check that logical reductions of different kinds do not end up using the same
+! reduction declaration
+
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l64 : !fir.logical<8> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
+
+subroutine test(a4, a8, sz)
+  integer :: sz
+  logical(4), dimension(sz) :: a4
+  logical(8), dimension(sz) :: a8
+
+  logical(4) :: res4 = .false.
+  logical(8) :: res8 = .false.
+  integer i
+
+! CHECK: omp.wsloop private({{.*}}) reduction(@or_reduction_l32 {{.*}}, @or_reduction_l64 {{.*}}) {
+  !$omp do reduction(.or.:res4, res8)
+  do i = 1,sz
+    res4 = res4 .or. a4(i)
+    res8 = res8 .or. a8(i)
+  enddo
+end subroutine
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 366dd67591cfd..8c869952547c6 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -149,7 +149,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index 66f624ff493fe..e0901bec6bc9c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -141,7 +141,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index a867d0a550bc7..bb09e2563e378 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -41,7 +41,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -88,7 +88,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -145,7 +145,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 592d4fa516bd7..5c9bcbf584d48 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index fdd13719c5b71..38954626e77ee 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: @_QPadd_test
 subroutine add_test(a,b,c)
diff --git a/flang/test/Lower/math-lowering/aint.f90 b/flang/test/Lower/math-lowering/aint.f90
index 6c7809f8ea1a3..b36fb421d7c8c 100644
--- a/flang/test/Lower/math-lowering/aint.f90
+++ b/flang/test/Lower/math-lowering/aint.f90
@@ -1,10 +1,10 @@
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
 
 function test_real4(x)
   real :: x, test_real4
diff --git a/flang/test/Lower/select-type-2.f90 b/flang/test/Lower/select-type-2.f90
index 7b4cf96069e53..59485c176c76a 100644
--- a/flang/test/Lower/select-type-2.f90
+++ b/flang/test/Lower/select-type-2.f90
@@ -30,8 +30,8 @@ subroutine select_type1(a)
 
 ! CHECK-LABEL: func.func @_QMselect_type_2Pselect_type1(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
-! CHECK:      %[[TDESC_P3_ADDR:.*]] = fir.address_of(@_QMselect_type_2E.dt.p3) : !fir.ref<!fir.type<{{.*}}>>
-! CHECK:      %[[TDESC_P3_CONV:.*]] = fir.convert %[[TDESC_P3_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CHECK:      %[[TDESC_P3_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_2Tp3
+! CHECK:      %[[TDESC_P3_CONV:.*]] = fir.convert %[[TDESC_P3_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CHECK:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CHECK:      %[[CLASS_IS_CMP:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P3_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CHECK:      cf.cond_br %[[CLASS_IS_CMP]], ^[[CLASS_IS_P3_BLK:.*]], ^[[NOT_CLASS_IS_P3_BLK:.*]]
@@ -40,8 +40,8 @@ subroutine select_type1(a)
 ! CHECK:    ^bb[[CLASS_IS_P1:[0-9]]]:
 ! CHECK:      cf.br ^bb[[END_SELECT_BLK:[0-9]]]
 ! CHECK:    ^[[NOT_CLASS_IS_P3_BLK]]:
-! CHECK:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_2E.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CHECK:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CHECK:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_2Tp1
+! CHECK:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CHECK:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CHECK:      %[[CLASS_IS_CMP:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CHECK:      cf.cond_br %[[CLASS_IS_CMP]], ^bb[[CLASS_IS_P1]], ^bb[[NOT_CLASS_IS_P1]]
diff --git a/flang/test/Lower/select-type.f90 b/flang/test/Lower/select-type.f90
index 64dd639731ab1..e2ca87ad447c9 100644
--- a/flang/test/Lower/select-type.f90
+++ b/flang/test/Lower/select-type.f90
@@ -68,15 +68,15 @@ subroutine select_type1(a)
 
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type1(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> index
+! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P1_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[TYPE_IS_P1_BLK:.*]], ^[[NOT_TYPE_IS_P1_BLK:.*]]
 ! CFG:    ^[[NOT_TYPE_IS_P1_BLK]]:
-! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p2) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp2{{.*}}
+! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.tdesc<{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P2_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^bb[[CLASS_IS_P2_BLK:.*]], ^[[NOT_CLASS_IS_P2_BLK:.*]]
@@ -87,8 +87,8 @@ subroutine select_type1(a)
 ! CFG:    ^bb[[CLASS_IS_P1_BLK:[0-9]]]:
 ! CFG:      cf.br ^[[END_SELECT_BLK:.*]]
 ! CFG:    ^[[NOT_CLASS_IS_P2_BLK]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc<{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^bb[[CLASS_IS_P1_BLK]], ^bb[[NOT_CLASS_IS_P1_BLK]]
@@ -126,15 +126,15 @@ subroutine select_type2()
 ! CFG:     %[[GET_CLASS:.*]] = fir.call @_QMselect_type_lower_testPget_class() {{.*}} : () -> !fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>
 ! CFG:     fir.save_result %[[GET_CLASS]] to %[[CLASS_ALLOCA]] : !fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>, !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>
 ! CFG:     %[[LOAD_CLASS:.*]] = fir.load %[[CLASS_ALLOCA]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>
-! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:     %[[CLASS_TDESC:.*]] = fir.box_tdesc %[[LOAD_CLASS]] : (!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>) -> !fir.tdesc<none>
-! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:     %[[BOX_TDESC_CONV:.*]] = fir.convert %[[CLASS_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:     %[[CLASS_TDESC:.*]] = fir.box_tdesc %[[LOAD_CLASS]] : (!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>) -> !fir.tdesc<{{.*}}>
+! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:     %[[BOX_TDESC_CONV:.*]] = fir.convert %[[CLASS_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:     %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P1_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:     cf.cond_br %[[TDESC_CMP]], ^[[TYPE_IS_P1_BLK:.*]], ^[[NOT_TYPE_IS_P1_BLK:.*]]
 ! CFG:   ^[[NOT_TYPE_IS_P1_BLK]]:
-! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:     %[[BOX_NONE:.*]] = fir.convert %[[LOAD_CLASS]] : (!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>) -> !fir.box<none>
 ! CFG:     %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:     cf.cond_br %[[CLASS_IS]], ^[[CLASS_IS_BLK:.*]], ^[[NOT_CLASS_IS_BLK:.*]]
@@ -176,15 +176,15 @@ subroutine select_type3(a)
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type3(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "a"}) {
 ! CFG:      %[[SELECTOR:.*]] = fir.embox %{{.*}} source_box %{{.*}} : (!fir.ref<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>, !fir.class<{{.*}}>) -> !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[SELECTOR_TDESC:.*]] = fir.box_tdesc %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[TDESC_CONV:.*]] = fir.convert %[[SELECTOR_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[SELECTOR_TDESC:.*]] = fir.box_tdesc %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:      %[[TDESC_CONV:.*]] = fir.convert %[[SELECTOR_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P1_CONV]], %[[TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[TYPE_IS_P1_BLK:.*]], ^[[NOT_TYPE_IS_P1_BLK:.*]]
 ! CFG:    ^[[NOT_TYPE_IS_P1_BLK]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^[[CLASS_IS_BLK:.*]], ^[[NOT_CLASS_IS:.*]]
@@ -222,25 +222,25 @@ subroutine select_type4(a)
 
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type4(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
-! CFG:      %[[TDESC_P3_8_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p3.8) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P3_8_CONV:.*]] = fir.convert %[[TDESC_P3_8_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P3_8_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp3K8
+! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
+! CFG:      %[[TDESC_P3_8_CONV:.*]] = fir.convert %[[TDESC_P3_8_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P3_8_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[P3_8_BLK:.*]], ^[[NOT_P3_8_BLK:.*]]
 ! CFG:    ^[[NOT_P3_8_BLK]]:
-! CFG:      %[[TDESC_P3_4_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p3.4) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P3_4_CONV:.*]] = fir.convert %[[TDESC_P3_4_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P3_4_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp3K4
+! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
+! CFG:      %[[TDESC_P3_4_CONV:.*]] = fir.convert %[[TDESC_P3_4_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P3_4_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[P3_4_BLK:.*]], ^[[NOT_P3_4_BLK:.*]]
 ! CFG:    ^[[P3_8_BLK]]:
 ! CFG:      _FortranAioOutputAscii
 ! CFG:      cf.br ^bb[[EXIT_SELECT_BLK:[0-9]]]
 ! CFG:    ^[[NOT_P3_4_BLK]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^[[P1_BLK:.*]], ^[[NOT_P1_BLK:.*]]
@@ -409,8 +409,8 @@ subroutine select_type7(a)
 
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type7(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"}) {
-! CFG:      %[[TDESC_P4_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p4) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P4_CONV:.*]] = fir.convert %[[TDESC_P4_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P4_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp4
+! CFG:      %[[TDESC_P4_CONV:.*]] = fir.convert %[[TDESC_P4_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<none>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS_P4:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P4_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS_P4]], ^[[CLASS_IS_P4_BLK:.*]], ^[[CLASS_NOT_P4_BLK:.*]]
@@ -419,16 +419,16 @@ subroutine select_type7(a)
 ! CFG:    ^bb[[CLASS_IS_P1_BLK:[0-9]]]:
 ! CFG:      cf.br ^[[EXIT_SELECT_BLK:.*]]
 ! CFG:    ^bb[[CLASS_NOT_P2_BLK:[0-9]]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<none>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS_P1:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS_P1]], ^bb[[CLASS_IS_P1_BLK]], ^bb[[CLASS_NOT_P1_BLK]]
 ! CFG:    ^bb[[CLASS_IS_P2_BLK:[0-9]]]:
 ! CFG:      cf.br ^[[EXIT_SELECT_BLK]]
 ! CFG:    ^[[CLASS_NOT_P4_BLK]]:
-! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p2) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp2
+! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<none>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS_P2:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P2_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS_P2]], ^bb[[CLASS_IS_P2_BLK]], ^bb[[CLASS_NOT_P2_BLK]]
diff --git a/flang/test/Parser/OpenMP/declare-variant.f90 b/flang/test/Parser/OpenMP/declare-variant.f90
index 1b97733ea9525..3366b143e62e6 100644
--- a/flang/test/Parser/OpenMP/declare-variant.f90
+++ b/flang/test/Parser/OpenMP/declare-variant.f90
@@ -102,3 +102,30 @@ subroutine vsub(v1, v2)
     type(c_ptr), value :: v1, v2
   end
 end subroutine
+
+subroutine f
+  real :: x, y
+  y = 2
+  !omp simd
+  call f2(x, y)
+  !omp end simd 
+contains
+  subroutine f1 (x, y)
+    real :: x, y
+  end
+
+  subroutine f2 (x, y)
+    real :: x, y
+    !$omp declare variant (f1) match (construct={simd(uniform(y))})
+  end
+end subroutine
+!CHECK: !$OMP DECLARE VARIANT (f1) MATCH(CONSTRUCT={SIMD(UNIFORM(y))})
+!PARSE-TREE: | | | | DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE-NEXT: | | | | | Verbatim
+!PARSE-TREE-NEXT: | | | | | Name = 'f1'
+!PARSE-TREE-NEXT: | | | | | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSelector
+!PARSE-TREE-NEXT: | | | | | | | OmpTraitSelectorName -> Value = Simd
+!PARSE-TREE-NEXT: | | | | | | | Properties
+!PARSE-TREE-NEXT: | | | | | | | | OmpTraitProperty -> OmpClause -> Uniform -> Name = 'y'
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct01.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..baffc2f6e2f1e
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,74 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with 1 nested loop.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct02.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
new file mode 100644
index 0000000000000..b50e7183841cc
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
@@ -0,0 +1,85 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with multiple nested loops.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP TILE
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END TILE
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct03.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
new file mode 100644
index 0000000000000..81f0de1b76263
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct03.f90
@@ -0,0 +1,75 @@
+! This reproducer hit an issue where when finding directive's, and end directive's would iterate over the block.end()
+! so Flang would crash. We should be able to parse this subroutine without flang crashing.
+! Reported in https://github.com/llvm/llvm-project/issues/147309 and https://github.com/llvm/llvm-project/pull/145917#issuecomment-3041570824
+
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct7
+implicit none
+real(kind=8), dimension(1:10, 2) :: a
+integer :: b,c
+
+!$omp target teams distribute parallel do collapse(2) private(b)
+do b = 1, 10
+  do c = 1, 10
+    a(b, 2) = a(c, 1)
+  end do
+end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = target teams distribute parallel do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList -> OmpClause -> Collapse -> Scalar -> Integer -> Constant -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | OmpClause -> Private -> OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | Block
+!CHECK-PARSE-NEXT: | | | | | ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '10_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '10'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'a(int(b,kind=8),2_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'b'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '2_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '2'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = 'a(int(c,kind=8),1_8)'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'a'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'c'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | EndSubroutineStmt ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct7
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  REAL(KIND=8_4), DIMENSION(1_4:10_4,2_4) :: a
+!CHECK-UNPARSE-NEXT:  INTEGER b, c
+!CHECK-UNPARSE-NEXT: !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO  COLLAPSE(2_4) PRIVATE(b)
+!CHECK-UNPARSE-NEXT:  DO b=1_4,10_4
+!CHECK-UNPARSE-NEXT:   DO c=1_4,10_4
+!CHECK-UNPARSE-NEXT:     a(int(b,kind=8),2_8)=a(int(c,kind=8),1_8)
+!CHECK-UNPARSE-NEXT:   END DO
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/issue-146362.1.f90 b/flang/test/Parser/issue-146362.1.f90
new file mode 100644
index 0000000000000..a37fd4b182a7f
--- /dev/null
+++ b/flang/test/Parser/issue-146362.1.f90
@@ -0,0 +1,5 @@
+! RUN: %flang_fc1 -fsyntax-only -cpp %s 2>&1
+#define UNITY(k) 1_ ## k
+PROGRAM REPRODUCER
+WRITE(*,*) UNITY(4)
+END PROGRAM REPRODUCER
\ No newline at end of file
diff --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
new file mode 100644
index 0000000000000..d9ad8c7d547f2
--- /dev/null
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -0,0 +1,18 @@
+!RUN: %flang_fc1 -cpp -fdebug-unparse %s | FileCheck %s
+PROGRAM P
+#if (1 && 2)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+#if ((1 || 2) != 3)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+END PROGRAM
+
diff --git a/flang/test/Preprocessing/bug1077.F90 b/flang/test/Preprocessing/bug1077.F90
new file mode 100644
index 0000000000000..dd7391813a357
--- /dev/null
+++ b/flang/test/Preprocessing/bug1077.F90
@@ -0,0 +1,7 @@
+!RUN: %flang -E %s 2>&1 | FileCheck %s
+!CHECK: print *,((1)+(2)),4
+#define foo(x,y) ((x)+(y))
+print *,&
+foo(1,2)&
+,4
+end
diff --git a/flang/test/Preprocessing/bug518.F b/flang/test/Preprocessing/bug518.F
index 0b680dd5751b9..beb33564e4232 100644
--- a/flang/test/Preprocessing/bug518.F
+++ b/flang/test/Preprocessing/bug518.F
@@ -1,5 +1,5 @@
 ! RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
 ! CHECK: k=1_4
                         k=                                            1_99999999
-     &4
+     & 4
       end
diff --git a/flang/test/Preprocessing/pp111.F90 b/flang/test/Preprocessing/pp111.F90
index 4da45ef35f5c0..bbf8709c3ab15 100644
--- a/flang/test/Preprocessing/pp111.F90
+++ b/flang/test/Preprocessing/pp111.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = IFLM(666)
 ! FLM call name split across continuation, no leading &, with & ! comment
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Preprocessing/pp112.F90 b/flang/test/Preprocessing/pp112.F90
index 16705527f68c3..a5244410f31af 100644
--- a/flang/test/Preprocessing/pp112.F90
+++ b/flang/test/Preprocessing/pp112.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = IFLM(666)
 ! ditto, but without & ! comment
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Preprocessing/pp115.F90 b/flang/test/Preprocessing/pp115.F90
index 4e4c621110ed8..eea42c53b936d 100644
--- a/flang/test/Preprocessing/pp115.F90
+++ b/flang/test/Preprocessing/pp115.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = ((666)+111)
 ! ditto, with & ! comment, no leading &
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Preprocessing/pp116.F90 b/flang/test/Preprocessing/pp116.F90
index e35a13cbf6489..39edf95763eab 100644
--- a/flang/test/Preprocessing/pp116.F90
+++ b/flang/test/Preprocessing/pp116.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = ((666)+111)
 ! FLM call split between name and (, no leading &
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Semantics/OpenACC/acc-branch.f90 b/flang/test/Semantics/OpenACC/acc-branch.f90
index b3692d0165890..a2d7b583f6794 100644
--- a/flang/test/Semantics/OpenACC/acc-branch.f90
+++ b/flang/test/Semantics/OpenACC/acc-branch.f90
@@ -74,7 +74,7 @@ subroutine openacc_clause_validity
   ! Exit branches out of parallel construct, attached to an OpenACC parallel construct.
   thisblk: BLOCK
     fortname: if (.true.) then
-      !PORTABILITY: The construct name 'name1' should be distinct at the subprogram level
+      !PORTABILITY: The construct name 'name1' should be distinct at the subprogram level [-Wbenign-name-clash]
       name1: do k = 1, N
         !$acc parallel
         !ERROR: EXIT to construct 'fortname' outside of PARALLEL construct is not allowed
diff --git a/flang/test/Semantics/OpenACC/acc-data.f90 b/flang/test/Semantics/OpenACC/acc-data.f90
index 473b91d555cea..98582f8174896 100644
--- a/flang/test/Semantics/OpenACC/acc-data.f90
+++ b/flang/test/Semantics/OpenACC/acc-data.f90
@@ -132,7 +132,7 @@ program openacc_data_validity
   !ERROR: At least one of COPYOUT, DELETE, DETACH clause must appear on the EXIT DATA directive
   !$acc exit data
 
-  !PORTABILITY: At least one of ATTACH, COPY, COPYIN, COPYOUT, CREATE, DEFAULT, DEVICEPTR, NO_CREATE, PRESENT clause should appear on the DATA directive
+  !PORTABILITY: At least one of ATTACH, COPY, COPYIN, COPYOUT, CREATE, DEFAULT, DEVICEPTR, NO_CREATE, PRESENT clause should appear on the DATA directive [-Wportability]
   !$acc data
   !$acc end data
 
diff --git a/flang/test/Semantics/OpenACC/acc-declare-validity.f90 b/flang/test/Semantics/OpenACC/acc-declare-validity.f90
index 7cdc69436704b..5ee61fac0660a 100644
--- a/flang/test/Semantics/OpenACC/acc-declare-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-declare-validity.f90
@@ -14,7 +14,7 @@ module openacc_declare_validity
 
   !$acc declare create(aa, bb)
 
-  !WARNING: 'aa' in the CREATE clause is already present in the same clause in this module
+  !WARNING: 'aa' in the CREATE clause is already present in the same clause in this module [-Wopen-acc-usage]
   !$acc declare create(aa)
 
   !$acc declare link(ab)
diff --git a/flang/test/Semantics/OpenACC/acc-serial.f90 b/flang/test/Semantics/OpenACC/acc-serial.f90
index 1f22003ed6b11..f3b81c992e6ea 100644
--- a/flang/test/Semantics/OpenACC/acc-serial.f90
+++ b/flang/test/Semantics/OpenACC/acc-serial.f90
@@ -84,15 +84,15 @@ program openacc_serial_validity
   !$acc serial wait(wait1) wait(wait2)
   !$acc end serial
 
-  !PORTABILITY: NUM_GANGS clause is not allowed on the SERIAL directive and will be ignored
+  !PORTABILITY: NUM_GANGS clause is not allowed on the SERIAL directive and will be ignored [-Wportability]
   !$acc serial num_gangs(8)
   !$acc end serial
 
-  !PORTABILITY: NUM_WORKERS clause is not allowed on the SERIAL directive and will be ignored
+  !PORTABILITY: NUM_WORKERS clause is not allowed on the SERIAL directive and will be ignored [-Wportability]
   !$acc serial num_workers(8)
   !$acc end serial
 
-  !PORTABILITY: VECTOR_LENGTH clause is not allowed on the SERIAL directive and will be ignored
+  !PORTABILITY: VECTOR_LENGTH clause is not allowed on the SERIAL directive and will be ignored [-Wportability]
   !$acc serial vector_length(128)
   !$acc end serial
 
diff --git a/flang/test/Semantics/OpenMP/allocate-align01.f90 b/flang/test/Semantics/OpenMP/allocate-align01.f90
index bc17d7047bbb5..508efa82f12a0 100644
--- a/flang/test/Semantics/OpenMP/allocate-align01.f90
+++ b/flang/test/Semantics/OpenMP/allocate-align01.f90
@@ -13,7 +13,7 @@ program allocate_align_tree
     z = 3
     !ERROR: The alignment value should be a constant positive integer
 !$omp allocate(j) align(xx)
-    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
+    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead. [-Wopen-mp-usage]
     !ERROR: The alignment value should be a constant positive integer
 !$omp allocate(xarray) align(-32) allocator(omp_large_cap_mem_alloc)
     allocate(j(z), xarray(t))
diff --git a/flang/test/Semantics/OpenMP/allocate01.f90 b/flang/test/Semantics/OpenMP/allocate01.f90
index b205b2c79d65b..e0b084ff0030b 100644
--- a/flang/test/Semantics/OpenMP/allocate01.f90
+++ b/flang/test/Semantics/OpenMP/allocate01.f90
@@ -19,7 +19,7 @@ subroutine sema()
     !$omp allocate(y)
         print *, a
 
-    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
+    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead. [-Wopen-mp-usage]
     !ERROR: List items must be declared in the same scoping unit in which the ALLOCATE directive appears
     !$omp allocate(x) allocator(omp_default_mem_alloc)
       allocate ( x(a), darray(a, b) )
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 6989a183e83ed..bc1371847b792 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -483,14 +483,14 @@
 ! 2.13.1 master
 
   !$omp parallel
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead. [-Wopen-mp-usage]
   !$omp master
   a=3.14
   !$omp end master
   !$omp end parallel
 
   !$omp parallel
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead. [-Wopen-mp-usage]
   !ERROR: NUM_THREADS clause is not allowed on the MASTER directive
   !$omp master num_threads(4)
   a=3.14
diff --git a/flang/test/Semantics/OpenMP/copying.f90 b/flang/test/Semantics/OpenMP/copying.f90
index 63fb39a0f26e5..d79027361f2ab 100644
--- a/flang/test/Semantics/OpenMP/copying.f90
+++ b/flang/test/Semantics/OpenMP/copying.f90
@@ -10,7 +10,7 @@
 subroutine firstprivate()
   class(*), allocatable, save :: x
 
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in FIRSTPRIVATE clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in FIRSTPRIVATE clause, the behavior is unspecified [-Wportability]
   !$omp parallel firstprivate(x)
     call sub()
   !$omp end parallel
@@ -20,7 +20,7 @@ subroutine firstprivate()
 subroutine lastprivate()
   class(*), allocatable, save :: x
 
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in LASTPRIVATE clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in LASTPRIVATE clause, the behavior is unspecified [-Wportability]
   !$omp do lastprivate(x)
   do i = 1, 10
     call sub()
@@ -33,7 +33,7 @@ subroutine copyin()
   class(*), allocatable, save :: x
   !$omp threadprivate(x)
 
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYIN clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYIN clause, the behavior is unspecified [-Wportability]
   !$omp parallel copyin(x)
     call sub()
   !$omp end parallel
@@ -46,7 +46,7 @@ subroutine copyprivate()
 
   !$omp single
     call sub()
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYPRIVATE clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYPRIVATE clause, the behavior is unspecified [-Wportability]
   !$omp end single copyprivate(x)
 
 end
diff --git a/flang/test/Semantics/OpenMP/declarative-directive01.f90 b/flang/test/Semantics/OpenMP/declarative-directive01.f90
index e8bf605565fad..9e6e6021ffab2 100644
--- a/flang/test/Semantics/OpenMP/declarative-directive01.f90
+++ b/flang/test/Semantics/OpenMP/declarative-directive01.f90
@@ -56,10 +56,10 @@ module m2
 contains
   subroutine foo
     !$omp declare target
-    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive
-    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive
+    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive [-Wopen-mp-usage]
+    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive [-Wopen-mp-usage]
     !$omp declare target (foo, N, M)
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
@@ -68,7 +68,7 @@ subroutine foo
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !$omp declare target enter(Q, S) link(R)
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: MAP clause is not allowed on the DECLARE TARGET directive
     !$omp declare target to(Q) map(from:Q)
diff --git a/flang/test/Semantics/OpenMP/declare-target01.f90 b/flang/test/Semantics/OpenMP/declare-target01.f90
index 0651d3b5d89c1..39185b171b6b4 100644
--- a/flang/test/Semantics/OpenMP/declare-target01.f90
+++ b/flang/test/Semantics/OpenMP/declare-target01.f90
@@ -51,84 +51,84 @@ module declare_target01
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target (y%KIND)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var)
 
   !$omp declare target enter (my_var)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var) device_type(host)
   
   !$omp declare target enter (my_var) device_type(host)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%t_i)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%t_i)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%t_arr)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%t_arr)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%kind_param)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%kind_param)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%len_param)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%len_param)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr)
 
   !$omp declare target enter (arr)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr) device_type(nohost)
 
   !$omp declare target enter (arr) device_type(nohost)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr(1))
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (arr(1))
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr(1:2))
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (arr(1:2))
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (x%KIND)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (x%KIND)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (w%LEN)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (w%LEN)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (y%KIND)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
diff --git a/flang/test/Semantics/OpenMP/declare-target02.f90 b/flang/test/Semantics/OpenMP/declare-target02.f90
index 0f12180587f83..5146ed8c89f08 100644
--- a/flang/test/Semantics/OpenMP/declare-target02.f90
+++ b/flang/test/Semantics/OpenMP/declare-target02.f90
@@ -16,17 +16,17 @@ program declare_target02
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target (a1)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr1_to)
 
   !$omp declare target enter (arr1_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (blk1_to)
 
   !$omp declare target enter (blk1_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target to (a1_to)
 
@@ -44,7 +44,7 @@ program declare_target02
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target (eq_a)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target to (eq_a)
 
@@ -57,7 +57,7 @@ program declare_target02
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target (eq_c)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target to (eq_c)
 
@@ -87,26 +87,26 @@ subroutine func()
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target (a3)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !$omp declare target to (arr2_to)
 
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !$omp declare target enter (arr2_to)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !$omp declare target to (arr3_to)
 
     !$omp declare target enter (arr3_to)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target to (a2_to)
 
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target enter (a2_to)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target to (a3_to)
 
@@ -137,16 +137,16 @@ module mod4
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target (a4)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr4_to)
 
   !$omp declare target enter (arr4_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (blk4_to)
   !$omp declare target enter (blk4_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target to (a4_to)
 
@@ -174,21 +174,21 @@ subroutine func5()
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target (a5)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target to (arr5_to)
   
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target enter (arr5_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target to (blk5_to)
 
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target enter (blk5_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target to (a5_to)
 
diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index bb1ed90e390f3..64a299d78224a 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -16,7 +16,7 @@ program main
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (main)
 end
diff --git a/flang/test/Semantics/OpenMP/declare-target06.f90 b/flang/test/Semantics/OpenMP/declare-target06.f90
index 7df0a73123094..08fe531af6953 100644
--- a/flang/test/Semantics/OpenMP/declare-target06.f90
+++ b/flang/test/Semantics/OpenMP/declare-target06.f90
@@ -12,7 +12,7 @@ module test_0
 !ERROR: No explicit type declared for 'no_implicit_materialization_2'
 !$omp declare target link(no_implicit_materialization_2)
 
-!WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+!WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
 !ERROR: No explicit type declared for 'no_implicit_materialization_3'
 !$omp declare target to(no_implicit_materialization_3)
 
diff --git a/flang/test/Semantics/OpenMP/deprecation.f90 b/flang/test/Semantics/OpenMP/deprecation.f90
index df15c3bcc0b13..bc1c96d595e71 100644
--- a/flang/test/Semantics/OpenMP/deprecation.f90
+++ b/flang/test/Semantics/OpenMP/deprecation.f90
@@ -4,7 +4,7 @@
 
 subroutine test_master()
   integer :: c = 1
-!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead. [-Wopen-mp-usage]
   !$omp master 
   c = c + 1
   !$omp end master
@@ -12,7 +12,7 @@ subroutine test_master()
 
 subroutine test_parallel_master
   integer :: c = 2
-!WARNING: OpenMP directive PARALLEL MASTER has been deprecated, please use PARALLEL MASKED instead.
+!WARNING: OpenMP directive PARALLEL MASTER has been deprecated, please use PARALLEL MASKED instead. [-Wopen-mp-usage]
   !$omp parallel master
   c = c + 2
   !$omp end parallel master
@@ -20,7 +20,7 @@ subroutine test_parallel_master
 
 subroutine test_master_taskloop_simd()
   integer :: i, j = 1
-!WARNING: OpenMP directive MASTER TASKLOOP SIMD has been deprecated, please use MASKED TASKLOOP SIMD instead.
+!WARNING: OpenMP directive MASTER TASKLOOP SIMD has been deprecated, please use MASKED TASKLOOP SIMD instead. [-Wopen-mp-usage]
   !$omp master taskloop simd 
   do i=1,10
    j = j + 1
@@ -30,7 +30,7 @@ subroutine test_master_taskloop_simd()
 
 subroutine test_master_taskloop
   integer :: i, j = 1
-!WARNING: OpenMP directive MASTER TASKLOOP has been deprecated, please use MASKED TASKLOOP instead.
+!WARNING: OpenMP directive MASTER TASKLOOP has been deprecated, please use MASKED TASKLOOP instead. [-Wopen-mp-usage]
   !$omp master taskloop
   do i=1,10
    j = j + 1
@@ -40,7 +40,7 @@ subroutine test_master_taskloop
 
 subroutine test_parallel_master_taskloop_simd
   integer :: i, j = 1
-!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP SIMD has been deprecated, please use PARALLEL_MASKED TASKLOOP SIMD instead.
+!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP SIMD has been deprecated, please use PARALLEL_MASKED TASKLOOP SIMD instead. [-Wopen-mp-usage]
   !$omp parallel master taskloop simd 
   do i=1,10
    j = j + 1
@@ -50,7 +50,7 @@ subroutine test_parallel_master_taskloop_simd
 
 subroutine test_parallel_master_taskloop
   integer :: i, j = 1
-!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP has been deprecated, please use PARALLEL MASKED TASKLOOP instead.
+!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP has been deprecated, please use PARALLEL MASKED TASKLOOP instead. [-Wopen-mp-usage]
   !$omp parallel master taskloop
   do i=1,10
    j = j + 1
diff --git a/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90 b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 0000000000000..f718efc32aabf
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,100 @@
+! Testing the Semantics of nested Loop Transformation Constructs
+
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
+
+subroutine loop_transformation_construct1
+  implicit none
+
+  !$omp do
+  !ERROR: A DO loop must follow the UNROLL directive
+  !$omp unroll
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !$omp end tile
+  !$omp end do
+  !ERROR: The END TILE directive must follow the DO loop associated with the loop construct
+  !$omp end tile
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs
+  !$omp parallel do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct3
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !ERROR: A DO loop must follow the TILE directive
+  !$omp tile
+end subroutine
+
+subroutine loop_transformation_construct4
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll full
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct5
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct6
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  !$omp unroll partial(2)
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/nested-target.f90 b/flang/test/Semantics/OpenMP/nested-target.f90
index 6a56a84f4f570..a82b6c01b1968 100644
--- a/flang/test/Semantics/OpenMP/nested-target.f90
+++ b/flang/test/Semantics/OpenMP/nested-target.f90
@@ -10,7 +10,7 @@ program main
   real, allocatable :: B(:)
 
   !$omp target
-  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target update from(arrayA) to(arrayB)
   do i = 1, 512
     arrayA(i) = arrayB(i)
@@ -20,7 +20,7 @@ program main
   !$omp parallel
   !$omp target
   !$omp parallel
-  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target update from(arrayA) to(arrayB)
   do i = 1, 512
     arrayA(i) = arrayB(i)
@@ -30,7 +30,7 @@ program main
   !$omp end parallel
 
   !$omp target
-  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target data map(to: a)
   do i = 1, N
     a = 3.14
@@ -40,12 +40,12 @@ program main
 
   allocate(B(N))
   !$omp target
-  !PORTABILITY: If TARGET ENTER DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET ENTER DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target enter data map(alloc:B)
   !$omp end target
 
   !$omp target
-  !PORTABILITY: If TARGET EXIT DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET EXIT DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target exit data map(delete:B)
   !$omp end target
   deallocate(B)
@@ -53,7 +53,7 @@ program main
   n1 = 10
   n2 = 10
   !$omp target teams map(to:a)
-  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !ERROR: Only `DISTRIBUTE`, `PARALLEL`, or `LOOP` regions are allowed to be strictly nested inside `TEAMS` region.
   !$omp target data map(n1,n2)
   do i=1, n1
@@ -65,7 +65,7 @@ program main
   !$omp end target teams
 
   !$omp target teams map(to:a) map(from:n1,n2)
-  !PORTABILITY: If TARGET TEAMS DISTRIBUTE PARALLEL DO directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET TEAMS DISTRIBUTE PARALLEL DO directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !ERROR: Only `DISTRIBUTE`, `PARALLEL`, or `LOOP` regions are allowed to be strictly nested inside `TEAMS` region.
   !$omp target teams distribute parallel do
   do i=1, n1
diff --git a/flang/test/Semantics/OpenMP/requires04.f90 b/flang/test/Semantics/OpenMP/requires04.f90
index 1fbb3aa6219bc..6eb0fb7f046a2 100644
--- a/flang/test/Semantics/OpenMP/requires04.f90
+++ b/flang/test/Semantics/OpenMP/requires04.f90
@@ -6,7 +6,7 @@
 
 subroutine f
   integer, save :: x
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to(x) device_type(nohost)
   !$omp declare target enter(x) device_type(nohost)
 end subroutine f
diff --git a/flang/test/Semantics/OpenMP/requires05.f90 b/flang/test/Semantics/OpenMP/requires05.f90
index f410f0104d899..0e2352ead0597 100644
--- a/flang/test/Semantics/OpenMP/requires05.f90
+++ b/flang/test/Semantics/OpenMP/requires05.f90
@@ -5,7 +5,7 @@
 ! device constructs, such as declare target with 'to' clause and no device_type.
 
 subroutine f
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to(f)
   !$omp declare target enter(f)
 end subroutine f
diff --git a/flang/test/Semantics/OpenMP/single03.f90 b/flang/test/Semantics/OpenMP/single03.f90
index dc2c2fd27eb04..789fcfdd88080 100644
--- a/flang/test/Semantics/OpenMP/single03.f90
+++ b/flang/test/Semantics/OpenMP/single03.f90
@@ -39,7 +39,7 @@ subroutine omp_single
         !$omp single private(j) copyprivate(j)
             print *, "omp single", j
         !ERROR: COPYPRIVATE variable 'j' may not appear on a PRIVATE or FIRSTPRIVATE clause on a SINGLE construct
-        !WARNING: The COPYPRIVATE clause with 'j' is already used on the SINGLE directive
+        !WARNING: The COPYPRIVATE clause with 'j' is already used on the SINGLE directive [-Wopen-mp-usage]
         !$omp end single copyprivate(j)
 
         !$omp single nowait
diff --git a/flang/test/Semantics/OpenMP/single04.f90 b/flang/test/Semantics/OpenMP/single04.f90
index 9505745c600e9..eabfe54b8ee3f 100644
--- a/flang/test/Semantics/OpenMP/single04.f90
+++ b/flang/test/Semantics/OpenMP/single04.f90
@@ -52,20 +52,20 @@ program single
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp single copyprivate(x) nowait
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
     !$omp end single copyprivate(x) nowait
 
     !$omp single copyprivate(x)
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp end single copyprivate(x) nowait
 
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp single copyprivate(x, y) nowait
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: 'z' appears in more than one COPYPRIVATE clause on the END SINGLE directive
     !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
     !$omp end single copyprivate(x, z) copyprivate(z) nowait
@@ -73,9 +73,9 @@ program single
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp single copyprivate(x) nowait copyprivate(y) copyprivate(z)
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
-    !WARNING: The COPYPRIVATE clause with 'y' is already used on the SINGLE directive
-    !WARNING: The COPYPRIVATE clause with 'z' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
+    !WARNING: The COPYPRIVATE clause with 'y' is already used on the SINGLE directive [-Wopen-mp-usage]
+    !WARNING: The COPYPRIVATE clause with 'z' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
     !$omp end single copyprivate(x, y, z) nowait
 end program
diff --git a/flang/test/Semantics/OpenMP/target01.f90 b/flang/test/Semantics/OpenMP/target01.f90
index 545cc8a7b69f8..f6e7c6a67d41f 100644
--- a/flang/test/Semantics/OpenMP/target01.f90
+++ b/flang/test/Semantics/OpenMP/target01.f90
@@ -39,19 +39,19 @@ subroutine bar(b1, b2, b3)
   type(c_ptr), pointer :: b2
   type(c_ptr), value :: b3
 
-  !WARNING: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(c)
     y = y + 1
   !$omp end target
-  !WARNING: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(b1)
     y = y + 1
   !$omp end target
-  !WARNING: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(b2)
     y = y + 1
   !$omp end target
-  !WARNING: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(b3)
     y = y + 1
   !$omp end target
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index b466a8e05e9c2..81e26ee327a9d 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -13,7 +13,7 @@ program main
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(main)
 
diff --git a/flang/test/Semantics/OpenMP/use_device_ptr1.f90 b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
index 5a5437e618450..ed0a1d238c96d 100644
--- a/flang/test/Semantics/OpenMP/use_device_ptr1.f90
+++ b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
@@ -27,7 +27,7 @@ subroutine omp_target_data
       a = arrayB
    !$omp end target data
 
-   !WARNING: Use of non-C_PTR type 'a' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead
+   !WARNING: Use of non-C_PTR type 'a' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead [-Wopen-mp-usage]
    !$omp target data map(tofrom: a) use_device_ptr(a)
       a = 2
    !$omp end target data
diff --git a/flang/test/Semantics/allocate09.f90 b/flang/test/Semantics/allocate09.f90
index c6b3b58773b02..15ede202d55fd 100644
--- a/flang/test/Semantics/allocate09.f90
+++ b/flang/test/Semantics/allocate09.f90
@@ -85,9 +85,9 @@ subroutine C946(param_ca_4_assumed, param_ta_4_assumed, param_ca_4_deferred)
 
   allocate(deferredChar, source="abcd")
   allocate(deferredChar, mold=deferredChar)
-  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD
+  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD [-Wallocate-to-other-length]
   allocate(char2, source="a")
-  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD
+  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD [-Wallocate-to-other-length]
   allocate(char2, source="abc")
   allocate(char2, mold=deferredChar)
 
diff --git a/flang/test/Semantics/argshape01.f90 b/flang/test/Semantics/argshape01.f90
index 19cca1ca4620a..a1766c216bdbd 100644
--- a/flang/test/Semantics/argshape01.f90
+++ b/flang/test/Semantics/argshape01.f90
@@ -113,7 +113,7 @@ program main
   !ERROR: Actual procedure argument has interface incompatible with dummy argument 's=': incompatible dummy argument #1: incompatible dummy data object shapes
   call s8c(s7)
   call s8c(s8)
-  !WARNING: Actual procedure argument has possible interface incompatibility with dummy argument 's=': possibly incompatible dummy argument #1: distinct dummy data object shapes
+  !WARNING: Actual procedure argument has possible interface incompatibility with dummy argument 's=': possibly incompatible dummy argument #1: distinct dummy data object shapes [-Wproc-dummy-arg-shapes]
   call s8c(s8b)
   call s9c(s9)
   call s9c(s9b)
@@ -154,7 +154,7 @@ program main
   !ERROR: Procedure pointer 'ps7' associated with incompatible procedure designator 's8': incompatible dummy argument #1: incompatible dummy data object shapes
   ps7 => s8
   ps8 => s8
-  !WARNING: pointer 'ps8' and s8b may not be completely compatible procedures: possibly incompatible dummy argument #1: distinct dummy data object shapes
+  !WARNING: pointer 'ps8' and s8b may not be completely compatible procedures: possibly incompatible dummy argument #1: distinct dummy data object shapes [-Wproc-dummy-arg-shapes]
   ps8 => s8b
   !ERROR: Procedure pointer 'ps8' associated with incompatible procedure designator 's6': incompatible dummy argument #1: incompatible dummy data object shapes
   ps8 => s6
diff --git a/flang/test/Semantics/assign02.f90 b/flang/test/Semantics/assign02.f90
index 9fa672025bfe7..f998197aeab58 100644
--- a/flang/test/Semantics/assign02.f90
+++ b/flang/test/Semantics/assign02.f90
@@ -85,7 +85,7 @@ subroutine s5
     real, pointer, volatile :: q
     p => x
     !ERROR: Pointer must be VOLATILE when target is a VOLATILE coarray
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p => y
     !ERROR: Pointer may not be VOLATILE when target is a non-VOLATILE coarray
     q => x
@@ -175,7 +175,7 @@ subroutine s12
     real, volatile, target :: x
     real, pointer :: p
     real, pointer, volatile :: q
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p => x
     q => x
   end
@@ -188,11 +188,11 @@ subroutine s13
     real, pointer, volatile :: q1
     type(t2), pointer, volatile :: q2
     type(t3), pointer, volatile :: q3
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p1 => y%t3Field%t2Field
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p2 => y%t3Field
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p3 => y
     !OK:
     q1 => y%t3Field%t2Field
@@ -200,9 +200,9 @@ subroutine s13
     q2 => y%t3Field
     !OK:
     q3 => y
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p3%t3FieldPtr => y%t3Field
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p3%t3FieldPtr%t2FieldPtr => y%t3Field%t2Field
     !OK
     q3%t3FieldPtr => y%t3Field
diff --git a/flang/test/Semantics/assign09.f90 b/flang/test/Semantics/assign09.f90
index b29e67a473e13..7c95e3a64ea57 100644
--- a/flang/test/Semantics/assign09.f90
+++ b/flang/test/Semantics/assign09.f90
@@ -49,9 +49,9 @@ elemental real function userElemental(a)
   realToRealProcPtr => noInterfaceExternal ! ok
   intToRealProcPtr => noInterfaceExternal !ok
   call sub1(noInterfaceExternal) ! ok
-  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface
+  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface [-Wimplicit-interface-actual]
   call sub2(noInterfaceExternal)
-  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface
+  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface [-Wimplicit-interface-actual]
   call sub3(noInterfaceExternal)
 
   !ERROR: Procedure pointer 'nointerfaceprocptr' with implicit interface may not be associated with procedure designator 'userelemental' with explicit interface that cannot be called via an implicit interface
diff --git a/flang/test/Semantics/associate01.f90 b/flang/test/Semantics/associate01.f90
index deafea695e84f..b51379bab3c30 100644
--- a/flang/test/Semantics/associate01.f90
+++ b/flang/test/Semantics/associate01.f90
@@ -13,7 +13,7 @@ module m1
   function iptr(n)
     integer, intent(in), target :: n
     integer, pointer :: iptr
-    !WARNING: Pointer target is not a definable variable
+    !WARNING: Pointer target is not a definable variable [-Wpointer-to-undefinable]
     !BECAUSE: 'n' is an INTENT(IN) dummy argument
     iptr => n
   end function
diff --git a/flang/test/Semantics/associated.f90 b/flang/test/Semantics/associated.f90
index 3f3f5488ad9fe..7cb6c240db226 100644
--- a/flang/test/Semantics/associated.f90
+++ b/flang/test/Semantics/associated.f90
@@ -123,15 +123,15 @@ subroutine test(assumedRank)
     lVar = associated(null(intAllocVar))
     lVar = associated(null()) !OK
     lVar = associated(null(intPointerVar1)) !OK
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
     !BECAUSE: 'NULL()' is a null pointer
     lVar = associated(null(), null()) !OK
     lVar = associated(intPointerVar1, null(intPointerVar2)) !OK
     lVar = associated(intPointerVar1, null()) !OK
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
     !BECAUSE: 'NULL()' is a null pointer
     lVar = associated(null(), null(intPointerVar1)) !OK
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(null(intPointerVar1), null()) !OK
     !ERROR: POINTER= argument of ASSOCIATED() must be a pointer
     lVar = associated(intVar)
@@ -180,18 +180,18 @@ subroutine test(assumedRank)
 
     ! Functions (other than NULL) returning pointers
     lVar = associated(objPtrFunc(targetIntVar1)) ! ok
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), targetIntVar1) ! ok
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), objPtrFunc(targetIntVar1)) ! ok
     lVar = associated(procPtrFunc()) ! ok
     lVar = associated(procPtrFunc(), intFunc) ! ok
     lVar = associated(procPtrFunc(), procPtrFunc()) ! ok
     !ERROR: POINTER= argument 'objptrfunc(targetintvar1)' is an object pointer but the TARGET= argument 'intfunc' is not a variable
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), intFunc)
     !ERROR: POINTER= argument 'objptrfunc(targetintvar1)' is an object pointer but the TARGET= argument 'procptrfunc()' is not a variable
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), procPtrFunc())
     !ERROR: POINTER= argument 'procptrfunc()' is a procedure pointer but the TARGET= argument 'objptrfunc(targetintvar1)' is not a procedure or procedure pointer
     lVar = associated(procPtrFunc(), objPtrFunc(targetIntVar1))
diff --git a/flang/test/Semantics/bind-c02.f90 b/flang/test/Semantics/bind-c02.f90
index 416d071542fe6..66e1f0b816b99 100644
--- a/flang/test/Semantics/bind-c02.f90
+++ b/flang/test/Semantics/bind-c02.f90
@@ -19,8 +19,8 @@ subroutine proc()
   !ERROR: Only variable and named common block can be in BIND statement
   bind(c) :: sub
 
-  !PORTABILITY: Global name 'm' conflicts with a module
-  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module
+  !PORTABILITY: Global name 'm' conflicts with a module [-Wbenign-name-clash]
+  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module [-Wbenign-name-clash]
   bind(c) :: m ! no error for implicit type variable
 
   type my_type
diff --git a/flang/test/Semantics/bind-c04.f90 b/flang/test/Semantics/bind-c04.f90
index 27119e375ce05..86a925f0090ee 100644
--- a/flang/test/Semantics/bind-c04.f90
+++ b/flang/test/Semantics/bind-c04.f90
@@ -37,7 +37,7 @@ subroutine aproc2() bind(c) ! ok
 
   procedure(proc), bind(c) :: y
 
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
   !ERROR: A procedure pointer may not have a BIND attribute with a name
   procedure(proc), bind(c, name="pc8"), bind(c), pointer :: pc8
 
diff --git a/flang/test/Semantics/bind-c06.f90 b/flang/test/Semantics/bind-c06.f90
index ff78a4743deee..608d193ee04de 100644
--- a/flang/test/Semantics/bind-c06.f90
+++ b/flang/test/Semantics/bind-c06.f90
@@ -40,7 +40,7 @@ program main
     procedure, nopass :: b => s
   end type
 
-  ! WARNING: A derived type with the BIND attribute should not be empty
+  ! WARNING: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, bind(c) :: t5
   end type
 
@@ -71,7 +71,7 @@ program main
   end type
 
   type, bind(c) :: t10
-    !WARNING: A CHARACTER component of an interoperable type should have length 1
+    !WARNING: A CHARACTER component of an interoperable type should have length 1 [-Wbind-c-char-length]
     character(len=2) x
   end type
   type, bind(c) :: t11
@@ -79,7 +79,7 @@ program main
     character(kind=2) x
   end type
   type, bind(c) :: t12
-    !PORTABILITY: A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL
+    !PORTABILITY: A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL [-Wlogical-vs-c-bool]
     logical(kind=8) x
   end type
   type, bind(c) :: t13
diff --git a/flang/test/Semantics/bind-c11.f90 b/flang/test/Semantics/bind-c11.f90
index 54021f68b3d8d..761a113fe7f76 100644
--- a/flang/test/Semantics/bind-c11.f90
+++ b/flang/test/Semantics/bind-c11.f90
@@ -8,7 +8,7 @@ module m
   real, allocatable, bind(c) :: x3(:)
  contains
   subroutine s1(x) bind(c)
-    !PORTABILITY: A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL
+    !PORTABILITY: A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL [-Wlogical-vs-c-bool]
     logical(2), intent(in), value :: x
   end
   subroutine s2(x) bind(c)
diff --git a/flang/test/Semantics/bind-c13.f90 b/flang/test/Semantics/bind-c13.f90
index 81815c1a95efa..867d63fc0e30d 100644
--- a/flang/test/Semantics/bind-c13.f90
+++ b/flang/test/Semantics/bind-c13.f90
@@ -5,8 +5,8 @@ subroutine interop(ptr,ashape,arank,eshape,asize) bind(c)
   real, pointer, contiguous :: ptr(:)
   real, contiguous :: ashape(:) ! ok
   real, contiguous :: arank(..) ! ok
-  !PORTABILITY: CONTIGUOUS entity 'eshape' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'eshape' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous :: eshape(10)
-  !PORTABILITY: CONTIGUOUS entity 'asize' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'asize' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous :: asize(*)
 end
diff --git a/flang/test/Semantics/bind-c17.f90 b/flang/test/Semantics/bind-c17.f90
index 8e0ecde67a0a5..34d21865d4e4c 100644
--- a/flang/test/Semantics/bind-c17.f90
+++ b/flang/test/Semantics/bind-c17.f90
@@ -4,7 +4,7 @@ module m
   end type
  contains
   subroutine sub(x) bind(c)
-    !PORTABILITY: The derived type of this interoperable object should be BIND(C)
+    !PORTABILITY: The derived type of this interoperable object should be BIND(C) [-Wportability]
     type(a), pointer, intent(in) :: x
   end
 end
diff --git a/flang/test/Semantics/bindings01.f90 b/flang/test/Semantics/bindings01.f90
index dc44db09c4a6f..3283ac30eb075 100644
--- a/flang/test/Semantics/bindings01.f90
+++ b/flang/test/Semantics/bindings01.f90
@@ -4,7 +4,7 @@
 
 module m
   !ERROR: An ABSTRACT derived type must be extensible
-  !PORTABILITY: A derived type with the BIND attribute should not be empty
+  !PORTABILITY: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, abstract, bind(c) :: badAbstract1
   end type
   !ERROR: An ABSTRACT derived type must be extensible
@@ -45,7 +45,7 @@ module m
   end type
   type, extends(intermediate) :: concrete2  ! ensure no false missing binding error
   end type
-  !WARNING: A derived type with the BIND attribute should not be empty
+  !WARNING: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, bind(c) :: inextensible1
   end type
   !ERROR: The parent type is not extensible
@@ -226,7 +226,7 @@ subroutine mysubr
   end subroutine
   subroutine test
     type(t) a(2)
-    !PORTABILITY: Base of NOPASS type-bound procedure reference should be scalar
+    !PORTABILITY: Base of NOPASS type-bound procedure reference should be scalar [-Wnopass-scalar-base]
     call a%tbp
     !ERROR: Base of procedure component reference must be scalar
     call a%pp
diff --git a/flang/test/Semantics/bindings03.f90 b/flang/test/Semantics/bindings03.f90
index baa8432a2701e..b03caf0ac452f 100644
--- a/flang/test/Semantics/bindings03.f90
+++ b/flang/test/Semantics/bindings03.f90
@@ -15,9 +15,9 @@ program test
   use m
   procedure(sub), pointer :: p
   type(t) x
-  !PORTABILITY: Procedure binding 'sub' used as target of a pointer assignment
+  !PORTABILITY: Procedure binding 'sub' used as target of a pointer assignment [-Wbinding-as-procedure]
   p => x%sub
-  !PORTABILITY: Procedure binding 'sub' passed as an actual argument
+  !PORTABILITY: Procedure binding 'sub' passed as an actual argument [-Wbinding-as-procedure]
   call sub2(x%sub)
  contains
   subroutine sub2(s)
diff --git a/flang/test/Semantics/block-data01.f90 b/flang/test/Semantics/block-data01.f90
index aa4ede787c17a..c75c63afcf8c2 100644
--- a/flang/test/Semantics/block-data01.f90
+++ b/flang/test/Semantics/block-data01.f90
@@ -7,7 +7,7 @@ block data foo
   !ERROR: An initialized variable in BLOCK DATA must be in a COMMON block
   integer :: notInCommon = 1
   integer :: uninitialized ! ok
-  !PORTABILITY: Procedure pointer 'q' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'q' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   !ERROR: 'q' may not appear in a BLOCK DATA subprogram
   procedure(sin), pointer :: q => cos
   !ERROR: 'p' may not be a procedure as it is in a COMMON block
diff --git a/flang/test/Semantics/boz-literal-constants.f90 b/flang/test/Semantics/boz-literal-constants.f90
index ee6919cb9ecd5..4d957d13f3f67 100644
--- a/flang/test/Semantics/boz-literal-constants.f90
+++ b/flang/test/Semantics/boz-literal-constants.f90
@@ -56,7 +56,7 @@ subroutine explicit(n, x, c)
   res = CMPLX (realpart, img, 4)
   res = CMPLX (B"0101", B"1111", 4)
 
-  !WARNING: underflow on REAL(8) to REAL(4) conversion
+  !WARNING: underflow on REAL(8) to REAL(4) conversion [-Wfolding-exception]
   dbl = DBLE(B"1111")
   dbl = DBLE(realpart)
 
diff --git a/flang/test/Semantics/bug1046.f90 b/flang/test/Semantics/bug1046.f90
new file mode 100644
index 0000000000000..dc5c470621387
--- /dev/null
+++ b/flang/test/Semantics/bug1046.f90
@@ -0,0 +1,15 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic -Werror
+!WARNING: INTEGER(4) 0**0 is not defined [-Wfolding-exception]
+print *, 0**0
+!WARNING: REAL/COMPLEX 0**0 is not defined [-Wfolding-exception]
+print *, 0**0.
+!WARNING: invalid argument on power with INTEGER exponent [-Wfolding-exception]
+print *, 0.0**0
+!WARNING: REAL/COMPLEX 0**0 is not defined [-Wfolding-exception]
+print *, 0.0**0.
+!WARNING: invalid argument on power with INTEGER exponent [-Wfolding-exception]
+print *, (0.0, 0.0)**0
+!WARNING: REAL/COMPLEX 0**0 is not defined [-Wfolding-exception]
+print *, (0.0, 0.0)**0.
+print *, (0.0, 0.0)**2.5
+end
diff --git a/flang/test/Semantics/bug1056.f90 b/flang/test/Semantics/bug1056.f90
new file mode 100644
index 0000000000000..b32270dab8f71
--- /dev/null
+++ b/flang/test/Semantics/bug1056.f90
@@ -0,0 +1,13 @@
+! RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
+program bug
+  integer, target :: ita(2) = [1,2], itb(2) = [3,4], itc(2) = [5,6]
+  type t1
+    integer, pointer :: p1(:) => ita, p2(:) => itb
+  end type
+  type t2
+    !CHECK: TYPE(t1) :: comp = t1(p1=itc,p2=itb)
+    type(t1) :: comp = t1(itc)
+  end type
+  integer, pointer :: p3(:) => itd
+  integer, target :: itd(2) = [7,8]
+end
diff --git a/flang/test/Semantics/bug1092.F90 b/flang/test/Semantics/bug1092.F90
new file mode 100644
index 0000000000000..ef108733fd78b
--- /dev/null
+++ b/flang/test/Semantics/bug1092.F90
@@ -0,0 +1,26 @@
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -DWHICH=1 -fsyntax-only -J%t %s
+!RUN: %flang_fc1 -DWHICH=2 -fsyntax-only -fhermetic-module-files -I%t -J%t %s
+!RUN: %flang_fc1 -fsyntax-only -I%t %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+
+#if WHICH == 1
+module bug1092a
+  type t
+  end type
+ contains
+  subroutine subr(x)
+    type(t) x
+  end
+end
+#elif WHICH == 2
+module bug1092b
+  use bug1092a, only: subr
+end
+#else
+use bug1092a, only: t
+use bug1092b, only: subr
+type(t) x
+call subr(x)
+end
+#endif
diff --git a/flang/test/Semantics/bug144453.f90 b/flang/test/Semantics/bug144453.f90
new file mode 100644
index 0000000000000..85c0892bffb5c
--- /dev/null
+++ b/flang/test/Semantics/bug144453.f90
@@ -0,0 +1,10 @@
+!RUN: %python %S/test_errors.py %s %flang_fc1
+function func(immutable)
+  logical func
+  logical, intent(in) :: immutable
+  !No warning about an undefined function result should appear
+  INQUIRE(file="/usr/local/games/adventure", EXIST=func)
+  !ERROR: EXIST variable 'immutable' is not definable
+  !BECAUSE: 'immutable' is an INTENT(IN) dummy argument
+  INQUIRE(file="/usr/local/games/adventure", EXIST=immutable)
+end
diff --git a/flang/test/Semantics/c7108.f90 b/flang/test/Semantics/c7108.f90
index c23a0abe3ee03..0b7148f4d6455 100644
--- a/flang/test/Semantics/c7108.f90
+++ b/flang/test/Semantics/c7108.f90
@@ -33,9 +33,9 @@ program p
   use m
   type(foo) x
   x = foo(); print *, x       ! ok, not ambiguous
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar0') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar0') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   x = foo(2); print *, x      ! ambigous
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar2') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar2') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   x = foo(3.); print *, x     ! ambiguous due to data conversion
   x = foo(.true.); print *, x ! ok, not ambigous
 end
diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index de5673f8dc1d9..e2d00536cdacb 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -46,12 +46,12 @@ program test
   call c_f_pointer(scalarC, multiDimIntF, shape=[1_8])
   !ERROR: SHAPE= argument to C_F_POINTER() must be a rank-one array.
   call c_f_pointer(scalarC, multiDimIntF, shape=rankTwoArray)
-  !WARNING: FPTR= argument to C_F_POINTER() should not be unlimited polymorphic
+  !WARNING: FPTR= argument to C_F_POINTER() should not be unlimited polymorphic [-Winteroperability]
   call c_f_pointer(scalarC, unlimited)
-  !PORTABILITY: FPTR= argument to C_F_POINTER() should not have a derived type that is not BIND(C)
+  !PORTABILITY: FPTR= argument to C_F_POINTER() should not have a derived type that is not BIND(C) [-Wportability]
   call c_f_pointer(scalarC, notBindC)
-  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable character length CHARACTER(KIND=1,LEN=2_8)
+  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable character length CHARACTER(KIND=1,LEN=2_8) [-Wcharacter-interoperability]
   call c_f_pointer(scalarC, c2ptr)
-  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8)
+  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8) [-Winteroperability]
   call c_f_pointer(scalarC, unicodePtr)
 end program
diff --git a/flang/test/Semantics/c_loc01.f90 b/flang/test/Semantics/c_loc01.f90
index a515a7a64f02a..da8a0e5bdd9e1 100644
--- a/flang/test/Semantics/c_loc01.f90
+++ b/flang/test/Semantics/c_loc01.f90
@@ -14,7 +14,7 @@ subroutine test(assumedType, poly, nclen, n)
     type(c_ptr) cp
     type(c_funptr) cfp
     real notATarget
-    !PORTABILITY: Procedure pointer 'pptr' should not have an ELEMENTAL intrinsic as its interface
+    !PORTABILITY: Procedure pointer 'pptr' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
     procedure(sin), pointer :: pptr
     real, target :: arr(3)
     type(hasLen(1)), target :: clen
@@ -41,9 +41,9 @@ subroutine test(assumedType, poly, nclen, n)
     cp = c_loc(nclen)
     !ERROR: C_LOC() argument may not be zero-length character
     cp = c_loc(ch(2:1))
-    !WARNING: C_LOC() argument has non-interoperable character length
+    !WARNING: C_LOC() argument has non-interoperable character length [-Wcharacter-interoperability]
     cp = c_loc(ch)
-    !WARNING: C_LOC() argument has non-interoperable intrinsic type or kind
+    !WARNING: C_LOC() argument has non-interoperable intrinsic type or kind [-Winteroperability]
     cp = c_loc(unicode)
     cp = c_loc(ch(1:1)) ! ok
     cp = c_loc(deferred) ! ok
diff --git a/flang/test/Semantics/call01.f90 b/flang/test/Semantics/call01.f90
index 67d14c3800671..6849c5f6500b1 100644
--- a/flang/test/Semantics/call01.f90
+++ b/flang/test/Semantics/call01.f90
@@ -119,16 +119,16 @@ end function nested
 end function
 
 subroutine s01(f1, f2, fp1, fp2, fp3)
-  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
   character*(*) :: f1, f3, fp1
   external :: f1, f3
   pointer :: fp1, fp3
-  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
   procedure(character*(*)), pointer :: fp2
   interface
     character*(*) function f2()
     end function
-    !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+    !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
     character*(*) function fp3()
     end function
     !ERROR: A function interface may not declare an assumed-length CHARACTER(*) result
diff --git a/flang/test/Semantics/call02.f90 b/flang/test/Semantics/call02.f90
index 0ec5530f98089..18cc0ecdfd852 100644
--- a/flang/test/Semantics/call02.f90
+++ b/flang/test/Semantics/call02.f90
@@ -8,7 +8,7 @@ elemental real function elem(x)
       real, intent(in), value :: x
     end function
     subroutine subr(dummy)
-      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
       procedure(sin) :: dummy
     end subroutine
     subroutine badsubr(dummy)
@@ -17,11 +17,11 @@ subroutine badsubr(dummy)
       procedure(elem) :: dummy
     end subroutine
     subroutine optionalsubr(dummy)
-      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
       procedure(sin), optional :: dummy
     end subroutine
     subroutine ptrsubr(dummy)
-      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
       procedure(sin), pointer, intent(in) :: dummy
     end subroutine
   end interface
diff --git a/flang/test/Semantics/call03.f90 b/flang/test/Semantics/call03.f90
index 59513557324e5..1721b59986862 100644
--- a/flang/test/Semantics/call03.f90
+++ b/flang/test/Semantics/call03.f90
@@ -163,9 +163,9 @@ subroutine test06 ! 15.5.2.4(4)
     type(pdtWithDefault(3)) :: defaultVar3
     type(pdtWithDefault(4)) :: defaultVar4
     character :: ch1
-    !ERROR: Actual argument variable length '1' is less than expected length '2'
+    !ERROR: Actual argument variable length '1' is less than expected length '2' [-Wshort-character-actual]
     call ch2(ch1)
-    !WARNING: Actual argument expression length '0' is less than expected length '2'
+    !WARNING: Actual argument expression length '0' is less than expected length '2' [-Wshort-character-actual]
     call ch2("")
     call pdtdefault(vardefault)
     !ERROR: Actual argument type 'pdt(n=3_4)' is not compatible with dummy argument type 'pdt'
@@ -300,10 +300,10 @@ subroutine test12 ! 15.5.2.4(21)
     !ERROR: Actual argument associated with INTENT(IN OUT) dummy argument 'x=' is not definable
     !BECAUSE: Variable 'a(int(j,kind=8))' has a vector subscript
     call intentinout_arr(a(j))
-    !WARNING: Actual argument associated with ASYNCHRONOUS dummy argument 'x=' is not definable
+    !WARNING: Actual argument associated with ASYNCHRONOUS dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: Variable 'a(int(j,kind=8))' has a vector subscript
     call asynchronous_arr(a(j))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'x=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: Variable 'a(int(j,kind=8))' has a vector subscript
     call volatile_arr(a(j))
   end subroutine
@@ -386,9 +386,9 @@ subroutine test16() ! C1540
     call contiguous(a) ! ok
     call pointer(a) ! ok
     call pointer(b) ! ok
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     call pointer(c) ! ok
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     call pointer(d) ! ok
     call valueassumedsize(a) ! ok
     call valueassumedsize(b) ! ok
diff --git a/flang/test/Semantics/call05.f90 b/flang/test/Semantics/call05.f90
index a06fe4f196c8c..b9b463a44979d 100644
--- a/flang/test/Semantics/call05.f90
+++ b/flang/test/Semantics/call05.f90
@@ -73,9 +73,9 @@ subroutine test
     call sma(ma) ! ok
     call spp(pp) ! ok
     call spa(pa) ! ok
-    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so
+    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so [-Wpolymorphic-actual-allocatable-or-pointer-to-monomorphic-dummy]
     call smp(pp)
-    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so
+    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so [-Wpolymorphic-actual-allocatable-or-pointer-to-monomorphic-dummy]
     call sma(pa)
     !ERROR: If a POINTER or ALLOCATABLE dummy or actual argument is polymorphic, both must be so
     call spp(mp)
diff --git a/flang/test/Semantics/call07.f90 b/flang/test/Semantics/call07.f90
index 92f2bdba882d5..7e29fb74dd615 100644
--- a/flang/test/Semantics/call07.f90
+++ b/flang/test/Semantics/call07.f90
@@ -19,17 +19,17 @@ subroutine s04(p)
   end subroutine
 
   subroutine test
-    !PORTABILITY: CONTIGUOUS entity 'a01' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'a01' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real, pointer, contiguous :: a01 ! C830
     real, pointer :: a02(:)
     real, target :: a03(10)
     real :: a04(10) ! not TARGET
-    !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real, contiguous :: scalar
     call s01(a03) ! ok
     !ERROR: CONTIGUOUS pointer dummy argument may not be associated with non-CONTIGUOUS pointer actual argument
     call s01(a02)
-    !WARNING: Target of CONTIGUOUS pointer association is not known to be contiguous
+    !WARNING: Target of CONTIGUOUS pointer association is not known to be contiguous [-Wpointer-to-possible-noncontiguous]
     call s01(a02(:))
     !ERROR: CONTIGUOUS pointer may not be associated with a discontiguous target
     call s01(a03(::2))
diff --git a/flang/test/Semantics/call09.f90 b/flang/test/Semantics/call09.f90
index 58b2382f600ef..e7ff72f74de02 100644
--- a/flang/test/Semantics/call09.f90
+++ b/flang/test/Semantics/call09.f90
@@ -37,7 +37,7 @@ subroutine s05(p)
   end subroutine
 
   subroutine selemental1(p)
-    !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+    !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
     procedure(cos) :: p ! ok
   end subroutine
 
diff --git a/flang/test/Semantics/call10.f90 b/flang/test/Semantics/call10.f90
index 47976ca9630cd..a9b760cc692e6 100644
--- a/flang/test/Semantics/call10.f90
+++ b/flang/test/Semantics/call10.f90
@@ -62,7 +62,7 @@ pure real function f03a(a)
     real, pointer :: a ! ok
   end function
   pure real function f04(a) ! C1583
-    !WARNING: non-POINTER dummy argument of pure function should be INTENT(IN) or VALUE
+    !WARNING: non-POINTER dummy argument of pure function should be INTENT(IN) or VALUE [-Wrelaxed-pure-dummy]
     real, intent(out) :: a
   end function
   pure real function f04a(a)
diff --git a/flang/test/Semantics/call14.f90 b/flang/test/Semantics/call14.f90
index fba11d35790b2..ad67caa41b4ed 100644
--- a/flang/test/Semantics/call14.f90
+++ b/flang/test/Semantics/call14.f90
@@ -9,7 +9,7 @@ module m
   !ERROR: VALUE attribute may apply only to a dummy data object
   subroutine C863(notData,assumedSize,coarray,coarrayComponent,assumedRank,assumedLen)
     external :: notData
-    !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+    !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
     real, value :: notADummy
     value :: notData
     !ERROR: VALUE attribute may not apply to an assumed-size array
@@ -20,7 +20,7 @@ subroutine C863(notData,assumedSize,coarray,coarrayComponent,assumedRank,assumed
     type(hasCoarray), value :: coarrayComponent
     !ERROR: VALUE attribute may not apply to an assumed-rank array
     real, value :: assumedRank(..)
-    !PORTABILITY: VALUE attribute on assumed-length CHARACTER may not be portable
+    !PORTABILITY: VALUE attribute on assumed-length CHARACTER may not be portable [-Wportability]
     character(*), value :: assumedLen
   end subroutine
   subroutine C864(allocatable, inout, out, pointer, volatile)
diff --git a/flang/test/Semantics/call24.f90 b/flang/test/Semantics/call24.f90
index c1053db93648f..bdcf80a74b84a 100644
--- a/flang/test/Semantics/call24.f90
+++ b/flang/test/Semantics/call24.f90
@@ -38,7 +38,7 @@ subroutine test()
 
   !ERROR: References to the procedure 'bar' require an explicit interface
   !BECAUSE: a dummy procedure is optional or a pointer
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Actual argument associated with procedure pointer dummy argument 'a_pointer=' is not a procedure pointer
   call bar(sin)
 
diff --git a/flang/test/Semantics/call27.f90 b/flang/test/Semantics/call27.f90
index f401eb254fde0..2559482cfe2b5 100644
--- a/flang/test/Semantics/call27.f90
+++ b/flang/test/Semantics/call27.f90
@@ -4,9 +4,9 @@ program test
   real, allocatable :: a
   !ERROR: NULL() actual argument 'NULL()' may not be associated with allocatable dummy argument dummy argument 'a=' that is INTENT(OUT) or INTENT(IN OUT)
   call foo0(null())
-  !WARNING: NULL() actual argument 'NULL()' should not be associated with allocatable dummy argument dummy argument 'a=' without INTENT(IN)
+  !WARNING: NULL() actual argument 'NULL()' should not be associated with allocatable dummy argument dummy argument 'a=' without INTENT(IN) [-Wnull-actual-for-default-intent-allocatable]
   call foo1(null())
-  !PORTABILITY: Allocatable dummy argument 'a=' is associated with NULL()
+  !PORTABILITY: Allocatable dummy argument 'a=' is associated with NULL() [-Wnull-actual-for-allocatable]
   call foo2(null())
   call foo3(null()) ! ok
   !ERROR: Actual argument associated with INTENT(IN OUT) dummy argument 'a=' is not definable
diff --git a/flang/test/Semantics/call30.f90 b/flang/test/Semantics/call30.f90
index 7aec5903606ec..1570eb02b6755 100644
--- a/flang/test/Semantics/call30.f90
+++ b/flang/test/Semantics/call30.f90
@@ -23,49 +23,49 @@ subroutine vol_dum_int_arr(my_int_arr)
   end subroutine vol_dum_int_arr
 
   subroutine test_all_subprograms()
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '6_4' is not a variable or pointer
     call vol_dum_int(6)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '18_4' is not a variable or pointer
     call vol_dum_int(6+12)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '72_4' is not a variable or pointer
     call vol_dum_int(6*12)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '-3_4' is not a variable or pointer
     call vol_dum_int(-6/2)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '3.1415927410125732421875_4' is not a variable or pointer
     call vol_dum_real(3.141592653)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '3.1415927410125732421875_4' is not a variable or pointer
     call vol_dum_real(3.141592653 + (-10.6e-11))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '3.3300884272335906644002534449100494384765625e-10_4' is not a variable or pointer
     call vol_dum_real(3.141592653 * 10.6e-11)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '-2.9637666816e10_4' is not a variable or pointer
     call vol_dum_real(3.141592653 / (-10.6e-11))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(1._4,3.2000000476837158203125_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(-1._4,6.340000152587890625_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2) + (-2., 3.14))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(-1.2048000335693359375e1_4,-3.2599999904632568359375_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2) * (-2., 3.14))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(5.80680549144744873046875e-1_4,-6.8833148479461669921875e-1_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2) / (-2., 3.14))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '[INTEGER(4)::1_4,2_4,3_4,4_4]' is not a variable or pointer
     call vol_dum_int_arr((/ 1, 2, 3, 4 /))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: 'reshape([INTEGER(4)::1_4,2_4,3_4,4_4],shape=[2,2])' is not a variable or pointer
     call vol_dum_int_arr(reshape((/ 1, 2, 3, 4 /), (/ 2, 2/)))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '[INTEGER(4)::1_4,2_4,3_4,4_4]' is not a variable or pointer
     call vol_dum_int_arr((/ 1, 2, 3, 4 /))
   end subroutine test_all_subprograms
diff --git a/flang/test/Semantics/call31.f90 b/flang/test/Semantics/call31.f90
index d9fd2cc60ec33..71fecd5363457 100644
--- a/flang/test/Semantics/call31.f90
+++ b/flang/test/Semantics/call31.f90
@@ -4,7 +4,7 @@
       module m
        contains
         subroutine subr(parg)
-          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
           procedure(character(*)), pointer :: parg
           !ERROR: An assumed (*) type parameter may be used only for a (non-statement function) dummy argument, associate name, character named constant, or external function result
           procedure(character(*)), pointer :: plocal
@@ -14,7 +14,7 @@ subroutine subr(parg)
         end subroutine
 
         subroutine subr_1(parg_1)
-          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
           procedure(character(*)), pointer :: parg_1
           print *, parg_1()
         end subroutine
diff --git a/flang/test/Semantics/call33.f90 b/flang/test/Semantics/call33.f90
index 285c4be98a9db..05b7dfb8d8c42 100644
--- a/flang/test/Semantics/call33.f90
+++ b/flang/test/Semantics/call33.f90
@@ -29,7 +29,7 @@ program test
   character(4) long, longarr(1)
   character(4), allocatable :: longalloc
   character(4), pointer :: longptr
-  !WARNING: Actual argument variable length '2' is less than expected length '3'
+  !WARNING: Actual argument variable length '2' is less than expected length '3' [-Wshort-character-actual]
   call s1(short)
   !ERROR: Actual argument array has fewer characters (2) than dummy argument 'x=' array (3)
   call s2(shortarr)
diff --git a/flang/test/Semantics/call34.f90 b/flang/test/Semantics/call34.f90
index 325a267309d46..a21536c6935fe 100644
--- a/flang/test/Semantics/call34.f90
+++ b/flang/test/Semantics/call34.f90
@@ -11,11 +11,11 @@ program test
   real, target :: a(1)
   real :: b(1)
   call foo(a) ! ok
-  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call must not be used afterwards, as 'b' is not a target
+  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call must not be used afterwards, as 'b' is not a target [-Wnon-target-passed-to-target]
   call foo(b)
-  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of '(a)' afterwards
+  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of '(a)' afterwards [-Wnon-target-passed-to-target]
   call foo((a))
-  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of 'a([INTEGER(8)::1_8])' afterwards
+  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of 'a([INTEGER(8)::1_8])' afterwards [-Wnon-target-passed-to-target]
   call foo(a([1]))
   !ERROR: Scalar actual argument may not be associated with assumed-shape dummy argument 'a='
   call foo(a(1))
diff --git a/flang/test/Semantics/call35.f90 b/flang/test/Semantics/call35.f90
index ff819481226d6..d500a3aef127d 100644
--- a/flang/test/Semantics/call35.f90
+++ b/flang/test/Semantics/call35.f90
@@ -5,14 +5,14 @@ subroutine s1
 end
 
 subroutine s2
-  !WARNING: Reference to the procedure 'ext' has an implicit interface that is distinct from another reference: distinct numbers of dummy arguments
+  !WARNING: Reference to the procedure 'ext' has an implicit interface that is distinct from another reference: distinct numbers of dummy arguments [-Wincompatible-implicit-interfaces]
   call ext(1.)
   call myerror('abcd') ! don't warn about distinct lengths
 end
 
 subroutine s3
   interface
-    !WARNING: The global subprogram 'ext' is not compatible with its local procedure declaration (incompatible procedure attributes: ImplicitInterface)
+    !WARNING: The global subprogram 'ext' is not compatible with its local procedure declaration (incompatible procedure attributes: ImplicitInterface) [-Wexternal-interface-mismatch]
     subroutine ext(n)
       integer n
     end
diff --git a/flang/test/Semantics/call36.f90 b/flang/test/Semantics/call36.f90
index 779aafdd2340a..387758408a841 100644
--- a/flang/test/Semantics/call36.f90
+++ b/flang/test/Semantics/call36.f90
@@ -15,7 +15,7 @@ subroutine test
     call intentInUnlimited(scalar)
     !ERROR: Actual argument associated with POINTER dummy argument 'x=' must also be POINTER unless INTENT(IN)
     call intentInOutUnlimited(scalar)
-    !PORTABILITY: If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both should be so
+    !PORTABILITY: If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both should be so [-Wrelaxed-intent-in-checking]
     call intentInUnlimited(arrayptr)
     !ERROR: If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both must be so
     call intentInOutUnlimited(arrayptr)
diff --git a/flang/test/Semantics/call37.f90 b/flang/test/Semantics/call37.f90
index 6018b8697ad5f..a536f62a1a7a8 100644
--- a/flang/test/Semantics/call37.f90
+++ b/flang/test/Semantics/call37.f90
@@ -28,15 +28,15 @@ subroutine exprLen(s)
 
 module m1
   interface
-    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: CHARACTER(KIND=1,LEN=1_8) vs CHARACTER(KIND=1,LEN=2_8))
+    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: CHARACTER(KIND=1,LEN=1_8) vs CHARACTER(KIND=1,LEN=2_8)) [-Wexternal-interface-mismatch]
     subroutine constLen(s)
       character(len=2) s
     end
-    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine assumedLen(s)
       character(len=2) s
     end
-    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments)
+    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments) [-Wexternal-interface-mismatch]
     subroutine exprLen(s)
       character(len=2) s
     end
@@ -45,11 +45,11 @@ subroutine exprLen(s)
 
 module m2
   interface
-    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine constLen(s)
       character(len=*) s
     end
-    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine exprLen(s)
       character(len=*) s
     end
@@ -58,12 +58,12 @@ subroutine exprLen(s)
 
 module m3
   interface
-    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments)
+    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments) [-Wexternal-interface-mismatch]
     subroutine constLen(s)
       common n
       character(len=n) s
     end
-    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine assumedLen(s)
       common n
       character(len=n) s
diff --git a/flang/test/Semantics/call38.f90 b/flang/test/Semantics/call38.f90
index b1a35973e35fe..4a00699779561 100644
--- a/flang/test/Semantics/call38.f90
+++ b/flang/test/Semantics/call38.f90
@@ -516,9 +516,9 @@ subroutine test
     !ERROR: Actual argument associated with coarray dummy argument 'a=' must be a coarray
     call coarray0(matrix11(1,1))
 
-    !WARNING: Actual argument variable length '1' is less than expected length '2'
+    !WARNING: Actual argument variable length '1' is less than expected length '2' [-Wshort-character-actual]
     call scalar(scalar0(1:1))
-    !WARNING: Actual argument expression length '1' is less than expected length '2'
+    !WARNING: Actual argument expression length '1' is less than expected length '2' [-Wshort-character-actual]
     call scalar('a')
   end
 end
diff --git a/flang/test/Semantics/call41.f90 b/flang/test/Semantics/call41.f90
index a4c7514d99ba5..c26ef2899576f 100644
--- a/flang/test/Semantics/call41.f90
+++ b/flang/test/Semantics/call41.f90
@@ -5,7 +5,7 @@ subroutine unlimited(x)
     class(*), intent(in) :: x
   end
   subroutine test
-    !PORTABILITY: passing Hollerith to unlimited polymorphic as if it were CHARACTER
+    !PORTABILITY: passing Hollerith to unlimited polymorphic as if it were CHARACTER [-Whollerith-polymorphic]
     call unlimited(6HHERMAN)
     call unlimited('abc') ! ok
   end
diff --git a/flang/test/Semantics/call42.f90 b/flang/test/Semantics/call42.f90
index 2d5303b58cb01..c264f8e842c5f 100644
--- a/flang/test/Semantics/call42.f90
+++ b/flang/test/Semantics/call42.f90
@@ -22,17 +22,17 @@ recursive subroutine typeOutAssumedRank(a,b,c,d)
     type(hasAlloc), intent(out) :: b(..)
     type(hasInit), intent(out) :: c(..)
     type(hasFinal), intent(out) :: d(..)
-    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call typeOutAssumedRank(a, b, c, d)
-    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call classOutAssumedRank(a, b, c, d)
-    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
@@ -95,17 +95,17 @@ subroutine typeAssumedSize(a,b,c,d)
     type(hasAlloc) b(*)
     type(hasInit) c(*)
     type(hasFinal) d(*)
-    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call typeOutAssumedRank(a,b,c,d)
-    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call classOutAssumedRank(a,b,c,d)
-    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
diff --git a/flang/test/Semantics/call43.f90 b/flang/test/Semantics/call43.f90
index d8cc543a4838a..938ecb9a2a7db 100644
--- a/flang/test/Semantics/call43.f90
+++ b/flang/test/Semantics/call43.f90
@@ -5,11 +5,11 @@ subroutine from(a, b, c, d)
   call to(a)
   call to(a(1)) ! ok
   call to(b) ! ok, passed via temp
-  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes
+  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes [-Wincompatible-implicit-interfaces]
   call to(b(1))
-  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes
+  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes [-Wincompatible-implicit-interfaces]
   call to(c)
-  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes
+  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes [-Wincompatible-implicit-interfaces]
   call to(1.)
   call to([1., 2.]) ! ok
   call to(d) ! ok
diff --git a/flang/test/Semantics/call44.f90 b/flang/test/Semantics/call44.f90
index f7c4c9093b432..6e52aa9de55f7 100644
--- a/flang/test/Semantics/call44.f90
+++ b/flang/test/Semantics/call44.f90
@@ -2,10 +2,10 @@
 subroutine assumedshape(normal, contig)
   real normal(:)
   real, contiguous :: contig(:)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Element of assumed-shape array may not be associated with a dummy argument 'assumedsize=' array
   call seqAssociate(normal(1))
-  !PORTABILITY: Element of contiguous assumed-shape array is accepted for storage sequence association
+  !PORTABILITY: Element of contiguous assumed-shape array is accepted for storage sequence association [-Wcontiguous-ok-for-seq-association]
   call seqAssociate(contig(1))
 end
 subroutine seqAssociate(assumedSize)
diff --git a/flang/test/Semantics/case01.f90 b/flang/test/Semantics/case01.f90
index 15bc30498ab51..c9631d299e49c 100644
--- a/flang/test/Semantics/case01.f90
+++ b/flang/test/Semantics/case01.f90
@@ -129,7 +129,7 @@ program selectCaseProg
   end select
 
   select case (grade2)
-     !WARNING: CASE has lower bound greater than upper bound
+     !WARNING: CASE has lower bound greater than upper bound [-Wempty-case]
      case (51:50)
      case (100:)
      case (:30)
@@ -183,13 +183,13 @@ subroutine test_overflow
   integer :: j
   select case(1_1)
   case (127)
-  !WARNING: CASE value (128_4) overflows type (INTEGER(1)) of SELECT CASE expression
+  !WARNING: CASE value (128_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
   case (128)
-  !WARNING: CASE value (129_4) overflows type (INTEGER(1)) of SELECT CASE expression
-  !WARNING: CASE value (130_4) overflows type (INTEGER(1)) of SELECT CASE expression
+  !WARNING: CASE value (129_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
+  !WARNING: CASE value (130_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
   case (129:130)
-  !WARNING: CASE value (-130_4) overflows type (INTEGER(1)) of SELECT CASE expression
-  !WARNING: CASE value (-129_4) overflows type (INTEGER(1)) of SELECT CASE expression
+  !WARNING: CASE value (-130_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
+  !WARNING: CASE value (-129_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
   case (-130:-129)
   case (-128)
   !ERROR: Must be a scalar value, but is a rank-1 array
diff --git a/flang/test/Semantics/common-blocks.f90 b/flang/test/Semantics/common-blocks.f90
index 65f17f6d3fe97..816a9039dd49f 100644
--- a/flang/test/Semantics/common-blocks.f90
+++ b/flang/test/Semantics/common-blocks.f90
@@ -7,7 +7,7 @@ subroutine init_1
   common x, y
   common /a/ xa, ya
   common /b/ xb, yb
-  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard
+  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard [-Wdata-stmt-extensions]
   data x /42./, xa /42./, yb/42./
 end subroutine
 
@@ -18,7 +18,7 @@ subroutine init_conflict
   common /a/ xa, ya
   common /b/ xb, yb
   equivalence (yb, yb_eq)
-  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard
+  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard [-Wdata-stmt-extensions]
   !ERROR: Multiple initialization of COMMON block /b/
   data x /66./, xa /66./, yb_eq /66./
 end subroutine
diff --git a/flang/test/Semantics/contiguous01.f90 b/flang/test/Semantics/contiguous01.f90
index 89382775261b8..eca5b4c69a601 100644
--- a/flang/test/Semantics/contiguous01.f90
+++ b/flang/test/Semantics/contiguous01.f90
@@ -5,30 +5,30 @@ module m0
 end
 module m
   use m0
-  !WARNING: Use-associated 'p1' already has 'CONTIGUOUS' attribute
+  !WARNING: Use-associated 'p1' already has 'CONTIGUOUS' attribute [-Wredundant-attribute]
   contiguous p1
   !ERROR: Cannot change CONTIGUOUS attribute on use-associated 'p2'
   contiguous p2
-  !PORTABILITY: CONTIGUOUS entity 'x' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'x' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous :: x
-  !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous, pointer :: scalar
-  !PORTABILITY: CONTIGUOUS entity 'allocatable' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'allocatable' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous, allocatable :: allocatable
  contains
-  !PORTABILITY: CONTIGUOUS entity 'func' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'func' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   function func(ashape,arank) result(r)
     real, contiguous :: ashape(:) ! ok
     real, contiguous :: arank(..) ! ok
-    !PORTABILITY: CONTIGUOUS entity 'r' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'r' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real :: r(10)
-    !PORTABILITY: CONTIGUOUS entity 'r2' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'r2' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real :: r2(10)
     contiguous func
     contiguous r
     contiguous e
     contiguous r2
-    !PORTABILITY: CONTIGUOUS entity 'e' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'e' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     entry e() result(r2)
     r2 = 0
   end
diff --git a/flang/test/Semantics/cuf01.cuf b/flang/test/Semantics/cuf01.cuf
index 574d345eae045..fc405fcd3e36a 100644
--- a/flang/test/Semantics/cuf01.cuf
+++ b/flang/test/Semantics/cuf01.cuf
@@ -2,15 +2,15 @@
 ! Test conflicting CUDA subprogram attributes
 module m1
  contains
-  !WARNING: ATTRIBUTES(Host) appears more than once
+  !WARNING: ATTRIBUTES(Host) appears more than once [-Wredundant-attribute]
   attributes(host,host) subroutine ok1; end
-  !WARNING: ATTRIBUTES(Host) appears more than once
+  !WARNING: ATTRIBUTES(Host) appears more than once [-Wredundant-attribute]
   attributes(host) attributes(host) subroutine ok2; end
   attributes(host,device) subroutine ok3; end
   attributes(device,host) subroutine ok4; end
-  !WARNING: ATTRIBUTES(Host) appears more than once
+  !WARNING: ATTRIBUTES(Host) appears more than once [-Wredundant-attribute]
   attributes(host,device,host) subroutine ok5; end
-  !WARNING: ATTRIBUTES(Device) appears more than once
+  !WARNING: ATTRIBUTES(Device) appears more than once [-Wredundant-attribute]
   attributes(device,host,device) subroutine ok6; end
   !ERROR: ATTRIBUTES(Global) conflicts with earlier ATTRIBUTES(Host)
   attributes(host,global) subroutine conflict1; end
diff --git a/flang/test/Semantics/cuf03.cuf b/flang/test/Semantics/cuf03.cuf
index fe9dd5b3ecf05..d2a189f419299 100644
--- a/flang/test/Semantics/cuf03.cuf
+++ b/flang/test/Semantics/cuf03.cuf
@@ -38,14 +38,14 @@ module m
   !ERROR: Object 'mmp' with ATTRIBUTES(MANAGED) must also be allocatable, automatic, explicit shape, or a dummy argument
   real, managed, pointer :: mmp(:)
   real, managed, target :: mmt
-  !WARNING: Object 'mp' with ATTRIBUTES(PINNED) should also be allocatable
+  !WARNING: Object 'mp' with ATTRIBUTES(PINNED) should also be allocatable [-Wcuda-usage]
   real, pinned :: mp
-  !WARNING: Object 'mpi' with ATTRIBUTES(PINNED) should also be allocatable
+  !WARNING: Object 'mpi' with ATTRIBUTES(PINNED) should also be allocatable [-Wcuda-usage]
   real, pinned :: mpi = 1.
   real, pinned, allocatable :: mpl ! ok
-  !ERROR: Object 'mpp' with ATTRIBUTES(PINNED) may not be a pointer
+  !ERROR: Object 'mpp' with ATTRIBUTES(PINNED) may not be a pointer [-Wcuda-usage]
   real, pinned, pointer :: mpp
-  !WARNING: Object 'mpt' with ATTRIBUTES(PINNED) should also be allocatable
+  !WARNING: Object 'mpt' with ATTRIBUTES(PINNED) should also be allocatable [-Wcuda-usage]
   real, pinned, target :: mpt ! ok
   !ERROR: ATTRIBUTES(TEXTURE) is obsolete and no longer supported
   real, texture, pointer :: mt
@@ -67,7 +67,7 @@ module m
     integer, intent(in) :: n
     real, device :: da(*) ! ok
     real, managed :: ma(n) ! ok
-    !WARNING: Pointer 'dp' may not be associated in a device subprogram
+    !WARNING: Pointer 'dp' may not be associated in a device subprogram [-Wcuda-usage]
     real, device, pointer :: dp
     real, constant :: rc ! ok
     real, shared :: rs ! ok
diff --git a/flang/test/Semantics/cuf04.cuf b/flang/test/Semantics/cuf04.cuf
index 2e2faa90b490d..3c9ee7388ab2d 100644
--- a/flang/test/Semantics/cuf04.cuf
+++ b/flang/test/Semantics/cuf04.cuf
@@ -4,9 +4,9 @@ module m
  contains
   attributes(device) subroutine devsubr(n)
     integer, intent(in) :: n
-    !WARNING: 'x1' should not have the SAVE attribute or initialization in a device subprogram
+    !WARNING: 'x1' should not have the SAVE attribute or initialization in a device subprogram [-Wcuda-usage]
     real, save :: x1
-    !WARNING: 'x2' should not have the SAVE attribute or initialization in a device subprogram
+    !WARNING: 'x2' should not have the SAVE attribute or initialization in a device subprogram [-Wcuda-usage]
     real :: x2 = 1.
     !ERROR: Device subprogram 'devsubr' cannot call itself
     if (n > 0) call devsubr(n-1)
diff --git a/flang/test/Semantics/cuf09.cuf b/flang/test/Semantics/cuf09.cuf
index 1e23819f9afe8..9178b0a63adbe 100644
--- a/flang/test/Semantics/cuf09.cuf
+++ b/flang/test/Semantics/cuf09.cuf
@@ -15,7 +15,7 @@ module m
     print*,'from device'
     print '(f10.5)', (x(ivar), ivar = 1, 10)
     write(*,*), "Hello world from device!"
-    !WARNING: I/O statement might not be supported on device
+    !WARNING: I/O statement might not be supported on device [-Wcuda-usage]
     write(12,'(10F4.1)'), x
   end
   attributes(global) subroutine devsub3(n)
diff --git a/flang/test/Semantics/data06.f90 b/flang/test/Semantics/data06.f90
index 08b4700028438..7ca514ec923f6 100644
--- a/flang/test/Semantics/data06.f90
+++ b/flang/test/Semantics/data06.f90
@@ -41,7 +41,7 @@ real function rfunc(x)
   data rp/rfunc/
   procedure(rfunc), pointer :: rpp
   real, target :: rt
-  !WARNING: Procedure pointer 'rpp' in a DATA statement is not standard
+  !WARNING: Procedure pointer 'rpp' in a DATA statement is not standard [-Wdata-stmt-extensions]
   !ERROR: Data object 'rt' may not be used to initialize 'rpp', which is a procedure pointer
   data rpp/rt/
   !ERROR: Initializer for 'rt' must not be a pointer
@@ -49,7 +49,7 @@ real function rfunc(x)
   !ERROR: Initializer for 'rt' must not be a procedure
   data rt/rfunc/
   integer :: jx, jy
-  !WARNING: DATA statement value initializes 'jx' of type 'INTEGER(4)' with CHARACTER
+  !WARNING: DATA statement value initializes 'jx' of type 'INTEGER(4)' with CHARACTER [-Wdata-stmt-extensions]
   data jx/'abc'/
   !ERROR: DATA statement value could not be converted to the type 'INTEGER(4)' of the object 'jx'
   data jx/t1()/
diff --git a/flang/test/Semantics/data17.f90 b/flang/test/Semantics/data17.f90
index 86f1c4c6c1269..2f4e5fb02e6a9 100644
--- a/flang/test/Semantics/data17.f90
+++ b/flang/test/Semantics/data17.f90
@@ -1,8 +1,8 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 character(4) a, b, c, d, e, f
-!WARNING: DATA statement value '"abcde"' for 'a' has the wrong length
+!WARNING: DATA statement value '"abcde"' for 'a' has the wrong length [-Wdata-length]
 data a(1:4)/'abcde'/
-!WARNING: DATA statement value '"abc"' for 'b' has the wrong length
+!WARNING: DATA statement value '"abc"' for 'b' has the wrong length [-Wdata-length]
 data b(1:4)/'abc'/
 data c/'abcde'/ ! not a substring, conforms
 data d/'abc'/ ! not a substring, conforms
diff --git a/flang/test/Semantics/declarations04.f90 b/flang/test/Semantics/declarations04.f90
index f0ca568e7a97c..5585b08904dd6 100644
--- a/flang/test/Semantics/declarations04.f90
+++ b/flang/test/Semantics/declarations04.f90
@@ -20,7 +20,7 @@ subroutine bar() bind(c,name="ext3")
 end
 
 block data ext3
-  !PORTABILITY: Global name 'ext4' conflicts with a module
+  !PORTABILITY: Global name 'ext4' conflicts with a module [-Wbenign-name-clash]
   common /ext4/ x
 end
 
diff --git a/flang/test/Semantics/declarations05.f90 b/flang/test/Semantics/declarations05.f90
index b1e3d3c773160..06191e165bce7 100644
--- a/flang/test/Semantics/declarations05.f90
+++ b/flang/test/Semantics/declarations05.f90
@@ -27,7 +27,7 @@ pure subroutine test
     !ERROR: 'x1' may not be a local variable in a pure subprogram
     !BECAUSE: 'x1' has an impure FINAL procedure 'final'
     type(t1) x1
-    !WARNING: 'x1a' of derived type 't1' does not have a FINAL subroutine for its rank (1)
+    !WARNING: 'x1a' of derived type 't1' does not have a FINAL subroutine for its rank (1) [-Wfinal]
     type(t1), allocatable :: x1a(:)
     type(t1), parameter :: namedConst = t1() ! ok
     !ERROR: 'x2' may not be a local variable in a pure subprogram
diff --git a/flang/test/Semantics/declarations07.f90 b/flang/test/Semantics/declarations07.f90
index 8c95c163b043b..75e5bcf22bafb 100644
--- a/flang/test/Semantics/declarations07.f90
+++ b/flang/test/Semantics/declarations07.f90
@@ -11,8 +11,8 @@ pure integer function mykind(x)
   real a, b
   integer, parameter :: ak = kind(a)
   integer, parameter :: br = rank(b)
-  !WARNING: 'a' appeared earlier as a scalar actual argument to a specification function
+  !WARNING: 'a' appeared earlier as a scalar actual argument to a specification function [-Wprevious-scalar-use]
   dimension a(1)
-  !WARNING: 'b' appeared earlier as a scalar actual argument to a specification function
+  !WARNING: 'b' appeared earlier as a scalar actual argument to a specification function [-Wprevious-scalar-use]
   dimension b(1)
 end
diff --git a/flang/test/Semantics/deferred01.f90 b/flang/test/Semantics/deferred01.f90
index ce406a72b8fab..8117b91eef434 100644
--- a/flang/test/Semantics/deferred01.f90
+++ b/flang/test/Semantics/deferred01.f90
@@ -19,7 +19,7 @@ module m2
   use m1
   type, extends(absBase) :: ext
    contains
-    !WARNING: Override of PRIVATE DEFERRED 'deferredtbp' should appear in its module
+    !WARNING: Override of PRIVATE DEFERRED 'deferredtbp' should appear in its module [-Winaccessible-deferred-override]
     procedure :: deferredTbp => implTbp
   end type
  contains
diff --git a/flang/test/Semantics/definable02.f90 b/flang/test/Semantics/definable02.f90
index 666fee91a97de..559efb0dbb003 100644
--- a/flang/test/Semantics/definable02.f90
+++ b/flang/test/Semantics/definable02.f90
@@ -36,9 +36,9 @@ program test
   x1(:) = [t1()] ! ok
   x2(:) = [t2()] ! ok
   x3(:) = [t3()] ! ok
-  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f1'
+  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f1' [-Wvector-subscript-finalization]
   x1([1]) = [t1()]
-  !PORTABILITY: Variable 'x2([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f2'
+  !PORTABILITY: Variable 'x2([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f2' [-Wvector-subscript-finalization]
   x2([1]) = [t2()]
   x3([1]) = [t3()] ! ok
 end
diff --git a/flang/test/Semantics/dim01.f90 b/flang/test/Semantics/dim01.f90
index aba9a15b95a41..c00cd39ecd097 100644
--- a/flang/test/Semantics/dim01.f90
+++ b/flang/test/Semantics/dim01.f90
@@ -16,21 +16,21 @@ function f0b(a)
   function f1(a,d)
     real, intent(in) :: a(:)
     integer, optional, intent(in) :: d
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f1 = sum(a,dim=d)
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f1 = norm2(a,dim=d)
   end function
   function f2(a,d)
     real, intent(in) :: a(:)
     integer, pointer, intent(in) :: d
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f2 = sum(a,dim=d)
   end function
   function f3(a,d)
     real, intent(in) :: a(:)
     integer, allocatable, intent(in) :: d
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f3 = sum(a,dim=d)
   end function
   function f10a(a)
@@ -49,23 +49,23 @@ function f11(a,d)
     real, intent(in) :: a(:,:)
     integer, optional, intent(in) :: d
     real, allocatable :: f11(:)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f11 = sum(a,dim=d)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f11 = norm2(a,dim=d)
   end function
   function f12(a,d)
     real, intent(in) :: a(:,:)
     integer, pointer, intent(in) :: d
     real, allocatable :: f12(:)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f12 = sum(a,dim=d)
   end function
   function f13(a,d)
     real, intent(in) :: a(:,:)
     integer, allocatable, intent(in) :: d
     real, allocatable :: f13(:)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f13 = sum(a,dim=d)
   end function
 end module
diff --git a/flang/test/Semantics/dosemantics02.f90 b/flang/test/Semantics/dosemantics02.f90
index d6075a8a3f8f8..2e7713db28ef4 100644
--- a/flang/test/Semantics/dosemantics02.f90
+++ b/flang/test/Semantics/dosemantics02.f90
@@ -23,12 +23,12 @@ SUBROUTINE s1()
   INTEGER, PARAMETER :: constInt = 0
 
   ! Warn on this one for backwards compatibility
-  !WARNING: DO step expression should not be zero
+  !WARNING: DO step expression should not be zero [-Wzero-do-step]
   DO 10 I = 1, 10, 0
   10 CONTINUE
 
   ! Warn on this one for backwards compatibility
-  !WARNING: DO step expression should not be zero
+  !WARNING: DO step expression should not be zero [-Wzero-do-step]
   DO 20 I = 1, 10, 5 - 5
   20 CONTINUE
 
diff --git a/flang/test/Semantics/dosemantics03.f90 b/flang/test/Semantics/dosemantics03.f90
index a36c86b202c48..a2ec1135d7b16 100644
--- a/flang/test/Semantics/dosemantics03.f90
+++ b/flang/test/Semantics/dosemantics03.f90
@@ -50,13 +50,13 @@ END FUNCTION ifunc
   END DO
 
 ! REAL DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO rvar = 1, 10, 3
     PRINT *, "rvar is: ", rvar
   END DO
 
 ! DOUBLE PRECISISON DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO dvar = 1, 10, 3
     PRINT *, "dvar is: ", dvar
   END DO
@@ -69,14 +69,14 @@ END FUNCTION ifunc
 
 ! Pointer to REAL DO variable
   ALLOCATE(prvar)
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO prvar = 1, 10, 3
     PRINT *, "prvar is: ", prvar
   END DO
 
 ! Pointer to DOUBLE PRECISION DO variable
   ALLOCATE(pdvar)
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO pdvar = 1, 10, 3
     PRINT *, "pdvar is: ", pdvar
   END DO
@@ -148,26 +148,26 @@ END FUNCTION ifunc
   END DO
 
 ! Shared association REAL DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO realvarshare = 1, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Shared association DOUBLE PRECISION DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO dpvarshare = 1, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Initial expressions
 ! REAL initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = rvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! DOUBLE PRECISION initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = dvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -178,13 +178,13 @@ END FUNCTION ifunc
   END DO
 
 ! Pointer to REAL initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = prvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Pointer to DOUBLE PRECISION initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = pdvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -221,13 +221,13 @@ END FUNCTION ifunc
 
 ! Final expression
 ! REAL final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, rvar, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! DOUBLE PRECISION final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, dvar, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -238,13 +238,13 @@ END FUNCTION ifunc
   END DO
 
 ! Pointer to REAL final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, prvar, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Pointer to DOUBLE PRECISION final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = pdvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -263,13 +263,13 @@ END FUNCTION ifunc
 
 ! Step expression
 ! REAL step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, rvar
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! DOUBLE PRECISION step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, dvar
     PRINT *, "ivar is: ", ivar
   END DO
@@ -280,13 +280,13 @@ END FUNCTION ifunc
   END DO
 
 ! Pointer to REAL step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, prvar
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Pointer to DOUBLE PRECISION step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, pdvar
     PRINT *, "ivar is: ", ivar
   END DO
diff --git a/flang/test/Semantics/dosemantics12.f90 b/flang/test/Semantics/dosemantics12.f90
index 1757ade4b7c8f..3416142dd1b69 100644
--- a/flang/test/Semantics/dosemantics12.f90
+++ b/flang/test/Semantics/dosemantics12.f90
@@ -316,7 +316,7 @@ subroutine s9()
   ! Technically non-conformant (F'2023 19.4 p8)
   do concurrent (ivar = 1:10)
     print *, "hello"
-    !PORTABILITY: Index variable 'ivar' should not also be an index in an enclosing FORALL or DO CONCURRENT
+    !PORTABILITY: Index variable 'ivar' should not also be an index in an enclosing FORALL or DO CONCURRENT [-Wodd-index-variable-restrictions]
     do concurrent (ivar = 1:10)
       print *, "hello"
     end do
@@ -392,7 +392,7 @@ subroutine s12()
 
   call intentInOutSub(jvar, ivar)
   do ivar = 1,10
-    !WARNING: Possible redefinition of DO variable 'ivar'
+    !WARNING: Possible redefinition of DO variable 'ivar' [-Windex-var-redefinition]
     call intentInOutSub(jvar, ivar)
   end do
 
@@ -437,7 +437,7 @@ subroutine s13()
   end do
 
   do ivar = 1, 10
-    !WARNING: Possible redefinition of DO variable 'ivar'
+    !WARNING: Possible redefinition of DO variable 'ivar' [-Windex-var-redefinition]
     jvar = intentInOutFunc(ivar)
   end do
 
diff --git a/flang/test/Semantics/expr-errors05.f90 b/flang/test/Semantics/expr-errors05.f90
index 0328165a7921e..43133c836f3b4 100644
--- a/flang/test/Semantics/expr-errors05.f90
+++ b/flang/test/Semantics/expr-errors05.f90
@@ -1,16 +1,16 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
-!PORTABILITY: nonstandard usage: generalized COMPLEX constructor
-!PORTABILITY: Real part of complex constructor is not scalar
+!PORTABILITY: nonstandard usage: generalized COMPLEX constructor [-Wcomplex-constructor]
+!PORTABILITY: Real part of complex constructor is not scalar [-Wcomplex-constructor]
 complex, parameter :: z1(*) = ([1.,2.], 3.)
-!PORTABILITY: nonstandard usage: generalized COMPLEX constructor
-!PORTABILITY: Imaginary part of complex constructor is not scalar
+!PORTABILITY: nonstandard usage: generalized COMPLEX constructor [-Wcomplex-constructor]
+!PORTABILITY: Imaginary part of complex constructor is not scalar [-Wcomplex-constructor]
 complex, parameter :: z2(*) = (4., [5.,6.])
 real, parameter :: aa(*) = [7.,8.]
-!PORTABILITY: Real part of complex literal constant is not scalar
+!PORTABILITY: Real part of complex literal constant is not scalar [-Wcomplex-constructor]
 complex, parameter :: z3(*) = (aa, 9.)
-!PORTABILITY: Imaginary part of complex literal constant is not scalar
+!PORTABILITY: Imaginary part of complex literal constant is not scalar [-Wcomplex-constructor]
 complex, parameter :: z4(*) = (10., aa)
 !We need a nonzero exit status to make test_errors.py look at messages :-(
-!WARNING: division by zero
+!WARNING: division by zero [-Wfolding-exception]
 real, parameter :: xxx = 1./0.
 end
diff --git a/flang/test/Semantics/expr-errors06.f90 b/flang/test/Semantics/expr-errors06.f90
index 7d4283d35cdcb..c366060691556 100644
--- a/flang/test/Semantics/expr-errors06.f90
+++ b/flang/test/Semantics/expr-errors06.f90
@@ -43,6 +43,6 @@ subroutine subr(da)
   print *, empty(1:0,1) ! ok
   print *, empty(:,1) ! ok
   print *, empty(i:j,k) ! ok
-  !WARNING: Empty array dimension 1 should not be subscripted as an element or non-empty array section
+  !WARNING: Empty array dimension 1 should not be subscripted as an element or non-empty array section [-Wsubscripted-empty-array]
   print *, empty(i,1)
 end
diff --git a/flang/test/Semantics/final03.f90 b/flang/test/Semantics/final03.f90
index 3c402540152bc..15335b2eeadcf 100644
--- a/flang/test/Semantics/final03.f90
+++ b/flang/test/Semantics/final03.f90
@@ -20,7 +20,7 @@ program test
   type(pdt(1)) x1(1)
   type(pdt(2)) x2(1)
   type(pdt(3)) x3(1)
-  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'finalarr'
+  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'finalarr' [-Wvector-subscript-finalization]
   x1([1]) = pdt(1)()
   x2([1]) = pdt(2)() ! ok, doesn't match either
   x3([1]) = pdt(3)() ! ok, calls finalElem
diff --git a/flang/test/Semantics/forall01.f90 b/flang/test/Semantics/forall01.f90
index 72ad9ecd39471..d95328709ef48 100644
--- a/flang/test/Semantics/forall01.f90
+++ b/flang/test/Semantics/forall01.f90
@@ -10,7 +10,7 @@ subroutine forall1
     a(i) = i
   end forall
   forall (j=1:8)
-    !PORTABILITY: Index variable 'j' should not also be an index in an enclosing FORALL or DO CONCURRENT
+    !PORTABILITY: Index variable 'j' should not also be an index in an enclosing FORALL or DO CONCURRENT [-Wodd-index-variable-restrictions]
     forall (j=1:9)
     end forall
   end forall
@@ -41,7 +41,7 @@ subroutine forall3
   forall(i=1:10)
     forall(j=1:10)
       !ERROR: Cannot redefine FORALL variable 'i'
-      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment [-Wunused-forall-index]
       i = 1
     end forall
   end forall
@@ -80,20 +80,20 @@ subroutine forall5
     x(i) = y(i)
   end forall
   forall(i=1:10)
-    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
     x = y
     forall(j=1:10)
-      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment [-Wunused-forall-index]
       x(i) = y(i)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       x(j) = y(j)
     endforall
   endforall
   do concurrent(i=1:10)
     x = y
     !Odd rule from F'2023 19.4 p8
-    !PORTABILITY: Index variable 'i' should not also be an index in an enclosing FORALL or DO CONCURRENT
-    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+    !PORTABILITY: Index variable 'i' should not also be an index in an enclosing FORALL or DO CONCURRENT [-Wodd-index-variable-restrictions]
+    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
     forall(i=1:10) x = y
   end do
 end
@@ -106,7 +106,7 @@ subroutine forall6
   real, target :: b(10)
   forall(i=1:10)
     a(i)%p => b(i)
-    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
     a(1)%p => b(i)
   end forall
 end
@@ -116,20 +116,20 @@ subroutine forall7(x)
   real :: a(10)
   class(*) :: x
   associate (j => iarr(1))
-    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     forall (j=1:size(a))
       a(j) = a(j) + 1
     end forall
   end associate
   associate (j => iarr(1) + 1)
-    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     forall (j=1:size(a))
       a(j) = a(j) + 1
     end forall
   end associate
   select type (j => x)
   type is (integer)
-    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     forall (j=1:size(a))
       a(j) = a(j) + 1
     end forall
diff --git a/flang/test/Semantics/forall02.f90 b/flang/test/Semantics/forall02.f90
index c4f4311a175a3..17e61769dc37a 100644
--- a/flang/test/Semantics/forall02.f90
+++ b/flang/test/Semantics/forall02.f90
@@ -35,7 +35,7 @@ subroutine s1()
 
     ! Error to invoke an IMPURE FINAL procedure in a FORALL
     forall (i = 1:10)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       !ERROR: Impure procedure 'impuresub' is referenced by finalization in a FORALL
       ifvar = ifvar1
     end forall
@@ -46,7 +46,7 @@ subroutine s1()
     end forall
 
     forall (i = 1:5)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       !ERROR: Impure procedure 'impuresubrank1' is referenced by finalization in a FORALL
       ifArr1 = if0
     end forall
@@ -57,7 +57,7 @@ subroutine s1()
     end forall
 
     forall (i = 1:5)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       !ERROR: Impure procedure 'impuresubrank2' is referenced by finalization in a FORALL
       ifArr2(:,:) = if0
     end forall
diff --git a/flang/test/Semantics/generic03.f90 b/flang/test/Semantics/generic03.f90
index 829780f52174f..c5a2156c365d2 100644
--- a/flang/test/Semantics/generic03.f90
+++ b/flang/test/Semantics/generic03.f90
@@ -15,7 +15,7 @@ integer function f1(x, j)
 
 program test
   use m1
-  !WARNING: Generic interface 'g1' has both a function and a subroutine
+  !WARNING: Generic interface 'g1' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface g1
     subroutine s1(x, a)
       import t1
diff --git a/flang/test/Semantics/generic06.f90 b/flang/test/Semantics/generic06.f90
index b47d442f3549e..30dc05fc09f90 100644
--- a/flang/test/Semantics/generic06.f90
+++ b/flang/test/Semantics/generic06.f90
@@ -1,11 +1,11 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 module m
-  !PORTABILITY: Specific procedure 'sin' of generic interface 'yintercept' should not be INTRINSIC
+  !PORTABILITY: Specific procedure 'sin' of generic interface 'yintercept' should not be INTRINSIC [-Wintrinsic-as-specific]
   intrinsic sin
   interface yIntercept
     procedure sin
   end interface
-  !PORTABILITY: Specific procedure 'cos' of generic interface 'xintercept' should not be INTRINSIC
+  !PORTABILITY: Specific procedure 'cos' of generic interface 'xintercept' should not be INTRINSIC [-Wintrinsic-as-specific]
   intrinsic cos
   generic :: xIntercept => cos
 end module
diff --git a/flang/test/Semantics/global01.f90 b/flang/test/Semantics/global01.f90
index ab3edf9ef805f..ee7b170f577bc 100644
--- a/flang/test/Semantics/global01.f90
+++ b/flang/test/Semantics/global01.f90
@@ -30,7 +30,7 @@ recursive function global6()
 
 program test
   interface
-    !WARNING: The global subprogram 'global1' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: INTEGER(4) vs REAL(4))
+    !WARNING: The global subprogram 'global1' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: INTEGER(4) vs REAL(4)) [-Wexternal-interface-mismatch]
     subroutine global1(x)
       real, intent(in) :: x
     end subroutine
@@ -43,7 +43,7 @@ subroutine global3(x) bind(c,name="abc")
     subroutine global4(x) ! not PURE, but that's ok
       integer, intent(in) :: x
     end subroutine
-    !WARNING: The global subprogram 'global5' is not compatible with its local procedure declaration (incompatible procedure attributes: Pure)
+    !WARNING: The global subprogram 'global5' is not compatible with its local procedure declaration (incompatible procedure attributes: Pure) [-Wexternal-interface-mismatch]
     pure subroutine global5(x)
       integer, intent(in) :: x
     end subroutine
diff --git a/flang/test/Semantics/ichar01.f90 b/flang/test/Semantics/ichar01.f90
index 5eb25a97e6d93..d3609652165b8 100644
--- a/flang/test/Semantics/ichar01.f90
+++ b/flang/test/Semantics/ichar01.f90
@@ -5,9 +5,9 @@
 print *, iachar('')
 print *, ichar('a')
 print *, iachar('a')
-!PORTABILITY: Character in intrinsic function ichar should have length one
+!PORTABILITY: Character in intrinsic function ichar should have length one [-Wportability]
 print *, ichar('ab')
-!PORTABILITY: Character in intrinsic function iachar should have length one
+!PORTABILITY: Character in intrinsic function iachar should have length one [-Wportability]
 print *, iachar('ab')
 end
 
diff --git a/flang/test/Semantics/ignore_tkr01.f90 b/flang/test/Semantics/ignore_tkr01.f90
index 2af4974b1c038..d069c97331bf6 100644
--- a/flang/test/Semantics/ignore_tkr01.f90
+++ b/flang/test/Semantics/ignore_tkr01.f90
@@ -53,13 +53,13 @@ subroutine t8(x)
 
     subroutine t9(x)
 !dir$ ignore_tkr x
-!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer
+!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer [-Wignore-tkr-usage]
       real, intent(in), allocatable :: x
     end
 
     subroutine t10(x)
 !dir$ ignore_tkr x
-!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer
+!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer [-Wignore-tkr-usage]
       real, intent(in), pointer :: x
     end
 
@@ -88,7 +88,7 @@ elemental subroutine t13(x)
 
     subroutine t14(x)
 !dir$ ignore_tkr(r) x
-!WARNING: !DIR$ IGNORE_TKR(R) should not apply to a dummy argument passed via descriptor
+!WARNING: !DIR$ IGNORE_TKR(R) should not apply to a dummy argument passed via descriptor [-Wignore-tkr-usage]
       real x(:)
     end
 
@@ -145,7 +145,7 @@ subroutine t20(x)
 
   subroutine t22(x)
 !dir$ ignore_tkr(r) x
-!WARNING: !DIR$ IGNORE_TKR(R) is not meaningful for an assumed-rank array
+!WARNING: !DIR$ IGNORE_TKR(R) is not meaningful for an assumed-rank array [-Wignore-tkr-usage]
     real x(..)
   end
 
@@ -198,7 +198,7 @@ program test
   !ERROR: Actual argument type 'INTEGER(4)' is not compatible with dummy argument type 'REAL(4)'
   call t3(1)
   call t3(dx)
-  !ERROR: passing Hollerith or character literal as if it were BOZ
+  !ERROR: passing Hollerith or character literal as if it were BOZ [-Whollerith-or-character-as-boz]
   call t3('a')
   !ERROR: Actual argument type 'COMPLEX(4)' is not compatible with dummy argument type 'REAL(4)'
   call t3((1.,2.))
diff --git a/flang/test/Semantics/int-literals.f90 b/flang/test/Semantics/int-literals.f90
index 6d4a63e4bba71..7a386e0103b41 100644
--- a/flang/test/Semantics/int-literals.f90
+++ b/flang/test/Semantics/int-literals.f90
@@ -30,9 +30,9 @@
 complex, parameter :: okz4a = (+2147483648_8, 0)
 
 complex, parameter :: okj4d = 2147483647, okz4d = (+2147483647, -2147483648)
-!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8)
+!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8) [-Wbig-int-literals]
 complex, parameter :: badj4dext = 2147483648
-!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8)
+!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8) [-Wbig-int-literals]
 complex, parameter :: badz4dext = (+2147483648, 0)
 
 complex, parameter :: okj8 = 9223372036854775807_8, okz8 = (+9223372036854775807_8, -9223372036854775808_8)
diff --git a/flang/test/Semantics/intrinsics02.f90 b/flang/test/Semantics/intrinsics02.f90
index 0b1f7c13a1564..615045d275812 100644
--- a/flang/test/Semantics/intrinsics02.f90
+++ b/flang/test/Semantics/intrinsics02.f90
@@ -17,7 +17,7 @@ pure real function mycos(x)
 end
 subroutine sameIntrinsic1
   use explicit
-  !WARNING: Use-associated 'cos' already has 'INTRINSIC' attribute
+  !WARNING: Use-associated 'cos' already has 'INTRINSIC' attribute [-Wredundant-attribute]
   intrinsic cos
   real :: one = cos(0.)
 end
diff --git a/flang/test/Semantics/kinds05b.f90 b/flang/test/Semantics/kinds05b.f90
index 3927829bbaab6..3f2d4605adbdf 100644
--- a/flang/test/Semantics/kinds05b.f90
+++ b/flang/test/Semantics/kinds05b.f90
@@ -8,8 +8,8 @@ subroutine s
   real :: realvar3 = 4.0_8
   real :: realvar4 = 4.0E6_4
   real :: realvar5 = 4.0E6_8
-  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
+  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard [-Wexponent-matching-kind-param]
   real :: realvar6 = 4.0D6_8
-  !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'd'
+  !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'd' [-Wexponent-matching-kind-param]
   real :: realvar7 = 4.0D6_4
 end subroutine s
diff --git a/flang/test/Semantics/label18.f90 b/flang/test/Semantics/label18.f90
index a9f3e50237f98..152a189876b8a 100644
--- a/flang/test/Semantics/label18.f90
+++ b/flang/test/Semantics/label18.f90
@@ -9,10 +9,10 @@ program main
 1 end if
   if (.true.) then
     do j = 1, 2
-      !WARNING: Label '1' is in a construct that should not be used as a branch target here
+      !WARNING: Label '1' is in a construct that should not be used as a branch target here [-Wbranch-into-construct]
       goto 1
     end do
   end if
-  !WARNING: Label '1' is in a construct that should not be used as a branch target here
+  !WARNING: Label '1' is in a construct that should not be used as a branch target here [-Wbranch-into-construct]
   goto 1
 end
diff --git a/flang/test/Semantics/local-vs-global.f90 b/flang/test/Semantics/local-vs-global.f90
index 3f7e9339639cc..21a3009ad7f29 100644
--- a/flang/test/Semantics/local-vs-global.f90
+++ b/flang/test/Semantics/local-vs-global.f90
@@ -46,24 +46,24 @@ function implicit_func_before_2(a)
 
 program test
   external justfine ! OK to name a BLOCK DATA if not called
-  !WARNING: The global entity 'module_before_1' corresponding to the local procedure 'module_before_1' is not a callable subprogram
+  !WARNING: The global entity 'module_before_1' corresponding to the local procedure 'module_before_1' is not a callable subprogram [-Wexternal-name-conflict]
   external module_before_1
-  !WARNING: The global entity 'block_data_before_1' corresponding to the local procedure 'block_data_before_1' is not a callable subprogram
+  !WARNING: The global entity 'block_data_before_1' corresponding to the local procedure 'block_data_before_1' is not a callable subprogram [-Wexternal-name-conflict]
   external block_data_before_1
-  !WARNING: The global subprogram 'explicit_before_1' should not be referenced via the implicit interface 'explicit_before_1'
+  !WARNING: The global subprogram 'explicit_before_1' should not be referenced via the implicit interface 'explicit_before_1' [-Wexternal-interface-mismatch]
   external explicit_before_1
   external implicit_before_1
-  !WARNING: The global subprogram 'explicit_func_before_1' should not be referenced via the implicit interface 'explicit_func_before_1'
+  !WARNING: The global subprogram 'explicit_func_before_1' should not be referenced via the implicit interface 'explicit_func_before_1' [-Wexternal-interface-mismatch]
   external explicit_func_before_1
   external implicit_func_before_1
-  !WARNING: The global entity 'module_after_1' corresponding to the local procedure 'module_after_1' is not a callable subprogram
+  !WARNING: The global entity 'module_after_1' corresponding to the local procedure 'module_after_1' is not a callable subprogram [-Wexternal-name-conflict]
   external module_after_1
-  !WARNING: The global entity 'block_data_after_1' corresponding to the local procedure 'block_data_after_1' is not a callable subprogram
+  !WARNING: The global entity 'block_data_after_1' corresponding to the local procedure 'block_data_after_1' is not a callable subprogram [-Wexternal-name-conflict]
   external block_data_after_1
-  !WARNING: The global subprogram 'explicit_after_1' should not be referenced via the implicit interface 'explicit_after_1'
+  !WARNING: The global subprogram 'explicit_after_1' should not be referenced via the implicit interface 'explicit_after_1' [-Wexternal-interface-mismatch]
   external explicit_after_1
   external implicit_after_1
-  !WARNING: The global subprogram 'explicit_func_after_1' should not be referenced via the implicit interface 'explicit_func_after_1'
+  !WARNING: The global subprogram 'explicit_func_after_1' should not be referenced via the implicit interface 'explicit_func_after_1' [-Wexternal-interface-mismatch]
   external explicit_func_after_1
   external implicit_func_after_1
   call module_before_1
@@ -76,20 +76,20 @@ program test
   !ERROR: References to the procedure 'explicit_before_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   call explicit_before_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_before_1
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_before_2
   print *, explicit_func_before_1(1.)
   !ERROR: References to the procedure 'explicit_func_before_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   print *, explicit_func_before_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_before_1()
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_before_2()
   call module_after_1
@@ -100,20 +100,20 @@ program test
   !ERROR: References to the procedure 'explicit_after_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   call explicit_after_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_after_1
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_after_2
   print *, explicit_func_after_1(1.)
   !ERROR: References to the procedure 'explicit_func_after_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   print *, explicit_func_after_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_after_1()
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_after_2()
 end program
diff --git a/flang/test/Semantics/long-name.f90 b/flang/test/Semantics/long-name.f90
index c406e366f76d4..44899b13edd5a 100644
--- a/flang/test/Semantics/long-name.f90
+++ b/flang/test/Semantics/long-name.f90
@@ -1,14 +1,14 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
 
-!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63
+!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
 program aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1
 
-  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63
+  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
   integer :: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2
 
   integer :: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg
 
-  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg3 has length 64, which is greater than the maximum name length 63
+  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg3 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
   call aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg3
 
 end
diff --git a/flang/test/Semantics/modfile43.f90 b/flang/test/Semantics/modfile43.f90
index 1a50e94698f63..ed66dff27fb03 100644
--- a/flang/test/Semantics/modfile43.f90
+++ b/flang/test/Semantics/modfile43.f90
@@ -5,7 +5,7 @@ module iso_fortran_env
 end module
 module m1
   use, intrinsic :: iso_fortran_env, only: int32
-  !PORTABILITY: Should not USE the non-intrinsic module 'iso_fortran_env' in the same scope as a USE of the intrinsic module
+  !PORTABILITY: Should not USE the non-intrinsic module 'iso_fortran_env' in the same scope as a USE of the intrinsic module [-Wmisc-use-extensions]
   use, non_intrinsic :: iso_fortran_env, only: user_defined_123
 end module
 module m2
diff --git a/flang/test/Semantics/null-init.f90 b/flang/test/Semantics/null-init.f90
index ad3f91679a427..d01ad75a75a11 100644
--- a/flang/test/Semantics/null-init.f90
+++ b/flang/test/Semantics/null-init.f90
@@ -37,7 +37,7 @@ module m6
 
 module m7
   interface
-    !WARNING: The external interface 'null' is not compatible with an earlier definition (incompatible procedure attributes: ImplicitInterface)
+    !WARNING: The external interface 'null' is not compatible with an earlier definition (incompatible procedure attributes: ImplicitInterface) [-Wexternal-interface-mismatch]
     function null() result(p)
       integer, pointer :: p
     end function
diff --git a/flang/test/Semantics/null01.f90 b/flang/test/Semantics/null01.f90
index b21ee91a0be0a..64c9881297308 100644
--- a/flang/test/Semantics/null01.f90
+++ b/flang/test/Semantics/null01.f90
@@ -103,9 +103,9 @@ function f3()
   dt3x = dt3(pps1=null(mold=dt2x%pps0))
   dt3x = dt3(pps1=null(mold=dt3x%pps1))
   dt4x = dt4(null()) ! ok
-  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0'
+  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0' [-Wnull-mold-allocatable-component-value]
   dt4x = dt4(null(rp0))
-  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0'
+  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0' [-Wnull-mold-allocatable-component-value]
   !ERROR: Rank-1 array value is not compatible with scalar component 'ra0'
   dt4x = dt4(null(rp1))
   !ERROR: A NULL procedure pointer may not be used as the value for component 'ra0'
@@ -133,7 +133,7 @@ function f3()
   print *, same_type_as(null(dt5p), null(dt4p)) ! ok
   !ERROR: A NULL() pointer is not allowed for 'source=' intrinsic argument
   print *, transfer(null(rp0),ip0)
-  !WARNING: Source of TRANSFER contains allocatable or pointer component %ra0
+  !WARNING: Source of TRANSFER contains allocatable or pointer component %ra0 [-Wpointer-component-transfer-arg]
   print *, transfer(dt4(null()),[0])
   !ERROR: NULL() may not be used as an expression in this context
   select case(null(ip0))
diff --git a/flang/test/Semantics/pointer01.f90 b/flang/test/Semantics/pointer01.f90
index cb860f3a3f437..eaa2426dd77e3 100644
--- a/flang/test/Semantics/pointer01.f90
+++ b/flang/test/Semantics/pointer01.f90
@@ -7,7 +7,7 @@ subroutine msubr
 end module
 program main
   use m
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   pointer main
   !ERROR: Cannot change POINTER attribute on use-associated 'mobj'
   pointer mobj
diff --git a/flang/test/Semantics/procinterface02.f90 b/flang/test/Semantics/procinterface02.f90
index 8b1becbe081e5..6aa5dc13bd9a5 100644
--- a/flang/test/Semantics/procinterface02.f90
+++ b/flang/test/Semantics/procinterface02.f90
@@ -12,7 +12,7 @@ real function foo_nonelemental(x)
     end function
   end interface
   real :: A(:), B(:)
-  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sqrt), pointer :: P
   !ERROR: Rank of dummy argument is 0, but actual argument has rank 1
   A = P(B)
diff --git a/flang/test/Semantics/procinterface04.f90 b/flang/test/Semantics/procinterface04.f90
index f59e53f90dab1..fb48c4b17b419 100644
--- a/flang/test/Semantics/procinterface04.f90
+++ b/flang/test/Semantics/procinterface04.f90
@@ -9,11 +9,11 @@ pure real function nonelemental(x)
       real, intent(in) :: x
     end
   end interface
-  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin) :: dp1
   !ERROR: A dummy procedure may not be ELEMENTAL
   procedure(elemental) :: dp2
-  !PORTABILITY: Procedure pointer 'pp1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'pp1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: pp1
   !ERROR: Procedure pointer 'pp2' may not be ELEMENTAL
   procedure(elemental), pointer :: pp2
diff --git a/flang/test/Semantics/resolve05.f90 b/flang/test/Semantics/resolve05.f90
index 4e50feb64e4c3..0c9877af9b4e2 100644
--- a/flang/test/Semantics/resolve05.f90
+++ b/flang/test/Semantics/resolve05.f90
@@ -1,18 +1,18 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 program p
-  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   integer :: p
 end
 module m
-  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module
+  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module [-Wbenign-name-clash]
   integer :: m
 end
 submodule(m) sm
-  !PORTABILITY: Name 'sm' declared in a submodule should not have the same name as the submodule
+  !PORTABILITY: Name 'sm' declared in a submodule should not have the same name as the submodule [-Wbenign-name-clash]
   integer :: sm
 end
 block data bd
-  !PORTABILITY: Name 'bd' declared in a BLOCK DATA subprogram should not have the same name as the BLOCK DATA subprogram
+  !PORTABILITY: Name 'bd' declared in a BLOCK DATA subprogram should not have the same name as the BLOCK DATA subprogram [-Wbenign-name-clash]
   type bd
   end type
 end
diff --git a/flang/test/Semantics/resolve108.f90 b/flang/test/Semantics/resolve108.f90
index 644b8fba86892..4ad53da5ddeb4 100644
--- a/flang/test/Semantics/resolve108.f90
+++ b/flang/test/Semantics/resolve108.f90
@@ -44,7 +44,7 @@ subroutine s2
   use :: m2, only: foo
   !If we got the type of foo right, this declaration will fail
   !due to an attempted division by zero.
-  !WARNING: INTEGER(4) division by zero
+  !WARNING: INTEGER(4) division by zero [-Wfolding-exception]
   !ERROR: Must be a constant value
   integer, parameter :: test = 1 / (kind(foo(1)) - kind(1.d0))
 end subroutine
@@ -54,7 +54,7 @@ module m3
  contains
   real(kind=kind(x)) function foo(x)
     real(kind=kind(1.0d0)) x
-    !WARNING: INTEGER(4) division by zero
+    !WARNING: INTEGER(4) division by zero [-Wfolding-exception]
     !ERROR: Must be a constant value
     integer, parameter :: test = 1 / (kind(foo) - kind(1.d0))
     foo = n
diff --git a/flang/test/Semantics/resolve11.f90 b/flang/test/Semantics/resolve11.f90
index 9ae4f52c4fd54..0c457f2c9f3f0 100644
--- a/flang/test/Semantics/resolve11.f90
+++ b/flang/test/Semantics/resolve11.f90
@@ -4,7 +4,7 @@ module m
   integer, private :: j
   !ERROR: The accessibility of 'i' has already been specified as PUBLIC
   private i
-  !WARNING: The accessibility of 'j' has already been specified as PRIVATE
+  !WARNING: The accessibility of 'j' has already been specified as PRIVATE [-Wredundant-attribute]
   private j
 end
 
@@ -66,7 +66,7 @@ subroutine s4
   !ERROR: 'fun' is PRIVATE in 'm4'
   use m4, only: foo, fun
   type(foo) x ! ok
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'fun') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'fun') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   print *, foo()
 end
 
diff --git a/flang/test/Semantics/resolve114.f90 b/flang/test/Semantics/resolve114.f90
index 6204e5fc2f3f4..05fb7aa5e4795 100644
--- a/flang/test/Semantics/resolve114.f90
+++ b/flang/test/Semantics/resolve114.f90
@@ -34,9 +34,9 @@ end module m2
 subroutine s2a
   use m1
   use m2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: p1 => sin
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iabs), pointer :: p2 => iabs
   procedure(ext1), pointer :: p3 => ext1
   procedure(ext2), pointer :: p4 => ext2
@@ -46,9 +46,9 @@ subroutine s2b
   use m1, only: x1 => sin, x2 => iabs, x3 => ext1, x4 => ext2
   use m2, only: x1 => sin, x2 => iabs, x3 => ext1, x4 => ext2
   use m1, only: iface1 => sin, iface2 => iabs, iface3 => ext1, iface4 => ext2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface1), pointer :: p1 => x1
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface2), pointer :: p2 => x2
   procedure(iface3), pointer :: p3 => x3
   procedure(iface4), pointer :: p4 => x4
@@ -60,9 +60,9 @@ module m3
 end module
 subroutine s3
   use m3
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: p1 => sin
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iabs), pointer :: p2 => iabs
   procedure(ext1), pointer :: p3 => ext1
   procedure(ext2), pointer :: p4 => ext2
@@ -75,9 +75,9 @@ module m4
 subroutine s4
   use m4
   use m1, only: iface1 => sin, iface2 => iabs, iface3 => ext1, iface4 => ext2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface1), pointer :: p1 => x1
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface2), pointer :: p2 => x2
   procedure(iface3), pointer :: p3 => x3
   procedure(iface4), pointer :: p4 => x4
@@ -87,10 +87,10 @@ subroutine s5
   use m1, only: x1 => sin, x2 => iabs, x3 => ext1, x4 => ext2
   use m2, only: x1 => tan, x2 => idim, x3 => ext2, x4 => ext1
   use m1, only: iface1 => sin, iface2 => iabs, iface3 => ext1, iface4 => ext2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   !ERROR: Reference to 'x1' is ambiguous
   procedure(iface1), pointer :: p1 => x1
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   !ERROR: Reference to 'x2' is ambiguous
   procedure(iface2), pointer :: p2 => x2
   !ERROR: Reference to 'x3' is ambiguous
diff --git a/flang/test/Semantics/resolve118.f90 b/flang/test/Semantics/resolve118.f90
index 024b67b5a471b..59199dfd6311b 100644
--- a/flang/test/Semantics/resolve118.f90
+++ b/flang/test/Semantics/resolve118.f90
@@ -17,7 +17,7 @@ module m3
   interface
     subroutine s1(x)
       use m1
-      !PORTABILITY: The same 't' is already present in this scope
+      !PORTABILITY: The same 't' is already present in this scope [-Wbenign-name-clash]
       import t
       type(t) x
     end
diff --git a/flang/test/Semantics/resolve17.f90 b/flang/test/Semantics/resolve17.f90
index 6a6e355abe0b8..173fb9c6c5b78 100644
--- a/flang/test/Semantics/resolve17.f90
+++ b/flang/test/Semantics/resolve17.f90
@@ -11,7 +11,7 @@ module m2
   interface s
   end interface
 contains
-  !WARNING: 's' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic
+  !WARNING: 's' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic [-Whomonymous-specific]
   subroutine s
   end subroutine
 end module
@@ -290,7 +290,7 @@ module m14d
  contains
   subroutine test
     real :: y
-    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name
+    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
     y = foo(1.0)
     x = foo(2)
   end subroutine
@@ -302,7 +302,7 @@ module m14e
  contains
   subroutine test
     real :: y
-    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name
+    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
     y = foo(1.0)
     x = foo(2)
   end subroutine
diff --git a/flang/test/Semantics/resolve18.f90 b/flang/test/Semantics/resolve18.f90
index 547db5e85714c..65c39931cc7cb 100644
--- a/flang/test/Semantics/resolve18.f90
+++ b/flang/test/Semantics/resolve18.f90
@@ -11,7 +11,7 @@ subroutine foo(x)
 module m2
   use m1
   implicit none
-  !WARNING: 'foo' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic
+  !WARNING: 'foo' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic [-Whomonymous-specific]
   interface foo
     module procedure s
   end interface
@@ -22,14 +22,14 @@ subroutine s(i)
 end module
 
 subroutine foo
-  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name
+  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name [-Wuse-association-into-same-name-subprogram]
   use m1
   !ERROR: Reference to 'foo' is ambiguous
   call foo
 end
 
 subroutine bar
-  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name
+  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name [-Wuse-association-into-same-name-subprogram]
   use m1, bar => foo
   !ERROR: Reference to 'bar' is ambiguous
   call bar
@@ -348,7 +348,7 @@ subroutine s_21_23
   use m21
   use m23
   type(foo) x ! Intel and NAG error
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'f1') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'f1') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   print *, foo(1.) ! Intel error
   print *, foo(1.,2.,3.) ! Intel error
   call ext(foo) ! GNU and Intel error
diff --git a/flang/test/Semantics/resolve20.f90 b/flang/test/Semantics/resolve20.f90
index 359bc0c1f9a3f..8b8d190206689 100644
--- a/flang/test/Semantics/resolve20.f90
+++ b/flang/test/Semantics/resolve20.f90
@@ -38,7 +38,7 @@ subroutine forward
   type :: bad3
   end type
 
-  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module
+  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module [-Wbenign-name-clash]
   type :: m
   end type m
 
@@ -49,7 +49,7 @@ subroutine forward
   external :: a, b, c, d
   !ERROR: EXTERNAL attribute not allowed on 'm'
   external :: m
-  !WARNING: EXTERNAL attribute was already specified on 'foo'
+  !WARNING: EXTERNAL attribute was already specified on 'foo' [-Wredundant-attribute]
   external :: foo
   !ERROR: EXTERNAL attribute not allowed on 'bar'
   external :: bar
diff --git a/flang/test/Semantics/resolve24.f90 b/flang/test/Semantics/resolve24.f90
index 72d6719665bb5..664fe0faeedc9 100644
--- a/flang/test/Semantics/resolve24.f90
+++ b/flang/test/Semantics/resolve24.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 subroutine test1
-  !WARNING: Generic interface 'foo' has both a function and a subroutine
+  !WARNING: Generic interface 'foo' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface foo
     subroutine s1(x)
     end subroutine
@@ -12,7 +12,7 @@ function f()
 end subroutine
 
 subroutine test2
-  !WARNING: Generic interface 'foo' has both a function and a subroutine
+  !WARNING: Generic interface 'foo' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface foo
     function t2f1(x)
     end function
@@ -24,7 +24,7 @@ function t2f2(x, y)
 end subroutine
 
 module test3
-  !WARNING: Generic interface 'foo' has both a function and a subroutine
+  !WARNING: Generic interface 'foo' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface foo
     module procedure s
     module procedure f
@@ -39,7 +39,7 @@ function f()
 subroutine test4
   type foo
   end type
-  !WARNING: Generic interface 'foo' should only contain functions due to derived type with same name
+  !WARNING: Generic interface 'foo' should only contain functions due to derived type with same name [-Wsubroutine-and-function-specifics]
   interface foo
     subroutine s()
     end subroutine
diff --git a/flang/test/Semantics/resolve30.f90 b/flang/test/Semantics/resolve30.f90
index 32108e89cdefe..28613dfe4ffd9 100644
--- a/flang/test/Semantics/resolve30.f90
+++ b/flang/test/Semantics/resolve30.f90
@@ -41,6 +41,6 @@ subroutine s4
 subroutine s5
   implicit none
   data x/1./
-  !PORTABILITY: 'x' appeared in a DATA statement before its type was declared under IMPLICIT NONE(TYPE)
+  !PORTABILITY: 'x' appeared in a DATA statement before its type was declared under IMPLICIT NONE(TYPE) [-Wforward-ref-implicit-none-data]
   real x
 end
diff --git a/flang/test/Semantics/resolve31.f90 b/flang/test/Semantics/resolve31.f90
index 6bf8e877a5156..eabd87776904d 100644
--- a/flang/test/Semantics/resolve31.f90
+++ b/flang/test/Semantics/resolve31.f90
@@ -49,9 +49,9 @@ module m4
   type :: t1
     private
     sequence
-    !WARNING: PRIVATE should not appear more than once in derived type components
+    !WARNING: PRIVATE should not appear more than once in derived type components [-Wredundant-attribute]
     private
-    !WARNING: SEQUENCE should not appear more than once in derived type components
+    !WARNING: SEQUENCE should not appear more than once in derived type components [-Wredundant-attribute]
     sequence
     real :: t1Field
   end type
@@ -68,7 +68,7 @@ module m4
   !ERROR: A sequence type may not have a CONTAINS statement
   contains
   end type
-  !WARNING: A sequence type should have at least one component
+  !WARNING: A sequence type should have at least one component [-Wempty-sequence-type]
   type :: emptyType
     sequence
   end type emptyType
@@ -85,7 +85,7 @@ module m4
     class(*), allocatable :: typeStarField
     !ERROR: A sequence type data component must either be of an intrinsic type or a derived sequence type
     type(plainType) :: testField1
-    !WARNING: A sequence type data component that is a pointer to a non-sequence type is not standard
+    !WARNING: A sequence type data component that is a pointer to a non-sequence type is not standard [-Wpointer-in-seq-type]
     type(plainType), pointer :: testField1p
     type(sequenceType) :: testField2
     procedure(real), pointer, nopass :: procField
diff --git a/flang/test/Semantics/resolve35.f90 b/flang/test/Semantics/resolve35.f90
index 2947b225978d1..f3f8409161df2 100644
--- a/flang/test/Semantics/resolve35.f90
+++ b/flang/test/Semantics/resolve35.f90
@@ -21,14 +21,14 @@ subroutine s3
   real :: a(10,10), b(10,10)
   type y; end type
   integer(8) :: x
-  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope
+  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
   !ERROR: Must have INTEGER type, but is REAL(4)
   forall(x=1:10, y=1:10)
     !ERROR: Must have INTEGER type, but is REAL(4)
     !ERROR: Must have INTEGER type, but is REAL(4)
     a(x, y) = b(x, y)
   end forall
-  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope
+  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
   !ERROR: Must have INTEGER type, but is REAL(4)
   !ERROR: Must have INTEGER type, but is REAL(4)
   !ERROR: Must have INTEGER type, but is REAL(4)
@@ -51,7 +51,7 @@ subroutine s4
     !ERROR: Must have INTEGER type, but is REAL(4)
     a(y) = b(y)
   end forall
-  !PORTABILITY: Index variable 'i' should be scalar in the enclosing scope
+  !PORTABILITY: Index variable 'i' should be scalar in the enclosing scope [-Wodd-index-variable-restrictions]
   forall(i=1:10)
     a(i) = b(i)
   end forall
@@ -61,7 +61,7 @@ subroutine s6
   integer, parameter :: n = 4
   real, dimension(n) :: x
   data(x(i), i=1, n) / n * 0.0 /
-  !PORTABILITY: Index variable 't' should be a scalar object or common block if it is present in the enclosing scope
+  !PORTABILITY: Index variable 't' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
   !ERROR: Must have INTEGER type, but is REAL(4)
   !ERROR: Must have INTEGER type, but is REAL(4)
   forall(t=1:n) x(t) = 0.0
@@ -86,7 +86,7 @@ subroutine s7
   do concurrent(integer::i=1:5) local(j, i) &
       !ERROR: 'j' is already declared in this scoping unit
       local_init(k, j) &
-      !WARNING: Variable 'a' with SHARED locality implicitly declared
+      !WARNING: Variable 'a' with SHARED locality implicitly declared [-Wimplicit-shared]
       shared(a)
     a = j + 1
   end do
diff --git a/flang/test/Semantics/resolve37.f90 b/flang/test/Semantics/resolve37.f90
index f8229f1f974a4..d35174e62d64a 100644
--- a/flang/test/Semantics/resolve37.f90
+++ b/flang/test/Semantics/resolve37.f90
@@ -25,25 +25,25 @@
 character(len=l) :: v
 !ERROR: Value of named constant 'o' (o) cannot be computed as a constant value
 real, parameter ::  o = o
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
 integer, parameter ::  p = 0/0
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
-!WARNING: INTEGER(4) division by zero
-!WARNING: INTEGER(4) division by zero
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 integer, parameter ::  q = 1+2*(1/0)
 integer not_constant
 !ERROR: Must be a constant value
 integer, parameter :: s1 = not_constant/2
 !ERROR: Must be a constant value
 integer, parameter :: s2 = 3/not_constant
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
 integer(kind=2/0) r
 integer, parameter :: sok(*)=[1,2]/[1,2]
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
 integer, parameter :: snok(*)=[1,2]/[1,0]
 end
diff --git a/flang/test/Semantics/resolve45.f90 b/flang/test/Semantics/resolve45.f90
index 7f6d8d6ae50b2..18a382a788b6b 100644
--- a/flang/test/Semantics/resolve45.f90
+++ b/flang/test/Semantics/resolve45.f90
@@ -36,13 +36,13 @@ subroutine s3(x)
 end
 
 subroutine s4
-  !WARNING: Explicit SAVE of 'z' is redundant due to global SAVE statement
+  !WARNING: Explicit SAVE of 'z' is redundant due to global SAVE statement [-Wredundant-attribute]
   save z
   save
   procedure(integer), pointer :: x
-  !WARNING: Explicit SAVE of 'x' is redundant due to global SAVE statement
+  !WARNING: Explicit SAVE of 'x' is redundant due to global SAVE statement [-Wredundant-attribute]
   save :: x
-  !WARNING: Explicit SAVE of 'y' is redundant due to global SAVE statement
+  !WARNING: Explicit SAVE of 'y' is redundant due to global SAVE statement [-Wredundant-attribute]
   integer, save :: y
 end
 
diff --git a/flang/test/Semantics/resolve46.f90 b/flang/test/Semantics/resolve46.f90
index 0acc20b19f0b7..d473226edaabe 100644
--- a/flang/test/Semantics/resolve46.f90
+++ b/flang/test/Semantics/resolve46.f90
@@ -20,7 +20,7 @@ logical function chrcmp(a,b)
      end function chrcmp
   end interface
 
-  !PORTABILITY: Procedure pointer 'p' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: p => cos
   !ERROR: Intrinsic procedure 'amin0' is not an unrestricted specific intrinsic permitted for use as the definition of the interface to procedure pointer 'q'
   procedure(amin0), pointer :: q
@@ -29,7 +29,7 @@ end function chrcmp
   !ERROR: Intrinsic procedure 'llt' is not an unrestricted specific intrinsic permitted for use as the initializer for procedure pointer 's'
   procedure(chrcmp), pointer :: s => llt
   !ERROR: Intrinsic procedure 'bessel_j0' is not an unrestricted specific intrinsic permitted for use as the initializer for procedure pointer 't'
-  !PORTABILITY: Procedure pointer 't' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 't' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(cos), pointer :: t => bessel_j0
   procedure(chrcmp), pointer :: u
   p => alog ! valid use of an unrestricted specific intrinsic
diff --git a/flang/test/Semantics/resolve55.f90 b/flang/test/Semantics/resolve55.f90
index 5f7a3044e834c..908fda8a83e15 100644
--- a/flang/test/Semantics/resolve55.f90
+++ b/flang/test/Semantics/resolve55.f90
@@ -94,3 +94,23 @@ subroutine s8(arg)
   do concurrent(i=1:5) local(arg)
   end do
 end subroutine s8
+
+subroutine s9()
+  type l3
+    integer, allocatable :: a
+  end type
+  type l2
+    type(l3) :: l2_3
+  end type
+  type l1
+    type(l2) :: l1_2
+  end type
+  type(l1) :: v
+  integer sum
+
+  sum = 0
+!ERROR: Derived type variable 'v' with ultimate ALLOCATABLE component '%l1_2%l2_3%a' not allowed in a LOCAL_INIT locality-spec
+  do concurrent (i = 1:10) local_init(v)
+    sum = sum + i
+  end do
+end subroutine s9
diff --git a/flang/test/Semantics/resolve58.f90 b/flang/test/Semantics/resolve58.f90
index 7686de4898404..ff9d6c358c69e 100644
--- a/flang/test/Semantics/resolve58.f90
+++ b/flang/test/Semantics/resolve58.f90
@@ -69,12 +69,12 @@ subroutine s6()
 
   !ERROR: Implied-shape array 'local1' must be a named constant or a dummy argument
   real, dimension (*) :: local1
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   real, intent(in) :: local2
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   procedure(), intent(in) :: p1
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   real, optional :: local3
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   procedure(), optional :: p2
 end subroutine
diff --git a/flang/test/Semantics/resolve59.f90 b/flang/test/Semantics/resolve59.f90
index 3bdcf67aa958a..a598fbe1849f3 100644
--- a/flang/test/Semantics/resolve59.f90
+++ b/flang/test/Semantics/resolve59.f90
@@ -59,10 +59,10 @@ real function rfunc(x)
     x = acos(f5)
   end function
   ! Sanity test: f18 handles C1560 violation by ignoring RESULT
-  !WARNING: The function name should not appear in RESULT; references to 'f6' inside the function will be considered as references to the result only
+  !WARNING: The function name should not appear in RESULT; references to 'f6' inside the function will be considered as references to the result only [-Whomonymous-result]
   function f6() result(f6)
   end function
-  !WARNING: The function name should not appear in RESULT; references to 'f7' inside the function will be considered as references to the result only
+  !WARNING: The function name should not appear in RESULT; references to 'f7' inside the function will be considered as references to the result only [-Whomonymous-result]
   function f7() result(f7)
     real :: x, f7
     !ERROR: Recursive call to 'f7' requires a distinct RESULT in its declaration
@@ -114,7 +114,7 @@ function f4() result(r)
   end function
   function f5(x) result(r)
     real :: x
-    !PORTABILITY: Procedure pointer 'r' should not have an ELEMENTAL intrinsic as its interface
+    !PORTABILITY: Procedure pointer 'r' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
     procedure(acos), pointer :: r
     r => acos
     !ERROR: Actual argument for 'x=' may not be a procedure
diff --git a/flang/test/Semantics/resolve60.f90 b/flang/test/Semantics/resolve60.f90
index ff988d54a8210..0192f830dad63 100644
--- a/flang/test/Semantics/resolve60.f90
+++ b/flang/test/Semantics/resolve60.f90
@@ -32,7 +32,7 @@
 
   enum, bind(C)
     !ERROR: Enumerator value could not be computed from the given expression
-    !WARNING: INTEGER(4) division by zero
+    !WARNING: INTEGER(4) division by zero [-Wfolding-exception]
     !ERROR: Must be a constant value
     enumerator :: wrong = 0/0
   end enum
diff --git a/flang/test/Semantics/resolve61.f90 b/flang/test/Semantics/resolve61.f90
index 2a1f584ffaf08..ffee32243af6e 100644
--- a/flang/test/Semantics/resolve61.f90
+++ b/flang/test/Semantics/resolve61.f90
@@ -114,7 +114,7 @@ subroutine p12
   type(t2) :: x2
   type(t3) :: x3
   pointer(a, x1)
-  !WARNING: Type of Cray pointee 'x2' is a derived type that is neither SEQUENCE nor BIND(C)
+  !WARNING: Type of Cray pointee 'x2' is a derived type that is neither SEQUENCE nor BIND(C) [-Wnon-sequence-cray-pointee]
   pointer(b, x2)
   pointer(c, x3)
 end
diff --git a/flang/test/Semantics/resolve65.f90 b/flang/test/Semantics/resolve65.f90
index b2815c4ed1c79..1f985125a152c 100644
--- a/flang/test/Semantics/resolve65.f90
+++ b/flang/test/Semantics/resolve65.f90
@@ -15,8 +15,8 @@ module m1
     procedure :: assign_t2
     procedure :: assign_t3
     !ERROR: Defined assignment subroutine 'assign_t2' must have two dummy arguments
-    !WARNING: In defined assignment subroutine 'assign_t3', second dummy argument 'y' should have INTENT(IN) or VALUE attribute
-    !WARNING: In defined assignment subroutine 'assign_t4', first dummy argument 'x' should have INTENT(OUT) or INTENT(INOUT)
+    !WARNING: In defined assignment subroutine 'assign_t3', second dummy argument 'y' should have INTENT(IN) or VALUE attribute [-Wdefined-operator-args]
+    !WARNING: In defined assignment subroutine 'assign_t4', first dummy argument 'x' should have INTENT(OUT) or INTENT(INOUT) [-Wdefined-operator-args]
     !ERROR: In defined assignment subroutine 'assign_t5', first dummy argument 'x' may not have INTENT(IN)
     !ERROR: In defined assignment subroutine 'assign_t6', second dummy argument 'y' may not have INTENT(OUT)
     generic :: assignment(=) => assign_t, assign_t2, assign_t3, assign_t4, assign_t5, assign_t6
diff --git a/flang/test/Semantics/resolve67.f90 b/flang/test/Semantics/resolve67.f90
index 677eef21d0a8f..7dd0f2adc9d7b 100644
--- a/flang/test/Semantics/resolve67.f90
+++ b/flang/test/Semantics/resolve67.f90
@@ -41,7 +41,7 @@ character(*) function divide(x, y)
     end
   end interface
   interface operator(<)
-    !WARNING: In OPERATOR(<) function 'lt1', dummy argument 'x' should have INTENT(IN) or VALUE attribute
+    !WARNING: In OPERATOR(<) function 'lt1', dummy argument 'x' should have INTENT(IN) or VALUE attribute [-Wdefined-operator-args]
     !ERROR: In OPERATOR(<) function 'lt1', dummy argument 'y' may not be OPTIONAL
     logical function lt1(x, y)
       logical :: x
@@ -90,7 +90,7 @@ real function plus(x)
     end
   end interface
   interface operator(.not.)
-    !WARNING: The external interface 'not1' is not compatible with an earlier definition (distinct numbers of dummy arguments)
+    !WARNING: The external interface 'not1' is not compatible with an earlier definition (distinct numbers of dummy arguments) [-Wexternal-interface-mismatch]
     real function not1(x)
       real, value :: x
     end
diff --git a/flang/test/Semantics/resolve69.f90 b/flang/test/Semantics/resolve69.f90
index 5acfd30604fe3..5af14b48a07eb 100644
--- a/flang/test/Semantics/resolve69.f90
+++ b/flang/test/Semantics/resolve69.f90
@@ -16,7 +16,7 @@ subroutine s1()
   !
   integer, parameter :: constVal = 1
   integer :: nonConstVal = 1
-!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved)
+!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved) [-Wsaved-local-in-spec-expr]
   character(nonConstVal) :: colonString1
   character(len=20, kind=constVal + 1) :: constKindString
   character(len=:, kind=constVal + 1), pointer :: constKindString1
@@ -53,13 +53,13 @@ function foo3()
 
   type (derived(constVal, 3)) :: constDerivedKind
 !ERROR: Value of KIND type parameter 'typekind' must be constant
-!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved)
+!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved) [-Wsaved-local-in-spec-expr]
   type (derived(nonConstVal, 3)) :: nonConstDerivedKind
 
   !OK because all type-params are constants
   type (derived(3, constVal)) :: constDerivedLen
 
-!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved)
+!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved) [-Wsaved-local-in-spec-expr]
   type (derived(3, nonConstVal)) :: nonConstDerivedLen
 !ERROR: 'colonderivedlen' has a type derived(typekind=3_4,typelen=:) with a deferred type parameter but is neither an allocatable nor an object pointer
   type (derived(3, :)) :: colonDerivedLen
diff --git a/flang/test/Semantics/resolve77.f90 b/flang/test/Semantics/resolve77.f90
index 0133fac3bfbc5..f58f2637c453e 100644
--- a/flang/test/Semantics/resolve77.f90
+++ b/flang/test/Semantics/resolve77.f90
@@ -60,13 +60,13 @@ pure integer function if2(n)
 block data
   common /blk2/ n
   data n/100/
-  !PORTABILITY: specification expression refers to local object 'n' (initialized and saved)
+  !PORTABILITY: specification expression refers to local object 'n' (initialized and saved) [-Wsaved-local-in-spec-expr]
   !ERROR: Automatic data object 'a' may not appear in a BLOCK DATA subprogram
   real a(n)
 end
 
 program main
   common /blk2/ n
-  !PORTABILITY: Automatic data object 'a' should not appear in the specification part of a main program
+  !PORTABILITY: Automatic data object 'a' should not appear in the specification part of a main program [-Wautomatic-in-main-program]
   real a(n)
 end
diff --git a/flang/test/Semantics/resolve78.f90 b/flang/test/Semantics/resolve78.f90
index 8c25f32e80cdc..c4163fe370310 100644
--- a/flang/test/Semantics/resolve78.f90
+++ b/flang/test/Semantics/resolve78.f90
@@ -11,21 +11,21 @@ module m
 !      POINTER
 
   type :: derived
-    !WARNING: Attribute 'PUBLIC' cannot be used more than once
+    !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
     real, public, allocatable, public :: field1
-    !WARNING: Attribute 'PRIVATE' cannot be used more than once
+    !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
     real, private, allocatable, private :: field2
     !ERROR: Attributes 'PUBLIC' and 'PRIVATE' conflict with each other
     real, public, allocatable, private :: field3
-    !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once
+    !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once [-Wredundant-attribute]
     real, allocatable, public, allocatable :: field4
     !ERROR: Attribute 'CODIMENSION' cannot be used more than once
     real, public, codimension[:], allocatable, codimension[:] :: field5
-    !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once
+    !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once [-Wredundant-attribute]
     real, public, contiguous, pointer, contiguous, dimension(:) :: field6
     !ERROR: Attribute 'DIMENSION' cannot be used more than once
     real, dimension(5), public, dimension(5) :: field7
-    !WARNING: Attribute 'POINTER' cannot be used more than once
+    !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
     real, pointer, public, pointer :: field8
   end type derived
 
diff --git a/flang/test/Semantics/resolve79.f90 b/flang/test/Semantics/resolve79.f90
index 037e107dc5874..87bd55a3bcfb5 100644
--- a/flang/test/Semantics/resolve79.f90
+++ b/flang/test/Semantics/resolve79.f90
@@ -12,17 +12,17 @@ module m
 !    PUBLIC, PRIVATE, NOPASS, PASS, POINTER
 
   type :: procComponentType
-    !WARNING: Attribute 'PUBLIC' cannot be used more than once
+    !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
     procedure(publicProc), public, pointer, public :: publicField
-    !WARNING: Attribute 'PRIVATE' cannot be used more than once
+    !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
     procedure(privateProc), private, pointer, private :: privateField
-    !WARNING: Attribute 'NOPASS' cannot be used more than once
+    !WARNING: Attribute 'NOPASS' cannot be used more than once [-Wredundant-attribute]
     procedure(nopassProc), nopass, pointer, nopass :: noPassField
-    !WARNING: Attribute 'PASS' cannot be used more than once
+    !WARNING: Attribute 'PASS' cannot be used more than once [-Wredundant-attribute]
     procedure(passProc), pass, pointer, pass :: passField
     !ERROR: Attributes 'PASS' and 'NOPASS' conflict with each other
     procedure(passNopassProc), pass, pointer, nopass :: passNopassField
-    !WARNING: Attribute 'POINTER' cannot be used more than once
+    !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
     procedure(pointerProc), pointer, public, pointer :: pointerField
     !ERROR: Procedure component 'nonpointerfield' must have POINTER attribute
     procedure(publicProc), public :: nonpointerField
diff --git a/flang/test/Semantics/resolve80.f90 b/flang/test/Semantics/resolve80.f90
index e787a4ba7f866..7df6a6c57d695 100644
--- a/flang/test/Semantics/resolve80.f90
+++ b/flang/test/Semantics/resolve80.f90
@@ -13,17 +13,17 @@ module m
 !
   type, abstract :: boundProcType
    contains
-    !WARNING: Attribute 'PUBLIC' cannot be used more than once
+    !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
     procedure(subPublic), public, deferred, public :: publicBinding
-    !WARNING: Attribute 'PRIVATE' cannot be used more than once
+    !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
     procedure(subPrivate), private, deferred, private :: privateBinding
-    !WARNING: Attribute 'DEFERRED' cannot be used more than once
+    !WARNING: Attribute 'DEFERRED' cannot be used more than once [-Wredundant-attribute]
     procedure(subDeferred), deferred, public, deferred :: deferredBinding
-    !WARNING: Attribute 'NON_OVERRIDABLE' cannot be used more than once
+    !WARNING: Attribute 'NON_OVERRIDABLE' cannot be used more than once [-Wredundant-attribute]
     procedure, non_overridable, public, non_overridable :: subNon_overridable;
-    !WARNING: Attribute 'NOPASS' cannot be used more than once
+    !WARNING: Attribute 'NOPASS' cannot be used more than once [-Wredundant-attribute]
     procedure(subNopass), nopass, deferred, nopass :: nopassBinding
-    !WARNING: Attribute 'PASS' cannot be used more than once
+    !WARNING: Attribute 'PASS' cannot be used more than once [-Wredundant-attribute]
     procedure(subPass), pass, deferred, pass :: passBinding
     !ERROR: Attributes 'PASS' and 'NOPASS' conflict with each other
     procedure(subPassNopass), pass, deferred, nopass :: passNopassBinding  ! C781
diff --git a/flang/test/Semantics/resolve81.f90 b/flang/test/Semantics/resolve81.f90
index db5b19f1155ea..23651cbb257e7 100644
--- a/flang/test/Semantics/resolve81.f90
+++ b/flang/test/Semantics/resolve81.f90
@@ -11,52 +11,52 @@
 !    VOLATILE
 module m
 
-  !WARNING: Attribute 'PUBLIC' cannot be used more than once
+  !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
   real, public, allocatable, public :: publicVar
-  !WARNING: Attribute 'PRIVATE' cannot be used more than once
+  !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
   real, private, allocatable, private :: privateVar
-  !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once
+  !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once [-Wredundant-attribute]
   real, allocatable, allocatable :: allocVar
-  !WARNING: Attribute 'ASYNCHRONOUS' cannot be used more than once
+  !WARNING: Attribute 'ASYNCHRONOUS' cannot be used more than once [-Wredundant-attribute]
   real, asynchronous, public, asynchronous :: asynchVar
   !ERROR: Attribute 'CODIMENSION' cannot be used more than once
   real, codimension[*], codimension[*] :: codimensionVar
-  !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once
+  !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once [-Wredundant-attribute]
   real, contiguous, pointer, contiguous :: contigVar(:)
   !ERROR: Attribute 'DIMENSION' cannot be used more than once
   real, dimension(5), dimension(5) :: arrayVar
-  !WARNING: Attribute 'EXTERNAL' cannot be used more than once
+  !WARNING: Attribute 'EXTERNAL' cannot be used more than once [-Wredundant-attribute]
   real, external, external :: externFunc
-  !WARNING: Attribute 'INTRINSIC' cannot be used more than once
+  !WARNING: Attribute 'INTRINSIC' cannot be used more than once [-Wredundant-attribute]
   !ERROR: 'cos' may not have both the BIND(C) and INTRINSIC attributes
   !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   real, intrinsic, bind(c), intrinsic :: cos
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
   integer, bind(c), volatile, bind(c) :: bindVar
-  !WARNING: Attribute 'PARAMETER' cannot be used more than once
+  !WARNING: Attribute 'PARAMETER' cannot be used more than once [-Wredundant-attribute]
   real, parameter, parameter :: realConst = 4.3
-  !WARNING: Attribute 'POINTER' cannot be used more than once
+  !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
   real, pointer, pointer :: realPtr
-  !WARNING: Attribute 'PROTECTED' cannot be used more than once
+  !WARNING: Attribute 'PROTECTED' cannot be used more than once [-Wredundant-attribute]
   real, protected, protected :: realProt
-  !WARNING: Attribute 'SAVE' cannot be used more than once
+  !WARNING: Attribute 'SAVE' cannot be used more than once [-Wredundant-attribute]
   real, save, save :: saveVar
-  !WARNING: Attribute 'TARGET' cannot be used more than once
+  !WARNING: Attribute 'TARGET' cannot be used more than once [-Wredundant-attribute]
   real, target, target :: targetVar
-  !WARNING: Attribute 'VOLATILE' cannot be used more than once
+  !WARNING: Attribute 'VOLATILE' cannot be used more than once [-Wredundant-attribute]
   real, volatile, volatile :: volatileVar
 
 contains
     subroutine testTypeDecl(arg1, arg2, arg3, arg4, arg5, arg6)
-      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once [-Wredundant-attribute]
       real, intent(in), intent(in) :: arg1
-      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once [-Wredundant-attribute]
       real, intent(out), intent(out) :: arg2
-      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once [-Wredundant-attribute]
       real, intent(inout), intent(inout) :: arg3
-      !WARNING: Attribute 'OPTIONAL' cannot be used more than once
+      !WARNING: Attribute 'OPTIONAL' cannot be used more than once [-Wredundant-attribute]
       integer, optional, intent(in), optional :: arg4
-      !WARNING: Attribute 'VALUE' cannot be used more than once
+      !WARNING: Attribute 'VALUE' cannot be used more than once [-Wredundant-attribute]
       integer, value, intent(in), value :: arg5
       !ERROR: Attributes 'INTENT(IN)' and 'INTENT(INOUT)' conflict with each other
       integer, intent(in), pointer, intent(inout) :: arg6
diff --git a/flang/test/Semantics/resolve82.f90 b/flang/test/Semantics/resolve82.f90
index 989ce1d837c70..c06edd6bb5d98 100644
--- a/flang/test/Semantics/resolve82.f90
+++ b/flang/test/Semantics/resolve82.f90
@@ -14,14 +14,14 @@ real function procFunc()
     end function procFunc
   end interface
 
-  !WARNING: Attribute 'PUBLIC' cannot be used more than once
+  !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
   procedure(procFunc), public, pointer, public :: proc1
-  !WARNING: Attribute 'PRIVATE' cannot be used more than once
+  !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
   procedure(procFunc), private, pointer, private :: proc2
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
   !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   procedure(procFunc), bind(c), pointer, bind(c) :: proc3
-  !WARNING: Attribute 'PROTECTED' cannot be used more than once
+  !WARNING: Attribute 'PROTECTED' cannot be used more than once [-Wredundant-attribute]
   procedure(procFunc), protected, pointer, protected :: proc4
   !ERROR: A PROTECTED entity must be a variable or pointer
   external extsub
@@ -39,21 +39,21 @@ end function procFunc
 contains
 
     subroutine testProcDecl(arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11)
-      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), intent(in), pointer, intent(in) :: arg4
-      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), intent(out), pointer, intent(out) :: arg5
-      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), intent(inout), pointer, intent(inout) :: arg6
       !ERROR: Attributes 'INTENT(INOUT)' and 'INTENT(OUT)' conflict with each other
       procedure(procFunc), intent(inout), pointer, intent(out) :: arg7
       !ERROR: Attributes 'INTENT(INOUT)' and 'INTENT(OUT)' conflict with each other
       procedure(procFunc), intent(out), pointer, intent(inout) :: arg8
-      !WARNING: Attribute 'OPTIONAL' cannot be used more than once
+      !WARNING: Attribute 'OPTIONAL' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), optional, pointer, optional :: arg9
-      !WARNING: Attribute 'POINTER' cannot be used more than once
+      !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), pointer, optional, pointer :: arg10
-      !WARNING: Attribute 'SAVE' cannot be used more than once
+      !WARNING: Attribute 'SAVE' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), save, pointer, save :: localProc
       !ERROR: A PROTECTED entity must be in the specification part of a module
       real x
diff --git a/flang/test/Semantics/resolve83.f90 b/flang/test/Semantics/resolve83.f90
index e9d53dd6bd81d..a60a9980293f9 100644
--- a/flang/test/Semantics/resolve83.f90
+++ b/flang/test/Semantics/resolve83.f90
@@ -3,7 +3,7 @@ module m
 
   ! For C1543
   interface intFace
-    !WARNING: Attribute 'MODULE' cannot be used more than once
+    !WARNING: Attribute 'MODULE' cannot be used more than once [-Wredundant-attribute]
     module pure module real function moduleFunc()
     end function moduleFunc
   end interface
@@ -27,17 +27,17 @@ end function moduleFunc
     real pure real function realFunc()
     end function realFunc
 
-    !WARNING: Attribute 'ELEMENTAL' cannot be used more than once
+    !WARNING: Attribute 'ELEMENTAL' cannot be used more than once [-Wredundant-attribute]
     elemental real elemental function elementalFunc(x)
       real, value :: x
       elementalFunc = x
     end function elementalFunc
 
-    !WARNING: Attribute 'IMPURE' cannot be used more than once
+    !WARNING: Attribute 'IMPURE' cannot be used more than once [-Wredundant-attribute]
     impure real impure function impureFunc()
     end function impureFunc
 
-    !WARNING: Attribute 'PURE' cannot be used more than once
+    !WARNING: Attribute 'PURE' cannot be used more than once [-Wredundant-attribute]
     pure real pure function pureFunc()
     end function pureFunc
 
@@ -45,11 +45,11 @@ end function pureFunc
     impure real pure function impurePureFunc()
     end function impurePureFunc
 
-    !WARNING: Attribute 'RECURSIVE' cannot be used more than once
+    !WARNING: Attribute 'RECURSIVE' cannot be used more than once [-Wredundant-attribute]
     recursive real recursive function recursiveFunc()
     end function recursiveFunc
 
-    !WARNING: Attribute 'NON_RECURSIVE' cannot be used more than once
+    !WARNING: Attribute 'NON_RECURSIVE' cannot be used more than once [-Wredundant-attribute]
     non_recursive real non_recursive function non_recursiveFunc()
     end function non_recursiveFunc
 
diff --git a/flang/test/Semantics/resolve85.f90 b/flang/test/Semantics/resolve85.f90
index 9b9358ecf4770..64b8081dd457b 100644
--- a/flang/test/Semantics/resolve85.f90
+++ b/flang/test/Semantics/resolve85.f90
@@ -7,15 +7,15 @@ module m
 !        TYPE [[, type-attr-spec-list] ::] type-name [( type-param-name-list )]
 !  type-attr-spec values are:
 !    ABSTRACT, PUBLIC, PRIVATE, BIND(C), EXTENDS(parent-type-name)
-  !WARNING: Attribute 'ABSTRACT' cannot be used more than once
+  !WARNING: Attribute 'ABSTRACT' cannot be used more than once [-Wredundant-attribute]
   type, abstract, public, abstract :: derived1
   end type derived1
 
-  !WARNING: Attribute 'PUBLIC' cannot be used more than once
+  !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
   type, public, abstract, public :: derived2
   end type derived2
 
-  !WARNING: Attribute 'PRIVATE' cannot be used more than once
+  !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
   type, private, abstract, private :: derived3
   end type derived3
 
@@ -23,8 +23,8 @@ module m
   type, public, abstract, private :: derived4
   end type derived4
 
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
-  !WARNING: A derived type with the BIND attribute should not be empty
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
+  !WARNING: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, bind(c), public, bind(c) :: derived5
   end type derived5
 
diff --git a/flang/test/Semantics/resolve90.f90 b/flang/test/Semantics/resolve90.f90
index baa108f4d0839..7d2b5e9d333ff 100644
--- a/flang/test/Semantics/resolve90.f90
+++ b/flang/test/Semantics/resolve90.f90
@@ -12,7 +12,7 @@ subroutine s()
     !ERROR: 'pointerallocatablefield' may not have both the POINTER and ALLOCATABLE attributes
     real, pointer, allocatable :: pointerAllocatableField
     real, dimension(:), contiguous, pointer :: goodContigField
-    !PORTABILITY: CONTIGUOUS component 'badcontigfield' should be an array with the POINTER attribute
+    !PORTABILITY: CONTIGUOUS component 'badcontigfield' should be an array with the POINTER attribute [-Wredundant-contiguous]
     real, dimension(:), contiguous, allocatable :: badContigField
     character :: charField * 3
     !ERROR: A length specifier cannot be used to declare the non-character entity 'realfield'
diff --git a/flang/test/Semantics/resolve99.f90 b/flang/test/Semantics/resolve99.f90
index e56022b61bfd8..796c354af1341 100644
--- a/flang/test/Semantics/resolve99.f90
+++ b/flang/test/Semantics/resolve99.f90
@@ -31,7 +31,7 @@ subroutine constructAssoc()
     integer, dimension(4) :: table
     integer :: localVar
     associate (assocVar => localVar)
-      !PORTABILITY: Index variable 'assocvar' should be a scalar object or common block if it is present in the enclosing scope
+      !PORTABILITY: Index variable 'assocvar' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
       FORALL (assocVar=1:4) table(assocVar) = 343
     end associate
   end subroutine constructAssoc
@@ -44,7 +44,7 @@ end subroutine commonSub
 
   subroutine mismatch()
     integer, dimension(4) :: table
-    !PORTABILITY: Index variable 'typename' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'typename' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     !ERROR: Must have INTEGER type, but is REAL(4)
     !ERROR: Must have INTEGER type, but is REAL(4)
     FORALL (typeName=1:4) table(typeName) = 343
diff --git a/flang/test/Semantics/separate-mp02.f90 b/flang/test/Semantics/separate-mp02.f90
index cb1e2687bad73..7d8ea6e0c3932 100644
--- a/flang/test/Semantics/separate-mp02.f90
+++ b/flang/test/Semantics/separate-mp02.f90
@@ -368,7 +368,7 @@ module subroutine s(x)
 end
 submodule(m11) sm11
  contains
-  !WARNING: Dummy procedure 'x' does not exactly match the corresponding argument in the interface body
+  !WARNING: Dummy procedure 'x' does not exactly match the corresponding argument in the interface body [-Wmismatching-dummy-procedure]
   module subroutine s(x)
     call x ! no error
   end
diff --git a/flang/test/Semantics/spec-expr.f90 b/flang/test/Semantics/spec-expr.f90
index 28ebea1109f1d..4d79f2ca188fc 100644
--- a/flang/test/Semantics/spec-expr.f90
+++ b/flang/test/Semantics/spec-expr.f90
@@ -117,7 +117,7 @@ subroutine s7biii(x, y)
   integer :: local = 5
   ! OK, since "localConst" is a constant
   real, dimension(localConst) :: realArray1
-  !PORTABILITY: specification expression refers to local object 'local' (initialized and saved)
+  !PORTABILITY: specification expression refers to local object 'local' (initialized and saved) [-Wsaved-local-in-spec-expr]
   real, dimension(local) :: realArray2
   real, dimension(size(realArray1)) :: realArray3 ! ok
   real, dimension(size(x)) :: realArray4 ! ok
diff --git a/flang/test/Semantics/stmt-func01.f90 b/flang/test/Semantics/stmt-func01.f90
index a87b0d7af52b4..d8ef9af25b389 100644
--- a/flang/test/Semantics/stmt-func01.f90
+++ b/flang/test/Semantics/stmt-func01.f90
@@ -10,14 +10,14 @@ program main
     pure integer function ifunc()
     end function
   end interface
-  !PORTABILITY: Automatic data object 'x1' should not appear in the specification part of a main program
+  !PORTABILITY: Automatic data object 'x1' should not appear in the specification part of a main program [-Wautomatic-in-main-program]
   type(t1(k=4,l=ifunc())) x1
-  !PORTABILITY: Statement function 'sf1' should not contain an array constructor
+  !PORTABILITY: Statement function 'sf1' should not contain an array constructor [-Wstatement-function-extensions]
   sf1(n) = sum([(j,j=1,n)])
   type(t1) sf2
-  !PORTABILITY: Statement function 'sf2' should not contain a structure constructor
+  !PORTABILITY: Statement function 'sf2' should not contain a structure constructor [-Wstatement-function-extensions]
   sf2(n) = t1(n)
-  !PORTABILITY: Statement function 'sf3' should not contain a type parameter inquiry
+  !PORTABILITY: Statement function 'sf3' should not contain a type parameter inquiry [-Wstatement-function-extensions]
   sf3(n) = x1%l
   !ERROR: Recursive call to statement function 'sf4' is not allowed
   sf4(n) = sf4(n)
@@ -26,10 +26,10 @@ pure integer function ifunc()
   real sf7
   !ERROR: Statement function 'sf6' may not reference another statement function 'sf7' that is defined later
   sf6(n) = sf7(n)
-  !PORTABILITY: Statement function 'sf7' should not reference function 'explicit' that requires an explicit interface
+  !PORTABILITY: Statement function 'sf7' should not reference function 'explicit' that requires an explicit interface [-Wstatement-function-extensions]
   sf7(n) = explicit(n)
   real :: a(3) = [1., 2., 3.]
-  !PORTABILITY: Statement function 'sf8' should not pass an array argument that is not a whole array
+  !PORTABILITY: Statement function 'sf8' should not pass an array argument that is not a whole array [-Wstatement-function-extensions]
   sf8(n) = sum(a(1:2))
   sf8a(n) = sum(a) ! ok
   integer :: sf9
@@ -56,7 +56,7 @@ pure function arr()
     arr = [1., 2.]
   end function
   subroutine foo
-    !PORTABILITY: An implicitly typed statement function should not appear when the same symbol is available in its host scope
+    !PORTABILITY: An implicitly typed statement function should not appear when the same symbol is available in its host scope [-Wstatement-function-extensions]
     sf14(x) = 2.*x
   end subroutine
 end
@@ -94,7 +94,7 @@ subroutine s4
 subroutine s5
   !ERROR: Invalid specification expression: reference to impure function 'k'
   real x(k())
-  !WARNING: Name 'k' from host scope should have a type declaration before its local statement function definition
+  !WARNING: Name 'k' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
   !ERROR: 'k' is already declared in this scoping unit
   k() = 0.0
 end
diff --git a/flang/test/Semantics/stmt-func02.f90 b/flang/test/Semantics/stmt-func02.f90
index bfed280ded58d..10166a0abf7b1 100644
--- a/flang/test/Semantics/stmt-func02.f90
+++ b/flang/test/Semantics/stmt-func02.f90
@@ -24,11 +24,11 @@ subroutine test1
     print *, x
   end
   subroutine test2
-    !PORTABILITY: Name 'rf' from host scope should have a type declaration before its local statement function definition
+    !PORTABILITY: Name 'rf' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
     rf(x) = 1.
   end
   subroutine test2b
-    !PORTABILITY: Name 'rf2' from host scope should have a type declaration before its local statement function definition
+    !PORTABILITY: Name 'rf2' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
     rf2(x) = 1.
   end
   subroutine test3
@@ -43,7 +43,7 @@ function f()
     f() = 1. ! statement function of same name as function
   end
   function g() result(r)
-    !WARNING: Name 'g' from host scope should have a type declaration before its local statement function definition
+    !WARNING: Name 'g' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
     !ERROR: 'g' is already declared in this scoping unit
     g() = 1. ! statement function of same name as function
   end
diff --git a/flang/test/Semantics/structconst03.f90 b/flang/test/Semantics/structconst03.f90
index ecd31723b12bb..006f21e5b29f7 100644
--- a/flang/test/Semantics/structconst03.f90
+++ b/flang/test/Semantics/structconst03.f90
@@ -183,7 +183,7 @@ impure real function ipf1(dummy1, dummy2, dummy3, dummy4)
     x1 = t1(0)(usedfrom1)
     x1 = t1(0)(modulevar1)
     x1 = t1(0)(commonvar1)
-    !WARNING: Pointer target is not a definable variable
+    !WARNING: Pointer target is not a definable variable [-Wpointer-to-undefinable]
     !BECAUSE: 'dummy1' is an INTENT(IN) dummy argument
     x1 = t1(0)(dummy1)
     x1 = t1(0)(dummy2)
diff --git a/flang/test/Semantics/structconst04.f90 b/flang/test/Semantics/structconst04.f90
index abddf6001726c..b44f87aeb13f0 100644
--- a/flang/test/Semantics/structconst04.f90
+++ b/flang/test/Semantics/structconst04.f90
@@ -177,7 +177,7 @@ impure real function ipf1(dummy1, dummy2, dummy3, dummy4)
     x1 = t1(usedfrom1)
     x1 = t1(modulevar1)
     x1 = t1(commonvar1)
-    !WARNING: Pointer target is not a definable variable
+    !WARNING: Pointer target is not a definable variable [-Wpointer-to-undefinable]
     !BECAUSE: 'dummy1' is an INTENT(IN) dummy argument
     x1 = t1(dummy1)
     x1 = t1(dummy2)
diff --git a/flang/test/Semantics/structconst08.f90 b/flang/test/Semantics/structconst08.f90
index 149c898dda9d4..78e44a49735d8 100644
--- a/flang/test/Semantics/structconst08.f90
+++ b/flang/test/Semantics/structconst08.f90
@@ -25,7 +25,7 @@ subroutine test1()
 !ERROR: Must be a constant value
     type(parent1) :: tp2 = parent1(j)
     type(parent1) :: tp3 = parent1(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     type(parent1) :: tp4 = parent1(null(ipp))
 
 !ERROR: Must be a constant value
@@ -33,7 +33,7 @@ subroutine test1()
 !ERROR: Must be a constant value
     type(parent2) :: tp6 = parent2(arr)
     type(parent2) :: tp7 = parent2(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     type(parent2) :: tp8 = parent2(null(rpp))
   end subroutine test1
 
@@ -47,20 +47,20 @@ subroutine test2()
     tp1 = parent1(3)
     tp1 = parent1(j)
     tp1 = parent1(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     tp1 = parent1(null(ipp))
 
     tp2 = parent2([1.1,2.1,3.1])
     tp2 = parent2(arr)
     tp2 = parent2(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     tp2 = parent2(null(rpp))
   end subroutine test2
 
   subroutine test3()
     real, pointer :: pp(:)
     type(child) :: tc1 = child(5, parent2(null()))
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     type(child) :: tc10 = child(5, parent2(null(pp)))
 !ERROR: Must be a constant value
     type(child) :: tc3 = child(5, parent2([1.1,1.2]))
diff --git a/flang/test/Semantics/structconst10.f90 b/flang/test/Semantics/structconst10.f90
index 582f8fc15704f..e6a22a9bfac55 100644
--- a/flang/test/Semantics/structconst10.f90
+++ b/flang/test/Semantics/structconst10.f90
@@ -14,12 +14,12 @@ end module m1
 program test
   use m1
   type(a3) v
-  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous
+  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous [-Wanonymous-parents]
   v=a3(a2(x1=18,x2=6),x3=6)
-  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous
+  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous [-Wanonymous-parents]
   v=a3(a1(x1=18),x2=6,x3=6)
-  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous
-  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous
+  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous [-Wanonymous-parents]
+  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous [-Wanonymous-parents]
   v=a3(a2(a1(x1=18),x2=6),x3=6)
   v=a3(a2=a2(a1=a1(x1=18),x2=6),x3=6) ! ok
 end
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index 97dc50a23845f..df10942e6af2d 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -43,6 +43,9 @@ subroutine iface
  !DEF: /m/pp6 EXTERNAL, POINTER, PUBLIC (Subroutine) ProcEntity
  !DEF: /m/modproc1 PUBLIC (Subroutine) Subprogram
  procedure(iface), pointer :: pp6 => modproc1
+ !DEF: /m/xx PUBLIC, TARGET ObjectEntity REAL(4)
+ !DEF: /m/yy PUBLIC, TARGET ObjectEntity REAL(4)
+ real, target :: xx, yy(2)
  !DEF: /m/t1 PUBLIC DerivedType
  type :: t1
   !DEF: /m/t1/opc1 POINTER ObjectEntity REAL(4)
@@ -51,11 +54,11 @@ subroutine iface
   !REF: /m/null
   real, pointer :: opc2 => null()
   !DEF: /m/t1/opc3 POINTER ObjectEntity REAL(4)
-  !REF: /m/x
-  real, pointer :: opc3 => x
+  !REF: /m/xx
+  real, pointer :: opc3 => xx
   !DEF: /m/t1/opc4 POINTER ObjectEntity REAL(4)
-  !REF: /m/y
-  real, pointer :: opc4 => y(1)
+  !REF: /m/yy
+  real, pointer :: opc4 => yy(1)
   !REF: /m/iface
   !DEF: /m/t1/ppc1 NOPASS, POINTER (Subroutine) ProcEntity
   procedure(iface), nopass, pointer :: ppc1
@@ -101,12 +104,12 @@ subroutine iface
   !REF: /m/null
   real, pointer :: opc2 => null()
   !DEF: /m/pdt1/opc3 POINTER ObjectEntity REAL(4)
-  !REF: /m/x
-  real, pointer :: opc3 => x
+  !REF: /m/xx
+  real, pointer :: opc3 => xx
   !DEF: /m/pdt1/opc4 POINTER ObjectEntity REAL(4)
-  !REF: /m/y
+  !REF: /m/yy
   !REF: /m/pdt1/k
-  real, pointer :: opc4 => y(k)
+  real, pointer :: opc4 => yy(k)
   !REF: /m/iface
   !DEF: /m/pdt1/ppc1 NOPASS, POINTER (Subroutine) ProcEntity
   procedure(iface), nopass, pointer :: ppc1
diff --git a/flang/test/Semantics/transfer01.f90 b/flang/test/Semantics/transfer01.f90
index 26f4f1b3eb62f..d58cf1a4408e7 100644
--- a/flang/test/Semantics/transfer01.f90
+++ b/flang/test/Semantics/transfer01.f90
@@ -20,17 +20,17 @@ subroutine subr(o)
   print *, transfer(1., empty2) ! ok
   !ERROR: Element size of MOLD= array may not be zero when SOURCE= is not empty
   print *, transfer(1., empty3)
-  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty
+  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty [-Wvoid-mold]
   print *, transfer(source, empty1)
   print *, transfer(source, empty2) ! ok
-  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty
+  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty [-Wvoid-mold]
   print *, transfer(source, empty3)
   !ERROR: SIZE= argument may not be the optional dummy argument 'o'
   print *, transfer(1., empty2, size=o)
-  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning
+  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning [-Wtransfer-size-presence]
   print *, transfer(1., empty2, size=ia)
-  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning
+  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning [-Wtransfer-size-presence]
   print *, transfer(1., empty2, size=ip)
-  !WARNING: Source of TRANSFER contains allocatable or pointer component %allocatable
+  !WARNING: Source of TRANSFER contains allocatable or pointer component %allocatable [-Wpointer-component-transfer-arg]
   print *, transfer(hasDesc, 1)
 end
diff --git a/flang/test/Semantics/undef-result01.f90 b/flang/test/Semantics/undef-result01.f90
index e1ae58dae7c0a..85cd181cd4413 100644
--- a/flang/test/Semantics/undef-result01.f90
+++ b/flang/test/Semantics/undef-result01.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 function basic()
 end
 
@@ -29,7 +29,7 @@ subroutine intentInPtr(p)
   end
 end
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 function notDefdByCall()
   call intentin(notDefdByCall)
  contains
@@ -38,7 +38,7 @@ subroutine intentin(n)
   end
 end
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 function basicAlloc()
   real, allocatable :: basicAlloc
   allocate(basicAlloc)
@@ -134,7 +134,7 @@ integer function defdBySize()
   inquire(6,status=defdByInquire)
 end
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 character(20) function notDefdByInquire()
   inquire(file=notDefdByInquire)
 end
diff --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
index 705fb1b816895..799a34e6b89c5 100644
--- a/flang/test/Semantics/windows.f90
+++ b/flang/test/Semantics/windows.f90
@@ -1,4 +1,4 @@
-! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if x86-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
 ! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
 
 subroutine uid
diff --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 0000000000000..6b895b732c42b
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable
diff --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 0000000000000..15dcf2389b21d
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>
diff --git a/flang/test/Transforms/tbaa-for-local-vars.fir b/flang/test/Transforms/tbaa-for-local-vars.fir
new file mode 100644
index 0000000000000..82058ffef290a
--- /dev/null
+++ b/flang/test/Transforms/tbaa-for-local-vars.fir
@@ -0,0 +1,92 @@
+// Test TBAA tags assignment for access off the fir.declare
+// placed in the middle of the routine (e.g. a temporary).
+// Original example:
+// module m
+//   type t
+//      real :: x
+//   end type t
+// contains
+//   subroutine bar(this)
+//     class(t), intent(out) :: this
+//     this%x = 1.0
+//   end subroutine bar
+//   function foo() result(res)
+//     type(t) :: res
+//     call bar(res)
+//   end function foo
+//   subroutine test(arg)
+//     type(t) :: var
+//     var = foo()
+//     arg = var%x
+//   end subroutine test
+// end module m
+//
+// The calls were manually inlined in FIR with fir.save_result's
+// destination operand being used instead of the temporary
+// alloca of the result inside foo. Runtime calls were removed
+// to reduce the test.
+// The temporary function result fir.declare is then dominated
+// by fir.dummy_scope of the foo function. If we use this scope
+// to assign the TBAA local-alloc tag to the accesses of
+// the temporary, then it won't conflict with the TBAA dummy tag
+// assigned to the accesses of this argument of bar.
+// That would be incorrect, because they access the same memory.
+// Check that the local-alloc tag is placed into the outermost
+// scope's TBAA tree.
+// RUN: fir-opt --fir-add-alias-tags %s | FileCheck %s
+
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QMmPtest - Scope 2">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QMmPtest">
+// CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
+// CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_1]], 0>}>
+// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_2]], 0>}>
+// CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_3]], 0>}>
+// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_4]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$ATTR_9:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmFbarEthis", members = {<#[[$ATTR_6]], 0>}>
+// CHECK: #[[$ATTR_10:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[$ATTR_7]], 0>}>
+// CHECK: #[[$ATTR_12:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_9]], access_type = #[[$ATTR_9]], offset = 0>
+// CHECK: #[[$ATTR_13:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_10]], access_type = #[[$ATTR_10]], offset = 0>
+
+// CHECK-LABEL:   func.func @_QMmPtest(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<f32> {fir.bindc_name = "arg"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTt{x:f32}> {bindc_name = ".result"}
+// CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_3:.*]] = fir.declare %[[ARG0]] dummy_scope %[[VAL_2]] {uniq_name = "_QMmFtestEarg"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_1]] {uniq_name = "_QMmFfooEres"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_8:.*]] = fir.embox %[[VAL_7]] : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.box<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.type<_QMmTt{x:f32}>>) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_10:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_11:.*]] = fir.declare %[[VAL_9]] dummy_scope %[[VAL_10]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QMmFbarEthis"} : (!fir.class<!fir.type<_QMmTt{x:f32}>>, !fir.dscope) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_12:.*]] = fir.coordinate_of %[[VAL_11]], x : (!fir.class<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+// CHECK:           fir.store %[[VAL_0]] to %[[VAL_12]] {tbaa = [#[[$ATTR_12]]]} : !fir.ref<f32>
+// CHECK:           %[[VAL_13:.*]] = fir.declare %[[VAL_1]] {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_14:.*]] = fir.coordinate_of %[[VAL_13]], x : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_14]] {tbaa = [#[[$ATTR_13]]]} : !fir.ref<f32>
+func.func @_QMmPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "arg"}) {
+  %cst = arith.constant 1.000000e+00 : f32
+  %0 = fir.alloca !fir.type<_QMmTt{x:f32}> {bindc_name = ".result"}
+  %1 = fir.dummy_scope : !fir.dscope
+  %2 = fir.declare %arg0 dummy_scope %1 {uniq_name = "_QMmFtestEarg"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+  %3 = fir.alloca !fir.type<_QMmTt{x:f32}> {bindc_name = "var", uniq_name = "_QMmFtestEvar"}
+  %4 = fir.declare %3 {uniq_name = "_QMmFtestEvar"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+  %5 = fir.dummy_scope : !fir.dscope
+  %6 = fir.declare %0 {uniq_name = "_QMmFfooEres"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+  %7 = fir.embox %6 : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.box<!fir.type<_QMmTt{x:f32}>>
+  %8 = fir.convert %7 : (!fir.box<!fir.type<_QMmTt{x:f32}>>) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+  %9 = fir.dummy_scope : !fir.dscope
+  %10 = fir.declare %8 dummy_scope %9 {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QMmFbarEthis"} : (!fir.class<!fir.type<_QMmTt{x:f32}>>, !fir.dscope) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+  %14 = fir.coordinate_of %10, x : (!fir.class<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+  fir.store %cst to %14 : !fir.ref<f32>
+  %15 = fir.declare %0 {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+  %16 = fir.coordinate_of %15, x : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+  %17 = fir.coordinate_of %4, x : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+  %18 = fir.load %16 : !fir.ref<f32>
+  fir.store %18 to %17 : !fir.ref<f32>
+  %19 = fir.load %17 : !fir.ref<f32>
+  fir.store %19 to %2 : !fir.ref<f32>
+  return
+}
+
diff --git a/flang/test/Transforms/tbaa-with-dummy-scope2.fir b/flang/test/Transforms/tbaa-with-dummy-scope2.fir
index 249471de458d3..fd711a4d70eb4 100644
--- a/flang/test/Transforms/tbaa-with-dummy-scope2.fir
+++ b/flang/test/Transforms/tbaa-with-dummy-scope2.fir
@@ -84,18 +84,17 @@ func.func @_QPtest2() attributes {noinline} {
   fir.store %c2_i32 to %2 : !fir.ref<i32>
   return
 }
-// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2 - Scope 1">
-// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2">
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2 - Scope 1">
 // CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_1]], 0>}>
 // CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_2]], 0>}>
 // CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_3]], 0>}>
 // CHECK: #[[$TARGETDATA_0:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_4]], 0>}>
-// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_4]], 0>}>
-// CHECK: #[[$TARGETDATA_1:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_5]], 0>}>
 // CHECK: #[[$LOCAL_ATTR_0:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[$TARGETDATA_0]], 0>}>
 // CHECK: #[[$ATTR_8:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtest2FinnerEx", members = {<#[[$ATTR_6]], 0>}>
-// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$TARGETDATA_1]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$TARGETDATA_0]], 0>}>
 // CHECK: #[[$ATTR_10:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_8]], access_type = #[[$ATTR_8]], offset = 0>
 // CHECK: #[[$LOCAL_ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QFtest2FinnerEy", members = {<#[[$LOCAL_ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_9:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmEglob", members = {<#[[$ATTR_7]], 0>}>
diff --git a/flang/test/lib/OpenACC/CMakeLists.txt b/flang/test/lib/OpenACC/CMakeLists.txt
index 1c0ac748f85e8..d8d0109ed5848 100644
--- a/flang/test/lib/OpenACC/CMakeLists.txt
+++ b/flang/test/lib/OpenACC/CMakeLists.txt
@@ -2,22 +2,26 @@ add_flang_library(FIRTestOpenACCInterfaces
   TestOpenACCInterfaces.cpp
 
   DEPENDS
+  HLFIRDialect
   FIRDialect
   FIROpenACCSupport
   FIRSupport
 
   LINK_LIBS
+  HLFIRDialect
   FIRDialect
   FIROpenACCSupport
   FIRSupport
 
   MLIR_DEPS
+  MLIRDLTIDialect
   MLIRIR
   MLIROpenACCDialect
   MLIRPass
   MLIRSupport
 
   MLIR_LIBS
+  MLIRDLTIDialect
   MLIRIR
   MLIROpenACCDialect
   MLIRPass
diff --git a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
index 90aabd7d40d44..e72b96fe7cd10 100644
--- a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
+++ b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/Support/DataLayout.h"
 
 using namespace mlir;
@@ -25,6 +29,11 @@ struct TestFIROpenACCInterfaces
   StringRef getDescription() const final {
     return "Test FIR implementation of the OpenACC interfaces.";
   }
+  void getDependentDialects(::mlir::DialectRegistry &registry) const override {
+    registry.insert<fir::FIROpsDialect, hlfir::hlfirDialect,
+        mlir::arith::ArithDialect, mlir::acc::OpenACCDialect,
+        mlir::DLTIDialect>();
+  }
   void runOnOperation() override {
     mlir::ModuleOp mod = getOperation();
     auto datalayout =
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 015c86604a1fd..d216069654bc4 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -223,6 +223,11 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
+static llvm::cl::opt<bool>
+    enableDoConcurrentOffload("fdoconcurrent-offload",
+                              llvm::cl::desc("enable do concurrent offload"),
+                              llvm::cl::init(false));
+
 static llvm::cl::opt<bool>
     disableCUDAWarpFunction("fcuda-disable-warp-function",
                             llvm::cl::desc("Disable CUDA Warp Function"),
@@ -434,6 +439,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setStackRepackArrays(stackRepackArrays);
   loweringOptions.setRepackArrays(repackArrays);
   loweringOptions.setRepackArraysWhole(repackArraysWhole);
+  loweringOptions.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
   if (enableCUDA)
     loweringOptions.setCUDARuntimeCheck(true);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
@@ -607,6 +613,11 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
+  if (enableDoConcurrentOffload) {
+    options.features.Enable(
+        Fortran::common::LanguageFeature::DoConcurrentOffload);
+  }
+
   if (disableCUDAWarpFunction) {
     options.features.Enable(
         Fortran::common::LanguageFeature::CudaWarpMatchFunction, false);
diff --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index eaf4bae088454..806cd3148d09f 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -70,6 +70,11 @@ static cl::opt<bool> codeGenLLVM(
     cl::desc("Run only CodeGen passes and translate FIR to LLVM IR"),
     cl::init(false));
 
+static cl::opt<bool> emitFinalMLIR(
+    "emit-final-mlir",
+    cl::desc("Only translate FIR to MLIR, do not lower to LLVM IR"),
+    cl::init(false));
+
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
 
@@ -149,13 +154,15 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
       fir::registerDefaultInlinerPass(config);
       fir::createMLIRToLLVMPassPipeline(pm, config);
     }
-    fir::addLLVMDialectToLLVMPass(pm, out.os());
+    if (!emitFinalMLIR)
+      fir::addLLVMDialectToLLVMPass(pm, out.os());
   }
 
   // run the pass manager
   if (mlir::succeeded(pm.run(*owningRef))) {
     // passes ran successfully, so keep the output
-    if ((emitFir || passPipeline.hasAnyOccurrences()) && !codeGenLLVM)
+    if ((emitFir || passPipeline.hasAnyOccurrences() || emitFinalMLIR) &&
+        !codeGenLLVM)
       printModule(*owningRef, out.os());
     out.keep();
     return mlir::success();
diff --git a/flang/unittests/Common/FortranFeaturesTest.cpp b/flang/unittests/Common/FortranFeaturesTest.cpp
index afd677f69a165..9408da0361e1d 100644
--- a/flang/unittests/Common/FortranFeaturesTest.cpp
+++ b/flang/unittests/Common/FortranFeaturesTest.cpp
@@ -558,4 +558,12 @@ TEST(FortranFeaturesTest, CamelCaseToLowerCaseHyphenated) {
       "non-volatile-pointer-to-volatile");
 }
 
+TEST(FortranFeaturesTest, HintLanguageControlFlag) {
+  LanguageFeatureControl control{};
+  EXPECT_EQ(control.getDefaultCliSpelling(LanguageFeature::BenignNameClash),
+      "benign-name-clash");
+  EXPECT_EQ(
+      control.getDefaultCliSpelling(UsageWarning::Portability), "portability");
+}
+
 } // namespace Fortran::common::details
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 1161ae260be2e..fa62f4d290c3a 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -201,6 +201,8 @@ if(LLVM_LIBC_FULL_BUILD)
     # setjmp.h entrypoints
     libc.src.setjmp.longjmp
     libc.src.setjmp.setjmp
+    libc.src.setjmp.siglongjmp
+    libc.src.setjmp.sigsetjmp
   )
 endif()
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 7a954a480e698..59c248871f83a 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -376,6 +376,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wcspbrk
     libc.src.wchar.wcsrchr
     libc.src.wchar.wcsspn
+    libc.src.wchar.wcscspn
     libc.src.wchar.wmemcmp
     libc.src.wchar.wmempcpy
     libc.src.wchar.wmemcpy
@@ -387,6 +388,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wmemchr
     libc.src.wchar.wcpcpy
     libc.src.wchar.wcpncpy
+    libc.src.wchar.wcstok
 
     # sys/uio.h entrypoints
     libc.src.sys.uio.writev
diff --git a/libc/include/llvm-libc-types/jmp_buf.h b/libc/include/llvm-libc-types/jmp_buf.h
index 2fec35dbc9163..8a7d2839e21d4 100644
--- a/libc/include/llvm-libc-types/jmp_buf.h
+++ b/libc/include/llvm-libc-types/jmp_buf.h
@@ -13,7 +13,7 @@
 // Issue: https://github.com/llvm/llvm-project/issues/136358
 #if defined(__linux__)
 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) ||        \
-    defined(__riscv)
+    defined(__arm__) || defined(__riscv)
 #define __LIBC_HAS_SIGJMP_BUF
 #endif
 #endif
diff --git a/libc/include/wchar.yaml b/libc/include/wchar.yaml
index 9e862ff984494..02e1ba331b7cc 100644
--- a/libc/include/wchar.yaml
+++ b/libc/include/wchar.yaml
@@ -104,6 +104,13 @@ functions:
     arguments:
       - type: const wchar_t *
       - type: const wchar_t *
+  - name: wcscspn
+    standards:
+      - stdc
+    return_type: size_t
+    arguments:
+      - type: const wchar_t *
+      - type: const wchar_t *
   - name: wmemcmp
     standards:
       - stdc
@@ -196,6 +203,14 @@ functions:
     arguments:
       - type: wchar_t *__restrict
       - type: const wchar_t *__restrict
+  - name: wcstok
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
+      - type: wchar_t** __restrict
   - name: wcpcpy
     standards:
       - stdc
diff --git a/libc/src/__support/CPP/new.h b/libc/src/__support/CPP/new.h
index 8694d9c475507..fe36de29468a8 100644
--- a/libc/src/__support/CPP/new.h
+++ b/libc/src/__support/CPP/new.h
@@ -29,6 +29,14 @@ enum class align_val_t : size_t {};
 
 namespace LIBC_NAMESPACE_DECL {
 
+namespace cpp {
+template <class T> [[nodiscard]] constexpr T *launder(T *p) {
+  static_assert(__has_builtin(__builtin_launder),
+                "cpp::launder requires __builtin_launder");
+  return __builtin_launder(p);
+}
+} // namespace cpp
+
 class AllocChecker {
   bool success = false;
 
diff --git a/libc/src/__support/GPU/allocator.cpp b/libc/src/__support/GPU/allocator.cpp
index 66ab155e5c299..7923fbb2c1c24 100644
--- a/libc/src/__support/GPU/allocator.cpp
+++ b/libc/src/__support/GPU/allocator.cpp
@@ -22,6 +22,7 @@
 #include "src/__support/GPU/utils.h"
 #include "src/__support/RPC/rpc_client.h"
 #include "src/__support/threads/sleep.h"
+#include "src/string/memory_utils/inline_memcpy.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -137,6 +138,11 @@ void uniform_memset(uint32_t *s, uint32_t c, uint32_t n, uint64_t uniform) {
     s[i] = c;
 }
 
+// Indicates that the provided value is a power of two.
+static inline constexpr bool is_pow2(uint64_t x) {
+  return x && (x & (x - 1)) == 0;
+}
+
 } // namespace impl
 
 /// A slab allocator used to hand out identically sized slabs of memory.
@@ -183,7 +189,9 @@ struct Slab {
 
   // Get the number of bytes needed to contain the bitfield bits.
   constexpr static uint32_t bitfield_bytes(uint32_t chunk_size) {
-    return ((num_chunks(chunk_size) + BITS_IN_WORD - 1) / BITS_IN_WORD) * 8;
+    return __builtin_align_up(
+        ((num_chunks(chunk_size) + BITS_IN_WORD - 1) / BITS_IN_WORD) * 8,
+        MIN_ALIGNMENT + 1);
   }
 
   // The actual amount of memory available excluding the bitfield and metadata.
@@ -540,15 +548,58 @@ void deallocate(void *ptr) {
     return;
 
   // All non-slab allocations will be aligned on a 2MiB boundary.
-  if ((reinterpret_cast<uintptr_t>(ptr) & SLAB_ALIGNMENT) == 0)
+  if (__builtin_is_aligned(ptr, SLAB_ALIGNMENT + 1))
     return impl::rpc_free(ptr);
 
   // The original slab pointer is the 2MiB boundary using the given pointer.
-  Slab *slab = reinterpret_cast<Slab *>(
-      (reinterpret_cast<uintptr_t>(ptr) & ~SLAB_ALIGNMENT));
+  Slab *slab = cpp::launder(reinterpret_cast<Slab *>(
+      (reinterpret_cast<uintptr_t>(ptr) & ~SLAB_ALIGNMENT)));
   slab->deallocate(ptr);
   release_slab(slab);
 }
 
+void *reallocate(void *ptr, uint64_t size) {
+  if (ptr == nullptr)
+    return gpu::allocate(size);
+
+  // Non-slab allocations are considered foreign pointers so we fail.
+  if (__builtin_is_aligned(ptr, SLAB_ALIGNMENT + 1))
+    return nullptr;
+
+  // The original slab pointer is the 2MiB boundary using the given pointer.
+  Slab *slab = cpp::launder(reinterpret_cast<Slab *>(
+      (reinterpret_cast<uintptr_t>(ptr) & ~SLAB_ALIGNMENT)));
+  if (slab->get_chunk_size() >= size)
+    return ptr;
+
+  // If we need a new chunk we reallocate and copy it over.
+  void *new_ptr = gpu::allocate(size);
+  inline_memcpy(new_ptr, ptr, slab->get_chunk_size());
+  gpu::deallocate(ptr);
+  return new_ptr;
+}
+
+void *aligned_allocate(uint32_t alignment, uint64_t size) {
+  // All alignment values must be a non-zero power of two.
+  if (!impl::is_pow2(alignment))
+    return nullptr;
+
+  // If the requested alignment is less than what we already provide this is
+  // just a normal allocation.
+  if (alignment <= MIN_ALIGNMENT + 1)
+    return gpu::allocate(size);
+
+  // We can't handle alignments greater than 2MiB so we simply fail.
+  if (alignment > SLAB_ALIGNMENT + 1)
+    return nullptr;
+
+  // Trying to handle allocation internally would break the assumption that each
+  // chunk is identical to eachother. Allocate enough memory with worst-case
+  // alignment and then round up. The index logic will round down properly.
+  uint64_t rounded = size + alignment - MIN_ALIGNMENT;
+  void *ptr = gpu::allocate(rounded);
+  return __builtin_align_up(ptr, alignment);
+}
+
 } // namespace gpu
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/GPU/allocator.h b/libc/src/__support/GPU/allocator.h
index 466009aa71981..a7cf8bceef272 100644
--- a/libc/src/__support/GPU/allocator.h
+++ b/libc/src/__support/GPU/allocator.h
@@ -17,6 +17,8 @@ namespace gpu {
 
 void *allocate(uint64_t size);
 void deallocate(void *ptr);
+void *reallocate(void *ptr, uint64_t size);
+void *aligned_allocate(uint32_t alignment, uint64_t size);
 
 } // namespace gpu
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index 86a47319f278a..d3fb58ed0c71c 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -13,6 +13,7 @@ add_object_library(
   SRCS 
     character_converter.cpp
   DEPENDS
+    libc.hdr.errno_macros
     libc.hdr.types.char8_t
     libc.hdr.types.char32_t
     libc.src.__support.error_or
diff --git a/libc/src/__support/wchar/character_converter.cpp b/libc/src/__support/wchar/character_converter.cpp
index c54a1b751f402..3cacfa5689e4d 100644
--- a/libc/src/__support/wchar/character_converter.cpp
+++ b/libc/src/__support/wchar/character_converter.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/errno_macros.h"
 #include "hdr/types/char32_t.h"
 #include "hdr/types/char8_t.h"
 #include "src/__support/CPP/bit.h"
@@ -76,7 +77,7 @@ int CharacterConverter::push(char8_t utf8_byte) {
     else {
       // bytes_stored and total_bytes will always be 0 here
       state->partial = static_cast<char32_t>(0);
-      return -1;
+      return EILSEQ;
     }
     state->partial = static_cast<char32_t>(utf8_byte);
     state->bytes_stored++;
@@ -93,7 +94,7 @@ int CharacterConverter::push(char8_t utf8_byte) {
   }
   // Invalid byte -> reset the state
   clear();
-  return -1;
+  return EILSEQ;
 }
 
 int CharacterConverter::push(char32_t utf32) {
@@ -115,7 +116,7 @@ int CharacterConverter::push(char32_t utf32) {
   // `utf32` contains a value that is too large to actually represent a valid
   // unicode character
   clear();
-  return -1;
+  return EILSEQ;
 }
 
 ErrorOr<char32_t> CharacterConverter::pop_utf32() {
diff --git a/libc/src/__support/wchar/mbrtowc.cpp b/libc/src/__support/wchar/mbrtowc.cpp
index 3b8f7666026c3..90ba934c42b69 100644
--- a/libc/src/__support/wchar/mbrtowc.cpp
+++ b/libc/src/__support/wchar/mbrtowc.cpp
@@ -32,8 +32,8 @@ ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict s,
   for (; i < n && !char_conv.isFull(); ++i) {
     int err = char_conv.push(static_cast<char8_t>(s[i]));
     // Encoding error
-    if (err == -1)
-      return Error(EILSEQ);
+    if (err == EILSEQ)
+      return Error(err);
   }
   auto wc = char_conv.pop_utf32();
   if (wc.has_value()) {
diff --git a/libc/src/__support/wchar/wcrtomb.cpp b/libc/src/__support/wchar/wcrtomb.cpp
index a74a6f3ec34a6..fc54bbfc93e0c 100644
--- a/libc/src/__support/wchar/wcrtomb.cpp
+++ b/libc/src/__support/wchar/wcrtomb.cpp
@@ -30,12 +30,9 @@ ErrorOr<size_t> wcrtomb(char *__restrict s, wchar_t wc,
   if (!cr.isValidState())
     return Error(EINVAL);
 
-  if (s == nullptr)
-    return Error(EILSEQ);
-
   int status = cr.push(static_cast<char32_t>(wc));
   if (status != 0)
-    return Error(EILSEQ);
+    return Error(status);
 
   size_t count = 0;
   while (!cr.isEmpty()) {
diff --git a/libc/src/setjmp/arm/CMakeLists.txt b/libc/src/setjmp/arm/CMakeLists.txt
index 55c80b0bede0d..77f8471c06bb8 100644
--- a/libc/src/setjmp/arm/CMakeLists.txt
+++ b/libc/src/setjmp/arm/CMakeLists.txt
@@ -8,6 +8,22 @@ add_entrypoint_object(
     libc.hdr.types.jmp_buf
 )
 
+if (TARGET libc.src.setjmp.sigsetjmp_epilogue)
+  add_entrypoint_object(
+    sigsetjmp
+    SRCS
+      sigsetjmp.cpp
+    HDRS
+      ../sigsetjmp.h
+    DEPENDS
+      libc.hdr.types.jmp_buf
+      libc.hdr.types.sigset_t
+      libc.hdr.offsetof_macros
+      libc.src.setjmp.sigsetjmp_epilogue
+      libc.src.setjmp.setjmp
+  )
+endif()
+
 add_entrypoint_object(
   longjmp
   SRCS
diff --git a/libc/src/setjmp/arm/sigsetjmp.cpp b/libc/src/setjmp/arm/sigsetjmp.cpp
new file mode 100644
index 0000000000000..41c86bda5d6c6
--- /dev/null
+++ b/libc/src/setjmp/arm/sigsetjmp.cpp
@@ -0,0 +1,65 @@
+//===-- Implementation of sigsetjmp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/setjmp/sigsetjmp.h"
+#include "hdr/offsetof_macros.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/setjmp/setjmp_impl.h"
+#include "src/setjmp/sigsetjmp_epilogue.h"
+
+namespace LIBC_NAMESPACE_DECL {
+[[gnu::naked]]
+LLVM_LIBC_FUNCTION(int, sigsetjmp, (sigjmp_buf buf)) {
+#if defined(__thumb__) && __ARM_ARCH_ISA_THUMB == 1
+  // Thumb1 does not support the high registers > r7 in stmia, so move them
+  // into lower GPRs first.
+  asm(R"(
+      tst r1, r1
+      bne .Ldosave
+      b %c[setjmp]
+.Ldosave:
+      str r4, [r0, #%c[extra]]
+      mov r4, lr
+      str r4, [r0, #%c[retaddr]]
+      mov r4, r0
+      bl %c[setjmp]
+      mov r1, r0
+      mov r0, r4
+      ldr r4, [r0, #%c[retaddr]]
+      mov lr, r4
+      ldr r4, [r0, #%c[extra]]
+      b %c[epilogue]
+  )" ::[retaddr] "i"(offsetof(__jmp_buf, sig_retaddr)),
+      [extra] "i"(offsetof(__jmp_buf, sig_extra)), [setjmp] "i"(setjmp),
+      [epilogue] "i"(sigsetjmp_epilogue)
+      : "r0", "r1", "r4");
+#else
+  // Some thumb2 linkers do not support conditional branch to PLT.
+  // We branch to local labels instead.
+  asm(R"(
+      tst r1, r1
+      bne .Ldosave
+      b %c[setjmp]
+.Ldosave:
+      str r4, [r0, #%c[extra]]
+      str lr, [r0, #%c[retaddr]]
+      mov r4, r0
+      bl %c[setjmp]
+      mov r1, r0
+      mov r0, r4
+      ldr lr, [r0, #%c[retaddr]]
+      ldr r4, [r0, #%c[extra]]
+      b %c[epilogue]
+  )" ::[retaddr] "i"(offsetof(__jmp_buf, sig_retaddr)),
+      [extra] "i"(offsetof(__jmp_buf, sig_extra)), [setjmp] "X"(setjmp),
+      [epilogue] "X"(sigsetjmp_epilogue)
+      : "r0", "r1", "r4");
+#endif
+}
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/gpu/aligned_alloc.cpp b/libc/src/stdlib/gpu/aligned_alloc.cpp
index cd2c7e55128fe..34a7eae618fec 100644
--- a/libc/src/stdlib/gpu/aligned_alloc.cpp
+++ b/libc/src/stdlib/gpu/aligned_alloc.cpp
@@ -15,15 +15,15 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, aligned_alloc, (size_t alignment, size_t size)) {
-  if ((alignment & -alignment) != alignment)
-    return nullptr;
-
-  void *ptr = gpu::allocate(size);
-  if ((reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)) != 0) {
-    gpu::deallocate(ptr);
-    return nullptr;
-  }
-  return ptr;
+  // FIXME: NVIDIA targets currently use the built-in 'malloc' which we cannot
+  // reason with. But we still need to provide this function for compatibility.
+#ifndef LIBC_TARGET_ARCH_IS_NVPTX
+  return gpu::aligned_allocate(static_cast<uint32_t>(alignment), size);
+#else
+  (void)alignment;
+  (void)size;
+  return nullptr;
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/gpu/realloc.cpp b/libc/src/stdlib/gpu/realloc.cpp
index 4fd4d6b278179..97ad1b3bbeff2 100644
--- a/libc/src/stdlib/gpu/realloc.cpp
+++ b/libc/src/stdlib/gpu/realloc.cpp
@@ -16,17 +16,15 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, realloc, (void *ptr, size_t size)) {
-  if (ptr == nullptr)
-    return gpu::allocate(size);
-
-  void *newmem = gpu::allocate(size);
-  if (newmem == nullptr)
-    return nullptr;
-
-  // This will copy garbage if it goes beyond the old allocation size.
-  inline_memcpy(newmem, ptr, size);
-  gpu::deallocate(ptr);
-  return newmem;
+  // FIXME: NVIDIA targets currently use the built-in 'malloc' which we cannot
+  // reason with. But we still need to provide this function for compatibility.
+#ifndef LIBC_TARGET_ARCH_IS_NVPTX
+  return gpu::reallocate(ptr, size);
+#else
+  (void)ptr;
+  (void)size;
+  return nullptr;
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/CMakeLists.txt b/libc/src/wchar/CMakeLists.txt
index 867aab6755bf6..f2f4b1d38f0f3 100644
--- a/libc/src/wchar/CMakeLists.txt
+++ b/libc/src/wchar/CMakeLists.txt
@@ -45,6 +45,16 @@ add_entrypoint_object(
     libc.src.__support.wctype_utils
 )
 
+add_entrypoint_object(
+  wcstok
+  SRCS
+    wcstok.cpp
+  HDRS
+    wcstok.h
+  DEPENDS
+    libc.hdr.types.wchar_t
+)
+
 add_entrypoint_object(
   wcrtomb
   SRCS
@@ -203,6 +213,17 @@ add_entrypoint_object(
     libc.hdr.types.size_t
 )
 
+add_entrypoint_object(
+  wcscspn
+  SRCS
+    wcscspn.cpp
+  HDRS
+    wcscspn.h
+  DEPENDS
+    libc.hdr.wchar_macros
+    libc.hdr.types.size_t
+)
+
 add_entrypoint_object(
   wmemcmp
   SRCS
diff --git a/libc/src/wchar/wcpncpy.h b/libc/src/wchar/wcpncpy.h
index d817239d2d0b1..1f67cd46f7753 100644
--- a/libc/src/wchar/wcpncpy.h
+++ b/libc/src/wchar/wcpncpy.h
@@ -20,4 +20,4 @@ wchar_t *wcpncpy(wchar_t *__restrict ws1, const wchar_t *__restrict ws2,
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_WCHAR_WCPNsCPY_H
+#endif // LLVM_LIBC_SRC_WCHAR_WCPNCPY_H
diff --git a/libc/src/wchar/wcscspn.cpp b/libc/src/wchar/wcscspn.cpp
new file mode 100644
index 0000000000000..8869d84cdfdee
--- /dev/null
+++ b/libc/src/wchar/wcscspn.cpp
@@ -0,0 +1,34 @@
+//===-- Implementation of wcscspn -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcscspn.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bool check(wchar_t c, const wchar_t *s2) {
+  for (int n = 0; s2[n]; ++n) {
+    if (s2[n] == c)
+      return false;
+  }
+  return true;
+}
+LLVM_LIBC_FUNCTION(size_t, wcscspn, (const wchar_t *s1, const wchar_t *s2)) {
+  size_t i = 0;
+  for (; s1[i]; ++i) {
+    if (!check(s1[i], s2))
+      return i;
+  }
+  return i;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcscspn.h b/libc/src/wchar/wcscspn.h
new file mode 100644
index 0000000000000..cffc2b98c3467
--- /dev/null
+++ b/libc/src/wchar/wcscspn.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcscspn ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSCSPN_H
+#define LLVM_LIBC_SRC_WCHAR_WCSCSPN_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+size_t wcscspn(const wchar_t *s1, const wchar_t *s2);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSCSPN_H
diff --git a/libc/src/wchar/wcsspn.h b/libc/src/wchar/wcsspn.h
index 6dbe65d9a4a98..6e21cbe594574 100644
--- a/libc/src/wchar/wcsspn.h
+++ b/libc/src/wchar/wcsspn.h
@@ -19,4 +19,4 @@ size_t wcsspn(const wchar_t *s1, const wchar_t *s2);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_WCHAR_WCSCHR_H
+#endif // LLVM_LIBC_SRC_WCHAR_WCSSPN_H
diff --git a/libc/src/wchar/wcstok.cpp b/libc/src/wchar/wcstok.cpp
new file mode 100644
index 0000000000000..291efc15e158a
--- /dev/null
+++ b/libc/src/wchar/wcstok.cpp
@@ -0,0 +1,50 @@
+//===-- Implementation of wcstok ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcstok.h"
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bool isADelimeter(wchar_t wc, const wchar_t *delimiters) {
+  for (const wchar_t *delim_ptr = delimiters; *delim_ptr != L'\0'; ++delim_ptr)
+    if (wc == *delim_ptr)
+      return true;
+  return false;
+}
+
+LLVM_LIBC_FUNCTION(wchar_t *, wcstok,
+                   (wchar_t *__restrict str, const wchar_t *__restrict delim,
+                    wchar_t **__restrict context)) {
+  if (str == nullptr) {
+    if (*context == nullptr)
+      return nullptr;
+
+    str = *context;
+  }
+
+  wchar_t *tok_start, *tok_end;
+  for (tok_start = str; *tok_start != L'\0' && isADelimeter(*tok_start, delim);
+       ++tok_start)
+    ;
+
+  for (tok_end = tok_start; *tok_end != L'\0' && !isADelimeter(*tok_end, delim);
+       ++tok_end)
+    ;
+
+  if (*tok_end != L'\0') {
+    *tok_end = L'\0';
+    ++tok_end;
+  }
+  *context = tok_end;
+  return *tok_start == L'\0' ? nullptr : tok_start;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcstok.h b/libc/src/wchar/wcstok.h
new file mode 100644
index 0000000000000..5e673ff4e89b9
--- /dev/null
+++ b/libc/src/wchar/wcstok.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcstok ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSTOK_H
+#define LLVM_LIBC_SRC_WCHAR_WCSTOK_H
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wcstok(wchar_t *__restrict str, const wchar_t *__restrict delim,
+                wchar_t **__restrict context);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSTOK_H
diff --git a/libc/test/integration/src/stdlib/gpu/CMakeLists.txt b/libc/test/integration/src/stdlib/gpu/CMakeLists.txt
index 26c877b1b6ae6..5f9a215bb8738 100644
--- a/libc/test/integration/src/stdlib/gpu/CMakeLists.txt
+++ b/libc/test/integration/src/stdlib/gpu/CMakeLists.txt
@@ -17,6 +17,36 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
       --blocks 1024
   )
 
+  add_integration_test(
+    realloc
+    SUITE
+      stdlib-gpu-integration-tests
+    SRCS
+      realloc.cpp
+    DEPENDS
+      libc.src.stdlib.malloc
+      libc.src.stdlib.free
+      libc.src.stdlib.realloc
+    LOADER_ARGS
+      --threads 256
+      --blocks 1024
+  )
+
+  add_integration_test(
+    aligned_alloc
+    SUITE
+      stdlib-gpu-integration-tests
+    SRCS
+      aligned_alloc.cpp
+    DEPENDS
+      libc.src.stdlib.aligned_alloc
+      libc.src.stdlib.malloc
+      libc.src.stdlib.free
+    LOADER_ARGS
+      --threads 256
+      --blocks 128
+  )
+
   add_integration_test(
     malloc_stress
     SUITE
diff --git a/libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp b/libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp
new file mode 100644
index 0000000000000..6e00eb86c680a
--- /dev/null
+++ b/libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp
@@ -0,0 +1,29 @@
+#include "test/IntegrationTest/test.h"
+
+#include "src/__support/GPU/utils.h"
+#include "src/stdlib/aligned_alloc.h" // Adjust path if needed
+#include "src/stdlib/free.h"
+
+using namespace LIBC_NAMESPACE;
+
+TEST_MAIN(int, char **, char **) {
+  // aligned_alloc with valid alignment and size
+  void *ptr = LIBC_NAMESPACE::aligned_alloc(32, 16);
+  EXPECT_NE(ptr, nullptr);
+  EXPECT_TRUE(__builtin_is_aligned(ptr, 32));
+
+  LIBC_NAMESPACE::free(ptr);
+
+  // aligned_alloc fails if alignment is not power of two
+  void *bad_align = LIBC_NAMESPACE::aligned_alloc(30, 99);
+  EXPECT_EQ(bad_align, nullptr);
+
+  // aligned_alloc with a divergent size.
+  size_t alignment = 1 << (__gpu_lane_id() % 8 + 1);
+  void *div =
+      LIBC_NAMESPACE::aligned_alloc(alignment, (gpu::get_thread_id() + 1) * 4);
+  EXPECT_NE(div, nullptr);
+  EXPECT_TRUE(__builtin_is_aligned(div, alignment));
+
+  return 0;
+}
diff --git a/libc/test/integration/src/stdlib/gpu/malloc.cpp b/libc/test/integration/src/stdlib/gpu/malloc.cpp
index 7880206b1aaaa..a02a6749258ca 100644
--- a/libc/test/integration/src/stdlib/gpu/malloc.cpp
+++ b/libc/test/integration/src/stdlib/gpu/malloc.cpp
@@ -24,6 +24,7 @@ TEST_MAIN(int, char **, char **) {
   int *divergent = reinterpret_cast<int *>(
       LIBC_NAMESPACE::malloc((gpu::get_thread_id() + 1) * 16));
   EXPECT_NE(divergent, nullptr);
+  EXPECT_TRUE(__builtin_is_aligned(divergent, 16));
   *divergent = 1;
   EXPECT_EQ(*divergent, 1);
   LIBC_NAMESPACE::free(divergent);
diff --git a/libc/test/integration/src/stdlib/gpu/realloc.cpp b/libc/test/integration/src/stdlib/gpu/realloc.cpp
new file mode 100644
index 0000000000000..7cdf29489928a
--- /dev/null
+++ b/libc/test/integration/src/stdlib/gpu/realloc.cpp
@@ -0,0 +1,44 @@
+#include "test/IntegrationTest/test.h"
+
+#include "src/__support/GPU/utils.h"
+#include "src/stdlib/free.h"
+#include "src/stdlib/malloc.h"
+#include "src/stdlib/realloc.h"
+
+using namespace LIBC_NAMESPACE;
+
+TEST_MAIN(int, char **, char **) {
+  // realloc(nullptr, size) is equivalent to malloc.
+  int *alloc = reinterpret_cast<int *>(LIBC_NAMESPACE::realloc(nullptr, 32));
+  EXPECT_NE(alloc, nullptr);
+  *alloc = 42;
+  EXPECT_EQ(*alloc, 42);
+
+  // realloc to same size returns the same pointer.
+  void *same = LIBC_NAMESPACE::realloc(alloc, 32);
+  EXPECT_EQ(same, alloc);
+  EXPECT_EQ(reinterpret_cast<int *>(same)[0], 42);
+
+  // realloc to smaller size returns same pointer.
+  void *smaller = LIBC_NAMESPACE::realloc(same, 16);
+  EXPECT_EQ(smaller, alloc);
+  EXPECT_EQ(reinterpret_cast<int *>(smaller)[0], 42);
+
+  // realloc to larger size returns new pointer and preserves contents.
+  int *larger = reinterpret_cast<int *>(LIBC_NAMESPACE::realloc(smaller, 128));
+  EXPECT_NE(larger, nullptr);
+  EXPECT_EQ(larger[0], 42);
+
+  // realloc works when called with a divergent size.
+  int *div = reinterpret_cast<int *>(
+      LIBC_NAMESPACE::malloc((gpu::get_thread_id() + 1) * 16));
+  EXPECT_NE(div, nullptr);
+  div[0] = static_cast<int>(gpu::get_thread_id());
+  int *div_realloc = reinterpret_cast<int *>(
+      LIBC_NAMESPACE::realloc(div, ((gpu::get_thread_id() + 1) * 32)));
+  EXPECT_NE(div_realloc, nullptr);
+  EXPECT_EQ(div_realloc[0], static_cast<int>(gpu::get_thread_id()));
+  LIBC_NAMESPACE::free(div_realloc);
+
+  return 0;
+}
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index 36ae7d689cc00..b419fb5d55414 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/errno_macros.h"
 #include "src/__support/error_or.h"
 #include "src/__support/wchar/character_converter.h"
 #include "src/__support/wchar/mbstate.h"
@@ -87,7 +88,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidByte) {
   LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
   int err = char_conv.push(static_cast<char8_t>(ch));
 
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
 }
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidMultiByte) {
@@ -100,12 +101,12 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidMultiByte) {
 
   LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
   int err = char_conv.push(static_cast<char8_t>(ch[0]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
   err = char_conv.push(static_cast<char8_t>(ch[1]));
   ASSERT_EQ(err, 0);
   // Prev byte was single byte so trying to push another should error.
   err = char_conv.push(static_cast<char8_t>(ch[2]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
   err = char_conv.push(static_cast<char8_t>(ch[3]));
   ASSERT_EQ(err, 0);
 }
@@ -127,7 +128,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidLastByte) {
   err = char_conv.push(static_cast<char8_t>(ch[2]));
   ASSERT_EQ(err, 0);
   err = char_conv.push(static_cast<char8_t>(ch[3]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
 }
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
@@ -144,7 +145,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
   ASSERT_EQ(err, 0);
   // Should produce an error on 3rd byte
   err = char_conv.push(static_cast<char8_t>(ch[2]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
 
   // Should produce an error since mbstate was reset
   auto wch = char_conv.pop_utf32();
diff --git a/libc/test/src/math/tanpif16_test.cpp b/libc/test/src/math/tanpif16_test.cpp
index 629ba963135c0..04d1f713574a1 100644
--- a/libc/test/src/math/tanpif16_test.cpp
+++ b/libc/test/src/math/tanpif16_test.cpp
@@ -1,4 +1,4 @@
-//===-- Exhaustive test for cospif16 --------------------------------------===//
+//===-- Exhaustive test for tanpif16 --------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index 02949c68d81dd..657343d072158 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -123,6 +123,16 @@ add_libc_test(
     libc.src.wchar.wcschr
 )
 
+add_libc_test(
+  wcstok_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcstok_test.cpp
+  DEPENDS
+    libc.src.wchar.wcstok
+)
+
 add_libc_test(
   wcsncmp_test
   SUITE
@@ -173,6 +183,16 @@ add_libc_test(
     libc.src.wchar.wcsspn
 )
 
+add_libc_test(
+  wcscspn_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcscspn_test.cpp
+  DEPENDS
+    libc.src.wchar.wcscspn
+)
+
 add_libc_test(
   wmemchr_test
   SUITE
diff --git a/libc/test/src/wchar/wcscspn_test.cpp b/libc/test/src/wchar/wcscspn_test.cpp
new file mode 100644
index 0000000000000..6318302d10057
--- /dev/null
+++ b/libc/test/src/wchar/wcscspn_test.cpp
@@ -0,0 +1,67 @@
+//===-- Unittests for wcscspn ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcscspn.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSCSpnTest, EmptyStringShouldReturnZeroLengthSpan) {
+  // The search should not include the null terminator.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"", L""), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"_", L""), size_t(1));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"", L"_"), size_t(0));
+}
+
+TEST(LlvmLibcWCSCSpnTest, ShouldNotSpanAnythingAfterNullTerminator) {
+  const wchar_t src[4] = {L'a', L'b', L'\0', L'c'};
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"de"), size_t(2));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"c"), size_t(2));
+
+  // Same goes for the segment to be searched for.
+  const wchar_t segment[4] = {L'1', L'2', L'\0', L'3'};
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"3", segment), size_t(1));
+}
+
+TEST(LlvmLibcWCSCSpnTest, SpanEachIndividualCharacter) {
+  const wchar_t *src = L"12345";
+  // These are all in the segment.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"1"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"2"), size_t(1));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"3"), size_t(2));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"4"), size_t(3));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"5"), size_t(4));
+}
+
+TEST(LlvmLibcWCSCSpnTest, UnmatchedCharacterShouldReturnLength) {
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"a", L"b"), size_t(1));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"abcdef", L"1"), size_t(6));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"123", L"4"), size_t(3));
+}
+
+TEST(LlvmLibcWCSCSpnTest, NonSequentialCharactersShouldNotSpan) {
+  const wchar_t *src = L"abc456789";
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"_1_abc_2_def_3_"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"67__34xyz12"), size_t(3));
+}
+
+TEST(LlvmLibcWCSCSpnTest, ReverseCharacters) {
+  // These are all in the string.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"12345", L"54321"), size_t(0));
+  // 1 is not in the span.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"12345", L"432"), size_t(1));
+  // 1 is in the span.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"12345", L"51"), size_t(0));
+}
+
+TEST(LlvmLibcWCSCSpnTest, DuplicatedCharactersToBeSearchedForShouldStillMatch) {
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"a", L"aa"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"aa", L"aa"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"aaa", L"bb"), size_t(3));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"aaaa", L"bb"), size_t(4));
+}
diff --git a/libc/test/src/wchar/wcstok_test.cpp b/libc/test/src/wchar/wcstok_test.cpp
new file mode 100644
index 0000000000000..7106e9f2fab5e
--- /dev/null
+++ b/libc/test/src/wchar/wcstok_test.cpp
@@ -0,0 +1,181 @@
+//===-- Unittests for wcstok ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcstok.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSTokReentrantTest, NoTokenFound) {
+  { // Empty source and delimiter string.
+    wchar_t empty[] = L"";
+    wchar_t *reserve = nullptr;
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"", &reserve), nullptr);
+    // Another call to ensure that 'reserve' is not in a bad state.
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"", &reserve), nullptr);
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L"", &reserve), nullptr);
+  }
+  { // Empty source and single character delimiter string.
+    wchar_t empty[] = L"";
+    wchar_t *reserve = nullptr;
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"_", &reserve), nullptr);
+    // Another call to ensure that 'reserve' is not in a bad state.
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"_", &reserve), nullptr);
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L"_", &reserve), nullptr);
+  }
+  { // Same character source and delimiter string.
+    wchar_t single[] = L"_";
+    wchar_t *reserve = nullptr;
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(single, L"_", &reserve), nullptr);
+    // Another call to ensure that 'reserve' is not in a bad state.
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(single, L"_", &reserve), nullptr);
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L"_", &reserve), nullptr);
+  }
+  { // Multiple character source and single character delimiter string.
+    wchar_t multiple[] = L"1,2";
+    wchar_t *reserve = nullptr;
+    wchar_t *tok = LIBC_NAMESPACE::wcstok(multiple, L":", &reserve);
+    ASSERT_TRUE(tok[0] == L'1');
+    ASSERT_TRUE(tok[1] == L',');
+    ASSERT_TRUE(tok[2] == L'2');
+    ASSERT_TRUE(tok[3] == L'\0');
+    // Another call to ensure that 'reserve' is not in a bad state.
+    tok = LIBC_NAMESPACE::wcstok(multiple, L":", &reserve);
+    ASSERT_TRUE(tok[0] == L'1');
+    ASSERT_TRUE(tok[1] == L',');
+    ASSERT_TRUE(tok[2] == L'2');
+    ASSERT_TRUE(tok[3] == L'\0');
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L":", &reserve), nullptr);
+  }
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimiterAsFirstCharacterShouldBeIgnored) {
+  wchar_t src[] = L".123";
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L".", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L".", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L".", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimiterIsMiddleCharacter) {
+  wchar_t src[] = L"12,34";
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L",", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimiterAsLastCharacterShouldBeIgnored) {
+  wchar_t src[] = L"1234:";
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L":", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'4');
+  ASSERT_TRUE(tok[4] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L":", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'4');
+  ASSERT_TRUE(tok[4] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L":", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, ShouldNotGoPastNullTerminator) {
+  wchar_t src[] = {L'1', L'2', L'\0', L',', L'3'};
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L",", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest,
+     ShouldReturnNullptrWhenBothSrcAndSaveptrAreNull) {
+  wchar_t *src = nullptr;
+  wchar_t *reserve = nullptr;
+  // Ensure that instead of crashing if src and reserve are null, nullptr is
+  // returned
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(src, L",", &reserve), nullptr);
+  // And that neither src nor reserve are changed when that happens
+  ASSERT_EQ(src, nullptr);
+  ASSERT_EQ(reserve, nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest,
+     SubsequentCallsShouldFindFollowingDelimiters) {
+  wchar_t src[] = L"12,34.56";
+  wchar_t *reserve = nullptr;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L",.", &reserve);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L",.", &reserve);
+  ASSERT_TRUE(token[0] == L'3');
+  ASSERT_TRUE(token[1] == L'4');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L",.", &reserve);
+  ASSERT_TRUE(token[0] == L'5');
+  ASSERT_TRUE(token[1] == L'6');
+  ASSERT_TRUE(token[2] == L'\0');
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &reserve);
+  ASSERT_EQ(token, nullptr);
+  // Subsequent calls after hitting the end of the string should also return
+  // nullptr.
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &reserve);
+  ASSERT_EQ(token, nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimitersShouldNotBeIncludedInToken) {
+  wchar_t src[] = L"__ab__:_cd__:__ef__:__";
+  wchar_t *reserve = nullptr;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L"_:", &reserve);
+  ASSERT_TRUE(token[0] == L'a');
+  ASSERT_TRUE(token[1] == L'b');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L":_", &reserve);
+  ASSERT_TRUE(token[0] == L'c');
+  ASSERT_TRUE(token[1] == L'd');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,", &reserve);
+  ASSERT_TRUE(token[0] == L'e');
+  ASSERT_TRUE(token[1] == L'f');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &reserve);
+  ASSERT_EQ(token, nullptr);
+}
diff --git a/libc/utils/MPFRWrapper/MPCommon.cpp b/libc/utils/MPFRWrapper/MPCommon.cpp
index ccd4d2d01a4e2..441d5420b278b 100644
--- a/libc/utils/MPFRWrapper/MPCommon.cpp
+++ b/libc/utils/MPFRWrapper/MPCommon.cpp
@@ -73,8 +73,7 @@ MPFRNumber MPFRNumber::acosh() const {
 MPFRNumber MPFRNumber::acospi() const {
   MPFRNumber result(*this);
 
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
   mpfr_acospi(result.value, value, mpfr_rounding);
   return result;
 #else
@@ -150,8 +149,7 @@ MPFRNumber MPFRNumber::cosh() const {
 MPFRNumber MPFRNumber::cospi() const {
   MPFRNumber result(*this);
 
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
   mpfr_cospi(result.value, value, mpfr_rounding);
   return result;
 #else
@@ -195,8 +193,7 @@ MPFRNumber MPFRNumber::exp2() const {
 
 MPFRNumber MPFRNumber::exp2m1() const {
   // TODO: Only use mpfr_exp2m1 once CI and buildbots get MPFR >= 4.2.0.
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
   MPFRNumber result(*this);
   mpfr_exp2m1(result.value, value, mpfr_rounding);
   return result;
@@ -231,8 +228,7 @@ MPFRNumber MPFRNumber::exp10() const {
 
 MPFRNumber MPFRNumber::exp10m1() const {
   // TODO: Only use mpfr_exp10m1 once CI and buildbots get MPFR >= 4.2.0.
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
   MPFRNumber result(*this);
   mpfr_exp10m1(result.value, value, mpfr_rounding);
   return result;
@@ -402,9 +398,7 @@ MPFRNumber MPFRNumber::sin() const {
 MPFRNumber MPFRNumber::sinpi() const {
   MPFRNumber result(*this);
 
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
   mpfr_sinpi(result.value, value, mpfr_rounding);
   return result;
 #else
@@ -463,9 +457,7 @@ MPFRNumber MPFRNumber::tanh() const {
 MPFRNumber MPFRNumber::tanpi() const {
   MPFRNumber result(*this);
 
-#if MPFR_VERSION_MAJOR > 4 ||                                                  \
-    (MPFR_VERSION_MAJOR == 4 && MPFR_VERSION_MINOR >= 2)
-
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
   mpfr_tanpi(result.value, value, mpfr_rounding);
   return result;
 #else
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index c98e2043464d9..e2871d1b01a16 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -84,6 +84,9 @@ else()
   endif()
 endif()
 
+# Setup the paths where libclc runtimes should be stored.
+set( LIBCLC_OUTPUT_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR} )
+
 if( EXISTS ${LIBCLC_CUSTOM_LLVM_TOOLS_BINARY_DIR} )
   message( WARNING "Using custom LLVM tools to build libclc: "
     "${LIBCLC_CUSTOM_LLVM_TOOLS_BINARY_DIR}, "
diff --git a/libclc/clc/include/clc/clcmacro.h b/libclc/clc/include/clc/clcmacro.h
index b712fe5cf326c..5c67c937cb1d4 100644
--- a/libclc/clc/include/clc/clcmacro.h
+++ b/libclc/clc/include/clc/clcmacro.h
@@ -12,111 +12,6 @@
 #include <clc/internal/clc.h>
 #include <clc/utils.h>
 
-#define _CLC_UNARY_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE)          \
-  DECLSPEC RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x) {                              \
-    return (RET_TYPE##2)(FUNCTION(x.s0), FUNCTION(x.s1));                      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x) {                              \
-    return (RET_TYPE##3)(FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2));      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x) {                              \
-    return (RET_TYPE##4)(FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),       \
-                         FUNCTION(x.s3));                                      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x) {                              \
-    return (RET_TYPE##8)(FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),       \
-                         FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),       \
-                         FUNCTION(x.s6), FUNCTION(x.s7));                      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x) {                            \
-    return (RET_TYPE##16)(                                                     \
-        FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3),        \
-        FUNCTION(x.s4), FUNCTION(x.s5), FUNCTION(x.s6), FUNCTION(x.s7),        \
-        FUNCTION(x.s8), FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb),        \
-        FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se), FUNCTION(x.sf));       \
-  }
-
-#define _CLC_BINARY_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,         \
-                              ARG2_TYPE)                                       \
-  DECLSPEC RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x, ARG2_TYPE##2 y) {              \
-    return (RET_TYPE##2)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1));          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x, ARG2_TYPE##3 y) {              \
-    return (RET_TYPE##3)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),           \
-                         FUNCTION(x.s2, y.s2));                                \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x, ARG2_TYPE##4 y) {              \
-    return (RET_TYPE##4)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),           \
-                         FUNCTION(x.s2, y.s2), FUNCTION(x.s3, y.s3));          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x, ARG2_TYPE##8 y) {              \
-    return (RET_TYPE##8)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),           \
-                         FUNCTION(x.s2, y.s2), FUNCTION(x.s3, y.s3),           \
-                         FUNCTION(x.s4, y.s4), FUNCTION(x.s5, y.s5),           \
-                         FUNCTION(x.s6, y.s6), FUNCTION(x.s7, y.s7));          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x, ARG2_TYPE##16 y) {           \
-    return (RET_TYPE##16)(                                                     \
-        FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1), FUNCTION(x.s2, y.s2),      \
-        FUNCTION(x.s3, y.s3), FUNCTION(x.s4, y.s4), FUNCTION(x.s5, y.s5),      \
-        FUNCTION(x.s6, y.s6), FUNCTION(x.s7, y.s7), FUNCTION(x.s8, y.s8),      \
-        FUNCTION(x.s9, y.s9), FUNCTION(x.sa, y.sa), FUNCTION(x.sb, y.sb),      \
-        FUNCTION(x.sc, y.sc), FUNCTION(x.sd, y.sd), FUNCTION(x.se, y.se),      \
-        FUNCTION(x.sf, y.sf));                                                 \
-  }
-
-#define _CLC_TERNARY_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,        \
-                               ARG2_TYPE, ARG3_TYPE)                           \
-  DECLSPEC RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x, ARG2_TYPE##2 y,                \
-                                ARG3_TYPE##2 z) {                              \
-    return (RET_TYPE##2)(FUNCTION(x.s0, y.s0, z.s0),                           \
-                         FUNCTION(x.s1, y.s1, z.s1));                          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x, ARG2_TYPE##3 y,                \
-                                ARG3_TYPE##3 z) {                              \
-    return (RET_TYPE##3)(FUNCTION(x.s0, y.s0, z.s0),                           \
-                         FUNCTION(x.s1, y.s1, z.s1),                           \
-                         FUNCTION(x.s2, y.s2, z.s2));                          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x, ARG2_TYPE##4 y,                \
-                                ARG3_TYPE##4 z) {                              \
-    return (RET_TYPE##4)(                                                      \
-        FUNCTION(x.s0, y.s0, z.s0), FUNCTION(x.s1, y.s1, z.s1),                \
-        FUNCTION(x.s2, y.s2, z.s2), FUNCTION(x.s3, y.s3, z.s3));               \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x, ARG2_TYPE##8 y,                \
-                                ARG3_TYPE##8 z) {                              \
-    return (RET_TYPE##8)(                                                      \
-        FUNCTION(x.s0, y.s0, z.s0), FUNCTION(x.s1, y.s1, z.s1),                \
-        FUNCTION(x.s2, y.s2, z.s2), FUNCTION(x.s3, y.s3, z.s3),                \
-        FUNCTION(x.s4, y.s4, z.s4), FUNCTION(x.s5, y.s5, z.s5),                \
-        FUNCTION(x.s6, y.s6, z.s6), FUNCTION(x.s7, y.s7, z.s7));               \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x, ARG2_TYPE##16 y,             \
-                                 ARG3_TYPE##16 z) {                            \
-    return (RET_TYPE##16)(                                                     \
-        FUNCTION(x.s0, y.s0, z.s0), FUNCTION(x.s1, y.s1, z.s1),                \
-        FUNCTION(x.s2, y.s2, z.s2), FUNCTION(x.s3, y.s3, z.s3),                \
-        FUNCTION(x.s4, y.s4, z.s4), FUNCTION(x.s5, y.s5, z.s5),                \
-        FUNCTION(x.s6, y.s6, z.s6), FUNCTION(x.s7, y.s7, z.s7),                \
-        FUNCTION(x.s8, y.s8, z.s8), FUNCTION(x.s9, y.s9, z.s9),                \
-        FUNCTION(x.sa, y.sa, z.sa), FUNCTION(x.sb, y.sb, z.sb),                \
-        FUNCTION(x.sc, y.sc, z.sc), FUNCTION(x.sd, y.sd, z.sd),                \
-        FUNCTION(x.se, y.se, z.se), FUNCTION(x.sf, y.sf, z.sf));               \
-  }
-
 #define _CLC_V_V_VP_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,         \
                               ADDR_SPACE, ARG2_TYPE)                           \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 2)                                          \
@@ -171,12 +66,4 @@
         FUNCTION(x.sf, ptr + 15));                                             \
   }
 
-#define _CLC_DEFINE_BINARY_BUILTIN(RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE,     \
-                                   ARG2_TYPE)                                  \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, RET_TYPE, FUNCTION, ARG1_TYPE, \
-                        ARG2_TYPE)
-
 #endif // __CLC_CLCMACRO_H__
diff --git a/libclc/clc/include/clc/common/clc_degrees.h b/libclc/clc/include/clc/common/clc_degrees.h
index 5f04fac1f2612..e633344c291c1 100644
--- a/libclc/clc/include/clc/common/clc_degrees.h
+++ b/libclc/clc/include/clc/common/clc_degrees.h
@@ -10,10 +10,10 @@
 #define __CLC_COMMON_CLC_DEGREES_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_degrees
+#define FUNCTION __clc_degrees
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_DEGREES_H__
diff --git a/libclc/clc/include/clc/common/clc_radians.h b/libclc/clc/include/clc/common/clc_radians.h
index 03d7c4b9e3014..0b075efbcce2f 100644
--- a/libclc/clc/include/clc/common/clc_radians.h
+++ b/libclc/clc/include/clc/common/clc_radians.h
@@ -10,10 +10,10 @@
 #define __CLC_COMMON_CLC_RADIANS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_radians
+#define FUNCTION __clc_radians
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_RADIANS_H__
diff --git a/libclc/clc/include/clc/common/clc_sign.h b/libclc/clc/include/clc/common/clc_sign.h
index 9737f1e54441b..0a8c4cc94d80b 100644
--- a/libclc/clc/include/clc/common/clc_sign.h
+++ b/libclc/clc/include/clc/common/clc_sign.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_COMMON_CLC_SIGN_H__
 #define __CLC_COMMON_CLC_SIGN_H__
 
-#define __CLC_FUNCTION __clc_sign
+#define FUNCTION __clc_sign
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_SIGN_H__
diff --git a/libclc/clc/include/clc/common/clc_step.h b/libclc/clc/include/clc/common/clc_step.h
index 6b093d06896c0..96b4759cabf77 100644
--- a/libclc/clc/include/clc/common/clc_step.h
+++ b/libclc/clc/include/clc/common/clc_step.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_COMMON_CLC_STEP_H__
 #define __CLC_COMMON_CLC_STEP_H__
 
-#define __CLC_FUNCTION __clc_step
+#define FUNCTION __clc_step
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_STEP_H__
diff --git a/libclc/clc/include/clc/geometric/binary_decl.inc b/libclc/clc/include/clc/geometric/binary_decl.inc
index 3f3aafdeae6c3..342d97a802659 100644
--- a/libclc/clc/include/clc/geometric/binary_decl.inc
+++ b/libclc/clc/include/clc/geometric/binary_decl.inc
@@ -10,7 +10,7 @@
 #if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
      __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
 
-_CLC_OVERLOAD _CLC_DECL __CLC_SCALAR_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                            __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_SCALAR_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                                      __CLC_GENTYPE b);
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/binary_def.inc b/libclc/clc/include/clc/geometric/binary_def.inc
index 8fc875a052210..ad4fa4c01683b 100644
--- a/libclc/clc/include/clc/geometric/binary_def.inc
+++ b/libclc/clc/include/clc/geometric/binary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // Geometric functions are only defined for scalar, vec2, vec3 and vec4
@@ -18,7 +18,7 @@
 
 _CLC_OVERLOAD _CLC_DEF __CLC_SCALAR_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                                      __CLC_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/clc_distance.h b/libclc/clc/include/clc/geometric/clc_distance.h
index 1acea411a08b6..666e2c881bc9a 100644
--- a/libclc/clc/include/clc/geometric/clc_distance.h
+++ b/libclc/clc/include/clc/geometric/clc_distance.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_DISTANCE_H__
 #define __CLC_GEOMETRIC_CLC_DISTANCE_H__
 
-#define __CLC_FUNCTION __clc_distance
+#define FUNCTION __clc_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_DISTANCE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_dot.h b/libclc/clc/include/clc/geometric/clc_dot.h
index a5a97d1bdd2bc..4bfed6e3abc51 100644
--- a/libclc/clc/include/clc/geometric/clc_dot.h
+++ b/libclc/clc/include/clc/geometric/clc_dot.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_DOT_H__
 #define __CLC_GEOMETRIC_CLC_DOT_H__
 
-#define __CLC_FUNCTION __clc_dot
+#define FUNCTION __clc_dot
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_DOT_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_distance.h b/libclc/clc/include/clc/geometric/clc_fast_distance.h
index 489bea4299516..2932d0147a6d8 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_distance.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_distance.h
@@ -10,12 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_fast_distance
+#define FUNCTION __clc_fast_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_length.h b/libclc/clc/include/clc/geometric/clc_fast_length.h
index 40552634fff94..903ecc8b91755 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_length.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_length.h
@@ -10,12 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_fast_length
+#define FUNCTION __clc_fast_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_normalize.h b/libclc/clc/include/clc/geometric/clc_fast_normalize.h
index 66eed8b83ab18..a91862fe61cb7 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_normalize.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_normalize.h
@@ -11,12 +11,11 @@
 
 #define __FLOAT_ONLY
 #define __CLC_GEOMETRIC_RET_GENTYPE
-#define __CLC_FUNCTION __clc_fast_normalize
+#define FUNCTION __clc_fast_normalize
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef __FLOAT_ONLY
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_length.h b/libclc/clc/include/clc/geometric/clc_length.h
index f8ce12db07871..35b206e6c3fbb 100644
--- a/libclc/clc/include/clc/geometric/clc_length.h
+++ b/libclc/clc/include/clc/geometric/clc_length.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_LENGTH_H__
 #define __CLC_GEOMETRIC_CLC_LENGTH_H__
 
-#define __CLC_FUNCTION __clc_length
+#define FUNCTION __clc_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_LENGTH_H__
diff --git a/libclc/clc/include/clc/geometric/clc_normalize.h b/libclc/clc/include/clc/geometric/clc_normalize.h
index 3058a72b2bbbe..745ed7f4887d1 100644
--- a/libclc/clc/include/clc/geometric/clc_normalize.h
+++ b/libclc/clc/include/clc/geometric/clc_normalize.h
@@ -10,11 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_NORMALIZE_H__
 
 #define __CLC_GEOMETRIC_RET_GENTYPE
-#define __CLC_FUNCTION __clc_normalize
+#define FUNCTION __clc_normalize
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
 
 #endif // __CLC_GEOMETRIC_CLC_NORMALIZE_H__
diff --git a/libclc/clc/include/clc/geometric/unary_decl.inc b/libclc/clc/include/clc/geometric/unary_decl.inc
index 6da16d2e8abf8..cff66000fcf77 100644
--- a/libclc/clc/include/clc/geometric/unary_decl.inc
+++ b/libclc/clc/include/clc/geometric/unary_decl.inc
@@ -16,6 +16,6 @@ _CLC_OVERLOAD _CLC_DECL
 #else
     __CLC_SCALAR_GENTYPE
 #endif
-    __CLC_FUNCTION(__CLC_GENTYPE a);
+    FUNCTION(__CLC_GENTYPE a);
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/unary_def.inc b/libclc/clc/include/clc/geometric/unary_def.inc
index 5bedefb662765..ea90de0ec581d 100644
--- a/libclc/clc/include/clc/geometric/unary_def.inc
+++ b/libclc/clc/include/clc/geometric/unary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // Geometric functions are only defined for scalar, vec2, vec3 and vec4
@@ -23,7 +23,7 @@ _CLC_OVERLOAD _CLC_DEF
     __CLC_SCALAR_GENTYPE
 #endif
     FUNCTION(__CLC_GENTYPE a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/integer/clc_add_sat.h b/libclc/clc/include/clc/integer/clc_add_sat.h
index 8c3495f6c5193..be095d38225e6 100644
--- a/libclc/clc/include/clc/integer/clc_add_sat.h
+++ b/libclc/clc/include/clc/integer/clc_add_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_ADD_SAT_H__
 #define __CLC_INTEGER_CLC_ADD_SAT_H__
 
-#define __CLC_FUNCTION __clc_add_sat
+#define FUNCTION __clc_add_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_ADD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_clz.h b/libclc/clc/include/clc/integer/clc_clz.h
index dd62cc935f8cd..3e0530bad6f11 100644
--- a/libclc/clc/include/clc/integer/clc_clz.h
+++ b/libclc/clc/include/clc/integer/clc_clz.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_CLZ_H__
 #define __CLC_INTEGER_CLC_CLZ_H__
 
-#define __CLC_FUNCTION __clc_clz
+#define FUNCTION __clc_clz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_CLZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_ctz.h b/libclc/clc/include/clc/integer/clc_ctz.h
index 3f2079e023f67..1495dc5560f1d 100644
--- a/libclc/clc/include/clc/integer/clc_ctz.h
+++ b/libclc/clc/include/clc/integer/clc_ctz.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_CTZ_H__
 #define __CLC_INTEGER_CLC_CTZ_H__
 
-#define __CLC_FUNCTION __clc_ctz
+#define FUNCTION __clc_ctz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_CTZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_hadd.h b/libclc/clc/include/clc/integer/clc_hadd.h
index 679369d3c9d4e..7aaee9b601a26 100644
--- a/libclc/clc/include/clc/integer/clc_hadd.h
+++ b/libclc/clc/include/clc/integer/clc_hadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_HADD_H__
 #define __CLC_INTEGER_CLC_HADD_H__
 
-#define __CLC_FUNCTION __clc_hadd
+#define FUNCTION __clc_hadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_HADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad24.h b/libclc/clc/include/clc/integer/clc_mad24.h
index e19bd905d22eb..121840f81d63e 100644
--- a/libclc/clc/include/clc/integer/clc_mad24.h
+++ b/libclc/clc/include/clc/integer/clc_mad24.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MAD24_H__
 #define __CLC_INTEGER_CLC_MAD24_H__
 
-#define __CLC_FUNCTION __clc_mad24
+#define FUNCTION __clc_mad24
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MAD24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad_sat.h b/libclc/clc/include/clc/integer/clc_mad_sat.h
index 845378b3e7c46..0c418068ca4ac 100644
--- a/libclc/clc/include/clc/integer/clc_mad_sat.h
+++ b/libclc/clc/include/clc/integer/clc_mad_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MAD_SAT_H__
 #define __CLC_INTEGER_CLC_MAD_SAT_H__
 
-#define __CLC_FUNCTION __clc_mad_sat
+#define FUNCTION __clc_mad_sat
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MAD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul24.h b/libclc/clc/include/clc/integer/clc_mul24.h
index f813de55c71e5..9310458d59817 100644
--- a/libclc/clc/include/clc/integer/clc_mul24.h
+++ b/libclc/clc/include/clc/integer/clc_mul24.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MUL24_H__
 #define __CLC_INTEGER_CLC_MUL24_H__
 
-#define __CLC_FUNCTION __clc_mul24
+#define FUNCTION __clc_mul24
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MUL24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul_hi.h b/libclc/clc/include/clc/integer/clc_mul_hi.h
index e4738e5570e81..6542f237d09bc 100644
--- a/libclc/clc/include/clc/integer/clc_mul_hi.h
+++ b/libclc/clc/include/clc/integer/clc_mul_hi.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MUL_HI_H__
 #define __CLC_INTEGER_CLC_MUL_HI_H__
 
-#define __CLC_FUNCTION __clc_mul_hi
+#define FUNCTION __clc_mul_hi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MUL_HI_H__
diff --git a/libclc/clc/include/clc/integer/clc_popcount.h b/libclc/clc/include/clc/integer/clc_popcount.h
index 55ca6ccfcec28..8ece2e998862d 100644
--- a/libclc/clc/include/clc/integer/clc_popcount.h
+++ b/libclc/clc/include/clc/integer/clc_popcount.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_INTEGER_CLC_POPCOUNT_H__
 #define __CLC_INTEGER_CLC_POPCOUNT_H__
 
-#define __CLC_FUNCTION __clc_popcount
+#define FUNCTION __clc_popcount
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
 #undef __CLC_INTRINSIC
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_POPCOUNT_H__
diff --git a/libclc/clc/include/clc/integer/clc_rhadd.h b/libclc/clc/include/clc/integer/clc_rhadd.h
index b8b0b1a3a56ed..882f950656dfb 100644
--- a/libclc/clc/include/clc/integer/clc_rhadd.h
+++ b/libclc/clc/include/clc/integer/clc_rhadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_RHADD_H__
 #define __CLC_INTEGER_CLC_RHADD_H__
 
-#define __CLC_FUNCTION __clc_rhadd
+#define FUNCTION __clc_rhadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_RHADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_rotate.h b/libclc/clc/include/clc/integer/clc_rotate.h
index 513bf98526233..61cf08788ef3b 100644
--- a/libclc/clc/include/clc/integer/clc_rotate.h
+++ b/libclc/clc/include/clc/integer/clc_rotate.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_ROTATE_H__
 #define __CLC_INTEGER_CLC_ROTATE_H__
 
-#define __CLC_FUNCTION __clc_rotate
+#define FUNCTION __clc_rotate
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_ROTATE_H__
diff --git a/libclc/clc/include/clc/integer/clc_sub_sat.h b/libclc/clc/include/clc/integer/clc_sub_sat.h
index 94a5484adefcf..1b694a4a1780a 100644
--- a/libclc/clc/include/clc/integer/clc_sub_sat.h
+++ b/libclc/clc/include/clc/integer/clc_sub_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_SUB_SAT_H__
 #define __CLC_INTEGER_CLC_SUB_SAT_H__
 
-#define __CLC_FUNCTION __clc_sub_sat
+#define FUNCTION __clc_sub_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_SUB_SAT_H__
diff --git a/libclc/clc/include/clc/internal/math/clc_sw_fma.h b/libclc/clc/include/clc/internal/math/clc_sw_fma.h
index 5d6c76879ceb9..8497f4efd1137 100644
--- a/libclc/clc/include/clc/internal/math/clc_sw_fma.h
+++ b/libclc/clc/include/clc/internal/math/clc_sw_fma.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 #define __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 
-#define __CLC_FUNCTION __clc_sw_fma
+#define FUNCTION __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
diff --git a/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc b/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
index 6694b20a29ec0..1b2b9ff33521b 100644
--- a/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_GENTYPE b);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                               __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                               __CLC_SCALAR_GENTYPE b);
diff --git a/libclc/clc/include/clc/math/binary_def_via_fp32.inc b/libclc/clc/include/clc/math/binary_def_via_fp32.inc
index 0b4108adb0610..b53a178bf0f10 100644
--- a/libclc/clc/include/clc/math/binary_def_via_fp32.inc
+++ b/libclc/clc/include/clc/math/binary_def_via_fp32.inc
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                    __CLC_GENTYPE y) {
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                              __CLC_GENTYPE y) {
   return __CLC_CONVERT_GENTYPE(
-      __CLC_FUNCTION(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y)));
+      FUNCTION(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y)));
 }
diff --git a/libclc/clc/include/clc/math/clc_acos.h b/libclc/clc/include/clc/math/clc_acos.h
index 1d484fa5b56c9..130e3111ec1ca 100644
--- a/libclc/clc/include/clc/math/clc_acos.h
+++ b/libclc/clc/include/clc/math/clc_acos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_acos
+#define FUNCTION __clc_acos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOS_H__
diff --git a/libclc/clc/include/clc/math/clc_acosh.h b/libclc/clc/include/clc/math/clc_acosh.h
index 9492724bb27ba..9b8c7184b9742 100644
--- a/libclc/clc/include/clc/math/clc_acosh.h
+++ b/libclc/clc/include/clc/math/clc_acosh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_acosh
+#define FUNCTION __clc_acosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOSH_H__
diff --git a/libclc/clc/include/clc/math/clc_acospi.h b/libclc/clc/include/clc/math/clc_acospi.h
index 90cfc06b7dffb..6d4dc694900a0 100644
--- a/libclc/clc/include/clc/math/clc_acospi.h
+++ b/libclc/clc/include/clc/math/clc_acospi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_acospi
+#define FUNCTION __clc_acospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOSPI_H__
diff --git a/libclc/clc/include/clc/math/clc_asin.h b/libclc/clc/include/clc/math/clc_asin.h
index f6246c49f8022..f3d69a7bf9956 100644
--- a/libclc/clc/include/clc/math/clc_asin.h
+++ b/libclc/clc/include/clc/math/clc_asin.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_asin
+#define FUNCTION __clc_asin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ASIN_H__
diff --git a/libclc/clc/include/clc/math/clc_asinh.h b/libclc/clc/include/clc/math/clc_asinh.h
index 05e44fa438cee..dc4530fa68b82 100644
--- a/libclc/clc/include/clc/math/clc_asinh.h
+++ b/libclc/clc/include/clc/math/clc_asinh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_asinh
+#define FUNCTION __clc_asinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ASINH_H__
diff --git a/libclc/clc/include/clc/math/clc_asinpi.h b/libclc/clc/include/clc/math/clc_asinpi.h
index 8f308dd06022e..19d8ec3ad0c50 100644
--- a/libclc/clc/include/clc/math/clc_asinpi.h
+++ b/libclc/clc/include/clc/math/clc_asinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_asinpi
+#define FUNCTION __clc_asinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ASINPI_H__
diff --git a/libclc/clc/include/clc/math/clc_atan.h b/libclc/clc/include/clc/math/clc_atan.h
index 203e46c2b8a4a..9f10efeae976c 100644
--- a/libclc/clc/include/clc/math/clc_atan.h
+++ b/libclc/clc/include/clc/math/clc_atan.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_atan
+#define FUNCTION __clc_atan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN_H__
diff --git a/libclc/clc/include/clc/math/clc_atan2.h b/libclc/clc/include/clc/math/clc_atan2.h
index 6edc9dda48f4d..99bd0760f3fe8 100644
--- a/libclc/clc/include/clc/math/clc_atan2.h
+++ b/libclc/clc/include/clc/math/clc_atan2.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN2_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_atan2
+#define FUNCTION __clc_atan2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN2_H__
diff --git a/libclc/clc/include/clc/math/clc_atan2pi.h b/libclc/clc/include/clc/math/clc_atan2pi.h
index 2f26d1d9ae985..ae044b7bbfafb 100644
--- a/libclc/clc/include/clc/math/clc_atan2pi.h
+++ b/libclc/clc/include/clc/math/clc_atan2pi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN2PI_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_atan2pi
+#define FUNCTION __clc_atan2pi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN2PI_H__
diff --git a/libclc/clc/include/clc/math/clc_atanh.h b/libclc/clc/include/clc/math/clc_atanh.h
index 85f9fc53be1a6..c1a95d4acba1b 100644
--- a/libclc/clc/include/clc/math/clc_atanh.h
+++ b/libclc/clc/include/clc/math/clc_atanh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_atanh
+#define FUNCTION __clc_atanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATANH_H__
diff --git a/libclc/clc/include/clc/math/clc_atanpi.h b/libclc/clc/include/clc/math/clc_atanpi.h
index 2b3c6b373ed42..43f752ecfe98b 100644
--- a/libclc/clc/include/clc/math/clc_atanpi.h
+++ b/libclc/clc/include/clc/math/clc_atanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_atanpi
+#define FUNCTION __clc_atanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATANPI_H__
diff --git a/libclc/clc/include/clc/math/clc_cbrt.inc b/libclc/clc/include/clc/math/clc_cbrt.inc
index fd9b750c8fd89..019b5174a9015 100644
--- a/libclc/clc/include/clc/math/clc_cbrt.inc
+++ b/libclc/clc/include/clc/math/clc_cbrt.inc
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_CBRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cbrt
+#define FUNCTION __clc_cbrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_CBRT_H__
diff --git a/libclc/clc/include/clc/math/clc_ceil.h b/libclc/clc/include/clc/math/clc_ceil.h
index 45668224c6176..c60b794c83c6c 100644
--- a/libclc/clc/include/clc/math/clc_ceil.h
+++ b/libclc/clc/include/clc/math/clc_ceil.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_CEIL_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_ceil
+#define FUNCTION __clc_ceil
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_CEIL_H__
diff --git a/libclc/clc/include/clc/math/clc_copysign.h b/libclc/clc/include/clc/math/clc_copysign.h
index 563dd2827839a..5976e42de250f 100644
--- a/libclc/clc/include/clc/math/clc_copysign.h
+++ b/libclc/clc/include/clc/math/clc_copysign.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COPYSIGN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_copysign
+#define FUNCTION __clc_copysign
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COPYSIGN_H__
diff --git a/libclc/clc/include/clc/math/clc_cos.h b/libclc/clc/include/clc/math/clc_cos.h
index 44681608efc37..3a1ae9342e265 100644
--- a/libclc/clc/include/clc/math/clc_cos.h
+++ b/libclc/clc/include/clc/math/clc_cos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cos
+#define FUNCTION __clc_cos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_cosh.h b/libclc/clc/include/clc/math/clc_cosh.h
index 7bcfa2ea26e06..3033d76000dc0 100644
--- a/libclc/clc/include/clc/math/clc_cosh.h
+++ b/libclc/clc/include/clc/math/clc_cosh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cosh
+#define FUNCTION __clc_cosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COSH_H__
diff --git a/libclc/clc/include/clc/math/clc_cospi.h b/libclc/clc/include/clc/math/clc_cospi.h
index 4d82100649b01..3e4e73f096a06 100644
--- a/libclc/clc/include/clc/math/clc_cospi.h
+++ b/libclc/clc/include/clc/math/clc_cospi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cospi
+#define FUNCTION __clc_cospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COSPI_H__
diff --git a/libclc/clc/include/clc/math/clc_erf.h b/libclc/clc/include/clc/math/clc_erf.h
index 01a21b36b352f..04b990c65133f 100644
--- a/libclc/clc/include/clc/math/clc_erf.h
+++ b/libclc/clc/include/clc/math/clc_erf.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ERF_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_erf
+#define FUNCTION __clc_erf
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ERF_H__
diff --git a/libclc/clc/include/clc/math/clc_erfc.h b/libclc/clc/include/clc/math/clc_erfc.h
index efd581542879f..be2578b4bfb04 100644
--- a/libclc/clc/include/clc/math/clc_erfc.h
+++ b/libclc/clc/include/clc/math/clc_erfc.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ERFC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_erfc
+#define FUNCTION __clc_erfc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ERFC_H__
diff --git a/libclc/clc/include/clc/math/clc_exp.h b/libclc/clc/include/clc/math/clc_exp.h
index 84369bb8fbdd6..355148934427f 100644
--- a/libclc/clc/include/clc/math/clc_exp.h
+++ b/libclc/clc/include/clc/math/clc_exp.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_exp
+#define FUNCTION __clc_exp
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_exp10.h b/libclc/clc/include/clc/math/clc_exp10.h
index 500271cb07429..c281ee84dd3a1 100644
--- a/libclc/clc/include/clc/math/clc_exp10.h
+++ b/libclc/clc/include/clc/math/clc_exp10.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_exp10
+#define FUNCTION __clc_exp10
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_exp2.h b/libclc/clc/include/clc/math/clc_exp2.h
index 4ad0ddea48a7c..bb10f12d34e2e 100644
--- a/libclc/clc/include/clc/math/clc_exp2.h
+++ b/libclc/clc/include/clc/math/clc_exp2.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_exp2
+#define FUNCTION __clc_exp2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_exp_helper.h b/libclc/clc/include/clc/math/clc_exp_helper.h
index 89c725ca0d5c9..0c5028bd23858 100644
--- a/libclc/clc/include/clc/math/clc_exp_helper.h
+++ b/libclc/clc/include/clc/math/clc_exp_helper.h
@@ -14,6 +14,4 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __DOUBLE_ONLY
-
 #endif // __CLC_MATH_CLC_EXP_HELPER
diff --git a/libclc/clc/include/clc/math/clc_expm1.h b/libclc/clc/include/clc/math/clc_expm1.h
index 5c60960ca5b03..7a199dca17cb3 100644
--- a/libclc/clc/include/clc/math/clc_expm1.h
+++ b/libclc/clc/include/clc/math/clc_expm1.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXPM1_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_expm1
+#define FUNCTION __clc_expm1
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXPM1_H__
diff --git a/libclc/clc/include/clc/math/clc_fabs.h b/libclc/clc/include/clc/math/clc_fabs.h
index 1ef38a8c75957..8af3bb572142f 100644
--- a/libclc/clc/include/clc/math/clc_fabs.h
+++ b/libclc/clc/include/clc/math/clc_fabs.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FABS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_fabs
+#define FUNCTION __clc_fabs
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FABS_H__
diff --git a/libclc/clc/include/clc/math/clc_fdim.h b/libclc/clc/include/clc/math/clc_fdim.h
index 50ad045db62cf..c678eb25f59ba 100644
--- a/libclc/clc/include/clc/math/clc_fdim.h
+++ b/libclc/clc/include/clc/math/clc_fdim.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FDIM_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_fdim
+#define FUNCTION __clc_fdim
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FDIM_H__
diff --git a/libclc/clc/include/clc/math/clc_floor.h b/libclc/clc/include/clc/math/clc_floor.h
index d243819e0a42c..b3fe8ea95ce13 100644
--- a/libclc/clc/include/clc/math/clc_floor.h
+++ b/libclc/clc/include/clc/math/clc_floor.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FLOOR_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_floor
+#define FUNCTION __clc_floor
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FLOOR_H__
diff --git a/libclc/clc/include/clc/math/clc_fma.h b/libclc/clc/include/clc/math/clc_fma.h
index 5e87f4c470850..efe20bd150eb5 100644
--- a/libclc/clc/include/clc/math/clc_fma.h
+++ b/libclc/clc/include/clc/math/clc_fma.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMA_H__
 #define __CLC_MATH_CLC_FMA_H__
 
-#define __CLC_FUNCTION __clc_fma
+#define FUNCTION __clc_fma
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMA_H__
diff --git a/libclc/clc/include/clc/math/clc_fmax.h b/libclc/clc/include/clc/math/clc_fmax.h
index f367a9b6ea1ca..fd5d38987ec11 100644
--- a/libclc/clc/include/clc/math/clc_fmax.h
+++ b/libclc/clc/include/clc/math/clc_fmax.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMAX_H__
 #define __CLC_MATH_CLC_FMAX_H__
 
-#define __CLC_FUNCTION __clc_fmax
+#define FUNCTION __clc_fmax
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMAX_H__
diff --git a/libclc/clc/include/clc/math/clc_fmin.h b/libclc/clc/include/clc/math/clc_fmin.h
index 47ed3fad2d8c2..9fcdb56475ff6 100644
--- a/libclc/clc/include/clc/math/clc_fmin.h
+++ b/libclc/clc/include/clc/math/clc_fmin.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMIN_H__
 #define __CLC_MATH_CLC_FMIN_H__
 
-#define __CLC_FUNCTION __clc_fmin
+#define FUNCTION __clc_fmin
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMIN_H__
diff --git a/libclc/clc/include/clc/math/clc_fmod.h b/libclc/clc/include/clc/math/clc_fmod.h
index 44179babd7fed..840bb850d73c9 100644
--- a/libclc/clc/include/clc/math/clc_fmod.h
+++ b/libclc/clc/include/clc/math/clc_fmod.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMOD_H__
 #define __CLC_MATH_CLC_FMOD_H__
 
-#define __CLC_FUNCTION __clc_fmod
+#define FUNCTION __clc_fmod
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMOD_H__
diff --git a/libclc/clc/include/clc/math/clc_fract.h b/libclc/clc/include/clc/math/clc_fract.h
index af762ecc59ea5..26c402c79641a 100644
--- a/libclc/clc/include/clc/math/clc_fract.h
+++ b/libclc/clc/include/clc/math/clc_fract.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FRACT_H__
 #define __CLC_MATH_CLC_FRACT_H__
 
-#define __CLC_FUNCTION __clc_fract
+#define FUNCTION __clc_fract
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FRACT_H__
diff --git a/libclc/clc/include/clc/math/clc_frexp.h b/libclc/clc/include/clc/math/clc_frexp.h
index d6ed2754d0d40..e3c9f6b363dd1 100644
--- a/libclc/clc/include/clc/math/clc_frexp.h
+++ b/libclc/clc/include/clc/math/clc_frexp.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_FREXP_H__
 #define __CLC_MATH_CLC_FREXP_H__
 
-#define __CLC_FUNCTION __clc_frexp
+#define FUNCTION __clc_frexp
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FREXP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_cos.h b/libclc/clc/include/clc/math/clc_half_cos.h
index d0efc45793af9..0f9f0b61443f9 100644
--- a/libclc/clc/include/clc/math/clc_half_cos.h
+++ b/libclc/clc/include/clc/math/clc_half_cos.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_COS_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_cos
+#define FUNCTION __clc_half_cos
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_half_divide.h b/libclc/clc/include/clc/math/clc_half_divide.h
index 5d1e7b9e33e39..44d90f39b4f61 100644
--- a/libclc/clc/include/clc/math/clc_half_divide.h
+++ b/libclc/clc/include/clc/math/clc_half_divide.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_DIVIDE_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_divide
+#define FUNCTION __clc_half_divide
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_DIVIDE_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp.h b/libclc/clc/include/clc/math/clc_half_exp.h
index e95f3d42085c8..158d28c16aabb 100644
--- a/libclc/clc/include/clc/math/clc_half_exp.h
+++ b/libclc/clc/include/clc/math/clc_half_exp.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_EXP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_exp
+#define FUNCTION __clc_half_exp
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp10.h b/libclc/clc/include/clc/math/clc_half_exp10.h
index e4cce6e63bf7a..4982b618e505a 100644
--- a/libclc/clc/include/clc/math/clc_half_exp10.h
+++ b/libclc/clc/include/clc/math/clc_half_exp10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_EXP10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_exp10
+#define FUNCTION __clc_half_exp10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp2.h b/libclc/clc/include/clc/math/clc_half_exp2.h
index 6fc99735f0928..dc0f67659bf9f 100644
--- a/libclc/clc/include/clc/math/clc_half_exp2.h
+++ b/libclc/clc/include/clc/math/clc_half_exp2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_EXP2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_exp2
+#define FUNCTION __clc_half_exp2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log.h b/libclc/clc/include/clc/math/clc_half_log.h
index e44e686499663..df527e1a8fb57 100644
--- a/libclc/clc/include/clc/math/clc_half_log.h
+++ b/libclc/clc/include/clc/math/clc_half_log.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_LOG_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_log
+#define FUNCTION __clc_half_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log10.h b/libclc/clc/include/clc/math/clc_half_log10.h
index 23e0ba116c728..47db4cd325931 100644
--- a/libclc/clc/include/clc/math/clc_half_log10.h
+++ b/libclc/clc/include/clc/math/clc_half_log10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_LOG10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_log10
+#define FUNCTION __clc_half_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log2.h b/libclc/clc/include/clc/math/clc_half_log2.h
index 8ea2439eb3db6..2b95b444b3f24 100644
--- a/libclc/clc/include/clc/math/clc_half_log2.h
+++ b/libclc/clc/include/clc/math/clc_half_log2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_LOG2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_log2
+#define FUNCTION __clc_half_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_half_powr.h b/libclc/clc/include/clc/math/clc_half_powr.h
index 252bfcd785043..2fd62213b8241 100644
--- a/libclc/clc/include/clc/math/clc_half_powr.h
+++ b/libclc/clc/include/clc/math/clc_half_powr.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_POWR_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_powr
+#define FUNCTION __clc_half_powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_half_recip.h b/libclc/clc/include/clc/math/clc_half_recip.h
index 1adb0bbe249cc..d2ace8b7a17e4 100644
--- a/libclc/clc/include/clc/math/clc_half_recip.h
+++ b/libclc/clc/include/clc/math/clc_half_recip.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_RECIP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_recip
+#define FUNCTION __clc_half_recip
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_RECIP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_rsqrt.h b/libclc/clc/include/clc/math/clc_half_rsqrt.h
index 6342739ed6f20..a50dabccdaafc 100644
--- a/libclc/clc/include/clc/math/clc_half_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_half_rsqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_RSQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_rsqrt
+#define FUNCTION __clc_half_rsqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_half_sin.h b/libclc/clc/include/clc/math/clc_half_sin.h
index 31378d290fecc..f79f6afc2b4ec 100644
--- a/libclc/clc/include/clc/math/clc_half_sin.h
+++ b/libclc/clc/include/clc/math/clc_half_sin.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_SIN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_sin
+#define FUNCTION __clc_half_sin
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_half_sqrt.h b/libclc/clc/include/clc/math/clc_half_sqrt.h
index 0e765fb3e7731..df5fc222a5c9f 100644
--- a/libclc/clc/include/clc/math/clc_half_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_half_sqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_SQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_sqrt
+#define FUNCTION __clc_half_sqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_half_tan.h b/libclc/clc/include/clc/math/clc_half_tan.h
index 6c890a952522c..0f6ef87907225 100644
--- a/libclc/clc/include/clc/math/clc_half_tan.h
+++ b/libclc/clc/include/clc/math/clc_half_tan.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_TAN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_tan
+#define FUNCTION __clc_half_tan
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_hypot.h b/libclc/clc/include/clc/math/clc_hypot.h
index 40b354154fa53..b2d69ebda96e2 100644
--- a/libclc/clc/include/clc/math/clc_hypot.h
+++ b/libclc/clc/include/clc/math/clc_hypot.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_HYPOT_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_hypot
+#define FUNCTION __clc_hypot
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HYPOT_H__
diff --git a/libclc/clc/include/clc/math/clc_ilogb.h b/libclc/clc/include/clc/math/clc_ilogb.h
index 6aeaf6be3ac6c..e02108ff1444b 100644
--- a/libclc/clc/include/clc/math/clc_ilogb.h
+++ b/libclc/clc/include/clc/math/clc_ilogb.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_ILOGB_H__
 #define __CLC_MATH_CLC_ILOGB_H__
 
-#define __CLC_FUNCTION __clc_ilogb
+#define FUNCTION __clc_ilogb
 #define __CLC_BODY <clc/math/unary_decl_with_int_return.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ILOGB_H__
diff --git a/libclc/clc/include/clc/math/clc_lgamma.h b/libclc/clc/include/clc/math/clc_lgamma.h
index f54c5e2b61b67..69b1ff9701ca5 100644
--- a/libclc/clc/include/clc/math/clc_lgamma.h
+++ b/libclc/clc/include/clc/math/clc_lgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_LGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_lgamma
+#define FUNCTION __clc_lgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LGAMMA_H__
diff --git a/libclc/clc/include/clc/math/clc_lgamma_r.h b/libclc/clc/include/clc/math/clc_lgamma_r.h
index 262e62372227a..a98f760d8032e 100644
--- a/libclc/clc/include/clc/math/clc_lgamma_r.h
+++ b/libclc/clc/include/clc/math/clc_lgamma_r.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LGAMMA_R_H__
 #define __CLC_MATH_CLC_LGAMMA_R_H__
 
-#define __CLC_FUNCTION __clc_lgamma_r
+#define FUNCTION __clc_lgamma_r
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LGAMMA_R_H__
diff --git a/libclc/clc/include/clc/math/clc_log.h b/libclc/clc/include/clc/math/clc_log.h
index 8b0e44b425a89..6bddb5ff746ec 100644
--- a/libclc/clc/include/clc/math/clc_log.h
+++ b/libclc/clc/include/clc/math/clc_log.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG_H__
 #define __CLC_MATH_CLC_LOG_H__
 
-#define __CLC_FUNCTION __clc_log
+#define FUNCTION __clc_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_log10.h b/libclc/clc/include/clc/math/clc_log10.h
index 6e086690c6ada..e3a5b19eb4184 100644
--- a/libclc/clc/include/clc/math/clc_log10.h
+++ b/libclc/clc/include/clc/math/clc_log10.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG10_H__
 #define __CLC_MATH_CLC_LOG10_H__
 
-#define __CLC_FUNCTION __clc_log10
+#define FUNCTION __clc_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_log1p.h b/libclc/clc/include/clc/math/clc_log1p.h
index 1be8c8548d488..f502802dafb22 100644
--- a/libclc/clc/include/clc/math/clc_log1p.h
+++ b/libclc/clc/include/clc/math/clc_log1p.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_LOG1P_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_log1p
+#define FUNCTION __clc_log1p
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG1P_H__
diff --git a/libclc/clc/include/clc/math/clc_log2.h b/libclc/clc/include/clc/math/clc_log2.h
index c5f9e37f39845..7344cabaeffaa 100644
--- a/libclc/clc/include/clc/math/clc_log2.h
+++ b/libclc/clc/include/clc/math/clc_log2.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG2_H__
 #define __CLC_MATH_CLC_LOG2_H__
 
-#define __CLC_FUNCTION __clc_log2
+#define FUNCTION __clc_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_logb.h b/libclc/clc/include/clc/math/clc_logb.h
index 75a2ce7e578b1..1339d5819171d 100644
--- a/libclc/clc/include/clc/math/clc_logb.h
+++ b/libclc/clc/include/clc/math/clc_logb.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_LOGB_H__
 #define __CLC_MATH_CLC_LOGB_H__
 
-#define __CLC_FUNCTION __clc_logb
+#define FUNCTION __clc_logb
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOGB_H__
diff --git a/libclc/clc/include/clc/math/clc_mad.h b/libclc/clc/include/clc/math/clc_mad.h
index 117e361380eda..72014b7c0ef7b 100644
--- a/libclc/clc/include/clc/math/clc_mad.h
+++ b/libclc/clc/include/clc/math/clc_mad.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MAD_H__
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define __CLC_FUNCTION __clc_mad
+#define FUNCTION __clc_mad
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MAD_H__
diff --git a/libclc/clc/include/clc/math/clc_maxmag.h b/libclc/clc/include/clc/math/clc_maxmag.h
index 857ad7867b05d..058230e6ebfc9 100644
--- a/libclc/clc/include/clc/math/clc_maxmag.h
+++ b/libclc/clc/include/clc/math/clc_maxmag.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MAXMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_maxmag
+#define FUNCTION __clc_maxmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MAXMAG_H__
diff --git a/libclc/clc/include/clc/math/clc_minmag.h b/libclc/clc/include/clc/math/clc_minmag.h
index 9615d029bfed2..581b957828235 100644
--- a/libclc/clc/include/clc/math/clc_minmag.h
+++ b/libclc/clc/include/clc/math/clc_minmag.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MINMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_minmag
+#define FUNCTION __clc_minmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MINMAG_H__
diff --git a/libclc/clc/include/clc/math/clc_modf.h b/libclc/clc/include/clc/math/clc_modf.h
index aa8f9e20892ab..fbdd3f8ffd4b8 100644
--- a/libclc/clc/include/clc/math/clc_modf.h
+++ b/libclc/clc/include/clc/math/clc_modf.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_MODF_H__
 #define __CLC_MATH_CLC_MODF_H__
 
-#define __CLC_FUNCTION __clc_modf
+#define FUNCTION __clc_modf
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MODF_H__
diff --git a/libclc/clc/include/clc/math/clc_nan.h b/libclc/clc/include/clc/math/clc_nan.h
index 91901fd9369f2..45d91184cf145 100644
--- a/libclc/clc/include/clc/math/clc_nan.h
+++ b/libclc/clc/include/clc/math/clc_nan.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_NAN_H__
 #define __CLC_MATH_CLC_NAN_H__
 
-#define __CLC_FUNCTION __clc_nan
+#define FUNCTION __clc_nan
 #define __CLC_BODY <clc/math/clc_nan.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NAN_H__
diff --git a/libclc/clc/include/clc/math/clc_nan.inc b/libclc/clc/include/clc/math/clc_nan.inc
index 637928f0839fd..22604e0a5785e 100644
--- a/libclc/clc/include/clc/math/clc_nan.inc
+++ b/libclc/clc/include/clc/math/clc_nan.inc
@@ -7,3 +7,4 @@
 //===----------------------------------------------------------------------===//
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_U_GENTYPE code);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_S_GENTYPE code);
diff --git a/libclc/clc/include/clc/math/clc_native_cos.h b/libclc/clc/include/clc/math/clc_native_cos.h
index f5837d5479133..08d51baca1f61 100644
--- a/libclc/clc/include/clc/math/clc_native_cos.h
+++ b/libclc/clc/include/clc/math/clc_native_cos.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_COS_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_cos
+#define FUNCTION __clc_native_cos
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_native_divide.h b/libclc/clc/include/clc/math/clc_native_divide.h
index ed8118775e98b..a10ed512f10d9 100644
--- a/libclc/clc/include/clc/math/clc_native_divide.h
+++ b/libclc/clc/include/clc/math/clc_native_divide.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_DIVIDE_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_divide
+#define FUNCTION __clc_native_divide
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_DIVIDE_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp.h b/libclc/clc/include/clc/math/clc_native_exp.h
index b9c4005585daa..30801fe2f8ad3 100644
--- a/libclc/clc/include/clc/math/clc_native_exp.h
+++ b/libclc/clc/include/clc/math/clc_native_exp.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_EXP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_exp
+#define FUNCTION __clc_native_exp
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp10.h b/libclc/clc/include/clc/math/clc_native_exp10.h
index 5c49e46330d60..91cf60ef6a2be 100644
--- a/libclc/clc/include/clc/math/clc_native_exp10.h
+++ b/libclc/clc/include/clc/math/clc_native_exp10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_EXP10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_exp10
+#define FUNCTION __clc_native_exp10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp2.h b/libclc/clc/include/clc/math/clc_native_exp2.h
index 2fa368f69fef4..fe47280536198 100644
--- a/libclc/clc/include/clc/math/clc_native_exp2.h
+++ b/libclc/clc/include/clc/math/clc_native_exp2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_EXP2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_exp2
+#define FUNCTION __clc_native_exp2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log.h b/libclc/clc/include/clc/math/clc_native_log.h
index 61fe0c0902d85..25a7054d2a599 100644
--- a/libclc/clc/include/clc/math/clc_native_log.h
+++ b/libclc/clc/include/clc/math/clc_native_log.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_LOG_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_log
+#define FUNCTION __clc_native_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log10.h b/libclc/clc/include/clc/math/clc_native_log10.h
index f9fca1d94aedb..d46124d655ee7 100644
--- a/libclc/clc/include/clc/math/clc_native_log10.h
+++ b/libclc/clc/include/clc/math/clc_native_log10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_LOG10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_log10
+#define FUNCTION __clc_native_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log2.h b/libclc/clc/include/clc/math/clc_native_log2.h
index 69aec752b832b..8998553b4984a 100644
--- a/libclc/clc/include/clc/math/clc_native_log2.h
+++ b/libclc/clc/include/clc/math/clc_native_log2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_LOG2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_log2
+#define FUNCTION __clc_native_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_native_powr.h b/libclc/clc/include/clc/math/clc_native_powr.h
index 8d50f21a3124d..932cfa66aeb9c 100644
--- a/libclc/clc/include/clc/math/clc_native_powr.h
+++ b/libclc/clc/include/clc/math/clc_native_powr.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_POWR_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_powr
+#define FUNCTION __clc_native_powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_native_recip.h b/libclc/clc/include/clc/math/clc_native_recip.h
index 94d494ad07a17..6b94cbc3d6c6e 100644
--- a/libclc/clc/include/clc/math/clc_native_recip.h
+++ b/libclc/clc/include/clc/math/clc_native_recip.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_RECIP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_recip
+#define FUNCTION __clc_native_recip
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_RECIP_H__
diff --git a/libclc/clc/include/clc/math/clc_native_rsqrt.h b/libclc/clc/include/clc/math/clc_native_rsqrt.h
index 5b58d6efd7584..5fa870b3f868a 100644
--- a/libclc/clc/include/clc/math/clc_native_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_native_rsqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_RSQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_rsqrt
+#define FUNCTION __clc_native_rsqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_native_sin.h b/libclc/clc/include/clc/math/clc_native_sin.h
index 27f5ecbf40e72..d25968c74409e 100644
--- a/libclc/clc/include/clc/math/clc_native_sin.h
+++ b/libclc/clc/include/clc/math/clc_native_sin.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_SIN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_sin
+#define FUNCTION __clc_native_sin
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_native_sqrt.h b/libclc/clc/include/clc/math/clc_native_sqrt.h
index a45f8b7f21a21..e8a28b7f70463 100644
--- a/libclc/clc/include/clc/math/clc_native_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_native_sqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_SQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_sqrt
+#define FUNCTION __clc_native_sqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_native_tan.h b/libclc/clc/include/clc/math/clc_native_tan.h
index 362088a5d58f3..36fdd6d8c011b 100644
--- a/libclc/clc/include/clc/math/clc_native_tan.h
+++ b/libclc/clc/include/clc/math/clc_native_tan.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_TAN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_tan
+#define FUNCTION __clc_native_tan
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_nextafter.h b/libclc/clc/include/clc/math/clc_nextafter.h
index f24742b9e9c37..d080a40a06f03 100644
--- a/libclc/clc/include/clc/math/clc_nextafter.h
+++ b/libclc/clc/include/clc/math/clc_nextafter.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_NEXTAFTER_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_nextafter
+#define FUNCTION __clc_nextafter
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NEXTAFTER_H__
diff --git a/libclc/clc/include/clc/math/clc_pow.h b/libclc/clc/include/clc/math/clc_pow.h
index 5e37e5bf6da65..b33391e10a42a 100644
--- a/libclc/clc/include/clc/math/clc_pow.h
+++ b/libclc/clc/include/clc/math/clc_pow.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POW_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_pow
+#define FUNCTION __clc_pow
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_POW_H__
diff --git a/libclc/clc/include/clc/math/clc_pown.h b/libclc/clc/include/clc/math/clc_pown.h
index 30628efb19001..473bbe1603c52 100644
--- a/libclc/clc/include/clc/math/clc_pown.h
+++ b/libclc/clc/include/clc/math/clc_pown.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POWN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
-#define __CLC_FUNCTION __clc_pown
+#define FUNCTION __clc_pown
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_POWN_H__
diff --git a/libclc/clc/include/clc/math/clc_powr.h b/libclc/clc/include/clc/math/clc_powr.h
index baa494cce6989..a820516f18660 100644
--- a/libclc/clc/include/clc/math/clc_powr.h
+++ b/libclc/clc/include/clc/math/clc_powr.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POWR_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_powr
+#define FUNCTION __clc_powr
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_remainder.h b/libclc/clc/include/clc/math/clc_remainder.h
index a7700c95103b4..bc53b043045cd 100644
--- a/libclc/clc/include/clc/math/clc_remainder.h
+++ b/libclc/clc/include/clc/math/clc_remainder.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_REMAINDER_H__
 #define __CLC_MATH_CLC_REMAINDER_H__
 
-#define __CLC_FUNCTION __clc_remainder
+#define FUNCTION __clc_remainder
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_REMAINDER_H__
diff --git a/libclc/clc/include/clc/math/clc_remquo.h b/libclc/clc/include/clc/math/clc_remquo.h
index 48a8844a6e384..b46f94ba94477 100644
--- a/libclc/clc/include/clc/math/clc_remquo.h
+++ b/libclc/clc/include/clc/math/clc_remquo.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_REMQUO_H__
 #define __CLC_MATH_CLC_REMQUO_H__
 
-#define __CLC_FUNCTION __clc_remquo
+#define FUNCTION __clc_remquo
 #define __CLC_BODY <clc/math/remquo_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_REMQUO_H__
diff --git a/libclc/clc/include/clc/math/clc_rint.h b/libclc/clc/include/clc/math/clc_rint.h
index 17b1549f439b8..c446c75764710 100644
--- a/libclc/clc/include/clc/math/clc_rint.h
+++ b/libclc/clc/include/clc/math/clc_rint.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_RINT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_rint
+#define FUNCTION __clc_rint
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_RINT_H__
diff --git a/libclc/clc/include/clc/math/clc_rootn.h b/libclc/clc/include/clc/math/clc_rootn.h
index 90a25ad52d867..fb7817ee250fd 100644
--- a/libclc/clc/include/clc/math/clc_rootn.h
+++ b/libclc/clc/include/clc/math/clc_rootn.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ROOTN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
-#define __CLC_FUNCTION __clc_rootn
+#define FUNCTION __clc_rootn
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ROOTN_H__
diff --git a/libclc/clc/include/clc/math/clc_round.h b/libclc/clc/include/clc/math/clc_round.h
index 78ebf4db99e9f..fd1872a5cf224 100644
--- a/libclc/clc/include/clc/math/clc_round.h
+++ b/libclc/clc/include/clc/math/clc_round.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ROUND_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_round
+#define FUNCTION __clc_round
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ROUND_H__
diff --git a/libclc/clc/include/clc/math/clc_rsqrt.h b/libclc/clc/include/clc/math/clc_rsqrt.h
index 59402fbead2b2..1690ede1b8926 100644
--- a/libclc/clc/include/clc/math/clc_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_rsqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_RSQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_rsqrt
+#define FUNCTION __clc_rsqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_sin.h b/libclc/clc/include/clc/math/clc_sin.h
index de4c722ca123f..89ae444eb6e30 100644
--- a/libclc/clc/include/clc/math/clc_sin.h
+++ b/libclc/clc/include/clc/math/clc_sin.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sin
+#define FUNCTION __clc_sin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_sincos.h b/libclc/clc/include/clc/math/clc_sincos.h
index e26dc7c024c9c..bf3e0806db381 100644
--- a/libclc/clc/include/clc/math/clc_sincos.h
+++ b/libclc/clc/include/clc/math/clc_sincos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINCOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
-#define __CLC_FUNCTION __clc_sincos
+#define FUNCTION __clc_sincos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SINCOS_H__
diff --git a/libclc/clc/include/clc/math/clc_sincos_helpers.h b/libclc/clc/include/clc/math/clc_sincos_helpers.h
index e3a2e1c0b605c..f9ceba3bf2cf7 100644
--- a/libclc/clc/include/clc/math/clc_sincos_helpers.h
+++ b/libclc/clc/include/clc/math/clc_sincos_helpers.h
@@ -14,13 +14,9 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-
 #define __DOUBLE_ONLY
 #define __CLC_BODY <clc/math/clc_sincos_helpers_fp64.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __DOUBLE_ONLY
-
 #endif // __CLC_MATH_CLC_SINCOS_HELPERS_H__
diff --git a/libclc/clc/include/clc/math/clc_sinh.h b/libclc/clc/include/clc/math/clc_sinh.h
index fa9cabe79cb0e..c94d8e37acc64 100644
--- a/libclc/clc/include/clc/math/clc_sinh.h
+++ b/libclc/clc/include/clc/math/clc_sinh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sinh
+#define FUNCTION __clc_sinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SINH_H__
diff --git a/libclc/clc/include/clc/math/clc_sinpi.h b/libclc/clc/include/clc/math/clc_sinpi.h
index db563c2e7e14a..725e559600edd 100644
--- a/libclc/clc/include/clc/math/clc_sinpi.h
+++ b/libclc/clc/include/clc/math/clc_sinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sinpi
+#define FUNCTION __clc_sinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SINPI_H__
diff --git a/libclc/clc/include/clc/math/clc_sqrt.h b/libclc/clc/include/clc/math/clc_sqrt.h
index 7084515533291..74aea8e858099 100644
--- a/libclc/clc/include/clc/math/clc_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_sqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sqrt
+#define FUNCTION __clc_sqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_tan.h b/libclc/clc/include/clc/math/clc_tan.h
index 028ff28ecd69d..d16370420a030 100644
--- a/libclc/clc/include/clc/math/clc_tan.h
+++ b/libclc/clc/include/clc/math/clc_tan.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tan
+#define FUNCTION __clc_tan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_tanh.h b/libclc/clc/include/clc/math/clc_tanh.h
index 88fce2340d99b..2c860c165ca67 100644
--- a/libclc/clc/include/clc/math/clc_tanh.h
+++ b/libclc/clc/include/clc/math/clc_tanh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tanh
+#define FUNCTION __clc_tanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TANH_H__
diff --git a/libclc/clc/include/clc/math/clc_tanpi.h b/libclc/clc/include/clc/math/clc_tanpi.h
index 596bb72b4b8de..2691fa696c0a5 100644
--- a/libclc/clc/include/clc/math/clc_tanpi.h
+++ b/libclc/clc/include/clc/math/clc_tanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tanpi
+#define FUNCTION __clc_tanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TANPI_H__
diff --git a/libclc/clc/include/clc/math/clc_tgamma.h b/libclc/clc/include/clc/math/clc_tgamma.h
index dd76e041875d5..d067703c2e9df 100644
--- a/libclc/clc/include/clc/math/clc_tgamma.h
+++ b/libclc/clc/include/clc/math/clc_tgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tgamma
+#define FUNCTION __clc_tgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TGAMMA_H__
diff --git a/libclc/clc/include/clc/math/clc_trunc.h b/libclc/clc/include/clc/math/clc_trunc.h
index 33445f9fe15d6..6a8353a4f6d45 100644
--- a/libclc/clc/include/clc/math/clc_trunc.h
+++ b/libclc/clc/include/clc/math/clc_trunc.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TRUNC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_trunc
+#define FUNCTION __clc_trunc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TRUNC_H__
diff --git a/libclc/clc/include/clc/math/gentype.inc b/libclc/clc/include/clc/math/gentype.inc
index a8d8317a9cacb..d74a9932bd0a6 100644
--- a/libclc/clc/include/clc/math/gentype.inc
+++ b/libclc/clc/include/clc/math/gentype.inc
@@ -349,3 +349,7 @@
 
 #undef __CLC_AS_GENTYPE
 #undef __CLC_CONVERT_GENTYPE
+
+#undef __HALF_ONLY
+#undef __FLOAT_ONLY
+#undef __DOUBLE_ONLY
diff --git a/libclc/clc/include/clc/math/remquo_decl.inc b/libclc/clc/include/clc/math/remquo_decl.inc
index cba28a7244eb4..24d96b0488059 100644
--- a/libclc/clc/include/clc/math/remquo_decl.inc
+++ b/libclc/clc/include/clc/math/remquo_decl.inc
@@ -6,19 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     private __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               private __CLC_INTN *q);
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     global __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               global __CLC_INTN *q);
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     local __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               local __CLC_INTN *q);
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     generic __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               generic __CLC_INTN *q);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_decl.inc b/libclc/clc/include/clc/math/unary_decl.inc
index 5d742de00b711..ce1a71b8284aa 100644
--- a/libclc/clc/include/clc/math/unary_decl.inc
+++ b/libclc/clc/include/clc/math/unary_decl.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc b/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
index 8bfe17c3c6817..d62046a5292cd 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     global __CLC_INTN *iptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     local __CLC_INTN *iptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     private __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               global __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               local __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               private __CLC_INTN *iptr);
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     generic __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               generic __CLC_INTN *iptr);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_decl_with_int_return.inc b/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
index 333a752ced39e..ed8d0a12506f2 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/math/unary_decl_with_ptr.inc b/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
index 63a20e3d21c5d..ca0077bec558f 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     global __CLC_GENTYPE *ptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     local __CLC_GENTYPE *ptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_GENTYPE x, private __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               global __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               local __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               private __CLC_GENTYPE *ptr);
 
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_GENTYPE x, generic __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               generic __CLC_GENTYPE *ptr);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_def_via_fp32.inc b/libclc/clc/include/clc/math/unary_def_via_fp32.inc
index aefee6170571d..f109e8ec4a111 100644
--- a/libclc/clc/include/clc/math/unary_def_via_fp32.inc
+++ b/libclc/clc/include/clc/math/unary_def_via_fp32.inc
@@ -6,6 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x) {
-  return __CLC_CONVERT_GENTYPE(__CLC_FUNCTION(__CLC_CONVERT_FLOATN(x)));
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x) {
+  return __CLC_CONVERT_GENTYPE(FUNCTION(__CLC_CONVERT_FLOATN(x)));
 }
diff --git a/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc b/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
index 464c6ffc7c8ca..a8ed34ad04bbf 100644
--- a/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
@@ -8,28 +8,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               private __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               global __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               local __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               generic __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 #endif
diff --git a/libclc/clc/include/clc/math/unary_def_with_int_return.inc b/libclc/clc/include/clc/math/unary_def_with_int_return.inc
index 33be58bf45842..c7bb84cf5f7c8 100644
--- a/libclc/clc/include/clc/math/unary_def_with_int_return.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_int_return.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_GENTYPE a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
diff --git a/libclc/clc/include/clc/math/unary_def_with_ptr.inc b/libclc/clc/include/clc/math/unary_def_with_ptr.inc
index fecec80dde834..c3faa9aab87c9 100644
--- a/libclc/clc/include/clc/math/unary_def_with_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_ptr.inc
@@ -8,28 +8,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               private __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               global __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               local __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               generic __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 #endif
diff --git a/libclc/clc/include/clc/misc/clc_shuffle.h b/libclc/clc/include/clc/misc/clc_shuffle.h
index ba3d7a2bddd62..1e9ec6828f95a 100644
--- a/libclc/clc/include/clc/misc/clc_shuffle.h
+++ b/libclc/clc/include/clc/misc/clc_shuffle.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MISC_CLC_SHUFFLE_H__
 #define __CLC_MISC_CLC_SHUFFLE_H__
 
-#define __CLC_FUNCTION __clc_shuffle
+#define FUNCTION __clc_shuffle
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MISC_CLC_SHUFFLE_H__
diff --git a/libclc/clc/include/clc/misc/clc_shuffle2.h b/libclc/clc/include/clc/misc/clc_shuffle2.h
index 84bafba12b211..fb5361cfeb95a 100644
--- a/libclc/clc/include/clc/misc/clc_shuffle2.h
+++ b/libclc/clc/include/clc/misc/clc_shuffle2.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MISC_CLC_SHUFFLE2_H__
 #define __CLC_MISC_CLC_SHUFFLE2_H__
 
-#define __CLC_FUNCTION __clc_shuffle2
+#define FUNCTION __clc_shuffle2
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MISC_CLC_SHUFFLE2_H__
diff --git a/libclc/clc/include/clc/misc/shuffle2_decl.inc b/libclc/clc/include/clc/misc/shuffle2_decl.inc
index 2fc992ef2c177..47fd5e28a3142 100644
--- a/libclc/clc/include/clc/misc/shuffle2_decl.inc
+++ b/libclc/clc/include/clc/misc/shuffle2_decl.inc
@@ -13,16 +13,16 @@
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask);
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle2_def.inc b/libclc/clc/include/clc/misc/shuffle2_def.inc
index 099a175d665dd..0415b8c187291 100644
--- a/libclc/clc/include/clc/misc/shuffle2_def.inc
+++ b/libclc/clc/include/clc/misc/shuffle2_def.inc
@@ -11,8 +11,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // The return type is same base type as the input type, with the same vector
@@ -21,22 +21,22 @@
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle_decl.inc b/libclc/clc/include/clc/misc/shuffle_decl.inc
index 5e7e5b24c2873..c6c64d5b5fede 100644
--- a/libclc/clc/include/clc/misc/shuffle_decl.inc
+++ b/libclc/clc/include/clc/misc/shuffle_decl.inc
@@ -13,12 +13,12 @@
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask);
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle_def.inc b/libclc/clc/include/clc/misc/shuffle_def.inc
index 84c873e6458ec..d2a088498fc6c 100644
--- a/libclc/clc/include/clc/misc/shuffle_def.inc
+++ b/libclc/clc/include/clc/misc/shuffle_def.inc
@@ -11,8 +11,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // The return type is same base type as the input type, with the same vector
@@ -20,19 +20,19 @@
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/relational/binary_decl.inc b/libclc/clc/include/clc/relational/binary_decl.inc
index b82fb85d9742c..bcdf5238b8f58 100644
--- a/libclc/clc/include/clc/relational/binary_decl.inc
+++ b/libclc/clc/include/clc/relational/binary_decl.inc
@@ -6,5 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_FLOATN a,
-                                                  __CLC_FLOATN b);
+_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_FLOATN a, __CLC_FLOATN b);
diff --git a/libclc/clc/include/clc/relational/clc_isfinite.h b/libclc/clc/include/clc/relational/clc_isfinite.h
index 37483edd37613..5e71ec7a0640a 100644
--- a/libclc/clc/include/clc/relational/clc_isfinite.h
+++ b/libclc/clc/include/clc/relational/clc_isfinite.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISFINITE_H__
 #define __CLC_RELATIONAL_CLC_ISFINITE_H__
 
-#define __CLC_FUNCTION __clc_isfinite
+#define FUNCTION __clc_isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISFINITE_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreater.h b/libclc/clc/include/clc/relational/clc_isgreater.h
index 4ce18ee631712..e2e6911a80cdd 100644
--- a/libclc/clc/include/clc/relational/clc_isgreater.h
+++ b/libclc/clc/include/clc/relational/clc_isgreater.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISGREATER_H__
 
-#define __CLC_FUNCTION __clc_isgreater
+#define FUNCTION __clc_isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreaterequal.h b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
index c0a430cc61e97..3fe8835aff9d5 100644
--- a/libclc/clc/include/clc/relational/clc_isgreaterequal.h
+++ b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 
-#define __CLC_FUNCTION __clc_isgreaterequal
+#define FUNCTION __clc_isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isless.h b/libclc/clc/include/clc/relational/clc_isless.h
index e680793970620..01384cf6fa4a0 100644
--- a/libclc/clc/include/clc/relational/clc_isless.h
+++ b/libclc/clc/include/clc/relational/clc_isless.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESS_H__
 #define __CLC_RELATIONAL_CLC_ISLESS_H__
 
-#define __CLC_FUNCTION __clc_isless
+#define FUNCTION __clc_isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESS_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessequal.h b/libclc/clc/include/clc/relational/clc_islessequal.h
index 8e95d04bf4752..a4b77a451b248 100644
--- a/libclc/clc/include/clc/relational/clc_islessequal.h
+++ b/libclc/clc/include/clc/relational/clc_islessequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 
-#define __CLC_FUNCTION __clc_islessequal
+#define FUNCTION __clc_islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessgreater.h b/libclc/clc/include/clc/relational/clc_islessgreater.h
index 69c43e40b463e..9fb6d641bfa14 100644
--- a/libclc/clc/include/clc/relational/clc_islessgreater.h
+++ b/libclc/clc/include/clc/relational/clc_islessgreater.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 
-#define __CLC_FUNCTION __clc_islessgreater
+#define FUNCTION __clc_islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnormal.h b/libclc/clc/include/clc/relational/clc_isnormal.h
index 4fee10a022444..d580fed5a7395 100644
--- a/libclc/clc/include/clc/relational/clc_isnormal.h
+++ b/libclc/clc/include/clc/relational/clc_isnormal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNORMAL_H__
 #define __CLC_RELATIONAL_CLC_ISNORMAL_H__
 
-#define __CLC_FUNCTION __clc_isnormal
+#define FUNCTION __clc_isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISNORMAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnotequal.h b/libclc/clc/include/clc/relational/clc_isnotequal.h
index 3aac82cfb9387..16982fc3c5aaa 100644
--- a/libclc/clc/include/clc/relational/clc_isnotequal.h
+++ b/libclc/clc/include/clc/relational/clc_isnotequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 
-#define __CLC_FUNCTION __clc_isnotequal
+#define FUNCTION __clc_isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isordered.h b/libclc/clc/include/clc/relational/clc_isordered.h
index 20b5adb0cce72..7ba26662105fc 100644
--- a/libclc/clc/include/clc/relational/clc_isordered.h
+++ b/libclc/clc/include/clc/relational/clc_isordered.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISORDERED_H__
 
-#define __CLC_FUNCTION __clc_isordered
+#define FUNCTION __clc_isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_isunordered.h b/libclc/clc/include/clc/relational/clc_isunordered.h
index 7e693233ba344..eac158d245191 100644
--- a/libclc/clc/include/clc/relational/clc_isunordered.h
+++ b/libclc/clc/include/clc/relational/clc_isunordered.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 
-#define __CLC_FUNCTION __clc_isunordered
+#define FUNCTION __clc_isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISUNORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_signbit.h b/libclc/clc/include/clc/relational/clc_signbit.h
index 55159c9cc56d1..892263a09e99c 100644
--- a/libclc/clc/include/clc/relational/clc_signbit.h
+++ b/libclc/clc/include/clc/relational/clc_signbit.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_SIGNBIT_H__
 #define __CLC_RELATIONAL_CLC_SIGNBIT_H__
 
-#define __CLC_FUNCTION __clc_signbit
+#define FUNCTION __clc_signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_SIGNBIT_H__
diff --git a/libclc/clc/include/clc/relational/unary_decl.inc b/libclc/clc/include/clc/relational/unary_decl.inc
index cff02d0cb9c47..b9fb36c905469 100644
--- a/libclc/clc/include/clc/relational/unary_decl.inc
+++ b/libclc/clc/include/clc/relational/unary_decl.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_FLOATN x);
+_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_FLOATN x);
diff --git a/libclc/clc/include/clc/shared/binary_decl.inc b/libclc/clc/include/clc/shared/binary_decl.inc
index 589a31a7305ac..ff4739dcdb8d1 100644
--- a/libclc/clc/include/clc/shared/binary_decl.inc
+++ b/libclc/clc/include/clc/shared/binary_decl.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               __CLC_GENTYPE y);
diff --git a/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc b/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
index 1a854abe11d7a..a86d89b4fc85f 100644
--- a/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
@@ -8,5 +8,4 @@
 
 #include <clc/utils.h>
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_INTN y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_INTN y);
diff --git a/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc b/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
index 6044902a5be7c..05a53393170f6 100644
--- a/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               __CLC_GENTYPE y);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_SCALAR_GENTYPE y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               __CLC_SCALAR_GENTYPE y);
 #endif
diff --git a/libclc/clc/include/clc/shared/binary_def.inc b/libclc/clc/include/clc/shared/binary_def.inc
index 646e1504f4826..9ab5a7c5a79e5 100644
--- a/libclc/clc/include/clc/shared/binary_def.inc
+++ b/libclc/clc/include/clc/shared/binary_def.inc
@@ -8,11 +8,11 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                               __CLC_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
diff --git a/libclc/clc/include/clc/shared/binary_def_scalarize.inc b/libclc/clc/include/clc/shared/binary_def_scalarize.inc
new file mode 100644
index 0000000000000..53d4d3fa23b13
--- /dev/null
+++ b/libclc/clc/include/clc/shared/binary_def_scalarize.inc
@@ -0,0 +1,131 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/utils.h>
+
+#ifdef __CLC_SCALAR
+
+#ifndef __CLC_MIN_VECSIZE
+#define __CLC_MIN_VECSIZE 2
+#endif
+
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION FUNCTION
+#endif
+
+#ifndef __CLC_DEF_SPEC
+#define __CLC_DEF_SPEC _CLC_DEF
+#endif
+
+#ifndef __CLC_RET_TYPE
+#define __CLC_RET_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG1_TYPE
+#define __CLC_ARG1_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG2_TYPE
+#define __CLC_ARG2_TYPE __CLC_GENTYPE
+#endif
+
+#if __CLC_MIN_VECSIZE == 1
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
+                                                     __CLC_ARG2_TYPE y) {
+  return __IMPL_FUNCTION(x, y);
+}
+#endif // __CLC_MIN_VECSIZE == 1
+
+#if __CLC_MIN_VECSIZE <= 2
+#define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
+#define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
+#define __CLC_ARG2_TYPE2 __CLC_XCONCAT(__CLC_ARG2_TYPE, 2)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
+                                                      __CLC_ARG2_TYPE2 y) {
+  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0, y.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1));
+}
+#undef __CLC_RET_TYPE2
+#undef __CLC_ARG1_TYPE2
+#undef __CLC_ARG2_TYPE2
+#endif // __CLC_MIN_VECSIZE <= 2
+
+#if __CLC_MIN_VECSIZE > 3
+#error "__CLC_MIN_VECSIZE > 3 isn't implemented"
+#endif
+
+#define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
+#define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
+#define __CLC_ARG2_TYPE3 __CLC_XCONCAT(__CLC_ARG2_TYPE, 3)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
+                                                      __CLC_ARG2_TYPE3 y) {
+  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0, y.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2));
+}
+#undef __CLC_RET_TYPE3
+#undef __CLC_ARG1_TYPE3
+#undef __CLC_ARG2_TYPE3
+
+#define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
+#define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
+#define __CLC_ARG2_TYPE4 __CLC_XCONCAT(__CLC_ARG2_TYPE, 4)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
+                                                      __CLC_ARG2_TYPE4 y) {
+  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0, y.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2),
+                           __IMPL_FUNCTION(x.s3, y.s3));
+}
+#undef __CLC_RET_TYPE4
+#undef __CLC_ARG1_TYPE4
+#undef __CLC_ARG2_TYPE4
+
+#define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
+#define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
+#define __CLC_ARG2_TYPE8 __CLC_XCONCAT(__CLC_ARG2_TYPE, 8)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
+                                                      __CLC_ARG2_TYPE8 y) {
+  return (
+      __CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0, y.s0), __IMPL_FUNCTION(x.s1, y.s1),
+                       __IMPL_FUNCTION(x.s2, y.s2), __IMPL_FUNCTION(x.s3, y.s3),
+                       __IMPL_FUNCTION(x.s4, y.s4), __IMPL_FUNCTION(x.s5, y.s5),
+                       __IMPL_FUNCTION(x.s6, y.s6),
+                       __IMPL_FUNCTION(x.s7, y.s7));
+}
+#undef __CLC_RET_TYPE8
+#undef __CLC_ARG1_TYPE8
+#undef __CLC_ARG2_TYPE8
+
+#define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
+#define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
+#define __CLC_ARG2_TYPE16 __CLC_XCONCAT(__CLC_ARG2_TYPE, 16)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x,
+                                                       __CLC_ARG2_TYPE16 y) {
+  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0, y.s0),
+                            __IMPL_FUNCTION(x.s1, y.s1),
+                            __IMPL_FUNCTION(x.s2, y.s2),
+                            __IMPL_FUNCTION(x.s3, y.s3),
+                            __IMPL_FUNCTION(x.s4, y.s4),
+                            __IMPL_FUNCTION(x.s5, y.s5),
+                            __IMPL_FUNCTION(x.s6, y.s6),
+                            __IMPL_FUNCTION(x.s7, y.s7),
+                            __IMPL_FUNCTION(x.s8, y.s8),
+                            __IMPL_FUNCTION(x.s9, y.s9),
+                            __IMPL_FUNCTION(x.sa, y.sa),
+                            __IMPL_FUNCTION(x.sb, y.sb),
+                            __IMPL_FUNCTION(x.sc, y.sc),
+                            __IMPL_FUNCTION(x.sd, y.sd),
+                            __IMPL_FUNCTION(x.se, y.se),
+                            __IMPL_FUNCTION(x.sf, y.sf));
+}
+#undef __CLC_RET_TYPE16
+#undef __CLC_ARG1_TYPE16
+#undef __CLC_ARG2_TYPE16
+
+#endif // __CLC_SCALAR
diff --git a/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc b/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
index 756bce6538ab4..262b6b21cc648 100644
--- a/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_INTN y) {
-  return __CLC_FUNCTION(FUNCTION)(x, y);
+  return __IMPL_FUNCTION(FUNCTION)(x, y);
 }
diff --git a/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc b/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
index 49fa7d2ce36e9..103f775ab87e5 100644
--- a/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
@@ -8,18 +8,18 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                               __CLC_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
 
 #ifndef __CLC_SCALAR
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                               __CLC_SCALAR_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, (__CLC_GENTYPE)b);
+  return __IMPL_FUNCTION(FUNCTION)(a, (__CLC_GENTYPE)b);
 }
 #endif
diff --git a/libclc/clc/include/clc/shared/ternary_decl.inc b/libclc/clc/include/clc/shared/ternary_decl.inc
index e734c0f898075..a76db1eefb6a4 100644
--- a/libclc/clc/include/clc/shared/ternary_decl.inc
+++ b/libclc/clc/include/clc/shared/ternary_decl.inc
@@ -6,6 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_GENTYPE b,
-                                                     __CLC_GENTYPE c);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
+                                               __CLC_GENTYPE c);
diff --git a/libclc/clc/include/clc/shared/ternary_def.inc b/libclc/clc/include/clc/shared/ternary_def.inc
index 36952136e74c0..92134e8cb6f27 100644
--- a/libclc/clc/include/clc/shared/ternary_def.inc
+++ b/libclc/clc/include/clc/shared/ternary_def.inc
@@ -8,11 +8,11 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
                                               __CLC_GENTYPE c) {
-  return __CLC_FUNCTION(FUNCTION)(a, b, c);
+  return __IMPL_FUNCTION(FUNCTION)(a, b, c);
 }
diff --git a/libclc/clc/include/clc/shared/ternary_def_scalarize.inc b/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
new file mode 100644
index 0000000000000..c216fa4db4cac
--- /dev/null
+++ b/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
@@ -0,0 +1,153 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/utils.h>
+
+#ifdef __CLC_SCALAR
+
+#ifndef __CLC_MIN_VECSIZE
+#define __CLC_MIN_VECSIZE 2
+#endif
+
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION FUNCTION
+#endif
+
+#ifndef __CLC_DEF_SPEC
+#define __CLC_DEF_SPEC _CLC_DEF
+#endif
+
+#ifndef __CLC_RET_TYPE
+#define __CLC_RET_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG1_TYPE
+#define __CLC_ARG1_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG2_TYPE
+#define __CLC_ARG2_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG3_TYPE
+#define __CLC_ARG3_TYPE __CLC_GENTYPE
+#endif
+
+#if __CLC_MIN_VECSIZE == 1
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
+                                                     __CLC_ARG2_TYPE y,
+                                                     __CLC_ARG3_TYPE z) {
+  return __IMPL_FUNCTION(x, y, z);
+}
+#endif // __CLC_MIN_VECSIZE == 1
+
+#if __CLC_MIN_VECSIZE <= 2
+#define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
+#define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
+#define __CLC_ARG2_TYPE2 __CLC_XCONCAT(__CLC_ARG2_TYPE, 2)
+#define __CLC_ARG3_TYPE2 __CLC_XCONCAT(__CLC_ARG3_TYPE, 2)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
+                                                      __CLC_ARG2_TYPE2 y,
+                                                      __CLC_ARG3_TYPE2 z) {
+  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1));
+}
+#undef __CLC_RET_TYPE2
+#undef __CLC_ARG1_TYPE2
+#undef __CLC_ARG2_TYPE2
+#undef __CLC_ARG3_TYPE2
+#endif // __CLC_MIN_VECSIZE <= 2
+
+#if __CLC_MIN_VECSIZE > 3
+#error "__CLC_MIN_VECSIZE > 3 isn't implemented"
+#endif
+
+#define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
+#define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
+#define __CLC_ARG2_TYPE3 __CLC_XCONCAT(__CLC_ARG2_TYPE, 3)
+#define __CLC_ARG3_TYPE3 __CLC_XCONCAT(__CLC_ARG3_TYPE, 3)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
+                                                      __CLC_ARG2_TYPE3 y,
+                                                      __CLC_ARG3_TYPE3 z) {
+  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2, z.s2));
+}
+#undef __CLC_RET_TYPE3
+#undef __CLC_ARG1_TYPE3
+#undef __CLC_ARG2_TYPE3
+#undef __CLC_ARG3_TYPE3
+
+#define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
+#define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
+#define __CLC_ARG2_TYPE4 __CLC_XCONCAT(__CLC_ARG2_TYPE, 4)
+#define __CLC_ARG3_TYPE4 __CLC_XCONCAT(__CLC_ARG3_TYPE, 4)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
+                                                      __CLC_ARG2_TYPE4 y,
+                                                      __CLC_ARG3_TYPE4 z) {
+  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                           __IMPL_FUNCTION(x.s3, y.s3, z.s3));
+}
+#undef __CLC_RET_TYPE4
+#undef __CLC_ARG1_TYPE4
+#undef __CLC_ARG2_TYPE4
+#undef __CLC_ARG3_TYPE4
+
+#define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
+#define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
+#define __CLC_ARG2_TYPE8 __CLC_XCONCAT(__CLC_ARG2_TYPE, 8)
+#define __CLC_ARG3_TYPE8 __CLC_XCONCAT(__CLC_ARG3_TYPE, 8)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
+                                                      __CLC_ARG2_TYPE8 y,
+                                                      __CLC_ARG3_TYPE8 z) {
+  return (__CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                           __IMPL_FUNCTION(x.s3, y.s3, z.s3),
+                           __IMPL_FUNCTION(x.s4, y.s4, z.s4),
+                           __IMPL_FUNCTION(x.s5, y.s5, z.s5),
+                           __IMPL_FUNCTION(x.s6, y.s6, z.s6),
+                           __IMPL_FUNCTION(x.s7, y.s7, z.s7));
+}
+#undef __CLC_RET_TYPE8
+#undef __CLC_ARG1_TYPE8
+#undef __CLC_ARG2_TYPE8
+#undef __CLC_ARG3_TYPE8
+
+#define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
+#define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
+#define __CLC_ARG2_TYPE16 __CLC_XCONCAT(__CLC_ARG2_TYPE, 16)
+#define __CLC_ARG3_TYPE16 __CLC_XCONCAT(__CLC_ARG3_TYPE, 16)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x,
+                                                       __CLC_ARG2_TYPE16 y,
+                                                       __CLC_ARG3_TYPE16 z) {
+  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                            __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                            __IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                            __IMPL_FUNCTION(x.s3, y.s3, z.s3),
+                            __IMPL_FUNCTION(x.s4, y.s4, z.s4),
+                            __IMPL_FUNCTION(x.s5, y.s5, z.s5),
+                            __IMPL_FUNCTION(x.s6, y.s6, z.s6),
+                            __IMPL_FUNCTION(x.s7, y.s7, z.s7),
+                            __IMPL_FUNCTION(x.s8, y.s8, z.s8),
+                            __IMPL_FUNCTION(x.s9, y.s9, z.s9),
+                            __IMPL_FUNCTION(x.sa, y.sa, z.sa),
+                            __IMPL_FUNCTION(x.sb, y.sb, z.sb),
+                            __IMPL_FUNCTION(x.sc, y.sc, z.sc),
+                            __IMPL_FUNCTION(x.sd, y.sd, z.sd),
+                            __IMPL_FUNCTION(x.se, y.se, z.se),
+                            __IMPL_FUNCTION(x.sf, y.sf, z.sf));
+}
+#undef __CLC_RET_TYPE16
+#undef __CLC_ARG1_TYPE16
+#undef __CLC_ARG2_TYPE16
+#undef __CLC_ARG3_TYPE16
+
+#endif // __CLC_SCALAR
diff --git a/libclc/clc/include/clc/shared/unary_decl.inc b/libclc/clc/include/clc/shared/unary_decl.inc
index 5d742de00b711..ce1a71b8284aa 100644
--- a/libclc/clc/include/clc/shared/unary_decl.inc
+++ b/libclc/clc/include/clc/shared/unary_decl.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/shared/unary_def.inc b/libclc/clc/include/clc/shared/unary_def.inc
index 3325d410ede53..e746a6bc89f32 100644
--- a/libclc/clc/include/clc/shared/unary_def.inc
+++ b/libclc/clc/include/clc/shared/unary_def.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
diff --git a/libclc/clc/include/clc/shared/unary_def_scalarize.inc b/libclc/clc/include/clc/shared/unary_def_scalarize.inc
new file mode 100644
index 0000000000000..6c82babad2b52
--- /dev/null
+++ b/libclc/clc/include/clc/shared/unary_def_scalarize.inc
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/utils.h>
+
+#ifdef __CLC_SCALAR
+
+#ifndef __CLC_MIN_VECSIZE
+#define __CLC_MIN_VECSIZE 2
+#endif
+
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION FUNCTION
+#endif
+
+#ifndef __CLC_RET_TYPE
+#define __CLC_RET_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG1_TYPE
+#define __CLC_ARG1_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG2_TYPE
+#define __CLC_ARG2_TYPE __CLC_GENTYPE
+#endif
+
+#if __CLC_MIN_VECSIZE == 1
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x) {
+  return __IMPL_FUNCTION(x);
+}
+#endif // __CLC_MIN_VECSIZE == 1
+
+#if __CLC_MIN_VECSIZE <= 2
+#define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
+#define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x) {
+  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1));
+}
+#undef __CLC_RET_TYPE2
+#undef __CLC_ARG1_TYPE2
+#endif // __CLC_MIN_VECSIZE <= 2
+
+#if __CLC_MIN_VECSIZE > 3
+#error "__CLC_MIN_VECSIZE > 3 isn't implemented"
+#endif
+
+#define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
+#define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x) {
+  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                           __IMPL_FUNCTION(x.s2));
+}
+#undef __CLC_RET_TYPE3
+#undef __CLC_ARG1_TYPE3
+
+#define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
+#define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x) {
+  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                           __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3));
+}
+#undef __CLC_RET_TYPE4
+#undef __CLC_ARG1_TYPE4
+
+#define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
+#define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x) {
+  return (__CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                           __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3),
+                           __IMPL_FUNCTION(x.s4), __IMPL_FUNCTION(x.s5),
+                           __IMPL_FUNCTION(x.s6), __IMPL_FUNCTION(x.s7));
+}
+#undef __CLC_RET_TYPE8
+#undef __CLC_ARG1_TYPE8
+
+#define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
+#define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x) {
+  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                            __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3),
+                            __IMPL_FUNCTION(x.s4), __IMPL_FUNCTION(x.s5),
+                            __IMPL_FUNCTION(x.s6), __IMPL_FUNCTION(x.s7),
+                            __IMPL_FUNCTION(x.s8), __IMPL_FUNCTION(x.s9),
+                            __IMPL_FUNCTION(x.sa), __IMPL_FUNCTION(x.sb),
+                            __IMPL_FUNCTION(x.sc), __IMPL_FUNCTION(x.sd),
+                            __IMPL_FUNCTION(x.se), __IMPL_FUNCTION(x.sf));
+}
+#undef __CLC_RET_TYPE16
+#undef __CLC_ARG1_TYPE16
+
+#endif // __CLC_SCALAR
diff --git a/libclc/clc/lib/amdgcn/math/clc_fmax.cl b/libclc/clc/lib/amdgcn/math/clc_fmax.cl
index 20bdcadb9eabf..cea90a7135d5a 100644
--- a/libclc/clc/lib/amdgcn/math/clc_fmax.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_fmax.cl
@@ -18,7 +18,6 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmax(float x, float y) {
   y = __builtin_canonicalizef(y);
   return __builtin_fmaxf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmax, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -29,8 +28,6 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmax(double x, double y) {
   y = __builtin_canonicalize(y);
   return __builtin_fmax(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmax, double,
-                      double)
 
 #endif
 #ifdef cl_khr_fp16
@@ -44,6 +41,9 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmax(half x, half y) {
     return x;
   return (y < x) ? x : y;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmax, half, half)
 
 #endif
+
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgcn/math/clc_fmin.cl b/libclc/clc/lib/amdgcn/math/clc_fmin.cl
index a5f66dfefa900..12bb0c64429fd 100644
--- a/libclc/clc/lib/amdgcn/math/clc_fmin.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_fmin.cl
@@ -18,7 +18,6 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmin(float x, float y) {
   y = __builtin_canonicalizef(y);
   return __builtin_fminf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmin, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -29,8 +28,6 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmin(double x, double y) {
   y = __builtin_canonicalize(y);
   return __builtin_fmin(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmin, double,
-                      double)
 
 #endif
 
@@ -45,6 +42,9 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmin(half x, half y) {
     return x;
   return (y < x) ? y : x;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmin, half, half)
 
 #endif
+
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl b/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
index 94a6d6ccfa9f8..b2990647a00a7 100644
--- a/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
@@ -10,14 +10,25 @@
 #include <clc/internal/clc.h>
 #include <clc/math/clc_ldexp.h>
 
+#define FUNCTION __clc_ldexp
+#define __CLC_ARG2_TYPE int
+#define __CLC_MIN_VECSIZE 1
+
 #ifdef __HAS_LDEXPF__
 // This defines all the ldexp(floatN, intN) variants.
-_CLC_DEFINE_BINARY_BUILTIN(float, __clc_ldexp, __builtin_amdgcn_ldexpf, float, int);
+#define __FLOAT_ONLY
+#define __IMPL_FUNCTION __builtin_amdgcn_ldexpf
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __IMPL_FUNCTION
 #endif
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 // This defines all the ldexp(doubleN, intN) variants.
-_CLC_DEFINE_BINARY_BUILTIN(double, __clc_ldexp, __builtin_amdgcn_ldexp, double,
-                           int);
+#define __DOUBLE_ONLY
+#define __IMPL_FUNCTION __builtin_amdgcn_ldexp
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __IMPL_FUNCTION
 #endif
diff --git a/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl b/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
index 76b1850fce574..63f28921dc177 100644
--- a/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
@@ -9,8 +9,9 @@
 #include <clc/clcmacro.h>
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF float __clc_native_exp2(float val) {
-  return __builtin_amdgcn_exp2f(val);
-}
-
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_native_exp2, float)
+#define __FLOAT_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_native_exp2
+#define __IMPL_FUNCTION __builtin_amdgcn_exp2f
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl b/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
index 576525bc47f6a..b7cb635a2ae8e 100644
--- a/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
@@ -43,6 +43,9 @@ _CLC_OVERLOAD _CLC_DEF double __clc_sqrt(double x) {
   return (x == __builtin_inf() || (x == 0.0)) ? v01 : v23;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_sqrt, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_sqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
 
 #endif
diff --git a/libclc/clc/lib/clspv/math/clc_sw_fma.cl b/libclc/clc/lib/clspv/math/clc_sw_fma.cl
index 82978380a63f6..266269644721a 100644
--- a/libclc/clc/lib/clspv/math/clc_sw_fma.cl
+++ b/libclc/clc/lib/clspv/math/clc_sw_fma.cl
@@ -269,5 +269,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
                         ((uint)st_fma.mantissa.lo & 0x7fffff));
 }
 
-_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_sw_fma, float,
-                       float, float)
+#define __FLOAT_ONLY
+#define FUNCTION __clc_sw_fma
+#define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_add_sat.cl b/libclc/clc/lib/generic/integer/clc_add_sat.cl
index b0cf51e61597f..8241ee5d547ac 100644
--- a/libclc/clc/lib/generic/integer/clc_add_sat.cl
+++ b/libclc/clc/lib/generic/integer/clc_add_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_add_sat
-#define __CLC_FUNCTION(x) __builtin_elementwise_add_sat
+#define __IMPL_FUNCTION(x) __builtin_elementwise_add_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_clz.cl b/libclc/clc/lib/generic/integer/clc_clz.cl
index 74f662375af6b..c6e1da680b7bd 100644
--- a/libclc/clc/lib/generic/integer/clc_clz.cl
+++ b/libclc/clc/lib/generic/integer/clc_clz.cl
@@ -42,11 +42,6 @@ _CLC_OVERLOAD _CLC_DEF ulong __clc_clz(ulong x) {
   return x ? __builtin_clzl(x) : 64;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, __clc_clz, char)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, __clc_clz, uchar)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, __clc_clz, short)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, __clc_clz, ushort)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, __clc_clz, int)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, __clc_clz, uint)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, __clc_clz, long)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, __clc_clz, ulong)
+#define FUNCTION __clc_clz
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_ctz.cl b/libclc/clc/lib/generic/integer/clc_ctz.cl
index 50fda4a214b24..d82d99d539df8 100644
--- a/libclc/clc/lib/generic/integer/clc_ctz.cl
+++ b/libclc/clc/lib/generic/integer/clc_ctz.cl
@@ -38,11 +38,6 @@ _CLC_OVERLOAD _CLC_DEF ulong __clc_ctz(ulong x) {
   return __builtin_ctzg(x, 64);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, __clc_ctz, char)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, __clc_ctz, uchar)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, __clc_ctz, short)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, __clc_ctz, ushort)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, __clc_ctz, int)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, __clc_ctz, uint)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, __clc_ctz, long)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, __clc_ctz, ulong)
+#define FUNCTION __clc_ctz
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_popcount.cl b/libclc/clc/lib/generic/integer/clc_popcount.cl
index f9e1cdf982049..078ceecc97691 100644
--- a/libclc/clc/lib/generic/integer/clc_popcount.cl
+++ b/libclc/clc/lib/generic/integer/clc_popcount.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_popcount
-#define __CLC_FUNCTION(x) __builtin_elementwise_popcount
+#define __IMPL_FUNCTION(x) __builtin_elementwise_popcount
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_sub_sat.cl b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
index a9aa5f67323f3..e664f04b17481 100644
--- a/libclc/clc/lib/generic/integer/clc_sub_sat.cl
+++ b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_sub_sat
-#define __CLC_FUNCTION(x) __builtin_elementwise_sub_sat
+#define __IMPL_FUNCTION(x) __builtin_elementwise_sub_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_ceil.cl b/libclc/clc/lib/generic/math/clc_ceil.cl
index c4df638d5ce26..b06cb364b58d9 100644
--- a/libclc/clc/lib/generic/math/clc_ceil.cl
+++ b/libclc/clc/lib/generic/math/clc_ceil.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_ceil
-#define __CLC_FUNCTION(x) __builtin_elementwise_ceil
+#define __IMPL_FUNCTION(x) __builtin_elementwise_ceil
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_copysign.cl b/libclc/clc/lib/generic/math/clc_copysign.cl
index b066c14bcf3f5..06f126b16dfde 100644
--- a/libclc/clc/lib/generic/math/clc_copysign.cl
+++ b/libclc/clc/lib/generic/math/clc_copysign.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_copysign
-#define __CLC_FUNCTION(x) __builtin_elementwise_copysign
+#define __IMPL_FUNCTION(x) __builtin_elementwise_copysign
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_erf.cl b/libclc/clc/lib/generic/math/clc_erf.cl
index 3808b9b81411a..17fee3e8d11f8 100644
--- a/libclc/clc/lib/generic/math/clc_erf.cl
+++ b/libclc/clc/lib/generic/math/clc_erf.cl
@@ -211,7 +211,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_erf(float x) {
   return ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_erf, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_erf
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -496,7 +500,11 @@ _CLC_OVERLOAD _CLC_DEF double __clc_erf(double y) {
   return y < 0.0 ? -ret : ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erf, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_erf
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
 
@@ -508,7 +516,7 @@ _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erf, double);
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_erf
+#define FUNCTION __clc_erf
 #define __CLC_BODY <clc/math/unary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_erfc.cl b/libclc/clc/lib/generic/math/clc_erfc.cl
index a9edfdbb72f23..77f8959831d27 100644
--- a/libclc/clc/lib/generic/math/clc_erfc.cl
+++ b/libclc/clc/lib/generic/math/clc_erfc.cl
@@ -211,7 +211,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_erfc(float x) {
   return ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_erfc, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_erfc
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -505,7 +509,11 @@ _CLC_OVERLOAD _CLC_DEF double __clc_erfc(double x) {
   return ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erfc, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_erfc
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
 
@@ -517,7 +525,7 @@ _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erfc, double);
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_erfc
+#define FUNCTION __clc_erfc
 #define __CLC_BODY <clc/math/unary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_fabs.cl b/libclc/clc/lib/generic/math/clc_fabs.cl
index f684c1e76bbbd..a0f0f3af07e25 100644
--- a/libclc/clc/lib/generic/math/clc_fabs.cl
+++ b/libclc/clc/lib/generic/math/clc_fabs.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_fabs
-#define __CLC_FUNCTION(x) __builtin_elementwise_abs
+#define __IMPL_FUNCTION(x) __builtin_elementwise_abs
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_floor.cl b/libclc/clc/lib/generic/math/clc_floor.cl
index 0626ba36f3ea8..a14adb98297e7 100644
--- a/libclc/clc/lib/generic/math/clc_floor.cl
+++ b/libclc/clc/lib/generic/math/clc_floor.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_floor
-#define __CLC_FUNCTION(x) __builtin_elementwise_floor
+#define __IMPL_FUNCTION(x) __builtin_elementwise_floor
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_fmax.cl b/libclc/clc/lib/generic/math/clc_fmax.cl
index 8ee369f57d38b..5ebbf1b28df9c 100644
--- a/libclc/clc/lib/generic/math/clc_fmax.cl
+++ b/libclc/clc/lib/generic/math/clc_fmax.cl
@@ -10,13 +10,29 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/clc_isnan.h>
 
-_CLC_DEFINE_BINARY_BUILTIN(float, __clc_fmax, __builtin_fmaxf, float, float);
+#define __FLOAT_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmax
+#define __IMPL_FUNCTION __builtin_fmaxf
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-_CLC_DEFINE_BINARY_BUILTIN(double, __clc_fmax, __builtin_fmax, double, double);
+#define __DOUBLE_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmax
+#define __IMPL_FUNCTION __builtin_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #endif
 
@@ -31,6 +47,12 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmax(half x, half y) {
     return x;
   return (x < y) ? y : x;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmax, half, half)
+
+#define __HALF_ONLY
+#define __CLC_SUPPORTED_VECSIZE_OR_1 2
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_fmin.cl b/libclc/clc/lib/generic/math/clc_fmin.cl
index 2f307274b9be5..5bddbb8634126 100644
--- a/libclc/clc/lib/generic/math/clc_fmin.cl
+++ b/libclc/clc/lib/generic/math/clc_fmin.cl
@@ -10,13 +10,29 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/clc_isnan.h>
 
-_CLC_DEFINE_BINARY_BUILTIN(float, __clc_fmin, __builtin_fminf, float, float);
+#define __FLOAT_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmin
+#define __IMPL_FUNCTION __builtin_fminf
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-_CLC_DEFINE_BINARY_BUILTIN(double, __clc_fmin, __builtin_fmin, double, double);
+#define __DOUBLE_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmin
+#define __IMPL_FUNCTION __builtin_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #endif
 
@@ -31,6 +47,11 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmin(half x, half y) {
     return x;
   return (y < x) ? y : x;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmin, half, half)
+
+#define __HALF_ONLY
+#define __CLC_SUPPORTED_VECSIZE_OR_1 2
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
 
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_fmod.cl b/libclc/clc/lib/generic/math/clc_fmod.cl
index 6af84a14f3d1a..3162ef60d46ef 100644
--- a/libclc/clc/lib/generic/math/clc_fmod.cl
+++ b/libclc/clc/lib/generic/math/clc_fmod.cl
@@ -63,7 +63,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y) {
 
   return xr;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_fmod, float, float);
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_fmod
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -170,8 +175,13 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y) {
 
   return ret;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_fmod, double,
-                      double);
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_fmod
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
+
 #endif
 
 #ifdef cl_khr_fp16
@@ -180,7 +190,7 @@ _CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_fmod, double,
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_fmod
+#define FUNCTION __clc_fmod
 #define __CLC_BODY <clc/math/binary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_half_cos.cl b/libclc/clc/lib/generic/math/clc_half_cos.cl
index c2064a00232f4..79a7ee7b483f4 100644
--- a/libclc/clc/lib/generic/math/clc_half_cos.cl
+++ b/libclc/clc/lib/generic/math/clc_half_cos.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_cos
-#define __CLC_FUNCTION(x) __clc_cos
+#define __IMPL_FUNCTION(x) __clc_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp.cl b/libclc/clc/lib/generic/math/clc_half_exp.cl
index 88336f9df011d..b53454580e3ce 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_exp
-#define __CLC_FUNCTION(x) __clc_exp
+#define __IMPL_FUNCTION(x) __clc_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp10.cl b/libclc/clc/lib/generic/math/clc_half_exp10.cl
index b3c0001a85278..7bd107bf59e85 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp10.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp10.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_exp10
-#define __CLC_FUNCTION(x) __clc_exp10
+#define __IMPL_FUNCTION(x) __clc_exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp2.cl b/libclc/clc/lib/generic/math/clc_half_exp2.cl
index f3ffcc6876b94..a20d79bfff6e3 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp2.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_exp2
-#define __CLC_FUNCTION(x) __clc_exp2
+#define __IMPL_FUNCTION(x) __clc_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log.cl b/libclc/clc/lib/generic/math/clc_half_log.cl
index 7371344ab2d1c..26b2c756c9889 100644
--- a/libclc/clc/lib/generic/math/clc_half_log.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_log
-#define __CLC_FUNCTION(x) __clc_log
+#define __IMPL_FUNCTION(x) __clc_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log10.cl b/libclc/clc/lib/generic/math/clc_half_log10.cl
index e9becad210ea9..36f5bf55bb531 100644
--- a/libclc/clc/lib/generic/math/clc_half_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log10.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_log10
-#define __CLC_FUNCTION(x) __clc_log10
+#define __IMPL_FUNCTION(x) __clc_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log2.cl b/libclc/clc/lib/generic/math/clc_half_log2.cl
index aa8d838afad7c..cbf1d350dd8e2 100644
--- a/libclc/clc/lib/generic/math/clc_half_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_log2
-#define __CLC_FUNCTION(x) __clc_log2
+#define __IMPL_FUNCTION(x) __clc_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_powr.cl b/libclc/clc/lib/generic/math/clc_half_powr.cl
index ade6731db1434..f7ef2074b840e 100644
--- a/libclc/clc/lib/generic/math/clc_half_powr.cl
+++ b/libclc/clc/lib/generic/math/clc_half_powr.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_powr
-#define __CLC_FUNCTION(x) __clc_powr
+#define __IMPL_FUNCTION(x) __clc_powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_rsqrt.cl b/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
index 07daf7fc36234..7e5d9e0526851 100644
--- a/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_rsqrt
-#define __CLC_FUNCTION(x) __clc_rsqrt
+#define __IMPL_FUNCTION(x) __clc_rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_sin.cl b/libclc/clc/lib/generic/math/clc_half_sin.cl
index c0af5e7bf4717..ef333dae3797c 100644
--- a/libclc/clc/lib/generic/math/clc_half_sin.cl
+++ b/libclc/clc/lib/generic/math/clc_half_sin.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_sin
-#define __CLC_FUNCTION(x) __clc_sin
+#define __IMPL_FUNCTION(x) __clc_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_sqrt.cl b/libclc/clc/lib/generic/math/clc_half_sqrt.cl
index b6b5097e965c5..293eb81196b63 100644
--- a/libclc/clc/lib/generic/math/clc_half_sqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_half_sqrt.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_sqrt
-#define __CLC_FUNCTION(x) __clc_sqrt
+#define __IMPL_FUNCTION(x) __clc_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_tan.cl b/libclc/clc/lib/generic/math/clc_half_tan.cl
index ca27f3c662aad..ecc9f8f328311 100644
--- a/libclc/clc/lib/generic/math/clc_half_tan.cl
+++ b/libclc/clc/lib/generic/math/clc_half_tan.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_tan
-#define __CLC_FUNCTION(x) __clc_tan
+#define __IMPL_FUNCTION(x) __clc_tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_ldexp.cl b/libclc/clc/lib/generic/math/clc_ldexp.cl
index 144a8fc897d99..cb4185d89c729 100644
--- a/libclc/clc/lib/generic/math/clc_ldexp.cl
+++ b/libclc/clc/lib/generic/math/clc_ldexp.cl
@@ -86,8 +86,6 @@ _CLC_DEF_ldexp _CLC_OVERLOAD float __clc_ldexp(float x, int n) {
   return val_f;
 }
 
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF_ldexp, float, __clc_ldexp, float, int);
-
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
@@ -121,8 +119,6 @@ _CLC_DEF_ldexp _CLC_OVERLOAD double __clc_ldexp(double x, int n) {
   return mr;
 }
 
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF_ldexp, double, __clc_ldexp, double, int);
-
 #endif
 
 #ifdef cl_khr_fp16
@@ -133,6 +129,10 @@ _CLC_OVERLOAD _CLC_DEF_ldexp half __clc_ldexp(half x, int n) {
   return (half)__clc_ldexp((float)x, n);
 }
 
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF_ldexp, half, __clc_ldexp, half, int);
-
 #endif
+
+#define FUNCTION __clc_ldexp
+#define __CLC_DEF_SPEC _CLC_DEF_ldexp
+#define __CLC_ARG2_TYPE int
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_lgamma_r.cl b/libclc/clc/lib/generic/math/clc_lgamma_r.cl
index 96a42bbb6e158..ffacfc17a4d01 100644
--- a/libclc/clc/lib/generic/math/clc_lgamma_r.cl
+++ b/libclc/clc/lib/generic/math/clc_lgamma_r.cl
@@ -611,3 +611,10 @@ _CLC_V_V_VP_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_lgamma_r, half,
 #define __CLC_BODY <clc_lgamma_r.inc>
 #include <clc/math/gentype.inc>
 #undef __CLC_ADDRSPACE
+
+#if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
+#define __CLC_ADDRSPACE generic
+#define __CLC_BODY <clc_lgamma_r.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_ADDRSPACE
+#endif
diff --git a/libclc/clc/lib/generic/math/clc_log.cl b/libclc/clc/lib/generic/math/clc_log.cl
index 85405faf8b8ca..cf5628f206cad 100644
--- a/libclc/clc/lib/generic/math/clc_log.cl
+++ b/libclc/clc/lib/generic/math/clc_log.cl
@@ -19,8 +19,6 @@ _CLC_OVERLOAD _CLC_DEF float __clc_log(float x) {
   return __clc_log2(x) * (1.0f / M_LOG2E_F);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_log, float);
-
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
@@ -29,8 +27,6 @@ _CLC_OVERLOAD _CLC_DEF double __clc_log(double x) {
   return __clc_log2(x) * (1.0 / M_LOG2E);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_log, double);
-
 #endif // cl_khr_fp64
 
 #ifdef cl_khr_fp16
@@ -41,6 +37,8 @@ _CLC_OVERLOAD _CLC_DEF half __clc_log(half x) {
   return (half)__clc_log2((float)x) * (1.0h / M_LOG2E_H);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_log, half);
-
 #endif // cl_khr_fp16
+
+#define FUNCTION __clc_log
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log10.cl b/libclc/clc/lib/generic/math/clc_log10.cl
index 689ec25b74e5f..f5f0e8cc70837 100644
--- a/libclc/clc/lib/generic/math/clc_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_log10.cl
@@ -22,12 +22,6 @@
 #include "clc_log_base.h"
 #undef COMPILING_LOG10
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_log10, float);
-
-#ifdef cl_khr_fp64
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_log10, double);
-#endif // cl_khr_fp64
-
-#ifdef cl_khr_fp16
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_log10, half);
-#endif // cl_khr_fp16
+#define FUNCTION __clc_log10
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log2.cl b/libclc/clc/lib/generic/math/clc_log2.cl
index ebd981c9d9955..335488af2f3de 100644
--- a/libclc/clc/lib/generic/math/clc_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_log2.cl
@@ -22,12 +22,6 @@
 #include "clc_log_base.h"
 #undef COMPILING_LOG2
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_log2, float);
-
-#ifdef cl_khr_fp64
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_log2, double);
-#endif // cl_khr_fp64
-
-#ifdef cl_khr_fp16
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_log2, half);
-#endif // cl_khr_fp16
+#define FUNCTION __clc_log2
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_nan.inc b/libclc/clc/lib/generic/math/clc_nan.inc
index e934416351db1..46e828ba48c7e 100644
--- a/libclc/clc/lib/generic/math/clc_nan.inc
+++ b/libclc/clc/lib/generic/math/clc_nan.inc
@@ -20,4 +20,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_U_GENTYPE code) {
   return __CLC_AS_GENTYPE(res);
 }
 
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_S_GENTYPE code) {
+  return __clc_nan(__CLC_AS_U_GENTYPE(code));
+}
+
 #undef NAN_MASK
diff --git a/libclc/clc/lib/generic/math/clc_native_cos.cl b/libclc/clc/lib/generic/math/clc_native_cos.cl
index de56fdec48d24..2e63e9c9493f1 100644
--- a/libclc/clc/lib/generic/math/clc_native_cos.cl
+++ b/libclc/clc/lib/generic/math/clc_native_cos.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_cos
-#define __CLC_FUNCTION(x) __builtin_elementwise_cos
+#define __IMPL_FUNCTION(x) __builtin_elementwise_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp.cl b/libclc/clc/lib/generic/math/clc_native_exp.cl
index 400270a6163a4..2f50a0aefac92 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_exp
-#define __CLC_FUNCTION(x) __builtin_elementwise_exp
+#define __IMPL_FUNCTION(x) __builtin_elementwise_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp2.cl b/libclc/clc/lib/generic/math/clc_native_exp2.cl
index 427d901fcdb19..25f557306c35b 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp2.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_exp2
-#define __CLC_FUNCTION(x) __builtin_elementwise_exp2
+#define __IMPL_FUNCTION(x) __builtin_elementwise_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log.cl b/libclc/clc/lib/generic/math/clc_native_log.cl
index 85f188b654282..b9b9d274f8b9f 100644
--- a/libclc/clc/lib/generic/math/clc_native_log.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_log
-#define __CLC_FUNCTION(x) __builtin_elementwise_log
+#define __IMPL_FUNCTION(x) __builtin_elementwise_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log10.cl b/libclc/clc/lib/generic/math/clc_native_log10.cl
index 624018e4481bf..221aa406466c4 100644
--- a/libclc/clc/lib/generic/math/clc_native_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log10.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_log10
-#define __CLC_FUNCTION(x) __builtin_elementwise_log10
+#define __IMPL_FUNCTION(x) __builtin_elementwise_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log2.cl b/libclc/clc/lib/generic/math/clc_native_log2.cl
index 2c8c18e61ca5d..c3008ce242c74 100644
--- a/libclc/clc/lib/generic/math/clc_native_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_log2
-#define __CLC_FUNCTION(x) __builtin_elementwise_log2
+#define __IMPL_FUNCTION(x) __builtin_elementwise_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_sin.cl b/libclc/clc/lib/generic/math/clc_native_sin.cl
index 22b988bf4375f..533f8d726d41a 100644
--- a/libclc/clc/lib/generic/math/clc_native_sin.cl
+++ b/libclc/clc/lib/generic/math/clc_native_sin.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_sin
-#define __CLC_FUNCTION(x) __builtin_elementwise_sin
+#define __IMPL_FUNCTION(x) __builtin_elementwise_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_sqrt.cl b/libclc/clc/lib/generic/math/clc_native_sqrt.cl
index ed022ef1fee1f..e0f028e988b42 100644
--- a/libclc/clc/lib/generic/math/clc_native_sqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_native_sqrt.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_sqrt
-#define __CLC_FUNCTION(x) __builtin_elementwise_sqrt
+#define __IMPL_FUNCTION(x) __builtin_elementwise_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_remainder.cl b/libclc/clc/lib/generic/math/clc_remainder.cl
index c79d271c624e2..2b3d185a8bee6 100644
--- a/libclc/clc/lib/generic/math/clc_remainder.cl
+++ b/libclc/clc/lib/generic/math/clc_remainder.cl
@@ -73,8 +73,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remainder(float x, float y) {
 
   return xr;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_remainder, float,
-                      float);
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_remainder
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -207,8 +211,13 @@ _CLC_DEF _CLC_OVERLOAD double __clc_remainder(double x, double y) {
 
   return ret;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_remainder, double,
-                      double);
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_remainder
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
+
 #endif
 
 #ifdef cl_khr_fp16
@@ -217,7 +226,7 @@ _CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_remainder, double,
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_remainder
+#define FUNCTION __clc_remainder
 #define __CLC_BODY <clc/math/binary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_rint.cl b/libclc/clc/lib/generic/math/clc_rint.cl
index 2188a7d901f01..b3f94490e3b5a 100644
--- a/libclc/clc/lib/generic/math/clc_rint.cl
+++ b/libclc/clc/lib/generic/math/clc_rint.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_rint
-#define __CLC_FUNCTION(x) __builtin_elementwise_rint
+#define __IMPL_FUNCTION(x) __builtin_elementwise_rint
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_round.cl b/libclc/clc/lib/generic/math/clc_round.cl
index e784dbeaf3fab..8c298d8c34835 100644
--- a/libclc/clc/lib/generic/math/clc_round.cl
+++ b/libclc/clc/lib/generic/math/clc_round.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_round
-#define __CLC_FUNCTION(x) __builtin_elementwise_round
+#define __IMPL_FUNCTION(x) __builtin_elementwise_round
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_sincos_helpers.cl b/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
index c1768e39243fa..0ea1195fffa70 100644
--- a/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
+++ b/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
@@ -32,8 +32,6 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
@@ -52,6 +50,4 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __DOUBLE_ONLY
-
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_sw_fma.cl b/libclc/clc/lib/generic/math/clc_sw_fma.cl
index df5ac9dbd49d7..ee4734078d698 100644
--- a/libclc/clc/lib/generic/math/clc_sw_fma.cl
+++ b/libclc/clc/lib/generic/math/clc_sw_fma.cl
@@ -160,5 +160,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
                         ((uint)st_fma.mantissa & 0x7fffff));
 }
 
-_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_sw_fma, float,
-                       float, float)
+#define __FLOAT_ONLY
+#define FUNCTION __clc_sw_fma
+#define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_tgamma.cl b/libclc/clc/lib/generic/math/clc_tgamma.cl
index 5c2d164abeee8..e0fed98026ca5 100644
--- a/libclc/clc/lib/generic/math/clc_tgamma.cl
+++ b/libclc/clc/lib/generic/math/clc_tgamma.cl
@@ -32,7 +32,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_tgamma(float x) {
   return g;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_tgamma, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_tgamma
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -55,7 +59,11 @@ _CLC_OVERLOAD _CLC_DEF double __clc_tgamma(double x) {
   return g;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_tgamma, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_tgamma
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
 
@@ -67,7 +75,7 @@ _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_tgamma, double);
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_tgamma
+#define FUNCTION __clc_tgamma
 #define __CLC_BODY <clc/math/unary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_trunc.cl b/libclc/clc/lib/generic/math/clc_trunc.cl
index 0bf2bc44c26f8..6261d04cd26de 100644
--- a/libclc/clc/lib/generic/math/clc_trunc.cl
+++ b/libclc/clc/lib/generic/math/clc_trunc.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_trunc
-#define __CLC_FUNCTION(x) __builtin_elementwise_trunc
+#define __IMPL_FUNCTION(x) __builtin_elementwise_trunc
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/r600/math/clc_fmax.cl b/libclc/clc/lib/r600/math/clc_fmax.cl
index 7e9c6df789eda..689e51a9829aa 100644
--- a/libclc/clc/lib/r600/math/clc_fmax.cl
+++ b/libclc/clc/lib/r600/math/clc_fmax.cl
@@ -17,7 +17,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmax(float x, float y) {
   y = __clc_flush_denormal_if_not_supported(y);
   return __builtin_fmaxf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmax, float, float)
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -26,7 +31,11 @@ _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmax, float, float)
 _CLC_DEF _CLC_OVERLOAD double __clc_fmax(double x, double y) {
   return __builtin_fmax(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmax, double,
-                      double)
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/r600/math/clc_fmin.cl b/libclc/clc/lib/r600/math/clc_fmin.cl
index cbcf849f7b7e8..22cb7046a4ce3 100644
--- a/libclc/clc/lib/r600/math/clc_fmin.cl
+++ b/libclc/clc/lib/r600/math/clc_fmin.cl
@@ -18,7 +18,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmin(float x, float y) {
   y = __clc_flush_denormal_if_not_supported(y);
   return __builtin_fminf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmin, float, float)
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -27,7 +32,11 @@ _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmin, float, float)
 _CLC_DEF _CLC_OVERLOAD double __clc_fmin(double x, double y) {
   return __builtin_fmin(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmin, double,
-                      double)
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/r600/math/clc_native_rsqrt.cl b/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
index ee09814eb1e76..b5966570804c9 100644
--- a/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
+++ b/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
@@ -13,4 +13,7 @@ _CLC_OVERLOAD _CLC_DEF float __clc_native_rsqrt(float x) {
   return __builtin_r600_recipsqrt_ieeef(x);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_native_rsqrt, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_native_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/r600/math/clc_rsqrt_override.cl b/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
index 91350d611f0ac..75355df56d32e 100644
--- a/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
+++ b/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
@@ -13,7 +13,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_rsqrt(float x) {
   return __builtin_r600_recipsqrt_ieeef(x);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_rsqrt, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -23,6 +27,10 @@ _CLC_OVERLOAD _CLC_DEF double __clc_rsqrt(double x) {
   return __builtin_r600_recipsqrt_ieee(x);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_rsqrt, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake
index 597bb642655e4..a8160e04d46a0 100644
--- a/libclc/cmake/modules/AddLibclc.cmake
+++ b/libclc/cmake/modules/AddLibclc.cmake
@@ -120,14 +120,14 @@ function(link_bc)
   endif()
 
   add_custom_command(
-    OUTPUT ${ARG_TARGET}.bc
-    COMMAND ${llvm-link_exe} ${link_flags} -o ${ARG_TARGET}.bc ${LINK_INPUT_ARG}
+    OUTPUT ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc
+    COMMAND ${llvm-link_exe} ${link_flags} -o ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc ${LINK_INPUT_ARG}
     DEPENDS ${llvm-link_target} ${ARG_DEPENDENCIES} ${ARG_INPUTS} ${RSP_FILE}
   )
 
-  add_custom_target( ${ARG_TARGET} ALL DEPENDS ${ARG_TARGET}.bc )
+  add_custom_target( ${ARG_TARGET} ALL DEPENDS ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc )
   set_target_properties( ${ARG_TARGET} PROPERTIES
-    TARGET_FILE ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.bc
+    TARGET_FILE ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc
     FOLDER "libclc/Device IR/Linking"
   )
 endfunction()
@@ -356,65 +356,92 @@ function(add_libclc_builtin_set)
 
   set( builtins_link_lib $<TARGET_PROPERTY:${builtins_link_lib_tgt},TARGET_FILE> )
 
+  # For SPIR-V targets we diverage at this point and generate SPIR-V using the
+  # llvm-spirv tool.
   if( ARG_ARCH STREQUAL spirv OR ARG_ARCH STREQUAL spirv64 )
-    set( spv_suffix ${ARG_ARCH_SUFFIX}.spv )
-    add_custom_command( OUTPUT ${spv_suffix}
-      COMMAND ${llvm-spirv_exe} ${spvflags} -o ${spv_suffix} ${builtins_link_lib}
+    set( obj_suffix ${ARG_ARCH_SUFFIX}.spv )
+    set( libclc_builtins_lib ${LIBCLC_OUTPUT_LIBRARY_DIR}/${obj_suffix} )
+    add_custom_command( OUTPUT ${libclc_builtins_lib}
+      COMMAND ${llvm-spirv_exe} ${spvflags} -o ${libclc_builtins_lib} ${builtins_link_lib}
       DEPENDS ${llvm-spirv_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
     )
-    add_custom_target( "prepare-${spv_suffix}" ALL DEPENDS "${spv_suffix}" )
-    set_target_properties( "prepare-${spv_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
-    install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${spv_suffix}
-       DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  else()
+    # Non-SPIR-V targets add an extra step to optimize the bytecode
+    set( builtins_opt_lib_tgt builtins.opt.${ARG_ARCH_SUFFIX} )
 
-    return()
+    add_custom_command( OUTPUT ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+      COMMAND ${opt_exe} ${ARG_OPT_FLAGS} -o ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+        ${builtins_link_lib}
+      DEPENDS ${opt_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
+    )
+    add_custom_target( ${builtins_opt_lib_tgt}
+      ALL DEPENDS ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+    )
+    set_target_properties( ${builtins_opt_lib_tgt} PROPERTIES
+      TARGET_FILE ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+      FOLDER "libclc/Device IR/Opt"
+    )
+
+    set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
+
+    set( obj_suffix ${ARG_ARCH_SUFFIX}.bc )
+    set( libclc_builtins_lib ${LIBCLC_OUTPUT_LIBRARY_DIR}/${obj_suffix} )
+    add_custom_command( OUTPUT ${libclc_builtins_lib}
+      COMMAND ${prepare_builtins_exe} -o ${libclc_builtins_lib} ${builtins_opt_lib}
+      DEPENDS ${builtins_opt_lib} ${builtins_opt_lib_tgt} ${prepare_builtins_target}
+    )
   endif()
 
-  set( builtins_opt_lib_tgt builtins.opt.${ARG_ARCH_SUFFIX} )
+  # Add a 'prepare' target
+  add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${libclc_builtins_lib} )
+  set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
 
-  # Add opt target
-  add_custom_command( OUTPUT ${builtins_opt_lib_tgt}.bc
-    COMMAND ${opt_exe} ${ARG_OPT_FLAGS} -o ${builtins_opt_lib_tgt}.bc
-      ${builtins_link_lib}
-    DEPENDS ${opt_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
-  )
-  add_custom_target( ${builtins_opt_lib_tgt}
-    ALL DEPENDS ${builtins_opt_lib_tgt}.bc
-  )
-  set_target_properties( ${builtins_opt_lib_tgt} PROPERTIES
-    TARGET_FILE ${CMAKE_CURRENT_BINARY_DIR}/${builtins_opt_lib_tgt}.bc
-    FOLDER "libclc/Device IR/Opt"
-  )
+  # Also add a 'prepare' target for the triple. Since a triple may have
+  # multiple devices, ensure we only try to create the triple target once. The
+  # triple's target will build all of the bytecode for its constituent devices.
+  if( NOT TARGET prepare-${ARG_TRIPLE} )
+    add_custom_target( prepare-${ARG_TRIPLE} ALL )
+  endif()
+  add_dependencies( prepare-${ARG_TRIPLE} prepare-${obj_suffix} )
 
-  set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
+  install(
+    FILES ${libclc_builtins_lib}
+    DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
+  )
 
-  # Add prepare target
-  set( obj_suffix ${ARG_ARCH_SUFFIX}.bc )
-  add_custom_command( OUTPUT ${obj_suffix}
-    COMMAND ${prepare_builtins_exe} -o ${obj_suffix} ${builtins_opt_lib}
-    DEPENDS ${builtins_opt_lib} ${builtins_opt_lib_tgt} ${prepare_builtins_target} )
-  add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${obj_suffix} )
-  set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
+  # SPIR-V targets can exit early here
+  if( ARG_ARCH STREQUAL spirv OR ARG_ARCH STREQUAL spirv64 )
+    return()
+  endif()
 
-  # nvptx-- targets don't include workitem builtins, and clspv targets don't
-  # include all OpenCL builtins
+  # Add a test for whether or not the libraries contain unresolved calls which
+  # would usually indicate a build problem. Note that we don't perform this
+  # test for all libclc targets:
+  # * nvptx-- targets don't include workitem builtins
+  # * clspv targets don't include all OpenCL builtins
   if( NOT ARG_ARCH MATCHES "^(nvptx|clspv)(64)?$" )
     add_test( NAME external-calls-${obj_suffix}
-      COMMAND ./check_external_calls.sh ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} ${LLVM_TOOLS_BINARY_DIR}
+      COMMAND ./check_external_calls.sh ${libclc_builtins_lib} ${LLVM_TOOLS_BINARY_DIR}
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
   endif()
 
-  install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
   foreach( a ${ARG_ALIASES} )
     set( alias_suffix "${a}-${ARG_TRIPLE}.bc" )
     add_custom_command(
-      OUTPUT ${alias_suffix}
-      COMMAND ${CMAKE_COMMAND} -E create_symlink ${obj_suffix} ${alias_suffix}
-      DEPENDS prepare-${obj_suffix} )
-    add_custom_target( alias-${alias_suffix} ALL DEPENDS ${alias_suffix} )
-    set_target_properties( alias-${alias_suffix} PROPERTIES FOLDER "libclc/Device IR/Aliases" )
-    install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${alias_suffix}
-             DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+      OUTPUT ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ${libclc_builtins_lib} ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+      DEPENDS prepare-${obj_suffix}
+    )
+    add_custom_target( alias-${alias_suffix} ALL
+      DEPENDS ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+    )
+    set_target_properties( alias-${alias_suffix}
+      PROPERTIES FOLDER "libclc/Device IR/Aliases"
+    )
+    install(
+      FILES ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+      DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
+    )
   endforeach( a )
 endfunction(add_libclc_builtin_set)
 
diff --git a/libclc/opencl/include/clc/opencl/as_type.h b/libclc/opencl/include/clc/opencl/as_type.h
index a68e8a96f252b..abe643df81046 100644
--- a/libclc/opencl/include/clc/opencl/as_type.h
+++ b/libclc/opencl/include/clc/opencl/as_type.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_AS_TYPE_H__
+#define __CLC_OPENCL_AS_TYPE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define as_char(x) __builtin_astype(x, char)
 #define as_uchar(x) __builtin_astype(x, uchar)
 #define as_short(x) __builtin_astype(x, short)
@@ -83,3 +88,5 @@
 #define as_half8(x) __builtin_astype(x, half8)
 #define as_half16(x) __builtin_astype(x, half16)
 #endif
+
+#endif // __CLC_OPENCL_AS_TYPE_H__
diff --git a/libclc/opencl/include/clc/opencl/async/async_work_group_copy.h b/libclc/opencl/include/clc/opencl/async/async_work_group_copy.h
index 14420b3c9eeac..1bd16a411554a 100644
--- a/libclc/opencl/include/clc/opencl/async/async_work_group_copy.h
+++ b/libclc/opencl/include/clc/opencl/async/async_work_group_copy.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ASYNC_ASYNC_WORK_GROUP_COPY_H__
+#define __CLC_OPENCL_ASYNC_ASYNC_WORK_GROUP_COPY_H__
+
 #define __CLC_DST_ADDR_SPACE local
 #define __CLC_SRC_ADDR_SPACE global
 #define __CLC_BODY <clc/opencl/async/async_work_group_copy.inc>
@@ -23,3 +26,5 @@
 #include <clc/math/gentype.inc>
 #undef __CLC_DST_ADDR_SPACE
 #undef __CLC_SRC_ADDR_SPACE
+
+#endif // __CLC_OPENCL_ASYNC_ASYNC_WORK_GROUP_COPY_H__
diff --git a/libclc/opencl/include/clc/opencl/async/async_work_group_strided_copy.h b/libclc/opencl/include/clc/opencl/async/async_work_group_strided_copy.h
index fd37e35f18c5b..5929ce73a9317 100644
--- a/libclc/opencl/include/clc/opencl/async/async_work_group_strided_copy.h
+++ b/libclc/opencl/include/clc/opencl/async/async_work_group_strided_copy.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ASYNC_ASYNC_WORK_GROUP_STRIDED_COPY_H__
+#define __CLC_OPENCL_ASYNC_ASYNC_WORK_GROUP_STRIDED_COPY_H__
+
 #define __CLC_DST_ADDR_SPACE local
 #define __CLC_SRC_ADDR_SPACE global
 #define __CLC_BODY <clc/opencl/async/async_work_group_strided_copy.inc>
@@ -23,3 +26,5 @@
 #include <clc/math/gentype.inc>
 #undef __CLC_DST_ADDR_SPACE
 #undef __CLC_SRC_ADDR_SPACE
+
+#endif // __CLC_OPENCL_ASYNC_ASYNC_WORK_GROUP_STRIDED_COPY_H__
diff --git a/libclc/opencl/include/clc/opencl/async/prefetch.h b/libclc/opencl/include/clc/opencl/async/prefetch.h
index 5075dcaee1409..1ba4414d9eb10 100644
--- a/libclc/opencl/include/clc/opencl/async/prefetch.h
+++ b/libclc/opencl/include/clc/opencl/async/prefetch.h
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ASYNC_PREFETCH_H__
+#define __CLC_OPENCL_ASYNC_PREFETCH_H__
+
 #define __CLC_BODY <clc/opencl/async/prefetch.inc>
 #include <clc/integer/gentype.inc>
 
 #define __CLC_BODY <clc/opencl/async/prefetch.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_ASYNC_PREFETCH_H__
diff --git a/libclc/opencl/include/clc/opencl/async/wait_group_events.h b/libclc/opencl/include/clc/opencl/async/wait_group_events.h
index 6aab22e5406ea..d3ace9c34a25d 100644
--- a/libclc/opencl/include/clc/opencl/async/wait_group_events.h
+++ b/libclc/opencl/include/clc/opencl/async/wait_group_events.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ASYNC_WAIT_GROUP_EVENTS_H__
+#define __CLC_OPENCL_ASYNC_WAIT_GROUP_EVENTS_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD void wait_group_events(int num_events,
                                                event_t *event_list);
+
+#endif // __CLC_OPENCL_ASYNC_WAIT_GROUP_EVENTS_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_add.h b/libclc/opencl/include/clc/opencl/atomic/atom_add.h
index 62c73d99371fb..7f68fca0fc3a5 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_add.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_ADD_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_ADD_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_base_atomics
-#define __CLC_FUNCTION atom_add
+#define FUNCTION atom_add
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define __CLC_FUNCTION atom_add
+#define FUNCTION atom_add
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define __CLC_FUNCTION atom_add
+#define FUNCTION atom_add
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_ADD_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_and.h b/libclc/opencl/include/clc/opencl/atomic/atom_and.h
index 9dd9466f62f86..5c40aa4db70ad 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_and.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_AND_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_AND_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_and
+#define FUNCTION atom_and
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_and
+#define FUNCTION atom_and
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_and
+#define FUNCTION atom_and
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_AND_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_cmpxchg.h b/libclc/opencl/include/clc/opencl/atomic/atom_cmpxchg.h
index 92d922b0d2766..83ee99f414241 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_cmpxchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_cmpxchg.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_CMPXCHG_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_CMPXCHG_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -37,3 +42,5 @@ _CLC_OVERLOAD _CLC_DECL unsigned long
 atom_cmpxchg(volatile local unsigned long *p, unsigned long cmp,
              unsigned long val);
 #endif // cl_khr_int64_base_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_CMPXCHG_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_dec.h b/libclc/opencl/include/clc/opencl/atomic/atom_dec.h
index afb829bb83159..f57d9c6afca5d 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_dec.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_dec.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_DEC_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_DEC_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -26,3 +31,5 @@ atom_dec(volatile global unsigned long *p);
 _CLC_OVERLOAD _CLC_DECL long atom_dec(volatile local long *p);
 _CLC_OVERLOAD _CLC_DECL unsigned long atom_dec(volatile local unsigned long *p);
 #endif // cl_khr_int64_base_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_DEC_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
index 866d8903db816..b8631ceac3a42 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
@@ -10,12 +10,12 @@
 #include <clc/clctypes.h>
 
 #define __CLC_DECLARE_ATOM(ADDRSPACE, TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 __CLC_DECLARE_ATOM(__CLC_ADDRESS_SPACE, int)
 __CLC_DECLARE_ATOM(__CLC_ADDRESS_SPACE, uint)
 
 #undef __CLC_DECLARE_ATOM
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_ADDRESS_SPACE
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
index 146de3412fc2e..3918a7cae006b 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
@@ -10,7 +10,7 @@
 #include <clc/clctypes.h>
 
 #define __CLC_DECLARE_ATOM(ADDRSPACE, TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 __CLC_DECLARE_ATOM(local, long)
 __CLC_DECLARE_ATOM(local, ulong)
@@ -19,4 +19,4 @@ __CLC_DECLARE_ATOM(global, ulong)
 
 #undef __CLC_DECLARE_ATOM
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_inc.h b/libclc/opencl/include/clc/opencl/atomic/atom_inc.h
index 227538b153ab5..96d3faa593fc9 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_inc.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_inc.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_INC_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_INC_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
@@ -26,3 +31,5 @@ atom_inc(volatile global unsigned long *p);
 _CLC_OVERLOAD _CLC_DECL long atom_inc(volatile local long *p);
 _CLC_OVERLOAD _CLC_DECL unsigned long atom_inc(volatile local unsigned long *p);
 #endif // cl_khr_int64_base_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_INC_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_max.h b/libclc/opencl/include/clc/opencl/atomic/atom_max.h
index 00b47e6b14fde..87c70f20058d9 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_max.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_MAX_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_MAX_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_max
+#define FUNCTION atom_max
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_max
+#define FUNCTION atom_max
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_max
+#define FUNCTION atom_max
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_MAX_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_min.h b/libclc/opencl/include/clc/opencl/atomic/atom_min.h
index 40bed0dc086f2..487364c617ab5 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_min.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_MIN_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_MIN_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_min
+#define FUNCTION atom_min
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_min
+#define FUNCTION atom_min
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_min
+#define FUNCTION atom_min
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_MIN_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_or.h b/libclc/opencl/include/clc/opencl/atomic/atom_or.h
index 2c0beb850b4d0..1c3ae6984eead 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_or.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_OR_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_OR_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_or
+#define FUNCTION atom_or
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_or
+#define FUNCTION atom_or
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_or
+#define FUNCTION atom_or
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_OR_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_sub.h b/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
index 36074aaef5a32..eb17aa8ce73fe 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_SUB_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_SUB_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_base_atomics
-#define __CLC_FUNCTION atom_sub
+#define FUNCTION atom_sub
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define __CLC_FUNCTION atom_sub
+#define FUNCTION atom_sub
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define __CLC_FUNCTION atom_sub
+#define FUNCTION atom_sub
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_SUB_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
index a5043aa4109ed..df17d31d6e67a 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_XCHG_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_XCHG_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_base_atomics
-#define __CLC_FUNCTION atom_xchg
+#define FUNCTION atom_xchg
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define __CLC_FUNCTION atom_xchg
+#define FUNCTION atom_xchg
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define __CLC_FUNCTION atom_xchg
+#define FUNCTION atom_xchg
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_XCHG_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_xor.h b/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
index 0c4b456b8f23f..8a97818821582 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
@@ -6,19 +6,26 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOM_XOR_H__
+#define __CLC_OPENCL_ATOMIC_ATOM_XOR_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_xor
+#define FUNCTION atom_xor
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_xor
+#define FUNCTION atom_xor
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_xor
+#define FUNCTION atom_xor
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
+
+#endif // __CLC_OPENCL_ATOMIC_ATOM_XOR_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
index 937def3f4a2ae..821ae7aab05bf 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_add
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_ADD_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_ADD_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_add
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_ADD_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
index 5f353f40318f6..d10cfed9b581a 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_and
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_AND_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_AND_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_and
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_AND_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_cmpxchg.h b/libclc/opencl/include/clc/opencl/atomic/atomic_cmpxchg.h
index fc7975910cb24..f784984299344 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_cmpxchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_cmpxchg.h
@@ -6,7 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_CMPXCHG_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_CMPXCHG_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_OVERLOAD _CLC_DECL int atomic_cmpxchg(volatile local int *, int, int);
 _CLC_OVERLOAD _CLC_DECL int atomic_cmpxchg(volatile global int *, int, int);
 _CLC_OVERLOAD _CLC_DECL uint atomic_cmpxchg(volatile local uint *, uint, uint);
 _CLC_OVERLOAD _CLC_DECL uint atomic_cmpxchg(volatile global uint *, uint, uint);
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_CMPXCHG_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_dec.h b/libclc/opencl/include/clc/opencl/atomic/atomic_dec.h
index 354a721003f56..253a64f491fb4 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_dec.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_dec.h
@@ -6,7 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_DEC_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_DEC_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_OVERLOAD _CLC_DECL int atomic_dec(volatile local int *);
 _CLC_OVERLOAD _CLC_DECL int atomic_dec(volatile global int *);
 _CLC_OVERLOAD _CLC_DECL uint atomic_dec(volatile local uint *);
 _CLC_OVERLOAD _CLC_DECL uint atomic_dec(volatile global uint *);
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_DEC_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
index 0cfd4c3eab5f6..e060e3aaea161 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE)                                  \
-  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 #define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE)                                   \
   __CLC_DECLARE_ATOMIC(global, TYPE)                                           \
@@ -19,4 +19,4 @@ __CLC_DECLARE_ATOMIC_ADDRSPACE(uint)
 #undef __CLC_DECLARE_ATOMIC_ADDRSPACE
 #undef __CLC_DECLARE_ATOMIC
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_inc.h b/libclc/opencl/include/clc/opencl/atomic/atomic_inc.h
index 2b81c23bf0e5e..75b3fe076cf21 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_inc.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_inc.h
@@ -6,7 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_INC_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_INC_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_OVERLOAD _CLC_DECL int atomic_inc(volatile local int *);
 _CLC_OVERLOAD _CLC_DECL int atomic_inc(volatile global int *);
 _CLC_OVERLOAD _CLC_DECL uint atomic_inc(volatile local uint *);
 _CLC_OVERLOAD _CLC_DECL uint atomic_inc(volatile global uint *);
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_INC_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
index 13832423cfeec..667fa36f16f9d 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_max
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_MAX_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_MAX_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_max
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_MAX_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
index 3bd47a3f7837d..91bb636eec875 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_min
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_MIN_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_MIN_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_min
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_MIN_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
index 13f84ea49a5ab..5c03fd157a2bc 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_or
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_OR_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_OR_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_or
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_OR_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
index eedc9d0e76054..25ffe9ff4a9b7 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_sub
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_SUB_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_SUB_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_sub
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_SUB_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
index ec322d56e23d1..6b4206dedb820 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
@@ -6,8 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_xchg
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_XCHG_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_XCHG_H__
 
-_CLC_OVERLOAD _CLC_DECL float __CLC_FUNCTION(volatile local float *, float);
-_CLC_OVERLOAD _CLC_DECL float __CLC_FUNCTION(volatile global float *, float);
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_xchg
+
+_CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile local float *, float);
+_CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile global float *, float);
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_XCHG_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
index 6f6b4564fdf3c..e94560cb6b9ed 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_xor
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_XOR_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_XOR_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION atomic_xor
 #include <clc/opencl/atomic/atomic_decl.inc>
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_XOR_H__
diff --git a/libclc/opencl/include/clc/opencl/clc.h b/libclc/opencl/include/clc/opencl/clc.h
deleted file mode 100644
index 5859a00c3158b..0000000000000
--- a/libclc/opencl/include/clc/opencl/clc.h
+++ /dev/null
@@ -1,268 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef __CLC_OPENCL_CLC_H__
-#define __CLC_OPENCL_CLC_H__
-
-#ifndef cl_clang_storage_class_specifiers
-#error Implementation requires cl_clang_storage_class_specifiers extension!
-#endif
-
-#pragma OPENCL EXTENSION cl_clang_storage_class_specifiers : enable
-
-#ifdef cl_khr_fp64
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-#endif
-
-#ifdef cl_khr_fp16
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#endif
-
-/* Function Attributes */
-#include <clc/clcfunc.h>
-
-/* 6.1 Supported Data Types */
-#include <clc/clctypes.h>
-
-/* 6.2.3 Explicit Conversions */
-#include <clc/opencl/convert.h>
-
-/* 6.2.4.2 Reinterpreting Types Using as_type() and as_typen() */
-#include <clc/opencl/as_type.h>
-
-/* 6.11.1 Work-Item Functions */
-#include <clc/opencl/workitem/get_global_id.h>
-#include <clc/opencl/workitem/get_global_offset.h>
-#include <clc/opencl/workitem/get_global_size.h>
-#include <clc/opencl/workitem/get_group_id.h>
-#include <clc/opencl/workitem/get_local_id.h>
-#include <clc/opencl/workitem/get_local_size.h>
-#include <clc/opencl/workitem/get_num_groups.h>
-#include <clc/opencl/workitem/get_work_dim.h>
-
-/* 6.11.2 Math Functions */
-#include <clc/opencl/math/acos.h>
-#include <clc/opencl/math/acosh.h>
-#include <clc/opencl/math/acospi.h>
-#include <clc/opencl/math/asin.h>
-#include <clc/opencl/math/asinh.h>
-#include <clc/opencl/math/asinpi.h>
-#include <clc/opencl/math/atan.h>
-#include <clc/opencl/math/atan2.h>
-#include <clc/opencl/math/atan2pi.h>
-#include <clc/opencl/math/atanh.h>
-#include <clc/opencl/math/atanpi.h>
-#include <clc/opencl/math/cbrt.h>
-#include <clc/opencl/math/ceil.h>
-#include <clc/opencl/math/copysign.h>
-#include <clc/opencl/math/cos.h>
-#include <clc/opencl/math/cosh.h>
-#include <clc/opencl/math/cospi.h>
-#include <clc/opencl/math/erf.h>
-#include <clc/opencl/math/erfc.h>
-#include <clc/opencl/math/exp.h>
-#include <clc/opencl/math/exp10.h>
-#include <clc/opencl/math/exp2.h>
-#include <clc/opencl/math/expm1.h>
-#include <clc/opencl/math/fabs.h>
-#include <clc/opencl/math/fdim.h>
-#include <clc/opencl/math/floor.h>
-#include <clc/opencl/math/fma.h>
-#include <clc/opencl/math/fmax.h>
-#include <clc/opencl/math/fmin.h>
-#include <clc/opencl/math/fmod.h>
-#include <clc/opencl/math/fract.h>
-#include <clc/opencl/math/frexp.h>
-#include <clc/opencl/math/half_cos.h>
-#include <clc/opencl/math/half_divide.h>
-#include <clc/opencl/math/half_exp.h>
-#include <clc/opencl/math/half_exp10.h>
-#include <clc/opencl/math/half_exp2.h>
-#include <clc/opencl/math/half_log.h>
-#include <clc/opencl/math/half_log10.h>
-#include <clc/opencl/math/half_log2.h>
-#include <clc/opencl/math/half_powr.h>
-#include <clc/opencl/math/half_recip.h>
-#include <clc/opencl/math/half_rsqrt.h>
-#include <clc/opencl/math/half_sin.h>
-#include <clc/opencl/math/half_sqrt.h>
-#include <clc/opencl/math/half_tan.h>
-#include <clc/opencl/math/hypot.h>
-#include <clc/opencl/math/ilogb.h>
-#include <clc/opencl/math/ldexp.h>
-#include <clc/opencl/math/lgamma.h>
-#include <clc/opencl/math/lgamma_r.h>
-#include <clc/opencl/math/log.h>
-#include <clc/opencl/math/log10.h>
-#include <clc/opencl/math/log1p.h>
-#include <clc/opencl/math/log2.h>
-#include <clc/opencl/math/logb.h>
-#include <clc/opencl/math/mad.h>
-#include <clc/opencl/math/maxmag.h>
-#include <clc/opencl/math/minmag.h>
-#include <clc/opencl/math/modf.h>
-#include <clc/opencl/math/nan.h>
-#include <clc/opencl/math/native_cos.h>
-#include <clc/opencl/math/native_divide.h>
-#include <clc/opencl/math/native_exp.h>
-#include <clc/opencl/math/native_exp10.h>
-#include <clc/opencl/math/native_exp2.h>
-#include <clc/opencl/math/native_log.h>
-#include <clc/opencl/math/native_log10.h>
-#include <clc/opencl/math/native_log2.h>
-#include <clc/opencl/math/native_powr.h>
-#include <clc/opencl/math/native_recip.h>
-#include <clc/opencl/math/native_rsqrt.h>
-#include <clc/opencl/math/native_sin.h>
-#include <clc/opencl/math/native_sqrt.h>
-#include <clc/opencl/math/native_tan.h>
-#include <clc/opencl/math/nextafter.h>
-#include <clc/opencl/math/pow.h>
-#include <clc/opencl/math/pown.h>
-#include <clc/opencl/math/powr.h>
-#include <clc/opencl/math/remainder.h>
-#include <clc/opencl/math/remquo.h>
-#include <clc/opencl/math/rint.h>
-#include <clc/opencl/math/rootn.h>
-#include <clc/opencl/math/round.h>
-#include <clc/opencl/math/rsqrt.h>
-#include <clc/opencl/math/sin.h>
-#include <clc/opencl/math/sincos.h>
-#include <clc/opencl/math/sinh.h>
-#include <clc/opencl/math/sinpi.h>
-#include <clc/opencl/math/sqrt.h>
-#include <clc/opencl/math/tan.h>
-#include <clc/opencl/math/tanh.h>
-#include <clc/opencl/math/tanpi.h>
-#include <clc/opencl/math/tgamma.h>
-#include <clc/opencl/math/trunc.h>
-
-/* 6.11.2.1 Floating-point macros */
-#include <clc/float/definitions.h>
-
-/* 6.11.3 Integer Functions */
-#include <clc/opencl/integer/abs.h>
-#include <clc/opencl/integer/abs_diff.h>
-#include <clc/opencl/integer/add_sat.h>
-#include <clc/opencl/integer/clz.h>
-#include <clc/opencl/integer/ctz.h>
-#include <clc/opencl/integer/hadd.h>
-#include <clc/opencl/integer/mad24.h>
-#include <clc/opencl/integer/mad_hi.h>
-#include <clc/opencl/integer/mad_sat.h>
-#include <clc/opencl/integer/mul24.h>
-#include <clc/opencl/integer/mul_hi.h>
-#include <clc/opencl/integer/popcount.h>
-#include <clc/opencl/integer/rhadd.h>
-#include <clc/opencl/integer/rotate.h>
-#include <clc/opencl/integer/sub_sat.h>
-#include <clc/opencl/integer/upsample.h>
-
-/* 6.11.3 Integer Definitions */
-#include <clc/integer/definitions.h>
-
-/* 6.11.2 and 6.11.3 Shared Integer/Math Functions */
-#include <clc/opencl/shared/clamp.h>
-#include <clc/opencl/shared/max.h>
-#include <clc/opencl/shared/min.h>
-#include <clc/opencl/shared/vload.h>
-#include <clc/opencl/shared/vstore.h>
-
-/* 6.11.4 Common Functions */
-#include <clc/opencl/common/degrees.h>
-#include <clc/opencl/common/mix.h>
-#include <clc/opencl/common/radians.h>
-#include <clc/opencl/common/sign.h>
-#include <clc/opencl/common/smoothstep.h>
-#include <clc/opencl/common/step.h>
-
-/* 6.11.5 Geometric Functions */
-#include <clc/opencl/geometric/cross.h>
-#include <clc/opencl/geometric/distance.h>
-#include <clc/opencl/geometric/dot.h>
-#include <clc/opencl/geometric/fast_distance.h>
-#include <clc/opencl/geometric/fast_length.h>
-#include <clc/opencl/geometric/fast_normalize.h>
-#include <clc/opencl/geometric/length.h>
-#include <clc/opencl/geometric/normalize.h>
-
-/* 6.11.6 Relational Functions */
-#include <clc/opencl/relational/all.h>
-#include <clc/opencl/relational/any.h>
-#include <clc/opencl/relational/bitselect.h>
-#include <clc/opencl/relational/isequal.h>
-#include <clc/opencl/relational/isfinite.h>
-#include <clc/opencl/relational/isgreater.h>
-#include <clc/opencl/relational/isgreaterequal.h>
-#include <clc/opencl/relational/isinf.h>
-#include <clc/opencl/relational/isless.h>
-#include <clc/opencl/relational/islessequal.h>
-#include <clc/opencl/relational/islessgreater.h>
-#include <clc/opencl/relational/isnan.h>
-#include <clc/opencl/relational/isnormal.h>
-#include <clc/opencl/relational/isnotequal.h>
-#include <clc/opencl/relational/isordered.h>
-#include <clc/opencl/relational/isunordered.h>
-#include <clc/opencl/relational/select.h>
-#include <clc/opencl/relational/signbit.h>
-
-#include <clc/opencl/synchronization/cl_mem_fence_flags.h>
-
-/* 6.11.8 Synchronization Functions */
-#include <clc/opencl/synchronization/barrier.h>
-
-/* 6.11.9 Explicit Memory Fence Functions */
-#include <clc/opencl/explicit_fence/explicit_memory_fence.h>
-
-/* 6.11.10 Async Copy and Prefetch Functions */
-#include <clc/opencl/async/async_work_group_copy.h>
-#include <clc/opencl/async/async_work_group_strided_copy.h>
-#include <clc/opencl/async/prefetch.h>
-#include <clc/opencl/async/wait_group_events.h>
-
-/* 6.11.11 Atomic Functions */
-#include <clc/opencl/atomic/atomic_add.h>
-#include <clc/opencl/atomic/atomic_and.h>
-#include <clc/opencl/atomic/atomic_cmpxchg.h>
-#include <clc/opencl/atomic/atomic_dec.h>
-#include <clc/opencl/atomic/atomic_inc.h>
-#include <clc/opencl/atomic/atomic_max.h>
-#include <clc/opencl/atomic/atomic_min.h>
-#include <clc/opencl/atomic/atomic_or.h>
-#include <clc/opencl/atomic/atomic_sub.h>
-#include <clc/opencl/atomic/atomic_xchg.h>
-#include <clc/opencl/atomic/atomic_xor.h>
-
-/* cl_khr_global_int32_base_atomics, cl_khr_local_int32_base_atomics and
- * cl_khr_int64_base_atomics Extension Functions */
-#include <clc/opencl/atomic/atom_add.h>
-#include <clc/opencl/atomic/atom_cmpxchg.h>
-#include <clc/opencl/atomic/atom_dec.h>
-#include <clc/opencl/atomic/atom_inc.h>
-#include <clc/opencl/atomic/atom_sub.h>
-#include <clc/opencl/atomic/atom_xchg.h>
-
-/* cl_khr_global_int32_extended_atomics, cl_khr_local_int32_extended_atomics and
- * cl_khr_int64_extended_atomics Extension Functions */
-#include <clc/opencl/atomic/atom_and.h>
-#include <clc/opencl/atomic/atom_max.h>
-#include <clc/opencl/atomic/atom_min.h>
-#include <clc/opencl/atomic/atom_or.h>
-#include <clc/opencl/atomic/atom_xor.h>
-
-/* 6.12.12 Miscellaneous Vector Functions */
-#include <clc/opencl/misc/shuffle.h>
-#include <clc/opencl/misc/shuffle2.h>
-
-/* 6.11.13 Image Read and Write Functions */
-#include <clc/opencl/image/image.h>
-#include <clc/opencl/image/image_defines.h>
-
-#pragma OPENCL EXTENSION all : disable
-
-#endif // __CLC_OPENCL_CLC_H__
diff --git a/libclc/opencl/include/clc/opencl/common/degrees.h b/libclc/opencl/include/clc/opencl/common/degrees.h
index f442c5fbda857..5bd6f4e8f7fd5 100644
--- a/libclc/opencl/include/clc/opencl/common/degrees.h
+++ b/libclc/opencl/include/clc/opencl/common/degrees.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION degrees
+#ifndef __CLC_OPENCL_COMMON_DEGREES_H__
+#define __CLC_OPENCL_COMMON_DEGREES_H__
+
+#define FUNCTION degrees
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_COMMON_DEGREES_H__
diff --git a/libclc/opencl/include/clc/opencl/common/mix.h b/libclc/opencl/include/clc/opencl/common/mix.h
index 76aeaaba665ac..75863cf69946f 100644
--- a/libclc/opencl/include/clc/opencl/common/mix.h
+++ b/libclc/opencl/include/clc/opencl/common/mix.h
@@ -6,5 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_COMMON_MIX_H__
+#define __CLC_OPENCL_COMMON_MIX_H__
+
 #define __CLC_BODY <clc/opencl/common/mix.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_COMMON_MIX_H__
diff --git a/libclc/opencl/include/clc/opencl/common/radians.h b/libclc/opencl/include/clc/opencl/common/radians.h
index 2772f7c2f0ae8..3761a7e875be3 100644
--- a/libclc/opencl/include/clc/opencl/common/radians.h
+++ b/libclc/opencl/include/clc/opencl/common/radians.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION radians
+#ifndef __CLC_OPENCL_COMMON_RADIANS_H__
+#define __CLC_OPENCL_COMMON_RADIANS_H__
+
+#define FUNCTION radians
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_COMMON_RADIANS_H__
diff --git a/libclc/opencl/include/clc/opencl/common/sign.h b/libclc/opencl/include/clc/opencl/common/sign.h
index 2bb95716358fd..619e68ff1e14d 100644
--- a/libclc/opencl/include/clc/opencl/common/sign.h
+++ b/libclc/opencl/include/clc/opencl/common/sign.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION sign
+#ifndef __CLC_OPENCL_COMMON_SIGN_H__
+#define __CLC_OPENCL_COMMON_SIGN_H__
+
+#define FUNCTION sign
 #define __CLC_BODY <clc/math/unary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_COMMON_SIGN_H__
diff --git a/libclc/opencl/include/clc/opencl/common/smoothstep.h b/libclc/opencl/include/clc/opencl/common/smoothstep.h
index 87a642f8947a1..79fe7d37fc1be 100644
--- a/libclc/opencl/include/clc/opencl/common/smoothstep.h
+++ b/libclc/opencl/include/clc/opencl/common/smoothstep.h
@@ -6,5 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_COMMON_SMOOTHSTEP_H__
+#define __CLC_OPENCL_COMMON_SMOOTHSTEP_H__
+
 #define __CLC_BODY <clc/opencl/common/smoothstep.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_COMMON_SMOOTHSTEP_H__
diff --git a/libclc/opencl/include/clc/opencl/common/step.h b/libclc/opencl/include/clc/opencl/common/step.h
index d25a66f16ac3a..aa3882a9f1abf 100644
--- a/libclc/opencl/include/clc/opencl/common/step.h
+++ b/libclc/opencl/include/clc/opencl/common/step.h
@@ -6,5 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_COMMON_STEP_H__
+#define __CLC_OPENCL_COMMON_STEP_H__
+
 #define __CLC_BODY <clc/opencl/common/step.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_COMMON_STEP_H__
diff --git a/libclc/opencl/include/clc/opencl/convert.h b/libclc/opencl/include/clc/opencl/convert.h
index cdd7cc7b1503c..1f71f673619b9 100644
--- a/libclc/opencl/include/clc/opencl/convert.h
+++ b/libclc/opencl/include/clc/opencl/convert.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_CONVERT_H__
+#define __CLC_OPENCL_CONVERT_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_CONVERT_DECL(FROM_TYPE, TO_TYPE, SUFFIX)                          \
   _CLC_OVERLOAD _CLC_DECL TO_TYPE convert_##TO_TYPE##SUFFIX(FROM_TYPE x);
 
@@ -99,3 +104,5 @@ _CLC_VECTOR_CONVERT_TO_SUFFIX()
 #undef _CLC_VECTOR_CONVERT_FROM1
 #undef _CLC_VECTOR_CONVERT_DECL
 #undef _CLC_CONVERT_DECL
+
+#endif // __CLC_OPENCL_CONVERT_H__
diff --git a/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h b/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h
index 3ad3757d43e7d..3ade970a33c61 100644
--- a/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h
+++ b/libclc/opencl/include/clc/opencl/explicit_fence/explicit_memory_fence.h
@@ -6,6 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_EXPLICIT_FENCE_EXPLICIT_MEMORY_FENCE_H__
+#define __CLC_OPENCL_EXPLICIT_FENCE_EXPLICIT_MEMORY_FENCE_H__
+
+#include <clc/opencl/opencl-base.h>
+#include <clc/opencl/synchronization/cl_mem_fence_flags.h>
+
 _CLC_DECL _CLC_OVERLOAD void mem_fence(cl_mem_fence_flags flags);
 _CLC_DECL _CLC_OVERLOAD void read_mem_fence(cl_mem_fence_flags flags);
 _CLC_DECL _CLC_OVERLOAD void write_mem_fence(cl_mem_fence_flags flags);
+
+#endif // __CLC_OPENCL_EXPLICIT_FENCE_EXPLICIT_MEMORY_FENCE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/cross.h b/libclc/opencl/include/clc/opencl/geometric/cross.h
index 9d479e6eed885..be137965aea4b 100644
--- a/libclc/opencl/include/clc/opencl/geometric/cross.h
+++ b/libclc/opencl/include/clc/opencl/geometric/cross.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_GEOMETRIC_CROSS_H__
+#define __CLC_OPENCL_GEOMETRIC_CROSS_H__
+
 _CLC_OVERLOAD _CLC_DECL float3 cross(float3 p0, float3 p1);
 _CLC_OVERLOAD _CLC_DECL float4 cross(float4 p0, float4 p1);
 
@@ -13,3 +16,5 @@ _CLC_OVERLOAD _CLC_DECL float4 cross(float4 p0, float4 p1);
 _CLC_OVERLOAD _CLC_DECL double3 cross(double3 p0, double3 p1);
 _CLC_OVERLOAD _CLC_DECL double4 cross(double4 p0, double4 p1);
 #endif
+
+#endif // __CLC_OPENCL_GEOMETRIC_CROSS_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/distance.h b/libclc/opencl/include/clc/opencl/geometric/distance.h
index 28ec8274302ef..f9c219c44a49b 100644
--- a/libclc/opencl/include/clc/opencl/geometric/distance.h
+++ b/libclc/opencl/include/clc/opencl/geometric/distance.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION distance
+#ifndef __CLC_OPENCL_GEOMETRIC_DISTANCE_H__
+#define __CLC_OPENCL_GEOMETRIC_DISTANCE_H__
+
+#define FUNCTION distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_GEOMETRIC_DISTANCE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/dot.h b/libclc/opencl/include/clc/opencl/geometric/dot.h
index 92d5b56d2e067..0b391a10aa075 100644
--- a/libclc/opencl/include/clc/opencl/geometric/dot.h
+++ b/libclc/opencl/include/clc/opencl/geometric/dot.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION dot
+#ifndef __CLC_OPENCL_GEOMETRIC_DOT_H__
+#define __CLC_OPENCL_GEOMETRIC_DOT_H__
+
+#define FUNCTION dot
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_GEOMETRIC_DOT_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_distance.h b/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
index ef401fdf7289f..95b4966dfac1a 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_GEOMETRIC_FAST_DISTANCE_H__
+#define __CLC_OPENCL_GEOMETRIC_FAST_DISTANCE_H__
+
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION fast_distance
+#define FUNCTION fast_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_GEOMETRIC_FAST_DISTANCE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_length.h b/libclc/opencl/include/clc/opencl/geometric/fast_length.h
index b6fba6a8ca30c..78ad1fea981c3 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_length.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_length.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_GEOMETRIC_FAST_LENGTH_H__
+#define __CLC_OPENCL_GEOMETRIC_FAST_LENGTH_H__
+
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION fast_length
+#define FUNCTION fast_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_GEOMETRIC_FAST_LENGTH_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h b/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
index 1af81c66973b7..7a70dd5df5196 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
@@ -6,13 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_GEOMETRIC_FAST_NORMALIZE_H__
+#define __CLC_OPENCL_GEOMETRIC_FAST_NORMALIZE_H__
+
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION fast_normalize
+#define FUNCTION fast_normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef __FLOAT_ONLY
+
+#endif // __CLC_OPENCL_GEOMETRIC_FAST_NORMALIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/length.h b/libclc/opencl/include/clc/opencl/geometric/length.h
index 2105a8f8cece0..5dd6ff5a9cad4 100644
--- a/libclc/opencl/include/clc/opencl/geometric/length.h
+++ b/libclc/opencl/include/clc/opencl/geometric/length.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION length
+#ifndef __CLC_OPENCL_GEOMETRIC_LENGTH_H__
+#define __CLC_OPENCL_GEOMETRIC_LENGTH_H__
+
+#define FUNCTION length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_GEOMETRIC_LENGTH_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/normalize.h b/libclc/opencl/include/clc/opencl/geometric/normalize.h
index a15b7003329b5..7d3e8fa98f8ed 100644
--- a/libclc/opencl/include/clc/opencl/geometric/normalize.h
+++ b/libclc/opencl/include/clc/opencl/geometric/normalize.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION normalize
+#ifndef __CLC_OPENCL_GEOMETRIC_NORMALIZE_H__
+#define __CLC_OPENCL_GEOMETRIC_NORMALIZE_H__
+
+#define FUNCTION normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_GEOMETRIC_NORMALIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/image/image.h b/libclc/opencl/include/clc/opencl/image/image.h
index 6435f3430e2fe..7a248304acff5 100644
--- a/libclc/opencl/include/clc/opencl/image/image.h
+++ b/libclc/opencl/include/clc/opencl/image/image.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_IMAGE_H__
+#define __CLC_OPENCL_IMAGE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #if defined(__opencl_c_images)
 
 _CLC_OVERLOAD _CLC_DECL int get_image_width(image2d_t image);
@@ -46,3 +51,5 @@ _CLC_OVERLOAD _CLC_DECL uint4 read_imageui(image2d_t image, sampler_t sampler,
                                            float2 coord);
 
 #endif
+
+#endif // __CLC_OPENCL_IMAGE_H__
diff --git a/libclc/opencl/include/clc/opencl/image/image_defines.h b/libclc/opencl/include/clc/opencl/image/image_defines.h
index f3589fdc4a89a..75e08dec3f044 100644
--- a/libclc/opencl/include/clc/opencl/image/image_defines.h
+++ b/libclc/opencl/include/clc/opencl/image/image_defines.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_IMAGE_IMAGE_DEFINES_H__
+#define __CLC_OPENCL_IMAGE_IMAGE_DEFINES_H__
+
 /* get_image_channel_data_type flags */
 #define CLK_SNORM_INT8 0x10D0
 #define CLK_SNORM_INT16 0x10D1
@@ -55,3 +58,5 @@
 #define CLK_FILTER_NEAREST 0x0000
 #define CLK_FILTER_LINEAR 0x0010
 #define __CLC_FILTER_MASK 0x0010
+
+#endif // __CLC_OPENCL_IMAGE_IMAGE_DEFINES_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/abs.h b/libclc/opencl/include/clc/opencl/integer/abs.h
index 22ba40fb40d42..3059bef9a7e4f 100644
--- a/libclc/opencl/include/clc/opencl/integer/abs.h
+++ b/libclc/opencl/include/clc/opencl/integer/abs.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_ABS_H__
+#define __CLC_OPENCL_INTEGER_ABS_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_BODY <clc/opencl/integer/abs.inc>
 #include <clc/integer/gentype.inc>
+
+#endif // __CLC_OPENCL_INTEGER_ABS_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/abs_diff.h b/libclc/opencl/include/clc/opencl/integer/abs_diff.h
index f22cc4d88afcf..9e8fe2b31fed7 100644
--- a/libclc/opencl/include/clc/opencl/integer/abs_diff.h
+++ b/libclc/opencl/include/clc/opencl/integer/abs_diff.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_ABS_DIFF_H__
+#define __CLC_OPENCL_INTEGER_ABS_DIFF_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_BODY <clc/opencl/integer/abs_diff.inc>
 #include <clc/integer/gentype.inc>
+
+#endif // __CLC_OPENCL_INTEGER_ABS_DIFF_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/add_sat.h b/libclc/opencl/include/clc/opencl/integer/add_sat.h
index 1982d11908008..cd2c17a096299 100644
--- a/libclc/opencl/include/clc/opencl/integer/add_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/add_sat.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION add_sat
+#ifndef __CLC_OPENCL_INTEGER_ADD_SAT_H__
+#define __CLC_OPENCL_INTEGER_ADD_SAT_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION add_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_ADD_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/clz.h b/libclc/opencl/include/clc/opencl/integer/clz.h
index b215fc1d0bc94..77032ceeaf0f4 100644
--- a/libclc/opencl/include/clc/opencl/integer/clz.h
+++ b/libclc/opencl/include/clc/opencl/integer/clz.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION clz
+#ifndef __CLC_OPENCL_INTEGER_CLZ_H__
+#define __CLC_OPENCL_INTEGER_CLZ_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION clz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_CLZ_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/ctz.h b/libclc/opencl/include/clc/opencl/integer/ctz.h
index 2ab6436440853..b861225846ee4 100644
--- a/libclc/opencl/include/clc/opencl/integer/ctz.h
+++ b/libclc/opencl/include/clc/opencl/integer/ctz.h
@@ -6,13 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_CTZ_H__
+#define __CLC_OPENCL_INTEGER_CTZ_H__
+
 #if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 
-#define __CLC_FUNCTION ctz
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION ctz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __OPENCL_C_VERSION__ >= CL_VERSION_2_0
+
+#endif // __CLC_OPENCL_INTEGER_CTZ_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/hadd.h b/libclc/opencl/include/clc/opencl/integer/hadd.h
index c05c9d50e67d3..c4c8f7daf4014 100644
--- a/libclc/opencl/include/clc/opencl/integer/hadd.h
+++ b/libclc/opencl/include/clc/opencl/integer/hadd.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION hadd
+#ifndef __CLC_OPENCL_INTEGER_HADD_H__
+#define __CLC_OPENCL_INTEGER_HADD_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION hadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_HADD_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad24.h b/libclc/opencl/include/clc/opencl/integer/mad24.h
index 9ff998a8604d4..daee9a0383477 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad24.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad24.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mad24
+#ifndef __CLC_OPENCL_INTEGER_MAD24_H__
+#define __CLC_OPENCL_INTEGER_MAD24_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mad24
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MAD24_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad_hi.h b/libclc/opencl/include/clc/opencl/integer/mad_hi.h
index e06ab8f3c11f1..bbcdcd9c82ebd 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad_hi.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad_hi.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mad_hi
+#ifndef __CLC_OPENCL_INTEGER_MAD_HI_H__
+#define __CLC_OPENCL_INTEGER_MAD_HI_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mad_hi
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MAD_HI_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad_sat.h b/libclc/opencl/include/clc/opencl/integer/mad_sat.h
index 0a9fd7869a6f4..28045477975ed 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad_sat.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mad_sat
+#ifndef __CLC_OPENCL_INTEGER_MAD_SAT_H__
+#define __CLC_OPENCL_INTEGER_MAD_SAT_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mad_sat
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MAD_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mul24.h b/libclc/opencl/include/clc/opencl/integer/mul24.h
index d2aca25fc3140..6aa799a28b7ec 100644
--- a/libclc/opencl/include/clc/opencl/integer/mul24.h
+++ b/libclc/opencl/include/clc/opencl/integer/mul24.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mul24
+#ifndef __CLC_OPENCL_INTEGER_MUL24_H__
+#define __CLC_OPENCL_INTEGER_MUL24_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mul24
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MUL24_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mul_hi.h b/libclc/opencl/include/clc/opencl/integer/mul_hi.h
index 4374ec73be32a..ed7ac7319cab8 100644
--- a/libclc/opencl/include/clc/opencl/integer/mul_hi.h
+++ b/libclc/opencl/include/clc/opencl/integer/mul_hi.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mul_hi
+#ifndef __CLC_OPENCL_INTEGER_MUL_HI_H__
+#define __CLC_OPENCL_INTEGER_MUL_HI_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mul_hi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MUL_HI_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/popcount.h b/libclc/opencl/include/clc/opencl/integer/popcount.h
index 4d43f26f489e6..43d5bc0f0b335 100644
--- a/libclc/opencl/include/clc/opencl/integer/popcount.h
+++ b/libclc/opencl/include/clc/opencl/integer/popcount.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION popcount
+#ifndef __CLC_OPENCL_INTEGER_POPCOUNT_H__
+#define __CLC_OPENCL_INTEGER_POPCOUNT_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION popcount
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_POPCOUNT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/rhadd.h b/libclc/opencl/include/clc/opencl/integer/rhadd.h
index 3da5524df206c..2c93c100cf02f 100644
--- a/libclc/opencl/include/clc/opencl/integer/rhadd.h
+++ b/libclc/opencl/include/clc/opencl/integer/rhadd.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION rhadd
+#ifndef __CLC_OPENCL_INTEGER_RHADD_H__
+#define __CLC_OPENCL_INTEGER_RHADD_H__
+
+#define FUNCTION rhadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_RHADD_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/rotate.h b/libclc/opencl/include/clc/opencl/integer/rotate.h
index 1e4de24cbf313..156155ae0638a 100644
--- a/libclc/opencl/include/clc/opencl/integer/rotate.h
+++ b/libclc/opencl/include/clc/opencl/integer/rotate.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION rotate
+#ifndef __CLC_OPENCL_INTEGER_ROTATE_H__
+#define __CLC_OPENCL_INTEGER_ROTATE_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION rotate
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_ROTATE_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/sub_sat.h b/libclc/opencl/include/clc/opencl/integer/sub_sat.h
index ee5d8c78f3ebe..a7861ee1ac4a1 100644
--- a/libclc/opencl/include/clc/opencl/integer/sub_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/sub_sat.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION sub_sat
+#ifndef __CLC_OPENCL_INTEGER_SUB_SAT_H__
+#define __CLC_OPENCL_INTEGER_SUB_SAT_H__
+
+#define FUNCTION sub_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_SUB_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/upsample.h b/libclc/opencl/include/clc/opencl/integer/upsample.h
index 0db67dfe4e973..a713f2b392f27 100644
--- a/libclc/opencl/include/clc/opencl/integer/upsample.h
+++ b/libclc/opencl/include/clc/opencl/integer/upsample.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_UPSAMPLE_H__
+#define __CLC_OPENCL_INTEGER_UPSAMPLE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                       \
   _CLC_OVERLOAD _CLC_DECL BGENTYPE upsample(GENTYPE hi, UGENTYPE lo);
 
@@ -30,3 +35,5 @@ __CLC_UPSAMPLE_TYPES()
 #undef __CLC_UPSAMPLE_TYPES
 #undef __CLC_UPSAMPLE_DECL
 #undef __CLC_UPSAMPLE_VEC
+
+#endif // __CLC_OPENCL_INTEGER_UPSAMPLE_H__
diff --git a/libclc/opencl/include/clc/opencl/math/acos.h b/libclc/opencl/include/clc/opencl/math/acos.h
index 695e6560478c8..f9e9b44bb9041 100644
--- a/libclc/opencl/include/clc/opencl/math/acos.h
+++ b/libclc/opencl/include/clc/opencl/math/acos.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ACOS_H__
+#define __CLC_OPENCL_MATH_ACOS_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION acos
+#define FUNCTION acos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ACOS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/acosh.h b/libclc/opencl/include/clc/opencl/math/acosh.h
index 52197d448d6be..241b305e6bc87 100644
--- a/libclc/opencl/include/clc/opencl/math/acosh.h
+++ b/libclc/opencl/include/clc/opencl/math/acosh.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ACOSH_H__
+#define __CLC_OPENCL_MATH_ACOSH_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION acosh
+#define FUNCTION acosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ACOSH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/acospi.h b/libclc/opencl/include/clc/opencl/math/acospi.h
index 47e4e5f3e7f15..7014889dfb7ef 100644
--- a/libclc/opencl/include/clc/opencl/math/acospi.h
+++ b/libclc/opencl/include/clc/opencl/math/acospi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ACOSPI_H__
+#define __CLC_OPENCL_MATH_ACOSPI_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION acospi
+#define FUNCTION acospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ACOSPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/asin.h b/libclc/opencl/include/clc/opencl/math/asin.h
index eb83d2d2743f3..1ba1f459a9024 100644
--- a/libclc/opencl/include/clc/opencl/math/asin.h
+++ b/libclc/opencl/include/clc/opencl/math/asin.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ASIN_H__
+#define __CLC_OPENCL_MATH_ASIN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION asin
+#define FUNCTION asin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ASIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/asinh.h b/libclc/opencl/include/clc/opencl/math/asinh.h
index 74ce5ba6731c6..9a530a51fa1ee 100644
--- a/libclc/opencl/include/clc/opencl/math/asinh.h
+++ b/libclc/opencl/include/clc/opencl/math/asinh.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ASINH_H__
+#define __CLC_OPENCL_MATH_ASINH_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION asinh
+#define FUNCTION asinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ASINH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/asinpi.h b/libclc/opencl/include/clc/opencl/math/asinpi.h
index a7bd6469c93a9..844848fff0433 100644
--- a/libclc/opencl/include/clc/opencl/math/asinpi.h
+++ b/libclc/opencl/include/clc/opencl/math/asinpi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ASINPI_H__
+#define __CLC_OPENCL_MATH_ASINPI_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION asinpi
+#define FUNCTION asinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ASINPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atan.h b/libclc/opencl/include/clc/opencl/math/atan.h
index 4c83c2d4c819b..f2312b8fb0737 100644
--- a/libclc/opencl/include/clc/opencl/math/atan.h
+++ b/libclc/opencl/include/clc/opencl/math/atan.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ATAN_H__
+#define __CLC_OPENCL_MATH_ATAN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION atan
+#define FUNCTION atan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ATAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atan2.h b/libclc/opencl/include/clc/opencl/math/atan2.h
index 5872e41218dc4..72412d44aafb1 100644
--- a/libclc/opencl/include/clc/opencl/math/atan2.h
+++ b/libclc/opencl/include/clc/opencl/math/atan2.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atan2
+#ifndef __CLC_OPENCL_MATH_ATAN2_H__
+#define __CLC_OPENCL_MATH_ATAN2_H__
+
+#define FUNCTION atan2
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ATAN2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atan2pi.h b/libclc/opencl/include/clc/opencl/math/atan2pi.h
index b68cf492e2925..f12a19776ef23 100644
--- a/libclc/opencl/include/clc/opencl/math/atan2pi.h
+++ b/libclc/opencl/include/clc/opencl/math/atan2pi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atan2pi
+#ifndef __CLC_OPENCL_MATH_ATAN2PI_H__
+#define __CLC_OPENCL_MATH_ATAN2PI_H__
+
+#define FUNCTION atan2pi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ATAN2PI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atanh.h b/libclc/opencl/include/clc/opencl/math/atanh.h
index 6ea99c2dedc40..147a76b543e61 100644
--- a/libclc/opencl/include/clc/opencl/math/atanh.h
+++ b/libclc/opencl/include/clc/opencl/math/atanh.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ATANH_H__
+#define __CLC_OPENCL_MATH_ATANH_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION atanh
+#define FUNCTION atanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ATANH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atanpi.h b/libclc/opencl/include/clc/opencl/math/atanpi.h
index 5c7d0a9130fd0..12e9ea2501705 100644
--- a/libclc/opencl/include/clc/opencl/math/atanpi.h
+++ b/libclc/opencl/include/clc/opencl/math/atanpi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ATANPI_H__
+#define __CLC_OPENCL_MATH_ATANPI_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION atanpi
+#define FUNCTION atanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ATANPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cbrt.h b/libclc/opencl/include/clc/opencl/math/cbrt.h
index f0599e0805ab7..fdb408ac96858 100644
--- a/libclc/opencl/include/clc/opencl/math/cbrt.h
+++ b/libclc/opencl/include/clc/opencl/math/cbrt.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_CBRT_H__
+#define __CLC_OPENCL_MATH_CBRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cbrt
+#define FUNCTION cbrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_CBRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/ceil.h b/libclc/opencl/include/clc/opencl/math/ceil.h
index 82e5dc4eec000..dad2ea4f69807 100644
--- a/libclc/opencl/include/clc/opencl/math/ceil.h
+++ b/libclc/opencl/include/clc/opencl/math/ceil.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_CEIL_H__
+#define __CLC_OPENCL_MATH_CEIL_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION ceil
+#define FUNCTION ceil
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_CEIL_H__
diff --git a/libclc/opencl/include/clc/opencl/math/copysign.h b/libclc/opencl/include/clc/opencl/math/copysign.h
index 3956dcd2fc19c..10d6c90edf81f 100644
--- a/libclc/opencl/include/clc/opencl/math/copysign.h
+++ b/libclc/opencl/include/clc/opencl/math/copysign.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION copysign
+#ifndef __CLC_OPENCL_MATH_COPYSIGN_H__
+#define __CLC_OPENCL_MATH_COPYSIGN_H__
+
+#define FUNCTION copysign
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_COPYSIGN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cos.h b/libclc/opencl/include/clc/opencl/math/cos.h
index 073282a3e0c1e..4e3aec6c69628 100644
--- a/libclc/opencl/include/clc/opencl/math/cos.h
+++ b/libclc/opencl/include/clc/opencl/math/cos.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_COS_H__
+#define __CLC_OPENCL_MATH_COS_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cos
+#define FUNCTION cos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_COS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cosh.h b/libclc/opencl/include/clc/opencl/math/cosh.h
index f156194cd9b14..f316433a23514 100644
--- a/libclc/opencl/include/clc/opencl/math/cosh.h
+++ b/libclc/opencl/include/clc/opencl/math/cosh.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_COSH_H__
+#define __CLC_OPENCL_MATH_COSH_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cosh
+#define FUNCTION cosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_COSH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cospi.h b/libclc/opencl/include/clc/opencl/math/cospi.h
index 2e9ae1b2ed376..b7dfb0dbafb58 100644
--- a/libclc/opencl/include/clc/opencl/math/cospi.h
+++ b/libclc/opencl/include/clc/opencl/math/cospi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_COSPI_H__
+#define __CLC_OPENCL_MATH_COSPI_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cospi
+#define FUNCTION cospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_COSPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/erf.h b/libclc/opencl/include/clc/opencl/math/erf.h
index cbbb58b6c35a7..8ce262f276e39 100644
--- a/libclc/opencl/include/clc/opencl/math/erf.h
+++ b/libclc/opencl/include/clc/opencl/math/erf.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ERF_H__
+#define __CLC_OPENCL_MATH_ERF_H__
+
 #undef erfc
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION erf
+#define FUNCTION erf
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ERF_H__
diff --git a/libclc/opencl/include/clc/opencl/math/erfc.h b/libclc/opencl/include/clc/opencl/math/erfc.h
index 74cef50ecc7c8..26afe1315df60 100644
--- a/libclc/opencl/include/clc/opencl/math/erfc.h
+++ b/libclc/opencl/include/clc/opencl/math/erfc.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ERFC_H__
+#define __CLC_OPENCL_MATH_ERFC_H__
+
 #undef erfc
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION erfc
+#define FUNCTION erfc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ERFC_H__
diff --git a/libclc/opencl/include/clc/opencl/math/exp.h b/libclc/opencl/include/clc/opencl/math/exp.h
index a6193a9eac844..fb8f674a8c7e5 100644
--- a/libclc/opencl/include/clc/opencl/math/exp.h
+++ b/libclc/opencl/include/clc/opencl/math/exp.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_EXP_H__
+#define __CLC_OPENCL_MATH_EXP_H__
+
 #undef exp
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION exp
+#define FUNCTION exp
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_EXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/exp10.h b/libclc/opencl/include/clc/opencl/math/exp10.h
index 7b48c1ce0348f..cfd5c6df9d15c 100644
--- a/libclc/opencl/include/clc/opencl/math/exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/exp10.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_EXP10_H__
+#define __CLC_OPENCL_MATH_EXP10_H__
+
 #undef exp10
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION exp10
+#define FUNCTION exp10
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_EXP10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/exp2.h b/libclc/opencl/include/clc/opencl/math/exp2.h
index 7081050769e9a..db0405eb13341 100644
--- a/libclc/opencl/include/clc/opencl/math/exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/exp2.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_EXP2_H__
+#define __CLC_OPENCL_MATH_EXP2_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION exp2
+#define FUNCTION exp2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_EXP2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/expm1.h b/libclc/opencl/include/clc/opencl/math/expm1.h
index 9f6a512df38e6..0b3628d48468e 100644
--- a/libclc/opencl/include/clc/opencl/math/expm1.h
+++ b/libclc/opencl/include/clc/opencl/math/expm1.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_EXPM1_H__
+#define __CLC_OPENCL_MATH_EXPM1_H__
+
 #undef exp
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION expm1
+#define FUNCTION expm1
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_EXPM1_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fabs.h b/libclc/opencl/include/clc/opencl/math/fabs.h
index 41212d6f597c8..f20016f4eb8ba 100644
--- a/libclc/opencl/include/clc/opencl/math/fabs.h
+++ b/libclc/opencl/include/clc/opencl/math/fabs.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_FABS_H__
+#define __CLC_OPENCL_MATH_FABS_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION fabs
+#define FUNCTION fabs
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FABS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fdim.h b/libclc/opencl/include/clc/opencl/math/fdim.h
index fdc261072c164..82f70fd832361 100644
--- a/libclc/opencl/include/clc/opencl/math/fdim.h
+++ b/libclc/opencl/include/clc/opencl/math/fdim.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION fdim
+#ifndef __CLC_OPENCL_MATH_FDIM_H__
+#define __CLC_OPENCL_MATH_FDIM_H__
+
+#define FUNCTION fdim
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FDIM_H__
diff --git a/libclc/opencl/include/clc/opencl/math/floor.h b/libclc/opencl/include/clc/opencl/math/floor.h
index 80f8fcc5ac95d..1930aeca74ff0 100644
--- a/libclc/opencl/include/clc/opencl/math/floor.h
+++ b/libclc/opencl/include/clc/opencl/math/floor.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_FLOOR_H__
+#define __CLC_OPENCL_MATH_FLOOR_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION floor
+#define FUNCTION floor
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FLOOR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fma.h b/libclc/opencl/include/clc/opencl/math/fma.h
index 9a201bb98400f..6fed3fc9b2560 100644
--- a/libclc/opencl/include/clc/opencl/math/fma.h
+++ b/libclc/opencl/include/clc/opencl/math/fma.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_FMA_H__
+#define __CLC_OPENCL_MATH_FMA_H__
+
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define __CLC_FUNCTION fma
+#define FUNCTION fma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FMA_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fmax.h b/libclc/opencl/include/clc/opencl/math/fmax.h
index 2144bb334f1ae..84c754e2c0c71 100644
--- a/libclc/opencl/include/clc/opencl/math/fmax.h
+++ b/libclc/opencl/include/clc/opencl/math/fmax.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_FMAX_H__
+#define __CLC_OPENCL_MATH_FMAX_H__
+
 #define __CLC_BODY <clc/math/binary_decl_with_scalar_second_arg.inc>
-#define __CLC_FUNCTION fmax
+#define FUNCTION fmax
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FMAX_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fmin.h b/libclc/opencl/include/clc/opencl/math/fmin.h
index 5a2af3725bf2b..b35573e4f6337 100644
--- a/libclc/opencl/include/clc/opencl/math/fmin.h
+++ b/libclc/opencl/include/clc/opencl/math/fmin.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_FMIN_H__
+#define __CLC_OPENCL_MATH_FMIN_H__
+
 #define __CLC_BODY <clc/math/binary_decl_with_scalar_second_arg.inc>
-#define __CLC_FUNCTION fmin
+#define FUNCTION fmin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FMIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fmod.h b/libclc/opencl/include/clc/opencl/math/fmod.h
index 10236aec3e9e3..13515006fd2be 100644
--- a/libclc/opencl/include/clc/opencl/math/fmod.h
+++ b/libclc/opencl/include/clc/opencl/math/fmod.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION fmod
+#ifndef __CLC_OPENCL_MATH_FMOD_H__
+#define __CLC_OPENCL_MATH_FMOD_H__
+
+#define FUNCTION fmod
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FMOD_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fract.h b/libclc/opencl/include/clc/opencl/math/fract.h
index 8443d4968c985..de953f78c5940 100644
--- a/libclc/opencl/include/clc/opencl/math/fract.h
+++ b/libclc/opencl/include/clc/opencl/math/fract.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION fract
+#ifndef __CLC_OPENCL_MATH_FRACT_H__
+#define __CLC_OPENCL_MATH_FRACT_H__
+
+#define FUNCTION fract
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FRACT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/frexp.h b/libclc/opencl/include/clc/opencl/math/frexp.h
index 796acca1cb75a..c7ed7cbd1cbc4 100644
--- a/libclc/opencl/include/clc/opencl/math/frexp.h
+++ b/libclc/opencl/include/clc/opencl/math/frexp.h
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION frexp
+#ifndef __CLC_OPENCL_MATH_FREXP_H__
+#define __CLC_OPENCL_MATH_FREXP_H__
+
+#define FUNCTION frexp
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_FREXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_cos.h b/libclc/opencl/include/clc/opencl/math/half_cos.h
index c9a5a88f80910..5ab0b69ca4224 100644
--- a/libclc/opencl/include/clc/opencl/math/half_cos.h
+++ b/libclc/opencl/include/clc/opencl/math/half_cos.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_COS_H__
+#define __CLC_OPENCL_MATH_HALF_COS_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_cos
+#define FUNCTION half_cos
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_COS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_divide.h b/libclc/opencl/include/clc/opencl/math/half_divide.h
index a05eab5b1c78e..ff506d0c9fd6d 100644
--- a/libclc/opencl/include/clc/opencl/math/half_divide.h
+++ b/libclc/opencl/include/clc/opencl/math/half_divide.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_DIVIDE_H__
+#define __CLC_OPENCL_MATH_HALF_DIVIDE_H__
+
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION half_divide
+#define FUNCTION half_divide
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_DIVIDE_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp.h b/libclc/opencl/include/clc/opencl/math/half_exp.h
index 65734878eb4e6..22780c144923c 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_EXP_H__
+#define __CLC_OPENCL_MATH_HALF_EXP_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_exp
+#define FUNCTION half_exp
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_EXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp10.h b/libclc/opencl/include/clc/opencl/math/half_exp10.h
index 4df20a7aa6222..1a8195e67eee0 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp10.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_EXP10_H__
+#define __CLC_OPENCL_MATH_HALF_EXP10_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_exp10
+#define FUNCTION half_exp10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_EXP10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp2.h b/libclc/opencl/include/clc/opencl/math/half_exp2.h
index 2eed851fa698f..2ca69aa1ea076 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp2.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_EXP2_H__
+#define __CLC_OPENCL_MATH_HALF_EXP2_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_exp2
+#define FUNCTION half_exp2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_EXP2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_log.h b/libclc/opencl/include/clc/opencl/math/half_log.h
index 3d44680570098..d2eef868790f5 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_LOG_H__
+#define __CLC_OPENCL_MATH_HALF_LOG_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_log
+#define FUNCTION half_log
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_LOG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_log10.h b/libclc/opencl/include/clc/opencl/math/half_log10.h
index 1c468f3f33985..e0f85fff3f3ea 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log10.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log10.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_LOG10_H__
+#define __CLC_OPENCL_MATH_HALF_LOG10_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_log10
+#define FUNCTION half_log10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_LOG10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_log2.h b/libclc/opencl/include/clc/opencl/math/half_log2.h
index 0695e2542962f..ba0105d883568 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log2.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log2.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_LOG2_H__
+#define __CLC_OPENCL_MATH_HALF_LOG2_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_log2
+#define FUNCTION half_log2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_LOG2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_powr.h b/libclc/opencl/include/clc/opencl/math/half_powr.h
index 27aa2b0fab666..4b792c0c6cba4 100644
--- a/libclc/opencl/include/clc/opencl/math/half_powr.h
+++ b/libclc/opencl/include/clc/opencl/math/half_powr.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_POWR_H__
+#define __CLC_OPENCL_MATH_HALF_POWR_H__
+
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION half_powr
+#define FUNCTION half_powr
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_POWR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_recip.h b/libclc/opencl/include/clc/opencl/math/half_recip.h
index 8e2d16099f9f7..3425df4e476d8 100644
--- a/libclc/opencl/include/clc/opencl/math/half_recip.h
+++ b/libclc/opencl/include/clc/opencl/math/half_recip.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_RECIP_H__
+#define __CLC_OPENCL_MATH_HALF_RECIP_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_recip
+#define FUNCTION half_recip
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_RECIP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_rsqrt.h b/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
index e93973bc64575..e22a7375cacda 100644
--- a/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
@@ -6,9 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_RSQRT_H__
+#define __CLC_OPENCL_MATH_HALF_RSQRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_rsqrt
+#define FUNCTION half_rsqrt
 #define __FLOAT_ONLY
 #include <clc/math/gentype.inc>
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_RSQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_sin.h b/libclc/opencl/include/clc/opencl/math/half_sin.h
index 50626f9d96fe6..cb0462c45436f 100644
--- a/libclc/opencl/include/clc/opencl/math/half_sin.h
+++ b/libclc/opencl/include/clc/opencl/math/half_sin.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_SIN_H__
+#define __CLC_OPENCL_MATH_HALF_SIN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_sin
+#define FUNCTION half_sin
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_SIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_sqrt.h b/libclc/opencl/include/clc/opencl/math/half_sqrt.h
index 7298570984baa..f755fee7b4be2 100644
--- a/libclc/opencl/include/clc/opencl/math/half_sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/half_sqrt.h
@@ -6,9 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_SQRT_H__
+#define __CLC_OPENCL_MATH_HALF_SQRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_sqrt
+#define FUNCTION half_sqrt
 #define __FLOAT_ONLY
 #include <clc/math/gentype.inc>
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_SQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_tan.h b/libclc/opencl/include/clc/opencl/math/half_tan.h
index 40ae6e6c9aba9..91b5b11ae1789 100644
--- a/libclc/opencl/include/clc/opencl/math/half_tan.h
+++ b/libclc/opencl/include/clc/opencl/math/half_tan.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_HALF_TAN_H__
+#define __CLC_OPENCL_MATH_HALF_TAN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_tan
+#define FUNCTION half_tan
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HALF_TAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/hypot.h b/libclc/opencl/include/clc/opencl/math/hypot.h
index a9d464af95c0a..76ff6ef0655a9 100644
--- a/libclc/opencl/include/clc/opencl/math/hypot.h
+++ b/libclc/opencl/include/clc/opencl/math/hypot.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION hypot
+#ifndef __CLC_OPENCL_MATH_HYPOT_H__
+#define __CLC_OPENCL_MATH_HYPOT_H__
+
+#define FUNCTION hypot
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_HYPOT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/ilogb.h b/libclc/opencl/include/clc/opencl/math/ilogb.h
index 45d8ea064b464..d9752a644bee4 100644
--- a/libclc/opencl/include/clc/opencl/math/ilogb.h
+++ b/libclc/opencl/include/clc/opencl/math/ilogb.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION ilogb
+#ifndef __CLC_OPENCL_MATH_ILOGB_H__
+#define __CLC_OPENCL_MATH_ILOGB_H__
+
+#define FUNCTION ilogb
 #define __CLC_BODY <clc/math/unary_decl_with_int_return.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ILOGB_H__
diff --git a/libclc/opencl/include/clc/opencl/math/ldexp.h b/libclc/opencl/include/clc/opencl/math/ldexp.h
index ca50ae6a98312..b0194d9c9adad 100644
--- a/libclc/opencl/include/clc/opencl/math/ldexp.h
+++ b/libclc/opencl/include/clc/opencl/math/ldexp.h
@@ -6,10 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION ldexp
+#ifndef __CLC_OPENCL_MATH_LDEXP_H__
+#define __CLC_OPENCL_MATH_LDEXP_H__
+
+#define FUNCTION ldexp
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #define __CLC_BODY <clc/opencl/math/ldexp.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_MATH_LDEXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/lgamma.h b/libclc/opencl/include/clc/opencl/math/lgamma.h
index 9f0be722b5f63..95ed07c3531ee 100644
--- a/libclc/opencl/include/clc/opencl/math/lgamma.h
+++ b/libclc/opencl/include/clc/opencl/math/lgamma.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_LGAMMA_H__
+#define __CLC_OPENCL_MATH_LGAMMA_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION lgamma
+#define FUNCTION lgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LGAMMA_H__
diff --git a/libclc/opencl/include/clc/opencl/math/lgamma_r.h b/libclc/opencl/include/clc/opencl/math/lgamma_r.h
index 78438b1be5514..4853d2dd5c283 100644
--- a/libclc/opencl/include/clc/opencl/math/lgamma_r.h
+++ b/libclc/opencl/include/clc/opencl/math/lgamma_r.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION lgamma_r
+#ifndef __CLC_OPENCL_MATH_LGAMMA_R_H__
+#define __CLC_OPENCL_MATH_LGAMMA_R_H__
+
+#define FUNCTION lgamma_r
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LGAMMA_R_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log.h b/libclc/opencl/include/clc/opencl/math/log.h
index 8fec73d7fb69b..2d536d6b09266 100644
--- a/libclc/opencl/include/clc/opencl/math/log.h
+++ b/libclc/opencl/include/clc/opencl/math/log.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_LOG_H__
+#define __CLC_OPENCL_MATH_LOG_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log
+#define FUNCTION log
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LOG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log10.h b/libclc/opencl/include/clc/opencl/math/log10.h
index 21cf2a976942d..0f83b8c2a03e9 100644
--- a/libclc/opencl/include/clc/opencl/math/log10.h
+++ b/libclc/opencl/include/clc/opencl/math/log10.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_LOG10_H__
+#define __CLC_OPENCL_MATH_LOG10_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log10
+#define FUNCTION log10
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LOG10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log1p.h b/libclc/opencl/include/clc/opencl/math/log1p.h
index 643207ebb9474..bd8adeda88d4e 100644
--- a/libclc/opencl/include/clc/opencl/math/log1p.h
+++ b/libclc/opencl/include/clc/opencl/math/log1p.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_LOG1P_H__
+#define __CLC_OPENCL_MATH_LOG1P_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log1p
+#define FUNCTION log1p
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LOG1P_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log2.h b/libclc/opencl/include/clc/opencl/math/log2.h
index 26b407f1019b1..e4f6cb619acf2 100644
--- a/libclc/opencl/include/clc/opencl/math/log2.h
+++ b/libclc/opencl/include/clc/opencl/math/log2.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_LOG2_H__
+#define __CLC_OPENCL_MATH_LOG2_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log2
+#define FUNCTION log2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LOG2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/logb.h b/libclc/opencl/include/clc/opencl/math/logb.h
index a0daa98063810..3bbc8476d2ffa 100644
--- a/libclc/opencl/include/clc/opencl/math/logb.h
+++ b/libclc/opencl/include/clc/opencl/math/logb.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_LOGB_H__
+#define __CLC_OPENCL_MATH_LOGB_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION logb
+#define FUNCTION logb
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_LOGB_H__
diff --git a/libclc/opencl/include/clc/opencl/math/mad.h b/libclc/opencl/include/clc/opencl/math/mad.h
index 3daf9b9f8a0e9..18732eb7be668 100644
--- a/libclc/opencl/include/clc/opencl/math/mad.h
+++ b/libclc/opencl/include/clc/opencl/math/mad.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_MAD_H__
+#define __CLC_OPENCL_MATH_MAD_H__
+
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define __CLC_FUNCTION mad
+#define FUNCTION mad
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_MAD_H__
diff --git a/libclc/opencl/include/clc/opencl/math/maxmag.h b/libclc/opencl/include/clc/opencl/math/maxmag.h
index d4616e60dc378..c8f3d5700fa28 100644
--- a/libclc/opencl/include/clc/opencl/math/maxmag.h
+++ b/libclc/opencl/include/clc/opencl/math/maxmag.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_MAXMAG_H__
+#define __CLC_OPENCL_MATH_MAXMAG_H__
+
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION maxmag
+#define FUNCTION maxmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_MAXMAG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/minmag.h b/libclc/opencl/include/clc/opencl/math/minmag.h
index 92c0f23a22b72..c90a9d9bdbeb1 100644
--- a/libclc/opencl/include/clc/opencl/math/minmag.h
+++ b/libclc/opencl/include/clc/opencl/math/minmag.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_MINMAG_H__
+#define __CLC_OPENCL_MATH_MINMAG_H__
+
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION minmag
+#define FUNCTION minmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_MINMAG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/modf.h b/libclc/opencl/include/clc/opencl/math/modf.h
index b582cae948f26..a336c9e242c1e 100644
--- a/libclc/opencl/include/clc/opencl/math/modf.h
+++ b/libclc/opencl/include/clc/opencl/math/modf.h
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION modf
+#ifndef __CLC_OPENCL_MATH_MODF_H__
+#define __CLC_OPENCL_MATH_MODF_H__
+
+#define FUNCTION modf
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_MODF_H__
diff --git a/libclc/opencl/include/clc/opencl/math/nan.h b/libclc/opencl/include/clc/opencl/math/nan.h
index c5be0af945a57..d98ac56603495 100644
--- a/libclc/opencl/include/clc/opencl/math/nan.h
+++ b/libclc/opencl/include/clc/opencl/math/nan.h
@@ -6,5 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NAN_H__
+#define __CLC_OPENCL_MATH_NAN_H__
+
 #define __CLC_BODY <clc/opencl/math/nan.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_MATH_NAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_cos.h b/libclc/opencl/include/clc/opencl/math/native_cos.h
index e301634322b69..566c32e95b32c 100644
--- a/libclc/opencl/include/clc/opencl/math/native_cos.h
+++ b/libclc/opencl/include/clc/opencl/math/native_cos.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_COS_H__
+#define __CLC_OPENCL_MATH_NATIVE_COS_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_cos
+#define FUNCTION native_cos
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_COS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_divide.h b/libclc/opencl/include/clc/opencl/math/native_divide.h
index 5fe0c32f67d8e..23360b4bdeb9b 100644
--- a/libclc/opencl/include/clc/opencl/math/native_divide.h
+++ b/libclc/opencl/include/clc/opencl/math/native_divide.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_DIVIDE_H__
+#define __CLC_OPENCL_MATH_NATIVE_DIVIDE_H__
+
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION native_divide
+#define FUNCTION native_divide
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_DIVIDE_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp.h b/libclc/opencl/include/clc/opencl/math/native_exp.h
index bfe24aa12808a..5133469509ffb 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_EXP_H__
+#define __CLC_OPENCL_MATH_NATIVE_EXP_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_exp
+#define FUNCTION native_exp
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_EXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp10.h b/libclc/opencl/include/clc/opencl/math/native_exp10.h
index f775478eef3a9..8a2788cd3bcad 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp10.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_EXP10_H__
+#define __CLC_OPENCL_MATH_NATIVE_EXP10_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_exp10
+#define FUNCTION native_exp10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_EXP10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp2.h b/libclc/opencl/include/clc/opencl/math/native_exp2.h
index c00632c99387b..5a6aed09a838e 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp2.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_EXP2_H__
+#define __CLC_OPENCL_MATH_NATIVE_EXP2_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_exp2
+#define FUNCTION native_exp2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_EXP2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_log.h b/libclc/opencl/include/clc/opencl/math/native_log.h
index 854f420723742..58164960644ff 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_LOG_H__
+#define __CLC_OPENCL_MATH_NATIVE_LOG_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_log
+#define FUNCTION native_log
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_LOG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_log10.h b/libclc/opencl/include/clc/opencl/math/native_log10.h
index 6862124b0f69a..c215c733c0422 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log10.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log10.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_LOG10_H__
+#define __CLC_OPENCL_MATH_NATIVE_LOG10_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_log10
+#define FUNCTION native_log10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_LOG10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_log2.h b/libclc/opencl/include/clc/opencl/math/native_log2.h
index 576f50079d269..783abe6cd8e46 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log2.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log2.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_LOG2_H__
+#define __CLC_OPENCL_MATH_NATIVE_LOG2_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_log2
+#define FUNCTION native_log2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_LOG2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_powr.h b/libclc/opencl/include/clc/opencl/math/native_powr.h
index a9210092ed38d..9c8e171afc21a 100644
--- a/libclc/opencl/include/clc/opencl/math/native_powr.h
+++ b/libclc/opencl/include/clc/opencl/math/native_powr.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_POWR_H__
+#define __CLC_OPENCL_MATH_NATIVE_POWR_H__
+
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION native_powr
+#define FUNCTION native_powr
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_POWR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_recip.h b/libclc/opencl/include/clc/opencl/math/native_recip.h
index 7bc99233fe008..1c88c076e03c7 100644
--- a/libclc/opencl/include/clc/opencl/math/native_recip.h
+++ b/libclc/opencl/include/clc/opencl/math/native_recip.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_RECIP_H__
+#define __CLC_OPENCL_MATH_NATIVE_RECIP_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_recip
+#define FUNCTION native_recip
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_RECIP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_rsqrt.h b/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
index 40a1f02cba075..2806f9128d94f 100644
--- a/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_RSQRT_H__
+#define __CLC_OPENCL_MATH_NATIVE_RSQRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_rsqrt
+#define FUNCTION native_rsqrt
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_RSQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_sin.h b/libclc/opencl/include/clc/opencl/math/native_sin.h
index 798d5b325f6f1..81f664153cee1 100644
--- a/libclc/opencl/include/clc/opencl/math/native_sin.h
+++ b/libclc/opencl/include/clc/opencl/math/native_sin.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_SIN_H__
+#define __CLC_OPENCL_MATH_NATIVE_SIN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_sin
+#define FUNCTION native_sin
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_SIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_sqrt.h b/libclc/opencl/include/clc/opencl/math/native_sqrt.h
index a41a0c104cc84..dca0e542e0ac7 100644
--- a/libclc/opencl/include/clc/opencl/math/native_sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/native_sqrt.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_SQRT_H__
+#define __CLC_OPENCL_MATH_NATIVE_SQRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_sqrt
+#define FUNCTION native_sqrt
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_SQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_tan.h b/libclc/opencl/include/clc/opencl/math/native_tan.h
index 658bb8f932743..cf942110e3124 100644
--- a/libclc/opencl/include/clc/opencl/math/native_tan.h
+++ b/libclc/opencl/include/clc/opencl/math/native_tan.h
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_NATIVE_TAN_H__
+#define __CLC_OPENCL_MATH_NATIVE_TAN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_tan
+#define FUNCTION native_tan
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NATIVE_TAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/nextafter.h b/libclc/opencl/include/clc/opencl/math/nextafter.h
index ce1d549c0659b..e338f5bf6c2a7 100644
--- a/libclc/opencl/include/clc/opencl/math/nextafter.h
+++ b/libclc/opencl/include/clc/opencl/math/nextafter.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION nextafter
+#ifndef __CLC_OPENCL_MATH_NEXTAFTER_H__
+#define __CLC_OPENCL_MATH_NEXTAFTER_H__
+
+#define FUNCTION nextafter
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_NEXTAFTER_H__
diff --git a/libclc/opencl/include/clc/opencl/math/pow.h b/libclc/opencl/include/clc/opencl/math/pow.h
index 5f8479332830e..3f4c1746faa91 100644
--- a/libclc/opencl/include/clc/opencl/math/pow.h
+++ b/libclc/opencl/include/clc/opencl/math/pow.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION pow
+#ifndef __CLC_OPENCL_MATH_POW_H__
+#define __CLC_OPENCL_MATH_POW_H__
+
+#define FUNCTION pow
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_POW_H__
diff --git a/libclc/opencl/include/clc/opencl/math/pown.h b/libclc/opencl/include/clc/opencl/math/pown.h
index bbdf8f8b6e91e..fd97a705c8c87 100644
--- a/libclc/opencl/include/clc/opencl/math/pown.h
+++ b/libclc/opencl/include/clc/opencl/math/pown.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION pown
+#ifndef __CLC_OPENCL_MATH_POWN_H__
+#define __CLC_OPENCL_MATH_POWN_H__
+
+#define FUNCTION pown
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_POWN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/powr.h b/libclc/opencl/include/clc/opencl/math/powr.h
index d128d79edb39f..47f653a73afa3 100644
--- a/libclc/opencl/include/clc/opencl/math/powr.h
+++ b/libclc/opencl/include/clc/opencl/math/powr.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION powr
+#ifndef __CLC_OPENCL_MATH_POWR_H__
+#define __CLC_OPENCL_MATH_POWR_H__
+
+#define FUNCTION powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_POWR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/remainder.h b/libclc/opencl/include/clc/opencl/math/remainder.h
index d366474f4192c..416a0422c9a62 100644
--- a/libclc/opencl/include/clc/opencl/math/remainder.h
+++ b/libclc/opencl/include/clc/opencl/math/remainder.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION remainder
+#ifndef __CLC_OPENCL_MATH_REMAINDER_H__
+#define __CLC_OPENCL_MATH_REMAINDER_H__
+
+#define FUNCTION remainder
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_REMAINDER_H__
diff --git a/libclc/opencl/include/clc/opencl/math/remquo.h b/libclc/opencl/include/clc/opencl/math/remquo.h
index 007d380c7cfc8..3525c6bd0f685 100644
--- a/libclc/opencl/include/clc/opencl/math/remquo.h
+++ b/libclc/opencl/include/clc/opencl/math/remquo.h
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION remquo
+#ifndef __CLC_OPENCL_MATH_REMQUO_H__
+#define __CLC_OPENCL_MATH_REMQUO_H__
+
+#define FUNCTION remquo
 
 #define __CLC_BODY <clc/math/remquo_decl.inc>
 #include <clc/math/gentype.inc>
@@ -18,4 +21,6 @@
 #undef __CLC_ADDRESS_SPACE
 #endif
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_REMQUO_H__
diff --git a/libclc/opencl/include/clc/opencl/math/rint.h b/libclc/opencl/include/clc/opencl/math/rint.h
index aa31c177882ca..8f08fd20c3c92 100644
--- a/libclc/opencl/include/clc/opencl/math/rint.h
+++ b/libclc/opencl/include/clc/opencl/math/rint.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_RINT_H__
+#define __CLC_OPENCL_MATH_RINT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION rint
+#define FUNCTION rint
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_RINT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/rootn.h b/libclc/opencl/include/clc/opencl/math/rootn.h
index 669aeefb273a9..20b52e7c01dc5 100644
--- a/libclc/opencl/include/clc/opencl/math/rootn.h
+++ b/libclc/opencl/include/clc/opencl/math/rootn.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ROOTN_H__
+#define __CLC_OPENCL_MATH_ROOTN_H__
+
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
-#define __CLC_FUNCTION rootn
+#define FUNCTION rootn
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ROOTN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/round.h b/libclc/opencl/include/clc/opencl/math/round.h
index f4209f631d6ce..65bfc265ced0e 100644
--- a/libclc/opencl/include/clc/opencl/math/round.h
+++ b/libclc/opencl/include/clc/opencl/math/round.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_ROUND_H__
+#define __CLC_OPENCL_MATH_ROUND_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION round
+#define FUNCTION round
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_ROUND_H__
diff --git a/libclc/opencl/include/clc/opencl/math/rsqrt.h b/libclc/opencl/include/clc/opencl/math/rsqrt.h
index 5f674b299adbd..41fb54ea9bb5f 100644
--- a/libclc/opencl/include/clc/opencl/math/rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/rsqrt.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_RSQRT_H__
+#define __CLC_OPENCL_MATH_RSQRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION rsqrt
+#define FUNCTION rsqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_RSQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sin.h b/libclc/opencl/include/clc/opencl/math/sin.h
index 5ef699395689f..fc7741e164fda 100644
--- a/libclc/opencl/include/clc/opencl/math/sin.h
+++ b/libclc/opencl/include/clc/opencl/math/sin.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_SIN_H__
+#define __CLC_OPENCL_MATH_SIN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sin
+#define FUNCTION sin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_SIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sincos.h b/libclc/opencl/include/clc/opencl/math/sincos.h
index 7426c23e88af9..0231560a6b9de 100644
--- a/libclc/opencl/include/clc/opencl/math/sincos.h
+++ b/libclc/opencl/include/clc/opencl/math/sincos.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_SINCOS_H__
+#define __CLC_OPENCL_MATH_SINCOS_H__
+
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
-#define __CLC_FUNCTION __clc_sincos
+#define FUNCTION sincos
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_SINCOS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sinh.h b/libclc/opencl/include/clc/opencl/math/sinh.h
index c6d7d9a936e2f..cbe30551ec641 100644
--- a/libclc/opencl/include/clc/opencl/math/sinh.h
+++ b/libclc/opencl/include/clc/opencl/math/sinh.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_SINH_H__
+#define __CLC_OPENCL_MATH_SINH_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sinh
+#define FUNCTION sinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_SINH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sinpi.h b/libclc/opencl/include/clc/opencl/math/sinpi.h
index 0d90dbf153511..3ef55a1cd42dc 100644
--- a/libclc/opencl/include/clc/opencl/math/sinpi.h
+++ b/libclc/opencl/include/clc/opencl/math/sinpi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_SINPI_H__
+#define __CLC_OPENCL_MATH_SINPI_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sinpi
+#define FUNCTION sinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_SINPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sqrt.h b/libclc/opencl/include/clc/opencl/math/sqrt.h
index 03faa0513c06e..8da37e94e756c 100644
--- a/libclc/opencl/include/clc/opencl/math/sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/sqrt.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_SQRT_H__
+#define __CLC_OPENCL_MATH_SQRT_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sqrt
+#define FUNCTION sqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_SQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tan.h b/libclc/opencl/include/clc/opencl/math/tan.h
index bbc0b79ba0bba..b7bab825c935c 100644
--- a/libclc/opencl/include/clc/opencl/math/tan.h
+++ b/libclc/opencl/include/clc/opencl/math/tan.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_TAN_H__
+#define __CLC_OPENCL_MATH_TAN_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tan
+#define FUNCTION tan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_TAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tanh.h b/libclc/opencl/include/clc/opencl/math/tanh.h
index 9017078740a0b..168c06d50cb29 100644
--- a/libclc/opencl/include/clc/opencl/math/tanh.h
+++ b/libclc/opencl/include/clc/opencl/math/tanh.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_TANH_H__
+#define __CLC_OPENCL_MATH_TANH_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tanh
+#define FUNCTION tanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_TANH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tanpi.h b/libclc/opencl/include/clc/opencl/math/tanpi.h
index f02b5a9be81ff..0475027086f63 100644
--- a/libclc/opencl/include/clc/opencl/math/tanpi.h
+++ b/libclc/opencl/include/clc/opencl/math/tanpi.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_TANPI_H__
+#define __CLC_OPENCL_MATH_TANPI_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tanpi
+#define FUNCTION tanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_TANPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tgamma.h b/libclc/opencl/include/clc/opencl/math/tgamma.h
index c74d67a857c5d..6629c02002455 100644
--- a/libclc/opencl/include/clc/opencl/math/tgamma.h
+++ b/libclc/opencl/include/clc/opencl/math/tgamma.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_TGAMMA_H__
+#define __CLC_OPENCL_MATH_TGAMMA_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tgamma
+#define FUNCTION tgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_TGAMMA_H__
diff --git a/libclc/opencl/include/clc/opencl/math/trunc.h b/libclc/opencl/include/clc/opencl/math/trunc.h
index 5832e9e93bd8f..0e61f6e1679f7 100644
--- a/libclc/opencl/include/clc/opencl/math/trunc.h
+++ b/libclc/opencl/include/clc/opencl/math/trunc.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_MATH_TRUNC_H__
+#define __CLC_OPENCL_MATH_TRUNC_H__
+
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION trunc
+#define FUNCTION trunc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MATH_TRUNC_H__
diff --git a/libclc/opencl/include/clc/opencl/misc/shuffle.h b/libclc/opencl/include/clc/opencl/misc/shuffle.h
index f9d84c9633d37..03c2718b0c4ff 100644
--- a/libclc/opencl/include/clc/opencl/misc/shuffle.h
+++ b/libclc/opencl/include/clc/opencl/misc/shuffle.h
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION shuffle
+#ifndef __CLC_OPENCL_MISC_SHUFFLE_H__
+#define __CLC_OPENCL_MISC_SHUFFLE_H__
+
+#define FUNCTION shuffle
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
@@ -16,4 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MISC_SHUFFLE_H__
diff --git a/libclc/opencl/include/clc/opencl/misc/shuffle2.h b/libclc/opencl/include/clc/opencl/misc/shuffle2.h
index 91548ad784fbb..5b6ba99686516 100644
--- a/libclc/opencl/include/clc/opencl/misc/shuffle2.h
+++ b/libclc/opencl/include/clc/opencl/misc/shuffle2.h
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION shuffle2
+#ifndef __CLC_OPENCL_MISC_SHUFFLE2_H__
+#define __CLC_OPENCL_MISC_SHUFFLE2_H__
+
+#define FUNCTION shuffle2
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
@@ -16,4 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_MISC_SHUFFLE2_H__
diff --git a/libclc/opencl/include/clc/opencl/opencl-base.h b/libclc/opencl/include/clc/opencl/opencl-base.h
new file mode 100644
index 0000000000000..ac0dd0f129707
--- /dev/null
+++ b/libclc/opencl/include/clc/opencl/opencl-base.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_OPENCL_BASE_H__
+#define __CLC_OPENCL_OPENCL_BASE_H__
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#endif
+
+/* Function Attributes */
+#include <clc/clcfunc.h>
+
+/* 6.1 Supported Data Types */
+#include <clc/clctypes.h>
+
+#endif // __CLC_OPENCL_OPENCL_BASE_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/all.h b/libclc/opencl/include/clc/opencl/relational/all.h
index 31c8e895f6661..6c19d83e25bbf 100644
--- a/libclc/opencl/include/clc/opencl/relational/all.h
+++ b/libclc/opencl/include/clc/opencl/relational/all.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_ALL_H__
+#define __CLC_OPENCL_RELATIONAL_ALL_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_ALL_DECL(TYPE) _CLC_OVERLOAD _CLC_DECL int all(TYPE v);
 
 #define _CLC_VECTOR_ALL_DECL(TYPE)                                             \
@@ -23,3 +28,5 @@ _CLC_VECTOR_ALL_DECL(long)
 
 #undef _CLC_ALL_DECL
 #undef _CLC_VECTOR_ALL_DECL
+
+#endif // __CLC_OPENCL_RELATIONAL_ALL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/any.h b/libclc/opencl/include/clc/opencl/relational/any.h
index 9c227e8f1600e..4f6b2e22460c1 100644
--- a/libclc/opencl/include/clc/opencl/relational/any.h
+++ b/libclc/opencl/include/clc/opencl/relational/any.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_ANY_H__
+#define __CLC_OPENCL_RELATIONAL_ANY_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_ANY_DECL(TYPE) _CLC_OVERLOAD _CLC_DECL int any(TYPE v);
 
 #define _CLC_VECTOR_ANY_DECL(TYPE)                                             \
@@ -23,3 +28,5 @@ _CLC_VECTOR_ANY_DECL(long)
 
 #undef _CLC_ANY_DECL
 #undef _CLC_VECTOR_ANY_DECL
+
+#endif // __CLC_OPENCL_RELATIONAL_ANY_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/bitselect.h b/libclc/opencl/include/clc/opencl/relational/bitselect.h
index 7a39d5468e5ff..9e967c6e2d642 100644
--- a/libclc/opencl/include/clc/opencl/relational/bitselect.h
+++ b/libclc/opencl/include/clc/opencl/relational/bitselect.h
@@ -6,7 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_BITSELECT_H__
+#define __CLC_OPENCL_RELATIONAL_BITSELECT_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_BODY <clc/opencl/relational/bitselect.inc>
 #include <clc/math/gentype.inc>
 #define __CLC_BODY <clc/opencl/relational/bitselect.inc>
 #include <clc/integer/gentype.inc>
+
+#endif // __CLC_OPENCL_RELATIONAL_BITSELECT_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isequal.h b/libclc/opencl/include/clc/opencl/relational/isequal.h
index 090b9928ede25..c679b2cfc3feb 100644
--- a/libclc/opencl/include/clc/opencl/relational/isequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isequal.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_ISEQUAL_H__
+#define __CLC_OPENCL_RELATIONAL_ISEQUAL_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_ISEQUAL_DECL(TYPE, RETTYPE)                                       \
   _CLC_OVERLOAD _CLC_DECL RETTYPE isequal(TYPE x, TYPE y);
 
@@ -30,3 +35,5 @@ _CLC_VECTOR_ISEQUAL_DECL(half, short)
 
 #undef _CLC_ISEQUAL_DECL
 #undef _CLC_VECTOR_ISEQUAL_DEC
+
+#endif // __CLC_OPENCL_RELATIONAL_ISEQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isfinite.h b/libclc/opencl/include/clc/opencl/relational/isfinite.h
index e92a4d40a8cbe..2548e6acf5109 100644
--- a/libclc/opencl/include/clc/opencl/relational/isfinite.h
+++ b/libclc/opencl/include/clc/opencl/relational/isfinite.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isfinite
+#ifndef __CLC_OPENCL_RELATIONAL_ISFINITE_H__
+#define __CLC_OPENCL_RELATIONAL_ISFINITE_H__
 
-#define __CLC_FUNCTION isfinite
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISFINITE_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isgreater.h b/libclc/opencl/include/clc/opencl/relational/isgreater.h
index bec1e81f48baa..6dfe6eb810e2a 100644
--- a/libclc/opencl/include/clc/opencl/relational/isgreater.h
+++ b/libclc/opencl/include/clc/opencl/relational/isgreater.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isgreater
+#ifndef __CLC_OPENCL_RELATIONAL_ISGREATER_H__
+#define __CLC_OPENCL_RELATIONAL_ISGREATER_H__
 
-#define __CLC_FUNCTION isgreater
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISGREATER_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h b/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
index da02167fc010a..1db2c5d58d062 100644
--- a/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isgreaterequal
+#ifndef __CLC_OPENCL_RELATIONAL_ISGREATEREQUAL_H__
+#define __CLC_OPENCL_RELATIONAL_ISGREATEREQUAL_H__
 
-#define __CLC_FUNCTION isgreaterequal
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISGREATEREQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isinf.h b/libclc/opencl/include/clc/opencl/relational/isinf.h
index da38b01e364b5..561cb1cda47b7 100644
--- a/libclc/opencl/include/clc/opencl/relational/isinf.h
+++ b/libclc/opencl/include/clc/opencl/relational/isinf.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_ISINF_H__
+#define __CLC_OPENCL_RELATIONAL_ISINF_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_ISINF_DECL(RET_TYPE, ARG_TYPE)                                    \
   _CLC_OVERLOAD _CLC_DECL RET_TYPE isinf(ARG_TYPE);
 
@@ -31,3 +36,5 @@ _CLC_VECTOR_ISINF_DECL(short, half)
 
 #undef _CLC_ISINF_DECL
 #undef _CLC_VECTOR_ISINF_DECL
+
+#endif // __CLC_OPENCL_RELATIONAL_ISINF_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isless.h b/libclc/opencl/include/clc/opencl/relational/isless.h
index c930b6df6d083..3e2afb32cddf4 100644
--- a/libclc/opencl/include/clc/opencl/relational/isless.h
+++ b/libclc/opencl/include/clc/opencl/relational/isless.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION isless
+#ifndef __CLC_OPENCL_RELATIONAL_ISLESS_H__
+#define __CLC_OPENCL_RELATIONAL_ISLESS_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISLESS_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/islessequal.h b/libclc/opencl/include/clc/opencl/relational/islessequal.h
index 335c29612446e..978e6a9052c16 100644
--- a/libclc/opencl/include/clc/opencl/relational/islessequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/islessequal.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION islessequal
+#ifndef __CLC_OPENCL_RELATIONAL_ISLESSEQUAL_H__
+#define __CLC_OPENCL_RELATIONAL_ISLESSEQUAL_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISLESSEQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/islessgreater.h b/libclc/opencl/include/clc/opencl/relational/islessgreater.h
index f9a5fa8b01221..56cce7db20770 100644
--- a/libclc/opencl/include/clc/opencl/relational/islessgreater.h
+++ b/libclc/opencl/include/clc/opencl/relational/islessgreater.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION islessgreater
+#ifndef __CLC_OPENCL_RELATIONAL_ISLESSGREATER_H__
+#define __CLC_OPENCL_RELATIONAL_ISLESSGREATER_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISLESSGREATER_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isnan.h b/libclc/opencl/include/clc/opencl/relational/isnan.h
index e712d6a5d93a9..fe1e5721c1d24 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnan.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnan.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_ISNAN_H__
+#define __CLC_OPENCL_RELATIONAL_ISNAN_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_ISNAN_DECL(RET_TYPE, ARG_TYPE)                                    \
   _CLC_OVERLOAD _CLC_DECL RET_TYPE isnan(ARG_TYPE);
 
@@ -31,3 +36,5 @@ _CLC_VECTOR_ISNAN_DECL(short, half)
 
 #undef _CLC_ISNAN_DECL
 #undef _CLC_VECTOR_ISNAN_DECL
+
+#endif // __CLC_OPENCL_RELATIONAL_ISNAN_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isnormal.h b/libclc/opencl/include/clc/opencl/relational/isnormal.h
index 33b7a7e894511..ee74a990b5eaf 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnormal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnormal.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isnormal
+#ifndef __CLC_OPENCL_RELATIONAL_ISNORMAL_H__
+#define __CLC_OPENCL_RELATIONAL_ISNORMAL_H__
 
-#define __CLC_FUNCTION isnormal
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISNORMAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isnotequal.h b/libclc/opencl/include/clc/opencl/relational/isnotequal.h
index dfb9de1da3a18..7cf94e3ceec5f 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnotequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnotequal.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isnotequal
+#ifndef __CLC_OPENCL_RELATIONAL_ISNOTEQUAL_H__
+#define __CLC_OPENCL_RELATIONAL_ISNOTEQUAL_H__
 
-#define __CLC_FUNCTION isnotequal
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISNOTEQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isordered.h b/libclc/opencl/include/clc/opencl/relational/isordered.h
index bc6dbdeba92a0..ad9770bd627f2 100644
--- a/libclc/opencl/include/clc/opencl/relational/isordered.h
+++ b/libclc/opencl/include/clc/opencl/relational/isordered.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isordered
+#ifndef __CLC_OPENCL_RELATIONAL_ISORDERED_H__
+#define __CLC_OPENCL_RELATIONAL_ISORDERED_H__
 
-#define __CLC_FUNCTION isordered
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISORDERED_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isunordered.h b/libclc/opencl/include/clc/opencl/relational/isunordered.h
index 73f05ce9edea8..01d2f53837317 100644
--- a/libclc/opencl/include/clc/opencl/relational/isunordered.h
+++ b/libclc/opencl/include/clc/opencl/relational/isunordered.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef isunordered
+#ifndef __CLC_OPENCL_RELATIONAL_ISUNORDERED_H__
+#define __CLC_OPENCL_RELATIONAL_ISUNORDERED_H__
 
-#define __CLC_FUNCTION isunordered
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_ISUNORDERED_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/select.h b/libclc/opencl/include/clc/opencl/relational/select.h
index 52f42eff3e4d4..e4cefd8bcde2e 100644
--- a/libclc/opencl/include/clc/opencl/relational/select.h
+++ b/libclc/opencl/include/clc/opencl/relational/select.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_RELATIONAL_SELECT_H__
+#define __CLC_OPENCL_RELATIONAL_SELECT_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_SELECT_FN select
 
 #define __CLC_BODY <clc/relational/clc_select_decl.inc>
@@ -14,3 +19,5 @@
 #include <clc/integer/gentype.inc>
 
 #undef __CLC_SELECT_FN
+
+#endif // __CLC_OPENCL_RELATIONAL_SELECT_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/signbit.h b/libclc/opencl/include/clc/opencl/relational/signbit.h
index 89785f5fc1030..29591c0c126a9 100644
--- a/libclc/opencl/include/clc/opencl/relational/signbit.h
+++ b/libclc/opencl/include/clc/opencl/relational/signbit.h
@@ -6,11 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#undef signbit
+#ifndef __CLC_OPENCL_RELATIONAL_SIGNBIT_H__
+#define __CLC_OPENCL_RELATIONAL_SIGNBIT_H__
 
-#define __CLC_FUNCTION signbit
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_RELATIONAL_SIGNBIT_H__
diff --git a/libclc/opencl/include/clc/opencl/shared/clamp.h b/libclc/opencl/include/clc/opencl/shared/clamp.h
index 4fe7897f56016..4bd365e9dd006 100644
--- a/libclc/opencl/include/clc/opencl/shared/clamp.h
+++ b/libclc/opencl/include/clc/opencl/shared/clamp.h
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SHARED_CLAMP_H__
+#define __CLC_OPENCL_SHARED_CLAMP_H__
+
 #define __CLC_BODY <clc/opencl/shared/clamp.inc>
 #include <clc/integer/gentype.inc>
 
 #define __CLC_BODY <clc/opencl/shared/clamp.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_SHARED_CLAMP_H__
diff --git a/libclc/opencl/include/clc/opencl/shared/max.h b/libclc/opencl/include/clc/opencl/shared/max.h
index 5ff01ba8082a2..512e696885817 100644
--- a/libclc/opencl/include/clc/opencl/shared/max.h
+++ b/libclc/opencl/include/clc/opencl/shared/max.h
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SHARED_MAX_H__
+#define __CLC_OPENCL_SHARED_MAX_H__
+
 #define __CLC_BODY <clc/opencl/shared/max.inc>
 #include <clc/integer/gentype.inc>
 
 #define __CLC_BODY <clc/opencl/shared/max.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_SHARED_MAX_H__
diff --git a/libclc/opencl/include/clc/opencl/shared/min.h b/libclc/opencl/include/clc/opencl/shared/min.h
index 573f837685be1..e20545210dcd2 100644
--- a/libclc/opencl/include/clc/opencl/shared/min.h
+++ b/libclc/opencl/include/clc/opencl/shared/min.h
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SHARED_MIN_H__
+#define __CLC_OPENCL_SHARED_MIN_H__
+
 #define __CLC_BODY <clc/opencl/shared/min.inc>
 #include <clc/integer/gentype.inc>
 
 #define __CLC_BODY <clc/opencl/shared/min.inc>
 #include <clc/math/gentype.inc>
+
+#endif // __CLC_OPENCL_SHARED_MIN_H__
diff --git a/libclc/opencl/include/clc/opencl/shared/vload.h b/libclc/opencl/include/clc/opencl/shared/vload.h
index 7b4092c0a1c32..2ec087d4f3c18 100644
--- a/libclc/opencl/include/clc/opencl/shared/vload.h
+++ b/libclc/opencl/include/clc/opencl/shared/vload.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SHARED_VLOAD_H__
+#define __CLC_OPENCL_SHARED_VLOAD_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_VLOAD_DECL(SUFFIX, MEM_TYPE, VEC_TYPE, WIDTH, ADDR_SPACE)         \
   _CLC_OVERLOAD _CLC_DECL VEC_TYPE vload##SUFFIX##WIDTH(                       \
       size_t offset, const ADDR_SPACE MEM_TYPE *x);
@@ -79,3 +84,5 @@ _CLC_VLOAD_DECL(a_half, half, float, , __generic)
 #undef _CLC_VECTOR_VLOAD_PRIM3
 #undef _CLC_VECTOR_VLOAD_PRIM1
 #undef _CLC_VECTOR_VLOAD_GENERIC_DECL
+
+#endif // __CLC_OPENCL_SHARED_VLOAD_H__
diff --git a/libclc/opencl/include/clc/opencl/shared/vstore.h b/libclc/opencl/include/clc/opencl/shared/vstore.h
index c1b78b734138f..00eaff41f77fb 100644
--- a/libclc/opencl/include/clc/opencl/shared/vstore.h
+++ b/libclc/opencl/include/clc/opencl/shared/vstore.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SHARED_VSTORE_H__
+#define __CLC_OPENCL_SHARED_VSTORE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define _CLC_VSTORE_DECL(SUFFIX, PRIM_TYPE, VEC_TYPE, WIDTH, ADDR_SPACE, RND)  \
   _CLC_OVERLOAD _CLC_DECL void vstore##SUFFIX##WIDTH##RND(                     \
       VEC_TYPE vec, size_t offset, ADDR_SPACE PRIM_TYPE *out);
@@ -82,3 +87,5 @@ _CLC_VECTOR_VSTORE_PRIM1(half)
 #undef _CLC_VECTOR_VSTORE_PRIM3
 #undef _CLC_VECTOR_VSTORE_PRIM1
 #undef _CLC_VECTOR_VSTORE_GENERIC_DECL
+
+#endif // __CLC_OPENCL_SHARED_VSTORE_H__
diff --git a/libclc/opencl/include/clc/opencl/synchronization/barrier.h b/libclc/opencl/include/clc/opencl/synchronization/barrier.h
index 5efe25c3194f7..728ba4ae8a532 100644
--- a/libclc/opencl/include/clc/opencl/synchronization/barrier.h
+++ b/libclc/opencl/include/clc/opencl/synchronization/barrier.h
@@ -6,4 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SYNCHRONIZATION_BARRIER_H__
+#define __CLC_OPENCL_SYNCHRONIZATION_BARRIER_H__
+
+#include <clc/opencl/opencl-base.h>
+#include <clc/opencl/synchronization/cl_mem_fence_flags.h>
+
 _CLC_DECL _CLC_OVERLOAD void barrier(cl_mem_fence_flags flags);
+
+#endif // __CLC_OPENCL_SYNCHRONIZATION_BARRIER_H__
diff --git a/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h b/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h
index f98b1e2a2bc51..6636515fca47d 100644
--- a/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h
+++ b/libclc/opencl/include/clc/opencl/synchronization/cl_mem_fence_flags.h
@@ -6,7 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_SYNCHRONIZATION_CL_MEM_FENCE_FLAGS_H__
+#define __CLC_OPENCL_SYNCHRONIZATION_CL_MEM_FENCE_FLAGS_H__
+
 typedef uint cl_mem_fence_flags;
 
 #define CLK_LOCAL_MEM_FENCE 1
 #define CLK_GLOBAL_MEM_FENCE 2
+
+#endif // __CLC_OPENCL_SYNCHRONIZATION_CL_MEM_FENCE_FLAGS_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_global_id.h b/libclc/opencl/include/clc/opencl/workitem/get_global_id.h
index c60e9bc8b9b78..98217c0ba6b4d 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_global_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_global_id.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_GLOBAL_ID_H__
+#define __CLC_OPENCL_WORKITEM_GET_GLOBAL_ID_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_global_id(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_GLOBAL_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h b/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h
index 7f06476048c52..8c820331e6ab2 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_GLOBAL_OFFSET_H__
+#define __CLC_OPENCL_WORKITEM_GET_GLOBAL_OFFSET_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_global_offset(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_GLOBAL_OFFSET_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_global_size.h b/libclc/opencl/include/clc/opencl/workitem/get_global_size.h
index e235d990c79fb..5c7426047d7d0 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_global_size.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_global_size.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_GLOBAL_SIZE_H__
+#define __CLC_OPENCL_WORKITEM_GET_GLOBAL_SIZE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_global_size(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_GLOBAL_SIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_group_id.h b/libclc/opencl/include/clc/opencl/workitem/get_group_id.h
index 78b78e8e56922..3d0f51aeb8dc2 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_group_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_group_id.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_GROUP_ID_H__
+#define __CLC_OPENCL_WORKITEM_GET_GROUP_ID_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_group_id(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_GROUP_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_local_id.h b/libclc/opencl/include/clc/opencl/workitem/get_local_id.h
index 82b569380d471..31fe9d2943f48 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_local_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_local_id.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_LOCAL_ID_H__
+#define __CLC_OPENCL_WORKITEM_GET_LOCAL_ID_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_local_id(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_LOCAL_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_local_size.h b/libclc/opencl/include/clc/opencl/workitem/get_local_size.h
index 9458ba3923f7b..c93a7a27bd7cf 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_local_size.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_local_size.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_LOCAL_SIZE_H__
+#define __CLC_OPENCL_WORKITEM_GET_LOCAL_SIZE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_local_size(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_LOCAL_SIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h b/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h
index 3f0d3cb2c4fbd..0c47a5ef462b3 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_NUM_GROUPS_H__
+#define __CLC_OPENCL_WORKITEM_GET_NUM_GROUPS_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD size_t get_num_groups(uint dim);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_NUM_GROUPS_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h b/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h
index dc6ae4e9f93bf..c4c3c1b224a9a 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h
@@ -6,4 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_WORKITEM_GET_WORK_DIM_H__
+#define __CLC_OPENCL_WORKITEM_GET_WORK_DIM_H__
+
+#include <clc/opencl/opencl-base.h>
+
 _CLC_DECL _CLC_OVERLOAD uint get_work_dim(void);
+
+#endif // __CLC_OPENCL_WORKITEM_GET_WORK_DIM_H__
diff --git a/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_global_size.cl b/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_global_size.cl
index 2d7fea91233aa..8f9b47978c6af 100644
--- a/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_global_size.cl
+++ b/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_global_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_size.h>
 
 #if __clang_major__ >= 8
 #define CONST_AS __constant
diff --git a/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_local_size.cl b/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_local_size.cl
index 84552e2e08fd7..fd30e6ab7a47c 100644
--- a/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_local_size.cl
+++ b/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_local_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 #if __clang_major__ >= 8
 #define CONST_AS __constant
diff --git a/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_num_groups.cl b/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_num_groups.cl
index 1ed16736376f6..a88dae97c7958 100644
--- a/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_num_groups.cl
+++ b/libclc/opencl/lib/amdgcn-amdhsa/workitem/get_num_groups.cl
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_size.h>
+#include <clc/opencl/workitem/get_local_size.h>
+#include <clc/opencl/workitem/get_num_groups.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_num_groups(uint dim) {
   size_t global_size = get_global_size(dim);
diff --git a/libclc/opencl/lib/amdgcn/mem_fence/fence.cl b/libclc/opencl/lib/amdgcn/mem_fence/fence.cl
index 5c5029a63cfc7..88b953005aae6 100644
--- a/libclc/opencl/lib/amdgcn/mem_fence/fence.cl
+++ b/libclc/opencl/lib/amdgcn/mem_fence/fence.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/explicit_fence/explicit_memory_fence.h>
 
 void __clc_amdgcn_s_waitcnt(unsigned flags);
 
diff --git a/libclc/opencl/lib/amdgcn/synchronization/barrier.cl b/libclc/opencl/lib/amdgcn/synchronization/barrier.cl
index f7c41aafa83c9..5203db72f484c 100644
--- a/libclc/opencl/lib/amdgcn/synchronization/barrier.cl
+++ b/libclc/opencl/lib/amdgcn/synchronization/barrier.cl
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/explicit_fence/explicit_memory_fence.h>
+#include <clc/opencl/synchronization/barrier.h>
 
 _CLC_DEF _CLC_OVERLOAD void barrier(cl_mem_fence_flags flags) {
   mem_fence(flags);
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_global_offset.cl b/libclc/opencl/lib/amdgcn/workitem/get_global_offset.cl
index a1b3ce4192793..55c09a0a2d4e9 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_global_offset.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_global_offset.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_offset.h>
 
 #if __clang_major__ >= 8
 #define CONST_AS __constant
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_global_size.cl b/libclc/opencl/lib/amdgcn/workitem/get_global_size.cl
index 8f1507765f934..f49d26eb684ac 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_global_size.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_global_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_size.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_global_size(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_group_id.cl b/libclc/opencl/lib/amdgcn/workitem/get_group_id.cl
index 446cc63ab759d..d0fae52f6f535 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_group_id.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_group_id.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_group_id.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_group_id(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_local_id.cl b/libclc/opencl/lib/amdgcn/workitem/get_local_id.cl
index cd07d8645cd53..5b8b8f40d095d 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_local_id.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_local_id.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_id.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_local_id(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_local_size.cl b/libclc/opencl/lib/amdgcn/workitem/get_local_size.cl
index cec2d358df8ac..aeaa383ce22f4 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_local_size.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_local_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 uint __clc_amdgcn_get_local_size_x(void) __asm("llvm.r600.read.local.size.x");
 uint __clc_amdgcn_get_local_size_y(void) __asm("llvm.r600.read.local.size.y");
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_num_groups.cl b/libclc/opencl/lib/amdgcn/workitem/get_num_groups.cl
index 0d8f0b8736585..3d602fb821dcd 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_num_groups.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_num_groups.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_num_groups.h>
 
 uint __clc_amdgcn_get_num_groups_x(void) __asm("llvm.r600.read.ngroups.x");
 uint __clc_amdgcn_get_num_groups_y(void) __asm("llvm.r600.read.ngroups.y");
diff --git a/libclc/opencl/lib/amdgcn/workitem/get_work_dim.cl b/libclc/opencl/lib/amdgcn/workitem/get_work_dim.cl
index 8ca8b0b61ce54..f7af4ee57432f 100644
--- a/libclc/opencl/lib/amdgcn/workitem/get_work_dim.cl
+++ b/libclc/opencl/lib/amdgcn/workitem/get_work_dim.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_work_dim.h>
 
 #if __clang_major__ >= 8
 #define CONST_AS __constant
diff --git a/libclc/opencl/lib/clspv/math/fma.cl b/libclc/opencl/lib/clspv/math/fma.cl
index 0f3141a0e09ee..83504f63772ca 100644
--- a/libclc/opencl/lib/clspv/math/fma.cl
+++ b/libclc/opencl/lib/clspv/math/fma.cl
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/internal/math/clc_sw_fma.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fma.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION fma
-#define __CLC_FUNCTION(x) __clc_sw_fma
+#define __IMPL_FUNCTION(x) __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/clspv/shared/vstore_half.cl b/libclc/opencl/lib/clspv/shared/vstore_half.cl
index 1694ebeebe689..df5f30a711df5 100644
--- a/libclc/opencl/lib/clspv/shared/vstore_half.cl
+++ b/libclc/opencl/lib/clspv/shared/vstore_half.cl
@@ -6,7 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/float/definitions.h>
+#include <clc/opencl/as_type.h>
+#include <clc/opencl/math/copysign.h>
+#include <clc/opencl/math/fabs.h>
+#include <clc/opencl/math/nextafter.h>
+#include <clc/opencl/relational/isinf.h>
+#include <clc/opencl/relational/isnan.h>
+#include <clc/opencl/shared/min.h>
+#include <clc/opencl/shared/vstore.h>
 
 #pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
 
diff --git a/libclc/opencl/lib/clspv/subnormal_config.cl b/libclc/opencl/lib/clspv/subnormal_config.cl
index 77d60ab9cbbde..114aabb2e9435 100644
--- a/libclc/opencl/lib/clspv/subnormal_config.cl
+++ b/libclc/opencl/lib/clspv/subnormal_config.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_subnormal_config.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/opencl-base.h>
 
 _CLC_DEF bool __clc_fp16_subnormals_supported() { return false; }
 
diff --git a/libclc/opencl/lib/generic/async/async_work_group_copy.cl b/libclc/opencl/lib/generic/async/async_work_group_copy.cl
index a0b9d7d1f52ec..67eadf5e2569a 100644
--- a/libclc/opencl/lib/generic/async/async_work_group_copy.cl
+++ b/libclc/opencl/lib/generic/async/async_work_group_copy.cl
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/async/async_work_group_copy.h>
+#include <clc/opencl/async/async_work_group_strided_copy.h>
 
 #define __CLC_BODY <async_work_group_copy.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/async/async_work_group_strided_copy.cl b/libclc/opencl/lib/generic/async/async_work_group_strided_copy.cl
index ac4d1da3bc215..2c17e2a685e8a 100644
--- a/libclc/opencl/lib/generic/async/async_work_group_strided_copy.cl
+++ b/libclc/opencl/lib/generic/async/async_work_group_strided_copy.cl
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/async/async_work_group_strided_copy.h>
+#include <clc/opencl/workitem/get_local_id.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 #define __CLC_BODY <async_work_group_strided_copy.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/async/prefetch.cl b/libclc/opencl/lib/generic/async/prefetch.cl
index 50abbc5083428..8ab2101205868 100644
--- a/libclc/opencl/lib/generic/async/prefetch.cl
+++ b/libclc/opencl/lib/generic/async/prefetch.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/async/clc_prefetch.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/async/prefetch.h>
 
 #define __CLC_BODY <prefetch.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/async/wait_group_events.cl b/libclc/opencl/lib/generic/async/wait_group_events.cl
index e353e48cfe09f..00b8ddf3342fb 100644
--- a/libclc/opencl/lib/generic/async/wait_group_events.cl
+++ b/libclc/opencl/lib/generic/async/wait_group_events.cl
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/async/wait_group_events.h>
+#include <clc/opencl/synchronization/barrier.h>
 
 _CLC_DEF _CLC_OVERLOAD void wait_group_events(int num_events,
                                               event_t *event_list) {
diff --git a/libclc/opencl/lib/generic/atomic/atom_add.cl b/libclc/opencl/lib/generic/atomic/atom_add.cl
index 9f60881c2717d..65d7ad595ace6 100644
--- a/libclc/opencl/lib/generic/atomic/atom_add.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_add.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_add.h>
+#include <clc/opencl/atomic/atomic_add.h>
 
 #ifdef cl_khr_global_int32_base_atomics
 #define __CLC_ATOMIC_OP add
diff --git a/libclc/opencl/lib/generic/atomic/atom_and.cl b/libclc/opencl/lib/generic/atomic/atom_and.cl
index 2653296d29e4f..371e5f0aa4d0d 100644
--- a/libclc/opencl/lib/generic/atomic/atom_and.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_and.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_and.h>
+#include <clc/opencl/atomic/atomic_and.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
 #define __CLC_ATOMIC_OP and
diff --git a/libclc/opencl/lib/generic/atomic/atom_int32_binary.inc b/libclc/opencl/lib/generic/atomic/atom_int32_binary.inc
index bee20d9146420..0afc2190263fa 100644
--- a/libclc/opencl/lib/generic/atomic/atom_int32_binary.inc
+++ b/libclc/opencl/lib/generic/atomic/atom_int32_binary.inc
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/opencl-base.h>
 #include <clc/utils.h>
 
 #define __CLC_ATOM_IMPL(AS, TYPE)                                              \
diff --git a/libclc/opencl/lib/generic/atomic/atom_max.cl b/libclc/opencl/lib/generic/atomic/atom_max.cl
index 744f39f6cb494..2542191d04f53 100644
--- a/libclc/opencl/lib/generic/atomic/atom_max.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_max.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_max.h>
+#include <clc/opencl/atomic/atomic_max.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
 #define __CLC_ATOMIC_OP max
diff --git a/libclc/opencl/lib/generic/atomic/atom_min.cl b/libclc/opencl/lib/generic/atomic/atom_min.cl
index 75e5fb02e1289..4e62804824d82 100644
--- a/libclc/opencl/lib/generic/atomic/atom_min.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_min.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_min.h>
+#include <clc/opencl/atomic/atomic_min.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
 #define __CLC_ATOMIC_OP min
diff --git a/libclc/opencl/lib/generic/atomic/atom_or.cl b/libclc/opencl/lib/generic/atomic/atom_or.cl
index 4b1ecac95d878..30ad8e52ca232 100644
--- a/libclc/opencl/lib/generic/atomic/atom_or.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_or.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_or.h>
+#include <clc/opencl/atomic/atomic_or.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
 #define __CLC_ATOMIC_OP or
diff --git a/libclc/opencl/lib/generic/atomic/atom_sub.cl b/libclc/opencl/lib/generic/atomic/atom_sub.cl
index 24c7e85ca73a2..f6ee94db7469f 100644
--- a/libclc/opencl/lib/generic/atomic/atom_sub.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_sub.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_sub.h>
+#include <clc/opencl/atomic/atomic_sub.h>
 
 #ifdef cl_khr_global_int32_base_atomics
 #define __CLC_ATOMIC_OP sub
diff --git a/libclc/opencl/lib/generic/atomic/atom_xchg.cl b/libclc/opencl/lib/generic/atomic/atom_xchg.cl
index f14b18070deb0..3fa17c950a82e 100644
--- a/libclc/opencl/lib/generic/atomic/atom_xchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_xchg.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_xchg.h>
+#include <clc/opencl/atomic/atomic_xchg.h>
 
 #ifdef cl_khr_global_int32_base_atomics
 #define __CLC_ATOMIC_OP xchg
diff --git a/libclc/opencl/lib/generic/atomic/atom_xor.cl b/libclc/opencl/lib/generic/atomic/atom_xor.cl
index 9373fae2487de..ac08ef63b1a4b 100644
--- a/libclc/opencl/lib/generic/atomic/atom_xor.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_xor.cl
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/opencl/atomic/atom_xor.h>
+#include <clc/opencl/atomic/atomic_xor.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
 #define __CLC_ATOMIC_OP xor
diff --git a/libclc/opencl/lib/generic/atomic/atomic_add.cl b/libclc/opencl/lib/generic/atomic/atomic_add.cl
index cb3935cc4f8a1..a0effced7dc62 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_add.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_add.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_add.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_add(volatile AS TYPE *p, TYPE val) {      \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_and.cl b/libclc/opencl/lib/generic/atomic/atomic_and.cl
index f5b6e97246c6b..629e6638d3bc2 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_and.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_and.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_and.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_and(volatile AS TYPE *p, TYPE val) {      \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl b/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl
index c2227ea5207df..db0495b004c1a 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_cmpxchg.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_cmpxchg(volatile AS TYPE *p, TYPE cmp,    \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_dec.cl b/libclc/opencl/lib/generic/atomic/atomic_dec.cl
index d67f6382fd07d..6f18cdf13428a 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_dec.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_dec.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_dec.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_dec(volatile AS TYPE *p) {                \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_inc.cl b/libclc/opencl/lib/generic/atomic/atomic_inc.cl
index 989c45c0e5d8a..13349e5432e5c 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_inc.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_inc.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_inc.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_inc(volatile AS TYPE *p) {                \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_max.cl b/libclc/opencl/lib/generic/atomic/atomic_max.cl
index dbd5aa049c5e7..ae929f0523dc2 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_max.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_max.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_max.h>
 
 #define IMPL(TYPE, AS, OP)                                                     \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_max(volatile AS TYPE *p, TYPE val) {      \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_min.cl b/libclc/opencl/lib/generic/atomic/atomic_min.cl
index d43f74c82ffc5..c7ebe71ee8295 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_min.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_min.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_min.h>
 
 #define IMPL(TYPE, AS, OP)                                                     \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_min(volatile AS TYPE *p, TYPE val) {      \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_or.cl b/libclc/opencl/lib/generic/atomic/atomic_or.cl
index dac7eb66033b4..45fb865689655 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_or.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_or.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_or.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_or(volatile AS TYPE *p, TYPE val) {       \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_sub.cl
index 5ebe1f0b13694..74977f15155fb 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_sub.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_sub.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_sub.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_sub(volatile AS TYPE *p, TYPE val) {      \
diff --git a/libclc/opencl/lib/generic/atomic/atomic_xchg.cl b/libclc/opencl/lib/generic/atomic/atomic_xchg.cl
index b3f631c21aec3..883132caf9fbb 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_xchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_xchg.cl
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/as_type.h>
+#include <clc/opencl/atomic/atomic_xchg.h>
 
 _CLC_OVERLOAD _CLC_DEF float atomic_xchg(volatile global float *p, float val) {
   return as_float(atomic_xchg((volatile global uint *)p, as_uint(val)));
diff --git a/libclc/opencl/lib/generic/atomic/atomic_xor.cl b/libclc/opencl/lib/generic/atomic/atomic_xor.cl
index 864e22b774538..246752c258374 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_xor.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_xor.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/atomic/atomic_xor.h>
 
 #define IMPL(TYPE, AS)                                                         \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_xor(volatile AS TYPE *p, TYPE val) {      \
diff --git a/libclc/opencl/lib/generic/common/degrees.cl b/libclc/opencl/lib/generic/common/degrees.cl
index a86003c170bff..d989fa6382edb 100644
--- a/libclc/opencl/lib/generic/common/degrees.cl
+++ b/libclc/opencl/lib/generic/common/degrees.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/common/clc_degrees.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/common/degrees.h>
 
 #define FUNCTION degrees
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/common/mix.cl b/libclc/opencl/lib/generic/common/mix.cl
index cc95dcd3b615c..014cc1a742f1c 100644
--- a/libclc/opencl/lib/generic/common/mix.cl
+++ b/libclc/opencl/lib/generic/common/mix.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_mad.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/common/mix.h>
 
 #define __CLC_BODY <mix.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/common/radians.cl b/libclc/opencl/lib/generic/common/radians.cl
index b45653be2e782..1da38920d43fb 100644
--- a/libclc/opencl/lib/generic/common/radians.cl
+++ b/libclc/opencl/lib/generic/common/radians.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/common/clc_radians.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/common/radians.h>
 
 #define FUNCTION radians
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/common/sign.cl b/libclc/opencl/lib/generic/common/sign.cl
index 0acab050af598..9afcb7375e96a 100644
--- a/libclc/opencl/lib/generic/common/sign.cl
+++ b/libclc/opencl/lib/generic/common/sign.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/common/clc_sign.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/common/sign.h>
 
 #define FUNCTION sign
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/common/smoothstep.cl b/libclc/opencl/lib/generic/common/smoothstep.cl
index 30ed1451ff302..fdf9a89dcbfdc 100644
--- a/libclc/opencl/lib/generic/common/smoothstep.cl
+++ b/libclc/opencl/lib/generic/common/smoothstep.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/common/clc_smoothstep.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/common/smoothstep.h>
 
 #define SMOOTHSTEP_SINGLE_DEF(X_TYPE)                                          \
   _CLC_OVERLOAD _CLC_DEF X_TYPE smoothstep(X_TYPE edge0, X_TYPE edge1,         \
diff --git a/libclc/opencl/lib/generic/common/step.cl b/libclc/opencl/lib/generic/common/step.cl
index 99a7af08c9264..d946771698150 100644
--- a/libclc/opencl/lib/generic/common/step.cl
+++ b/libclc/opencl/lib/generic/common/step.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/common/clc_step.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/common/step.h>
 
 #define __CLC_BODY <step.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/cross.cl b/libclc/opencl/lib/generic/geometric/cross.cl
index bd4f86bb15d84..7553c80e6b253 100644
--- a/libclc/opencl/lib/generic/geometric/cross.cl
+++ b/libclc/opencl/lib/generic/geometric/cross.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_cross.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/cross.h>
 
 _CLC_OVERLOAD _CLC_DEF float3 cross(float3 p0, float3 p1) {
   return __clc_cross(p0, p1);
diff --git a/libclc/opencl/lib/generic/geometric/distance.cl b/libclc/opencl/lib/generic/geometric/distance.cl
index b2194abd60e97..54c78d3609ba7 100644
--- a/libclc/opencl/lib/generic/geometric/distance.cl
+++ b/libclc/opencl/lib/generic/geometric/distance.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_distance.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/distance.h>
 
 #define FUNCTION distance
 #define __CLC_BODY <clc/geometric/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/geometric/dot.cl b/libclc/opencl/lib/generic/geometric/dot.cl
index 785767f27aacf..05425aafb0404 100644
--- a/libclc/opencl/lib/generic/geometric/dot.cl
+++ b/libclc/opencl/lib/generic/geometric/dot.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_dot.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/dot.h>
 
 #define FUNCTION dot
 #define __CLC_BODY <clc/geometric/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/geometric/fast_distance.cl b/libclc/opencl/lib/generic/geometric/fast_distance.cl
index 2a19418a175b9..bc30c298a2b9a 100644
--- a/libclc/opencl/lib/generic/geometric/fast_distance.cl
+++ b/libclc/opencl/lib/generic/geometric/fast_distance.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_fast_distance.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/fast_distance.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION fast_distance
diff --git a/libclc/opencl/lib/generic/geometric/fast_length.cl b/libclc/opencl/lib/generic/geometric/fast_length.cl
index 3af751af5425d..407080900cceb 100644
--- a/libclc/opencl/lib/generic/geometric/fast_length.cl
+++ b/libclc/opencl/lib/generic/geometric/fast_length.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_fast_length.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/fast_length.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION fast_length
diff --git a/libclc/opencl/lib/generic/geometric/fast_normalize.cl b/libclc/opencl/lib/generic/geometric/fast_normalize.cl
index 938b30093b8de..195081371a5b2 100644
--- a/libclc/opencl/lib/generic/geometric/fast_normalize.cl
+++ b/libclc/opencl/lib/generic/geometric/fast_normalize.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_fast_normalize.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/fast_normalize.h>
 
 #define FUNCTION fast_normalize
 #define __FLOAT_ONLY
diff --git a/libclc/opencl/lib/generic/geometric/length.cl b/libclc/opencl/lib/generic/geometric/length.cl
index 5645ade4c9172..0b0ebaa74d951 100644
--- a/libclc/opencl/lib/generic/geometric/length.cl
+++ b/libclc/opencl/lib/generic/geometric/length.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_length.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/length.h>
 
 #define FUNCTION length
 #define __CLC_BODY <clc/geometric/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/geometric/normalize.cl b/libclc/opencl/lib/generic/geometric/normalize.cl
index 914f493f2cb28..eb84ffd382129 100644
--- a/libclc/opencl/lib/generic/geometric/normalize.cl
+++ b/libclc/opencl/lib/generic/geometric/normalize.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/geometric/clc_normalize.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/geometric/normalize.h>
 
 #define FUNCTION normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
diff --git a/libclc/opencl/lib/generic/integer/abs.cl b/libclc/opencl/lib/generic/integer/abs.cl
index 5d354d97e71c5..0771ec7934330 100644
--- a/libclc/opencl/lib/generic/integer/abs.cl
+++ b/libclc/opencl/lib/generic/integer/abs.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_abs.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/abs.h>
 
 #define __CLC_BODY <abs.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/abs_diff.cl b/libclc/opencl/lib/generic/integer/abs_diff.cl
index 73d6631b3a4a3..d65b0468a7f21 100644
--- a/libclc/opencl/lib/generic/integer/abs_diff.cl
+++ b/libclc/opencl/lib/generic/integer/abs_diff.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_abs_diff.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/abs_diff.h>
 
 #define __CLC_BODY <abs_diff.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/add_sat.cl b/libclc/opencl/lib/generic/integer/add_sat.cl
index 8fe1d32a82216..50d989a47211b 100644
--- a/libclc/opencl/lib/generic/integer/add_sat.cl
+++ b/libclc/opencl/lib/generic/integer/add_sat.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_add_sat.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/add_sat.h>
 
 #define FUNCTION add_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/clz.cl b/libclc/opencl/lib/generic/integer/clz.cl
index cf12a9838c6d7..58110df1eec24 100644
--- a/libclc/opencl/lib/generic/integer/clz.cl
+++ b/libclc/opencl/lib/generic/integer/clz.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_clz.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/clz.h>
 
 #define FUNCTION clz
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/ctz.cl b/libclc/opencl/lib/generic/integer/ctz.cl
index 1b0470e08fd36..cbb167275a4a6 100644
--- a/libclc/opencl/lib/generic/integer/ctz.cl
+++ b/libclc/opencl/lib/generic/integer/ctz.cl
@@ -9,7 +9,7 @@
 #if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 
 #include <clc/integer/clc_ctz.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/ctz.h>
 
 #define FUNCTION ctz
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/hadd.cl b/libclc/opencl/lib/generic/integer/hadd.cl
index 100cb7bf207a5..e9edc88bce02c 100644
--- a/libclc/opencl/lib/generic/integer/hadd.cl
+++ b/libclc/opencl/lib/generic/integer/hadd.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_hadd.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/hadd.h>
 
 #define FUNCTION hadd
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad24.cl b/libclc/opencl/lib/generic/integer/mad24.cl
index d42873f1dd15e..5e9dfb3b3031a 100644
--- a/libclc/opencl/lib/generic/integer/mad24.cl
+++ b/libclc/opencl/lib/generic/integer/mad24.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mad24.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mad24.h>
 
 #define FUNCTION mad24
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad_hi.cl b/libclc/opencl/lib/generic/integer/mad_hi.cl
index 68e610d400cc1..786614e42bc8b 100644
--- a/libclc/opencl/lib/generic/integer/mad_hi.cl
+++ b/libclc/opencl/lib/generic/integer/mad_hi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mad_hi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mad_hi.h>
 
 #define FUNCTION mad_hi
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad_sat.cl b/libclc/opencl/lib/generic/integer/mad_sat.cl
index d9daf56632b07..54ddc0eed2d17 100644
--- a/libclc/opencl/lib/generic/integer/mad_sat.cl
+++ b/libclc/opencl/lib/generic/integer/mad_sat.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mad_sat.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mad_sat.h>
 
 #define FUNCTION mad_sat
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mul24.cl b/libclc/opencl/lib/generic/integer/mul24.cl
index b0a9ff80f3528..e6db6b89c5d98 100644
--- a/libclc/opencl/lib/generic/integer/mul24.cl
+++ b/libclc/opencl/lib/generic/integer/mul24.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mul24.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mul24.h>
 
 #define FUNCTION mul24
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mul_hi.cl b/libclc/opencl/lib/generic/integer/mul_hi.cl
index 40cb359f5b296..4c7b8c2caa994 100644
--- a/libclc/opencl/lib/generic/integer/mul_hi.cl
+++ b/libclc/opencl/lib/generic/integer/mul_hi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mul_hi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mul_hi.h>
 
 #define FUNCTION mul_hi
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/popcount.cl b/libclc/opencl/lib/generic/integer/popcount.cl
index a349f454813ef..3c33c82302c39 100644
--- a/libclc/opencl/lib/generic/integer/popcount.cl
+++ b/libclc/opencl/lib/generic/integer/popcount.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_popcount.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/popcount.h>
 
 #define FUNCTION popcount
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/rhadd.cl b/libclc/opencl/lib/generic/integer/rhadd.cl
index 539def845488d..f835511899429 100644
--- a/libclc/opencl/lib/generic/integer/rhadd.cl
+++ b/libclc/opencl/lib/generic/integer/rhadd.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_rhadd.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/rhadd.h>
 
 #define FUNCTION rhadd
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/rotate.cl b/libclc/opencl/lib/generic/integer/rotate.cl
index a340ccf77b2ea..282270f3b3ff2 100644
--- a/libclc/opencl/lib/generic/integer/rotate.cl
+++ b/libclc/opencl/lib/generic/integer/rotate.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_rotate.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/rotate.h>
 
 #define FUNCTION rotate
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/sub_sat.cl b/libclc/opencl/lib/generic/integer/sub_sat.cl
index 46596724358ab..0b45df1118f14 100644
--- a/libclc/opencl/lib/generic/integer/sub_sat.cl
+++ b/libclc/opencl/lib/generic/integer/sub_sat.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_sub_sat.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/sub_sat.h>
 
 #define FUNCTION sub_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/upsample.cl b/libclc/opencl/lib/generic/integer/upsample.cl
index 32cb3cffc9ebd..5f95c811e058d 100644
--- a/libclc/opencl/lib/generic/integer/upsample.cl
+++ b/libclc/opencl/lib/generic/integer/upsample.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_upsample.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/upsample.h>
 
 #define __CLC_UPSAMPLE_IMPL(BGENTYPE, GENTYPE, UGENTYPE)                       \
   _CLC_OVERLOAD _CLC_DEF BGENTYPE upsample(GENTYPE hi, UGENTYPE lo) {          \
diff --git a/libclc/opencl/lib/generic/math/acos.cl b/libclc/opencl/lib/generic/math/acos.cl
index 3d290225a2c29..040b2b82f6f98 100644
--- a/libclc/opencl/lib/generic/math/acos.cl
+++ b/libclc/opencl/lib/generic/math/acos.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_acos.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/acos.h>
 
 #define FUNCTION acos
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/acosh.cl b/libclc/opencl/lib/generic/math/acosh.cl
index fea40caa3e20c..55b60bc99614a 100644
--- a/libclc/opencl/lib/generic/math/acosh.cl
+++ b/libclc/opencl/lib/generic/math/acosh.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_acosh.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/acosh.h>
 
 #define FUNCTION acosh
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/acospi.cl b/libclc/opencl/lib/generic/math/acospi.cl
index 6c5ba2982ed0b..2537c992e2ada 100644
--- a/libclc/opencl/lib/generic/math/acospi.cl
+++ b/libclc/opencl/lib/generic/math/acospi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_acospi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/acospi.h>
 
 #define FUNCTION acospi
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/asin.cl b/libclc/opencl/lib/generic/math/asin.cl
index b8d02cef7d3ac..08dde1201392e 100644
--- a/libclc/opencl/lib/generic/math/asin.cl
+++ b/libclc/opencl/lib/generic/math/asin.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_asin.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/asin.h>
 
 #define FUNCTION asin
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/asinh.cl b/libclc/opencl/lib/generic/math/asinh.cl
index f043a33fe3177..189392f50b5ee 100644
--- a/libclc/opencl/lib/generic/math/asinh.cl
+++ b/libclc/opencl/lib/generic/math/asinh.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_asinh.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/asinh.h>
 
 #define FUNCTION asinh
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/asinpi.cl b/libclc/opencl/lib/generic/math/asinpi.cl
index 688da90337ea9..b9327faaccba5 100644
--- a/libclc/opencl/lib/generic/math/asinpi.cl
+++ b/libclc/opencl/lib/generic/math/asinpi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_asinpi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/asinpi.h>
 
 #define FUNCTION asinpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/atan.cl b/libclc/opencl/lib/generic/math/atan.cl
index 006fc9e65d45e..e2b1530f9e4cf 100644
--- a/libclc/opencl/lib/generic/math/atan.cl
+++ b/libclc/opencl/lib/generic/math/atan.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_atan.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/atan.h>
 
 #define FUNCTION atan
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/atan2.cl b/libclc/opencl/lib/generic/math/atan2.cl
index ae277e7dc9567..9f3d4a965e58a 100644
--- a/libclc/opencl/lib/generic/math/atan2.cl
+++ b/libclc/opencl/lib/generic/math/atan2.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/math/clc_atan2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/atan2.h>
 
 #define FUNCTION atan2
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/atan2pi.cl b/libclc/opencl/lib/generic/math/atan2pi.cl
index 06f0974d4325a..9b3fea163f685 100644
--- a/libclc/opencl/lib/generic/math/atan2pi.cl
+++ b/libclc/opencl/lib/generic/math/atan2pi.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/math/clc_atan2pi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/atan2pi.h>
 
 #define FUNCTION atan2pi
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/atanh.cl b/libclc/opencl/lib/generic/math/atanh.cl
index ad4c54b81d21d..5e4564d347026 100644
--- a/libclc/opencl/lib/generic/math/atanh.cl
+++ b/libclc/opencl/lib/generic/math/atanh.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_atanh.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/atanh.h>
 
 #define FUNCTION atanh
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/atanpi.cl b/libclc/opencl/lib/generic/math/atanpi.cl
index 3eba081d1b3f9..ddeb38e64e993 100644
--- a/libclc/opencl/lib/generic/math/atanpi.cl
+++ b/libclc/opencl/lib/generic/math/atanpi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_atanpi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/atanpi.h>
 
 #define FUNCTION atanpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/cbrt.cl b/libclc/opencl/lib/generic/math/cbrt.cl
index edccc1aaa187d..dccb49a47dc03 100644
--- a/libclc/opencl/lib/generic/math/cbrt.cl
+++ b/libclc/opencl/lib/generic/math/cbrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_cbrt.inc>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/cbrt.h>
 
 #define FUNCTION cbrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/ceil.cl b/libclc/opencl/lib/generic/math/ceil.cl
index 981fa80a44430..e312281b98a23 100644
--- a/libclc/opencl/lib/generic/math/ceil.cl
+++ b/libclc/opencl/lib/generic/math/ceil.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_ceil.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/ceil.h>
 
 #define FUNCTION ceil
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/copysign.cl b/libclc/opencl/lib/generic/math/copysign.cl
index 9a34aa3ea94f8..7aa4cb2da0eec 100644
--- a/libclc/opencl/lib/generic/math/copysign.cl
+++ b/libclc/opencl/lib/generic/math/copysign.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_copysign.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/copysign.h>
 
 #define FUNCTION copysign
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/cos.cl b/libclc/opencl/lib/generic/math/cos.cl
index 01f419ebc2878..fb40b6c00e3c8 100644
--- a/libclc/opencl/lib/generic/math/cos.cl
+++ b/libclc/opencl/lib/generic/math/cos.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_cos.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/cos.h>
 
 #define FUNCTION cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/cosh.cl b/libclc/opencl/lib/generic/math/cosh.cl
index c2ee2375ce5a8..7106fc9f18fa5 100644
--- a/libclc/opencl/lib/generic/math/cosh.cl
+++ b/libclc/opencl/lib/generic/math/cosh.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_cosh.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/cosh.h>
 
 #define FUNCTION cosh
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/cospi.cl b/libclc/opencl/lib/generic/math/cospi.cl
index 05de33f74373c..0e242ad0d4779 100644
--- a/libclc/opencl/lib/generic/math/cospi.cl
+++ b/libclc/opencl/lib/generic/math/cospi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_cospi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/cospi.h>
 
 #define FUNCTION cospi
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/erf.cl b/libclc/opencl/lib/generic/math/erf.cl
index ba60e4c47355f..83c38867e0460 100644
--- a/libclc/opencl/lib/generic/math/erf.cl
+++ b/libclc/opencl/lib/generic/math/erf.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_erf.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/erf.h>
 
 #define FUNCTION erf
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/erfc.cl b/libclc/opencl/lib/generic/math/erfc.cl
index 4624a04525668..1c473999bba93 100644
--- a/libclc/opencl/lib/generic/math/erfc.cl
+++ b/libclc/opencl/lib/generic/math/erfc.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_erfc.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/erfc.h>
 
 #define FUNCTION erfc
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/exp.cl b/libclc/opencl/lib/generic/math/exp.cl
index 8b20e26716a3a..8125ce314e8e4 100644
--- a/libclc/opencl/lib/generic/math/exp.cl
+++ b/libclc/opencl/lib/generic/math/exp.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_exp.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/exp.h>
 
 #define FUNCTION exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/exp10.cl b/libclc/opencl/lib/generic/math/exp10.cl
index d5bcf1960d7d9..f8c0f94a990bc 100644
--- a/libclc/opencl/lib/generic/math/exp10.cl
+++ b/libclc/opencl/lib/generic/math/exp10.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_exp10.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/exp10.h>
 
 #define FUNCTION exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/exp2.cl b/libclc/opencl/lib/generic/math/exp2.cl
index 130dbbd77be8a..dc75c2201a460 100644
--- a/libclc/opencl/lib/generic/math/exp2.cl
+++ b/libclc/opencl/lib/generic/math/exp2.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_exp2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/exp2.h>
 
 #define FUNCTION exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/expm1.cl b/libclc/opencl/lib/generic/math/expm1.cl
index 26c3eac9e678b..7507786c2ad09 100644
--- a/libclc/opencl/lib/generic/math/expm1.cl
+++ b/libclc/opencl/lib/generic/math/expm1.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_expm1.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/expm1.h>
 
 #define FUNCTION expm1
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/fabs.cl b/libclc/opencl/lib/generic/math/fabs.cl
index 4f9cf117a85f2..0d3128f3b2a1f 100644
--- a/libclc/opencl/lib/generic/math/fabs.cl
+++ b/libclc/opencl/lib/generic/math/fabs.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_fabs.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fabs.h>
 
 #define FUNCTION fabs
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/fdim.cl b/libclc/opencl/lib/generic/math/fdim.cl
index 6b9a46c1bbe1a..ecdcd3aef3a7d 100644
--- a/libclc/opencl/lib/generic/math/fdim.cl
+++ b/libclc/opencl/lib/generic/math/fdim.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_fdim.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fdim.h>
 
 #define FUNCTION fdim
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/floor.cl b/libclc/opencl/lib/generic/math/floor.cl
index 9d5f5691a7059..cda8363f91360 100644
--- a/libclc/opencl/lib/generic/math/floor.cl
+++ b/libclc/opencl/lib/generic/math/floor.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_floor.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/floor.h>
 
 #define FUNCTION floor
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/fma.cl b/libclc/opencl/lib/generic/math/fma.cl
index c077357a44f0d..867b975069013 100644
--- a/libclc/opencl/lib/generic/math/fma.cl
+++ b/libclc/opencl/lib/generic/math/fma.cl
@@ -8,7 +8,7 @@
 
 #include <clc/math/clc_fma.h>
 #include <clc/math/math.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fma.h>
 
 #define FUNCTION fma
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/fmax.cl b/libclc/opencl/lib/generic/math/fmax.cl
index 6bc22823757a6..7dbd6fe0bd7f4 100644
--- a/libclc/opencl/lib/generic/math/fmax.cl
+++ b/libclc/opencl/lib/generic/math/fmax.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_fmax.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fmax.h>
 
 #define FUNCTION fmax
 #define __CLC_BODY <clc/shared/binary_def_with_scalar_second_arg.inc>
diff --git a/libclc/opencl/lib/generic/math/fmin.cl b/libclc/opencl/lib/generic/math/fmin.cl
index 2aea160b975e4..ee0dca480073e 100644
--- a/libclc/opencl/lib/generic/math/fmin.cl
+++ b/libclc/opencl/lib/generic/math/fmin.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_fmin.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fmin.h>
 
 #define FUNCTION fmin
 #define __CLC_BODY <clc/shared/binary_def_with_scalar_second_arg.inc>
diff --git a/libclc/opencl/lib/generic/math/fmod.cl b/libclc/opencl/lib/generic/math/fmod.cl
index ddd071fdab95b..ec1904d213d87 100644
--- a/libclc/opencl/lib/generic/math/fmod.cl
+++ b/libclc/opencl/lib/generic/math/fmod.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_fmod.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fmod.h>
 
 #define FUNCTION fmod
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/fract.cl b/libclc/opencl/lib/generic/math/fract.cl
index c3e8b0afee78d..9567072bf4c39 100644
--- a/libclc/opencl/lib/generic/math/fract.cl
+++ b/libclc/opencl/lib/generic/math/fract.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_fract.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fract.h>
 
 #define FUNCTION fract
 #define __CLC_BODY <clc/math/unary_def_with_ptr.inc>
diff --git a/libclc/opencl/lib/generic/math/frexp.cl b/libclc/opencl/lib/generic/math/frexp.cl
index 940a8e37b5b59..3ac9be792ae65 100644
--- a/libclc/opencl/lib/generic/math/frexp.cl
+++ b/libclc/opencl/lib/generic/math/frexp.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_frexp.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/frexp.h>
 
 #define FUNCTION frexp
 #define __CLC_BODY <clc/math/unary_def_with_int_ptr.inc>
diff --git a/libclc/opencl/lib/generic/math/half_cos.cl b/libclc/opencl/lib/generic/math/half_cos.cl
index 5bb91b8addbca..377c4d1642599 100644
--- a/libclc/opencl/lib/generic/math/half_cos.cl
+++ b/libclc/opencl/lib/generic/math/half_cos.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_cos.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_cos.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_cos
diff --git a/libclc/opencl/lib/generic/math/half_divide.cl b/libclc/opencl/lib/generic/math/half_divide.cl
index 8b5d8eddd67a7..730617ba67fb3 100644
--- a/libclc/opencl/lib/generic/math/half_divide.cl
+++ b/libclc/opencl/lib/generic/math/half_divide.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_divide.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_divide.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_divide
diff --git a/libclc/opencl/lib/generic/math/half_exp.cl b/libclc/opencl/lib/generic/math/half_exp.cl
index 97507ab2e189a..05b8753b2faf4 100644
--- a/libclc/opencl/lib/generic/math/half_exp.cl
+++ b/libclc/opencl/lib/generic/math/half_exp.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_exp.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_exp.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_exp
diff --git a/libclc/opencl/lib/generic/math/half_exp10.cl b/libclc/opencl/lib/generic/math/half_exp10.cl
index 6925035d05f7c..b4e46aaa771cd 100644
--- a/libclc/opencl/lib/generic/math/half_exp10.cl
+++ b/libclc/opencl/lib/generic/math/half_exp10.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_exp10.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_exp10.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_exp10
diff --git a/libclc/opencl/lib/generic/math/half_exp2.cl b/libclc/opencl/lib/generic/math/half_exp2.cl
index 5bd244482ffa4..51fe9f72477a9 100644
--- a/libclc/opencl/lib/generic/math/half_exp2.cl
+++ b/libclc/opencl/lib/generic/math/half_exp2.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_exp2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_exp2.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_exp2
diff --git a/libclc/opencl/lib/generic/math/half_log.cl b/libclc/opencl/lib/generic/math/half_log.cl
index 28c6964166983..b21835204a173 100644
--- a/libclc/opencl/lib/generic/math/half_log.cl
+++ b/libclc/opencl/lib/generic/math/half_log.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_log.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_log.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_log
diff --git a/libclc/opencl/lib/generic/math/half_log10.cl b/libclc/opencl/lib/generic/math/half_log10.cl
index fcd0a818308c0..acb50f99af192 100644
--- a/libclc/opencl/lib/generic/math/half_log10.cl
+++ b/libclc/opencl/lib/generic/math/half_log10.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_log10.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_log10.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_log10
diff --git a/libclc/opencl/lib/generic/math/half_log2.cl b/libclc/opencl/lib/generic/math/half_log2.cl
index 05214e6f5a629..c97e9e093878d 100644
--- a/libclc/opencl/lib/generic/math/half_log2.cl
+++ b/libclc/opencl/lib/generic/math/half_log2.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_log2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_log2.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_log2
diff --git a/libclc/opencl/lib/generic/math/half_powr.cl b/libclc/opencl/lib/generic/math/half_powr.cl
index 3562d3e286faf..d61a18ab91321 100644
--- a/libclc/opencl/lib/generic/math/half_powr.cl
+++ b/libclc/opencl/lib/generic/math/half_powr.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_powr.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_powr.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_powr
diff --git a/libclc/opencl/lib/generic/math/half_recip.cl b/libclc/opencl/lib/generic/math/half_recip.cl
index e2650662372c8..1917448c4e695 100644
--- a/libclc/opencl/lib/generic/math/half_recip.cl
+++ b/libclc/opencl/lib/generic/math/half_recip.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_recip.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_recip.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_recip
diff --git a/libclc/opencl/lib/generic/math/half_rsqrt.cl b/libclc/opencl/lib/generic/math/half_rsqrt.cl
index 4f287d72d6417..d6ffa651fbe6a 100644
--- a/libclc/opencl/lib/generic/math/half_rsqrt.cl
+++ b/libclc/opencl/lib/generic/math/half_rsqrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_rsqrt.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_rsqrt.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_rsqrt
diff --git a/libclc/opencl/lib/generic/math/half_sin.cl b/libclc/opencl/lib/generic/math/half_sin.cl
index c1e5696185102..baba0cca49849 100644
--- a/libclc/opencl/lib/generic/math/half_sin.cl
+++ b/libclc/opencl/lib/generic/math/half_sin.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_sin.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_sin.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_sin
diff --git a/libclc/opencl/lib/generic/math/half_sqrt.cl b/libclc/opencl/lib/generic/math/half_sqrt.cl
index c327881a553b8..7a59744ab11d0 100644
--- a/libclc/opencl/lib/generic/math/half_sqrt.cl
+++ b/libclc/opencl/lib/generic/math/half_sqrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_sqrt.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_sqrt.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_sqrt
diff --git a/libclc/opencl/lib/generic/math/half_tan.cl b/libclc/opencl/lib/generic/math/half_tan.cl
index 0d16b08dc2233..acd4b011173db 100644
--- a/libclc/opencl/lib/generic/math/half_tan.cl
+++ b/libclc/opencl/lib/generic/math/half_tan.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_half_tan.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/half_tan.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION half_tan
diff --git a/libclc/opencl/lib/generic/math/hypot.cl b/libclc/opencl/lib/generic/math/hypot.cl
index 3ac2983efc4bf..4c59e4696e5ae 100644
--- a/libclc/opencl/lib/generic/math/hypot.cl
+++ b/libclc/opencl/lib/generic/math/hypot.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_hypot.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/hypot.h>
 
 #define FUNCTION hypot
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/ilogb.cl b/libclc/opencl/lib/generic/math/ilogb.cl
index 147f82d95d1f0..3829ce68fb577 100644
--- a/libclc/opencl/lib/generic/math/ilogb.cl
+++ b/libclc/opencl/lib/generic/math/ilogb.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_ilogb.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/ilogb.h>
 
 #define FUNCTION ilogb
 #define __CLC_BODY <clc/math/unary_def_with_int_return.inc>
diff --git a/libclc/opencl/lib/generic/math/ldexp.cl b/libclc/opencl/lib/generic/math/ldexp.cl
index 069ba8251feba..88c60716d7829 100644
--- a/libclc/opencl/lib/generic/math/ldexp.cl
+++ b/libclc/opencl/lib/generic/math/ldexp.cl
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_ldexp.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/ldexp.h>
 
 #define FUNCTION ldexp
-#define __CLC_FUNCTION(x) __clc_ldexp
+#define __IMPL_FUNCTION(x) __clc_ldexp
 #define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/lgamma.cl b/libclc/opencl/lib/generic/math/lgamma.cl
index 002a08153a3a3..253ce31c15771 100644
--- a/libclc/opencl/lib/generic/math/lgamma.cl
+++ b/libclc/opencl/lib/generic/math/lgamma.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_lgamma.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/lgamma.h>
 
 #define FUNCTION lgamma
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/lgamma_r.cl b/libclc/opencl/lib/generic/math/lgamma_r.cl
index c44051cb4c39a..8753f6c75d8b1 100644
--- a/libclc/opencl/lib/generic/math/lgamma_r.cl
+++ b/libclc/opencl/lib/generic/math/lgamma_r.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_lgamma_r.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/lgamma_r.h>
 
 #define FUNCTION lgamma_r
 #define __CLC_BODY <clc/math/unary_def_with_int_ptr.inc>
diff --git a/libclc/opencl/lib/generic/math/log.cl b/libclc/opencl/lib/generic/math/log.cl
index 78430644a55ee..dbe3e35727033 100644
--- a/libclc/opencl/lib/generic/math/log.cl
+++ b/libclc/opencl/lib/generic/math/log.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/math/clc_log.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/log.h>
 
 #define FUNCTION log
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/log10.cl b/libclc/opencl/lib/generic/math/log10.cl
index 01113e7c00c92..d5137a7cab7ab 100644
--- a/libclc/opencl/lib/generic/math/log10.cl
+++ b/libclc/opencl/lib/generic/math/log10.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/math/clc_log10.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/log10.h>
 
 #define FUNCTION log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/log1p.cl b/libclc/opencl/lib/generic/math/log1p.cl
index 26c4df015bc88..d389ab72908c8 100644
--- a/libclc/opencl/lib/generic/math/log1p.cl
+++ b/libclc/opencl/lib/generic/math/log1p.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_log1p.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/log1p.h>
 
 #define FUNCTION log1p
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/log2.cl b/libclc/opencl/lib/generic/math/log2.cl
index d70f16d6f7d89..d1433bc825813 100644
--- a/libclc/opencl/lib/generic/math/log2.cl
+++ b/libclc/opencl/lib/generic/math/log2.cl
@@ -8,7 +8,7 @@
 
 #include <clc/clcmacro.h>
 #include <clc/math/clc_log2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/log2.h>
 
 #define FUNCTION log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/logb.cl b/libclc/opencl/lib/generic/math/logb.cl
index 21046f2ac98cf..037380a26a727 100644
--- a/libclc/opencl/lib/generic/math/logb.cl
+++ b/libclc/opencl/lib/generic/math/logb.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_logb.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/logb.h>
 
 #define FUNCTION logb
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/mad.cl b/libclc/opencl/lib/generic/math/mad.cl
index 39aa8e884cc03..3fe2dd9bed5fb 100644
--- a/libclc/opencl/lib/generic/math/mad.cl
+++ b/libclc/opencl/lib/generic/math/mad.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_mad.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/mad.h>
 
 #define FUNCTION mad
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/maxmag.cl b/libclc/opencl/lib/generic/math/maxmag.cl
index cf4cfc7ded3a5..4f9dbeb8d27e7 100644
--- a/libclc/opencl/lib/generic/math/maxmag.cl
+++ b/libclc/opencl/lib/generic/math/maxmag.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_maxmag.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/maxmag.h>
 
 #define FUNCTION maxmag
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/minmag.cl b/libclc/opencl/lib/generic/math/minmag.cl
index e636ca7e2c40a..efe72b6f545c2 100644
--- a/libclc/opencl/lib/generic/math/minmag.cl
+++ b/libclc/opencl/lib/generic/math/minmag.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_minmag.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/minmag.h>
 
 #define FUNCTION minmag
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/modf.cl b/libclc/opencl/lib/generic/math/modf.cl
index 7ad3a2c7550d9..de6524e02f73f 100644
--- a/libclc/opencl/lib/generic/math/modf.cl
+++ b/libclc/opencl/lib/generic/math/modf.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_modf.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/modf.h>
 
 #define FUNCTION modf
 #define __CLC_BODY <clc/math/unary_def_with_ptr.inc>
diff --git a/libclc/opencl/lib/generic/math/native_cos.cl b/libclc/opencl/lib/generic/math/native_cos.cl
index d0df037716943..85944a03c5468 100644
--- a/libclc/opencl/lib/generic/math/native_cos.cl
+++ b/libclc/opencl/lib/generic/math/native_cos.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_cos.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_cos.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_cos
diff --git a/libclc/opencl/lib/generic/math/native_divide.cl b/libclc/opencl/lib/generic/math/native_divide.cl
index 6bcc5ecf4d05b..8efd8cc21b5d8 100644
--- a/libclc/opencl/lib/generic/math/native_divide.cl
+++ b/libclc/opencl/lib/generic/math/native_divide.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_divide.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_divide.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_divide
diff --git a/libclc/opencl/lib/generic/math/native_exp.cl b/libclc/opencl/lib/generic/math/native_exp.cl
index 23b72fc82f8c0..42208371e195c 100644
--- a/libclc/opencl/lib/generic/math/native_exp.cl
+++ b/libclc/opencl/lib/generic/math/native_exp.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_exp.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_exp.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_exp
diff --git a/libclc/opencl/lib/generic/math/native_exp10.cl b/libclc/opencl/lib/generic/math/native_exp10.cl
index 76cbc57cb3ba5..aa2585ff4311d 100644
--- a/libclc/opencl/lib/generic/math/native_exp10.cl
+++ b/libclc/opencl/lib/generic/math/native_exp10.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_exp10.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_exp10.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_exp10
diff --git a/libclc/opencl/lib/generic/math/native_exp2.cl b/libclc/opencl/lib/generic/math/native_exp2.cl
index 107db022b6e2b..8955b28b367cb 100644
--- a/libclc/opencl/lib/generic/math/native_exp2.cl
+++ b/libclc/opencl/lib/generic/math/native_exp2.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_exp2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_exp2.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_exp2
diff --git a/libclc/opencl/lib/generic/math/native_log.cl b/libclc/opencl/lib/generic/math/native_log.cl
index 3faf70674fac8..334f7c04e3899 100644
--- a/libclc/opencl/lib/generic/math/native_log.cl
+++ b/libclc/opencl/lib/generic/math/native_log.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_log.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_log.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_log
diff --git a/libclc/opencl/lib/generic/math/native_log10.cl b/libclc/opencl/lib/generic/math/native_log10.cl
index e71ac4448df25..a65938ee4c6cb 100644
--- a/libclc/opencl/lib/generic/math/native_log10.cl
+++ b/libclc/opencl/lib/generic/math/native_log10.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_log10.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_log10.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_log10
diff --git a/libclc/opencl/lib/generic/math/native_log2.cl b/libclc/opencl/lib/generic/math/native_log2.cl
index c971013c4021d..f10b533f91f40 100644
--- a/libclc/opencl/lib/generic/math/native_log2.cl
+++ b/libclc/opencl/lib/generic/math/native_log2.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_log2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_log2.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_log2
diff --git a/libclc/opencl/lib/generic/math/native_powr.cl b/libclc/opencl/lib/generic/math/native_powr.cl
index ffc7e5a174377..8301443cfbcfd 100644
--- a/libclc/opencl/lib/generic/math/native_powr.cl
+++ b/libclc/opencl/lib/generic/math/native_powr.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_powr.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_powr.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_powr
diff --git a/libclc/opencl/lib/generic/math/native_recip.cl b/libclc/opencl/lib/generic/math/native_recip.cl
index cad339ecb4f78..145845b455147 100644
--- a/libclc/opencl/lib/generic/math/native_recip.cl
+++ b/libclc/opencl/lib/generic/math/native_recip.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_recip.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_recip.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_recip
diff --git a/libclc/opencl/lib/generic/math/native_rsqrt.cl b/libclc/opencl/lib/generic/math/native_rsqrt.cl
index 69a387387f336..cbb5a754fc967 100644
--- a/libclc/opencl/lib/generic/math/native_rsqrt.cl
+++ b/libclc/opencl/lib/generic/math/native_rsqrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_rsqrt.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_rsqrt.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_rsqrt
diff --git a/libclc/opencl/lib/generic/math/native_sin.cl b/libclc/opencl/lib/generic/math/native_sin.cl
index c1a3faed39a26..dfcc882614a63 100644
--- a/libclc/opencl/lib/generic/math/native_sin.cl
+++ b/libclc/opencl/lib/generic/math/native_sin.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_sin.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_sin.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_sin
diff --git a/libclc/opencl/lib/generic/math/native_sqrt.cl b/libclc/opencl/lib/generic/math/native_sqrt.cl
index 5b5487cfea934..a0be41d33fa43 100644
--- a/libclc/opencl/lib/generic/math/native_sqrt.cl
+++ b/libclc/opencl/lib/generic/math/native_sqrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_sqrt.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_sqrt.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_sqrt
diff --git a/libclc/opencl/lib/generic/math/native_tan.cl b/libclc/opencl/lib/generic/math/native_tan.cl
index 000d6c5f366a1..86eee4f5913f0 100644
--- a/libclc/opencl/lib/generic/math/native_tan.cl
+++ b/libclc/opencl/lib/generic/math/native_tan.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_native_tan.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/native_tan.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION native_tan
diff --git a/libclc/opencl/lib/generic/math/nextafter.cl b/libclc/opencl/lib/generic/math/nextafter.cl
index 6a5a745f82526..9c30e5668a56b 100644
--- a/libclc/opencl/lib/generic/math/nextafter.cl
+++ b/libclc/opencl/lib/generic/math/nextafter.cl
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_nextafter.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/nextafter.h>
 
 #define FUNCTION nextafter
-#define __CLC_FUNCTION(x) __clc_nextafter
+#define __IMPL_FUNCTION(x) __clc_nextafter
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/pow.cl b/libclc/opencl/lib/generic/math/pow.cl
index b3a77f5a2a71e..2fddb7e889649 100644
--- a/libclc/opencl/lib/generic/math/pow.cl
+++ b/libclc/opencl/lib/generic/math/pow.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_pow.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/pow.h>
 
 #define FUNCTION pow
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/pown.cl b/libclc/opencl/lib/generic/math/pown.cl
index 115bae3406f0e..0a5ee893a45a1 100644
--- a/libclc/opencl/lib/generic/math/pown.cl
+++ b/libclc/opencl/lib/generic/math/pown.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_pown.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/pown.h>
 
 #define FUNCTION pown
 #define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
diff --git a/libclc/opencl/lib/generic/math/powr.cl b/libclc/opencl/lib/generic/math/powr.cl
index 87707198aae15..13af03ff13274 100644
--- a/libclc/opencl/lib/generic/math/powr.cl
+++ b/libclc/opencl/lib/generic/math/powr.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_powr.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/powr.h>
 
 #define FUNCTION powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/remainder.cl b/libclc/opencl/lib/generic/math/remainder.cl
index aeba5d59e89a9..224a66bc9b279 100644
--- a/libclc/opencl/lib/generic/math/remainder.cl
+++ b/libclc/opencl/lib/generic/math/remainder.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_remainder.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/remainder.h>
 
 #define FUNCTION remainder
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/remquo.cl b/libclc/opencl/lib/generic/math/remquo.cl
index 588c5f1cb1100..de102af58fe9f 100644
--- a/libclc/opencl/lib/generic/math/remquo.cl
+++ b/libclc/opencl/lib/generic/math/remquo.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_remquo.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/remquo.h>
 
 #define __CLC_BODY <remquo.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/rint.cl b/libclc/opencl/lib/generic/math/rint.cl
index 84ce99c4d216c..489883c97c141 100644
--- a/libclc/opencl/lib/generic/math/rint.cl
+++ b/libclc/opencl/lib/generic/math/rint.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_rint.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/rint.h>
 
 #define FUNCTION rint
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/rootn.cl b/libclc/opencl/lib/generic/math/rootn.cl
index 0e1acc95470df..e8dba17b9395d 100644
--- a/libclc/opencl/lib/generic/math/rootn.cl
+++ b/libclc/opencl/lib/generic/math/rootn.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_rootn.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/rootn.h>
 
 #define FUNCTION rootn
 #define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
diff --git a/libclc/opencl/lib/generic/math/round.cl b/libclc/opencl/lib/generic/math/round.cl
index 0ce44e72b00aa..c45e681b1c222 100644
--- a/libclc/opencl/lib/generic/math/round.cl
+++ b/libclc/opencl/lib/generic/math/round.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_round.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/round.h>
 
 #define FUNCTION round
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/rsqrt.cl b/libclc/opencl/lib/generic/math/rsqrt.cl
index 7d6dcfaa3de7d..47e4f450ed1a6 100644
--- a/libclc/opencl/lib/generic/math/rsqrt.cl
+++ b/libclc/opencl/lib/generic/math/rsqrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_rsqrt.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/rsqrt.h>
 
 #define FUNCTION rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/sin.cl b/libclc/opencl/lib/generic/math/sin.cl
index 293a2636c3070..ff7db2a5b3cb8 100644
--- a/libclc/opencl/lib/generic/math/sin.cl
+++ b/libclc/opencl/lib/generic/math/sin.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_sin.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/sin.h>
 
 #define FUNCTION sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/sincos.cl b/libclc/opencl/lib/generic/math/sincos.cl
index d87da4d7df912..c8871fd2b0020 100644
--- a/libclc/opencl/lib/generic/math/sincos.cl
+++ b/libclc/opencl/lib/generic/math/sincos.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_sincos.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/sincos.h>
 
 #define FUNCTION sincos
 #define __CLC_BODY <clc/math/unary_def_with_ptr.inc>
diff --git a/libclc/opencl/lib/generic/math/sinh.cl b/libclc/opencl/lib/generic/math/sinh.cl
index aa559f9330889..bd6ea20e7e898 100644
--- a/libclc/opencl/lib/generic/math/sinh.cl
+++ b/libclc/opencl/lib/generic/math/sinh.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_sinh.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/sinh.h>
 
 #define FUNCTION sinh
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/sinpi.cl b/libclc/opencl/lib/generic/math/sinpi.cl
index 8b48426a0ef84..badecd0909856 100644
--- a/libclc/opencl/lib/generic/math/sinpi.cl
+++ b/libclc/opencl/lib/generic/math/sinpi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_sinpi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/sinpi.h>
 
 #define FUNCTION sinpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/sqrt.cl b/libclc/opencl/lib/generic/math/sqrt.cl
index fd031daf5bfbd..3b05b25864485 100644
--- a/libclc/opencl/lib/generic/math/sqrt.cl
+++ b/libclc/opencl/lib/generic/math/sqrt.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_sqrt.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/sqrt.h>
 
 #define FUNCTION sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/tan.cl b/libclc/opencl/lib/generic/math/tan.cl
index 6df22b9c36959..95409fc5b092b 100644
--- a/libclc/opencl/lib/generic/math/tan.cl
+++ b/libclc/opencl/lib/generic/math/tan.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_tan.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/tan.h>
 
 #define FUNCTION tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/tanh.cl b/libclc/opencl/lib/generic/math/tanh.cl
index ef7cb1ec5d861..a77878bb79667 100644
--- a/libclc/opencl/lib/generic/math/tanh.cl
+++ b/libclc/opencl/lib/generic/math/tanh.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_tanh.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/tanh.h>
 
 #define FUNCTION tanh
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/tanpi.cl b/libclc/opencl/lib/generic/math/tanpi.cl
index 2195830faf32d..6b5805ec3cd04 100644
--- a/libclc/opencl/lib/generic/math/tanpi.cl
+++ b/libclc/opencl/lib/generic/math/tanpi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_tanpi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/tanpi.h>
 
 #define FUNCTION tanpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/tgamma.cl b/libclc/opencl/lib/generic/math/tgamma.cl
index 634ed7a506118..8f35d22426a70 100644
--- a/libclc/opencl/lib/generic/math/tgamma.cl
+++ b/libclc/opencl/lib/generic/math/tgamma.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_tgamma.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/tgamma.h>
 
 #define FUNCTION tgamma
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/math/trunc.cl b/libclc/opencl/lib/generic/math/trunc.cl
index 58f414dcc784c..ea56329bc8dea 100644
--- a/libclc/opencl/lib/generic/math/trunc.cl
+++ b/libclc/opencl/lib/generic/math/trunc.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_trunc.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/trunc.h>
 
 #define FUNCTION trunc
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/misc/shuffle.cl b/libclc/opencl/lib/generic/misc/shuffle.cl
index f8c5d16d21946..15295bf7d907a 100644
--- a/libclc/opencl/lib/generic/misc/shuffle.cl
+++ b/libclc/opencl/lib/generic/misc/shuffle.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/misc/clc_shuffle.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/misc/shuffle.h>
 
 #define FUNCTION shuffle
 
diff --git a/libclc/opencl/lib/generic/misc/shuffle2.cl b/libclc/opencl/lib/generic/misc/shuffle2.cl
index 74487d07db3d9..7e4c5f0922682 100644
--- a/libclc/opencl/lib/generic/misc/shuffle2.cl
+++ b/libclc/opencl/lib/generic/misc/shuffle2.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/misc/clc_shuffle2.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/misc/shuffle2.h>
 
 #define FUNCTION shuffle2
 
diff --git a/libclc/opencl/lib/generic/relational/all.cl b/libclc/opencl/lib/generic/relational/all.cl
index 43a3236bafe65..5ac4a465365a4 100644
--- a/libclc/opencl/lib/generic/relational/all.cl
+++ b/libclc/opencl/lib/generic/relational/all.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/all.h>
 #include <clc/relational/clc_all.h>
 
 #define ALL_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int all(TYPE v)
diff --git a/libclc/opencl/lib/generic/relational/any.cl b/libclc/opencl/lib/generic/relational/any.cl
index ae78c623f53f1..507f2b170693e 100644
--- a/libclc/opencl/lib/generic/relational/any.cl
+++ b/libclc/opencl/lib/generic/relational/any.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/any.h>
 #include <clc/relational/clc_any.h>
 
 #define ANY_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int any(TYPE v)
diff --git a/libclc/opencl/lib/generic/relational/binary_def.inc b/libclc/opencl/lib/generic/relational/binary_def.inc
index a3b90bbd6d417..54bb237b8f8f5 100644
--- a/libclc/opencl/lib/generic/relational/binary_def.inc
+++ b/libclc/opencl/lib/generic/relational/binary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 
 _CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_FLOATN a, __CLC_FLOATN b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
diff --git a/libclc/opencl/lib/generic/relational/bitselect.cl b/libclc/opencl/lib/generic/relational/bitselect.cl
index fc0a15768c5f5..0be7e5321caf0 100644
--- a/libclc/opencl/lib/generic/relational/bitselect.cl
+++ b/libclc/opencl/lib/generic/relational/bitselect.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/bitselect.h>
 #include <clc/relational/clc_bitselect.h>
 
 #define __CLC_BODY <bitselect.inc>
diff --git a/libclc/opencl/lib/generic/relational/isequal.cl b/libclc/opencl/lib/generic/relational/isequal.cl
index 71109d0e0bf3e..94f83f9452666 100644
--- a/libclc/opencl/lib/generic/relational/isequal.cl
+++ b/libclc/opencl/lib/generic/relational/isequal.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isequal.h>
 #include <clc/relational/clc_isequal.h>
 
 #define FUNCTION isequal
diff --git a/libclc/opencl/lib/generic/relational/isfinite.cl b/libclc/opencl/lib/generic/relational/isfinite.cl
index 7ee3ff2262f54..695ffea806d5c 100644
--- a/libclc/opencl/lib/generic/relational/isfinite.cl
+++ b/libclc/opencl/lib/generic/relational/isfinite.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isfinite.h>
 #include <clc/relational/clc_isfinite.h>
 
 #define FUNCTION isfinite
diff --git a/libclc/opencl/lib/generic/relational/isgreater.cl b/libclc/opencl/lib/generic/relational/isgreater.cl
index 13ecfa70fa4f6..fb46ff20ac608 100644
--- a/libclc/opencl/lib/generic/relational/isgreater.cl
+++ b/libclc/opencl/lib/generic/relational/isgreater.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isgreater.h>
 #include <clc/relational/clc_isgreater.h>
 
 #define FUNCTION isgreater
diff --git a/libclc/opencl/lib/generic/relational/isgreaterequal.cl b/libclc/opencl/lib/generic/relational/isgreaterequal.cl
index 458c4a6f6b747..b8edde2a05b77 100644
--- a/libclc/opencl/lib/generic/relational/isgreaterequal.cl
+++ b/libclc/opencl/lib/generic/relational/isgreaterequal.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isgreaterequal.h>
 #include <clc/relational/clc_isgreaterequal.h>
 
 #define FUNCTION isgreaterequal
diff --git a/libclc/opencl/lib/generic/relational/isinf.cl b/libclc/opencl/lib/generic/relational/isinf.cl
index 65e987940ff50..2c15f1f826762 100644
--- a/libclc/opencl/lib/generic/relational/isinf.cl
+++ b/libclc/opencl/lib/generic/relational/isinf.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isinf.h>
 #include <clc/relational/clc_isinf.h>
 
 #define FUNCTION isinf
diff --git a/libclc/opencl/lib/generic/relational/isless.cl b/libclc/opencl/lib/generic/relational/isless.cl
index e1b931aede695..0af1f53e71042 100644
--- a/libclc/opencl/lib/generic/relational/isless.cl
+++ b/libclc/opencl/lib/generic/relational/isless.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isless.h>
 #include <clc/relational/clc_isless.h>
 
 #define FUNCTION isless
diff --git a/libclc/opencl/lib/generic/relational/islessequal.cl b/libclc/opencl/lib/generic/relational/islessequal.cl
index 91d0891e21223..9e32afc718ab2 100644
--- a/libclc/opencl/lib/generic/relational/islessequal.cl
+++ b/libclc/opencl/lib/generic/relational/islessequal.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/islessequal.h>
 #include <clc/relational/clc_islessequal.h>
 
 #define FUNCTION islessequal
diff --git a/libclc/opencl/lib/generic/relational/islessgreater.cl b/libclc/opencl/lib/generic/relational/islessgreater.cl
index dc519cc602752..c36a857dc3dfc 100644
--- a/libclc/opencl/lib/generic/relational/islessgreater.cl
+++ b/libclc/opencl/lib/generic/relational/islessgreater.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/islessgreater.h>
 #include <clc/relational/clc_islessgreater.h>
 
 #define FUNCTION islessgreater
diff --git a/libclc/opencl/lib/generic/relational/isnan.cl b/libclc/opencl/lib/generic/relational/isnan.cl
index 6761851caffe4..8b03930c5312f 100644
--- a/libclc/opencl/lib/generic/relational/isnan.cl
+++ b/libclc/opencl/lib/generic/relational/isnan.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isnan.h>
 #include <clc/relational/clc_isnan.h>
 
 #define FUNCTION isnan
diff --git a/libclc/opencl/lib/generic/relational/isnormal.cl b/libclc/opencl/lib/generic/relational/isnormal.cl
index 42f3e1f4f4349..4ba21cc3e17fc 100644
--- a/libclc/opencl/lib/generic/relational/isnormal.cl
+++ b/libclc/opencl/lib/generic/relational/isnormal.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isnormal.h>
 #include <clc/relational/clc_isnormal.h>
 
 #define FUNCTION isnormal
diff --git a/libclc/opencl/lib/generic/relational/isnotequal.cl b/libclc/opencl/lib/generic/relational/isnotequal.cl
index f28317b8fe0f5..928923b9b2a5e 100644
--- a/libclc/opencl/lib/generic/relational/isnotequal.cl
+++ b/libclc/opencl/lib/generic/relational/isnotequal.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isnotequal.h>
 #include <clc/relational/clc_isnotequal.h>
 
 #define FUNCTION isnotequal
diff --git a/libclc/opencl/lib/generic/relational/isordered.cl b/libclc/opencl/lib/generic/relational/isordered.cl
index 83117765d5261..60ca4d67ff1ea 100644
--- a/libclc/opencl/lib/generic/relational/isordered.cl
+++ b/libclc/opencl/lib/generic/relational/isordered.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isordered.h>
 #include <clc/relational/clc_isordered.h>
 
 #define FUNCTION isordered
diff --git a/libclc/opencl/lib/generic/relational/isunordered.cl b/libclc/opencl/lib/generic/relational/isunordered.cl
index 26deb195a7406..3392d77856ced 100644
--- a/libclc/opencl/lib/generic/relational/isunordered.cl
+++ b/libclc/opencl/lib/generic/relational/isunordered.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/isunordered.h>
 #include <clc/relational/clc_isunordered.h>
 
 #define FUNCTION isunordered
diff --git a/libclc/opencl/lib/generic/relational/select.cl b/libclc/opencl/lib/generic/relational/select.cl
index c29109b1d3c2c..47a7b8e5ad114 100644
--- a/libclc/opencl/lib/generic/relational/select.cl
+++ b/libclc/opencl/lib/generic/relational/select.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/select.h>
 #include <clc/relational/clc_select.h>
 #include <clc/utils.h>
 
diff --git a/libclc/opencl/lib/generic/relational/signbit.cl b/libclc/opencl/lib/generic/relational/signbit.cl
index 2b58922bf8705..26feb8d43fa25 100644
--- a/libclc/opencl/lib/generic/relational/signbit.cl
+++ b/libclc/opencl/lib/generic/relational/signbit.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/relational/signbit.h>
 #include <clc/relational/clc_signbit.h>
 
 #define FUNCTION signbit
diff --git a/libclc/opencl/lib/generic/relational/unary_def.inc b/libclc/opencl/lib/generic/relational/unary_def.inc
index 80dea211bb30f..47bb33ef2da3d 100644
--- a/libclc/opencl/lib/generic/relational/unary_def.inc
+++ b/libclc/opencl/lib/generic/relational/unary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 
 _CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_FLOATN a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
diff --git a/libclc/opencl/lib/generic/shared/clamp.cl b/libclc/opencl/lib/generic/shared/clamp.cl
index 4d00f6f8c3a68..6e2bedc0704ff 100644
--- a/libclc/opencl/lib/generic/shared/clamp.cl
+++ b/libclc/opencl/lib/generic/shared/clamp.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/shared/clamp.h>
 #include <clc/shared/clc_clamp.h>
 
 #define __CLC_BODY <clamp.inc>
diff --git a/libclc/opencl/lib/generic/shared/max.cl b/libclc/opencl/lib/generic/shared/max.cl
index 59f0b7e3b4c8e..5cb5ff124bd01 100644
--- a/libclc/opencl/lib/generic/shared/max.cl
+++ b/libclc/opencl/lib/generic/shared/max.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/shared/max.h>
 #include <clc/shared/clc_max.h>
 
 #define __CLC_BODY <max.inc>
diff --git a/libclc/opencl/lib/generic/shared/min.cl b/libclc/opencl/lib/generic/shared/min.cl
index 6c491c5eb1cbf..2a1ded3aa0288 100644
--- a/libclc/opencl/lib/generic/shared/min.cl
+++ b/libclc/opencl/lib/generic/shared/min.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/shared/min.h>
 #include <clc/shared/clc_min.h>
 
 #define __CLC_BODY <min.inc>
diff --git a/libclc/opencl/lib/generic/shared/vload.cl b/libclc/opencl/lib/generic/shared/vload.cl
index ad22839580132..6f1ca06ccb8be 100644
--- a/libclc/opencl/lib/generic/shared/vload.cl
+++ b/libclc/opencl/lib/generic/shared/vload.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/shared/vload.h>
 #include <clc/shared/clc_vload.h>
 
 #define __CLC_BODY "vload.inc"
diff --git a/libclc/opencl/lib/generic/shared/vstore.cl b/libclc/opencl/lib/generic/shared/vstore.cl
index 145658f873dc5..1d62de810ff75 100644
--- a/libclc/opencl/lib/generic/shared/vstore.cl
+++ b/libclc/opencl/lib/generic/shared/vstore.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/shared/vstore.h>
 #include <clc/shared/clc_vstore.h>
 
 #define __CLC_BODY "vstore.inc"
diff --git a/libclc/opencl/lib/generic/subnormal_config.cl b/libclc/opencl/lib/generic/subnormal_config.cl
index a32af9b0cf2e7..aa2e30935e5f0 100644
--- a/libclc/opencl/lib/generic/subnormal_config.cl
+++ b/libclc/opencl/lib/generic/subnormal_config.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_subnormal_config.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/opencl-base.h>
 
 _CLC_DEF bool __clc_fp16_subnormals_supported() { return false; }
 
diff --git a/libclc/opencl/lib/generic/workitem/get_global_id.cl b/libclc/opencl/lib/generic/workitem/get_global_id.cl
index 26c3bf528cd4d..5e417bef0d2bd 100644
--- a/libclc/opencl/lib/generic/workitem/get_global_id.cl
+++ b/libclc/opencl/lib/generic/workitem/get_global_id.cl
@@ -6,7 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_id.h>
+#include <clc/opencl/workitem/get_global_offset.h>
+#include <clc/opencl/workitem/get_group_id.h>
+#include <clc/opencl/workitem/get_local_id.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_global_id(uint dim) {
   return get_group_id(dim) * get_local_size(dim) + get_local_id(dim) +
diff --git a/libclc/opencl/lib/generic/workitem/get_global_size.cl b/libclc/opencl/lib/generic/workitem/get_global_size.cl
index 747115d524885..d62e9e91888ac 100644
--- a/libclc/opencl/lib/generic/workitem/get_global_size.cl
+++ b/libclc/opencl/lib/generic/workitem/get_global_size.cl
@@ -6,7 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_size.h>
+#include <clc/opencl/workitem/get_local_size.h>
+#include <clc/opencl/workitem/get_num_groups.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_global_size(uint dim) {
   return get_num_groups(dim) * get_local_size(dim);
diff --git a/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl b/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl
index 284a0517d3308..d24569ecda1bc 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/mem_fence/fence.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/explicit_fence/explicit_memory_fence.h>
 
 _CLC_DEF _CLC_OVERLOAD void mem_fence(cl_mem_fence_flags flags) {
   if (flags & (CLK_GLOBAL_MEM_FENCE | CLK_LOCAL_MEM_FENCE))
diff --git a/libclc/opencl/lib/ptx-nvidiacl/synchronization/barrier.cl b/libclc/opencl/lib/ptx-nvidiacl/synchronization/barrier.cl
index 173d771ccaa39..7c57478795dda 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/synchronization/barrier.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/synchronization/barrier.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/synchronization/barrier.h>
 
 _CLC_DEF _CLC_OVERLOAD void barrier(cl_mem_fence_flags flags) {
   __syncthreads();
diff --git a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_global_id.cl b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_global_id.cl
index 2f1f7cd4250fe..bcfddbe70cb63 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_global_id.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_global_id.cl
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_id.h>
+#include <clc/opencl/workitem/get_group_id.h>
+#include <clc/opencl/workitem/get_local_id.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_global_id(uint dim) {
   return get_group_id(dim) * get_local_size(dim) + get_local_id(dim);
diff --git a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_group_id.cl b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_group_id.cl
index 0dad4c2061fe6..7b3e463fd423a 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_group_id.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_group_id.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_group_id.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_group_id(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_id.cl b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_id.cl
index 199b4610bdb7b..b5133848e9a2e 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_id.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_id.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_id.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_local_id(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_size.cl b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_size.cl
index a93fa0a3c9649..814ac462a2bc2 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_size.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_local_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_local_size(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_num_groups.cl b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_num_groups.cl
index 4c934968df865..e9e4e4d06930a 100644
--- a/libclc/opencl/lib/ptx-nvidiacl/workitem/get_num_groups.cl
+++ b/libclc/opencl/lib/ptx-nvidiacl/workitem/get_num_groups.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_num_groups.h>
 
 _CLC_DEF _CLC_OVERLOAD size_t get_num_groups(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/r600/image/get_image_channel_data_type.cl b/libclc/opencl/lib/r600/image/get_image_channel_data_type.cl
index 813572ae5d2d7..f28b7d458ca4f 100644
--- a/libclc/opencl/lib/r600/image/get_image_channel_data_type.cl
+++ b/libclc/opencl/lib/r600/image/get_image_channel_data_type.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL int __clc_get_image_channel_data_type_2d(image2d_t);
 _CLC_DECL int __clc_get_image_channel_data_type_3d(image3d_t);
diff --git a/libclc/opencl/lib/r600/image/get_image_channel_order.cl b/libclc/opencl/lib/r600/image/get_image_channel_order.cl
index 0e5882b155079..da6d783e7f01a 100644
--- a/libclc/opencl/lib/r600/image/get_image_channel_order.cl
+++ b/libclc/opencl/lib/r600/image/get_image_channel_order.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL int __clc_get_image_channel_order_2d(image2d_t);
 _CLC_DECL int __clc_get_image_channel_order_3d(image3d_t);
diff --git a/libclc/opencl/lib/r600/image/get_image_depth.cl b/libclc/opencl/lib/r600/image/get_image_depth.cl
index 431889b6ca258..1731d85dc4c3d 100644
--- a/libclc/opencl/lib/r600/image/get_image_depth.cl
+++ b/libclc/opencl/lib/r600/image/get_image_depth.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL int __clc_get_image_depth_3d(image3d_t);
 
diff --git a/libclc/opencl/lib/r600/image/get_image_dim.cl b/libclc/opencl/lib/r600/image/get_image_dim.cl
index 5a123425a2910..6929d12ad1a5a 100644
--- a/libclc/opencl/lib/r600/image/get_image_dim.cl
+++ b/libclc/opencl/lib/r600/image/get_image_dim.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_OVERLOAD _CLC_DEF int2 get_image_dim(image2d_t image) {
   return (int2)(get_image_width(image), get_image_height(image));
diff --git a/libclc/opencl/lib/r600/image/get_image_height.cl b/libclc/opencl/lib/r600/image/get_image_height.cl
index 0c498bed8e608..14a21823a826d 100644
--- a/libclc/opencl/lib/r600/image/get_image_height.cl
+++ b/libclc/opencl/lib/r600/image/get_image_height.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL int __clc_get_image_height_2d(image2d_t);
 _CLC_DECL int __clc_get_image_height_3d(image3d_t);
diff --git a/libclc/opencl/lib/r600/image/get_image_width.cl b/libclc/opencl/lib/r600/image/get_image_width.cl
index c54f66176e764..fa54ec3628f2d 100644
--- a/libclc/opencl/lib/r600/image/get_image_width.cl
+++ b/libclc/opencl/lib/r600/image/get_image_width.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL int __clc_get_image_width_2d(image2d_t);
 _CLC_DECL int __clc_get_image_width_3d(image3d_t);
diff --git a/libclc/opencl/lib/r600/image/read_imagef.cl b/libclc/opencl/lib/r600/image/read_imagef.cl
index 54983363e33c5..fc597b762e52e 100644
--- a/libclc/opencl/lib/r600/image/read_imagef.cl
+++ b/libclc/opencl/lib/r600/image/read_imagef.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL float4 __clc_read_imagef_tex(image2d_t, sampler_t, float2);
 
diff --git a/libclc/opencl/lib/r600/image/read_imagei.cl b/libclc/opencl/lib/r600/image/read_imagei.cl
index 918a444f90a11..bc54a631eabb5 100644
--- a/libclc/opencl/lib/r600/image/read_imagei.cl
+++ b/libclc/opencl/lib/r600/image/read_imagei.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL float4 __clc_read_imagef_tex(image2d_t, sampler_t, float2);
 
diff --git a/libclc/opencl/lib/r600/image/read_imageui.cl b/libclc/opencl/lib/r600/image/read_imageui.cl
index 8236b542ea0e8..8c266b9f2f854 100644
--- a/libclc/opencl/lib/r600/image/read_imageui.cl
+++ b/libclc/opencl/lib/r600/image/read_imageui.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL float4 __clc_read_imagef_tex(image2d_t, sampler_t, float2);
 
diff --git a/libclc/opencl/lib/r600/image/write_imagef.cl b/libclc/opencl/lib/r600/image/write_imagef.cl
index d9784f5322c94..0e828acad0823 100644
--- a/libclc/opencl/lib/r600/image/write_imagef.cl
+++ b/libclc/opencl/lib/r600/image/write_imagef.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL void __clc_write_imagef_2d(image2d_t image, int2 coord, float4 color);
 
diff --git a/libclc/opencl/lib/r600/image/write_imagei.cl b/libclc/opencl/lib/r600/image/write_imagei.cl
index ced65184071b3..88ea35634abcd 100644
--- a/libclc/opencl/lib/r600/image/write_imagei.cl
+++ b/libclc/opencl/lib/r600/image/write_imagei.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL void __clc_write_imagei_2d(image2d_t image, int2 coord, int4 color);
 
diff --git a/libclc/opencl/lib/r600/image/write_imageui.cl b/libclc/opencl/lib/r600/image/write_imageui.cl
index ef2ee5a835ec7..40f3f33345836 100644
--- a/libclc/opencl/lib/r600/image/write_imageui.cl
+++ b/libclc/opencl/lib/r600/image/write_imageui.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/image/image.h>
 
 _CLC_DECL void __clc_write_imageui_2d(image2d_t image, int2 coord, uint4 color);
 
diff --git a/libclc/opencl/lib/r600/synchronization/barrier.cl b/libclc/opencl/lib/r600/synchronization/barrier.cl
index 83438efa26498..69e87d9497fde 100644
--- a/libclc/opencl/lib/r600/synchronization/barrier.cl
+++ b/libclc/opencl/lib/r600/synchronization/barrier.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/explicit_fence/explicit_memory_fence.h>
 
 _CLC_DEF void __clc_r600_barrier(void) __asm("llvm.r600.group.barrier");
 
diff --git a/libclc/opencl/lib/r600/workitem/get_global_offset.cl b/libclc/opencl/lib/r600/workitem/get_global_offset.cl
index cc42fe4beef0c..988e482de2391 100644
--- a/libclc/opencl/lib/r600/workitem/get_global_offset.cl
+++ b/libclc/opencl/lib/r600/workitem/get_global_offset.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_offset.h>
 
 _CLC_DEF _CLC_OVERLOAD uint get_global_offset(uint dim) {
   __attribute__((address_space(7))) uint *ptr =
diff --git a/libclc/opencl/lib/r600/workitem/get_global_size.cl b/libclc/opencl/lib/r600/workitem/get_global_size.cl
index 1c93568f80c74..298487f87dc1e 100644
--- a/libclc/opencl/lib/r600/workitem/get_global_size.cl
+++ b/libclc/opencl/lib/r600/workitem/get_global_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_global_size.h>
 
 uint __clc_r600_get_global_size_x(void) __asm("llvm.r600.read.global.size.x");
 uint __clc_r600_get_global_size_y(void) __asm("llvm.r600.read.global.size.y");
diff --git a/libclc/opencl/lib/r600/workitem/get_group_id.cl b/libclc/opencl/lib/r600/workitem/get_group_id.cl
index c771e835cda12..196f0f21241e8 100644
--- a/libclc/opencl/lib/r600/workitem/get_group_id.cl
+++ b/libclc/opencl/lib/r600/workitem/get_group_id.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_group_id.h>
 
 _CLC_DEF _CLC_OVERLOAD uint get_group_id(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/r600/workitem/get_local_id.cl b/libclc/opencl/lib/r600/workitem/get_local_id.cl
index 9d8e044812f24..1321e1e8362e2 100644
--- a/libclc/opencl/lib/r600/workitem/get_local_id.cl
+++ b/libclc/opencl/lib/r600/workitem/get_local_id.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_id.h>
 
 _CLC_DEF _CLC_OVERLOAD uint get_local_id(uint dim) {
   switch (dim) {
diff --git a/libclc/opencl/lib/r600/workitem/get_local_size.cl b/libclc/opencl/lib/r600/workitem/get_local_size.cl
index 3755cf5e8b0e6..c0e4e15542769 100644
--- a/libclc/opencl/lib/r600/workitem/get_local_size.cl
+++ b/libclc/opencl/lib/r600/workitem/get_local_size.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_local_size.h>
 
 uint __clc_r600_get_local_size_x(void) __asm("llvm.r600.read.local.size.x");
 uint __clc_r600_get_local_size_y(void) __asm("llvm.r600.read.local.size.y");
diff --git a/libclc/opencl/lib/r600/workitem/get_num_groups.cl b/libclc/opencl/lib/r600/workitem/get_num_groups.cl
index 088a055aabfce..098a9cb86a6dc 100644
--- a/libclc/opencl/lib/r600/workitem/get_num_groups.cl
+++ b/libclc/opencl/lib/r600/workitem/get_num_groups.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_num_groups.h>
 
 uint __clc_r600_get_num_groups_x(void) __asm("llvm.r600.read.ngroups.x");
 uint __clc_r600_get_num_groups_y(void) __asm("llvm.r600.read.ngroups.y");
diff --git a/libclc/opencl/lib/r600/workitem/get_work_dim.cl b/libclc/opencl/lib/r600/workitem/get_work_dim.cl
index b59dd3af6c334..a7917b864e407 100644
--- a/libclc/opencl/lib/r600/workitem/get_work_dim.cl
+++ b/libclc/opencl/lib/r600/workitem/get_work_dim.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/opencl/clc.h>
+#include <clc/opencl/workitem/get_work_dim.h>
 
 _CLC_DEF _CLC_OVERLOAD uint get_work_dim(void) {
   __attribute__((address_space(7))) uint *ptr =
diff --git a/libclc/opencl/lib/spirv/math/fma.cl b/libclc/opencl/lib/spirv/math/fma.cl
index 0f3141a0e09ee..83504f63772ca 100644
--- a/libclc/opencl/lib/spirv/math/fma.cl
+++ b/libclc/opencl/lib/spirv/math/fma.cl
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/internal/math/clc_sw_fma.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/math/fma.h>
 
 #define __FLOAT_ONLY
 #define FUNCTION fma
-#define __CLC_FUNCTION(x) __clc_sw_fma
+#define __IMPL_FUNCTION(x) __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/spirv/subnormal_config.cl b/libclc/opencl/lib/spirv/subnormal_config.cl
index 77d60ab9cbbde..114aabb2e9435 100644
--- a/libclc/opencl/lib/spirv/subnormal_config.cl
+++ b/libclc/opencl/lib/spirv/subnormal_config.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/math/clc_subnormal_config.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/opencl-base.h>
 
 _CLC_DEF bool __clc_fp16_subnormals_supported() { return false; }
 
diff --git a/libclc/utils/gen_convert.py b/libclc/utils/gen_convert.py
index 02893dbad6a37..737ab9d1a5cd6 100644
--- a/libclc/utils/gen_convert.py
+++ b/libclc/utils/gen_convert.py
@@ -167,7 +167,7 @@ def conditional_guard(src, dst):
 nl = "\n"
 includes = []
 if not clc:
-    includes = ["<clc/opencl/clc.h>"]
+    includes = ["<clc/opencl/convert.h>"]
 else:
     includes = sorted(
         [
diff --git a/libcxx/.clang-format b/libcxx/.clang-format
index fe2f2ef0119fa..f372ac9619997 100644
--- a/libcxx/.clang-format
+++ b/libcxx/.clang-format
@@ -16,8 +16,11 @@ AllowShortLambdasOnASingleLine: All
 AttributeMacros: [
                   '_ALIGNAS_TYPE',
                   '_ALIGNAS',
+                  '_LIBCPP_ACQUIRE_CAPABILITY',
+                  '_LIBCPP_ACQUIRE_SHARED_CAPABILITY',
                   '_LIBCPP_ALIGNOF',
                   '_LIBCPP_ALWAYS_INLINE',
+                  '_LIBCPP_CAPABILITY',
                   '_LIBCPP_CONSTEXPR_SINCE_CXX14',
                   '_LIBCPP_CONSTEXPR_SINCE_CXX17',
                   '_LIBCPP_CONSTEXPR_SINCE_CXX20',
@@ -41,9 +44,13 @@ AttributeMacros: [
                   '_LIBCPP_NO_UNIQUE_ADDRESS',
                   '_LIBCPP_NOALIAS',
                   '_LIBCPP_OVERRIDABLE_FUNC_VIS',
+                  '_LIBCPP_RELEASE_CAPABILITY',
+                  '_LIBCPP_REQUIRES_CAPABILITY',
+                  '_LIBCPP_SCOPED_LOCKABLE',
                   '_LIBCPP_STANDALONE_DEBUG',
                   '_LIBCPP_TEMPLATE_DATA_VIS',
-                  '_LIBCPP_THREAD_SAFETY_ANNOTATION',
+                  '_LIBCPP_TRY_ACQUIRE_CAPABILITY',
+                  '_LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY',
                   '_LIBCPP_USING_IF_EXISTS',
                   '_LIBCPP_WEAK',
                  ]
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 3c635e5e46bbd..61805726a4ff0 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -198,6 +198,10 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_char8_t``                                      ``201907L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_common_reference``                             ``202302L``
+    ---------------------------------------------------------- -----------------
+    ``__cpp_lib_common_reference_wrapper``                     ``202302L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_concepts``                                     ``202002L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_algorithms``                         ``201806L``
diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index c52bc54f412bd..2cb560b3852f3 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -47,10 +47,11 @@ Implemented Papers
 - P1222R4: A Standard ``flat_set`` (`Github <https://github.com/llvm/llvm-project/issues/105193>`__)
 - P2897R7: ``aligned_accessor``: An mdspan accessor expressing pointer over-alignment (`Github <https://github.com/llvm/llvm-project/issues/118372>`__)
 - P3247R2: Deprecate the notion of trivial types (`Github <https://github.com/llvm/llvm-project/issues/118387>`__)
-- P3372R3: ``constexpr`` containers and adaptors (`Github <https://github.com/llvm/llvm-project/issues/128673>`__) (Only ``constexpr flat_map`` is implemented)
+- P3372R3: ``constexpr`` containers and adaptors (`Github <https://github.com/llvm/llvm-project/issues/127876>`__) (``forward_list``, ``list``, ``priority_queue``, ``flat_map``, and ``flat_set`` are implemented)
 - P2441R2: ``views::join_with`` (`Github <https://github.com/llvm/llvm-project/issues/105185>`__)
 - P2711R1: Making multi-param constructors of ``views`` ``explicit`` (`Github <https://github.com/llvm/llvm-project/issues/105252>`__)
 - P2770R0: Stashing stashing ``iterators`` for proper flattening (`Github <https://github.com/llvm/llvm-project/issues/105250>`__)
+- P2655R3: ``common_reference_t`` of ``reference_wrapper`` Should Be a Reference Type (`Github <https://github.com/llvm/llvm-project/issues/105260>`__)
 
 Improvements and New Features
 -----------------------------
@@ -81,6 +82,9 @@ Improvements and New Features
 - The ``bitset::to_string`` function has been optimized, resulting in a performance improvement of up to 8.3x for bitsets
   with uniformly distributed zeros and ones, and up to 13.5x and 16.1x for sparse and dense bitsets, respectively.
 
+- The ``flat_map::insert`` and ``flat_set::insert_range`` have been optimized, resulting in a performance improvement of up
+  to 10x for inserting elements into a ``flat_map`` when the input range is a ``flat_map`` or a ``zip_view``.
+
 Deprecations and Removals
 -------------------------
 
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 574675175a4cf..e4fa07d82289d 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -113,7 +113,7 @@
 "`P2693R1 <https://wg21.link/P2693R1>`__","Formatting ``thread::id`` and ``stacktrace``","2023-02 (Issaquah)","|Partial|","","The formatter for ``stacktrace`` is not implemented, since ``stacktrace`` is not implemented yet"
 "`P2679R2 <https://wg21.link/P2679R2>`__","Fixing ``std::start_lifetime_as`` for arrays","2023-02 (Issaquah)","","",""
 "`P2674R1 <https://wg21.link/P2674R1>`__","A trait for implicit lifetime types","2023-02 (Issaquah)","|Complete|","20",""
-"`P2655R3 <https://wg21.link/P2655R3>`__","``common_reference_t`` of ``reference_wrapper`` Should Be a Reference Type","2023-02 (Issaquah)","","",""
+"`P2655R3 <https://wg21.link/P2655R3>`__","``common_reference_t`` of ``reference_wrapper`` Should Be a Reference Type","2023-02 (Issaquah)","|Complete|","21","The paper is implemented as a DR to C++20"
 "`P2652R2 <https://wg21.link/P2652R2>`__","Disallow User Specialization of ``allocator_traits``","2023-02 (Issaquah)","|Complete|","19",""
 "`P2787R1 <https://wg21.link/P2787R1>`__","``pmr::generator`` - Promise Types are not Values","2023-02 (Issaquah)","","",""
 "`P2614R2 <https://wg21.link/P2614R2>`__","Deprecate ``numeric_limits::has_denorm``","2023-02 (Issaquah)","|Complete|","18",""
diff --git a/libcxx/docs/UserDocumentation.rst b/libcxx/docs/UserDocumentation.rst
index f82554a4869fa..79f5908bdc6b6 100644
--- a/libcxx/docs/UserDocumentation.rst
+++ b/libcxx/docs/UserDocumentation.rst
@@ -114,11 +114,6 @@ enable or disable extended libc++ behavior.
   ensure that the appropriate experimental library (usually ``libc++experimental.a``)
   is linked into their program.
 
-**_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS**:
-  This macro is used to enable -Wthread-safety annotations on libc++'s
-  ``std::mutex`` and ``std::lock_guard``. By default, these annotations are
-  disabled and must be manually enabled by the user.
-
 **_LIBCPP_HARDENING_MODE**:
   This macro is used to choose the :ref:`hardening mode <using-hardening-modes>`.
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index e386f31386b60..248aed666e27e 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -498,6 +498,7 @@ set(files
   __iterator/ostreambuf_iterator.h
   __iterator/permutable.h
   __iterator/prev.h
+  __iterator/product_iterator.h
   __iterator/projected.h
   __iterator/ranges_iterator_traits.h
   __iterator/readable_traits.h
@@ -510,13 +511,19 @@ set(files
   __iterator/unreachable_sentinel.h
   __iterator/wrap_iter.h
   __locale
+  __locale_dir/check_grouping.h
+  __locale_dir/get_c_locale.h
   __locale_dir/locale_base_api.h
   __locale_dir/locale_base_api/android.h
   __locale_dir/locale_base_api/bsd_locale_fallbacks.h
   __locale_dir/locale_base_api/ibm.h
   __locale_dir/locale_base_api/musl.h
   __locale_dir/locale_base_api/openbsd.h
+  __locale_dir/messages.h
+  __locale_dir/money.h
+  __locale_dir/num.h
   __locale_dir/pad_and_output.h
+  __locale_dir/scan_keyword.h
   __locale_dir/support/apple.h
   __locale_dir/support/bsd_like.h
   __locale_dir/support/freebsd.h
@@ -525,6 +532,9 @@ set(files
   __locale_dir/support/no_locale/characters.h
   __locale_dir/support/no_locale/strtonum.h
   __locale_dir/support/windows.h
+  __locale_dir/time.h
+  __locale_dir/wbuffer_convert.h
+  __locale_dir/wstring_convert.h
   __math/abs.h
   __math/copysign.h
   __math/error_functions.h
@@ -777,9 +787,8 @@ set(files
   __tuple/tuple_size.h
   __tuple/tuple_types.h
   __type_traits/add_cv_quals.h
-  __type_traits/add_lvalue_reference.h
   __type_traits/add_pointer.h
-  __type_traits/add_rvalue_reference.h
+  __type_traits/add_reference.h
   __type_traits/aligned_storage.h
   __type_traits/aligned_union.h
   __type_traits/alignment_of.h
diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h
index 86b606d27847b..49758397f6185 100644
--- a/libcxx/include/__chrono/formatter.h
+++ b/libcxx/include/__chrono/formatter.h
@@ -50,12 +50,14 @@
 #  include <__format/formatter.h>
 #  include <__format/parser_std_format_spec.h>
 #  include <__format/write_escaped.h>
+#  include <__iterator/istreambuf_iterator.h>
+#  include <__iterator/ostreambuf_iterator.h>
+#  include <__locale_dir/time.h>
 #  include <__memory/addressof.h>
 #  include <__type_traits/is_specialization.h>
 #  include <cmath>
 #  include <ctime>
 #  include <limits>
-#  include <locale>
 #  include <sstream>
 #  include <string_view>
 
diff --git a/libcxx/include/__concepts/common_with.h b/libcxx/include/__concepts/common_with.h
index 85abb05efbc29..09082b2295def 100644
--- a/libcxx/include/__concepts/common_with.h
+++ b/libcxx/include/__concepts/common_with.h
@@ -12,7 +12,7 @@
 #include <__concepts/common_reference_with.h>
 #include <__concepts/same_as.h>
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/common_reference.h>
 #include <__type_traits/common_type.h>
 #include <__utility/declval.h>
diff --git a/libcxx/include/__config b/libcxx/include/__config
index af8a297fdf3fd..58ca9bab937e4 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -903,23 +903,6 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS
 #  endif
 
-// Work around the attribute handling in clang.  When both __declspec and
-// __attribute__ are present, the processing goes awry preventing the definition
-// of the types. In MinGW mode, __declspec evaluates to __attribute__, and thus
-// combining the two does work.
-#  if defined(_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS) && defined(__clang__) &&                                       \
-      __has_attribute(acquire_capability) && !defined(_MSC_VER)
-#    define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS 1
-#  else
-#    define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS 0
-#  endif
-
-#  if _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
-#    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x) __attribute__((x))
-#  else
-#    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x)
-#  endif
-
 #  if _LIBCPP_STD_VER >= 20
 #    define _LIBCPP_CONSTINIT constinit
 #  elif __has_attribute(__require_constant_initialization__)
@@ -1147,6 +1130,60 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_PREFERRED_NAME(x)
 #  endif
 
+#  if __has_cpp_attribute(_Clang::__scoped_lockable__)
+#    define _LIBCPP_SCOPED_LOCKABLE [[_Clang::__scoped_lockable__]]
+#  else
+#    define _LIBCPP_SCOPED_LOCKABLE
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__capability__)
+#    define _LIBCPP_CAPABILITY(...) [[_Clang::__capability__(__VA_ARGS__)]]
+#  else
+#    define _LIBCPP_CAPABILITY(...)
+#  endif
+
+#  if __has_attribute(__acquire_capability__)
+#    define _LIBCPP_ACQUIRE_CAPABILITY(...) __attribute__((__acquire_capability__(__VA_ARGS__)))
+#  else
+#    define _LIBCPP_ACQUIRE_CAPABILITY(...)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__try_acquire_capability__)
+#    define _LIBCPP_TRY_ACQUIRE_CAPABILITY(...) [[_Clang::__try_acquire_capability__(__VA_ARGS__)]]
+#  else
+#    define _LIBCPP_TRY_ACQUIRE_CAPABILITY(...)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__acquire_shared_capability__)
+#    define _LIBCPP_ACQUIRE_SHARED_CAPABILITY [[_Clang::__acquire_shared_capability__]]
+#  else
+#    define _LIBCPP_ACQUIRE_SHARED_CAPABILITY
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__try_acquire_shared_capability__)
+#    define _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(...) [[_Clang::__try_acquire_shared_capability__(__VA_ARGS__)]]
+#  else
+#    define _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(...)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__release_capability__)
+#    define _LIBCPP_RELEASE_CAPABILITY [[_Clang::__release_capability__]]
+#  else
+#    define _LIBCPP_RELEASE_CAPABILITY
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__release_shared_capability__)
+#    define _LIBCPP_RELEASE_SHARED_CAPABILITY [[_Clang::__release_shared_capability__]]
+#  else
+#    define _LIBCPP_RELEASE_SHARED_CAPABILITY
+#  endif
+
+#  if __has_attribute(__requires_capability__)
+#    define _LIBCPP_REQUIRES_CAPABILITY(...) __attribute__((__requires_capability__(__VA_ARGS__)))
+#  else
+#    define _LIBCPP_REQUIRES_CAPABILITY(...)
+#  endif
+
 #  if defined(_LIBCPP_ABI_MICROSOFT) && __has_declspec_attribute(empty_bases)
 #    define _LIBCPP_DECLSPEC_EMPTY_BASES __declspec(empty_bases)
 #  else
diff --git a/libcxx/include/__coroutine/coroutine_handle.h b/libcxx/include/__coroutine/coroutine_handle.h
index 88848544cf6bd..b7add258510eb 100644
--- a/libcxx/include/__coroutine/coroutine_handle.h
+++ b/libcxx/include/__coroutine/coroutine_handle.h
@@ -172,6 +172,6 @@ struct hash<coroutine_handle<_Tp>> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__coroutine/coroutine_traits.h b/libcxx/include/__coroutine/coroutine_traits.h
index 78f05341f7486..60f7ca2311c6b 100644
--- a/libcxx/include/__coroutine/coroutine_traits.h
+++ b/libcxx/include/__coroutine/coroutine_traits.h
@@ -43,6 +43,6 @@ struct coroutine_traits : public __coroutine_traits_sfinae<_Ret> {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
diff --git a/libcxx/include/__coroutine/noop_coroutine_handle.h b/libcxx/include/__coroutine/noop_coroutine_handle.h
index c95d85d5484c4..2b2838b6bf49b 100644
--- a/libcxx/include/__coroutine/noop_coroutine_handle.h
+++ b/libcxx/include/__coroutine/noop_coroutine_handle.h
@@ -94,6 +94,6 @@ inline _LIBCPP_HIDE_FROM_ABI noop_coroutine_handle noop_coroutine() noexcept { r
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__coroutine/trivial_awaitables.h b/libcxx/include/__coroutine/trivial_awaitables.h
index b604bd3c2d8ad..7cb7f4dfeb78c 100644
--- a/libcxx/include/__coroutine/trivial_awaitables.h
+++ b/libcxx/include/__coroutine/trivial_awaitables.h
@@ -35,6 +35,6 @@ struct suspend_always {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
diff --git a/libcxx/include/__exception/exception_ptr.h b/libcxx/include/__exception/exception_ptr.h
index b1fe9a1299ec7..796fa924be121 100644
--- a/libcxx/include/__exception/exception_ptr.h
+++ b/libcxx/include/__exception/exception_ptr.h
@@ -15,6 +15,7 @@
 #include <__memory/addressof.h>
 #include <__memory/construct_at.h>
 #include <__type_traits/decay.h>
+#include <__type_traits/is_pointer.h>
 #include <cstdlib>
 #include <typeinfo>
 
@@ -62,7 +63,7 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   static exception_ptr __from_native_exception_pointer(void*) _NOEXCEPT;
 
   template <class _Ep>
-  friend _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT;
+  friend _LIBCPP_HIDE_FROM_ABI exception_ptr __make_exception_ptr_explicit(_Ep&) _NOEXCEPT;
 
 public:
   // exception_ptr is basically a COW string so it is trivially relocatable.
@@ -91,25 +92,21 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
   friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
 };
 
-template <class _Ep>
-_LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
 #  if _LIBCPP_HAS_EXCEPTIONS
-#    if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION && __cplusplus >= 201103L
+#    if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI exception_ptr __make_exception_ptr_explicit(_Ep& __e) _NOEXCEPT {
   using _Ep2 = __decay_t<_Ep>;
-
   void* __ex = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ep));
 #      ifdef __wasm__
-  // In Wasm, a destructor returns its argument
-  (void)__cxxabiv1::__cxa_init_primary_exception(
-      __ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) -> void* {
+  auto __cleanup = [](void* __p) -> void* {
+    std::__destroy_at(static_cast<_Ep2*>(__p));
+    return __p;
+  };
 #      else
-  (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), [](void* __p) {
-#      endif
-        std::__destroy_at(static_cast<_Ep2*>(__p));
-#      ifdef __wasm__
-        return __p;
+  auto __cleanup = [](void* __p) { std::__destroy_at(static_cast<_Ep2*>(__p)); };
 #      endif
-      });
+  (void)__cxxabiv1::__cxa_init_primary_exception(__ex, const_cast<std::type_info*>(&typeid(_Ep)), __cleanup);
 
   try {
     ::new (__ex) _Ep2(__e);
@@ -118,18 +115,47 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
     __cxxabiv1::__cxa_free_exception(__ex);
     return current_exception();
   }
-#    else
+}
+#    endif
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI exception_ptr __make_exception_ptr_via_throw(_Ep& __e) _NOEXCEPT {
   try {
     throw __e;
   } catch (...) {
     return current_exception();
   }
+}
+
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
+  // Objective-C exceptions are thrown via pointer. When throwing an Objective-C exception,
+  // Clang generates a call to `objc_exception_throw` instead of the usual `__cxa_throw`.
+  // That function creates an exception with a special Objective-C typeinfo instead of
+  // the usual C++ typeinfo, since that is needed to implement the behavior documented
+  // at [1]).
+  //
+  // Because of this special behavior, we can't create an exception via `__cxa_init_primary_exception`
+  // for Objective-C exceptions, otherwise we'd bypass `objc_exception_throw`. See https://llvm.org/PR135089.
+  //
+  // [1]:
+  // https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/Exceptions/Articles/Exceptions64Bit.html
+  if _LIBCPP_CONSTEXPR (is_pointer<_Ep>::value) {
+    return std::__make_exception_ptr_via_throw(__e);
+  }
+
+#    if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION && !defined(_LIBCPP_CXX03_LANG)
+  return std::__make_exception_ptr_explicit(__e);
+#    else
+  return std::__make_exception_ptr_via_throw(__e);
 #    endif
-#  else
-  ((void)__e);
+}
+#  else  // !_LIBCPP_HAS_EXCEPTIONS
+template <class _Ep>
+_LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT {
   std::abort();
-#  endif
 }
+#  endif // _LIBCPP_HAS_EXCEPTIONS
 
 #else // _LIBCPP_ABI_MICROSOFT
 
diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h
index a2c28bfd79bb9..381e5678a5855 100644
--- a/libcxx/include/__filesystem/path.h
+++ b/libcxx/include/__filesystem/path.h
@@ -29,7 +29,6 @@
 
 #if _LIBCPP_HAS_LOCALIZATION
 #  include <iomanip> // for quoted
-#  include <locale>
 #endif
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__filesystem/u8path.h b/libcxx/include/__filesystem/u8path.h
index e13980298d9e9..a701425e42919 100644
--- a/libcxx/include/__filesystem/u8path.h
+++ b/libcxx/include/__filesystem/u8path.h
@@ -13,14 +13,9 @@
 #include <__algorithm/unwrap_iter.h>
 #include <__config>
 #include <__filesystem/path.h>
+#include <__locale>
 #include <string>
 
-// Only required on Windows for __widen_from_utf8, and included conservatively
-// because it requires support for localization.
-#if defined(_LIBCPP_WIN32API)
-#  include <locale>
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
diff --git a/libcxx/include/__flat_map/key_value_iterator.h b/libcxx/include/__flat_map/key_value_iterator.h
index f163dfc28706d..d04a23d1f8606 100644
--- a/libcxx/include/__flat_map/key_value_iterator.h
+++ b/libcxx/include/__flat_map/key_value_iterator.h
@@ -13,9 +13,12 @@
 #include <__compare/three_way_comparable.h>
 #include <__concepts/convertible_to.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/iterator_traits.h>
+#include <__iterator/product_iterator.h>
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
+#include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
 
@@ -57,6 +60,8 @@ struct __key_value_iterator {
   template <class, class, class, bool>
   friend struct __key_value_iterator;
 
+  friend struct __product_iterator_traits<__key_value_iterator>;
+
 public:
   using iterator_concept = random_access_iterator_tag;
   // `__key_value_iterator` only satisfy "Cpp17InputIterator" named requirements, because
@@ -181,6 +186,29 @@ struct __key_value_iterator {
   }
 };
 
+template <class _Owner, class _KeyContainer, class _MappedContainer, bool _Const>
+struct __product_iterator_traits<__key_value_iterator<_Owner, _KeyContainer, _MappedContainer, _Const>> {
+  static constexpr size_t __size = 2;
+
+  template <size_t _Nth, class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static decltype(auto) __get_iterator_element(_Iter&& __it)
+    requires(_Nth <= 1)
+  {
+    if constexpr (_Nth == 0) {
+      return std::forward<_Iter>(__it).__key_iter_;
+    } else {
+      return std::forward<_Iter>(__it).__mapped_iter_;
+    }
+  }
+
+  template <class _KeyIter, class _MappedIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto
+  __make_product_iterator(_KeyIter&& __key_iter, _MappedIter&& __mapped_iter) {
+    return __key_value_iterator<_Owner, _KeyContainer, _MappedContainer, _Const>(
+        std::forward<_KeyIter>(__key_iter), std::forward<_MappedIter>(__mapped_iter));
+  }
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 23
diff --git a/libcxx/include/__flat_map/utils.h b/libcxx/include/__flat_map/utils.h
index 27687ae8de3bc..3a05c715660dc 100644
--- a/libcxx/include/__flat_map/utils.h
+++ b/libcxx/include/__flat_map/utils.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___FLAT_MAP_UTILS_H
 
 #include <__config>
+#include <__iterator/product_iterator.h>
 #include <__type_traits/container_traits.h>
 #include <__utility/exception_guard.h>
 #include <__utility/forward.h>
@@ -79,8 +80,6 @@ struct __flat_map_utils {
     return typename _Map::iterator(std::move(__key_it), std::move(__mapped_it));
   }
 
-  // TODO: We could optimize this, see
-  // https://github.com/llvm/llvm-project/issues/108624
   template <class _Map, class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static typename _Map::size_type
   __append(_Map& __map, _InputIterator __first, _Sentinel __last) {
@@ -93,6 +92,25 @@ struct __flat_map_utils {
     }
     return __num_appended;
   }
+
+  template <class _Map, class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static typename _Map::size_type
+  __append(_Map& __map, _InputIterator __first, _InputIterator __last)
+    requires __is_product_iterator_of_size<_InputIterator, 2>::value
+  {
+    auto __s1 = __map.__containers_.keys.size();
+    __map.__containers_.keys.insert(
+        __map.__containers_.keys.end(),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<0>(__first),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<0>(__last));
+
+    __map.__containers_.values.insert(
+        __map.__containers_.values.end(),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<1>(__first),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<1>(__last));
+
+    return __map.__containers_.keys.size() - __s1;
+  }
 };
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__flat_set/flat_set.h b/libcxx/include/__flat_set/flat_set.h
index b8edd8eee0cfa..95cb998459bc8 100644
--- a/libcxx/include/__flat_set/flat_set.h
+++ b/libcxx/include/__flat_set/flat_set.h
@@ -107,13 +107,13 @@ class flat_set {
 
 public:
   // [flat.set.cons], construct/copy/destroy
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set() noexcept(is_nothrow_default_constructible_v<_KeyContainer> && is_nothrow_default_constructible_v<_Compare>)
       : __keys_(), __compare_() {}
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(const flat_set&) = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const flat_set&) = default;
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(flat_set&& __other) noexcept(
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(flat_set&& __other) noexcept(
       is_nothrow_move_constructible_v<_KeyContainer> && is_nothrow_move_constructible_v<_Compare>)
 #  if _LIBCPP_HAS_EXCEPTIONS
       try
@@ -130,14 +130,17 @@ class flat_set {
 #  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
-  _LIBCPP_HIDE_FROM_ABI explicit flat_set(const key_compare& __comp) : __keys_(), __compare_(__comp) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_set(const key_compare& __comp)
+      : __keys_(), __compare_(__comp) {}
 
-  _LIBCPP_HIDE_FROM_ABI explicit flat_set(container_type __keys, const key_compare& __comp = key_compare())
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_set(container_type __keys, const key_compare& __comp = key_compare())
       : __keys_(std::move(__keys)), __compare_(__comp) {
     __sort_and_unique();
   }
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(sorted_unique_t, container_type __keys, const key_compare& __comp = key_compare())
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(sorted_unique_t, container_type __keys, const key_compare& __comp = key_compare())
       : __keys_(std::move(__keys)), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys_), "Either the key container is not sorted or it contains duplicates");
@@ -145,7 +148,7 @@ class flat_set {
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(_InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
       : __keys_(), __compare_(__comp) {
     insert(__first, __last);
@@ -153,7 +156,7 @@ class flat_set {
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, _InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
       : __keys_(__first, __last), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
@@ -161,48 +164,52 @@ class flat_set {
   }
 
   template <_ContainerCompatibleRange<value_type> _Range>
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(from_range_t, _Range&& __rg)
       : flat_set(from_range, std::forward<_Range>(__rg), key_compare()) {}
 
   template <_ContainerCompatibleRange<value_type> _Range>
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg, const key_compare& __comp) : flat_set(__comp) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(from_range_t, _Range&& __rg, const key_compare& __comp)
+      : flat_set(__comp) {
     insert_range(std::forward<_Range>(__rg));
   }
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : flat_set(__il.begin(), __il.end(), __comp) {}
 
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : flat_set(sorted_unique, __il.begin(), __il.end(), __comp) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI explicit flat_set(const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_set(const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const key_compare& __comp, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const container_type& __keys, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const container_type& __keys, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_() {
     __sort_and_unique();
   }
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_(__comp) {
     __sort_and_unique();
   }
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(sorted_unique_t, const container_type& __keys, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(sorted_unique_t, const container_type& __keys, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_() {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys_), "Either the key container is not sorted or it contains duplicates");
@@ -210,7 +217,7 @@ class flat_set {
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
@@ -219,13 +226,13 @@ class flat_set {
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const flat_set& __other, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const flat_set& __other, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __other.__keys_)),
         __compare_(__other.__compare_) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(flat_set&& __other, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(flat_set&& __other, const _Allocator& __alloc)
 #  if _LIBCPP_HAS_EXCEPTIONS
       try
 #  endif // _LIBCPP_HAS_EXCEPTIONS
@@ -241,14 +248,15 @@ class flat_set {
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI flat_set(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {
     insert(__first, __last);
   }
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(_InputIterator __first, _InputIterator __last, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {
     insert(__first, __last);
@@ -256,7 +264,7 @@ class flat_set {
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, _InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __first, __last)), __compare_() {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
@@ -265,12 +273,12 @@ class flat_set {
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI
-  flat_set(sorted_unique_t,
-           _InputIterator __first,
-           _InputIterator __last,
-           const key_compare& __comp,
-           const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(
+      sorted_unique_t,
+      _InputIterator __first,
+      _InputIterator __last,
+      const key_compare& __comp,
+      const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __first, __last)), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys_), "Either the key container is not sorted or it contains duplicates");
@@ -278,49 +286,52 @@ class flat_set {
 
   template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(from_range_t, _Range&& __rg, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {
     insert_range(std::forward<_Range>(__rg));
   }
 
   template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {
     insert_range(std::forward<_Range>(__rg));
   }
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(initializer_list<value_type> __il, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(initializer_list<value_type> __il, const _Allocator& __alloc)
       : flat_set(__il.begin(), __il.end(), __alloc) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
       : flat_set(__il.begin(), __il.end(), __comp, __alloc) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(sorted_unique_t, initializer_list<value_type> __il, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(sorted_unique_t, initializer_list<value_type> __il, const _Allocator& __alloc)
       : flat_set(sorted_unique, __il.begin(), __il.end(), __alloc) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
       : flat_set(sorted_unique, __il.begin(), __il.end(), __comp, __alloc) {}
 
-  _LIBCPP_HIDE_FROM_ABI flat_set& operator=(initializer_list<value_type> __il) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set& operator=(initializer_list<value_type> __il) {
     clear();
     insert(__il);
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI flat_set& operator=(const flat_set&) = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set& operator=(const flat_set&) = default;
 
-  _LIBCPP_HIDE_FROM_ABI flat_set& operator=(flat_set&& __other) noexcept(
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set& operator=(flat_set&& __other) noexcept(
       is_nothrow_move_assignable_v<_KeyContainer> && is_nothrow_move_assignable_v<_Compare>) {
     // No matter what happens, we always want to clear the other container before returning
     // since we moved from it
@@ -336,31 +347,53 @@ class flat_set {
   }
 
   // iterators
-  _LIBCPP_HIDE_FROM_ABI iterator begin() noexcept { return iterator(std::as_const(__keys_).begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept { return const_iterator(__keys_.begin()); }
-  _LIBCPP_HIDE_FROM_ABI iterator end() noexcept { return iterator(std::as_const(__keys_).end()); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept { return const_iterator(__keys_.end()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
+    return iterator(std::as_const(__keys_).begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
+    return const_iterator(__keys_.begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
+    return iterator(std::as_const(__keys_).end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
+    return const_iterator(__keys_.end());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
+    return const_reverse_iterator(begin());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return begin(); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return end(); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept { return end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
+    return const_reverse_iterator(begin());
+  }
 
   // [flat.set.capacity], capacity
-  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __keys_.empty(); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool empty() const noexcept {
+    return __keys_.empty();
+  }
 
-  _LIBCPP_HIDE_FROM_ABI size_type size() const noexcept { return __keys_.size(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept { return __keys_.size(); }
 
-  _LIBCPP_HIDE_FROM_ABI size_type max_size() const noexcept { return __keys_.max_size(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept { return __keys_.max_size(); }
 
   // [flat.set.modifiers], modifiers
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> emplace(_Args&&... __args) {
     if constexpr (sizeof...(__args) == 1 && (is_same_v<remove_cvref_t<_Args>, _Key> && ...)) {
       return __emplace(std::forward<_Args>(__args)...);
     } else {
@@ -369,7 +402,7 @@ class flat_set {
   }
 
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
     if constexpr (sizeof...(__args) == 1 && (is_same_v<remove_cvref_t<_Args>, _Key> && ...)) {
       return __emplace_hint(std::move(__hint), std::forward<_Args>(__args)...);
     } else {
@@ -377,32 +410,36 @@ class flat_set {
     }
   }
 
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return emplace(__x); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> insert(const value_type& __x) {
+    return emplace(__x);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __x) { return emplace(std::move(__x)); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> insert(value_type&& __x) {
+    return emplace(std::move(__x));
+  }
 
   template <class _Kp>
     requires(__is_transparent_v<_Compare> && is_constructible_v<value_type, _Kp>)
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_Kp&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> insert(_Kp&& __x) {
     return __emplace(std::forward<_Kp>(__x));
   }
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, const value_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, const value_type& __x) {
     return emplace_hint(__hint, __x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, value_type&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, value_type&& __x) {
     return emplace_hint(__hint, std::move(__x));
   }
 
   template <class _Kp>
     requires(__is_transparent_v<_Compare> && is_constructible_v<value_type, _Kp>)
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _Kp&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, _Kp&& __x) {
     return __emplace_hint(__hint, std::forward<_Kp>(__x));
   }
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(_InputIterator __first, _InputIterator __last) {
     if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
       __reserve(__last - __first);
     }
@@ -411,7 +448,8 @@ class flat_set {
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI void insert(sorted_unique_t, _InputIterator __first, _InputIterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
+  insert(sorted_unique_t, _InputIterator __first, _InputIterator __last) {
     if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
       __reserve(__last - __first);
     }
@@ -420,7 +458,7 @@ class flat_set {
   }
 
   template <_ContainerCompatibleRange<value_type> _Range>
-  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(_Range&& __range) {
     if constexpr (ranges::sized_range<_Range>) {
       __reserve(ranges::size(__range));
     }
@@ -428,19 +466,21 @@ class flat_set {
     __append_sort_merge_unique</*WasSorted = */ false>(std::forward<_Range>(__range));
   }
 
-  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(initializer_list<value_type> __il) {
+    insert(__il.begin(), __il.end());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI void insert(sorted_unique_t, initializer_list<value_type> __il) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(sorted_unique_t, initializer_list<value_type> __il) {
     insert(sorted_unique, __il.begin(), __il.end());
   }
 
-  _LIBCPP_HIDE_FROM_ABI container_type extract() && {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 container_type extract() && {
     auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
     auto __ret   = std::move(__keys_);
     return __ret;
   }
 
-  _LIBCPP_HIDE_FROM_ABI void replace(container_type&& __keys) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void replace(container_type&& __keys) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys), "Either the key container is not sorted or it contains duplicates");
     auto __guard = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
@@ -448,7 +488,7 @@ class flat_set {
     __guard.__complete();
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __position) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(iterator __position) {
     auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
     auto __key_iter   = __keys_.erase(__position.__base());
     __on_failure.__complete();
@@ -458,7 +498,7 @@ class flat_set {
   // The following overload is the same as the iterator overload
   // iterator erase(const_iterator __position);
 
-  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(const key_type& __x) {
     auto __iter = find(__x);
     if (__iter != end()) {
       erase(__iter);
@@ -470,21 +510,21 @@ class flat_set {
   template <class _Kp>
     requires(__is_transparent_v<_Compare> && !is_convertible_v<_Kp &&, iterator> &&
              !is_convertible_v<_Kp &&, const_iterator>)
-  _LIBCPP_HIDE_FROM_ABI size_type erase(_Kp&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(_Kp&& __x) {
     auto [__first, __last] = equal_range(__x);
     auto __res             = __last - __first;
     erase(__first, __last);
     return __res;
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(const_iterator __first, const_iterator __last) {
     auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
     auto __key_it     = __keys_.erase(__first.__base(), __last.__base());
     __on_failure.__complete();
     return iterator(std::move(__key_it));
   }
 
-  _LIBCPP_HIDE_FROM_ABI void swap(flat_set& __y) noexcept {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_set& __y) noexcept {
     // warning: The spec has unconditional noexcept, which means that
     // if any of the following functions throw an exception,
     // std::terminate will be called.
@@ -493,121 +533,134 @@ class flat_set {
     ranges::swap(__keys_, __y.__keys_);
   }
 
-  _LIBCPP_HIDE_FROM_ABI void clear() noexcept { __keys_.clear(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void clear() noexcept { __keys_.clear(); }
 
   // observers
-  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __compare_; }
-  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __compare_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const { return __compare_; }
 
   // set operations
-  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __x) { return __find_impl(*this, __x); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
+    return __find_impl(*this, __x);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __x) const { return __find_impl(*this, __x); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
+    return __find_impl(*this, __x);
+  }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI iterator find(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
     return __find_impl(*this, __x);
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
     return __find_impl(*this, __x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __x) const { return contains(__x) ? 1 : 0; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
+    return contains(__x) ? 1 : 0;
+  }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI size_type count(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
     return contains(__x) ? 1 : 0;
   }
 
-  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __x) const { return find(__x) != end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
+    return find(__x) != end();
+  }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI bool contains(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
     return find(__x) != end();
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
     const auto& __keys = __keys_;
     return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const key_type& __x) const {
     return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
     const auto& __keys = __keys_;
     return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
     return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
     const auto& __keys = __keys_;
     return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const key_type& __x) const {
     return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
     const auto& __keys = __keys_;
     return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
     return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const key_type& __x) {
     return __equal_range_impl(*this, __x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
+  equal_range(const key_type& __x) const {
     return __equal_range_impl(*this, __x);
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const _Kp& __x) {
     return __equal_range_impl(*this, __x);
   }
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
+  equal_range(const _Kp& __x) const {
     return __equal_range_impl(*this, __x);
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const flat_set& __x, const flat_set& __y) {
+  friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool operator==(const flat_set& __x, const flat_set& __y) {
     return ranges::equal(__x, __y);
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI auto operator<=>(const flat_set& __x, const flat_set& __y) {
+  friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 auto
+  operator<=>(const flat_set& __x, const flat_set& __y) {
     return std::lexicographical_compare_three_way(
         __x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI void swap(flat_set& __x, flat_set& __y) noexcept { __x.swap(__y); }
+  friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_set& __x, flat_set& __y) noexcept {
+    __x.swap(__y);
+  }
 
 private:
-  _LIBCPP_HIDE_FROM_ABI bool __is_sorted_and_unique(auto&& __key_container) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_sorted_and_unique(auto&& __key_container) const {
     auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) { return !__compare_(__x, __y); };
     return ranges::adjacent_find(__key_container, __greater_or_equal_to) == ranges::end(__key_container);
   }
@@ -615,14 +668,14 @@ class flat_set {
   // This function is only used in constructors. So there is not exception handling in this function.
   // If the function exits via an exception, there will be no flat_set object constructed, thus, there
   // is no invariant state to preserve
-  _LIBCPP_HIDE_FROM_ABI void __sort_and_unique() {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __sort_and_unique() {
     ranges::sort(__keys_, __compare_);
     auto __dup_start = ranges::unique(__keys_, __key_equiv(__compare_)).begin();
     __keys_.erase(__dup_start, __keys_.end());
   }
 
   template <bool _WasSorted, class... _Args>
-  _LIBCPP_HIDE_FROM_ABI void __append_sort_merge_unique(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __append_sort_merge_unique(_Args&&... __args) {
     auto __on_failure    = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
     size_type __old_size = size();
     __flat_set_utils::__append(*this, std::forward<_Args>(__args)...);
@@ -642,7 +695,7 @@ class flat_set {
   }
 
   template <class _Self, class _Kp>
-  _LIBCPP_HIDE_FROM_ABI static auto __find_impl(_Self&& __self, const _Kp& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __find_impl(_Self&& __self, const _Kp& __key) {
     auto __it   = __self.lower_bound(__key);
     auto __last = __self.end();
     if (__it == __last || __self.__compare_(__key, *__it)) {
@@ -652,7 +705,7 @@ class flat_set {
   }
 
   template <class _Self, class _Kp>
-  _LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
     using __iter = _If<is_const_v<__libcpp_remove_reference_t<_Self>>, const_iterator, iterator>;
     auto __it    = std::lower_bound(__self.__keys_.begin(), __self.__keys_.end(), __key, __self.__compare_);
     auto __last  = __self.__keys_.end();
@@ -663,7 +716,7 @@ class flat_set {
   }
 
   template <class _Kp>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace(_Kp&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> __emplace(_Kp&& __key) {
     auto __it = lower_bound(__key);
     if (__it == end() || __compare_(__key, *__it)) {
       return pair<iterator, bool>(__flat_set_utils::__emplace_exact_pos(*this, __it, std::forward<_Kp>(__key)), true);
@@ -673,7 +726,7 @@ class flat_set {
   }
 
   template <class _Kp>
-  _LIBCPP_HIDE_FROM_ABI bool __is_hint_correct(const_iterator __hint, _Kp&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_hint_correct(const_iterator __hint, _Kp&& __key) {
     if (__hint != cbegin() && !__compare_(*std::prev(__hint), __key)) {
       return false;
     }
@@ -684,7 +737,7 @@ class flat_set {
   }
 
   template <class _Kp>
-  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint(const_iterator __hint, _Kp&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator __emplace_hint(const_iterator __hint, _Kp&& __key) {
     if (__is_hint_correct(__hint, __key)) {
       if (__hint == cend() || __compare_(__key, *__hint)) {
         return __flat_set_utils::__emplace_exact_pos(*this, __hint, std::forward<_Kp>(__key));
@@ -697,22 +750,23 @@ class flat_set {
     }
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __reserve(size_t __size) {
     if constexpr (__container_traits<_KeyContainer>::__reservable) {
       __keys_.reserve(__size);
     }
   }
 
   template <class _Key2, class _Compare2, class _KeyContainer2, class _Predicate>
-  friend typename flat_set<_Key2, _Compare2, _KeyContainer2>::size_type
+  friend typename flat_set<_Key2, _Compare2, _KeyContainer2>::size_type _LIBCPP_CONSTEXPR_SINCE_CXX26
   erase_if(flat_set<_Key2, _Compare2, _KeyContainer2>&, _Predicate);
 
   _KeyContainer __keys_;
   _LIBCPP_NO_UNIQUE_ADDRESS key_compare __compare_;
 
   struct __key_equiv {
-    _LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
-    _LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 __key_equiv(key_compare __c) : __comp_(__c) {}
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
+    operator()(const_reference __x, const_reference __y) const {
       return !__comp_(__x, __y) && !__comp_(__y, __x);
     }
     key_compare __comp_;
@@ -799,7 +853,7 @@ struct uses_allocator<flat_set<_Key, _Compare, _KeyContainer>, _Allocator>
     : bool_constant<uses_allocator_v<_KeyContainer, _Allocator>> {};
 
 template <class _Key, class _Compare, class _KeyContainer, class _Predicate>
-_LIBCPP_HIDE_FROM_ABI typename flat_set<_Key, _Compare, _KeyContainer>::size_type
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 typename flat_set<_Key, _Compare, _KeyContainer>::size_type
 erase_if(flat_set<_Key, _Compare, _KeyContainer>& __flat_set, _Predicate __pred) {
   auto __guard = std::__make_exception_guard([&] { __flat_set.clear(); });
   auto __it    = std::remove_if(__flat_set.__keys_.begin(), __flat_set.__keys_.end(), [&](const auto& __e) -> bool {
diff --git a/libcxx/include/__flat_set/utils.h b/libcxx/include/__flat_set/utils.h
index 542bfd886aef5..8a7f93a00465d 100644
--- a/libcxx/include/__flat_set/utils.h
+++ b/libcxx/include/__flat_set/utils.h
@@ -38,7 +38,8 @@ struct __flat_set_utils {
   // When an exception is thrown during the emplacement, the function will clear the set if the container does not
   // have strong exception safety guarantee on emplacement.
   template <class _Set, class _Iter, class _KeyArg>
-  _LIBCPP_HIDE_FROM_ABI static auto __emplace_exact_pos(_Set& __set, _Iter&& __iter, _KeyArg&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto
+  __emplace_exact_pos(_Set& __set, _Iter&& __iter, _KeyArg&& __key) {
     using _KeyContainer = typename decay_t<_Set>::container_type;
     auto __on_failure   = std::__make_exception_guard([&]() noexcept {
       if constexpr (!__container_traits<_KeyContainer>::__emplacement_has_strong_exception_safety_guarantee) {
@@ -51,12 +52,13 @@ struct __flat_set_utils {
   }
 
   template <class _Set, class _InputIterator>
-  _LIBCPP_HIDE_FROM_ABI static void __append(_Set& __set, _InputIterator __first, _InputIterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static void
+  __append(_Set& __set, _InputIterator __first, _InputIterator __last) {
     __set.__keys_.insert(__set.__keys_.end(), std::move(__first), std::move(__last));
   }
 
   template <class _Set, class _Range>
-  _LIBCPP_HIDE_FROM_ABI static void __append(_Set& __set, _Range&& __rng) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static void __append(_Set& __set, _Range&& __rng) {
     if constexpr (requires { __set.__keys_.insert_range(__set.__keys_.end(), std::forward<_Range>(__rng)); }) {
       // C++23 Sequence Container should have insert_range member function
       // Note that not all Sequence Containers provide append_range.
diff --git a/libcxx/include/__functional/boyer_moore_searcher.h b/libcxx/include/__functional/boyer_moore_searcher.h
index 7889232f4b919..6c51c32d2f89b 100644
--- a/libcxx/include/__functional/boyer_moore_searcher.h
+++ b/libcxx/include/__functional/boyer_moore_searcher.h
@@ -17,12 +17,10 @@
 #include <__config>
 #include <__functional/hash.h>
 #include <__functional/operations.h>
-#include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/shared_ptr.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/pair.h>
-#include <__vector/vector.h>
 #include <array>
 #include <limits>
 #include <unordered_map>
@@ -125,8 +123,8 @@ class boyer_moore_searcher {
   template <class _RandomAccessIterator2>
   _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
   operator()(_RandomAccessIterator2 __first, _RandomAccessIterator2 __last) const {
-    static_assert(__is_same_uncvref<typename iterator_traits<_RandomAccessIterator1>::value_type,
-                                    typename iterator_traits<_RandomAccessIterator2>::value_type>::value,
+    static_assert(is_same_v<__remove_cvref_t<typename iterator_traits<_RandomAccessIterator1>::value_type>,
+                            __remove_cvref_t<typename iterator_traits<_RandomAccessIterator2>::value_type>>,
                   "Corpus and Pattern iterators must point to the same type");
     if (__first == __last)
       return std::make_pair(__last, __last);
@@ -196,7 +194,7 @@ class boyer_moore_searcher {
     if (__count == 0)
       return;
 
-    vector<difference_type> __scratch(__count);
+    auto __scratch = std::make_unique<difference_type[]>(__count);
 
     __compute_bm_prefix(__first, __last, __pred, __scratch);
     for (size_t __i = 0; __i <= __count; ++__i)
@@ -256,8 +254,8 @@ class boyer_moore_horspool_searcher {
   template <class _RandomAccessIterator2>
   _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
   operator()(_RandomAccessIterator2 __first, _RandomAccessIterator2 __last) const {
-    static_assert(__is_same_uncvref<typename std::iterator_traits<_RandomAccessIterator1>::value_type,
-                                    typename std::iterator_traits<_RandomAccessIterator2>::value_type>::value,
+    static_assert(is_same_v<__remove_cvref_t<typename std::iterator_traits<_RandomAccessIterator1>::value_type>,
+                            __remove_cvref_t<typename std::iterator_traits<_RandomAccessIterator2>::value_type>>,
                   "Corpus and Pattern iterators must point to the same type");
     if (__first == __last)
       return std::make_pair(__last, __last);
diff --git a/libcxx/include/__functional/reference_wrapper.h b/libcxx/include/__functional/reference_wrapper.h
index c46203a4ca9a4..148703b21d84a 100644
--- a/libcxx/include/__functional/reference_wrapper.h
+++ b/libcxx/include/__functional/reference_wrapper.h
@@ -11,14 +11,18 @@
 #define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 
 #include <__compare/synth_three_way.h>
+#include <__concepts/convertible_to.h>
 #include <__config>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
+#include <__type_traits/common_reference.h>
 #include <__type_traits/desugars_to.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/is_core_convertible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_specialization.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
@@ -45,7 +49,7 @@ class reference_wrapper : public __weak_result_type<_Tp> {
 public:
   template <class _Up,
             class = __void_t<decltype(__fun(std::declval<_Up>()))>,
-            __enable_if_t<!__is_same_uncvref<_Up, reference_wrapper>::value, int> = 0>
+            __enable_if_t<!is_same<__remove_cvref_t<_Up>, reference_wrapper>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper(_Up&& __u)
       _NOEXCEPT_(noexcept(__fun(std::declval<_Up>()))) {
     type& __f = static_cast<_Up&&>(__u);
@@ -155,6 +159,32 @@ template <class _CanonicalTag, class _Operation, class... _Args>
 inline const bool __desugars_to_v<_CanonicalTag, reference_wrapper<_Operation>, _Args...> =
     __desugars_to_v<_CanonicalTag, _Operation, _Args...>;
 
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+inline constexpr bool __is_ref_wrapper = __is_specialization_v<_Tp, reference_wrapper>;
+
+template <class _Rp, class _Tp, class _RpQual, class _TpQual>
+concept __ref_wrap_common_reference_exists_with = __is_ref_wrapper<_Rp> && requires {
+  typename common_reference_t<typename _Rp::type&, _TpQual>;
+} && convertible_to<_RpQual, common_reference_t<typename _Rp::type&, _TpQual>>;
+
+template <class _Rp, class _Tp, template <class> class _RpQual, template <class> class _TpQual>
+  requires(__ref_wrap_common_reference_exists_with<_Rp, _Tp, _RpQual<_Rp>, _TpQual<_Tp>> &&
+           !__ref_wrap_common_reference_exists_with<_Tp, _Rp, _TpQual<_Tp>, _RpQual<_Rp>>)
+struct basic_common_reference<_Rp, _Tp, _RpQual, _TpQual> {
+  using type _LIBCPP_NODEBUG = common_reference_t<typename _Rp::type&, _TpQual<_Tp>>;
+};
+
+template <class _Tp, class _Rp, template <class> class _TpQual, template <class> class _RpQual>
+  requires(__ref_wrap_common_reference_exists_with<_Rp, _Tp, _RpQual<_Rp>, _TpQual<_Tp>> &&
+           !__ref_wrap_common_reference_exists_with<_Tp, _Rp, _TpQual<_Tp>, _RpQual<_Rp>>)
+struct basic_common_reference<_Tp, _Rp, _TpQual, _RpQual> {
+  using type _LIBCPP_NODEBUG = common_reference_t<typename _Rp::type&, _TpQual<_Tp>>;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index cc5f6d1348e41..c59d8d5bf383a 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -207,12 +207,12 @@ struct __hash_key_value_types<__hash_value_type<_Key, _Tp> > {
 
   _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(__container_value_type const& __v) { return __v.first; }
 
-  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __node_value_type>::value, int> = 0>
+  template <class _Up, __enable_if_t<is_same<__remove_cvref_t<_Up>, __node_value_type>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
     return __t.__get_value();
   }
 
-  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __container_value_type>::value, int> = 0>
+  template <class _Up, __enable_if_t<is_same<__remove_cvref_t<_Up>, __container_value_type>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
     return __t;
   }
@@ -856,7 +856,7 @@ public:
     return __emplace_unique_key_args(_NodeTypes::__get_key(__x), std::move(__x));
   }
 
-  template <class _Pp, __enable_if_t<!__is_same_uncvref<_Pp, __container_value_type>::value, int> = 0>
+  template <class _Pp, __enable_if_t<!is_same<__remove_cvref_t<_Pp>, __container_value_type>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(_Pp&& __x) {
     return __emplace_unique(std::forward<_Pp>(__x));
   }
@@ -1831,9 +1831,9 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
 template <class _Key>
 typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) {
-  size_t __hash  = hash_function()(__k);
   size_type __bc = bucket_count();
-  if (__bc != 0) {
+  if (__bc != 0 && size() != 0) {
+    size_t __hash       = hash_function()(__k);
     size_t __chash      = std::__constrain_hash(__hash, __bc);
     __next_pointer __nd = __bucket_list_[__chash];
     if (__nd != nullptr) {
@@ -1852,9 +1852,9 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
 template <class _Key>
 typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) const {
-  size_t __hash  = hash_function()(__k);
   size_type __bc = bucket_count();
-  if (__bc != 0) {
+  if (__bc != 0 && size() != 0) {
+    size_t __hash       = hash_function()(__k);
     size_t __chash      = std::__constrain_hash(__hash, __bc);
     __next_pointer __nd = __bucket_list_[__chash];
     if (__nd != nullptr) {
diff --git a/libcxx/include/__iterator/product_iterator.h b/libcxx/include/__iterator/product_iterator.h
new file mode 100644
index 0000000000000..1db673bafd337
--- /dev/null
+++ b/libcxx/include/__iterator/product_iterator.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_PRODUCT_ITERATOR_H
+#define _LIBCPP___ITERATOR_PRODUCT_ITERATOR_H
+
+// Product iterators are iterators that contain two or more underlying iterators.
+//
+// For example, std::flat_map stores its data into two separate containers, and its iterator
+// is a proxy over two separate underlying iterators. The concept of product iterators
+// allows algorithms to operate over these underlying iterators separately, opening the
+// door to various optimizations.
+//
+// If __product_iterator_traits can be instantiated, the following functions and associated types must be provided:
+// - static constexpr size_t Traits::__size
+//   The number of underlying iterators inside the product iterator.
+//
+// - template <size_t _N>
+//   static decltype(auto) Traits::__get_iterator_element(It&& __it)
+//   Returns the _Nth iterator element of the given product iterator.
+//
+// - template <class... _Iters>
+//   static _Iterator __make_product_iterator(_Iters&&...);
+//   Creates a product iterator from the given underlying iterators.
+
+#include <__config>
+#include <__cstddef/size_t.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iterator>
+struct __product_iterator_traits;
+/* exposition-only:
+{
+  static constexpr size_t __size = ...;
+
+  template <size_t _N, class _Iter>
+  static decltype(auto) __get_iterator_element(_Iter&&);
+
+  template <class... _Iters>
+  static _Iterator __make_product_iterator(_Iters&&...);
+};
+*/
+
+template <class _Tp, size_t = 0>
+struct __is_product_iterator : false_type {};
+
+template <class _Tp>
+struct __is_product_iterator<_Tp, sizeof(__product_iterator_traits<_Tp>) * 0> : true_type {};
+
+template <class _Tp, size_t _Size, class = void>
+struct __is_product_iterator_of_size : false_type {};
+
+template <class _Tp, size_t _Size>
+struct __is_product_iterator_of_size<_Tp, _Size, __enable_if_t<__product_iterator_traits<_Tp>::__size == _Size> >
+    : true_type {};
+
+template <class _Iterator, size_t _Nth>
+using __product_iterator_element_t _LIBCPP_NODEBUG =
+    decltype(__product_iterator_traits<_Iterator>::template __get_iterator_element<_Nth>(std::declval<_Iterator>()));
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_PRODUCT_ITERATOR_H
diff --git a/libcxx/include/__locale_dir/check_grouping.h b/libcxx/include/__locale_dir/check_grouping.h
new file mode 100644
index 0000000000000..93e9e404bb5f3
--- /dev/null
+++ b/libcxx/include/__locale_dir/check_grouping.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_CHECK_GROUPING_H
+#define _LIBCPP___LOCALE_DIR_CHECK_GROUPING_H
+
+#include <__config>
+#include <__fwd/string.h>
+#include <ios>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_EXPORTED_FROM_ABI void
+__check_grouping(const string& __grouping, unsigned* __g, unsigned* __g_end, ios_base::iostate& __err);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_CHECK_GROUPING_H
diff --git a/libcxx/include/__locale_dir/get_c_locale.h b/libcxx/include/__locale_dir/get_c_locale.h
new file mode 100644
index 0000000000000..e8bac9a87095b
--- /dev/null
+++ b/libcxx/include/__locale_dir/get_c_locale.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_GET_C_LOCALE_H
+#define _LIBCPP___LOCALE_DIR_GET_C_LOCALE_H
+
+#include <__config>
+#include <__locale_dir/locale_base_api.h>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// FIXME: This should really be part of the locale base API
+
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    define _LIBCPP_GET_C_LOCALE 0
+#  elif defined(__NetBSD__)
+#    define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
+#  else
+#    define _LIBCPP_GET_C_LOCALE __cloc()
+// Get the C locale object
+_LIBCPP_EXPORTED_FROM_ABI __locale::__locale_t __cloc();
+#    define __cloc_defined
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_GET_C_LOCALE_H
diff --git a/libcxx/include/__locale_dir/messages.h b/libcxx/include/__locale_dir/messages.h
new file mode 100644
index 0000000000000..c04bf04025ff0
--- /dev/null
+++ b/libcxx/include/__locale_dir/messages.h
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_MESSAGES_H
+#define _LIBCPP___LOCALE_DIR_MESSAGES_H
+
+#include <__config>
+#include <__iterator/back_insert_iterator.h>
+#include <__locale>
+#include <string>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
+// Most unix variants have catopen.  These are the specific ones that don't.
+#    if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
+#      define _LIBCPP_HAS_CATOPEN 1
+#      include <nl_types.h>
+#    else
+#      define _LIBCPP_HAS_CATOPEN 0
+#    endif
+#  else
+#    define _LIBCPP_HAS_CATOPEN 0
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI messages_base {
+public:
+  typedef intptr_t catalog;
+
+  _LIBCPP_HIDE_FROM_ABI messages_base() {}
+};
+
+template <class _CharT>
+class messages : public locale::facet, public messages_base {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<_CharT> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI catalog open(const basic_string<char>& __nm, const locale& __loc) const {
+    return do_open(__nm, __loc);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI string_type get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
+    return do_get(__c, __set, __msgid, __dflt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void close(catalog __c) const { do_close(__c); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages() override {}
+
+  virtual catalog do_open(const basic_string<char>&, const locale&) const;
+  virtual string_type do_get(catalog, int __set, int __msgid, const string_type& __dflt) const;
+  virtual void do_close(catalog) const;
+};
+
+template <class _CharT>
+locale::id messages<_CharT>::id;
+
+template <class _CharT>
+typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
+#  if _LIBCPP_HAS_CATOPEN
+  return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__nm;
+  return -1;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+template <class _CharT>
+typename messages<_CharT>::string_type
+messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
+#  if _LIBCPP_HAS_CATOPEN
+  string __ndflt;
+  __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
+      std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
+  nl_catd __cat = (nl_catd)__c;
+  static_assert(sizeof(catalog) >= sizeof(nl_catd), "Unexpected nl_catd type");
+  char* __n = catgets(__cat, __set, __msgid, __ndflt.c_str());
+  string_type __w;
+  __widen_from_utf8<sizeof(char_type) * __CHAR_BIT__>()(std::back_inserter(__w), __n, __n + std::strlen(__n));
+  return __w;
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__c;
+  (void)__set;
+  (void)__msgid;
+  return __dflt;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+template <class _CharT>
+void messages<_CharT>::do_close(catalog __c) const {
+#  if _LIBCPP_HAS_CATOPEN
+  catclose((nl_catd)__c);
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__c;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<wchar_t>;
+#  endif
+
+template <class _CharT>
+class messages_byname : public messages<_CharT> {
+public:
+  typedef messages_base::catalog catalog;
+  typedef basic_string<_CharT> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const char*, size_t __refs = 0) : messages<_CharT>(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const string&, size_t __refs = 0) : messages<_CharT>(__refs) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages_byname() override {}
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_MESSAGES_H
diff --git a/libcxx/include/__locale_dir/money.h b/libcxx/include/__locale_dir/money.h
new file mode 100644
index 0000000000000..c1296665505e1
--- /dev/null
+++ b/libcxx/include/__locale_dir/money.h
@@ -0,0 +1,873 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_MONEY_H
+#define _LIBCPP___LOCALE_DIR_MONEY_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/find.h>
+#include <__algorithm/reverse.h>
+#include <__config>
+#include <__locale>
+#include <__locale_dir/check_grouping.h>
+#include <__locale_dir/get_c_locale.h>
+#include <__locale_dir/pad_and_output.h>
+#include <__memory/unique_ptr.h>
+#include <ios>
+#include <string>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// money_base
+
+class _LIBCPP_EXPORTED_FROM_ABI money_base {
+public:
+  enum part { none, space, symbol, sign, value };
+  struct pattern {
+    char field[4];
+  };
+
+  _LIBCPP_HIDE_FROM_ABI money_base() {}
+};
+
+// moneypunct
+
+template <class _CharT, bool _International = false>
+class moneypunct : public locale::facet, public money_base {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
+  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
+  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
+  _LIBCPP_HIDE_FROM_ABI string_type curr_symbol() const { return do_curr_symbol(); }
+  _LIBCPP_HIDE_FROM_ABI string_type positive_sign() const { return do_positive_sign(); }
+  _LIBCPP_HIDE_FROM_ABI string_type negative_sign() const { return do_negative_sign(); }
+  _LIBCPP_HIDE_FROM_ABI int frac_digits() const { return do_frac_digits(); }
+  _LIBCPP_HIDE_FROM_ABI pattern pos_format() const { return do_pos_format(); }
+  _LIBCPP_HIDE_FROM_ABI pattern neg_format() const { return do_neg_format(); }
+
+  static locale::id id;
+  static const bool intl = _International;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct() override {}
+
+  virtual char_type do_decimal_point() const { return numeric_limits<char_type>::max(); }
+  virtual char_type do_thousands_sep() const { return numeric_limits<char_type>::max(); }
+  virtual string do_grouping() const { return string(); }
+  virtual string_type do_curr_symbol() const { return string_type(); }
+  virtual string_type do_positive_sign() const { return string_type(); }
+  virtual string_type do_negative_sign() const { return string_type(1, '-'); }
+  virtual int do_frac_digits() const { return 0; }
+  virtual pattern do_pos_format() const {
+    pattern __p = {{symbol, sign, none, value}};
+    return __p;
+  }
+  virtual pattern do_neg_format() const {
+    pattern __p = {{symbol, sign, none, value}};
+    return __p;
+  }
+};
+
+template <class _CharT, bool _International>
+locale::id moneypunct<_CharT, _International>::id;
+
+template <class _CharT, bool _International>
+const bool moneypunct<_CharT, _International>::intl;
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, true>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, true>;
+#  endif
+
+// moneypunct_byname
+
+template <class _CharT, bool _International = false>
+class moneypunct_byname : public moneypunct<_CharT, _International> {
+public:
+  typedef money_base::pattern pattern;
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const char* __nm, size_t __refs = 0)
+      : moneypunct<_CharT, _International>(__refs) {
+    init(__nm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const string& __nm, size_t __refs = 0)
+      : moneypunct<_CharT, _International>(__refs) {
+    init(__nm.c_str());
+  }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct_byname() override {}
+
+  char_type do_decimal_point() const override { return __decimal_point_; }
+  char_type do_thousands_sep() const override { return __thousands_sep_; }
+  string do_grouping() const override { return __grouping_; }
+  string_type do_curr_symbol() const override { return __curr_symbol_; }
+  string_type do_positive_sign() const override { return __positive_sign_; }
+  string_type do_negative_sign() const override { return __negative_sign_; }
+  int do_frac_digits() const override { return __frac_digits_; }
+  pattern do_pos_format() const override { return __pos_format_; }
+  pattern do_neg_format() const override { return __neg_format_; }
+
+private:
+  char_type __decimal_point_;
+  char_type __thousands_sep_;
+  string __grouping_;
+  string_type __curr_symbol_;
+  string_type __positive_sign_;
+  string_type __negative_sign_;
+  int __frac_digits_;
+  pattern __pos_format_;
+  pattern __neg_format_;
+
+  void init(const char*);
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, false>::init(const char*);
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, true>::init(const char*);
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, true>;
+
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, false>::init(const char*);
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, true>::init(const char*);
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, true>;
+#  endif
+
+// money_get
+
+template <class _CharT>
+class __money_get {
+protected:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI __money_get() {}
+
+  static void __gather_info(
+      bool __intl,
+      const locale& __loc,
+      money_base::pattern& __pat,
+      char_type& __dp,
+      char_type& __ts,
+      string& __grp,
+      string_type& __sym,
+      string_type& __psn,
+      string_type& __nsn,
+      int& __fd);
+};
+
+template <class _CharT>
+void __money_get<_CharT>::__gather_info(
+    bool __intl,
+    const locale& __loc,
+    money_base::pattern& __pat,
+    char_type& __dp,
+    char_type& __ts,
+    string& __grp,
+    string_type& __sym,
+    string_type& __psn,
+    string_type& __nsn,
+    int& __fd) {
+  if (__intl) {
+    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
+    __pat                                   = __mp.neg_format();
+    __nsn                                   = __mp.negative_sign();
+    __psn                                   = __mp.positive_sign();
+    __dp                                    = __mp.decimal_point();
+    __ts                                    = __mp.thousands_sep();
+    __grp                                   = __mp.grouping();
+    __sym                                   = __mp.curr_symbol();
+    __fd                                    = __mp.frac_digits();
+  } else {
+    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
+    __pat                                    = __mp.neg_format();
+    __nsn                                    = __mp.negative_sign();
+    __psn                                    = __mp.positive_sign();
+    __dp                                     = __mp.decimal_point();
+    __ts                                     = __mp.thousands_sep();
+    __grp                                    = __mp.grouping();
+    __sym                                    = __mp.curr_symbol();
+    __fd                                     = __mp.frac_digits();
+  }
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<wchar_t>;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class money_get : public locale::facet, private __money_get<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit money_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return do_get(__b, __e, __intl, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
+    return do_get(__b, __e, __intl, __iob, __err, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_get() override {}
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const;
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const;
+
+private:
+  static bool __do_get(
+      iter_type& __b,
+      iter_type __e,
+      bool __intl,
+      const locale& __loc,
+      ios_base::fmtflags __flags,
+      ios_base::iostate& __err,
+      bool& __neg,
+      const ctype<char_type>& __ct,
+      unique_ptr<char_type, void (*)(void*)>& __wb,
+      char_type*& __wn,
+      char_type* __we);
+};
+
+template <class _CharT, class _InputIterator>
+locale::id money_get<_CharT, _InputIterator>::id;
+
+_LIBCPP_EXPORTED_FROM_ABI void __do_nothing(void*);
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __double_or_nothing(unique_ptr<_Tp, void (*)(void*)>& __b, _Tp*& __n, _Tp*& __e) {
+  bool __owns      = __b.get_deleter() != __do_nothing;
+  size_t __cur_cap = static_cast<size_t>(__e - __b.get()) * sizeof(_Tp);
+  size_t __new_cap = __cur_cap < numeric_limits<size_t>::max() / 2 ? 2 * __cur_cap : numeric_limits<size_t>::max();
+  if (__new_cap == 0)
+    __new_cap = sizeof(_Tp);
+  size_t __n_off = static_cast<size_t>(__n - __b.get());
+  _Tp* __t       = (_Tp*)std::realloc(__owns ? __b.get() : 0, __new_cap);
+  if (__t == 0)
+    std::__throw_bad_alloc();
+  if (__owns)
+    __b.release();
+  else
+    std::memcpy(__t, __b.get(), __cur_cap);
+  __b = unique_ptr<_Tp, void (*)(void*)>(__t, free);
+  __new_cap /= sizeof(_Tp);
+  __n = __b.get() + __n_off;
+  __e = __b.get() + __new_cap;
+}
+
+// true == success
+template <class _CharT, class _InputIterator>
+bool money_get<_CharT, _InputIterator>::__do_get(
+    iter_type& __b,
+    iter_type __e,
+    bool __intl,
+    const locale& __loc,
+    ios_base::fmtflags __flags,
+    ios_base::iostate& __err,
+    bool& __neg,
+    const ctype<char_type>& __ct,
+    unique_ptr<char_type, void (*)(void*)>& __wb,
+    char_type*& __wn,
+    char_type* __we) {
+  if (__b == __e) {
+    __err |= ios_base::failbit;
+    return false;
+  }
+  const unsigned __bz = 100;
+  unsigned __gbuf[__bz];
+  unique_ptr<unsigned, void (*)(void*)> __gb(__gbuf, __do_nothing);
+  unsigned* __gn = __gb.get();
+  unsigned* __ge = __gn + __bz;
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __psn;
+  string_type __nsn;
+  // Capture the spaces read into money_base::{space,none} so they
+  // can be compared to initial spaces in __sym.
+  string_type __spaces;
+  int __fd;
+  __money_get<_CharT>::__gather_info(__intl, __loc, __pat, __dp, __ts, __grp, __sym, __psn, __nsn, __fd);
+  const string_type* __trailing_sign = 0;
+  __wn                               = __wb.get();
+  for (unsigned __p = 0; __p < 4 && __b != __e; ++__p) {
+    switch (__pat.field[__p]) {
+    case money_base::space:
+      if (__p != 3) {
+        if (__ct.is(ctype_base::space, *__b))
+          __spaces.push_back(*__b++);
+        else {
+          __err |= ios_base::failbit;
+          return false;
+        }
+      }
+      [[__fallthrough__]];
+    case money_base::none:
+      if (__p != 3) {
+        while (__b != __e && __ct.is(ctype_base::space, *__b))
+          __spaces.push_back(*__b++);
+      }
+      break;
+    case money_base::sign:
+      if (__psn.size() > 0 && *__b == __psn[0]) {
+        ++__b;
+        __neg = false;
+        if (__psn.size() > 1)
+          __trailing_sign = std::addressof(__psn);
+        break;
+      }
+      if (__nsn.size() > 0 && *__b == __nsn[0]) {
+        ++__b;
+        __neg = true;
+        if (__nsn.size() > 1)
+          __trailing_sign = std::addressof(__nsn);
+        break;
+      }
+      if (__psn.size() > 0 && __nsn.size() > 0) { // sign is required
+        __err |= ios_base::failbit;
+        return false;
+      }
+      if (__psn.size() == 0 && __nsn.size() == 0)
+        // locale has no way of specifying a sign. Use the initial value of __neg as a default
+        break;
+      __neg = (__nsn.size() == 0);
+      break;
+    case money_base::symbol: {
+      bool __more_needed =
+          __trailing_sign || (__p < 2) || (__p == 2 && __pat.field[3] != static_cast<char>(money_base::none));
+      bool __sb = (__flags & ios_base::showbase) != 0;
+      if (__sb || __more_needed) {
+        typename string_type::const_iterator __sym_space_end = __sym.begin();
+        if (__p > 0 && (__pat.field[__p - 1] == money_base::none || __pat.field[__p - 1] == money_base::space)) {
+          // Match spaces we've already read against spaces at
+          // the beginning of __sym.
+          while (__sym_space_end != __sym.end() && __ct.is(ctype_base::space, *__sym_space_end))
+            ++__sym_space_end;
+          const size_t __num_spaces = __sym_space_end - __sym.begin();
+          if (__num_spaces > __spaces.size() ||
+              !std::equal(__spaces.end() - __num_spaces, __spaces.end(), __sym.begin())) {
+            // No match. Put __sym_space_end back at the
+            // beginning of __sym, which will prevent a
+            // match in the next loop.
+            __sym_space_end = __sym.begin();
+          }
+        }
+        typename string_type::const_iterator __sym_curr_char = __sym_space_end;
+        while (__sym_curr_char != __sym.end() && __b != __e && *__b == *__sym_curr_char) {
+          ++__b;
+          ++__sym_curr_char;
+        }
+        if (__sb && __sym_curr_char != __sym.end()) {
+          __err |= ios_base::failbit;
+          return false;
+        }
+      }
+    } break;
+    case money_base::value: {
+      unsigned __ng = 0;
+      for (; __b != __e; ++__b) {
+        char_type __c = *__b;
+        if (__ct.is(ctype_base::digit, __c)) {
+          if (__wn == __we)
+            std::__double_or_nothing(__wb, __wn, __we);
+          *__wn++ = __c;
+          ++__ng;
+        } else if (__grp.size() > 0 && __ng > 0 && __c == __ts) {
+          if (__gn == __ge)
+            std::__double_or_nothing(__gb, __gn, __ge);
+          *__gn++ = __ng;
+          __ng    = 0;
+        } else
+          break;
+      }
+      if (__gb.get() != __gn && __ng > 0) {
+        if (__gn == __ge)
+          std::__double_or_nothing(__gb, __gn, __ge);
+        *__gn++ = __ng;
+      }
+      if (__fd > 0) {
+        if (__b == __e || *__b != __dp) {
+          __err |= ios_base::failbit;
+          return false;
+        }
+        for (++__b; __fd > 0; --__fd, ++__b) {
+          if (__b == __e || !__ct.is(ctype_base::digit, *__b)) {
+            __err |= ios_base::failbit;
+            return false;
+          }
+          if (__wn == __we)
+            std::__double_or_nothing(__wb, __wn, __we);
+          *__wn++ = *__b;
+        }
+      }
+      if (__wn == __wb.get()) {
+        __err |= ios_base::failbit;
+        return false;
+      }
+    } break;
+    }
+  }
+  if (__trailing_sign) {
+    for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, ++__b) {
+      if (__b == __e || *__b != (*__trailing_sign)[__i]) {
+        __err |= ios_base::failbit;
+        return false;
+      }
+    }
+  }
+  if (__gb.get() != __gn) {
+    ios_base::iostate __et = ios_base::goodbit;
+    __check_grouping(__grp, __gb.get(), __gn, __et);
+    if (__et) {
+      __err |= ios_base::failbit;
+      return false;
+    }
+  }
+  return true;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator money_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+  const int __bz = 100;
+  char_type __wbuf[__bz];
+  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
+  char_type* __wn;
+  char_type* __we              = __wbuf + __bz;
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = false;
+  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
+    const char __src[] = "0123456789";
+    char_type __atoms[sizeof(__src) - 1];
+    __ct.widen(__src, __src + (sizeof(__src) - 1), __atoms);
+    char __nbuf[__bz];
+    char* __nc          = __nbuf;
+    const char* __nc_in = __nc;
+    unique_ptr<char, void (*)(void*)> __h(nullptr, free);
+    if (__wn - __wb.get() > __bz - 2) {
+      __h.reset((char*)malloc(static_cast<size_t>(__wn - __wb.get() + 2)));
+      if (__h.get() == nullptr)
+        std::__throw_bad_alloc();
+      __nc    = __h.get();
+      __nc_in = __nc;
+    }
+    if (__neg)
+      *__nc++ = '-';
+    for (const char_type* __w = __wb.get(); __w < __wn; ++__w, ++__nc)
+      *__nc = __src[std::find(__atoms, std::end(__atoms), *__w) - __atoms];
+    *__nc = char();
+    if (sscanf(__nc_in, "%Lf", &__v) != 1)
+      std::__throw_runtime_error("money_get error");
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator money_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
+  const int __bz = 100;
+  char_type __wbuf[__bz];
+  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
+  char_type* __wn;
+  char_type* __we              = __wbuf + __bz;
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = false;
+  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
+    __v.clear();
+    if (__neg)
+      __v.push_back(__ct.widen('-'));
+    char_type __z = __ct.widen('0');
+    char_type* __w;
+    for (__w = __wb.get(); __w < __wn - 1; ++__w)
+      if (*__w != __z)
+        break;
+    __v.append(__w, __wn);
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<wchar_t>;
+#  endif
+
+// money_put
+
+template <class _CharT>
+class __money_put {
+protected:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI __money_put() {}
+
+  static void __gather_info(
+      bool __intl,
+      bool __neg,
+      const locale& __loc,
+      money_base::pattern& __pat,
+      char_type& __dp,
+      char_type& __ts,
+      string& __grp,
+      string_type& __sym,
+      string_type& __sn,
+      int& __fd);
+  static void __format(
+      char_type* __mb,
+      char_type*& __mi,
+      char_type*& __me,
+      ios_base::fmtflags __flags,
+      const char_type* __db,
+      const char_type* __de,
+      const ctype<char_type>& __ct,
+      bool __neg,
+      const money_base::pattern& __pat,
+      char_type __dp,
+      char_type __ts,
+      const string& __grp,
+      const string_type& __sym,
+      const string_type& __sn,
+      int __fd);
+};
+
+template <class _CharT>
+void __money_put<_CharT>::__gather_info(
+    bool __intl,
+    bool __neg,
+    const locale& __loc,
+    money_base::pattern& __pat,
+    char_type& __dp,
+    char_type& __ts,
+    string& __grp,
+    string_type& __sym,
+    string_type& __sn,
+    int& __fd) {
+  if (__intl) {
+    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
+    if (__neg) {
+      __pat = __mp.neg_format();
+      __sn  = __mp.negative_sign();
+    } else {
+      __pat = __mp.pos_format();
+      __sn  = __mp.positive_sign();
+    }
+    __dp  = __mp.decimal_point();
+    __ts  = __mp.thousands_sep();
+    __grp = __mp.grouping();
+    __sym = __mp.curr_symbol();
+    __fd  = __mp.frac_digits();
+  } else {
+    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
+    if (__neg) {
+      __pat = __mp.neg_format();
+      __sn  = __mp.negative_sign();
+    } else {
+      __pat = __mp.pos_format();
+      __sn  = __mp.positive_sign();
+    }
+    __dp  = __mp.decimal_point();
+    __ts  = __mp.thousands_sep();
+    __grp = __mp.grouping();
+    __sym = __mp.curr_symbol();
+    __fd  = __mp.frac_digits();
+  }
+}
+
+template <class _CharT>
+void __money_put<_CharT>::__format(
+    char_type* __mb,
+    char_type*& __mi,
+    char_type*& __me,
+    ios_base::fmtflags __flags,
+    const char_type* __db,
+    const char_type* __de,
+    const ctype<char_type>& __ct,
+    bool __neg,
+    const money_base::pattern& __pat,
+    char_type __dp,
+    char_type __ts,
+    const string& __grp,
+    const string_type& __sym,
+    const string_type& __sn,
+    int __fd) {
+  __me = __mb;
+  for (char __p : __pat.field) {
+    switch (__p) {
+    case money_base::none:
+      __mi = __me;
+      break;
+    case money_base::space:
+      __mi    = __me;
+      *__me++ = __ct.widen(' ');
+      break;
+    case money_base::sign:
+      if (!__sn.empty())
+        *__me++ = __sn[0];
+      break;
+    case money_base::symbol:
+      if (!__sym.empty() && (__flags & ios_base::showbase))
+        __me = std::copy(__sym.begin(), __sym.end(), __me);
+      break;
+    case money_base::value: {
+      // remember start of value so we can reverse it
+      char_type* __t = __me;
+      // find beginning of digits
+      if (__neg)
+        ++__db;
+      // find end of digits
+      const char_type* __d;
+      for (__d = __db; __d < __de; ++__d)
+        if (!__ct.is(ctype_base::digit, *__d))
+          break;
+      // print fractional part
+      if (__fd > 0) {
+        int __f;
+        for (__f = __fd; __d > __db && __f > 0; --__f)
+          *__me++ = *--__d;
+        char_type __z = __f > 0 ? __ct.widen('0') : char_type();
+        for (; __f > 0; --__f)
+          *__me++ = __z;
+        *__me++ = __dp;
+      }
+      // print units part
+      if (__d == __db) {
+        *__me++ = __ct.widen('0');
+      } else {
+        unsigned __ng = 0;
+        unsigned __ig = 0;
+        unsigned __gl = __grp.empty() ? numeric_limits<unsigned>::max() : static_cast<unsigned>(__grp[__ig]);
+        while (__d != __db) {
+          if (__ng == __gl) {
+            *__me++ = __ts;
+            __ng    = 0;
+            if (++__ig < __grp.size())
+              __gl = __grp[__ig] == numeric_limits<char>::max()
+                       ? numeric_limits<unsigned>::max()
+                       : static_cast<unsigned>(__grp[__ig]);
+          }
+          *__me++ = *--__d;
+          ++__ng;
+        }
+      }
+      // reverse it
+      std::reverse(__t, __me);
+    } break;
+    }
+  }
+  // print rest of sign, if any
+  if (__sn.size() > 1)
+    __me = std::copy(__sn.begin() + 1, __sn.end(), __me);
+  // set alignment
+  if ((__flags & ios_base::adjustfield) == ios_base::left)
+    __mi = __me;
+  else if ((__flags & ios_base::adjustfield) != ios_base::internal)
+    __mi = __mb;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class money_put : public locale::facet, private __money_put<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit money_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
+    return do_put(__s, __intl, __iob, __fl, __units);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
+    return do_put(__s, __intl, __iob, __fl, __digits);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_put() override {}
+
+  virtual iter_type do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const;
+  virtual iter_type
+  do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const;
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id money_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
+  // convert to char
+  const size_t __bs = 100;
+  char __buf[__bs];
+  char* __bb = __buf;
+  char_type __digits[__bs];
+  char_type* __db = __digits;
+  int __n         = snprintf(__bb, __bs, "%.0Lf", __units);
+  unique_ptr<char, void (*)(void*)> __hn(nullptr, free);
+  unique_ptr<char_type, void (*)(void*)> __hd(0, free);
+  // secure memory for digit storage
+  if (static_cast<size_t>(__n) > __bs - 1) {
+    __n = __locale::__asprintf(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units);
+    if (__n == -1)
+      std::__throw_bad_alloc();
+    __hn.reset(__bb);
+    __hd.reset((char_type*)malloc(static_cast<size_t>(__n) * sizeof(char_type)));
+    if (__hd == nullptr)
+      std::__throw_bad_alloc();
+    __db = __hd.get();
+  }
+  // gather info
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  __ct.widen(__bb, __bb + __n, __db);
+  bool __neg = __n > 0 && __bb[0] == '-';
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __sn;
+  int __fd;
+  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  // secure memory for formatting
+  char_type __mbuf[__bs];
+  char_type* __mb = __mbuf;
+  unique_ptr<char_type, void (*)(void*)> __hw(0, free);
+  size_t __exn = __n > __fd ? (static_cast<size_t>(__n) - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() +
+                                  static_cast<size_t>(__fd) + 1
+                            : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
+  if (__exn > __bs) {
+    __hw.reset((char_type*)malloc(__exn * sizeof(char_type)));
+    __mb = __hw.get();
+    if (__mb == 0)
+      std::__throw_bad_alloc();
+  }
+  // format
+  char_type* __mi;
+  char_type* __me;
+  this->__format(
+      __mb, __mi, __me, __iob.flags(), __db, __db + __n, __ct, __neg, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
+  // gather info
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = __digits.size() > 0 && __digits[0] == __ct.widen('-');
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __sn;
+  int __fd;
+  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  // secure memory for formatting
+  char_type __mbuf[100];
+  char_type* __mb = __mbuf;
+  unique_ptr<char_type, void (*)(void*)> __h(0, free);
+  size_t __exn =
+      static_cast<int>(__digits.size()) > __fd
+          ? (__digits.size() - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() + static_cast<size_t>(__fd) +
+                1
+          : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
+  if (__exn > 100) {
+    __h.reset((char_type*)malloc(__exn * sizeof(char_type)));
+    __mb = __h.get();
+    if (__mb == 0)
+      std::__throw_bad_alloc();
+  }
+  // format
+  char_type* __mi;
+  char_type* __me;
+  this->__format(
+      __mb,
+      __mi,
+      __me,
+      __iob.flags(),
+      __digits.data(),
+      __digits.data() + __digits.size(),
+      __ct,
+      __neg,
+      __pat,
+      __dp,
+      __ts,
+      __grp,
+      __sym,
+      __sn,
+      __fd);
+  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_MONEY_H
diff --git a/libcxx/include/__locale_dir/num.h b/libcxx/include/__locale_dir/num.h
new file mode 100644
index 0000000000000..7ca8ffe348959
--- /dev/null
+++ b/libcxx/include/__locale_dir/num.h
@@ -0,0 +1,1072 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_NUM_H
+#define _LIBCPP___LOCALE_DIR_NUM_H
+
+#include <__algorithm/find.h>
+#include <__algorithm/reverse.h>
+#include <__charconv/to_chars_integral.h>
+#include <__charconv/traits.h>
+#include <__config>
+#include <__iterator/istreambuf_iterator.h>
+#include <__iterator/ostreambuf_iterator.h>
+#include <__locale_dir/check_grouping.h>
+#include <__locale_dir/get_c_locale.h>
+#include <__locale_dir/pad_and_output.h>
+#include <__locale_dir/scan_keyword.h>
+#include <__memory/unique_ptr.h>
+#include <__system_error/errc.h>
+#include <cerrno>
+#include <ios>
+#include <streambuf>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+// TODO: Properly qualify calls now that the locale base API defines functions instead of macros
+// NOLINTBEGIN(libcpp-robust-against-adl)
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_EXPORTED_FROM_ABI __num_get_base {
+  static const int __num_get_buf_sz = 40;
+
+  static int __get_base(ios_base&);
+  static const char __src[33]; // "0123456789abcdefABCDEFxX+-pPiInN"
+  // count of leading characters in __src used for parsing integers ("012..X+-")
+  static const size_t __int_chr_cnt = 26;
+  // count of leading characters in __src used for parsing floating-point values ("012..-pP")
+  static const size_t __fp_chr_cnt = 28;
+};
+
+template <class _CharT>
+struct __num_get : protected __num_get_base {
+  static string __stage2_float_prep(ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep);
+
+  static int __stage2_float_loop(
+      _CharT __ct,
+      bool& __in_units,
+      char& __exp,
+      char* __a,
+      char*& __a_end,
+      _CharT __decimal_point,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      unsigned& __dc,
+      _CharT* __atoms);
+
+  [[__deprecated__("This exists only for ABI compatibility")]] static string
+  __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
+  static int __stage2_int_loop(
+      _CharT __ct,
+      int __base,
+      char* __a,
+      char*& __a_end,
+      unsigned& __dc,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      _CharT* __atoms);
+
+  _LIBCPP_HIDE_FROM_ABI static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
+    locale __loc                 = __iob.getloc();
+    const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
+    __thousands_sep              = __np.thousands_sep();
+    return __np.grouping();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const {
+    return __do_widen_p(__iob, __atoms);
+  }
+
+private:
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
+    locale __loc = __iob.getloc();
+    use_facet<ctype<_Tp> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
+    return __atoms;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
+    (void)__iob;
+    (void)__atoms;
+    return __src;
+  }
+};
+
+template <class _CharT>
+string __num_get<_CharT>::__stage2_float_prep(
+    ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep) {
+  locale __loc = __iob.getloc();
+  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __fp_chr_cnt, __atoms);
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
+  __decimal_point              = __np.decimal_point();
+  __thousands_sep              = __np.thousands_sep();
+  return __np.grouping();
+}
+
+template <class _CharT>
+int __num_get<_CharT>::__stage2_int_loop(
+    _CharT __ct,
+    int __base,
+    char* __a,
+    char*& __a_end,
+    unsigned& __dc,
+    _CharT __thousands_sep,
+    const string& __grouping,
+    unsigned* __g,
+    unsigned*& __g_end,
+    _CharT* __atoms) {
+  if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
+    *__a_end++ = __ct == __atoms[24] ? '+' : '-';
+    __dc       = 0;
+    return 0;
+  }
+  if (__grouping.size() != 0 && __ct == __thousands_sep) {
+    if (__g_end - __g < __num_get_buf_sz) {
+      *__g_end++ = __dc;
+      __dc       = 0;
+    }
+    return 0;
+  }
+  ptrdiff_t __f = std::find(__atoms, __atoms + __int_chr_cnt, __ct) - __atoms;
+  if (__f >= 24)
+    return -1;
+  switch (__base) {
+  case 8:
+  case 10:
+    if (__f >= __base)
+      return -1;
+    break;
+  case 16:
+    if (__f < 22)
+      break;
+    if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
+      __dc       = 0;
+      *__a_end++ = __src[__f];
+      return 0;
+    }
+    return -1;
+  }
+  *__a_end++ = __src[__f];
+  ++__dc;
+  return 0;
+}
+
+template <class _CharT>
+int __num_get<_CharT>::__stage2_float_loop(
+    _CharT __ct,
+    bool& __in_units,
+    char& __exp,
+    char* __a,
+    char*& __a_end,
+    _CharT __decimal_point,
+    _CharT __thousands_sep,
+    const string& __grouping,
+    unsigned* __g,
+    unsigned*& __g_end,
+    unsigned& __dc,
+    _CharT* __atoms) {
+  if (__ct == __decimal_point) {
+    if (!__in_units)
+      return -1;
+    __in_units = false;
+    *__a_end++ = '.';
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
+      *__g_end++ = __dc;
+    return 0;
+  }
+  if (__ct == __thousands_sep && __grouping.size() != 0) {
+    if (!__in_units)
+      return -1;
+    if (__g_end - __g < __num_get_buf_sz) {
+      *__g_end++ = __dc;
+      __dc       = 0;
+    }
+    return 0;
+  }
+  ptrdiff_t __f = std::find(__atoms, __atoms + __num_get_base::__fp_chr_cnt, __ct) - __atoms;
+  if (__f >= static_cast<ptrdiff_t>(__num_get_base::__fp_chr_cnt))
+    return -1;
+  char __x = __src[__f];
+  if (__x == '-' || __x == '+') {
+    if (__a_end == __a || (std::toupper(__a_end[-1]) == std::toupper(__exp))) {
+      *__a_end++ = __x;
+      return 0;
+    }
+    return -1;
+  }
+  if (__x == 'x' || __x == 'X')
+    __exp = 'P';
+  else if (std::toupper(__x) == __exp) {
+    __exp = std::tolower(__exp);
+    if (__in_units) {
+      __in_units = false;
+      if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
+        *__g_end++ = __dc;
+    }
+  }
+  *__a_end++ = __x;
+  if (__f >= 22)
+    return 0;
+  ++__dc;
+  return 0;
+}
+
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<wchar_t>;
+#  endif
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __do_strtod(const char* __a, char** __p2);
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI float __do_strtod<float>(const char* __a, char** __p2) {
+  return __locale::__strtof(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI double __do_strtod<double>(const char* __a, char** __p2) {
+  return __locale::__strtod(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI long double __do_strtod<long double>(const char* __a, char** __p2) {
+  return __locale::__strtold(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __num_get_float(const char* __a, const char* __a_end, ios_base::iostate& __err) {
+  if (__a != __a_end) {
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    _Tp __ld                                                     = std::__do_strtod<_Tp>(__a, &__p2);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE)
+      __err = ios_base::failbit;
+    return __ld;
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
+  if (__a != __a_end) {
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    long long __ll = __locale::__strtoll(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE || __ll < numeric_limits<_Tp>::min() || numeric_limits<_Tp>::max() < __ll) {
+      __err = ios_base::failbit;
+      if (__ll > 0)
+        return numeric_limits<_Tp>::max();
+      else
+        return numeric_limits<_Tp>::min();
+    }
+    return static_cast<_Tp>(__ll);
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
+  if (__a != __a_end) {
+    const bool __negate = *__a == '-';
+    if (__negate && ++__a == __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    }
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    unsigned long long __ll = __locale::__strtoull(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE || numeric_limits<_Tp>::max() < __ll) {
+      __err = ios_base::failbit;
+      return numeric_limits<_Tp>::max();
+    }
+    _Tp __res = static_cast<_Tp>(__ll);
+    if (__negate)
+      __res = -__res;
+    return __res;
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class num_get : public locale::facet, private __num_get<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit num_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_get() override {}
+
+  template <class _Fp>
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  __do_get_floating_point(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Fp& __v) const {
+    // Stage 1, nothing to do
+    // Stage 2
+    char_type __atoms[__num_get_base::__fp_chr_cnt];
+    char_type __decimal_point;
+    char_type __thousands_sep;
+    string __grouping = this->__stage2_float_prep(__iob, __atoms, __decimal_point, __thousands_sep);
+    string __buf;
+    __buf.resize(__buf.capacity());
+    char* __a     = &__buf[0];
+    char* __a_end = __a;
+    unsigned __g[__num_get_base::__num_get_buf_sz];
+    unsigned* __g_end        = __g;
+    unsigned __dc            = 0;
+    bool __in_units          = true;
+    char __exp               = 'E';
+    bool __is_leading_parsed = false;
+    for (; __b != __e; ++__b) {
+      if (__a_end == __a + __buf.size()) {
+        size_t __tmp = __buf.size();
+        __buf.resize(2 * __buf.size());
+        __buf.resize(__buf.capacity());
+        __a     = &__buf[0];
+        __a_end = __a + __tmp;
+      }
+      if (this->__stage2_float_loop(
+              *__b,
+              __in_units,
+              __exp,
+              __a,
+              __a_end,
+              __decimal_point,
+              __thousands_sep,
+              __grouping,
+              __g,
+              __g_end,
+              __dc,
+              __atoms))
+        break;
+
+      // the leading character excluding the sign must be a decimal digit
+      if (!__is_leading_parsed) {
+        if (__a_end - __a >= 1 && __a[0] != '-' && __a[0] != '+') {
+          if (('0' <= __a[0] && __a[0] <= '9') || __a[0] == '.')
+            __is_leading_parsed = true;
+          else
+            break;
+        } else if (__a_end - __a >= 2 && (__a[0] == '-' || __a[0] == '+')) {
+          if (('0' <= __a[1] && __a[1] <= '9') || __a[1] == '.')
+            __is_leading_parsed = true;
+          else
+            break;
+        }
+      }
+    }
+    if (__grouping.size() != 0 && __in_units && __g_end - __g < __num_get_base::__num_get_buf_sz)
+      *__g_end++ = __dc;
+    // Stage 3
+    __v = std::__num_get_float<_Fp>(__a, __a_end, __err);
+    // Digit grouping checked
+    __check_grouping(__grouping, __g, __g_end, __err);
+    // EOF checked
+    if (__b == __e)
+      __err |= ios_base::eofbit;
+    return __b;
+  }
+
+  template <class _Signed>
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  __do_get_signed(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const {
+    // Stage 1
+    int __base = this->__get_base(__iob);
+    // Stage 2
+    char_type __thousands_sep;
+    const int __atoms_size = __num_get_base::__int_chr_cnt;
+    char_type __atoms1[__atoms_size];
+    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
+    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
+    string __buf;
+    __buf.resize(__buf.capacity());
+    char* __a     = &__buf[0];
+    char* __a_end = __a;
+    unsigned __g[__num_get_base::__num_get_buf_sz];
+    unsigned* __g_end = __g;
+    unsigned __dc     = 0;
+    for (; __b != __e; ++__b) {
+      if (__a_end == __a + __buf.size()) {
+        size_t __tmp = __buf.size();
+        __buf.resize(2 * __buf.size());
+        __buf.resize(__buf.capacity());
+        __a     = &__buf[0];
+        __a_end = __a + __tmp;
+      }
+      if (this->__stage2_int_loop(
+              *__b,
+              __base,
+              __a,
+              __a_end,
+              __dc,
+              __thousands_sep,
+              __grouping,
+              __g,
+              __g_end,
+              const_cast<char_type*>(__atoms)))
+        break;
+    }
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
+      *__g_end++ = __dc;
+    // Stage 3
+    __v = std::__num_get_signed_integral<_Signed>(__a, __a_end, __err, __base);
+    // Digit grouping checked
+    __check_grouping(__grouping, __g, __g_end, __err);
+    // EOF checked
+    if (__b == __e)
+      __err |= ios_base::eofbit;
+    return __b;
+  }
+
+  template <class _Unsigned>
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  __do_get_unsigned(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const {
+    // Stage 1
+    int __base = this->__get_base(__iob);
+    // Stage 2
+    char_type __thousands_sep;
+    const int __atoms_size = __num_get_base::__int_chr_cnt;
+    char_type __atoms1[__atoms_size];
+    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
+    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
+    string __buf;
+    __buf.resize(__buf.capacity());
+    char* __a     = &__buf[0];
+    char* __a_end = __a;
+    unsigned __g[__num_get_base::__num_get_buf_sz];
+    unsigned* __g_end = __g;
+    unsigned __dc     = 0;
+    for (; __b != __e; ++__b) {
+      if (__a_end == __a + __buf.size()) {
+        size_t __tmp = __buf.size();
+        __buf.resize(2 * __buf.size());
+        __buf.resize(__buf.capacity());
+        __a     = &__buf[0];
+        __a_end = __a + __tmp;
+      }
+      if (this->__stage2_int_loop(
+              *__b,
+              __base,
+              __a,
+              __a_end,
+              __dc,
+              __thousands_sep,
+              __grouping,
+              __g,
+              __g_end,
+              const_cast<char_type*>(__atoms)))
+        break;
+    }
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
+      *__g_end++ = __dc;
+    // Stage 3
+    __v = std::__num_get_unsigned_integral<_Unsigned>(__a, __a_end, __err, __base);
+    // Digit grouping checked
+    __check_grouping(__grouping, __g, __g_end, __err);
+    // EOF checked
+    if (__b == __e)
+      __err |= ios_base::eofbit;
+    return __b;
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const;
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
+    return this->__do_get_signed(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
+    return this->__do_get_signed(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const;
+};
+
+template <class _CharT, class _InputIterator>
+locale::id num_get<_CharT, _InputIterator>::id;
+
+template <class _CharT, class _InputIterator>
+_InputIterator num_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
+  if ((__iob.flags() & ios_base::boolalpha) == 0) {
+    long __lv = -1;
+    __b       = do_get(__b, __e, __iob, __err, __lv);
+    switch (__lv) {
+    case 0:
+      __v = false;
+      break;
+    case 1:
+      __v = true;
+      break;
+    default:
+      __v   = true;
+      __err = ios_base::failbit;
+      break;
+    }
+    return __b;
+  }
+  const ctype<_CharT>& __ct    = std::use_facet<ctype<_CharT> >(__iob.getloc());
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__iob.getloc());
+  typedef typename numpunct<_CharT>::string_type string_type;
+  const string_type __names[2] = {__np.truename(), __np.falsename()};
+  const string_type* __i       = std::__scan_keyword(__b, __e, __names, __names + 2, __ct, __err);
+  __v                          = __i == __names;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator num_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
+  // Stage 1
+  int __base = 16;
+  // Stage 2
+  char_type __atoms[__num_get_base::__int_chr_cnt];
+  char_type __thousands_sep = char_type();
+  string __grouping;
+  std::use_facet<ctype<_CharT> >(__iob.getloc())
+      .widen(__num_get_base::__src, __num_get_base::__src + __num_get_base::__int_chr_cnt, __atoms);
+  string __buf;
+  __buf.resize(__buf.capacity());
+  char* __a     = &__buf[0];
+  char* __a_end = __a;
+  unsigned __g[__num_get_base::__num_get_buf_sz];
+  unsigned* __g_end = __g;
+  unsigned __dc     = 0;
+  for (; __b != __e; ++__b) {
+    if (__a_end == __a + __buf.size()) {
+      size_t __tmp = __buf.size();
+      __buf.resize(2 * __buf.size());
+      __buf.resize(__buf.capacity());
+      __a     = &__buf[0];
+      __a_end = __a + __tmp;
+    }
+    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+      break;
+  }
+  // Stage 3
+  __buf.resize(__a_end - __a);
+  if (__locale::__sscanf(__buf.c_str(), _LIBCPP_GET_C_LOCALE, "%p", &__v) != 1)
+    __err = ios_base::failbit;
+  // EOF checked
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<wchar_t>;
+#  endif
+
+struct _LIBCPP_EXPORTED_FROM_ABI __num_put_base {
+protected:
+  static void __format_int(char* __fmt, const char* __len, bool __signd, ios_base::fmtflags __flags);
+  static bool __format_float(char* __fmt, const char* __len, ios_base::fmtflags __flags);
+  static char* __identify_padding(char* __nb, char* __ne, const ios_base& __iob);
+};
+
+template <class _CharT>
+struct __num_put : protected __num_put_base {
+  static void __widen_and_group_int(
+      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
+  static void __widen_and_group_float(
+      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
+};
+
+template <class _CharT>
+void __num_put<_CharT>::__widen_and_group_int(
+    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
+  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
+  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
+  string __grouping             = __npt.grouping();
+  if (__grouping.empty()) {
+    __ct.widen(__nb, __ne, __ob);
+    __oe = __ob + (__ne - __nb);
+  } else {
+    __oe       = __ob;
+    char* __nf = __nb;
+    if (*__nf == '-' || *__nf == '+')
+      *__oe++ = __ct.widen(*__nf++);
+    if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
+      *__oe++ = __ct.widen(*__nf++);
+      *__oe++ = __ct.widen(*__nf++);
+    }
+    std::reverse(__nf, __ne);
+    _CharT __thousands_sep = __npt.thousands_sep();
+    unsigned __dc          = 0;
+    unsigned __dg          = 0;
+    for (char* __p = __nf; __p < __ne; ++__p) {
+      if (static_cast<unsigned>(__grouping[__dg]) > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
+        *__oe++ = __thousands_sep;
+        __dc    = 0;
+        if (__dg < __grouping.size() - 1)
+          ++__dg;
+      }
+      *__oe++ = __ct.widen(*__p);
+      ++__dc;
+    }
+    std::reverse(__ob + (__nf - __nb), __oe);
+  }
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __ob + (__np - __nb);
+}
+
+template <class _CharT>
+void __num_put<_CharT>::__widen_and_group_float(
+    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
+  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
+  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
+  string __grouping             = __npt.grouping();
+  __oe                          = __ob;
+  char* __nf                    = __nb;
+  if (*__nf == '-' || *__nf == '+')
+    *__oe++ = __ct.widen(*__nf++);
+  char* __ns;
+  if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
+    *__oe++ = __ct.widen(*__nf++);
+    *__oe++ = __ct.widen(*__nf++);
+    for (__ns = __nf; __ns < __ne; ++__ns)
+      if (!__locale::__isxdigit(*__ns, _LIBCPP_GET_C_LOCALE))
+        break;
+  } else {
+    for (__ns = __nf; __ns < __ne; ++__ns)
+      if (!__locale::__isdigit(*__ns, _LIBCPP_GET_C_LOCALE))
+        break;
+  }
+  if (__grouping.empty()) {
+    __ct.widen(__nf, __ns, __oe);
+    __oe += __ns - __nf;
+  } else {
+    std::reverse(__nf, __ns);
+    _CharT __thousands_sep = __npt.thousands_sep();
+    unsigned __dc          = 0;
+    unsigned __dg          = 0;
+    for (char* __p = __nf; __p < __ns; ++__p) {
+      if (__grouping[__dg] > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
+        *__oe++ = __thousands_sep;
+        __dc    = 0;
+        if (__dg < __grouping.size() - 1)
+          ++__dg;
+      }
+      *__oe++ = __ct.widen(*__p);
+      ++__dc;
+    }
+    std::reverse(__ob + (__nf - __nb), __oe);
+  }
+  for (__nf = __ns; __nf < __ne; ++__nf) {
+    if (*__nf == '.') {
+      *__oe++ = __npt.decimal_point();
+      ++__nf;
+      break;
+    } else
+      *__oe++ = __ct.widen(*__nf);
+  }
+  __ct.widen(__nf, __ne, __oe);
+  __oe += __ne - __nf;
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __ob + (__np - __nb);
+}
+
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class num_put : public locale::facet, private __num_put<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit num_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_put() override {}
+
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const;
+
+  template <class _Integral>
+  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
+  __do_put_integral(iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const;
+
+  template <class _Float>
+  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
+  __do_put_floating_point(iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const;
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id num_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
+  if ((__iob.flags() & ios_base::boolalpha) == 0)
+    return do_put(__s, __iob, __fl, (unsigned long)__v);
+  const numpunct<char_type>& __np = std::use_facet<numpunct<char_type> >(__iob.getloc());
+  typedef typename numpunct<char_type>::string_type string_type;
+  string_type __nm = __v ? __np.truename() : __np.falsename();
+  for (typename string_type::iterator __i = __nm.begin(); __i != __nm.end(); ++__i, ++__s)
+    *__s = *__i;
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+template <class _Integral>
+_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_integral(
+    iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const {
+  // Stage 1 - Get number in narrow char
+
+  // Worst case is octal, with showbase enabled. Note that octal is always
+  // printed as an unsigned value.
+  using _Unsigned = typename make_unsigned<_Integral>::type;
+  _LIBCPP_CONSTEXPR const unsigned __buffer_size =
+      (numeric_limits<_Unsigned>::digits / 3)          // 1 char per 3 bits
+      + ((numeric_limits<_Unsigned>::digits % 3) != 0) // round up
+      + 2;                                             // base prefix + terminating null character
+
+  char __char_buffer[__buffer_size];
+  char* __buffer_ptr = __char_buffer;
+
+  auto __flags = __iob.flags();
+
+  auto __basefield = (__flags & ios_base::basefield);
+
+  // Extract base
+  int __base = 10;
+  if (__basefield == ios_base::oct)
+    __base = 8;
+  else if (__basefield == ios_base::hex)
+    __base = 16;
+
+  // Print '-' and make the argument unsigned
+  auto __uval = std::__to_unsigned_like(__v);
+  if (__basefield != ios_base::oct && __basefield != ios_base::hex && __v < 0) {
+    *__buffer_ptr++ = '-';
+    __uval          = std::__complement(__uval);
+  }
+
+  // Maybe add '+' prefix
+  if (std::is_signed<_Integral>::value && (__flags & ios_base::showpos) && __basefield != ios_base::oct &&
+      __basefield != ios_base::hex && __v >= 0)
+    *__buffer_ptr++ = '+';
+
+  // Add base prefix
+  if (__v != 0 && __flags & ios_base::showbase) {
+    if (__basefield == ios_base::oct) {
+      *__buffer_ptr++ = '0';
+    } else if (__basefield == ios_base::hex) {
+      *__buffer_ptr++ = '0';
+      *__buffer_ptr++ = (__flags & ios_base::uppercase ? 'X' : 'x');
+    }
+  }
+
+  auto __res = std::__to_chars_integral(__buffer_ptr, __char_buffer + __buffer_size, __uval, __base);
+  _LIBCPP_ASSERT_INTERNAL(__res.__ec == std::errc(0), "to_chars: invalid maximum buffer size computed?");
+
+  // Make letters uppercase
+  if (__flags & ios_base::hex && __flags & ios_base::uppercase) {
+    for (; __buffer_ptr != __res.__ptr; ++__buffer_ptr)
+      *__buffer_ptr = std::__hex_to_upper(*__buffer_ptr);
+  }
+
+  char* __np = this->__identify_padding(__char_buffer, __res.__ptr, __iob);
+  // Stage 2 - Widen __nar while adding thousands separators
+  char_type __o[2 * (__buffer_size - 1) - 1];
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  this->__widen_and_group_int(__char_buffer, __np, __res.__ptr, __o, __op, __oe, __iob.getloc());
+  // [__o, __oe) contains thousands_sep'd wide number
+  // Stage 3 & 4
+  return std::__pad_and_output(__s, __o, __op, __oe, __iob, __fl);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+template <class _Float>
+_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_floating_point(
+    iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const {
+  // Stage 1 - Get number in narrow char
+  char __fmt[8]            = {'%', 0};
+  bool __specify_precision = this->__format_float(__fmt + 1, __len, __iob.flags());
+  const unsigned __nbuf    = 30;
+  char __nar[__nbuf];
+  char* __nb = __nar;
+  int __nc;
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  if (__specify_precision)
+    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
+  else
+    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
+  unique_ptr<char, void (*)(void*)> __nbh(nullptr, free);
+  if (__nc > static_cast<int>(__nbuf - 1)) {
+    if (__specify_precision)
+      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
+    else
+      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
+    if (__nc == -1)
+      std::__throw_bad_alloc();
+    __nbh.reset(__nb);
+  }
+  _LIBCPP_DIAGNOSTIC_POP
+  char* __ne = __nb + __nc;
+  char* __np = this->__identify_padding(__nb, __ne, __iob);
+  // Stage 2 - Widen __nar while adding thousands separators
+  char_type __o[2 * (__nbuf - 1) - 1];
+  char_type* __ob = __o;
+  unique_ptr<char_type, void (*)(void*)> __obh(0, free);
+  if (__nb != __nar) {
+    __ob = (char_type*)malloc(2 * static_cast<size_t>(__nc) * sizeof(char_type));
+    if (__ob == 0)
+      std::__throw_bad_alloc();
+    __obh.reset(__ob);
+  }
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  this->__widen_and_group_float(__nb, __np, __ne, __ob, __op, __oe, __iob.getloc());
+  // [__o, __oe) contains thousands_sep'd wide number
+  // Stage 3 & 4
+  __s = std::__pad_and_output(__s, __ob, __op, __oe, __iob, __fl);
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
+  return this->__do_put_floating_point(__s, __iob, __fl, __v, "");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
+  return this->__do_put_floating_point(__s, __iob, __fl, __v, "L");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
+  auto __flags = __iob.flags();
+  __iob.flags((__flags & ~ios_base::basefield & ~ios_base::uppercase) | ios_base::hex | ios_base::showbase);
+  auto __res = __do_put_integral(__s, __iob, __fl, reinterpret_cast<uintptr_t>(__v));
+  __iob.flags(__flags);
+  return __res;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+// NOLINTEND(libcpp-robust-against-adl)
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_NUM_H
diff --git a/libcxx/include/__locale_dir/scan_keyword.h b/libcxx/include/__locale_dir/scan_keyword.h
new file mode 100644
index 0000000000000..78dd0a46a8ee1
--- /dev/null
+++ b/libcxx/include/__locale_dir/scan_keyword.h
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_SCAN_KEYWORD_H
+#define _LIBCPP___LOCALE_DIR_SCAN_KEYWORD_H
+
+#include <__config>
+#include <__memory/unique_ptr.h>
+#include <ios>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __scan_keyword
+// Scans [__b, __e) until a match is found in the basic_strings range
+//  [__kb, __ke) or until it can be shown that there is no match in [__kb, __ke).
+//  __b will be incremented (visibly), consuming CharT until a match is found
+//  or proved to not exist.  A keyword may be "", in which will match anything.
+//  If one keyword is a prefix of another, and the next CharT in the input
+//  might match another keyword, the algorithm will attempt to find the longest
+//  matching keyword.  If the longer matching keyword ends up not matching, then
+//  no keyword match is found.  If no keyword match is found, __ke is returned
+//  and failbit is set in __err.
+//  Else an iterator pointing to the matching keyword is found.  If more than
+//  one keyword matches, an iterator to the first matching keyword is returned.
+//  If on exit __b == __e, eofbit is set in __err.  If __case_sensitive is false,
+//  __ct is used to force to lower case before comparing characters.
+//  Examples:
+//  Keywords:  "a", "abb"
+//  If the input is "a", the first keyword matches and eofbit is set.
+//  If the input is "abc", no match is found and "ab" are consumed.
+template <class _InputIterator, class _ForwardIterator, class _Ctype>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator __scan_keyword(
+    _InputIterator& __b,
+    _InputIterator __e,
+    _ForwardIterator __kb,
+    _ForwardIterator __ke,
+    const _Ctype& __ct,
+    ios_base::iostate& __err,
+    bool __case_sensitive = true) {
+  typedef typename iterator_traits<_InputIterator>::value_type _CharT;
+  size_t __nkw                       = static_cast<size_t>(std::distance(__kb, __ke));
+  const unsigned char __doesnt_match = '\0';
+  const unsigned char __might_match  = '\1';
+  const unsigned char __does_match   = '\2';
+  unsigned char __statbuf[100];
+  unsigned char* __status = __statbuf;
+  unique_ptr<unsigned char, void (*)(void*)> __stat_hold(nullptr, free);
+  if (__nkw > sizeof(__statbuf)) {
+    __status = (unsigned char*)malloc(__nkw);
+    if (__status == nullptr)
+      std::__throw_bad_alloc();
+    __stat_hold.reset(__status);
+  }
+  size_t __n_might_match = __nkw; // At this point, any keyword might match
+  size_t __n_does_match  = 0;     // but none of them definitely do
+  // Initialize all statuses to __might_match, except for "" keywords are __does_match
+  unsigned char* __st = __status;
+  for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+    if (!__ky->empty())
+      *__st = __might_match;
+    else {
+      *__st = __does_match;
+      --__n_might_match;
+      ++__n_does_match;
+    }
+  }
+  // While there might be a match, test keywords against the next CharT
+  for (size_t __indx = 0; __b != __e && __n_might_match > 0; ++__indx) {
+    // Peek at the next CharT but don't consume it
+    _CharT __c = *__b;
+    if (!__case_sensitive)
+      __c = __ct.toupper(__c);
+    bool __consume = false;
+    // For each keyword which might match, see if the __indx character is __c
+    // If a match if found, consume __c
+    // If a match is found, and that is the last character in the keyword,
+    //    then that keyword matches.
+    // If the keyword doesn't match this character, then change the keyword
+    //    to doesn't match
+    __st = __status;
+    for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+      if (*__st == __might_match) {
+        _CharT __kc = (*__ky)[__indx];
+        if (!__case_sensitive)
+          __kc = __ct.toupper(__kc);
+        if (__c == __kc) {
+          __consume = true;
+          if (__ky->size() == __indx + 1) {
+            *__st = __does_match;
+            --__n_might_match;
+            ++__n_does_match;
+          }
+        } else {
+          *__st = __doesnt_match;
+          --__n_might_match;
+        }
+      }
+    }
+    // consume if we matched a character
+    if (__consume) {
+      ++__b;
+      // If we consumed a character and there might be a matched keyword that
+      //   was marked matched on a previous iteration, then such keywords
+      //   which are now marked as not matching.
+      if (__n_might_match + __n_does_match > 1) {
+        __st = __status;
+        for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+          if (*__st == __does_match && __ky->size() != __indx + 1) {
+            *__st = __doesnt_match;
+            --__n_does_match;
+          }
+        }
+      }
+    }
+  }
+  // We've exited the loop because we hit eof and/or we have no more "might matches".
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  // Return the first matching result
+  for (__st = __status; __kb != __ke; ++__kb, (void)++__st)
+    if (*__st == __does_match)
+      break;
+  if (__kb == __ke)
+    __err |= ios_base::failbit;
+  return __kb;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_SCAN_KEYWORD_H
diff --git a/libcxx/include/__locale_dir/time.h b/libcxx/include/__locale_dir/time.h
new file mode 100644
index 0000000000000..5f60d5f36b349
--- /dev/null
+++ b/libcxx/include/__locale_dir/time.h
@@ -0,0 +1,766 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_TIME_H
+#define _LIBCPP___LOCALE_DIR_TIME_H
+
+#include <__algorithm/copy.h>
+#include <__config>
+#include <__locale_dir/get_c_locale.h>
+#include <__locale_dir/scan_keyword.h>
+#include <ios>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _InputIterator>
+_LIBCPP_HIDE_FROM_ABI int __get_up_to_n_digits(
+    _InputIterator& __b, _InputIterator __e, ios_base::iostate& __err, const ctype<_CharT>& __ct, int __n) {
+  // Precondition:  __n >= 1
+  if (__b == __e) {
+    __err |= ios_base::eofbit | ios_base::failbit;
+    return 0;
+  }
+  // get first digit
+  _CharT __c = *__b;
+  if (!__ct.is(ctype_base::digit, __c)) {
+    __err |= ios_base::failbit;
+    return 0;
+  }
+  int __r = __ct.narrow(__c, 0) - '0';
+  for (++__b, (void)--__n; __b != __e && __n > 0; ++__b, (void)--__n) {
+    // get next digit
+    __c = *__b;
+    if (!__ct.is(ctype_base::digit, __c))
+      return __r;
+    __r = __r * 10 + __ct.narrow(__c, 0) - '0';
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __r;
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI time_base {
+public:
+  enum dateorder { no_order, dmy, mdy, ymd, ydm };
+};
+
+template <class _CharT>
+class __time_get_c_storage {
+protected:
+  typedef basic_string<_CharT> string_type;
+
+  virtual const string_type* __weeks() const;
+  virtual const string_type* __months() const;
+  virtual const string_type* __am_pm() const;
+  virtual const string_type& __c() const;
+  virtual const string_type& __r() const;
+  virtual const string_type& __x() const;
+  virtual const string_type& __X() const;
+
+  _LIBCPP_HIDE_FROM_ABI ~__time_get_c_storage() {}
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__weeks() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__months() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__am_pm() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__c() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__r() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__x() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__X() const;
+
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__weeks() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__months() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__am_pm() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__c() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__r() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__x() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__X() const;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class time_get : public locale::facet, public time_base, private __time_get_c_storage<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+  typedef time_base::dateorder dateorder;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI dateorder date_order() const { return this->do_date_order(); }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_time(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_date(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_weekday(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_monthname(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_year(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod = 0)
+      const {
+    return do_get(__b, __e, __iob, __err, __tm, __fmt, __mod);
+  }
+
+  iter_type
+  get(iter_type __b,
+      iter_type __e,
+      ios_base& __iob,
+      ios_base::iostate& __err,
+      tm* __tm,
+      const char_type* __fmtb,
+      const char_type* __fmte) const;
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get() override {}
+
+  virtual dateorder do_date_order() const;
+  virtual iter_type
+  do_get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type do_get(
+      iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod) const;
+
+private:
+  void __get_white_space(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_percent(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+
+  void __get_weekdayname(
+      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_monthname(
+      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_day(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_month(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_year(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_year4(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_hour(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_12_hour(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_am_pm(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_minute(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_second(int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_weekday(int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_day_year_num(
+      int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+};
+
+template <class _CharT, class _InputIterator>
+locale::id time_get<_CharT, _InputIterator>::id;
+
+// time_get primitives
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_weekdayname(
+    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  // Note:  ignoring case comes from the POSIX strptime spec
+  const string_type* __wk = this->__weeks();
+  ptrdiff_t __i           = std::__scan_keyword(__b, __e, __wk, __wk + 14, __ct, __err, false) - __wk;
+  if (__i < 14)
+    __w = __i % 7;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_monthname(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  // Note:  ignoring case comes from the POSIX strptime spec
+  const string_type* __month = this->__months();
+  ptrdiff_t __i              = std::__scan_keyword(__b, __e, __month, __month + 24, __ct, __err, false) - __month;
+  if (__i < 24)
+    __m = __i % 12;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_day(
+    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 31)
+    __d = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_month(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2) - 1;
+  if (!(__err & ios_base::failbit) && 0 <= __t && __t <= 11)
+    __m = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_year(
+    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
+  if (!(__err & ios_base::failbit)) {
+    if (__t < 69)
+      __t += 2000;
+    else if (69 <= __t && __t <= 99)
+      __t += 1900;
+    __y = __t - 1900;
+  }
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_year4(
+    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
+  if (!(__err & ios_base::failbit))
+    __y = __t - 1900;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_hour(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 23)
+    __h = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_12_hour(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 12)
+    __h = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_minute(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 59)
+    __m = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_second(
+    int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 60)
+    __s = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_weekday(
+    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 1);
+  if (!(__err & ios_base::failbit) && __t <= 6)
+    __w = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_day_year_num(
+    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 3);
+  if (!(__err & ios_base::failbit) && __t <= 365)
+    __d = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_white_space(
+    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
+    ;
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_am_pm(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  const string_type* __ap = this->__am_pm();
+  if (__ap[0].size() + __ap[1].size() == 0) {
+    __err |= ios_base::failbit;
+    return;
+  }
+  ptrdiff_t __i = std::__scan_keyword(__b, __e, __ap, __ap + 2, __ct, __err, false) - __ap;
+  if (__i == 0 && __h == 12)
+    __h = 0;
+  else if (__i == 1 && __h < 12)
+    __h += 12;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_percent(
+    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  if (__b == __e) {
+    __err |= ios_base::eofbit | ios_base::failbit;
+    return;
+  }
+  if (__ct.narrow(*__b, 0) != '%')
+    __err |= ios_base::failbit;
+  else if (++__b == __e)
+    __err |= ios_base::eofbit;
+}
+
+// time_get end primitives
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::get(
+    iter_type __b,
+    iter_type __e,
+    ios_base& __iob,
+    ios_base::iostate& __err,
+    tm* __tm,
+    const char_type* __fmtb,
+    const char_type* __fmte) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __err                        = ios_base::goodbit;
+  while (__fmtb != __fmte && __err == ios_base::goodbit) {
+    if (__b == __e) {
+      __err = ios_base::failbit;
+      break;
+    }
+    if (__ct.narrow(*__fmtb, 0) == '%') {
+      if (++__fmtb == __fmte) {
+        __err = ios_base::failbit;
+        break;
+      }
+      char __cmd = __ct.narrow(*__fmtb, 0);
+      char __opt = '\0';
+      if (__cmd == 'E' || __cmd == '0') {
+        if (++__fmtb == __fmte) {
+          __err = ios_base::failbit;
+          break;
+        }
+        __opt = __cmd;
+        __cmd = __ct.narrow(*__fmtb, 0);
+      }
+      __b = do_get(__b, __e, __iob, __err, __tm, __cmd, __opt);
+      ++__fmtb;
+    } else if (__ct.is(ctype_base::space, *__fmtb)) {
+      for (++__fmtb; __fmtb != __fmte && __ct.is(ctype_base::space, *__fmtb); ++__fmtb)
+        ;
+      for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
+        ;
+    } else if (__ct.toupper(*__b) == __ct.toupper(*__fmtb)) {
+      ++__b;
+      ++__fmtb;
+    } else
+      __err = ios_base::failbit;
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+typename time_get<_CharT, _InputIterator>::dateorder time_get<_CharT, _InputIterator>::do_date_order() const {
+  return mdy;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_time(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const char_type __fmt[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
+  return get(__b, __e, __iob, __err, __tm, __fmt, __fmt + sizeof(__fmt) / sizeof(__fmt[0]));
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_date(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const string_type& __fmt = this->__x();
+  return get(__b, __e, __iob, __err, __tm, __fmt.data(), __fmt.data() + __fmt.size());
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_weekday(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_monthname(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_year(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_year(__tm->tm_year, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char) const {
+  __err                        = ios_base::goodbit;
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  switch (__fmt) {
+  case 'a':
+  case 'A':
+    __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
+    break;
+  case 'b':
+  case 'B':
+  case 'h':
+    __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
+    break;
+  case 'c': {
+    const string_type& __fm = this->__c();
+    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
+  } break;
+  case 'd':
+  case 'e':
+    __get_day(__tm->tm_mday, __b, __e, __err, __ct);
+    break;
+  case 'D': {
+    const char_type __fm[] = {'%', 'm', '/', '%', 'd', '/', '%', 'y'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'F': {
+    const char_type __fm[] = {'%', 'Y', '-', '%', 'm', '-', '%', 'd'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'H':
+    __get_hour(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'I':
+    __get_12_hour(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'j':
+    __get_day_year_num(__tm->tm_yday, __b, __e, __err, __ct);
+    break;
+  case 'm':
+    __get_month(__tm->tm_mon, __b, __e, __err, __ct);
+    break;
+  case 'M':
+    __get_minute(__tm->tm_min, __b, __e, __err, __ct);
+    break;
+  case 'n':
+  case 't':
+    __get_white_space(__b, __e, __err, __ct);
+    break;
+  case 'p':
+    __get_am_pm(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'r': {
+    const char_type __fm[] = {'%', 'I', ':', '%', 'M', ':', '%', 'S', ' ', '%', 'p'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'R': {
+    const char_type __fm[] = {'%', 'H', ':', '%', 'M'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'S':
+    __get_second(__tm->tm_sec, __b, __e, __err, __ct);
+    break;
+  case 'T': {
+    const char_type __fm[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'w':
+    __get_weekday(__tm->tm_wday, __b, __e, __err, __ct);
+    break;
+  case 'x':
+    return do_get_date(__b, __e, __iob, __err, __tm);
+  case 'X': {
+    const string_type& __fm = this->__X();
+    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
+  } break;
+  case 'y':
+    __get_year(__tm->tm_year, __b, __e, __err, __ct);
+    break;
+  case 'Y':
+    __get_year4(__tm->tm_year, __b, __e, __err, __ct);
+    break;
+  case '%':
+    __get_percent(__b, __e, __err, __ct);
+    break;
+  default:
+    __err |= ios_base::failbit;
+  }
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<wchar_t>;
+#  endif
+
+class _LIBCPP_EXPORTED_FROM_ABI __time_get {
+protected:
+  __locale::__locale_t __loc_;
+
+  __time_get(const char* __nm);
+  __time_get(const string& __nm);
+  ~__time_get();
+};
+
+template <class _CharT>
+class __time_get_storage : public __time_get {
+protected:
+  typedef basic_string<_CharT> string_type;
+
+  string_type __weeks_[14];
+  string_type __months_[24];
+  string_type __am_pm_[2];
+  string_type __c_;
+  string_type __r_;
+  string_type __x_;
+  string_type __X_;
+
+  explicit __time_get_storage(const char* __nm);
+  explicit __time_get_storage(const string& __nm);
+
+  _LIBCPP_HIDE_FROM_ABI ~__time_get_storage() {}
+
+  time_base::dateorder __do_date_order() const;
+
+private:
+  void init(const ctype<_CharT>&);
+  string_type __analyze(char __fmt, const ctype<_CharT>&);
+};
+
+#  define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                      \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;                \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                             \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                           \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                             \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(           \
+        char, const ctype<_CharT>&);                                                                                   \
+    extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()       \
+        const;                                                                                                         \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);             \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);           \
+    extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);             \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                  \
+    __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);
+
+_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t)
+#  endif
+#  undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class time_get_byname : public time_get<_CharT, _InputIterator>, private __time_get_storage<_CharT> {
+public:
+  typedef time_base::dateorder dateorder;
+  typedef _InputIterator iter_type;
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const char* __nm, size_t __refs = 0)
+      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
+  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const string& __nm, size_t __refs = 0)
+      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get_byname() override {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL dateorder do_date_order() const override { return this->__do_date_order(); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __weeks() const override { return this->__weeks_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __months() const override { return this->__months_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __am_pm() const override { return this->__am_pm_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __c() const override { return this->__c_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __r() const override { return this->__r_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __x() const override { return this->__x_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __X() const override { return this->__X_; }
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<wchar_t>;
+#  endif
+
+class _LIBCPP_EXPORTED_FROM_ABI __time_put {
+  __locale::__locale_t __loc_;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI __time_put() : __loc_(_LIBCPP_GET_C_LOCALE) {}
+  __time_put(const char* __nm);
+  __time_put(const string& __nm);
+  ~__time_put();
+  void __do_put(char* __nb, char*& __ne, const tm* __tm, char __fmt, char __mod) const;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+  void __do_put(wchar_t* __wb, wchar_t*& __we, const tm* __tm, char __fmt, char __mod) const;
+#  endif
+};
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class time_put : public locale::facet, private __time_put {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  iter_type
+  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
+      const;
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, char __fmt, char __mod = 0) const {
+    return do_put(__s, __iob, __fl, __tm, __fmt, __mod);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put() override {}
+  virtual iter_type do_put(iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(const char* __nm, size_t __refs) : locale::facet(__refs), __time_put(__nm) {}
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(const string& __nm, size_t __refs)
+      : locale::facet(__refs), __time_put(__nm) {}
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id time_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator time_put<_CharT, _OutputIterator>::put(
+    iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
+    const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  for (; __pb != __pe; ++__pb) {
+    if (__ct.narrow(*__pb, 0) == '%') {
+      if (++__pb == __pe) {
+        *__s++ = __pb[-1];
+        break;
+      }
+      char __mod = 0;
+      char __fmt = __ct.narrow(*__pb, 0);
+      if (__fmt == 'E' || __fmt == 'O') {
+        if (++__pb == __pe) {
+          *__s++ = __pb[-2];
+          *__s++ = __pb[-1];
+          break;
+        }
+        __mod = __fmt;
+        __fmt = __ct.narrow(*__pb, 0);
+      }
+      __s = do_put(__s, __iob, __fl, __tm, __fmt, __mod);
+    } else
+      *__s++ = *__pb;
+  }
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator time_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const {
+  char_type __nar[100];
+  char_type* __nb = __nar;
+  char_type* __ne = __nb + 100;
+  __do_put(__nb, __ne, __tm, __fmt, __mod);
+  return std::copy(__nb, __ne, __s);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class time_put_byname : public time_put<_CharT, _OutputIterator> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const char* __nm, size_t __refs = 0)
+      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const string& __nm, size_t __refs = 0)
+      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put_byname() override {}
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_TIME_H
diff --git a/libcxx/include/__locale_dir/wbuffer_convert.h b/libcxx/include/__locale_dir/wbuffer_convert.h
new file mode 100644
index 0000000000000..a6818aadf5d0b
--- /dev/null
+++ b/libcxx/include/__locale_dir/wbuffer_convert.h
@@ -0,0 +1,430 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_WBUFFER_CONVERT_H
+#define _LIBCPP___LOCALE_DIR_WBUFFER_CONVERT_H
+
+#include <__algorithm/reverse.h>
+#include <__config>
+#include <__string/char_traits.h>
+#include <ios>
+#include <streambuf>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+_LIBCPP_PUSH_MACROS
+#    include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Codecvt, class _Elem = wchar_t, class _Tr = char_traits<_Elem> >
+class _LIBCPP_DEPRECATED_IN_CXX17 wbuffer_convert : public basic_streambuf<_Elem, _Tr> {
+public:
+  // types:
+  typedef _Elem char_type;
+  typedef _Tr traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef typename _Codecvt::state_type state_type;
+
+private:
+  char* __extbuf_;
+  const char* __extbufnext_;
+  const char* __extbufend_;
+  char __extbuf_min_[8];
+  size_t __ebs_;
+  char_type* __intbuf_;
+  size_t __ibs_;
+  streambuf* __bufptr_;
+  _Codecvt* __cv_;
+  state_type __st_;
+  ios_base::openmode __cm_;
+  bool __owns_eb_;
+  bool __owns_ib_;
+  bool __always_noconv_;
+
+public:
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wbuffer_convert() : wbuffer_convert(nullptr) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
+#    else
+  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  wbuffer_convert(streambuf* __bytebuf = nullptr, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI ~wbuffer_convert();
+
+  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf() const { return __bufptr_; }
+  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf(streambuf* __bytebuf) {
+    streambuf* __r = __bufptr_;
+    __bufptr_      = __bytebuf;
+    return __r;
+  }
+
+  wbuffer_convert(const wbuffer_convert&)            = delete;
+  wbuffer_convert& operator=(const wbuffer_convert&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __st_; }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type underflow();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type pbackfail(int_type __c = traits_type::eof());
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type overflow(int_type __c = traits_type::eof());
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual basic_streambuf<char_type, traits_type>* setbuf(char_type* __s, streamsize __n);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
+  seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int sync();
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool __read_mode();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __write_mode();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL wbuffer_convert* __close();
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>::wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt, state_type __state)
+    : __extbuf_(nullptr),
+      __extbufnext_(nullptr),
+      __extbufend_(nullptr),
+      __ebs_(0),
+      __intbuf_(0),
+      __ibs_(0),
+      __bufptr_(__bytebuf),
+      __cv_(__pcvt),
+      __st_(__state),
+      __cm_(0),
+      __owns_eb_(false),
+      __owns_ib_(false),
+      __always_noconv_(__cv_ ? __cv_->always_noconv() : false) {
+  setbuf(0, 4096);
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>::~wbuffer_convert() {
+  __close();
+  delete __cv_;
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow() {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || __bufptr_ == nullptr)
+    return traits_type::eof();
+  bool __initial = __read_mode();
+  char_type __1buf;
+  if (this->gptr() == 0)
+    this->setg(std::addressof(__1buf), std::addressof(__1buf) + 1, std::addressof(__1buf) + 1);
+  const size_t __unget_sz = __initial ? 0 : std::min<size_t>((this->egptr() - this->eback()) / 2, 4);
+  int_type __c            = traits_type::eof();
+  if (this->gptr() == this->egptr()) {
+    std::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
+    if (__always_noconv_) {
+      streamsize __nmemb = static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz);
+      __nmemb            = __bufptr_->sgetn((char*)this->eback() + __unget_sz, __nmemb);
+      if (__nmemb != 0) {
+        this->setg(this->eback(), this->eback() + __unget_sz, this->eback() + __unget_sz + __nmemb);
+        __c = *this->gptr();
+      }
+    } else {
+      if (__extbufend_ != __extbufnext_) {
+        _LIBCPP_ASSERT_NON_NULL(__extbufnext_ != nullptr, "underflow moving from nullptr");
+        _LIBCPP_ASSERT_NON_NULL(__extbuf_ != nullptr, "underflow moving into nullptr");
+        std::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
+      }
+      __extbufnext_      = __extbuf_ + (__extbufend_ - __extbufnext_);
+      __extbufend_       = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
+      streamsize __nmemb = std::min(static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz),
+                                    static_cast<streamsize>(__extbufend_ - __extbufnext_));
+      codecvt_base::result __r;
+      // FIXME: Do we ever need to restore the state here?
+      // state_type __svs = __st_;
+      streamsize __nr = __bufptr_->sgetn(const_cast<char*>(__extbufnext_), __nmemb);
+      if (__nr != 0) {
+        __extbufend_ = __extbufnext_ + __nr;
+        char_type* __inext;
+        __r = __cv_->in(
+            __st_, __extbuf_, __extbufend_, __extbufnext_, this->eback() + __unget_sz, this->egptr(), __inext);
+        if (__r == codecvt_base::noconv) {
+          this->setg((char_type*)__extbuf_, (char_type*)__extbuf_, (char_type*)const_cast<char*>(__extbufend_));
+          __c = *this->gptr();
+        } else if (__inext != this->eback() + __unget_sz) {
+          this->setg(this->eback(), this->eback() + __unget_sz, __inext);
+          __c = *this->gptr();
+        }
+      }
+    }
+  } else
+    __c = *this->gptr();
+  if (this->eback() == std::addressof(__1buf))
+    this->setg(0, 0, 0);
+  return __c;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::pbackfail(int_type __c) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ != 0 && __bufptr_ && this->eback() < this->gptr()) {
+    if (traits_type::eq_int_type(__c, traits_type::eof())) {
+      this->gbump(-1);
+      return traits_type::not_eof(__c);
+    }
+    if (traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
+      this->gbump(-1);
+      *this->gptr() = traits_type::to_char_type(__c);
+      return __c;
+    }
+  }
+  return traits_type::eof();
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::overflow(int_type __c) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || !__bufptr_)
+    return traits_type::eof();
+  __write_mode();
+  char_type __1buf;
+  char_type* __pb_save  = this->pbase();
+  char_type* __epb_save = this->epptr();
+  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
+    if (this->pptr() == 0)
+      this->setp(std::addressof(__1buf), std::addressof(__1buf) + 1);
+    *this->pptr() = traits_type::to_char_type(__c);
+    this->pbump(1);
+  }
+  if (this->pptr() != this->pbase()) {
+    if (__always_noconv_) {
+      streamsize __nmemb = static_cast<streamsize>(this->pptr() - this->pbase());
+      if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
+        return traits_type::eof();
+    } else {
+      char* __extbe = __extbuf_;
+      codecvt_base::result __r;
+      do {
+        const char_type* __e;
+        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+        if (__e == this->pbase())
+          return traits_type::eof();
+        if (__r == codecvt_base::noconv) {
+          streamsize __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
+          if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
+            return traits_type::eof();
+        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
+          streamsize __nmemb = static_cast<size_t>(__extbe - __extbuf_);
+          if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
+            return traits_type::eof();
+          if (__r == codecvt_base::partial) {
+            this->setp(const_cast<char_type*>(__e), this->pptr());
+            this->__pbump(this->epptr() - this->pbase());
+          }
+        } else
+          return traits_type::eof();
+      } while (__r == codecvt_base::partial);
+    }
+    this->setp(__pb_save, __epb_save);
+  }
+  return traits_type::not_eof(__c);
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+basic_streambuf<_Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::setbuf(char_type* __s, streamsize __n) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  this->setg(0, 0, 0);
+  this->setp(0, 0);
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+  __ebs_ = __n;
+  if (__ebs_ > sizeof(__extbuf_min_)) {
+    if (__always_noconv_ && __s) {
+      __extbuf_  = (char*)__s;
+      __owns_eb_ = false;
+    } else {
+      __extbuf_  = new char[__ebs_];
+      __owns_eb_ = true;
+    }
+  } else {
+    __extbuf_  = __extbuf_min_;
+    __ebs_     = sizeof(__extbuf_min_);
+    __owns_eb_ = false;
+  }
+  if (!__always_noconv_) {
+    __ibs_ = max<streamsize>(__n, sizeof(__extbuf_min_));
+    if (__s && __ibs_ >= sizeof(__extbuf_min_)) {
+      __intbuf_  = __s;
+      __owns_ib_ = false;
+    } else {
+      __intbuf_  = new char_type[__ibs_];
+      __owns_ib_ = true;
+    }
+  } else {
+    __ibs_     = 0;
+    __intbuf_  = 0;
+    __owns_ib_ = false;
+  }
+  return this;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __om) {
+  int __width = __cv_->encoding();
+  if (__cv_ == 0 || !__bufptr_ || (__width <= 0 && __off != 0) || sync())
+    return pos_type(off_type(-1));
+  // __width > 0 || __off == 0, now check __way
+  if (__way != ios_base::beg && __way != ios_base::cur && __way != ios_base::end)
+    return pos_type(off_type(-1));
+  pos_type __r = __bufptr_->pubseekoff(__width * __off, __way, __om);
+  __r.state(__st_);
+  return __r;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::seekpos(pos_type __sp, ios_base::openmode __wch) {
+  if (__cv_ == 0 || !__bufptr_ || sync())
+    return pos_type(off_type(-1));
+  if (__bufptr_->pubseekpos(__sp, __wch) == pos_type(off_type(-1)))
+    return pos_type(off_type(-1));
+  return __sp;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+int wbuffer_convert<_Codecvt, _Elem, _Tr>::sync() {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || !__bufptr_)
+    return 0;
+  if (__cm_ & ios_base::out) {
+    if (this->pptr() != this->pbase())
+      if (overflow() == traits_type::eof())
+        return -1;
+    codecvt_base::result __r;
+    do {
+      char* __extbe;
+      __r                = __cv_->unshift(__st_, __extbuf_, __extbuf_ + __ebs_, __extbe);
+      streamsize __nmemb = static_cast<streamsize>(__extbe - __extbuf_);
+      if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
+        return -1;
+    } while (__r == codecvt_base::partial);
+    if (__r == codecvt_base::error)
+      return -1;
+    if (__bufptr_->pubsync())
+      return -1;
+  } else if (__cm_ & ios_base::in) {
+    off_type __c;
+    if (__always_noconv_)
+      __c = this->egptr() - this->gptr();
+    else {
+      int __width = __cv_->encoding();
+      __c         = __extbufend_ - __extbufnext_;
+      if (__width > 0)
+        __c += __width * (this->egptr() - this->gptr());
+      else {
+        if (this->gptr() != this->egptr()) {
+          std::reverse(this->gptr(), this->egptr());
+          codecvt_base::result __r;
+          const char_type* __e = this->gptr();
+          char* __extbe;
+          do {
+            __r = __cv_->out(__st_, __e, this->egptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+            switch (__r) {
+            case codecvt_base::noconv:
+              __c += this->egptr() - this->gptr();
+              break;
+            case codecvt_base::ok:
+            case codecvt_base::partial:
+              __c += __extbe - __extbuf_;
+              break;
+            default:
+              return -1;
+            }
+          } while (__r == codecvt_base::partial);
+        }
+      }
+    }
+    if (__bufptr_->pubseekoff(-__c, ios_base::cur, __cm_) == pos_type(off_type(-1)))
+      return -1;
+    this->setg(0, 0, 0);
+    __cm_ = 0;
+  }
+  return 0;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+bool wbuffer_convert<_Codecvt, _Elem, _Tr>::__read_mode() {
+  if (!(__cm_ & ios_base::in)) {
+    this->setp(0, 0);
+    if (__always_noconv_)
+      this->setg((char_type*)__extbuf_, (char_type*)__extbuf_ + __ebs_, (char_type*)__extbuf_ + __ebs_);
+    else
+      this->setg(__intbuf_, __intbuf_ + __ibs_, __intbuf_ + __ibs_);
+    __cm_ = ios_base::in;
+    return true;
+  }
+  return false;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+void wbuffer_convert<_Codecvt, _Elem, _Tr>::__write_mode() {
+  if (!(__cm_ & ios_base::out)) {
+    this->setg(0, 0, 0);
+    if (__ebs_ > sizeof(__extbuf_min_)) {
+      if (__always_noconv_)
+        this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__ebs_ - 1));
+      else
+        this->setp(__intbuf_, __intbuf_ + (__ibs_ - 1));
+    } else
+      this->setp(0, 0);
+    __cm_ = ios_base::out;
+  }
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::__close() {
+  wbuffer_convert* __rt = nullptr;
+  if (__cv_ != nullptr && __bufptr_ != nullptr) {
+    __rt = this;
+    if ((__cm_ & ios_base::out) && sync())
+      __rt = nullptr;
+  }
+  return __rt;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_WBUFFER_CONVERT_H
diff --git a/libcxx/include/__locale_dir/wstring_convert.h b/libcxx/include/__locale_dir/wstring_convert.h
new file mode 100644
index 0000000000000..42a56eb8573ea
--- /dev/null
+++ b/libcxx/include/__locale_dir/wstring_convert.h
@@ -0,0 +1,254 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_WSTRING_CONVERT_H
+#define _LIBCPP___LOCALE_DIR_WSTRING_CONVERT_H
+
+#include <__config>
+#include <__locale>
+#include <__memory/allocator.h>
+#include <string>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+_LIBCPP_PUSH_MACROS
+#    include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Codecvt,
+          class _Elem      = wchar_t,
+          class _WideAlloc = allocator<_Elem>,
+          class _ByteAlloc = allocator<char> >
+class _LIBCPP_DEPRECATED_IN_CXX17 wstring_convert {
+public:
+  typedef basic_string<char, char_traits<char>, _ByteAlloc> byte_string;
+  typedef basic_string<_Elem, char_traits<_Elem>, _WideAlloc> wide_string;
+  typedef typename _Codecvt::state_type state_type;
+  typedef typename wide_string::traits_type::int_type int_type;
+
+private:
+  byte_string __byte_err_string_;
+  wide_string __wide_err_string_;
+  _Codecvt* __cvtptr_;
+  state_type __cvtstate_;
+  size_t __cvtcount_;
+
+public:
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wstring_convert() : wstring_convert(new _Codecvt) {}
+  _LIBCPP_HIDE_FROM_ABI explicit wstring_convert(_Codecvt* __pcvt);
+#    else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_EXPLICIT_SINCE_CXX14 wstring_convert(_Codecvt* __pcvt = new _Codecvt);
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI wstring_convert(_Codecvt* __pcvt, state_type __state);
+  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  wstring_convert(const byte_string& __byte_err, const wide_string& __wide_err = wide_string());
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wstring_convert(wstring_convert&& __wc);
+#    endif
+  _LIBCPP_HIDE_FROM_ABI ~wstring_convert();
+
+  wstring_convert(const wstring_convert& __wc)            = delete;
+  wstring_convert& operator=(const wstring_convert& __wc) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(char __byte) { return from_bytes(&__byte, &__byte + 1); }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __ptr) {
+    return from_bytes(__ptr, __ptr + char_traits<char>::length(__ptr));
+  }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const byte_string& __str) {
+    return from_bytes(__str.data(), __str.data() + __str.size());
+  }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __first, const char* __last);
+
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(_Elem __wchar) {
+    return to_bytes(std::addressof(__wchar), std::addressof(__wchar) + 1);
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __wptr) {
+    return to_bytes(__wptr, __wptr + char_traits<_Elem>::length(__wptr));
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const wide_string& __wstr) {
+    return to_bytes(__wstr.data(), __wstr.data() + __wstr.size());
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __first, const _Elem* __last);
+
+  _LIBCPP_HIDE_FROM_ABI size_t converted() const _NOEXCEPT { return __cvtcount_; }
+  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __cvtstate_; }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt)
+    : __cvtptr_(__pcvt), __cvtstate_(), __cvtcount_(0) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt, state_type __state)
+    : __cvtptr_(__pcvt), __cvtstate_(__state), __cvtcount_(0) {}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(
+    const byte_string& __byte_err, const wide_string& __wide_err)
+    : __byte_err_string_(__byte_err), __wide_err_string_(__wide_err), __cvtstate_(), __cvtcount_(0) {
+  __cvtptr_ = new _Codecvt;
+}
+
+#    ifndef _LIBCPP_CXX03_LANG
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(wstring_convert&& __wc)
+    : __byte_err_string_(std::move(__wc.__byte_err_string_)),
+      __wide_err_string_(std::move(__wc.__wide_err_string_)),
+      __cvtptr_(__wc.__cvtptr_),
+      __cvtstate_(__wc.__cvtstate_),
+      __cvtcount_(__wc.__cvtcount_) {
+  __wc.__cvtptr_ = nullptr;
+}
+
+#    endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::~wstring_convert() {
+  delete __cvtptr_;
+}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wide_string
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::from_bytes(const char* __frm, const char* __frm_end) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  __cvtcount_ = 0;
+  if (__cvtptr_ != nullptr) {
+    wide_string __ws(2 * (__frm_end - __frm), _Elem());
+    if (__frm != __frm_end)
+      __ws.resize(__ws.capacity());
+    codecvt_base::result __r = codecvt_base::ok;
+    state_type __st          = __cvtstate_;
+    if (__frm != __frm_end) {
+      _Elem* __to     = std::addressof(__ws[0]);
+      _Elem* __to_end = __to + __ws.size();
+      const char* __frm_nxt;
+      do {
+        _Elem* __to_nxt;
+        __r = __cvtptr_->in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+        __cvtcount_ += __frm_nxt - __frm;
+        if (__frm_nxt == __frm) {
+          __r = codecvt_base::error;
+        } else if (__r == codecvt_base::noconv) {
+          __ws.resize(__to - std::addressof(__ws[0]));
+          // This only gets executed if _Elem is char
+          __ws.append((const _Elem*)__frm, (const _Elem*)__frm_end);
+          __frm = __frm_nxt;
+          __r   = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __ws.resize(__to_nxt - std::addressof(__ws[0]));
+          __frm = __frm_nxt;
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __s = __to_nxt - std::addressof(__ws[0]);
+          __ws.resize(2 * __s);
+          __to     = std::addressof(__ws[0]) + __s;
+          __to_end = std::addressof(__ws[0]) + __ws.size();
+          __frm    = __frm_nxt;
+        }
+      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
+    }
+    if (__r == codecvt_base::ok)
+      return __ws;
+  }
+
+  if (__wide_err_string_.empty())
+    std::__throw_range_error("wstring_convert: from_bytes error");
+
+  return __wide_err_string_;
+}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::byte_string
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::to_bytes(const _Elem* __frm, const _Elem* __frm_end) {
+  __cvtcount_ = 0;
+  if (__cvtptr_ != nullptr) {
+    byte_string __bs(2 * (__frm_end - __frm), char());
+    if (__frm != __frm_end)
+      __bs.resize(__bs.capacity());
+    codecvt_base::result __r = codecvt_base::ok;
+    state_type __st          = __cvtstate_;
+    if (__frm != __frm_end) {
+      char* __to     = std::addressof(__bs[0]);
+      char* __to_end = __to + __bs.size();
+      const _Elem* __frm_nxt;
+      do {
+        char* __to_nxt;
+        __r = __cvtptr_->out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+        __cvtcount_ += __frm_nxt - __frm;
+        if (__frm_nxt == __frm) {
+          __r = codecvt_base::error;
+        } else if (__r == codecvt_base::noconv) {
+          __bs.resize(__to - std::addressof(__bs[0]));
+          // This only gets executed if _Elem is char
+          __bs.append((const char*)__frm, (const char*)__frm_end);
+          __frm = __frm_nxt;
+          __r   = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __bs.resize(__to_nxt - std::addressof(__bs[0]));
+          __frm = __frm_nxt;
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __s = __to_nxt - std::addressof(__bs[0]);
+          __bs.resize(2 * __s);
+          __to     = std::addressof(__bs[0]) + __s;
+          __to_end = std::addressof(__bs[0]) + __bs.size();
+          __frm    = __frm_nxt;
+        }
+      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
+    }
+    if (__r == codecvt_base::ok) {
+      size_t __s = __bs.size();
+      __bs.resize(__bs.capacity());
+      char* __to     = std::addressof(__bs[0]) + __s;
+      char* __to_end = __to + __bs.size();
+      do {
+        char* __to_nxt;
+        __r = __cvtptr_->unshift(__st, __to, __to_end, __to_nxt);
+        if (__r == codecvt_base::noconv) {
+          __bs.resize(__to - std::addressof(__bs[0]));
+          __r = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __bs.resize(__to_nxt - std::addressof(__bs[0]));
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __sp = __to_nxt - std::addressof(__bs[0]);
+          __bs.resize(2 * __sp);
+          __to     = std::addressof(__bs[0]) + __sp;
+          __to_end = std::addressof(__bs[0]) + __bs.size();
+        }
+      } while (__r == codecvt_base::partial);
+      if (__r == codecvt_base::ok)
+        return __bs;
+    }
+  }
+
+  if (__byte_err_string_.empty())
+    std::__throw_range_error("wstring_convert: to_bytes error");
+
+  return __byte_err_string_;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_WSTRING_CONVERT_H
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index f1b2e2dbfc0cd..0cbd995105671 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -34,7 +34,7 @@
 #include <__memory/shared_count.h>
 #include <__memory/uninitialized_algorithms.h>
 #include <__memory/unique_ptr.h>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/disjunction.h>
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 3e8d3cd6a4097..eff24546cdc01 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -24,7 +24,7 @@
 #include <__memory/auto_ptr.h>
 #include <__memory/compressed_pair.h>
 #include <__memory/pointer_traits.h>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/dependent_type.h>
diff --git a/libcxx/include/__mutex/lock_guard.h b/libcxx/include/__mutex/lock_guard.h
index a529c552a1df0..67a25ed2f8a89 100644
--- a/libcxx/include/__mutex/lock_guard.h
+++ b/libcxx/include/__mutex/lock_guard.h
@@ -19,7 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Mutex>
-class _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) lock_guard {
+class _LIBCPP_SCOPED_LOCKABLE lock_guard {
 public:
   typedef _Mutex mutex_type;
 
@@ -27,16 +27,14 @@ class _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) lock_guard {
   mutex_type& __m_;
 
 public:
-  [[__nodiscard__]]
-  _LIBCPP_HIDE_FROM_ABI explicit lock_guard(mutex_type& __m) _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability(__m))
+  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI explicit lock_guard(mutex_type& __m) _LIBCPP_ACQUIRE_CAPABILITY(__m)
       : __m_(__m) {
     __m_.lock();
   }
 
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI lock_guard(mutex_type& __m, adopt_lock_t)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(requires_capability(__m))
+  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI lock_guard(mutex_type& __m, adopt_lock_t) _LIBCPP_REQUIRES_CAPABILITY(__m)
       : __m_(__m) {}
-  _LIBCPP_HIDE_FROM_ABI ~lock_guard() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()) { __m_.unlock(); }
+  _LIBCPP_RELEASE_CAPABILITY _LIBCPP_HIDE_FROM_ABI ~lock_guard() { __m_.unlock(); }
 
   lock_guard(lock_guard const&)            = delete;
   lock_guard& operator=(lock_guard const&) = delete;
diff --git a/libcxx/include/__mutex/mutex.h b/libcxx/include/__mutex/mutex.h
index 317320287902f..68c8842b35eda 100644
--- a/libcxx/include/__mutex/mutex.h
+++ b/libcxx/include/__mutex/mutex.h
@@ -21,7 +21,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(capability("mutex")) mutex {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("mutex") mutex {
   __libcpp_mutex_t __m_ = _LIBCPP_MUTEX_INITIALIZER;
 
 public:
@@ -36,9 +36,9 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(capability("mut
   ~mutex() _NOEXCEPT;
 #  endif
 
-  void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability());
-  bool try_lock() _NOEXCEPT _LIBCPP_THREAD_SAFETY_ANNOTATION(try_acquire_capability(true));
-  void unlock() _NOEXCEPT _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability());
+  _LIBCPP_ACQUIRE_CAPABILITY() void lock();
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT;
+  _LIBCPP_RELEASE_CAPABILITY void unlock() _NOEXCEPT;
 
   typedef __libcpp_mutex_t* native_handle_type;
   _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
diff --git a/libcxx/include/__ostream/basic_ostream.h b/libcxx/include/__ostream/basic_ostream.h
index f7473a36d8ccc..effeef491f341 100644
--- a/libcxx/include/__ostream/basic_ostream.h
+++ b/libcxx/include/__ostream/basic_ostream.h
@@ -15,6 +15,9 @@
 
 #  include <__exception/operations.h>
 #  include <__fwd/memory.h>
+#  include <__iterator/ostreambuf_iterator.h>
+#  include <__locale_dir/num.h>
+#  include <__locale_dir/pad_and_output.h>
 #  include <__memory/addressof.h>
 #  include <__memory/unique_ptr.h>
 #  include <__new/exceptions.h>
@@ -27,7 +30,6 @@
 #  include <__utility/declval.h>
 #  include <bitset>
 #  include <ios>
-#  include <locale>
 #  include <streambuf>
 #  include <string_view>
 
diff --git a/libcxx/include/__ostream/print.h b/libcxx/include/__ostream/print.h
index a4d7506cffc48..c5906c41b95b5 100644
--- a/libcxx/include/__ostream/print.h
+++ b/libcxx/include/__ostream/print.h
@@ -18,8 +18,8 @@
 #  include <__ostream/basic_ostream.h>
 #  include <format>
 #  include <ios>
-#  include <locale>
 #  include <print>
+#  include <streambuf>
 
 #  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #    pragma GCC system_header
diff --git a/libcxx/include/__ranges/zip_view.h b/libcxx/include/__ranges/zip_view.h
index 6d31f5ce0993c..e2a194efcfb4c 100644
--- a/libcxx/include/__ranges/zip_view.h
+++ b/libcxx/include/__ranges/zip_view.h
@@ -23,6 +23,7 @@
 #include <__iterator/iter_move.h>
 #include <__iterator/iter_swap.h>
 #include <__iterator/iterator_traits.h>
+#include <__iterator/product_iterator.h>
 #include <__ranges/access.h>
 #include <__ranges/all.h>
 #include <__ranges/concepts.h>
@@ -251,6 +252,10 @@ class zip_view<_Views...>::__iterator : public __zip_view_iterator_category_base
 
   friend class zip_view<_Views...>;
 
+  static constexpr bool __is_zip_view_iterator = true;
+
+  friend struct __product_iterator_traits<__iterator>;
+
 public:
   using iterator_concept = decltype(ranges::__get_zip_view_iterator_tag<_Const, _Views...>());
   using value_type       = tuple<range_value_t<__maybe_const<_Const, _Views>>...>;
@@ -468,6 +473,23 @@ inline constexpr auto zip = __zip::__fn{};
 } // namespace views
 } // namespace ranges
 
+template <class _Iterator>
+  requires _Iterator::__is_zip_view_iterator
+struct __product_iterator_traits<_Iterator> {
+  static constexpr size_t __size = tuple_size<decltype(std::declval<_Iterator>().__current_)>::value;
+
+  template <size_t _Nth, class _Iter>
+    requires(_Nth < __size)
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __get_iterator_element(_Iter&& __it) {
+    return std::get<_Nth>(std::forward<_Iter>(__it).__current_);
+  }
+
+  template <class... _Iters>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Iterator __make_product_iterator(_Iters&&... __iters) {
+    return _Iterator(std::tuple(std::forward<_Iters>(__iters)...));
+  }
+};
+
 #endif // _LIBCPP_STD_VER >= 23
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__thread/thread.h b/libcxx/include/__thread/thread.h
index 40ea83096e738..1b51571ce302e 100644
--- a/libcxx/include/__thread/thread.h
+++ b/libcxx/include/__thread/thread.h
@@ -16,6 +16,7 @@
 #include <__exception/terminate.h>
 #include <__functional/hash.h>
 #include <__functional/unary_function.h>
+#include <__locale>
 #include <__memory/addressof.h>
 #include <__memory/unique_ptr.h>
 #include <__mutex/mutex.h>
@@ -30,7 +31,6 @@
 #include <tuple>
 
 #if _LIBCPP_HAS_LOCALIZATION
-#  include <locale>
 #  include <sstream>
 #endif
 
diff --git a/libcxx/include/__type_traits/add_pointer.h b/libcxx/include/__type_traits/add_pointer.h
index f37f9cf5a93c0..fa9376699a74a 100644
--- a/libcxx/include/__type_traits/add_pointer.h
+++ b/libcxx/include/__type_traits/add_pointer.h
@@ -20,10 +20,20 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__add_pointer) && !defined(_LIBCPP_COMPILER_GCC)
+#if !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS)
 
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS add_pointer {
+  using type _LIBCPP_NODEBUG = __add_pointer(_Tp);
+};
+
+#  ifdef _LIBCPP_COMPILER_GCC
+template <class _Tp>
+using __add_pointer_t _LIBCPP_NODEBUG = typename add_pointer<_Tp>::type;
+#  else
 template <class _Tp>
 using __add_pointer_t _LIBCPP_NODEBUG = __add_pointer(_Tp);
+#  endif
 
 #else
 template <class _Tp, bool = __is_referenceable_v<_Tp> || is_void<_Tp>::value>
@@ -38,13 +48,13 @@ struct __add_pointer_impl<_Tp, false> {
 template <class _Tp>
 using __add_pointer_t = typename __add_pointer_impl<_Tp>::type;
 
-#endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__add_pointer)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS add_pointer {
   using type _LIBCPP_NODEBUG = __add_pointer_t<_Tp>;
 };
 
+#endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS)
+
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
 using add_pointer_t = __add_pointer_t<_Tp>;
diff --git a/libcxx/include/__type_traits/add_lvalue_reference.h b/libcxx/include/__type_traits/add_reference.h
similarity index 51%
rename from libcxx/include/__type_traits/add_lvalue_reference.h
rename to libcxx/include/__type_traits/add_reference.h
index 42f91623f980c..c6f5d63d26dea 100644
--- a/libcxx/include/__type_traits/add_lvalue_reference.h
+++ b/libcxx/include/__type_traits/add_reference.h
@@ -6,11 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_ADD_REFERENCE_H
 
 #include <__config>
-#include <__type_traits/is_referenceable.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -18,37 +17,42 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__add_lvalue_reference) && !defined(_LIBCPP_COMPILER_GCC)
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS add_lvalue_reference {
+  using type _LIBCPP_NODEBUG = __add_lvalue_reference(_Tp);
+};
 
+#ifdef _LIBCPP_COMPILER_GCC
+template <class _Tp>
+using __add_lvalue_reference_t _LIBCPP_NODEBUG = typename add_lvalue_reference<_Tp>::type;
+#else
 template <class _Tp>
 using __add_lvalue_reference_t _LIBCPP_NODEBUG = __add_lvalue_reference(_Tp);
+#endif
 
-#else
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
+#endif
 
-template <class _Tp, bool = __is_referenceable_v<_Tp>>
-struct __add_lvalue_reference_impl {
-  using type _LIBCPP_NODEBUG = _Tp;
-};
-template <class _Tp >
-struct __add_lvalue_reference_impl<_Tp, true> {
-  using type _LIBCPP_NODEBUG = _Tp&;
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS add_rvalue_reference {
+  using type _LIBCPP_NODEBUG = __add_rvalue_reference(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-using __add_lvalue_reference_t = typename __add_lvalue_reference_impl<_Tp>::type;
-
-#endif // __has_builtin(__add_lvalue_reference)
-
+using __add_rvalue_reference_t _LIBCPP_NODEBUG = typename add_rvalue_reference<_Tp>::type;
+#else
 template <class _Tp>
-struct _LIBCPP_NO_SPECIALIZATIONS add_lvalue_reference {
-  using type _LIBCPP_NODEBUG = __add_lvalue_reference_t<_Tp>;
-};
+using __add_rvalue_reference_t _LIBCPP_NODEBUG = __add_rvalue_reference(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
-using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
+using add_rvalue_reference_t = __add_rvalue_reference_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
+#endif // _LIBCPP___TYPE_TRAITS_ADD_REFERENCE_H
diff --git a/libcxx/include/__type_traits/add_rvalue_reference.h b/libcxx/include/__type_traits/add_rvalue_reference.h
deleted file mode 100644
index 33cc00180cc13..0000000000000
--- a/libcxx/include/__type_traits/add_rvalue_reference.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
-
-#include <__config>
-#include <__type_traits/is_referenceable.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#if __has_builtin(__add_rvalue_reference) && !defined(_LIBCPP_COMPILER_GCC)
-
-template <class _Tp>
-using __add_rvalue_reference_t _LIBCPP_NODEBUG = __add_rvalue_reference(_Tp);
-
-#else
-
-template <class _Tp, bool = __is_referenceable_v<_Tp> >
-struct __add_rvalue_reference_impl {
-  using type _LIBCPP_NODEBUG = _Tp;
-};
-template <class _Tp >
-struct __add_rvalue_reference_impl<_Tp, true> {
-  using type _LIBCPP_NODEBUG = _Tp&&;
-};
-
-template <class _Tp>
-using __add_rvalue_reference_t = typename __add_rvalue_reference_impl<_Tp>::type;
-
-#endif // __has_builtin(__add_rvalue_reference)
-
-template <class _Tp>
-struct _LIBCPP_NO_SPECIALIZATIONS add_rvalue_reference {
-  using type _LIBCPP_NODEBUG = __add_rvalue_reference_t<_Tp>;
-};
-
-#if _LIBCPP_STD_VER >= 14
-template <class _Tp>
-using add_rvalue_reference_t = __add_rvalue_reference_t<_Tp>;
-#endif
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
diff --git a/libcxx/include/__type_traits/common_reference.h b/libcxx/include/__type_traits/common_reference.h
index 7df55f7fecfd5..59badb64267de 100644
--- a/libcxx/include/__type_traits/common_reference.h
+++ b/libcxx/include/__type_traits/common_reference.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
 
 #include <__config>
+#include <__type_traits/add_pointer.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/copy_cv.h>
 #include <__type_traits/copy_cvref.h>
@@ -139,13 +140,17 @@ struct __common_reference_sub_bullet2 : __common_reference_sub_bullet3<_Tp, _Up>
 template <class _Tp, class _Up>
 struct __common_reference_sub_bullet1 : __common_reference_sub_bullet2<_Tp, _Up> {};
 
-// sub-bullet 1 - If T1 and T2 are reference types and COMMON-REF(T1, T2) is well-formed, then
-// the member typedef `type` denotes that type.
+// sub-bullet 1 - Let R be COMMON-REF(T1, T2). If T1 and T2 are reference types, R is well-formed, and
+// is_convertible_v<add_pointer_t<T1>, add_pointer_t<R>> && is_convertible_v<add_pointer_t<T2>, add_pointer_t<R>> is
+// true, then the member typedef type denotes R.
+
 template <class _Tp, class _Up>
 struct common_reference<_Tp, _Up> : __common_reference_sub_bullet1<_Tp, _Up> {};
 
 template <class _Tp, class _Up>
-  requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; }
+  requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; } &&
+           is_convertible_v<add_pointer_t<_Tp>, add_pointer_t<__common_ref_t<_Tp, _Up>>> &&
+           is_convertible_v<add_pointer_t<_Up>, add_pointer_t<__common_ref_t<_Tp, _Up>>>
 struct __common_reference_sub_bullet1<_Tp, _Up> {
   using type _LIBCPP_NODEBUG = __common_ref_t<_Tp, _Up>;
 };
diff --git a/libcxx/include/__type_traits/copy_cvref.h b/libcxx/include/__type_traits/copy_cvref.h
index 158e5a5d78bb3..39002347d4a94 100644
--- a/libcxx/include/__type_traits/copy_cvref.h
+++ b/libcxx/include/__type_traits/copy_cvref.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/copy_cv.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/decay.h b/libcxx/include/__type_traits/decay.h
index 1e663fdcd0eda..661fba53dfaa4 100644
--- a/libcxx/include/__type_traits/decay.h
+++ b/libcxx/include/__type_traits/decay.h
@@ -10,14 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_DECAY_H
 
 #include <__config>
-#include <__type_traits/add_pointer.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_function.h>
-#include <__type_traits/is_referenceable.h>
-#include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_extent.h>
-#include <__type_traits/remove_reference.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -25,42 +17,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__decay) && !defined(_LIBCPP_COMPILER_GCC)
-template <class _Tp>
-using __decay_t _LIBCPP_NODEBUG = __decay(_Tp);
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS decay {
-  using type _LIBCPP_NODEBUG = __decay_t<_Tp>;
-};
-
-#else
-template <class _Up, bool>
-struct __decay {
-  using type _LIBCPP_NODEBUG = __remove_cv_t<_Up>;
-};
-
-template <class _Up>
-struct __decay<_Up, true> {
-public:
-  using type _LIBCPP_NODEBUG =
-      __conditional_t<is_array<_Up>::value,
-                      __add_pointer_t<__remove_extent_t<_Up> >,
-                      __conditional_t<is_function<_Up>::value, typename add_pointer<_Up>::type, __remove_cv_t<_Up> > >;
+  using type _LIBCPP_NODEBUG = __decay(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-struct decay {
-private:
-  using _Up _LIBCPP_NODEBUG = __libcpp_remove_reference_t<_Tp>;
-
-public:
-  using type _LIBCPP_NODEBUG = typename __decay<_Up, __is_referenceable_v<_Up> >::type;
-};
-
+using __decay_t _LIBCPP_NODEBUG = typename decay<_Tp>::type;
+#else
 template <class _Tp>
-using __decay_t = typename decay<_Tp>::type;
-#endif // __has_builtin(__decay)
+using __decay_t _LIBCPP_NODEBUG = __decay(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/has_unique_object_representation.h b/libcxx/include/__type_traits/has_unique_object_representation.h
index ba81be6aa19b0..af9abfe81d41f 100644
--- a/libcxx/include/__type_traits/has_unique_object_representation.h
+++ b/libcxx/include/__type_traits/has_unique_object_representation.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__type_traits/integral_constant.h>
-#include <__type_traits/remove_all_extents.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -23,12 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS has_unique_object_representations
-    // TODO: We work around a Clang and GCC bug in __has_unique_object_representations by using remove_all_extents
-    //       even though it should not be necessary. This was reported to the compilers:
-    //         - Clang: https://github.com/llvm/llvm-project/issues/95311
-    //         - GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115476
-    //       remove_all_extents_t can be removed once all the compilers we support have fixed this bug.
-    : public integral_constant<bool, __has_unique_object_representations(remove_all_extents_t<_Tp>)> {};
+    : integral_constant<bool, __has_unique_object_representations(_Tp)> {};
 
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool has_unique_object_representations_v =
diff --git a/libcxx/include/__type_traits/invoke.h b/libcxx/include/__type_traits/invoke.h
index ccf86209d2295..ac38b32b0cd76 100644
--- a/libcxx/include/__type_traits/invoke.h
+++ b/libcxx/include/__type_traits/invoke.h
@@ -22,6 +22,7 @@
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/nat.h>
+#include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
 
@@ -61,6 +62,112 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#if __has_builtin(__builtin_invoke)
+
+template <class... _Args>
+using __invoke_result_t _LIBCPP_NODEBUG = decltype(__builtin_invoke(std::declval<_Args>()...));
+
+template <class, class... _Args>
+struct __invoke_result_impl {};
+
+template <class... _Args>
+struct __invoke_result_impl<__void_t<__invoke_result_t<_Args...> >, _Args...> {
+  using type _LIBCPP_NODEBUG = __invoke_result_t<_Args...>;
+};
+
+template <class... _Args>
+using __invoke_result _LIBCPP_NODEBUG = __invoke_result_impl<void, _Args...>;
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __invoke_result_t<_Args...> __invoke(_Args&&... __args)
+    _NOEXCEPT_(noexcept(__builtin_invoke(std::forward<_Args>(__args)...))) {
+  return __builtin_invoke(std::forward<_Args>(__args)...);
+}
+
+template <class _Void, class... _Args>
+inline const bool __is_invocable_impl = false;
+
+template <class... _Args>
+inline const bool __is_invocable_impl<__void_t<__invoke_result_t<_Args...> >, _Args...> = true;
+
+template <class... _Args>
+inline const bool __is_invocable_v = __is_invocable_impl<void, _Args...>;
+
+template <class... _Args>
+struct __is_invocable : integral_constant<bool, __is_invocable_v<_Args...> > {};
+
+template <class _Ret, bool, class... _Args>
+inline const bool __is_invocable_r_impl = false;
+
+template <class _Ret, class... _Args>
+inline const bool __is_invocable_r_impl<_Ret, true, _Args...> =
+    __is_core_convertible<__invoke_result_t<_Args...>, _Ret>::value || is_void<_Ret>::value;
+
+template <class _Ret, class... _Args>
+inline const bool __is_invocable_r_v = __is_invocable_r_impl<_Ret, __is_invocable_v<_Args...>, _Args...>;
+
+template <bool __is_invocable, class... _Args>
+inline const bool __is_nothrow_invocable_impl = false;
+
+template <class... _Args>
+inline const bool __is_nothrow_invocable_impl<true, _Args...> = noexcept(__builtin_invoke(std::declval<_Args>()...));
+
+template <class... _Args>
+inline const bool __is_nothrow_invocable_v = __is_nothrow_invocable_impl<__is_invocable_v<_Args...>, _Args...>;
+
+template <bool __is_invocable, class _Ret, class... _Args>
+inline const bool __is_nothrow_invocable_r_impl = false;
+
+template <class _Ret, class... _Args>
+inline const bool __is_nothrow_invocable_r_impl<true, _Ret, _Args...> =
+    __is_nothrow_core_convertible_v<__invoke_result_t<_Args...>, _Ret> || is_void<_Ret>::value;
+
+template <class _Ret, class... _Args>
+inline const bool __is_nothrow_invocable_r_v =
+    __is_nothrow_invocable_r_impl<__is_nothrow_invocable_v<_Args...>, _Ret, _Args...>;
+
+#  if _LIBCPP_STD_VER >= 17
+
+// is_invocable
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_invocable : bool_constant<__is_invocable_v<_Fn, _Args...> > {};
+
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_invocable_r : bool_constant<__is_invocable_r_v<_Ret, _Fn, _Args...>> {};
+
+template <class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_invocable_v = __is_invocable_v<_Fn, _Args...>;
+
+template <class _Ret, class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_invocable_r_v = is_invocable_r<_Ret, _Fn, _Args...>::value;
+
+// is_nothrow_invocable
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_nothrow_invocable : bool_constant<__is_nothrow_invocable_v<_Fn, _Args...> > {};
+
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_nothrow_invocable_r
+    : integral_constant<bool, __is_nothrow_invocable_r_v<_Ret, _Fn, _Args...>> {};
+
+template <class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_nothrow_invocable_v = __is_nothrow_invocable_v<_Fn, _Args...>;
+
+template <class _Ret, class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_nothrow_invocable_r_v =
+    __is_nothrow_invocable_r_v<_Ret, _Fn, _Args...>;
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS invoke_result : __invoke_result<_Fn, _Args...> {};
+
+template <class _Fn, class... _Args>
+using invoke_result_t = __invoke_result_t<_Fn, _Args...>;
+
+#  endif // _LIBCPP_STD_VER >= 17
+
+#else // __has_builtin(__builtin_invoke)
+
 template <class _DecayedFp>
 struct __member_pointer_class_type {};
 
@@ -211,21 +318,21 @@ struct __nothrow_invokable_r_imp<true, false, _Ret, _Fp, _Args...> {
   template <class _Tp>
   static void __test_noexcept(_Tp) _NOEXCEPT;
 
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   static const bool value = false;
-#else
+#  else
   static const bool value =
       noexcept(_ThisT::__test_noexcept<_Ret>(std::__invoke(std::declval<_Fp>(), std::declval<_Args>()...)));
-#endif
+#  endif
 };
 
 template <class _Ret, class _Fp, class... _Args>
 struct __nothrow_invokable_r_imp<true, true, _Ret, _Fp, _Args...> {
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   static const bool value = false;
-#else
+#  else
   static const bool value = noexcept(std::__invoke(std::declval<_Fp>(), std::declval<_Args>()...));
-#endif
+#  endif
 };
 
 template <class _Ret, class _Fp, class... _Args>
@@ -236,22 +343,6 @@ template <class _Fp, class... _Args>
 using __nothrow_invokable _LIBCPP_NODEBUG =
     __nothrow_invokable_r_imp<__is_invocable<_Fp, _Args...>::value, true, void, _Fp, _Args...>;
 
-template <class _Ret, bool = is_void<_Ret>::value>
-struct __invoke_void_return_wrapper {
-  template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
-    return std::__invoke(std::forward<_Args>(__args)...);
-  }
-};
-
-template <class _Ret>
-struct __invoke_void_return_wrapper<_Ret, true> {
-  template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
-    std::__invoke(std::forward<_Args>(__args)...);
-  }
-};
-
 template <class _Func, class... _Args>
 inline const bool __is_invocable_v = __is_invocable<_Func, _Args...>::value;
 
@@ -268,12 +359,7 @@ struct __invoke_result
 template <class _Func, class... _Args>
 using __invoke_result_t _LIBCPP_NODEBUG = typename __invoke_result<_Func, _Args...>::type;
 
-template <class _Ret, class... _Args>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ret __invoke_r(_Args&&... __args) {
-  return __invoke_void_return_wrapper<_Ret>::__call(std::forward<_Args>(__args)...);
-}
-
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
 // is_invocable
 
@@ -311,7 +397,30 @@ struct _LIBCPP_NO_SPECIALIZATIONS invoke_result : __invoke_result<_Fn, _Args...>
 template <class _Fn, class... _Args>
 using invoke_result_t = typename invoke_result<_Fn, _Args...>::type;
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
+
+#endif // __has_builtin(__builtin_invoke_r)
+
+template <class _Ret, bool = is_void<_Ret>::value>
+struct __invoke_void_return_wrapper {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
+    return std::__invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Ret>
+struct __invoke_void_return_wrapper<_Ret, true> {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
+    std::__invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Ret, class... _Args>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ret __invoke_r(_Args&&... __args) {
+  return __invoke_void_return_wrapper<_Ret>::__call(std::forward<_Args>(__args)...);
+}
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_array.h b/libcxx/include/__type_traits/is_array.h
index 1d94370378c61..e734d1a3043ee 100644
--- a/libcxx/include/__type_traits/is_array.h
+++ b/libcxx/include/__type_traits/is_array.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -19,32 +18,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_array) &&                                                                                       \
-    (!defined(_LIBCPP_COMPILER_CLANG_BASED) || (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1900))
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_array : _BoolConstant<__is_array(_Tp)> {};
 
-#  if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_array_v = __is_array(_Tp);
-#  endif
-
-#else
-
-template <class _Tp>
-struct is_array : public false_type {};
-template <class _Tp>
-struct is_array<_Tp[]> : public true_type {};
-template <class _Tp, size_t _Np>
-struct is_array<_Tp[_Np]> : public true_type {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_array_v = is_array<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_array)
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_assignable.h b/libcxx/include/__type_traits/is_assignable.h
index 253e86ba774ea..da9080db3b76f 100644
--- a/libcxx/include/__type_traits/is_assignable.h
+++ b/libcxx/include/__type_traits/is_assignable.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_bounded_array.h b/libcxx/include/__type_traits/is_bounded_array.h
index aec488826ec44..8a41e07aa019b 100644
--- a/libcxx/include/__type_traits/is_bounded_array.h
+++ b/libcxx/include/__type_traits/is_bounded_array.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -19,26 +18,16 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class>
-inline const bool __is_bounded_array_v = false;
-template <class _Tp, size_t _Np>
-inline const bool __is_bounded_array_v<_Tp[_Np]> = true;
+template <class _Tp>
+inline const bool __is_bounded_array_v = __is_bounded_array(_Tp);
 
 #if _LIBCPP_STD_VER >= 20
 
-template <class>
-struct _LIBCPP_NO_SPECIALIZATIONS is_bounded_array : false_type {};
-
-_LIBCPP_DIAGNOSTIC_PUSH
-#  if __has_warning("-Winvalid-specialization")
-_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-specialization")
-#  endif
-template <class _Tp, size_t _Np>
-struct is_bounded_array<_Tp[_Np]> : true_type {};
-_LIBCPP_DIAGNOSTIC_POP
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS is_bounded_array : bool_constant<__is_bounded_array(_Tp)> {};
 
 template <class _Tp>
-_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_bounded_array_v = is_bounded_array<_Tp>::value;
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_bounded_array_v = __is_bounded_array(_Tp);
 
 #endif
 
diff --git a/libcxx/include/__type_traits/is_const.h b/libcxx/include/__type_traits/is_const.h
index 4ec354d7f9bf3..85cc32bff1eaf 100644
--- a/libcxx/include/__type_traits/is_const.h
+++ b/libcxx/include/__type_traits/is_const.h
@@ -18,29 +18,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_const)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_const : _BoolConstant<__is_const(_Tp)> {};
 
-#  if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_const_v = __is_const(_Tp);
-#  endif
-
-#else
-
-template <class _Tp>
-struct is_const : public false_type {};
-template <class _Tp>
-struct is_const<_Tp const> : public true_type {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_const_v = is_const<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_const)
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_constructible.h b/libcxx/include/__type_traits/is_constructible.h
index feafb70b1f684..28d3048719b03 100644
--- a/libcxx/include/__type_traits/is_constructible.h
+++ b/libcxx/include/__type_traits/is_constructible.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_core_convertible.h b/libcxx/include/__type_traits/is_core_convertible.h
index ca3a346c17cd7..0b8e94e101db1 100644
--- a/libcxx/include/__type_traits/is_core_convertible.h
+++ b/libcxx/include/__type_traits/is_core_convertible.h
@@ -37,6 +37,15 @@ concept __core_convertible_to = __is_core_convertible<_Tp, _Up>::value;
 
 #endif // _LIBCPP_STD_VER >= 20
 
+template <class _Tp, class _Up, bool = __is_core_convertible<_Tp, _Up>::value>
+inline const bool __is_nothrow_core_convertible_v = false;
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp, class _Up>
+inline const bool __is_nothrow_core_convertible_v<_Tp, _Up, true> =
+    noexcept(static_cast<void (*)(_Up) noexcept>(0)(static_cast<_Tp (*)() noexcept>(0)()));
+#endif
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
diff --git a/libcxx/include/__type_traits/is_nothrow_assignable.h b/libcxx/include/__type_traits/is_nothrow_assignable.h
index 903dead4435e8..15ad047c566e3 100644
--- a/libcxx/include/__type_traits/is_nothrow_assignable.h
+++ b/libcxx/include/__type_traits/is_nothrow_assignable.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_nothrow_constructible.h b/libcxx/include/__type_traits/is_nothrow_constructible.h
index bd14c1c40516f..bc4c2ee45b9f5 100644
--- a/libcxx/include/__type_traits/is_nothrow_constructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_constructible.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_pointer.h b/libcxx/include/__type_traits/is_pointer.h
index 3c58656e0e61b..1c3a93d1256c9 100644
--- a/libcxx/include/__type_traits/is_pointer.h
+++ b/libcxx/include/__type_traits/is_pointer.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__type_traits/integral_constant.h>
-#include <__type_traits/remove_cv.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -19,8 +18,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_pointer)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_pointer : _BoolConstant<__is_pointer(_Tp)> {};
 
@@ -29,36 +26,6 @@ template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_pointer_v = __is_pointer(_Tp);
 #  endif
 
-#else // __has_builtin(__is_pointer)
-
-template <class _Tp>
-struct __libcpp_is_pointer : false_type {};
-template <class _Tp>
-struct __libcpp_is_pointer<_Tp*> : true_type {};
-
-template <class _Tp>
-struct __libcpp_remove_objc_qualifiers {
-  typedef _Tp type;
-};
-#  if __has_feature(objc_arc)
-// clang-format off
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __strong> { typedef _Tp type; };
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __weak> { typedef _Tp type; };
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __autoreleasing> { typedef _Tp type; };
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __unsafe_unretained> { typedef _Tp type; };
-// clang-format on
-#  endif
-
-template <class _Tp>
-struct is_pointer : __libcpp_is_pointer<typename __libcpp_remove_objc_qualifiers<__remove_cv_t<_Tp> >::type> {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_pointer_v = is_pointer<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_pointer)
-
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_IS_POINTER_H
diff --git a/libcxx/include/__type_traits/is_swappable.h b/libcxx/include/__type_traits/is_swappable.h
index 2afe784e8d896..5c060402f16d1 100644
--- a/libcxx/include/__type_traits/is_swappable.h
+++ b/libcxx/include/__type_traits/is_swappable.h
@@ -11,7 +11,7 @@
 
 #include <__config>
 #include <__cstddef/size_t.h>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_assignable.h>
diff --git a/libcxx/include/__type_traits/is_trivially_assignable.h b/libcxx/include/__type_traits/is_trivially_assignable.h
index 6548de991be4c..c95ed9dd57be4 100644
--- a/libcxx/include/__type_traits/is_trivially_assignable.h
+++ b/libcxx/include/__type_traits/is_trivially_assignable.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_trivially_constructible.h b/libcxx/include/__type_traits/is_trivially_constructible.h
index 4e36c910cc042..820b04287c517 100644
--- a/libcxx/include/__type_traits/is_trivially_constructible.h
+++ b/libcxx/include/__type_traits/is_trivially_constructible.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_volatile.h b/libcxx/include/__type_traits/is_volatile.h
index 22e9c297c800e..c654ebba9299e 100644
--- a/libcxx/include/__type_traits/is_volatile.h
+++ b/libcxx/include/__type_traits/is_volatile.h
@@ -18,29 +18,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_volatile)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_volatile : _BoolConstant<__is_volatile(_Tp)> {};
 
-#  if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_volatile_v = __is_volatile(_Tp);
-#  endif
-
-#else
-
-template <class _Tp>
-struct is_volatile : public false_type {};
-template <class _Tp>
-struct is_volatile<_Tp volatile> : public true_type {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_volatile_v = is_volatile<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_volatile)
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/remove_all_extents.h b/libcxx/include/__type_traits/remove_all_extents.h
index 2c7aea2bfd182..a9aaad488078a 100644
--- a/libcxx/include/__type_traits/remove_all_extents.h
+++ b/libcxx/include/__type_traits/remove_all_extents.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -18,31 +17,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__remove_all_extents) && !defined(_LIBCPP_COMPILER_GCC)
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS remove_all_extents {
   using type _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-using __remove_all_extents_t _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
+using __remove_all_extents_t _LIBCPP_NODEBUG = typename remove_all_extents<_Tp>::type;
 #else
 template <class _Tp>
-struct remove_all_extents {
-  typedef _Tp type;
-};
-template <class _Tp>
-struct remove_all_extents<_Tp[]> {
-  typedef typename remove_all_extents<_Tp>::type type;
-};
-template <class _Tp, size_t _Np>
-struct remove_all_extents<_Tp[_Np]> {
-  typedef typename remove_all_extents<_Tp>::type type;
-};
-
-template <class _Tp>
-using __remove_all_extents_t = typename remove_all_extents<_Tp>::type;
-#endif // __has_builtin(__remove_all_extents)
+using __remove_all_extents_t _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/remove_cvref.h b/libcxx/include/__type_traits/remove_cvref.h
index 760deddca946d..c568e8df3099d 100644
--- a/libcxx/include/__type_traits/remove_cvref.h
+++ b/libcxx/include/__type_traits/remove_cvref.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
 
 #include <__config>
-#include <__type_traits/is_same.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -31,9 +30,6 @@ template <class _Tp>
 using __remove_cvref_t _LIBCPP_NODEBUG = __remove_cvref(_Tp);
 #endif // __has_builtin(__remove_cvref)
 
-template <class _Tp, class _Up>
-using __is_same_uncvref _LIBCPP_NODEBUG = _IsSame<__remove_cvref_t<_Tp>, __remove_cvref_t<_Up> >;
-
 #if _LIBCPP_STD_VER >= 20
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS remove_cvref {
diff --git a/libcxx/include/__type_traits/remove_extent.h b/libcxx/include/__type_traits/remove_extent.h
index 7745af14bdb17..108b867bd0b49 100644
--- a/libcxx/include/__type_traits/remove_extent.h
+++ b/libcxx/include/__type_traits/remove_extent.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -18,31 +17,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__remove_extent) && !defined(_LIBCPP_COMPILER_GCC)
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS remove_extent {
   using type _LIBCPP_NODEBUG = __remove_extent(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-using __remove_extent_t _LIBCPP_NODEBUG = __remove_extent(_Tp);
+using __remove_extent_t _LIBCPP_NODEBUG = typename remove_extent<_Tp>::type;
 #else
 template <class _Tp>
-struct remove_extent {
-  typedef _Tp type;
-};
-template <class _Tp>
-struct remove_extent<_Tp[]> {
-  typedef _Tp type;
-};
-template <class _Tp, size_t _Np>
-struct remove_extent<_Tp[_Np]> {
-  typedef _Tp type;
-};
-
-template <class _Tp>
-using __remove_extent_t = typename remove_extent<_Tp>::type;
-#endif // __has_builtin(__remove_extent)
+using __remove_extent_t _LIBCPP_NODEBUG = __remove_extent(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/experimental/iterator b/libcxx/include/experimental/iterator
index 565bb83903ac3..0ed95188d7715 100644
--- a/libcxx/include/experimental/iterator
+++ b/libcxx/include/experimental/iterator
@@ -130,6 +130,11 @@ _LIBCPP_POP_MACROS
 #    include <optional>
 #    include <type_traits>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_EXPERIMENTAL_ITERATOR
diff --git a/libcxx/include/experimental/memory b/libcxx/include/experimental/memory
index 7ebae2847e5ea..ffbd8f3a065a6 100644
--- a/libcxx/include/experimental/memory
+++ b/libcxx/include/experimental/memory
@@ -57,8 +57,8 @@ public:
 #  include <__cstddef/size_t.h>
 #  include <__functional/hash.h>
 #  include <__functional/operations.h>
-#  include <__type_traits/add_lvalue_reference.h>
 #  include <__type_traits/add_pointer.h>
+#  include <__type_traits/add_reference.h>
 #  include <__type_traits/common_type.h>
 #  include <__type_traits/enable_if.h>
 #  include <__type_traits/is_convertible.h>
diff --git a/libcxx/include/flat_map b/libcxx/include/flat_map
index 2552450081734..eea9896165f06 100644
--- a/libcxx/include/flat_map
+++ b/libcxx/include/flat_map
@@ -72,6 +72,15 @@ namespace std {
 #  include <version>
 
 // standard required includes
+
+// [iterator.range]
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
+
+// [flat.map.syn]
 #  include <compare>
 #  include <initializer_list>
 
diff --git a/libcxx/include/flat_set b/libcxx/include/flat_set
index ebbb3a0247f3e..66041a42b79cf 100644
--- a/libcxx/include/flat_set
+++ b/libcxx/include/flat_set
@@ -65,6 +65,15 @@ namespace std {
 #  include <version>
 
 // standard required includes
+
+// [iterator.range]
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
+
+// [flat.set.syn]
 #  include <compare>
 #  include <initializer_list>
 
diff --git a/libcxx/include/functional b/libcxx/include/functional
index b121a19d6bcb3..9ebcd818ec840 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -565,6 +565,7 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #    include <__functional/bind_front.h>
 #    include <__functional/identity.h>
 #    include <__functional/ranges_operations.h>
+#    include <__type_traits/common_reference.h>
 #    include <__type_traits/unwrap_ref.h>
 #  endif
 
@@ -599,6 +600,10 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #    include <utility>
 #    include <vector>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 23
+#    include <__vector/vector.h>
+#  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_FUNCTIONAL
diff --git a/libcxx/include/future b/libcxx/include/future
index f49ed34a9c767..31932aeeb6b82 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -393,7 +393,7 @@ template <class R, class Alloc> struct uses_allocator<packaged_task<R>, Alloc>;
 #    include <__system_error/error_code.h>
 #    include <__system_error/error_condition.h>
 #    include <__thread/thread.h>
-#    include <__type_traits/add_lvalue_reference.h>
+#    include <__type_traits/add_reference.h>
 #    include <__type_traits/aligned_storage.h>
 #    include <__type_traits/conditional.h>
 #    include <__type_traits/decay.h>
diff --git a/libcxx/include/iomanip b/libcxx/include/iomanip
index a97b4a94d232f..d65d66d716597 100644
--- a/libcxx/include/iomanip
+++ b/libcxx/include/iomanip
@@ -49,10 +49,12 @@ template <class charT, class traits, class Allocator>
 
 #  if _LIBCPP_HAS_LOCALIZATION
 
+#    include <__iterator/istreambuf_iterator.h>
+#    include <__locale_dir/money.h>
+#    include <__locale_dir/time.h>
 #    include <__ostream/put_character_sequence.h>
 #    include <ios>
 #    include <iosfwd>
-#    include <locale>
 #    include <version>
 
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -564,6 +566,11 @@ _LIBCPP_END_NAMESPACE_STD
 #    include <unordered_map>
 #    include <vector>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_IOMANIP
diff --git a/libcxx/include/istream b/libcxx/include/istream
index 02546902494e3..93def61a8b477 100644
--- a/libcxx/include/istream
+++ b/libcxx/include/istream
@@ -167,6 +167,7 @@ template <class Stream, class T>
 
 #    include <__fwd/istream.h>
 #    include <__iterator/istreambuf_iterator.h>
+#    include <__locale_dir/num.h>
 #    include <__ostream/basic_ostream.h>
 #    include <__type_traits/conjunction.h>
 #    include <__type_traits/enable_if.h>
@@ -176,7 +177,7 @@ template <class Stream, class T>
 #    include <__utility/forward.h>
 #    include <bitset>
 #    include <ios>
-#    include <locale>
+#    include <streambuf>
 #    include <version>
 
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -1411,6 +1412,10 @@ _LIBCPP_POP_MACROS
 #    include <type_traits>
 #  endif
 
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_ISTREAM
diff --git a/libcxx/include/locale b/libcxx/include/locale
index b206bf8f162f9..b50f9c87c7281 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -194,3496 +194,20 @@ template <class charT> class messages_byname;
 
 #  if _LIBCPP_HAS_LOCALIZATION
 
-#    include <__algorithm/copy.h>
-#    include <__algorithm/equal.h>
-#    include <__algorithm/find.h>
-#    include <__algorithm/max.h>
-#    include <__algorithm/reverse.h>
-#    include <__algorithm/unwrap_iter.h>
-#    include <__assert>
-#    include <__charconv/to_chars_integral.h>
-#    include <__charconv/traits.h>
-#    include <__iterator/access.h>
-#    include <__iterator/back_insert_iterator.h>
-#    include <__iterator/istreambuf_iterator.h>
-#    include <__iterator/ostreambuf_iterator.h>
 #    include <__locale>
-#    include <__locale_dir/pad_and_output.h>
-#    include <__memory/addressof.h>
-#    include <__memory/unique_ptr.h>
-#    include <__new/exceptions.h>
-#    include <__system_error/errc.h>
-#    include <__type_traits/make_unsigned.h>
-#    include <cerrno>
-#    include <cstdio>
-#    include <cstdlib>
-#    include <ctime>
+#    include <__locale_dir/messages.h>
+#    include <__locale_dir/money.h>
+#    include <__locale_dir/num.h>
+#    include <__locale_dir/time.h>
+#    include <__locale_dir/wbuffer_convert.h>
+#    include <__locale_dir/wstring_convert.h>
 #    include <ios>
-#    include <limits>
-#    include <streambuf>
 #    include <version>
 
-// TODO: Properly qualify calls now that the locale base API defines functions instead of macros
-// NOLINTBEGIN(libcpp-robust-against-adl)
-
-#    if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
-// Most unix variants have catopen.  These are the specific ones that don't.
-#      if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
-#        define _LIBCPP_HAS_CATOPEN 1
-#        include <nl_types.h>
-#      else
-#        define _LIBCPP_HAS_CATOPEN 0
-#      endif
-#    else
-#      define _LIBCPP_HAS_CATOPEN 0
-#    endif
-
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #      pragma GCC system_header
 #    endif
 
-_LIBCPP_PUSH_MACROS
-#    include <__undef_macros>
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#    if defined(__APPLE__) || defined(__FreeBSD__)
-#      define _LIBCPP_GET_C_LOCALE 0
-#    elif defined(__NetBSD__)
-#      define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
-#    else
-#      define _LIBCPP_GET_C_LOCALE __cloc()
-// Get the C locale object
-_LIBCPP_EXPORTED_FROM_ABI __locale::__locale_t __cloc();
-#      define __cloc_defined
-#    endif
-
-// __scan_keyword
-// Scans [__b, __e) until a match is found in the basic_strings range
-//  [__kb, __ke) or until it can be shown that there is no match in [__kb, __ke).
-//  __b will be incremented (visibly), consuming CharT until a match is found
-//  or proved to not exist.  A keyword may be "", in which will match anything.
-//  If one keyword is a prefix of another, and the next CharT in the input
-//  might match another keyword, the algorithm will attempt to find the longest
-//  matching keyword.  If the longer matching keyword ends up not matching, then
-//  no keyword match is found.  If no keyword match is found, __ke is returned
-//  and failbit is set in __err.
-//  Else an iterator pointing to the matching keyword is found.  If more than
-//  one keyword matches, an iterator to the first matching keyword is returned.
-//  If on exit __b == __e, eofbit is set in __err.  If __case_sensitive is false,
-//  __ct is used to force to lower case before comparing characters.
-//  Examples:
-//  Keywords:  "a", "abb"
-//  If the input is "a", the first keyword matches and eofbit is set.
-//  If the input is "abc", no match is found and "ab" are consumed.
-template <class _InputIterator, class _ForwardIterator, class _Ctype>
-_LIBCPP_HIDE_FROM_ABI _ForwardIterator __scan_keyword(
-    _InputIterator& __b,
-    _InputIterator __e,
-    _ForwardIterator __kb,
-    _ForwardIterator __ke,
-    const _Ctype& __ct,
-    ios_base::iostate& __err,
-    bool __case_sensitive = true) {
-  typedef typename iterator_traits<_InputIterator>::value_type _CharT;
-  size_t __nkw                       = static_cast<size_t>(std::distance(__kb, __ke));
-  const unsigned char __doesnt_match = '\0';
-  const unsigned char __might_match  = '\1';
-  const unsigned char __does_match   = '\2';
-  unsigned char __statbuf[100];
-  unsigned char* __status = __statbuf;
-  unique_ptr<unsigned char, void (*)(void*)> __stat_hold(nullptr, free);
-  if (__nkw > sizeof(__statbuf)) {
-    __status = (unsigned char*)malloc(__nkw);
-    if (__status == nullptr)
-      std::__throw_bad_alloc();
-    __stat_hold.reset(__status);
-  }
-  size_t __n_might_match = __nkw; // At this point, any keyword might match
-  size_t __n_does_match  = 0;     // but none of them definitely do
-  // Initialize all statuses to __might_match, except for "" keywords are __does_match
-  unsigned char* __st = __status;
-  for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
-    if (!__ky->empty())
-      *__st = __might_match;
-    else {
-      *__st = __does_match;
-      --__n_might_match;
-      ++__n_does_match;
-    }
-  }
-  // While there might be a match, test keywords against the next CharT
-  for (size_t __indx = 0; __b != __e && __n_might_match > 0; ++__indx) {
-    // Peek at the next CharT but don't consume it
-    _CharT __c = *__b;
-    if (!__case_sensitive)
-      __c = __ct.toupper(__c);
-    bool __consume = false;
-    // For each keyword which might match, see if the __indx character is __c
-    // If a match if found, consume __c
-    // If a match is found, and that is the last character in the keyword,
-    //    then that keyword matches.
-    // If the keyword doesn't match this character, then change the keyword
-    //    to doesn't match
-    __st = __status;
-    for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
-      if (*__st == __might_match) {
-        _CharT __kc = (*__ky)[__indx];
-        if (!__case_sensitive)
-          __kc = __ct.toupper(__kc);
-        if (__c == __kc) {
-          __consume = true;
-          if (__ky->size() == __indx + 1) {
-            *__st = __does_match;
-            --__n_might_match;
-            ++__n_does_match;
-          }
-        } else {
-          *__st = __doesnt_match;
-          --__n_might_match;
-        }
-      }
-    }
-    // consume if we matched a character
-    if (__consume) {
-      ++__b;
-      // If we consumed a character and there might be a matched keyword that
-      //   was marked matched on a previous iteration, then such keywords
-      //   which are now marked as not matching.
-      if (__n_might_match + __n_does_match > 1) {
-        __st = __status;
-        for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
-          if (*__st == __does_match && __ky->size() != __indx + 1) {
-            *__st = __doesnt_match;
-            --__n_does_match;
-          }
-        }
-      }
-    }
-  }
-  // We've exited the loop because we hit eof and/or we have no more "might matches".
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  // Return the first matching result
-  for (__st = __status; __kb != __ke; ++__kb, (void)++__st)
-    if (*__st == __does_match)
-      break;
-  if (__kb == __ke)
-    __err |= ios_base::failbit;
-  return __kb;
-}
-
-struct _LIBCPP_EXPORTED_FROM_ABI __num_get_base {
-  static const int __num_get_buf_sz = 40;
-
-  static int __get_base(ios_base&);
-  static const char __src[33]; // "0123456789abcdefABCDEFxX+-pPiInN"
-  // count of leading characters in __src used for parsing integers ("012..X+-")
-  static const size_t __int_chr_cnt = 26;
-  // count of leading characters in __src used for parsing floating-point values ("012..-pP")
-  static const size_t __fp_chr_cnt = 28;
-};
-
-_LIBCPP_EXPORTED_FROM_ABI void
-__check_grouping(const string& __grouping, unsigned* __g, unsigned* __g_end, ios_base::iostate& __err);
-
-template <class _CharT>
-struct __num_get : protected __num_get_base {
-  static string __stage2_float_prep(ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep);
-
-  static int __stage2_float_loop(
-      _CharT __ct,
-      bool& __in_units,
-      char& __exp,
-      char* __a,
-      char*& __a_end,
-      _CharT __decimal_point,
-      _CharT __thousands_sep,
-      const string& __grouping,
-      unsigned* __g,
-      unsigned*& __g_end,
-      unsigned& __dc,
-      _CharT* __atoms);
-
-  [[__deprecated__("This exists only for ABI compatibility")]] static string
-  __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
-  static int __stage2_int_loop(
-      _CharT __ct,
-      int __base,
-      char* __a,
-      char*& __a_end,
-      unsigned& __dc,
-      _CharT __thousands_sep,
-      const string& __grouping,
-      unsigned* __g,
-      unsigned*& __g_end,
-      _CharT* __atoms);
-
-  _LIBCPP_HIDE_FROM_ABI static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
-    locale __loc                 = __iob.getloc();
-    const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
-    __thousands_sep              = __np.thousands_sep();
-    return __np.grouping();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const {
-    return __do_widen_p(__iob, __atoms);
-  }
-
-private:
-  template <typename _Tp>
-  _LIBCPP_HIDE_FROM_ABI const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
-    locale __loc = __iob.getloc();
-    use_facet<ctype<_Tp> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
-    return __atoms;
-  }
-
-  _LIBCPP_HIDE_FROM_ABI const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
-    (void)__iob;
-    (void)__atoms;
-    return __src;
-  }
-};
-
-template <class _CharT>
-string __num_get<_CharT>::__stage2_float_prep(
-    ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep) {
-  locale __loc = __iob.getloc();
-  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __fp_chr_cnt, __atoms);
-  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
-  __decimal_point              = __np.decimal_point();
-  __thousands_sep              = __np.thousands_sep();
-  return __np.grouping();
-}
-
-template <class _CharT>
-int __num_get<_CharT>::__stage2_int_loop(
-    _CharT __ct,
-    int __base,
-    char* __a,
-    char*& __a_end,
-    unsigned& __dc,
-    _CharT __thousands_sep,
-    const string& __grouping,
-    unsigned* __g,
-    unsigned*& __g_end,
-    _CharT* __atoms) {
-  if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
-    *__a_end++ = __ct == __atoms[24] ? '+' : '-';
-    __dc       = 0;
-    return 0;
-  }
-  if (__grouping.size() != 0 && __ct == __thousands_sep) {
-    if (__g_end - __g < __num_get_buf_sz) {
-      *__g_end++ = __dc;
-      __dc       = 0;
-    }
-    return 0;
-  }
-  ptrdiff_t __f = std::find(__atoms, __atoms + __int_chr_cnt, __ct) - __atoms;
-  if (__f >= 24)
-    return -1;
-  switch (__base) {
-  case 8:
-  case 10:
-    if (__f >= __base)
-      return -1;
-    break;
-  case 16:
-    if (__f < 22)
-      break;
-    if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
-      __dc       = 0;
-      *__a_end++ = __src[__f];
-      return 0;
-    }
-    return -1;
-  }
-  *__a_end++ = __src[__f];
-  ++__dc;
-  return 0;
-}
-
-template <class _CharT>
-int __num_get<_CharT>::__stage2_float_loop(
-    _CharT __ct,
-    bool& __in_units,
-    char& __exp,
-    char* __a,
-    char*& __a_end,
-    _CharT __decimal_point,
-    _CharT __thousands_sep,
-    const string& __grouping,
-    unsigned* __g,
-    unsigned*& __g_end,
-    unsigned& __dc,
-    _CharT* __atoms) {
-  if (__ct == __decimal_point) {
-    if (!__in_units)
-      return -1;
-    __in_units = false;
-    *__a_end++ = '.';
-    if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
-      *__g_end++ = __dc;
-    return 0;
-  }
-  if (__ct == __thousands_sep && __grouping.size() != 0) {
-    if (!__in_units)
-      return -1;
-    if (__g_end - __g < __num_get_buf_sz) {
-      *__g_end++ = __dc;
-      __dc       = 0;
-    }
-    return 0;
-  }
-  ptrdiff_t __f = std::find(__atoms, __atoms + __num_get_base::__fp_chr_cnt, __ct) - __atoms;
-  if (__f >= static_cast<ptrdiff_t>(__num_get_base::__fp_chr_cnt))
-    return -1;
-  char __x = __src[__f];
-  if (__x == '-' || __x == '+') {
-    if (__a_end == __a || (std::toupper(__a_end[-1]) == std::toupper(__exp))) {
-      *__a_end++ = __x;
-      return 0;
-    }
-    return -1;
-  }
-  if (__x == 'x' || __x == 'X')
-    __exp = 'P';
-  else if (std::toupper(__x) == __exp) {
-    __exp = std::tolower(__exp);
-    if (__in_units) {
-      __in_units = false;
-      if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
-        *__g_end++ = __dc;
-    }
-  }
-  *__a_end++ = __x;
-  if (__f >= 22)
-    return 0;
-  ++__dc;
-  return 0;
-}
-
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<wchar_t>;
-#    endif
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __do_strtod(const char* __a, char** __p2);
-
-template <>
-inline _LIBCPP_HIDE_FROM_ABI float __do_strtod<float>(const char* __a, char** __p2) {
-  return __locale::__strtof(__a, __p2, _LIBCPP_GET_C_LOCALE);
-}
-
-template <>
-inline _LIBCPP_HIDE_FROM_ABI double __do_strtod<double>(const char* __a, char** __p2) {
-  return __locale::__strtod(__a, __p2, _LIBCPP_GET_C_LOCALE);
-}
-
-template <>
-inline _LIBCPP_HIDE_FROM_ABI long double __do_strtod<long double>(const char* __a, char** __p2) {
-  return __locale::__strtold(__a, __p2, _LIBCPP_GET_C_LOCALE);
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __num_get_float(const char* __a, const char* __a_end, ios_base::iostate& __err) {
-  if (__a != __a_end) {
-    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
-    errno                                                     = 0;
-    char* __p2;
-    _Tp __ld                                                     = std::__do_strtod<_Tp>(__a, &__p2);
-    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
-    if (__current_errno == 0)
-      errno = __save_errno;
-    if (__p2 != __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    } else if (__current_errno == ERANGE)
-      __err = ios_base::failbit;
-    return __ld;
-  }
-  __err = ios_base::failbit;
-  return 0;
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
-  if (__a != __a_end) {
-    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
-    errno                                                     = 0;
-    char* __p2;
-    long long __ll = __locale::__strtoll(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
-    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
-    if (__current_errno == 0)
-      errno = __save_errno;
-    if (__p2 != __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    } else if (__current_errno == ERANGE || __ll < numeric_limits<_Tp>::min() || numeric_limits<_Tp>::max() < __ll) {
-      __err = ios_base::failbit;
-      if (__ll > 0)
-        return numeric_limits<_Tp>::max();
-      else
-        return numeric_limits<_Tp>::min();
-    }
-    return static_cast<_Tp>(__ll);
-  }
-  __err = ios_base::failbit;
-  return 0;
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
-  if (__a != __a_end) {
-    const bool __negate = *__a == '-';
-    if (__negate && ++__a == __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    }
-    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
-    errno                                                     = 0;
-    char* __p2;
-    unsigned long long __ll = __locale::__strtoull(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
-    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
-    if (__current_errno == 0)
-      errno = __save_errno;
-    if (__p2 != __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    } else if (__current_errno == ERANGE || numeric_limits<_Tp>::max() < __ll) {
-      __err = ios_base::failbit;
-      return numeric_limits<_Tp>::max();
-    }
-    _Tp __res = static_cast<_Tp>(__ll);
-    if (__negate)
-      __res = -__res;
-    return __res;
-  }
-  __err = ios_base::failbit;
-  return 0;
-}
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class num_get : public locale::facet, private __num_get<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _InputIterator iter_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit num_get(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_get() override {}
-
-  template <class _Fp>
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  __do_get_floating_point(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Fp& __v) const {
-    // Stage 1, nothing to do
-    // Stage 2
-    char_type __atoms[__num_get_base::__fp_chr_cnt];
-    char_type __decimal_point;
-    char_type __thousands_sep;
-    string __grouping = this->__stage2_float_prep(__iob, __atoms, __decimal_point, __thousands_sep);
-    string __buf;
-    __buf.resize(__buf.capacity());
-    char* __a     = &__buf[0];
-    char* __a_end = __a;
-    unsigned __g[__num_get_base::__num_get_buf_sz];
-    unsigned* __g_end        = __g;
-    unsigned __dc            = 0;
-    bool __in_units          = true;
-    char __exp               = 'E';
-    bool __is_leading_parsed = false;
-    for (; __b != __e; ++__b) {
-      if (__a_end == __a + __buf.size()) {
-        size_t __tmp = __buf.size();
-        __buf.resize(2 * __buf.size());
-        __buf.resize(__buf.capacity());
-        __a     = &__buf[0];
-        __a_end = __a + __tmp;
-      }
-      if (this->__stage2_float_loop(
-              *__b,
-              __in_units,
-              __exp,
-              __a,
-              __a_end,
-              __decimal_point,
-              __thousands_sep,
-              __grouping,
-              __g,
-              __g_end,
-              __dc,
-              __atoms))
-        break;
-
-      // the leading character excluding the sign must be a decimal digit
-      if (!__is_leading_parsed) {
-        if (__a_end - __a >= 1 && __a[0] != '-' && __a[0] != '+') {
-          if (('0' <= __a[0] && __a[0] <= '9') || __a[0] == '.')
-            __is_leading_parsed = true;
-          else
-            break;
-        } else if (__a_end - __a >= 2 && (__a[0] == '-' || __a[0] == '+')) {
-          if (('0' <= __a[1] && __a[1] <= '9') || __a[1] == '.')
-            __is_leading_parsed = true;
-          else
-            break;
-        }
-      }
-    }
-    if (__grouping.size() != 0 && __in_units && __g_end - __g < __num_get_base::__num_get_buf_sz)
-      *__g_end++ = __dc;
-    // Stage 3
-    __v = std::__num_get_float<_Fp>(__a, __a_end, __err);
-    // Digit grouping checked
-    __check_grouping(__grouping, __g, __g_end, __err);
-    // EOF checked
-    if (__b == __e)
-      __err |= ios_base::eofbit;
-    return __b;
-  }
-
-  template <class _Signed>
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  __do_get_signed(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const {
-    // Stage 1
-    int __base = this->__get_base(__iob);
-    // Stage 2
-    char_type __thousands_sep;
-    const int __atoms_size = __num_get_base::__int_chr_cnt;
-    char_type __atoms1[__atoms_size];
-    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
-    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-    string __buf;
-    __buf.resize(__buf.capacity());
-    char* __a     = &__buf[0];
-    char* __a_end = __a;
-    unsigned __g[__num_get_base::__num_get_buf_sz];
-    unsigned* __g_end = __g;
-    unsigned __dc     = 0;
-    for (; __b != __e; ++__b) {
-      if (__a_end == __a + __buf.size()) {
-        size_t __tmp = __buf.size();
-        __buf.resize(2 * __buf.size());
-        __buf.resize(__buf.capacity());
-        __a     = &__buf[0];
-        __a_end = __a + __tmp;
-      }
-      if (this->__stage2_int_loop(
-              *__b,
-              __base,
-              __a,
-              __a_end,
-              __dc,
-              __thousands_sep,
-              __grouping,
-              __g,
-              __g_end,
-              const_cast<char_type*>(__atoms)))
-        break;
-    }
-    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
-      *__g_end++ = __dc;
-    // Stage 3
-    __v = std::__num_get_signed_integral<_Signed>(__a, __a_end, __err, __base);
-    // Digit grouping checked
-    __check_grouping(__grouping, __g, __g_end, __err);
-    // EOF checked
-    if (__b == __e)
-      __err |= ios_base::eofbit;
-    return __b;
-  }
-
-  template <class _Unsigned>
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  __do_get_unsigned(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const {
-    // Stage 1
-    int __base = this->__get_base(__iob);
-    // Stage 2
-    char_type __thousands_sep;
-    const int __atoms_size = __num_get_base::__int_chr_cnt;
-    char_type __atoms1[__atoms_size];
-    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
-    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-    string __buf;
-    __buf.resize(__buf.capacity());
-    char* __a     = &__buf[0];
-    char* __a_end = __a;
-    unsigned __g[__num_get_base::__num_get_buf_sz];
-    unsigned* __g_end = __g;
-    unsigned __dc     = 0;
-    for (; __b != __e; ++__b) {
-      if (__a_end == __a + __buf.size()) {
-        size_t __tmp = __buf.size();
-        __buf.resize(2 * __buf.size());
-        __buf.resize(__buf.capacity());
-        __a     = &__buf[0];
-        __a_end = __a + __tmp;
-      }
-      if (this->__stage2_int_loop(
-              *__b,
-              __base,
-              __a,
-              __a_end,
-              __dc,
-              __thousands_sep,
-              __grouping,
-              __g,
-              __g_end,
-              const_cast<char_type*>(__atoms)))
-        break;
-    }
-    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
-      *__g_end++ = __dc;
-    // Stage 3
-    __v = std::__num_get_unsigned_integral<_Unsigned>(__a, __a_end, __err, __base);
-    // Digit grouping checked
-    __check_grouping(__grouping, __g, __g_end, __err);
-    // EOF checked
-    if (__b == __e)
-      __err |= ios_base::eofbit;
-    return __b;
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const;
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
-    return this->__do_get_signed(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
-    return this->__do_get_signed(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
-    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
-    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const;
-};
-
-template <class _CharT, class _InputIterator>
-locale::id num_get<_CharT, _InputIterator>::id;
-
-template <class _CharT, class _InputIterator>
-_InputIterator num_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
-  if ((__iob.flags() & ios_base::boolalpha) == 0) {
-    long __lv = -1;
-    __b       = do_get(__b, __e, __iob, __err, __lv);
-    switch (__lv) {
-    case 0:
-      __v = false;
-      break;
-    case 1:
-      __v = true;
-      break;
-    default:
-      __v   = true;
-      __err = ios_base::failbit;
-      break;
-    }
-    return __b;
-  }
-  const ctype<_CharT>& __ct    = std::use_facet<ctype<_CharT> >(__iob.getloc());
-  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__iob.getloc());
-  typedef typename numpunct<_CharT>::string_type string_type;
-  const string_type __names[2] = {__np.truename(), __np.falsename()};
-  const string_type* __i       = std::__scan_keyword(__b, __e, __names, __names + 2, __ct, __err);
-  __v                          = __i == __names;
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator num_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
-  // Stage 1
-  int __base = 16;
-  // Stage 2
-  char_type __atoms[__num_get_base::__int_chr_cnt];
-  char_type __thousands_sep = char_type();
-  string __grouping;
-  std::use_facet<ctype<_CharT> >(__iob.getloc())
-      .widen(__num_get_base::__src, __num_get_base::__src + __num_get_base::__int_chr_cnt, __atoms);
-  string __buf;
-  __buf.resize(__buf.capacity());
-  char* __a     = &__buf[0];
-  char* __a_end = __a;
-  unsigned __g[__num_get_base::__num_get_buf_sz];
-  unsigned* __g_end = __g;
-  unsigned __dc     = 0;
-  for (; __b != __e; ++__b) {
-    if (__a_end == __a + __buf.size()) {
-      size_t __tmp = __buf.size();
-      __buf.resize(2 * __buf.size());
-      __buf.resize(__buf.capacity());
-      __a     = &__buf[0];
-      __a_end = __a + __tmp;
-    }
-    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
-      break;
-  }
-  // Stage 3
-  __buf.resize(__a_end - __a);
-  if (__locale::__sscanf(__buf.c_str(), _LIBCPP_GET_C_LOCALE, "%p", &__v) != 1)
-    __err = ios_base::failbit;
-  // EOF checked
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<wchar_t>;
-#    endif
-
-struct _LIBCPP_EXPORTED_FROM_ABI __num_put_base {
-protected:
-  static void __format_int(char* __fmt, const char* __len, bool __signd, ios_base::fmtflags __flags);
-  static bool __format_float(char* __fmt, const char* __len, ios_base::fmtflags __flags);
-  static char* __identify_padding(char* __nb, char* __ne, const ios_base& __iob);
-};
-
-template <class _CharT>
-struct __num_put : protected __num_put_base {
-  static void __widen_and_group_int(
-      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
-  static void __widen_and_group_float(
-      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
-};
-
-template <class _CharT>
-void __num_put<_CharT>::__widen_and_group_int(
-    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
-  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
-  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
-  string __grouping             = __npt.grouping();
-  if (__grouping.empty()) {
-    __ct.widen(__nb, __ne, __ob);
-    __oe = __ob + (__ne - __nb);
-  } else {
-    __oe       = __ob;
-    char* __nf = __nb;
-    if (*__nf == '-' || *__nf == '+')
-      *__oe++ = __ct.widen(*__nf++);
-    if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
-      *__oe++ = __ct.widen(*__nf++);
-      *__oe++ = __ct.widen(*__nf++);
-    }
-    std::reverse(__nf, __ne);
-    _CharT __thousands_sep = __npt.thousands_sep();
-    unsigned __dc          = 0;
-    unsigned __dg          = 0;
-    for (char* __p = __nf; __p < __ne; ++__p) {
-      if (static_cast<unsigned>(__grouping[__dg]) > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
-        *__oe++ = __thousands_sep;
-        __dc    = 0;
-        if (__dg < __grouping.size() - 1)
-          ++__dg;
-      }
-      *__oe++ = __ct.widen(*__p);
-      ++__dc;
-    }
-    std::reverse(__ob + (__nf - __nb), __oe);
-  }
-  if (__np == __ne)
-    __op = __oe;
-  else
-    __op = __ob + (__np - __nb);
-}
-
-template <class _CharT>
-void __num_put<_CharT>::__widen_and_group_float(
-    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
-  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
-  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
-  string __grouping             = __npt.grouping();
-  __oe                          = __ob;
-  char* __nf                    = __nb;
-  if (*__nf == '-' || *__nf == '+')
-    *__oe++ = __ct.widen(*__nf++);
-  char* __ns;
-  if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
-    *__oe++ = __ct.widen(*__nf++);
-    *__oe++ = __ct.widen(*__nf++);
-    for (__ns = __nf; __ns < __ne; ++__ns)
-      if (!__locale::__isxdigit(*__ns, _LIBCPP_GET_C_LOCALE))
-        break;
-  } else {
-    for (__ns = __nf; __ns < __ne; ++__ns)
-      if (!__locale::__isdigit(*__ns, _LIBCPP_GET_C_LOCALE))
-        break;
-  }
-  if (__grouping.empty()) {
-    __ct.widen(__nf, __ns, __oe);
-    __oe += __ns - __nf;
-  } else {
-    std::reverse(__nf, __ns);
-    _CharT __thousands_sep = __npt.thousands_sep();
-    unsigned __dc          = 0;
-    unsigned __dg          = 0;
-    for (char* __p = __nf; __p < __ns; ++__p) {
-      if (__grouping[__dg] > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
-        *__oe++ = __thousands_sep;
-        __dc    = 0;
-        if (__dg < __grouping.size() - 1)
-          ++__dg;
-      }
-      *__oe++ = __ct.widen(*__p);
-      ++__dc;
-    }
-    std::reverse(__ob + (__nf - __nb), __oe);
-  }
-  for (__nf = __ns; __nf < __ne; ++__nf) {
-    if (*__nf == '.') {
-      *__oe++ = __npt.decimal_point();
-      ++__nf;
-      break;
-    } else
-      *__oe++ = __ct.widen(*__nf);
-  }
-  __ct.widen(__nf, __ne, __oe);
-  __oe += __ne - __nf;
-  if (__np == __ne)
-    __op = __oe;
-  else
-    __op = __ob + (__np - __nb);
-}
-
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class num_put : public locale::facet, private __num_put<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _OutputIterator iter_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit num_put(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_put() override {}
-
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const;
-
-  template <class _Integral>
-  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
-  __do_put_integral(iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const;
-
-  template <class _Float>
-  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
-  __do_put_floating_point(iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const;
-};
-
-template <class _CharT, class _OutputIterator>
-locale::id num_put<_CharT, _OutputIterator>::id;
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
-  if ((__iob.flags() & ios_base::boolalpha) == 0)
-    return do_put(__s, __iob, __fl, (unsigned long)__v);
-  const numpunct<char_type>& __np = std::use_facet<numpunct<char_type> >(__iob.getloc());
-  typedef typename numpunct<char_type>::string_type string_type;
-  string_type __nm = __v ? __np.truename() : __np.falsename();
-  for (typename string_type::iterator __i = __nm.begin(); __i != __nm.end(); ++__i, ++__s)
-    *__s = *__i;
-  return __s;
-}
-
-template <class _CharT, class _OutputIterator>
-template <class _Integral>
-_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_integral(
-    iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const {
-  // Stage 1 - Get number in narrow char
-
-  // Worst case is octal, with showbase enabled. Note that octal is always
-  // printed as an unsigned value.
-  using _Unsigned = typename make_unsigned<_Integral>::type;
-  _LIBCPP_CONSTEXPR const unsigned __buffer_size =
-      (numeric_limits<_Unsigned>::digits / 3)          // 1 char per 3 bits
-      + ((numeric_limits<_Unsigned>::digits % 3) != 0) // round up
-      + 2;                                             // base prefix + terminating null character
-
-  char __char_buffer[__buffer_size];
-  char* __buffer_ptr = __char_buffer;
-
-  auto __flags = __iob.flags();
-
-  auto __basefield = (__flags & ios_base::basefield);
-
-  // Extract base
-  int __base = 10;
-  if (__basefield == ios_base::oct)
-    __base = 8;
-  else if (__basefield == ios_base::hex)
-    __base = 16;
-
-  // Print '-' and make the argument unsigned
-  auto __uval = std::__to_unsigned_like(__v);
-  if (__basefield != ios_base::oct && __basefield != ios_base::hex && __v < 0) {
-    *__buffer_ptr++ = '-';
-    __uval          = std::__complement(__uval);
-  }
-
-  // Maybe add '+' prefix
-  if (std::is_signed<_Integral>::value && (__flags & ios_base::showpos) && __basefield != ios_base::oct &&
-      __basefield != ios_base::hex && __v >= 0)
-    *__buffer_ptr++ = '+';
-
-  // Add base prefix
-  if (__v != 0 && __flags & ios_base::showbase) {
-    if (__basefield == ios_base::oct) {
-      *__buffer_ptr++ = '0';
-    } else if (__basefield == ios_base::hex) {
-      *__buffer_ptr++ = '0';
-      *__buffer_ptr++ = (__flags & ios_base::uppercase ? 'X' : 'x');
-    }
-  }
-
-  auto __res = std::__to_chars_integral(__buffer_ptr, __char_buffer + __buffer_size, __uval, __base);
-  _LIBCPP_ASSERT_INTERNAL(__res.__ec == std::errc(0), "to_chars: invalid maximum buffer size computed?");
-
-  // Make letters uppercase
-  if (__flags & ios_base::hex && __flags & ios_base::uppercase) {
-    for (; __buffer_ptr != __res.__ptr; ++__buffer_ptr)
-      *__buffer_ptr = std::__hex_to_upper(*__buffer_ptr);
-  }
-
-  char* __np = this->__identify_padding(__char_buffer, __res.__ptr, __iob);
-  // Stage 2 - Widen __nar while adding thousands separators
-  char_type __o[2 * (__buffer_size - 1) - 1];
-  char_type* __op; // pad here
-  char_type* __oe; // end of output
-  this->__widen_and_group_int(__char_buffer, __np, __res.__ptr, __o, __op, __oe, __iob.getloc());
-  // [__o, __oe) contains thousands_sep'd wide number
-  // Stage 3 & 4
-  return std::__pad_and_output(__s, __o, __op, __oe, __iob, __fl);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-template <class _Float>
-_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_floating_point(
-    iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const {
-  // Stage 1 - Get number in narrow char
-  char __fmt[8]            = {'%', 0};
-  bool __specify_precision = this->__format_float(__fmt + 1, __len, __iob.flags());
-  const unsigned __nbuf    = 30;
-  char __nar[__nbuf];
-  char* __nb = __nar;
-  int __nc;
-  _LIBCPP_DIAGNOSTIC_PUSH
-  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
-  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
-  if (__specify_precision)
-    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
-  else
-    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
-  unique_ptr<char, void (*)(void*)> __nbh(nullptr, free);
-  if (__nc > static_cast<int>(__nbuf - 1)) {
-    if (__specify_precision)
-      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
-    else
-      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
-    if (__nc == -1)
-      std::__throw_bad_alloc();
-    __nbh.reset(__nb);
-  }
-  _LIBCPP_DIAGNOSTIC_POP
-  char* __ne = __nb + __nc;
-  char* __np = this->__identify_padding(__nb, __ne, __iob);
-  // Stage 2 - Widen __nar while adding thousands separators
-  char_type __o[2 * (__nbuf - 1) - 1];
-  char_type* __ob = __o;
-  unique_ptr<char_type, void (*)(void*)> __obh(0, free);
-  if (__nb != __nar) {
-    __ob = (char_type*)malloc(2 * static_cast<size_t>(__nc) * sizeof(char_type));
-    if (__ob == 0)
-      std::__throw_bad_alloc();
-    __obh.reset(__ob);
-  }
-  char_type* __op; // pad here
-  char_type* __oe; // end of output
-  this->__widen_and_group_float(__nb, __np, __ne, __ob, __op, __oe, __iob.getloc());
-  // [__o, __oe) contains thousands_sep'd wide number
-  // Stage 3 & 4
-  __s = std::__pad_and_output(__s, __ob, __op, __oe, __iob, __fl);
-  return __s;
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
-  return this->__do_put_floating_point(__s, __iob, __fl, __v, "");
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
-  return this->__do_put_floating_point(__s, __iob, __fl, __v, "L");
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
-  auto __flags = __iob.flags();
-  __iob.flags((__flags & ~ios_base::basefield & ~ios_base::uppercase) | ios_base::hex | ios_base::showbase);
-  auto __res = __do_put_integral(__s, __iob, __fl, reinterpret_cast<uintptr_t>(__v));
-  __iob.flags(__flags);
-  return __res;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _InputIterator>
-_LIBCPP_HIDE_FROM_ABI int __get_up_to_n_digits(
-    _InputIterator& __b, _InputIterator __e, ios_base::iostate& __err, const ctype<_CharT>& __ct, int __n) {
-  // Precondition:  __n >= 1
-  if (__b == __e) {
-    __err |= ios_base::eofbit | ios_base::failbit;
-    return 0;
-  }
-  // get first digit
-  _CharT __c = *__b;
-  if (!__ct.is(ctype_base::digit, __c)) {
-    __err |= ios_base::failbit;
-    return 0;
-  }
-  int __r = __ct.narrow(__c, 0) - '0';
-  for (++__b, (void)--__n; __b != __e && __n > 0; ++__b, (void)--__n) {
-    // get next digit
-    __c = *__b;
-    if (!__ct.is(ctype_base::digit, __c))
-      return __r;
-    __r = __r * 10 + __ct.narrow(__c, 0) - '0';
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __r;
-}
-
-class _LIBCPP_EXPORTED_FROM_ABI time_base {
-public:
-  enum dateorder { no_order, dmy, mdy, ymd, ydm };
-};
-
-template <class _CharT>
-class __time_get_c_storage {
-protected:
-  typedef basic_string<_CharT> string_type;
-
-  virtual const string_type* __weeks() const;
-  virtual const string_type* __months() const;
-  virtual const string_type* __am_pm() const;
-  virtual const string_type& __c() const;
-  virtual const string_type& __r() const;
-  virtual const string_type& __x() const;
-  virtual const string_type& __X() const;
-
-  _LIBCPP_HIDE_FROM_ABI ~__time_get_c_storage() {}
-};
-
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__weeks() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__months() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__am_pm() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__c() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__r() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__x() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__X() const;
-
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__weeks() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__months() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__am_pm() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__c() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__r() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__x() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__X() const;
-#    endif
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class time_get : public locale::facet, public time_base, private __time_get_c_storage<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _InputIterator iter_type;
-  typedef time_base::dateorder dateorder;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_get(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI dateorder date_order() const { return this->do_date_order(); }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_time(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_date(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_weekday(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_monthname(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_year(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod = 0)
-      const {
-    return do_get(__b, __e, __iob, __err, __tm, __fmt, __mod);
-  }
-
-  iter_type
-  get(iter_type __b,
-      iter_type __e,
-      ios_base& __iob,
-      ios_base::iostate& __err,
-      tm* __tm,
-      const char_type* __fmtb,
-      const char_type* __fmte) const;
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get() override {}
-
-  virtual dateorder do_date_order() const;
-  virtual iter_type
-  do_get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type do_get(
-      iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod) const;
-
-private:
-  void __get_white_space(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_percent(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-
-  void __get_weekdayname(
-      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_monthname(
-      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_day(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_month(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_year(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_year4(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_hour(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_12_hour(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_am_pm(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_minute(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_second(int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_weekday(int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_day_year_num(
-      int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-};
-
-template <class _CharT, class _InputIterator>
-locale::id time_get<_CharT, _InputIterator>::id;
-
-// time_get primitives
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_weekdayname(
-    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  // Note:  ignoring case comes from the POSIX strptime spec
-  const string_type* __wk = this->__weeks();
-  ptrdiff_t __i           = std::__scan_keyword(__b, __e, __wk, __wk + 14, __ct, __err, false) - __wk;
-  if (__i < 14)
-    __w = __i % 7;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_monthname(
-    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  // Note:  ignoring case comes from the POSIX strptime spec
-  const string_type* __month = this->__months();
-  ptrdiff_t __i              = std::__scan_keyword(__b, __e, __month, __month + 24, __ct, __err, false) - __month;
-  if (__i < 24)
-    __m = __i % 12;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_day(
-    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 31)
-    __d = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_month(
-    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2) - 1;
-  if (!(__err & ios_base::failbit) && 0 <= __t && __t <= 11)
-    __m = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_year(
-    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
-  if (!(__err & ios_base::failbit)) {
-    if (__t < 69)
-      __t += 2000;
-    else if (69 <= __t && __t <= 99)
-      __t += 1900;
-    __y = __t - 1900;
-  }
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_year4(
-    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
-  if (!(__err & ios_base::failbit))
-    __y = __t - 1900;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_hour(
-    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && __t <= 23)
-    __h = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_12_hour(
-    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 12)
-    __h = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_minute(
-    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && __t <= 59)
-    __m = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_second(
-    int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && __t <= 60)
-    __s = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_weekday(
-    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 1);
-  if (!(__err & ios_base::failbit) && __t <= 6)
-    __w = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_day_year_num(
-    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 3);
-  if (!(__err & ios_base::failbit) && __t <= 365)
-    __d = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_white_space(
-    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
-    ;
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_am_pm(
-    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  const string_type* __ap = this->__am_pm();
-  if (__ap[0].size() + __ap[1].size() == 0) {
-    __err |= ios_base::failbit;
-    return;
-  }
-  ptrdiff_t __i = std::__scan_keyword(__b, __e, __ap, __ap + 2, __ct, __err, false) - __ap;
-  if (__i == 0 && __h == 12)
-    __h = 0;
-  else if (__i == 1 && __h < 12)
-    __h += 12;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_percent(
-    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  if (__b == __e) {
-    __err |= ios_base::eofbit | ios_base::failbit;
-    return;
-  }
-  if (__ct.narrow(*__b, 0) != '%')
-    __err |= ios_base::failbit;
-  else if (++__b == __e)
-    __err |= ios_base::eofbit;
-}
-
-// time_get end primitives
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::get(
-    iter_type __b,
-    iter_type __e,
-    ios_base& __iob,
-    ios_base::iostate& __err,
-    tm* __tm,
-    const char_type* __fmtb,
-    const char_type* __fmte) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __err                        = ios_base::goodbit;
-  while (__fmtb != __fmte && __err == ios_base::goodbit) {
-    if (__b == __e) {
-      __err = ios_base::failbit;
-      break;
-    }
-    if (__ct.narrow(*__fmtb, 0) == '%') {
-      if (++__fmtb == __fmte) {
-        __err = ios_base::failbit;
-        break;
-      }
-      char __cmd = __ct.narrow(*__fmtb, 0);
-      char __opt = '\0';
-      if (__cmd == 'E' || __cmd == '0') {
-        if (++__fmtb == __fmte) {
-          __err = ios_base::failbit;
-          break;
-        }
-        __opt = __cmd;
-        __cmd = __ct.narrow(*__fmtb, 0);
-      }
-      __b = do_get(__b, __e, __iob, __err, __tm, __cmd, __opt);
-      ++__fmtb;
-    } else if (__ct.is(ctype_base::space, *__fmtb)) {
-      for (++__fmtb; __fmtb != __fmte && __ct.is(ctype_base::space, *__fmtb); ++__fmtb)
-        ;
-      for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
-        ;
-    } else if (__ct.toupper(*__b) == __ct.toupper(*__fmtb)) {
-      ++__b;
-      ++__fmtb;
-    } else
-      __err = ios_base::failbit;
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-typename time_get<_CharT, _InputIterator>::dateorder time_get<_CharT, _InputIterator>::do_date_order() const {
-  return mdy;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_time(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const char_type __fmt[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
-  return get(__b, __e, __iob, __err, __tm, __fmt, __fmt + sizeof(__fmt) / sizeof(__fmt[0]));
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_date(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const string_type& __fmt = this->__x();
-  return get(__b, __e, __iob, __err, __tm, __fmt.data(), __fmt.data() + __fmt.size());
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_weekday(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_monthname(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_year(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __get_year(__tm->tm_year, __b, __e, __err, __ct);
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char) const {
-  __err                        = ios_base::goodbit;
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  switch (__fmt) {
-  case 'a':
-  case 'A':
-    __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
-    break;
-  case 'b':
-  case 'B':
-  case 'h':
-    __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
-    break;
-  case 'c': {
-    const string_type& __fm = this->__c();
-    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
-  } break;
-  case 'd':
-  case 'e':
-    __get_day(__tm->tm_mday, __b, __e, __err, __ct);
-    break;
-  case 'D': {
-    const char_type __fm[] = {'%', 'm', '/', '%', 'd', '/', '%', 'y'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'F': {
-    const char_type __fm[] = {'%', 'Y', '-', '%', 'm', '-', '%', 'd'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'H':
-    __get_hour(__tm->tm_hour, __b, __e, __err, __ct);
-    break;
-  case 'I':
-    __get_12_hour(__tm->tm_hour, __b, __e, __err, __ct);
-    break;
-  case 'j':
-    __get_day_year_num(__tm->tm_yday, __b, __e, __err, __ct);
-    break;
-  case 'm':
-    __get_month(__tm->tm_mon, __b, __e, __err, __ct);
-    break;
-  case 'M':
-    __get_minute(__tm->tm_min, __b, __e, __err, __ct);
-    break;
-  case 'n':
-  case 't':
-    __get_white_space(__b, __e, __err, __ct);
-    break;
-  case 'p':
-    __get_am_pm(__tm->tm_hour, __b, __e, __err, __ct);
-    break;
-  case 'r': {
-    const char_type __fm[] = {'%', 'I', ':', '%', 'M', ':', '%', 'S', ' ', '%', 'p'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'R': {
-    const char_type __fm[] = {'%', 'H', ':', '%', 'M'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'S':
-    __get_second(__tm->tm_sec, __b, __e, __err, __ct);
-    break;
-  case 'T': {
-    const char_type __fm[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'w':
-    __get_weekday(__tm->tm_wday, __b, __e, __err, __ct);
-    break;
-  case 'x':
-    return do_get_date(__b, __e, __iob, __err, __tm);
-  case 'X': {
-    const string_type& __fm = this->__X();
-    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
-  } break;
-  case 'y':
-    __get_year(__tm->tm_year, __b, __e, __err, __ct);
-    break;
-  case 'Y':
-    __get_year4(__tm->tm_year, __b, __e, __err, __ct);
-    break;
-  case '%':
-    __get_percent(__b, __e, __err, __ct);
-    break;
-  default:
-    __err |= ios_base::failbit;
-  }
-  return __b;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<wchar_t>;
-#    endif
-
-class _LIBCPP_EXPORTED_FROM_ABI __time_get {
-protected:
-  __locale::__locale_t __loc_;
-
-  __time_get(const char* __nm);
-  __time_get(const string& __nm);
-  ~__time_get();
-};
-
-template <class _CharT>
-class __time_get_storage : public __time_get {
-protected:
-  typedef basic_string<_CharT> string_type;
-
-  string_type __weeks_[14];
-  string_type __months_[24];
-  string_type __am_pm_[2];
-  string_type __c_;
-  string_type __r_;
-  string_type __x_;
-  string_type __X_;
-
-  explicit __time_get_storage(const char* __nm);
-  explicit __time_get_storage(const string& __nm);
-
-  _LIBCPP_HIDE_FROM_ABI ~__time_get_storage() {}
-
-  time_base::dateorder __do_date_order() const;
-
-private:
-  void init(const ctype<_CharT>&);
-  string_type __analyze(char __fmt, const ctype<_CharT>&);
-};
-
-#    define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                    \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;              \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                           \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                         \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                           \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(         \
-          char, const ctype<_CharT>&);                                                                                 \
-      extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()     \
-          const;                                                                                                       \
-      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);           \
-      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);         \
-      extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);           \
-      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                \
-      __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);                                               \
-      /**/
-
-_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t)
-#    endif
-#    undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class time_get_byname : public time_get<_CharT, _InputIterator>, private __time_get_storage<_CharT> {
-public:
-  typedef time_base::dateorder dateorder;
-  typedef _InputIterator iter_type;
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const char* __nm, size_t __refs = 0)
-      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
-  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const string& __nm, size_t __refs = 0)
-      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get_byname() override {}
-
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL dateorder do_date_order() const override { return this->__do_date_order(); }
-
-private:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __weeks() const override { return this->__weeks_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __months() const override { return this->__months_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __am_pm() const override { return this->__am_pm_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __c() const override { return this->__c_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __r() const override { return this->__r_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __x() const override { return this->__x_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __X() const override { return this->__X_; }
-};
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<wchar_t>;
-#    endif
-
-class _LIBCPP_EXPORTED_FROM_ABI __time_put {
-  __locale::__locale_t __loc_;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI __time_put() : __loc_(_LIBCPP_GET_C_LOCALE) {}
-  __time_put(const char* __nm);
-  __time_put(const string& __nm);
-  ~__time_put();
-  void __do_put(char* __nb, char*& __ne, const tm* __tm, char __fmt, char __mod) const;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-  void __do_put(wchar_t* __wb, wchar_t*& __we, const tm* __tm, char __fmt, char __mod) const;
-#    endif
-};
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class time_put : public locale::facet, private __time_put {
-public:
-  typedef _CharT char_type;
-  typedef _OutputIterator iter_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_put(size_t __refs = 0) : locale::facet(__refs) {}
-
-  iter_type
-  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
-      const;
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, char __fmt, char __mod = 0) const {
-    return do_put(__s, __iob, __fl, __tm, __fmt, __mod);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put() override {}
-  virtual iter_type do_put(iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_put(const char* __nm, size_t __refs) : locale::facet(__refs), __time_put(__nm) {}
-  _LIBCPP_HIDE_FROM_ABI explicit time_put(const string& __nm, size_t __refs)
-      : locale::facet(__refs), __time_put(__nm) {}
-};
-
-template <class _CharT, class _OutputIterator>
-locale::id time_put<_CharT, _OutputIterator>::id;
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator time_put<_CharT, _OutputIterator>::put(
-    iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
-    const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  for (; __pb != __pe; ++__pb) {
-    if (__ct.narrow(*__pb, 0) == '%') {
-      if (++__pb == __pe) {
-        *__s++ = __pb[-1];
-        break;
-      }
-      char __mod = 0;
-      char __fmt = __ct.narrow(*__pb, 0);
-      if (__fmt == 'E' || __fmt == 'O') {
-        if (++__pb == __pe) {
-          *__s++ = __pb[-2];
-          *__s++ = __pb[-1];
-          break;
-        }
-        __mod = __fmt;
-        __fmt = __ct.narrow(*__pb, 0);
-      }
-      __s = do_put(__s, __iob, __fl, __tm, __fmt, __mod);
-    } else
-      *__s++ = *__pb;
-  }
-  return __s;
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator time_put<_CharT, _OutputIterator>::do_put(
-    iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const {
-  char_type __nar[100];
-  char_type* __nb = __nar;
-  char_type* __ne = __nb + 100;
-  __do_put(__nb, __ne, __tm, __fmt, __mod);
-  return std::copy(__nb, __ne, __s);
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class time_put_byname : public time_put<_CharT, _OutputIterator> {
-public:
-  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const char* __nm, size_t __refs = 0)
-      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const string& __nm, size_t __refs = 0)
-      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put_byname() override {}
-};
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<wchar_t>;
-#    endif
-
-// money_base
-
-class _LIBCPP_EXPORTED_FROM_ABI money_base {
-public:
-  enum part { none, space, symbol, sign, value };
-  struct pattern {
-    char field[4];
-  };
-
-  _LIBCPP_HIDE_FROM_ABI money_base() {}
-};
-
-// moneypunct
-
-template <class _CharT, bool _International = false>
-class moneypunct : public locale::facet, public money_base {
-public:
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit moneypunct(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
-  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
-  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
-  _LIBCPP_HIDE_FROM_ABI string_type curr_symbol() const { return do_curr_symbol(); }
-  _LIBCPP_HIDE_FROM_ABI string_type positive_sign() const { return do_positive_sign(); }
-  _LIBCPP_HIDE_FROM_ABI string_type negative_sign() const { return do_negative_sign(); }
-  _LIBCPP_HIDE_FROM_ABI int frac_digits() const { return do_frac_digits(); }
-  _LIBCPP_HIDE_FROM_ABI pattern pos_format() const { return do_pos_format(); }
-  _LIBCPP_HIDE_FROM_ABI pattern neg_format() const { return do_neg_format(); }
-
-  static locale::id id;
-  static const bool intl = _International;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct() override {}
-
-  virtual char_type do_decimal_point() const { return numeric_limits<char_type>::max(); }
-  virtual char_type do_thousands_sep() const { return numeric_limits<char_type>::max(); }
-  virtual string do_grouping() const { return string(); }
-  virtual string_type do_curr_symbol() const { return string_type(); }
-  virtual string_type do_positive_sign() const { return string_type(); }
-  virtual string_type do_negative_sign() const { return string_type(1, '-'); }
-  virtual int do_frac_digits() const { return 0; }
-  virtual pattern do_pos_format() const {
-    pattern __p = {{symbol, sign, none, value}};
-    return __p;
-  }
-  virtual pattern do_neg_format() const {
-    pattern __p = {{symbol, sign, none, value}};
-    return __p;
-  }
-};
-
-template <class _CharT, bool _International>
-locale::id moneypunct<_CharT, _International>::id;
-
-template <class _CharT, bool _International>
-const bool moneypunct<_CharT, _International>::intl;
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, true>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, true>;
-#    endif
-
-// moneypunct_byname
-
-template <class _CharT, bool _International = false>
-class moneypunct_byname : public moneypunct<_CharT, _International> {
-public:
-  typedef money_base::pattern pattern;
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const char* __nm, size_t __refs = 0)
-      : moneypunct<_CharT, _International>(__refs) {
-    init(__nm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const string& __nm, size_t __refs = 0)
-      : moneypunct<_CharT, _International>(__refs) {
-    init(__nm.c_str());
-  }
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct_byname() override {}
-
-  char_type do_decimal_point() const override { return __decimal_point_; }
-  char_type do_thousands_sep() const override { return __thousands_sep_; }
-  string do_grouping() const override { return __grouping_; }
-  string_type do_curr_symbol() const override { return __curr_symbol_; }
-  string_type do_positive_sign() const override { return __positive_sign_; }
-  string_type do_negative_sign() const override { return __negative_sign_; }
-  int do_frac_digits() const override { return __frac_digits_; }
-  pattern do_pos_format() const override { return __pos_format_; }
-  pattern do_neg_format() const override { return __neg_format_; }
-
-private:
-  char_type __decimal_point_;
-  char_type __thousands_sep_;
-  string __grouping_;
-  string_type __curr_symbol_;
-  string_type __positive_sign_;
-  string_type __negative_sign_;
-  int __frac_digits_;
-  pattern __pos_format_;
-  pattern __neg_format_;
-
-  void init(const char*);
-};
-
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, false>::init(const char*);
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, true>::init(const char*);
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, true>;
-
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, false>::init(const char*);
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, true>::init(const char*);
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, true>;
-#    endif
-
-// money_get
-
-template <class _CharT>
-class __money_get {
-protected:
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI __money_get() {}
-
-  static void __gather_info(
-      bool __intl,
-      const locale& __loc,
-      money_base::pattern& __pat,
-      char_type& __dp,
-      char_type& __ts,
-      string& __grp,
-      string_type& __sym,
-      string_type& __psn,
-      string_type& __nsn,
-      int& __fd);
-};
-
-template <class _CharT>
-void __money_get<_CharT>::__gather_info(
-    bool __intl,
-    const locale& __loc,
-    money_base::pattern& __pat,
-    char_type& __dp,
-    char_type& __ts,
-    string& __grp,
-    string_type& __sym,
-    string_type& __psn,
-    string_type& __nsn,
-    int& __fd) {
-  if (__intl) {
-    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
-    __pat                                   = __mp.neg_format();
-    __nsn                                   = __mp.negative_sign();
-    __psn                                   = __mp.positive_sign();
-    __dp                                    = __mp.decimal_point();
-    __ts                                    = __mp.thousands_sep();
-    __grp                                   = __mp.grouping();
-    __sym                                   = __mp.curr_symbol();
-    __fd                                    = __mp.frac_digits();
-  } else {
-    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
-    __pat                                    = __mp.neg_format();
-    __nsn                                    = __mp.negative_sign();
-    __psn                                    = __mp.positive_sign();
-    __dp                                     = __mp.decimal_point();
-    __ts                                     = __mp.thousands_sep();
-    __grp                                    = __mp.grouping();
-    __sym                                    = __mp.curr_symbol();
-    __fd                                     = __mp.frac_digits();
-  }
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<wchar_t>;
-#    endif
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class money_get : public locale::facet, private __money_get<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _InputIterator iter_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit money_get(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-    return do_get(__b, __e, __intl, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
-    return do_get(__b, __e, __intl, __iob, __err, __v);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_get() override {}
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const;
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const;
-
-private:
-  static bool __do_get(
-      iter_type& __b,
-      iter_type __e,
-      bool __intl,
-      const locale& __loc,
-      ios_base::fmtflags __flags,
-      ios_base::iostate& __err,
-      bool& __neg,
-      const ctype<char_type>& __ct,
-      unique_ptr<char_type, void (*)(void*)>& __wb,
-      char_type*& __wn,
-      char_type* __we);
-};
-
-template <class _CharT, class _InputIterator>
-locale::id money_get<_CharT, _InputIterator>::id;
-
-_LIBCPP_EXPORTED_FROM_ABI void __do_nothing(void*);
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void __double_or_nothing(unique_ptr<_Tp, void (*)(void*)>& __b, _Tp*& __n, _Tp*& __e) {
-  bool __owns      = __b.get_deleter() != __do_nothing;
-  size_t __cur_cap = static_cast<size_t>(__e - __b.get()) * sizeof(_Tp);
-  size_t __new_cap = __cur_cap < numeric_limits<size_t>::max() / 2 ? 2 * __cur_cap : numeric_limits<size_t>::max();
-  if (__new_cap == 0)
-    __new_cap = sizeof(_Tp);
-  size_t __n_off = static_cast<size_t>(__n - __b.get());
-  _Tp* __t       = (_Tp*)std::realloc(__owns ? __b.get() : 0, __new_cap);
-  if (__t == 0)
-    std::__throw_bad_alloc();
-  if (__owns)
-    __b.release();
-  else
-    std::memcpy(__t, __b.get(), __cur_cap);
-  __b = unique_ptr<_Tp, void (*)(void*)>(__t, free);
-  __new_cap /= sizeof(_Tp);
-  __n = __b.get() + __n_off;
-  __e = __b.get() + __new_cap;
-}
-
-// true == success
-template <class _CharT, class _InputIterator>
-bool money_get<_CharT, _InputIterator>::__do_get(
-    iter_type& __b,
-    iter_type __e,
-    bool __intl,
-    const locale& __loc,
-    ios_base::fmtflags __flags,
-    ios_base::iostate& __err,
-    bool& __neg,
-    const ctype<char_type>& __ct,
-    unique_ptr<char_type, void (*)(void*)>& __wb,
-    char_type*& __wn,
-    char_type* __we) {
-  if (__b == __e) {
-    __err |= ios_base::failbit;
-    return false;
-  }
-  const unsigned __bz = 100;
-  unsigned __gbuf[__bz];
-  unique_ptr<unsigned, void (*)(void*)> __gb(__gbuf, __do_nothing);
-  unsigned* __gn = __gb.get();
-  unsigned* __ge = __gn + __bz;
-  money_base::pattern __pat;
-  char_type __dp;
-  char_type __ts;
-  string __grp;
-  string_type __sym;
-  string_type __psn;
-  string_type __nsn;
-  // Capture the spaces read into money_base::{space,none} so they
-  // can be compared to initial spaces in __sym.
-  string_type __spaces;
-  int __fd;
-  __money_get<_CharT>::__gather_info(__intl, __loc, __pat, __dp, __ts, __grp, __sym, __psn, __nsn, __fd);
-  const string_type* __trailing_sign = 0;
-  __wn                               = __wb.get();
-  for (unsigned __p = 0; __p < 4 && __b != __e; ++__p) {
-    switch (__pat.field[__p]) {
-    case money_base::space:
-      if (__p != 3) {
-        if (__ct.is(ctype_base::space, *__b))
-          __spaces.push_back(*__b++);
-        else {
-          __err |= ios_base::failbit;
-          return false;
-        }
-      }
-      [[__fallthrough__]];
-    case money_base::none:
-      if (__p != 3) {
-        while (__b != __e && __ct.is(ctype_base::space, *__b))
-          __spaces.push_back(*__b++);
-      }
-      break;
-    case money_base::sign:
-      if (__psn.size() > 0 && *__b == __psn[0]) {
-        ++__b;
-        __neg = false;
-        if (__psn.size() > 1)
-          __trailing_sign = std::addressof(__psn);
-        break;
-      }
-      if (__nsn.size() > 0 && *__b == __nsn[0]) {
-        ++__b;
-        __neg = true;
-        if (__nsn.size() > 1)
-          __trailing_sign = std::addressof(__nsn);
-        break;
-      }
-      if (__psn.size() > 0 && __nsn.size() > 0) { // sign is required
-        __err |= ios_base::failbit;
-        return false;
-      }
-      if (__psn.size() == 0 && __nsn.size() == 0)
-        // locale has no way of specifying a sign. Use the initial value of __neg as a default
-        break;
-      __neg = (__nsn.size() == 0);
-      break;
-    case money_base::symbol: {
-      bool __more_needed =
-          __trailing_sign || (__p < 2) || (__p == 2 && __pat.field[3] != static_cast<char>(money_base::none));
-      bool __sb = (__flags & ios_base::showbase) != 0;
-      if (__sb || __more_needed) {
-        typename string_type::const_iterator __sym_space_end = __sym.begin();
-        if (__p > 0 && (__pat.field[__p - 1] == money_base::none || __pat.field[__p - 1] == money_base::space)) {
-          // Match spaces we've already read against spaces at
-          // the beginning of __sym.
-          while (__sym_space_end != __sym.end() && __ct.is(ctype_base::space, *__sym_space_end))
-            ++__sym_space_end;
-          const size_t __num_spaces = __sym_space_end - __sym.begin();
-          if (__num_spaces > __spaces.size() ||
-              !std::equal(__spaces.end() - __num_spaces, __spaces.end(), __sym.begin())) {
-            // No match. Put __sym_space_end back at the
-            // beginning of __sym, which will prevent a
-            // match in the next loop.
-            __sym_space_end = __sym.begin();
-          }
-        }
-        typename string_type::const_iterator __sym_curr_char = __sym_space_end;
-        while (__sym_curr_char != __sym.end() && __b != __e && *__b == *__sym_curr_char) {
-          ++__b;
-          ++__sym_curr_char;
-        }
-        if (__sb && __sym_curr_char != __sym.end()) {
-          __err |= ios_base::failbit;
-          return false;
-        }
-      }
-    } break;
-    case money_base::value: {
-      unsigned __ng = 0;
-      for (; __b != __e; ++__b) {
-        char_type __c = *__b;
-        if (__ct.is(ctype_base::digit, __c)) {
-          if (__wn == __we)
-            std::__double_or_nothing(__wb, __wn, __we);
-          *__wn++ = __c;
-          ++__ng;
-        } else if (__grp.size() > 0 && __ng > 0 && __c == __ts) {
-          if (__gn == __ge)
-            std::__double_or_nothing(__gb, __gn, __ge);
-          *__gn++ = __ng;
-          __ng    = 0;
-        } else
-          break;
-      }
-      if (__gb.get() != __gn && __ng > 0) {
-        if (__gn == __ge)
-          std::__double_or_nothing(__gb, __gn, __ge);
-        *__gn++ = __ng;
-      }
-      if (__fd > 0) {
-        if (__b == __e || *__b != __dp) {
-          __err |= ios_base::failbit;
-          return false;
-        }
-        for (++__b; __fd > 0; --__fd, ++__b) {
-          if (__b == __e || !__ct.is(ctype_base::digit, *__b)) {
-            __err |= ios_base::failbit;
-            return false;
-          }
-          if (__wn == __we)
-            std::__double_or_nothing(__wb, __wn, __we);
-          *__wn++ = *__b;
-        }
-      }
-      if (__wn == __wb.get()) {
-        __err |= ios_base::failbit;
-        return false;
-      }
-    } break;
-    }
-  }
-  if (__trailing_sign) {
-    for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, ++__b) {
-      if (__b == __e || *__b != (*__trailing_sign)[__i]) {
-        __err |= ios_base::failbit;
-        return false;
-      }
-    }
-  }
-  if (__gb.get() != __gn) {
-    ios_base::iostate __et = ios_base::goodbit;
-    __check_grouping(__grp, __gb.get(), __gn, __et);
-    if (__et) {
-      __err |= ios_base::failbit;
-      return false;
-    }
-  }
-  return true;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator money_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-  const int __bz = 100;
-  char_type __wbuf[__bz];
-  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
-  char_type* __wn;
-  char_type* __we              = __wbuf + __bz;
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  bool __neg                   = false;
-  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
-    const char __src[] = "0123456789";
-    char_type __atoms[sizeof(__src) - 1];
-    __ct.widen(__src, __src + (sizeof(__src) - 1), __atoms);
-    char __nbuf[__bz];
-    char* __nc          = __nbuf;
-    const char* __nc_in = __nc;
-    unique_ptr<char, void (*)(void*)> __h(nullptr, free);
-    if (__wn - __wb.get() > __bz - 2) {
-      __h.reset((char*)malloc(static_cast<size_t>(__wn - __wb.get() + 2)));
-      if (__h.get() == nullptr)
-        std::__throw_bad_alloc();
-      __nc    = __h.get();
-      __nc_in = __nc;
-    }
-    if (__neg)
-      *__nc++ = '-';
-    for (const char_type* __w = __wb.get(); __w < __wn; ++__w, ++__nc)
-      *__nc = __src[std::find(__atoms, std::end(__atoms), *__w) - __atoms];
-    *__nc = char();
-    if (sscanf(__nc_in, "%Lf", &__v) != 1)
-      std::__throw_runtime_error("money_get error");
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator money_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
-  const int __bz = 100;
-  char_type __wbuf[__bz];
-  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
-  char_type* __wn;
-  char_type* __we              = __wbuf + __bz;
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  bool __neg                   = false;
-  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
-    __v.clear();
-    if (__neg)
-      __v.push_back(__ct.widen('-'));
-    char_type __z = __ct.widen('0');
-    char_type* __w;
-    for (__w = __wb.get(); __w < __wn - 1; ++__w)
-      if (*__w != __z)
-        break;
-    __v.append(__w, __wn);
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<wchar_t>;
-#    endif
-
-// money_put
-
-template <class _CharT>
-class __money_put {
-protected:
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI __money_put() {}
-
-  static void __gather_info(
-      bool __intl,
-      bool __neg,
-      const locale& __loc,
-      money_base::pattern& __pat,
-      char_type& __dp,
-      char_type& __ts,
-      string& __grp,
-      string_type& __sym,
-      string_type& __sn,
-      int& __fd);
-  static void __format(
-      char_type* __mb,
-      char_type*& __mi,
-      char_type*& __me,
-      ios_base::fmtflags __flags,
-      const char_type* __db,
-      const char_type* __de,
-      const ctype<char_type>& __ct,
-      bool __neg,
-      const money_base::pattern& __pat,
-      char_type __dp,
-      char_type __ts,
-      const string& __grp,
-      const string_type& __sym,
-      const string_type& __sn,
-      int __fd);
-};
-
-template <class _CharT>
-void __money_put<_CharT>::__gather_info(
-    bool __intl,
-    bool __neg,
-    const locale& __loc,
-    money_base::pattern& __pat,
-    char_type& __dp,
-    char_type& __ts,
-    string& __grp,
-    string_type& __sym,
-    string_type& __sn,
-    int& __fd) {
-  if (__intl) {
-    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
-    if (__neg) {
-      __pat = __mp.neg_format();
-      __sn  = __mp.negative_sign();
-    } else {
-      __pat = __mp.pos_format();
-      __sn  = __mp.positive_sign();
-    }
-    __dp  = __mp.decimal_point();
-    __ts  = __mp.thousands_sep();
-    __grp = __mp.grouping();
-    __sym = __mp.curr_symbol();
-    __fd  = __mp.frac_digits();
-  } else {
-    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
-    if (__neg) {
-      __pat = __mp.neg_format();
-      __sn  = __mp.negative_sign();
-    } else {
-      __pat = __mp.pos_format();
-      __sn  = __mp.positive_sign();
-    }
-    __dp  = __mp.decimal_point();
-    __ts  = __mp.thousands_sep();
-    __grp = __mp.grouping();
-    __sym = __mp.curr_symbol();
-    __fd  = __mp.frac_digits();
-  }
-}
-
-template <class _CharT>
-void __money_put<_CharT>::__format(
-    char_type* __mb,
-    char_type*& __mi,
-    char_type*& __me,
-    ios_base::fmtflags __flags,
-    const char_type* __db,
-    const char_type* __de,
-    const ctype<char_type>& __ct,
-    bool __neg,
-    const money_base::pattern& __pat,
-    char_type __dp,
-    char_type __ts,
-    const string& __grp,
-    const string_type& __sym,
-    const string_type& __sn,
-    int __fd) {
-  __me = __mb;
-  for (char __p : __pat.field) {
-    switch (__p) {
-    case money_base::none:
-      __mi = __me;
-      break;
-    case money_base::space:
-      __mi    = __me;
-      *__me++ = __ct.widen(' ');
-      break;
-    case money_base::sign:
-      if (!__sn.empty())
-        *__me++ = __sn[0];
-      break;
-    case money_base::symbol:
-      if (!__sym.empty() && (__flags & ios_base::showbase))
-        __me = std::copy(__sym.begin(), __sym.end(), __me);
-      break;
-    case money_base::value: {
-      // remember start of value so we can reverse it
-      char_type* __t = __me;
-      // find beginning of digits
-      if (__neg)
-        ++__db;
-      // find end of digits
-      const char_type* __d;
-      for (__d = __db; __d < __de; ++__d)
-        if (!__ct.is(ctype_base::digit, *__d))
-          break;
-      // print fractional part
-      if (__fd > 0) {
-        int __f;
-        for (__f = __fd; __d > __db && __f > 0; --__f)
-          *__me++ = *--__d;
-        char_type __z = __f > 0 ? __ct.widen('0') : char_type();
-        for (; __f > 0; --__f)
-          *__me++ = __z;
-        *__me++ = __dp;
-      }
-      // print units part
-      if (__d == __db) {
-        *__me++ = __ct.widen('0');
-      } else {
-        unsigned __ng = 0;
-        unsigned __ig = 0;
-        unsigned __gl = __grp.empty() ? numeric_limits<unsigned>::max() : static_cast<unsigned>(__grp[__ig]);
-        while (__d != __db) {
-          if (__ng == __gl) {
-            *__me++ = __ts;
-            __ng    = 0;
-            if (++__ig < __grp.size())
-              __gl = __grp[__ig] == numeric_limits<char>::max()
-                       ? numeric_limits<unsigned>::max()
-                       : static_cast<unsigned>(__grp[__ig]);
-          }
-          *__me++ = *--__d;
-          ++__ng;
-        }
-      }
-      // reverse it
-      std::reverse(__t, __me);
-    } break;
-    }
-  }
-  // print rest of sign, if any
-  if (__sn.size() > 1)
-    __me = std::copy(__sn.begin() + 1, __sn.end(), __me);
-  // set alignment
-  if ((__flags & ios_base::adjustfield) == ios_base::left)
-    __mi = __me;
-  else if ((__flags & ios_base::adjustfield) != ios_base::internal)
-    __mi = __mb;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class money_put : public locale::facet, private __money_put<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _OutputIterator iter_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit money_put(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
-    return do_put(__s, __intl, __iob, __fl, __units);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
-    return do_put(__s, __intl, __iob, __fl, __digits);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_put() override {}
-
-  virtual iter_type do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const;
-  virtual iter_type
-  do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const;
-};
-
-template <class _CharT, class _OutputIterator>
-locale::id money_put<_CharT, _OutputIterator>::id;
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
-    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
-  // convert to char
-  const size_t __bs = 100;
-  char __buf[__bs];
-  char* __bb = __buf;
-  char_type __digits[__bs];
-  char_type* __db = __digits;
-  int __n         = snprintf(__bb, __bs, "%.0Lf", __units);
-  unique_ptr<char, void (*)(void*)> __hn(nullptr, free);
-  unique_ptr<char_type, void (*)(void*)> __hd(0, free);
-  // secure memory for digit storage
-  if (static_cast<size_t>(__n) > __bs - 1) {
-    __n = __locale::__asprintf(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units);
-    if (__n == -1)
-      std::__throw_bad_alloc();
-    __hn.reset(__bb);
-    __hd.reset((char_type*)malloc(static_cast<size_t>(__n) * sizeof(char_type)));
-    if (__hd == nullptr)
-      std::__throw_bad_alloc();
-    __db = __hd.get();
-  }
-  // gather info
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  __ct.widen(__bb, __bb + __n, __db);
-  bool __neg = __n > 0 && __bb[0] == '-';
-  money_base::pattern __pat;
-  char_type __dp;
-  char_type __ts;
-  string __grp;
-  string_type __sym;
-  string_type __sn;
-  int __fd;
-  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
-  // secure memory for formatting
-  char_type __mbuf[__bs];
-  char_type* __mb = __mbuf;
-  unique_ptr<char_type, void (*)(void*)> __hw(0, free);
-  size_t __exn = __n > __fd ? (static_cast<size_t>(__n) - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() +
-                                  static_cast<size_t>(__fd) + 1
-                            : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
-  if (__exn > __bs) {
-    __hw.reset((char_type*)malloc(__exn * sizeof(char_type)));
-    __mb = __hw.get();
-    if (__mb == 0)
-      std::__throw_bad_alloc();
-  }
-  // format
-  char_type* __mi;
-  char_type* __me;
-  this->__format(
-      __mb, __mi, __me, __iob.flags(), __db, __db + __n, __ct, __neg, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
-  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
-    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
-  // gather info
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  bool __neg                   = __digits.size() > 0 && __digits[0] == __ct.widen('-');
-  money_base::pattern __pat;
-  char_type __dp;
-  char_type __ts;
-  string __grp;
-  string_type __sym;
-  string_type __sn;
-  int __fd;
-  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
-  // secure memory for formatting
-  char_type __mbuf[100];
-  char_type* __mb = __mbuf;
-  unique_ptr<char_type, void (*)(void*)> __h(0, free);
-  size_t __exn =
-      static_cast<int>(__digits.size()) > __fd
-          ? (__digits.size() - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() + static_cast<size_t>(__fd) +
-                1
-          : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
-  if (__exn > 100) {
-    __h.reset((char_type*)malloc(__exn * sizeof(char_type)));
-    __mb = __h.get();
-    if (__mb == 0)
-      std::__throw_bad_alloc();
-  }
-  // format
-  char_type* __mi;
-  char_type* __me;
-  this->__format(
-      __mb,
-      __mi,
-      __me,
-      __iob.flags(),
-      __digits.data(),
-      __digits.data() + __digits.size(),
-      __ct,
-      __neg,
-      __pat,
-      __dp,
-      __ts,
-      __grp,
-      __sym,
-      __sn,
-      __fd);
-  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<wchar_t>;
-#    endif
-
-// messages
-
-class _LIBCPP_EXPORTED_FROM_ABI messages_base {
-public:
-  typedef intptr_t catalog;
-
-  _LIBCPP_HIDE_FROM_ABI messages_base() {}
-};
-
-template <class _CharT>
-class messages : public locale::facet, public messages_base {
-public:
-  typedef _CharT char_type;
-  typedef basic_string<_CharT> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit messages(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI catalog open(const basic_string<char>& __nm, const locale& __loc) const {
-    return do_open(__nm, __loc);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI string_type get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
-    return do_get(__c, __set, __msgid, __dflt);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI void close(catalog __c) const { do_close(__c); }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages() override {}
-
-  virtual catalog do_open(const basic_string<char>&, const locale&) const;
-  virtual string_type do_get(catalog, int __set, int __msgid, const string_type& __dflt) const;
-  virtual void do_close(catalog) const;
-};
-
-template <class _CharT>
-locale::id messages<_CharT>::id;
-
-template <class _CharT>
-typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
-#    if _LIBCPP_HAS_CATOPEN
-  return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
-#    else  // !_LIBCPP_HAS_CATOPEN
-  (void)__nm;
-  return -1;
-#    endif // _LIBCPP_HAS_CATOPEN
-}
-
-template <class _CharT>
-typename messages<_CharT>::string_type
-messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
-#    if _LIBCPP_HAS_CATOPEN
-  string __ndflt;
-  __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
-      std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
-  nl_catd __cat = (nl_catd)__c;
-  static_assert(sizeof(catalog) >= sizeof(nl_catd), "Unexpected nl_catd type");
-  char* __n = catgets(__cat, __set, __msgid, __ndflt.c_str());
-  string_type __w;
-  __widen_from_utf8<sizeof(char_type) * __CHAR_BIT__>()(std::back_inserter(__w), __n, __n + std::strlen(__n));
-  return __w;
-#    else  // !_LIBCPP_HAS_CATOPEN
-  (void)__c;
-  (void)__set;
-  (void)__msgid;
-  return __dflt;
-#    endif // _LIBCPP_HAS_CATOPEN
-}
-
-template <class _CharT>
-void messages<_CharT>::do_close(catalog __c) const {
-#    if _LIBCPP_HAS_CATOPEN
-  catclose((nl_catd)__c);
-#    else  // !_LIBCPP_HAS_CATOPEN
-  (void)__c;
-#    endif // _LIBCPP_HAS_CATOPEN
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<wchar_t>;
-#    endif
-
-template <class _CharT>
-class messages_byname : public messages<_CharT> {
-public:
-  typedef messages_base::catalog catalog;
-  typedef basic_string<_CharT> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const char*, size_t __refs = 0) : messages<_CharT>(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const string&, size_t __refs = 0) : messages<_CharT>(__refs) {}
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages_byname() override {}
-};
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<wchar_t>;
-#    endif
-
-#    if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
-
-template <class _Codecvt,
-          class _Elem      = wchar_t,
-          class _WideAlloc = allocator<_Elem>,
-          class _ByteAlloc = allocator<char> >
-class _LIBCPP_DEPRECATED_IN_CXX17 wstring_convert {
-public:
-  typedef basic_string<char, char_traits<char>, _ByteAlloc> byte_string;
-  typedef basic_string<_Elem, char_traits<_Elem>, _WideAlloc> wide_string;
-  typedef typename _Codecvt::state_type state_type;
-  typedef typename wide_string::traits_type::int_type int_type;
-
-private:
-  byte_string __byte_err_string_;
-  wide_string __wide_err_string_;
-  _Codecvt* __cvtptr_;
-  state_type __cvtstate_;
-  size_t __cvtcount_;
-
-public:
-#      ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI wstring_convert() : wstring_convert(new _Codecvt) {}
-  _LIBCPP_HIDE_FROM_ABI explicit wstring_convert(_Codecvt* __pcvt);
-#      else
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_EXPLICIT_SINCE_CXX14 wstring_convert(_Codecvt* __pcvt = new _Codecvt);
-#      endif
-
-  _LIBCPP_HIDE_FROM_ABI wstring_convert(_Codecvt* __pcvt, state_type __state);
-  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
-  wstring_convert(const byte_string& __byte_err, const wide_string& __wide_err = wide_string());
-#      ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI wstring_convert(wstring_convert&& __wc);
-#      endif
-  _LIBCPP_HIDE_FROM_ABI ~wstring_convert();
-
-  wstring_convert(const wstring_convert& __wc)            = delete;
-  wstring_convert& operator=(const wstring_convert& __wc) = delete;
-
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(char __byte) { return from_bytes(&__byte, &__byte + 1); }
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __ptr) {
-    return from_bytes(__ptr, __ptr + char_traits<char>::length(__ptr));
-  }
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const byte_string& __str) {
-    return from_bytes(__str.data(), __str.data() + __str.size());
-  }
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __first, const char* __last);
-
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(_Elem __wchar) { return to_bytes(std::addressof(__wchar), std::addressof(__wchar) + 1); }
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __wptr) {
-    return to_bytes(__wptr, __wptr + char_traits<_Elem>::length(__wptr));
-  }
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const wide_string& __wstr) {
-    return to_bytes(__wstr.data(), __wstr.data() + __wstr.size());
-  }
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __first, const _Elem* __last);
-
-  _LIBCPP_HIDE_FROM_ABI size_t converted() const _NOEXCEPT { return __cvtcount_; }
-  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __cvtstate_; }
-};
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt)
-    : __cvtptr_(__pcvt), __cvtstate_(), __cvtcount_(0) {}
-_LIBCPP_SUPPRESS_DEPRECATED_POP
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt, state_type __state)
-    : __cvtptr_(__pcvt), __cvtstate_(__state), __cvtcount_(0) {}
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(
-    const byte_string& __byte_err, const wide_string& __wide_err)
-    : __byte_err_string_(__byte_err), __wide_err_string_(__wide_err), __cvtstate_(), __cvtcount_(0) {
-  __cvtptr_ = new _Codecvt;
-}
-
-#      ifndef _LIBCPP_CXX03_LANG
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(wstring_convert&& __wc)
-    : __byte_err_string_(std::move(__wc.__byte_err_string_)),
-      __wide_err_string_(std::move(__wc.__wide_err_string_)),
-      __cvtptr_(__wc.__cvtptr_),
-      __cvtstate_(__wc.__cvtstate_),
-      __cvtcount_(__wc.__cvtcount_) {
-  __wc.__cvtptr_ = nullptr;
-}
-
-#      endif // _LIBCPP_CXX03_LANG
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::~wstring_convert() {
-  delete __cvtptr_;
-}
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wide_string
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::from_bytes(const char* __frm, const char* __frm_end) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  __cvtcount_ = 0;
-  if (__cvtptr_ != nullptr) {
-    wide_string __ws(2 * (__frm_end - __frm), _Elem());
-    if (__frm != __frm_end)
-      __ws.resize(__ws.capacity());
-    codecvt_base::result __r = codecvt_base::ok;
-    state_type __st          = __cvtstate_;
-    if (__frm != __frm_end) {
-      _Elem* __to     = std::addressof(__ws[0]);
-      _Elem* __to_end = __to + __ws.size();
-      const char* __frm_nxt;
-      do {
-        _Elem* __to_nxt;
-        __r = __cvtptr_->in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
-        __cvtcount_ += __frm_nxt - __frm;
-        if (__frm_nxt == __frm) {
-          __r = codecvt_base::error;
-        } else if (__r == codecvt_base::noconv) {
-          __ws.resize(__to - std::addressof(__ws[0]));
-          // This only gets executed if _Elem is char
-          __ws.append((const _Elem*)__frm, (const _Elem*)__frm_end);
-          __frm = __frm_nxt;
-          __r   = codecvt_base::ok;
-        } else if (__r == codecvt_base::ok) {
-          __ws.resize(__to_nxt - std::addressof(__ws[0]));
-          __frm = __frm_nxt;
-        } else if (__r == codecvt_base::partial) {
-          ptrdiff_t __s = __to_nxt - std::addressof(__ws[0]);
-          __ws.resize(2 * __s);
-          __to     = std::addressof(__ws[0]) + __s;
-          __to_end = std::addressof(__ws[0]) + __ws.size();
-          __frm    = __frm_nxt;
-        }
-      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
-    }
-    if (__r == codecvt_base::ok)
-      return __ws;
-  }
-
-  if (__wide_err_string_.empty())
-    std::__throw_range_error("wstring_convert: from_bytes error");
-
-  return __wide_err_string_;
-}
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::byte_string
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::to_bytes(const _Elem* __frm, const _Elem* __frm_end) {
-  __cvtcount_ = 0;
-  if (__cvtptr_ != nullptr) {
-    byte_string __bs(2 * (__frm_end - __frm), char());
-    if (__frm != __frm_end)
-      __bs.resize(__bs.capacity());
-    codecvt_base::result __r = codecvt_base::ok;
-    state_type __st          = __cvtstate_;
-    if (__frm != __frm_end) {
-      char* __to     = std::addressof(__bs[0]);
-      char* __to_end = __to + __bs.size();
-      const _Elem* __frm_nxt;
-      do {
-        char* __to_nxt;
-        __r = __cvtptr_->out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
-        __cvtcount_ += __frm_nxt - __frm;
-        if (__frm_nxt == __frm) {
-          __r = codecvt_base::error;
-        } else if (__r == codecvt_base::noconv) {
-          __bs.resize(__to - std::addressof(__bs[0]));
-          // This only gets executed if _Elem is char
-          __bs.append((const char*)__frm, (const char*)__frm_end);
-          __frm = __frm_nxt;
-          __r   = codecvt_base::ok;
-        } else if (__r == codecvt_base::ok) {
-          __bs.resize(__to_nxt - std::addressof(__bs[0]));
-          __frm = __frm_nxt;
-        } else if (__r == codecvt_base::partial) {
-          ptrdiff_t __s = __to_nxt - std::addressof(__bs[0]);
-          __bs.resize(2 * __s);
-          __to     = std::addressof(__bs[0]) + __s;
-          __to_end = std::addressof(__bs[0]) + __bs.size();
-          __frm    = __frm_nxt;
-        }
-      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
-    }
-    if (__r == codecvt_base::ok) {
-      size_t __s = __bs.size();
-      __bs.resize(__bs.capacity());
-      char* __to     = std::addressof(__bs[0]) + __s;
-      char* __to_end = __to + __bs.size();
-      do {
-        char* __to_nxt;
-        __r = __cvtptr_->unshift(__st, __to, __to_end, __to_nxt);
-        if (__r == codecvt_base::noconv) {
-          __bs.resize(__to - std::addressof(__bs[0]));
-          __r = codecvt_base::ok;
-        } else if (__r == codecvt_base::ok) {
-          __bs.resize(__to_nxt - std::addressof(__bs[0]));
-        } else if (__r == codecvt_base::partial) {
-          ptrdiff_t __sp = __to_nxt - std::addressof(__bs[0]);
-          __bs.resize(2 * __sp);
-          __to     = std::addressof(__bs[0]) + __sp;
-          __to_end = std::addressof(__bs[0]) + __bs.size();
-        }
-      } while (__r == codecvt_base::partial);
-      if (__r == codecvt_base::ok)
-        return __bs;
-    }
-  }
-
-  if (__byte_err_string_.empty())
-    std::__throw_range_error("wstring_convert: to_bytes error");
-
-  return __byte_err_string_;
-}
-
-template <class _Codecvt, class _Elem = wchar_t, class _Tr = char_traits<_Elem> >
-class _LIBCPP_DEPRECATED_IN_CXX17 wbuffer_convert : public basic_streambuf<_Elem, _Tr> {
-public:
-  // types:
-  typedef _Elem char_type;
-  typedef _Tr traits_type;
-  typedef typename traits_type::int_type int_type;
-  typedef typename traits_type::pos_type pos_type;
-  typedef typename traits_type::off_type off_type;
-  typedef typename _Codecvt::state_type state_type;
-
-private:
-  char* __extbuf_;
-  const char* __extbufnext_;
-  const char* __extbufend_;
-  char __extbuf_min_[8];
-  size_t __ebs_;
-  char_type* __intbuf_;
-  size_t __ibs_;
-  streambuf* __bufptr_;
-  _Codecvt* __cv_;
-  state_type __st_;
-  ios_base::openmode __cm_;
-  bool __owns_eb_;
-  bool __owns_ib_;
-  bool __always_noconv_;
-
-public:
-#      ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI wbuffer_convert() : wbuffer_convert(nullptr) {}
-  explicit _LIBCPP_HIDE_FROM_ABI
-  wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
-#      else
-  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
-  wbuffer_convert(streambuf* __bytebuf = nullptr, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
-#      endif
-
-  _LIBCPP_HIDE_FROM_ABI ~wbuffer_convert();
-
-  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf() const { return __bufptr_; }
-  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf(streambuf* __bytebuf) {
-    streambuf* __r = __bufptr_;
-    __bufptr_      = __bytebuf;
-    return __r;
-  }
-
-  wbuffer_convert(const wbuffer_convert&)            = delete;
-  wbuffer_convert& operator=(const wbuffer_convert&) = delete;
-
-  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __st_; }
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type underflow();
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type pbackfail(int_type __c = traits_type::eof());
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type overflow(int_type __c = traits_type::eof());
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual basic_streambuf<char_type, traits_type>* setbuf(char_type* __s, streamsize __n);
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
-  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out);
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
-  seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out);
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int sync();
-
-private:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool __read_mode();
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __write_mode();
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL wbuffer_convert* __close();
-};
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-wbuffer_convert<_Codecvt, _Elem, _Tr>::wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt, state_type __state)
-    : __extbuf_(nullptr),
-      __extbufnext_(nullptr),
-      __extbufend_(nullptr),
-      __ebs_(0),
-      __intbuf_(0),
-      __ibs_(0),
-      __bufptr_(__bytebuf),
-      __cv_(__pcvt),
-      __st_(__state),
-      __cm_(0),
-      __owns_eb_(false),
-      __owns_ib_(false),
-      __always_noconv_(__cv_ ? __cv_->always_noconv() : false) {
-  setbuf(0, 4096);
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-wbuffer_convert<_Codecvt, _Elem, _Tr>::~wbuffer_convert() {
-  __close();
-  delete __cv_;
-  if (__owns_eb_)
-    delete[] __extbuf_;
-  if (__owns_ib_)
-    delete[] __intbuf_;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow() {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ == 0 || __bufptr_ == nullptr)
-    return traits_type::eof();
-  bool __initial = __read_mode();
-  char_type __1buf;
-  if (this->gptr() == 0)
-    this->setg(std::addressof(__1buf), std::addressof(__1buf) + 1, std::addressof(__1buf) + 1);
-  const size_t __unget_sz = __initial ? 0 : std::min<size_t>((this->egptr() - this->eback()) / 2, 4);
-  int_type __c            = traits_type::eof();
-  if (this->gptr() == this->egptr()) {
-    std::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
-    if (__always_noconv_) {
-      streamsize __nmemb = static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz);
-      __nmemb            = __bufptr_->sgetn((char*)this->eback() + __unget_sz, __nmemb);
-      if (__nmemb != 0) {
-        this->setg(this->eback(), this->eback() + __unget_sz, this->eback() + __unget_sz + __nmemb);
-        __c = *this->gptr();
-      }
-    } else {
-      if (__extbufend_ != __extbufnext_) {
-        _LIBCPP_ASSERT_NON_NULL(__extbufnext_ != nullptr, "underflow moving from nullptr");
-        _LIBCPP_ASSERT_NON_NULL(__extbuf_ != nullptr, "underflow moving into nullptr");
-        std::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
-      }
-      __extbufnext_      = __extbuf_ + (__extbufend_ - __extbufnext_);
-      __extbufend_       = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
-      streamsize __nmemb = std::min(static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz),
-                                    static_cast<streamsize>(__extbufend_ - __extbufnext_));
-      codecvt_base::result __r;
-      // FIXME: Do we ever need to restore the state here?
-      // state_type __svs = __st_;
-      streamsize __nr = __bufptr_->sgetn(const_cast<char*>(__extbufnext_), __nmemb);
-      if (__nr != 0) {
-        __extbufend_ = __extbufnext_ + __nr;
-        char_type* __inext;
-        __r = __cv_->in(
-            __st_, __extbuf_, __extbufend_, __extbufnext_, this->eback() + __unget_sz, this->egptr(), __inext);
-        if (__r == codecvt_base::noconv) {
-          this->setg((char_type*)__extbuf_, (char_type*)__extbuf_, (char_type*)const_cast<char*>(__extbufend_));
-          __c = *this->gptr();
-        } else if (__inext != this->eback() + __unget_sz) {
-          this->setg(this->eback(), this->eback() + __unget_sz, __inext);
-          __c = *this->gptr();
-        }
-      }
-    }
-  } else
-    __c = *this->gptr();
-  if (this->eback() == std::addressof(__1buf))
-    this->setg(0, 0, 0);
-  return __c;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type
-wbuffer_convert<_Codecvt, _Elem, _Tr>::pbackfail(int_type __c) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ != 0 && __bufptr_ && this->eback() < this->gptr()) {
-    if (traits_type::eq_int_type(__c, traits_type::eof())) {
-      this->gbump(-1);
-      return traits_type::not_eof(__c);
-    }
-    if (traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
-      this->gbump(-1);
-      *this->gptr() = traits_type::to_char_type(__c);
-      return __c;
-    }
-  }
-  return traits_type::eof();
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::overflow(int_type __c) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ == 0 || !__bufptr_)
-    return traits_type::eof();
-  __write_mode();
-  char_type __1buf;
-  char_type* __pb_save  = this->pbase();
-  char_type* __epb_save = this->epptr();
-  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
-    if (this->pptr() == 0)
-      this->setp(std::addressof(__1buf), std::addressof(__1buf) + 1);
-    *this->pptr() = traits_type::to_char_type(__c);
-    this->pbump(1);
-  }
-  if (this->pptr() != this->pbase()) {
-    if (__always_noconv_) {
-      streamsize __nmemb = static_cast<streamsize>(this->pptr() - this->pbase());
-      if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
-        return traits_type::eof();
-    } else {
-      char* __extbe = __extbuf_;
-      codecvt_base::result __r;
-      do {
-        const char_type* __e;
-        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
-        if (__e == this->pbase())
-          return traits_type::eof();
-        if (__r == codecvt_base::noconv) {
-          streamsize __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
-          if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
-            return traits_type::eof();
-        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
-          streamsize __nmemb = static_cast<size_t>(__extbe - __extbuf_);
-          if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
-            return traits_type::eof();
-          if (__r == codecvt_base::partial) {
-            this->setp(const_cast<char_type*>(__e), this->pptr());
-            this->__pbump(this->epptr() - this->pbase());
-          }
-        } else
-          return traits_type::eof();
-      } while (__r == codecvt_base::partial);
-    }
-    this->setp(__pb_save, __epb_save);
-  }
-  return traits_type::not_eof(__c);
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-basic_streambuf<_Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::setbuf(char_type* __s, streamsize __n) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  this->setg(0, 0, 0);
-  this->setp(0, 0);
-  if (__owns_eb_)
-    delete[] __extbuf_;
-  if (__owns_ib_)
-    delete[] __intbuf_;
-  __ebs_ = __n;
-  if (__ebs_ > sizeof(__extbuf_min_)) {
-    if (__always_noconv_ && __s) {
-      __extbuf_  = (char*)__s;
-      __owns_eb_ = false;
-    } else {
-      __extbuf_  = new char[__ebs_];
-      __owns_eb_ = true;
-    }
-  } else {
-    __extbuf_  = __extbuf_min_;
-    __ebs_     = sizeof(__extbuf_min_);
-    __owns_eb_ = false;
-  }
-  if (!__always_noconv_) {
-    __ibs_ = max<streamsize>(__n, sizeof(__extbuf_min_));
-    if (__s && __ibs_ >= sizeof(__extbuf_min_)) {
-      __intbuf_  = __s;
-      __owns_ib_ = false;
-    } else {
-      __intbuf_  = new char_type[__ibs_];
-      __owns_ib_ = true;
-    }
-  } else {
-    __ibs_     = 0;
-    __intbuf_  = 0;
-    __owns_ib_ = false;
-  }
-  return this;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
-wbuffer_convert<_Codecvt, _Elem, _Tr>::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __om) {
-  int __width = __cv_->encoding();
-  if (__cv_ == 0 || !__bufptr_ || (__width <= 0 && __off != 0) || sync())
-    return pos_type(off_type(-1));
-  // __width > 0 || __off == 0, now check __way
-  if (__way != ios_base::beg && __way != ios_base::cur && __way != ios_base::end)
-    return pos_type(off_type(-1));
-  pos_type __r = __bufptr_->pubseekoff(__width * __off, __way, __om);
-  __r.state(__st_);
-  return __r;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
-wbuffer_convert<_Codecvt, _Elem, _Tr>::seekpos(pos_type __sp, ios_base::openmode __wch) {
-  if (__cv_ == 0 || !__bufptr_ || sync())
-    return pos_type(off_type(-1));
-  if (__bufptr_->pubseekpos(__sp, __wch) == pos_type(off_type(-1)))
-    return pos_type(off_type(-1));
-  return __sp;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-int wbuffer_convert<_Codecvt, _Elem, _Tr>::sync() {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ == 0 || !__bufptr_)
-    return 0;
-  if (__cm_ & ios_base::out) {
-    if (this->pptr() != this->pbase())
-      if (overflow() == traits_type::eof())
-        return -1;
-    codecvt_base::result __r;
-    do {
-      char* __extbe;
-      __r                = __cv_->unshift(__st_, __extbuf_, __extbuf_ + __ebs_, __extbe);
-      streamsize __nmemb = static_cast<streamsize>(__extbe - __extbuf_);
-      if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
-        return -1;
-    } while (__r == codecvt_base::partial);
-    if (__r == codecvt_base::error)
-      return -1;
-    if (__bufptr_->pubsync())
-      return -1;
-  } else if (__cm_ & ios_base::in) {
-    off_type __c;
-    if (__always_noconv_)
-      __c = this->egptr() - this->gptr();
-    else {
-      int __width = __cv_->encoding();
-      __c         = __extbufend_ - __extbufnext_;
-      if (__width > 0)
-        __c += __width * (this->egptr() - this->gptr());
-      else {
-        if (this->gptr() != this->egptr()) {
-          std::reverse(this->gptr(), this->egptr());
-          codecvt_base::result __r;
-          const char_type* __e = this->gptr();
-          char* __extbe;
-          do {
-            __r = __cv_->out(__st_, __e, this->egptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
-            switch (__r) {
-            case codecvt_base::noconv:
-              __c += this->egptr() - this->gptr();
-              break;
-            case codecvt_base::ok:
-            case codecvt_base::partial:
-              __c += __extbe - __extbuf_;
-              break;
-            default:
-              return -1;
-            }
-          } while (__r == codecvt_base::partial);
-        }
-      }
-    }
-    if (__bufptr_->pubseekoff(-__c, ios_base::cur, __cm_) == pos_type(off_type(-1)))
-      return -1;
-    this->setg(0, 0, 0);
-    __cm_ = 0;
-  }
-  return 0;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-bool wbuffer_convert<_Codecvt, _Elem, _Tr>::__read_mode() {
-  if (!(__cm_ & ios_base::in)) {
-    this->setp(0, 0);
-    if (__always_noconv_)
-      this->setg((char_type*)__extbuf_, (char_type*)__extbuf_ + __ebs_, (char_type*)__extbuf_ + __ebs_);
-    else
-      this->setg(__intbuf_, __intbuf_ + __ibs_, __intbuf_ + __ibs_);
-    __cm_ = ios_base::in;
-    return true;
-  }
-  return false;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-void wbuffer_convert<_Codecvt, _Elem, _Tr>::__write_mode() {
-  if (!(__cm_ & ios_base::out)) {
-    this->setg(0, 0, 0);
-    if (__ebs_ > sizeof(__extbuf_min_)) {
-      if (__always_noconv_)
-        this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__ebs_ - 1));
-      else
-        this->setp(__intbuf_, __intbuf_ + (__ibs_ - 1));
-    } else
-      this->setp(0, 0);
-    __cm_ = ios_base::out;
-  }
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-wbuffer_convert<_Codecvt, _Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::__close() {
-  wbuffer_convert* __rt = nullptr;
-  if (__cv_ != nullptr && __bufptr_ != nullptr) {
-    __rt = this;
-    if ((__cm_ & ios_base::out) && sync())
-      __rt = nullptr;
-  }
-  return __rt;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_POP
-
-#    endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
-
-_LIBCPP_END_NAMESPACE_STD
-
-_LIBCPP_POP_MACROS
-
-// NOLINTEND(libcpp-robust-against-adl)
-
 #  endif // _LIBCPP_HAS_LOCALIZATION
 
 #  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
diff --git a/libcxx/include/map b/libcxx/include/map
index 6c271392911dc..2070c91cfc46d 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -972,31 +972,15 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
 
-  _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) {
-#  ifndef _LIBCPP_CXX03_LANG
-    __tree_ = __m.__tree_;
-#  else
-    if (this != std::addressof(__m)) {
-      __tree_.clear();
-      __tree_.value_comp() = __m.__tree_.value_comp();
-      __tree_.__copy_assign_alloc(__m.__tree_);
-      insert(__m.begin(), __m.end());
-    }
-#  endif
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__m.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) {
-    __tree_ = std::move(__m.__tree_);
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : __tree_(__vc(__comp)) {
@@ -1659,31 +1643,16 @@ public:
     insert(__m.begin(), __m.end());
   }
 
-  _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) {
-#  ifndef _LIBCPP_CXX03_LANG
-    __tree_ = __m.__tree_;
-#  else
-    if (this != std::addressof(__m)) {
-      __tree_.clear();
-      __tree_.value_comp() = __m.__tree_.value_comp();
-      __tree_.__copy_assign_alloc(__m.__tree_);
-      insert(__m.begin(), __m.end());
-    }
-#  endif
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__m.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) {
-    __tree_ = std::move(__m.__tree_);
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multimap&
+  operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : __tree_(__vc(__comp)) {
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index adf80f2ac9acb..45b9c72a05b82 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -65,9 +65,8 @@ module std_core [system] {
 
   module type_traits {
     module add_cv_quals                               { header "__type_traits/add_cv_quals.h" }
-    module add_lvalue_reference                       { header "__type_traits/add_lvalue_reference.h" }
     module add_pointer                                { header "__type_traits/add_pointer.h" }
-    module add_rvalue_reference                       { header "__type_traits/add_rvalue_reference.h" }
+    module add_reference                              { header "__type_traits/add_reference.h" }
     module aligned_storage                            { header "__type_traits/aligned_storage.h" }
     module aligned_union                              { header "__type_traits/aligned_union.h" }
     module alignment_of                               { header "__type_traits/alignment_of.h" }
@@ -869,6 +868,7 @@ module std [system] {
 
     header "array"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module atomic {
@@ -934,7 +934,10 @@ module std [system] {
     module chars_format               { header "__charconv/chars_format.h" }
     module from_chars_floating_point  { header "__charconv/from_chars_floating_point.h" }
     module from_chars_integral        { header "__charconv/from_chars_integral.h" }
-    module from_chars_result          { header "__charconv/from_chars_result.h" }
+    module from_chars_result          {
+      header "__charconv/from_chars_result.h"
+      export std.system_error.errc
+    }
     module tables                     { header "__charconv/tables.h" }
     module to_chars                   { header "__charconv/to_chars.h" }
     module to_chars_base_10           { header "__charconv/to_chars_base_10.h" }
@@ -1223,6 +1226,7 @@ module std [system] {
 
     header "deque"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module exception {
@@ -1299,6 +1303,7 @@ module std [system] {
     header "flat_map"
     export *
     export std.algorithm.ranges_sort
+    export std.iterator.reverse_iterator
     export std.ranges.zip_view
     export std.tuple
   }
@@ -1317,6 +1322,10 @@ module std [system] {
     export std.flat_map.sorted_unique
     export std.flat_map.sorted_equivalent
     export *
+    export std.algorithm.ranges_sort
+    export std.iterator.reverse_iterator
+    export std.ranges.zip_view
+    export std.tuple
   }
 
   module format {
@@ -1372,6 +1381,7 @@ module std [system] {
   module forward_list {
     header "forward_list"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module fstream {
@@ -1522,6 +1532,7 @@ module std [system] {
     }
     module permutable                 { header "__iterator/permutable.h" }
     module prev                       { header "__iterator/prev.h" }
+    module product_iterator           { header "__iterator/product_iterator.h" }
     module projected                  { header "__iterator/projected.h" }
     module ranges_iterator_traits     { header "__iterator/ranges_iterator_traits.h" }
     module readable_traits            { header "__iterator/readable_traits.h" }
@@ -1546,11 +1557,21 @@ module std [system] {
   module list {
     header "list"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module locale {
     header "locale"
-    module pad_and_output { header "__locale_dir/pad_and_output.h" }
+    module check_grouping  { header "__locale_dir/check_grouping.h" }
+    module get_c_locale    { header "__locale_dir/get_c_locale.h" }
+    module messages        { header "__locale_dir/messages.h" }
+    module money           { header "__locale_dir/money.h" }
+    module num             { header "__locale_dir/num.h" }
+    module pad_and_output  { header "__locale_dir/pad_and_output.h" }
+    module scan_keyword    { header "__locale_dir/scan_keyword.h" }
+    module time            { header "__locale_dir/time.h" }
+    module wbuffer_convert { header "__locale_dir/wbuffer_convert.h" }
+    module wstring_convert { header "__locale_dir/wstring_convert.h" }
 
     module support {
       header "__locale_dir/locale_base_api.h"
@@ -1588,6 +1609,7 @@ module std [system] {
     module fwd { header "__fwd/map.h" }
     header "map"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module mdspan {
@@ -1934,6 +1956,7 @@ module std [system] {
   module regex {
     header "regex"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module scoped_allocator {
@@ -1950,6 +1973,7 @@ module std [system] {
     module fwd { header "__fwd/set.h" }
     header "set"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module shared_mutex {
@@ -1967,6 +1991,7 @@ module std [system] {
 
     header "span"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module sstream {
@@ -2019,6 +2044,7 @@ module std [system] {
 
     header "string"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module string_view {
@@ -2026,6 +2052,7 @@ module std [system] {
 
     header "string_view"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module strstream {
@@ -2107,11 +2134,13 @@ module std [system] {
   module unordered_map {
     header "unordered_map"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module unordered_set {
     header "unordered_set"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module utility {
@@ -2200,6 +2229,7 @@ module std [system] {
     }
 
     header "vector"
+    export std.iterator.reverse_iterator
     export *
   }
 
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index f616bad3ac171..dc8e711f04878 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -434,7 +434,7 @@ public:
 };
 
 template <class _Mutex>
-class _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) scoped_lock<_Mutex> {
+class _LIBCPP_SCOPED_LOCKABLE scoped_lock<_Mutex> {
 public:
   typedef _Mutex mutex_type;
 
@@ -442,16 +442,15 @@ private:
   mutex_type& __m_;
 
 public:
-  [[nodiscard]]
-  _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(mutex_type& __m) _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability(__m))
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(mutex_type& __m) _LIBCPP_ACQUIRE_CAPABILITY(__m)
       : __m_(__m) {
     __m_.lock();
   }
 
-  ~scoped_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()) { __m_.unlock(); }
+  _LIBCPP_RELEASE_CAPABILITY _LIBCPP_HIDE_FROM_ABI ~scoped_lock() { __m_.unlock(); }
 
-  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(adopt_lock_t, mutex_type& __m)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(requires_capability(__m))
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(adopt_lock_t, mutex_type& __m) _LIBCPP_REQUIRES_CAPABILITY(__m)
       : __m_(__m) {}
 
   scoped_lock(scoped_lock const&)            = delete;
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index 4a627a07a4a2f..8ec17b68503f8 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -205,6 +205,11 @@ void vprint_nonunicode(ostream& os, string_view fmt, format_args args);
 #    include <stdexcept>
 #    include <type_traits>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_OSTREAM
diff --git a/libcxx/include/set b/libcxx/include/set
index aeea98adf582b..480aa0ca0b0c1 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -662,14 +662,10 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
 
-  _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) {
-    __tree_ = __s.__tree_;
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__s.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 #  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
@@ -1129,14 +1125,10 @@ public:
     insert(__s.begin(), __s.end());
   }
 
-  _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) {
-    __tree_ = __s.__tree_;
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__s.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
 #  endif // _LIBCPP_CXX03_LANG
diff --git a/libcxx/include/shared_mutex b/libcxx/include/shared_mutex
index 6469c02ca5874..8c02e348e4de7 100644
--- a/libcxx/include/shared_mutex
+++ b/libcxx/include/shared_mutex
@@ -183,7 +183,7 @@ struct _LIBCPP_EXPORTED_FROM_ABI __shared_mutex_base {
 };
 
 #      if _LIBCPP_STD_VER >= 17
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_mutex")) shared_mutex {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("shared_mutex") shared_mutex {
   __shared_mutex_base __base_;
 
 public:
@@ -194,35 +194,23 @@ public:
   shared_mutex& operator=(const shared_mutex&) = delete;
 
   // Exclusive ownership
-  _LIBCPP_HIDE_FROM_ABI void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_capability__()) {
-    return __base_.lock();
-  }
-  _LIBCPP_HIDE_FROM_ABI bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
-    return __base_.try_lock();
-  }
-  _LIBCPP_HIDE_FROM_ABI void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_capability__()) {
-    return __base_.unlock();
-  }
+  _LIBCPP_ACQUIRE_CAPABILITY() _LIBCPP_HIDE_FROM_ABI void lock() { return __base_.lock(); }
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool try_lock() { return __base_.try_lock(); }
+  _LIBCPP_RELEASE_CAPABILITY _LIBCPP_HIDE_FROM_ABI void unlock() { return __base_.unlock(); }
 
   // Shared ownership
-  _LIBCPP_HIDE_FROM_ABI void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_shared_capability__()) {
-    return __base_.lock_shared();
-  }
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared()
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+  _LIBCPP_ACQUIRE_SHARED_CAPABILITY _LIBCPP_HIDE_FROM_ABI void lock_shared() { return __base_.lock_shared(); }
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool try_lock_shared() {
     return __base_.try_lock_shared();
   }
-  _LIBCPP_HIDE_FROM_ABI void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_shared_capability__()) {
-    return __base_.unlock_shared();
-  }
+  _LIBCPP_RELEASE_SHARED_CAPABILITY _LIBCPP_HIDE_FROM_ABI void unlock_shared() { return __base_.unlock_shared(); }
 
   //     typedef __shared_mutex_base::native_handle_type native_handle_type;
   //     _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return __base::unlock_shared(); }
 };
 #      endif
 
-class _LIBCPP_EXPORTED_FROM_ABI
-_LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_timed_mutex")) shared_timed_mutex {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("shared_timed_mutex") shared_timed_mutex {
   __shared_mutex_base __base_;
 
 public:
@@ -233,17 +221,17 @@ public:
   shared_timed_mutex& operator=(const shared_timed_mutex&) = delete;
 
   // Exclusive ownership
-  void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_capability__());
-  bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true));
+  void lock() _LIBCPP_ACQUIRE_CAPABILITY();
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock();
   template <class _Rep, class _Period>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __rel_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_for(const chrono::duration<_Rep, _Period>& __rel_time) {
     return try_lock_until(chrono::steady_clock::now() + __rel_time);
   }
 
   template <class _Clock, class _Duration>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time) {
     unique_lock<mutex> __lk(__base_.__mut_);
     if (__base_.__state_ & __base_.__write_entered_) {
       while (true) {
@@ -270,20 +258,20 @@ public:
     return true;
   }
 
-  void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_capability__());
+  _LIBCPP_RELEASE_CAPABILITY void unlock();
 
   // Shared ownership
-  void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_shared_capability__());
-  bool try_lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true));
+  _LIBCPP_ACQUIRE_SHARED_CAPABILITY void lock_shared();
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) bool try_lock_shared();
   template <class _Rep, class _Period>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared_for(const chrono::duration<_Rep, _Period>& __rel_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_shared_for(const chrono::duration<_Rep, _Period>& __rel_time) {
     return try_lock_shared_until(chrono::steady_clock::now() + __rel_time);
   }
 
   template <class _Clock, class _Duration>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared_until(const chrono::time_point<_Clock, _Duration>& __abs_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_shared_until(const chrono::time_point<_Clock, _Duration>& __abs_time) {
     unique_lock<mutex> __lk(__base_.__mut_);
     if ((__base_.__state_ & __base_.__write_entered_) ||
         (__base_.__state_ & __base_.__n_readers_) == __base_.__n_readers_) {
@@ -302,7 +290,7 @@ public:
     return true;
   }
 
-  void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_shared_capability__());
+  _LIBCPP_RELEASE_SHARED_CAPABILITY void unlock_shared();
 };
 
 template <class _Mutex>
diff --git a/libcxx/include/sstream b/libcxx/include/sstream
index 8462f75853408..682a28fd4dbb8 100644
--- a/libcxx/include/sstream
+++ b/libcxx/include/sstream
@@ -325,7 +325,7 @@ typedef basic_stringstream<wchar_t> wstringstream;
 #    include <__utility/swap.h>
 #    include <ios>
 #    include <istream>
-#    include <locale>
+#    include <streambuf>
 #    include <string>
 #    include <string_view>
 #    include <version>
diff --git a/libcxx/include/string b/libcxx/include/string
index c450ddcb41914..514dd91c7c172 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -1126,7 +1126,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
   basic_string(const _Tp& __t, size_type __pos, size_type __n, const allocator_type& __a = allocator_type())
@@ -1138,7 +1138,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1147,7 +1147,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t, const allocator_type& __a)
       : __alloc_(__a) {
@@ -1206,7 +1206,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1343,7 +1343,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string >::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string >::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1372,7 +1372,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1383,7 +1383,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   append(const _Tp& __t, size_type __pos, size_type __n = npos) {
@@ -1510,7 +1510,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   assign(const _Tp& __t, size_type __pos, size_type __n = npos) {
@@ -1580,7 +1580,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   insert(size_type __pos1, const _Tp& __t, size_type __pos2, size_type __n = npos) {
@@ -1661,7 +1661,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   replace(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) {
@@ -2006,7 +2006,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
   compare(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) const {
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index 9db7b2afb0cf3..a6e0c1867566b 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -463,9 +463,8 @@ namespace std
 #else
 #  include <__config>
 #  include <__type_traits/add_cv_quals.h>
-#  include <__type_traits/add_lvalue_reference.h>
 #  include <__type_traits/add_pointer.h>
-#  include <__type_traits/add_rvalue_reference.h>
+#  include <__type_traits/add_reference.h>
 #  include <__type_traits/aligned_storage.h>
 #  include <__type_traits/aligned_union.h>
 #  include <__type_traits/alignment_of.h>
diff --git a/libcxx/include/version b/libcxx/include/version
index 91fe48351e161..d98049bd57046 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -59,6 +59,8 @@ __cpp_lib_char8_t                                       201907L <atomic> <filesy
 __cpp_lib_chrono                                        201611L <chrono>
 __cpp_lib_chrono_udls                                   201304L <chrono>
 __cpp_lib_clamp                                         201603L <algorithm>
+__cpp_lib_common_reference                              202302L <type_traits>
+__cpp_lib_common_reference_wrapper                      202302L <functional>
 __cpp_lib_complex_udls                                  201309L <complex>
 __cpp_lib_concepts                                      202002L <concepts>
 __cpp_lib_constexpr_algorithms                          202306L <algorithm> <utility>
@@ -402,6 +404,8 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # if _LIBCPP_HAS_CHAR8_T
 #   define __cpp_lib_char8_t                            201907L
 # endif
+# define __cpp_lib_common_reference                     202302L
+# define __cpp_lib_common_reference_wrapper             202302L
 # define __cpp_lib_concepts                             202002L
 # define __cpp_lib_constexpr_algorithms                 201806L
 # define __cpp_lib_constexpr_complex                    201711L
diff --git a/libcxx/modules/std/functional.inc b/libcxx/modules/std/functional.inc
index ddc7d023ee6dc..9ef8f584611fc 100644
--- a/libcxx/modules/std/functional.inc
+++ b/libcxx/modules/std/functional.inc
@@ -14,6 +14,10 @@ export namespace std {
   using std::invoke_r;
 #endif
 
+#if _LIBCPP_STD_VER >= 20
+  using std::basic_common_reference;
+#endif
+
   // [refwrap], reference_wrapper
   using std::reference_wrapper;
 
diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp
index c1af8d6f95aae..903084da053a1 100644
--- a/libcxx/src/atomic.cpp
+++ b/libcxx/src/atomic.cpp
@@ -151,7 +151,10 @@ __libcpp_contention_monitor_for_wait(__cxx_atomic_contention_t volatile* /*__con
 static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __contention_state,
                                      __cxx_atomic_contention_t const volatile* __platform_state,
                                      __cxx_contention_t __old_value) {
-  __cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_seq_cst);
+  __cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_relaxed);
+  // https://github.com/llvm/llvm-project/issues/109290
+  // There are no platform guarantees of a memory barrier in the platform wait implementation
+  __cxx_atomic_thread_fence(memory_order_seq_cst);
   // We sleep as long as the monitored value hasn't changed.
   __libcpp_platform_wait_on_address(__platform_state, __old_value);
   __cxx_atomic_fetch_sub(__contention_state, __cxx_contention_t(1), memory_order_release);
@@ -163,7 +166,7 @@ static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __conte
 static void __libcpp_atomic_notify(void const volatile* __location) {
   auto const __entry = __libcpp_contention_state(__location);
   // The value sequence laundering happens on the next line below.
-  __cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_release);
+  __cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_seq_cst);
   __libcpp_contention_notify(
       &__entry->__contention_state,
       &__entry->__platform_state,
diff --git a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
index fb4455c4aa9da..0ff7f15164d8a 100644
--- a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
+++ b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
@@ -13,12 +13,14 @@
 #include <iterator>
 #include <random>
 #include <string>
+#include <ranges>
 #include <type_traits>
 #include <utility>
 #include <vector>
 
 #include "benchmark/benchmark.h"
 #include "../../GenerateInput.h"
+#include "test_macros.h"
 
 namespace support {
 
@@ -57,7 +59,7 @@ void associative_container_benchmarks(std::string container) {
   auto get_key = [](Value const& v) { return adapt_operations<Container>::key_from_value(v); };
 
   auto bench = [&](std::string operation, auto f) {
-    benchmark::RegisterBenchmark(container + "::" + operation, f)->Arg(32)->Arg(1024)->Arg(8192);
+    benchmark::RegisterBenchmark(container + "::" + operation, f)->Arg(0)->Arg(32)->Arg(1024)->Arg(8192);
   };
 
   static constexpr bool is_multi_key_container =
@@ -66,6 +68,8 @@ void associative_container_benchmarks(std::string container) {
 
   static constexpr bool is_ordered_container = requires(Container c, Key k) { c.lower_bound(k); };
 
+  static constexpr bool is_map_like = requires { typename Container::mapped_type; };
+
   // These benchmarks are structured to perform the operation being benchmarked
   // a small number of times at each iteration, in order to offset the cost of
   // PauseTiming() and ResumeTiming().
@@ -172,7 +176,7 @@ void associative_container_benchmarks(std::string container) {
   // Insertion
   /////////////////////////
   bench("insert(value) (already present)", [=](auto& st) {
-    const std::size_t size = st.range(0);
+    const std::size_t size = st.range(0) ? st.range(0) : 1;
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Value to_insert        = in[in.size() / 2]; // pick any existing value
     std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
@@ -321,11 +325,53 @@ void associative_container_benchmarks(std::string container) {
     }
   });
 
+  if constexpr (is_map_like) {
+    bench("insert(iterator, iterator) (product_iterator from same type)", [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Value> in  = make_value_types(generate_unique_keys(size + (size / 10)));
+      Container source(in.begin(), in.end());
+
+      Container c;
+
+      for ([[maybe_unused]] auto _ : st) {
+        c.insert(source.begin(), source.end());
+        benchmark::DoNotOptimize(c);
+        benchmark::ClobberMemory();
+
+        st.PauseTiming();
+        c = Container();
+        st.ResumeTiming();
+      }
+    });
+
+#if TEST_STD_VER >= 23
+    bench("insert(iterator, iterator) (product_iterator from zip_view)", [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Key> keys  = generate_unique_keys(size + (size / 10));
+      std::sort(keys.begin(), keys.end());
+      std::vector<typename Container::mapped_type> mapped(keys.size());
+
+      auto source = std::views::zip(keys, mapped);
+
+      Container c;
+
+      for ([[maybe_unused]] auto _ : st) {
+        c.insert(source.begin(), source.end());
+        benchmark::DoNotOptimize(c);
+        benchmark::ClobberMemory();
+
+        st.PauseTiming();
+        c = Container();
+        st.ResumeTiming();
+      }
+    });
+#endif
+  }
   /////////////////////////
   // Erasure
   /////////////////////////
   bench("erase(key) (existent)", [=](auto& st) {
-    const std::size_t size = st.range(0);
+    const std::size_t size = st.range(0) ? st.range(0) : 1; // avoid empty container
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Value element          = in[in.size() / 2]; // pick any element
     std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
@@ -369,7 +415,7 @@ void associative_container_benchmarks(std::string container) {
   });
 
   bench("erase(iterator)", [=](auto& st) {
-    const std::size_t size = st.range(0);
+    const std::size_t size = st.range(0) ? st.range(0) : 1; // avoid empty container
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Value element          = in[in.size() / 2]; // pick any element
 
@@ -448,7 +494,7 @@ void associative_container_benchmarks(std::string container) {
       Container c(in.begin(), in.end());
 
       while (st.KeepRunningBatch(BatchSize)) {
-        for (std::size_t i = 0; i != BatchSize; ++i) {
+        for (std::size_t i = 0; i != keys.size(); ++i) { // possible empty keys when Arg(0)
           auto result = func(c, keys[i]);
           benchmark::DoNotOptimize(c);
           benchmark::DoNotOptimize(result);
diff --git a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
index 82902d50f31e6..f3b86554802ca 100644
--- a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: std-at-least-c++26
+// REQUIRES: std-at-least-c++23
 
 #include <flat_map>
 #include <utility>
diff --git a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
index f752f79b3b454..80eaa549042c6 100644
--- a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: std-at-least-c++26
+// REQUIRES: std-at-least-c++23
 
 #include <flat_map>
 
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
index 56420bdaadfbf..89443a597e85a 100644
--- a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
@@ -8,6 +8,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
+#include <string>
 #include <unordered_set>
 #include <utility>
 
@@ -27,6 +28,7 @@ struct support::adapt_operations<std::unordered_set<K>> {
 
 int main(int argc, char** argv) {
   support::associative_container_benchmarks<std::unordered_set<int>>("std::unordered_set<int>");
+  support::associative_container_benchmarks<std::unordered_set<std::string>>("std::unordered_set<std::string>");
 
   benchmark::Initialize(&argc, argv);
   benchmark::RunSpecifiedBenchmarks();
diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in
index d7d588669032d..20b7c1e9bc357 100644
--- a/libcxx/test/configs/cmake-bridge.cfg.in
+++ b/libcxx/test/configs/cmake-bridge.cfg.in
@@ -12,6 +12,8 @@
 #
 
 import os, site
+import shlex
+import sys
 site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
 import libcxx.test.format
 
@@ -32,3 +34,4 @@ config.substitutions.append(('%{lib-dir}', '@LIBCXX_TESTING_INSTALL_PREFIX@/@LIB
 config.substitutions.append(('%{module-dir}', '@LIBCXX_TESTING_INSTALL_PREFIX@/@LIBCXX_INSTALL_MODULES_DIR@'))
 config.substitutions.append(('%{test-tools-dir}', '@LIBCXX_TEST_TOOLS_PATH@'))
 config.substitutions.append(('%{benchmark_flags}', '-I @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/include -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib64 -l benchmark'))
+config.substitutions.append(("%{python}", shlex.quote(sys.executable)))
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp
index e71f35129ddac..22e0662cf2c63 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp
@@ -21,7 +21,7 @@
 #include "../flat_helpers.h"
 #include "test_macros.h"
 
-bool test() {
+constexpr bool test() {
   using M = std::flat_set<TrackCopyMove>;
   {
     M m;
@@ -43,6 +43,9 @@ bool test() {
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp
index 35b9737e85106..c9dcd71127e2b 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp
@@ -24,23 +24,33 @@
 #include <vector>
 
 #include "../flat_helpers.h"
+#include "test_iterators.h"
 #include "test_macros.h"
 
-void test() {
+constexpr bool test() {
   NotQuiteSequenceContainer<int> v;
   std::flat_set s(v);
-  std::istringstream ints("0 1 1 0");
-  auto r = std::ranges::subrange(std::istream_iterator<int>(ints), std::istream_iterator<int>()) |
-           std::views::transform([](int i) { return i * i; });
+
+  int ar[]   = {0, 1, 1, 0};
+  using Iter = cpp20_input_iterator<const int*>;
+  using Sent = sentinel_wrapper<Iter>;
+  using R    = std::ranges::subrange<Iter, Sent>;
+  auto r     = R(Iter(ar), Sent(Iter(ar + 4)));
+
   static_assert(
       ![](auto& t) { return requires { t.insert_range(t.end(), r); }; }(v),
       "This test is to test the case where the underlying container does not provide insert_range");
   s.insert_range(r);
   assert(std::ranges::equal(s, std::vector<int>{0, 1}));
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp b/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
index c04250987e8e2..f17c1483c4a99 100644
--- a/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
+++ b/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
@@ -13,7 +13,7 @@
 
 // The fix for issue 57964 requires an updated dylib due to explicit
 // instantiations. That means Apple backdeployment targets remain broken.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <ios>
 
diff --git a/libcxx/test/libcxx/iterators/product_iterator.pass.cpp b/libcxx/test/libcxx/iterators/product_iterator.pass.cpp
new file mode 100644
index 0000000000000..604f3667c36e3
--- /dev/null
+++ b/libcxx/test/libcxx/iterators/product_iterator.pass.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// gcc 15 does not seem to recognize the __product_iterator_traits specializations
+// UNSUPPORTED: gcc
+
+#include <flat_map>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_iterators.h"
+
+constexpr bool test() {
+  {
+    // Test that the __get_iterator_element can handle a non-copyable iterator
+    int Date[] = {1, 2, 3, 4};
+    cpp20_input_iterator<int*> iter(Date);
+    sentinel_wrapper<cpp20_input_iterator<int*>> sent{cpp20_input_iterator<int*>(Date + 4)};
+    std::ranges::subrange r1(std::move(iter), std::move(sent));
+    auto v  = std::views::zip(std::move(r1), std::views::iota(0, 4));
+    auto it = v.begin();
+
+    using Iter = decltype(it);
+
+    static_assert(!std::is_copy_constructible_v<Iter>);
+
+    static_assert(std::__product_iterator_traits<Iter>::__size == 2);
+    std::same_as<cpp20_input_iterator<int*>&> decltype(auto) it1 =
+        std::__product_iterator_traits<Iter>::__get_iterator_element<0>(it);
+
+    assert(*it1 == 1);
+  }
+  if (!std::is_constant_evaluated()) {
+    // Test __make_product_iterator
+    using M = std::flat_map<int, int>;
+    M m{{1, 1}, {2, 2}, {3, 3}};
+    using Iter         = std::ranges::iterator_t<const M>;
+    const auto& keys   = m.keys();
+    const auto& values = m.values();
+
+    auto it_keys   = std::ranges::begin(keys);
+    auto it_values = std::ranges::begin(values);
+
+    auto it = std::__product_iterator_traits<Iter>::__make_product_iterator(it_keys, it_values);
+    assert(it->first == 1);
+    assert(it->second == 1);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp b/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp
index 1ecf378de5b61..5f618e92375ae 100644
--- a/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp
+++ b/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp
@@ -35,8 +35,11 @@
 //                const _Ctype& __ct, ios_base::iostate& __err,
 //                bool __case_sensitive = true);
 
-#include <locale>
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
+#include <__locale_dir/scan_keyword.h>
 #include <cassert>
+#include <locale>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
index 5d40fa71a4152..f43ad5b8c188a 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
@@ -11,9 +11,8 @@
 
 // <mutex>
 
-// This test does not define _LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS so it
-// should compile without any warnings or errors even though this pattern is not
-// understood by the thread safety annotations.
+// This test does not set -Wthread-safety so it should compile without any warnings or errors even though this pattern
+// is not understood by the thread safety annotations.
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
index 1c1f36459c837..9633982774e1f 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
index ddfe3269fdfa7..965d1444a4d5d 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
index bab686e7b4ca2..4ae22219b0308 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
@@ -6,21 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
 std::mutex m;
 
-void f() {
-  m.lock();
-} // expected-error {{mutex 'm' is still held at the end of function}}
+void f() { m.lock(); } // expected-warning {{mutex 'm' is still held at the end of function}}
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
index 81ce3b25a8394..e9794acc89414 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp b/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp
index fee940fff847b..d87a70927819b 100644
--- a/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp
+++ b/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp
@@ -8,12 +8,11 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
 
 // <shared_mutex>
 //
@@ -51,13 +50,13 @@ void f() {
   {
     m.lock_shared();
     read(data); // ok
-    ++data; // expected-error {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
+    ++data;     // expected-warning {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
     m.unlock_shared();
   }
   {
     if (m.try_lock_shared()) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
diff --git a/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp b/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp
index baa8bc614f149..9c16caa3631fb 100644
--- a/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp
+++ b/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp
@@ -8,12 +8,11 @@
 
 // UNSUPPORTED: c++03, c++11
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
 
 // <shared_mutex>
 //
@@ -68,27 +67,27 @@ void f(std::chrono::time_point<std::chrono::steady_clock> tp, std::chrono::milli
   {
     m.lock_shared();
     read(data); // ok
-    ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+    ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
     m.unlock_shared();
   }
   {
     if (m.try_lock_shared()) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
   {
     if (m.try_lock_shared_for(d)) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
   {
     if (m.try_lock_shared_until(tp)) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index ce8f0261f2b27..5f906338f4b7c 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -82,7 +82,6 @@ ccomplex ios
 ccomplex iosfwd
 ccomplex istream
 ccomplex limits
-ccomplex locale
 ccomplex ratio
 ccomplex sstream
 ccomplex stdexcept
@@ -120,7 +119,6 @@ chrono ios
 chrono iosfwd
 chrono istream
 chrono limits
-chrono locale
 chrono optional
 chrono ratio
 chrono sstream
@@ -177,7 +175,6 @@ complex ios
 complex iosfwd
 complex istream
 complex limits
-complex locale
 complex ratio
 complex sstream
 complex stdexcept
@@ -226,7 +223,6 @@ ctgmath ios
 ctgmath iosfwd
 ctgmath istream
 ctgmath limits
-ctgmath locale
 ctgmath ratio
 ctgmath sstream
 ctgmath stdexcept
@@ -278,7 +274,6 @@ experimental/iterator ios
 experimental/iterator iosfwd
 experimental/iterator iterator
 experimental/iterator limits
-experimental/iterator locale
 experimental/iterator ratio
 experimental/iterator stdexcept
 experimental/iterator streambuf
@@ -324,10 +319,8 @@ filesystem iomanip
 filesystem ios
 filesystem iosfwd
 filesystem limits
-filesystem locale
 filesystem ratio
 filesystem stdexcept
-filesystem streambuf
 filesystem string
 filesystem string_view
 filesystem tuple
@@ -412,7 +405,6 @@ fstream ios
 fstream iosfwd
 fstream istream
 fstream limits
-fstream locale
 fstream ratio
 fstream stdexcept
 fstream streambuf
@@ -455,7 +447,6 @@ future ios
 future iosfwd
 future istream
 future limits
-future locale
 future ratio
 future sstream
 future stdexcept
@@ -483,10 +474,8 @@ iomanip initializer_list
 iomanip ios
 iomanip iosfwd
 iomanip limits
-iomanip locale
 iomanip ratio
 iomanip stdexcept
-iomanip streambuf
 iomanip string
 iomanip string_view
 iomanip tuple
@@ -538,7 +527,6 @@ iostream ios
 iostream iosfwd
 iostream istream
 iostream limits
-iostream locale
 iostream optional
 iostream ostream
 iostream print
@@ -568,7 +556,6 @@ istream initializer_list
 istream ios
 istream iosfwd
 istream limits
-istream locale
 istream ratio
 istream stdexcept
 istream streambuf
@@ -724,7 +711,6 @@ ostream initializer_list
 ostream ios
 ostream iosfwd
 ostream limits
-ostream locale
 ostream optional
 ostream print
 ostream ratio
@@ -905,7 +891,6 @@ sstream ios
 sstream iosfwd
 sstream istream
 sstream limits
-sstream locale
 sstream ratio
 sstream stdexcept
 sstream streambuf
@@ -1004,7 +989,6 @@ strstream ios
 strstream iosfwd
 strstream istream
 strstream limits
-strstream locale
 strstream ratio
 strstream stdexcept
 strstream streambuf
@@ -1034,7 +1018,6 @@ syncstream initializer_list
 syncstream ios
 syncstream iosfwd
 syncstream limits
-syncstream locale
 syncstream map
 syncstream optional
 syncstream ostream
@@ -1086,7 +1069,6 @@ thread ios
 thread iosfwd
 thread istream
 thread limits
-thread locale
 thread ratio
 thread sstream
 thread stdexcept
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/and_then.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/and_then.mandates.verify.cpp
index c46ab633295c1..fbd2317ebeee2 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/and_then.mandates.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/and_then.mandates.verify.cpp
@@ -55,6 +55,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(value()) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
@@ -74,6 +75,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(value()) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
@@ -94,6 +96,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(std::move(value())) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
@@ -113,6 +116,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(std::move(value())) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/or_else.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/or_else.mandates.verify.cpp
index af1fa53307960..553ac4c6033a5 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/or_else.mandates.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/or_else.mandates.verify.cpp
@@ -55,6 +55,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(error()) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<G:value_type, T>
@@ -74,6 +75,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(error()) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<G:value_type, T>
@@ -93,6 +95,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(std::move(error())) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<G:value_type, T>
@@ -112,6 +115,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f(std::move(error())) must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<G:value_type, T>
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp
index d2b62a9b6ab99..91a7db1d9a7c6 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/value.observers.verify.cpp
@@ -124,8 +124,8 @@ void test() {
 #if _LIBCPP_HAS_EXCEPTIONS
   // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
   // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
-  // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
-  // expected-error-re@*:* {{call to deleted constructor of{{.*}}}}
+  // expected-error-re@*:* 1-2{{call to deleted constructor of{{.*}}}}
+  // expected-error-re@*:* 0-2{{call to deleted constructor of{{.*}}}}
 #endif
 }
 // clang-format on
diff --git a/libcxx/test/libcxx/utilities/expected/expected.void/and_then.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.void/and_then.mandates.verify.cpp
index e1a7ec5ff4d72..1df6cbf543b4f 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.void/and_then.mandates.verify.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.void/and_then.mandates.verify.cpp
@@ -55,6 +55,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f() must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
@@ -74,6 +75,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f() must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
@@ -93,6 +95,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f() must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
@@ -112,6 +115,7 @@ void test() {
       // expected-error-re@*:* {{static assertion failed {{.*}}The result of f() must be a specialization of std::expected}}
       // expected-error-re@*:* {{{{.*}}cannot be used prior to '::' because it has no members}}
       // expected-error-re@*:* {{no matching constructor for initialization of{{.*}}}}
+      // expected-error@*:* 0-1{{excess elements in struct initializer}}
     }
 
     // !std::is_same_v<U:error_type, E>
diff --git a/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp b/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp
index 48460d1488fd7..32f8d45fbbc95 100644
--- a/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp
@@ -254,8 +254,9 @@ struct ReferenceWrapper {
     static void fun(Type&) noexcept;
     static void fun(Type&&) = delete;
 
-    template <class Type2,
-              class = typename std::enable_if<!std::__is_same_uncvref<Type2, ReferenceWrapper>::value>::type>
+    template <
+        class Type2,
+        class = typename std::enable_if<!std::is_same<std::__remove_cvref_t<Type2>, ReferenceWrapper>::value>::type>
     constexpr ReferenceWrapper(Type2&& t) noexcept : ptr(&t) {}
 
     constexpr Type& get() const noexcept { return *ptr; }
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp b/libcxx/test/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp
rename to libcxx/test/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/lit.local.cfg b/libcxx/test/selftest/additional_compile_flags/lit.local.cfg
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/lit.local.cfg
rename to libcxx/test/selftest/additional_compile_flags/lit.local.cfg
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp b/libcxx/test/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
rename to libcxx/test/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-run.sh.cpp b/libcxx/test/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
rename to libcxx/test/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.fail.cpp/compile-error.compile.fail.cpp b/libcxx/test/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
rename to libcxx/test/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.fail.cpp/compile-success.compile.fail.cpp b/libcxx/test/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
rename to libcxx/test/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/compile-error.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/compile-success.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/link-error.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/link-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/link-error.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/link-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/run-error.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/run-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/run-error.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/run-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/compile-error.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/compile-error.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/compile-error.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/compile-error.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/compile-success.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/compile-success.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/compile-success.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/compile-success.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/link-error.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/link-error.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/link-error.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/link-error.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/run-error.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/run-error.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/run-error.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/run-error.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/convenience_substitutions/build_run.sh.cpp b/libcxx/test/selftest/convenience_substitutions/build_run.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/convenience_substitutions/build_run.sh.cpp
rename to libcxx/test/selftest/convenience_substitutions/build_run.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/convenience_substitutions/verify.sh.cpp b/libcxx/test/selftest/convenience_substitutions/verify.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/convenience_substitutions/verify.sh.cpp
rename to libcxx/test/selftest/convenience_substitutions/verify.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/selftest/dsl/dsl.sh.py
similarity index 99%
rename from libcxx/test/libcxx/selftest/dsl/dsl.sh.py
rename to libcxx/test/selftest/dsl/dsl.sh.py
index 012759436d891..6d4406b7858e6 100644
--- a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
+++ b/libcxx/test/selftest/dsl/dsl.sh.py
@@ -26,7 +26,7 @@
 
 # Allow importing 'lit' and the 'libcxx' module. Make sure we put the lit
 # path first so we don't find any system-installed version.
-monorepoRoot = dirname(dirname(dirname(dirname(dirname(dirname(__file__))))))
+monorepoRoot = dirname(dirname(dirname(dirname(dirname(__file__)))))
 sys.path = [
     os.path.join(monorepoRoot, "libcxx", "utils"),
     os.path.join(monorepoRoot, "llvm", "utils", "lit"),
diff --git a/libcxx/test/libcxx/selftest/dsl/lit.local.cfg b/libcxx/test/selftest/dsl/lit.local.cfg
similarity index 100%
rename from libcxx/test/libcxx/selftest/dsl/lit.local.cfg
rename to libcxx/test/selftest/dsl/lit.local.cfg
diff --git a/libcxx/test/selftest/file_dependencies/a.txt b/libcxx/test/selftest/file_dependencies/a.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp b/libcxx/test/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
rename to libcxx/test/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
diff --git a/libcxx/test/selftest/file_dependencies/dir/b.txt b/libcxx/test/selftest/file_dependencies/dir/b.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/substitute-in-dependencies.sh.cpp b/libcxx/test/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
rename to libcxx/test/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp b/libcxx/test/selftest/gen.cpp/empty.gen.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
rename to libcxx/test/selftest/gen.cpp/empty.gen.cpp
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp b/libcxx/test/selftest/gen.cpp/one.gen.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp
rename to libcxx/test/selftest/gen.cpp/one.gen.cpp
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp b/libcxx/test/selftest/gen.cpp/two.gen.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp
rename to libcxx/test/selftest/gen.cpp/two.gen.cpp
diff --git a/libcxx/test/libcxx/selftest/link.fail.cpp/compile-error.link.fail.cpp b/libcxx/test/selftest/link.fail.cpp/compile-error.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.fail.cpp/compile-error.link.fail.cpp
rename to libcxx/test/selftest/link.fail.cpp/compile-error.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/link.fail.cpp/link-error.link.fail.cpp b/libcxx/test/selftest/link.fail.cpp/link-error.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.fail.cpp/link-error.link.fail.cpp
rename to libcxx/test/selftest/link.fail.cpp/link-error.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/link.fail.cpp/link-success.link.fail.cpp b/libcxx/test/selftest/link.fail.cpp/link-success.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.fail.cpp/link-success.link.fail.cpp
rename to libcxx/test/selftest/link.fail.cpp/link-success.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/compile-error.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/compile-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/compile-error.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/compile-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/link-error.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/link-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/link-error.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/link-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/link-success.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/link-success.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/link-success.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/link-success.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/run-error.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/run-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/run-error.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/run-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/compile-error.link.pass.mm b/libcxx/test/selftest/link.pass.mm/compile-error.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/compile-error.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/compile-error.link.pass.mm
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/link-error.link.pass.mm b/libcxx/test/selftest/link.pass.mm/link-error.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/link-error.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/link-error.link.pass.mm
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/link-success.link.pass.mm b/libcxx/test/selftest/link.pass.mm/link-success.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/link-success.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/link-success.link.pass.mm
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/run-error.link.pass.mm b/libcxx/test/selftest/link.pass.mm/run-error.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/run-error.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/run-error.link.pass.mm
diff --git a/libcxx/test/libcxx/lit.local.cfg b/libcxx/test/selftest/lit.local.cfg
similarity index 100%
rename from libcxx/test/libcxx/lit.local.cfg
rename to libcxx/test/selftest/lit.local.cfg
diff --git a/libcxx/test/libcxx/selftest/modules/no-modules.sh.cpp b/libcxx/test/selftest/modules/no-modules.sh.cpp
similarity index 73%
rename from libcxx/test/libcxx/selftest/modules/no-modules.sh.cpp
rename to libcxx/test/selftest/modules/no-modules.sh.cpp
index 74e65a9072f7b..762b38ccaaebf 100644
--- a/libcxx/test/libcxx/selftest/modules/no-modules.sh.cpp
+++ b/libcxx/test/selftest/modules/no-modules.sh.cpp
@@ -6,6 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // Make sure that the compile flags contain no module information.
 
 // MODULE_DEPENDENCIES:
diff --git a/libcxx/test/libcxx/selftest/modules/std-and-std.compat-module.sh.cpp b/libcxx/test/selftest/modules/std-and-std.compat-module.sh.cpp
similarity index 82%
rename from libcxx/test/libcxx/selftest/modules/std-and-std.compat-module.sh.cpp
rename to libcxx/test/selftest/modules/std-and-std.compat-module.sh.cpp
index d56ebb2961d4a..455674d6b9949 100644
--- a/libcxx/test/libcxx/selftest/modules/std-and-std.compat-module.sh.cpp
+++ b/libcxx/test/selftest/modules/std-and-std.compat-module.sh.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: clang-modules-build
 // UNSUPPORTED: gcc
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // XFAIL: has-no-cxx-module-support
 
 // Make sure that the compile flags contain the expected elements.
diff --git a/libcxx/test/libcxx/selftest/modules/std-module.sh.cpp b/libcxx/test/selftest/modules/std-module.sh.cpp
similarity index 82%
rename from libcxx/test/libcxx/selftest/modules/std-module.sh.cpp
rename to libcxx/test/selftest/modules/std-module.sh.cpp
index ec43994fa1ef9..e96c1e64b7275 100644
--- a/libcxx/test/libcxx/selftest/modules/std-module.sh.cpp
+++ b/libcxx/test/selftest/modules/std-module.sh.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: clang-modules-build
 // UNSUPPORTED: gcc
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // XFAIL: has-no-cxx-module-support
 
 // Make sure that the compile flags contain the expected elements.
diff --git a/libcxx/test/libcxx/selftest/modules/std.compat-module.sh.cpp b/libcxx/test/selftest/modules/std.compat-module.sh.cpp
similarity index 82%
rename from libcxx/test/libcxx/selftest/modules/std.compat-module.sh.cpp
rename to libcxx/test/selftest/modules/std.compat-module.sh.cpp
index e84709853fbca..4ec0ad8323d8d 100644
--- a/libcxx/test/libcxx/selftest/modules/std.compat-module.sh.cpp
+++ b/libcxx/test/selftest/modules/std.compat-module.sh.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: clang-modules-build
 // UNSUPPORTED: gcc
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // XFAIL: has-no-cxx-module-support
 
 // Make sure that the compile flags contain the expected elements.
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp b/libcxx/test/selftest/pass.cpp/compile-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp
rename to libcxx/test/selftest/pass.cpp/compile-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp b/libcxx/test/selftest/pass.cpp/link-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp
rename to libcxx/test/selftest/pass.cpp/link-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/run-error.pass.cpp b/libcxx/test/selftest/pass.cpp/run-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/run-error.pass.cpp
rename to libcxx/test/selftest/pass.cpp/run-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/run-success.pass.cpp b/libcxx/test/selftest/pass.cpp/run-success.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/run-success.pass.cpp
rename to libcxx/test/selftest/pass.cpp/run-success.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp b/libcxx/test/selftest/pass.cpp/werror.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp
rename to libcxx/test/selftest/pass.cpp/werror.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm b/libcxx/test/selftest/pass.mm/compile-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm
rename to libcxx/test/selftest/pass.mm/compile-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm b/libcxx/test/selftest/pass.mm/link-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm
rename to libcxx/test/selftest/pass.mm/link-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/no-arc.pass.mm b/libcxx/test/selftest/pass.mm/no-arc.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/no-arc.pass.mm
rename to libcxx/test/selftest/pass.mm/no-arc.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/run-error.pass.mm b/libcxx/test/selftest/pass.mm/run-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/run-error.pass.mm
rename to libcxx/test/selftest/pass.mm/run-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/run-success.pass.mm b/libcxx/test/selftest/pass.mm/run-success.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/run-success.pass.mm
rename to libcxx/test/selftest/pass.mm/run-success.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/use-objective-cxx.pass.mm b/libcxx/test/selftest/pass.mm/use-objective-cxx.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/use-objective-cxx.pass.mm
rename to libcxx/test/selftest/pass.mm/use-objective-cxx.pass.mm
diff --git a/libcxx/test/libcxx/selftest/remote-substitutions.sh.cpp b/libcxx/test/selftest/remote-substitutions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/remote-substitutions.sh.cpp
rename to libcxx/test/selftest/remote-substitutions.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/run-error.sh.cpp b/libcxx/test/selftest/sh.cpp/run-error.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/run-error.sh.cpp
rename to libcxx/test/selftest/sh.cpp/run-error.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp b/libcxx/test/selftest/sh.cpp/run-success.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
rename to libcxx/test/selftest/sh.cpp/run-success.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/substitutions.sh.cpp b/libcxx/test/selftest/sh.cpp/substitutions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/substitutions.sh.cpp
rename to libcxx/test/selftest/sh.cpp/substitutions.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/werror.sh.cpp b/libcxx/test/selftest/sh.cpp/werror.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/werror.sh.cpp
rename to libcxx/test/selftest/sh.cpp/werror.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/shell-no-escape-builtins.sh.cpp b/libcxx/test/selftest/shell-no-escape-builtins.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/shell-no-escape-builtins.sh.cpp
rename to libcxx/test/selftest/shell-no-escape-builtins.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp b/libcxx/test/selftest/stdin-is-piped.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp
rename to libcxx/test/selftest/stdin-is-piped.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/test_macros.pass.cpp b/libcxx/test/selftest/test_macros.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/test_macros.pass.cpp
rename to libcxx/test/selftest/test_macros.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/tmpdir-exists.sh.cpp b/libcxx/test/selftest/tmpdir-exists.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/tmpdir-exists.sh.cpp
rename to libcxx/test/selftest/tmpdir-exists.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp b/libcxx/test/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
rename to libcxx/test/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics.verify.cpp b/libcxx/test/selftest/verify.cpp/no-diagnostics.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics.verify.cpp
rename to libcxx/test/selftest/verify.cpp/no-diagnostics.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/no-werror.verify.cpp b/libcxx/test/selftest/verify.cpp/no-werror.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/no-werror.verify.cpp
rename to libcxx/test/selftest/verify.cpp/no-werror.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/right-diagnostic.verify.cpp b/libcxx/test/selftest/verify.cpp/right-diagnostic.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/right-diagnostic.verify.cpp
rename to libcxx/test/selftest/verify.cpp/right-diagnostic.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/wrong-diagnostic.verify.cpp b/libcxx/test/selftest/verify.cpp/wrong-diagnostic.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/wrong-diagnostic.verify.cpp
rename to libcxx/test/selftest/verify.cpp/wrong-diagnostic.verify.cpp
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
index 66bd4022f321a..9bcdc9c1f2b9e 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
@@ -18,6 +18,7 @@
 #include <cassert>
 #include <functional>
 #include <deque>
+#include <ranges>
 
 #include "MinSequenceContainer.h"
 #include "../helpers.h"
@@ -95,6 +96,26 @@ constexpr void test() {
       });
 }
 
+constexpr void test_product_iterator() {
+  using M = std::flat_map<int, int>;
+  {
+    M m1{{1, 1}, {2, 1}, {3, 1}};
+    M m2{{4, 1}, {5, 1}, {6, 1}};
+    m1.insert(m2.begin(), m2.end());
+    M expected{{1, 1}, {2, 1}, {3, 1}, {4, 1}, {5, 1}, {6, 1}};
+    assert(m1 == expected);
+  }
+  {
+    std::vector<int> keys{1, 2, 3};
+    std::vector<int> values{1, 1, 1};
+    auto zv = std::views::zip(keys, values);
+    M m;
+    m.insert(zv.begin(), zv.end());
+    M expected{{1, 1}, {2, 1}, {3, 1}};
+    assert(m == expected);
+  }
+}
+
 constexpr bool test() {
   test<std::vector<int>, std::vector<double>>();
 #ifndef __cpp_lib_constexpr_deque
@@ -105,7 +126,7 @@ constexpr bool test() {
   }
   test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
   test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
-
+  test_product_iterator();
   if (!TEST_IS_CONSTANT_EVALUATED) {
     auto insert_func = [](auto& m, const auto& newValues) { m.insert(newValues.begin(), newValues.end()); };
     test_insert_range_exception_guarantee(insert_func);
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp
index 223b92fc3e8e8..e0afa752b5a95 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<int>, KeyContainer>;
   M m;
@@ -38,15 +38,23 @@ void test_one() {
   assert(m.empty());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp
index dde1f7092e5b1..e2f52ab458c4c 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp
@@ -24,7 +24,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-void test() {
+constexpr bool test() {
   {
     using A1 = limited_allocator<int, 10>;
     using C  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
@@ -59,10 +59,15 @@ void test() {
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(std::allocator<char>()));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp
index 9f5ffdd066351..4c4de89f58ba3 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp
@@ -16,6 +16,7 @@
 #include <deque>
 #include <flat_set>
 #include <functional>
+#include <type_traits>
 #include <vector>
 
 #include "MinSequenceContainer.h"
@@ -23,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, std::less<int>, KeyContainer>;
   using S = typename M::size_type;
   {
@@ -46,7 +47,7 @@ void test_one() {
   }
   {
     M m;
-    S s = 1000000;
+    S s = TEST_IS_CONSTANT_EVALUATED ? 100 : 1000000;
     for (auto i = 0u; i < s; ++i) {
       m.emplace(i);
     }
@@ -56,15 +57,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp
index d14e883dd5e93..3b7a3550391f5 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp
@@ -14,15 +14,18 @@
 //   explicit flat_set(const Allocator& a);
 
 #include <cassert>
+#include <deque>
 #include <flat_set>
 #include <functional>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "../../../test_compare.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // The constructors in this subclause shall not participate in overload
     // resolution unless uses_allocator_v<container_type, Alloc> is true.
@@ -30,8 +33,8 @@ void test() {
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
     using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
+    using V1 = KeyContainer<int, A1>;
+    using V2 = KeyContainer<int, A2>;
     using M1 = std::flat_set<int, C, V1>;
     using M2 = std::flat_set<int, C, V2>;
     static_assert(std::is_constructible_v<M1, const A1&>);
@@ -41,14 +44,14 @@ void test() {
   }
   {
     // explicit
-    using M = std::flat_set<int, std::less<int>, std::vector<int, test_allocator<int>>>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, test_allocator<int>>>;
 
     static_assert(std::is_constructible_v<M, test_allocator<int>>);
     static_assert(!std::is_convertible_v<test_allocator<int>, M>);
   }
   {
     using A = test_allocator<short>;
-    using M = std::flat_set<int, std::less<int>, std::vector<int, test_allocator<int>>>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, test_allocator<int>>>;
     M m(A(0, 5));
     assert(m.empty());
     assert(m.begin() == m.end());
@@ -57,8 +60,21 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp
index ad49b62149036..388a515893bd9 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -45,17 +45,25 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp
index 110757a1bb9ab..712f33a7b7f8a 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp
@@ -23,27 +23,14 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 #include "test_allocator.h"
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, const C&, const A1&>);
-  }
+template <class KeyContainer>
+constexpr void test_compare() {
   {
     using C = test_less<int>;
-    auto m  = std::flat_set<int, C>(C(3));
+    auto m  = std::flat_set<int, C, KeyContainer>(C(3));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(3));
@@ -51,16 +38,20 @@ void test() {
   {
     // The one-argument ctor is explicit.
     using C = test_less<int>;
-    static_assert(std::is_constructible_v<std::flat_set<int, C>, C>);
-    static_assert(!std::is_convertible_v<C, std::flat_set<int, C>>);
+    static_assert(std::is_constructible_v<std::flat_set<int, C, KeyContainer>, C>);
+    static_assert(!std::is_convertible_v<C, std::flat_set<int, C, KeyContainer>>);
 
-    static_assert(std::is_constructible_v<std::flat_set<int>, std::less<int>>);
-    static_assert(!std::is_convertible_v<std::less<int>, std::flat_set<int>>);
+    static_assert(std::is_constructible_v<std::flat_set<int, std::less<int>, KeyContainer>, std::less<int>>);
+    static_assert(!std::is_convertible_v<std::less<int>, std::flat_set<int, std::less<int>, KeyContainer>>);
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_compare_alloc() {
   {
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    auto m   = std::flat_set<int, C, std::vector<int, A1>>(C(4), A1(5));
+    auto m   = std::flat_set<int, C, KeyContainer<int, A1>>(C(4), A1(5));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
@@ -69,9 +60,9 @@ void test() {
   }
   {
     // explicit(false)
-    using C                                      = test_less<int>;
-    using A1                                     = test_allocator<int>;
-    std::flat_set<int, C, std::deque<int, A1>> m = {C(4), A1(5)};
+    using C                                        = test_less<int>;
+    using A1                                       = test_allocator<int>;
+    std::flat_set<int, C, KeyContainer<int, A1>> m = {C(4), A1(5)};
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
@@ -80,8 +71,45 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const C&, const A1&>);
+  }
+  test_compare<std::vector<int>>();
+  test_compare<MinSequenceContainer<int>>();
+  test_compare<std::vector<int, min_allocator<int>>>();
+
+  test_compare_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_compare<std::deque<int>>();
+    test_compare_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp
index 6b1246885bf52..ea775ebd19bec 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp
@@ -36,44 +36,24 @@ void conversion_test(T);
 template <class T, class... Args>
 concept ImplicitlyConstructible = requires(Args&&... args) { conversion_test<T>({std::forward<Args>(args)...}); };
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, const V1&, const A1&>);
-    static_assert(std::is_constructible_v<M2, const V2&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, const V1&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, const V2&, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, const V1&, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, const V2&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, const V1&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, const V2&, const C&, const A1&>);
-  }
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // flat_set(container_type)
-    using M             = std::flat_set<int>;
-    std::vector<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
-    auto m              = M(ks);
+    using M              = std::flat_set<int, std::less<int>, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    auto m               = M(ks);
     assert(std::ranges::equal(m, std::vector<int>{1, 2, 3}));
 
     // explicit
-    static_assert(std::is_constructible_v<M, const std::vector<int>&>);
-    static_assert(!ImplicitlyConstructible<M, const std::vector<int>&>);
+    static_assert(std::is_constructible_v<M, const KeyContainer<int>&>);
+    static_assert(!ImplicitlyConstructible<M, const KeyContainer<int>&>);
   }
   {
     // flat_set(container_type)
     // move-only
     MoveOnly expected[] = {3, 2, 1};
-    using Ks            = std::deque<MoveOnly, min_allocator<MoveOnly>>;
+    using Ks            = KeyContainer<MoveOnly, min_allocator<MoveOnly>>;
     using M             = std::flat_set<MoveOnly, std::greater<MoveOnly>, Ks>;
     Ks ks;
     ks.push_back(1);
@@ -87,8 +67,8 @@ void test() {
     // flat_set(container_type)
     // container's allocator is used
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
     auto m  = M(std::move(ks));
     assert(ks.empty()); // it was moved-from
     assert((m == M{1, 2, 3}));
@@ -97,22 +77,22 @@ void test() {
   }
   {
     // flat_set(container_type , key_compare)
-    using C             = test_less<int>;
-    using M             = std::flat_set<int, C>;
-    std::vector<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
-    auto m              = M(ks, C(4));
+    using C              = test_less<int>;
+    using M              = std::flat_set<int, C, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    auto m               = M(ks, C(4));
     assert(std::ranges::equal(m, std::vector<int>{1, 2, 3}));
     assert(m.key_comp() == C(4));
 
     // explicit
-    static_assert(std::is_constructible_v<M, const std::vector<int>&, const C&>);
-    static_assert(!ImplicitlyConstructible<M, const std::vector<int>&, const C&>);
+    static_assert(std::is_constructible_v<M, const KeyContainer<int>&, const C&>);
+    static_assert(!ImplicitlyConstructible<M, const KeyContainer<int>&, const C&>);
   }
   {
     // flat_set(container_type , const Allocator&)
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
     auto m  = M(ks, A(4)); // replaces the allocators
     assert(!ks.empty());   // it was an lvalue above
     assert((m == M{1, 2, 3}));
@@ -120,7 +100,7 @@ void test() {
     assert(keys.get_allocator() == A(4));
 
     // explicit(false)
-    static_assert(ImplicitlyConstructible<M, const std::deque<int, A>&, const A&>);
+    static_assert(ImplicitlyConstructible<M, const KeyContainer<int, A>&, const A&>);
     M m2 = {ks, A(4)};   // implicit ctor
     assert(!ks.empty()); // it was an lvalue above
     assert(m2 == m);
@@ -129,11 +109,11 @@ void test() {
   }
   {
     // flat_set(container_type , key_compare, const Allocator&)
-    using C                = test_less<int>;
-    using A                = test_allocator<int>;
-    using M                = std::flat_set<int, C, std::vector<int, A>>;
-    std::vector<int, A> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
-    auto m                 = M(ks, C(4), A(5));
+    using C                 = test_less<int>;
+    using A                 = test_allocator<int>;
+    using M                 = std::flat_set<int, C, KeyContainer<int, A>>;
+    KeyContainer<int, A> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    auto m                  = M(ks, C(4), A(5));
     assert(std::ranges::equal(m, std::vector<int, A>{1, 2, 3}));
     assert(m.key_comp() == C(4));
     auto m_copy = m;
@@ -141,7 +121,7 @@ void test() {
     assert(keys.get_allocator() == A(5));
 
     // explicit(false)
-    static_assert(ImplicitlyConstructible<M, const std::vector<int, A>&, const A&>);
+    static_assert(ImplicitlyConstructible<M, const KeyContainer<int, A>&, const A&>);
     M m2 = {ks, C(4), A(5)};
     assert(m2 == m);
     assert(m2.key_comp() == C(4));
@@ -150,8 +130,44 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, const V1&, const A1&>);
+    static_assert(std::is_constructible_v<M2, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V2&, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, const V1&, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V2&, const C&, const A1&>);
+  }
+
+  test<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp
index 1ba550d98f01f..5864d7d751b12 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <deque>
 #include <flat_set>
 #include <vector>
 
@@ -21,10 +22,11 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     using C = test_less<int>;
-    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    KeyContainer<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
     using M = std::flat_set<int, C, decltype(ks)>;
     auto mo = M(ks, C(5));
     auto m  = mo;
@@ -42,7 +44,7 @@ void test() {
   }
   {
     using C  = test_less<int>;
-    using Ks = std::vector<int, other_allocator<int>>;
+    using Ks = KeyContainer<int, other_allocator<int>>;
     auto ks  = Ks({1, 3, 5}, other_allocator<int>(6));
     using M  = std::flat_set<int, C, Ks>;
     auto mo  = M(Ks(ks, other_allocator<int>(6)), C(5));
@@ -61,8 +63,21 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp
index 5011bd2003064..108a3cf1c4968 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp
@@ -23,7 +23,27 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
+  using C = test_less<int>;
+  KeyContainer<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+  using M = std::flat_set<int, C, decltype(ks)>;
+  auto mo = M(ks, C(5));
+  auto m  = M(mo, test_allocator<int>(3));
+
+  assert(m.key_comp() == C(5));
+  assert(std::ranges::equal(m, ks));
+  auto keys = std::move(m).extract();
+  assert(keys.get_allocator() == test_allocator<int>(3));
+
+  // mo is unchanged
+  assert(mo.key_comp() == C(5));
+  assert(std::ranges::equal(mo, ks));
+  auto keys2 = std::move(mo).extract();
+  assert(keys2.get_allocator() == test_allocator<int>(6));
+}
+
+constexpr bool test() {
   {
     // The constructors in this subclause shall not participate in overload
     // resolution unless uses_allocator_v<container_type, Alloc> is true.
@@ -40,28 +60,21 @@ void test() {
     static_assert(!std::is_constructible_v<M1, const M1&, const A2&>);
     static_assert(!std::is_constructible_v<M2, const M2&, const A1&>);
   }
-  {
-    using C = test_less<int>;
-    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
-    using M = std::flat_set<int, C, decltype(ks)>;
-    auto mo = M(ks, C(5));
-    auto m  = M(mo, test_allocator<int>(3));
 
-    assert(m.key_comp() == C(5));
-    assert(std::ranges::equal(m, ks));
-    auto keys = std::move(m).extract();
-    assert(keys.get_allocator() == test_allocator<int>(3));
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
 
-    // mo is unchanged
-    assert(mo.key_comp() == C(5));
-    assert(std::ranges::equal(mo, ks));
-    auto keys2 = std::move(mo).extract();
-    assert(keys2.get_allocator() == test_allocator<int>(6));
-  }
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp
index 695363e3aeaba..59caa9c1f8a48 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp
@@ -13,6 +13,7 @@
 // flat_set& operator=(const flat_set& m);
 
 #include <algorithm>
+#include <deque>
 #include <flat_set>
 #include <functional>
 #include <vector>
@@ -22,11 +23,12 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // test_allocator is not propagated
     using C = test_less<int>;
-    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    KeyContainer<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
     using M = std::flat_set<int, C, decltype(ks)>;
     auto mo = M(ks, C(5));
     auto m  = M({{3, 4, 5}}, C(3), test_allocator<int>(2));
@@ -46,7 +48,7 @@ void test() {
   {
     // other_allocator is propagated
     using C  = test_less<int>;
-    using Ks = std::vector<int, other_allocator<int>>;
+    using Ks = KeyContainer<int, other_allocator<int>>;
     auto ks  = Ks({1, 3, 5}, other_allocator<int>(6));
     using M  = std::flat_set<int, C, Ks>;
     auto mo  = M(Ks(ks, other_allocator<int>(6)), C(5));
@@ -64,9 +66,9 @@ void test() {
     auto keys2 = std::move(mo).extract();
     assert(keys2.get_allocator() == other_allocator<int>(6));
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // comparator is copied and invariant is preserved
-    using M = std::flat_set<int, std::function<bool(int, int)>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int>>;
     M mo    = M({1, 2}, std::less<int>());
     M m     = M({1, 2}, std::greater<int>());
     assert(m.key_comp()(2, 1) == true);
@@ -77,7 +79,7 @@ void test() {
   }
   {
     // self-assignment
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     M m     = {{1, 2}};
     m       = static_cast<const M&>(m);
     assert((m == M{{1, 2}}));
@@ -93,8 +95,21 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp
index 5cbd557eb8562..75eba28b61bd5 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp
@@ -25,28 +25,29 @@
 #include "test_macros.h"
 
 struct DefaultCtableComp {
-  explicit DefaultCtableComp() { default_constructed_ = true; }
-  bool operator()(int, int) const { return false; }
+  constexpr explicit DefaultCtableComp() { default_constructed_ = true; }
+  constexpr bool operator()(int, int) const { return false; }
   bool default_constructed_ = false;
 };
 
 struct ThrowingCtorComp {
-  ThrowingCtorComp() noexcept(false) {}
-  bool operator()(const auto&, const auto&) const { return false; }
+  constexpr ThrowingCtorComp() noexcept(false) {}
+  constexpr bool operator()(const auto&, const auto&) const { return false; }
 };
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
-    std::flat_set<int> m;
+    std::flat_set<int, std::less<int>, KeyContainer<int>> m;
     assert(m.empty());
   }
   {
     // explicit(false)
-    std::flat_set<int> m = {};
+    std::flat_set<int, std::less<int>, KeyContainer<int>> m = {};
     assert(m.empty());
   }
   {
-    std::flat_set<int, DefaultCtableComp, std::deque<int, min_allocator<int>>> m;
+    std::flat_set<int, DefaultCtableComp, KeyContainer<int, min_allocator<int>>> m;
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp().default_constructed_);
@@ -54,43 +55,56 @@ void test() {
   {
     using A1 = explicit_allocator<int>;
     {
-      std::flat_set<int, DefaultCtableComp, std::vector<int, A1>> m;
+      std::flat_set<int, DefaultCtableComp, KeyContainer<int, A1>> m;
       assert(m.empty());
       assert(m.key_comp().default_constructed_);
     }
     {
       A1 a1;
-      std::flat_set<int, DefaultCtableComp, std::vector<int, A1>> m(a1);
+      std::flat_set<int, DefaultCtableComp, KeyContainer<int, A1>> m(a1);
       assert(m.empty());
       assert(m.key_comp().default_constructed_);
     }
   }
 #if defined(_LIBCPP_VERSION)
   {
-    using C = std::flat_set<MoveOnly>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly>>;
     static_assert(std::is_nothrow_default_constructible_v<C>);
     C c;
   }
   {
-    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, test_allocator<MoveOnly>>>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly, test_allocator<MoveOnly>>>;
     static_assert(std::is_nothrow_default_constructible_v<C>);
     C c;
   }
 #endif // _LIBCPP_VERSION
   {
-    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, other_allocator<MoveOnly>>>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly, other_allocator<MoveOnly>>>;
     static_assert(!std::is_nothrow_default_constructible_v<C>);
     C c;
   }
   {
-    using C = std::flat_set<MoveOnly, ThrowingCtorComp>;
+    using C = std::flat_set<MoveOnly, ThrowingCtorComp, KeyContainer<MoveOnly>>;
     static_assert(!std::is_nothrow_default_constructible_v<C>);
     C c;
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp
index fa1e2478af459..810b13b0a5b0b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp
@@ -23,39 +23,53 @@
 #include "test_allocator.h"
 
 struct ThrowingDtorComp {
-  bool operator()(const auto&, const auto&) const;
-  ~ThrowingDtorComp() noexcept(false) {}
+  constexpr bool operator()(const auto&, const auto&) const;
+  constexpr ~ThrowingDtorComp() noexcept(false) {}
 };
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
-    using C = std::flat_set<MoveOnly, MoveOnly>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly>>;
     static_assert(std::is_nothrow_destructible_v<C>);
     C c;
   }
   {
-    using V = std::vector<MoveOnly, test_allocator<MoveOnly>>;
+    using V = KeyContainer<MoveOnly, test_allocator<MoveOnly>>;
     using C = std::flat_set<MoveOnly, std::less<MoveOnly>, V>;
     static_assert(std::is_nothrow_destructible_v<C>);
     C c;
   }
   {
-    using V = std::deque<MoveOnly, other_allocator<MoveOnly>>;
+    using V = KeyContainer<MoveOnly, other_allocator<MoveOnly>>;
     using C = std::flat_set<MoveOnly, std::greater<MoveOnly>, V>;
     static_assert(std::is_nothrow_destructible_v<C>);
     C c;
   }
 #if defined(_LIBCPP_VERSION)
   {
-    using C = std::flat_set<MoveOnly, ThrowingDtorComp>;
+    using C = std::flat_set<MoveOnly, ThrowingDtorComp, KeyContainer<MoveOnly>>;
     static_assert(!std::is_nothrow_destructible_v<C>);
     C c;
   }
 #endif // _LIBCPP_VERSION
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp
index 9aed5c88ee726..2a8e4c40ebe2b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp
@@ -30,57 +30,17 @@
 #include "../../../test_compare.h"
 
 struct DefaultCtableComp {
-  explicit DefaultCtableComp() { default_constructed_ = true; }
-  bool operator()(int, int) const { return false; }
+  constexpr explicit DefaultCtableComp() { default_constructed_ = true; }
+  constexpr bool operator()(int, int) const { return false; }
   bool default_constructed_ = false;
 };
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    using IL = std::initializer_list<int>;
-    static_assert(std::is_constructible_v<M1, IL, const A1&>);
-    static_assert(std::is_constructible_v<M2, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M1, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M2, IL, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, IL, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, IL, const C&, const A1&>);
-  }
-
-  {
-    // initializer_list<value_type> needs to match exactly
-    using M = std::flat_set<int>;
-    using C = typename M::key_compare;
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>>);
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C>);
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C, std::allocator<int>>);
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
-  }
-
+template <template <class...> class KeyContainer>
+constexpr void test() {
   int expected[] = {1, 2, 3, 5};
   {
     // flat_set(initializer_list<value_type>);
-    using M                       = std::flat_set<int>;
+    using M                       = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     std::initializer_list<int> il = {5, 2, 2, 3, 1, 3};
     M m(il);
     assert(std::equal(m.begin(), m.end(), expected, expected + 4));
@@ -88,13 +48,13 @@ void test() {
   {
     // flat_set(initializer_list<value_type>);
     // explicit(false)
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     M m     = {5, 2, 2, 3, 1, 3};
     assert(std::equal(m.begin(), m.end(), expected, expected + 4));
   }
   {
     // flat_set(initializer_list<value_type>);
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer<int, min_allocator<int>>>;
     M m     = {5, 2, 2, 3, 1, 3};
     assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
   }
@@ -103,15 +63,14 @@ void test() {
     {
       // flat_set(initializer_list<value_type>);
       // different comparator
-      using M = std::flat_set<int, DefaultCtableComp, std::vector<int, A>>;
+      using M = std::flat_set<int, DefaultCtableComp, KeyContainer<int, A>>;
       M m     = {1, 2, 3};
       assert(m.size() == 1);
-      LIBCPP_ASSERT(*m.begin() == 1);
       assert(m.key_comp().default_constructed_);
     }
     {
       // flat_set(initializer_list<value_type>, const Allocator&);
-      using M = std::flat_set<int, std::greater<int>, std::deque<int, A>>;
+      using M = std::flat_set<int, std::greater<int>, KeyContainer<int, A>>;
       A a;
       M m({5, 2, 2, 3, 1, 3}, a);
       assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
@@ -120,7 +79,7 @@ void test() {
   {
     // flat_set(initializer_list<value_type>, const key_compare&);
     using C = test_less<int>;
-    using M = std::flat_set<int, C>;
+    using M = std::flat_set<int, C, KeyContainer<int>>;
     auto m  = M({5, 2, 2, 3, 1, 3}, C(10));
     assert(std::equal(m.begin(), m.end(), expected, expected + 4));
     assert(m.key_comp() == C(10));
@@ -130,10 +89,10 @@ void test() {
     assert(m2 == m);
     assert(m2.key_comp() == C(10));
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // flat_set(initializer_list<value_type>, const key_compare&);
     // Sorting uses the comparator that was passed in
-    using M = std::flat_set<int, std::function<bool(int, int)>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int, min_allocator<int>>>;
     auto m  = M({5, 2, 2, 1, 3, 1}, std::greater<int>());
     assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
     assert(m.key_comp()(2, 1) == true);
@@ -141,15 +100,69 @@ void test() {
   {
     // flat_set(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
     using A = explicit_allocator<int>;
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, A>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer<int, A>>;
     A a;
     M m({5, 2, 2, 3, 1, 3}, {}, a);
     assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    using IL = std::initializer_list<int>;
+    static_assert(std::is_constructible_v<M1, IL, const A1&>);
+    static_assert(std::is_constructible_v<M2, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M1, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, IL, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const C&, const A1&>);
+  }
+
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_set<int>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C, std::allocator<int>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp
index 2d0b07c9155fd..256f01a7b2f12 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp
@@ -23,43 +23,21 @@
 #include <functional>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C     = test_less<int>;
-    using A1    = test_allocator<int>;
-    using A2    = other_allocator<int>;
-    using V1    = std::vector<int, A1>;
-    using V2    = std::vector<int, A2>;
-    using M1    = std::flat_set<int, C, V1>;
-    using M2    = std::flat_set<int, C, V2>;
-    using Iter1 = typename M1::iterator;
-    using Iter2 = typename M2::iterator;
-    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const A1&>);
-    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const A2&>);
-    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const A2&>);
-    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const C&, const A1&>);
-  }
-
+template <class KeyContainer>
+constexpr void test() {
   int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
   int expected[] = {1, 2, 3};
   {
     // flat_set(InputIterator , InputIterator)
     // cpp17_input_iterator
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
     auto m  = M(cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + 9));
     assert(std::ranges::equal(m, expected));
 
@@ -70,21 +48,21 @@ void test() {
   {
     // flat_set(InputIterator , InputIterator)
     // greater
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer>;
     auto m  = M(cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + 9));
-    assert(std::ranges::equal(m, std::deque<int, min_allocator<int>>{3, 2, 1}));
+    assert(std::ranges::equal(m, KeyContainer{3, 2, 1}));
   }
   {
     // flat_set(InputIterator , InputIterator)
     // Test when the operands are of array type (also contiguous iterator type)
-    using M = std::flat_set<int, std::greater<int>, std::vector<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer>;
     auto m  = M(ar, ar);
     assert(m.empty());
   }
   {
     // flat_set(InputIterator , InputIterator, const key_compare&)
     using C = test_less<int>;
-    using M = std::flat_set<int, C, std::vector<int>>;
+    using M = std::flat_set<int, C, KeyContainer>;
     auto m  = M(ar, ar + 9, C(3));
     assert(std::ranges::equal(m, expected));
     assert(m.key_comp() == C(3));
@@ -94,10 +72,17 @@ void test() {
     assert(m2 == m);
     assert(m2.key_comp() == C(3));
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_alloc() {
+  int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+  int expected[] = {1, 2, 3};
+
   {
     // flat_set(InputIterator , InputIterator, const Allocator&)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     auto m   = M(ar, ar + 9, A1(5));
     assert(std::ranges::equal(m, expected));
     assert(std::move(m).extract().get_allocator() == A1(5));
@@ -106,7 +91,7 @@ void test() {
     // flat_set(InputIterator , InputIterator, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     M m      = {ar, ar + 9, A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
     assert(std::move(m).extract().get_allocator() == A1(5));
@@ -115,7 +100,7 @@ void test() {
     // flat_set(InputIterator , InputIterator, const key_compare&, const Allocator&)
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, C, std::vector<int, A1>>;
+    using M  = std::flat_set<int, C, KeyContainer<int, A1>>;
     auto m   = M(ar, ar + 9, C(3), A1(5));
     assert(std::ranges::equal(m, expected));
     assert(m.key_comp() == C(3));
@@ -125,7 +110,7 @@ void test() {
     // flat_set(InputIterator , InputIterator, const key_compare&, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::deque<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     M m      = {ar, ar + 9, {}, A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
     LIBCPP_ASSERT(std::ranges::equal(m, expected));
@@ -133,8 +118,53 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_set<int, C, V1>;
+    using M2    = std::flat_set<int, C, V2>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const A1&>);
+    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const C&, const A1&>);
+  }
+
+  test<std::vector<int>>();
+  test<MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+
+  test_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque<int>>();
+    test_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp
index e26f8c8f25ce1..fcc15edc2935b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp
@@ -25,11 +25,12 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     using C = test_less<int>;
     using A = test_allocator<int>;
-    using M = std::flat_set<int, C, std::deque<int, A>>;
+    using M = std::flat_set<int, C, KeyContainer<int, A>>;
     M mo    = M({1, 2, 3}, C(5), A(7));
     M m     = std::move(mo);
     assert((m == M{1, 2, 3}));
@@ -43,7 +44,7 @@ void test() {
   {
     using C = test_less<int>;
     using A = min_allocator<int>;
-    using M = std::flat_set<int, C, std::vector<int, A>>;
+    using M = std::flat_set<int, C, KeyContainer<int, A>>;
     M mo    = M({1, 2, 3}, C(5), A());
     M m     = std::move(mo);
     assert((m == M{1, 2, 3}));
@@ -54,9 +55,9 @@ void test() {
     assert(mo.key_comp() == C(5));
     assert(std::move(mo).extract().get_allocator() == A());
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // A moved-from flat_set maintains its class invariant in the presence of moved-from comparators.
-    using M = std::flat_set<int, std::function<bool(int, int)>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int>>;
     M mo    = M({1, 2, 3}, std::less<int>());
     M m     = std::move(mo);
     assert(m.size() == 3);
@@ -86,47 +87,35 @@ struct ThrowingMoveAllocator {
   using value_type                                    = T;
   explicit ThrowingMoveAllocator()                    = default;
   ThrowingMoveAllocator(const ThrowingMoveAllocator&) = default;
-  ThrowingMoveAllocator(ThrowingMoveAllocator&&) noexcept(false) {}
-  T* allocate(std::ptrdiff_t n) { return std::allocator<T>().allocate(n); }
-  void deallocate(T* p, std::ptrdiff_t n) { return std::allocator<T>().deallocate(p, n); }
+  constexpr ThrowingMoveAllocator(ThrowingMoveAllocator&&) noexcept(false) {}
+  constexpr T* allocate(std::ptrdiff_t n) { return std::allocator<T>().allocate(n); }
+  constexpr void deallocate(T* p, std::ptrdiff_t n) { return std::allocator<T>().deallocate(p, n); }
   friend bool operator==(ThrowingMoveAllocator, ThrowingMoveAllocator) = default;
 };
 
 struct ThrowingMoveComp {
   ThrowingMoveComp() = default;
-  ThrowingMoveComp(const ThrowingMoveComp&) noexcept(true) {}
-  ThrowingMoveComp(ThrowingMoveComp&&) noexcept(false) {}
-  bool operator()(const auto&, const auto&) const { return false; }
-};
-
-struct MoveSensitiveComp {
-  MoveSensitiveComp() noexcept(false)                  = default;
-  MoveSensitiveComp(const MoveSensitiveComp&) noexcept = default;
-  MoveSensitiveComp(MoveSensitiveComp&& rhs) { rhs.is_moved_from_ = true; }
-  MoveSensitiveComp& operator=(const MoveSensitiveComp&) noexcept(false) = default;
-  MoveSensitiveComp& operator=(MoveSensitiveComp&& rhs) {
-    rhs.is_moved_from_ = true;
-    return *this;
-  }
-  bool operator()(const auto&, const auto&) const { return false; }
-  bool is_moved_from_ = false;
+  constexpr ThrowingMoveComp(const ThrowingMoveComp&) noexcept(true) {}
+  constexpr ThrowingMoveComp(ThrowingMoveComp&&) noexcept(false) {}
+  constexpr bool operator()(const auto&, const auto&) const { return false; }
 };
 
-void test_move_noexcept() {
+template <template <class...> class KeyContainer>
+constexpr void test_move_noexcept() {
   {
-    using C = std::flat_set<int>;
+    using C = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
     C c;
     C d = std::move(c);
   }
   {
-    using C = std::flat_set<int, std::less<int>, std::deque<int, test_allocator<int>>>;
+    using C = std::flat_set<int, std::less<int>, KeyContainer<int, test_allocator<int>>>;
     LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
     C c;
     C d = std::move(c);
   }
 #if _LIBCPP_VERSION
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // Container fails to be nothrow-move-constructible; this relies on libc++'s support for non-nothrow-copyable allocators
     using C = std::flat_set<int, std::less<int>, std::deque<int, ThrowingMoveAllocator<int>>>;
     static_assert(!std::is_nothrow_move_constructible_v<std::deque<int, ThrowingMoveAllocator<int>>>);
@@ -137,13 +126,26 @@ void test_move_noexcept() {
 #endif // _LIBCPP_VERSION
   {
     // Comparator fails to be nothrow-move-constructible
-    using C = std::flat_set<int, ThrowingMoveComp>;
+    using C = std::flat_set<int, ThrowingMoveComp, KeyContainer<int>>;
     static_assert(!std::is_nothrow_move_constructible_v<C>);
     C c;
     C d = std::move(c);
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+  test_move_noexcept<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque>();
+    test_move_noexcept<std::deque>();
+  }
+  return true;
+}
+
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
 static int countdown = 0;
 
@@ -179,10 +181,12 @@ void test_move_exception() {
 
 int main(int, char**) {
   test();
-  test_move_noexcept();
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   test_move_exception();
 #endif
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp
index 489b6ff36324b..9e88cbb002da5 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp
@@ -24,28 +24,13 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, M1&&, const A1&>);
-    static_assert(std::is_constructible_v<M2, M2&&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, M1&&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, M2&&, const A1&>);
-  }
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     int expected[] = {1, 2, 3};
     using C        = test_less<int>;
     using A        = test_allocator<int>;
-    using M        = std::flat_set<int, C, std::deque<int, A>>;
+    using M        = std::flat_set<int, C, KeyContainer<int, A>>;
     auto mo        = M(expected, expected + 3, C(5), A(7));
     auto m         = M(std::move(mo), A(3));
 
@@ -72,8 +57,38 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, M1&&, const A1&>);
+    static_assert(std::is_constructible_v<M2, M2&&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, M1&&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, M2&&, const A1&>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp
index 0e0ab0aa135f9..1c3affe05d10f 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp
@@ -30,9 +30,9 @@
 struct MoveNegates {
   int value_    = 0;
   MoveNegates() = default;
-  MoveNegates(int v) : value_(v) {}
-  MoveNegates(MoveNegates&& rhs) : value_(rhs.value_) { rhs.value_ = -rhs.value_; }
-  MoveNegates& operator=(MoveNegates&& rhs) {
+  constexpr MoveNegates(int v) : value_(v) {}
+  constexpr MoveNegates(MoveNegates&& rhs) : value_(rhs.value_) { rhs.value_ = -rhs.value_; }
+  constexpr MoveNegates& operator=(MoveNegates&& rhs) {
     value_     = rhs.value_;
     rhs.value_ = -rhs.value_;
     return *this;
@@ -44,9 +44,9 @@ struct MoveNegates {
 struct MoveClears {
   int value_   = 0;
   MoveClears() = default;
-  MoveClears(int v) : value_(v) {}
-  MoveClears(MoveClears&& rhs) : value_(rhs.value_) { rhs.value_ = 0; }
-  MoveClears& operator=(MoveClears&& rhs) {
+  constexpr MoveClears(int v) : value_(v) {}
+  constexpr MoveClears(MoveClears&& rhs) : value_(rhs.value_) { rhs.value_ = 0; }
+  constexpr MoveClears& operator=(MoveClears&& rhs) {
     value_     = rhs.value_;
     rhs.value_ = 0;
     return *this;
@@ -68,11 +68,12 @@ struct MoveAssignThrows : std::vector<int> {
 };
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-void test_move_assign_clears() {
+template <template <class...> class KeyContainer>
+constexpr void test_move_assign_clears() {
   // Preserves the class invariant for the moved-from flat_set.
   {
     const int expected[] = {1, 2, 3, 4, 5, 6, 7, 8};
-    using M              = std::flat_set<MoveNegates, std::less<MoveNegates>>;
+    using M              = std::flat_set<MoveNegates, std::less<MoveNegates>, KeyContainer<MoveNegates>>;
     M m                  = M(expected, expected + 8);
     M m2                 = M(expected, expected + 3);
 
@@ -89,7 +90,7 @@ void test_move_assign_clears() {
   }
   {
     const int expected[] = {1, 2, 3, 4, 5, 6, 7, 8};
-    using M              = std::flat_set<MoveClears, std::less<MoveClears>>;
+    using M              = std::flat_set<MoveClears, std::less<MoveClears>, KeyContainer<MoveClears>>;
     M m                  = M(expected, expected + 8);
     M m2                 = M(expected, expected + 3);
 
@@ -106,7 +107,7 @@ void test_move_assign_clears() {
   }
   {
     // moved-from object maintains invariant if one of underlying container does not clear after move
-    using M = std::flat_set<int, std::less<>, std::vector<int>>;
+    using M = std::flat_set<int, std::less<>, CopyOnlyVector<int>>;
     M m1    = M({1, 2, 3});
     M m2    = M({1, 2});
     m2      = std::move(m1);
@@ -115,7 +116,7 @@ void test_move_assign_clears() {
     LIBCPP_ASSERT(m1.empty());
   }
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     using M = std::flat_set<int, std::less<>, MoveAssignThrows>;
     M m1    = {1, 2, 3};
     M m2    = {1, 2};
@@ -189,11 +190,12 @@ void test_move_assign_no_except() {
   }
 }
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     using C                           = test_less<int>;
     using A1                          = test_allocator<int>;
-    using M                           = std::flat_set<int, C, std::vector<int, A1>>;
+    using M                           = std::flat_set<int, C, KeyContainer<int, A1>>;
     M mo                              = M({1, 2, 3}, C(5), A1(7));
     M m                               = M({}, C(3), A1(7));
     std::same_as<M&> decltype(auto) r = m = std::move(mo);
@@ -207,7 +209,7 @@ void test() {
   {
     using C                           = test_less<int>;
     using A1                          = other_allocator<int>;
-    using M                           = std::flat_set<int, C, std::deque<int, A1>>;
+    using M                           = std::flat_set<int, C, KeyContainer<int, A1>>;
     M mo                              = M({4, 5}, C(5), A1(7));
     M m                               = M({1, 2, 3, 4}, C(3), A1(7));
     std::same_as<M&> decltype(auto) r = m = std::move(mo);
@@ -220,7 +222,7 @@ void test() {
   }
   {
     using A                           = min_allocator<int>;
-    using M                           = std::flat_set<int, std::greater<int>, std::vector<int, A>>;
+    using M                           = std::flat_set<int, std::greater<int>, KeyContainer<int, A>>;
     M mo                              = M({5, 4, 3}, A());
     M m                               = M({4, 3, 2, 1}, A());
     std::same_as<M&> decltype(auto) r = m = std::move(mo);
@@ -232,10 +234,27 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+  test_move_assign_clears<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque>();
+    test_move_assign_clears<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
-  test_move_assign_clears();
   test_move_assign_no_except();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp
index bd7b5c12432e9..031c035bf5bf8 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp
@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
@@ -56,35 +57,14 @@ static_assert(
     !std::
         is_constructible_v<Set, std::from_range_t, RangeOf<std::pair<int, int>>, std::less<int>, std::allocator<int>>);
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A1&>);
-  }
-
+template <class KeyContainer>
+constexpr void test() {
   int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
   int expected[] = {1, 2, 3};
   {
     // flat_set(from_range_t, R&&)
     // input_range && !common
-    using M    = std::flat_set<int>;
+    using M    = std::flat_set<int, std::less<int>, KeyContainer>;
     using Iter = cpp20_input_iterator<const int*>;
     using Sent = sentinel_wrapper<Iter>;
     using R    = std::ranges::subrange<Iter, Sent>;
@@ -99,17 +79,17 @@ void test() {
   {
     // flat_set(from_range_t, R&&)
     // greater
-    using M    = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M    = std::flat_set<int, std::greater<int>, KeyContainer>;
     using Iter = cpp20_input_iterator<const int*>;
     using Sent = sentinel_wrapper<Iter>;
     using R    = std::ranges::subrange<Iter, Sent>;
     auto m     = M(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
-    assert(std::ranges::equal(m, std::deque<int, min_allocator<int>>{3, 2, 1}));
+    assert(std::ranges::equal(m, KeyContainer{3, 2, 1}));
   }
   {
     // flat_set(from_range_t, R&&)
     // contiguous range
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
     using R = std::ranges::subrange<const int*>;
     auto m  = M(std::from_range, R(ar, ar + 9));
     assert(std::ranges::equal(m, expected));
@@ -117,7 +97,7 @@ void test() {
   {
     // flat_set(from_range_t, R&&, const key_compare&)
     using C = test_less<int>;
-    using M = std::flat_set<int, C, std::vector<int>>;
+    using M = std::flat_set<int, C, KeyContainer>;
     using R = std::ranges::subrange<const int*>;
     auto m  = M(std::from_range, R(ar, ar + 9), C(3));
     assert(std::ranges::equal(m, expected));
@@ -128,10 +108,16 @@ void test() {
     assert(m2 == m);
     assert(m2.key_comp() == C(3));
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_alloc() {
+  int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+  int expected[] = {1, 2, 3};
   {
     // flat_set(from_range_t, R&&, const Allocator&)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     auto m   = M(std::from_range, R(ar, ar + 9), A1(5));
     assert(std::ranges::equal(m, expected));
@@ -141,7 +127,7 @@ void test() {
     // flat_set(from_range_t, R&&, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::deque<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     M m      = {std::from_range, R(ar, ar + 9), A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
@@ -151,7 +137,7 @@ void test() {
     // flat_set(from_range_t, R&&, const key_compare&, const Allocator&)
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, C, std::vector<int, A1>>;
+    using M  = std::flat_set<int, C, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     auto m   = M(std::from_range, R(ar, ar + 9), C(3), A1(5));
     assert(std::ranges::equal(m, expected));
@@ -162,7 +148,7 @@ void test() {
     // flat_set(from_range_t, R&&, const key_compare&, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::deque<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     M m      = {std::from_range, R(ar, ar + 9), {}, A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
@@ -170,8 +156,51 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A1&>);
+  }
+
+  test<std::vector<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+  test<MinSequenceContainer<int>>();
+
+  test_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque<int>>();
+    test_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp
index 873ff32b62936..5cf6d4c23f4e9 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp
@@ -30,33 +30,13 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A1&>);
-  }
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // flat_set(sorted_unique_t, container_type)
-    using M             = std::flat_set<int>;
-    std::vector<int> ks = {1, 2, 4, 10};
-    auto ks2            = ks;
+    using M              = std::flat_set<int, std::less<int>, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 2, 4, 10};
+    auto ks2             = ks;
 
     auto m = M(std::sorted_unique, ks);
     assert((m == M{1, 2, 4, 10}));
@@ -71,7 +51,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, container_type)
     // non-default container, comparator and allocator type
-    using Ks = std::deque<int, min_allocator<int>>;
+    using Ks = KeyContainer<int, min_allocator<int>>;
     using M  = std::flat_set<int, std::greater<int>, Ks>;
     Ks ks    = {10, 4, 2, 1};
     auto m   = M(std::sorted_unique, ks);
@@ -84,8 +64,8 @@ void test() {
     // flat_set(sorted_unique_t, container_type)
     // allocator copied into the containers
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 2, 4, 10}, A(4));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 2, 4, 10}, A(4));
     auto m  = M(std::sorted_unique, std::move(ks));
     assert(ks.empty()); // it was moved-from
     assert((m == M{1, 2, 4, 10}));
@@ -93,9 +73,9 @@ void test() {
   }
   {
     // flat_set(sorted_unique_t, container_type ,  key_compare)
-    using C             = test_less<int>;
-    using M             = std::flat_set<int, C>;
-    std::vector<int> ks = {1, 2, 4, 10};
+    using C              = test_less<int>;
+    using M              = std::flat_set<int, C, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 2, 4, 10};
 
     auto m = M(std::sorted_unique, ks, C(4));
     assert((m == M{1, 2, 4, 10}));
@@ -108,11 +88,11 @@ void test() {
   }
   {
     // flat_set(sorted_unique_t, container_type , key_compare, const Allocator&)
-    using C                = test_less<int>;
-    using A                = test_allocator<int>;
-    using M                = std::flat_set<int, C, std::vector<int, A>>;
-    std::vector<int, A> ks = {1, 2, 4, 10};
-    auto m                 = M(std::sorted_unique, ks, C(4), A(5));
+    using C                 = test_less<int>;
+    using A                 = test_allocator<int>;
+    using M                 = std::flat_set<int, C, KeyContainer<int, A>>;
+    KeyContainer<int, A> ks = {1, 2, 4, 10};
+    auto m                  = M(std::sorted_unique, ks, C(4), A(5));
     assert((m == M{1, 2, 4, 10}));
     assert(m.key_comp() == C(4));
     assert(M(m).extract().get_allocator() == A(5));
@@ -126,8 +106,8 @@ void test() {
   {
     // flat_set(sorted_unique_t, container_type , const Allocator&)
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 2, 4, 10}, A(4));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 2, 4, 10}, A(4));
     auto m  = M(std::sorted_unique, ks, A(6)); // replaces the allocators
     assert(!ks.empty());                       // it was an lvalue above
     assert((m == M{1, 2, 4, 10}));
@@ -140,8 +120,43 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A1&>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp
index a8dac35aefee8..00cebb60950fc 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp
@@ -31,60 +31,16 @@
 #include "../../../test_compare.h"
 
 template <class T>
-std::initializer_list<T> il = {1, 2, 4, 5};
+constexpr std::initializer_list<T> il = {1, 2, 4, 5};
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   const auto il1 = il<int>;
   const auto il2 = il<short>;
 
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    using IL = std::initializer_list<int>;
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A1&>);
-  }
-  {
-    // initializer_list<value_type> needs to match exactly
-    using M = std::flat_set<int>;
-    using C = typename M::key_compare;
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>>);
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C>);
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C, std::allocator<int>>);
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
-  }
-
   {
     // flat_set(sorted_unique_t, initializer_list<value_type>);
-    using M       = std::flat_set<int>;
+    using M       = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     auto m        = M(std::sorted_unique, il1);
     auto expected = M{1, 2, 4, 5};
     assert(m == expected);
@@ -93,9 +49,9 @@ void test() {
     M m2 = {std::sorted_unique, il1};
     assert(m2 == m);
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&);
-    using M = std::flat_set<int, std::function<bool(int, int)>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int>>;
     auto m  = M(std::sorted_unique, il1, std::less<int>());
     assert(m == M({1, 2, 4, 5}, std::less<>()));
     assert(m.key_comp()(1, 2) == true);
@@ -107,7 +63,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&);
     // greater
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer<int, min_allocator<int>>>;
     std::initializer_list<int> il4{5, 4, 2, 1};
     auto m = M(std::sorted_unique, il4, std::greater<int>());
     assert((m == M{5, 4, 2, 1}));
@@ -115,7 +71,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, initializer_list<value_type>,  const Allocator&)
     using A1      = test_allocator<short>;
-    using M       = std::flat_set<short, std::less<int>, std::deque<short, A1>>;
+    using M       = std::flat_set<short, std::less<int>, KeyContainer<short, A1>>;
     auto m        = M(std::sorted_unique, il2, A1(5));
     auto expected = M{1, 2, 4, 5};
     assert(m == expected);
@@ -130,7 +86,7 @@ void test() {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&, const Allocator&);
     using C  = test_less<int>;
     using A1 = test_allocator<short>;
-    using M  = std::flat_set<short, C, std::vector<short, A1>>;
+    using M  = std::flat_set<short, C, KeyContainer<short, A1>>;
     auto m   = M(std::sorted_unique, il2, C(3), A1(5));
     assert((m == M{1, 2, 4, 5}));
     assert(m.key_comp() == C(3));
@@ -140,15 +96,73 @@ void test() {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&, const Allocator&);
     // explicit(false)
     using A1 = test_allocator<short>;
-    using M  = std::flat_set<short, std::less<int>, std::deque<short, A1>>;
+    using M  = std::flat_set<short, std::less<int>, KeyContainer<short, A1>>;
     M m      = {std::sorted_unique, il2, {}, A1(5)}; // implicit ctor
     assert((m == M{1, 2, 4, 5}));
     assert(std::move(m).extract().get_allocator() == A1(5));
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    using IL = std::initializer_list<int>;
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A1&>);
+  }
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_set<int>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C, std::allocator<int>>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp
index b184ee9c3f5ab..7ffb09e7347c4 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp
@@ -22,40 +22,20 @@
 #include <functional>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C     = test_less<int>;
-    using A1    = test_allocator<int>;
-    using A2    = other_allocator<int>;
-    using V1    = std::vector<int, A1>;
-    using V2    = std::vector<int, A2>;
-    using M1    = std::flat_set<int, C, V1>;
-    using M2    = std::flat_set<int, C, V2>;
-    using Iter1 = typename M1::iterator;
-    using Iter2 = typename M2::iterator;
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A1&>);
-  }
+template <class KeyContainer>
+constexpr void test() {
   {
     // flat_set(sorted_unique_t, InputIterator, InputIterator);
     // cpp17_input_iterator
-    using M  = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
+
     int ar[] = {1, 2, 4, 5};
     auto m   = M(std::sorted_unique, cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + 4));
     auto expected = M{1, 2, 4, 5};
@@ -69,16 +49,16 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator);
     // contiguous iterator
     using C       = test_less<int>;
-    using M       = std::flat_set<int, C, std::vector<int, min_allocator<int>>>;
+    using M       = std::flat_set<int, C, KeyContainer>;
     int ar[]      = {1, 2, 4, 5};
     auto m        = M(std::sorted_unique, ar, ar + 4);
     auto expected = M{1, 2, 4, 5};
     assert(m == expected);
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
     // cpp_17_input_iterator
-    using M  = std::flat_set<int, std::function<bool(int, int)>>;
+    using M  = std::flat_set<int, std::function<bool(int, int)>, KeyContainer>;
     int ar[] = {1, 2, 4, 5};
     auto m   = M(std::sorted_unique,
                cpp17_input_iterator<const int*>(ar),
@@ -97,7 +77,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
     // greater
-    using M  = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M  = std::flat_set<int, std::greater<int>, KeyContainer>;
     int ar[] = {5, 4, 2, 1};
     auto m   = M(std::sorted_unique,
                cpp17_input_iterator<const int*>(ar),
@@ -109,16 +89,20 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
     // contiguous iterator
     using C   = test_less<int>;
-    using M   = std::flat_set<int, C, std::vector<int, min_allocator<int>>>;
+    using M   = std::flat_set<int, C, KeyContainer>;
     int ar[1] = {42};
     auto m    = M(std::sorted_unique, ar, ar, C(5));
     assert(m.empty());
     assert(m.key_comp() == C(5));
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_alloc() {
   {
     // flat_set(sorted_unique_t, InputIterator , InputIterator, const Allocator&)
     using A1      = test_allocator<int>;
-    using M       = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M       = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     int ar[]      = {1, 2, 4, 5};
     auto m        = M(std::sorted_unique, ar, ar + 4, A1(5));
     auto expected = M{1, 2, 4, 5};
@@ -134,7 +118,7 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, C, std::deque<int, A1>>;
+    using M  = std::flat_set<int, C, KeyContainer<int, A1>>;
     int ar[] = {1, 2, 4, 5};
     auto m   = M(std::sorted_unique, ar, ar + 4, C(3), A1(5));
     assert((m == M{1, 2, 4, 5}));
@@ -145,7 +129,7 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
     // explicit(false)
     using A1 = test_allocator<short>;
-    using M  = std::flat_set<short, std::less<int>, std::deque<short, A1>>;
+    using M  = std::flat_set<short, std::less<int>, KeyContainer<short, A1>>;
     int ar[] = {1, 2, 4, 5};
     M m      = {std::sorted_unique, ar, ar + 4, {}, A1(5)}; // implicit ctor
     assert((m == M{1, 2, 4, 5}));
@@ -153,8 +137,52 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_set<int, C, V1>;
+    using M2    = std::flat_set<int, C, V2>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A1&>);
+  }
+
+  test<std::vector<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+  test<MinSequenceContainer<int>>();
+  test_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque<int>>();
+    test_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp
index d7d7a98f49d77..924fbdab5de98 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp
@@ -32,7 +32,7 @@ static_assert(HasStdErase<std::vector<int>>);
 static_assert(!HasStdErase<std::flat_set<int>>);
 
 template <class M>
-M make(std::initializer_list<int> vals) {
+constexpr M make(std::initializer_list<int> vals) {
   M ret;
   for (int v : vals)
     ret.emplace(v);
@@ -40,8 +40,8 @@ M make(std::initializer_list<int> vals) {
 }
 
 template <class M, class Pred>
-void test0(
-    std::initializer_list<int> vals, Pred p, std::initializer_list<int> expected, std::size_t expected_erased_count) {
+constexpr void
+test0(std::initializer_list<int> vals, Pred p, std::initializer_list<int> expected, std::size_t expected_erased_count) {
   M s = make<M>(vals);
   ASSERT_SAME_TYPE(typename M::size_type, decltype(std::erase_if(s, p)));
   assert(expected_erased_count == std::erase_if(s, p));
@@ -50,11 +50,11 @@ void test0(
 
 struct NotBool {
   bool b;
-  explicit operator bool() const { return b; }
+  constexpr explicit operator bool() const { return b; }
 };
 
 template <class S>
-void test_one() {
+constexpr void test_one() {
   // Test all the plausible signatures for this predicate.
   auto is1        = [](typename S::const_reference v) { return v == 1; };
   auto is2        = [](typename S::value_type v) { return v == 2; };
@@ -84,18 +84,28 @@ void test_one() {
   test0<S>({1, 2, 3}, nonBoolIs1, {2, 3}, 1);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::flat_set<int>>();
   test_one<std::flat_set<int, std::less<int>, std::vector<int, min_allocator<int>>>>();
   test_one<std::flat_set<int, std::greater<int>, std::vector<int, test_allocator<int>>>>();
-  test_one<std::flat_set<int, std::less<int>, std::deque<int, min_allocator<int>>>>();
-  test_one<std::flat_set<int, std::greater<int>, std::deque<int, test_allocator<int>>>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_one<std::flat_set<int, std::less<int>, std::deque<int, min_allocator<int>>>>();
+    test_one<std::flat_set<int, std::greater<int>, std::deque<int, test_allocator<int>>>>();
+  }
   test_one<std::flat_set<long>>();
   test_one<std::flat_set<double>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp
index 846bcff63d773..8aafa97818bc7 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp
@@ -30,7 +30,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -67,9 +67,12 @@ void test_one() {
   assert(i == m.begin());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -88,10 +91,15 @@ void test() {
     assert(!(ii1 != cii));
     assert(!(cii != ii1));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp
index ad1bc8facc92e..fbc519cba98de 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using I   = M::iterator;
@@ -141,15 +141,23 @@ void test_one() {
   assert(cri2 <=> cri1 == std::strong_ordering::greater);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp
index 4c2482cd0f3de..3725333f1f109 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp
@@ -25,19 +25,22 @@
 
 #include <iterator>
 
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
 #include "test_macros.h"
 
-void test() {
+template <class KeyContainer>
+constexpr void test_one() {
   {
-    using M     = std::flat_set<int, std::less<int>, std::deque<int>>;
+    using M     = std::flat_set<int, std::less<int>, KeyContainer>;
     M m         = {1, 2, 3, 4};
     const M& cm = m;
-    ASSERT_SAME_TYPE(decltype(m.rbegin()), M::reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(m.crbegin()), M::const_reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(cm.rbegin()), M::const_reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(m.rend()), M::reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(m.crend()), M::const_reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(cm.rend()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.rbegin()), typename M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crbegin()), typename M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rbegin()), typename M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.rend()), typename M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crend()), typename M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rend()), typename M::const_reverse_iterator);
     static_assert(noexcept(m.rbegin()));
     static_assert(noexcept(cm.rbegin()));
     static_assert(noexcept(m.crbegin()));
@@ -49,12 +52,12 @@ void test() {
     assert(std::distance(cm.rbegin(), cm.rend()) == 4);
     assert(std::distance(m.crbegin(), m.crend()) == 4);
     assert(std::distance(cm.crbegin(), cm.crend()) == 4);
-    M::reverse_iterator i; // default-construct
+    typename M::reverse_iterator i; // default-construct
     ASSERT_SAME_TYPE(decltype(*i), const int&);
-    i                           = m.rbegin(); // move-assignment
-    M::const_reverse_iterator k = i;          // converting constructor
-    assert(i == k);                           // comparison
-    for (int j = 4; j >= 1; --j, ++i) {       // pre-increment
+    i                                    = m.rbegin(); // move-assignment
+    typename M::const_reverse_iterator k = i;          // converting constructor
+    assert(i == k);                                    // comparison
+    for (int j = 4; j >= 1; --j, ++i) {                // pre-increment
       assert(*i == j);
     }
     assert(i == m.rend());
@@ -65,11 +68,11 @@ void test() {
     assert(i == m.rbegin());
   }
   {
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
     // N3644 testing
-    using C = std::flat_set<int>;
-    C::reverse_iterator ii1{}, ii2{};
-    C::reverse_iterator ii4 = ii1;
-    C::const_reverse_iterator cii{};
+    typename M::reverse_iterator ii1{}, ii2{};
+    typename M::reverse_iterator ii4 = ii1;
+    typename M::const_reverse_iterator cii{};
     assert(ii1 == ii2);
     assert(ii1 == ii4);
     assert(!(ii1 != ii2));
@@ -81,8 +84,23 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test_one<std::vector<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
+  test_one<MinSequenceContainer<int>>();
+  test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp
index 5ecdc429b3c58..f92609fc34550 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp
@@ -38,7 +38,7 @@ static_assert(NoExceptClear<std::flat_set<int, std::less<int>, ThrowOnMoveContai
 #endif
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -58,17 +58,24 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp
index 79800e894afbf..de8c05c48c244 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp
@@ -28,7 +28,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = std::pair<typename M::iterator, bool>;
@@ -98,7 +98,7 @@ void test_one() {
 }
 
 template <class KeyContainer>
-void test_emplaceable() {
+constexpr void test_emplaceable() {
   using M = std::flat_set<Emplaceable, std::less<Emplaceable>, KeyContainer>;
   using R = std::pair<typename M::iterator, bool>;
 
@@ -121,16 +121,24 @@ void test_emplaceable() {
   assert(*m.begin() == Emplaceable(1, 3.5));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
   test_emplaceable<std::vector<Emplaceable>>();
-  test_emplaceable<std::deque<Emplaceable>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_emplaceable<std::deque<Emplaceable>>();
   test_emplaceable<MinSequenceContainer<Emplaceable>>();
   test_emplaceable<std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -141,6 +149,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp
index b3bd8adf0c35d..0b5aab1b1c61e 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp
@@ -27,7 +27,7 @@
 #include "../helpers.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = M::iterator;
@@ -114,7 +114,7 @@ void test_one() {
 }
 
 template <class KeyContainer>
-void test_emplaceable() {
+constexpr void test_emplaceable() {
   using M = std::flat_set<Emplaceable, std::less<Emplaceable>, KeyContainer>;
   using R = M::iterator;
 
@@ -134,16 +134,24 @@ void test_emplaceable() {
   assert(*m.begin() == Emplaceable(1, 3.5));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
   test_emplaceable<std::vector<Emplaceable>>();
-  test_emplaceable<std::vector<Emplaceable>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_emplaceable<std::deque<Emplaceable>>();
   test_emplaceable<MinSequenceContainer<Emplaceable>>();
   test_emplaceable<std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -154,6 +162,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp
index 27c6fadb5341e..c908f5d63b879 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp
@@ -27,7 +27,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using I   = M::iterator;
@@ -94,11 +94,16 @@ void test_one() {
   assert(i8 == m.end());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -109,6 +114,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp
index f44a8864bdbc8..bc53e306851b8 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp
@@ -26,7 +26,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using I   = M::iterator;
@@ -78,11 +78,16 @@ void test_one() {
   assert(i5 == m.end());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -93,6 +98,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp
index c50b219895b17..c5d4cd1052b1f 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp
@@ -26,7 +26,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer, class Compare = std::less<>>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, Compare, KeyContainer>;
 
   auto make = [](std::initializer_list<int> il) {
@@ -74,12 +74,17 @@ void test_one() {
   assert(m.empty());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<int>, std::greater<>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -95,6 +100,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
index f4b373364cab2..b6701e36a55a9 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
@@ -38,10 +38,10 @@ static_assert(!CanErase<const NonTransparentSet>);
 
 template <class Key, class It>
 struct HeterogeneousKey {
-  explicit HeterogeneousKey(Key key, It it) : key_(key), it_(it) {}
-  operator It() && { return it_; }
-  auto operator<=>(Key key) const { return key_ <=> key; }
-  friend bool operator<(const HeterogeneousKey&, const HeterogeneousKey&) {
+  constexpr explicit HeterogeneousKey(Key key, It it) : key_(key), it_(it) {}
+  constexpr operator It() && { return it_; }
+  constexpr auto operator<=>(Key key) const { return key_ <=> key; }
+  constexpr friend bool operator<(const HeterogeneousKey&, const HeterogeneousKey&) {
     assert(false);
     return false;
   }
@@ -50,7 +50,7 @@ struct HeterogeneousKey {
 };
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -70,7 +70,7 @@ void test_one() {
 }
 
 template <class KeyContainer>
-void test_transparent_comparator() {
+constexpr void test_transparent_comparator() {
   using M = std::flat_set<std::string, TransparentComparator, KeyContainer>;
   {
     M m = {"alpha", "beta", "epsilon", "eta", "gamma"};
@@ -95,14 +95,20 @@ void test_transparent_comparator() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
   test_transparent_comparator<std::vector<std::string>>();
-  test_transparent_comparator<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_transparent_comparator<std::deque<std::string>>();
   test_transparent_comparator<MinSequenceContainer<std::string>>();
   test_transparent_comparator<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -145,6 +151,8 @@ void test() {
     auto n = m.erase("beta");
     assert(n == 1);
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -159,6 +167,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp
index 12e63e0d23e1d..b554ef8154909 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp
@@ -33,7 +33,7 @@ static_assert(!CanExtract<std::flat_set<int> const&>);
 static_assert(!CanExtract<std::flat_set<int> const&&>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, std::less<int>, KeyContainer>;
   {
     M m = M({1, 2, 3});
@@ -55,9 +55,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -70,6 +73,8 @@ void test() {
     check_invariant(m);
     LIBCPP_ASSERT(m.empty());
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -97,6 +102,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp
index e0cb80f74462c..2e59818b48286 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = std::pair<typename M::iterator, bool>;
@@ -59,11 +59,16 @@ void test_one() {
   assert(*r.first == 3);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -78,6 +83,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp
index 992c96ecba1a2..ad6f379b091ef 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using V   = typename M::value_type;
@@ -67,11 +67,16 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -87,6 +92,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp
index 76b4d5f0eb7cc..710f83f78ecb3 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = typename M::iterator;
@@ -55,11 +55,16 @@ void test_one() {
   assert(*r == 3);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -75,6 +80,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
index 18bdb798de01f..af25de9b1af39 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
@@ -37,7 +37,7 @@ static_assert(!CanInsert<Set, int, int>);
 static_assert(!CanInsert<Set, cpp20_input_iterator<int*>, cpp20_input_iterator<int*>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, std::less<int>, KeyContainer>;
 
   int ar1[] = {
@@ -75,9 +75,12 @@ void test_one() {
   assert(m == expected2);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   {
@@ -86,6 +89,8 @@ void test() {
     m.insert(v.begin(), v.end());
     assert(std::ranges::equal(m, std::vector<int>{1, 2, 3, 4}));
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -96,6 +101,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp
index d29de98e4d3dd..a1fb10115bffa 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp
@@ -22,7 +22,7 @@
 #include "test_macros.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using V   = Key;
@@ -49,15 +49,22 @@ void test_one() {
   assert(*r == V(3));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<MoveOnly>>();
-  test_one<std::deque<int>>();
-  test_one<std::deque<MoveOnly>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_one<std::deque<int>>();
+    test_one<std::deque<MoveOnly>>();
+  }
   test_one<MinSequenceContainer<int>>();
   test_one<MinSequenceContainer<MoveOnly>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -73,6 +80,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp
index 966a696c096b9..a624f9161a985 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp
@@ -39,7 +39,7 @@ static_assert(!CanInsertRange<Set, std::ranges::subrange<std::pair<int, int>*>>)
 static_assert(!CanInsertRange<Set, std::ranges::subrange<std::pair<short, short>*>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
 
   {
@@ -65,7 +65,7 @@ void test_one() {
   {
     // The "uniquing" part uses the comparator, not operator==.
     struct ModTen {
-      bool operator()(int a, int b) const { return (a % 10) < (b % 10); }
+      constexpr bool operator()(int a, int b) const { return (a % 10) < (b % 10); }
     };
     using M  = std::flat_set<Key, ModTen, KeyContainer>;
     M m      = {21, 43, 15, 37};
@@ -83,9 +83,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   {
@@ -96,6 +99,8 @@ void test() {
     MoveOnly expected[] = {1, 3, 4, 5};
     assert(std::ranges::equal(m, expected));
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -106,6 +111,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp
index a179632d64211..b5272f2ae6e2b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp
@@ -25,7 +25,7 @@
 #include "../helpers.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
   using R   = std::pair<typename M::iterator, bool>;
@@ -57,15 +57,22 @@ void test_one() {
   assert(*r.first == V(3));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<MoveOnly>>();
-  test_one<std::deque<int>>();
-  test_one<std::deque<MoveOnly>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_one<std::deque<int>>();
+    test_one<std::deque<MoveOnly>>();
+  }
   test_one<MinSequenceContainer<int>>();
   test_one<MinSequenceContainer<MoveOnly>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -81,6 +88,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp
index fb0a8b049e72a..bc6476bfaaacb 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using V   = Key;
@@ -47,11 +47,16 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -67,6 +72,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp
index 6258c4dbfdcbb..5287dbec19294 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanInsert<Set, std::sorted_unique_t, int, int>);
 static_assert(!CanInsert<Set, std::sorted_unique_t, cpp20_input_iterator<int*>, cpp20_input_iterator<int*>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -60,11 +60,16 @@ void test_one() {
   assert(m == expected2);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -77,6 +82,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
index 4b5779a10ee7a..3d5c954c5e316 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
@@ -46,7 +46,7 @@ static_assert(!CanInsert<NonTransparentSet, NonConvertibleTransparent<int>>);
 static_assert(!CanInsert<NonTransparentSet, NonTransparentSetIter, NonConvertibleTransparent<int>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -105,9 +105,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -131,6 +134,8 @@ void test() {
     assert(!inserted);
     assert(iter == m.begin() + 1);
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -153,6 +158,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp
index 591def4c0f938..c15266676c61b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp
@@ -31,7 +31,7 @@ static_assert(CanReplace<Set, std::vector<int>>);
 static_assert(!CanReplace<Set, const std::vector<int>&>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -51,11 +51,16 @@ void test_one() {
   assert(std::ranges::equal(m, expected_keys));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -81,6 +86,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp
index 686fe93a593f5..6b4a65e4701b2 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp
@@ -38,7 +38,7 @@ static_assert(NoExceptAdlSwap<std::flat_set<int, std::less<int>, ThrowOnMoveCont
 #endif
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -84,15 +84,22 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
-  test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp
index 1eac55d768ce0..a3b605ee94cfb 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp
@@ -37,7 +37,7 @@ static_assert(NoExceptMemberSwap<std::flat_set<int, std::less<int>, ThrowOnMoveC
 #endif
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -82,15 +82,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp
index ba2c428c9e30e..b1c667a879cef 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp
@@ -21,7 +21,7 @@
 
 #include "test_macros.h"
 
-void test() {
+constexpr bool test() {
   {
     using M    = std::flat_set<int>;
     using Comp = std::less<int>; // the default
@@ -36,7 +36,7 @@ void test() {
     assert(vc(1, 2));
     assert(!vc(2, 1));
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     using Comp = std::function<bool(int, int)>;
     using M    = std::flat_set<int, Comp>;
     Comp comp  = std::greater<int>();
@@ -67,10 +67,15 @@ void test() {
     assert(vc(1, 2));
     assert(!vc(2, 1));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp
index e0416fc261ffc..cb2b3da891384 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -66,15 +66,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
index 91f49305cd71b..d7e45f6b706b1 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!CanContains<NonTransparentSet>);
 static_assert(!CanContains<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -60,9 +60,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -82,10 +85,15 @@ void test() {
     assert(m.contains("beta"));
     assert(!m.contains("eta2"));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp
index 51aec21ca212b..b2c646e3ecf22 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using S   = typename KeyContainer::size_type;
 
@@ -66,15 +66,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
index 84e7b9a18b83b..8686db3d400e5 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!CanCount<NonTransparentSet>);
 static_assert(!CanCount<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
   {
@@ -59,9 +59,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -81,10 +84,15 @@ void test() {
     assert(m.count("beta") == 1);
     assert(m.count("eta2") == 0);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp
index 7a8b19273289d..69e5f1b18cb28 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M  = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -74,15 +74,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
index 31081eb970397..e6667c363963a 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanEqualRange<NonTransparentSet>);
 static_assert(!CanEqualRange<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -87,9 +87,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -110,10 +113,15 @@ void test() {
     assert(first == m.begin() + 1);
     assert(last == m.begin() + 2);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp
index 953b8fe638547..daf9b70fdcc34 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp
@@ -25,7 +25,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<>, KeyContainer>;
   {
@@ -50,15 +50,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
index b8d454c58f54c..d952ba941896b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanFind<NonTransparentSet>);
 static_assert(!CanFind<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -77,9 +77,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -101,10 +104,15 @@ void test() {
     auto it2 = m.find("beta2");
     assert(it2 == m.end());
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp
index 32796b7a164f7..f5e6c7c06e462 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -66,15 +66,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
index 0c3e25f795f33..3b24e051e9991 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanLowerBound<NonTransparentSet>);
 static_assert(!CanLowerBound<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -83,9 +83,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -107,10 +110,15 @@ void test() {
     auto it2 = m.lower_bound("beta2");
     assert(it2 == m.begin() + 2);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp
index 6a967cb2371ba..c669746862dd5 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -67,15 +67,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
index 36f2a20946496..dda8858f2683a 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanUpperBound<NonTransparentSet>);
 static_assert(!CanUpperBound<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -83,9 +83,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -107,10 +110,15 @@ void test() {
     auto it2 = m.upper_bound("beta2");
     assert(it2 == m.begin() + 2);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h b/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h
index 4cc720311cf01..3f0dc403e4bc1 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h
@@ -20,7 +20,7 @@
 #include "test_macros.h"
 
 template <class... Args>
-void check_invariant(const std::flat_set<Args...>& m) {
+constexpr void check_invariant(const std::flat_set<Args...>& m) {
   assert(std::is_sorted(m.begin(), m.end(), m.key_comp()));
   auto key_equal = [&](const auto& x, const auto& y) {
     const auto& c = m.key_comp();
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp
index faf746861df30..8c47a63631efa 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp
@@ -15,6 +15,7 @@
 
 #include <flat_set>
 #include <vector>
+#include "test_macros.h"
 
 struct A {
   using Set = std::flat_set<A>;
@@ -25,12 +26,18 @@ struct A {
 };
 
 // Implement the operator< required in order to instantiate flat_set<A>
-bool operator<(A const& L, A const& R) { return L.data < R.data; }
+constexpr bool operator<(A const& L, A const& R) { return L.data < R.data; }
 
-void test() { A a; }
+constexpr bool test() {
+  A a;
+  return true;
+}
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp
index 3e7aecee77fdd..a8284389efd1b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp
@@ -31,7 +31,7 @@
 #include "test_container_comparisons.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
 
   {
@@ -69,9 +69,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -86,7 +89,7 @@ void test() {
   {
     // Comparisons use value_type's native operators, not the comparator
     struct StrongComp {
-      bool operator()(double a, double b) const { return std::strong_order(a, b) < 0; }
+      constexpr bool operator()(double a, double b) const { return std::strong_order(a, b) < 0; }
     };
     using C = std::flat_set<double, StrongComp>;
     C s1    = {1};
@@ -101,10 +104,15 @@ void test() {
     assert(s1 != s2);
     assert((s1 <=> s2) == std::partial_ordering::unordered);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat_helpers.h b/libcxx/test/std/containers/container.adaptors/flat_helpers.h
index 57c4595ccfeb0..45860bf15c644 100644
--- a/libcxx/test/std/containers/container.adaptors/flat_helpers.h
+++ b/libcxx/test/std/containers/container.adaptors/flat_helpers.h
@@ -9,6 +9,7 @@
 #ifndef TEST_STD_CONTAINERS_CONTAINER_ADAPTORS_FLAT_HELPERS_H
 #define TEST_STD_CONTAINERS_CONTAINER_ADAPTORS_FLAT_HELPERS_H
 
+#include <cstdint>
 #include <vector>
 
 #include "test_macros.h"
@@ -17,7 +18,7 @@ template <class T>
 struct CopyOnlyVector : std::vector<T> {
   using std::vector<T>::vector;
 
-  CopyOnlyVector(const CopyOnlyVector&) = default;
+  constexpr CopyOnlyVector(const CopyOnlyVector&) = default;
   constexpr CopyOnlyVector(CopyOnlyVector&& other) : CopyOnlyVector(other) {}
   constexpr CopyOnlyVector(CopyOnlyVector&& other, std::vector<T>::allocator_type alloc)
       : CopyOnlyVector(other, alloc) {}
@@ -30,14 +31,14 @@ template <class T>
 struct SillyReserveVector : std::vector<T> {
   using std::vector<T>::vector;
 
-  void reserve(size_t) { this->clear(); }
+  constexpr void reserve(std::size_t) { this->clear(); }
 };
 
 template <class T, bool ConvertibleToT = false>
 struct Transparent {
   T t;
 
-  TEST_CONSTEXPR explicit operator T() const
+  constexpr explicit operator T() const
     requires ConvertibleToT
   {
     return t;
@@ -58,10 +59,10 @@ struct TransparentComparator {
 
   bool* transparent_used  = nullptr;
   TransparentComparator() = default;
-  TEST_CONSTEXPR TransparentComparator(bool& used) : transparent_used(&used) {}
+  constexpr TransparentComparator(bool& used) : transparent_used(&used) {}
 
   template <class T, bool Convertible>
-  TEST_CONSTEXPR bool operator()(const T& t, const Transparent<T, Convertible>& transparent) const {
+  constexpr bool operator()(const T& t, const Transparent<T, Convertible>& transparent) const {
     if (transparent_used != nullptr) {
       *transparent_used = true;
     }
@@ -69,7 +70,7 @@ struct TransparentComparator {
   }
 
   template <class T, bool Convertible>
-  TEST_CONSTEXPR bool operator()(const Transparent<T, Convertible>& transparent, const T& t) const {
+  constexpr bool operator()(const Transparent<T, Convertible>& transparent, const T& t) const {
     if (transparent_used != nullptr) {
       *transparent_used = true;
     }
@@ -77,7 +78,7 @@ struct TransparentComparator {
   }
 
   template <class T>
-  TEST_CONSTEXPR bool operator()(const T& t1, const T& t2) const {
+  constexpr bool operator()(const T& t1, const T& t2) const {
     return t1 < t2;
   }
 };
diff --git a/libcxx/test/std/containers/sequences/array/array.creation/to_array.verify.cpp b/libcxx/test/std/containers/sequences/array/array.creation/to_array.verify.cpp
index 9e7b456bce18b..e3efef988f0f4 100644
--- a/libcxx/test/std/containers/sequences/array/array.creation/to_array.verify.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.creation/to_array.verify.cpp
@@ -28,14 +28,14 @@ int main(int, char**) {
   {
     MoveOnly mo[] = {MoveOnly{3}};
     // expected-error@array:* {{to_array requires copy constructible elements}}
-    // expected-error-re@array:* {{{{(call to implicitly-deleted copy constructor of 'MoveOnly')|(call to deleted constructor of 'MoveOnly')}}}}
+    // expected-error-re@array:* 1-2{{{{(call to implicitly-deleted copy constructor of 'MoveOnly')|(call to deleted constructor of 'MoveOnly')}}}}
     std::to_array(mo); // expected-note {{requested here}}
   }
 
   {
     const MoveOnly cmo[] = {MoveOnly{3}};
     // expected-error@array:* {{to_array requires move constructible elements}}
-    // expected-error-re@array:* {{{{(call to implicitly-deleted copy constructor of 'MoveOnly')|(call to deleted constructor of 'MoveOnly')}}}}
+    // expected-error-re@array:* 0-1{{{{(call to implicitly-deleted copy constructor of 'MoveOnly')|(call to deleted constructor of 'MoveOnly')}}}}
     std::to_array(std::move(cmo)); // expected-note {{requested here}}
   }
 
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
index d46ba6ffbe5ec..cd4d4d9a7c9da 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
@@ -43,6 +43,7 @@ void cant_construct_data_handle_type() {
   int data;
   std::mdspan<int, std::extents<int>, std::layout_right, convertible_accessor_but_not_handle<int>> m_nc(&data);
   // expected-error-re@*:* {{{{.*}}no matching constructor for initialization of {{.*}} (aka 'not_const_convertible_handle<const int>')}}
+  // expected-error@*:* 0-1{{no matching constructor for initialization of 'not_const_convertible_handle<const int>'}}
   // expected-error-re@*:* {{static assertion failed {{.*}}mdspan: incompatible data_handle_type for mdspan construction}}
   [[maybe_unused]] std::
       mdspan<const int, std::extents<int>, std::layout_right, convertible_accessor_but_not_handle<const int>> m_c(m_nc);
@@ -52,5 +53,6 @@ void mapping_constructible_despite_extents_compatibility() {
   int data;
   std::mdspan<int, std::extents<int>, always_convertible_layout> m(&data);
   // expected-error-re@*:* {{static assertion failed {{.*}}mdspan: incompatible extents for mdspan construction}}
+  // expected-error@*:* 0-1{{no matching constructor for initialization of 'std::extents<int, 5>'}}
   [[maybe_unused]] std::mdspan<int, std::extents<int, 5>, always_convertible_layout> m2(m);
 }
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp
index d7c4088f5be00..4e3afc17ad495 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp
@@ -15,7 +15,7 @@
 
 // This test requires the fix to https://github.com/llvm/llvm-project/issues/60509 in the dylib,
 // which landed in 5afb937d8a30445642ccaf33866ee4cdd0713222.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 #include <fstream>
 #include <cstddef>
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
index d468d7154dad5..88672851f7b6b 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
@@ -15,9 +15,10 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+#include <cassert>
 #include <iomanip>
 #include <istream>
-#include <cassert>
+#include <streambuf>
 
 #include "test_macros.h"
 #include "platform_support.h" // locale name macros
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp
index addf53fcdeb30..c6e3e43109a2f 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp
@@ -12,9 +12,11 @@
 
 // template <class charT> T9 get_time(struct tm* tmb, const charT* fmt);
 
+#include <cassert>
+#include <ctime>
 #include <iomanip>
 #include <istream>
-#include <cassert>
+#include <streambuf>
 
 #include "test_macros.h"
 #include "platform_support.h" // locale name macros
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp
index 7de93bef0f7b8..5b35b7d1cc81b 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp
@@ -12,9 +12,10 @@
 
 // template <class charT> T10 put_time(const struct tm* tmb, const charT* fmt);
 
+#include <cassert>
+#include <ctime>
 #include <iomanip>
 #include <ostream>
-#include <cassert>
 
 #include "test_macros.h"
 #include "platform_support.h" // locale name macros
diff --git a/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp b/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp
index 3b4354d0916c1..3b685950d36a6 100644
--- a/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp
@@ -16,7 +16,7 @@
 // The fix for bug 51497 and bug 51499 require and updated dylib due to
 // explicit instantiations. That means Apple backdeployment targets remain
 // broken.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 #include <istream>
 #include <cassert>
diff --git a/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py b/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py
new file mode 100644
index 0000000000000..46290b1ed52a0
--- /dev/null
+++ b/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py
@@ -0,0 +1,180 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+# In addition to being available via inclusion of the <iterator> header,
+# the function templates in [iterator.range] are available when any of the following
+# headers are included: <array>, <deque>, <flat_map>, <flat_set>, <forward_list>,
+# <list>, <map>, <regex>, <set>, <span>, <string>, <string_view>, <unordered_map>,
+# <unordered_set>, <vector>.
+
+# RUN: %{python} %s %{libcxx-dir}/utils
+# END.
+
+import sys
+
+sys.path.append(sys.argv[1])
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    Header,
+)
+
+headers = list(
+    map(
+        Header,
+        [
+            "array",
+            "deque",
+            "flat_map",
+            "flat_set",
+            "forward_list",
+            "list",
+            "map",
+            "regex",
+            "set",
+            "span",
+            "string",
+            "string_view",
+            "unordered_map",
+            "unordered_set",
+            "vector",
+        ],
+    )
+)
+
+for header in headers:
+    print(
+        f"""\
+//--- {header}.pass.cpp
+{lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
+// UNSUPPORTED: c++03
+
+#include <{header}>
+#include <cassert>
+#include <initializer_list>
+
+#include "test_macros.h"
+
+struct Container {{
+  int a[3] = {{1, 2, 3}};
+
+  int* begin() {{ return &a[0]; }}
+  const int* begin() const {{ return &a[0]; }}
+  int* rbegin() {{ return &a[2]; }}
+  const int* rbegin() const {{ return &a[2]; }}
+  int* end() {{ return &a[3]; }}
+  const int* end() const {{ return &a[3]; }}
+  int* rend() {{ return (&a[0]) - 1; }}
+  const int* rend() const {{ return (&a[0]) - 1; }}
+  std::size_t size() const {{ return 3; }}
+  bool empty() const {{ return false; }}
+  int* data() {{return &a[0]; }}
+  const int* data() const {{ return &a[0]; }}
+}};
+
+int main(int, char**)  {{
+  {{
+    Container c;
+    const auto& cc = c;
+    assert(std::begin(c) == c.begin());
+    assert(std::begin(cc) == cc.begin());
+    assert(std::end(c) == c.end());
+    assert(std::end(cc) == cc.end());
+#if TEST_STD_VER >= 14
+    assert(std::cbegin(c) == cc.begin());
+    assert(std::cbegin(cc) == cc.begin());
+    assert(std::cend(c) == cc.end());
+    assert(std::cend(cc) == cc.end());
+    assert(std::rbegin(c) == c.rbegin());
+    assert(std::rbegin(cc) == cc.rbegin());
+    assert(std::rend(c) == cc.rend());
+    assert(std::rend(cc) == cc.rend());
+    assert(std::crbegin(c) == cc.rbegin());
+    assert(std::crbegin(cc) == cc.rbegin());
+    assert(std::crend(c) == cc.rend());
+    assert(std::crend(cc) == cc.rend());
+#endif
+#if TEST_STD_VER >= 17
+    assert(std::data(c) == c.data());
+    assert(std::data(cc) == cc.data());
+    assert(std::size(cc) == cc.size());
+    assert(std::empty(cc) == cc.empty());
+#endif
+#if TEST_STD_VER >= 20
+    assert(std::ssize(cc) == 3);
+#endif
+  }}
+  {{
+    int a[]        = {{1, 2, 3}};
+    const auto& ca = a;
+    assert(std::begin(a) == &a[0]);
+    assert(std::begin(ca) == &ca[0]);
+    assert(std::end(a) == &a[3]);
+    assert(std::end(ca) == &ca[3]);
+#if TEST_STD_VER >= 14
+    assert(std::cbegin(a) == &a[0]);
+    assert(std::cbegin(ca) == &ca[0]);
+    assert(std::cend(a) == &a[3]);
+    assert(std::cend(ca) == &ca[3]);
+    assert(std::rbegin(a) == std::reverse_iterator<int*>(std::end(a)));
+    assert(std::rbegin(ca) == std::reverse_iterator<const int*>(std::end(ca)));
+    assert(std::rend(a) == std::reverse_iterator<int*>(std::begin(a)));
+    assert(std::rend(ca) == std::reverse_iterator<const int*>(std::begin(ca)));
+    assert(std::crbegin(a) == std::reverse_iterator<const int*>(std::end(a)));
+    assert(std::crbegin(ca) == std::reverse_iterator<const int*>(std::end(ca)));
+    assert(std::crend(a) == std::reverse_iterator<const int*>(std::begin(a)));
+    assert(std::crend(ca) == std::reverse_iterator<const int*>(std::begin(ca)));
+#endif
+#if TEST_STD_VER >= 17
+    assert(std::size(ca) == 3);
+    assert(!std::empty(ca));
+    assert(std::data(a) == &a[0]);
+    assert(std::data(ca) == &ca[0]);
+#endif
+#if TEST_STD_VER >= 20
+    assert(std::ssize(ca) == 3);
+#endif
+  }}
+  {{
+    auto il         = {{1, 2, 3}};
+    const auto& cil = il;
+    assert(std::begin(il) == il.begin());
+    assert(std::begin(cil) == cil.begin());
+    assert(std::end(il) == il.end());
+    assert(std::end(cil) == cil.end());
+#if TEST_STD_VER >= 14
+    assert(std::cbegin(il) == cil.begin());
+    assert(std::cbegin(cil) == cil.begin());
+    assert(std::cend(il) == cil.end());
+    assert(std::cend(cil) == cil.end());
+    assert(std::rbegin(il) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::rbegin(cil) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::rend(il) == std::reverse_iterator<const int*>(std::begin(il)));
+    assert(std::rend(cil) == std::reverse_iterator<const int*>(std::begin(il)));
+    assert(std::crbegin(il) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::crbegin(cil) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::crend(il) == std::reverse_iterator<const int*>(std::begin(il)));
+    assert(std::crend(cil) == std::reverse_iterator<const int*>(std::begin(il)));
+#endif
+#if TEST_STD_VER >= 17
+    assert(std::size(cil) == 3);
+    assert(!std::empty(cil));
+    assert(std::data(il) == &*std::begin(il));
+    assert(std::data(cil) == &*std::begin(il));
+#endif
+#if TEST_STD_VER >= 20
+    assert(std::ssize(cil) == 3);
+#endif
+  }}
+
+  return 0;
+}}
+
+"""
+    )
diff --git a/libcxx/test/std/language.support/support.exception/propagation/make_exception_ptr.objc.pass.mm b/libcxx/test/std/language.support/support.exception/propagation/make_exception_ptr.objc.pass.mm
new file mode 100644
index 0000000000000..05a6698ea1a59
--- /dev/null
+++ b/libcxx/test/std/language.support/support.exception/propagation/make_exception_ptr.objc.pass.mm
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// This test ensures that we can catch an Objective-C++ exception by type when
+// throwing an exception created via `std::make_exception_ptr`.
+// See http://llvm.org/PR135089.
+
+// UNSUPPORTED: no-exceptions
+// UNSUPPORTED: c++03
+
+// This test requires the Objective-C ARC, which is (only?) available on Darwin
+// out-of-the-box.
+// REQUIRES: has-fobjc-arc && darwin
+
+// ADDITIONAL_COMPILE_FLAGS: -fobjc-arc
+
+#include <cassert>
+#include <exception>
+
+#import <Foundation/Foundation.h>
+
+NSError* RecoverException(const std::exception_ptr& exc) {
+  try {
+    std::rethrow_exception(exc);
+  } catch (NSError* error) {
+    return error;
+  } catch (...) {
+  }
+  return nullptr;
+}
+
+int main(int, char**) {
+  NSError* error         = [NSError errorWithDomain:NSPOSIXErrorDomain code:EPERM userInfo:nil];
+  std::exception_ptr exc = std::make_exception_ptr(error);
+  NSError* recov         = RecoverException(exc);
+  assert(recov != nullptr);
+
+  return 0;
+}
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
index 8dbbbf432ddbf..8c0820681188d 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
@@ -32,6 +32,10 @@
 #    error "__cpp_lib_boyer_moore_searcher should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -94,6 +98,10 @@
 #    error "__cpp_lib_boyer_moore_searcher should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -165,6 +173,10 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -245,6 +257,13 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++20"
 #  endif
 
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++20"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++20"
 #  endif
@@ -337,6 +356,13 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++23"
 #  endif
 
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++23"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++23"
 #  endif
@@ -441,6 +467,13 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
index 45c51b5807c69..e6c0940ab7fd5 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
@@ -28,6 +28,10 @@
 #    error "__cpp_lib_bounded_array_traits should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should not be defined before c++17"
 #  endif
@@ -130,6 +134,10 @@
 #    error "__cpp_lib_bounded_array_traits should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should not be defined before c++17"
 #  endif
@@ -250,6 +258,10 @@
 #    error "__cpp_lib_bounded_array_traits should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++17"
 #  endif
@@ -394,6 +406,13 @@
 #    error "__cpp_lib_bounded_array_traits should have the value 201902L in c++20"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++20"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++20"
 #  endif
@@ -568,6 +587,13 @@
 #    error "__cpp_lib_bounded_array_traits should have the value 201902L in c++23"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++23"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++23"
 #  endif
@@ -763,6 +789,13 @@
 #    error "__cpp_lib_bounded_array_traits should have the value 201902L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index e546719142231..222d562a19d63 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -164,6 +164,14 @@
 #    error "__cpp_lib_clamp should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should not be defined before c++14"
 #  endif
@@ -1057,6 +1065,14 @@
 #    error "__cpp_lib_clamp should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++14"
 #  endif
@@ -2055,6 +2071,14 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++17"
 #  endif
@@ -3281,6 +3305,20 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++20"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++20"
+#  endif
+
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++20"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++20"
 #  endif
@@ -4726,6 +4764,20 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++23"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++23"
+#  endif
+
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++23"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++23"
 #  endif
@@ -6399,6 +6451,20 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++26"
+#  endif
+
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
index 4a5b3ee1ca87d..ea6b07934510a 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
@@ -9,7 +9,7 @@
 // TODO(mordante) Investigate
 // UNSUPPORTED: apple-clang
 
-// XFAIL darwin
+// XFAIL: darwin
 
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
index 251f6b996571a..14745996b9fd1 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
@@ -9,7 +9,7 @@
 // TODO(mordante) Investigate
 // UNSUPPORTED: apple-clang
 
-// XFAIL darwin
+// XFAIL: darwin
 
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
index 1e3b4a6588826..630b2739c88a8 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
@@ -9,7 +9,7 @@
 // TODO(mordante) Investigate
 // UNSUPPORTED: apple-clang
 
-// XFAIL darwin
+// XFAIL: darwin
 //
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp
index f32f7b0c18470..612d3738a373f 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp
@@ -11,7 +11,7 @@
 
 // The fix for LWG2381 (https://github.com/llvm/llvm-project/pull/77948) changed behavior of
 // FP parsing. This requires 3e15c97fa3812993bdc319827a5c6d867b765ae8 in the dylib.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp
index f063e67e65a0a..58bc9e5abef87 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp
@@ -11,7 +11,7 @@
 
 // The fix for LWG2381 (https://github.com/llvm/llvm-project/pull/77948) changed behavior of
 // FP parsing. This requires 3e15c97fa3812993bdc319827a5c6d867b765ae8 in the dylib.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp
index 29fc7a382fe41..bf8bb651d6bce 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp
@@ -11,7 +11,7 @@
 
 // The fix for LWG2381 (https://github.com/llvm/llvm-project/pull/77948) changed behavior of
 // FP parsing. This requires 3e15c97fa3812993bdc319827a5c6d867b765ae8 in the dylib.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
index 0749617d056ab..3f53490c1baee 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
@@ -22,8 +22,10 @@
 // get_date(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
index 1b19f5c944999..fbffafc5bcee7 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
@@ -20,8 +20,10 @@
 // get_monthname(iter_type s, iter_type end, ios_base& str,
 //               ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
index 35dba465917e3..513f92bc3fafd 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
@@ -23,8 +23,10 @@
 // iter_type get(iter_type s, iter_type end, ios_base& f,
 //               ios_base::iostate& err, tm *t, char format, char modifier = 0) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp
index aa5b0d04caa1f..651c1e16092e4 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp
@@ -19,9 +19,11 @@
 // get_time(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
index da64b573a41cd..0df0b10c9d7ca 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
@@ -19,8 +19,10 @@
 // get_weekday(iter_type s, iter_type end, ios_base& str,
 //             ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp
index ec846093583db..ef0d9230e78ba 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp
@@ -18,9 +18,11 @@
 // iter_type get_year(iter_type s, iter_type end, ios_base& str,
 //                    ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp
index a6c7f7816fdc4..1eac586d413ff 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp
@@ -14,9 +14,11 @@
 // get_date(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp
index e8ca74ee4aa04..4a4443d38be32 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp
@@ -14,9 +14,11 @@
 // get(iter_type s, iter_type end, ios_base& f, ios_base::iostate& err, tm *t,
 //     const char_type *fmt, const char_type *fmtend) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp
index 10769f92c50db..10d91715991b2 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp
@@ -14,8 +14,10 @@
 // get_monthname(iter_type s, iter_type end, ios_base& str,
 //               ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp
index a4ef9bd9f12c4..16a8eaa20b24d 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp
@@ -13,8 +13,10 @@
 // iter_type get(iter_type s, iter_type end, ios_base& f,
 //               ios_base::iostate& err, tm *t, char format, char modifier = 0) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp
index 009f94d26ee53..4d5b5edcdb693 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp
@@ -14,8 +14,10 @@
 // get_time(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp
index 23d53e32ef0d6..21780e59081db 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp
@@ -14,8 +14,10 @@
 // get_weekday(iter_type s, iter_type end, ios_base& str,
 //             ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp
index 663f7647d5ac7..c59dca140bfea 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp
@@ -13,9 +13,11 @@
 // iter_type get_year(iter_type s, iter_type end, ios_base& str,
 //                    ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
index f1c067cd322b7..4e087fd4226d1 100644
--- a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
@@ -16,9 +16,13 @@
 // There is no way to limit the number of threads on windows
 // UNSUPPORTED: windows
 
-// AIX and macOS seem to limit the number of processes, not threads via RLIMIT_NPROC
+// AIX, macOS and FreeBSD seem to limit the number of processes, not threads via RLIMIT_NPROC
 // XFAIL: target={{.+}}-aix{{.*}}
 // XFAIL: target={{.+}}-apple-{{.*}}
+// XFAIL: freebsd
+
+// z/OS does not have mechanism to limit the number of threads
+// XFAIL: target={{.+}}-zos{{.*}}
 
 // This test makes sure that we fail gracefully in care the thread creation fails. This is only reliably possible on
 // systems that allow limiting the number of threads that can be created. See https://llvm.org/PR125428 for more details
diff --git a/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp b/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp
index 8260ec3dfcaf4..7384a195bb360 100644
--- a/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp
@@ -20,22 +20,23 @@
 #include <atomic>
 #include <future>
 #include <mutex>
+#include <condition_variable>
+#include <thread>
 
 std::mutex mux;
 
 int main(int, char**) {
-  using namespace std::chrono_literals;
+  std::condition_variable cond;
   std::unique_lock lock(mux);
-  std::atomic<bool> in_async = false;
-  auto v                     = std::async(std::launch::async, [&in_async, value = 1]() mutable {
-    in_async = true;
-    in_async.notify_all();
-    std::scoped_lock thread_lock(mux);
+  auto v = std::async(std::launch::async, [&cond, value = 1]() mutable {
+    std::unique_lock thread_lock(mux);
+    cond.notify_all();
+    thread_lock.unlock();
+
     value = 4;
     (void)value;
   });
-  in_async.wait(true);
-  lock.unlock();
+  cond.wait(lock);
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp
index 25eb97a2df4d3..16c6986ae670e 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp
@@ -16,14 +16,14 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(CanCompare<std::expected<int, int>, int>);
-static_assert(CanCompare<std::expected<int, int>, EqualityComparable>);
-static_assert(!CanCompare<std::expected<int, int>, NonComparable>);
+static_assert(HasOperatorEqual<std::expected<int, int>, int>);
+static_assert(HasOperatorEqual<std::expected<int, int>, EqualityComparable>);
+static_assert(!HasOperatorEqual<std::expected<int, int>, NonComparable>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp
index f0f549b6b7772..218f0c39740e5 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp
@@ -17,26 +17,26 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 // Test constraint
-static_assert(!CanCompare<NonComparable, NonComparable>);
+static_assert(!HasOperatorEqual<NonComparable, NonComparable>);
 
-static_assert(CanCompare<std::expected<int, int>, std::expected<int, int>>);
-static_assert(CanCompare<std::expected<int, int>, std::expected<short, short>>);
+static_assert(HasOperatorEqual<std::expected<int, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<int, int>, std::expected<short, short>>);
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(!CanCompare<std::expected<int, int>, std::expected<void, int>>);
-static_assert(CanCompare<std::expected<int, int>, std::expected<int, int>>);
-static_assert(!CanCompare<std::expected<NonComparable, int>, std::expected<NonComparable, int>>);
-static_assert(!CanCompare<std::expected<int, NonComparable>, std::expected<int, NonComparable>>);
-static_assert(!CanCompare<std::expected<NonComparable, int>, std::expected<int, NonComparable>>);
-static_assert(!CanCompare<std::expected<int, NonComparable>, std::expected<NonComparable, int>>);
+static_assert(!HasOperatorEqual<std::expected<int, int>, std::expected<void, int>>);
+static_assert(HasOperatorEqual<std::expected<int, int>, std::expected<int, int>>);
+static_assert(!HasOperatorEqual<std::expected<NonComparable, int>, std::expected<NonComparable, int>>);
+static_assert(!HasOperatorEqual<std::expected<int, NonComparable>, std::expected<int, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<NonComparable, int>, std::expected<int, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<int, NonComparable>, std::expected<NonComparable, int>>);
 #else
 // Note this is true because other overloads in expected<non-void> are unconstrained
-static_assert(CanCompare<std::expected<void, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<int, int>>);
 #endif
 constexpr bool test() {
   // x.has_value() && y.has_value()
diff --git a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp
index 6c7d2f39514e7..153cbbddf3062 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp
@@ -16,14 +16,14 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(CanCompare<std::expected<EqualityComparable, EqualityComparable>, std::unexpected<int>>);
-static_assert(CanCompare<std::expected<EqualityComparable, int>, std::unexpected<EqualityComparable>>);
-static_assert(!CanCompare<std::expected<EqualityComparable, NonComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<EqualityComparable, EqualityComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<EqualityComparable, int>, std::unexpected<EqualityComparable>>);
+static_assert(!HasOperatorEqual<std::expected<EqualityComparable, NonComparable>, std::unexpected<int>>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp b/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp
index b6c3d8deee644..af53d0c00f578 100644
--- a/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp
@@ -17,26 +17,26 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 struct Foo{};
-static_assert(!CanCompare<Foo, Foo>);
+static_assert(!HasOperatorEqual<Foo, Foo>);
 
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, int>>);
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, short>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, short>>);
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(!CanCompare<std::expected<void, int>, std::expected<int, int>>);
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, int>>);
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, int>>);
-static_assert(!CanCompare<std::expected<void, NonComparable>, std::expected<void, NonComparable>>);
-static_assert(!CanCompare<std::expected<void, int>, std::expected<void, NonComparable>>);
-static_assert(!CanCompare<std::expected<void, NonComparable>, std::expected<void, int>>);
+static_assert(!HasOperatorEqual<std::expected<void, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, int>>);
+static_assert(!HasOperatorEqual<std::expected<void, NonComparable>, std::expected<void, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<void, int>, std::expected<void, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<void, NonComparable>, std::expected<void, int>>);
 #else
 // Note this is true because other overloads in expected<non-void> are unconstrained
-static_assert(CanCompare<std::expected<void, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<int, int>>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp b/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp
index f37f38bb71512..8d040d2ab7c0a 100644
--- a/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp
@@ -16,14 +16,14 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(CanCompare<std::expected<void, EqualityComparable>, std::unexpected<int>>);
-static_assert(CanCompare<std::expected<void, int>, std::unexpected<EqualityComparable>>);
-static_assert(!CanCompare<std::expected<void, NonComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<void, EqualityComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::unexpected<EqualityComparable>>);
+static_assert(!HasOperatorEqual<std::expected<void, NonComparable>, std::unexpected<int>>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/types.h b/libcxx/test/std/utilities/expected/types.h
index 11473ca3d97de..df73ebdfe495e 100644
--- a/libcxx/test/std/utilities/expected/types.h
+++ b/libcxx/test/std/utilities/expected/types.h
@@ -336,17 +336,4 @@ struct CheckForInvalidWrites : public CheckForInvalidWritesBase<WithPaddedExpect
   }
 };
 
-struct NonComparable {};
-
-struct EqualityComparable {
-  int i;
-  constexpr EqualityComparable(int ii) : i(ii) {}
-
-  friend constexpr bool operator==(const EqualityComparable& data, int ii) { return data.i == ii; }
-};
-
-// Test constraint
-template <class T1, class T2>
-concept CanCompare = requires(T1 t1, T2 t2) { t1 == t2; };
-
 #endif // TEST_STD_UTILITIES_EXPECTED_TYPES_H
diff --git a/libcxx/test/std/utilities/function.objects/func.bind.partial/bind_back.verify.cpp b/libcxx/test/std/utilities/function.objects/func.bind.partial/bind_back.verify.cpp
index eb100c15f580d..6a63b65bf148e 100644
--- a/libcxx/test/std/utilities/function.objects/func.bind.partial/bind_back.verify.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.bind.partial/bind_back.verify.cpp
@@ -69,6 +69,8 @@ void test() {
     auto f = std::bind_back([](const Arg&) {}, x);
     // expected-error-re@*:* {{static assertion failed{{.*}}bind_back requires all decay_t<Args> to be constructible from respective Args}}
     // expected-error@*:* {{no matching constructor for initialization}}
+    // expected-error@*:* 0-1{{call to deleted constructor of 'F'}}
+    // expected-error@*:* 0-1{{call to deleted constructor of 'Arg'}}
   }
 
   { // Mandates: (is_move_constructible_v<decay_t<Args>> && ...)
diff --git a/libcxx/test/std/utilities/function.objects/func.bind_front/bind_front.verify.cpp b/libcxx/test/std/utilities/function.objects/func.bind_front/bind_front.verify.cpp
index 5100259bd5b29..27a1fba90329a 100644
--- a/libcxx/test/std/utilities/function.objects/func.bind_front/bind_front.verify.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.bind_front/bind_front.verify.cpp
@@ -42,4 +42,5 @@ void f() {
     auto d = std::bind_front(do_nothing, n); // expected-error {{no matching function for call to 'bind_front'}}
 
     auto t = std::bind_front(testNotMoveConst, NotMoveConst(0)); // expected-error {{no matching function for call to 'bind_front'}}
+    // expected-error@*:* 0-1{{call to deleted constructor of 'NotMoveConst'}}
 }
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp
new file mode 100644
index 0000000000000..a3630cb0017b6
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp
@@ -0,0 +1,152 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <functional>
+//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+//
+// common_reference specializations for reference_wrapper
+
+#include <concepts>
+#include <functional>
+#include <type_traits>
+
+template <class T>
+concept HasType = requires { typename T::type; };
+
+template <class Result, class T1, class T2>
+concept check_XY = std::same_as<Result, std::common_reference_t<T1, T2>>;
+
+template <class Result, class T1, class T2>
+concept check_YX = std::same_as<Result, std::common_reference_t<T2, T1>>;
+
+template <class Result, class T1, class T2>
+concept check = check_XY<Result, T1, T2> && check_YX<Result, T1, T2>;
+
+template <class T1, class T2>
+concept check_none_XY = !HasType<std::common_reference<T1, T2>>;
+template <class T1, class T2>
+concept check_none_YX = !HasType<std::common_reference<T2, T1>>;
+
+template <class T1, class T2>
+concept check_none = check_none_XY<T1, T2> && check_none_YX<T1, T2>;
+
+// https://eel.is/c++draft/meta.trans#other-2.4
+template <class X, class Y>
+using CondRes = decltype(false ? std::declval<X (&)()>()() : std::declval<Y (&)()>()());
+
+template <class X, class Y>
+struct Ternary {};
+
+template <class X, class Y>
+  requires requires() { typename CondRes<X, Y>; }
+struct Ternary<X, Y> {
+  using type = CondRes<X, Y>;
+};
+template <class X, class Y>
+using Ternary_t = typename Ternary<X, Y>::type;
+
+template <class T>
+using Ref = std::reference_wrapper<T>;
+
+using std::common_reference_t;
+using std::same_as;
+
+// clang-format off
+static_assert(check<int &     , Ref<int      >, int &      >);
+static_assert(check<int const&, Ref<int      >, int const& >);
+static_assert(check<int const&, Ref<int const>, int &      >);
+static_assert(check<int const&, Ref<int const>, int const& >);
+static_assert(check<int&,       Ref<int> const&, int& >);
+static_assert(check<const volatile int&, Ref<const volatile int>, const volatile int&>);
+
+// derived-base and implicit convertibles
+struct B {};
+struct D : B {};
+struct C {
+    operator B&() const;
+};
+
+static_assert(check<B&      , Ref<B>,       D &     >);
+static_assert(check<B const&, Ref<B>,       D const&>);
+static_assert(check<B const&, Ref<B const>, D const&>);
+
+static_assert(check<B&      , Ref<D>,       B &     >);
+static_assert(check<B const&, Ref<D>,       B const&>);
+static_assert(check<B const&, Ref<D const>, B const&>);
+
+static_assert(std::same_as<B&,       CondRes<Ref<D>,       B&>>);
+static_assert(std::same_as<B const&, CondRes<Ref<D>,       B const &>>);
+static_assert(std::same_as<B const&, CondRes<Ref<D const>, B const&>>);
+
+static_assert( check<B&        , Ref<B>      , C&      >);
+static_assert( check<B&        , Ref<B>      , C       >);
+static_assert( check<B const&  , Ref<B const>, C       >);
+static_assert(!check<B&        , Ref<C>      , B&      >); // Ref<C> cannot be converted to B&
+static_assert( check<B&        , Ref<B>      , C const&>); // was const B& before P2655R3
+
+
+using Ri   = Ref<int>;
+using RRi  = Ref<Ref<int>>;
+using RRRi = Ref<Ref<Ref<int>>>;
+static_assert(check<Ri&,  Ri&,  RRi>);
+static_assert(check<RRi&, RRi&, RRRi>);
+static_assert(check<Ri,   Ri,   RRi>);
+static_assert(check<RRi,  RRi,  RRRi>);
+
+static_assert(check_none<int&, RRi>);
+static_assert(check_none<int,  RRi>);
+static_assert(check_none<int&, RRRi>);
+static_assert(check_none<int,  RRRi>);
+
+static_assert(check_none<Ri&, RRRi>);
+static_assert(check_none<Ri,  RRRi>);
+
+
+template <typename T>
+struct Test {
+  // Check that reference_wrapper<T> behaves the same as T& in common_reference.
+
+  using R1 = common_reference_t<T&, T&>;
+  using R2 = common_reference_t<T&, T const&>;
+  using R3 = common_reference_t<T&, T&&>;
+  using R4 = common_reference_t<T&, T const&&>;
+  using R5 = common_reference_t<T&, T>;
+
+  static_assert(same_as<R1, common_reference_t<Ref<T>, T&>>);
+  static_assert(same_as<R2, common_reference_t<Ref<T>, T const&>>);
+  static_assert(same_as<R3, common_reference_t<Ref<T>, T&&>>);
+  static_assert(same_as<R4, common_reference_t<Ref<T>, T const&&>>);
+  static_assert(same_as<R5, common_reference_t<Ref<T>, T>>);
+
+  // commute:
+  static_assert(same_as<R1, common_reference_t<T&,        Ref<T>>>);
+  static_assert(same_as<R2, common_reference_t<T const&,  Ref<T>>>);
+  static_assert(same_as<R3, common_reference_t<T&&,       Ref<T>>>);
+  static_assert(same_as<R4, common_reference_t<T const&&, Ref<T>>>);
+  static_assert(same_as<R5, common_reference_t<T,         Ref<T>>>);
+
+  // reference qualification of reference_wrapper is irrelevant 
+  static_assert(same_as<R1, common_reference_t<Ref<T>&,        T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T> ,        T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T> const&,  T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T>&&,       T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T> const&&, T&>>);
+};
+
+// clang-format on
+// Instantiate above checks:
+template struct Test<int>;
+template struct Test<std::reference_wrapper<int>>;
+
+// reference_wrapper as both args is unaffected.
+// subject to simple first rule of
+static_assert(check<Ref<int>&, Ref<int>&, Ref<int>&>);
+
+// double wrap is unaffected.
+static_assert(check<Ref<int>&, Ref<Ref<int>>, Ref<int>&>);
diff --git a/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp b/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp
index e802776f52bfc..8e3cad797d5a7 100644
--- a/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp
@@ -18,9 +18,7 @@
 #include "test_macros.h"
 
 template <class T>
-constexpr bool has_type = requires {
-  typename T::type;
-};
+constexpr bool has_type = requires { typename T::type; };
 
 // A slightly simplified variation of std::tuple
 template <class...>
@@ -74,8 +72,10 @@ static_assert(std::is_same_v<std::common_reference_t<void (&&)()>, void (&&)()>)
 //  -- Otherwise, if sizeof...(T) is two, let T1 and T2 denote the two types in
 //     the pack T. Then
 // (6.3.1)
-//    -- If T1 and T2 are reference types and COMMON_REF(T1, T2) is well-formed,
-//       then the member typedef type denotes that type.
+//    -- Let R be COMMON-REF(T1, T2). If T1 and T2 are reference types, R is well-formed,
+//       and is_convertible_v<add_pointer_t<T1>, add_pointer_t<R>> && is_convertible_v<add_pointer_t<T2>, add_pointer_t<R>>
+//       is true, then the member typedef type denotes R.
+
 struct B {};
 struct D : B {};
 static_assert(std::is_same_v<std::common_reference_t<B&, D&>, B&>);
@@ -99,7 +99,19 @@ static_assert(std::is_same_v<std::common_reference_t<int const&, int volatile&>,
 static_assert(std::is_same_v<std::common_reference_t<int const volatile&&, int volatile&&>, int const volatile&&>);
 
 static_assert(std::is_same_v<std::common_reference_t<int (&)[10], int (&&)[10]>, int const (&)[10]>);
-static_assert(std::is_same_v<std::common_reference_t<int const (&)[10], int volatile (&)[10]>, int const volatile (&)[10]>);
+static_assert(
+    std::is_same_v<std::common_reference_t<int const (&)[10], int volatile (&)[10]>, int const volatile (&)[10]>);
+
+// when conversion from pointers are not true
+struct E {};
+struct F {
+  operator E&() const;
+};
+
+static_assert(!std::is_convertible_v<F*, E*>);
+
+// The following should not use 6.3.1, but fallback to 6.3.3
+static_assert(std::is_same_v<std::common_reference_t<E&, F>, E&>);
 
 // (6.3.2)
 //    -- Otherwise, if basic_common_reference<remove_cvref_t<T1>,
@@ -136,8 +148,8 @@ static_assert(std::is_same_v<std::common_reference_t<int&, MyIntRef>, MyIntRef>)
 //    -- Otherwise, if common_type_t<T1, T2> is well-formed, then the member
 //       typedef type denotes that type.
 struct moveonly {
-  moveonly() = default;
-  moveonly(moveonly&&) = default;
+  moveonly()                      = default;
+  moveonly(moveonly&&)            = default;
   moveonly& operator=(moveonly&&) = default;
 };
 struct moveonly2 : moveonly {};
@@ -169,14 +181,17 @@ static_assert(!has_type<std::common_reference<int, short, int, char*> >);
 
 #if TEST_STD_VER > 20
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, int>>, std::tuple<int, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, long>, std::tuple<long, int>>, std::tuple<long, long>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::tuple<int, long>, std::tuple<long, int>>, std::tuple<long, long>>);
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int&, const int&>, std::tuple<const int&, int>>,
                              std::tuple<const int&, int>>);
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int&, volatile int&>, std::tuple<volatile int&, int>>,
                              std::tuple<volatile int&, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::tuple<int&, const volatile int&>, std::tuple<const volatile int&, int>>,
-                             std::tuple<const volatile int&, int>>);
-static_assert(!has_type<std::common_reference_t<std::tuple<const int&, volatile int&>, std::tuple<volatile int&, const int&>>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::tuple<int&, const volatile int&>, std::tuple<const volatile int&, int>>,
+                   std::tuple<const volatile int&, int>>);
+static_assert(
+    !has_type<std::common_reference_t<std::tuple<const int&, volatile int&>, std::tuple<volatile int&, const int&>>>);
 
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, X2>, std::tuple<int, Y2>>, std::tuple<int, Z2>>);
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, X2>, std::tuple<int, Y2>>, std::tuple<int, Z2>>);
@@ -185,26 +200,25 @@ static_assert(!has_type<std::common_reference<std::tuple<int, X2>, std::tuple<fl
 static_assert(!has_type<std::common_reference<std::tuple<int, X2>, int, X2>>);
 
 struct A {};
-template <template<class> class TQual, template<class> class UQual>
+template <template <class> class TQual, template <class> class UQual>
 struct std::basic_common_reference<A, std::tuple<B>, TQual, UQual> {
   using type = tuple<UQual<B>>;
 };
 
 static_assert(std::is_same_v<std::common_reference_t<A, std::tuple<B>, std::tuple<D>>, std::tuple<B>>);
 
-
-static_assert(std::is_same_v<std::common_reference_t<std::pair<int, int>>,
-                             std::pair<int, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::pair<int, long>, std::pair<long, int>>,
-                             std::pair<long, long>>);
+static_assert(std::is_same_v<std::common_reference_t<std::pair<int, int>>, std::pair<int, int>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::pair<int, long>, std::pair<long, int>>, std::pair<long, long>>);
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int&, const int&>, std::pair<const int&, int>>,
                              std::pair<const int&, int>>);
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int&, volatile int&>, std::pair<volatile int&, int>>,
                              std::pair<volatile int&, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::pair<int&, const volatile int&>, std::pair<const volatile int&, int>>,
-                             std::pair<const volatile int&, int>>);
-static_assert(!has_type<std::common_reference_t<std::pair<const int&, volatile int&>,
-                        std::pair<volatile int&, const int&>>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::pair<int&, const volatile int&>, std::pair<const volatile int&, int>>,
+                   std::pair<const volatile int&, int>>);
+static_assert(
+    !has_type<std::common_reference_t<std::pair<const int&, volatile int&>, std::pair<volatile int&, const int&>>>);
 
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int, X2>, std::pair<int, Y2>>, std::pair<int, Z2>>);
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int, X2>, std::pair<int, Y2>>, std::pair<int, Z2>>);
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp
index c472906c5ed7f..226a3587a47f4 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp
@@ -21,28 +21,15 @@
 #include <cassert>
 #include <concepts>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
 
 #if TEST_STD_VER >= 26
 
 // Test SFINAE.
 
-struct EqualityComparable {
-  constexpr EqualityComparable(int value) : value_{value} {};
-
-  friend constexpr bool operator==(const EqualityComparable&, const EqualityComparable&) noexcept = default;
-
-  int value_;
-};
-
-static_assert(std::equality_comparable<EqualityComparable>);
-
 static_assert(std::equality_comparable<std::pair<EqualityComparable, EqualityComparable>>);
 
-struct NonComparable {};
-
-static_assert(!std::equality_comparable<NonComparable>);
-
 static_assert(!std::equality_comparable<std::pair<EqualityComparable, NonComparable>>);
 static_assert(!std::equality_comparable<std::pair<NonComparable, EqualityComparable>>);
 
diff --git a/libcxx/test/support/concat_macros.h b/libcxx/test/support/concat_macros.h
index 50b6562197392..efec4029f4025 100644
--- a/libcxx/test/support/concat_macros.h
+++ b/libcxx/test/support/concat_macros.h
@@ -9,6 +9,7 @@
 #ifndef TEST_SUPPORT_CONCAT_MACROS_H
 #define TEST_SUPPORT_CONCAT_MACROS_H
 
+#include <algorithm>
 #include <cstdio>
 #include <string>
 #include <source_location>
diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
index 897da8b769e6a..8ec029f7dba2d 100644
--- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
+++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
@@ -101,8 +101,10 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match
             // Looks at the common locations where headers store their data
             // * header
             // * __header/*.h
+            // * __header_dir/*.h // Used for transitioning from __header
             // * __fwd/header.h
             anyOf(isExpansionInFileMatching(("v1/__" + filename_ + "/").str()),
+                  isExpansionInFileMatching(("v1/__" + filename_ + "_dir/").str()),
                   isExpansionInFileMatching(extra_header_),
                   isExpansionInFileMatching(("v1/__fwd/" + filename_ + "\\.h$").str()),
                   isExpansionInFileMatching(("v1/" + filename_ + "$").str())),
diff --git a/libcxx/utils/ci/buildkite-pipeline-trigger.sh b/libcxx/utils/ci/buildkite-pipeline-trigger.sh
new file mode 100755
index 0000000000000..4661cd54fad42
--- /dev/null
+++ b/libcxx/utils/ci/buildkite-pipeline-trigger.sh
@@ -0,0 +1,34 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+#
+# This script determines whether the libc++ Buildkite pipeline should be triggered
+# on a change. This is required because Buildkite gets notified for every PR in the
+# LLVM monorepo, and we make it a no-op unless the libc++ pipeline needs to be triggered.
+#
+
+# Set by buildkite
+: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
+
+# Fetch origin to have an up to date merge base for the diff.
+git fetch origin
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
+
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
+
+# If libc++ or one of the runtimes directories changed, trigger the libc++ Buildkite pipeline.
+if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
+    buildkite-agent pipeline upload libcxx/utils/ci/buildkite-pipeline.yml
+else
+    echo "No Buildkite jobs to trigger"
+fi
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index edd7b124a1fb3..18eb8a8623748 100644
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -313,6 +313,16 @@ def add_version_header(tc):
             "values": {"c++17": 201603},
             "headers": ["algorithm"],
         },
+        {
+            "name": "__cpp_lib_common_reference",
+            "values": {"c++20": 202302},
+            "headers": ["type_traits"],
+        },
+        {
+            "name": "__cpp_lib_common_reference_wrapper",
+            "values": {"c++20": 202302},
+            "headers": ["functional"],
+        },
         {
             "name": "__cpp_lib_complex_udls",
             "values": {"c++14": 201309},
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 74746e37d3bc4..0cb81546665d4 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -134,11 +134,6 @@ def _mingwSupportsModules(cfg):
     Feature(name=lambda cfg: "msvc-{}".format(*_msvcVersion(cfg)), when=_isMSVC),
     Feature(name=lambda cfg: "msvc-{}.{}".format(*_msvcVersion(cfg)), when=_isMSVC),
 
-    Feature(
-        name="thread-safety",
-        when=lambda cfg: hasCompileFlag(cfg, "-Werror=thread-safety"),
-        actions=[AddCompileFlag("-Werror=thread-safety")],
-    ),
     Feature(
         name="diagnose-if-support",
         when=lambda cfg: hasCompileFlag(cfg, "-Wuser-defined-warnings"),
@@ -336,12 +331,22 @@ def _mingwSupportsModules(cfg):
         or platform.system().lower().startswith("aix")
         # Avoid building on platforms that don't support modules properly.
         or not hasCompileFlag(cfg, "-Wno-reserved-module-identifier")
-        or not sourceBuilds(
-            cfg,
-            """
+        # older versions don't support extern "C++", newer versions don't support main in named module.
+        or not (
+            sourceBuilds(
+                cfg,
+                """
+            export module test;
+            extern "C++" int main(int, char**) { return 0; }
+          """,
+            )
+            or sourceBuilds(
+                cfg,
+                """
             export module test;
             int main(int, char**) { return 0; }
           """,
+            )
         ),
     ),
     # The time zone validation tests compare the output of zdump against the
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index fc34009d0a551..adfb2a9f69508 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -170,7 +170,6 @@ def getSuitableClangTidy(cfg):
             [
                 AddFeature("target={}".format(triple)),
                 AddFlagIfSupported("--target={}".format(triple)),
-                AddSubstitution("%{triple}", triple),
             ],
         ),
     ),
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index 3e2f80b818450..975bd75cbb9bd 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -192,6 +192,8 @@ if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
   set(LIBCXXABI_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LIBCXXABI_TARGET_SUBDIR})
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LLVM_LIBDIR_SUFFIX}/${LIBCXXABI_TARGET_SUBDIR} CACHE STRING
       "Path where built libc++abi libraries should be installed.")
+  set(LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR "${CMAKE_INSTALL_INCLUDEDIR}/${LIBCXXABI_TARGET_SUBDIR}/c++/v1" CACHE STRING
+      "Path where target-specific libc++abi headers should be installed.")
   unset(LIBCXXABI_TARGET_SUBDIR)
 else()
   if(LLVM_LIBRARY_OUTPUT_INTDIR)
@@ -203,6 +205,8 @@ else()
   endif()
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LIBCXXABI_LIBDIR_SUFFIX} CACHE STRING
       "Path where built libc++abi libraries should be installed.")
+  set(LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR "${LIBCXXABI_INSTALL_INCLUDE_DIR}" CACHE STRING
+      "Path where target-specific libc++abi headers should be installed.")
 endif()
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LIBCXXABI_LIBRARY_DIR})
diff --git a/libcxxabi/test/configs/cmake-bridge.cfg.in b/libcxxabi/test/configs/cmake-bridge.cfg.in
index 1bdba32405ddd..f81dd8afb1091 100644
--- a/libcxxabi/test/configs/cmake-bridge.cfg.in
+++ b/libcxxabi/test/configs/cmake-bridge.cfg.in
@@ -32,7 +32,7 @@ config.substitutions.append(('%{llvm}', '@LLVM_MAIN_SRC_DIR@'))
 config.substitutions.append(('%{install-prefix}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@'))
 config.substitutions.append(('%{include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/include'))
 config.substitutions.append(('%{cxx-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_DIR@'))
-config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/include/%{triple}/c++/v1'))
+config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR@'))
 config.substitutions.append(('%{lib}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_LIBRARY_DIR@'))
 config.substitutions.append(('%{benchmark_flags}', ''))
 
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 473943a239422..79b63e5b7236f 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -307,6 +307,7 @@ struct Configuration {
   bool warnDebugInfoUnusable = true;
   bool warnLongSectionNames = true;
   bool warnStdcallFixup = true;
+  bool warnExportedDllMain = true;
   bool incremental = true;
   bool integrityCheck = false;
   bool killAt = false;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 8a016d44d4636..283aeed1a19cd 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -1643,6 +1643,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         config->warnLocallyDefinedImported = false;
       else if (s == "longsections")
         config->warnLongSectionNames = false;
+      else if (s == "exporteddllmain")
+        config->warnExportedDllMain = false;
       // Other warning numbers are ignored.
     }
   }
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 760ce00dda0d6..0b7dbea8cdd99 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -114,6 +114,35 @@ static coff_symbol_generic *cloneSymbol(COFFSymbolRef sym) {
   }
 }
 
+// Skip importing DllMain thunks from import libraries.
+static bool fixupDllMain(COFFLinkerContext &ctx, llvm::object::Archive *file,
+                         const Archive::Symbol &sym, bool &skipDllMain) {
+  if (skipDllMain)
+    return true;
+  const Archive::Child &c =
+      CHECK(sym.getMember(), file->getFileName() +
+                                 ": could not get the member for symbol " +
+                                 toCOFFString(ctx, sym));
+  MemoryBufferRef mb =
+      CHECK(c.getMemoryBufferRef(),
+            file->getFileName() +
+                ": could not get the buffer for a child buffer of the archive");
+  if (identify_magic(mb.getBuffer()) == file_magic::coff_import_library) {
+    if (ctx.config.warnExportedDllMain) {
+      // We won't place DllMain symbols in the symbol table if they are
+      // coming from a import library. This message can be ignored with the flag
+      // '/ignore:exporteddllmain'
+      Warn(ctx)
+          << file->getFileName()
+          << ": skipping exported DllMain symbol [exporteddllmain]\nNOTE: this "
+             "might be a mistake when the DLL/library was produced.";
+    }
+    skipDllMain = true;
+    return true;
+  }
+  return false;
+}
+
 ArchiveFile::ArchiveFile(COFFLinkerContext &ctx, MemoryBufferRef m)
     : InputFile(ctx.symtab, ArchiveKind, m) {}
 
@@ -176,8 +205,17 @@ void ArchiveFile::parse() {
   }
 
   // Read the symbol table to construct Lazy objects.
-  for (const Archive::Symbol &sym : file->symbols())
+  bool skipDllMain = false;
+  for (const Archive::Symbol &sym : file->symbols()) {
+    // If the DllMain symbol was exported by mistake, skip importing it
+    // otherwise we might end up with a import thunk in the final binary which
+    // is wrong.
+    if (sym.getName() == "__imp_DllMain" || sym.getName() == "DllMain") {
+      if (fixupDllMain(ctx, file.get(), sym, skipDllMain))
+        continue;
+    }
     archiveSymtab->addLazyArchive(this, sym);
+  }
 }
 
 // Returns a buffer pointing to a member file containing a given symbol.
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index 3c4ad53af1b51..fe804cbb0e690 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -39,8 +39,13 @@ class LoongArch final : public TargetInfo {
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
   bool relaxOnce(int pass) const override;
+  RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
   void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
   void finalizeRelax(int passes) const override;
+
+private:
+  void tlsdescToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
+  void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
 };
 } // end anonymous namespace
 
@@ -58,6 +63,7 @@ enum Op {
   LU12I_W = 0x14000000,
   PCADDI = 0x18000000,
   PCADDU12I = 0x1c000000,
+  PCALAU12I = 0x1a000000,
   LD_W = 0x28800000,
   LD_D = 0x28c00000,
   JIRL = 0x4c000000,
@@ -69,6 +75,7 @@ enum Reg {
   R_ZERO = 0,
   R_RA = 1,
   R_TP = 2,
+  R_A0 = 4,
   R_T0 = 12,
   R_T1 = 13,
   R_T2 = 14,
@@ -961,7 +968,8 @@ static bool relax(Ctx &ctx, InputSection &sec) {
     case R_LARCH_TLS_LD_PC_HI20:
     case R_LARCH_TLS_DESC_PC_HI20:
       // The overflow check for i+2 will be carried out in isPairRelaxable.
-      if (isPairRelaxable(relocs, i))
+      if (r.expr != RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC &&
+          r.expr != R_RELAX_TLS_GD_TO_LE && isPairRelaxable(relocs, i))
         relaxPCHi20Lo12(ctx, sec, i, loc, r, relocs[i + 2], remove);
       break;
     case R_LARCH_CALL36:
@@ -1046,6 +1054,104 @@ static void tlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) {
   }
 }
 
+// Convert TLSDESC GD/LD to IE.
+// In normal or medium code model, there are two forms of code sequences:
+//  * pcalau12i  $a0, %desc_pc_hi20(sym_desc)
+//  * addi.d     $a0, $a0, %desc_pc_lo12(sym_desc)
+//  * ld.d       $ra, $a0, %desc_ld(sym_desc)
+//  * jirl       $ra, $ra, %desc_call(sym_desc)
+//  ------
+//  * pcaddi $a0, %desc_pcrel_20(a)
+//  * load $ra, $a0, %desc_ld(a)
+//  * jirl $ra, $ra, %desc_call(a)
+//
+// The code sequence obtained is as follows:
+//  * pcalau12i $a0, %ie_pc_hi20(sym_ie)
+//  * ld.[wd]   $a0, $a0, %ie_pc_lo12(sym_ie)
+//
+// Simplicity, whether tlsdescToIe or tlsdescToLe, we always tend to convert the
+// preceding instructions to NOPs, due to both forms of code sequence
+// (corresponding to relocation combinations:
+// R_LARCH_TLS_DESC_PC_HI20+R_LARCH_TLS_DESC_PC_LO12 and
+// R_LARCH_TLS_DESC_PCREL20_S2) have same process.
+//
+// When relaxation enables, redundant NOPs can be removed.
+void LoongArch::tlsdescToIe(uint8_t *loc, const Relocation &rel,
+                            uint64_t val) const {
+  switch (rel.type) {
+  case R_LARCH_TLS_DESC_PC_HI20:
+  case R_LARCH_TLS_DESC_PC_LO12:
+  case R_LARCH_TLS_DESC_PCREL20_S2:
+    write32le(loc, insn(ANDI, R_ZERO, R_ZERO, 0)); // nop
+    break;
+  case R_LARCH_TLS_DESC_LD:
+    write32le(loc, insn(PCALAU12I, R_A0, 0, 0)); // pcalau12i $a0, %ie_pc_hi20
+    relocateNoSym(loc, R_LARCH_TLS_IE_PC_HI20, val);
+    break;
+  case R_LARCH_TLS_DESC_CALL:
+    write32le(loc, insn(ctx.arg.is64 ? LD_D : LD_W, R_A0, R_A0,
+                        0)); // ld.[wd] $a0, $a0, %ie_pc_lo12
+    relocateNoSym(loc, R_LARCH_TLS_IE_PC_LO12, val);
+    break;
+  default:
+    llvm_unreachable("unsupported relocation for TLSDESC to IE");
+  }
+}
+
+// Convert TLSDESC GD/LD to LE.
+// The code sequence obtained in the normal or medium code model is as follows:
+//  * lu12i.w   $a0, %le_hi20(sym)      # le_hi20 != 0, otherwise NOP
+//  * ori       $a0, src, %le_lo12(sym) # le_hi20 != 0, src = $a0,
+//                                      # otherwise,    src = $zero
+// See the comment in tlsdescToIe for detailed information.
+void LoongArch::tlsdescToLe(uint8_t *loc, const Relocation &rel,
+                            uint64_t val) const {
+  assert(isInt<32>(val) &&
+         "val exceeds the range of medium code model in tlsdescToLe");
+
+  bool isUInt12 = isUInt<12>(val);
+  switch (rel.type) {
+  case R_LARCH_TLS_DESC_PC_HI20:
+  case R_LARCH_TLS_DESC_PC_LO12:
+  case R_LARCH_TLS_DESC_PCREL20_S2:
+    write32le(loc, insn(ANDI, R_ZERO, R_ZERO, 0)); // nop
+    break;
+  case R_LARCH_TLS_DESC_LD:
+    if (isUInt12)
+      write32le(loc, insn(ANDI, R_ZERO, R_ZERO, 0)); // nop
+    else
+      write32le(loc, insn(LU12I_W, R_A0, extractBits(val, 31, 12),
+                          0)); // lu12i.w $a0, %le_hi20
+    break;
+  case R_LARCH_TLS_DESC_CALL:
+    if (isUInt12)
+      write32le(loc, insn(ORI, R_A0, R_ZERO, val)); // ori $a0, $zero, %le_lo12
+    else
+      write32le(loc,
+                insn(ORI, R_A0, R_A0, lo12(val))); // ori $a0, $a0, %le_lo12
+    break;
+  default:
+    llvm_unreachable("unsupported relocation for TLSDESC to LE");
+  }
+}
+
+// During TLSDESC GD_TO_IE, the converted code sequence always includes an
+// instruction related to the Lo12 relocation (ld.[wd]). To obtain correct val
+// in `getRelocTargetVA`, expr of this instruction should be adjusted to
+// R_RELAX_TLS_GD_TO_IE_ABS, while expr of other instructions related to the
+// Hi20 relocation (pcalau12i) should be adjusted to
+// RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC. Specifically, in the normal or
+// medium code model, the instruction with relocation R_LARCH_TLS_DESC_CALL is
+// the candidate of Lo12 relocation.
+RelExpr LoongArch::adjustTlsExpr(RelType type, RelExpr expr) const {
+  if (expr == R_RELAX_TLS_GD_TO_IE) {
+    if (type != R_LARCH_TLS_DESC_CALL)
+      return RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC;
+    return R_RELAX_TLS_GD_TO_IE_ABS;
+  }
+  return expr;
+}
+
 void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
   const unsigned bits = ctx.arg.is64 ? 64 : 32;
   uint64_t secAddr = sec.getOutputSection()->addr;
@@ -1074,7 +1180,7 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
         //  * i+2 -- R_LARCH_TLS_IE64_PC_LO20
         //  * i+3 -- R_LARCH_TLS_IE64_PC_HI12
         isExtreme =
-            (i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_IE64_PC_LO20);
+            i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_IE64_PC_LO20;
       }
       if (isExtreme) {
         rel.expr = getRelExpr(rel.type, *rel.sym, loc);
@@ -1088,6 +1194,47 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
         tlsIeToLe(loc, rel, val);
       }
       continue;
+    case RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC:
+      if (rel.type == R_LARCH_TLS_DESC_PC_HI20) {
+        // LoongArch does not support TLSDESC GD/LD to LE/IE optimization in the
+        // extreme code model. In these cases, the relocs are as follows:
+        //
+        //  * i   -- R_LARCH_TLS_DESC_PC_HI20
+        //  * i+1 -- R_LARCH_TLS_DESC_PC_LO12
+        //  * i+2 -- R_LARCH_TLS_DESC64_PC_LO20
+        //  * i+3 -- R_LARCH_TLS_DESC64_PC_HI12
+        isExtreme =
+            i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_DESC64_PC_LO20;
+      }
+      [[fallthrough]];
+    case R_RELAX_TLS_GD_TO_IE_ABS:
+      if (isExtreme) {
+        if (rel.type == R_LARCH_TLS_DESC_CALL)
+          continue;
+        rel.expr = getRelExpr(rel.type, *rel.sym, loc);
+        val = SignExtend64(sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset),
+                           bits);
+        relocateNoSym(loc, rel.type, val);
+      } else {
+        tlsdescToIe(loc, rel, val);
+      }
+      continue;
+    case R_RELAX_TLS_GD_TO_LE:
+      if (rel.type == R_LARCH_TLS_DESC_PC_HI20) {
+        isExtreme =
+            i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_DESC64_PC_LO20;
+      }
+      if (isExtreme) {
+        if (rel.type == R_LARCH_TLS_DESC_CALL)
+          continue;
+        rel.expr = getRelExpr(rel.type, *rel.sym, loc);
+        val = SignExtend64(sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset),
+                           bits);
+        relocateNoSym(loc, rel.type, val);
+      } else {
+        tlsdescToLe(loc, rel, val);
+      }
+      continue;
     default:
       break;
     }
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 163505102d0ec..488f4803b2cb4 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -320,6 +320,8 @@ bool X86_64::deleteFallThruJmpInsn(InputSection &is, InputFile *file,
 bool X86_64::relaxOnce(int pass) const {
   uint64_t minVA = UINT64_MAX, maxVA = 0;
   for (OutputSection *osec : ctx.outputSections) {
+    if (!(osec->flags & SHF_ALLOC))
+      continue;
     minVA = std::min(minVA, osec->addr);
     maxVA = std::max(maxVA, osec->addr + osec->size);
   }
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 88bda41d36487..4c771e47a0e72 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -275,6 +275,10 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 1e0b5988343a6..a194ae71d559a 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1396,6 +1396,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
   ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
   ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
   ctx.arg.ehFrameHdr =
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index f8786265029e8..68e3feb1bd048 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -830,6 +830,7 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
   case R_GOTPLT_PC:
     return r.sym->getGotPltVA(ctx) + a - p;
   case RE_LOONGARCH_GOT_PAGE_PC:
+  case RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC:
     if (r.sym->hasFlag(NEEDS_TLSGD))
       return getLoongArchPageDelta(ctx.in.got->getGlobalDynAddr(*r.sym) + a, p,
                                    r.type);
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 82a7463446a94..8d4a6c9e3a81e 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNew),
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index d7e3313167007..cc91680550b4b 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -714,7 +714,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
   HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
-
+def thinlto_distributor_eq: JJ<"thinlto-distributor=">,
+  HelpText<"Distributor to use for ThinLTO backend compilations. If specified, "
+  "ThinLTO backend compilations will be distributed">;
+defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
+  "pass to the ThinLTO distributor">;
+def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">,
+  HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
+  "compilations">;
+defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler "
+  "arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm fat_lto_objects: BB<"fat-lto-objects",
     "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
     "Ignore the .llvm.lto section in relocatable object files (default).">;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 43f19186f0981..cebd564036b2c 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1340,22 +1340,10 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
   if (ctx.arg.emachine == EM_MIPS)
     return handleMipsTlsRelocation(ctx, type, sym, *sec, offset, addend, expr);
 
-  // LoongArch does not yet implement transition from TLSDESC to LE/IE, so
-  // generate TLSDESC dynamic relocation for the dynamic linker to handle.
-  if (ctx.arg.emachine == EM_LOONGARCH &&
-      oneof<RE_LOONGARCH_TLSDESC_PAGE_PC, R_TLSDESC, R_TLSDESC_PC,
-            R_TLSDESC_CALL>(expr)) {
-    if (expr != R_TLSDESC_CALL) {
-      sym.setFlags(NEEDS_TLSDESC);
-      sec->addReloc({expr, type, offset, addend, &sym});
-    }
-    return 1;
-  }
-
   bool isRISCV = ctx.arg.emachine == EM_RISCV;
 
   if (oneof<RE_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
-            R_TLSDESC_GOTPLT>(expr) &&
+            R_TLSDESC_GOTPLT, RE_LOONGARCH_TLSDESC_PAGE_PC>(expr) &&
       ctx.arg.shared) {
     // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12_I,CALL} reference a label. Do not
     // set NEEDS_TLSDESC on the label.
@@ -1369,10 +1357,14 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
     return 1;
   }
 
-  // LoongArch supports IE to LE optimization in non-extreme code model.
+  // LoongArch supports IE to LE, DESC GD/LD to IE/LE optimizations in
+  // non-extreme code model.
   bool execOptimizeInLoongArch =
       ctx.arg.emachine == EM_LOONGARCH &&
-      (type == R_LARCH_TLS_IE_PC_HI20 || type == R_LARCH_TLS_IE_PC_LO12);
+      (type == R_LARCH_TLS_IE_PC_HI20 || type == R_LARCH_TLS_IE_PC_LO12 ||
+       type == R_LARCH_TLS_DESC_PC_HI20 || type == R_LARCH_TLS_DESC_PC_LO12 ||
+       type == R_LARCH_TLS_DESC_LD || type == R_LARCH_TLS_DESC_CALL ||
+       type == R_LARCH_TLS_DESC_PCREL20_S2);
 
   // ARM, Hexagon, LoongArch and RISC-V do not support GD/LD to IE/LE
   // optimizations.
@@ -1431,9 +1423,23 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
     return 1;
   }
 
+  // LoongArch does not support transition from TLSDESC to LE/IE in the extreme
+  // code model, in which NEEDS_TLSDESC should set, rather than NEEDS_TLSGD. So
+  // we check independently.
+  if (ctx.arg.emachine == EM_LOONGARCH &&
+      oneof<RE_LOONGARCH_TLSDESC_PAGE_PC, R_TLSDESC, R_TLSDESC_PC,
+            R_TLSDESC_CALL>(expr) &&
+      !execOptimize) {
+    if (expr != R_TLSDESC_CALL) {
+      sym.setFlags(NEEDS_TLSDESC);
+      sec->addReloc({expr, type, offset, addend, &sym});
+    }
+    return 1;
+  }
+
   if (oneof<RE_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSDESC_GOTPLT, R_TLSGD_GOT, R_TLSGD_GOTPLT, R_TLSGD_PC,
-            RE_LOONGARCH_TLSGD_PAGE_PC>(expr)) {
+            RE_LOONGARCH_TLSGD_PAGE_PC, RE_LOONGARCH_TLSDESC_PAGE_PC>(expr)) {
     if (!execOptimize) {
       sym.setFlags(NEEDS_TLSGD);
       sec->addReloc({expr, type, offset, addend, &sym});
diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h
index d2a77bc953109..02ddf707fd950 100644
--- a/lld/ELF/Relocations.h
+++ b/lld/ELF/Relocations.h
@@ -131,6 +131,7 @@ enum RelExpr {
   RE_LOONGARCH_GOT_PAGE_PC,
   RE_LOONGARCH_TLSGD_PAGE_PC,
   RE_LOONGARCH_TLSDESC_PAGE_PC,
+  RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC,
 };
 
 // Architecture-neutral representation of relocation.
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 593d5636f2455..ddfa24d9cacf5 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -1229,6 +1229,8 @@ SymbolAssignment *ScriptParser::readSymbolAssignment(StringRef name) {
 // This is an operator-precedence parser to parse a linker
 // script expression.
 Expr ScriptParser::readExpr() {
+  if (atEOF())
+    return []() { return 0; };
   // Our lexer is context-aware. Set the in-expression bit so that
   // they apply different tokenization rules.
   SaveAndRestore saved(lexState, State::Expr);
diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst
new file mode 100644
index 0000000000000..985decf6c7db8
--- /dev/null
+++ b/lld/docs/DTLTO.rst
@@ -0,0 +1,42 @@
+Integrated Distributed ThinLTO (DTLTO)
+======================================
+
+Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend
+ThinLTO compilations via external distribution systems, such as Incredibuild,
+during the traditional link step.
+
+The implementation is documented here: https://llvm.org/docs/DTLTO.html.
+
+Currently, DTLTO is only supported in ELF LLD. Support will be added to other
+LLD flavours in the future.
+
+ELF LLD
+-------
+
+The command-line interface is as follows:
+
+- ``--thinlto-distributor=<path>``  
+  Specifies the file to execute as the distributor process. If specified,
+  ThinLTO backend compilations will be distributed.
+
+- ``--thinlto-remote-compiler=<path>``  
+  Specifies the path to the compiler that the distributor process will use for
+  backend compilations. The compiler invoked must match the version of LLD.
+
+- ``--thinlto-distributor-arg=<arg>``  
+  Specifies ``<arg>`` on the command line when invoking the distributor.
+  Can be specified multiple times.
+
+- ``--thinlto-remote-compiler-arg=<arg>``  
+  Appends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
+  Options that introduce extra input/output files may cause miscompilation if
+  the distribution system does not automatically handle pushing/fetching them to
+  remote nodes. In such cases, configure the distributor - possibly using
+  ``--thinlto-distributor-arg=`` - to manage these dependencies. See the
+  distributor documentation for details.
+
+Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
+Currently, other options are silently accepted but do not have the intended
+effect. Support for such options will be expanded in the future.
diff --git a/lld/docs/index.rst b/lld/docs/index.rst
index 8260461c36905..69792e3b575be 100644
--- a/lld/docs/index.rst
+++ b/lld/docs/index.rst
@@ -147,3 +147,4 @@ document soon.
    ELF/start-stop-gc
    ELF/warn_backrefs
    MachO/index
+   DTLTO
diff --git a/lld/test/COFF/exported-dllmain.test b/lld/test/COFF/exported-dllmain.test
new file mode 100644
index 0000000000000..fcf6ed1005379
--- /dev/null
+++ b/lld/test/COFF/exported-dllmain.test
@@ -0,0 +1,57 @@
+REQUIRES: x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows a.s -o a.obj
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows b1.s -o b1.obj
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows b2.s -o b2.obj
+
+### This is the line where our problem occurs. Here, we export the DllMain symbol which shouldn't happen normally.
+RUN: lld-link b1.obj b2.obj -out:b.dll -dll -implib:b.lib -entry:DllMain -export:bar -export:DllMain
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows c.s -o c.obj
+RUN: lld-link -lib c.obj -out:c.lib
+
+### Later, if b.lib is provided before other libs/objs that export DllMain statically, we previously were using the dllimported DllMain from b.lib, which is wrong.
+RUN: lld-link a.obj b.lib c.lib -dll -out:out.dll -entry:DllMain 2>&1 | FileCheck -check-prefix=WARN %s
+RUN: lld-link a.obj b.lib c.lib -dll -out:out.dll -entry:DllMain -ignore:exporteddllmain 2>&1 | FileCheck -check-prefix=IGNORED --allow-empty %s
+RUN: llvm-objdump --private-headers -d out.dll | FileCheck -check-prefix=DISASM %s
+
+WARN: lld-link: warning: b.lib: skipping exported DllMain symbol [exporteddllmain]
+IGNORED-NOT: lld-link: warning: b.lib: skipping exported DllMain symbol [exporteddllmain]
+
+DISASM: The Import Tables:
+DISASM: DLL Name: b.dll
+DISASM-NOT: DllMain
+DISASM: bar
+DISASM: Disassembly of section .text:
+DISASM-EMPTY:
+DISASM:      b8 01 00 00 00               movl    $0x1, %eax
+DISASM-NEXT: c3                           retq
+
+#--- a.s
+        .text
+        .globl foo
+foo:
+        call *__imp_bar(%rip)
+        ret
+
+#--- b1.s
+        .text
+        .globl bar
+bar:
+        ret
+
+#--- b2.s
+        .text
+        .globl DllMain
+DllMain:
+        xor %eax, %eax
+        ret
+
+#--- c.s
+        .text
+        .globl DllMain
+DllMain:
+        movl $1, %eax
+        ret
diff --git a/lld/test/ELF/dtlto/files.test b/lld/test/ELF/dtlto/files.test
new file mode 100644
index 0000000000000..54bcf587f46b9
--- /dev/null
+++ b/lld/test/ELF/dtlto/files.test
@@ -0,0 +1,99 @@
+# REQUIRES: x86
+
+## Test that the LLD options --save-temps, --thinlto-emit-index-files,
+## and --thinlto-emit-imports-files function correctly with DTLTO.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+
+## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the
+## linker.
+RUN: opt -thinlto-bc t1.ll -o t1.bc
+RUN: opt -thinlto-bc t2.ll -o t2.bc
+RUN: cp t1.bc t3.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o
+RUN: llc t2.ll --filetype=obj -o t2.o
+
+## Create response file containing shared ThinLTO linker arguments.
+## --start-lib/--end-lib is used to test the special case where unused lazy
+## bitcode inputs result in empty index/imports files.
+## Note that mock.py does not do any compilation; instead, it simply writes
+## the contents of the object files supplied on the command line into the
+## output object files in job order.
+RUN: echo "t1.bc t2.bc --start-lib t3.bc --end-lib -o my.elf \
+RUN:   --thinlto-distributor=\"%python\" \
+RUN:   --thinlto-distributor-arg=\"%llvm_src_root/utils/dtlto/mock.py\" \
+RUN:   --thinlto-distributor-arg=t1.o \
+RUN:   --thinlto-distributor-arg=t2.o" > l.rsp
+
+## Check that without extra flags, no index/imports files are produced and
+## backend temp files are removed.
+RUN: ld.lld @l.rsp
+RUN: ls | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that index files are created with --thinlto-emit-index-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that imports files are created with --thinlto-emit-imports-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,IMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that both index and imports files are emitted with both flags.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## Check that backend temp files are retained with --save-temps.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that all files are emitted when all options are enabled.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## JSON jobs description, retained with --save-temps.
+## Note that DTLTO temporary files include a PID component.
+NOBACKEND-NOT:      {{^}}my.[[#]].dist-file.json{{$}}
+BACKEND:            {{^}}my.[[#]].dist-file.json{{$}}
+
+## Index/imports files for t1.bc.
+NOIMPORTSFILES-NOT: {{^}}t1.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t1.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t1.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t1.bc.thinlto.bc{{$}}
+
+## Index/imports files for t2.bc.
+NOIMPORTSFILES-NOT: {{^}}t2.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t2.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t2.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t2.bc.thinlto.bc{{$}}
+
+## Empty index/imports files for unused t3.bc.
+NOEMPTYIMPORTS-NOT: {{^}}t3.bc.imports{{$}}
+EMPTYIMPORTS:       {{^}}t3.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t3.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t3.bc.thinlto.bc{{$}}
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/options.test b/lld/test/ELF/dtlto/options.test
new file mode 100644
index 0000000000000..1a10d18140f27
--- /dev/null
+++ b/lld/test/ELF/dtlto/options.test
@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO options are passed correctly to the distributor and
+## remote compiler.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt -thinlto-bc foo.ll -o foo.o
+
+## Note: validate.py does not perform any compilation. Instead, it validates the
+## received JSON, pretty-prints the JSON and the supplied arguments, and then
+## exits with an error. This allows FileCheck directives to verify the
+## distributor inputs.
+RUN: not ld.lld foo.o \
+RUN:   -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+RUN:   --thinlto-distributor-arg=darg1=10 \
+RUN:   --thinlto-distributor-arg=darg2=20 \
+RUN:   --thinlto-remote-compiler=my_clang.exe \
+RUN:   --thinlto-remote-compiler-arg=carg1=20 \
+RUN:   --thinlto-remote-compiler-arg=carg2=30 2>&1 | FileCheck %s
+
+CHECK: distributor_args=['darg1=10', 'darg2=20']
+
+CHECK: "linker_output": "my.elf"
+
+CHECK: "my_clang.exe"
+CHECK: "carg1=20"
+CHECK: "carg2=30"
+
+CHECK: error: DTLTO backend compilation: cannot open native object file:
+
+#--- foo.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/partitions.test b/lld/test/ELF/dtlto/partitions.test
new file mode 100644
index 0000000000000..fbcbc0713ce23
--- /dev/null
+++ b/lld/test/ELF/dtlto/partitions.test
@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO works with more than one LTO partition.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@f/@t1/g' f.ll > t1.ll
+RUN: sed 's/@f/@t2/g' f.ll > t2.ll
+
+## Generate bitcode.
+RUN: opt f.ll -o full.bc
+RUN: opt -thinlto-bc t1.ll -o thin1.bc
+RUN: opt -thinlto-bc t2.ll -o thin2.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o thin1.o
+RUN: llc t2.ll --filetype=obj -o thin2.o
+
+## Link with 3 LTO partitions.
+RUN: ld.lld full.bc thin1.bc thin2.bc \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=thin1.o \
+RUN:   --thinlto-distributor-arg=thin2.o \
+RUN:   --save-temps \
+RUN:   --lto-partitions=3
+
+## DTLTO temporary object files include the task number and a PID component. The
+## task number should incorporate the LTO partition number.
+RUN: ls | sort | FileCheck %s
+CHECK: {{^}}thin1.3.[[PID:[a-zA-Z0-9_]+]].native.o{{$}}
+CHECK: {{^}}thin2.4.[[PID]].native.o{{$}}
+
+#--- f.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+  ret void
+}
diff --git a/lld/test/ELF/linkerscript/align-section.test b/lld/test/ELF/linkerscript/align-section.test
index 7a28fef2076ed..1324be24a9328 100644
--- a/lld/test/ELF/linkerscript/align-section.test
+++ b/lld/test/ELF/linkerscript/align-section.test
@@ -1,7 +1,24 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux /dev/null -o %t.o
-# RUN: ld.lld -o %t --script %s %t.o -shared
+# RUN: rm -rf %t && split-file %s %t && cd %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux /dev/null -o a.o
+# RUN: ld.lld --script a.t a.o -shared
 
 # lld shouldn't crash.
 
+#--- a.t
 SECTIONS { .foo : ALIGN(2M) {} }
+
+# RUN: not ld.lld --script b.t 2>&1 | FileCheck %s --match-full-lines --strict-whitespace
+
+# lld should not crash and report the error properly.
+
+# CHECK:{{.*}} error: b.t:3: malformed number: :
+# CHECK:>>>   S :ALIGN(4096) {}
+# CHECK:>>>     ^
+
+#--- b.t
+SECTIONS
+{
+  S :ALIGN(4096) {}
+}
diff --git a/lld/test/ELF/loongarch-relax-tlsdesc.s b/lld/test/ELF/loongarch-relax-tlsdesc.s
index 9ce7c5881ca96..5f4368343471c 100644
--- a/lld/test/ELF/loongarch-relax-tlsdesc.s
+++ b/lld/test/ELF/loongarch-relax-tlsdesc.s
@@ -9,19 +9,18 @@
 # RUN: llvm-readobj -r -x .got a.64.so | FileCheck --check-prefix=GD64-RELA %s
 # RUN: llvm-objdump --no-show-raw-insn -dr -h a.64.so | FileCheck %s --check-prefix=GD64
 
-## FIXME: The transition from TLSDESC to IE/LE has not yet been implemented.
-## Keep the dynamic relocations and hand them over to dynamic linker.
-
-# RUN: ld.lld --relax -e 0 -z now a.64.o c.64.o -o a.64.le
-# RUN: llvm-readobj -r -x .got a.64.le | FileCheck --check-prefix=LE64-RELA %s
-# RUN: llvm-objdump --no-show-raw-insn -d -h a.64.le | FileCheck %s --check-prefix=LE64
+## FIXME: IE/LE relaxation have not yet been implemented, --relax/--no-relax obtain the same results.
+## Transition from TLSDESC to IE/LE. Also check --emit-relocs.
+# RUN: ld.lld -e 0 -z now --emit-relocs a.64.o c.64.o -o a.64.le
+# RUN: llvm-readobj -r -x .got a.64.le 2>&1 | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -dr -h a.64.le | FileCheck %s --check-prefix=LE64
 
 # RUN: ld.lld --no-relax -e 0 -z now a.64.o c.64.o -o a.64.le.norelax
 # RUN: llvm-objdump --no-show-raw-insn -d -h a.64.le.norelax | FileCheck %s --check-prefix=LE64-NORELAX
 
-# RUN: ld.lld --relax -e 0 -z now a.64.o c.64.so -o a.64.ie
+# RUN: ld.lld --relax -e 0 -z now --emit-relocs a.64.o c.64.so -o a.64.ie
 # RUN: llvm-readobj -r -x .got a.64.ie | FileCheck --check-prefix=IE64-RELA %s
-# RUN: llvm-objdump --no-show-raw-insn -d -h a.64.ie | FileCheck %s --check-prefix=IE64
+# RUN: llvm-objdump --no-show-raw-insn -dr -h a.64.ie | FileCheck %s --check-prefix=IE64
 
 # RUN: ld.lld --no-relax -e 0 -z now a.64.o c.64.so -o a.64.ie.norelax
 # RUN: llvm-objdump --no-show-raw-insn -d -h a.64.ie.norelax | FileCheck %s --check-prefix=IE64-NORELAX
@@ -71,172 +70,199 @@
 # GD64-NEXT:          jirl    $ra, $ra, 0
 # GD64-NEXT:          add.d   $a4, $a0, $tp
 
-# LE64-RELA:      .rela.dyn {
-# LE64-RELA-NEXT:   0x30280 R_LARCH_TLS_DESC64 - 0x8
-# LE64-RELA-NEXT:   0x30290 R_LARCH_TLS_DESC64 - 0x800
-# LE64-RELA-NEXT:   0x302A0 R_LARCH_TLS_DESC64 - 0x1000
-# LE64-RELA-NEXT:   0x302B0 R_LARCH_TLS_DESC64 - 0x7FF
-# LE64-RELA-NEXT: }
-# LE64-RELA:      Hex dump of section '.got':
-# LE64-RELA-NEXT: 0x00030280 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030290 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x000302a0 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x000302b0 00000000 00000000 00000000 00000000 .
-
-# LE64:   .got    00000040 0000000000030280
-
-## &.got[a]-. = 0x30280 - 0x20228 = 16406<<2
-# LE64:        20228: pcaddi  $a0, 16406
-# LE64-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NEXT:          jirl    $ra, $ra, 0
+# LE64-RELA: could not find section '.got'
+
+## a@tprel = 0x8
+# LE64:        20158: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          ori     $a0, $zero, 8
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
 # LE64-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30280+48 - 0x20238: 0x10 pages, page offset 0x2b0
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# LE64:        20238: pcalau12i $a0, 16
-# LE64-NEXT:          addi.d  $a0, $a0, 688
-# LE64-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# LE64:        2016c: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 b
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 b
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD b
+# LE64-NEXT:          ori     $a0, $zero, 2047
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL b
 # LE64-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30280+16 - 0x2024c: 0x10 pages, page offset 0x290
+## c@tprel = 0x800
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# LE64:        2024c: pcalau12i $a0, 16
+# LE64:        20180: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 c
 # LE64-NEXT:          addi.d  $t0, $zero, 0
-# LE64-NEXT:          addi.d  $a0, $a0, 656
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 c
 # LE64-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NEXT:          ld.d    $ra, $a0, 0
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD c
 # LE64-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NEXT:          jirl    $ra, $ra, 0
+# LE64-NEXT:          ori     $a0, $zero, 2048
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL c
 # LE64-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30280+32 - 0x2026c = 16397<<2
-# LE64:        2026c: pcaddi  $a0, 16397
-# LE64-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NEXT:          jirl    $ra, $ra, 0
+## d@tprel = 0x1000
+# LE64:        201a0: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 d
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 d
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          lu12i.w $a0, 1
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD d
+# LE64-NEXT:          ori     $a0, $a0, 0
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL d
 # LE64-NEXT:          add.d   $a4, $a0, $tp
 
-# LE64-NORELAX: .got    00000040 0000000000030288
-
-## &.got[a]-. = 0x30288 - 0x20228 = 0x10 pages, page offset 0x288
-# LE64-NORELAX:        20228: pcalau12i $a0, 16
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 648
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## a@tprel = 0x8
+# LE64-NORELAX:        20158: nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          ori     $a0, $zero, 8
 # LE64-NORELAX-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30288+48 - 0x2023c: 0x10 pages, page offset 0x2b8
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# LE64-NORELAX:        2023c: pcalau12i $a0, 16
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 696
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# LE64-NORELAX:        2016c: nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          ori     $a0, $zero, 2047
 # LE64-NORELAX-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30288+16 - 0x20250: 0x10 pages, page offset 0x298
+## c@tprel = 0x800
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# LE64-NORELAX:        20250: pcalau12i $a0, 16
+# LE64-NORELAX:        20180: nop
 # LE64-NORELAX-NEXT:          addi.d  $t0, $zero, 0
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 664
+# LE64-NORELAX-NEXT:          nop
 # LE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
+# LE64-NORELAX-NEXT:          nop
 # LE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+# LE64-NORELAX-NEXT:          ori     $a0, $zero, 2048
 # LE64-NORELAX-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30288+32 - 0x20270: 0x10 pages, page offset 0x2a8
-# LE64-NORELAX:        20270: pcalau12i $a0, 16
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 680
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## d@tprel = 0x1000
+# LE64-NORELAX:        201a0: nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          lu12i.w $a0, 1
+# LE64-NORELAX-NEXT:          ori     $a0, $a0, 0
 # LE64-NORELAX-NEXT:          add.d   $a4, $a0, $tp
 
 # IE64-RELA:      .rela.dyn {
-# IE64-RELA-NEXT:   0x30430 R_LARCH_TLS_DESC64 - 0x8
-# IE64-RELA-NEXT:   0x30460 R_LARCH_TLS_DESC64 - 0x7FF
-# IE64-RELA-NEXT:   0x30440 R_LARCH_TLS_DESC64 c 0x0
-# IE64-RELA-NEXT:   0x30450 R_LARCH_TLS_DESC64 d 0x0
+# IE64-RELA-NEXT:   0x30408 R_LARCH_TLS_TPREL64 c 0x0
+# IE64-RELA-NEXT:   0x30410 R_LARCH_TLS_TPREL64 d 0x0
 # IE64-RELA-NEXT: }
 # IE64-RELA:      Hex dump of section '.got':
-# IE64-RELA-NEXT: 0x00030430 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x00030440 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x00030450 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x00030460 00000000 00000000 00000000 00000000 .
+# IE64-RELA-NEXT: 0x00030408 00000000 00000000 00000000 00000000 .
 
-# IE64:   .got           00000040 0000000000030430
+# IE64:   .got           00000010 0000000000030408
 
 ## a and b are optimized to use LE. c and d are optimized to IE.
-## &.got[a]-. = 0x30430 - 0x202f8 = 16462<<2
-# IE64:        202f8: pcaddi  $a0, 16462
-# IE64-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NEXT:          jirl    $ra, $ra, 0
+## a@tprel = 0x8
+# IE64:        202c8: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          ori     $a0, $zero, 8
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
 # IE64-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30430+48 - 0x20308: 0x10 pages, page offset 0x460
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# IE64:        20308: pcalau12i $a0, 16
-# IE64-NEXT:          addi.d  $a0, $a0, 1120
-# IE64-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# IE64:        202dc: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 b
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 b
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD b
+# IE64-NEXT:          ori     $a0, $zero, 2047
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL b
 # IE64-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30430+16 - 0x2031c: 0x10 pages, page offset 0x440
+## &.got[c]-. = 0x30408 - 0x20300: 0x10 pages, page offset 0x408
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# IE64:        2031c: pcalau12i $a0, 16
+# IE64:        202f0: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 c
 # IE64-NEXT:          addi.d  $t0, $zero, 0
-# IE64-NEXT:          addi.d  $a0, $a0, 1088
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 c
 # IE64-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NEXT:          ld.d    $ra, $a0, 0
+# IE64-NEXT:          pcalau12i $a0, 16
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD c
 # IE64-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NEXT:          jirl    $ra, $ra, 0
+# IE64-NEXT:          ld.d    $a0, $a0, 1032
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL c
 # IE64-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30430+32 - 0x2033c = 16453<<2
-# IE64:        2033c: pcaddi  $a0, 16453
-# IE64-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NEXT:          jirl    $ra, $ra, 0
+## &.got[d]-. = 0x30408+8 - 0x20318: 0x10 pages, page offset 0x410
+# IE64:        20310: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 d
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 d
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          pcalau12i $a0, 16
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD d
+# IE64-NEXT:          ld.d    $a0, $a0, 1040
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL d
 # IE64-NEXT:          add.d   $a4, $a0, $tp
 
-# IE64-NORELAX: .got    00000040 0000000000030438
+# IE64-NORELAX: .got    00000010 0000000000030408
 
-## &.got[a]-. = 0x30438 - 0x202f8 = 0x10 pages, page offset 0x438
-# IE64-NORELAX:        202f8: pcalau12i $a0, 16
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1080
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## a@tprel = 0x8
+# IE64-NORELAX:        202c8: nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          ori     $a0, $zero, 8
 # IE64-NORELAX-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30438+48 - 0x2030c: 0x10 pages, page offset 0x468
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# IE64-NORELAX:        2030c: pcalau12i $a0, 16
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1128
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# IE64-NORELAX:        202dc: nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          ori     $a0, $zero, 2047
 # IE64-NORELAX-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30438+16 - 0x20320: 0x10 pages, page offset 0x448
+## &.got[c]-. = 0x30408 - 0x20300: 0x10 pages, page offset 0x408
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# IE64-NORELAX:        20320: pcalau12i $a0, 16
+# IE64-NORELAX:        202f0: nop
 # IE64-NORELAX-NEXT:          addi.d  $t0, $zero, 0
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1096
+# IE64-NORELAX-NEXT:          nop
 # IE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
+# IE64-NORELAX-NEXT:          pcalau12i $a0, 16
 # IE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+# IE64-NORELAX-NEXT:          ld.d    $a0, $a0, 1032
 # IE64-NORELAX-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30438+32 - 0x20340: 0x10 pages, page offset 0x458
-# IE64-NORELAX:        20340: pcalau12i $a0, 16
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1112
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## &.got[d]-. = 0x30408+8 - 0x20318: 0x10 pages, page offset 0x410
+# IE64-NORELAX:        20310: nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          pcalau12i $a0, 16
+# IE64-NORELAX-NEXT:          ld.d    $a0, $a0, 1040
 # IE64-NORELAX-NEXT:          add.d   $a4, $a0, $tp
 
 #--- a.s
 la.tls.desc $a0, a
 add.d $a1, $a0, $tp
 
-# ADDI.D does not have R_LARCH_RELAX. No relaxation.
+# ADDI.D does not have R_LARCH_RELAX. No relaxation when it is not optimized to IE/LE (--shared).
 pcalau12i $a0, %desc_pc_hi20(b)
 .reloc .-4, R_LARCH_RELAX, 0
 addi.d $a0, $a0, %desc_pc_lo12(b)
diff --git a/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s b/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s
index 99e21d9935197..422592980d28f 100644
--- a/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s
+++ b/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s
@@ -14,14 +14,14 @@
 # RUN: ld.lld -shared -z now a.64.o c.64.o -o rel.64.so -z rel
 # RUN: llvm-readobj -r -x .got rel.64.so | FileCheck --check-prefix=GD64-REL %s
 
-## FIXME: The transition frome TLSDESC to IE/LE has not yet been implemented.
-## Keep the dynamic relocations and hand them over to dynamic linker.
-
+## Transition from TLSDESC to IE/LE.
 # RUN: ld.lld -e 0 -z now a.64.o c.64.o -o a.64.le
-# RUN: llvm-readobj -r -x .got a.64.le | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-readobj -r -x .got a.64.le 2>&1 | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.le | FileCheck --check-prefix=LE64 %s
 
 # RUN: ld.lld -e 0 -z now a.64.o c.64.so -o a.64.ie
 # RUN: llvm-readobj -r -x .got a.64.ie | FileCheck --check-prefix=IE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.ie | FileCheck --check-prefix=IE64 %s
 
 ## 32-bit code is mostly the same. We only test a few variants.
 
@@ -68,25 +68,46 @@
 # GD64-NEXT:          jirl $ra, $ra, 0
 # GD64-NEXT:          add.d $a3, $a0, $tp
 
-# LE64-RELA:      .rela.dyn {
-# LE64-RELA-NEXT:   0x30240 R_LARCH_TLS_DESC64 - 0x8
-# LE64-RELA-NEXT:   0x30250 R_LARCH_TLS_DESC64 - 0x800
-# LE64-RELA-NEXT:   0x30260 R_LARCH_TLS_DESC64 - 0x7FF
-# LE64-RELA-NEXT: }
-# LE64-RELA:      Hex dump of section '.got':
-# LE64-RELA-NEXT: 0x00030240 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030250 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030260 00000000 00000000 00000000 00000000 .
+# LE64-RELA: could not find section '.got'
+
+# LE64-LABEL: <.text>:
+## st_value(a) = 8
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 8
+# LE64-NEXT:         add.d   $a1, $a0, $tp
+## st_value(b) = 2047
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2047
+# LE64-NEXT:         add.d   $a2, $a0, $tp
+## st_value(c) = 2048
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2048
+# LE64-NEXT:         add.d   $a3, $a0, $tp
 
 # IE64-RELA:      .rela.dyn {
-# IE64-RELA-NEXT:   0x303C8 R_LARCH_TLS_DESC64 - 0x8
-# IE64-RELA-NEXT:   0x303E8 R_LARCH_TLS_DESC64 - 0x7FF
-# IE64-RELA-NEXT:   0x303D8 R_LARCH_TLS_DESC64 c 0x0
+# IE64-RELA-NEXT:   0x30398 R_LARCH_TLS_TPREL64 c 0x0
 # IE64-RELA-NEXT: }
 # IE64-RELA:      Hex dump of section '.got':
-# IE64-RELA-NEXT: 0x000303c8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303d8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303e8 00000000 00000000 00000000 00000000 .
+# IE64-RELA-NEXT: 0x00030398 00000000 00000000                  .
+
+## a and b are optimized to use LE. c is optimized to IE.
+# IE64-LABEL: <.text>:
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 8
+# IE64-NEXT:         add.d   $a1, $a0, $tp
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 2047
+# IE64-NEXT:         add.d   $a2, $a0, $tp
+## &.got[c]-. = 0x30398 - 0x202ac: 0x10 pages, page offset 0x398
+# IE64-NEXT:         nop
+# IE64-NEXT:  202ac: pcalau12i $a0, 16
+# IE64-NEXT:         ld.d      $a0, $a0, 920
+# IE64-NEXT:         add.d   $a3, $a0, $tp
 
 # GD32-REL:      .rel.dyn {
 # GD32-REL-NEXT:    0x20264 R_LARCH_TLS_DESC32 -
diff --git a/lld/test/ELF/loongarch-tlsdesc.s b/lld/test/ELF/loongarch-tlsdesc.s
index bf09b1e4bbae9..3dc31210d7ddb 100644
--- a/lld/test/ELF/loongarch-tlsdesc.s
+++ b/lld/test/ELF/loongarch-tlsdesc.s
@@ -14,14 +14,14 @@
 # RUN: ld.lld -shared -z now a.64.o c.64.o -o rel.64.so -z rel
 # RUN: llvm-readobj -r -x .got rel.64.so | FileCheck --check-prefix=GD64-REL %s
 
-## FIXME: The transition frome TLSDESC to IE/LE has not yet been implemented.
-## Keep the dynamic relocations and hand them over to dynamic linker.
-
+## Transition from TLSDESC to IE/LE.
 # RUN: ld.lld -e 0 -z now a.64.o c.64.o -o a.64.le
-# RUN: llvm-readobj -r -x .got a.64.le | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-readobj -r -x .got a.64.le 2>&1 | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.le | FileCheck --check-prefix=LE64 %s
 
 # RUN: ld.lld -e 0 -z now a.64.o c.64.so -o a.64.ie
 # RUN: llvm-readobj -r -x .got a.64.ie | FileCheck --check-prefix=IE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.ie | FileCheck --check-prefix=IE64 %s
 
 ## 32-bit code is mostly the same. We only test a few variants.
 
@@ -71,25 +71,52 @@
 # GD64-NEXT:          jirl $ra, $ra, 0
 # GD64-NEXT:          add.d $a3, $a0, $tp
 
-# LE64-RELA:      .rela.dyn {
-# LE64-RELA-NEXT:   0x30250 R_LARCH_TLS_DESC64 - 0x8
-# LE64-RELA-NEXT:   0x30260 R_LARCH_TLS_DESC64 - 0x800
-# LE64-RELA-NEXT:   0x30270 R_LARCH_TLS_DESC64 - 0x7FF
-# LE64-RELA-NEXT: }
-# LE64-RELA:      Hex dump of section '.got':
-# LE64-RELA-NEXT: 0x00030250 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030260 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030270 00000000 00000000 00000000 00000000 .
+# LE64-RELA: could not find section '.got'
+
+# LE64-LABEL: <.text>:
+## st_value(a) = 8
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 8
+# LE64-NEXT:         add.d   $a1, $a0, $tp
+## st_value(b) = 2047
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2047
+# LE64-NEXT:         add.d   $a2, $a0, $tp
+## st_value(c) = 2048
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2048
+# LE64-NEXT:         add.d   $a3, $a0, $tp
 
 # IE64-RELA:      .rela.dyn {
-# IE64-RELA-NEXT:   0x303D8 R_LARCH_TLS_DESC64 - 0x8
-# IE64-RELA-NEXT:   0x303F8 R_LARCH_TLS_DESC64 - 0x7FF
-# IE64-RELA-NEXT:   0x303E8 R_LARCH_TLS_DESC64 c 0x0
+# IE64-RELA-NEXT:   0x303A8 R_LARCH_TLS_TPREL64 c 0x0
 # IE64-RELA-NEXT: }
 # IE64-RELA:      Hex dump of section '.got':
-# IE64-RELA-NEXT: 0x000303d8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303e8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303f8 00000000 00000000 00000000 00000000 .
+# IE64-RELA-NEXT: 0x000303a8 00000000 00000000                  .
+
+## a and b are optimized to use LE. c is optimized to IE.
+# IE64-LABEL: <.text>:
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 8
+# IE64-NEXT:         add.d   $a1, $a0, $tp
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 2047
+# IE64-NEXT:         add.d   $a2, $a0, $tp
+## &.got[c]-. = 0x303a8 - 0x202b8: 0x10 pages, page offset 0x3a8
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:  202b8: pcalau12i $a0, 16
+# IE64-NEXT:         ld.d      $a0, $a0, 936
+# IE64-NEXT:         add.d   $a3, $a0, $tp
 
 # GD32-REL:      .rel.dyn {
 # GD32-REL-NEXT:    0x20270 R_LARCH_TLS_DESC32 -
diff --git a/lld/test/ELF/lto/lazy-debug.ll b/lld/test/ELF/lto/lazy-debug.ll
new file mode 100644
index 0000000000000..48d176f0f6bf6
--- /dev/null
+++ b/lld/test/ELF/lto/lazy-debug.ll
@@ -0,0 +1,107 @@
+; REQUIRES: aarch64
+;; Regression test for a case in lazy debuginfo loading.
+;; The bug would cause ld.lld to crash.
+
+; RUN: split-file %s %t
+; RUN: llvm-as %t/hda_codec.s -o %t/hda_codec.o
+; RUN: llvm-as %t/hda_bind.s -o %t/hda_bind.o
+; RUN: ld.lld -EL -maarch64elf -r %t/hda_bind.o %t/hda_codec.o -o %t/hda_codec
+
+;--- hda_codec.s
+; ModuleID = 'hda_codec.o'
+source_filename = "hda_codec.i"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+%struct.anon = type { i32 }
+
+@hda_set_power_state_codec = hidden local_unnamed_addr global %struct.anon zeroinitializer, align 4, !dbg !0
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable
+define hidden void @snd_hda_codec_shutdown() local_unnamed_addr #0 !dbg !19 {
+entry:
+  ret void, !dbg !22
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12, !13, !14, !15, !16, !17}
+!llvm.ident = !{!18}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "hda_set_power_state_codec", scope: !2, file: !5, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C11, file: !3, producer: "clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "hda_codec.i", directory: "/tmp")
+!4 = !{!0}
+!5 = !DIFile(filename: "hda_codec.i", directory: "/tmp", checksumkind: CSK_MD5, checksum: "c192644b468953345ff9647026173a7b")
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !5, line: 1, size: i64 32, offset: i64 0, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "mfg", scope: !6, file: !5, line: 2, baseType: !9, size: i64 32, offset: i64 0)
+!9 = !DIBasicType(name: "int", size: i64 32, encoding: DW_ATE_signed)
+!10 = !{i32 7, !"Dwarf Version", i32 5}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{i32 8, !"PIC Level", i32 2}
+!14 = !{i32 7, !"PIE Level", i32 2}
+!15 = !{i32 7, !"uwtable", i32 2}
+!16 = !{i32 7, !"frame-pointer", i32 1}
+!17 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!18 = !{!"clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)"}
+!19 = distinct !DISubprogram(name: "snd_hda_codec_shutdown", scope: !5, file: !5, line: 4, type: !20, scopeLine: 4, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!20 = !DISubroutineType(types: !21)
+!21 = !{null}
+!22 = !DILocation(line: 4, column: 32, scope: !19)
+
+^0 = module: (path: "hda_codec.o", hash: (1120894731, 3099354915, 309166549, 2100129435, 1932081428))
+^1 = gv: (name: "snd_hda_codec_shutdown", summaries: (function: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 1, funcFlags: (readNone: 1, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0)))) ; guid = 1539195202824839354
+^2 = gv: (name: "hda_set_power_state_codec", summaries: (variable: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), varFlags: (readonly: 1, writeonly: 1, constant: 0)))) ; guid = 10300548032946263328
+^3 = flags: 8
+^4 = blockcount: 0
+
+;--- hda_bind.s
+; ModuleID = 'hda_bind.o'
+source_filename = "hda_bind.i"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define hidden void @hda_codec_driver_shutdown() local_unnamed_addr #0 !dbg !11 {
+entry:
+  tail call void @snd_hda_codec_shutdown() #2, !dbg !15
+  ret void, !dbg !16
+}
+
+declare void @snd_hda_codec_shutdown(...) local_unnamed_addr #1
+
+attributes #0 = { nounwind uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+attributes #1 = { "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "hda_bind.i", directory: "/tmp")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"PIE Level", i32 2}
+!7 = !{i32 7, !"uwtable", i32 2}
+!8 = !{i32 7, !"frame-pointer", i32 1}
+!9 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!10 = !{!"clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)"}
+!11 = distinct !DISubprogram(name: "hda_codec_driver_shutdown", scope: !12, file: !12, line: 2, type: !13, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!12 = !DIFile(filename: "hda_bind.i", directory: "/tmp", checksumkind: CSK_MD5, checksum: "5907dd04e8964940b57448f37db201c6")
+!13 = !DISubroutineType(types: !14)
+!14 = !{null}
+!15 = !DILocation(line: 2, column: 36, scope: !11)
+!16 = !DILocation(line: 2, column: 62, scope: !11)
+
+^0 = module: (path: "hda_bind.o", hash: (1958332034, 2012675483, 855691486, 2017350850, 2779827776))
+^1 = gv: (name: "snd_hda_codec_shutdown") ; guid = 1539195202824839354
+^2 = gv: (name: "hda_codec_driver_shutdown", summaries: (function: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1, tail: 1))))) ; guid = 12817427500962331703
+^3 = flags: 8
+^4 = blockcount: 0
diff --git a/lld/test/MachO/icf-safe-thunks-dwarf.ll b/lld/test/MachO/icf-safe-thunks-dwarf.ll
index 248f1424a8081..7fae4124f793b 100644
--- a/lld/test/MachO/icf-safe-thunks-dwarf.ll
+++ b/lld/test/MachO/icf-safe-thunks-dwarf.ll
@@ -186,7 +186,7 @@ entry:
   ret i64 add (i64 add (i64 ptrtoint (ptr @func_A to i64), i64 ptrtoint (ptr @func_B to i64)), i64 ptrtoint (ptr @func_C to i64)), !dbg !27
 }
 
-attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!6, !7, !8, !9, !10, !11}
@@ -254,7 +254,7 @@ entry:
   ret i64 add (i64 add (i64 ptrtoint (ptr @func_D to i64), i64 ptrtoint (ptr @func_E to i64)), i64 ptrtoint (ptr @func_F to i64)), !dbg !27
 }
 
-attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!6, !7, !8, !9, !10, !11}
diff --git a/lld/test/MachO/icf-safe-thunks.ll b/lld/test/MachO/icf-safe-thunks.ll
index 6505930038732..12f1e81bdf3e8 100644
--- a/lld/test/MachO/icf-safe-thunks.ll
+++ b/lld/test/MachO/icf-safe-thunks.ll
@@ -244,8 +244,8 @@ define void @take_func_addr() local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { mustprogress nofree noinline norecurse nounwind ssp memory(readwrite, argmem: none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
-attributes #1 = { mustprogress nofree noinline norecurse nounwind ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress nofree noinline norecurse nounwind ssp memory(readwrite, argmem: none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+attributes #1 = { mustprogress nofree noinline norecurse nounwind ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.module.flags = !{!0, !1, !2, !3}
 
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index 9e6b0e839d9a8..10f556567cdc8 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -36,6 +36,7 @@
 
 llvm_config.use_default_substitutions()
 llvm_config.use_lld()
+config.substitutions.append(("%llvm_src_root", config.llvm_src_root))
 
 tool_patterns = [
     "llc",
diff --git a/lld/test/wasm/compress-relocs.ll b/lld/test/wasm/compress-relocs.ll
deleted file mode 100644
index cea9f3476e996..0000000000000
--- a/lld/test/wasm/compress-relocs.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc -filetype=obj %s -o %t.o
-; RUN: llvm-mc -mattr=+call-indirect-overlong -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/call-indirect.s -o %t2.o
-; RUN: wasm-ld --export-dynamic -o %t.wasm %t2.o %t.o
-; RUN: obj2yaml %t.wasm | FileCheck %s
-; RUN: wasm-ld --export-dynamic -O2 -o %t-opt.wasm %t2.o %t.o
-; RUN: obj2yaml %t-opt.wasm | FileCheck %s
-; RUN: not wasm-ld --compress-relocations -o %t-compressed.wasm %t2.o %t.o 2>&1 | FileCheck %s -check-prefix=ERROR
-; RUN: wasm-ld --export-dynamic --strip-debug --compress-relocations -o %t-compressed.wasm %t2.o %t.o
-; RUN: obj2yaml %t-compressed.wasm | FileCheck %s -check-prefix=COMPRESS
-
-target triple = "wasm32-unknown-unknown-wasm"
-
-define i32 @foo() {
-entry:
-  ret i32 2
-}
-
-define void @_start() local_unnamed_addr {
-entry:
-  ret void
-}
-
-; ERROR: wasm-ld: error: --compress-relocations is incompatible with output debug information. Please pass --strip-debug or --strip-all
-
-; CHECK:    Body:            41002802808880800011808080800080808080001A41002802848880800011818080800080808080001A0B
-; COMPRESS: Body:            4100280280081100001A4100280284081101001A0B
diff --git a/lld/test/wasm/compress-relocs.s b/lld/test/wasm/compress-relocs.s
new file mode 100644
index 0000000000000..41d4ff567d501
--- /dev/null
+++ b/lld/test/wasm/compress-relocs.s
@@ -0,0 +1,129 @@
+# RUN: llvm-mc -mattr=+reference-types,+exception-handling -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
+# RUN: wasm-ld --export-dynamic -o %t.wasm %t.o
+# RUN: llvm-objdump -d %t.wasm | FileCheck %s
+# RUN: wasm-ld --export-dynamic -O2 -o %t-opt.wasm %t.o
+# RUN: llvm-objdump -d %t-opt.wasm | FileCheck %s
+# RUN: not wasm-ld --compress-relocations -o %t-compressed.wasm %t.o 2>&1 | FileCheck %s -check-prefix=ERROR
+# RUN: wasm-ld --export-dynamic --strip-debug --compress-relocations -o %t-compressed.wasm %t.o
+# RUN: llvm-objdump -d %t-compressed.wasm | FileCheck %s -check-prefix=COMPRESS
+
+  .globl _start
+_start:
+  .functype _start () -> ()
+  end_function
+
+  .globl func_ret_i64
+func_ret_i64:
+  .functype func_ret_i64 () -> (i64)
+  i64.const 1
+  end_function
+
+  .globl func_ret_i32
+func_ret_i32:
+  .functype func_ret_i32 () -> (i32)
+  i32.const 2
+  end_function
+
+i32_global:
+  .globaltype i32_global, i32
+
+i32_tag:
+  .tagtype i32_tag i32
+
+  .globl test_memory_and_indirect_call_relocs
+test_memory_and_indirect_call_relocs:
+  .functype test_memory_and_indirect_call_relocs () -> ()
+  i32.const indirect_func_ret_i64 # R_WASM_MEMORY_ADDR_SLEB
+  i32.load  0
+  call_indirect () -> (i64)       # R_WASM_TYPE_INDEX_LEB, R_WASM_TABLE_NUMBER_LEB
+  drop
+  i32.const 0
+  i32.load  indirect_func_ret_i32 # R_WASM_MEMORY_ADDR_LEB
+  call_indirect () -> (i32)
+  drop
+  i32.const func_ret_i64          # R_WASM_TABLE_INDEX_SLEB
+  call_indirect () -> (i64)
+  drop
+  end_function
+
+# CHECK:    test_memory_and_indirect_call_relocs
+# CHECK:      41 90 88 80 80 00                 i32.const      1040
+# CHECK:      11 80 80 80 80 00 80 80 80 80 00  call_indirect  0
+# CHECK:      28 02 94 88 80 80 00              i32.load       1044
+# CHECK:      11 81 80 80 80 00 80 80 80 80 00  call_indirect  1
+# CHECK:      41 81 80 80 80 00                 i32.const      1
+# CHECK:      11 80 80 80 80 00 80 80 80 80 00  call_indirect  0
+# COMPRESS: test_memory_and_indirect_call_relocs
+# COMPRESS:   41 90 08                          i32.const      1040
+# COMPRESS:   11 00 00                          call_indirect  0
+# COMPRESS:   28 02 94 08                       i32.load       1044
+# COMPRESS:   11 01 00                          call_indirect  1
+# COMPRESS:   41 01                             i32.const      1
+# COMPRESS:   11 00 00                          call_indirect  0
+
+  .globl test_simple_index_relocs
+test_simple_index_relocs:
+  .functype test_simple_index_relocs () -> ()
+  call       func_ret_i32 # R_WASM_FUNCTION_INDEX_LEB
+  global.set i32_global   # R_WASM_GLOBAL_INDEX_LEB
+  i32.const  0
+  throw      i32_tag      # R_WASM_TAG_INDEX_LEB
+  end_function
+
+# CHECK:    test_simple_index_relocs
+# CHECK:      10 82 80 80 80 00  call        2
+# CHECK:      24 81 80 80 80 00  global.set  1
+# CHECK:      08 80 80 80 80 00  throw       0
+# COMPRESS: test_simple_index_relocs
+# COMPRESS:   10 02              call        2
+# COMPRESS:   24 01              global.set  1
+# COMPRESS:   08 00              throw       0
+
+  .globl test_relative_relocs
+test_relative_relocs:
+  .functype test_relative_relocs () -> ()
+  i32.const indirect_func_ret_i64@MBREL # R_WASM_MEMORY_ADDR_REL_SLEB
+  drop
+  i32.const func_ret_i32@TBREL          # R_WASM_TABLE_INDEX_REL_SLEB
+  drop
+  i32.const i32_tls_data@TLSREL         # R_WASM_MEMORY_ADDR_TLS_SLEB
+  drop
+  end_function
+
+# CHECK:    test_relative_relocs
+# CHECK:      41 90 88 80 80 00  i32.const  1040
+# CHECK:      41 81 80 80 80 00  i32.const  1
+# CHECK:      41 83 80 80 80 00  i32.const  3
+# COMPRESS: test_relative_relocs
+# COMPRESS:   41 90 08           i32.const  1040
+# COMPRESS:   41 01              i32.const  1
+# COMPRESS:   41 03              i32.const  3
+
+# ERROR: wasm-ld: error: --compress-relocations is incompatible with output debug information. Please pass --strip-debug or --strip-all
+
+  .section .tdata,"T",@
+  .int8 0
+  .int8 0
+  .int8 0
+i32_tls_data:
+  .int32 65
+  .size  i32_tls_data, 4
+
+  .section .data,"",@
+  .p2align 4
+indirect_func_ret_i64:
+  .int32 func_ret_i64
+  .size  indirect_func_ret_i64, 4
+
+indirect_func_ret_i32:
+  .int32 func_ret_i32
+  .size  indirect_func_ret_i32, 4
+
+.section .custom_section.target_features,"",@
+  .int8 2
+  .int8 43
+  .int8 7
+  .ascii  "atomics"
+  .int8 43
+  .int8 11
+  .ascii  "bulk-memory"
diff --git a/lld/test/wasm/compress-relocs64.s b/lld/test/wasm/compress-relocs64.s
new file mode 100644
index 0000000000000..44e7a089275bb
--- /dev/null
+++ b/lld/test/wasm/compress-relocs64.s
@@ -0,0 +1,92 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm64-unknown-unknown %s -o %t.o
+# RUN: wasm-ld -mwasm64 --export-dynamic -o %t.wasm %t.o
+# RUN: llvm-objdump -d %t.wasm | FileCheck %s
+# RUN: wasm-ld -mwasm64 --export-dynamic -O2 -o %t-opt.wasm %t.o
+# RUN: llvm-objdump -d %t-opt.wasm | FileCheck %s
+# RUN: wasm-ld -mwasm64 --export-dynamic --strip-debug --compress-relocations -o %t-compressed.wasm %t.o
+# RUN: llvm-objdump -d %t-compressed.wasm | FileCheck %s -check-prefix=COMPRESS
+
+  .globl _start
+_start:
+  .functype _start () -> ()
+  end_function
+
+  .globl func_ret_i64
+func_ret_i64:
+  .functype func_ret_i64 () -> (i64)
+  i64.const 1
+  end_function
+
+  .globl func_ret_i32
+func_ret_i32:
+  .functype func_ret_i32 () -> (i32)
+  i32.const 2
+  end_function
+
+  .globl test_memory_and_indirect_call_relocs
+test_memory_and_indirect_call_relocs:
+  .functype test_memory_and_indirect_call_relocs () -> ()
+  i64.const indirect_func_ret_i64 # R_WASM_MEMORY_ADDR_SLEB64
+  drop
+  i64.const 0
+  i64.load  indirect_func_ret_i32 # R_WASM_MEMORY_ADDR_LEB64
+  drop
+  i64.const func_ret_i64          # R_WASM_TABLE_INDEX_SLEB64
+  drop
+  end_function
+
+# CHECK:    test_memory_and_indirect_call_relocs
+# CHECK:      42 90 88 80 80 80 80 80 80 80 00     i64.const 1040
+# CHECK:      29 03 98 88 80 80 80 80 80 80 80 00  i64.load  1048
+# CHECK:      42 81 80 80 80 80 80 80 80 80 00     i64.const 1
+# COMPRESS: test_memory_and_indirect_call_relocs
+# COMPRESS:   42 90 08                             i64.const 1040
+# COMPRESS:   29 03 98 08                          i64.load  1048
+# COMPRESS:   42 01                                i64.const 1
+
+  .globl test_relative_relocs
+test_relative_relocs:
+  .functype test_relative_relocs () -> ()
+  i64.const indirect_func_ret_i64@MBREL # R_WASM_MEMORY_ADDR_REL_SLEB64
+  drop
+  i64.const func_ret_i32@TBREL          # R_WASM_TABLE_INDEX_REL_SLEB64
+  drop
+  i64.const i32_tls_data@TLSREL         # R_WASM_MEMORY_ADDR_TLS_SLEB64
+  drop
+  end_function
+
+# CHECK:    test_relative_relocs
+# CHECK:      42 90 88 80 80 80 80 80 80 80 00  i64.const 1040
+# CHECK:      42 81 80 80 80 80 80 80 80 80 00  i64.const 1
+# CHECK:      42 83 80 80 80 80 80 80 80 80 00  i64.const 3
+# COMPRESS: test_relative_relocs
+# COMPRESS:   42 90 08                          i64.const 1040
+# COMPRESS:   42 01                             i64.const 1
+# COMPRESS:   42 03                             i64.const 3
+
+  .section .tdata,"T",@
+  .int8 0
+  .int8 0
+  .int8 0
+i32_tls_data:
+  .int32 65
+  .size  i32_tls_data, 4
+
+  .section .data,"",@
+  .p2align 4
+indirect_func_ret_i64:
+  .int64 func_ret_i64
+  .size  indirect_func_ret_i64, 8
+
+indirect_func_ret_i32:
+  .int64 func_ret_i32
+  .size  indirect_func_ret_i32, 8
+
+.section .custom_section.target_features,"",@
+  .int8 2
+  .int8 43
+  .int8 7
+  .ascii  "atomics"
+  .int8 43
+  .int8 11
+  .ascii  "bulk-memory"
diff --git a/lld/test/wasm/lto/Inputs/libcall-archive.ll b/lld/test/wasm/lto/Inputs/libcall-archive.ll
index 30764af83e673..9722b283e692e 100644
--- a/lld/test/wasm/lto/Inputs/libcall-archive.ll
+++ b/lld/test/wasm/lto/Inputs/libcall-archive.ll
@@ -5,4 +5,12 @@ define void @memcpy() #0 {
   ret void
 }
 
+define float @acosf(float %x) {
+  ret float %x
+}
+
+define i128 @__umodti3(i128 %a, i128 %b) {
+  ret i128 %a
+}
+
 attributes #0 = { "target-features"="-bulk-memory,-bulk-memory-opt" }
diff --git a/lld/test/wasm/lto/libcall-archive.ll b/lld/test/wasm/lto/libcall-archive.ll
index 0cee9a5de29f6..90e7f1b26c1b7 100644
--- a/lld/test/wasm/lto/libcall-archive.ll
+++ b/lld/test/wasm/lto/libcall-archive.ll
@@ -8,12 +8,28 @@
 target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
 target triple = "wasm32-unknown-unknown"
 
+@llvm.used = appending global [2 x ptr] [ptr @test_acosf, ptr @test___umodti3]
+
 define void @_start(ptr %a, ptr %b) #0 {
 entry:
   call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %b, i64 1024, i1 false)
   ret void
 }
 
+; Emit acosf, which currently happens to be the first runtime libcall
+; entry.
+define float @test_acosf(float %x) {
+  %acos = call float @llvm.acos.f32(float %x)
+  ret float %acos
+}
+
+; Emit __umodti3, which currently happens to be the last runtime
+; libcall entry.
+define i128 @test___umodti3(i128 %a, i128 %b) {
+  %urem = urem i128 %a, %b
+  ret i128 %urem
+}
+
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1)
 
 attributes #0 = { "target-features"="-bulk-memory,-bulk-memory-opt" }
@@ -22,6 +38,12 @@ attributes #0 = { "target-features"="-bulk-memory,-bulk-memory-opt" }
 ; CHECK-NEXT:    Name:            name
 ; CHECK-NEXT:    FunctionNames:
 ; CHECK-NEXT:      - Index:           0
-; CHECK-NEXT:        Name:            _start
+; CHECK-NEXT:        Name:            test_acosf
 ; CHECK-NEXT:      - Index:           1
-; CHECK-NEXT:        Name:            memcpy
+; CHECK-NEXT:        Name:            acosf
+; CHECK-NEXT:      - Index:           2
+; CHECK-NEXT:        Name:            test___umodti3
+; CHECK-NEXT:      - Index:           3
+; CHECK-NEXT:        Name:            __umodti3
+; CHECK-NEXT:      - Index:           4
+; CHECK-NEXT:        Name:            _start
diff --git a/lld/test/wasm/page-size.s b/lld/test/wasm/page-size.s
index 9f5826109d27c..a2bf694936476 100644
--- a/lld/test/wasm/page-size.s
+++ b/lld/test/wasm/page-size.s
@@ -41,3 +41,21 @@ foo:
 # CHECK-DEFAULT-DIS:      <_start>:
 # CHECK-DEFAULT-DIS:          i32.const 65536
 # CHECK-DEFAULT-DIS-NEXT:     end
+
+# RUN: wasm-ld -no-gc-sections -o %t.custom-import.wasm %t.o --page-size=1 --import-memory
+# RUN: obj2yaml %t.custom-import.wasm | FileCheck %s --check-prefix=CHECK-CUSTOM-IMPORT
+
+# CHECK-CUSTOM-IMPORT:      Imports:
+# CHECK-CUSTOM-IMPORT-NEXT:   - Module:          env
+# CHECK-CUSTOM-IMPORT-NEXT:     Field:           memory
+# CHECK-CUSTOM-IMPORT-NEXT:     Kind:            MEMORY
+# CHECK-CUSTOM-IMPORT-NEXT:     Memory:
+# CHECK-CUSTOM-IMPORT-NEXT:       Flags:           [ HAS_PAGE_SIZE ]
+# CHECK-CUSTOM-IMPORT-NEXT:       Minimum:         0x10410
+# CHECK-CUSTOM-IMPORT-NEXT:       PageSize:        0x1
+
+# RUN: llvm-objdump --disassemble-symbols=_start %t.custom-import.wasm | FileCheck %s --check-prefix=CHECK-CUSTOM-IMPORT-DIS
+
+# CHECK-CUSTOM-IMPORT-DIS:      <_start>:
+# CHECK-CUSTOM-IMPORT-DIS:          i32.const 1
+# CHECK-CUSTOM-IMPORT-DIS-NEXT:     end
diff --git a/lld/wasm/InputChunks.cpp b/lld/wasm/InputChunks.cpp
index 855589353e4f6..181221a77b106 100644
--- a/lld/wasm/InputChunks.cpp
+++ b/lld/wasm/InputChunks.cpp
@@ -228,7 +228,7 @@ void InputFunction::setTableIndex(uint32_t index) {
 // witten.
 static unsigned writeCompressedReloc(uint8_t *buf, const WasmRelocation &rel,
                                      uint64_t value) {
-  switch (rel.Type) {
+  switch (rel.getType()) {
   case R_WASM_TYPE_INDEX_LEB:
   case R_WASM_FUNCTION_INDEX_LEB:
   case R_WASM_GLOBAL_INDEX_LEB:
@@ -239,16 +239,33 @@ static unsigned writeCompressedReloc(uint8_t *buf, const WasmRelocation &rel,
     return encodeULEB128(value, buf);
   case R_WASM_TABLE_INDEX_SLEB:
   case R_WASM_TABLE_INDEX_SLEB64:
+  case R_WASM_TABLE_INDEX_REL_SLEB64:
   case R_WASM_MEMORY_ADDR_SLEB:
   case R_WASM_MEMORY_ADDR_SLEB64:
+  case R_WASM_MEMORY_ADDR_REL_SLEB:
+  case R_WASM_MEMORY_ADDR_REL_SLEB64:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB64:
+  case R_WASM_TABLE_INDEX_REL_SLEB:
     return encodeSLEB128(static_cast<int64_t>(value), buf);
-  default:
-    llvm_unreachable("unexpected relocation type");
+  case R_WASM_TABLE_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I32:
+  case R_WASM_FUNCTION_OFFSET_I32:
+  case R_WASM_SECTION_OFFSET_I32:
+  case R_WASM_GLOBAL_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I64:
+  case R_WASM_TABLE_INDEX_I64:
+  case R_WASM_FUNCTION_OFFSET_I64:
+  case R_WASM_MEMORY_ADDR_LOCREL_I32:
+  case R_WASM_FUNCTION_INDEX_I32:
+    fatal("relocation compression not supported for " +
+          relocTypeToString(rel.Type));
   }
+  llvm_unreachable("unhandled relocation type");
 }
 
 static unsigned getRelocWidthPadded(const WasmRelocation &rel) {
-  switch (rel.Type) {
+  switch (rel.getType()) {
   case R_WASM_TYPE_INDEX_LEB:
   case R_WASM_FUNCTION_INDEX_LEB:
   case R_WASM_GLOBAL_INDEX_LEB:
@@ -256,15 +273,32 @@ static unsigned getRelocWidthPadded(const WasmRelocation &rel) {
   case R_WASM_MEMORY_ADDR_LEB:
   case R_WASM_TABLE_NUMBER_LEB:
   case R_WASM_TABLE_INDEX_SLEB:
+  case R_WASM_TABLE_INDEX_REL_SLEB:
   case R_WASM_MEMORY_ADDR_SLEB:
+  case R_WASM_MEMORY_ADDR_REL_SLEB:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB:
     return 5;
   case R_WASM_TABLE_INDEX_SLEB64:
+  case R_WASM_TABLE_INDEX_REL_SLEB64:
   case R_WASM_MEMORY_ADDR_LEB64:
   case R_WASM_MEMORY_ADDR_SLEB64:
+  case R_WASM_MEMORY_ADDR_REL_SLEB64:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB64:
     return 10;
-  default:
-    llvm_unreachable("unexpected relocation type");
+  case R_WASM_TABLE_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I32:
+  case R_WASM_FUNCTION_OFFSET_I32:
+  case R_WASM_SECTION_OFFSET_I32:
+  case R_WASM_GLOBAL_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I64:
+  case R_WASM_TABLE_INDEX_I64:
+  case R_WASM_FUNCTION_OFFSET_I64:
+  case R_WASM_MEMORY_ADDR_LOCREL_I32:
+  case R_WASM_FUNCTION_INDEX_I32:
+    fatal("relocation compression not supported for " +
+          relocTypeToString(rel.Type));
   }
+  llvm_unreachable("unhandled relocation type");
 }
 
 static unsigned getRelocWidth(const WasmRelocation &rel, uint64_t value) {
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index 76719596c62e0..e1192706ea913 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -258,6 +258,10 @@ void ImportSection::writeBody() {
       import.Memory.Flags |= WASM_LIMITS_FLAG_IS_SHARED;
     if (is64)
       import.Memory.Flags |= WASM_LIMITS_FLAG_IS_64;
+    if (ctx.arg.pageSize != WasmDefaultPageSize) {
+      import.Memory.Flags |= WASM_LIMITS_FLAG_HAS_PAGE_SIZE;
+      import.Memory.PageSize = ctx.arg.pageSize;
+    }
     writeImport(os, import);
   }
 
diff --git a/lldb/cmake/modules/LLDBFramework.cmake b/lldb/cmake/modules/LLDBFramework.cmake
index 70010ffbf738c..bbd717a982cf3 100644
--- a/lldb/cmake/modules/LLDBFramework.cmake
+++ b/lldb/cmake/modules/LLDBFramework.cmake
@@ -112,7 +112,8 @@ file(GLOB lldb_framework_header_staging_list ${lldb_framework_header_staging}/*)
 foreach(header ${lldb_framework_header_staging_list})
 
   set(input_header ${header})
-  set(output_header $<TARGET_FILE_DIR:liblldb>/Headers/${input_header})
+  get_filename_component(header_basename ${input_header} NAME)
+  set(output_header $<TARGET_FILE_DIR:liblldb>/Headers/${header_basename})
 
   add_custom_command(TARGET liblldb POST_BUILD
     COMMAND ${LLDB_SOURCE_DIR}/scripts/framework-header-fix.py -f lldb_main -i ${input_header} -o ${output_header} -p ${unifdef_EXECUTABLE} USWIG
diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index 1ffdb08a1ca2c..d49d0d8d5a733 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -135,6 +135,7 @@ interesting areas to contribute to lldb.
    use/ondemand
    use/aarch64-linux
    use/symbolfilejson
+   use/mcp
    use/troubleshooting
    use/links
    Man Page <man/lldb>
diff --git a/lldb/docs/resources/contributing.rst b/lldb/docs/resources/contributing.rst
index e7d46c9061d53..7e84df41c3fbd 100644
--- a/lldb/docs/resources/contributing.rst
+++ b/lldb/docs/resources/contributing.rst
@@ -104,7 +104,7 @@ making sense for the particular pair of plugins). Unfortunately, due to historic
 reasons, not all plugin dependencies follow this rule, which is why we have
 another category called `LLDB_TOLERATED_PLUGIN_DEPENDENCIES`. New dependencies
 are forbidden (even though they are accepted by CMake) and existing ones should
-be removed whereever possible.
+be removed wherever possible.
 
 .. _Error handling:
 
diff --git a/lldb/docs/use/mcp.md b/lldb/docs/use/mcp.md
new file mode 100644
index 0000000000000..375c164fe771c
--- /dev/null
+++ b/lldb/docs/use/mcp.md
@@ -0,0 +1,85 @@
+# Model Context Protocol (MCP)
+
+LLDB supports the [Model Context Protocol](https://modelcontextprotocol.io)
+(MCP). This structured, machine-friendly protocol allows AI models to access
+and interact with external tools, for example debuggers. Using MCP, an AI agent
+can execute LLDB commands to control the debugger: set breakpoints, inspect
+memory, step through code. This can range from helping you run a specific
+command you cannot immediately remember, to a fully agent-driven debugging
+experience.
+
+## MCP Server
+
+To start the MCP server in LLDB, use the `protocol-server start` command.
+Specify `MCP` as the protocol and provide a URI to listen on. For example, to
+start listening for local TCP connections on port `59999`, use the following
+command:
+
+```
+(lldb) protocol-server start MCP listen://localhost:59999
+MCP server started with connection listeners: connection://[::1]:59999, connection://[127.0.0.1]:59999
+```
+
+The server will automatically stop when exiting LLDB, or it can be stopped
+explicitly with the `protocol-server stop` command.
+
+```
+(lldb) protocol-server stop MCP
+```
+
+The commands will fail if a server is already running or not running
+respectively.
+
+## MCP Client
+
+MCP uses standard input/output (stdio) for communication between client and
+server. The exact configuration depends on the client, but most applications
+allow you to specify an MCP server as a binary and arguments. This means that
+you need to use something like `netcat` to connect to LLDB's MCP server and
+forward communication over stdio over the network connection.
+
+```
+┌──────────┐               ┌──────────┐               ┌──────────┐
+│          │               │          │               │          │
+│   LLDB   ├─────socket────┤  netcat  ├─────stdio─────┤MCP Client│
+│          │               │          │               │          │
+└──────────┘               └──────────┘               └──────────┘
+```
+
+Configuration example for [Claude Code](https://modelcontextprotocol.io/quickstart/user):
+
+```json
+{
+  "mcpServers": {
+    "tool": {
+      "command": "/usr/bin/nc",
+      "args": ["localhost", "59999"]
+    }
+  }
+}
+```
+
+Configuration example for [Visual Studio Code](https://code.visualstudio.com/docs/copilot/chat/mcp-servers):
+
+```json
+{
+  "mcp": {
+    "servers": {
+      "lldb": {
+        "type": "stdio",
+        "command": "/usr/bin/nc",
+        "args": ["localhost", "59999"]
+      }
+    }
+  }
+}
+```
+
+### Troubleshooting
+
+The MCP server uses the `Host` log channel. You can enable logging with the
+`log enable` command.
+
+```
+(lldb) log enable lldb host
+```
diff --git a/lldb/include/lldb/API/SBBreakpoint.h b/lldb/include/lldb/API/SBBreakpoint.h
index e08df3b6d5ab0..18ed3e7226d3b 100644
--- a/lldb/include/lldb/API/SBBreakpoint.h
+++ b/lldb/include/lldb/API/SBBreakpoint.h
@@ -148,6 +148,11 @@ class LLDB_API SBBreakpoint {
 
   bool IsHardware() const;
 
+  /// Make this breakpoint a hardware breakpoint. This will replace all existing
+  /// breakpoint locations with hardware breakpoints. Returns an error if this
+  /// fails, e.g. when there aren't enough hardware resources available.
+  lldb::SBError SetIsHardware(bool is_hardware);
+
   // Can only be called from a ScriptedBreakpointResolver...
   SBError
   AddLocation(SBAddress &address);
diff --git a/lldb/include/lldb/Breakpoint/Breakpoint.h b/lldb/include/lldb/Breakpoint/Breakpoint.h
index f623a2e0c295b..b200a1e4893df 100644
--- a/lldb/include/lldb/Breakpoint/Breakpoint.h
+++ b/lldb/include/lldb/Breakpoint/Breakpoint.h
@@ -519,6 +519,8 @@ class Breakpoint : public std::enable_shared_from_this<Breakpoint>,
 
   bool IsHardware() const { return m_hardware; }
 
+  llvm::Error SetIsHardware(bool is_hardware);
+
   lldb::BreakpointResolverSP GetResolver() { return m_resolver_sp; }
 
   lldb::SearchFilterSP GetSearchFilter() { return m_filter_sp; }
diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index 3592291bb2d06..66274e8825ee2 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -69,7 +69,7 @@ class BreakpointLocation
   // The next section deals with various breakpoint options.
 
   /// If \a enabled is \b true, enable the breakpoint, if \b false disable it.
-  void SetEnabled(bool enabled);
+  bool SetEnabled(bool enabled);
 
   /// Check the Enable/Disable state.
   ///
@@ -235,7 +235,7 @@ class BreakpointLocation
   ///     \b true if the target should stop at this breakpoint and \b
   ///     false not.
   bool InvokeCallback(StoppointCallbackContext *context);
-  
+
   /// Report whether the callback for this location is synchronous or not.
   ///
   /// \return
@@ -371,11 +371,8 @@ class BreakpointLocation
   ///     The thread for which this breakpoint location is valid, or
   ///     LLDB_INVALID_THREAD_ID if it is valid for all threads.
   ///
-  /// \param[in] hardware
-  ///     \b true if a hardware breakpoint is requested.
-
   BreakpointLocation(lldb::break_id_t bid, Breakpoint &owner,
-                     const Address &addr, lldb::tid_t tid, bool hardware,
+                     const Address &addr, lldb::tid_t tid,
                      bool check_for_resolver = true);
 
   // Data members:
diff --git a/lldb/include/lldb/Breakpoint/BreakpointName.h b/lldb/include/lldb/Breakpoint/BreakpointName.h
index c1c2ba110c39e..6ddca88644e46 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointName.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointName.h
@@ -32,18 +32,20 @@ class BreakpointName {
   class Permissions
   {
   public:
-  
-    enum PermissionKinds { listPerm = 0, disablePerm = 1, 
-                       deletePerm = 2, allPerms = 3 };
+    enum PermissionKinds {
+      listPerm = 0,
+      disablePerm = 1,
+      deletePerm = 2,
+      allPerms = 3
+    };
 
-    Permissions(bool in_list, bool in_disable, bool in_delete) 
-    {
+    Permissions(bool in_list, bool in_disable, bool in_delete) {
       m_permissions[listPerm]    = in_list;
       m_permissions[disablePerm] = in_disable;
       m_permissions[deletePerm]  = in_delete;
       m_set_mask.Set(permissions_mask[allPerms]);
     }
-    
+
     Permissions(const Permissions &rhs)
     {
       m_permissions[listPerm]    = rhs.m_permissions[listPerm];
@@ -51,15 +53,14 @@ class BreakpointName {
       m_permissions[deletePerm]  = rhs.m_permissions[deletePerm];
       m_set_mask = rhs.m_set_mask;
     }
-    
-    Permissions() 
-    {
+
+    Permissions() {
       m_permissions[listPerm]    = true;
       m_permissions[disablePerm] = true;
       m_permissions[deletePerm]  = true;
       m_set_mask.Clear();
     }
-    
+
     const Permissions &operator= (const Permissions &rhs)
     {
       if (this != &rhs) {
@@ -70,11 +71,11 @@ class BreakpointName {
       }
       return *this;
     }
-    
+
     void Clear() {
       *this = Permissions();
     }
-    
+
     // Merge the permissions from incoming into this set of permissions. Only
     // merge set permissions, and most restrictive permission wins.
     void MergeInto(const Permissions &incoming)
@@ -86,13 +87,14 @@ class BreakpointName {
 
     bool GetAllowList() const { return GetPermission(listPerm); }
     bool SetAllowList(bool value) { return SetPermission(listPerm, value); }
-    
+
     bool GetAllowDelete() const { return GetPermission(deletePerm); }
     bool SetAllowDelete(bool value) { return SetPermission(deletePerm, value); }
-    
+
     bool GetAllowDisable() const { return GetPermission(disablePerm); }
-    bool SetAllowDisable(bool value) { return SetPermission(disablePerm, 
-                                                            value); }
+    bool SetAllowDisable(bool value) {
+      return SetPermission(disablePerm, value);
+    }
 
     bool GetPermission(enum PermissionKinds permission) const
     {
@@ -105,17 +107,17 @@ class BreakpointName {
     {
       return m_set_mask.Test(permissions_mask[permission]);
     }
-    
+
     bool AnySet() {
       return m_set_mask.AnySet(permissions_mask[allPerms]);
     }
-    
+
   private:
     static const Flags::ValueType permissions_mask[allPerms + 1];
-    
+
     bool m_permissions[allPerms];
     Flags m_set_mask;
-    
+
     bool SetPermission(enum PermissionKinds permission, bool value)
     {
       bool old_value = m_permissions[permission];
@@ -123,11 +125,10 @@ class BreakpointName {
       m_set_mask.Set(permissions_mask[permission]);
       return old_value;
     }
-    
+
     // If either side disallows the permission, the resultant disallows it.
-    void MergePermission(const Permissions &incoming, 
-                         enum PermissionKinds permission)
-    {
+    void MergePermission(const Permissions &incoming,
+                         enum PermissionKinds permission) {
       if (incoming.IsSet(permission))
       {
         SetPermission(permission, !(m_permissions[permission] |
@@ -135,37 +136,37 @@ class BreakpointName {
       }
     }
   };
-  
+
   BreakpointName(ConstString name, const char *help = nullptr) :
       m_name(name), m_options(false)
    {
      SetHelp(help);
    }
-  
+
   BreakpointName(const BreakpointName &rhs) :
       m_name(rhs.m_name), m_options(rhs.m_options),
       m_permissions(rhs.m_permissions), m_help(rhs.m_help)
   {}
-      
+
   ConstString GetName() const { return m_name; }
   BreakpointOptions &GetOptions() { return m_options; }
   const BreakpointOptions &GetOptions() const { return m_options; }
-  
+
   void SetOptions(const BreakpointOptions &options) {
     m_options = options;
   }
-  
+
   Permissions &GetPermissions() { return m_permissions; }
   const Permissions &GetPermissions() const { return m_permissions; }
   void SetPermissions(const Permissions &permissions) {
     m_permissions = permissions;
   }
-  
+
   bool GetPermission(Permissions::PermissionKinds permission) const
   {
     return m_permissions.GetPermission(permission);
   }
-  
+
   void SetHelp(const char *description)
   {
     if (description)
@@ -173,17 +174,17 @@ class BreakpointName {
     else
       m_help.clear();
   }
-  
+
   const char *GetHelp()
   {
     return m_help.c_str();
   }
-  
+
   // Returns true if any options were set in the name
   bool GetDescription(Stream *s, lldb::DescriptionLevel level);
-  
+
   void ConfigureBreakpoint(lldb::BreakpointSP bp_sp);
-  
+
 private:
   ConstString        m_name;
   BreakpointOptions  m_options;
diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolver.h b/lldb/include/lldb/Breakpoint/BreakpointResolver.h
index a0b37c94ac2b4..52cd70e934e6d 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointResolver.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointResolver.h
@@ -148,8 +148,7 @@ class BreakpointResolver : public Searcher {
   enum ResolverTy GetResolverTy() {
     if (SubclassID > ResolverTy::LastKnownResolverType)
       return ResolverTy::UnknownResolver;
-    else
-      return (enum ResolverTy)SubclassID;
+    return (enum ResolverTy)SubclassID;
   }
 
   const char *GetResolverName() { return ResolverTyToName(GetResolverTy()); }
diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h b/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h
index 16182e36d7095..0e4a1d733e107 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h
@@ -60,7 +60,7 @@ class BreakpointResolverScripted : public BreakpointResolver {
 private:
   void CreateImplementationIfNeeded(lldb::BreakpointSP bkpt);
   ScriptInterpreter *GetScriptInterpreter();
-  
+
   std::string m_class_name;
   lldb::SearchDepth m_depth;
   StructuredDataImpl m_args;
diff --git a/lldb/include/lldb/Breakpoint/Watchpoint.h b/lldb/include/lldb/Breakpoint/Watchpoint.h
index 16b83fbf7f705..ca48a0114888a 100644
--- a/lldb/include/lldb/Breakpoint/Watchpoint.h
+++ b/lldb/include/lldb/Breakpoint/Watchpoint.h
@@ -73,7 +73,7 @@ class Watchpoint : public std::enable_shared_from_this<Watchpoint>,
   bool IsHardware() const override;
 
   bool ShouldStop(StoppointCallbackContext *context) override;
-  
+
   bool WatchpointRead() const;
   bool WatchpointWrite() const;
   bool WatchpointModify() const;
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index 2087ef2a11562..504f936fe317a 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -227,6 +227,8 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   const char *GetIOHandlerHelpPrologue();
 
+  void RefreshIOHandler();
+
   void ClearIOHandlers();
 
   bool EnableLog(llvm::StringRef channel,
diff --git a/lldb/include/lldb/Core/EmulateInstruction.h b/lldb/include/lldb/Core/EmulateInstruction.h
index b459476883fc5..a9fd4543cbbcb 100644
--- a/lldb/include/lldb/Core/EmulateInstruction.h
+++ b/lldb/include/lldb/Core/EmulateInstruction.h
@@ -22,6 +22,8 @@
 #include "lldb/lldb-private-types.h"
 #include "lldb/lldb-types.h"
 
+#include "llvm/Support/Error.h"
+
 #include <cstddef>
 #include <cstdint>
 
@@ -32,6 +34,38 @@ class RegisterValue;
 class Stream;
 class Target;
 class UnwindPlan;
+class EmulateInstruction;
+
+using BreakpointLocations = std::vector<lldb::addr_t>;
+
+class SingleStepBreakpointLocationsPredictor {
+public:
+  SingleStepBreakpointLocationsPredictor(
+      std::unique_ptr<EmulateInstruction> emulator_up)
+      : m_emulator_up{std::move(emulator_up)} {}
+
+  virtual BreakpointLocations GetBreakpointLocations(Status &status);
+
+  virtual llvm::Expected<unsigned>
+  GetBreakpointSize([[maybe_unused]] lldb::addr_t bp_addr) {
+    return 4;
+  }
+
+  virtual ~SingleStepBreakpointLocationsPredictor() = default;
+
+protected:
+  // This function retrieves the address of the next instruction as it appears
+  // in the binary file. Essentially, it reads the value of the PC register,
+  // determines the size of the current instruction (where the PC is pointing),
+  // and returns the sum of these two values.
+  lldb::addr_t GetNextInstructionAddress(Status &error);
+
+  lldb::addr_t GetBreakpointLocationAddress(lldb::addr_t entry_pc,
+                                            Status &error);
+
+  std::unique_ptr<EmulateInstruction> m_emulator_up;
+  bool m_emulation_result = false;
+};
 
 /// \class EmulateInstruction EmulateInstruction.h
 /// "lldb/Core/EmulateInstruction.h"
@@ -497,7 +531,19 @@ class EmulateInstruction : public PluginInterface {
   static uint32_t GetInternalRegisterNumber(RegisterContext *reg_ctx,
                                             const RegisterInfo &reg_info);
 
+  static std::unique_ptr<SingleStepBreakpointLocationsPredictor>
+  CreateBreakpointLocationPredictor(
+      std::unique_ptr<EmulateInstruction> emulator_up);
+
+  // Helper functions
+  std::optional<lldb::addr_t> ReadPC();
+  bool WritePC(lldb::addr_t addr);
+
 protected:
+  using BreakpointLocationsPredictorCreator =
+      std::function<std::unique_ptr<SingleStepBreakpointLocationsPredictor>(
+          std::unique_ptr<EmulateInstruction>)>;
+
   ArchSpec m_arch;
   void *m_baton = nullptr;
   ReadMemoryCallback m_read_mem_callback = &ReadMemoryDefault;
@@ -508,6 +554,21 @@ class EmulateInstruction : public PluginInterface {
   Opcode m_opcode;
 
 private:
+  virtual BreakpointLocationsPredictorCreator
+  GetSingleStepBreakpointLocationsPredictorCreator() {
+    if (!m_arch.IsMIPS() && !m_arch.GetTriple().isPPC64() &&
+        !m_arch.GetTriple().isLoongArch()) {
+      // Unsupported architecture
+      return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+        return nullptr;
+      };
+    }
+    return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+      return std::make_unique<SingleStepBreakpointLocationsPredictor>(
+          std::move(emulator_up));
+    };
+  }
+
   // For EmulateInstruction only
   EmulateInstruction(const EmulateInstruction &) = delete;
   const EmulateInstruction &operator=(const EmulateInstruction &) = delete;
diff --git a/lldb/include/lldb/Core/IOHandler.h b/lldb/include/lldb/Core/IOHandler.h
index 2fb3d7a7c9cc3..2672bbe5da2b3 100644
--- a/lldb/include/lldb/Core/IOHandler.h
+++ b/lldb/include/lldb/Core/IOHandler.h
@@ -90,6 +90,8 @@ class IOHandler {
 
   virtual void TerminalSizeChanged() {}
 
+  virtual void Refresh() {}
+
   virtual const char *GetPrompt() {
     // Prompt support isn't mandatory
     return nullptr;
@@ -404,6 +406,8 @@ class IOHandlerEditline : public IOHandler {
 
   void PrintAsync(const char *s, size_t len, bool is_stdout) override;
 
+  void Refresh() override;
+
 private:
 #if LLDB_ENABLE_LIBEDIT
   bool IsInputCompleteCallback(Editline *editline, StringList &lines);
diff --git a/lldb/include/lldb/Core/Statusline.h b/lldb/include/lldb/Core/Statusline.h
index 521b9f2526f6b..6bda153f822d2 100644
--- a/lldb/include/lldb/Core/Statusline.h
+++ b/lldb/include/lldb/Core/Statusline.h
@@ -36,12 +36,10 @@ class Statusline {
   /// Draw the statusline with the given text.
   void Draw(std::string msg);
 
-  /// Update terminal dimensions.
-  void UpdateTerminalProperties();
-
   enum ScrollWindowMode {
     EnableStatusline,
     DisableStatusline,
+    ResizeStatusline,
   };
 
   /// Set the scroll window for the given mode.
diff --git a/lldb/include/lldb/DataFormatters/FormattersHelpers.h b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
index 42699d0a0b1b1..6a47cf37eac64 100644
--- a/lldb/include/lldb/DataFormatters/FormattersHelpers.h
+++ b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
@@ -55,6 +55,14 @@ void AddFilter(TypeCategoryImpl::SharedPointer category_sp,
 
 std::optional<size_t> ExtractIndexFromString(const char *item_name);
 
+/// Prints the summary for the pointer value of a C++
+/// std::unique_ptr/std::shared_ptr/std::weak_ptr.
+void DumpCxxSmartPtrPointerSummary(Stream &stream, ValueObject &ptr,
+                                   const TypeSummaryOptions &options);
+
+bool ContainerSizeSummaryProvider(ValueObject &valobj, Stream &stream,
+                                  const TypeSummaryOptions &options);
+
 Address GetArrayAddressOrPointerValue(ValueObject &valobj);
 
 time_t GetOSXEpoch();
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index 11a4ca2cd8330..b147d66def730 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -56,10 +56,10 @@ class SyntheticChildrenFrontEnd {
   /// This function is assumed to always succeed and if it fails, the front-end
   /// should know to deal with it in the correct way (most probably, by refusing
   /// to return any children). The return value of \ref Update should actually
-  /// be interpreted as "ValueObjectSyntheticFilter cache is good/bad". If this
+  /// be interpreted as "ValueObjectSynthetic cache is good/bad". If this
   /// function returns \ref lldb::ChildCacheState::eReuse, \ref
-  /// ValueObjectSyntheticFilter is allowed to use the children it fetched
-  /// previously and cached. Otherwise, \ref ValueObjectSyntheticFilter must
+  /// ValueObjectSynthetic is allowed to use the children it fetched
+  /// previously and cached. Otherwise, \ref ValueObjectSynthetic must
   /// throw away its cache, and query again for children.
   virtual lldb::ChildCacheState Update() = 0;
 
diff --git a/lldb/include/lldb/Expression/DWARFExpression.h b/lldb/include/lldb/Expression/DWARFExpression.h
index 0adbe3e8df2ee..37853c0b5a8fc 100644
--- a/lldb/include/lldb/Expression/DWARFExpression.h
+++ b/lldb/include/lldb/Expression/DWARFExpression.h
@@ -35,6 +35,8 @@ namespace lldb_private {
 /// location expression or a location list and interprets it.
 class DWARFExpression {
 public:
+  using Stack = std::vector<Value>;
+
   class Delegate {
   public:
     Delegate() = default;
@@ -53,7 +55,7 @@ class DWARFExpression {
     virtual bool ParseVendorDWARFOpcode(uint8_t op,
                                         const DataExtractor &opcodes,
                                         lldb::offset_t &offset,
-                                        std::vector<Value> &stack) const = 0;
+                                        Stack &stack) const = 0;
 
     Delegate(const Delegate &) = delete;
     Delegate &operator=(const Delegate &) = delete;
diff --git a/lldb/include/lldb/Expression/IRMemoryMap.h b/lldb/include/lldb/Expression/IRMemoryMap.h
index abec5442793c6..58b95c56c1c36 100644
--- a/lldb/include/lldb/Expression/IRMemoryMap.h
+++ b/lldb/include/lldb/Expression/IRMemoryMap.h
@@ -12,6 +12,7 @@
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-public.h"
+#include "llvm/Support/Error.h"
 
 #include <map>
 
@@ -50,8 +51,13 @@ class IRMemoryMap {
                                  ///only in the process.
   };
 
-  lldb::addr_t Malloc(size_t size, uint8_t alignment, uint32_t permissions,
-                      AllocationPolicy policy, bool zero_memory, Status &error);
+  // If 'policy' is 'eAllocationPolicyMirror' but it is impossible to allocate
+  // memory in the process, 'eAllocationPolicyHostOnly' will be used instead.
+  // The actual policy is returned via 'used_policy'.
+  llvm::Expected<lldb::addr_t> Malloc(size_t size, uint8_t alignment,
+                                      uint32_t permissions,
+                                      AllocationPolicy policy, bool zero_memory,
+                                      AllocationPolicy *used_policy = nullptr);
   void Leak(lldb::addr_t process_address, Status &error);
   void Free(lldb::addr_t process_address, Status &error);
 
diff --git a/lldb/include/lldb/Host/Editline.h b/lldb/include/lldb/Host/Editline.h
index c202a76758e13..947ad3bfe5ec6 100644
--- a/lldb/include/lldb/Host/Editline.h
+++ b/lldb/include/lldb/Host/Editline.h
@@ -267,6 +267,8 @@ class Editline {
 
   size_t GetTerminalHeight() { return m_terminal_height; }
 
+  void Refresh();
+
 private:
   /// Sets the lowest line number for multi-line editing sessions.  A value of
   /// zero suppresses line number printing in the prompt.
diff --git a/lldb/include/lldb/Host/posix/HostInfoPosix.h b/lldb/include/lldb/Host/posix/HostInfoPosix.h
index 779b67bd66eda..c4a7f13674f40 100644
--- a/lldb/include/lldb/Host/posix/HostInfoPosix.h
+++ b/lldb/include/lldb/Host/posix/HostInfoPosix.h
@@ -39,6 +39,8 @@ class HostInfoPosix : public HostInfoBase {
   static llvm::VersionTuple GetOSVersion();
   static std::optional<std::string> GetOSBuildString();
 
+  static llvm::Expected<llvm::StringRef> GetSDKRoot(SDKOptions options);
+
 protected:
   static bool ComputeSupportExeDirectory(FileSpec &file_spec);
   static bool ComputeHeaderDirectory(FileSpec &file_spec);
diff --git a/lldb/include/lldb/Host/windows/MainLoopWindows.h b/lldb/include/lldb/Host/windows/MainLoopWindows.h
index 3937a24645d95..705e7e78ba48a 100644
--- a/lldb/include/lldb/Host/windows/MainLoopWindows.h
+++ b/lldb/include/lldb/Host/windows/MainLoopWindows.h
@@ -17,6 +17,8 @@
 
 namespace lldb_private {
 
+using handle_t = void *;
+
 // Windows-specific implementation of the MainLoopBase class. It can monitor
 // socket descriptors for readability using WSAEventSelect. Non-socket file
 // descriptors are not supported.
@@ -31,17 +33,30 @@ class MainLoopWindows : public MainLoopBase {
 
   Status Run() override;
 
+  class IOEvent {
+  public:
+    IOEvent(handle_t event) : m_event(event) {}
+    virtual ~IOEvent() {}
+    virtual void WillPoll() {}
+    virtual void DidPoll() {}
+    virtual void Disarm() {}
+    handle_t GetHandle() { return m_event; }
+
+  protected:
+    handle_t m_event;
+  };
+  using IOEventUP = std::unique_ptr<IOEvent>;
+
 protected:
   void UnregisterReadObject(IOObject::WaitableHandle handle) override;
 
   void Interrupt() override;
 
 private:
-  void ProcessReadObject(IOObject::WaitableHandle handle);
   llvm::Expected<size_t> Poll();
 
   struct FdInfo {
-    void *event;
+    IOEventUP event;
     Callback callback;
   };
   llvm::DenseMap<IOObject::WaitableHandle, FdInfo> m_read_fds;
diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index 43567592dd447..1b9ae1fb31a69 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -18,6 +18,7 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/UUID.h"
 #include "lldb/lldb-private.h"
 #include "llvm/Support/Threading.h"
@@ -544,9 +545,9 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
     return false;
   }
 
-  /// Get metadata about threads from the corefile.
+  /// Get metadata about thread ids from the corefile.
   ///
-  /// The corefile may have metadata (e.g. a Mach-O "thread extrainfo"
+  /// The corefile may have metadata (e.g. a Mach-O "process metadata"
   /// LC_NOTE) which for the threads in the process; this method tries
   /// to retrieve them.
   ///
@@ -568,6 +569,18 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
     return false;
   }
 
+  /// Get process metadata from the corefile in a StructuredData dictionary.
+  ///
+  /// The corefile may have notes (e.g. a Mach-O "process metadata" LC_NOTE)
+  /// which provide metadata about the process and threads in a JSON or
+  /// similar format.
+  ///
+  /// \return
+  ///     A StructuredData object with the metadata in the note, if there is
+  ///     one.  An empty shared pointer is returned if not metadata is found,
+  ///     or a problem parsing it.
+  virtual StructuredData::ObjectSP GetCorefileProcessMetadata() { return {}; }
+
   virtual lldb::RegisterContextSP
   GetThreadContextAtIndex(uint32_t idx, lldb_private::Thread &thread) {
     return lldb::RegisterContextSP();
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index b0b608d0a5e79..e95f95553c17c 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -467,8 +467,11 @@ class SymbolFile : public PluginInterface {
   ///     If true, then only return separate debug info files that encountered
   ///     errors during loading. If false, then return all expected separate
   ///     debug info files, regardless of whether they were successfully loaded.
+  /// \param load_all_debug_info
+  ///     If true, force loading any symbol files if they are not yet loaded.
   virtual bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
-                                    bool errors_only) {
+                                    bool errors_only,
+                                    bool load_all_debug_info = false) {
     return false;
   };
 
diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
index ba4a7f09afeaa..6e3c2477d1769 100644
--- a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
+++ b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
@@ -223,9 +223,10 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
     return m_sym_file_impl->SetDebugInfoHadFrameVariableErrors();
   }
 
-  bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
-                            bool errors_only) override {
-    return m_sym_file_impl->GetSeparateDebugInfo(d, errors_only);
+  bool GetSeparateDebugInfo(StructuredData::Dictionary &d, bool errors_only,
+                            bool load_all_debug_info = false) override {
+    return m_sym_file_impl->GetSeparateDebugInfo(d, errors_only,
+                                                 load_all_debug_info);
   }
 
   lldb::TypeSP MakeType(lldb::user_id_t uid, ConstString name,
diff --git a/lldb/include/lldb/Utility/IOObject.h b/lldb/include/lldb/Utility/IOObject.h
index 8cf42992e7be5..de6532a637083 100644
--- a/lldb/include/lldb/Utility/IOObject.h
+++ b/lldb/include/lldb/Utility/IOObject.h
@@ -14,6 +14,7 @@
 #include <sys/types.h>
 
 #include "lldb/lldb-private.h"
+#include "lldb/lldb-types.h"
 
 namespace lldb_private {
 
@@ -24,9 +25,9 @@ class IOObject {
     eFDTypeSocket, // Socket requiring send/recv
   };
 
-  // TODO: On Windows this should be a HANDLE, and wait should use
-  // WaitForMultipleObjects
-  typedef int WaitableHandle;
+  // A handle for integrating with the host event loop model.
+  using WaitableHandle = lldb::file_t;
+
   static const WaitableHandle kInvalidHandleValue;
 
   IOObject(FDType type) : m_fd_type(type) {}
diff --git a/lldb/include/lldb/ValueObject/DILEval.h b/lldb/include/lldb/ValueObject/DILEval.h
index 2a0cb548a810f..45e29b3ddcd7b 100644
--- a/lldb/include/lldb/ValueObject/DILEval.h
+++ b/lldb/include/lldb/ValueObject/DILEval.h
@@ -25,8 +25,7 @@ namespace lldb_private::dil {
 /// evaluating).
 lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
                                      std::shared_ptr<StackFrame> frame_sp,
-                                     lldb::DynamicValueType use_dynamic,
-                                     CompilerType *scope_ptr = nullptr);
+                                     lldb::DynamicValueType use_dynamic);
 
 /// Given the name of an identifier, check to see if it matches the name of a
 /// global variable. If so, find the ValueObject for that global variable, and
@@ -35,8 +34,7 @@ lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
 lldb::ValueObjectSP LookupGlobalIdentifier(llvm::StringRef name_ref,
                                            std::shared_ptr<StackFrame> frame_sp,
                                            lldb::TargetSP target_sp,
-                                           lldb::DynamicValueType use_dynamic,
-                                           CompilerType *scope_ptr = nullptr);
+                                           lldb::DynamicValueType use_dynamic);
 
 class Interpreter : Visitor {
 public:
diff --git a/lldb/include/lldb/ValueObject/ValueObjectSyntheticFilter.h b/lldb/include/lldb/ValueObject/ValueObjectSynthetic.h
similarity index 98%
rename from lldb/include/lldb/ValueObject/ValueObjectSyntheticFilter.h
rename to lldb/include/lldb/ValueObject/ValueObjectSynthetic.h
index cdb2b1ddc3646..063d796ee4eec 100644
--- a/lldb/include/lldb/ValueObject/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectSynthetic.h
@@ -1,4 +1,4 @@
-//===-- ValueObjectSyntheticFilter.h ----------------------------*- C++ -*-===//
+//===-- ValueObjectSynthetic.h ----------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/lldb/include/lldb/lldb-types.h b/lldb/include/lldb/lldb-types.h
index 15781ad626efb..e309fc8833ce9 100644
--- a/lldb/include/lldb/lldb-types.h
+++ b/lldb/include/lldb/lldb-types.h
@@ -42,7 +42,7 @@ typedef void *rwlock_t;
 typedef void *process_t;                          // Process type is HANDLE
 typedef void *thread_t;                           // Host thread type
 typedef void *file_t;                             // Host file type
-typedef unsigned int __w64 socket_t;              // Host socket type
+typedef uintptr_t socket_t;                       // Host socket type
 typedef void *thread_arg_t;                       // Host thread argument type
 typedef unsigned thread_result_t;                 // Host thread result type
 typedef thread_result_t (*thread_func_t)(void *); // Host thread function type
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index d7f274ac4f60e..90c8e32afa507 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -831,6 +831,46 @@ def checkLibstdcxxSupport():
     configuration.skip_categories.append("libstdcxx")
 
 
+def canRunMsvcStlTests():
+    from lldbsuite.test import lldbplatformutil
+
+    platform = lldbplatformutil.getPlatform()
+    if platform != "windows":
+        return False, f"Don't know how to build with MSVC's STL on {platform}"
+
+    with tempfile.NamedTemporaryFile() as f:
+        cmd = [configuration.compiler, "-xc++", "-o", f.name, "-E", "-"]
+        p = subprocess.Popen(
+            cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        _, stderr = p.communicate(
+            """
+            #include <yvals_core.h>
+            #ifndef _MSVC_STL_VERSION
+            #error _MSVC_STL_VERSION not defined
+            #endif
+            """
+        )
+        if not p.returncode:
+            return True, "Compiling with MSVC STL"
+        return (False, f"Not compiling with MSVC STL: {stderr}")
+
+
+def checkMsvcStlSupport():
+    result, reason = canRunMsvcStlTests()
+    if result:
+        return  # msvcstl supported
+    if "msvcstl" in configuration.categories_list:
+        return  # msvcstl category explicitly requested, let it run.
+    if configuration.verbose:
+        print(f"msvcstl tests will not be run because: {reason}")
+    configuration.skip_categories.append("msvcstl")
+
+
 def canRunWatchpointTests():
     from lldbsuite.test import lldbplatformutil
 
@@ -1044,6 +1084,7 @@ def run_suite():
 
     checkLibcxxSupport()
     checkLibstdcxxSupport()
+    checkMsvcStlSupport()
     checkWatchpointSupport()
     checkDebugInfoSupport()
     checkDebugServerSupport()
diff --git a/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h b/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
index 6dc53a4e88ffd..35649b10ce45e 100644
--- a/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
+++ b/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
@@ -58,7 +58,7 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>,
 
   _T1 &first() { return static_cast<_Base1 &>(*this).__get(); }
 };
-#elif COMPRESSED_PAIR_REV == 1
+#elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2
 // From libc++ datasizeof.h
 template <class _Tp> struct _FirstPaddingByte {
   _LLDB_NO_UNIQUE_ADDRESS _Tp __v_;
@@ -72,6 +72,9 @@ inline const size_t __datasizeof_v =
 template <class _Tp>
 struct __lldb_is_final : public integral_constant<bool, __is_final(_Tp)> {};
 
+// The legacy layout has been patched, see
+// https://github.com/llvm/llvm-project/pull/142516.
+#if COMPRESSED_PAIR_REV == 1
 template <class _ToPad> class __compressed_pair_padding {
   char __padding_[((is_empty<_ToPad>::value &&
                     !__lldb_is_final<_ToPad>::value) ||
@@ -79,6 +82,25 @@ template <class _ToPad> class __compressed_pair_padding {
                       ? 0
                       : sizeof(_ToPad) - __datasizeof_v<_ToPad>];
 };
+#else
+template <class _ToPad>
+inline const bool __is_reference_or_unpadded_object =
+    (std::is_empty<_ToPad>::value && !__lldb_is_final<_ToPad>::value) ||
+    sizeof(_ToPad) == __datasizeof_v<_ToPad>;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp &> = true;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp &&> = true;
+
+template <class _ToPad, bool _Empty = __is_reference_or_unpadded_object<_ToPad>>
+class __compressed_pair_padding {
+  char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>] = {};
+};
+
+template <class _ToPad> class __compressed_pair_padding<_ToPad, true> {};
+#endif
 
 #define _LLDB_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)              \
   [[__gnu__::__aligned__(                                                      \
@@ -96,7 +118,7 @@ template <class _ToPad> class __compressed_pair_padding {
   _LLDB_NO_UNIQUE_ADDRESS __compressed_pair_padding<T2> __padding2_;           \
   _LLDB_NO_UNIQUE_ADDRESS T3 Initializer3;                                     \
   _LLDB_NO_UNIQUE_ADDRESS __compressed_pair_padding<T3> __padding3_;
-#elif COMPRESSED_PAIR_REV == 2
+#elif COMPRESSED_PAIR_REV == 3
 #define _LLDB_COMPRESSED_PAIR(T1, Name1, T2, Name2)                            \
   _LLDB_NO_UNIQUE_ADDRESS T1 Name1;                                            \
   _LLDB_NO_UNIQUE_ADDRESS T2 Name2
diff --git a/lldb/packages/Python/lldbsuite/test/test_categories.py b/lldb/packages/Python/lldbsuite/test/test_categories.py
index b585f695adeab..1f6e8a78e0c0d 100644
--- a/lldb/packages/Python/lldbsuite/test/test_categories.py
+++ b/lldb/packages/Python/lldbsuite/test/test_categories.py
@@ -33,6 +33,7 @@
     "lldb-server": "Tests related to lldb-server",
     "lldb-dap": "Tests for the Debug Adapter Protocol with lldb-dap",
     "llgs": "Tests for the gdb-server functionality of lldb-server",
+    "msvcstl": "Test for MSVC STL data formatters",
     "pexpect": "Tests requiring the pexpect library to be available",
     "objc": "Tests related to the Objective-C programming language support",
     "pyapi": "Tests related to the Python API",
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 0fe36cd4bc71f..d227a66a703c1 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -199,8 +199,8 @@ def _read_packet_thread(self):
         finally:
             dump_dap_log(self.log_file)
 
-    def get_modules(self):
-        module_list = self.request_modules()["body"]["modules"]
+    def get_modules(self, startModule: int = 0, moduleCount: int = 0):
+        module_list = self.request_modules(startModule, moduleCount)["body"]["modules"]
         modules = {}
         for module in module_list:
             modules[module["name"]] = module
@@ -1143,8 +1143,14 @@ def request_completions(self, text, frameId=None):
         }
         return self.send_recv(command_dict)
 
-    def request_modules(self):
-        return self.send_recv({"command": "modules", "type": "request"})
+    def request_modules(self, startModule: int, moduleCount: int):
+        return self.send_recv(
+            {
+                "command": "modules",
+                "type": "request",
+                "arguments": {"startModule": startModule, "moduleCount": moduleCount},
+            }
+        )
 
     def request_stackTrace(
         self, threadId=None, startFrame=None, levels=None, format=None, dump=False
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index 3b54d598c3509..6299caf7631af 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -344,7 +344,12 @@ def stepOver(
         granularity="statement",
         timeout=DEFAULT_TIMEOUT,
     ):
-        self.dap_server.request_next(threadId=threadId, granularity=granularity)
+        response = self.dap_server.request_next(
+            threadId=threadId, granularity=granularity
+        )
+        self.assertTrue(
+            response["success"], f"next request failed: response {response}"
+        )
         if waitForStop:
             return self.dap_server.wait_for_stopped(timeout)
         return None
diff --git a/lldb/scripts/framework-header-fix.py b/lldb/scripts/framework-header-fix.py
index d8c38511f19af..6ea8df4c24dd4 100755
--- a/lldb/scripts/framework-header-fix.py
+++ b/lldb/scripts/framework-header-fix.py
@@ -113,6 +113,10 @@ def main():
     # arguments in and of themself, so they need to passed in without dashes.
     unifdef_guards = ["-" + guard for guard in args.unifdef_guards]
 
+    # Create the framework's header dir if it doesn't already exist
+    if not os.path.exists(os.path.dirname(output_file_path)):
+        os.makedirs(os.path.dirname(output_file_path))
+
     if framework_version == "lldb_main":
         modify_main_includes(input_file_path, output_file_path)
     if framework_version == "lldb_rpc":
diff --git a/lldb/source/API/SBBreakpoint.cpp b/lldb/source/API/SBBreakpoint.cpp
index 87fadbcec4f26..397afc1f10f94 100644
--- a/lldb/source/API/SBBreakpoint.cpp
+++ b/lldb/source/API/SBBreakpoint.cpp
@@ -781,6 +781,18 @@ bool SBBreakpoint::IsHardware() const {
   return false;
 }
 
+lldb::SBError SBBreakpoint::SetIsHardware(bool is_hardware) {
+  LLDB_INSTRUMENT_VA(this, is_hardware);
+
+  BreakpointSP bkpt_sp = GetSP();
+  if (bkpt_sp) {
+    std::lock_guard<std::recursive_mutex> guard(
+        bkpt_sp->GetTarget().GetAPIMutex());
+    return SBError(Status::FromError(bkpt_sp->SetIsHardware(is_hardware)));
+  }
+  return SBError();
+}
+
 BreakpointSP SBBreakpoint::GetSP() const { return m_opaque_wp.lock(); }
 
 // This is simple collection of breakpoint id's and their target.
diff --git a/lldb/source/Breakpoint/Breakpoint.cpp b/lldb/source/Breakpoint/Breakpoint.cpp
index 2ed0c9314e3e1..8fc93cc8e0e51 100644
--- a/lldb/source/Breakpoint/Breakpoint.cpp
+++ b/lldb/source/Breakpoint/Breakpoint.cpp
@@ -61,7 +61,7 @@ Breakpoint::Breakpoint(Target &new_target, const Breakpoint &source_bp)
 Breakpoint::~Breakpoint() = default;
 
 BreakpointSP Breakpoint::CopyFromBreakpoint(TargetSP new_target,
-    const Breakpoint& bp_to_copy_from) {
+                                            const Breakpoint &bp_to_copy_from) {
   if (!new_target)
     return BreakpointSP();
 
@@ -163,7 +163,7 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
         std::make_shared<SearchFilterForUnconstrainedSearches>(target_sp);
   else {
     filter_sp = SearchFilter::CreateFromStructuredData(target_sp, *filter_dict,
-        create_error);
+                                                       create_error);
     if (create_error.Fail()) {
       error = Status::FromErrorStringWithFormat(
           "Error creating breakpoint filter from data: %s.",
@@ -174,7 +174,7 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
 
   std::unique_ptr<BreakpointOptions> options_up;
   StructuredData::Dictionary *options_dict;
-  Target& target = *target_sp;
+  Target &target = *target_sp;
   success = breakpoint_dict->GetValueForKeyAsDictionary(
       BreakpointOptions::GetSerializationKey(), options_dict);
   if (success) {
@@ -192,8 +192,8 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
   success = breakpoint_dict->GetValueForKeyAsBoolean(
       Breakpoint::GetKey(OptionNames::Hardware), hardware);
 
-  result_sp = target.CreateBreakpoint(filter_sp, resolver_sp, false,
-                                      hardware, true);
+  result_sp =
+      target.CreateBreakpoint(filter_sp, resolver_sp, false, hardware, true);
 
   if (result_sp && options_up) {
     result_sp->m_options = *options_up;
@@ -251,6 +251,45 @@ const lldb::TargetSP Breakpoint::GetTargetSP() {
 
 bool Breakpoint::IsInternal() const { return LLDB_BREAK_ID_IS_INTERNAL(m_bid); }
 
+llvm::Error Breakpoint::SetIsHardware(bool is_hardware) {
+  if (is_hardware == m_hardware)
+    return llvm::Error::success();
+
+  // Disable all non-hardware breakpoint locations.
+  std::vector<BreakpointLocationSP> locations;
+  for (BreakpointLocationSP location_sp : m_locations.BreakpointLocations()) {
+    if (!location_sp || !location_sp->IsEnabled())
+      continue;
+
+    lldb::BreakpointSiteSP breakpoint_site_sp =
+        location_sp->GetBreakpointSite();
+    if (!breakpoint_site_sp ||
+        breakpoint_site_sp->GetType() == BreakpointSite::eHardware)
+      continue;
+
+    locations.push_back(location_sp);
+    location_sp->SetEnabled(false);
+  }
+
+  // Toggle the hardware mode.
+  m_hardware = is_hardware;
+
+  // Re-enable all breakpoint locations.
+  size_t num_failures = 0;
+  for (BreakpointLocationSP location_sp : locations) {
+    if (!location_sp->SetEnabled(true))
+      num_failures++;
+  }
+
+  if (num_failures != 0)
+    return llvm::createStringError(
+        "%ull out of %ull breakpoint locations left disabled because they "
+        "couldn't be converted to hardware",
+        num_failures, locations.size());
+
+  return llvm::Error::success();
+}
+
 BreakpointLocationSP Breakpoint::AddLocation(const Address &addr,
                                              bool *new_location) {
   return m_locations.AddLocation(addr, m_resolve_indirect_symbols,
@@ -347,8 +386,7 @@ void Breakpoint::SetThreadID(lldb::tid_t thread_id) {
 lldb::tid_t Breakpoint::GetThreadID() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return LLDB_INVALID_THREAD_ID;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetTID();
+  return m_options.GetThreadSpecNoCreate()->GetTID();
 }
 
 void Breakpoint::SetThreadIndex(uint32_t index) {
@@ -362,8 +400,7 @@ void Breakpoint::SetThreadIndex(uint32_t index) {
 uint32_t Breakpoint::GetThreadIndex() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return 0;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetIndex();
+  return m_options.GetThreadSpecNoCreate()->GetIndex();
 }
 
 void Breakpoint::SetThreadName(const char *thread_name) {
@@ -378,8 +415,7 @@ void Breakpoint::SetThreadName(const char *thread_name) {
 const char *Breakpoint::GetThreadName() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return nullptr;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetName();
+  return m_options.GetThreadSpecNoCreate()->GetName();
 }
 
 void Breakpoint::SetQueueName(const char *queue_name) {
@@ -394,8 +430,7 @@ void Breakpoint::SetQueueName(const char *queue_name) {
 const char *Breakpoint::GetQueueName() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return nullptr;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetQueueName();
+  return m_options.GetThreadSpecNoCreate()->GetQueueName();
 }
 
 void Breakpoint::SetCondition(const char *condition) {
@@ -851,8 +886,8 @@ void Breakpoint::GetDescription(Stream *s, lldb::DescriptionLevel level,
     if (level == eDescriptionLevelBrief) {
       s->PutCString(GetBreakpointKind());
       return;
-    } else
-      s->Printf("Kind: %s\n", GetBreakpointKind());
+    }
+    s->Printf("Kind: %s\n", GetBreakpointKind());
   }
 
   const size_t num_locations = GetNumLocations();
@@ -956,8 +991,7 @@ void Breakpoint::GetResolverDescription(Stream *s) {
     m_resolver_sp->GetDescription(s);
 }
 
-bool Breakpoint::GetMatchingFileLine(ConstString filename,
-                                     uint32_t line_number,
+bool Breakpoint::GetMatchingFileLine(ConstString filename, uint32_t line_number,
                                      BreakpointLocationCollection &loc_coll) {
   // TODO: To be correct, this method needs to fill the breakpoint location
   // collection
@@ -1014,19 +1048,32 @@ void Breakpoint::SendBreakpointChangedEvent(
 
 const char *Breakpoint::BreakpointEventTypeAsCString(BreakpointEventType type) {
   switch (type) {
-    case eBreakpointEventTypeInvalidType: return "invalid";
-    case eBreakpointEventTypeAdded: return "breakpoint added";
-    case eBreakpointEventTypeRemoved: return "breakpoint removed";
-    case eBreakpointEventTypeLocationsAdded: return "locations added";
-    case eBreakpointEventTypeLocationsRemoved: return "locations removed";
-    case eBreakpointEventTypeLocationsResolved: return "locations resolved";
-    case eBreakpointEventTypeEnabled: return "breakpoint enabled";
-    case eBreakpointEventTypeDisabled: return "breakpoint disabled";
-    case eBreakpointEventTypeCommandChanged: return "command changed";
-    case eBreakpointEventTypeConditionChanged: return "condition changed";
-    case eBreakpointEventTypeIgnoreChanged: return "ignore count changed";
-    case eBreakpointEventTypeThreadChanged: return "thread changed";
-    case eBreakpointEventTypeAutoContinueChanged: return "autocontinue changed";
+  case eBreakpointEventTypeInvalidType:
+    return "invalid";
+  case eBreakpointEventTypeAdded:
+    return "breakpoint added";
+  case eBreakpointEventTypeRemoved:
+    return "breakpoint removed";
+  case eBreakpointEventTypeLocationsAdded:
+    return "locations added";
+  case eBreakpointEventTypeLocationsRemoved:
+    return "locations removed";
+  case eBreakpointEventTypeLocationsResolved:
+    return "locations resolved";
+  case eBreakpointEventTypeEnabled:
+    return "breakpoint enabled";
+  case eBreakpointEventTypeDisabled:
+    return "breakpoint disabled";
+  case eBreakpointEventTypeCommandChanged:
+    return "command changed";
+  case eBreakpointEventTypeConditionChanged:
+    return "condition changed";
+  case eBreakpointEventTypeIgnoreChanged:
+    return "ignore count changed";
+  case eBreakpointEventTypeThreadChanged:
+    return "thread changed";
+  case eBreakpointEventTypeAutoContinueChanged:
+    return "autocontinue changed";
   };
   llvm_unreachable("Fully covered switch above!");
 }
@@ -1064,7 +1111,7 @@ void Breakpoint::BreakpointEventData::Dump(Stream *s) const {
   BreakpointEventType event_type = GetBreakpointEventType();
   break_id_t bkpt_id = GetBreakpoint()->GetID();
   s->Format("bkpt: {0} type: {1}", bkpt_id,
-      BreakpointEventTypeAsCString(event_type));
+            BreakpointEventTypeAsCString(event_type));
 }
 
 const Breakpoint::BreakpointEventData *
@@ -1085,8 +1132,7 @@ Breakpoint::BreakpointEventData::GetBreakpointEventTypeFromEvent(
 
   if (data == nullptr)
     return eBreakpointEventTypeInvalidType;
-  else
-    return data->GetBreakpointEventType();
+  return data->GetBreakpointEventType();
 }
 
 BreakpointSP Breakpoint::BreakpointEventData::GetBreakpointFromEvent(
diff --git a/lldb/source/Breakpoint/BreakpointIDList.cpp b/lldb/source/Breakpoint/BreakpointIDList.cpp
index 5fc9f95a75db1..8c46b39c02a9e 100644
--- a/lldb/source/Breakpoint/BreakpointIDList.cpp
+++ b/lldb/source/Breakpoint/BreakpointIDList.cpp
@@ -99,8 +99,8 @@ llvm::Error BreakpointIDList::FindAndReplaceIDRanges(
         new_args.Clear();
         return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                        error.AsCString());
-      } else
-        names_found.insert(std::string(current_arg));
+      }
+      names_found.insert(std::string(current_arg));
     } else if ((i + 2 < old_args.size()) &&
                BreakpointID::IsRangeIdentifier(old_args[i + 1].ref()) &&
                BreakpointID::IsValidIDExpression(current_arg) &&
@@ -256,7 +256,7 @@ llvm::Error BreakpointIDList::FindAndReplaceIDRanges(
       else
         iter++;
     }
-    
+
     if (!names_found.empty()) {
       for (BreakpointSP bkpt_sp : target->GetBreakpointList().Breakpoints()) {
         for (const std::string &name : names_found) {
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index c7ea50407ae1c..7553315946043 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -31,7 +31,7 @@ using namespace lldb_private;
 
 BreakpointLocation::BreakpointLocation(break_id_t loc_id, Breakpoint &owner,
                                        const Address &addr, lldb::tid_t tid,
-                                       bool hardware, bool check_for_resolver)
+                                       bool check_for_resolver)
     : m_should_resolve_indirect_functions(false), m_is_reexported(false),
       m_is_indirect(false), m_address(addr), m_owner(owner),
       m_condition_hash(0), m_loc_id(loc_id), m_hit_counter() {
@@ -55,8 +55,7 @@ const BreakpointOptions &BreakpointLocation::GetOptionsSpecifyingKind(
     BreakpointOptions::OptionKind kind) const {
   if (m_options_up && m_options_up->IsOptionSet(kind))
     return *m_options_up;
-  else
-    return m_owner.GetOptions();
+  return m_owner.GetOptions();
 }
 
 Address &BreakpointLocation::GetAddress() { return m_address; }
@@ -68,29 +67,25 @@ Target &BreakpointLocation::GetTarget() { return m_owner.GetTarget(); }
 bool BreakpointLocation::IsEnabled() const {
   if (!m_owner.IsEnabled())
     return false;
-  else if (m_options_up != nullptr)
+  if (m_options_up != nullptr)
     return m_options_up->IsEnabled();
-  else
-    return true;
+  return true;
 }
 
-void BreakpointLocation::SetEnabled(bool enabled) {
+bool BreakpointLocation::SetEnabled(bool enabled) {
   GetLocationOptions().SetEnabled(enabled);
-  if (enabled) {
-    ResolveBreakpointSite();
-  } else {
-    ClearBreakpointSite();
-  }
+  const bool success =
+      enabled ? ResolveBreakpointSite() : ClearBreakpointSite();
   SendBreakpointLocationChangedEvent(enabled ? eBreakpointEventTypeEnabled
                                              : eBreakpointEventTypeDisabled);
+  return success;
 }
 
 bool BreakpointLocation::IsAutoContinue() const {
   if (m_options_up &&
       m_options_up->IsOptionSet(BreakpointOptions::eAutoContinue))
     return m_options_up->IsAutoContinue();
-  else
-    return m_owner.IsAutoContinue();
+  return m_owner.IsAutoContinue();
 }
 
 void BreakpointLocation::SetAutoContinue(bool auto_continue) {
@@ -109,8 +104,7 @@ lldb::tid_t BreakpointLocation::GetThreadID() {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetTID();
-  else
-    return LLDB_INVALID_THREAD_ID;
+  return LLDB_INVALID_THREAD_ID;
 }
 
 void BreakpointLocation::SetThreadIndex(uint32_t index) {
@@ -131,8 +125,7 @@ uint32_t BreakpointLocation::GetThreadIndex() const {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetIndex();
-  else
-    return 0;
+  return 0;
 }
 
 void BreakpointLocation::SetThreadName(const char *thread_name) {
@@ -153,8 +146,7 @@ const char *BreakpointLocation::GetThreadName() const {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetName();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 void BreakpointLocation::SetQueueName(const char *queue_name) {
@@ -175,22 +167,19 @@ const char *BreakpointLocation::GetQueueName() const {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetQueueName();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 bool BreakpointLocation::InvokeCallback(StoppointCallbackContext *context) {
   if (m_options_up != nullptr && m_options_up->HasCallback())
     return m_options_up->InvokeCallback(context, m_owner.GetID(), GetID());
-  else
-    return m_owner.InvokeCallback(context, GetID());
+  return m_owner.InvokeCallback(context, GetID());
 }
 
 bool BreakpointLocation::IsCallbackSynchronous() {
   if (m_options_up != nullptr && m_options_up->HasCallback())
     return m_options_up->IsCallbackSynchronous();
-  else
-    return m_owner.GetOptions().IsCallbackSynchronous();
+  return m_owner.GetOptions().IsCallbackSynchronous();
 }
 
 void BreakpointLocation::SetCallback(BreakpointHitCallback callback,
@@ -445,10 +434,10 @@ bool BreakpointLocation::ResolveBreakpointSite() {
       process->CreateBreakpointSite(shared_from_this(), m_owner.IsHardware());
 
   if (new_id == LLDB_INVALID_BREAK_ID) {
-    Log *log = GetLog(LLDBLog::Breakpoints);
-    if (log)
-      log->Warning("Failed to add breakpoint site at 0x%" PRIx64,
-                   m_address.GetOpcodeLoadAddress(&m_owner.GetTarget()));
+    LLDB_LOGF(GetLog(LLDBLog::Breakpoints),
+              "Failed to add breakpoint site at 0x%" PRIx64 "(resolved=%s)",
+              m_address.GetOpcodeLoadAddress(&m_owner.GetTarget()),
+              IsResolved() ? "yes" : "no");
   }
 
   return IsResolved();
diff --git a/lldb/source/Breakpoint/BreakpointLocationCollection.cpp b/lldb/source/Breakpoint/BreakpointLocationCollection.cpp
index 28a3639e95bb6..81bec0bd7583d 100644
--- a/lldb/source/Breakpoint/BreakpointLocationCollection.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocationCollection.cpp
@@ -121,7 +121,7 @@ bool BreakpointLocationCollection::ShouldStop(
   size_t prev_size = GetSize();
   while (i < prev_size) {
     // ShouldStop can remove the breakpoint from the list, or even delete
-    // it, so we should 
+    // it, so we should
     BreakpointLocationSP cur_loc_sp = GetByIndex(i);
     BreakpointSP keep_bkpt_alive_sp = cur_loc_sp->GetBreakpoint().shared_from_this();
     if (cur_loc_sp->ShouldStop(context))
diff --git a/lldb/source/Breakpoint/BreakpointLocationList.cpp b/lldb/source/Breakpoint/BreakpointLocationList.cpp
index 0240305d6f292..1d8b4c1ccfaeb 100644
--- a/lldb/source/Breakpoint/BreakpointLocationList.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocationList.cpp
@@ -32,7 +32,7 @@ BreakpointLocationList::Create(const Address &addr,
   lldb::break_id_t bp_loc_id = ++m_next_id;
   BreakpointLocationSP bp_loc_sp(
       new BreakpointLocation(bp_loc_id, m_owner, addr, LLDB_INVALID_THREAD_ID,
-                             m_owner.IsHardware(), resolve_indirect_symbols));
+                             resolve_indirect_symbols));
   m_locations.push_back(bp_loc_sp);
   m_address_to_location[addr] = bp_loc_sp;
   return bp_loc_sp;
@@ -72,8 +72,7 @@ BreakpointLocationList::FindByID(lldb::break_id_t break_id) const {
       llvm::lower_bound(m_locations, break_id, Compare);
   if (pos != end && (*pos)->GetID() == break_id)
     return *(pos);
-  else
-    return BreakpointLocationSP();
+  return BreakpointLocationSP();
 }
 
 size_t BreakpointLocationList::FindInModule(
diff --git a/lldb/source/Breakpoint/BreakpointName.cpp b/lldb/source/Breakpoint/BreakpointName.cpp
index 5a4aad172fc02..8249d8f82236e 100644
--- a/lldb/source/Breakpoint/BreakpointName.cpp
+++ b/lldb/source/Breakpoint/BreakpointName.cpp
@@ -21,12 +21,8 @@ using namespace lldb;
 using namespace lldb_private;
 
 const Flags::ValueType BreakpointName::Permissions::permissions_mask
-    [BreakpointName::Permissions::PermissionKinds::allPerms + 1] =  { 
-      (1u << 0),
-      (1u << 1),
-      (1u << 2),
-      (0x5u)
-};
+    [BreakpointName::Permissions::PermissionKinds::allPerms + 1] = {
+        (1u << 0), (1u << 1), (1u << 2), (0x5u)};
 
 bool BreakpointName::Permissions::GetDescription(Stream *s,
                                                  lldb::DescriptionLevel level) {
@@ -36,10 +32,10 @@ bool BreakpointName::Permissions::GetDescription(Stream *s,
     s->Indent();
     if (IsSet(listPerm))
       s->Printf("list: %s", GetAllowList() ? "allowed" : "disallowed");
-  
+
     if (IsSet(disablePerm))
       s->Printf("disable: %s", GetAllowDisable() ? "allowed" : "disallowed");
-  
+
     if (IsSet(deletePerm))
       s->Printf("delete: %s", GetAllowDelete() ? "allowed" : "disallowed");
     s->IndentLess();
diff --git a/lldb/source/Breakpoint/BreakpointOptions.cpp b/lldb/source/Breakpoint/BreakpointOptions.cpp
index 242b5b30168c5..08e48c4921078 100644
--- a/lldb/source/Breakpoint/BreakpointOptions.cpp
+++ b/lldb/source/Breakpoint/BreakpointOptions.cpp
@@ -99,8 +99,8 @@ BreakpointOptions::CommandData::CreateFromStructuredData(
 
 const char *BreakpointOptions::g_option_names[(
     size_t)BreakpointOptions::OptionNames::LastOptionName]{
-    "ConditionText", "IgnoreCount", 
-    "EnabledState", "OneShotState", "AutoContinue"};
+    "ConditionText", "IgnoreCount", "EnabledState", "OneShotState",
+    "AutoContinue"};
 
 // BreakpointOptions constructor
 BreakpointOptions::BreakpointOptions(bool all_flags_set)
@@ -113,7 +113,7 @@ BreakpointOptions::BreakpointOptions(bool all_flags_set)
 }
 
 BreakpointOptions::BreakpointOptions(const char *condition, bool enabled,
-                                     int32_t ignore, bool one_shot, 
+                                     int32_t ignore, bool one_shot,
                                      bool auto_continue)
     : m_callback(nullptr), m_baton_is_command_baton(false),
       m_callback_is_synchronous(false), m_enabled(enabled),
@@ -247,7 +247,7 @@ std::unique_ptr<BreakpointOptions> BreakpointOptions::CreateFromStructuredData(
       }
       set_options.Set(eOneShot);
   }
-  
+
   key = GetKey(OptionNames::AutoContinue);
   if (key && options_dict.HasKey(key)) {
     success = options_dict.GetValueForKeyAsBoolean(key, auto_continue);
@@ -258,7 +258,7 @@ std::unique_ptr<BreakpointOptions> BreakpointOptions::CreateFromStructuredData(
       }
       set_options.Set(eAutoContinue);
   }
-  
+
   key = GetKey(OptionNames::IgnoreCount);
   if (key && options_dict.HasKey(key)) {
     success = options_dict.GetValueForKeyAsInteger(key, ignore_count);
@@ -297,8 +297,8 @@ std::unique_ptr<BreakpointOptions> BreakpointOptions::CreateFromStructuredData(
   }
 
   auto bp_options = std::make_unique<BreakpointOptions>(
-      condition_ref.str().c_str(), enabled, 
-      ignore_count, one_shot, auto_continue);
+      condition_ref.str().c_str(), enabled, ignore_count, one_shot,
+      auto_continue);
   if (cmd_data_up) {
     if (cmd_data_up->interpreter == eScriptLanguageNone)
       bp_options->SetCommandDataCallback(cmd_data_up);
@@ -364,7 +364,7 @@ StructuredData::ObjectSP BreakpointOptions::SerializeToStructuredData() {
   if (m_set_flags.Test(eCondition))
     options_dict_sp->AddStringItem(GetKey(OptionNames::ConditionText),
                                    m_condition_text);
-         
+
   if (m_set_flags.Test(eCallback) && m_baton_is_command_baton) {
     auto cmd_baton =
         std::static_pointer_cast<CommandBaton>(m_callback_baton_sp);
@@ -438,7 +438,8 @@ bool BreakpointOptions::InvokeCallback(StoppointCallbackContext *context,
         return m_callback(m_callback_baton_sp ? m_callback_baton_sp->data()
                                           : nullptr,
                       context, break_id, break_loc_id);
-    } else if (IsCallbackSynchronous()) {
+    }
+    if (IsCallbackSynchronous()) {
       return false;
     }
   }
@@ -482,9 +483,8 @@ const char *BreakpointOptions::GetConditionText(size_t *hash) const {
       *hash = m_condition_text_hash;
 
     return m_condition_text.c_str();
-  } else {
-    return nullptr;
   }
+  return nullptr;
 }
 
 const ThreadSpec *BreakpointOptions::GetThreadSpecNoCreate() const {
diff --git a/lldb/source/Breakpoint/BreakpointResolverName.cpp b/lldb/source/Breakpoint/BreakpointResolverName.cpp
index b9c8a1635d844..edde1c91b789c 100644
--- a/lldb/source/Breakpoint/BreakpointResolverName.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverName.cpp
@@ -124,14 +124,14 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData(
     return std::make_shared<BreakpointResolverName>(
         nullptr, RegularExpression(regex_text), language, offset,
         skip_prologue);
-  } else {
-    StructuredData::Array *names_array;
-    success = options_dict.GetValueForKeyAsArray(
-        GetKey(OptionNames::SymbolNameArray), names_array);
-    if (!success) {
-      error = Status::FromErrorString("BRN::CFSD: Missing symbol names entry.");
-      return nullptr;
-    }
+  }
+  StructuredData::Array *names_array;
+  success = options_dict.GetValueForKeyAsArray(
+      GetKey(OptionNames::SymbolNameArray), names_array);
+  if (!success) {
+    error = Status::FromErrorString("BRN::CFSD: Missing symbol names entry.");
+    return nullptr;
+  }
     StructuredData::Array *names_mask_array;
     success = options_dict.GetValueForKeyAsArray(
         GetKey(OptionNames::NameMaskArray), names_mask_array);
@@ -182,7 +182,6 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData(
       resolver_sp->AddNameLookup(ConstString(names[i]), name_masks[i]);
     }
     return resolver_sp;
-  }
 }
 
 StructuredData::ObjectSP BreakpointResolverName::SerializeToStructuredData() {
diff --git a/lldb/source/Breakpoint/Watchpoint.cpp b/lldb/source/Breakpoint/Watchpoint.cpp
index 2df848aaa0576..f1366ca538075 100644
--- a/lldb/source/Breakpoint/Watchpoint.cpp
+++ b/lldb/source/Breakpoint/Watchpoint.cpp
@@ -477,8 +477,7 @@ void Watchpoint::SetCondition(const char *condition) {
 const char *Watchpoint::GetConditionText() const {
   if (m_condition_up)
     return m_condition_up->GetUserText();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 void Watchpoint::SendWatchpointChangedEvent(
@@ -534,8 +533,7 @@ Watchpoint::WatchpointEventData::GetWatchpointEventTypeFromEvent(
 
   if (data == nullptr)
     return eWatchpointEventTypeInvalidType;
-  else
-    return data->GetWatchpointEventType();
+  return data->GetWatchpointEventType();
 }
 
 WatchpointSP Watchpoint::WatchpointEventData::GetWatchpointFromEvent(
diff --git a/lldb/source/Breakpoint/WatchpointOptions.cpp b/lldb/source/Breakpoint/WatchpointOptions.cpp
index c27cd62e02206..321aab0700195 100644
--- a/lldb/source/Breakpoint/WatchpointOptions.cpp
+++ b/lldb/source/Breakpoint/WatchpointOptions.cpp
@@ -92,8 +92,8 @@ bool WatchpointOptions::InvokeCallback(StoppointCallbackContext *context,
     return m_callback(m_callback_baton_sp ? m_callback_baton_sp->data()
                                           : nullptr,
                       context, watch_id);
-  } else
-    return true;
+  }
+  return true;
 }
 
 bool WatchpointOptions::HasCallback() {
diff --git a/lldb/source/Commands/CommandObjectProtocolServer.cpp b/lldb/source/Commands/CommandObjectProtocolServer.cpp
index 55bd42ed1a533..f11e27f01c8a8 100644
--- a/lldb/source/Commands/CommandObjectProtocolServer.cpp
+++ b/lldb/source/Commands/CommandObjectProtocolServer.cpp
@@ -75,9 +75,12 @@ class CommandObjectProtocolServerStart : public CommandObjectParsed {
 
     ProtocolServer::Connection connection;
     connection.protocol = protocol_and_mode->first;
-    connection.name =
-        formatv("[{0}]:{1}", uri->hostname.empty() ? "0.0.0.0" : uri->hostname,
-                uri->port.value_or(0));
+    if (connection.protocol == Socket::SocketProtocol::ProtocolUnixDomain)
+      connection.name = uri->path;
+    else
+      connection.name = formatv(
+          "[{0}]:{1}", uri->hostname.empty() ? "0.0.0.0" : uri->hostname,
+          uri->port.value_or(0));
 
     if (llvm::Error error = server->Start(connection)) {
       result.AppendErrorWithFormatv("{0}", llvm::fmt_consume(std::move(error)));
@@ -90,6 +93,7 @@ class CommandObjectProtocolServerStart : public CommandObjectParsed {
       result.AppendMessageWithFormatv(
           "{0} server started with connection listeners: {1}", protocol,
           address);
+      result.SetStatus(eReturnStatusSuccessFinishNoResult);
     }
   }
 };
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index a4ced37649ea0..dbebbbd38093e 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -1412,11 +1412,13 @@ static bool DumpModuleSymbolFile(Stream &strm, Module *module) {
 }
 
 static bool GetSeparateDebugInfoList(StructuredData::Array &list,
-                                     Module *module, bool errors_only) {
+                                     Module *module, bool errors_only,
+                                     bool load_all_debug_info) {
   if (module) {
     if (SymbolFile *symbol_file = module->GetSymbolFile(/*can_create=*/true)) {
       StructuredData::Dictionary d;
-      if (symbol_file->GetSeparateDebugInfo(d, errors_only)) {
+      if (symbol_file->GetSeparateDebugInfo(d, errors_only,
+                                            load_all_debug_info)) {
         list.AddItem(
             std::make_shared<StructuredData::Dictionary>(std::move(d)));
         return true;
@@ -2522,6 +2524,10 @@ class CommandObjectTargetModulesDumpSeparateDebugInfoFiles
       const int short_option = m_getopt_table[option_idx].val;
 
       switch (short_option) {
+      case 'f':
+        m_load_all_debug_info.SetCurrentValue(true);
+        m_load_all_debug_info.SetOptionWasSet();
+        break;
       case 'j':
         m_json.SetCurrentValue(true);
         m_json.SetOptionWasSet();
@@ -2539,6 +2545,7 @@ class CommandObjectTargetModulesDumpSeparateDebugInfoFiles
     void OptionParsingStarting(ExecutionContext *execution_context) override {
       m_json.Clear();
       m_errors_only.Clear();
+      m_load_all_debug_info.Clear();
     }
 
     llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
@@ -2547,6 +2554,7 @@ class CommandObjectTargetModulesDumpSeparateDebugInfoFiles
 
     OptionValueBoolean m_json = false;
     OptionValueBoolean m_errors_only = false;
+    OptionValueBoolean m_load_all_debug_info = false;
   };
 
 protected:
@@ -2578,7 +2586,8 @@ class CommandObjectTargetModulesDumpSeparateDebugInfoFiles
 
         if (GetSeparateDebugInfoList(separate_debug_info_lists_by_module,
                                      module_sp.get(),
-                                     bool(m_options.m_errors_only)))
+                                     bool(m_options.m_errors_only),
+                                     bool(m_options.m_load_all_debug_info)))
           num_dumped++;
       }
     } else {
@@ -2599,7 +2608,8 @@ class CommandObjectTargetModulesDumpSeparateDebugInfoFiles
               break;
             Module *module = module_list.GetModulePointerAtIndex(i);
             if (GetSeparateDebugInfoList(separate_debug_info_lists_by_module,
-                                         module, bool(m_options.m_errors_only)))
+                                         module, bool(m_options.m_errors_only),
+                                         bool(m_options.m_load_all_debug_info)))
               num_dumped++;
           }
         } else
@@ -4086,8 +4096,6 @@ class CommandObjectTargetModulesLookup : public CommandObjectParsed {
     // Dump all sections for all modules images
 
     if (command.GetArgumentCount() == 0) {
-      ModuleSP current_module;
-
       // Where it is possible to look in the current symbol context first,
       // try that.  If this search was successful and --all was not passed,
       // don't print anything else.
@@ -4110,8 +4118,7 @@ class CommandObjectTargetModulesLookup : public CommandObjectParsed {
       }
 
       for (ModuleSP module_sp : target_modules.ModulesNoLocking()) {
-        if (module_sp != current_module &&
-            LookupInModule(m_interpreter, module_sp.get(), result,
+        if (LookupInModule(m_interpreter, module_sp.get(), result,
                            syntax_error)) {
           result.GetOutputStream().EOL();
           num_successful_lookups++;
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index 5327647d65cc4..e543566e4ff1e 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -13,6 +13,9 @@ let Command = "target modules dump separate debug info" in {
     Desc<"Output the details in JSON format.">;
   def tm_errors_only : Option<"errors-only", "e">, Group<1>,
     Desc<"Filter to show only debug info files with errors.">;
+  def tm_force_load_all_debug_info : Option<"force-load-all-debug-info", "f">,
+                                     Group<1>,
+                                     Desc<"Load all debug info files.">;
 }
 
 let Command = "help" in {
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 445baf1f63785..ed674ee1275c7 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -1445,6 +1445,13 @@ bool Debugger::PopIOHandler(const IOHandlerSP &pop_reader_sp) {
   return true;
 }
 
+void Debugger::RefreshIOHandler() {
+  std::lock_guard<std::recursive_mutex> guard(m_io_handler_stack.GetMutex());
+  IOHandlerSP reader_sp(m_io_handler_stack.Top());
+  if (reader_sp)
+    reader_sp->Refresh();
+}
+
 StreamUP Debugger::GetAsyncOutputStream() {
   return std::make_unique<StreamAsynchronousIO>(*this,
                                                 StreamAsynchronousIO::STDOUT);
diff --git a/lldb/source/Core/EmulateInstruction.cpp b/lldb/source/Core/EmulateInstruction.cpp
index d240b4d3b3310..634ce1075d54b 100644
--- a/lldb/source/Core/EmulateInstruction.cpp
+++ b/lldb/source/Core/EmulateInstruction.cpp
@@ -588,7 +588,99 @@ EmulateInstruction::GetInternalRegisterNumber(RegisterContext *reg_ctx,
   return LLDB_INVALID_REGNUM;
 }
 
+std::unique_ptr<SingleStepBreakpointLocationsPredictor>
+EmulateInstruction::CreateBreakpointLocationPredictor(
+    std::unique_ptr<EmulateInstruction> emulator_up) {
+  auto creator =
+      emulator_up->GetSingleStepBreakpointLocationsPredictorCreator();
+  return creator(std::move(emulator_up));
+}
+
+std::optional<lldb::addr_t> EmulateInstruction::ReadPC() {
+  bool success = false;
+  auto addr = ReadRegisterUnsigned(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC,
+                                   LLDB_INVALID_ADDRESS, &success);
+  return success ? std::optional<addr_t>(addr) : std::nullopt;
+}
+
+bool EmulateInstruction::WritePC(lldb::addr_t addr) {
+  EmulateInstruction::Context ctx;
+  ctx.type = eContextAdvancePC;
+  ctx.SetNoArgs();
+  return WriteRegisterUnsigned(ctx, eRegisterKindGeneric,
+                               LLDB_REGNUM_GENERIC_PC, addr);
+}
+
 bool EmulateInstruction::CreateFunctionEntryUnwind(UnwindPlan &unwind_plan) {
   unwind_plan.Clear();
   return false;
 }
+
+BreakpointLocations
+SingleStepBreakpointLocationsPredictor::GetBreakpointLocations(Status &status) {
+  if (!m_emulator_up->ReadInstruction()) {
+    // try to get at least the size of next instruction to set breakpoint.
+    lldb::addr_t next_pc = GetNextInstructionAddress(status);
+    return BreakpointLocations{next_pc};
+  }
+
+  auto entry_pc = m_emulator_up->ReadPC();
+  if (!entry_pc) {
+    status = Status("Can't read PC");
+    return {};
+  }
+
+  m_emulation_result = m_emulator_up->EvaluateInstruction(
+      eEmulateInstructionOptionAutoAdvancePC);
+
+  lldb::addr_t next_pc = GetBreakpointLocationAddress(*entry_pc, status);
+  return BreakpointLocations{next_pc};
+}
+
+lldb::addr_t SingleStepBreakpointLocationsPredictor::GetNextInstructionAddress(
+    Status &error) {
+  auto instr_size = m_emulator_up->GetLastInstrSize();
+  if (!instr_size) {
+    error = Status("Read instruction failed!");
+    return LLDB_INVALID_ADDRESS;
+  }
+
+  auto pc = m_emulator_up->ReadPC();
+  if (!pc) {
+    error = Status("Can't read PC");
+    return LLDB_INVALID_ADDRESS;
+  }
+
+  lldb::addr_t next_pc = *pc + *instr_size;
+  return next_pc;
+}
+
+lldb::addr_t
+SingleStepBreakpointLocationsPredictor::GetBreakpointLocationAddress(
+    lldb::addr_t entry_pc, Status &error) {
+  auto addr = m_emulator_up->ReadPC();
+  if (!addr) {
+    error = Status("Can't read PC");
+    return LLDB_INVALID_ADDRESS;
+  }
+  lldb::addr_t pc = *addr;
+
+  if (m_emulation_result) {
+    assert(entry_pc != pc && "Emulation was successfull but PC wasn't updated");
+    return pc;
+  }
+
+  if (entry_pc == pc) {
+    // Emulate instruction failed and it hasn't changed PC. Advance PC with
+    // the size of the current opcode because the emulation of all
+    // PC modifying instruction should be successful. The failure most
+    // likely caused by an unsupported instruction which does not modify PC.
+    return pc + m_emulator_up->GetOpcode().GetByteSize();
+  }
+
+  // The instruction emulation failed after it modified the PC. It is an
+  // unknown error where we can't continue because the next instruction is
+  // modifying the PC but we don't  know how.
+  error = Status("Instruction emulation failed unexpectedly.");
+  return LLDB_INVALID_ADDRESS;
+}
diff --git a/lldb/source/Core/IOHandler.cpp b/lldb/source/Core/IOHandler.cpp
index 8aac507eaa0c2..f65a1113f3592 100644
--- a/lldb/source/Core/IOHandler.cpp
+++ b/lldb/source/Core/IOHandler.cpp
@@ -663,3 +663,10 @@ void IOHandlerEditline::PrintAsync(const char *s, size_t len, bool is_stdout) {
 #endif
   }
 }
+
+void IOHandlerEditline::Refresh() {
+#if LLDB_ENABLE_LIBEDIT
+  if (m_editline_up)
+    m_editline_up->Refresh();
+#endif
+}
diff --git a/lldb/source/Core/Statusline.cpp b/lldb/source/Core/Statusline.cpp
index 8a8640805cac0..393d427241021 100644
--- a/lldb/source/Core/Statusline.cpp
+++ b/lldb/source/Core/Statusline.cpp
@@ -24,6 +24,7 @@
 #define ANSI_SAVE_CURSOR ESCAPE "7"
 #define ANSI_RESTORE_CURSOR ESCAPE "8"
 #define ANSI_CLEAR_BELOW ESCAPE "[J"
+#define ANSI_CLEAR_SCREEN ESCAPE "[2J"
 #define ANSI_SET_SCROLL_ROWS ESCAPE "[1;%ur"
 #define ANSI_TO_START_OF_ROW ESCAPE "[%u;1f"
 #define ANSI_REVERSE_VIDEO ESCAPE "[7m"
@@ -41,10 +42,12 @@ Statusline::Statusline(Debugger &debugger)
 Statusline::~Statusline() { Disable(); }
 
 void Statusline::TerminalSizeChanged() {
-  UpdateTerminalProperties();
+  m_terminal_width = m_debugger.GetTerminalWidth();
+  m_terminal_height = m_debugger.GetTerminalHeight();
+
+  UpdateScrollWindow(ResizeStatusline);
 
-  // This definitely isn't signal safe, but the best we can do, until we
-  // have proper signal-catching thread.
+  // Draw the old statusline.
   Redraw(/*update=*/false);
 }
 
@@ -85,13 +88,6 @@ void Statusline::Draw(std::string str) {
   locked_stream << ANSI_RESTORE_CURSOR;
 }
 
-void Statusline::UpdateTerminalProperties() {
-  UpdateScrollWindow(DisableStatusline);
-  m_terminal_width = m_debugger.GetTerminalWidth();
-  m_terminal_height = m_debugger.GetTerminalHeight();
-  UpdateScrollWindow(EnableStatusline);
-}
-
 void Statusline::UpdateScrollWindow(ScrollWindowMode mode) {
   assert(m_terminal_width != 0 && m_terminal_height != 0);
 
@@ -99,24 +95,36 @@ void Statusline::UpdateScrollWindow(ScrollWindowMode mode) {
   if (!stream_sp)
     return;
 
-  const unsigned scroll_height =
-      (mode == DisableStatusline) ? m_terminal_height : m_terminal_height - 1;
-
+  const unsigned reduced_scroll_window = m_terminal_height - 1;
   LockedStreamFile locked_stream = stream_sp->Lock();
-  locked_stream << ANSI_SAVE_CURSOR;
-  locked_stream.Printf(ANSI_SET_SCROLL_ROWS, scroll_height);
-  locked_stream << ANSI_RESTORE_CURSOR;
+
   switch (mode) {
   case EnableStatusline:
     // Move everything on the screen up.
-    locked_stream.Printf(ANSI_UP_ROWS, 1);
     locked_stream << '\n';
+    locked_stream.Printf(ANSI_UP_ROWS, 1);
+    // Reduce the scroll window.
+    locked_stream << ANSI_SAVE_CURSOR;
+    locked_stream.Printf(ANSI_SET_SCROLL_ROWS, reduced_scroll_window);
+    locked_stream << ANSI_RESTORE_CURSOR;
     break;
   case DisableStatusline:
+    // Reset the scroll window.
+    locked_stream << ANSI_SAVE_CURSOR;
+    locked_stream.Printf(ANSI_SET_SCROLL_ROWS, 0);
+    locked_stream << ANSI_RESTORE_CURSOR;
     // Clear the screen below to hide the old statusline.
     locked_stream << ANSI_CLEAR_BELOW;
     break;
+  case ResizeStatusline:
+    // Clear the screen and update the scroll window.
+    // FIXME: Find a better solution (#146919).
+    locked_stream << ANSI_CLEAR_SCREEN;
+    locked_stream.Printf(ANSI_SET_SCROLL_ROWS, reduced_scroll_window);
+    break;
   }
+
+  m_debugger.RefreshIOHandler();
 }
 
 void Statusline::Redraw(bool update) {
diff --git a/lldb/source/Core/Telemetry.cpp b/lldb/source/Core/Telemetry.cpp
index 6434b11a63ad4..2f2e1c1d56d52 100644
--- a/lldb/source/Core/Telemetry.cpp
+++ b/lldb/source/Core/Telemetry.cpp
@@ -119,6 +119,19 @@ llvm::Error TelemetryManager::preDispatch(TelemetryInfo *entry) {
   return llvm::Error::success();
 }
 
+// Helper for extracting time field from a Dictionary.
+static std::optional<std::chrono::nanoseconds>
+GetAsNanosec(StructuredData::Dictionary *dict, llvm::StringRef key) {
+  auto value = dict->GetValueForKey(key);
+  if (!value->IsValid()) {
+    LLDB_LOG(GetLog(LLDBLog::Object),
+             "Cannot determine {0} from client-telemetry entry", key);
+    return std::nullopt;
+  }
+
+  return std::chrono::nanoseconds(value->GetUnsignedIntegerValue(0));
+}
+
 void TelemetryManager::DispatchClientTelemetry(
     const lldb_private::StructuredDataImpl &entry, Debugger *debugger) {
   if (!m_config->enable_client_telemetry)
@@ -148,23 +161,12 @@ void TelemetryManager::DispatchClientTelemetry(
     LLDB_LOG(GetLog(LLDBLog::Object),
              "Cannot determine client_data from client-telemetry entry");
 
-  int64_t start_time;
-  if (dict->GetValueForKeyAsInteger("start_time", start_time)) {
-    client_info.start_time +=
-        std::chrono::nanoseconds(static_cast<size_t>(start_time));
-  } else {
-    LLDB_LOG(GetLog(LLDBLog::Object),
-             "Cannot determine start-time from client-telemetry entry");
-  }
+  if (auto maybe_start_time = GetAsNanosec(dict, "start_time"))
+    client_info.start_time += *maybe_start_time;
 
-  int64_t end_time;
-  if (dict->GetValueForKeyAsInteger("end_time", end_time)) {
+  if (auto maybe_end_time = GetAsNanosec(dict, "end_time")) {
     SteadyTimePoint epoch;
-    client_info.end_time =
-        epoch + std::chrono::nanoseconds(static_cast<size_t>(end_time));
-  } else {
-    LLDB_LOG(GetLog(LLDBLog::Object),
-             "Cannot determine end-time from client-telemetry entry");
+    client_info.end_time = epoch + *maybe_end_time;
   }
 
   llvm::StringRef error_msg;
diff --git a/lldb/source/DataFormatters/FormattersHelpers.cpp b/lldb/source/DataFormatters/FormattersHelpers.cpp
index d7b058d91c4a3..028fc5da73dc0 100644
--- a/lldb/source/DataFormatters/FormattersHelpers.cpp
+++ b/lldb/source/DataFormatters/FormattersHelpers.cpp
@@ -126,3 +126,28 @@ lldb_private::formatters::GetArrayAddressOrPointerValue(ValueObject &valobj) {
 
   return data_addr.address;
 }
+
+void lldb_private::formatters::DumpCxxSmartPtrPointerSummary(
+    Stream &stream, ValueObject &ptr, const TypeSummaryOptions &options) {
+  if (ptr.GetValueAsUnsigned(0) == 0) {
+    stream.Printf("nullptr");
+    return;
+  }
+
+  Status error;
+  ValueObjectSP pointee_sp = ptr.Dereference(error);
+  if (!pointee_sp || !error.Success())
+    return;
+
+  if (!pointee_sp->DumpPrintableRepresentation(
+          stream, ValueObject::eValueObjectRepresentationStyleSummary,
+          lldb::eFormatInvalid,
+          ValueObject::PrintableRepresentationSpecialCases::eDisable, false))
+    stream.Printf("ptr = 0x%" PRIx64, ptr.GetValueAsUnsigned(0));
+}
+
+bool lldb_private::formatters::ContainerSizeSummaryProvider(
+    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  return FormatEntity::FormatStringRef("size=${svar%#}", stream, nullptr,
+                                       nullptr, nullptr, &valobj, false, false);
+}
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 661324338e801..52891fcefd68b 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -36,7 +36,8 @@
 #include "lldb/Target/StackID.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -81,7 +82,7 @@ void DWARFExpression::DumpLocation(Stream *s, lldb::DescriptionLevel level,
   llvm::DIDumpOptions DumpOpts;
   DumpOpts.GetNameForDWARFReg = GetRegName;
   llvm::DWARFExpression E(m_data.GetAsLLVM(), m_data.GetAddressByteSize());
-  llvm::DWARFExpressionPrinter::print(&E, s->AsRawOstream(), DumpOpts, nullptr);
+  llvm::printDwarfExpression(&E, s->AsRawOstream(), DumpOpts, nullptr);
 }
 
 RegisterKind DWARFExpression::GetRegisterKind() const { return m_reg_kind; }
@@ -588,7 +589,7 @@ bool DWARFExpression::LinkThreadLocalStorage(
   return true;
 }
 
-static llvm::Error Evaluate_DW_OP_entry_value(std::vector<Value> &stack,
+static llvm::Error Evaluate_DW_OP_entry_value(DWARFExpression::Stack &stack,
                                               ExecutionContext *exe_ctx,
                                               RegisterContext *reg_ctx,
                                               const DataExtractor &opcodes,
@@ -653,7 +654,7 @@ static llvm::Error Evaluate_DW_OP_entry_value(std::vector<Value> &stack,
   addr_t return_pc = LLDB_INVALID_ADDRESS;
   uint32_t current_frame_idx = current_frame->GetFrameIndex();
 
-  for (uint32_t parent_frame_idx = current_frame_idx + 1;;parent_frame_idx++) {
+  for (uint32_t parent_frame_idx = current_frame_idx + 1;; parent_frame_idx++) {
     parent_frame = thread->GetStackFrameAtIndex(parent_frame_idx);
     // If this is null, we're at the end of the stack.
     if (!parent_frame)
@@ -859,6 +860,66 @@ ResolveLoadAddress(ExecutionContext *exe_ctx, lldb::ModuleSP &module_sp,
   return load_addr;
 }
 
+static llvm::Error Evaluate_DW_OP_deref(DWARFExpression::Stack &stack,
+                                        ExecutionContext *exe_ctx,
+                                        lldb::ModuleSP module_sp,
+                                        Process *process) {
+  if (stack.empty())
+    return llvm::createStringError("expression stack empty for DW_OP_deref");
+
+  const Value::ValueType value_type = stack.back().GetValueType();
+  switch (value_type) {
+  case Value::ValueType::HostAddress: {
+    void *src = (void *)stack.back().GetScalar().ULongLong();
+    intptr_t ptr;
+    ::memcpy(&ptr, src, sizeof(void *));
+    stack.back().GetScalar() = ptr;
+    stack.back().ClearContext();
+  } break;
+  case Value::ValueType::FileAddress: {
+    auto file_addr = stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
+    Address so_addr;
+    auto maybe_load_addr = ResolveLoadAddress(exe_ctx, module_sp, "DW_OP_deref",
+                                              file_addr, so_addr);
+    if (!maybe_load_addr)
+      return maybe_load_addr.takeError();
+    stack.back().GetScalar() = *maybe_load_addr;
+    // Fall through to load address promotion code below.
+  }
+    [[fallthrough]];
+  case Value::ValueType::Scalar:
+    // Promote Scalar to LoadAddress and fall through.
+    stack.back().SetValueType(Value::ValueType::LoadAddress);
+    [[fallthrough]];
+  case Value::ValueType::LoadAddress: {
+    if (!exe_ctx)
+      return llvm::createStringError("NULL execution context for DW_OP_deref");
+    if (!process)
+      return llvm::createStringError("NULL process for DW_OP_deref");
+    lldb::addr_t pointer_addr =
+        stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
+    Status error;
+    lldb::addr_t pointer_value =
+        process->ReadPointerFromMemory(pointer_addr, error);
+    if (pointer_value == LLDB_INVALID_ADDRESS)
+      return llvm::joinErrors(
+          llvm::createStringError(
+              "Failed to dereference pointer from 0x%" PRIx64
+              " for DW_OP_deref",
+              pointer_addr),
+          error.takeError());
+    if (ABISP abi_sp = process->GetABI())
+      pointer_value = abi_sp->FixCodeAddress(pointer_value);
+    stack.back().GetScalar() = pointer_value;
+    stack.back().ClearContext();
+  } break;
+  case Value::ValueType::Invalid:
+    return llvm::createStringError("invalid value type for DW_OP_deref");
+  }
+
+  return llvm::Error::success();
+}
+
 /// Helper function to move common code used to load sized data from a uint8_t
 /// buffer.
 ///
@@ -889,7 +950,8 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
   if (opcodes.GetByteSize() == 0)
     return llvm::createStringError(
         "no location, value may have been optimized out");
-  std::vector<Value> stack;
+
+  Stack stack;
 
   Process *process = nullptr;
   StackFrame *frame = nullptr;
@@ -1018,69 +1080,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // retrieved from the dereferenced address is the size of an address on the
     // target machine.
     case DW_OP_deref: {
-      if (stack.empty())
-        return llvm::createStringError(
-            "expression stack empty for DW_OP_deref");
-      Value::ValueType value_type = stack.back().GetValueType();
-      switch (value_type) {
-      case Value::ValueType::HostAddress: {
-        void *src = (void *)stack.back().GetScalar().ULongLong();
-        intptr_t ptr;
-        ::memcpy(&ptr, src, sizeof(void *));
-        stack.back().GetScalar() = ptr;
-        stack.back().ClearContext();
-      } break;
-      case Value::ValueType::FileAddress: {
-        auto file_addr = stack.back().GetScalar().ULongLong(
-            LLDB_INVALID_ADDRESS);
-
-        Address so_addr;
-        auto maybe_load_addr = ResolveLoadAddress(
-            exe_ctx, module_sp, "DW_OP_deref", file_addr, so_addr);
-
-        if (!maybe_load_addr)
-          return maybe_load_addr.takeError();
-
-        stack.back().GetScalar() = *maybe_load_addr;
-        // Fall through to load address promotion code below.
-      }
-        [[fallthrough]];
-      case Value::ValueType::Scalar:
-        // Promote Scalar to LoadAddress and fall through.
-        stack.back().SetValueType(Value::ValueType::LoadAddress);
-        [[fallthrough]];
-      case Value::ValueType::LoadAddress:
-        if (exe_ctx) {
-          if (process) {
-            lldb::addr_t pointer_addr =
-                stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
-            Status error;
-            lldb::addr_t pointer_value =
-                process->ReadPointerFromMemory(pointer_addr, error);
-            if (pointer_value != LLDB_INVALID_ADDRESS) {
-              if (ABISP abi_sp = process->GetABI())
-                pointer_value = abi_sp->FixCodeAddress(pointer_value);
-              stack.back().GetScalar() = pointer_value;
-              stack.back().ClearContext();
-            } else {
-              return llvm::createStringError(
-                  "Failed to dereference pointer from 0x%" PRIx64
-                  " for DW_OP_deref: %s\n",
-                  pointer_addr, error.AsCString());
-            }
-          } else {
-            return llvm::createStringError("NULL process for DW_OP_deref");
-          }
-        } else {
-          return llvm::createStringError(
-              "NULL execution context for DW_OP_deref");
-        }
-        break;
-
-      case Value::ValueType::Invalid:
-        return llvm::createStringError("invalid value type for DW_OP_deref");
-      }
-
+      if (llvm::Error err =
+              Evaluate_DW_OP_deref(stack, exe_ctx, module_sp, process))
+        return err;
     } break;
 
     // OPCODE: DW_OP_deref_size
@@ -1965,8 +1967,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
           case Value::ValueType::Scalar: {
             uint32_t bit_size = piece_byte_size * 8;
             uint32_t bit_offset = 0;
-            if (!scalar.ExtractBitfield(
-                    bit_size, bit_offset)) {
+            if (!scalar.ExtractBitfield(bit_size, bit_offset)) {
               return llvm::createStringError(
                   "unable to extract %" PRIu64 " bytes from a %" PRIu64
                   " byte scalar value.",
@@ -2408,8 +2409,7 @@ bool DWARFExpression::MatchesOperand(
     return MatchUnaryOp(
         MatchOpType(Instruction::Operand::Type::Dereference),
         MatchBinaryOp(MatchOpType(Instruction::Operand::Type::Sum),
-                      MatchRegOp(*reg),
-                      MatchImmOp(offset)))(operand);
+                      MatchRegOp(*reg), MatchImmOp(offset)))(operand);
   } else {
     return MatchRegOp(*reg)(operand);
   }
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index 06b0cb7769f64..e445fa8833022 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -57,12 +57,14 @@ IRExecutionUnit::IRExecutionUnit(std::unique_ptr<llvm::LLVMContext> &context_up,
 lldb::addr_t IRExecutionUnit::WriteNow(const uint8_t *bytes, size_t size,
                                        Status &error) {
   const bool zero_memory = false;
-  lldb::addr_t allocation_process_addr =
+  auto address_or_error =
       Malloc(size, 8, lldb::ePermissionsWritable | lldb::ePermissionsReadable,
-             eAllocationPolicyMirror, zero_memory, error);
-
-  if (!error.Success())
+             eAllocationPolicyMirror, zero_memory);
+  if (!address_or_error) {
+    error = Status::FromError(address_or_error.takeError());
     return LLDB_INVALID_ADDRESS;
+  }
+  lldb::addr_t allocation_process_addr = *address_or_error;
 
   WriteMemory(allocation_process_addr, bytes, size, error);
 
@@ -1102,9 +1104,12 @@ bool IRExecutionUnit::CommitOneAllocation(lldb::ProcessSP &process_sp,
     break;
   default:
     const bool zero_memory = false;
-    record.m_process_address =
-        Malloc(record.m_size, record.m_alignment, record.m_permissions,
-               eAllocationPolicyProcessOnly, zero_memory, error);
+    if (auto address_or_error =
+            Malloc(record.m_size, record.m_alignment, record.m_permissions,
+                   eAllocationPolicyProcessOnly, zero_memory))
+      record.m_process_address = *address_or_error;
+    else
+      error = Status::FromError(address_or_error.takeError());
     break;
   }
 
diff --git a/lldb/source/Expression/IRMemoryMap.cpp b/lldb/source/Expression/IRMemoryMap.cpp
index 65b5d11413c85..150699352a2e3 100644
--- a/lldb/source/Expression/IRMemoryMap.cpp
+++ b/lldb/source/Expression/IRMemoryMap.cpp
@@ -319,11 +319,11 @@ IRMemoryMap::Allocation::Allocation(lldb::addr_t process_alloc,
   }
 }
 
-lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
-                                 uint32_t permissions, AllocationPolicy policy,
-                                 bool zero_memory, Status &error) {
+llvm::Expected<lldb::addr_t>
+IRMemoryMap::Malloc(size_t size, uint8_t alignment, uint32_t permissions,
+                    AllocationPolicy policy, bool zero_memory,
+                    AllocationPolicy *used_policy) {
   lldb_private::Log *log(GetLog(LLDBLog::Expressions));
-  error.Clear();
 
   lldb::ProcessSP process_sp;
   lldb::addr_t allocation_address = LLDB_INVALID_ADDRESS;
@@ -347,15 +347,14 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
 
   switch (policy) {
   default:
-    error =
-        Status::FromErrorString("Couldn't malloc: invalid allocation policy");
-    return LLDB_INVALID_ADDRESS;
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        "Couldn't malloc: invalid allocation policy");
   case eAllocationPolicyHostOnly:
     allocation_address = FindSpace(allocation_size);
-    if (allocation_address == LLDB_INVALID_ADDRESS) {
-      error = Status::FromErrorString("Couldn't malloc: address space is full");
-      return LLDB_INVALID_ADDRESS;
-    }
+    if (allocation_address == LLDB_INVALID_ADDRESS)
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Couldn't malloc: address space is full");
     break;
   case eAllocationPolicyMirror:
     process_sp = m_process_wp.lock();
@@ -366,6 +365,7 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
               process_sp && process_sp->CanJIT() ? "true" : "false",
               process_sp && process_sp->IsAlive() ? "true" : "false");
     if (process_sp && process_sp->CanJIT() && process_sp->IsAlive()) {
+      Status error;
       if (!zero_memory)
         allocation_address =
             process_sp->AllocateMemory(allocation_size, permissions, error);
@@ -374,7 +374,7 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
             process_sp->CallocateMemory(allocation_size, permissions, error);
 
       if (!error.Success())
-        return LLDB_INVALID_ADDRESS;
+        return error.takeError();
     } else {
       LLDB_LOGF(log,
                 "IRMemoryMap::%s switching to eAllocationPolicyHostOnly "
@@ -382,17 +382,17 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
                 __FUNCTION__);
       policy = eAllocationPolicyHostOnly;
       allocation_address = FindSpace(allocation_size);
-      if (allocation_address == LLDB_INVALID_ADDRESS) {
-        error =
-            Status::FromErrorString("Couldn't malloc: address space is full");
-        return LLDB_INVALID_ADDRESS;
-      }
+      if (allocation_address == LLDB_INVALID_ADDRESS)
+        return llvm::createStringError(
+            llvm::inconvertibleErrorCode(),
+            "Couldn't malloc: address space is full");
     }
     break;
   case eAllocationPolicyProcessOnly:
     process_sp = m_process_wp.lock();
     if (process_sp) {
       if (process_sp->CanJIT() && process_sp->IsAlive()) {
+        Status error;
         if (!zero_memory)
           allocation_address =
               process_sp->AllocateMemory(allocation_size, permissions, error);
@@ -401,17 +401,16 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
               process_sp->CallocateMemory(allocation_size, permissions, error);
 
         if (!error.Success())
-          return LLDB_INVALID_ADDRESS;
+          return error.takeError();
       } else {
-        error = Status::FromErrorString(
+        return llvm::createStringError(
+            llvm::inconvertibleErrorCode(),
             "Couldn't malloc: process doesn't support allocating memory");
-        return LLDB_INVALID_ADDRESS;
       }
     } else {
-      error = Status::FromErrorString(
-          "Couldn't malloc: process doesn't exist, and this "
-          "memory must be in the process");
-      return LLDB_INVALID_ADDRESS;
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Couldn't malloc: process doesn't exist, "
+                                     "and this memory must be in the process");
     }
     break;
   }
@@ -455,6 +454,9 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
               (uint64_t)permissions, policy_string, aligned_address);
   }
 
+  if (used_policy)
+    *used_policy = policy;
+
   return aligned_address;
 }
 
diff --git a/lldb/source/Expression/LLVMUserExpression.cpp b/lldb/source/Expression/LLVMUserExpression.cpp
index fac3ce6f5799d..2d59194027b57 100644
--- a/lldb/source/Expression/LLVMUserExpression.cpp
+++ b/lldb/source/Expression/LLVMUserExpression.cpp
@@ -313,25 +313,22 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
 
   if (m_jit_start_addr != LLDB_INVALID_ADDRESS || m_can_interpret) {
     if (m_materialized_address == LLDB_INVALID_ADDRESS) {
-      Status alloc_error;
-
       IRMemoryMap::AllocationPolicy policy =
           m_can_interpret ? IRMemoryMap::eAllocationPolicyHostOnly
                           : IRMemoryMap::eAllocationPolicyMirror;
 
       const bool zero_memory = false;
-
-      m_materialized_address = m_execution_unit_sp->Malloc(
-          m_materializer_up->GetStructByteSize(),
-          m_materializer_up->GetStructAlignment(),
-          lldb::ePermissionsReadable | lldb::ePermissionsWritable, policy,
-          zero_memory, alloc_error);
-
-      if (!alloc_error.Success()) {
+      if (auto address_or_error = m_execution_unit_sp->Malloc(
+              m_materializer_up->GetStructByteSize(),
+              m_materializer_up->GetStructAlignment(),
+              lldb::ePermissionsReadable | lldb::ePermissionsWritable, policy,
+              zero_memory)) {
+        m_materialized_address = *address_or_error;
+      } else {
         diagnostic_manager.Printf(
             lldb::eSeverityError,
             "Couldn't allocate space for materialized struct: %s",
-            alloc_error.AsCString());
+            toString(address_or_error.takeError()).c_str());
         return false;
       }
     }
@@ -339,8 +336,6 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
     struct_address = m_materialized_address;
 
     if (m_can_interpret && m_stack_frame_bottom == LLDB_INVALID_ADDRESS) {
-      Status alloc_error;
-
       size_t stack_frame_size = target->GetExprAllocSize();
       if (stack_frame_size == 0) {
         ABISP abi_sp;
@@ -351,19 +346,17 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
       }
 
       const bool zero_memory = false;
-
-      m_stack_frame_bottom = m_execution_unit_sp->Malloc(
-          stack_frame_size, 8,
-          lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-          IRMemoryMap::eAllocationPolicyHostOnly, zero_memory, alloc_error);
-
-      m_stack_frame_top = m_stack_frame_bottom + stack_frame_size;
-
-      if (!alloc_error.Success()) {
+      if (auto address_or_error = m_execution_unit_sp->Malloc(
+              stack_frame_size, 8,
+              lldb::ePermissionsReadable | lldb::ePermissionsWritable,
+              IRMemoryMap::eAllocationPolicyHostOnly, zero_memory)) {
+        m_stack_frame_bottom = *address_or_error;
+        m_stack_frame_top = m_stack_frame_bottom + stack_frame_size;
+      } else {
         diagnostic_manager.Printf(
             lldb::eSeverityError,
             "Couldn't allocate space for the stack frame: %s",
-            alloc_error.AsCString());
+            toString(address_or_error.takeError()).c_str());
         return false;
       }
     }
@@ -382,4 +375,3 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
   }
   return true;
 }
-
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 79c804c6c4214..17ea1596806d0 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -74,22 +74,21 @@ class EntityPersistentVariable : public Materializer::Entity {
     // Allocate a spare memory area to store the persistent variable's
     // contents.
 
-    Status allocate_error;
     const bool zero_memory = false;
-
-    lldb::addr_t mem = map.Malloc(
+    IRMemoryMap::AllocationPolicy used_policy;
+    auto address_or_error = map.Malloc(
         llvm::expectedToOptional(m_persistent_variable_sp->GetByteSize())
             .value_or(0),
         8, lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-        IRMemoryMap::eAllocationPolicyMirror, zero_memory, allocate_error);
-
-    if (!allocate_error.Success()) {
+        IRMemoryMap::eAllocationPolicyMirror, zero_memory, &used_policy);
+    if (!address_or_error) {
       err = Status::FromErrorStringWithFormat(
           "couldn't allocate a memory area to store %s: %s",
           m_persistent_variable_sp->GetName().GetCString(),
-          allocate_error.AsCString());
+          toString(address_or_error.takeError()).c_str());
       return;
     }
+    lldb::addr_t mem = *address_or_error;
 
     LLDB_LOGF(log, "Allocated %s (0x%" PRIx64 ") successfully",
               m_persistent_variable_sp->GetName().GetCString(), mem);
@@ -103,14 +102,22 @@ class EntityPersistentVariable : public Materializer::Entity {
         m_persistent_variable_sp->GetName(), mem, eAddressTypeLoad,
         map.GetAddressByteSize());
 
-    // Clear the flag if the variable will never be deallocated.
-
     if (m_persistent_variable_sp->m_flags &
         ExpressionVariable::EVKeepInTarget) {
-      Status leak_error;
-      map.Leak(mem, leak_error);
-      m_persistent_variable_sp->m_flags &=
-          ~ExpressionVariable::EVNeedsAllocation;
+      if (used_policy == IRMemoryMap::eAllocationPolicyMirror) {
+        // Clear the flag if the variable will never be deallocated.
+        Status leak_error;
+        map.Leak(mem, leak_error);
+        m_persistent_variable_sp->m_flags &=
+            ~ExpressionVariable::EVNeedsAllocation;
+      } else {
+        // If the variable cannot be kept in target, clear this flag...
+        m_persistent_variable_sp->m_flags &=
+            ~ExpressionVariable::EVKeepInTarget;
+        // ...and set the flag to copy the value during dematerialization.
+        m_persistent_variable_sp->m_flags |=
+            ExpressionVariable::EVNeedsFreezeDry;
+      }
     }
 
     // Write the contents of the variable to the area.
@@ -329,22 +336,10 @@ class EntityPersistentVariable : public Materializer::Entity {
       return;
     }
 
-    lldb::ProcessSP process_sp =
-        map.GetBestExecutionContextScope()->CalculateProcess();
-    if (!process_sp || !process_sp->CanJIT()) {
-      // Allocations are not persistent so persistent variables cannot stay
-      // materialized.
-
-      m_persistent_variable_sp->m_flags |=
-          ExpressionVariable::EVNeedsAllocation;
-
-      DestroyAllocation(map, err);
-      if (!err.Success())
-        return;
-    } else if (m_persistent_variable_sp->m_flags &
-                   ExpressionVariable::EVNeedsAllocation &&
-               !(m_persistent_variable_sp->m_flags &
-                 ExpressionVariable::EVKeepInTarget)) {
+    if (m_persistent_variable_sp->m_flags &
+            ExpressionVariable::EVNeedsAllocation &&
+        !(m_persistent_variable_sp->m_flags &
+          ExpressionVariable::EVKeepInTarget)) {
       DestroyAllocation(map, err);
       if (!err.Success())
         return;
@@ -565,26 +560,25 @@ class EntityVariableBase : public Materializer::Entity {
 
         size_t byte_align = (*opt_bit_align + 7) / 8;
 
-        Status alloc_error;
         const bool zero_memory = false;
-
-        m_temporary_allocation = map.Malloc(
-            data.GetByteSize(), byte_align,
-            lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-            IRMemoryMap::eAllocationPolicyMirror, zero_memory, alloc_error);
+        if (auto address_or_error = map.Malloc(
+                data.GetByteSize(), byte_align,
+                lldb::ePermissionsReadable | lldb::ePermissionsWritable,
+                IRMemoryMap::eAllocationPolicyMirror, zero_memory)) {
+          m_temporary_allocation = *address_or_error;
+        } else {
+          err = Status::FromErrorStringWithFormat(
+              "couldn't allocate a temporary region for %s: %s",
+              GetName().AsCString(),
+              toString(address_or_error.takeError()).c_str());
+          return;
+        }
 
         m_temporary_allocation_size = data.GetByteSize();
 
         m_original_data = std::make_shared<DataBufferHeap>(data.GetDataStart(),
                                                            data.GetByteSize());
 
-        if (!alloc_error.Success()) {
-          err = Status::FromErrorStringWithFormat(
-              "couldn't allocate a temporary region for %s: %s",
-              GetName().AsCString(), alloc_error.AsCString());
-          return;
-        }
-
         Status write_error;
 
         map.WriteMemory(m_temporary_allocation, data.GetDataStart(),
@@ -967,22 +961,21 @@ class EntityResultVariable : public Materializer::Entity {
 
       size_t byte_align = (*opt_bit_align + 7) / 8;
 
-      Status alloc_error;
       const bool zero_memory = true;
-
-      m_temporary_allocation = map.Malloc(
-          byte_size, byte_align,
-          lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-          IRMemoryMap::eAllocationPolicyMirror, zero_memory, alloc_error);
-      m_temporary_allocation_size = byte_size;
-
-      if (!alloc_error.Success()) {
+      if (auto address_or_error = map.Malloc(
+              byte_size, byte_align,
+              lldb::ePermissionsReadable | lldb::ePermissionsWritable,
+              IRMemoryMap::eAllocationPolicyMirror, zero_memory)) {
+        m_temporary_allocation = *address_or_error;
+      } else {
         err = Status::FromErrorStringWithFormat(
             "couldn't allocate a temporary region for the result: %s",
-            alloc_error.AsCString());
+            toString(address_or_error.takeError()).c_str());
         return;
       }
 
+      m_temporary_allocation_size = byte_size;
+
       Status pointer_write_error;
 
       map.WritePointerToMemory(load_addr, m_temporary_allocation,
@@ -1086,9 +1079,8 @@ class EntityResultVariable : public Materializer::Entity {
       m_delegate->DidDematerialize(ret);
     }
 
-    bool can_persist =
-        (m_is_program_reference && process_sp && process_sp->CanJIT() &&
-         !(address >= frame_bottom && address < frame_top));
+    bool can_persist = m_is_program_reference &&
+                       !(address >= frame_bottom && address < frame_top);
 
     if (can_persist && m_keep_in_memory) {
       ret->m_live_sp = ValueObjectConstResult::Create(exe_scope, m_type, name,
@@ -1118,7 +1110,9 @@ class EntityResultVariable : public Materializer::Entity {
         map.Free(m_temporary_allocation, free_error);
       }
     } else {
-      ret->m_flags |= ExpressionVariable::EVIsLLDBAllocated;
+      ret->m_flags |= m_is_program_reference
+                          ? ExpressionVariable::EVIsProgramReference
+                          : ExpressionVariable::EVIsLLDBAllocated;
     }
 
     m_temporary_allocation = LLDB_INVALID_ADDRESS;
diff --git a/lldb/source/Host/common/Editline.cpp b/lldb/source/Host/common/Editline.cpp
index 431f931d5f3fc..5ed30fbb231d8 100644
--- a/lldb/source/Host/common/Editline.cpp
+++ b/lldb/source/Host/common/Editline.cpp
@@ -1709,6 +1709,13 @@ void Editline::PrintAsync(lldb::LockableStreamFileSP stream_sp, const char *s,
   }
 }
 
+void Editline::Refresh() {
+  if (!m_editline || !m_output_stream_sp)
+    return;
+  LockedStreamFile locked_stream = m_output_stream_sp->Lock();
+  el_set(m_editline, EL_REFRESH);
+}
+
 bool Editline::CompleteCharacter(char ch, EditLineGetCharType &out) {
 #if !LLDB_EDITLINE_USE_WCHAR
   if (ch == (char)EOF)
diff --git a/lldb/source/Host/common/File.cpp b/lldb/source/Host/common/File.cpp
index 9aa95ffda44cb..23b6dc9fe850d 100644
--- a/lldb/source/Host/common/File.cpp
+++ b/lldb/source/Host/common/File.cpp
@@ -274,7 +274,11 @@ int NativeFile::GetDescriptor() const {
 }
 
 IOObject::WaitableHandle NativeFile::GetWaitableHandle() {
+#ifdef _WIN32
+  return (HANDLE)_get_osfhandle(GetDescriptor());
+#else
   return GetDescriptor();
+#endif
 }
 
 FILE *NativeFile::GetStream() {
diff --git a/lldb/source/Host/common/JSONTransport.cpp b/lldb/source/Host/common/JSONTransport.cpp
index 1a0851d5c4365..546c12c8f7114 100644
--- a/lldb/source/Host/common/JSONTransport.cpp
+++ b/lldb/source/Host/common/JSONTransport.cpp
@@ -42,7 +42,8 @@ ReadFull(IOObject &descriptor, size_t length,
   if (timeout && timeout_supported) {
     SelectHelper sh;
     sh.SetTimeout(*timeout);
-    sh.FDSetRead(descriptor.GetWaitableHandle());
+    sh.FDSetRead(
+        reinterpret_cast<lldb::socket_t>(descriptor.GetWaitableHandle()));
     Status status = sh.Select();
     if (status.Fail()) {
       // Convert timeouts into a specific error.
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index 802ff9a1b5d1d..3511cde8bb36f 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -313,8 +313,7 @@ Socket::DecodeHostAndPort(llvm::StringRef host_and_port) {
 }
 
 IOObject::WaitableHandle Socket::GetWaitableHandle() {
-  // TODO: On Windows, use WSAEventSelect
-  return m_socket;
+  return (IOObject::WaitableHandle)m_socket;
 }
 
 Status Socket::Read(void *buf, size_t &num_bytes) {
diff --git a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
index 16e3f9910eefc..e8bf04e308447 100644
--- a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
+++ b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
@@ -272,13 +272,11 @@ size_t ConnectionFileDescriptor::Read(void *dst, size_t dst_len,
   error = m_io_sp->Read(dst, bytes_read);
 
   if (log) {
-    LLDB_LOGF(log,
-              "%p ConnectionFileDescriptor::Read()  fd = %" PRIu64
-              ", dst = %p, dst_len = %" PRIu64 ") => %" PRIu64 ", error = %s",
-              static_cast<void *>(this),
-              static_cast<uint64_t>(m_io_sp->GetWaitableHandle()),
-              static_cast<void *>(dst), static_cast<uint64_t>(dst_len),
-              static_cast<uint64_t>(bytes_read), error.AsCString());
+    LLDB_LOG(log,
+             "{0} ConnectionFileDescriptor::Read()  fd = {1}"
+             ", dst = {2}, dst_len = {3}) => {4}, error = {5}",
+             this, m_io_sp->GetWaitableHandle(), dst, dst_len, bytes_read,
+             error.AsCString());
   }
 
   if (bytes_read == 0) {
@@ -376,13 +374,11 @@ size_t ConnectionFileDescriptor::Write(const void *src, size_t src_len,
   error = m_io_sp->Write(src, bytes_sent);
 
   if (log) {
-    LLDB_LOGF(log,
-              "%p ConnectionFileDescriptor::Write(fd = %" PRIu64
-              ", src = %p, src_len = %" PRIu64 ") => %" PRIu64 " (error = %s)",
-              static_cast<void *>(this),
-              static_cast<uint64_t>(m_io_sp->GetWaitableHandle()),
-              static_cast<const void *>(src), static_cast<uint64_t>(src_len),
-              static_cast<uint64_t>(bytes_sent), error.AsCString());
+    LLDB_LOG(log,
+             "{0} ConnectionFileDescriptor::Write(fd = {1}"
+             ", src = {2}, src_len = {3}) => {4} (error = {5})",
+             this, m_io_sp->GetWaitableHandle(), src, src_len, bytes_sent,
+             error.AsCString());
   }
 
   if (error_ptr)
@@ -451,7 +447,8 @@ ConnectionFileDescriptor::BytesAvailable(const Timeout<std::micro> &timeout,
     if (timeout)
       select_helper.SetTimeout(*timeout);
 
-    select_helper.FDSetRead(handle);
+    // FIXME: Migrate to MainLoop.
+    select_helper.FDSetRead(reinterpret_cast<socket_t>(handle));
 #if defined(_WIN32)
     // select() won't accept pipes on Windows.  The entire Windows codepath
     // needs to be converted over to using WaitForMultipleObjects and event
@@ -493,7 +490,7 @@ ConnectionFileDescriptor::BytesAvailable(const Timeout<std::micro> &timeout,
           break; // Lets keep reading to until we timeout
         }
       } else {
-        if (select_helper.FDIsSetRead(handle))
+        if (select_helper.FDIsSetRead((lldb::socket_t)handle))
           return eConnectionStatusSuccess;
 
         if (select_helper.FDIsSetRead(pipe_fd)) {
diff --git a/lldb/source/Host/posix/HostInfoPosix.cpp b/lldb/source/Host/posix/HostInfoPosix.cpp
index 879dccfd353be..704f77891c829 100644
--- a/lldb/source/Host/posix/HostInfoPosix.cpp
+++ b/lldb/source/Host/posix/HostInfoPosix.cpp
@@ -140,6 +140,12 @@ std::optional<std::string> PosixUserIDResolver::DoGetGroupName(id_t gid) {
   return std::nullopt;
 }
 
+/// The SDK is the directory where the system C headers, libraries, can be
+/// found. On POSIX platforms this is simply the root directory.
+llvm::Expected<llvm::StringRef> HostInfoPosix::GetSDKRoot(SDKOptions options) {
+  return "/";
+}
+
 static llvm::ManagedStatic<PosixUserIDResolver> g_user_id_resolver;
 
 UserIDResolver &HostInfoPosix::GetUserIDResolver() {
diff --git a/lldb/source/Host/windows/MainLoopWindows.cpp b/lldb/source/Host/windows/MainLoopWindows.cpp
index f3ab2a710cd01..a1de895c0ba98 100644
--- a/lldb/source/Host/windows/MainLoopWindows.cpp
+++ b/lldb/source/Host/windows/MainLoopWindows.cpp
@@ -8,14 +8,20 @@
 
 #include "lldb/Host/windows/MainLoopWindows.h"
 #include "lldb/Host/Config.h"
+#include "lldb/Host/Socket.h"
+#include "lldb/Host/windows/windows.h"
 #include "lldb/Utility/Status.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/Support/WindowsError.h"
 #include <algorithm>
 #include <cassert>
-#include <cerrno>
-#include <csignal>
 #include <ctime>
+#include <io.h>
+#include <synchapi.h>
+#include <thread>
 #include <vector>
+#include <winbase.h>
+#include <winerror.h>
 #include <winsock2.h>
 
 using namespace lldb;
@@ -31,6 +37,139 @@ static DWORD ToTimeout(std::optional<MainLoopWindows::TimePoint> point) {
   return ceil<milliseconds>(dur).count();
 }
 
+namespace {
+
+class PipeEvent : public MainLoopWindows::IOEvent {
+public:
+  explicit PipeEvent(HANDLE handle)
+      : IOEvent(CreateEventW(NULL, /*bManualReset=*/TRUE,
+                             /*bInitialState=*/FALSE, NULL)),
+        m_handle(handle), m_ready(CreateEventW(NULL, /*bManualReset=*/TRUE,
+                                               /*bInitialState=*/FALSE, NULL)) {
+    assert(m_event && m_ready);
+    m_monitor_thread = std::thread(&PipeEvent::Monitor, this);
+  }
+
+  ~PipeEvent() override {
+    if (m_monitor_thread.joinable()) {
+      m_stopped = true;
+      SetEvent(m_ready);
+      // Keep trying to cancel ReadFile() until the thread exits.
+      do {
+        CancelIoEx(m_handle, /*lpOverlapped=*/NULL);
+      } while (WaitForSingleObject(m_monitor_thread.native_handle(), 1) ==
+               WAIT_TIMEOUT);
+      m_monitor_thread.join();
+    }
+    CloseHandle(m_event);
+    CloseHandle(m_ready);
+  }
+
+  void WillPoll() override {
+    if (WaitForSingleObject(m_event, /*dwMilliseconds=*/0) != WAIT_TIMEOUT) {
+      // The thread has already signalled that the data is available. No need
+      // for further polling until we consume that event.
+      return;
+    }
+    if (WaitForSingleObject(m_ready, /*dwMilliseconds=*/0) != WAIT_TIMEOUT) {
+      // The thread is already waiting for data to become available.
+      return;
+    }
+    // Start waiting.
+    SetEvent(m_ready);
+  }
+
+  void Disarm() override { ResetEvent(m_event); }
+
+  /// Monitors the handle performing a zero byte read to determine when data is
+  /// avaiable.
+  void Monitor() {
+    // Wait until the MainLoop tells us to start.
+    WaitForSingleObject(m_ready, INFINITE);
+
+    do {
+      char buf[1];
+      DWORD bytes_read = 0;
+      OVERLAPPED ov;
+      ZeroMemory(&ov, sizeof(ov));
+      // Block on a 0-byte read; this will only resume when data is
+      // available in the pipe. The pipe must be PIPE_WAIT or this thread
+      // will spin.
+      BOOL success =
+          ReadFile(m_handle, buf, /*nNumberOfBytesToRead=*/0, &bytes_read, &ov);
+      DWORD bytes_available = 0;
+      DWORD err = GetLastError();
+      if (!success && err == ERROR_IO_PENDING) {
+        success = GetOverlappedResult(m_handle, &ov, &bytes_read,
+                                      /*bWait=*/TRUE);
+        err = GetLastError();
+      }
+      if (success) {
+        success =
+            PeekNamedPipe(m_handle, NULL, 0, NULL, &bytes_available, NULL);
+        err = GetLastError();
+      }
+      if (success) {
+        if (bytes_available == 0) {
+          // This can happen with a zero-byte write. Try again.
+          continue;
+        }
+      } else if (err == ERROR_NO_DATA) {
+        // The pipe is nonblocking. Try again.
+        Sleep(0);
+        continue;
+      } else if (err == ERROR_OPERATION_ABORTED) {
+        // Read may have been cancelled, try again.
+        continue;
+      }
+
+      // Notify that data is available on the pipe. It's important to set this
+      // before clearing m_ready to avoid a race with WillPoll.
+      SetEvent(m_event);
+      // Stop polling until we're told to resume.
+      ResetEvent(m_ready);
+
+      // Wait until the current read is consumed before doing the next read.
+      WaitForSingleObject(m_ready, INFINITE);
+    } while (!m_stopped);
+  }
+
+private:
+  HANDLE m_handle;
+  HANDLE m_ready;
+  std::thread m_monitor_thread;
+  std::atomic<bool> m_stopped = false;
+};
+
+class SocketEvent : public MainLoopWindows::IOEvent {
+public:
+  explicit SocketEvent(SOCKET socket)
+      : IOEvent(WSACreateEvent()), m_socket(socket) {
+    assert(m_event != WSA_INVALID_EVENT);
+  }
+
+  ~SocketEvent() override { WSACloseEvent(m_event); }
+
+  void WillPoll() override {
+    int result =
+        WSAEventSelect(m_socket, m_event, FD_READ | FD_ACCEPT | FD_CLOSE);
+    assert(result == 0);
+    UNUSED_IF_ASSERT_DISABLED(result);
+  }
+
+  void DidPoll() override {
+    int result = WSAEventSelect(m_socket, WSA_INVALID_EVENT, 0);
+    assert(result == 0);
+    UNUSED_IF_ASSERT_DISABLED(result);
+  }
+
+  void Disarm() override { WSAResetEvent(m_event); }
+
+  SOCKET m_socket;
+};
+
+} // namespace
+
 MainLoopWindows::MainLoopWindows() {
   m_interrupt_event = WSACreateEvent();
   assert(m_interrupt_event != WSA_INVALID_EVENT);
@@ -44,14 +183,11 @@ MainLoopWindows::~MainLoopWindows() {
 }
 
 llvm::Expected<size_t> MainLoopWindows::Poll() {
-  std::vector<WSAEVENT> events;
+  std::vector<HANDLE> events;
   events.reserve(m_read_fds.size() + 1);
-  for (auto &[fd, info] : m_read_fds) {
-    int result = WSAEventSelect(fd, info.event, FD_READ | FD_ACCEPT | FD_CLOSE);
-    assert(result == 0);
-    UNUSED_IF_ASSERT_DISABLED(result);
-
-    events.push_back(info.event);
+  for (auto &[_, fd_info] : m_read_fds) {
+    fd_info.event->WillPoll();
+    events.push_back(fd_info.event->GetHandle());
   }
   events.push_back(m_interrupt_event);
 
@@ -59,11 +195,8 @@ llvm::Expected<size_t> MainLoopWindows::Poll() {
       WSAWaitForMultipleEvents(events.size(), events.data(), FALSE,
                                ToTimeout(GetNextWakeupTime()), FALSE);
 
-  for (auto &fd : m_read_fds) {
-    int result = WSAEventSelect(fd.first, WSA_INVALID_EVENT, 0);
-    assert(result == 0);
-    UNUSED_IF_ASSERT_DISABLED(result);
-  }
+  for (auto &[_, fd_info] : m_read_fds)
+    fd_info.event->DidPoll();
 
   if (result >= WSA_WAIT_EVENT_0 && result < WSA_WAIT_EVENT_0 + events.size())
     return result - WSA_WAIT_EVENT_0;
@@ -83,49 +216,42 @@ MainLoopWindows::RegisterReadObject(const IOObjectSP &object_sp,
     error = Status::FromErrorString("IO object is not valid.");
     return nullptr;
   }
-  if (object_sp->GetFdType() != IOObject::eFDTypeSocket) {
-    error = Status::FromErrorString(
-        "MainLoopWindows: non-socket types unsupported on Windows");
-    return nullptr;
-  }
 
-  WSAEVENT event = WSACreateEvent();
-  if (event == WSA_INVALID_EVENT) {
-    error =
-        Status::FromErrorStringWithFormat("Cannot create monitoring event.");
-    return nullptr;
-  }
+  IOObject::WaitableHandle waitable_handle = object_sp->GetWaitableHandle();
+  assert(waitable_handle != IOObject::kInvalidHandleValue);
 
-  const bool inserted =
-      m_read_fds
-          .try_emplace(object_sp->GetWaitableHandle(), FdInfo{event, callback})
-          .second;
-  if (!inserted) {
-    WSACloseEvent(event);
+  if (m_read_fds.find(waitable_handle) != m_read_fds.end()) {
     error = Status::FromErrorStringWithFormat(
-        "File descriptor %d already monitored.",
-        object_sp->GetWaitableHandle());
+        "File descriptor %d already monitored.", waitable_handle);
     return nullptr;
   }
 
+  if (object_sp->GetFdType() == IOObject::eFDTypeSocket) {
+    m_read_fds[waitable_handle] = {
+        std::make_unique<SocketEvent>(
+            reinterpret_cast<SOCKET>(waitable_handle)),
+        callback};
+  } else {
+    DWORD file_type = GetFileType(waitable_handle);
+    if (file_type != FILE_TYPE_PIPE) {
+      error = Status::FromErrorStringWithFormat("Unsupported file type %d",
+                                                file_type);
+      return nullptr;
+    }
+
+    m_read_fds[waitable_handle] = {std::make_unique<PipeEvent>(waitable_handle),
+                                   callback};
+  }
+
   return CreateReadHandle(object_sp);
 }
 
 void MainLoopWindows::UnregisterReadObject(IOObject::WaitableHandle handle) {
   auto it = m_read_fds.find(handle);
   assert(it != m_read_fds.end());
-  BOOL result = WSACloseEvent(it->second.event);
-  assert(result == TRUE);
-  UNUSED_IF_ASSERT_DISABLED(result);
   m_read_fds.erase(it);
 }
 
-void MainLoopWindows::ProcessReadObject(IOObject::WaitableHandle handle) {
-  auto it = m_read_fds.find(handle);
-  if (it != m_read_fds.end())
-    it->second.callback(*this); // Do the work
-}
-
 Status MainLoopWindows::Run() {
   m_terminate_request = false;
 
@@ -138,8 +264,8 @@ Status MainLoopWindows::Run() {
 
     if (*signaled_event < m_read_fds.size()) {
       auto &KV = *std::next(m_read_fds.begin(), *signaled_event);
-      WSAResetEvent(KV.second.event);
-      ProcessReadObject(KV.first);
+      KV.second.event->Disarm();
+      KV.second.callback(*this); // Do the work.
     } else {
       assert(*signaled_event == m_read_fds.size());
       WSAResetEvent(m_interrupt_event);
diff --git a/lldb/source/Interpreter/OptionArgParser.cpp b/lldb/source/Interpreter/OptionArgParser.cpp
index 2d393a57452ee..616f6e3dc8820 100644
--- a/lldb/source/Interpreter/OptionArgParser.cpp
+++ b/lldb/source/Interpreter/OptionArgParser.cpp
@@ -175,8 +175,7 @@ lldb::addr_t OptionArgParser::ToAddress(const ExecutionContext *exe_ctx,
   lldb::addr_t addr = *maybe_addr;
 
   if (Process *process = exe_ctx->GetProcessPtr())
-    if (ABISP abi_sp = process->GetABI())
-      addr = abi_sp->FixCodeAddress(addr);
+    addr = process->FixAnyAddress(addr);
 
   return addr;
 }
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
index 32975d88af46e..89da4d200699f 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
@@ -14471,3 +14471,16 @@ bool EmulateInstructionARM::CreateFunctionEntryUnwind(UnwindPlan &unwind_plan) {
   unwind_plan.SetReturnAddressRegister(dwarf_lr);
   return true;
 }
+
+llvm::Expected<unsigned>
+ARMSingleStepBreakpointLocationsPredictor::GetBreakpointSize(
+    lldb::addr_t bp_addr) {
+  auto flags = m_emulator_up->ReadRegisterUnsigned(
+      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS, LLDB_INVALID_ADDRESS,
+      nullptr);
+  if (flags == LLDB_INVALID_ADDRESS)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Reading flags failed!");
+
+  return (flags & 0x20) ? /* Thumb mode */ 2 : /* Arm mode */ 4;
+}
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
index 9eca0288607c1..7931e0e648fb0 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
@@ -16,6 +16,16 @@
 
 namespace lldb_private {
 
+class ARMSingleStepBreakpointLocationsPredictor
+    : public SingleStepBreakpointLocationsPredictor {
+public:
+  ARMSingleStepBreakpointLocationsPredictor(
+      std::unique_ptr<EmulateInstruction> emulator_up)
+      : SingleStepBreakpointLocationsPredictor{std::move(emulator_up)} {}
+
+  llvm::Expected<unsigned> GetBreakpointSize(lldb::addr_t bp_addr) override;
+};
+
 // ITSession - Keep track of the IT Block progression.
 class ITSession {
 public:
@@ -770,6 +780,14 @@ class EmulateInstructionARM : public EmulateInstruction {
   // B6.2.13 SUBS PC, LR and related instructions
   bool EmulateSUBSPcLrEtc(const uint32_t opcode, const ARMEncoding encoding);
 
+  BreakpointLocationsPredictorCreator
+  GetSingleStepBreakpointLocationsPredictorCreator() override {
+    return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+      return std::make_unique<ARMSingleStepBreakpointLocationsPredictor>(
+          std::move(emulator_up));
+    };
+  }
+
   uint32_t m_arm_isa;
   Mode m_opcode_mode;
   uint32_t m_opcode_cpsr;
diff --git a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp
index 37f08592d62b9..db32bf8a4dd51 100644
--- a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp
+++ b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp
@@ -86,7 +86,6 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
   uint32_t inst_size = m_opcode.GetByteSize();
   uint32_t inst = m_opcode.GetOpcode32();
   bool increase_pc = options & eEmulateInstructionOptionAutoAdvancePC;
-  bool success = false;
 
   Opcode *opcode_data = GetOpcodeForInstruction(inst);
   if (!opcode_data)
@@ -94,9 +93,10 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
 
   lldb::addr_t old_pc = 0;
   if (increase_pc) {
-    old_pc = ReadPC(&success);
-    if (!success)
+    auto addr = ReadPC();
+    if (!addr)
       return false;
+    old_pc = *addr;
   }
 
   // Call the Emulate... function.
@@ -104,9 +104,10 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
     return false;
 
   if (increase_pc) {
-    lldb::addr_t new_pc = ReadPC(&success);
-    if (!success)
+    auto addr = ReadPC();
+    if (!addr)
       return false;
+    lldb::addr_t new_pc = *addr;
 
     if (new_pc == old_pc && !WritePC(old_pc + inst_size))
       return false;
@@ -115,13 +116,14 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
 }
 
 bool EmulateInstructionLoongArch::ReadInstruction() {
-  bool success = false;
-  m_addr = ReadPC(&success);
-  if (!success) {
+  auto addr = ReadPC();
+  if (!addr) {
     m_addr = LLDB_INVALID_ADDRESS;
     return false;
   }
+  m_addr = *addr;
 
+  bool success = false;
   Context ctx;
   ctx.type = eContextReadOpcode;
   ctx.SetNoArgs();
@@ -131,19 +133,6 @@ bool EmulateInstructionLoongArch::ReadInstruction() {
   return true;
 }
 
-lldb::addr_t EmulateInstructionLoongArch::ReadPC(bool *success) {
-  return ReadRegisterUnsigned(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC,
-                              LLDB_INVALID_ADDRESS, success);
-}
-
-bool EmulateInstructionLoongArch::WritePC(lldb::addr_t pc) {
-  EmulateInstruction::Context ctx;
-  ctx.type = eContextAdvancePC;
-  ctx.SetNoArgs();
-  return WriteRegisterUnsigned(ctx, eRegisterKindGeneric,
-                               LLDB_REGNUM_GENERIC_PC, pc);
-}
-
 std::optional<RegisterInfo>
 EmulateInstructionLoongArch::GetRegisterInfo(lldb::RegisterKind reg_kind,
                                              uint32_t reg_index) {
@@ -273,9 +262,12 @@ bool EmulateInstructionLoongArch::EmulateNonJMP(uint32_t inst) { return false; }
 bool EmulateInstructionLoongArch::EmulateBEQZ64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -293,9 +285,12 @@ bool EmulateInstructionLoongArch::EmulateBEQZ64(uint32_t inst) {
 bool EmulateInstructionLoongArch::EmulateBNEZ64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -313,9 +308,12 @@ bool EmulateInstructionLoongArch::EmulateBNEZ64(uint32_t inst) {
 bool EmulateInstructionLoongArch::EmulateBCEQZ64(uint32_t inst) {
   bool success = false;
   uint32_t cj = Bits32(inst, 7, 5) + fpr_fcc0_loongarch;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint8_t cj_val =
       (uint8_t)ReadRegisterUnsigned(eRegisterKindLLDB, cj, 0, &success);
@@ -335,9 +333,12 @@ bool EmulateInstructionLoongArch::EmulateBCEQZ64(uint32_t inst) {
 bool EmulateInstructionLoongArch::EmulateBCNEZ64(uint32_t inst) {
   bool success = false;
   uint32_t cj = Bits32(inst, 7, 5) + fpr_fcc0_loongarch;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint8_t cj_val =
       (uint8_t)ReadRegisterUnsigned(eRegisterKindLLDB, cj, 0, &success);
@@ -358,9 +359,12 @@ bool EmulateInstructionLoongArch::EmulateJIRL64(uint32_t inst) {
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
   bool success = false;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   EmulateInstruction::Context ctx;
   if (!WriteRegisterUnsigned(ctx, eRegisterKindLLDB, rd, pc + 4))
     return false;
@@ -374,10 +378,11 @@ bool EmulateInstructionLoongArch::EmulateJIRL64(uint32_t inst) {
 // b offs26
 // PC = PC + SignExtend({offs26, 2' b0}, GRLEN)
 bool EmulateInstructionLoongArch::EmulateB64(uint32_t inst) {
-  bool success = false;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs26 = Bits32(inst, 25, 10) + (Bits32(inst, 9, 0) << 16);
   uint64_t next_pc = pc + llvm::SignExtend64<28>(offs26 << 2);
   return WritePC(next_pc);
@@ -387,10 +392,11 @@ bool EmulateInstructionLoongArch::EmulateB64(uint32_t inst) {
 // GR[1] = PC + 4
 // PC = PC + SignExtend({offs26, 2'b0}, GRLEN)
 bool EmulateInstructionLoongArch::EmulateBL64(uint32_t inst) {
-  bool success = false;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   EmulateInstruction::Context ctx;
   if (!WriteRegisterUnsigned(ctx, eRegisterKindLLDB, gpr_r1_loongarch, pc + 4))
     return false;
@@ -406,9 +412,12 @@ bool EmulateInstructionLoongArch::EmulateBEQ64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
@@ -429,9 +438,12 @@ bool EmulateInstructionLoongArch::EmulateBNE64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
@@ -452,9 +464,12 @@ bool EmulateInstructionLoongArch::EmulateBLT64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   int64_t rj_val =
       (int64_t)ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -477,9 +492,12 @@ bool EmulateInstructionLoongArch::EmulateBGE64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   int64_t rj_val =
       (int64_t)ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -502,9 +520,12 @@ bool EmulateInstructionLoongArch::EmulateBLTU64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
@@ -525,9 +546,12 @@ bool EmulateInstructionLoongArch::EmulateBGEU64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
diff --git a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h
index 47fe454fcd88c..2a8acba42f49e 100644
--- a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h
+++ b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h
@@ -57,8 +57,6 @@ class EmulateInstructionLoongArch : public EmulateInstruction {
 
   std::optional<RegisterInfo> GetRegisterInfo(lldb::RegisterKind reg_kind,
                                               uint32_t reg_num) override;
-  lldb::addr_t ReadPC(bool *success);
-  bool WritePC(lldb::addr_t pc);
   bool IsLoongArch64() { return m_arch_subtype == llvm::Triple::loongarch64; }
   bool TestExecute(uint32_t inst);
 
diff --git a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp
index badc7ba36f011..2adde02aca3a1 100644
--- a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp
+++ b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp
@@ -650,9 +650,10 @@ std::optional<DecodeResult> EmulateInstructionRISCV::Decode(uint32_t inst) {
   for (const InstrPattern &pat : PATTERNS) {
     if ((inst & pat.type_mask) == pat.eigen &&
         (inst_type & pat.inst_type) != 0) {
-      LLDB_LOGF(
-          log, "EmulateInstructionRISCV::%s: inst(%x at %" PRIx64 ") was decoded to %s",
-          __FUNCTION__, inst, m_addr, pat.name);
+      LLDB_LOGF(log,
+                "EmulateInstructionRISCV::%s: inst(%x at %" PRIx64
+                ") was decoded to %s",
+                __FUNCTION__, inst, m_addr, pat.name);
       auto decoded = is_16b ? pat.decode(try_rvc) : pat.decode(inst);
       return DecodeResult{decoded, inst, is_16b, pat};
     }
@@ -1649,21 +1650,6 @@ bool EmulateInstructionRISCV::ReadInstruction() {
   return true;
 }
 
-std::optional<addr_t> EmulateInstructionRISCV::ReadPC() {
-  bool success = false;
-  auto addr = ReadRegisterUnsigned(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC,
-                                   LLDB_INVALID_ADDRESS, &success);
-  return success ? std::optional<addr_t>(addr) : std::nullopt;
-}
-
-bool EmulateInstructionRISCV::WritePC(addr_t pc) {
-  EmulateInstruction::Context ctx;
-  ctx.type = eContextAdvancePC;
-  ctx.SetNoArgs();
-  return WriteRegisterUnsigned(ctx, eRegisterKindGeneric,
-                               LLDB_REGNUM_GENERIC_PC, pc);
-}
-
 RoundingMode EmulateInstructionRISCV::GetRoundingMode() {
   bool success = false;
   auto fcsr = ReadRegisterUnsigned(eRegisterKindLLDB, fpr_fcsr_riscv,
@@ -1792,4 +1778,125 @@ bool EmulateInstructionRISCV::SupportsThisArch(const ArchSpec &arch) {
   return arch.GetTriple().isRISCV();
 }
 
+BreakpointLocations
+RISCVSingleStepBreakpointLocationsPredictor::GetBreakpointLocations(
+    Status &status) {
+  EmulateInstructionRISCV *riscv_emulator =
+      static_cast<EmulateInstructionRISCV *>(m_emulator_up.get());
+
+  auto pc = riscv_emulator->ReadPC();
+  if (!pc) {
+    status = Status("Can't read PC");
+    return {};
+  }
+
+  auto inst = riscv_emulator->ReadInstructionAt(*pc);
+  if (!inst) {
+    // Can't read instruction. Try default handler.
+    return SingleStepBreakpointLocationsPredictor::GetBreakpointLocations(
+        status);
+  }
+
+  if (FoundLoadReserve(inst->decoded))
+    return HandleAtomicSequence(*pc, status);
+
+  if (FoundStoreConditional(inst->decoded)) {
+    // Ill-formed atomic sequence (SC doesn't have corresponding LR
+    // instruction). Consider SC instruction like a non-atomic store and set a
+    // breakpoint at the next instruction.
+    Log *log = GetLog(LLDBLog::Unwind);
+    LLDB_LOGF(log,
+              "RISCVSingleStepBreakpointLocationsPredictor::%s: can't find "
+              "corresponding load reserve insturuction",
+              __FUNCTION__);
+    return {*pc + (inst->is_rvc ? 2u : 4u)};
+  }
+
+  return SingleStepBreakpointLocationsPredictor::GetBreakpointLocations(status);
+}
+
+llvm::Expected<unsigned>
+RISCVSingleStepBreakpointLocationsPredictor::GetBreakpointSize(
+    lldb::addr_t bp_addr) {
+  EmulateInstructionRISCV *riscv_emulator =
+      static_cast<EmulateInstructionRISCV *>(m_emulator_up.get());
+
+  if (auto inst = riscv_emulator->ReadInstructionAt(bp_addr); inst)
+    return inst->is_rvc ? 2 : 4;
+
+  // Try last instruction size.
+  if (auto last_instr_size = riscv_emulator->GetLastInstrSize();
+      last_instr_size)
+    return *last_instr_size;
+
+  // Just place non-compressed software trap.
+  return 4;
+}
+
+BreakpointLocations
+RISCVSingleStepBreakpointLocationsPredictor::HandleAtomicSequence(
+    lldb::addr_t pc, Status &error) {
+  EmulateInstructionRISCV *riscv_emulator =
+      static_cast<EmulateInstructionRISCV *>(m_emulator_up.get());
+
+  // Handle instructions between LR and SC. According to unprivilleged
+  // RISC-V ISA there can be at most 16 instructions in the sequence.
+
+  lldb::addr_t entry_pc = pc; // LR instruction address
+  auto lr_inst = riscv_emulator->ReadInstructionAt(entry_pc);
+  pc += lr_inst->is_rvc ? 2 : 4;
+
+  size_t atomic_length = 0;
+  std::optional<DecodeResult> inst;
+  std::vector<lldb::addr_t> bp_addrs;
+  do {
+    inst = riscv_emulator->ReadInstructionAt(pc);
+    if (!inst) {
+      error = Status("Can't read instruction");
+      return {};
+    }
+
+    if (B *branch = std::get_if<B>(&inst->decoded))
+      bp_addrs.push_back(pc + SignExt(branch->imm));
+
+    unsigned addent = inst->is_rvc ? 2 : 4;
+    pc += addent;
+    atomic_length += addent;
+  } while ((atomic_length < s_max_atomic_sequence_length) &&
+           !FoundStoreConditional(inst->decoded));
+
+  if (atomic_length >= s_max_atomic_sequence_length) {
+    // Ill-formed atomic sequence (LR doesn't have corresponding SC
+    // instruction). In this case consider LR like a non-atomic load instruction
+    // and set a breakpoint at the next after LR instruction.
+    Log *log = GetLog(LLDBLog::Unwind);
+    LLDB_LOGF(log,
+              "RISCVSingleStepBreakpointLocationsPredictor::%s: can't find "
+              "corresponding store conditional insturuction",
+              __FUNCTION__);
+    return {entry_pc + (lr_inst->is_rvc ? 2u : 4u)};
+  }
+
+  lldb::addr_t exit_pc = pc;
+
+  // Check if we have a branch to the start of the atomic sequence after SC
+  // instruction. If we have such branch, consider it as a part of the atomic
+  // sequence.
+  inst = riscv_emulator->ReadInstructionAt(exit_pc);
+  if (inst) {
+    B *branch = std::get_if<B>(&inst->decoded);
+    if (branch && (exit_pc + SignExt(branch->imm)) == entry_pc)
+      exit_pc += inst->is_rvc ? 2 : 4;
+  }
+
+  // Set breakpoints at the jump addresses of the forward branches that points
+  // after the end of the atomic sequence.
+  llvm::erase_if(
+      bp_addrs, [exit_pc](lldb::addr_t bp_addr) { return exit_pc >= bp_addr; });
+
+  // Set breakpoint at the end of atomic sequence.
+  bp_addrs.push_back(exit_pc);
+  return bp_addrs;
+}
+
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h
index 53ac11c2e1102..3578a4ab03053 100644
--- a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h
+++ b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h
@@ -20,6 +20,33 @@
 
 namespace lldb_private {
 
+class RISCVSingleStepBreakpointLocationsPredictor
+    : public SingleStepBreakpointLocationsPredictor {
+public:
+  RISCVSingleStepBreakpointLocationsPredictor(
+      std::unique_ptr<EmulateInstruction> emulator)
+      : SingleStepBreakpointLocationsPredictor{std::move(emulator)} {}
+
+  BreakpointLocations GetBreakpointLocations(Status &status) override;
+
+  llvm::Expected<unsigned> GetBreakpointSize(lldb::addr_t bp_addr) override;
+
+private:
+  static bool FoundLoadReserve(const RISCVInst &inst) {
+    return std::holds_alternative<LR_W>(inst) ||
+           std::holds_alternative<LR_D>(inst);
+  }
+
+  static bool FoundStoreConditional(const RISCVInst &inst) {
+    return std::holds_alternative<SC_W>(inst) ||
+           std::holds_alternative<SC_D>(inst);
+  }
+
+  BreakpointLocations HandleAtomicSequence(lldb::addr_t pc, Status &error);
+
+  static constexpr size_t s_max_atomic_sequence_length = 64;
+};
+
 class EmulateInstructionRISCV : public EmulateInstruction {
 public:
   static llvm::StringRef GetPluginNameStatic() { return "riscv"; }
@@ -67,9 +94,6 @@ class EmulateInstructionRISCV : public EmulateInstruction {
   std::optional<RegisterInfo> GetRegisterInfo(lldb::RegisterKind reg_kind,
                                               uint32_t reg_num) override;
 
-  std::optional<lldb::addr_t> ReadPC();
-  bool WritePC(lldb::addr_t pc);
-
   std::optional<DecodeResult> ReadInstructionAt(lldb::addr_t addr);
   std::optional<DecodeResult> Decode(uint32_t inst);
   bool Execute(DecodeResult inst, bool ignore_cond);
@@ -98,6 +122,13 @@ class EmulateInstructionRISCV : public EmulateInstruction {
   bool SetAccruedExceptions(llvm::APFloatBase::opStatus);
 
 private:
+  BreakpointLocationsPredictorCreator
+  GetSingleStepBreakpointLocationsPredictorCreator() override {
+    return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+      return std::make_unique<RISCVSingleStepBreakpointLocationsPredictor>(
+          std::move(emulator_up));
+    };
+  }
   /// Last decoded instruction from m_opcode
   DecodeResult m_decoded;
   /// Last decoded instruction size estimate.
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt b/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
index 5ba2567c80cc3..3ec3cad4b8178 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
+++ b/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
@@ -12,6 +12,7 @@ add_lldb_library(lldbPluginCPlusPlusLanguage PLUGIN
   CPlusPlusLanguage.cpp
   CPlusPlusNameParser.cpp
   CxxStringTypes.cpp
+  Generic.cpp
   GenericBitset.cpp
   GenericOptional.cpp
   LibCxx.cpp
@@ -32,6 +33,7 @@ add_lldb_library(lldbPluginCPlusPlusLanguage PLUGIN
   LibStdcpp.cpp
   LibStdcppTuple.cpp
   LibStdcppUniquePointer.cpp
+  MsvcStl.cpp
   MSVCUndecoratedNameParser.cpp
 
   LINK_COMPONENTS
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 7ae2e141a63e0..17963c0273ba8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -46,6 +46,7 @@
 #include "LibCxxVariant.h"
 #include "LibStdcpp.h"
 #include "MSVCUndecoratedNameParser.h"
+#include "MsvcStl.h"
 #include "lldb/lldb-enumerations.h"
 
 using namespace lldb;
@@ -376,9 +377,9 @@ static bool PrintDemangledArgumentList(Stream &s, const SymbolContext &sc) {
 
   auto info_or_err = GetAndValidateInfo(sc);
   if (!info_or_err) {
-    LLDB_LOG_ERROR(
-        GetLog(LLDBLog::Language), info_or_err.takeError(),
-        "Failed to handle ${{function.basename}} frame-format variable: {0}");
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Language), info_or_err.takeError(),
+                   "Failed to handle ${{function.formatted-arguments}} "
+                   "frame-format variable: {0}");
     return false;
   }
   auto [demangled_name, info] = *info_or_err;
@@ -1016,15 +1017,15 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   stl_summary_flags.SetDontShowChildren(false);
   stl_summary_flags.SetSkipPointers(false);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::bitset summary provider",
                 "^std::__[[:alnum:]]+::bitset<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::vector summary provider",
                 "^std::__[[:alnum:]]+::vector<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::valarray summary provider",
                 "^std::__[[:alnum:]]+::valarray<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
@@ -1033,16 +1034,16 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "^std::__[[:alnum:]]+::slice_array<.+>$", stl_summary_flags,
                 true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ summary provider for the valarray proxy arrays",
                 "^std::__[[:alnum:]]+::(gslice|mask|indirect)_array<.+>$",
                 stl_summary_flags, true);
   AddCXXSummary(
-      cpp_category_sp, lldb_private::formatters::LibcxxContainerSummaryProvider,
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
       "libc++ std::list summary provider",
       "^std::__[[:alnum:]]+::forward_list<.+>$", stl_summary_flags, true);
   AddCXXSummary(
-      cpp_category_sp, lldb_private::formatters::LibcxxContainerSummaryProvider,
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
       "libc++ std::list summary provider",
       // A POSIX variant of: "^std::__(?!cxx11:)[[:alnum:]]+::list<.+>$"
       // so that it does not clash with: "^std::(__cxx11::)?list<.+>$"
@@ -1050,35 +1051,35 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       "cxx11[[:alnum:]])[[:alnum:]]*::list<.+>$",
       stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::map summary provider",
                 "^std::__[[:alnum:]]+::map<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::deque summary provider",
                 "^std::__[[:alnum:]]+::deque<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::queue summary provider",
                 "^std::__[[:alnum:]]+::queue<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::set summary provider",
                 "^std::__[[:alnum:]]+::set<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::multiset summary provider",
                 "^std::__[[:alnum:]]+::multiset<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::multimap summary provider",
                 "^std::__[[:alnum:]]+::multimap<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::unordered containers summary provider",
                 "^std::__[[:alnum:]]+::unordered_(multi)?(map|set)<.+> >$",
                 stl_summary_flags, true);
-  AddCXXSummary(cpp_category_sp, LibcxxContainerSummaryProvider,
+  AddCXXSummary(cpp_category_sp, ContainerSizeSummaryProvider,
                 "libc++ std::tuple summary provider",
                 "^std::__[[:alnum:]]+::tuple<.*>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
@@ -1094,7 +1095,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "libc++ std::variant summary provider",
                 "^std::__[[:alnum:]]+::variant<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::span summary provider",
                 "^std::__[[:alnum:]]+::span<.+>$", stl_summary_flags, true);
 
@@ -1331,6 +1332,37 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
           "${var.__y_} ${var.__m_} ${var.__wdl_}")));
 }
 
+static void RegisterStdStringSummaryProvider(
+    const lldb::TypeCategoryImplSP &category_sp, llvm::StringRef string_ty,
+    llvm::StringRef char_ty, lldb::TypeSummaryImplSP summary_sp) {
+  auto makeSpecifier = [](llvm::StringRef name) {
+    return std::make_shared<lldb_private::TypeNameSpecifierImpl>(
+        name, eFormatterMatchExact);
+  };
+
+  category_sp->AddTypeSummary(makeSpecifier(string_ty), summary_sp);
+
+  // std::basic_string<char>
+  category_sp->AddTypeSummary(
+      makeSpecifier(llvm::formatv("std::basic_string<{}>", char_ty).str()),
+      summary_sp);
+  // std::basic_string<char,std::char_traits<char>,std::allocator<char> >
+  category_sp->AddTypeSummary(
+      makeSpecifier(llvm::formatv("std::basic_string<{0},std::char_traits<{0}>,"
+                                  "std::allocator<{0}> >",
+                                  char_ty)
+                        .str()),
+      summary_sp);
+  // std::basic_string<char, std::char_traits<char>, std::allocator<char> >
+  category_sp->AddTypeSummary(
+      makeSpecifier(
+          llvm::formatv("std::basic_string<{0}, std::char_traits<{0}>, "
+                        "std::allocator<{0}> >",
+                        char_ty)
+              .str()),
+      summary_sp);
+}
+
 static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   if (!cpp_category_sp)
     return;
@@ -1344,62 +1376,26 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       .SetShowMembersOneLiner(false)
       .SetHideItemNames(false);
 
-  lldb::TypeSummaryImplSP std_string_summary_sp(
-      new StringSummaryFormat(stl_summary_flags, "${var._M_dataplus._M_p}"));
-
-  lldb::TypeSummaryImplSP cxx11_string_summary_sp(new CXXFunctionSummaryFormat(
+  lldb::TypeSummaryImplSP string_summary_sp(new CXXFunctionSummaryFormat(
       stl_summary_flags, LibStdcppStringSummaryProvider,
-      "libstdc++ c++11 std::string summary provider"));
-  lldb::TypeSummaryImplSP cxx11_wstring_summary_sp(new CXXFunctionSummaryFormat(
-      stl_summary_flags, LibStdcppWStringSummaryProvider,
-      "libstdc++ c++11 std::wstring summary provider"));
-
-  cpp_category_sp->AddTypeSummary("std::string", eFormatterMatchExact,
-                                  std_string_summary_sp);
-  cpp_category_sp->AddTypeSummary("std::basic_string<char>",
-                                  eFormatterMatchExact, std_string_summary_sp);
-  cpp_category_sp->AddTypeSummary(
-      "std::basic_string<char,std::char_traits<char>,std::allocator<char> >",
-      eFormatterMatchExact, std_string_summary_sp);
-  cpp_category_sp->AddTypeSummary(
-      "std::basic_string<char, std::char_traits<char>, std::allocator<char> >",
-      eFormatterMatchExact, std_string_summary_sp);
-
+      "libstdc++ std::(w)string summary provider"));
   cpp_category_sp->AddTypeSummary("std::__cxx11::string", eFormatterMatchExact,
-                                  cxx11_string_summary_sp);
+                                  string_summary_sp);
   cpp_category_sp->AddTypeSummary(
       "std::__cxx11::basic_string<char, std::char_traits<char>, "
       "std::allocator<char> >",
-      eFormatterMatchExact, cxx11_string_summary_sp);
+      eFormatterMatchExact, string_summary_sp);
   cpp_category_sp->AddTypeSummary("std::__cxx11::basic_string<unsigned char, "
                                   "std::char_traits<unsigned char>, "
                                   "std::allocator<unsigned char> >",
-                                  eFormatterMatchExact,
-                                  cxx11_string_summary_sp);
-
-  // making sure we force-pick the summary for printing wstring (_M_p is a
-  // wchar_t*)
-  lldb::TypeSummaryImplSP std_wstring_summary_sp(
-      new StringSummaryFormat(stl_summary_flags, "${var._M_dataplus._M_p%S}"));
-
-  cpp_category_sp->AddTypeSummary("std::wstring", eFormatterMatchExact,
-                                  std_wstring_summary_sp);
-  cpp_category_sp->AddTypeSummary("std::basic_string<wchar_t>",
-                                  eFormatterMatchExact, std_wstring_summary_sp);
-  cpp_category_sp->AddTypeSummary("std::basic_string<wchar_t,std::char_traits<"
-                                  "wchar_t>,std::allocator<wchar_t> >",
-                                  eFormatterMatchExact, std_wstring_summary_sp);
-  cpp_category_sp->AddTypeSummary(
-      "std::basic_string<wchar_t, std::char_traits<wchar_t>, "
-      "std::allocator<wchar_t> >",
-      eFormatterMatchExact, std_wstring_summary_sp);
+                                  eFormatterMatchExact, string_summary_sp);
 
   cpp_category_sp->AddTypeSummary("std::__cxx11::wstring", eFormatterMatchExact,
-                                  cxx11_wstring_summary_sp);
+                                  string_summary_sp);
   cpp_category_sp->AddTypeSummary(
       "std::__cxx11::basic_string<wchar_t, std::char_traits<wchar_t>, "
       "std::allocator<wchar_t> >",
-      eFormatterMatchExact, cxx11_wstring_summary_sp);
+      eFormatterMatchExact, string_summary_sp);
 
   SyntheticChildren::Flags stl_synth_flags;
   stl_synth_flags.SetCascades(true).SetSkipPointers(false).SetSkipReferences(
@@ -1463,44 +1459,58 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
 
   stl_summary_flags.SetDontShowChildren(false);
   stl_summary_flags.SetSkipPointers(false);
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?bitset<.+>(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?vector<.+>(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?map<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?set<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?deque<.+>(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?multimap<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?multiset<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary(
-      "^std::(__debug::)?unordered_(multi)?(map|set)<.+> >$",
-      eFormatterMatchRegex,
-      TypeSummaryImplSP(
-          new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary(
-      "^std::((__debug::)?|(__cxx11::)?)list<.+>(( )?&)?$",
-      eFormatterMatchRegex,
-      TypeSummaryImplSP(
-          new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::bitset summary provider",
+      "^std::(__debug::)?bitset<.+>(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::vector summary provider",
+      "^std::(__debug::)?vector<.+>(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::map summary provider",
+      "^std::(__debug::)?map<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::set summary provider",
+      "^std::(__debug::)?set<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::deque summary provider",
+      "^std::(__debug::)?deque<.+>(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::multimap summary provider",
+      "^std::(__debug::)?multimap<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::multiset summary provider",
+      "^std::(__debug::)?multiset<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(cpp_category_sp,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
+                "libstdc++ std unordered container summary provider",
+                "^std::(__debug::)?unordered_(multi)?(map|set)<.+> >$",
+                stl_summary_flags, true);
+
+  AddCXXSummary(cpp_category_sp,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
+                "libstdc++ std::list summary provider",
+                "^std::((__debug::)?|(__cxx11::)?)list<.+>(( )?&)?$",
+                stl_summary_flags, true);
+
+  AddCXXSummary(cpp_category_sp, ContainerSizeSummaryProvider,
+                "libstdc++ std::tuple summary provider",
+                "^std::tuple<.*>(( )?&)?$", stl_summary_flags, true);
+
   cpp_category_sp->AddTypeSummary(
       "^std::((__debug::)?|(__cxx11::)?)forward_list<.+>(( )?&)?$",
       eFormatterMatchRegex,
@@ -1522,8 +1532,8 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   AddCXXSynthetic(
       cpp_category_sp,
       lldb_private::formatters::LibstdcppMapIteratorSyntheticFrontEndCreator,
-      "std::map iterator synthetic children", "^std::_Rb_tree_iterator<.+>$",
-      stl_synth_flags, true);
+      "std::map iterator synthetic children",
+      "^std::_Rb_tree_(const_)?iterator<.+>$", stl_synth_flags, true);
 
   AddCXXSynthetic(
       cpp_category_sp,
@@ -1543,7 +1553,7 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   AddCXXSynthetic(
       cpp_category_sp,
       lldb_private::formatters::LibStdcppTupleSyntheticFrontEndCreator,
-      "std::tuple synthetic children", "^std::tuple<.+>(( )?&)?$",
+      "std::tuple synthetic children", "^std::tuple<.*>(( )?&)?$",
       stl_synth_flags, true);
 
   static constexpr const char *const libstdcpp_std_coroutine_handle_regex =
@@ -1588,6 +1598,81 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "^std::optional<.+>(( )?&)?$", stl_summary_flags, true);
 }
 
+/// Load formatters that are formatting types from more than one STL
+static void LoadCommonStlFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
+  if (!cpp_category_sp)
+    return;
+
+  TypeSummaryImpl::Flags stl_summary_flags;
+  stl_summary_flags.SetCascades(true)
+      .SetSkipPointers(false)
+      .SetSkipReferences(false)
+      .SetDontShowChildren(true)
+      .SetDontShowValue(false)
+      .SetShowMembersOneLiner(false)
+      .SetHideItemNames(false);
+  using StringElementType = StringPrinter::StringElementType;
+
+  RegisterStdStringSummaryProvider(
+      cpp_category_sp, "std::string", "char",
+      std::make_shared<CXXFunctionSummaryFormat>(
+          stl_summary_flags,
+          [](ValueObject &valobj, Stream &stream,
+             const TypeSummaryOptions &options) {
+            if (IsMsvcStlStringType(valobj))
+              return MsvcStlStringSummaryProvider<StringElementType::ASCII>(
+                  valobj, stream, options);
+            return LibStdcppStringSummaryProvider(valobj, stream, options);
+          },
+          "MSVC STL/libstdc++ std::string summary provider"));
+  RegisterStdStringSummaryProvider(
+      cpp_category_sp, "std::wstring", "wchar_t",
+      std::make_shared<CXXFunctionSummaryFormat>(
+          stl_summary_flags,
+          [](ValueObject &valobj, Stream &stream,
+             const TypeSummaryOptions &options) {
+            if (IsMsvcStlStringType(valobj))
+              return MsvcStlWStringSummaryProvider(valobj, stream, options);
+            return LibStdcppStringSummaryProvider(valobj, stream, options);
+          },
+          "MSVC STL/libstdc++ std::wstring summary provider"));
+}
+
+static void LoadMsvcStlFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
+  if (!cpp_category_sp)
+    return;
+
+  TypeSummaryImpl::Flags stl_summary_flags;
+  stl_summary_flags.SetCascades(true)
+      .SetSkipPointers(false)
+      .SetSkipReferences(false)
+      .SetDontShowChildren(true)
+      .SetDontShowValue(false)
+      .SetShowMembersOneLiner(false)
+      .SetHideItemNames(false);
+
+  using StringElementType = StringPrinter::StringElementType;
+
+  RegisterStdStringSummaryProvider(
+      cpp_category_sp, "std::u8string", "char8_t",
+      std::make_shared<CXXFunctionSummaryFormat>(
+          stl_summary_flags,
+          MsvcStlStringSummaryProvider<StringElementType::UTF8>,
+          "MSVC STL std::u8string summary provider"));
+  RegisterStdStringSummaryProvider(
+      cpp_category_sp, "std::u16string", "char16_t",
+      std::make_shared<CXXFunctionSummaryFormat>(
+          stl_summary_flags,
+          MsvcStlStringSummaryProvider<StringElementType::UTF16>,
+          "MSVC STL std::u16string summary provider"));
+  RegisterStdStringSummaryProvider(
+      cpp_category_sp, "std::u32string", "char32_t",
+      std::make_shared<CXXFunctionSummaryFormat>(
+          stl_summary_flags,
+          MsvcStlStringSummaryProvider<StringElementType::UTF32>,
+          "MSVC STL std::u32string summary provider"));
+}
+
 static void LoadSystemFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   if (!cpp_category_sp)
     return;
@@ -1702,6 +1787,8 @@ lldb::TypeCategoryImplSP CPlusPlusLanguage::GetFormatters() {
       // LLDB prioritizes the last loaded matching formatter.
       LoadLibCxxFormatters(g_category);
       LoadLibStdcppFormatters(g_category);
+      LoadMsvcStlFormatters(g_category);
+      LoadCommonStlFormatters(g_category);
       LoadSystemFormatters(g_category);
     }
   });
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Generic.cpp b/lldb/source/Plugins/Language/CPlusPlus/Generic.cpp
new file mode 100644
index 0000000000000..b237a8a27090c
--- /dev/null
+++ b/lldb/source/Plugins/Language/CPlusPlus/Generic.cpp
@@ -0,0 +1,22 @@
+//===-- Generic.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "Generic.h"
+
+lldb::ValueObjectSP lldb_private::formatters::GetDesugaredSmartPointerValue(
+    ValueObject &ptr, ValueObject &container) {
+  auto container_type = container.GetCompilerType().GetNonReferenceType();
+  if (!container_type)
+    return nullptr;
+
+  auto arg = container_type.GetTypeTemplateArgument(0);
+  if (!arg)
+    return nullptr;
+
+  return ptr.Cast(arg.GetPointerType());
+}
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Generic.h b/lldb/source/Plugins/Language/CPlusPlus/Generic.h
index 34efef9e82e5c..f3946225ed48d 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Generic.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Generic.h
@@ -19,6 +19,11 @@ namespace formatters {
 bool GenericOptionalSummaryProvider(ValueObject &valobj, Stream &stream,
                                     const TypeSummaryOptions &options);
 
+/// Return the ValueObjectSP of the underlying pointer member whose type
+/// is a desugared 'std::shared_ptr::element_type *'.
+lldb::ValueObjectSP GetDesugaredSmartPointerValue(ValueObject &ptr,
+                                                  ValueObject &container);
+
 } // namespace formatters
 } // namespace lldb_private
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 7143089209dd3..a7874047942c4 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -25,6 +25,7 @@
 #include "lldb/ValueObject/ValueObjectConstResult.h"
 
 #include "Plugins/Language/CPlusPlus/CxxStringTypes.h"
+#include "Plugins/Language/CPlusPlus/Generic.h"
 #include "Plugins/LanguageRuntime/CPlusPlus/CPPLanguageRuntime.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/lldb-enumerations.h"
@@ -157,39 +158,43 @@ bool lldb_private::formatters::LibcxxSmartPointerSummaryProvider(
   ValueObjectSP valobj_sp(valobj.GetNonSyntheticValue());
   if (!valobj_sp)
     return false;
-  ValueObjectSP ptr_sp(valobj_sp->GetChildMemberWithName("__ptr_"));
-  ValueObjectSP count_sp(
-      valobj_sp->GetChildAtNamePath({"__cntrl_", "__shared_owners_"}));
-  ValueObjectSP weakcount_sp(
-      valobj_sp->GetChildAtNamePath({"__cntrl_", "__shared_weak_owners_"}));
 
-  if (!ptr_sp)
+  ValueObjectSP ptr_sp(valobj_sp->GetChildMemberWithName("__ptr_"));
+  ValueObjectSP ctrl_sp(valobj_sp->GetChildMemberWithName("__cntrl_"));
+  if (!ctrl_sp || !ptr_sp)
     return false;
 
-  if (ptr_sp->GetValueAsUnsigned(0) == 0) {
-    stream.Printf("nullptr");
+  DumpCxxSmartPtrPointerSummary(stream, *ptr_sp, options);
+
+  bool success;
+  uint64_t ctrl_addr = ctrl_sp->GetValueAsUnsigned(0, &success);
+  // Empty control field. We're done.
+  if (!success || ctrl_addr == 0)
     return true;
-  } else {
-    bool print_pointee = false;
-    Status error;
-    ValueObjectSP pointee_sp = ptr_sp->Dereference(error);
-    if (pointee_sp && error.Success()) {
-      if (pointee_sp->DumpPrintableRepresentation(
-              stream, ValueObject::eValueObjectRepresentationStyleSummary,
-              lldb::eFormatInvalid,
-              ValueObject::PrintableRepresentationSpecialCases::eDisable,
-              false))
-        print_pointee = true;
-    }
-    if (!print_pointee)
-      stream.Printf("ptr = 0x%" PRIx64, ptr_sp->GetValueAsUnsigned(0));
+
+  if (auto count_sp = ctrl_sp->GetChildMemberWithName("__shared_owners_")) {
+    bool success;
+    uint64_t count = count_sp->GetValueAsUnsigned(0, &success);
+    if (!success)
+      return false;
+
+    // std::shared_ptr releases the underlying resource when the
+    // __shared_owners_ count hits -1. So `__shared_owners_ == 0` indicates 1
+    // owner. Hence add +1 here.
+    stream.Printf(" strong=%" PRIu64, count + 1);
   }
 
-  if (count_sp)
-    stream.Printf(" strong=%" PRIu64, 1 + count_sp->GetValueAsUnsigned(0));
+  if (auto weak_count_sp =
+          ctrl_sp->GetChildMemberWithName("__shared_weak_owners_")) {
+    bool success;
+    uint64_t count = weak_count_sp->GetValueAsUnsigned(0, &success);
+    if (!success)
+      return false;
 
-  if (weakcount_sp)
-    stream.Printf(" weak=%" PRIu64, 1 + weakcount_sp->GetValueAsUnsigned(0));
+    // Unlike __shared_owners_, __shared_weak_owners_ indicates the exact
+    // std::weak_ptr reference count.
+    stream.Printf(" weak=%" PRIu64, count);
+  }
 
   return true;
 }
@@ -210,24 +215,7 @@ bool lldb_private::formatters::LibcxxUniquePointerSummaryProvider(
   if (!ptr_sp)
     return false;
 
-  if (ptr_sp->GetValueAsUnsigned(0) == 0) {
-    stream.Printf("nullptr");
-    return true;
-  } else {
-    bool print_pointee = false;
-    Status error;
-    ValueObjectSP pointee_sp = ptr_sp->Dereference(error);
-    if (pointee_sp && error.Success()) {
-      if (pointee_sp->DumpPrintableRepresentation(
-              stream, ValueObject::eValueObjectRepresentationStyleSummary,
-              lldb::eFormatInvalid,
-              ValueObject::PrintableRepresentationSpecialCases::eDisable,
-              false))
-        print_pointee = true;
-    }
-    if (!print_pointee)
-      stream.Printf("ptr = 0x%" PRIx64, ptr_sp->GetValueAsUnsigned(0));
-  }
+  DumpCxxSmartPtrPointerSummary(stream, *ptr_sp, options);
 
   return true;
 }
@@ -251,7 +239,8 @@ lldb_private::formatters::LibCxxVectorIteratorSyntheticFrontEndCreator(
 
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp)
-    : SyntheticChildrenFrontEnd(*valobj_sp), m_cntrl(nullptr) {
+    : SyntheticChildrenFrontEnd(*valobj_sp), m_cntrl(nullptr),
+      m_ptr_obj(nullptr) {
   if (valobj_sp)
     Update();
 }
@@ -264,7 +253,7 @@ llvm::Expected<uint32_t> lldb_private::formatters::
 lldb::ValueObjectSP
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  if (!m_cntrl)
+  if (!m_cntrl || !m_ptr_obj)
     return lldb::ValueObjectSP();
 
   ValueObjectSP valobj_sp = m_backend.GetSP();
@@ -272,20 +261,13 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex(
     return lldb::ValueObjectSP();
 
   if (idx == 0)
-    return valobj_sp->GetChildMemberWithName("__ptr_");
+    return m_ptr_obj->GetSP();
 
   if (idx == 1) {
-    if (auto ptr_sp = valobj_sp->GetChildMemberWithName("__ptr_")) {
-      Status status;
-      auto value_type_sp =
-            valobj_sp->GetCompilerType()
-              .GetTypeTemplateArgument(0).GetPointerType();
-      ValueObjectSP cast_ptr_sp = ptr_sp->Cast(value_type_sp);
-      ValueObjectSP value_sp = cast_ptr_sp->Dereference(status);
-      if (status.Success()) {
-        return value_sp;
-      }
-    }
+    Status status;
+    ValueObjectSP value_sp = m_ptr_obj->Dereference(status);
+    if (status.Success())
+      return value_sp;
   }
 
   return lldb::ValueObjectSP();
@@ -294,6 +276,7 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex(
 lldb::ChildCacheState
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::Update() {
   m_cntrl = nullptr;
+  m_ptr_obj = nullptr;
 
   ValueObjectSP valobj_sp = m_backend.GetSP();
   if (!valobj_sp)
@@ -303,6 +286,16 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::Update() {
   if (!target_sp)
     return lldb::ChildCacheState::eRefetch;
 
+  auto ptr_obj_sp = valobj_sp->GetChildMemberWithName("__ptr_");
+  if (!ptr_obj_sp)
+    return lldb::ChildCacheState::eRefetch;
+
+  auto cast_ptr_sp = GetDesugaredSmartPointerValue(*ptr_obj_sp, *valobj_sp);
+  if (!cast_ptr_sp)
+    return lldb::ChildCacheState::eRefetch;
+
+  m_ptr_obj = cast_ptr_sp->Clone(ConstString("pointer")).get();
+
   lldb::ValueObjectSP cntrl_sp(valobj_sp->GetChildMemberWithName("__cntrl_"));
 
   m_cntrl = cntrl_sp.get(); // need to store the raw pointer to avoid a circular
@@ -313,10 +306,12 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::Update() {
 llvm::Expected<size_t>
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
-  if (name == "__ptr_")
+  if (name == "pointer")
     return 0;
-  if (name == "$$dereference$$")
+
+  if (name == "object" || name == "$$dereference$$")
     return 1;
+
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
 }
@@ -417,18 +412,12 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
     return 0;
   if (name == "deleter")
     return 1;
-  if (name == "$$dereference$$")
+  if (name == "obj" || name == "object" || name == "$$dereference$$")
     return 2;
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
 }
 
-bool lldb_private::formatters::LibcxxContainerSummaryProvider(
-    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  return FormatEntity::FormatStringRef("size=${svar%#}", stream, nullptr,
-                                       nullptr, nullptr, &valobj, false, false);
-}
-
 /// The field layout in a libc++ string (cap, side, data or data, size, cap).
 namespace {
 enum class StringLayout { CSD, DSC };
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index 56501302d116f..d88a6ecb1fa89 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -81,9 +81,6 @@ SyntheticChildrenFrontEnd *
 LibcxxVectorBoolSyntheticFrontEndCreator(CXXSyntheticChildren *,
                                          lldb::ValueObjectSP);
 
-bool LibcxxContainerSummaryProvider(ValueObject &valobj, Stream &stream,
-                                    const TypeSummaryOptions &options);
-
 /// Formatter for libc++ std::span<>.
 bool LibcxxSpanSummaryProvider(ValueObject &valobj, Stream &stream,
                                const TypeSummaryOptions &options);
@@ -108,6 +105,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 
 private:
   ValueObject *m_cntrl;
+  ValueObject *m_ptr_obj;
 };
 
 class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 77e0755607a06..c80a52d0f9ed6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -9,7 +9,9 @@
 #include "LibStdcpp.h"
 #include "LibCxx.h"
 
+#include "Plugins/Language/CPlusPlus/Generic.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
+#include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/DataFormatters/VectorIterator.h"
 #include "lldb/Target/Target.h"
@@ -77,8 +79,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
   // objects are only destroyed when every shared pointer to any of them
   // is destroyed, so we must not store a shared pointer to any ValueObject
   // derived from our backend ValueObject (since we're in the same cluster).
-  ValueObject* m_ptr_obj = nullptr; // Underlying pointer (held, not owned)
-  ValueObject* m_obj_obj = nullptr; // Underlying object (held, not owned)
+  ValueObject *m_ptr_obj = nullptr; // Underlying pointer (held, not owned)
 };
 
 } // end of anonymous namespace
@@ -239,122 +240,12 @@ VectorIteratorSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
 
 bool lldb_private::formatters::LibStdcppStringSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  const bool scalar_is_load_addr = true;
-  auto [addr_of_string, addr_type] =
-      valobj.IsPointerOrReferenceType()
-          ? valobj.GetPointerValue()
-          : valobj.GetAddressOf(scalar_is_load_addr);
-  if (addr_of_string != LLDB_INVALID_ADDRESS) {
-    switch (addr_type) {
-    case eAddressTypeLoad: {
-      ProcessSP process_sp(valobj.GetProcessSP());
-      if (!process_sp)
-        return false;
-
-      StringPrinter::ReadStringAndDumpToStreamOptions options(valobj);
-      Status error;
-      lldb::addr_t addr_of_data =
-          process_sp->ReadPointerFromMemory(addr_of_string, error);
-      if (error.Fail() || addr_of_data == 0 ||
-          addr_of_data == LLDB_INVALID_ADDRESS)
-        return false;
-      options.SetLocation(addr_of_data);
-      options.SetTargetSP(valobj.GetTargetSP());
-      options.SetStream(&stream);
-      options.SetNeedsZeroTermination(false);
-      options.SetBinaryZeroIsTerminator(true);
-      lldb::addr_t size_of_data = process_sp->ReadPointerFromMemory(
-          addr_of_string + process_sp->GetAddressByteSize(), error);
-      if (error.Fail())
-        return false;
-      options.SetSourceSize(size_of_data);
-      options.SetHasSourceSize(true);
-
-      if (!StringPrinter::ReadStringAndDumpToStream<
-              StringPrinter::StringElementType::UTF8>(options)) {
-        stream.Printf("Summary Unavailable");
-        return true;
-      } else
-        return true;
-    } break;
-    case eAddressTypeHost:
-      break;
-    case eAddressTypeInvalid:
-    case eAddressTypeFile:
-      break;
-    }
-  }
-  return false;
-}
+  ValueObjectSP ptr = valobj.GetChildAtNamePath({"_M_dataplus", "_M_p"});
+  if (!ptr)
+    return false;
 
-bool lldb_private::formatters::LibStdcppWStringSummaryProvider(
-    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  const bool scalar_is_load_addr = true;
-  auto [addr_of_string, addr_type] = valobj.GetAddressOf(scalar_is_load_addr);
-  if (addr_of_string != LLDB_INVALID_ADDRESS) {
-    switch (addr_type) {
-    case eAddressTypeLoad: {
-      ProcessSP process_sp(valobj.GetProcessSP());
-      if (!process_sp)
-        return false;
-
-      CompilerType wchar_compiler_type =
-          valobj.GetCompilerType().GetBasicTypeFromAST(lldb::eBasicTypeWChar);
-
-      if (!wchar_compiler_type)
-        return false;
-
-      // Safe to pass nullptr for exe_scope here.
-      std::optional<uint64_t> size =
-          llvm::expectedToOptional(wchar_compiler_type.GetBitSize(nullptr));
-      if (!size)
-        return false;
-      const uint32_t wchar_size = *size;
-
-      StringPrinter::ReadStringAndDumpToStreamOptions options(valobj);
-      Status error;
-      lldb::addr_t addr_of_data =
-          process_sp->ReadPointerFromMemory(addr_of_string, error);
-      if (error.Fail() || addr_of_data == 0 ||
-          addr_of_data == LLDB_INVALID_ADDRESS)
-        return false;
-      options.SetLocation(addr_of_data);
-      options.SetTargetSP(valobj.GetTargetSP());
-      options.SetStream(&stream);
-      options.SetNeedsZeroTermination(false);
-      options.SetBinaryZeroIsTerminator(false);
-      lldb::addr_t size_of_data = process_sp->ReadPointerFromMemory(
-          addr_of_string + process_sp->GetAddressByteSize(), error);
-      if (error.Fail())
-        return false;
-      options.SetSourceSize(size_of_data);
-      options.SetHasSourceSize(true);
-      options.SetPrefixToken("L");
-
-      switch (wchar_size) {
-      case 8:
-        return StringPrinter::ReadStringAndDumpToStream<
-            StringPrinter::StringElementType::UTF8>(options);
-      case 16:
-        return StringPrinter::ReadStringAndDumpToStream<
-            StringPrinter::StringElementType::UTF16>(options);
-      case 32:
-        return StringPrinter::ReadStringAndDumpToStream<
-            StringPrinter::StringElementType::UTF32>(options);
-      default:
-        stream.Printf("size for wchar_t is not valid");
-        return true;
-      }
-      return true;
-    } break;
-    case eAddressTypeHost:
-      break;
-    case eAddressTypeInvalid:
-    case eAddressTypeFile:
-      break;
-    }
-  }
-  return false;
+  stream << ptr->GetSummaryAsCString();
+  return true;
 }
 
 LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
@@ -376,15 +267,16 @@ LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
 
   if (idx == 0)
     return m_ptr_obj->GetSP();
+
   if (idx == 1) {
-    if (m_ptr_obj && !m_obj_obj) {
-      Status error;
-      ValueObjectSP obj_obj = m_ptr_obj->Dereference(error);
-      if (error.Success())
-        m_obj_obj = obj_obj->Clone(ConstString("object")).get();
-    }
-    if (m_obj_obj)
-      return m_obj_obj->GetSP();
+    ValueObjectSP valobj_sp = m_backend.GetSP();
+    if (!valobj_sp)
+      return nullptr;
+
+    Status status;
+    ValueObjectSP value_sp = m_ptr_obj->Dereference(status);
+    if (status.Success())
+      return value_sp;
   }
   return lldb::ValueObjectSP();
 }
@@ -402,8 +294,11 @@ lldb::ChildCacheState LibStdcppSharedPtrSyntheticFrontEnd::Update() {
   if (!ptr_obj_sp)
     return lldb::ChildCacheState::eRefetch;
 
-  m_ptr_obj = ptr_obj_sp->Clone(ConstString("pointer")).get();
-  m_obj_obj = nullptr;
+  auto cast_ptr_sp = GetDesugaredSmartPointerValue(*ptr_obj_sp, *valobj_sp);
+  if (!cast_ptr_sp)
+    return lldb::ChildCacheState::eRefetch;
+
+  m_ptr_obj = cast_ptr_sp->Clone(ConstString("pointer")).get();
 
   return lldb::ChildCacheState::eRefetch;
 }
@@ -412,8 +307,10 @@ llvm::Expected<size_t>
 LibStdcppSharedPtrSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   if (name == "pointer")
     return 0;
+
   if (name == "object" || name == "$$dereference$$")
     return 1;
+
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
 }
@@ -435,29 +332,37 @@ bool lldb_private::formatters::LibStdcppSmartPointerSummaryProvider(
   if (!ptr_sp)
     return false;
 
-  ValueObjectSP usecount_sp(
-      valobj_sp->GetChildAtNamePath({"_M_refcount", "_M_pi", "_M_use_count"}));
-  if (!usecount_sp)
+  DumpCxxSmartPtrPointerSummary(stream, *ptr_sp, options);
+
+  ValueObjectSP pi_sp = valobj_sp->GetChildAtNamePath({"_M_refcount", "_M_pi"});
+  if (!pi_sp)
     return false;
 
-  if (ptr_sp->GetValueAsUnsigned(0) == 0 ||
-      usecount_sp->GetValueAsUnsigned(0) == 0) {
-    stream.Printf("nullptr");
+  bool success;
+  uint64_t pi_addr = pi_sp->GetValueAsUnsigned(0, &success);
+  // Empty control field. We're done.
+  if (!success || pi_addr == 0)
     return true;
+
+  int64_t shared_count = 0;
+  if (auto count_sp = pi_sp->GetChildMemberWithName("_M_use_count")) {
+    bool success;
+    shared_count = count_sp->GetValueAsSigned(0, &success);
+    if (!success)
+      return false;
+
+    stream.Printf(" strong=%" PRId64, shared_count);
   }
 
-  Status error;
-  ValueObjectSP pointee_sp = ptr_sp->Dereference(error);
-  if (pointee_sp && error.Success()) {
-    if (pointee_sp->DumpPrintableRepresentation(
-            stream, ValueObject::eValueObjectRepresentationStyleSummary,
-            lldb::eFormatInvalid,
-            ValueObject::PrintableRepresentationSpecialCases::eDisable,
-            false)) {
-      return true;
-    }
+  // _M_weak_count is the number of weak references + (_M_use_count != 0).
+  if (auto weak_count_sp = pi_sp->GetChildMemberWithName("_M_weak_count")) {
+    bool success;
+    int64_t count = weak_count_sp->GetValueAsUnsigned(0, &success);
+    if (!success)
+      return false;
+
+    stream.Printf(" weak=%" PRId64, count - (shared_count != 0));
   }
 
-  stream.Printf("ptr = 0x%" PRIx64, ptr_sp->GetValueAsUnsigned(0));
   return true;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
index c7b2130275ad8..8d4d777edee88 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
@@ -18,11 +18,7 @@ namespace lldb_private {
 namespace formatters {
 bool LibStdcppStringSummaryProvider(
     ValueObject &valobj, Stream &stream,
-    const TypeSummaryOptions &options); // libcstdc++ c++11 std::string
-
-bool LibStdcppWStringSummaryProvider(
-    ValueObject &valobj, Stream &stream,
-    const TypeSummaryOptions &options); // libcstdc++ c++11 std::wstring
+    const TypeSummaryOptions &options); // libstdc++ std::string
 
 bool LibStdcppSmartPointerSummaryProvider(
     ValueObject &valobj, Stream &stream,
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index b47ff579b6a5e..4daba2e364168 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -42,8 +42,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
   // objects are only destroyed when every shared pointer to any of them
   // is destroyed, so we must not store a shared pointer to any ValueObject
   // derived from our backend ValueObject (since we're in the same cluster).
-  ValueObject* m_ptr_obj = nullptr;
-  ValueObject* m_obj_obj = nullptr;
+  ValueObject *m_ptr_obj = nullptr;
   ValueObject* m_del_obj = nullptr;
 
   ValueObjectSP GetTuple();
@@ -107,7 +106,6 @@ lldb::ChildCacheState LibStdcppUniquePtrSyntheticFrontEnd::Update() {
     if (del_obj)
       m_del_obj = del_obj->Clone(ConstString("deleter")).get();
   }
-  m_obj_obj = nullptr;
 
   return lldb::ChildCacheState::eRefetch;
 }
@@ -119,15 +117,13 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   if (idx == 1 && m_del_obj)
     return m_del_obj->GetSP();
   if (idx == 2) {
-    if (m_ptr_obj && !m_obj_obj) {
-      Status error;
-      ValueObjectSP obj_obj = m_ptr_obj->Dereference(error);
-      if (error.Success()) {
-        m_obj_obj = obj_obj->Clone(ConstString("object")).get();
+    if (m_ptr_obj) {
+      Status status;
+      auto value_sp = m_ptr_obj->Dereference(status);
+      if (status.Success()) {
+        return value_sp;
       }
     }
-    if (m_obj_obj)
-      return m_obj_obj->GetSP();
   }
   return lldb::ValueObjectSP();
 }
@@ -156,14 +152,8 @@ bool LibStdcppUniquePtrSyntheticFrontEnd::GetSummary(
   if (!m_ptr_obj)
     return false;
 
-  bool success;
-  uint64_t ptr_value = m_ptr_obj->GetValueAsUnsigned(0, &success);
-  if (!success)
-    return false;
-  if (ptr_value == 0)
-    stream.Printf("nullptr");
-  else
-    stream.Printf("0x%" PRIx64, ptr_value);
+  DumpCxxSmartPtrPointerSummary(stream, *m_ptr_obj, options);
+
   return true;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/MsvcStl.cpp b/lldb/source/Plugins/Language/CPlusPlus/MsvcStl.cpp
new file mode 100644
index 0000000000000..5d0f20b1beaae
--- /dev/null
+++ b/lldb/source/Plugins/Language/CPlusPlus/MsvcStl.cpp
@@ -0,0 +1,155 @@
+//===-- MsvcStl.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MsvcStl.h"
+
+#include "lldb/Core/Debugger.h"
+#include "lldb/Core/FormatEntity.h"
+#include "lldb/DataFormatters/StringPrinter.h"
+#include "lldb/DataFormatters/TypeSummary.h"
+#include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/Status.h"
+#include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+
+#include "Plugins/Language/CPlusPlus/CxxStringTypes.h"
+
+#include "lldb/lldb-forward.h"
+#include <optional>
+#include <tuple>
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::formatters;
+
+using StringElementType = StringPrinter::StringElementType;
+
+template <StringElementType element_type>
+static constexpr uint64_t StringElementByteSize() {
+  switch (element_type) {
+  case StringElementType::ASCII:
+  case StringElementType::UTF8:
+    return 1;
+  case StringElementType::UTF16:
+    return 2;
+  case StringElementType::UTF32:
+    return 3;
+  }
+  return 0;
+}
+
+static ValueObjectSP ExtractMsvcStlStringData(ValueObject &valobj) {
+  return valobj.GetChildAtNamePath({"_Mypair", "_Myval2"});
+}
+
+/// Determine the size in bytes of \p valobj (a MSVC STL std::string object) and
+/// extract its data payload. Return the size + payload pair.
+static std::optional<std::pair<uint64_t, ValueObjectSP>>
+ExtractMsvcStlStringInfo(ValueObject &valobj, uint64_t element_size) {
+  ValueObjectSP valobj_pair_sp = ExtractMsvcStlStringData(valobj);
+  if (!valobj_pair_sp || !valobj_pair_sp->GetError().Success())
+    return {};
+
+  ValueObjectSP size_sp = valobj_pair_sp->GetChildMemberWithName("_Mysize");
+  ValueObjectSP capacity_sp = valobj_pair_sp->GetChildMemberWithName("_Myres");
+  ValueObjectSP bx_sp = valobj_pair_sp->GetChildMemberWithName("_Bx");
+  if (!size_sp || !capacity_sp || !bx_sp)
+    return {};
+
+  bool success = false;
+  uint64_t size = size_sp->GetValueAsUnsigned(0, &success);
+  if (!success)
+    return {};
+  uint64_t capacity = capacity_sp->GetValueAsUnsigned(0, &success);
+  if (!success)
+    return {};
+
+  size_t bufSize = std::max<size_t>(16 / element_size, 1);
+  bool isShortString = capacity < bufSize;
+
+  if (isShortString) {
+    ValueObjectSP buf_sp = bx_sp->GetChildMemberWithName("_Buf");
+    if (buf_sp)
+      return std::make_pair(size, buf_sp);
+    return {};
+  }
+  ValueObjectSP ptr_sp = bx_sp->GetChildMemberWithName("_Ptr");
+  if (ptr_sp)
+    return std::make_pair(size, ptr_sp);
+  return {};
+}
+
+template <StringPrinter::StringElementType element_type>
+static bool
+MsvcStlStringSummaryProviderImpl(ValueObject &valobj, Stream &stream,
+                                 const TypeSummaryOptions &summary_options,
+                                 std::string prefix_token) {
+  auto string_info =
+      ExtractMsvcStlStringInfo(valobj, StringElementByteSize<element_type>());
+  if (!string_info)
+    return false;
+  auto [size, location_sp] = *string_info;
+
+  return StringBufferSummaryProvider<element_type>(
+      stream, summary_options, location_sp, size, prefix_token);
+}
+template <StringPrinter::StringElementType element_type>
+static bool formatStringImpl(ValueObject &valobj, Stream &stream,
+                             const TypeSummaryOptions &summary_options,
+                             std::string prefix_token) {
+  StreamString scratch_stream;
+  const bool success = MsvcStlStringSummaryProviderImpl<element_type>(
+      valobj, scratch_stream, summary_options, prefix_token);
+  if (success)
+    stream << scratch_stream.GetData();
+  else
+    stream << "Summary Unavailable";
+  return true;
+}
+
+bool lldb_private::formatters::IsMsvcStlStringType(ValueObject &valobj) {
+  std::vector<uint32_t> indexes;
+  return valobj.GetCompilerType().GetIndexOfChildMemberWithName("_Mypair", true,
+                                                                indexes) > 0;
+}
+
+bool lldb_private::formatters::MsvcStlWStringSummaryProvider(
+    ValueObject &valobj, Stream &stream,
+    const TypeSummaryOptions &summary_options) {
+  return formatStringImpl<StringElementType::UTF16>(valobj, stream,
+                                                    summary_options, "L");
+}
+
+template <>
+bool lldb_private::formatters::MsvcStlStringSummaryProvider<
+    StringElementType::ASCII>(ValueObject &valobj, Stream &stream,
+                              const TypeSummaryOptions &summary_options) {
+  return MsvcStlStringSummaryProviderImpl<StringElementType::ASCII>(
+      valobj, stream, summary_options, "");
+}
+template <>
+bool lldb_private::formatters::MsvcStlStringSummaryProvider<
+    StringElementType::UTF8>(ValueObject &valobj, Stream &stream,
+                             const TypeSummaryOptions &summary_options) {
+  return MsvcStlStringSummaryProviderImpl<StringElementType::UTF8>(
+      valobj, stream, summary_options, "u8");
+}
+template <>
+bool lldb_private::formatters::MsvcStlStringSummaryProvider<
+    StringElementType::UTF16>(ValueObject &valobj, Stream &stream,
+                              const TypeSummaryOptions &summary_options) {
+  return MsvcStlStringSummaryProviderImpl<StringElementType::UTF16>(
+      valobj, stream, summary_options, "u");
+}
+template <>
+bool lldb_private::formatters::MsvcStlStringSummaryProvider<
+    StringElementType::UTF32>(ValueObject &valobj, Stream &stream,
+                              const TypeSummaryOptions &summary_options) {
+  return MsvcStlStringSummaryProviderImpl<StringElementType::UTF32>(
+      valobj, stream, summary_options, "U");
+}
diff --git a/lldb/source/Plugins/Language/CPlusPlus/MsvcStl.h b/lldb/source/Plugins/Language/CPlusPlus/MsvcStl.h
new file mode 100644
index 0000000000000..e4ed923033aa7
--- /dev/null
+++ b/lldb/source/Plugins/Language/CPlusPlus/MsvcStl.h
@@ -0,0 +1,35 @@
+//===-- MsvcStl.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_MSVCSTL_H
+#define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_MSVCSTL_H
+
+#include "lldb/DataFormatters/StringPrinter.h"
+#include "lldb/DataFormatters/TypeSummary.h"
+#include "lldb/Utility/Stream.h"
+#include "lldb/ValueObject/ValueObject.h"
+
+namespace lldb_private {
+namespace formatters {
+
+bool IsMsvcStlStringType(ValueObject &valobj);
+
+template <StringPrinter::StringElementType element_type>
+bool MsvcStlStringSummaryProvider(
+    ValueObject &valobj, Stream &stream,
+    const TypeSummaryOptions
+        &summary_options); // VC 2015+ std::string,u8string,u16string,u32string
+
+bool MsvcStlWStringSummaryProvider(
+    ValueObject &valobj, Stream &stream,
+    const TypeSummaryOptions &options); // VC 2015+ std::wstring
+
+} // namespace formatters
+} // namespace lldb_private
+
+#endif
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index b1741926b74aa..13df6e2f26b53 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -11,7 +11,6 @@
 
 #include "Plugins/Process/Utility/RegisterContextDarwin_arm.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_arm64.h"
-#include "Plugins/Process/Utility/RegisterContextDarwin_i386.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_riscv32.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_x86_64.h"
 #include "lldb/Core/Debugger.h"
@@ -81,9 +80,6 @@
 #ifdef CPU_TYPE_ARM64_32
 #undef CPU_TYPE_ARM64_32
 #endif
-#ifdef CPU_TYPE_I386
-#undef CPU_TYPE_I386
-#endif
 #ifdef CPU_TYPE_X86_64
 #undef CPU_TYPE_X86_64
 #endif
@@ -184,46 +180,32 @@ class RegisterContextDarwin_x86_64_Mach : public RegisterContextDarwin_x86_64 {
     SetError(GPRRegSet, Read, -1);
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
-    bool done = false;
 
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       if (flavor == 0)
-        done = true;
-      else {
-        uint32_t i;
-        uint32_t count = data.GetU32(&offset);
-        switch (flavor) {
-        case GPRRegSet:
-          for (i = 0; i < count; ++i)
-            (&gpr.rax)[i] = data.GetU64(&offset);
-          SetError(GPRRegSet, Read, 0);
-          done = true;
-
-          break;
-        case FPURegSet:
-          // TODO: fill in FPU regs....
-          // SetError (FPURegSet, Read, -1);
-          done = true;
-
-          break;
-        case EXCRegSet:
-          exc.trapno = data.GetU32(&offset);
-          exc.err = data.GetU32(&offset);
-          exc.faultvaddr = data.GetU64(&offset);
-          SetError(EXCRegSet, Read, 0);
-          done = true;
-          break;
-        case 7:
-        case 8:
-        case 9:
-          // fancy flavors that encapsulate of the above flavors...
-          break;
-
-        default:
-          done = true;
-          break;
-        }
+        break;
+      uint32_t count = data.GetU32(&offset);
+      switch (flavor) {
+      case GPRRegSet: {
+        uint32_t *gpr_data = reinterpret_cast<uint32_t *>(&gpr.rax);
+        for (uint32_t i = 0; i < count && offset < data.GetByteSize(); ++i)
+          gpr_data[i] = data.GetU32(&offset);
+        SetError(GPRRegSet, Read, 0);
+      } break;
+      case FPURegSet:
+        // TODO: fill in FPU regs....
+        SetError(FPURegSet, Read, -1);
+        break;
+      case EXCRegSet:
+        exc.trapno = data.GetU32(&offset);
+        exc.err = data.GetU32(&offset);
+        exc.faultvaddr = data.GetU64(&offset);
+        SetError(EXCRegSet, Read, 0);
+        break;
+      default:
+        offset += count * 4;
+        break;
       }
     }
   }
@@ -353,127 +335,11 @@ class RegisterContextDarwin_x86_64_Mach : public RegisterContextDarwin_x86_64 {
   }
 
 protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override { return 0; }
-
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override { return 0; }
-
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override { return 0; }
-
-  int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override {
-    return 0;
-  }
-
-  int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) override {
-    return 0;
-  }
-
-  int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) override {
-    return 0;
-  }
-};
-
-class RegisterContextDarwin_i386_Mach : public RegisterContextDarwin_i386 {
-public:
-  RegisterContextDarwin_i386_Mach(lldb_private::Thread &thread,
-                                  const DataExtractor &data)
-      : RegisterContextDarwin_i386(thread, 0) {
-    SetRegisterDataFrom_LC_THREAD(data);
-  }
-
-  void InvalidateAllRegisters() override {
-    // Do nothing... registers are always valid...
-  }
-
-  void SetRegisterDataFrom_LC_THREAD(const DataExtractor &data) {
-    lldb::offset_t offset = 0;
-    SetError(GPRRegSet, Read, -1);
-    SetError(FPURegSet, Read, -1);
-    SetError(EXCRegSet, Read, -1);
-    bool done = false;
-
-    while (!done) {
-      int flavor = data.GetU32(&offset);
-      if (flavor == 0)
-        done = true;
-      else {
-        uint32_t i;
-        uint32_t count = data.GetU32(&offset);
-        switch (flavor) {
-        case GPRRegSet:
-          for (i = 0; i < count; ++i)
-            (&gpr.eax)[i] = data.GetU32(&offset);
-          SetError(GPRRegSet, Read, 0);
-          done = true;
-
-          break;
-        case FPURegSet:
-          // TODO: fill in FPU regs....
-          // SetError (FPURegSet, Read, -1);
-          done = true;
-
-          break;
-        case EXCRegSet:
-          exc.trapno = data.GetU32(&offset);
-          exc.err = data.GetU32(&offset);
-          exc.faultvaddr = data.GetU32(&offset);
-          SetError(EXCRegSet, Read, 0);
-          done = true;
-          break;
-        case 7:
-        case 8:
-        case 9:
-          // fancy flavors that encapsulate of the above flavors...
-          break;
-
-        default:
-          done = true;
-          break;
-        }
-      }
-    }
-  }
-
-  static bool Create_LC_THREAD(Thread *thread, Stream &data) {
-    RegisterContextSP reg_ctx_sp(thread->GetRegisterContext());
-    if (reg_ctx_sp) {
-      RegisterContext *reg_ctx = reg_ctx_sp.get();
-
-      data.PutHex32(GPRRegSet); // Flavor
-      data.PutHex32(GPRWordCount);
-      PrintRegisterValue(reg_ctx, "eax", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ebx", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ecx", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "edx", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "edi", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "esi", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ebp", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "esp", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ss", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "eflags", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "eip", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "cs", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ds", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "es", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "fs", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "gs", nullptr, 4, data);
-
-      // Write out the EXC registers
-      data.PutHex32(EXCRegSet);
-      data.PutHex32(EXCWordCount);
-      PrintRegisterValue(reg_ctx, "trapno", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "err", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "faultvaddr", nullptr, 4, data);
-      return true;
-    }
-    return false;
-  }
-
-protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override { return 0; }
+  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override { return -1; }
 
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override { return 0; }
+  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override { return -1; }
 
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override { return 0; }
+  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override { return -1; }
 
   int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override {
     return 0;
@@ -505,12 +371,11 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
     SetError(GPRRegSet, Read, -1);
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
-    bool done = false;
 
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       uint32_t count = data.GetU32(&offset);
-      lldb::offset_t next_thread_state = offset + (count * 4);
+      offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRAltRegSet:
       case GPRRegSet: {
@@ -524,9 +389,7 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
 
           SetError(GPRRegSet, Read, 0);
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
 
       case FPURegSet: {
         uint8_t *fpu_reg_buf = (uint8_t *)&fpu.floats;
@@ -536,12 +399,8 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
           offset += fpu_reg_buf_size;
           fpu.fpscr = data.GetU32(&offset);
           SetError(FPURegSet, Read, 0);
-        } else {
-          done = true;
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
 
       case EXCRegSet:
         if (count == 3) {
@@ -550,14 +409,9 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
           exc.far = data.GetU32(&offset);
           SetError(EXCRegSet, Read, 0);
         }
-        done = true;
-        offset = next_thread_state;
         break;
-
-      // Unknown register set flavor, stop trying to parse.
-      default:
-        done = true;
       }
+      offset = next_thread_state;
     }
   }
 
@@ -640,11 +494,10 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
     SetError(GPRRegSet, Read, -1);
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
-    bool done = false;
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       uint32_t count = data.GetU32(&offset);
-      lldb::offset_t next_thread_state = offset + (count * 4);
+      offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRRegSet:
         // x0-x29 + fp + lr + sp + pc (== 33 64-bit registers) plus cpsr (1
@@ -659,7 +512,6 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
           gpr.cpsr = data.GetU32(&offset);
           SetError(GPRRegSet, Read, 0);
         }
-        offset = next_thread_state;
         break;
       case FPURegSet: {
         uint8_t *fpu_reg_buf = (uint8_t *)&fpu.v[0];
@@ -668,12 +520,8 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
             data.ExtractBytes(offset, fpu_reg_buf_size, eByteOrderLittle,
                               fpu_reg_buf) == fpu_reg_buf_size) {
           SetError(FPURegSet, Read, 0);
-        } else {
-          done = true;
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
       case EXCRegSet:
         if (count == 4) {
           exc.far = data.GetU64(&offset);
@@ -681,12 +529,9 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
           exc.exception = data.GetU32(&offset);
           SetError(EXCRegSet, Read, 0);
         }
-        offset = next_thread_state;
-        break;
-      default:
-        done = true;
         break;
       }
+      offset = next_thread_state;
     }
   }
 
@@ -789,11 +634,10 @@ class RegisterContextDarwin_riscv32_Mach
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
     SetError(CSRRegSet, Read, -1);
-    bool done = false;
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       uint32_t count = data.GetU32(&offset);
-      lldb::offset_t next_thread_state = offset + (count * 4);
+      offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRRegSet:
         // x0-x31 + pc
@@ -803,7 +647,6 @@ class RegisterContextDarwin_riscv32_Mach
           gpr.pc = data.GetU32(&offset);
           SetError(GPRRegSet, Read, 0);
         }
-        offset = next_thread_state;
         break;
       case FPURegSet: {
         // f0-f31 + fcsr
@@ -813,9 +656,7 @@ class RegisterContextDarwin_riscv32_Mach
           fpr.fcsr = data.GetU32(&offset);
           SetError(FPURegSet, Read, 0);
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
       case EXCRegSet:
         if (count == 3) {
           exc.exception = data.GetU32(&offset);
@@ -823,12 +664,9 @@ class RegisterContextDarwin_riscv32_Mach
           exc.far = data.GetU32(&offset);
           SetError(EXCRegSet, Read, 0);
         }
-        offset = next_thread_state;
-        break;
-      default:
-        done = true;
         break;
       }
+      offset = next_thread_state;
     }
   }
 
@@ -5422,16 +5260,6 @@ lldb_private::Address ObjectFileMachO::GetEntryPointAddress() {
               done = true;
             }
             break;
-          case llvm::MachO::CPU_TYPE_I386:
-            if (flavor ==
-                1) // x86_THREAD_STATE32 from mach/i386/thread_status.h
-            {
-              offset += 40; // This is the offset of eip in the GPR thread state
-                            // data structure.
-              start_address = m_data.GetU32(&offset);
-              done = true;
-            }
-            break;
           case llvm::MachO::CPU_TYPE_X86_64:
             if (flavor ==
                 4) // x86_THREAD_STATE64 from mach/i386/thread_status.h
@@ -5794,27 +5622,8 @@ bool ObjectFileMachO::GetCorefileThreadExtraInfos(
     std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
 
     Log *log(GetLog(LLDBLog::Object | LLDBLog::Process | LLDBLog::Thread));
-    auto lc_notes = FindLC_NOTEByName("process metadata");
-    for (auto lc_note : lc_notes) {
-      offset_t payload_offset = std::get<0>(lc_note);
-      offset_t strsize = std::get<1>(lc_note);
-      std::string buf(strsize, '\0');
-      if (m_data.CopyData(payload_offset, strsize, buf.data()) != strsize) {
-        LLDB_LOGF(log,
-                  "Unable to read %" PRIu64
-                  " bytes of 'process metadata' LC_NOTE JSON contents",
-                  strsize);
-        return false;
-      }
-      while (buf.back() == '\0')
-        buf.resize(buf.size() - 1);
-      StructuredData::ObjectSP object_sp = StructuredData::ParseJSON(buf);
+    if (StructuredData::ObjectSP object_sp = GetCorefileProcessMetadata()) {
       StructuredData::Dictionary *dict = object_sp->GetAsDictionary();
-      if (!dict) {
-        LLDB_LOGF(log, "Unable to read 'process metadata' LC_NOTE, did not "
-                       "get a dictionary.");
-        return false;
-      }
       StructuredData::Array *threads;
       if (!dict->GetValueForKeyAsArray("threads", threads) || !threads) {
         LLDB_LOGF(log,
@@ -5857,6 +5666,49 @@ bool ObjectFileMachO::GetCorefileThreadExtraInfos(
   return false;
 }
 
+StructuredData::ObjectSP ObjectFileMachO::GetCorefileProcessMetadata() {
+  ModuleSP module_sp(GetModule());
+  if (!module_sp)
+    return {};
+
+  Log *log(GetLog(LLDBLog::Object | LLDBLog::Process | LLDBLog::Thread));
+  std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
+  auto lc_notes = FindLC_NOTEByName("process metadata");
+  if (lc_notes.size() == 0)
+    return {};
+
+  if (lc_notes.size() > 1)
+    LLDB_LOGF(
+        log,
+        "Multiple 'process metadata' LC_NOTEs found, only using the first.");
+
+  auto [payload_offset, strsize] = lc_notes[0];
+  std::string buf(strsize, '\0');
+  if (m_data.CopyData(payload_offset, strsize, buf.data()) != strsize) {
+    LLDB_LOGF(log,
+              "Unable to read %" PRIu64
+              " bytes of 'process metadata' LC_NOTE JSON contents",
+              strsize);
+    return {};
+  }
+  while (buf.back() == '\0')
+    buf.resize(buf.size() - 1);
+  StructuredData::ObjectSP object_sp = StructuredData::ParseJSON(buf);
+  if (!object_sp) {
+    LLDB_LOGF(log, "Unable to read 'process metadata' LC_NOTE, did not "
+                   "parse as valid JSON.");
+    return {};
+  }
+  StructuredData::Dictionary *dict = object_sp->GetAsDictionary();
+  if (!dict) {
+    LLDB_LOGF(log, "Unable to read 'process metadata' LC_NOTE, did not "
+                   "get a dictionary.");
+    return {};
+  }
+
+  return object_sp;
+}
+
 lldb::RegisterContextSP
 ObjectFileMachO::GetThreadContextAtIndex(uint32_t idx,
                                          lldb_private::Thread &thread) {
@@ -5887,11 +5739,6 @@ ObjectFileMachO::GetThreadContextAtIndex(uint32_t idx,
             std::make_shared<RegisterContextDarwin_arm_Mach>(thread, data);
         break;
 
-      case llvm::MachO::CPU_TYPE_I386:
-        reg_ctx_sp =
-            std::make_shared<RegisterContextDarwin_i386_Mach>(thread, data);
-        break;
-
       case llvm::MachO::CPU_TYPE_X86_64:
         reg_ctx_sp =
             std::make_shared<RegisterContextDarwin_x86_64_Mach>(thread, data);
@@ -6759,11 +6606,6 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
                   thread_sp.get(), LC_THREAD_datas[thread_idx]);
               break;
 
-            case llvm::MachO::CPU_TYPE_I386:
-              RegisterContextDarwin_i386_Mach::Create_LC_THREAD(
-                  thread_sp.get(), LC_THREAD_datas[thread_idx]);
-              break;
-
             case llvm::MachO::CPU_TYPE_X86_64:
               RegisterContextDarwin_x86_64_Mach::Create_LC_THREAD(
                   thread_sp.get(), LC_THREAD_datas[thread_idx]);
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index 7f67f5e04f1d6..7e3a6754dd0b8 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -133,6 +133,8 @@ class ObjectFileMachO : public lldb_private::ObjectFile {
 
   bool GetCorefileThreadExtraInfos(std::vector<lldb::tid_t> &tids) override;
 
+  lldb_private::StructuredData::ObjectSP GetCorefileProcessMetadata() override;
+
   bool LoadCoreFileImages(lldb_private::Process &process) override;
 
   lldb::RegisterContextSP
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index 262a7dc731713..1db7bc78013d7 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -1030,6 +1030,40 @@ PlatformDarwin::ExtractAppSpecificInfo(Process &process) {
   return dict_sp;
 }
 
+static llvm::Expected<lldb_private::FileSpec>
+ResolveSDKPathFromDebugInfo(lldb_private::Target *target) {
+
+  ModuleSP exe_module_sp = target->GetExecutableModule();
+  if (!exe_module_sp)
+    return llvm::createStringError("Failed to get module from target");
+
+  SymbolFile *sym_file = exe_module_sp->GetSymbolFile();
+  if (!sym_file)
+    return llvm::createStringError("Failed to get symbol file from module");
+
+  XcodeSDK merged_sdk;
+  for (unsigned i = 0; i < sym_file->GetNumCompileUnits(); ++i) {
+    if (auto cu_sp = sym_file->GetCompileUnitAtIndex(i)) {
+      auto cu_sdk = sym_file->ParseXcodeSDK(*cu_sp);
+      merged_sdk.Merge(cu_sdk);
+    }
+  }
+
+  // TODO: The result of this loop is almost equivalent to deriving the SDK
+  // from the target triple, which would be a lot cheaper.
+  FileSpec sdk_path = merged_sdk.GetSysroot();
+  if (FileSystem::Instance().Exists(sdk_path)) {
+    return sdk_path;
+  }
+  auto path_or_err = HostInfo::GetSDKRoot(HostInfo::SDKOptions{merged_sdk});
+  if (!path_or_err)
+    return llvm::createStringError(
+        llvm::formatv("Failed to resolve SDK path: {0}",
+                      llvm::toString(path_or_err.takeError())));
+
+  return FileSpec(*path_or_err);
+}
+
 void PlatformDarwin::AddClangModuleCompilationOptionsForSDKType(
     Target *target, std::vector<std::string> &options, XcodeSDK::Type sdk_type) {
   const std::vector<std::string> apple_arguments = {
@@ -1129,15 +1163,13 @@ void PlatformDarwin::AddClangModuleCompilationOptionsForSDKType(
   FileSpec sysroot_spec;
 
   if (target) {
-    if (ModuleSP exe_module_sp = target->GetExecutableModule()) {
-      auto path_or_err = ResolveSDKPathFromDebugInfo(*exe_module_sp);
-      if (path_or_err) {
-        sysroot_spec = FileSpec(*path_or_err);
-      } else {
-        LLDB_LOG_ERROR(GetLog(LLDBLog::Types | LLDBLog::Host),
-                       path_or_err.takeError(),
-                       "Failed to resolve SDK path: {0}");
-      }
+    auto sysroot_spec_or_err = ::ResolveSDKPathFromDebugInfo(target);
+    if (!sysroot_spec_or_err) {
+      LLDB_LOG_ERROR(GetLog(LLDBLog::Types | LLDBLog::Host),
+                     sysroot_spec_or_err.takeError(),
+                     "Failed to resolve sysroot: {0}");
+    } else {
+      sysroot_spec = *sysroot_spec_or_err;
     }
   }
 
diff --git a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
index bf552e19742c4..a22083a8a0903 100644
--- a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
@@ -324,12 +324,14 @@ void NativeProcessFreeBSD::MonitorSIGTRAP(lldb::pid_t pid) {
         auto thread_info =
             m_threads_stepping_with_breakpoint.find(thread->GetID());
         if (thread_info != m_threads_stepping_with_breakpoint.end() &&
-            thread_info->second == regctx.GetPC()) {
+            llvm::is_contained(thread_info->second, regctx.GetPC())) {
           thread->SetStoppedByTrace();
-          Status brkpt_error = RemoveBreakpoint(thread_info->second);
-          if (brkpt_error.Fail())
-            LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
-                     thread_info->first, brkpt_error);
+          for (auto &&bp_addr : thread_info->second) {
+            Status brkpt_error = RemoveBreakpoint(bp_addr);
+            if (brkpt_error.Fail())
+              LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
+                       thread_info->first, brkpt_error);
+          }
           m_threads_stepping_with_breakpoint.erase(thread_info);
         } else
           thread->SetStoppedByBreakpoint();
diff --git a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
index 7f2aba0e4eb2c..9c798cb1cc8f2 100644
--- a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
@@ -833,7 +833,7 @@ void NativeProcessLinux::MonitorBreakpoint(NativeThreadLinux &thread) {
   auto stepping_with_bp_it =
       m_threads_stepping_with_breakpoint.find(thread.GetID());
   if (stepping_with_bp_it != m_threads_stepping_with_breakpoint.end() &&
-      stepping_with_bp_it->second == reg_ctx.GetPC())
+      llvm::is_contained(stepping_with_bp_it->second, reg_ctx.GetPC()))
     thread.SetStoppedByTrace();
 
   StopRunningThreads(thread.GetID());
@@ -1960,10 +1960,12 @@ void NativeProcessLinux::SignalIfAllThreadsStopped() {
   // Clear any temporary breakpoints we used to implement software single
   // stepping.
   for (const auto &thread_info : m_threads_stepping_with_breakpoint) {
-    Status error = RemoveBreakpoint(thread_info.second);
-    if (error.Fail())
-      LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
-               thread_info.first, error);
+    for (auto &&bp_addr : thread_info.second) {
+      Status error = RemoveBreakpoint(bp_addr);
+      if (error.Fail())
+        LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
+                 thread_info.first, error);
+    }
   }
   m_threads_stepping_with_breakpoint.clear();
 
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt b/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt
index ddce25c62046a..f26e14f580cb7 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt
@@ -12,7 +12,6 @@ add_lldb_library(lldbPluginProcessMacOSXKernel PLUGIN
   ProcessKDPLog.cpp
   RegisterContextKDP_arm.cpp
   RegisterContextKDP_arm64.cpp
-  RegisterContextKDP_i386.cpp
   RegisterContextKDP_x86_64.cpp
   ThreadKDP.cpp
 
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp
deleted file mode 100644
index 61dfeae6ddf43..0000000000000
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-//===-- RegisterContextKDP_i386.cpp ---------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "RegisterContextKDP_i386.h"
-#include "ProcessKDP.h"
-#include "ThreadKDP.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-RegisterContextKDP_i386::RegisterContextKDP_i386(ThreadKDP &thread,
-                                                 uint32_t concrete_frame_idx)
-    : RegisterContextDarwin_i386(thread, concrete_frame_idx),
-      m_kdp_thread(thread) {}
-
-RegisterContextKDP_i386::~RegisterContextKDP_i386() = default;
-
-int RegisterContextKDP_i386::DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestReadRegisters(tid, GPRRegSet, &gpr, sizeof(gpr),
-                                      error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestReadRegisters(tid, FPURegSet, &fpu, sizeof(fpu),
-                                      error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestReadRegisters(tid, EXCRegSet, &exc, sizeof(exc),
-                                      error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoWriteGPR(lldb::tid_t tid, int flavor,
-                                        const GPR &gpr) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestWriteRegisters(tid, GPRRegSet, &gpr, sizeof(gpr),
-                                       error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoWriteFPU(lldb::tid_t tid, int flavor,
-                                        const FPU &fpu) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestWriteRegisters(tid, FPURegSet, &fpu, sizeof(fpu),
-                                       error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoWriteEXC(lldb::tid_t tid, int flavor,
-                                        const EXC &exc) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestWriteRegisters(tid, EXCRegSet, &exc, sizeof(exc),
-                                       error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h b/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h
deleted file mode 100644
index 04868e96191fd..0000000000000
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===-- RegisterContextKDP_i386.h -------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_PROCESS_MACOSX_KERNEL_REGISTERCONTEXTKDP_I386_H
-#define LLDB_SOURCE_PLUGINS_PROCESS_MACOSX_KERNEL_REGISTERCONTEXTKDP_I386_H
-
-#include "Plugins/Process/Utility/RegisterContextDarwin_i386.h"
-
-class ThreadKDP;
-
-class RegisterContextKDP_i386 : public RegisterContextDarwin_i386 {
-public:
-  RegisterContextKDP_i386(ThreadKDP &thread, uint32_t concrete_frame_idx);
-
-  ~RegisterContextKDP_i386() override;
-
-protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override;
-
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override;
-
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override;
-
-  int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override;
-
-  int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) override;
-
-  int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) override;
-
-  ThreadKDP &m_kdp_thread;
-};
-
-#endif // LLDB_SOURCE_PLUGINS_PROCESS_MACOSX_KERNEL_REGISTERCONTEXTKDP_I386_H
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
index 1349de879c69a..69d4cfdae4ad5 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
@@ -26,7 +26,6 @@
 #include "ProcessKDPLog.h"
 #include "RegisterContextKDP_arm.h"
 #include "RegisterContextKDP_arm64.h"
-#include "RegisterContextKDP_i386.h"
 #include "RegisterContextKDP_x86_64.h"
 
 #include <memory>
@@ -105,10 +104,6 @@ ThreadKDP::CreateRegisterContextForFrame(StackFrame *frame) {
         reg_ctx_sp = std::make_shared<RegisterContextKDP_arm64>(
             *this, concrete_frame_idx);
         break;
-      case llvm::MachO::CPU_TYPE_I386:
-        reg_ctx_sp = std::make_shared<RegisterContextKDP_i386>(
-            *this, concrete_frame_idx);
-        break;
       case llvm::MachO::CPU_TYPE_X86_64:
         reg_ctx_sp = std::make_shared<RegisterContextKDP_x86_64>(
             *this, concrete_frame_idx);
diff --git a/lldb/source/Plugins/Process/Utility/CMakeLists.txt b/lldb/source/Plugins/Process/Utility/CMakeLists.txt
index 48646b784f931..5d99c22dafe15 100644
--- a/lldb/source/Plugins/Process/Utility/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/Utility/CMakeLists.txt
@@ -22,7 +22,6 @@ add_lldb_library(lldbPluginProcessUtility
   RegisterContext_x86.cpp
   RegisterContextDarwin_arm.cpp
   RegisterContextDarwin_arm64.cpp
-  RegisterContextDarwin_i386.cpp
   RegisterContextDarwin_riscv32.cpp
   RegisterContextDarwin_x86_64.cpp
   RegisterContextDummy.cpp
@@ -35,7 +34,6 @@ add_lldb_library(lldbPluginProcessUtility
   RegisterContextLinux_x86_64.cpp
   RegisterContextLinux_s390x.cpp
   RegisterContextMach_arm.cpp
-  RegisterContextMach_i386.cpp
   RegisterContextMach_x86_64.cpp
   RegisterContextMemory.cpp
   RegisterContextNetBSD_i386.cpp
diff --git a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
index 31d59e387d25b..eddf4b97babae 100644
--- a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
+++ b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
@@ -87,34 +87,10 @@ static size_t WriteMemoryCallback(EmulateInstruction *instruction, void *baton,
   return length;
 }
 
-static lldb::addr_t ReadFlags(NativeRegisterContext &regsiter_context) {
-  const RegisterInfo *flags_info = regsiter_context.GetRegisterInfo(
-      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS);
-  return regsiter_context.ReadRegisterAsUnsigned(flags_info,
-                                                 LLDB_INVALID_ADDRESS);
-}
-
-static int GetSoftwareBreakpointSize(const ArchSpec &arch,
-                                     lldb::addr_t next_flags) {
-  if (arch.GetMachine() == llvm::Triple::arm) {
-    if (next_flags & 0x20)
-      // Thumb mode
-      return 2;
-    // Arm mode
-    return 4;
-  }
-  if (arch.IsMIPS() || arch.GetTriple().isPPC64() ||
-      arch.GetTriple().isRISCV() || arch.GetTriple().isLoongArch())
-    return 4;
-  return 0;
-}
-
-static Status SetSoftwareBreakpointOnPC(const ArchSpec &arch, lldb::addr_t pc,
-                                        lldb::addr_t next_flags,
-                                        NativeProcessProtocol &process) {
-  int size_hint = GetSoftwareBreakpointSize(arch, next_flags);
+static Status SetSoftwareBreakpoint(lldb::addr_t bp_addr, unsigned bp_size,
+                                    NativeProcessProtocol &process) {
   Status error;
-  error = process.SetBreakpoint(pc, size_hint, /*hardware=*/false);
+  error = process.SetBreakpoint(bp_addr, bp_size, /*hardware=*/false);
 
   // If setting the breakpoint fails because pc is out of the address
   // space, ignore it and let the debugee segfault.
@@ -136,7 +112,6 @@ Status NativeProcessSoftwareSingleStep::SetupSoftwareSingleStepping(
   std::unique_ptr<EmulateInstruction> emulator_up(
       EmulateInstruction::FindPlugin(arch, eInstructionTypePCModifying,
                                      nullptr));
-
   if (emulator_up == nullptr)
     return Status::FromErrorString("Instruction emulator not found!");
 
@@ -147,65 +122,24 @@ Status NativeProcessSoftwareSingleStep::SetupSoftwareSingleStepping(
   emulator_up->SetWriteMemCallback(&WriteMemoryCallback);
   emulator_up->SetWriteRegCallback(&WriteRegisterCallback);
 
-  if (!emulator_up->ReadInstruction()) {
-    // try to get at least the size of next instruction to set breakpoint.
-    auto instr_size = emulator_up->GetLastInstrSize();
-    if (!instr_size)
-      return Status::FromErrorString("Read instruction failed!");
-    bool success = false;
-    auto pc = emulator_up->ReadRegisterUnsigned(eRegisterKindGeneric,
-                                                LLDB_REGNUM_GENERIC_PC,
-                                                LLDB_INVALID_ADDRESS, &success);
-    if (!success)
-      return Status::FromErrorString("Reading pc failed!");
-    lldb::addr_t next_pc = pc + *instr_size;
-    auto result =
-        SetSoftwareBreakpointOnPC(arch, next_pc, /* next_flags */ 0x0, process);
-    m_threads_stepping_with_breakpoint.insert({thread.GetID(), next_pc});
-    return result;
-  }
+  auto bp_locaions_predictor =
+      EmulateInstruction::CreateBreakpointLocationPredictor(
+          std::move(emulator_up));
+
+  auto bp_locations = bp_locaions_predictor->GetBreakpointLocations(error);
+  if (error.Fail())
+    return error;
+
+  for (auto &&bp_addr : bp_locations) {
+    auto bp_size = bp_locaions_predictor->GetBreakpointSize(bp_addr);
+    if (auto err = bp_size.takeError())
+      return Status(toString(std::move(err)));
 
-  bool emulation_result =
-      emulator_up->EvaluateInstruction(eEmulateInstructionOptionAutoAdvancePC);
-
-  const RegisterInfo *reg_info_pc = register_context.GetRegisterInfo(
-      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC);
-  const RegisterInfo *reg_info_flags = register_context.GetRegisterInfo(
-      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS);
-
-  auto pc_it =
-      baton.m_register_values.find(reg_info_pc->kinds[eRegisterKindDWARF]);
-  auto flags_it = reg_info_flags == nullptr
-                      ? baton.m_register_values.end()
-                      : baton.m_register_values.find(
-                            reg_info_flags->kinds[eRegisterKindDWARF]);
-
-  lldb::addr_t next_pc;
-  lldb::addr_t next_flags;
-  if (emulation_result) {
-    assert(pc_it != baton.m_register_values.end() &&
-           "Emulation was successfull but PC wasn't updated");
-    next_pc = pc_it->second.GetAsUInt64();
-
-    if (flags_it != baton.m_register_values.end())
-      next_flags = flags_it->second.GetAsUInt64();
-    else
-      next_flags = ReadFlags(register_context);
-  } else if (pc_it == baton.m_register_values.end()) {
-    // Emulate instruction failed and it haven't changed PC. Advance PC with
-    // the size of the current opcode because the emulation of all
-    // PC modifying instruction should be successful. The failure most
-    // likely caused by a not supported instruction which don't modify PC.
-    next_pc = register_context.GetPC() + emulator_up->GetOpcode().GetByteSize();
-    next_flags = ReadFlags(register_context);
-  } else {
-    // The instruction emulation failed after it modified the PC. It is an
-    // unknown error where we can't continue because the next instruction is
-    // modifying the PC but we don't  know how.
-    return Status::FromErrorString(
-        "Instruction emulation failed unexpectedly.");
+    error = SetSoftwareBreakpoint(bp_addr, *bp_size, process);
+    if (error.Fail())
+      return error;
   }
-  auto result = SetSoftwareBreakpointOnPC(arch, next_pc, next_flags, process);
-  m_threads_stepping_with_breakpoint.insert({thread.GetID(), next_pc});
-  return result;
+
+  m_threads_stepping_with_breakpoint.insert({thread.GetID(), bp_locations});
+  return error;
 }
diff --git a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
index f9435b7a84ba4..e30e9594359fa 100644
--- a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
+++ b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
@@ -23,7 +23,8 @@ class NativeProcessSoftwareSingleStep {
 protected:
   // List of thread ids stepping with a breakpoint with the address of
   // the relevan breakpoint
-  std::map<lldb::tid_t, lldb::addr_t> m_threads_stepping_with_breakpoint;
+  std::map<lldb::tid_t, std::vector<lldb::addr_t>>
+      m_threads_stepping_with_breakpoint;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp
deleted file mode 100644
index 174a5a567d2dd..0000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp
+++ /dev/null
@@ -1,958 +0,0 @@
-//===-- RegisterContextDarwin_i386.cpp ------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Utility/DataBufferHeap.h"
-#include "lldb/Utility/DataExtractor.h"
-#include "lldb/Utility/Endian.h"
-#include "lldb/Utility/Log.h"
-#include "lldb/Utility/RegisterValue.h"
-#include "lldb/Utility/Scalar.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Compiler.h"
-
-#include <cstddef>
-
-#include <memory>
-
-#include "RegisterContextDarwin_i386.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-enum {
-  gpr_eax = 0,
-  gpr_ebx,
-  gpr_ecx,
-  gpr_edx,
-  gpr_edi,
-  gpr_esi,
-  gpr_ebp,
-  gpr_esp,
-  gpr_ss,
-  gpr_eflags,
-  gpr_eip,
-  gpr_cs,
-  gpr_ds,
-  gpr_es,
-  gpr_fs,
-  gpr_gs,
-
-  fpu_fcw,
-  fpu_fsw,
-  fpu_ftw,
-  fpu_fop,
-  fpu_ip,
-  fpu_cs,
-  fpu_dp,
-  fpu_ds,
-  fpu_mxcsr,
-  fpu_mxcsrmask,
-  fpu_stmm0,
-  fpu_stmm1,
-  fpu_stmm2,
-  fpu_stmm3,
-  fpu_stmm4,
-  fpu_stmm5,
-  fpu_stmm6,
-  fpu_stmm7,
-  fpu_xmm0,
-  fpu_xmm1,
-  fpu_xmm2,
-  fpu_xmm3,
-  fpu_xmm4,
-  fpu_xmm5,
-  fpu_xmm6,
-  fpu_xmm7,
-
-  exc_trapno,
-  exc_err,
-  exc_faultvaddr,
-
-  k_num_registers,
-
-  // Aliases
-  fpu_fctrl = fpu_fcw,
-  fpu_fstat = fpu_fsw,
-  fpu_ftag = fpu_ftw,
-  fpu_fiseg = fpu_cs,
-  fpu_fioff = fpu_ip,
-  fpu_foseg = fpu_ds,
-  fpu_fooff = fpu_dp
-};
-
-enum {
-  ehframe_eax = 0,
-  ehframe_ecx,
-  ehframe_edx,
-  ehframe_ebx,
-  ehframe_ebp,
-  ehframe_esp,
-  ehframe_esi,
-  ehframe_edi,
-  ehframe_eip,
-  ehframe_eflags
-};
-
-enum {
-  dwarf_eax = 0,
-  dwarf_ecx,
-  dwarf_edx,
-  dwarf_ebx,
-  dwarf_esp,
-  dwarf_ebp,
-  dwarf_esi,
-  dwarf_edi,
-  dwarf_eip,
-  dwarf_eflags,
-  dwarf_stmm0 = 11,
-  dwarf_stmm1,
-  dwarf_stmm2,
-  dwarf_stmm3,
-  dwarf_stmm4,
-  dwarf_stmm5,
-  dwarf_stmm6,
-  dwarf_stmm7,
-  dwarf_xmm0 = 21,
-  dwarf_xmm1,
-  dwarf_xmm2,
-  dwarf_xmm3,
-  dwarf_xmm4,
-  dwarf_xmm5,
-  dwarf_xmm6,
-  dwarf_xmm7
-};
-
-#define GPR_OFFSET(reg)                                                        \
-  (LLVM_EXTENSION offsetof(RegisterContextDarwin_i386::GPR, reg))
-#define FPU_OFFSET(reg)                                                        \
-  (LLVM_EXTENSION offsetof(RegisterContextDarwin_i386::FPU, reg) +             \
-   sizeof(RegisterContextDarwin_i386::GPR))
-#define EXC_OFFSET(reg)                                                        \
-  (LLVM_EXTENSION offsetof(RegisterContextDarwin_i386::EXC, reg) +             \
-   sizeof(RegisterContextDarwin_i386::GPR) +                                   \
-   sizeof(RegisterContextDarwin_i386::FPU))
-
-// These macros will auto define the register name, alt name, register size,
-// register offset, encoding, format and native register. This ensures that the
-// register state structures are defined correctly and have the correct sizes
-// and offsets.
-#define DEFINE_GPR(reg, alt)                                                   \
-  #reg, alt, sizeof(((RegisterContextDarwin_i386::GPR *) NULL)->reg),          \
-                    GPR_OFFSET(reg), eEncodingUint, eFormatHex
-#define DEFINE_FPU_UINT(reg)                                                   \
-  #reg, NULL, sizeof(((RegisterContextDarwin_i386::FPU *) NULL)->reg),         \
-                     FPU_OFFSET(reg), eEncodingUint, eFormatHex
-#define DEFINE_FPU_VECT(reg, i)                                                \
-  #reg #i, NULL,                                                               \
-      sizeof(((RegisterContextDarwin_i386::FPU *) NULL)->reg[i].bytes),        \
-              FPU_OFFSET(reg[i]), eEncodingVector, eFormatVectorOfUInt8,       \
-                         {LLDB_INVALID_REGNUM, dwarf_##reg##i,                 \
-                          LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,            \
-                          fpu_##reg##i },                                      \
-                          nullptr, nullptr, nullptr,
-
-#define DEFINE_EXC(reg)                                                        \
-  #reg, NULL, sizeof(((RegisterContextDarwin_i386::EXC *) NULL)->reg),         \
-                     EXC_OFFSET(reg), eEncodingUint, eFormatHex
-#define REG_CONTEXT_SIZE                                                       \
-  (sizeof(RegisterContextDarwin_i386::GPR) +                                   \
-   sizeof(RegisterContextDarwin_i386::FPU) +                                   \
-   sizeof(RegisterContextDarwin_i386::EXC))
-
-static RegisterInfo g_register_infos[] = {
-    {DEFINE_GPR(eax, nullptr),
-     {ehframe_eax, dwarf_eax, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_eax},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ebx, nullptr),
-     {ehframe_ebx, dwarf_ebx, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_ebx},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ecx, nullptr),
-     {ehframe_ecx, dwarf_ecx, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_ecx},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(edx, nullptr),
-     {ehframe_edx, dwarf_edx, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_edx},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(edi, nullptr),
-     {ehframe_edi, dwarf_edi, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_edi},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(esi, nullptr),
-     {ehframe_esi, dwarf_esi, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_esi},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ebp, "fp"),
-     {ehframe_ebp, dwarf_ebp, LLDB_REGNUM_GENERIC_FP, LLDB_INVALID_REGNUM,
-      gpr_ebp},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(esp, "sp"),
-     {ehframe_esp, dwarf_esp, LLDB_REGNUM_GENERIC_SP, LLDB_INVALID_REGNUM,
-      gpr_esp},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ss, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_ss},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(eflags, "flags"),
-     {ehframe_eflags, dwarf_eflags, LLDB_REGNUM_GENERIC_FLAGS,
-      LLDB_INVALID_REGNUM, gpr_eflags},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(eip, "pc"),
-     {ehframe_eip, dwarf_eip, LLDB_REGNUM_GENERIC_PC, LLDB_INVALID_REGNUM,
-      gpr_eip},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(cs, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_cs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ds, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_ds},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(es, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_es},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(fs, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_fs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(gs, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_gs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-
-    {DEFINE_FPU_UINT(fcw),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_fcw},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(fsw),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_fsw},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(ftw),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_ftw},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(fop),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_fop},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(ip),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_ip},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(cs),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_cs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(dp),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_dp},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(ds),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_ds},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(mxcsr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_mxcsr},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(mxcsrmask),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_mxcsrmask},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_VECT(stmm, 0)},
-    {DEFINE_FPU_VECT(stmm, 1)},
-    {DEFINE_FPU_VECT(stmm, 2)},
-    {DEFINE_FPU_VECT(stmm, 3)},
-    {DEFINE_FPU_VECT(stmm, 4)},
-    {DEFINE_FPU_VECT(stmm, 5)},
-    {DEFINE_FPU_VECT(stmm, 6)},
-    {DEFINE_FPU_VECT(stmm, 7)},
-    {DEFINE_FPU_VECT(xmm, 0)},
-    {DEFINE_FPU_VECT(xmm, 1)},
-    {DEFINE_FPU_VECT(xmm, 2)},
-    {DEFINE_FPU_VECT(xmm, 3)},
-    {DEFINE_FPU_VECT(xmm, 4)},
-    {DEFINE_FPU_VECT(xmm, 5)},
-    {DEFINE_FPU_VECT(xmm, 6)},
-    {DEFINE_FPU_VECT(xmm, 7)},
-
-    {DEFINE_EXC(trapno),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, exc_trapno},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_EXC(err),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, exc_err},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_EXC(faultvaddr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, exc_faultvaddr},
-     nullptr,
-     nullptr,
-     nullptr,
-     }};
-
-static size_t k_num_register_infos = std::size(g_register_infos);
-
-RegisterContextDarwin_i386::RegisterContextDarwin_i386(
-    Thread &thread, uint32_t concrete_frame_idx)
-    : RegisterContext(thread, concrete_frame_idx), gpr(), fpu(), exc() {
-  uint32_t i;
-  for (i = 0; i < kNumErrors; i++) {
-    gpr_errs[i] = -1;
-    fpu_errs[i] = -1;
-    exc_errs[i] = -1;
-  }
-}
-
-RegisterContextDarwin_i386::~RegisterContextDarwin_i386() = default;
-
-void RegisterContextDarwin_i386::InvalidateAllRegisters() {
-  InvalidateAllRegisterStates();
-}
-
-size_t RegisterContextDarwin_i386::GetRegisterCount() {
-  assert(k_num_register_infos == k_num_registers);
-  return k_num_registers;
-}
-
-const RegisterInfo *
-RegisterContextDarwin_i386::GetRegisterInfoAtIndex(size_t reg) {
-  assert(k_num_register_infos == k_num_registers);
-  if (reg < k_num_registers)
-    return &g_register_infos[reg];
-  return nullptr;
-}
-
-size_t RegisterContextDarwin_i386::GetRegisterInfosCount() {
-  return k_num_register_infos;
-}
-
-const RegisterInfo *RegisterContextDarwin_i386::GetRegisterInfos() {
-  return g_register_infos;
-}
-
-// General purpose registers
-static uint32_t g_gpr_regnums[] = {
-    gpr_eax, gpr_ebx,    gpr_ecx, gpr_edx, gpr_edi, gpr_esi, gpr_ebp, gpr_esp,
-    gpr_ss,  gpr_eflags, gpr_eip, gpr_cs,  gpr_ds,  gpr_es,  gpr_fs,  gpr_gs};
-
-// Floating point registers
-static uint32_t g_fpu_regnums[] = {
-    fpu_fcw,   fpu_fsw,   fpu_ftw,   fpu_fop,       fpu_ip,    fpu_cs,
-    fpu_dp,    fpu_ds,    fpu_mxcsr, fpu_mxcsrmask, fpu_stmm0, fpu_stmm1,
-    fpu_stmm2, fpu_stmm3, fpu_stmm4, fpu_stmm5,     fpu_stmm6, fpu_stmm7,
-    fpu_xmm0,  fpu_xmm1,  fpu_xmm2,  fpu_xmm3,      fpu_xmm4,  fpu_xmm5,
-    fpu_xmm6,  fpu_xmm7};
-
-// Exception registers
-
-static uint32_t g_exc_regnums[] = {exc_trapno, exc_err, exc_faultvaddr};
-
-// Number of registers in each register set
-const size_t k_num_gpr_registers = std::size(g_gpr_regnums);
-const size_t k_num_fpu_registers = std::size(g_fpu_regnums);
-const size_t k_num_exc_registers = std::size(g_exc_regnums);
-
-// Register set definitions. The first definitions at register set index of
-// zero is for all registers, followed by other registers sets. The register
-// information for the all register set need not be filled in.
-static const RegisterSet g_reg_sets[] = {
-    {
-        "General Purpose Registers", "gpr", k_num_gpr_registers, g_gpr_regnums,
-    },
-    {"Floating Point Registers", "fpu", k_num_fpu_registers, g_fpu_regnums},
-    {"Exception State Registers", "exc", k_num_exc_registers, g_exc_regnums}};
-
-const size_t k_num_regsets = std::size(g_reg_sets);
-
-size_t RegisterContextDarwin_i386::GetRegisterSetCount() {
-  return k_num_regsets;
-}
-
-const RegisterSet *RegisterContextDarwin_i386::GetRegisterSet(size_t reg_set) {
-  if (reg_set < k_num_regsets)
-    return &g_reg_sets[reg_set];
-  return nullptr;
-}
-
-// Register information definitions for 32 bit i386.
-int RegisterContextDarwin_i386::GetSetForNativeRegNum(int reg_num) {
-  if (reg_num < fpu_fcw)
-    return GPRRegSet;
-  else if (reg_num < exc_trapno)
-    return FPURegSet;
-  else if (reg_num < k_num_registers)
-    return EXCRegSet;
-  return -1;
-}
-
-void RegisterContextDarwin_i386::LogGPR(Log *log, const char *title) {
-  if (log) {
-    if (title)
-      LLDB_LOGF(log, "%s", title);
-    for (uint32_t i = 0; i < k_num_gpr_registers; i++) {
-      uint32_t reg = gpr_eax + i;
-      LLDB_LOGF(log, "%12s = 0x%8.8x", g_register_infos[reg].name,
-                (&gpr.eax)[reg]);
-    }
-  }
-}
-
-int RegisterContextDarwin_i386::ReadGPR(bool force) {
-  int set = GPRRegSet;
-  if (force || !RegisterSetIsCached(set)) {
-    SetError(set, Read, DoReadGPR(GetThreadID(), set, gpr));
-  }
-  return GetError(set, Read);
-}
-
-int RegisterContextDarwin_i386::ReadFPU(bool force) {
-  int set = FPURegSet;
-  if (force || !RegisterSetIsCached(set)) {
-    SetError(set, Read, DoReadFPU(GetThreadID(), set, fpu));
-  }
-  return GetError(set, Read);
-}
-
-int RegisterContextDarwin_i386::ReadEXC(bool force) {
-  int set = EXCRegSet;
-  if (force || !RegisterSetIsCached(set)) {
-    SetError(set, Read, DoReadEXC(GetThreadID(), set, exc));
-  }
-  return GetError(set, Read);
-}
-
-int RegisterContextDarwin_i386::WriteGPR() {
-  int set = GPRRegSet;
-  if (!RegisterSetIsCached(set)) {
-    SetError(set, Write, -1);
-    return -1;
-  }
-  SetError(set, Write, DoWriteGPR(GetThreadID(), set, gpr));
-  SetError(set, Read, -1);
-  return GetError(set, Write);
-}
-
-int RegisterContextDarwin_i386::WriteFPU() {
-  int set = FPURegSet;
-  if (!RegisterSetIsCached(set)) {
-    SetError(set, Write, -1);
-    return -1;
-  }
-  SetError(set, Write, DoWriteFPU(GetThreadID(), set, fpu));
-  SetError(set, Read, -1);
-  return GetError(set, Write);
-}
-
-int RegisterContextDarwin_i386::WriteEXC() {
-  int set = EXCRegSet;
-  if (!RegisterSetIsCached(set)) {
-    SetError(set, Write, -1);
-    return -1;
-  }
-  SetError(set, Write, DoWriteEXC(GetThreadID(), set, exc));
-  SetError(set, Read, -1);
-  return GetError(set, Write);
-}
-
-int RegisterContextDarwin_i386::ReadRegisterSet(uint32_t set, bool force) {
-  switch (set) {
-  case GPRRegSet:
-    return ReadGPR(force);
-  case FPURegSet:
-    return ReadFPU(force);
-  case EXCRegSet:
-    return ReadEXC(force);
-  default:
-    break;
-  }
-  return -1;
-}
-
-int RegisterContextDarwin_i386::WriteRegisterSet(uint32_t set) {
-  // Make sure we have a valid context to set.
-  if (RegisterSetIsCached(set)) {
-    switch (set) {
-    case GPRRegSet:
-      return WriteGPR();
-    case FPURegSet:
-      return WriteFPU();
-    case EXCRegSet:
-      return WriteEXC();
-    default:
-      break;
-    }
-  }
-  return -1;
-}
-
-bool RegisterContextDarwin_i386::ReadRegister(const RegisterInfo *reg_info,
-                                              RegisterValue &value) {
-  const uint32_t reg = reg_info->kinds[eRegisterKindLLDB];
-  int set = RegisterContextDarwin_i386::GetSetForNativeRegNum(reg);
-
-  if (set == -1)
-    return false;
-
-  if (ReadRegisterSet(set, false) != 0)
-    return false;
-
-  switch (reg) {
-  case gpr_eax:
-  case gpr_ebx:
-  case gpr_ecx:
-  case gpr_edx:
-  case gpr_edi:
-  case gpr_esi:
-  case gpr_ebp:
-  case gpr_esp:
-  case gpr_ss:
-  case gpr_eflags:
-  case gpr_eip:
-  case gpr_cs:
-  case gpr_ds:
-  case gpr_es:
-  case gpr_fs:
-  case gpr_gs:
-    value = (&gpr.eax)[reg - gpr_eax];
-    break;
-
-  case fpu_fcw:
-    value = fpu.fcw;
-    break;
-
-  case fpu_fsw:
-    value = fpu.fsw;
-    break;
-
-  case fpu_ftw:
-    value = fpu.ftw;
-    break;
-
-  case fpu_fop:
-    value = fpu.fop;
-    break;
-
-  case fpu_ip:
-    value = fpu.ip;
-    break;
-
-  case fpu_cs:
-    value = fpu.cs;
-    break;
-
-  case fpu_dp:
-    value = fpu.dp;
-    break;
-
-  case fpu_ds:
-    value = fpu.ds;
-    break;
-
-  case fpu_mxcsr:
-    value = fpu.mxcsr;
-    break;
-
-  case fpu_mxcsrmask:
-    value = fpu.mxcsrmask;
-    break;
-
-  case fpu_stmm0:
-  case fpu_stmm1:
-  case fpu_stmm2:
-  case fpu_stmm3:
-  case fpu_stmm4:
-  case fpu_stmm5:
-  case fpu_stmm6:
-  case fpu_stmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    //::memcpy (reg_value.value.vector.uint8, fpu.stmm[reg - fpu_stmm0].bytes,
-    //10);
-    return false;
-
-  case fpu_xmm0:
-  case fpu_xmm1:
-  case fpu_xmm2:
-  case fpu_xmm3:
-  case fpu_xmm4:
-  case fpu_xmm5:
-  case fpu_xmm6:
-  case fpu_xmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    //::memcpy (reg_value.value.vector.uint8, fpu.xmm[reg - fpu_xmm0].bytes,
-    //16);
-    return false;
-
-  case exc_trapno:
-    value = exc.trapno;
-    break;
-
-  case exc_err:
-    value = exc.err;
-    break;
-
-  case exc_faultvaddr:
-    value = exc.faultvaddr;
-    break;
-
-  default:
-    return false;
-  }
-  return true;
-}
-
-bool RegisterContextDarwin_i386::WriteRegister(const RegisterInfo *reg_info,
-                                               const RegisterValue &value) {
-  const uint32_t reg = reg_info->kinds[eRegisterKindLLDB];
-  int set = GetSetForNativeRegNum(reg);
-
-  if (set == -1)
-    return false;
-
-  if (ReadRegisterSet(set, false) != 0)
-    return false;
-
-  switch (reg) {
-  case gpr_eax:
-  case gpr_ebx:
-  case gpr_ecx:
-  case gpr_edx:
-  case gpr_edi:
-  case gpr_esi:
-  case gpr_ebp:
-  case gpr_esp:
-  case gpr_ss:
-  case gpr_eflags:
-  case gpr_eip:
-  case gpr_cs:
-  case gpr_ds:
-  case gpr_es:
-  case gpr_fs:
-  case gpr_gs:
-    (&gpr.eax)[reg - gpr_eax] = value.GetAsUInt32();
-    break;
-
-  case fpu_fcw:
-    fpu.fcw = value.GetAsUInt16();
-    break;
-
-  case fpu_fsw:
-    fpu.fsw = value.GetAsUInt16();
-    break;
-
-  case fpu_ftw:
-    fpu.ftw = value.GetAsUInt8();
-    break;
-
-  case fpu_fop:
-    fpu.fop = value.GetAsUInt16();
-    break;
-
-  case fpu_ip:
-    fpu.ip = value.GetAsUInt32();
-    break;
-
-  case fpu_cs:
-    fpu.cs = value.GetAsUInt16();
-    break;
-
-  case fpu_dp:
-    fpu.dp = value.GetAsUInt32();
-    break;
-
-  case fpu_ds:
-    fpu.ds = value.GetAsUInt16();
-    break;
-
-  case fpu_mxcsr:
-    fpu.mxcsr = value.GetAsUInt32();
-    break;
-
-  case fpu_mxcsrmask:
-    fpu.mxcsrmask = value.GetAsUInt32();
-    break;
-
-  case fpu_stmm0:
-  case fpu_stmm1:
-  case fpu_stmm2:
-  case fpu_stmm3:
-  case fpu_stmm4:
-  case fpu_stmm5:
-  case fpu_stmm6:
-  case fpu_stmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    ::memcpy(fpu.stmm[reg - fpu_stmm0].bytes, value.GetBytes(),
-             value.GetByteSize());
-    return false;
-
-  case fpu_xmm0:
-  case fpu_xmm1:
-  case fpu_xmm2:
-  case fpu_xmm3:
-  case fpu_xmm4:
-  case fpu_xmm5:
-  case fpu_xmm6:
-  case fpu_xmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    ::memcpy(fpu.xmm[reg - fpu_xmm0].bytes, value.GetBytes(),
-             value.GetByteSize());
-    return false;
-
-  case exc_trapno:
-    exc.trapno = value.GetAsUInt32();
-    break;
-
-  case exc_err:
-    exc.err = value.GetAsUInt32();
-    break;
-
-  case exc_faultvaddr:
-    exc.faultvaddr = value.GetAsUInt32();
-    break;
-
-  default:
-    return false;
-  }
-  return WriteRegisterSet(set) == 0;
-}
-
-bool RegisterContextDarwin_i386::ReadAllRegisterValues(
-    lldb::WritableDataBufferSP &data_sp) {
-  data_sp = std::make_shared<DataBufferHeap>(REG_CONTEXT_SIZE, 0);
-  if (ReadGPR(false) == 0 && ReadFPU(false) == 0 && ReadEXC(false) == 0) {
-    uint8_t *dst = data_sp->GetBytes();
-    ::memcpy(dst, &gpr, sizeof(gpr));
-    dst += sizeof(gpr);
-
-    ::memcpy(dst, &fpu, sizeof(fpu));
-    dst += sizeof(gpr);
-
-    ::memcpy(dst, &exc, sizeof(exc));
-    return true;
-  }
-  return false;
-}
-
-bool RegisterContextDarwin_i386::WriteAllRegisterValues(
-    const lldb::DataBufferSP &data_sp) {
-  if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) {
-    const uint8_t *src = data_sp->GetBytes();
-    ::memcpy(&gpr, src, sizeof(gpr));
-    src += sizeof(gpr);
-
-    ::memcpy(&fpu, src, sizeof(fpu));
-    src += sizeof(gpr);
-
-    ::memcpy(&exc, src, sizeof(exc));
-    uint32_t success_count = 0;
-    if (WriteGPR() == 0)
-      ++success_count;
-    if (WriteFPU() == 0)
-      ++success_count;
-    if (WriteEXC() == 0)
-      ++success_count;
-    return success_count == 3;
-  }
-  return false;
-}
-
-uint32_t RegisterContextDarwin_i386::ConvertRegisterKindToRegisterNumber(
-    lldb::RegisterKind kind, uint32_t reg) {
-  if (kind == eRegisterKindGeneric) {
-    switch (reg) {
-    case LLDB_REGNUM_GENERIC_PC:
-      return gpr_eip;
-    case LLDB_REGNUM_GENERIC_SP:
-      return gpr_esp;
-    case LLDB_REGNUM_GENERIC_FP:
-      return gpr_ebp;
-    case LLDB_REGNUM_GENERIC_FLAGS:
-      return gpr_eflags;
-    case LLDB_REGNUM_GENERIC_RA:
-    default:
-      break;
-    }
-  } else if (kind == eRegisterKindEHFrame || kind == eRegisterKindDWARF) {
-    switch (reg) {
-    case dwarf_eax:
-      return gpr_eax;
-    case dwarf_ecx:
-      return gpr_ecx;
-    case dwarf_edx:
-      return gpr_edx;
-    case dwarf_ebx:
-      return gpr_ebx;
-    case dwarf_esp:
-      return gpr_esp;
-    case dwarf_ebp:
-      return gpr_ebp;
-    case dwarf_esi:
-      return gpr_esi;
-    case dwarf_edi:
-      return gpr_edi;
-    case dwarf_eip:
-      return gpr_eip;
-    case dwarf_eflags:
-      return gpr_eflags;
-    case dwarf_stmm0:
-      return fpu_stmm0;
-    case dwarf_stmm1:
-      return fpu_stmm1;
-    case dwarf_stmm2:
-      return fpu_stmm2;
-    case dwarf_stmm3:
-      return fpu_stmm3;
-    case dwarf_stmm4:
-      return fpu_stmm4;
-    case dwarf_stmm5:
-      return fpu_stmm5;
-    case dwarf_stmm6:
-      return fpu_stmm6;
-    case dwarf_stmm7:
-      return fpu_stmm7;
-    case dwarf_xmm0:
-      return fpu_xmm0;
-    case dwarf_xmm1:
-      return fpu_xmm1;
-    case dwarf_xmm2:
-      return fpu_xmm2;
-    case dwarf_xmm3:
-      return fpu_xmm3;
-    case dwarf_xmm4:
-      return fpu_xmm4;
-    case dwarf_xmm5:
-      return fpu_xmm5;
-    case dwarf_xmm6:
-      return fpu_xmm6;
-    case dwarf_xmm7:
-      return fpu_xmm7;
-    default:
-      break;
-    }
-  } else if (kind == eRegisterKindLLDB) {
-    return reg;
-  }
-  return LLDB_INVALID_REGNUM;
-}
-
-bool RegisterContextDarwin_i386::HardwareSingleStep(bool enable) {
-  if (ReadGPR(false) != 0)
-    return false;
-
-  const uint32_t trace_bit = 0x100u;
-  if (enable) {
-    // If the trace bit is already set, there is nothing to do
-    if (gpr.eflags & trace_bit)
-      return true;
-    else
-      gpr.eflags |= trace_bit;
-  } else {
-    // If the trace bit is already cleared, there is nothing to do
-    if (gpr.eflags & trace_bit)
-      gpr.eflags &= ~trace_bit;
-    else
-      return true;
-  }
-
-  return WriteGPR() == 0;
-}
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h b/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h
deleted file mode 100644
index be933f3be2661..0000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h
+++ /dev/null
@@ -1,208 +0,0 @@
-//===-- RegisterContextDarwin_i386.h ----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTDARWIN_I386_H
-#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTDARWIN_I386_H
-
-#include "lldb/Target/RegisterContext.h"
-#include "lldb/lldb-private.h"
-
-class RegisterContextDarwin_i386 : public lldb_private::RegisterContext {
-public:
-  RegisterContextDarwin_i386(lldb_private::Thread &thread,
-                             uint32_t concrete_frame_idx);
-
-  ~RegisterContextDarwin_i386() override;
-
-  void InvalidateAllRegisters() override;
-
-  size_t GetRegisterCount() override;
-
-  const lldb_private::RegisterInfo *GetRegisterInfoAtIndex(size_t reg) override;
-
-  size_t GetRegisterSetCount() override;
-
-  const lldb_private::RegisterSet *GetRegisterSet(size_t set) override;
-
-  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
-                    lldb_private::RegisterValue &value) override;
-
-  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
-                     const lldb_private::RegisterValue &value) override;
-
-  bool ReadAllRegisterValues(lldb::WritableDataBufferSP &data_sp) override;
-
-  bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override;
-
-  uint32_t ConvertRegisterKindToRegisterNumber(lldb::RegisterKind kind,
-                                               uint32_t num) override;
-
-  bool HardwareSingleStep(bool enable) override;
-
-  struct GPR {
-    uint32_t eax;
-    uint32_t ebx;
-    uint32_t ecx;
-    uint32_t edx;
-    uint32_t edi;
-    uint32_t esi;
-    uint32_t ebp;
-    uint32_t esp;
-    uint32_t ss;
-    uint32_t eflags;
-    uint32_t eip;
-    uint32_t cs;
-    uint32_t ds;
-    uint32_t es;
-    uint32_t fs;
-    uint32_t gs;
-  };
-
-  struct MMSReg {
-    uint8_t bytes[10];
-    uint8_t pad[6];
-  };
-
-  struct XMMReg {
-    uint8_t bytes[16];
-  };
-
-  struct FPU {
-    uint32_t pad[2];
-    uint16_t fcw;
-    uint16_t fsw;
-    uint8_t ftw;
-    uint8_t pad1;
-    uint16_t fop;
-    uint32_t ip;
-    uint16_t cs;
-    uint16_t pad2;
-    uint32_t dp;
-    uint16_t ds;
-    uint16_t pad3;
-    uint32_t mxcsr;
-    uint32_t mxcsrmask;
-    MMSReg stmm[8];
-    XMMReg xmm[8];
-    uint8_t pad4[14 * 16];
-    int pad5;
-  };
-
-  struct EXC {
-    uint32_t trapno;
-    uint32_t err;
-    uint32_t faultvaddr;
-  };
-
-protected:
-  enum { GPRRegSet = 1, FPURegSet = 2, EXCRegSet = 3 };
-
-  enum {
-    GPRWordCount = sizeof(GPR) / sizeof(uint32_t),
-    FPUWordCount = sizeof(FPU) / sizeof(uint32_t),
-    EXCWordCount = sizeof(EXC) / sizeof(uint32_t)
-  };
-
-  enum { Read = 0, Write = 1, kNumErrors = 2 };
-
-  GPR gpr;
-  FPU fpu;
-  EXC exc;
-  int gpr_errs[2]; // Read/Write errors
-  int fpu_errs[2]; // Read/Write errors
-  int exc_errs[2]; // Read/Write errors
-
-  void InvalidateAllRegisterStates() {
-    SetError(GPRRegSet, Read, -1);
-    SetError(FPURegSet, Read, -1);
-    SetError(EXCRegSet, Read, -1);
-  }
-
-  int GetError(int flavor, uint32_t err_idx) const {
-    if (err_idx < kNumErrors) {
-      switch (flavor) {
-      // When getting all errors, just OR all values together to see if
-      // we got any kind of error.
-      case GPRRegSet:
-        return gpr_errs[err_idx];
-      case FPURegSet:
-        return fpu_errs[err_idx];
-      case EXCRegSet:
-        return exc_errs[err_idx];
-      default:
-        break;
-      }
-    }
-    return -1;
-  }
-
-  bool SetError(int flavor, uint32_t err_idx, int err) {
-    if (err_idx < kNumErrors) {
-      switch (flavor) {
-      case GPRRegSet:
-        gpr_errs[err_idx] = err;
-        return true;
-
-      case FPURegSet:
-        fpu_errs[err_idx] = err;
-        return true;
-
-      case EXCRegSet:
-        exc_errs[err_idx] = err;
-        return true;
-
-      default:
-        break;
-      }
-    }
-    return false;
-  }
-
-  bool RegisterSetIsCached(int set) const { return GetError(set, Read) == 0; }
-
-  void LogGPR(lldb_private::Log *log, const char *title);
-
-  int ReadGPR(bool force);
-
-  int ReadFPU(bool force);
-
-  int ReadEXC(bool force);
-
-  int WriteGPR();
-
-  int WriteFPU();
-
-  int WriteEXC();
-
-  // Subclasses override these to do the actual reading.
-  virtual int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) = 0;
-
-  virtual int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) = 0;
-
-  virtual int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) = 0;
-
-  virtual int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) = 0;
-
-  virtual int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) = 0;
-
-  virtual int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) = 0;
-
-  int ReadRegisterSet(uint32_t set, bool force);
-
-  int WriteRegisterSet(uint32_t set);
-
-  static uint32_t GetRegisterNumber(uint32_t reg_kind, uint32_t reg_num);
-
-  static int GetSetForNativeRegNum(int reg_num);
-
-  static size_t GetRegisterInfosCount();
-
-  static const lldb_private::RegisterInfo *GetRegisterInfos();
-};
-
-#endif // LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTDARWIN_I386_H
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp
deleted file mode 100644
index fe5cecef1b0c7..0000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- RegisterContextMach_i386.cpp --------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(__APPLE__)
-
-#include <mach/thread_act.h>
-
-#include "RegisterContextMach_i386.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-RegisterContextMach_i386::RegisterContextMach_i386(Thread &thread,
-                                                   uint32_t concrete_frame_idx)
-    : RegisterContextDarwin_i386(thread, concrete_frame_idx) {}
-
-RegisterContextMach_i386::~RegisterContextMach_i386() = default;
-
-int RegisterContextMach_i386::DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) {
-  mach_msg_type_number_t count = GPRWordCount;
-  return ::thread_get_state(tid, flavor, (thread_state_t)&gpr, &count);
-}
-
-int RegisterContextMach_i386::DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) {
-  mach_msg_type_number_t count = FPUWordCount;
-  return ::thread_get_state(tid, flavor, (thread_state_t)&fpu, &count);
-}
-
-int RegisterContextMach_i386::DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) {
-  mach_msg_type_number_t count = EXCWordCount;
-  return ::thread_get_state(tid, flavor, (thread_state_t)&exc, &count);
-}
-
-int RegisterContextMach_i386::DoWriteGPR(lldb::tid_t tid, int flavor,
-                                         const GPR &gpr) {
-  return ::thread_set_state(
-      tid, flavor, reinterpret_cast<thread_state_t>(const_cast<GPR *>(&gpr)),
-      GPRWordCount);
-}
-
-int RegisterContextMach_i386::DoWriteFPU(lldb::tid_t tid, int flavor,
-                                         const FPU &fpu) {
-  return ::thread_set_state(
-      tid, flavor, reinterpret_cast<thread_state_t>(const_cast<FPU *>(&fpu)),
-      FPUWordCount);
-}
-
-int RegisterContextMach_i386::DoWriteEXC(lldb::tid_t tid, int flavor,
-                                         const EXC &exc) {
-  return ::thread_set_state(
-      tid, flavor, reinterpret_cast<thread_state_t>(const_cast<EXC *>(&exc)),
-      EXCWordCount);
-}
-
-#endif
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h b/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h
deleted file mode 100644
index 8bdac083863d4..0000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===-- RegisterContextMach_i386.h ------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTMACH_I386_H
-#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTMACH_I386_H
-
-#include "RegisterContextDarwin_i386.h"
-
-class RegisterContextMach_i386 : public RegisterContextDarwin_i386 {
-public:
-  RegisterContextMach_i386(lldb_private::Thread &thread,
-                           uint32_t concrete_frame_idx);
-
-  ~RegisterContextMach_i386() override;
-
-protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override;
-
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override;
-
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override;
-
-  int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override;
-
-  int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) override;
-
-  int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) override;
-};
-
-#endif // LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTMACH_I386_H
diff --git a/lldb/source/Plugins/Process/mach-core/CMakeLists.txt b/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
index a1ea85ec4c728..926f06d23c64b 100644
--- a/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_library(lldbPluginProcessMachCore PLUGIN
   ProcessMachCore.cpp
   ThreadMachCore.cpp
+  RegisterContextUnifiedCore.cpp
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp
new file mode 100644
index 0000000000000..449f85bc3f52a
--- /dev/null
+++ b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp
@@ -0,0 +1,312 @@
+//===-- RegisterContextUnifiedCore.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegisterContextUnifiedCore.h"
+#include "lldb/Target/DynamicRegisterInfo.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Utility/DataExtractor.h"
+#include "lldb/Utility/RegisterValue.h"
+#include "lldb/Utility/StructuredData.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+RegisterContextUnifiedCore::RegisterContextUnifiedCore(
+    Thread &thread, uint32_t concrete_frame_idx,
+    RegisterContextSP core_thread_regctx_sp,
+    StructuredData::ObjectSP metadata_thread_registers)
+    : RegisterContext(thread, concrete_frame_idx) {
+
+  ProcessSP process_sp(thread.GetProcess());
+  Target &target = process_sp->GetTarget();
+  StructuredData::Dictionary *metadata_registers_dict = nullptr;
+
+  // If we have thread metadata, check if the keys for register
+  // definitions are present; if not, clear the ObjectSP.
+  if (metadata_thread_registers &&
+      metadata_thread_registers->GetAsDictionary() &&
+      metadata_thread_registers->GetAsDictionary()->HasKey("register_info")) {
+    metadata_registers_dict = metadata_thread_registers->GetAsDictionary()
+                                  ->GetValueForKey("register_info")
+                                  ->GetAsDictionary();
+    if (metadata_registers_dict)
+      if (!metadata_registers_dict->HasKey("sets") ||
+          !metadata_registers_dict->HasKey("registers"))
+        metadata_registers_dict = nullptr;
+  }
+
+  // When creating a register set list from the two sources,
+  // the LC_THREAD aka core_thread_regctx_sp register sets
+  // will be used at the same indexes.
+  // Any additional sets named by the thread metadata registers
+  // will be added after them.  If the thread metadata
+  // specify a set with the same name as LC_THREAD, the already-used
+  // index from the core register context will be used in
+  // the RegisterInfo.
+  std::map<size_t, size_t> metadata_regset_to_combined_regset;
+
+  // Calculate the total size of the register store buffer we need
+  // for all registers.  The corefile register definitions may include
+  // RegisterInfo descriptions of registers that aren't actually
+  // available.  For simplicity, calculate the size of all registers
+  // as if they are available, so we can maintain the same offsets into
+  // the buffer.
+  uint32_t core_buffer_end = 0;
+  for (size_t idx = 0; idx < core_thread_regctx_sp->GetRegisterCount(); idx++) {
+    const RegisterInfo *reginfo =
+        core_thread_regctx_sp->GetRegisterInfoAtIndex(idx);
+    core_buffer_end =
+        std::max(reginfo->byte_offset + reginfo->byte_size, core_buffer_end);
+  }
+
+  // Add metadata register sizes to the total buffer size.
+  uint32_t combined_buffer_end = core_buffer_end;
+  if (metadata_registers_dict) {
+    StructuredData::Array *registers = nullptr;
+    if (metadata_registers_dict->GetValueForKeyAsArray("registers", registers))
+      registers->ForEach(
+          [&combined_buffer_end](StructuredData::Object *ent) -> bool {
+            uint32_t bitsize;
+            if (!ent->GetAsDictionary()->GetValueForKeyAsInteger("bitsize",
+                                                                 bitsize))
+              return false;
+            combined_buffer_end += (bitsize / 8);
+            return true;
+          });
+  }
+  m_register_data.resize(combined_buffer_end, 0);
+
+  // Copy the core register values into our combined data buffer,
+  // skip registers that are contained within another (e.g. w0 vs. x0)
+  // and registers that return as "unavailable".
+  for (size_t idx = 0; idx < core_thread_regctx_sp->GetRegisterCount(); idx++) {
+    const RegisterInfo *reginfo =
+        core_thread_regctx_sp->GetRegisterInfoAtIndex(idx);
+    RegisterValue val;
+    if (!reginfo->value_regs &&
+        core_thread_regctx_sp->ReadRegister(reginfo, val))
+      memcpy(m_register_data.data() + reginfo->byte_offset, val.GetBytes(),
+             val.GetByteSize());
+  }
+
+  // Set 'offset' fields for each register definition into our combined
+  // register data buffer. DynamicRegisterInfo needs this field set to
+  // parse the JSON.
+  // Also copy the values of the registers into our register data buffer.
+  if (metadata_registers_dict) {
+    size_t offset = core_buffer_end;
+    ByteOrder byte_order = core_thread_regctx_sp->GetByteOrder();
+    StructuredData::Array *registers;
+    if (metadata_registers_dict->GetValueForKeyAsArray("registers", registers))
+      registers->ForEach([this, &offset,
+                          byte_order](StructuredData::Object *ent) -> bool {
+        uint64_t bitsize;
+        uint64_t value;
+        if (!ent->GetAsDictionary()->GetValueForKeyAsInteger("bitsize",
+                                                             bitsize))
+          return false;
+        if (!ent->GetAsDictionary()->GetValueForKeyAsInteger("value", value)) {
+          // We had a bitsize but no value, so move the offset forward I guess.
+          offset += (bitsize / 8);
+          return false;
+        }
+        ent->GetAsDictionary()->AddIntegerItem("offset", offset);
+        Status error;
+        const int bytesize = bitsize / 8;
+        switch (bytesize) {
+        case 2: {
+          Scalar value_scalar((uint16_t)value);
+          value_scalar.GetAsMemoryData(m_register_data.data() + offset,
+                                       bytesize, byte_order, error);
+          offset += bytesize;
+        } break;
+        case 4: {
+          Scalar value_scalar((uint32_t)value);
+          value_scalar.GetAsMemoryData(m_register_data.data() + offset,
+                                       bytesize, byte_order, error);
+          offset += bytesize;
+        } break;
+        case 8: {
+          Scalar value_scalar((uint64_t)value);
+          value_scalar.GetAsMemoryData(m_register_data.data() + offset,
+                                       bytesize, byte_order, error);
+          offset += bytesize;
+        } break;
+        }
+        return true;
+      });
+  }
+
+  // Create a DynamicRegisterInfo from the metadata JSON.
+  std::unique_ptr<DynamicRegisterInfo> additional_reginfo_up;
+  if (metadata_registers_dict)
+    additional_reginfo_up = DynamicRegisterInfo::Create(
+        *metadata_registers_dict, target.GetArchitecture());
+
+  // Put the RegisterSet names in the constant string pool,
+  // to sidestep lifetime issues of char*'s.
+  auto copy_regset_name = [](RegisterSet &dst, const RegisterSet &src) {
+    dst.name = ConstString(src.name).AsCString();
+    if (src.short_name)
+      dst.short_name = ConstString(src.short_name).AsCString();
+    else
+      dst.short_name = nullptr;
+  };
+
+  // Copy the core thread register sets into our combined register set list.
+  // RegisterSet indexes will be identical for the LC_THREAD RegisterContext.
+  for (size_t idx = 0; idx < core_thread_regctx_sp->GetRegisterSetCount();
+       idx++) {
+    RegisterSet new_set;
+    const RegisterSet *old_set = core_thread_regctx_sp->GetRegisterSet(idx);
+    copy_regset_name(new_set, *old_set);
+    m_register_sets.push_back(new_set);
+  }
+
+  // Add any additional metadata RegisterSets to our combined RegisterSet array.
+  if (additional_reginfo_up) {
+    for (size_t idx = 0; idx < additional_reginfo_up->GetNumRegisterSets();
+         idx++) {
+      // See if this metadata RegisterSet name matches one already present
+      // from the LC_THREAD RegisterContext.
+      bool found_match = false;
+      const RegisterSet *old_set = additional_reginfo_up->GetRegisterSet(idx);
+      for (size_t jdx = 0; jdx < m_register_sets.size(); jdx++) {
+        if (strcmp(m_register_sets[jdx].name, old_set->name) == 0) {
+          metadata_regset_to_combined_regset[idx] = jdx;
+          found_match = true;
+          break;
+        }
+      }
+      // This metadata RegisterSet is a new one.
+      // Add it to the combined RegisterSet array.
+      if (!found_match) {
+        RegisterSet new_set;
+        copy_regset_name(new_set, *old_set);
+        metadata_regset_to_combined_regset[idx] = m_register_sets.size();
+        m_register_sets.push_back(new_set);
+      }
+    }
+  }
+
+  // Set up our combined RegisterInfo array, one RegisterSet at a time.
+  for (size_t combined_regset_idx = 0;
+       combined_regset_idx < m_register_sets.size(); combined_regset_idx++) {
+    uint32_t registers_this_regset = 0;
+
+    // Copy all LC_THREAD RegisterInfos that have a value into our
+    // combined RegisterInfo array.  (the LC_THREAD RegisterContext
+    // may describe registers that were not provided in this thread)
+    //
+    // LC_THREAD register set indexes are identical to the combined
+    // register set indexes.  The combined register set array may have
+    // additional entries.
+    if (combined_regset_idx < core_thread_regctx_sp->GetRegisterSetCount()) {
+      const RegisterSet *regset =
+          core_thread_regctx_sp->GetRegisterSet(combined_regset_idx);
+      // Copy all the registers that have values in.
+      for (size_t j = 0; j < regset->num_registers; j++) {
+        uint32_t reg_idx = regset->registers[j];
+        const RegisterInfo *reginfo =
+            core_thread_regctx_sp->GetRegisterInfoAtIndex(reg_idx);
+        RegisterValue val;
+        if (!reginfo->value_regs &&
+            core_thread_regctx_sp->ReadRegister(reginfo, val)) {
+          m_regset_regnum_collection[combined_regset_idx].push_back(
+              m_register_infos.size());
+          m_register_infos.push_back(*reginfo);
+          registers_this_regset++;
+        }
+      }
+    }
+
+    // Copy all the metadata RegisterInfos into our combined combined
+    // RegisterInfo array.
+    // The metadata may add registers to one of the LC_THREAD register sets,
+    // or its own newly added register sets.  metadata_regset_to_combined_regset
+    // has the association of the RegisterSet indexes between the two.
+    if (additional_reginfo_up) {
+      // Find the register set in the metadata that matches this register
+      // set, then copy all its RegisterInfos.
+      for (size_t setidx = 0;
+           setidx < additional_reginfo_up->GetNumRegisterSets(); setidx++) {
+        if (metadata_regset_to_combined_regset[setidx] == combined_regset_idx) {
+          const RegisterSet *regset =
+              additional_reginfo_up->GetRegisterSet(setidx);
+          for (size_t j = 0; j < regset->num_registers; j++) {
+            uint32_t reg_idx = regset->registers[j];
+            const RegisterInfo *reginfo =
+                additional_reginfo_up->GetRegisterInfoAtIndex(reg_idx);
+            m_regset_regnum_collection[combined_regset_idx].push_back(
+                m_register_infos.size());
+            m_register_infos.push_back(*reginfo);
+            registers_this_regset++;
+          }
+        }
+      }
+    }
+    m_register_sets[combined_regset_idx].num_registers = registers_this_regset;
+    m_register_sets[combined_regset_idx].registers =
+        m_regset_regnum_collection[combined_regset_idx].data();
+  }
+}
+
+size_t RegisterContextUnifiedCore::GetRegisterCount() {
+  return m_register_infos.size();
+}
+
+const RegisterInfo *
+RegisterContextUnifiedCore::GetRegisterInfoAtIndex(size_t reg) {
+  if (reg < m_register_infos.size())
+    return &m_register_infos[reg];
+  return nullptr;
+}
+
+size_t RegisterContextUnifiedCore::GetRegisterSetCount() {
+  return m_register_sets.size();
+}
+
+const RegisterSet *RegisterContextUnifiedCore::GetRegisterSet(size_t set) {
+  if (set < m_register_sets.size())
+    return &m_register_sets[set];
+  return nullptr;
+}
+
+bool RegisterContextUnifiedCore::ReadRegister(
+    const lldb_private::RegisterInfo *reg_info,
+    lldb_private::RegisterValue &value) {
+  if (!reg_info)
+    return false;
+  if (ProcessSP process_sp = m_thread.GetProcess()) {
+    DataExtractor regdata(m_register_data.data(), m_register_data.size(),
+                          process_sp->GetByteOrder(),
+                          process_sp->GetAddressByteSize());
+    offset_t offset = reg_info->byte_offset;
+    switch (reg_info->byte_size) {
+    case 2:
+      value.SetUInt16(regdata.GetU16(&offset));
+      break;
+    case 4:
+      value.SetUInt32(regdata.GetU32(&offset));
+      break;
+    case 8:
+      value.SetUInt64(regdata.GetU64(&offset));
+      break;
+    default:
+      return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+bool RegisterContextUnifiedCore::WriteRegister(
+    const lldb_private::RegisterInfo *reg_info,
+    const lldb_private::RegisterValue &value) {
+  return false;
+}
diff --git a/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h
new file mode 100644
index 0000000000000..971d5cb568462
--- /dev/null
+++ b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h
@@ -0,0 +1,57 @@
+//===-- RegisterContextUnifiedCore.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_REGISTERCONTEXT_UNIFIED_CORE_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_REGISTERCONTEXT_UNIFIED_CORE_H
+
+#include <string>
+#include <vector>
+
+#include "lldb/Target/RegisterContext.h"
+#include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/StructuredData.h"
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-private.h"
+
+namespace lldb_private {
+
+class RegisterContextUnifiedCore : public RegisterContext {
+public:
+  RegisterContextUnifiedCore(
+      Thread &thread, uint32_t concrete_frame_idx,
+      lldb::RegisterContextSP core_thread_regctx_sp,
+      lldb_private::StructuredData::ObjectSP metadata_thread_registers);
+
+  void InvalidateAllRegisters() override {};
+
+  size_t GetRegisterCount() override;
+
+  const lldb_private::RegisterInfo *GetRegisterInfoAtIndex(size_t reg) override;
+
+  size_t GetRegisterSetCount() override;
+
+  const lldb_private::RegisterSet *GetRegisterSet(size_t set) override;
+
+  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
+                    lldb_private::RegisterValue &value) override;
+
+  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
+                     const lldb_private::RegisterValue &value) override;
+
+private:
+  std::vector<lldb_private::RegisterSet> m_register_sets;
+  std::vector<lldb_private::RegisterInfo> m_register_infos;
+  /// For each register set, an array of register numbers included.
+  std::map<size_t, std::vector<uint32_t>> m_regset_regnum_collection;
+  /// Bytes of the register contents.
+  std::vector<uint8_t> m_register_data;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_REGISTERCONTEXT_UNIFIED_CORE_H
diff --git a/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
index 003cdd0fcaa57..de7de9ee7c5ce 100644
--- a/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
+++ b/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
@@ -6,12 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "RegisterContextUnifiedCore.h"
 #include "ThreadMachCore.h"
 
 #include "lldb/Breakpoint/Watchpoint.h"
 #include "lldb/Host/SafeMachO.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Target/AppleArm64ExceptionClass.h"
+#include "lldb/Target/DynamicRegisterInfo.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Target/StopInfo.h"
@@ -22,6 +28,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/Utility/StructuredData.h"
 
 #include "ProcessMachCore.h"
 //#include "RegisterContextKDP_arm.h"
@@ -70,27 +77,50 @@ lldb::RegisterContextSP ThreadMachCore::GetRegisterContext() {
 
 lldb::RegisterContextSP
 ThreadMachCore::CreateRegisterContextForFrame(StackFrame *frame) {
-  lldb::RegisterContextSP reg_ctx_sp;
   uint32_t concrete_frame_idx = 0;
 
   if (frame)
     concrete_frame_idx = frame->GetConcreteFrameIndex();
+  if (concrete_frame_idx > 0)
+    return GetUnwinder().CreateRegisterContextForFrame(frame);
 
-  if (concrete_frame_idx == 0) {
-    if (!m_thread_reg_ctx_sp) {
-      ProcessSP process_sp(GetProcess());
+  if (m_thread_reg_ctx_sp)
+    return m_thread_reg_ctx_sp;
 
-      ObjectFile *core_objfile =
-          static_cast<ProcessMachCore *>(process_sp.get())->GetCoreObjectFile();
-      if (core_objfile)
-        m_thread_reg_ctx_sp = core_objfile->GetThreadContextAtIndex(
-            m_objfile_lc_thread_idx, *this);
+  ProcessSP process_sp(GetProcess());
+  assert(process_sp);
+
+  ObjectFile *core_objfile =
+      static_cast<ProcessMachCore *>(process_sp.get())->GetCoreObjectFile();
+  if (!core_objfile)
+    return {};
+
+  RegisterContextSP core_thread_regctx_sp =
+      core_objfile->GetThreadContextAtIndex(m_objfile_lc_thread_idx, *this);
+
+  if (!core_thread_regctx_sp)
+    return {};
+
+  StructuredData::ObjectSP process_md_sp =
+      core_objfile->GetCorefileProcessMetadata();
+
+  StructuredData::ObjectSP thread_md_sp;
+  if (process_md_sp && process_md_sp->GetAsDictionary() &&
+      process_md_sp->GetAsDictionary()->HasKey("threads")) {
+    StructuredData::Array *threads = process_md_sp->GetAsDictionary()
+                                         ->GetValueForKey("threads")
+                                         ->GetAsArray();
+    if (threads && threads->GetSize() == core_objfile->GetNumThreadContexts()) {
+      StructuredData::ObjectSP thread_sp =
+          threads->GetItemAtIndex(m_objfile_lc_thread_idx);
+      if (thread_sp && thread_sp->GetAsDictionary())
+        thread_md_sp = thread_sp;
     }
-    reg_ctx_sp = m_thread_reg_ctx_sp;
-  } else {
-    reg_ctx_sp = GetUnwinder().CreateRegisterContextForFrame(frame);
   }
-  return reg_ctx_sp;
+  m_thread_reg_ctx_sp = std::make_shared<RegisterContextUnifiedCore>(
+      *this, concrete_frame_idx, core_thread_regctx_sp, thread_md_sp);
+
+  return m_thread_reg_ctx_sp;
 }
 
 static bool IsCrashExceptionClass(AppleArm64ExceptionClass EC) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
index c5876502b8f29..a0b8acb585635 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
@@ -7,20 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "DWARFDataExtractor.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/ArrayRef.h"
 
 namespace lldb_private {
 
-uint64_t
-DWARFDataExtractor::GetDWARFInitialLength(lldb::offset_t *offset_ptr) const {
-  return GetU32(offset_ptr);
-}
-
-dw_offset_t
-DWARFDataExtractor::GetDWARFOffset(lldb::offset_t *offset_ptr) const {
-  return GetMaxU64(offset_ptr, GetDWARFSizeOfOffset());
-}
-
 llvm::DWARFDataExtractor DWARFDataExtractor::GetAsLLVMDWARF() const {
   return llvm::DWARFDataExtractor(llvm::ArrayRef(GetDataStart(), GetByteSize()),
                                   GetByteOrder() == lldb::eByteOrderLittle,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
index 41b8e9ad0217b..f6bcf9b2d27f7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
@@ -9,7 +9,6 @@
 #ifndef LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDATAEXTRACTOR_H
 #define LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDATAEXTRACTOR_H
 
-#include "lldb/Core/dwarf.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 
@@ -23,13 +22,6 @@ class DWARFDataExtractor : public DataExtractor {
                      lldb::offset_t length)
       : DataExtractor(data, offset, length) {}
 
-  uint64_t GetDWARFInitialLength(lldb::offset_t *offset_ptr) const;
-
-  dw_offset_t GetDWARFOffset(lldb::offset_t *offset_ptr) const;
-
-  size_t GetDWARFSizeofInitialLength() const { return 4; }
-  size_t GetDWARFSizeOfOffset() const { return 4; }
-
   llvm::DWARFDataExtractor GetAsLLVMDWARF() const;
   llvm::DataExtractor GetAsLLVM() const;
 };
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
index fd3d45cef4c5e..a159b923f8df4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
@@ -36,7 +36,6 @@ bool DWARFFormValue::ExtractValue(const DWARFDataExtractor &data,
   bool indirect = false;
   bool is_block = false;
   m_value.data = nullptr;
-  uint8_t ref_addr_size;
   // Read the value for the form into value and follow and DW_FORM_indirect
   // instances we run into
   do {
@@ -77,7 +76,9 @@ bool DWARFFormValue::ExtractValue(const DWARFDataExtractor &data,
     case DW_FORM_strp:
     case DW_FORM_line_strp:
     case DW_FORM_sec_offset:
-      m_value.uval = data.GetMaxU64(offset_ptr, 4);
+      assert(m_unit);
+      m_value.uval = data.GetMaxU64(
+          offset_ptr, m_unit->GetFormParams().getDwarfOffsetByteSize());
       break;
     case DW_FORM_addrx1:
     case DW_FORM_strx1:
@@ -119,11 +120,8 @@ bool DWARFFormValue::ExtractValue(const DWARFDataExtractor &data,
       break;
     case DW_FORM_ref_addr:
       assert(m_unit);
-      if (m_unit->GetVersion() <= 2)
-        ref_addr_size = m_unit->GetAddressByteSize();
-      else
-        ref_addr_size = 4;
-      m_value.uval = data.GetMaxU64(offset_ptr, ref_addr_size);
+      m_value.uval = data.GetMaxU64(
+          offset_ptr, m_unit->GetFormParams().getRefAddrByteSize());
       break;
     case DW_FORM_indirect:
       m_form = static_cast<dw_form_t>(data.GetULEB128(offset_ptr));
@@ -165,7 +163,7 @@ static FormSize g_form_sizes[] = {
     {1, 1}, // 0x0b DW_FORM_data1
     {1, 1}, // 0x0c DW_FORM_flag
     {0, 0}, // 0x0d DW_FORM_sdata
-    {1, 4}, // 0x0e DW_FORM_strp
+    {0, 0}, // 0x0e DW_FORM_strp (4 bytes for DWARF32, 8 bytes for DWARF64)
     {0, 0}, // 0x0f DW_FORM_udata
     {0, 0}, // 0x10 DW_FORM_ref_addr (addr size for DWARF2 and earlier, 4 bytes
             // for DWARF32, 8 bytes for DWARF32 in DWARF 3 and later
@@ -175,7 +173,7 @@ static FormSize g_form_sizes[] = {
     {1, 8},  // 0x14 DW_FORM_ref8
     {0, 0},  // 0x15 DW_FORM_ref_udata
     {0, 0},  // 0x16 DW_FORM_indirect
-    {1, 4},  // 0x17 DW_FORM_sec_offset
+    {0, 0}, // 0x17 DW_FORM_sec_offset (4 bytes for DWARF32,8 bytes for DWARF64)
     {0, 0},  // 0x18 DW_FORM_exprloc
     {1, 0},  // 0x19 DW_FORM_flag_present
     {0, 0},  // 0x1a DW_FORM_strx (ULEB128)
@@ -183,7 +181,7 @@ static FormSize g_form_sizes[] = {
     {1, 4},  // 0x1c DW_FORM_ref_sup4
     {0, 0},  // 0x1d DW_FORM_strp_sup (4 bytes for DWARF32, 8 bytes for DWARF64)
     {1, 16}, // 0x1e DW_FORM_data16
-    {1, 4},  // 0x1f DW_FORM_line_strp
+    {0, 0}, // 0x1f DW_FORM_line_strp (4 bytes for DWARF32, 8 bytes for DWARF64)
     {1, 8},  // 0x20 DW_FORM_ref_sig8
 };
 
@@ -209,7 +207,6 @@ bool DWARFFormValue::SkipValue(dw_form_t form,
                                const DWARFDataExtractor &debug_info_data,
                                lldb::offset_t *offset_ptr,
                                const DWARFUnit *unit) {
-  uint8_t ref_addr_size;
   switch (form) {
   // Blocks if inlined data that have a length field and the data bytes inlined
   // in the .debug_info
@@ -246,14 +243,9 @@ bool DWARFFormValue::SkipValue(dw_form_t form,
     return true;
 
   case DW_FORM_ref_addr:
-    ref_addr_size = 4;
     assert(unit); // Unit must be valid for DW_FORM_ref_addr objects or we will
                   // get this wrong
-    if (unit->GetVersion() <= 2)
-      ref_addr_size = unit->GetAddressByteSize();
-    else
-      ref_addr_size = 4;
-    *offset_ptr += ref_addr_size;
+    *offset_ptr += unit->GetFormParams().getRefAddrByteSize();
     return true;
 
   // 0 bytes values (implied from DW_FORM)
@@ -288,7 +280,8 @@ bool DWARFFormValue::SkipValue(dw_form_t form,
     case DW_FORM_sec_offset:
     case DW_FORM_strp:
     case DW_FORM_line_strp:
-      *offset_ptr += 4;
+      assert(unit);
+      *offset_ptr += unit->GetFormParams().getDwarfOffsetByteSize();
       return true;
 
     // 4 byte values
@@ -419,12 +412,8 @@ void DWARFFormValue::Dump(Stream &s) const {
   case DW_FORM_ref_addr: {
     assert(m_unit); // Unit must be valid for DW_FORM_ref_addr objects or we
                     // will get this wrong
-    if (m_unit->GetVersion() <= 2)
-      DumpAddress(s.AsRawOstream(), uvalue, sizeof(uint64_t) * 2);
-    else
-      DumpAddress(s.AsRawOstream(), uvalue,
-                  4 * 2); // 4 for DWARF32, 8 for DWARF64, but we don't
-                          // support DWARF64 yet
+    DumpAddress(s.AsRawOstream(), uvalue,
+                m_unit->GetFormParams().getRefAddrByteSize());
     break;
   }
   case DW_FORM_ref1:
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index ffd6f1dd52aff..f216ab13e8936 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -1073,20 +1073,7 @@ const lldb_private::DWARFDataExtractor &DWARFUnit::GetData() const {
              : m_dwarf.GetDWARFContext().getOrLoadDebugInfoData();
 }
 
-uint32_t DWARFUnit::GetHeaderByteSize() const {
-  switch (m_header.getUnitType()) {
-  case llvm::dwarf::DW_UT_compile:
-  case llvm::dwarf::DW_UT_partial:
-    return GetVersion() < 5 ? 11 : 12;
-  case llvm::dwarf::DW_UT_skeleton:
-  case llvm::dwarf::DW_UT_split_compile:
-    return 20;
-  case llvm::dwarf::DW_UT_type:
-  case llvm::dwarf::DW_UT_split_type:
-    return GetVersion() < 5 ? 23 : 24;
-  }
-  llvm_unreachable("invalid UnitType.");
-}
+uint32_t DWARFUnit::GetHeaderByteSize() const { return m_header.getSize(); }
 
 std::optional<uint64_t>
 DWARFUnit::GetStringOffsetSectionItem(uint32_t index) const {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index c05bba36ed74b..f55400eeaa448 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -119,6 +119,9 @@ class DWARFUnit : public DWARFExpression::Delegate, public UserID {
   // Size of the CU data incl. header but without initial length.
   dw_offset_t GetLength() const { return m_header.getLength(); }
   uint16_t GetVersion() const override { return m_header.getVersion(); }
+  const llvm::dwarf::FormParams &GetFormParams() const {
+    return m_header.getFormParams();
+  }
   const llvm::DWARFAbbreviationDeclarationSet *GetAbbreviations() const;
   dw_offset_t GetAbbrevOffset() const;
   uint8_t GetAddressByteSize() const override {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index c83779c40a05b..5b16ce5f75138 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -4139,7 +4139,8 @@ void SymbolFileDWARF::DumpClangAST(Stream &s, llvm::StringRef filter) {
 }
 
 bool SymbolFileDWARF::GetSeparateDebugInfo(StructuredData::Dictionary &d,
-                                           bool errors_only) {
+                                           bool errors_only,
+                                           bool load_all_debug_info) {
   StructuredData::Array separate_debug_info_files;
   DWARFDebugInfo &info = DebugInfo();
   const size_t num_cus = info.GetNumUnits();
@@ -4182,7 +4183,7 @@ bool SymbolFileDWARF::GetSeparateDebugInfo(StructuredData::Dictionary &d,
 
     // If we have a DWO symbol file, that means we were able to successfully
     // load it.
-    SymbolFile *dwo_symfile = dwarf_cu->GetDwoSymbolFile();
+    SymbolFile *dwo_symfile = dwarf_cu->GetDwoSymbolFile(load_all_debug_info);
     if (dwo_symfile) {
       dwo_data->AddStringItem(
           "resolved_dwo_path",
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 8335f11712872..2dc862cccca14 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -279,8 +279,8 @@ class SymbolFileDWARF : public SymbolFileCommon {
   void DumpClangAST(Stream &s, llvm::StringRef filter) override;
 
   /// List separate dwo files.
-  bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
-                            bool errors_only) override;
+  bool GetSeparateDebugInfo(StructuredData::Dictionary &d, bool errors_only,
+                            bool load_all_debug_info = false) override;
 
   // Gets a pair of loaded and total dwo file counts.
   // For split-dwarf files, this reports the counts for successfully loaded DWO
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
index f3a940b2ee396..dd94f0b36f2b2 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
@@ -1278,7 +1278,8 @@ void SymbolFileDWARFDebugMap::DumpClangAST(Stream &s, llvm::StringRef filter) {
 }
 
 bool SymbolFileDWARFDebugMap::GetSeparateDebugInfo(
-    lldb_private::StructuredData::Dictionary &d, bool errors_only) {
+    lldb_private::StructuredData::Dictionary &d, bool errors_only,
+    bool load_all_debug_info) {
   StructuredData::Array separate_debug_info_files;
   const uint32_t cu_count = GetNumCompileUnits();
   for (uint32_t cu_idx = 0; cu_idx < cu_count; ++cu_idx) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
index 35cbdbbb1692f..f074b17082e62 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
@@ -132,8 +132,8 @@ class SymbolFileDWARFDebugMap : public SymbolFileCommon {
   void DumpClangAST(Stream &s, llvm::StringRef filter) override;
 
   /// List separate oso files.
-  bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
-                            bool errors_only) override;
+  bool GetSeparateDebugInfo(StructuredData::Dictionary &d, bool errors_only,
+                            bool load_all_debug_info = false) override;
 
   // PluginInterface protocol
   llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
diff --git a/lldb/source/Symbol/UnwindPlan.cpp b/lldb/source/Symbol/UnwindPlan.cpp
index e9ac6b6cde295..9245e52732061 100644
--- a/lldb/source/Symbol/UnwindPlan.cpp
+++ b/lldb/source/Symbol/UnwindPlan.cpp
@@ -17,7 +17,8 @@
 #include "lldb/Utility/Log.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/DebugInfo/DIContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include <optional>
 
 using namespace lldb;
@@ -89,8 +90,7 @@ static void DumpDWARFExpr(Stream &s, llvm::ArrayRef<uint8_t> expr, Thread *threa
                              order_and_width->second);
     llvm::DWARFExpression E(data, order_and_width->second,
                             llvm::dwarf::DWARF32);
-    llvm::DWARFExpressionPrinter::print(&E, s.AsRawOstream(),
-                                        llvm::DIDumpOptions(), nullptr);
+    printDwarfExpression(&E, s.AsRawOstream(), llvm::DIDumpOptions(), nullptr);
   } else
     s.PutCString("dwarf-expr");
 }
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 52a54020bc468..d97a814952186 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -562,6 +562,7 @@ ValueObjectSP StackFrame::DILGetValueForVariableExpressionPath(
     return ValueObjectConstResult::Create(nullptr, std::move(error));
   }
 
+  var_sp = (*valobj_or_error)->GetVariable();
   return *valobj_or_error;
 }
 
diff --git a/lldb/source/Utility/IOObject.cpp b/lldb/source/Utility/IOObject.cpp
index 964edce0ce10d..c0c07cc0b68e3 100644
--- a/lldb/source/Utility/IOObject.cpp
+++ b/lldb/source/Utility/IOObject.cpp
@@ -8,7 +8,16 @@
 
 #include "lldb/Utility/IOObject.h"
 
+#ifdef _WIN32
+#include "lldb/Host/windows/windows.h"
+#endif
+
 using namespace lldb_private;
 
+#ifdef _WIN32
+const IOObject::WaitableHandle IOObject::kInvalidHandleValue =
+    INVALID_HANDLE_VALUE;
+#else
 const IOObject::WaitableHandle IOObject::kInvalidHandleValue = -1;
+#endif
 IOObject::~IOObject() = default;
diff --git a/lldb/source/Utility/SelectHelper.cpp b/lldb/source/Utility/SelectHelper.cpp
index 34c1b62228fdc..3044cdae110ed 100644
--- a/lldb/source/Utility/SelectHelper.cpp
+++ b/lldb/source/Utility/SelectHelper.cpp
@@ -177,7 +177,7 @@ lldb_private::Status SelectHelper::Select() {
 #endif
   // Set the FD bits in the fdsets for read/write/error
   for (auto &pair : m_fd_map) {
-    const lldb::socket_t fd = pair.first;
+    lldb::socket_t fd = pair.first;
 
     if (pair.second.read_set)
       FD_SET(fd, read_fdset_ptr);
diff --git a/lldb/source/ValueObject/CMakeLists.txt b/lldb/source/ValueObject/CMakeLists.txt
index db590f757972a..2a61407521bec 100644
--- a/lldb/source/ValueObject/CMakeLists.txt
+++ b/lldb/source/ValueObject/CMakeLists.txt
@@ -14,7 +14,7 @@ add_lldb_library(lldbValueObject
   ValueObjectList.cpp
   ValueObjectMemory.cpp
   ValueObjectRegister.cpp
-  ValueObjectSyntheticFilter.cpp
+  ValueObjectSynthetic.cpp
   ValueObjectUpdater.cpp
   ValueObjectVariable.cpp
   ValueObjectVTable.cpp
diff --git a/lldb/source/ValueObject/DILEval.cpp b/lldb/source/ValueObject/DILEval.cpp
index 4293990208c77..8ca9b4215985d 100644
--- a/lldb/source/ValueObject/DILEval.cpp
+++ b/lldb/source/ValueObject/DILEval.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/ValueObject/DILEval.h"
+#include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/ValueObject/DILAST.h"
@@ -18,70 +19,26 @@
 
 namespace lldb_private::dil {
 
-static lldb::ValueObjectSP LookupStaticIdentifier(
-    VariableList &variable_list, std::shared_ptr<StackFrame> exe_scope,
-    llvm::StringRef name_ref, llvm::StringRef unqualified_name) {
-  // First look for an exact match to the (possibly) qualified name.
-  for (const lldb::VariableSP &var_sp : variable_list) {
-    lldb::ValueObjectSP valobj_sp(
-        ValueObjectVariable::Create(exe_scope.get(), var_sp));
-    if (valobj_sp && valobj_sp->GetVariable() &&
-        (valobj_sp->GetVariable()->NameMatches(ConstString(name_ref))))
-      return valobj_sp;
-  }
-
-  // If the qualified name is the same as the unqualfied name, there's nothing
-  // more to be done.
-  if (name_ref == unqualified_name)
-    return nullptr;
-
-  // We didn't match the qualified name; try to match the unqualified name.
-  for (const lldb::VariableSP &var_sp : variable_list) {
-    lldb::ValueObjectSP valobj_sp(
-        ValueObjectVariable::Create(exe_scope.get(), var_sp));
-    if (valobj_sp && valobj_sp->GetVariable() &&
-        (valobj_sp->GetVariable()->NameMatches(ConstString(unqualified_name))))
-      return valobj_sp;
-  }
-
-  return nullptr;
-}
-
 static lldb::VariableSP DILFindVariable(ConstString name,
-                                        lldb::VariableListSP variable_list) {
+                                        VariableList &variable_list) {
   lldb::VariableSP exact_match;
   std::vector<lldb::VariableSP> possible_matches;
 
-  for (lldb::VariableSP var_sp : *variable_list) {
+  for (lldb::VariableSP var_sp : variable_list) {
     llvm::StringRef str_ref_name = var_sp->GetName().GetStringRef();
-    // Check for global vars, which might start with '::'.
-    str_ref_name.consume_front("::");
 
-    if (str_ref_name == name.GetStringRef())
-      possible_matches.push_back(var_sp);
-    else if (var_sp->NameMatches(name))
-      possible_matches.push_back(var_sp);
-  }
-
-  // Look for exact matches (favors local vars over global vars)
-  auto exact_match_it =
-      llvm::find_if(possible_matches, [&](lldb::VariableSP var_sp) {
-        return var_sp->GetName() == name;
-      });
-
-  if (exact_match_it != possible_matches.end())
-    return *exact_match_it;
-
-  // Look for a global var exact match.
-  for (auto var_sp : possible_matches) {
-    llvm::StringRef str_ref_name = var_sp->GetName().GetStringRef();
     str_ref_name.consume_front("::");
+    // Check for the exact same match
     if (str_ref_name == name.GetStringRef())
       return var_sp;
+
+    // Check for possible matches by base name
+    if (var_sp->NameMatches(name))
+      possible_matches.push_back(var_sp);
   }
 
-  // If there's a single non-exact match, take it.
-  if (possible_matches.size() == 1)
+  // If there's a non-exact match, take it.
+  if (possible_matches.size() > 0)
     return possible_matches[0];
 
   return nullptr;
@@ -89,38 +46,23 @@ static lldb::VariableSP DILFindVariable(ConstString name,
 
 lldb::ValueObjectSP LookupGlobalIdentifier(
     llvm::StringRef name_ref, std::shared_ptr<StackFrame> stack_frame,
-    lldb::TargetSP target_sp, lldb::DynamicValueType use_dynamic,
-    CompilerType *scope_ptr) {
-  // First look for match in "local" global variables.
-  lldb::VariableListSP variable_list(stack_frame->GetInScopeVariableList(true));
-  name_ref.consume_front("::");
+    lldb::TargetSP target_sp, lldb::DynamicValueType use_dynamic) {
+  // Get a global variables list without the locals from the current frame
+  SymbolContext symbol_context =
+      stack_frame->GetSymbolContext(lldb::eSymbolContextCompUnit);
+  lldb::VariableListSP variable_list =
+      symbol_context.comp_unit->GetVariableList(true);
 
+  name_ref.consume_front("::");
   lldb::ValueObjectSP value_sp;
   if (variable_list) {
     lldb::VariableSP var_sp =
-        DILFindVariable(ConstString(name_ref), variable_list);
+        DILFindVariable(ConstString(name_ref), *variable_list);
     if (var_sp)
       value_sp =
           stack_frame->GetValueObjectForFrameVariable(var_sp, use_dynamic);
   }
 
-  if (value_sp)
-    return value_sp;
-
-  // Also check for static global vars.
-  if (variable_list) {
-    const char *type_name = "";
-    if (scope_ptr)
-      type_name = scope_ptr->GetCanonicalType().GetTypeName().AsCString();
-    std::string name_with_type_prefix =
-        llvm::formatv("{0}::{1}", type_name, name_ref).str();
-    value_sp = LookupStaticIdentifier(*variable_list, stack_frame,
-                                      name_with_type_prefix, name_ref);
-    if (!value_sp)
-      value_sp = LookupStaticIdentifier(*variable_list, stack_frame, name_ref,
-                                        name_ref);
-  }
-
   if (value_sp)
     return value_sp;
 
@@ -129,28 +71,22 @@ lldb::ValueObjectSP LookupGlobalIdentifier(
   target_sp->GetImages().FindGlobalVariables(
       ConstString(name_ref), std::numeric_limits<uint32_t>::max(),
       modules_var_list);
-  if (modules_var_list.Empty())
-    return nullptr;
 
-  for (const lldb::VariableSP &var_sp : modules_var_list) {
-    std::string qualified_name = llvm::formatv("::{0}", name_ref).str();
-    if (var_sp->NameMatches(ConstString(name_ref)) ||
-        var_sp->NameMatches(ConstString(qualified_name))) {
+  if (!modules_var_list.Empty()) {
+    lldb::VariableSP var_sp =
+        DILFindVariable(ConstString(name_ref), modules_var_list);
+    if (var_sp)
       value_sp = ValueObjectVariable::Create(stack_frame.get(), var_sp);
-      break;
-    }
-  }
-
-  if (value_sp)
-    return value_sp;
 
+    if (value_sp)
+      return value_sp;
+  }
   return nullptr;
 }
 
 lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
                                      std::shared_ptr<StackFrame> stack_frame,
-                                     lldb::DynamicValueType use_dynamic,
-                                     CompilerType *scope_ptr) {
+                                     lldb::DynamicValueType use_dynamic) {
   // Support $rax as a special syntax for accessing registers.
   // Will return an invalid value in case the requested register doesn't exist.
   if (name_ref.consume_front("$")) {
@@ -164,38 +100,34 @@ lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
     return nullptr;
   }
 
-  lldb::VariableListSP variable_list(
-      stack_frame->GetInScopeVariableList(false));
-
   if (!name_ref.contains("::")) {
-    if (!scope_ptr || !scope_ptr->IsValid()) {
-      // Lookup in the current frame.
-      // Try looking for a local variable in current scope.
-      lldb::ValueObjectSP value_sp;
-      if (variable_list) {
-        lldb::VariableSP var_sp =
-            DILFindVariable(ConstString(name_ref), variable_list);
-        if (var_sp)
-          value_sp =
-              stack_frame->GetValueObjectForFrameVariable(var_sp, use_dynamic);
-      }
-      if (!value_sp)
-        value_sp = stack_frame->FindVariable(ConstString(name_ref));
-
-      if (value_sp)
-        return value_sp;
-
-      // Try looking for an instance variable (class member).
-      SymbolContext sc = stack_frame->GetSymbolContext(
-          lldb::eSymbolContextFunction | lldb::eSymbolContextBlock);
-      llvm::StringRef ivar_name = sc.GetInstanceVariableName();
-      value_sp = stack_frame->FindVariable(ConstString(ivar_name));
-      if (value_sp)
-        value_sp = value_sp->GetChildMemberWithName(name_ref);
-
-      if (value_sp)
-        return value_sp;
+    // Lookup in the current frame.
+    // Try looking for a local variable in current scope.
+    lldb::VariableListSP variable_list(
+        stack_frame->GetInScopeVariableList(false));
+
+    lldb::ValueObjectSP value_sp;
+    if (variable_list) {
+      lldb::VariableSP var_sp =
+          variable_list->FindVariable(ConstString(name_ref));
+      if (var_sp)
+        value_sp =
+            stack_frame->GetValueObjectForFrameVariable(var_sp, use_dynamic);
     }
+
+    if (value_sp)
+      return value_sp;
+
+    // Try looking for an instance variable (class member).
+    SymbolContext sc = stack_frame->GetSymbolContext(
+        lldb::eSymbolContextFunction | lldb::eSymbolContextBlock);
+    llvm::StringRef ivar_name = sc.GetInstanceVariableName();
+    value_sp = stack_frame->FindVariable(ConstString(ivar_name));
+    if (value_sp)
+      value_sp = value_sp->GetChildMemberWithName(name_ref);
+
+    if (value_sp)
+      return value_sp;
   }
   return nullptr;
 }
@@ -363,8 +295,8 @@ Interpreter::Visit(const MemberOfNode *node) {
 
     if (!m_use_synthetic || !field_obj) {
       std::string errMsg = llvm::formatv(
-          "no member named '{0}' in {1}", node->GetFieldName(),
-          base->GetCompilerType().GetFullyUnqualifiedType().TypeDescription());
+          "\"{0}\" is not a member of \"({1}) {2}\"", node->GetFieldName(),
+          base->GetTypeName().AsCString("<invalid type>"), base->GetName());
       return llvm::make_error<DILDiagnosticError>(
           m_expr, errMsg, node->GetLocation(), node->GetFieldName().size());
     }
@@ -383,9 +315,9 @@ Interpreter::Visit(const MemberOfNode *node) {
   CompilerType base_type = base->GetCompilerType();
   if (node->GetIsArrow() && base->IsPointerType())
     base_type = base_type.GetPointeeType();
-  std::string errMsg =
-      llvm::formatv("no member named '{0}' in {1}", node->GetFieldName(),
-                    base_type.GetFullyUnqualifiedType().TypeDescription());
+  std::string errMsg = llvm::formatv(
+      "\"{0}\" is not a member of \"({1}) {2}\"", node->GetFieldName(),
+      base->GetTypeName().AsCString("<invalid type>"), base->GetName());
   return llvm::make_error<DILDiagnosticError>(
       m_expr, errMsg, node->GetLocation(), node->GetFieldName().size());
 }
diff --git a/lldb/source/ValueObject/DILParser.cpp b/lldb/source/ValueObject/DILParser.cpp
index 5abbfeabcea3d..eac41fab90763 100644
--- a/lldb/source/ValueObject/DILParser.cpp
+++ b/lldb/source/ValueObject/DILParser.cpp
@@ -183,12 +183,14 @@ ASTNodeUP DILParser::ParsePostfixExpression() {
 //    "(" expression ")"
 //
 ASTNodeUP DILParser::ParsePrimaryExpression() {
-  if (CurToken().IsOneOf({Token::coloncolon, Token::identifier})) {
+  if (CurToken().IsOneOf(
+          {Token::coloncolon, Token::identifier, Token::l_paren})) {
     // Save the source location for the diagnostics message.
     uint32_t loc = CurToken().GetLocation();
-    auto identifier = ParseIdExpression();
+    std::string identifier = ParseIdExpression();
 
-    return std::make_unique<IdentifierNode>(loc, identifier);
+    if (!identifier.empty())
+      return std::make_unique<IdentifierNode>(loc, identifier);
   }
 
   if (CurToken().Is(Token::l_paren)) {
@@ -232,16 +234,15 @@ std::string DILParser::ParseNestedNameSpecifier() {
         m_dil_lexer.LookAhead(4).Is(Token::coloncolon)) {
       m_dil_lexer.Advance(4);
 
-      assert(
-          (CurToken().Is(Token::identifier) || CurToken().Is(Token::l_paren)) &&
-          "Expected an identifier or anonymous namespace, but not found.");
+      Expect(Token::coloncolon);
+      m_dil_lexer.Advance();
+      if (!CurToken().Is(Token::identifier) && !CurToken().Is(Token::l_paren)) {
+        BailOut("Expected an identifier or anonymous namespace, but not found.",
+                CurToken().GetLocation(), CurToken().GetSpelling().length());
+      }
       // Continue parsing the nested_namespace_specifier.
       std::string identifier2 = ParseNestedNameSpecifier();
-      if (identifier2.empty()) {
-        Expect(Token::identifier);
-        identifier2 = CurToken().GetSpelling();
-        m_dil_lexer.Advance();
-      }
+
       return "(anonymous namespace)::" + identifier2;
     }
 
@@ -301,6 +302,9 @@ std::string DILParser::ParseIdExpression() {
                          nested_name_specifier, unqualified_id);
   }
 
+  if (!CurToken().Is(Token::identifier))
+    return "";
+
   // No nested_name_specifier, but with global scope -- this is also a
   // qualified_id production. Follow the second production rule.
   if (global_scope) {
diff --git a/lldb/source/ValueObject/ValueObject.cpp b/lldb/source/ValueObject/ValueObject.cpp
index accfd08b30205..a6fb3fad65251 100644
--- a/lldb/source/ValueObject/ValueObject.cpp
+++ b/lldb/source/ValueObject/ValueObject.cpp
@@ -46,7 +46,7 @@
 #include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "lldb/ValueObject/ValueObjectDynamicValue.h"
 #include "lldb/ValueObject/ValueObjectMemory.h"
-#include "lldb/ValueObject/ValueObjectSyntheticFilter.h"
+#include "lldb/ValueObject/ValueObjectSynthetic.h"
 #include "lldb/ValueObject/ValueObjectVTable.h"
 #include "lldb/lldb-private-types.h"
 
diff --git a/lldb/source/ValueObject/ValueObjectSyntheticFilter.cpp b/lldb/source/ValueObject/ValueObjectSynthetic.cpp
similarity index 99%
rename from lldb/source/ValueObject/ValueObjectSyntheticFilter.cpp
rename to lldb/source/ValueObject/ValueObjectSynthetic.cpp
index df774284e4c1b..f673c51a88412 100644
--- a/lldb/source/ValueObject/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/ValueObject/ValueObjectSynthetic.cpp
@@ -1,4 +1,4 @@
-//===-- ValueObjectSyntheticFilter.cpp ------------------------------------===//
+//===-- ValueObjectSynthetic.cpp ------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/ValueObject/ValueObjectSyntheticFilter.h"
+#include "lldb/ValueObject/ValueObjectSynthetic.h"
 
 #include "lldb/Core/Value.h"
 #include "lldb/DataFormatters/TypeSynthetic.h"
diff --git a/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
index ca2d2d0177eec..5c2f1f454c494 100644
--- a/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
+++ b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
@@ -12,7 +12,6 @@ class ExprCommandCallFunctionTestCase(TestBase):
     @expectedFailureAll(
         compiler="icc", bugnumber="llvm.org/pr14437, fails with ICC 13.1"
     )
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr21765")
     @skipIf(compiler="clang", compiler_version=["<", "9.0"])
     def test_with(self):
         """Test calling std::String member function."""
diff --git a/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py b/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
index 9f9848912a115..c22987550efa8 100644
--- a/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
+++ b/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
@@ -7,16 +7,10 @@
 
 
 class ExprSyscallTestCase(TestBase):
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr21765, getpid() does not exist on Windows",
-    )
     @expectedFailureNetBSD
     def test_setpgid(self):
         self.build()
-        self.expr_syscall()
 
-    def expr_syscall(self):
         # Create a target by the debugger.
         target = self.createTestTarget()
 
@@ -65,7 +59,12 @@ def expr_syscall(self):
 
         # try evaluating a couple of expressions in this state
         self.expect_expr("release_flag = 1", result_value="1")
-        self.expect_expr("(int)getpid()", result_value=str(process.GetProcessID()))
+        func = (
+            "GetCurrentProcessId"
+            if lldbplatformutil.getPlatform() == "windows"
+            else "getpid"
+        )
+        self.expect_expr(f"(int){func}()", result_value=str(process.GetProcessID()))
 
         # and run the process to completion
         process.Continue()
diff --git a/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
index 168c4994b39ba..85755a62018c0 100644
--- a/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
@@ -23,7 +23,7 @@ def test(self):
         self.expect_expr(
             "s",
             result_type="std::shared_ptr<Foo>",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("s->a", result_type="int", result_value="3")
         self.expect_expr("s->a = 5", result_type="int", result_value="5")
diff --git a/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
index 1704099cfd85a..35d62ae50f6a8 100644
--- a/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
@@ -10,7 +10,7 @@
 class TestSharedPtr(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=['<', '17.0'])
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     @skipUnlessDarwin
     def test(self):
         self.build()
@@ -24,8 +24,8 @@ def test(self):
         self.expect_expr(
             "s",
             result_type="std::shared_ptr<int>",
-            result_summary="3 strong=1 weak=1",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_summary="3 strong=1 weak=0",
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("*s", result_type="element_type", result_value="3")
         self.expect_expr("*s = 5", result_type="element_type", result_value="5")
diff --git a/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
index f07b0633d32ad..e95214df3b38c 100644
--- a/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
@@ -10,7 +10,7 @@
 class TestDbgInfoContentWeakPtr(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=['<', '17.0'])
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     @skipUnlessDarwin
     def test(self):
         self.build()
@@ -24,7 +24,7 @@ def test(self):
         self.expect_expr(
             "w",
             result_type="std::weak_ptr<Foo>",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("*w.lock()", result_type="element_type")
         self.expect_expr("w.lock()->a", result_type="int", result_value="3")
diff --git a/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
index a9c53548a93c8..0bc0fd466acd5 100644
--- a/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
@@ -10,7 +10,7 @@
 class TestSharedPtr(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=['<', '17.0'])
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     @skipUnlessDarwin
     def test(self):
         self.build()
@@ -24,8 +24,8 @@ def test(self):
         self.expect_expr(
             "w",
             result_type="std::weak_ptr<int>",
-            result_summary="3 strong=1 weak=2",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_summary="3 strong=1 weak=1",
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("*w.lock()", result_type="element_type", result_value="3")
         self.expect_expr("*w.lock() = 5", result_type="element_type", result_value="5")
diff --git a/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
index 0515876734ca4..23188ef898d56 100644
--- a/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
+++ b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
@@ -104,9 +104,11 @@ def build_and_run(self):
 
         self.runCmd("run", RUN_SUCCEEDED)
 
+    @expectedFailureAll(
+        oslist=["windows"],
+        bugnumber="llvm.org/pr21765",
+    )
     @add_test_categories(["pyapi"])
-    # getpid() is POSIX, among other problems, see bug
-    @expectedFailureAll(oslist=["windows"], bugnumber="http://llvm.org/pr21765")
     def test_ir_interpreter(self):
         self.build_and_run()
 
@@ -137,6 +139,14 @@ def test_ir_interpreter(self):
         for expression in set_up_expressions:
             self.frame().EvaluateExpression(expression, options)
 
+        func_call = "(int)getpid()"
+        if lldbplatformutil.getPlatform() == "windows":
+            func_call = "(int)GetCurrentProcessId()"
+
+        for expression in expressions:
+            interp_expression = expression
+            jit_expression = func_call + "; " + expression
+
         for expression in expressions:
             interp_expression = expression
             jit_expression = "(int)getpid(); " + expression
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile
index 99998b20bcb05..c39bce9a94dcf 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile
@@ -1,3 +1,3 @@
-CXX_SOURCES := main.cpp
+CXX_SOURCES := main.cpp extern.cpp
 
 include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py
index edb013c7b3526..9cb166e3d0dc1 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py
@@ -18,6 +18,7 @@ class TestFrameVarDILGlobalVariableLookup(TestBase):
     # each debug info format.
     NO_DEBUG_INFO_TESTCASE = True
 
+    @skipIf(macos_version=["<", "15.0"], archs=["arm64", "arm64e"])
     def test_frame_var(self):
         self.build()
         lldbutil.run_to_source_breakpoint(
@@ -32,20 +33,20 @@ def test_frame_var(self):
         self.expect_var_path("::globalPtr", type="int *")
         self.expect_var_path("::globalRef", type="int &")
 
-        self.expect(
-            "frame variable 'externGlobalVar'",
-            error=True,
-            substrs=["use of undeclared identifier"],
-        )  # 0x00C0FFEE
-        self.expect(
-            "frame variable '::externGlobalVar'",
-            error=True,
-            substrs=["use of undeclared identifier"],
-        )  # ["12648430"])
-
         self.expect_var_path("ns::globalVar", value="13")
         self.expect_var_path("ns::globalPtr", type="int *")
         self.expect_var_path("ns::globalRef", type="int &")
         self.expect_var_path("::ns::globalVar", value="13")
         self.expect_var_path("::ns::globalPtr", type="int *")
         self.expect_var_path("::ns::globalRef", type="int &")
+
+        self.expect_var_path("externGlobalVar", value="2")
+        self.expect_var_path("::externGlobalVar", value="2")
+        self.expect_var_path("ext::externGlobalVar", value="4")
+        self.expect_var_path("::ext::externGlobalVar", value="4")
+
+        self.expect_var_path("ExtStruct::static_inline", value="16")
+
+        # Test local variable priority over global
+        self.expect_var_path("foo", value="1")
+        self.expect_var_path("::foo", value="2")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp
new file mode 100644
index 0000000000000..c451191dafc57
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp
@@ -0,0 +1,10 @@
+int externGlobalVar = 2;
+
+namespace ext {
+int externGlobalVar = 4;
+} // namespace ext
+
+struct ExtStruct {
+private:
+  static constexpr inline int static_inline = 16;
+} es;
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp
index 5bae4fd423e32..a8ecbe2d8fc44 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp
@@ -10,6 +10,9 @@ int *globalPtr = &globalVar;
 int &globalRef = globalVar;
 } // namespace ns
 
+int foo = 2;
+
 int main(int argc, char **argv) {
+  int foo = 1;
   return 0; // Set a breakpoint here
 }
diff --git a/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py b/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py
index bb16c1f82489d..ca6754a556d89 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py
@@ -31,12 +31,19 @@ def test_frame_var(self):
         self.expect_var_path("sp->x", value="1")
         self.expect_var_path("sp->r", type="int &")
 
-        self.expect("frame variable 'sp->foo'", error=True,
-                    substrs=["no member named 'foo' in 'Sx *'"])
-
-        self.expect("frame variable 'sp.x'", error=True,
-                    substrs=["member reference type 'Sx *' is a "
-                             "pointer; did you mean to use '->'"])
+        self.expect(
+            "frame variable 'sp->foo'",
+            error=True,
+            substrs=['"foo" is not a member of "(Sx *) sp"'],
+        )
+
+        self.expect(
+            "frame variable 'sp.x'",
+            error=True,
+            substrs=[
+                "member reference type 'Sx *' is a pointer; did you mean to use '->'"
+            ],
+        )
 
         # Test for record typedefs.
         self.expect_var_path("sa.x", value="3")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py b/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py
index 1bde4706da90f..856b3a069eec8 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py
@@ -27,8 +27,11 @@ def test_frame_var(self):
         self.expect_var_path("a.x", value="1")
         self.expect_var_path("a.y", value="2")
 
-        self.expect("frame variable 'b.x'", error=True,
-                    substrs=["no member named 'x' in 'B'"])
+        self.expect(
+            "frame variable 'b.x'",
+            error=True,
+            substrs=['"x" is not a member of "(B) b"'],
+        )
         #self.expect_var_path("b.y", value="0")
         self.expect_var_path("b.z", value="3")
         self.expect_var_path("b.w", value="4")
@@ -43,19 +46,31 @@ def test_frame_var(self):
         self.expect_var_path("d.z", value="9")
         self.expect_var_path("d.w", value="10")
 
-        self.expect("frame variable 'e.x'", error=True,
-                    substrs=["no member named 'x' in 'E'"])
-        self.expect("frame variable 'f.x'", error=True,
-                    substrs=["no member named 'x' in 'F'"])
+        self.expect(
+            "frame variable 'e.x'",
+            error=True,
+            substrs=['"x" is not a member of "(E) e"'],
+        )
+        self.expect(
+            "frame variable 'f.x'",
+            error=True,
+            substrs=['"x" is not a member of "(F) f"'],
+        )
         self.expect_var_path("f.named_field.x", value="12")
 
         self.expect_var_path("unnamed_derived.y", value="2")
         self.expect_var_path("unnamed_derived.z", value="13")
 
-        self.expect("frame variable 'derb.x'", error=True,
-                    substrs=["no member named 'x' in 'DerivedB'"])
-        self.expect("frame variable 'derb.y'", error=True,
-                    substrs=["no member named 'y' in 'DerivedB'"])
+        self.expect(
+            "frame variable 'derb.x'",
+            error=True,
+            substrs=['"x" is not a member of "(DerivedB) derb"'],
+        )
+        self.expect(
+            "frame variable 'derb.y'",
+            error=True,
+            substrs=['"y" is not a member of "(DerivedB) derb"'],
+        )
         self.expect_var_path("derb.w", value="14")
         self.expect_var_path("derb.k", value="15")
         self.expect_var_path("derb.a.x", value="1")
diff --git a/lldb/test/API/commands/protocol/TestMCPUnixSocket.py b/lldb/test/API/commands/protocol/TestMCPUnixSocket.py
new file mode 100644
index 0000000000000..ea9255cc60ef5
--- /dev/null
+++ b/lldb/test/API/commands/protocol/TestMCPUnixSocket.py
@@ -0,0 +1,34 @@
+import os
+import tempfile
+import unittest
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+
+# To be safe and portable, Unix domain socket paths should be kept at or below
+# 108 characters on Linux, and around 104 characters on macOS:
+MAX_SOCKET_PATH_LENGTH = 104
+
+
+class MCPUnixSocketCommandTestCase(TestBase):
+    @skipIfWindows
+    @skipIfRemote
+    @no_debug_info_test
+    def test_unix_socket(self):
+        """
+        Test if we can start an MCP protocol-server accepting unix sockets
+        """
+
+        temp_directory = tempfile.TemporaryDirectory()
+        socket_file = os.path.join(temp_directory.name, "mcp.sock")
+
+        if len(socket_file) >= MAX_SOCKET_PATH_LENGTH:
+            self.skipTest(
+                f"Socket path {socket_file} exceeds the {MAX_SOCKET_PATH_LENGTH} character limit"
+            )
+
+        self.expect(
+            f"protocol-server start MCP accept://{socket_file}",
+            startstr="MCP server started with connection listeners:",
+            substrs=[f"unix-connect://{socket_file}"],
+        )
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile
index 99b3fb3bd7762..00bbef5df3004 100644
--- a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile
@@ -1,4 +1,4 @@
 include Makefile.rules
 
 a.out:
-	$(CC) -target x86_64-pc-linux-elf -g -gsplit-dwarf -o $@ $(SRCDIR)/main.c $(SRCDIR)/foo.c
+	$(CC) $(CFLAGS) -target x86_64-pc-linux-elf -g -gsplit-dwarf -o $@ $(SRCDIR)/main.c $(SRCDIR)/foo.c
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py
index 13d12e3686a17..d819b5ed9ca87 100644
--- a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py
@@ -24,9 +24,9 @@ def get_dwos_from_json_output(self):
             result[symfile_entry["symfile"]] = dwo_dict
         return result
 
-    def build_and_skip_if_error(self):
+    def build_and_skip_if_error(self, debug_info=None):
         try:
-            self.build()
+            self.build(debug_info=debug_info)
         except BuildError as e:
             self.skipTest(f"Skipping test due to build exception: {e}")
 
@@ -145,6 +145,75 @@ def test_dwos_loaded_symbols_on_demand(self):
         self.runCmd("target modules dump separate-debug-info --json")
 
         # Check the output
+        output = self.get_dwos_from_json_output()
+        self.assertFalse(output[exe]["a.out-main.dwo"]["loaded"])
+        self.assertFalse(output[exe]["a.out-foo.dwo"]["loaded"])
+        # Set a breakpoint in main(). All DWO files should be loaded now
+        self.runCmd("b main")
+        self.runCmd("target modules dump separate-debug-info --json")
+        output = self.get_dwos_from_json_output()
+        self.assertTrue(output[exe]["a.out-main.dwo"]["loaded"])
+        self.assertTrue(output[exe]["a.out-foo.dwo"]["loaded"])
+
+    def test_dwos_load_json_with_debug_names_default(self):
+        """
+        Test that DWO files are lazily loaded, and target module dump gives the expected output.
+        """
+        # Build with split DWARF, debug_names, and gpubnames
+        self.build_and_skip_if_error(debug_info=["debug_names"])
+        exe = self.getBuildArtifact("a.out")
+
+        main_dwo = self.getBuildArtifact("a.out-main.dwo")
+        foo_dwo = self.getBuildArtifact("a.out-foo.dwo")
+
+        # Make sure dwo files exist
+        self.assertTrue(os.path.exists(main_dwo), f'Make sure "{main_dwo}" file exists')
+        self.assertTrue(os.path.exists(foo_dwo), f'Make sure "{foo_dwo}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.runCmd("target modules dump separate-debug-info --j")
+
+        # Check the output
+        output = self.get_dwos_from_json_output()
+        self.assertFalse(output[exe]["a.out-main.dwo"]["loaded"])
+        self.assertFalse(output[exe]["a.out-foo.dwo"]["loaded"])
+
+        # Set a breakpoint in main(). a.out-main.dwo should be loaded now
+        self.runCmd("b main")
+        self.runCmd("target modules dump separate-debug-info --j")
+        output = self.get_dwos_from_json_output()
+        self.assertTrue(output[exe]["a.out-main.dwo"]["loaded"])
+        self.assertFalse(output[exe]["a.out-foo.dwo"]["loaded"])
+
+        # Set a breakpoint in foo(). a.out-foo.dwo should be loaded now
+        self.runCmd("b foo")
+        self.runCmd("target modules dump separate-debug-info --j")
+        output = self.get_dwos_from_json_output()
+        self.assertTrue(output[exe]["a.out-main.dwo"]["loaded"])
+        self.assertTrue(output[exe]["a.out-foo.dwo"]["loaded"])
+
+    def test_dwos_load_json_with_debug_names_force_load_all(self):
+        """
+        Test that DWO files are lazily loaded, and target module dump gives the expected output.
+        """
+        # Build with split DWARF, debug_names, and gpubnames
+        self.build_and_skip_if_error(debug_info=["debug_names"])
+        exe = self.getBuildArtifact("a.out")
+
+        main_dwo = self.getBuildArtifact("a.out-main.dwo")
+        foo_dwo = self.getBuildArtifact("a.out-foo.dwo")
+
+        # Make sure dwo files exist
+        self.assertTrue(os.path.exists(main_dwo), f'Make sure "{main_dwo}" file exists')
+        self.assertTrue(os.path.exists(foo_dwo), f'Make sure "{foo_dwo}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.runCmd("target modules dump separate-debug-info --j --f")
+
         output = self.get_dwos_from_json_output()
         self.assertTrue(output[exe]["a.out-main.dwo"]["loaded"])
         self.assertTrue(output[exe]["a.out-foo.dwo"]["loaded"])
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
index 0ab5dd0f910f2..0ad903befc65e 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
@@ -14,5 +14,15 @@ def supports_hw_breakpoints(self):
         self.runCmd("breakpoint set -b main --hardware")
         self.runCmd("run")
         if "stopped" in self.res.GetOutput():
+            return True
+        return False
+
+    def hw_breakpoints_supported(self):
+        if self.supports_hw_breakpoints():
             return "Hardware breakpoints are supported"
         return None
+
+    def hw_breakpoints_unsupported(self):
+        if not self.supports_hw_breakpoints():
+            return "Hardware breakpoints are unsupported"
+        return None
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
index 1a0515aa04c07..4632c3bed1899 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
@@ -12,16 +12,13 @@
 
 
 class HardwareBreakpointMultiThreadTestCase(HardwareBreakpointTestBase):
-    def does_not_support_hw_breakpoints(self):
-        return not super().supports_hw_breakpoints()
-
-    @skipTestIfFn(does_not_support_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_unsupported)
     def test_hw_break_set_delete_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
         self.break_multi_thread("delete")
 
-    @skipTestIfFn(does_not_support_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_unsupported)
     def test_hw_break_set_disable_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
index 89d57683a8007..a8c9cdeea9362 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
@@ -26,7 +26,7 @@ def test_breakpoint(self):
         breakpoint = target.BreakpointCreateByLocation("main.c", 1)
         self.assertTrue(breakpoint.IsHardware())
 
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_range(self):
         """Test stepping when hardware breakpoints are required."""
         self.build()
@@ -49,7 +49,7 @@ def test_step_range(self):
             "Could not create hardware breakpoint for thread plan", error.GetCString()
         )
 
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_out(self):
         """Test stepping out when hardware breakpoints are required."""
         self.build()
@@ -71,7 +71,7 @@ def test_step_out(self):
             "Could not create hardware breakpoint for thread plan", error.GetCString()
         )
 
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_over(self):
         """Test stepping over when hardware breakpoints are required."""
         self.build()
@@ -91,7 +91,7 @@ def test_step_over(self):
 
     # Was reported to sometimes pass on certain hardware.
     @skipIf(oslist=["linux"], archs=["arm$"])
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_until(self):
         """Test stepping until when hardware breakpoints are required."""
         self.build()
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile
new file mode 100644
index 0000000000000..304633c2dca1f
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile
@@ -0,0 +1,7 @@
+C_SOURCES := main.c
+
+ifeq ($(CC_TYPE), icc)
+    CFLAGS_EXTRAS := -debug inline-debug-info
+endif
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py
new file mode 100644
index 0000000000000..ccbb23507c1be
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py
@@ -0,0 +1,50 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+from functionalities.breakpoint.hardware_breakpoints.base import *
+
+
+class SimpleHWBreakpointTest(HardwareBreakpointTestBase):
+    def does_not_support_hw_breakpoints(self):
+        # FIXME: Use HardwareBreakpointTestBase.supports_hw_breakpoints
+        if super().supports_hw_breakpoints() is None:
+            return "Hardware breakpoints are unsupported"
+        return None
+
+    @skipTestIfFn(does_not_support_hw_breakpoints)
+    def test(self):
+        """Test SBBreakpoint::SetIsHardware"""
+        self.build()
+
+        # Set a breakpoint on main.
+        target, process, _, main_bp = lldbutil.run_to_source_breakpoint(
+            self, "main", lldb.SBFileSpec("main.c")
+        )
+
+        break_on_me_bp = target.BreakpointCreateByLocation("main.c", 1)
+
+        self.assertFalse(main_bp.IsHardware())
+        self.assertFalse(break_on_me_bp.IsHardware())
+        self.assertGreater(break_on_me_bp.GetNumResolvedLocations(), 0)
+
+        error = break_on_me_bp.SetIsHardware(True)
+
+        # Regardless of whether we succeeded in updating all the locations, the
+        # breakpoint will be marked as a hardware breakpoint.
+        self.assertTrue(break_on_me_bp.IsHardware())
+
+        if super().supports_hw_breakpoints():
+            self.assertSuccess(error)
+
+            # Continue to our Hardware breakpoint and verify that's the reason
+            # we're stopped.
+            process.Continue()
+            self.expect(
+                "thread list",
+                STOPPED_DUE_TO_BREAKPOINT,
+                substrs=["stopped", "stop reason = breakpoint"],
+            )
+        else:
+            self.assertFailure(error)
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c
new file mode 100644
index 0000000000000..4f1c9df56200a
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c
@@ -0,0 +1,7 @@
+int break_on_me() {
+  int i = 10;
+  i++;
+  return i;
+}
+
+int main() { return break_on_me(); }
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
index e4e25ae877ded..c82ae24a6d9ab 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
@@ -12,10 +12,9 @@
 
 
 class WriteMemoryWithHWBreakpoint(HardwareBreakpointTestBase):
-    def does_not_support_hw_breakpoints(self):
-        return not super().supports_hw_breakpoints()
 
-    @skipTestIfFn(does_not_support_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skip
     def test_copy_memory_with_hw_break(self):
         self.build()
         exe = self.getBuildArtifact("a.out")
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
index 2c0a89f983994..91b249b677c73 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
@@ -10,10 +10,6 @@
 
 
 class SkipSummaryDataFormatterTestCase(TestBase):
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr24462, Data formatters have problems on Windows",
-    )
     def test_with_run_command(self):
         """Test data formatter commands."""
         self.build()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile
similarity index 75%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile
index d87cf7d402787..99998b20bcb05 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile
@@ -1,4 +1,3 @@
 CXX_SOURCES := main.cpp
-USE_LIBCPP := 1
-include Makefile.rules
 
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/TestDataFormatterStdAtomic.py
similarity index 89%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/TestDataFormatterStdAtomic.py
index c6592ede03147..8186e1d66985b 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/TestDataFormatterStdAtomic.py
@@ -9,18 +9,15 @@
 from lldbsuite.test import lldbutil
 
 
-class LibCxxAtomicTestCase(TestBase):
+class StdAtomicTestCase(TestBase):
     def get_variable(self, name):
         var = self.frame().FindVariable(name)
         var.SetPreferDynamicValue(lldb.eDynamicCanRunTarget)
         var.SetPreferSyntheticValue(True)
         return var
 
-    @skipIf(compiler=["gcc"])
-    @add_test_categories(["libc++"])
-    def test(self):
-        """Test that std::atomic as defined by libc++ is correctly printed by LLDB"""
-        self.build()
+    def do_test(self):
+        """Test that std::atomic is correctly printed by LLDB"""
         self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
 
         bkpt = self.target().FindBreakpointByID(
@@ -31,8 +28,6 @@ def test(self):
 
         self.runCmd("run", RUN_SUCCEEDED)
 
-        lldbutil.skip_if_library_missing(self, self.target(), re.compile(r"libc\+\+"))
-
         # The stop reason of the thread should be breakpoint.
         self.expect(
             "thread list",
@@ -66,3 +61,9 @@ def test(self):
         self.expect(
             "frame var p.child.parent", substrs=["p.child.parent = {\n  Value = 0x"]
         )
+
+    @skipIf(compiler=["gcc"])
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/main.cpp
similarity index 51%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/main.cpp
index 760d2384a772b..ee77a880a0c5d 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/main.cpp
@@ -6,28 +6,26 @@ struct Child {
   // This should point to the parent which in turn owns this
   // child instance. This cycle should not cause LLDB to infinite loop
   // during printing.
-  std::atomic<Parent*> parent{nullptr};
+  std::atomic<Parent *> parent{nullptr};
 };
 struct Parent {
   Child child;
 };
 
 struct S {
-    int x = 1;
-    int y = 2;
+  int x = 1;
+  int y = 2;
 };
 
-int main ()
-{
-    std::atomic<S> s;
-    s.store(S());
-    std::atomic<int> i;
-    i.store(5);
+int main() {
+  std::atomic<S> s;
+  s.store(S());
+  std::atomic<int> i;
+  i.store(5);
 
-    Parent p;
-    // Let the child node know what its parent is.
-    p.child.parent = &p;
+  Parent p;
+  // Let the child node know what its parent is.
+  p.child.parent = &p;
 
-    return 0; // Set break point at this line.
+  return 0; // Set break point at this line.
 }
-
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/Makefile
similarity index 82%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/Makefile
index 57540fc1a2816..4f79c0a900c3a 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/Makefile
@@ -1,6 +1,4 @@
 CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
 CXXFLAGS_EXTRAS := -std=c++20
+
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/TestDataFormatterStdChrono.py
similarity index 98%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/TestDataFormatterStdChrono.py
index 0737a5bc7e6eb..013a1787bc4bf 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/TestDataFormatterStdChrono.py
@@ -2,20 +2,16 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibcxxChronoDataFormatterTestCase(TestBase):
-    @add_test_categories(["libc++"])
-    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
-    def test_with_run_command(self):
+class StdChronoDataFormatterTestCase(TestBase):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
         isNotWindowsHost = lldbplatformutil.getHostPlatform() != "windows"
-        self.build()
         (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.cpp", False)
         )
@@ -433,3 +429,9 @@ def test_with_run_command(self):
                 "ymwdl_2024_last_tuesday_january = year=2024 month=January weekday=Tuesday index=last"
             ],
         )
+
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/main.cpp
similarity index 100%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/main.cpp
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py
index abe2a2b7895de..f52f6f7ec3ffb 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py
@@ -21,9 +21,44 @@ def check_size(self, var_name, size):
         var = self.findVariable(var_name)
         self.assertEqual(var.GetNumChildren(), size)
 
+    def check_numbers(self, var_name, show_ptr=False):
+        patterns = []
+        substrs = [
+            "[0] = 1",
+            "[1] = 12",
+            "[2] = 123",
+            "[3] = 1234",
+            "[4] = 12345",
+            "[5] = 123456",
+            "[6] = 1234567",
+            "}",
+        ]
+        if show_ptr:
+            patterns = [var_name + " = 0x.* size=7"]
+        else:
+            substrs.insert(0, var_name + " = size=7")
+        self.expect(
+            "frame variable " + var_name,
+            patterns=patterns,
+            substrs=substrs,
+        )
+        self.expect_expr(
+            var_name,
+            result_summary="size=7",
+            result_children=[
+                ValueCheck(value="1"),
+                ValueCheck(value="12"),
+                ValueCheck(value="123"),
+                ValueCheck(value="1234"),
+                ValueCheck(value="12345"),
+                ValueCheck(value="123456"),
+                ValueCheck(value="1234567"),
+            ],
+        )
+
     def do_test(self, stdlib_type):
         self.build(dictionary={stdlib_type: "1"})
-        lldbutil.run_to_source_breakpoint(
+        (_, process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.cpp")
         )
 
@@ -83,6 +118,21 @@ def do_test(self, stdlib_type):
                 ],
             )
 
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        # first value added
+        self.expect("frame variable empty", substrs=["empty = size=1", "[0] = 1", "}"])
+
+        # add remaining values
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        self.check_numbers("empty")
+
+        # clear out the deque
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        self.expect_expr("empty", result_children=[])
+
     @add_test_categories(["libstdcxx"])
     def test_libstdcpp(self):
         self.do_test(USE_LIBSTDCPP)
@@ -90,3 +140,25 @@ def test_libstdcpp(self):
     @add_test_categories(["libc++"])
     def test_libcpp(self):
         self.do_test(USE_LIBCPP)
+
+    def do_test_ref_and_ptr(self, stdlib_type: str):
+        """Test formatting of std::deque& and std::deque*"""
+        self.build(dictionary={stdlib_type: "1"})
+        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "stop here", lldb.SBFileSpec("main.cpp", False)
+        )
+
+        # The reference should display the same was as the value did
+        self.check_numbers("ref", True)
+
+        # The pointer should just show the right number of elements:
+        self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
+        self.expect("expression ptr", substrs=["$", "size=7"])
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcpp_ref_and_ptr(self):
+        self.do_test_ref_and_ptr(USE_LIBSTDCPP)
+
+    @add_test_categories(["libc++"])
+    def test_libcpp_ref_and_ptr(self):
+        self.do_test_ref_and_ptr(USE_LIBCPP)
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp
index b948fe1b4b375..a1c5b443daae6 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp
@@ -26,6 +26,11 @@ template <typename T> T fill(T deque) {
   return deque;
 }
 
+void by_ref_and_ptr(std::deque<int> &ref, std::deque<int> *ptr) {
+  puts("stop here");
+  return;
+}
+
 int main() {
   std::deque<int> empty;
   std::deque<int> deque_1 = {1};
@@ -37,5 +42,23 @@ int main() {
   std::deque<Foo_large> deque_200_large;
   deque_200_large = fill<std::deque<Foo_large>>(deque_200_large);
 
-  return empty.size() + deque_1.front() + deque_3.front(); // break here
+  puts("break here");
+
+  empty.push_back(1);
+  puts("break here");
+
+  (empty.push_back(12));
+  (empty.push_back(123));
+  (empty.push_back(1234));
+  (empty.push_back(12345));
+  (empty.push_back(123456));
+  (empty.push_back(1234567));
+  puts("break here");
+
+  by_ref_and_ptr(empty, &empty);
+
+  empty.clear();
+  puts("break here");
+
+  return empty.size() + deque_1.front() + deque_3.front();
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile
similarity index 75%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile
index c5ac05605bdcb..99998b20bcb05 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile
@@ -1,4 +1,3 @@
-USE_LIBCPP := 1
 CXX_SOURCES := main.cpp
 
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/TestDataFormatterStdMap.py
similarity index 94%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/TestDataFormatterStdMap.py
index b2b83a3b46114..5851588b59b5f 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/TestDataFormatterStdMap.py
@@ -2,14 +2,13 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibcxxMapDataFormatterTestCase(TestBase):
+class StdMapDataFormatterTestCase(TestBase):
     def setUp(self):
         TestBase.setUp(self)
         ns = "ndk" if lldbplatformutil.target_is_android() else ""
@@ -22,10 +21,8 @@ def check_pair(self, first_value, second_value):
         ]
         return ValueCheck(children=pair_children)
 
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
         self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
 
         bkpt = self.target().FindBreakpointByID(
@@ -326,3 +323,23 @@ def cleanup():
         lldbutil.continue_to_breakpoint(self.process(), bkpt)
 
         self.expect("frame variable ss", substrs=["%s::map" % ns, "size=0", "{}"])
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @expectedFailureAll(
+        bugnumber="Don't support formatting __gnu_debug::_Safe_iterator yet"
+    )
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp
new file mode 100644
index 0000000000000..91bdf0b58d9e4
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp
@@ -0,0 +1,80 @@
+#include <map>
+#include <string>
+
+#define intint_map std::map<int, int>
+#define strint_map std::map<std::string, int>
+#define intstr_map std::map<int, std::string>
+#define strstr_map std::map<std::string, std::string>
+
+int g_the_foo = 0;
+
+int thefoo_rw(int arg = 1) {
+  if (arg < 0)
+    arg = 0;
+  if (!arg)
+    arg = 1;
+  g_the_foo += arg;
+  return g_the_foo;
+}
+
+int main() {
+  intint_map ii;
+
+  ii[0] = 0; // Set break point at this line.
+  ii[1] = 1;
+
+  intint_map::iterator it = ii.begin();
+  intint_map::const_iterator const_it = ii.cbegin();
+  std::printf("%d %d\n", it->second, const_it->second);
+
+  thefoo_rw(1); // Set break point at this line.
+  ii[2] = 0;
+  ii[3] = 1;
+  thefoo_rw(1); // Set break point at this line.
+  ii[4] = 0;
+  ii[5] = 1;
+  ii[6] = 0;
+  ii[7] = 1;
+  thefoo_rw(1); // Set break point at this line.
+  ii[85] = 1234567;
+
+  ii.clear();
+
+  strint_map si;
+  thefoo_rw(1); // Set break point at this line.
+
+  si["zero"] = 0;
+  thefoo_rw(1); // Set break point at this line.
+  si["one"] = 1;
+  si["two"] = 2;
+  si["three"] = 3;
+  thefoo_rw(1); // Set break point at this line.
+  si["four"] = 4;
+
+  si.clear();
+  thefoo_rw(1); // Set break point at this line.
+
+  intstr_map is;
+  thefoo_rw(1); // Set break point at this line.
+  is[85] = "goofy";
+  is[1] = "is";
+  is[2] = "smart";
+  is[3] = "!!!";
+  thefoo_rw(1); // Set break point at this line.
+
+  is.clear();
+  thefoo_rw(1); // Set break point at this line.
+
+  strstr_map ss;
+  thefoo_rw(1); // Set break point at this line.
+
+  ss["ciao"] = "hello";
+  ss["casa"] = "house";
+  ss["gatto"] = "cat";
+  thefoo_rw(1); // Set break point at this line.
+  ss["a Mac.."] = "..is always a Mac!";
+
+  ss.clear();
+  thefoo_rw(1); // Set break point at this line.
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/Makefile
similarity index 75%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/Makefile
index 680e1abfbef58..99998b20bcb05 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/Makefile
@@ -1,4 +1,3 @@
 CXX_SOURCES := main.cpp
 
-USE_LIBCPP := 1
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/TestDataFormatterStdQueue.py
similarity index 90%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/TestDataFormatterStdQueue.py
index 53eaf5a23cc07..ac729c5bacb0f 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/TestDataFormatterStdQueue.py
@@ -2,14 +2,13 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class TestDataFormatterLibcxxQueue(TestBase):
+class TestDataFormatterStdQueue(TestBase):
     def setUp(self):
         TestBase.setUp(self)
         self.namespace = "std"
@@ -30,9 +29,9 @@ def check_variable(self, name):
         bugnumber="llvm.org/pr36109", debug_info="gmodules", triple=".*-android"
     )
     @add_test_categories(["libc++"])
-    def test(self):
+    def test_libcxx(self):
         """Test that std::queue is displayed correctly"""
-        self.build()
+        self.build(dictionary={"USE_LIBCPP": 1})
         lldbutil.run_to_source_breakpoint(
             self, "// break here", lldb.SBFileSpec("main.cpp", False)
         )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/main.cpp
similarity index 54%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/main.cpp
index f3b5f32817970..406b437914932 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/queue/main.cpp
@@ -2,8 +2,8 @@
 #include <vector>
 
 int main() {
-  std::queue<int> q1{{1,2,3,4,5}};
-  std::queue<int, std::vector<int>> q2{{1,2,3,4,5}};
+  std::queue<int> q1{{1, 2, 3, 4, 5}};
+  std::queue<int, std::vector<int>> q2{{1, 2, 3, 4, 5}};
   int ret = q1.size() + q2.size(); // break here
   return ret;
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py
index 03e18ff4335e5..d3d6ef55c6ff0 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py
@@ -137,7 +137,7 @@ def test_with_run_command_libcpp(self):
 
     def do_test_ref_and_ptr(self, stdlib_type):
         """Test that the data formatters work on ref and ptr."""
-        self.build()
+        self.build(dictionary={stdlib_type: "1"})
         (self.target, process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self,
             "Stop here to check by ref and ptr.",
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/TestDataFormatterStdSharedPtr.py
similarity index 59%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/TestDataFormatterStdSharedPtr.py
index 61c050b3bfa01..3d8569da0332e 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/TestDataFormatterStdSharedPtr.py
@@ -1,8 +1,7 @@
 """
-Test lldb data formatter for libc++ std::shared_ptr.
+Test lldb data formatter for std::shared_ptr.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -10,12 +9,9 @@
 
 
 class TestCase(TestBase):
-    @add_test_categories(["libc++"])
-    def test_shared_ptr_variables(self):
+    def do_test(self):
         """Test `frame variable` output for `std::shared_ptr` types."""
-        self.build()
-
-        lldbutil.run_to_source_breakpoint(
+        (_, process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "// break here", lldb.SBFileSpec("main.cpp")
         )
 
@@ -23,7 +19,7 @@ def test_shared_ptr_variables(self):
             "sp_empty",
             type="std::shared_ptr<int>",
             summary="nullptr",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
         self.assertEqual(
             valobj.child[0].GetValueAsUnsigned(lldb.LLDB_INVALID_ADDRESS), 0
@@ -36,45 +32,37 @@ def test_shared_ptr_variables(self):
         valobj = self.expect_var_path(
             "sp_int",
             type="std::shared_ptr<int>",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
-        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=1$")
+        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=0$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
         valobj = self.expect_var_path(
             "sp_int_ref",
             type="std::shared_ptr<int> &",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
-        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=1$")
+        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=0$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
         valobj = self.expect_var_path(
             "sp_int_ref_ref",
             type="std::shared_ptr<int> &&",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
-        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=1$")
+        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=0$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
-        if self.expectedCompiler(["clang"]) and self.expectedCompilerVersion(
-            [">", "16.0"]
-        ):
-            string_type = "std::string"
-        else:
-            string_type = "std::basic_string<char, std::char_traits<char>, std::allocator<char> > "
-
         valobj = self.expect_var_path(
             "sp_str",
-            type="std::shared_ptr<" + string_type + ">",
-            children=[ValueCheck(name="__ptr_", summary='"hello"')],
+            children=[ValueCheck(name="pointer", summary='"hello"')],
         )
-        self.assertRegex(valobj.summary, r'^"hello"( strong=1)? weak=1$')
+        self.assertRegex(valobj.summary, r'^"hello"( strong=1)? weak=0$')
 
         valobj = self.expect_var_path("sp_user", type="std::shared_ptr<User>")
         self.assertRegex(
             valobj.summary,
-            "^std(::__[^:]*)?::shared_ptr<User>::element_type @ 0x0*[1-9a-f][0-9a-f]+( strong=1)? weak=1",
+            "element_type @ 0x0*[1-9a-f][0-9a-f]+( strong=1)? weak=0",
         )
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
@@ -86,13 +74,47 @@ def test_shared_ptr_variables(self):
                 ValueCheck(name="name", summary='"steph"'),
             ],
         )
-        self.assertEqual(str(valobj), '(User) *__ptr_ = (id = 30, name = "steph")')
+        self.assertEqual(str(valobj), '(User) *pointer = (id = 30, name = "steph")')
 
         self.expect_var_path("sp_user->id", type="int", value="30")
         self.expect_var_path("sp_user->name", type="std::string", summary='"steph"')
 
+        valobj = self.expect_var_path(
+            "si", type="std::shared_ptr<int>", summary="47 strong=2 weak=0"
+        )
+
+        valobj = self.expect_var_path(
+            "sie", type="std::shared_ptr<int>", summary="nullptr strong=2 weak=0"
+        )
+
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        valobj = self.expect_var_path(
+            "si", type="std::shared_ptr<int>", summary="47 strong=2 weak=2"
+        )
+        valobj = self.expect_var_path(
+            "sie", type="std::shared_ptr<int>", summary="nullptr strong=2 weak=2"
+        )
+        valobj = self.expect_var_path(
+            "wie", type="std::weak_ptr<int>", summary="nullptr strong=2 weak=2"
+        )
+
+        self.expect_var_path("si.pointer", type="int *")
+        self.expect_var_path("*si.pointer", type="int", value="47")
+        self.expect_var_path("si.object", type="int", value="47")
+
         self.runCmd("settings set target.experimental.use-DIL true")
         self.expect_var_path("ptr_node->value", value="1")
         self.expect_var_path("ptr_node->next->value", value="2")
         self.expect_var_path("(*ptr_node).value", value="1")
         self.expect_var_path("(*(*ptr_node).next).value", value="2")
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp
similarity index 69%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp
index 84a02ec13df1c..cfefaf37a5dbd 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp
@@ -1,3 +1,4 @@
+#include <cstdio>
 #include <memory>
 #include <string>
 
@@ -22,5 +23,16 @@ int main() {
       std::shared_ptr<NodeS>(new NodeS{nullptr, 2});
   ptr_node = std::shared_ptr<NodeS>(new NodeS{std::move(ptr_node), 1});
 
-  return 0; // break here
+  // Construct empty shared_ptr with non-null control field.
+  std::shared_ptr<int> si(new int(47));
+  std::shared_ptr<int> sie(si, nullptr);
+
+  std::puts("// break here");
+
+  std::weak_ptr<int> wie = sie;
+  std::weak_ptr<int> wie2 = sie;
+
+  std::puts("// break here");
+
+  return 0;
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py
similarity index 52%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py
index 32764629d65a7..7ae5687af2cf4 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py
@@ -10,7 +10,7 @@
 from lldbsuite.test import lldbutil
 
 
-class LibcxxStringDataFormatterTestCase(TestBase):
+class StdStringDataFormatterTestCase(TestBase):
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -18,17 +18,8 @@ def setUp(self):
         self.main_spec = lldb.SBFileSpec("main.cpp")
         self.namespace = "std"
 
-    @add_test_categories(["libc++"])
-    @expectedFailureAll(
-        bugnumber="llvm.org/pr36109", debug_info="gmodules", triple=".*-android"
-    )
-    # Inline namespace is randomly ignored as Clang due to broken lookup inside
-    # the std namespace.
-    @expectedFailureAll(debug_info="gmodules")
-    def test_with_run_command(self):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
-
         (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "Set break point at this line.", self.main_spec
         )
@@ -42,49 +33,18 @@ def cleanup():
             self.runCmd("type filter clear", check=False)
             self.runCmd("type synth clear", check=False)
 
-        is_64_bit = self.process().GetAddressByteSize() == 8
-
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
 
         ns = self.namespace
 
-        self.expect(
-            "frame variable",
-            substrs=[
-                '(%s::wstring) wempty = L""' % ns,
-                '(%s::wstring) s = L"hello world! מזל טוב!"' % ns,
-                '(%s::wstring) S = L"!!!!"' % ns,
-                "(const wchar_t *) mazeltov = 0x",
-                'L"מזל טוב"',
-                '(%s::string) empty = ""' % ns,
-                '(%s::string) q = "hello world"' % ns,
-                '(%s::string) Q = "quite a long std::strin with lots of info inside it"'
-                % ns,
-                '(%s::string) IHaveEmbeddedZeros = "a\\0b\\0c\\0d"' % ns,
-                '(%s::wstring) IHaveEmbeddedZerosToo = L"hello world!\\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監"'
-                % ns,
-                '(%s::u16string) u16_string = u"ß水氶"' % ns,
-                '(%s::u16string) u16_empty = u""' % ns,
-                '(%s::u32string) u32_string = U"🍄🍅🍆🍌"' % ns,
-                '(%s::u32string) u32_empty = U""' % ns,
-                "(%s::string *) null_str = nullptr" % ns,
-            ],
-        )
+        # Check 'S' pre-assignment.
+        self.expect("frame variable S", substrs=['(%s::wstring) S = L"!!!!"' % ns])
 
         thread.StepOver()
 
         TheVeryLongOne = frame.FindVariable("TheVeryLongOne")
         summaryOptions = lldb.SBTypeSummaryOptions()
-        summaryOptions.SetCapping(lldb.eTypeSummaryUncapped)
-        uncappedSummaryStream = lldb.SBStream()
-        TheVeryLongOne.GetSummary(uncappedSummaryStream, summaryOptions)
-        uncappedSummary = uncappedSummaryStream.GetData()
-        self.assertGreater(
-            uncappedSummary.find("someText"),
-            0,
-            "uncappedSummary does not include the full string",
-        )
         summaryOptions.SetCapping(lldb.eTypeSummaryCapped)
         cappedSummaryStream = lldb.SBStream()
         TheVeryLongOne.GetSummary(cappedSummaryStream, summaryOptions)
@@ -110,50 +70,145 @@ def cleanup():
         self.expect(
             "frame variable",
             substrs=[
+                '(%s::wstring) wempty = L""' % ns,
+                '(%s::wstring) s = L"hello world! מזל טוב!"' % ns,
                 '(%s::wstring) S = L"!!!!!"' % ns,
                 "(const wchar_t *) mazeltov = 0x",
                 'L"מזל טוב"',
+                '(%s::string) empty = ""' % ns,
                 '(%s::string) q = "hello world"' % ns,
                 '(%s::string) Q = "quite a long std::strin with lots of info inside it"'
                 % ns,
+                "(%s::string *) null_str = nullptr" % ns,
+            ],
+        )
+
+        # Test references and pointers to std::string.
+        var_rq = frame.FindVariable("rq")
+        var_rQ = frame.FindVariable("rQ")
+        var_pq = frame.FindVariable("pq")
+        var_pQ = frame.FindVariable("pQ")
+
+        self.assertEqual(var_rq.GetSummary(), '"hello world"', "rq summary wrong")
+        self.assertEqual(
+            var_rQ.GetSummary(),
+            '"quite a long std::strin with lots of info inside it"',
+            "rQ summary wrong",
+        )
+        self.assertEqual(var_pq.GetSummary(), '"hello world"', "pq summary wrong")
+        self.assertEqual(
+            var_pQ.GetSummary(),
+            '"quite a long std::strin with lots of info inside it"',
+            "pQ summary wrong",
+        )
+
+    @expectedFailureAll(
+        bugnumber="llvm.org/pr36109", debug_info="gmodules", triple=".*-android"
+    )
+    # Inline namespace is randomly ignored as Clang due to broken lookup inside
+    # the std namespace.
+    @expectedFailureAll(debug_info="gmodules")
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @expectedFailureAll(
+        bugnumber="llvm.org/pr36109", debug_info="gmodules", triple=".*-android"
+    )
+    # Inline namespace is randomly ignored as Clang due to broken lookup inside
+    # the std namespace.
+    @expectedFailureAll(debug_info="gmodules")
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    def do_test_multibyte(self):
+        lldbutil.run_to_source_breakpoint(
+            self, "Set break point at this line.", self.main_spec
+        )
+
+        ns = self.namespace
+
+        self.expect(
+            "frame variable",
+            substrs=[
                 '(%s::string) IHaveEmbeddedZeros = "a\\0b\\0c\\0d"' % ns,
                 '(%s::wstring) IHaveEmbeddedZerosToo = L"hello world!\\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監"'
                 % ns,
                 '(%s::u16string) u16_string = u"ß水氶"' % ns,
+                '(%s::u16string) u16_empty = u""' % ns,
                 '(%s::u32string) u32_string = U"🍄🍅🍆🍌"' % ns,
                 '(%s::u32string) u32_empty = U""' % ns,
-                "(%s::string *) null_str = nullptr" % ns,
             ],
         )
 
-        # The test assumes that std::string is in its cap-size-data layout.
-        is_alternate_layout = (
-            "arm" in self.getArchitecture()
-        ) and self.platformIsDarwin()
-        if is_64_bit and not is_alternate_layout:
-            self.expect(
-                "frame variable garbage1", substrs=["garbage1 = Summary Unavailable"]
-            )
-            self.expect(
-                "frame variable garbage2", substrs=[r'garbage2 = "\xfa\xfa\xfa\xfa"']
-            )
-            self.expect("frame variable garbage3", substrs=[r'garbage3 = "\xf0\xf0"'])
-            self.expect(
-                "frame variable garbage4", substrs=["garbage4 = Summary Unavailable"]
-            )
-            self.expect(
-                "frame variable garbage5", substrs=["garbage5 = Summary Unavailable"]
-            )
-
-        # Finally, make sure that if the string is not readable, we give an error:
-        bkpt_2 = target.BreakpointCreateBySourceRegex(
-            "Break here to look at bad string", self.main_spec
+    @add_test_categories(["libc++"])
+    def test_multibyte_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_multibyte()
+
+    @expectedFailureAll(
+        bugnumber="libstdc++ formatters don't support UTF-16/UTF-32 strings yet."
+    )
+    @add_test_categories(["libstdcxx"])
+    def test_multibyte_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_multibyte()
+
+    def do_test_uncapped_summary(self):
+        (_, _, thread, _) = lldbutil.run_to_source_breakpoint(
+            self, "Set break point at this line.", self.main_spec
+        )
+
+        TheVeryLongOne = thread.frames[0].FindVariable("TheVeryLongOne")
+        summaryOptions = lldb.SBTypeSummaryOptions()
+        summaryOptions.SetCapping(lldb.eTypeSummaryUncapped)
+        uncappedSummaryStream = lldb.SBStream()
+        TheVeryLongOne.GetSummary(uncappedSummaryStream, summaryOptions)
+        uncappedSummary = uncappedSummaryStream.GetData()
+        self.assertGreater(
+            uncappedSummary.find("someText"),
+            0,
+            "uncappedSummary does not include the full string",
         )
-        self.assertEqual(bkpt_2.GetNumLocations(), 1, "Got one location")
-        threads = lldbutil.continue_to_breakpoint(process, bkpt_2)
-        self.assertEqual(len(threads), 1, "Stopped at second breakpoint")
-        frame = threads[0].frames[0]
-        var = frame.FindVariable("in_str")
-        self.assertTrue(var.GetError().Success(), "Made variable")
+
+    @add_test_categories(["libc++"])
+    def test_uncapped_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_uncapped_summary()
+
+    @expectedFailureAll(
+        bugnumber="libstdc++ std::string summary provider doesn't obey summary options."
+    )
+    @add_test_categories(["libstdcxx"])
+    def test_uncapped_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_uncapped_summary()
+
+    def do_test_summary_unavailable(self):
+        """
+        Make sure that if the string is not readable, we give an error.
+        """
+        (_, _, thread, _) = lldbutil.run_to_source_breakpoint(
+            self, "Break here to look at bad string", self.main_spec
+        )
+
+        var = thread.frames[0].FindVariable("in_str")
+        self.assertTrue(var.GetError().Success(), "Found variable")
         summary = var.GetSummary()
         self.assertEqual(summary, "Summary Unavailable", "No summary for bad value")
+
+    @add_test_categories(["libc++"])
+    def test_unavailable_summary_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_summary_unavailable()
+
+    @expectedFailureAll(
+        bugnumber="libstdc++ std::string summary provider doesn't output a user-friendly message for invalid strings."
+    )
+    @add_test_categories(["libstdcxx"])
+    def test_unavailable_summary_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_summary_unavailable()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/main.cpp
new file mode 100644
index 0000000000000..db695ed06f015
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/main.cpp
@@ -0,0 +1,104 @@
+#include <stdint.h>
+#include <string>
+
+size_t touch_string(std::string &in_str) {
+  return in_str.size(); // Break here to look at bad string
+}
+
+int main() {
+  std::wstring wempty(L"");
+  std::wstring s(L"hello world! מזל טוב!");
+  std::wstring S(L"!!!!");
+  const wchar_t *mazeltov = L"מזל טוב";
+  std::string empty("");
+  std::string q("hello world");
+  std::string Q("quite a long std::strin with lots of info inside it");
+  std::string TheVeryLongOne(
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "9012345678901234567890123456789012345678901234567890someText123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "123456789012345678901234567890123456789012345678901234567890123456789012"
+      "345678901234567890123456789012345678901234567890123456789012345678901234"
+      "567890123456789012345678901234567890123456789012345678901234567890123456"
+      "789012345678901234567890123456789012345678901234567890123456789012345678"
+      "901234567890123456789012345678901234567890123456789012345678901234567890"
+      "1234567890123456789012345678901234567890");
+  std::string IHaveEmbeddedZeros("a\0b\0c\0d", 7);
+  std::wstring IHaveEmbeddedZerosToo(
+      L"hello world!\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監", 38);
+  std::u16string u16_string(u"ß水氶");
+  std::u16string u16_empty(u"");
+  std::u32string u32_string(U"🍄🍅🍆🍌");
+  std::u32string u32_empty(U"");
+  std::string *null_str = nullptr;
+  auto &rq = q;
+  auto &rQ = Q;
+  std::string *pq = &q;
+  std::string *pQ = &Q;
+
+  S.assign(L"!!!!!"); // Set break point at this line.
+  std::string *not_a_string = (std::string *)0x0;
+  touch_string(*not_a_string);
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/TestDataFormatterStdTuple.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/TestDataFormatterStdTuple.py
new file mode 100644
index 0000000000000..ade502e12b928
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/TestDataFormatterStdTuple.py
@@ -0,0 +1,76 @@
+"""
+Test lldb data formatter subsystem.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestDataFormatterStdTuple(TestBase):
+    def setUp(self):
+        TestBase.setUp(self)
+        self.line = line_number("main.cpp", "// break here")
+        self.namespace = "std"
+
+    def do_test(self):
+        """Test that std::tuple is displayed correctly"""
+        lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp", False)
+        )
+
+        tuple_name = self.namespace + "::tuple"
+        self.expect("frame variable empty", substrs=[tuple_name, "size=0", "{}"])
+
+        self.expect(
+            "frame variable one_elt",
+            substrs=[tuple_name, "size=1", "{", "[0] = 47", "}"],
+        )
+
+        self.expect(
+            "frame variable three_elts",
+            substrs=[
+                tuple_name,
+                "size=3",
+                "{",
+                "[0] = 1",
+                "[1] = 47",
+                '[2] = "foo"',
+                "}",
+            ],
+        )
+
+        frame = self.frame()
+        self.assertTrue(frame.IsValid())
+
+        self.assertEqual(
+            47, frame.GetValueForVariablePath("one_elt[0]").GetValueAsUnsigned()
+        )
+        self.assertFalse(frame.GetValueForVariablePath("one_elt[1]").IsValid())
+
+        self.assertEqual(
+            '"foobar"', frame.GetValueForVariablePath("string_elt[0]").GetSummary()
+        )
+        self.assertFalse(frame.GetValueForVariablePath("string_elt[1]").IsValid())
+
+        self.assertEqual(
+            1, frame.GetValueForVariablePath("three_elts[0]").GetValueAsUnsigned()
+        )
+        self.assertEqual(
+            47, frame.GetValueForVariablePath("three_elts[1]").GetValueAsUnsigned()
+        )
+        self.assertEqual(
+            '"foo"', frame.GetValueForVariablePath("three_elts[2]").GetSummary()
+        )
+        self.assertFalse(frame.GetValueForVariablePath("three_elts[3]").IsValid())
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/main.cpp
similarity index 80%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/main.cpp
index beb44cd960005..d49dbe8a5f1af 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/tuple/main.cpp
@@ -1,9 +1,10 @@
-#include <tuple>
 #include <string>
+#include <tuple>
 
 int main() {
   std::tuple<> empty;
   std::tuple<int> one_elt{47};
+  std::tuple<std::string> string_elt{"foobar"};
   std::tuple<int, long, std::string> three_elts{1, 47l, "foo"};
   return 0; // break here
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile
new file mode 100644
index 0000000000000..a27336ffd9acd
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile
@@ -0,0 +1,4 @@
+CXX_SOURCES := main.cpp
+CXXFLAGS_EXTRAS := -std=c++14
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py
similarity index 55%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py
index 25a1cd82a4baa..1b204c7d6ef02 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py
@@ -1,8 +1,7 @@
 """
-Test lldb data formatter for libc++ std::unique_ptr.
+Test lldb data formatter for std::unique_ptr.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -10,32 +9,8 @@
 
 
 class TestCase(TestBase):
-    def make_expected_type(self, pointee_type: str, qualifiers: str = "") -> str:
-        if qualifiers:
-            qualifiers = " " + qualifiers
-
-        if self.expectedCompiler(["clang"]) and self.expectedCompilerVersion(
-            [">", "16.0"]
-        ):
-            return f"std::unique_ptr<{pointee_type}>{qualifiers}"
-        else:
-            return f"std::unique_ptr<{pointee_type}, std::default_delete<{pointee_type}> >{qualifiers}"
-
-    def make_expected_basic_string_ptr(self) -> str:
-        if self.expectedCompiler(["clang"]) and self.expectedCompilerVersion(
-            [">", "16.0"]
-        ):
-            return f"std::unique_ptr<std::string>"
-        else:
-            return (
-                "std::unique_ptr<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, "
-                "std::default_delete<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >"
-            )
-
-    @add_test_categories(["libc++"])
-    def test_unique_ptr_variables(self):
+    def do_test(self):
         """Test `frame variable` output for `std::unique_ptr` types."""
-        self.build()
 
         lldbutil.run_to_source_breakpoint(
             self, "// break here", lldb.SBFileSpec("main.cpp")
@@ -43,7 +18,6 @@ def test_unique_ptr_variables(self):
 
         valobj = self.expect_var_path(
             "up_empty",
-            type=self.make_expected_type("int"),
             summary="nullptr",
             children=[ValueCheck(name="pointer")],
         )
@@ -57,7 +31,6 @@ def test_unique_ptr_variables(self):
 
         valobj = self.expect_var_path(
             "up_int",
-            type=self.make_expected_type("int"),
             summary="10",
             children=[ValueCheck(name="pointer")],
         )
@@ -65,7 +38,6 @@ def test_unique_ptr_variables(self):
 
         valobj = self.expect_var_path(
             "up_int_ref",
-            type=self.make_expected_type("int", qualifiers="&"),
             summary="10",
             children=[ValueCheck(name="pointer")],
         )
@@ -73,7 +45,6 @@ def test_unique_ptr_variables(self):
 
         valobj = self.expect_var_path(
             "up_int_ref_ref",
-            type=self.make_expected_type("int", qualifiers="&&"),
             summary="10",
             children=[ValueCheck(name="pointer")],
         )
@@ -81,12 +52,11 @@ def test_unique_ptr_variables(self):
 
         valobj = self.expect_var_path(
             "up_str",
-            type=self.make_expected_basic_string_ptr(),
             summary='"hello"',
             children=[ValueCheck(name="pointer", summary='"hello"')],
         )
 
-        valobj = self.expect_var_path("up_user", type=self.make_expected_type("User"))
+        valobj = self.expect_var_path("up_user")
         self.assertRegex(valobj.summary, "^User @ 0x0*[1-9a-f][0-9a-f]+$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
@@ -121,3 +91,67 @@ def test_unique_ptr_variables(self):
         self.expect_var_path("ptr_node->next->value", value="2")
         self.expect_var_path("(*ptr_node).value", value="1")
         self.expect_var_path("(*(*ptr_node).next).value", value="2")
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    def do_test_recursive_unique_ptr(self):
+        # Tests that LLDB can handle when we have a loop in the unique_ptr
+        # reference chain and that it correctly handles the different options
+        # for the frame variable command in this case.
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+
+        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
+        self.runCmd("run", RUN_SUCCEEDED)
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        self.expect("frame variable f1->next", substrs=["next = NodeU @"])
+        self.expect(
+            "frame variable --ptr-depth=1 f1->next",
+            substrs=["next = NodeU @", "value = 2"],
+        )
+        self.expect(
+            "frame variable --ptr-depth=2 f1->next",
+            substrs=["next = NodeU @", "value = 1", "value = 2"],
+        )
+
+        frame = self.frame()
+        self.assertTrue(frame.IsValid())
+        self.assertEqual(
+            2,
+            frame.GetValueForVariablePath("f1->next.object.value").GetValueAsUnsigned(),
+        )
+        self.assertEqual(
+            2, frame.GetValueForVariablePath("f1->next->value").GetValueAsUnsigned()
+        )
+        self.assertEqual(
+            1,
+            frame.GetValueForVariablePath(
+                "f1->next.object.next.obj.value"
+            ).GetValueAsUnsigned(),
+        )
+        self.assertEqual(
+            1,
+            frame.GetValueForVariablePath("f1->next->next->value").GetValueAsUnsigned(),
+        )
+
+    @add_test_categories(["libstdcxx"])
+    def test_recursive_unique_ptr_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_recursive_unique_ptr()
+
+    @add_test_categories(["libc++"])
+    def test_recursive_unique_ptr_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_recursive_unique_ptr()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/main.cpp
similarity index 74%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/main.cpp
index afdddf0bbaf16..15e9f70dbd6aa 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/main.cpp
@@ -15,11 +15,20 @@ struct NodeU {
 // representation when the type of the second element is an empty class. So
 // we need a deleter class with a dummy data member to trigger the other path.
 struct NonEmptyIntDeleter {
-  void operator()(int* ptr) { delete ptr; }
+  void operator()(int *ptr) { delete ptr; }
 
   int dummy_ = 9999;
 };
 
+static void recursive() {
+  // Set up a structure where we have a loop in the unique_ptr chain.
+  NodeU *f1 = new NodeU{nullptr, 1};
+  NodeU *f2 = new NodeU{nullptr, 2};
+  f1->next.reset(f2);
+  f2->next.reset(f1);
+  std::puts("Set break point at this line.");
+}
+
 int main() {
   std::unique_ptr<int> up_empty;
   std::unique_ptr<int> up_int = std::make_unique<int>(10);
@@ -33,5 +42,9 @@ int main() {
       std::unique_ptr<NodeU>(new NodeU{nullptr, 2});
   ptr_node = std::unique_ptr<NodeU>(new NodeU{std::move(ptr_node), 1});
 
-  return 0; // break here
+  std::puts("// break here");
+
+  recursive();
+
+  return 0;
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/Makefile
similarity index 82%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/Makefile
index 7eeff7407804d..d5f5fec8441b5 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/Makefile
@@ -1,6 +1,4 @@
 CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
 CXXFLAGS_EXTRAS := -std=c++17
+
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/TestDataFormatterStdVariant.py
similarity index 50%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/TestDataFormatterStdVariant.py
index 394e221809f7c..9365cfc96783e 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/TestDataFormatterStdVariant.py
@@ -1,26 +1,30 @@
 """
-Test lldb data formatter for LibStdC++ std::variant.
+Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibStdcxxVariantDataFormatterTestCase(TestBase):
-    @add_test_categories(["libstdcxx"])
-    def test_with_run_command(self):
-        """Test LibStdC++ std::variant data formatter works correctly."""
-        self.build()
+class StdVariantDataFormatterTestCase(TestBase):
+    def do_test(self):
+        """Test that that file and class static variables display correctly."""
+
+        def cleanup():
+            self.runCmd("type format clear", check=False)
+            self.runCmd("type summary clear", check=False)
+            self.runCmd("type filter clear", check=False)
+            self.runCmd("type synth clear", check=False)
+
+        # Execute the cleanup function during test case tear down.
+        self.addTearDownHook(cleanup)
 
         (self.target, self.process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "// break here", lldb.SBFileSpec("main.cpp", False)
         )
 
-        lldbutil.continue_to_breakpoint(self.process, bkpt)
-
         for name in ["v1", "v1_typedef"]:
             self.expect(
                 "frame variable " + name,
@@ -62,40 +66,20 @@ def test_with_run_command(self):
             "frame variable v3",
             substrs=["v3 =  Active Type = char  {", "Value = 'A'", "}"],
         )
-        """
-        TODO: temporarily disable No Value tests as they seem to fail on ubuntu/debian
-        bots. Pending reproduce and investigation.
 
-        self.expect("frame variable v_no_value", substrs=["v_no_value =  No Value"])
+        self.expect("frame variable v_valueless", substrs=["v_valueless =  No Value"])
 
         self.expect(
-            "frame variable v_many_types_no_value",
-            substrs=["v_many_types_no_value =  No Value"],
+            "frame variable v_300_types_valueless",
+            substrs=["v_300_types_valueless =  No Value"],
         )
-        """
-
-    @add_test_categories(["libstdcxx"])
-    def test_invalid_variant_index(self):
-        """Test LibStdC++ data formatter for std::variant with invalid index."""
-        self.build()
 
-        (self.target, self.process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
-            self, "// break here", lldb.SBFileSpec("main.cpp", False)
-        )
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
 
-        lldbutil.continue_to_breakpoint(self.process, bkpt)
-
-        self.expect(
-            "frame variable v1",
-            substrs=["v1 =  Active Type = int  {", "Value = 12", "}"],
-        )
-
-        var_v1 = thread.frames[0].FindVariable("v1")
-        var_v1_raw_obj = var_v1.GetNonSyntheticValue()
-        index_obj = var_v1_raw_obj.GetChildMemberWithName("_M_index")
-        self.assertTrue(index_obj and index_obj.IsValid())
-
-        INVALID_INDEX = "100"
-        index_obj.SetValueFromCString(INVALID_INDEX)
-
-        self.expect("frame variable v1", substrs=["v1 =  <Invalid>"])
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/main.cpp
similarity index 72%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/main.cpp
index 36e0f74f831f8..620b97b7306f9 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/variant/main.cpp
@@ -4,16 +4,15 @@
 #include <vector>
 
 struct S {
-  operator int() { throw 42; }
+  S() = default;
+  S(S &&) { throw 42; }
+  S &operator=(S &&) = default;
 };
 
 int main() {
-  bool has_variant = true;
-
-  printf("%d\n", has_variant); // break here
-
   std::variant<int, double, char> v1;
   std::variant<int, double, char> &v1_ref = v1;
+
   using V1_typedef = std::variant<int, double, char>;
   V1_typedef v1_typedef;
   V1_typedef &v1_typedef_ref = v1_typedef;
@@ -21,8 +20,8 @@ int main() {
   std::variant<int, double, char> v2;
   std::variant<int, double, char> v3;
   std::variant<std::variant<int, double, char>> v_v1;
-  std::variant<int, double, char> v_no_value;
-  // The next variant has many types, meaning the type index does not fit in
+  std::variant<int, char, S> v_valueless = 5;
+  // The next variant has 300 types, meaning the type index does not fit in
   // a byte and must be `unsigned short` instead of `unsigned char` when
   // using the unstable libc++ ABI. With stable libc++ ABI, the type index
   // is always just `unsigned int`.
@@ -43,8 +42,15 @@ int main() {
       int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
       int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
       int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-      int, int, int, int, int, int, int, int, int, int, int, int>
-      v_many_types_no_value;
+      int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
+      int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
+      int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
+      int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
+      S>
+      v_300_types_valueless;
+
+  v_valueless = 5;
+  v_300_types_valueless.emplace<0>(10);
 
   v1 = 12; // v contains int
   v1_typedef = v1;
@@ -67,18 +73,22 @@ int main() {
   printf("%f\n", d); // break here
 
   try {
-    v_no_value.emplace<0>(S());
+    // Exception in type-changing move-assignment is guaranteed to put
+    // std::variant into a valueless state.
+    v_valueless = S();
   } catch (...) {
   }
 
-  printf("%zu\n", v_no_value.index());
+  printf("%d\n", v_valueless.valueless_by_exception());
 
   try {
-    v_many_types_no_value.emplace<0>(S());
+    // Exception in move-assignment is guaranteed to put std::variant into a
+    // valueless state.
+    v_300_types_valueless = S();
   } catch (...) {
   }
 
-  printf("%zu\n", v_many_types_no_value.index());
+  printf("%d\n", v_300_types_valueless.valueless_by_exception());
 
   return 0; // break here
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/Makefile
new file mode 100644
index 0000000000000..65e9dd2fa9e33
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/Makefile
@@ -0,0 +1,4 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
+
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/TestDataFormatterStdVBool.py
similarity index 81%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/TestDataFormatterStdVBool.py
index f3371bc014b17..56c86d1edde25 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/TestDataFormatterStdVBool.py
@@ -2,8 +2,6 @@
 Test lldb data formatter subsystem.
 """
 
-
-from typing import Optional
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -17,19 +15,8 @@ def setUp(self):
         # Find the line number to break at.
         self.line = line_number("main.cpp", "// Set break point at this line.")
 
-    @skip
-    @add_test_categories(["libstdcxx"])
-    def test_with_run_command(self):
-        self.with_run_command()
-
-    @add_test_categories(["libstdcxx"])
-    def test_with_run_command_debug(self):
-        build_args = {"CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
-        self.with_run_command(build_args)
-
-    def with_run_command(self, dictionary: Optional[dict] = None):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build(dictionary=dictionary)
         self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
 
         lldbutil.run_break_set_by_file_and_line(
@@ -52,6 +39,7 @@ def cleanup():
             self.runCmd("type summary clear", check=False)
             self.runCmd("type filter clear", check=False)
             self.runCmd("type synth clear", check=False)
+            self.runCmd("settings set target.max-children-count 24", check=False)
 
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
@@ -83,3 +71,20 @@ def cleanup():
                 "[48] = true",
             ],
         )
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp
new file mode 100644
index 0000000000000..22fc6c89ca8a2
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp
@@ -0,0 +1,65 @@
+#include <cstdio>
+#include <string>
+#include <vector>
+
+int main() {
+  std::vector<bool> vBool;
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(true);
+
+  std::puts("// Set break point at this line.");
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/TestDataFormatterStdVector.py
similarity index 84%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/TestDataFormatterStdVector.py
index 13341a9b274be..ba8b10450f4fc 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/TestDataFormatterStdVector.py
@@ -2,14 +2,13 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibcxxVectorDataFormatterTestCase(TestBase):
+class StdVectorDataFormatterTestCase(TestBase):
     def check_numbers(self, var_name, show_ptr=False):
         patterns = []
         substrs = [
@@ -52,10 +51,8 @@ def check_numbers(self, var_name, show_ptr=False):
         self.expect("frame variable " + var_name + "[2]", substrs=["123"])
         self.expect("frame variable " + var_name + "[3]", substrs=["1234"])
 
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
         (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.cpp", False)
         )
@@ -170,10 +167,25 @@ def cleanup():
 
         self.expect("frame variable strings", substrs=["vector has 0 items"])
 
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test()
+
     @add_test_categories(["libc++"])
-    def test_ref_and_ptr(self):
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    def do_test_ref_and_ptr(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
         (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "Stop here to check by ref", lldb.SBFileSpec("main.cpp", False)
         )
@@ -186,3 +198,20 @@ def test_ref_and_ptr(self):
         self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
 
         self.expect("expression ptr", substrs=["$", "size=7"])
+
+    @add_test_categories(["libstdcxx"])
+    def test_ref_and_ptr_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_ref_and_ptr()
+
+    @add_test_categories(["libstdcxx"])
+    def test_ref_and_ptr_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test_ref_and_ptr()
+
+    @add_test_categories(["libc++"])
+    def test_ref_and_ptr_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_ref_and_ptr()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp
new file mode 100644
index 0000000000000..ec7ef7f84310e
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp
@@ -0,0 +1,40 @@
+#include <stdio.h>
+#include <string>
+#include <vector>
+typedef std::vector<int> int_vect;
+typedef std::vector<std::string> string_vect;
+
+template <class T>
+void by_ref_and_ptr(std::vector<T> &ref, std::vector<T> *ptr) {
+  // Stop here to check by ref
+  return;
+}
+
+int main() {
+  int_vect numbers;
+  numbers.push_back(1);  // break here
+  numbers.push_back(12); // break here
+  numbers.push_back(123);
+  numbers.push_back(1234);
+  numbers.push_back(12345); // break here
+  numbers.push_back(123456);
+  numbers.push_back(1234567);
+  by_ref_and_ptr(numbers, &numbers);
+
+  printf("break here");
+  numbers.clear();
+
+  numbers.push_back(7); // break here
+
+  string_vect strings;
+  strings.push_back(std::string("goofy"));
+  strings.push_back(std::string("is"));
+  strings.push_back(std::string("smart"));
+  printf("break here");
+  strings.push_back(std::string("!!!"));
+
+  printf("break here");
+  strings.clear();
+
+  return 0; // break here
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
index 3a745883fb49c..f10811817c0d2 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
@@ -1,4 +1,4 @@
-#define COMPRESSED_PAIR_REV 2
+#define COMPRESSED_PAIR_REV 3
 #include <libcxx-simulators-common/compressed_pair.h>
 
 namespace std {
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py
index c996fff4fb7aa..c8d9c2e389a05 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py
@@ -28,7 +28,7 @@ def _run_test(self, defines):
 
 for v in [None, "ALTERNATE_LAYOUT"]:
     for r in range(6):
-        for c in range(3):
+        for c in range(4):
             name = "test_r%d_c%d" % (r, c)
             defines = ["REVISION=%d" % r, "COMPRESSED_PAIR_REV=%d" % c]
             if v:
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp
index fdf8d53a34138..cf431e524069c 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp
@@ -209,7 +209,7 @@ template <class _CharT, class _Traits, class _Allocator> class basic_string {
   __long &getLongRep() {
 #if COMPRESSED_PAIR_REV == 0
     return __r_.first().__l;
-#elif COMPRESSED_PAIR_REV <= 2
+#elif COMPRESSED_PAIR_REV <= 3
     return __rep_.__l;
 #endif
   }
@@ -217,14 +217,14 @@ template <class _CharT, class _Traits, class _Allocator> class basic_string {
   __short &getShortRep() {
 #if COMPRESSED_PAIR_REV == 0
     return __r_.first().__s;
-#elif COMPRESSED_PAIR_REV <= 2
+#elif COMPRESSED_PAIR_REV <= 3
     return __rep_.__s;
 #endif
   }
 
 #if COMPRESSED_PAIR_REV == 0
   std::__lldb::__compressed_pair<__rep, allocator_type> __r_;
-#elif COMPRESSED_PAIR_REV <= 2
+#elif COMPRESSED_PAIR_REV <= 3
   _LLDB_COMPRESSED_PAIR(__rep, __rep_, allocator_type, __alloc_);
 #endif
 
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py
index 4ee97e10de84c..e623c3a1413b6 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py
@@ -26,7 +26,7 @@ def _run_test(self, defines):
         )
 
 
-for r in range(3):
+for r in range(4):
     name = "test_r%d" % r
     defines = ["COMPRESSED_PAIR_REV=%d" % r]
 
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp
index 91a019566affb..3d174a91cc262 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp
@@ -20,7 +20,8 @@ template <class _Tp, class _Dp = default_delete<_Tp>> class unique_ptr {
   std::__lldb::__compressed_pair<pointer, deleter_type> __ptr_;
   explicit unique_ptr(pointer __p) noexcept
       : __ptr_(__p, std::__lldb::__value_init_tag()) {}
-#elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2
+#elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2 ||                  \
+    COMPRESSED_PAIR_REV == 3
   _LLDB_COMPRESSED_PAIR(pointer, __ptr_, deleter_type, __deleter_);
   explicit unique_ptr(pointer __p) noexcept : __ptr_(__p), __deleter_() {}
 #endif
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
deleted file mode 100644
index b016f006747da..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-CXX_SOURCES := main.cpp
-CXXFLAGS_EXTRAS := -std=c++11
-USE_LIBCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
deleted file mode 100644
index ece1e4f5f049b..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
+++ /dev/null
@@ -1,87 +0,0 @@
-"""
-Test LLDB's data formatter for libcxx's std::deque.
-"""
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class LibcxxDequeDataFormatterTestCase(TestBase):
-    def check_numbers(self, var_name, show_ptr=False):
-        patterns = []
-        substrs = [
-            "[0] = 1",
-            "[1] = 12",
-            "[2] = 123",
-            "[3] = 1234",
-            "[4] = 12345",
-            "[5] = 123456",
-            "[6] = 1234567",
-            "}",
-        ]
-        if show_ptr:
-            patterns = [var_name + " = 0x.* size=7"]
-        else:
-            substrs.insert(0, var_name + " = size=7")
-        self.expect(
-            "frame variable " + var_name,
-            patterns=patterns,
-            substrs=substrs,
-        )
-        self.expect_expr(
-            var_name,
-            result_summary="size=7",
-            result_children=[
-                ValueCheck(value="1"),
-                ValueCheck(value="12"),
-                ValueCheck(value="123"),
-                ValueCheck(value="1234"),
-                ValueCheck(value="12345"),
-                ValueCheck(value="123456"),
-                ValueCheck(value="1234567"),
-            ],
-        )
-
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
-        """Test basic formatting of std::deque"""
-        self.build()
-        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
-            self, "break here", lldb.SBFileSpec("main.cpp", False)
-        )
-
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-        lldbutil.continue_to_breakpoint(process, bkpt)
-
-        # first value added
-        self.expect(
-            "frame variable numbers", substrs=["numbers = size=1", "[0] = 1", "}"]
-        )
-
-        # add remaining values
-        lldbutil.continue_to_breakpoint(process, bkpt)
-
-        self.check_numbers("numbers")
-
-        # clear out the deque
-        lldbutil.continue_to_breakpoint(process, bkpt)
-
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-    @add_test_categories(["libc++"])
-    def test_ref_and_ptr(self):
-        """Test formatting of std::deque& and std::deque*"""
-        self.build()
-        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
-            self, "stop here", lldb.SBFileSpec("main.cpp", False)
-        )
-
-        # The reference should display the same was as the value did
-        self.check_numbers("ref", True)
-
-        # The pointer should just show the right number of elements:
-        self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
-        self.expect("expression ptr", substrs=["$", "size=7"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp
deleted file mode 100644
index 025c823d1f5b6..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-#include <cstdio>
-#include <deque>
-typedef std::deque<int> int_deq;
-
-void by_ref_and_ptr(std::deque<int> &ref, std::deque<int> *ptr) {
-  printf("stop here");
-  return;
-}
-
-int main() {
-  int_deq numbers;
-  printf("break here");
-
-  (numbers.push_back(1));
-  printf("break here");
-
-  (numbers.push_back(12));
-  (numbers.push_back(123));
-  (numbers.push_back(1234));
-  (numbers.push_back(12345));
-  (numbers.push_back(123456));
-  (numbers.push_back(1234567));
-  by_ref_and_ptr(numbers, &numbers);
-  printf("break here");
-
-  numbers.clear();
-  printf("break here");
-
-  return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/Makefile
similarity index 98%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/Makefile
index 680e1abfbef58..c5df567e01a2a 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/Makefile
@@ -1,4 +1,5 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py
new file mode 100644
index 0000000000000..ae8e0ac08c2b0
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py
@@ -0,0 +1,38 @@
+"""
+Test lldb behaves sanely when formatting corrupted `std::string`s.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class LibcxxInvalidStringDataFormatterTestCase(TestBase):
+    @add_test_categories(["libc++"])
+    @skipIf(oslist=[lldbplatformutil.getDarwinOSTriples()], archs=["arm", "aarch64"])
+    def test(self):
+        self.build()
+
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Set break point at this line.", lldb.SBFileSpec("main.cpp")
+        )
+        frame = thread.frames[0]
+
+        if not self.process().GetAddressByteSize() == 8:
+            self.skip()
+
+        # The test assumes that std::string is in its cap-size-data layout.
+        self.expect(
+            "frame variable garbage1", substrs=["garbage1 = Summary Unavailable"]
+        )
+        self.expect(
+            "frame variable garbage2", substrs=[r'garbage2 = "\xfa\xfa\xfa\xfa"']
+        )
+        self.expect("frame variable garbage3", substrs=[r'garbage3 = "\xf0\xf0"'])
+        self.expect(
+            "frame variable garbage4", substrs=["garbage4 = Summary Unavailable"]
+        )
+        self.expect(
+            "frame variable garbage5", substrs=["garbage5 = Summary Unavailable"]
+        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp
new file mode 100644
index 0000000000000..eb3efe1bcb7ef
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp
@@ -0,0 +1,110 @@
+#include <cstdio>
+#include <cstdlib>
+#include <stdint.h>
+#include <string>
+
+// For more information about libc++'s std::string ABI, see:
+//
+//   https://joellaity.com/2020/01/31/string.html
+
+// A corrupt string which hits the SSO code path, but has an invalid size.
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  // Set the size of this short-mode string to 116. Note that in short mode,
+  // the size is encoded as `size << 1`.
+  unsigned char size = 232;
+
+  // 23 garbage bytes for the inline string payload.
+  char inline_buf[23] = {0};
+#else  // _LIBCPP_ABI_VERSION == 1
+  // Like above, but data comes first, and use bitfields to indicate size.
+  char inline_buf[23] = {0};
+  unsigned char size : 7 = 116;
+  unsigned char is_long : 1 = 0;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_short_mode;
+
+// A corrupt libcxx string in long mode with a payload that contains a utf8
+// sequence that's inherently too long.
+static unsigned char garbage_utf8_payload1[] = {
+    250, // This means that we expect a 5-byte sequence, this is invalid. LLDB
+         // should fall back to ASCII printing.
+    250, 250, 250};
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 5;
+  uint64_t size = 4;
+  unsigned char *data = &garbage_utf8_payload1[0];
+#else  // _LIBCPP_ABI_VERSION == 1
+  unsigned char *data = &garbage_utf8_payload1[0];
+  uint64_t size = 4;
+  uint64_t cap : 63 = 4;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode1;
+
+// A corrupt libcxx string in long mode with a payload that contains a utf8
+// sequence that's too long to fit in the buffer.
+static unsigned char garbage_utf8_payload2[] = {
+    240, // This means that we expect a 4-byte sequence, but the buffer is too
+         // small for this. LLDB should fall back to ASCII printing.
+    240};
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 3;
+  uint64_t size = 2;
+  unsigned char *data = &garbage_utf8_payload2[0];
+#else  // _LIBCPP_ABI_VERSION == 1
+  unsigned char *data = &garbage_utf8_payload2[0];
+  uint64_t size = 2;
+  uint64_t cap : 63 = 3;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode2;
+
+// A corrupt libcxx string which has an invalid size (i.e. a size greater than
+// the capacity of the string).
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 5;
+  uint64_t size = 7;
+  const char *data = "foo";
+#else  // _LIBCPP_ABI_VERSION == 1
+  const char *data = "foo";
+  uint64_t size = 7;
+  uint64_t cap : 63 = 5;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode3;
+
+// A corrupt libcxx string in long mode with a payload that would trigger a
+// buffer overflow.
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 5;
+  uint64_t size = 2;
+  uint64_t data = 0xfffffffffffffffeULL;
+#else  // _LIBCPP_ABI_VERSION == 1
+  uint64_t data = 0xfffffffffffffffeULL;
+  uint64_t size = 2;
+  uint64_t cap : 63 = 5;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode4;
+
+int main() {
+  std::string garbage1, garbage2, garbage3, garbage4, garbage5;
+  if (sizeof(std::string) == sizeof(garbage_string_short_mode))
+    memcpy((void *)&garbage1, &garbage_string_short_mode, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode1))
+    memcpy((void *)&garbage2, &garbage_string_long_mode1, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode2))
+    memcpy((void *)&garbage3, &garbage_string_long_mode2, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode3))
+    memcpy((void *)&garbage4, &garbage_string_long_mode3, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode4))
+    memcpy((void *)&garbage5, &garbage_string_long_mode4, sizeof(std::string));
+
+  std::puts("// Set break point at this line.");
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
deleted file mode 100644
index 43ce6aeefd2a2..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-#include <string>
-#include <map>
-
-#define intint_map std::map<int, int> 
-#define strint_map std::map<std::string, int> 
-#define intstr_map std::map<int, std::string> 
-#define strstr_map std::map<std::string, std::string> 
-
-int g_the_foo = 0;
-
-int thefoo_rw(int arg = 1)
-{
-	if (arg < 0)
-		arg = 0;
-	if (!arg)
-		arg = 1;
-	g_the_foo += arg;
-	return g_the_foo;
-}
-
-int main()
-{
-    intint_map ii;
-    
-    ii[0] = 0; // Set break point at this line.
-    ii[1] = 1;
-
-    intint_map::iterator it = ii.begin();
-    intint_map::const_iterator const_it = ii.cbegin();
-    std::printf("%d %d\n", it->second, const_it->second);
-
-    thefoo_rw(1); // Set break point at this line.
-    ii[2] = 0;
-    ii[3] = 1;
-	thefoo_rw(1);  // Set break point at this line.
-	ii[4] = 0;
-    ii[5] = 1;
-    ii[6] = 0;
-    ii[7] = 1;
-    thefoo_rw(1);  // Set break point at this line.
-    ii[85] = 1234567;
-
-    ii.clear();
-    
-    strint_map si;
-    thefoo_rw(1);  // Set break point at this line.
-	
-    si["zero"] = 0;
-	thefoo_rw(1);  // Set break point at this line.
-    si["one"] = 1;
-    si["two"] = 2;
-    si["three"] = 3;
-	thefoo_rw(1);  // Set break point at this line.
-    si["four"] = 4;
-
-    si.clear();
-    thefoo_rw(1);  // Set break point at this line.
-	
-    intstr_map is;
-    thefoo_rw(1);  // Set break point at this line.
-    is[85] = "goofy";
-    is[1] = "is";
-    is[2] = "smart";
-    is[3] = "!!!";
-    thefoo_rw(1);  // Set break point at this line.
-	
-    is.clear();
-    thefoo_rw(1);  // Set break point at this line.
-	
-    strstr_map ss;
-    thefoo_rw(1);  // Set break point at this line.
-	
-    ss["ciao"] = "hello";
-    ss["casa"] = "house";
-    ss["gatto"] = "cat";
-    thefoo_rw(1);  // Set break point at this line.
-    ss["a Mac.."] = "..is always a Mac!";
-    
-    ss.clear();
-    thefoo_rw(1);  // Set break point at this line.    
-    return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile
deleted file mode 100644
index c1c8b4a2a0a53..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-# We need debug info tuning for lldb in order to emit the preferred name for
-# std::string. See https://reviews.llvm.org/D145803.
-CXXFLAGS_EXTRAS := -std=c++14 -glldb
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
deleted file mode 100644
index c7c91da728d1e..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-CXXFLAGS_EXTRAS := -std=c++11 -O0
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
deleted file mode 100644
index f9f1c0802e518..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
+++ /dev/null
@@ -1,134 +0,0 @@
-#include <string>
-#include <stdint.h>
-
-// For more information about libc++'s std::string ABI, see:
-//
-//   https://joellaity.com/2020/01/31/string.html
-
-// A corrupt string which hits the SSO code path, but has an invalid size.
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  // Set the size of this short-mode string to 116. Note that in short mode,
-  // the size is encoded as `size << 1`.
-  unsigned char size = 232;
-
-  // 23 garbage bytes for the inline string payload.
-  char inline_buf[23] = {0};
-#else  // _LIBCPP_ABI_VERSION == 1
-  // Like above, but data comes first, and use bitfields to indicate size.
-  char inline_buf[23] = {0};
-  unsigned char size : 7 = 116;
-  unsigned char is_long : 1 = 0;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_short_mode;
-
-// A corrupt libcxx string in long mode with a payload that contains a utf8
-// sequence that's inherently too long.
-static unsigned char garbage_utf8_payload1[] = {
-  250, // This means that we expect a 5-byte sequence, this is invalid. LLDB
-       // should fall back to ASCII printing.
-  250, 250, 250
-};
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 5;
-  uint64_t size = 4;
-  unsigned char *data = &garbage_utf8_payload1[0];
-#else  // _LIBCPP_ABI_VERSION == 1
-  unsigned char *data = &garbage_utf8_payload1[0];
-  uint64_t size = 4;
-  uint64_t cap : 63 = 4;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode1;
-
-// A corrupt libcxx string in long mode with a payload that contains a utf8
-// sequence that's too long to fit in the buffer.
-static unsigned char garbage_utf8_payload2[] = {
-  240, // This means that we expect a 4-byte sequence, but the buffer is too
-       // small for this. LLDB should fall back to ASCII printing.
-  240
-};
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 3;
-  uint64_t size = 2;
-  unsigned char *data = &garbage_utf8_payload2[0];
-#else  // _LIBCPP_ABI_VERSION == 1
-  unsigned char *data = &garbage_utf8_payload2[0];
-  uint64_t size = 2;
-  uint64_t cap : 63 = 3;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode2;
-
-// A corrupt libcxx string which has an invalid size (i.e. a size greater than
-// the capacity of the string).
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 5;
-  uint64_t size = 7;
-  const char *data = "foo";
-#else  // _LIBCPP_ABI_VERSION == 1
-  const char *data = "foo";
-  uint64_t size = 7;
-  uint64_t cap : 63 = 5;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode3;
-
-// A corrupt libcxx string in long mode with a payload that would trigger a
-// buffer overflow.
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 5;
-  uint64_t size = 2;
-  uint64_t data = 0xfffffffffffffffeULL;
-#else  // _LIBCPP_ABI_VERSION == 1
-  uint64_t data = 0xfffffffffffffffeULL;
-  uint64_t size = 2;
-  uint64_t cap : 63 = 5;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode4;
-
-size_t touch_string(std::string &in_str)
-{
-  return in_str.size(); // Break here to look at bad string
-}
-
-int main()
-{
-    std::wstring wempty(L"");
-    std::wstring s(L"hello world! מזל טוב!");
-    std::wstring S(L"!!!!");
-    const wchar_t *mazeltov = L"מזל טוב";
-    std::string empty("");
-    std::string q("hello world");
-    std::string Q("quite a long std::strin with lots of info inside it");
-    std::string TheVeryLongOne("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890someText1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890");
-    std::string IHaveEmbeddedZeros("a\0b\0c\0d",7);
-    std::wstring IHaveEmbeddedZerosToo(L"hello world!\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監", 38);
-    std::u16string u16_string(u"ß水氶");
-    std::u16string u16_empty(u"");
-    std::u32string u32_string(U"🍄🍅🍆🍌");
-    std::u32string u32_empty(U"");
-    std::string *null_str = nullptr;
-
-    std::string garbage1, garbage2, garbage3, garbage4, garbage5;
-    if (sizeof(std::string) == sizeof(garbage_string_short_mode))
-      memcpy((void *)&garbage1, &garbage_string_short_mode, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode1))
-      memcpy((void *)&garbage2, &garbage_string_long_mode1, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode2))
-      memcpy((void *)&garbage3, &garbage_string_long_mode2, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode3))
-      memcpy((void *)&garbage4, &garbage_string_long_mode3, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode4))
-      memcpy((void *)&garbage5, &garbage_string_long_mode4, sizeof(std::string));
-
-    S.assign(L"!!!!!"); // Set break point at this line.
-    std::string *not_a_string = (std::string *) 0x0;
-    touch_string(*not_a_string);
-    return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py
deleted file mode 100644
index 5b6dfaf5e5d9d..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class TestDataFormatterLibcxxTuple(TestBase):
-    def setUp(self):
-        TestBase.setUp(self)
-        self.line = line_number("main.cpp", "// break here")
-        self.namespace = "std"
-
-    @add_test_categories(["libc++"])
-    def test(self):
-        """Test that std::tuple is displayed correctly"""
-        self.build()
-        lldbutil.run_to_source_breakpoint(
-            self, "// break here", lldb.SBFileSpec("main.cpp", False)
-        )
-
-        tuple_name = self.namespace + "::tuple"
-        self.expect("frame variable empty", substrs=[tuple_name, "size=0", "{}"])
-
-        self.expect(
-            "frame variable one_elt",
-            substrs=[tuple_name, "size=1", "{", "[0] = 47", "}"],
-        )
-
-        self.expect(
-            "frame variable three_elts",
-            substrs=[
-                tuple_name,
-                "size=3",
-                "{",
-                "[0] = 1",
-                "[1] = 47",
-                '[2] = "foo"',
-                "}",
-            ],
-        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile
deleted file mode 100644
index c1c8b4a2a0a53..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-# We need debug info tuning for lldb in order to emit the preferred name for
-# std::string. See https://reviews.llvm.org/D145803.
-CXXFLAGS_EXTRAS := -std=c++14 -glldb
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/Makefile
similarity index 100%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/Makefile
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/TestDataFormatterLibccUnorderedMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/TestDataFormatterLibccUnorderedMap.py
similarity index 58%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/TestDataFormatterLibccUnorderedMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/TestDataFormatterLibccUnorderedMap.py
index 2b1bd676a5b34..a338e3c12598e 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/TestDataFormatterLibccUnorderedMap.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/TestDataFormatterLibccUnorderedMap.py
@@ -9,21 +9,28 @@
 
 
 class LibcxxUnorderedMapDataFormatterTestCase(TestBase):
-    def check_reference(self, var_name: str, expected_type: str):
-        self.expect_var_path(
-            var_name,
-            summary="size=1",
-            type=expected_type,
-            children=[
-                ValueCheck(
-                    name="[0]",
-                    children=[
-                        ValueCheck(name="first", summary='"Hello"'),
-                        ValueCheck(name="second", summary='"World"'),
-                    ],
-                ),
-            ],
-        )
+    def check_ptr_or_ref(self, var_name: str):
+        var = self.frame().FindVariable(var_name)
+        self.assertTrue(var)
+
+        pair = var.GetChildAtIndex(0)
+        self.assertTrue(pair)
+
+        self.assertEqual(pair.GetChildAtIndex(0).summary, '"Hello"')
+        self.assertEqual(pair.GetChildAtIndex(1).summary, '"World"')
+
+    def check_ptr_ptr(self, var_name: str):
+        var = self.frame().FindVariable(var_name)
+        self.assertTrue(var)
+
+        ptr = var.GetChildAtIndex(0)
+        self.assertTrue(ptr)
+
+        pair = ptr.GetChildAtIndex(0)
+        self.assertTrue(pair)
+
+        self.assertEqual(pair.GetChildAtIndex(0).summary, '"Hello"')
+        self.assertEqual(pair.GetChildAtIndex(1).summary, '"World"')
 
     @add_test_categories(["libc++"])
     def test_iterator_formatters(self):
@@ -84,12 +91,12 @@ def test_iterator_formatters(self):
         lldbutil.continue_to_breakpoint(process, bkpt)
 
         # Test references to std::unordered_map
-        self.check_reference("ref1", "const StringMapT &")
-        self.check_reference("ref2", "StringMapT &")
-        self.check_reference("ref3", "StringMapTRef")
-        self.check_reference("ref4", "const StringMapT &")
-        self.check_reference("ref5", "const StringMapT &&")
-        self.check_reference("ref6", "StringMapT &&")
+        self.check_ptr_or_ref("ref1")
+        self.check_ptr_or_ref("ref2")
+        self.check_ptr_or_ref("ref3")
+        self.check_ptr_or_ref("ref4")
+        self.check_ptr_or_ref("ref5")
+        self.check_ptr_or_ref("ref6")
 
         # FIXME: we're getting this wrong.
         self.expect_var_path(
@@ -97,3 +104,26 @@ def test_iterator_formatters(self):
             summary="size=0",
             type="const StringMapT *const &",
         )
+
+    @expectedFailureAll(
+        bugnumber="https://github.com/llvm/llvm-project/issues/146040",
+        compiler="clang",
+        compiler_version=["<", "21"],
+    )
+    @add_test_categories(["libc++"])
+    def test_ptr_formatters(self):
+        """
+        Test that pointers to std::unordered_map are formatted correctly.
+        """
+
+        self.build()
+        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Stop here", lldb.SBFileSpec("main.cpp", False)
+        )
+
+        self.check_ptr_or_ref("ptr1")
+        self.check_ptr_or_ref("ptr2")
+        self.check_ptr_or_ref("ptr3")
+        self.check_ptr_ptr("ptr4")
+        self.check_ptr_ptr("ptr5")
+        self.check_ptr_ptr("ptr6")
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp
similarity index 72%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp
index c581fded1ec5f..5397e54d99545 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp
@@ -4,6 +4,7 @@
 
 using StringMapT = std::unordered_map<std::string, std::string>;
 using StringMapTRef = const StringMapT &;
+using StringMapTPtr = const StringMapT *;
 
 static void check_references(const StringMapT &ref1, StringMapT &ref2,
                              StringMapTRef ref3, StringMapTRef &ref4,
@@ -12,6 +13,12 @@ static void check_references(const StringMapT &ref1, StringMapT &ref2,
   std::printf("Break here");
 }
 
+static void check_pointer(const StringMapT *ptr1, StringMapT *ptr2,
+                          StringMapTPtr ptr3, StringMapTPtr *ptr4,
+                          const StringMapT *const *ptr5, StringMapT **ptr6) {
+  std::printf("Stop here");
+}
+
 int main() {
   StringMapT string_map;
   {
@@ -35,6 +42,11 @@ int main() {
     StringMapT tmp{{"Hello", "World"}};
     check_references(tmp, tmp, tmp, tmp, StringMapT{tmp}, StringMapT{tmp},
                      &tmp);
+
+    StringMapT *tmp_ptr = &tmp;
+    const StringMapT *const_tmp_ptr = &tmp;
+    check_pointer(tmp_ptr, tmp_ptr, tmp_ptr, &const_tmp_ptr, &tmp_ptr,
+                  &tmp_ptr);
   }
 
   return 0;
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile
deleted file mode 100644
index 564cbada74e08..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-CXXFLAGS_EXTRAS := -O0
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
deleted file mode 100644
index 47e07a5ce3f5b..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
+++ /dev/null
@@ -1,88 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class LibcxxVariantDataFormatterTestCase(TestBase):
-    @add_test_categories(["libc++"])
-    ## Clang 7.0 is the oldest Clang that can reliably parse newer libc++ versions
-    ## with -std=c++17.
-    @skipIf(
-        oslist=no_match(["macosx"]), compiler="clang", compiler_version=["<", "7.0"]
-    )
-    ## We are skipping gcc version less that 5.1 since this test requires -std=c++17
-    @skipIf(compiler="gcc", compiler_version=["<", "5.1"])
-    ## std::get is unavailable for std::variant before macOS 10.14
-    @skipIf(macos_version=["<", "10.14"])
-    def test_with_run_command(self):
-        """Test that that file and class static variables display correctly."""
-        self.build()
-
-        (self.target, self.process, _, bkpt) = lldbutil.run_to_source_breakpoint(
-            self, "// break here", lldb.SBFileSpec("main.cpp", False)
-        )
-
-        self.runCmd("frame variable has_variant")
-
-        output = self.res.GetOutput()
-
-        ## The variable has_variant tells us if the test program
-        ## detected we have a sufficient libc++ version to support variant
-        ## false means we do not and therefore should skip the test
-        if output.find("(bool) has_variant = false") != -1:
-            self.skipTest("std::variant not supported")
-
-        lldbutil.continue_to_breakpoint(self.process, bkpt)
-
-        self.expect(
-            "frame variable v1",
-            substrs=["v1 =  Active Type = int  {", "Value = 12", "}"],
-        )
-
-        self.expect(
-            "frame variable v1_ref",
-            patterns=["v1_ref = 0x.* Active Type = int : {", "Value = 12", "}"],
-        )
-
-        self.expect(
-            "frame variable v_v1",
-            substrs=[
-                "v_v1 =  Active Type = std::variant<int, double, char>  {",
-                "Value =  Active Type = int  {",
-                "Value = 12",
-                "}",
-                "}",
-            ],
-        )
-
-        lldbutil.continue_to_breakpoint(self.process, bkpt)
-
-        self.expect(
-            "frame variable v1",
-            substrs=["v1 =  Active Type = double  {", "Value = 2", "}"],
-        )
-
-        lldbutil.continue_to_breakpoint(self.process, bkpt)
-
-        self.expect(
-            "frame variable v2",
-            substrs=["v2 =  Active Type = double  {", "Value = 2", "}"],
-        )
-
-        self.expect(
-            "frame variable v3",
-            substrs=["v3 =  Active Type = char  {", "Value = 'A'", "}"],
-        )
-
-        self.expect("frame variable v_no_value", substrs=["v_no_value =  No Value"])
-
-        self.expect(
-            "frame variable v_300_types_no_value",
-            substrs=["v_300_types_no_value =  No Value"],
-        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp
deleted file mode 100644
index 560ec692f30ed..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-#include <cstdio>
-#include <string>
-#include <vector>
-
-// If we have libc++ 4.0 or greater we should have <variant>
-// According to libc++ C++1z status page https://libcxx.llvm.org/cxx1z_status.html
-#if _LIBCPP_VERSION >= 4000
-#include <variant>
-#define HAVE_VARIANT 1
-#else
-#define HAVE_VARIANT 0
-#endif
-
-struct S {
-  operator int() { throw 42; }
-} ;
-
-
-int main()
-{
-    bool has_variant = HAVE_VARIANT ;
-
-    printf( "%d\n", has_variant ) ; // break here
-
-#if HAVE_VARIANT == 1
-    std::variant<int, double, char> v1;
-    std::variant<int, double, char> &v1_ref = v1;
-    std::variant<int, double, char> v2;
-    std::variant<int, double, char> v3;
-    std::variant<std::variant<int,double,char>> v_v1 ;
-    std::variant<int, double, char> v_no_value;
-    // The next variant has 300 types, meaning the type index does not fit in
-    // a byte and must be `unsigned short` instead of `unsigned char` when
-    // using the unstable libc++ ABI. With stable libc++ ABI, the type index
-    // is always just `unsigned int`.
-    std::variant<
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-        int, int, int, int, int, int>
-        v_300_types_no_value;
-
-    v1 = 12; // v contains int
-    v_v1 = v1 ;
-    int i = std::get<int>(v1);
-    printf( "%d\n", i ); // break here
-
-    v2 = 2.0 ;
-    double d = std::get<double>(v2) ;
-    printf( "%f\n", d );
-
-    v3 = 'A' ;
-    char c = std::get<char>(v3) ;
-    printf( "%d\n", c );
-
-    // Checking v1 above and here to make sure we done maintain the incorrect
-    // state when we change its value.
-    v1 = 2.0;
-    d = std::get<double>(v1) ;
-    printf( "%f\n", d ); // break here
-
-     try {
-       v_no_value.emplace<0>(S());
-     } catch( ... ) {}
-
-     printf( "%zu\n", v_no_value.index() ) ;
-
-     try {
-       v_300_types_no_value.emplace<0>(S());
-     } catch (...) {
-     }
-
-     printf("%zu\n", v_300_types_no_value.index());
-#endif
-
-    return 0; // break here
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
deleted file mode 100644
index 24dddee62e1e7..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class LibcxxVBoolDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
-        """Test that that file and class static variables display correctly."""
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1
-        )
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-            self.runCmd("settings set target.max-children-count 24", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        self.expect(
-            "frame variable -A vBool",
-            substrs=[
-                "size=49",
-                "[0] = false",
-                "[1] = true",
-                "[18] = false",
-                "[27] = true",
-                "[36] = false",
-                "[47] = true",
-                "[48] = true",
-            ],
-        )
-
-        self.expect(
-            "expr -A -- vBool",
-            substrs=[
-                "size=49",
-                "[0] = false",
-                "[1] = true",
-                "[18] = false",
-                "[27] = true",
-                "[36] = false",
-                "[47] = true",
-                "[48] = true",
-            ],
-        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
deleted file mode 100644
index 026cfc863f2c0..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-#include <string>
-#include <vector>
-
-int main()
-{
-    std::vector<bool> vBool;
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(true);
-
-    printf ("size: %d", (int) vBool.size()); // Set break point at this line.
-    return 0; 
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
deleted file mode 100644
index 564cbada74e08..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-CXXFLAGS_EXTRAS := -O0
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
deleted file mode 100644
index 0e1dbe4f03e2b..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-#include <stdio.h>
-#include <string>
-#include <vector>
-typedef std::vector<int> int_vect;
-typedef std::vector<std::string> string_vect;
-
-template <class T>
-void by_ref_and_ptr(std::vector<T> &ref, std::vector<T> *ptr) {
-  // Stop here to check by ref
-  return;  
-}
-
-int main()
-{
-    int_vect numbers;
-    (numbers.push_back(1));  // break here
-    (numbers.push_back(12));  // break here
-    (numbers.push_back(123));
-    (numbers.push_back(1234));
-    (numbers.push_back(12345)); // break here
-    (numbers.push_back(123456));
-    (numbers.push_back(1234567));
-    by_ref_and_ptr(numbers, &numbers);
-    
-    printf("break here");
-    numbers.clear();
-    
-    (numbers.push_back(7)); // break here
-
-    string_vect strings;
-    (strings.push_back(std::string("goofy")));
-    (strings.push_back(std::string("is")));
-    (strings.push_back(std::string("smart")));
-    printf("break here");
-    (strings.push_back(std::string("!!!")));
-     
-    printf("break here");
-    strings.clear();
-    
-    return 0;  // break here
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/Makefile
similarity index 100%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/Makefile
index 104f82809c7a3..8c50abfbb5349 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
-
-USE_LIBSTDCPP := 1
 CXXFLAGS_EXTRAS := -std=c++17
+USE_LIBSTDCPP := 1
+
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/TestDataFormatterInvalidStdVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/TestDataFormatterInvalidStdVariant.py
new file mode 100644
index 0000000000000..14bb3463fe435
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/TestDataFormatterInvalidStdVariant.py
@@ -0,0 +1,30 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class InvalidStdVariantDataFormatterTestCase(TestBase):
+    @add_test_categories(["libstdcxx"])
+    def test(self):
+        """Test STL data formatters for std::variant with invalid index."""
+        self.build()
+
+        (_, _, thread, _) = lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp", False)
+        )
+
+        self.expect(
+            "frame variable v1",
+            substrs=["v1 =  Active Type = char  {", "Value = 'x'", "}"],
+        )
+
+        var_v1 = thread.frames[0].FindVariable("v1")
+        var_v1_raw_obj = var_v1.GetNonSyntheticValue()
+        index_obj = var_v1_raw_obj.GetChildMemberWithName("_M_index")
+        self.assertTrue(index_obj and index_obj.IsValid())
+
+        INVALID_INDEX = "100"
+        index_obj.SetValueFromCString(INVALID_INDEX)
+
+        self.expect("frame variable v1", substrs=["v1 =  <Invalid>"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/main.cpp
new file mode 100644
index 0000000000000..0d64cbf17bd2b
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/invalid-variant/main.cpp
@@ -0,0 +1,11 @@
+#include <cstdio>
+#include <variant>
+
+int main() {
+  std::variant<int, double, char> v1;
+  v1 = 'x';
+
+  std::puts("// break here");
+
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
deleted file mode 100644
index bf8e6b8703f36..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
deleted file mode 100644
index 2d76c02657a7d..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
+++ /dev/null
@@ -1,301 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-from typing import Optional
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdMapDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.with_run_command()
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command_debug(self):
-        build_args = {"CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
-        self.with_run_command("__debug::", build_args)
-
-    def with_run_command(self, namespace: str = "", dictionary: Optional[dict] = None):
-        """Test that that file and class static variables display correctly."""
-        self.build(dictionary=dictionary)
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        self.runCmd("frame variable ii --show-types")
-
-        self.runCmd(
-            f'type summary add -x "std::{namespace}map<" --summary-string "map has ${{svar%#}} items" -e'
-        )
-
-        self.expect("frame variable ii", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ii",
-            substrs=[
-                "map has 2 items",
-                "[0] = ",
-                "first = 0",
-                "second = 0",
-                "[1] = ",
-                "first = 1",
-                "second = 1",
-            ],
-        )
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ii",
-            substrs=[
-                "map has 4 items",
-                "[2] = ",
-                "first = 2",
-                "second = 0",
-                "[3] = ",
-                "first = 3",
-                "second = 1",
-            ],
-        )
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ii",
-            substrs=[
-                "map has 9 items",
-                "[5] = ",
-                "first = 5",
-                "second = 0",
-                "[7] = ",
-                "first = 7",
-                "second = 1",
-            ],
-        )
-
-        self.expect(
-            "expression ii",
-            substrs=[
-                "map has 9 items",
-                "[5] = ",
-                "first = 5",
-                "second = 0",
-                "[7] = ",
-                "first = 7",
-                "second = 1",
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable ii[0]", substrs=["first = 0", "second = 0"])
-        self.expect("frame variable ii[3]", substrs=["first =", "second ="])
-
-        self.expect("frame variable ii[8]", matching=True, substrs=["1234567"])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("ii").MightHaveChildren(),
-            "ii.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression ii[8]", matching=False, error=True,
-        #            substrs = ['1234567'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable ii", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("frame variable si --show-types")
-
-        self.expect("frame variable si", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable si",
-            substrs=["map has 1 items", "[0] = ", 'first = "zero"', "second = 0"],
-        )
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable si",
-            substrs=[
-                "map has 5 items",
-                '[0] = (first = "four", second = 4)',
-                '[1] = (first = "one", second = 1)',
-                '[2] = (first = "three", second = 3)',
-                '[3] = (first = "two", second = 2)',
-                '[4] = (first = "zero", second = 0)',
-            ],
-        )
-
-        self.expect(
-            "expression si",
-            substrs=[
-                "map has 5 items",
-                '[0] = (first = "four", second = 4)',
-                '[1] = (first = "one", second = 1)',
-                '[2] = (first = "three", second = 3)',
-                '[3] = (first = "two", second = 2)',
-                '[4] = (first = "zero", second = 0)',
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable si[0]", substrs=["first = ", "four", "second = 4"])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("si").MightHaveChildren(),
-            "si.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression si[0]", matching=False, error=True,
-        #            substrs = ['first = ', 'zero'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable si", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("frame variable is --show-types")
-
-        self.expect("frame variable is", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable is",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = 1, second = "is")',
-                '[1] = (first = 2, second = "smart")',
-                '[2] = (first = 3, second = "!!!")',
-                '[3] = (first = 85, second = "goofy")',
-            ],
-        )
-
-        self.expect(
-            "expression is",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = 1, second = "is")',
-                '[1] = (first = 2, second = "smart")',
-                '[2] = (first = 3, second = "!!!")',
-                '[3] = (first = 85, second = "goofy")',
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable is[0]", substrs=["first = ", "second ="])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("is").MightHaveChildren(),
-            "is.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression is[0]", matching=False, error=True,
-        #            substrs = ['first = ', 'goofy'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable is", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("frame variable ss --show-types")
-
-        self.expect("frame variable ss", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ss",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = "a Mac..", second = "..is always a Mac!")',
-                '[1] = (first = "casa", second = "house")',
-                '[2] = (first = "ciao", second = "hello")',
-                '[3] = (first = "gatto", second = "cat")',
-            ],
-        )
-
-        self.expect(
-            "expression ss",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = "a Mac..", second = "..is always a Mac!")',
-                '[1] = (first = "casa", second = "house")',
-                '[2] = (first = "ciao", second = "hello")',
-                '[3] = (first = "gatto", second = "cat")',
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable ss[3]", substrs=["gatto", "cat"])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("ss").MightHaveChildren(),
-            "ss.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression ss[3]", matching=False, error=True,
-        #            substrs = ['gatto'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable ss", substrs=["map has 0 items", "{}"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
deleted file mode 100644
index d5e5b212782d8..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-#include <map>
-#include <string>
-
-#define intint_map std::map<int, int> 
-#define strint_map std::map<std::string, int> 
-#define intstr_map std::map<int, std::string> 
-#define strstr_map std::map<std::string, std::string> 
-
-
-int main()
-{
-    intint_map ii;
-    
-    ii[0] = 0; // Set break point at this line.
-    ii[1] = 1;
-    ii[2] = 0;// Set break point at this line.
-    ii[3] = 1;
-    ii[4] = 0;// Set break point at this line.
-    ii[5] = 1;
-    ii[6] = 0;
-    ii[7] = 1;
-    ii[85] = 1234567;
-    
-    ii.clear();// Set break point at this line.
-    
-    strint_map si;
-    
-    si["zero"] = 0;// Set break point at this line.
-    si["one"] = 1;// Set break point at this line.
-    si["two"] = 2;
-    si["three"] = 3;
-    si["four"] = 4;
-
-    si.clear();// Set break point at this line.
-    
-    intstr_map is;
-    
-    is[85] = "goofy";// Set break point at this line.
-    is[1] = "is";
-    is[2] = "smart";
-    is[3] = "!!!";
-    
-    is.clear();// Set break point at this line.
-    
-    strstr_map ss;
-    
-    ss["ciao"] = "hello";// Set break point at this line.
-    ss["casa"] = "house";
-    ss["gatto"] = "cat";
-    ss["a Mac.."] = "..is always a Mac!";
-    
-    ss.clear();// Set break point at this line.
-    
-    return 0;// Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
deleted file mode 100644
index 654e4b15bd568..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CXXFLAGS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
deleted file mode 100644
index a6856177858a3..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdSmartPtrDataFormatterTestCase(TestBase):
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        self.expect("frame variable nsp", substrs=["nsp = nullptr"])
-        self.expect("frame variable isp", substrs=["isp = 123"])
-        self.expect("frame variable ssp", substrs=['ssp = "foobar"'])
-
-        self.expect("frame variable nwp", substrs=["nwp = nullptr"])
-        self.expect("frame variable iwp", substrs=["iwp = 123"])
-        self.expect("frame variable swp", substrs=['swp = "foobar"'])
-
-        self.expect("frame variable *nsp", substrs=["*nsp = <parent is NULL>"])
-        self.expect("frame variable *isp", substrs=["*isp = 123"])
-        self.expect("frame variable *ssp", substrs=['*ssp = "foobar"'])
-        self.expect("frame variable *fsp", substrs=["*fsp = (mem = 5)"])
-
-        self.expect("frame variable fsp->mem", substrs=["(int) fsp->mem = 5"])
-
-        self.runCmd("continue")
-
-        self.expect("frame variable nsp", substrs=["nsp = nullptr"])
-        self.expect("frame variable isp", substrs=["isp = nullptr"])
-        self.expect("frame variable ssp", substrs=["ssp = nullptr"])
-
-        self.expect("frame variable nwp", substrs=["nwp = nullptr"])
-        self.expect("frame variable iwp", substrs=["iwp = nullptr"])
-        self.expect("frame variable swp", substrs=["swp = nullptr"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
deleted file mode 100644
index 6e4b869e1c8e0..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
+++ /dev/null
@@ -1,26 +0,0 @@
-#include <memory>
-#include <string>
-
-struct Foo {
-  int mem = 5;
-};
-
-int
-main()
-{
-    std::shared_ptr<char> nsp;
-    std::shared_ptr<int> isp(new int{123});
-    std::shared_ptr<std::string> ssp = std::make_shared<std::string>("foobar");
-    std::shared_ptr<Foo> fsp = std::make_shared<Foo>();
-
-    std::weak_ptr<char> nwp;
-    std::weak_ptr<int> iwp = isp;
-    std::weak_ptr<std::string> swp = ssp;
-
-    nsp.reset(); // Set break point at this line.
-    isp.reset();
-    ssp.reset();
-    fsp.reset();
-
-    return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
deleted file mode 100644
index c825977b1a5dc..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CFLAGS_EXTRAS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
deleted file mode 100644
index 0f66fe89e67b3..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# coding=utf8
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdStringDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        """Test that that file and class static variables display correctly."""
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1
-        )
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        var_wempty = self.frame().FindVariable("wempty")
-        var_s = self.frame().FindVariable("s")
-        var_S = self.frame().FindVariable("S")
-        var_mazeltov = self.frame().FindVariable("mazeltov")
-        var_empty = self.frame().FindVariable("empty")
-        var_q = self.frame().FindVariable("q")
-        var_Q = self.frame().FindVariable("Q")
-        var_rq = self.frame().FindVariable("rq")
-        var_rQ = self.frame().FindVariable("rQ")
-        var_pq = self.frame().FindVariable("pq")
-        var_pQ = self.frame().FindVariable("pQ")
-
-        self.assertEqual(var_wempty.GetSummary(), 'L""', "wempty summary wrong")
-        self.assertEqual(
-            var_s.GetSummary(), 'L"hello world! מזל טוב!"', "s summary wrong"
-        )
-        self.assertEqual(var_S.GetSummary(), 'L"!!!!"', "S summary wrong")
-        self.assertEqual(
-            var_mazeltov.GetSummary(), 'L"מזל טוב"', "mazeltov summary wrong"
-        )
-        self.assertEqual(var_empty.GetSummary(), '""', "empty summary wrong")
-        self.assertEqual(var_q.GetSummary(), '"hello world"', "q summary wrong")
-        self.assertEqual(
-            var_Q.GetSummary(),
-            '"quite a long std::strin with lots of info inside it"',
-            "Q summary wrong",
-        )
-        self.assertEqual(var_rq.GetSummary(), '"hello world"', "rq summary wrong")
-        self.assertEqual(
-            var_rQ.GetSummary(),
-            '"quite a long std::strin with lots of info inside it"',
-            "rQ summary wrong",
-        )
-        self.assertEqual(var_pq.GetSummary(), '"hello world"', "pq summary wrong")
-        self.assertEqual(
-            var_pQ.GetSummary(),
-            '"quite a long std::strin with lots of info inside it"',
-            "pQ summary wrong",
-        )
-
-        self.runCmd("next")
-
-        self.assertEqual(var_S.GetSummary(), 'L"!!!!!"', "new S summary wrong")
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
deleted file mode 100644
index eefb96c4573e4..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
+++ /dev/null
@@ -1,16 +0,0 @@
-#include <string>
-
-int main()
-{
-    std::wstring wempty(L"");
-    std::wstring s(L"hello world! מזל טוב!");
-    std::wstring S(L"!!!!");
-    const wchar_t *mazeltov = L"מזל טוב";
-    std::string empty("");
-    std::string q("hello world");
-    std::string Q("quite a long std::strin with lots of info inside it");
-    auto &rq = q, &rQ = Q;
-    std::string *pq = &q, *pQ = &Q;
-    S.assign(L"!!!!!"); // Set break point at this line.
-    return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile
deleted file mode 100644
index bf8e6b8703f36..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py
deleted file mode 100644
index 1433b5bc1acb8..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py
+++ /dev/null
@@ -1,47 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdTupleDataFormatterTestCase(TestBase):
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        frame = self.frame()
-        self.assertTrue(frame.IsValid())
-
-        self.expect("frame variable ti", substrs=["[0] = 1"])
-        self.expect("frame variable ts", substrs=['[0] = "foobar"'])
-        self.expect("frame variable tt", substrs=["[0] = 1", '[1] = "baz"', "[2] = 2"])
-
-        self.assertEqual(1, frame.GetValueForVariablePath("ti[0]").GetValueAsUnsigned())
-        self.assertFalse(frame.GetValueForVariablePath("ti[1]").IsValid())
-
-        self.assertEqual(
-            '"foobar"', frame.GetValueForVariablePath("ts[0]").GetSummary()
-        )
-        self.assertFalse(frame.GetValueForVariablePath("ts[1]").IsValid())
-
-        self.assertEqual(1, frame.GetValueForVariablePath("tt[0]").GetValueAsUnsigned())
-        self.assertEqual('"baz"', frame.GetValueForVariablePath("tt[1]").GetSummary())
-        self.assertEqual(2, frame.GetValueForVariablePath("tt[2]").GetValueAsUnsigned())
-        self.assertFalse(frame.GetValueForVariablePath("tt[3]").IsValid())
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp
deleted file mode 100644
index 7247742ee6bb5..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp
+++ /dev/null
@@ -1,9 +0,0 @@
-#include <memory>
-#include <string>
-
-int main() {
-  std::tuple<int> ti{1};
-  std::tuple<std::string> ts{"foobar"};
-  std::tuple<int, std::string, int> tt{1, "baz", 2};
-  return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
deleted file mode 100644
index bf8e6b8703f36..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
deleted file mode 100644
index 2301e5edfbd90..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
+++ /dev/null
@@ -1,134 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdUniquePtrDataFormatterTestCase(TestBase):
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    @skipIf(oslist=["linux"], archs=["arm$", "aarch64"])
-    def test_with_run_command(self):
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        frame = self.frame()
-        self.assertTrue(frame.IsValid())
-
-        self.expect("frame variable nup", substrs=["nup = nullptr"])
-        self.expect("frame variable iup", substrs=["iup = 0x"])
-        self.expect("frame variable sup", substrs=["sup = 0x"])
-
-        self.expect("frame variable ndp", substrs=["ndp = nullptr"])
-        self.expect(
-            "frame variable idp", substrs=["idp = 0x", "deleter = ", "a = 1", "b = 2"]
-        )
-        self.expect(
-            "frame variable sdp", substrs=["sdp = 0x", "deleter = ", "a = 3", "b = 4"]
-        )
-
-        self.assertEqual(
-            123, frame.GetValueForVariablePath("iup.object").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            123, frame.GetValueForVariablePath("*iup").GetValueAsUnsigned()
-        )
-        self.assertFalse(frame.GetValueForVariablePath("iup.deleter").IsValid())
-
-        self.assertEqual(
-            '"foobar"', frame.GetValueForVariablePath("sup.object").GetSummary()
-        )
-        self.assertEqual('"foobar"', frame.GetValueForVariablePath("*sup").GetSummary())
-        self.assertFalse(frame.GetValueForVariablePath("sup.deleter").IsValid())
-
-        self.assertEqual(
-            456, frame.GetValueForVariablePath("idp.object").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            456, frame.GetValueForVariablePath("*idp").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            '"baz"', frame.GetValueForVariablePath("sdp.object").GetSummary()
-        )
-        self.assertEqual('"baz"', frame.GetValueForVariablePath("*sdp").GetSummary())
-
-        idp_deleter = frame.GetValueForVariablePath("idp.deleter")
-        self.assertTrue(idp_deleter.IsValid())
-        self.assertEqual(
-            1, idp_deleter.GetChildMemberWithName("a").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            2, idp_deleter.GetChildMemberWithName("b").GetValueAsUnsigned()
-        )
-
-        sdp_deleter = frame.GetValueForVariablePath("sdp.deleter")
-        self.assertTrue(sdp_deleter.IsValid())
-        self.assertEqual(
-            3, sdp_deleter.GetChildMemberWithName("a").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            4, sdp_deleter.GetChildMemberWithName("b").GetValueAsUnsigned()
-        )
-
-    @skipIfFreeBSD
-    @skipIfWindows  # libstdcpp not ported to Windows
-    @skipIfDarwin  # doesn't compile on Darwin
-    @skipIfwatchOS  # libstdcpp not ported to watchos
-    @skipIf(oslist=["linux"], archs=["arm$", "aarch64"])
-    @add_test_categories(["libstdcxx"])
-    def test_recursive_unique_ptr(self):
-        # Tests that LLDB can handle when we have a loop in the unique_ptr
-        # reference chain and that it correctly handles the different options
-        # for the frame variable command in this case.
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        self.expect("frame variable f1->fp", substrs=["fp = 0x"])
-        self.expect(
-            "frame variable --ptr-depth=1 f1->fp", substrs=["data = 2", "fp = 0x"]
-        )
-        self.expect(
-            "frame variable --ptr-depth=2 f1->fp",
-            substrs=["data = 2", "fp = 0x", "data = 1"],
-        )
-
-        frame = self.frame()
-        self.assertTrue(frame.IsValid())
-        self.assertEqual(
-            2, frame.GetValueForVariablePath("f1->fp.object.data").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            2, frame.GetValueForVariablePath("f1->fp->data").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            1,
-            frame.GetValueForVariablePath(
-                "f1->fp.object.fp.object.data"
-            ).GetValueAsUnsigned(),
-        )
-        self.assertEqual(
-            1, frame.GetValueForVariablePath("f1->fp->fp->data").GetValueAsUnsigned()
-        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
deleted file mode 100644
index 9cacfce989769..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import lldbsuite.test.lldbinline as lldbinline
-from lldbsuite.test.decorators import *
-
-lldbinline.MakeInlineTest(__file__, globals(), [no_debug_info_test])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
deleted file mode 100644
index b12cab231695b..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
+++ /dev/null
@@ -1,11 +0,0 @@
-// Test that we don't crash when trying to pretty-print structures that don't
-// have the layout our data formatters expect.
-namespace std {
-template<typename T, typename Deleter = void>
-class unique_ptr {};
-}
-
-int main() {
-  std::unique_ptr<int> U;
-  return 0; //% self.expect("frame variable U", substrs=["unique_ptr", "{}"])
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
deleted file mode 100644
index dd0072764d4e6..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-#include <memory>
-#include <string>
-
-struct Deleter {
-  void operator()(void *) {}
-
-  int a;
-  int b;
-};
-
-struct Foo {
-  int data;
-  std::unique_ptr<Foo> fp;
-};
-
-int main() {
-  std::unique_ptr<char> nup;
-  std::unique_ptr<int> iup(new int{123});
-  std::unique_ptr<std::string> sup(new std::string("foobar"));
-
-  std::unique_ptr<char, Deleter> ndp;
-  std::unique_ptr<int, Deleter> idp(new int{456}, Deleter{1, 2});
-  std::unique_ptr<std::string, Deleter> sdp(new std::string("baz"),
-                                            Deleter{3, 4});
-
-  std::unique_ptr<Foo> fp(new Foo{3});
-
-  // Set up a structure where we have a loop in the unique_ptr chain.
-  Foo* f1 = new Foo{1};
-  Foo* f2 = new Foo{2};
-  f1->fp.reset(f2);
-  f2->fp.reset(f1);
-
-  return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
deleted file mode 100644
index c825977b1a5dc..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CFLAGS_EXTRAS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
deleted file mode 100644
index 73956dd3fda31..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-#include <vector>
-
-int main()
-{
-    std::vector<bool> vBool;
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(true);
-
-    return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
deleted file mode 100644
index 654e4b15bd568..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CXXFLAGS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
deleted file mode 100644
index 6cfd17a39304e..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
+++ /dev/null
@@ -1,220 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-from typing import Optional
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdVectorDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.with_run_command()
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command_debug(self):
-        build_args = {"CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
-        self.with_run_command(build_args)
-
-    def with_run_command(self, dictionary: Optional[dict] = None):
-        """Test that that file and class static variables display correctly."""
-        self.build(dictionary=dictionary)
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        # empty vectors (and storage pointers SHOULD BOTH BE NULL..)
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-        self.runCmd("c")
-
-        # first value added
-        self.expect(
-            "frame variable numbers", substrs=["numbers = size=1", "[0] = 1", "}"]
-        )
-
-        # add some more data
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable numbers",
-            substrs=[
-                "numbers = size=4",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "}",
-            ],
-        )
-
-        self.expect(
-            "expression numbers",
-            substrs=[
-                "$",
-                "size=4",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "}",
-            ],
-        )
-
-        # check access to synthetic children
-        self.runCmd(
-            'type summary add --summary-string "item 0 is ${var[0]}" std::int_vect int_vect'
-        )
-        self.expect("frame variable numbers", substrs=["item 0 is 1"])
-
-        self.runCmd(
-            'type summary add --summary-string "item 0 is ${svar[0]}" std::int_vect int_vect'
-        )
-        # import time
-        # time.sleep(19)
-        self.expect("frame variable numbers", substrs=["item 0 is 1"])
-        # move on with synths
-        self.runCmd("type summary delete std::int_vect")
-        self.runCmd("type summary delete int_vect")
-
-        # add some more data
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable numbers",
-            substrs=[
-                "numbers = size=7",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "[4] = 12345",
-                "[5] = 123456",
-                "[6] = 1234567",
-                "}",
-            ],
-        )
-
-        self.expect(
-            "expression numbers",
-            substrs=[
-                "$",
-                "size=7",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "[4] = 12345",
-                "[5] = 123456",
-                "[6] = 1234567",
-                "}",
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable numbers[0]", substrs=["1"])
-        self.expect("frame variable numbers[1]", substrs=["12"])
-        self.expect("frame variable numbers[2]", substrs=["123"])
-        self.expect("frame variable numbers[3]", substrs=["1234"])
-
-        # but check that expression does not rely on us
-        # (when expression gets to call into STL code correctly, we will have to find
-        # another way to check this)
-        self.expect(
-            "expression numbers[6]", matching=False, error=True, substrs=["1234567"]
-        )
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("numbers").MightHaveChildren(),
-            "numbers.MightHaveChildren() says False for non empty!",
-        )
-
-        # clear out the vector and see that we do the right thing once again
-        self.runCmd("c")
-
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-        self.runCmd("c")
-
-        # first value added
-        self.expect(
-            "frame variable numbers", substrs=["numbers = size=1", "[0] = 7", "}"]
-        )
-
-        # check if we can display strings
-        self.runCmd("c")
-
-        self.expect("frame variable strings", substrs=["goofy", "is", "smart"])
-
-        self.expect("expression strings", substrs=["goofy", "is", "smart"])
-
-        # test summaries based on synthetic children
-        self.runCmd(
-            'type summary add std::string_vect string_vect --summary-string "vector has ${svar%#} items" -e'
-        )
-        self.expect(
-            "frame variable strings",
-            substrs=["vector has 3 items", "goofy", "is", "smart"],
-        )
-
-        self.expect(
-            "expression strings", substrs=["vector has 3 items", "goofy", "is", "smart"]
-        )
-
-        self.runCmd("c")
-
-        self.expect("frame variable strings", substrs=["vector has 4 items"])
-
-        # check access-by-index
-        self.expect("frame variable strings[0]", substrs=["goofy"])
-        self.expect("frame variable strings[1]", substrs=["is"])
-
-        # but check that expression does not rely on us
-        # (when expression gets to call into STL code correctly, we will have to find
-        # another way to check this)
-        self.expect(
-            "expression strings[0]", matching=False, error=True, substrs=["goofy"]
-        )
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("strings").MightHaveChildren(),
-            "strings.MightHaveChildren() says False for non empty!",
-        )
-
-        self.runCmd("c")
-
-        self.expect("frame variable strings", substrs=["vector has 0 items"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
deleted file mode 100644
index 010917995e401..0000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-#include <string>
-#include <vector>
-typedef std::vector<int> int_vect;
-typedef std::vector<std::string> string_vect;
-
-int main()
-{
-    int_vect numbers;
-    numbers.push_back(1);  // Set break point at this line.
-    numbers.push_back(12);  // Set break point at this line.
-    numbers.push_back(123);
-    numbers.push_back(1234);
-    numbers.push_back(12345);  // Set break point at this line.
-    numbers.push_back(123456);
-    numbers.push_back(1234567);
-    
-    numbers.clear();  // Set break point at this line.
-    
-    numbers.push_back(7);  // Set break point at this line.
-
-    string_vect strings;  // Set break point at this line.
-    strings.push_back(std::string("goofy"));
-    strings.push_back(std::string("is"));
-    strings.push_back(std::string("smart"));
-    
-    strings.push_back(std::string("!!!"));  // Set break point at this line.
-    
-    strings.clear();  // Set break point at this line.
-    
-    return 0;// Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/TestDataFormatterMsvcStlString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/TestDataFormatterMsvcStlString.py
new file mode 100644
index 0000000000000..e49ea84dbdf47
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/TestDataFormatterMsvcStlString.py
@@ -0,0 +1,120 @@
+# coding=utf8
+"""
+Test std::*string summaries with MSVC's STL.
+"""
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class MsvcStlStringDataFormatterTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @add_test_categories(["msvcstl"])
+    def test_with_run_command(self):
+        """Test that that file and class static variables display correctly."""
+        self.build()
+
+        main_spec = lldb.SBFileSpec("main.cpp")
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Set break point at this line.", main_spec
+        )
+        frame = thread.frames[0]
+
+        # This is the function to remove the custom formats in order to have a
+        # clean slate for the next test case.
+        def cleanup():
+            self.runCmd("type format clear", check=False)
+            self.runCmd("type summary clear", check=False)
+            self.runCmd("type filter clear", check=False)
+            self.runCmd("type synth clear", check=False)
+
+        # Execute the cleanup function during test case tear down.
+        self.addTearDownHook(cleanup)
+
+        self.expect(
+            "frame variable",
+            substrs=[
+                '(std::wstring) wempty = L""',
+                '(std::wstring) s = L"hello world! מזל טוב!"',
+                '(std::wstring) S = L"!!!!"',
+                "(const wchar_t *) mazeltov = 0x",
+                'L"מזל טוב"',
+                '(std::string) empty = ""',
+                '(std::string) q = "hello world"',
+                '(std::string) Q = "quite a long std::strin with lots of info inside it"',
+                '(std::string) overwritten_zero = "abc"',
+                '(std::string) IHaveEmbeddedZeros = "a\\0b\\0c\\0d"',
+                '(std::wstring) IHaveEmbeddedZerosToo = L"hello world!\\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監"',
+                '(std::u16string) u16_string = u"ß水氶"',
+                '(std::u16string) u16_empty = u""',
+                '(std::u32string) u32_string = U"🍄🍅🍆🍌"',
+                '(std::u32string) u32_empty = U""',
+                "(std::string *) null_str = nullptr",
+            ],
+        )
+
+        thread.StepOver()
+
+        TheVeryLongOne = frame.FindVariable("TheVeryLongOne")
+        summaryOptions = lldb.SBTypeSummaryOptions()
+        summaryOptions.SetCapping(lldb.eTypeSummaryUncapped)
+        uncappedSummaryStream = lldb.SBStream()
+        TheVeryLongOne.GetSummary(uncappedSummaryStream, summaryOptions)
+        uncappedSummary = uncappedSummaryStream.GetData()
+        self.assertGreater(
+            uncappedSummary.find("someText"),
+            0,
+            "uncappedSummary does not include the full string",
+        )
+        summaryOptions.SetCapping(lldb.eTypeSummaryCapped)
+        cappedSummaryStream = lldb.SBStream()
+        TheVeryLongOne.GetSummary(cappedSummaryStream, summaryOptions)
+        cappedSummary = cappedSummaryStream.GetData()
+        self.assertLessEqual(
+            cappedSummary.find("someText"), 0, "cappedSummary includes the full string"
+        )
+
+        self.expect_expr(
+            "s", result_type="std::wstring", result_summary='L"hello world! מזל טוב!"'
+        )
+
+        self.expect_expr("q", result_type="std::string", result_summary='"hello world"')
+
+        self.expect_expr(
+            "Q",
+            result_type="std::string",
+            result_summary='"quite a long std::strin with lots of info inside it"',
+        )
+
+        self.expect(
+            "frame variable",
+            substrs=[
+                '(std::wstring) S = L"!!!!!"',
+                "(const wchar_t *) mazeltov = 0x",
+                'L"מזל טוב"',
+                '(std::string) q = "hello world"',
+                '(std::string) Q = "quite a long std::strin with lots of info inside it"',
+                '(std::string) IHaveEmbeddedZeros = "a\\0b\\0c\\0d"',
+                '(std::wstring) IHaveEmbeddedZerosToo = L"hello world!\\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監"',
+                '(std::u16string) u16_string = u"ß水氶"',
+                '(std::u32string) u32_string = U"🍄🍅🍆🍌"',
+                '(std::u32string) u32_empty = U""',
+                "(std::string *) null_str = nullptr",
+            ],
+        )
+
+        # Finally, make sure that if the string is not readable, we give an error:
+        bkpt_2 = target.BreakpointCreateBySourceRegex(
+            "Break here to look at bad string", main_spec
+        )
+        self.assertEqual(bkpt_2.GetNumLocations(), 1, "Got one location")
+        threads = lldbutil.continue_to_breakpoint(process, bkpt_2)
+        self.assertEqual(len(threads), 1, "Stopped at second breakpoint")
+        frame = threads[0].frames[0]
+        var = frame.FindVariable("in_str")
+        self.assertTrue(var.GetError().Success(), "Made variable")
+        self.assertIsNone(var.GetSummary())
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/main.cpp
new file mode 100644
index 0000000000000..fcfb5d48e9bb1
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/string/main.cpp
@@ -0,0 +1,40 @@
+#include <stdint.h>
+
+#include <string>
+
+#ifndef _MSVC_STL_VERSION
+// this is more of a sanity check that the categories work as expected
+#error Not using MSVC STL
+#endif
+
+static size_t touch_string(std::string &in_str) {
+  return in_str.size(); // Break here to look at bad string
+}
+
+int main() {
+  std::wstring wempty(L"");
+  std::wstring s(L"hello world! מזל טוב!");
+  std::wstring S(L"!!!!");
+  const wchar_t *mazeltov = L"מזל טוב";
+  std::string empty("");
+  std::string q("hello world");
+  std::string Q("quite a long std::strin with lots of info inside it");
+  std::string overwritten_zero("abc");
+  const_cast<char *>(overwritten_zero.data())[3] = 'd';
+  // clang-format off
+  std::string TheVeryLongOne("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890someText1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890");
+  // clang-format on
+  std::string IHaveEmbeddedZeros("a\0b\0c\0d", 7);
+  std::wstring IHaveEmbeddedZerosToo(
+      L"hello world!\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監", 38);
+  std::u16string u16_string(u"ß水氶");
+  std::u16string u16_empty(u"");
+  std::u32string u32_string(U"🍄🍅🍆🍌");
+  std::u32string u32_empty(U"");
+  std::string *null_str = nullptr;
+
+  S.assign(L"!!!!!"); // Set break point at this line.
+  std::string *not_a_string = (std::string *)0x0;
+  touch_string(*not_a_string);
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/Makefile
new file mode 100644
index 0000000000000..58558e6e15f78
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/Makefile
@@ -0,0 +1,4 @@
+CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -std=c++20 -O0
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/TestDataFormatterMsvcStlU8String.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/TestDataFormatterMsvcStlU8String.py
new file mode 100644
index 0000000000000..f181f520f5d85
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/TestDataFormatterMsvcStlU8String.py
@@ -0,0 +1,33 @@
+# coding=utf8
+"""
+Test std::u8string summary with MSVC's STL.
+"""
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class MsvcStlU8StringDataFormatterTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @add_test_categories(["msvcstl"])
+    def test_with_run_command(self):
+        """Test that that file and class static variables display correctly."""
+        self.build()
+
+        lldbutil.run_to_source_breakpoint(
+            self, "Set break point at this line.", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.expect(
+            "frame variable",
+            substrs=[
+                '(std::u8string) u8_string_small = u8"🍄"',
+                '(std::u8string) u8_string = u8"❤️👍📄📁😃🧑‍🌾"',
+                '(std::u8string) u8_empty = u8""',
+                '(std::u8string) u8_text = u8"ABC"',
+            ],
+        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/main.cpp
new file mode 100644
index 0000000000000..e01af9fa08e7e
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/msvcstl/u8string/main.cpp
@@ -0,0 +1,14 @@
+#include <string>
+
+#ifndef _MSVC_STL_VERSION
+// this is more of a sanity check that the categories work as expected
+#error Not using MSVC STL
+#endif
+
+int main() {
+  std::u8string u8_string_small(u8"🍄");
+  std::u8string u8_string(u8"❤️👍📄📁😃🧑‍🌾");
+  std::u8string u8_empty(u8"");
+  std::u8string u8_text(u8"ABC");
+  u8_text.assign(u8"ABCd"); // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/data-formatter/stringprinter/TestStringPrinter.py b/lldb/test/API/functionalities/data-formatter/stringprinter/TestStringPrinter.py
index 27285949c52bc..0e3bfc2733a56 100644
--- a/lldb/test/API/functionalities/data-formatter/stringprinter/TestStringPrinter.py
+++ b/lldb/test/API/functionalities/data-formatter/stringprinter/TestStringPrinter.py
@@ -4,5 +4,4 @@
 lldbinline.MakeInlineTest(
     __file__,
     globals(),
-    [decorators.expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24772")],
 )
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py b/lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py
new file mode 100644
index 0000000000000..dd03a0cc836a7
--- /dev/null
+++ b/lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py
@@ -0,0 +1,52 @@
+"""
+Test evaluating expressions when debugging core file.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+@skipIfLLVMTargetMissing("X86")
+class CoreExprTestCase(TestBase):
+    def setUp(self):
+        TestBase.setUp(self)
+        self.target = self.dbg.CreateTarget("linux-x86_64.out")
+        self.process = self.target.LoadCore("linux-x86_64.core")
+        self.assertTrue(self.process, PROCESS_IS_VALID)
+
+    def test_result_var(self):
+        """Test that the result variable can be used in subsequent expressions."""
+
+        self.expect_expr(
+            "outer",
+            result_type="Outer",
+            result_children=[ValueCheck(name="inner", type="Inner")],
+        )
+        self.expect_expr(
+            "$0.inner",
+            result_type="Inner",
+            result_children=[ValueCheck(name="val", type="int", value="5")],
+        )
+        self.expect_expr("$1.val", result_type="int", result_value="5")
+
+    def test_persist_var(self):
+        """Test that user-defined variables can be used in subsequent expressions."""
+
+        self.target.EvaluateExpression("int $my_int = 5")
+        self.expect_expr("$my_int * 2", result_type="int", result_value="10")
+
+    def test_context_object(self):
+        """Test expression evaluation in context of an object."""
+
+        val_outer = self.expect_expr("outer", result_type="Outer")
+
+        val_inner = val_outer.EvaluateExpression("inner")
+        self.assertTrue(val_inner.IsValid())
+        self.assertEqual("Inner", val_inner.GetDisplayTypeName())
+
+        val_val = val_inner.EvaluateExpression("this->val")
+        self.assertTrue(val_val.IsValid())
+        self.assertEqual("int", val_val.GetDisplayTypeName())
+        self.assertEqual(val_val.GetValueAsSigned(), 5)
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.core b/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.core
new file mode 100644
index 0000000000000..3bd2916c64e9f
Binary files /dev/null and b/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.core differ
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.out b/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.out
new file mode 100755
index 0000000000000..33e60f025d210
Binary files /dev/null and b/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.out differ
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp b/lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp
new file mode 100644
index 0000000000000..32a9c63e9b716
--- /dev/null
+++ b/lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp
@@ -0,0 +1,15 @@
+struct Inner {
+  Inner(int val) : val(val) {}
+  int val;
+};
+
+struct Outer {
+  Outer(int val) : inner(val) {}
+  Inner inner;
+};
+
+extern "C" void _start(void) {
+  Outer outer(5);
+  char *boom = (char *)0;
+  *boom = 47;
+}
diff --git a/lldb/test/API/functionalities/statusline/TestStatusline.py b/lldb/test/API/functionalities/statusline/TestStatusline.py
index 68c5ad3090cfd..087e62b387d77 100644
--- a/lldb/test/API/functionalities/statusline/TestStatusline.py
+++ b/lldb/test/API/functionalities/statusline/TestStatusline.py
@@ -27,10 +27,12 @@ def do_setup(self):
         self.expect("run", substrs=["stop reason"])
         self.resize()
 
-    def resize(self):
+    def resize(self, height=None, width=None):
+        height = self.TERMINAL_HEIGHT if not height else height
+        width = self.TERMINAL_WIDTH if not width else width
         # Change the terminal dimensions. When we launch the tests, we reset
         # all the settings, leaving the terminal dimensions unset.
-        self.child.setwinsize(self.TERMINAL_HEIGHT, self.TERMINAL_WIDTH)
+        self.child.setwinsize(height, width)
 
     def test(self):
         """Basic test for the statusline."""
@@ -65,9 +67,7 @@ def test(self):
         self.expect('set set separator "| "')
 
         # Hide the statusline and check or the control character.
-        self.expect(
-            "set set show-statusline false", ["\x1b[1;{}r".format(self.TERMINAL_HEIGHT)]
-        )
+        self.expect("set set show-statusline false", ["\x1b[1;0r"])
 
     def test_no_color(self):
         """Basic test for the statusline with colors disabled."""
@@ -104,3 +104,14 @@ def test_no_target(self):
         self.resize()
 
         self.expect("set set show-statusline true", ["no target"])
+
+    @skipIfEditlineSupportMissing
+    def test_resize(self):
+        """Test that move the cursor when resizing."""
+        self.launch(timeout=self.TIMEOUT)
+        self.resize()
+        self.expect("set set show-statusline true", ["no target"])
+        self.resize(20, 60)
+        # Check for the escape code to resize the scroll window.
+        self.child.expect(re.escape("\x1b[1;19r"))
+        self.child.expect("(lldb)")
diff --git a/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py b/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
index 4eb5351eefc82..08f09b317b217 100644
--- a/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
+++ b/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
@@ -24,9 +24,7 @@ def test_without_process(self):
 
         self.expect_expr("a", result_type="char8_t", result_summary="0x61 u8'a'")
         self.expect_expr("ab", result_type="const char8_t *", result_summary='u8"你好"')
-
-        # FIXME: This should work too.
-        self.expect("expr abc", substrs=['u8"你好"'], matching=False)
+        self.expect_expr("abc", result_type="char8_t[9]", result_summary='u8"你好"')
 
     @skipIf(compiler="clang", compiler_version=["<", "7.0"])
     def test_with_process(self):
diff --git a/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py b/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py
index 4190ea3ac3318..6754288a65e1a 100644
--- a/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py
+++ b/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py
@@ -1,6 +1,5 @@
 """Test that Mach-O armv7/arm64 corefile register contexts are read by lldb."""
 
-
 import os
 import re
 import subprocess
@@ -44,6 +43,11 @@ def test_armv7_corefile(self):
         self.assertTrue(exception.IsValid())
         self.assertEqual(exception.GetValueAsUnsigned(), 0x00003F5C)
 
+        # read 4 bytes starting at $sp-1 (an odd/unaligned address on this arch),
+        # formatted hex.
+        # aka `mem read -f x -s 1 -c 4 $sp-1`
+        self.expect("x/4bx $sp-1", substrs=["0x000dffff", "0x1f 0x20 0x21 0x22"])
+
     def test_arm64_corefile(self):
         ### Create corefile
         retcode = call(self.create_corefile + " arm64 " + self.corefile, shell=True)
diff --git a/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp b/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp
index 5517a2397ae52..db39f12ecfb7e 100644
--- a/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp
+++ b/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp
@@ -10,15 +10,30 @@
 // only defines the ARM register context constants when building on
 // an arm system.  We're creating fake corefiles, and might be
 // creating them on an intel system.
+#ifndef ARM_THREAD_STATE
 #define ARM_THREAD_STATE 1
+#endif
+#ifndef ARM_THREAD_STATE_COUNT
 #define ARM_THREAD_STATE_COUNT 17
+#endif
+#ifndef ARM_EXCEPTION_STATE
 #define ARM_EXCEPTION_STATE 3
+#endif
+#ifndef ARM_EXCEPTION_STATE_COUNT
 #define ARM_EXCEPTION_STATE_COUNT 3
+#endif
+#ifndef ARM_THREAD_STATE64
 #define ARM_THREAD_STATE64 6
+#endif
+#ifndef ARM_THREAD_STATE64_COUNT
 #define ARM_THREAD_STATE64_COUNT 68
+#endif
+#ifndef ARM_EXCEPTION_STATE64
 #define ARM_EXCEPTION_STATE64 7
+#endif
+#ifndef ARM_EXCEPTION_STATE64_COUNT
 #define ARM_EXCEPTION_STATE64_COUNT 4
-
+#endif
 
 union uint32_buf {
   uint8_t bytebuf[4];
@@ -129,6 +144,27 @@ std::vector<uint8_t> arm64_lc_thread_load_command() {
   return data;
 }
 
+std::vector<uint8_t> lc_segment(uint32_t fileoff,
+                                uint32_t lc_segment_data_size) {
+  std::vector<uint8_t> data;
+  // 0x000e0000 is the value of $sp in the armv7 LC_THREAD
+  uint32_t start_vmaddr = 0x000e0000 - (lc_segment_data_size / 2);
+  add_uint32(data, LC_SEGMENT);                     // segment_command.cmd
+  add_uint32(data, sizeof(struct segment_command)); // segment_command.cmdsize
+  for (int i = 0; i < 16; i++)
+    data.push_back(0);                    // segment_command.segname[16]
+  add_uint32(data, start_vmaddr);         // segment_command.vmaddr
+  add_uint32(data, lc_segment_data_size); // segment_command.vmsize
+  add_uint32(data, fileoff);              // segment_command.fileoff
+  add_uint32(data, lc_segment_data_size); // segment_command.filesize
+  add_uint32(data, 3);                    // segment_command.maxprot
+  add_uint32(data, 3);                    // segment_command.initprot
+  add_uint32(data, 0);                    // segment_command.nsects
+  add_uint32(data, 0);                    // segment_command.flags
+
+  return data;
+}
+
 enum arch { unspecified, armv7, arm64 };
 
 int main(int argc, char **argv) {
@@ -157,10 +193,12 @@ int main(int argc, char **argv) {
 
   // First add all the load commands / payload so we can figure out how large
   // the load commands will actually be.
-  if (arch == armv7)
+  if (arch == armv7) {
     load_commands.push_back(armv7_lc_thread_load_command());
-  else if (arch == arm64)
+    load_commands.push_back(lc_segment(0, 0));
+  } else if (arch == arm64) {
     load_commands.push_back(arm64_lc_thread_load_command());
+  }
 
   int size_of_load_commands = 0;
   for (const auto &lc : load_commands)
@@ -174,19 +212,33 @@ int main(int argc, char **argv) {
   load_commands.clear();
   payload.clear();
 
-  if (arch == armv7)
+  int payload_fileoff = (header_and_load_cmd_room + 4096 - 1) & ~(4096 - 1);
+
+  const int lc_segment_data_size = 64;
+  if (arch == armv7) {
     load_commands.push_back(armv7_lc_thread_load_command());
-  else if (arch == arm64)
+    load_commands.push_back(lc_segment(payload_fileoff, lc_segment_data_size));
+  } else if (arch == arm64) {
     load_commands.push_back(arm64_lc_thread_load_command());
+  }
+
+  if (arch == armv7)
+    for (int i = 0; i < lc_segment_data_size;
+         i++) // from segment_command.filesize
+      payload.push_back(i);
 
   struct mach_header_64 mh;
-  mh.magic = MH_MAGIC_64;
+  int header_size;
   if (arch == armv7) {
+    mh.magic = MH_MAGIC;
     mh.cputype = CPU_TYPE_ARM;
     mh.cpusubtype = CPU_SUBTYPE_ARM_V7M;
+    header_size = sizeof(struct mach_header);
   } else if (arch == arm64) {
+    mh.magic = MH_MAGIC_64;
     mh.cputype = CPU_TYPE_ARM64;
     mh.cpusubtype = CPU_SUBTYPE_ARM64_ALL;
+    header_size = sizeof(struct mach_header_64);
   }
   mh.filetype = MH_CORE;
   mh.ncmds = load_commands.size();
@@ -201,12 +253,12 @@ int main(int argc, char **argv) {
     exit(1);
   }
 
-  fwrite(&mh, sizeof(struct mach_header_64), 1, f);
+  fwrite(&mh, header_size, 1, f);
 
   for (const auto &lc : load_commands)
     fwrite(lc.data(), lc.size(), 1, f);
 
-  fseek(f, header_and_load_cmd_room, SEEK_SET);
+  fseek(f, payload_fileoff, SEEK_SET);
 
   fwrite(payload.data(), payload.size(), 1, f);
 
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/Makefile b/lldb/test/API/macosx/lc-note/additional-registers/Makefile
new file mode 100644
index 0000000000000..e232ec5d69675
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/Makefile
@@ -0,0 +1,11 @@
+MAKE_DSYM := NO
+C_SOURCES := main.c
+CXXFLAGS_EXTRAS := -std=c++17
+
+all: a.out add-lcnote
+
+add-lcnote:
+	"$(MAKE)" -f "$(MAKEFILE_RULES)" EXE=add-lcnote \
+	    CXX=$(CC) CXXFLAGS_EXTRAS="$(CXXFLAGS_EXTRAS)" CXX_SOURCES=add-lcnote.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py b/lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py
new file mode 100644
index 0000000000000..cf9f055bd78fd
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py
@@ -0,0 +1,150 @@
+"""Test that lldb will read additional registers from Mach-O LC_NOTE metadata."""
+
+import os
+import re
+import subprocess
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestMetadataRegisters(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipUnlessDarwin
+    @skipIfRemote
+    def test_add_registers_via_metadata(self):
+        self.build()
+        self.aout_exe = self.getBuildArtifact("a.out")
+        lldb_corefile = self.getBuildArtifact("lldb.core")
+        metadata_corefile = self.getBuildArtifact("metadata.core")
+        bad_metadata1_corefile = self.getBuildArtifact("bad-metadata1.core")
+        bad_metadata2_corefile = self.getBuildArtifact("bad-metadata2.core")
+        add_lcnote = self.getBuildArtifact("add-lcnote")
+
+        (target, process, t, bp) = lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.c")
+        )
+
+        self.assertTrue(process.IsValid())
+
+        if self.TraceOn():
+            self.runCmd("bt")
+            self.runCmd("reg read -a")
+
+        self.runCmd("process save-core " + lldb_corefile)
+        process.Kill()
+        target.Clear()
+
+        cmd = (
+            add_lcnote
+            + " -r"
+            + " -i '%s'" % lldb_corefile
+            + " -o '%s'" % metadata_corefile
+            + " -n 'process metadata' "
+            + " -s '"
+            + """{"threads":[{"register_info":
+                   {"sets":["Special Registers", "General Purpose Registers"],
+                    "registers":[
+                       {"name":"jar", "value":10, "bitsize": 32, "set": 0},
+                       {"name":"bar", "value":65537, "bitsize":16, "set":0},
+                       {"name":"mar", "value":65537, "bitsize":32, "set":0},
+                       {"name":"anotherpc", "value":55, "bitsize":64, "set": 1}]}}]}"""
+            + "'"
+        )
+        call(cmd, shell=True)
+
+        cmd = (
+            add_lcnote
+            + " -r"
+            + " -i '%s'" % lldb_corefile
+            + " -o '%s'" % bad_metadata1_corefile
+            + " -n 'process metadata' "
+            + " -s '"
+            + """{lol im bad json}"""
+            + "'"
+        )
+        call(cmd, shell=True)
+
+        cmd = (
+            add_lcnote
+            + " -r"
+            + " -i '%s'" % lldb_corefile
+            + " -o '%s'" % bad_metadata2_corefile
+            + " -n 'process metadata' "
+            + " -s '"
+            + """{"threads":[
+                   {"register_info":
+                     {"sets":["a"],"registers":[{"name":"a", "value":1, "bitsize": 32, "set": 0}]}},
+                   {"register_info":
+                     {"sets":["a"],"registers":[{"name":"a", "value":1, "bitsize": 32, "set": 0}]}}
+                 ]}"""
+            + "'"
+        )
+        call(cmd, shell=True)
+
+        # Now load the corefile
+        target = self.dbg.CreateTarget("")
+        process = target.LoadCore(metadata_corefile)
+        self.assertTrue(process.IsValid())
+        if self.TraceOn():
+            self.runCmd("bt")
+            self.runCmd("reg read -a")
+
+        thread = process.GetSelectedThread()
+        frame = thread.GetFrameAtIndex(0)
+
+        # Register sets will be
+        #    from LC_THREAD:
+        #       General Purpose Registers
+        #       Floating Point Registers
+        #       Exception State Registers
+        #    from LC_NOTE metadata:
+        #       Special Registers
+        self.assertEqual(frame.registers[0].GetName(), "General Purpose Registers")
+        self.assertEqual(frame.registers[3].GetName(), "Special Registers")
+
+        anotherpc = frame.registers[0].GetChildMemberWithName("anotherpc")
+        self.assertTrue(anotherpc.IsValid())
+        self.assertEqual(anotherpc.GetValueAsUnsigned(), 0x37)
+
+        jar = frame.registers[3].GetChildMemberWithName("jar")
+        self.assertTrue(jar.IsValid())
+        self.assertEqual(jar.GetValueAsUnsigned(), 10)
+        self.assertEqual(jar.GetByteSize(), 4)
+
+        bar = frame.registers[3].GetChildMemberWithName("bar")
+        self.assertTrue(bar.IsValid())
+        self.assertEqual(bar.GetByteSize(), 2)
+
+        mar = frame.registers[3].GetChildMemberWithName("mar")
+        self.assertTrue(mar.IsValid())
+        self.assertEqual(mar.GetValueAsUnsigned(), 0x10001)
+        self.assertEqual(mar.GetByteSize(), 4)
+
+        # bad1 has invalid JSON, no additional registers
+        target = self.dbg.CreateTarget("")
+        process = target.LoadCore(bad_metadata1_corefile)
+        self.assertTrue(process.IsValid())
+        thread = process.GetSelectedThread()
+        frame = thread.GetFrameAtIndex(0)
+        gpr_found = False
+        for regset in frame.registers:
+            if regset.GetName() == "General Purpose Registers":
+                gpr_found = True
+        self.assertTrue(gpr_found)
+
+        # bad2 has invalid JSON, more process metadata threads than
+        # LC_THREADs, and should be rejected.
+        target = self.dbg.CreateTarget("")
+        process = target.LoadCore(bad_metadata2_corefile)
+        self.assertTrue(process.IsValid())
+        thread = process.GetSelectedThread()
+        frame = thread.GetFrameAtIndex(0)
+        metadata_regset_found = False
+        for regset in frame.registers:
+            if regset.GetName() == "a" or regset.GetName() == "b":
+                metadata_regset_found = True
+        self.assertFalse(metadata_regset_found)
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp b/lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp
new file mode 100644
index 0000000000000..c5ab732c0c7cc
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp
@@ -0,0 +1,385 @@
+#include <getopt.h>
+#include <mach-o/loader.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include <iostream>
+#include <optional>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+[[noreturn]] void print_help(void) {
+  fprintf(stderr, "Append an LC_NOTE to a corefile.  Usage: \n");
+  fprintf(stderr, "   -i|--input <corefile>\n");
+  fprintf(stderr, "   -o|--output <corefile>\n");
+  fprintf(stderr, "   -n|--name <LC_NOTE name>\n");
+  fprintf(
+      stderr,
+      "   -r|--remove-dups  remove existing LC_NOTEs with this same name\n");
+  fprintf(stderr, "  One of:\n");
+  fprintf(stderr, "   -f|--file <file to embed as LC_NOTE payload>\n");
+  fprintf(stderr, "   -s|--str <string to embed as LC_NOTE payload>\n");
+  exit(1);
+}
+
+void parse_args(int argc, char **argv, string &infile, string &outfile,
+                string &note_name, vector<uint8_t> &payload,
+                bool &remove_dups) {
+  const char *const short_opts = "i:o:n:f:s:hr";
+  const option long_opts[] = {{"input", required_argument, nullptr, 'i'},
+                              {"output", required_argument, nullptr, 'o'},
+                              {"name", required_argument, nullptr, 'n'},
+                              {"file", required_argument, nullptr, 'f'},
+                              {"str", required_argument, nullptr, 's'},
+                              {"remove-dups", no_argument, nullptr, 'r'},
+                              {"help", no_argument, nullptr, 'h'},
+                              {nullptr, no_argument, nullptr, 0}};
+  optional<string> infile_str, outfile_str, name_str, payload_file_str,
+      payload_str;
+  remove_dups = false;
+  while (true) {
+    const auto opt = getopt_long(argc, argv, short_opts, long_opts, nullptr);
+    if (opt == -1)
+      break;
+    switch (opt) {
+    case 'i':
+      infile_str = optarg;
+      break;
+    case 'o':
+      outfile_str = optarg;
+      break;
+    case 'n':
+      name_str = optarg;
+      break;
+    case 'f':
+      payload_file_str = optarg;
+      break;
+    case 's':
+      payload_str = optarg;
+      break;
+    case 'r':
+      remove_dups = true;
+      break;
+    case 'h':
+      print_help();
+    }
+  }
+
+  if (!infile_str || !outfile_str || !name_str ||
+      (!payload_file_str && !payload_str))
+    print_help();
+
+  infile = *infile_str;
+  outfile = *outfile_str;
+  note_name = *name_str;
+  if (payload_str) {
+    payload.resize(payload_str->size(), 0);
+    memcpy(payload.data(), payload_str->c_str(), payload_str->size());
+  } else {
+    struct stat sb;
+    if (stat(payload_file_str->c_str(), &sb)) {
+      fprintf(stderr, "File '%s' does not exist.\n", payload_file_str->c_str());
+      exit(1);
+    }
+    payload.resize(sb.st_size, 0);
+    FILE *f = fopen(payload_file_str->c_str(), "r");
+    fread(payload.data(), 1, sb.st_size, f);
+    fclose(f);
+  }
+}
+
+struct all_image_infos_header {
+  uint32_t version;         // currently 1
+  uint32_t imgcount;        // number of binary images
+  uint64_t entries_fileoff; // file offset in the corefile of where the array of
+                            // struct entry's begin.
+  uint32_t entry_size;      // size of 'struct entry'.
+  uint32_t unused;          // set to 0
+};
+
+struct image_entry {
+  uint64_t filepath_offset; // corefile offset of the c-string filepath,
+                            // if available, else this should be set
+                            // to UINT64_MAX.
+  uuid_t uuid;              // uint8_t[16].  should be set to all zeroes if
+                            // uuid is unknown.
+  uint64_t
+      load_address; // virtual addr of mach-o header, UINT64_MAX if unknown.
+  uint64_t seg_addrs_offset; // corefile offset to the array of struct
+                             // segment_vmaddr's, UINT64_MAX if none.
+  uint32_t segment_count; // The number of segments for this binary, 0 if none.
+  uint32_t
+      executing; // Set to 0 if executing status is unknown by corefile
+                 // creator.
+                 // Set to 1 if this binary was executing on any thread,
+                 // so it can be force-loaded by the corefile reader.
+                 // Set to 2 if this binary was not executing on any thread.
+};
+
+int count_lc_notes_with_name(FILE *in, std::string name) {
+  fseeko(in, 0, SEEK_SET);
+
+  uint8_t magic[4];
+  if (fread(magic, 1, 4, in) != 4) {
+    printf("Failed to read magic number\n");
+    return 0;
+  }
+  uint8_t magic_32_le[] = {0xce, 0xfa, 0xed, 0xfe};
+  uint8_t magic_64_le[] = {0xcf, 0xfa, 0xed, 0xfe};
+
+  if (memcmp(magic, magic_32_le, 4) != 0 &&
+      memcmp(magic, magic_64_le, 4) != 0) {
+    return 0;
+  }
+
+  fseeko(in, 0, SEEK_SET);
+
+  int number_of_load_cmds = 0;
+  size_t size_of_mach_header = 0;
+  if (memcmp(magic, magic_64_le, 4) == 0) {
+    struct mach_header_64 mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return 0;
+    }
+    number_of_load_cmds = mh.ncmds;
+  } else {
+    struct mach_header mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return 0;
+    }
+    number_of_load_cmds = mh.ncmds;
+  }
+
+  int notes_seen = 0;
+  fseeko(in, size_of_mach_header, SEEK_SET);
+  for (int i = 0; i < number_of_load_cmds; i++) {
+    off_t cmd_start = ftello(in);
+    uint32_t cmd, cmdsize;
+    fread(&cmd, sizeof(uint32_t), 1, in);
+    fread(&cmdsize, sizeof(uint32_t), 1, in);
+
+    fseeko(in, cmd_start, SEEK_SET);
+    off_t next_cmd = cmd_start + cmdsize;
+    if (cmd == LC_NOTE) {
+      struct note_command note;
+      fread(&note, sizeof(note), 1, in);
+      if (strncmp(name.c_str(), note.data_owner, 16) == 0)
+        notes_seen++;
+    }
+    fseeko(in, next_cmd, SEEK_SET);
+  }
+  return notes_seen;
+}
+
+void copy_and_add_note(FILE *in, FILE *out, std::string lc_note_name,
+                       vector<uint8_t> payload_data, bool remove_dups) {
+  int number_of_load_cmds = 0;
+  off_t header_start = ftello(in);
+
+  int notes_to_remove = 0;
+  if (remove_dups)
+    notes_to_remove = count_lc_notes_with_name(in, lc_note_name);
+  fseeko(in, header_start, SEEK_SET);
+
+  uint8_t magic[4];
+  if (fread(magic, 1, 4, in) != 4) {
+    printf("Failed to read magic number\n");
+    return;
+  }
+  uint8_t magic_32_le[] = {0xce, 0xfa, 0xed, 0xfe};
+  uint8_t magic_64_le[] = {0xcf, 0xfa, 0xed, 0xfe};
+
+  if (memcmp(magic, magic_32_le, 4) != 0 &&
+      memcmp(magic, magic_64_le, 4) != 0) {
+    return;
+  }
+
+  fseeko(in, header_start, SEEK_SET);
+
+  off_t end_of_infine_loadcmds;
+  size_t size_of_mach_header = 0;
+  if (memcmp(magic, magic_64_le, 4) == 0) {
+    struct mach_header_64 mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return;
+    }
+    number_of_load_cmds = mh.ncmds;
+    end_of_infine_loadcmds = sizeof(mh) + mh.sizeofcmds;
+
+    mh.ncmds += 1;
+    mh.ncmds -= notes_to_remove;
+    mh.sizeofcmds += sizeof(struct note_command);
+    mh.sizeofcmds -= notes_to_remove * sizeof(struct note_command);
+    fseeko(out, header_start, SEEK_SET);
+    fwrite(&mh, sizeof(mh), 1, out);
+  } else {
+    struct mach_header mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return;
+    }
+    number_of_load_cmds = mh.ncmds;
+    end_of_infine_loadcmds = sizeof(mh) + mh.sizeofcmds;
+
+    mh.ncmds += 1;
+    mh.ncmds -= notes_to_remove;
+    mh.sizeofcmds += sizeof(struct note_command);
+    mh.sizeofcmds -= notes_to_remove * sizeof(struct note_command);
+    fseeko(out, header_start, SEEK_SET);
+    fwrite(&mh, sizeof(mh), 1, out);
+  }
+
+  off_t start_of_infile_load_cmds = ftello(in);
+  fseek(in, 0, SEEK_END);
+  off_t infile_size = ftello(in);
+
+  // LC_SEGMENT may be aligned to 4k boundaries, let's maintain
+  // that alignment by putting 4096 minus the size of the added
+  // LC_NOTE load command after the output file's load commands.
+  off_t end_of_outfile_loadcmds =
+      end_of_infine_loadcmds - (notes_to_remove * sizeof(struct note_command)) +
+      4096 - sizeof(struct note_command);
+  off_t slide = end_of_outfile_loadcmds - end_of_infine_loadcmds;
+
+  off_t all_image_infos_infile_offset = 0;
+
+  fseek(in, start_of_infile_load_cmds, SEEK_SET);
+  fseek(out, start_of_infile_load_cmds, SEEK_SET);
+  // Copy all the load commands from IN to OUT, updating any file offsets by
+  // SLIDE.
+  for (int cmd_num = 0; cmd_num < number_of_load_cmds; cmd_num++) {
+    off_t cmd_start = ftello(in);
+    uint32_t cmd, cmdsize;
+    fread(&cmd, sizeof(uint32_t), 1, in);
+    fread(&cmdsize, sizeof(uint32_t), 1, in);
+
+    fseeko(in, cmd_start, SEEK_SET);
+    off_t next_cmd = cmd_start + cmdsize;
+
+    switch (cmd) {
+    case LC_SEGMENT: {
+      struct segment_command segcmd;
+      fread(&segcmd, sizeof(segcmd), 1, in);
+      segcmd.fileoff += slide;
+      fwrite(&segcmd, cmdsize, 1, out);
+    } break;
+    case LC_SEGMENT_64: {
+      struct segment_command_64 segcmd;
+      fread(&segcmd, sizeof(segcmd), 1, in);
+      segcmd.fileoff += slide;
+      fwrite(&segcmd, cmdsize, 1, out);
+    } break;
+    case LC_NOTE: {
+      struct note_command notecmd;
+      fread(&notecmd, sizeof(notecmd), 1, in);
+      if ((strncmp(lc_note_name.c_str(), notecmd.data_owner, 16) == 0) &&
+          remove_dups) {
+        fseeko(in, next_cmd, SEEK_SET);
+        continue;
+      }
+      if (strncmp("all image infos", notecmd.data_owner, 16) == 0)
+        all_image_infos_infile_offset = notecmd.offset;
+      notecmd.offset += slide;
+      fwrite(&notecmd, cmdsize, 1, out);
+    } break;
+    default: {
+      vector<uint8_t> buf(cmdsize);
+      fread(buf.data(), cmdsize, 1, in);
+      fwrite(buf.data(), cmdsize, 1, out);
+    }
+    }
+    fseeko(in, next_cmd, SEEK_SET);
+  }
+
+  // Now add our additional LC_NOTE load command.
+  struct note_command note;
+  note.cmd = LC_NOTE;
+  note.cmdsize = sizeof(struct note_command);
+  memset(&note.data_owner, 0, 16);
+  // data_owner may not be nul terminated if all 16 characters
+  // are used, intentionally using strncpy here.
+  strncpy(note.data_owner, lc_note_name.c_str(), 16);
+  note.offset = infile_size + slide;
+  note.size = payload_data.size();
+  fwrite(&note, sizeof(struct note_command), 1, out);
+
+  fseeko(in, end_of_infine_loadcmds, SEEK_SET);
+  fseeko(out, end_of_outfile_loadcmds, SEEK_SET);
+
+  // Copy the rest of the corefile contents
+  vector<uint8_t> data_buf(1024 * 1024);
+  while (!feof(in)) {
+    size_t read_bytes = fread(data_buf.data(), 1, data_buf.size(), in);
+    if (read_bytes > 0) {
+      fwrite(data_buf.data(), read_bytes, 1, out);
+    } else {
+      break;
+    }
+  }
+
+  fwrite(payload_data.data(), payload_data.size(), 1, out);
+
+  // The "all image infos" LC_NOTE payload has file offsets hardcoded
+  // in it, unfortunately.  We've shifted the contents of the corefile
+  // and these offsets need to be updated in the ouput file.
+  // Re-copy them into the outfile with corrected file offsets.
+  off_t infile_image_entry_base = 0;
+  if (all_image_infos_infile_offset != 0) {
+    off_t all_image_infos_outfile_offset =
+        all_image_infos_infile_offset + slide;
+    fseeko(in, all_image_infos_infile_offset, SEEK_SET);
+    struct all_image_infos_header header;
+    fread(&header, sizeof(header), 1, in);
+    infile_image_entry_base = header.entries_fileoff;
+    header.entries_fileoff += slide;
+    fseeko(out, all_image_infos_outfile_offset, SEEK_SET);
+    fwrite(&header, sizeof(header), 1, out);
+
+    for (int i = 0; i < header.imgcount; i++) {
+      off_t infile_entries_fileoff = header.entries_fileoff - slide;
+      off_t outfile_entries_fileoff = header.entries_fileoff;
+
+      struct image_entry ent;
+      fseeko(in, infile_entries_fileoff + (header.entry_size * i), SEEK_SET);
+      fread(&ent, sizeof(ent), 1, in);
+      ent.filepath_offset += slide;
+      ent.seg_addrs_offset += slide;
+      fseeko(out, outfile_entries_fileoff + (header.entry_size * i), SEEK_SET);
+      fwrite(&ent, sizeof(ent), 1, out);
+    }
+  }
+}
+
+int main(int argc, char **argv) {
+  string infile, outfile, name;
+  vector<uint8_t> payload;
+  bool remove_dups;
+  parse_args(argc, argv, infile, outfile, name, payload, remove_dups);
+
+  FILE *in = fopen(infile.c_str(), "r");
+  if (!in) {
+    fprintf(stderr, "Unable to open %s for reading\n", infile.c_str());
+    exit(1);
+  }
+  FILE *out = fopen(outfile.c_str(), "w");
+  if (!out) {
+    fprintf(stderr, "Unable to open %s for reading\n", outfile.c_str());
+    exit(1);
+  }
+
+  copy_and_add_note(in, out, name, payload, remove_dups);
+
+  fclose(in);
+  fclose(out);
+}
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/main.c b/lldb/test/API/macosx/lc-note/additional-registers/main.c
new file mode 100644
index 0000000000000..70679942daf06
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/main.c
@@ -0,0 +1,11 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char **argv) {
+
+  char *heap_buf = (char *)malloc(80);
+  strcpy(heap_buf, "this is a string on the heap");
+
+  return 0; // break here
+}
diff --git a/lldb/test/API/python_api/sbvalue_persist/TestSBValuePersist.py b/lldb/test/API/python_api/sbvalue_persist/TestSBValuePersist.py
index d441a9d121709..8b58dc6d37935 100644
--- a/lldb/test/API/python_api/sbvalue_persist/TestSBValuePersist.py
+++ b/lldb/test/API/python_api/sbvalue_persist/TestSBValuePersist.py
@@ -10,7 +10,6 @@
 class SBValuePersistTestCase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24772")
     def test(self):
         """Test SBValue::Persist"""
         self.build()
diff --git a/lldb/test/API/riscv/step/Makefile b/lldb/test/API/riscv/step/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/riscv/step/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/riscv/step/TestSoftwareStep.py b/lldb/test/API/riscv/step/TestSoftwareStep.py
new file mode 100644
index 0000000000000..279c4c1b797e0
--- /dev/null
+++ b/lldb/test/API/riscv/step/TestSoftwareStep.py
@@ -0,0 +1,79 @@
+"""
+Test software step-inst, also known as instruction level single step, in risc-v atomic sequence.
+For more information about atomic sequences, see the RISC-V Unprivileged ISA specification.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestSoftwareStep(TestBase):
+    def do_sequence_test(self, filename, bkpt_name):
+        source_file = filename + ".c"
+        exe_file = filename + ".x"
+
+        self.build(dictionary={"C_SOURCES": source_file, "EXE": exe_file})
+        (target, process, cur_thread, bkpt) = lldbutil.run_to_name_breakpoint(
+            self, bkpt_name, exe_name=exe_file
+        )
+        entry_pc = cur_thread.GetFrameAtIndex(0).GetPC()
+
+        self.runCmd("thread step-inst")
+        self.expect(
+            "thread list",
+            substrs=["stopped", "stop reason = instruction step into"],
+        )
+
+        pc = cur_thread.GetFrameAtIndex(0).GetPC()
+
+        return pc - entry_pc
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_cas(self):
+        """
+        This test verifies LLDB instruction step handling of a proper lr/sc pair.
+        """
+        difference = self.do_sequence_test("main", "cas")
+        self.assertEqual(difference, 0x1A)
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_branch_cas(self):
+        """
+        LLDB cannot predict the actual state of registers within a critical section (i.e., inside an atomic
+        sequence). Therefore, it should identify all forward branches inside the atomic sequence and set
+        breakpoints at every jump address that lies beyond the end of the sequence (after the sc instruction).
+        This ensures that if any such branch is taken, execution will pause at its target address.
+
+        This test includes an lr/sc sequence containing an active forward branch with a jump address located
+        after the end of the atomic section. LLDB should correctly stop at this branch's target address. The
+        test is nearly identical to the previous one, except for the branch condition, which is inverted and
+        will result in a taken jump.
+        """
+        difference = self.do_sequence_test("branch", "branch_cas")
+        self.assertEqual(difference, 0x1A)
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_incomplete_sequence_without_lr(self):
+        """
+        This test verifies the behavior of a standalone sc instruction without a preceding lr. Since the sc
+        lacks the required lr pairing, LLDB should treat it as a non-atomic store rather than part of an
+        atomic sequence.
+        """
+        difference = self.do_sequence_test(
+            "incomplete_sequence_without_lr", "incomplete_cas"
+        )
+        self.assertEqual(difference, 0x4)
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_incomplete_sequence_without_sc(self):
+        """
+        This test checks the behavior of a standalone lr instruction without a subsequent sc. Since the lr
+        lacks its required sc counterpart, LLDB should treat it as a non-atomic load rather than part of an
+        atomic sequence.
+        """
+        difference = self.do_sequence_test(
+            "incomplete_sequence_without_sc", "incomplete_cas"
+        )
+        self.assertEqual(difference, 0x4)
diff --git a/lldb/test/API/riscv/step/branch.c b/lldb/test/API/riscv/step/branch.c
new file mode 100644
index 0000000000000..79bf0ca005db1
--- /dev/null
+++ b/lldb/test/API/riscv/step/branch.c
@@ -0,0 +1,22 @@
+void __attribute__((naked)) branch_cas(int *a, int *b) {
+  // Stop at the first instruction. The atomic sequence contains active forward
+  // branch (bne a5, a1, 2f). After step instruction lldb should stop at the
+  // branch's target address (ret instruction).
+  asm volatile("1:\n\t"
+               "lr.w a2, (a0)\n\t"
+               "and a5, a2, a4\n\t"
+               "bne a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "beqz a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  branch_cas(&a, &b);
+}
diff --git a/lldb/test/API/riscv/step/incomplete_sequence_without_lr.c b/lldb/test/API/riscv/step/incomplete_sequence_without_lr.c
new file mode 100644
index 0000000000000..eda313c6d69c4
--- /dev/null
+++ b/lldb/test/API/riscv/step/incomplete_sequence_without_lr.c
@@ -0,0 +1,22 @@
+void __attribute__((naked)) incomplete_cas(int *a, int *b) {
+  // Stop at the first instruction (an sc without a corresponding lr), then make
+  // a step instruction and ensure that execution stops at the next instruction
+  // (and).
+  asm volatile("1:\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "and a5, a2, a4\n\t"
+               "beq a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "bnez a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  incomplete_cas(&a, &b);
+}
diff --git a/lldb/test/API/riscv/step/incomplete_sequence_without_sc.c b/lldb/test/API/riscv/step/incomplete_sequence_without_sc.c
new file mode 100644
index 0000000000000..952dc7d2804e8
--- /dev/null
+++ b/lldb/test/API/riscv/step/incomplete_sequence_without_sc.c
@@ -0,0 +1,21 @@
+void __attribute__((naked)) incomplete_cas(int *a, int *b) {
+  // Stop at the first instruction (an lr without a corresponding sc), then make
+  // a step instruction and ensure that execution stops at the next instruction
+  // (and).
+  asm volatile("1:\n\t"
+               "lr.w a2, (a0)\n\t"
+               "and a5, a2, a4\n\t"
+               "beq a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "bnez a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  incomplete_cas(&a, &b);
+}
diff --git a/lldb/test/API/riscv/step/main.c b/lldb/test/API/riscv/step/main.c
new file mode 100644
index 0000000000000..35e8aee2cae4b
--- /dev/null
+++ b/lldb/test/API/riscv/step/main.c
@@ -0,0 +1,22 @@
+void __attribute__((naked)) cas(int *a, int *b) {
+  // This atomic sequence implements a copy-and-swap function. This test should
+  // at the first instruction, and after step instruction, we should stop at the
+  // end of the sequence (on the ret instruction).
+  asm volatile("1:\n\t"
+               "lr.w a2, (a0)\n\t"
+               "and a5, a2, a4\n\t"
+               "beq a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "beqz a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  cas(&a, &b);
+}
diff --git a/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py b/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py
index 1ef2f2a8235a4..64ed4154b035d 100644
--- a/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py
+++ b/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py
@@ -32,7 +32,7 @@ def test_module_event(self):
 
         # Make sure we got a module event for libother.
         event = self.dap_server.wait_for_event("module", 5)
-        self.assertTrue(event, "didn't get a module event")
+        self.assertIsNotNone(event, "didn't get a module event")
         module_name = event["body"]["module"]["name"]
         module_id = event["body"]["module"]["id"]
         self.assertEqual(event["body"]["reason"], "new")
@@ -43,13 +43,20 @@ def test_module_event(self):
 
         # Make sure we got a module event for libother.
         event = self.dap_server.wait_for_event("module", 5)
-        self.assertTrue(event, "didn't get a module event")
+        self.assertIsNotNone(event, "didn't get a module event")
         reason = event["body"]["reason"]
-        self.assertEqual(event["body"]["reason"], "removed")
+        self.assertEqual(reason, "removed")
         self.assertEqual(event["body"]["module"]["id"], module_id)
 
-        # The removed module event should omit everything but the module id.
-        # Check that there's no module name in the event.
-        self.assertNotIn("name", event["body"]["module"])
+        # The removed module event should omit everything but the module id and name
+        # as they are required fields.
+        module_data = event["body"]["module"]
+        required_keys = ["id", "name"]
+        self.assertListEqual(list(module_data.keys()), required_keys)
+        self.assertEqual(module_data["name"], "", "expects empty name.")
+
+        # Make sure we do not send another event
+        event = self.dap_server.wait_for_event("module", 3)
+        self.assertIsNone(event, "expects no events.")
 
         self.continue_to_exit()
diff --git a/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py b/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
index 4fc221668a8ee..c9091df64f487 100644
--- a/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
+++ b/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
@@ -45,16 +45,20 @@ def run_test(self, symbol_basename, expect_debug_info_size):
             context="repl",
         )
 
-        def checkSymbolsLoadedWithSize():
+        def check_symbols_loaded_with_size():
             active_modules = self.dap_server.get_modules()
             program_module = active_modules[program_basename]
             self.assertIn("symbolFilePath", program_module)
             self.assertIn(symbols_path, program_module["symbolFilePath"])
-            symbol_regex = re.compile(r"[0-9]+(\.[0-9]*)?[KMG]?B")
-            return symbol_regex.match(program_module["symbolStatus"])
+            size_regex = re.compile(r"[0-9]+(\.[0-9]*)?[KMG]?B")
+            return size_regex.match(program_module["debugInfoSize"])
 
         if expect_debug_info_size:
-            self.waitUntil(checkSymbolsLoadedWithSize)
+            self.assertTrue(
+                self.waitUntil(check_symbols_loaded_with_size),
+                "expect has debug info size",
+            )
+
         active_modules = self.dap_server.get_modules()
         program_module = active_modules[program_basename]
         self.assertEqual(program_basename, program_module["name"])
@@ -83,6 +87,7 @@ def checkSymbolsLoadedWithSize():
         # symbols got added.
         self.assertNotEqual(len(module_changed_names), 0)
         self.assertIn(program_module["name"], module_changed_names)
+        self.continue_to_exit()
 
     @skipIfWindows
     def test_modules(self):
@@ -124,3 +129,5 @@ def test_compile_units(self):
         self.assertTrue(response["body"])
         cu_paths = [cu["compileUnitPath"] for cu in response["body"]["compileUnits"]]
         self.assertIn(main_source_path, cu_paths, "Real path to main.cpp matches")
+
+        self.continue_to_exit()
diff --git a/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py b/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
index 592a4cfb0a88b..e01320c25b155 100644
--- a/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
+++ b/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
@@ -101,7 +101,9 @@ def test_server_interrupt(self):
         # Interrupt the server which should disconnect all clients.
         process.send_signal(signal.SIGINT)
 
-        self.dap_server.wait_for_terminated()
+        # Wait for both events since they can happen in any order.
+        self.dap_server.wait_for_event(["terminated", "exited"])
+        self.dap_server.wait_for_event(["terminated", "exited"])
         self.assertIsNotNone(
             self.dap_server.exit_status,
             "Process exited before interrupting lldb-dap server",
diff --git a/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py b/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
index 42a39e3c8c080..5c055f679aedc 100644
--- a/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
+++ b/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
@@ -83,3 +83,41 @@ def test_step(self):
 
                     # only step one thread that is at the breakpoint and stop
                     break
+
+    def test_step_over_inlined_function(self):
+        """
+        Test stepping over when the program counter is in another file.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.cpp"
+        breakpoint_lines = [line_number(source, "// breakpoint 2")]
+        step_over_pos = line_number(source, "// position_after_step_over")
+        breakpoint_ids = self.set_source_breakpoints(source, breakpoint_lines)
+        self.assertEqual(
+            len(breakpoint_ids),
+            len(breakpoint_lines),
+            "expect correct number of breakpoints.",
+        )
+        self.continue_to_breakpoints(breakpoint_ids)
+
+        thread_id = self.dap_server.get_thread_id()
+        self.stepOver(thread_id)
+        levels = 1
+        frames = self.get_stackFrames(thread_id, 0, levels)
+        self.assertEqual(len(frames), levels, "expect current number of frame levels.")
+        top_frame = frames[0]
+        self.assertEqual(
+            top_frame["source"]["name"], source, "expect we are in the same file."
+        )
+        self.assertTrue(
+            top_frame["source"]["path"].endswith(source),
+            f"expect path ending with '{source}'.",
+        )
+        self.assertEqual(
+            top_frame["line"],
+            step_over_pos,
+            f"expect step_over on line {step_over_pos}",
+        )
+
+        self.continue_to_exit()
diff --git a/lldb/test/API/tools/lldb-dap/step/main.cpp b/lldb/test/API/tools/lldb-dap/step/main.cpp
index 8905beb5e7eff..7320e83154f5b 100644
--- a/lldb/test/API/tools/lldb-dap/step/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/step/main.cpp
@@ -1,3 +1,5 @@
+#include "other.h"
+
 int function(int x) {
   if ((x % 2) == 0)
     return function(x - 1) + x; // breakpoint 1
@@ -5,4 +7,14 @@ int function(int x) {
     return x;
 }
 
-int main(int argc, char const *argv[]) { return function(2); }
+int function2() {
+  int volatile value = 3; // breakpoint 2
+  inlined_fn();           // position_after_step_over
+
+  return value;
+}
+
+int main(int argc, char const *argv[]) {
+  int func_result = function2();
+  return function(2) - func_result; // returns 0
+}
diff --git a/lldb/test/API/tools/lldb-dap/step/other.h b/lldb/test/API/tools/lldb-dap/step/other.h
new file mode 100644
index 0000000000000..c71cc373fbdff
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/step/other.h
@@ -0,0 +1,7 @@
+#ifndef OTHER_H
+#define OTHER_H
+
+__attribute__((noinline)) void not_inlined_fn() {};
+
+__attribute__((always_inline)) inline void inlined_fn() { not_inlined_fn(); }
+#endif // OTHER_H
\ No newline at end of file
diff --git a/lldb/test/Shell/RPC/Generator/Inputs/Server/CheckBasicIncludesEmit.h b/lldb/test/Shell/RPC/Generator/Inputs/Server/CheckBasicIncludesEmit.h
new file mode 100644
index 0000000000000..77394aba12f7a
--- /dev/null
+++ b/lldb/test/Shell/RPC/Generator/Inputs/Server/CheckBasicIncludesEmit.h
@@ -0,0 +1,6 @@
+// This ia a basic header file used to check that the server-side emitter
+// for rpc-gen emits an expected set of includes in a generated source file.
+#ifndef LLDB_API_SBRPC_CHECKBASICINCLUDE_H
+#define LLDB_API_SBRPC_CHECKBASICINCLUDE_H
+
+#endif // LLDB_API_SBRPC_CHECKBASICINCLUDE_H
diff --git a/lldb/test/Shell/RPC/Generator/Inputs/Server/CheckConstCharPointer.h b/lldb/test/Shell/RPC/Generator/Inputs/Server/CheckConstCharPointer.h
new file mode 100644
index 0000000000000..37121cd445267
--- /dev/null
+++ b/lldb/test/Shell/RPC/Generator/Inputs/Server/CheckConstCharPointer.h
@@ -0,0 +1,14 @@
+#ifndef LLDB_API_SBRPC_CHECKCONSTCHARPOINTER_H
+#define LLDB_API_SBRPC_CHECKCONSTCHARPOINTER_H
+
+namespace lldb {
+class LLDB_API SBRPC_CHECKCONSTCHARPOINTER {
+public:
+  // const char * parameters must decoded as rpc_common::ConstCharPointer in server side
+  // source files.
+  int CheckConstCharPointer(char *string);
+
+}; // class SBRPC_CHECKCONSTCHARPOINTER
+} // namespace lldb
+
+#endif // LLDB_API_SBRPC_CHECKCONSTCHARPOINTER_H
diff --git a/lldb/test/Shell/RPC/Generator/Tests/Server/CheckBasicIncludesEmit.test b/lldb/test/Shell/RPC/Generator/Tests/Server/CheckBasicIncludesEmit.test
new file mode 100644
index 0000000000000..fd21da93cb823
--- /dev/null
+++ b/lldb/test/Shell/RPC/Generator/Tests/Server/CheckBasicIncludesEmit.test
@@ -0,0 +1,15 @@
+# Disabling until the lldb-rpc-gen tool lands.
+XFAIL: *
+RUN: mkdir -p %t/server
+RUN: mkdir -p %t/lib
+RUN: %lldb-rpc-gen --output-dir=%t %S/../../Inputs/CheckBasicIncludesEmit.h
+
+RUN: cat %t/lib/CheckBasicIncludesEmit.cpp | FileCheck %s
+
+# All server-side source files must have these includes at the top of their files.
+CHECK: #include "RPCUserServer.h"
+CHECK: #include "SBAPI.h"
+CHECK: #include <lldb-rpc/common/RPCArgument.h>
+CHECK: #include <lldb-rpc/common/RPCCommon.h>
+CHECK: #include <lldb-rpc/common/RPCFunction.h>
+CHECK: #include <lldb/API/LLDB.h>
diff --git a/lldb/test/Shell/RPC/Generator/Tests/Server/CheckConstCharPointer.test b/lldb/test/Shell/RPC/Generator/Tests/Server/CheckConstCharPointer.test
new file mode 100644
index 0000000000000..4f953882f7d81
--- /dev/null
+++ b/lldb/test/Shell/RPC/Generator/Tests/Server/CheckConstCharPointer.test
@@ -0,0 +1,11 @@
+# Disabling until the lldb-rpc-gen tool lands.
+XFAIL: *
+RUN: mkdir -p %t/server
+RUN: mkdir -p %t/lib
+RUN: %lldb-rpc-gen --output-dir=%t %S/../../Inputs/CheckConstCharPointer.h
+
+RUN: cat %t/lib/CheckConstCharPointer.cpp | FileCheck %s
+
+# const char * pointers must be decoded as rpc_common::ConstCharPointer objects
+# in server side source files.
+CHECK: rpc_common::ConstCharPointer string
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index cd97458bd4aa8..60a32393a9b91 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -16,6 +16,7 @@
 #include "LLDBUtils.h"
 #include "OutputRedirector.h"
 #include "Protocol/ProtocolBase.h"
+#include "Protocol/ProtocolEvents.h"
 #include "Protocol/ProtocolRequests.h"
 #include "Protocol/ProtocolTypes.h"
 #include "ProtocolUtils.h"
@@ -623,6 +624,17 @@ ReplMode DAP::DetectReplMode(lldb::SBFrame frame, std::string &expression,
   llvm_unreachable("enum cases exhausted.");
 }
 
+std::optional<protocol::Source> DAP::ResolveSource(const lldb::SBFrame &frame) {
+  if (!frame.IsValid())
+    return std::nullopt;
+
+  const lldb::SBAddress frame_pc = frame.GetPCAddress();
+  if (DisplayAssemblySource(debugger, frame_pc))
+    return ResolveAssemblySource(frame_pc);
+
+  return CreateSource(frame.GetLineEntry().GetFileSpec());
+}
+
 std::optional<protocol::Source> DAP::ResolveSource(lldb::SBAddress address) {
   if (DisplayAssemblySource(debugger, address))
     return ResolveAssemblySource(address);
@@ -1342,37 +1354,37 @@ void DAP::EventThread() {
             event_mask & lldb::SBTarget::eBroadcastBitSymbolsChanged) {
           const uint32_t num_modules =
               lldb::SBTarget::GetNumModulesFromEvent(event);
+          const bool remove_module =
+              event_mask & lldb::SBTarget::eBroadcastBitModulesUnloaded;
+
           std::lock_guard<std::mutex> guard(modules_mutex);
           for (uint32_t i = 0; i < num_modules; ++i) {
             lldb::SBModule module =
                 lldb::SBTarget::GetModuleAtIndexFromEvent(i, event);
-            if (!module.IsValid())
-              continue;
-            llvm::StringRef module_id = module.GetUUIDString();
-            if (module_id.empty())
+
+            std::optional<protocol::Module> p_module =
+                CreateModule(target, module, remove_module);
+            if (!p_module)
               continue;
 
-            llvm::StringRef reason;
-            bool id_only = false;
-            if (modules.contains(module_id)) {
-              if (event_mask & lldb::SBTarget::eBroadcastBitModulesUnloaded) {
-                modules.erase(module_id);
-                reason = "removed";
-                id_only = true;
-              } else {
-                reason = "changed";
-              }
-            } else {
+            llvm::StringRef module_id = p_module->id;
+
+            const bool module_exists = modules.contains(module_id);
+            if (remove_module && module_exists) {
+              modules.erase(module_id);
+              Send(protocol::Event{
+                  "module", ModuleEventBody{std::move(p_module).value(),
+                                            ModuleEventBody::eReasonRemoved}});
+            } else if (module_exists) {
+              Send(protocol::Event{
+                  "module", ModuleEventBody{std::move(p_module).value(),
+                                            ModuleEventBody::eReasonChanged}});
+            } else if (!remove_module) {
               modules.insert(module_id);
-              reason = "new";
+              Send(protocol::Event{
+                  "module", ModuleEventBody{std::move(p_module).value(),
+                                            ModuleEventBody::eReasonNew}});
             }
-
-            llvm::json::Object body;
-            body.try_emplace("reason", reason);
-            body.try_emplace("module", CreateModule(target, module, id_only));
-            llvm::json::Object module_event = CreateEventObject("module");
-            module_event.try_emplace("body", std::move(body));
-            SendJSON(llvm::json::Value(std::move(module_event)));
           }
         }
       } else if (lldb::SBBreakpoint::EventIsBreakpointEvent(event)) {
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 0e9a9e0eb674c..af4aabaafaae8 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -254,6 +254,17 @@ struct DAP {
   ReplMode DetectReplMode(lldb::SBFrame frame, std::string &expression,
                           bool partial_expression);
 
+  /// Create a `protocol::Source` object as described in the debug adapter
+  /// definition.
+  ///
+  /// \param[in] frame
+  ///     The frame to use when populating the "Source" object.
+  ///
+  /// \return
+  ///     A `protocol::Source` object that follows the formal JSON
+  ///     definition outlined by Microsoft.
+  std::optional<protocol::Source> ResolveSource(const lldb::SBFrame &frame);
+
   /// Create a "Source" JSON object as described in the debug adapter
   /// definition.
   ///
diff --git a/lldb/tools/lldb-dap/Handler/ModulesRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/ModulesRequestHandler.cpp
index d37f302b06958..697166a2a1a92 100644
--- a/lldb/tools/lldb-dap/Handler/ModulesRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/ModulesRequestHandler.cpp
@@ -7,64 +7,38 @@
 //===----------------------------------------------------------------------===//
 
 #include "DAP.h"
-#include "EventHelper.h"
-#include "JSONUtils.h"
+#include "ProtocolUtils.h"
 #include "RequestHandler.h"
 
+using namespace lldb_dap::protocol;
 namespace lldb_dap {
 
-// "modulesRequest": {
-//   "allOf": [ { "$ref": "#/definitions/Request" }, {
-//     "type": "object",
-//     "description": "Modules request; value of command field is
-//                     'modules'.",
-//     "properties": {
-//       "command": {
-//         "type": "string",
-//         "enum": [ "modules" ]
-//       },
-//     },
-//     "required": [ "command" ]
-//   }]
-// },
-// "modulesResponse": {
-//   "allOf": [ { "$ref": "#/definitions/Response" }, {
-//     "type": "object",
-//     "description": "Response to 'modules' request.",
-//     "properties": {
-//       "body": {
-//         "description": "Response to 'modules' request. Array of
-//                         module objects."
-//       }
-//     }
-//   }]
-// }
-void ModulesRequestHandler::operator()(
-    const llvm::json::Object &request) const {
-  llvm::json::Object response;
-  FillResponse(request, response);
-
-  llvm::json::Array modules;
-
-  {
-    std::lock_guard<std::mutex> guard(dap.modules_mutex);
-    for (size_t i = 0; i < dap.target.GetNumModules(); i++) {
-      lldb::SBModule module = dap.target.GetModuleAtIndex(i);
-      if (!module.IsValid())
-        continue;
-
-      llvm::StringRef module_id = module.GetUUIDString();
-      if (!module_id.empty())
-        dap.modules.insert(module_id);
-
-      modules.emplace_back(CreateModule(dap.target, module));
+/// Modules can be retrieved from the debug adapter with this request which can
+/// either return all modules or a range of modules to support paging.
+///
+/// Clients should only call this request if the corresponding capability
+/// `supportsModulesRequest` is true.
+llvm::Expected<ModulesResponseBody>
+ModulesRequestHandler::Run(const std::optional<ModulesArguments> &args) const {
+  ModulesResponseBody response;
+
+  std::vector<Module> &modules = response.modules;
+  std::lock_guard<std::mutex> guard(dap.modules_mutex);
+  const uint32_t total_modules = dap.target.GetNumModules();
+  response.totalModules = total_modules;
+
+  modules.reserve(total_modules);
+  for (uint32_t i = 0; i < total_modules; i++) {
+    lldb::SBModule module = dap.target.GetModuleAtIndex(i);
+
+    std::optional<Module> result = CreateModule(dap.target, module);
+    if (result && !result->id.empty()) {
+      dap.modules.insert(result->id);
+      modules.emplace_back(std::move(result).value());
     }
   }
 
-  llvm::json::Object body;
-  body.try_emplace("modules", std::move(modules));
-  response.try_emplace("body", std::move(body));
-  dap.SendJSON(llvm::json::Value(std::move(response)));
+  return response;
 }
 
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Handler/RequestHandler.h b/lldb/tools/lldb-dap/Handler/RequestHandler.h
index e35b9830ab60f..3b910abe81992 100644
--- a/lldb/tools/lldb-dap/Handler/RequestHandler.h
+++ b/lldb/tools/lldb-dap/Handler/RequestHandler.h
@@ -466,14 +466,17 @@ class CompileUnitsRequestHandler : public LegacyRequestHandler {
   void operator()(const llvm::json::Object &request) const override;
 };
 
-class ModulesRequestHandler : public LegacyRequestHandler {
+class ModulesRequestHandler final
+    : public RequestHandler<std::optional<protocol::ModulesArguments>,
+                            llvm::Expected<protocol::ModulesResponseBody>> {
 public:
-  using LegacyRequestHandler::LegacyRequestHandler;
+  using RequestHandler::RequestHandler;
   static llvm::StringLiteral GetCommand() { return "modules"; }
   FeatureSet GetSupportedFeatures() const override {
     return {protocol::eAdapterFeatureModulesRequest};
   }
-  void operator()(const llvm::json::Object &request) const override;
+  llvm::Expected<protocol::ModulesResponseBody>
+  Run(const std::optional<protocol::ModulesArguments> &args) const override;
 };
 
 class PauseRequestHandler : public LegacyRequestHandler {
diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index 08e65ab835a57..553c52605c998 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -341,101 +341,6 @@ llvm::json::Value CreateScope(const llvm::StringRef name,
   return llvm::json::Value(std::move(object));
 }
 
-static uint64_t GetDebugInfoSizeInSection(lldb::SBSection section) {
-  uint64_t debug_info_size = 0;
-  llvm::StringRef section_name(section.GetName());
-  if (section_name.starts_with(".debug") ||
-      section_name.starts_with("__debug") ||
-      section_name.starts_with(".apple") || section_name.starts_with("__apple"))
-    debug_info_size += section.GetFileByteSize();
-  size_t num_sub_sections = section.GetNumSubSections();
-  for (size_t i = 0; i < num_sub_sections; i++) {
-    debug_info_size +=
-        GetDebugInfoSizeInSection(section.GetSubSectionAtIndex(i));
-  }
-  return debug_info_size;
-}
-
-static uint64_t GetDebugInfoSize(lldb::SBModule module) {
-  uint64_t debug_info_size = 0;
-  size_t num_sections = module.GetNumSections();
-  for (size_t i = 0; i < num_sections; i++) {
-    debug_info_size += GetDebugInfoSizeInSection(module.GetSectionAtIndex(i));
-  }
-  return debug_info_size;
-}
-
-static std::string ConvertDebugInfoSizeToString(uint64_t debug_info) {
-  std::ostringstream oss;
-  oss << std::fixed << std::setprecision(1);
-  if (debug_info < 1024) {
-    oss << debug_info << "B";
-  } else if (debug_info < 1024 * 1024) {
-    double kb = double(debug_info) / 1024.0;
-    oss << kb << "KB";
-  } else if (debug_info < 1024 * 1024 * 1024) {
-    double mb = double(debug_info) / (1024.0 * 1024.0);
-    oss << mb << "MB";
-  } else {
-    double gb = double(debug_info) / (1024.0 * 1024.0 * 1024.0);
-    oss << gb << "GB";
-  }
-  return oss.str();
-}
-
-llvm::json::Value CreateModule(lldb::SBTarget &target, lldb::SBModule &module,
-                               bool id_only) {
-  llvm::json::Object object;
-  if (!target.IsValid() || !module.IsValid())
-    return llvm::json::Value(std::move(object));
-
-  const char *uuid = module.GetUUIDString();
-  object.try_emplace("id", uuid ? std::string(uuid) : std::string(""));
-
-  if (id_only)
-    return llvm::json::Value(std::move(object));
-
-  object.try_emplace("name", std::string(module.GetFileSpec().GetFilename()));
-  char module_path_arr[PATH_MAX];
-  module.GetFileSpec().GetPath(module_path_arr, sizeof(module_path_arr));
-  std::string module_path(module_path_arr);
-  object.try_emplace("path", module_path);
-  if (module.GetNumCompileUnits() > 0) {
-    std::string symbol_str = "Symbols loaded.";
-    std::string debug_info_size;
-    uint64_t debug_info = GetDebugInfoSize(module);
-    if (debug_info > 0) {
-      debug_info_size = ConvertDebugInfoSizeToString(debug_info);
-    }
-    object.try_emplace("symbolStatus", symbol_str);
-    object.try_emplace("debugInfoSize", debug_info_size);
-    char symbol_path_arr[PATH_MAX];
-    module.GetSymbolFileSpec().GetPath(symbol_path_arr,
-                                       sizeof(symbol_path_arr));
-    std::string symbol_path(symbol_path_arr);
-    object.try_emplace("symbolFilePath", symbol_path);
-  } else {
-    object.try_emplace("symbolStatus", "Symbols not found.");
-  }
-  std::string load_address =
-      llvm::formatv("{0:x}",
-                    module.GetObjectFileHeaderAddress().GetLoadAddress(target))
-          .str();
-  object.try_emplace("addressRange", load_address);
-  std::string version_str;
-  uint32_t version_nums[3];
-  uint32_t num_versions =
-      module.GetVersion(version_nums, sizeof(version_nums) / sizeof(uint32_t));
-  for (uint32_t i = 0; i < num_versions; ++i) {
-    if (!version_str.empty())
-      version_str += ".";
-    version_str += std::to_string(version_nums[i]);
-  }
-  if (!version_str.empty())
-    object.try_emplace("version", version_str);
-  return llvm::json::Value(std::move(object));
-}
-
 // "Event": {
 //   "allOf": [ { "$ref": "#/definitions/ProtocolMessage" }, {
 //     "type": "object",
@@ -572,9 +477,7 @@ llvm::json::Value CreateStackFrame(DAP &dap, lldb::SBFrame &frame,
 
   EmplaceSafeString(object, "name", frame_name);
 
-  auto target = frame.GetThread().GetProcess().GetTarget();
-  std::optional<protocol::Source> source =
-      dap.ResolveSource(frame.GetPCAddress());
+  std::optional<protocol::Source> source = dap.ResolveSource(frame);
 
   if (source && !IsAssemblySource(*source)) {
     // This is a normal source with a valid line entry.
@@ -586,8 +489,7 @@ llvm::json::Value CreateStackFrame(DAP &dap, lldb::SBFrame &frame,
     // This is a source where the disassembly is used, but there is a valid
     // symbol. Calculate the line of the current PC from the start of the
     // current symbol.
-    lldb::SBTarget target = frame.GetThread().GetProcess().GetTarget();
-    lldb::SBInstructionList inst_list = target.ReadInstructions(
+    lldb::SBInstructionList inst_list = dap.target.ReadInstructions(
         frame.GetSymbol().GetStartAddress(), frame.GetPCAddress(), nullptr);
     size_t inst_line = inst_list.GetSize();
 
diff --git a/lldb/tools/lldb-dap/JSONUtils.h b/lldb/tools/lldb-dap/JSONUtils.h
index fd9a06931ebff..0424438ad5b72 100644
--- a/lldb/tools/lldb-dap/JSONUtils.h
+++ b/lldb/tools/lldb-dap/JSONUtils.h
@@ -196,24 +196,6 @@ GetStringMap(const llvm::json::Object &obj, llvm::StringRef key);
 void FillResponse(const llvm::json::Object &request,
                   llvm::json::Object &response);
 
-/// Converts a LLDB module to a VS Code DAP module for use in "modules" events.
-///
-/// \param[in] target
-///     A LLDB target object to convert into a JSON value.
-///
-/// \param[in] module
-///     A LLDB module object to convert into a JSON value
-///
-/// \param[in] id_only
-///     Only include the module ID in the JSON value. This is used when sending
-///     a "removed" module event.
-///
-/// \return
-///     A "Module" JSON object with that follows the formal JSON
-///     definition outlined by Microsoft.
-llvm::json::Value CreateModule(lldb::SBTarget &target, lldb::SBModule &module,
-                               bool id_only = false);
-
 /// Create a "Event" JSON object using \a event_name as the event name
 ///
 /// \param[in] event_name
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp
index ad6e305d09fed..4faf65567c3ea 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolEvents.cpp
@@ -17,4 +17,20 @@ json::Value toJSON(const CapabilitiesEventBody &CEB) {
   return json::Object{{"capabilities", CEB.capabilities}};
 }
 
+json::Value toJSON(const ModuleEventBody::Reason &MEBR) {
+  switch (MEBR) {
+  case ModuleEventBody::eReasonNew:
+    return "new";
+  case ModuleEventBody::eReasonChanged:
+    return "changed";
+  case ModuleEventBody::eReasonRemoved:
+    return "removed";
+  }
+  llvm_unreachable("unhandled module event reason!.");
+}
+
+json::Value toJSON(const ModuleEventBody &MEB) {
+  return json::Object{{"reason", MEB.reason}, {"module", MEB.module}};
+}
+
 } // namespace lldb_dap::protocol
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h b/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h
index 512106222362c..ee9e03c499eae 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolEvents.h
@@ -41,6 +41,21 @@ struct CapabilitiesEventBody {
 };
 llvm::json::Value toJSON(const CapabilitiesEventBody &);
 
+/// The event indicates that some information about a module has changed.
+struct ModuleEventBody {
+  enum Reason : unsigned { eReasonNew, eReasonChanged, eReasonRemoved };
+
+  /// The new, changed, or removed module. In case of `removed` only the module
+  /// id is used.
+  Module module;
+
+  /// The reason for the event.
+  /// Values: 'new', 'changed', 'removed'
+  Reason reason;
+};
+llvm::json::Value toJSON(const ModuleEventBody::Reason &);
+llvm::json::Value toJSON(const ModuleEventBody &);
+
 } // end namespace lldb_dap::protocol
 
 #endif
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
index 9bd84a6c898f9..124dbace97dd6 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
@@ -517,4 +517,18 @@ json::Value toJSON(const ReadMemoryResponseBody &RMR) {
   return result;
 }
 
+bool fromJSON(const json::Value &Params, ModulesArguments &MA, json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.mapOptional("startModule", MA.startModule) &&
+         O.mapOptional("moduleCount", MA.moduleCount);
+}
+
+json::Value toJSON(const ModulesResponseBody &MR) {
+  json::Object result{{"modules", MR.modules}};
+  if (MR.totalModules != 0)
+    result.insert({"totalModules", MR.totalModules});
+
+  return result;
+}
+
 } // namespace lldb_dap::protocol
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index d4b816c72679b..26eb3cb396ad1 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -875,6 +875,27 @@ struct ReadMemoryResponseBody {
 };
 llvm::json::Value toJSON(const ReadMemoryResponseBody &);
 
+/// Arguments for `modules` request.
+struct ModulesArguments {
+  /// The index of the first module to return; if omitted modules start at 0.
+  uint32_t startModule = 0;
+
+  /// The number of modules to return. If `moduleCount` is not specified or 0,
+  /// all modules are returned.
+  uint32_t moduleCount = 0;
+};
+bool fromJSON(const llvm::json::Value &, ModulesArguments &, llvm::json::Path);
+
+/// Response to `modules` request.
+struct ModulesResponseBody {
+  /// All modules or range of modules.
+  std::vector<Module> modules;
+
+  /// The total number of modules available.
+  uint32_t totalModules = 0;
+};
+llvm::json::Value toJSON(const ModulesResponseBody &);
+
 } // namespace lldb_dap::protocol
 
 #endif
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
index f3635202175a7..9b5c9ef348ca4 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
@@ -8,6 +8,7 @@
 
 #include "Protocol/ProtocolTypes.h"
 #include "JSONUtils.h"
+#include "ProtocolUtils.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -926,4 +927,30 @@ llvm::json::Value toJSON(const DisassembledInstruction &DI) {
   return result;
 }
 
+json::Value toJSON(const Module &M) {
+  json::Object result{{"id", M.id}, {"name", M.name}};
+
+  if (!M.path.empty())
+    result.insert({"path", M.path});
+  if (M.isOptimized)
+    result.insert({"isOptimized", M.isOptimized});
+  if (M.isUserCode)
+    result.insert({"isUserCode", M.isUserCode});
+  if (!M.version.empty())
+    result.insert({"version", M.version});
+  if (!M.symbolStatus.empty())
+    result.insert({"symbolStatus", M.symbolStatus});
+  if (!M.symbolFilePath.empty())
+    result.insert({"symbolFilePath", M.symbolFilePath});
+  if (!M.dateTimeStamp.empty())
+    result.insert({"dateTimeStamp", M.dateTimeStamp});
+  if (!M.addressRange.empty())
+    result.insert({"addressRange", M.addressRange});
+  if (M.debugInfoSizeBytes != 0)
+    result.insert(
+        {"debugInfoSize", ConvertDebugInfoSizeToString(M.debugInfoSizeBytes)});
+
+  return result;
+}
+
 } // namespace lldb_dap::protocol
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
index 6adfe3b7211b1..2bb765e956256 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
@@ -743,6 +743,52 @@ bool fromJSON(const llvm::json::Value &, DisassembledInstruction &,
               llvm::json::Path);
 llvm::json::Value toJSON(const DisassembledInstruction &);
 
+struct Module {
+  /// Unique identifier for the module.
+  std::string id;
+
+  /// A name of the module.
+  std::string name;
+
+  /// Logical full path to the module. The exact definition is implementation
+  /// defined, but usually this would be a full path to the on-disk file for the
+  /// module.
+  std::string path;
+
+  /// True if the module is optimized.
+  bool isOptimized = false;
+
+  /// True if the module is considered 'user code' by a debugger that supports
+  /// 'Just My Code'.
+  bool isUserCode = false;
+
+  /// Version of Module.
+  std::string version;
+
+  /// User-understandable description of if symbols were found for the module
+  /// (ex: 'Symbols Loaded', 'Symbols not found', etc.)
+  std::string symbolStatus;
+
+  /// Logical full path to the symbol file. The exact definition is
+  /// implementation defined.
+  std::string symbolFilePath;
+
+  /// Module created or modified, encoded as an RFC 3339 timestamp.
+  std::string dateTimeStamp;
+
+  /// Address range covered by this module.
+  std::string addressRange;
+
+  /// Custom fields
+  /// @{
+
+  /// Size of the debug_info sections in the module in bytes.
+  uint64_t debugInfoSizeBytes = 0;
+
+  //// @}
+};
+llvm::json::Value toJSON(const Module &);
+
 } // namespace lldb_dap::protocol
 
 #endif
diff --git a/lldb/tools/lldb-dap/ProtocolUtils.cpp b/lldb/tools/lldb-dap/ProtocolUtils.cpp
index 724d851107928..f9e373db74618 100644
--- a/lldb/tools/lldb-dap/ProtocolUtils.cpp
+++ b/lldb/tools/lldb-dap/ProtocolUtils.cpp
@@ -16,7 +16,10 @@
 #include "lldb/API/SBTarget.h"
 #include "lldb/API/SBThread.h"
 #include "lldb/Host/PosixApi.h" // Adds PATH_MAX for windows
+
+#include <iomanip>
 #include <optional>
+#include <sstream>
 
 using namespace lldb_dap::protocol;
 namespace lldb_dap {
@@ -46,6 +49,106 @@ static bool ShouldDisplayAssemblySource(
   return false;
 }
 
+static uint64_t GetDebugInfoSizeInSection(lldb::SBSection section) {
+  uint64_t debug_info_size = 0;
+  const llvm::StringRef section_name(section.GetName());
+  if (section_name.starts_with(".debug") ||
+      section_name.starts_with("__debug") ||
+      section_name.starts_with(".apple") || section_name.starts_with("__apple"))
+    debug_info_size += section.GetFileByteSize();
+
+  const size_t num_sub_sections = section.GetNumSubSections();
+  for (size_t i = 0; i < num_sub_sections; i++)
+    debug_info_size +=
+        GetDebugInfoSizeInSection(section.GetSubSectionAtIndex(i));
+
+  return debug_info_size;
+}
+
+static uint64_t GetDebugInfoSize(lldb::SBModule module) {
+  uint64_t debug_info_size = 0;
+  const size_t num_sections = module.GetNumSections();
+  for (size_t i = 0; i < num_sections; i++)
+    debug_info_size += GetDebugInfoSizeInSection(module.GetSectionAtIndex(i));
+
+  return debug_info_size;
+}
+
+std::string ConvertDebugInfoSizeToString(uint64_t debug_size) {
+  std::ostringstream oss;
+  oss << std::fixed << std::setprecision(1);
+  if (debug_size < 1024) {
+    oss << debug_size << "B";
+  } else if (debug_size < static_cast<uint64_t>(1024 * 1024)) {
+    double kb = double(debug_size) / 1024.0;
+    oss << kb << "KB";
+  } else if (debug_size < 1024 * 1024 * 1024) {
+    double mb = double(debug_size) / (1024.0 * 1024.0);
+    oss << mb << "MB";
+  } else {
+    double gb = double(debug_size) / (1024.0 * 1024.0 * 1024.0);
+    oss << gb << "GB";
+  }
+  return oss.str();
+}
+
+std::optional<protocol::Module> CreateModule(const lldb::SBTarget &target,
+                                             lldb::SBModule &module,
+                                             bool id_only) {
+  if (!target.IsValid() || !module.IsValid())
+    return std::nullopt;
+
+  const llvm::StringRef uuid = module.GetUUIDString();
+  if (uuid.empty())
+    return std::nullopt;
+
+  protocol::Module p_module;
+  p_module.id = uuid;
+
+  if (id_only)
+    return p_module;
+
+  std::array<char, PATH_MAX> path_buffer{};
+  if (const lldb::SBFileSpec file_spec = module.GetFileSpec()) {
+    p_module.name = file_spec.GetFilename();
+
+    const uint32_t path_size =
+        file_spec.GetPath(path_buffer.data(), path_buffer.size());
+    p_module.path = std::string(path_buffer.data(), path_size);
+  }
+
+  if (const uint32_t num_compile_units = module.GetNumCompileUnits();
+      num_compile_units > 0) {
+    p_module.symbolStatus = "Symbols loaded.";
+
+    p_module.debugInfoSizeBytes = GetDebugInfoSize(module);
+
+    if (const lldb::SBFileSpec symbol_fspec = module.GetSymbolFileSpec()) {
+      const uint32_t path_size =
+          symbol_fspec.GetPath(path_buffer.data(), path_buffer.size());
+      p_module.symbolFilePath = std::string(path_buffer.data(), path_size);
+    }
+  } else {
+    p_module.symbolStatus = "Symbols not found.";
+  }
+
+  const auto load_address = module.GetObjectFileHeaderAddress();
+  if (const lldb::addr_t raw_address = load_address.GetLoadAddress(target);
+      raw_address != LLDB_INVALID_ADDRESS)
+    p_module.addressRange = llvm::formatv("{0:x}", raw_address);
+
+  std::array<uint32_t, 3> version_nums{};
+  const uint32_t num_versions =
+      module.GetVersion(version_nums.data(), version_nums.size());
+  if (num_versions > 0) {
+    p_module.version = llvm::formatv(
+        "{:$[.]}", llvm::make_range(version_nums.begin(),
+                                    version_nums.begin() + num_versions));
+  }
+
+  return p_module;
+}
+
 std::optional<protocol::Source> CreateSource(const lldb::SBFileSpec &file) {
   if (!file.IsValid())
     return std::nullopt;
diff --git a/lldb/tools/lldb-dap/ProtocolUtils.h b/lldb/tools/lldb-dap/ProtocolUtils.h
index f36bf0fb60a87..d906d8e881158 100644
--- a/lldb/tools/lldb-dap/ProtocolUtils.h
+++ b/lldb/tools/lldb-dap/ProtocolUtils.h
@@ -20,6 +20,26 @@
 
 namespace lldb_dap {
 
+/// Converts a LLDB module to a DAP protocol module for use in `module events or
+/// request.
+///
+/// \param[in] target
+///     The target that has the module
+///
+/// \param[in] module
+///     A LLDB module object to convert into a protocol module
+///
+/// \param[in] id_only
+///     Only initialize the module ID in the return type. This is used when
+///     sending a "removed" module event.
+///
+/// \return
+///     A `protocol::Module` that follows the formal Module
+///     definition outlined by the DAP protocol.
+std::optional<protocol::Module> CreateModule(const lldb::SBTarget &target,
+                                             lldb::SBModule &module,
+                                             bool id_only = false);
+
 /// Create a "Source" JSON object as described in the debug adapter definition.
 ///
 /// \param[in] file
@@ -76,6 +96,16 @@ std::vector<protocol::Thread> GetThreads(lldb::SBProcess process,
 protocol::ExceptionBreakpointsFilter
 CreateExceptionBreakpointFilter(const ExceptionBreakpoint &bp);
 
+/// Converts a size in bytes to a human-readable string format.
+///
+/// \param[in] debug_size
+///     Size of the debug information in bytes (uint64_t).
+///
+/// \return
+///     A string representing the size in a readable format (e.g., "1 KB",
+///     "2 MB").
+std::string ConvertDebugInfoSizeToString(uint64_t debug_size);
+
 } // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-dap/tool/lldb-dap.cpp b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
index 2799d10ae84ba..8bba4162aa7bf 100644
--- a/lldb/tools/lldb-dap/tool/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
@@ -351,8 +351,6 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
                      << " disconnected failed: "
                      << llvm::toString(std::move(error)) << "\n";
       }
-      // Close the socket to ensure the DAP::Loop read finishes.
-      sock->Close();
     }
   }
 
diff --git a/lldb/tools/lldb-rpc/lldb-rpc-gen/lldb-rpc-gen.cpp b/lldb/tools/lldb-rpc/lldb-rpc-gen/lldb-rpc-gen.cpp
new file mode 100644
index 0000000000000..e6b601ea13012
--- /dev/null
+++ b/lldb/tools/lldb-rpc/lldb-rpc-gen/lldb-rpc-gen.cpp
@@ -0,0 +1,414 @@
+//===-- lldb-rpc-gen.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RPCCommon.h"
+#include "RPCServerHeaderEmitter.h"
+#include "RPCServerSourceEmitter.h"
+
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/CodeGen/ObjectFilePCHContainerWriter.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Serialization/ObjectFilePCHContainerReader.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+using namespace clang::driver;
+using namespace clang::tooling;
+
+static llvm::cl::OptionCategory RPCGenCategory("Tool for generating LLDBRPC");
+
+static llvm::cl::opt<std::string>
+    OutputDir("output-dir",
+              llvm::cl::desc("Directory to output generated files to"),
+              llvm::cl::init(""), llvm::cl::cat(RPCGenCategory));
+
+static std::string GetLibraryOutputDirectory() {
+  llvm::SmallString<128> Path(OutputDir.getValue());
+  llvm::sys::path::append(Path, "lib");
+  return std::string(Path);
+}
+
+static std::unique_ptr<llvm::ToolOutputFile>
+CreateOutputFile(llvm::StringRef OutputDir, llvm::StringRef Filename) {
+  llvm::SmallString<128> Path(OutputDir);
+  llvm::sys::path::append(Path, Filename);
+
+  std::error_code EC;
+  auto OutputFile =
+      std::make_unique<llvm::ToolOutputFile>(Path, EC, llvm::sys::fs::OF_None);
+  if (EC) {
+    llvm::errs() << "Failed to create output file: " << Path << "!\n";
+    return nullptr;
+  }
+  return OutputFile;
+}
+
+struct GeneratedByproducts {
+  std::set<std::string> ClassNames;
+  std::set<std::string> MangledMethodNames;
+  std::set<std::string> SkippedMethodNames;
+  std::set<lldb_rpc_gen::Method> CallbackMethods;
+};
+
+enum SupportLevel {
+  eUnsupported,
+  eUnimplemented,
+  eImplemented,
+};
+
+class SBVisitor : public RecursiveASTVisitor<SBVisitor> {
+public:
+  SBVisitor(GeneratedByproducts &Byproducts, SourceManager &Manager,
+            ASTContext &Context,
+            std::unique_ptr<llvm::ToolOutputFile> &&ServerMethodOutputFile,
+            std::unique_ptr<llvm::ToolOutputFile> &&ServerHeaderOutputFile)
+      : Byproducts(Byproducts), Manager(Manager), Context(Context),
+        ServerSourceEmitter(std::move(ServerMethodOutputFile)),
+        ServerHeaderEmitter(std::move(ServerHeaderOutputFile)) {}
+
+  ~SBVisitor() {}
+
+  bool VisitCXXRecordDecl(CXXRecordDecl *RDecl) {
+    if (ShouldSkipRecord(RDecl))
+      return true;
+
+    const std::string ClassName = RDecl->getNameAsString();
+    Byproducts.ClassNames.insert(ClassName);
+
+    // Print 'bool' instead of '_Bool'.
+    PrintingPolicy Policy(Context.getLangOpts());
+    Policy.Bool = true;
+
+    for (CXXMethodDecl *MDecl : RDecl->methods()) {
+      const std::string MangledName =
+          lldb_rpc_gen::GetMangledName(Context, MDecl);
+      const bool IsDisallowed = lldb_rpc_gen::MethodIsDisallowed(MangledName);
+      const bool HasCallbackParameter =
+          lldb_rpc_gen::HasCallbackParameter(MDecl);
+      SupportLevel MethodSupportLevel = GetMethodSupportLevel(MDecl);
+      if (MethodSupportLevel == eImplemented && !IsDisallowed) {
+        const lldb_rpc_gen::Method Method(MDecl, Policy, Context);
+        ServerSourceEmitter.EmitMethod(Method);
+        ServerHeaderEmitter.EmitMethod(Method);
+        Byproducts.MangledMethodNames.insert(MangledName);
+      } else if (MethodSupportLevel == eUnimplemented)
+        Byproducts.SkippedMethodNames.insert(MangledName);
+    }
+    return true;
+  }
+
+private:
+  /// Determines whether we should skip a RecordDecl.
+  /// Conditions for skipping:
+  ///   - Anything not in the header itself
+  ///   - Certain inconvenient classes
+  ///   - Records without definitions (forward declarations)
+  bool ShouldSkipRecord(CXXRecordDecl *Decl) {
+    const Type *DeclType = Decl->getTypeForDecl();
+    QualType CanonicalType = DeclType->getCanonicalTypeInternal();
+    return !Manager.isInMainFile(Decl->getBeginLoc()) ||
+           !Decl->hasDefinition() || Decl->getDefinition() != Decl ||
+           lldb_rpc_gen::TypeIsDisallowedClass(CanonicalType);
+  }
+
+  /// Check the support level for a type
+  /// Known unsupported types:
+  ///  - FILE * (We do not want to expose this primitive)
+  ///  - Types that are internal to LLDB
+  SupportLevel GetTypeSupportLevel(QualType Type) {
+    const std::string TypeName = Type.getAsString();
+    if (TypeName == "FILE *" || lldb_rpc_gen::TypeIsFromLLDBPrivate(Type))
+      return eUnsupported;
+
+    if (lldb_rpc_gen::TypeIsDisallowedClass(Type))
+      return eUnsupported;
+
+    return eImplemented;
+  }
+
+  /// Determine the support level of a given method.
+  /// Known unsupported methods:
+  ///   - Non-public methods (lldb-rpc is a client and can only see public
+  ///     things)
+  ///   - Copy assignment operators (the client side will handle this)
+  ///   - Move assignment operators (the client side will handle this)
+  ///   - Methods involving unsupported types.
+  /// Known unimplemented methods:
+  ///   - No variadic functions, e.g. Printf
+  SupportLevel GetMethodSupportLevel(CXXMethodDecl *MDecl) {
+    AccessSpecifier AS = MDecl->getAccess();
+    if (AS != AccessSpecifier::AS_public)
+      return eUnsupported;
+    if (MDecl->isCopyAssignmentOperator())
+      return eUnsupported;
+    if (MDecl->isMoveAssignmentOperator())
+      return eUnsupported;
+
+    if (MDecl->isVariadic())
+      return eUnimplemented;
+
+    SupportLevel ReturnTypeLevel = GetTypeSupportLevel(MDecl->getReturnType());
+    if (ReturnTypeLevel != eImplemented)
+      return ReturnTypeLevel;
+
+    for (auto *ParamDecl : MDecl->parameters()) {
+      SupportLevel ParamTypeLevel = GetTypeSupportLevel(ParamDecl->getType());
+      if (ParamTypeLevel != eImplemented)
+        return ParamTypeLevel;
+    }
+
+    // FIXME: If a callback does not take a `void *baton` parameter, it is
+    // considered unsupported at this time. On the server-side, we hijack the
+    // baton argument in order to pass additional information to the server-side
+    // callback so we can correctly perform a reverse RPC call back to the
+    // client. Without this baton, we would need the server-side callback to
+    // have some side channel by which it obtained that information, and
+    // spending time designing that doesn't outweight the cost of doing it at
+    // the moment.
+    bool HasCallbackParameter = false;
+    bool HasBatonParameter = false;
+    auto End = MDecl->parameters().end();
+    for (auto Iter = MDecl->parameters().begin(); Iter != End; Iter++) {
+      if ((*Iter)->getType()->isFunctionPointerType()) {
+        HasCallbackParameter = true;
+        continue;
+      }
+
+      // FIXME: We assume that if we have a function pointer and a void pointer
+      // together in the same parameter list, that it is not followed by a
+      // length argument. If that changes, we will need to revisit this
+      // implementation.
+      if ((*Iter)->getType()->isVoidPointerType())
+        HasBatonParameter = true;
+    }
+
+    if (HasCallbackParameter && !HasBatonParameter)
+      return eUnimplemented;
+
+    return eImplemented;
+  }
+
+  GeneratedByproducts &Byproducts;
+  SourceManager &Manager;
+  ASTContext &Context;
+  lldb_rpc_gen::RPCServerSourceEmitter ServerSourceEmitter;
+  lldb_rpc_gen::RPCServerHeaderEmitter ServerHeaderEmitter;
+};
+
+class SBConsumer : public ASTConsumer {
+public:
+  SBConsumer(GeneratedByproducts &Byproducts, SourceManager &Manager,
+             ASTContext &Context,
+             std::unique_ptr<llvm::ToolOutputFile> &&ServerMethodOutputFile,
+             std::unique_ptr<llvm::ToolOutputFile> &&ServerHeaderOutputFile)
+      : Visitor(Byproducts, Manager, Context, std::move(ServerMethodOutputFile),
+                std::move(ServerHeaderOutputFile)) {}
+  bool HandleTopLevelDecl(DeclGroupRef DR) override {
+    for (Decl *D : DR)
+      Visitor.TraverseDecl(D);
+
+    return true;
+  }
+
+private:
+  SBVisitor Visitor;
+};
+
+class SBAction : public ASTFrontendAction {
+public:
+  SBAction(GeneratedByproducts &Byproducts) : Byproducts(Byproducts) {}
+
+  std::unique_ptr<ASTConsumer>
+  CreateASTConsumer(CompilerInstance &CI, llvm::StringRef File) override {
+    llvm::StringRef FilenameNoExt =
+        llvm::sys::path::stem(llvm::sys::path::filename(File));
+
+    const std::string ServerMethodFilename =
+        "Server_" + FilenameNoExt.str() + ".cpp";
+    std::unique_ptr<llvm::ToolOutputFile> ServerMethodOutputFile =
+        CreateOutputFile(GetServerOutputDirectory(), ServerMethodFilename);
+    if (!ServerMethodOutputFile)
+      return nullptr;
+
+    const std::string ServerHeaderFilename =
+        "Server_" + FilenameNoExt.str() + ".h";
+    std::unique_ptr<llvm::ToolOutputFile> ServerHeaderOutputFile =
+        CreateOutputFile(GetServerOutputDirectory(), ServerHeaderFilename);
+    if (!ServerHeaderOutputFile)
+      return nullptr;
+
+    ServerMethodOutputFile->keep();
+    ServerHeaderOutputFile->keep();
+    return std::make_unique<SBConsumer>(
+        Byproducts, CI.getSourceManager(), CI.getASTContext(),
+        std::move(ServerMethodOutputFile), std::move(ServerHeaderOutputFile));
+  }
+
+private:
+  GeneratedByproducts &Byproducts;
+};
+
+class SBActionFactory : public FrontendActionFactory {
+public:
+  SBActionFactory(GeneratedByproducts &Byproducts) : Byproducts(Byproducts) {}
+
+  std::unique_ptr<FrontendAction> create() override {
+    return std::make_unique<SBAction>(Byproducts);
+  }
+
+private:
+  GeneratedByproducts &Byproducts;
+};
+
+bool EmitAmalgamatedServerHeader(const std::vector<std::string> &Files) {
+  // Create the file
+  static constexpr llvm::StringLiteral AmalgamatedServerHeaderName = "SBAPI.h";
+  std::unique_ptr<llvm::ToolOutputFile> AmalgamatedServerHeader =
+      CreateOutputFile(GetServerOutputDirectory(), AmalgamatedServerHeaderName);
+  if (!AmalgamatedServerHeader)
+    return false;
+
+  // Write the header
+  AmalgamatedServerHeader->os()
+      << "#ifndef GENERATED_LLDB_RPC_SERVER_SBAPI_H\n";
+  AmalgamatedServerHeader->os()
+      << "#define GENERATED_LLDB_RPC_SERVER_SBAPI_H\n";
+  for (const auto &File : Files) {
+    llvm::StringRef FilenameNoExt =
+        llvm::sys::path::stem(llvm::sys::path::filename(File));
+    const std::string ServerHeaderFilename =
+        "Server_" + FilenameNoExt.str() + ".h";
+
+    AmalgamatedServerHeader->os()
+        << "#include \"" + ServerHeaderFilename + "\"\n";
+  }
+  AmalgamatedServerHeader->os() << "#include \"SBAPIExtensions.h\"\n";
+  AmalgamatedServerHeader->os()
+      << "#endif // GENERATED_LLDB_RPC_SERVER_SBAPI_H\n";
+  AmalgamatedServerHeader->keep();
+  return true;
+}
+
+bool EmitClassNamesFile(std::set<std::string> &ClassNames) {
+  static constexpr llvm::StringLiteral ClassNamesFileName = "SBClasses.def";
+  std::unique_ptr<llvm::ToolOutputFile> ClassNamesFile =
+      CreateOutputFile(OutputDir.getValue(), ClassNamesFileName);
+  if (!ClassNamesFile)
+    return false;
+
+  ClassNamesFile->os() << "#ifndef SBCLASS\n";
+  ClassNamesFile->os() << "#error \"SBClass must be defined\"\n";
+  ClassNamesFile->os() << "#endif\n";
+
+  for (const auto &ClassName : ClassNames) {
+    if (ClassName == "SBStream" || ClassName == "SBProgress")
+      ClassNamesFile->os() << "#if !defined(SBCLASS_EXCLUDE_NONCOPYABLE)\n";
+    else if (ClassName == "SBReproducer")
+      ClassNamesFile->os() << "#if !defined(SBCLASS_EXCLUDE_STATICONLY)\n";
+
+    ClassNamesFile->os() << "SBCLASS(" << ClassName << ")\n";
+    if (ClassName == "SBStream" || ClassName == "SBReproducer" ||
+        ClassName == "SBProgress")
+      ClassNamesFile->os() << "#endif\n";
+  }
+  ClassNamesFile->keep();
+  return true;
+}
+
+bool EmitMethodNamesFile(std::set<std::string> &MangledMethodNames) {
+  static constexpr llvm::StringLiteral MethodNamesFileName = "SBAPI.def";
+  std::unique_ptr<llvm::ToolOutputFile> MethodNamesFile =
+      CreateOutputFile(OutputDir.getValue(), MethodNamesFileName);
+  if (!MethodNamesFile)
+    return false;
+
+  MethodNamesFile->os() << "#ifndef GENERATE_SBAPI\n";
+  MethodNamesFile->os() << "#error \"GENERATE_SBAPI must be defined\"\n";
+  MethodNamesFile->os() << "#endif\n";
+
+  for (const auto &MangledName : MangledMethodNames) {
+    MethodNamesFile->os() << "GENERATE_SBAPI(" << MangledName << ")\n";
+  }
+  MethodNamesFile->keep();
+  return true;
+}
+
+bool EmitSkippedMethodsFile(std::set<std::string> &SkippedMethodNames) {
+  static constexpr llvm::StringLiteral FileName = "SkippedMethods.txt";
+  std::unique_ptr<llvm::ToolOutputFile> File =
+      CreateOutputFile(OutputDir.getValue(), FileName);
+  if (!File)
+    return false;
+
+  for (const auto &Skipped : SkippedMethodNames) {
+    File->os() << Skipped << "\n";
+  }
+  File->keep();
+  return true;
+}
+
+int main(int argc, const char *argv[]) {
+  auto ExpectedParser = CommonOptionsParser::create(
+      argc, argv, RPCGenCategory, llvm::cl::OneOrMore,
+      "Tool for generating LLDBRPC interfaces and implementations");
+
+  if (!ExpectedParser) {
+    llvm::errs() << ExpectedParser.takeError();
+    return 1;
+  }
+
+  if (OutputDir.empty()) {
+    llvm::errs() << "Please specify an output directory for the generated "
+                    "files with --output-dir!\n";
+    return 1;
+  }
+
+  CommonOptionsParser &OP = ExpectedParser.get();
+  auto PCHOpts = std::make_shared<PCHContainerOperations>();
+  PCHOpts->registerWriter(std::make_unique<ObjectFilePCHContainerWriter>());
+  PCHOpts->registerReader(std::make_unique<ObjectFilePCHContainerReader>());
+
+  ClangTool T(OP.getCompilations(), OP.getSourcePathList(), PCHOpts);
+
+  if (!EmitAmalgamatedServerHeader(OP.getSourcePathList())) {
+    llvm::errs() << "Failed to create amalgamated server header\n";
+    return 1;
+  }
+
+  GeneratedByproducts Byproducts;
+
+  SBActionFactory Factory(Byproducts);
+  auto Result = T.run(&Factory);
+  if (!EmitClassNamesFile(Byproducts.ClassNames)) {
+    llvm::errs() << "Failed to create SB Class file\n";
+    return 1;
+  }
+  if (!EmitMethodNamesFile(Byproducts.MangledMethodNames)) {
+    llvm::errs() << "Failed to create Method Names file\n";
+    return 1;
+  }
+  if (!EmitSkippedMethodsFile(Byproducts.SkippedMethodNames)) {
+    llvm::errs() << "Failed to create Skipped Methods file\n";
+    return 1;
+  }
+
+  return Result;
+}
diff --git a/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerHeaderEmitter.cpp b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerHeaderEmitter.cpp
new file mode 100644
index 0000000000000..30a18e3c7044b
--- /dev/null
+++ b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerHeaderEmitter.cpp
@@ -0,0 +1,75 @@
+//===-- RPCServerHeaderEmitter.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RPCServerHeaderEmitter.h"
+#include "RPCCommon.h"
+
+#include "clang/AST/AST.h"
+#include "clang/AST/Mangle.h"
+#include "clang/Frontend/CompilerInstance.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+using namespace lldb_rpc_gen;
+
+void RPCServerHeaderEmitter::EmitMethod(const Method &method) {
+  // We'll be using the mangled name in order to disambiguate
+  // overloaded methods.
+  const std::string &MangledName = method.MangledName;
+
+  EmitLine("class " + MangledName +
+           " : public rpc_common::RPCFunctionInstance {");
+  EmitLine("public:");
+  IndentLevel++;
+  EmitConstructor(MangledName);
+  EmitDestructor(MangledName);
+  EmitHandleRPCCall();
+  IndentLevel--;
+  EmitLine("};");
+}
+
+void RPCServerHeaderEmitter::EmitHandleRPCCall() {
+  EmitLine("bool HandleRPCCall(rpc_common::Connection &connection, "
+           "rpc_common::RPCStream &send, rpc_common::RPCStream &response) "
+           "override;");
+}
+
+void RPCServerHeaderEmitter::EmitConstructor(const std::string &MangledName) {
+  EmitLine(MangledName + "() : RPCFunctionInstance(\"" + MangledName +
+           "\") {}");
+}
+
+void RPCServerHeaderEmitter::EmitDestructor(const std::string &MangledName) {
+  EmitLine("~" + MangledName + "() override {}");
+}
+
+std::string RPCServerHeaderEmitter::GetHeaderGuard() {
+  const std::string UpperFilenameNoExt =
+      llvm::sys::path::stem(
+          llvm::sys::path::filename(OutputFile->getFilename()))
+          .upper();
+  return "GENERATED_LLDB_RPC_SERVER_" + UpperFilenameNoExt + "_H";
+}
+
+void RPCServerHeaderEmitter::Begin() {
+  const std::string HeaderGuard = GetHeaderGuard();
+  EmitLine("#ifndef " + HeaderGuard);
+  EmitLine("#define " + HeaderGuard);
+  EmitLine("");
+  EmitLine("#include <lldb-rpc/common/RPCFunction.h>");
+  EmitLine("");
+  EmitLine("namespace rpc_server {");
+}
+
+void RPCServerHeaderEmitter::End() {
+  EmitLine("} // namespace rpc_server");
+  EmitLine("#endif // " + GetHeaderGuard());
+}
diff --git a/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerHeaderEmitter.h b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerHeaderEmitter.h
new file mode 100644
index 0000000000000..a0e4f47d4f570
--- /dev/null
+++ b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerHeaderEmitter.h
@@ -0,0 +1,47 @@
+//===-- RPCServerHeaderEmitter.h ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_RPC_GEN_RPCSERVERHEADEREMITTER_H
+#define LLDB_RPC_GEN_RPCSERVERHEADEREMITTER_H
+
+#include "RPCCommon.h"
+
+#include "clang/AST/AST.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+using namespace clang;
+
+namespace lldb_rpc_gen {
+/// Emit the source code for server-side *.h files.
+class RPCServerHeaderEmitter : public FileEmitter {
+public:
+  RPCServerHeaderEmitter(std::unique_ptr<llvm::ToolOutputFile> &&OutputFile)
+      : FileEmitter(std::move(OutputFile)) {
+    Begin();
+  }
+
+  ~RPCServerHeaderEmitter() { End(); }
+
+  void EmitMethod(const Method &method);
+
+private:
+  void EmitHandleRPCCall();
+
+  void EmitConstructor(const std::string &MangledName);
+
+  void EmitDestructor(const std::string &MangledName);
+
+  std::string GetHeaderGuard();
+
+  void Begin();
+
+  void End();
+};
+} // namespace lldb_rpc_gen
+
+#endif // LLDB_RPC_GEN_RPCSERVERHEADEREMITTER_H
diff --git a/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerSourceEmitter.cpp b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerSourceEmitter.cpp
new file mode 100644
index 0000000000000..a6d2528fe2c3c
--- /dev/null
+++ b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerSourceEmitter.cpp
@@ -0,0 +1,584 @@
+//===-- RPCServerSourceEmitter.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RPCServerSourceEmitter.h"
+#include "RPCCommon.h"
+
+#include "clang/AST/AST.h"
+#include "clang/Frontend/CompilerInstance.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <map>
+
+using namespace clang;
+using namespace lldb_rpc_gen;
+
+// For methods with pointer return types, it's important that we know how big
+// the type of the pointee is. We must correctly size a buffer (in the form of a
+// Bytes object) before we can actually use it.
+static const std::map<llvm::StringRef, size_t> MethodsWithPointerReturnTypes = {
+    {"_ZN4lldb12SBModuleSpec12GetUUIDBytesEv", 16}, // sizeof(uuid_t) -> 16
+    {"_ZNK4lldb8SBModule12GetUUIDBytesEv", 16},     // sizeof(uuid_t) -> 16
+};
+
+void RPCServerSourceEmitter::EmitMethod(const Method &method) {
+  if (method.ContainsFunctionPointerParameter)
+    EmitCallbackFunction(method);
+
+  EmitCommentHeader(method);
+  EmitFunctionHeader(method);
+  EmitFunctionBody(method);
+  EmitFunctionFooter();
+}
+
+void RPCServerSourceEmitter::EmitCommentHeader(const Method &method) {
+  std::string CommentLine;
+  llvm::raw_string_ostream CommentStream(CommentLine);
+
+  CommentStream << "// " << method.QualifiedName << "("
+                << method.CreateParamListAsString(eServer) << ")";
+  if (method.IsConst)
+    CommentStream << " const";
+
+  EmitLine("//------------------------------------------------------------");
+  EmitLine(CommentLine);
+  EmitLine("//------------------------------------------------------------");
+}
+
+void RPCServerSourceEmitter::EmitFunctionHeader(const Method &method) {
+  std::string FunctionHeader;
+  llvm::raw_string_ostream FunctionHeaderStream(FunctionHeader);
+  FunctionHeaderStream
+      << "bool rpc_server::" << method.MangledName
+      << "::HandleRPCCall(rpc_common::Connection &connection, RPCStream "
+         "&send, RPCStream &response) {";
+  EmitLine(FunctionHeader);
+  IndentLevel++;
+}
+
+void RPCServerSourceEmitter::EmitFunctionBody(const Method &method) {
+  EmitLine("// 1) Make local storage for incoming function arguments");
+  EmitStorageForParameters(method);
+  EmitLine("// 2) Decode all function arguments");
+  EmitDecodeForParameters(method);
+  EmitLine("// 3) Call the method and encode the return value");
+  EmitMethodCallAndEncode(method);
+}
+
+void RPCServerSourceEmitter::EmitFunctionFooter() {
+  EmitLine("return true;");
+  IndentLevel--;
+  EmitLine("}");
+}
+
+void RPCServerSourceEmitter::EmitStorageForParameters(const Method &method) {
+  // If we have an instance method and it isn't a constructor, we'll need to
+  // emit a "this" pointer.
+  if (method.IsInstance && !method.IsCtor)
+    EmitStorageForOneParameter(method.ThisType, "this_ptr", method.Policy,
+                               /* IsFollowedByLen = */ false);
+  for (auto Iter = method.Params.begin(); Iter != method.Params.end(); Iter++) {
+    EmitStorageForOneParameter(Iter->Type, Iter->Name, method.Policy,
+                               Iter->IsFollowedByLen);
+    // Skip over the length parameter, we don't emit it.
+    if (!lldb_rpc_gen::TypeIsConstCharPtrPtr(Iter->Type) &&
+        Iter->IsFollowedByLen)
+      Iter++;
+  }
+}
+
+void RPCServerSourceEmitter::EmitStorageForOneParameter(
+    QualType ParamType, const std::string &ParamName,
+    const PrintingPolicy &Policy, bool IsFollowedByLen) {
+  // First, we consider `const char *`, `const char **`. They have special
+  // server-side types.
+  if (TypeIsConstCharPtr(ParamType)) {
+    EmitLine("rpc_common::ConstCharPointer " + ParamName + ";");
+    return;
+  } else if (TypeIsConstCharPtrPtr(ParamType)) {
+    EmitLine("rpc_common::StringList " + ParamName + ";");
+    return;
+  }
+
+  QualType UnderlyingType =
+      lldb_rpc_gen::GetUnqualifiedUnderlyingType(ParamType);
+  const bool IsSBClass = lldb_rpc_gen::TypeIsSBClass(UnderlyingType);
+
+  if (ParamType->isPointerType() && !IsSBClass) {
+    // Void pointer with no length is usually a baton for a callback. We're
+    // going to hold onto the pointer value so we can send it back to the
+    // client-side when we implement callbacks.
+    if (ParamType->isVoidPointerType() && !IsFollowedByLen) {
+      EmitLine("void * " + ParamName + " = nullptr;");
+      return;
+    }
+
+    if (!ParamType->isFunctionPointerType()) {
+      EmitLine("Bytes " + ParamName + ";");
+      return;
+    }
+
+    assert(ParamType->isFunctionPointerType() && "Unhandled pointer type");
+    EmitLine("rpc_common::function_ptr_t " + ParamName + " = nullptr;");
+    return;
+  }
+
+  std::string StorageDeclaration;
+  llvm::raw_string_ostream StorageDeclarationStream(StorageDeclaration);
+
+  UnderlyingType.print(StorageDeclarationStream, Policy);
+  StorageDeclarationStream << " ";
+  if (IsSBClass)
+    StorageDeclarationStream << "*";
+  StorageDeclarationStream << ParamName;
+  if (IsSBClass)
+    StorageDeclarationStream << " = nullptr";
+  else
+    StorageDeclarationStream << " = {}";
+  StorageDeclarationStream << ";";
+  EmitLine(StorageDeclaration);
+}
+
+void RPCServerSourceEmitter::EmitDecodeForParameters(const Method &method) {
+  if (method.IsInstance && !method.IsCtor)
+    EmitDecodeForOneParameter(method.ThisType, "this_ptr", method.Policy);
+  for (auto Iter = method.Params.begin(); Iter != method.Params.end(); Iter++) {
+    EmitDecodeForOneParameter(Iter->Type, Iter->Name, method.Policy);
+    if (!lldb_rpc_gen::TypeIsConstCharPtrPtr(Iter->Type) &&
+        Iter->IsFollowedByLen)
+      Iter++;
+  }
+}
+
+void RPCServerSourceEmitter::EmitDecodeForOneParameter(
+    QualType ParamType, const std::string &ParamName,
+    const PrintingPolicy &Policy) {
+  QualType UnderlyingType =
+      lldb_rpc_gen::GetUnqualifiedUnderlyingType(ParamType);
+
+  if (TypeIsSBClass(UnderlyingType)) {
+    std::string DecodeLine;
+    llvm::raw_string_ostream DecodeLineStream(DecodeLine);
+    DecodeLineStream << ParamName << " = "
+                     << "RPCServerObjectDecoder<";
+    UnderlyingType.print(DecodeLineStream, Policy);
+    DecodeLineStream << ">(send, rpc_common::RPCPacket::ValueType::Argument);";
+    EmitLine(DecodeLine);
+    EmitLine("if (!" + ParamName + ")");
+    IndentLevel++;
+    EmitLine("return false;");
+    IndentLevel--;
+  } else {
+    EmitLine("if (!RPCValueDecoder(send, "
+             "rpc_common::RPCPacket::ValueType::Argument, " +
+             ParamName + "))");
+    IndentLevel++;
+    EmitLine("return false;");
+    IndentLevel--;
+  }
+}
+
+std::string RPCServerSourceEmitter::CreateMethodCall(const Method &method) {
+  std::string MethodCall;
+  llvm::raw_string_ostream MethodCallStream(MethodCall);
+  if (method.IsInstance) {
+    if (!method.IsCtor)
+      MethodCallStream << "this_ptr->";
+    MethodCallStream << method.BaseName;
+  } else
+    MethodCallStream << method.QualifiedName;
+
+  std::vector<std::string> Args;
+  std::string FunctionPointerName;
+  for (auto Iter = method.Params.begin(); Iter != method.Params.end(); Iter++) {
+    std::string Arg;
+    // We must check for `const char *` and `const char **` first.
+    if (TypeIsConstCharPtr(Iter->Type)) {
+      // `const char *` is stored server-side as rpc_common::ConstCharPointer
+      Arg = Iter->Name + ".c_str()";
+    } else if (TypeIsConstCharPtrPtr(Iter->Type)) {
+      // `const char **` is stored server-side as rpc_common::StringList
+      Arg = Iter->Name + ".argv()";
+    } else if (lldb_rpc_gen::TypeIsSBClass(Iter->Type)) {
+      Arg = Iter->Name;
+      if (!Iter->Type->isPointerType())
+        Arg = "*" + Iter->Name;
+    } else if (Iter->Type->isPointerType() &&
+               !Iter->Type->isFunctionPointerType() &&
+               (!Iter->Type->isVoidPointerType() || Iter->IsFollowedByLen)) {
+      // We move pointers between the server and client as 'Bytes' objects.
+      // Pointers with length arguments will have their length filled in below.
+      // Pointers with no length arguments are assumed to behave like an array
+      // with length of 1, except for void pointers which are handled
+      // differently.
+      Arg = "(" + Iter->Type.getAsString(method.Policy) + ")" + Iter->Name +
+            ".GetData()";
+    } else if (Iter->Type->isFunctionPointerType()) {
+      // If we have a function pointer, we only want to pass something along if
+      // we got a real pointer.
+      Arg = Iter->Name + " ? " + method.MangledName + "_callback : nullptr";
+      FunctionPointerName = Iter->Name;
+    } else if (Iter->Type->isVoidPointerType() && !Iter->IsFollowedByLen &&
+               method.ContainsFunctionPointerParameter) {
+      // Assumptions:
+      //  - This is assumed to be the baton for the function pointer.
+      //  - This is assumed to come after the function pointer parameter.
+      // We always produce this regardless of the value of the baton argument.
+      Arg = "new CallbackInfo(" + FunctionPointerName + ", " + Iter->Name +
+            ", connection.GetConnectionID())";
+    } else
+      Arg = Iter->Name;
+
+    if (Iter->Type->isRValueReferenceType())
+      Arg = "std::move(" + Arg + ")";
+    Args.push_back(Arg);
+
+    if (!lldb_rpc_gen::TypeIsConstCharPtrPtr(Iter->Type) &&
+        Iter->IsFollowedByLen) {
+      std::string LengthArg = Iter->Name + ".GetSize()";
+      if (!Iter->Type->isVoidPointerType()) {
+        QualType UUT = lldb_rpc_gen::GetUnqualifiedUnderlyingType(Iter->Type);
+        LengthArg += " / sizeof(" + UUT.getAsString(method.Policy) + ")";
+      }
+      Args.push_back(LengthArg);
+      Iter++;
+    }
+  }
+  MethodCallStream << "(" << llvm::join(Args, ", ") << ")";
+
+  return MethodCall;
+}
+
+std::string RPCServerSourceEmitter::CreateEncodeLine(const std::string &Value,
+                                                     bool IsEncodingSBClass) {
+  std::string EncodeLine;
+  llvm::raw_string_ostream EncodeLineStream(EncodeLine);
+
+  if (IsEncodingSBClass)
+    EncodeLineStream << "RPCServerObjectEncoder(";
+  else
+    EncodeLineStream << "RPCValueEncoder(";
+
+  EncodeLineStream
+      << "response, rpc_common::RPCPacket::ValueType::ReturnValue, ";
+  EncodeLineStream << Value;
+  EncodeLineStream << ");";
+  return EncodeLine;
+}
+
+// There are 4 cases to consider:
+// - const SBClass &: No need to do anything.
+// - const foo &: No need to do anything.
+// - SBClass &: The server and the client hold on to IDs to refer to specific
+//   instances, so there's no need to send any information back to the client.
+// - foo &: The client is sending us a value over the wire, but because the type
+//   is mutable, we must send the changed value back in case the method call
+//   mutated it.
+//
+// Updating a mutable reference is done as a return value from the RPC
+// perspective. These return values need to be emitted after the method's return
+// value, and they are emitted in the order in which they occur in the
+// declaration.
+void RPCServerSourceEmitter::EmitEncodesForMutableParameters(
+    const std::vector<Param> &Params) {
+  for (auto Iter = Params.begin(); Iter != Params.end(); Iter++) {
+    // No need to manually update an SBClass
+    if (lldb_rpc_gen::TypeIsSBClass(Iter->Type))
+      continue;
+
+    if (!Iter->Type->isReferenceType() && !Iter->Type->isPointerType())
+      continue;
+
+    // If we have a void pointer with no length, there's nothing to update. This
+    // is likely a baton for a callback. The same goes for function pointers.
+    if (Iter->Type->isFunctionPointerType() ||
+        (Iter->Type->isVoidPointerType() && !Iter->IsFollowedByLen))
+      continue;
+
+    // No need to update pointers and references to const-qualified data.
+    QualType UnderlyingType = lldb_rpc_gen::GetUnderlyingType(Iter->Type);
+    if (UnderlyingType.isConstQualified())
+      continue;
+
+    const std::string EncodeLine =
+        CreateEncodeLine(Iter->Name, /* IsEncodingSBClass = */ false);
+    EmitLine(EncodeLine);
+  }
+}
+
+// There are 3 possible scenarios that this method can encounter:
+// 1. The method has no return value and is not a constructor.
+//    Only the method call itself is emitted.
+// 2. The method is a constructor.
+//    The call to the constructor is emitted in the encode line.
+// 3. The method has a return value.
+//    The method call is emitted and the return value is captured in a variable.
+//    After that, an encode call is emitted with the variable that captured the
+//    return value.
+void RPCServerSourceEmitter::EmitMethodCallAndEncode(const Method &method) {
+  const std::string MethodCall = CreateMethodCall(method);
+
+  // If this function returns nothing, we just emit the call and update any
+  // mutable references. Note that constructors have return type `void` so we
+  // must explicitly check for that here.
+  if (!method.IsCtor && method.ReturnType->isVoidType()) {
+    EmitLine(MethodCall + ";");
+    EmitEncodesForMutableParameters(method.Params);
+    return;
+  }
+
+  static constexpr llvm::StringLiteral ReturnVariableName("__result");
+
+  // If this isn't a constructor, we'll need to store the result of the method
+  // call in a result variable.
+  if (!method.IsCtor) {
+    // We need to determine what the appropriate return type is. Here is the
+    // strategy:
+    // 1.) `SBFoo` -> `SBFoo &&`
+    // 2.) If the type is a pointer other than `const char *` or `const char **`
+    //     or `void *`, the return type will be `Bytes` (e.g. `const uint8_t *`
+    //     -> `Bytes`).
+    // 3.) Otherwise, emit the exact same return type.
+    std::string ReturnTypeName;
+    std::string AssignLine;
+    llvm::raw_string_ostream AssignLineStream(AssignLine);
+    if (method.ReturnType->isPointerType() &&
+        !lldb_rpc_gen::TypeIsConstCharPtr(method.ReturnType) &&
+        !lldb_rpc_gen::TypeIsConstCharPtrPtr(method.ReturnType) &&
+        !method.ReturnType->isVoidPointerType()) {
+      llvm::StringRef MangledNameRef(method.MangledName);
+      auto Pos = MethodsWithPointerReturnTypes.find(MangledNameRef);
+      assert(Pos != MethodsWithPointerReturnTypes.end() &&
+             "Unable to determine the size of the return buffer");
+      if (Pos == MethodsWithPointerReturnTypes.end()) {
+        EmitLine(
+            "// Intentionally inserting a compiler error. lldb-rpc-gen "
+            "was unable to determine how large the return buffer should be.");
+        EmitLine("#error: \"unable to determine size of return buffer\"");
+        return;
+      }
+      AssignLineStream << "Bytes " << ReturnVariableName << "(" << MethodCall
+                       << ", " << Pos->second << ");";
+    } else {
+      if (lldb_rpc_gen::TypeIsSBClass(method.ReturnType)) {
+        // We want to preserve constness, so we don't strip qualifications from
+        // the underlying type
+        QualType UnderlyingReturnType =
+            lldb_rpc_gen::GetUnderlyingType(method.ReturnType);
+        ReturnTypeName =
+            UnderlyingReturnType.getAsString(method.Policy) + " &&";
+      } else
+        ReturnTypeName = method.ReturnType.getAsString(method.Policy);
+
+      AssignLineStream << ReturnTypeName << " " << ReturnVariableName << " = "
+                       << MethodCall << ";";
+    }
+    EmitLine(AssignLine);
+  }
+
+  const bool IsEncodingSBClass =
+      lldb_rpc_gen::TypeIsSBClass(method.ReturnType) || method.IsCtor;
+
+  std::string ValueToEncode;
+  if (IsEncodingSBClass) {
+    if (method.IsCtor)
+      ValueToEncode = MethodCall;
+    else
+      ValueToEncode = "std::move(" + ReturnVariableName.str() + ")";
+  } else
+    ValueToEncode = ReturnVariableName.str();
+
+  const std::string ReturnValueEncodeLine =
+      CreateEncodeLine(ValueToEncode, IsEncodingSBClass);
+  EmitLine(ReturnValueEncodeLine);
+  EmitEncodesForMutableParameters(method.Params);
+}
+
+// NOTE: This contains most of the same knowledge as RPCLibrarySourceEmitter. I
+// have chosen not to re-use code here because the needs are different enough
+// that it would be more work to re-use than just reimplement portions of it.
+// Specifically:
+//  - Callbacks do not neatly fit into a `Method` object, which currently
+//    assumes that you have a CXXMethodDecl (We have a FunctionDecl at most).
+//  - We only generate callbacks that have a `void *` baton parameter. We hijack
+//    those baton parameters and treat them differently.
+//  - Callbacks need to do something special for moving SB class references back
+//    to the client-side.
+void RPCServerSourceEmitter::EmitCallbackFunction(const Method &method) {
+  // Check invariants and locate necessary resources
+  Param FuncPointerParam;
+  Param BatonParam;
+  for (const auto &Param : method.Params)
+    if (Param.Type->isFunctionPointerType())
+      FuncPointerParam = Param;
+    else if (Param.Type->isVoidPointerType())
+      BatonParam = Param;
+
+  assert(FuncPointerParam.Type->isFunctionPointerType() &&
+         "Emitting callback function with no function pointer");
+  assert(BatonParam.Type->isVoidPointerType() &&
+         "Emitting callback function with no baton");
+
+  QualType FuncType = FuncPointerParam.Type->getPointeeType();
+  const auto *FuncProtoType = FuncType->getAs<FunctionProtoType>();
+  assert(FuncProtoType && "Emitting callback with no parameter information");
+  if (!FuncProtoType)
+    return; // If asserts are off, we'll just fail to compile.
+
+  std::vector<Param> CallbackParams;
+  std::vector<std::string> CallbackParamsAsStrings;
+  uint8_t ArgIdx = 0;
+  for (QualType ParamType : FuncProtoType->param_types()) {
+    Param CallbackParam;
+    CallbackParam.IsFollowedByLen = false;
+    CallbackParam.Type = ParamType;
+    if (ParamType->isVoidPointerType())
+      CallbackParam.Name = "baton";
+    else
+      CallbackParam.Name = "arg" + std::to_string(ArgIdx++);
+
+    CallbackParams.push_back(CallbackParam);
+    CallbackParamsAsStrings.push_back(ParamType.getAsString(method.Policy) +
+                                      " " + CallbackParam.Name);
+  }
+  const std::string CallbackReturnTypeName =
+      FuncProtoType->getReturnType().getAsString(method.Policy);
+  const std::string CallbackName = method.MangledName + "_callback";
+
+  // Emit Function Header
+  std::string Header;
+  llvm::raw_string_ostream HeaderStream(Header);
+  HeaderStream << "static " << CallbackReturnTypeName << " " << CallbackName
+               << "(" << llvm::join(CallbackParamsAsStrings, ", ") << ") {";
+  EmitLine(Header);
+  IndentLevel++;
+
+  // Emit Function Body
+  EmitLine("// RPC connection setup and sanity checking");
+  EmitLine("CallbackInfo *callback_info = (CallbackInfo *)baton;");
+  EmitLine("rpc_common::ConnectionSP connection_sp = "
+           "rpc_common::Connection::GetConnectionFromID(callback_info->"
+           "connection_id);");
+  EmitLine("if (!connection_sp)");
+  IndentLevel++;
+  if (FuncProtoType->getReturnType()->isVoidType())
+    EmitLine("return;");
+  else
+    EmitLine("return {};");
+  IndentLevel--;
+
+  EmitLine("// Preparing to make the call");
+  EmitLine("static RPCFunctionInfo g_func(\"" + CallbackName + "\");");
+  EmitLine("RPCStream send;");
+  EmitLine("RPCStream response;");
+  EmitLine("g_func.Encode(send);");
+
+  EmitLine("// The first thing we encode is the callback address so that the "
+           "client-side can know where the callback is");
+  EmitLine("RPCValueEncoder(send, rpc_common::RPCPacket::ValueType::Argument, "
+           "callback_info->callback);");
+  EmitLine("// Encode all the arguments");
+  for (const Param &CallbackParam : CallbackParams) {
+    if (lldb_rpc_gen::TypeIsSBClass(CallbackParam.Type)) {
+
+      // FIXME: SB class server references are stored as non-const references so
+      // that we can actually change them as needed. If a parameter is marked
+      // const, we will fail to compile because we cannot make an
+      // SBFooServerReference from a `const SBFoo &`.
+      // To work around this issue, we'll apply a `const_cast` if needed so we
+      // can continue to generate callbacks for now, but we really should
+      // rethink the way we store object IDs server-side to support
+      // const-qualified parameters.
+      QualType UnderlyingSBClass =
+          lldb_rpc_gen::GetUnderlyingType(CallbackParam.Type);
+      QualType UnqualifiedUnderlyingSBClass =
+          UnderlyingSBClass.getUnqualifiedType();
+
+      std::string SBClassName = GetSBClassNameFromType(UnderlyingSBClass);
+      llvm::StringRef SBClassNameRef(SBClassName);
+      SBClassNameRef.consume_front("lldb::");
+
+      std::string ServerReferenceLine;
+      llvm::raw_string_ostream ServerReferenceLineStream(ServerReferenceLine);
+      ServerReferenceLineStream << "rpc_server::" << SBClassNameRef
+                                << "ServerReference " << CallbackParam.Name
+                                << "_ref(";
+
+      if (UnderlyingSBClass.isConstQualified()) {
+        QualType NonConstSBType =
+            method.Context.getLValueReferenceType(UnqualifiedUnderlyingSBClass);
+        ServerReferenceLineStream << "const_cast<" << NonConstSBType << ">(";
+      }
+      ServerReferenceLineStream << CallbackParam.Name;
+      if (UnderlyingSBClass.isConstQualified())
+        ServerReferenceLineStream << ")";
+
+      ServerReferenceLineStream << ");";
+      EmitLine(ServerReferenceLine);
+      EmitLine(
+          CallbackParam.Name +
+          "_ref.Encode(send, rpc_common::RPCPacket::ValueType::Argument);");
+    } else {
+      std::string ParamName;
+      if (CallbackParam.Type->isVoidPointerType())
+        ParamName = "callback_info->baton";
+      else
+        ParamName = CallbackParam.Name;
+      EmitLine(
+          "RPCValueEncoder(send, rpc_common::RPCPacket::ValueType::Argument, " +
+          ParamName + ");");
+    }
+  }
+
+  if (!FuncProtoType->getReturnType()->isVoidType()) {
+    EmitLine("// Storage for return value");
+    const bool ReturnsSBClass =
+        lldb_rpc_gen::TypeIsSBClass(FuncProtoType->getReturnType());
+    std::string ReturnValueLine = CallbackReturnTypeName;
+    llvm::raw_string_ostream ReturnValueLineStream(ReturnValueLine);
+
+    if (ReturnsSBClass)
+      ReturnValueLineStream << " *";
+    ReturnValueLineStream << " __result = ";
+    if (ReturnsSBClass)
+      ReturnValueLineStream << "nullptr";
+    else
+      ReturnValueLineStream << "{}";
+    ReturnValueLineStream << ";";
+    EmitLine(ReturnValueLine);
+  }
+
+  EmitLine(
+      "if (connection_sp->SendRPCCallAndWaitForResponse(send, response)) {");
+  IndentLevel++;
+  if (!FuncProtoType->getReturnType()->isVoidType()) {
+    if (lldb_rpc_gen::TypeIsSBClass(FuncProtoType->getReturnType())) {
+      EmitLine("__result = rpc_server::RPCServerObjectDecoder<" +
+               CallbackReturnTypeName +
+               ">(response, rpc_common::RPCPacket::ValueType::ReturnValue);");
+    } else
+      EmitLine("RPCValueDecoder(response, "
+               "rpc_common::RPCPacket::ValueType::ReturnValue, __result);");
+  }
+  IndentLevel--;
+  EmitLine("}");
+  if (!FuncProtoType->getReturnType()->isVoidType()) {
+    if (lldb_rpc_gen::TypeIsSBClass(FuncProtoType->getReturnType()))
+      EmitLine("return *__result;");
+    else
+      EmitLine("return __result;");
+  }
+
+  // Emit Function Footer;
+  IndentLevel--;
+  EmitLine("};");
+}
diff --git a/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerSourceEmitter.h b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerSourceEmitter.h
new file mode 100644
index 0000000000000..a371dc100ff7a
--- /dev/null
+++ b/lldb/tools/lldb-rpc/lldb-rpc-gen/server/RPCServerSourceEmitter.h
@@ -0,0 +1,81 @@
+//===-- RPCServerSourceEmitter.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+#ifndef LLDB_RPC_GEN_RPCSERVERMETHODEMITTER_H
+#define LLDB_RPC_GEN_RPCSERVERMETHODEMITTER_H
+
+#include "RPCCommon.h"
+
+#include "clang/AST/AST.h"
+
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+
+namespace lldb_rpc_gen {
+/// Emit the source code for server-side *.cpp files.
+class RPCServerSourceEmitter : public FileEmitter {
+public:
+  RPCServerSourceEmitter(std::unique_ptr<llvm::ToolOutputFile> &&OutputFile)
+      : FileEmitter(std::move(OutputFile)) {
+    Begin();
+  }
+
+  /// Given a Method, emits a server-side implementation of the method
+  /// for lldb-rpc-server
+  void EmitMethod(const Method &method);
+
+private:
+  void EmitCommentHeader(const Method &method);
+
+  void EmitFunctionHeader(const Method &method);
+
+  void EmitFunctionBody(const Method &method);
+
+  void EmitFunctionFooter();
+
+  void EmitStorageForParameters(const Method &method);
+
+  void EmitStorageForOneParameter(QualType ParamType,
+                                  const std::string &ParamName,
+                                  const PrintingPolicy &Policy,
+                                  bool IsFollowedByLen);
+
+  void EmitDecodeForParameters(const Method &method);
+
+  void EmitDecodeForOneParameter(QualType ParamType,
+                                 const std::string &ParamName,
+                                 const PrintingPolicy &Policy);
+
+  std::string CreateMethodCall(const Method &method);
+
+  std::string CreateEncodeLine(const std::string &value,
+                               bool IsEncodingSBClass);
+
+  void EmitEncodesForMutableParameters(const std::vector<Param> &Params);
+
+  void EmitMethodCallAndEncode(const Method &method);
+
+  void EmitCallbackFunction(const Method &method);
+
+  void Begin() {
+    EmitLine("#include \"RPCUserServer.h\"");
+    EmitLine("#include \"SBAPI.h\"");
+    EmitLine("#include <lldb-rpc/common/RPCArgument.h>");
+    EmitLine("#include <lldb-rpc/common/RPCCommon.h>");
+    EmitLine("#include <lldb-rpc/common/RPCFunction.h>");
+    EmitLine("#include <lldb/API/LLDB.h>");
+    EmitLine("");
+    EmitLine("using namespace rpc_common;");
+    EmitLine("using namespace lldb;");
+  }
+};
+} // namespace lldb_rpc_gen
+
+#endif // LLDB_RPC_GEN_RPCSERVERMETHODEMITTER_H
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index 6743ce543dc66..3f198d963a93b 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -1112,13 +1112,13 @@ bool opts::irmemorymap::evalMalloc(StringRef Line,
   // Issue the malloc in the target process with "-rw" permissions.
   const uint32_t Permissions = 0x3;
   const bool ZeroMemory = false;
-  Status ST;
-  addr_t Addr =
-      State.Map.Malloc(Size, Alignment, Permissions, AP, ZeroMemory, ST);
-  if (ST.Fail()) {
-    outs() << formatv("Malloc error: {0}\n", ST);
+  auto AddrOrErr =
+      State.Map.Malloc(Size, Alignment, Permissions, AP, ZeroMemory);
+  if (!AddrOrErr) {
+    outs() << formatv("Malloc error: {0}\n", toString(AddrOrErr.takeError()));
     return true;
   }
+  addr_t Addr = *AddrOrErr;
 
   // Print the result of the allocation before checking its validity.
   outs() << formatv("Malloc: address = {0:x}\n", Addr);
diff --git a/lldb/unittests/DAP/CMakeLists.txt b/lldb/unittests/DAP/CMakeLists.txt
index d5824f4b38a5e..156cd625546bd 100644
--- a/lldb/unittests/DAP/CMakeLists.txt
+++ b/lldb/unittests/DAP/CMakeLists.txt
@@ -7,6 +7,7 @@ add_lldb_unittest(DAPTests
   JSONUtilsTest.cpp
   LLDBUtilsTest.cpp
   ProtocolTypesTest.cpp
+  ProtocolUtilsTest.cpp
   TestBase.cpp
   VariablesTest.cpp
 
diff --git a/lldb/unittests/DAP/JSONUtilsTest.cpp b/lldb/unittests/DAP/JSONUtilsTest.cpp
index ce4be085965d5..876980eb4bf4a 100644
--- a/lldb/unittests/DAP/JSONUtilsTest.cpp
+++ b/lldb/unittests/DAP/JSONUtilsTest.cpp
@@ -139,17 +139,6 @@ TEST(JSONUtilsTest, GetInteger_DifferentTypes) {
   EXPECT_EQ(result.value(), static_cast<int16_t>(789));
 }
 
-TEST(JSONUtilsTest, CreateModule) {
-  SBTarget target;
-  SBModule module;
-
-  json::Value value = CreateModule(target, module);
-  json::Object *object = value.getAsObject();
-
-  ASSERT_NE(object, nullptr);
-  EXPECT_EQ(object->size(), 0UL);
-}
-
 TEST(JSONUtilsTest, GetStrings_EmptyArray) {
   llvm::json::Object obj;
   obj.try_emplace("key", llvm::json::Array());
diff --git a/lldb/unittests/DAP/ProtocolTypesTest.cpp b/lldb/unittests/DAP/ProtocolTypesTest.cpp
index 085348ffc519d..b5cf06bd6f0b6 100644
--- a/lldb/unittests/DAP/ProtocolTypesTest.cpp
+++ b/lldb/unittests/DAP/ProtocolTypesTest.cpp
@@ -792,3 +792,94 @@ TEST(ProtocolTypesTest, ReadMemoryResponseBody) {
   ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
   EXPECT_EQ(pp(*expected), pp(response));
 }
+
+TEST(ProtocolTypesTest, Modules) {
+  Module module;
+  module.id = "AC805E8E-B6A4-CD92-4B05-5CFA7CE24AE8-8926C776";
+  module.name = "libm.so.6";
+  module.path = "/some/path/to/libm.so.6";
+  module.isOptimized = true;
+  module.isUserCode = true;
+  module.version = "0.0.1";
+  module.symbolStatus = "Symbol not found.";
+  module.symbolFilePath = "/some/file/path/to/the/symbol/module";
+  module.dateTimeStamp = "2020-12-09T16:09:53+00:00";
+  module.addressRange = "0xcafeface";
+  module.debugInfoSizeBytes = 1572864;
+
+  Expected<json::Value> expected = json::parse(
+      R"({
+                  "id" : "AC805E8E-B6A4-CD92-4B05-5CFA7CE24AE8-8926C776",
+                  "name": "libm.so.6",
+                  "path": "/some/path/to/libm.so.6",
+                  "isOptimized": true,
+                  "isUserCode": true,
+                  "version": "0.0.1",
+                  "symbolStatus": "Symbol not found.",
+                  "symbolFilePath": "/some/file/path/to/the/symbol/module",
+                  "dateTimeStamp": "2020-12-09T16:09:53+00:00",
+                  "addressRange": "0xcafeface",
+                  "debugInfoSize": "1.5MB" })");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(pp(*expected), pp(module));
+
+  // Test without optional values.
+  module.path.clear();
+  module.isOptimized = false;
+  module.isUserCode = false;
+  module.version.clear();
+  module.symbolStatus.clear();
+  module.symbolFilePath.clear();
+  module.dateTimeStamp.clear();
+  module.addressRange.clear();
+  module.debugInfoSizeBytes = 0;
+  EXPECT_NE(pp(*expected), pp(module));
+
+  Expected<json::Value> expected_no_opt = json::parse(
+      R"({
+                  "id" : "AC805E8E-B6A4-CD92-4B05-5CFA7CE24AE8-8926C776",
+                  "name": "libm.so.6"})");
+  ASSERT_THAT_EXPECTED(expected_no_opt, llvm::Succeeded());
+  EXPECT_EQ(pp(*expected_no_opt), pp(module));
+}
+
+TEST(ProtocolTypesTest, ModulesArguments) {
+  ModulesArguments args;
+
+  llvm::Expected<ModulesArguments> expected = parse<ModulesArguments>(R"({})");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(args.startModule, expected->startModule);
+  EXPECT_EQ(args.moduleCount, expected->moduleCount);
+
+  // Non Default values.
+  args.startModule = 1;
+  args.moduleCount = 2;
+  llvm::Expected<ModulesArguments> expected_no_default =
+      parse<ModulesArguments>(R"({ "startModule": 1, "moduleCount": 2})");
+  ASSERT_THAT_EXPECTED(expected_no_default, llvm::Succeeded());
+  EXPECT_EQ(args.startModule, expected_no_default->startModule);
+  EXPECT_EQ(args.moduleCount, expected_no_default->moduleCount);
+}
+
+TEST(ProtocolTypesTest, ModulesResponseBody) {
+  ModulesResponseBody response;
+  Module module1;
+  module1.id = "first id";
+  module1.name = "first name";
+
+  Module module2;
+  module2.id = "second id";
+  module2.name = "second name";
+  response.modules = {std::move(module1), std::move(module2)};
+  response.totalModules = 2;
+
+  Expected<json::Value> expected = json::parse(
+      R"({
+                  "modules": [
+                    { "id": "first id", "name": "first name"},
+                    { "id": "second id", "name": "second name"}
+                  ],
+                  "totalModules": 2 })");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(pp(*expected), pp(response));
+}
diff --git a/lldb/unittests/DAP/ProtocolUtilsTest.cpp b/lldb/unittests/DAP/ProtocolUtilsTest.cpp
new file mode 100644
index 0000000000000..62bd885af594e
--- /dev/null
+++ b/lldb/unittests/DAP/ProtocolUtilsTest.cpp
@@ -0,0 +1,24 @@
+//===-- ProtocolUtilsTest.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProtocolUtils.h"
+#include "JSONUtils.h"
+#include "lldb/API/LLDB.h"
+#include "gtest/gtest.h"
+#include <optional>
+
+using namespace lldb;
+using namespace lldb_dap;
+
+TEST(ProtocolUtilsTest, CreateModule) {
+  SBTarget target;
+  SBModule module;
+
+  std::optional<protocol::Module> module_opt = CreateModule(target, module);
+  EXPECT_EQ(module_opt, std::nullopt);
+}
diff --git a/lldb/unittests/DAP/TestBase.cpp b/lldb/unittests/DAP/TestBase.cpp
index 27ad42686fbbf..d5d36158d68e0 100644
--- a/lldb/unittests/DAP/TestBase.cpp
+++ b/lldb/unittests/DAP/TestBase.cpp
@@ -29,7 +29,7 @@ using lldb_private::NativeFile;
 using lldb_private::Pipe;
 
 void TransportBase::SetUp() {
-  PipeTest::SetUp();
+  PipePairTest::SetUp();
   to_dap = std::make_unique<Transport>(
       "to_dap", nullptr,
       std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
diff --git a/lldb/unittests/DAP/TestBase.h b/lldb/unittests/DAP/TestBase.h
index 25d37013954d5..50884b1d7feb9 100644
--- a/lldb/unittests/DAP/TestBase.h
+++ b/lldb/unittests/DAP/TestBase.h
@@ -18,7 +18,7 @@ namespace lldb_dap_tests {
 
 /// A base class for tests that need transport configured for communicating DAP
 /// messages.
-class TransportBase : public PipeTest {
+class TransportBase : public PipePairTest {
 protected:
   std::unique_ptr<lldb_dap::Transport> to_dap;
   std::unique_ptr<lldb_dap::Transport> from_dap;
diff --git a/lldb/unittests/Host/FileTest.cpp b/lldb/unittests/Host/FileTest.cpp
index 35c87bb200fad..d973d19430596 100644
--- a/lldb/unittests/Host/FileTest.cpp
+++ b/lldb/unittests/Host/FileTest.cpp
@@ -14,6 +14,10 @@
 #include "llvm/Support/Program.h"
 #include "gtest/gtest.h"
 
+#ifdef _WIN32
+#include "lldb/Host/windows/windows.h"
+#endif
+
 using namespace lldb;
 using namespace lldb_private;
 
@@ -32,7 +36,11 @@ TEST(File, GetWaitableHandleFileno) {
   ASSERT_TRUE(stream);
 
   NativeFile file(stream, true);
-  EXPECT_EQ(file.GetWaitableHandle(), fd);
+#ifdef _WIN32
+  EXPECT_EQ(file.GetWaitableHandle(), (HANDLE)_get_osfhandle(fd));
+#else
+  EXPECT_EQ(file.GetWaitableHandle(), (file_t)fd);
+#endif
 }
 
 TEST(File, GetStreamFromDescriptor) {
@@ -53,5 +61,9 @@ TEST(File, GetStreamFromDescriptor) {
   ASSERT_TRUE(stream != NULL);
 
   EXPECT_EQ(file.GetDescriptor(), fd);
-  EXPECT_EQ(file.GetWaitableHandle(), fd);
+#ifdef _WIN32
+  EXPECT_EQ(file.GetWaitableHandle(), (HANDLE)_get_osfhandle(fd));
+#else
+  EXPECT_EQ(file.GetWaitableHandle(), (file_t)fd);
+#endif
 }
diff --git a/lldb/unittests/Host/JSONTransportTest.cpp b/lldb/unittests/Host/JSONTransportTest.cpp
index 4621869887ac8..2f0846471688c 100644
--- a/lldb/unittests/Host/JSONTransportTest.cpp
+++ b/lldb/unittests/Host/JSONTransportTest.cpp
@@ -14,12 +14,12 @@ using namespace llvm;
 using namespace lldb_private;
 
 namespace {
-template <typename T> class JSONTransportTest : public PipeTest {
+template <typename T> class JSONTransportTest : public PipePairTest {
 protected:
   std::unique_ptr<JSONTransport> transport;
 
   void SetUp() override {
-    PipeTest::SetUp();
+    PipePairTest::SetUp();
     transport = std::make_unique<T>(
         std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
                                      File::eOpenOptionReadOnly,
diff --git a/lldb/unittests/Host/MainLoopTest.cpp b/lldb/unittests/Host/MainLoopTest.cpp
index e18489978e90c..448ad9cfd20f0 100644
--- a/lldb/unittests/Host/MainLoopTest.cpp
+++ b/lldb/unittests/Host/MainLoopTest.cpp
@@ -10,6 +10,7 @@
 #include "TestingSupport/SubsystemRAII.h"
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/FileSystem.h"
+#include "lldb/Host/MainLoopBase.h"
 #include "lldb/Host/PseudoTerminal.h"
 #include "lldb/Host/common/TCPSocket.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
@@ -64,7 +65,7 @@ class MainLoopTest : public testing::Test {
 };
 } // namespace
 
-TEST_F(MainLoopTest, ReadObject) {
+TEST_F(MainLoopTest, ReadSocketObject) {
   char X = 'X';
   size_t len = sizeof(X);
   ASSERT_TRUE(socketpair[0]->Write(&X, len).Success());
@@ -79,7 +80,207 @@ TEST_F(MainLoopTest, ReadObject) {
   ASSERT_EQ(1u, callback_count);
 }
 
-TEST_F(MainLoopTest, NoSpuriousReads) {
+TEST_F(MainLoopTest, ReadPipeObject) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  MainLoop loop;
+
+  char X = 'X';
+  size_t len = sizeof(X);
+  ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+
+  Status error;
+  auto handle = loop.RegisterReadObject(
+      std::make_shared<NativeFile>(pipe.GetReadFileDescriptor(),
+                                   File::eOpenOptionReadOnly, false),
+      make_callback(), error);
+  ASSERT_TRUE(error.Success());
+  ASSERT_TRUE(handle);
+  ASSERT_TRUE(loop.Run().Success());
+  ASSERT_EQ(1u, callback_count);
+}
+
+TEST_F(MainLoopTest, MultipleReadsPipeObject) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  MainLoop loop;
+
+  std::future<void> async_writer = std::async(std::launch::async, [&] {
+    for (int i = 0; i < 5; ++i) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(500));
+      char X = 'X';
+      size_t len = sizeof(X);
+      ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+    }
+  });
+
+  Status error;
+  lldb::FileSP file = std::make_shared<NativeFile>(
+      pipe.GetReadFileDescriptor(), File::eOpenOptionReadOnly, false);
+  auto handle = loop.RegisterReadObject(
+      file,
+      [&](MainLoopBase &loop) {
+        callback_count++;
+        if (callback_count == 5)
+          loop.RequestTermination();
+
+        // Read some data to ensure the handle is not in a readable state.
+        char buf[1024] = {0};
+        size_t len = sizeof(buf);
+        ASSERT_THAT_ERROR(file->Read(buf, len).ToError(), llvm::Succeeded());
+        EXPECT_EQ(len, 1U);
+        EXPECT_EQ(buf[0], 'X');
+      },
+      error);
+  ASSERT_TRUE(error.Success());
+  ASSERT_TRUE(handle);
+  ASSERT_TRUE(loop.Run().Success());
+  ASSERT_EQ(5u, callback_count);
+  async_writer.wait();
+}
+
+TEST_F(MainLoopTest, PipeDelayBetweenRegisterAndRun) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  MainLoop loop;
+
+  Status error;
+  lldb::FileSP file = std::make_shared<NativeFile>(
+      pipe.GetReadFileDescriptor(), File::eOpenOptionReadOnly, false);
+  auto handle = loop.RegisterReadObject(
+      file,
+      [&](MainLoopBase &loop) {
+        callback_count++;
+
+        // Read some data to ensure the handle is not in a readable state.
+        char buf[1024] = {0};
+        size_t len = sizeof(buf);
+        ASSERT_THAT_ERROR(file->Read(buf, len).ToError(), llvm::Succeeded());
+        EXPECT_EQ(len, 2U);
+        EXPECT_EQ(buf[0], 'X');
+        EXPECT_EQ(buf[1], 'X');
+      },
+      error);
+  auto cb = [&](MainLoopBase &) {
+    callback_count++;
+    char X = 'X';
+    size_t len = sizeof(X);
+    // Write twice and ensure we coalesce into a single read.
+    ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+    ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+  };
+  // Add a write that triggers a read events.
+  loop.AddCallback(cb, std::chrono::milliseconds(500));
+  loop.AddCallback([](MainLoopBase &loop) { loop.RequestTermination(); },
+                   std::chrono::milliseconds(1000));
+  ASSERT_TRUE(error.Success());
+  ASSERT_TRUE(handle);
+
+  // Write between RegisterReadObject / Run should NOT invoke the callback.
+  cb(loop);
+  ASSERT_EQ(1u, callback_count);
+
+  ASSERT_TRUE(loop.Run().Success());
+  ASSERT_EQ(4u, callback_count);
+}
+
+TEST_F(MainLoopTest, NoSelfTriggersDuringPipeHandler) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  MainLoop loop;
+
+  Status error;
+  lldb::FileSP file = std::make_shared<NativeFile>(
+      pipe.GetReadFileDescriptor(), File::eOpenOptionReadOnly, false);
+  auto handle = loop.RegisterReadObject(
+      file,
+      [&](MainLoopBase &lop) {
+        callback_count++;
+
+        char X = 'Y';
+        size_t len = sizeof(X);
+        // writes / reads during the handler callback should NOT trigger itself.
+        ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+
+        char buf[1024] = {0};
+        len = sizeof(buf);
+        ASSERT_THAT_ERROR(file->Read(buf, len).ToError(), llvm::Succeeded());
+        EXPECT_EQ(len, 2U);
+        EXPECT_EQ(buf[0], 'X');
+        EXPECT_EQ(buf[1], 'Y');
+
+        if (callback_count == 2)
+          loop.RequestTermination();
+      },
+      error);
+  // Add a write that triggers a read event.
+  loop.AddPendingCallback([&](MainLoopBase &) {
+    char X = 'X';
+    size_t len = sizeof(X);
+    ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+  });
+  loop.AddCallback(
+      [&](MainLoopBase &) {
+        char X = 'X';
+        size_t len = sizeof(X);
+        ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+      },
+      std::chrono::milliseconds(500));
+  ASSERT_TRUE(error.Success());
+  ASSERT_TRUE(handle);
+  ASSERT_TRUE(loop.Run().Success());
+  ASSERT_EQ(2u, callback_count);
+}
+
+TEST_F(MainLoopTest, NoSpuriousPipeReads) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  char X = 'X';
+  size_t len = sizeof(X);
+  ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::Succeeded());
+
+  lldb::IOObjectSP r = std::make_shared<NativeFile>(
+      pipe.GetReadFileDescriptor(), File::eOpenOptionReadOnly, false);
+
+  MainLoop loop;
+
+  Status error;
+  auto handle = loop.RegisterReadObject(
+      r,
+      [&](MainLoopBase &) {
+        if (callback_count == 0) {
+          // Read the byte back the first time we're called. After that, the
+          // pipe is empty, and we should not be called anymore.
+          char X;
+          size_t len = sizeof(X);
+          ASSERT_THAT_ERROR(r->Read(&X, len).ToError(), llvm::Succeeded());
+          EXPECT_EQ(len, sizeof(X));
+          EXPECT_EQ(X, 'X');
+        }
+        ++callback_count;
+      },
+      error);
+  ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
+  // Terminate the loop after one second.
+  loop.AddCallback([](MainLoopBase &loop) { loop.RequestTermination(); },
+                   std::chrono::seconds(1));
+  ASSERT_THAT_ERROR(loop.Run().ToError(), llvm::Succeeded());
+
+  // Make sure the callback was called only once.
+  ASSERT_EQ(1u, callback_count);
+}
+
+TEST_F(MainLoopTest, NoSpuriousSocketReads) {
   // Write one byte into the socket.
   char X = 'X';
   size_t len = sizeof(X);
@@ -99,6 +300,7 @@ TEST_F(MainLoopTest, NoSpuriousReads) {
           EXPECT_THAT_ERROR(socketpair[1]->Read(&X, len).ToError(),
                             llvm::Succeeded());
           EXPECT_EQ(len, sizeof(X));
+          EXPECT_EQ(X, 'X');
         }
         ++callback_count;
       },
@@ -164,9 +366,8 @@ TEST_F(MainLoopTest, PendingCallbackCalledOnlyOnce) {
       [&](MainLoopBase &loop) {
         // Add one pending callback on the first iteration.
         if (callback_count == 0) {
-          loop.AddPendingCallback([&](MainLoopBase &loop) {
-            callback_count++;
-          });
+          loop.AddPendingCallback(
+              [&](MainLoopBase &loop) { callback_count++; });
         }
         // Terminate the loop on second iteration.
         if (callback_count++ >= 1)
@@ -321,7 +522,7 @@ TEST_F(MainLoopTest, UnmonitoredSignal) {
   MainLoop loop;
   Status error;
   struct sigaction sa;
-  sa.sa_sigaction = [](int, siginfo_t *, void *) { };
+  sa.sa_sigaction = [](int, siginfo_t *, void *) {};
   sa.sa_flags = SA_SIGINFO; // important: no SA_RESTART
   sigemptyset(&sa.sa_mask);
   ASSERT_EQ(0, sigaction(SIGUSR2, &sa, nullptr));
diff --git a/lldb/unittests/Host/posix/SupportTest.cpp b/lldb/unittests/Host/posix/SupportTest.cpp
index e4d7ba89fece6..5393ad83ad8ac 100644
--- a/lldb/unittests/Host/posix/SupportTest.cpp
+++ b/lldb/unittests/Host/posix/SupportTest.cpp
@@ -7,10 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/posix/Support.h"
-#include "lldb/Host/aix/Support.h"
 #include "llvm/Support/Threading.h"
 #include "gtest/gtest.h"
 
+#if defined(_AIX)
+#include "lldb/Host/aix/Support.h"
+#elif defined(__linux__)
+#include "lldb/Host/linux/Support.h"
+#endif
+
 using namespace lldb_private;
 
 #ifndef __APPLE__
@@ -21,10 +26,16 @@ TEST(Support, getProcFile_Pid) {
 }
 #endif // #ifndef __APPLE__
 
-#if defined(_AIX) && defined(LLVM_ENABLE_THREADING)
+#if defined(_AIX) || defined(__linux__)
 TEST(Support, getProcFile_Tid) {
-  auto BufferOrError = getProcFile(getpid(), llvm::get_threadid(), "lwpstatus");
+  auto BufferOrError = getProcFile(getpid(), llvm::get_threadid(),
+#ifdef _AIX
+                                   "lwpstatus"
+#else
+                                   "status"
+#endif
+  );
   ASSERT_TRUE(BufferOrError);
   ASSERT_TRUE(*BufferOrError);
 }
-#endif // #ifdef _AIX && LLVM_ENABLE_THREADING
+#endif // #if defined(_AIX) || defined(__linux__)
diff --git a/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp b/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp
index f9372ded0133b..f3b0d65c2cbfe 100644
--- a/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp
+++ b/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp
@@ -166,7 +166,6 @@ using EncoderBZcond = uint32_t (*)(uint32_t rj, int32_t offs21);
 using EncoderBCZcond = uint32_t (*)(uint8_t cj, int32_t offs21);
 
 TEST_F(LoongArch64EmulatorTester, testJIRL) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   WritePC(old_pc);
   // JIRL r1, r12, 0x10
@@ -177,14 +176,13 @@ TEST_F(LoongArch64EmulatorTester, testJIRL) {
   gpr.gpr[12] = 0x12000400;
   ASSERT_TRUE(TestExecute(inst));
   auto r1 = gpr.gpr[1];
-  auto pc = ReadPC(&success);
-  ASSERT_TRUE(success);
+  auto pc = ReadPC();
+  ASSERT_TRUE(pc);
   ASSERT_EQ(r1, old_pc + 4);
-  ASSERT_EQ(pc, gpr.gpr[12] + (offs16 * 4));
+  ASSERT_EQ(*pc, gpr.gpr[12] + (offs16 * 4));
 }
 
 TEST_F(LoongArch64EmulatorTester, testB) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   WritePC(old_pc);
   // B  0x10010
@@ -193,13 +191,12 @@ TEST_F(LoongArch64EmulatorTester, testB) {
   uint32_t inst = 0b01010000000000000100000000000001;
   uint32_t offs26 = 0x10010;
   ASSERT_TRUE(TestExecute(inst));
-  auto pc = ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (offs26 * 4));
+  auto pc = ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (offs26 * 4));
 }
 
 TEST_F(LoongArch64EmulatorTester, testBL) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   WritePC(old_pc);
   // BL  0x10010
@@ -209,16 +206,15 @@ TEST_F(LoongArch64EmulatorTester, testBL) {
   uint32_t offs26 = 0x10010;
   ASSERT_TRUE(TestExecute(inst));
   auto r1 = gpr.gpr[1];
-  auto pc = ReadPC(&success);
-  ASSERT_TRUE(success);
+  auto pc = ReadPC();
+  ASSERT_TRUE(pc);
   ASSERT_EQ(r1, old_pc + 4);
-  ASSERT_EQ(pc, old_pc + (offs26 * 4));
+  ASSERT_EQ(*pc, old_pc + (offs26 * 4));
 }
 
 static void testBcondBranch(LoongArch64EmulatorTester *tester,
                             EncoderBcond encoder, bool branched,
                             uint64_t rj_val, uint64_t rd_val) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   tester->WritePC(old_pc);
   tester->gpr.gpr[12] = rj_val;
@@ -226,39 +222,37 @@ static void testBcondBranch(LoongArch64EmulatorTester *tester,
   // b<cmp> r12, r13, (-256)
   uint32_t inst = encoder(12, 13, -256);
   ASSERT_TRUE(tester->TestExecute(inst));
-  auto pc = tester->ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (branched ? (-256 * 4) : 4));
+  auto pc = tester->ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (branched ? (-256 * 4) : 4));
 }
 
 static void testBZcondBranch(LoongArch64EmulatorTester *tester,
                              EncoderBZcond encoder, bool branched,
                              uint64_t rj_val) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   tester->WritePC(old_pc);
   tester->gpr.gpr[4] = rj_val;
   // b<cmp>z  r4, (-256)
   uint32_t inst = encoder(4, -256);
   ASSERT_TRUE(tester->TestExecute(inst));
-  auto pc = tester->ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (branched ? (-256 * 4) : 4));
+  auto pc = tester->ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (branched ? (-256 * 4) : 4));
 }
 
 static void testBCZcondBranch(LoongArch64EmulatorTester *tester,
                               EncoderBCZcond encoder, bool branched,
                               uint32_t cj_val) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   tester->WritePC(old_pc);
   tester->fpr.fcc = cj_val;
   // bc<cmp>z fcc0, 256
   uint32_t inst = encoder(0, 256);
   ASSERT_TRUE(tester->TestExecute(inst));
-  auto pc = tester->ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (branched ? (256 * 4) : 4));
+  auto pc = tester->ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (branched ? (256 * 4) : 4));
 }
 
 GEN_BCOND_TEST(64, BEQ, 1, 1, 0)
diff --git a/lldb/unittests/Symbol/PostfixExpressionTest.cpp b/lldb/unittests/Symbol/PostfixExpressionTest.cpp
index 1e437da5133d9..f60b5d2c389ed 100644
--- a/lldb/unittests/Symbol/PostfixExpressionTest.cpp
+++ b/lldb/unittests/Symbol/PostfixExpressionTest.cpp
@@ -11,7 +11,8 @@
 #include "lldb/Utility/StreamString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/DebugInfo/DIContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gmock/gmock.h"
@@ -160,7 +161,7 @@ static std::string ParseAndGenerateDWARF(llvm::StringRef expr) {
   std::string result;
   llvm::raw_string_ostream os(result);
   llvm::DWARFExpression E(extractor, addr_size, llvm::dwarf::DWARF32);
-  llvm::DWARFExpressionPrinter::print(&E, os, llvm::DIDumpOptions(), nullptr);
+  llvm::printDwarfExpression(&E, os, llvm::DIDumpOptions(), nullptr);
   return result;
 }
 
diff --git a/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt b/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
index cf8702209a7c3..eb2e00adba64b 100644
--- a/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
+++ b/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
@@ -6,6 +6,7 @@ add_lldb_unittest(SymbolFileDWARFTests
   DWARFUnitTest.cpp
   SymbolFileDWARFTests.cpp
   XcodeSDKModuleTests.cpp
+  DWARF64UnitTest.cpp
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp b/lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp
new file mode 100644
index 0000000000000..2303f68674e4c
--- /dev/null
+++ b/lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp
@@ -0,0 +1,86 @@
+//===-- DWARF64UnitTest.cpp------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/SymbolFile/DWARF/DWARFDebugInfo.h"
+#include "Plugins/SymbolFile/DWARF/DWARFUnit.h"
+#include "TestingSupport/Symbol/YAMLModuleTester.h"
+
+using namespace lldb_private;
+using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
+
+TEST(DWARF64UnitTest, DWARF64DebugInfoAndCU) {
+  const char *yamldata = R"(
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_PPC64
+DWARF:
+  debug_str:
+    - 'clang version 18.1.8 (clang-18.1.8-1)'
+    - 'main'
+  debug_abbrev:
+    - Table:
+        - Code:            0x1
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_producer
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_data2
+            - Attribute:       DW_AT_stmt_list 
+              Form:            DW_FORM_sec_offset 
+        - Code:            0x02
+          Tag:             DW_TAG_subprogram 
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_name 
+              Form:            DW_FORM_strp
+  debug_info:
+    - Format:          DWARF64
+      Version:         4
+      AbbrOffset:      0x0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x1
+          Values:
+            - Value:           0x0
+            - Value:           0x04
+            - Value:           0x0
+        - AbbrCode:        0x2
+          Values:
+            - Value:           0x1
+        - AbbrCode:	   0x0
+)";
+
+  YAMLModuleTester t(yamldata);
+  auto *symbol_file =
+      llvm::cast<SymbolFileDWARF>(t.GetModule()->GetSymbolFile());
+  DWARFUnit *unit = symbol_file->DebugInfo().GetUnitAtIndex(0);
+  ASSERT_TRUE(unit);
+  ASSERT_EQ(unit->GetFormParams().Format, DwarfFormat::DWARF64);
+  ASSERT_EQ(unit->GetVersion(), 4);
+  ASSERT_EQ(unit->GetAddressByteSize(), 8);
+
+  DWARFFormValue form_value;
+  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
+  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
+  ASSERT_EQ(unit->GetProducer(), eProducerClang);
+  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
+  auto attrs = cu_entry->GetAttributes(unit, DWARFDebugInfoEntry::Recurse::yes);
+  attrs.ExtractFormValueAtIndex(2, form_value); // Validate DW_AT_stmt_list
+  ASSERT_EQ(form_value.Unsigned(), 0UL);
+
+  DWARFDIE cu_die(unit, cu_entry);
+  auto declaration = cu_die.GetFirstChild();
+  ASSERT_TRUE(declaration.IsValid());
+  ASSERT_EQ(declaration.Tag(), DW_TAG_subprogram);
+}
diff --git a/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp b/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
index d746e04f8a9fc..c60688ef22939 100644
--- a/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
+++ b/lldb/unittests/SymbolFile/NativePDB/PdbFPOProgramToDWARFExpressionTests.cpp
@@ -16,7 +16,8 @@
 #include "lldb/Utility/StreamBuffer.h"
 #include "lldb/Utility/StreamString.h"
 #include "llvm/DebugInfo/DIContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -40,7 +41,7 @@ CheckValidProgramTranslation(llvm::StringRef fpo_program,
   std::string result;
   llvm::raw_string_ostream os(result);
   llvm::DWARFExpression E(extractor, /*AddressSize=*/4, llvm::dwarf::DWARF32);
-  llvm::DWARFExpressionPrinter::print(&E, os, llvm::DIDumpOptions(), nullptr);
+  llvm::printDwarfExpression(&E, os, llvm::DIDumpOptions(), nullptr);
 
   // actual check
   ASSERT_EQ(expected_dwarf_expression, result);
diff --git a/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h b/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
index 859cd4fd132a2..4d86e7962201a 100644
--- a/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
+++ b/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
@@ -14,7 +14,7 @@
 #include "gtest/gtest.h"
 
 /// A base class for tests that need a pair of pipes for communication.
-class PipeTest : public testing::Test {
+class PipePairTest : public testing::Test {
 protected:
   lldb_private::Pipe input;
   lldb_private::Pipe output;
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 22877bf916abf..3f8201fa426fe 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -581,7 +581,7 @@ set(LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING "DISABLED" CACHE STRING
 set_property(CACHE LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING PROPERTY STRINGS DISABLED COVERAGE COVERAGE_AND_ORIGIN)
 
 option(LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
-  "Add additional fields to DILocations to support Key Instructions" OFF)
+  "Add additional fields to DILocations to support Key Instructions" ON)
 
 set(WINDOWS_PREFER_FORWARD_SLASH_DEFAULT OFF)
 if (MINGW)
@@ -1071,6 +1071,10 @@ if (LLVM_BUILD_LLVM_DYLIB OR LLVM_BUILD_SHARED_LIBS OR LLVM_ENABLE_PLUGINS)
   set(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS ON)
 endif()
 
+# Because LLVM-C is built into the LLVM library, for now export its symbols
+# whenever LLVM symbols are exported.
+set(LLVM_ENABLE_LLVM_C_EXPORT_ANNOTATIONS ${LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS})
+
 set(LLVM_ENABLE_NEW_PASS_MANAGER TRUE CACHE BOOL
   "Enable the new pass manager by default.")
 if(NOT LLVM_ENABLE_NEW_PASS_MANAGER)
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index 7bf722d5f18aa..9a7eb344ad4fb 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -263,8 +263,6 @@ amaclean@nvidia.com (email), [AlexMaclean](https://github.com/AlexMaclean) (GitH
 
 #### PowerPC backend
 
-Zheng Chen \
-czhengsz@cn.ibm.com (email), [chenzheng1030](https://github.com/chenzheng1030) (GitHub)
 
 #### RISCV backend
 
@@ -300,9 +298,6 @@ marukawa@nec.com (email), [kaz7](https://github.com/kaz7) (GitHub)
 
 #### WebAssembly backend
 
-Dan Gohman \
-llvm@sunfishcode.online (email), [sunfishcode](https://github.com/sunfishcode) (GitHub)
-
 #### X86 backend
 
 Simon Pilgrim \
@@ -522,7 +517,9 @@ Hans Wennborg (hans@chromium.org, [zmodem](https://github.com/zmodem)) -- Releas
 Kostya Serebryany ([kcc](https://github.com/kcc)) -- Sanitizers \
 Michael Spencer (bigcheesegs@gmail.com), [Bigcheese](https://github.com/Bigcheese)) -- Windows support in object tools \
 Alexei Starovoitov (alexei.starovoitov@gmail.com, [4ast](https://github.com/4ast)) -- BPF backend \
-Evgeniy Stepanov ([eugenis](https://github.com/eugenis)) -- Sanitizers
+Evgeniy Stepanov ([eugenis](https://github.com/eugenis)) -- Sanitizers \
+Zheng Chen (czhengsz@cn.ibm.com, [chenzheng1030](https://github.com/chenzheng1030)) -- PowerPC backend
+Dan Gohman (llvm@sunfishcode.online, [sunfishcode](https://github.com/sunfishcode)) -- WebAssembly backend
 
 ### Former maintainers of removed components
 
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 8d8a94d1cddc4..83772ed8d2b13 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -137,7 +137,7 @@ function(add_llvm_symbol_exports target_name export_file)
       set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                    LINK_FLAGS "  -Wl,--version-script,\"${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}\"")
     endif()
-  elseif(WIN32)
+  elseif(WIN32 OR CYGWIN)
     set(native_export_file "${target_name}.def")
 
     add_custom_command(OUTPUT ${native_export_file}
@@ -153,7 +153,7 @@ function(add_llvm_symbol_exports target_name export_file)
     elseif(CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
       # clang in msvc mode, calling a link.exe/lld-link style linker
       set(export_file_linker_flag "-Wl,/DEF:${export_file_linker_flag}")
-    elseif(MINGW)
+    elseif(MINGW OR CYGWIN)
       # ${export_file_linker_flag}, which is the plain file name, works as is
       # when passed to the compiler driver, which then passes it on to the
       # linker as an input file.
@@ -666,7 +666,7 @@ function(llvm_add_library name)
     # When building shared objects for each target there are some internal APIs
     # that are used across shared objects which we can't hide.
     if (LLVM_BUILD_LLVM_DYLIB_VIS AND NOT BUILD_SHARED_LIBS AND NOT APPLE AND
-        (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
+        (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
         NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
         NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
 
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 5aa047a33ba6a..c126b0d073322 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -465,7 +465,7 @@ if( LLVM_ENABLE_PIC )
 endif()
 
 if((NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX")) AND
-   (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")))
+   (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")))
   # GCC for MinGW does nothing about -fvisibility-inlines-hidden, but warns
   # about use of the attributes. As long as we don't use the attributes (to
   # override the default) we shouldn't set the command line options either.
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index fc8223560c91c..2f03421c92230 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -710,7 +710,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   For example, ar will be symlinked to llvm-ar.
 
 **LLVM_INSTALL_CCTOOLS_SYMLINKS**:BOOL
-  Install symliks from the cctools tool names to the corresponding LLVM tools.
+  Install symlinks from the cctools tool names to the corresponding LLVM tools.
   For example, lipo will be symlinked to llvm-lipo.
 
 **LLVM_INSTALL_OCAMLDOC_HTML_DIR**:STRING
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 7d275a511de1a..c614a6d7ace9e 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -684,7 +684,7 @@ something notionally equivalent. Examples:
   };
 
   // The Foo constructor call is reading a file, don't use braces to call it.
-  std::fill(foo.begin(), foo.end(), Foo("name"));
+  llvm::fill(foo, Foo("name"));
 
   // The pair is being constructed like an aggregate, use braces.
   bar_map.insert({my_key, my_value});
diff --git a/llvm/docs/DeveloperPolicy.rst b/llvm/docs/DeveloperPolicy.rst
index 6b60812df83bd..eb59c4953dc2d 100644
--- a/llvm/docs/DeveloperPolicy.rst
+++ b/llvm/docs/DeveloperPolicy.rst
@@ -16,90 +16,88 @@ to eliminate miscommunication, rework, and confusion that might arise from the
 distributed nature of LLVM's development.  By stating the policy in clear terms,
 we hope each developer can know ahead of time what to expect when making LLVM
 contributions.  This policy covers all llvm.org subprojects, including Clang,
-LLDB, libc++, etc.
+LLDB, libc++, MLIR, etc.
 
-This policy is also designed to accomplish the following objectives:
+The developer policy supports the following LLVM project objectives:
 
-#. Attract both users and developers to the LLVM project.
+#. Attract both users and new contributors to the LLVM project.
 
-#. Make life as simple and easy for contributors as possible.
+#. Help people contribute to LLVM by documenting our development practices.
 
-#. Keep the top of tree as stable as possible.
+#. Maintain the stability, performance, and quality of the ``main`` branch.
 
-#. Establish awareness of the project's :ref:`copyright, license, and patent
-   policies <copyright-license-patents>` with contributors to the project.
-
-This policy is aimed at frequent contributors to LLVM. People interested in
-contributing one-off patches can do so in an informal way by sending them to the
-`llvm-commits mailing list
-<http://lists.llvm.org/mailman/listinfo/llvm-commits>`_ and engaging another
-developer to see it through the process.
+#. Establish the project's :ref:`copyright, license, and patent
+   policies <copyright-license-patents>` policies.
 
 Developer Policies
 ==================
 
-This section contains policies that pertain to frequent LLVM developers.  We
-always welcome `one-off patches`_ from people who do not routinely contribute to
-LLVM, but we expect more from frequent contributors to keep the system as
-efficient as possible for everyone.  Frequent LLVM contributors are expected to
-meet the following requirements in order for LLVM to maintain a high standard of
-quality.
-
-Stay Informed
--------------
-
-Developers should stay informed by reading the `LLVM Discourse forums`_ and subscribing
-to the categories of interest for notifications.
-
-Paying attention to changes being made by others is a good way to see what other people
-are interested in and watching the flow of the project as a whole.
+Communication Channels
+----------------------
 
-Contributions to the project are made through :ref:`GitHub Pull Requests <github-reviews>`.
-You can subscribe to notification for areas of the codebase by joining
+LLVM is a large project with many subcomponents, and it has a wide array of
+communication channels that you can use to keep track of recent developments,
+upcoming projects, new designs, enhancements, and other community business.
+
+First and foremost is the `LLVM Discourse forums`_, which is the successor
+to our former mailing lists (llvm-dev@, cfe-dev@, lldb-dev@, etc). This is
+probably the most vital and active communication channel to our highly
+distributed open source project. It enables long-form asynchronous text
+communication, and this is where people tend to propose major changes or
+propose new designs in the form of RFCs (Request For Comment), which are
+described later. Please be aware that the Discourse forums are public and
+archived, and that notices of confidentiality or non-disclosure cannot be
+respected.
+
+We accept code contributions as :ref:`GitHub Pull Requests <github-reviews>`.
+Our project is generally too large to subscribe to all github notifications, so
+if you want to be notified of pull requests affecting a specific parts of the
+code, you can join
 one of the `pr-subscribers-* <https://github.com/orgs/llvm/teams?query=pr-subscribers>`_
 GitHub teams. This `mapping <https://github.com/llvm/llvm-project/blob/main/.github/new-prs-labeler.yml>`_
-indicates which team is associated with a particular paths in the repository.
-
-You can also subscribe to the "commits" mailing list for a subproject you're interested in,
-such as `llvm-commits
-<http://lists.llvm.org/mailman/listinfo/llvm-commits>`_, `cfe-commits
-<http://lists.llvm.org/mailman/listinfo/cfe-commits>`_, or `lldb-commits
-<http://lists.llvm.org/mailman/listinfo/lldb-commits>`_.
-
-Missing features and bugs are tracked through our `GitHub issue tracker <https://github.com/llvm/llvm-project/issues>`_
-and assigned labels. We recommend that active developers monitor incoming issues.
-You can subscribe for notification for specific components by joining
-one of the `issue-subscribers-* <https://github.com/orgs/llvm/teams?query=issue-subscribers>`_
-teams.
-You may also subscribe to the `llvm-bugs
-<http://lists.llvm.org/mailman/listinfo/llvm-bugs>`_ email list to keep track
-of bugs and enhancements occurring in the entire project.  We really appreciate people
-who are proactive at catching incoming bugs in their components and dealing with them
-promptly.
-
-Please be aware that all public LLVM mailing lists and discourse forums are public and archived, and
-that notices of confidentiality or non-disclosure cannot be respected.
+documents the paths that trigger notifications for each of the listed teams.
+
+Missing features and bugs are tracked through our `GitHub issue tracker
+<https://github.com/llvm/llvm-project/issues>`_. You can subscribe for
+notification for specific components by joining one of the `issue-subscribers-*
+<https://github.com/orgs/llvm/teams?query=issue-subscribers>`_ teams. You may
+also subscribe to the `llvm-bugs
+<http://lists.llvm.org/mailman/listinfo/llvm-bugs>`_ email list to subscribe to
+the firehose of all issue notifications, which some community members use to
+perform custom filtering.
+
+Beyond the asynchronous written communication channels, LLVM has a Discord
+server for real-time chat communication, as well as a community calendar with
+many regular workgroup video calls and office hours. See :doc:`GettingInvolved`
+for more information on other ways to engage with the community.
 
 .. _patch:
-.. _one-off patches:
 
 Making and Submitting a Patch
 -----------------------------
 
-When making a patch for review, the goal is to make it as easy for the reviewer
-to read it as possible.  As such, we recommend that you:
+Patches are submitted to GitHub and reviewed using Pull Requests. Follow the
+:ref:`Getting Started Guide <sources>` to check out sources, make a patch, and
+then follow the :ref:`GitHub Pull Request <github-reviews>` guide to upload a
+pull request.
+
+Here are some tips to enable a successful code review:
 
-#. Make your patch against git main, not a branch, and not an old version
-   of LLVM.  This makes it easy to apply the patch.  For information on how to
-   clone from git, please see the :ref:`Getting Started Guide <sources>`.
+* :ref:`Include a test <include a testcase>`. This tends to be one of the first
+  things a reviewer will ask for and look at to understand what a new patch
+  does.
 
-#. Similarly, patches should be submitted soon after they are generated.  Old
-   patches may not apply correctly if the underlying code changes between the
-   time the patch was created and the time it is applied.
+* Identify 2-3 individuals to review the patch. Look through the relevant
+  :ref:`Maintainers` file or browse git blame for likely stakeholders for the
+  code you want to modify, and add ``@username`` to a PR comment to notify them
+  of your PR if you are unable to add reviewers yourself due to GitHub permissions.
 
-#. Once you have created your patch, create a
-   :ref:`GitHub Pull Request <github-reviews>` for
-   it (or commit it directly if applicable).
+* To avoid precommit CI failures due to merge conflicts, base your patches on a
+  recent commit from ``main``. If you want to make changes to a release branch,
+  land a change in ``main`` first and then follow the
+  :ref:`backporting instructions <backporting>`.
+
+See :doc:`CodeReview` for more info on what to expect.
 
 When submitting patches, please do not add confidentiality or non-disclosure
 notices to the patches themselves.  These notices conflict with the LLVM
@@ -115,19 +113,20 @@ GitHub platform about their past contributions. Primarily, our buildbot
 infrastructure uses emails to contact contributors about build and test
 failures.
 
-Therefore, the LLVM community requires contributors to have a public
-email address associated with their GitHub commits, so please ensure that "Keep
-my email addresses private" is disabled in your
-`account settings <https://github.com/settings/emails>`_.
+Therefore, the LLVM community requires contributors to have a public email
+address associated with their GitHub commits, so please ensure that "Keep my
+email addresses private" is disabled in your `account settings
+<https://github.com/settings/emails>`_. There are many free email forwarding
+services available if you wish to keep your identity private.
 
 .. _code review:
 
 Code Reviews
 ------------
 
-LLVM has a code-review policy. Code review is one way to increase the quality of
-software. Please see :doc:`CodeReview` for more information on LLVM's code-review
-process.
+LLVM uses code review, which is a generally accepted software engineering best
+practice for maintaining high code quality. Please see :doc:`CodeReview` for
+more information on LLVM's code review process.
 
 .. _maintainers:
 
@@ -147,17 +146,17 @@ Community members can find active and inactive maintainers for a project in the
 Maintainers are volunteering to take on the following shared responsibilities
 within an area of a project:
 
-    * ensure that commits receive high-quality review, either by the maintainer
-      or by someone else,
-    * help to confirm and comment on issues,
-    * mediate code review disagreements through collaboration with other
-      maintainers (and other reviewers) to come to a consensus on how best to
-      proceed with disputed changes,
-    * actively engage with relevant RFCs,
-    * aid release managers with backporting and other release-related
-      activities,
-    * be a point of contact for contributors who need help (answering questions
-      on Discord/Discourse or holding office hours).
+* ensure that commits receive high-quality review, either by the maintainer
+  or by someone else,
+* help to confirm and comment on issues,
+* mediate code review disagreements through collaboration with other
+  maintainers (and other reviewers) to come to a consensus on how best to
+  proceed with disputed changes,
+* actively engage with relevant RFCs,
+* aid release managers with backporting and other release-related
+  activities,
+* be a point of contact for contributors who need help (answering questions
+  on Discord/Discourse or holding office hours).
 
 Each top-level project in the monorepo will specify one or more
 lead maintainers who are responsible for ensuring community needs are
@@ -173,10 +172,10 @@ project should be discontinued.
 All contributors with commit access to the LLVM Project are eligible to be a
 maintainer. However, we are looking for people who can commit to:
 
-    * engaging in their responsibilities the majority of the days in a month,
-    * ensuring that they, and the community members they interact with, abide by
-      the LLVM Community Code of Conduct, and
-    * performing these duties for at least three months.
+* engaging in their responsibilities the majority of the days in a month,
+* ensuring that they, and the community members they interact with, abide by the
+  :ref:`LLVM Community Code of Conduct`, and
+* performing these duties for at least three months.
 
 We recognize that priorities shift, job changes happen, burnout is real,
 extended vacations are a blessing, and people's lives are generally complex.
@@ -220,27 +219,47 @@ Test Cases
 Developers are required to create test cases for any bugs fixed and any new
 features added.  Some tips for getting your testcase approved:
 
-* All feature and regression test cases are added to the ``llvm/test``
-  directory. The appropriate sub-directory should be selected (see the
+* All feature and regression test cases are added to the ``test`` subdirectory
+  of each LLVM subproject, i.e. ``llvm-project/llvm/test`` for LLVM itself. The
+  appropriate sub-directory should be selected (see the
   :doc:`Testing Guide <TestingGuide>` for details).
 
-* Test cases should be written in :doc:`LLVM assembly language <LangRef>`.
-
-* Test cases, especially for regressions, should be reduced as much as possible,
-  by :doc:`bugpoint <Bugpoint>` or manually. It is unacceptable to place an
-  entire failing program into ``llvm/test`` as this creates a *time-to-test*
-  burden on all developers. Please keep them short.
+* We prefer that functional changes are tested using ``FileCheck`` and the tool
+  that fits most closely with the code being modified. For example, ``opt`` is
+  used to test IR transformations, ``llc`` for backend changes, and ``clang``
+  for frontend changes. Some components have scripts for generating and
+  updating golden tests in the ``utils/`` subproject directory, i.e.
+  `mlir/utils/generate-test-checks.py <https://github.com/llvm/llvm-project/blob/main/mlir/utils/generate-test-checks.py>`_
+  and `llvm/utils/update_llc_test_checks.py <https://github.com/llvm/llvm-project/blob/main/llvm/utils/update_llc_test_checks.py>`_
+
+* Some subprojects such as ``clang`` and ``clangd`` have project specific
+  testing tools, like the ``clang -verify`` flag (`docs
+  <https://clang.llvm.org/docs/InternalsManual.html#verifying-diagnostics>`_)
+  and the ``clangd -lit-test``
+  flag, which are preferred over ``FileCheck``.
+
+* Changes to libraries, such as Support, which are not directly observable
+  through tool invocations, are often best tested with unit tests. Unit tests
+  are located under the ``unittests`` subdirectory of each subproject.
+
+* Test cases should be targeted. Large inputs exhibiting bugs should be reduced
+  with tools like ``llvm-reduce`` before committing them to the suite. It is not
+  acceptable to place an entire failing program into ``llvm/test`` as this
+  creates a *time-to-test* burden on all developers. Please keep them short.
 
 * Avoid adding links to resources that are not available to the entire
   community, such as links to private bug trackers, internal corporate
   documentation, etc. Instead, add sufficient comments to the test to provide
   the context behind such links.
 
-Note that llvm/test and clang/test are designed for regression and small feature
-tests only. More extensive test cases (e.g., entire applications, benchmarks,
-etc) should be added to the ``llvm-test`` test suite.  The llvm-test suite is
-for coverage (correctness, performance, etc) testing, not feature or regression
-testing.
+As a project, we prefer to separate tests into small in-tree tests, and large
+out-of-tree integration tests. More extensive integration test cases (e.g.,
+entire applications, benchmarks, etc) should be added to the `llvm-test-suite
+<https://github.com/llvm/llvm-test-suite>`_ repository.  The
+``llvm-test-suite`` repository is for integration and application testing
+(correctness, performance, etc) testing, not feature or regression testing. It
+also serves to separate out third party code that falls under a different
+license.
 
 Release Notes
 -------------
@@ -336,12 +355,12 @@ you follow these guidelines to help review, search in logs, email formatting
 and so on. These guidelines are very similar to rules used by other open source
 projects.
 
-Most importantly, the contents of the message should be carefully written to
-convey the rationale of the change (without delving too much in detail). It
-also should avoid being vague or overly specific. For example, "bits were not
-set right" will leave the reviewer wondering about which bits, and why they
-weren't right, while "Correctly set overflow bits in TargetInfo" conveys almost
-all there is to the change.
+Commit messages should communicate briefly what the change does, but they
+should really emphasize why a change is being made and provide useful context.
+Commit messages should be thoughtfully written and specific, rather than vague.
+For example, "bits were not set right" will leave the reviewer wondering about
+which bits, and why they weren't right, while "Correctly set overflow bits in
+TargetInfo" conveys almost all there is to the change.
 
 Below are some guidelines about the format of the message itself:
 
@@ -369,8 +388,6 @@ Below are some guidelines about the format of the message itself:
   or "[OpenMP] ...". This helps email filters and searches for post-commit
   reviews.
 
-* The body, if it exists, should be separated from the title by an empty line.
-
 * The body should be concise, but explanatory, including a complete
   reasoning.  Unless it is required to understand the change, examples,
   code snippets and gory details should be left to bug comments, web
@@ -382,7 +399,7 @@ Below are some guidelines about the format of the message itself:
 * If the commit is a bug fix on top of another recently committed patch, or a
   revert or reapply of a patch, include the git commit hash of the prior
   related commit. This could be as simple as "Revert commit NNNN because it
-  caused PR#".
+  caused issue #".
 
 * If the patch has been reviewed, add a link to its review page, as shown
   `here <https://www.llvm.org/docs/Phabricator.html#committing-a-change>`__.
@@ -397,9 +414,15 @@ Below are some guidelines about the format of the message itself:
   message self-explanatory. Note that such non-public links should not be
   included in the submitted code.
 
+LLVM uses a squash workflow for pull requests, so as the pull request evolves
+during review, it's important to update the pull request description over the
+course of a review. GitHub uses the initial commit message to create the pull
+request description, but it ignores all subsequent commit messages. Authors and
+reviewers should make a final editing pass over the squashed commit description when
+squashing and merging PRs.
+
 For minor violations of these recommendations, the community normally favors
-reminding the contributor of this policy over reverting. Minor corrections and
-omissions can be handled by sending a reply to the commits mailing list.
+reminding the contributor of this policy over reverting.
 
 .. _revert_policy:
 
@@ -512,11 +535,12 @@ someone with commit access commits on your behalf. When doing so, please
 provide the name and email address you would like to use in the Author
 property of the commit.
 
-For external tracking purposes, committed changes are automatically reflected
-on a commits mailing list soon after the commit lands (e.g. llvm-commits_).
-Note that these mailing lists are moderated, and it is not unusual for a large
-commit to require a moderator to approve the email, so do not be concerned if a
-commit does not immediately appear in the archives.
+For external tracking purposes, committed changes are automatically reflected on
+a commits mailing list soon after the commit lands (e.g.
+llvm-commits@lists.llvm.org). Note that these mailing lists are moderated, and
+it is not unusual for a large commit to require a moderator to approve the
+email, so do not be concerned if a commit does not immediately appear in the
+archives.
 
 If you have recently been granted commit access, these policies apply:
 
@@ -550,28 +574,69 @@ after they are committed, depending on the nature of the change).  You are
 encouraged to review other peoples' patches as well, but you aren't required
 to do so.
 
-.. _discuss the change/gather consensus:
+Obtaining Other Access or Permissions
+-------------------------------------
 
-Making a Major Change
----------------------
+To obtain access other than commit access, you can raise an issue like the one
+for obtaining commit access. However, instead of including PRs you have authored,
+include evidence of your need for the type of access you want.
 
-When a developer begins a major new project with the aim of contributing it back
-to LLVM, they should inform the community with a post to the `LLVM Discourse forums`_, to the extent
-possible. The reason for this is to:
+For example, if you are helping to triage issues and want the ability to add
+labels, include links to issues you have triaged previously and explain how
+having this ability would help that work.
 
-#. keep the community informed about future changes to LLVM,
+.. _discuss the change/gather consensus:
 
-#. avoid duplication of effort by preventing multiple parties working on the
-   same thing and not knowing about it, and
+Proposing Major Changes (RFCs)
+------------------------------
 
-#. ensure that any technical issues around the proposed work are discussed and
-   resolved before any significant work is done.
+LLVM is a large community with many stakeholders, and before landing any major
+change, it is important to discuss the design of a change publicly with the
+community. This is done by posting an Request For Comments (RFC) on the `LLVM
+Discourse forums`_.
 
 The design of LLVM is carefully controlled to ensure that all the pieces fit
 together well and are as consistent as possible. If you plan to make a major
 change to the way LLVM works or want to add a major new extension, it is a good
-idea to get consensus with the development community before you start working on
-it.
+idea to get consensus with the development community before you invest
+significant effort in an implementation. Prototype implementations, however, can
+often be helpful in making design discussions more concrete by demonstrating
+what is possible.
+
+These are some suggestions for how to get a major change accepted:
+
+* Make it targeted, and avoid touching components irrelevant to the task.
+
+* Explain how the change improves LLVM for other stakeholders rather than
+  focusing on your specific use case.
+
+* As discussion evolves, periodically summarize the current state of the
+  discussion and clearly separate points where consensus seems to emerge from
+  those where further discussion is necessary.
+
+* Compilers are foundational infrastructure, so there is a high quality bar,
+  and the burden of proof is on the proposer. If reviewers repeatedly ask for
+  an unreasonable amount of evidence or data, proposal authors can escalate to
+  the area team to resolve disagreements.
+
+After posting a major proposal, it is common to receive lots of conflicting
+feedback from different parties, or no feedback at all, leaving authors without
+clear next steps. As a community, we are aiming for `"rough consensus"
+<https://en.wikipedia.org/wiki/Rough_consensus>`_, similar in spirit to what is
+described in `IETF RFC7282 <https://datatracker.ietf.org/doc/html/rfc7282>`_.
+This requires considering and addressing all of the objections to the RFC, and
+confirming that we can all live with the tradeoffs embodied in the proposal.
+
+The LLVM Area Teams (defined in `LP0004
+<https://github.com/llvm/llvm-www/blob/main/proposals/LP0004-project-governance.md>`_)
+are responsible for facilitating project decision making. In cases were there
+isn't obvious agreement, area teams should step in to restate their perceived
+consensus. In cases of deeper disagreement, area teams should try to identify
+the next steps for the proposal, such as gathering more data, changing the
+proposal, or rejecting it outright. They can also act as moderators by
+scheduling calls for participants to speak directly to resolve disagreements,
+subject to normal :ref:`Code of Conduct <LLVM Community Code of Conduct>`
+guidelines.
 
 Once the design of the new feature is finalized, the work itself should be done
 as a series of `incremental changes`_, not as a long-term development branch.
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index ea267842cdc35..f92eac3209c21 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -430,27 +430,6 @@ Example:
    .uleb128  .LBB_END0_1-.LBB0_1          # BB_1 size
    .byte     y                            # BB_1 metadata
 
-Version 0: basic block address offsets are computed relative to the function
-address. This uses the unversioned ``SHT_LLVM_BB_ADDR_MAP_V0`` section type and
-is semantically equivalent to using ``SHT_LLVM_BB_ADDR_MAP`` with a zero
-version field.
-
-Example:
-
-.. code-block:: gas
-
-  .section  ".llvm_bb_addr_map","",@llvm_bb_addr_map_v0
-  .quad     .Lfunc_begin0                 # address of the function
-  .byte     2                             # number of basic blocks
-  # BB record for BB_0
-   .uleb128  .Lfunc_beign0-.Lfunc_begin0  # BB_0 offset relative to the function entry (always zero)
-   .uleb128  .LBB_END0_0-.Lfunc_begin0    # BB_0 size
-   .byte     x                            # BB_0 metadata
-  # BB record for BB_1
-   .uleb128  .LBB0_1-.Lfunc_begin0        # BB_1 offset relative to the function entry
-   .uleb128  .LBB_END0_1-.LBB0_1          # BB_1 size
-   .byte     y                            # BB_1 metadata
-
 PGO Analysis Map
 """"""""""""""""
 
diff --git a/llvm/docs/GitHub.rst b/llvm/docs/GitHub.rst
index 31dee0bbb17ba..f43792be6f93d 100644
--- a/llvm/docs/GitHub.rst
+++ b/llvm/docs/GitHub.rst
@@ -537,6 +537,8 @@ See more in-depth information about how to contribute in the following documenta
 Releases
 ========
 
+.. _backporting:
+
 Backporting Fixes to the Release Branches
 -----------------------------------------
 You can use special comments on issues or pull requests to make backport
diff --git a/llvm/docs/HowToUpdateDebugInfo.rst b/llvm/docs/HowToUpdateDebugInfo.rst
index a87efe7e6e43f..c3262a96b62e4 100644
--- a/llvm/docs/HowToUpdateDebugInfo.rst
+++ b/llvm/docs/HowToUpdateDebugInfo.rst
@@ -420,7 +420,41 @@ tests. Changes to this pass are not allowed to break existing tests.
    check lines. In cases where this can't be avoided (say, if a test wouldn't
    be precise enough), moving the test to its own file is preferred.
 
-.. _MIRDebugify:
+Using Coverage Tracking to remove false positives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As described :ref:`above<WhenToDropLocation>`, there are valid reasons for
+instructions to not have source locations. Therefore, when detecting dropped or
+not-generated source locations, it may be preferable to avoid detecting cases
+where the missing source location is intentional. For this, you can use the
+"coverage tracking" feature in LLVM to prevent these from appearing in the
+``debugify`` output. This is enabled in a build of LLVM by setting the CMake
+flag ``-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=COVERAGE``. When this has been
+set, LLVM will enable runtime tracking of
+:ref:`DebugLoc annotations<NewInstLocations>`, allowing ``debugify`` to ignore
+instructions that have an explicitly recorded reason given for not having a
+source location.
+
+For triaging source location bugs detected with ``debugify``, you may find it
+helpful to instead set the CMake flag to enable "origin tracking",
+``-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=COVERAGE_AND_ORIGIN``. This flag adds
+more detail to ``debugify``'s output, by including one or more stacktraces with
+every missing source location, capturing the point at which the empty source
+location was created, and every point at which it was copied to an instruction,
+making it trivial in most cases to find the origin of the underlying bug. If
+using origin tracking, it is recommended to also build LLVM with debug info
+enabled, so that the stacktrace can be accurately symbolized.
+
+.. note::
+
+   The coverage tracking feature has been designed primarily for use with the
+   :ref:`original debug info preservation<OriginalDI>` mode of ``debugify``, and
+   so may not be reliable in other settings. When using this mode, the
+   stacktraces produced by the ``COVERAGE_AND_ORIGIN`` setting will be printed
+   in an easy-to-read format as part of the reports generated by the
+   ``llvm-original-di-preservation.py`` script.
+
+.. _OriginalDI:
 
 Test original debug info preservation in optimizations
 ------------------------------------------------------
@@ -478,6 +512,8 @@ as follows:
 Please do note that there are some known false positives, for source locations
 and debug record checking, so that will be addressed as a future work.
 
+.. _MIRDebugify:
+
 Mutation testing for MIR-level transformations
 ----------------------------------------------
 
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index d77c659055236..d2a1821efd698 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1234,8 +1234,7 @@ Currently, only the following parameter attributes are defined:
     ``byval`` parameters). This is not a valid attribute for return
     values.
 
-    The byval type argument indicates the in-memory value type, and
-    must be the same as the pointee type of the argument.
+    The byval type argument indicates the in-memory value type.
 
     The byval attribute also supports specifying an alignment with the
     align attribute. It indicates the alignment of the stack slot to
@@ -1283,8 +1282,7 @@ Currently, only the following parameter attributes are defined:
     any parameter must have a ``"preallocated"`` operand bundle. A ``musttail``
     function call cannot have a ``"preallocated"`` operand bundle.
 
-    The preallocated attribute requires a type argument, which must be
-    the same as the pointee type of the argument.
+    The preallocated attribute requires a type argument.
 
     The preallocated attribute also supports specifying an alignment with the
     align attribute. It indicates the alignment of the stack slot to
@@ -1318,8 +1316,7 @@ Currently, only the following parameter attributes are defined:
     must be cleared off with :ref:`llvm.stackrestore
     <int_stackrestore>`.
 
-    The inalloca attribute requires a type argument, which must be the
-    same as the pointee type of the argument.
+    The inalloca attribute requires a type argument.
 
     See :doc:`InAlloca` for more information on how to use this
     attribute.
@@ -1331,8 +1328,7 @@ Currently, only the following parameter attributes are defined:
     loads and stores to the structure may be assumed by the callee not
     to trap and to be properly aligned.
 
-    The sret type argument specifies the in memory type, which must be
-    the same as the pointee type of the argument.
+    The sret type argument specifies the in memory type.
 
     A function that accepts an ``sret`` argument must return ``void``.
     A return value may not be ``sret``.
@@ -1741,6 +1737,23 @@ Currently, only the following parameter attributes are defined:
 
     This attribute cannot be applied to return values.
 
+``dead_on_return``
+    This attribute indicates that the memory pointed to by the argument is dead
+    upon function return, both upon normal return and if the calls unwinds, meaning
+    that the caller will not depend on its contents. Stores that would be observable
+    either on the return path or on the unwind path may be elided.
+
+    Specifically, the behavior is as-if any memory written through the pointer
+    during the execution of the function is overwritten with a poison value
+    upon function return. The caller may access the memory, but any load
+    not preceded by a store will return poison.
+
+    This attribute does not imply aliasing properties. For pointer arguments that
+    do not alias other memory locations, ``noalias`` attribute may be used in
+    conjunction. Conversely, this attribute always implies ``dead_on_unwind``.
+
+    This attribute cannot be applied to return values.
+
 ``range(<ty> <a>, <b>)``
     This attribute expresses the possible range of the parameter or return value.
     If the value is not in the specified range, it is converted to poison.
@@ -6975,16 +6988,25 @@ appear in the included source file.
 DILabel
 """""""
 
-``DILabel`` nodes represent labels within a :ref:`DISubprogram`. All fields of
-a ``DILabel`` are mandatory. The ``scope:`` field must be one of either a
-:ref:`DILexicalBlockFile`, a :ref:`DILexicalBlock`, or a :ref:`DISubprogram`.
-The ``name:`` field is the label identifier. The ``file:`` field is the
-:ref:`DIFile` the label is present in. The ``line:`` field is the source line
+``DILabel`` nodes represent labels within a :ref:`DISubprogram`. The ``scope:``
+field must be one of either a :ref:`DILexicalBlockFile`, a
+:ref:`DILexicalBlock`, or a :ref:`DISubprogram`. The ``name:`` field is the
+label identifier. The ``file:`` field is the :ref:`DIFile` the label is
+present in. The ``line:`` and ``column:`` field are the source line and column
 within the file where the label is declared.
 
+Furthermore, a label can be marked as artificial, i.e. compiler-generated,
+using ``isArtificial:``. Such artificial labels are generated, e.g., by
+the ``CoroSplit`` pass. In addition, the ``CoroSplit`` pass also uses the
+``coroSuspendIdx:`` field to identify the coroutine suspend points.
+
+``scope:``, ``name:``, ``file:`` and ``line:`` are mandatory. The remaining
+fields are optional.
+
 .. code-block:: text
 
-  !2 = !DILabel(scope: !0, name: "foo", file: !1, line: 7)
+  !2 = !DILabel(scope: !0, name: "foo", file: !1, line: 7, column: 4)
+  !3 = !DILabel(scope: !0, name: "__coro_resume_3", file: !1, line: 9, column: 3, isArtificial: true, coroSuspendIdx: 3)
 
 DICommonBlock
 """""""""""""
@@ -13161,7 +13183,7 @@ This instruction requires several arguments:
    -  All ABI-impacting function attributes, such as sret, byval, inreg,
       returned, and inalloca, must match.
    -  The caller and callee prototypes must match. Pointer types of parameters
-      or return types may differ in pointee type, but not in address space.
+      or return types do not differ in address space.
 
    On the other hand, if the calling convention is `swifttailcc` or `tailcc`:
 
diff --git a/llvm/docs/MLGO.rst b/llvm/docs/MLGO.rst
index 4f8fb3f59ca19..ed0769bebeac3 100644
--- a/llvm/docs/MLGO.rst
+++ b/llvm/docs/MLGO.rst
@@ -15,11 +15,13 @@ Currently the following heuristics feature such integration:
 
 This document is an outline of the tooling and APIs facilitating MLGO.
 
-Note that tools for orchestrating ML training are not part of LLVM, as they are
-dependency-heavy - both on the ML infrastructure choice, as well as choices of
-distributed computing. For the training scenario, LLVM only contains facilities
-enabling it, such as corpus extraction, training data extraction, and evaluation
-of models during training.
+.. note::
+    
+  The tools for orchestrating ML training are not part of LLVM, as they are
+  dependency-heavy - both on the ML infrastructure choice, as well as choices of
+  distributed computing. For the training scenario, LLVM only contains facilities
+  enabling it, such as corpus extraction, training data extraction, and evaluation
+  of models during training.
 
 
 .. contents::
@@ -329,8 +331,10 @@ We currently feature 4 implementations:
   the neural network, together with its weights (essentially, loops performing
   matrix multiplications)
 
-NOTE: we are actively working on replacing this with an EmitC implementation
-requiring no out of tree build-time dependencies.
+.. note::
+    
+  we are actively working on replacing this with an EmitC implementation
+  requiring no out of tree build-time dependencies.
 
 - ``InteractiveModelRunner``. This is intended for training scenarios where the
   training algorithm drives compilation. This model runner has no special
@@ -448,7 +452,16 @@ downstream tasks, including ML-guided compiler optimizations.
 
 The core components are:
   - **Vocabulary**: A mapping from IR entities (opcodes, types, etc.) to their
-    vector representations. This is managed by ``IR2VecVocabAnalysis``.
+    vector representations. This is managed by ``IR2VecVocabAnalysis``. The 
+    vocabulary (.json file) contains three sections -- Opcodes, Types, and 
+    Arguments, each containing the representations of the corresponding 
+    entities.
+
+    .. note::
+      
+      It is mandatory to have these three sections present in the vocabulary file 
+      for it to be valid; order in which they appear does not matter.
+
   - **Embedder**: A class (``ir2vec::Embedder``) that uses the vocabulary to
     compute embeddings for instructions, basic blocks, and functions.
 
@@ -479,14 +492,9 @@ embeddings can be computed and accessed via an ``ir2vec::Embedder`` instance.
 
       // Assuming F is an llvm::Function&
       // For example, using IR2VecKind::Symbolic:
-      Expected<std::unique_ptr<ir2vec::Embedder>> EmbOrErr =
+      std::unique_ptr<ir2vec::Embedder> Emb =
           ir2vec::Embedder::create(IR2VecKind::Symbolic, F, Vocabulary);
 
-      if (auto Err = EmbOrErr.takeError()) {
-        // Handle error in embedder creation
-        return;
-      }
-      std::unique_ptr<ir2vec::Embedder> Emb = std::move(*EmbOrErr);
 
 3. **Compute and Access Embeddings**:
    Call ``getFunctionVector()`` to get the embedding for the function. 
@@ -522,9 +530,11 @@ implementation details.
 Building with ML support
 ========================
 
-**NOTE** For up to date information on custom builds, see the ``ml-*``
-`build bots <http://lab.llvm.org>`_. They are set up using 
-`like this <https://github.com/google/ml-compiler-opt/blob/main/buildbot/buildbot_init.sh>`_.
+.. note::
+  
+  For up to date information on custom builds, see the ``ml-*``
+  `build bots <http://lab.llvm.org>`_. They are set up using 
+  `like this <https://github.com/google/ml-compiler-opt/blob/main/buildbot/buildbot_init.sh>`_.
 
 Embed pre-trained models (aka "release" mode)
 ---------------------------------------------
@@ -558,9 +568,11 @@ You can also specify a URL for the path, and it is also possible to pre-compile
 the header and object and then just point to the precompiled artifacts. See for
 example ``LLVM_OVERRIDE_MODEL_HEADER_INLINERSIZEMODEL``.
 
-**Note** that we are transitioning away from the AOT compiler shipping with the
-tensorflow package, and to a EmitC, in-tree solution, so these details will
-change soon.
+.. note::
+
+  We are transitioning away from the AOT compiler shipping with the
+  tensorflow package, and to a EmitC, in-tree solution, so these details will
+  change soon.
 
 Using TFLite (aka "development" mode)
 -------------------------------------
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 81684ba30f12c..158a20cce7f85 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -498,6 +498,9 @@ The current vendor extensions supported are:
 ``experimental-Xqcisync``
   LLVM implements `version 0.3 of the Qualcomm uC Sync Delay extension specification <https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.13.0>`__ by Qualcomm. These instructions are only available for riscv32.
 
+``Xmipscbop``
+  LLVM implements MIPS prefetch extension `p8700 processor <https://mips.com/products/hardware/p8700/>`__ by MIPS.
+
 ``Xmipscmov``
   LLVM implements conditional move for the `p8700 processor <https://mips.com/products/hardware/p8700/>`__ by MIPS.
 
@@ -516,6 +519,9 @@ The current vendor extensions supported are:
 ``XAndesVBFHCvt``
   LLVM implements `version 5.0.0 of the Andes Vector BFLOAT16 Conversion Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
 
+``XAndesVSINTLoad``
+  LLVM implements `version 5.0.0 of the Andes Vector INT4 Load Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
+
 ``XAndesVPackFPH``
   LLVM implements `version 5.0.0 of the Andes Vector Packed FP16 Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
 
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 73ae2ee599640..daf822388a2ff 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -213,6 +213,7 @@ Changes to the RISC-V Backend
 * Adds assembler support for the Andes `XAndesvbfhcvt` (Andes Vector BFLOAT16 Conversion extension).
 * `-mcpu=andes-ax45mpv` was added.
 * Removed -mattr=+no-rvc-hints that could be used to disable parsing and generation of RVC hints.
+* Adds assembler support for the Andes `XAndesvsintload` (Andes Vector INT4 Load extension).
 
 Changes to the WebAssembly Backend
 ----------------------------------
@@ -273,6 +274,7 @@ Changes to the LLVM tools
 * In llvm-objcopy/llvm-strip's ELF port, `--discard-locals` and `--discard-all` now allow and preserve symbols referenced by relocations.
   ([#47468](https://github.com/llvm/llvm-project/issues/47468))
 * llvm-addr2line now supports a `+` prefix when specifying an address.
+* Support for `SHT_LLVM_BB_ADDR_MAP` versions 0 and 1 has been dropped.
 
 Changes to LLDB
 ---------------------------------
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 1858bda6160d4..1f563fbfb725a 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -217,6 +217,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
      - Adds an instruction to compute the matrix product of an M x K matrix with a K x N matrix and then add an M x N matrix. 
    * - ``SPV_INTEL_int4``
      - Adds support for 4-bit integer type, and allow this type to be used in cooperative matrices.
+   * - ``SPV_KHR_float_controls2``
+     - Adds ability to specify the floating-point environment in shaders. It can be used on whole modules and individual instructions.
 
 To enable multiple extensions, list them separated by comma. For example, to enable support for atomic operations on floating-point numbers and arbitrary precision integers, use:
 
@@ -255,6 +257,7 @@ using target extension types and are represented as follows:
      SPIR-V Type        LLVM type name          LLVM type arguments
      ================== ======================= ===========================================================================================
      OpTypeImage        ``spirv.Image``         sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
+     OpTypeImage        ``spirv.SignedImage``   sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
      OpTypeSampler      ``spirv.Sampler``       (none)
      OpTypeSampledImage ``spirv.SampledImage``  sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
      OpTypeEvent        ``spirv.Event``         (none)
@@ -275,6 +278,12 @@ parameters of its underlying image type, so that a sampled image for the
 previous type has the representation
 ``target("spirv.SampledImage, void, 1, 1, 0, 0, 0, 0, 0)``.
 
+The differences between ``spirv.Image`` and ``spirv.SignedImage`` is that the
+backend will generate code assuming that the format of the image is a signed
+integer instead of unsigned. This is required because llvm-ir will create the
+same sampled type for signed and unsigned integers. If the image format is
+unknown, the backend cannot distinguish the two case.
+
 See `wg-hlsl proposal 0018 <https://github.com/llvm/wg-hlsl/blob/main/proposals/0018-spirv-resource-representation.md>`_
 for details on ``spirv.VulkanBuffer``.
 
diff --git a/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp b/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp
index c55aa73d50277..84f17871c03e4 100644
--- a/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp
+++ b/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp
@@ -169,7 +169,9 @@ Expected<ThreadSafeModule> loadModule(StringRef Path,
 
   MemoryBufferRef BitcodeBufferRef = (**BitcodeBuffer).getMemBufferRef();
   Expected<std::unique_ptr<Module>> M =
-      parseBitcodeFile(BitcodeBufferRef, *TSCtx.getContext());
+      TSCtx.withContextDo([&](LLVMContext *Ctx) {
+        return parseBitcodeFile(BitcodeBufferRef, *Ctx);
+      });
   if (!M)
     return M.takeError();
 
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c
index 286fa8baac4f8..b95462f340f2f 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c
@@ -22,11 +22,8 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Create a new LLVM module.
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
@@ -57,6 +54,9 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   //  - Free the builder.
   LLVMDisposeBuilder(Builder);
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
+
   // Our demo module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c
index 1b4102625fa1b..42a27c7054d47 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c
@@ -31,8 +31,7 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  LLVMContextRef Ctx = LLVMContextCreate();
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
   LLVMTypeRef ParamTypes[] = {LLVMInt32Type(), LLVMInt32Type()};
   LLVMTypeRef SumFunctionType =
@@ -45,6 +44,8 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   LLVMValueRef SumArg1 = LLVMGetParam(SumFunction, 1);
   LLVMValueRef Result = LLVMBuildAdd(Builder, SumArg0, SumArg1, "result");
   LLVMBuildRet(Builder, Result);
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
   LLVMOrcDisposeThreadSafeContext(TSCtx);
   return TSM;
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c
index 41ae6e53db1d6..62904d006da61 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c
@@ -32,8 +32,7 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  LLVMContextRef Ctx = LLVMContextCreate();
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
   LLVMTypeRef ParamTypes[] = {LLVMInt32Type(), LLVMInt32Type()};
   LLVMTypeRef SumFunctionType =
@@ -47,6 +46,7 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   LLVMValueRef Result = LLVMBuildAdd(Builder, SumArg0, SumArg1, "result");
   LLVMBuildRet(Builder, Result);
   LLVMDisposeBuilder(Builder);
+  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
   LLVMOrcDisposeThreadSafeContext(TSCtx);
   return TSM;
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c
index 33398c8cb9816..9c31f93899201 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c
@@ -67,11 +67,9 @@ const char MainMod[] =
 LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
                                 const char *Name,
                                 LLVMOrcThreadSafeModuleRef *TSM) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
 
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext for the Module.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Wrap Source in a MemoryBuffer
   LLVMMemoryBufferRef MB =
@@ -85,6 +83,10 @@ LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
     // TODO: LLVMDisposeMessage(ErrMsg);
   }
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
+
   // Our module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   *TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c
index f85430bcfda4a..6962c6980e787 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c
@@ -150,11 +150,8 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Create a new LLVM module.
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
@@ -182,6 +179,10 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   //  - Build the return instruction.
   LLVMBuildRet(Builder, Result);
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
+
   // Our demo module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c
index 7f84a3d413435..7f8a9cd334c6b 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c
@@ -22,11 +22,8 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Create a new LLVM module.
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
@@ -57,6 +54,10 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   //  - Free the builder.
   LLVMDisposeBuilder(Builder);
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
+
   // Our demo module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c
index 85651f728399e..3c1ff8392eff4 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c
@@ -74,11 +74,8 @@ LLVMErrorRef applyDataLayout(void *Ctx, LLVMModuleRef M) {
 LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
                                 const char *Name,
                                 LLVMOrcThreadSafeModuleRef *TSM) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Wrap Source in a MemoryBuffer
   LLVMMemoryBufferRef MB =
@@ -93,6 +90,9 @@ LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
     return Err;
   }
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
+
   // Our module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   *TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/include/llvm-c/Analysis.h b/llvm/include/llvm-c/Analysis.h
index 270b145a4d27e..01e1667b5232b 100644
--- a/llvm/include/llvm-c/Analysis.h
+++ b/llvm/include/llvm-c/Analysis.h
@@ -21,6 +21,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -41,17 +42,19 @@ typedef enum {
 /* Verifies that a module is valid, taking the specified action if not.
    Optionally returns a human-readable description of any invalid constructs.
    OutMessage must be disposed with LLVMDisposeMessage. */
-LLVMBool LLVMVerifyModule(LLVMModuleRef M, LLVMVerifierFailureAction Action,
-                          char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMVerifyModule(LLVMModuleRef M,
+                                     LLVMVerifierFailureAction Action,
+                                     char **OutMessage);
 
 /* Verifies that a single function is valid, taking the specified action. Useful
    for debugging. */
-LLVMBool LLVMVerifyFunction(LLVMValueRef Fn, LLVMVerifierFailureAction Action);
+LLVM_C_ABI LLVMBool LLVMVerifyFunction(LLVMValueRef Fn,
+                                       LLVMVerifierFailureAction Action);
 
 /* Open up a ghostview window that displays the CFG of the current function.
    Useful for debugging. */
-void LLVMViewFunctionCFG(LLVMValueRef Fn);
-void LLVMViewFunctionCFGOnly(LLVMValueRef Fn);
+LLVM_C_ABI void LLVMViewFunctionCFG(LLVMValueRef Fn);
+LLVM_C_ABI void LLVMViewFunctionCFGOnly(LLVMValueRef Fn);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/BitReader.h b/llvm/include/llvm-c/BitReader.h
index 088107468d4fb..9dcdbf436454f 100644
--- a/llvm/include/llvm-c/BitReader.h
+++ b/llvm/include/llvm-c/BitReader.h
@@ -21,6 +21,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -36,30 +37,33 @@ LLVM_C_EXTERN_C_BEGIN
    Optionally returns a human-readable error message via OutMessage.
 
    This is deprecated. Use LLVMParseBitcode2. */
-LLVMBool LLVMParseBitcode(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutModule,
-                          char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMParseBitcode(LLVMMemoryBufferRef MemBuf,
+                                     LLVMModuleRef *OutModule,
+                                     char **OutMessage);
 
 /* Builds a module from the bitcode in the specified memory buffer, returning a
    reference to the module via the OutModule parameter. Returns 0 on success. */
-LLVMBool LLVMParseBitcode2(LLVMMemoryBufferRef MemBuf,
-                           LLVMModuleRef *OutModule);
+LLVM_C_ABI LLVMBool LLVMParseBitcode2(LLVMMemoryBufferRef MemBuf,
+                                      LLVMModuleRef *OutModule);
 
 /* This is deprecated. Use LLVMParseBitcodeInContext2. */
-LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef,
-                                   LLVMMemoryBufferRef MemBuf,
-                                   LLVMModuleRef *OutModule, char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMParseBitcodeInContext(LLVMContextRef ContextRef,
+                                              LLVMMemoryBufferRef MemBuf,
+                                              LLVMModuleRef *OutModule,
+                                              char **OutMessage);
 
-LLVMBool LLVMParseBitcodeInContext2(LLVMContextRef ContextRef,
-                                    LLVMMemoryBufferRef MemBuf,
-                                    LLVMModuleRef *OutModule);
+LLVM_C_ABI LLVMBool LLVMParseBitcodeInContext2(LLVMContextRef ContextRef,
+                                               LLVMMemoryBufferRef MemBuf,
+                                               LLVMModuleRef *OutModule);
 
 /** Reads a module from the specified path, returning via the OutMP parameter
     a module provider which performs lazy deserialization. Returns 0 on success.
     Optionally returns a human-readable error message via OutMessage.
     This is deprecated. Use LLVMGetBitcodeModuleInContext2. */
-LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef,
-                                       LLVMMemoryBufferRef MemBuf,
-                                       LLVMModuleRef *OutM, char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef,
+                                                  LLVMMemoryBufferRef MemBuf,
+                                                  LLVMModuleRef *OutM,
+                                                  char **OutMessage);
 
 /** Reads a module from the given memory buffer, returning via the OutMP
  * parameter a module provider which performs lazy deserialization.
@@ -68,15 +72,17 @@ LLVMBool LLVMGetBitcodeModuleInContext(LLVMContextRef ContextRef,
  *
  * Takes ownership of \p MemBuf if (and only if) the module was read
  * successfully. */
-LLVMBool LLVMGetBitcodeModuleInContext2(LLVMContextRef ContextRef,
-                                        LLVMMemoryBufferRef MemBuf,
-                                        LLVMModuleRef *OutM);
+LLVM_C_ABI LLVMBool LLVMGetBitcodeModuleInContext2(LLVMContextRef ContextRef,
+                                                   LLVMMemoryBufferRef MemBuf,
+                                                   LLVMModuleRef *OutM);
 
 /* This is deprecated. Use LLVMGetBitcodeModule2. */
-LLVMBool LLVMGetBitcodeModule(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutM,
-                              char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMGetBitcodeModule(LLVMMemoryBufferRef MemBuf,
+                                         LLVMModuleRef *OutM,
+                                         char **OutMessage);
 
-LLVMBool LLVMGetBitcodeModule2(LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutM);
+LLVM_C_ABI LLVMBool LLVMGetBitcodeModule2(LLVMMemoryBufferRef MemBuf,
+                                          LLVMModuleRef *OutM);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/BitWriter.h b/llvm/include/llvm-c/BitWriter.h
index ea84b6593d126..71d482b6be070 100644
--- a/llvm/include/llvm-c/BitWriter.h
+++ b/llvm/include/llvm-c/BitWriter.h
@@ -21,6 +21,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -34,18 +35,18 @@ LLVM_C_EXTERN_C_BEGIN
 /*===-- Operations on modules ---------------------------------------------===*/
 
 /** Writes a module to the specified path. Returns 0 on success. */
-int LLVMWriteBitcodeToFile(LLVMModuleRef M, const char *Path);
+LLVM_C_ABI int LLVMWriteBitcodeToFile(LLVMModuleRef M, const char *Path);
 
 /** Writes a module to an open file descriptor. Returns 0 on success. */
-int LLVMWriteBitcodeToFD(LLVMModuleRef M, int FD, int ShouldClose,
-                         int Unbuffered);
+LLVM_C_ABI int LLVMWriteBitcodeToFD(LLVMModuleRef M, int FD, int ShouldClose,
+                                    int Unbuffered);
 
 /** Deprecated for LLVMWriteBitcodeToFD. Writes a module to an open file
     descriptor. Returns 0 on success. Closes the Handle. */
-int LLVMWriteBitcodeToFileHandle(LLVMModuleRef M, int Handle);
+LLVM_C_ABI int LLVMWriteBitcodeToFileHandle(LLVMModuleRef M, int Handle);
 
 /** Writes a module to a new memory buffer and returns it. */
-LLVMMemoryBufferRef LLVMWriteBitcodeToMemoryBuffer(LLVMModuleRef M);
+LLVM_C_ABI LLVMMemoryBufferRef LLVMWriteBitcodeToMemoryBuffer(LLVMModuleRef M);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Comdat.h b/llvm/include/llvm-c/Comdat.h
index 8002bc0581af7..cf13f8507440e 100644
--- a/llvm/include/llvm-c/Comdat.h
+++ b/llvm/include/llvm-c/Comdat.h
@@ -16,6 +16,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -43,35 +44,37 @@ typedef enum {
  *
  * @see llvm::Module::getOrInsertComdat()
  */
-LLVMComdatRef LLVMGetOrInsertComdat(LLVMModuleRef M, const char *Name);
+LLVM_C_ABI LLVMComdatRef LLVMGetOrInsertComdat(LLVMModuleRef M,
+                                               const char *Name);
 
 /**
  * Get the Comdat assigned to the given global object.
  *
  * @see llvm::GlobalObject::getComdat()
  */
-LLVMComdatRef LLVMGetComdat(LLVMValueRef V);
+LLVM_C_ABI LLVMComdatRef LLVMGetComdat(LLVMValueRef V);
 
 /**
  * Assign the Comdat to the given global object.
  *
  * @see llvm::GlobalObject::setComdat()
  */
-void LLVMSetComdat(LLVMValueRef V, LLVMComdatRef C);
+LLVM_C_ABI void LLVMSetComdat(LLVMValueRef V, LLVMComdatRef C);
 
 /*
  * Get the conflict resolution selection kind for the Comdat.
  *
  * @see llvm::Comdat::getSelectionKind()
  */
-LLVMComdatSelectionKind LLVMGetComdatSelectionKind(LLVMComdatRef C);
+LLVM_C_ABI LLVMComdatSelectionKind LLVMGetComdatSelectionKind(LLVMComdatRef C);
 
 /*
  * Set the conflict resolution selection kind for the Comdat.
  *
  * @see llvm::Comdat::setSelectionKind()
  */
-void LLVMSetComdatSelectionKind(LLVMComdatRef C, LLVMComdatSelectionKind Kind);
+LLVM_C_ABI void LLVMSetComdatSelectionKind(LLVMComdatRef C,
+                                           LLVMComdatSelectionKind Kind);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 3f30ed92997b4..d645646289025 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -18,6 +18,7 @@
 #include "llvm-c/Deprecated.h"
 #include "llvm-c/ErrorHandling.h"
 #include "llvm-c/ExternC.h"
+#include "llvm-c/Visibility.h"
 
 #include "llvm-c/Types.h"
 
@@ -536,7 +537,7 @@ typedef unsigned LLVMGEPNoWrapFlags;
 /** Deallocate and destroy all ManagedStatic variables.
     @see llvm::llvm_shutdown
     @see ManagedStatic */
-void LLVMShutdown(void);
+LLVM_C_ABI void LLVMShutdown(void);
 
 /*===-- Version query -----------------------------------------------------===*/
 
@@ -546,12 +547,13 @@ void LLVMShutdown(void);
  * The version components are returned via the function's three output
  * parameters or skipped if a NULL pointer was supplied.
  */
-void LLVMGetVersion(unsigned *Major, unsigned *Minor, unsigned *Patch);
+LLVM_C_ABI void LLVMGetVersion(unsigned *Major, unsigned *Minor,
+                               unsigned *Patch);
 
 /*===-- Error handling ----------------------------------------------------===*/
 
-char *LLVMCreateMessage(const char *Message);
-void LLVMDisposeMessage(char *Message);
+LLVM_C_ABI char *LLVMCreateMessage(const char *Message);
+LLVM_C_ABI void LLVMDisposeMessage(char *Message);
 
 /**
  * @defgroup LLVMCCoreContext Contexts
@@ -574,44 +576,46 @@ typedef void (*LLVMYieldCallback)(LLVMContextRef, void *);
  * Every call to this function should be paired with a call to
  * LLVMContextDispose() or the context will leak memory.
  */
-LLVMContextRef LLVMContextCreate(void);
+LLVM_C_ABI LLVMContextRef LLVMContextCreate(void);
 
 /**
  * Obtain the global context instance.
  */
-LLVMContextRef LLVMGetGlobalContext(void);
+LLVM_C_ABI LLVMContextRef LLVMGetGlobalContext(void);
 
 /**
  * Set the diagnostic handler for this context.
  */
-void LLVMContextSetDiagnosticHandler(LLVMContextRef C,
-                                     LLVMDiagnosticHandler Handler,
-                                     void *DiagnosticContext);
+LLVM_C_ABI void LLVMContextSetDiagnosticHandler(LLVMContextRef C,
+                                                LLVMDiagnosticHandler Handler,
+                                                void *DiagnosticContext);
 
 /**
  * Get the diagnostic handler of this context.
  */
-LLVMDiagnosticHandler LLVMContextGetDiagnosticHandler(LLVMContextRef C);
+LLVM_C_ABI LLVMDiagnosticHandler
+LLVMContextGetDiagnosticHandler(LLVMContextRef C);
 
 /**
  * Get the diagnostic context of this context.
  */
-void *LLVMContextGetDiagnosticContext(LLVMContextRef C);
+LLVM_C_ABI void *LLVMContextGetDiagnosticContext(LLVMContextRef C);
 
 /**
  * Set the yield callback function for this context.
  *
  * @see LLVMContext::setYieldCallback()
  */
-void LLVMContextSetYieldCallback(LLVMContextRef C, LLVMYieldCallback Callback,
-                                 void *OpaqueHandle);
+LLVM_C_ABI void LLVMContextSetYieldCallback(LLVMContextRef C,
+                                            LLVMYieldCallback Callback,
+                                            void *OpaqueHandle);
 
 /**
  * Retrieve whether the given context is set to discard all value names.
  *
  * @see LLVMContext::shouldDiscardValueNames()
  */
-LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C);
+LLVM_C_ABI LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C);
 
 /**
  * Set whether the given context discards all value names.
@@ -621,7 +625,8 @@ LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C);
  *
  * @see LLVMContext::setDiscardValueNames()
  */
-void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard);
+LLVM_C_ABI void LLVMContextSetDiscardValueNames(LLVMContextRef C,
+                                                LLVMBool Discard);
 
 /**
  * Destroy a context instance.
@@ -629,7 +634,7 @@ void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard);
  * This should be called for every call to LLVMContextCreate() or memory
  * will be leaked.
  */
-void LLVMContextDispose(LLVMContextRef C);
+LLVM_C_ABI void LLVMContextDispose(LLVMContextRef C);
 
 /**
  * Return a string representation of the DiagnosticInfo. Use
@@ -637,23 +642,25 @@ void LLVMContextDispose(LLVMContextRef C);
  *
  * @see DiagnosticInfo::print()
  */
-char *LLVMGetDiagInfoDescription(LLVMDiagnosticInfoRef DI);
+LLVM_C_ABI char *LLVMGetDiagInfoDescription(LLVMDiagnosticInfoRef DI);
 
 /**
  * Return an enum LLVMDiagnosticSeverity.
  *
  * @see DiagnosticInfo::getSeverity()
  */
-LLVMDiagnosticSeverity LLVMGetDiagInfoSeverity(LLVMDiagnosticInfoRef DI);
+LLVM_C_ABI LLVMDiagnosticSeverity
+LLVMGetDiagInfoSeverity(LLVMDiagnosticInfoRef DI);
 
-unsigned LLVMGetMDKindIDInContext(LLVMContextRef C, const char *Name,
-                                  unsigned SLen);
-unsigned LLVMGetMDKindID(const char *Name, unsigned SLen);
+LLVM_C_ABI unsigned LLVMGetMDKindIDInContext(LLVMContextRef C, const char *Name,
+                                             unsigned SLen);
+LLVM_C_ABI unsigned LLVMGetMDKindID(const char *Name, unsigned SLen);
 
 /**
  * Maps a synchronization scope name to a ID unique within this context.
  */
-unsigned LLVMGetSyncScopeID(LLVMContextRef C, const char *Name, size_t SLen);
+LLVM_C_ABI unsigned LLVMGetSyncScopeID(LLVMContextRef C, const char *Name,
+                                       size_t SLen);
 
 /**
  * Return an unique id given the name of a enum attribute,
@@ -666,36 +673,39 @@ unsigned LLVMGetSyncScopeID(LLVMContextRef C, const char *Name, size_t SLen);
  * NB: Attribute names and/or id are subject to change without
  * going through the C API deprecation cycle.
  */
-unsigned LLVMGetEnumAttributeKindForName(const char *Name, size_t SLen);
-unsigned LLVMGetLastEnumAttributeKind(void);
+LLVM_C_ABI unsigned LLVMGetEnumAttributeKindForName(const char *Name,
+                                                    size_t SLen);
+LLVM_C_ABI unsigned LLVMGetLastEnumAttributeKind(void);
 
 /**
  * Create an enum attribute.
  */
-LLVMAttributeRef LLVMCreateEnumAttribute(LLVMContextRef C, unsigned KindID,
-                                         uint64_t Val);
+LLVM_C_ABI LLVMAttributeRef LLVMCreateEnumAttribute(LLVMContextRef C,
+                                                    unsigned KindID,
+                                                    uint64_t Val);
 
 /**
  * Get the unique id corresponding to the enum attribute
  * passed as argument.
  */
-unsigned LLVMGetEnumAttributeKind(LLVMAttributeRef A);
+LLVM_C_ABI unsigned LLVMGetEnumAttributeKind(LLVMAttributeRef A);
 
 /**
  * Get the enum attribute's value. 0 is returned if none exists.
  */
-uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A);
+LLVM_C_ABI uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A);
 
 /**
  * Create a type attribute
  */
-LLVMAttributeRef LLVMCreateTypeAttribute(LLVMContextRef C, unsigned KindID,
-                                         LLVMTypeRef type_ref);
+LLVM_C_ABI LLVMAttributeRef LLVMCreateTypeAttribute(LLVMContextRef C,
+                                                    unsigned KindID,
+                                                    LLVMTypeRef type_ref);
 
 /**
  * Get the type attribute's value.
  */
-LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A);
+LLVM_C_ABI LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A);
 
 /**
  * Create a ConstantRange attribute.
@@ -703,40 +713,42 @@ LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A);
  * LowerWords and UpperWords need to be NumBits divided by 64 rounded up
  * elements long.
  */
-LLVMAttributeRef LLVMCreateConstantRangeAttribute(LLVMContextRef C,
-                                                  unsigned KindID,
-                                                  unsigned NumBits,
-                                                  const uint64_t LowerWords[],
-                                                  const uint64_t UpperWords[]);
+LLVM_C_ABI LLVMAttributeRef LLVMCreateConstantRangeAttribute(
+    LLVMContextRef C, unsigned KindID, unsigned NumBits,
+    const uint64_t LowerWords[], const uint64_t UpperWords[]);
 
 /**
  * Create a string attribute.
  */
-LLVMAttributeRef LLVMCreateStringAttribute(LLVMContextRef C,
-                                           const char *K, unsigned KLength,
-                                           const char *V, unsigned VLength);
+LLVM_C_ABI LLVMAttributeRef LLVMCreateStringAttribute(LLVMContextRef C,
+                                                      const char *K,
+                                                      unsigned KLength,
+                                                      const char *V,
+                                                      unsigned VLength);
 
 /**
  * Get the string attribute's kind.
  */
-const char *LLVMGetStringAttributeKind(LLVMAttributeRef A, unsigned *Length);
+LLVM_C_ABI const char *LLVMGetStringAttributeKind(LLVMAttributeRef A,
+                                                  unsigned *Length);
 
 /**
  * Get the string attribute's value.
  */
-const char *LLVMGetStringAttributeValue(LLVMAttributeRef A, unsigned *Length);
+LLVM_C_ABI const char *LLVMGetStringAttributeValue(LLVMAttributeRef A,
+                                                   unsigned *Length);
 
 /**
  * Check for the different types of attributes.
  */
-LLVMBool LLVMIsEnumAttribute(LLVMAttributeRef A);
-LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A);
-LLVMBool LLVMIsTypeAttribute(LLVMAttributeRef A);
+LLVM_C_ABI LLVMBool LLVMIsEnumAttribute(LLVMAttributeRef A);
+LLVM_C_ABI LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A);
+LLVM_C_ABI LLVMBool LLVMIsTypeAttribute(LLVMAttributeRef A);
 
 /**
  * Obtain a Type from a context by its registered name.
  */
-LLVMTypeRef LLVMGetTypeByName2(LLVMContextRef C, const char *Name);
+LLVM_C_ABI LLVMTypeRef LLVMGetTypeByName2(LLVMContextRef C, const char *Name);
 
 /**
  * @}
@@ -761,7 +773,7 @@ LLVMTypeRef LLVMGetTypeByName2(LLVMContextRef C, const char *Name);
  * Every invocation should be paired with LLVMDisposeModule() or memory
  * will be leaked.
  */
-LLVMModuleRef LLVMModuleCreateWithName(const char *ModuleID);
+LLVM_C_ABI LLVMModuleRef LLVMModuleCreateWithName(const char *ModuleID);
 
 /**
  * Create a new, empty module in a specific context.
@@ -769,12 +781,12 @@ LLVMModuleRef LLVMModuleCreateWithName(const char *ModuleID);
  * Every invocation should be paired with LLVMDisposeModule() or memory
  * will be leaked.
  */
-LLVMModuleRef LLVMModuleCreateWithNameInContext(const char *ModuleID,
-                                                LLVMContextRef C);
+LLVM_C_ABI LLVMModuleRef LLVMModuleCreateWithNameInContext(const char *ModuleID,
+                                                           LLVMContextRef C);
 /**
  * Return an exact copy of the specified module.
  */
-LLVMModuleRef LLVMCloneModule(LLVMModuleRef M);
+LLVM_C_ABI LLVMModuleRef LLVMCloneModule(LLVMModuleRef M);
 
 /**
  * Destroy a module instance.
@@ -782,7 +794,7 @@ LLVMModuleRef LLVMCloneModule(LLVMModuleRef M);
  * This must be called for every created module or memory will be
  * leaked.
  */
-void LLVMDisposeModule(LLVMModuleRef M);
+LLVM_C_ABI void LLVMDisposeModule(LLVMModuleRef M);
 
 /**
  * Soon to be deprecated.
@@ -792,7 +804,7 @@ void LLVMDisposeModule(LLVMModuleRef M);
  * non-instruction debug records instead of debug intrinsics for variable
  * location tracking.
  */
-LLVMBool LLVMIsNewDbgInfoFormat(LLVMModuleRef M);
+LLVM_C_ABI LLVMBool LLVMIsNewDbgInfoFormat(LLVMModuleRef M);
 
 /**
  * Soon to be deprecated.
@@ -800,7 +812,8 @@ LLVMBool LLVMIsNewDbgInfoFormat(LLVMModuleRef M);
  *
  * Convert module into desired debug info format.
  */
-void LLVMSetIsNewDbgInfoFormat(LLVMModuleRef M, LLVMBool UseNewFormat);
+LLVM_C_ABI void LLVMSetIsNewDbgInfoFormat(LLVMModuleRef M,
+                                          LLVMBool UseNewFormat);
 
 /**
  * Obtain the identifier of a module.
@@ -810,7 +823,7 @@ void LLVMSetIsNewDbgInfoFormat(LLVMModuleRef M, LLVMBool UseNewFormat);
  * @return The identifier of M.
  * @see Module::getModuleIdentifier()
  */
-const char *LLVMGetModuleIdentifier(LLVMModuleRef M, size_t *Len);
+LLVM_C_ABI const char *LLVMGetModuleIdentifier(LLVMModuleRef M, size_t *Len);
 
 /**
  * Set the identifier of a module to a string Ident with length Len.
@@ -820,7 +833,8 @@ const char *LLVMGetModuleIdentifier(LLVMModuleRef M, size_t *Len);
  * @param Len Length of Ident
  * @see Module::setModuleIdentifier()
  */
-void LLVMSetModuleIdentifier(LLVMModuleRef M, const char *Ident, size_t Len);
+LLVM_C_ABI void LLVMSetModuleIdentifier(LLVMModuleRef M, const char *Ident,
+                                        size_t Len);
 
 /**
  * Obtain the module's original source file name.
@@ -830,7 +844,7 @@ void LLVMSetModuleIdentifier(LLVMModuleRef M, const char *Ident, size_t Len);
  * @return The original source file name of M
  * @see Module::getSourceFileName()
  */
-const char *LLVMGetSourceFileName(LLVMModuleRef M, size_t *Len);
+LLVM_C_ABI const char *LLVMGetSourceFileName(LLVMModuleRef M, size_t *Len);
 
 /**
  * Set the original source file name of a module to a string Name with length
@@ -841,7 +855,8 @@ const char *LLVMGetSourceFileName(LLVMModuleRef M, size_t *Len);
  * @param Len Length of Name
  * @see Module::setSourceFileName()
  */
-void LLVMSetSourceFileName(LLVMModuleRef M, const char *Name, size_t Len);
+LLVM_C_ABI void LLVMSetSourceFileName(LLVMModuleRef M, const char *Name,
+                                      size_t Len);
 
 /**
  * Obtain the data layout for a module.
@@ -852,29 +867,29 @@ void LLVMSetSourceFileName(LLVMModuleRef M, const char *Name, size_t Len);
  * but match the name of another method on the module. Prefer the use
  * of LLVMGetDataLayoutStr, which is not ambiguous.
  */
-const char *LLVMGetDataLayoutStr(LLVMModuleRef M);
-const char *LLVMGetDataLayout(LLVMModuleRef M);
+LLVM_C_ABI const char *LLVMGetDataLayoutStr(LLVMModuleRef M);
+LLVM_C_ABI const char *LLVMGetDataLayout(LLVMModuleRef M);
 
 /**
  * Set the data layout for a module.
  *
  * @see Module::setDataLayout()
  */
-void LLVMSetDataLayout(LLVMModuleRef M, const char *DataLayoutStr);
+LLVM_C_ABI void LLVMSetDataLayout(LLVMModuleRef M, const char *DataLayoutStr);
 
 /**
  * Obtain the target triple for a module.
  *
  * @see Module::getTargetTriple()
  */
-const char *LLVMGetTarget(LLVMModuleRef M);
+LLVM_C_ABI const char *LLVMGetTarget(LLVMModuleRef M);
 
 /**
  * Set the target triple for a module.
  *
  * @see Module::setTargetTriple()
  */
-void LLVMSetTarget(LLVMModuleRef M, const char *Triple);
+LLVM_C_ABI void LLVMSetTarget(LLVMModuleRef M, const char *Triple);
 
 /**
  * Returns the module flags as an array of flag-key-value triples.  The caller
@@ -883,37 +898,37 @@ void LLVMSetTarget(LLVMModuleRef M, const char *Triple);
  *
  * @see Module::getModuleFlagsMetadata()
  */
-LLVMModuleFlagEntry *LLVMCopyModuleFlagsMetadata(LLVMModuleRef M, size_t *Len);
+LLVM_C_ABI LLVMModuleFlagEntry *LLVMCopyModuleFlagsMetadata(LLVMModuleRef M,
+                                                            size_t *Len);
 
 /**
  * Destroys module flags metadata entries.
  */
-void LLVMDisposeModuleFlagsMetadata(LLVMModuleFlagEntry *Entries);
+LLVM_C_ABI void LLVMDisposeModuleFlagsMetadata(LLVMModuleFlagEntry *Entries);
 
 /**
  * Returns the flag behavior for a module flag entry at a specific index.
  *
  * @see Module::ModuleFlagEntry::Behavior
  */
-LLVMModuleFlagBehavior
-LLVMModuleFlagEntriesGetFlagBehavior(LLVMModuleFlagEntry *Entries,
-                                     unsigned Index);
+LLVM_C_ABI LLVMModuleFlagBehavior LLVMModuleFlagEntriesGetFlagBehavior(
+    LLVMModuleFlagEntry *Entries, unsigned Index);
 
 /**
  * Returns the key for a module flag entry at a specific index.
  *
  * @see Module::ModuleFlagEntry::Key
  */
-const char *LLVMModuleFlagEntriesGetKey(LLVMModuleFlagEntry *Entries,
-                                        unsigned Index, size_t *Len);
+LLVM_C_ABI const char *LLVMModuleFlagEntriesGetKey(LLVMModuleFlagEntry *Entries,
+                                                   unsigned Index, size_t *Len);
 
 /**
  * Returns the metadata for a module flag entry at a specific index.
  *
  * @see Module::ModuleFlagEntry::Val
  */
-LLVMMetadataRef LLVMModuleFlagEntriesGetMetadata(LLVMModuleFlagEntry *Entries,
-                                                 unsigned Index);
+LLVM_C_ABI LLVMMetadataRef
+LLVMModuleFlagEntriesGetMetadata(LLVMModuleFlagEntry *Entries, unsigned Index);
 
 /**
  * Add a module-level flag to the module-level flags metadata if it doesn't
@@ -921,8 +936,8 @@ LLVMMetadataRef LLVMModuleFlagEntriesGetMetadata(LLVMModuleFlagEntry *Entries,
  *
  * @see Module::getModuleFlag()
  */
-LLVMMetadataRef LLVMGetModuleFlag(LLVMModuleRef M,
-                                  const char *Key, size_t KeyLen);
+LLVM_C_ABI LLVMMetadataRef LLVMGetModuleFlag(LLVMModuleRef M, const char *Key,
+                                             size_t KeyLen);
 
 /**
  * Add a module-level flag to the module-level flags metadata if it doesn't
@@ -930,16 +945,17 @@ LLVMMetadataRef LLVMGetModuleFlag(LLVMModuleRef M,
  *
  * @see Module::addModuleFlag()
  */
-void LLVMAddModuleFlag(LLVMModuleRef M, LLVMModuleFlagBehavior Behavior,
-                       const char *Key, size_t KeyLen,
-                       LLVMMetadataRef Val);
+LLVM_C_ABI void LLVMAddModuleFlag(LLVMModuleRef M,
+                                  LLVMModuleFlagBehavior Behavior,
+                                  const char *Key, size_t KeyLen,
+                                  LLVMMetadataRef Val);
 
 /**
  * Dump a representation of a module to stderr.
  *
  * @see Module::dump()
  */
-void LLVMDumpModule(LLVMModuleRef M);
+LLVM_C_ABI void LLVMDumpModule(LLVMModuleRef M);
 
 /**
  * Print a representation of a module to a file. The ErrorMessage needs to be
@@ -947,8 +963,8 @@ void LLVMDumpModule(LLVMModuleRef M);
  *
  * @see Module::print()
  */
-LLVMBool LLVMPrintModuleToFile(LLVMModuleRef M, const char *Filename,
-                               char **ErrorMessage);
+LLVM_C_ABI LLVMBool LLVMPrintModuleToFile(LLVMModuleRef M, const char *Filename,
+                                          char **ErrorMessage);
 
 /**
  * Return a string representation of the module. Use
@@ -956,58 +972,61 @@ LLVMBool LLVMPrintModuleToFile(LLVMModuleRef M, const char *Filename,
  *
  * @see Module::print()
  */
-char *LLVMPrintModuleToString(LLVMModuleRef M);
+LLVM_C_ABI char *LLVMPrintModuleToString(LLVMModuleRef M);
 
 /**
  * Get inline assembly for a module.
  *
  * @see Module::getModuleInlineAsm()
  */
-const char *LLVMGetModuleInlineAsm(LLVMModuleRef M, size_t *Len);
+LLVM_C_ABI const char *LLVMGetModuleInlineAsm(LLVMModuleRef M, size_t *Len);
 
 /**
  * Set inline assembly for a module.
  *
  * @see Module::setModuleInlineAsm()
  */
-void LLVMSetModuleInlineAsm2(LLVMModuleRef M, const char *Asm, size_t Len);
+LLVM_C_ABI void LLVMSetModuleInlineAsm2(LLVMModuleRef M, const char *Asm,
+                                        size_t Len);
 
 /**
  * Append inline assembly to a module.
  *
  * @see Module::appendModuleInlineAsm()
  */
-void LLVMAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, size_t Len);
+LLVM_C_ABI void LLVMAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm,
+                                          size_t Len);
 
 /**
  * Create the specified uniqued inline asm string.
  *
  * @see InlineAsm::get()
  */
-LLVMValueRef LLVMGetInlineAsm(LLVMTypeRef Ty, const char *AsmString,
-                              size_t AsmStringSize, const char *Constraints,
-                              size_t ConstraintsSize, LLVMBool HasSideEffects,
-                              LLVMBool IsAlignStack,
-                              LLVMInlineAsmDialect Dialect, LLVMBool CanThrow);
+LLVM_C_ABI LLVMValueRef LLVMGetInlineAsm(
+    LLVMTypeRef Ty, const char *AsmString, size_t AsmStringSize,
+    const char *Constraints, size_t ConstraintsSize, LLVMBool HasSideEffects,
+    LLVMBool IsAlignStack, LLVMInlineAsmDialect Dialect, LLVMBool CanThrow);
 
 /**
  * Get the template string used for an inline assembly snippet
  *
  */
-const char *LLVMGetInlineAsmAsmString(LLVMValueRef InlineAsmVal, size_t *Len);
+LLVM_C_ABI const char *LLVMGetInlineAsmAsmString(LLVMValueRef InlineAsmVal,
+                                                 size_t *Len);
 
 /**
  * Get the raw constraint string for an inline assembly snippet
  *
  */
-const char *LLVMGetInlineAsmConstraintString(LLVMValueRef InlineAsmVal,
-                                             size_t *Len);
+LLVM_C_ABI const char *
+LLVMGetInlineAsmConstraintString(LLVMValueRef InlineAsmVal, size_t *Len);
 
 /**
  * Get the dialect used by the inline asm snippet
  *
  */
-LLVMInlineAsmDialect LLVMGetInlineAsmDialect(LLVMValueRef InlineAsmVal);
+LLVM_C_ABI LLVMInlineAsmDialect
+LLVMGetInlineAsmDialect(LLVMValueRef InlineAsmVal);
 
 /**
  * Get the function type of the inline assembly snippet. The same type that
@@ -1016,49 +1035,50 @@ LLVMInlineAsmDialect LLVMGetInlineAsmDialect(LLVMValueRef InlineAsmVal);
  * @see LLVMGetInlineAsm
  *
  */
-LLVMTypeRef LLVMGetInlineAsmFunctionType(LLVMValueRef InlineAsmVal);
+LLVM_C_ABI LLVMTypeRef LLVMGetInlineAsmFunctionType(LLVMValueRef InlineAsmVal);
 
 /**
  * Get if the inline asm snippet has side effects
  *
  */
-LLVMBool LLVMGetInlineAsmHasSideEffects(LLVMValueRef InlineAsmVal);
+LLVM_C_ABI LLVMBool LLVMGetInlineAsmHasSideEffects(LLVMValueRef InlineAsmVal);
 
 /**
  * Get if the inline asm snippet needs an aligned stack
  *
  */
-LLVMBool LLVMGetInlineAsmNeedsAlignedStack(LLVMValueRef InlineAsmVal);
+LLVM_C_ABI LLVMBool
+LLVMGetInlineAsmNeedsAlignedStack(LLVMValueRef InlineAsmVal);
 
 /**
  * Get if the inline asm snippet may unwind the stack
  *
  */
-LLVMBool LLVMGetInlineAsmCanUnwind(LLVMValueRef InlineAsmVal);
+LLVM_C_ABI LLVMBool LLVMGetInlineAsmCanUnwind(LLVMValueRef InlineAsmVal);
 
 /**
  * Obtain the context to which this module is associated.
  *
  * @see Module::getContext()
  */
-LLVMContextRef LLVMGetModuleContext(LLVMModuleRef M);
+LLVM_C_ABI LLVMContextRef LLVMGetModuleContext(LLVMModuleRef M);
 
 /** Deprecated: Use LLVMGetTypeByName2 instead. */
-LLVMTypeRef LLVMGetTypeByName(LLVMModuleRef M, const char *Name);
+LLVM_C_ABI LLVMTypeRef LLVMGetTypeByName(LLVMModuleRef M, const char *Name);
 
 /**
  * Obtain an iterator to the first NamedMDNode in a Module.
  *
  * @see llvm::Module::named_metadata_begin()
  */
-LLVMNamedMDNodeRef LLVMGetFirstNamedMetadata(LLVMModuleRef M);
+LLVM_C_ABI LLVMNamedMDNodeRef LLVMGetFirstNamedMetadata(LLVMModuleRef M);
 
 /**
  * Obtain an iterator to the last NamedMDNode in a Module.
  *
  * @see llvm::Module::named_metadata_end()
  */
-LLVMNamedMDNodeRef LLVMGetLastNamedMetadata(LLVMModuleRef M);
+LLVM_C_ABI LLVMNamedMDNodeRef LLVMGetLastNamedMetadata(LLVMModuleRef M);
 
 /**
  * Advance a NamedMDNode iterator to the next NamedMDNode.
@@ -1066,7 +1086,8 @@ LLVMNamedMDNodeRef LLVMGetLastNamedMetadata(LLVMModuleRef M);
  * Returns NULL if the iterator was already at the end and there are no more
  * named metadata nodes.
  */
-LLVMNamedMDNodeRef LLVMGetNextNamedMetadata(LLVMNamedMDNodeRef NamedMDNode);
+LLVM_C_ABI LLVMNamedMDNodeRef
+LLVMGetNextNamedMetadata(LLVMNamedMDNodeRef NamedMDNode);
 
 /**
  * Decrement a NamedMDNode iterator to the previous NamedMDNode.
@@ -1074,7 +1095,8 @@ LLVMNamedMDNodeRef LLVMGetNextNamedMetadata(LLVMNamedMDNodeRef NamedMDNode);
  * Returns NULL if the iterator was already at the beginning and there are
  * no previous named metadata nodes.
  */
-LLVMNamedMDNodeRef LLVMGetPreviousNamedMetadata(LLVMNamedMDNodeRef NamedMDNode);
+LLVM_C_ABI LLVMNamedMDNodeRef
+LLVMGetPreviousNamedMetadata(LLVMNamedMDNodeRef NamedMDNode);
 
 /**
  * Retrieve a NamedMDNode with the given name, returning NULL if no such
@@ -1082,8 +1104,9 @@ LLVMNamedMDNodeRef LLVMGetPreviousNamedMetadata(LLVMNamedMDNodeRef NamedMDNode);
  *
  * @see llvm::Module::getNamedMetadata()
  */
-LLVMNamedMDNodeRef LLVMGetNamedMetadata(LLVMModuleRef M,
-                                        const char *Name, size_t NameLen);
+LLVM_C_ABI LLVMNamedMDNodeRef LLVMGetNamedMetadata(LLVMModuleRef M,
+                                                   const char *Name,
+                                                   size_t NameLen);
 
 /**
  * Retrieve a NamedMDNode with the given name, creating a new node if no such
@@ -1091,24 +1114,25 @@ LLVMNamedMDNodeRef LLVMGetNamedMetadata(LLVMModuleRef M,
  *
  * @see llvm::Module::getOrInsertNamedMetadata()
  */
-LLVMNamedMDNodeRef LLVMGetOrInsertNamedMetadata(LLVMModuleRef M,
-                                                const char *Name,
-                                                size_t NameLen);
+LLVM_C_ABI LLVMNamedMDNodeRef LLVMGetOrInsertNamedMetadata(LLVMModuleRef M,
+                                                           const char *Name,
+                                                           size_t NameLen);
 
 /**
  * Retrieve the name of a NamedMDNode.
  *
  * @see llvm::NamedMDNode::getName()
  */
-const char *LLVMGetNamedMetadataName(LLVMNamedMDNodeRef NamedMD,
-                                     size_t *NameLen);
+LLVM_C_ABI const char *LLVMGetNamedMetadataName(LLVMNamedMDNodeRef NamedMD,
+                                                size_t *NameLen);
 
 /**
  * Obtain the number of operands for named metadata in a module.
  *
  * @see llvm::Module::getNamedMetadata()
  */
-unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char *Name);
+LLVM_C_ABI unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M,
+                                                    const char *Name);
 
 /**
  * Obtain the named metadata operands for a module.
@@ -1121,8 +1145,8 @@ unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char *Name);
  * @see llvm::Module::getNamedMetadata()
  * @see llvm::MDNode::getOperand()
  */
-void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char *Name,
-                                  LLVMValueRef *Dest);
+LLVM_C_ABI void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char *Name,
+                                             LLVMValueRef *Dest);
 
 /**
  * Add an operand to named metadata.
@@ -1130,8 +1154,8 @@ void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char *Name,
  * @see llvm::Module::getNamedMetadata()
  * @see llvm::MDNode::addOperand()
  */
-void LLVMAddNamedMetadataOperand(LLVMModuleRef M, const char *Name,
-                                 LLVMValueRef Val);
+LLVM_C_ABI void LLVMAddNamedMetadataOperand(LLVMModuleRef M, const char *Name,
+                                            LLVMValueRef Val);
 
 /**
  * Return the directory of the debug location for this value, which must be
@@ -1141,7 +1165,8 @@ void LLVMAddNamedMetadataOperand(LLVMModuleRef M, const char *Name,
  * @see llvm::GlobalVariable::getDebugInfo()
  * @see llvm::Function::getSubprogram()
  */
-const char *LLVMGetDebugLocDirectory(LLVMValueRef Val, unsigned *Length);
+LLVM_C_ABI const char *LLVMGetDebugLocDirectory(LLVMValueRef Val,
+                                                unsigned *Length);
 
 /**
  * Return the filename of the debug location for this value, which must be
@@ -1151,7 +1176,8 @@ const char *LLVMGetDebugLocDirectory(LLVMValueRef Val, unsigned *Length);
  * @see llvm::GlobalVariable::getDebugInfo()
  * @see llvm::Function::getSubprogram()
  */
-const char *LLVMGetDebugLocFilename(LLVMValueRef Val, unsigned *Length);
+LLVM_C_ABI const char *LLVMGetDebugLocFilename(LLVMValueRef Val,
+                                               unsigned *Length);
 
 /**
  * Return the line number of the debug location for this value, which must be
@@ -1161,7 +1187,7 @@ const char *LLVMGetDebugLocFilename(LLVMValueRef Val, unsigned *Length);
  * @see llvm::GlobalVariable::getDebugInfo()
  * @see llvm::Function::getSubprogram()
  */
-unsigned LLVMGetDebugLocLine(LLVMValueRef Val);
+LLVM_C_ABI unsigned LLVMGetDebugLocLine(LLVMValueRef Val);
 
 /**
  * Return the column number of the debug location for this value, which must be
@@ -1169,15 +1195,15 @@ unsigned LLVMGetDebugLocLine(LLVMValueRef Val);
  *
  * @see llvm::Instruction::getDebugLoc()
  */
-unsigned LLVMGetDebugLocColumn(LLVMValueRef Val);
+LLVM_C_ABI unsigned LLVMGetDebugLocColumn(LLVMValueRef Val);
 
 /**
  * Add a function to a module under a specified name.
  *
  * @see llvm::Function::Create()
  */
-LLVMValueRef LLVMAddFunction(LLVMModuleRef M, const char *Name,
-                             LLVMTypeRef FunctionTy);
+LLVM_C_ABI LLVMValueRef LLVMAddFunction(LLVMModuleRef M, const char *Name,
+                                        LLVMTypeRef FunctionTy);
 
 /**
  * Obtain a Function value from a Module by its name.
@@ -1186,7 +1212,7 @@ LLVMValueRef LLVMAddFunction(LLVMModuleRef M, const char *Name,
  *
  * @see llvm::Module::getFunction()
  */
-LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name);
 
 /**
  * Obtain a Function value from a Module by its name.
@@ -1195,22 +1221,23 @@ LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name);
  *
  * @see llvm::Module::getFunction()
  */
-LLVMValueRef LLVMGetNamedFunctionWithLength(LLVMModuleRef M, const char *Name,
-                                            size_t Length);
+LLVM_C_ABI LLVMValueRef LLVMGetNamedFunctionWithLength(LLVMModuleRef M,
+                                                       const char *Name,
+                                                       size_t Length);
 
 /**
  * Obtain an iterator to the first Function in a Module.
  *
  * @see llvm::Module::begin()
  */
-LLVMValueRef LLVMGetFirstFunction(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetFirstFunction(LLVMModuleRef M);
 
 /**
  * Obtain an iterator to the last Function in a Module.
  *
  * @see llvm::Module::end()
  */
-LLVMValueRef LLVMGetLastFunction(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetLastFunction(LLVMModuleRef M);
 
 /**
  * Advance a Function iterator to the next Function.
@@ -1218,7 +1245,7 @@ LLVMValueRef LLVMGetLastFunction(LLVMModuleRef M);
  * Returns NULL if the iterator was already at the end and there are no more
  * functions.
  */
-LLVMValueRef LLVMGetNextFunction(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetNextFunction(LLVMValueRef Fn);
 
 /**
  * Decrement a Function iterator to the previous Function.
@@ -1226,10 +1253,10 @@ LLVMValueRef LLVMGetNextFunction(LLVMValueRef Fn);
  * Returns NULL if the iterator was already at the beginning and there are
  * no previous functions.
  */
-LLVMValueRef LLVMGetPreviousFunction(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetPreviousFunction(LLVMValueRef Fn);
 
 /** Deprecated: Use LLVMSetModuleInlineAsm2 instead. */
-void LLVMSetModuleInlineAsm(LLVMModuleRef M, const char *Asm);
+LLVM_C_ABI void LLVMSetModuleInlineAsm(LLVMModuleRef M, const char *Asm);
 
 /**
  * @}
@@ -1269,7 +1296,7 @@ void LLVMSetModuleInlineAsm(LLVMModuleRef M, const char *Asm);
  *
  * @see llvm::Type:getTypeID()
  */
-LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty);
 
 /**
  * Whether the type has a known size.
@@ -1278,21 +1305,21 @@ LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty);
  *
  * @see llvm::Type::isSized()
  */
-LLVMBool LLVMTypeIsSized(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMBool LLVMTypeIsSized(LLVMTypeRef Ty);
 
 /**
  * Obtain the context to which this type instance is associated.
  *
  * @see llvm::Type::getContext()
  */
-LLVMContextRef LLVMGetTypeContext(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMContextRef LLVMGetTypeContext(LLVMTypeRef Ty);
 
 /**
  * Dump a representation of a type to stderr.
  *
  * @see llvm::Type::dump()
  */
-void LLVMDumpType(LLVMTypeRef Val);
+LLVM_C_ABI void LLVMDumpType(LLVMTypeRef Val);
 
 /**
  * Return a string representation of the type. Use
@@ -1300,7 +1327,7 @@ void LLVMDumpType(LLVMTypeRef Val);
  *
  * @see llvm::Type::print()
  */
-char *LLVMPrintTypeToString(LLVMTypeRef Val);
+LLVM_C_ABI char *LLVMPrintTypeToString(LLVMTypeRef Val);
 
 /**
  * @defgroup LLVMCCoreTypeInt Integer Types
@@ -1313,26 +1340,26 @@ char *LLVMPrintTypeToString(LLVMTypeRef Val);
 /**
  * Obtain an integer type from a context with specified bit width.
  */
-LLVMTypeRef LLVMInt1TypeInContext(LLVMContextRef C);
-LLVMTypeRef LLVMInt8TypeInContext(LLVMContextRef C);
-LLVMTypeRef LLVMInt16TypeInContext(LLVMContextRef C);
-LLVMTypeRef LLVMInt32TypeInContext(LLVMContextRef C);
-LLVMTypeRef LLVMInt64TypeInContext(LLVMContextRef C);
-LLVMTypeRef LLVMInt128TypeInContext(LLVMContextRef C);
-LLVMTypeRef LLVMIntTypeInContext(LLVMContextRef C, unsigned NumBits);
+LLVM_C_ABI LLVMTypeRef LLVMInt1TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMInt8TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMInt16TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMInt32TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMInt64TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMInt128TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMIntTypeInContext(LLVMContextRef C, unsigned NumBits);
 
 /**
  * Obtain an integer type from the global context with a specified bit
  * width.
  */
-LLVMTypeRef LLVMInt1Type(void);
-LLVMTypeRef LLVMInt8Type(void);
-LLVMTypeRef LLVMInt16Type(void);
-LLVMTypeRef LLVMInt32Type(void);
-LLVMTypeRef LLVMInt64Type(void);
-LLVMTypeRef LLVMInt128Type(void);
-LLVMTypeRef LLVMIntType(unsigned NumBits);
-unsigned LLVMGetIntTypeWidth(LLVMTypeRef IntegerTy);
+LLVM_C_ABI LLVMTypeRef LLVMInt1Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMInt8Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMInt16Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMInt32Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMInt64Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMInt128Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMIntType(unsigned NumBits);
+LLVM_C_ABI unsigned LLVMGetIntTypeWidth(LLVMTypeRef IntegerTy);
 
 /**
  * @}
@@ -1347,51 +1374,51 @@ unsigned LLVMGetIntTypeWidth(LLVMTypeRef IntegerTy);
 /**
  * Obtain a 16-bit floating point type from a context.
  */
-LLVMTypeRef LLVMHalfTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMHalfTypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a 16-bit brain floating point type from a context.
  */
-LLVMTypeRef LLVMBFloatTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMBFloatTypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a 32-bit floating point type from a context.
  */
-LLVMTypeRef LLVMFloatTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMFloatTypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a 64-bit floating point type from a context.
  */
-LLVMTypeRef LLVMDoubleTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMDoubleTypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a 80-bit floating point type (X87) from a context.
  */
-LLVMTypeRef LLVMX86FP80TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMX86FP80TypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a 128-bit floating point type (112-bit mantissa) from a
  * context.
  */
-LLVMTypeRef LLVMFP128TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMFP128TypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a 128-bit floating point type (two 64-bits) from a context.
  */
-LLVMTypeRef LLVMPPCFP128TypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMPPCFP128TypeInContext(LLVMContextRef C);
 
 /**
  * Obtain a floating point type from the global context.
  *
  * These map to the functions in this group of the same name.
  */
-LLVMTypeRef LLVMHalfType(void);
-LLVMTypeRef LLVMBFloatType(void);
-LLVMTypeRef LLVMFloatType(void);
-LLVMTypeRef LLVMDoubleType(void);
-LLVMTypeRef LLVMX86FP80Type(void);
-LLVMTypeRef LLVMFP128Type(void);
-LLVMTypeRef LLVMPPCFP128Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMHalfType(void);
+LLVM_C_ABI LLVMTypeRef LLVMBFloatType(void);
+LLVM_C_ABI LLVMTypeRef LLVMFloatType(void);
+LLVM_C_ABI LLVMTypeRef LLVMDoubleType(void);
+LLVM_C_ABI LLVMTypeRef LLVMX86FP80Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMFP128Type(void);
+LLVM_C_ABI LLVMTypeRef LLVMPPCFP128Type(void);
 
 /**
  * @}
@@ -1409,24 +1436,24 @@ LLVMTypeRef LLVMPPCFP128Type(void);
  * The function is defined as a tuple of a return Type, a list of
  * parameter types, and whether the function is variadic.
  */
-LLVMTypeRef LLVMFunctionType(LLVMTypeRef ReturnType,
-                             LLVMTypeRef *ParamTypes, unsigned ParamCount,
-                             LLVMBool IsVarArg);
+LLVM_C_ABI LLVMTypeRef LLVMFunctionType(LLVMTypeRef ReturnType,
+                                        LLVMTypeRef *ParamTypes,
+                                        unsigned ParamCount, LLVMBool IsVarArg);
 
 /**
  * Returns whether a function type is variadic.
  */
-LLVMBool LLVMIsFunctionVarArg(LLVMTypeRef FunctionTy);
+LLVM_C_ABI LLVMBool LLVMIsFunctionVarArg(LLVMTypeRef FunctionTy);
 
 /**
  * Obtain the Type this function Type returns.
  */
-LLVMTypeRef LLVMGetReturnType(LLVMTypeRef FunctionTy);
+LLVM_C_ABI LLVMTypeRef LLVMGetReturnType(LLVMTypeRef FunctionTy);
 
 /**
  * Obtain the number of parameters this function accepts.
  */
-unsigned LLVMCountParamTypes(LLVMTypeRef FunctionTy);
+LLVM_C_ABI unsigned LLVMCountParamTypes(LLVMTypeRef FunctionTy);
 
 /**
  * Obtain the types of a function's parameters.
@@ -1439,7 +1466,7 @@ unsigned LLVMCountParamTypes(LLVMTypeRef FunctionTy);
  * @param FunctionTy The function type to operate on.
  * @param Dest Memory address of an array to be filled with result.
  */
-void LLVMGetParamTypes(LLVMTypeRef FunctionTy, LLVMTypeRef *Dest);
+LLVM_C_ABI void LLVMGetParamTypes(LLVMTypeRef FunctionTy, LLVMTypeRef *Dest);
 
 /**
  * @}
@@ -1463,45 +1490,49 @@ void LLVMGetParamTypes(LLVMTypeRef FunctionTy, LLVMTypeRef *Dest);
  *
  * @see llvm::StructType::create()
  */
-LLVMTypeRef LLVMStructTypeInContext(LLVMContextRef C, LLVMTypeRef *ElementTypes,
-                                    unsigned ElementCount, LLVMBool Packed);
+LLVM_C_ABI LLVMTypeRef LLVMStructTypeInContext(LLVMContextRef C,
+                                               LLVMTypeRef *ElementTypes,
+                                               unsigned ElementCount,
+                                               LLVMBool Packed);
 
 /**
  * Create a new structure type in the global context.
  *
  * @see llvm::StructType::create()
  */
-LLVMTypeRef LLVMStructType(LLVMTypeRef *ElementTypes, unsigned ElementCount,
-                           LLVMBool Packed);
+LLVM_C_ABI LLVMTypeRef LLVMStructType(LLVMTypeRef *ElementTypes,
+                                      unsigned ElementCount, LLVMBool Packed);
 
 /**
  * Create an empty structure in a context having a specified name.
  *
  * @see llvm::StructType::create()
  */
-LLVMTypeRef LLVMStructCreateNamed(LLVMContextRef C, const char *Name);
+LLVM_C_ABI LLVMTypeRef LLVMStructCreateNamed(LLVMContextRef C,
+                                             const char *Name);
 
 /**
  * Obtain the name of a structure.
  *
  * @see llvm::StructType::getName()
  */
-const char *LLVMGetStructName(LLVMTypeRef Ty);
+LLVM_C_ABI const char *LLVMGetStructName(LLVMTypeRef Ty);
 
 /**
  * Set the contents of a structure type.
  *
  * @see llvm::StructType::setBody()
  */
-void LLVMStructSetBody(LLVMTypeRef StructTy, LLVMTypeRef *ElementTypes,
-                       unsigned ElementCount, LLVMBool Packed);
+LLVM_C_ABI void LLVMStructSetBody(LLVMTypeRef StructTy,
+                                  LLVMTypeRef *ElementTypes,
+                                  unsigned ElementCount, LLVMBool Packed);
 
 /**
  * Get the number of elements defined inside the structure.
  *
  * @see llvm::StructType::getNumElements()
  */
-unsigned LLVMCountStructElementTypes(LLVMTypeRef StructTy);
+LLVM_C_ABI unsigned LLVMCountStructElementTypes(LLVMTypeRef StructTy);
 
 /**
  * Get the elements within a structure.
@@ -1513,35 +1544,37 @@ unsigned LLVMCountStructElementTypes(LLVMTypeRef StructTy);
  * of the structure type itself, which is the lifetime of the context it
  * is contained in.
  */
-void LLVMGetStructElementTypes(LLVMTypeRef StructTy, LLVMTypeRef *Dest);
+LLVM_C_ABI void LLVMGetStructElementTypes(LLVMTypeRef StructTy,
+                                          LLVMTypeRef *Dest);
 
 /**
  * Get the type of the element at a given index in the structure.
  *
  * @see llvm::StructType::getTypeAtIndex()
  */
-LLVMTypeRef LLVMStructGetTypeAtIndex(LLVMTypeRef StructTy, unsigned i);
+LLVM_C_ABI LLVMTypeRef LLVMStructGetTypeAtIndex(LLVMTypeRef StructTy,
+                                                unsigned i);
 
 /**
  * Determine whether a structure is packed.
  *
  * @see llvm::StructType::isPacked()
  */
-LLVMBool LLVMIsPackedStruct(LLVMTypeRef StructTy);
+LLVM_C_ABI LLVMBool LLVMIsPackedStruct(LLVMTypeRef StructTy);
 
 /**
  * Determine whether a structure is opaque.
  *
  * @see llvm::StructType::isOpaque()
  */
-LLVMBool LLVMIsOpaqueStruct(LLVMTypeRef StructTy);
+LLVM_C_ABI LLVMBool LLVMIsOpaqueStruct(LLVMTypeRef StructTy);
 
 /**
  * Determine whether a structure is literal.
  *
  * @see llvm::StructType::isLiteral()
  */
-LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy);
+LLVM_C_ABI LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy);
 
 /**
  * @}
@@ -1561,21 +1594,21 @@ LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy);
  *
  * @see llvm::SequentialType::getElementType()
  */
-LLVMTypeRef LLVMGetElementType(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMTypeRef LLVMGetElementType(LLVMTypeRef Ty);
 
 /**
  * Returns type's subtypes
  *
  * @see llvm::Type::subtypes()
  */
-void LLVMGetSubtypes(LLVMTypeRef Tp, LLVMTypeRef *Arr);
+LLVM_C_ABI void LLVMGetSubtypes(LLVMTypeRef Tp, LLVMTypeRef *Arr);
 
 /**
  *  Return the number of types in the derived type.
  *
  * @see llvm::Type::getNumContainedTypes()
  */
-unsigned LLVMGetNumContainedTypes(LLVMTypeRef Tp);
+LLVM_C_ABI unsigned LLVMGetNumContainedTypes(LLVMTypeRef Tp);
 
 /**
  * Create a fixed size array type that refers to a specific type.
@@ -1587,7 +1620,8 @@ unsigned LLVMGetNumContainedTypes(LLVMTypeRef Tp);
  * LLVMArrayType2
  * @see llvm::ArrayType::get()
  */
-LLVMTypeRef LLVMArrayType(LLVMTypeRef ElementType, unsigned ElementCount);
+LLVM_C_ABI LLVMTypeRef LLVMArrayType(LLVMTypeRef ElementType,
+                                     unsigned ElementCount);
 
 /**
  * Create a fixed size array type that refers to a specific type.
@@ -1597,7 +1631,8 @@ LLVMTypeRef LLVMArrayType(LLVMTypeRef ElementType, unsigned ElementCount);
  *
  * @see llvm::ArrayType::get()
  */
-LLVMTypeRef LLVMArrayType2(LLVMTypeRef ElementType, uint64_t ElementCount);
+LLVM_C_ABI LLVMTypeRef LLVMArrayType2(LLVMTypeRef ElementType,
+                                      uint64_t ElementCount);
 
 /**
  * Obtain the length of an array type.
@@ -1608,7 +1643,7 @@ LLVMTypeRef LLVMArrayType2(LLVMTypeRef ElementType, uint64_t ElementCount);
  * LLVMGetArrayLength2
  * @see llvm::ArrayType::getNumElements()
  */
-unsigned LLVMGetArrayLength(LLVMTypeRef ArrayTy);
+LLVM_C_ABI unsigned LLVMGetArrayLength(LLVMTypeRef ArrayTy);
 
 /**
  * Obtain the length of an array type.
@@ -1617,7 +1652,7 @@ unsigned LLVMGetArrayLength(LLVMTypeRef ArrayTy);
  *
  * @see llvm::ArrayType::getNumElements()
  */
-uint64_t LLVMGetArrayLength2(LLVMTypeRef ArrayTy);
+LLVM_C_ABI uint64_t LLVMGetArrayLength2(LLVMTypeRef ArrayTy);
 
 /**
  * Create a pointer type that points to a defined type.
@@ -1627,7 +1662,8 @@ uint64_t LLVMGetArrayLength2(LLVMTypeRef ArrayTy);
  *
  * @see llvm::PointerType::get()
  */
-LLVMTypeRef LLVMPointerType(LLVMTypeRef ElementType, unsigned AddressSpace);
+LLVM_C_ABI LLVMTypeRef LLVMPointerType(LLVMTypeRef ElementType,
+                                       unsigned AddressSpace);
 
 /**
  * Determine whether a pointer is opaque.
@@ -1636,14 +1672,15 @@ LLVMTypeRef LLVMPointerType(LLVMTypeRef ElementType, unsigned AddressSpace);
  *
  * @see llvm::Type::isOpaquePointerTy()
  */
-LLVMBool LLVMPointerTypeIsOpaque(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMBool LLVMPointerTypeIsOpaque(LLVMTypeRef Ty);
 
 /**
  * Create an opaque pointer type in a context.
  *
  * @see llvm::PointerType::get()
  */
-LLVMTypeRef LLVMPointerTypeInContext(LLVMContextRef C, unsigned AddressSpace);
+LLVM_C_ABI LLVMTypeRef LLVMPointerTypeInContext(LLVMContextRef C,
+                                                unsigned AddressSpace);
 
 /**
  * Obtain the address space of a pointer type.
@@ -1652,7 +1689,7 @@ LLVMTypeRef LLVMPointerTypeInContext(LLVMContextRef C, unsigned AddressSpace);
  *
  * @see llvm::PointerType::getAddressSpace()
  */
-unsigned LLVMGetPointerAddressSpace(LLVMTypeRef PointerTy);
+LLVM_C_ABI unsigned LLVMGetPointerAddressSpace(LLVMTypeRef PointerTy);
 
 /**
  * Create a vector type that contains a defined type and has a specific
@@ -1663,7 +1700,8 @@ unsigned LLVMGetPointerAddressSpace(LLVMTypeRef PointerTy);
  *
  * @see llvm::VectorType::get()
  */
-LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount);
+LLVM_C_ABI LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType,
+                                      unsigned ElementCount);
 
 /**
  * Create a vector type that contains a defined type and has a scalable
@@ -1674,8 +1712,8 @@ LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount);
  *
  * @see llvm::ScalableVectorType::get()
  */
-LLVMTypeRef LLVMScalableVectorType(LLVMTypeRef ElementType,
-                                   unsigned ElementCount);
+LLVM_C_ABI LLVMTypeRef LLVMScalableVectorType(LLVMTypeRef ElementType,
+                                              unsigned ElementCount);
 
 /**
  * Obtain the (possibly scalable) number of elements in a vector type.
@@ -1684,28 +1722,29 @@ LLVMTypeRef LLVMScalableVectorType(LLVMTypeRef ElementType,
  *
  * @see llvm::VectorType::getNumElements()
  */
-unsigned LLVMGetVectorSize(LLVMTypeRef VectorTy);
+LLVM_C_ABI unsigned LLVMGetVectorSize(LLVMTypeRef VectorTy);
 
 /**
  * Get the pointer value for the associated ConstantPtrAuth constant.
  *
  * @see llvm::ConstantPtrAuth::getPointer
  */
-LLVMValueRef LLVMGetConstantPtrAuthPointer(LLVMValueRef PtrAuth);
+LLVM_C_ABI LLVMValueRef LLVMGetConstantPtrAuthPointer(LLVMValueRef PtrAuth);
 
 /**
  * Get the key value for the associated ConstantPtrAuth constant.
  *
  * @see llvm::ConstantPtrAuth::getKey
  */
-LLVMValueRef LLVMGetConstantPtrAuthKey(LLVMValueRef PtrAuth);
+LLVM_C_ABI LLVMValueRef LLVMGetConstantPtrAuthKey(LLVMValueRef PtrAuth);
 
 /**
  * Get the discriminator value for the associated ConstantPtrAuth constant.
  *
  * @see llvm::ConstantPtrAuth::getDiscriminator
  */
-LLVMValueRef LLVMGetConstantPtrAuthDiscriminator(LLVMValueRef PtrAuth);
+LLVM_C_ABI LLVMValueRef
+LLVMGetConstantPtrAuthDiscriminator(LLVMValueRef PtrAuth);
 
 /**
  * Get the address discriminator value for the associated ConstantPtrAuth
@@ -1713,7 +1752,8 @@ LLVMValueRef LLVMGetConstantPtrAuthDiscriminator(LLVMValueRef PtrAuth);
  *
  * @see llvm::ConstantPtrAuth::getAddrDiscriminator
  */
-LLVMValueRef LLVMGetConstantPtrAuthAddrDiscriminator(LLVMValueRef PtrAuth);
+LLVM_C_ABI LLVMValueRef
+LLVMGetConstantPtrAuthAddrDiscriminator(LLVMValueRef PtrAuth);
 
 /**
  * @}
@@ -1728,80 +1768,79 @@ LLVMValueRef LLVMGetConstantPtrAuthAddrDiscriminator(LLVMValueRef PtrAuth);
 /**
  * Create a void type in a context.
  */
-LLVMTypeRef LLVMVoidTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMVoidTypeInContext(LLVMContextRef C);
 
 /**
  * Create a label type in a context.
  */
-LLVMTypeRef LLVMLabelTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMLabelTypeInContext(LLVMContextRef C);
 
 /**
  * Create a X86 AMX type in a context.
  */
-LLVMTypeRef LLVMX86AMXTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMX86AMXTypeInContext(LLVMContextRef C);
 
 /**
  * Create a token type in a context.
  */
-LLVMTypeRef LLVMTokenTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMTokenTypeInContext(LLVMContextRef C);
 
 /**
  * Create a metadata type in a context.
  */
-LLVMTypeRef LLVMMetadataTypeInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMTypeRef LLVMMetadataTypeInContext(LLVMContextRef C);
 
 /**
  * These are similar to the above functions except they operate on the
  * global context.
  */
-LLVMTypeRef LLVMVoidType(void);
-LLVMTypeRef LLVMLabelType(void);
-LLVMTypeRef LLVMX86AMXType(void);
+LLVM_C_ABI LLVMTypeRef LLVMVoidType(void);
+LLVM_C_ABI LLVMTypeRef LLVMLabelType(void);
+LLVM_C_ABI LLVMTypeRef LLVMX86AMXType(void);
 
 /**
  * Create a target extension type in LLVM context.
  */
-LLVMTypeRef LLVMTargetExtTypeInContext(LLVMContextRef C, const char *Name,
-                                       LLVMTypeRef *TypeParams,
-                                       unsigned TypeParamCount,
-                                       unsigned *IntParams,
-                                       unsigned IntParamCount);
+LLVM_C_ABI LLVMTypeRef LLVMTargetExtTypeInContext(
+    LLVMContextRef C, const char *Name, LLVMTypeRef *TypeParams,
+    unsigned TypeParamCount, unsigned *IntParams, unsigned IntParamCount);
 
 /**
  * Obtain the name for this target extension type.
  *
  * @see llvm::TargetExtType::getName()
  */
-const char *LLVMGetTargetExtTypeName(LLVMTypeRef TargetExtTy);
+LLVM_C_ABI const char *LLVMGetTargetExtTypeName(LLVMTypeRef TargetExtTy);
 
 /**
  * Obtain the number of type parameters for this target extension type.
  *
  * @see llvm::TargetExtType::getNumTypeParameters()
  */
-unsigned LLVMGetTargetExtTypeNumTypeParams(LLVMTypeRef TargetExtTy);
+LLVM_C_ABI unsigned LLVMGetTargetExtTypeNumTypeParams(LLVMTypeRef TargetExtTy);
 
 /**
  * Get the type parameter at the given index for the target extension type.
  *
  * @see llvm::TargetExtType::getTypeParameter()
  */
-LLVMTypeRef LLVMGetTargetExtTypeTypeParam(LLVMTypeRef TargetExtTy,
-                                          unsigned Idx);
+LLVM_C_ABI LLVMTypeRef LLVMGetTargetExtTypeTypeParam(LLVMTypeRef TargetExtTy,
+                                                     unsigned Idx);
 
 /**
  * Obtain the number of int parameters for this target extension type.
  *
  * @see llvm::TargetExtType::getNumIntParameters()
  */
-unsigned LLVMGetTargetExtTypeNumIntParams(LLVMTypeRef TargetExtTy);
+LLVM_C_ABI unsigned LLVMGetTargetExtTypeNumIntParams(LLVMTypeRef TargetExtTy);
 
 /**
  * Get the int parameter at the given index for the target extension type.
  *
  * @see llvm::TargetExtType::getIntParameter()
  */
-unsigned LLVMGetTargetExtTypeIntParam(LLVMTypeRef TargetExtTy, unsigned Idx);
+LLVM_C_ABI unsigned LLVMGetTargetExtTypeIntParam(LLVMTypeRef TargetExtTy,
+                                                 unsigned Idx);
 
 /**
  * @}
@@ -1944,35 +1983,36 @@ unsigned LLVMGetTargetExtTypeIntParam(LLVMTypeRef TargetExtTy, unsigned Idx);
  *
  * @see llvm::Value::getType()
  */
-LLVMTypeRef LLVMTypeOf(LLVMValueRef Val);
+LLVM_C_ABI LLVMTypeRef LLVMTypeOf(LLVMValueRef Val);
 
 /**
  * Obtain the enumerated type of a Value instance.
  *
  * @see llvm::Value::getValueID()
  */
-LLVMValueKind LLVMGetValueKind(LLVMValueRef Val);
+LLVM_C_ABI LLVMValueKind LLVMGetValueKind(LLVMValueRef Val);
 
 /**
  * Obtain the string name of a value.
  *
  * @see llvm::Value::getName()
  */
-const char *LLVMGetValueName2(LLVMValueRef Val, size_t *Length);
+LLVM_C_ABI const char *LLVMGetValueName2(LLVMValueRef Val, size_t *Length);
 
 /**
  * Set the string name of a value.
  *
  * @see llvm::Value::setName()
  */
-void LLVMSetValueName2(LLVMValueRef Val, const char *Name, size_t NameLen);
+LLVM_C_ABI void LLVMSetValueName2(LLVMValueRef Val, const char *Name,
+                                  size_t NameLen);
 
 /**
  * Dump a representation of a value to stderr.
  *
  * @see llvm::Value::dump()
  */
-void LLVMDumpValue(LLVMValueRef Val);
+LLVM_C_ABI void LLVMDumpValue(LLVMValueRef Val);
 
 /**
  * Return a string representation of the value. Use
@@ -1980,14 +2020,14 @@ void LLVMDumpValue(LLVMValueRef Val);
  *
  * @see llvm::Value::print()
  */
-char *LLVMPrintValueToString(LLVMValueRef Val);
+LLVM_C_ABI char *LLVMPrintValueToString(LLVMValueRef Val);
 
 /**
  * Obtain the context to which this value is associated.
  *
  * @see llvm::Value::getContext()
  */
-LLVMContextRef LLVMGetValueContext(LLVMValueRef Val);
+LLVM_C_ABI LLVMContextRef LLVMGetValueContext(LLVMValueRef Val);
 
 /**
  * Return a string representation of the DbgRecord. Use
@@ -1995,29 +2035,30 @@ LLVMContextRef LLVMGetValueContext(LLVMValueRef Val);
  *
  * @see llvm::DbgRecord::print()
  */
-char *LLVMPrintDbgRecordToString(LLVMDbgRecordRef Record);
+LLVM_C_ABI char *LLVMPrintDbgRecordToString(LLVMDbgRecordRef Record);
 
 /**
  * Replace all uses of a value with another one.
  *
  * @see llvm::Value::replaceAllUsesWith()
  */
-void LLVMReplaceAllUsesWith(LLVMValueRef OldVal, LLVMValueRef NewVal);
+LLVM_C_ABI void LLVMReplaceAllUsesWith(LLVMValueRef OldVal,
+                                       LLVMValueRef NewVal);
 
 /**
  * Determine whether the specified value instance is constant.
  */
-LLVMBool LLVMIsConstant(LLVMValueRef Val);
+LLVM_C_ABI LLVMBool LLVMIsConstant(LLVMValueRef Val);
 
 /**
  * Determine whether a value instance is undefined.
  */
-LLVMBool LLVMIsUndef(LLVMValueRef Val);
+LLVM_C_ABI LLVMBool LLVMIsUndef(LLVMValueRef Val);
 
 /**
  * Determine whether a value instance is poisonous.
  */
-LLVMBool LLVMIsPoison(LLVMValueRef Val);
+LLVM_C_ABI LLVMBool LLVMIsPoison(LLVMValueRef Val);
 
 /**
  * Convert value instances between types.
@@ -2030,18 +2071,18 @@ LLVMBool LLVMIsPoison(LLVMValueRef Val);
  *
  * @see llvm::dyn_cast_or_null<>
  */
-#define LLVM_DECLARE_VALUE_CAST(name) \
-  LLVMValueRef LLVMIsA##name(LLVMValueRef Val);
+#define LLVM_DECLARE_VALUE_CAST(name)                                          \
+  LLVM_C_ABI LLVMValueRef LLVMIsA##name(LLVMValueRef Val);
 LLVM_FOR_EACH_VALUE_SUBCLASS(LLVM_DECLARE_VALUE_CAST)
 
-LLVMValueRef LLVMIsAMDNode(LLVMValueRef Val);
-LLVMValueRef LLVMIsAValueAsMetadata(LLVMValueRef Val);
-LLVMValueRef LLVMIsAMDString(LLVMValueRef Val);
+LLVM_C_ABI LLVMValueRef LLVMIsAMDNode(LLVMValueRef Val);
+LLVM_C_ABI LLVMValueRef LLVMIsAValueAsMetadata(LLVMValueRef Val);
+LLVM_C_ABI LLVMValueRef LLVMIsAMDString(LLVMValueRef Val);
 
 /** Deprecated: Use LLVMGetValueName2 instead. */
-const char *LLVMGetValueName(LLVMValueRef Val);
+LLVM_C_ABI const char *LLVMGetValueName(LLVMValueRef Val);
 /** Deprecated: Use LLVMSetValueName2 instead. */
-void LLVMSetValueName(LLVMValueRef Val, const char *Name);
+LLVM_C_ABI void LLVMSetValueName(LLVMValueRef Val, const char *Name);
 
 /**
  * @}
@@ -2070,7 +2111,7 @@ void LLVMSetValueName(LLVMValueRef Val, const char *Name);
  *
  * @see llvm::Value::use_begin()
  */
-LLVMUseRef LLVMGetFirstUse(LLVMValueRef Val);
+LLVM_C_ABI LLVMUseRef LLVMGetFirstUse(LLVMValueRef Val);
 
 /**
  * Obtain the next use of a value.
@@ -2078,7 +2119,7 @@ LLVMUseRef LLVMGetFirstUse(LLVMValueRef Val);
  * This effectively advances the iterator. It returns NULL if you are on
  * the final use and no more are available.
  */
-LLVMUseRef LLVMGetNextUse(LLVMUseRef U);
+LLVM_C_ABI LLVMUseRef LLVMGetNextUse(LLVMUseRef U);
 
 /**
  * Obtain the user value for a user.
@@ -2087,14 +2128,14 @@ LLVMUseRef LLVMGetNextUse(LLVMUseRef U);
  *
  * @see llvm::Use::getUser()
  */
-LLVMValueRef LLVMGetUser(LLVMUseRef U);
+LLVM_C_ABI LLVMValueRef LLVMGetUser(LLVMUseRef U);
 
 /**
  * Obtain the value this use corresponds to.
  *
  * @see llvm::Use::get().
  */
-LLVMValueRef LLVMGetUsedValue(LLVMUseRef U);
+LLVM_C_ABI LLVMValueRef LLVMGetUsedValue(LLVMUseRef U);
 
 /**
  * @}
@@ -2115,28 +2156,29 @@ LLVMValueRef LLVMGetUsedValue(LLVMUseRef U);
  *
  * @see llvm::User::getOperand()
  */
-LLVMValueRef LLVMGetOperand(LLVMValueRef Val, unsigned Index);
+LLVM_C_ABI LLVMValueRef LLVMGetOperand(LLVMValueRef Val, unsigned Index);
 
 /**
  * Obtain the use of an operand at a specific index in a llvm::User value.
  *
  * @see llvm::User::getOperandUse()
  */
-LLVMUseRef LLVMGetOperandUse(LLVMValueRef Val, unsigned Index);
+LLVM_C_ABI LLVMUseRef LLVMGetOperandUse(LLVMValueRef Val, unsigned Index);
 
 /**
  * Set an operand at a specific index in a llvm::User value.
  *
  * @see llvm::User::setOperand()
  */
-void LLVMSetOperand(LLVMValueRef User, unsigned Index, LLVMValueRef Val);
+LLVM_C_ABI void LLVMSetOperand(LLVMValueRef User, unsigned Index,
+                               LLVMValueRef Val);
 
 /**
  * Obtain the number of operands in a llvm::User value.
  *
  * @see llvm::User::getNumOperands()
  */
-int LLVMGetNumOperands(LLVMValueRef Val);
+LLVM_C_ABI int LLVMGetNumOperands(LLVMValueRef Val);
 
 /**
  * @}
@@ -2159,7 +2201,7 @@ int LLVMGetNumOperands(LLVMValueRef Val);
  *
  * @see llvm::Constant::getNullValue()
  */
-LLVMValueRef LLVMConstNull(LLVMTypeRef Ty); /* all zeroes */
+LLVM_C_ABI LLVMValueRef LLVMConstNull(LLVMTypeRef Ty); /* all zeroes */
 
 /**
  * Obtain a constant value referring to the instance of a type
@@ -2169,34 +2211,34 @@ LLVMValueRef LLVMConstNull(LLVMTypeRef Ty); /* all zeroes */
  *
  * @see llvm::Constant::getAllOnesValue()
  */
-LLVMValueRef LLVMConstAllOnes(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMValueRef LLVMConstAllOnes(LLVMTypeRef Ty);
 
 /**
  * Obtain a constant value referring to an undefined value of a type.
  *
  * @see llvm::UndefValue::get()
  */
-LLVMValueRef LLVMGetUndef(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMValueRef LLVMGetUndef(LLVMTypeRef Ty);
 
 /**
  * Obtain a constant value referring to a poison value of a type.
  *
  * @see llvm::PoisonValue::get()
  */
-LLVMValueRef LLVMGetPoison(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMValueRef LLVMGetPoison(LLVMTypeRef Ty);
 
 /**
  * Determine whether a value instance is null.
  *
  * @see llvm::Constant::isNullValue()
  */
-LLVMBool LLVMIsNull(LLVMValueRef Val);
+LLVM_C_ABI LLVMBool LLVMIsNull(LLVMValueRef Val);
 
 /**
  * Obtain a constant that is a constant pointer pointing to NULL for a
  * specified type.
  */
-LLVMValueRef LLVMConstPointerNull(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMValueRef LLVMConstPointerNull(LLVMTypeRef Ty);
 
 /**
  * @defgroup LLVMCCoreValueConstantScalar Scalar constants
@@ -2225,17 +2267,16 @@ LLVMValueRef LLVMConstPointerNull(LLVMTypeRef Ty);
  * @param N The value the returned instance should refer to.
  * @param SignExtend Whether to sign extend the produced value.
  */
-LLVMValueRef LLVMConstInt(LLVMTypeRef IntTy, unsigned long long N,
-                          LLVMBool SignExtend);
+LLVM_C_ABI LLVMValueRef LLVMConstInt(LLVMTypeRef IntTy, unsigned long long N,
+                                     LLVMBool SignExtend);
 
 /**
  * Obtain a constant value for an integer of arbitrary precision.
  *
  * @see llvm::ConstantInt::get()
  */
-LLVMValueRef LLVMConstIntOfArbitraryPrecision(LLVMTypeRef IntTy,
-                                              unsigned NumWords,
-                                              const uint64_t Words[]);
+LLVM_C_ABI LLVMValueRef LLVMConstIntOfArbitraryPrecision(
+    LLVMTypeRef IntTy, unsigned NumWords, const uint64_t Words[]);
 
 /**
  * Obtain a constant value for an integer parsed from a string.
@@ -2246,8 +2287,8 @@ LLVMValueRef LLVMConstIntOfArbitraryPrecision(LLVMTypeRef IntTy,
  *
  * @see llvm::ConstantInt::get()
  */
-LLVMValueRef LLVMConstIntOfString(LLVMTypeRef IntTy, const char *Text,
-                                  uint8_t Radix);
+LLVM_C_ABI LLVMValueRef LLVMConstIntOfString(LLVMTypeRef IntTy,
+                                             const char *Text, uint8_t Radix);
 
 /**
  * Obtain a constant value for an integer parsed from a string with
@@ -2255,13 +2296,15 @@ LLVMValueRef LLVMConstIntOfString(LLVMTypeRef IntTy, const char *Text,
  *
  * @see llvm::ConstantInt::get()
  */
-LLVMValueRef LLVMConstIntOfStringAndSize(LLVMTypeRef IntTy, const char *Text,
-                                         unsigned SLen, uint8_t Radix);
+LLVM_C_ABI LLVMValueRef LLVMConstIntOfStringAndSize(LLVMTypeRef IntTy,
+                                                    const char *Text,
+                                                    unsigned SLen,
+                                                    uint8_t Radix);
 
 /**
  * Obtain a constant value referring to a double floating point value.
  */
-LLVMValueRef LLVMConstReal(LLVMTypeRef RealTy, double N);
+LLVM_C_ABI LLVMValueRef LLVMConstReal(LLVMTypeRef RealTy, double N);
 
 /**
  * Obtain a constant for a floating point value parsed from a string.
@@ -2269,27 +2312,30 @@ LLVMValueRef LLVMConstReal(LLVMTypeRef RealTy, double N);
  * A similar API, LLVMConstRealOfStringAndSize is also available. It
  * should be used if the input string's length is known.
  */
-LLVMValueRef LLVMConstRealOfString(LLVMTypeRef RealTy, const char *Text);
+LLVM_C_ABI LLVMValueRef LLVMConstRealOfString(LLVMTypeRef RealTy,
+                                              const char *Text);
 
 /**
  * Obtain a constant for a floating point value parsed from a string.
  */
-LLVMValueRef LLVMConstRealOfStringAndSize(LLVMTypeRef RealTy, const char *Text,
-                                          unsigned SLen);
+LLVM_C_ABI LLVMValueRef LLVMConstRealOfStringAndSize(LLVMTypeRef RealTy,
+                                                     const char *Text,
+                                                     unsigned SLen);
 
 /**
  * Obtain the zero extended value for an integer constant value.
  *
  * @see llvm::ConstantInt::getZExtValue()
  */
-unsigned long long LLVMConstIntGetZExtValue(LLVMValueRef ConstantVal);
+LLVM_C_ABI unsigned long long
+LLVMConstIntGetZExtValue(LLVMValueRef ConstantVal);
 
 /**
  * Obtain the sign extended value for an integer constant value.
  *
  * @see llvm::ConstantInt::getSExtValue()
  */
-long long LLVMConstIntGetSExtValue(LLVMValueRef ConstantVal);
+LLVM_C_ABI long long LLVMConstIntGetSExtValue(LLVMValueRef ConstantVal);
 
 /**
  * Obtain the double value for an floating point constant value.
@@ -2297,7 +2343,8 @@ long long LLVMConstIntGetSExtValue(LLVMValueRef ConstantVal);
  *
  * @see llvm::ConstantFP::getDoubleValue
  */
-double LLVMConstRealGetDouble(LLVMValueRef ConstantVal, LLVMBool *losesInfo);
+LLVM_C_ABI double LLVMConstRealGetDouble(LLVMValueRef ConstantVal,
+                                         LLVMBool *losesInfo);
 
 /**
  * @}
@@ -2318,17 +2365,20 @@ double LLVMConstRealGetDouble(LLVMValueRef ConstantVal, LLVMBool *losesInfo);
  * accurate LLVMConstStringInContext2
  * @see llvm::ConstantDataArray::getString()
  */
-LLVMValueRef LLVMConstStringInContext(LLVMContextRef C, const char *Str,
-                                      unsigned Length, LLVMBool DontNullTerminate);
+LLVM_C_ABI LLVMValueRef LLVMConstStringInContext(LLVMContextRef C,
+                                                 const char *Str,
+                                                 unsigned Length,
+                                                 LLVMBool DontNullTerminate);
 
 /**
  * Create a ConstantDataSequential and initialize it with a string.
  *
  * @see llvm::ConstantDataArray::getString()
  */
-LLVMValueRef LLVMConstStringInContext2(LLVMContextRef C, const char *Str,
-                                       size_t Length,
-                                       LLVMBool DontNullTerminate);
+LLVM_C_ABI LLVMValueRef LLVMConstStringInContext2(LLVMContextRef C,
+                                                  const char *Str,
+                                                  size_t Length,
+                                                  LLVMBool DontNullTerminate);
 
 /**
  * Create a ConstantDataSequential with string content in the global context.
@@ -2339,22 +2389,22 @@ LLVMValueRef LLVMConstStringInContext2(LLVMContextRef C, const char *Str,
  * @see LLVMConstStringInContext()
  * @see llvm::ConstantDataArray::getString()
  */
-LLVMValueRef LLVMConstString(const char *Str, unsigned Length,
-                             LLVMBool DontNullTerminate);
+LLVM_C_ABI LLVMValueRef LLVMConstString(const char *Str, unsigned Length,
+                                        LLVMBool DontNullTerminate);
 
 /**
  * Returns true if the specified constant is an array of i8.
  *
  * @see ConstantDataSequential::getAsString()
  */
-LLVMBool LLVMIsConstantString(LLVMValueRef c);
+LLVM_C_ABI LLVMBool LLVMIsConstantString(LLVMValueRef c);
 
 /**
  * Get the given constant data sequential as a string.
  *
  * @see ConstantDataSequential::getAsString()
  */
-const char *LLVMGetAsString(LLVMValueRef c, size_t *Length);
+LLVM_C_ABI const char *LLVMGetAsString(LLVMValueRef c, size_t *Length);
 
 /**
  * Get the raw, underlying bytes of the given constant data sequential.
@@ -2364,16 +2414,18 @@ const char *LLVMGetAsString(LLVMValueRef c, size_t *Length);
  *
  * @see ConstantDataSequential::getRawDataValues()
  */
-const char *LLVMGetRawDataValues(LLVMValueRef c, size_t *SizeInBytes);
+LLVM_C_ABI const char *LLVMGetRawDataValues(LLVMValueRef c,
+                                            size_t *SizeInBytes);
 
 /**
  * Create an anonymous ConstantStruct with the specified values.
  *
  * @see llvm::ConstantStruct::getAnon()
  */
-LLVMValueRef LLVMConstStructInContext(LLVMContextRef C,
-                                      LLVMValueRef *ConstantVals,
-                                      unsigned Count, LLVMBool Packed);
+LLVM_C_ABI LLVMValueRef LLVMConstStructInContext(LLVMContextRef C,
+                                                 LLVMValueRef *ConstantVals,
+                                                 unsigned Count,
+                                                 LLVMBool Packed);
 
 /**
  * Create a ConstantStruct in the global Context.
@@ -2383,8 +2435,8 @@ LLVMValueRef LLVMConstStructInContext(LLVMContextRef C,
  *
  * @see LLVMConstStructInContext()
  */
-LLVMValueRef LLVMConstStruct(LLVMValueRef *ConstantVals, unsigned Count,
-                             LLVMBool Packed);
+LLVM_C_ABI LLVMValueRef LLVMConstStruct(LLVMValueRef *ConstantVals,
+                                        unsigned Count, LLVMBool Packed);
 
 /**
  * Create a ConstantArray from values.
@@ -2393,16 +2445,18 @@ LLVMValueRef LLVMConstStruct(LLVMValueRef *ConstantVals, unsigned Count,
  * LLVMConstArray2
  * @see llvm::ConstantArray::get()
  */
-LLVMValueRef LLVMConstArray(LLVMTypeRef ElementTy,
-                            LLVMValueRef *ConstantVals, unsigned Length);
+LLVM_C_ABI LLVMValueRef LLVMConstArray(LLVMTypeRef ElementTy,
+                                       LLVMValueRef *ConstantVals,
+                                       unsigned Length);
 
 /**
  * Create a ConstantArray from values.
  *
  * @see llvm::ConstantArray::get()
  */
-LLVMValueRef LLVMConstArray2(LLVMTypeRef ElementTy, LLVMValueRef *ConstantVals,
-                             uint64_t Length);
+LLVM_C_ABI LLVMValueRef LLVMConstArray2(LLVMTypeRef ElementTy,
+                                        LLVMValueRef *ConstantVals,
+                                        uint64_t Length);
 
 /**
  * Create a ConstantDataArray from raw values.
@@ -2413,17 +2467,18 @@ LLVMValueRef LLVMConstArray2(LLVMTypeRef ElementTy, LLVMValueRef *ConstantVals,
  *
  * @see llvm::ConstantDataArray::getRaw()
  */
-LLVMValueRef LLVMConstDataArray(LLVMTypeRef ElementTy, const char *Data,
-                                size_t SizeInBytes);
+LLVM_C_ABI LLVMValueRef LLVMConstDataArray(LLVMTypeRef ElementTy,
+                                           const char *Data,
+                                           size_t SizeInBytes);
 
 /**
  * Create a non-anonymous ConstantStruct from values.
  *
  * @see llvm::ConstantStruct::get()
  */
-LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy,
-                                  LLVMValueRef *ConstantVals,
-                                  unsigned Count);
+LLVM_C_ABI LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy,
+                                             LLVMValueRef *ConstantVals,
+                                             unsigned Count);
 
 /**
  * Get element of a constant aggregate (struct, array or vector) at the
@@ -2433,14 +2488,14 @@ LLVMValueRef LLVMConstNamedStruct(LLVMTypeRef StructTy,
  *
  * @see llvm::Constant::getAggregateElement()
  */
-LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx);
+LLVM_C_ABI LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx);
 
 /**
  * Get an element at specified index as a constant.
  *
  * @see ConstantDataSequential::getElementAsConstant()
  */
-LLVM_ATTRIBUTE_C_DEPRECATED(
+LLVM_C_ABI LLVM_ATTRIBUTE_C_DEPRECATED(
     LLVMValueRef LLVMGetElementAsConstant(LLVMValueRef C, unsigned idx),
     "Use LLVMGetAggregateElement instead");
 
@@ -2449,15 +2504,17 @@ LLVM_ATTRIBUTE_C_DEPRECATED(
  *
  * @see llvm::ConstantVector::get()
  */
-LLVMValueRef LLVMConstVector(LLVMValueRef *ScalarConstantVals, unsigned Size);
+LLVM_C_ABI LLVMValueRef LLVMConstVector(LLVMValueRef *ScalarConstantVals,
+                                        unsigned Size);
 
 /**
  * Create a ConstantPtrAuth constant with the given values.
  *
  * @see llvm::ConstantPtrAuth::get()
  */
-LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key,
-                                 LLVMValueRef Disc, LLVMValueRef AddrDisc);
+LLVM_C_ABI LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key,
+                                            LLVMValueRef Disc,
+                                            LLVMValueRef AddrDisc);
 
 /**
  * @}
@@ -2472,71 +2529,86 @@ LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key,
  *
  * @{
  */
-LLVMOpcode LLVMGetConstOpcode(LLVMValueRef ConstantVal);
-LLVMValueRef LLVMAlignOf(LLVMTypeRef Ty);
-LLVMValueRef LLVMSizeOf(LLVMTypeRef Ty);
-LLVMValueRef LLVMConstNeg(LLVMValueRef ConstantVal);
-LLVMValueRef LLVMConstNSWNeg(LLVMValueRef ConstantVal);
-LLVM_ATTRIBUTE_C_DEPRECATED(
+LLVM_C_ABI LLVMOpcode LLVMGetConstOpcode(LLVMValueRef ConstantVal);
+LLVM_C_ABI LLVMValueRef LLVMAlignOf(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMValueRef LLVMSizeOf(LLVMTypeRef Ty);
+LLVM_C_ABI LLVMValueRef LLVMConstNeg(LLVMValueRef ConstantVal);
+LLVM_C_ABI LLVMValueRef LLVMConstNSWNeg(LLVMValueRef ConstantVal);
+LLVM_C_ABI LLVM_ATTRIBUTE_C_DEPRECATED(
     LLVMValueRef LLVMConstNUWNeg(LLVMValueRef ConstantVal),
     "Use LLVMConstNull instead.");
-LLVMValueRef LLVMConstNot(LLVMValueRef ConstantVal);
-LLVMValueRef LLVMConstAdd(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstNSWAdd(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstNUWAdd(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstSub(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstNSWSub(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstNUWSub(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstXor(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
-                           LLVMValueRef *ConstantIndices, unsigned NumIndices);
-LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
-                                   LLVMValueRef *ConstantIndices,
-                                   unsigned NumIndices);
+LLVM_C_ABI LLVMValueRef LLVMConstNot(LLVMValueRef ConstantVal);
+LLVM_C_ABI LLVMValueRef LLVMConstAdd(LLVMValueRef LHSConstant,
+                                     LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstNSWAdd(LLVMValueRef LHSConstant,
+                                        LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstNUWAdd(LLVMValueRef LHSConstant,
+                                        LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstSub(LLVMValueRef LHSConstant,
+                                     LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstNSWSub(LLVMValueRef LHSConstant,
+                                        LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstNUWSub(LLVMValueRef LHSConstant,
+                                        LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstXor(LLVMValueRef LHSConstant,
+                                     LLVMValueRef RHSConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
+                                      LLVMValueRef *ConstantIndices,
+                                      unsigned NumIndices);
+LLVM_C_ABI LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty,
+                                              LLVMValueRef ConstantVal,
+                                              LLVMValueRef *ConstantIndices,
+                                              unsigned NumIndices);
 /**
  * Creates a constant GetElementPtr expression. Similar to LLVMConstGEP2, but
  * allows specifying the no-wrap flags.
  *
  * @see llvm::ConstantExpr::getGetElementPtr()
  */
-LLVMValueRef LLVMConstGEPWithNoWrapFlags(LLVMTypeRef Ty,
-                                         LLVMValueRef ConstantVal,
-                                         LLVMValueRef *ConstantIndices,
-                                         unsigned NumIndices,
-                                         LLVMGEPNoWrapFlags NoWrapFlags);
-LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
-LLVMValueRef LLVMConstPtrToInt(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
-LLVMValueRef LLVMConstIntToPtr(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
-LLVMValueRef LLVMConstBitCast(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
-LLVMValueRef LLVMConstAddrSpaceCast(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
-LLVMValueRef LLVMConstTruncOrBitCast(LLVMValueRef ConstantVal,
-                                     LLVMTypeRef ToType);
-LLVMValueRef LLVMConstPointerCast(LLVMValueRef ConstantVal,
-                                  LLVMTypeRef ToType);
-LLVMValueRef LLVMConstExtractElement(LLVMValueRef VectorConstant,
-                                     LLVMValueRef IndexConstant);
-LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant,
-                                    LLVMValueRef ElementValueConstant,
-                                    LLVMValueRef IndexConstant);
-LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
-                                    LLVMValueRef VectorBConstant,
-                                    LLVMValueRef MaskConstant);
-LLVMValueRef LLVMBlockAddress(LLVMValueRef F, LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMConstGEPWithNoWrapFlags(
+    LLVMTypeRef Ty, LLVMValueRef ConstantVal, LLVMValueRef *ConstantIndices,
+    unsigned NumIndices, LLVMGEPNoWrapFlags NoWrapFlags);
+LLVM_C_ABI LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal,
+                                       LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstPtrToInt(LLVMValueRef ConstantVal,
+                                          LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstIntToPtr(LLVMValueRef ConstantVal,
+                                          LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstBitCast(LLVMValueRef ConstantVal,
+                                         LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstAddrSpaceCast(LLVMValueRef ConstantVal,
+                                               LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstTruncOrBitCast(LLVMValueRef ConstantVal,
+                                                LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstPointerCast(LLVMValueRef ConstantVal,
+                                             LLVMTypeRef ToType);
+LLVM_C_ABI LLVMValueRef LLVMConstExtractElement(LLVMValueRef VectorConstant,
+                                                LLVMValueRef IndexConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstInsertElement(
+    LLVMValueRef VectorConstant, LLVMValueRef ElementValueConstant,
+    LLVMValueRef IndexConstant);
+LLVM_C_ABI LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
+                                               LLVMValueRef VectorBConstant,
+                                               LLVMValueRef MaskConstant);
+LLVM_C_ABI LLVMValueRef LLVMBlockAddress(LLVMValueRef F, LLVMBasicBlockRef BB);
 
 /**
  * Gets the function associated with a given BlockAddress constant value.
  */
-LLVMValueRef LLVMGetBlockAddressFunction(LLVMValueRef BlockAddr);
+LLVM_C_ABI LLVMValueRef LLVMGetBlockAddressFunction(LLVMValueRef BlockAddr);
 
 /**
  * Gets the basic block associated with a given BlockAddress constant value.
  */
-LLVMBasicBlockRef LLVMGetBlockAddressBasicBlock(LLVMValueRef BlockAddr);
+LLVM_C_ABI LLVMBasicBlockRef
+LLVMGetBlockAddressBasicBlock(LLVMValueRef BlockAddr);
 
 /** Deprecated: Use LLVMGetInlineAsm instead. */
-LLVMValueRef LLVMConstInlineAsm(LLVMTypeRef Ty,
-                                const char *AsmString, const char *Constraints,
-                                LLVMBool HasSideEffects, LLVMBool IsAlignStack);
+LLVM_C_ABI LLVMValueRef LLVMConstInlineAsm(LLVMTypeRef Ty,
+                                           const char *AsmString,
+                                           const char *Constraints,
+                                           LLVMBool HasSideEffects,
+                                           LLVMBool IsAlignStack);
 
 /**
  * @}
@@ -2553,18 +2625,20 @@ LLVMValueRef LLVMConstInlineAsm(LLVMTypeRef Ty,
  * @{
  */
 
-LLVMModuleRef LLVMGetGlobalParent(LLVMValueRef Global);
-LLVMBool LLVMIsDeclaration(LLVMValueRef Global);
-LLVMLinkage LLVMGetLinkage(LLVMValueRef Global);
-void LLVMSetLinkage(LLVMValueRef Global, LLVMLinkage Linkage);
-const char *LLVMGetSection(LLVMValueRef Global);
-void LLVMSetSection(LLVMValueRef Global, const char *Section);
-LLVMVisibility LLVMGetVisibility(LLVMValueRef Global);
-void LLVMSetVisibility(LLVMValueRef Global, LLVMVisibility Viz);
-LLVMDLLStorageClass LLVMGetDLLStorageClass(LLVMValueRef Global);
-void LLVMSetDLLStorageClass(LLVMValueRef Global, LLVMDLLStorageClass Class);
-LLVMUnnamedAddr LLVMGetUnnamedAddress(LLVMValueRef Global);
-void LLVMSetUnnamedAddress(LLVMValueRef Global, LLVMUnnamedAddr UnnamedAddr);
+LLVM_C_ABI LLVMModuleRef LLVMGetGlobalParent(LLVMValueRef Global);
+LLVM_C_ABI LLVMBool LLVMIsDeclaration(LLVMValueRef Global);
+LLVM_C_ABI LLVMLinkage LLVMGetLinkage(LLVMValueRef Global);
+LLVM_C_ABI void LLVMSetLinkage(LLVMValueRef Global, LLVMLinkage Linkage);
+LLVM_C_ABI const char *LLVMGetSection(LLVMValueRef Global);
+LLVM_C_ABI void LLVMSetSection(LLVMValueRef Global, const char *Section);
+LLVM_C_ABI LLVMVisibility LLVMGetVisibility(LLVMValueRef Global);
+LLVM_C_ABI void LLVMSetVisibility(LLVMValueRef Global, LLVMVisibility Viz);
+LLVM_C_ABI LLVMDLLStorageClass LLVMGetDLLStorageClass(LLVMValueRef Global);
+LLVM_C_ABI void LLVMSetDLLStorageClass(LLVMValueRef Global,
+                                       LLVMDLLStorageClass Class);
+LLVM_C_ABI LLVMUnnamedAddr LLVMGetUnnamedAddress(LLVMValueRef Global);
+LLVM_C_ABI void LLVMSetUnnamedAddress(LLVMValueRef Global,
+                                      LLVMUnnamedAddr UnnamedAddr);
 
 /**
  * Returns the "value type" of a global value.  This differs from the formal
@@ -2573,12 +2647,13 @@ void LLVMSetUnnamedAddress(LLVMValueRef Global, LLVMUnnamedAddr UnnamedAddr);
  * @see llvm::GlobalValue::getValueType()
  * @see llvm::Function::getFunctionType()
  */
-LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global);
+LLVM_C_ABI LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global);
 
 /** Deprecated: Use LLVMGetUnnamedAddress instead. */
-LLVMBool LLVMHasUnnamedAddr(LLVMValueRef Global);
+LLVM_C_ABI LLVMBool LLVMHasUnnamedAddr(LLVMValueRef Global);
 /** Deprecated: Use LLVMSetUnnamedAddress instead. */
-void LLVMSetUnnamedAddr(LLVMValueRef Global, LLVMBool HasUnnamedAddr);
+LLVM_C_ABI void LLVMSetUnnamedAddr(LLVMValueRef Global,
+                                   LLVMBool HasUnnamedAddr);
 
 /**
  * @defgroup LLVMCCoreValueWithAlignment Values with alignment
@@ -2596,7 +2671,7 @@ void LLVMSetUnnamedAddr(LLVMValueRef Global, LLVMBool HasUnnamedAddr);
  * @see llvm::AtomicCmpXchgInst::setAlignment()
  * @see llvm::GlobalValue::getAlignment()
  */
-unsigned LLVMGetAlignment(LLVMValueRef V);
+LLVM_C_ABI unsigned LLVMGetAlignment(LLVMValueRef V);
 
 /**
  * Set the preferred alignment of the value.
@@ -2607,7 +2682,7 @@ unsigned LLVMGetAlignment(LLVMValueRef V);
  * @see llvm::AtomicCmpXchgInst::setAlignment()
  * @see llvm::GlobalValue::setAlignment()
  */
-void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes);
+LLVM_C_ABI void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes);
 
 /**
  * Sets a metadata attachment, erasing the existing metadata attachment if
@@ -2615,22 +2690,22 @@ void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes);
  *
  * @see llvm::GlobalObject::setMetadata()
  */
-void LLVMGlobalSetMetadata(LLVMValueRef Global, unsigned Kind,
-                           LLVMMetadataRef MD);
+LLVM_C_ABI void LLVMGlobalSetMetadata(LLVMValueRef Global, unsigned Kind,
+                                      LLVMMetadataRef MD);
 
 /**
  * Erases a metadata attachment of the given kind if it exists.
  *
  * @see llvm::GlobalObject::eraseMetadata()
  */
-void LLVMGlobalEraseMetadata(LLVMValueRef Global, unsigned Kind);
+LLVM_C_ABI void LLVMGlobalEraseMetadata(LLVMValueRef Global, unsigned Kind);
 
 /**
  * Removes all metadata attachments from this value.
  *
  * @see llvm::GlobalObject::clearMetadata()
  */
-void LLVMGlobalClearMetadata(LLVMValueRef Global);
+LLVM_C_ABI void LLVMGlobalClearMetadata(LLVMValueRef Global);
 
 /**
  * Retrieves an array of metadata entries representing the metadata attached to
@@ -2639,27 +2714,28 @@ void LLVMGlobalClearMetadata(LLVMValueRef Global);
  *
  * @see llvm::GlobalObject::getAllMetadata()
  */
-LLVMValueMetadataEntry *LLVMGlobalCopyAllMetadata(LLVMValueRef Value,
-                                                  size_t *NumEntries);
+LLVM_C_ABI LLVMValueMetadataEntry *
+LLVMGlobalCopyAllMetadata(LLVMValueRef Value, size_t *NumEntries);
 
 /**
  * Destroys value metadata entries.
  */
-void LLVMDisposeValueMetadataEntries(LLVMValueMetadataEntry *Entries);
+LLVM_C_ABI void
+LLVMDisposeValueMetadataEntries(LLVMValueMetadataEntry *Entries);
 
 /**
  * Returns the kind of a value metadata entry at a specific index.
  */
-unsigned LLVMValueMetadataEntriesGetKind(LLVMValueMetadataEntry *Entries,
-                                         unsigned Index);
+LLVM_C_ABI unsigned
+LLVMValueMetadataEntriesGetKind(LLVMValueMetadataEntry *Entries,
+                                unsigned Index);
 
 /**
  * Returns the underlying metadata node of a value metadata entry at a
  * specific index.
  */
-LLVMMetadataRef
-LLVMValueMetadataEntriesGetMetadata(LLVMValueMetadataEntry *Entries,
-                                    unsigned Index);
+LLVM_C_ABI LLVMMetadataRef LLVMValueMetadataEntriesGetMetadata(
+    LLVMValueMetadataEntry *Entries, unsigned Index);
 
 /**
  * @}
@@ -2674,28 +2750,36 @@ LLVMValueMetadataEntriesGetMetadata(LLVMValueMetadataEntry *Entries,
  *
  * @{
  */
-LLVMValueRef LLVMAddGlobal(LLVMModuleRef M, LLVMTypeRef Ty, const char *Name);
-LLVMValueRef LLVMAddGlobalInAddressSpace(LLVMModuleRef M, LLVMTypeRef Ty,
-                                         const char *Name,
-                                         unsigned AddressSpace);
-LLVMValueRef LLVMGetNamedGlobal(LLVMModuleRef M, const char *Name);
-LLVMValueRef LLVMGetNamedGlobalWithLength(LLVMModuleRef M, const char *Name,
-                                          size_t Length);
-LLVMValueRef LLVMGetFirstGlobal(LLVMModuleRef M);
-LLVMValueRef LLVMGetLastGlobal(LLVMModuleRef M);
-LLVMValueRef LLVMGetNextGlobal(LLVMValueRef GlobalVar);
-LLVMValueRef LLVMGetPreviousGlobal(LLVMValueRef GlobalVar);
-void LLVMDeleteGlobal(LLVMValueRef GlobalVar);
-LLVMValueRef LLVMGetInitializer(LLVMValueRef GlobalVar);
-void LLVMSetInitializer(LLVMValueRef GlobalVar, LLVMValueRef ConstantVal);
-LLVMBool LLVMIsThreadLocal(LLVMValueRef GlobalVar);
-void LLVMSetThreadLocal(LLVMValueRef GlobalVar, LLVMBool IsThreadLocal);
-LLVMBool LLVMIsGlobalConstant(LLVMValueRef GlobalVar);
-void LLVMSetGlobalConstant(LLVMValueRef GlobalVar, LLVMBool IsConstant);
-LLVMThreadLocalMode LLVMGetThreadLocalMode(LLVMValueRef GlobalVar);
-void LLVMSetThreadLocalMode(LLVMValueRef GlobalVar, LLVMThreadLocalMode Mode);
-LLVMBool LLVMIsExternallyInitialized(LLVMValueRef GlobalVar);
-void LLVMSetExternallyInitialized(LLVMValueRef GlobalVar, LLVMBool IsExtInit);
+LLVM_C_ABI LLVMValueRef LLVMAddGlobal(LLVMModuleRef M, LLVMTypeRef Ty,
+                                      const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMAddGlobalInAddressSpace(LLVMModuleRef M,
+                                                    LLVMTypeRef Ty,
+                                                    const char *Name,
+                                                    unsigned AddressSpace);
+LLVM_C_ABI LLVMValueRef LLVMGetNamedGlobal(LLVMModuleRef M, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMGetNamedGlobalWithLength(LLVMModuleRef M,
+                                                     const char *Name,
+                                                     size_t Length);
+LLVM_C_ABI LLVMValueRef LLVMGetFirstGlobal(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetLastGlobal(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetNextGlobal(LLVMValueRef GlobalVar);
+LLVM_C_ABI LLVMValueRef LLVMGetPreviousGlobal(LLVMValueRef GlobalVar);
+LLVM_C_ABI void LLVMDeleteGlobal(LLVMValueRef GlobalVar);
+LLVM_C_ABI LLVMValueRef LLVMGetInitializer(LLVMValueRef GlobalVar);
+LLVM_C_ABI void LLVMSetInitializer(LLVMValueRef GlobalVar,
+                                   LLVMValueRef ConstantVal);
+LLVM_C_ABI LLVMBool LLVMIsThreadLocal(LLVMValueRef GlobalVar);
+LLVM_C_ABI void LLVMSetThreadLocal(LLVMValueRef GlobalVar,
+                                   LLVMBool IsThreadLocal);
+LLVM_C_ABI LLVMBool LLVMIsGlobalConstant(LLVMValueRef GlobalVar);
+LLVM_C_ABI void LLVMSetGlobalConstant(LLVMValueRef GlobalVar,
+                                      LLVMBool IsConstant);
+LLVM_C_ABI LLVMThreadLocalMode LLVMGetThreadLocalMode(LLVMValueRef GlobalVar);
+LLVM_C_ABI void LLVMSetThreadLocalMode(LLVMValueRef GlobalVar,
+                                       LLVMThreadLocalMode Mode);
+LLVM_C_ABI LLVMBool LLVMIsExternallyInitialized(LLVMValueRef GlobalVar);
+LLVM_C_ABI void LLVMSetExternallyInitialized(LLVMValueRef GlobalVar,
+                                             LLVMBool IsExtInit);
 
 /**
  * @}
@@ -2716,9 +2800,9 @@ void LLVMSetExternallyInitialized(LLVMValueRef GlobalVar, LLVMBool IsExtInit);
  *
  * @see llvm::GlobalAlias::create()
  */
-LLVMValueRef LLVMAddAlias2(LLVMModuleRef M, LLVMTypeRef ValueTy,
-                           unsigned AddrSpace, LLVMValueRef Aliasee,
-                           const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMAddAlias2(LLVMModuleRef M, LLVMTypeRef ValueTy,
+                                      unsigned AddrSpace, LLVMValueRef Aliasee,
+                                      const char *Name);
 
 /**
  * Obtain a GlobalAlias value from a Module by its name.
@@ -2727,22 +2811,23 @@ LLVMValueRef LLVMAddAlias2(LLVMModuleRef M, LLVMTypeRef ValueTy,
  *
  * @see llvm::Module::getNamedAlias()
  */
-LLVMValueRef LLVMGetNamedGlobalAlias(LLVMModuleRef M,
-                                     const char *Name, size_t NameLen);
+LLVM_C_ABI LLVMValueRef LLVMGetNamedGlobalAlias(LLVMModuleRef M,
+                                                const char *Name,
+                                                size_t NameLen);
 
 /**
  * Obtain an iterator to the first GlobalAlias in a Module.
  *
  * @see llvm::Module::alias_begin()
  */
-LLVMValueRef LLVMGetFirstGlobalAlias(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetFirstGlobalAlias(LLVMModuleRef M);
 
 /**
  * Obtain an iterator to the last GlobalAlias in a Module.
  *
  * @see llvm::Module::alias_end()
  */
-LLVMValueRef LLVMGetLastGlobalAlias(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetLastGlobalAlias(LLVMModuleRef M);
 
 /**
  * Advance a GlobalAlias iterator to the next GlobalAlias.
@@ -2750,7 +2835,7 @@ LLVMValueRef LLVMGetLastGlobalAlias(LLVMModuleRef M);
  * Returns NULL if the iterator was already at the end and there are no more
  * global aliases.
  */
-LLVMValueRef LLVMGetNextGlobalAlias(LLVMValueRef GA);
+LLVM_C_ABI LLVMValueRef LLVMGetNextGlobalAlias(LLVMValueRef GA);
 
 /**
  * Decrement a GlobalAlias iterator to the previous GlobalAlias.
@@ -2758,17 +2843,17 @@ LLVMValueRef LLVMGetNextGlobalAlias(LLVMValueRef GA);
  * Returns NULL if the iterator was already at the beginning and there are
  * no previous global aliases.
  */
-LLVMValueRef LLVMGetPreviousGlobalAlias(LLVMValueRef GA);
+LLVM_C_ABI LLVMValueRef LLVMGetPreviousGlobalAlias(LLVMValueRef GA);
 
 /**
  * Retrieve the target value of an alias.
  */
-LLVMValueRef LLVMAliasGetAliasee(LLVMValueRef Alias);
+LLVM_C_ABI LLVMValueRef LLVMAliasGetAliasee(LLVMValueRef Alias);
 
 /**
  * Set the target value of an alias.
  */
-void LLVMAliasSetAliasee(LLVMValueRef Alias, LLVMValueRef Aliasee);
+LLVM_C_ABI void LLVMAliasSetAliasee(LLVMValueRef Alias, LLVMValueRef Aliasee);
 
 /**
  * @}
@@ -2790,42 +2875,43 @@ void LLVMAliasSetAliasee(LLVMValueRef Alias, LLVMValueRef Aliasee);
  *
  * @see llvm::Function::eraseFromParent()
  */
-void LLVMDeleteFunction(LLVMValueRef Fn);
+LLVM_C_ABI void LLVMDeleteFunction(LLVMValueRef Fn);
 
 /**
  * Check whether the given function has a personality function.
  *
  * @see llvm::Function::hasPersonalityFn()
  */
-LLVMBool LLVMHasPersonalityFn(LLVMValueRef Fn);
+LLVM_C_ABI LLVMBool LLVMHasPersonalityFn(LLVMValueRef Fn);
 
 /**
  * Obtain the personality function attached to the function.
  *
  * @see llvm::Function::getPersonalityFn()
  */
-LLVMValueRef LLVMGetPersonalityFn(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetPersonalityFn(LLVMValueRef Fn);
 
 /**
  * Set the personality function attached to the function.
  *
  * @see llvm::Function::setPersonalityFn()
  */
-void LLVMSetPersonalityFn(LLVMValueRef Fn, LLVMValueRef PersonalityFn);
+LLVM_C_ABI void LLVMSetPersonalityFn(LLVMValueRef Fn,
+                                     LLVMValueRef PersonalityFn);
 
 /**
  * Obtain the intrinsic ID number which matches the given function name.
  *
  * @see llvm::Intrinsic::lookupIntrinsicID()
  */
-unsigned LLVMLookupIntrinsicID(const char *Name, size_t NameLen);
+LLVM_C_ABI unsigned LLVMLookupIntrinsicID(const char *Name, size_t NameLen);
 
 /**
  * Obtain the ID number from a function instance.
  *
  * @see llvm::Function::getIntrinsicID()
  */
-unsigned LLVMGetIntrinsicID(LLVMValueRef Fn);
+LLVM_C_ABI unsigned LLVMGetIntrinsicID(LLVMValueRef Fn);
 
 /**
  * Get or insert the declaration of an intrinsic.  For overloaded intrinsics,
@@ -2833,10 +2919,10 @@ unsigned LLVMGetIntrinsicID(LLVMValueRef Fn);
  *
  * @see llvm::Intrinsic::getOrInsertDeclaration()
  */
-LLVMValueRef LLVMGetIntrinsicDeclaration(LLVMModuleRef Mod,
-                                         unsigned ID,
-                                         LLVMTypeRef *ParamTypes,
-                                         size_t ParamCount);
+LLVM_C_ABI LLVMValueRef LLVMGetIntrinsicDeclaration(LLVMModuleRef Mod,
+                                                    unsigned ID,
+                                                    LLVMTypeRef *ParamTypes,
+                                                    size_t ParamCount);
 
 /**
  * Retrieves the type of an intrinsic.  For overloaded intrinsics, parameter
@@ -2844,19 +2930,22 @@ LLVMValueRef LLVMGetIntrinsicDeclaration(LLVMModuleRef Mod,
  *
  * @see llvm::Intrinsic::getType()
  */
-LLVMTypeRef LLVMIntrinsicGetType(LLVMContextRef Ctx, unsigned ID,
-                                 LLVMTypeRef *ParamTypes, size_t ParamCount);
+LLVM_C_ABI LLVMTypeRef LLVMIntrinsicGetType(LLVMContextRef Ctx, unsigned ID,
+                                            LLVMTypeRef *ParamTypes,
+                                            size_t ParamCount);
 
 /**
  * Retrieves the name of an intrinsic.
  *
  * @see llvm::Intrinsic::getName()
  */
-const char *LLVMIntrinsicGetName(unsigned ID, size_t *NameLength);
+LLVM_C_ABI const char *LLVMIntrinsicGetName(unsigned ID, size_t *NameLength);
 
 /** Deprecated: Use LLVMIntrinsicCopyOverloadedName2 instead. */
-char *LLVMIntrinsicCopyOverloadedName(unsigned ID, LLVMTypeRef *ParamTypes,
-                                      size_t ParamCount, size_t *NameLength);
+LLVM_C_ABI char *LLVMIntrinsicCopyOverloadedName(unsigned ID,
+                                                 LLVMTypeRef *ParamTypes,
+                                                 size_t ParamCount,
+                                                 size_t *NameLength);
 
 /**
  * Copies the name of an overloaded intrinsic identified by a given list of
@@ -2869,16 +2958,18 @@ char *LLVMIntrinsicCopyOverloadedName(unsigned ID, LLVMTypeRef *ParamTypes,
  *
  * @see llvm::Intrinsic::getName()
  */
-char *LLVMIntrinsicCopyOverloadedName2(LLVMModuleRef Mod, unsigned ID,
-                                       LLVMTypeRef *ParamTypes,
-                                       size_t ParamCount, size_t *NameLength);
+LLVM_C_ABI char *LLVMIntrinsicCopyOverloadedName2(LLVMModuleRef Mod,
+                                                  unsigned ID,
+                                                  LLVMTypeRef *ParamTypes,
+                                                  size_t ParamCount,
+                                                  size_t *NameLength);
 
 /**
  * Obtain if the intrinsic identified by the given ID is overloaded.
  *
  * @see llvm::Intrinsic::isOverloaded()
  */
-LLVMBool LLVMIntrinsicIsOverloaded(unsigned ID);
+LLVM_C_ABI LLVMBool LLVMIntrinsicIsOverloaded(unsigned ID);
 
 /**
  * Obtain the calling function of a function.
@@ -2887,7 +2978,7 @@ LLVMBool LLVMIntrinsicIsOverloaded(unsigned ID);
  *
  * @see llvm::Function::getCallingConv()
  */
-unsigned LLVMGetFunctionCallConv(LLVMValueRef Fn);
+LLVM_C_ABI unsigned LLVMGetFunctionCallConv(LLVMValueRef Fn);
 
 /**
  * Set the calling convention of a function.
@@ -2897,7 +2988,7 @@ unsigned LLVMGetFunctionCallConv(LLVMValueRef Fn);
  * @param Fn Function to operate on
  * @param CC LLVMCallConv to set calling convention to
  */
-void LLVMSetFunctionCallConv(LLVMValueRef Fn, unsigned CC);
+LLVM_C_ABI void LLVMSetFunctionCallConv(LLVMValueRef Fn, unsigned CC);
 
 /**
  * Obtain the name of the garbage collector to use during code
@@ -2905,80 +2996,83 @@ void LLVMSetFunctionCallConv(LLVMValueRef Fn, unsigned CC);
  *
  * @see llvm::Function::getGC()
  */
-const char *LLVMGetGC(LLVMValueRef Fn);
+LLVM_C_ABI const char *LLVMGetGC(LLVMValueRef Fn);
 
 /**
  * Define the garbage collector to use during code generation.
  *
  * @see llvm::Function::setGC()
  */
-void LLVMSetGC(LLVMValueRef Fn, const char *Name);
+LLVM_C_ABI void LLVMSetGC(LLVMValueRef Fn, const char *Name);
 
 /**
  * Gets the prefix data associated with a function. Only valid on functions, and
  * only if LLVMHasPrefixData returns true.
  * See https://llvm.org/docs/LangRef.html#prefix-data
  */
-LLVMValueRef LLVMGetPrefixData(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetPrefixData(LLVMValueRef Fn);
 
 /**
  * Check if a given function has prefix data. Only valid on functions.
  * See https://llvm.org/docs/LangRef.html#prefix-data
  */
-LLVMBool LLVMHasPrefixData(LLVMValueRef Fn);
+LLVM_C_ABI LLVMBool LLVMHasPrefixData(LLVMValueRef Fn);
 
 /**
  * Sets the prefix data for the function. Only valid on functions.
  * See https://llvm.org/docs/LangRef.html#prefix-data
  */
-void LLVMSetPrefixData(LLVMValueRef Fn, LLVMValueRef prefixData);
+LLVM_C_ABI void LLVMSetPrefixData(LLVMValueRef Fn, LLVMValueRef prefixData);
 
 /**
  * Gets the prologue data associated with a function. Only valid on functions,
  * and only if LLVMHasPrologueData returns true.
  * See https://llvm.org/docs/LangRef.html#prologue-data
  */
-LLVMValueRef LLVMGetPrologueData(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetPrologueData(LLVMValueRef Fn);
 
 /**
  * Check if a given function has prologue data. Only valid on functions.
  * See https://llvm.org/docs/LangRef.html#prologue-data
  */
-LLVMBool LLVMHasPrologueData(LLVMValueRef Fn);
+LLVM_C_ABI LLVMBool LLVMHasPrologueData(LLVMValueRef Fn);
 
 /**
  * Sets the prologue data for the function. Only valid on functions.
  * See https://llvm.org/docs/LangRef.html#prologue-data
  */
-void LLVMSetPrologueData(LLVMValueRef Fn, LLVMValueRef prologueData);
+LLVM_C_ABI void LLVMSetPrologueData(LLVMValueRef Fn, LLVMValueRef prologueData);
 
 /**
  * Add an attribute to a function.
  *
  * @see llvm::Function::addAttribute()
  */
-void LLVMAddAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx,
-                             LLVMAttributeRef A);
-unsigned LLVMGetAttributeCountAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx);
-void LLVMGetAttributesAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx,
-                              LLVMAttributeRef *Attrs);
-LLVMAttributeRef LLVMGetEnumAttributeAtIndex(LLVMValueRef F,
-                                             LLVMAttributeIndex Idx,
-                                             unsigned KindID);
-LLVMAttributeRef LLVMGetStringAttributeAtIndex(LLVMValueRef F,
+LLVM_C_ABI void LLVMAddAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx,
+                                        LLVMAttributeRef A);
+LLVM_C_ABI unsigned LLVMGetAttributeCountAtIndex(LLVMValueRef F,
+                                                 LLVMAttributeIndex Idx);
+LLVM_C_ABI void LLVMGetAttributesAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx,
+                                         LLVMAttributeRef *Attrs);
+LLVM_C_ABI LLVMAttributeRef LLVMGetEnumAttributeAtIndex(LLVMValueRef F,
+                                                        LLVMAttributeIndex Idx,
+                                                        unsigned KindID);
+LLVM_C_ABI LLVMAttributeRef LLVMGetStringAttributeAtIndex(
+    LLVMValueRef F, LLVMAttributeIndex Idx, const char *K, unsigned KLen);
+LLVM_C_ABI void LLVMRemoveEnumAttributeAtIndex(LLVMValueRef F,
                                                LLVMAttributeIndex Idx,
-                                               const char *K, unsigned KLen);
-void LLVMRemoveEnumAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx,
-                                    unsigned KindID);
-void LLVMRemoveStringAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx,
-                                      const char *K, unsigned KLen);
+                                               unsigned KindID);
+LLVM_C_ABI void LLVMRemoveStringAttributeAtIndex(LLVMValueRef F,
+                                                 LLVMAttributeIndex Idx,
+                                                 const char *K, unsigned KLen);
 
 /**
  * Add a target-dependent attribute to a function
  * @see llvm::AttrBuilder::addAttribute()
  */
-void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn, const char *A,
-                                        const char *V);
+LLVM_C_ABI void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn,
+                                                   const char *A,
+                                                   const char *V);
 
 /**
  * @defgroup LLVMCCoreValueFunctionParameters Function Parameters
@@ -2996,7 +3090,7 @@ void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn, const char *A,
  *
  * @see llvm::Function::arg_size()
  */
-unsigned LLVMCountParams(LLVMValueRef Fn);
+LLVM_C_ABI unsigned LLVMCountParams(LLVMValueRef Fn);
 
 /**
  * Obtain the parameters in a function.
@@ -3009,7 +3103,7 @@ unsigned LLVMCountParams(LLVMValueRef Fn);
  *
  * @see llvm::Function::arg_begin()
  */
-void LLVMGetParams(LLVMValueRef Fn, LLVMValueRef *Params);
+LLVM_C_ABI void LLVMGetParams(LLVMValueRef Fn, LLVMValueRef *Params);
 
 /**
  * Obtain the parameter at the specified index.
@@ -3018,7 +3112,7 @@ void LLVMGetParams(LLVMValueRef Fn, LLVMValueRef *Params);
  *
  * @see llvm::Function::arg_begin()
  */
-LLVMValueRef LLVMGetParam(LLVMValueRef Fn, unsigned Index);
+LLVM_C_ABI LLVMValueRef LLVMGetParam(LLVMValueRef Fn, unsigned Index);
 
 /**
  * Obtain the function to which this argument belongs.
@@ -3029,21 +3123,21 @@ LLVMValueRef LLVMGetParam(LLVMValueRef Fn, unsigned Index);
  * The returned LLVMValueRef is the llvm::Function to which this
  * argument belongs.
  */
-LLVMValueRef LLVMGetParamParent(LLVMValueRef Inst);
+LLVM_C_ABI LLVMValueRef LLVMGetParamParent(LLVMValueRef Inst);
 
 /**
  * Obtain the first parameter to a function.
  *
  * @see llvm::Function::arg_begin()
  */
-LLVMValueRef LLVMGetFirstParam(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetFirstParam(LLVMValueRef Fn);
 
 /**
  * Obtain the last parameter to a function.
  *
  * @see llvm::Function::arg_end()
  */
-LLVMValueRef LLVMGetLastParam(LLVMValueRef Fn);
+LLVM_C_ABI LLVMValueRef LLVMGetLastParam(LLVMValueRef Fn);
 
 /**
  * Obtain the next parameter to a function.
@@ -3052,14 +3146,14 @@ LLVMValueRef LLVMGetLastParam(LLVMValueRef Fn);
  * actually a wrapped iterator) and obtains the next parameter from the
  * underlying iterator.
  */
-LLVMValueRef LLVMGetNextParam(LLVMValueRef Arg);
+LLVM_C_ABI LLVMValueRef LLVMGetNextParam(LLVMValueRef Arg);
 
 /**
  * Obtain the previous parameter to a function.
  *
  * This is the opposite of LLVMGetNextParam().
  */
-LLVMValueRef LLVMGetPreviousParam(LLVMValueRef Arg);
+LLVM_C_ABI LLVMValueRef LLVMGetPreviousParam(LLVMValueRef Arg);
 
 /**
  * Set the alignment for a function parameter.
@@ -3067,7 +3161,7 @@ LLVMValueRef LLVMGetPreviousParam(LLVMValueRef Arg);
  * @see llvm::Argument::addAttr()
  * @see llvm::AttrBuilder::addAlignmentAttr()
  */
-void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned Align);
+LLVM_C_ABI void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned Align);
 
 /**
  * @}
@@ -3089,10 +3183,10 @@ void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned Align);
  *
  * @see llvm::GlobalIFunc::create()
  */
-LLVMValueRef LLVMAddGlobalIFunc(LLVMModuleRef M,
-                                const char *Name, size_t NameLen,
-                                LLVMTypeRef Ty, unsigned AddrSpace,
-                                LLVMValueRef Resolver);
+LLVM_C_ABI LLVMValueRef LLVMAddGlobalIFunc(LLVMModuleRef M, const char *Name,
+                                           size_t NameLen, LLVMTypeRef Ty,
+                                           unsigned AddrSpace,
+                                           LLVMValueRef Resolver);
 
 /**
  * Obtain a GlobalIFunc value from a Module by its name.
@@ -3101,22 +3195,23 @@ LLVMValueRef LLVMAddGlobalIFunc(LLVMModuleRef M,
  *
  * @see llvm::Module::getNamedIFunc()
  */
-LLVMValueRef LLVMGetNamedGlobalIFunc(LLVMModuleRef M,
-                                     const char *Name, size_t NameLen);
+LLVM_C_ABI LLVMValueRef LLVMGetNamedGlobalIFunc(LLVMModuleRef M,
+                                                const char *Name,
+                                                size_t NameLen);
 
 /**
  * Obtain an iterator to the first GlobalIFunc in a Module.
  *
  * @see llvm::Module::ifunc_begin()
  */
-LLVMValueRef LLVMGetFirstGlobalIFunc(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetFirstGlobalIFunc(LLVMModuleRef M);
 
 /**
  * Obtain an iterator to the last GlobalIFunc in a Module.
  *
  * @see llvm::Module::ifunc_end()
  */
-LLVMValueRef LLVMGetLastGlobalIFunc(LLVMModuleRef M);
+LLVM_C_ABI LLVMValueRef LLVMGetLastGlobalIFunc(LLVMModuleRef M);
 
 /**
  * Advance a GlobalIFunc iterator to the next GlobalIFunc.
@@ -3124,7 +3219,7 @@ LLVMValueRef LLVMGetLastGlobalIFunc(LLVMModuleRef M);
  * Returns NULL if the iterator was already at the end and there are no more
  * global aliases.
  */
-LLVMValueRef LLVMGetNextGlobalIFunc(LLVMValueRef IFunc);
+LLVM_C_ABI LLVMValueRef LLVMGetNextGlobalIFunc(LLVMValueRef IFunc);
 
 /**
  * Decrement a GlobalIFunc iterator to the previous GlobalIFunc.
@@ -3132,7 +3227,7 @@ LLVMValueRef LLVMGetNextGlobalIFunc(LLVMValueRef IFunc);
  * Returns NULL if the iterator was already at the beginning and there are
  * no previous global aliases.
  */
-LLVMValueRef LLVMGetPreviousGlobalIFunc(LLVMValueRef IFunc);
+LLVM_C_ABI LLVMValueRef LLVMGetPreviousGlobalIFunc(LLVMValueRef IFunc);
 
 /**
  * Retrieves the resolver function associated with this indirect function, or
@@ -3140,21 +3235,22 @@ LLVMValueRef LLVMGetPreviousGlobalIFunc(LLVMValueRef IFunc);
  *
  * @see llvm::GlobalIFunc::getResolver()
  */
-LLVMValueRef LLVMGetGlobalIFuncResolver(LLVMValueRef IFunc);
+LLVM_C_ABI LLVMValueRef LLVMGetGlobalIFuncResolver(LLVMValueRef IFunc);
 
 /**
  * Sets the resolver function associated with this indirect function.
  *
  * @see llvm::GlobalIFunc::setResolver()
  */
-void LLVMSetGlobalIFuncResolver(LLVMValueRef IFunc, LLVMValueRef Resolver);
+LLVM_C_ABI void LLVMSetGlobalIFuncResolver(LLVMValueRef IFunc,
+                                           LLVMValueRef Resolver);
 
 /**
  * Remove a global indirect function from its parent module and delete it.
  *
  * @see llvm::GlobalIFunc::eraseFromParent()
  */
-void LLVMEraseGlobalIFunc(LLVMValueRef IFunc);
+LLVM_C_ABI void LLVMEraseGlobalIFunc(LLVMValueRef IFunc);
 
 /**
  * Remove a global indirect function from its parent module.
@@ -3164,7 +3260,7 @@ void LLVMEraseGlobalIFunc(LLVMValueRef IFunc);
  *
  * @see llvm::GlobalIFunc::removeFromParent()
  */
-void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc);
+LLVM_C_ABI void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc);
 
 /**
  * @}
@@ -3196,26 +3292,28 @@ void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc);
  *
  * @see llvm::MDString::get()
  */
-LLVMMetadataRef LLVMMDStringInContext2(LLVMContextRef C, const char *Str,
-                                       size_t SLen);
+LLVM_C_ABI LLVMMetadataRef LLVMMDStringInContext2(LLVMContextRef C,
+                                                  const char *Str, size_t SLen);
 
 /**
  * Create an MDNode value with the given array of operands.
  *
  * @see llvm::MDNode::get()
  */
-LLVMMetadataRef LLVMMDNodeInContext2(LLVMContextRef C, LLVMMetadataRef *MDs,
-                                     size_t Count);
+LLVM_C_ABI LLVMMetadataRef LLVMMDNodeInContext2(LLVMContextRef C,
+                                                LLVMMetadataRef *MDs,
+                                                size_t Count);
 
 /**
  * Obtain a Metadata as a Value.
  */
-LLVMValueRef LLVMMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD);
+LLVM_C_ABI LLVMValueRef LLVMMetadataAsValue(LLVMContextRef C,
+                                            LLVMMetadataRef MD);
 
 /**
  * Obtain a Value as a Metadata.
  */
-LLVMMetadataRef LLVMValueAsMetadata(LLVMValueRef Val);
+LLVM_C_ABI LLVMMetadataRef LLVMValueAsMetadata(LLVMValueRef Val);
 
 /**
  * Obtain the underlying string from a MDString value.
@@ -3224,7 +3322,7 @@ LLVMMetadataRef LLVMValueAsMetadata(LLVMValueRef Val);
  * @param Length Memory address which will hold length of returned string.
  * @return String data in MDString.
  */
-const char *LLVMGetMDString(LLVMValueRef V, unsigned *Length);
+LLVM_C_ABI const char *LLVMGetMDString(LLVMValueRef V, unsigned *Length);
 
 /**
  * Obtain the number of operands from an MDNode value.
@@ -3232,7 +3330,7 @@ const char *LLVMGetMDString(LLVMValueRef V, unsigned *Length);
  * @param V MDNode to get number of operands from.
  * @return Number of operands of the MDNode.
  */
-unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V);
+LLVM_C_ABI unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V);
 
 /**
  * Obtain the given MDNode's operands.
@@ -3245,26 +3343,26 @@ unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V);
  * @param V MDNode to get the operands from.
  * @param Dest Destination array for operands.
  */
-void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest);
+LLVM_C_ABI void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest);
 
 /**
  * Replace an operand at a specific index in a llvm::MDNode value.
  *
  * @see llvm::MDNode::replaceOperandWith()
  */
-void LLVMReplaceMDNodeOperandWith(LLVMValueRef V, unsigned Index,
-                                  LLVMMetadataRef Replacement);
+LLVM_C_ABI void LLVMReplaceMDNodeOperandWith(LLVMValueRef V, unsigned Index,
+                                             LLVMMetadataRef Replacement);
 
 /** Deprecated: Use LLVMMDStringInContext2 instead. */
-LLVMValueRef LLVMMDStringInContext(LLVMContextRef C, const char *Str,
-                                   unsigned SLen);
+LLVM_C_ABI LLVMValueRef LLVMMDStringInContext(LLVMContextRef C, const char *Str,
+                                              unsigned SLen);
 /** Deprecated: Use LLVMMDStringInContext2 instead. */
-LLVMValueRef LLVMMDString(const char *Str, unsigned SLen);
+LLVM_C_ABI LLVMValueRef LLVMMDString(const char *Str, unsigned SLen);
 /** Deprecated: Use LLVMMDNodeInContext2 instead. */
-LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals,
-                                 unsigned Count);
+LLVM_C_ABI LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C,
+                                            LLVMValueRef *Vals, unsigned Count);
 /** Deprecated: Use LLVMMDNodeInContext2 instead. */
-LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count);
+LLVM_C_ABI LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count);
 
 /**
  * @}
@@ -3292,9 +3390,10 @@ LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count);
  * @param Args Memory address of an array of bundle operands
  * @param NumArgs Length of Args
  */
-LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag, size_t TagLen,
-                                             LLVMValueRef *Args,
-                                             unsigned NumArgs);
+LLVM_C_ABI LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag,
+                                                        size_t TagLen,
+                                                        LLVMValueRef *Args,
+                                                        unsigned NumArgs);
 
 /**
  * Destroy an operand bundle.
@@ -3302,7 +3401,7 @@ LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag, size_t TagLen,
  * This must be called for every created operand bundle or memory will be
  * leaked.
  */
-void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle);
+LLVM_C_ABI void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle);
 
 /**
  * Obtain the tag of an operand bundle as a string.
@@ -3312,7 +3411,8 @@ void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle);
  * @return The tag name of Bundle.
  * @see OperandBundleDef::getTag()
  */
-const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle, size_t *Len);
+LLVM_C_ABI const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle,
+                                               size_t *Len);
 
 /**
  * Obtain the number of operands for an operand bundle.
@@ -3321,7 +3421,7 @@ const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle, size_t *Len);
  * @return The number of operands.
  * @see OperandBundleDef::input_size()
  */
-unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle);
+LLVM_C_ABI unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle);
 
 /**
  * Obtain the operand for an operand bundle at the given index.
@@ -3331,8 +3431,8 @@ unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle);
  * LLVMGetNumOperandBundleArgs().
  * @return The operand.
  */
-LLVMValueRef LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle,
-                                            unsigned Index);
+LLVM_C_ABI LLVMValueRef
+LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle, unsigned Index);
 
 /**
  * @}
@@ -3358,29 +3458,29 @@ LLVMValueRef LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle,
 /**
  * Convert a basic block instance to a value type.
  */
-LLVMValueRef LLVMBasicBlockAsValue(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMBasicBlockAsValue(LLVMBasicBlockRef BB);
 
 /**
  * Determine whether an LLVMValueRef is itself a basic block.
  */
-LLVMBool LLVMValueIsBasicBlock(LLVMValueRef Val);
+LLVM_C_ABI LLVMBool LLVMValueIsBasicBlock(LLVMValueRef Val);
 
 /**
  * Convert an LLVMValueRef to an LLVMBasicBlockRef instance.
  */
-LLVMBasicBlockRef LLVMValueAsBasicBlock(LLVMValueRef Val);
+LLVM_C_ABI LLVMBasicBlockRef LLVMValueAsBasicBlock(LLVMValueRef Val);
 
 /**
  * Obtain the string name of a basic block.
  */
-const char *LLVMGetBasicBlockName(LLVMBasicBlockRef BB);
+LLVM_C_ABI const char *LLVMGetBasicBlockName(LLVMBasicBlockRef BB);
 
 /**
  * Obtain the function to which a basic block belongs.
  *
  * @see llvm::BasicBlock::getParent()
  */
-LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB);
 
 /**
  * Obtain the terminator instruction for a basic block.
@@ -3392,14 +3492,14 @@ LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB);
  *
  * @see llvm::BasicBlock::getTerminator()
  */
-LLVMValueRef LLVMGetBasicBlockTerminator(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMGetBasicBlockTerminator(LLVMBasicBlockRef BB);
 
 /**
  * Obtain the number of basic blocks in a function.
  *
  * @param Fn Function value to operate on.
  */
-unsigned LLVMCountBasicBlocks(LLVMValueRef Fn);
+LLVM_C_ABI unsigned LLVMCountBasicBlocks(LLVMValueRef Fn);
 
 /**
  * Obtain all of the basic blocks in a function.
@@ -3409,7 +3509,8 @@ unsigned LLVMCountBasicBlocks(LLVMValueRef Fn);
  * LLVMCountBasicBlocks() in length. This array is populated with
  * LLVMBasicBlockRef instances.
  */
-void LLVMGetBasicBlocks(LLVMValueRef Fn, LLVMBasicBlockRef *BasicBlocks);
+LLVM_C_ABI void LLVMGetBasicBlocks(LLVMValueRef Fn,
+                                   LLVMBasicBlockRef *BasicBlocks);
 
 /**
  * Obtain the first basic block in a function.
@@ -3419,24 +3520,24 @@ void LLVMGetBasicBlocks(LLVMValueRef Fn, LLVMBasicBlockRef *BasicBlocks);
  *
  * @see llvm::Function::begin()
  */
-LLVMBasicBlockRef LLVMGetFirstBasicBlock(LLVMValueRef Fn);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetFirstBasicBlock(LLVMValueRef Fn);
 
 /**
  * Obtain the last basic block in a function.
  *
  * @see llvm::Function::end()
  */
-LLVMBasicBlockRef LLVMGetLastBasicBlock(LLVMValueRef Fn);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetLastBasicBlock(LLVMValueRef Fn);
 
 /**
  * Advance a basic block iterator.
  */
-LLVMBasicBlockRef LLVMGetNextBasicBlock(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetNextBasicBlock(LLVMBasicBlockRef BB);
 
 /**
  * Go backwards in a basic block iterator.
  */
-LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB);
 
 /**
  * Obtain the basic block that corresponds to the entry point of a
@@ -3444,7 +3545,7 @@ LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB);
  *
  * @see llvm::Function::getEntryBlock()
  */
-LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn);
 
 /**
  * Insert the given basic block after the insertion point of the given builder.
@@ -3453,33 +3554,34 @@ LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn);
  *
  * @see llvm::Function::BasicBlockListType::insertAfter()
  */
-void LLVMInsertExistingBasicBlockAfterInsertBlock(LLVMBuilderRef Builder,
-                                                  LLVMBasicBlockRef BB);
+LLVM_C_ABI void
+LLVMInsertExistingBasicBlockAfterInsertBlock(LLVMBuilderRef Builder,
+                                             LLVMBasicBlockRef BB);
 
 /**
  * Append the given basic block to the basic block list of the given function.
  *
  * @see llvm::Function::BasicBlockListType::push_back()
  */
-void LLVMAppendExistingBasicBlock(LLVMValueRef Fn,
-                                  LLVMBasicBlockRef BB);
+LLVM_C_ABI void LLVMAppendExistingBasicBlock(LLVMValueRef Fn,
+                                             LLVMBasicBlockRef BB);
 
 /**
  * Create a new basic block without inserting it into a function.
  *
  * @see llvm::BasicBlock::Create()
  */
-LLVMBasicBlockRef LLVMCreateBasicBlockInContext(LLVMContextRef C,
-                                                const char *Name);
+LLVM_C_ABI LLVMBasicBlockRef LLVMCreateBasicBlockInContext(LLVMContextRef C,
+                                                           const char *Name);
 
 /**
  * Append a basic block to the end of a function.
  *
  * @see llvm::BasicBlock::Create()
  */
-LLVMBasicBlockRef LLVMAppendBasicBlockInContext(LLVMContextRef C,
-                                                LLVMValueRef Fn,
-                                                const char *Name);
+LLVM_C_ABI LLVMBasicBlockRef LLVMAppendBasicBlockInContext(LLVMContextRef C,
+                                                           LLVMValueRef Fn,
+                                                           const char *Name);
 
 /**
  * Append a basic block to the end of a function using the global
@@ -3487,7 +3589,8 @@ LLVMBasicBlockRef LLVMAppendBasicBlockInContext(LLVMContextRef C,
  *
  * @see llvm::BasicBlock::Create()
  */
-LLVMBasicBlockRef LLVMAppendBasicBlock(LLVMValueRef Fn, const char *Name);
+LLVM_C_ABI LLVMBasicBlockRef LLVMAppendBasicBlock(LLVMValueRef Fn,
+                                                  const char *Name);
 
 /**
  * Insert a basic block in a function before another basic block.
@@ -3497,17 +3600,17 @@ LLVMBasicBlockRef LLVMAppendBasicBlock(LLVMValueRef Fn, const char *Name);
  *
  * @see llvm::BasicBlock::Create()
  */
-LLVMBasicBlockRef LLVMInsertBasicBlockInContext(LLVMContextRef C,
-                                                LLVMBasicBlockRef BB,
-                                                const char *Name);
+LLVM_C_ABI LLVMBasicBlockRef LLVMInsertBasicBlockInContext(LLVMContextRef C,
+                                                           LLVMBasicBlockRef BB,
+                                                           const char *Name);
 
 /**
  * Insert a basic block in a function using the global context.
  *
  * @see llvm::BasicBlock::Create()
  */
-LLVMBasicBlockRef LLVMInsertBasicBlock(LLVMBasicBlockRef InsertBeforeBB,
-                                       const char *Name);
+LLVM_C_ABI LLVMBasicBlockRef
+LLVMInsertBasicBlock(LLVMBasicBlockRef InsertBeforeBB, const char *Name);
 
 /**
  * Remove a basic block from a function and delete it.
@@ -3517,7 +3620,7 @@ LLVMBasicBlockRef LLVMInsertBasicBlock(LLVMBasicBlockRef InsertBeforeBB,
  *
  * @see llvm::BasicBlock::eraseFromParent()
  */
-void LLVMDeleteBasicBlock(LLVMBasicBlockRef BB);
+LLVM_C_ABI void LLVMDeleteBasicBlock(LLVMBasicBlockRef BB);
 
 /**
  * Remove a basic block from a function.
@@ -3527,21 +3630,23 @@ void LLVMDeleteBasicBlock(LLVMBasicBlockRef BB);
  *
  * @see llvm::BasicBlock::removeFromParent()
  */
-void LLVMRemoveBasicBlockFromParent(LLVMBasicBlockRef BB);
+LLVM_C_ABI void LLVMRemoveBasicBlockFromParent(LLVMBasicBlockRef BB);
 
 /**
  * Move a basic block to before another one.
  *
  * @see llvm::BasicBlock::moveBefore()
  */
-void LLVMMoveBasicBlockBefore(LLVMBasicBlockRef BB, LLVMBasicBlockRef MovePos);
+LLVM_C_ABI void LLVMMoveBasicBlockBefore(LLVMBasicBlockRef BB,
+                                         LLVMBasicBlockRef MovePos);
 
 /**
  * Move a basic block to after another one.
  *
  * @see llvm::BasicBlock::moveAfter()
  */
-void LLVMMoveBasicBlockAfter(LLVMBasicBlockRef BB, LLVMBasicBlockRef MovePos);
+LLVM_C_ABI void LLVMMoveBasicBlockAfter(LLVMBasicBlockRef BB,
+                                        LLVMBasicBlockRef MovePos);
 
 /**
  * Obtain the first instruction in a basic block.
@@ -3549,14 +3654,14 @@ void LLVMMoveBasicBlockAfter(LLVMBasicBlockRef BB, LLVMBasicBlockRef MovePos);
  * The returned LLVMValueRef corresponds to a llvm::Instruction
  * instance.
  */
-LLVMValueRef LLVMGetFirstInstruction(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMGetFirstInstruction(LLVMBasicBlockRef BB);
 
 /**
  * Obtain the last instruction in a basic block.
  *
  * The returned LLVMValueRef corresponds to an LLVM:Instruction.
  */
-LLVMValueRef LLVMGetLastInstruction(LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMGetLastInstruction(LLVMBasicBlockRef BB);
 
 /**
  * @}
@@ -3582,17 +3687,18 @@ LLVMValueRef LLVMGetLastInstruction(LLVMBasicBlockRef BB);
 /**
  * Determine whether an instruction has any metadata attached.
  */
-int LLVMHasMetadata(LLVMValueRef Val);
+LLVM_C_ABI int LLVMHasMetadata(LLVMValueRef Val);
 
 /**
  * Return metadata associated with an instruction value.
  */
-LLVMValueRef LLVMGetMetadata(LLVMValueRef Val, unsigned KindID);
+LLVM_C_ABI LLVMValueRef LLVMGetMetadata(LLVMValueRef Val, unsigned KindID);
 
 /**
  * Set metadata associated with an instruction value.
  */
-void LLVMSetMetadata(LLVMValueRef Val, unsigned KindID, LLVMValueRef Node);
+LLVM_C_ABI void LLVMSetMetadata(LLVMValueRef Val, unsigned KindID,
+                                LLVMValueRef Node);
 
 /**
  * Returns the metadata associated with an instruction value, but filters out
@@ -3600,7 +3706,7 @@ void LLVMSetMetadata(LLVMValueRef Val, unsigned KindID, LLVMValueRef Node);
  *
  * @see llvm::Instruction::getAllMetadataOtherThanDebugLoc()
  */
-LLVMValueMetadataEntry *
+LLVM_C_ABI LLVMValueMetadataEntry *
 LLVMInstructionGetAllMetadataOtherThanDebugLoc(LLVMValueRef Instr,
                                                size_t *NumEntries);
 
@@ -3609,7 +3715,7 @@ LLVMInstructionGetAllMetadataOtherThanDebugLoc(LLVMValueRef Instr,
  *
  * @see llvm::Instruction::getParent()
  */
-LLVMBasicBlockRef LLVMGetInstructionParent(LLVMValueRef Inst);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetInstructionParent(LLVMValueRef Inst);
 
 /**
  * Obtain the instruction that occurs after the one specified.
@@ -3619,7 +3725,7 @@ LLVMBasicBlockRef LLVMGetInstructionParent(LLVMValueRef Inst);
  * If this is the last instruction in a basic block, NULL will be
  * returned.
  */
-LLVMValueRef LLVMGetNextInstruction(LLVMValueRef Inst);
+LLVM_C_ABI LLVMValueRef LLVMGetNextInstruction(LLVMValueRef Inst);
 
 /**
  * Obtain the instruction that occurred before this one.
@@ -3627,7 +3733,7 @@ LLVMValueRef LLVMGetNextInstruction(LLVMValueRef Inst);
  * If the instruction is the first instruction in a basic block, NULL
  * will be returned.
  */
-LLVMValueRef LLVMGetPreviousInstruction(LLVMValueRef Inst);
+LLVM_C_ABI LLVMValueRef LLVMGetPreviousInstruction(LLVMValueRef Inst);
 
 /**
  * Remove an instruction.
@@ -3637,7 +3743,7 @@ LLVMValueRef LLVMGetPreviousInstruction(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::removeFromParent()
  */
-void LLVMInstructionRemoveFromParent(LLVMValueRef Inst);
+LLVM_C_ABI void LLVMInstructionRemoveFromParent(LLVMValueRef Inst);
 
 /**
  * Remove and delete an instruction.
@@ -3647,7 +3753,7 @@ void LLVMInstructionRemoveFromParent(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::eraseFromParent()
  */
-void LLVMInstructionEraseFromParent(LLVMValueRef Inst);
+LLVM_C_ABI void LLVMInstructionEraseFromParent(LLVMValueRef Inst);
 
 /**
  * Delete an instruction.
@@ -3657,14 +3763,14 @@ void LLVMInstructionEraseFromParent(LLVMValueRef Inst);
  *
  * @see llvm::Value::deleteValue()
  */
-void LLVMDeleteInstruction(LLVMValueRef Inst);
+LLVM_C_ABI void LLVMDeleteInstruction(LLVMValueRef Inst);
 
 /**
  * Obtain the code opcode for an individual instruction.
  *
  * @see llvm::Instruction::getOpCode()
  */
-LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst);
+LLVM_C_ABI LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst);
 
 /**
  * Obtain the predicate of an instruction.
@@ -3673,7 +3779,7 @@ LLVMOpcode LLVMGetInstructionOpcode(LLVMValueRef Inst);
  *
  * @see llvm::ICmpInst::getPredicate()
  */
-LLVMIntPredicate LLVMGetICmpPredicate(LLVMValueRef Inst);
+LLVM_C_ABI LLVMIntPredicate LLVMGetICmpPredicate(LLVMValueRef Inst);
 
 /**
  * Get whether or not an icmp instruction has the samesign flag.
@@ -3682,7 +3788,7 @@ LLVMIntPredicate LLVMGetICmpPredicate(LLVMValueRef Inst);
  *
  * @see llvm::ICmpInst::hasSameSign()
  */
-LLVMBool LLVMGetICmpSameSign(LLVMValueRef Inst);
+LLVM_C_ABI LLVMBool LLVMGetICmpSameSign(LLVMValueRef Inst);
 
 /**
  * Set the samesign flag on an icmp instruction.
@@ -3691,7 +3797,7 @@ LLVMBool LLVMGetICmpSameSign(LLVMValueRef Inst);
  *
  * @see llvm::ICmpInst::setSameSign()
  */
-void LLVMSetICmpSameSign(LLVMValueRef Inst, LLVMBool SameSign);
+LLVM_C_ABI void LLVMSetICmpSameSign(LLVMValueRef Inst, LLVMBool SameSign);
 
 /**
  * Obtain the float predicate of an instruction.
@@ -3700,7 +3806,7 @@ void LLVMSetICmpSameSign(LLVMValueRef Inst, LLVMBool SameSign);
  *
  * @see llvm::FCmpInst::getPredicate()
  */
-LLVMRealPredicate LLVMGetFCmpPredicate(LLVMValueRef Inst);
+LLVM_C_ABI LLVMRealPredicate LLVMGetFCmpPredicate(LLVMValueRef Inst);
 
 /**
  * Create a copy of 'this' instruction that is identical in all ways
@@ -3710,7 +3816,7 @@ LLVMRealPredicate LLVMGetFCmpPredicate(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::clone()
  */
-LLVMValueRef LLVMInstructionClone(LLVMValueRef Inst);
+LLVM_C_ABI LLVMValueRef LLVMInstructionClone(LLVMValueRef Inst);
 
 /**
  * Determine whether an instruction is a terminator. This routine is named to
@@ -3719,7 +3825,7 @@ LLVMValueRef LLVMInstructionClone(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::isTerminator()
  */
-LLVMValueRef LLVMIsATerminatorInst(LLVMValueRef Inst);
+LLVM_C_ABI LLVMValueRef LLVMIsATerminatorInst(LLVMValueRef Inst);
 
 /**
  * Obtain the first debug record attached to an instruction.
@@ -3731,7 +3837,7 @@ LLVMValueRef LLVMIsATerminatorInst(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::getDbgRecordRange()
  */
-LLVMDbgRecordRef LLVMGetFirstDbgRecord(LLVMValueRef Inst);
+LLVM_C_ABI LLVMDbgRecordRef LLVMGetFirstDbgRecord(LLVMValueRef Inst);
 
 /**
  * Obtain the last debug record attached to an instruction.
@@ -3740,21 +3846,22 @@ LLVMDbgRecordRef LLVMGetFirstDbgRecord(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::getDbgRecordRange()
  */
-LLVMDbgRecordRef LLVMGetLastDbgRecord(LLVMValueRef Inst);
+LLVM_C_ABI LLVMDbgRecordRef LLVMGetLastDbgRecord(LLVMValueRef Inst);
 
 /**
  * Obtain the next DbgRecord in the sequence or NULL if there are no more.
  *
  * @see llvm::Instruction::getDbgRecordRange()
  */
-LLVMDbgRecordRef LLVMGetNextDbgRecord(LLVMDbgRecordRef DbgRecord);
+LLVM_C_ABI LLVMDbgRecordRef LLVMGetNextDbgRecord(LLVMDbgRecordRef DbgRecord);
 
 /**
  * Obtain the previous DbgRecord in the sequence or NULL if there are no more.
  *
  * @see llvm::Instruction::getDbgRecordRange()
  */
-LLVMDbgRecordRef LLVMGetPreviousDbgRecord(LLVMDbgRecordRef DbgRecord);
+LLVM_C_ABI LLVMDbgRecordRef
+LLVMGetPreviousDbgRecord(LLVMDbgRecordRef DbgRecord);
 
 /**
  * @defgroup LLVMCCoreValueInstructionCall Call Sites and Invocations
@@ -3776,7 +3883,7 @@ LLVMDbgRecordRef LLVMGetPreviousDbgRecord(LLVMDbgRecordRef DbgRecord);
  * @see llvm::InvokeInst::getNumArgOperands()
  * @see llvm::FuncletPadInst::getNumArgOperands()
  */
-unsigned LLVMGetNumArgOperands(LLVMValueRef Instr);
+LLVM_C_ABI unsigned LLVMGetNumArgOperands(LLVMValueRef Instr);
 
 /**
  * Set the calling convention for a call instruction.
@@ -3787,7 +3894,7 @@ unsigned LLVMGetNumArgOperands(LLVMValueRef Instr);
  * @see llvm::CallInst::setCallingConv()
  * @see llvm::InvokeInst::setCallingConv()
  */
-void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC);
+LLVM_C_ABI void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC);
 
 /**
  * Obtain the calling convention for a call instruction.
@@ -3797,33 +3904,37 @@ void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC);
  *
  * @see LLVMSetInstructionCallConv()
  */
-unsigned LLVMGetInstructionCallConv(LLVMValueRef Instr);
-
-void LLVMSetInstrParamAlignment(LLVMValueRef Instr, LLVMAttributeIndex Idx,
-                                unsigned Align);
-
-void LLVMAddCallSiteAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
-                              LLVMAttributeRef A);
-unsigned LLVMGetCallSiteAttributeCount(LLVMValueRef C, LLVMAttributeIndex Idx);
-void LLVMGetCallSiteAttributes(LLVMValueRef C, LLVMAttributeIndex Idx,
-                               LLVMAttributeRef *Attrs);
-LLVMAttributeRef LLVMGetCallSiteEnumAttribute(LLVMValueRef C,
-                                              LLVMAttributeIndex Idx,
-                                              unsigned KindID);
-LLVMAttributeRef LLVMGetCallSiteStringAttribute(LLVMValueRef C,
+LLVM_C_ABI unsigned LLVMGetInstructionCallConv(LLVMValueRef Instr);
+
+LLVM_C_ABI void LLVMSetInstrParamAlignment(LLVMValueRef Instr,
+                                           LLVMAttributeIndex Idx,
+                                           unsigned Align);
+
+LLVM_C_ABI void LLVMAddCallSiteAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
+                                         LLVMAttributeRef A);
+LLVM_C_ABI unsigned LLVMGetCallSiteAttributeCount(LLVMValueRef C,
+                                                  LLVMAttributeIndex Idx);
+LLVM_C_ABI void LLVMGetCallSiteAttributes(LLVMValueRef C,
+                                          LLVMAttributeIndex Idx,
+                                          LLVMAttributeRef *Attrs);
+LLVM_C_ABI LLVMAttributeRef LLVMGetCallSiteEnumAttribute(LLVMValueRef C,
+                                                         LLVMAttributeIndex Idx,
+                                                         unsigned KindID);
+LLVM_C_ABI LLVMAttributeRef LLVMGetCallSiteStringAttribute(
+    LLVMValueRef C, LLVMAttributeIndex Idx, const char *K, unsigned KLen);
+LLVM_C_ABI void LLVMRemoveCallSiteEnumAttribute(LLVMValueRef C,
                                                 LLVMAttributeIndex Idx,
-                                                const char *K, unsigned KLen);
-void LLVMRemoveCallSiteEnumAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
-                                     unsigned KindID);
-void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
-                                       const char *K, unsigned KLen);
+                                                unsigned KindID);
+LLVM_C_ABI void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C,
+                                                  LLVMAttributeIndex Idx,
+                                                  const char *K, unsigned KLen);
 
 /**
  * Obtain the function type called by this instruction.
  *
  * @see llvm::CallBase::getFunctionType()
  */
-LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C);
+LLVM_C_ABI LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C);
 
 /**
  * Obtain the pointer to the function invoked by this instruction.
@@ -3834,7 +3945,7 @@ LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C);
  * @see llvm::CallInst::getCalledOperand()
  * @see llvm::InvokeInst::getCalledOperand()
  */
-LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr);
+LLVM_C_ABI LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr);
 
 /**
  * Obtain the number of operand bundles attached to this instruction.
@@ -3843,7 +3954,7 @@ LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr);
  *
  * @see llvm::CallBase::getNumOperandBundles()
  */
-unsigned LLVMGetNumOperandBundles(LLVMValueRef C);
+LLVM_C_ABI unsigned LLVMGetNumOperandBundles(LLVMValueRef C);
 
 /**
  * Obtain the operand bundle attached to this instruction at the given index.
@@ -3851,8 +3962,8 @@ unsigned LLVMGetNumOperandBundles(LLVMValueRef C);
  *
  * This only works on llvm::CallInst and llvm::InvokeInst instructions.
  */
-LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C,
-                                                 unsigned Index);
+LLVM_C_ABI LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C,
+                                                            unsigned Index);
 
 /**
  * Obtain whether a call instruction is a tail call.
@@ -3861,7 +3972,7 @@ LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C,
  *
  * @see llvm::CallInst::isTailCall()
  */
-LLVMBool LLVMIsTailCall(LLVMValueRef CallInst);
+LLVM_C_ABI LLVMBool LLVMIsTailCall(LLVMValueRef CallInst);
 
 /**
  * Set whether a call instruction is a tail call.
@@ -3870,21 +3981,22 @@ LLVMBool LLVMIsTailCall(LLVMValueRef CallInst);
  *
  * @see llvm::CallInst::setTailCall()
  */
-void LLVMSetTailCall(LLVMValueRef CallInst, LLVMBool IsTailCall);
+LLVM_C_ABI void LLVMSetTailCall(LLVMValueRef CallInst, LLVMBool IsTailCall);
 
 /**
  * Obtain a tail call kind of the call instruction.
  *
  * @see llvm::CallInst::setTailCallKind()
  */
-LLVMTailCallKind LLVMGetTailCallKind(LLVMValueRef CallInst);
+LLVM_C_ABI LLVMTailCallKind LLVMGetTailCallKind(LLVMValueRef CallInst);
 
 /**
  * Set the call kind of the call instruction.
  *
  * @see llvm::CallInst::getTailCallKind()
  */
-void LLVMSetTailCallKind(LLVMValueRef CallInst, LLVMTailCallKind kind);
+LLVM_C_ABI void LLVMSetTailCallKind(LLVMValueRef CallInst,
+                                    LLVMTailCallKind kind);
 
 /**
  * Return the normal destination basic block.
@@ -3893,7 +4005,7 @@ void LLVMSetTailCallKind(LLVMValueRef CallInst, LLVMTailCallKind kind);
  *
  * @see llvm::InvokeInst::getNormalDest()
  */
-LLVMBasicBlockRef LLVMGetNormalDest(LLVMValueRef InvokeInst);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetNormalDest(LLVMValueRef InvokeInst);
 
 /**
  * Return the unwind destination basic block.
@@ -3905,7 +4017,7 @@ LLVMBasicBlockRef LLVMGetNormalDest(LLVMValueRef InvokeInst);
  * @see llvm::CleanupReturnInst::getUnwindDest()
  * @see llvm::CatchSwitchInst::getUnwindDest()
  */
-LLVMBasicBlockRef LLVMGetUnwindDest(LLVMValueRef InvokeInst);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetUnwindDest(LLVMValueRef InvokeInst);
 
 /**
  * Set the normal destination basic block.
@@ -3914,7 +4026,7 @@ LLVMBasicBlockRef LLVMGetUnwindDest(LLVMValueRef InvokeInst);
  *
  * @see llvm::InvokeInst::setNormalDest()
  */
-void LLVMSetNormalDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
+LLVM_C_ABI void LLVMSetNormalDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
 
 /**
  * Set the unwind destination basic block.
@@ -3926,14 +4038,14 @@ void LLVMSetNormalDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
  * @see llvm::CleanupReturnInst::setUnwindDest()
  * @see llvm::CatchSwitchInst::setUnwindDest()
  */
-void LLVMSetUnwindDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
+LLVM_C_ABI void LLVMSetUnwindDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
 
 /**
  * Get the default destination of a CallBr instruction.
  *
  * @see llvm::CallBrInst::getDefaultDest()
  */
-LLVMBasicBlockRef LLVMGetCallBrDefaultDest(LLVMValueRef CallBr);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetCallBrDefaultDest(LLVMValueRef CallBr);
 
 /**
  * Get the number of indirect destinations of a CallBr instruction.
@@ -3941,14 +4053,15 @@ LLVMBasicBlockRef LLVMGetCallBrDefaultDest(LLVMValueRef CallBr);
  * @see llvm::CallBrInst::getNumIndirectDests()
 
  */
-unsigned LLVMGetCallBrNumIndirectDests(LLVMValueRef CallBr);
+LLVM_C_ABI unsigned LLVMGetCallBrNumIndirectDests(LLVMValueRef CallBr);
 
 /**
  * Get the indirect destination of a CallBr instruction at the given index.
  *
  * @see llvm::CallBrInst::getIndirectDest()
  */
-LLVMBasicBlockRef LLVMGetCallBrIndirectDest(LLVMValueRef CallBr, unsigned Idx);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetCallBrIndirectDest(LLVMValueRef CallBr,
+                                                       unsigned Idx);
 
 /**
  * @}
@@ -3968,21 +4081,22 @@ LLVMBasicBlockRef LLVMGetCallBrIndirectDest(LLVMValueRef CallBr, unsigned Idx);
  *
  * @see llvm::Instruction::getNumSuccessors
  */
-unsigned LLVMGetNumSuccessors(LLVMValueRef Term);
+LLVM_C_ABI unsigned LLVMGetNumSuccessors(LLVMValueRef Term);
 
 /**
  * Return the specified successor.
  *
  * @see llvm::Instruction::getSuccessor
  */
-LLVMBasicBlockRef LLVMGetSuccessor(LLVMValueRef Term, unsigned i);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetSuccessor(LLVMValueRef Term, unsigned i);
 
 /**
  * Update the specified successor to point at the provided block.
  *
  * @see llvm::Instruction::setSuccessor
  */
-void LLVMSetSuccessor(LLVMValueRef Term, unsigned i, LLVMBasicBlockRef block);
+LLVM_C_ABI void LLVMSetSuccessor(LLVMValueRef Term, unsigned i,
+                                 LLVMBasicBlockRef block);
 
 /**
  * Return if a branch is conditional.
@@ -3991,7 +4105,7 @@ void LLVMSetSuccessor(LLVMValueRef Term, unsigned i, LLVMBasicBlockRef block);
  *
  * @see llvm::BranchInst::isConditional
  */
-LLVMBool LLVMIsConditional(LLVMValueRef Branch);
+LLVM_C_ABI LLVMBool LLVMIsConditional(LLVMValueRef Branch);
 
 /**
  * Return the condition of a branch instruction.
@@ -4000,7 +4114,7 @@ LLVMBool LLVMIsConditional(LLVMValueRef Branch);
  *
  * @see llvm::BranchInst::getCondition
  */
-LLVMValueRef LLVMGetCondition(LLVMValueRef Branch);
+LLVM_C_ABI LLVMValueRef LLVMGetCondition(LLVMValueRef Branch);
 
 /**
  * Set the condition of a branch instruction.
@@ -4009,7 +4123,7 @@ LLVMValueRef LLVMGetCondition(LLVMValueRef Branch);
  *
  * @see llvm::BranchInst::setCondition
  */
-void LLVMSetCondition(LLVMValueRef Branch, LLVMValueRef Cond);
+LLVM_C_ABI void LLVMSetCondition(LLVMValueRef Branch, LLVMValueRef Cond);
 
 /**
  * Obtain the default destination basic block of a switch instruction.
@@ -4018,7 +4132,7 @@ void LLVMSetCondition(LLVMValueRef Branch, LLVMValueRef Cond);
  *
  * @see llvm::SwitchInst::getDefaultDest()
  */
-LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef SwitchInstr);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef SwitchInstr);
 
 /**
  * @}
@@ -4036,7 +4150,7 @@ LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef SwitchInstr);
 /**
  * Obtain the type that is being allocated by the alloca instruction.
  */
-LLVMTypeRef LLVMGetAllocatedType(LLVMValueRef Alloca);
+LLVM_C_ABI LLVMTypeRef LLVMGetAllocatedType(LLVMValueRef Alloca);
 
 /**
  * @}
@@ -4054,31 +4168,32 @@ LLVMTypeRef LLVMGetAllocatedType(LLVMValueRef Alloca);
 /**
  * Check whether the given GEP operator is inbounds.
  */
-LLVMBool LLVMIsInBounds(LLVMValueRef GEP);
+LLVM_C_ABI LLVMBool LLVMIsInBounds(LLVMValueRef GEP);
 
 /**
  * Set the given GEP instruction to be inbounds or not.
  */
-void LLVMSetIsInBounds(LLVMValueRef GEP, LLVMBool InBounds);
+LLVM_C_ABI void LLVMSetIsInBounds(LLVMValueRef GEP, LLVMBool InBounds);
 
 /**
  * Get the source element type of the given GEP operator.
  */
-LLVMTypeRef LLVMGetGEPSourceElementType(LLVMValueRef GEP);
+LLVM_C_ABI LLVMTypeRef LLVMGetGEPSourceElementType(LLVMValueRef GEP);
 
 /**
  * Get the no-wrap related flags for the given GEP instruction.
  *
  * @see llvm::GetElementPtrInst::getNoWrapFlags
  */
-LLVMGEPNoWrapFlags LLVMGEPGetNoWrapFlags(LLVMValueRef GEP);
+LLVM_C_ABI LLVMGEPNoWrapFlags LLVMGEPGetNoWrapFlags(LLVMValueRef GEP);
 
 /**
  * Set the no-wrap related flags for the given GEP instruction.
  *
  * @see llvm::GetElementPtrInst::setNoWrapFlags
  */
-void LLVMGEPSetNoWrapFlags(LLVMValueRef GEP, LLVMGEPNoWrapFlags NoWrapFlags);
+LLVM_C_ABI void LLVMGEPSetNoWrapFlags(LLVMValueRef GEP,
+                                      LLVMGEPNoWrapFlags NoWrapFlags);
 
 /**
  * @}
@@ -4096,23 +4211,27 @@ void LLVMGEPSetNoWrapFlags(LLVMValueRef GEP, LLVMGEPNoWrapFlags NoWrapFlags);
 /**
  * Add an incoming value to the end of a PHI list.
  */
-void LLVMAddIncoming(LLVMValueRef PhiNode, LLVMValueRef *IncomingValues,
-                     LLVMBasicBlockRef *IncomingBlocks, unsigned Count);
+LLVM_C_ABI void LLVMAddIncoming(LLVMValueRef PhiNode,
+                                LLVMValueRef *IncomingValues,
+                                LLVMBasicBlockRef *IncomingBlocks,
+                                unsigned Count);
 
 /**
  * Obtain the number of incoming basic blocks to a PHI node.
  */
-unsigned LLVMCountIncoming(LLVMValueRef PhiNode);
+LLVM_C_ABI unsigned LLVMCountIncoming(LLVMValueRef PhiNode);
 
 /**
  * Obtain an incoming value to a PHI node as an LLVMValueRef.
  */
-LLVMValueRef LLVMGetIncomingValue(LLVMValueRef PhiNode, unsigned Index);
+LLVM_C_ABI LLVMValueRef LLVMGetIncomingValue(LLVMValueRef PhiNode,
+                                             unsigned Index);
 
 /**
  * Obtain an incoming value to a PHI node as an LLVMBasicBlockRef.
  */
-LLVMBasicBlockRef LLVMGetIncomingBlock(LLVMValueRef PhiNode, unsigned Index);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetIncomingBlock(LLVMValueRef PhiNode,
+                                                  unsigned Index);
 
 /**
  * @}
@@ -4132,12 +4251,12 @@ LLVMBasicBlockRef LLVMGetIncomingBlock(LLVMValueRef PhiNode, unsigned Index);
  * Obtain the number of indices.
  * NB: This also works on GEP operators.
  */
-unsigned LLVMGetNumIndices(LLVMValueRef Inst);
+LLVM_C_ABI unsigned LLVMGetNumIndices(LLVMValueRef Inst);
 
 /**
  * Obtain the indices as an array.
  */
-const unsigned *LLVMGetIndices(LLVMValueRef Inst);
+LLVM_C_ABI const unsigned *LLVMGetIndices(LLVMValueRef Inst);
 
 /**
  * @}
@@ -4160,37 +4279,43 @@ const unsigned *LLVMGetIndices(LLVMValueRef Inst);
  * @{
  */
 
-LLVMBuilderRef LLVMCreateBuilderInContext(LLVMContextRef C);
-LLVMBuilderRef LLVMCreateBuilder(void);
+LLVM_C_ABI LLVMBuilderRef LLVMCreateBuilderInContext(LLVMContextRef C);
+LLVM_C_ABI LLVMBuilderRef LLVMCreateBuilder(void);
 /**
  * Set the builder position before Instr but after any attached debug records,
  * or if Instr is null set the position to the end of Block.
  */
-void LLVMPositionBuilder(LLVMBuilderRef Builder, LLVMBasicBlockRef Block,
-                         LLVMValueRef Instr);
+LLVM_C_ABI void LLVMPositionBuilder(LLVMBuilderRef Builder,
+                                    LLVMBasicBlockRef Block,
+                                    LLVMValueRef Instr);
 /**
  * Set the builder position before Instr and any attached debug records,
  * or if Instr is null set the position to the end of Block.
  */
-void LLVMPositionBuilderBeforeDbgRecords(LLVMBuilderRef Builder,
-                                         LLVMBasicBlockRef Block,
-                                         LLVMValueRef Inst);
+LLVM_C_ABI void LLVMPositionBuilderBeforeDbgRecords(LLVMBuilderRef Builder,
+                                                    LLVMBasicBlockRef Block,
+                                                    LLVMValueRef Inst);
 /**
  * Set the builder position before Instr but after any attached debug records.
  */
-void LLVMPositionBuilderBefore(LLVMBuilderRef Builder, LLVMValueRef Instr);
+LLVM_C_ABI void LLVMPositionBuilderBefore(LLVMBuilderRef Builder,
+                                          LLVMValueRef Instr);
 /**
  * Set the builder position before Instr and any attached debug records.
  */
-void LLVMPositionBuilderBeforeInstrAndDbgRecords(LLVMBuilderRef Builder,
-                                                 LLVMValueRef Instr);
-void LLVMPositionBuilderAtEnd(LLVMBuilderRef Builder, LLVMBasicBlockRef Block);
-LLVMBasicBlockRef LLVMGetInsertBlock(LLVMBuilderRef Builder);
-void LLVMClearInsertionPosition(LLVMBuilderRef Builder);
-void LLVMInsertIntoBuilder(LLVMBuilderRef Builder, LLVMValueRef Instr);
-void LLVMInsertIntoBuilderWithName(LLVMBuilderRef Builder, LLVMValueRef Instr,
-                                   const char *Name);
-void LLVMDisposeBuilder(LLVMBuilderRef Builder);
+LLVM_C_ABI void
+LLVMPositionBuilderBeforeInstrAndDbgRecords(LLVMBuilderRef Builder,
+                                            LLVMValueRef Instr);
+LLVM_C_ABI void LLVMPositionBuilderAtEnd(LLVMBuilderRef Builder,
+                                         LLVMBasicBlockRef Block);
+LLVM_C_ABI LLVMBasicBlockRef LLVMGetInsertBlock(LLVMBuilderRef Builder);
+LLVM_C_ABI void LLVMClearInsertionPosition(LLVMBuilderRef Builder);
+LLVM_C_ABI void LLVMInsertIntoBuilder(LLVMBuilderRef Builder,
+                                      LLVMValueRef Instr);
+LLVM_C_ABI void LLVMInsertIntoBuilderWithName(LLVMBuilderRef Builder,
+                                              LLVMValueRef Instr,
+                                              const char *Name);
+LLVM_C_ABI void LLVMDisposeBuilder(LLVMBuilderRef Builder);
 
 /* Metadata */
 
@@ -4199,7 +4324,7 @@ void LLVMDisposeBuilder(LLVMBuilderRef Builder);
  *
  * @see llvm::IRBuilder::getCurrentDebugLocation()
  */
-LLVMMetadataRef LLVMGetCurrentDebugLocation2(LLVMBuilderRef Builder);
+LLVM_C_ABI LLVMMetadataRef LLVMGetCurrentDebugLocation2(LLVMBuilderRef Builder);
 
 /**
  * Set location information used by debugging information.
@@ -4208,7 +4333,8 @@ LLVMMetadataRef LLVMGetCurrentDebugLocation2(LLVMBuilderRef Builder);
  *
  * @see llvm::IRBuilder::SetCurrentDebugLocation()
  */
-void LLVMSetCurrentDebugLocation2(LLVMBuilderRef Builder, LLVMMetadataRef Loc);
+LLVM_C_ABI void LLVMSetCurrentDebugLocation2(LLVMBuilderRef Builder,
+                                             LLVMMetadataRef Loc);
 
 /**
  * Attempts to set the debug location for the given instruction using the
@@ -4220,21 +4346,24 @@ void LLVMSetCurrentDebugLocation2(LLVMBuilderRef Builder, LLVMMetadataRef Loc);
  *
  * @see llvm::IRBuilder::SetInstDebugLocation()
  */
-void LLVMSetInstDebugLocation(LLVMBuilderRef Builder, LLVMValueRef Inst);
+LLVM_C_ABI void LLVMSetInstDebugLocation(LLVMBuilderRef Builder,
+                                         LLVMValueRef Inst);
 
 /**
  * Adds the metadata registered with the given builder to the given instruction.
  *
  * @see llvm::IRBuilder::AddMetadataToInst()
  */
-void LLVMAddMetadataToInst(LLVMBuilderRef Builder, LLVMValueRef Inst);
+LLVM_C_ABI void LLVMAddMetadataToInst(LLVMBuilderRef Builder,
+                                      LLVMValueRef Inst);
 
 /**
  * Get the dafult floating-point math metadata for a given builder.
  *
  * @see llvm::IRBuilder::getDefaultFPMathTag()
  */
-LLVMMetadataRef LLVMBuilderGetDefaultFPMathTag(LLVMBuilderRef Builder);
+LLVM_C_ABI LLVMMetadataRef
+LLVMBuilderGetDefaultFPMathTag(LLVMBuilderRef Builder);
 
 /**
  * Set the default floating-point math metadata for the given builder.
@@ -4243,101 +4372,115 @@ LLVMMetadataRef LLVMBuilderGetDefaultFPMathTag(LLVMBuilderRef Builder);
  *
  * @see llvm::IRBuilder::setDefaultFPMathTag()
  */
-void LLVMBuilderSetDefaultFPMathTag(LLVMBuilderRef Builder,
-                                    LLVMMetadataRef FPMathTag);
+LLVM_C_ABI void LLVMBuilderSetDefaultFPMathTag(LLVMBuilderRef Builder,
+                                               LLVMMetadataRef FPMathTag);
 
 /**
  * Obtain the context to which this builder is associated.
  *
  * @see llvm::IRBuilder::getContext()
  */
-LLVMContextRef LLVMGetBuilderContext(LLVMBuilderRef Builder);
+LLVM_C_ABI LLVMContextRef LLVMGetBuilderContext(LLVMBuilderRef Builder);
 
 /**
  * Deprecated: Passing the NULL location will crash.
  * Use LLVMGetCurrentDebugLocation2 instead.
  */
-void LLVMSetCurrentDebugLocation(LLVMBuilderRef Builder, LLVMValueRef L);
+LLVM_C_ABI void LLVMSetCurrentDebugLocation(LLVMBuilderRef Builder,
+                                            LLVMValueRef L);
 /**
  * Deprecated: Returning the NULL location will crash.
  * Use LLVMGetCurrentDebugLocation2 instead.
  */
-LLVMValueRef LLVMGetCurrentDebugLocation(LLVMBuilderRef Builder);
+LLVM_C_ABI LLVMValueRef LLVMGetCurrentDebugLocation(LLVMBuilderRef Builder);
 
 /* Terminators */
-LLVMValueRef LLVMBuildRetVoid(LLVMBuilderRef);
-LLVMValueRef LLVMBuildRet(LLVMBuilderRef, LLVMValueRef V);
-LLVMValueRef LLVMBuildAggregateRet(LLVMBuilderRef, LLVMValueRef *RetVals,
-                                   unsigned N);
-LLVMValueRef LLVMBuildBr(LLVMBuilderRef, LLVMBasicBlockRef Dest);
-LLVMValueRef LLVMBuildCondBr(LLVMBuilderRef, LLVMValueRef If,
-                             LLVMBasicBlockRef Then, LLVMBasicBlockRef Else);
-LLVMValueRef LLVMBuildSwitch(LLVMBuilderRef, LLVMValueRef V,
-                             LLVMBasicBlockRef Else, unsigned NumCases);
-LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr,
-                                 unsigned NumDests);
-LLVMValueRef LLVMBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
-                             LLVMBasicBlockRef DefaultDest,
-                             LLVMBasicBlockRef *IndirectDests,
-                             unsigned NumIndirectDests, LLVMValueRef *Args,
-                             unsigned NumArgs, LLVMOperandBundleRef *Bundles,
-                             unsigned NumBundles, const char *Name);
-LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn,
-                              LLVMValueRef *Args, unsigned NumArgs,
-                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
-                              const char *Name);
-LLVMValueRef LLVMBuildInvokeWithOperandBundles(
+LLVM_C_ABI LLVMValueRef LLVMBuildRetVoid(LLVMBuilderRef);
+LLVM_C_ABI LLVMValueRef LLVMBuildRet(LLVMBuilderRef, LLVMValueRef V);
+LLVM_C_ABI LLVMValueRef LLVMBuildAggregateRet(LLVMBuilderRef,
+                                              LLVMValueRef *RetVals,
+                                              unsigned N);
+LLVM_C_ABI LLVMValueRef LLVMBuildBr(LLVMBuilderRef, LLVMBasicBlockRef Dest);
+LLVM_C_ABI LLVMValueRef LLVMBuildCondBr(LLVMBuilderRef, LLVMValueRef If,
+                                        LLVMBasicBlockRef Then,
+                                        LLVMBasicBlockRef Else);
+LLVM_C_ABI LLVMValueRef LLVMBuildSwitch(LLVMBuilderRef, LLVMValueRef V,
+                                        LLVMBasicBlockRef Else,
+                                        unsigned NumCases);
+LLVM_C_ABI LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr,
+                                            unsigned NumDests);
+LLVM_C_ABI LLVMValueRef LLVMBuildCallBr(
+    LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+    LLVMBasicBlockRef DefaultDest, LLVMBasicBlockRef *IndirectDests,
+    unsigned NumIndirectDests, LLVMValueRef *Args, unsigned NumArgs,
+    LLVMOperandBundleRef *Bundles, unsigned NumBundles, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty,
+                                         LLVMValueRef Fn, LLVMValueRef *Args,
+                                         unsigned NumArgs,
+                                         LLVMBasicBlockRef Then,
+                                         LLVMBasicBlockRef Catch,
+                                         const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildInvokeWithOperandBundles(
     LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args,
     unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
     LLVMOperandBundleRef *Bundles, unsigned NumBundles, const char *Name);
-LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef);
+LLVM_C_ABI LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef);
 
 /* Exception Handling */
-LLVMValueRef LLVMBuildResume(LLVMBuilderRef B, LLVMValueRef Exn);
-LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty,
-                                 LLVMValueRef PersFn, unsigned NumClauses,
-                                 const char *Name);
-LLVMValueRef LLVMBuildCleanupRet(LLVMBuilderRef B, LLVMValueRef CatchPad,
-                                 LLVMBasicBlockRef BB);
-LLVMValueRef LLVMBuildCatchRet(LLVMBuilderRef B, LLVMValueRef CatchPad,
-                               LLVMBasicBlockRef BB);
-LLVMValueRef LLVMBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad,
-                               LLVMValueRef *Args, unsigned NumArgs,
-                               const char *Name);
-LLVMValueRef LLVMBuildCleanupPad(LLVMBuilderRef B, LLVMValueRef ParentPad,
-                                 LLVMValueRef *Args, unsigned NumArgs,
-                                 const char *Name);
-LLVMValueRef LLVMBuildCatchSwitch(LLVMBuilderRef B, LLVMValueRef ParentPad,
-                                  LLVMBasicBlockRef UnwindBB,
-                                  unsigned NumHandlers, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildResume(LLVMBuilderRef B, LLVMValueRef Exn);
+LLVM_C_ABI LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                            LLVMValueRef PersFn,
+                                            unsigned NumClauses,
+                                            const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildCleanupRet(LLVMBuilderRef B,
+                                            LLVMValueRef CatchPad,
+                                            LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMBuildCatchRet(LLVMBuilderRef B,
+                                          LLVMValueRef CatchPad,
+                                          LLVMBasicBlockRef BB);
+LLVM_C_ABI LLVMValueRef LLVMBuildCatchPad(LLVMBuilderRef B,
+                                          LLVMValueRef ParentPad,
+                                          LLVMValueRef *Args, unsigned NumArgs,
+                                          const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildCleanupPad(LLVMBuilderRef B,
+                                            LLVMValueRef ParentPad,
+                                            LLVMValueRef *Args,
+                                            unsigned NumArgs, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildCatchSwitch(LLVMBuilderRef B,
+                                             LLVMValueRef ParentPad,
+                                             LLVMBasicBlockRef UnwindBB,
+                                             unsigned NumHandlers,
+                                             const char *Name);
 
 /* Add a case to the switch instruction */
-void LLVMAddCase(LLVMValueRef Switch, LLVMValueRef OnVal,
-                 LLVMBasicBlockRef Dest);
+LLVM_C_ABI void LLVMAddCase(LLVMValueRef Switch, LLVMValueRef OnVal,
+                            LLVMBasicBlockRef Dest);
 
 /* Add a destination to the indirectbr instruction */
-void LLVMAddDestination(LLVMValueRef IndirectBr, LLVMBasicBlockRef Dest);
+LLVM_C_ABI void LLVMAddDestination(LLVMValueRef IndirectBr,
+                                   LLVMBasicBlockRef Dest);
 
 /* Get the number of clauses on the landingpad instruction */
-unsigned LLVMGetNumClauses(LLVMValueRef LandingPad);
+LLVM_C_ABI unsigned LLVMGetNumClauses(LLVMValueRef LandingPad);
 
 /* Get the value of the clause at index Idx on the landingpad instruction */
-LLVMValueRef LLVMGetClause(LLVMValueRef LandingPad, unsigned Idx);
+LLVM_C_ABI LLVMValueRef LLVMGetClause(LLVMValueRef LandingPad, unsigned Idx);
 
 /* Add a catch or filter clause to the landingpad instruction */
-void LLVMAddClause(LLVMValueRef LandingPad, LLVMValueRef ClauseVal);
+LLVM_C_ABI void LLVMAddClause(LLVMValueRef LandingPad, LLVMValueRef ClauseVal);
 
 /* Get the 'cleanup' flag in the landingpad instruction */
-LLVMBool LLVMIsCleanup(LLVMValueRef LandingPad);
+LLVM_C_ABI LLVMBool LLVMIsCleanup(LLVMValueRef LandingPad);
 
 /* Set the 'cleanup' flag in the landingpad instruction */
-void LLVMSetCleanup(LLVMValueRef LandingPad, LLVMBool Val);
+LLVM_C_ABI void LLVMSetCleanup(LLVMValueRef LandingPad, LLVMBool Val);
 
 /* Add a destination to the catchswitch instruction */
-void LLVMAddHandler(LLVMValueRef CatchSwitch, LLVMBasicBlockRef Dest);
+LLVM_C_ABI void LLVMAddHandler(LLVMValueRef CatchSwitch,
+                               LLVMBasicBlockRef Dest);
 
 /* Get the number of handlers on the catchswitch instruction */
-unsigned LLVMGetNumHandlers(LLVMValueRef CatchSwitch);
+LLVM_C_ABI unsigned LLVMGetNumHandlers(LLVMValueRef CatchSwitch);
 
 /**
  * Obtain the basic blocks acting as handlers for a catchswitch instruction.
@@ -4350,15 +4493,17 @@ unsigned LLVMGetNumHandlers(LLVMValueRef CatchSwitch);
  * @param CatchSwitch The catchswitch instruction to operate on.
  * @param Handlers Memory address of an array to be filled with basic blocks.
  */
-void LLVMGetHandlers(LLVMValueRef CatchSwitch, LLVMBasicBlockRef *Handlers);
+LLVM_C_ABI void LLVMGetHandlers(LLVMValueRef CatchSwitch,
+                                LLVMBasicBlockRef *Handlers);
 
 /* Funclets */
 
 /* Get the number of funcletpad arguments. */
-LLVMValueRef LLVMGetArgOperand(LLVMValueRef Funclet, unsigned i);
+LLVM_C_ABI LLVMValueRef LLVMGetArgOperand(LLVMValueRef Funclet, unsigned i);
 
 /* Set a funcletpad argument at the given index. */
-void LLVMSetArgOperand(LLVMValueRef Funclet, unsigned i, LLVMValueRef value);
+LLVM_C_ABI void LLVMSetArgOperand(LLVMValueRef Funclet, unsigned i,
+                                  LLVMValueRef value);
 
 /**
  * Get the parent catchswitch instruction of a catchpad instruction.
@@ -4367,7 +4512,7 @@ void LLVMSetArgOperand(LLVMValueRef Funclet, unsigned i, LLVMValueRef value);
  *
  * @see llvm::CatchPadInst::getCatchSwitch()
  */
-LLVMValueRef LLVMGetParentCatchSwitch(LLVMValueRef CatchPad);
+LLVM_C_ABI LLVMValueRef LLVMGetParentCatchSwitch(LLVMValueRef CatchPad);
 
 /**
  * Set the parent catchswitch instruction of a catchpad instruction.
@@ -4376,91 +4521,95 @@ LLVMValueRef LLVMGetParentCatchSwitch(LLVMValueRef CatchPad);
  *
  * @see llvm::CatchPadInst::setCatchSwitch()
  */
-void LLVMSetParentCatchSwitch(LLVMValueRef CatchPad, LLVMValueRef CatchSwitch);
+LLVM_C_ABI void LLVMSetParentCatchSwitch(LLVMValueRef CatchPad,
+                                         LLVMValueRef CatchSwitch);
 
 /* Arithmetic */
-LLVMValueRef LLVMBuildAdd(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                          const char *Name);
-LLVMValueRef LLVMBuildNSWAdd(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                             const char *Name);
-LLVMValueRef LLVMBuildNUWAdd(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                             const char *Name);
-LLVMValueRef LLVMBuildFAdd(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildSub(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                          const char *Name);
-LLVMValueRef LLVMBuildNSWSub(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                             const char *Name);
-LLVMValueRef LLVMBuildNUWSub(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                             const char *Name);
-LLVMValueRef LLVMBuildFSub(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildMul(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                          const char *Name);
-LLVMValueRef LLVMBuildNSWMul(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                             const char *Name);
-LLVMValueRef LLVMBuildNUWMul(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                             const char *Name);
-LLVMValueRef LLVMBuildFMul(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildUDiv(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildExactUDiv(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                                const char *Name);
-LLVMValueRef LLVMBuildSDiv(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildExactSDiv(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                                const char *Name);
-LLVMValueRef LLVMBuildFDiv(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildURem(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildSRem(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildFRem(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildShl(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildLShr(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildAShr(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildAnd(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                          const char *Name);
-LLVMValueRef LLVMBuildOr(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                          const char *Name);
-LLVMValueRef LLVMBuildXor(LLVMBuilderRef, LLVMValueRef LHS, LLVMValueRef RHS,
-                          const char *Name);
-LLVMValueRef LLVMBuildBinOp(LLVMBuilderRef B, LLVMOpcode Op,
-                            LLVMValueRef LHS, LLVMValueRef RHS,
-                            const char *Name);
-LLVMValueRef LLVMBuildNeg(LLVMBuilderRef, LLVMValueRef V, const char *Name);
-LLVMValueRef LLVMBuildNSWNeg(LLVMBuilderRef B, LLVMValueRef V,
-                             const char *Name);
-LLVM_ATTRIBUTE_C_DEPRECATED(LLVMValueRef LLVMBuildNUWNeg(LLVMBuilderRef B,
-                                                         LLVMValueRef V,
-                                                         const char *Name),
-                            "Use LLVMBuildNeg + LLVMSetNUW instead.");
-LLVMValueRef LLVMBuildFNeg(LLVMBuilderRef, LLVMValueRef V, const char *Name);
-LLVMValueRef LLVMBuildNot(LLVMBuilderRef, LLVMValueRef V, const char *Name);
-
-LLVMBool LLVMGetNUW(LLVMValueRef ArithInst);
-void LLVMSetNUW(LLVMValueRef ArithInst, LLVMBool HasNUW);
-LLVMBool LLVMGetNSW(LLVMValueRef ArithInst);
-void LLVMSetNSW(LLVMValueRef ArithInst, LLVMBool HasNSW);
-LLVMBool LLVMGetExact(LLVMValueRef DivOrShrInst);
-void LLVMSetExact(LLVMValueRef DivOrShrInst, LLVMBool IsExact);
+LLVM_C_ABI LLVMValueRef LLVMBuildAdd(LLVMBuilderRef, LLVMValueRef LHS,
+                                     LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNSWAdd(LLVMBuilderRef, LLVMValueRef LHS,
+                                        LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNUWAdd(LLVMBuilderRef, LLVMValueRef LHS,
+                                        LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFAdd(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSub(LLVMBuilderRef, LLVMValueRef LHS,
+                                     LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNSWSub(LLVMBuilderRef, LLVMValueRef LHS,
+                                        LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNUWSub(LLVMBuilderRef, LLVMValueRef LHS,
+                                        LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFSub(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildMul(LLVMBuilderRef, LLVMValueRef LHS,
+                                     LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNSWMul(LLVMBuilderRef, LLVMValueRef LHS,
+                                        LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNUWMul(LLVMBuilderRef, LLVMValueRef LHS,
+                                        LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFMul(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildUDiv(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildExactUDiv(LLVMBuilderRef, LLVMValueRef LHS,
+                                           LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSDiv(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildExactSDiv(LLVMBuilderRef, LLVMValueRef LHS,
+                                           LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFDiv(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildURem(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSRem(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFRem(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildShl(LLVMBuilderRef, LLVMValueRef LHS,
+                                     LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildLShr(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildAShr(LLVMBuilderRef, LLVMValueRef LHS,
+                                      LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildAnd(LLVMBuilderRef, LLVMValueRef LHS,
+                                     LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildOr(LLVMBuilderRef, LLVMValueRef LHS,
+                                    LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildXor(LLVMBuilderRef, LLVMValueRef LHS,
+                                     LLVMValueRef RHS, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildBinOp(LLVMBuilderRef B, LLVMOpcode Op,
+                                       LLVMValueRef LHS, LLVMValueRef RHS,
+                                       const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNeg(LLVMBuilderRef, LLVMValueRef V,
+                                     const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNSWNeg(LLVMBuilderRef B, LLVMValueRef V,
+                                        const char *Name);
+LLVM_C_ABI LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildNUWNeg(LLVMBuilderRef B, LLVMValueRef V,
+                                 const char *Name),
+    "Use LLVMBuildNeg + LLVMSetNUW instead.");
+LLVM_C_ABI LLVMValueRef LLVMBuildFNeg(LLVMBuilderRef, LLVMValueRef V,
+                                      const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildNot(LLVMBuilderRef, LLVMValueRef V,
+                                     const char *Name);
+
+LLVM_C_ABI LLVMBool LLVMGetNUW(LLVMValueRef ArithInst);
+LLVM_C_ABI void LLVMSetNUW(LLVMValueRef ArithInst, LLVMBool HasNUW);
+LLVM_C_ABI LLVMBool LLVMGetNSW(LLVMValueRef ArithInst);
+LLVM_C_ABI void LLVMSetNSW(LLVMValueRef ArithInst, LLVMBool HasNSW);
+LLVM_C_ABI LLVMBool LLVMGetExact(LLVMValueRef DivOrShrInst);
+LLVM_C_ABI void LLVMSetExact(LLVMValueRef DivOrShrInst, LLVMBool IsExact);
 
 /**
  * Gets if the instruction has the non-negative flag set.
  * Only valid for zext instructions.
  */
-LLVMBool LLVMGetNNeg(LLVMValueRef NonNegInst);
+LLVM_C_ABI LLVMBool LLVMGetNNeg(LLVMValueRef NonNegInst);
 /**
  * Sets the non-negative flag for the instruction.
  * Only valid for zext instructions.
  */
-void LLVMSetNNeg(LLVMValueRef NonNegInst, LLVMBool IsNonNeg);
+LLVM_C_ABI void LLVMSetNNeg(LLVMValueRef NonNegInst, LLVMBool IsNonNeg);
 
 /**
  * Get the flags for which fast-math-style optimizations are allowed for this
@@ -4469,7 +4618,7 @@ void LLVMSetNNeg(LLVMValueRef NonNegInst, LLVMBool IsNonNeg);
  * Only valid on floating point instructions.
  * @see LLVMCanValueUseFastMathFlags
  */
-LLVMFastMathFlags LLVMGetFastMathFlags(LLVMValueRef FPMathInst);
+LLVM_C_ABI LLVMFastMathFlags LLVMGetFastMathFlags(LLVMValueRef FPMathInst);
 
 /**
  * Sets the flags for which fast-math-style optimizations are allowed for this
@@ -4478,7 +4627,8 @@ LLVMFastMathFlags LLVMGetFastMathFlags(LLVMValueRef FPMathInst);
  * Only valid on floating point instructions.
  * @see LLVMCanValueUseFastMathFlags
  */
-void LLVMSetFastMathFlags(LLVMValueRef FPMathInst, LLVMFastMathFlags FMF);
+LLVM_C_ABI void LLVMSetFastMathFlags(LLVMValueRef FPMathInst,
+                                     LLVMFastMathFlags FMF);
 
 /**
  * Check if a given value can potentially have fast math flags.
@@ -4487,23 +4637,25 @@ void LLVMSetFastMathFlags(LLVMValueRef FPMathInst, LLVMFastMathFlags FMF);
  * phi, and call instructions whose type is a floating point type, or a vector
  * or array thereof. See https://llvm.org/docs/LangRef.html#fast-math-flags
  */
-LLVMBool LLVMCanValueUseFastMathFlags(LLVMValueRef Inst);
+LLVM_C_ABI LLVMBool LLVMCanValueUseFastMathFlags(LLVMValueRef Inst);
 
 /**
  * Gets whether the instruction has the disjoint flag set.
  * Only valid for or instructions.
  */
-LLVMBool LLVMGetIsDisjoint(LLVMValueRef Inst);
+LLVM_C_ABI LLVMBool LLVMGetIsDisjoint(LLVMValueRef Inst);
 /**
  * Sets the disjoint flag for the instruction.
  * Only valid for or instructions.
  */
-void LLVMSetIsDisjoint(LLVMValueRef Inst, LLVMBool IsDisjoint);
+LLVM_C_ABI void LLVMSetIsDisjoint(LLVMValueRef Inst, LLVMBool IsDisjoint);
 
 /* Memory */
-LLVMValueRef LLVMBuildMalloc(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
-LLVMValueRef LLVMBuildArrayMalloc(LLVMBuilderRef, LLVMTypeRef Ty,
-                                  LLVMValueRef Val, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildMalloc(LLVMBuilderRef, LLVMTypeRef Ty,
+                                        const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildArrayMalloc(LLVMBuilderRef, LLVMTypeRef Ty,
+                                             LLVMValueRef Val,
+                                             const char *Name);
 
 /**
  * Creates and inserts a memset to the specified pointer and the
@@ -4511,202 +4663,225 @@ LLVMValueRef LLVMBuildArrayMalloc(LLVMBuilderRef, LLVMTypeRef Ty,
  *
  * @see llvm::IRRBuilder::CreateMemSet()
  */
-LLVMValueRef LLVMBuildMemSet(LLVMBuilderRef B, LLVMValueRef Ptr,
-                             LLVMValueRef Val, LLVMValueRef Len,
-                             unsigned Align);
+LLVM_C_ABI LLVMValueRef LLVMBuildMemSet(LLVMBuilderRef B, LLVMValueRef Ptr,
+                                        LLVMValueRef Val, LLVMValueRef Len,
+                                        unsigned Align);
 /**
  * Creates and inserts a memcpy between the specified pointers.
  *
  * @see llvm::IRRBuilder::CreateMemCpy()
  */
-LLVMValueRef LLVMBuildMemCpy(LLVMBuilderRef B,
-                             LLVMValueRef Dst, unsigned DstAlign,
-                             LLVMValueRef Src, unsigned SrcAlign,
-                             LLVMValueRef Size);
+LLVM_C_ABI LLVMValueRef LLVMBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst,
+                                        unsigned DstAlign, LLVMValueRef Src,
+                                        unsigned SrcAlign, LLVMValueRef Size);
 /**
  * Creates and inserts a memmove between the specified pointers.
  *
  * @see llvm::IRRBuilder::CreateMemMove()
  */
-LLVMValueRef LLVMBuildMemMove(LLVMBuilderRef B,
-                              LLVMValueRef Dst, unsigned DstAlign,
-                              LLVMValueRef Src, unsigned SrcAlign,
-                              LLVMValueRef Size);
-
-LLVMValueRef LLVMBuildAlloca(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
-LLVMValueRef LLVMBuildArrayAlloca(LLVMBuilderRef, LLVMTypeRef Ty,
-                                  LLVMValueRef Val, const char *Name);
-LLVMValueRef LLVMBuildFree(LLVMBuilderRef, LLVMValueRef PointerVal);
-LLVMValueRef LLVMBuildLoad2(LLVMBuilderRef, LLVMTypeRef Ty,
-                            LLVMValueRef PointerVal, const char *Name);
-LLVMValueRef LLVMBuildStore(LLVMBuilderRef, LLVMValueRef Val, LLVMValueRef Ptr);
-LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
-                           LLVMValueRef Pointer, LLVMValueRef *Indices,
-                           unsigned NumIndices, const char *Name);
-LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
-                                   LLVMValueRef Pointer, LLVMValueRef *Indices,
-                                   unsigned NumIndices, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildMemMove(LLVMBuilderRef B, LLVMValueRef Dst,
+                                         unsigned DstAlign, LLVMValueRef Src,
+                                         unsigned SrcAlign, LLVMValueRef Size);
+
+LLVM_C_ABI LLVMValueRef LLVMBuildAlloca(LLVMBuilderRef, LLVMTypeRef Ty,
+                                        const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildArrayAlloca(LLVMBuilderRef, LLVMTypeRef Ty,
+                                             LLVMValueRef Val,
+                                             const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFree(LLVMBuilderRef, LLVMValueRef PointerVal);
+LLVM_C_ABI LLVMValueRef LLVMBuildLoad2(LLVMBuilderRef, LLVMTypeRef Ty,
+                                       LLVMValueRef PointerVal,
+                                       const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildStore(LLVMBuilderRef, LLVMValueRef Val,
+                                       LLVMValueRef Ptr);
+LLVM_C_ABI LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                      LLVMValueRef Pointer,
+                                      LLVMValueRef *Indices,
+                                      unsigned NumIndices, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                              LLVMValueRef Pointer,
+                                              LLVMValueRef *Indices,
+                                              unsigned NumIndices,
+                                              const char *Name);
 /**
  * Creates a GetElementPtr instruction. Similar to LLVMBuildGEP2, but allows
  * specifying the no-wrap flags.
  *
  * @see llvm::IRBuilder::CreateGEP()
  */
-LLVMValueRef LLVMBuildGEPWithNoWrapFlags(LLVMBuilderRef B, LLVMTypeRef Ty,
-                                         LLVMValueRef Pointer,
-                                         LLVMValueRef *Indices,
-                                         unsigned NumIndices, const char *Name,
-                                         LLVMGEPNoWrapFlags NoWrapFlags);
-LLVMValueRef LLVMBuildStructGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
-                                 LLVMValueRef Pointer, unsigned Idx,
-                                 const char *Name);
-LLVMValueRef LLVMBuildGlobalString(LLVMBuilderRef B, const char *Str,
-                                   const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildGEPWithNoWrapFlags(
+    LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Pointer,
+    LLVMValueRef *Indices, unsigned NumIndices, const char *Name,
+    LLVMGEPNoWrapFlags NoWrapFlags);
+LLVM_C_ABI LLVMValueRef LLVMBuildStructGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                            LLVMValueRef Pointer, unsigned Idx,
+                                            const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildGlobalString(LLVMBuilderRef B, const char *Str,
+                                              const char *Name);
 /**
  * Deprecated: Use LLVMBuildGlobalString instead, which has identical behavior.
  */
-LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B, const char *Str,
-                                      const char *Name);
-LLVMBool LLVMGetVolatile(LLVMValueRef MemoryAccessInst);
-void LLVMSetVolatile(LLVMValueRef MemoryAccessInst, LLVMBool IsVolatile);
-LLVMBool LLVMGetWeak(LLVMValueRef CmpXchgInst);
-void LLVMSetWeak(LLVMValueRef CmpXchgInst, LLVMBool IsWeak);
-LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemoryAccessInst);
-void LLVMSetOrdering(LLVMValueRef MemoryAccessInst, LLVMAtomicOrdering Ordering);
-LLVMAtomicRMWBinOp LLVMGetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst);
-void LLVMSetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst, LLVMAtomicRMWBinOp BinOp);
+LLVM_C_ABI LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B,
+                                                 const char *Str,
+                                                 const char *Name);
+LLVM_C_ABI LLVMBool LLVMGetVolatile(LLVMValueRef MemoryAccessInst);
+LLVM_C_ABI void LLVMSetVolatile(LLVMValueRef MemoryAccessInst,
+                                LLVMBool IsVolatile);
+LLVM_C_ABI LLVMBool LLVMGetWeak(LLVMValueRef CmpXchgInst);
+LLVM_C_ABI void LLVMSetWeak(LLVMValueRef CmpXchgInst, LLVMBool IsWeak);
+LLVM_C_ABI LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemoryAccessInst);
+LLVM_C_ABI void LLVMSetOrdering(LLVMValueRef MemoryAccessInst,
+                                LLVMAtomicOrdering Ordering);
+LLVM_C_ABI LLVMAtomicRMWBinOp LLVMGetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst);
+LLVM_C_ABI void LLVMSetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst,
+                                      LLVMAtomicRMWBinOp BinOp);
 
 /* Casts */
-LLVMValueRef LLVMBuildTrunc(LLVMBuilderRef, LLVMValueRef Val,
-                            LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildZExt(LLVMBuilderRef, LLVMValueRef Val,
-                           LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildSExt(LLVMBuilderRef, LLVMValueRef Val,
-                           LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildFPToUI(LLVMBuilderRef, LLVMValueRef Val,
-                             LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildFPToSI(LLVMBuilderRef, LLVMValueRef Val,
-                             LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildUIToFP(LLVMBuilderRef, LLVMValueRef Val,
-                             LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildSIToFP(LLVMBuilderRef, LLVMValueRef Val,
-                             LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildFPTrunc(LLVMBuilderRef, LLVMValueRef Val,
-                              LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildFPExt(LLVMBuilderRef, LLVMValueRef Val,
-                            LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildPtrToInt(LLVMBuilderRef, LLVMValueRef Val,
-                               LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildIntToPtr(LLVMBuilderRef, LLVMValueRef Val,
-                               LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildBitCast(LLVMBuilderRef, LLVMValueRef Val,
-                              LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildAddrSpaceCast(LLVMBuilderRef, LLVMValueRef Val,
-                                    LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildZExtOrBitCast(LLVMBuilderRef, LLVMValueRef Val,
-                                    LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildSExtOrBitCast(LLVMBuilderRef, LLVMValueRef Val,
-                                    LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildTruncOrBitCast(LLVMBuilderRef, LLVMValueRef Val,
-                                     LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildCast(LLVMBuilderRef B, LLVMOpcode Op, LLVMValueRef Val,
-                           LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildPointerCast(LLVMBuilderRef, LLVMValueRef Val,
-                                  LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildIntCast2(LLVMBuilderRef, LLVMValueRef Val,
-                               LLVMTypeRef DestTy, LLVMBool IsSigned,
-                               const char *Name);
-LLVMValueRef LLVMBuildFPCast(LLVMBuilderRef, LLVMValueRef Val,
-                             LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildTrunc(LLVMBuilderRef, LLVMValueRef Val,
+                                       LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildZExt(LLVMBuilderRef, LLVMValueRef Val,
+                                      LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSExt(LLVMBuilderRef, LLVMValueRef Val,
+                                      LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFPToUI(LLVMBuilderRef, LLVMValueRef Val,
+                                        LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFPToSI(LLVMBuilderRef, LLVMValueRef Val,
+                                        LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildUIToFP(LLVMBuilderRef, LLVMValueRef Val,
+                                        LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSIToFP(LLVMBuilderRef, LLVMValueRef Val,
+                                        LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFPTrunc(LLVMBuilderRef, LLVMValueRef Val,
+                                         LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFPExt(LLVMBuilderRef, LLVMValueRef Val,
+                                       LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildPtrToInt(LLVMBuilderRef, LLVMValueRef Val,
+                                          LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildIntToPtr(LLVMBuilderRef, LLVMValueRef Val,
+                                          LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildBitCast(LLVMBuilderRef, LLVMValueRef Val,
+                                         LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildAddrSpaceCast(LLVMBuilderRef, LLVMValueRef Val,
+                                               LLVMTypeRef DestTy,
+                                               const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildZExtOrBitCast(LLVMBuilderRef, LLVMValueRef Val,
+                                               LLVMTypeRef DestTy,
+                                               const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSExtOrBitCast(LLVMBuilderRef, LLVMValueRef Val,
+                                               LLVMTypeRef DestTy,
+                                               const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildTruncOrBitCast(LLVMBuilderRef,
+                                                LLVMValueRef Val,
+                                                LLVMTypeRef DestTy,
+                                                const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildCast(LLVMBuilderRef B, LLVMOpcode Op,
+                                      LLVMValueRef Val, LLVMTypeRef DestTy,
+                                      const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildPointerCast(LLVMBuilderRef, LLVMValueRef Val,
+                                             LLVMTypeRef DestTy,
+                                             const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildIntCast2(LLVMBuilderRef, LLVMValueRef Val,
+                                          LLVMTypeRef DestTy, LLVMBool IsSigned,
+                                          const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFPCast(LLVMBuilderRef, LLVMValueRef Val,
+                                        LLVMTypeRef DestTy, const char *Name);
 
 /** Deprecated: This cast is always signed. Use LLVMBuildIntCast2 instead. */
-LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef, LLVMValueRef Val, /*Signed cast!*/
-                              LLVMTypeRef DestTy, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef,
+                                         LLVMValueRef Val, /*Signed cast!*/
+                                         LLVMTypeRef DestTy, const char *Name);
 
-LLVMOpcode LLVMGetCastOpcode(LLVMValueRef Src, LLVMBool SrcIsSigned,
-                             LLVMTypeRef DestTy, LLVMBool DestIsSigned);
+LLVM_C_ABI LLVMOpcode LLVMGetCastOpcode(LLVMValueRef Src, LLVMBool SrcIsSigned,
+                                        LLVMTypeRef DestTy,
+                                        LLVMBool DestIsSigned);
 
 /* Comparisons */
-LLVMValueRef LLVMBuildICmp(LLVMBuilderRef, LLVMIntPredicate Op,
-                           LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
-LLVMValueRef LLVMBuildFCmp(LLVMBuilderRef, LLVMRealPredicate Op,
-                           LLVMValueRef LHS, LLVMValueRef RHS,
-                           const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildICmp(LLVMBuilderRef, LLVMIntPredicate Op,
+                                      LLVMValueRef LHS, LLVMValueRef RHS,
+                                      const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFCmp(LLVMBuilderRef, LLVMRealPredicate Op,
+                                      LLVMValueRef LHS, LLVMValueRef RHS,
+                                      const char *Name);
 
 /* Miscellaneous instructions */
-LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
-LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn,
-                            LLVMValueRef *Args, unsigned NumArgs,
-                            const char *Name);
-LLVMValueRef
-LLVMBuildCallWithOperandBundles(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn,
-                                LLVMValueRef *Args, unsigned NumArgs,
-                                LLVMOperandBundleRef *Bundles,
-                                unsigned NumBundles, const char *Name);
-LLVMValueRef LLVMBuildSelect(LLVMBuilderRef, LLVMValueRef If,
-                             LLVMValueRef Then, LLVMValueRef Else,
-                             const char *Name);
-LLVMValueRef LLVMBuildVAArg(LLVMBuilderRef, LLVMValueRef List, LLVMTypeRef Ty,
-                            const char *Name);
-LLVMValueRef LLVMBuildExtractElement(LLVMBuilderRef, LLVMValueRef VecVal,
-                                     LLVMValueRef Index, const char *Name);
-LLVMValueRef LLVMBuildInsertElement(LLVMBuilderRef, LLVMValueRef VecVal,
-                                    LLVMValueRef EltVal, LLVMValueRef Index,
-                                    const char *Name);
-LLVMValueRef LLVMBuildShuffleVector(LLVMBuilderRef, LLVMValueRef V1,
-                                    LLVMValueRef V2, LLVMValueRef Mask,
-                                    const char *Name);
-LLVMValueRef LLVMBuildExtractValue(LLVMBuilderRef, LLVMValueRef AggVal,
-                                   unsigned Index, const char *Name);
-LLVMValueRef LLVMBuildInsertValue(LLVMBuilderRef, LLVMValueRef AggVal,
-                                  LLVMValueRef EltVal, unsigned Index,
-                                  const char *Name);
-LLVMValueRef LLVMBuildFreeze(LLVMBuilderRef, LLVMValueRef Val,
-                             const char *Name);
-
-LLVMValueRef LLVMBuildIsNull(LLVMBuilderRef, LLVMValueRef Val,
-                             const char *Name);
-LLVMValueRef LLVMBuildIsNotNull(LLVMBuilderRef, LLVMValueRef Val,
-                                const char *Name);
-LLVMValueRef LLVMBuildPtrDiff2(LLVMBuilderRef, LLVMTypeRef ElemTy,
-                               LLVMValueRef LHS, LLVMValueRef RHS,
-                               const char *Name);
-LLVMValueRef LLVMBuildFence(LLVMBuilderRef B, LLVMAtomicOrdering ordering,
-                            LLVMBool singleThread, const char *Name);
-LLVMValueRef LLVMBuildFenceSyncScope(LLVMBuilderRef B,
-                                     LLVMAtomicOrdering ordering, unsigned SSID,
+LLVM_C_ABI LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty,
                                      const char *Name);
-LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B, LLVMAtomicRMWBinOp op,
-                                LLVMValueRef PTR, LLVMValueRef Val,
-                                LLVMAtomicOrdering ordering,
-                                LLVMBool singleThread);
-LLVMValueRef LLVMBuildAtomicRMWSyncScope(LLVMBuilderRef B,
-                                         LLVMAtomicRMWBinOp op,
-                                         LLVMValueRef PTR, LLVMValueRef Val,
-                                         LLVMAtomicOrdering ordering,
-                                         unsigned SSID);
-LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr,
-                                    LLVMValueRef Cmp, LLVMValueRef New,
-                                    LLVMAtomicOrdering SuccessOrdering,
-                                    LLVMAtomicOrdering FailureOrdering,
-                                    LLVMBool SingleThread);
-LLVMValueRef LLVMBuildAtomicCmpXchgSyncScope(LLVMBuilderRef B, LLVMValueRef Ptr,
-                                             LLVMValueRef Cmp, LLVMValueRef New,
-                                             LLVMAtomicOrdering SuccessOrdering,
-                                             LLVMAtomicOrdering FailureOrdering,
-                                             unsigned SSID);
+LLVM_C_ABI LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef,
+                                       LLVMValueRef Fn, LLVMValueRef *Args,
+                                       unsigned NumArgs, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildCallWithOperandBundles(
+    LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn, LLVMValueRef *Args,
+    unsigned NumArgs, LLVMOperandBundleRef *Bundles, unsigned NumBundles,
+    const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildSelect(LLVMBuilderRef, LLVMValueRef If,
+                                        LLVMValueRef Then, LLVMValueRef Else,
+                                        const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildVAArg(LLVMBuilderRef, LLVMValueRef List,
+                                       LLVMTypeRef Ty, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildExtractElement(LLVMBuilderRef,
+                                                LLVMValueRef VecVal,
+                                                LLVMValueRef Index,
+                                                const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildInsertElement(LLVMBuilderRef,
+                                               LLVMValueRef VecVal,
+                                               LLVMValueRef EltVal,
+                                               LLVMValueRef Index,
+                                               const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildShuffleVector(LLVMBuilderRef, LLVMValueRef V1,
+                                               LLVMValueRef V2,
+                                               LLVMValueRef Mask,
+                                               const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildExtractValue(LLVMBuilderRef,
+                                              LLVMValueRef AggVal,
+                                              unsigned Index, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildInsertValue(LLVMBuilderRef,
+                                             LLVMValueRef AggVal,
+                                             LLVMValueRef EltVal,
+                                             unsigned Index, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFreeze(LLVMBuilderRef, LLVMValueRef Val,
+                                        const char *Name);
+
+LLVM_C_ABI LLVMValueRef LLVMBuildIsNull(LLVMBuilderRef, LLVMValueRef Val,
+                                        const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildIsNotNull(LLVMBuilderRef, LLVMValueRef Val,
+                                           const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildPtrDiff2(LLVMBuilderRef, LLVMTypeRef ElemTy,
+                                          LLVMValueRef LHS, LLVMValueRef RHS,
+                                          const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFence(LLVMBuilderRef B,
+                                       LLVMAtomicOrdering ordering,
+                                       LLVMBool singleThread, const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildFenceSyncScope(LLVMBuilderRef B,
+                                                LLVMAtomicOrdering ordering,
+                                                unsigned SSID,
+                                                const char *Name);
+LLVM_C_ABI LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,
+                                           LLVMAtomicRMWBinOp op,
+                                           LLVMValueRef PTR, LLVMValueRef Val,
+                                           LLVMAtomicOrdering ordering,
+                                           LLVMBool singleThread);
+LLVM_C_ABI LLVMValueRef LLVMBuildAtomicRMWSyncScope(
+    LLVMBuilderRef B, LLVMAtomicRMWBinOp op, LLVMValueRef PTR, LLVMValueRef Val,
+    LLVMAtomicOrdering ordering, unsigned SSID);
+LLVM_C_ABI LLVMValueRef LLVMBuildAtomicCmpXchg(
+    LLVMBuilderRef B, LLVMValueRef Ptr, LLVMValueRef Cmp, LLVMValueRef New,
+    LLVMAtomicOrdering SuccessOrdering, LLVMAtomicOrdering FailureOrdering,
+    LLVMBool SingleThread);
+LLVM_C_ABI LLVMValueRef LLVMBuildAtomicCmpXchgSyncScope(
+    LLVMBuilderRef B, LLVMValueRef Ptr, LLVMValueRef Cmp, LLVMValueRef New,
+    LLVMAtomicOrdering SuccessOrdering, LLVMAtomicOrdering FailureOrdering,
+    unsigned SSID);
 
 /**
  * Get the number of elements in the mask of a ShuffleVector instruction.
  */
-unsigned LLVMGetNumMaskElements(LLVMValueRef ShuffleVectorInst);
+LLVM_C_ABI unsigned LLVMGetNumMaskElements(LLVMValueRef ShuffleVectorInst);
 
 /**
  * \returns a constant that specifies that the result of a \c ShuffleVectorInst
  * is undefined.
  */
-int LLVMGetUndefMaskElem(void);
+LLVM_C_ABI int LLVMGetUndefMaskElem(void);
 
 /**
  * Get the mask value at position Elt in the mask of a ShuffleVector
@@ -4715,33 +4890,37 @@ int LLVMGetUndefMaskElem(void);
  * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is
  * poison at that position.
  */
-int LLVMGetMaskValue(LLVMValueRef ShuffleVectorInst, unsigned Elt);
+LLVM_C_ABI int LLVMGetMaskValue(LLVMValueRef ShuffleVectorInst, unsigned Elt);
 
-LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst);
-void LLVMSetAtomicSingleThread(LLVMValueRef AtomicInst, LLVMBool SingleThread);
+LLVM_C_ABI LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst);
+LLVM_C_ABI void LLVMSetAtomicSingleThread(LLVMValueRef AtomicInst,
+                                          LLVMBool SingleThread);
 
 /**
  * Returns whether an instruction is an atomic instruction, e.g., atomicrmw,
  * cmpxchg, fence, or loads and stores with atomic ordering.
  */
-LLVMBool LLVMIsAtomic(LLVMValueRef Inst);
+LLVM_C_ABI LLVMBool LLVMIsAtomic(LLVMValueRef Inst);
 
 /**
  * Returns the synchronization scope ID of an atomic instruction.
  */
-unsigned LLVMGetAtomicSyncScopeID(LLVMValueRef AtomicInst);
+LLVM_C_ABI unsigned LLVMGetAtomicSyncScopeID(LLVMValueRef AtomicInst);
 
 /**
  * Sets the synchronization scope ID of an atomic instruction.
  */
-void LLVMSetAtomicSyncScopeID(LLVMValueRef AtomicInst, unsigned SSID);
+LLVM_C_ABI void LLVMSetAtomicSyncScopeID(LLVMValueRef AtomicInst,
+                                         unsigned SSID);
 
-LLVMAtomicOrdering LLVMGetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst);
-void LLVMSetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst,
-                                   LLVMAtomicOrdering Ordering);
-LLVMAtomicOrdering LLVMGetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst);
-void LLVMSetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst,
-                                   LLVMAtomicOrdering Ordering);
+LLVM_C_ABI LLVMAtomicOrdering
+LLVMGetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst);
+LLVM_C_ABI void LLVMSetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst,
+                                              LLVMAtomicOrdering Ordering);
+LLVM_C_ABI LLVMAtomicOrdering
+LLVMGetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst);
+LLVM_C_ABI void LLVMSetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst,
+                                              LLVMAtomicOrdering Ordering);
 
 /**
  * @}
@@ -4757,13 +4936,13 @@ void LLVMSetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst,
  * Changes the type of M so it can be passed to FunctionPassManagers and the
  * JIT.  They take ModuleProviders for historical reasons.
  */
-LLVMModuleProviderRef
+LLVM_C_ABI LLVMModuleProviderRef
 LLVMCreateModuleProviderForExistingModule(LLVMModuleRef M);
 
 /**
  * Destroys the module M.
  */
-void LLVMDisposeModuleProvider(LLVMModuleProviderRef M);
+LLVM_C_ABI void LLVMDisposeModuleProvider(LLVMModuleProviderRef M);
 
 /**
  * @}
@@ -4775,21 +4954,18 @@ void LLVMDisposeModuleProvider(LLVMModuleProviderRef M);
  * @{
  */
 
-LLVMBool LLVMCreateMemoryBufferWithContentsOfFile(const char *Path,
-                                                  LLVMMemoryBufferRef *OutMemBuf,
-                                                  char **OutMessage);
-LLVMBool LLVMCreateMemoryBufferWithSTDIN(LLVMMemoryBufferRef *OutMemBuf,
-                                         char **OutMessage);
-LLVMMemoryBufferRef LLVMCreateMemoryBufferWithMemoryRange(const char *InputData,
-                                                          size_t InputDataLength,
-                                                          const char *BufferName,
-                                                          LLVMBool RequiresNullTerminator);
-LLVMMemoryBufferRef LLVMCreateMemoryBufferWithMemoryRangeCopy(const char *InputData,
-                                                              size_t InputDataLength,
-                                                              const char *BufferName);
-const char *LLVMGetBufferStart(LLVMMemoryBufferRef MemBuf);
-size_t LLVMGetBufferSize(LLVMMemoryBufferRef MemBuf);
-void LLVMDisposeMemoryBuffer(LLVMMemoryBufferRef MemBuf);
+LLVM_C_ABI LLVMBool LLVMCreateMemoryBufferWithContentsOfFile(
+    const char *Path, LLVMMemoryBufferRef *OutMemBuf, char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMCreateMemoryBufferWithSTDIN(
+    LLVMMemoryBufferRef *OutMemBuf, char **OutMessage);
+LLVM_C_ABI LLVMMemoryBufferRef LLVMCreateMemoryBufferWithMemoryRange(
+    const char *InputData, size_t InputDataLength, const char *BufferName,
+    LLVMBool RequiresNullTerminator);
+LLVM_C_ABI LLVMMemoryBufferRef LLVMCreateMemoryBufferWithMemoryRangeCopy(
+    const char *InputData, size_t InputDataLength, const char *BufferName);
+LLVM_C_ABI const char *LLVMGetBufferStart(LLVMMemoryBufferRef MemBuf);
+LLVM_C_ABI size_t LLVMGetBufferSize(LLVMMemoryBufferRef MemBuf);
+LLVM_C_ABI void LLVMDisposeMemoryBuffer(LLVMMemoryBufferRef MemBuf);
 
 /**
  * @}
@@ -4805,43 +4981,46 @@ void LLVMDisposeMemoryBuffer(LLVMMemoryBufferRef MemBuf);
 /** Constructs a new whole-module pass pipeline. This type of pipeline is
     suitable for link-time optimization and whole-module transformations.
     @see llvm::PassManager::PassManager */
-LLVMPassManagerRef LLVMCreatePassManager(void);
+LLVM_C_ABI LLVMPassManagerRef LLVMCreatePassManager(void);
 
 /** Constructs a new function-by-function pass pipeline over the module
     provider. It does not take ownership of the module provider. This type of
     pipeline is suitable for code generation and JIT compilation tasks.
     @see llvm::FunctionPassManager::FunctionPassManager */
-LLVMPassManagerRef LLVMCreateFunctionPassManagerForModule(LLVMModuleRef M);
+LLVM_C_ABI LLVMPassManagerRef
+LLVMCreateFunctionPassManagerForModule(LLVMModuleRef M);
 
 /** Deprecated: Use LLVMCreateFunctionPassManagerForModule instead. */
-LLVMPassManagerRef LLVMCreateFunctionPassManager(LLVMModuleProviderRef MP);
+LLVM_C_ABI LLVMPassManagerRef
+LLVMCreateFunctionPassManager(LLVMModuleProviderRef MP);
 
 /** Initializes, executes on the provided module, and finalizes all of the
     passes scheduled in the pass manager. Returns 1 if any of the passes
     modified the module, 0 otherwise.
     @see llvm::PassManager::run(Module&) */
-LLVMBool LLVMRunPassManager(LLVMPassManagerRef PM, LLVMModuleRef M);
+LLVM_C_ABI LLVMBool LLVMRunPassManager(LLVMPassManagerRef PM, LLVMModuleRef M);
 
 /** Initializes all of the function passes scheduled in the function pass
     manager. Returns 1 if any of the passes modified the module, 0 otherwise.
     @see llvm::FunctionPassManager::doInitialization */
-LLVMBool LLVMInitializeFunctionPassManager(LLVMPassManagerRef FPM);
+LLVM_C_ABI LLVMBool LLVMInitializeFunctionPassManager(LLVMPassManagerRef FPM);
 
 /** Executes all of the function passes scheduled in the function pass manager
     on the provided function. Returns 1 if any of the passes modified the
     function, false otherwise.
     @see llvm::FunctionPassManager::run(Function&) */
-LLVMBool LLVMRunFunctionPassManager(LLVMPassManagerRef FPM, LLVMValueRef F);
+LLVM_C_ABI LLVMBool LLVMRunFunctionPassManager(LLVMPassManagerRef FPM,
+                                               LLVMValueRef F);
 
 /** Finalizes all of the function passes scheduled in the function pass
     manager. Returns 1 if any of the passes modified the module, 0 otherwise.
     @see llvm::FunctionPassManager::doFinalization */
-LLVMBool LLVMFinalizeFunctionPassManager(LLVMPassManagerRef FPM);
+LLVM_C_ABI LLVMBool LLVMFinalizeFunctionPassManager(LLVMPassManagerRef FPM);
 
 /** Frees the memory of a pass pipeline. For function pipelines, does not free
     the module provider.
     @see llvm::PassManagerBase::~PassManagerBase. */
-void LLVMDisposePassManager(LLVMPassManagerRef PM);
+LLVM_C_ABI void LLVMDisposePassManager(LLVMPassManagerRef PM);
 
 /**
  * @}
@@ -4858,15 +5037,15 @@ void LLVMDisposePassManager(LLVMPassManagerRef PM);
 /** Deprecated: Multi-threading can only be enabled/disabled with the compile
     time define LLVM_ENABLE_THREADS.  This function always returns
     LLVMIsMultithreaded(). */
-LLVMBool LLVMStartMultithreaded(void);
+LLVM_C_ABI LLVMBool LLVMStartMultithreaded(void);
 
 /** Deprecated: Multi-threading can only be enabled/disabled with the compile
     time define LLVM_ENABLE_THREADS. */
-void LLVMStopMultithreaded(void);
+LLVM_C_ABI void LLVMStopMultithreaded(void);
 
 /** Check whether LLVM is executing in thread-safe mode or not.
     @see llvm::llvm_is_multithreaded */
-LLVMBool LLVMIsMultithreaded(void);
+LLVM_C_ABI LLVMBool LLVMIsMultithreaded(void);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h
index ef89a88617bea..7d1caa6438906 100644
--- a/llvm/include/llvm-c/DebugInfo.h
+++ b/llvm/include/llvm-c/DebugInfo.h
@@ -18,6 +18,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -223,12 +224,12 @@ typedef enum {
 /**
  * The current debug metadata version number.
  */
-unsigned LLVMDebugMetadataVersion(void);
+LLVM_C_ABI unsigned LLVMDebugMetadataVersion(void);
 
 /**
  * The version of debug metadata that's present in the provided \c Module.
  */
-unsigned LLVMGetModuleDebugMetadataVersion(LLVMModuleRef Module);
+LLVM_C_ABI unsigned LLVMGetModuleDebugMetadataVersion(LLVMModuleRef Module);
 
 /**
  * Strip debug info in the module if it exists.
@@ -236,38 +237,39 @@ unsigned LLVMGetModuleDebugMetadataVersion(LLVMModuleRef Module);
  * metadata for debugging. We also remove debug locations for instructions.
  * Return true if module is modified.
  */
-LLVMBool LLVMStripModuleDebugInfo(LLVMModuleRef Module);
+LLVM_C_ABI LLVMBool LLVMStripModuleDebugInfo(LLVMModuleRef Module);
 
 /**
  * Construct a builder for a module, and do not allow for unresolved nodes
  * attached to the module.
  */
-LLVMDIBuilderRef LLVMCreateDIBuilderDisallowUnresolved(LLVMModuleRef M);
+LLVM_C_ABI LLVMDIBuilderRef
+LLVMCreateDIBuilderDisallowUnresolved(LLVMModuleRef M);
 
 /**
  * Construct a builder for a module and collect unresolved nodes attached
  * to the module in order to resolve cycles during a call to
  * \c LLVMDIBuilderFinalize.
  */
-LLVMDIBuilderRef LLVMCreateDIBuilder(LLVMModuleRef M);
+LLVM_C_ABI LLVMDIBuilderRef LLVMCreateDIBuilder(LLVMModuleRef M);
 
 /**
  * Deallocates the \c DIBuilder and everything it owns.
  * @note You must call \c LLVMDIBuilderFinalize before this
  */
-void LLVMDisposeDIBuilder(LLVMDIBuilderRef Builder);
+LLVM_C_ABI void LLVMDisposeDIBuilder(LLVMDIBuilderRef Builder);
 
 /**
  * Construct any deferred debug info descriptors.
  */
-void LLVMDIBuilderFinalize(LLVMDIBuilderRef Builder);
+LLVM_C_ABI void LLVMDIBuilderFinalize(LLVMDIBuilderRef Builder);
 
 /**
  * Finalize a specific subprogram.
  * No new variables may be added to this subprogram afterwards.
  */
-void LLVMDIBuilderFinalizeSubprogram(LLVMDIBuilderRef Builder,
-                                     LLVMMetadataRef Subprogram);
+LLVM_C_ABI void LLVMDIBuilderFinalizeSubprogram(LLVMDIBuilderRef Builder,
+                                                LLVMMetadataRef Subprogram);
 
 /**
  * A CompileUnit provides an anchor for all debugging
@@ -301,7 +303,7 @@ void LLVMDIBuilderFinalizeSubprogram(LLVMDIBuilderRef Builder,
  * \param SDK           The SDK. On Darwin, the last component of the sysroot.
  * \param SDKLen        The length of the C string passed to \c SDK.
  */
-LLVMMetadataRef LLVMDIBuilderCreateCompileUnit(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateCompileUnit(
     LLVMDIBuilderRef Builder, LLVMDWARFSourceLanguage Lang,
     LLVMMetadataRef FileRef, const char *Producer, size_t ProducerLen,
     LLVMBool isOptimized, const char *Flags, size_t FlagsLen,
@@ -318,10 +320,11 @@ LLVMMetadataRef LLVMDIBuilderCreateCompileUnit(
  * \param Directory    Directory.
  * \param DirectoryLen The length of the C string passed to \c Directory.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateFile(LLVMDIBuilderRef Builder, const char *Filename,
-                        size_t FilenameLen, const char *Directory,
-                        size_t DirectoryLen);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateFile(LLVMDIBuilderRef Builder,
+                                                   const char *Filename,
+                                                   size_t FilenameLen,
+                                                   const char *Directory,
+                                                   size_t DirectoryLen);
 
 /**
  * Creates a new descriptor for a module with the specified parent scope.
@@ -337,12 +340,11 @@ LLVMDIBuilderCreateFile(LLVMDIBuilderRef Builder, const char *Filename,
  * \param APINotesFile    The path to an API notes file for the module.
  * \param APINotesFileLen The length of the C string passed to \c APINotestFile.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateModule(LLVMDIBuilderRef Builder, LLVMMetadataRef ParentScope,
-                          const char *Name, size_t NameLen,
-                          const char *ConfigMacros, size_t ConfigMacrosLen,
-                          const char *IncludePath, size_t IncludePathLen,
-                          const char *APINotesFile, size_t APINotesFileLen);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateModule(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef ParentScope, const char *Name,
+    size_t NameLen, const char *ConfigMacros, size_t ConfigMacrosLen,
+    const char *IncludePath, size_t IncludePathLen, const char *APINotesFile,
+    size_t APINotesFileLen);
 
 /**
  * Creates a new descriptor for a namespace with the specified parent scope.
@@ -353,11 +355,9 @@ LLVMDIBuilderCreateModule(LLVMDIBuilderRef Builder, LLVMMetadataRef ParentScope,
  * \param ExportSymbols    Whether or not the namespace exports symbols, e.g.
  *                         this is true of C++ inline namespaces.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateNameSpace(LLVMDIBuilderRef Builder,
-                             LLVMMetadataRef ParentScope,
-                             const char *Name, size_t NameLen,
-                             LLVMBool ExportSymbols);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateNameSpace(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef ParentScope, const char *Name,
+    size_t NameLen, LLVMBool ExportSymbols);
 
 /**
  * Create a new descriptor for the specified subprogram.
@@ -377,12 +377,12 @@ LLVMDIBuilderCreateNameSpace(LLVMDIBuilderRef Builder,
  *                        used to emit dwarf attributes.
  * \param IsOptimized     True if optimization is ON.
  */
-LLVMMetadataRef LLVMDIBuilderCreateFunction(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateFunction(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, const char *LinkageName, size_t LinkageNameLen,
     LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty,
-    LLVMBool IsLocalToUnit, LLVMBool IsDefinition,
-    unsigned ScopeLine, LLVMDIFlags Flags, LLVMBool IsOptimized);
+    LLVMBool IsLocalToUnit, LLVMBool IsDefinition, unsigned ScopeLine,
+    LLVMDIFlags Flags, LLVMBool IsOptimized);
 
 /**
  * Create a descriptor for a lexical block with the specified parent context.
@@ -392,9 +392,9 @@ LLVMMetadataRef LLVMDIBuilderCreateFunction(
  * \param Line         The line in the source file.
  * \param Column       The column in the source file.
  */
-LLVMMetadataRef LLVMDIBuilderCreateLexicalBlock(
-    LLVMDIBuilderRef Builder, LLVMMetadataRef Scope,
-    LLVMMetadataRef File, unsigned Line, unsigned Column);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateLexicalBlock(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef File,
+    unsigned Line, unsigned Column);
 
 /**
  * Create a descriptor for a lexical block with a new file attached.
@@ -403,11 +403,9 @@ LLVMMetadataRef LLVMDIBuilderCreateLexicalBlock(
  * \param File           Source file.
  * \param Discriminator  DWARF path discriminator value.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateLexicalBlockFile(LLVMDIBuilderRef Builder,
-                                    LLVMMetadataRef Scope,
-                                    LLVMMetadataRef File,
-                                    unsigned Discriminator);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateLexicalBlockFile(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef File,
+    unsigned Discriminator);
 
 /**
  * Create a descriptor for an imported namespace. Suitable for e.g. C++
@@ -417,12 +415,9 @@ LLVMDIBuilderCreateLexicalBlockFile(LLVMDIBuilderRef Builder,
  * \param File       File where the declaration is located.
  * \param Line       Line number of the declaration.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateImportedModuleFromNamespace(LLVMDIBuilderRef Builder,
-                                               LLVMMetadataRef Scope,
-                                               LLVMMetadataRef NS,
-                                               LLVMMetadataRef File,
-                                               unsigned Line);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromNamespace(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef NS,
+    LLVMMetadataRef File, unsigned Line);
 
 /**
  * Create a descriptor for an imported module that aliases another
@@ -435,7 +430,7 @@ LLVMDIBuilderCreateImportedModuleFromNamespace(LLVMDIBuilderRef Builder,
  * \param Elements       Renamed elements.
  * \param NumElements    Number of renamed elements.
  */
-LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromAlias(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromAlias(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope,
     LLVMMetadataRef ImportedEntity, LLVMMetadataRef File, unsigned Line,
     LLVMMetadataRef *Elements, unsigned NumElements);
@@ -450,7 +445,7 @@ LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromAlias(
  * \param Elements       Renamed elements.
  * \param NumElements    Number of renamed elements.
  */
-LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromModule(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromModule(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef M,
     LLVMMetadataRef File, unsigned Line, LLVMMetadataRef *Elements,
     unsigned NumElements);
@@ -470,7 +465,7 @@ LLVMMetadataRef LLVMDIBuilderCreateImportedModuleFromModule(
  * \param Elements       Renamed elements.
  * \param NumElements    Number of renamed elements.
  */
-LLVMMetadataRef LLVMDIBuilderCreateImportedDeclaration(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateImportedDeclaration(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef Decl,
     LLVMMetadataRef File, unsigned Line, const char *Name, size_t NameLen,
     LLVMMetadataRef *Elements, unsigned NumElements);
@@ -485,10 +480,9 @@ LLVMMetadataRef LLVMDIBuilderCreateImportedDeclaration(
  * \note If the item to which this location is attached cannot be
  *       attributed to a source line, pass 0 for the line and column.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateDebugLocation(LLVMContextRef Ctx, unsigned Line,
-                                 unsigned Column, LLVMMetadataRef Scope,
-                                 LLVMMetadataRef InlinedAt);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateDebugLocation(
+    LLVMContextRef Ctx, unsigned Line, unsigned Column, LLVMMetadataRef Scope,
+    LLVMMetadataRef InlinedAt);
 
 /**
  * Get the line number of this debug location.
@@ -496,7 +490,7 @@ LLVMDIBuilderCreateDebugLocation(LLVMContextRef Ctx, unsigned Line,
  *
  * @see DILocation::getLine()
  */
-unsigned LLVMDILocationGetLine(LLVMMetadataRef Location);
+LLVM_C_ABI unsigned LLVMDILocationGetLine(LLVMMetadataRef Location);
 
 /**
  * Get the column number of this debug location.
@@ -504,7 +498,7 @@ unsigned LLVMDILocationGetLine(LLVMMetadataRef Location);
  *
  * @see DILocation::getColumn()
  */
-unsigned LLVMDILocationGetColumn(LLVMMetadataRef Location);
+LLVM_C_ABI unsigned LLVMDILocationGetColumn(LLVMMetadataRef Location);
 
 /**
  * Get the local scope associated with this debug location.
@@ -512,7 +506,7 @@ unsigned LLVMDILocationGetColumn(LLVMMetadataRef Location);
  *
  * @see DILocation::getScope()
  */
-LLVMMetadataRef LLVMDILocationGetScope(LLVMMetadataRef Location);
+LLVM_C_ABI LLVMMetadataRef LLVMDILocationGetScope(LLVMMetadataRef Location);
 
 /**
  * Get the "inline at" location associated with this debug location.
@@ -520,7 +514,7 @@ LLVMMetadataRef LLVMDILocationGetScope(LLVMMetadataRef Location);
  *
  * @see DILocation::getInlinedAt()
  */
-LLVMMetadataRef LLVMDILocationGetInlinedAt(LLVMMetadataRef Location);
+LLVM_C_ABI LLVMMetadataRef LLVMDILocationGetInlinedAt(LLVMMetadataRef Location);
 
 /**
  * Get the metadata of the file associated with a given scope.
@@ -528,7 +522,7 @@ LLVMMetadataRef LLVMDILocationGetInlinedAt(LLVMMetadataRef Location);
  *
  * @see DIScope::getFile()
  */
-LLVMMetadataRef LLVMDIScopeGetFile(LLVMMetadataRef Scope);
+LLVM_C_ABI LLVMMetadataRef LLVMDIScopeGetFile(LLVMMetadataRef Scope);
 
 /**
  * Get the directory of a given file.
@@ -537,7 +531,8 @@ LLVMMetadataRef LLVMDIScopeGetFile(LLVMMetadataRef Scope);
  *
  * @see DIFile::getDirectory()
  */
-const char *LLVMDIFileGetDirectory(LLVMMetadataRef File, unsigned *Len);
+LLVM_C_ABI const char *LLVMDIFileGetDirectory(LLVMMetadataRef File,
+                                              unsigned *Len);
 
 /**
  * Get the name of a given file.
@@ -546,7 +541,8 @@ const char *LLVMDIFileGetDirectory(LLVMMetadataRef File, unsigned *Len);
  *
  * @see DIFile::getFilename()
  */
-const char *LLVMDIFileGetFilename(LLVMMetadataRef File, unsigned *Len);
+LLVM_C_ABI const char *LLVMDIFileGetFilename(LLVMMetadataRef File,
+                                             unsigned *Len);
 
 /**
  * Get the source of a given file.
@@ -555,7 +551,7 @@ const char *LLVMDIFileGetFilename(LLVMMetadataRef File, unsigned *Len);
  *
  * @see DIFile::getSource()
  */
-const char *LLVMDIFileGetSource(LLVMMetadataRef File, unsigned *Len);
+LLVM_C_ABI const char *LLVMDIFileGetSource(LLVMMetadataRef File, unsigned *Len);
 
 /**
  * Create a type array.
@@ -563,9 +559,8 @@ const char *LLVMDIFileGetSource(LLVMMetadataRef File, unsigned *Len);
  * \param Data           The type elements.
  * \param NumElements    Number of type elements.
  */
-LLVMMetadataRef LLVMDIBuilderGetOrCreateTypeArray(LLVMDIBuilderRef Builder,
-                                                  LLVMMetadataRef *Data,
-                                                  size_t NumElements);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderGetOrCreateTypeArray(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef *Data, size_t NumElements);
 
 /**
  * Create subroutine type.
@@ -577,12 +572,10 @@ LLVMMetadataRef LLVMDIBuilderGetOrCreateTypeArray(LLVMDIBuilderRef Builder,
  * \param Flags           E.g.: \c LLVMDIFlagLValueReference.
  *                        These flags are used to emit dwarf attributes.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateSubroutineType(LLVMDIBuilderRef Builder,
-                                  LLVMMetadataRef File,
-                                  LLVMMetadataRef *ParameterTypes,
-                                  unsigned NumParameterTypes,
-                                  LLVMDIFlags Flags);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateSubroutineType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef File,
+    LLVMMetadataRef *ParameterTypes, unsigned NumParameterTypes,
+    LLVMDIFlags Flags);
 
 /**
  * Create debugging information entry for a macro.
@@ -595,12 +588,10 @@ LLVMDIBuilderCreateSubroutineType(LLVMDIBuilderRef Builder,
  * @param Value           Macro value.
  * @param ValueLen        Macro value length.
  */
-LLVMMetadataRef LLVMDIBuilderCreateMacro(LLVMDIBuilderRef Builder,
-                                         LLVMMetadataRef ParentMacroFile,
-                                         unsigned Line,
-                                         LLVMDWARFMacinfoRecordType RecordType,
-                                         const char *Name, size_t NameLen,
-                                         const char *Value, size_t ValueLen);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateMacro(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef ParentMacroFile, unsigned Line,
+    LLVMDWARFMacinfoRecordType RecordType, const char *Name, size_t NameLen,
+    const char *Value, size_t ValueLen);
 
 /**
  * Create debugging information temporary entry for a macro file.
@@ -611,10 +602,9 @@ LLVMMetadataRef LLVMDIBuilderCreateMacro(LLVMDIBuilderRef Builder,
  * @param Line            Source line number where the macro file is included.
  * @param File            File descriptor containing the name of the macro file.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateTempMacroFile(LLVMDIBuilderRef Builder,
-                                 LLVMMetadataRef ParentMacroFile, unsigned Line,
-                                 LLVMMetadataRef File);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateTempMacroFile(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef ParentMacroFile, unsigned Line,
+    LLVMMetadataRef File);
 
 /**
  * Create debugging information entry for an enumerator.
@@ -624,10 +614,9 @@ LLVMDIBuilderCreateTempMacroFile(LLVMDIBuilderRef Builder,
  * @param Value          Enumerator value.
  * @param IsUnsigned     True if the value is unsigned.
  */
-LLVMMetadataRef LLVMDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder,
-                                              const char *Name, size_t NameLen,
-                                              int64_t Value,
-                                              LLVMBool IsUnsigned);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateEnumerator(
+    LLVMDIBuilderRef Builder, const char *Name, size_t NameLen, int64_t Value,
+    LLVMBool IsUnsigned);
 
 /**
  * Create debugging information entry for an enumerator of arbitrary precision.
@@ -638,7 +627,7 @@ LLVMMetadataRef LLVMDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder,
  * @param Words          The words that make up the value.
  * @param IsUnsigned     True if the value is unsigned.
  */
-LLVMMetadataRef LLVMDIBuilderCreateEnumeratorOfArbitraryPrecision(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateEnumeratorOfArbitraryPrecision(
     LLVMDIBuilderRef Builder, const char *Name, size_t NameLen,
     uint64_t SizeInBits, const uint64_t Words[], LLVMBool IsUnsigned);
 
@@ -656,7 +645,7 @@ LLVMMetadataRef LLVMDIBuilderCreateEnumeratorOfArbitraryPrecision(
  * \param NumElements    Number of enumeration elements.
  * \param ClassTy        Underlying type of a C++11/ObjC fixed enum.
  */
-LLVMMetadataRef LLVMDIBuilderCreateEnumerationType(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateEnumerationType(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, LLVMMetadataRef File, unsigned LineNumber,
     uint64_t SizeInBits, uint32_t AlignInBits, LLVMMetadataRef *Elements,
@@ -679,14 +668,13 @@ LLVMMetadataRef LLVMDIBuilderCreateEnumerationType(
  * \param UniqueId     A unique identifier for the union.
  * \param UniqueIdLen  Length of unique identifier.
  */
-LLVMMetadataRef LLVMDIBuilderCreateUnionType(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateUnionType(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, LLVMMetadataRef File, unsigned LineNumber,
     uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags,
     LLVMMetadataRef *Elements, unsigned NumElements, unsigned RunTimeLang,
     const char *UniqueId, size_t UniqueIdLen);
 
-
 /**
  * Create debugging information entry for an array.
  * \param Builder      The DIBuilder.
@@ -696,11 +684,9 @@ LLVMMetadataRef LLVMDIBuilderCreateUnionType(
  * \param Subscripts   Subscripts.
  * \param NumSubscripts Number of subscripts.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateArrayType(LLVMDIBuilderRef Builder, uint64_t Size,
-                             uint32_t AlignInBits, LLVMMetadataRef Ty,
-                             LLVMMetadataRef *Subscripts,
-                             unsigned NumSubscripts);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateArrayType(
+    LLVMDIBuilderRef Builder, uint64_t Size, uint32_t AlignInBits,
+    LLVMMetadataRef Ty, LLVMMetadataRef *Subscripts, unsigned NumSubscripts);
 
 /**
  * Create debugging information entry for a vector type.
@@ -711,11 +697,9 @@ LLVMDIBuilderCreateArrayType(LLVMDIBuilderRef Builder, uint64_t Size,
  * \param Subscripts   Subscripts.
  * \param NumSubscripts Number of subscripts.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateVectorType(LLVMDIBuilderRef Builder, uint64_t Size,
-                              uint32_t AlignInBits, LLVMMetadataRef Ty,
-                              LLVMMetadataRef *Subscripts,
-                              unsigned NumSubscripts);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateVectorType(
+    LLVMDIBuilderRef Builder, uint64_t Size, uint32_t AlignInBits,
+    LLVMMetadataRef Ty, LLVMMetadataRef *Subscripts, unsigned NumSubscripts);
 
 /**
  * Create a DWARF unspecified type.
@@ -723,9 +707,8 @@ LLVMDIBuilderCreateVectorType(LLVMDIBuilderRef Builder, uint64_t Size,
  * \param Name      The unspecified type's name.
  * \param NameLen   Length of type name.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateUnspecifiedType(LLVMDIBuilderRef Builder, const char *Name,
-                                   size_t NameLen);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateUnspecifiedType(
+    LLVMDIBuilderRef Builder, const char *Name, size_t NameLen);
 
 /**
  * Create debugging information entry for a basic
@@ -737,11 +720,9 @@ LLVMDIBuilderCreateUnspecifiedType(LLVMDIBuilderRef Builder, const char *Name,
  * \param Encoding    DWARF encoding code, e.g. \c LLVMDWARFTypeEncoding_float.
  * \param Flags       Flags to encode optional attribute like endianity
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateBasicType(LLVMDIBuilderRef Builder, const char *Name,
-                             size_t NameLen, uint64_t SizeInBits,
-                             LLVMDWARFTypeEncoding Encoding,
-                             LLVMDIFlags Flags);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateBasicType(
+    LLVMDIBuilderRef Builder, const char *Name, size_t NameLen,
+    uint64_t SizeInBits, LLVMDWARFTypeEncoding Encoding, LLVMDIFlags Flags);
 
 /**
  * Create debugging information entry for a pointer.
@@ -753,10 +734,10 @@ LLVMDIBuilderCreateBasicType(LLVMDIBuilderRef Builder, const char *Name,
  * \param Name              Pointer type name. (optional)
  * \param NameLen           Length of pointer type name. (optional)
  */
-LLVMMetadataRef LLVMDIBuilderCreatePointerType(
-    LLVMDIBuilderRef Builder, LLVMMetadataRef PointeeTy,
-    uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace,
-    const char *Name, size_t NameLen);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreatePointerType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef PointeeTy, uint64_t SizeInBits,
+    uint32_t AlignInBits, unsigned AddressSpace, const char *Name,
+    size_t NameLen);
 
 /**
  * Create debugging information entry for a struct.
@@ -776,7 +757,7 @@ LLVMMetadataRef LLVMDIBuilderCreatePointerType(
  * \param UniqueId     A unique identifier for the struct.
  * \param UniqueIdLen  Length of the unique identifier for the struct.
  */
-LLVMMetadataRef LLVMDIBuilderCreateStructType(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateStructType(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, LLVMMetadataRef File, unsigned LineNumber,
     uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags,
@@ -798,11 +779,11 @@ LLVMMetadataRef LLVMDIBuilderCreateStructType(
  * \param Flags        Flags to encode member attribute, e.g. private
  * \param Ty           Parent type.
  */
-LLVMMetadataRef LLVMDIBuilderCreateMemberType(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateMemberType(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
-    size_t NameLen, LLVMMetadataRef File, unsigned LineNo,
-    uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
-    LLVMDIFlags Flags, LLVMMetadataRef Ty);
+    size_t NameLen, LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits,
+    uint32_t AlignInBits, uint64_t OffsetInBits, LLVMDIFlags Flags,
+    LLVMMetadataRef Ty);
 
 /**
  * Create debugging information entry for a
@@ -818,8 +799,7 @@ LLVMMetadataRef LLVMDIBuilderCreateMemberType(
  * \param ConstantVal  Const initializer of the member.
  * \param AlignInBits  Member alignment.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateStaticMemberType(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateStaticMemberType(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, LLVMMetadataRef File, unsigned LineNumber,
     LLVMMetadataRef Type, LLVMDIFlags Flags, LLVMValueRef ConstantVal,
@@ -834,13 +814,10 @@ LLVMDIBuilderCreateStaticMemberType(
  * \param AlignInBits  Alignment.
  * \param Flags        Flags.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateMemberPointerType(LLVMDIBuilderRef Builder,
-                                     LLVMMetadataRef PointeeType,
-                                     LLVMMetadataRef ClassType,
-                                     uint64_t SizeInBits,
-                                     uint32_t AlignInBits,
-                                     LLVMDIFlags Flags);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateMemberPointerType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef PointeeType,
+    LLVMMetadataRef ClassType, uint64_t SizeInBits, uint32_t AlignInBits,
+    LLVMDIFlags Flags);
 /**
  * Create debugging information entry for Objective-C instance variable.
  * \param Builder      The DIBuilder.
@@ -855,13 +832,11 @@ LLVMDIBuilderCreateMemberPointerType(LLVMDIBuilderRef Builder,
  * \param Ty           Parent type.
  * \param PropertyNode Property associated with this ivar.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateObjCIVar(LLVMDIBuilderRef Builder,
-                            const char *Name, size_t NameLen,
-                            LLVMMetadataRef File, unsigned LineNo,
-                            uint64_t SizeInBits, uint32_t AlignInBits,
-                            uint64_t OffsetInBits, LLVMDIFlags Flags,
-                            LLVMMetadataRef Ty, LLVMMetadataRef PropertyNode);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateObjCIVar(
+    LLVMDIBuilderRef Builder, const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits,
+    uint32_t AlignInBits, uint64_t OffsetInBits, LLVMDIFlags Flags,
+    LLVMMetadataRef Ty, LLVMMetadataRef PropertyNode);
 
 /**
  * Create debugging information entry for Objective-C property.
@@ -877,14 +852,11 @@ LLVMDIBuilderCreateObjCIVar(LLVMDIBuilderRef Builder,
  * \param PropertyAttributes Objective C property attributes.
  * \param Ty                 Type.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateObjCProperty(LLVMDIBuilderRef Builder,
-                                const char *Name, size_t NameLen,
-                                LLVMMetadataRef File, unsigned LineNo,
-                                const char *GetterName, size_t GetterNameLen,
-                                const char *SetterName, size_t SetterNameLen,
-                                unsigned PropertyAttributes,
-                                LLVMMetadataRef Ty);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateObjCProperty(
+    LLVMDIBuilderRef Builder, const char *Name, size_t NameLen,
+    LLVMMetadataRef File, unsigned LineNo, const char *GetterName,
+    size_t GetterNameLen, const char *SetterName, size_t SetterNameLen,
+    unsigned PropertyAttributes, LLVMMetadataRef Ty);
 
 /**
  * Create a uniqued DIType* clone with FlagObjectPointer. If \c Implicit
@@ -894,9 +866,8 @@ LLVMDIBuilderCreateObjCProperty(LLVMDIBuilderRef Builder,
  * \param Implicit  Indicates whether this pointer was implicitly generated
  *                  (i.e., not spelled out in source).
  */
-LLVMMetadataRef LLVMDIBuilderCreateObjectPointerType(LLVMDIBuilderRef Builder,
-                                                     LLVMMetadataRef Type,
-                                                     LLVMBool Implicit);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateObjectPointerType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Type, LLVMBool Implicit);
 
 /**
  * Create debugging information entry for a qualified
@@ -906,9 +877,8 @@ LLVMMetadataRef LLVMDIBuilderCreateObjectPointerType(LLVMDIBuilderRef Builder,
  *                    e.g. LLVMDWARFTypeQualifier_volatile_type
  * \param Type        Base Type.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateQualifiedType(LLVMDIBuilderRef Builder, unsigned Tag,
-                                 LLVMMetadataRef Type);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateQualifiedType(
+    LLVMDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Type);
 
 /**
  * Create debugging information entry for a c++
@@ -917,15 +887,14 @@ LLVMDIBuilderCreateQualifiedType(LLVMDIBuilderRef Builder, unsigned Tag,
  * \param Tag       Tag identifying type,
  * \param Type      Base Type.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateReferenceType(LLVMDIBuilderRef Builder, unsigned Tag,
-                                 LLVMMetadataRef Type);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateReferenceType(
+    LLVMDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Type);
 
 /**
  * Create C++11 nullptr type.
  * \param Builder   The DIBuilder.
  */
-LLVMMetadataRef
+LLVM_C_ABI LLVMMetadataRef
 LLVMDIBuilderCreateNullPtrType(LLVMDIBuilderRef Builder);
 
 /**
@@ -937,11 +906,10 @@ LLVMDIBuilderCreateNullPtrType(LLVMDIBuilderRef Builder);
  * \param LineNo     Line number.
  * \param Scope      The surrounding context for the typedef.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateTypedef(LLVMDIBuilderRef Builder, LLVMMetadataRef Type,
-                           const char *Name, size_t NameLen,
-                           LLVMMetadataRef File, unsigned LineNo,
-                           LLVMMetadataRef Scope, uint32_t AlignInBits);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateTypedef(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name,
+    size_t NameLen, LLVMMetadataRef File, unsigned LineNo,
+    LLVMMetadataRef Scope, uint32_t AlignInBits);
 
 /**
  * Create debugging information entry to establish inheritance relationship
@@ -953,11 +921,9 @@ LLVMDIBuilderCreateTypedef(LLVMDIBuilderRef Builder, LLVMMetadataRef Type,
  * \param VBPtrOffset  Virtual base pointer offset.
  * \param Flags         Flags to describe inheritance attribute, e.g. private
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateInheritance(LLVMDIBuilderRef Builder,
-                               LLVMMetadataRef Ty, LLVMMetadataRef BaseTy,
-                               uint64_t BaseOffset, uint32_t VBPtrOffset,
-                               LLVMDIFlags Flags);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateInheritance(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Ty, LLVMMetadataRef BaseTy,
+    uint64_t BaseOffset, uint32_t VBPtrOffset, LLVMDIFlags Flags);
 
 /**
  * Create a permanent forward-declared type.
@@ -975,9 +941,9 @@ LLVMDIBuilderCreateInheritance(LLVMDIBuilderRef Builder,
  * \param UniqueIdentifier    A unique identifier for the type.
  * \param UniqueIdentifierLen Length of the unique identifier.
  */
-LLVMMetadataRef LLVMDIBuilderCreateForwardDecl(
-    LLVMDIBuilderRef Builder, unsigned Tag, const char *Name,
-    size_t NameLen, LLVMMetadataRef Scope, LLVMMetadataRef File, unsigned Line,
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateForwardDecl(
+    LLVMDIBuilderRef Builder, unsigned Tag, const char *Name, size_t NameLen,
+    LLVMMetadataRef Scope, LLVMMetadataRef File, unsigned Line,
     unsigned RuntimeLang, uint64_t SizeInBits, uint32_t AlignInBits,
     const char *UniqueIdentifier, size_t UniqueIdentifierLen);
 
@@ -998,10 +964,9 @@ LLVMMetadataRef LLVMDIBuilderCreateForwardDecl(
  * \param UniqueIdentifier    A unique identifier for the type.
  * \param UniqueIdentifierLen Length of the unique identifier.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateReplaceableCompositeType(
-    LLVMDIBuilderRef Builder, unsigned Tag, const char *Name,
-    size_t NameLen, LLVMMetadataRef Scope, LLVMMetadataRef File, unsigned Line,
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateReplaceableCompositeType(
+    LLVMDIBuilderRef Builder, unsigned Tag, const char *Name, size_t NameLen,
+    LLVMMetadataRef Scope, LLVMMetadataRef File, unsigned Line,
     unsigned RuntimeLang, uint64_t SizeInBits, uint32_t AlignInBits,
     LLVMDIFlags Flags, const char *UniqueIdentifier,
     size_t UniqueIdentifierLen);
@@ -1020,15 +985,11 @@ LLVMDIBuilderCreateReplaceableCompositeType(
  * \param Flags               Flags to encode member attribute.
  * \param Type                Parent type.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateBitFieldMemberType(LLVMDIBuilderRef Builder,
-                                      LLVMMetadataRef Scope,
-                                      const char *Name, size_t NameLen,
-                                      LLVMMetadataRef File, unsigned LineNumber,
-                                      uint64_t SizeInBits,
-                                      uint64_t OffsetInBits,
-                                      uint64_t StorageOffsetInBits,
-                                      LLVMDIFlags Flags, LLVMMetadataRef Type);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateBitFieldMemberType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
+    size_t NameLen, LLVMMetadataRef File, unsigned LineNumber,
+    uint64_t SizeInBits, uint64_t OffsetInBits, uint64_t StorageOffsetInBits,
+    LLVMDIFlags Flags, LLVMMetadataRef Type);
 
 /**
  * Create debugging information entry for a class.
@@ -1052,23 +1013,22 @@ LLVMDIBuilderCreateBitFieldMemberType(LLVMDIBuilderRef Builder,
  * \param UniqueIdentifier    A unique identifier for the type.
  * \param UniqueIdentifierLen Length of the unique identifier.
  */
-LLVMMetadataRef LLVMDIBuilderCreateClassType(LLVMDIBuilderRef Builder,
-    LLVMMetadataRef Scope, const char *Name, size_t NameLen,
-    LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits,
-    uint32_t AlignInBits, uint64_t OffsetInBits, LLVMDIFlags Flags,
-    LLVMMetadataRef DerivedFrom,
-    LLVMMetadataRef *Elements, unsigned NumElements,
-    LLVMMetadataRef VTableHolder, LLVMMetadataRef TemplateParamsNode,
-    const char *UniqueIdentifier, size_t UniqueIdentifierLen);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateClassType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
+    size_t NameLen, LLVMMetadataRef File, unsigned LineNumber,
+    uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+    LLVMDIFlags Flags, LLVMMetadataRef DerivedFrom, LLVMMetadataRef *Elements,
+    unsigned NumElements, LLVMMetadataRef VTableHolder,
+    LLVMMetadataRef TemplateParamsNode, const char *UniqueIdentifier,
+    size_t UniqueIdentifierLen);
 
 /**
  * Create a uniqued DIType* clone with FlagArtificial set.
  * \param Builder     The DIBuilder.
  * \param Type        The underlying type.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateArtificialType(LLVMDIBuilderRef Builder,
-                                  LLVMMetadataRef Type);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateArtificialType(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef Type);
 
 /**
  * Get the name of this DIType.
@@ -1077,7 +1037,7 @@ LLVMDIBuilderCreateArtificialType(LLVMDIBuilderRef Builder,
  *
  * @see DIType::getName()
  */
-const char *LLVMDITypeGetName(LLVMMetadataRef DType, size_t *Length);
+LLVM_C_ABI const char *LLVMDITypeGetName(LLVMMetadataRef DType, size_t *Length);
 
 /**
  * Get the size of this DIType in bits.
@@ -1085,7 +1045,7 @@ const char *LLVMDITypeGetName(LLVMMetadataRef DType, size_t *Length);
  *
  * @see DIType::getSizeInBits()
  */
-uint64_t LLVMDITypeGetSizeInBits(LLVMMetadataRef DType);
+LLVM_C_ABI uint64_t LLVMDITypeGetSizeInBits(LLVMMetadataRef DType);
 
 /**
  * Get the offset of this DIType in bits.
@@ -1093,7 +1053,7 @@ uint64_t LLVMDITypeGetSizeInBits(LLVMMetadataRef DType);
  *
  * @see DIType::getOffsetInBits()
  */
-uint64_t LLVMDITypeGetOffsetInBits(LLVMMetadataRef DType);
+LLVM_C_ABI uint64_t LLVMDITypeGetOffsetInBits(LLVMMetadataRef DType);
 
 /**
  * Get the alignment of this DIType in bits.
@@ -1101,7 +1061,7 @@ uint64_t LLVMDITypeGetOffsetInBits(LLVMMetadataRef DType);
  *
  * @see DIType::getAlignInBits()
  */
-uint32_t LLVMDITypeGetAlignInBits(LLVMMetadataRef DType);
+LLVM_C_ABI uint32_t LLVMDITypeGetAlignInBits(LLVMMetadataRef DType);
 
 /**
  * Get the source line where this DIType is declared.
@@ -1109,7 +1069,7 @@ uint32_t LLVMDITypeGetAlignInBits(LLVMMetadataRef DType);
  *
  * @see DIType::getLine()
  */
-unsigned LLVMDITypeGetLine(LLVMMetadataRef DType);
+LLVM_C_ABI unsigned LLVMDITypeGetLine(LLVMMetadataRef DType);
 
 /**
  * Get the flags associated with this DIType.
@@ -1117,7 +1077,7 @@ unsigned LLVMDITypeGetLine(LLVMMetadataRef DType);
  *
  * @see DIType::getFlags()
  */
-LLVMDIFlags LLVMDITypeGetFlags(LLVMMetadataRef DType);
+LLVM_C_ABI LLVMDIFlags LLVMDITypeGetFlags(LLVMMetadataRef DType);
 
 /**
  * Create a descriptor for a value range.
@@ -1125,9 +1085,8 @@ LLVMDIFlags LLVMDITypeGetFlags(LLVMMetadataRef DType);
  * \param LowerBound Lower bound of the subrange, e.g. 0 for C, 1 for Fortran.
  * \param Count      Count of elements in the subrange.
  */
-LLVMMetadataRef LLVMDIBuilderGetOrCreateSubrange(LLVMDIBuilderRef Builder,
-                                                 int64_t LowerBound,
-                                                 int64_t Count);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderGetOrCreateSubrange(
+    LLVMDIBuilderRef Builder, int64_t LowerBound, int64_t Count);
 
 /**
  * Create an array of DI Nodes.
@@ -1135,9 +1094,8 @@ LLVMMetadataRef LLVMDIBuilderGetOrCreateSubrange(LLVMDIBuilderRef Builder,
  * \param Data           The DI Node elements.
  * \param NumElements    Number of DI Node elements.
  */
-LLVMMetadataRef LLVMDIBuilderGetOrCreateArray(LLVMDIBuilderRef Builder,
-                                              LLVMMetadataRef *Data,
-                                              size_t NumElements);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderGetOrCreateArray(
+    LLVMDIBuilderRef Builder, LLVMMetadataRef *Data, size_t NumElements);
 
 /**
  * Create a new descriptor for the specified variable which has a complex
@@ -1146,8 +1104,8 @@ LLVMMetadataRef LLVMDIBuilderGetOrCreateArray(LLVMDIBuilderRef Builder,
  * \param Addr        An array of complex address operations.
  * \param Length      Length of the address operation array.
  */
-LLVMMetadataRef LLVMDIBuilderCreateExpression(LLVMDIBuilderRef Builder,
-                                              uint64_t *Addr, size_t Length);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateExpression(
+    LLVMDIBuilderRef Builder, uint64_t *Addr, size_t Length);
 
 /**
  * Create a new descriptor for the specified variable that does not have an
@@ -1155,9 +1113,8 @@ LLVMMetadataRef LLVMDIBuilderCreateExpression(LLVMDIBuilderRef Builder,
  * \param Builder     The DIBuilder.
  * \param Value       The constant value.
  */
-LLVMMetadataRef
-LLVMDIBuilderCreateConstantValueExpression(LLVMDIBuilderRef Builder,
-                                           uint64_t Value);
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateConstantValueExpression(
+    LLVMDIBuilderRef Builder, uint64_t Value);
 
 /**
  * Create a new descriptor for the specified variable.
@@ -1178,17 +1135,16 @@ LLVMDIBuilderCreateConstantValueExpression(LLVMDIBuilderRef Builder,
  * \param AlignInBits Variable alignment(or 0 if no alignment attr was
  *                    specified)
  */
-LLVMMetadataRef LLVMDIBuilderCreateGlobalVariableExpression(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateGlobalVariableExpression(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, const char *Linkage, size_t LinkLen, LLVMMetadataRef File,
     unsigned LineNo, LLVMMetadataRef Ty, LLVMBool LocalToUnit,
     LLVMMetadataRef Expr, LLVMMetadataRef Decl, uint32_t AlignInBits);
 
-
 /**
  * Get the dwarf::Tag of a DINode
  */
-uint16_t LLVMGetDINodeTag(LLVMMetadataRef MD);
+LLVM_C_ABI uint16_t LLVMGetDINodeTag(LLVMMetadataRef MD);
 
 /**
  * Retrieves the \c DIVariable associated with this global variable expression.
@@ -1196,7 +1152,8 @@ uint16_t LLVMGetDINodeTag(LLVMMetadataRef MD);
  *
  * @see llvm::DIGlobalVariableExpression::getVariable()
  */
-LLVMMetadataRef LLVMDIGlobalVariableExpressionGetVariable(LLVMMetadataRef GVE);
+LLVM_C_ABI LLVMMetadataRef
+LLVMDIGlobalVariableExpressionGetVariable(LLVMMetadataRef GVE);
 
 /**
  * Retrieves the \c DIExpression associated with this global variable expression.
@@ -1204,8 +1161,8 @@ LLVMMetadataRef LLVMDIGlobalVariableExpressionGetVariable(LLVMMetadataRef GVE);
  *
  * @see llvm::DIGlobalVariableExpression::getExpression()
  */
-LLVMMetadataRef LLVMDIGlobalVariableExpressionGetExpression(
-    LLVMMetadataRef GVE);
+LLVM_C_ABI LLVMMetadataRef
+LLVMDIGlobalVariableExpressionGetExpression(LLVMMetadataRef GVE);
 
 /**
  * Get the metadata of the file associated with a given variable.
@@ -1213,7 +1170,7 @@ LLVMMetadataRef LLVMDIGlobalVariableExpressionGetExpression(
  *
  * @see DIVariable::getFile()
  */
-LLVMMetadataRef LLVMDIVariableGetFile(LLVMMetadataRef Var);
+LLVM_C_ABI LLVMMetadataRef LLVMDIVariableGetFile(LLVMMetadataRef Var);
 
 /**
  * Get the metadata of the scope associated with a given variable.
@@ -1221,7 +1178,7 @@ LLVMMetadataRef LLVMDIVariableGetFile(LLVMMetadataRef Var);
  *
  * @see DIVariable::getScope()
  */
-LLVMMetadataRef LLVMDIVariableGetScope(LLVMMetadataRef Var);
+LLVM_C_ABI LLVMMetadataRef LLVMDIVariableGetScope(LLVMMetadataRef Var);
 
 /**
  * Get the source line where this \c DIVariable is declared.
@@ -1229,7 +1186,7 @@ LLVMMetadataRef LLVMDIVariableGetScope(LLVMMetadataRef Var);
  *
  * @see DIVariable::getLine()
  */
-unsigned LLVMDIVariableGetLine(LLVMMetadataRef Var);
+LLVM_C_ABI unsigned LLVMDIVariableGetLine(LLVMMetadataRef Var);
 
 /**
  * Create a new temporary \c MDNode.  Suitable for use in constructing cyclic
@@ -1239,8 +1196,9 @@ unsigned LLVMDIVariableGetLine(LLVMMetadataRef Var);
  * \param Data           The metadata elements.
  * \param NumElements    Number of metadata elements.
  */
-LLVMMetadataRef LLVMTemporaryMDNode(LLVMContextRef Ctx, LLVMMetadataRef *Data,
-                                    size_t NumElements);
+LLVM_C_ABI LLVMMetadataRef LLVMTemporaryMDNode(LLVMContextRef Ctx,
+                                               LLVMMetadataRef *Data,
+                                               size_t NumElements);
 
 /**
  * Deallocate a temporary node.
@@ -1249,15 +1207,16 @@ LLVMMetadataRef LLVMTemporaryMDNode(LLVMContextRef Ctx, LLVMMetadataRef *Data,
  * references will be reset.
  * \param TempNode    The temporary metadata node.
  */
-void LLVMDisposeTemporaryMDNode(LLVMMetadataRef TempNode);
+LLVM_C_ABI void LLVMDisposeTemporaryMDNode(LLVMMetadataRef TempNode);
 
 /**
  * Replace all uses of temporary metadata.
  * \param TempTargetMetadata    The temporary metadata node.
  * \param Replacement           The replacement metadata node.
  */
-void LLVMMetadataReplaceAllUsesWith(LLVMMetadataRef TempTargetMetadata,
-                                    LLVMMetadataRef Replacement);
+LLVM_C_ABI void
+LLVMMetadataReplaceAllUsesWith(LLVMMetadataRef TempTargetMetadata,
+                               LLVMMetadataRef Replacement);
 
 /**
  * Create a new descriptor for the specified global variable that is temporary
@@ -1276,7 +1235,7 @@ void LLVMMetadataReplaceAllUsesWith(LLVMMetadataRef TempTargetMetadata,
  * \param AlignInBits Variable alignment(or 0 if no alignment attr was
  *                    specified)
  */
-LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, const char *Linkage, size_t LnkLen, LLVMMetadataRef File,
     unsigned LineNo, LLVMMetadataRef Ty, LLVMBool LocalToUnit,
@@ -1297,7 +1256,7 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl(
  * \param DebugLoc    Debug info location.
  * \param Instr       Instruction acting as a location for the new record.
  */
-LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore(
+LLVM_C_ABI LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore(
     LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo,
     LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr);
 
@@ -1318,7 +1277,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore(
  * \param DebugLoc    Debug info location.
  * \param Block       Basic block acting as a location for the new record.
  */
-LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd(
+LLVM_C_ABI LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd(
     LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo,
     LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block);
 
@@ -1337,7 +1296,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd(
  * \param DebugLoc    Debug info location.
  * \param Instr       Instruction acting as a location for the new record.
  */
-LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore(
+LLVM_C_ABI LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore(
     LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo,
     LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr);
 
@@ -1358,7 +1317,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore(
  * \param DebugLoc    Debug info location.
  * \param Block       Basic block acting as a location for the new record.
  */
-LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd(
+LLVM_C_ABI LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd(
     LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo,
     LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block);
 
@@ -1375,7 +1334,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd(
  * \param Flags           Flags.
  * \param AlignInBits     Variable alignment.
  */
-LLVMMetadataRef LLVMDIBuilderCreateAutoVariable(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateAutoVariable(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty,
     LLVMBool AlwaysPreserve, LLVMDIFlags Flags, uint32_t AlignInBits);
@@ -1393,7 +1352,7 @@ LLVMMetadataRef LLVMDIBuilderCreateAutoVariable(
  * \param AlwaysPreserve  If true, this descriptor will survive optimizations.
  * \param Flags           Flags.
  */
-LLVMMetadataRef LLVMDIBuilderCreateParameterVariable(
+LLVM_C_ABI LLVMMetadataRef LLVMDIBuilderCreateParameterVariable(
     LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name,
     size_t NameLen, unsigned ArgNo, LLVMMetadataRef File, unsigned LineNo,
     LLVMMetadataRef Ty, LLVMBool AlwaysPreserve, LLVMDIFlags Flags);
@@ -1403,14 +1362,14 @@ LLVMMetadataRef LLVMDIBuilderCreateParameterVariable(
  *
  * @see llvm::Function::getSubprogram()
  */
-LLVMMetadataRef LLVMGetSubprogram(LLVMValueRef Func);
+LLVM_C_ABI LLVMMetadataRef LLVMGetSubprogram(LLVMValueRef Func);
 
 /**
  * Set the subprogram attached to a function.
  *
  * @see llvm::Function::setSubprogram()
  */
-void LLVMSetSubprogram(LLVMValueRef Func, LLVMMetadataRef SP);
+LLVM_C_ABI void LLVMSetSubprogram(LLVMValueRef Func, LLVMMetadataRef SP);
 
 /**
  * Get the line associated with a given subprogram.
@@ -1418,7 +1377,7 @@ void LLVMSetSubprogram(LLVMValueRef Func, LLVMMetadataRef SP);
  *
  * @see DISubprogram::getLine()
  */
-unsigned LLVMDISubprogramGetLine(LLVMMetadataRef Subprogram);
+LLVM_C_ABI unsigned LLVMDISubprogramGetLine(LLVMMetadataRef Subprogram);
 
 /**
  * Replace the subprogram subroutine type.
@@ -1427,15 +1386,15 @@ unsigned LLVMDISubprogramGetLine(LLVMMetadataRef Subprogram);
  *
  * @see DISubprogram::replaceType()
  */
-void LLVMDISubprogramReplaceType(LLVMMetadataRef Subprogram,
-                                 LLVMMetadataRef SubroutineType);
+LLVM_C_ABI void LLVMDISubprogramReplaceType(LLVMMetadataRef Subprogram,
+                                            LLVMMetadataRef SubroutineType);
 
 /**
  * Get the debug location for the given instruction.
  *
  * @see llvm::Instruction::getDebugLoc()
  */
-LLVMMetadataRef LLVMInstructionGetDebugLoc(LLVMValueRef Inst);
+LLVM_C_ABI LLVMMetadataRef LLVMInstructionGetDebugLoc(LLVMValueRef Inst);
 
 /**
  * Set the debug location for the given instruction.
@@ -1444,7 +1403,8 @@ LLVMMetadataRef LLVMInstructionGetDebugLoc(LLVMValueRef Inst);
  *
  * @see llvm::Instruction::setDebugLoc()
  */
-void LLVMInstructionSetDebugLoc(LLVMValueRef Inst, LLVMMetadataRef Loc);
+LLVM_C_ABI void LLVMInstructionSetDebugLoc(LLVMValueRef Inst,
+                                           LLVMMetadataRef Loc);
 
 /**
  * Create a new descriptor for a label
@@ -1459,10 +1419,10 @@ void LLVMInstructionSetDebugLoc(LLVMValueRef Inst, LLVMMetadataRef Loc);
  *
  * @see llvm::DIBuilder::createLabel()
  */
-LLVMMetadataRef LLVMDIBuilderCreateLabel(
-    LLVMDIBuilderRef Builder,
-    LLVMMetadataRef Context, const char *Name, size_t NameLen,
-    LLVMMetadataRef File, unsigned LineNo, LLVMBool AlwaysPreserve);
+LLVM_C_ABI LLVMMetadataRef
+LLVMDIBuilderCreateLabel(LLVMDIBuilderRef Builder, LLVMMetadataRef Context,
+                         const char *Name, size_t NameLen, LLVMMetadataRef File,
+                         unsigned LineNo, LLVMBool AlwaysPreserve);
 
 /**
  * Insert a new llvm.dbg.label intrinsic call
@@ -1474,7 +1434,7 @@ LLVMMetadataRef LLVMDIBuilderCreateLabel(
  *
  * @see llvm::DIBuilder::insertLabel()
  */
-LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(
+LLVM_C_ABI LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(
     LLVMDIBuilderRef Builder, LLVMMetadataRef LabelInfo,
     LLVMMetadataRef Location, LLVMValueRef InsertBefore);
 
@@ -1488,7 +1448,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(
  *
  * @see llvm::DIBuilder::insertLabel()
  */
-LLVMDbgRecordRef LLVMDIBuilderInsertLabelAtEnd(
+LLVM_C_ABI LLVMDbgRecordRef LLVMDIBuilderInsertLabelAtEnd(
     LLVMDIBuilderRef Builder, LLVMMetadataRef LabelInfo,
     LLVMMetadataRef Location, LLVMBasicBlockRef InsertAtEnd);
 
@@ -1497,7 +1457,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertLabelAtEnd(
  *
  * @see llvm::Metadata::getMetadataID()
  */
-LLVMMetadataKind LLVMGetMetadataKind(LLVMMetadataRef Metadata);
+LLVM_C_ABI LLVMMetadataKind LLVMGetMetadataKind(LLVMMetadataRef Metadata);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Disassembler.h b/llvm/include/llvm-c/Disassembler.h
index 4bc6b04dd6eaf..c53367945cb35 100644
--- a/llvm/include/llvm-c/Disassembler.h
+++ b/llvm/include/llvm-c/Disassembler.h
@@ -17,6 +17,7 @@
 
 #include "llvm-c/DisassemblerTypes.h"
 #include "llvm-c/ExternC.h"
+#include "llvm-c/Visibility.h"
 
 /**
  * @defgroup LLVMCDisassembler Disassembler
@@ -35,9 +36,9 @@ LLVM_C_EXTERN_C_BEGIN
  * returns NULL. This function is equivalent to calling
  * LLVMCreateDisasmCPUFeatures() with an empty CPU name and feature set.
  */
-LLVMDisasmContextRef LLVMCreateDisasm(const char *TripleName, void *DisInfo,
-                                      int TagType, LLVMOpInfoCallback GetOpInfo,
-                                      LLVMSymbolLookupCallback SymbolLookUp);
+LLVM_C_ABI LLVMDisasmContextRef LLVMCreateDisasm(
+    const char *TripleName, void *DisInfo, int TagType,
+    LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp);
 
 /**
  * Create a disassembler for the TripleName and a specific CPU.  Symbolic
@@ -47,10 +48,9 @@ LLVMDisasmContextRef LLVMCreateDisasm(const char *TripleName, void *DisInfo,
  * disassembler context.  If not, it returns NULL. This function is equivalent
  * to calling LLVMCreateDisasmCPUFeatures() with an empty feature set.
  */
-LLVMDisasmContextRef LLVMCreateDisasmCPU(const char *Triple, const char *CPU,
-                                         void *DisInfo, int TagType,
-                                         LLVMOpInfoCallback GetOpInfo,
-                                         LLVMSymbolLookupCallback SymbolLookUp);
+LLVM_C_ABI LLVMDisasmContextRef LLVMCreateDisasmCPU(
+    const char *Triple, const char *CPU, void *DisInfo, int TagType,
+    LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp);
 
 /**
  * Create a disassembler for the TripleName, a specific CPU and specific feature
@@ -59,17 +59,16 @@ LLVMDisasmContextRef LLVMCreateDisasmCPU(const char *Triple, const char *CPU,
  * described above.  These can all be passed * as NULL.  If successful, this
  * returns a disassembler context.  If not, it returns NULL.
  */
-LLVMDisasmContextRef
-LLVMCreateDisasmCPUFeatures(const char *Triple, const char *CPU,
-                            const char *Features, void *DisInfo, int TagType,
-                            LLVMOpInfoCallback GetOpInfo,
-                            LLVMSymbolLookupCallback SymbolLookUp);
+LLVM_C_ABI LLVMDisasmContextRef LLVMCreateDisasmCPUFeatures(
+    const char *Triple, const char *CPU, const char *Features, void *DisInfo,
+    int TagType, LLVMOpInfoCallback GetOpInfo,
+    LLVMSymbolLookupCallback SymbolLookUp);
 
 /**
  * Set the disassembler's options.  Returns 1 if it can set the Options and 0
  * otherwise.
  */
-int LLVMSetDisasmOptions(LLVMDisasmContextRef DC, uint64_t Options);
+LLVM_C_ABI int LLVMSetDisasmOptions(LLVMDisasmContextRef DC, uint64_t Options);
 
 /* The option to produce marked up assembly. */
 #define LLVMDisassembler_Option_UseMarkup 1
@@ -87,7 +86,7 @@ int LLVMSetDisasmOptions(LLVMDisasmContextRef DC, uint64_t Options);
 /**
  * Dispose of a disassembler context.
  */
-void LLVMDisasmDispose(LLVMDisasmContextRef DC);
+LLVM_C_ABI void LLVMDisasmDispose(LLVMDisasmContextRef DC);
 
 /**
  * Disassemble a single instruction using the disassembler context specified in
@@ -99,9 +98,9 @@ void LLVMDisasmDispose(LLVMDisasmContextRef DC);
  * function returns the number of bytes in the instruction or zero if there was
  * no valid instruction.
  */
-size_t LLVMDisasmInstruction(LLVMDisasmContextRef DC, uint8_t *Bytes,
-                             uint64_t BytesSize, uint64_t PC,
-                             char *OutString, size_t OutStringSize);
+LLVM_C_ABI size_t LLVMDisasmInstruction(LLVMDisasmContextRef DC, uint8_t *Bytes,
+                                        uint64_t BytesSize, uint64_t PC,
+                                        char *OutString, size_t OutStringSize);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Error.h b/llvm/include/llvm-c/Error.h
index 874bbcfe8f21a..1a07c1b64d819 100644
--- a/llvm/include/llvm-c/Error.h
+++ b/llvm/include/llvm-c/Error.h
@@ -15,6 +15,7 @@
 #define LLVM_C_ERROR_H
 
 #include "llvm-c/ExternC.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -41,7 +42,7 @@ typedef const void *LLVMErrorTypeId;
  * Returns the type id for the given error instance, which must be a failure
  * value (i.e. non-null).
  */
-LLVMErrorTypeId LLVMGetErrorTypeId(LLVMErrorRef Err);
+LLVM_C_ABI LLVMErrorTypeId LLVMGetErrorTypeId(LLVMErrorRef Err);
 
 /**
  * Dispose of the given error without handling it. This operation consumes the
@@ -49,7 +50,7 @@ LLVMErrorTypeId LLVMGetErrorTypeId(LLVMErrorRef Err);
  * Note: This method *only* needs to be called if the error is not being passed
  * to some other consuming operation, e.g. LLVMGetErrorMessage.
  */
-void LLVMConsumeError(LLVMErrorRef Err);
+LLVM_C_ABI void LLVMConsumeError(LLVMErrorRef Err);
 
 /**
  * Report a fatal error if Err is a failure value.
@@ -57,7 +58,7 @@ void LLVMConsumeError(LLVMErrorRef Err);
  * This function can be used to wrap calls to fallible functions ONLY when it is
  * known that the Error will always be a success value.
  */
-void LLVMCantFail(LLVMErrorRef Err);
+LLVM_C_ABI void LLVMCantFail(LLVMErrorRef Err);
 
 /**
  * Returns the given string's error message. This operation consumes the error,
@@ -65,22 +66,22 @@ void LLVMCantFail(LLVMErrorRef Err);
  * The caller is responsible for disposing of the string by calling
  * LLVMDisposeErrorMessage.
  */
-char *LLVMGetErrorMessage(LLVMErrorRef Err);
+LLVM_C_ABI char *LLVMGetErrorMessage(LLVMErrorRef Err);
 
 /**
  * Dispose of the given error message.
  */
-void LLVMDisposeErrorMessage(char *ErrMsg);
+LLVM_C_ABI void LLVMDisposeErrorMessage(char *ErrMsg);
 
 /**
  * Returns the type id for llvm StringError.
  */
-LLVMErrorTypeId LLVMGetStringErrorTypeId(void);
+LLVM_C_ABI LLVMErrorTypeId LLVMGetStringErrorTypeId(void);
 
 /**
  * Create a StringError.
  */
-LLVMErrorRef LLVMCreateStringError(const char *ErrMsg);
+LLVM_C_ABI LLVMErrorRef LLVMCreateStringError(const char *ErrMsg);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/ErrorHandling.h b/llvm/include/llvm-c/ErrorHandling.h
index d9b9f22752b8f..8cd43f9245018 100644
--- a/llvm/include/llvm-c/ErrorHandling.h
+++ b/llvm/include/llvm-c/ErrorHandling.h
@@ -15,6 +15,7 @@
 #define LLVM_C_ERRORHANDLING_H
 
 #include "llvm-c/ExternC.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -33,20 +34,20 @@ typedef void (*LLVMFatalErrorHandler)(const char *Reason);
  * function allows you to install a callback that will be invoked prior to the
  * call to exit(1).
  */
-void LLVMInstallFatalErrorHandler(LLVMFatalErrorHandler Handler);
+LLVM_C_ABI void LLVMInstallFatalErrorHandler(LLVMFatalErrorHandler Handler);
 
 /**
  * Reset the fatal error handler. This resets LLVM's fatal error handling
  * behavior to the default.
  */
-void LLVMResetFatalErrorHandler(void);
+LLVM_C_ABI void LLVMResetFatalErrorHandler(void);
 
 /**
  * Enable LLVM's built-in stack trace code. This intercepts the OS's crash
  * signals and prints which component of LLVM you were in at the time if the
  * crash.
  */
-void LLVMEnablePrettyStackTrace(void);
+LLVM_C_ABI void LLVMEnablePrettyStackTrace(void);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/ExecutionEngine.h b/llvm/include/llvm-c/ExecutionEngine.h
index 2062cbf470d8b..184787ae1d682 100644
--- a/llvm/include/llvm-c/ExecutionEngine.h
+++ b/llvm/include/llvm-c/ExecutionEngine.h
@@ -23,6 +23,7 @@
 #include "llvm-c/Target.h"
 #include "llvm-c/TargetMachine.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -37,12 +38,12 @@ LLVM_C_EXTERN_C_BEGIN
  * Empty function used to force the linker to link MCJIT.
  * Has no effect when called on a pre-built library (dylib interface).
  */
-void LLVMLinkInMCJIT(void);
+LLVM_C_ABI void LLVMLinkInMCJIT(void);
 /**
  * Empty function used to force the linker to link the LLVM interpreter.
  * Has no effect when called on a pre-built library (dylib interface).
  */
-void LLVMLinkInInterpreter(void);
+LLVM_C_ABI void LLVMLinkInInterpreter(void);
 
 typedef struct LLVMOpaqueGenericValue *LLVMGenericValueRef;
 typedef struct LLVMOpaqueExecutionEngine *LLVMExecutionEngineRef;
@@ -58,42 +59,42 @@ struct LLVMMCJITCompilerOptions {
 
 /*===-- Operations on generic values --------------------------------------===*/
 
-LLVMGenericValueRef LLVMCreateGenericValueOfInt(LLVMTypeRef Ty,
-                                                unsigned long long N,
-                                                LLVMBool IsSigned);
+LLVM_C_ABI LLVMGenericValueRef LLVMCreateGenericValueOfInt(LLVMTypeRef Ty,
+                                                           unsigned long long N,
+                                                           LLVMBool IsSigned);
 
-LLVMGenericValueRef LLVMCreateGenericValueOfPointer(void *P);
+LLVM_C_ABI LLVMGenericValueRef LLVMCreateGenericValueOfPointer(void *P);
 
-LLVMGenericValueRef LLVMCreateGenericValueOfFloat(LLVMTypeRef Ty, double N);
+LLVM_C_ABI LLVMGenericValueRef LLVMCreateGenericValueOfFloat(LLVMTypeRef Ty,
+                                                             double N);
 
-unsigned LLVMGenericValueIntWidth(LLVMGenericValueRef GenValRef);
+LLVM_C_ABI unsigned LLVMGenericValueIntWidth(LLVMGenericValueRef GenValRef);
 
-unsigned long long LLVMGenericValueToInt(LLVMGenericValueRef GenVal,
-                                         LLVMBool IsSigned);
+LLVM_C_ABI unsigned long long LLVMGenericValueToInt(LLVMGenericValueRef GenVal,
+                                                    LLVMBool IsSigned);
 
-void *LLVMGenericValueToPointer(LLVMGenericValueRef GenVal);
+LLVM_C_ABI void *LLVMGenericValueToPointer(LLVMGenericValueRef GenVal);
 
-double LLVMGenericValueToFloat(LLVMTypeRef TyRef, LLVMGenericValueRef GenVal);
+LLVM_C_ABI double LLVMGenericValueToFloat(LLVMTypeRef TyRef,
+                                          LLVMGenericValueRef GenVal);
 
-void LLVMDisposeGenericValue(LLVMGenericValueRef GenVal);
+LLVM_C_ABI void LLVMDisposeGenericValue(LLVMGenericValueRef GenVal);
 
 /*===-- Operations on execution engines -----------------------------------===*/
 
-LLVMBool LLVMCreateExecutionEngineForModule(LLVMExecutionEngineRef *OutEE,
-                                            LLVMModuleRef M,
-                                            char **OutError);
+LLVM_C_ABI LLVMBool LLVMCreateExecutionEngineForModule(
+    LLVMExecutionEngineRef *OutEE, LLVMModuleRef M, char **OutError);
 
-LLVMBool LLVMCreateInterpreterForModule(LLVMExecutionEngineRef *OutInterp,
-                                        LLVMModuleRef M,
-                                        char **OutError);
+LLVM_C_ABI LLVMBool LLVMCreateInterpreterForModule(
+    LLVMExecutionEngineRef *OutInterp, LLVMModuleRef M, char **OutError);
 
-LLVMBool LLVMCreateJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
-                                        LLVMModuleRef M,
-                                        unsigned OptLevel,
-                                        char **OutError);
+LLVM_C_ABI LLVMBool
+LLVMCreateJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, LLVMModuleRef M,
+                               unsigned OptLevel, char **OutError);
 
-void LLVMInitializeMCJITCompilerOptions(
-  struct LLVMMCJITCompilerOptions *Options, size_t SizeOfOptions);
+LLVM_C_ABI void
+LLVMInitializeMCJITCompilerOptions(struct LLVMMCJITCompilerOptions *Options,
+                                   size_t SizeOfOptions);
 
 /**
  * Create an MCJIT execution engine for a module, with the given options. It is
@@ -112,55 +113,60 @@ void LLVMInitializeMCJITCompilerOptions(
  *
  * LLVMCreateMCJITCompilerForModule(&jit, mod, 0, 0, &error);
  */
-LLVMBool LLVMCreateMCJITCompilerForModule(
-  LLVMExecutionEngineRef *OutJIT, LLVMModuleRef M,
-  struct LLVMMCJITCompilerOptions *Options, size_t SizeOfOptions,
-  char **OutError);
+LLVM_C_ABI LLVMBool LLVMCreateMCJITCompilerForModule(
+    LLVMExecutionEngineRef *OutJIT, LLVMModuleRef M,
+    struct LLVMMCJITCompilerOptions *Options, size_t SizeOfOptions,
+    char **OutError);
 
-void LLVMDisposeExecutionEngine(LLVMExecutionEngineRef EE);
+LLVM_C_ABI void LLVMDisposeExecutionEngine(LLVMExecutionEngineRef EE);
 
-void LLVMRunStaticConstructors(LLVMExecutionEngineRef EE);
+LLVM_C_ABI void LLVMRunStaticConstructors(LLVMExecutionEngineRef EE);
 
-void LLVMRunStaticDestructors(LLVMExecutionEngineRef EE);
+LLVM_C_ABI void LLVMRunStaticDestructors(LLVMExecutionEngineRef EE);
 
-int LLVMRunFunctionAsMain(LLVMExecutionEngineRef EE, LLVMValueRef F,
-                          unsigned ArgC, const char * const *ArgV,
-                          const char * const *EnvP);
+LLVM_C_ABI int LLVMRunFunctionAsMain(LLVMExecutionEngineRef EE, LLVMValueRef F,
+                                     unsigned ArgC, const char *const *ArgV,
+                                     const char *const *EnvP);
 
-LLVMGenericValueRef LLVMRunFunction(LLVMExecutionEngineRef EE, LLVMValueRef F,
-                                    unsigned NumArgs,
-                                    LLVMGenericValueRef *Args);
+LLVM_C_ABI LLVMGenericValueRef LLVMRunFunction(LLVMExecutionEngineRef EE,
+                                               LLVMValueRef F, unsigned NumArgs,
+                                               LLVMGenericValueRef *Args);
 
-void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F);
+LLVM_C_ABI void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE,
+                                               LLVMValueRef F);
 
-void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M);
+LLVM_C_ABI void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M);
 
-LLVMBool LLVMRemoveModule(LLVMExecutionEngineRef EE, LLVMModuleRef M,
-                          LLVMModuleRef *OutMod, char **OutError);
+LLVM_C_ABI LLVMBool LLVMRemoveModule(LLVMExecutionEngineRef EE, LLVMModuleRef M,
+                                     LLVMModuleRef *OutMod, char **OutError);
 
-LLVMBool LLVMFindFunction(LLVMExecutionEngineRef EE, const char *Name,
-                          LLVMValueRef *OutFn);
+LLVM_C_ABI LLVMBool LLVMFindFunction(LLVMExecutionEngineRef EE,
+                                     const char *Name, LLVMValueRef *OutFn);
 
-void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE,
-                                     LLVMValueRef Fn);
+LLVM_C_ABI void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE,
+                                                LLVMValueRef Fn);
 
-LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE);
-LLVMTargetMachineRef
+LLVM_C_ABI LLVMTargetDataRef
+LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE);
+LLVM_C_ABI LLVMTargetMachineRef
 LLVMGetExecutionEngineTargetMachine(LLVMExecutionEngineRef EE);
 
-void LLVMAddGlobalMapping(LLVMExecutionEngineRef EE, LLVMValueRef Global,
-                          void* Addr);
+LLVM_C_ABI void LLVMAddGlobalMapping(LLVMExecutionEngineRef EE,
+                                     LLVMValueRef Global, void *Addr);
 
-void *LLVMGetPointerToGlobal(LLVMExecutionEngineRef EE, LLVMValueRef Global);
+LLVM_C_ABI void *LLVMGetPointerToGlobal(LLVMExecutionEngineRef EE,
+                                        LLVMValueRef Global);
 
-uint64_t LLVMGetGlobalValueAddress(LLVMExecutionEngineRef EE, const char *Name);
+LLVM_C_ABI uint64_t LLVMGetGlobalValueAddress(LLVMExecutionEngineRef EE,
+                                              const char *Name);
 
-uint64_t LLVMGetFunctionAddress(LLVMExecutionEngineRef EE, const char *Name);
+LLVM_C_ABI uint64_t LLVMGetFunctionAddress(LLVMExecutionEngineRef EE,
+                                           const char *Name);
 
 /// Returns true on error, false on success. If true is returned then the error
 /// message is copied to OutStr and cleared in the ExecutionEngine instance.
-LLVMBool LLVMExecutionEngineGetErrMsg(LLVMExecutionEngineRef EE,
-                                      char **OutError);
+LLVM_C_ABI LLVMBool LLVMExecutionEngineGetErrMsg(LLVMExecutionEngineRef EE,
+                                                 char **OutError);
 
 /*===-- Operations on memory managers -------------------------------------===*/
 
@@ -185,21 +191,21 @@ typedef void (*LLVMMemoryManagerDestroyCallback)(void *Opaque);
  * @param FinalizeMemory Set page permissions and flush cache. Return 0 on
  *   success, 1 on error.
  */
-LLVMMCJITMemoryManagerRef LLVMCreateSimpleMCJITMemoryManager(
-  void *Opaque,
-  LLVMMemoryManagerAllocateCodeSectionCallback AllocateCodeSection,
-  LLVMMemoryManagerAllocateDataSectionCallback AllocateDataSection,
-  LLVMMemoryManagerFinalizeMemoryCallback FinalizeMemory,
-  LLVMMemoryManagerDestroyCallback Destroy);
+LLVM_C_ABI LLVMMCJITMemoryManagerRef LLVMCreateSimpleMCJITMemoryManager(
+    void *Opaque,
+    LLVMMemoryManagerAllocateCodeSectionCallback AllocateCodeSection,
+    LLVMMemoryManagerAllocateDataSectionCallback AllocateDataSection,
+    LLVMMemoryManagerFinalizeMemoryCallback FinalizeMemory,
+    LLVMMemoryManagerDestroyCallback Destroy);
 
-void LLVMDisposeMCJITMemoryManager(LLVMMCJITMemoryManagerRef MM);
+LLVM_C_ABI void LLVMDisposeMCJITMemoryManager(LLVMMCJITMemoryManagerRef MM);
 
 /*===-- JIT Event Listener functions -------------------------------------===*/
 
-LLVMJITEventListenerRef LLVMCreateGDBRegistrationListener(void);
-LLVMJITEventListenerRef LLVMCreateIntelJITEventListener(void);
-LLVMJITEventListenerRef LLVMCreateOProfileJITEventListener(void);
-LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void);
+LLVM_C_ABI LLVMJITEventListenerRef LLVMCreateGDBRegistrationListener(void);
+LLVM_C_ABI LLVMJITEventListenerRef LLVMCreateIntelJITEventListener(void);
+LLVM_C_ABI LLVMJITEventListenerRef LLVMCreateOProfileJITEventListener(void);
+LLVM_C_ABI LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/IRReader.h b/llvm/include/llvm-c/IRReader.h
index 905b84fa5a869..9e5615b5be596 100644
--- a/llvm/include/llvm-c/IRReader.h
+++ b/llvm/include/llvm-c/IRReader.h
@@ -16,6 +16,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -35,9 +36,10 @@ LLVM_C_EXTERN_C_BEGIN
  *
  * @see llvm::ParseIR()
  */
-LLVMBool LLVMParseIRInContext(LLVMContextRef ContextRef,
-                              LLVMMemoryBufferRef MemBuf, LLVMModuleRef *OutM,
-                              char **OutMessage);
+LLVM_C_ABI LLVMBool LLVMParseIRInContext(LLVMContextRef ContextRef,
+                                         LLVMMemoryBufferRef MemBuf,
+                                         LLVMModuleRef *OutM,
+                                         char **OutMessage);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/LLJIT.h b/llvm/include/llvm-c/LLJIT.h
index a58c3b8bbef70..414fe34224596 100644
--- a/llvm/include/llvm-c/LLJIT.h
+++ b/llvm/include/llvm-c/LLJIT.h
@@ -28,6 +28,7 @@
 #include "llvm-c/Orc.h"
 #include "llvm-c/TargetMachine.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -71,14 +72,14 @@ typedef struct LLVMOrcOpaqueLLJIT *LLVMOrcLLJITRef;
  * The client owns the resulting LLJITBuilder and should dispose of it using
  * LLVMOrcDisposeLLJITBuilder once they are done with it.
  */
-LLVMOrcLLJITBuilderRef LLVMOrcCreateLLJITBuilder(void);
+LLVM_C_ABI LLVMOrcLLJITBuilderRef LLVMOrcCreateLLJITBuilder(void);
 
 /**
  * Dispose of an LLVMOrcLLJITBuilderRef. This should only be called if ownership
  * has not been passed to LLVMOrcCreateLLJIT (e.g. because some error prevented
  * that function from being called).
  */
-void LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder);
+LLVM_C_ABI void LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder);
 
 /**
  * Set the JITTargetMachineBuilder to be used when constructing the LLJIT
@@ -89,13 +90,13 @@ void LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder);
  * This function takes ownership of the JTMB argument: clients should not
  * dispose of the JITTargetMachineBuilder after calling this function.
  */
-void LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(
+LLVM_C_ABI void LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(
     LLVMOrcLLJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB);
 
 /**
  * Set an ObjectLinkingLayer creator function for this LLJIT instance.
  */
-void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
+LLVM_C_ABI void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
     LLVMOrcLLJITBuilderRef Builder,
     LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F, void *Ctx);
 
@@ -113,13 +114,13 @@ void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
  * LLJIT instance will free all memory managed by the JIT, including JIT'd code
  * and not-yet compiled modules.
  */
-LLVMErrorRef LLVMOrcCreateLLJIT(LLVMOrcLLJITRef *Result,
-                                LLVMOrcLLJITBuilderRef Builder);
+LLVM_C_ABI LLVMErrorRef LLVMOrcCreateLLJIT(LLVMOrcLLJITRef *Result,
+                                           LLVMOrcLLJITBuilderRef Builder);
 
 /**
  * Dispose of an LLJIT instance.
  */
-LLVMErrorRef LLVMOrcDisposeLLJIT(LLVMOrcLLJITRef J);
+LLVM_C_ABI LLVMErrorRef LLVMOrcDisposeLLJIT(LLVMOrcLLJITRef J);
 
 /**
  * Get a reference to the ExecutionSession for this LLJIT instance.
@@ -127,7 +128,8 @@ LLVMErrorRef LLVMOrcDisposeLLJIT(LLVMOrcLLJITRef J);
  * The ExecutionSession is owned by the LLJIT instance. The client is not
  * responsible for managing its memory.
  */
-LLVMOrcExecutionSessionRef LLVMOrcLLJITGetExecutionSession(LLVMOrcLLJITRef J);
+LLVM_C_ABI LLVMOrcExecutionSessionRef
+LLVMOrcLLJITGetExecutionSession(LLVMOrcLLJITRef J);
 
 /**
  * Return a reference to the Main JITDylib.
@@ -135,18 +137,18 @@ LLVMOrcExecutionSessionRef LLVMOrcLLJITGetExecutionSession(LLVMOrcLLJITRef J);
  * The JITDylib is owned by the LLJIT instance. The client is not responsible
  * for managing its memory.
  */
-LLVMOrcJITDylibRef LLVMOrcLLJITGetMainJITDylib(LLVMOrcLLJITRef J);
+LLVM_C_ABI LLVMOrcJITDylibRef LLVMOrcLLJITGetMainJITDylib(LLVMOrcLLJITRef J);
 
 /**
  * Return the target triple for this LLJIT instance. This string is owned by
  * the LLJIT instance and should not be freed by the client.
  */
-const char *LLVMOrcLLJITGetTripleString(LLVMOrcLLJITRef J);
+LLVM_C_ABI const char *LLVMOrcLLJITGetTripleString(LLVMOrcLLJITRef J);
 
 /**
  * Returns the global prefix character according to the LLJIT's DataLayout.
  */
-char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J);
+LLVM_C_ABI char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J);
 
 /**
  * Mangles the given string according to the LLJIT instance's DataLayout, then
@@ -155,7 +157,7 @@ char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J);
  * decrement the ref-count on the pool entry once they are finished with this
  * value.
  */
-LLVMOrcSymbolStringPoolEntryRef
+LLVM_C_ABI LLVMOrcSymbolStringPoolEntryRef
 LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName);
 
 /**
@@ -167,8 +169,8 @@ LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName);
  * Resources associated with the given object will be tracked by the given
  * JITDylib's default resource tracker.
  */
-LLVMErrorRef LLVMOrcLLJITAddObjectFile(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD,
-                                       LLVMMemoryBufferRef ObjBuffer);
+LLVM_C_ABI LLVMErrorRef LLVMOrcLLJITAddObjectFile(
+    LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, LLVMMemoryBufferRef ObjBuffer);
 
 /**
  * Add a buffer representing an object file to the given ResourceTracker's
@@ -179,9 +181,9 @@ LLVMErrorRef LLVMOrcLLJITAddObjectFile(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD,
  * Resources associated with the given object will be tracked by ResourceTracker
  * RT.
  */
-LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcLLJITRef J,
-                                             LLVMOrcResourceTrackerRef RT,
-                                             LLVMMemoryBufferRef ObjBuffer);
+LLVM_C_ABI LLVMErrorRef
+LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcLLJITRef J, LLVMOrcResourceTrackerRef RT,
+                                LLVMMemoryBufferRef ObjBuffer);
 
 /**
  * Add an IR module to the given JITDylib in the given LLJIT instance. This
@@ -192,9 +194,8 @@ LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcLLJITRef J,
  * Resources associated with the given Module will be tracked by the given
  * JITDylib's default resource tracker.
  */
-LLVMErrorRef LLVMOrcLLJITAddLLVMIRModule(LLVMOrcLLJITRef J,
-                                         LLVMOrcJITDylibRef JD,
-                                         LLVMOrcThreadSafeModuleRef TSM);
+LLVM_C_ABI LLVMErrorRef LLVMOrcLLJITAddLLVMIRModule(
+    LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, LLVMOrcThreadSafeModuleRef TSM);
 
 /**
  * Add an IR module to the given ResourceTracker's JITDylib in the given LLJIT
@@ -205,34 +206,36 @@ LLVMErrorRef LLVMOrcLLJITAddLLVMIRModule(LLVMOrcLLJITRef J,
  * Resources associated with the given Module will be tracked by ResourceTracker
  * RT.
  */
-LLVMErrorRef LLVMOrcLLJITAddLLVMIRModuleWithRT(LLVMOrcLLJITRef J,
-                                               LLVMOrcResourceTrackerRef JD,
-                                               LLVMOrcThreadSafeModuleRef TSM);
+LLVM_C_ABI LLVMErrorRef LLVMOrcLLJITAddLLVMIRModuleWithRT(
+    LLVMOrcLLJITRef J, LLVMOrcResourceTrackerRef JD,
+    LLVMOrcThreadSafeModuleRef TSM);
 
 /**
  * Look up the given symbol in the main JITDylib of the given LLJIT instance.
  *
  * This operation does not take ownership of the Name argument.
  */
-LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J,
-                                LLVMOrcExecutorAddress *Result,
-                                const char *Name);
+LLVM_C_ABI LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J,
+                                           LLVMOrcExecutorAddress *Result,
+                                           const char *Name);
 
 /**
  * Returns a non-owning reference to the LLJIT instance's object linking layer.
  */
-LLVMOrcObjectLayerRef LLVMOrcLLJITGetObjLinkingLayer(LLVMOrcLLJITRef J);
+LLVM_C_ABI LLVMOrcObjectLayerRef
+LLVMOrcLLJITGetObjLinkingLayer(LLVMOrcLLJITRef J);
 
 /**
  * Returns a non-owning reference to the LLJIT instance's object linking layer.
  */
-LLVMOrcObjectTransformLayerRef
+LLVM_C_ABI LLVMOrcObjectTransformLayerRef
 LLVMOrcLLJITGetObjTransformLayer(LLVMOrcLLJITRef J);
 
 /**
  * Returns a non-owning reference to the LLJIT instance's IR transform layer.
  */
-LLVMOrcIRTransformLayerRef LLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J);
+LLVM_C_ABI LLVMOrcIRTransformLayerRef
+LLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J);
 
 /**
  * Get the LLJIT instance's default data layout string.
@@ -240,7 +243,7 @@ LLVMOrcIRTransformLayerRef LLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J);
  * This string is owned by the LLJIT instance and does not need to be freed
  * by the caller.
  */
-const char *LLVMOrcLLJITGetDataLayoutStr(LLVMOrcLLJITRef J);
+LLVM_C_ABI const char *LLVMOrcLLJITGetDataLayoutStr(LLVMOrcLLJITRef J);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/LLJITUtils.h b/llvm/include/llvm-c/LLJITUtils.h
index 940097432b78f..4064d5907bc8a 100644
--- a/llvm/include/llvm-c/LLJITUtils.h
+++ b/llvm/include/llvm-c/LLJITUtils.h
@@ -27,6 +27,7 @@
 #define LLVM_C_LLJITUTILS_H
 
 #include "llvm-c/LLJIT.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -41,7 +42,7 @@ LLVM_C_EXTERN_C_BEGIN
  * Install the plugin that submits debug objects to the executor. Executors must
  * expose the llvm_orc_registerJITLoaderGDBWrapper symbol.
  */
-LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J);
+LLVM_C_ABI LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Linker.h b/llvm/include/llvm-c/Linker.h
index acff5d5e22253..dfcb67e817426 100644
--- a/llvm/include/llvm-c/Linker.h
+++ b/llvm/include/llvm-c/Linker.h
@@ -16,6 +16,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -38,7 +39,7 @@ typedef enum {
  * The return value is true if an error occurred, false otherwise.
  * Use the diagnostic handler to get any diagnostic message.
 */
-LLVMBool LLVMLinkModules2(LLVMModuleRef Dest, LLVMModuleRef Src);
+LLVM_C_ABI LLVMBool LLVMLinkModules2(LLVMModuleRef Dest, LLVMModuleRef Src);
 
 LLVM_C_EXTERN_C_END
 
diff --git a/llvm/include/llvm-c/Object.h b/llvm/include/llvm-c/Object.h
index f422c1ad224d2..0fc4ebe969a06 100644
--- a/llvm/include/llvm-c/Object.h
+++ b/llvm/include/llvm-c/Object.h
@@ -21,6 +21,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/Config/llvm-config.h"
 
 LLVM_C_EXTERN_C_BEGIN
@@ -73,9 +74,9 @@ typedef enum {
  *
  * @see llvm::object::createBinary
  */
-LLVMBinaryRef LLVMCreateBinary(LLVMMemoryBufferRef MemBuf,
-                               LLVMContextRef Context,
-                               char **ErrorMessage);
+LLVM_C_ABI LLVMBinaryRef LLVMCreateBinary(LLVMMemoryBufferRef MemBuf,
+                                          LLVMContextRef Context,
+                                          char **ErrorMessage);
 
 /**
  * Dispose of a binary file.
@@ -83,7 +84,7 @@ LLVMBinaryRef LLVMCreateBinary(LLVMMemoryBufferRef MemBuf,
  * The binary file does not own its backing buffer.  It is the responsibilty
  * of the caller to free it with \c LLVMDisposeMemoryBuffer.
  */
-void LLVMDisposeBinary(LLVMBinaryRef BR);
+LLVM_C_ABI void LLVMDisposeBinary(LLVMBinaryRef BR);
 
 /**
  * Retrieves a copy of the memory buffer associated with this object file.
@@ -94,14 +95,14 @@ void LLVMDisposeBinary(LLVMBinaryRef BR);
  *
  * @see llvm::object::getMemoryBufferRef
  */
-LLVMMemoryBufferRef LLVMBinaryCopyMemoryBuffer(LLVMBinaryRef BR);
+LLVM_C_ABI LLVMMemoryBufferRef LLVMBinaryCopyMemoryBuffer(LLVMBinaryRef BR);
 
 /**
  * Retrieve the specific type of a binary.
  *
  * @see llvm::object::Binary::getType
  */
-LLVMBinaryType LLVMBinaryGetType(LLVMBinaryRef BR);
+LLVM_C_ABI LLVMBinaryType LLVMBinaryGetType(LLVMBinaryRef BR);
 
 /*
  * For a Mach-O universal binary file, retrieves the object file corresponding
@@ -114,10 +115,8 @@ LLVMBinaryType LLVMBinaryGetType(LLVMBinaryRef BR);
  * It is the responsiblity of the caller to free the returned object file by
  * calling \c LLVMDisposeBinary.
  */
-LLVMBinaryRef LLVMMachOUniversalBinaryCopyObjectForArch(LLVMBinaryRef BR,
-                                                        const char *Arch,
-                                                        size_t ArchLen,
-                                                        char **ErrorMessage);
+LLVM_C_ABI LLVMBinaryRef LLVMMachOUniversalBinaryCopyObjectForArch(
+    LLVMBinaryRef BR, const char *Arch, size_t ArchLen, char **ErrorMessage);
 
 /**
  * Retrieve a copy of the section iterator for this object file.
@@ -130,15 +129,16 @@ LLVMBinaryRef LLVMMachOUniversalBinaryCopyObjectForArch(LLVMBinaryRef BR,
  *
  * @see llvm::object::sections()
  */
-LLVMSectionIteratorRef LLVMObjectFileCopySectionIterator(LLVMBinaryRef BR);
+LLVM_C_ABI LLVMSectionIteratorRef
+LLVMObjectFileCopySectionIterator(LLVMBinaryRef BR);
 
 /**
  * Returns whether the given section iterator is at the end.
  *
  * @see llvm::object::section_end
  */
-LLVMBool LLVMObjectFileIsSectionIteratorAtEnd(LLVMBinaryRef BR,
-                                              LLVMSectionIteratorRef SI);
+LLVM_C_ABI LLVMBool LLVMObjectFileIsSectionIteratorAtEnd(
+    LLVMBinaryRef BR, LLVMSectionIteratorRef SI);
 
 /**
  * Retrieve a copy of the symbol iterator for this object file.
@@ -151,78 +151,81 @@ LLVMBool LLVMObjectFileIsSectionIteratorAtEnd(LLVMBinaryRef BR,
  *
  * @see llvm::object::symbols()
  */
-LLVMSymbolIteratorRef LLVMObjectFileCopySymbolIterator(LLVMBinaryRef BR);
+LLVM_C_ABI LLVMSymbolIteratorRef
+LLVMObjectFileCopySymbolIterator(LLVMBinaryRef BR);
 
 /**
  * Returns whether the given symbol iterator is at the end.
  *
  * @see llvm::object::symbol_end
  */
-LLVMBool LLVMObjectFileIsSymbolIteratorAtEnd(LLVMBinaryRef BR,
-                                             LLVMSymbolIteratorRef SI);
+LLVM_C_ABI LLVMBool
+LLVMObjectFileIsSymbolIteratorAtEnd(LLVMBinaryRef BR, LLVMSymbolIteratorRef SI);
 
-void LLVMDisposeSectionIterator(LLVMSectionIteratorRef SI);
+LLVM_C_ABI void LLVMDisposeSectionIterator(LLVMSectionIteratorRef SI);
 
-void LLVMMoveToNextSection(LLVMSectionIteratorRef SI);
-void LLVMMoveToContainingSection(LLVMSectionIteratorRef Sect,
-                                 LLVMSymbolIteratorRef Sym);
+LLVM_C_ABI void LLVMMoveToNextSection(LLVMSectionIteratorRef SI);
+LLVM_C_ABI void LLVMMoveToContainingSection(LLVMSectionIteratorRef Sect,
+                                            LLVMSymbolIteratorRef Sym);
 
 // ObjectFile Symbol iterators
-void LLVMDisposeSymbolIterator(LLVMSymbolIteratorRef SI);
-void LLVMMoveToNextSymbol(LLVMSymbolIteratorRef SI);
+LLVM_C_ABI void LLVMDisposeSymbolIterator(LLVMSymbolIteratorRef SI);
+LLVM_C_ABI void LLVMMoveToNextSymbol(LLVMSymbolIteratorRef SI);
 
 // SectionRef accessors
-const char *LLVMGetSectionName(LLVMSectionIteratorRef SI);
-uint64_t LLVMGetSectionSize(LLVMSectionIteratorRef SI);
-const char *LLVMGetSectionContents(LLVMSectionIteratorRef SI);
-uint64_t LLVMGetSectionAddress(LLVMSectionIteratorRef SI);
-LLVMBool LLVMGetSectionContainsSymbol(LLVMSectionIteratorRef SI,
-                                 LLVMSymbolIteratorRef Sym);
+LLVM_C_ABI const char *LLVMGetSectionName(LLVMSectionIteratorRef SI);
+LLVM_C_ABI uint64_t LLVMGetSectionSize(LLVMSectionIteratorRef SI);
+LLVM_C_ABI const char *LLVMGetSectionContents(LLVMSectionIteratorRef SI);
+LLVM_C_ABI uint64_t LLVMGetSectionAddress(LLVMSectionIteratorRef SI);
+LLVM_C_ABI LLVMBool LLVMGetSectionContainsSymbol(LLVMSectionIteratorRef SI,
+                                                 LLVMSymbolIteratorRef Sym);
 
 // Section Relocation iterators
-LLVMRelocationIteratorRef LLVMGetRelocations(LLVMSectionIteratorRef Section);
-void LLVMDisposeRelocationIterator(LLVMRelocationIteratorRef RI);
-LLVMBool LLVMIsRelocationIteratorAtEnd(LLVMSectionIteratorRef Section,
-                                       LLVMRelocationIteratorRef RI);
-void LLVMMoveToNextRelocation(LLVMRelocationIteratorRef RI);
-
+LLVM_C_ABI LLVMRelocationIteratorRef
+LLVMGetRelocations(LLVMSectionIteratorRef Section);
+LLVM_C_ABI void LLVMDisposeRelocationIterator(LLVMRelocationIteratorRef RI);
+LLVM_C_ABI LLVMBool LLVMIsRelocationIteratorAtEnd(
+    LLVMSectionIteratorRef Section, LLVMRelocationIteratorRef RI);
+LLVM_C_ABI void LLVMMoveToNextRelocation(LLVMRelocationIteratorRef RI);
 
 // SymbolRef accessors
-const char *LLVMGetSymbolName(LLVMSymbolIteratorRef SI);
-uint64_t LLVMGetSymbolAddress(LLVMSymbolIteratorRef SI);
-uint64_t LLVMGetSymbolSize(LLVMSymbolIteratorRef SI);
+LLVM_C_ABI const char *LLVMGetSymbolName(LLVMSymbolIteratorRef SI);
+LLVM_C_ABI uint64_t LLVMGetSymbolAddress(LLVMSymbolIteratorRef SI);
+LLVM_C_ABI uint64_t LLVMGetSymbolSize(LLVMSymbolIteratorRef SI);
 
 // RelocationRef accessors
-uint64_t LLVMGetRelocationOffset(LLVMRelocationIteratorRef RI);
-LLVMSymbolIteratorRef LLVMGetRelocationSymbol(LLVMRelocationIteratorRef RI);
-uint64_t LLVMGetRelocationType(LLVMRelocationIteratorRef RI);
+LLVM_C_ABI uint64_t LLVMGetRelocationOffset(LLVMRelocationIteratorRef RI);
+LLVM_C_ABI LLVMSymbolIteratorRef
+LLVMGetRelocationSymbol(LLVMRelocationIteratorRef RI);
+LLVM_C_ABI uint64_t LLVMGetRelocationType(LLVMRelocationIteratorRef RI);
 // NOTE: Caller takes ownership of returned string of the two
 // following functions.
-const char *LLVMGetRelocationTypeName(LLVMRelocationIteratorRef RI);
-const char *LLVMGetRelocationValueString(LLVMRelocationIteratorRef RI);
+LLVM_C_ABI const char *LLVMGetRelocationTypeName(LLVMRelocationIteratorRef RI);
+LLVM_C_ABI const char *
+LLVMGetRelocationValueString(LLVMRelocationIteratorRef RI);
 
 /** Deprecated: Use LLVMBinaryRef instead. */
 typedef struct LLVMOpaqueObjectFile *LLVMObjectFileRef;
 
 /** Deprecated: Use LLVMCreateBinary instead. */
-LLVMObjectFileRef LLVMCreateObjectFile(LLVMMemoryBufferRef MemBuf);
+LLVM_C_ABI LLVMObjectFileRef LLVMCreateObjectFile(LLVMMemoryBufferRef MemBuf);
 
 /** Deprecated: Use LLVMDisposeBinary instead. */
-void LLVMDisposeObjectFile(LLVMObjectFileRef ObjectFile);
+LLVM_C_ABI void LLVMDisposeObjectFile(LLVMObjectFileRef ObjectFile);
 
 /** Deprecated: Use LLVMObjectFileCopySectionIterator instead. */
-LLVMSectionIteratorRef LLVMGetSections(LLVMObjectFileRef ObjectFile);
+LLVM_C_ABI LLVMSectionIteratorRef LLVMGetSections(LLVMObjectFileRef ObjectFile);
 
 /** Deprecated: Use LLVMObjectFileIsSectionIteratorAtEnd instead. */
-LLVMBool LLVMIsSectionIteratorAtEnd(LLVMObjectFileRef ObjectFile,
-                                    LLVMSectionIteratorRef SI);
+LLVM_C_ABI LLVMBool LLVMIsSectionIteratorAtEnd(LLVMObjectFileRef ObjectFile,
+                                               LLVMSectionIteratorRef SI);
 
 /** Deprecated: Use LLVMObjectFileCopySymbolIterator instead. */
-LLVMSymbolIteratorRef LLVMGetSymbols(LLVMObjectFileRef ObjectFile);
+LLVM_C_ABI LLVMSymbolIteratorRef LLVMGetSymbols(LLVMObjectFileRef ObjectFile);
 
 /** Deprecated: Use LLVMObjectFileIsSymbolIteratorAtEnd instead. */
-LLVMBool LLVMIsSymbolIteratorAtEnd(LLVMObjectFileRef ObjectFile,
-                                   LLVMSymbolIteratorRef SI);
+LLVM_C_ABI LLVMBool LLVMIsSymbolIteratorAtEnd(LLVMObjectFileRef ObjectFile,
+                                              LLVMSymbolIteratorRef SI);
 /**
  * @}
  */
diff --git a/llvm/include/llvm-c/Orc.h b/llvm/include/llvm-c/Orc.h
index 743ba1d581782..ce8ee0de2d407 100644
--- a/llvm/include/llvm-c/Orc.h
+++ b/llvm/include/llvm-c/Orc.h
@@ -30,6 +30,7 @@
 #include "llvm-c/Error.h"
 #include "llvm-c/TargetMachine.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -488,7 +489,7 @@ typedef struct LLVMOrcOpaqueDumpObjects *LLVMOrcDumpObjectsRef;
  * requesting definitions from the JIT will typically be delivered a
  * FailureToMaterialize error instead).
  */
-void LLVMOrcExecutionSessionSetErrorReporter(
+LLVM_C_ABI void LLVMOrcExecutionSessionSetErrorReporter(
     LLVMOrcExecutionSessionRef ES, LLVMOrcErrorReporterFunction ReportError,
     void *Ctx);
 
@@ -498,7 +499,7 @@ void LLVMOrcExecutionSessionSetErrorReporter(
  * Ownership of the pool remains with the ExecutionSession: The caller is
  * not required to free the pool.
  */
-LLVMOrcSymbolStringPoolRef
+LLVM_C_ABI LLVMOrcSymbolStringPoolRef
 LLVMOrcExecutionSessionGetSymbolStringPool(LLVMOrcExecutionSessionRef ES);
 
 /**
@@ -511,7 +512,8 @@ LLVMOrcExecutionSessionGetSymbolStringPool(LLVMOrcExecutionSessionRef ES);
  * entries will have become unreferenced, e.g. after removing a module or
  * closing a JITDylib.
  */
-void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP);
+LLVM_C_ABI void
+LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP);
 
 /**
  * Intern a string in the ExecutionSession's SymbolStringPool and return a
@@ -525,7 +527,7 @@ void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP);
  *
  * Note that this function does not perform linker-mangling on the string.
  */
-LLVMOrcSymbolStringPoolEntryRef
+LLVM_C_ABI LLVMOrcSymbolStringPoolEntryRef
 LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name);
 
 /**
@@ -574,7 +576,7 @@ typedef void (*LLVMOrcExecutionSessionLookupHandleResultFunction)(
  *
  * THIS API IS EXPERIMENTAL AND LIKELY TO CHANGE IN THE NEAR FUTURE!
  */
-void LLVMOrcExecutionSessionLookup(
+LLVM_C_ABI void LLVMOrcExecutionSessionLookup(
     LLVMOrcExecutionSessionRef ES, LLVMOrcLookupKind K,
     LLVMOrcCJITDylibSearchOrder SearchOrder, size_t SearchOrderSize,
     LLVMOrcCLookupSet Symbols, size_t SymbolsSize,
@@ -583,49 +585,56 @@ void LLVMOrcExecutionSessionLookup(
 /**
  * Increments the ref-count for a SymbolStringPool entry.
  */
-void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S);
+LLVM_C_ABI void
+LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S);
 
 /**
  * Reduces the ref-count for of a SymbolStringPool entry.
  */
-void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S);
+LLVM_C_ABI void
+LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S);
 
 /**
  * Return the c-string for the given symbol. This string will remain valid until
  * the entry is freed (once all LLVMOrcSymbolStringPoolEntryRefs have been
  * released).
  */
-const char *LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S);
+LLVM_C_ABI const char *
+LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S);
 
 /**
  * Reduces the ref-count of a ResourceTracker.
  */
-void LLVMOrcReleaseResourceTracker(LLVMOrcResourceTrackerRef RT);
+LLVM_C_ABI void LLVMOrcReleaseResourceTracker(LLVMOrcResourceTrackerRef RT);
 
 /**
  * Transfers tracking of all resources associated with resource tracker SrcRT
  * to resource tracker DstRT.
  */
-void LLVMOrcResourceTrackerTransferTo(LLVMOrcResourceTrackerRef SrcRT,
-                                      LLVMOrcResourceTrackerRef DstRT);
+LLVM_C_ABI void
+LLVMOrcResourceTrackerTransferTo(LLVMOrcResourceTrackerRef SrcRT,
+                                 LLVMOrcResourceTrackerRef DstRT);
 
 /**
  * Remove all resources associated with the given tracker. See
  * ResourceTracker::remove().
  */
-LLVMErrorRef LLVMOrcResourceTrackerRemove(LLVMOrcResourceTrackerRef RT);
+LLVM_C_ABI LLVMErrorRef
+LLVMOrcResourceTrackerRemove(LLVMOrcResourceTrackerRef RT);
 
 /**
  * Dispose of a JITDylib::DefinitionGenerator. This should only be called if
  * ownership has not been passed to a JITDylib (e.g. because some error
  * prevented the client from calling LLVMOrcJITDylibAddGenerator).
  */
-void LLVMOrcDisposeDefinitionGenerator(LLVMOrcDefinitionGeneratorRef DG);
+LLVM_C_ABI void
+LLVMOrcDisposeDefinitionGenerator(LLVMOrcDefinitionGeneratorRef DG);
 
 /**
  * Dispose of a MaterializationUnit.
  */
-void LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU);
+LLVM_C_ABI void
+LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU);
 
 /**
  * Create a custom MaterializationUnit.
@@ -660,7 +669,7 @@ void LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU);
  * either passing it to a JITDylib (via LLVMOrcJITDylibDefine) or disposing
  * of it by calling LLVMOrcDisposeMaterializationUnit.
  */
-LLVMOrcMaterializationUnitRef LLVMOrcCreateCustomMaterializationUnit(
+LLVM_C_ABI LLVMOrcMaterializationUnitRef LLVMOrcCreateCustomMaterializationUnit(
     const char *Name, void *Ctx, LLVMOrcCSymbolFlagsMapPairs Syms,
     size_t NumSyms, LLVMOrcSymbolStringPoolEntryRef InitSym,
     LLVMOrcMaterializationUnitMaterializeFunction Materialize,
@@ -688,7 +697,7 @@ LLVMOrcMaterializationUnitRef LLVMOrcCreateCustomMaterializationUnit(
  * If a client wishes to reuse elements of the Sym array after this call they
  * must explicitly retain each of the elements for themselves.
  */
-LLVMOrcMaterializationUnitRef
+LLVM_C_ABI LLVMOrcMaterializationUnitRef
 LLVMOrcAbsoluteSymbols(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs);
 
 /**
@@ -712,7 +721,7 @@ LLVMOrcAbsoluteSymbols(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs);
  * If a client wishes to reuse elements of the CallableAliases array after this call they
  * must explicitly retain each of the elements for themselves.
  */
-LLVMOrcMaterializationUnitRef LLVMOrcLazyReexports(
+LLVM_C_ABI LLVMOrcMaterializationUnitRef LLVMOrcLazyReexports(
     LLVMOrcLazyCallThroughManagerRef LCTM, LLVMOrcIndirectStubsManagerRef ISM,
     LLVMOrcJITDylibRef SourceRef, LLVMOrcCSymbolAliasMapPairs CallableAliases,
     size_t NumPairs);
@@ -727,19 +736,20 @@ LLVMOrcMaterializationUnitRef LLVMOrcLazyReexports(
  * LLVMOrcMaterializationResponsibilityNotifyEmitted) or failed (via
  * LLVMOrcMaterializationResponsibilityFailMaterialization).
  */
-void LLVMOrcDisposeMaterializationResponsibility(
+LLVM_C_ABI void LLVMOrcDisposeMaterializationResponsibility(
     LLVMOrcMaterializationResponsibilityRef MR);
 
 /**
  * Returns the target JITDylib that these symbols are being materialized into.
  */
-LLVMOrcJITDylibRef LLVMOrcMaterializationResponsibilityGetTargetDylib(
+LLVM_C_ABI LLVMOrcJITDylibRef
+LLVMOrcMaterializationResponsibilityGetTargetDylib(
     LLVMOrcMaterializationResponsibilityRef MR);
 
 /**
  * Returns the ExecutionSession for this MaterializationResponsibility.
  */
-LLVMOrcExecutionSessionRef
+LLVM_C_ABI LLVMOrcExecutionSessionRef
 LLVMOrcMaterializationResponsibilityGetExecutionSession(
     LLVMOrcMaterializationResponsibilityRef MR);
 
@@ -753,7 +763,8 @@ LLVMOrcMaterializationResponsibilityGetExecutionSession(
  * MaterializationResponsibility requires the caller to retain the symbols
  * explicitly.
  */
-LLVMOrcCSymbolFlagsMapPairs LLVMOrcMaterializationResponsibilityGetSymbols(
+LLVM_C_ABI LLVMOrcCSymbolFlagsMapPairs
+LLVMOrcMaterializationResponsibilityGetSymbols(
     LLVMOrcMaterializationResponsibilityRef MR, size_t *NumPairs);
 
 /**
@@ -761,7 +772,8 @@ LLVMOrcCSymbolFlagsMapPairs LLVMOrcMaterializationResponsibilityGetSymbols(
  *
  * Does not release the entries themselves.
  */
-void LLVMOrcDisposeCSymbolFlagsMap(LLVMOrcCSymbolFlagsMapPairs Pairs);
+LLVM_C_ABI void
+LLVMOrcDisposeCSymbolFlagsMap(LLVMOrcCSymbolFlagsMapPairs Pairs);
 
 /**
  * Returns the initialization pseudo-symbol, if any. This symbol will also
@@ -771,7 +783,7 @@ void LLVMOrcDisposeCSymbolFlagsMap(LLVMOrcCSymbolFlagsMapPairs Pairs);
  * The returned symbol is not retained over any mutating operation of the
  * MaterializationResponsbility or beyond the lifetime thereof.
  */
-LLVMOrcSymbolStringPoolEntryRef
+LLVM_C_ABI LLVMOrcSymbolStringPoolEntryRef
 LLVMOrcMaterializationResponsibilityGetInitializerSymbol(
     LLVMOrcMaterializationResponsibilityRef MR);
 
@@ -781,7 +793,7 @@ LLVMOrcMaterializationResponsibilityGetInitializerSymbol(
  * information can be used to return responsibility for unrequested symbols
  * back to the JITDylib via the delegate method.
  */
-LLVMOrcSymbolStringPoolEntryRef *
+LLVM_C_ABI LLVMOrcSymbolStringPoolEntryRef *
 LLVMOrcMaterializationResponsibilityGetRequestedSymbols(
     LLVMOrcMaterializationResponsibilityRef MR, size_t *NumSymbols);
 
@@ -790,7 +802,7 @@ LLVMOrcMaterializationResponsibilityGetRequestedSymbols(
  *
  * Does not release the symbols themselves.
  */
-void LLVMOrcDisposeSymbols(LLVMOrcSymbolStringPoolEntryRef *Symbols);
+LLVM_C_ABI void LLVMOrcDisposeSymbols(LLVMOrcSymbolStringPoolEntryRef *Symbols);
 
 /**
  * Notifies the target JITDylib that the given symbols have been resolved.
@@ -808,7 +820,7 @@ void LLVMOrcDisposeSymbols(LLVMOrcSymbolStringPoolEntryRef *Symbols);
  * MaterializationResponsibility then this method is guaranteed to return
  * LLVMErrorSuccess.
  */
-LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyResolved(
+LLVM_C_ABI LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyResolved(
     LLVMOrcMaterializationResponsibilityRef MR, LLVMOrcCSymbolMapPairs Symbols,
     size_t NumPairs);
 
@@ -838,7 +850,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyResolved(
  * MaterializationResponsibility then this method is guaranteed to return
  * LLVMErrorSuccess.
  */
-LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyEmitted(
+LLVM_C_ABI LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyEmitted(
     LLVMOrcMaterializationResponsibilityRef MR,
     LLVMOrcCSymbolDependenceGroup *SymbolDepGroups, size_t NumSymbolDepGroups);
 
@@ -855,7 +867,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyEmitted(
  * additional symbols at materialization time (e.g. stubs, compile
  * callbacks, metadata)
  */
-LLVMErrorRef LLVMOrcMaterializationResponsibilityDefineMaterializing(
+LLVM_C_ABI LLVMErrorRef LLVMOrcMaterializationResponsibilityDefineMaterializing(
     LLVMOrcMaterializationResponsibilityRef MR,
     LLVMOrcCSymbolFlagsMapPairs Pairs, size_t NumPairs);
 
@@ -866,7 +878,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityDefineMaterializing(
  * from the target JITDylib, and send an error to any queries waiting on
  * these symbols.
  */
-void LLVMOrcMaterializationResponsibilityFailMaterialization(
+LLVM_C_ABI void LLVMOrcMaterializationResponsibilityFailMaterialization(
     LLVMOrcMaterializationResponsibilityRef MR);
 
 /**
@@ -876,7 +888,7 @@ void LLVMOrcMaterializationResponsibilityFailMaterialization(
  * by introspecting which symbols have actually been looked up and
  * materializing only those).
  */
-LLVMErrorRef LLVMOrcMaterializationResponsibilityReplace(
+LLVM_C_ABI LLVMErrorRef LLVMOrcMaterializationResponsibilityReplace(
     LLVMOrcMaterializationResponsibilityRef MR,
     LLVMOrcMaterializationUnitRef MU);
 
@@ -888,7 +900,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityReplace(
  * The caller retains responsibility of the the passed
  * MaterializationResponsibility.
  */
-LLVMErrorRef LLVMOrcMaterializationResponsibilityDelegate(
+LLVM_C_ABI LLVMErrorRef LLVMOrcMaterializationResponsibilityDelegate(
     LLVMOrcMaterializationResponsibilityRef MR,
     LLVMOrcSymbolStringPoolEntryRef *Symbols, size_t NumSymbols,
     LLVMOrcMaterializationResponsibilityRef *Result);
@@ -902,9 +914,8 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityDelegate(
  * This call does not install any library code or symbols into the newly
  * created JITDylib. The client is responsible for all configuration.
  */
-LLVMOrcJITDylibRef
-LLVMOrcExecutionSessionCreateBareJITDylib(LLVMOrcExecutionSessionRef ES,
-                                          const char *Name);
+LLVM_C_ABI LLVMOrcJITDylibRef LLVMOrcExecutionSessionCreateBareJITDylib(
+    LLVMOrcExecutionSessionRef ES, const char *Name);
 
 /**
  * Create a JITDylib.
@@ -918,25 +929,23 @@ LLVMOrcExecutionSessionCreateBareJITDylib(LLVMOrcExecutionSessionRef ES,
  * call is equivalent to LLVMExecutionSessionRefCreateBareJITDylib and will
  * always return success.
  */
-LLVMErrorRef
-LLVMOrcExecutionSessionCreateJITDylib(LLVMOrcExecutionSessionRef ES,
-                                      LLVMOrcJITDylibRef *Result,
-                                      const char *Name);
+LLVM_C_ABI LLVMErrorRef LLVMOrcExecutionSessionCreateJITDylib(
+    LLVMOrcExecutionSessionRef ES, LLVMOrcJITDylibRef *Result,
+    const char *Name);
 
 /**
  * Returns the JITDylib with the given name, or NULL if no such JITDylib
  * exists.
  */
-LLVMOrcJITDylibRef
-LLVMOrcExecutionSessionGetJITDylibByName(LLVMOrcExecutionSessionRef ES,
-                                         const char *Name);
+LLVM_C_ABI LLVMOrcJITDylibRef LLVMOrcExecutionSessionGetJITDylibByName(
+    LLVMOrcExecutionSessionRef ES, const char *Name);
 
 /**
  * Return a reference to a newly created resource tracker associated with JD.
  * The tracker is returned with an initial ref-count of 1, and must be released
  * with LLVMOrcReleaseResourceTracker when no longer needed.
  */
-LLVMOrcResourceTrackerRef
+LLVM_C_ABI LLVMOrcResourceTrackerRef
 LLVMOrcJITDylibCreateResourceTracker(LLVMOrcJITDylibRef JD);
 
 /**
@@ -944,7 +953,7 @@ LLVMOrcJITDylibCreateResourceTracker(LLVMOrcJITDylibRef JD);
  * This operation will increase the retain count of the tracker: Clients should
  * call LLVMOrcReleaseResourceTracker when the result is no longer needed.
  */
-LLVMOrcResourceTrackerRef
+LLVM_C_ABI LLVMOrcResourceTrackerRef
 LLVMOrcJITDylibGetDefaultResourceTracker(LLVMOrcJITDylibRef JD);
 
 /**
@@ -954,14 +963,14 @@ LLVMOrcJITDylibGetDefaultResourceTracker(LLVMOrcJITDylibRef JD);
  * If the operation fails then ownership remains with the caller who should
  * call LLVMOrcDisposeMaterializationUnit to destroy it.
  */
-LLVMErrorRef LLVMOrcJITDylibDefine(LLVMOrcJITDylibRef JD,
-                                   LLVMOrcMaterializationUnitRef MU);
+LLVM_C_ABI LLVMErrorRef LLVMOrcJITDylibDefine(LLVMOrcJITDylibRef JD,
+                                              LLVMOrcMaterializationUnitRef MU);
 
 /**
  * Calls remove on all trackers associated with this JITDylib, see
  * JITDylib::clear().
  */
-LLVMErrorRef LLVMOrcJITDylibClear(LLVMOrcJITDylibRef JD);
+LLVM_C_ABI LLVMErrorRef LLVMOrcJITDylibClear(LLVMOrcJITDylibRef JD);
 
 /**
  * Add a DefinitionGenerator to the given JITDylib.
@@ -969,8 +978,8 @@ LLVMErrorRef LLVMOrcJITDylibClear(LLVMOrcJITDylibRef JD);
  * The JITDylib will take ownership of the given generator: The client is no
  * longer responsible for managing its memory.
  */
-void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD,
-                                 LLVMOrcDefinitionGeneratorRef DG);
+LLVM_C_ABI void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD,
+                                            LLVMOrcDefinitionGeneratorRef DG);
 
 /**
  * Create a custom generator.
@@ -985,7 +994,8 @@ void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD,
  * Dispose is the disposal function for Ctx. This argument is permitted to be
  * null (in which case the client is responsible for the lifetime of Ctx).
  */
-LLVMOrcDefinitionGeneratorRef LLVMOrcCreateCustomCAPIDefinitionGenerator(
+LLVM_C_ABI LLVMOrcDefinitionGeneratorRef
+LLVMOrcCreateCustomCAPIDefinitionGenerator(
     LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction F, void *Ctx,
     LLVMOrcDisposeCAPIDefinitionGeneratorFunction Dispose);
 
@@ -993,8 +1003,8 @@ LLVMOrcDefinitionGeneratorRef LLVMOrcCreateCustomCAPIDefinitionGenerator(
  * Continue a lookup that was suspended in a generator (see
  * LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction).
  */
-void LLVMOrcLookupStateContinueLookup(LLVMOrcLookupStateRef S,
-                                      LLVMErrorRef Err);
+LLVM_C_ABI void LLVMOrcLookupStateContinueLookup(LLVMOrcLookupStateRef S,
+                                                 LLVMErrorRef Err);
 
 /**
  * Get a DynamicLibrarySearchGenerator that will reflect process symbols into
@@ -1014,7 +1024,7 @@ void LLVMOrcLookupStateContinueLookup(LLVMOrcLookupStateRef S,
  * function is the full mangled symbol: The client is responsible for stripping
  * the global prefix if present.
  */
-LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(
+LLVM_C_ABI LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(
     LLVMOrcDefinitionGeneratorRef *Result, char GlobalPrefx,
     LLVMOrcSymbolPredicate Filter, void *FilterCtx);
 
@@ -1039,7 +1049,7 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(
  * THIS API IS EXPERIMENTAL AND LIKELY TO CHANGE IN THE NEAR FUTURE!
  * 
  */
-LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath(
+LLVM_C_ABI LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath(
     LLVMOrcDefinitionGeneratorRef *Result, const char *FileName,
     char GlobalPrefix, LLVMOrcSymbolPredicate Filter, void *FilterCtx);
 
@@ -1057,30 +1067,43 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath(
  * THIS API IS EXPERIMENTAL AND LIKELY TO CHANGE IN THE NEAR FUTURE!
  * 
  */
-LLVMErrorRef LLVMOrcCreateStaticLibrarySearchGeneratorForPath(
+LLVM_C_ABI LLVMErrorRef LLVMOrcCreateStaticLibrarySearchGeneratorForPath(
     LLVMOrcDefinitionGeneratorRef *Result, LLVMOrcObjectLayerRef ObjLayer,
     const char *FileName);
 
 /**
- * Create a ThreadSafeContext containing a new LLVMContext.
+ * Create a ThreadSafeContextRef containing a new LLVMContext.
  *
  * Ownership of the underlying ThreadSafeContext data is shared: Clients
- * can and should dispose of their ThreadSafeContext as soon as they no longer
- * need to refer to it directly. Other references (e.g. from ThreadSafeModules)
- * will keep the data alive as long as it is needed.
+ * can and should dispose of their ThreadSafeContextRef as soon as they no
+ * longer need to refer to it directly. Other references (e.g. from
+ * ThreadSafeModules) will keep the underlying data alive as long as it is
+ * needed.
  */
-LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void);
+LLVM_C_ABI LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void);
 
 /**
- * Get a reference to the wrapped LLVMContext.
+ * Create a ThreadSafeContextRef from a given LLVMContext, which must not be
+ * associated with any existing ThreadSafeContext.
+ *
+ * The underlying ThreadSafeContext will take ownership of the LLVMContext
+ * object, so clients should not free the LLVMContext passed to this
+ * function.
+ *
+ * Ownership of the underlying ThreadSafeContext data is shared: Clients
+ * can and should dispose of their ThreadSafeContextRef as soon as they no
+ * longer need to refer to it directly. Other references (e.g. from
+ * ThreadSafeModules) will keep the underlying data alive as long as it is
+ * needed.
  */
-LLVMContextRef
-LLVMOrcThreadSafeContextGetContext(LLVMOrcThreadSafeContextRef TSCtx);
+LLVM_C_ABI LLVMOrcThreadSafeContextRef
+LLVMOrcCreateNewThreadSafeContextFromLLVMContext(LLVMContextRef Ctx);
 
 /**
  * Dispose of a ThreadSafeContext.
  */
-void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx);
+LLVM_C_ABI void
+LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx);
 
 /**
  * Create a ThreadSafeModule wrapper around the given LLVM module. This takes
@@ -1092,24 +1115,22 @@ void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx);
  * responsible for it. If it is not transferred to the JIT then the client
  * should call LLVMOrcDisposeThreadSafeModule to dispose of it.
  */
-LLVMOrcThreadSafeModuleRef
-LLVMOrcCreateNewThreadSafeModule(LLVMModuleRef M,
-                                 LLVMOrcThreadSafeContextRef TSCtx);
+LLVM_C_ABI LLVMOrcThreadSafeModuleRef LLVMOrcCreateNewThreadSafeModule(
+    LLVMModuleRef M, LLVMOrcThreadSafeContextRef TSCtx);
 
 /**
  * Dispose of a ThreadSafeModule. This should only be called if ownership has
  * not been passed to LLJIT (e.g. because some error prevented the client from
  * adding this to the JIT).
  */
-void LLVMOrcDisposeThreadSafeModule(LLVMOrcThreadSafeModuleRef TSM);
+LLVM_C_ABI void LLVMOrcDisposeThreadSafeModule(LLVMOrcThreadSafeModuleRef TSM);
 
 /**
  * Apply the given function to the module contained in this ThreadSafeModule.
  */
-LLVMErrorRef
-LLVMOrcThreadSafeModuleWithModuleDo(LLVMOrcThreadSafeModuleRef TSM,
-                                    LLVMOrcGenericIRModuleOperationFunction F,
-                                    void *Ctx);
+LLVM_C_ABI LLVMErrorRef LLVMOrcThreadSafeModuleWithModuleDo(
+    LLVMOrcThreadSafeModuleRef TSM, LLVMOrcGenericIRModuleOperationFunction F,
+    void *Ctx);
 
 /**
  * Create a JITTargetMachineBuilder by detecting the host.
@@ -1119,7 +1140,7 @@ LLVMOrcThreadSafeModuleWithModuleDo(LLVMOrcThreadSafeModuleRef TSM,
  * LLVMOrcLLJITBuilderSetJITTargetMachineBuilder) or disposed of by calling
  * LLVMOrcDisposeJITTargetMachineBuilder.
  */
-LLVMErrorRef LLVMOrcJITTargetMachineBuilderDetectHost(
+LLVM_C_ABI LLVMErrorRef LLVMOrcJITTargetMachineBuilderDetectHost(
     LLVMOrcJITTargetMachineBuilderRef *Result);
 
 /**
@@ -1131,14 +1152,14 @@ LLVMErrorRef LLVMOrcJITTargetMachineBuilderDetectHost(
  * LLVMOrcLLJITBuilderSetJITTargetMachineBuilder) or disposed of by calling
  * LLVMOrcDisposeJITTargetMachineBuilder.
  */
-LLVMOrcJITTargetMachineBuilderRef
+LLVM_C_ABI LLVMOrcJITTargetMachineBuilderRef
 LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM);
 
 /**
  * Dispose of a JITTargetMachineBuilder.
  */
-void LLVMOrcDisposeJITTargetMachineBuilder(
-    LLVMOrcJITTargetMachineBuilderRef JTMB);
+LLVM_C_ABI void
+LLVMOrcDisposeJITTargetMachineBuilder(LLVMOrcJITTargetMachineBuilderRef JTMB);
 
 /**
  * Returns the target triple for the given JITTargetMachineBuilder as a string.
@@ -1146,14 +1167,14 @@ void LLVMOrcDisposeJITTargetMachineBuilder(
  * The caller owns the resulting string as must dispose of it by calling
  * LLVMDisposeMessage
  */
-char *LLVMOrcJITTargetMachineBuilderGetTargetTriple(
+LLVM_C_ABI char *LLVMOrcJITTargetMachineBuilderGetTargetTriple(
     LLVMOrcJITTargetMachineBuilderRef JTMB);
 
 /**
  * Sets the target triple for the given JITTargetMachineBuilder to the given
  * string.
  */
-void LLVMOrcJITTargetMachineBuilderSetTargetTriple(
+LLVM_C_ABI void LLVMOrcJITTargetMachineBuilderSetTargetTriple(
     LLVMOrcJITTargetMachineBuilderRef JTMB, const char *TargetTriple);
 
 /**
@@ -1167,9 +1188,9 @@ void LLVMOrcJITTargetMachineBuilderSetTargetTriple(
  * Resources associated with the given object will be tracked by the given
  * JITDylib's default ResourceTracker.
  */
-LLVMErrorRef LLVMOrcObjectLayerAddObjectFile(LLVMOrcObjectLayerRef ObjLayer,
-                                             LLVMOrcJITDylibRef JD,
-                                             LLVMMemoryBufferRef ObjBuffer);
+LLVM_C_ABI LLVMErrorRef LLVMOrcObjectLayerAddObjectFile(
+    LLVMOrcObjectLayerRef ObjLayer, LLVMOrcJITDylibRef JD,
+    LLVMMemoryBufferRef ObjBuffer);
 
 /**
  * Add an object to an ObjectLayer using the given ResourceTracker.
@@ -1182,10 +1203,9 @@ LLVMErrorRef LLVMOrcObjectLayerAddObjectFile(LLVMOrcObjectLayerRef ObjLayer,
  * Resources associated with the given object will be tracked by
  * ResourceTracker RT.
  */
-LLVMErrorRef
-LLVMOrcObjectLayerAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer,
-                                      LLVMOrcResourceTrackerRef RT,
-                                      LLVMMemoryBufferRef ObjBuffer);
+LLVM_C_ABI LLVMErrorRef LLVMOrcObjectLayerAddObjectFileWithRT(
+    LLVMOrcObjectLayerRef ObjLayer, LLVMOrcResourceTrackerRef RT,
+    LLVMMemoryBufferRef ObjBuffer);
 
 /**
  * Emit an object buffer to an ObjectLayer.
@@ -1193,31 +1213,33 @@ LLVMOrcObjectLayerAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer,
  * Ownership of the responsibility object and object buffer pass to this
  * function. The client is not responsible for cleanup.
  */
-void LLVMOrcObjectLayerEmit(LLVMOrcObjectLayerRef ObjLayer,
-                            LLVMOrcMaterializationResponsibilityRef R,
-                            LLVMMemoryBufferRef ObjBuffer);
+LLVM_C_ABI void
+LLVMOrcObjectLayerEmit(LLVMOrcObjectLayerRef ObjLayer,
+                       LLVMOrcMaterializationResponsibilityRef R,
+                       LLVMMemoryBufferRef ObjBuffer);
 
 /**
  * Dispose of an ObjectLayer.
  */
-void LLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer);
+LLVM_C_ABI void LLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer);
 
-void LLVMOrcIRTransformLayerEmit(LLVMOrcIRTransformLayerRef IRTransformLayer,
-                                 LLVMOrcMaterializationResponsibilityRef MR,
-                                 LLVMOrcThreadSafeModuleRef TSM);
+LLVM_C_ABI void
+LLVMOrcIRTransformLayerEmit(LLVMOrcIRTransformLayerRef IRTransformLayer,
+                            LLVMOrcMaterializationResponsibilityRef MR,
+                            LLVMOrcThreadSafeModuleRef TSM);
 
 /**
  * Set the transform function of the provided transform layer, passing through a
  * pointer to user provided context.
  */
-void LLVMOrcIRTransformLayerSetTransform(
+LLVM_C_ABI void LLVMOrcIRTransformLayerSetTransform(
     LLVMOrcIRTransformLayerRef IRTransformLayer,
     LLVMOrcIRTransformLayerTransformFunction TransformFunction, void *Ctx);
 
 /**
  * Set the transform function on an LLVMOrcObjectTransformLayer.
  */
-void LLVMOrcObjectTransformLayerSetTransform(
+LLVM_C_ABI void LLVMOrcObjectTransformLayerSetTransform(
     LLVMOrcObjectTransformLayerRef ObjTransformLayer,
     LLVMOrcObjectTransformLayerTransformFunction TransformFunction, void *Ctx);
 
@@ -1227,15 +1249,16 @@ void LLVMOrcObjectTransformLayerSetTransform(
  * The resulting IndirectStubsManager is owned by the client
  * and must be disposed of by calling LLVMOrcDisposeDisposeIndirectStubsManager.
  */
-LLVMOrcIndirectStubsManagerRef
+LLVM_C_ABI LLVMOrcIndirectStubsManagerRef
 LLVMOrcCreateLocalIndirectStubsManager(const char *TargetTriple);
 
 /**
  * Dispose of an IndirectStubsManager.
  */
-void LLVMOrcDisposeIndirectStubsManager(LLVMOrcIndirectStubsManagerRef ISM);
+LLVM_C_ABI void
+LLVMOrcDisposeIndirectStubsManager(LLVMOrcIndirectStubsManagerRef ISM);
 
-LLVMErrorRef LLVMOrcCreateLocalLazyCallThroughManager(
+LLVM_C_ABI LLVMErrorRef LLVMOrcCreateLocalLazyCallThroughManager(
     const char *TargetTriple, LLVMOrcExecutionSessionRef ES,
     LLVMOrcJITTargetAddress ErrorHandlerAddr,
     LLVMOrcLazyCallThroughManagerRef *LCTM);
@@ -1243,8 +1266,8 @@ LLVMErrorRef LLVMOrcCreateLocalLazyCallThroughManager(
 /**
  * Dispose of an LazyCallThroughManager.
  */
-void LLVMOrcDisposeLazyCallThroughManager(
-    LLVMOrcLazyCallThroughManagerRef LCTM);
+LLVM_C_ABI void
+LLVMOrcDisposeLazyCallThroughManager(LLVMOrcLazyCallThroughManagerRef LCTM);
 
 /**
  * Create a DumpObjects instance.
@@ -1260,19 +1283,19 @@ void LLVMOrcDisposeLazyCallThroughManager(
  * <ident>.2.o, <ident>.3.o, and so on). IdentifierOverride should not contain
  * an extension, as a .o suffix will be added by DumpObjects.
  */
-LLVMOrcDumpObjectsRef LLVMOrcCreateDumpObjects(const char *DumpDir,
-                                               const char *IdentifierOverride);
+LLVM_C_ABI LLVMOrcDumpObjectsRef
+LLVMOrcCreateDumpObjects(const char *DumpDir, const char *IdentifierOverride);
 
 /**
  * Dispose of a DumpObjects instance.
  */
-void LLVMOrcDisposeDumpObjects(LLVMOrcDumpObjectsRef DumpObjects);
+LLVM_C_ABI void LLVMOrcDisposeDumpObjects(LLVMOrcDumpObjectsRef DumpObjects);
 
 /**
  * Dump the contents of the given MemoryBuffer.
  */
-LLVMErrorRef LLVMOrcDumpObjects_CallOperator(LLVMOrcDumpObjectsRef DumpObjects,
-                                             LLVMMemoryBufferRef *ObjBuffer);
+LLVM_C_ABI LLVMErrorRef LLVMOrcDumpObjects_CallOperator(
+    LLVMOrcDumpObjectsRef DumpObjects, LLVMMemoryBufferRef *ObjBuffer);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/OrcEE.h b/llvm/include/llvm-c/OrcEE.h
index d451187aaef59..fcec3a47186ac 100644
--- a/llvm/include/llvm-c/OrcEE.h
+++ b/llvm/include/llvm-c/OrcEE.h
@@ -29,6 +29,7 @@
 #include "llvm-c/Orc.h"
 #include "llvm-c/TargetMachine.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -46,7 +47,7 @@ typedef void (*LLVMMemoryManagerNotifyTerminatingCallback)(void *CtxCtx);
  * Create a RTDyldObjectLinkingLayer instance using the standard
  * SectionMemoryManager for memory management.
  */
-LLVMOrcObjectLayerRef
+LLVM_C_ABI LLVMOrcObjectLayerRef
 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(
     LLVMOrcExecutionSessionRef ES);
 
@@ -74,7 +75,7 @@ LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(
  * This scheme simply reuses the CreateContextCtx pointer as the one-and-only
  * allocation context.
  */
-LLVMOrcObjectLayerRef
+LLVM_C_ABI LLVMOrcObjectLayerRef
 LLVMOrcCreateRTDyldObjectLinkingLayerWithMCJITMemoryManagerLikeCallbacks(
     LLVMOrcExecutionSessionRef ES, void *CreateContextCtx,
     LLVMMemoryManagerCreateContextCallback CreateContext,
@@ -90,7 +91,7 @@ LLVMOrcCreateRTDyldObjectLinkingLayerWithMCJITMemoryManagerLikeCallbacks(
  * Note: Layer must be an RTDyldObjectLinkingLayer instance or
  * behavior is undefined.
  */
-void LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(
+LLVM_C_ABI void LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(
     LLVMOrcObjectLayerRef RTDyldObjLinkingLayer,
     LLVMJITEventListenerRef Listener);
 
diff --git a/llvm/include/llvm-c/Remarks.h b/llvm/include/llvm-c/Remarks.h
index ffe647a6554aa..b3468a55e5046 100644
--- a/llvm/include/llvm-c/Remarks.h
+++ b/llvm/include/llvm-c/Remarks.h
@@ -17,6 +17,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 #ifdef __cplusplus
 #include <cstddef>
 #else
@@ -61,14 +62,15 @@ typedef struct LLVMRemarkOpaqueString *LLVMRemarkStringRef;
  *
  * \since REMARKS_API_VERSION=0
  */
-extern const char *LLVMRemarkStringGetData(LLVMRemarkStringRef String);
+LLVM_C_ABI extern const char *
+LLVMRemarkStringGetData(LLVMRemarkStringRef String);
 
 /**
  * Returns the size of the string.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern uint32_t LLVMRemarkStringGetLen(LLVMRemarkStringRef String);
+LLVM_C_ABI extern uint32_t LLVMRemarkStringGetLen(LLVMRemarkStringRef String);
 
 /**
  * DebugLoc containing File, Line and Column.
@@ -82,7 +84,7 @@ typedef struct LLVMRemarkOpaqueDebugLoc *LLVMRemarkDebugLocRef;
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkStringRef
+LLVM_C_ABI extern LLVMRemarkStringRef
 LLVMRemarkDebugLocGetSourceFilePath(LLVMRemarkDebugLocRef DL);
 
 /**
@@ -90,14 +92,16 @@ LLVMRemarkDebugLocGetSourceFilePath(LLVMRemarkDebugLocRef DL);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern uint32_t LLVMRemarkDebugLocGetSourceLine(LLVMRemarkDebugLocRef DL);
+LLVM_C_ABI extern uint32_t
+LLVMRemarkDebugLocGetSourceLine(LLVMRemarkDebugLocRef DL);
 
 /**
  * Return the column in the source file for a debug location.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern uint32_t LLVMRemarkDebugLocGetSourceColumn(LLVMRemarkDebugLocRef DL);
+LLVM_C_ABI extern uint32_t
+LLVMRemarkDebugLocGetSourceColumn(LLVMRemarkDebugLocRef DL);
 
 /**
  * Element of the "Args" list. The key might give more information about what
@@ -114,14 +118,15 @@ typedef struct LLVMRemarkOpaqueArg *LLVMRemarkArgRef;
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkStringRef LLVMRemarkArgGetKey(LLVMRemarkArgRef Arg);
+LLVM_C_ABI extern LLVMRemarkStringRef LLVMRemarkArgGetKey(LLVMRemarkArgRef Arg);
 
 /**
  * Returns the value of an argument. This is a string that can contain newlines.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkStringRef LLVMRemarkArgGetValue(LLVMRemarkArgRef Arg);
+LLVM_C_ABI extern LLVMRemarkStringRef
+LLVMRemarkArgGetValue(LLVMRemarkArgRef Arg);
 
 /**
  * Returns the debug location that is attached to the value of this argument.
@@ -130,7 +135,8 @@ extern LLVMRemarkStringRef LLVMRemarkArgGetValue(LLVMRemarkArgRef Arg);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkDebugLocRef LLVMRemarkArgGetDebugLoc(LLVMRemarkArgRef Arg);
+LLVM_C_ABI extern LLVMRemarkDebugLocRef
+LLVMRemarkArgGetDebugLoc(LLVMRemarkArgRef Arg);
 
 /**
  * A remark emitted by the compiler.
@@ -144,7 +150,7 @@ typedef struct LLVMRemarkOpaqueEntry *LLVMRemarkEntryRef;
  *
  * \since REMARKS_API_VERSION=0
  */
-extern void LLVMRemarkEntryDispose(LLVMRemarkEntryRef Remark);
+LLVM_C_ABI extern void LLVMRemarkEntryDispose(LLVMRemarkEntryRef Remark);
 
 /**
  * The type of the remark. For example, it can allow users to only keep the
@@ -152,14 +158,15 @@ extern void LLVMRemarkEntryDispose(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern enum LLVMRemarkType LLVMRemarkEntryGetType(LLVMRemarkEntryRef Remark);
+LLVM_C_ABI extern enum LLVMRemarkType
+LLVMRemarkEntryGetType(LLVMRemarkEntryRef Remark);
 
 /**
  * Get the name of the pass that emitted this remark.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkStringRef
+LLVM_C_ABI extern LLVMRemarkStringRef
 LLVMRemarkEntryGetPassName(LLVMRemarkEntryRef Remark);
 
 /**
@@ -167,7 +174,7 @@ LLVMRemarkEntryGetPassName(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkStringRef
+LLVM_C_ABI extern LLVMRemarkStringRef
 LLVMRemarkEntryGetRemarkName(LLVMRemarkEntryRef Remark);
 
 /**
@@ -175,7 +182,7 @@ LLVMRemarkEntryGetRemarkName(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkStringRef
+LLVM_C_ABI extern LLVMRemarkStringRef
 LLVMRemarkEntryGetFunctionName(LLVMRemarkEntryRef Remark);
 
 /**
@@ -185,7 +192,7 @@ LLVMRemarkEntryGetFunctionName(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkDebugLocRef
+LLVM_C_ABI extern LLVMRemarkDebugLocRef
 LLVMRemarkEntryGetDebugLoc(LLVMRemarkEntryRef Remark);
 
 /**
@@ -195,14 +202,14 @@ LLVMRemarkEntryGetDebugLoc(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern uint64_t LLVMRemarkEntryGetHotness(LLVMRemarkEntryRef Remark);
+LLVM_C_ABI extern uint64_t LLVMRemarkEntryGetHotness(LLVMRemarkEntryRef Remark);
 
 /**
  * The number of arguments the remark holds.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern uint32_t LLVMRemarkEntryGetNumArgs(LLVMRemarkEntryRef Remark);
+LLVM_C_ABI extern uint32_t LLVMRemarkEntryGetNumArgs(LLVMRemarkEntryRef Remark);
 
 /**
  * Get a new iterator to iterate over a remark's argument.
@@ -213,7 +220,8 @@ extern uint32_t LLVMRemarkEntryGetNumArgs(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkArgRef LLVMRemarkEntryGetFirstArg(LLVMRemarkEntryRef Remark);
+LLVM_C_ABI extern LLVMRemarkArgRef
+LLVMRemarkEntryGetFirstArg(LLVMRemarkEntryRef Remark);
 
 /**
  * Get the next argument in \p Remark from the position of \p It.
@@ -224,8 +232,8 @@ extern LLVMRemarkArgRef LLVMRemarkEntryGetFirstArg(LLVMRemarkEntryRef Remark);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkArgRef LLVMRemarkEntryGetNextArg(LLVMRemarkArgRef It,
-                                                  LLVMRemarkEntryRef Remark);
+LLVM_C_ABI extern LLVMRemarkArgRef
+LLVMRemarkEntryGetNextArg(LLVMRemarkArgRef It, LLVMRemarkEntryRef Remark);
 
 typedef struct LLVMRemarkOpaqueParser *LLVMRemarkParserRef;
 
@@ -240,8 +248,8 @@ typedef struct LLVMRemarkOpaqueParser *LLVMRemarkParserRef;
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkParserRef LLVMRemarkParserCreateYAML(const void *Buf,
-                                                      uint64_t Size);
+LLVM_C_ABI extern LLVMRemarkParserRef
+LLVMRemarkParserCreateYAML(const void *Buf, uint64_t Size);
 
 /**
  * Creates a remark parser that can be used to parse the buffer located in \p
@@ -254,8 +262,8 @@ extern LLVMRemarkParserRef LLVMRemarkParserCreateYAML(const void *Buf,
  *
  * \since REMARKS_API_VERSION=1
  */
-extern LLVMRemarkParserRef LLVMRemarkParserCreateBitstream(const void *Buf,
-                                                           uint64_t Size);
+LLVM_C_ABI extern LLVMRemarkParserRef
+LLVMRemarkParserCreateBitstream(const void *Buf, uint64_t Size);
 
 /**
  * Returns the next remark in the file.
@@ -299,14 +307,15 @@ extern LLVMRemarkParserRef LLVMRemarkParserCreateBitstream(const void *Buf,
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMRemarkEntryRef LLVMRemarkParserGetNext(LLVMRemarkParserRef Parser);
+LLVM_C_ABI extern LLVMRemarkEntryRef
+LLVMRemarkParserGetNext(LLVMRemarkParserRef Parser);
 
 /**
  * Returns `1` if the parser encountered an error while parsing the buffer.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern LLVMBool LLVMRemarkParserHasError(LLVMRemarkParserRef Parser);
+LLVM_C_ABI extern LLVMBool LLVMRemarkParserHasError(LLVMRemarkParserRef Parser);
 
 /**
  * Returns a null-terminated string containing an error message.
@@ -319,14 +328,15 @@ extern LLVMBool LLVMRemarkParserHasError(LLVMRemarkParserRef Parser);
  *
  * \since REMARKS_API_VERSION=0
  */
-extern const char *LLVMRemarkParserGetErrorMessage(LLVMRemarkParserRef Parser);
+LLVM_C_ABI extern const char *
+LLVMRemarkParserGetErrorMessage(LLVMRemarkParserRef Parser);
 
 /**
  * Releases all the resources used by \p Parser.
  *
  * \since REMARKS_API_VERSION=0
  */
-extern void LLVMRemarkParserDispose(LLVMRemarkParserRef Parser);
+LLVM_C_ABI extern void LLVMRemarkParserDispose(LLVMRemarkParserRef Parser);
 
 /**
  * Returns the version of the remarks library.
diff --git a/llvm/include/llvm-c/Support.h b/llvm/include/llvm-c/Support.h
index 17657861b32b9..a3adfdf2834cc 100644
--- a/llvm/include/llvm-c/Support.h
+++ b/llvm/include/llvm-c/Support.h
@@ -17,6 +17,7 @@
 #include "llvm-c/DataTypes.h"
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -32,7 +33,7 @@ LLVM_C_EXTERN_C_BEGIN
  *
  * @see sys::DynamicLibrary::LoadLibraryPermanently()
   */
-LLVMBool LLVMLoadLibraryPermanently(const char* Filename);
+LLVM_C_ABI LLVMBool LLVMLoadLibraryPermanently(const char *Filename);
 
 /**
  * This function parses the given arguments using the LLVM command line parser.
@@ -42,8 +43,8 @@ LLVMBool LLVMLoadLibraryPermanently(const char* Filename);
  *
  * @see llvm::cl::ParseCommandLineOptions()
  */
-void LLVMParseCommandLineOptions(int argc, const char *const *argv,
-                                 const char *Overview);
+LLVM_C_ABI void LLVMParseCommandLineOptions(int argc, const char *const *argv,
+                                            const char *Overview);
 
 /**
  * This function will search through all previously loaded dynamic
@@ -52,7 +53,7 @@ void LLVMParseCommandLineOptions(int argc, const char *const *argv,
  *
  * @see sys::DynamicLibrary::SearchForAddressOfSymbol()
  */
-void *LLVMSearchForAddressOfSymbol(const char *symbolName);
+LLVM_C_ABI void *LLVMSearchForAddressOfSymbol(const char *symbolName);
 
 /**
  * This functions permanently adds the symbol \p symbolName with the
@@ -61,7 +62,7 @@ void *LLVMSearchForAddressOfSymbol(const char *symbolName);
  *
  * @see sys::DynamicLibrary::AddSymbol()
  */
-void LLVMAddSymbol(const char *symbolName, void *symbolValue);
+LLVM_C_ABI void LLVMAddSymbol(const char *symbolName, void *symbolValue);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Target.h b/llvm/include/llvm-c/Target.h
index aef06a5f2486d..03a3b563a6392 100644
--- a/llvm/include/llvm-c/Target.h
+++ b/llvm/include/llvm-c/Target.h
@@ -21,6 +21,7 @@
 
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/Config/llvm-config.h"
 
 LLVM_C_EXTERN_C_BEGIN
@@ -38,35 +39,36 @@ typedef struct LLVMOpaqueTargetData *LLVMTargetDataRef;
 typedef struct LLVMOpaqueTargetLibraryInfotData *LLVMTargetLibraryInfoRef;
 
 /* Declare all of the target-initialization functions that are available. */
-#define LLVM_TARGET(TargetName) \
-  void LLVMInitialize##TargetName##TargetInfo(void);
+#define LLVM_TARGET(TargetName)                                                \
+  LLVM_C_ABI void LLVMInitialize##TargetName##TargetInfo(void);
 #include "llvm/Config/Targets.def"
 #undef LLVM_TARGET  /* Explicit undef to make SWIG happier */
 
-#define LLVM_TARGET(TargetName) void LLVMInitialize##TargetName##Target(void);
+#define LLVM_TARGET(TargetName)                                                \
+  LLVM_C_ABI void LLVMInitialize##TargetName##Target(void);
 #include "llvm/Config/Targets.def"
 #undef LLVM_TARGET  /* Explicit undef to make SWIG happier */
 
-#define LLVM_TARGET(TargetName) \
-  void LLVMInitialize##TargetName##TargetMC(void);
+#define LLVM_TARGET(TargetName)                                                \
+  LLVM_C_ABI void LLVMInitialize##TargetName##TargetMC(void);
 #include "llvm/Config/Targets.def"
 #undef LLVM_TARGET  /* Explicit undef to make SWIG happier */
 
 /* Declare all of the available assembly printer initialization functions. */
-#define LLVM_ASM_PRINTER(TargetName) \
-  void LLVMInitialize##TargetName##AsmPrinter(void);
+#define LLVM_ASM_PRINTER(TargetName)                                           \
+  LLVM_C_ABI void LLVMInitialize##TargetName##AsmPrinter(void);
 #include "llvm/Config/AsmPrinters.def"
 #undef LLVM_ASM_PRINTER  /* Explicit undef to make SWIG happier */
 
 /* Declare all of the available assembly parser initialization functions. */
-#define LLVM_ASM_PARSER(TargetName) \
-  void LLVMInitialize##TargetName##AsmParser(void);
+#define LLVM_ASM_PARSER(TargetName)                                            \
+  LLVM_C_ABI void LLVMInitialize##TargetName##AsmParser(void);
 #include "llvm/Config/AsmParsers.def"
 #undef LLVM_ASM_PARSER  /* Explicit undef to make SWIG happier */
 
 /* Declare all of the available disassembler initialization functions. */
-#define LLVM_DISASSEMBLER(TargetName) \
-  void LLVMInitialize##TargetName##Disassembler(void);
+#define LLVM_DISASSEMBLER(TargetName)                                          \
+  LLVM_C_ABI void LLVMInitialize##TargetName##Disassembler(void);
 #include "llvm/Config/Disassemblers.def"
 #undef LLVM_DISASSEMBLER  /* Explicit undef to make SWIG happier */
 
@@ -183,105 +185,115 @@ static inline LLVMBool LLVMInitializeNativeDisassembler(void) {
  *
  * @see Module::getDataLayout()
  */
-LLVMTargetDataRef LLVMGetModuleDataLayout(LLVMModuleRef M);
+LLVM_C_ABI LLVMTargetDataRef LLVMGetModuleDataLayout(LLVMModuleRef M);
 
 /**
  * Set the data layout for a module.
  *
  * @see Module::setDataLayout()
  */
-void LLVMSetModuleDataLayout(LLVMModuleRef M, LLVMTargetDataRef DL);
+LLVM_C_ABI void LLVMSetModuleDataLayout(LLVMModuleRef M, LLVMTargetDataRef DL);
 
 /** Creates target data from a target layout string.
     See the constructor llvm::DataLayout::DataLayout. */
-LLVMTargetDataRef LLVMCreateTargetData(const char *StringRep);
+LLVM_C_ABI LLVMTargetDataRef LLVMCreateTargetData(const char *StringRep);
 
 /** Deallocates a TargetData.
     See the destructor llvm::DataLayout::~DataLayout. */
-void LLVMDisposeTargetData(LLVMTargetDataRef TD);
+LLVM_C_ABI void LLVMDisposeTargetData(LLVMTargetDataRef TD);
 
 /** Adds target library information to a pass manager. This does not take
     ownership of the target library info.
     See the method llvm::PassManagerBase::add. */
-void LLVMAddTargetLibraryInfo(LLVMTargetLibraryInfoRef TLI,
-                              LLVMPassManagerRef PM);
+LLVM_C_ABI void LLVMAddTargetLibraryInfo(LLVMTargetLibraryInfoRef TLI,
+                                         LLVMPassManagerRef PM);
 
 /** Converts target data to a target layout string. The string must be disposed
     with LLVMDisposeMessage.
     See the constructor llvm::DataLayout::DataLayout. */
-char *LLVMCopyStringRepOfTargetData(LLVMTargetDataRef TD);
+LLVM_C_ABI char *LLVMCopyStringRepOfTargetData(LLVMTargetDataRef TD);
 
 /** Returns the byte order of a target, either LLVMBigEndian or
     LLVMLittleEndian.
     See the method llvm::DataLayout::isLittleEndian. */
-enum LLVMByteOrdering LLVMByteOrder(LLVMTargetDataRef TD);
+LLVM_C_ABI enum LLVMByteOrdering LLVMByteOrder(LLVMTargetDataRef TD);
 
 /** Returns the pointer size in bytes for a target.
     See the method llvm::DataLayout::getPointerSize. */
-unsigned LLVMPointerSize(LLVMTargetDataRef TD);
+LLVM_C_ABI unsigned LLVMPointerSize(LLVMTargetDataRef TD);
 
 /** Returns the pointer size in bytes for a target for a specified
     address space.
     See the method llvm::DataLayout::getPointerSize. */
-unsigned LLVMPointerSizeForAS(LLVMTargetDataRef TD, unsigned AS);
+LLVM_C_ABI unsigned LLVMPointerSizeForAS(LLVMTargetDataRef TD, unsigned AS);
 
 /** Returns the integer type that is the same size as a pointer on a target.
     See the method llvm::DataLayout::getIntPtrType. */
-LLVMTypeRef LLVMIntPtrType(LLVMTargetDataRef TD);
+LLVM_C_ABI LLVMTypeRef LLVMIntPtrType(LLVMTargetDataRef TD);
 
 /** Returns the integer type that is the same size as a pointer on a target.
     This version allows the address space to be specified.
     See the method llvm::DataLayout::getIntPtrType. */
-LLVMTypeRef LLVMIntPtrTypeForAS(LLVMTargetDataRef TD, unsigned AS);
+LLVM_C_ABI LLVMTypeRef LLVMIntPtrTypeForAS(LLVMTargetDataRef TD, unsigned AS);
 
 /** Returns the integer type that is the same size as a pointer on a target.
     See the method llvm::DataLayout::getIntPtrType. */
-LLVMTypeRef LLVMIntPtrTypeInContext(LLVMContextRef C, LLVMTargetDataRef TD);
+LLVM_C_ABI LLVMTypeRef LLVMIntPtrTypeInContext(LLVMContextRef C,
+                                               LLVMTargetDataRef TD);
 
 /** Returns the integer type that is the same size as a pointer on a target.
     This version allows the address space to be specified.
     See the method llvm::DataLayout::getIntPtrType. */
-LLVMTypeRef LLVMIntPtrTypeForASInContext(LLVMContextRef C, LLVMTargetDataRef TD,
-                                         unsigned AS);
+LLVM_C_ABI LLVMTypeRef LLVMIntPtrTypeForASInContext(LLVMContextRef C,
+                                                    LLVMTargetDataRef TD,
+                                                    unsigned AS);
 
 /** Computes the size of a type in bits for a target.
     See the method llvm::DataLayout::getTypeSizeInBits. */
-unsigned long long LLVMSizeOfTypeInBits(LLVMTargetDataRef TD, LLVMTypeRef Ty);
+LLVM_C_ABI unsigned long long LLVMSizeOfTypeInBits(LLVMTargetDataRef TD,
+                                                   LLVMTypeRef Ty);
 
 /** Computes the storage size of a type in bytes for a target.
     See the method llvm::DataLayout::getTypeStoreSize. */
-unsigned long long LLVMStoreSizeOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty);
+LLVM_C_ABI unsigned long long LLVMStoreSizeOfType(LLVMTargetDataRef TD,
+                                                  LLVMTypeRef Ty);
 
 /** Computes the ABI size of a type in bytes for a target.
     See the method llvm::DataLayout::getTypeAllocSize. */
-unsigned long long LLVMABISizeOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty);
+LLVM_C_ABI unsigned long long LLVMABISizeOfType(LLVMTargetDataRef TD,
+                                                LLVMTypeRef Ty);
 
 /** Computes the ABI alignment of a type in bytes for a target.
     See the method llvm::DataLayout::getTypeABISize. */
-unsigned LLVMABIAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty);
+LLVM_C_ABI unsigned LLVMABIAlignmentOfType(LLVMTargetDataRef TD,
+                                           LLVMTypeRef Ty);
 
 /** Computes the call frame alignment of a type in bytes for a target.
     See the method llvm::DataLayout::getTypeABISize. */
-unsigned LLVMCallFrameAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty);
+LLVM_C_ABI unsigned LLVMCallFrameAlignmentOfType(LLVMTargetDataRef TD,
+                                                 LLVMTypeRef Ty);
 
 /** Computes the preferred alignment of a type in bytes for a target.
     See the method llvm::DataLayout::getTypeABISize. */
-unsigned LLVMPreferredAlignmentOfType(LLVMTargetDataRef TD, LLVMTypeRef Ty);
+LLVM_C_ABI unsigned LLVMPreferredAlignmentOfType(LLVMTargetDataRef TD,
+                                                 LLVMTypeRef Ty);
 
 /** Computes the preferred alignment of a global variable in bytes for a target.
     See the method llvm::DataLayout::getPreferredAlignment. */
-unsigned LLVMPreferredAlignmentOfGlobal(LLVMTargetDataRef TD,
-                                        LLVMValueRef GlobalVar);
+LLVM_C_ABI unsigned LLVMPreferredAlignmentOfGlobal(LLVMTargetDataRef TD,
+                                                   LLVMValueRef GlobalVar);
 
 /** Computes the structure element that contains the byte offset for a target.
     See the method llvm::StructLayout::getElementContainingOffset. */
-unsigned LLVMElementAtOffset(LLVMTargetDataRef TD, LLVMTypeRef StructTy,
-                             unsigned long long Offset);
+LLVM_C_ABI unsigned LLVMElementAtOffset(LLVMTargetDataRef TD,
+                                        LLVMTypeRef StructTy,
+                                        unsigned long long Offset);
 
 /** Computes the byte offset of the indexed struct element for a target.
     See the method llvm::StructLayout::getElementContainingOffset. */
-unsigned long long LLVMOffsetOfElement(LLVMTargetDataRef TD,
-                                       LLVMTypeRef StructTy, unsigned Element);
+LLVM_C_ABI unsigned long long LLVMOffsetOfElement(LLVMTargetDataRef TD,
+                                                  LLVMTypeRef StructTy,
+                                                  unsigned Element);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/TargetMachine.h b/llvm/include/llvm-c/TargetMachine.h
index cbe8913803e02..1c2d1ed9bfff8 100644
--- a/llvm/include/llvm-c/TargetMachine.h
+++ b/llvm/include/llvm-c/TargetMachine.h
@@ -22,6 +22,7 @@
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Target.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 LLVM_C_EXTERN_C_BEGIN
 
@@ -74,35 +75,36 @@ typedef enum {
 } LLVMGlobalISelAbortMode;
 
 /** Returns the first llvm::Target in the registered targets list. */
-LLVMTargetRef LLVMGetFirstTarget(void);
+LLVM_C_ABI LLVMTargetRef LLVMGetFirstTarget(void);
 /** Returns the next llvm::Target given a previous one (or null if there's none) */
-LLVMTargetRef LLVMGetNextTarget(LLVMTargetRef T);
+LLVM_C_ABI LLVMTargetRef LLVMGetNextTarget(LLVMTargetRef T);
 
 /*===-- Target ------------------------------------------------------------===*/
 /** Finds the target corresponding to the given name and stores it in \p T.
   Returns 0 on success. */
-LLVMTargetRef LLVMGetTargetFromName(const char *Name);
+LLVM_C_ABI LLVMTargetRef LLVMGetTargetFromName(const char *Name);
 
 /** Finds the target corresponding to the given triple and stores it in \p T.
   Returns 0 on success. Optionally returns any error in ErrorMessage.
   Use LLVMDisposeMessage to dispose the message. */
-LLVMBool LLVMGetTargetFromTriple(const char* Triple, LLVMTargetRef *T,
-                                 char **ErrorMessage);
+LLVM_C_ABI LLVMBool LLVMGetTargetFromTriple(const char *Triple,
+                                            LLVMTargetRef *T,
+                                            char **ErrorMessage);
 
 /** Returns the name of a target. See llvm::Target::getName */
-const char *LLVMGetTargetName(LLVMTargetRef T);
+LLVM_C_ABI const char *LLVMGetTargetName(LLVMTargetRef T);
 
 /** Returns the description  of a target. See llvm::Target::getDescription */
-const char *LLVMGetTargetDescription(LLVMTargetRef T);
+LLVM_C_ABI const char *LLVMGetTargetDescription(LLVMTargetRef T);
 
 /** Returns if the target has a JIT */
-LLVMBool LLVMTargetHasJIT(LLVMTargetRef T);
+LLVM_C_ABI LLVMBool LLVMTargetHasJIT(LLVMTargetRef T);
 
 /** Returns if the target has a TargetMachine associated */
-LLVMBool LLVMTargetHasTargetMachine(LLVMTargetRef T);
+LLVM_C_ABI LLVMBool LLVMTargetHasTargetMachine(LLVMTargetRef T);
 
 /** Returns if the target as an ASM backend (required for emitting output) */
-LLVMBool LLVMTargetHasAsmBackend(LLVMTargetRef T);
+LLVM_C_ABI LLVMBool LLVMTargetHasAsmBackend(LLVMTargetRef T);
 
 /*===-- Target Machine ----------------------------------------------------===*/
 /**
@@ -112,35 +114,42 @@ LLVMBool LLVMTargetHasAsmBackend(LLVMTargetRef T);
  * LLVMDisposeTargetMachineOptions() after the call to
  * LLVMCreateTargetMachineWithOptions().
  */
-LLVMTargetMachineOptionsRef LLVMCreateTargetMachineOptions(void);
+LLVM_C_ABI LLVMTargetMachineOptionsRef LLVMCreateTargetMachineOptions(void);
 
 /**
  * Dispose of an LLVMTargetMachineOptionsRef instance.
  */
-void LLVMDisposeTargetMachineOptions(LLVMTargetMachineOptionsRef Options);
+LLVM_C_ABI void
+LLVMDisposeTargetMachineOptions(LLVMTargetMachineOptionsRef Options);
 
-void LLVMTargetMachineOptionsSetCPU(LLVMTargetMachineOptionsRef Options,
-                                    const char *CPU);
+LLVM_C_ABI void
+LLVMTargetMachineOptionsSetCPU(LLVMTargetMachineOptionsRef Options,
+                               const char *CPU);
 
 /**
  * Set the list of features for the target machine.
  *
  * \param Features a comma-separated list of features.
  */
-void LLVMTargetMachineOptionsSetFeatures(LLVMTargetMachineOptionsRef Options,
-                                         const char *Features);
+LLVM_C_ABI void
+LLVMTargetMachineOptionsSetFeatures(LLVMTargetMachineOptionsRef Options,
+                                    const char *Features);
 
-void LLVMTargetMachineOptionsSetABI(LLVMTargetMachineOptionsRef Options,
-                                    const char *ABI);
+LLVM_C_ABI void
+LLVMTargetMachineOptionsSetABI(LLVMTargetMachineOptionsRef Options,
+                               const char *ABI);
 
-void LLVMTargetMachineOptionsSetCodeGenOptLevel(
-    LLVMTargetMachineOptionsRef Options, LLVMCodeGenOptLevel Level);
+LLVM_C_ABI void
+LLVMTargetMachineOptionsSetCodeGenOptLevel(LLVMTargetMachineOptionsRef Options,
+                                           LLVMCodeGenOptLevel Level);
 
-void LLVMTargetMachineOptionsSetRelocMode(LLVMTargetMachineOptionsRef Options,
-                                          LLVMRelocMode Reloc);
+LLVM_C_ABI void
+LLVMTargetMachineOptionsSetRelocMode(LLVMTargetMachineOptionsRef Options,
+                                     LLVMRelocMode Reloc);
 
-void LLVMTargetMachineOptionsSetCodeModel(LLVMTargetMachineOptionsRef Options,
-                                          LLVMCodeModel CodeModel);
+LLVM_C_ABI void
+LLVMTargetMachineOptionsSetCodeModel(LLVMTargetMachineOptionsRef Options,
+                                     LLVMCodeModel CodeModel);
 
 /**
  * Create a new llvm::TargetMachine.
@@ -150,90 +159,95 @@ void LLVMTargetMachineOptionsSetCodeModel(LLVMTargetMachineOptionsRef Options,
  * \param Options additional configuration (see
  *                LLVMCreateTargetMachineOptions()).
  */
-LLVMTargetMachineRef
-LLVMCreateTargetMachineWithOptions(LLVMTargetRef T, const char *Triple,
-                                   LLVMTargetMachineOptionsRef Options);
+LLVM_C_ABI LLVMTargetMachineRef LLVMCreateTargetMachineWithOptions(
+    LLVMTargetRef T, const char *Triple, LLVMTargetMachineOptionsRef Options);
 
 /** Creates a new llvm::TargetMachine. See llvm::Target::createTargetMachine */
-LLVMTargetMachineRef LLVMCreateTargetMachine(LLVMTargetRef T,
-  const char *Triple, const char *CPU, const char *Features,
-  LLVMCodeGenOptLevel Level, LLVMRelocMode Reloc, LLVMCodeModel CodeModel);
+LLVM_C_ABI LLVMTargetMachineRef LLVMCreateTargetMachine(
+    LLVMTargetRef T, const char *Triple, const char *CPU, const char *Features,
+    LLVMCodeGenOptLevel Level, LLVMRelocMode Reloc, LLVMCodeModel CodeModel);
 
 /** Dispose the LLVMTargetMachineRef instance generated by
   LLVMCreateTargetMachine. */
-void LLVMDisposeTargetMachine(LLVMTargetMachineRef T);
+LLVM_C_ABI void LLVMDisposeTargetMachine(LLVMTargetMachineRef T);
 
 /** Returns the Target used in a TargetMachine */
-LLVMTargetRef LLVMGetTargetMachineTarget(LLVMTargetMachineRef T);
+LLVM_C_ABI LLVMTargetRef LLVMGetTargetMachineTarget(LLVMTargetMachineRef T);
 
 /** Returns the triple used creating this target machine. See
   llvm::TargetMachine::getTriple. The result needs to be disposed with
   LLVMDisposeMessage. */
-char *LLVMGetTargetMachineTriple(LLVMTargetMachineRef T);
+LLVM_C_ABI char *LLVMGetTargetMachineTriple(LLVMTargetMachineRef T);
 
 /** Returns the cpu used creating this target machine. See
   llvm::TargetMachine::getCPU. The result needs to be disposed with
   LLVMDisposeMessage. */
-char *LLVMGetTargetMachineCPU(LLVMTargetMachineRef T);
+LLVM_C_ABI char *LLVMGetTargetMachineCPU(LLVMTargetMachineRef T);
 
 /** Returns the feature string used creating this target machine. See
   llvm::TargetMachine::getFeatureString. The result needs to be disposed with
   LLVMDisposeMessage. */
-char *LLVMGetTargetMachineFeatureString(LLVMTargetMachineRef T);
+LLVM_C_ABI char *LLVMGetTargetMachineFeatureString(LLVMTargetMachineRef T);
 
 /** Create a DataLayout based on the targetMachine. */
-LLVMTargetDataRef LLVMCreateTargetDataLayout(LLVMTargetMachineRef T);
+LLVM_C_ABI LLVMTargetDataRef LLVMCreateTargetDataLayout(LLVMTargetMachineRef T);
 
 /** Set the target machine's ASM verbosity. */
-void LLVMSetTargetMachineAsmVerbosity(LLVMTargetMachineRef T,
-                                      LLVMBool VerboseAsm);
+LLVM_C_ABI void LLVMSetTargetMachineAsmVerbosity(LLVMTargetMachineRef T,
+                                                 LLVMBool VerboseAsm);
 
 /** Enable fast-path instruction selection. */
-void LLVMSetTargetMachineFastISel(LLVMTargetMachineRef T, LLVMBool Enable);
+LLVM_C_ABI void LLVMSetTargetMachineFastISel(LLVMTargetMachineRef T,
+                                             LLVMBool Enable);
 
 /** Enable global instruction selection. */
-void LLVMSetTargetMachineGlobalISel(LLVMTargetMachineRef T, LLVMBool Enable);
+LLVM_C_ABI void LLVMSetTargetMachineGlobalISel(LLVMTargetMachineRef T,
+                                               LLVMBool Enable);
 
 /** Set abort behaviour when global instruction selection fails to lower/select
  * an instruction. */
-void LLVMSetTargetMachineGlobalISelAbort(LLVMTargetMachineRef T,
-                                         LLVMGlobalISelAbortMode Mode);
+LLVM_C_ABI void
+LLVMSetTargetMachineGlobalISelAbort(LLVMTargetMachineRef T,
+                                    LLVMGlobalISelAbortMode Mode);
 
 /** Enable the MachineOutliner pass. */
-void LLVMSetTargetMachineMachineOutliner(LLVMTargetMachineRef T,
-                                         LLVMBool Enable);
+LLVM_C_ABI void LLVMSetTargetMachineMachineOutliner(LLVMTargetMachineRef T,
+                                                    LLVMBool Enable);
 
 /** Emits an asm or object file for the given module to the filename. This
   wraps several c++ only classes (among them a file stream). Returns any
   error in ErrorMessage. Use LLVMDisposeMessage to dispose the message. */
-LLVMBool LLVMTargetMachineEmitToFile(LLVMTargetMachineRef T, LLVMModuleRef M,
-                                     const char *Filename,
-                                     LLVMCodeGenFileType codegen,
-                                     char **ErrorMessage);
+LLVM_C_ABI LLVMBool LLVMTargetMachineEmitToFile(LLVMTargetMachineRef T,
+                                                LLVMModuleRef M,
+                                                const char *Filename,
+                                                LLVMCodeGenFileType codegen,
+                                                char **ErrorMessage);
 
 /** Compile the LLVM IR stored in \p M and store the result in \p OutMemBuf. */
-LLVMBool LLVMTargetMachineEmitToMemoryBuffer(LLVMTargetMachineRef T, LLVMModuleRef M,
-  LLVMCodeGenFileType codegen, char** ErrorMessage, LLVMMemoryBufferRef *OutMemBuf);
+LLVM_C_ABI LLVMBool LLVMTargetMachineEmitToMemoryBuffer(
+    LLVMTargetMachineRef T, LLVMModuleRef M, LLVMCodeGenFileType codegen,
+    char **ErrorMessage, LLVMMemoryBufferRef *OutMemBuf);
 
 /*===-- Triple ------------------------------------------------------------===*/
 /** Get a triple for the host machine as a string. The result needs to be
   disposed with LLVMDisposeMessage. */
-char* LLVMGetDefaultTargetTriple(void);
+LLVM_C_ABI char *LLVMGetDefaultTargetTriple(void);
 
 /** Normalize a target triple. The result needs to be disposed with
   LLVMDisposeMessage. */
-char* LLVMNormalizeTargetTriple(const char* triple);
+LLVM_C_ABI char *LLVMNormalizeTargetTriple(const char *triple);
 
 /** Get the host CPU as a string. The result needs to be disposed with
   LLVMDisposeMessage. */
-char* LLVMGetHostCPUName(void);
+LLVM_C_ABI char *LLVMGetHostCPUName(void);
 
 /** Get the host CPU's features as a string. The result needs to be disposed
   with LLVMDisposeMessage. */
-char* LLVMGetHostCPUFeatures(void);
+LLVM_C_ABI char *LLVMGetHostCPUFeatures(void);
 
 /** Adds the target-specific analysis passes to the pass manager. */
-void LLVMAddAnalysisPasses(LLVMTargetMachineRef T, LLVMPassManagerRef PM);
+LLVM_C_ABI void LLVMAddAnalysisPasses(LLVMTargetMachineRef T,
+                                      LLVMPassManagerRef PM);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Transforms/PassBuilder.h b/llvm/include/llvm-c/Transforms/PassBuilder.h
index d297b57cadd07..7921f8f4ac286 100644
--- a/llvm/include/llvm-c/Transforms/PassBuilder.h
+++ b/llvm/include/llvm-c/Transforms/PassBuilder.h
@@ -17,6 +17,7 @@
 #include "llvm-c/Error.h"
 #include "llvm-c/TargetMachine.h"
 #include "llvm-c/Types.h"
+#include "llvm-c/Visibility.h"
 
 /**
  * @defgroup LLVMCCoreNewPM New Pass Manager
@@ -46,9 +47,9 @@ typedef struct LLVMOpaquePassBuilderOptions *LLVMPassBuilderOptionsRef;
  * pipelines may also be invoked using `default<O3>` and friends. See opt for
  * full reference of the Passes format.
  */
-LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
-                           LLVMTargetMachineRef TM,
-                           LLVMPassBuilderOptionsRef Options);
+LLVM_C_ABI LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
+                                      LLVMTargetMachineRef TM,
+                                      LLVMPassBuilderOptionsRef Options);
 
 /**
  * Construct and run a set of passes over a function.
@@ -56,9 +57,9 @@ LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
  * This function behaves the same as LLVMRunPasses, but operates on a single
  * function instead of an entire module.
  */
-LLVMErrorRef LLVMRunPassesOnFunction(LLVMValueRef F, const char *Passes,
-                                     LLVMTargetMachineRef TM,
-                                     LLVMPassBuilderOptionsRef Options);
+LLVM_C_ABI LLVMErrorRef LLVMRunPassesOnFunction(
+    LLVMValueRef F, const char *Passes, LLVMTargetMachineRef TM,
+    LLVMPassBuilderOptionsRef Options);
 
 /**
  * Create a new set of options for a PassBuilder
@@ -67,63 +68,75 @@ LLVMErrorRef LLVMRunPassesOnFunction(LLVMValueRef F, const char *Passes,
  * responsible for it. The client should call LLVMDisposePassBuilderOptions
  * to free the pass builder options.
  */
-LLVMPassBuilderOptionsRef LLVMCreatePassBuilderOptions(void);
+LLVM_C_ABI LLVMPassBuilderOptionsRef LLVMCreatePassBuilderOptions(void);
 
 /**
  * Toggle adding the VerifierPass for the PassBuilder, ensuring all functions
  * inside the module is valid.
  */
-void LLVMPassBuilderOptionsSetVerifyEach(LLVMPassBuilderOptionsRef Options,
-                                         LLVMBool VerifyEach);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetVerifyEach(LLVMPassBuilderOptionsRef Options,
+                                    LLVMBool VerifyEach);
 
 /**
  * Toggle debug logging when running the PassBuilder
  */
-void LLVMPassBuilderOptionsSetDebugLogging(LLVMPassBuilderOptionsRef Options,
-                                           LLVMBool DebugLogging);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetDebugLogging(LLVMPassBuilderOptionsRef Options,
+                                      LLVMBool DebugLogging);
 
 /**
  * Specify a custom alias analysis pipeline for the PassBuilder to be used
  * instead of the default one. The string argument is not copied; the caller
  * is responsible for ensuring it outlives the PassBuilderOptions instance.
  */
-void LLVMPassBuilderOptionsSetAAPipeline(LLVMPassBuilderOptionsRef Options,
-                                         const char *AAPipeline);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetAAPipeline(LLVMPassBuilderOptionsRef Options,
+                                    const char *AAPipeline);
 
-void LLVMPassBuilderOptionsSetLoopInterleaving(
-    LLVMPassBuilderOptionsRef Options, LLVMBool LoopInterleaving);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetLoopInterleaving(LLVMPassBuilderOptionsRef Options,
+                                          LLVMBool LoopInterleaving);
 
-void LLVMPassBuilderOptionsSetLoopVectorization(
-    LLVMPassBuilderOptionsRef Options, LLVMBool LoopVectorization);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetLoopVectorization(LLVMPassBuilderOptionsRef Options,
+                                           LLVMBool LoopVectorization);
 
-void LLVMPassBuilderOptionsSetSLPVectorization(
-    LLVMPassBuilderOptionsRef Options, LLVMBool SLPVectorization);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetSLPVectorization(LLVMPassBuilderOptionsRef Options,
+                                          LLVMBool SLPVectorization);
 
-void LLVMPassBuilderOptionsSetLoopUnrolling(LLVMPassBuilderOptionsRef Options,
-                                            LLVMBool LoopUnrolling);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetLoopUnrolling(LLVMPassBuilderOptionsRef Options,
+                                       LLVMBool LoopUnrolling);
 
-void LLVMPassBuilderOptionsSetForgetAllSCEVInLoopUnroll(
+LLVM_C_ABI void LLVMPassBuilderOptionsSetForgetAllSCEVInLoopUnroll(
     LLVMPassBuilderOptionsRef Options, LLVMBool ForgetAllSCEVInLoopUnroll);
 
-void LLVMPassBuilderOptionsSetLicmMssaOptCap(LLVMPassBuilderOptionsRef Options,
-                                             unsigned LicmMssaOptCap);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetLicmMssaOptCap(LLVMPassBuilderOptionsRef Options,
+                                        unsigned LicmMssaOptCap);
 
-void LLVMPassBuilderOptionsSetLicmMssaNoAccForPromotionCap(
+LLVM_C_ABI void LLVMPassBuilderOptionsSetLicmMssaNoAccForPromotionCap(
     LLVMPassBuilderOptionsRef Options, unsigned LicmMssaNoAccForPromotionCap);
 
-void LLVMPassBuilderOptionsSetCallGraphProfile(
-    LLVMPassBuilderOptionsRef Options, LLVMBool CallGraphProfile);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetCallGraphProfile(LLVMPassBuilderOptionsRef Options,
+                                          LLVMBool CallGraphProfile);
 
-void LLVMPassBuilderOptionsSetMergeFunctions(LLVMPassBuilderOptionsRef Options,
-                                             LLVMBool MergeFunctions);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetMergeFunctions(LLVMPassBuilderOptionsRef Options,
+                                        LLVMBool MergeFunctions);
 
-void LLVMPassBuilderOptionsSetInlinerThreshold(
-    LLVMPassBuilderOptionsRef Options, int Threshold);
+LLVM_C_ABI void
+LLVMPassBuilderOptionsSetInlinerThreshold(LLVMPassBuilderOptionsRef Options,
+                                          int Threshold);
 
 /**
  * Dispose of a heap-allocated PassBuilderOptions instance
  */
-void LLVMDisposePassBuilderOptions(LLVMPassBuilderOptionsRef Options);
+LLVM_C_ABI void
+LLVMDisposePassBuilderOptions(LLVMPassBuilderOptionsRef Options);
 
 /**
  * @}
diff --git a/llvm/include/llvm-c/Visibility.h b/llvm/include/llvm-c/Visibility.h
new file mode 100644
index 0000000000000..371b981b6d32c
--- /dev/null
+++ b/llvm/include/llvm-c/Visibility.h
@@ -0,0 +1,44 @@
+/*===-- llvm-c/Visibility.h - Visibility macros for llvm-c ------*- C++ -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This header defines visibility macros used for the LLVM C interface. These *|
+|* macros are used to annotate C functions that should be exported as part of *|
+|* a shared library or DLL.                                                   *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_C_VISIBILITY_H
+#define LLVM_C_VISIBILITY_H
+
+#include "llvm/Config/llvm-config.h"
+
+/// LLVM_C_ABI is the export/visibility macro used to mark symbols declared in
+/// llvm-c as exported when built as a shared library.
+
+#if !defined(LLVM_ABI_GENERATING_ANNOTATIONS)
+// TODO(https://github.com/llvm/llvm-project/issues/145406): eliminate need for
+// two preprocessor definitions to gate LLVM_ABI macro definitions.
+#if defined(LLVM_ENABLE_LLVM_C_EXPORT_ANNOTATIONS) &&                          \
+    !defined(LLVM_BUILD_STATIC)
+#if defined(_WIN32) && !defined(__MINGW32__)
+#if defined(LLVM_EXPORTS)
+#define LLVM_C_ABI __declspec(dllexport)
+#else
+#define LLVM_C_ABI __declspec(dllimport)
+#endif
+#elif defined(__has_attribute) && __has_attribute(visibility)
+#define LLVM_C_ABI __attribute__((visibility("default")))
+#endif
+#endif
+#if !defined(LLVM_C_ABI)
+#define LLVM_C_ABI
+#endif
+#endif
+
+#endif
diff --git a/llvm/include/llvm-c/blake3.h b/llvm/include/llvm-c/blake3.h
index 679477c3aa7f0..ad3eb08464496 100644
--- a/llvm/include/llvm-c/blake3.h
+++ b/llvm/include/llvm-c/blake3.h
@@ -17,6 +17,7 @@
 #ifndef LLVM_C_BLAKE3_H
 #define LLVM_C_BLAKE3_H
 
+#include "llvm-c/Visibility.h"
 #include <stddef.h>
 #include <stdint.h>
 
@@ -54,23 +55,24 @@ typedef struct {
   uint8_t cv_stack[(LLVM_BLAKE3_MAX_DEPTH + 1) * LLVM_BLAKE3_OUT_LEN];
 } llvm_blake3_hasher;
 
-const char *llvm_blake3_version(void);
-void llvm_blake3_hasher_init(llvm_blake3_hasher *self);
-void llvm_blake3_hasher_init_keyed(llvm_blake3_hasher *self,
-                                   const uint8_t key[LLVM_BLAKE3_KEY_LEN]);
-void llvm_blake3_hasher_init_derive_key(llvm_blake3_hasher *self,
-                                        const char *context);
-void llvm_blake3_hasher_init_derive_key_raw(llvm_blake3_hasher *self,
-                                            const void *context,
-                                            size_t context_len);
-void llvm_blake3_hasher_update(llvm_blake3_hasher *self, const void *input,
-                               size_t input_len);
-void llvm_blake3_hasher_finalize(const llvm_blake3_hasher *self, uint8_t *out,
-                                 size_t out_len);
-void llvm_blake3_hasher_finalize_seek(const llvm_blake3_hasher *self,
-                                      uint64_t seek, uint8_t *out,
-                                      size_t out_len);
-void llvm_blake3_hasher_reset(llvm_blake3_hasher *self);
+LLVM_C_ABI const char *llvm_blake3_version(void);
+LLVM_C_ABI void llvm_blake3_hasher_init(llvm_blake3_hasher *self);
+LLVM_C_ABI void
+llvm_blake3_hasher_init_keyed(llvm_blake3_hasher *self,
+                              const uint8_t key[LLVM_BLAKE3_KEY_LEN]);
+LLVM_C_ABI void llvm_blake3_hasher_init_derive_key(llvm_blake3_hasher *self,
+                                                   const char *context);
+LLVM_C_ABI void llvm_blake3_hasher_init_derive_key_raw(llvm_blake3_hasher *self,
+                                                       const void *context,
+                                                       size_t context_len);
+LLVM_C_ABI void llvm_blake3_hasher_update(llvm_blake3_hasher *self,
+                                          const void *input, size_t input_len);
+LLVM_C_ABI void llvm_blake3_hasher_finalize(const llvm_blake3_hasher *self,
+                                            uint8_t *out, size_t out_len);
+LLVM_C_ABI void llvm_blake3_hasher_finalize_seek(const llvm_blake3_hasher *self,
+                                                 uint64_t seek, uint8_t *out,
+                                                 size_t out_len);
+LLVM_C_ABI void llvm_blake3_hasher_reset(llvm_blake3_hasher *self);
 
 #ifdef __cplusplus
 }
diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index 3b05cfb326c82..ff8bdb8b4fec4 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -158,6 +158,20 @@ namespace llvm {
       return Data[Length-1];
     }
 
+    /// consume_front() - Returns the first element and drops it from ArrayRef.
+    const T &consume_front() {
+      const T &Ret = front();
+      *this = drop_front();
+      return Ret;
+    }
+
+    /// consume_back() - Returns the last element and drops it from ArrayRef.
+    const T &consume_back() {
+      const T &Ret = back();
+      *this = drop_back();
+      return Ret;
+    }
+
     // copy - Allocate copy in Allocator and return ArrayRef<T> to it.
     template <typename Allocator> MutableArrayRef<T> copy(Allocator &A) {
       T *Buff = A.template Allocate<T>(Length);
@@ -304,7 +318,8 @@ namespace llvm {
     /*implicit*/ MutableArrayRef() = default;
 
     /// Construct an empty MutableArrayRef from std::nullopt.
-    /*implicit*/ MutableArrayRef(std::nullopt_t) : ArrayRef<T>() {}
+    /*implicit*/ LLVM_DEPRECATED("Use {} or MutableArrayRef<T>() instead", "{}")
+    MutableArrayRef(std::nullopt_t) : ArrayRef<T>() {}
 
     /// Construct a MutableArrayRef from a single element.
     /*implicit*/ MutableArrayRef(T &OneElt) : ArrayRef<T>(OneElt) {}
@@ -327,11 +342,6 @@ namespace llvm {
                   void>>
     /*implicit*/ constexpr MutableArrayRef(const C &V) : ArrayRef<T>(V) {}
 
-    /// Construct a MutableArrayRef from a std::array
-    template <size_t N>
-    /*implicit*/ constexpr MutableArrayRef(std::array<T, N> &Arr)
-        : ArrayRef<T>(Arr) {}
-
     /// Construct a MutableArrayRef from a C array.
     template <size_t N>
     /*implicit*/ constexpr MutableArrayRef(T (&Arr)[N]) : ArrayRef<T>(Arr) {}
@@ -356,6 +366,20 @@ namespace llvm {
       return data()[this->size()-1];
     }
 
+    /// consume_front() - Returns the first element and drops it from ArrayRef.
+    T &consume_front() {
+      T &Ret = front();
+      *this = drop_front();
+      return Ret;
+    }
+
+    /// consume_back() - Returns the last element and drops it from ArrayRef.
+    T &consume_back() {
+      T &Ret = back();
+      *this = drop_back();
+      return Ret;
+    }
+
     /// slice(n, m) - Chop off the first N elements of the array, and keep M
     /// elements in the array.
     MutableArrayRef<T> slice(size_t N, size_t M) const {
@@ -541,6 +565,27 @@ namespace llvm {
     return !(LHS == RHS);
   }
 
+  template <typename T>
+  inline bool operator<(ArrayRef<T> LHS, ArrayRef<T> RHS) {
+    return std::lexicographical_compare(LHS.begin(), LHS.end(), RHS.begin(),
+                                        RHS.end());
+  }
+
+  template <typename T>
+  inline bool operator>(ArrayRef<T> LHS, ArrayRef<T> RHS) {
+    return RHS < LHS;
+  }
+
+  template <typename T>
+  inline bool operator<=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
+    return !(LHS > RHS);
+  }
+
+  template <typename T>
+  inline bool operator>=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
+    return !(LHS < RHS);
+  }
+
   /// @}
 
   template <typename T> hash_code hash_value(ArrayRef<T> S) {
diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h
index 0eaa77b6dd81c..70de4c239c1c7 100644
--- a/llvm/include/llvm/ADT/BitVector.h
+++ b/llvm/include/llvm/ADT/BitVector.h
@@ -795,9 +795,7 @@ class BitVector {
     set_unused_bits(false);
   }
 
-  void init_words(bool t) {
-    std::fill(Bits.begin(), Bits.end(), 0 - (BitWord)t);
-  }
+  void init_words(bool t) { llvm::fill(Bits, 0 - (BitWord)t); }
 
   template<bool AddBits, bool InvertMask>
   void applyMask(const uint32_t *Mask, unsigned MaskWords) {
diff --git a/llvm/include/llvm/ADT/Bitset.h b/llvm/include/llvm/ADT/Bitset.h
index 4ba5c63535b2d..55b96e47f2ed6 100644
--- a/llvm/include/llvm/ADT/Bitset.h
+++ b/llvm/include/llvm/ADT/Bitset.h
@@ -50,7 +50,7 @@ class Bitset {
   }
 
   Bitset &set() {
-    std::fill(std::begin(Bits), std::end(Bits), -BitWord(0));
+    llvm::fill(Bits, -BitWord(0));
     return *this;
   }
 
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index ca010f836de6f..07015b4a8dd80 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_ADT_DYNAMICAPINT_H
 #define LLVM_ADT_DYNAMICAPINT_H
 
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/SlowDynamicAPInt.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
@@ -116,6 +117,14 @@ class DynamicAPInt {
       : ValSmall(Val) {
     ValLarge.Val.BitWidth = 0;
   }
+  LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(const APInt &Val) {
+    if (Val.getBitWidth() <= 64) {
+      ValSmall = Val.getSExtValue();
+      ValLarge.Val.BitWidth = 0;
+    } else {
+      new (&ValLarge) detail::SlowDynamicAPInt(Val);
+    }
+  }
   LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt() : DynamicAPInt(0) {}
   LLVM_ATTRIBUTE_ALWAYS_INLINE ~DynamicAPInt() {
     if (LLVM_UNLIKELY(isLarge()))
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index eea06cfb99ba2..b23188cbdadeb 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1759,6 +1759,12 @@ bool none_of(R &&Range, UnaryPredicate P) {
   return std::none_of(adl_begin(Range), adl_end(Range), P);
 }
 
+/// Provide wrappers to std::fill which take ranges instead of having to pass
+/// begin/end explicitly.
+template <typename R, typename T> void fill(R &&Range, T &&Value) {
+  std::fill(adl_begin(Range), adl_end(Range), std::forward<T>(Value));
+}
+
 /// Provide wrappers to std::find which take ranges instead of having to pass
 /// begin/end explicitly.
 template <typename R, typename T> auto find(R &&Range, const T &Val) {
diff --git a/llvm/include/llvm/ADT/STLForwardCompat.h b/llvm/include/llvm/ADT/STLForwardCompat.h
index 75a0d4acf67f1..7bd2c8705f393 100644
--- a/llvm/include/llvm/ADT/STLForwardCompat.h
+++ b/llvm/include/llvm/ADT/STLForwardCompat.h
@@ -36,6 +36,19 @@ template <typename T>
 using remove_cvref_t // NOLINT(readability-identifier-naming)
     = typename llvm::remove_cvref<T>::type;
 
+// TODO: Remove this in favor of std::type_identity<T> once we switch to C++23.
+template <typename T>
+struct type_identity // NOLINT(readability-identifier-naming)
+{
+  using type = T;
+};
+
+// TODO: Remove this in favor of std::type_identity_t<T> once we switch to
+// C++23.
+template <typename T>
+using type_identity_t // NOLINT(readability-identifier-naming)
+    = typename llvm::type_identity<T>::type;
+
 //===----------------------------------------------------------------------===//
 //     Features from C++23
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index 2ce934321c456..892040ef4cab1 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -80,7 +80,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   explicit SmallPtrSetImplBase(const void **SmallStorage, unsigned SmallSize)
       : CurArray(SmallStorage), CurArraySize(SmallSize), NumNonEmpty(0),
         NumTombstones(0), IsSmall(true) {
-    assert(SmallSize && (SmallSize & (SmallSize-1)) == 0 &&
+    assert(llvm::has_single_bit(SmallSize) &&
            "Initial size must be a power of two!");
   }
 
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 66ea5ecbc4bfb..c9de5a4fa9213 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -72,9 +72,6 @@ namespace bfi_detail {
 
 struct IrreducibleGraph;
 
-// This is part of a workaround for a GCC 4.7 crash on lambdas.
-template <class BT> struct BlockEdgesAdder;
-
 /// Mass of a block.
 ///
 /// This class implements a sort of fixed-point fraction always between 0.0 and
@@ -843,9 +840,6 @@ void IrreducibleGraph::addEdges(const BlockNode &Node,
 ///         (Running this until fixed point would "solve" the geometric
 ///         series by simulation.)
 template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
-  // This is part of a workaround for a GCC 4.7 crash on lambdas.
-  friend struct bfi_detail::BlockEdgesAdder<BT>;
-
   using BlockT = typename bfi_detail::TypeMap<BT>::BlockT;
   using BlockKeyT = typename bfi_detail::TypeMap<BT>::BlockKeyT;
   using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT;
@@ -1632,29 +1626,6 @@ BlockFrequencyInfoImplBase::Scaled64 BlockFrequencyInfoImpl<BT>::discrepancy(
 }
 #endif
 
-/// \note This should be a lambda, but that crashes GCC 4.7.
-namespace bfi_detail {
-
-template <class BT> struct BlockEdgesAdder {
-  using BlockT = BT;
-  using LoopData = BlockFrequencyInfoImplBase::LoopData;
-  using Successor = GraphTraits<const BlockT *>;
-
-  const BlockFrequencyInfoImpl<BT> &BFI;
-
-  explicit BlockEdgesAdder(const BlockFrequencyInfoImpl<BT> &BFI)
-      : BFI(BFI) {}
-
-  void operator()(IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr,
-                  const LoopData *OuterLoop) {
-    const BlockT *BB = BFI.RPOT[Irr.Node.Index];
-    for (const auto *Succ : children<const BlockT *>(BB))
-      G.addEdge(Irr, BFI.getNode(Succ), OuterLoop);
-  }
-};
-
-} // end namespace bfi_detail
-
 template <class BT>
 void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass(
     LoopData *OuterLoop, std::list<LoopData>::iterator Insert) {
@@ -1665,9 +1636,12 @@ void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass(
 
   using namespace bfi_detail;
 
-  // Ideally, addBlockEdges() would be declared here as a lambda, but that
-  // crashes GCC 4.7.
-  BlockEdgesAdder<BT> addBlockEdges(*this);
+  auto addBlockEdges = [&](IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr,
+                           const LoopData *OuterLoop) {
+    const BlockT *BB = RPOT[Irr.Node.Index];
+    for (const auto *Succ : children<const BlockT *>(BB))
+      G.addEdge(Irr, getNode(Succ), OuterLoop);
+  };
   IrreducibleGraph G(*this, OuterLoop, addBlockEdges);
 
   for (auto &L : analyzeIrreducible(G, OuterLoop, Insert))
diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index 83a92c5eb39a3..cf57f2c95b55b 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -411,8 +411,8 @@ class BranchProbabilityInfo {
   std::optional<uint32_t> getInitialEstimatedBlockWeight(const BasicBlock *BB);
 
   // Computes estimated weights for all blocks in \p F.
-  void computeEestimateBlockWeight(const Function &F, DominatorTree *DT,
-                                   PostDominatorTree *PDT);
+  void estimateBlockWeights(const Function &F, DominatorTree *DT,
+                            PostDominatorTree *PDT);
 
   /// Based on computed weights by \p computeEstimatedBlockWeight set
   /// probabilities on branches.
diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
index babb6d9d6cf0c..7c6eca5bb8470 100644
--- a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
+++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
@@ -15,6 +15,7 @@
 #define LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Analysis/IR2Vec.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Compiler.h"
@@ -32,17 +33,19 @@ class FunctionPropertiesInfo {
   void updateAggregateStats(const Function &F, const LoopInfo &LI);
   void reIncludeBB(const BasicBlock &BB);
 
+  ir2vec::Embedding FunctionEmbedding = ir2vec::Embedding(0.0);
+  std::optional<ir2vec::Vocab> IR2VecVocab;
+
 public:
   LLVM_ABI static FunctionPropertiesInfo
   getFunctionPropertiesInfo(const Function &F, const DominatorTree &DT,
-                            const LoopInfo &LI);
+                            const LoopInfo &LI,
+                            const IR2VecVocabResult *VocabResult);
 
   LLVM_ABI static FunctionPropertiesInfo
   getFunctionPropertiesInfo(Function &F, FunctionAnalysisManager &FAM);
 
-  bool operator==(const FunctionPropertiesInfo &FPI) const {
-    return std::memcmp(this, &FPI, sizeof(FunctionPropertiesInfo)) == 0;
-  }
+  LLVM_ABI bool operator==(const FunctionPropertiesInfo &FPI) const;
 
   bool operator!=(const FunctionPropertiesInfo &FPI) const {
     return !(*this == FPI);
@@ -137,6 +140,19 @@ class FunctionPropertiesInfo {
   int64_t CallReturnsVectorPointerCount = 0;
   int64_t CallWithManyArgumentsCount = 0;
   int64_t CallWithPointerArgumentCount = 0;
+
+  const ir2vec::Embedding &getFunctionEmbedding() const {
+    return FunctionEmbedding;
+  }
+
+  const std::optional<ir2vec::Vocab> &getIR2VecVocab() const {
+    return IR2VecVocab;
+  }
+
+  // Helper intended to be useful for unittests
+  void setFunctionEmbeddingForTest(const ir2vec::Embedding &Embedding) {
+    FunctionEmbedding = Embedding;
+  }
 };
 
 // Analysis pass
@@ -193,6 +209,7 @@ class FunctionPropertiesUpdater {
   DominatorTree &getUpdatedDominatorTree(FunctionAnalysisManager &FAM) const;
 
   DenseSet<const BasicBlock *> Successors;
+  DenseSet<const BasicBlock *> CallUsers;
 
   // Edges we might potentially need to remove from the dominator tree.
   SmallVector<DominatorTree::UpdateType, 2> DomTreeUpdates;
diff --git a/llvm/include/llvm/Analysis/IR2Vec.h b/llvm/include/llvm/Analysis/IR2Vec.h
index 1eb4a9b8aaf9e..16fb07bdbbe9f 100644
--- a/llvm/include/llvm/Analysis/IR2Vec.h
+++ b/llvm/include/llvm/Analysis/IR2Vec.h
@@ -108,7 +108,11 @@ struct Embedding {
 
   /// Arithmetic operators
   LLVM_ABI Embedding &operator+=(const Embedding &RHS);
+  LLVM_ABI Embedding operator+(const Embedding &RHS) const;
   LLVM_ABI Embedding &operator-=(const Embedding &RHS);
+  LLVM_ABI Embedding operator-(const Embedding &RHS) const;
+  LLVM_ABI Embedding &operator*=(double Factor);
+  LLVM_ABI Embedding operator*(double Factor) const;
 
   /// Adds Src Embedding scaled by Factor with the called Embedding.
   /// Called_Embedding += Src * Factor
@@ -117,7 +121,9 @@ struct Embedding {
   /// Returns true if the embedding is approximately equal to the RHS embedding
   /// within the specified tolerance.
   LLVM_ABI bool approximatelyEquals(const Embedding &RHS,
-                                    double Tolerance = 1e-6) const;
+                                    double Tolerance = 1e-4) const;
+
+  LLVM_ABI void print(raw_ostream &OS) const;
 };
 
 using InstEmbeddingsMap = DenseMap<const Instruction *, Embedding>;
@@ -169,7 +175,7 @@ class Embedder {
   virtual ~Embedder() = default;
 
   /// Factory method to create an Embedder object.
-  LLVM_ABI static Expected<std::unique_ptr<Embedder>>
+  LLVM_ABI static std::unique_ptr<Embedder>
   create(IR2VecKind Mode, const Function &F, const Vocab &Vocabulary);
 
   /// Returns a map containing instructions and the corresponding embeddings for
@@ -236,6 +242,8 @@ class IR2VecVocabResult {
 class IR2VecVocabAnalysis : public AnalysisInfoMixin<IR2VecVocabAnalysis> {
   ir2vec::Vocab Vocabulary;
   Error readVocabulary();
+  Error parseVocabSection(StringRef Key, const json::Value &ParsedVocabValue,
+                          ir2vec::Vocab &TargetVocab, unsigned &Dim);
   void emitError(Error Err, LLVMContext &Ctx);
 
 public:
@@ -251,7 +259,6 @@ class IR2VecVocabAnalysis : public AnalysisInfoMixin<IR2VecVocabAnalysis> {
 /// functions.
 class IR2VecPrinterPass : public PassInfoMixin<IR2VecPrinterPass> {
   raw_ostream &OS;
-  void printVector(const ir2vec::Embedding &Vec) const;
 
 public:
   explicit IR2VecPrinterPass(raw_ostream &OS) : OS(OS) {}
@@ -259,6 +266,16 @@ class IR2VecPrinterPass : public PassInfoMixin<IR2VecPrinterPass> {
   static bool isRequired() { return true; }
 };
 
+/// This pass prints the embeddings in the vocabulary
+class IR2VecVocabPrinterPass : public PassInfoMixin<IR2VecVocabPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit IR2VecVocabPrinterPass(raw_ostream &OS) : OS(OS) {}
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  static bool isRequired() { return true; }
+};
+
 } // namespace llvm
 
 #endif // LLVM_ANALYSIS_IR2VEC_H
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 463249461483f..3b92cbff28de4 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -54,6 +54,9 @@ enum class RecurKind {
   FMulAdd,  ///< Sum of float products with llvm.fmuladd(a * b + sum).
   AnyOf,    ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
             ///< loop invariant, and both x and y are integer type.
+  FindFirstIVSMin, /// FindFirst reduction with select(icmp(),x,y) where one of
+                   ///< (x,y) is a decreasing loop induction, and both x and y
+                   ///< are integer type, producing a SMin reduction.
   FindLastIVSMax, ///< FindLast reduction with select(cmp(),x,y) where one of
                   ///< (x,y) is increasing loop induction, and both x and y
                   ///< are integer type, producing a SMax reduction.
@@ -165,13 +168,13 @@ class RecurrenceDescriptor {
   /// Returns a struct describing whether the instruction is either a
   ///   Select(ICmp(A, B), X, Y), or
   ///   Select(FCmp(A, B), X, Y)
-  /// where one of (X, Y) is an increasing loop induction variable, and the
-  /// other is a PHI value.
+  /// where one of (X, Y) is an increasing (FindLast) or decreasing (FindFirst)
+  /// loop induction variable, and the other is a PHI value.
   // TODO: Support non-monotonic variable. FindLast does not need be restricted
   // to increasing loop induction variables.
-  LLVM_ABI static InstDesc isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
-                                               Instruction *I,
-                                               ScalarEvolution &SE);
+  LLVM_ABI static InstDesc isFindIVPattern(RecurKind Kind, Loop *TheLoop,
+                                           PHINode *OrigPhi, Instruction *I,
+                                           ScalarEvolution &SE);
 
   /// Returns a struct describing if the instruction is a
   /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
@@ -259,6 +262,12 @@ class RecurrenceDescriptor {
     return Kind == RecurKind::AnyOf;
   }
 
+  /// Returns true if the recurrence kind is of the form
+  ///   select(cmp(),x,y) where one of (x,y) is decreasing loop induction.
+  static bool isFindFirstIVRecurrenceKind(RecurKind Kind) {
+    return Kind == RecurKind::FindFirstIVSMin;
+  }
+
   /// Returns true if the recurrence kind is of the form
   ///   select(cmp(),x,y) where one of (x,y) is increasing loop induction.
   static bool isFindLastIVRecurrenceKind(RecurKind Kind) {
@@ -269,22 +278,35 @@ class RecurrenceDescriptor {
   /// Returns true if recurrece kind is a signed redux kind.
   static bool isSignedRecurrenceKind(RecurKind Kind) {
     return Kind == RecurKind::SMax || Kind == RecurKind::SMin ||
+           Kind == RecurKind::FindFirstIVSMin ||
            Kind == RecurKind::FindLastIVSMax;
   }
 
+  /// Returns true if the recurrence kind is of the form
+  ///   select(cmp(),x,y) where one of (x,y) is an increasing or decreasing loop
+  ///   induction.
+  static bool isFindIVRecurrenceKind(RecurKind Kind) {
+    return isFindFirstIVRecurrenceKind(Kind) ||
+           isFindLastIVRecurrenceKind(Kind);
+  }
+
   /// Returns the type of the recurrence. This type can be narrower than the
   /// actual type of the Phi if the recurrence has been type-promoted.
   Type *getRecurrenceType() const { return RecurrenceType; }
 
-  /// Returns the sentinel value for FindLastIV recurrences to replace the start
-  /// value.
+  /// Returns the sentinel value for FindFirstIV & FindLastIV recurrences to
+  /// replace the start value.
   Value *getSentinelValue() const {
-    assert(isFindLastIVRecurrenceKind(Kind) && "Unexpected recurrence kind");
     Type *Ty = StartValue->getType();
     unsigned BW = Ty->getIntegerBitWidth();
+    if (isFindLastIVRecurrenceKind(Kind)) {
+      return ConstantInt::get(Ty, isSignedRecurrenceKind(Kind)
+                                      ? APInt::getSignedMinValue(BW)
+                                      : APInt::getMinValue(BW));
+    }
     return ConstantInt::get(Ty, isSignedRecurrenceKind(Kind)
-                                    ? APInt::getSignedMinValue(BW)
-                                    : APInt::getMinValue(BW));
+                                    ? APInt::getSignedMaxValue(BW)
+                                    : APInt::getMaxValue(BW));
   }
 
   /// Returns a reference to the instructions used for type-promoting the
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 9d15136e81d10..50ba3c13da70f 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -331,6 +331,10 @@ class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
   };
 
   Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
+
+private:
+  static bool initializeIR2VecVocabIfRequested(Module &M,
+                                               ModuleAnalysisManager &MAM);
 };
 
 /// Printer pass for the InlineAdvisorAnalysis results.
diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
index 961d5091bf9f3..25a35df3efe2c 100644
--- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
+++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
@@ -132,6 +132,11 @@ constexpr bool isHeuristicInlineCostFeature(InlineCostFeatureIndex Feature) {
     "not "                                                                     \
     "fully inlined by ElimAvailExtern)")
 
+// Not all features listed in FeatureIndex are used by the ML model.
+// Specifically, callee_embedding and caller_embedding are used only when the
+// usage of IR2Vec embeddings is explicitly enabled. Meaning, the size/number of
+// features is not static. So, we cannot determine number of features based on
+// the number of elements in this enum.
 // clang-format off
 enum class FeatureIndex : size_t {
 #define POPULATE_INDICES(DTYPE, SHAPE, NAME, COMMENT) NAME,
@@ -142,7 +147,11 @@ enum class FeatureIndex : size_t {
   INLINE_FEATURE_ITERATOR(POPULATE_INDICES)
 #undef POPULATE_INDICES
 
-  NumberOfFeatures
+// IR2Vec embeddings
+// Dimensions of embeddings are not known in the compile time (until vocab is 
+// read). Hence macros cannot be used here.
+  callee_embedding,
+  caller_embedding
 };
 // clang-format on
 
@@ -151,10 +160,7 @@ inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature) {
   return static_cast<FeatureIndex>(static_cast<size_t>(Feature));
 }
 
-constexpr size_t NumberOfFeatures =
-    static_cast<size_t>(FeatureIndex::NumberOfFeatures);
-
-LLVM_ABI extern const std::vector<TensorSpec> FeatureMap;
+LLVM_ABI extern std::vector<TensorSpec> FeatureMap;
 
 LLVM_ABI extern const char *const DecisionName;
 LLVM_ABI extern const TensorSpec InlineDecisionSpec;
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 1faf279ae2012..1415da14a3494 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -396,8 +396,6 @@ class MemoryDepChecker {
     uint64_t MaxStride;
     std::optional<uint64_t> CommonStride;
 
-    bool ShouldRetryWithRuntimeCheck;
-
     /// TypeByteSize is either the common store size of both accesses, or 0 when
     /// store sizes mismatch.
     uint64_t TypeByteSize;
@@ -407,11 +405,9 @@ class MemoryDepChecker {
 
     DepDistanceStrideAndSizeInfo(const SCEV *Dist, uint64_t MaxStride,
                                  std::optional<uint64_t> CommonStride,
-                                 bool ShouldRetryWithRuntimeCheck,
                                  uint64_t TypeByteSize, bool AIsWrite,
                                  bool BIsWrite)
         : Dist(Dist), MaxStride(MaxStride), CommonStride(CommonStride),
-          ShouldRetryWithRuntimeCheck(ShouldRetryWithRuntimeCheck),
           TypeByteSize(TypeByteSize), AIsWrite(AIsWrite), BIsWrite(BIsWrite) {}
   };
 
diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
index 580dd5e95d760..8262dd0846ede 100644
--- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
@@ -82,6 +82,7 @@ class MLInlineAdvisor : public InlineAdvisor {
   int64_t NodeCount = 0;
   int64_t EdgeCount = 0;
   int64_t EdgesOfLastSeenNodes = 0;
+  const bool UseIR2Vec;
 
   std::map<const LazyCallGraph::Node *, unsigned> FunctionLevels;
   const int32_t InitialIRSize = 0;
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
index 8e9d7e0b72142..09e3945f5a8ff 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
@@ -64,6 +64,9 @@ inline class_match<const SCEV> m_SCEV() { return class_match<const SCEV>(); }
 inline class_match<const SCEVConstant> m_SCEVConstant() {
   return class_match<const SCEVConstant>();
 }
+inline class_match<const SCEVVScale> m_SCEVVScale() {
+  return class_match<const SCEVVScale>();
+}
 
 template <typename Class> struct bind_ty {
   Class *&VR;
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index 58ffac0a6e207..c7bd1618e2f9b 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -136,7 +136,7 @@ class TargetLibraryInfoImpl {
     AMDLIBM      // AMD Math Vector library.
   };
 
-  LLVM_ABI TargetLibraryInfoImpl() = delete;
+  TargetLibraryInfoImpl() = delete;
   LLVM_ABI explicit TargetLibraryInfoImpl(const Triple &T);
 
   // Provide value semantics.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 90d92e0fcf55c..98b793aace7a3 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1325,9 +1325,21 @@ class TargetTransformInfo {
   /// \return The cost of a partial reduction, which is a reduction from a
   /// vector to another vector with fewer elements of larger size. They are
   /// represented by the llvm.experimental.partial.reduce.add intrinsic, which
-  /// takes an accumulator and a binary operation operand that itself is fed by
-  /// two extends. An example of an operation that uses a partial reduction is a
-  /// dot product, which reduces two vectors to another of 4 times fewer and 4
+  /// takes an accumulator of type \p AccumType and a second vector operand to
+  /// be accumulated, whose element count is specified by \p VF. The type of
+  /// reduction is specified by \p Opcode. The second operand passed to the
+  /// intrinsic could be the result of an extend, such as sext or zext. In
+  /// this case \p BinOp is nullopt, \p InputTypeA represents the type being
+  /// extended and \p OpAExtend the operation, i.e. sign- or zero-extend.
+  /// Also, \p InputTypeB should be nullptr and OpBExtend should be None.
+  /// Alternatively, the second operand could be the result of a binary
+  /// operation performed on two extends, i.e.
+  ///   mul(zext i8 %a -> i32, zext i8 %b -> i32).
+  /// In this case \p BinOp may specify the opcode of the binary operation,
+  /// \p InputTypeA and \p InputTypeB the types being extended, and
+  /// \p OpAExtend, \p OpBExtend the form of extensions. An example of an
+  /// operation that uses a partial reduction is a dot product, which reduces
+  /// two vectors in binary mul operation to another of 4 times fewer and 4
   /// times larger elements.
   LLVM_ABI InstructionCost getPartialReductionCost(
       unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
@@ -1690,12 +1702,13 @@ class TargetTransformInfo {
   /// unordered-atomic memory intrinsic.
   LLVM_ABI unsigned getAtomicMemIntrinsicMaxElementSize() const;
 
-  /// \returns A value which is the result of the given memory intrinsic.  New
-  /// instructions may be created to extract the result from the given intrinsic
-  /// memory operation.  Returns nullptr if the target cannot create a result
-  /// from the given intrinsic.
-  LLVM_ABI Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                                    Type *ExpectedType) const;
+  /// \returns A value which is the result of the given memory intrinsic. If \p
+  /// CanCreate is true, new instructions may be created to extract the result
+  /// from the given intrinsic memory operation. Returns nullptr if the target
+  /// cannot create a result from the given intrinsic.
+  LLVM_ABI Value *
+  getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType,
+                                    bool CanCreate = true) const;
 
   /// \returns The type to use in a loop expansion of a memcpy call.
   LLVM_ABI Type *getMemcpyLoopLoweringType(
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 12f87226c5f57..ddc8a5eaffa94 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -983,8 +983,9 @@ class TargetTransformInfoImplBase {
     return 0;
   }
 
-  virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                                   Type *ExpectedType) const {
+  virtual Value *
+  getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType,
+                                    bool CanCreate = true) const {
     return nullptr;
   }
 
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 15e23de3878dc..02990a3cb44f7 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -21,7 +21,6 @@
 #include "llvm/IR/FMF.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/Compiler.h"
 #include <cassert>
@@ -42,6 +41,7 @@ class LoopInfo;
 class MDNode;
 class StringRef;
 class TargetLibraryInfo;
+class IntrinsicInst;
 template <typename T> class ArrayRef;
 
 constexpr unsigned MaxAnalysisRecursionDepth = 6;
@@ -727,6 +727,9 @@ LLVM_ABI bool isGuaranteedToExecuteForEveryIteration(const Instruction *I,
 /// getGuaranteedNonPoisonOp.
 LLVM_ABI bool propagatesPoison(const Use &PoisonOp);
 
+/// Return whether this intrinsic propagates poison for all operands.
+LLVM_ABI bool intrinsicPropagatesPoison(Intrinsic::ID IID);
+
 /// Return true if the given instruction must trigger undefined behavior
 /// when I is executed with any operands which appear in KnownPoison holding
 /// a poison value at the point of execution.
diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 45170f550faf1..89abca02efefa 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -157,23 +157,33 @@ enum class FeatureFlags : uint64_t {
 static_assert((uint64_t)FeatureFlags::NextUnusedBit <= 1ull << 63,
               "Shader flag bits exceed enum size.");
 
-#define ROOT_ELEMENT_FLAG(Num, Val) Val = Num,
-enum class RootElementFlag : uint32_t {
+#define ROOT_SIGNATURE_FLAG(Num, Val) Val = Num,
+enum class RootFlags : uint32_t {
 #include "DXContainerConstants.def"
+
+  LLVM_MARK_AS_BITMASK_ENUM(SamplerHeapDirectlyIndexed)
 };
 
-#define ROOT_DESCRIPTOR_FLAG(Num, Val) Val = Num,
-enum class RootDescriptorFlag : uint32_t {
+LLVM_ABI ArrayRef<EnumEntry<RootFlags>> getRootFlags();
+
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag) Enum = Num,
+enum class RootDescriptorFlags : uint32_t {
 #include "DXContainerConstants.def"
+
+  LLVM_MARK_AS_BITMASK_ENUM(DataStatic)
 };
 
-#define DESCRIPTOR_RANGE_FLAG(Num, Val) Val = Num,
-enum class DescriptorRangeFlag : uint32_t {
+LLVM_ABI ArrayRef<EnumEntry<RootDescriptorFlags>> getRootDescriptorFlags();
+
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag) Enum = Num,
+enum class DescriptorRangeFlags : uint32_t {
 #include "DXContainerConstants.def"
 
-  LLVM_MARK_AS_BITMASK_ENUM(DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
+  LLVM_MARK_AS_BITMASK_ENUM(DescriptorsStaticKeepingBufferBoundsChecks)
 };
 
+LLVM_ABI ArrayRef<EnumEntry<DescriptorRangeFlags>> getDescriptorRangeFlags();
+
 #define ROOT_PARAMETER(Val, Enum) Enum = Val,
 enum class RootParameterType : uint32_t {
 #include "DXContainerConstants.def"
@@ -215,26 +225,34 @@ inline bool isValidShaderVisibility(uint32_t V) {
   return false;
 }
 
-#define STATIC_SAMPLER_FILTER(Val, Enum) Enum = Val,
-enum class StaticSamplerFilter : uint32_t {
+#define FILTER(Val, Enum) Enum = Val,
+enum class SamplerFilter : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+LLVM_ABI ArrayRef<EnumEntry<SamplerFilter>> getSamplerFilters();
+
 #define TEXTURE_ADDRESS_MODE(Val, Enum) Enum = Val,
 enum class TextureAddressMode : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-#define COMPARISON_FUNCTION(Val, Enum) Enum = Val,
-enum class SamplersComparisonFunction : uint32_t {
+LLVM_ABI ArrayRef<EnumEntry<TextureAddressMode>> getTextureAddressModes();
+
+#define COMPARISON_FUNC(Val, Enum) Enum = Val,
+enum class ComparisonFunc : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+LLVM_ABI ArrayRef<EnumEntry<ComparisonFunc>> getComparisonFuncs();
+
 #define STATIC_BORDER_COLOR(Val, Enum) Enum = Val,
-enum class SamplersBorderColor : uint32_t {
+enum class StaticBorderColor : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+LLVM_ABI ArrayRef<EnumEntry<StaticBorderColor>> getStaticBorderColors();
+
 LLVM_ABI PartType parsePartType(StringRef S);
 
 struct VertexPSVInfo {
diff --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index a281256a44ae0..8730820dd8b06 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -55,47 +55,52 @@ SHADER_FEATURE_FLAG(33, 39, NextUnusedBit, "Next reserved shader flag bit (not a
 #undef SHADER_FEATURE_FLAG
 #endif // SHADER_FEATURE_FLAG
 
-
-// ROOT_ELEMENT_FLAG(bit offset for the flag, name).
-#ifdef ROOT_ELEMENT_FLAG
-
-ROOT_ELEMENT_FLAG(0, NONE)
-ROOT_ELEMENT_FLAG(0x1, AllowInputAssemblerInputLayout)
-ROOT_ELEMENT_FLAG(0x2, DenyVertexShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x4, DenyHullShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x8, DenyDomainShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x10, DenyGeometryShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x20, DenyPixelShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x40, AllowStreamOutput)
-ROOT_ELEMENT_FLAG(0x80, LocalRootSignature)
-ROOT_ELEMENT_FLAG(0x100, DenyAmplificationShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x200, DenyMeshShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x400, CBVSRVUAVHeapDirectlyIndexed)
-ROOT_ELEMENT_FLAG(0x800, SamplerHeapDirectlyIndexed)
-#undef ROOT_ELEMENT_FLAG
-#endif // ROOT_ELEMENT_FLAG
-
+// Below is the definition of root signature enumerations and flags. The
+// definitions of all values here correspond to their description in the
+// d3d12.h header and are carried over from their values in DXC. For reference:
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/
+
+// ROOT_SIGNATURE_FLAG(flag value, name).
+#ifdef ROOT_SIGNATURE_FLAG
+
+ROOT_SIGNATURE_FLAG(0, None)
+ROOT_SIGNATURE_FLAG(0x1, AllowInputAssemblerInputLayout)
+ROOT_SIGNATURE_FLAG(0x2, DenyVertexShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x4, DenyHullShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x8, DenyDomainShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x10, DenyGeometryShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x20, DenyPixelShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x40, AllowStreamOutput)
+ROOT_SIGNATURE_FLAG(0x80, LocalRootSignature)
+ROOT_SIGNATURE_FLAG(0x100, DenyAmplificationShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x200, DenyMeshShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x400, CBVSRVUAVHeapDirectlyIndexed)
+ROOT_SIGNATURE_FLAG(0x800, SamplerHeapDirectlyIndexed)
+
+#undef ROOT_SIGNATURE_FLAG
+#endif // ROOT_SIGNATURE_FLAG
  
-// ROOT_DESCRIPTOR_FLAG(bit offset for the flag, name).
+// ROOT_DESCRIPTOR_FLAG(flag value, name, flag).
 #ifdef ROOT_DESCRIPTOR_FLAG
 
-ROOT_DESCRIPTOR_FLAG(0, NONE)
-ROOT_DESCRIPTOR_FLAG(0x2, DATA_VOLATILE)
-ROOT_DESCRIPTOR_FLAG(0x4, DATA_STATIC_WHILE_SET_AT_EXECUTE)
-ROOT_DESCRIPTOR_FLAG(0x8, DATA_STATIC)
+ROOT_DESCRIPTOR_FLAG(0, None, NONE)
+ROOT_DESCRIPTOR_FLAG(0x2, DataVolatile, DATA_VOLATILE)
+ROOT_DESCRIPTOR_FLAG(0x4, DataStaticWhileSetAtExecute, DATA_STATIC_WHILE_SET_AT_EXECUTE)
+ROOT_DESCRIPTOR_FLAG(0x8, DataStatic, DATA_STATIC)
+
 #undef ROOT_DESCRIPTOR_FLAG
 #endif // ROOT_DESCRIPTOR_FLAG
 
-
-// DESCRIPTOR_RANGE_FLAG(bit offset for the flag, name).
+// DESCRIPTOR_RANGE_FLAG(flag value, name, flag).
 #ifdef DESCRIPTOR_RANGE_FLAG
 
-DESCRIPTOR_RANGE_FLAG(0, NONE)
-DESCRIPTOR_RANGE_FLAG(0x1, DESCRIPTORS_VOLATILE)
-DESCRIPTOR_RANGE_FLAG(0x2, DATA_VOLATILE)
-DESCRIPTOR_RANGE_FLAG(0x4, DATA_STATIC_WHILE_SET_AT_EXECUTE)
-DESCRIPTOR_RANGE_FLAG(0x8, DATA_STATIC)
-DESCRIPTOR_RANGE_FLAG(0x10000, DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
+DESCRIPTOR_RANGE_FLAG(0, None, NONE)
+DESCRIPTOR_RANGE_FLAG(0x1, DescriptorsVolatile, DESCRIPTORS_VOLATILE)
+DESCRIPTOR_RANGE_FLAG(0x2, DataVolatile, DATA_VOLATILE)
+DESCRIPTOR_RANGE_FLAG(0x4, DataStaticWhileSetAtExecute, DATA_STATIC_WHILE_SET_AT_EXECUTE)
+DESCRIPTOR_RANGE_FLAG(0x8, DataStatic, DATA_STATIC)
+DESCRIPTOR_RANGE_FLAG(0x10000, DescriptorsStaticKeepingBufferBoundsChecks, DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
+
 #undef DESCRIPTOR_RANGE_FLAG
 #endif // DESCRIPTOR_RANGE_FLAG
 
@@ -119,6 +124,7 @@ ROOT_PARAMETER(4, UAV)
 #undef ROOT_PARAMETER
 #endif // ROOT_PARAMETER
 
+// SHADER_VISIBILITY(value, enum).
 #ifdef SHADER_VISIBILITY
 
 SHADER_VISIBILITY(0, All)
@@ -129,79 +135,89 @@ SHADER_VISIBILITY(4, Geometry)
 SHADER_VISIBILITY(5, Pixel)
 SHADER_VISIBILITY(6, Amplification)
 SHADER_VISIBILITY(7, Mesh)
+
 #undef SHADER_VISIBILITY
 #endif // SHADER_VISIBILITY
 
-#ifdef STATIC_SAMPLER_FILTER
-
-STATIC_SAMPLER_FILTER(0, MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x1, MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x4, MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x5, MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x10, MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x11, MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x14, MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x15, MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x55, ANISOTROPIC)
-STATIC_SAMPLER_FILTER(0x80, COMPARISON_MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x81, COMPARISON_MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x84, COMPARISON_MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x85, COMPARISON_MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x90, COMPARISON_MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x91, COMPARISON_MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x94, COMPARISON_MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x95, COMPARISON_MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0xd5, COMPARISON_ANISOTROPIC)
-STATIC_SAMPLER_FILTER(0x100, MINIMUM_MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x101, MINIMUM_MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x104, MINIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x105, MINIMUM_MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x110, MINIMUM_MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x111, MINIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x114, MINIMUM_MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x115, MINIMUM_MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x155, MINIMUM_ANISOTROPIC)
-STATIC_SAMPLER_FILTER(0x180, MAXIMUM_MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x181, MAXIMUM_MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x184, MAXIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x185, MAXIMUM_MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x190, MAXIMUM_MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x191, MAXIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x194, MAXIMUM_MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x195, MAXIMUM_MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x1d5, MAXIMUM_ANISOTROPIC)
-#undef STATIC_SAMPLER_FILTER
-#endif // STATIC_SAMPLER_FILTER
-
+// FILTER(value, enum).
+#ifdef FILTER
+
+FILTER(0, MinMagMipPoint)
+FILTER(0x1, MinMagPointMipLinear)
+FILTER(0x4, MinPointMagLinearMipPoint)
+FILTER(0x5, MinPointMagMipLinear)
+FILTER(0x10, MinLinearMagMipPoint)
+FILTER(0x11, MinLinearMagPointMipLinear)
+FILTER(0x14, MinMagLinearMipPoint)
+FILTER(0x15, MinMagMipLinear)
+FILTER(0x55, Anisotropic)
+FILTER(0x80, ComparisonMinMagMipPoint)
+FILTER(0x81, ComparisonMinMagPointMipLinear)
+FILTER(0x84, ComparisonMinPointMagLinearMipPoint)
+FILTER(0x85, ComparisonMinPointMagMipLinear)
+FILTER(0x90, ComparisonMinLinearMagMipPoint)
+FILTER(0x91, ComparisonMinLinearMagPointMipLinear)
+FILTER(0x94, ComparisonMinMagLinearMipPoint)
+FILTER(0x95, ComparisonMinMagMipLinear)
+FILTER(0xd5, ComparisonAnisotropic)
+FILTER(0x100, MinimumMinMagMipPoint)
+FILTER(0x101, MinimumMinMagPointMipLinear)
+FILTER(0x104, MinimumMinPointMagLinearMipPoint)
+FILTER(0x105, MinimumMinPointMagMipLinear)
+FILTER(0x110, MinimumMinLinearMagMipPoint)
+FILTER(0x111, MinimumMinLinearMagPointMipLinear)
+FILTER(0x114, MinimumMinMagLinearMipPoint)
+FILTER(0x115, MinimumMinMagMipLinear)
+FILTER(0x155, MinimumAnisotropic)
+FILTER(0x180, MaximumMinMagMipPoint)
+FILTER(0x181, MaximumMinMagPointMipLinear)
+FILTER(0x184, MaximumMinPointMagLinearMipPoint)
+FILTER(0x185, MaximumMinPointMagMipLinear)
+FILTER(0x190, MaximumMinLinearMagMipPoint)
+FILTER(0x191, MaximumMinLinearMagPointMipLinear)
+FILTER(0x194, MaximumMinMagLinearMipPoint)
+FILTER(0x195, MaximumMinMagMipLinear)
+FILTER(0x1d5, MaximumAnisotropic)
+
+#undef FILTER
+#endif // FILTER
+
+// TEXTURE_ADDRESS_MODE(value, enum).
 #ifdef TEXTURE_ADDRESS_MODE
 
-TEXTURE_ADDRESS_MODE( 1,  Wrap)
-TEXTURE_ADDRESS_MODE( 2,  Mirror)
-TEXTURE_ADDRESS_MODE( 3,  Clamp)
-TEXTURE_ADDRESS_MODE( 4,  Border)
-TEXTURE_ADDRESS_MODE( 5,  MirrorOnce)
+TEXTURE_ADDRESS_MODE(1,  Wrap)
+TEXTURE_ADDRESS_MODE(2,  Mirror)
+TEXTURE_ADDRESS_MODE(3,  Clamp)
+TEXTURE_ADDRESS_MODE(4,  Border)
+TEXTURE_ADDRESS_MODE(5,  MirrorOnce)
+
 #undef TEXTURE_ADDRESS_MODE
 #endif // TEXTURE_ADDRESS_MODE
 
-#ifdef COMPARISON_FUNCTION
+// COMPARISON_FUNC(value, enum).
+#ifdef COMPARISON_FUNC
+
+COMPARISON_FUNC(1,  Never)
+COMPARISON_FUNC(2,  Less)
+COMPARISON_FUNC(3,  Equal)
+COMPARISON_FUNC(4,  LessEqual)
+COMPARISON_FUNC(5,  Greater)
+COMPARISON_FUNC(6,  NotEqual)
+COMPARISON_FUNC(7,  GreaterEqual)
+COMPARISON_FUNC(8,  Always)
 
-COMPARISON_FUNCTION( 1,  Never)
-COMPARISON_FUNCTION( 2,  Less)
-COMPARISON_FUNCTION( 3,  Equal)
-COMPARISON_FUNCTION( 4,  LessEqual)
-COMPARISON_FUNCTION( 5,  Greater)
-COMPARISON_FUNCTION( 6,  NotEqual)
-COMPARISON_FUNCTION( 7,  GreaterEqual)
-COMPARISON_FUNCTION( 8,  Always)
-#undef COMPARISON_FUNCTION
-#endif // COMPARISON_FUNCTION
+#undef COMPARISON_FUNC
+#endif // COMPARISON_FUNC
 
+// STATIC_BORDER_COLOR(value, enum).
 #ifdef STATIC_BORDER_COLOR
-STATIC_BORDER_COLOR( 0,  TransparentBlack)
-STATIC_BORDER_COLOR( 1,  OpaqueBlack)
-STATIC_BORDER_COLOR( 2,  OpaqueWhite)
-STATIC_BORDER_COLOR( 3,  OpaqueBlackUint)
-STATIC_BORDER_COLOR( 4,  OpaqueWhiteUint)
+
+STATIC_BORDER_COLOR(0,  TransparentBlack)
+STATIC_BORDER_COLOR(1,  OpaqueBlack)
+STATIC_BORDER_COLOR(2,  OpaqueWhite)
+STATIC_BORDER_COLOR(3,  OpaqueBlackUint)
+STATIC_BORDER_COLOR(4,  OpaqueWhiteUint)
+
 #undef STATIC_BORDER_COLOR
 #endif // STATIC_BORDER_COLOR
 
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index 9cccea4fcaa58..48b33478d5045 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -624,6 +624,7 @@ HANDLE_DW_AT(0x3e09, LLVM_ptrauth_authenticates_null_values, 0, LLVM)
 HANDLE_DW_AT(0x3e0a, LLVM_ptrauth_authentication_mode, 0, LLVM)
 HANDLE_DW_AT(0x3e0b, LLVM_num_extra_inhabitants, 0, LLVM)
 HANDLE_DW_AT(0x3e0c, LLVM_stmt_sequence, 0, LLVM)
+HANDLE_DW_AT(0x3e0d, LLVM_coro_suspend_idx, 0, LLVM)
 
 // Apple extensions.
 
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index f5f236cf98064..ebb257ab33821 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1153,9 +1153,7 @@ enum : unsigned {
   SHT_LLVM_SYMPART = 0x6fff4c05,   // Symbol partition specification.
   SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition.
   SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition.
-  SHT_LLVM_BB_ADDR_MAP_V0 =
-      0x6fff4c08, // LLVM Basic Block Address Map (old version kept for
-                  // backward-compatibility).
+  // SHT_LLVM_BB_ADDR_MAP_V0 = 0x6fff4c08,  // Do not use.
   SHT_LLVM_CALL_GRAPH_PROFILE = 0x6fff4c09, // LLVM Call Graph Profile.
   SHT_LLVM_BB_ADDR_MAP = 0x6fff4c0a,        // LLVM Basic Block Address Map.
   SHT_LLVM_OFFLOADING = 0x6fff4c0b,         // LLVM device offloading data.
diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h
index 604104ff3cca3..cf90a43d0d7e8 100644
--- a/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -253,7 +253,7 @@ const unsigned WASM_SYMBOL_ABSOLUTE = 0x200;
 
 #define WASM_RELOC(name, value) name = value,
 
-enum : unsigned {
+enum WasmRelocType : unsigned {
 #include "WasmRelocs.def"
 };
 
@@ -452,6 +452,8 @@ struct WasmRelocation {
   uint32_t Index;  // Index into either symbol or type index space.
   uint64_t Offset; // Offset from the start of the section.
   int64_t Addend;  // A value to add to the symbol.
+
+  WasmRelocType getType() const { return static_cast<WasmRelocType>(Type); }
 };
 
 struct WasmInitFunc {
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index b362a88963f6c..dc78eb4164acf 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -798,6 +798,7 @@ enum AttributeKindCodes {
   ATTR_KIND_NO_DIVERGENCE_SOURCE = 100,
   ATTR_KIND_SANITIZE_TYPE = 101,
   ATTR_KIND_CAPTURES = 102,
+  ATTR_KIND_DEAD_ON_RETURN = 103,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index bf958e100f2ac..2b9be43eadb7a 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2243,6 +2243,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::log2:
       ISD = ISD::FLOG2;
       break;
+    case Intrinsic::ldexp:
+      ISD = ISD::FLDEXP;
+      break;
     case Intrinsic::fabs:
       ISD = ISD::FABS;
       break;
@@ -2297,6 +2300,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::roundeven:
       ISD = ISD::FROUNDEVEN;
       break;
+    case Intrinsic::lround:
+      ISD = ISD::LROUND;
+      break;
+    case Intrinsic::llround:
+      ISD = ISD::LLROUND;
+      break;
     case Intrinsic::pow:
       ISD = ISD::FPOW;
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index c15263e0b06f8..7d7b5364d6b68 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -693,18 +693,19 @@ class CombinerHelper {
   /// feeding a G_AND instruction \p MI.
   bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const;
 
-  /// Given an G_UDIV \p MI expressing a divide by constant, return an
-  /// expression that implements it by multiplying by a magic number.
+  /// Given an G_UDIV \p MI or G_UREM \p MI expressing a divide by constant,
+  /// return an expression that implements it by multiplying by a magic number.
   /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
-  MachineInstr *buildUDivUsingMul(MachineInstr &MI) const;
-  /// Combine G_UDIV by constant into a multiply by magic constant.
-  bool matchUDivByConst(MachineInstr &MI) const;
-  void applyUDivByConst(MachineInstr &MI) const;
+  MachineInstr *buildUDivorURemUsingMul(MachineInstr &MI) const;
+  /// Combine G_UDIV or G_UREM by constant into a multiply by magic constant.
+  bool matchUDivorURemByConst(MachineInstr &MI) const;
+  void applyUDivorURemByConst(MachineInstr &MI) const;
 
   /// Given an G_SDIV \p MI expressing a signed divide by constant, return an
   /// expression that implements it by multiplying by a magic number.
   /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
   MachineInstr *buildSDivUsingMul(MachineInstr &MI) const;
+  /// Combine G_SDIV by constant into a multiply by magic constant.
   bool matchSDivByConst(MachineInstr &MI) const;
   void applySDivByConst(MachineInstr &MI) const;
 
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 119786f045ed9..e07cdc678579e 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -141,6 +141,8 @@ template <> struct ScalarEnumerationTraits<MachineJumpTableInfo::JTEntryKind> {
                 MachineJumpTableInfo::EK_LabelDifference64);
     IO.enumCase(EntryKind, "inline", MachineJumpTableInfo::EK_Inline);
     IO.enumCase(EntryKind, "custom32", MachineJumpTableInfo::EK_Custom32);
+    IO.enumCase(EntryKind, "coff-imgrel32",
+                MachineJumpTableInfo::EK_CoffImgRel32);
   }
 };
 
diff --git a/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h b/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h
index 1dd2371bd4582..750d37c4eff96 100644
--- a/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineJumpTableInfo.h
@@ -85,7 +85,12 @@ class MachineJumpTableInfo {
 
     /// EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the
     /// TargetLowering::LowerCustomJumpTableEntry hook.
-    EK_Custom32
+    EK_Custom32,
+
+    // EK_CoffImgRel32 - In PE/COFF (Windows) images, each entry is a 32-bit
+    // unsigned offset that is added to the image base.
+    //       .word LBB123@IMGREL
+    EK_CoffImgRel32,
   };
 
 private:
@@ -100,6 +105,9 @@ class MachineJumpTableInfo {
   LLVM_ABI unsigned getEntrySize(const DataLayout &TD) const;
   /// getEntryAlignment - Return the alignment of each entry in the jump table.
   LLVM_ABI unsigned getEntryAlignment(const DataLayout &TD) const;
+  /// getEntryIsSigned - Return true if the load for the jump table index
+  /// should use signed extension, false if zero extension (unsigned)
+  LLVM_ABI bool getEntryIsSigned() const;
 
   /// createJumpTableIndex - Create a new jump table.
   ///
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 18df5d657064a..714285eb5d5d4 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -619,7 +619,7 @@ LLVM_ABI FunctionPass *createSelectOptimizePass();
 LLVM_ABI FunctionPass *createCallBrPass();
 
 /// Creates Windows Secure Hot Patch pass. \see WindowsSecureHotPatching.cpp
-ModulePass *createWindowsSecureHotPatchingPass();
+LLVM_ABI ModulePass *createWindowsSecureHotPatchingPass();
 
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
index 7481ed5b80b3f..f865b8c1583f8 100644
--- a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
@@ -56,6 +56,10 @@ LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
 /// UNKNOWN_LIBCALL if there is none.
 LLVM_ABI Libcall getPOWI(EVT RetVT);
 
+/// getPOW - Return the POW_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+LLVM_ABI Libcall getPOW(EVT RetVT);
+
 /// getLDEXP - Return the LDEXP_* value for the given types, or
 /// UNKNOWN_LIBCALL if there is none.
 LLVM_ABI Libcall getLDEXP(EVT RetVT);
@@ -64,6 +68,14 @@ LLVM_ABI Libcall getLDEXP(EVT RetVT);
 /// UNKNOWN_LIBCALL if there is none.
 LLVM_ABI Libcall getFREXP(EVT RetVT);
 
+/// Return the SIN_* value for the given types, or UNKNOWN_LIBCALL if there is
+/// none.
+LLVM_ABI Libcall getSIN(EVT RetVT);
+
+/// Return the COS_* value for the given types, or UNKNOWN_LIBCALL if there is
+/// none.
+LLVM_ABI Libcall getCOS(EVT RetVT);
+
 /// getSINCOS - Return the SINCOS_* value for the given types, or
 /// UNKNOWN_LIBCALL if there is none.
 LLVM_ABI Libcall getSINCOS(EVT RetVT);
diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index 83187b4a0241c..2967532226197 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -294,6 +294,34 @@ inline auto m_SpecificVT(EVT RefVT) {
 inline auto m_Glue() { return m_SpecificVT(MVT::Glue); }
 inline auto m_OtherVT() { return m_SpecificVT(MVT::Other); }
 
+/// Match a scalar ValueType.
+template <typename Pattern>
+inline auto m_SpecificScalarVT(EVT RefVT, const Pattern &P) {
+  return ValueType_match{[=](EVT VT) { return VT.getScalarType() == RefVT; },
+                         P};
+}
+inline auto m_SpecificScalarVT(EVT RefVT) {
+  return ValueType_match{[=](EVT VT) { return VT.getScalarType() == RefVT; },
+                         m_Value()};
+}
+
+/// Match a vector ValueType.
+template <typename Pattern>
+inline auto m_SpecificVectorElementVT(EVT RefVT, const Pattern &P) {
+  return ValueType_match{[=](EVT VT) {
+                           return VT.isVector() &&
+                                  VT.getVectorElementType() == RefVT;
+                         },
+                         P};
+}
+inline auto m_SpecificVectorElementVT(EVT RefVT) {
+  return ValueType_match{[=](EVT VT) {
+                           return VT.isVector() &&
+                                  VT.getVectorElementType() == RefVT;
+                         },
+                         m_Value()};
+}
+
 /// Match any integer ValueTypes.
 template <typename Pattern> inline auto m_IntegerVT(const Pattern &P) {
   return ValueType_match{[](EVT VT) { return VT.isInteger(); }, P};
@@ -627,6 +655,21 @@ struct MaxMin_match {
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
+    auto MatchMinMax = [&](SDValue L, SDValue R, SDValue TrueValue,
+                           SDValue FalseValue, ISD::CondCode CC) {
+      if ((TrueValue != L || FalseValue != R) &&
+          (TrueValue != R || FalseValue != L))
+        return false;
+
+      ISD::CondCode Cond =
+          TrueValue == L ? CC : getSetCCInverse(CC, L.getValueType());
+      if (!Pred_t::match(Cond))
+        return false;
+
+      return (LHS.match(Ctx, L) && RHS.match(Ctx, R)) ||
+             (Commutable && LHS.match(Ctx, R) && RHS.match(Ctx, L));
+    };
+
     if (sd_context_match(N, Ctx, m_Opc(ISD::SELECT)) ||
         sd_context_match(N, Ctx, m_Opc(ISD::VSELECT))) {
       EffectiveOperands<ExcludeChain> EO_SELECT(N, Ctx);
@@ -642,23 +685,22 @@ struct MaxMin_match {
         SDValue R = Cond->getOperand(EO_SETCC.FirstIndex + 1);
         auto *CondNode =
             cast<CondCodeSDNode>(Cond->getOperand(EO_SETCC.FirstIndex + 2));
-
-        if ((TrueValue != L || FalseValue != R) &&
-            (TrueValue != R || FalseValue != L)) {
-          return false;
-        }
-
-        ISD::CondCode Cond =
-            TrueValue == L ? CondNode->get()
-                           : getSetCCInverse(CondNode->get(), L.getValueType());
-        if (!Pred_t::match(Cond)) {
-          return false;
-        }
-        return (LHS.match(Ctx, L) && RHS.match(Ctx, R)) ||
-               (Commutable && LHS.match(Ctx, R) && RHS.match(Ctx, L));
+        return MatchMinMax(L, R, TrueValue, FalseValue, CondNode->get());
       }
     }
 
+    if (sd_context_match(N, Ctx, m_Opc(ISD::SELECT_CC))) {
+      EffectiveOperands<ExcludeChain> EO_SELECT(N, Ctx);
+      assert(EO_SELECT.Size == 5);
+      SDValue L = N->getOperand(EO_SELECT.FirstIndex);
+      SDValue R = N->getOperand(EO_SELECT.FirstIndex + 1);
+      SDValue TrueValue = N->getOperand(EO_SELECT.FirstIndex + 2);
+      SDValue FalseValue = N->getOperand(EO_SELECT.FirstIndex + 3);
+      auto *CondNode =
+          cast<CondCodeSDNode>(N->getOperand(EO_SELECT.FirstIndex + 4));
+      return MatchMinMax(L, R, TrueValue, FalseValue, CondNode->get());
+    }
+
     return false;
   }
 };
@@ -1072,19 +1114,46 @@ inline SpecificInt_match m_SpecificInt(uint64_t V) {
   return SpecificInt_match(APInt(64, V));
 }
 
-inline SpecificInt_match m_Zero() { return m_SpecificInt(0U); }
-inline SpecificInt_match m_One() { return m_SpecificInt(1U); }
+struct Zero_match {
+  bool AllowUndefs;
+
+  explicit Zero_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
+
+  template <typename MatchContext>
+  bool match(const MatchContext &, SDValue N) const {
+    return isZeroOrZeroSplat(N, AllowUndefs);
+  }
+};
+
+struct Ones_match {
+  bool AllowUndefs;
+
+  Ones_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
+
+  template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
+    return isOnesOrOnesSplat(N, AllowUndefs);
+  }
+};
 
 struct AllOnes_match {
+  bool AllowUndefs;
 
-  AllOnes_match() = default;
+  AllOnes_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
 
   template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
-    return isAllOnesOrAllOnesSplat(N);
+    return isAllOnesOrAllOnesSplat(N, AllowUndefs);
   }
 };
 
-inline AllOnes_match m_AllOnes() { return AllOnes_match(); }
+inline Ones_match m_One(bool AllowUndefs = false) {
+  return Ones_match(AllowUndefs);
+}
+inline Zero_match m_Zero(bool AllowUndefs = false) {
+  return Zero_match(AllowUndefs);
+}
+inline AllOnes_match m_AllOnes(bool AllowUndefs = false) {
+  return AllOnes_match(AllowUndefs);
+}
 
 /// Match true boolean value based on the information provided by
 /// TargetLowering.
@@ -1161,7 +1230,7 @@ inline CondCode_match m_SpecificCondCode(ISD::CondCode CC) {
 
 /// Match a negate as a sub(0, v)
 template <typename ValTy>
-inline BinaryOpc_match<SpecificInt_match, ValTy> m_Neg(const ValTy &V) {
+inline BinaryOpc_match<Zero_match, ValTy, false> m_Neg(const ValTy &V) {
   return m_Sub(m_Zero(), V);
 }
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index a98e46c587273..b856b4786573b 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -238,6 +238,8 @@ class SelectionDAG {
   LLVMContext *Context;
   CodeGenOptLevel OptLevel;
 
+  bool DivergentTarget = false;
+
   UniformityInfo *UA = nullptr;
   FunctionLoweringInfo * FLI = nullptr;
 
@@ -471,14 +473,16 @@ class SelectionDAG {
                      Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
                      UniformityInfo *UA, ProfileSummaryInfo *PSIin,
                      BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI,
-                     FunctionVarLocs const *FnVarLocs);
+                     FunctionVarLocs const *FnVarLocs, bool HasDivergency);
 
   void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
             MachineFunctionAnalysisManager &AM,
             const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA,
             ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin,
-            MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs) {
-    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs);
+            MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs,
+            bool HasDivergency) {
+    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs,
+         HasDivergency);
     MFAM = &AM;
   }
 
@@ -2030,6 +2034,11 @@ class SelectionDAG {
   LLVM_ABI SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
                                       ArrayRef<SDValue> Ops);
 
+  /// Fold BUILD_VECTOR of constants/undefs to the destination type
+  /// BUILD_VECTOR of constants/undefs elements.
+  LLVM_ABI SDValue FoldConstantBuildVector(BuildVectorSDNode *BV,
+                                           const SDLoc &DL, EVT DstEltVT);
+
   /// Constant fold a setcc to true or false.
   LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
                              const SDLoc &dl);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 5241a51dd8cd8..a6a3928230c3d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -57,9 +57,7 @@ class SelectionDAGISel {
   AssumptionCache *AC = nullptr;
   GCFunctionInfo *GFI = nullptr;
   SSPLayoutInfo *SP = nullptr;
-#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   TargetTransformInfo *TTI = nullptr;
-#endif
   CodeGenOptLevel OptLevel;
   const TargetInstrInfo *TII;
   const TargetLowering *TLI;
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index a3675eecfea3f..5d9937f832396 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1937,6 +1937,16 @@ LLVM_ABI bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
 /// Does not permit build vector implicit truncation.
 LLVM_ABI bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
 
+/// Return true if the value is a constant 1 integer or a splatted vector of a
+/// constant 1 integer (with no undefs).
+/// Does not permit build vector implicit truncation.
+LLVM_ABI bool isOnesOrOnesSplat(SDValue N, bool AllowUndefs = false);
+
+/// Return true if the value is a constant 0 integer or a splatted vector of a
+/// constant 0 integer (with no undefs).
+/// Does not permit build vector implicit truncation.
+LLVM_ABI bool isZeroOrZeroSplat(SDValue N, bool AllowUndefs = false);
+
 /// Return true if \p V is either a integer or FP constant.
 inline bool isIntOrFPConstant(SDValue V) {
   return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 84c53e1e45452..fee94cc167363 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1116,10 +1116,7 @@ class LLVM_ABI TargetLoweringBase {
     LegalizeTypeAction ValueTypeActions[MVT::VALUETYPE_SIZE];
 
   public:
-    ValueTypeActionImpl() {
-      std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
-                TypeLegal);
-    }
+    ValueTypeActionImpl() { llvm::fill(ValueTypeActions, TypeLegal); }
 
     LegalizeTypeAction getTypeAction(MVT VT) const {
       return ValueTypeActions[VT.SimpleTy];
@@ -3562,6 +3559,11 @@ class LLVM_ABI TargetLoweringBase {
     Libcalls.setLibcallImpl(Call, Impl);
   }
 
+  /// Get the libcall impl routine name for the specified libcall.
+  RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const {
+    return Libcalls.getLibcallImpl(Call);
+  }
+
   /// Get the libcall routine name for the specified libcall.
   const char *getLibcallName(RTLIB::Libcall Call) const {
     return Libcalls.getLibcallName(Call);
@@ -3584,11 +3586,18 @@ class LLVM_ABI TargetLoweringBase {
   }
 
   /// Set the CallingConv that should be used for the specified libcall.
-  void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
-    Libcalls.setLibcallCallingConv(Call, CC);
+  void setLibcallImplCallingConv(RTLIB::LibcallImpl Call, CallingConv::ID CC) {
+    Libcalls.setLibcallImplCallingConv(Call, CC);
+  }
+
+  /// Get the CallingConv that should be used for the specified libcall
+  /// implementation.
+  CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const {
+    return Libcalls.getLibcallImplCallingConv(Call);
   }
 
   /// Get the CallingConv that should be used for the specified libcall.
+  // FIXME: Remove this wrapper and directly use the used LibcallImpl
   CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
     return Libcalls.getLibcallCallingConv(Call);
   }
@@ -5086,9 +5095,6 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
     return nullptr;
   }
 
-  bool verifyReturnAddressArgumentIsConstant(SDValue Op,
-                                             SelectionDAG &DAG) const;
-
   //===--------------------------------------------------------------------===//
   // Inline Asm Support hooks
   //
diff --git a/llvm/include/llvm/Config/abi-breaking.h.cmake b/llvm/include/llvm/Config/abi-breaking.h.cmake
index 2d27e02b1d545..318bd015f80d2 100644
--- a/llvm/include/llvm/Config/abi-breaking.h.cmake
+++ b/llvm/include/llvm/Config/abi-breaking.h.cmake
@@ -12,6 +12,9 @@
 #ifndef LLVM_ABI_BREAKING_CHECKS_H
 #define LLVM_ABI_BREAKING_CHECKS_H
 
+// Compiler.h is required for LLVM_ABI definition.
+#include "llvm/Support/Compiler.h"
+
 /* Define to enable checks that alter the LLVM C++ ABI */
 #cmakedefine01 LLVM_ENABLE_ABI_BREAKING_CHECKS
 
@@ -43,12 +46,12 @@
 #endif
 namespace llvm {
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
-extern int EnableABIBreakingChecks;
+LLVM_ABI extern int EnableABIBreakingChecks;
 LLVM_HIDDEN_VISIBILITY
 __attribute__((weak)) int *VerifyEnableABIBreakingChecks =
     &EnableABIBreakingChecks;
 #else
-extern int DisableABIBreakingChecks;
+LLVM_ABI extern int DisableABIBreakingChecks;
 LLVM_HIDDEN_VISIBILITY
 __attribute__((weak)) int *VerifyDisableABIBreakingChecks =
     &DisableABIBreakingChecks;
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 0d8db0b6b52f1..a6832299e5b6b 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -113,6 +113,9 @@
 /* Define if exporting LLVM public interface for shared library */
 #cmakedefine LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS
 
+/* Define if exporting LLVM-C public interface for shared library */
+#cmakedefine LLVM_ENABLE_LLVM_C_EXPORT_ANNOTATIONS
+
 /* Define if building LLVM with LLVM_FORCE_USE_OLD_TOOLCHAIN_LIBS */
 #cmakedefine LLVM_FORCE_USE_OLD_TOOLCHAIN ${LLVM_FORCE_USE_OLD_TOOLCHAIN}
 
diff --git a/llvm/include/llvm/DWARFLinker/AddressesMap.h b/llvm/include/llvm/DWARFLinker/AddressesMap.h
index a232aafadc5ce..e2215c70dc34e 100644
--- a/llvm/include/llvm/DWARFLinker/AddressesMap.h
+++ b/llvm/include/llvm/DWARFLinker/AddressesMap.h
@@ -12,8 +12,8 @@
 #include "llvm/ADT/AddressRanges.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include <cstdint>
 
 namespace llvm {
diff --git a/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h
index b12d96812108e..5b9535380aebf 100644
--- a/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h
+++ b/llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h
@@ -20,7 +20,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Compiler.h"
 #include <map>
 
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h
index 17a18c4b029f4..99aeb0df076e6 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h
+++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h
@@ -16,7 +16,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Compiler.h"
 #include <map>
 namespace llvm {
diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
index f5f6fe69430cc..c2897891544af 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
@@ -240,8 +240,7 @@ struct BinaryAnnotationIterator
     if (Annotations.empty())
       return -1;
 
-    uint8_t FirstByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t FirstByte = Annotations.consume_front();
 
     if ((FirstByte & 0x80) == 0x00)
       return FirstByte;
@@ -249,8 +248,7 @@ struct BinaryAnnotationIterator
     if (Annotations.empty())
       return -1;
 
-    uint8_t SecondByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t SecondByte = Annotations.consume_front();
 
     if ((FirstByte & 0xC0) == 0x80)
       return ((FirstByte & 0x3F) << 8) | SecondByte;
@@ -258,14 +256,12 @@ struct BinaryAnnotationIterator
     if (Annotations.empty())
       return -1;
 
-    uint8_t ThirdByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t ThirdByte = Annotations.consume_front();
 
     if (Annotations.empty())
       return -1;
 
-    uint8_t FourthByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t FourthByte = Annotations.consume_front();
 
     if ((FirstByte & 0xE0) == 0xC0)
       return ((FirstByte & 0x1F) << 24) | (SecondByte << 16) |
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h
index 4723f00a4d241..7d4bf8d923097 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_DEBUGINFO_DWARF_DWARFCFIPRINTER_H
 #define LLVM_DEBUGINFO_DWARF_DWARFCFIPRINTER_H
 
-#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
deleted file mode 100644
index 1d89ac3578c10..0000000000000
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
+++ /dev/null
@@ -1,153 +0,0 @@
-//===- DWARFCFIProgram.h ----------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
-#define LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/iterator.h"
-#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/Error.h"
-#include "llvm/TargetParser/Triple.h"
-#include <map>
-#include <memory>
-#include <vector>
-
-namespace llvm {
-
-namespace dwarf {
-
-/// Represent a sequence of Call Frame Information instructions that, when read
-/// in order, construct a table mapping PC to frame state. This can also be
-/// referred to as "CFI rules" in DWARF literature to avoid confusion with
-/// computer programs in the broader sense, and in this context each instruction
-/// would be a rule to establish the mapping. Refer to pg. 172 in the DWARF5
-/// manual, "6.4.1 Structure of Call Frame Information".
-class CFIProgram {
-public:
-  static constexpr size_t MaxOperands = 3;
-  typedef SmallVector<uint64_t, MaxOperands> Operands;
-
-  /// An instruction consists of a DWARF CFI opcode and an optional sequence of
-  /// operands. If it refers to an expression, then this expression has its own
-  /// sequence of operations and operands handled separately by DWARFExpression.
-  struct Instruction {
-    Instruction(uint8_t Opcode) : Opcode(Opcode) {}
-
-    uint8_t Opcode;
-    Operands Ops;
-    // Associated DWARF expression in case this instruction refers to one
-    std::optional<DWARFExpression> Expression;
-
-    LLVM_ABI Expected<uint64_t> getOperandAsUnsigned(const CFIProgram &CFIP,
-                                                     uint32_t OperandIdx) const;
-
-    LLVM_ABI Expected<int64_t> getOperandAsSigned(const CFIProgram &CFIP,
-                                                  uint32_t OperandIdx) const;
-  };
-
-  using InstrList = std::vector<Instruction>;
-  using iterator = InstrList::iterator;
-  using const_iterator = InstrList::const_iterator;
-
-  iterator begin() { return Instructions.begin(); }
-  const_iterator begin() const { return Instructions.begin(); }
-  iterator end() { return Instructions.end(); }
-  const_iterator end() const { return Instructions.end(); }
-
-  unsigned size() const { return (unsigned)Instructions.size(); }
-  bool empty() const { return Instructions.empty(); }
-  uint64_t codeAlign() const { return CodeAlignmentFactor; }
-  int64_t dataAlign() const { return DataAlignmentFactor; }
-  Triple::ArchType triple() const { return Arch; }
-
-  CFIProgram(uint64_t CodeAlignmentFactor, int64_t DataAlignmentFactor,
-             Triple::ArchType Arch)
-      : CodeAlignmentFactor(CodeAlignmentFactor),
-        DataAlignmentFactor(DataAlignmentFactor), Arch(Arch) {}
-
-  /// Parse and store a sequence of CFI instructions from Data,
-  /// starting at *Offset and ending at EndOffset. *Offset is updated
-  /// to EndOffset upon successful parsing, or indicates the offset
-  /// where a problem occurred in case an error is returned.
-  LLVM_ABI Error parse(DWARFDataExtractor Data, uint64_t *Offset,
-                       uint64_t EndOffset);
-
-  void addInstruction(const Instruction &I) { Instructions.push_back(I); }
-
-  /// Get a DWARF CFI call frame string for the given DW_CFA opcode.
-  LLVM_ABI StringRef callFrameString(unsigned Opcode) const;
-
-  /// Types of operands to CFI instructions
-  /// In DWARF, this type is implicitly tied to a CFI instruction opcode and
-  /// thus this type doesn't need to be explicitly written to the file (this is
-  /// not a DWARF encoding). The relationship of instrs to operand types can
-  /// be obtained from getOperandTypes() and is only used to simplify
-  /// instruction printing and error messages.
-  enum OperandType {
-    OT_Unset,
-    OT_None,
-    OT_Address,
-    OT_Offset,
-    OT_FactoredCodeOffset,
-    OT_SignedFactDataOffset,
-    OT_UnsignedFactDataOffset,
-    OT_Register,
-    OT_AddressSpace,
-    OT_Expression
-  };
-
-  /// Get the OperandType as a "const char *".
-  LLVM_ABI static const char *operandTypeString(OperandType OT);
-
-  /// Retrieve the array describing the types of operands according to the enum
-  /// above. This is indexed by opcode.
-  LLVM_ABI static ArrayRef<OperandType[MaxOperands]> getOperandTypes();
-
-private:
-  std::vector<Instruction> Instructions;
-  const uint64_t CodeAlignmentFactor;
-  const int64_t DataAlignmentFactor;
-  Triple::ArchType Arch;
-
-  /// Convenience method to add a new instruction with the given opcode.
-  void addInstruction(uint8_t Opcode) {
-    Instructions.push_back(Instruction(Opcode));
-  }
-
-  /// Add a new single-operand instruction.
-  void addInstruction(uint8_t Opcode, uint64_t Operand1) {
-    Instructions.push_back(Instruction(Opcode));
-    Instructions.back().Ops.push_back(Operand1);
-  }
-
-  /// Add a new instruction that has two operands.
-  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2) {
-    Instructions.push_back(Instruction(Opcode));
-    Instructions.back().Ops.push_back(Operand1);
-    Instructions.back().Ops.push_back(Operand2);
-  }
-
-  /// Add a new instruction that has three operands.
-  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2,
-                      uint64_t Operand3) {
-    Instructions.push_back(Instruction(Opcode));
-    Instructions.back().Ops.push_back(Operand1);
-    Instructions.back().Ops.push_back(Operand2);
-    Instructions.back().Ops.push_back(Operand3);
-  }
-};
-
-} // end namespace dwarf
-
-} // end namespace llvm
-
-#endif // LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
index 005ccd208cdda..d2f05f19b376c 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
@@ -10,10 +10,10 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTOR_H
 
 #include "llvm/BinaryFormat/Dwarf.h"
-#include "llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h"
 #include "llvm/DebugInfo/DWARF/DWARFObject.h"
 #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h"
 #include "llvm/DebugInfo/DWARF/DWARFSection.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
@@ -42,8 +42,8 @@ class DWARFDataExtractor : public DWARFDataExtractorBase<DWARFDataExtractor> {
 
   /// Extracts a value and applies a relocation to the result if
   /// one exists for the given offset.
-  LLVM_ABI uint64_t getRelocatedValueImpl(uint32_t Size, uint64_t *Off,
-                                          uint64_t *SecNdx, Error *Err) const {
+  uint64_t getRelocatedValueImpl(uint32_t Size, uint64_t *Off, uint64_t *SecNdx,
+                                 Error *Err) const {
     if (SecNdx)
       *SecNdx = object::SectionedAddress::UndefSection;
     if (!Section)
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
index 3b367009a379d..65189916b41f1 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
@@ -9,15 +9,14 @@
 #ifndef LLVM_DEBUGINFO_DWARF_DWARFDEBUGFRAME_H
 #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGFRAME_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/iterator.h"
-#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/TargetParser/Triple.h"
-#include <map>
 #include <memory>
 #include <vector>
 
@@ -30,354 +29,6 @@ struct DIDumpOptions;
 
 namespace dwarf {
 
-constexpr uint32_t InvalidRegisterNumber = UINT32_MAX;
-
-/// A class that represents a location for the Call Frame Address (CFA) or a
-/// register. This is decoded from the DWARF Call Frame Information
-/// instructions and put into an UnwindRow.
-class UnwindLocation {
-public:
-  enum Location {
-    /// Not specified.
-    Unspecified,
-    /// Register is not available and can't be recovered.
-    Undefined,
-    /// Register value is in the register, nothing needs to be done to unwind
-    /// it:
-    ///   reg = reg
-    Same,
-    /// Register is in or at the CFA plus an offset:
-    ///   reg = CFA + offset
-    ///   reg = defef(CFA + offset)
-    CFAPlusOffset,
-    /// Register or CFA is in or at a register plus offset, optionally in
-    /// an address space:
-    ///   reg = reg + offset [in addrspace]
-    ///   reg = deref(reg + offset [in addrspace])
-    RegPlusOffset,
-    /// Register or CFA value is in or at a value found by evaluating a DWARF
-    /// expression:
-    ///   reg = eval(dwarf_expr)
-    ///   reg = deref(eval(dwarf_expr))
-    DWARFExpr,
-    /// Value is a constant value contained in "Offset":
-    ///   reg = Offset
-    Constant,
-  };
-
-private:
-  Location Kind;   /// The type of the location that describes how to unwind it.
-  uint32_t RegNum; /// The register number for Kind == RegPlusOffset.
-  int32_t Offset;  /// The offset for Kind == CFAPlusOffset or RegPlusOffset.
-  std::optional<uint32_t> AddrSpace;   /// The address space for Kind ==
-                                       /// RegPlusOffset for CFA.
-  std::optional<DWARFExpression> Expr; /// The DWARF expression for Kind ==
-                                       /// DWARFExpression.
-  bool Dereference; /// If true, the resulting location must be dereferenced
-                    /// after the location value is computed.
-
-  // Constructors are private to force people to use the create static
-  // functions.
-  UnwindLocation(Location K)
-      : Kind(K), RegNum(InvalidRegisterNumber), Offset(0),
-        AddrSpace(std::nullopt), Dereference(false) {}
-
-  UnwindLocation(Location K, uint32_t Reg, int32_t Off,
-                 std::optional<uint32_t> AS, bool Deref)
-      : Kind(K), RegNum(Reg), Offset(Off), AddrSpace(AS), Dereference(Deref) {}
-
-  UnwindLocation(DWARFExpression E, bool Deref)
-      : Kind(DWARFExpr), RegNum(InvalidRegisterNumber), Offset(0), Expr(E),
-        Dereference(Deref) {}
-
-public:
-  /// Create a location whose rule is set to Unspecified. This means the
-  /// register value might be in the same register but it wasn't specified in
-  /// the unwind opcodes.
-  LLVM_ABI static UnwindLocation createUnspecified();
-  /// Create a location where the value is undefined and not available. This can
-  /// happen when a register is volatile and can't be recovered.
-  LLVM_ABI static UnwindLocation createUndefined();
-  /// Create a location where the value is known to be in the register itself.
-  LLVM_ABI static UnwindLocation createSame();
-  /// Create a location that is in (Deref == false) or at (Deref == true) the
-  /// CFA plus an offset. Most registers that are spilled onto the stack use
-  /// this rule. The rule for the register will use this rule and specify a
-  /// unique offset from the CFA with \a Deref set to true. This value will be
-  /// relative to a CFA value which is typically defined using the register
-  /// plus offset location. \see createRegisterPlusOffset(...) for more
-  /// information.
-  LLVM_ABI static UnwindLocation createIsCFAPlusOffset(int32_t Off);
-  LLVM_ABI static UnwindLocation createAtCFAPlusOffset(int32_t Off);
-  /// Create a location where the saved value is in (Deref == false) or at
-  /// (Deref == true) a regiser plus an offset and, optionally, in the specified
-  /// address space (used mostly for the CFA).
-  ///
-  /// The CFA is usually defined using this rule by using the stack pointer or
-  /// frame pointer as the register, with an offset that accounts for all
-  /// spilled registers and all local variables in a function, and Deref ==
-  /// false.
-  LLVM_ABI static UnwindLocation
-  createIsRegisterPlusOffset(uint32_t Reg, int32_t Off,
-                             std::optional<uint32_t> AddrSpace = std::nullopt);
-  LLVM_ABI static UnwindLocation
-  createAtRegisterPlusOffset(uint32_t Reg, int32_t Off,
-                             std::optional<uint32_t> AddrSpace = std::nullopt);
-  /// Create a location whose value is the result of evaluating a DWARF
-  /// expression. This allows complex expressions to be evaluated in order to
-  /// unwind a register or CFA value.
-  LLVM_ABI static UnwindLocation createIsDWARFExpression(DWARFExpression Expr);
-  LLVM_ABI static UnwindLocation createAtDWARFExpression(DWARFExpression Expr);
-  LLVM_ABI static UnwindLocation createIsConstant(int32_t Value);
-
-  Location getLocation() const { return Kind; }
-  uint32_t getRegister() const { return RegNum; }
-  int32_t getOffset() const { return Offset; }
-  uint32_t getAddressSpace() const {
-    assert(Kind == RegPlusOffset && AddrSpace);
-    return *AddrSpace;
-  }
-  int32_t getConstant() const { return Offset; }
-  /// Some opcodes will modify the CFA location's register only, so we need
-  /// to be able to modify the CFA register when evaluating DWARF Call Frame
-  /// Information opcodes.
-  void setRegister(uint32_t NewRegNum) { RegNum = NewRegNum; }
-  /// Some opcodes will modify the CFA location's offset only, so we need
-  /// to be able to modify the CFA offset when evaluating DWARF Call Frame
-  /// Information opcodes.
-  void setOffset(int32_t NewOffset) { Offset = NewOffset; }
-  /// Some opcodes modify a constant value and we need to be able to update
-  /// the constant value (DW_CFA_GNU_window_save which is also known as
-  // DW_CFA_AARCH64_negate_ra_state).
-  void setConstant(int32_t Value) { Offset = Value; }
-
-  std::optional<DWARFExpression> getDWARFExpressionBytes() const {
-    return Expr;
-  }
-  /// Dump a location expression as text and use the register information if
-  /// some is provided.
-  ///
-  /// \param OS the stream to use for output.
-  ///
-  /// \param MRI register information that helps emit register names insteead
-  /// of raw register numbers.
-  ///
-  /// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
-  /// instead of from .debug_frame. This is needed for register number
-  /// conversion because some register numbers differ between the two sections
-  /// for certain architectures like x86.
-  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts) const;
-
-  LLVM_ABI bool operator==(const UnwindLocation &RHS) const;
-};
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindLocation &R);
-
-/// A class that can track all registers with locations in a UnwindRow object.
-///
-/// Register locations use a map where the key is the register number and the
-/// the value is a UnwindLocation.
-///
-/// The register maps are put into a class so that all register locations can
-/// be copied when parsing the unwind opcodes DW_CFA_remember_state and
-/// DW_CFA_restore_state.
-class RegisterLocations {
-  std::map<uint32_t, UnwindLocation> Locations;
-
-public:
-  /// Return the location for the register in \a RegNum if there is a location.
-  ///
-  /// \param RegNum the register number to find a location for.
-  ///
-  /// \returns A location if one is available for \a RegNum, or std::nullopt
-  /// otherwise.
-  std::optional<UnwindLocation> getRegisterLocation(uint32_t RegNum) const {
-    auto Pos = Locations.find(RegNum);
-    if (Pos == Locations.end())
-      return std::nullopt;
-    return Pos->second;
-  }
-
-  /// Set the location for the register in \a RegNum to \a Location.
-  ///
-  /// \param RegNum the register number to set the location for.
-  ///
-  /// \param Location the UnwindLocation that describes how to unwind the value.
-  void setRegisterLocation(uint32_t RegNum, const UnwindLocation &Location) {
-    Locations.erase(RegNum);
-    Locations.insert(std::make_pair(RegNum, Location));
-  }
-
-  /// Removes any rule for the register in \a RegNum.
-  ///
-  /// \param RegNum the register number to remove the location for.
-  void removeRegisterLocation(uint32_t RegNum) { Locations.erase(RegNum); }
-
-  /// Dump all registers + locations that are currently defined in this object.
-  ///
-  /// \param OS the stream to use for output.
-  ///
-  /// \param MRI register information that helps emit register names insteead
-  /// of raw register numbers.
-  ///
-  /// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
-  /// instead of from .debug_frame. This is needed for register number
-  /// conversion because some register numbers differ between the two sections
-  /// for certain architectures like x86.
-  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts) const;
-
-  /// Returns true if we have any register locations in this object.
-  bool hasLocations() const { return !Locations.empty(); }
-
-  size_t size() const { return Locations.size(); }
-
-  bool operator==(const RegisterLocations &RHS) const {
-    return Locations == RHS.Locations;
-  }
-};
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RegisterLocations &RL);
-
-/// A class that represents a single row in the unwind table that is decoded by
-/// parsing the DWARF Call Frame Information opcodes.
-///
-/// The row consists of an optional address, the rule to unwind the CFA and all
-/// rules to unwind any registers. If the address doesn't have a value, this
-/// row represents the initial instructions for a CIE. If the address has a
-/// value the UnwindRow represents a row in the UnwindTable for a FDE. The
-/// address is the first address for which the CFA location and register rules
-/// are valid within a function.
-///
-/// UnwindRow objects are created by parsing opcodes in the DWARF Call Frame
-/// Information and UnwindRow objects are lazily populated and pushed onto a
-/// stack in the UnwindTable when evaluating this state machine. Accessors are
-/// needed for the address, CFA value, and register locations as the opcodes
-/// encode a state machine that produces a sorted array of UnwindRow objects
-/// \see UnwindTable.
-class UnwindRow {
-  /// The address will be valid when parsing the instructions in a FDE. If
-  /// invalid, this object represents the initial instructions of a CIE.
-  std::optional<uint64_t> Address; ///< Address for row in FDE, invalid for CIE.
-  UnwindLocation CFAValue;    ///< How to unwind the Call Frame Address (CFA).
-  RegisterLocations RegLocs;  ///< How to unwind all registers in this list.
-
-public:
-  UnwindRow() : CFAValue(UnwindLocation::createUnspecified()) {}
-
-  /// Returns true if the address is valid in this object.
-  bool hasAddress() const { return Address.has_value(); }
-
-  /// Get the address for this row.
-  ///
-  /// Clients should only call this function after verifying it has a valid
-  /// address with a call to \see hasAddress().
-  uint64_t getAddress() const { return *Address; }
-
-  /// Set the address for this UnwindRow.
-  ///
-  /// The address represents the first address for which the CFAValue and
-  /// RegLocs are valid within a function.
-  void setAddress(uint64_t Addr) { Address = Addr; }
-
-  /// Offset the address for this UnwindRow.
-  ///
-  /// The address represents the first address for which the CFAValue and
-  /// RegLocs are valid within a function. Clients must ensure that this object
-  /// already has an address (\see hasAddress()) prior to calling this
-  /// function.
-  void slideAddress(uint64_t Offset) { *Address += Offset; }
-  UnwindLocation &getCFAValue() { return CFAValue; }
-  const UnwindLocation &getCFAValue() const { return CFAValue; }
-  RegisterLocations &getRegisterLocations() { return RegLocs; }
-  const RegisterLocations &getRegisterLocations() const { return RegLocs; }
-
-  /// Dump the UnwindRow to the stream.
-  ///
-  /// \param OS the stream to use for output.
-  ///
-  /// \param MRI register information that helps emit register names insteead
-  /// of raw register numbers.
-  ///
-  /// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
-  /// instead of from .debug_frame. This is needed for register number
-  /// conversion because some register numbers differ between the two sections
-  /// for certain architectures like x86.
-  ///
-  /// \param IndentLevel specify the indent level as an integer. The UnwindRow
-  /// will be output to the stream preceded by 2 * IndentLevel number of spaces.
-  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-                     unsigned IndentLevel = 0) const;
-};
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindRow &Row);
-
-/// A class that contains all UnwindRow objects for an FDE or a single unwind
-/// row for a CIE. To unwind an address the rows, which are sorted by start
-/// address, can be searched to find the UnwindRow with the lowest starting
-/// address that is greater than or equal to the address that is being looked
-/// up.
-class UnwindTable {
-public:
-  using RowContainer = std::vector<UnwindRow>;
-  using iterator = RowContainer::iterator;
-  using const_iterator = RowContainer::const_iterator;
-
-  UnwindTable(RowContainer &&Rows) : Rows(std::move(Rows)) {}
-
-  size_t size() const { return Rows.size(); }
-  iterator begin() { return Rows.begin(); }
-  const_iterator begin() const { return Rows.begin(); }
-  iterator end() { return Rows.end(); }
-  const_iterator end() const { return Rows.end(); }
-  const UnwindRow &operator[](size_t Index) const {
-    assert(Index < size());
-    return Rows[Index];
-  }
-
-  /// Dump the UnwindTable to the stream.
-  ///
-  /// \param OS the stream to use for output.
-  ///
-  /// \param MRI register information that helps emit register names instead
-  /// of raw register numbers.
-  ///
-  /// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
-  /// instead of from .debug_frame. This is needed for register number
-  /// conversion because some register numbers differ between the two sections
-  /// for certain architectures like x86.
-  ///
-  /// \param IndentLevel specify the indent level as an integer. The UnwindRow
-  /// will be output to the stream preceded by 2 * IndentLevel number of spaces.
-  LLVM_ABI void dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-                     unsigned IndentLevel = 0) const;
-
-private:
-  RowContainer Rows;
-};
-
-/// Parse the information in the CFIProgram and update the CurrRow object
-/// that the state machine describes.
-///
-/// This function emulates the state machine described in the DWARF Call Frame
-/// Information opcodes and will push CurrRow onto a RowContainer when needed.
-///
-/// \param CFIP the CFI program that contains the opcodes from a CIE or FDE.
-///
-/// \param CurrRow the current row to modify while parsing the state machine.
-///
-/// \param InitialLocs If non-NULL, we are parsing a FDE and this contains
-/// the initial register locations from the CIE. If NULL, then a CIE's
-/// opcodes are being parsed and this is not needed. This is used for the
-/// DW_CFA_restore and DW_CFA_restore_extended opcodes.
-///
-/// \returns An error if the DWARF Call Frame Information opcodes have state
-/// machine errors, or the accumulated rows otherwise.
-LLVM_ABI Expected<UnwindTable::RowContainer>
-parseRows(const CFIProgram &CFIP, UnwindRow &CurrRow,
-          const RegisterLocations *InitialLocs);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindTable &Rows);
-
 class CIE;
 
 /// Create an UnwindTable from a Common Information Entry (CIE).
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h
new file mode 100644
index 0000000000000..0aad492d47551
--- /dev/null
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h
@@ -0,0 +1,66 @@
+//===--- DWARFExpressionPRinter.h - DWARF Expression Printing ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARF_DWARFEXPRESSIONPRINTER_H
+#define LLVM_DEBUGINFO_DWARF_DWARFEXPRESSIONPRINTER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+
+// This functionality is separated from the main data structure so that nothing
+// in DWARFExpression.cpp needs build-time dependencies on DWARFUnit or other
+// higher-level Dwarf structures. This approach creates better layering and
+// allows DWARFExpression to be used from code which can't have dependencies on
+// those higher-level structures.
+
+class DWARFUnit;
+struct DIDumpOptions;
+class raw_ostream;
+
+/// Print a Dwarf expression/
+/// \param E to be printed
+/// \param OS to this stream
+/// \param GetNameForDWARFReg callback to return dwarf register name
+LLVM_ABI void printDwarfExpression(const DWARFExpression *E, raw_ostream &OS,
+                                   DIDumpOptions DumpOpts, DWARFUnit *U,
+                                   bool IsEH = false);
+
+/// Print the expression in a format intended to be compact and useful to a
+/// user, but not perfectly unambiguous, or capable of representing every
+/// valid DWARF expression. Returns true if the expression was sucessfully
+/// printed.
+///
+/// \param E to be printed
+/// \param OS to this stream
+/// \param GetNameForDWARFReg callback to return dwarf register name
+///
+/// \returns true if the expression was successfully printed
+LLVM_ABI bool printDwarfExpressionCompact(
+    const DWARFExpression *E, raw_ostream &OS,
+    std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg =
+        nullptr);
+
+/// Pretty print a register opcode and operands.
+/// \param U within the context of this Dwarf unit, if any.
+/// \param OS to this stream
+/// \param DumpOpts with these options
+/// \param Opcode to print
+/// \param Operands to the opcode
+///
+/// returns true if the Op was successfully printed
+LLVM_ABI bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
+                                    DIDumpOptions DumpOpts, uint8_t Opcode,
+                                    ArrayRef<uint64_t> Operands);
+
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_DWARF_DWARFEXPRESSIONPRINTER_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnwindTablePrinter.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnwindTablePrinter.h
new file mode 100644
index 0000000000000..242eceeed604a
--- /dev/null
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnwindTablePrinter.h
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARF_DWARFUNWINDTABLEPRINTER_H
+#define LLVM_DEBUGINFO_DWARF_DWARFUNWINDTABLEPRINTER_H
+
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+
+struct DIDumpOptions;
+
+namespace dwarf {
+
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindLocation &R);
+
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RegisterLocations &RL);
+
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindRow &Row);
+
+/// Print a UnwindTable to the stream.
+///
+/// \param Rows the UnwindTable to print.
+///
+/// \param OS the stream to use for output.
+///
+/// \param MRI register information that helps emit register names instead
+/// of raw register numbers.
+///
+/// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
+/// instead of from .debug_frame. This is needed for register number
+/// conversion because some register numbers differ between the two sections
+/// for certain architectures like x86.
+///
+/// \param IndentLevel specify the indent level as an integer. The UnwindRow
+/// will be output to the stream preceded by 2 * IndentLevel number of spaces.
+LLVM_ABI void printUnwindTable(const UnwindTable &Rows, raw_ostream &OS,
+                               DIDumpOptions DumpOpts,
+                               unsigned IndentLevel = 0);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const UnwindTable &Rows);
+
+} // end namespace dwarf
+
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_DWARF_DWARFUNWINDTABLEPRINTER_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
index 8e1b9589a5d76..f667496ab1059 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
@@ -14,8 +14,8 @@
 #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h"
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnitIndex.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <map>
diff --git a/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h
new file mode 100644
index 0000000000000..c571112344254
--- /dev/null
+++ b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h
@@ -0,0 +1,295 @@
+//===- DWARFCFIProgram.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFCFIPROGRAM_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFCFIPROGRAM_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Error.h"
+#include "llvm/TargetParser/Triple.h"
+#include <map>
+#include <memory>
+#include <vector>
+
+namespace llvm {
+
+namespace dwarf {
+
+/// Represent a sequence of Call Frame Information instructions that, when read
+/// in order, construct a table mapping PC to frame state. This can also be
+/// referred to as "CFI rules" in DWARF literature to avoid confusion with
+/// computer programs in the broader sense, and in this context each instruction
+/// would be a rule to establish the mapping. Refer to pg. 172 in the DWARF5
+/// manual, "6.4.1 Structure of Call Frame Information".
+class CFIProgram {
+public:
+  static constexpr size_t MaxOperands = 3;
+  typedef SmallVector<uint64_t, MaxOperands> Operands;
+
+  /// An instruction consists of a DWARF CFI opcode and an optional sequence of
+  /// operands. If it refers to an expression, then this expression has its own
+  /// sequence of operations and operands handled separately by DWARFExpression.
+  struct Instruction {
+    Instruction(uint8_t Opcode) : Opcode(Opcode) {}
+
+    uint8_t Opcode;
+    Operands Ops;
+    // Associated DWARF expression in case this instruction refers to one
+    std::optional<DWARFExpression> Expression;
+
+    LLVM_ABI Expected<uint64_t> getOperandAsUnsigned(const CFIProgram &CFIP,
+                                                     uint32_t OperandIdx) const;
+
+    LLVM_ABI Expected<int64_t> getOperandAsSigned(const CFIProgram &CFIP,
+                                                  uint32_t OperandIdx) const;
+  };
+
+  using InstrList = std::vector<Instruction>;
+  using iterator = InstrList::iterator;
+  using const_iterator = InstrList::const_iterator;
+
+  iterator begin() { return Instructions.begin(); }
+  const_iterator begin() const { return Instructions.begin(); }
+  iterator end() { return Instructions.end(); }
+  const_iterator end() const { return Instructions.end(); }
+
+  unsigned size() const { return (unsigned)Instructions.size(); }
+  bool empty() const { return Instructions.empty(); }
+  uint64_t codeAlign() const { return CodeAlignmentFactor; }
+  int64_t dataAlign() const { return DataAlignmentFactor; }
+  Triple::ArchType triple() const { return Arch; }
+
+  CFIProgram(uint64_t CodeAlignmentFactor, int64_t DataAlignmentFactor,
+             Triple::ArchType Arch)
+      : CodeAlignmentFactor(CodeAlignmentFactor),
+        DataAlignmentFactor(DataAlignmentFactor), Arch(Arch) {}
+
+  /// Parse and store a sequence of CFI instructions from Data,
+  /// starting at *Offset and ending at EndOffset. *Offset is updated
+  /// to EndOffset upon successful parsing, or indicates the offset
+  /// where a problem occurred in case an error is returned.
+  template <typename T>
+  Error parse(DWARFDataExtractorBase<T> &Data, uint64_t *Offset,
+              uint64_t EndOffset) {
+    // See DWARF standard v3, section 7.23
+    const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
+    const uint8_t DWARF_CFI_PRIMARY_OPERAND_MASK = 0x3f;
+
+    DataExtractor::Cursor C(*Offset);
+    while (C && C.tell() < EndOffset) {
+      uint8_t Opcode = Data.getRelocatedValue(C, 1);
+      if (!C)
+        break;
+
+      // Some instructions have a primary opcode encoded in the top bits.
+      if (uint8_t Primary = Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) {
+        // If it's a primary opcode, the first operand is encoded in the
+        // bottom bits of the opcode itself.
+        uint64_t Op1 = Opcode & DWARF_CFI_PRIMARY_OPERAND_MASK;
+        switch (Primary) {
+        case DW_CFA_advance_loc:
+        case DW_CFA_restore:
+          addInstruction(Primary, Op1);
+          break;
+        case DW_CFA_offset:
+          addInstruction(Primary, Op1, Data.getULEB128(C));
+          break;
+        default:
+          llvm_unreachable("invalid primary CFI opcode");
+        }
+        continue;
+      }
+
+      // Extended opcode - its value is Opcode itself.
+      switch (Opcode) {
+      default:
+        return createStringError(errc::illegal_byte_sequence,
+                                 "invalid extended CFI opcode 0x%" PRIx8,
+                                 Opcode);
+      case DW_CFA_nop:
+      case DW_CFA_remember_state:
+      case DW_CFA_restore_state:
+      case DW_CFA_GNU_window_save:
+      case DW_CFA_AARCH64_negate_ra_state_with_pc:
+        // No operands
+        addInstruction(Opcode);
+        break;
+      case DW_CFA_set_loc:
+        // Operands: Address
+        addInstruction(Opcode, Data.getRelocatedAddress(C));
+        break;
+      case DW_CFA_advance_loc1:
+        // Operands: 1-byte delta
+        addInstruction(Opcode, Data.getRelocatedValue(C, 1));
+        break;
+      case DW_CFA_advance_loc2:
+        // Operands: 2-byte delta
+        addInstruction(Opcode, Data.getRelocatedValue(C, 2));
+        break;
+      case DW_CFA_advance_loc4:
+        // Operands: 4-byte delta
+        addInstruction(Opcode, Data.getRelocatedValue(C, 4));
+        break;
+      case DW_CFA_restore_extended:
+      case DW_CFA_undefined:
+      case DW_CFA_same_value:
+      case DW_CFA_def_cfa_register:
+      case DW_CFA_def_cfa_offset:
+      case DW_CFA_GNU_args_size:
+        // Operands: ULEB128
+        addInstruction(Opcode, Data.getULEB128(C));
+        break;
+      case DW_CFA_def_cfa_offset_sf:
+        // Operands: SLEB128
+        addInstruction(Opcode, Data.getSLEB128(C));
+        break;
+      case DW_CFA_LLVM_def_aspace_cfa:
+      case DW_CFA_LLVM_def_aspace_cfa_sf: {
+        auto RegNum = Data.getULEB128(C);
+        auto CfaOffset = Opcode == DW_CFA_LLVM_def_aspace_cfa
+                             ? Data.getULEB128(C)
+                             : Data.getSLEB128(C);
+        auto AddressSpace = Data.getULEB128(C);
+        addInstruction(Opcode, RegNum, CfaOffset, AddressSpace);
+        break;
+      }
+      case DW_CFA_offset_extended:
+      case DW_CFA_register:
+      case DW_CFA_def_cfa:
+      case DW_CFA_val_offset: {
+        // Operands: ULEB128, ULEB128
+        // Note: We can not embed getULEB128 directly into function
+        // argument list. getULEB128 changes Offset and order of evaluation
+        // for arguments is unspecified.
+        uint64_t op1 = Data.getULEB128(C);
+        uint64_t op2 = Data.getULEB128(C);
+        addInstruction(Opcode, op1, op2);
+        break;
+      }
+      case DW_CFA_offset_extended_sf:
+      case DW_CFA_def_cfa_sf:
+      case DW_CFA_val_offset_sf: {
+        // Operands: ULEB128, SLEB128
+        // Note: see comment for the previous case
+        uint64_t op1 = Data.getULEB128(C);
+        uint64_t op2 = (uint64_t)Data.getSLEB128(C);
+        addInstruction(Opcode, op1, op2);
+        break;
+      }
+      case DW_CFA_def_cfa_expression: {
+        uint64_t ExprLength = Data.getULEB128(C);
+        addInstruction(Opcode, 0);
+        StringRef Expression = Data.getBytes(C, ExprLength);
+
+        DataExtractor Extractor(Expression, Data.isLittleEndian(),
+                                Data.getAddressSize());
+        // Note. We do not pass the DWARF format to DWARFExpression, because
+        // DW_OP_call_ref, the only operation which depends on the format, is
+        // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
+        Instructions.back().Expression =
+            DWARFExpression(Extractor, Data.getAddressSize());
+        break;
+      }
+      case DW_CFA_expression:
+      case DW_CFA_val_expression: {
+        uint64_t RegNum = Data.getULEB128(C);
+        addInstruction(Opcode, RegNum, 0);
+
+        uint64_t BlockLength = Data.getULEB128(C);
+        StringRef Expression = Data.getBytes(C, BlockLength);
+        DataExtractor Extractor(Expression, Data.isLittleEndian(),
+                                Data.getAddressSize());
+        // Note. We do not pass the DWARF format to DWARFExpression, because
+        // DW_OP_call_ref, the only operation which depends on the format, is
+        // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
+        Instructions.back().Expression =
+            DWARFExpression(Extractor, Data.getAddressSize());
+        break;
+      }
+      }
+    }
+
+    *Offset = C.tell();
+    return C.takeError();
+  }
+
+  void addInstruction(const Instruction &I) { Instructions.push_back(I); }
+
+  /// Get a DWARF CFI call frame string for the given DW_CFA opcode.
+  LLVM_ABI StringRef callFrameString(unsigned Opcode) const;
+
+  /// Types of operands to CFI instructions
+  /// In DWARF, this type is implicitly tied to a CFI instruction opcode and
+  /// thus this type doesn't need to be explicitly written to the file (this is
+  /// not a DWARF encoding). The relationship of instrs to operand types can
+  /// be obtained from getOperandTypes() and is only used to simplify
+  /// instruction printing and error messages.
+  enum OperandType {
+    OT_Unset,
+    OT_None,
+    OT_Address,
+    OT_Offset,
+    OT_FactoredCodeOffset,
+    OT_SignedFactDataOffset,
+    OT_UnsignedFactDataOffset,
+    OT_Register,
+    OT_AddressSpace,
+    OT_Expression
+  };
+
+  /// Get the OperandType as a "const char *".
+  LLVM_ABI static const char *operandTypeString(OperandType OT);
+
+  /// Retrieve the array describing the types of operands according to the enum
+  /// above. This is indexed by opcode.
+  LLVM_ABI static ArrayRef<OperandType[MaxOperands]> getOperandTypes();
+
+  /// Convenience method to add a new instruction with the given opcode.
+  void addInstruction(uint8_t Opcode) {
+    Instructions.push_back(Instruction(Opcode));
+  }
+
+  /// Add a new single-operand instruction.
+  void addInstruction(uint8_t Opcode, uint64_t Operand1) {
+    Instructions.push_back(Instruction(Opcode));
+    Instructions.back().Ops.push_back(Operand1);
+  }
+
+  /// Add a new instruction that has two operands.
+  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2) {
+    Instructions.push_back(Instruction(Opcode));
+    Instructions.back().Ops.push_back(Operand1);
+    Instructions.back().Ops.push_back(Operand2);
+  }
+
+  /// Add a new instruction that has three operands.
+  void addInstruction(uint8_t Opcode, uint64_t Operand1, uint64_t Operand2,
+                      uint64_t Operand3) {
+    Instructions.push_back(Instruction(Opcode));
+    Instructions.back().Ops.push_back(Operand1);
+    Instructions.back().Ops.push_back(Operand2);
+    Instructions.back().Ops.push_back(Operand3);
+  }
+
+private:
+  std::vector<Instruction> Instructions;
+  const uint64_t CodeAlignmentFactor;
+  const int64_t DataAlignmentFactor;
+  Triple::ArchType Arch;
+};
+
+} // end namespace dwarf
+
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFCFIPROGRAM_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h
similarity index 84%
rename from llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
rename to llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h
index 4c9a81093ca8f..52af205257627 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTORSIMPLE_H
-#define LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTORSIMPLE_H
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFDATAEXTRACTORSIMPLE_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFDATAEXTRACTORSIMPLE_H
 
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Compiler.h"
@@ -39,26 +39,24 @@ class DWARFDataExtractorBase : public DataExtractor {
                       Other.getAddressSize()) {}
 
   /// Extracts a value and returns it as adjusted by the Relocator
-  LLVM_ABI uint64_t getRelocatedValue(uint32_t Size, uint64_t *Off,
-                                      uint64_t *SectionIndex = nullptr,
-                                      Error *Err = nullptr) const {
+  uint64_t getRelocatedValue(uint32_t Size, uint64_t *Off,
+                             uint64_t *SectionIndex = nullptr,
+                             Error *Err = nullptr) const {
     return static_cast<const Relocator *>(this)->getRelocatedValueImpl(
         Size, Off, SectionIndex, Err);
   }
 
-  LLVM_ABI uint64_t getRelocatedValue(Cursor &C, uint32_t Size,
-                                      uint64_t *SectionIndex = nullptr) const {
+  uint64_t getRelocatedValue(Cursor &C, uint32_t Size,
+                             uint64_t *SectionIndex = nullptr) const {
     return getRelocatedValue(Size, &getOffset(C), SectionIndex, &getError(C));
   }
 
   /// Extracts an address-sized value.
-  LLVM_ABI uint64_t getRelocatedAddress(uint64_t *Off,
-                                        uint64_t *SecIx = nullptr) const {
+  uint64_t getRelocatedAddress(uint64_t *Off, uint64_t *SecIx = nullptr) const {
     return getRelocatedValue(getAddressSize(), Off, SecIx);
   }
 
-  LLVM_ABI uint64_t getRelocatedAddress(Cursor &C,
-                                        uint64_t *SecIx = nullptr) const {
+  uint64_t getRelocatedAddress(Cursor &C, uint64_t *SecIx = nullptr) const {
     return getRelocatedValue(getAddressSize(), &getOffset(C), SecIx,
                              &getError(C));
   }
@@ -68,7 +66,7 @@ class DWARFDataExtractorBase : public DataExtractor {
   /// 64-bit length. Returns the actual length, and the DWARF format which is
   /// encoded in the field. In case of errors, it returns {0, DWARF32} and
   /// leaves the offset unchanged.
-  LLVM_ABI std::pair<uint64_t, dwarf::DwarfFormat>
+  std::pair<uint64_t, dwarf::DwarfFormat>
   getInitialLength(uint64_t *Off, Error *Err = nullptr) const {
     ErrorAsOutParameter ErrAsOut(Err);
     if (Err && *Err)
@@ -100,8 +98,7 @@ class DWARFDataExtractorBase : public DataExtractor {
     return {0, dwarf::DWARF32};
   }
 
-  LLVM_ABI std::pair<uint64_t, dwarf::DwarfFormat>
-  getInitialLength(Cursor &C) const {
+  std::pair<uint64_t, dwarf::DwarfFormat> getInitialLength(Cursor &C) const {
     return getInitialLength(&getOffset(C), &getError(C));
   }
 
@@ -109,9 +106,8 @@ class DWARFDataExtractorBase : public DataExtractor {
   /// There is a DWARF encoding that uses a PC-relative adjustment.
   /// For these values, \p AbsPosOffset is used to fix them, which should
   /// reflect the absolute address of this pointer.
-  LLVM_ABI std::optional<uint64_t>
-  getEncodedPointer(uint64_t *Offset, uint8_t Encoding,
-                    uint64_t PCRelOffset) const {
+  std::optional<uint64_t> getEncodedPointer(uint64_t *Offset, uint8_t Encoding,
+                                            uint64_t PCRelOffset) const {
     if (Encoding == dwarf::DW_EH_PE_omit)
       return std::nullopt;
 
@@ -195,4 +191,4 @@ class DWARFDataExtractorSimple
 };
 
 } // end namespace llvm
-#endif // LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTOR_H
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFDATAEXTRACTORSIMPLE_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h
similarity index 65%
rename from llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
rename to llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h
index ea414278c35d6..458a3a4e49e8b 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
-#define LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFEXPRESSION_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFEXPRESSION_H
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
@@ -65,7 +65,7 @@ class DWARFExpression {
 
     /// Description of the encoding of one expression Op.
     struct Description {
-      DwarfVersion Version; ///< Dwarf version where the Op was introduced.
+      DwarfVersion Version;     ///< Dwarf version where the Op was introduced.
       SmallVector<Encoding> Op; ///< Encoding for Op operands.
 
       template <typename... Ts>
@@ -77,7 +77,6 @@ class DWARFExpression {
 
   private:
     friend class DWARFExpression::iterator;
-    friend class DWARFExpressionPrinter;
     friend class DWARFVerifier;
 
     uint8_t Opcode; ///< The Op Opcode, DW_OP_<something>.
@@ -156,7 +155,6 @@ class DWARFExpression {
 
   StringRef getData() const { return Data.getData(); }
 
-  friend class DWARFExpressionPrinter;
   friend class DWARFVerifier;
 
 private:
@@ -170,65 +168,6 @@ inline bool operator==(const DWARFExpression::iterator &LHS,
   return LHS.Expr == RHS.Expr && LHS.Offset == RHS.Offset;
 }
 
-// This functionality is separated from the main data structure so that nothing
-// in DWARFExpression.cpp needs build-time dependencies on DWARFUnit or other
-// higher-level Dwarf structures. This approach creates better layering and
-// allows DWARFExpression to be used from code which can't have dependencies on
-// those higher-level structures.
-
-class DWARFUnit;
-struct DIDumpOptions;
-class raw_ostream;
-
-class DWARFExpressionPrinter {
-public:
-  /// Print a Dwarf expression/
-  /// \param E to be printed
-  /// \param OS to this stream
-  /// \param GetNameForDWARFReg callback to return dwarf register name
-  LLVM_ABI static void print(const DWARFExpression *E, raw_ostream &OS,
-                             DIDumpOptions DumpOpts, DWARFUnit *U,
-                             bool IsEH = false);
-
-  /// Print the expression in a format intended to be compact and useful to a
-  /// user, but not perfectly unambiguous, or capable of representing every
-  /// valid DWARF expression. Returns true if the expression was sucessfully
-  /// printed.
-  ///
-  /// \param E to be printed
-  /// \param OS to this stream
-  /// \param GetNameForDWARFReg callback to return dwarf register name
-  ///
-  /// \returns true if the expression was successfully printed
-  LLVM_ABI static bool printCompact(
-      const DWARFExpression *E, raw_ostream &OS,
-      std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg =
-          nullptr);
-
-  /// Pretty print a register opcode and operands.
-  /// \param U within the context of this Dwarf unit, if any.
-  /// \param OS to this stream
-  /// \param DumpOpts with these options
-  /// \param Opcode to print
-  /// \param Operands to the opcode
-  ///
-  /// returns true if the Op was successfully printed
-  LLVM_ABI static bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
-                                             DIDumpOptions DumpOpts,
-                                             uint8_t Opcode,
-                                             ArrayRef<uint64_t> Operands);
-
-private:
-  static bool printOp(const DWARFExpression::Operation *Op, raw_ostream &OS,
-                      DIDumpOptions DumpOpts, const DWARFExpression *Expr,
-                      DWARFUnit *U);
-
-  static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
-                                     DIDumpOptions DumpOpts,
-                                     ArrayRef<uint64_t> Operands,
-                                     unsigned Operand);
-};
-
 } // end namespace llvm
 
-#endif // LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFEXPRESSION_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.h b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.h
new file mode 100644
index 0000000000000..828dea12d168c
--- /dev/null
+++ b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.h
@@ -0,0 +1,321 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFUNWINDTABLE_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFUNWINDTABLE_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Error.h"
+#include <map>
+#include <vector>
+
+namespace llvm {
+
+namespace dwarf {
+constexpr uint32_t InvalidRegisterNumber = UINT32_MAX;
+
+/// A class that represents a location for the Call Frame Address (CFA) or a
+/// register. This is decoded from the DWARF Call Frame Information
+/// instructions and put into an UnwindRow.
+class UnwindLocation {
+public:
+  enum Location {
+    /// Not specified.
+    Unspecified,
+    /// Register is not available and can't be recovered.
+    Undefined,
+    /// Register value is in the register, nothing needs to be done to unwind
+    /// it:
+    ///   reg = reg
+    Same,
+    /// Register is in or at the CFA plus an offset:
+    ///   reg = CFA + offset
+    ///   reg = defef(CFA + offset)
+    CFAPlusOffset,
+    /// Register or CFA is in or at a register plus offset, optionally in
+    /// an address space:
+    ///   reg = reg + offset [in addrspace]
+    ///   reg = deref(reg + offset [in addrspace])
+    RegPlusOffset,
+    /// Register or CFA value is in or at a value found by evaluating a DWARF
+    /// expression:
+    ///   reg = eval(dwarf_expr)
+    ///   reg = deref(eval(dwarf_expr))
+    DWARFExpr,
+    /// Value is a constant value contained in "Offset":
+    ///   reg = Offset
+    Constant,
+  };
+
+private:
+  Location Kind;   /// The type of the location that describes how to unwind it.
+  uint32_t RegNum; /// The register number for Kind == RegPlusOffset.
+  int32_t Offset;  /// The offset for Kind == CFAPlusOffset or RegPlusOffset.
+  std::optional<uint32_t> AddrSpace;   /// The address space for Kind ==
+                                       /// RegPlusOffset for CFA.
+  std::optional<DWARFExpression> Expr; /// The DWARF expression for Kind ==
+                                       /// DWARFExpression.
+  bool Dereference; /// If true, the resulting location must be dereferenced
+                    /// after the location value is computed.
+
+  // Constructors are private to force people to use the create static
+  // functions.
+  UnwindLocation(Location K)
+      : Kind(K), RegNum(InvalidRegisterNumber), Offset(0),
+        AddrSpace(std::nullopt), Dereference(false) {}
+
+  UnwindLocation(Location K, uint32_t Reg, int32_t Off,
+                 std::optional<uint32_t> AS, bool Deref)
+      : Kind(K), RegNum(Reg), Offset(Off), AddrSpace(AS), Dereference(Deref) {}
+
+  UnwindLocation(DWARFExpression E, bool Deref)
+      : Kind(DWARFExpr), RegNum(InvalidRegisterNumber), Offset(0), Expr(E),
+        Dereference(Deref) {}
+
+public:
+  /// Create a location whose rule is set to Unspecified. This means the
+  /// register value might be in the same register but it wasn't specified in
+  /// the unwind opcodes.
+  LLVM_ABI static UnwindLocation createUnspecified();
+  /// Create a location where the value is undefined and not available. This can
+  /// happen when a register is volatile and can't be recovered.
+  LLVM_ABI static UnwindLocation createUndefined();
+  /// Create a location where the value is known to be in the register itself.
+  LLVM_ABI static UnwindLocation createSame();
+  /// Create a location that is in (Deref == false) or at (Deref == true) the
+  /// CFA plus an offset. Most registers that are spilled onto the stack use
+  /// this rule. The rule for the register will use this rule and specify a
+  /// unique offset from the CFA with \a Deref set to true. This value will be
+  /// relative to a CFA value which is typically defined using the register
+  /// plus offset location. \see createRegisterPlusOffset(...) for more
+  /// information.
+  LLVM_ABI static UnwindLocation createIsCFAPlusOffset(int32_t Off);
+  LLVM_ABI static UnwindLocation createAtCFAPlusOffset(int32_t Off);
+  /// Create a location where the saved value is in (Deref == false) or at
+  /// (Deref == true) a regiser plus an offset and, optionally, in the specified
+  /// address space (used mostly for the CFA).
+  ///
+  /// The CFA is usually defined using this rule by using the stack pointer or
+  /// frame pointer as the register, with an offset that accounts for all
+  /// spilled registers and all local variables in a function, and Deref ==
+  /// false.
+  LLVM_ABI static UnwindLocation
+  createIsRegisterPlusOffset(uint32_t Reg, int32_t Off,
+                             std::optional<uint32_t> AddrSpace = std::nullopt);
+  LLVM_ABI static UnwindLocation
+  createAtRegisterPlusOffset(uint32_t Reg, int32_t Off,
+                             std::optional<uint32_t> AddrSpace = std::nullopt);
+  /// Create a location whose value is the result of evaluating a DWARF
+  /// expression. This allows complex expressions to be evaluated in order to
+  /// unwind a register or CFA value.
+  LLVM_ABI static UnwindLocation createIsDWARFExpression(DWARFExpression Expr);
+  LLVM_ABI static UnwindLocation createAtDWARFExpression(DWARFExpression Expr);
+  LLVM_ABI static UnwindLocation createIsConstant(int32_t Value);
+
+  Location getLocation() const { return Kind; }
+  uint32_t getRegister() const { return RegNum; }
+  int32_t getOffset() const { return Offset; }
+  bool hasAddressSpace() const {
+    if (AddrSpace)
+      return true;
+    return false;
+  }
+  uint32_t getAddressSpace() const {
+    assert(Kind == RegPlusOffset && AddrSpace);
+    return *AddrSpace;
+  }
+  int32_t getConstant() const { return Offset; }
+  bool getDereference() const { return Dereference; }
+
+  /// Some opcodes will modify the CFA location's register only, so we need
+  /// to be able to modify the CFA register when evaluating DWARF Call Frame
+  /// Information opcodes.
+  void setRegister(uint32_t NewRegNum) { RegNum = NewRegNum; }
+  /// Some opcodes will modify the CFA location's offset only, so we need
+  /// to be able to modify the CFA offset when evaluating DWARF Call Frame
+  /// Information opcodes.
+  void setOffset(int32_t NewOffset) { Offset = NewOffset; }
+  /// Some opcodes modify a constant value and we need to be able to update
+  /// the constant value (DW_CFA_GNU_window_save which is also known as
+  // DW_CFA_AARCH64_negate_ra_state).
+  void setConstant(int32_t Value) { Offset = Value; }
+
+  std::optional<DWARFExpression> getDWARFExpressionBytes() const {
+    return Expr;
+  }
+
+  LLVM_ABI bool operator==(const UnwindLocation &RHS) const;
+};
+
+/// A class that can track all registers with locations in a UnwindRow object.
+///
+/// Register locations use a map where the key is the register number and the
+/// the value is a UnwindLocation.
+///
+/// The register maps are put into a class so that all register locations can
+/// be copied when parsing the unwind opcodes DW_CFA_remember_state and
+/// DW_CFA_restore_state.
+class RegisterLocations {
+  std::map<uint32_t, UnwindLocation> Locations;
+
+public:
+  /// Return the location for the register in \a RegNum if there is a location.
+  ///
+  /// \param RegNum the register number to find a location for.
+  ///
+  /// \returns A location if one is available for \a RegNum, or std::nullopt
+  /// otherwise.
+  std::optional<UnwindLocation> getRegisterLocation(uint32_t RegNum) const {
+    auto Pos = Locations.find(RegNum);
+    if (Pos == Locations.end())
+      return std::nullopt;
+    return Pos->second;
+  }
+
+  SmallVector<uint32_t, 4> getRegisters() const {
+    SmallVector<uint32_t, 4> Registers;
+    for (auto &&[Register, _] : Locations)
+      Registers.push_back(Register);
+    return Registers;
+  }
+
+  /// Set the location for the register in \a RegNum to \a Location.
+  ///
+  /// \param RegNum the register number to set the location for.
+  ///
+  /// \param Location the UnwindLocation that describes how to unwind the value.
+  void setRegisterLocation(uint32_t RegNum, const UnwindLocation &Location) {
+    Locations.erase(RegNum);
+    Locations.insert(std::make_pair(RegNum, Location));
+  }
+
+  /// Removes any rule for the register in \a RegNum.
+  ///
+  /// \param RegNum the register number to remove the location for.
+  void removeRegisterLocation(uint32_t RegNum) { Locations.erase(RegNum); }
+
+  /// Returns true if we have any register locations in this object.
+  bool hasLocations() const { return !Locations.empty(); }
+
+  size_t size() const { return Locations.size(); }
+
+  bool operator==(const RegisterLocations &RHS) const {
+    return Locations == RHS.Locations;
+  }
+};
+
+/// A class that represents a single row in the unwind table that is decoded by
+/// parsing the DWARF Call Frame Information opcodes.
+///
+/// The row consists of an optional address, the rule to unwind the CFA and all
+/// rules to unwind any registers. If the address doesn't have a value, this
+/// row represents the initial instructions for a CIE. If the address has a
+/// value the UnwindRow represents a row in the UnwindTable for a FDE. The
+/// address is the first address for which the CFA location and register rules
+/// are valid within a function.
+///
+/// UnwindRow objects are created by parsing opcodes in the DWARF Call Frame
+/// Information and UnwindRow objects are lazily populated and pushed onto a
+/// stack in the UnwindTable when evaluating this state machine. Accessors are
+/// needed for the address, CFA value, and register locations as the opcodes
+/// encode a state machine that produces a sorted array of UnwindRow objects
+/// \see UnwindTable.
+class UnwindRow {
+  /// The address will be valid when parsing the instructions in a FDE. If
+  /// invalid, this object represents the initial instructions of a CIE.
+  std::optional<uint64_t> Address; ///< Address for row in FDE, invalid for CIE.
+  UnwindLocation CFAValue;   ///< How to unwind the Call Frame Address (CFA).
+  RegisterLocations RegLocs; ///< How to unwind all registers in this list.
+
+public:
+  UnwindRow() : CFAValue(UnwindLocation::createUnspecified()) {}
+
+  /// Returns true if the address is valid in this object.
+  bool hasAddress() const { return Address.has_value(); }
+
+  /// Get the address for this row.
+  ///
+  /// Clients should only call this function after verifying it has a valid
+  /// address with a call to \see hasAddress().
+  uint64_t getAddress() const { return *Address; }
+
+  /// Set the address for this UnwindRow.
+  ///
+  /// The address represents the first address for which the CFAValue and
+  /// RegLocs are valid within a function.
+  void setAddress(uint64_t Addr) { Address = Addr; }
+
+  /// Offset the address for this UnwindRow.
+  ///
+  /// The address represents the first address for which the CFAValue and
+  /// RegLocs are valid within a function. Clients must ensure that this object
+  /// already has an address (\see hasAddress()) prior to calling this
+  /// function.
+  void slideAddress(uint64_t Offset) { *Address += Offset; }
+  UnwindLocation &getCFAValue() { return CFAValue; }
+  const UnwindLocation &getCFAValue() const { return CFAValue; }
+  RegisterLocations &getRegisterLocations() { return RegLocs; }
+  const RegisterLocations &getRegisterLocations() const { return RegLocs; }
+};
+
+/// A class that contains all UnwindRow objects for an FDE or a single unwind
+/// row for a CIE. To unwind an address the rows, which are sorted by start
+/// address, can be searched to find the UnwindRow with the lowest starting
+/// address that is greater than or equal to the address that is being looked
+/// up.
+class UnwindTable {
+public:
+  using RowContainer = std::vector<UnwindRow>;
+  using iterator = RowContainer::iterator;
+  using const_iterator = RowContainer::const_iterator;
+
+  UnwindTable(RowContainer &&Rows) : Rows(std::move(Rows)) {}
+
+  size_t size() const { return Rows.size(); }
+  iterator begin() { return Rows.begin(); }
+  const_iterator begin() const { return Rows.begin(); }
+  iterator end() { return Rows.end(); }
+  const_iterator end() const { return Rows.end(); }
+  const UnwindRow &operator[](size_t Index) const {
+    assert(Index < size());
+    return Rows[Index];
+  }
+
+private:
+  RowContainer Rows;
+};
+
+/// Parse the information in the CFIProgram and update the CurrRow object
+/// that the state machine describes.
+///
+/// This function emulates the state machine described in the DWARF Call Frame
+/// Information opcodes and will push CurrRow onto a RowContainer when needed.
+///
+/// \param CFIP the CFI program that contains the opcodes from a CIE or FDE.
+///
+/// \param CurrRow the current row to modify while parsing the state machine.
+///
+/// \param InitialLocs If non-NULL, we are parsing a FDE and this contains
+/// the initial register locations from the CIE. If NULL, then a CIE's
+/// opcodes are being parsed and this is not needed. This is used for the
+/// DW_CFA_restore and DW_CFA_restore_extended opcodes.
+///
+/// \returns An error if the DWARF Call Frame Information opcodes have state
+/// machine errors, or the accumulated rows otherwise.
+LLVM_ABI Expected<UnwindTable::RowContainer>
+parseRows(const CFIProgram &CFIP, UnwindRow &CurrRow,
+          const RegisterLocations *InitialLocs);
+
+} // end namespace dwarf
+
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFUNWINDTABLE_H
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
index d72fb47cd9b04..a9cfe726a73d3 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -226,35 +226,51 @@ enum class TagKind { Class, Struct, Union, Enum };
 
 enum class NodeKind {
   Unknown,
-  Md5Symbol,
-  PrimitiveType,
-  FunctionSignature,
-  Identifier,
-  NamedIdentifier,
-  VcallThunkIdentifier,
-  LocalStaticGuardIdentifier,
-  IntrinsicFunctionIdentifier,
-  ConversionOperatorIdentifier,
+
+  SymbolStart,
+  Md5Symbol = SymbolStart,
+  EncodedStringLiteral,
+  FunctionSymbol,
+  LocalStaticGuardVariable,
+  SpecialTableSymbol,
+  VariableSymbol,
+  SymbolEnd = VariableSymbol,
+
+  IdentifierStart,
+  ConversionOperatorIdentifier = IdentifierStart,
   DynamicStructorIdentifier,
-  StructorIdentifier,
+  IntrinsicFunctionIdentifier,
   LiteralOperatorIdentifier,
+  LocalStaticGuardIdentifier,
+  NamedIdentifier,
+  RttiBaseClassDescriptor,
+  StructorIdentifier,
+  VcallThunkIdentifier,
+  IdentifierEnd = VcallThunkIdentifier,
+
+  TypeStart,
+  ArrayType = TypeStart,
+  Custom,
+
+  FunctionSignature,
   ThunkSignature,
+  FunctionSignatureEnd = ThunkSignature,
+
+  IntrinsicType,
   PointerType,
+  PrimitiveType,
   TagType,
-  ArrayType,
-  Custom,
-  IntrinsicType,
+  TypeEnd = TagType,
+
+  PointerAuthQualifier,
+
+  IntegerLiteral,
+
   NodeArray,
+
   QualifiedName,
+
   TemplateParameterReference,
-  EncodedStringLiteral,
-  IntegerLiteral,
-  RttiBaseClassDescriptor,
-  LocalStaticGuardVariable,
-  FunctionSymbol,
-  VariableSymbol,
-  SpecialTableSymbol,
-  PointerAuthQualifier,
 };
 
 struct Node {
@@ -309,6 +325,10 @@ struct TypeNode : public Node {
     outputPost(OB, Flags);
   }
 
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::TypeStart && N->kind() <= NodeKind::TypeEnd;
+  }
+
   Qualifiers Quals = Q_None;
 };
 
@@ -319,6 +339,10 @@ struct PrimitiveTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override {}
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::PrimitiveType;
+  }
+
   PrimitiveKind PrimKind;
 };
 
@@ -329,6 +353,11 @@ struct FunctionSignatureNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::FunctionSignature &&
+           N->kind() <= NodeKind::FunctionSignatureEnd;
+  }
+
   // Valid if this FunctionTypeNode is the Pointee of a PointerType or
   // MemberPointerType.
   PointerAffinity Affinity = PointerAffinity::None;
@@ -357,6 +386,11 @@ struct FunctionSignatureNode : public TypeNode {
 struct IdentifierNode : public Node {
   explicit IdentifierNode(NodeKind K) : Node(K) {}
 
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::IdentifierStart &&
+           N->kind() <= NodeKind::IdentifierEnd;
+  }
+
   NodeArrayNode *TemplateParams = nullptr;
 
 protected:
@@ -368,6 +402,10 @@ struct VcallThunkIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::VcallThunkIdentifier;
+  }
+
   uint64_t OffsetInVTable = 0;
 };
 
@@ -377,6 +415,10 @@ struct DynamicStructorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::DynamicStructorIdentifier;
+  }
+
   VariableSymbolNode *Variable = nullptr;
   QualifiedNameNode *Name = nullptr;
   bool IsDestructor = false;
@@ -387,6 +429,10 @@ struct NamedIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::NamedIdentifier;
+  }
+
   std::string_view Name;
 };
 
@@ -397,6 +443,10 @@ struct IntrinsicFunctionIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntrinsicFunctionIdentifier;
+  }
+
   IntrinsicFunctionKind Operator;
 };
 
@@ -406,6 +456,10 @@ struct LiteralOperatorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::LiteralOperatorIdentifier;
+  }
+
   std::string_view Name;
 };
 
@@ -415,6 +469,10 @@ struct LocalStaticGuardIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::LocalStaticGuardIdentifier;
+  }
+
   bool IsThread = false;
   uint32_t ScopeIndex = 0;
 };
@@ -425,6 +483,10 @@ struct ConversionOperatorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ConversionOperatorIdentifier;
+  }
+
   // The type that this operator converts too.
   TypeNode *TargetType = nullptr;
 };
@@ -437,6 +499,10 @@ struct StructorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::StructorIdentifier;
+  }
+
   // The name of the class that this is a structor of.
   IdentifierNode *Class = nullptr;
   bool IsDestructor = false;
@@ -448,6 +514,10 @@ struct ThunkSignatureNode : public FunctionSignatureNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ThunkSignature;
+  }
+
   struct ThisAdjustor {
     uint32_t StaticOffset = 0;
     int32_t VBPtrOffset = 0;
@@ -463,6 +533,10 @@ struct PointerTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::PointerType;
+  }
+
   // Is this a pointer, reference, or rvalue-reference?
   PointerAffinity Affinity = PointerAffinity::None;
 
@@ -482,6 +556,8 @@ struct TagTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) { return N->kind() == NodeKind::TagType; }
+
   QualifiedNameNode *QualifiedName = nullptr;
   TagKind Tag;
 };
@@ -495,6 +571,10 @@ struct ArrayTypeNode : public TypeNode {
   void outputDimensionsImpl(OutputBuffer &OB, OutputFlags Flags) const;
   void outputOneDimension(OutputBuffer &OB, OutputFlags Flags, Node *N) const;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ArrayType;
+  }
+
   // A list of array dimensions.  e.g. [3,4,5] in `int Foo[3][4][5]`
   NodeArrayNode *Dimensions = nullptr;
 
@@ -505,6 +585,10 @@ struct ArrayTypeNode : public TypeNode {
 struct IntrinsicNode : public TypeNode {
   IntrinsicNode() : TypeNode(NodeKind::IntrinsicType) {}
   void output(OutputBuffer &OB, OutputFlags Flags) const override {}
+
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntrinsicType;
+  }
 };
 
 struct CustomTypeNode : public TypeNode {
@@ -513,6 +597,8 @@ struct CustomTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) { return N->kind() == NodeKind::Custom; }
+
   IdentifierNode *Identifier = nullptr;
 };
 
@@ -524,6 +610,10 @@ struct NodeArrayNode : public Node {
   void output(OutputBuffer &OB, OutputFlags Flags,
               std::string_view Separator) const;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::NodeArray;
+  }
+
   Node **Nodes = nullptr;
   size_t Count = 0;
 };
@@ -533,6 +623,10 @@ struct QualifiedNameNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::QualifiedName;
+  }
+
   NodeArrayNode *Components = nullptr;
 
   IdentifierNode *getUnqualifiedIdentifier() {
@@ -547,6 +641,10 @@ struct TemplateParameterReferenceNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::TemplateParameterReference;
+  }
+
   SymbolNode *Symbol = nullptr;
 
   int ThunkOffsetCount = 0;
@@ -562,6 +660,10 @@ struct IntegerLiteralNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntegerLiteral;
+  }
+
   uint64_t Value = 0;
   bool IsNegative = false;
 };
@@ -572,6 +674,10 @@ struct RttiBaseClassDescriptorNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::RttiBaseClassDescriptor;
+  }
+
   uint32_t NVOffset = 0;
   int32_t VBPtrOffset = 0;
   uint32_t VBTableOffset = 0;
@@ -581,6 +687,12 @@ struct RttiBaseClassDescriptorNode : public IdentifierNode {
 struct SymbolNode : public Node {
   explicit SymbolNode(NodeKind K) : Node(K) {}
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
+
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::SymbolStart &&
+           N->kind() <= NodeKind::SymbolEnd;
+  }
+
   QualifiedNameNode *Name = nullptr;
 };
 
@@ -589,6 +701,11 @@ struct SpecialTableSymbolNode : public SymbolNode {
       : SymbolNode(NodeKind::SpecialTableSymbol) {}
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
+
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::SpecialTableSymbol;
+  }
+
   QualifiedNameNode *TargetName = nullptr;
   Qualifiers Quals = Qualifiers::Q_None;
 };
@@ -599,6 +716,10 @@ struct LocalStaticGuardVariableNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::LocalStaticGuardVariable;
+  }
+
   bool IsVisible = false;
 };
 
@@ -607,6 +728,10 @@ struct EncodedStringLiteralNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::EncodedStringLiteral;
+  }
+
   std::string_view DecodedString;
   bool IsTruncated = false;
   CharKind Char = CharKind::Char;
@@ -617,6 +742,10 @@ struct VariableSymbolNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::VariableSymbol;
+  }
+
   StorageClass SC = StorageClass::None;
   TypeNode *Type = nullptr;
 };
@@ -626,6 +755,10 @@ struct FunctionSymbolNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::FunctionSymbol;
+  }
+
   FunctionSignatureNode *Signature = nullptr;
 };
 
@@ -641,6 +774,10 @@ struct PointerAuthQualifierNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::PointerAuthQualifier;
+  }
+
   // List of arguments.
   NodeArrayNode *Components = nullptr;
 };
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
index b61c8b8563a1a..03165895b85d0 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
@@ -36,16 +36,6 @@ class ThreadSafeContext {
   };
 
 public:
-  // RAII based lock for ThreadSafeContext.
-  class [[nodiscard]] Lock {
-  public:
-    Lock(std::shared_ptr<State> S) : S(std::move(S)), L(this->S->Mutex) {}
-
-  private:
-    std::shared_ptr<State> S;
-    std::unique_lock<std::recursive_mutex> L;
-  };
-
   /// Construct a null context.
   ThreadSafeContext() = default;
 
@@ -56,17 +46,20 @@ class ThreadSafeContext {
            "Can not construct a ThreadSafeContext from a nullptr");
   }
 
-  /// Returns a pointer to the LLVMContext that was used to construct this
-  /// instance, or null if the instance was default constructed.
-  LLVMContext *getContext() { return S ? S->Ctx.get() : nullptr; }
-
-  /// Returns a pointer to the LLVMContext that was used to construct this
-  /// instance, or null if the instance was default constructed.
-  const LLVMContext *getContext() const { return S ? S->Ctx.get() : nullptr; }
+  template <typename Func> decltype(auto) withContextDo(Func &&F) {
+    if (auto TmpS = S) {
+      std::lock_guard<std::recursive_mutex> Lock(TmpS->Mutex);
+      return F(TmpS->Ctx.get());
+    } else
+      return F((LLVMContext *)nullptr);
+  }
 
-  Lock getLock() const {
-    assert(S && "Can not lock an empty ThreadSafeContext");
-    return Lock(S);
+  template <typename Func> decltype(auto) withContextDo(Func &&F) const {
+    if (auto TmpS = S) {
+      std::lock_guard<std::recursive_mutex> Lock(TmpS->Mutex);
+      return F(const_cast<const LLVMContext *>(TmpS->Ctx.get()));
+    } else
+      return F((const LLVMContext *)nullptr);
   }
 
 private:
@@ -89,10 +82,7 @@ class ThreadSafeModule {
     // *before* the context that it depends on.
     // We also need to lock the context to make sure the module tear-down
     // does not overlap any other work on the context.
-    if (M) {
-      auto L = TSCtx.getLock();
-      M = nullptr;
-    }
+    TSCtx.withContextDo([this](LLVMContext *Ctx) { M = nullptr; });
     M = std::move(Other.M);
     TSCtx = std::move(Other.TSCtx);
     return *this;
@@ -111,45 +101,39 @@ class ThreadSafeModule {
 
   ~ThreadSafeModule() {
     // We need to lock the context while we destruct the module.
-    if (M) {
-      auto L = TSCtx.getLock();
-      M = nullptr;
-    }
+    TSCtx.withContextDo([this](LLVMContext *Ctx) { M = nullptr; });
   }
 
   /// Boolean conversion: This ThreadSafeModule will evaluate to true if it
   /// wraps a non-null module.
-  explicit operator bool() const {
-    if (M) {
-      assert(TSCtx.getContext() &&
-             "Non-null module must have non-null context");
-      return true;
-    }
-    return false;
-  }
+  explicit operator bool() const { return !!M; }
 
   /// Locks the associated ThreadSafeContext and calls the given function
   /// on the contained Module.
   template <typename Func> decltype(auto) withModuleDo(Func &&F) {
-    assert(M && "Can not call on null module");
-    auto Lock = TSCtx.getLock();
-    return F(*M);
+    return TSCtx.withContextDo([&](LLVMContext *) {
+      assert(M && "Can not call on null module");
+      return F(*M);
+    });
   }
 
   /// Locks the associated ThreadSafeContext and calls the given function
   /// on the contained Module.
   template <typename Func> decltype(auto) withModuleDo(Func &&F) const {
-    assert(M && "Can not call on null module");
-    auto Lock = TSCtx.getLock();
-    return F(*M);
+    return TSCtx.withContextDo([&](const LLVMContext *) {
+      assert(M && "Can not call on null module");
+      return F(*M);
+    });
   }
 
   /// Locks the associated ThreadSafeContext and calls the given function,
   /// passing the contained std::unique_ptr<Module>. The given function should
   /// consume the Module.
   template <typename Func> decltype(auto) consumingModuleDo(Func &&F) {
-    auto Lock = TSCtx.getLock();
-    return F(std::move(M));
+    return TSCtx.withContextDo([&](LLVMContext *) {
+      assert(M && "Can not call on null module");
+      return F(std::move(M));
+    });
   }
 
   /// Get a raw pointer to the contained module without locking the context.
@@ -169,6 +153,12 @@ class ThreadSafeModule {
 using GVPredicate = std::function<bool(const GlobalValue &)>;
 using GVModifier = std::function<void(GlobalValue &)>;
 
+/// Clones teh given module onto the given context.
+LLVM_ABI ThreadSafeModule
+cloneToContext(const ThreadSafeModule &TSMW, ThreadSafeContext TSCtx,
+               GVPredicate ShouldCloneDef = GVPredicate(),
+               GVModifier UpdateClonedDefSource = GVModifier());
+
 /// Clones the given module on to a new context.
 LLVM_ABI ThreadSafeModule cloneToNewContext(
     const ThreadSafeModule &TSMW, GVPredicate ShouldCloneDef = GVPredicate(),
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
index 9dfbd3cb68928..e44612af071bc 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
+++ b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
@@ -14,8 +14,10 @@
 #ifndef LLVM_FRONTEND_HLSL_HLSLROOTSIGNATURE_H
 #define LLVM_FRONTEND_HLSL_HLSLROOTSIGNATURE_H
 
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DXILABI.h"
+#include "llvm/Support/raw_ostream.h"
 #include <limits>
 #include <variant>
 
@@ -23,126 +25,6 @@ namespace llvm {
 namespace hlsl {
 namespace rootsig {
 
-// Definition of the various enumerations and flags. The definitions of all
-// values here correspond to their description in the d3d12.h header and are
-// carried over from their values in DXC. For reference:
-// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/
-
-enum class RootFlags : uint32_t {
-  None = 0,
-  AllowInputAssemblerInputLayout = 0x1,
-  DenyVertexShaderRootAccess = 0x2,
-  DenyHullShaderRootAccess = 0x4,
-  DenyDomainShaderRootAccess = 0x8,
-  DenyGeometryShaderRootAccess = 0x10,
-  DenyPixelShaderRootAccess = 0x20,
-  AllowStreamOutput = 0x40,
-  LocalRootSignature = 0x80,
-  DenyAmplificationShaderRootAccess = 0x100,
-  DenyMeshShaderRootAccess = 0x200,
-  CBVSRVUAVHeapDirectlyIndexed = 0x400,
-  SamplerHeapDirectlyIndexed = 0x800,
-  ValidFlags = 0x00000fff
-};
-
-enum class RootDescriptorFlags : unsigned {
-  None = 0,
-  DataVolatile = 0x2,
-  DataStaticWhileSetAtExecute = 0x4,
-  DataStatic = 0x8,
-  ValidFlags = 0xe,
-};
-
-enum class DescriptorRangeFlags : unsigned {
-  None = 0,
-  DescriptorsVolatile = 0x1,
-  DataVolatile = 0x2,
-  DataStaticWhileSetAtExecute = 0x4,
-  DataStatic = 0x8,
-  DescriptorsStaticKeepingBufferBoundsChecks = 0x10000,
-  ValidFlags = 0x1000f,
-  ValidSamplerFlags = DescriptorsVolatile,
-};
-
-enum class ShaderVisibility {
-  All = 0,
-  Vertex = 1,
-  Hull = 2,
-  Domain = 3,
-  Geometry = 4,
-  Pixel = 5,
-  Amplification = 6,
-  Mesh = 7,
-};
-
-// D3D12_FILTER enumeration:
-// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_filter
-enum class SamplerFilter {
-  MinMagMipPoint = 0,
-  MinMagPointMipLinear = 0x1,
-  MinPointMagLinearMipPoint = 0x4,
-  MinPointMagMipLinear = 0x5,
-  MinLinearMagMipPoint = 0x10,
-  MinLinearMagPointMipLinear = 0x11,
-  MinMagLinearMipPoint = 0x14,
-  MinMagMipLinear = 0x15,
-  Anisotropic = 0x55,
-  ComparisonMinMagMipPoint = 0x80,
-  ComparisonMinMagPointMipLinear = 0x81,
-  ComparisonMinPointMagLinearMipPoint = 0x84,
-  ComparisonMinPointMagMipLinear = 0x85,
-  ComparisonMinLinearMagMipPoint = 0x90,
-  ComparisonMinLinearMagPointMipLinear = 0x91,
-  ComparisonMinMagLinearMipPoint = 0x94,
-  ComparisonMinMagMipLinear = 0x95,
-  ComparisonAnisotropic = 0xd5,
-  MinimumMinMagMipPoint = 0x100,
-  MinimumMinMagPointMipLinear = 0x101,
-  MinimumMinPointMagLinearMipPoint = 0x104,
-  MinimumMinPointMagMipLinear = 0x105,
-  MinimumMinLinearMagMipPoint = 0x110,
-  MinimumMinLinearMagPointMipLinear = 0x111,
-  MinimumMinMagLinearMipPoint = 0x114,
-  MinimumMinMagMipLinear = 0x115,
-  MinimumAnisotropic = 0x155,
-  MaximumMinMagMipPoint = 0x180,
-  MaximumMinMagPointMipLinear = 0x181,
-  MaximumMinPointMagLinearMipPoint = 0x184,
-  MaximumMinPointMagMipLinear = 0x185,
-  MaximumMinLinearMagMipPoint = 0x190,
-  MaximumMinLinearMagPointMipLinear = 0x191,
-  MaximumMinMagLinearMipPoint = 0x194,
-  MaximumMinMagMipLinear = 0x195,
-  MaximumAnisotropic = 0x1d5
-};
-
-enum class TextureAddressMode {
-  Wrap = 1,
-  Mirror = 2,
-  Clamp = 3,
-  Border = 4,
-  MirrorOnce = 5
-};
-
-enum class ComparisonFunc : unsigned {
-  Never = 1,
-  Less = 2,
-  Equal = 3,
-  LessEqual = 4,
-  Greater = 5,
-  NotEqual = 6,
-  GreaterEqual = 7,
-  Always = 8
-};
-
-enum class StaticBorderColor {
-  TransparentBlack = 0,
-  OpaqueBlack = 1,
-  OpaqueWhite = 2,
-  OpaqueBlackUint = 3,
-  OpaqueWhiteUint = 4
-};
-
 // Definitions of the in-memory data layout structures
 
 // Models the different registers: bReg | tReg | uReg | sReg
@@ -157,7 +39,7 @@ struct RootConstants {
   uint32_t Num32BitConstants;
   Register Reg;
   uint32_t Space = 0;
-  ShaderVisibility Visibility = ShaderVisibility::All;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
 };
 
 enum class DescriptorType : uint8_t { SRV = 0, UAV, CBuffer };
@@ -166,17 +48,24 @@ struct RootDescriptor {
   DescriptorType Type;
   Register Reg;
   uint32_t Space = 0;
-  ShaderVisibility Visibility = ShaderVisibility::All;
-  RootDescriptorFlags Flags;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
+  dxbc::RootDescriptorFlags Flags;
 
-  void setDefaultFlags() {
+  void setDefaultFlags(dxbc::RootSignatureVersion Version) {
+    if (Version == dxbc::RootSignatureVersion::V1_0) {
+      Flags = dxbc::RootDescriptorFlags::DataVolatile;
+      return;
+    }
+
+    assert(Version == llvm::dxbc::RootSignatureVersion::V1_1 &&
+           "Specified an invalid root signature version");
     switch (Type) {
     case DescriptorType::CBuffer:
     case DescriptorType::SRV:
-      Flags = RootDescriptorFlags::DataStaticWhileSetAtExecute;
+      Flags = dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute;
       break;
     case DescriptorType::UAV:
-      Flags = RootDescriptorFlags::DataVolatile;
+      Flags = dxbc::RootDescriptorFlags::DataVolatile;
       break;
     }
   }
@@ -184,7 +73,7 @@ struct RootDescriptor {
 
 // Models the end of a descriptor table and stores its visibility
 struct DescriptorTable {
-  ShaderVisibility Visibility = ShaderVisibility::All;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
   // Denotes that the previous NumClauses in the RootElement array
   // are the clauses in the table.
   uint32_t NumClauses = 0;
@@ -200,19 +89,28 @@ struct DescriptorTableClause {
   uint32_t NumDescriptors = 1;
   uint32_t Space = 0;
   uint32_t Offset = DescriptorTableOffsetAppend;
-  DescriptorRangeFlags Flags;
+  dxbc::DescriptorRangeFlags Flags;
+
+  void setDefaultFlags(dxbc::RootSignatureVersion Version) {
+    if (Version == dxbc::RootSignatureVersion::V1_0) {
+      Flags = dxbc::DescriptorRangeFlags::DescriptorsVolatile;
+      if (Type != ClauseType::Sampler)
+        Flags |= dxbc::DescriptorRangeFlags::DataVolatile;
+      return;
+    }
 
-  void setDefaultFlags() {
+    assert(Version == dxbc::RootSignatureVersion::V1_1 &&
+           "Specified an invalid root signature version");
     switch (Type) {
     case ClauseType::CBuffer:
     case ClauseType::SRV:
-      Flags = DescriptorRangeFlags::DataStaticWhileSetAtExecute;
+      Flags = dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute;
       break;
     case ClauseType::UAV:
-      Flags = DescriptorRangeFlags::DataVolatile;
+      Flags = dxbc::DescriptorRangeFlags::DataVolatile;
       break;
     case ClauseType::Sampler:
-      Flags = DescriptorRangeFlags::None;
+      Flags = dxbc::DescriptorRangeFlags::None;
       break;
     }
   }
@@ -220,18 +118,18 @@ struct DescriptorTableClause {
 
 struct StaticSampler {
   Register Reg;
-  SamplerFilter Filter = SamplerFilter::Anisotropic;
-  TextureAddressMode AddressU = TextureAddressMode::Wrap;
-  TextureAddressMode AddressV = TextureAddressMode::Wrap;
-  TextureAddressMode AddressW = TextureAddressMode::Wrap;
+  dxbc::SamplerFilter Filter = dxbc::SamplerFilter::Anisotropic;
+  dxbc::TextureAddressMode AddressU = dxbc::TextureAddressMode::Wrap;
+  dxbc::TextureAddressMode AddressV = dxbc::TextureAddressMode::Wrap;
+  dxbc::TextureAddressMode AddressW = dxbc::TextureAddressMode::Wrap;
   float MipLODBias = 0.f;
   uint32_t MaxAnisotropy = 16;
-  ComparisonFunc CompFunc = ComparisonFunc::LessEqual;
-  StaticBorderColor BorderColor = StaticBorderColor::OpaqueWhite;
+  dxbc::ComparisonFunc CompFunc = dxbc::ComparisonFunc::LessEqual;
+  dxbc::StaticBorderColor BorderColor = dxbc::StaticBorderColor::OpaqueWhite;
   float MinLOD = 0.f;
   float MaxLOD = std::numeric_limits<float>::max();
   uint32_t Space = 0;
-  ShaderVisibility Visibility = ShaderVisibility::All;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
 };
 
 /// Models RootElement : RootFlags | RootConstants | RootParam
@@ -251,8 +149,23 @@ struct StaticSampler {
 /// RootElements in the array, and it holds a data member for the Visibility
 /// parameter.
 using RootElement =
-    std::variant<RootFlags, RootConstants, RootDescriptor, DescriptorTable,
-                 DescriptorTableClause, StaticSampler>;
+    std::variant<dxbc::RootFlags, RootConstants, RootDescriptor,
+                 DescriptorTable, DescriptorTableClause, StaticSampler>;
+
+/// The following contains the serialization interface for root elements
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const dxbc::RootFlags &Flags);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const RootConstants &Constants);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const DescriptorTableClause &Clause);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const RootDescriptor &Descriptor);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const StaticSampler &StaticSampler);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element);
+
+LLVM_ABI void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements);
 
 } // namespace rootsig
 } // namespace hlsl
diff --git a/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
new file mode 100644
index 0000000000000..0aa122f668ef1
--- /dev/null
+++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
@@ -0,0 +1,56 @@
+//===- RootSignatureMetadata.h - HLSL Root Signature helpers --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains a library for working with HLSL Root Signatures and
+/// their metadata representation.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H
+#define LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H
+
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+
+namespace llvm {
+class LLVMContext;
+class MDNode;
+class Metadata;
+
+namespace hlsl {
+namespace rootsig {
+
+class MetadataBuilder {
+public:
+  MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
+      : Ctx(Ctx), Elements(Elements) {}
+
+  /// Iterates through elements and dispatches onto the correct Build* method
+  ///
+  /// Accumulates the root signature and returns the Metadata node that is just
+  /// a list of all the elements
+  LLVM_ABI MDNode *BuildRootSignature();
+
+private:
+  /// Define the various builders for the different metadata types
+  MDNode *BuildRootFlags(const dxbc::RootFlags &Flags);
+  MDNode *BuildRootConstants(const RootConstants &Constants);
+  MDNode *BuildRootDescriptor(const RootDescriptor &Descriptor);
+  MDNode *BuildDescriptorTable(const DescriptorTable &Table);
+  MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
+  MDNode *BuildStaticSampler(const StaticSampler &Sampler);
+
+  llvm::LLVMContext &Ctx;
+  ArrayRef<RootElement> Elements;
+  SmallVector<Metadata *> GeneratedMetadata;
+};
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h
similarity index 54%
rename from llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
rename to llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h
index 43c8805ce06b6..9b68a524432cc 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
+++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h
@@ -1,4 +1,4 @@
-//===- HLSLRootSignatureUtils.h - HLSL Root Signature helpers -------------===//
+//===- RootSignatureValidations.h - HLSL Root Signature helpers -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,67 +11,33 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
-#define LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
+#ifndef LLVM_FRONTEND_HLSL_ROOTSIGNATUREVALIDATIONS_H
+#define LLVM_FRONTEND_HLSL_ROOTSIGNATUREVALIDATIONS_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IntervalMap.h"
 #include "llvm/Frontend/HLSL/HLSLRootSignature.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
-class LLVMContext;
-class MDNode;
-class Metadata;
-
 namespace hlsl {
 namespace rootsig {
 
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RootFlags &Flags);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const RootConstants &Constants);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const DescriptorTableClause &Clause);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const RootDescriptor &Descriptor);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const StaticSampler &StaticSampler);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element);
-
-LLVM_ABI void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements);
-
-class MetadataBuilder {
-public:
-  MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
-      : Ctx(Ctx), Elements(Elements) {}
-
-  /// Iterates through the elements and dispatches onto the correct Build method
-  ///
-  /// Accumulates the root signature and returns the Metadata node that is just
-  /// a list of all the elements
-  LLVM_ABI MDNode *BuildRootSignature();
-
-private:
-  /// Define the various builders for the different metadata types
-  MDNode *BuildRootFlags(const RootFlags &Flags);
-  MDNode *BuildRootConstants(const RootConstants &Constants);
-  MDNode *BuildRootDescriptor(const RootDescriptor &Descriptor);
-  MDNode *BuildDescriptorTable(const DescriptorTable &Table);
-  MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
-  MDNode *BuildStaticSampler(const StaticSampler &Sampler);
-
-  llvm::LLVMContext &Ctx;
-  ArrayRef<RootElement> Elements;
-  SmallVector<Metadata *> GeneratedMetadata;
-};
+// Basic verification of RootElements
+
+bool verifyRootFlag(uint32_t Flags);
+bool verifyVersion(uint32_t Version);
+bool verifyRegisterValue(uint32_t RegisterValue);
+bool verifyRegisterSpace(uint32_t RegisterSpace);
+bool verifyDescriptorFlag(uint32_t Flags);
+bool verifyRangeType(uint32_t Type);
+bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type,
+                               uint32_t FlagsVal);
+bool verifySamplerFilter(uint32_t Value);
+bool verifyAddress(uint32_t Address);
+bool verifyMipLODBias(float MipLODBias);
+bool verifyMaxAnisotropy(uint32_t MaxAnisotropy);
+bool verifyComparisonFunc(uint32_t ComparisonFunc);
+bool verifyBorderColor(uint32_t BorderColor);
+bool verifyLOD(float LOD);
 
 struct RangeInfo {
   const static uint32_t Unbounded = ~0u;
@@ -83,7 +49,7 @@ struct RangeInfo {
   // Information retained for diagnostics
   llvm::dxil::ResourceClass Class;
   uint32_t Space;
-  ShaderVisibility Visibility;
+  llvm::dxbc::ShaderVisibility Visibility;
 };
 
 class ResourceRange {
@@ -141,4 +107,4 @@ class ResourceRange {
 } // namespace hlsl
 } // namespace llvm
 
-#endif // LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
+#endif // LLVM_FRONTEND_HLSL_ROOTSIGNATUREVALIDATIONS_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h b/llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h
new file mode 100644
index 0000000000000..fbcdd51d7f64e
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h
@@ -0,0 +1,78 @@
+//===- DirectiveNameParser.h  ------------------------------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
+#define LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/Compiler.h"
+
+#include <memory>
+
+namespace llvm::omp {
+/// Parser class for OpenMP directive names. It only recognizes names listed
+/// in OMP.td, in particular it does not recognize Fortran's end-directives
+/// if they are not explicitly listed in OMP.td.
+///
+/// The class itself may be a singleton, once it's constructed it never
+/// changes.
+///
+/// Usage:
+/// {
+///   DirectiveNameParser Parser;   // Could be static const.
+///
+///   DirectiveNameParser::State *S = Parser.initial();
+///   for (StringRef Token : Tokens)
+///     S = Parser.consume(S, Token); // Passing nullptr is ok.
+///
+///   if (S == nullptr) {
+///     // Error: ended up in a state from which there is no possible path
+///     // to a successful parse.
+///   } else if (S->Value == OMPD_unknown) {
+///     // Parsed a sequence of tokens that are not a complete name, but
+///     // parsing more tokens could lead to a successful parse.
+///   } else {
+///     // Success.
+///     ParsedId = S->Value;
+///   }
+/// }
+struct DirectiveNameParser {
+  LLVM_ABI DirectiveNameParser(SourceLanguage L = SourceLanguage::C);
+
+  struct State {
+    Directive Value = Directive::OMPD_unknown;
+
+  private:
+    using TransitionMapTy = StringMap<State>;
+    std::unique_ptr<TransitionMapTy> Transition;
+
+    State *next(StringRef Tok);
+    const State *next(StringRef Tok) const;
+    bool isValid() const {
+      return Value != Directive::OMPD_unknown || !Transition->empty();
+    }
+    friend struct DirectiveNameParser;
+  };
+
+  const State *initial() const { return &InitialState; }
+  LLVM_ABI const State *consume(const State *Current, StringRef Tok) const;
+
+  LLVM_ABI static SmallVector<StringRef> tokenize(StringRef N);
+
+private:
+  void insertName(StringRef Name, Directive D);
+  State *insertTransition(State *From, StringRef Tok);
+
+  State InitialState;
+};
+} // namespace llvm::omp
+
+#endif // LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h
index 60854b17094bf..b9a73b3eb5fc2 100644
--- a/llvm/include/llvm/IR/Argument.h
+++ b/llvm/include/llvm/IR/Argument.h
@@ -78,6 +78,9 @@ class Argument final : public Value {
   /// Return true if this argument has the byval attribute.
   LLVM_ABI bool hasByValAttr() const;
 
+  /// Return true if this argument has the dead_on_return attribute.
+  LLVM_ABI bool hasDeadOnReturnAttr() const;
+
   /// Return true if this argument has the byref attribute.
   LLVM_ABI bool hasByRefAttr() const;
 
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 0bcd15eeed879..112853965407c 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -198,6 +198,9 @@ def NoFree : EnumAttr<"nofree", IntersectAnd, [FnAttr, ParamAttr]>;
 /// Argument is dead if the call unwinds.
 def DeadOnUnwind : EnumAttr<"dead_on_unwind", IntersectAnd, [ParamAttr]>;
 
+/// Argument is dead upon function return.
+def DeadOnReturn : EnumAttr<"dead_on_return", IntersectAnd, [ParamAttr]>;
+
 /// Disable implicit floating point insts.
 def NoImplicitFloat : EnumAttr<"noimplicitfloat", IntersectPreserve, [FnAttr]>;
 
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index 99f7491b1b9b5..25cbc38b61272 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -901,7 +901,10 @@ namespace llvm {
     /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually
     /// leads to a \a DISubprogram.
     LLVM_ABI DILabel *createLabel(DIScope *Scope, StringRef Name, DIFile *File,
-                                  unsigned LineNo, bool AlwaysPreserve = false);
+                                  unsigned LineNo, unsigned Column,
+                                  bool IsArtificial,
+                                  std::optional<unsigned> CoroSuspendIdx,
+                                  bool AlwaysPreserve = false);
 
     /// Create a new descriptor for a parameter variable.
     ///
@@ -950,6 +953,8 @@ namespace llvm {
     /// \param Annotations   Attribute Annotations.
     /// \param TargetFuncName The name of the target function if this is
     ///                       a trampoline.
+    /// \param UseKeyInstructions Instruct DWARF emission to interpret Key
+    /// Instructions metadata on instructions to determine is_stmt placement.
     LLVM_ABI DISubprogram *createFunction(
         DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
         unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine,
@@ -957,7 +962,8 @@ namespace llvm {
         DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
         DITemplateParameterArray TParams = nullptr,
         DISubprogram *Decl = nullptr, DITypeArray ThrownTypes = nullptr,
-        DINodeArray Annotations = nullptr, StringRef TargetFuncName = "");
+        DINodeArray Annotations = nullptr, StringRef TargetFuncName = "",
+        bool UseKeyInstructions = false);
 
     /// Identical to createFunction,
     /// except that the resulting DbgNode is meant to be RAUWed.
@@ -988,15 +994,15 @@ namespace llvm {
     /// \param SPFlags       Additional flags specific to subprograms.
     /// \param TParams       Function template parameters.
     /// \param ThrownTypes   Exception types this function may throw.
-    LLVM_ABI DISubprogram *
-    createMethod(DIScope *Scope, StringRef Name, StringRef LinkageName,
-                 DIFile *File, unsigned LineNo, DISubroutineType *Ty,
-                 unsigned VTableIndex = 0, int ThisAdjustment = 0,
-                 DIType *VTableHolder = nullptr,
-                 DINode::DIFlags Flags = DINode::FlagZero,
-                 DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
-                 DITemplateParameterArray TParams = nullptr,
-                 DITypeArray ThrownTypes = nullptr);
+    /// \param UseKeyInstructions Enable Key Instructions debug info.
+    LLVM_ABI DISubprogram *createMethod(
+        DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
+        unsigned LineNo, DISubroutineType *Ty, unsigned VTableIndex = 0,
+        int ThisAdjustment = 0, DIType *VTableHolder = nullptr,
+        DINode::DIFlags Flags = DINode::FlagZero,
+        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
+        DITemplateParameterArray TParams = nullptr,
+        DITypeArray ThrownTypes = nullptr, bool UseKeyInstructions = false);
 
     /// Create common block entry for a Fortran common block.
     /// \param Scope       Scope of this common block.
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index f80e44ce3abbc..9345f95015301 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -870,22 +870,22 @@ class DIBasicType : public DIType {
                    SizeInBits, AlignInBits, Encoding, NumExtraInhabitants,
                    Flags, Storage, ShouldCreate);
   }
-  LLVM_ABI static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
-                                       MDString *Name, uint64_t SizeInBits,
-                                       uint32_t AlignInBits, unsigned Encoding,
-                                       uint32_t NumExtraInhabitants,
-                                       DIFlags Flags, StorageType Storage,
-                                       bool ShouldCreate = true) {
+  static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
+                              MDString *Name, uint64_t SizeInBits,
+                              uint32_t AlignInBits, unsigned Encoding,
+                              uint32_t NumExtraInhabitants, DIFlags Flags,
+                              StorageType Storage, bool ShouldCreate = true) {
     auto *SizeInBitsNode = ConstantAsMetadata::get(
         ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
     return getImpl(Context, Tag, Name, SizeInBitsNode, AlignInBits, Encoding,
                    NumExtraInhabitants, Flags, Storage, ShouldCreate);
   }
-  static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
-                              MDString *Name, Metadata *SizeInBits,
-                              uint32_t AlignInBits, unsigned Encoding,
-                              uint32_t NumExtraInhabitants, DIFlags Flags,
-                              StorageType Storage, bool ShouldCreate = true);
+  LLVM_ABI static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
+                                       MDString *Name, Metadata *SizeInBits,
+                                       uint32_t AlignInBits, unsigned Encoding,
+                                       uint32_t NumExtraInhabitants,
+                                       DIFlags Flags, StorageType Storage,
+                                       bool ShouldCreate = true);
 
   TempDIBasicType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getRawName(),
@@ -1005,7 +1005,7 @@ class DIFixedPointType : public DIBasicType {
                    SizeInBits, AlignInBits, Encoding, Flags, Kind, Factor,
                    Numerator, Denominator, Storage, ShouldCreate);
   }
-  LLVM_ABI static DIFixedPointType *
+  static DIFixedPointType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
           uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
           DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
@@ -1016,7 +1016,7 @@ class DIFixedPointType : public DIBasicType {
                    Flags, Kind, Factor, Numerator, Denominator, Storage,
                    ShouldCreate);
   }
-  static DIFixedPointType *
+  LLVM_ABI static DIFixedPointType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
           Metadata *SizeInBits, uint32_t AlignInBits, unsigned Encoding,
           DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
@@ -1127,23 +1127,23 @@ class DIStringType : public DIType {
                    StringLength, StrLenExp, StrLocationExp, SizeInBitsNode,
                    AlignInBits, Encoding, Storage, ShouldCreate);
   }
-  LLVM_ABI static DIStringType *
-  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
-          Metadata *StringLength, Metadata *StrLenExp, Metadata *StrLocationExp,
-          uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
-          StorageType Storage, bool ShouldCreate = true) {
+  static DIStringType *getImpl(LLVMContext &Context, unsigned Tag,
+                               MDString *Name, Metadata *StringLength,
+                               Metadata *StrLenExp, Metadata *StrLocationExp,
+                               uint64_t SizeInBits, uint32_t AlignInBits,
+                               unsigned Encoding, StorageType Storage,
+                               bool ShouldCreate = true) {
     auto *SizeInBitsNode = ConstantAsMetadata::get(
         ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
     return getImpl(Context, Tag, Name, StringLength, StrLenExp, StrLocationExp,
                    SizeInBitsNode, AlignInBits, Encoding, Storage,
                    ShouldCreate);
   }
-  static DIStringType *getImpl(LLVMContext &Context, unsigned Tag,
-                               MDString *Name, Metadata *StringLength,
-                               Metadata *StrLenExp, Metadata *StrLocationExp,
-                               Metadata *SizeInBits, uint32_t AlignInBits,
-                               unsigned Encoding, StorageType Storage,
-                               bool ShouldCreate = true);
+  LLVM_ABI static DIStringType *
+  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
+          Metadata *StringLength, Metadata *StrLenExp, Metadata *StrLocationExp,
+          Metadata *SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDIStringType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getRawName(),
@@ -2217,7 +2217,7 @@ class DILocalScope : public DIScope {
   }
 };
 
-/// Subprogram description.
+/// Subprogram description. Uses SubclassData1.
 class DISubprogram : public DILocalScope {
   friend class LLVMContextImpl;
   friend class MDNode;
@@ -2264,7 +2264,8 @@ class DISubprogram : public DILocalScope {
 
   DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line,
                unsigned ScopeLine, unsigned VirtualIndex, int ThisAdjustment,
-               DIFlags Flags, DISPFlags SPFlags, ArrayRef<Metadata *> Ops);
+               DIFlags Flags, DISPFlags SPFlags, bool UsesKeyInstructions,
+               ArrayRef<Metadata *> Ops);
   ~DISubprogram() = default;
 
   static DISubprogram *
@@ -2276,15 +2277,17 @@ class DISubprogram : public DILocalScope {
           DITemplateParameterArray TemplateParams, DISubprogram *Declaration,
           DINodeArray RetainedNodes, DITypeArray ThrownTypes,
           DINodeArray Annotations, StringRef TargetFuncName,
-          StorageType Storage, bool ShouldCreate = true) {
+          bool UsesKeyInstructions, StorageType Storage,
+          bool ShouldCreate = true) {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name),
                    getCanonicalMDString(Context, LinkageName), File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags, Unit, TemplateParams.get(), Declaration,
                    RetainedNodes.get(), ThrownTypes.get(), Annotations.get(),
                    getCanonicalMDString(Context, TargetFuncName),
-                   Storage, ShouldCreate);
+                   UsesKeyInstructions, Storage, ShouldCreate);
   }
+
   LLVM_ABI static DISubprogram *
   getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
           MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
@@ -2292,8 +2295,8 @@ class DISubprogram : public DILocalScope {
           int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, Metadata *Unit,
           Metadata *TemplateParams, Metadata *Declaration,
           Metadata *RetainedNodes, Metadata *ThrownTypes, Metadata *Annotations,
-          MDString *TargetFuncName, StorageType Storage,
-          bool ShouldCreate = true);
+          MDString *TargetFuncName, bool UsesKeyInstructions,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDISubprogram cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getLinkageName(),
@@ -2302,7 +2305,7 @@ class DISubprogram : public DILocalScope {
                         getThisAdjustment(), getFlags(), getSPFlags(),
                         getUnit(), getTemplateParams(), getDeclaration(),
                         getRetainedNodes(), getThrownTypes(), getAnnotations(),
-                        getTargetFuncName());
+                        getTargetFuncName(), getKeyInstructionsEnabled());
   }
 
 public:
@@ -2315,10 +2318,11 @@ class DISubprogram : public DILocalScope {
        DITemplateParameterArray TemplateParams = nullptr,
        DISubprogram *Declaration = nullptr, DINodeArray RetainedNodes = nullptr,
        DITypeArray ThrownTypes = nullptr, DINodeArray Annotations = nullptr,
-       StringRef TargetFuncName = ""),
+       StringRef TargetFuncName = "", bool UsesKeyInstructions = false),
       (Scope, Name, LinkageName, File, Line, Type, ScopeLine, ContainingType,
        VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams,
-       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName))
+       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName,
+       UsesKeyInstructions))
 
   DEFINE_MDNODE_GET(
       DISubprogram,
@@ -2328,10 +2332,12 @@ class DISubprogram : public DILocalScope {
        DIFlags Flags, DISPFlags SPFlags, Metadata *Unit,
        Metadata *TemplateParams = nullptr, Metadata *Declaration = nullptr,
        Metadata *RetainedNodes = nullptr, Metadata *ThrownTypes = nullptr,
-       Metadata *Annotations = nullptr, MDString *TargetFuncName = nullptr),
+       Metadata *Annotations = nullptr, MDString *TargetFuncName = nullptr,
+       bool UsesKeyInstructions = false),
       (Scope, Name, LinkageName, File, Line, Type, ScopeLine, ContainingType,
        VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams,
-       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName))
+       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName,
+       UsesKeyInstructions))
 
   TempDISubprogram clone() const { return cloneImpl(); }
 
@@ -2342,6 +2348,8 @@ class DISubprogram : public DILocalScope {
     return NewSP;
   }
 
+  bool getKeyInstructionsEnabled() const { return SubclassData1; }
+
 public:
   unsigned getLine() const { return Line; }
   unsigned getVirtuality() const { return getSPFlags() & SPFlagVirtuality; }
@@ -4104,35 +4112,49 @@ class DILabel : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  DILabel(LLVMContext &C, StorageType Storage, unsigned Line,
+  unsigned Column;
+  std::optional<unsigned> CoroSuspendIdx;
+  bool IsArtificial;
+
+  DILabel(LLVMContext &C, StorageType Storage, unsigned Line, unsigned Column,
+          bool IsArtificial, std::optional<unsigned> CoroSuspendIdx,
           ArrayRef<Metadata *> Ops);
   ~DILabel() = default;
 
   static DILabel *getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name,
-                          DIFile *File, unsigned Line, StorageType Storage,
-                          bool ShouldCreate = true) {
+                          DIFile *File, unsigned Line, unsigned Column,
+                          bool IsArtificial,
+                          std::optional<unsigned> CoroSuspendIdx,
+                          StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name), File,
-                   Line, Storage, ShouldCreate);
+                   Line, Column, IsArtificial, CoroSuspendIdx, Storage,
+                   ShouldCreate);
   }
-  LLVM_ABI static DILabel *getImpl(LLVMContext &Context, Metadata *Scope,
-                                   MDString *Name, Metadata *File,
-                                   unsigned Line, StorageType Storage,
-                                   bool ShouldCreate = true);
+  LLVM_ABI static DILabel *
+  getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, Metadata *File,
+          unsigned Line, unsigned Column, bool IsArtificial,
+          std::optional<unsigned> CoroSuspendIdx, StorageType Storage,
+          bool ShouldCreate = true);
 
   TempDILabel cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getFile(),
-                        getLine());
+                        getLine(), getColumn(), isArtificial(),
+                        getCoroSuspendIdx());
   }
 
 public:
   DEFINE_MDNODE_GET(DILabel,
                     (DILocalScope * Scope, StringRef Name, DIFile *File,
-                     unsigned Line),
-                    (Scope, Name, File, Line))
+                     unsigned Line, unsigned Column, bool IsArtificial,
+                     std::optional<unsigned> CoroSuspendIdx),
+                    (Scope, Name, File, Line, Column, IsArtificial,
+                     CoroSuspendIdx))
   DEFINE_MDNODE_GET(DILabel,
                     (Metadata * Scope, MDString *Name, Metadata *File,
-                     unsigned Line),
-                    (Scope, Name, File, Line))
+                     unsigned Line, unsigned Column, bool IsArtificial,
+                     std::optional<unsigned> CoroSuspendIdx),
+                    (Scope, Name, File, Line, Column, IsArtificial,
+                     CoroSuspendIdx))
 
   TempDILabel clone() const { return cloneImpl(); }
 
@@ -4143,8 +4165,11 @@ class DILabel : public DINode {
     return cast_or_null<DILocalScope>(getRawScope());
   }
   unsigned getLine() const { return SubclassData32; }
+  unsigned getColumn() const { return Column; }
   StringRef getName() const { return getStringOperand(1); }
   DIFile *getFile() const { return cast_or_null<DIFile>(getRawFile()); }
+  bool isArtificial() const { return IsArtificial; }
+  std::optional<unsigned> getCoroSuspendIdx() const { return CoroSuspendIdx; }
 
   Metadata *getRawScope() const { return getOperand(0); }
   MDString *getRawName() const { return getOperandAs<MDString>(1); }
diff --git a/llvm/include/llvm/IR/DebugLoc.h b/llvm/include/llvm/IR/DebugLoc.h
index 999e03b6374a5..6d79aa6b2aa01 100644
--- a/llvm/include/llvm/IR/DebugLoc.h
+++ b/llvm/include/llvm/IR/DebugLoc.h
@@ -27,6 +27,21 @@ namespace llvm {
   class Function;
 
 #if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+  struct DbgLocOrigin {
+    static constexpr unsigned long MaxDepth = 16;
+    using StackTracesTy =
+        SmallVector<std::pair<int, std::array<void *, MaxDepth>>, 0>;
+    StackTracesTy StackTraces;
+    DbgLocOrigin(bool ShouldCollectTrace);
+    void addTrace();
+    const StackTracesTy &getOriginStackTraces() const { return StackTraces; };
+  };
+#else
+  struct DbgLocOrigin {
+    DbgLocOrigin(bool) {}
+  };
+#endif
   // Used to represent different "kinds" of DebugLoc, expressing that the
   // instruction it is part of is either normal and should contain a valid
   // DILocation, or otherwise describing the reason why the instruction does
@@ -55,22 +70,29 @@ namespace llvm {
     Temporary
   };
 
-  // Extends TrackingMDNodeRef to also store a DebugLocKind, allowing Debugify
-  // to ignore intentionally-empty DebugLocs.
-  class DILocAndCoverageTracking : public TrackingMDNodeRef {
+  // Extends TrackingMDNodeRef to also store a DebugLocKind and Origin,
+  // allowing Debugify to ignore intentionally-empty DebugLocs and display the
+  // code responsible for generating unintentionally-empty DebugLocs.
+  // Currently we only need to track the Origin of this DILoc when using a
+  // DebugLoc that is not annotated (i.e. has DebugLocKind::Normal) and has a
+  // null DILocation, so only collect the origin stacktrace in those cases.
+  class DILocAndCoverageTracking : public TrackingMDNodeRef,
+                                   public DbgLocOrigin {
   public:
     DebugLocKind Kind;
     // Default constructor for empty DebugLocs.
     DILocAndCoverageTracking()
-        : TrackingMDNodeRef(nullptr), Kind(DebugLocKind::Normal) {}
-    // Valid or nullptr MDNode*, normal DebugLocKind.
+        : TrackingMDNodeRef(nullptr), DbgLocOrigin(true),
+          Kind(DebugLocKind::Normal) {}
+    // Valid or nullptr MDNode*, no annotative DebugLocKind.
     DILocAndCoverageTracking(const MDNode *Loc)
-        : TrackingMDNodeRef(const_cast<MDNode *>(Loc)),
+        : TrackingMDNodeRef(const_cast<MDNode *>(Loc)), DbgLocOrigin(!Loc),
           Kind(DebugLocKind::Normal) {}
     LLVM_ABI DILocAndCoverageTracking(const DILocation *Loc);
     // Explicit DebugLocKind, which always means a nullptr MDNode*.
     DILocAndCoverageTracking(DebugLocKind Kind)
-        : TrackingMDNodeRef(nullptr), Kind(Kind) {}
+        : TrackingMDNodeRef(nullptr),
+          DbgLocOrigin(Kind == DebugLocKind::Normal), Kind(Kind) {}
   };
   template <> struct simplify_type<DILocAndCoverageTracking> {
     using SimpleType = MDNode *;
@@ -187,6 +209,19 @@ namespace llvm {
 #endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
     }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+    const DbgLocOrigin::StackTracesTy &getOriginStackTraces() const {
+      return Loc.getOriginStackTraces();
+    }
+    DebugLoc getCopied() const {
+      DebugLoc NewDL = *this;
+      NewDL.Loc.addTrace();
+      return NewDL;
+    }
+#else
+    DebugLoc getCopied() const { return *this; }
+#endif
+
     /// Get the underlying \a DILocation.
     ///
     /// \pre !*this or \c isa<DILocation>(getAsMDNode()).
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 8fe610835dca5..7c600e762a451 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -960,7 +960,7 @@ class IRBuilderBase {
                                               const Twine &Name = "");
 
   /// Create a call to llvm.vscale.<Ty>().
-  LLVM_ABI Value *CreateVScale(Type *Ty, const Twine &Name = "") {
+  Value *CreateVScale(Type *Ty, const Twine &Name = "") {
     return CreateIntrinsic(Intrinsic::vscale, {Ty}, {}, {}, Name);
   }
 
@@ -1575,10 +1575,14 @@ class IRBuilderBase {
     return Accum;
   }
 
-  Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
+  Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "",
+                  bool IsDisjoint = false) {
     if (auto *V = Folder.FoldBinOp(Instruction::Or, LHS, RHS))
       return V;
-    return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
+    return Insert(
+        IsDisjoint ? BinaryOperator::CreateDisjoint(Instruction::Or, LHS, RHS)
+                   : BinaryOperator::CreateOr(LHS, RHS),
+        Name);
   }
 
   Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 8e1ef24226789..ef382a9168f24 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -507,7 +507,7 @@ class Instruction : public User,
   LLVM_ABI bool extractProfTotalWeight(uint64_t &TotalVal) const;
 
   /// Set the debug location information for this instruction.
-  void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); }
+  void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc).getCopied(); }
 
   /// Return the debug location for this node as a DebugLoc.
   const DebugLoc &getDebugLoc() const { return DbgLoc; }
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 0ec5f5163118e..ca6e2128812f7 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -612,7 +612,7 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
                   [IntrReadMem, IntrArgMemOnly]>;
   class AdvSIMD_1Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 
   class AdvSIMD_2Vec_Load_Intrinsic
     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
@@ -626,11 +626,11 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
   class AdvSIMD_2Vec_Store_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                      llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
   class AdvSIMD_2Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                  llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 
   class AdvSIMD_3Vec_Load_Intrinsic
     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
@@ -644,12 +644,12 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
   class AdvSIMD_3Vec_Store_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                      LLVMMatchType<0>, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
   class AdvSIMD_3Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,
                  LLVMMatchType<0>, LLVMMatchType<0>,
                  llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 
   class AdvSIMD_4Vec_Load_Intrinsic
     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
@@ -667,12 +667,12 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                  LLVMMatchType<0>, LLVMMatchType<0>,
                  llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
   class AdvSIMD_4Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                  LLVMMatchType<0>, LLVMMatchType<0>,
                  llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 }
 
 // Memory ops
@@ -761,7 +761,7 @@ let TargetPrefix = "aarch64" in {
   class RNDR_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>;
   class FPMR_Set_Intrinsic
-    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrInaccessibleMemOnly]>; 
+    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>;
 }
 
 // FP environment registers.
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 6f974c97361de..16885f331e9dd 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2996,6 +2996,12 @@ class AMDGPULoadIntrinsic<LLVMType ptr_ty>:
 
 def int_amdgcn_global_load_tr_b64  : AMDGPULoadIntrinsic<global_ptr_ty>;
 def int_amdgcn_global_load_tr_b128 : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_global_load_tr4_b64 : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_global_load_tr6_b96 : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_ds_load_tr8_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
+def int_amdgcn_ds_load_tr16_b128   : AMDGPULoadIntrinsic<local_ptr_ty>;
+def int_amdgcn_ds_load_tr4_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
+def int_amdgcn_ds_load_tr6_b96     : AMDGPULoadIntrinsic<local_ptr_ty>;
 def int_amdgcn_ds_read_tr4_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
 def int_amdgcn_ds_read_tr6_b96     : AMDGPULoadIntrinsic<local_ptr_ty>;
 def int_amdgcn_ds_read_tr8_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
@@ -3396,6 +3402,12 @@ def int_amdgcn_cvt_f32_fp8 : ClangBuiltin<"__builtin_amdgcn_cvt_f32_fp8">,
             [llvm_i32_ty, llvm_i32_ty],
             [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
+// llvm.amdgcn.cvt.f32.fp8.e5m3 float vdst, int srcA, imm byte_sel [0..3]
+def int_amdgcn_cvt_f32_fp8_e5m3 : ClangBuiltin<"__builtin_amdgcn_cvt_f32_fp8_e5m3">,
+  DefaultAttrsIntrinsic<[llvm_float_ty],
+            [llvm_i32_ty, llvm_i32_ty],
+            [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
 // llvm.amdgcn.cvt.pk.f32.bf8 float2 vdst, int srcA, imm word_sel
 // word_sel = 1 selects 2 high bytes, 0 selects 2 low bytes.
 def int_amdgcn_cvt_pk_f32_bf8 : ClangBuiltin<"__builtin_amdgcn_cvt_pk_f32_bf8">,
@@ -3494,6 +3506,40 @@ def int_amdgcn_ashr_pk_u8_i32 : ClangBuiltin<"__builtin_amdgcn_ashr_pk_u8_i32">,
   DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
+//===----------------------------------------------------------------------===//
+// gfx1250 intrinsics
+// ===----------------------------------------------------------------------===//
+
+def int_amdgcn_ds_atomic_async_barrier_arrive_b64 :
+  ClangBuiltin<"__builtin_amdgcn_ds_atomic_async_barrier_arrive_b64">,
+  Intrinsic<[], [local_ptr_ty],
+            // Atomically updates LDS and also ASYNC_CNT which is modeled as InaccessibleMem.
+            [IntrConvergent, IntrWillReturn, IntrInaccessibleMemOrArgMemOnly],
+            "", [SDNPMemOperand]>;
+
+def int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 :
+  ClangBuiltin<"__builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64">,
+  Intrinsic<[llvm_i64_ty], [local_ptr_ty, llvm_i64_ty],
+            [IntrConvergent, IntrWillReturn, IntrArgMemOnly, NoCapture<ArgIndex<0>>],
+            "", [SDNPMemOperand]>;
+
+def int_amdgcn_s_monitor_sleep :
+  ClangBuiltin<"__builtin_amdgcn_s_monitor_sleep">,
+  DefaultAttrsIntrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem,
+                                            IntrHasSideEffects]>;
+
+// llvm.amdgcn.cvt.f16.fp8 half vdst, int srcA, imm byte_sel [0..3]
+def int_amdgcn_cvt_f16_fp8 : ClangBuiltin<"__builtin_amdgcn_cvt_f16_fp8">,
+  DefaultAttrsIntrinsic<[llvm_half_ty],
+            [llvm_i32_ty, llvm_i32_ty],
+            [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
+// llvm.amdgcn.cvt.f16.bf8 half vdst, int srcA, imm byte_sel [0..3]
+def int_amdgcn_cvt_f16_bf8 : ClangBuiltin<"__builtin_amdgcn_cvt_f16_bf8">,
+  DefaultAttrsIntrinsic<[llvm_half_ty],
+            [llvm_i32_ty, llvm_i32_ty],
+            [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
 //===----------------------------------------------------------------------===//
 // Special Intrinsics for backend internal use only. No frontend
 // should emit calls to these.
@@ -3540,6 +3586,41 @@ def int_amdgcn_fdiv_fast : DefaultAttrsIntrinsic<
   [IntrNoMem, IntrSpeculatable]
 >;
 
+class AMDGPUTensorLoadStore:
+  Intrinsic<
+    [],
+    [llvm_v4i32_ty, // D# group 0
+     llvm_v8i32_ty, // D# group 1
+     llvm_v4i32_ty, // D# group 2
+     llvm_v4i32_ty, // D# group 3
+     llvm_i32_ty],  // cachepolicy:
+                    //   bits [0-2] = th
+                    //   bits [3-4] = scope
+    [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<4>>, IntrWillReturn, IntrConvergent, IntrNoCallback, IntrNoFree],
+    "", [SDNPMemOperand]
+  >;
+
+class AMDGPUTensorLoadStoreD2:
+  Intrinsic<
+    [],
+    [llvm_v4i32_ty,  // D# group 0
+     llvm_v8i32_ty,  // D# group 1
+     llvm_i32_ty],   // cachepolicy:
+                     //   bits [0-2] = th
+                     //   bits [3-4] = scope
+    [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<2>>, IntrWillReturn, IntrConvergent, IntrNoCallback, IntrNoFree],
+    "", [SDNPMemOperand]
+  >;
+
+def int_amdgcn_tensor_load_to_lds :
+  ClangBuiltin<"__builtin_amdgcn_tensor_load_to_lds">, AMDGPUTensorLoadStore;
+def int_amdgcn_tensor_store_from_lds :
+  ClangBuiltin<"__builtin_amdgcn_tensor_store_from_lds">, AMDGPUTensorLoadStore;
+def int_amdgcn_tensor_load_to_lds_d2 :
+  ClangBuiltin<"__builtin_amdgcn_tensor_load_to_lds_d2">, AMDGPUTensorLoadStoreD2;
+def int_amdgcn_tensor_store_from_lds_d2 :
+  ClangBuiltin<"__builtin_amdgcn_tensor_store_from_lds_d2">, AMDGPUTensorLoadStoreD2;
+
 /// Emit an addrspacecast without null pointer checking.
 /// Should only be inserted by a pass based on analysis of an addrspacecast's src.
 def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic<
diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td
index 3ee69b72cc5cd..1219ce2f86da8 100644
--- a/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -451,9 +451,6 @@ class Neon_3Arg_Long_Intrinsic
                            LLVMTruncatedType<0>],
                           [IntrNoMem]>;
 
-class Neon_1FloatArg_Intrinsic
-  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
-
 class Neon_CvtFxToFP_Intrinsic
   : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
                           [IntrNoMem]>;
@@ -677,14 +674,6 @@ def int_arm_neon_vtbx2 : Neon_Tbl4Arg_Intrinsic;
 def int_arm_neon_vtbx3 : Neon_Tbl5Arg_Intrinsic;
 def int_arm_neon_vtbx4 : Neon_Tbl6Arg_Intrinsic;
 
-// Vector and Scalar Rounding.
-def int_arm_neon_vrintn : Neon_1FloatArg_Intrinsic;
-def int_arm_neon_vrintx : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrinta : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrintz : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrintm : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrintp : Neon_1Arg_Intrinsic;
-
 // De-interleaving vector loads from N-element structures.
 // Source operands are the address and alignment.
 def int_arm_neon_vld1 : DefaultAttrsIntrinsic<
@@ -744,32 +733,33 @@ def int_arm_neon_vld4dup : DefaultAttrsIntrinsic<
 // Interleaving vector stores from N-element structures.
 // Source operands are: the address, the N vectors, and the alignment.
 def int_arm_neon_vst1 : DefaultAttrsIntrinsic<
-    [], [llvm_anyptr_ty, llvm_anyvector_ty, llvm_i32_ty], [IntrArgMemOnly]>;
+    [], [llvm_anyptr_ty, llvm_anyvector_ty, llvm_i32_ty],
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst2 : DefaultAttrsIntrinsic<
     [], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst3 : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst4 : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      LLVMMatchType<1>, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 
 def int_arm_neon_vst1x2 : DefaultAttrsIntrinsic<
     [], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>],
-    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 def int_arm_neon_vst1x3 : DefaultAttrsIntrinsic<
     [], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
-    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 def int_arm_neon_vst1x4 : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      LLVMMatchType<1>],
-    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 
 // Vector store N-element structure from one lane.
 // Source operands are: the address, the N vectors, the lane number, and
@@ -778,17 +768,17 @@ def int_arm_neon_vst2lane : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty,
      llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst3lane : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      llvm_i32_ty, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst4lane : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      LLVMMatchType<1>, llvm_i32_ty, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 
 // Vector bitwise select.
 def int_arm_neon_vbsl : DefaultAttrsIntrinsic<
diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h
index 1e5b92636d0aa..d25bcafc9cc05 100644
--- a/llvm/include/llvm/IR/LegacyPassManagers.h
+++ b/llvm/include/llvm/IR/LegacyPassManagers.h
@@ -333,8 +333,7 @@ class LLVM_ABI PMDataManager {
   /// Initialize available analysis information.
   void initializeAnalysisInfo() {
     AvailableAnalysis.clear();
-    for (auto &IA : InheritedAnalysis)
-      IA = nullptr;
+    llvm::fill(InheritedAnalysis, nullptr);
   }
 
   // Return true if P preserves high level analysis used by other
diff --git a/llvm/include/llvm/IR/OptBisect.h b/llvm/include/llvm/IR/OptBisect.h
index be6aef3298b23..ea3c1defeb100 100644
--- a/llvm/include/llvm/IR/OptBisect.h
+++ b/llvm/include/llvm/IR/OptBisect.h
@@ -28,8 +28,8 @@ class OptPassGate {
 
   /// IRDescription is a textual description of the IR unit the pass is running
   /// over.
-  virtual bool shouldRunPass(const StringRef PassName,
-                             StringRef IRDescription) {
+  virtual bool shouldRunPass(StringRef PassName,
+                             StringRef IRDescription) const {
     return true;
   }
 
@@ -62,8 +62,8 @@ class LLVM_ABI OptBisect : public OptPassGate {
   /// Most passes should not call this routine directly. Instead, it is called
   /// through helper routines provided by the base classes of the pass. For
   /// instance, function passes should call FunctionPass::skipFunction().
-  bool shouldRunPass(const StringRef PassName,
-                     StringRef IRDescription) override;
+  bool shouldRunPass(StringRef PassName,
+                     StringRef IRDescription) const override;
 
   /// isEnabled() should return true before calling shouldRunPass().
   bool isEnabled() const override { return BisectLimit != Disabled; }
@@ -75,11 +75,11 @@ class LLVM_ABI OptBisect : public OptPassGate {
     LastBisectNum = 0;
   }
 
-  static const int Disabled = std::numeric_limits<int>::max();
+  static constexpr int Disabled = std::numeric_limits<int>::max();
 
 private:
   int BisectLimit = Disabled;
-  int LastBisectNum = 0;
+  mutable int LastBisectNum = 0;
 };
 
 /// Singleton instance of the OptBisect class, so multiple pass managers don't
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 1f86cdfd94e17..ed9b83d5d4361 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -2119,6 +2119,13 @@ m_IntToPtr(const OpTy &Op) {
   return CastOperator_match<OpTy, Instruction::IntToPtr>(Op);
 }
 
+/// Matches any cast or self. Used to ignore casts.
+template <typename OpTy>
+inline match_combine_or<CastInst_match<OpTy, CastInst>, OpTy>
+m_CastOrSelf(const OpTy &Op) {
+  return m_CombineOr(CastInst_match<OpTy, CastInst>(Op), Op);
+}
+
 /// Matches Trunc.
 template <typename OpTy>
 inline CastInst_match<OpTy, TruncInst> m_Trunc(const OpTy &Op) {
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index 5c0e08b03c245..ca56e4aa81575 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -22,11 +22,12 @@
 
 namespace llvm {
 struct MDProfLabels {
-  static const char *BranchWeights;
-  static const char *ValueProfile;
-  static const char *FunctionEntryCount;
-  static const char *SyntheticFunctionEntryCount;
-  static const char *ExpectedBranchWeights;
+  LLVM_ABI static const char *BranchWeights;
+  LLVM_ABI static const char *ValueProfile;
+  LLVM_ABI static const char *FunctionEntryCount;
+  LLVM_ABI static const char *SyntheticFunctionEntryCount;
+  LLVM_ABI static const char *ExpectedBranchWeights;
+  LLVM_ABI static const char *UnknownBranchWeightsMarker;
 };
 
 /// Checks if an Instruction has MD_prof Metadata
@@ -35,6 +36,9 @@ LLVM_ABI bool hasProfMD(const Instruction &I);
 /// Checks if an MDNode contains Branch Weight Metadata
 LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData);
 
+/// Checks if an MDNode contains value profiling Metadata
+LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData);
+
 /// Checks if an instructions has Branch Weight Metadata
 ///
 /// \param I The instruction to check
@@ -140,6 +144,18 @@ LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
 LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
                                bool IsExpected);
 
+/// Specify that the branch weights for this terminator cannot be known at
+/// compile time. This should only be called by passes, and never as a default
+/// behavior in e.g. MDBuilder. The goal is to use this info to validate passes
+/// do not accidentally drop profile info, and this API is called in cases where
+/// the pass explicitly cannot provide that info. Defaulting it in would hide
+/// bugs where the pass forgets to transfer over or otherwise specify profile
+/// info.
+LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I);
+
+LLVM_ABI bool isExplicitlyUnknownBranchWeightsMetadata(const MDNode &MD);
+LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I);
+
 /// Scaling the profile data attached to 'I' using the ratio of S/T.
 LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
 
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
index 50b9791111ce2..131aa8bf7faef 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -37,6 +37,10 @@ template <> struct enum_iteration_traits<RTLIB::Libcall> {
   static constexpr bool is_iterable = true;
 };
 
+template <> struct enum_iteration_traits<RTLIB::LibcallImpl> {
+  static constexpr bool is_iterable = true;
+};
+
 namespace RTLIB {
 
 // Return an iterator over all Libcall values.
@@ -44,6 +48,10 @@ static inline auto libcalls() {
   return enum_seq(static_cast<RTLIB::Libcall>(0), RTLIB::UNKNOWN_LIBCALL);
 }
 
+static inline auto libcall_impls() {
+  return enum_seq(static_cast<RTLIB::LibcallImpl>(1), RTLIB::NumLibcallImpls);
+}
+
 /// A simple container for information about the supported runtime calls.
 struct RuntimeLibcallsInfo {
   explicit RuntimeLibcallsInfo(
@@ -52,7 +60,14 @@ struct RuntimeLibcallsInfo {
       FloatABI::ABIType FloatABI = FloatABI::Default,
       EABI EABIVersion = EABI::Default, StringRef ABIName = "") {
     initSoftFloatCmpLibcallPredicates();
-    initDefaultLibCallImpls();
+
+    // FIXME: The ExceptionModel parameter is to handle the field in
+    // TargetOptions. This interface fails to distinguish the forced disable
+    // case for targets which support exceptions by default. This should
+    // probably be a module flag and removed from TargetOptions.
+    if (ExceptionModel == ExceptionHandling::None)
+      ExceptionModel = TT.getDefaultExceptionHandling();
+
     initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion, ABIName);
   }
 
@@ -77,26 +92,32 @@ struct RuntimeLibcallsInfo {
     return LibcallImpls[Call];
   }
 
-  /// Set the CallingConv that should be used for the specified libcall.
-  // FIXME: This should be a function of RTLIB::LibcallImpl
-  void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
-    LibcallCallingConvs[Call] = CC;
+  /// Set the CallingConv that should be used for the specified libcall
+  /// implementation
+  void setLibcallImplCallingConv(RTLIB::LibcallImpl Call, CallingConv::ID CC) {
+    LibcallImplCallingConvs[Call] = CC;
   }
 
-  /// Get the CallingConv that should be used for the specified libcall.
-  // FIXME: This should be a function of RTLIB::LibcallImpl
+  // FIXME: Remove this wrapper in favor of directly using
+  // getLibcallImplCallingConv
   CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
-    return LibcallCallingConvs[Call];
+    return LibcallImplCallingConvs[LibcallImpls[Call]];
+  }
+
+  /// Get the CallingConv that should be used for the specified libcall.
+  CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const {
+    return LibcallImplCallingConvs[Call];
   }
 
   ArrayRef<RTLIB::LibcallImpl> getLibcallImpls() const {
-    // Trim Unsupported from the start
-    return ArrayRef(LibcallImpls).drop_front();
+    // Trim UNKNOWN_LIBCALL from the back
+    return ArrayRef(LibcallImpls).drop_back();
   }
 
   /// Get the comparison predicate that's to be used to test the result of the
   /// comparison libcall against zero. This should only be used with
   /// floating-point compare libcalls.
+  // FIXME: This should be a function of RTLIB::LibcallImpl
   CmpInst::Predicate
   getSoftFloatCmpLibcallPredicate(RTLIB::Libcall Call) const {
     return SoftFloatCompareLibcallPredicates[Call];
@@ -130,8 +151,9 @@ struct RuntimeLibcallsInfo {
   static_assert(static_cast<int>(CallingConv::C) == 0,
                 "default calling conv should be encoded as 0");
 
-  /// Stores the CallingConv that should be used for each libcall.
-  CallingConv::ID LibcallCallingConvs[RTLIB::UNKNOWN_LIBCALL] = {};
+  /// Stores the CallingConv that should be used for each libcall
+  /// implementation.;
+  CallingConv::ID LibcallImplCallingConvs[RTLIB::NumLibcallImpls] = {};
 
   /// The condition type that should be used to test the result of each of the
   /// soft floating-point comparison libcall against integer zero.
@@ -143,13 +165,15 @@ struct RuntimeLibcallsInfo {
 
   /// Names of concrete implementations of runtime calls. e.g. __ashlsi3 for
   /// SHL_I32
-  static const char *const LibCallImplNames[RTLIB::NumLibcallImpls];
+  LLVM_ABI static const char *const LibCallImplNames[RTLIB::NumLibcallImpls];
 
   /// Map from a concrete LibcallImpl implementation to its RTLIB::Libcall kind.
   static const RTLIB::Libcall ImplToLibcall[RTLIB::NumLibcallImpls];
 
   static bool darwinHasSinCosStret(const Triple &TT) {
-    assert(TT.isOSDarwin() && "should be called with darwin triple");
+    if (!TT.isOSDarwin())
+      return false;
+
     // Don't bother with 32 bit x86.
     if (TT.getArch() == Triple::x86)
       return false;
@@ -171,18 +195,12 @@ struct RuntimeLibcallsInfo {
            (TT.isAndroid() && !TT.isAndroidVersionLT(9));
   }
 
-  void initDefaultLibCallImpls();
-
-  /// Generated by tablegen.
-  void setPPCLibCallNameOverrides();
-
-  /// Generated by tablegen.
-  void setZOSLibCallNameOverrides();
+  LLVM_ABI void initDefaultLibCallImpls();
 
   /// Generated by tablegen.
-  void setWindowsArm64LibCallNameOverrides();
+  void setTargetRuntimeLibcallSets(const Triple &TT);
 
-  void initSoftFloatCmpLibcallPredicates();
+  LLVM_ABI void initSoftFloatCmpLibcallPredicates();
 
   /// Set default libcall names. If a target wants to opt-out of a libcall it
   /// should be placed here.
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td
index 1b54be8dca418..c15ffa0653335 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.td
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.td
@@ -15,6 +15,13 @@ include "llvm/IR/RuntimeLibcallsImpl.td"
 class DuplicateLibcallImplWithPrefix<RuntimeLibcallImpl Impl, string prefix>
     : RuntimeLibcallImpl<Impl.Provides, prefix#Impl.LibCallFuncName>;
 
+/// Libcall Predicates
+def isOSDarwin : RuntimeLibcallPredicate<"TT.isOSDarwin()">;
+
+def darwinHasSinCosStret : RuntimeLibcallPredicate<"darwinHasSinCosStret(TT)">;
+def darwinHasExp10 : RuntimeLibcallPredicate<"darwinHasExp10(TT)">;
+def hasSinCos : RuntimeLibcallPredicate<"hasSinCos(TT)">;
+
 //--------------------------------------------------------------------
 // Declare all kinds of used libcalls
 //--------------------------------------------------------------------
@@ -383,7 +390,7 @@ def MEMCPY_ALIGN_4 : RuntimeLibcall;
 // Define implementation default libcalls
 //--------------------------------------------------------------------
 
-defset list<RuntimeLibcallImpl> DefaultRuntimeLibcallImpls = {
+defset list<RuntimeLibcallImpl> AllDefaultRuntimeLibcallImpls = {
 let IsDefault = true in {
 
 //--------------------------------------------------------------------
@@ -889,7 +896,7 @@ def memset : RuntimeLibcallImpl<MEMSET>;
 def calloc : RuntimeLibcallImpl<CALLOC>;
 
 } // End let IsDefault = true
-} // End defset DefaultRuntimeLibcallImpls
+} // End defset AllDefaultRuntimeLibcallImpls
 
 //--------------------------------------------------------------------
 // Define implementation other libcalls
@@ -915,58 +922,114 @@ def _Unwind_SjLj_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
 // F128 libm Runtime Libcalls
 //===----------------------------------------------------------------------===//
 
-def logf128 : RuntimeLibcallImpl<LOG_F128>;
-def log2f128 : RuntimeLibcallImpl<LOG2_F128>;
-def log10f128 : RuntimeLibcallImpl<LOG10_F128>;
-def expf128 : RuntimeLibcallImpl<EXP_F128>;
-def exp2f128 : RuntimeLibcallImpl<EXP2_F128>;
-def exp10f128 : RuntimeLibcallImpl<EXP10_F128>;
-def sinf128 : RuntimeLibcallImpl<SIN_F128>;
-def cosf128 : RuntimeLibcallImpl<COS_F128>;
-def tanf128 : RuntimeLibcallImpl<TAN_F128>;
-def tanhf128 : RuntimeLibcallImpl<TANH_F128>;
-def sincosf128 : RuntimeLibcallImpl<SINCOS_F128>;
-def powf128 : RuntimeLibcallImpl<POW_F128>;
-def fminf128 : RuntimeLibcallImpl<FMIN_F128>;
-def fmaxf128 : RuntimeLibcallImpl<FMAX_F128>;
-def fmodf128 : RuntimeLibcallImpl<REM_F128>;
-def sqrtf128 : RuntimeLibcallImpl<SQRT_F128>;
-def ceilf128 : RuntimeLibcallImpl<CEIL_F128>;
-def floorf128 : RuntimeLibcallImpl<FLOOR_F128>;
-def truncf128 : RuntimeLibcallImpl<TRUNC_F128>;
-def roundf128 : RuntimeLibcallImpl<ROUND_F128>;
-def lroundf128 : RuntimeLibcallImpl<LROUND_F128>;
-def llroundf128 : RuntimeLibcallImpl<LLROUND_F128>;
-def rintf128 : RuntimeLibcallImpl<RINT_F128>;
-def lrintf128 : RuntimeLibcallImpl<LRINT_F128>;
-def llrintf128 : RuntimeLibcallImpl<LLRINT_F128>;
-def nearbyintf128 : RuntimeLibcallImpl<NEARBYINT_F128>;
-def fmaf128 : RuntimeLibcallImpl<FMA_F128>;
-def frexpf128 : RuntimeLibcallImpl<FREXP_F128>;
-
-def cbrtf128 : RuntimeLibcallImpl<CBRT_F128>;
-def fminimumf128 : RuntimeLibcallImpl<FMINIMUM_F128>;
-def fmaximumf128 : RuntimeLibcallImpl<FMAXIMUM_F128>;
-def fminimum_numf128 : RuntimeLibcallImpl<FMINIMUM_NUM_F128>;
-def fmaximum_numf128 : RuntimeLibcallImpl<FMAXIMUM_NUM_F128>;
-def asinf128 : RuntimeLibcallImpl<ASIN_F128>;
-def acosf128 : RuntimeLibcallImpl<ACOS_F128>;
-def atanf128 : RuntimeLibcallImpl<ATAN_F128>;
-def atan2f128 : RuntimeLibcallImpl<ATAN2_F128>;
-def ldexpf128 : RuntimeLibcallImpl<LDEXP_F128>;
-def roundevenf128 : RuntimeLibcallImpl<ROUNDEVEN_F128>;
-def modff128 : RuntimeLibcallImpl<MODF_F128>;
-def sinhf128 : RuntimeLibcallImpl<SINH_F128>;
-def coshf128 : RuntimeLibcallImpl<COSH_F128>;
-def copysignf128 : RuntimeLibcallImpl<COPYSIGN_F128>;
-
-def __logf128_finite : RuntimeLibcallImpl<LOG_FINITE_F128>;
-def __log2f128_finite : RuntimeLibcallImpl<LOG2_FINITE_F128>;
-def __log10f128_finite : RuntimeLibcallImpl<LOG10_FINITE_F128>;
-def __expf128_finite : RuntimeLibcallImpl<EXP_FINITE_F128>;
-def __exp2f128_finite : RuntimeLibcallImpl<EXP2_FINITE_F128>;
-def __exp10f128_finite : RuntimeLibcallImpl<EXP10_FINITE_F128>;
-def __powf128_finite : RuntimeLibcallImpl<POW_FINITE_F128>;
+defset list<RuntimeLibcallImpl> LibmF128Libcalls = {
+  def logf128 : RuntimeLibcallImpl<LOG_F128>;
+  def log2f128 : RuntimeLibcallImpl<LOG2_F128>;
+  def log10f128 : RuntimeLibcallImpl<LOG10_F128>;
+  def expf128 : RuntimeLibcallImpl<EXP_F128>;
+  def exp2f128 : RuntimeLibcallImpl<EXP2_F128>;
+  def exp10f128 : RuntimeLibcallImpl<EXP10_F128>;
+  def sinf128 : RuntimeLibcallImpl<SIN_F128>;
+  def cosf128 : RuntimeLibcallImpl<COS_F128>;
+  def tanf128 : RuntimeLibcallImpl<TAN_F128>;
+  def tanhf128 : RuntimeLibcallImpl<TANH_F128>;
+  def sincosf128 : RuntimeLibcallImpl<SINCOS_F128>;
+  def powf128 : RuntimeLibcallImpl<POW_F128>;
+  def fminf128 : RuntimeLibcallImpl<FMIN_F128>;
+  def fmaxf128 : RuntimeLibcallImpl<FMAX_F128>;
+  def fmodf128 : RuntimeLibcallImpl<REM_F128>;
+  def sqrtf128 : RuntimeLibcallImpl<SQRT_F128>;
+  def ceilf128 : RuntimeLibcallImpl<CEIL_F128>;
+  def floorf128 : RuntimeLibcallImpl<FLOOR_F128>;
+  def truncf128 : RuntimeLibcallImpl<TRUNC_F128>;
+  def roundf128 : RuntimeLibcallImpl<ROUND_F128>;
+  def lroundf128 : RuntimeLibcallImpl<LROUND_F128>;
+  def llroundf128 : RuntimeLibcallImpl<LLROUND_F128>;
+  def rintf128 : RuntimeLibcallImpl<RINT_F128>;
+  def lrintf128 : RuntimeLibcallImpl<LRINT_F128>;
+  def llrintf128 : RuntimeLibcallImpl<LLRINT_F128>;
+  def nearbyintf128 : RuntimeLibcallImpl<NEARBYINT_F128>;
+  def fmaf128 : RuntimeLibcallImpl<FMA_F128>;
+  def frexpf128 : RuntimeLibcallImpl<FREXP_F128>;
+  def cbrtf128 : RuntimeLibcallImpl<CBRT_F128>;
+  def fminimumf128 : RuntimeLibcallImpl<FMINIMUM_F128>;
+  def fmaximumf128 : RuntimeLibcallImpl<FMAXIMUM_F128>;
+  def fminimum_numf128 : RuntimeLibcallImpl<FMINIMUM_NUM_F128>;
+  def fmaximum_numf128 : RuntimeLibcallImpl<FMAXIMUM_NUM_F128>;
+  def asinf128 : RuntimeLibcallImpl<ASIN_F128>;
+  def acosf128 : RuntimeLibcallImpl<ACOS_F128>;
+  def atanf128 : RuntimeLibcallImpl<ATAN_F128>;
+  def atan2f128 : RuntimeLibcallImpl<ATAN2_F128>;
+  def ldexpf128 : RuntimeLibcallImpl<LDEXP_F128>;
+  def roundevenf128 : RuntimeLibcallImpl<ROUNDEVEN_F128>;
+  def modff128 : RuntimeLibcallImpl<MODF_F128>;
+  def sinhf128 : RuntimeLibcallImpl<SINH_F128>;
+  def coshf128 : RuntimeLibcallImpl<COSH_F128>;
+  def copysignf128 : RuntimeLibcallImpl<COPYSIGN_F128>;
+}
+
+defset list<RuntimeLibcallImpl> LibmF128FiniteLibcalls = {
+  def __logf128_finite : RuntimeLibcallImpl<LOG_FINITE_F128>;
+  def __log2f128_finite : RuntimeLibcallImpl<LOG2_FINITE_F128>;
+  def __log10f128_finite : RuntimeLibcallImpl<LOG10_FINITE_F128>;
+  def __expf128_finite : RuntimeLibcallImpl<EXP_FINITE_F128>;
+  def __exp2f128_finite : RuntimeLibcallImpl<EXP2_FINITE_F128>;
+  def __exp10f128_finite : RuntimeLibcallImpl<EXP10_FINITE_F128>;
+  def __powf128_finite : RuntimeLibcallImpl<POW_FINITE_F128>;
+}
+
+//===----------------------------------------------------------------------===//
+// Common Libcall Sets
+//===----------------------------------------------------------------------===//
+
+// FIXME: Should move to explicit opt-in to different sets of libcalls
+// instead of trying to remove from a default set. We have
+// unreasonable defaults like reporting f80 calls on most targets when
+// they are relevant to only one.
+
+defvar AllDefaultLibCalls =
+    !foreach(entry, AllDefaultRuntimeLibcallImpls, entry.Provides);
+
+// Exist in libgcc and compiler-rt for 64-bit targets, or if
+// COMPILER_RT_ENABLE_SOFTWARE_INT128.
+defvar Int128RTLibcalls = [
+   __ashlti3, __lshrti3, __ashrti3, __multi3, __mulodi4
+];
+
+// Only available in compiler-rt
+defvar CompilerRTOnlyInt128Libcalls = [
+  __mulodi4
+];
+
+defvar DefaultRuntimeLibcallImpls =
+  !listremove(!listremove(AllDefaultRuntimeLibcallImpls,
+                          Int128RTLibcalls),
+                          CompilerRTOnlyInt128Libcalls);
+
+defvar DefaultRuntimeLibcallImpls_f128 =
+    !filter(entry, DefaultRuntimeLibcallImpls,
+            !match(!cast<string>(entry.Provides), "_F128"));
+
+defvar DefaultRuntimeLibcallImpls_atomic =
+    !filter(entry, DefaultRuntimeLibcallImpls,
+            !match(!cast<string>(entry.Provides), "ATOMIC"));
+
+/// Default set of libcall impls for 32-bit architectures.
+defvar DefaultLibcallImpls32 = DefaultRuntimeLibcallImpls;
+
+/// Default set of libcall impls for 64-bit architectures.
+defvar DefaultLibcallImpls64 = !listconcat(DefaultRuntimeLibcallImpls,
+                                           Int128RTLibcalls);
+
+defvar DarwinSinCosStret = LibcallImpls<(add __sincosf_stret, __sincos_stret),
+                                        darwinHasSinCosStret>;
+defvar DarwinExp10 = LibcallImpls<(add __exp10f, __exp10), darwinHasExp10>;
+
+defvar LibmHasSinCosF32 = LibcallImpls<(add sincosf), hasSinCos>;
+defvar LibmHasSinCosF64 =  LibcallImpls<(add sincos), hasSinCos>;
+defvar LibmHasSinCosF80 = LibcallImpls<(add sincos_f80), hasSinCos>;
+defvar LibmHasSinCosF128 = LibcallImpls<(add sincos_f128), hasSinCos>;
+defvar LibmHasSinCosPPCF128 = LibcallImpls<(add sincos_ppcf128), hasSinCos>;
 
 //===----------------------------------------------------------------------===//
 // AArch64 Runtime Libcalls
@@ -994,15 +1057,48 @@ defset list<RuntimeLibcallImpl> AArch64LibcallImpls = {
   def __arm_sc_memcpy : RuntimeLibcallImpl<SC_MEMCPY>;
   def __arm_sc_memmove : RuntimeLibcallImpl<SC_MEMMOVE>;
   def __arm_sc_memset : RuntimeLibcallImpl<SC_MEMSET>;
-}
+} // End AArch64LibcallImpls
+
+def isAArch64_ExceptArm64EC
+    : RuntimeLibcallPredicate<"(TT.isAArch64() && !TT.isWindowsArm64EC())">;
+def isWindowsArm64EC : RuntimeLibcallPredicate<"TT.isWindowsArm64EC()">;
+def isAArch64_ILP64  : RuntimeLibcallPredicate<"TT.isAArch64(64)">;
+
+
+def AArch64SystemLibrary : SystemRuntimeLibrary<
+  isAArch64_ExceptArm64EC,
+  (add DefaultRuntimeLibcallImpls,
+       AArch64LibcallImpls,
+       LibcallImpls<(add Int128RTLibcalls), isAArch64_ILP64>,
+       LibcallImpls<(add bzero), isOSDarwin>,
+       DarwinExp10, DarwinSinCosStret,
+       LibmHasSinCosF32, LibmHasSinCosF64, LibmHasSinCosF128)
+>;
+
+// Prepend a # to every name
+defset list<RuntimeLibcallImpl> WinArm64ECDefaultRuntimeLibcallImpls = {
+  foreach libcall = DefaultLibcallImpls64 in {
+    def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+  }
 
-foreach libcall = AArch64LibcallImpls in {
-  def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+  foreach libcall = AArch64LibcallImpls in {
+    def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+  }
 }
 
-foreach libcall = DefaultRuntimeLibcallImpls in {
-  def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
-}
+def WindowsARM64ECSystemLibrary
+    : SystemRuntimeLibrary<isWindowsArm64EC,
+                           (add WinArm64ECDefaultRuntimeLibcallImpls)>;
+
+//===----------------------------------------------------------------------===//
+// AMDGPU Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+def isAMDGPU : RuntimeLibcallPredicate<"TT.isAMDGPU()">;
+
+// No calls, except for dummy atomic calls to avoid crashes.
+def AMDGPUSystemLibrary
+    : SystemRuntimeLibrary<isAMDGPU, (add DefaultRuntimeLibcallImpls_atomic)>;
 
 //===----------------------------------------------------------------------===//
 // ARM Runtime Libcalls
@@ -1215,11 +1311,12 @@ def __gnu_h2f_ieee : RuntimeLibcallImpl<FPEXT_F16_F32>;
 //===----------------------------------------------------------------------===//
 
 // Several of the runtime library functions use a special calling conv
-def __divmodqi4 : RuntimeLibcallImpl<SDIVREM_I8>; // CallingConv::AVR_BUILTIN
-def __divmodhi4 : RuntimeLibcallImpl<SDIVREM_I16>; // CallingConv::AVR_BUILTIN
+def __divmodqi4 : RuntimeLibcallImpl<SDIVREM_I8>;
+def __divmodhi4 : RuntimeLibcallImpl<SDIVREM_I16>;
+def __udivmodqi4 : RuntimeLibcallImpl<UDIVREM_I8>;
+def __udivmodhi4 : RuntimeLibcallImpl<UDIVREM_I16>;
+
 //def __divmodsi4 : RuntimeLibcallImpl<SDIVREM_I32>;
-def __udivmodqi4 : RuntimeLibcallImpl<UDIVREM_I8>; // CallingConv::AVR_BUILTIN
-def __udivmodhi4 : RuntimeLibcallImpl<UDIVREM_I16>; // CallingConv::AVR_BUILTIN
 //def __udivmodsi4 : RuntimeLibcallImpl<UDIVREM_I32>;
 
 // Standard sinf/cosf name replaced with "sin" and "cos". Define a
@@ -1227,6 +1324,33 @@ def __udivmodhi4 : RuntimeLibcallImpl<UDIVREM_I16>; // CallingConv::AVR_BUILTIN
 def avr_sin : RuntimeLibcallImpl<SIN_F32, "sin">;
 def avr_cos : RuntimeLibcallImpl<COS_F32, "cos">;
 
+def isAVR : RuntimeLibcallPredicate<"TT.getArch() == Triple::avr">;
+
+def AVRSystemLibrary
+    : SystemRuntimeLibrary<
+          isAVR,
+          (add (sub DefaultRuntimeLibcallImpls,
+
+               // Division rtlib functions (not supported), use divmod
+               // functions instead
+               __divqi3, __divhi3, __divsi3, __udivqi3, __udivhi3, __udivsi3,
+
+               // Modulus rtlib functions (not supported), use divmod functions
+               // instead
+               __modqi3, __modhi3, __modsi3, __umodqi3, __umodhi3, __umodsi3,
+
+               // Standard f64 names are replaced
+               sin, cos, sinf, cosf),
+
+              // Several of the runtime library functions use a special calling
+              // conv
+              LibcallsWithCC<(add __divmodqi4, __divmodhi4, __udivmodqi4,
+                                 __udivmodhi4),
+                             AVR_BUILTIN>,
+              __divmodsi4, __udivmodsi4,
+              // Trigonometric rtlib functions
+              avr_sin, avr_cos)>;
+
 //===----------------------------------------------------------------------===//
 // Hexagon Runtime Libcalls
 //===----------------------------------------------------------------------===//
@@ -1265,6 +1389,18 @@ def __hexagon_fast2_sqrtdf2 : RuntimeLibcallImpl<SQRT_F64>;
 def __hexagon_memcpy_likely_aligned_min32bytes_mult8bytes
     : RuntimeLibcallImpl<HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES>;
 
+//===----------------------------------------------------------------------===//
+// Lanai Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+def isLanai : RuntimeLibcallPredicate<"TT.getArch() == Triple::lanai">;
+
+// Use fast calling convention for library functions.
+def LanaiSystemLibrary
+    : SystemRuntimeLibrary<isLanai, (add DefaultRuntimeLibcallImpls)> {
+  let DefaultLibcallCallingConv = FASTCC;
+}
+
 //===----------------------------------------------------------------------===//
 // Mips16 Runtime Libcalls
 //===----------------------------------------------------------------------===//
@@ -1446,53 +1582,202 @@ def __mspabi_mpyll : RuntimeLibcallImpl<MUL_I64>;
 
 // setLibcallCallingConv(MUL_I64, CallingConv::MSP430_BUILTIN);
 
+def isMSP430 : RuntimeLibcallPredicate<"TT.getArch() == Triple::msp430">;
+
+defvar MSP430DefaultOptOut = [
+  __addsf3, __divsf3, __extendsfdf2, __truncdfsf2, __fixsfsi,
+  __fixsfdi, __fixunssfsi, __mulsf3, __eqsf2, __gesf2, __gtsf2,
+  __divhi3, __divsi3, __ashlsi3, __floatsidf, __floatsisf,
+  __ashrsi3, __modhi3, __udivsi3, __fixdfsi, __fixunssfdi,
+  __udivhi3, __umodsi3, __nesf2, __lesf2, __floatundisf,
+  __fixdfdi, __fixunsdfsi, __modsi3, __floatunsisf,
+  __fixunsdfdi, __ltsf2, __floatdisf, __floatdidf,
+  __lshrsi3, __subsf3, __umodhi3, __floatunsidf,
+  __floatundidf
+];
+
+// EABI Libcalls - EABI Section 6.2
+def MSP430SystemLibrary
+    : SystemRuntimeLibrary<isMSP430,
+    (add (sub DefaultRuntimeLibcallImpls, MSP430DefaultOptOut),
+
+      // Floating point conversions - EABI Table 6
+      __mspabi_cvtdf,
+      __mspabi_cvtfd,
+      // The following is NOT implemented in libgcc
+      //__mspabi_fixdi,
+      __mspabi_fixdli,
+      __mspabi_fixdlli,
+      // The following is NOT implemented in libgcc
+      // __mspabi_fixdu
+      __mspabi_fixdul,
+      __mspabi_fixdull,
+      // The following is NOT implemented in libgcc
+      //__mspabi_fixfi,
+      __mspabi_fixfli,
+      __mspabi_fixflli,
+      // The following is NOT implemented in libgcc
+      //__mspabi_fixfu,
+      __mspabi_fixful,
+      __mspabi_fixfull,
+      // TODO The following IS implemented in libgcc
+      //__mspabi_fltid
+      __mspabi_fltlid,
+      // TODO The following IS implemented in libgcc but is not in the EABI
+      __mspabi_fltllid,
+      // TODO The following IS implemented in libgcc
+      //__mspabi_fltud,
+      __mspabi_fltuld,
+      // The following IS implemented in libgcc but is not in the EABI
+      __mspabi_fltulld,
+      // TODO The following IS implemented in libgcc
+      //__mspabi_fltif,
+      __mspabi_fltlif,
+      // TODO The following IS implemented in libgcc but is not in the EABI
+      __mspabi_fltllif,
+      // TODO The following IS implemented in libgcc
+      //__mspabi_fltuf,
+      __mspabi_fltulf,
+      // The following IS implemented in libgcc but is not in the EABI
+      __mspabi_fltullf,
+
+      // Floating point comparisons - EABI Table 7
+      LibcallsWithCC<(add __mspabi_cmpd__oeq,
+                          __mspabi_cmpd__une,
+                          __mspabi_cmpd__oge,
+                          __mspabi_cmpd__olt,
+                          __mspabi_cmpd__ole,
+                          __mspabi_cmpd__ogt), MSP430_BUILTIN>,
+      __mspabi_cmpf__oeq,
+      __mspabi_cmpf__une,
+      __mspabi_cmpf__oge,
+      __mspabi_cmpf__olt,
+      __mspabi_cmpf__ole,
+      __mspabi_cmpf__ogt,
+
+      // Floating point arithmetic - EABI Table 8
+      LibcallsWithCC<(add __mspabi_addd,
+      __mspabi_subd,
+      __mspabi_mpyd,
+      __mspabi_divd), MSP430_BUILTIN>,
+
+      __mspabi_addf,
+      __mspabi_subf,
+      __mspabi_mpyf,
+      __mspabi_divf,
+
+      // The following are NOT implemented in libgcc
+      // __mspabi_negd,
+      // __mspabi_negf,
+
+      // Universal Integer Operations - EABI Table 9
+      __mspabi_divi,
+      __mspabi_divli,
+      LibcallsWithCC<(add __mspabi_divlli), MSP430_BUILTIN>,
+      __mspabi_divu,
+      __mspabi_divul,
+      LibcallsWithCC<(add __mspabi_divull), MSP430_BUILTIN>,
+      __mspabi_remi,
+      __mspabi_remli,
+      LibcallsWithCC<(add __mspabi_remlli), MSP430_BUILTIN>,
+      __mspabi_remu,
+      __mspabi_remul,
+      LibcallsWithCC<(add __mspabi_remull), MSP430_BUILTIN>,
+
+      // Bitwise Operations - EABI Table 10
+      // TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
+      __mspabi_srll,
+      __mspabi_sral,
+      __mspabi_slll
+      // __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
+  )
+>;
+
 //===----------------------------------------------------------------------===//
-// PPC Runtime Libcalls
+// NVPTX Runtime Libcalls
 //===----------------------------------------------------------------------===//
 
-class PPCRuntimeLibcallImpl<RuntimeLibcall P, string Name = NAME>
-    : RuntimeLibcallImpl<P, Name>;
+def isNVPTX : RuntimeLibcallPredicate<"TT.isNVPTX()">;
+
+// No calls, except for dummy atomic calls to avoid crashes.
+def NVPTXSystemLibrary
+    : SystemRuntimeLibrary<isNVPTX, (add DefaultRuntimeLibcallImpls_atomic)>;
+
+//===----------------------------------------------------------------------===//
+// PPC Runtime Libcalls
+//===----------------------------------------------------------------------===//
 
 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
-def __addkf3 : PPCRuntimeLibcallImpl<ADD_F128>;
-def __subkf3 : PPCRuntimeLibcallImpl<SUB_F128>;
-def __mulkf3 : PPCRuntimeLibcallImpl<MUL_F128>;
-def __divkf3 : PPCRuntimeLibcallImpl<DIV_F128>;
-def __powikf2 : PPCRuntimeLibcallImpl<POWI_F128>;
-def __extendsfkf2 : PPCRuntimeLibcallImpl<FPEXT_F32_F128>;
-def __extenddfkf2 : PPCRuntimeLibcallImpl<FPEXT_F64_F128>;
-def __trunckfhf2 : PPCRuntimeLibcallImpl<FPROUND_F128_F16>;
-def __trunckfsf2 : PPCRuntimeLibcallImpl<FPROUND_F128_F32>;
-def __trunckfdf2 : PPCRuntimeLibcallImpl<FPROUND_F128_F64>;
-def __fixkfsi : PPCRuntimeLibcallImpl<FPTOSINT_F128_I32>;
-def __fixkfdi : PPCRuntimeLibcallImpl<FPTOSINT_F128_I64>;
-def __fixkfti : PPCRuntimeLibcallImpl<FPTOSINT_F128_I128>;
-def __fixunskfsi : PPCRuntimeLibcallImpl<FPTOUINT_F128_I32>;
-def __fixunskfdi : PPCRuntimeLibcallImpl<FPTOUINT_F128_I64>;
-def __fixunskfti : PPCRuntimeLibcallImpl<FPTOUINT_F128_I128>;
-def __floatsikf : PPCRuntimeLibcallImpl<SINTTOFP_I32_F128>;
-def __floatdikf : PPCRuntimeLibcallImpl<SINTTOFP_I64_F128>;
-def __floattikf : PPCRuntimeLibcallImpl<SINTTOFP_I128_F128>;
-def __floatunsikf : PPCRuntimeLibcallImpl<UINTTOFP_I32_F128>;
-def __floatundikf : PPCRuntimeLibcallImpl<UINTTOFP_I64_F128>;
-def __floatuntikf : PPCRuntimeLibcallImpl<UINTTOFP_I128_F128>;
-def __eqkf2 : PPCRuntimeLibcallImpl<OEQ_F128>;
-def __nekf2 : PPCRuntimeLibcallImpl<UNE_F128>;
-def __gekf2 : PPCRuntimeLibcallImpl<OGE_F128>;
-def __ltkf2 : PPCRuntimeLibcallImpl<OLT_F128>;
-def __lekf2 : PPCRuntimeLibcallImpl<OLE_F128>;
-def __gtkf2 : PPCRuntimeLibcallImpl<OGT_F128>;
-def __unordkf2 : PPCRuntimeLibcallImpl<UO_F128>;
-
-// PPC64 && Subtarget.isAIXABI()
-def ___memmove64 : RuntimeLibcallImpl<MEMCPY>;
-def ___memset64 : RuntimeLibcallImpl<MEMSET>;
-def ___bzero64 : RuntimeLibcallImpl<BZERO>;
-
-// !PPC64 && Subtarget.isAIXABI()
-def ___memmove : RuntimeLibcallImpl<MEMMOVE>;
-def ___memset : RuntimeLibcallImpl<MEMSET>;
-def ___bzero : RuntimeLibcallImpl<BZERO>;
+defset list<RuntimeLibcallImpl> PPCRuntimeLibcalls = {
+  def __addkf3 : RuntimeLibcallImpl<ADD_F128>;
+  def __subkf3 : RuntimeLibcallImpl<SUB_F128>;
+  def __mulkf3 : RuntimeLibcallImpl<MUL_F128>;
+  def __divkf3 : RuntimeLibcallImpl<DIV_F128>;
+  def __powikf2 : RuntimeLibcallImpl<POWI_F128>;
+  def __extendsfkf2 : RuntimeLibcallImpl<FPEXT_F32_F128>;
+  def __extenddfkf2 : RuntimeLibcallImpl<FPEXT_F64_F128>;
+  def __trunckfhf2 : RuntimeLibcallImpl<FPROUND_F128_F16>;
+  def __trunckfsf2 : RuntimeLibcallImpl<FPROUND_F128_F32>;
+  def __trunckfdf2 : RuntimeLibcallImpl<FPROUND_F128_F64>;
+  def __fixkfsi : RuntimeLibcallImpl<FPTOSINT_F128_I32>;
+  def __fixkfdi : RuntimeLibcallImpl<FPTOSINT_F128_I64>;
+  def __fixkfti : RuntimeLibcallImpl<FPTOSINT_F128_I128>;
+  def __fixunskfsi : RuntimeLibcallImpl<FPTOUINT_F128_I32>;
+  def __fixunskfdi : RuntimeLibcallImpl<FPTOUINT_F128_I64>;
+  def __fixunskfti : RuntimeLibcallImpl<FPTOUINT_F128_I128>;
+  def __floatsikf : RuntimeLibcallImpl<SINTTOFP_I32_F128>;
+  def __floatdikf : RuntimeLibcallImpl<SINTTOFP_I64_F128>;
+  def __floattikf : RuntimeLibcallImpl<SINTTOFP_I128_F128>;
+  def __floatunsikf : RuntimeLibcallImpl<UINTTOFP_I32_F128>;
+  def __floatundikf : RuntimeLibcallImpl<UINTTOFP_I64_F128>;
+  def __floatuntikf : RuntimeLibcallImpl<UINTTOFP_I128_F128>;
+  def __eqkf2 : RuntimeLibcallImpl<OEQ_F128>;
+  def __nekf2 : RuntimeLibcallImpl<UNE_F128>;
+  def __gekf2 : RuntimeLibcallImpl<OGE_F128>;
+  def __ltkf2 : RuntimeLibcallImpl<OLT_F128>;
+  def __lekf2 : RuntimeLibcallImpl<OLE_F128>;
+  def __gtkf2 : RuntimeLibcallImpl<OGT_F128>;
+  def __unordkf2 : RuntimeLibcallImpl<UO_F128>;
+}
+
+defset list<RuntimeLibcallImpl> PPC64AIXCallList = {
+  def ___memmove64 : RuntimeLibcallImpl<MEMCPY>;
+  def ___memset64 : RuntimeLibcallImpl<MEMSET>;
+  def ___bzero64 : RuntimeLibcallImpl<BZERO>;
+}
+
+defset list<RuntimeLibcallImpl> PPC32AIXCallList = {
+  def ___memmove : RuntimeLibcallImpl<MEMMOVE>;
+  def ___memset : RuntimeLibcallImpl<MEMSET>;
+  def ___bzero : RuntimeLibcallImpl<BZERO>;
+}
+
+defvar PPCOverrides = !foreach(entry, PPCRuntimeLibcalls, entry.Provides);
+
+def isPPC : RuntimeLibcallPredicate<"TT.isPPC()">;
+def isPPC32 : RuntimeLibcallPredicate<"TT.isPPC32()">;
+def isPPC64 : RuntimeLibcallPredicate<"TT.isPPC64()">;
+def isAIX : RuntimeLibcallPredicate<"TT.isOSAIX()">;
+def isNotAIX : RuntimeLibcallPredicate<"!TT.isOSAIX()">;
+def isPPC32_AIX : RuntimeLibcallPredicate<"(TT.isPPC32() && TT.isOSAIX())">;
+def isPPC64_AIX : RuntimeLibcallPredicate<"(TT.isPPC64() && TT.isOSAIX())">;
+
+def AIX32Calls : LibcallImpls<(add PPC32AIXCallList), isPPC32_AIX>;
+def AIX64Calls : LibcallImpls<(add PPC64AIXCallList), isPPC64_AIX>;
+
+// FIXME: Current emission behavior with multiple implementations is
+// janky. We need to filter out the conflicting cases with different
+// f128 names, and then add the overrides. We should switch to
+// explicitly adding subsets of the default calls.
+def PPCSystemLibrary
+    : SystemRuntimeLibrary<isPPC,
+      (add PPCRuntimeLibcalls,
+           (sub DefaultRuntimeLibcallImpls, memcpy,
+                DefaultRuntimeLibcallImpls_f128),
+           __extendkftf2, __trunctfkf2,
+           LibmF128Libcalls, AIX32Calls, AIX64Calls,
+           AvailableIf<memcpy, isNotAIX>,
+           LibcallImpls<(add Int128RTLibcalls), isPPC64>)>;
 
 //===----------------------------------------------------------------------===//
 // SPARC Runtime Libcalls
@@ -1550,11 +1835,12 @@ def _Q_ulltoq : RuntimeLibcallImpl<UINTTOFP_I64_F128>;
 //===----------------------------------------------------------------------===//
 
 // TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment()
-def _alldiv : RuntimeLibcallImpl<SDIV_I64>; // CallingConv::X86_StdCall
-def _aulldiv : RuntimeLibcallImpl<UDIV_I64>; // CallingConv::X86_StdCall
-def _allrem : RuntimeLibcallImpl<SREM_I64>; // CallingConv::X86_StdCall
-def _aullrem : RuntimeLibcallImpl<UREM_I64>; // CallingConv::X86_StdCall
-def _allmul : RuntimeLibcallImpl<MUL_I64>; // CallingConv::X86_StdCall
+
+def _alldiv : RuntimeLibcallImpl<SDIV_I64>;
+def _aulldiv : RuntimeLibcallImpl<UDIV_I64>;
+def _allrem : RuntimeLibcallImpl<SREM_I64>;
+def _aullrem : RuntimeLibcallImpl<UREM_I64>;
+def _allmul : RuntimeLibcallImpl<MUL_I64>;
 
 //===----------------------------------------------------------------------===//
 // XCore Runtime Libcalls
@@ -1562,94 +1848,105 @@ def _allmul : RuntimeLibcallImpl<MUL_I64>; // CallingConv::X86_StdCall
 
 def __memcpy_4 : RuntimeLibcallImpl<MEMCPY_ALIGN_4>;
 
+def isXCore : RuntimeLibcallPredicate<"TT.getArch() == Triple::xcore">;
+def XCoreSystemLibrary
+    : SystemRuntimeLibrary<isXCore, (add DefaultRuntimeLibcallImpls,
+                                        __memcpy_4)>;
+
 //===----------------------------------------------------------------------===//
 // ZOS Runtime Libcalls
 //===----------------------------------------------------------------------===//
 
-class ZOSRuntimeLibcallImpl<RuntimeLibcall P, string Name = NAME>
-    : RuntimeLibcallImpl<P, Name>;
-
-def zos___TRNC_B : ZOSRuntimeLibcallImpl<TRUNC_F64, "@@TRNC@B">;
-def zos___FTRC_B : ZOSRuntimeLibcallImpl<TRUNC_F32, "@@FTRC@B">;
-def zos___LTRC_B : ZOSRuntimeLibcallImpl<TRUNC_F128, "@@LTRC@B">;
-def zos___WSQT_B : ZOSRuntimeLibcallImpl<SQRT_F64, "@@WSQT@B">;
-def zos___FSQT_B : ZOSRuntimeLibcallImpl<SQRT_F32, "@@FSQT@B">;
-def zos___LSQT_B : ZOSRuntimeLibcallImpl<SQRT_F128, "@@LSQT@B">;
-def zos___SSIN_B : ZOSRuntimeLibcallImpl<SIN_F64, "@@SSIN@B">;
-def zos___FSIN_B : ZOSRuntimeLibcallImpl<SIN_F32, "@@FSIN@B">;
-def zos___LSIN_B : ZOSRuntimeLibcallImpl<SIN_F128, "@@LSIN@B">;
-def zos___ROUN_B : ZOSRuntimeLibcallImpl<ROUND_F64, "@@ROUN@B">;
-def zos___ROUNFB : ZOSRuntimeLibcallImpl<ROUND_F32, "@@ROUNFB">;
-def zos___ROUNLB : ZOSRuntimeLibcallImpl<ROUND_F128, "@@ROUNLB">;
-def zos___SRNT_B : ZOSRuntimeLibcallImpl<RINT_F64, "@@SRNT@B">;
-def zos___RINTFB : ZOSRuntimeLibcallImpl<RINT_F32, "@@RINTFB">;
-def zos___RINTLB : ZOSRuntimeLibcallImpl<RINT_F128, "@@RINTLB">;
-def zos___WFMD_B : ZOSRuntimeLibcallImpl<REM_F64, "@@WFMD@B">;
-def zos___FFMD_B : ZOSRuntimeLibcallImpl<REM_F32, "@@FFMD@B">;
-def zos___LFMD_B : ZOSRuntimeLibcallImpl<REM_F128, "@@LFMD@B">;
-def zos___WPOW_B : ZOSRuntimeLibcallImpl<POW_F64, "@@WPOW@B">;
-def zos___FPOW_B : ZOSRuntimeLibcallImpl<POW_F32, "@@FPOW@B">;
-def zos___LPOW_B : ZOSRuntimeLibcallImpl<POW_F128, "@@LPOW@B">;
-def zos___NBYI_B : ZOSRuntimeLibcallImpl<NEARBYINT_F64, "@@NBYI@B">;
-def zos___NBYIFB : ZOSRuntimeLibcallImpl<NEARBYINT_F32, "@@NBYIFB">;
-def zos___NBYILB : ZOSRuntimeLibcallImpl<NEARBYINT_F128, "@@NBYILB">;
-def zos___ROND_B : ZOSRuntimeLibcallImpl<LROUND_F64, "@@ROND@B">;
-def zos___FRND_B : ZOSRuntimeLibcallImpl<LROUND_F32, "@@FRND@B">;
-def zos___LRND_B : ZOSRuntimeLibcallImpl<LROUND_F128, "@@LRND@B">;
-def zos___LRNT_B : ZOSRuntimeLibcallImpl<LRINT_F64, "@@LRNT@B">;
-def zos___LRNTFB : ZOSRuntimeLibcallImpl<LRINT_F32, "@@LRNTFB">;
-def zos___LRNTLB : ZOSRuntimeLibcallImpl<LRINT_F128, "@@LRNTLB">;
-def zos___WLOG_B : ZOSRuntimeLibcallImpl<LOG_F64, "@@WLOG@B">;
-def zos___FLOG_B : ZOSRuntimeLibcallImpl<LOG_F32, "@@FLOG@B">;
-def zos___LLOG_B : ZOSRuntimeLibcallImpl<LOG_F128, "@@LLOG@B">;
-def zos___LOG2_B : ZOSRuntimeLibcallImpl<LOG2_F64, "@@LOG2@B">;
-def zos___FLG2_B : ZOSRuntimeLibcallImpl<LOG2_F32, "@@FLG2@B">;
-def zos___LLG2_B : ZOSRuntimeLibcallImpl<LOG2_F128, "@@LLG2@B">;
-def zos___WLG1_B : ZOSRuntimeLibcallImpl<LOG10_F64, "@@WLG1@B">;
-def zos___FLG1_B : ZOSRuntimeLibcallImpl<LOG10_F32, "@@FLG1@B">;
-def zos___LLG1_B : ZOSRuntimeLibcallImpl<LOG10_F128, "@@LLG1@B">;
-def zos___LLRD_B : ZOSRuntimeLibcallImpl<LLROUND_F64, "@@LLRD@B">;
-def zos___LLRDFB : ZOSRuntimeLibcallImpl<LLROUND_F32, "@@LLRDFB">;
-def zos___LLRDLB : ZOSRuntimeLibcallImpl<LLROUND_F128, "@@LLRDLB">;
-def zos___LLRT_B : ZOSRuntimeLibcallImpl<LLRINT_F64, "@@LLRT@B">;
-def zos___LLRTFB : ZOSRuntimeLibcallImpl<LLRINT_F32, "@@LLRTFB">;
-def zos___LLRTLB : ZOSRuntimeLibcallImpl<LLRINT_F128, "@@LLRTLB">;
-def zos___SLXP_B : ZOSRuntimeLibcallImpl<LDEXP_F64, "@@SLXP@B">;
-def zos___FLXP_B : ZOSRuntimeLibcallImpl<LDEXP_F32, "@@FLXP@B">;
-def zos___LLXP_B : ZOSRuntimeLibcallImpl<LDEXP_F128, "@@LLXP@B">;
-def zos___SFXP_B : ZOSRuntimeLibcallImpl<FREXP_F64, "@@SFXP@B">;
-def zos___FFXP_B : ZOSRuntimeLibcallImpl<FREXP_F32, "@@FFXP@B">;
-def zos___LFXP_B : ZOSRuntimeLibcallImpl<FREXP_F128, "@@LFXP@B">;
-def zos___FMIN_B : ZOSRuntimeLibcallImpl<FMIN_F64, "@@FMIN@B">;
-def zos___FMINFB : ZOSRuntimeLibcallImpl<FMIN_F32, "@@FMINFB">;
-def zos___FMINLB : ZOSRuntimeLibcallImpl<FMIN_F128, "@@FMINLB">;
-def zos___FMA_B : ZOSRuntimeLibcallImpl<FMA_F64, "@@FMA@B">;
-def zos___FMAFB : ZOSRuntimeLibcallImpl<FMA_F32, "@@FMAFB">;
-def zos___FMALB : ZOSRuntimeLibcallImpl<FMA_F128, "@@FMALB">;
-def zos___FMAX_B : ZOSRuntimeLibcallImpl<FMAX_F64, "@@FMAX@B">;
-def zos___FMAXFB : ZOSRuntimeLibcallImpl<FMAX_F32, "@@FMAXFB">;
-def zos___FMAXLB : ZOSRuntimeLibcallImpl<FMAX_F128, "@@FMAXLB">;
-def zos___SFLR_B : ZOSRuntimeLibcallImpl<FLOOR_F64, "@@SFLR@B">;
-def zos___FFLR_B : ZOSRuntimeLibcallImpl<FLOOR_F32, "@@FFLR@B">;
-def zos___LFLR_B : ZOSRuntimeLibcallImpl<FLOOR_F128, "@@LFLR@B">;
-def zos___WEXP_B : ZOSRuntimeLibcallImpl<EXP_F64, "@@WEXP@B">;
-def zos___FEXP_B : ZOSRuntimeLibcallImpl<EXP_F32, "@@FEXP@B">;
-def zos___LEXP_B : ZOSRuntimeLibcallImpl<EXP_F128, "@@LEXP@B">;
-def zos___EXP2_B : ZOSRuntimeLibcallImpl<EXP2_F64, "@@EXP2@B">;
-def zos___FXP2_B : ZOSRuntimeLibcallImpl<EXP2_F32, "@@FXP2@B">;
-def zos___LXP2_B : ZOSRuntimeLibcallImpl<EXP2_F128, "@@LXP2@B">;
-def zos___SCOS_B : ZOSRuntimeLibcallImpl<COS_F64, "@@SCOS@B">;
-def zos___FCOS_B : ZOSRuntimeLibcallImpl<COS_F32, "@@FCOS@B">;
-def zos___LCOS_B : ZOSRuntimeLibcallImpl<COS_F128, "@@LCOS@B">;
-def zos___DCPY_B : ZOSRuntimeLibcallImpl<COPYSIGN_F64, "@@DCPY@B">;
-def zos___FCPY_B : ZOSRuntimeLibcallImpl<COPYSIGN_F32, "@@FCPY@B">;
-def zos___LCPY_B : ZOSRuntimeLibcallImpl<COPYSIGN_F128, "@@LCPY@B">;
-def zos___SCEL_B : ZOSRuntimeLibcallImpl<CEIL_F64, "@@SCEL@B">;
-def zos___FCEL_B : ZOSRuntimeLibcallImpl<CEIL_F32, "@@FCEL@B">;
-def zos___LCEL_B : ZOSRuntimeLibcallImpl<CEIL_F128, "@@LCEL@B">;
-def zos___SCRT_B : ZOSRuntimeLibcallImpl<CBRT_F64, "@@SCRT@B">;
-def zos___FCBT_B : ZOSRuntimeLibcallImpl<CBRT_F32, "@@FCBT@B">;
-def zos___LCBT_B : ZOSRuntimeLibcallImpl<CBRT_F128, "@@LCBT@B">;
+defset list<RuntimeLibcallImpl> ZOSRuntimeLibcalls = {
+  def zos___TRNC_B : RuntimeLibcallImpl<TRUNC_F64, "@@TRNC@B">;
+  def zos___FTRC_B : RuntimeLibcallImpl<TRUNC_F32, "@@FTRC@B">;
+  def zos___LTRC_B : RuntimeLibcallImpl<TRUNC_F128, "@@LTRC@B">;
+  def zos___WSQT_B : RuntimeLibcallImpl<SQRT_F64, "@@WSQT@B">;
+  def zos___FSQT_B : RuntimeLibcallImpl<SQRT_F32, "@@FSQT@B">;
+  def zos___LSQT_B : RuntimeLibcallImpl<SQRT_F128, "@@LSQT@B">;
+  def zos___SSIN_B : RuntimeLibcallImpl<SIN_F64, "@@SSIN@B">;
+  def zos___FSIN_B : RuntimeLibcallImpl<SIN_F32, "@@FSIN@B">;
+  def zos___LSIN_B : RuntimeLibcallImpl<SIN_F128, "@@LSIN@B">;
+  def zos___ROUN_B : RuntimeLibcallImpl<ROUND_F64, "@@ROUN@B">;
+  def zos___ROUNFB : RuntimeLibcallImpl<ROUND_F32, "@@ROUNFB">;
+  def zos___ROUNLB : RuntimeLibcallImpl<ROUND_F128, "@@ROUNLB">;
+  def zos___SRNT_B : RuntimeLibcallImpl<RINT_F64, "@@SRNT@B">;
+  def zos___RINTFB : RuntimeLibcallImpl<RINT_F32, "@@RINTFB">;
+  def zos___RINTLB : RuntimeLibcallImpl<RINT_F128, "@@RINTLB">;
+  def zos___WFMD_B : RuntimeLibcallImpl<REM_F64, "@@WFMD@B">;
+  def zos___FFMD_B : RuntimeLibcallImpl<REM_F32, "@@FFMD@B">;
+  def zos___LFMD_B : RuntimeLibcallImpl<REM_F128, "@@LFMD@B">;
+  def zos___WPOW_B : RuntimeLibcallImpl<POW_F64, "@@WPOW@B">;
+  def zos___FPOW_B : RuntimeLibcallImpl<POW_F32, "@@FPOW@B">;
+  def zos___LPOW_B : RuntimeLibcallImpl<POW_F128, "@@LPOW@B">;
+  def zos___NBYI_B : RuntimeLibcallImpl<NEARBYINT_F64, "@@NBYI@B">;
+  def zos___NBYIFB : RuntimeLibcallImpl<NEARBYINT_F32, "@@NBYIFB">;
+  def zos___NBYILB : RuntimeLibcallImpl<NEARBYINT_F128, "@@NBYILB">;
+  def zos___ROND_B : RuntimeLibcallImpl<LROUND_F64, "@@ROND@B">;
+  def zos___FRND_B : RuntimeLibcallImpl<LROUND_F32, "@@FRND@B">;
+  def zos___LRND_B : RuntimeLibcallImpl<LROUND_F128, "@@LRND@B">;
+  def zos___LRNT_B : RuntimeLibcallImpl<LRINT_F64, "@@LRNT@B">;
+  def zos___LRNTFB : RuntimeLibcallImpl<LRINT_F32, "@@LRNTFB">;
+  def zos___LRNTLB : RuntimeLibcallImpl<LRINT_F128, "@@LRNTLB">;
+  def zos___WLOG_B : RuntimeLibcallImpl<LOG_F64, "@@WLOG@B">;
+  def zos___FLOG_B : RuntimeLibcallImpl<LOG_F32, "@@FLOG@B">;
+  def zos___LLOG_B : RuntimeLibcallImpl<LOG_F128, "@@LLOG@B">;
+  def zos___LOG2_B : RuntimeLibcallImpl<LOG2_F64, "@@LOG2@B">;
+  def zos___FLG2_B : RuntimeLibcallImpl<LOG2_F32, "@@FLG2@B">;
+  def zos___LLG2_B : RuntimeLibcallImpl<LOG2_F128, "@@LLG2@B">;
+  def zos___WLG1_B : RuntimeLibcallImpl<LOG10_F64, "@@WLG1@B">;
+  def zos___FLG1_B : RuntimeLibcallImpl<LOG10_F32, "@@FLG1@B">;
+  def zos___LLG1_B : RuntimeLibcallImpl<LOG10_F128, "@@LLG1@B">;
+  def zos___LLRD_B : RuntimeLibcallImpl<LLROUND_F64, "@@LLRD@B">;
+  def zos___LLRDFB : RuntimeLibcallImpl<LLROUND_F32, "@@LLRDFB">;
+  def zos___LLRDLB : RuntimeLibcallImpl<LLROUND_F128, "@@LLRDLB">;
+  def zos___LLRT_B : RuntimeLibcallImpl<LLRINT_F64, "@@LLRT@B">;
+  def zos___LLRTFB : RuntimeLibcallImpl<LLRINT_F32, "@@LLRTFB">;
+  def zos___LLRTLB : RuntimeLibcallImpl<LLRINT_F128, "@@LLRTLB">;
+  def zos___SLXP_B : RuntimeLibcallImpl<LDEXP_F64, "@@SLXP@B">;
+  def zos___FLXP_B : RuntimeLibcallImpl<LDEXP_F32, "@@FLXP@B">;
+  def zos___LLXP_B : RuntimeLibcallImpl<LDEXP_F128, "@@LLXP@B">;
+  def zos___SFXP_B : RuntimeLibcallImpl<FREXP_F64, "@@SFXP@B">;
+  def zos___FFXP_B : RuntimeLibcallImpl<FREXP_F32, "@@FFXP@B">;
+  def zos___LFXP_B : RuntimeLibcallImpl<FREXP_F128, "@@LFXP@B">;
+  def zos___FMIN_B : RuntimeLibcallImpl<FMIN_F64, "@@FMIN@B">;
+  def zos___FMINFB : RuntimeLibcallImpl<FMIN_F32, "@@FMINFB">;
+  def zos___FMINLB : RuntimeLibcallImpl<FMIN_F128, "@@FMINLB">;
+  def zos___FMA_B : RuntimeLibcallImpl<FMA_F64, "@@FMA@B">;
+  def zos___FMAFB : RuntimeLibcallImpl<FMA_F32, "@@FMAFB">;
+  def zos___FMALB : RuntimeLibcallImpl<FMA_F128, "@@FMALB">;
+  def zos___FMAX_B : RuntimeLibcallImpl<FMAX_F64, "@@FMAX@B">;
+  def zos___FMAXFB : RuntimeLibcallImpl<FMAX_F32, "@@FMAXFB">;
+  def zos___FMAXLB : RuntimeLibcallImpl<FMAX_F128, "@@FMAXLB">;
+  def zos___SFLR_B : RuntimeLibcallImpl<FLOOR_F64, "@@SFLR@B">;
+  def zos___FFLR_B : RuntimeLibcallImpl<FLOOR_F32, "@@FFLR@B">;
+  def zos___LFLR_B : RuntimeLibcallImpl<FLOOR_F128, "@@LFLR@B">;
+  def zos___WEXP_B : RuntimeLibcallImpl<EXP_F64, "@@WEXP@B">;
+  def zos___FEXP_B : RuntimeLibcallImpl<EXP_F32, "@@FEXP@B">;
+  def zos___LEXP_B : RuntimeLibcallImpl<EXP_F128, "@@LEXP@B">;
+  def zos___EXP2_B : RuntimeLibcallImpl<EXP2_F64, "@@EXP2@B">;
+  def zos___FXP2_B : RuntimeLibcallImpl<EXP2_F32, "@@FXP2@B">;
+  def zos___LXP2_B : RuntimeLibcallImpl<EXP2_F128, "@@LXP2@B">;
+  def zos___SCOS_B : RuntimeLibcallImpl<COS_F64, "@@SCOS@B">;
+  def zos___FCOS_B : RuntimeLibcallImpl<COS_F32, "@@FCOS@B">;
+  def zos___LCOS_B : RuntimeLibcallImpl<COS_F128, "@@LCOS@B">;
+  def zos___DCPY_B : RuntimeLibcallImpl<COPYSIGN_F64, "@@DCPY@B">;
+  def zos___FCPY_B : RuntimeLibcallImpl<COPYSIGN_F32, "@@FCPY@B">;
+  def zos___LCPY_B : RuntimeLibcallImpl<COPYSIGN_F128, "@@LCPY@B">;
+  def zos___SCEL_B : RuntimeLibcallImpl<CEIL_F64, "@@SCEL@B">;
+  def zos___FCEL_B : RuntimeLibcallImpl<CEIL_F32, "@@FCEL@B">;
+  def zos___LCEL_B : RuntimeLibcallImpl<CEIL_F128, "@@LCEL@B">;
+  def zos___SCRT_B : RuntimeLibcallImpl<CBRT_F64, "@@SCRT@B">;
+  def zos___FCBT_B : RuntimeLibcallImpl<CBRT_F32, "@@FCBT@B">;
+  def zos___LCBT_B : RuntimeLibcallImpl<CBRT_F128, "@@LCBT@B">;
+}
+
+def isSystemZZOS : RuntimeLibcallPredicate<"(TT.isSystemZ() && TT.isOSzOS())">;
+def isZOS : RuntimeLibcallPredicate<"TT.isOSzOS()">;
+def SystemZZOSSystemLibrary
+    : SystemRuntimeLibrary<
+          isSystemZZOS, (add DefaultLibcallImpls64,
+                            LibcallImpls<(add ZOSRuntimeLibcalls), isZOS>)>;
 
 //===----------------------------------------------------------------------===//
 // WebAssembly Runtime Libcalls
@@ -1659,3 +1956,14 @@ def zos___LCBT_B : ZOSRuntimeLibcallImpl<CBRT_F128, "@@LCBT@B">;
 // TODO: when implementing other Wasm backends, make this generic or only do
 // this on emscripten depending on what they end up doing.
 def emscripten_return_address : RuntimeLibcallImpl<RETURN_ADDRESS>;
+
+def isWasm : RuntimeLibcallPredicate<"TT.isWasm()">;
+
+// Define the emscripten name for return address helper.
+// TODO: when implementing other Wasm backends, make this generic or only do
+// this on emscripten depending on what they end up doing.
+def WasmSystemLibrary
+    : SystemRuntimeLibrary<isWasm,
+      (add DefaultRuntimeLibcallImpls, Int128RTLibcalls,
+           CompilerRTOnlyInt128Libcalls,
+           emscripten_return_address)>;
diff --git a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
index 9444a0a838cdd..601c291daf89d 100644
--- a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
+++ b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
@@ -6,6 +6,37 @@
 //
 //===----------------------------------------------------------------------===//
 
+include "llvm/TableGen/SetTheory.td"
+
+// Predicate for whether a libcall exists for the target ABI. This is
+// a module level property that should only be computed based on the
+// triple.
+class RuntimeLibcallPredicate<code cond> {
+  // Expression of an llvm::Triple named TT for whether a libcall
+  // should exist.
+  code Cond = cond;
+}
+
+// Predicate for whether a libcall should be used for the current
+// function/subtarget.
+class LibcallLoweringPredicate<code cond> { code Cond = cond; }
+
+def AlwaysAvailable : RuntimeLibcallPredicate<[{}]>;
+
+class LibcallCallingConv<code CC = [{}]> {
+  // Enum value for the calling convention. Empty string defaults to
+  // CallingConv::C.
+  code CallingConv = CC;
+}
+
+def DefaultCC : LibcallCallingConv;
+def FASTCC : LibcallCallingConv<[{CallingConv::Fast}]>;
+def MSP430_BUILTIN : LibcallCallingConv<[{CallingConv::MSP430_BUILTIN}]>;
+def ARM_AAPCS : LibcallCallingConv<[{CallingConv::ARM_AAPCS}]>;
+def ARM_AAPCS_VFP : LibcallCallingConv<[{CallingConv::ARM_AAPCS_VFP}]>;
+def X86_STDCALL : LibcallCallingConv<[{CallingConv::X86_StdCall}]>;
+def AVR_BUILTIN : LibcallCallingConv<[{CallingConv::AVR_BUILTIN}]>;
+
 /// Abstract definition for functionality the compiler may need to
 /// emit a call to. Emits the RTLIB::Libcall enum - This enum defines
 /// all of the runtime library calls the backend can emit. The various
@@ -24,9 +55,39 @@ class RuntimeLibcall {
 // Concrete implementation of a libcall, which may have a different
 // name and only be valid on some subset of targets.
 
-// TODO: Do we need a way to conditionally select calling convention?
 class RuntimeLibcallImpl<RuntimeLibcall P, string Name = NAME> {
   RuntimeLibcall Provides = P;
   string LibCallFuncName = Name;
+  list<LibcallLoweringPredicate> LoweringPredicates;
   bit IsDefault = false;
 }
+
+class LibcallImpls<dag funcList,
+                   RuntimeLibcallPredicate Pred = AlwaysAvailable> {
+  // Function of the triple where this applies
+  RuntimeLibcallPredicate AvailabilityPredicate = Pred;
+  dag MemberList = funcList;
+
+  // If set, applies calling convention to every entry instead of
+  // taking the system default.
+  LibcallCallingConv CallingConv = ?;
+}
+
+/// Convenience wrapper around LibcallImplSet to make a single libcall
+/// implementation conditionally conditionally available.
+class AvailableIf<RuntimeLibcallImpl Impl, RuntimeLibcallPredicate Pred>
+    : LibcallImpls<(add Impl), Pred>;
+
+class LibcallsWithCC<dag funcList, LibcallCallingConv CC,
+                     RuntimeLibcallPredicate Pred = AlwaysAvailable>
+    : LibcallImpls<funcList, Pred> {
+  LibcallCallingConv CallingConv = CC;
+}
+
+/// Define a complete top level set of runtime libcalls for a target.
+class SystemRuntimeLibrary<RuntimeLibcallPredicate Pred, dag funcList> {
+  /// Set the default calling convention assumed for RuntimeLibcallImpl members.
+  LibcallCallingConv DefaultLibcallCallingConv = DefaultCC;
+  RuntimeLibcallPredicate TriplePred = Pred;
+  LibcallImpls MemberList = LibcallImpls<funcList>;
+}
diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h
index fd987179645dd..74dd490729741 100644
--- a/llvm/include/llvm/IR/Type.h
+++ b/llvm/include/llvm/IR/Type.h
@@ -515,7 +515,7 @@ inline Type **unwrap(LLVMTypeRef* Tys) {
 }
 
 inline LLVMTypeRef *wrap(Type **Tys) {
-  return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
+  return reinterpret_cast<LLVMTypeRef *>(Tys);
 }
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/DXContainerRootSignature.h b/llvm/include/llvm/MC/DXContainerRootSignature.h
index 4c2ca063158c4..4b6b42f7d74f7 100644
--- a/llvm/include/llvm/MC/DXContainerRootSignature.h
+++ b/llvm/include/llvm/MC/DXContainerRootSignature.h
@@ -68,7 +68,7 @@ struct RootParametersContainer {
     Tables.push_back(Table);
   }
 
-  const std::pair<uint32_t, uint32_t>
+  std::pair<uint32_t, uint32_t>
   getTypeAndLocForParameter(uint32_t Location) const {
     const RootParameterInfo &Info = ParametersInfo[Location];
     return {Info.Header.ParameterType, Info.Location};
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index e49e786a10f58..6b81bdba25e67 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -28,17 +28,34 @@ class MCSymbol;
 class MCAssembler;
 class MCContext;
 struct MCDwarfFrameInfo;
-struct MCFixupKindInfo;
 class MCInst;
 class MCObjectStreamer;
 class MCObjectTargetWriter;
 class MCObjectWriter;
+class MCOperand;
 class MCSubtargetInfo;
 class MCValue;
 class raw_pwrite_stream;
 class StringRef;
 class raw_ostream;
 
+/// Target independent information on a fixup kind.
+struct MCFixupKindInfo {
+  /// A target specific name for the fixup kind. The names will be unique for
+  /// distinct kinds on any given target.
+  const char *Name;
+
+  /// The bit offset to write the relocation into.
+  uint8_t TargetOffset;
+
+  /// The number of bits written by this fixup. The bits are assumed to be
+  /// contiguous.
+  uint8_t TargetSize;
+
+  /// Flags describing additional information on this fixup kind.
+  unsigned Flags;
+};
+
 /// Generic interface to target specific assembler backends.
 class LLVM_ABI MCAsmBackend {
 protected: // Can only create subclasses.
@@ -91,11 +108,6 @@ class LLVM_ABI MCAsmBackend {
   /// Get information on a fixup kind.
   virtual MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const;
 
-  // Hook used by the default `addReloc` to check if a relocation is needed.
-  virtual bool shouldForceRelocation(const MCFixup &, const MCValue &) {
-    return false;
-  }
-
   /// Hook to check if extra nop bytes must be inserted for alignment directive.
   /// For some targets this may be necessary in order to support linker
   /// relaxation. The number of bytes to insert are returned in Size.
@@ -111,14 +123,18 @@ class LLVM_ABI MCAsmBackend {
     return false;
   }
 
-  virtual bool evaluateTargetFixup(const MCFixup &Fixup, const MCValue &Target,
-                                   uint64_t &Value) {
-    llvm_unreachable("Need to implement hook if target has custom fixups");
+  // Evaluate a fixup, returning std::nullopt to use default handling for
+  // `Value` and `IsResolved`. Otherwise, returns `IsResolved` with the
+  // expectation that the hook updates `Value`.
+  virtual std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &,
+                                            MCValue &, uint64_t &) {
+    return {};
   }
 
-  virtual bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                        uint64_t &FixedValue, bool IsResolved);
+  void maybeAddReloc(const MCFragment &, const MCFixup &, const MCValue &,
+                     uint64_t &Value, bool IsResolved);
 
+  /// Determine if a relocation is required. In addition,
   /// Apply the \p Value for given \p Fixup into the provided data fragment, at
   /// the offset specified by the fixup and following the fixup kind as
   /// appropriate. Errors (such as an out of range fixup value) should be
@@ -132,12 +148,9 @@ class LLVM_ABI MCAsmBackend {
   /// \name Target Relaxation Interfaces
   /// @{
 
-  /// Check whether the given instruction may need relaxation.
-  ///
-  /// \param Inst - The instruction to test.
-  /// \param STI - The MCSubtargetInfo in effect when the instruction was
-  /// encoded.
-  virtual bool mayNeedRelaxation(const MCInst &Inst,
+  /// Check whether the given instruction (encoded as Opcode+Operands) may need
+  /// relaxation.
+  virtual bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                                  const MCSubtargetInfo &STI) const {
     return false;
   }
@@ -159,7 +172,11 @@ class LLVM_ABI MCAsmBackend {
   /// instruction.
   /// \param STI the subtarget information for the associated instruction.
   virtual void relaxInstruction(MCInst &Inst,
-                                const MCSubtargetInfo &STI) const {};
+                                const MCSubtargetInfo &STI) const {
+    llvm_unreachable(
+        "Needed if fixupNeedsRelaxation/fixupNeedsRelaxationAdvanced may "
+        "return true");
+  }
 
   // Defined by linker relaxation targets.
   virtual bool relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 5e009090311c4..1015992cedf29 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -101,21 +101,18 @@ class MCAssembler {
   /// out.
   /// \param RecordReloc Record relocation if needed.
   /// relocation.
-  bool evaluateFixup(const MCFragment *F, const MCFixup &Fixup, MCValue &Target,
+  bool evaluateFixup(const MCFragment &F, MCFixup &Fixup, MCValue &Target,
                      uint64_t &Value, bool RecordReloc,
                      MutableArrayRef<char> Contents) const;
 
   /// Check whether a fixup can be satisfied, or whether it needs to be relaxed
   /// (increased in size, in order to hold its value correctly).
-  bool fixupNeedsRelaxation(const MCFixup &Fixup, const MCRelaxableFragment *DF) const;
-
-  /// Check whether the given fragment needs relaxation.
-  bool fragmentNeedsRelaxation(const MCRelaxableFragment *IF) const;
+  bool fixupNeedsRelaxation(const MCRelaxableFragment &, const MCFixup &) const;
 
   void layoutSection(MCSection &Sec);
-  /// Perform one layout iteration and return true if any offsets
-  /// were adjusted.
-  bool relaxOnce();
+  /// Perform one layout iteration and return the index of the first stable
+  /// section for subsequent optimization.
+  unsigned relaxOnce(unsigned FirstStable);
 
   /// Perform relaxation on a single fragment.
   bool relaxFragment(MCFragment &F);
diff --git a/llvm/include/llvm/MC/MCCodeView.h b/llvm/include/llvm/MC/MCCodeView.h
index 2a57e04b2c88c..88f84a2462841 100644
--- a/llvm/include/llvm/MC/MCCodeView.h
+++ b/llvm/include/llvm/MC/MCCodeView.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include <deque>
 #include <map>
 #include <vector>
 
@@ -268,6 +269,10 @@ class CodeViewContext {
   /// Indicate whether we have already laid out the checksum table addresses or
   /// not.
   bool ChecksumOffsetsAssigned = false;
+
+  /// Append-only storage of MCCVDefRangeFragment::Ranges.
+  std::deque<SmallVector<std::pair<const MCSymbol *, const MCSymbol *>, 0>>
+      DefRangeStorage;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index d7b81af4a785a..9f31e821d0923 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -612,12 +612,14 @@ class MCContext {
   getELFUniqueIDForEntsize(StringRef SectionName, unsigned Flags,
                            unsigned EntrySize);
 
-  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
-                                GOFF::SDAttr SDAttributes);
-  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
-                                GOFF::EDAttr EDAttributes, MCSection *Parent);
-  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
-                                GOFF::PRAttr PRAttributes, MCSection *Parent);
+  LLVM_ABI MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                         GOFF::SDAttr SDAttributes);
+  LLVM_ABI MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                         GOFF::EDAttr EDAttributes,
+                                         MCSection *Parent);
+  LLVM_ABI MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                         GOFF::PRAttr PRAttributes,
+                                         MCSection *Parent);
 
   LLVM_ABI MCSectionCOFF *
   getCOFFSection(StringRef Section, unsigned Characteristics,
diff --git a/llvm/include/llvm/MC/MCDXContainerStreamer.h b/llvm/include/llvm/MC/MCDXContainerStreamer.h
index 8d29bb3d03f61..37e3c7ddaad2c 100644
--- a/llvm/include/llvm/MC/MCDXContainerStreamer.h
+++ b/llvm/include/llvm/MC/MCDXContainerStreamer.h
@@ -35,9 +35,6 @@ class MCDXContainerStreamer : public MCObjectStreamer {
 
   bool emitSymbolAttribute(MCSymbol *, MCSymbolAttr) override { return false; }
   void emitCommonSymbol(MCSymbol *, uint64_t, Align) override {}
-
-private:
-  void emitInstToData(const MCInst &, const MCSubtargetInfo &) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index 8d66ed22875df..e460c95ea4dff 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -33,6 +33,10 @@ class MCSymbolRefExpr;
 /// needed for parsing.
 class MCExpr {
 public:
+  // Allow MC classes to access the private `print` function.
+  friend class MCAsmInfo;
+  friend class MCFragment;
+  friend class MCOperand;
   enum ExprKind : uint8_t {
     Binary,    ///< Binary expressions.
     Constant,  ///< Constant expressions.
@@ -53,10 +57,13 @@ class MCExpr {
   unsigned SubclassData : NumSubclassDataBits;
   SMLoc Loc;
 
+  void print(raw_ostream &OS, const MCAsmInfo *MAI,
+             int SurroundingPrec = 0) const;
   bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
                           bool InSet) const;
 
 protected:
+  using Spec = uint16_t;
   explicit MCExpr(ExprKind Kind, SMLoc Loc, unsigned SubclassData = 0)
       : Kind(Kind), SubclassData(SubclassData), Loc(Loc) {
     assert(SubclassData < (1 << NumSubclassDataBits) &&
@@ -82,9 +89,6 @@ class MCExpr {
   /// \name Utility Methods
   /// @{
 
-  // TODO: Make this private. Users should call MCAsmInfo::printExpr instead.
-  LLVM_ABI void print(raw_ostream &OS, const MCAsmInfo *MAI,
-                      int SurroundingPrec = 0) const;
   LLVM_ABI void dump() const;
 
   /// @}
@@ -134,11 +138,6 @@ class MCExpr {
                                            MCValue &);
 };
 
-inline raw_ostream &operator<<(raw_ostream &OS, const MCExpr &E) {
-  E.print(OS, nullptr);
-  return OS;
-}
-
 ////  Represent a constant integer expression.
 class MCConstantExpr : public MCExpr {
   int64_t Value;
@@ -193,15 +192,10 @@ class MCSymbolRefExpr : public MCExpr {
   // VariantKind isn't ideal for encoding relocation operators because:
   // (a) other expressions, like MCConstantExpr (e.g., 4@l) and MCBinaryExpr
   // (e.g., (a+1)@l), also need it; (b) semantics become unclear (e.g., folding
-  // expressions with @). MCTargetExpr, as used by AArch64 and RISC-V, offers a
-  // cleaner approach.
+  // expressions with @). MCSpecifierExpr, as used by AArch64 and RISC-V, offers
+  // a cleaner approach.
   enum VariantKind : uint16_t {
-    VK_None,
-
-    VK_SECREL,
-    VK_WEAKREF, // The link between the symbols in .weakref foo, bar
-
-    VK_COFF_IMGREL32, // symbol@imgrel (image-relative)
+    VK_COFF_IMGREL32 = 3, // symbol@imgrel (image-relative)
 
     FirstTargetSpecifier,
   };
@@ -210,7 +204,7 @@ class MCSymbolRefExpr : public MCExpr {
   /// The symbol being referenced.
   const MCSymbol *Symbol;
 
-  explicit MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind,
+  explicit MCSymbolRefExpr(const MCSymbol *Symbol, Spec specifier,
                            const MCAsmInfo *MAI, SMLoc Loc = SMLoc());
 
 public:
@@ -219,17 +213,12 @@ class MCSymbolRefExpr : public MCExpr {
 
   static const MCSymbolRefExpr *create(const MCSymbol *Symbol, MCContext &Ctx,
                                        SMLoc Loc = SMLoc()) {
-    return MCSymbolRefExpr::create(Symbol, VK_None, Ctx, Loc);
+    return MCSymbolRefExpr::create(Symbol, 0, Ctx, Loc);
   }
 
   LLVM_ABI static const MCSymbolRefExpr *create(const MCSymbol *Symbol,
-                                                VariantKind Kind,
-                                                MCContext &Ctx,
+                                                Spec specifier, MCContext &Ctx,
                                                 SMLoc Loc = SMLoc());
-  static const MCSymbolRefExpr *create(const MCSymbol *Symbol, uint16_t Kind,
-                                       MCContext &Ctx, SMLoc Loc = SMLoc()) {
-    return MCSymbolRefExpr::create(Symbol, VariantKind(Kind), Ctx, Loc);
-  }
 
   /// @}
   /// \name Accessors
@@ -352,8 +341,8 @@ class MCBinaryExpr : public MCExpr {
                                              SMLoc Loc = SMLoc());
 
   static const MCBinaryExpr *createAdd(const MCExpr *LHS, const MCExpr *RHS,
-                                       MCContext &Ctx) {
-    return create(Add, LHS, RHS, Ctx);
+                                       MCContext &Ctx, SMLoc Loc = SMLoc()) {
+    return create(Add, LHS, RHS, Ctx, Loc);
   }
 
   static const MCBinaryExpr *createAnd(const MCExpr *LHS, const MCExpr *RHS,
@@ -505,7 +494,6 @@ class LLVM_ABI MCTargetExpr : public MCExpr {
 /// MCExprs are bump pointer allocated and not destructed.
 class LLVM_ABI MCSpecifierExpr : public MCExpr {
 protected:
-  using Spec = uint16_t;
   const MCExpr *Expr;
   // Target-specific relocation specifier code
   const Spec specifier;
diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index 7575202eaec57..4002a862e4d79 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_MC_MCFIXUP_H
 #define LLVM_MC_MCFIXUP_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SMLoc.h"
@@ -18,7 +19,8 @@ namespace llvm {
 class MCExpr;
 
 /// Extensible enumeration to represent the type of a fixup.
-enum MCFixupKind : uint16_t {
+using MCFixupKind = uint16_t;
+enum {
   // [0, FirstLiteralRelocationKind) encodes raw relocation types.
 
   // [FirstLiteralRelocationKind, FK_NONE) encodes raw relocation types coming
@@ -34,10 +36,6 @@ enum MCFixupKind : uint16_t {
   FK_Data_4,      ///< A four-byte fixup.
   FK_Data_8,      ///< A eight-byte fixup.
   FK_Data_leb128, ///< A leb128 fixup.
-  FK_PCRel_1,     ///< A one-byte pc relative fixup.
-  FK_PCRel_2,     ///< A two-byte pc relative fixup.
-  FK_PCRel_4,     ///< A four-byte pc relative fixup.
-  FK_PCRel_8,     ///< A eight-byte pc relative fixup.
   FK_SecRel_1,    ///< A one-byte section relative fixup.
   FK_SecRel_2,    ///< A two-byte section relative fixup.
   FK_SecRel_4,    ///< A four-byte section relative fixup.
@@ -73,31 +71,29 @@ class MCFixup {
   /// determine how the operand value should be encoded into the instruction.
   MCFixupKind Kind = FK_NONE;
 
+  /// True if this is a PC-relative fixup. The relocatable expression is
+  /// typically resolved When SymB is nullptr and SymA is a local symbol defined
+  /// within the current section.
+  bool PCRel = false;
+
   /// Used by RISC-V style linker relaxation. Whether the fixup is
   /// linker-relaxable.
   bool LinkerRelaxable = false;
 
   /// Consider bit fields if we need more flags.
 
-  /// The source location which gave rise to the fixup, if any.
-  SMLoc Loc;
 public:
-  static MCFixup create(uint32_t Offset, const MCExpr *Value,
-                        MCFixupKind Kind, SMLoc Loc = SMLoc()) {
+  static MCFixup create(uint32_t Offset, const MCExpr *Value, MCFixupKind Kind,
+                        bool PCRel = false) {
     MCFixup FI;
     FI.Value = Value;
     FI.Offset = Offset;
     FI.Kind = Kind;
-    FI.Loc = Loc;
+    FI.PCRel = PCRel;
     return FI;
   }
-  static MCFixup create(uint32_t Offset, const MCExpr *Value, unsigned Kind,
-                        SMLoc Loc = SMLoc()) {
-    return create(Offset, Value, MCFixupKind(Kind), Loc);
-  }
 
   MCFixupKind getKind() const { return Kind; }
-
   unsigned getTargetKind() const { return Kind; }
 
   uint32_t getOffset() const { return Offset; }
@@ -105,6 +101,8 @@ class MCFixup {
 
   const MCExpr *getValue() const { return Value; }
 
+  bool isPCRel() const { return PCRel; }
+  void setPCRel() { PCRel = true; }
   bool isLinkerRelaxable() const { return LinkerRelaxable; }
   void setLinkerRelaxable() { LinkerRelaxable = true; }
 
@@ -124,7 +122,7 @@ class MCFixup {
     }
   }
 
-  SMLoc getLoc() const { return Loc; }
+  LLVM_ABI SMLoc getLoc() const;
 };
 
 namespace mc {
diff --git a/llvm/include/llvm/MC/MCFixupKindInfo.h b/llvm/include/llvm/MC/MCFixupKindInfo.h
deleted file mode 100644
index f11cd5caa1d8c..0000000000000
--- a/llvm/include/llvm/MC/MCFixupKindInfo.h
+++ /dev/null
@@ -1,47 +0,0 @@
-//===-- llvm/MC/MCFixupKindInfo.h - Fixup Descriptors -----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCFIXUPKINDINFO_H
-#define LLVM_MC_MCFIXUPKINDINFO_H
-
-#include <cstdint>
-
-namespace llvm {
-
-/// Target independent information on a fixup kind.
-struct MCFixupKindInfo {
-  enum FixupKindFlags {
-    /// Is this fixup kind PCrelative? This is used by the assembler backend to
-    /// evaluate fixup values in a target independent manner when possible.
-    FKF_IsPCRel = (1 << 0),
-
-    /// Should this fixup kind force a 4-byte aligned effective PC value?
-    FKF_IsAlignedDownTo32Bits = (1 << 1),
-
-    /// Should this fixup be evaluated in a target dependent manner?
-    FKF_IsTarget = (1 << 2),
-  };
-
-  /// A target specific name for the fixup kind. The names will be unique for
-  /// distinct kinds on any given target.
-  const char *Name;
-
-  /// The bit offset to write the relocation into.
-  uint8_t TargetOffset;
-
-  /// The number of bits written by this fixup. The bits are assumed to be
-  /// contiguous.
-  uint8_t TargetSize;
-
-  /// Flags describing additional information on this fixup kind.
-  unsigned Flags;
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
deleted file mode 100644
index 473dec6477df6..0000000000000
--- a/llvm/include/llvm/MC/MCFragment.h
+++ /dev/null
@@ -1,576 +0,0 @@
-//===- MCFragment.h - Fragment type hierarchy -------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCFRAGMENT_H
-#define LLVM_MC_MCFRAGMENT_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/ilist_node.h"
-#include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/Support/Alignment.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/SMLoc.h"
-#include <cstdint>
-#include <utility>
-
-namespace llvm {
-
-class MCAssembler;
-class MCObjectStreamer;
-class MCSection;
-class MCSubtargetInfo;
-class MCSymbol;
-
-class MCFragment {
-  friend class MCAssembler;
-  friend class MCObjectStreamer;
-  friend class MCSection;
-
-public:
-  enum FragmentType : uint8_t {
-    FT_Align,
-    FT_Data,
-    FT_Fill,
-    FT_Nops,
-    FT_Relaxable,
-    FT_Org,
-    FT_Dwarf,
-    FT_DwarfFrame,
-    FT_LEB,
-    FT_BoundaryAlign,
-    FT_SymbolId,
-    FT_CVInlineLines,
-    FT_CVDefRange,
-    FT_PseudoProbe,
-  };
-
-private:
-  // The next fragment within the section.
-  MCFragment *Next = nullptr;
-
-  /// The data for the section this fragment is in.
-  MCSection *Parent = nullptr;
-
-  /// The offset of this fragment in its section.
-  uint64_t Offset = 0;
-
-  /// The layout order of this fragment.
-  unsigned LayoutOrder = 0;
-
-  FragmentType Kind;
-
-protected:
-  /// Used by subclasses for better packing.
-  ///
-  /// MCEncodedFragment
-  bool HasInstructions : 1;
-  bool AlignToBundleEnd : 1;
-  /// MCDataFragment
-  bool LinkerRelaxable : 1;
-  /// MCRelaxableFragment: x86-specific
-  bool AllowAutoPadding : 1;
-
-  LLVM_ABI MCFragment(FragmentType Kind, bool HasInstructions);
-
-public:
-  MCFragment() = delete;
-  MCFragment(const MCFragment &) = delete;
-  MCFragment &operator=(const MCFragment &) = delete;
-
-  /// Destroys the current fragment.
-  ///
-  /// This must be used instead of delete as MCFragment is non-virtual.
-  /// This method will dispatch to the appropriate subclass.
-  LLVM_ABI void destroy();
-
-  MCFragment *getNext() const { return Next; }
-
-  FragmentType getKind() const { return Kind; }
-
-  MCSection *getParent() const { return Parent; }
-  void setParent(MCSection *Value) { Parent = Value; }
-
-  LLVM_ABI const MCSymbol *getAtom() const;
-
-  unsigned getLayoutOrder() const { return LayoutOrder; }
-  void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
-
-  /// Does this fragment have instructions emitted into it? By default
-  /// this is false, but specific fragment types may set it to true.
-  bool hasInstructions() const { return HasInstructions; }
-
-  LLVM_ABI void dump() const;
-};
-
-/// Interface implemented by fragments that contain encoded instructions and/or
-/// data.
-///
-class MCEncodedFragment : public MCFragment {
-  uint8_t BundlePadding = 0;
-
-protected:
-  MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions)
-      : MCFragment(FType, HasInstructions) {}
-
-  /// The MCSubtargetInfo in effect when the instruction was encoded.
-  /// It must be non-null for instructions.
-  const MCSubtargetInfo *STI = nullptr;
-
-public:
-  static bool classof(const MCFragment *F) {
-    MCFragment::FragmentType Kind = F->getKind();
-    switch (Kind) {
-    default:
-      return false;
-    case MCFragment::FT_Relaxable:
-    case MCFragment::FT_Data:
-    case MCFragment::FT_Dwarf:
-    case MCFragment::FT_DwarfFrame:
-    case MCFragment::FT_PseudoProbe:
-      return true;
-    }
-  }
-
-  /// Should this fragment be placed at the end of an aligned bundle?
-  bool alignToBundleEnd() const { return AlignToBundleEnd; }
-  void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; }
-
-  /// Get the padding size that must be inserted before this fragment.
-  /// Used for bundling. By default, no padding is inserted.
-  /// Note that padding size is restricted to 8 bits. This is an optimization
-  /// to reduce the amount of space used for each fragment. In practice, larger
-  /// padding should never be required.
-  uint8_t getBundlePadding() const { return BundlePadding; }
-
-  /// Set the padding size for this fragment. By default it's a no-op,
-  /// and only some fragments have a meaningful implementation.
-  void setBundlePadding(uint8_t N) { BundlePadding = N; }
-
-  /// Retrieve the MCSubTargetInfo in effect when the instruction was encoded.
-  /// Guaranteed to be non-null if hasInstructions() == true
-  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
-
-  /// Record that the fragment contains instructions with the MCSubtargetInfo in
-  /// effect when the instruction was encoded.
-  void setHasInstructions(const MCSubtargetInfo &STI) {
-    HasInstructions = true;
-    this->STI = &STI;
-  }
-};
-
-/// Interface implemented by fragments that contain encoded instructions and/or
-/// data and also have fixups registered.
-///
-template <unsigned ContentsSize, unsigned FixupsSize>
-class MCEncodedFragmentWithFixups : public MCEncodedFragment {
-  SmallVector<char, ContentsSize> Contents;
-
-  /// The list of fixups in this fragment.
-  SmallVector<MCFixup, FixupsSize> Fixups;
-
-protected:
-  MCEncodedFragmentWithFixups(MCFragment::FragmentType FType,
-                              bool HasInstructions)
-      : MCEncodedFragment(FType, HasInstructions) {}
-
-public:
-  SmallVectorImpl<char> &getContents() { return Contents; }
-  const SmallVectorImpl<char> &getContents() const { return Contents; }
-
-  void appendContents(ArrayRef<char> C) { Contents.append(C.begin(), C.end()); }
-  void appendContents(size_t Num, char Elt) { Contents.append(Num, Elt); }
-  void setContents(ArrayRef<char> C) { Contents.assign(C.begin(), C.end()); }
-
-  SmallVectorImpl<MCFixup> &getFixups() { return Fixups; }
-  const SmallVectorImpl<MCFixup> &getFixups() const { return Fixups; }
-
-  static bool classof(const MCFragment *F) {
-    MCFragment::FragmentType Kind = F->getKind();
-    return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data ||
-           Kind == MCFragment::FT_CVDefRange || Kind == MCFragment::FT_Dwarf ||
-           Kind == MCFragment::FT_DwarfFrame;
-  }
-};
-
-/// Fragment for data and encoded instructions.
-///
-class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> {
-public:
-  MCDataFragment() : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false) {}
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Data;
-  }
-
-  bool isLinkerRelaxable() const { return LinkerRelaxable; }
-  void setLinkerRelaxable() { LinkerRelaxable = true; }
-};
-
-/// A relaxable fragment holds on to its MCInst, since it may need to be
-/// relaxed during the assembler layout and relaxation stage.
-///
-class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The instruction this is a fragment for.
-  MCInst Inst;
-
-public:
-  MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI)
-      : MCEncodedFragmentWithFixups(FT_Relaxable, true), Inst(Inst) {
-    this->STI = &STI;
-  }
-
-  const MCInst &getInst() const { return Inst; }
-  void setInst(const MCInst &Value) { Inst = Value; }
-
-  bool getAllowAutoPadding() const { return AllowAutoPadding; }
-  void setAllowAutoPadding(bool V) { AllowAutoPadding = V; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Relaxable;
-  }
-};
-
-class MCAlignFragment : public MCFragment {
-  /// The alignment to ensure, in bytes.
-  Align Alignment;
-
-  /// Flag to indicate that (optimal) NOPs should be emitted instead
-  /// of using the provided value. The exact interpretation of this flag is
-  /// target dependent.
-  bool EmitNops : 1;
-
-  /// Value to use for filling padding bytes.
-  int64_t Value;
-
-  /// The size of the integer (in bytes) of \p Value.
-  unsigned ValueSize;
-
-  /// The maximum number of bytes to emit; if the alignment
-  /// cannot be satisfied in this width then this fragment is ignored.
-  unsigned MaxBytesToEmit;
-
-  /// When emitting Nops some subtargets have specific nop encodings.
-  const MCSubtargetInfo *STI = nullptr;
-
-public:
-  MCAlignFragment(Align Alignment, int64_t Value, unsigned ValueSize,
-                  unsigned MaxBytesToEmit)
-      : MCFragment(FT_Align, false), Alignment(Alignment), EmitNops(false),
-        Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {}
-
-  Align getAlignment() const { return Alignment; }
-
-  int64_t getValue() const { return Value; }
-
-  unsigned getValueSize() const { return ValueSize; }
-
-  unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; }
-
-  bool hasEmitNops() const { return EmitNops; }
-  void setEmitNops(bool Value, const MCSubtargetInfo *STI) {
-    EmitNops = Value;
-    this->STI = STI;
-  }
-
-  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Align;
-  }
-};
-
-class MCFillFragment : public MCFragment {
-  uint8_t ValueSize;
-  /// Value to use for filling bytes.
-  uint64_t Value;
-  /// The number of bytes to insert.
-  const MCExpr &NumValues;
-  uint64_t Size = 0;
-
-  /// Source location of the directive that this fragment was created for.
-  SMLoc Loc;
-
-public:
-  MCFillFragment(uint64_t Value, uint8_t VSize, const MCExpr &NumValues,
-                 SMLoc Loc)
-      : MCFragment(FT_Fill, false), ValueSize(VSize), Value(Value),
-        NumValues(NumValues), Loc(Loc) {}
-
-  uint64_t getValue() const { return Value; }
-  uint8_t getValueSize() const { return ValueSize; }
-  const MCExpr &getNumValues() const { return NumValues; }
-  uint64_t getSize() const { return Size; }
-  void setSize(uint64_t Value) { Size = Value; }
-
-  SMLoc getLoc() const { return Loc; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Fill;
-  }
-};
-
-class MCNopsFragment : public MCFragment {
-  /// The number of bytes to insert.
-  int64_t Size;
-  /// Maximum number of bytes allowed in each NOP instruction.
-  int64_t ControlledNopLength;
-
-  /// Source location of the directive that this fragment was created for.
-  SMLoc Loc;
-
-  /// When emitting Nops some subtargets have specific nop encodings.
-  const MCSubtargetInfo &STI;
-
-public:
-  MCNopsFragment(int64_t NumBytes, int64_t ControlledNopLength, SMLoc L,
-                 const MCSubtargetInfo &STI)
-      : MCFragment(FT_Nops, false), Size(NumBytes),
-        ControlledNopLength(ControlledNopLength), Loc(L), STI(STI) {}
-
-  int64_t getNumBytes() const { return Size; }
-  int64_t getControlledNopLength() const { return ControlledNopLength; }
-
-  SMLoc getLoc() const { return Loc; }
-
-  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Nops;
-  }
-};
-
-class MCOrgFragment : public MCFragment {
-  /// Value to use for filling bytes.
-  int8_t Value;
-
-  /// The offset this fragment should start at.
-  const MCExpr *Offset;
-
-  /// Source location of the directive that this fragment was created for.
-  SMLoc Loc;
-
-public:
-  MCOrgFragment(const MCExpr &Offset, int8_t Value, SMLoc Loc)
-      : MCFragment(FT_Org, false), Value(Value), Offset(&Offset), Loc(Loc) {}
-
-  const MCExpr &getOffset() const { return *Offset; }
-
-  uint8_t getValue() const { return Value; }
-
-  SMLoc getLoc() const { return Loc; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Org;
-  }
-};
-
-class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> {
-  /// True if this is a sleb128, false if uleb128.
-  bool IsSigned;
-
-  /// The value this fragment should contain.
-  const MCExpr *Value;
-
-public:
-  MCLEBFragment(const MCExpr &Value, bool IsSigned)
-      : MCEncodedFragmentWithFixups<8, 0>(FT_LEB, false), IsSigned(IsSigned),
-        Value(&Value) {
-    getContents().push_back(0);
-  }
-
-  const MCExpr &getValue() const { return *Value; }
-  void setValue(const MCExpr *Expr) { Value = Expr; }
-
-  bool isSigned() const { return IsSigned; }
-
-  /// @}
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_LEB;
-  }
-};
-
-class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The value of the difference between the two line numbers
-  /// between two .loc dwarf directives.
-  int64_t LineDelta;
-
-  /// The expression for the difference of the two symbols that
-  /// make up the address delta between two .loc dwarf directives.
-  const MCExpr *AddrDelta;
-
-public:
-  MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta)
-      : MCEncodedFragmentWithFixups<8, 1>(FT_Dwarf, false),
-        LineDelta(LineDelta), AddrDelta(&AddrDelta) {}
-
-  int64_t getLineDelta() const { return LineDelta; }
-
-  const MCExpr &getAddrDelta() const { return *AddrDelta; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Dwarf;
-  }
-};
-
-class MCDwarfCallFrameFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The expression for the difference of the two symbols that
-  /// make up the address delta between two .cfi_* dwarf directives.
-  const MCExpr *AddrDelta;
-
-public:
-  MCDwarfCallFrameFragment(const MCExpr &AddrDelta)
-      : MCEncodedFragmentWithFixups<8, 1>(FT_DwarfFrame, false),
-        AddrDelta(&AddrDelta) {}
-
-  const MCExpr &getAddrDelta() const { return *AddrDelta; }
-  void setAddrDelta(const MCExpr *E) { AddrDelta = E; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_DwarfFrame;
-  }
-};
-
-/// Represents a symbol table index fragment.
-class MCSymbolIdFragment : public MCFragment {
-  const MCSymbol *Sym;
-
-public:
-  MCSymbolIdFragment(const MCSymbol *Sym)
-      : MCFragment(FT_SymbolId, false), Sym(Sym) {}
-
-  const MCSymbol *getSymbol() { return Sym; }
-  const MCSymbol *getSymbol() const { return Sym; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_SymbolId;
-  }
-};
-
-/// Fragment representing the binary annotations produced by the
-/// .cv_inline_linetable directive.
-class MCCVInlineLineTableFragment : public MCFragment {
-  unsigned SiteFuncId;
-  unsigned StartFileId;
-  unsigned StartLineNum;
-  const MCSymbol *FnStartSym;
-  const MCSymbol *FnEndSym;
-  SmallString<8> Contents;
-
-  /// CodeViewContext has the real knowledge about this format, so let it access
-  /// our members.
-  friend class CodeViewContext;
-
-public:
-  MCCVInlineLineTableFragment(unsigned SiteFuncId, unsigned StartFileId,
-                              unsigned StartLineNum, const MCSymbol *FnStartSym,
-                              const MCSymbol *FnEndSym)
-      : MCFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId),
-        StartFileId(StartFileId), StartLineNum(StartLineNum),
-        FnStartSym(FnStartSym), FnEndSym(FnEndSym) {}
-
-  const MCSymbol *getFnStartSym() const { return FnStartSym; }
-  const MCSymbol *getFnEndSym() const { return FnEndSym; }
-
-  SmallString<8> &getContents() { return Contents; }
-  const SmallString<8> &getContents() const { return Contents; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_CVInlineLines;
-  }
-};
-
-/// Fragment representing the .cv_def_range directive.
-class MCCVDefRangeFragment : public MCEncodedFragmentWithFixups<32, 4> {
-  SmallVector<std::pair<const MCSymbol *, const MCSymbol *>, 2> Ranges;
-  SmallString<32> FixedSizePortion;
-
-  /// CodeViewContext has the real knowledge about this format, so let it access
-  /// our members.
-  friend class CodeViewContext;
-
-public:
-  MCCVDefRangeFragment(
-      ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
-      StringRef FixedSizePortion)
-      : MCEncodedFragmentWithFixups<32, 4>(FT_CVDefRange, false),
-        Ranges(Ranges), FixedSizePortion(FixedSizePortion) {}
-
-  ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> getRanges() const {
-    return Ranges;
-  }
-
-  StringRef getFixedSizePortion() const { return FixedSizePortion.str(); }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_CVDefRange;
-  }
-};
-
-/// Represents required padding such that a particular other set of fragments
-/// does not cross a particular power-of-two boundary. The other fragments must
-/// follow this one within the same section.
-class MCBoundaryAlignFragment : public MCFragment {
-  /// The alignment requirement of the branch to be aligned.
-  Align AlignBoundary;
-  /// The last fragment in the set of fragments to be aligned.
-  const MCFragment *LastFragment = nullptr;
-  /// The size of the fragment.  The size is lazily set during relaxation, and
-  /// is not meaningful before that.
-  uint64_t Size = 0;
-
-  /// When emitting Nops some subtargets have specific nop encodings.
-  const MCSubtargetInfo &STI;
-
-public:
-  MCBoundaryAlignFragment(Align AlignBoundary, const MCSubtargetInfo &STI)
-      : MCFragment(FT_BoundaryAlign, false), AlignBoundary(AlignBoundary),
-        STI(STI) {}
-
-  uint64_t getSize() const { return Size; }
-  void setSize(uint64_t Value) { Size = Value; }
-
-  Align getAlignment() const { return AlignBoundary; }
-  void setAlignment(Align Value) { AlignBoundary = Value; }
-
-  const MCFragment *getLastFragment() const { return LastFragment; }
-  void setLastFragment(const MCFragment *F) {
-    assert(!F || getParent() == F->getParent());
-    LastFragment = F;
-  }
-
-  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_BoundaryAlign;
-  }
-};
-
-class MCPseudoProbeAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The expression for the difference of the two symbols that
-  /// make up the address delta between two .pseudoprobe directives.
-  const MCExpr *AddrDelta;
-
-public:
-  MCPseudoProbeAddrFragment(const MCExpr *AddrDelta)
-      : MCEncodedFragmentWithFixups<8, 1>(FT_PseudoProbe, false),
-        AddrDelta(AddrDelta) {}
-
-  const MCExpr &getAddrDelta() const { return *AddrDelta; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_PseudoProbe;
-  }
-};
-} // end namespace llvm
-
-#endif // LLVM_MC_MCFRAGMENT_H
diff --git a/llvm/include/llvm/MC/MCGOFFStreamer.h b/llvm/include/llvm/MC/MCGOFFStreamer.h
index b89d516e1452b..6d029f6bd4a29 100644
--- a/llvm/include/llvm/MC/MCGOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCGOFFStreamer.h
@@ -35,7 +35,6 @@ class MCGOFFStreamer : public MCObjectStreamer {
   }
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override {}
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override {}
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCInst.h b/llvm/include/llvm/MC/MCInst.h
index 581dc609b913f..b0db6b8408da5 100644
--- a/llvm/include/llvm/MC/MCInst.h
+++ b/llvm/include/llvm/MC/MCInst.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_MC_MCINST_H
 #define LLVM_MC_MCINST_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/bit.h"
@@ -27,6 +28,7 @@
 
 namespace llvm {
 
+class MCContext;
 class MCExpr;
 class MCInst;
 class MCInstPrinter;
@@ -175,8 +177,7 @@ class MCOperand {
     return Op;
   }
 
-  LLVM_ABI void print(raw_ostream &OS,
-                      const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void print(raw_ostream &OS, const MCContext *Ctx = nullptr) const;
   LLVM_ABI void dump() const;
   LLVM_ABI bool isBareSymbolRef() const;
   LLVM_ABI bool evaluateAsConstantImm(int64_t &Imm) const;
@@ -210,7 +211,11 @@ class MCInst {
   MCOperand &getOperand(unsigned i) { return Operands[i]; }
   unsigned getNumOperands() const { return Operands.size(); }
 
+  ArrayRef<MCOperand> getOperands() const { return Operands; }
   void addOperand(const MCOperand Op) { Operands.push_back(Op); }
+  void setOperands(ArrayRef<MCOperand> Ops) {
+    Operands.assign(Ops.begin(), Ops.end());
+  }
 
   using iterator = SmallVectorImpl<MCOperand>::iterator;
   using const_iterator = SmallVectorImpl<MCOperand>::const_iterator;
@@ -228,8 +233,7 @@ class MCInst {
     return Operands.insert(I, Op);
   }
 
-  LLVM_ABI void print(raw_ostream &OS,
-                      const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void print(raw_ostream &OS, const MCContext *Ctx = nullptr) const;
   LLVM_ABI void dump() const;
 
   /// Dump the MCInst as prettily as possible using the additional MC
@@ -238,10 +242,10 @@ class MCInst {
   LLVM_ABI void dump_pretty(raw_ostream &OS,
                             const MCInstPrinter *Printer = nullptr,
                             StringRef Separator = " ",
-                            const MCRegisterInfo *RegInfo = nullptr) const;
+                            const MCContext *Ctx = nullptr) const;
   LLVM_ABI void dump_pretty(raw_ostream &OS, StringRef Name,
                             StringRef Separator = " ",
-                            const MCRegisterInfo *RegInfo = nullptr) const;
+                            const MCContext *Ctx = nullptr) const;
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const MCOperand &MO) {
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index a6ccdf06eb91d..51e4df59bea40 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -191,8 +191,6 @@ class LLVM_ABI MachObjectWriter final : public MCObjectWriter {
   /// \name Utility Methods
   /// @{
 
-  bool isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind);
-
   std::vector<IndirectSymbolData> &getIndirectSymbols() {
     return IndirectSymbols;
   }
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index c987bc2426e9f..58c18af406158 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -12,7 +12,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 
@@ -60,7 +59,7 @@ class MCObjectStreamer : public MCStreamer {
   DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>>
       pendingAssignments;
 
-  virtual void emitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0;
+  virtual void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &);
   void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
   void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
   void emitInstructionImpl(const MCInst &Inst, const MCSubtargetInfo &STI);
@@ -126,7 +125,7 @@ class MCObjectStreamer : public MCStreamer {
 
   /// Emit an instruction to a special fragment, because this instruction
   /// can change its size during relaxation.
-  virtual void emitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &);
+  void emitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &);
 
   void emitBundleAlignMode(Align Alignment) override;
   void emitBundleLock(bool AlignToEnd) override;
diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index fd52fbac5199b..27ed2a6276496 100644
--- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -16,6 +16,7 @@
 
 namespace llvm {
 
+class MCAsmInfo;
 class MCRegister;
 class raw_ostream;
 
@@ -89,20 +90,11 @@ class LLVM_ABI MCParsedAsmOperand {
   virtual SMLoc getOffsetOfLoc() const { return SMLoc(); }
 
   /// print - Print a debug representation of the operand to the given stream.
-  virtual void print(raw_ostream &OS) const = 0;
+  virtual void print(raw_ostream &, const MCAsmInfo &) const = 0;
 
   /// dump - Print to the debug stream.
   virtual void dump() const;
 };
-
-//===----------------------------------------------------------------------===//
-// Debugging Support
-
-inline raw_ostream& operator<<(raw_ostream &OS, const MCParsedAsmOperand &MO) {
-  MO.print(OS);
-  return OS;
-}
-
 } // end namespace llvm
 
 #endif // LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H
diff --git a/llvm/include/llvm/MC/MCSPIRVStreamer.h b/llvm/include/llvm/MC/MCSPIRVStreamer.h
index ee0544147073f..42559f7522b50 100644
--- a/llvm/include/llvm/MC/MCSPIRVStreamer.h
+++ b/llvm/include/llvm/MC/MCSPIRVStreamer.h
@@ -35,9 +35,6 @@ class MCSPIRVStreamer : public MCObjectStreamer {
   }
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override {}
-
-private:
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7a94a7d65613d..d5bf6b97bfeae 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -13,8 +13,12 @@
 #ifndef LLVM_MC_MCSECTION_H
 #define LLVM_MC_MCSECTION_H
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/MC/MCFragment.h"
+#include "llvm/MC/MCFixup.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Compiler.h"
@@ -29,15 +33,101 @@ class MCContext;
 class MCExpr;
 class MCObjectStreamer;
 class MCSymbol;
+class MCSection;
+class MCSubtargetInfo;
 class raw_ostream;
 class Triple;
 
+// Represents a contiguous piece of code or data within a section. Its size is
+// determined by MCAssembler::layout. All subclasses must have trivial
+// destructors.
+//
+// Declaration order: MCFragment, MCSection, then MCFragment's derived classes.
+// This allows MCSection's inline functions to access MCFragment members and
+// allows MCFragment's derived classes to access MCSection.
+class MCFragment {
+  friend class MCAssembler;
+  friend class MCObjectStreamer;
+  friend class MCSection;
+
+public:
+  enum FragmentType : uint8_t {
+    FT_Data,
+    FT_Relaxable,
+    FT_Align,
+    FT_Fill,
+    FT_LEB,
+    FT_Nops,
+    FT_Org,
+    FT_Dwarf,
+    FT_DwarfFrame,
+    FT_BoundaryAlign,
+    FT_SymbolId,
+    FT_CVInlineLines,
+    FT_CVDefRange,
+    FT_PseudoProbe,
+  };
+
+private:
+  // The next fragment within the section.
+  MCFragment *Next = nullptr;
+
+  /// The data for the section this fragment is in.
+  MCSection *Parent = nullptr;
+
+  /// The offset of this fragment in its section.
+  uint64_t Offset = 0;
+
+  /// The layout order of this fragment.
+  unsigned LayoutOrder = 0;
+
+  FragmentType Kind;
+
+protected:
+  /// Used by subclasses for better packing.
+  ///
+  /// MCEncodedFragment
+  bool HasInstructions : 1;
+  bool AlignToBundleEnd : 1;
+  /// MCDataFragment
+  bool LinkerRelaxable : 1;
+  /// MCRelaxableFragment: x86-specific
+  bool AllowAutoPadding : 1;
+
+  LLVM_ABI MCFragment(FragmentType Kind, bool HasInstructions);
+
+public:
+  MCFragment() = delete;
+  MCFragment(const MCFragment &) = delete;
+  MCFragment &operator=(const MCFragment &) = delete;
+
+  MCFragment *getNext() const { return Next; }
+
+  FragmentType getKind() const { return Kind; }
+
+  MCSection *getParent() const { return Parent; }
+  void setParent(MCSection *Value) { Parent = Value; }
+
+  LLVM_ABI const MCSymbol *getAtom() const;
+
+  unsigned getLayoutOrder() const { return LayoutOrder; }
+  void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
+
+  /// Does this fragment have instructions emitted into it? By default
+  /// this is false, but specific fragment types may set it to true.
+  bool hasInstructions() const { return HasInstructions; }
+
+  LLVM_ABI void dump() const;
+};
+
 /// Instances of this class represent a uniqued identifier for a section in the
 /// current translation unit.  The MCContext class uniques and creates these.
 class LLVM_ABI MCSection {
 public:
   friend MCAssembler;
   friend MCObjectStreamer;
+  friend class MCEncodedFragment;
+  friend class MCRelaxableFragment;
   static constexpr unsigned NonUniqueID = ~0U;
 
   enum SectionVariant {
@@ -115,6 +205,11 @@ class LLVM_ABI MCSection {
   // subsections.
   SmallVector<std::pair<unsigned, FragList>, 1> Subsections;
 
+  // Content and fixup storage for fragments
+  SmallVector<char, 0> ContentStorage;
+  SmallVector<MCFixup, 0> FixupStorage;
+  SmallVector<MCOperand, 0> MCOperandStorage;
+
 protected:
   // TODO Make Name private when possible.
   StringRef Name;
@@ -122,7 +217,8 @@ class LLVM_ABI MCSection {
 
   MCSection(SectionVariant V, StringRef Name, bool IsText, bool IsVirtual,
             MCSymbol *Begin);
-  ~MCSection();
+  // Protected non-virtual dtor prevents destroy through a base class pointer.
+  ~MCSection() {}
 
 public:
   MCSection(const MCSection &) = delete;
@@ -182,7 +278,8 @@ class LLVM_ABI MCSection {
   iterator begin() const { return iterator(CurFragList->Head); }
   iterator end() const { return {}; }
 
-  void dump() const;
+  void dump(DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>>
+                *FragToSyms = nullptr) const;
 
   virtual void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
                                     raw_ostream &OS,
@@ -199,6 +296,504 @@ class LLVM_ABI MCSection {
   virtual StringRef getVirtualSectionKind() const;
 };
 
+/// Interface implemented by fragments that contain encoded instructions and/or
+/// data.
+class MCEncodedFragment : public MCFragment {
+  uint8_t BundlePadding = 0;
+  uint32_t ContentStart = 0;
+  uint32_t ContentEnd = 0;
+  uint32_t FixupStart = 0;
+  uint32_t FixupEnd = 0;
+
+protected:
+  MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions)
+      : MCFragment(FType, HasInstructions) {}
+
+  /// The MCSubtargetInfo in effect when the instruction was encoded.
+  /// It must be non-null for instructions.
+  const MCSubtargetInfo *STI = nullptr;
+
+public:
+  static bool classof(const MCFragment *F) {
+    MCFragment::FragmentType Kind = F->getKind();
+    switch (Kind) {
+    default:
+      return false;
+    case MCFragment::FT_Relaxable:
+    case MCFragment::FT_Data:
+    case MCFragment::FT_Dwarf:
+    case MCFragment::FT_DwarfFrame:
+    case MCFragment::FT_LEB:
+    case MCFragment::FT_PseudoProbe:
+    case MCFragment::FT_CVInlineLines:
+    case MCFragment::FT_CVDefRange:
+      return true;
+    }
+  }
+
+  /// Should this fragment be placed at the end of an aligned bundle?
+  bool alignToBundleEnd() const { return AlignToBundleEnd; }
+  void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; }
+
+  /// Get the padding size that must be inserted before this fragment.
+  /// Used for bundling. By default, no padding is inserted.
+  /// Note that padding size is restricted to 8 bits. This is an optimization
+  /// to reduce the amount of space used for each fragment. In practice, larger
+  /// padding should never be required.
+  uint8_t getBundlePadding() const { return BundlePadding; }
+
+  /// Set the padding size for this fragment. By default it's a no-op,
+  /// and only some fragments have a meaningful implementation.
+  void setBundlePadding(uint8_t N) { BundlePadding = N; }
+
+  /// Retrieve the MCSubTargetInfo in effect when the instruction was encoded.
+  /// Guaranteed to be non-null if hasInstructions() == true
+  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
+
+  /// Record that the fragment contains instructions with the MCSubtargetInfo in
+  /// effect when the instruction was encoded.
+  void setHasInstructions(const MCSubtargetInfo &STI) {
+    HasInstructions = true;
+    this->STI = &STI;
+  }
+
+  // Content-related functions manage parent's storage using ContentStart and
+  // ContentSize.
+  void clearContents() { ContentEnd = ContentStart; }
+  // Get a SmallVector reference. The caller should call doneAppending to update
+  // `ContentEnd`.
+  SmallVectorImpl<char> &getContentsForAppending() {
+    SmallVectorImpl<char> &S = getParent()->ContentStorage;
+    if (LLVM_UNLIKELY(ContentEnd != S.size())) {
+      // Move the elements to the end. Reserve space to avoid invalidating
+      // S.begin()+I for `append`.
+      auto Size = ContentEnd - ContentStart;
+      auto I = std::exchange(ContentStart, S.size());
+      S.reserve(S.size() + Size);
+      S.append(S.begin() + I, S.begin() + I + Size);
+    }
+    return S;
+  }
+  void doneAppending() { ContentEnd = getParent()->ContentStorage.size(); }
+  void appendContents(ArrayRef<char> Contents) {
+    getContentsForAppending().append(Contents.begin(), Contents.end());
+    doneAppending();
+  }
+  void appendContents(size_t Num, char Elt) {
+    getContentsForAppending().append(Num, Elt);
+    doneAppending();
+  }
+  LLVM_ABI void setContents(ArrayRef<char> Contents);
+  MutableArrayRef<char> getContents() {
+    return MutableArrayRef(getParent()->ContentStorage)
+        .slice(ContentStart, ContentEnd - ContentStart);
+  }
+  ArrayRef<char> getContents() const {
+    return ArrayRef(getParent()->ContentStorage)
+        .slice(ContentStart, ContentEnd - ContentStart);
+  }
+
+  // Fixup-related functions manage parent's storage using FixupStart and
+  // FixupSize.
+  void clearFixups() { FixupEnd = FixupStart; }
+  LLVM_ABI void addFixup(MCFixup Fixup);
+  LLVM_ABI void appendFixups(ArrayRef<MCFixup> Fixups);
+  LLVM_ABI void setFixups(ArrayRef<MCFixup> Fixups);
+  MutableArrayRef<MCFixup> getFixups() {
+    return MutableArrayRef(getParent()->FixupStorage)
+        .slice(FixupStart, FixupEnd - FixupStart);
+  }
+  ArrayRef<MCFixup> getFixups() const {
+    return ArrayRef(getParent()->FixupStorage)
+        .slice(FixupStart, FixupEnd - FixupStart);
+  }
+};
+
+/// Fragment for data and encoded instructions.
+///
+class MCDataFragment : public MCEncodedFragment {
+public:
+  MCDataFragment() : MCEncodedFragment(FT_Data, false) {}
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Data;
+  }
+
+  bool isLinkerRelaxable() const { return LinkerRelaxable; }
+  void setLinkerRelaxable() { LinkerRelaxable = true; }
+};
+
+/// A relaxable fragment holds on to its MCInst, since it may need to be
+/// relaxed during the assembler layout and relaxation stage.
+///
+class MCRelaxableFragment : public MCEncodedFragment {
+  uint32_t Opcode = 0;
+  uint32_t Flags = 0;
+  uint32_t OperandStart = 0;
+  uint32_t OperandSize = 0;
+
+public:
+  MCRelaxableFragment(const MCSubtargetInfo &STI)
+      : MCEncodedFragment(FT_Relaxable, true) {
+    this->STI = &STI;
+  }
+
+  unsigned getOpcode() const { return Opcode; }
+  ArrayRef<MCOperand> getOperands() const {
+    return MutableArrayRef(getParent()->MCOperandStorage)
+        .slice(OperandStart, OperandSize);
+  }
+  MCInst getInst() const {
+    MCInst Inst;
+    Inst.setOpcode(Opcode);
+    Inst.setFlags(Flags);
+    Inst.setOperands(ArrayRef(getParent()->MCOperandStorage)
+                         .slice(OperandStart, OperandSize));
+    return Inst;
+  }
+  void setInst(const MCInst &Inst) {
+    Opcode = Inst.getOpcode();
+    Flags = Inst.getFlags();
+    auto &S = getParent()->MCOperandStorage;
+    if (Inst.getNumOperands() > OperandSize) {
+      OperandStart = S.size();
+      S.resize_for_overwrite(S.size() + Inst.getNumOperands());
+    }
+    OperandSize = Inst.getNumOperands();
+    llvm::copy(Inst, S.begin() + OperandStart);
+  }
+
+  bool getAllowAutoPadding() const { return AllowAutoPadding; }
+  void setAllowAutoPadding(bool V) { AllowAutoPadding = V; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Relaxable;
+  }
+};
+
+class MCAlignFragment : public MCFragment {
+  /// The alignment to ensure, in bytes.
+  Align Alignment;
+
+  /// Flag to indicate that (optimal) NOPs should be emitted instead
+  /// of using the provided value. The exact interpretation of this flag is
+  /// target dependent.
+  bool EmitNops : 1;
+
+  /// Value to use for filling padding bytes.
+  int64_t Value;
+
+  /// The size of the integer (in bytes) of \p Value.
+  unsigned ValueSize;
+
+  /// The maximum number of bytes to emit; if the alignment
+  /// cannot be satisfied in this width then this fragment is ignored.
+  unsigned MaxBytesToEmit;
+
+  /// When emitting Nops some subtargets have specific nop encodings.
+  const MCSubtargetInfo *STI = nullptr;
+
+public:
+  MCAlignFragment(Align Alignment, int64_t Value, unsigned ValueSize,
+                  unsigned MaxBytesToEmit)
+      : MCFragment(FT_Align, false), Alignment(Alignment), EmitNops(false),
+        Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {}
+
+  Align getAlignment() const { return Alignment; }
+
+  int64_t getValue() const { return Value; }
+
+  unsigned getValueSize() const { return ValueSize; }
+
+  unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; }
+
+  bool hasEmitNops() const { return EmitNops; }
+  void setEmitNops(bool Value, const MCSubtargetInfo *STI) {
+    EmitNops = Value;
+    this->STI = STI;
+  }
+
+  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Align;
+  }
+};
+
+class MCFillFragment : public MCFragment {
+  uint8_t ValueSize;
+  /// Value to use for filling bytes.
+  uint64_t Value;
+  /// The number of bytes to insert.
+  const MCExpr &NumValues;
+  uint64_t Size = 0;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+public:
+  MCFillFragment(uint64_t Value, uint8_t VSize, const MCExpr &NumValues,
+                 SMLoc Loc)
+      : MCFragment(FT_Fill, false), ValueSize(VSize), Value(Value),
+        NumValues(NumValues), Loc(Loc) {}
+
+  uint64_t getValue() const { return Value; }
+  uint8_t getValueSize() const { return ValueSize; }
+  const MCExpr &getNumValues() const { return NumValues; }
+  uint64_t getSize() const { return Size; }
+  void setSize(uint64_t Value) { Size = Value; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Fill;
+  }
+};
+
+class MCNopsFragment : public MCFragment {
+  /// The number of bytes to insert.
+  int64_t Size;
+  /// Maximum number of bytes allowed in each NOP instruction.
+  int64_t ControlledNopLength;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+  /// When emitting Nops some subtargets have specific nop encodings.
+  const MCSubtargetInfo &STI;
+
+public:
+  MCNopsFragment(int64_t NumBytes, int64_t ControlledNopLength, SMLoc L,
+                 const MCSubtargetInfo &STI)
+      : MCFragment(FT_Nops, false), Size(NumBytes),
+        ControlledNopLength(ControlledNopLength), Loc(L), STI(STI) {}
+
+  int64_t getNumBytes() const { return Size; }
+  int64_t getControlledNopLength() const { return ControlledNopLength; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Nops;
+  }
+};
+
+class MCOrgFragment : public MCFragment {
+  /// Value to use for filling bytes.
+  int8_t Value;
+
+  /// The offset this fragment should start at.
+  const MCExpr *Offset;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+public:
+  MCOrgFragment(const MCExpr &Offset, int8_t Value, SMLoc Loc)
+      : MCFragment(FT_Org, false), Value(Value), Offset(&Offset), Loc(Loc) {}
+
+  const MCExpr &getOffset() const { return *Offset; }
+
+  uint8_t getValue() const { return Value; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Org;
+  }
+};
+
+class MCLEBFragment final : public MCEncodedFragment {
+  /// True if this is a sleb128, false if uleb128.
+  bool IsSigned;
+
+  /// The value this fragment should contain.
+  const MCExpr *Value;
+
+public:
+  MCLEBFragment(const MCExpr &Value, bool IsSigned)
+      : MCEncodedFragment(FT_LEB, false), IsSigned(IsSigned), Value(&Value) {}
+
+  const MCExpr &getValue() const { return *Value; }
+  void setValue(const MCExpr *Expr) { Value = Expr; }
+
+  bool isSigned() const { return IsSigned; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_LEB;
+  }
+};
+
+class MCDwarfLineAddrFragment : public MCEncodedFragment {
+  /// The value of the difference between the two line numbers
+  /// between two .loc dwarf directives.
+  int64_t LineDelta;
+
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .loc dwarf directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta)
+      : MCEncodedFragment(FT_Dwarf, false), LineDelta(LineDelta),
+        AddrDelta(&AddrDelta) {}
+
+  int64_t getLineDelta() const { return LineDelta; }
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Dwarf;
+  }
+};
+
+class MCDwarfCallFrameFragment : public MCEncodedFragment {
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .cfi_* dwarf directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCDwarfCallFrameFragment(const MCExpr &AddrDelta)
+      : MCEncodedFragment(FT_DwarfFrame, false), AddrDelta(&AddrDelta) {}
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+  void setAddrDelta(const MCExpr *E) { AddrDelta = E; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_DwarfFrame;
+  }
+};
+
+/// Represents a symbol table index fragment.
+class MCSymbolIdFragment : public MCFragment {
+  const MCSymbol *Sym;
+
+public:
+  MCSymbolIdFragment(const MCSymbol *Sym)
+      : MCFragment(FT_SymbolId, false), Sym(Sym) {}
+
+  const MCSymbol *getSymbol() { return Sym; }
+  const MCSymbol *getSymbol() const { return Sym; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_SymbolId;
+  }
+};
+
+/// Fragment representing the binary annotations produced by the
+/// .cv_inline_linetable directive.
+class MCCVInlineLineTableFragment : public MCEncodedFragment {
+  unsigned SiteFuncId;
+  unsigned StartFileId;
+  unsigned StartLineNum;
+  const MCSymbol *FnStartSym;
+  const MCSymbol *FnEndSym;
+
+  /// CodeViewContext has the real knowledge about this format, so let it access
+  /// our members.
+  friend class CodeViewContext;
+
+public:
+  MCCVInlineLineTableFragment(unsigned SiteFuncId, unsigned StartFileId,
+                              unsigned StartLineNum, const MCSymbol *FnStartSym,
+                              const MCSymbol *FnEndSym)
+      : MCEncodedFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId),
+        StartFileId(StartFileId), StartLineNum(StartLineNum),
+        FnStartSym(FnStartSym), FnEndSym(FnEndSym) {}
+
+  const MCSymbol *getFnStartSym() const { return FnStartSym; }
+  const MCSymbol *getFnEndSym() const { return FnEndSym; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_CVInlineLines;
+  }
+};
+
+/// Fragment representing the .cv_def_range directive.
+class MCCVDefRangeFragment : public MCEncodedFragment {
+  ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges;
+  StringRef FixedSizePortion;
+
+  /// CodeViewContext has the real knowledge about this format, so let it access
+  /// our members.
+  friend class CodeViewContext;
+
+public:
+  MCCVDefRangeFragment(
+      ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
+      StringRef FixedSizePortion)
+      : MCEncodedFragment(FT_CVDefRange, false),
+        Ranges(Ranges.begin(), Ranges.end()),
+        FixedSizePortion(FixedSizePortion) {}
+
+  ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> getRanges() const {
+    return Ranges;
+  }
+
+  StringRef getFixedSizePortion() const { return FixedSizePortion; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_CVDefRange;
+  }
+};
+
+/// Represents required padding such that a particular other set of fragments
+/// does not cross a particular power-of-two boundary. The other fragments must
+/// follow this one within the same section.
+class MCBoundaryAlignFragment : public MCFragment {
+  /// The alignment requirement of the branch to be aligned.
+  Align AlignBoundary;
+  /// The last fragment in the set of fragments to be aligned.
+  const MCFragment *LastFragment = nullptr;
+  /// The size of the fragment.  The size is lazily set during relaxation, and
+  /// is not meaningful before that.
+  uint64_t Size = 0;
+
+  /// When emitting Nops some subtargets have specific nop encodings.
+  const MCSubtargetInfo &STI;
+
+public:
+  MCBoundaryAlignFragment(Align AlignBoundary, const MCSubtargetInfo &STI)
+      : MCFragment(FT_BoundaryAlign, false), AlignBoundary(AlignBoundary),
+        STI(STI) {}
+
+  uint64_t getSize() const { return Size; }
+  void setSize(uint64_t Value) { Size = Value; }
+
+  Align getAlignment() const { return AlignBoundary; }
+  void setAlignment(Align Value) { AlignBoundary = Value; }
+
+  const MCFragment *getLastFragment() const { return LastFragment; }
+  void setLastFragment(const MCFragment *F) {
+    assert(!F || getParent() == F->getParent());
+    LastFragment = F;
+  }
+
+  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_BoundaryAlign;
+  }
+};
+
+class MCPseudoProbeAddrFragment : public MCEncodedFragment {
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .pseudoprobe directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCPseudoProbeAddrFragment(const MCExpr *AddrDelta)
+      : MCEncodedFragment(FT_PseudoProbe, false), AddrDelta(AddrDelta) {}
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_PseudoProbe;
+  }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_MC_MCSECTION_H
diff --git a/llvm/include/llvm/MC/MCSectionGOFF.h b/llvm/include/llvm/MC/MCSectionGOFF.h
index 1e6a331202f3c..9e3f95e82a14c 100644
--- a/llvm/include/llvm/MC/MCSectionGOFF.h
+++ b/llvm/include/llvm/MC/MCSectionGOFF.h
@@ -18,6 +18,7 @@
 #include "llvm/BinaryFormat/GOFF.h"
 #include "llvm/MC/MCGOFFAttributes.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -25,7 +26,7 @@ namespace llvm {
 
 class MCExpr;
 
-class MCSectionGOFF final : public MCSection {
+class LLVM_ABI MCSectionGOFF final : public MCSection {
   // Parent of this section. Implies that the parent is emitted first.
   MCSectionGOFF *Parent;
 
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 8f2e137ea0c84..00bec1f582eab 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -19,9 +19,9 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCPseudoProbe.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCWinEH.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
@@ -45,11 +45,9 @@ class MCAsmBackend;
 class MCAssembler;
 class MCContext;
 class MCExpr;
-class MCFragment;
 class MCInst;
 class MCInstPrinter;
 class MCRegister;
-class MCSection;
 class MCStreamer;
 class MCSubtargetInfo;
 class MCSymbol;
diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index b02bc111955ab..dc191f881921a 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -16,7 +16,7 @@
 #include "llvm/ADT/StringMapEntry.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSymbolTableEntry.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/llvm/include/llvm/MC/MCValue.h b/llvm/include/llvm/MC/MCValue.h
index 417c4689d1b47..c9a680ec327a6 100644
--- a/llvm/include/llvm/MC/MCValue.h
+++ b/llvm/include/llvm/MC/MCValue.h
@@ -42,6 +42,7 @@ class MCValue {
   friend class MCExpr;
   MCValue() = default;
   int64_t getConstant() const { return Cst; }
+  void setConstant(int64_t C) { Cst = C; }
   uint32_t getSpecifier() const { return Specifier; }
   void setSpecifier(uint32_t S) { Specifier = S; }
 
diff --git a/llvm/include/llvm/MC/MCWasmStreamer.h b/llvm/include/llvm/MC/MCWasmStreamer.h
index bf5b5f6c3447d..2598c261ea02a 100644
--- a/llvm/include/llvm/MC/MCWasmStreamer.h
+++ b/llvm/include/llvm/MC/MCWasmStreamer.h
@@ -58,8 +58,6 @@ class MCWasmStreamer : public MCObjectStreamer {
   void finishImpl() override;
 
 private:
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
-
   bool SeenIdent;
 };
 
diff --git a/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
index b474ececb21d6..05def07a7d9dd 100644
--- a/llvm/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
@@ -74,8 +74,6 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
 protected:
   const MCSymbol *CurSymbol;
 
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &STI) override;
-
   void finalizeCGProfileEntry(const MCSymbolRefExpr *&S);
 
 private:
diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h
index 684c229407a80..870d48fe4200c 100644
--- a/llvm/include/llvm/MC/MCXCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h
@@ -25,7 +25,6 @@ class MCXCOFFStreamer : public MCObjectStreamer {
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override;
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
   void emitXCOFFLocalCommonSymbol(MCSymbol *LabelSym, uint64_t Size,
                                   MCSymbol *CsectSym, Align Alignment) override;
   void emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol,
diff --git a/llvm/include/llvm/MCA/CodeEmitter.h b/llvm/include/llvm/MCA/CodeEmitter.h
index cc978c78244fd..c28a5cd2fec5c 100644
--- a/llvm/include/llvm/MCA/CodeEmitter.h
+++ b/llvm/include/llvm/MCA/CodeEmitter.h
@@ -34,7 +34,6 @@ namespace mca {
 /// strings. Encodings are cached internally for later usage.
 class CodeEmitter {
   const MCSubtargetInfo &STI;
-  const MCAsmBackend &MAB;
   const MCCodeEmitter &MCE;
 
   SmallString<256> Code;
@@ -52,7 +51,7 @@ class CodeEmitter {
 public:
   CodeEmitter(const MCSubtargetInfo &ST, const MCAsmBackend &AB,
               const MCCodeEmitter &CE, ArrayRef<MCInst> S)
-      : STI(ST), MAB(AB), MCE(CE), Sequence(S), Encodings(S.size()) {}
+      : STI(ST), MCE(CE), Sequence(S), Encodings(S.size()) {}
 
   StringRef getEncoding(unsigned MCID) {
     EncodingInfo EI = getOrCreateEncodingInfo(MCID);
diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index c235112dacf7c..7e0a4c6b07039 100644
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -87,7 +87,7 @@ struct RootDescriptorYaml {
 
   LLVM_ABI uint32_t getEncodedFlags() const;
 
-#define ROOT_DESCRIPTOR_FLAG(Num, Val) bool Val = false;
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag) bool Enum = false;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
@@ -100,7 +100,7 @@ struct DescriptorRangeYaml {
 
   LLVM_ABI uint32_t getEncodedFlags() const;
 
-#define DESCRIPTOR_RANGE_FLAG(Num, Val) bool Val = false;
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag) bool Enum = false;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
@@ -165,16 +165,16 @@ struct RootParameterYamlDesc {
 };
 
 struct StaticSamplerYamlDesc {
-  uint32_t Filter = llvm::to_underlying(dxbc::StaticSamplerFilter::ANISOTROPIC);
+  uint32_t Filter = llvm::to_underlying(dxbc::SamplerFilter::Anisotropic);
   uint32_t AddressU = llvm::to_underlying(dxbc::TextureAddressMode::Wrap);
   uint32_t AddressV = llvm::to_underlying(dxbc::TextureAddressMode::Wrap);
   uint32_t AddressW = llvm::to_underlying(dxbc::TextureAddressMode::Wrap);
   float MipLODBias = 0.f;
   uint32_t MaxAnisotropy = 16u;
   uint32_t ComparisonFunc =
-      llvm::to_underlying(dxbc::SamplersComparisonFunction::LessEqual);
+      llvm::to_underlying(dxbc::ComparisonFunc::LessEqual);
   uint32_t BorderColor =
-      llvm::to_underlying(dxbc::SamplersBorderColor::OpaqueWhite);
+      llvm::to_underlying(dxbc::StaticBorderColor::OpaqueWhite);
   float MinLOD = 0.f;
   float MaxLOD = std::numeric_limits<float>::max();
   uint32_t ShaderRegister;
@@ -203,7 +203,7 @@ struct RootSignatureYamlDesc {
   LLVM_ABI static llvm::Expected<DXContainerYAML::RootSignatureYamlDesc>
   create(const object::DirectX::RootSignature &Data);
 
-#define ROOT_ELEMENT_FLAG(Num, Val) bool Val = false;
+#define ROOT_SIGNATURE_FLAG(Num, Val) bool Val = false;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h
index 34f4f7f804a38..2ecd47dd10bde 100644
--- a/llvm/include/llvm/Pass.h
+++ b/llvm/include/llvm/Pass.h
@@ -271,7 +271,7 @@ class LLVM_ABI ModulePass : public Pass {
 protected:
   /// Optional passes call this function to check whether the pass should be
   /// skipped. This is the case when optimization bisect is over the limit.
-  bool skipModule(Module &M) const;
+  bool skipModule(const Module &M) const;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 09dc7a2944159..4e1f1131b710f 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -289,8 +289,8 @@ struct LineLocation {
   LLVM_ABI void serialize(raw_ostream &OS);
 
   bool operator<(const LineLocation &O) const {
-    return LineOffset < O.LineOffset ||
-           (LineOffset == O.LineOffset && Discriminator < O.Discriminator);
+    return std::tie(LineOffset, Discriminator) <
+           std::tie(O.LineOffset, O.Discriminator);
   }
 
   bool operator==(const LineLocation &O) const {
diff --git a/llvm/include/llvm/SandboxIR/Value.h b/llvm/include/llvm/SandboxIR/Value.h
index dd0bc76db3e37..8e50614d6a6d0 100644
--- a/llvm/include/llvm/SandboxIR/Value.h
+++ b/llvm/include/llvm/SandboxIR/Value.h
@@ -33,6 +33,7 @@ class IntrinsicInst;
 class Operator;
 class OverflowingBinaryOperator;
 class FPMathOperator;
+class Region;
 
 /// Iterator for the `Use` edges of a Value's users.
 /// \Returns a `Use` when dereferenced.
diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h
index 5489b3ff78f51..adaa75cc6c348 100644
--- a/llvm/include/llvm/Support/CommandLine.h
+++ b/llvm/include/llvm/Support/CommandLine.h
@@ -1496,6 +1496,12 @@ class opt
     return this->getValue();
   }
 
+  template <class T> DataType &operator=(T &&Val) {
+    this->getValue() = std::forward<T>(Val);
+    Callback(this->getValue());
+    return this->getValue();
+  }
+
   template <class... Mods>
   explicit opt(const Mods &... Ms)
       : Option(llvm::cl::Optional, NotHidden), Parser(*this) {
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index f46a7aeccd0aa..417948dd45887 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -133,7 +133,7 @@
 #endif
 
 #if (!(defined(_WIN32) || defined(__CYGWIN__)) ||                              \
-     (defined(__MINGW32__) && defined(__clang__)))
+     ((defined(__MINGW32__) || defined(__CYGWIN__)) && defined(__clang__)))
 #define LLVM_LIBRARY_VISIBILITY LLVM_ATTRIBUTE_VISIBILITY_HIDDEN
 // Clang compilers older then 15 do not support gnu style attributes on
 // namespaces.
@@ -171,10 +171,6 @@
 /// for both functions and classes. On windows its turned in to dllimport for
 /// library consumers, for other platforms its a default visibility attribute.
 ///
-/// LLVM_C_ABI is used to annotated functions and data that need to be exported
-/// for the libllvm-c API. This used both for the llvm-c headers and for the
-/// functions declared in the different Target's c++ source files that don't
-/// include the header forward declaring them.
 #ifndef LLVM_ABI_GENERATING_ANNOTATIONS
 // Marker to add to classes or functions in public headers that should not have
 // export macros added to them by the clang tool
@@ -214,7 +210,6 @@
 #define LLVM_EXPORT_TEMPLATE
 #define LLVM_ABI_EXPORT
 #endif
-#define LLVM_C_ABI LLVM_ABI
 #endif
 
 #if defined(__GNUC__)
diff --git a/llvm/include/llvm/Support/DataExtractor.h b/llvm/include/llvm/Support/DataExtractor.h
index 1f7e45d43ca7a..3792f53407dd9 100644
--- a/llvm/include/llvm/Support/DataExtractor.h
+++ b/llvm/include/llvm/Support/DataExtractor.h
@@ -407,6 +407,18 @@ class DataExtractor {
     getU8(C, Dst.data(), Count);
   }
 
+  /// Extract a int8_t value from \a *OffsetPtr. In case of an extraction error,
+  /// or if error is already set, zero is returned and the offset is left
+  /// unmodified.
+  int8_t getS8(uint64_t *OffsetPtr, Error *Err = nullptr) const {
+    return static_cast<int8_t>(getU8(OffsetPtr, Err));
+  }
+
+  /// Extract a int8_t value from \a *OffsetPtr. In case of an extraction error,
+  /// or if the cursor is already in an error state, zero is returned and the
+  /// offset is left unmodified.
+  int8_t getS8(Cursor &C) const { return static_cast<int8_t>(getU8(C)); }
+
   //------------------------------------------------------------------
   /// Extract a uint16_t value from \a *offset_ptr.
   ///
@@ -462,6 +474,18 @@ class DataExtractor {
   LLVM_ABI uint16_t *getU16(uint64_t *offset_ptr, uint16_t *dst,
                             uint32_t count) const;
 
+  /// Extract a int16_t value from \a *OffsetPtr. In case of an extraction
+  /// error, or if error is already set, zero is returned and the offset is left
+  /// unmodified.
+  int16_t getS16(uint64_t *OffsetPtr, Error *Err = nullptr) const {
+    return static_cast<int16_t>(getU16(OffsetPtr, Err));
+  }
+
+  /// Extract a int16_t value from \a *OffsetPtr. In case of an extraction
+  /// error, or if the cursor is already in an error state, zero is returned and
+  /// the offset is left unmodified.
+  int16_t getS16(Cursor &C) const { return static_cast<int16_t>(getU16(C)); }
+
   /// Extract a 24-bit unsigned value from \a *offset_ptr and return it
   /// in a uint32_t.
   ///
@@ -543,6 +567,18 @@ class DataExtractor {
   LLVM_ABI uint32_t *getU32(uint64_t *offset_ptr, uint32_t *dst,
                             uint32_t count) const;
 
+  /// Extract a int32_t value from \a *OffsetPtr. In case of an extraction
+  /// error, or if error is already set, zero is returned and the offset is left
+  /// unmodified.
+  int32_t getS32(uint64_t *OffsetPtr, Error *Err = nullptr) const {
+    return static_cast<int32_t>(getU32(OffsetPtr, Err));
+  }
+
+  /// Extract a int32_t value from \a *OffsetPtr. In case of an extraction
+  /// error, or if the cursor is already in an error state, zero is returned and
+  /// the offset is left unmodified.
+  int32_t getS32(Cursor &C) const { return static_cast<int32_t>(getU32(C)); }
+
   /// Extract a uint64_t value from \a *offset_ptr.
   ///
   /// Extract a single uint64_t from the binary data at the offset
@@ -596,6 +632,18 @@ class DataExtractor {
   LLVM_ABI uint64_t *getU64(uint64_t *offset_ptr, uint64_t *dst,
                             uint32_t count) const;
 
+  /// Extract a int64_t value from \a *OffsetPtr. In case of an extraction
+  /// error, or if error is already set, zero is returned and the offset is left
+  /// unmodified.
+  int64_t getS64(uint64_t *OffsetPtr, Error *Err = nullptr) const {
+    return static_cast<int64_t>(getU64(OffsetPtr, Err));
+  }
+
+  /// Extract a int64_t value from \a *OffsetPtr. In case of an extraction
+  /// error, or if the cursor is already in an error state, zero is returned and
+  /// the offset is left unmodified.
+  int64_t getS64(Cursor &C) const { return static_cast<int64_t>(getU64(C)); }
+
   /// Extract a signed LEB128 value from \a *offset_ptr.
   ///
   /// Extracts an signed LEB128 number from this object's data
diff --git a/llvm/include/llvm/Support/Path.h b/llvm/include/llvm/Support/Path.h
index 0d8881359b806..0cb517146c04b 100644
--- a/llvm/include/llvm/Support/Path.h
+++ b/llvm/include/llvm/Support/Path.h
@@ -223,6 +223,8 @@ LLVM_ABI bool remove_dots(SmallVectorImpl<char> &path,
 ///   /foo  + bar/f => /foo/bar/f
 ///   /foo/ + bar/f => /foo/bar/f
 ///   foo   + bar/f => foo/bar/f
+///   foo   + /bar/f => foo/bar/f (FIXME: will be changed to /bar/f to align
+///                                with C++17 std::filesystem::path::append)
 /// @endcode
 ///
 /// @param path Set to \a path + \a component.
diff --git a/llvm/include/llvm/Support/Signals.h b/llvm/include/llvm/Support/Signals.h
index 6ce26acdd458e..21b425fffef53 100644
--- a/llvm/include/llvm/Support/Signals.h
+++ b/llvm/include/llvm/Support/Signals.h
@@ -14,10 +14,23 @@
 #ifndef LLVM_SUPPORT_SIGNALS_H
 #define LLVM_SUPPORT_SIGNALS_H
 
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <string>
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+namespace llvm {
+// Typedefs that are convenient but only used by the stack-trace-collection code
+// added if DebugLoc origin-tracking is enabled.
+using AddressSet = DenseSet<void *>;
+using SymbolizedAddressMap = DenseMap<void *, SmallVector<std::string, 0>>;
+} // namespace llvm
+#endif
+
 namespace llvm {
 class StringRef;
 class raw_ostream;
@@ -57,6 +70,28 @@ LLVM_ABI void DisableSystemDialogsOnCrash();
 ///        specified, the entire frame is printed.
 LLVM_ABI void PrintStackTrace(raw_ostream &OS, int Depth = 0);
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#ifdef NDEBUG
+#error DebugLoc origin-tracking should not be enabled in Release builds.
+#endif
+/// Populates the given array with a stack trace of the current program, up to
+/// MaxDepth frames. Returns the number of frames returned, which will be
+/// inserted into \p StackTrace from index 0. All entries after the returned
+/// depth will be unmodified. NB: This is only intended to be used for
+/// introspection of LLVM by Debugify, will not be enabled in release builds,
+/// and should not be relied on for other purposes.
+template <unsigned long MaxDepth>
+int getStackTrace(std::array<void *, MaxDepth> &StackTrace);
+
+/// Takes a set of \p Addresses, symbolizes them and stores the result in the
+/// provided \p SymbolizedAddresses map.
+/// NB: This is only intended to be used for introspection of LLVM by
+/// Debugify, will not be enabled in release builds, and should not be relied
+/// on for other purposes.
+void symbolizeAddresses(AddressSet &Addresses,
+                        SymbolizedAddressMap &SymbolizedAddresses);
+#endif
+
 // Run all registered signal handlers.
 LLVM_ABI void RunSignalHandlers();
 
diff --git a/llvm/include/llvm/Support/TrailingObjects.h b/llvm/include/llvm/Support/TrailingObjects.h
index f25f2311a81a4..d7211a930ae49 100644
--- a/llvm/include/llvm/Support/TrailingObjects.h
+++ b/llvm/include/llvm/Support/TrailingObjects.h
@@ -228,12 +228,18 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
 
   using ParentType::getTrailingObjectsImpl;
 
-  // This function contains only a static_assert BaseTy is final. The
-  // static_assert must be in a function, and not at class-level
-  // because BaseTy isn't complete at class instantiation time, but
-  // will be by the time this function is instantiated.
-  static void verifyTrailingObjectsAssertions() {
+  template <bool Strict> static void verifyTrailingObjectsAssertions() {
+    // The static_assert for BaseTy must be in a function, and not at
+    // class-level  because BaseTy isn't complete at class instantiation time,
+    // but will be by the time this function is instantiated.
     static_assert(std::is_final<BaseTy>(), "BaseTy must be final.");
+
+    // Verify that templated getTrailingObjects() is used only with multiple
+    // trailing types. Use getTrailingObjectsNonStrict() which does not check
+    // this.
+    static_assert(!Strict || sizeof...(TrailingTys) > 1,
+                  "Use templated getTrailingObjects() only when there are "
+                  "multiple trailing types");
   }
 
   // These two methods are the base of the recursion for this method.
@@ -283,7 +289,7 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
   /// (which must be one of those specified in the class template). The
   /// array may have zero or more elements in it.
   template <typename T> const T *getTrailingObjects() const {
-    verifyTrailingObjectsAssertions();
+    verifyTrailingObjectsAssertions<true>();
     // Forwards to an impl function with overloads, since member
     // function templates can't be specialized.
     return this->getTrailingObjectsImpl(
@@ -295,7 +301,7 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
   /// (which must be one of those specified in the class template). The
   /// array may have zero or more elements in it.
   template <typename T> T *getTrailingObjects() {
-    verifyTrailingObjectsAssertions();
+    verifyTrailingObjectsAssertions<true>();
     // Forwards to an impl function with overloads, since member
     // function templates can't be specialized.
     return this->getTrailingObjectsImpl(
@@ -310,14 +316,20 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
     static_assert(sizeof...(TrailingTys) == 1,
                   "Can use non-templated getTrailingObjects() only when there "
                   "is a single trailing type");
-    return getTrailingObjects<FirstTrailingType>();
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<const BaseTy *>(this),
+        TrailingObjectsBase::OverloadToken<FirstTrailingType>());
   }
 
   FirstTrailingType *getTrailingObjects() {
     static_assert(sizeof...(TrailingTys) == 1,
                   "Can use non-templated getTrailingObjects() only when there "
                   "is a single trailing type");
-    return getTrailingObjects<FirstTrailingType>();
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<BaseTy *>(this),
+        TrailingObjectsBase::OverloadToken<FirstTrailingType>());
   }
 
   // Functions that return the trailing objects as ArrayRefs.
@@ -337,6 +349,31 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
     return ArrayRef(getTrailingObjects(), N);
   }
 
+  // Non-strict forms of templated `getTrailingObjects` that work with single
+  // trailing type.
+  template <typename T> const T *getTrailingObjectsNonStrict() const {
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<const BaseTy *>(this),
+        TrailingObjectsBase::OverloadToken<T>());
+  }
+
+  template <typename T> T *getTrailingObjectsNonStrict() {
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<BaseTy *>(this), TrailingObjectsBase::OverloadToken<T>());
+  }
+
+  template <typename T>
+  MutableArrayRef<T> getTrailingObjectsNonStrict(size_t N) {
+    return MutableArrayRef(getTrailingObjectsNonStrict<T>(), N);
+  }
+
+  template <typename T>
+  ArrayRef<T> getTrailingObjectsNonStrict(size_t N) const {
+    return ArrayRef(getTrailingObjectsNonStrict<T>(), N);
+  }
+
   /// Returns the size of the trailing data, if an object were
   /// allocated with the given counts (The counts are in the same order
   /// as the template arguments). This does not include the size of the
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index 5849344bcb0b5..a2b86eb8e7cad 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -868,7 +868,7 @@ class UnOpInit final : public OpInit, public FoldingSetNode {
   UnaryOp getOpcode() const { return (UnaryOp)Opc; }
   const Init *getOperand() const { return LHS; }
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec, bool IsFinal = false) const;
 
@@ -940,7 +940,7 @@ class BinOpInit final : public OpInit, public FoldingSetNode {
   std::optional<bool> CompareInit(unsigned Opc, const Init *LHS,
                                   const Init *RHS) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec) const;
 
@@ -990,7 +990,7 @@ class TernOpInit final : public OpInit, public FoldingSetNode {
   const Init *getMHS() const { return MHS; }
   const Init *getRHS() const { return RHS; }
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec) const;
 
@@ -1096,7 +1096,7 @@ class FoldOpInit final : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec) const;
 
@@ -1129,7 +1129,7 @@ class IsAOpInit final : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold() const;
 
@@ -1163,7 +1163,7 @@ class ExistsOpInit final : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec, bool IsFinal = false) const;
 
@@ -1412,8 +1412,8 @@ class FieldInit final : public TypedInit {
   }
 };
 
-/// (v a, b) - Represent a DAG tree value.  DAG inits are required
-/// to have at least one value then a (possibly empty) list of arguments.  Each
+/// (v a, b) - Represent a DAG tree value. DAG inits are required
+/// to have at least one value then a (possibly empty) list of arguments. Each
 /// argument can have a name associated with it.
 class DagInit final
     : public TypedInit,
diff --git a/llvm/include/llvm/TableGen/SetTheory.h b/llvm/include/llvm/TableGen/SetTheory.h
index 771dcff2f214c..53c78e4572927 100644
--- a/llvm/include/llvm/TableGen/SetTheory.h
+++ b/llvm/include/llvm/TableGen/SetTheory.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements the SetTheory class that computes ordered sets of
-// Records from DAG expressions.  Operators for standard set operations are
+// Records from DAG expressions. Operators for standard set operations are
 // predefined, and it is possible to add special purpose set operators as well.
 //
 // The user may define named sets as Records of predefined classes. Set
@@ -112,7 +112,7 @@ class SetTheory {
   void addExpander(StringRef ClassName, std::unique_ptr<Expander>);
 
   /// addFieldExpander - Add an expander for ClassName that simply evaluates
-  /// FieldName in the Record to get the set elements.  That is all that is
+  /// FieldName in the Record to get the set elements. That is all that is
   /// needed for a class like:
   ///
   ///   class Set<dag d> {
@@ -134,7 +134,7 @@ class SetTheory {
       evaluate(*begin++, Elts, Loc);
   }
 
-  /// expand - Expand a record into a set of elements if possible.  Return a
+  /// expand - Expand a record into a set of elements if possible. Return a
   /// pointer to the expanded elements, or NULL if Set cannot be expanded
   /// further.
   const RecVec *expand(const Record *Set);
diff --git a/llvm/include/llvm/TableGen/SetTheory.td b/llvm/include/llvm/TableGen/SetTheory.td
new file mode 100644
index 0000000000000..38b89671ee541
--- /dev/null
+++ b/llvm/include/llvm/TableGen/SetTheory.td
@@ -0,0 +1,27 @@
+//===- SetTheory.td - DAG set operator declarations --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// FIXME: This is not used everywhere, and different files declare
+// different subsets of used operators.
+//
+// It just happens TargetSelectionDAG.td defines records with the same
+// names as the tablegen DAG operators for SelectionDAG operators.
+
+// Target.td separately declares the special set operators.
+
+def add; // Forward declare
+def sub;
+def and;
+def shl;
+// def trunc; // FIXME: Name collision
+def rotl;
+def rotr;
+
+def sequence;
+def decimate;
+def interleave;
diff --git a/llvm/include/llvm/TableGen/StringMatcher.h b/llvm/include/llvm/TableGen/StringMatcher.h
index 49769883a98b4..70554de1a6f17 100644
--- a/llvm/include/llvm/TableGen/StringMatcher.h
+++ b/llvm/include/llvm/TableGen/StringMatcher.h
@@ -26,7 +26,7 @@ class raw_ostream;
 /// simple switch tree to classify the input string.
 ///
 /// If a match is found, the code in Matches[i].second is executed; control must
-/// not exit this code fragment.  If nothing matches, execution falls through.
+/// not exit this code fragment. If nothing matches, execution falls through.
 class StringMatcher {
 public:
   using StringPair = std::pair<std::string, std::string>;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 4a92dc16c1bf4..6033d80e717d3 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1132,8 +1132,8 @@ def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg,
 def udiv_by_const : GICombineRule<
   (defs root:$root),
   (match (wip_match_opcode G_UDIV):$root,
-   [{ return Helper.matchUDivByConst(*${root}); }]),
-  (apply [{ Helper.applyUDivByConst(*${root}); }])>;
+   [{ return Helper.matchUDivorURemByConst(*${root}); }]),
+  (apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
 
 def sdiv_by_const : GICombineRule<
   (defs root:$root),
@@ -1156,6 +1156,14 @@ def udiv_by_pow2 : GICombineRule<
 def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const,
                                       sdiv_by_pow2, udiv_by_pow2]>;
 
+def urem_by_const : GICombineRule<
+  (defs root:$root),
+  (match (G_UREM $dst, $x, $y):$root,
+   [{ return Helper.matchUDivorURemByConst(*${root}); }]),
+  (apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
+
+def intrem_combines : GICombineGroup<[urem_by_const]>;
+
 def reassoc_ptradd : GICombineRule<
   (defs root:$root, build_fn_matchinfo:$matchinfo),
   (match (wip_match_opcode G_PTR_ADD):$root,
@@ -2048,7 +2056,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
     constant_fold_cast_op, fabs_fneg_fold,
     intdiv_combines, mulh_combines, redundant_neg_operands,
     and_or_disjoint_mask, fma_combines, fold_binop_into_select,
-    sub_add_reg, select_to_minmax,
+    intrem_combines, sub_add_reg, select_to_minmax,
     fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
     simplify_neg_minmax, combine_concat_vector,
     sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index ce9a2b2751968..4c83f8a580aa0 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -538,11 +538,11 @@ include "llvm/Target/TargetInstrPredicate.td"
 include "llvm/Target/TargetSchedule.td"
 
 class InstructionEncoding {
-  // Size of encoded instruction.
+  // Size of encoded instruction in bytes.
   int Size;
 
-  // The "namespace" in which this instruction exists, on targets like ARM
-  // which multiple ISA namespaces exist.
+  // The "namespace" in which this instruction exists (on targets like ARM
+  // where multiple ISA namespaces exist).
   string DecoderNamespace = "";
 
   // List of predicates which will be turned into isel matching code.
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index b286efdea3c19..bf4e490554723 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -38,6 +38,7 @@ using ModulePassManager = PassManager<Module>;
 
 class Function;
 class GlobalValue;
+class MachineInstr;
 class MachineModuleInfoWrapperPass;
 struct MachineSchedContext;
 class Mangler;
@@ -242,6 +243,15 @@ class LLVM_ABI TargetMachine {
   const MCInstrInfo *getMCInstrInfo() const { return MII.get(); }
   const MCSubtargetInfo *getMCSubtargetInfo() const { return STI.get(); }
 
+  /// Return the ExceptionHandling to use, considering TargetOptions and the
+  /// Triple's default.
+  ExceptionHandling getExceptionModel() const {
+    // FIXME: This interface fails to distinguish default from not supported.
+    return Options.ExceptionModel == ExceptionHandling::None
+               ? TargetTriple.getDefaultExceptionHandling()
+               : Options.ExceptionModel;
+  }
+
   bool requiresStructuredCFG() const { return RequireStructuredCFG; }
   void setRequiresStructuredCFG(bool Value) { RequireStructuredCFG = Value; }
 
@@ -519,6 +529,15 @@ class LLVM_ABI TargetMachine {
 
   // MachineRegisterInfo callback function
   virtual void registerMachineRegisterInfoCallback(MachineFunction &MF) const {}
+
+  /// Remove all Linker Optimization Hints (LOH) associated with instructions in
+  /// \p MIs and \return the number of hints removed. This is useful in
+  /// transformations that cause these hints to be illegal, like in the machine
+  /// outliner.
+  virtual size_t clearLinkerOptimizationHints(
+      const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+    return 0;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/TargetParser/SubtargetFeature.h b/llvm/include/llvm/TargetParser/SubtargetFeature.h
index cdcfcdd0e802e..a48b18745352a 100644
--- a/llvm/include/llvm/TargetParser/SubtargetFeature.h
+++ b/llvm/include/llvm/TargetParser/SubtargetFeature.h
@@ -56,7 +56,7 @@ class FeatureBitset {
   }
 
   FeatureBitset &set() {
-    std::fill(std::begin(Bits), std::end(Bits), -1ULL);
+    llvm::fill(Bits, -1ULL);
     return *this;
   }
 
diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
index 9b8d1f3c31121..2f68d66dee90f 100644
--- a/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/llvm/include/llvm/TargetParser/TargetParser.h
@@ -40,89 +40,89 @@ enum GPUKind : uint32_t {
   GK_NONE = 0,
 
   // R600-based processors.
-  GK_R600 = 1,
-  GK_R630 = 2,
-  GK_RS880 = 3,
-  GK_RV670 = 4,
-  GK_RV710 = 5,
-  GK_RV730 = 6,
-  GK_RV770 = 7,
-  GK_CEDAR = 8,
-  GK_CYPRESS = 9,
-  GK_JUNIPER = 10,
-  GK_REDWOOD = 11,
-  GK_SUMO = 12,
-  GK_BARTS = 13,
-  GK_CAICOS = 14,
-  GK_CAYMAN = 15,
-  GK_TURKS = 16,
+  GK_R600,
+  GK_R630,
+  GK_RS880,
+  GK_RV670,
+  GK_RV710,
+  GK_RV730,
+  GK_RV770,
+  GK_CEDAR,
+  GK_CYPRESS,
+  GK_JUNIPER,
+  GK_REDWOOD,
+  GK_SUMO,
+  GK_BARTS,
+  GK_CAICOS,
+  GK_CAYMAN,
+  GK_TURKS,
 
   GK_R600_FIRST = GK_R600,
   GK_R600_LAST = GK_TURKS,
 
   // AMDGCN-based processors.
-  GK_GFX600 = 32,
-  GK_GFX601 = 33,
-  GK_GFX602 = 34,
-
-  GK_GFX700 = 40,
-  GK_GFX701 = 41,
-  GK_GFX702 = 42,
-  GK_GFX703 = 43,
-  GK_GFX704 = 44,
-  GK_GFX705 = 45,
-
-  GK_GFX801 = 50,
-  GK_GFX802 = 51,
-  GK_GFX803 = 52,
-  GK_GFX805 = 53,
-  GK_GFX810 = 54,
-
-  GK_GFX900 = 60,
-  GK_GFX902 = 61,
-  GK_GFX904 = 62,
-  GK_GFX906 = 63,
-  GK_GFX908 = 64,
-  GK_GFX909 = 65,
-  GK_GFX90A = 66,
-  GK_GFX90C = 67,
-  GK_GFX942 = 70,
-  GK_GFX950 = 71,
-
-  GK_GFX1010 = 72,
-  GK_GFX1011 = 73,
-  GK_GFX1012 = 74,
-  GK_GFX1013 = 75,
-  GK_GFX1030 = 76,
-  GK_GFX1031 = 77,
-  GK_GFX1032 = 78,
-  GK_GFX1033 = 79,
-  GK_GFX1034 = 80,
-  GK_GFX1035 = 81,
-  GK_GFX1036 = 82,
-
-  GK_GFX1100 = 90,
-  GK_GFX1101 = 91,
-  GK_GFX1102 = 92,
-  GK_GFX1103 = 93,
-  GK_GFX1150 = 94,
-  GK_GFX1151 = 95,
-  GK_GFX1152 = 96,
-  GK_GFX1153 = 97,
-
-  GK_GFX1200 = 100,
-  GK_GFX1201 = 101,
-  GK_GFX1250 = 102,
+  GK_GFX600,
+  GK_GFX601,
+  GK_GFX602,
+
+  GK_GFX700,
+  GK_GFX701,
+  GK_GFX702,
+  GK_GFX703,
+  GK_GFX704,
+  GK_GFX705,
+
+  GK_GFX801,
+  GK_GFX802,
+  GK_GFX803,
+  GK_GFX805,
+  GK_GFX810,
+
+  GK_GFX900,
+  GK_GFX902,
+  GK_GFX904,
+  GK_GFX906,
+  GK_GFX908,
+  GK_GFX909,
+  GK_GFX90A,
+  GK_GFX90C,
+  GK_GFX942,
+  GK_GFX950,
+
+  GK_GFX1010,
+  GK_GFX1011,
+  GK_GFX1012,
+  GK_GFX1013,
+  GK_GFX1030,
+  GK_GFX1031,
+  GK_GFX1032,
+  GK_GFX1033,
+  GK_GFX1034,
+  GK_GFX1035,
+  GK_GFX1036,
+
+  GK_GFX1100,
+  GK_GFX1101,
+  GK_GFX1102,
+  GK_GFX1103,
+  GK_GFX1150,
+  GK_GFX1151,
+  GK_GFX1152,
+  GK_GFX1153,
+
+  GK_GFX1200,
+  GK_GFX1201,
+  GK_GFX1250,
 
   GK_AMDGCN_FIRST = GK_GFX600,
   GK_AMDGCN_LAST = GK_GFX1250,
 
-  GK_GFX9_GENERIC = 192,
-  GK_GFX10_1_GENERIC = 193,
-  GK_GFX10_3_GENERIC = 194,
-  GK_GFX11_GENERIC = 195,
-  GK_GFX12_GENERIC = 196,
-  GK_GFX9_4_GENERIC = 197,
+  GK_GFX9_GENERIC,
+  GK_GFX10_1_GENERIC,
+  GK_GFX10_3_GENERIC,
+  GK_GFX11_GENERIC,
+  GK_GFX12_GENERIC,
+  GK_GFX9_4_GENERIC,
 
   GK_AMDGCN_GENERIC_FIRST = GK_GFX9_GENERIC,
   GK_AMDGCN_GENERIC_LAST = GK_GFX9_4_GENERIC,
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index cbf85b2ff74f5..657f4230379e8 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -20,6 +20,7 @@
 #undef sparc
 
 namespace llvm {
+enum class ExceptionHandling;
 class Twine;
 
 /// Triple - Helper class for working with autoconf configuration names. For
@@ -1321,6 +1322,8 @@ class Triple {
   /// Returns whether an OS version is invalid and would not map to an Apple OS.
   LLVM_ABI static bool isValidVersionForOS(OSType OSKind,
                                            const VersionTuple &Version);
+
+  LLVM_ABI ExceptionHandling getDefaultExceptionHandling() const;
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index e6eb756df987d..719c0ee136236 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -99,6 +99,7 @@
 
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/IntervalMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
@@ -1229,7 +1230,7 @@ struct InformationCache {
           });
   }
 
-  ~InformationCache() {
+  virtual ~InformationCache() {
     // The FunctionInfo objects are allocated via a BumpPtrAllocator, we call
     // the destructor manually.
     for (auto &It : FuncInfoMap)
@@ -1355,6 +1356,8 @@ struct InformationCache {
   /// Return the flat address space if the associated target has.
   LLVM_ABI std::optional<unsigned> getFlatAddressSpace() const;
 
+  virtual unsigned getMaxAddrSpace() const { return ~0U; }
+
 private:
   struct FunctionInfo {
     LLVM_ABI ~FunctionInfo();
@@ -6335,6 +6338,47 @@ struct AAUnderlyingObjects : AbstractAttribute {
                           AA::ValueScope Scope = AA::Interprocedural) const = 0;
 };
 
+/// An abstract interface for identifying pointers from which loads can be
+/// marked invariant.
+struct AAInvariantLoadPointer : public AbstractAttribute {
+  AAInvariantLoadPointer(const IRPosition &IRP) : AbstractAttribute(IRP) {}
+
+  /// See AbstractAttribute::isValidIRPositionForInit
+  static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+    if (!IRP.getAssociatedType()->isPointerTy())
+      return false;
+
+    return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+  }
+
+  /// Create an abstract attribute view for the position \p IRP.
+  LLVM_ABI static AAInvariantLoadPointer &
+  createForPosition(const IRPosition &IRP, Attributor &A);
+
+  /// Return true if the pointer's contents are known to remain invariant.
+  virtual bool isKnownInvariant() const = 0;
+  virtual bool isKnownLocallyInvariant() const = 0;
+
+  /// Return true if the pointer's contents are assumed to remain invariant.
+  virtual bool isAssumedInvariant() const = 0;
+  virtual bool isAssumedLocallyInvariant() const = 0;
+
+  /// See AbstractAttribute::getName().
+  StringRef getName() const override { return "AAInvariantLoadPointer"; }
+
+  /// See AbstractAttribute::getIdAddr().
+  const char *getIdAddr() const override { return &ID; }
+
+  /// This function should return true if the type of the \p AA is
+  /// AAInvariantLoadPointer
+  static bool classof(const AbstractAttribute *AA) {
+    return (AA->getIdAddr() == &ID);
+  }
+
+  /// Unique ID (due to the unique address).
+  LLVM_ABI static const char ID;
+};
+
 /// An abstract interface for address space information.
 struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
   AAAddressSpace(const IRPosition &IRP, Attributor &A)
@@ -6379,6 +6423,47 @@ struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
   static const uint32_t InvalidAddressSpace = ~0U;
 };
 
+/// An abstract interface for potential address space information.
+struct AANoAliasAddrSpace
+    : public StateWrapper<BooleanState, AbstractAttribute> {
+  using Base = StateWrapper<BooleanState, AbstractAttribute>;
+  using RangeMap = IntervalMap<unsigned, bool>;
+  AANoAliasAddrSpace(const IRPosition &IRP, Attributor &A)
+      : Base(IRP), Map(Allocator) {}
+
+  /// See AbstractAttribute::isValidIRPositionForInit
+  static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+    if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+      return false;
+    return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+  }
+
+  /// See AbstractAttribute::requiresCallersForArgOrFunction
+  static bool requiresCallersForArgOrFunction() { return true; }
+
+  /// Create an abstract attribute view for the position \p IRP.
+  LLVM_ABI static AANoAliasAddrSpace &createForPosition(const IRPosition &IRP,
+                                                        Attributor &A);
+  /// See AbstractAttribute::getName()
+  StringRef getName() const override { return "AANoAliasAddrSpace"; }
+
+  /// See AbstractAttribute::getIdAddr()
+  const char *getIdAddr() const override { return &ID; }
+
+  /// This function should return true if the type of the \p AA is
+  /// AAAssumptionInfo
+  static bool classof(const AbstractAttribute *AA) {
+    return (AA->getIdAddr() == &ID);
+  }
+
+  /// Unique ID (due to the unique address)
+  LLVM_ABI static const char ID;
+
+protected:
+  RangeMap::Allocator Allocator;
+  RangeMap Map;
+};
+
 struct AAAllocationInfo : public StateWrapper<BooleanState, AbstractAttribute> {
   AAAllocationInfo(const IRPosition &IRP, Attributor &A)
       : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
diff --git a/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h b/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
index 7b095ea24b4b0..37bc728779836 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
@@ -27,6 +27,7 @@ class LowerAllowCheckPass : public PassInfoMixin<LowerAllowCheckPass> {
 public:
   struct Options {
     std::vector<unsigned int> cutoffs;
+    unsigned int runtime_check = 0;
   };
 
   explicit LowerAllowCheckPass(LowerAllowCheckPass::Options Opts)
diff --git a/llvm/include/llvm/Transforms/Utils/DeclareRuntimeLibcalls.h b/llvm/include/llvm/Transforms/Utils/DeclareRuntimeLibcalls.h
new file mode 100644
index 0000000000000..295e3dec31415
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/DeclareRuntimeLibcalls.h
@@ -0,0 +1,23 @@
+//===- DeclareRuntimeLibcalls.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_DECLARERUNTIMELIBCALLS_H
+#define LLVM_TRANSFORMS_UTILS_DECLARERUNTIMELIBCALLS_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+class DeclareRuntimeLibcallsPass
+    : public PassInfoMixin<DeclareRuntimeLibcallsPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_DECLARERUNTIMELIBCALLS_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index d654ac3ec9273..cba37363d0474 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -301,6 +301,14 @@ class LoopVectorizationLegality {
   /// Returns the reduction variables found in the loop.
   const ReductionList &getReductionVars() const { return Reductions; }
 
+  /// Returns the recurrence descriptor associated with a given phi node \p PN,
+  /// expecting one to exist.
+  const RecurrenceDescriptor &getRecurrenceDescriptor(PHINode *PN) const {
+    assert(isReductionVariable(PN) &&
+           "only reductions have recurrence descriptors");
+    return Reductions.find(PN)->second;
+  }
+
   /// Returns the induction variables found in the loop.
   const InductionList &getInductionVars() const { return Inductions; }
 
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
index 6d2144b14bb00..ec70350691abe 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
@@ -307,7 +307,8 @@ class SeedCollector {
   }
 
 public:
-  LLVM_ABI SeedCollector(BasicBlock *BB, ScalarEvolution &SE);
+  LLVM_ABI SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
+                         bool CollectStores, bool CollectLoads);
   LLVM_ABI ~SeedCollector();
 
   iterator_range<SeedContainer::iterator> getStoreSeeds() {
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 0ff16dbec24e0..671e023415a6e 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -798,8 +798,9 @@ BranchProbabilityInfo::getInitialEstimatedBlockWeight(const BasicBlock *BB) {
 // Does RPO traversal over all blocks in \p F and assigns weights to
 // 'unreachable', 'noreturn', 'cold', 'unwind' blocks. In addition it does its
 // best to propagate the weight to up/down the IR.
-void BranchProbabilityInfo::computeEestimateBlockWeight(
-    const Function &F, DominatorTree *DT, PostDominatorTree *PDT) {
+void BranchProbabilityInfo::estimateBlockWeights(const Function &F,
+                                                 DominatorTree *DT,
+                                                 PostDominatorTree *PDT) {
   SmallVector<BasicBlock *, 8> BlockWorkList;
   SmallVector<LoopBlock, 8> LoopWorkList;
   SmallDenseMap<LoopData, SmallVector<BasicBlock *, 4>> LoopExitBlocks;
@@ -1244,7 +1245,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI,
     PDT = PDTPtr.get();
   }
 
-  computeEestimateBlockWeight(F, DT, PDT);
+  estimateBlockWeights(F, DT, PDT);
 
   // Walk the basic blocks in post-order so that we can build up state about
   // the successors of a block iteratively.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 9e3c271f7d93f..6e469c034d9c8 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1963,7 +1963,7 @@ inline bool llvm_fenv_testexcept() {
   return false;
 }
 
-static const APFloat FTZPreserveSign(const APFloat &V) {
+static APFloat FTZPreserveSign(const APFloat &V) {
   if (V.isDenormal())
     return APFloat::getZero(V.getSemantics(), V.isNegative());
   return V;
@@ -2229,17 +2229,6 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
     return nullptr;
   }
 
-  if (isa<PoisonValue>(Operands[0])) {
-    // TODO: All of these operations should probably propagate poison.
-    switch (IntrinsicID) {
-    case Intrinsic::canonicalize:
-    case Intrinsic::sqrt:
-      return PoisonValue::get(Ty);
-    default:
-      break;
-    }
-  }
-
   if (isa<UndefValue>(Operands[0])) {
     // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN.
     // ctpop() is between 0 and bitwidth, pick 0 for undef.
@@ -3228,11 +3217,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
     case Intrinsic::smin:
     case Intrinsic::umax:
     case Intrinsic::umin:
-      // This is the same as for binary ops - poison propagates.
-      // TODO: Poison handling should be consolidated.
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 && !C1)
         return UndefValue::get(Ty);
       if (!C0 || !C1)
@@ -3245,9 +3229,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
 
     case Intrinsic::scmp:
     case Intrinsic::ucmp:
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 || !C1)
         return ConstantInt::get(Ty, 0);
 
@@ -3314,11 +3295,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
     }
     case Intrinsic::uadd_sat:
     case Intrinsic::sadd_sat:
-      // This is the same as for binary ops - poison propagates.
-      // TODO: Poison handling should be consolidated.
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 && !C1)
         return UndefValue::get(Ty);
       if (!C0 || !C1)
@@ -3329,11 +3305,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
         return ConstantInt::get(Ty, C0->sadd_sat(*C1));
     case Intrinsic::usub_sat:
     case Intrinsic::ssub_sat:
-      // This is the same as for binary ops - poison propagates.
-      // TODO: Poison handling should be consolidated.
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 && !C1)
         return UndefValue::get(Ty);
       if (!C0 || !C1)
@@ -3592,11 +3563,6 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
 
   if (IntrinsicID == Intrinsic::smul_fix ||
       IntrinsicID == Intrinsic::smul_fix_sat) {
-    // poison * C -> poison
-    // C * poison -> poison
-    if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-      return PoisonValue::get(Ty);
-
     const APInt *C0, *C1;
     if (!getConstIntOrUndef(Operands[0], C0) ||
         !getConstIntOrUndef(Operands[1], C1))
@@ -3670,6 +3636,11 @@ static Constant *ConstantFoldScalarCall(StringRef Name,
                                         ArrayRef<Constant *> Operands,
                                         const TargetLibraryInfo *TLI,
                                         const CallBase *Call) {
+  if (IntrinsicID != Intrinsic::not_intrinsic &&
+      any_of(Operands, IsaPred<PoisonValue>) &&
+      intrinsicPropagatesPoison(IntrinsicID))
+    return PoisonValue::get(Ty);
+
   if (Operands.size() == 1)
     return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call);
 
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index c1b1d002c9979..5b85060f9caa1 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -2372,20 +2372,17 @@ bool DependenceInfo::testMIV(const SCEV *Src, const SCEV *Dst,
     banerjeeMIVtest(Src, Dst, Loops, Result);
 }
 
-
 // Given a product, e.g., 10*X*Y, returns the first constant operand,
-// in this case 10. If there is no constant part, returns NULL.
-static
-const SCEVConstant *getConstantPart(const SCEV *Expr) {
+// in this case 10. If there is no constant part, returns std::nullopt.
+static std::optional<APInt> getConstantPart(const SCEV *Expr) {
   if (const auto *Constant = dyn_cast<SCEVConstant>(Expr))
-    return Constant;
-  else if (const auto *Product = dyn_cast<SCEVMulExpr>(Expr))
+    return Constant->getAPInt();
+  if (const auto *Product = dyn_cast<SCEVMulExpr>(Expr))
     if (const auto *Constant = dyn_cast<SCEVConstant>(Product->getOperand(0)))
-      return Constant;
-  return nullptr;
+      return Constant->getAPInt();
+  return std::nullopt;
 }
 
-
 //===----------------------------------------------------------------------===//
 // gcdMIVtest -
 // Tests an MIV subscript pair for dependence.
@@ -2421,11 +2418,10 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
     const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
-    const auto *Constant = getConstantPart(Coeff);
-    if (!Constant)
+    std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+    if (!ConstCoeff)
       return false;
-    APInt ConstCoeff = Constant->getAPInt();
-    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
     Coefficients = AddRec->getStart();
   }
   const SCEV *SrcConst = Coefficients;
@@ -2440,11 +2436,10 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
     const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
-    const auto *Constant = getConstantPart(Coeff);
-    if (!Constant)
+    std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+    if (!ConstCoeff)
       return false;
-    APInt ConstCoeff = Constant->getAPInt();
-    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
     Coefficients = AddRec->getStart();
   }
   const SCEV *DstConst = Coefficients;
@@ -2463,12 +2458,10 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) {
         // Search for constant operand to participate in GCD;
         // If none found; return false.
-        const SCEVConstant *ConstOp = getConstantPart(Product);
+        std::optional<APInt> ConstOp = getConstantPart(Product);
         if (!ConstOp)
           return false;
-        APInt ConstOpValue = ConstOp->getAPInt();
-        ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD,
-                                                   ConstOpValue.abs());
+        ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD, ConstOp->abs());
       }
       else
         return false;
@@ -2520,11 +2513,11 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       else {
         // If the coefficient is the product of a constant and other stuff,
         // we can use the constant in the GCD computation.
-        Constant = getConstantPart(Coeff);
-        if (!Constant)
+        std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+        if (!ConstCoeff)
           return false;
-        APInt ConstCoeff = Constant->getAPInt();
-        RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+        RunningGCD =
+            APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
       }
       Inner = AddRec->getStart();
     }
@@ -2537,24 +2530,23 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       else {
         // If the coefficient is the product of a constant and other stuff,
         // we can use the constant in the GCD computation.
-        Constant = getConstantPart(Coeff);
-        if (!Constant)
+        std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+        if (!ConstCoeff)
           return false;
-        APInt ConstCoeff = Constant->getAPInt();
-        RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+        RunningGCD =
+            APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
       }
       Inner = AddRec->getStart();
     }
     Delta = SE->getMinusSCEV(SrcCoeff, DstCoeff);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
-    Constant = getConstantPart(Delta);
-    if (!Constant)
+    std::optional<APInt> ConstCoeff = getConstantPart(Delta);
+    if (!ConstCoeff)
       // The difference of the two coefficients might not be a product
       // or constant, in which case we give up on this direction.
       continue;
-    APInt ConstCoeff = Constant->getAPInt();
-    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
     LLVM_DEBUG(dbgs() << "\tRunningGCD = " << RunningGCD << "\n");
     if (RunningGCD != 0) {
       Remainder = ConstDelta.srem(RunningGCD);
@@ -4188,67 +4180,65 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
     }
   }
 
-  if (Coupled.count()) {
-    // test coupled subscript groups
-    SmallVector<Constraint, 4> Constraints(MaxLevels + 1);
-    for (unsigned II = 0; II <= MaxLevels; ++II)
-      Constraints[II].setAny(SE);
-    for (unsigned SI : Coupled.set_bits()) {
-      SmallBitVector Group(Pair[SI].Group);
-      SmallBitVector Sivs(Pairs);
-      SmallBitVector Mivs(Pairs);
-      SmallBitVector ConstrainedLevels(MaxLevels + 1);
-      for (unsigned SJ : Group.set_bits()) {
-        if (Pair[SJ].Classification == Subscript::SIV)
-          Sivs.set(SJ);
-        else
-          Mivs.set(SJ);
+  assert(!Coupled.empty() && "coupled expected non-empty");
+
+  // test coupled subscript groups
+  SmallVector<Constraint, 4> Constraints(MaxLevels + 1);
+  for (unsigned II = 0; II <= MaxLevels; ++II)
+    Constraints[II].setAny(SE);
+  for (unsigned SI : Coupled.set_bits()) {
+    SmallBitVector Group(Pair[SI].Group);
+    SmallBitVector Sivs(Pairs);
+    SmallBitVector Mivs(Pairs);
+    SmallBitVector ConstrainedLevels(MaxLevels + 1);
+    for (unsigned SJ : Group.set_bits()) {
+      if (Pair[SJ].Classification == Subscript::SIV)
+        Sivs.set(SJ);
+      else
+        Mivs.set(SJ);
+    }
+    while (Sivs.any()) {
+      bool Changed = false;
+      for (unsigned SJ : Sivs.set_bits()) {
+        // SJ is an SIV subscript that's part of the current coupled group
+        unsigned Level;
+        const SCEV *SplitIter = nullptr;
+        (void)testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level, Result, NewConstraint,
+                      SplitIter);
+        if (Level == SplitLevel && SplitIter)
+          return SplitIter;
+        ConstrainedLevels.set(Level);
+        if (intersectConstraints(&Constraints[Level], &NewConstraint))
+          Changed = true;
+        Sivs.reset(SJ);
       }
-      while (Sivs.any()) {
-        bool Changed = false;
-        for (unsigned SJ : Sivs.set_bits()) {
-          // SJ is an SIV subscript that's part of the current coupled group
-          unsigned Level;
-          const SCEV *SplitIter = nullptr;
-          (void) testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level,
-                         Result, NewConstraint, SplitIter);
-          if (Level == SplitLevel && SplitIter)
-            return SplitIter;
-          ConstrainedLevels.set(Level);
-          if (intersectConstraints(&Constraints[Level], &NewConstraint))
-            Changed = true;
-          Sivs.reset(SJ);
-        }
-        if (Changed) {
-          // propagate, possibly creating new SIVs and ZIVs
-          for (unsigned SJ : Mivs.set_bits()) {
-            // SJ is an MIV subscript that's part of the current coupled group
-            if (propagate(Pair[SJ].Src, Pair[SJ].Dst,
-                          Pair[SJ].Loops, Constraints, Result.Consistent)) {
-              Pair[SJ].Classification =
-                classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()),
-                             Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()),
-                             Pair[SJ].Loops);
-              switch (Pair[SJ].Classification) {
-              case Subscript::ZIV:
-                Mivs.reset(SJ);
-                break;
-              case Subscript::SIV:
-                Sivs.set(SJ);
-                Mivs.reset(SJ);
-                break;
-              case Subscript::RDIV:
-              case Subscript::MIV:
-                break;
-              default:
-                llvm_unreachable("bad subscript classification");
-              }
-            }
-          }
+      if (!Changed)
+        continue;
+      // propagate, possibly creating new SIVs and ZIVs
+      for (unsigned SJ : Mivs.set_bits()) {
+        // SJ is an MIV subscript that's part of the current coupled group
+        if (!propagate(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops, Constraints,
+                       Result.Consistent))
+          continue;
+        Pair[SJ].Classification = classifyPair(
+            Pair[SJ].Src, LI->getLoopFor(Src->getParent()), Pair[SJ].Dst,
+            LI->getLoopFor(Dst->getParent()), Pair[SJ].Loops);
+        switch (Pair[SJ].Classification) {
+        case Subscript::ZIV:
+          Mivs.reset(SJ);
+          break;
+        case Subscript::SIV:
+          Sivs.set(SJ);
+          Mivs.reset(SJ);
+          break;
+        case Subscript::RDIV:
+        case Subscript::MIV:
+          break;
+        default:
+          llvm_unreachable("bad subscript classification");
         }
       }
     }
   }
   llvm_unreachable("somehow reached end of routine");
-  return nullptr;
 }
diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
index e7e8f2ac1ff25..790e00e1b3b06 100644
--- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -27,8 +27,8 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ManagedStatic.h"
 
-#include <vector>
 #include <optional>
+#include <vector>
 
 using namespace llvm;
 
@@ -260,9 +260,8 @@ static const std::vector<TensorSpec> TrainingOnlyFeatures{
 
 static const std::vector<TensorSpec> getInputFeatures() {
   std::vector<TensorSpec> InputSpecs;
-  for (size_t I = 0; I < NumberOfFeatures; ++I)
-    InputSpecs.push_back(
-        TensorSpec(TFFeedPrefix + FeatureMap[I].name(), FeatureMap[I]));
+  for (const auto &Feature : FeatureMap)
+    InputSpecs.push_back(TensorSpec(TFFeedPrefix + Feature.name(), Feature));
   append_range(InputSpecs, TrainingOnlyFeatures);
   return InputSpecs;
 }
@@ -299,7 +298,8 @@ void TrainingLogger::logInlineEvent(const InlineEvent &Event,
                                     const MLModelRunner &ModelRunner) {
   L->startObservation();
   size_t CurrentFeature = 0;
-  for (; CurrentFeature < NumberOfFeatures; ++CurrentFeature)
+  size_t FeatureMapSize = FeatureMap.size();
+  for (; CurrentFeature < FeatureMapSize; ++CurrentFeature)
     L->logTensorValue(CurrentFeature,
                       reinterpret_cast<const char *>(
                           ModelRunner.getTensorUntyped(CurrentFeature)));
diff --git a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
index 9d044c8a35910..dd4eb7f0df053 100644
--- a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
+++ b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
@@ -199,6 +199,25 @@ void FunctionPropertiesInfo::updateForBB(const BasicBlock &BB,
 #undef CHECK_OPERAND
     }
   }
+
+  if (IR2VecVocab) {
+    // We instantiate the IR2Vec embedder each time, as having an unique
+    // pointer to the embedder as member of the class would make it
+    // non-copyable. Instantiating the embedder in itself is not costly.
+    auto Embedder = ir2vec::Embedder::create(IR2VecKind::Symbolic,
+                                             *BB.getParent(), *IR2VecVocab);
+    if (!Embedder) {
+      BB.getContext().emitError("Error creating IR2Vec embeddings");
+      return;
+    }
+    const auto &BBEmbedding = Embedder->getBBVector(BB);
+    // Subtract BBEmbedding from Function embedding if the direction is -1,
+    // and add it if the direction is +1.
+    if (Direction == -1)
+      FunctionEmbedding -= BBEmbedding;
+    else
+      FunctionEmbedding += BBEmbedding;
+  }
 }
 
 void FunctionPropertiesInfo::updateAggregateStats(const Function &F,
@@ -220,14 +239,24 @@ void FunctionPropertiesInfo::updateAggregateStats(const Function &F,
 
 FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo(
     Function &F, FunctionAnalysisManager &FAM) {
+  // We use the cached result of the IR2VecVocabAnalysis run by
+  // InlineAdvisorAnalysis. If the IR2VecVocabAnalysis is not run, we don't
+  // use IR2Vec embeddings.
+  auto VocabResult = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                         .getCachedResult<IR2VecVocabAnalysis>(*F.getParent());
   return getFunctionPropertiesInfo(F, FAM.getResult<DominatorTreeAnalysis>(F),
-                                   FAM.getResult<LoopAnalysis>(F));
+                                   FAM.getResult<LoopAnalysis>(F), VocabResult);
 }
 
 FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo(
-    const Function &F, const DominatorTree &DT, const LoopInfo &LI) {
+    const Function &F, const DominatorTree &DT, const LoopInfo &LI,
+    const IR2VecVocabResult *VocabResult) {
 
   FunctionPropertiesInfo FPI;
+  if (VocabResult && VocabResult->isValid()) {
+    FPI.IR2VecVocab = VocabResult->getVocabulary();
+    FPI.FunctionEmbedding = ir2vec::Embedding(VocabResult->getDimension(), 0.0);
+  }
   for (const auto &BB : F)
     if (DT.isReachableFromEntry(&BB))
       FPI.reIncludeBB(BB);
@@ -235,6 +264,66 @@ FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo(
   return FPI;
 }
 
+bool FunctionPropertiesInfo::operator==(
+    const FunctionPropertiesInfo &FPI) const {
+  if (BasicBlockCount != FPI.BasicBlockCount ||
+      BlocksReachedFromConditionalInstruction !=
+          FPI.BlocksReachedFromConditionalInstruction ||
+      Uses != FPI.Uses ||
+      DirectCallsToDefinedFunctions != FPI.DirectCallsToDefinedFunctions ||
+      LoadInstCount != FPI.LoadInstCount ||
+      StoreInstCount != FPI.StoreInstCount ||
+      MaxLoopDepth != FPI.MaxLoopDepth ||
+      TopLevelLoopCount != FPI.TopLevelLoopCount ||
+      TotalInstructionCount != FPI.TotalInstructionCount ||
+      BasicBlocksWithSingleSuccessor != FPI.BasicBlocksWithSingleSuccessor ||
+      BasicBlocksWithTwoSuccessors != FPI.BasicBlocksWithTwoSuccessors ||
+      BasicBlocksWithMoreThanTwoSuccessors !=
+          FPI.BasicBlocksWithMoreThanTwoSuccessors ||
+      BasicBlocksWithSinglePredecessor !=
+          FPI.BasicBlocksWithSinglePredecessor ||
+      BasicBlocksWithTwoPredecessors != FPI.BasicBlocksWithTwoPredecessors ||
+      BasicBlocksWithMoreThanTwoPredecessors !=
+          FPI.BasicBlocksWithMoreThanTwoPredecessors ||
+      BigBasicBlocks != FPI.BigBasicBlocks ||
+      MediumBasicBlocks != FPI.MediumBasicBlocks ||
+      SmallBasicBlocks != FPI.SmallBasicBlocks ||
+      CastInstructionCount != FPI.CastInstructionCount ||
+      FloatingPointInstructionCount != FPI.FloatingPointInstructionCount ||
+      IntegerInstructionCount != FPI.IntegerInstructionCount ||
+      ConstantIntOperandCount != FPI.ConstantIntOperandCount ||
+      ConstantFPOperandCount != FPI.ConstantFPOperandCount ||
+      ConstantOperandCount != FPI.ConstantOperandCount ||
+      InstructionOperandCount != FPI.InstructionOperandCount ||
+      BasicBlockOperandCount != FPI.BasicBlockOperandCount ||
+      GlobalValueOperandCount != FPI.GlobalValueOperandCount ||
+      InlineAsmOperandCount != FPI.InlineAsmOperandCount ||
+      ArgumentOperandCount != FPI.ArgumentOperandCount ||
+      UnknownOperandCount != FPI.UnknownOperandCount ||
+      CriticalEdgeCount != FPI.CriticalEdgeCount ||
+      ControlFlowEdgeCount != FPI.ControlFlowEdgeCount ||
+      UnconditionalBranchCount != FPI.UnconditionalBranchCount ||
+      IntrinsicCount != FPI.IntrinsicCount ||
+      DirectCallCount != FPI.DirectCallCount ||
+      IndirectCallCount != FPI.IndirectCallCount ||
+      CallReturnsIntegerCount != FPI.CallReturnsIntegerCount ||
+      CallReturnsFloatCount != FPI.CallReturnsFloatCount ||
+      CallReturnsPointerCount != FPI.CallReturnsPointerCount ||
+      CallReturnsVectorIntCount != FPI.CallReturnsVectorIntCount ||
+      CallReturnsVectorFloatCount != FPI.CallReturnsVectorFloatCount ||
+      CallReturnsVectorPointerCount != FPI.CallReturnsVectorPointerCount ||
+      CallWithManyArgumentsCount != FPI.CallWithManyArgumentsCount ||
+      CallWithPointerArgumentCount != FPI.CallWithPointerArgumentCount) {
+    return false;
+  }
+  // Check the equality of the function embeddings. We don't check the equality
+  // of Vocabulary as it remains the same.
+  if (!FunctionEmbedding.approximatelyEquals(FPI.FunctionEmbedding))
+    return false;
+
+  return true;
+}
+
 void FunctionPropertiesInfo::print(raw_ostream &OS) const {
 #define PRINT_PROPERTY(PROP_NAME) OS << #PROP_NAME ": " << PROP_NAME << "\n";
 
@@ -322,6 +411,16 @@ FunctionPropertiesUpdater::FunctionPropertiesUpdater(
   // The caller's entry BB may change due to new alloca instructions.
   LikelyToChangeBBs.insert(&*Caller.begin());
 
+  // The users of the value returned by call instruction can change
+  // leading to the change in embeddings being computed, when used.
+  // We conservatively add the BBs with such uses to LikelyToChangeBBs.
+  for (const auto *User : CB.users())
+    CallUsers.insert(dyn_cast<Instruction>(User)->getParent());
+  // CallSiteBB can be removed from CallUsers if present, it's taken care
+  // separately.
+  CallUsers.erase(&CallSiteBB);
+  LikelyToChangeBBs.insert_range(CallUsers);
+
   // The successors may become unreachable in the case of `invoke` inlining.
   // We track successors separately, too, because they form a boundary, together
   // with the CB BB ('Entry') between which the inlined callee will be pasted.
@@ -435,6 +534,9 @@ void FunctionPropertiesUpdater::finish(FunctionAnalysisManager &FAM) const {
   if (&CallSiteBB != &*Caller.begin())
     Reinclude.insert(&*Caller.begin());
 
+  // Reinclude the BBs which use the values returned by call instruction
+  Reinclude.insert_range(CallUsers);
+
   // Distribute the successors to the 2 buckets.
   for (const auto *Succ : Successors)
     if (DT.isReachableFromEntry(Succ))
@@ -486,6 +588,9 @@ bool FunctionPropertiesUpdater::isUpdateValid(Function &F,
     return false;
   DominatorTree DT(F);
   LoopInfo LI(DT);
-  auto Fresh = FunctionPropertiesInfo::getFunctionPropertiesInfo(F, DT, LI);
+  auto VocabResult = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                         .getCachedResult<IR2VecVocabAnalysis>(*F.getParent());
+  auto Fresh =
+      FunctionPropertiesInfo::getFunctionPropertiesInfo(F, DT, LI, VocabResult);
   return FPI == Fresh;
 }
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 1a5dfbe4bed6e..2cc3ad5f18482 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -247,16 +247,10 @@ KnownBits ValueEvolution::compute(const Value *V) {
 }
 
 bool ValueEvolution::computeEvolutions(ArrayRef<PhiStepPair> PhiEvolutions) {
-  for (unsigned I = 0; I < TripCount; ++I) {
-    for (auto [Phi, Step] : PhiEvolutions) {
-      KnownBits KnownAtIter = computeInstr(Step);
-      if (KnownAtIter.getBitWidth() < I + 1) {
-        ErrStr = "Loop iterations exceed bitwidth of result";
-        return false;
-      }
-      KnownPhis.emplace_or_assign(Phi, KnownAtIter);
-    }
-  }
+  for (unsigned I = 0; I < TripCount; ++I)
+    for (auto [Phi, Step] : PhiEvolutions)
+      KnownPhis.emplace_or_assign(Phi, computeInstr(Step));
+
   return ErrStr.empty();
 }
 
@@ -369,7 +363,7 @@ RecurrenceInfo::digRecurrence(Instruction *V,
 /// A Conditional Recurrence is a recurrence of the form:
 ///
 /// loop:
-///    %rec = [%start, %entry], [%step, %loop]
+///    %rec = phi [%start, %entry], [%step, %loop]
 ///    ...
 ///    %step = select _, %tv, %fv
 ///
@@ -497,42 +491,46 @@ CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
   return Table;
 }
 
-/// Checks if \p Reference is reachable from \p Needle on the use-def chain, and
-/// that there are no stray PHI nodes while digging the use-def chain. \p
-/// BOToMatch is a CRC peculiarity: at least one of the Users of Needle needs to
-/// match this OpCode, which is XOR for CRC.
-static bool arePHIsIntertwined(
-    const PHINode *Needle, const PHINode *Reference, const Loop &L,
-    Instruction::BinaryOps BOToMatch = Instruction::BinaryOpsEnd) {
-  // Initialize the worklist with Users of the Needle.
+/// Checks that \p P1 and \p P2 are used together in an XOR in the use-def chain
+/// of \p SI's condition, ignoring any casts. The purpose of this function is to
+/// ensure that LHSAux from the SimpleRecurrence is used correctly in the CRC
+/// computation. We cannot check the correctness of casts at this point, and
+/// rely on the KnownBits propagation to check correctness of the CRC
+/// computation.
+///
+/// In other words, it checks for the following pattern:
+///
+/// loop:
+///   %P1 = phi [_, %entry], [%P1.next, %loop]
+///   %P2 = phi [_, %entry], [%P2.next, %loop]
+///   ...
+///   %xor = xor (CastOrSelf %P1), (CastOrSelf %P2)
+///
+/// where %xor is in the use-def chain of \p SI's condition.
+static bool isConditionalOnXorOfPHIs(const SelectInst *SI, const PHINode *P1,
+                                     const PHINode *P2, const Loop &L) {
   SmallVector<const Instruction *> Worklist;
-  for (const User *U : Needle->users()) {
-    if (auto *UI = dyn_cast<Instruction>(U))
-      if (L.contains(UI))
-        Worklist.push_back(UI);
-  }
 
-  // BOToMatch is usually XOR for CRC.
-  if (BOToMatch != Instruction::BinaryOpsEnd) {
-    if (count_if(Worklist, [BOToMatch](const Instruction *I) {
-          return I->getOpcode() == BOToMatch;
-        }) != 1)
-      return false;
-  }
+  // matchConditionalRecurrence has already ensured that the SelectInst's
+  // condition is an Instruction.
+  Worklist.push_back(cast<Instruction>(SI->getCondition()));
 
   while (!Worklist.empty()) {
     const Instruction *I = Worklist.pop_back_val();
 
-    // Since Needle is never pushed onto the Worklist, I must either be the
-    // Reference PHI node (in which case we're done), or a stray PHI node (in
-    // which case we abort).
+    // Don't add a PHI's operands to the Worklist.
     if (isa<PHINode>(I))
-      return I == Reference;
+      continue;
 
+    // If we match an XOR of the two PHIs ignoring casts, we're done.
+    if (match(I, m_c_Xor(m_CastOrSelf(m_Specific(P1)),
+                         m_CastOrSelf(m_Specific(P2)))))
+      return true;
+
+    // Continue along the use-def chain.
     for (const Use &U : I->operands())
       if (auto *UI = dyn_cast<Instruction>(U))
-        // Don't push Needle back onto the Worklist.
-        if (UI != Needle && L.contains(UI))
+        if (L.contains(UI))
           Worklist.push_back(UI);
   }
   return false;
@@ -586,11 +584,28 @@ HashRecognize::recognizeCRC() const {
   if (SimpleRecurrence) {
     if (isBigEndianBitShift(SimpleRecurrence.BO, SE) != ByteOrderSwapped)
       return "Loop with non-unit bitshifts";
-    if (!arePHIsIntertwined(SimpleRecurrence.Phi, ConditionalRecurrence.Phi, L,
-                            Instruction::BinaryOps::Xor))
-      return "Simple recurrence doesn't use conditional recurrence with XOR";
+
+    // Ensure that the PHIs have exactly two uses:
+    // the bit-shift, and the XOR (or a cast feeding into the XOR).
+    if (!ConditionalRecurrence.Phi->hasNUses(2) ||
+        !SimpleRecurrence.Phi->hasNUses(2))
+      return "Recurrences have stray uses";
+
+    // Check that the SelectInst ConditionalRecurrence.Step is conditional on
+    // the XOR of SimpleRecurrence.Phi and ConditionalRecurrence.Phi.
+    if (!isConditionalOnXorOfPHIs(cast<SelectInst>(ConditionalRecurrence.Step),
+                                  SimpleRecurrence.Phi,
+                                  ConditionalRecurrence.Phi, L))
+      return "Recurrences not intertwined with XOR";
   }
 
+  // Make sure that the TC doesn't exceed the bitwidth of LHSAux, or LHS.
+  Value *LHS = ConditionalRecurrence.Start;
+  Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
+  if (TC > (LHSAux ? LHSAux->getType()->getIntegerBitWidth()
+                   : LHS->getType()->getIntegerBitWidth()))
+    return "Loop iterations exceed bitwidth of data";
+
   // Make sure that the computed value is used in the exit block: this should be
   // true even if it is only really used in an outer loop's exit block, since
   // the loop is in LCSSA form.
@@ -619,13 +634,13 @@ HashRecognize::recognizeCRC() const {
     return VE.getError();
   KnownBits ResultBits = VE.KnownPhis.at(ConditionalRecurrence.Phi);
 
+  unsigned N = std::min(TC, ResultBits.getBitWidth());
   auto IsZero = [](const KnownBits &K) { return K.isZero(); };
-  if (!checkExtractBits(ResultBits, TC, IsZero, *ByteOrderSwapped))
+  if (!checkExtractBits(ResultBits, N, IsZero, *ByteOrderSwapped))
     return ErrBits(ResultBits, TC, *ByteOrderSwapped);
 
-  Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
-  return PolynomialInfo(TC, ConditionalRecurrence.Start, GenPoly, ComputedValue,
-                        *ByteOrderSwapped, LHSAux);
+  return PolynomialInfo(TC, LHS, GenPoly, ComputedValue, *ByteOrderSwapped,
+                        LHSAux);
 }
 
 void CRCTable::print(raw_ostream &OS) const {
diff --git a/llvm/lib/Analysis/IR2Vec.cpp b/llvm/lib/Analysis/IR2Vec.cpp
index fa38c35796a0e..bf456102bb618 100644
--- a/llvm/lib/Analysis/IR2Vec.cpp
+++ b/llvm/lib/Analysis/IR2Vec.cpp
@@ -70,7 +70,6 @@ inline bool fromJSON(const llvm::json::Value &E, Embedding &Out,
 // ==----------------------------------------------------------------------===//
 // Embedding
 //===----------------------------------------------------------------------===//
-
 Embedding &Embedding::operator+=(const Embedding &RHS) {
   assert(this->size() == RHS.size() && "Vectors must have the same dimension");
   std::transform(this->begin(), this->end(), RHS.begin(), this->begin(),
@@ -78,6 +77,12 @@ Embedding &Embedding::operator+=(const Embedding &RHS) {
   return *this;
 }
 
+Embedding Embedding::operator+(const Embedding &RHS) const {
+  Embedding Result(*this);
+  Result += RHS;
+  return Result;
+}
+
 Embedding &Embedding::operator-=(const Embedding &RHS) {
   assert(this->size() == RHS.size() && "Vectors must have the same dimension");
   std::transform(this->begin(), this->end(), RHS.begin(), this->begin(),
@@ -85,6 +90,24 @@ Embedding &Embedding::operator-=(const Embedding &RHS) {
   return *this;
 }
 
+Embedding Embedding::operator-(const Embedding &RHS) const {
+  Embedding Result(*this);
+  Result -= RHS;
+  return Result;
+}
+
+Embedding &Embedding::operator*=(double Factor) {
+  std::transform(this->begin(), this->end(), this->begin(),
+                 [Factor](double Elem) { return Elem * Factor; });
+  return *this;
+}
+
+Embedding Embedding::operator*(double Factor) const {
+  Embedding Result(*this);
+  Result *= Factor;
+  return Result;
+}
+
 Embedding &Embedding::scaleAndAdd(const Embedding &Src, float Factor) {
   assert(this->size() == Src.size() && "Vectors must have the same dimension");
   for (size_t Itr = 0; Itr < this->size(); ++Itr)
@@ -101,6 +124,13 @@ bool Embedding::approximatelyEquals(const Embedding &RHS,
   return true;
 }
 
+void Embedding::print(raw_ostream &OS) const {
+  OS << " [";
+  for (const auto &Elem : Data)
+    OS << " " << format("%.2f", Elem) << " ";
+  OS << "]\n";
+}
+
 // ==----------------------------------------------------------------------===//
 // Embedder and its subclasses
 //===----------------------------------------------------------------------===//
@@ -110,13 +140,13 @@ Embedder::Embedder(const Function &F, const Vocab &Vocabulary)
       Dimension(Vocabulary.begin()->second.size()), OpcWeight(::OpcWeight),
       TypeWeight(::TypeWeight), ArgWeight(::ArgWeight) {}
 
-Expected<std::unique_ptr<Embedder>>
-Embedder::create(IR2VecKind Mode, const Function &F, const Vocab &Vocabulary) {
+std::unique_ptr<Embedder> Embedder::create(IR2VecKind Mode, const Function &F,
+                                           const Vocab &Vocabulary) {
   switch (Mode) {
   case IR2VecKind::Symbolic:
     return std::make_unique<SymbolicEmbedder>(F, Vocabulary);
   }
-  return make_error<StringError>("Unknown IR2VecKind", errc::invalid_argument);
+  return nullptr;
 }
 
 // FIXME: Currently lookups are string based. Use numeric Keys
@@ -196,18 +226,12 @@ void SymbolicEmbedder::computeEmbeddings(const BasicBlock &BB) const {
   for (const auto &I : BB.instructionsWithoutDebug()) {
     Embedding InstVector(Dimension, 0);
 
-    const auto OpcVec = lookupVocab(I.getOpcodeName());
-    InstVector.scaleAndAdd(OpcVec, OpcWeight);
-
     // FIXME: Currently lookups are string based. Use numeric Keys
     // for efficiency.
-    const auto Type = I.getType();
-    const auto TypeVec = getTypeEmbedding(Type);
-    InstVector.scaleAndAdd(TypeVec, TypeWeight);
-
+    InstVector += lookupVocab(I.getOpcodeName());
+    InstVector += getTypeEmbedding(I.getType());
     for (const auto &Op : I.operands()) {
-      const auto OperandVec = getOperandEmbedding(Op.get());
-      InstVector.scaleAndAdd(OperandVec, ArgWeight);
+      InstVector += getOperandEmbedding(Op.get());
     }
     InstVecMap[&I] = InstVector;
     BBVector += InstVector;
@@ -251,6 +275,43 @@ bool IR2VecVocabResult::invalidate(
   return !(PAC.preservedWhenStateless());
 }
 
+Error IR2VecVocabAnalysis::parseVocabSection(
+    StringRef Key, const json::Value &ParsedVocabValue,
+    ir2vec::Vocab &TargetVocab, unsigned &Dim) {
+  json::Path::Root Path("");
+  const json::Object *RootObj = ParsedVocabValue.getAsObject();
+  if (!RootObj)
+    return createStringError(errc::invalid_argument,
+                             "JSON root is not an object");
+
+  const json::Value *SectionValue = RootObj->get(Key);
+  if (!SectionValue)
+    return createStringError(errc::invalid_argument,
+                             "Missing '" + std::string(Key) +
+                                 "' section in vocabulary file");
+  if (!json::fromJSON(*SectionValue, TargetVocab, Path))
+    return createStringError(errc::illegal_byte_sequence,
+                             "Unable to parse '" + std::string(Key) +
+                                 "' section from vocabulary");
+
+  Dim = TargetVocab.begin()->second.size();
+  if (Dim == 0)
+    return createStringError(errc::illegal_byte_sequence,
+                             "Dimension of '" + std::string(Key) +
+                                 "' section of the vocabulary is zero");
+
+  if (!std::all_of(TargetVocab.begin(), TargetVocab.end(),
+                   [Dim](const std::pair<StringRef, Embedding> &Entry) {
+                     return Entry.second.size() == Dim;
+                   }))
+    return createStringError(
+        errc::illegal_byte_sequence,
+        "All vectors in the '" + std::string(Key) +
+            "' section of the vocabulary are not of the same dimension");
+
+  return Error::success();
+}
+
 // FIXME: Make this optional. We can avoid file reads
 // by auto-generating a default vocabulary during the build time.
 Error IR2VecVocabAnalysis::readVocabulary() {
@@ -259,32 +320,40 @@ Error IR2VecVocabAnalysis::readVocabulary() {
     return createFileError(VocabFile, BufOrError.getError());
 
   auto Content = BufOrError.get()->getBuffer();
-  json::Path::Root Path("");
+
   Expected<json::Value> ParsedVocabValue = json::parse(Content);
   if (!ParsedVocabValue)
     return ParsedVocabValue.takeError();
 
-  bool Res = json::fromJSON(*ParsedVocabValue, Vocabulary, Path);
-  if (!Res)
-    return createStringError(errc::illegal_byte_sequence,
-                             "Unable to parse the vocabulary");
+  ir2vec::Vocab OpcodeVocab, TypeVocab, ArgVocab;
+  unsigned OpcodeDim = 0, TypeDim = 0, ArgDim = 0;
+  if (auto Err = parseVocabSection("Opcodes", *ParsedVocabValue, OpcodeVocab,
+                                   OpcodeDim))
+    return Err;
 
-  if (Vocabulary.empty())
-    return createStringError(errc::illegal_byte_sequence,
-                             "Vocabulary is empty");
+  if (auto Err =
+          parseVocabSection("Types", *ParsedVocabValue, TypeVocab, TypeDim))
+    return Err;
 
-  unsigned Dim = Vocabulary.begin()->second.size();
-  if (Dim == 0)
+  if (auto Err =
+          parseVocabSection("Arguments", *ParsedVocabValue, ArgVocab, ArgDim))
+    return Err;
+
+  if (!(OpcodeDim == TypeDim && TypeDim == ArgDim))
     return createStringError(errc::illegal_byte_sequence,
-                             "Dimension of vocabulary is zero");
+                             "Vocabulary sections have different dimensions");
 
-  if (!std::all_of(Vocabulary.begin(), Vocabulary.end(),
-                   [Dim](const std::pair<StringRef, Embedding> &Entry) {
-                     return Entry.second.size() == Dim;
-                   }))
-    return createStringError(
-        errc::illegal_byte_sequence,
-        "All vectors in the vocabulary are not of the same dimension");
+  auto scaleVocabSection = [](ir2vec::Vocab &Vocab, double Weight) {
+    for (auto &Entry : Vocab)
+      Entry.second *= Weight;
+  };
+  scaleVocabSection(OpcodeVocab, OpcWeight);
+  scaleVocabSection(TypeVocab, TypeWeight);
+  scaleVocabSection(ArgVocab, ArgWeight);
+
+  Vocabulary.insert(OpcodeVocab.begin(), OpcodeVocab.end());
+  Vocabulary.insert(TypeVocab.begin(), TypeVocab.end());
+  Vocabulary.insert(ArgVocab.begin(), ArgVocab.end());
 
   return Error::success();
 }
@@ -304,7 +373,6 @@ void IR2VecVocabAnalysis::emitError(Error Err, LLVMContext &Ctx) {
 IR2VecVocabAnalysis::Result
 IR2VecVocabAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
   auto Ctx = &M.getContext();
-  // FIXME: Scale the vocabulary once. This would avoid scaling per use later.
   // If vocabulary is already populated by the constructor, use it.
   if (!Vocabulary.empty())
     return IR2VecVocabResult(std::move(Vocabulary));
@@ -323,16 +391,9 @@ IR2VecVocabAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
 }
 
 // ==----------------------------------------------------------------------===//
-// IR2VecPrinterPass
+// Printer Passes
 //===----------------------------------------------------------------------===//
 
-void IR2VecPrinterPass::printVector(const Embedding &Vec) const {
-  OS << " [";
-  for (const auto &Elem : Vec)
-    OS << " " << format("%.2f", Elem) << " ";
-  OS << "]\n";
-}
-
 PreservedAnalyses IR2VecPrinterPass::run(Module &M,
                                          ModuleAnalysisManager &MAM) {
   auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);
@@ -340,20 +401,16 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
 
   auto Vocab = IR2VecVocabResult.getVocabulary();
   for (Function &F : M) {
-    Expected<std::unique_ptr<Embedder>> EmbOrErr =
+    std::unique_ptr<Embedder> Emb =
         Embedder::create(IR2VecKind::Symbolic, F, Vocab);
-    if (auto Err = EmbOrErr.takeError()) {
-      handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
-        OS << "Error creating IR2Vec embeddings: " << EI.message() << "\n";
-      });
+    if (!Emb) {
+      OS << "Error creating IR2Vec embeddings \n";
       continue;
     }
 
-    std::unique_ptr<Embedder> Emb = std::move(*EmbOrErr);
-
     OS << "IR2Vec embeddings for function " << F.getName() << ":\n";
     OS << "Function vector: ";
-    printVector(Emb->getFunctionVector());
+    Emb->getFunctionVector().print(OS);
 
     OS << "Basic block vectors:\n";
     const auto &BBMap = Emb->getBBVecMap();
@@ -361,7 +418,7 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
       auto It = BBMap.find(&BB);
       if (It != BBMap.end()) {
         OS << "Basic block: " << BB.getName() << ":\n";
-        printVector(It->second);
+        It->second.print(OS);
       }
     }
 
@@ -373,10 +430,24 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
         if (It != InstMap.end()) {
           OS << "Instruction: ";
           I.print(OS);
-          printVector(It->second);
+          It->second.print(OS);
         }
       }
     }
   }
   return PreservedAnalyses::all();
 }
+
+PreservedAnalyses IR2VecVocabPrinterPass::run(Module &M,
+                                              ModuleAnalysisManager &MAM) {
+  auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);
+  assert(IR2VecVocabResult.isValid() && "IR2Vec Vocabulary is invalid");
+
+  auto Vocab = IR2VecVocabResult.getVocabulary();
+  for (const auto &Entry : Vocab) {
+    OS << "Key: " << Entry.first << ": ";
+    Entry.second.print(OS);
+  }
+
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index c8e97e5ec0e58..b275b1064cef2 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -50,6 +50,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
   case RecurKind::UMax:
   case RecurKind::UMin:
   case RecurKind::AnyOf:
+  case RecurKind::FindFirstIVSMin:
   case RecurKind::FindLastIVSMax:
   case RecurKind::FindLastIVUMax:
     return true;
@@ -684,8 +685,9 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
 // value of the data type or a non-constant value by using mask and multiple
 // reduction operations.
 RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
-                                          Instruction *I, ScalarEvolution &SE) {
+RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
+                                      PHINode *OrigPhi, Instruction *I,
+                                      ScalarEvolution &SE) {
   // TODO: Support the vectorization of FindLastIV when the reduction phi is
   // used by more than one select instruction. This vectorization is only
   // performed when the SCEV of each increasing induction variable used by the
@@ -713,25 +715,42 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
       return std::nullopt;
 
     const SCEV *Step = AR->getStepRecurrence(SE);
-    if (!SE.isKnownPositive(Step))
+    if ((isFindFirstIVRecurrenceKind(Kind) && !SE.isKnownNegative(Step)) ||
+        (isFindLastIVRecurrenceKind(Kind) && !SE.isKnownPositive(Step)))
       return std::nullopt;
 
     // Keep the minimum value of the recurrence type as the sentinel value.
     // The maximum acceptable range for the increasing induction variable,
     // called the valid range, will be defined as
+
+    // Keep the minimum (FindLast) or maximum (FindFirst) value of the
+    // recurrence type as the sentinel value. The maximum acceptable range for
+    // the induction variable, called the valid range, will be defined as
     //   [<sentinel value> + 1, <sentinel value>)
-    // where <sentinel value> is [Signed|Unsigned]Min(<recurrence type>)
+    // where <sentinel value> is [Signed|Unsigned]Min(<recurrence type>) for
+    // FindLastIV or [Signed|Unsigned]Max(<recurrence type>) for FindFirstIV.
     // TODO: This range restriction can be lifted by adding an additional
     // virtual OR reduction.
     auto CheckRange = [&](bool IsSigned) {
       const ConstantRange IVRange =
           IsSigned ? SE.getSignedRange(AR) : SE.getUnsignedRange(AR);
       unsigned NumBits = Ty->getIntegerBitWidth();
-      const APInt Sentinel = IsSigned ? APInt::getSignedMinValue(NumBits)
-                                      : APInt::getMinValue(NumBits);
-      const ConstantRange ValidRange =
-          ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
-      LLVM_DEBUG(dbgs() << "LV: FindLastIV valid range is " << ValidRange
+      ConstantRange ValidRange = ConstantRange::getEmpty(NumBits);
+      if (isFindLastIVRecurrenceKind(Kind)) {
+        APInt Sentinel = IsSigned ? APInt::getSignedMinValue(NumBits)
+                                  : APInt::getMinValue(NumBits);
+        ValidRange = ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
+      } else {
+        assert(IsSigned && "Only FindFirstIV with SMax is supported currently");
+        ValidRange =
+            ConstantRange::getNonEmpty(APInt::getSignedMinValue(NumBits),
+                                       APInt::getSignedMaxValue(NumBits) - 1);
+      }
+
+      LLVM_DEBUG(dbgs() << "LV: "
+                        << (isFindLastIVRecurrenceKind(Kind) ? "FindLastIV"
+                                                             : "FindFirstIV")
+                        << " valid range is " << ValidRange
                         << ", and the range of " << *AR << " is " << IVRange
                         << "\n");
 
@@ -739,10 +758,18 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
       // its range is fully contained within the valid range.
       return ValidRange.contains(IVRange);
     };
+    if (isFindLastIVRecurrenceKind(Kind)) {
+      if (CheckRange(true))
+        return RecurKind::FindLastIVSMax;
+      if (CheckRange(false))
+        return RecurKind::FindLastIVUMax;
+      return std::nullopt;
+    }
+    assert(isFindFirstIVRecurrenceKind(Kind) &&
+           "Kind must either be a FindLastIV or FindFirstIV");
+
     if (CheckRange(true))
-      return RecurKind::FindLastIVSMax;
-    if (CheckRange(false))
-      return RecurKind::FindLastIVUMax;
+      return RecurKind::FindFirstIVSMin;
     return std::nullopt;
   };
 
@@ -888,8 +915,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
     if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
         Kind == RecurKind::Add || Kind == RecurKind::Mul)
       return isConditionalRdxPattern(I);
-    if (isFindLastIVRecurrenceKind(Kind) && SE)
-      return isFindLastIVPattern(L, OrigPhi, I, *SE);
+    if (isFindIVRecurrenceKind(Kind) && SE)
+      return isFindIVPattern(Kind, L, OrigPhi, I, *SE);
     [[fallthrough]];
   case Instruction::FCmp:
   case Instruction::ICmp:
@@ -1003,6 +1030,11 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
     LLVM_DEBUG(dbgs() << "Found a FindLastIV reduction PHI." << *Phi << "\n");
     return true;
   }
+  if (AddReductionVar(Phi, RecurKind::FindFirstIVSMin, TheLoop, FMF, RedDes, DB,
+                      AC, DT, SE)) {
+    LLVM_DEBUG(dbgs() << "Found a FindFirstIV reduction PHI." << *Phi << "\n");
+    return true;
+  }
   if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT,
                       SE)) {
     LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
@@ -1150,6 +1182,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
   case RecurKind::Mul:
     return Instruction::Mul;
   case RecurKind::AnyOf:
+  case RecurKind::FindFirstIVSMin:
   case RecurKind::FindLastIVSMax:
   case RecurKind::FindLastIVUMax:
   case RecurKind::Or:
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index 3d30f3d10a9d0..28b14c2562df1 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/EphemeralValuesCache.h"
+#include "llvm/Analysis/IR2Vec.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
@@ -64,6 +65,13 @@ static cl::opt<bool>
                         cl::desc("If true, annotate inline advisor remarks "
                                  "with LTO and pass information."));
 
+// This flag is used to enable IR2Vec embeddings in the ML inliner; Only valid
+// with ML inliner. The vocab file is used to initialize the embeddings.
+static cl::opt<std::string> IR2VecVocabFile(
+    "ml-inliner-ir2vec-vocab-file", cl::Hidden,
+    cl::desc("Vocab file for IR2Vec; Setting this enables "
+             "configuring the model to use IR2Vec embeddings."));
+
 namespace llvm {
 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
 } // namespace llvm
@@ -206,6 +214,20 @@ void InlineAdvice::recordInliningWithCalleeDeleted() {
 AnalysisKey InlineAdvisorAnalysis::Key;
 AnalysisKey PluginInlineAdvisorAnalysis::Key;
 
+bool InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(
+    Module &M, ModuleAnalysisManager &MAM) {
+  if (!IR2VecVocabFile.empty()) {
+    auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);
+    if (!IR2VecVocabResult.isValid()) {
+      M.getContext().emitError("Failed to load IR2Vec vocabulary");
+      return false;
+    }
+  }
+  // No vocab file specified is OK; We just don't use IR2Vec
+  // embeddings.
+  return true;
+}
+
 bool InlineAdvisorAnalysis::Result::tryCreate(
     InlineParams Params, InliningAdvisorMode Mode,
     const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
@@ -231,14 +253,21 @@ bool InlineAdvisorAnalysis::Result::tryCreate(
                                              /* EmitRemarks =*/true, IC);
     }
     break;
+    // Run IR2VecVocabAnalysis once per module to get the vocabulary.
+    // We run it here because it is immutable and we want to avoid running it
+    // multiple times.
   case InliningAdvisorMode::Development:
 #ifdef LLVM_HAVE_TFLITE
     LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
+    if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
+      return false;
     Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
 #endif
     break;
   case InliningAdvisorMode::Release:
     LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
+    if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
+      return false;
     Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
     break;
   }
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index cb1dae92faf92..82530e7d5b6c6 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6484,9 +6484,6 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
     break;
   }
   case Intrinsic::ptrmask: {
-    if (isa<PoisonValue>(Op0) || isa<PoisonValue>(Op1))
-      return PoisonValue::get(Op0->getType());
-
     // NOTE: We can't apply this simplifications based on the value of Op1
     // because we need to preserve provenance.
     if (Q.isUndefValue(Op0) || match(Op0, m_Zero()))
@@ -6533,10 +6530,6 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
     if (match(Op0, m_ImmConstant()))
       std::swap(Op0, Op1);
 
-    // Propagate poison.
-    if (isa<PoisonValue>(Op1))
-      return Op1;
-
     // Assume undef is the limit value.
     if (Q.isUndefValue(Op1))
       return ConstantInt::get(
@@ -6696,9 +6689,6 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
       return Op1;
     break;
   case Intrinsic::is_fpclass: {
-    if (isa<PoisonValue>(Op0))
-      return PoisonValue::get(ReturnType);
-
     uint64_t Mask = cast<ConstantInt>(Op1)->getZExtValue();
     // If all tests are made, it doesn't matter what the value is.
     if ((Mask & fcAllFlags) == fcAllFlags)
@@ -6793,6 +6783,9 @@ static Value *simplifyIntrinsic(CallBase *Call, Value *Callee,
   Function *F = cast<Function>(Callee);
   Intrinsic::ID IID = F->getIntrinsicID();
 
+  if (IID != Intrinsic::not_intrinsic && intrinsicPropagatesPoison(IID) &&
+      any_of(Args, IsaPred<PoisonValue>))
+    return PoisonValue::get(F->getReturnType());
   // Most of the intrinsics with no operands have some kind of side effect.
   // Don't simplify.
   if (!NumOperands) {
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 2a562484fc374..922f25de54e9d 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -1745,14 +1745,10 @@ LazyValueInfoImpl &LazyValueInfo::getOrCreateImpl(const Module *M) {
         Intrinsic::getDeclarationIfExists(M, Intrinsic::experimental_guard);
     PImpl = new LazyValueInfoImpl(AC, DL, GuardDecl);
   }
-  return *static_cast<LazyValueInfoImpl *>(PImpl);
+  return *PImpl;
 }
 
-LazyValueInfoImpl *LazyValueInfo::getImpl() {
-  if (!PImpl)
-    return nullptr;
-  return static_cast<LazyValueInfoImpl *>(PImpl);
-}
+LazyValueInfoImpl *LazyValueInfo::getImpl() { return PImpl; }
 
 LazyValueInfo::~LazyValueInfo() { releaseMemory(); }
 
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 23dfbe0209e45..7b71dad23948c 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -949,18 +949,12 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
   int64_t Size = AllocSize.getFixedValue();
 
   // Huge step value - give up.
-  if (APStepVal->getBitWidth() > 64)
+  std::optional<int64_t> StepVal = APStepVal->trySExtValue();
+  if (!StepVal)
     return std::nullopt;
 
-  int64_t StepVal = APStepVal->getSExtValue();
-
   // Strided access.
-  int64_t Stride = StepVal / Size;
-  int64_t Rem = StepVal % Size;
-  if (Rem)
-    return std::nullopt;
-
-  return Stride;
+  return *StepVal % Size ? std::nullopt : std::make_optional(*StepVal / Size);
 }
 
 /// Check whether \p AR is a non-wrapping AddRec. If \p Ptr is not nullptr, use
@@ -2087,14 +2081,14 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
   if (StrideAScaled == StrideBScaled)
     CommonStride = StrideAScaled;
 
-  // TODO: Historically, we don't retry with runtime checks unless the
-  // (unscaled) strides are the same. Fix this once the condition for runtime
-  // checks in isDependent is fixed.
-  bool ShouldRetryWithRuntimeCheck = StrideAPtrInt == StrideBPtrInt;
+  // TODO: FoundNonConstantDistanceDependence is used as a necessary condition
+  // to consider retrying with runtime checks. Historically, we did not set it
+  // when (unscaled) strides were different but there is no inherent reason to.
+  if (!isa<SCEVConstant>(Dist))
+    FoundNonConstantDistanceDependence |= StrideAPtrInt == StrideBPtrInt;
 
   return DepDistanceStrideAndSizeInfo(Dist, MaxStride, CommonStride,
-                                      ShouldRetryWithRuntimeCheck, TypeByteSize,
-                                      AIsWrite, BIsWrite);
+                                      TypeByteSize, AIsWrite, BIsWrite);
 }
 
 MemoryDepChecker::Dependence::DepType
@@ -2109,15 +2103,11 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
   if (std::holds_alternative<Dependence::DepType>(Res))
     return std::get<Dependence::DepType>(Res);
 
-  auto &[Dist, MaxStride, CommonStride, ShouldRetryWithRuntimeCheck,
-         TypeByteSize, AIsWrite, BIsWrite] =
+  auto &[Dist, MaxStride, CommonStride, TypeByteSize, AIsWrite, BIsWrite] =
       std::get<DepDistanceStrideAndSizeInfo>(Res);
   bool HasSameSize = TypeByteSize > 0;
 
   if (isa<SCEVCouldNotCompute>(Dist)) {
-    // TODO: Relax requirement that there is a common unscaled stride to retry
-    // with non-constant distance dependencies.
-    FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
     LLVM_DEBUG(dbgs() << "LAA: Dependence because of uncomputable distance.\n");
     return Dependence::Unknown;
   }
@@ -2135,15 +2125,18 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
           DL, SE, *(PSE.getSymbolicMaxBackedgeTakenCount()), *Dist, MaxStride))
     return Dependence::NoDep;
 
-  // Attempt to prove strided accesses independent.
-  const APInt *ConstDist = nullptr;
-  if (match(Dist, m_scev_APInt(ConstDist))) {
-    uint64_t Distance = ConstDist->abs().getZExtValue();
+  // The rest of this function relies on ConstDist being at most 64-bits, which
+  // is checked earlier. Will assert if the calling code changes.
+  const APInt *APDist = nullptr;
+  uint64_t ConstDist =
+      match(Dist, m_scev_APInt(APDist)) ? APDist->abs().getZExtValue() : 0;
 
+  // Attempt to prove strided accesses independent.
+  if (APDist) {
     // If the distance between accesses and their strides are known constants,
     // check whether the accesses interlace each other.
-    if (Distance > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
-        areStridedAccessesIndependent(Distance, *CommonStride, TypeByteSize)) {
+    if (ConstDist > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
+        areStridedAccessesIndependent(ConstDist, *CommonStride, TypeByteSize)) {
       LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
       return Dependence::NoDep;
     }
@@ -2176,16 +2169,10 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     // forward dependency will allow vectorization using any width.
 
     if (IsTrueDataDependence && EnableForwardingConflictDetection) {
-      if (!ConstDist) {
-        // TODO: FoundNonConstantDistanceDependence is used as a necessary
-        // condition to consider retrying with runtime checks. Historically, we
-        // did not set it when strides were different but there is no inherent
-        // reason to.
-        FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
+      if (!ConstDist)
         return Dependence::Unknown;
-      }
-      if (!HasSameSize || couldPreventStoreLoadForward(
-                              ConstDist->abs().getZExtValue(), TypeByteSize)) {
+      if (!HasSameSize ||
+          couldPreventStoreLoadForward(ConstDist, TypeByteSize)) {
         LLVM_DEBUG(
             dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
         return Dependence::ForwardButPreventsForwarding;
@@ -2198,22 +2185,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
 
   int64_t MinDistance = SE.getSignedRangeMin(Dist).getSExtValue();
   // Below we only handle strictly positive distances.
-  if (MinDistance <= 0) {
-    FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
+  if (MinDistance <= 0)
     return Dependence::Unknown;
-  }
-
-  if (!ConstDist) {
-    // Previously this case would be treated as Unknown, possibly setting
-    // FoundNonConstantDistanceDependence to force re-trying with runtime
-    // checks. Until the TODO below is addressed, set it here to preserve
-    // original behavior w.r.t. re-trying with runtime checks.
-    // TODO: FoundNonConstantDistanceDependence is used as a necessary
-    // condition to consider retrying with runtime checks. Historically, we
-    // did not set it when strides were different but there is no inherent
-    // reason to.
-    FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
-  }
 
   if (!HasSameSize) {
     LLVM_DEBUG(dbgs() << "LAA: ReadWrite-Write positive dependency with "
@@ -2283,22 +2256,6 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     return Dependence::Backward;
   }
 
-  // Positive distance bigger than max vectorization factor.
-  // FIXME: Should use max factor instead of max distance in bytes, which could
-  // not handle different types.
-  // E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
-  //      void foo (int *A, char *B) {
-  //        for (unsigned i = 0; i < 1024; i++) {
-  //          A[i+2] = A[i] + 1;
-  //          B[i+2] = B[i] + 1;
-  //        }
-  //      }
-  //
-  // This case is currently unsafe according to the max safe distance. If we
-  // analyze the two accesses on array B, the max safe dependence distance
-  // is 2. Then we analyze the accesses on array A, the minimum distance needed
-  // is 8, which is less than 2 and forbidden vectorization, But actually
-  // both A and B could be vectorized by 2 iterations.
   MinDepDistBytes =
       std::min(static_cast<uint64_t>(MinDistance), MinDepDistBytes);
 
diff --git a/llvm/lib/Analysis/LoopPass.cpp b/llvm/lib/Analysis/LoopPass.cpp
index 5d0f9abbb5ce7..d8680aac74b22 100644
--- a/llvm/lib/Analysis/LoopPass.cpp
+++ b/llvm/lib/Analysis/LoopPass.cpp
@@ -373,7 +373,7 @@ bool LoopPass::skipLoop(const Loop *L) const {
   if (!F)
     return false;
   // Check the opt bisect limit.
-  OptPassGate &Gate = F->getContext().getOptPassGate();
+  const OptPassGate &Gate = F->getContext().getOptPassGate();
   if (Gate.isEnabled() &&
       !Gate.shouldRunPass(this->getPassName(), getDescription(*L)))
     return true;
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
index 81a3bc94a6ad8..8853a13972bea 100644
--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -107,7 +107,7 @@ static cl::opt<bool> KeepFPICache(
     cl::init(false));
 
 // clang-format off
-const std::vector<TensorSpec> llvm::FeatureMap{
+std::vector<TensorSpec> llvm::FeatureMap{
 #define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
 // InlineCost features - these must come first
   INLINE_COST_FEATURE_ITERATOR(POPULATE_NAMES)
@@ -144,6 +144,7 @@ MLInlineAdvisor::MLInlineAdvisor(
           M, MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
       ModelRunner(std::move(Runner)), GetDefaultAdvice(GetDefaultAdvice),
       CG(MAM.getResult<LazyCallGraphAnalysis>(M)),
+      UseIR2Vec(MAM.getCachedResult<IR2VecVocabAnalysis>(M) != nullptr),
       InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize),
       PSI(MAM.getResult<ProfileSummaryAnalysis>(M)) {
   assert(ModelRunner);
@@ -186,6 +187,19 @@ MLInlineAdvisor::MLInlineAdvisor(
     EdgeCount += getLocalCalls(KVP.first->getFunction());
   }
   NodeCount = AllNodes.size();
+
+  if (auto IR2VecVocabResult = MAM.getCachedResult<IR2VecVocabAnalysis>(M)) {
+    if (!IR2VecVocabResult->isValid()) {
+      M.getContext().emitError("IR2VecVocabAnalysis is not valid");
+      return;
+    }
+    // Add the IR2Vec features to the feature map
+    auto IR2VecDim = IR2VecVocabResult->getDimension();
+    FeatureMap.push_back(
+        TensorSpec::createSpec<float>("callee_embedding", {IR2VecDim}));
+    FeatureMap.push_back(
+        TensorSpec::createSpec<float>("caller_embedding", {IR2VecDim}));
+  }
 }
 
 unsigned MLInlineAdvisor::getInitialFunctionLevel(const Function &F) const {
@@ -433,6 +447,22 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
   *ModelRunner->getTensor<int64_t>(FeatureIndex::is_caller_avail_external) =
       Caller.hasAvailableExternallyLinkage();
 
+  if (UseIR2Vec) {
+    // Python side expects float embeddings. The IR2Vec embeddings are doubles
+    // as of now due to the restriction of fromJSON method used by the
+    // readVocabulary method in ir2vec::Embeddings.
+    auto setEmbedding = [&](const ir2vec::Embedding &Embedding,
+                            FeatureIndex Index) {
+      llvm::transform(Embedding, ModelRunner->getTensor<float>(Index),
+                      [](double Val) { return static_cast<float>(Val); });
+    };
+
+    setEmbedding(CalleeBefore.getFunctionEmbedding(),
+                 FeatureIndex::callee_embedding);
+    setEmbedding(CallerBefore.getFunctionEmbedding(),
+                 FeatureIndex::caller_embedding);
+  }
+
   // Add the cost features
   for (size_t I = 0;
        I < static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures); ++I) {
@@ -441,8 +471,7 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
   }
   // This one would have been set up to be right at the end.
   if (!InteractiveChannelBaseName.empty() && InteractiveIncludeDefault)
-    *ModelRunner->getTensor<int64_t>(FeatureIndex::NumberOfFeatures) =
-        GetDefaultAdvice(CB);
+    *ModelRunner->getTensor<int64_t>(FeatureMap.size()) = GetDefaultAdvice(CB);
   return getAdviceFromModel(CB, ORE);
 }
 
@@ -520,7 +549,7 @@ void MLInlineAdvice::reportContextForRemark(
     DiagnosticInfoOptimizationBase &OR) {
   using namespace ore;
   OR << NV("Callee", Callee->getName());
-  for (size_t I = 0; I < NumberOfFeatures; ++I)
+  for (size_t I = 0; I < FeatureMap.size(); ++I)
     OR << NV(FeatureMap[I].name(),
              *getAdvisor()->getModelRunner().getTensor<int64_t>(I));
   OR << NV("ShouldInline", isInliningRecommended());
diff --git a/llvm/lib/Analysis/RegionPass.cpp b/llvm/lib/Analysis/RegionPass.cpp
index 9ea7d711918f5..ae1d84659de86 100644
--- a/llvm/lib/Analysis/RegionPass.cpp
+++ b/llvm/lib/Analysis/RegionPass.cpp
@@ -282,7 +282,7 @@ static std::string getDescription(const Region &R) {
 
 bool RegionPass::skipRegion(Region &R) const {
   Function &F = *R.getEntry()->getParent();
-  OptPassGate &Gate = F.getContext().getOptPassGate();
+  const OptPassGate &Gate = F.getContext().getOptPassGate();
   if (Gate.isEnabled() &&
       !Gate.shouldRunPass(this->getPassName(), getDescription(R)))
     return true;
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index dd309bc2c54a8..1c66f5c877f59 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -1559,9 +1559,8 @@ ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
   Ty = getEffectiveSCEVType(Ty);
 
   FoldID ID(scZeroExtend, Op, Ty);
-  auto Iter = FoldCache.find(ID);
-  if (Iter != FoldCache.end())
-    return Iter->second;
+  if (const SCEV *S = FoldCache.lookup(ID))
+    return S;
 
   const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
   if (!isa<SCEVZeroExtendExpr>(S))
@@ -1894,9 +1893,8 @@ ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
   Ty = getEffectiveSCEVType(Ty);
 
   FoldID ID(scSignExtend, Op, Ty);
-  auto Iter = FoldCache.find(ID);
-  if (Iter != FoldCache.end())
-    return Iter->second;
+  if (const SCEV *S = FoldCache.lookup(ID))
+    return S;
 
   const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
   if (!isa<SCEVSignExtendExpr>(S))
@@ -11376,11 +11374,9 @@ bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
   if (HasSameValue(LHS, RHS))
     return ICmpInst::isTrueWhenEqual(Pred);
 
-  // This code is split out from isKnownPredicate because it is called from
-  // within isLoopEntryGuardedByCond.
-
-  auto CheckRanges = [&](const ConstantRange &RangeLHS,
-                         const ConstantRange &RangeRHS) {
+  auto CheckRange = [&](bool IsSigned) {
+    auto RangeLHS = IsSigned ? getSignedRange(LHS) : getUnsignedRange(LHS);
+    auto RangeRHS = IsSigned ? getSignedRange(RHS) : getUnsignedRange(RHS);
     return RangeLHS.icmp(Pred, RangeRHS);
   };
 
@@ -11390,27 +11386,13 @@ bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
     return false;
 
   if (Pred == CmpInst::ICMP_NE) {
-    auto SL = getSignedRange(LHS);
-    auto SR = getSignedRange(RHS);
-    if (CheckRanges(SL, SR))
-      return true;
-    auto UL = getUnsignedRange(LHS);
-    auto UR = getUnsignedRange(RHS);
-    if (CheckRanges(UL, UR))
+    if (CheckRange(true) || CheckRange(false))
       return true;
     auto *Diff = getMinusSCEV(LHS, RHS);
     return !isa<SCEVCouldNotCompute>(Diff) && isKnownNonZero(Diff);
   }
 
-  if (CmpInst::isSigned(Pred)) {
-    auto SL = getSignedRange(LHS);
-    auto SR = getSignedRange(RHS);
-    return CheckRanges(SL, SR);
-  }
-
-  auto UL = getUnsignedRange(LHS);
-  auto UR = getUnsignedRange(RHS);
-  return CheckRanges(UL, UR);
+  return CheckRange(CmpInst::isSigned(Pred));
 }
 
 bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
@@ -12499,21 +12481,14 @@ bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
                                             const SCEV *FoundLHS,
                                             const SCEV *FoundRHS,
                                             const Instruction *CtxI) {
-  if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, Pred, FoundLHS, FoundRHS))
-    return true;
-
-  if (isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS, FoundRHS))
-    return true;
-
-  if (isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS))
-    return true;
-
-  if (isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
-                                          CtxI))
-    return true;
-
-  return isImpliedCondOperandsHelper(Pred, LHS, RHS,
-                                     FoundLHS, FoundRHS);
+  return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, Pred, FoundLHS,
+                                        FoundRHS) ||
+         isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
+                                            FoundRHS) ||
+         isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
+         isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
+                                             CtxI) ||
+         isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
 }
 
 /// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
@@ -14521,15 +14496,15 @@ void ScalarEvolution::verify() const {
 
   for (const auto &KV : ExprValueMap) {
     for (Value *V : KV.second) {
-      auto It = ValueExprMap.find_as(V);
-      if (It == ValueExprMap.end()) {
+      const SCEV *S = ValueExprMap.lookup(V);
+      if (!S) {
         dbgs() << "Value " << *V
                << " is in ExprValueMap but not in ValueExprMap\n";
         std::abort();
       }
-      if (It->second != KV.first) {
-        dbgs() << "Value " << *V << " mapped to " << *It->second
-               << " rather than " << *KV.first << "\n";
+      if (S != KV.first) {
+        dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
+               << *KV.first << "\n";
         std::abort();
       }
     }
@@ -14648,15 +14623,15 @@ void ScalarEvolution::verify() const {
   }
   for (auto [Expr, IDs] : FoldCacheUser) {
     for (auto &FoldID : IDs) {
-      auto I = FoldCache.find(FoldID);
-      if (I == FoldCache.end()) {
+      const SCEV *S = FoldCache.lookup(FoldID);
+      if (!S) {
         dbgs() << "Missing entry in FoldCache for expression " << *Expr
                << "!\n";
         std::abort();
       }
-      if (I->second != Expr) {
-        dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: "
-               << *I->second << " != " << *Expr << "!\n";
+      if (S != Expr) {
+        dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
+               << " != " << *Expr << "!\n";
         std::abort();
       }
     }
@@ -15624,8 +15599,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock(
     // existing rewrite because we want to chain further rewrites onto the
     // already rewritten value. Otherwise returns \p S.
     auto GetMaybeRewritten = [&](const SCEV *S) {
-      auto I = RewriteMap.find(S);
-      return I != RewriteMap.end() ? I->second : S;
+      return RewriteMap.lookup_or(S, S);
     };
 
     // Check for the SCEV expression (A /u B) * B while B is a constant, inside
@@ -15928,9 +15902,8 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
              Bitwidth > Op->getType()->getScalarSizeInBits()) {
         Type *NarrowTy = IntegerType::get(SE.getContext(), Bitwidth);
         auto *NarrowExt = SE.getZeroExtendExpr(Op, NarrowTy);
-        auto I = Map.find(NarrowExt);
-        if (I != Map.end())
-          return SE.getZeroExtendExpr(I->second, Ty);
+        if (const SCEV *S = Map.lookup(NarrowExt))
+          return SE.getZeroExtendExpr(S, Ty);
         Bitwidth = Bitwidth / 2;
       }
 
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 3ebd9d487ba04..8a470ebf85a16 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1294,8 +1294,9 @@ unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
 }
 
 Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
-    IntrinsicInst *Inst, Type *ExpectedType) const {
-  return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
+    IntrinsicInst *Inst, Type *ExpectedType, bool CanCreate) const {
+  return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType,
+                                                    CanCreate);
 }
 
 Type *TargetTransformInfo::getMemcpyLoopLoweringType(
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index e576f4899810a..21f844c4d2f45 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -3036,14 +3036,12 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         return isKnownNonZero(TI->getOperand(0), DemandedElts, Q, Depth);
     break;
 
+  // Iff x - y != 0, then x ^ y != 0
+  // Therefore we can do the same exact checks
+  case Instruction::Xor:
   case Instruction::Sub:
     return isNonZeroSub(DemandedElts, Q, BitWidth, I->getOperand(0),
                         I->getOperand(1), Depth);
-  case Instruction::Xor:
-    // (X ^ (X != 0)) is non zero
-    if (matchOpWithOpEqZero(I->getOperand(0), I->getOperand(1)))
-      return true;
-    break;
   case Instruction::Or:
     // (X | (X != 0)) is non zero
     if (matchOpWithOpEqZero(I->getOperand(0), I->getOperand(1)))
@@ -7879,6 +7877,51 @@ bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
   llvm_unreachable("Instruction not contained in its own parent basic block.");
 }
 
+bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
+  switch (IID) {
+  // TODO: Add more intrinsics.
+  case Intrinsic::sadd_with_overflow:
+  case Intrinsic::ssub_with_overflow:
+  case Intrinsic::smul_with_overflow:
+  case Intrinsic::uadd_with_overflow:
+  case Intrinsic::usub_with_overflow:
+  case Intrinsic::umul_with_overflow:
+    // If an input is a vector containing a poison element, the
+    // two output vectors (calculated results, overflow bits)'
+    // corresponding lanes are poison.
+    return true;
+  case Intrinsic::ctpop:
+  case Intrinsic::ctlz:
+  case Intrinsic::cttz:
+  case Intrinsic::abs:
+  case Intrinsic::smax:
+  case Intrinsic::smin:
+  case Intrinsic::umax:
+  case Intrinsic::umin:
+  case Intrinsic::scmp:
+  case Intrinsic::is_fpclass:
+  case Intrinsic::ptrmask:
+  case Intrinsic::ucmp:
+  case Intrinsic::bitreverse:
+  case Intrinsic::bswap:
+  case Intrinsic::sadd_sat:
+  case Intrinsic::ssub_sat:
+  case Intrinsic::sshl_sat:
+  case Intrinsic::uadd_sat:
+  case Intrinsic::usub_sat:
+  case Intrinsic::ushl_sat:
+  case Intrinsic::smul_fix:
+  case Intrinsic::smul_fix_sat:
+  case Intrinsic::pow:
+  case Intrinsic::powi:
+  case Intrinsic::canonicalize:
+  case Intrinsic::sqrt:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool llvm::propagatesPoison(const Use &PoisonOp) {
   const Operator *I = cast<Operator>(PoisonOp.getUser());
   switch (I->getOpcode()) {
@@ -7889,38 +7932,8 @@ bool llvm::propagatesPoison(const Use &PoisonOp) {
   case Instruction::Select:
     return PoisonOp.getOperandNo() == 0;
   case Instruction::Call:
-    if (auto *II = dyn_cast<IntrinsicInst>(I)) {
-      switch (II->getIntrinsicID()) {
-      // TODO: Add more intrinsics.
-      case Intrinsic::sadd_with_overflow:
-      case Intrinsic::ssub_with_overflow:
-      case Intrinsic::smul_with_overflow:
-      case Intrinsic::uadd_with_overflow:
-      case Intrinsic::usub_with_overflow:
-      case Intrinsic::umul_with_overflow:
-        // If an input is a vector containing a poison element, the
-        // two output vectors (calculated results, overflow bits)'
-        // corresponding lanes are poison.
-        return true;
-      case Intrinsic::ctpop:
-      case Intrinsic::ctlz:
-      case Intrinsic::cttz:
-      case Intrinsic::abs:
-      case Intrinsic::smax:
-      case Intrinsic::smin:
-      case Intrinsic::umax:
-      case Intrinsic::umin:
-      case Intrinsic::bitreverse:
-      case Intrinsic::bswap:
-      case Intrinsic::sadd_sat:
-      case Intrinsic::ssub_sat:
-      case Intrinsic::sshl_sat:
-      case Intrinsic::uadd_sat:
-      case Intrinsic::usub_sat:
-      case Intrinsic::ushl_sat:
-        return true;
-      }
-    }
+    if (auto *II = dyn_cast<IntrinsicInst>(I))
+      return intrinsicPropagatesPoison(II->getIntrinsicID());
     return false;
   case Instruction::ICmp:
   case Instruction::FCmp:
@@ -9274,9 +9287,9 @@ isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
     return Res;
   if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
     LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(LPred)
-                                : static_cast<CmpInst::Predicate>(LPred);
+                                : LPred.dropSameSign();
     RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(RPred)
-                                : static_cast<CmpInst::Predicate>(RPred);
+                                : RPred.dropSameSign();
     return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(LPred, LCR),
                          RPred);
   }
diff --git a/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json b/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json
index 65ca240deba13..a0d539afd5294 100644
--- a/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json
+++ b/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json
@@ -1,65 +1,72 @@
 {
-"add":[0.09571152, 0.08334279, 0.07070262, 0.14084256, 0.04825167, -0.12893851, -0.12180215, -0.07194936, -0.05329844, -0.28276998, 0.00202254, 0.07488817, -0.16139749, 0.07618549, -0.06084673, 0.10536429, -0.03031596, 0.16402271, 0.22730531, -0.0015432, 0.01986631, -0.15808797, 0.03030382, -0.04201249, 0.01689876, -0.30041003, -0.06479812, 0.1282431, -0.09032831, 0.14007935, -0.23540203, 0.00552223, 0.15325953, 0.13147154, -0.15204638, 0.1616615, 0.05758473, -0.1385157, 0.16438901, -0.14482859, 0.07895052, 0.18407233, 0.15283448, -0.00081216, -0.17934133, -0.16779658, 0.09044863, -0.18453072, -0.00552684, 0.01191218, 0.18504514, 0.13140059, -0.0174882, -0.18127315, 0.02269725, -0.02048657, 0.10858727, 0.0074029, 0.09485064, -0.13431476, 0.07491371, -0.17498694, -0.32914585, -0.14159656, -0.03594542, 0.04091231, 0.00298631, -0.08933277, -0.07178984, 0.04144038, 0.12151367, -0.09504163, 0.13691336, 0.07825345, 0.13958281],
-"alloca":[0.12644756, -0.20912014, 0.07298578, 0.13963142, 0.03932726, -0.12778169, -0.13084207, -0.09090705, -0.01497083, 0.0646746, 0.14847252, 0.08546949, -0.11579383, 0.07812478, -0.16431178, 0.08578802, 0.06094746, -0.1239017, -0.02789196, -0.01074964, -0.1738938, -0.1629063, -0.07137732, -0.02845656, 0.1728318, 0.13779363, -0.06250989, 0.03479109, -0.08423121, 0.14009888, 0.09620709, -0.02287545, -0.04616966, -0.19591278, -0.19636007, 0.21825573, 0.07949521, -0.14614704, 0.17752613, -0.15099092, -0.04518029, 0.17774306, 0.18947133, -0.00197501, -0.12619424, -0.18458585, -0.09960615, 0.01162648, 0.21306573, 0.0468024, 0.201505, 0.09970672, 0.06599383, 0.17757463, -0.11899275, 0.10299123, 0.06038174, -0.07116731, -0.15743989, -0.1258558, 0.1535076, 0.01872367, -0.14336765, -0.19708565, -0.26796287, 0.02628843, -0.0009325, -0.08809827, -0.12970725, 0.17223337, -0.10651242, -0.09162157, 0.03264611, 0.0911452, 0.09506542],
-"and":[-0.12902537, 0.07848564, 0.07048505, 0.13578993, 0.03570583, -0.1242408, -0.12619697, -0.06914084, -0.0515872, 0.13225996, 0.01673339, 0.0763608, 0.18325369, 0.07690141, -0.09591792, 0.10353354, -0.02401061, 0.16493416, 0.21942355, -0.00400911, 0.03811587, -0.15406959, -0.07134877, -0.02946596, -0.03854588, 0.28656727, -0.06625008, 0.12780443, -0.04631379, 0.13919763, -0.15808977, -0.00312698, 0.14692454, 0.18495218, -0.14863448, 0.13449706, 0.06471325, -0.13883992, 0.17630532, -0.16833898, 0.07391811, 0.17909151, 0.18229873, -0.00381102, -0.17680968, -0.1645554, 0.10016466, -0.07963493, 0.00130218, 0.05646244, 0.18222143, 0.10511146, -0.0191175, -0.08713559, 0.25423968, -0.02557301, 0.04319789, 0.03259414, 0.07209402, -0.13169754, 0.07424775, -0.17216511, -0.32057068, -0.13833733, -0.0658454, 0.02420194, -0.04393166, -0.08238692, -0.07023077, 0.04014502, 0.20101993, -0.09093616, 0.13076238, 0.09114857, -0.06483845],
-"ashr":[0.16414718, -0.02078147, 0.07353837, 0.14210321, 0.08068018, -0.13586298, -0.15728961, -0.10365791, 0.00359197, 0.04608767, 0.35770142, 0.08003625, 0.00944533, 0.08471741, 0.05809571, 0.09202059, -0.18349345, 0.22169511, 0.24418135, 0.19192688, -0.1956347, -0.17401719, -0.07583863, -0.02781139, 0.0952112, -0.01444751, -0.27259794, 0.14392436, -0.13541143, 0.1410963, 0.04314299, -0.01641751, -0.05448177, -0.22287542, -0.2131822, 0.25112826, 0.06918604, -0.14414005, 0.060288, -0.09658266, -0.09122665, -0.02779044, 0.20349248, 0.0041391, 0.10610974, -0.20890503, -0.09881835, -0.07368057, 0.22467837, 0.00075097, 0.2147348, -0.02612463, -0.01696278, -0.0649786, 0.15041149, 0.02361087, 0.0211603, -0.03706622, 0.18296233, -0.14298625, 0.14614436, 0.02273145, 0.0209446, -0.21062987, 0.16911499, -0.03668665, 0.00197532, -0.09607943, -0.08398084, 0.02006913, 0.05739584, -0.07919859, 0.19634944, 0.11082727, -0.06584227],
-"bitcast":[0.1675692, -0.12870269, 0.04048132, -0.1670965, -0.1279611, 0.02615386, -0.16829294, -0.09034907, 0.10913523, -0.07819421, 0.23986322, -0.05966561, 0.08410738, 0.19072439, 0.06047394, 0.02999627, -0.16747619, -0.06076627, -0.02673951, -0.1619169, 0.06443421, -0.13788716, -0.05644303, 0.01361013, -0.06858975, -0.06005004, 0.10011288, -0.05508338, -0.10613093, -0.11281271, -0.00758647, -0.12425531, 0.05333719, -0.16881412, -0.20088236, 0.06015657, -0.16405901, 0.06226884, 0.09171099, -0.09500738, 0.07907875, 0.0776544, -0.18457057, -0.11278627, -0.12111131, -0.10180638, 0.18328871, 0.18770072, 0.20346186, 0.10305139, -0.18344335, 0.10693213, -0.10920919, -0.05994263, 0.20354497, -0.03093485, 0.14214055, 0.00580597, 0.10480052, -0.09955201, -0.134185, -0.02904563, 0.00175069, 0.17646657, 0.01348841, -0.02030338, -0.06537742, 0.10032237, 0.15315783, 0.0102667, 0.07717734, -0.01060431, 0.14727928, -0.16261302, -0.06770234],
-"br":[0.11499645, 0.0824301, 0.07218035, 0.13258332, -0.13419574, -0.12916718, -0.12626709, -0.06741403, -0.02857593, -0.2543893, 0.02193022, 0.0760875, -0.1562702, 0.07712954, 0.3149361, 0.10217083, -0.041038, 0.16601022, 0.01906607, -0.02043359, 0.05471838, -0.15233372, -0.06945753, -0.02313732, -0.07342829, 0.3331809, -0.05246306, -0.0269839, -0.05435036, 0.13908924, 0.32604694, 0.00170966, 0.14997493, 0.13026518, -0.14908995, 0.12238151, -0.06773318, -0.13566032, 0.16068587, -0.12842499, 0.04970508, 0.17827724, 0.09729939, -0.00447832, -0.1739753, -0.16429187, 0.09886666, -0.08058207, 0.00714044, 0.04585538, 0.13424252, 0.11376464, -0.01675582, 0.17901348, -0.00653374, -0.01570439, 0.13032894, 0.01734108, 0.16833901, -0.1173776, 0.07662185, -0.15942436, -0.21173944, -0.10505079, 0.0597497, 0.03491669, 0.00338842, -0.04969047, -0.07644061, 0.04528612, 0.254365, -0.09514527, 0.12015092, 0.08262096, -0.02352029],
-"call":[0.10815012, -0.12419116, 0.18759736, -0.1905027, 0.01619313, -0.13483052, -0.12278763, -0.07051246, -0.0083437, 0.25107145, -0.16601063, 0.08127163, -0.17432374, -0.18380919, 0.24335551, 0.07208319, -0.04401246, 0.11606008, -0.02733191, 0.02098145, 0.019888, -0.13705409, -0.07569158, -0.03072285, 0.16870692, -0.09787013, -0.09340432, 0.01931342, -0.03557841, 0.14359893, -0.1592094, -0.00055867, 0.159316, 0.1099042, -0.11837319, 0.08741318, 0.03364393, -0.12831019, 0.10450637, -0.12699029, -0.20213994, 0.18390144, 0.11092624, -0.00209971, -0.13063665, 0.19996215, 0.09006448, -0.07840014, 0.22549215, 0.02587176, 0.13374338, 0.11009877, -0.01874998, -0.21446206, 0.02377797, -0.01036531, 0.05427047, 0.01418843, 0.00771817, -0.12639529, -0.10334941, -0.12244401, 0.30014148, -0.09857437, 0.21212636, 0.03429029, -0.04947309, 0.1023307, -0.07743628, 0.03006962, -0.24868701, -0.02357339, 0.11574048, 0.06895301, -0.363474],
-"constant":[-0.2850312, -0.22839, 0.12669143, -0.0674703, -0.12639391, -0.00477266, 0.04786542, 0.06336267, 0.08660185, 0.12805316, -0.07146342, 0.21539183, -0.0624397, -0.02638953, -0.28688517, 0.28374302, 0.05338082, 0.05559688, -0.13133128, 0.12440272, -0.03583231, 0.29848817, 0.13930812, 0.15453401, 0.0538353, -0.06874479, 0.00262802, 0.27964258, 0.19028014, -0.16371843, -0.05762961, 0.20059372, -0.20804578, -0.06549844, 0.09732475, -0.01551855, 0.21226783, 0.05889762, -0.07560658, 0.11312829, -0.04594622, -0.27309528, -0.05293005, 0.18953343, 0.05463868, -0.31045213, -0.04364616, 0.11005993, 0.12489324, -0.05413342, -0.05814561, -0.26131225, -0.18863814, 0.31165487, -0.08796364, -0.19958755, -0.10849535, 0.14899114, -0.01385941, 0.29359573, -0.01349372, 0.0562498, 0.10977754, 0.08993197, 0.06231657, -0.13509868, -0.20968516, 0.03578237, 0.15356435, -0.17766887, -0.11509016, 0.06898113, -0.06665445, -0.14065051, 0.34711906],
-"extractelement":[-1.62098512e-01, -2.00751424e-02, 1.71776459e-01, 1.38723463e-01, -1.60769299e-01, 9.30800363e-02, -1.24304645e-01, 1.45934001e-01, -5.04420400e-02, -7.54220188e-02, -5.30924797e-02, 8.55294541e-02, 1.17488159e-02, -1.82809234e-01, 9.37484950e-02, 8.38199258e-02, 1.26266479e-01, 9.31843743e-02, -2.26742271e-02, 6.50950940e-03, 5.02749532e-03, -1.14133619e-01, 1.80842891e-01, -9.63811725e-02, 1.67923152e-01, -5.84629439e-02, 1.04362026e-01, 8.92093012e-05, 7.93750137e-02, 1.39107972e-01, -8.40696543e-02, -1.59506593e-02, 1.99361086e-01, 1.93857521e-01, -1.46850392e-01, -1.78695560e-01, 9.88712162e-03, -1.40836969e-01, 1.77736521e-01, -1.61047727e-01, 3.51648539e-01, 1.79638579e-01, 1.49385989e-01, -6.06128015e-03, -1.72102928e-01, -1.81016047e-02, 1.01466164e-01, -1.28312945e-01, -8.05212185e-02, 6.28385469e-02, 1.15654446e-01, 1.91848025e-01, 3.03963851e-02, 3.55794169e-02, 1.40873834e-01, -1.44319711e-02, 1.85423180e-01, 1.16111919e-01, -7.47816712e-02, -1.14719503e-01, 1.02934733e-01, -1.83810964e-01, -2.64400076e-02, -8.99282843e-02, -1.90383971e-01, 7.31386840e-02, -4.36249487e-02, -4.71482053e-02, 1.07486300e-01, 1.09736443e-01, 4.15226035e-02, -1.42309800e-01, 8.96709636e-02, 6.64985999e-02, -6.13647103e-02],
-"extractvalue":[0.08820406, -0.0182279, 0.01493346, -0.17219813, 0.02927338, -0.17379265, -0.05663937, -0.06805898, -0.21235467, -0.01969833, -0.15152055, -0.18049374, 0.01062911, 0.07935719, -0.01993761, 0.12405304, -0.03198355, 0.13872959, 0.18017697, -0.0100032, 0.22617207, -0.21553156, -0.18403597, 0.10906533, 0.17709404, 0.07438782, 0.0875822, -0.22567064, -0.00636844, 0.06691552, 0.05859367, -0.06149556, 0.15721355, 0.01889951, 0.2164152, -0.07916643, -0.03927743, -0.02665933, 0.13756634, -0.08835335, -0.01159343, -0.19749148, 0.09438482, -0.1696074, -0.18197437, 0.19907822, -0.20855112, 0.21903005, -0.2204043, -0.07439119, 0.1308343, 0.10343243, -0.10401972, 0.02784402, -0.12545246, -0.18498465, 0.02385085, 0.22105947, -0.01296441, 0.19779074, -0.09320201, -0.07345647, -0.06985376, -0.0657003, -0.15406364, 0.14981052, 0.01336148, 0.18544021, -0.07248794, 0.0511543, -0.00308717, 0.01915094, 0.079139, 0.19975829, -0.06093699],
-"fadd":[0.0851364, -0.01757606, 0.20865884, 0.17407735, -0.17661206, 0.16181652, -0.11895371, -0.06142977, -0.05126655, 0.04448467, -0.03589934, 0.11921146, 0.0143434, -0.18255684, 0.05196553, 0.10967412, 0.15810761, 0.13785522, -0.01255003, -0.00134187, 0.03563518, -0.12446801, 0.18817455, -0.08609993, 0.1893248, 0.13434686, 0.25903776, -0.00467659, 0.09800702, 0.14749499, 0.03458502, -0.00238901, -0.1395337, 0.19306736, -0.11638073, 0.10491668, 0.04428702, 0.18128034, 0.1797104, -0.1929282, 0.07353631, 0.19062985, 0.10042762, -0.0040207, -0.18379262, 0.07060277, 0.09594815, -0.18272707, -0.12928157, 0.07631373, 0.13005428, 0.24108389, -0.01617561, -0.05803563, 0.02686582, -0.02223067, 0.21217403, 0.17594706, 0.00500477, -0.07345455, -0.08176229, -0.19486704, 0.02177307, -0.09052874, -0.14158368, 0.0890988, -0.06339125, -0.04612265, 0.10199347, 0.11777309, 0.07302366, -0.13303186, 0.09683178, 0.05031883, -0.05881981],
-"fcmp":[1.02981135e-01, -2.43355539e-02, 5.85255250e-02, 1.09401904e-01, -1.87941462e-01, 1.74530044e-01, -1.20533399e-01, -8.64934921e-02, -5.54629341e-02, 1.41877215e-02, 4.80354428e-02, 1.56707644e-01, 2.56910548e-02, -1.94340080e-01, 5.11265621e-02, 8.82050097e-02, 1.67082191e-01, 1.14665776e-01, -2.00159680e-02, -1.82088744e-02, 2.09244549e-01, -1.15076765e-01, 2.03621030e-01, -8.38057026e-02, -4.32270877e-02, 6.39017820e-02, -6.50197491e-02, -2.72039324e-03, 8.68055448e-02, 1.25798717e-01, -4.21847135e-01, 2.64590848e-02, 1.56372517e-01, 2.14883089e-01, -1.12701654e-01, 9.43767577e-02, -1.97456375e-01, -1.72886148e-01, 2.04119727e-01, -1.56766623e-01, 7.38008767e-02, 2.05716744e-01, 3.48394439e-02, -2.48072352e-02, -2.03255862e-01, 5.07165631e-03, 1.12850599e-01, -1.94697857e-01, -6.29984289e-02, 8.21878910e-02, 1.23685144e-01, 1.40430763e-01, -2.20402889e-02, -6.55216798e-02, 2.90598303e-01, -7.66792744e-02, 2.16097474e-01, 6.28568381e-02, 2.69129931e-04, -1.17002167e-01, -1.60454452e-01, -2.07048669e-01, 2.62669493e-02, -9.61467475e-02, -1.88501909e-01, 8.80606323e-02, -5.39420024e-02, -5.08916602e-02, 1.09891914e-01, 1.21375419e-01, 5.10863326e-02, -1.43003076e-01, 8.39024931e-02, 9.51330177e-03, -6.65674359e-02],
-"fdiv":[-0.14435205, -0.02003789, 0.07479589, 0.13465217, -0.18792528, 0.16504793, -0.12187403, 0.00537306, -0.04864043, 0.04958175, -0.14890797, 0.10156095, 0.01710029, -0.1809697, 0.06765524, 0.18452686, 0.16923915, 0.11264159, -0.01412142, -0.00287484, 0.01297035, 0.06755029, 0.18517323, -0.08975757, 0.16194478, 0.04174679, 0.10903918, 0.00072231, 0.10999232, 0.14989018, 0.05669819, -0.00733533, 0.20606744, 0.22580206, -0.14587933, 0.13480443, 0.05984581, -0.19580479, 0.20258778, -0.17146486, 0.07523733, 0.19139282, 0.03738374, -0.01550991, -0.21507922, 0.07425626, 0.10224851, -0.18437587, -0.06662318, 0.07977687, 0.12355862, 0.1434375, -0.01888226, 0.03870944, 0.07555477, -0.01191964, 0.23488866, 0.06922436, -0.07203218, -0.06884275, 0.07191673, -0.19515742, 0.02168754, -0.08983592, -0.20972523, 0.08903123, -0.05095004, -0.04967143, 0.0985181, 0.15771264, 0.09441628, -0.14769785, -0.16405083, 0.05066825, -0.058015],
-"floatTy":[-0.04556743, 0.03822431, -0.08842288, -0.12055657, 0.0960574, 0.2371231, 0.09947137, 0.11064581, -0.12371849, -0.06109007, 0.06440615, 0.14924356, 0.13218448, -0.04400141, 0.01761996, -0.13240574, 0.21056518, -0.14215694, -0.14700417, -0.16416645, -0.07137518, 0.05240471, -0.10071051, 0.12556827, 0.08208757, 0.06504779, 0.1578807, -0.09501757, 0.00659264, -0.00527758, -0.01085964, -0.17290565, 0.00501871, -0.10288252, 0.09658113, -0.107755, -0.1718967, 0.16306138, -0.080802, 0.10203532, -0.02934794, -0.02899184, -0.17915869, -0.19883338, -0.05124965, 0.16747124, -0.05587617, 0.14325564, -0.18054038, -0.161391, -0.1149613, 0.01600737, -0.0704658, 0.01935361, -0.04516762, -0.1305524, 0.04487818, -0.07881694, -0.18929696, 0.05364102, -0.17827097, 0.09934752, -0.04212811, 0.15036745, -0.18782069, 0.19947147, 0.1102478, -0.1304183, 0.17145109, 0.22485495, 0.02270225, 0.08804839, -0.12520337, -0.07780997, 0.00057234],
-"fmul":[-0.05690098, -0.02700638, 0.20705073, 0.16630849, -0.17719169, 0.16012336, -0.12955493, 0.19698587, -0.04035136, -0.03197788, 0.08116172, 0.1013689, 0.01416616, -0.17855197, 0.15503113, 0.13990149, 0.15355295, 0.12397105, -0.01912402, 0.01684221, 0.00669227, -0.11869009, 0.17604592, -0.05788622, 0.10467764, -0.0807798, 0.10300152, 0.10780353, 0.08334652, 0.1446782, 0.03546653, -0.00421023, 0.19682531, 0.19603632, -0.1466306, 0.12923796, 0.06296455, -0.14397427, 0.183406, -0.19625223, 0.08244827, 0.1860209, 0.10110238, -0.002166, -0.18034802, 0.06406105, 0.09568714, -0.17962225, -0.05463478, 0.04498994, 0.11928298, 0.1694295, -0.01222703, 0.14453876, 0.13506988, -0.01149808, 0.20273286, 0.05309585, -0.07139173, -0.07370458, 0.09957068, -0.19369006, 0.02577178, -0.09011577, -0.21375039, 0.07624438, -0.04969844, -0.04844297, 0.08409201, 0.11113621, 0.14570779, -0.13936704, 0.07202481, 0.05272412, -0.05747499],
-"fneg":[-0.04075071, -0.09381824, 0.17104743, 0.0744117, -0.15771168, 0.16444896, -0.11586417, 0.00200867, 0.17323531, -0.10421973, -0.05619403, 0.07496857, 0.05573901, -0.18903743, 0.06586835, 0.03499747, 0.17244707, 0.02476844, -0.02552369, 0.00063759, 0.04312319, -0.04501259, 0.1850581, -0.14339973, -0.04433612, 0.01609048, 0.10461161, -0.12243931, 0.07718915, 0.15313269, 0.12116063, -0.03062805, 0.21040255, 0.21582894, -0.10389283, 0.09344483, -0.20219979, -0.20113027, 0.19945832, -0.17248249, 0.07845841, 0.19814597, 0.06085683, -0.02476168, -0.18729973, 0.0674377, 0.1058675, -0.04973035, -0.10124692, 0.12834774, 0.0900365, -0.17527112, 0.09901146, -0.05764115, 0.03708475, 0.13003437, 0.2325445, 0.19712777, -0.07403741, -0.05580256, 0.18192469, -0.19122703, -0.05592606, 0.09128745, -0.15161441, 0.08734331, -0.0963953, -0.0377482, 0.18287936, 0.16522603, 0.03490613, -0.13519889, 0.08505893, 0.02470117, -0.06158587],
-"fpext":[0.1312062, -0.0272035, 0.1562131, 0.11956131, -0.18370572, 0.16904335, -0.18538451, -0.0905571, -0.05687085, -0.12067703, 0.15717801, 0.06977441, 0.04604319, -0.18692835, 0.07965986, 0.03840762, 0.17120989, 0.03048515, -0.02593113, -0.02722386, 0.0637762, -0.08012746, 0.03968937, -0.09751038, -0.04150679, -0.05797326, 0.09053179, -0.06100635, 0.01466092, 0.1521789, 0.0178679, 0.00468684, 0.09077137, 0.22244066, -0.21890686, 0.09471557, -0.18849488, 0.06589299, 0.20329474, -0.20006016, 0.06712949, 0.19578859, -0.00594554, -0.03142307, -0.18011327, 0.00516408, 0.18754548, -0.18884791, 0.2184627, 0.09512166, 0.07513344, 0.12641825, -0.0237517, -0.06020509, 0.15404698, -0.02384854, 0.22717056, 0.11279562, 0.16759154, -0.09342691, -0.09703359, -0.19393681, -0.06580188, 0.190172, -0.21666776, 0.09756065, -0.06633368, -0.04663929, 0.16539453, 0.16365078, 0.04338961, -0.1387515, 0.07042016, 0.03850469, -0.06256048],
-"fptosi":[-0.09471355, -0.01413281, 0.20098655, 0.17478812, -0.0103814, 0.11870264, -0.10561193, 0.19406088, -0.04215302, 0.09856935, -0.05858651, 0.11069191, -0.02667271, -0.17638545, 0.00591833, 0.12538631, 0.1609231, 0.143028, 0.00411847, 0.06113226, -0.1577999, -0.12172183, 0.18763137, -0.04490714, 0.0262733, 0.12005143, 0.08305117, -0.00109107, 0.03972085, 0.13256785, 0.12486281, -0.00920427, 0.19794717, 0.13488762, -0.08073806, 0.11790548, -0.1865304, -0.18761554, 0.15602915, -0.1440169, -0.01538679, 0.18490645, 0.05900477, -0.01250085, -0.17872328, 0.07583775, 0.09906318, -0.16909951, -0.12062778, 0.03089247, 0.11974704, 0.23940024, -0.00561649, 0.03862159, 0.13437317, -0.0128777, 0.2149731, 0.07683202, -0.10922348, -0.04304254, -0.20104879, -0.16371554, -0.08326109, -0.08942039, -0.03989649, 0.07596397, 0.18141732, 0.08706582, -0.06605583, 0.09984156, -0.05919264, -0.13918152, -0.11102735, -0.04074531, -0.05989955],
-"fptoui":[0.13186027, -0.02009563, 0.05399049, 0.22092278, 0.0764356, -0.12116565, -0.10575569, -0.08263665, -0.05279351, 0.00392524, -0.06554782, 0.15926358, 0.01012308, -0.18726377, 0.02134197, 0.12730621, -0.05603928, 0.1433069, -0.21253656, -0.1455532, 0.23767456, -0.13623604, 0.21878564, -0.05533312, -0.02895407, 0.03911315, -0.2773476, -0.00738798, 0.08483762, -0.14270262, 0.04603437, 0.21681121, -0.20425414, 0.12532364, -0.04144387, -0.1759266, 0.06860236, -0.14378744, 0.16322674, -0.09162073, -0.04966446, 0.1867569, 0.11250684, 0.04158355, -0.20126882, 0.0963328, 0.06281191, -0.17627244, -0.16096021, 0.00596877, 0.12892367, 0.2650723, 0.0032763, -0.05754196, 0.14840715, -0.09938947, 0.13039768, 0.04905574, -0.04759435, -0.18545668, -0.22984591, -0.22227132, -0.02666637, -0.09771203, -0.15587328, 0.07147411, 0.21007161, 0.09932306, -0.06597851, 0.04478595, -0.00686691, -0.14754876, -0.16366987, -0.22182341, -0.05699158],
-"fptrunc":[0.1180348, -0.01865118, 0.19759397, 0.00099745, -0.20069243, 0.15863162, -0.10404988, -0.03822596, -0.04113388, 0.02389156, -0.15024376, 0.10156121, 0.01125149, -0.17805247, -0.01032893, 0.11541142, 0.16151612, 0.08652765, -0.01421094, -0.02742836, 0.03890061, 0.16992694, -0.19120258, -0.08296357, 0.16136265, 0.13252915, 0.08675185, -0.05066127, 0.14796199, 0.16224207, 0.07839199, -0.0724185, 0.17859218, 0.20112462, -0.11284997, 0.04566038, -0.19799039, -0.17690767, 0.18034197, -0.14002973, -0.0558032, 0.18783137, -0.14857374, -0.03776158, -0.21491641, 0.04907086, 0.10290487, -0.17882703, -0.06566726, 0.09571292, -0.1965507, 0.1673165, -0.03019422, 0.04259395, 0.10474471, -0.01224911, 0.2133804, 0.08484804, -0.01221132, -0.04942748, -0.06522352, -0.16705054, -0.07432696, -0.01365143, -0.14369024, 0.09237107, -0.06833915, -0.04292256, 0.10805372, 0.10829192, -0.04953781, -0.16152975, 0.08096681, 0.12343118, -0.04967185],
-"freeze":[-0.10947239, 0.16156663, 0.0653074, -0.16102187, 0.05866997, -0.11313032, 0.18013522, 0.00849657, -0.13109621, 0.00250287, 0.12335391, -0.19474623, 0.00160397, 0.18526912, -0.02094408, 0.10131565, -0.17163642, 0.1624736, 0.20699912, 0.1198829, -0.15380894, -0.13946483, -0.0666218, -0.02034315, 0.06200019, 0.04227962, 0.07729523, 0.1244237, -0.07141764, 0.12154905, 0.0300131, 0.01618693, 0.13966976, 0.08918943, -0.07834358, 0.19349128, -0.06925047, -0.11661758, 0.01764905, -0.06631834, -0.19618054, 0.07797705, -0.18398666, -0.08675893, -0.14833811, -0.1468038, 0.18818453, -0.06729261, -0.00497526, -0.08958972, 0.16964634, -0.10325264, -0.10212341, -0.04745837, 0.12241088, -0.03629779, -0.00606445, 0.01766711, 0.0036858, -0.10732682, -0.13172524, -0.10182981, -0.06431175, -0.13082966, 0.04738441, 0.02668242, 0.15197244, -0.12880892, -0.06579075, 0.03165649, -0.00145413, -0.02683991, 0.13963282, -0.14597963, -0.05104417],
-"fsub":[-0.15878558, -0.01853204, 0.07496438, 0.13546987, -0.1904581, 0.16564278, -0.1120265, -0.05418859, -0.08075086, 0.1190263, -0.030006, 0.07342444, 0.01767328, -0.18650489, 0.04966565, 0.17190282, 0.1616953, 0.10582477, -0.00931773, 0.00157398, 0.00862398, 0.08480153, 0.17171694, -0.0916861, 0.17196383, 0.03703162, 0.11399966, -0.12855959, 0.05810672, 0.15144408, 0.05618297, -0.00800864, 0.20814295, 0.20056027, -0.1153914, 0.10849892, -0.1970011, -0.1813781, 0.1921156, -0.16984123, 0.07333071, 0.19433162, -0.01313439, -0.02830642, -0.17687869, 0.05937865, 0.1070236, -0.18605019, -0.07227098, 0.10597191, 0.1225975, 0.12388534, -0.02131494, -0.02420847, 0.00146665, -0.01353394, 0.22151577, 0.19731705, -0.07575841, 0.08710405, -0.07411117, -0.17442936, -0.1831972, -0.0886184, -0.17084508, 0.10038229, -0.06385624, -0.04644644, 0.10368951, 0.12130242, 0.24595715, -0.14661786, 0.09260331, 0.0455667, -0.05950425],
-"function":[-0.08280793, -0.18925415, -0.1866685, -0.04932962, 0.13478681, 0.203617, 0.05739383, 0.09856346, -0.17600478, 0.07619468, -0.12438924, -0.02529626, -0.07468224, 0.03905433, 0.11325707, -0.04229602, -0.16354711, -0.08952031, -0.15966323, 0.11114867, 0.07319006, 0.02678379, -0.13364255, 0.15032487, 0.0822423, 0.03195171, -0.00871139, -0.1128016, 0.19261838, -0.07204764, -0.05823177, 0.1990249, -0.2057403, -0.08260711, 0.09775644, -0.12436705, 0.15814295, 0.0463977, -0.0493853, 0.1450742, -0.07108644, -0.03192589, -0.11917686, -0.19753116, 0.09248564, 0.13522607, 0.16347136, 0.11065657, -0.0850426, -0.19110996, -0.15060848, -0.25829738, 0.11464069, -0.15613398, -0.07256057, 0.16283846, -0.11428182, -0.1879146, -0.10537492, 0.03637636, -0.02642156, 0.0344548, -0.00722412, 0.13349552, 0.05913915, 0.17941767, -0.20852967, 0.04403022, -0.13914339, -0.17848521, -0.07228794, 0.23070297, -0.11848032, -0.14098541, -0.23762096],
-"getelementptr":[0.09870283, 0.07089636, 0.07129486, 0.13141522, 0.0512748, -0.12164738, -0.10822044, -0.06830852, -0.03589667, 0.23281692, 0.00221914, 0.07347875, -0.14414017, -0.1762956, 0.2280886, 0.09720317, -0.02157139, 0.1186724, 0.16809724, 0.01354192, 0.00843806, -0.1289462, -0.06982945, -0.03921828, 0.11689314, -0.27946517, -0.06115843, 0.0435549, -0.04346889, 0.13182928, -0.13756175, -0.00165133, 0.14864644, 0.1256485, -0.10831792, 0.1638358, 0.05359713, -0.12713233, 0.15040766, -0.14046451, 0.06688947, 0.17097251, 0.11004995, 0.00303981, -0.17120391, -0.15561967, -0.0996208, -0.07566627, -0.0092614, 0.03340579, 0.12288038, 0.11154807, -0.01677553, 0.161112, 0.01968472, -0.01360116, 0.05185669, 0.03974737, 0.08288419, -0.11881893, 0.09566113, -0.14562541, 0.27193576, -0.08737413, -0.13493136, 0.06975145, -0.01905861, -0.04629398, -0.06382514, 0.03648283, 0.32102475, -0.07868497, 0.11246872, 0.06408555, 0.21986632],
-"icmp":[0.10752141, 0.08221146, 0.07407488, 0.13139327, -0.17970721, 0.02754223, -0.13538423, -0.06698897, -0.02524848, 0.29738554, 0.00894943, 0.07728788, 0.01191065, 0.07896648, 0.26208735, 0.10498706, -0.03250087, 0.1684631, 0.23128033, 0.00553169, 0.05936556, -0.15531996, -0.06056314, -0.03261358, 0.09216364, -0.31626984, -0.07434817, 0.12870958, -0.08664963, 0.14354071, 0.2156426, -0.0043262, 0.16025512, 0.13890502, -0.153901, 0.12319393, 0.01295959, -0.14369431, 0.17033054, -0.12931693, 0.07678536, 0.19203088, 0.11493351, -0.00601692, -0.18671934, -0.16845967, 0.11059918, -0.08381938, -0.00690406, 0.04286021, 0.13590424, 0.12681794, -0.01967513, -0.21233313, 0.02998303, -0.01612004, 0.17794448, 0.02153118, 0.09134272, -0.12414944, 0.08073039, -0.17339677, -0.0417021, -0.1026428, -0.0270442, 0.02959586, -0.05658696, -0.05483485, -0.07289126, 0.036765, 0.20820136, -0.0922353, 0.1438483, 0.06928346, -0.37188163],
-"insertelement":[-0.08810953, 0.19707826, 0.23040843, 0.15133125, -0.22085261, 0.18197243, -0.13767323, 0.00611759, -0.03750934, 0.05303819, -0.00140541, 0.13628025, 0.01222425, -0.19765897, 0.02197562, 0.17957552, 0.17369562, 0.13818526, -0.01018569, 0.07335348, -0.00122391, -0.05268616, 0.03798695, -0.07208619, 0.13593383, -0.08763747, 0.10625125, 0.12024429, -0.01286297, 0.16619073, 0.03609107, 0.00169814, 0.21330379, 0.22023544, -0.11971844, 0.23055589, -0.21922874, -0.18983133, 0.21032487, -0.19545083, 0.39695147, 0.2116961, 0.03465794, -0.01642283, -0.23929499, 0.05254361, 0.1186865, -0.20114459, -0.01001151, 0.06816892, 0.13270052, 0.13516793, -0.02193176, 0.13708963, 0.15222688, -0.01366894, 0.22591114, 0.07127486, -0.08461929, -0.08140475, 0.06961198, -0.18172547, 0.03001983, -0.09530003, -0.03038635, 0.09877217, -0.06264398, -0.06664777, 0.08449566, 0.13536829, 0.04701871, -0.1011354, 0.08900025, 0.0484806, -0.0840841],
-"insertvalue":[-0.18927452, 0.0174884, -0.14182499, 0.13622768, 0.0860026, 0.09214608, 0.19103362, -0.21386355, -0.08520557, 0.0039752, 0.11396001, -0.20266375, 0.17875974, -0.19441572, 0.06652557, -0.1076987, -0.15872131, -0.20631677, -0.06141828, 0.01331364, 0.02354101, 0.08883859, 0.18617646, -0.1524484, -0.00485444, -0.06658852, -0.06168855, -0.15898797, 0.02933339, -0.19443747, -0.08171376, -0.10284599, -0.14775087, 0.20585667, -0.03085143, -0.20518455, 0.05644068, 0.18632361, -0.14686832, 0.03718346, 0.10764548, -0.01129938, 0.19590198, 0.02194332, -0.04761516, 0.0276675, 0.01006511, -0.05134218, -0.10854474, 0.23528893, 0.09621219, -0.17192629, 0.10312149, -0.14351319, 0.15347119, 0.19321586, -0.20537563, 0.19100618, 0.11280359, 0.07242113, 0.18901348, 0.19133772, 0.02062779, -0.07726028, 0.00578185, -0.02625765, -0.10374335, -0.02086248, 0.18915856, 0.00862997, 0.04699488, -0.0631949, 0.18942626, 0.01894571, 0.02351614],
-"integerTy":[-0.02392217, 0.13147527, -0.12684862, -0.00286194, 0.18320721, -0.03383226, 0.10688385, 0.20315196, 0.16910973, 0.01749469, 0.09163074, 0.18359211, 0.08457439, 0.17821081, -0.01359864, -0.09173124, -0.1060688, -0.0317139, 0.089481, -0.13937937, 0.1258349, -0.00968946, -0.00211832, -0.13140695, -0.11206195, -0.0249014, -0.09646855, 0.16041876, -0.20260467, -0.1008801, -0.02906649, -0.148828, -0.01750175, -0.10806648, 0.10094456, -0.09347772, -0.11941694, 0.0714565, -0.13869469, 0.09965917, -0.02859109, -0.04663063, -0.03677037, -0.17767252, 0.19819601, -0.00728577, -0.07156438, 0.14130634, -0.16161495, -0.14514765, -0.0230404, 0.14609018, 0.12496789, -0.02587292, 0.06375326, -0.1703029, -0.14651431, -0.15148057, -0.11655853, 0.00689279, -0.21302195, 0.10692985, 0.00105841, 0.05685481, -0.05294221, 0.13358746, 0.17953119, 0.01200722, 0.00904744, 0.13893746, -0.01024614, 0.14703822, -0.08035436, 0.18355256, -0.00407319],
-"inttoptr":[0.04721976, 0.16880296, 0.06448112, -0.17660472, 0.02998716, -0.1156636, 0.13852204, -0.06320232, -0.03162635, -0.06644673, -0.136013, -0.06916367, -0.02801188, 0.10646518, 0.05828808, 0.03782522, -0.02227561, 0.02234134, 0.16369314, -0.08557447, 0.03853939, -0.14319475, -0.05599201, -0.01060566, -0.1136315, 0.0201032, 0.01836221, -0.05016, -0.1216938, 0.12626994, 0.03450989, -0.19574732, 0.14352062, -0.17880194, -0.13339539, -0.04630446, 0.03136644, -0.13006681, 0.1445991, -0.09556036, 0.09764113, 0.16694036, 0.09833785, 0.05145761, 0.17970908, -0.14575475, -0.13704927, -0.06876626, -0.00955433, -0.16089469, -0.14152728, 0.1012365, -0.18291287, -0.07371435, -0.13809073, 0.02210419, 0.090786, 0.02507522, 0.15311833, -0.11663677, 0.10619268, 0.1758742, -0.04052401, 0.19525726, 0.15682434, -0.19402866, 0.08792165, 0.19677114, 0.1986662, -0.2065629, 0.04007441, -0.01697999, 0.17015004, 0.07224851, -0.06042317],
-"invoke":[0.11604629, -0.02348489, -0.11863736, -0.16840768, -0.01534824, -0.13232489, -0.09663513, -0.1576129, -0.2348036, 0.00802987, -0.0517984, -0.05321365, 0.01174416, -0.11655734, 0.05259984, -0.16816719, -0.09482966, -0.00228002, -0.12152751, -0.00498575, 0.03465938, -0.02950593, 0.04545004, -0.2172498, -0.07073424, -0.08918925, 0.0623108, -0.20881031, -0.04182759, -0.20031597, -0.1351732, 0.069332, -0.00361818, 0.11065516, 0.13967985, -0.08796342, -0.00256544, -0.11508698, 0.09178918, 0.2073513, 0.04010937, -0.07716485, 0.06940974, 0.20694868, -0.08160415, 0.1820004, -0.12159622, 0.04531517, -0.00216078, -0.11625812, -0.10868325, 0.11549117, 0.19036727, 0.0365869, -0.09580094, 0.22367325, 0.11756624, -0.2184826, 0.27060437, 0.14232084, 0.01686368, -0.10716032, 0.02151413, -0.02634864, 0.19014603, -0.22087607, -0.05267305, -0.2422757, -0.06234193, -0.0299236, 0.01296355, 0.0901266, 0.09916983, -0.00811885, -0.02482844],
-"label":[-0.06294985, -0.07393633, -0.07793023, 0.19617933, -0.25141802, -0.16856542, 0.05546749, 0.2207709, -0.12299901, -0.10727913, -0.04701709, -0.05162717, 0.2220566, 0.14470658, -0.07780463, -0.02452197, 0.21929401, 0.10171317, -0.09900579, -0.11624218, -0.21602261, 0.11818817, 0.23674524, -0.0676048, -0.15250057, -0.06562828, 0.14570533, -0.0063743, 0.12755615, 0.18377739, -0.05544094, -0.03191924, -0.12098482, -0.04439923, 0.0412157, -0.13413347, -0.03934726, 0.04211318, -0.02061427, 0.04409805, -0.06535667, -0.10373748, -0.11245634, -0.07654405, 0.1734968, -0.01024228, -0.05300638, 0.08171037, 0.1142247, 0.13233529, -0.04136537, -0.00692873, -0.13803534, 0.0900873, -0.18191165, 0.05589029, -0.05146271, -0.11442295, 0.17379795, 0.11092487, 0.18033165, 0.03263773, -0.09051663, 0.05547883, 0.09321848, -0.13325489, 0.07476224, -0.16582265, -0.14871986, -0.11099493, 0.08359679, -0.2101298, -0.05699859, -0.08379642, 0.10380454],
-"landingpad":[0.12086448, 0.16615215, -0.11424457, -0.08484724, -0.09069939, -0.15767984, 0.10221493, -0.17647022, -0.18533581, -0.02190816, 0.1541339, -0.15779817, 0.03144602, -0.10866319, -0.01497394, -0.1721984, -0.07559271, -0.11889227, -0.1177194, -0.02792813, 0.04734257, -0.03713308, -0.17093344, 0.17252314, 0.02917121, 0.01523044, -0.08545186, -0.20184867, -0.08280238, -0.04850229, 0.03705949, 0.07665683, 0.01846915, 0.00205453, 0.1709319, -0.10548064, -0.07096241, 0.12575251, -0.1177155, 0.17698564, 0.03965896, -0.10821396, 0.07917166, 0.17810985, 0.01715738, 0.16967225, -0.16826203, 0.16891196, -0.15257767, 0.10498544, -0.10490264, 0.06432237, 0.18670914, -0.00099873, -0.13853344, 0.21604276, 0.13351974, -0.15664832, -0.08585881, 0.1024329, 0.02737338, 0.11940238, -0.01797596, -0.00314033, 0.12023867, 0.11616865, 0.05652065, 0.1415307, -0.15827312, -0.03440882, 0.02146849, 0.01478402, -0.13532415, -0.02099262, 0.00462938],
-"load":[0.06383128, 0.0803676, 0.07070765, 0.1284488, 0.053152, -0.11878661, -0.09626942, -0.05779593, -0.04309646, -0.31707463, -0.01623556, 0.0741919, -0.0051238, 0.07403108, -0.25466156, 0.09440999, -0.01665624, 0.14732535, 0.17144501, -0.00878042, 0.04688795, -0.15015227, -0.06462591, -0.04178058, -0.0378243, 0.33024567, -0.07477789, -0.04902143, -0.0476392, 0.13162148, -0.17537315, -0.01750547, 0.14473993, 0.1732234, -0.1423584, 0.11269526, 0.03678195, -0.1315726, 0.15341991, -0.10186778, 0.00435855, 0.16657132, 0.17959148, -0.00966169, -0.16982812, -0.02318787, 0.09485581, -0.07465204, -0.06222337, 0.01743078, 0.12776802, 0.09864764, -0.01487866, 0.03529362, 0.13232034, -0.01328148, 0.09440556, 0.06241368, 0.03618008, -0.11342508, 0.09918994, -0.15651965, -0.07195813, -0.09057574, -0.10321549, 0.06875013, -0.00129774, -0.04545839, -0.07164564, 0.05073486, 0.34142092, -0.08255292, 0.12089149, 0.08211686, 0.21363957],
-"lshr":[0.1238757, -0.02033522, 0.05549098, 0.11646183, 0.08761991, -0.12349915, -0.18941326, -0.08920491, 0.09122052, 0.03471606, 0.2897465, 0.07269037, 0.01238082, 0.07473343, 0.02699653, 0.08085709, -0.04658483, 0.2076412, 0.22355735, -0.03676752, 0.18093167, -0.15845503, -0.0710993, -0.02861892, -0.24865082, -0.00123379, -0.23955399, 0.13371961, -0.11877617, 0.07246234, 0.01312172, 0.01923358, -0.04966315, 0.1200467, -0.2137276, 0.10946998, 0.07409713, -0.12598129, 0.02809012, -0.09355168, 0.06742698, 0.08053191, 0.18937954, 0.00296521, 0.10038757, -0.19544551, 0.08368544, -0.07828917, 0.20687084, 0.01422358, 0.19612609, 0.03323235, -0.01736557, -0.2952406, 0.15087438, -0.07250606, 0.01719518, -0.01060284, 0.16445734, -0.16287296, 0.07179926, -0.14751296, 0.02352273, -0.17620452, 0.12871586, 0.01027796, -0.00240841, -0.08637159, -0.07440444, 0.02873684, 0.05644969, -0.07749362, 0.17137784, 0.09177881, -0.06086624],
-"mul":[9.14543942e-02, 7.25680292e-02, 1.87143609e-01, 1.56122670e-01, 8.18791837e-02, -1.13938227e-01, -1.17102735e-01, 1.78440675e-01, -5.71394851e-03, 3.98688577e-02, -1.22035742e-02, 9.62941721e-02, 9.87940375e-03, 6.54331893e-02, -1.18242744e-02, 1.01057030e-01, -3.37237865e-02, 1.82104349e-01, 1.98254153e-01, 4.89455797e-02, -1.38579145e-01, -1.43827185e-01, -7.30315745e-02, -2.81286202e-02, -3.69295813e-02, -7.32784346e-02, -7.52561465e-02, 1.15158796e-01, -8.05188417e-02, 1.22851163e-01, 1.18959583e-01, -6.19116612e-03, 1.14142060e-01, 1.15443885e-01, -1.32043362e-01, 1.91231743e-01, 6.71850219e-02, -1.18121214e-01, 1.39134541e-01, -9.38180089e-02, 7.00822175e-02, 1.52467072e-01, 1.72531351e-01, 2.39370289e-04, 8.84519666e-02, -1.74602866e-01, 7.58597329e-02, -7.24562407e-02, 2.30801646e-02, 2.31582299e-02, 1.77945897e-01, 2.19754413e-01, -1.05254715e-02, -5.34405783e-02, 1.28701046e-01, -8.88467114e-03, 2.39838846e-02, -2.52107698e-02, 4.09773886e-02, -1.43636853e-01, 9.37281698e-02, -1.36706889e-01, 3.18117648e-01, -1.72676995e-01, 9.86104552e-03, 1.59371197e-02, -2.31163911e-04, -7.84823969e-02, -7.37200677e-02, 3.27852108e-02, 1.85986701e-02, -7.93214887e-02, 1.19079717e-01, 9.70832258e-02, -5.25254011e-02],
-"or":[-0.15830125, 0.08118854, 0.06019896, 0.1351512, 0.0696514, -0.12829824, -0.12040509, -0.07483122, -0.05365232, -0.21453764, 0.01100464, 0.07939477, 0.01512126, 0.07680329, 0.20671816, 0.10159217, -0.03416578, 0.18844754, 0.2232867, 0.00390219, 0.17268041, -0.16189471, -0.07818128, -0.02745846, -0.03785938, -0.28434664, -0.23963155, 0.13414037, -0.03098747, 0.1290441, -0.04272285, 0.00627589, 0.15120292, 0.13505426, -0.14509544, 0.1633035, 0.07651295, -0.1345261, 0.15650204, -0.10334036, 0.08041331, 0.17568573, 0.18555732, -0.00240171, 0.09882613, -0.1697568, 0.08491044, -0.07938255, -0.00502023, 0.04242367, 0.19246665, 0.10571583, -0.01717717, -0.0940207, 0.19319834, -0.06771892, 0.03723063, 0.00987004, 0.08289062, -0.14470372, 0.07547856, -0.14796427, 0.03079064, -0.18788183, -0.06784894, 0.02374648, 0.00680319, -0.08787614, -0.0762485, 0.04015802, -0.31986576, -0.08643831, 0.14352895, 0.1116055, -0.09564826],
-"phi":[-0.17533, -0.14122052, 0.25121832, -0.20197222, -0.02265841, 0.08878156, -0.12254399, -0.05334119, 0.17813319, 0.25582585, -0.24652013, 0.08741698, -0.20831233, -0.12509026, -0.19707216, 0.15587404, -0.07037015, 0.00634299, 0.02692973, 0.20398706, 0.09076547, 0.2106457, 0.02665563, -0.01256549, 0.20547326, -0.01408568, -0.11248078, 0.17035946, 0.00720961, -0.24428545, -0.19424082, 0.02588499, -0.09168262, 0.12671804, -0.13895495, 0.11016023, 0.06885135, -0.1585114, 0.1442796, -0.09209647, -0.25246575, -0.18171434, 0.07803512, 0.01651475, -0.05719611, -0.18895395, 0.13336438, -0.09516547, 0.04140934, -0.13787958, 0.14149408, -0.15089944, -0.02157312, 0.2192603, 0.02856141, -0.05521443, 0.0077216, 0.2731221, -0.09575493, 0.2032519, -0.12885517, -0.18518618, 0.16917716, -0.11144329, 0.25175345, -0.30578047, -0.07883886, 0.14204925, 0.26059705, 0.01635692, -0.2599762, 0.1671248, 0.07727495, 0.06118969, 0.27382395],
-"pointer":[0.01122629, 0.03439064, 0.12280786, -0.03425089, 0.17276576, 0.21717595, 0.09126496, -0.17714174, 0.27793115, 0.08274158, -0.08821795, -0.16903219, -0.06300986, -0.23226759, 0.1235508, -0.11923615, -0.21793933, -0.17721081, 0.07182363, 0.27753174, 0.0920836, 0.06713749, -0.04808864, 0.15279293, 0.05296317, 0.04695908, -0.00205181, -0.14607918, 0.177891, -0.01903534, -0.05020404, 0.20370306, 0.03707821, -0.08771795, 0.10304545, 0.13130909, 0.18650576, 0.05210593, -0.09720453, 0.14389311, -0.05903525, 0.01644695, -0.11466363, 0.18700723, 0.07347484, 0.13623913, 0.13876125, 0.10103971, 0.11578573, -0.21065098, -0.07865446, -0.02068869, 0.16991298, -0.16359806, -0.07672877, 0.11908785, -0.14677979, 0.16077666, -0.21696989, 0.03678338, -0.02173937, 0.08764295, 0.18778177, 0.0974953, 0.05379327, -0.10313212, -0.2190201, 0.25079602, -0.1224342, -0.17659691, -0.10354815, 0.12724441, -0.11835013, -0.15596658, -0.262892],
-"pointerTy":[-5.16731367e-02, 1.41998947e-01, -1.03678465e-01, -2.35795856e-01, 1.67232469e-01, -3.50435115e-02, 1.31282926e-01, 8.77518952e-02, -1.61656141e-01, -1.89439468e-02, 6.89205006e-02, -1.47259831e-01, -1.85588151e-02, 1.95201755e-01, 1.53539265e-02, -1.11738130e-01, 7.46774301e-02, -1.60889581e-01, -1.51122361e-01, -1.64823771e-01, -7.43334070e-02, 1.79255735e-02, -8.14272910e-02, -1.60337687e-01, 7.73334429e-02, 1.45058706e-02, -1.51032144e-02, -1.70947760e-01, -1.48217022e-01, -7.63990656e-02, -8.62834305e-02, -1.66838810e-01, 4.88951942e-03, -1.25664428e-01, 1.13860749e-01, -2.08933145e-01, -1.30075082e-01, 7.92812705e-02, -7.94121996e-02, 1.08121462e-01, -2.62738355e-02, -4.07571979e-02, -9.88764390e-02, 1.00676276e-01, -2.82900538e-02, 7.71642849e-02, -2.50683606e-01, -1.28315210e-01, -1.78229675e-01, -2.19585538e-01, -8.51736143e-02, -1.43008991e-04, 1.66052416e-01, 6.89304620e-02, -1.85903355e-01, 1.39382124e-01, -1.14323251e-01, 6.72841072e-02, -1.73793092e-01, 1.37479026e-02, -6.61165267e-02, 1.28709331e-01, -3.70565802e-02, 1.15480699e-01, -1.72476366e-01, 1.86700657e-01, 1.71808466e-01, 1.25800788e-01, 1.42911309e-02, -1.19691528e-01, 1.48811145e-02, -1.66594520e-01, -1.34164989e-01, 1.62748635e-01, 3.45790684e-02],
-"ptrtoint":[0.13572109, 0.07109733, -0.01410782, 0.13761328, 0.06249858, -0.12742682, -0.03786812, -0.08852433, -0.00890332, -0.12647949, -0.23919484, 0.07803643, -0.29866624, 0.07854372, 0.0690124, 0.08534926, -0.03230691, -0.11528994, 0.2259676, -0.09829737, 0.10256737, -0.16430771, -0.0642257, -0.01210832, -0.12542038, 0.01087978, -0.2651048, 0.10121775, -0.12749597, -0.00404411, 0.08437401, -0.01521534, -0.05364794, -0.20650637, -0.15205501, 0.16177334, 0.08448336, -0.13946855, 0.15794401, -0.08243193, -0.05839634, 0.17580664, 0.19553919, 0.02601502, 0.14116347, -0.17160966, -0.09532923, -0.07157483, 0.20540062, -0.00341362, 0.20416199, 0.09911527, -0.01078249, -0.0624228, 0.14551571, 0.07151687, 0.04763724, -0.02011446, 0.16194499, -0.18160547, 0.13115011, 0.01976459, -0.01537652, -0.19834353, -0.14665273, -0.20727639, 0.03673945, -0.09134132, -0.15628532, -0.04004635, -0.02747078, 0.14147465, 0.17462312, 0.11279432, -0.07561264],
-"resume":[0.16280797, -0.14593223, -0.12683636, -0.1562535, -0.03300981, -0.13968605, 0.16718441, -0.19515003, 0.01863662, -0.05534162, -0.01442544, 0.0425216, 0.0093952, -0.16576275, 0.03578432, -0.15065056, -0.06406866, -0.16523188, -0.05909818, 0.06044476, -0.0871481, 0.15487999, -0.18122765, 0.1956721, -0.04706432, 0.08488349, 0.04219154, 0.02599989, -0.02276746, -0.18513387, 0.12803178, -0.09826294, -0.11191379, 0.04426758, 0.11935913, -0.14695078, -0.00907954, 0.15606298, -0.13066523, 0.01243626, 0.07249606, -0.13709828, 0.10189614, 0.18291259, -0.04105699, 0.1736963, -0.12169949, 0.09442408, 0.20398152, 0.22682573, -0.08286698, -0.13903227, 0.09133334, 0.03095248, 0.11325783, 0.1569758, -0.17479569, -0.12558746, -0.12676957, -0.12449513, 0.17823087, 0.16940698, -0.06061083, -0.1153774, 0.11211671, 0.03377679, -0.05404395, 0.10726969, -0.120761, 0.01132012, 0.00449077, 0.00602859, -0.13664234, 0.00466423, -0.02339442],
-"ret":[0.14368142, 0.0712266, 0.05669672, 0.13478386, -0.14595962, -0.12663847, -0.12403987, -0.09207936, 0.00078773, -0.20975904, 0.08747889, 0.07683876, -0.14989862, -0.18156855, 0.38113984, 0.05973466, -0.05483145, 0.00884139, -0.013859, -0.03841434, 0.05703019, -0.04955313, 0.04577729, -0.05112582, -0.25303054, 0.13709044, -0.06985793, -0.04175208, -0.06085019, 0.12376258, 0.08259401, -0.01146284, 0.05020906, 0.18525098, -0.15137392, 0.07154816, 0.06121736, -0.14056544, 0.16508178, -0.12329083, 0.22687763, 0.17450114, 0.18714464, 0.01847176, -0.02590629, -0.15461633, 0.0910584, -0.07975882, 0.03339317, 0.08106077, 0.10348453, 0.12415495, -0.01464526, -0.20166221, 0.23424618, -0.0058443, 0.13251036, -0.01039419, 0.19171973, -0.14341427, 0.11032798, -0.17445254, -0.21265155, -0.11035369, 0.09180369, -0.01410731, -0.05983369, -0.0549278, 0.11081824, 0.02046819, -0.04516792, -0.09715447, 0.14870624, 0.05914663, 0.10917712],
-"sdiv":[7.54674971e-02, 2.70023555e-01, 2.15832815e-01, 1.51102781e-01, 3.86169888e-02, -1.27587810e-01, -1.64553642e-01, 2.08858803e-01, 2.49577500e-03, 3.39335836e-02, 2.95089424e-01, 7.44504854e-02, 1.75000988e-02, 7.48216212e-02, 4.57024798e-02, 1.14481322e-01, -5.15110120e-02, 1.97094589e-01, 2.28497684e-01, 1.89727977e-01, -2.03567356e-01, -1.53853878e-01, -7.04708546e-02, -2.87290998e-02, -4.15551029e-02, -8.37634057e-02, 2.48096928e-01, 1.39052629e-01, -1.18507713e-01, 1.37168303e-01, 4.37297896e-02, -2.56327417e-04, 1.03771754e-01, 1.18692242e-01, -1.97971940e-01, 2.48502731e-01, 5.65373003e-02, -8.69503245e-02, -9.03292373e-03, -2.25797057e-01, 7.20400363e-02, 4.09315303e-02, 1.83778912e-01, -2.47538835e-03, -2.83727050e-02, -2.00868785e-01, 8.99481177e-02, -7.83472061e-02, 4.22467515e-02, 2.67557669e-02, 1.77338138e-01, -1.42202228e-02, -1.59657858e-02, -6.30925298e-02, 3.13569121e-02, -2.22823601e-02, 6.28621271e-03, -1.85963176e-02, 1.17826000e-01, -1.24646917e-01, 1.02576114e-01, -1.52709767e-01, 2.57164501e-02, -1.40907198e-01, 1.74567476e-01, 1.61048099e-02, -5.03884954e-03, -8.63459408e-02, -7.52566457e-02, 3.38615589e-02, 1.36013612e-01, -8.77554864e-02, 1.03453219e-01, 7.68808499e-02, -6.49084374e-02],
-"select":[0.05093412, 0.07767349, 0.07669216, 0.1312176, 0.0579063, 0.1616888, -0.11760561, 0.00115628, -0.04131674, -0.29780418, -0.01691337, 0.0748211, 0.0864482, 0.07403006, 0.25878662, 0.10279866, -0.01487916, 0.15694329, 0.22207598, 0.01221279, 0.02520609, -0.1521685, -0.06417656, -0.03086487, 0.12268542, 0.3565569, -0.08125142, 0.1072576, -0.02930279, 0.14090922, -0.14309497, -0.00379189, 0.15369308, 0.12072479, -0.11386253, 0.17081785, 0.0359251, -0.13158213, 0.15140608, -0.08365479, 0.04333765, 0.17864218, 0.09735203, -0.01021584, -0.1817566, -0.17009453, 0.10480749, -0.07720621, -0.06324099, 0.01645014, 0.13028447, 0.10660475, -0.02038397, -0.05975712, 0.02713404, -0.01552912, 0.07295278, 0.05679769, 0.02740115, -0.11240587, 0.07162624, -0.15834542, -0.32627442, -0.08725446, -0.07952213, 0.07846557, -0.01865849, -0.04569499, -0.0728619, 0.0374458, -0.27716625, -0.07809193, 0.11488006, 0.06656368, -0.06777868],
-"sext":[-0.03718835, 0.08533189, 0.21354397, 0.13909821, 0.04430787, -0.13043992, -0.12707381, 0.14811471, -0.01397946, -0.24312808, -0.01538438, 0.08007284, -0.20212536, 0.07967917, 0.16868502, 0.10560705, -0.01859606, 0.16425593, 0.23104599, 0.04822871, -0.15584451, -0.16572693, -0.07367796, -0.02386475, -0.04161641, -0.28961626, 0.26639512, 0.13398732, -0.09589424, 0.14068675, 0.16089684, -0.01488031, 0.16961882, 0.11251253, -0.15109386, 0.20157905, 0.04388804, -0.14243662, 0.17028852, -0.17495492, 0.08312923, 0.18138759, 0.19254005, -0.00167047, 0.1016985, -0.17693631, 0.09644604, -0.07967581, -0.01462245, 0.00063264, 0.19925497, 0.10426001, -0.01645718, -0.0897459, 0.02190982, -0.00871549, 0.03981987, 0.01151098, 0.07275884, -0.13166158, 0.11194995, -0.15796044, -0.02378252, -0.17802656, -0.05159309, 0.03235529, 0.00206553, -0.0869719, -0.07645953, 0.05517219, 0.01411109, -0.08648386, 0.1488923, 0.11236069, -0.14168048],
-"shl":[-1.21411718e-01, 8.12682733e-02, 7.26464316e-02, 1.38484403e-01, 8.17928985e-02, -1.24320365e-01, -1.29878476e-01, 1.89510286e-01, -6.82524173e-03, 4.69798781e-02, 7.76179358e-02, 7.87331015e-02, -1.99450791e-01, 7.40546957e-02, 1.56282917e-01, 1.49098694e-01, -3.80089432e-02, 1.97095275e-01, 2.20074877e-01, 9.85120013e-02, -1.51519790e-01, -1.56306505e-01, -6.62787706e-02, -2.89262943e-02, -8.50795880e-02, -3.77611667e-01, -8.08443353e-02, 1.29505768e-01, -9.56492350e-02, 1.34987608e-01, 1.17273629e-01, -8.01561028e-03, 2.01825440e-01, 1.25738055e-01, -1.46153197e-01, 2.18217447e-01, 7.04046711e-02, -1.39466003e-01, 6.33412227e-02, -1.05604395e-01, 7.39852190e-02, 1.70712352e-01, 1.86249197e-01, 1.75760695e-04, 9.54905152e-02, -1.82715610e-01, 8.42701495e-02, -7.70764053e-02, 3.44463770e-04, 2.22381260e-02, 1.92770571e-01, 6.70530349e-02, -8.68750829e-03, -5.87612167e-02, 1.35470673e-01, -9.91571508e-03, 3.54958773e-02, -1.26978569e-02, 3.80189577e-03, -1.57232955e-01, 1.05788536e-01, -1.48979709e-01, -6.13744035e-02, -1.79459959e-01, 1.58743337e-02, 1.66407768e-02, 3.78276110e-02, -8.67255181e-02, -7.59592950e-02, 3.41174081e-02, -1.17456488e-01, -8.46017748e-02, 1.42491326e-01, 8.54357481e-02, -6.05630279e-02],
-"shufflevector":[-1.64276063e-01, -1.40854018e-02, 2.16989160e-01, 1.48834497e-01, -1.81232363e-01, 1.71089604e-01, -1.51962206e-01, 9.34694987e-03, -6.01843521e-02, 1.57123819e-01, 1.49263054e-01, 1.57006048e-02, 8.54564831e-03, -1.46307275e-01, -9.97107551e-02, 2.03836277e-01, -1.10085299e-02, 1.58312604e-01, 3.98286656e-02, 1.22957818e-01, -4.54752985e-03, -1.51905298e-01, 2.56421398e-02, -9.66223106e-02, 1.35936990e-01, 1.13613196e-01, 8.88981670e-02, 1.39288589e-01, 2.35020537e-02, 1.54158428e-01, 1.38836756e-01, -7.45062644e-05, 1.85534820e-01, 2.31408417e-01, -1.86368376e-01, 2.18553007e-01, -1.25940442e-01, 6.19565435e-02, 2.09610596e-01, -2.28297710e-01, 2.30249181e-01, 1.89029738e-01, -6.89853504e-02, -3.19687016e-02, -1.80848792e-01, 4.24141204e-03, 1.89942136e-01, -1.91565320e-01, -2.52943840e-02, 5.65835461e-02, 1.32085189e-01, 1.23422973e-01, -2.14130040e-02, -2.95120943e-02, 1.43252522e-01, -2.44392790e-02, 1.89641267e-01, 3.69141959e-02, 5.39628556e-03, -1.15757883e-01, 6.42104596e-02, -2.11238548e-01, 1.83569789e-02, -9.20821503e-02, -9.48524103e-02, 8.56385827e-02, -5.02256379e-02, -8.02270398e-02, 8.46109912e-02, 7.74170980e-02, 3.27805057e-02, -1.08585857e-01, 1.16238609e-01, 5.87845854e-02, 1.14851862e-01],
-"sitofp":[0.05548016, 0.16714495, 0.21498686, 0.14356636, -0.16821964, 0.16527581, -0.15967155, 0.21070401, -0.01250376, -0.11125261, -0.06026782, 0.07956001, -0.13129172, -0.18479359, 0.08652773, 0.08131526, 0.15925482, 0.08148654, -0.00736642, 0.19463325, -0.20742258, -0.11578189, 0.02654804, -0.08633579, -0.05612461, 0.01452674, 0.11379433, 0.03724489, -0.12955493, 0.15072383, -0.0232376, -0.01797887, 0.20284739, 0.20505717, -0.18686672, 0.25398365, -0.11697914, -0.09353511, 0.19472948, -0.18606324, 0.13722429, 0.19458178, 0.03767281, -0.01490554, -0.15784514, -0.0030121, 0.1061831, -0.18870111, -0.04370302, 0.06840985, 0.12218294, 0.133711, -0.01636076, -0.05479606, -0.09695652, -0.01025795, 0.2093504, 0.08853228, -0.07362932, -0.06125485, 0.10777287, -0.17920133, 0.03073885, -0.09303376, -0.18211918, 0.09700941, -0.05694851, -0.0481168, 0.10195274, 0.11961144, 0.14820483, -0.10259698, 0.07829086, 0.06143153, -0.06575584],
-"srem":[0.12748273, -0.01354557, 0.07656507, 0.13586837, 0.09215222, -0.12957662, -0.22408909, 0.21526249, 0.0899937, 0.03087448, 0.23878594, 0.07816898, 0.0165252, 0.07721527, 0.0565106, 0.08858381, -0.13782543, 0.2185999, 0.2350998, 0.20486632, -0.22521928, -0.16895708, -0.07120457, -0.03142672, -0.193668, -0.06093899, 0.098878, 0.13780232, -0.13592266, 0.13956556, 0.03245056, -0.01644238, 0.1208498, 0.11854354, -0.20298819, 0.27215648, 0.06262159, -0.13207191, -0.22367987, 0.18101601, 0.07210222, -0.035157, 0.19500516, -0.0082911, 0.04821047, -0.2052546, 0.0944626, -0.08101162, 0.05265743, -0.00286297, 0.2012179, -0.02560825, -0.01475919, -0.06332223, -0.21597023, -0.00943558, 0.00891592, -0.04312498, 0.11415429, 0.08646774, 0.10881357, -0.15501237, 0.02134495, -0.15064926, 0.14675559, 0.01146032, 0.0020576, 0.09925999, -0.07733848, 0.03300545, 0.05376984, -0.08198962, 0.17412202, 0.08533774, -0.06574512],
-"store":[0.09998547, 0.07416256, 0.06472399, 0.12126696, 0.03879403, -0.12036075, -0.11504382, -0.06913467, -0.03769127, -0.33070236, -0.00644616, 0.07156026, -0.02137089, 0.07216891, -0.05470073, 0.09415297, -0.04082007, 0.1056991, -0.0013265, -0.00926145, 0.04998971, -0.14578499, -0.05182505, -0.02615017, 0.08348413, 0.31976137, -0.07019657, -0.05163203, -0.04972167, 0.12658256, 0.04289434, -0.0016057, 0.14311627, 0.12263075, -0.13941614, 0.13760482, 0.04816094, -0.12741822, 0.14878556, -0.11523844, 0.05825484, 0.1649721, 0.09197503, -0.0012878, -0.1629092, -0.1414969, 0.09324066, -0.07395344, -0.00317431, 0.01789259, 0.12094813, 0.10764453, -0.01553114, 0.11882594, 0.02442449, -0.01568008, 0.11022279, 0.03307666, 0.08506706, -0.10825977, 0.06488108, -0.15164001, -0.30984077, -0.08653892, -0.06201477, 0.01810287, -0.04473237, -0.04719033, -0.06153039, 0.03287021, 0.29479045, -0.07162003, 0.13225412, 0.05114779, -0.33301073],
-"structTy":[-5.22783101e-02, 1.48193806e-01, -1.73157901e-01, -8.37061405e-02, 1.75303683e-01, 1.41080663e-01, 1.32847771e-01, 9.91027430e-02, -1.22380503e-01, -5.10943905e-02, 7.63119832e-02, -1.28850952e-01, 1.66698098e-01, 1.32489190e-01, -4.90079597e-02, -1.37256354e-01, 4.85256799e-02, -1.78637013e-01, -1.86148569e-01, -1.64286211e-01, -6.26875609e-02, 8.13696086e-02, -1.20131724e-01, -1.59142628e-01, 3.36435549e-02, 2.20832378e-02, -2.72354912e-02, -1.38390273e-01, -1.38179898e-01, -1.23813316e-01, -9.40988287e-02, -1.55397400e-01, -1.58612743e-01, -1.14575408e-01, 1.55034050e-01, -1.50801614e-01, -1.07854791e-01, 1.55085072e-01, -1.68802753e-01, 1.61272466e-01, -2.05100384e-02, -8.52123126e-02, -9.58884135e-02, -1.21472001e-01, 7.77167231e-02, 1.36054471e-01, 1.28906265e-01, 1.30557746e-01, -1.81647420e-01, 1.45003811e-01, -1.11549668e-01, -1.79680258e-01, 1.72399670e-01, 1.51102387e-04, -4.99001816e-02, 1.28895119e-01, -1.17917232e-01, -9.13353860e-02, -1.21147677e-01, 9.76096392e-02, -1.77910581e-01, 1.50892287e-01, -2.25245636e-02, 1.24061532e-01, -1.16433218e-01, -6.76782504e-02, 1.45374849e-01, 2.35193707e-02, -1.35173321e-01, 1.46973729e-01, -1.22685850e-01, 1.27605781e-01, -1.51997164e-01, 1.60292000e-01, 7.63390437e-02],
-"sub":[-0.13264543, 0.07865644, 0.0717167, 0.1342318, 0.05076935, -0.12322044, -0.12266772, 0.00524008, -0.00201612, 0.05315073, 0.00274359, 0.0742974, 0.0123115, 0.07594631, 0.21857306, 0.10421466, -0.02275163, 0.17884067, 0.21873002, 0.00927466, 0.00893607, -0.154156, -0.06820294, -0.02856713, -0.03866831, -0.2793768, -0.06824332, 0.13080892, -0.08803294, 0.13524993, 0.21245252, -0.01117309, 0.20007995, 0.12509634, -0.14541206, 0.19173518, 0.0613077, -0.13841055, 0.15438707, -0.14274515, 0.07382213, 0.17015299, 0.18052574, -0.00206174, -0.15260881, -0.17803495, 0.08986183, -0.07603207, -0.0163452, 0.02869452, 0.186938, 0.07764658, -0.01459899, 0.10072613, 0.02469735, -0.01351501, 0.03415474, 0.0135162, 0.04113324, -0.12325168, 0.11012445, -0.14946231, 0.04155808, -0.1440628, -0.06376708, 0.01838643, 0.00198335, -0.07941123, -0.07066227, 0.0376416, 0.24301553, -0.0862716, 0.1351057, 0.0891021, -0.3352],
-"switch":[-0.16398937, -0.1353046, 0.08524002, 0.1561178, -0.22488637, -0.14185904, -0.10160676, 0.17193311, -0.05567684, -0.01794963, -0.22830375, 0.08948597, 0.1698675, 0.08543108, -0.18870668, 0.13391507, 0.1591491, 0.21046144, 0.02533695, -0.05875827, -0.15551412, 0.19997434, 0.19250119, -0.03397122, -0.05150964, -0.15923595, 0.0701151, 0.14852029, -0.01560177, 0.15765266, -0.2304326, -0.01416684, -0.05115065, 0.12840182, -0.17300242, -0.03878057, 0.07419615, -0.16339742, 0.18437031, -0.14087205, -0.2714784, -0.17547078, -0.01152283, -0.00396952, 0.11226111, -0.18184493, 0.1166721, -0.09108577, 0.23214185, 0.07350235, 0.14699937, 0.08840676, -0.01411979, 0.2082167, -0.10539865, -0.01378542, 0.04486391, -0.01463741, 0.14504644, 0.19353367, 0.11889812, -0.19718692, -0.04798679, -0.12421229, 0.29124823, -0.32081923, -0.00218598, -0.1037329, -0.08444938, 0.04092982, -0.03067664, -0.1606894, 0.07095047, 0.08663001, 0.25936154],
-"trunc":[0.11878826, 0.08377917, 0.05513672, 0.14270523, 0.07370453, -0.14921571, -0.16542384, -0.07493053, -0.00377195, -0.20950364, 0.12958477, 0.07888171, -0.00595982, 0.07865481, 0.16519356, 0.10834014, -0.04326443, 0.19355299, 0.23304398, 0.07385943, -0.11257231, -0.16919497, -0.082918, -0.0285878, -0.05886409, -0.25552744, 0.08462781, 0.13503358, -0.09682789, 0.12824969, 0.08024148, -0.01201469, 0.15677036, 0.14443342, -0.1907149, 0.23856063, -0.00528661, -0.14044978, 0.16222948, -0.10748056, 0.08127003, 0.17631039, 0.19577822, -0.01176626, 0.09836395, -0.16917923, 0.10726047, -0.10182234, 0.20931682, 0.04668214, 0.2017859, 0.11306392, -0.018347, -0.06246869, 0.24337131, -0.07117373, 0.03053473, -0.00658053, 0.03026612, -0.16877551, 0.10491424, -0.17555256, 0.14634104, -0.18189284, -0.11786141, 0.02876267, -0.00210879, -0.08721989, -0.07824761, 0.03958495, 0.20243248, -0.09376816, 0.11082744, 0.12096685, -0.12131985],
-"udiv":[0.08154505, 0.17510523, -0.19362096, 0.08671232, 0.11330891, -0.19157314, -0.2145219, -0.06051004, 0.1371807, 0.04572224, -0.06733968, 0.07146493, 0.00681373, 0.07493957, -0.02997438, 0.08688152, -0.04591569, 0.23223788, 0.22175628, 0.01500544, 0.14273156, -0.16111672, -0.08531193, 0.19467805, -0.00517966, -0.06020893, -0.06077334, 0.13201268, -0.12358853, 0.00868457, 0.06033499, 0.01196855, 0.14756525, 0.11571166, -0.11228922, 0.13696799, 0.21417981, 0.20094916, -0.21627706, 0.18220317, -0.06209835, 0.1076472, 0.20395155, 0.00197578, 0.13577849, -0.20195886, 0.0833538, -0.07554612, 0.00410905, 0.00158071, 0.1938367, 0.02883399, -0.01710817, 0.03711469, 0.13843457, -0.06804645, -0.00379174, -0.00144552, 0.13146468, -0.18377277, 0.07351629, -0.14530504, 0.01042824, -0.1766743, 0.01462858, 0.01206683, -0.12541275, 0.09634796, -0.07832152, 0.02791307, 0.02765011, 0.19757621, 0.16267568, 0.12346987, -0.0413983],
-"uitofp":[-0.15078814, 0.1826288, 0.11165173, 0.13953921, -0.1842304, 0.16399114, -0.11256697, -0.05694728, -0.04820754, -0.17459138, -0.12022546, 0.08202647, 0.05924561, -0.17365019, 0.1614662, 0.07272313, 0.14744736, 0.09281041, -0.00596566, -0.01817355, 0.09689283, -0.0900236, 0.03450679, -0.0848825, 0.17167987, -0.07416291, 0.09088556, 0.07786386, 0.08914284, 0.14832556, -0.07537761, 0.00234627, 0.15249911, 0.19451788, -0.15017514, 0.14780688, 0.11764529, -0.02274906, 0.1920297, -0.17293543, 0.22891366, 0.1852529, 0.0128901, -0.00331702, -0.18161494, -0.0123833, 0.10250985, -0.1877487, -0.02335341, 0.06747307, 0.12578565, 0.12541758, -0.01838141, -0.08433985, 0.26602918, -0.01791255, 0.18907419, 0.04615575, 0.22383852, -0.13418758, 0.07736338, -0.18702815, 0.06279426, -0.09547485, -0.14973089, 0.09295561, -0.04817062, -0.05371717, -0.01254407, 0.12052637, 0.14356695, -0.1381116, 0.17813013, 0.0863996, -0.10018482],
-"unreachable":[1.6576199e-01, -1.7907834e-02, -5.7712890e-02, 9.4339609e-02, -9.3380272e-02, -1.9464681e-01, 7.7050277e-03, -1.9517663e-01, 2.6435368e-03, 9.1913566e-02, 1.3639191e-02, 7.0288680e-02, -3.2517979e-01, 7.4262276e-02, -8.9507200e-02, -1.7191246e-01, -5.2555885e-02, -1.4688279e-01, 1.0220077e-01, -4.8814032e-02, -1.3367063e-01, -6.1610699e-02, -1.8777388e-01, 2.0192388e-01, 1.7102915e-01, 1.3610463e-01, -2.6177135e-01, -3.3939917e-02, -5.4787207e-02, 1.1302987e-01, 1.0523112e-01, 8.6424664e-02, -3.6340270e-02, 4.7679577e-02, 1.0370419e-02, 3.9357133e-03, 1.5754176e-02, -4.6523277e-02, -1.2651871e-01, -5.0250064e-03, -2.1445632e-01, -1.5226135e-01, 1.7948903e-01, 1.9091401e-01, 1.1402745e-01, -1.9978039e-01, -8.4826022e-02, -3.7182060e-03, 2.1397528e-01, -1.9490053e-03, 3.8303059e-02, 5.5778958e-02, -1.1436568e-02, 7.7554718e-02, 3.7896037e-02, 7.5804755e-02, -1.7246161e-02, -1.4188807e-01, 4.9074650e-02, -1.9401214e-01, 1.9030276e-01, 1.1096356e-01, -1.5253071e-01, -1.9243342e-01, 1.6415399e-01, 2.7355237e-02, 7.6516271e-03, -9.3317330e-02, -1.7084104e-01, 3.9767768e-02, -1.1136789e-01, -9.4884530e-02, 2.1568926e-04, 1.3515340e-01, 8.2443848e-02],
-"urem":[0.1196936, 0.18855006, -0.20257552, 0.01738752, 0.15161008, -0.14784957, -0.02992235, -0.0674265, 0.14330001, -0.00208072, -0.08096982, -0.06099883, 0.01042207, 0.1028357, 0.05679855, 0.08465453, -0.05227959, 0.21925992, 0.22556421, -0.059273, 0.1825322, 0.17631407, -0.08938511, -0.0277707, -0.2572769, -0.06195913, -0.24406035, 0.13713413, -0.13801506, 0.02009896, 0.02613169, 0.01552499, 0.14668229, 0.12454724, -0.14865053, 0.09922966, 0.2130675, -0.11827406, 0.01006878, -0.09671234, 0.06660412, 0.0945213, 0.20740281, 0.00239918, 0.14173026, -0.1632753, 0.08587474, -0.07923763, 0.02992029, 0.2015828, 0.2007573, -0.11056472, -0.01871637, -0.06330916, 0.14348209, -0.07483656, 0.00636646, -0.01837422, 0.17499712, -0.17495239, 0.07653105, -0.1475833, 0.02116938, -0.17941074, 0.1734715, -0.01114943, 0.03780977, -0.15019819, -0.07816306, -0.08282368, 0.05129486, 0.19702028, 0.18443331, 0.10329096, -0.06307837],
-"variable":[-0.00726075, -0.19133486, 0.11386976, -0.02404008, 0.18218324, 0.2147853, 0.05924276, -0.14287908, 0.2636781, 0.08127253, -0.08004244, -0.151082, -0.05468801, -0.14080277, 0.09922418, -0.09297107, -0.2022257, -0.0631255, 0.08962703, 0.2616532, -0.05850444, 0.00352755, -0.1646196, 0.12995376, 0.04720275, -0.04833887, 0.0210924, 0.23275656, 0.20620209, -0.01313339, -0.04938933, 0.18796188, 0.03722439, -0.0570973, 0.07931395, 0.12533784, 0.19275506, 0.04549298, -0.06122055, -0.27613187, -0.03735771, 0.0223406, -0.09128818, 0.15596932, -0.02494176, 0.10621183, -0.03121548, 0.09133425, 0.11494032, 0.1616893, -0.05987769, -0.01591593, -0.15986548, -0.13643607, -0.07331984, -0.21545175, -0.13974416, 0.14965075, -0.19167605, 0.03849319, -0.01465273, 0.04467848, 0.16016744, 0.08067682, 0.05409591, -0.1172289, -0.20374107, 0.21127956, 0.14046785, -0.15151982, -0.07432669, 0.1618207, -0.09742998, -0.14227344, 0.07137881],
-"vectorTy":[-0.03294902, 0.14949119, -0.09443742, 0.01450515, 0.19603784, 0.25139496, 0.07845286, 0.11937284, 0.16359204, 0.06099452, 0.09465881, -0.1069911, 0.09603978, 0.16709542, -0.00486909, -0.07722043, 0.22341618, -0.03857102, 0.09912027, -0.12520577, 0.13405597, 0.01585144, 0.08918274, -0.19392243, 0.02501151, 0.00475938, 0.15233898, 0.17524292, -0.19524728, -0.01515534, 0.08875454, -0.14749287, 0.01111115, -0.0826562, 0.09034086, -0.04994484, -0.10947519, 0.11695202, -0.15125127, 0.10647643, 0.03638166, -0.01116771, -0.14471221, -0.14233877, -0.05823282, 0.16917565, -0.06123812, -0.2417991, -0.17791323, -0.11769169, -0.07800104, -0.08785082, 0.11596312, -0.02590095, 0.06086191, -0.18129952, -0.15834333, -0.13579826, -0.14008556, 0.05263453, -0.1810565, 0.05645358, 0.05425795, 0.11036766, -0.15751451, 0.20713197, 0.15679651, -0.17410725, 0.00938685, 0.2128002, 0.05759892, 0.03854589, -0.12531556, 0.19853209, -0.00149427],
-"voidTy":[0.01605093, -0.0537584, -0.20252232, -0.08176173, -0.01439075, -0.0382277, 0.15691687, -0.104895, 0.06642336, -0.06327856, 0.09566005, 0.1357376, -0.06892086, -0.00750719, 0.00474408, -0.20349212, -0.05127234, -0.17626588, -0.17354688, -0.18703477, -0.10801363, 0.09327657, 0.10076355, 0.12450968, -0.14411868, 0.09044765, 0.01168226, -0.15345524, -0.14508787, -0.04046924, 0.04365833, -0.11735097, -0.1433241, -0.1147697, 0.18176553, -0.18558215, -0.13961768, 0.12200661, -0.13001354, 0.16781498, 0.01012573, -0.09670691, -0.10543029, 0.09294511, 0.08037758, 0.14874886, -0.1797814, -0.12390587, -0.0998297, 0.12955485, -0.15814209, -0.03254232, 0.17421001, 0.10237212, -0.07504516, -0.112073, -0.11584745, -0.20020956, 0.11506493, 0.01263643, -0.10729685, 0.12432116, -0.1172537, 0.10430267, 0.06826, -0.06138111, 0.13390729, -0.1675781, 0.00966638, 0.13989863, -0.00030718, 0.10629188, -0.15517116, -0.08125519, 0.03691618],
-"xor":[0.10254283, 0.18058334, 0.05268734, 0.1032732, 0.10089786, -0.1597949, -0.11678528, -0.07392009, 0.00164178, -0.3411041, -0.05760791, 0.14810716, 0.01395016, 0.07759612, 0.0478027, 0.10309431, -0.04646098, 0.22321403, 0.22373965, -0.0018288, 0.20699704, -0.16503152, -0.18555428, -0.02829313, -0.04340088, -0.09629976, -0.07565032, 0.12829013, -0.1184216, 0.05327054, -0.2091556, 0.01997479, 0.14617224, 0.12249397, -0.11211042, 0.17021945, 0.21050954, -0.1381692, 0.0287271, -0.09731391, 0.07712973, 0.0957914, 0.19754209, 0.00280226, -0.22004068, -0.19650316, 0.08103254, -0.07884329, 0.01431314, 0.01774853, 0.20215395, 0.04173042, -0.01757454, -0.06132229, 0.14705223, -0.07607429, 0.01276137, -0.02355135, 0.1687547, -0.16499719, 0.07755739, -0.15042275, 0.04714211, -0.19073205, 0.01791589, 0.00800123, -0.00147503, -0.08897253, -0.07817551, 0.02938494, 0.07296588, -0.08095124, 0.19651005, 0.12599285, -0.06209126],
-"zext":[0.07005657, 0.08003729, 0.07546207, 0.13528231, 0.02506938, -0.12476956, -0.14581475, 0.00614589, -0.00503257, -0.24861531, 0.09526423, 0.07557298, 0.01302374, 0.07628334, 0.04175617, 0.09726538, -0.02312024, 0.16171065, 0.22212435, 0.0079543, 0.07804493, -0.15814242, -0.06840046, -0.03052379, -0.0414926, 0.26214772, -0.07542547, 0.12756911, -0.08823611, 0.13954116, -0.18143456, -0.01209466, 0.15410358, 0.20636739, -0.1831482, 0.19437483, 0.06973958, -0.13698913, 0.19432417, -0.16362838, 0.09032086, 0.18127388, 0.18287036, -0.01740397, -0.17513882, -0.16302226, 0.10199076, -0.07994232, 0.029813, 0.04575716, 0.19189098, 0.10111669, -0.01755602, -0.0832833, 0.1540967, -0.01496344, 0.04155953, 0.00536162, 0.0699449, -0.13122603, 0.10352977, -0.18122241, 0.17052825, -0.15459219, -0.06903189, 0.05887637, -0.00081859, -0.08415917, -0.07304011, 0.08401199, -0.25388402, -0.09779947, 0.12506351, 0.09557163, -0.12431429]
+    "Opcodes":{
+        "add":[0.09571152, 0.08334279, 0.07070262, 0.14084256, 0.04825167, -0.12893851, -0.12180215, -0.07194936, -0.05329844, -0.28276998, 0.00202254, 0.07488817, -0.16139749, 0.07618549, -0.06084673, 0.10536429, -0.03031596, 0.16402271, 0.22730531, -0.0015432, 0.01986631, -0.15808797, 0.03030382, -0.04201249, 0.01689876, -0.30041003, -0.06479812, 0.1282431, -0.09032831, 0.14007935, -0.23540203, 0.00552223, 0.15325953, 0.13147154, -0.15204638, 0.1616615, 0.05758473, -0.1385157, 0.16438901, -0.14482859, 0.07895052, 0.18407233, 0.15283448, -0.00081216, -0.17934133, -0.16779658, 0.09044863, -0.18453072, -0.00552684, 0.01191218, 0.18504514, 0.13140059, -0.0174882, -0.18127315, 0.02269725, -0.02048657, 0.10858727, 0.0074029, 0.09485064, -0.13431476, 0.07491371, -0.17498694, -0.32914585, -0.14159656, -0.03594542, 0.04091231, 0.00298631, -0.08933277, -0.07178984, 0.04144038, 0.12151367, -0.09504163, 0.13691336, 0.07825345, 0.13958281],
+        "alloca":[0.12644756, -0.20912014, 0.07298578, 0.13963142, 0.03932726, -0.12778169, -0.13084207, -0.09090705, -0.01497083, 0.0646746, 0.14847252, 0.08546949, -0.11579383, 0.07812478, -0.16431178, 0.08578802, 0.06094746, -0.1239017, -0.02789196, -0.01074964, -0.1738938, -0.1629063, -0.07137732, -0.02845656, 0.1728318, 0.13779363, -0.06250989, 0.03479109, -0.08423121, 0.14009888, 0.09620709, -0.02287545, -0.04616966, -0.19591278, -0.19636007, 0.21825573, 0.07949521, -0.14614704, 0.17752613, -0.15099092, -0.04518029, 0.17774306, 0.18947133, -0.00197501, -0.12619424, -0.18458585, -0.09960615, 0.01162648, 0.21306573, 0.0468024, 0.201505, 0.09970672, 0.06599383, 0.17757463, -0.11899275, 0.10299123, 0.06038174, -0.07116731, -0.15743989, -0.1258558, 0.1535076, 0.01872367, -0.14336765, -0.19708565, -0.26796287, 0.02628843, -0.0009325, -0.08809827, -0.12970725, 0.17223337, -0.10651242, -0.09162157, 0.03264611, 0.0911452, 0.09506542],
+        "and":[-0.12902537, 0.07848564, 0.07048505, 0.13578993, 0.03570583, -0.1242408, -0.12619697, -0.06914084, -0.0515872, 0.13225996, 0.01673339, 0.0763608, 0.18325369, 0.07690141, -0.09591792, 0.10353354, -0.02401061, 0.16493416, 0.21942355, -0.00400911, 0.03811587, -0.15406959, -0.07134877, -0.02946596, -0.03854588, 0.28656727, -0.06625008, 0.12780443, -0.04631379, 0.13919763, -0.15808977, -0.00312698, 0.14692454, 0.18495218, -0.14863448, 0.13449706, 0.06471325, -0.13883992, 0.17630532, -0.16833898, 0.07391811, 0.17909151, 0.18229873, -0.00381102, -0.17680968, -0.1645554, 0.10016466, -0.07963493, 0.00130218, 0.05646244, 0.18222143, 0.10511146, -0.0191175, -0.08713559, 0.25423968, -0.02557301, 0.04319789, 0.03259414, 0.07209402, -0.13169754, 0.07424775, -0.17216511, -0.32057068, -0.13833733, -0.0658454, 0.02420194, -0.04393166, -0.08238692, -0.07023077, 0.04014502, 0.20101993, -0.09093616, 0.13076238, 0.09114857, -0.06483845],
+        "ashr":[0.16414718, -0.02078147, 0.07353837, 0.14210321, 0.08068018, -0.13586298, -0.15728961, -0.10365791, 0.00359197, 0.04608767, 0.35770142, 0.08003625, 0.00944533, 0.08471741, 0.05809571, 0.09202059, -0.18349345, 0.22169511, 0.24418135, 0.19192688, -0.1956347, -0.17401719, -0.07583863, -0.02781139, 0.0952112, -0.01444751, -0.27259794, 0.14392436, -0.13541143, 0.1410963, 0.04314299, -0.01641751, -0.05448177, -0.22287542, -0.2131822, 0.25112826, 0.06918604, -0.14414005, 0.060288, -0.09658266, -0.09122665, -0.02779044, 0.20349248, 0.0041391, 0.10610974, -0.20890503, -0.09881835, -0.07368057, 0.22467837, 0.00075097, 0.2147348, -0.02612463, -0.01696278, -0.0649786, 0.15041149, 0.02361087, 0.0211603, -0.03706622, 0.18296233, -0.14298625, 0.14614436, 0.02273145, 0.0209446, -0.21062987, 0.16911499, -0.03668665, 0.00197532, -0.09607943, -0.08398084, 0.02006913, 0.05739584, -0.07919859, 0.19634944, 0.11082727, -0.06584227],
+        "bitcast":[0.1675692, -0.12870269, 0.04048132, -0.1670965, -0.1279611, 0.02615386, -0.16829294, -0.09034907, 0.10913523, -0.07819421, 0.23986322, -0.05966561, 0.08410738, 0.19072439, 0.06047394, 0.02999627, -0.16747619, -0.06076627, -0.02673951, -0.1619169, 0.06443421, -0.13788716, -0.05644303, 0.01361013, -0.06858975, -0.06005004, 0.10011288, -0.05508338, -0.10613093, -0.11281271, -0.00758647, -0.12425531, 0.05333719, -0.16881412, -0.20088236, 0.06015657, -0.16405901, 0.06226884, 0.09171099, -0.09500738, 0.07907875, 0.0776544, -0.18457057, -0.11278627, -0.12111131, -0.10180638, 0.18328871, 0.18770072, 0.20346186, 0.10305139, -0.18344335, 0.10693213, -0.10920919, -0.05994263, 0.20354497, -0.03093485, 0.14214055, 0.00580597, 0.10480052, -0.09955201, -0.134185, -0.02904563, 0.00175069, 0.17646657, 0.01348841, -0.02030338, -0.06537742, 0.10032237, 0.15315783, 0.0102667, 0.07717734, -0.01060431, 0.14727928, -0.16261302, -0.06770234],
+        "br":[0.11499645, 0.0824301, 0.07218035, 0.13258332, -0.13419574, -0.12916718, -0.12626709, -0.06741403, -0.02857593, -0.2543893, 0.02193022, 0.0760875, -0.1562702, 0.07712954, 0.3149361, 0.10217083, -0.041038, 0.16601022, 0.01906607, -0.02043359, 0.05471838, -0.15233372, -0.06945753, -0.02313732, -0.07342829, 0.3331809, -0.05246306, -0.0269839, -0.05435036, 0.13908924, 0.32604694, 0.00170966, 0.14997493, 0.13026518, -0.14908995, 0.12238151, -0.06773318, -0.13566032, 0.16068587, -0.12842499, 0.04970508, 0.17827724, 0.09729939, -0.00447832, -0.1739753, -0.16429187, 0.09886666, -0.08058207, 0.00714044, 0.04585538, 0.13424252, 0.11376464, -0.01675582, 0.17901348, -0.00653374, -0.01570439, 0.13032894, 0.01734108, 0.16833901, -0.1173776, 0.07662185, -0.15942436, -0.21173944, -0.10505079, 0.0597497, 0.03491669, 0.00338842, -0.04969047, -0.07644061, 0.04528612, 0.254365, -0.09514527, 0.12015092, 0.08262096, -0.02352029],
+        "call":[0.10815012, -0.12419116, 0.18759736, -0.1905027, 0.01619313, -0.13483052, -0.12278763, -0.07051246, -0.0083437, 0.25107145, -0.16601063, 0.08127163, -0.17432374, -0.18380919, 0.24335551, 0.07208319, -0.04401246, 0.11606008, -0.02733191, 0.02098145, 0.019888, -0.13705409, -0.07569158, -0.03072285, 0.16870692, -0.09787013, -0.09340432, 0.01931342, -0.03557841, 0.14359893, -0.1592094, -0.00055867, 0.159316, 0.1099042, -0.11837319, 0.08741318, 0.03364393, -0.12831019, 0.10450637, -0.12699029, -0.20213994, 0.18390144, 0.11092624, -0.00209971, -0.13063665, 0.19996215, 0.09006448, -0.07840014, 0.22549215, 0.02587176, 0.13374338, 0.11009877, -0.01874998, -0.21446206, 0.02377797, -0.01036531, 0.05427047, 0.01418843, 0.00771817, -0.12639529, -0.10334941, -0.12244401, 0.30014148, -0.09857437, 0.21212636, 0.03429029, -0.04947309, 0.1023307, -0.07743628, 0.03006962, -0.24868701, -0.02357339, 0.11574048, 0.06895301, -0.363474],
+        "extractelement":[-1.62098512e-01, -2.00751424e-02, 1.71776459e-01, 1.38723463e-01, -1.60769299e-01, 9.30800363e-02, -1.24304645e-01, 1.45934001e-01, -5.04420400e-02, -7.54220188e-02, -5.30924797e-02, 8.55294541e-02, 1.17488159e-02, -1.82809234e-01, 9.37484950e-02, 8.38199258e-02, 1.26266479e-01, 9.31843743e-02, -2.26742271e-02, 6.50950940e-03, 5.02749532e-03, -1.14133619e-01, 1.80842891e-01, -9.63811725e-02, 1.67923152e-01, -5.84629439e-02, 1.04362026e-01, 8.92093012e-05, 7.93750137e-02, 1.39107972e-01, -8.40696543e-02, -1.59506593e-02, 1.99361086e-01, 1.93857521e-01, -1.46850392e-01, -1.78695560e-01, 9.88712162e-03, -1.40836969e-01, 1.77736521e-01, -1.61047727e-01, 3.51648539e-01, 1.79638579e-01, 1.49385989e-01, -6.06128015e-03, -1.72102928e-01, -1.81016047e-02, 1.01466164e-01, -1.28312945e-01, -8.05212185e-02, 6.28385469e-02, 1.15654446e-01, 1.91848025e-01, 3.03963851e-02, 3.55794169e-02, 1.40873834e-01, -1.44319711e-02, 1.85423180e-01, 1.16111919e-01, -7.47816712e-02, -1.14719503e-01, 1.02934733e-01, -1.83810964e-01, -2.64400076e-02, -8.99282843e-02, -1.90383971e-01, 7.31386840e-02, -4.36249487e-02, -4.71482053e-02, 1.07486300e-01, 1.09736443e-01, 4.15226035e-02, -1.42309800e-01, 8.96709636e-02, 6.64985999e-02, -6.13647103e-02],
+        "extractvalue":[0.08820406, -0.0182279, 0.01493346, -0.17219813, 0.02927338, -0.17379265, -0.05663937, -0.06805898, -0.21235467, -0.01969833, -0.15152055, -0.18049374, 0.01062911, 0.07935719, -0.01993761, 0.12405304, -0.03198355, 0.13872959, 0.18017697, -0.0100032, 0.22617207, -0.21553156, -0.18403597, 0.10906533, 0.17709404, 0.07438782, 0.0875822, -0.22567064, -0.00636844, 0.06691552, 0.05859367, -0.06149556, 0.15721355, 0.01889951, 0.2164152, -0.07916643, -0.03927743, -0.02665933, 0.13756634, -0.08835335, -0.01159343, -0.19749148, 0.09438482, -0.1696074, -0.18197437, 0.19907822, -0.20855112, 0.21903005, -0.2204043, -0.07439119, 0.1308343, 0.10343243, -0.10401972, 0.02784402, -0.12545246, -0.18498465, 0.02385085, 0.22105947, -0.01296441, 0.19779074, -0.09320201, -0.07345647, -0.06985376, -0.0657003, -0.15406364, 0.14981052, 0.01336148, 0.18544021, -0.07248794, 0.0511543, -0.00308717, 0.01915094, 0.079139, 0.19975829, -0.06093699],
+        "fadd":[0.0851364, -0.01757606, 0.20865884, 0.17407735, -0.17661206, 0.16181652, -0.11895371, -0.06142977, -0.05126655, 0.04448467, -0.03589934, 0.11921146, 0.0143434, -0.18255684, 0.05196553, 0.10967412, 0.15810761, 0.13785522, -0.01255003, -0.00134187, 0.03563518, -0.12446801, 0.18817455, -0.08609993, 0.1893248, 0.13434686, 0.25903776, -0.00467659, 0.09800702, 0.14749499, 0.03458502, -0.00238901, -0.1395337, 0.19306736, -0.11638073, 0.10491668, 0.04428702, 0.18128034, 0.1797104, -0.1929282, 0.07353631, 0.19062985, 0.10042762, -0.0040207, -0.18379262, 0.07060277, 0.09594815, -0.18272707, -0.12928157, 0.07631373, 0.13005428, 0.24108389, -0.01617561, -0.05803563, 0.02686582, -0.02223067, 0.21217403, 0.17594706, 0.00500477, -0.07345455, -0.08176229, -0.19486704, 0.02177307, -0.09052874, -0.14158368, 0.0890988, -0.06339125, -0.04612265, 0.10199347, 0.11777309, 0.07302366, -0.13303186, 0.09683178, 0.05031883, -0.05881981],
+        "fcmp":[1.02981135e-01, -2.43355539e-02, 5.85255250e-02, 1.09401904e-01, -1.87941462e-01, 1.74530044e-01, -1.20533399e-01, -8.64934921e-02, -5.54629341e-02, 1.41877215e-02, 4.80354428e-02, 1.56707644e-01, 2.56910548e-02, -1.94340080e-01, 5.11265621e-02, 8.82050097e-02, 1.67082191e-01, 1.14665776e-01, -2.00159680e-02, -1.82088744e-02, 2.09244549e-01, -1.15076765e-01, 2.03621030e-01, -8.38057026e-02, -4.32270877e-02, 6.39017820e-02, -6.50197491e-02, -2.72039324e-03, 8.68055448e-02, 1.25798717e-01, -4.21847135e-01, 2.64590848e-02, 1.56372517e-01, 2.14883089e-01, -1.12701654e-01, 9.43767577e-02, -1.97456375e-01, -1.72886148e-01, 2.04119727e-01, -1.56766623e-01, 7.38008767e-02, 2.05716744e-01, 3.48394439e-02, -2.48072352e-02, -2.03255862e-01, 5.07165631e-03, 1.12850599e-01, -1.94697857e-01, -6.29984289e-02, 8.21878910e-02, 1.23685144e-01, 1.40430763e-01, -2.20402889e-02, -6.55216798e-02, 2.90598303e-01, -7.66792744e-02, 2.16097474e-01, 6.28568381e-02, 2.69129931e-04, -1.17002167e-01, -1.60454452e-01, -2.07048669e-01, 2.62669493e-02, -9.61467475e-02, -1.88501909e-01, 8.80606323e-02, -5.39420024e-02, -5.08916602e-02, 1.09891914e-01, 1.21375419e-01, 5.10863326e-02, -1.43003076e-01, 8.39024931e-02, 9.51330177e-03, -6.65674359e-02],
+        "fdiv":[-0.14435205, -0.02003789, 0.07479589, 0.13465217, -0.18792528, 0.16504793, -0.12187403, 0.00537306, -0.04864043, 0.04958175, -0.14890797, 0.10156095, 0.01710029, -0.1809697, 0.06765524, 0.18452686, 0.16923915, 0.11264159, -0.01412142, -0.00287484, 0.01297035, 0.06755029, 0.18517323, -0.08975757, 0.16194478, 0.04174679, 0.10903918, 0.00072231, 0.10999232, 0.14989018, 0.05669819, -0.00733533, 0.20606744, 0.22580206, -0.14587933, 0.13480443, 0.05984581, -0.19580479, 0.20258778, -0.17146486, 0.07523733, 0.19139282, 0.03738374, -0.01550991, -0.21507922, 0.07425626, 0.10224851, -0.18437587, -0.06662318, 0.07977687, 0.12355862, 0.1434375, -0.01888226, 0.03870944, 0.07555477, -0.01191964, 0.23488866, 0.06922436, -0.07203218, -0.06884275, 0.07191673, -0.19515742, 0.02168754, -0.08983592, -0.20972523, 0.08903123, -0.05095004, -0.04967143, 0.0985181, 0.15771264, 0.09441628, -0.14769785, -0.16405083, 0.05066825, -0.058015],
+        "fmul":[-0.05690098, -0.02700638, 0.20705073, 0.16630849, -0.17719169, 0.16012336, -0.12955493, 0.19698587, -0.04035136, -0.03197788, 0.08116172, 0.1013689, 0.01416616, -0.17855197, 0.15503113, 0.13990149, 0.15355295, 0.12397105, -0.01912402, 0.01684221, 0.00669227, -0.11869009, 0.17604592, -0.05788622, 0.10467764, -0.0807798, 0.10300152, 0.10780353, 0.08334652, 0.1446782, 0.03546653, -0.00421023, 0.19682531, 0.19603632, -0.1466306, 0.12923796, 0.06296455, -0.14397427, 0.183406, -0.19625223, 0.08244827, 0.1860209, 0.10110238, -0.002166, -0.18034802, 0.06406105, 0.09568714, -0.17962225, -0.05463478, 0.04498994, 0.11928298, 0.1694295, -0.01222703, 0.14453876, 0.13506988, -0.01149808, 0.20273286, 0.05309585, -0.07139173, -0.07370458, 0.09957068, -0.19369006, 0.02577178, -0.09011577, -0.21375039, 0.07624438, -0.04969844, -0.04844297, 0.08409201, 0.11113621, 0.14570779, -0.13936704, 0.07202481, 0.05272412, -0.05747499],
+        "fneg":[-0.04075071, -0.09381824, 0.17104743, 0.0744117, -0.15771168, 0.16444896, -0.11586417, 0.00200867, 0.17323531, -0.10421973, -0.05619403, 0.07496857, 0.05573901, -0.18903743, 0.06586835, 0.03499747, 0.17244707, 0.02476844, -0.02552369, 0.00063759, 0.04312319, -0.04501259, 0.1850581, -0.14339973, -0.04433612, 0.01609048, 0.10461161, -0.12243931, 0.07718915, 0.15313269, 0.12116063, -0.03062805, 0.21040255, 0.21582894, -0.10389283, 0.09344483, -0.20219979, -0.20113027, 0.19945832, -0.17248249, 0.07845841, 0.19814597, 0.06085683, -0.02476168, -0.18729973, 0.0674377, 0.1058675, -0.04973035, -0.10124692, 0.12834774, 0.0900365, -0.17527112, 0.09901146, -0.05764115, 0.03708475, 0.13003437, 0.2325445, 0.19712777, -0.07403741, -0.05580256, 0.18192469, -0.19122703, -0.05592606, 0.09128745, -0.15161441, 0.08734331, -0.0963953, -0.0377482, 0.18287936, 0.16522603, 0.03490613, -0.13519889, 0.08505893, 0.02470117, -0.06158587],
+        "fpext":[0.1312062, -0.0272035, 0.1562131, 0.11956131, -0.18370572, 0.16904335, -0.18538451, -0.0905571, -0.05687085, -0.12067703, 0.15717801, 0.06977441, 0.04604319, -0.18692835, 0.07965986, 0.03840762, 0.17120989, 0.03048515, -0.02593113, -0.02722386, 0.0637762, -0.08012746, 0.03968937, -0.09751038, -0.04150679, -0.05797326, 0.09053179, -0.06100635, 0.01466092, 0.1521789, 0.0178679, 0.00468684, 0.09077137, 0.22244066, -0.21890686, 0.09471557, -0.18849488, 0.06589299, 0.20329474, -0.20006016, 0.06712949, 0.19578859, -0.00594554, -0.03142307, -0.18011327, 0.00516408, 0.18754548, -0.18884791, 0.2184627, 0.09512166, 0.07513344, 0.12641825, -0.0237517, -0.06020509, 0.15404698, -0.02384854, 0.22717056, 0.11279562, 0.16759154, -0.09342691, -0.09703359, -0.19393681, -0.06580188, 0.190172, -0.21666776, 0.09756065, -0.06633368, -0.04663929, 0.16539453, 0.16365078, 0.04338961, -0.1387515, 0.07042016, 0.03850469, -0.06256048],
+        "fptosi":[-0.09471355, -0.01413281, 0.20098655, 0.17478812, -0.0103814, 0.11870264, -0.10561193, 0.19406088, -0.04215302, 0.09856935, -0.05858651, 0.11069191, -0.02667271, -0.17638545, 0.00591833, 0.12538631, 0.1609231, 0.143028, 0.00411847, 0.06113226, -0.1577999, -0.12172183, 0.18763137, -0.04490714, 0.0262733, 0.12005143, 0.08305117, -0.00109107, 0.03972085, 0.13256785, 0.12486281, -0.00920427, 0.19794717, 0.13488762, -0.08073806, 0.11790548, -0.1865304, -0.18761554, 0.15602915, -0.1440169, -0.01538679, 0.18490645, 0.05900477, -0.01250085, -0.17872328, 0.07583775, 0.09906318, -0.16909951, -0.12062778, 0.03089247, 0.11974704, 0.23940024, -0.00561649, 0.03862159, 0.13437317, -0.0128777, 0.2149731, 0.07683202, -0.10922348, -0.04304254, -0.20104879, -0.16371554, -0.08326109, -0.08942039, -0.03989649, 0.07596397, 0.18141732, 0.08706582, -0.06605583, 0.09984156, -0.05919264, -0.13918152, -0.11102735, -0.04074531, -0.05989955],
+        "fptoui":[0.13186027, -0.02009563, 0.05399049, 0.22092278, 0.0764356, -0.12116565, -0.10575569, -0.08263665, -0.05279351, 0.00392524, -0.06554782, 0.15926358, 0.01012308, -0.18726377, 0.02134197, 0.12730621, -0.05603928, 0.1433069, -0.21253656, -0.1455532, 0.23767456, -0.13623604, 0.21878564, -0.05533312, -0.02895407, 0.03911315, -0.2773476, -0.00738798, 0.08483762, -0.14270262, 0.04603437, 0.21681121, -0.20425414, 0.12532364, -0.04144387, -0.1759266, 0.06860236, -0.14378744, 0.16322674, -0.09162073, -0.04966446, 0.1867569, 0.11250684, 0.04158355, -0.20126882, 0.0963328, 0.06281191, -0.17627244, -0.16096021, 0.00596877, 0.12892367, 0.2650723, 0.0032763, -0.05754196, 0.14840715, -0.09938947, 0.13039768, 0.04905574, -0.04759435, -0.18545668, -0.22984591, -0.22227132, -0.02666637, -0.09771203, -0.15587328, 0.07147411, 0.21007161, 0.09932306, -0.06597851, 0.04478595, -0.00686691, -0.14754876, -0.16366987, -0.22182341, -0.05699158],
+        "fptrunc":[0.1180348, -0.01865118, 0.19759397, 0.00099745, -0.20069243, 0.15863162, -0.10404988, -0.03822596, -0.04113388, 0.02389156, -0.15024376, 0.10156121, 0.01125149, -0.17805247, -0.01032893, 0.11541142, 0.16151612, 0.08652765, -0.01421094, -0.02742836, 0.03890061, 0.16992694, -0.19120258, -0.08296357, 0.16136265, 0.13252915, 0.08675185, -0.05066127, 0.14796199, 0.16224207, 0.07839199, -0.0724185, 0.17859218, 0.20112462, -0.11284997, 0.04566038, -0.19799039, -0.17690767, 0.18034197, -0.14002973, -0.0558032, 0.18783137, -0.14857374, -0.03776158, -0.21491641, 0.04907086, 0.10290487, -0.17882703, -0.06566726, 0.09571292, -0.1965507, 0.1673165, -0.03019422, 0.04259395, 0.10474471, -0.01224911, 0.2133804, 0.08484804, -0.01221132, -0.04942748, -0.06522352, -0.16705054, -0.07432696, -0.01365143, -0.14369024, 0.09237107, -0.06833915, -0.04292256, 0.10805372, 0.10829192, -0.04953781, -0.16152975, 0.08096681, 0.12343118, -0.04967185],
+        "freeze":[-0.10947239, 0.16156663, 0.0653074, -0.16102187, 0.05866997, -0.11313032, 0.18013522, 0.00849657, -0.13109621, 0.00250287, 0.12335391, -0.19474623, 0.00160397, 0.18526912, -0.02094408, 0.10131565, -0.17163642, 0.1624736, 0.20699912, 0.1198829, -0.15380894, -0.13946483, -0.0666218, -0.02034315, 0.06200019, 0.04227962, 0.07729523, 0.1244237, -0.07141764, 0.12154905, 0.0300131, 0.01618693, 0.13966976, 0.08918943, -0.07834358, 0.19349128, -0.06925047, -0.11661758, 0.01764905, -0.06631834, -0.19618054, 0.07797705, -0.18398666, -0.08675893, -0.14833811, -0.1468038, 0.18818453, -0.06729261, -0.00497526, -0.08958972, 0.16964634, -0.10325264, -0.10212341, -0.04745837, 0.12241088, -0.03629779, -0.00606445, 0.01766711, 0.0036858, -0.10732682, -0.13172524, -0.10182981, -0.06431175, -0.13082966, 0.04738441, 0.02668242, 0.15197244, -0.12880892, -0.06579075, 0.03165649, -0.00145413, -0.02683991, 0.13963282, -0.14597963, -0.05104417],
+        "fsub":[-0.15878558, -0.01853204, 0.07496438, 0.13546987, -0.1904581, 0.16564278, -0.1120265, -0.05418859, -0.08075086, 0.1190263, -0.030006, 0.07342444, 0.01767328, -0.18650489, 0.04966565, 0.17190282, 0.1616953, 0.10582477, -0.00931773, 0.00157398, 0.00862398, 0.08480153, 0.17171694, -0.0916861, 0.17196383, 0.03703162, 0.11399966, -0.12855959, 0.05810672, 0.15144408, 0.05618297, -0.00800864, 0.20814295, 0.20056027, -0.1153914, 0.10849892, -0.1970011, -0.1813781, 0.1921156, -0.16984123, 0.07333071, 0.19433162, -0.01313439, -0.02830642, -0.17687869, 0.05937865, 0.1070236, -0.18605019, -0.07227098, 0.10597191, 0.1225975, 0.12388534, -0.02131494, -0.02420847, 0.00146665, -0.01353394, 0.22151577, 0.19731705, -0.07575841, 0.08710405, -0.07411117, -0.17442936, -0.1831972, -0.0886184, -0.17084508, 0.10038229, -0.06385624, -0.04644644, 0.10368951, 0.12130242, 0.24595715, -0.14661786, 0.09260331, 0.0455667, -0.05950425],
+        "getelementptr":[0.09870283, 0.07089636, 0.07129486, 0.13141522, 0.0512748, -0.12164738, -0.10822044, -0.06830852, -0.03589667, 0.23281692, 0.00221914, 0.07347875, -0.14414017, -0.1762956, 0.2280886, 0.09720317, -0.02157139, 0.1186724, 0.16809724, 0.01354192, 0.00843806, -0.1289462, -0.06982945, -0.03921828, 0.11689314, -0.27946517, -0.06115843, 0.0435549, -0.04346889, 0.13182928, -0.13756175, -0.00165133, 0.14864644, 0.1256485, -0.10831792, 0.1638358, 0.05359713, -0.12713233, 0.15040766, -0.14046451, 0.06688947, 0.17097251, 0.11004995, 0.00303981, -0.17120391, -0.15561967, -0.0996208, -0.07566627, -0.0092614, 0.03340579, 0.12288038, 0.11154807, -0.01677553, 0.161112, 0.01968472, -0.01360116, 0.05185669, 0.03974737, 0.08288419, -0.11881893, 0.09566113, -0.14562541, 0.27193576, -0.08737413, -0.13493136, 0.06975145, -0.01905861, -0.04629398, -0.06382514, 0.03648283, 0.32102475, -0.07868497, 0.11246872, 0.06408555, 0.21986632],
+        "icmp":[0.10752141, 0.08221146, 0.07407488, 0.13139327, -0.17970721, 0.02754223, -0.13538423, -0.06698897, -0.02524848, 0.29738554, 0.00894943, 0.07728788, 0.01191065, 0.07896648, 0.26208735, 0.10498706, -0.03250087, 0.1684631, 0.23128033, 0.00553169, 0.05936556, -0.15531996, -0.06056314, -0.03261358, 0.09216364, -0.31626984, -0.07434817, 0.12870958, -0.08664963, 0.14354071, 0.2156426, -0.0043262, 0.16025512, 0.13890502, -0.153901, 0.12319393, 0.01295959, -0.14369431, 0.17033054, -0.12931693, 0.07678536, 0.19203088, 0.11493351, -0.00601692, -0.18671934, -0.16845967, 0.11059918, -0.08381938, -0.00690406, 0.04286021, 0.13590424, 0.12681794, -0.01967513, -0.21233313, 0.02998303, -0.01612004, 0.17794448, 0.02153118, 0.09134272, -0.12414944, 0.08073039, -0.17339677, -0.0417021, -0.1026428, -0.0270442, 0.02959586, -0.05658696, -0.05483485, -0.07289126, 0.036765, 0.20820136, -0.0922353, 0.1438483, 0.06928346, -0.37188163],
+        "insertelement":[-0.08810953, 0.19707826, 0.23040843, 0.15133125, -0.22085261, 0.18197243, -0.13767323, 0.00611759, -0.03750934, 0.05303819, -0.00140541, 0.13628025, 0.01222425, -0.19765897, 0.02197562, 0.17957552, 0.17369562, 0.13818526, -0.01018569, 0.07335348, -0.00122391, -0.05268616, 0.03798695, -0.07208619, 0.13593383, -0.08763747, 0.10625125, 0.12024429, -0.01286297, 0.16619073, 0.03609107, 0.00169814, 0.21330379, 0.22023544, -0.11971844, 0.23055589, -0.21922874, -0.18983133, 0.21032487, -0.19545083, 0.39695147, 0.2116961, 0.03465794, -0.01642283, -0.23929499, 0.05254361, 0.1186865, -0.20114459, -0.01001151, 0.06816892, 0.13270052, 0.13516793, -0.02193176, 0.13708963, 0.15222688, -0.01366894, 0.22591114, 0.07127486, -0.08461929, -0.08140475, 0.06961198, -0.18172547, 0.03001983, -0.09530003, -0.03038635, 0.09877217, -0.06264398, -0.06664777, 0.08449566, 0.13536829, 0.04701871, -0.1011354, 0.08900025, 0.0484806, -0.0840841],
+        "insertvalue":[-0.18927452, 0.0174884, -0.14182499, 0.13622768, 0.0860026, 0.09214608, 0.19103362, -0.21386355, -0.08520557, 0.0039752, 0.11396001, -0.20266375, 0.17875974, -0.19441572, 0.06652557, -0.1076987, -0.15872131, -0.20631677, -0.06141828, 0.01331364, 0.02354101, 0.08883859, 0.18617646, -0.1524484, -0.00485444, -0.06658852, -0.06168855, -0.15898797, 0.02933339, -0.19443747, -0.08171376, -0.10284599, -0.14775087, 0.20585667, -0.03085143, -0.20518455, 0.05644068, 0.18632361, -0.14686832, 0.03718346, 0.10764548, -0.01129938, 0.19590198, 0.02194332, -0.04761516, 0.0276675, 0.01006511, -0.05134218, -0.10854474, 0.23528893, 0.09621219, -0.17192629, 0.10312149, -0.14351319, 0.15347119, 0.19321586, -0.20537563, 0.19100618, 0.11280359, 0.07242113, 0.18901348, 0.19133772, 0.02062779, -0.07726028, 0.00578185, -0.02625765, -0.10374335, -0.02086248, 0.18915856, 0.00862997, 0.04699488, -0.0631949, 0.18942626, 0.01894571, 0.02351614],
+        "inttoptr":[0.04721976, 0.16880296, 0.06448112, -0.17660472, 0.02998716, -0.1156636, 0.13852204, -0.06320232, -0.03162635, -0.06644673, -0.136013, -0.06916367, -0.02801188, 0.10646518, 0.05828808, 0.03782522, -0.02227561, 0.02234134, 0.16369314, -0.08557447, 0.03853939, -0.14319475, -0.05599201, -0.01060566, -0.1136315, 0.0201032, 0.01836221, -0.05016, -0.1216938, 0.12626994, 0.03450989, -0.19574732, 0.14352062, -0.17880194, -0.13339539, -0.04630446, 0.03136644, -0.13006681, 0.1445991, -0.09556036, 0.09764113, 0.16694036, 0.09833785, 0.05145761, 0.17970908, -0.14575475, -0.13704927, -0.06876626, -0.00955433, -0.16089469, -0.14152728, 0.1012365, -0.18291287, -0.07371435, -0.13809073, 0.02210419, 0.090786, 0.02507522, 0.15311833, -0.11663677, 0.10619268, 0.1758742, -0.04052401, 0.19525726, 0.15682434, -0.19402866, 0.08792165, 0.19677114, 0.1986662, -0.2065629, 0.04007441, -0.01697999, 0.17015004, 0.07224851, -0.06042317],
+        "invoke":[0.11604629, -0.02348489, -0.11863736, -0.16840768, -0.01534824, -0.13232489, -0.09663513, -0.1576129, -0.2348036, 0.00802987, -0.0517984, -0.05321365, 0.01174416, -0.11655734, 0.05259984, -0.16816719, -0.09482966, -0.00228002, -0.12152751, -0.00498575, 0.03465938, -0.02950593, 0.04545004, -0.2172498, -0.07073424, -0.08918925, 0.0623108, -0.20881031, -0.04182759, -0.20031597, -0.1351732, 0.069332, -0.00361818, 0.11065516, 0.13967985, -0.08796342, -0.00256544, -0.11508698, 0.09178918, 0.2073513, 0.04010937, -0.07716485, 0.06940974, 0.20694868, -0.08160415, 0.1820004, -0.12159622, 0.04531517, -0.00216078, -0.11625812, -0.10868325, 0.11549117, 0.19036727, 0.0365869, -0.09580094, 0.22367325, 0.11756624, -0.2184826, 0.27060437, 0.14232084, 0.01686368, -0.10716032, 0.02151413, -0.02634864, 0.19014603, -0.22087607, -0.05267305, -0.2422757, -0.06234193, -0.0299236, 0.01296355, 0.0901266, 0.09916983, -0.00811885, -0.02482844],
+        "landingpad":[0.12086448, 0.16615215, -0.11424457, -0.08484724, -0.09069939, -0.15767984, 0.10221493, -0.17647022, -0.18533581, -0.02190816, 0.1541339, -0.15779817, 0.03144602, -0.10866319, -0.01497394, -0.1721984, -0.07559271, -0.11889227, -0.1177194, -0.02792813, 0.04734257, -0.03713308, -0.17093344, 0.17252314, 0.02917121, 0.01523044, -0.08545186, -0.20184867, -0.08280238, -0.04850229, 0.03705949, 0.07665683, 0.01846915, 0.00205453, 0.1709319, -0.10548064, -0.07096241, 0.12575251, -0.1177155, 0.17698564, 0.03965896, -0.10821396, 0.07917166, 0.17810985, 0.01715738, 0.16967225, -0.16826203, 0.16891196, -0.15257767, 0.10498544, -0.10490264, 0.06432237, 0.18670914, -0.00099873, -0.13853344, 0.21604276, 0.13351974, -0.15664832, -0.08585881, 0.1024329, 0.02737338, 0.11940238, -0.01797596, -0.00314033, 0.12023867, 0.11616865, 0.05652065, 0.1415307, -0.15827312, -0.03440882, 0.02146849, 0.01478402, -0.13532415, -0.02099262, 0.00462938],
+        "load":[0.06383128, 0.0803676, 0.07070765, 0.1284488, 0.053152, -0.11878661, -0.09626942, -0.05779593, -0.04309646, -0.31707463, -0.01623556, 0.0741919, -0.0051238, 0.07403108, -0.25466156, 0.09440999, -0.01665624, 0.14732535, 0.17144501, -0.00878042, 0.04688795, -0.15015227, -0.06462591, -0.04178058, -0.0378243, 0.33024567, -0.07477789, -0.04902143, -0.0476392, 0.13162148, -0.17537315, -0.01750547, 0.14473993, 0.1732234, -0.1423584, 0.11269526, 0.03678195, -0.1315726, 0.15341991, -0.10186778, 0.00435855, 0.16657132, 0.17959148, -0.00966169, -0.16982812, -0.02318787, 0.09485581, -0.07465204, -0.06222337, 0.01743078, 0.12776802, 0.09864764, -0.01487866, 0.03529362, 0.13232034, -0.01328148, 0.09440556, 0.06241368, 0.03618008, -0.11342508, 0.09918994, -0.15651965, -0.07195813, -0.09057574, -0.10321549, 0.06875013, -0.00129774, -0.04545839, -0.07164564, 0.05073486, 0.34142092, -0.08255292, 0.12089149, 0.08211686, 0.21363957],
+        "lshr":[0.1238757, -0.02033522, 0.05549098, 0.11646183, 0.08761991, -0.12349915, -0.18941326, -0.08920491, 0.09122052, 0.03471606, 0.2897465, 0.07269037, 0.01238082, 0.07473343, 0.02699653, 0.08085709, -0.04658483, 0.2076412, 0.22355735, -0.03676752, 0.18093167, -0.15845503, -0.0710993, -0.02861892, -0.24865082, -0.00123379, -0.23955399, 0.13371961, -0.11877617, 0.07246234, 0.01312172, 0.01923358, -0.04966315, 0.1200467, -0.2137276, 0.10946998, 0.07409713, -0.12598129, 0.02809012, -0.09355168, 0.06742698, 0.08053191, 0.18937954, 0.00296521, 0.10038757, -0.19544551, 0.08368544, -0.07828917, 0.20687084, 0.01422358, 0.19612609, 0.03323235, -0.01736557, -0.2952406, 0.15087438, -0.07250606, 0.01719518, -0.01060284, 0.16445734, -0.16287296, 0.07179926, -0.14751296, 0.02352273, -0.17620452, 0.12871586, 0.01027796, -0.00240841, -0.08637159, -0.07440444, 0.02873684, 0.05644969, -0.07749362, 0.17137784, 0.09177881, -0.06086624],
+        "mul":[9.14543942e-02, 7.25680292e-02, 1.87143609e-01, 1.56122670e-01, 8.18791837e-02, -1.13938227e-01, -1.17102735e-01, 1.78440675e-01, -5.71394851e-03, 3.98688577e-02, -1.22035742e-02, 9.62941721e-02, 9.87940375e-03, 6.54331893e-02, -1.18242744e-02, 1.01057030e-01, -3.37237865e-02, 1.82104349e-01, 1.98254153e-01, 4.89455797e-02, -1.38579145e-01, -1.43827185e-01, -7.30315745e-02, -2.81286202e-02, -3.69295813e-02, -7.32784346e-02, -7.52561465e-02, 1.15158796e-01, -8.05188417e-02, 1.22851163e-01, 1.18959583e-01, -6.19116612e-03, 1.14142060e-01, 1.15443885e-01, -1.32043362e-01, 1.91231743e-01, 6.71850219e-02, -1.18121214e-01, 1.39134541e-01, -9.38180089e-02, 7.00822175e-02, 1.52467072e-01, 1.72531351e-01, 2.39370289e-04, 8.84519666e-02, -1.74602866e-01, 7.58597329e-02, -7.24562407e-02, 2.30801646e-02, 2.31582299e-02, 1.77945897e-01, 2.19754413e-01, -1.05254715e-02, -5.34405783e-02, 1.28701046e-01, -8.88467114e-03, 2.39838846e-02, -2.52107698e-02, 4.09773886e-02, -1.43636853e-01, 9.37281698e-02, -1.36706889e-01, 3.18117648e-01, -1.72676995e-01, 9.86104552e-03, 1.59371197e-02, -2.31163911e-04, -7.84823969e-02, -7.37200677e-02, 3.27852108e-02, 1.85986701e-02, -7.93214887e-02, 1.19079717e-01, 9.70832258e-02, -5.25254011e-02],
+        "or":[-0.15830125, 0.08118854, 0.06019896, 0.1351512, 0.0696514, -0.12829824, -0.12040509, -0.07483122, -0.05365232, -0.21453764, 0.01100464, 0.07939477, 0.01512126, 0.07680329, 0.20671816, 0.10159217, -0.03416578, 0.18844754, 0.2232867, 0.00390219, 0.17268041, -0.16189471, -0.07818128, -0.02745846, -0.03785938, -0.28434664, -0.23963155, 0.13414037, -0.03098747, 0.1290441, -0.04272285, 0.00627589, 0.15120292, 0.13505426, -0.14509544, 0.1633035, 0.07651295, -0.1345261, 0.15650204, -0.10334036, 0.08041331, 0.17568573, 0.18555732, -0.00240171, 0.09882613, -0.1697568, 0.08491044, -0.07938255, -0.00502023, 0.04242367, 0.19246665, 0.10571583, -0.01717717, -0.0940207, 0.19319834, -0.06771892, 0.03723063, 0.00987004, 0.08289062, -0.14470372, 0.07547856, -0.14796427, 0.03079064, -0.18788183, -0.06784894, 0.02374648, 0.00680319, -0.08787614, -0.0762485, 0.04015802, -0.31986576, -0.08643831, 0.14352895, 0.1116055, -0.09564826],
+        "phi":[-0.17533, -0.14122052, 0.25121832, -0.20197222, -0.02265841, 0.08878156, -0.12254399, -0.05334119, 0.17813319, 0.25582585, -0.24652013, 0.08741698, -0.20831233, -0.12509026, -0.19707216, 0.15587404, -0.07037015, 0.00634299, 0.02692973, 0.20398706, 0.09076547, 0.2106457, 0.02665563, -0.01256549, 0.20547326, -0.01408568, -0.11248078, 0.17035946, 0.00720961, -0.24428545, -0.19424082, 0.02588499, -0.09168262, 0.12671804, -0.13895495, 0.11016023, 0.06885135, -0.1585114, 0.1442796, -0.09209647, -0.25246575, -0.18171434, 0.07803512, 0.01651475, -0.05719611, -0.18895395, 0.13336438, -0.09516547, 0.04140934, -0.13787958, 0.14149408, -0.15089944, -0.02157312, 0.2192603, 0.02856141, -0.05521443, 0.0077216, 0.2731221, -0.09575493, 0.2032519, -0.12885517, -0.18518618, 0.16917716, -0.11144329, 0.25175345, -0.30578047, -0.07883886, 0.14204925, 0.26059705, 0.01635692, -0.2599762, 0.1671248, 0.07727495, 0.06118969, 0.27382395],
+        "ptrtoint":[0.13572109, 0.07109733, -0.01410782, 0.13761328, 0.06249858, -0.12742682, -0.03786812, -0.08852433, -0.00890332, -0.12647949, -0.23919484, 0.07803643, -0.29866624, 0.07854372, 0.0690124, 0.08534926, -0.03230691, -0.11528994, 0.2259676, -0.09829737, 0.10256737, -0.16430771, -0.0642257, -0.01210832, -0.12542038, 0.01087978, -0.2651048, 0.10121775, -0.12749597, -0.00404411, 0.08437401, -0.01521534, -0.05364794, -0.20650637, -0.15205501, 0.16177334, 0.08448336, -0.13946855, 0.15794401, -0.08243193, -0.05839634, 0.17580664, 0.19553919, 0.02601502, 0.14116347, -0.17160966, -0.09532923, -0.07157483, 0.20540062, -0.00341362, 0.20416199, 0.09911527, -0.01078249, -0.0624228, 0.14551571, 0.07151687, 0.04763724, -0.02011446, 0.16194499, -0.18160547, 0.13115011, 0.01976459, -0.01537652, -0.19834353, -0.14665273, -0.20727639, 0.03673945, -0.09134132, -0.15628532, -0.04004635, -0.02747078, 0.14147465, 0.17462312, 0.11279432, -0.07561264],
+        "resume":[0.16280797, -0.14593223, -0.12683636, -0.1562535, -0.03300981, -0.13968605, 0.16718441, -0.19515003, 0.01863662, -0.05534162, -0.01442544, 0.0425216, 0.0093952, -0.16576275, 0.03578432, -0.15065056, -0.06406866, -0.16523188, -0.05909818, 0.06044476, -0.0871481, 0.15487999, -0.18122765, 0.1956721, -0.04706432, 0.08488349, 0.04219154, 0.02599989, -0.02276746, -0.18513387, 0.12803178, -0.09826294, -0.11191379, 0.04426758, 0.11935913, -0.14695078, -0.00907954, 0.15606298, -0.13066523, 0.01243626, 0.07249606, -0.13709828, 0.10189614, 0.18291259, -0.04105699, 0.1736963, -0.12169949, 0.09442408, 0.20398152, 0.22682573, -0.08286698, -0.13903227, 0.09133334, 0.03095248, 0.11325783, 0.1569758, -0.17479569, -0.12558746, -0.12676957, -0.12449513, 0.17823087, 0.16940698, -0.06061083, -0.1153774, 0.11211671, 0.03377679, -0.05404395, 0.10726969, -0.120761, 0.01132012, 0.00449077, 0.00602859, -0.13664234, 0.00466423, -0.02339442],
+        "ret":[0.14368142, 0.0712266, 0.05669672, 0.13478386, -0.14595962, -0.12663847, -0.12403987, -0.09207936, 0.00078773, -0.20975904, 0.08747889, 0.07683876, -0.14989862, -0.18156855, 0.38113984, 0.05973466, -0.05483145, 0.00884139, -0.013859, -0.03841434, 0.05703019, -0.04955313, 0.04577729, -0.05112582, -0.25303054, 0.13709044, -0.06985793, -0.04175208, -0.06085019, 0.12376258, 0.08259401, -0.01146284, 0.05020906, 0.18525098, -0.15137392, 0.07154816, 0.06121736, -0.14056544, 0.16508178, -0.12329083, 0.22687763, 0.17450114, 0.18714464, 0.01847176, -0.02590629, -0.15461633, 0.0910584, -0.07975882, 0.03339317, 0.08106077, 0.10348453, 0.12415495, -0.01464526, -0.20166221, 0.23424618, -0.0058443, 0.13251036, -0.01039419, 0.19171973, -0.14341427, 0.11032798, -0.17445254, -0.21265155, -0.11035369, 0.09180369, -0.01410731, -0.05983369, -0.0549278, 0.11081824, 0.02046819, -0.04516792, -0.09715447, 0.14870624, 0.05914663, 0.10917712],
+        "sdiv":[7.54674971e-02, 2.70023555e-01, 2.15832815e-01, 1.51102781e-01, 3.86169888e-02, -1.27587810e-01, -1.64553642e-01, 2.08858803e-01, 2.49577500e-03, 3.39335836e-02, 2.95089424e-01, 7.44504854e-02, 1.75000988e-02, 7.48216212e-02, 4.57024798e-02, 1.14481322e-01, -5.15110120e-02, 1.97094589e-01, 2.28497684e-01, 1.89727977e-01, -2.03567356e-01, -1.53853878e-01, -7.04708546e-02, -2.87290998e-02, -4.15551029e-02, -8.37634057e-02, 2.48096928e-01, 1.39052629e-01, -1.18507713e-01, 1.37168303e-01, 4.37297896e-02, -2.56327417e-04, 1.03771754e-01, 1.18692242e-01, -1.97971940e-01, 2.48502731e-01, 5.65373003e-02, -8.69503245e-02, -9.03292373e-03, -2.25797057e-01, 7.20400363e-02, 4.09315303e-02, 1.83778912e-01, -2.47538835e-03, -2.83727050e-02, -2.00868785e-01, 8.99481177e-02, -7.83472061e-02, 4.22467515e-02, 2.67557669e-02, 1.77338138e-01, -1.42202228e-02, -1.59657858e-02, -6.30925298e-02, 3.13569121e-02, -2.22823601e-02, 6.28621271e-03, -1.85963176e-02, 1.17826000e-01, -1.24646917e-01, 1.02576114e-01, -1.52709767e-01, 2.57164501e-02, -1.40907198e-01, 1.74567476e-01, 1.61048099e-02, -5.03884954e-03, -8.63459408e-02, -7.52566457e-02, 3.38615589e-02, 1.36013612e-01, -8.77554864e-02, 1.03453219e-01, 7.68808499e-02, -6.49084374e-02],
+        "select":[0.05093412, 0.07767349, 0.07669216, 0.1312176, 0.0579063, 0.1616888, -0.11760561, 0.00115628, -0.04131674, -0.29780418, -0.01691337, 0.0748211, 0.0864482, 0.07403006, 0.25878662, 0.10279866, -0.01487916, 0.15694329, 0.22207598, 0.01221279, 0.02520609, -0.1521685, -0.06417656, -0.03086487, 0.12268542, 0.3565569, -0.08125142, 0.1072576, -0.02930279, 0.14090922, -0.14309497, -0.00379189, 0.15369308, 0.12072479, -0.11386253, 0.17081785, 0.0359251, -0.13158213, 0.15140608, -0.08365479, 0.04333765, 0.17864218, 0.09735203, -0.01021584, -0.1817566, -0.17009453, 0.10480749, -0.07720621, -0.06324099, 0.01645014, 0.13028447, 0.10660475, -0.02038397, -0.05975712, 0.02713404, -0.01552912, 0.07295278, 0.05679769, 0.02740115, -0.11240587, 0.07162624, -0.15834542, -0.32627442, -0.08725446, -0.07952213, 0.07846557, -0.01865849, -0.04569499, -0.0728619, 0.0374458, -0.27716625, -0.07809193, 0.11488006, 0.06656368, -0.06777868],
+        "sext":[-0.03718835, 0.08533189, 0.21354397, 0.13909821, 0.04430787, -0.13043992, -0.12707381, 0.14811471, -0.01397946, -0.24312808, -0.01538438, 0.08007284, -0.20212536, 0.07967917, 0.16868502, 0.10560705, -0.01859606, 0.16425593, 0.23104599, 0.04822871, -0.15584451, -0.16572693, -0.07367796, -0.02386475, -0.04161641, -0.28961626, 0.26639512, 0.13398732, -0.09589424, 0.14068675, 0.16089684, -0.01488031, 0.16961882, 0.11251253, -0.15109386, 0.20157905, 0.04388804, -0.14243662, 0.17028852, -0.17495492, 0.08312923, 0.18138759, 0.19254005, -0.00167047, 0.1016985, -0.17693631, 0.09644604, -0.07967581, -0.01462245, 0.00063264, 0.19925497, 0.10426001, -0.01645718, -0.0897459, 0.02190982, -0.00871549, 0.03981987, 0.01151098, 0.07275884, -0.13166158, 0.11194995, -0.15796044, -0.02378252, -0.17802656, -0.05159309, 0.03235529, 0.00206553, -0.0869719, -0.07645953, 0.05517219, 0.01411109, -0.08648386, 0.1488923, 0.11236069, -0.14168048],
+        "shl":[-1.21411718e-01, 8.12682733e-02, 7.26464316e-02, 1.38484403e-01, 8.17928985e-02, -1.24320365e-01, -1.29878476e-01, 1.89510286e-01, -6.82524173e-03, 4.69798781e-02, 7.76179358e-02, 7.87331015e-02, -1.99450791e-01, 7.40546957e-02, 1.56282917e-01, 1.49098694e-01, -3.80089432e-02, 1.97095275e-01, 2.20074877e-01, 9.85120013e-02, -1.51519790e-01, -1.56306505e-01, -6.62787706e-02, -2.89262943e-02, -8.50795880e-02, -3.77611667e-01, -8.08443353e-02, 1.29505768e-01, -9.56492350e-02, 1.34987608e-01, 1.17273629e-01, -8.01561028e-03, 2.01825440e-01, 1.25738055e-01, -1.46153197e-01, 2.18217447e-01, 7.04046711e-02, -1.39466003e-01, 6.33412227e-02, -1.05604395e-01, 7.39852190e-02, 1.70712352e-01, 1.86249197e-01, 1.75760695e-04, 9.54905152e-02, -1.82715610e-01, 8.42701495e-02, -7.70764053e-02, 3.44463770e-04, 2.22381260e-02, 1.92770571e-01, 6.70530349e-02, -8.68750829e-03, -5.87612167e-02, 1.35470673e-01, -9.91571508e-03, 3.54958773e-02, -1.26978569e-02, 3.80189577e-03, -1.57232955e-01, 1.05788536e-01, -1.48979709e-01, -6.13744035e-02, -1.79459959e-01, 1.58743337e-02, 1.66407768e-02, 3.78276110e-02, -8.67255181e-02, -7.59592950e-02, 3.41174081e-02, -1.17456488e-01, -8.46017748e-02, 1.42491326e-01, 8.54357481e-02, -6.05630279e-02],
+        "shufflevector":[-1.64276063e-01, -1.40854018e-02, 2.16989160e-01, 1.48834497e-01, -1.81232363e-01, 1.71089604e-01, -1.51962206e-01, 9.34694987e-03, -6.01843521e-02, 1.57123819e-01, 1.49263054e-01, 1.57006048e-02, 8.54564831e-03, -1.46307275e-01, -9.97107551e-02, 2.03836277e-01, -1.10085299e-02, 1.58312604e-01, 3.98286656e-02, 1.22957818e-01, -4.54752985e-03, -1.51905298e-01, 2.56421398e-02, -9.66223106e-02, 1.35936990e-01, 1.13613196e-01, 8.88981670e-02, 1.39288589e-01, 2.35020537e-02, 1.54158428e-01, 1.38836756e-01, -7.45062644e-05, 1.85534820e-01, 2.31408417e-01, -1.86368376e-01, 2.18553007e-01, -1.25940442e-01, 6.19565435e-02, 2.09610596e-01, -2.28297710e-01, 2.30249181e-01, 1.89029738e-01, -6.89853504e-02, -3.19687016e-02, -1.80848792e-01, 4.24141204e-03, 1.89942136e-01, -1.91565320e-01, -2.52943840e-02, 5.65835461e-02, 1.32085189e-01, 1.23422973e-01, -2.14130040e-02, -2.95120943e-02, 1.43252522e-01, -2.44392790e-02, 1.89641267e-01, 3.69141959e-02, 5.39628556e-03, -1.15757883e-01, 6.42104596e-02, -2.11238548e-01, 1.83569789e-02, -9.20821503e-02, -9.48524103e-02, 8.56385827e-02, -5.02256379e-02, -8.02270398e-02, 8.46109912e-02, 7.74170980e-02, 3.27805057e-02, -1.08585857e-01, 1.16238609e-01, 5.87845854e-02, 1.14851862e-01],
+        "sitofp":[0.05548016, 0.16714495, 0.21498686, 0.14356636, -0.16821964, 0.16527581, -0.15967155, 0.21070401, -0.01250376, -0.11125261, -0.06026782, 0.07956001, -0.13129172, -0.18479359, 0.08652773, 0.08131526, 0.15925482, 0.08148654, -0.00736642, 0.19463325, -0.20742258, -0.11578189, 0.02654804, -0.08633579, -0.05612461, 0.01452674, 0.11379433, 0.03724489, -0.12955493, 0.15072383, -0.0232376, -0.01797887, 0.20284739, 0.20505717, -0.18686672, 0.25398365, -0.11697914, -0.09353511, 0.19472948, -0.18606324, 0.13722429, 0.19458178, 0.03767281, -0.01490554, -0.15784514, -0.0030121, 0.1061831, -0.18870111, -0.04370302, 0.06840985, 0.12218294, 0.133711, -0.01636076, -0.05479606, -0.09695652, -0.01025795, 0.2093504, 0.08853228, -0.07362932, -0.06125485, 0.10777287, -0.17920133, 0.03073885, -0.09303376, -0.18211918, 0.09700941, -0.05694851, -0.0481168, 0.10195274, 0.11961144, 0.14820483, -0.10259698, 0.07829086, 0.06143153, -0.06575584],
+        "srem":[0.12748273, -0.01354557, 0.07656507, 0.13586837, 0.09215222, -0.12957662, -0.22408909, 0.21526249, 0.0899937, 0.03087448, 0.23878594, 0.07816898, 0.0165252, 0.07721527, 0.0565106, 0.08858381, -0.13782543, 0.2185999, 0.2350998, 0.20486632, -0.22521928, -0.16895708, -0.07120457, -0.03142672, -0.193668, -0.06093899, 0.098878, 0.13780232, -0.13592266, 0.13956556, 0.03245056, -0.01644238, 0.1208498, 0.11854354, -0.20298819, 0.27215648, 0.06262159, -0.13207191, -0.22367987, 0.18101601, 0.07210222, -0.035157, 0.19500516, -0.0082911, 0.04821047, -0.2052546, 0.0944626, -0.08101162, 0.05265743, -0.00286297, 0.2012179, -0.02560825, -0.01475919, -0.06332223, -0.21597023, -0.00943558, 0.00891592, -0.04312498, 0.11415429, 0.08646774, 0.10881357, -0.15501237, 0.02134495, -0.15064926, 0.14675559, 0.01146032, 0.0020576, 0.09925999, -0.07733848, 0.03300545, 0.05376984, -0.08198962, 0.17412202, 0.08533774, -0.06574512],
+        "store":[0.09998547, 0.07416256, 0.06472399, 0.12126696, 0.03879403, -0.12036075, -0.11504382, -0.06913467, -0.03769127, -0.33070236, -0.00644616, 0.07156026, -0.02137089, 0.07216891, -0.05470073, 0.09415297, -0.04082007, 0.1056991, -0.0013265, -0.00926145, 0.04998971, -0.14578499, -0.05182505, -0.02615017, 0.08348413, 0.31976137, -0.07019657, -0.05163203, -0.04972167, 0.12658256, 0.04289434, -0.0016057, 0.14311627, 0.12263075, -0.13941614, 0.13760482, 0.04816094, -0.12741822, 0.14878556, -0.11523844, 0.05825484, 0.1649721, 0.09197503, -0.0012878, -0.1629092, -0.1414969, 0.09324066, -0.07395344, -0.00317431, 0.01789259, 0.12094813, 0.10764453, -0.01553114, 0.11882594, 0.02442449, -0.01568008, 0.11022279, 0.03307666, 0.08506706, -0.10825977, 0.06488108, -0.15164001, -0.30984077, -0.08653892, -0.06201477, 0.01810287, -0.04473237, -0.04719033, -0.06153039, 0.03287021, 0.29479045, -0.07162003, 0.13225412, 0.05114779, -0.33301073],
+        "sub":[-0.13264543, 0.07865644, 0.0717167, 0.1342318, 0.05076935, -0.12322044, -0.12266772, 0.00524008, -0.00201612, 0.05315073, 0.00274359, 0.0742974, 0.0123115, 0.07594631, 0.21857306, 0.10421466, -0.02275163, 0.17884067, 0.21873002, 0.00927466, 0.00893607, -0.154156, -0.06820294, -0.02856713, -0.03866831, -0.2793768, -0.06824332, 0.13080892, -0.08803294, 0.13524993, 0.21245252, -0.01117309, 0.20007995, 0.12509634, -0.14541206, 0.19173518, 0.0613077, -0.13841055, 0.15438707, -0.14274515, 0.07382213, 0.17015299, 0.18052574, -0.00206174, -0.15260881, -0.17803495, 0.08986183, -0.07603207, -0.0163452, 0.02869452, 0.186938, 0.07764658, -0.01459899, 0.10072613, 0.02469735, -0.01351501, 0.03415474, 0.0135162, 0.04113324, -0.12325168, 0.11012445, -0.14946231, 0.04155808, -0.1440628, -0.06376708, 0.01838643, 0.00198335, -0.07941123, -0.07066227, 0.0376416, 0.24301553, -0.0862716, 0.1351057, 0.0891021, -0.3352],
+        "switch":[-0.16398937, -0.1353046, 0.08524002, 0.1561178, -0.22488637, -0.14185904, -0.10160676, 0.17193311, -0.05567684, -0.01794963, -0.22830375, 0.08948597, 0.1698675, 0.08543108, -0.18870668, 0.13391507, 0.1591491, 0.21046144, 0.02533695, -0.05875827, -0.15551412, 0.19997434, 0.19250119, -0.03397122, -0.05150964, -0.15923595, 0.0701151, 0.14852029, -0.01560177, 0.15765266, -0.2304326, -0.01416684, -0.05115065, 0.12840182, -0.17300242, -0.03878057, 0.07419615, -0.16339742, 0.18437031, -0.14087205, -0.2714784, -0.17547078, -0.01152283, -0.00396952, 0.11226111, -0.18184493, 0.1166721, -0.09108577, 0.23214185, 0.07350235, 0.14699937, 0.08840676, -0.01411979, 0.2082167, -0.10539865, -0.01378542, 0.04486391, -0.01463741, 0.14504644, 0.19353367, 0.11889812, -0.19718692, -0.04798679, -0.12421229, 0.29124823, -0.32081923, -0.00218598, -0.1037329, -0.08444938, 0.04092982, -0.03067664, -0.1606894, 0.07095047, 0.08663001, 0.25936154],
+        "trunc":[0.11878826, 0.08377917, 0.05513672, 0.14270523, 0.07370453, -0.14921571, -0.16542384, -0.07493053, -0.00377195, -0.20950364, 0.12958477, 0.07888171, -0.00595982, 0.07865481, 0.16519356, 0.10834014, -0.04326443, 0.19355299, 0.23304398, 0.07385943, -0.11257231, -0.16919497, -0.082918, -0.0285878, -0.05886409, -0.25552744, 0.08462781, 0.13503358, -0.09682789, 0.12824969, 0.08024148, -0.01201469, 0.15677036, 0.14443342, -0.1907149, 0.23856063, -0.00528661, -0.14044978, 0.16222948, -0.10748056, 0.08127003, 0.17631039, 0.19577822, -0.01176626, 0.09836395, -0.16917923, 0.10726047, -0.10182234, 0.20931682, 0.04668214, 0.2017859, 0.11306392, -0.018347, -0.06246869, 0.24337131, -0.07117373, 0.03053473, -0.00658053, 0.03026612, -0.16877551, 0.10491424, -0.17555256, 0.14634104, -0.18189284, -0.11786141, 0.02876267, -0.00210879, -0.08721989, -0.07824761, 0.03958495, 0.20243248, -0.09376816, 0.11082744, 0.12096685, -0.12131985],
+        "udiv":[0.08154505, 0.17510523, -0.19362096, 0.08671232, 0.11330891, -0.19157314, -0.2145219, -0.06051004, 0.1371807, 0.04572224, -0.06733968, 0.07146493, 0.00681373, 0.07493957, -0.02997438, 0.08688152, -0.04591569, 0.23223788, 0.22175628, 0.01500544, 0.14273156, -0.16111672, -0.08531193, 0.19467805, -0.00517966, -0.06020893, -0.06077334, 0.13201268, -0.12358853, 0.00868457, 0.06033499, 0.01196855, 0.14756525, 0.11571166, -0.11228922, 0.13696799, 0.21417981, 0.20094916, -0.21627706, 0.18220317, -0.06209835, 0.1076472, 0.20395155, 0.00197578, 0.13577849, -0.20195886, 0.0833538, -0.07554612, 0.00410905, 0.00158071, 0.1938367, 0.02883399, -0.01710817, 0.03711469, 0.13843457, -0.06804645, -0.00379174, -0.00144552, 0.13146468, -0.18377277, 0.07351629, -0.14530504, 0.01042824, -0.1766743, 0.01462858, 0.01206683, -0.12541275, 0.09634796, -0.07832152, 0.02791307, 0.02765011, 0.19757621, 0.16267568, 0.12346987, -0.0413983],
+        "uitofp":[-0.15078814, 0.1826288, 0.11165173, 0.13953921, -0.1842304, 0.16399114, -0.11256697, -0.05694728, -0.04820754, -0.17459138, -0.12022546, 0.08202647, 0.05924561, -0.17365019, 0.1614662, 0.07272313, 0.14744736, 0.09281041, -0.00596566, -0.01817355, 0.09689283, -0.0900236, 0.03450679, -0.0848825, 0.17167987, -0.07416291, 0.09088556, 0.07786386, 0.08914284, 0.14832556, -0.07537761, 0.00234627, 0.15249911, 0.19451788, -0.15017514, 0.14780688, 0.11764529, -0.02274906, 0.1920297, -0.17293543, 0.22891366, 0.1852529, 0.0128901, -0.00331702, -0.18161494, -0.0123833, 0.10250985, -0.1877487, -0.02335341, 0.06747307, 0.12578565, 0.12541758, -0.01838141, -0.08433985, 0.26602918, -0.01791255, 0.18907419, 0.04615575, 0.22383852, -0.13418758, 0.07736338, -0.18702815, 0.06279426, -0.09547485, -0.14973089, 0.09295561, -0.04817062, -0.05371717, -0.01254407, 0.12052637, 0.14356695, -0.1381116, 0.17813013, 0.0863996, -0.10018482],
+        "unreachable":[1.6576199e-01, -1.7907834e-02, -5.7712890e-02, 9.4339609e-02, -9.3380272e-02, -1.9464681e-01, 7.7050277e-03, -1.9517663e-01, 2.6435368e-03, 9.1913566e-02, 1.3639191e-02, 7.0288680e-02, -3.2517979e-01, 7.4262276e-02, -8.9507200e-02, -1.7191246e-01, -5.2555885e-02, -1.4688279e-01, 1.0220077e-01, -4.8814032e-02, -1.3367063e-01, -6.1610699e-02, -1.8777388e-01, 2.0192388e-01, 1.7102915e-01, 1.3610463e-01, -2.6177135e-01, -3.3939917e-02, -5.4787207e-02, 1.1302987e-01, 1.0523112e-01, 8.6424664e-02, -3.6340270e-02, 4.7679577e-02, 1.0370419e-02, 3.9357133e-03, 1.5754176e-02, -4.6523277e-02, -1.2651871e-01, -5.0250064e-03, -2.1445632e-01, -1.5226135e-01, 1.7948903e-01, 1.9091401e-01, 1.1402745e-01, -1.9978039e-01, -8.4826022e-02, -3.7182060e-03, 2.1397528e-01, -1.9490053e-03, 3.8303059e-02, 5.5778958e-02, -1.1436568e-02, 7.7554718e-02, 3.7896037e-02, 7.5804755e-02, -1.7246161e-02, -1.4188807e-01, 4.9074650e-02, -1.9401214e-01, 1.9030276e-01, 1.1096356e-01, -1.5253071e-01, -1.9243342e-01, 1.6415399e-01, 2.7355237e-02, 7.6516271e-03, -9.3317330e-02, -1.7084104e-01, 3.9767768e-02, -1.1136789e-01, -9.4884530e-02, 2.1568926e-04, 1.3515340e-01, 8.2443848e-02],
+        "urem":[0.1196936, 0.18855006, -0.20257552, 0.01738752, 0.15161008, -0.14784957, -0.02992235, -0.0674265, 0.14330001, -0.00208072, -0.08096982, -0.06099883, 0.01042207, 0.1028357, 0.05679855, 0.08465453, -0.05227959, 0.21925992, 0.22556421, -0.059273, 0.1825322, 0.17631407, -0.08938511, -0.0277707, -0.2572769, -0.06195913, -0.24406035, 0.13713413, -0.13801506, 0.02009896, 0.02613169, 0.01552499, 0.14668229, 0.12454724, -0.14865053, 0.09922966, 0.2130675, -0.11827406, 0.01006878, -0.09671234, 0.06660412, 0.0945213, 0.20740281, 0.00239918, 0.14173026, -0.1632753, 0.08587474, -0.07923763, 0.02992029, 0.2015828, 0.2007573, -0.11056472, -0.01871637, -0.06330916, 0.14348209, -0.07483656, 0.00636646, -0.01837422, 0.17499712, -0.17495239, 0.07653105, -0.1475833, 0.02116938, -0.17941074, 0.1734715, -0.01114943, 0.03780977, -0.15019819, -0.07816306, -0.08282368, 0.05129486, 0.19702028, 0.18443331, 0.10329096, -0.06307837],
+        "xor":[0.10254283, 0.18058334, 0.05268734, 0.1032732, 0.10089786, -0.1597949, -0.11678528, -0.07392009, 0.00164178, -0.3411041, -0.05760791, 0.14810716, 0.01395016, 0.07759612, 0.0478027, 0.10309431, -0.04646098, 0.22321403, 0.22373965, -0.0018288, 0.20699704, -0.16503152, -0.18555428, -0.02829313, -0.04340088, -0.09629976, -0.07565032, 0.12829013, -0.1184216, 0.05327054, -0.2091556, 0.01997479, 0.14617224, 0.12249397, -0.11211042, 0.17021945, 0.21050954, -0.1381692, 0.0287271, -0.09731391, 0.07712973, 0.0957914, 0.19754209, 0.00280226, -0.22004068, -0.19650316, 0.08103254, -0.07884329, 0.01431314, 0.01774853, 0.20215395, 0.04173042, -0.01757454, -0.06132229, 0.14705223, -0.07607429, 0.01276137, -0.02355135, 0.1687547, -0.16499719, 0.07755739, -0.15042275, 0.04714211, -0.19073205, 0.01791589, 0.00800123, -0.00147503, -0.08897253, -0.07817551, 0.02938494, 0.07296588, -0.08095124, 0.19651005, 0.12599285, -0.06209126],
+        "zext":[0.07005657, 0.08003729, 0.07546207, 0.13528231, 0.02506938, -0.12476956, -0.14581475, 0.00614589, -0.00503257, -0.24861531, 0.09526423, 0.07557298, 0.01302374, 0.07628334, 0.04175617, 0.09726538, -0.02312024, 0.16171065, 0.22212435, 0.0079543, 0.07804493, -0.15814242, -0.06840046, -0.03052379, -0.0414926, 0.26214772, -0.07542547, 0.12756911, -0.08823611, 0.13954116, -0.18143456, -0.01209466, 0.15410358, 0.20636739, -0.1831482, 0.19437483, 0.06973958, -0.13698913, 0.19432417, -0.16362838, 0.09032086, 0.18127388, 0.18287036, -0.01740397, -0.17513882, -0.16302226, 0.10199076, -0.07994232, 0.029813, 0.04575716, 0.19189098, 0.10111669, -0.01755602, -0.0832833, 0.1540967, -0.01496344, 0.04155953, 0.00536162, 0.0699449, -0.13122603, 0.10352977, -0.18122241, 0.17052825, -0.15459219, -0.06903189, 0.05887637, -0.00081859, -0.08415917, -0.07304011, 0.08401199, -0.25388402, -0.09779947, 0.12506351, 0.09557163, -0.12431429]
+    },
+    "Types": {
+        "floatTy":[-0.04556743, 0.03822431, -0.08842288, -0.12055657, 0.0960574, 0.2371231, 0.09947137, 0.11064581, -0.12371849, -0.06109007, 0.06440615, 0.14924356, 0.13218448, -0.04400141, 0.01761996, -0.13240574, 0.21056518, -0.14215694, -0.14700417, -0.16416645, -0.07137518, 0.05240471, -0.10071051, 0.12556827, 0.08208757, 0.06504779, 0.1578807, -0.09501757, 0.00659264, -0.00527758, -0.01085964, -0.17290565, 0.00501871, -0.10288252, 0.09658113, -0.107755, -0.1718967, 0.16306138, -0.080802, 0.10203532, -0.02934794, -0.02899184, -0.17915869, -0.19883338, -0.05124965, 0.16747124, -0.05587617, 0.14325564, -0.18054038, -0.161391, -0.1149613, 0.01600737, -0.0704658, 0.01935361, -0.04516762, -0.1305524, 0.04487818, -0.07881694, -0.18929696, 0.05364102, -0.17827097, 0.09934752, -0.04212811, 0.15036745, -0.18782069, 0.19947147, 0.1102478, -0.1304183, 0.17145109, 0.22485495, 0.02270225, 0.08804839, -0.12520337, -0.07780997, 0.00057234],
+        "integerTy":[-0.02392217, 0.13147527, -0.12684862, -0.00286194, 0.18320721, -0.03383226, 0.10688385, 0.20315196, 0.16910973, 0.01749469, 0.09163074, 0.18359211, 0.08457439, 0.17821081, -0.01359864, -0.09173124, -0.1060688, -0.0317139, 0.089481, -0.13937937, 0.1258349, -0.00968946, -0.00211832, -0.13140695, -0.11206195, -0.0249014, -0.09646855, 0.16041876, -0.20260467, -0.1008801, -0.02906649, -0.148828, -0.01750175, -0.10806648, 0.10094456, -0.09347772, -0.11941694, 0.0714565, -0.13869469, 0.09965917, -0.02859109, -0.04663063, -0.03677037, -0.17767252, 0.19819601, -0.00728577, -0.07156438, 0.14130634, -0.16161495, -0.14514765, -0.0230404, 0.14609018, 0.12496789, -0.02587292, 0.06375326, -0.1703029, -0.14651431, -0.15148057, -0.11655853, 0.00689279, -0.21302195, 0.10692985, 0.00105841, 0.05685481, -0.05294221, 0.13358746, 0.17953119, 0.01200722, 0.00904744, 0.13893746, -0.01024614, 0.14703822, -0.08035436, 0.18355256, -0.00407319],
+        "pointerTy":[-5.16731367e-02, 1.41998947e-01, -1.03678465e-01, -2.35795856e-01, 1.67232469e-01, -3.50435115e-02, 1.31282926e-01, 8.77518952e-02, -1.61656141e-01, -1.89439468e-02, 6.89205006e-02, -1.47259831e-01, -1.85588151e-02, 1.95201755e-01, 1.53539265e-02, -1.11738130e-01, 7.46774301e-02, -1.60889581e-01, -1.51122361e-01, -1.64823771e-01, -7.43334070e-02, 1.79255735e-02, -8.14272910e-02, -1.60337687e-01, 7.73334429e-02, 1.45058706e-02, -1.51032144e-02, -1.70947760e-01, -1.48217022e-01, -7.63990656e-02, -8.62834305e-02, -1.66838810e-01, 4.88951942e-03, -1.25664428e-01, 1.13860749e-01, -2.08933145e-01, -1.30075082e-01, 7.92812705e-02, -7.94121996e-02, 1.08121462e-01, -2.62738355e-02, -4.07571979e-02, -9.88764390e-02, 1.00676276e-01, -2.82900538e-02, 7.71642849e-02, -2.50683606e-01, -1.28315210e-01, -1.78229675e-01, -2.19585538e-01, -8.51736143e-02, -1.43008991e-04, 1.66052416e-01, 6.89304620e-02, -1.85903355e-01, 1.39382124e-01, -1.14323251e-01, 6.72841072e-02, -1.73793092e-01, 1.37479026e-02, -6.61165267e-02, 1.28709331e-01, -3.70565802e-02, 1.15480699e-01, -1.72476366e-01, 1.86700657e-01, 1.71808466e-01, 1.25800788e-01, 1.42911309e-02, -1.19691528e-01, 1.48811145e-02, -1.66594520e-01, -1.34164989e-01, 1.62748635e-01, 3.45790684e-02],
+        "structTy":[-5.22783101e-02, 1.48193806e-01, -1.73157901e-01, -8.37061405e-02, 1.75303683e-01, 1.41080663e-01, 1.32847771e-01, 9.91027430e-02, -1.22380503e-01, -5.10943905e-02, 7.63119832e-02, -1.28850952e-01, 1.66698098e-01, 1.32489190e-01, -4.90079597e-02, -1.37256354e-01, 4.85256799e-02, -1.78637013e-01, -1.86148569e-01, -1.64286211e-01, -6.26875609e-02, 8.13696086e-02, -1.20131724e-01, -1.59142628e-01, 3.36435549e-02, 2.20832378e-02, -2.72354912e-02, -1.38390273e-01, -1.38179898e-01, -1.23813316e-01, -9.40988287e-02, -1.55397400e-01, -1.58612743e-01, -1.14575408e-01, 1.55034050e-01, -1.50801614e-01, -1.07854791e-01, 1.55085072e-01, -1.68802753e-01, 1.61272466e-01, -2.05100384e-02, -8.52123126e-02, -9.58884135e-02, -1.21472001e-01, 7.77167231e-02, 1.36054471e-01, 1.28906265e-01, 1.30557746e-01, -1.81647420e-01, 1.45003811e-01, -1.11549668e-01, -1.79680258e-01, 1.72399670e-01, 1.51102387e-04, -4.99001816e-02, 1.28895119e-01, -1.17917232e-01, -9.13353860e-02, -1.21147677e-01, 9.76096392e-02, -1.77910581e-01, 1.50892287e-01, -2.25245636e-02, 1.24061532e-01, -1.16433218e-01, -6.76782504e-02, 1.45374849e-01, 2.35193707e-02, -1.35173321e-01, 1.46973729e-01, -1.22685850e-01, 1.27605781e-01, -1.51997164e-01, 1.60292000e-01, 7.63390437e-02],
+        "vectorTy":[-0.03294902, 0.14949119, -0.09443742, 0.01450515, 0.19603784, 0.25139496, 0.07845286, 0.11937284, 0.16359204, 0.06099452, 0.09465881, -0.1069911, 0.09603978, 0.16709542, -0.00486909, -0.07722043, 0.22341618, -0.03857102, 0.09912027, -0.12520577, 0.13405597, 0.01585144, 0.08918274, -0.19392243, 0.02501151, 0.00475938, 0.15233898, 0.17524292, -0.19524728, -0.01515534, 0.08875454, -0.14749287, 0.01111115, -0.0826562, 0.09034086, -0.04994484, -0.10947519, 0.11695202, -0.15125127, 0.10647643, 0.03638166, -0.01116771, -0.14471221, -0.14233877, -0.05823282, 0.16917565, -0.06123812, -0.2417991, -0.17791323, -0.11769169, -0.07800104, -0.08785082, 0.11596312, -0.02590095, 0.06086191, -0.18129952, -0.15834333, -0.13579826, -0.14008556, 0.05263453, -0.1810565, 0.05645358, 0.05425795, 0.11036766, -0.15751451, 0.20713197, 0.15679651, -0.17410725, 0.00938685, 0.2128002, 0.05759892, 0.03854589, -0.12531556, 0.19853209, -0.00149427],
+        "voidTy":[0.01605093, -0.0537584, -0.20252232, -0.08176173, -0.01439075, -0.0382277, 0.15691687, -0.104895, 0.06642336, -0.06327856, 0.09566005, 0.1357376, -0.06892086, -0.00750719, 0.00474408, -0.20349212, -0.05127234, -0.17626588, -0.17354688, -0.18703477, -0.10801363, 0.09327657, 0.10076355, 0.12450968, -0.14411868, 0.09044765, 0.01168226, -0.15345524, -0.14508787, -0.04046924, 0.04365833, -0.11735097, -0.1433241, -0.1147697, 0.18176553, -0.18558215, -0.13961768, 0.12200661, -0.13001354, 0.16781498, 0.01012573, -0.09670691, -0.10543029, 0.09294511, 0.08037758, 0.14874886, -0.1797814, -0.12390587, -0.0998297, 0.12955485, -0.15814209, -0.03254232, 0.17421001, 0.10237212, -0.07504516, -0.112073, -0.11584745, -0.20020956, 0.11506493, 0.01263643, -0.10729685, 0.12432116, -0.1172537, 0.10430267, 0.06826, -0.06138111, 0.13390729, -0.1675781, 0.00966638, 0.13989863, -0.00030718, 0.10629188, -0.15517116, -0.08125519, 0.03691618]
+        
+    },
+    "Arguments": {
+        "constant":[-0.2850312, -0.22839, 0.12669143, -0.0674703, -0.12639391, -0.00477266, 0.04786542, 0.06336267, 0.08660185, 0.12805316, -0.07146342, 0.21539183, -0.0624397, -0.02638953, -0.28688517, 0.28374302, 0.05338082, 0.05559688, -0.13133128, 0.12440272, -0.03583231, 0.29848817, 0.13930812, 0.15453401, 0.0538353, -0.06874479, 0.00262802, 0.27964258, 0.19028014, -0.16371843, -0.05762961, 0.20059372, -0.20804578, -0.06549844, 0.09732475, -0.01551855, 0.21226783, 0.05889762, -0.07560658, 0.11312829, -0.04594622, -0.27309528, -0.05293005, 0.18953343, 0.05463868, -0.31045213, -0.04364616, 0.11005993, 0.12489324, -0.05413342, -0.05814561, -0.26131225, -0.18863814, 0.31165487, -0.08796364, -0.19958755, -0.10849535, 0.14899114, -0.01385941, 0.29359573, -0.01349372, 0.0562498, 0.10977754, 0.08993197, 0.06231657, -0.13509868, -0.20968516, 0.03578237, 0.15356435, -0.17766887, -0.11509016, 0.06898113, -0.06665445, -0.14065051, 0.34711906],
+        "function":[-0.08280793, -0.18925415, -0.1866685, -0.04932962, 0.13478681, 0.203617, 0.05739383, 0.09856346, -0.17600478, 0.07619468, -0.12438924, -0.02529626, -0.07468224, 0.03905433, 0.11325707, -0.04229602, -0.16354711, -0.08952031, -0.15966323, 0.11114867, 0.07319006, 0.02678379, -0.13364255, 0.15032487, 0.0822423, 0.03195171, -0.00871139, -0.1128016, 0.19261838, -0.07204764, -0.05823177, 0.1990249, -0.2057403, -0.08260711, 0.09775644, -0.12436705, 0.15814295, 0.0463977, -0.0493853, 0.1450742, -0.07108644, -0.03192589, -0.11917686, -0.19753116, 0.09248564, 0.13522607, 0.16347136, 0.11065657, -0.0850426, -0.19110996, -0.15060848, -0.25829738, 0.11464069, -0.15613398, -0.07256057, 0.16283846, -0.11428182, -0.1879146, -0.10537492, 0.03637636, -0.02642156, 0.0344548, -0.00722412, 0.13349552, 0.05913915, 0.17941767, -0.20852967, 0.04403022, -0.13914339, -0.17848521, -0.07228794, 0.23070297, -0.11848032, -0.14098541, -0.23762096],
+        "label":[-0.06294985, -0.07393633, -0.07793023, 0.19617933, -0.25141802, -0.16856542, 0.05546749, 0.2207709, -0.12299901, -0.10727913, -0.04701709, -0.05162717, 0.2220566, 0.14470658, -0.07780463, -0.02452197, 0.21929401, 0.10171317, -0.09900579, -0.11624218, -0.21602261, 0.11818817, 0.23674524, -0.0676048, -0.15250057, -0.06562828, 0.14570533, -0.0063743, 0.12755615, 0.18377739, -0.05544094, -0.03191924, -0.12098482, -0.04439923, 0.0412157, -0.13413347, -0.03934726, 0.04211318, -0.02061427, 0.04409805, -0.06535667, -0.10373748, -0.11245634, -0.07654405, 0.1734968, -0.01024228, -0.05300638, 0.08171037, 0.1142247, 0.13233529, -0.04136537, -0.00692873, -0.13803534, 0.0900873, -0.18191165, 0.05589029, -0.05146271, -0.11442295, 0.17379795, 0.11092487, 0.18033165, 0.03263773, -0.09051663, 0.05547883, 0.09321848, -0.13325489, 0.07476224, -0.16582265, -0.14871986, -0.11099493, 0.08359679, -0.2101298, -0.05699859, -0.08379642, 0.10380454],
+        "pointer":[0.01122629, 0.03439064, 0.12280786, -0.03425089, 0.17276576, 0.21717595, 0.09126496, -0.17714174, 0.27793115, 0.08274158, -0.08821795, -0.16903219, -0.06300986, -0.23226759, 0.1235508, -0.11923615, -0.21793933, -0.17721081, 0.07182363, 0.27753174, 0.0920836, 0.06713749, -0.04808864, 0.15279293, 0.05296317, 0.04695908, -0.00205181, -0.14607918, 0.177891, -0.01903534, -0.05020404, 0.20370306, 0.03707821, -0.08771795, 0.10304545, 0.13130909, 0.18650576, 0.05210593, -0.09720453, 0.14389311, -0.05903525, 0.01644695, -0.11466363, 0.18700723, 0.07347484, 0.13623913, 0.13876125, 0.10103971, 0.11578573, -0.21065098, -0.07865446, -0.02068869, 0.16991298, -0.16359806, -0.07672877, 0.11908785, -0.14677979, 0.16077666, -0.21696989, 0.03678338, -0.02173937, 0.08764295, 0.18778177, 0.0974953, 0.05379327, -0.10313212, -0.2190201, 0.25079602, -0.1224342, -0.17659691, -0.10354815, 0.12724441, -0.11835013, -0.15596658, -0.262892],
+        "variable":[-0.00726075, -0.19133486, 0.11386976, -0.02404008, 0.18218324, 0.2147853, 0.05924276, -0.14287908, 0.2636781, 0.08127253, -0.08004244, -0.151082, -0.05468801, -0.14080277, 0.09922418, -0.09297107, -0.2022257, -0.0631255, 0.08962703, 0.2616532, -0.05850444, 0.00352755, -0.1646196, 0.12995376, 0.04720275, -0.04833887, 0.0210924, 0.23275656, 0.20620209, -0.01313339, -0.04938933, 0.18796188, 0.03722439, -0.0570973, 0.07931395, 0.12533784, 0.19275506, 0.04549298, -0.06122055, -0.27613187, -0.03735771, 0.0223406, -0.09128818, 0.15596932, -0.02494176, 0.10621183, -0.03121548, 0.09133425, 0.11494032, 0.1616893, -0.05987769, -0.01591593, -0.15986548, -0.13643607, -0.07331984, -0.21545175, -0.13974416, 0.14965075, -0.19167605, 0.03849319, -0.01465273, 0.04467848, 0.16016744, 0.08067682, 0.05409591, -0.1172289, -0.20374107, 0.21127956, 0.14046785, -0.15151982, -0.07432669, 0.1618207, -0.09742998, -0.14227344, 0.07137881]
+    }
 }
\ No newline at end of file
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index c5e166cef6da6..b7f6950f679ef 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1234,14 +1234,12 @@ bool LLParser::parseAliasOrIFunc(const std::string &Name, unsigned NameID,
   std::unique_ptr<GlobalIFunc> GI;
   GlobalValue *GV;
   if (IsAlias) {
-    GA.reset(GlobalAlias::create(Ty, AddrSpace,
-                                 (GlobalValue::LinkageTypes)Linkage, Name,
-                                 Aliasee, /*Parent*/ nullptr));
+    GA.reset(GlobalAlias::create(Ty, AddrSpace, Linkage, Name, Aliasee,
+                                 /*Parent=*/nullptr));
     GV = GA.get();
   } else {
-    GI.reset(GlobalIFunc::create(Ty, AddrSpace,
-                                 (GlobalValue::LinkageTypes)Linkage, Name,
-                                 Aliasee, /*Parent*/ nullptr));
+    GI.reset(GlobalIFunc::create(Ty, AddrSpace, Linkage, Name, Aliasee,
+                                 /*Parent=*/nullptr));
     GV = GI.get();
   }
   GV->setThreadLocalMode(TLM);
@@ -5876,7 +5874,8 @@ bool LLParser::parseDISubprogram(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(retainedNodes, MDField, );                                          \
   OPTIONAL(thrownTypes, MDField, );                                            \
   OPTIONAL(annotations, MDField, );                                            \
-  OPTIONAL(targetFuncName, MDStringField, );
+  OPTIONAL(targetFuncName, MDStringField, );                                   \
+  OPTIONAL(keyInstructions, MDBoolField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
@@ -5896,7 +5895,7 @@ bool LLParser::parseDISubprogram(MDNode *&Result, bool IsDistinct) {
        type.Val, scopeLine.Val, containingType.Val, virtualIndex.Val,
        thisAdjustment.Val, flags.Val, SPFlags, unit.Val, templateParams.Val,
        declaration.Val, retainedNodes.Val, thrownTypes.Val, annotations.Val,
-       targetFuncName.Val));
+       targetFuncName.Val, keyInstructions.Val));
   return false;
 }
 
@@ -6116,18 +6115,26 @@ bool LLParser::parseDILocalVariable(MDNode *&Result, bool IsDistinct) {
 }
 
 /// parseDILabel:
-///   ::= !DILabel(scope: !0, name: "foo", file: !1, line: 7)
+///   ::= !DILabel(scope: !0, name: "foo", file: !1, line: 7, column: 4)
 bool LLParser::parseDILabel(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(scope, MDField, (/* AllowNull */ false));                           \
   REQUIRED(name, MDStringField, );                                             \
   REQUIRED(file, MDField, );                                                   \
-  REQUIRED(line, LineField, );
+  REQUIRED(line, LineField, );                                                 \
+  OPTIONAL(column, ColumnField, );                                             \
+  OPTIONAL(isArtificial, MDBoolField, );                                       \
+  OPTIONAL(coroSuspendIdx, MDUnsignedField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
+  std::optional<unsigned> CoroSuspendIdx =
+      coroSuspendIdx.Seen ? std::optional<unsigned>(coroSuspendIdx.Val)
+                          : std::nullopt;
+
   Result = GET_OR_DISTINCT(DILabel,
-                           (Context, scope.Val, name.Val, file.Val, line.Val));
+                           (Context, scope.Val, name.Val, file.Val, line.Val,
+                            column.Val, isArtificial.Val, CoroSuspendIdx));
   return false;
 }
 
diff --git a/llvm/lib/BinaryFormat/DXContainer.cpp b/llvm/lib/BinaryFormat/DXContainer.cpp
index 8e7b7d313706a..36d10d0b63078 100644
--- a/llvm/lib/BinaryFormat/DXContainer.cpp
+++ b/llvm/lib/BinaryFormat/DXContainer.cpp
@@ -60,6 +60,35 @@ ArrayRef<EnumEntry<SigComponentType>> dxbc::getSigComponentTypes() {
   return ArrayRef(SigComponentTypes);
 }
 
+static const EnumEntry<RootFlags> RootFlagNames[] = {
+#define ROOT_SIGNATURE_FLAG(Val, Enum) {#Enum, RootFlags::Enum},
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<RootFlags>> dxbc::getRootFlags() {
+  return ArrayRef(RootFlagNames);
+}
+
+static const EnumEntry<RootDescriptorFlags> RootDescriptorFlagNames[] = {
+#define ROOT_DESCRIPTOR_FLAG(Val, Enum, Flag)                                  \
+  {#Enum, RootDescriptorFlags::Enum},
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<RootDescriptorFlags>> dxbc::getRootDescriptorFlags() {
+  return ArrayRef(RootDescriptorFlagNames);
+}
+
+static const EnumEntry<DescriptorRangeFlags> DescriptorRangeFlagNames[] = {
+#define DESCRIPTOR_RANGE_FLAG(Val, Enum, Flag)                                 \
+  {#Enum, DescriptorRangeFlags::Enum},
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<DescriptorRangeFlags>> dxbc::getDescriptorRangeFlags() {
+  return ArrayRef(DescriptorRangeFlagNames);
+}
+
 #define SHADER_VISIBILITY(Val, Enum) {#Enum, ShaderVisibility::Enum},
 
 static const EnumEntry<ShaderVisibility> ShaderVisibilityValues[] = {
@@ -70,6 +99,46 @@ ArrayRef<EnumEntry<ShaderVisibility>> dxbc::getShaderVisibility() {
   return ArrayRef(ShaderVisibilityValues);
 }
 
+#define FILTER(Val, Enum) {#Enum, SamplerFilter::Enum},
+
+static const EnumEntry<SamplerFilter> SamplerFilterNames[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<SamplerFilter>> dxbc::getSamplerFilters() {
+  return ArrayRef(SamplerFilterNames);
+}
+
+#define TEXTURE_ADDRESS_MODE(Val, Enum) {#Enum, TextureAddressMode::Enum},
+
+static const EnumEntry<TextureAddressMode> TextureAddressModeNames[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<TextureAddressMode>> dxbc::getTextureAddressModes() {
+  return ArrayRef(TextureAddressModeNames);
+}
+
+#define COMPARISON_FUNC(Val, Enum) {#Enum, ComparisonFunc::Enum},
+
+static const EnumEntry<ComparisonFunc> ComparisonFuncNames[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<ComparisonFunc>> dxbc::getComparisonFuncs() {
+  return ArrayRef(ComparisonFuncNames);
+}
+
+#define STATIC_BORDER_COLOR(Val, Enum) {#Enum, StaticBorderColor::Enum},
+
+static const EnumEntry<StaticBorderColor> StaticBorderColorValues[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<StaticBorderColor>> dxbc::getStaticBorderColors() {
+  return ArrayRef(StaticBorderColorValues);
+}
+
 #define ROOT_PARAMETER(Val, Enum) {#Enum, RootParameterType::Enum},
 
 static const EnumEntry<RootParameterType> RootParameterTypes[] = {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index c6549e72d48c0..66ecc69c9874d 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2244,6 +2244,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::NoExt;
   case bitc::ATTR_KIND_CAPTURES:
     return Attribute::Captures;
+  case bitc::ATTR_KIND_DEAD_ON_RETURN:
+    return Attribute::DeadOnReturn;
   }
 }
 
@@ -5082,7 +5084,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
 
       unsigned Line = Record[0], Col = Record[1];
       unsigned ScopeID = Record[2], IAID = Record[3];
-      bool isImplicitCode = Record.size() == 5 && Record[4];
+      bool isImplicitCode = Record.size() >= 5 && Record[4];
+      uint64_t AtomGroup = Record.size() == 7 ? Record[5] : 0;
+      uint8_t AtomRank = Record.size() == 7 ? Record[6] : 0;
 
       MDNode *Scope = nullptr, *IA = nullptr;
       if (ScopeID) {
@@ -5097,8 +5101,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
         if (!IA)
           return error("Invalid record");
       }
+
       LastLoc = DILocation::get(Scope->getContext(), Line, Col, Scope, IA,
-                                isImplicitCode);
+                                isImplicitCode, AtomGroup, AtomRank);
       I->setDebugLoc(LastLoc);
       I = nullptr;
       continue;
@@ -7511,11 +7516,9 @@ std::vector<FunctionSummary::ParamAccess>
 ModuleSummaryIndexBitcodeReader::parseParamAccesses(ArrayRef<uint64_t> Record) {
   auto ReadRange = [&]() {
     APInt Lower(FunctionSummary::ParamAccess::RangeWidth,
-                BitcodeReader::decodeSignRotatedValue(Record.front()));
-    Record = Record.drop_front();
+                BitcodeReader::decodeSignRotatedValue(Record.consume_front()));
     APInt Upper(FunctionSummary::ParamAccess::RangeWidth,
-                BitcodeReader::decodeSignRotatedValue(Record.front()));
-    Record = Record.drop_front();
+                BitcodeReader::decodeSignRotatedValue(Record.consume_front()));
     ConstantRange Range{Lower, Upper};
     assert(!Range.isFullSet());
     assert(!Range.isUpperSignWrapped());
@@ -7526,16 +7529,13 @@ ModuleSummaryIndexBitcodeReader::parseParamAccesses(ArrayRef<uint64_t> Record) {
   while (!Record.empty()) {
     PendingParamAccesses.emplace_back();
     FunctionSummary::ParamAccess &ParamAccess = PendingParamAccesses.back();
-    ParamAccess.ParamNo = Record.front();
-    Record = Record.drop_front();
+    ParamAccess.ParamNo = Record.consume_front();
     ParamAccess.Use = ReadRange();
-    ParamAccess.Calls.resize(Record.front());
-    Record = Record.drop_front();
+    ParamAccess.Calls.resize(Record.consume_front());
     for (auto &Call : ParamAccess.Calls) {
-      Call.ParamNo = Record.front();
-      Record = Record.drop_front();
-      Call.Callee = std::get<0>(getValueInfoFromValueId(Record.front()));
-      Record = Record.drop_front();
+      Call.ParamNo = Record.consume_front();
+      Call.Callee =
+          std::get<0>(getValueInfoFromValueId(Record.consume_front()));
       Call.Offsets = ReadRange();
     }
   }
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index a9467d16c9a14..738e47b8b16c4 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -1156,8 +1156,10 @@ void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata(
   assert(ID >= MDStringRef.size() && "Unexpected lazy-loading of MDString");
   // Lookup first if the metadata hasn't already been loaded.
   if (auto *MD = MetadataList.lookup(ID)) {
-    auto *N = cast<MDNode>(MD);
-    if (!N->isTemporary())
+    auto *N = dyn_cast<MDNode>(MD);
+    // If the node is not an MDNode, or if it is not temporary, then
+    // we're done.
+    if (!N || !N->isTemporary())
       return;
   }
   SmallVector<uint64_t, 64> Record;
@@ -1424,7 +1426,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_LOCATION: {
-    if (Record.size() != 5 && Record.size() != 6)
+    // 5: inlinedAt, 6: isImplicit, 8: Key Instructions fields.
+    if (Record.size() != 5 && Record.size() != 6 && Record.size() != 8)
       return error("Invalid record");
 
     IsDistinct = Record[0];
@@ -1432,10 +1435,12 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     unsigned Column = Record[2];
     Metadata *Scope = getMD(Record[3]);
     Metadata *InlinedAt = getMDOrNull(Record[4]);
-    bool ImplicitCode = Record.size() == 6 && Record[5];
+    bool ImplicitCode = Record.size() >= 6 && Record[5];
+    uint64_t AtomGroup = Record.size() == 8 ? Record[6] : 0;
+    uint8_t AtomRank = Record.size() == 8 ? Record[7] : 0;
     MetadataList.assignValue(
         GET_OR_DISTINCT(DILocation, (Context, Line, Column, Scope, InlinedAt,
-                                     ImplicitCode)),
+                                     ImplicitCode, AtomGroup, AtomRank)),
         NextMetadataNo);
     NextMetadataNo++;
     break;
@@ -1888,7 +1893,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_SUBPROGRAM: {
-    if (Record.size() < 18 || Record.size() > 21)
+    if (Record.size() < 18 || Record.size() > 22)
       return error("Invalid record");
 
     bool HasSPFlags = Record[0] & 4;
@@ -1940,6 +1945,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     bool HasTargetFuncName = false;
     unsigned OffsetA = 0;
     unsigned OffsetB = 0;
+    // Key instructions won't be enabled in old-format bitcode, so only
+    // check it if HasSPFlags is true.
+    bool UsesKeyInstructions = false;
     if (!HasSPFlags) {
       OffsetA = 2;
       OffsetB = 2;
@@ -1952,7 +1960,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     } else {
       HasAnnotations = Record.size() >= 19;
       HasTargetFuncName = Record.size() >= 20;
+      UsesKeyInstructions = Record.size() >= 21 ? Record[20] : 0;
     }
+
     Metadata *CUorFn = getMDOrNull(Record[12 + OffsetB]);
     DISubprogram *SP = GET_OR_DISTINCT(
         DISubprogram,
@@ -1978,8 +1988,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
          HasAnnotations ? getMDOrNull(Record[18 + OffsetB])
                         : nullptr, // annotations
          HasTargetFuncName ? getMDString(Record[19 + OffsetB])
-                           : nullptr // targetFuncName
-         ));
+                           : nullptr, // targetFuncName
+         UsesKeyInstructions));
     MetadataList.assignValue(SP, NextMetadataNo);
     NextMetadataNo++;
 
@@ -2240,14 +2250,28 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_LABEL: {
-    if (Record.size() != 5)
+    if (Record.size() < 5 || Record.size() > 7)
       return error("Invalid record");
 
     IsDistinct = Record[0] & 1;
+    uint64_t Line = Record[4];
+    uint64_t Column = Record.size() > 5 ? Record[5] : 0;
+    bool IsArtificial = Record[0] & 2;
+    std::optional<unsigned> CoroSuspendIdx;
+    if (Record.size() > 6) {
+      uint64_t RawSuspendIdx = Record[6];
+      if (RawSuspendIdx != std::numeric_limits<uint64_t>::max()) {
+        if (RawSuspendIdx > (uint64_t)std::numeric_limits<unsigned>::max())
+          return error("CoroSuspendIdx value is too large");
+        CoroSuspendIdx = RawSuspendIdx;
+      }
+    }
+
     MetadataList.assignValue(
-        GET_OR_DISTINCT(DILabel, (Context, getMDOrNull(Record[1]),
-                                  getMDString(Record[2]),
-                                  getMDOrNull(Record[3]), Record[4])),
+        GET_OR_DISTINCT(DILabel,
+                        (Context, getMDOrNull(Record[1]),
+                         getMDString(Record[2]), getMDOrNull(Record[3]), Line,
+                         Column, IsArtificial, CoroSuspendIdx)),
         NextMetadataNo);
     NextMetadataNo++;
     break;
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 2a2dd085a9461..7e0d81ff4b196 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -141,6 +141,7 @@ enum {
 
   // FUNCTION_BLOCK abbrev id's.
   FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
+  FUNCTION_INST_STORE_ABBREV,
   FUNCTION_INST_UNOP_ABBREV,
   FUNCTION_INST_UNOP_FLAGS_ABBREV,
   FUNCTION_INST_BINOP_ABBREV,
@@ -149,9 +150,14 @@ enum {
   FUNCTION_INST_CAST_FLAGS_ABBREV,
   FUNCTION_INST_RET_VOID_ABBREV,
   FUNCTION_INST_RET_VAL_ABBREV,
+  FUNCTION_INST_BR_UNCOND_ABBREV,
+  FUNCTION_INST_BR_COND_ABBREV,
   FUNCTION_INST_UNREACHABLE_ABBREV,
   FUNCTION_INST_GEP_ABBREV,
+  FUNCTION_INST_CMP_ABBREV,
+  FUNCTION_INST_CMP_FLAGS_ABBREV,
   FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
+  FUNCTION_DEBUG_LOC_ABBREV,
 };
 
 /// Abstract class to manage the bitcode writing, subclassed for each bitcode
@@ -938,6 +944,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_NO_EXT;
   case Attribute::Captures:
     return bitc::ATTR_KIND_CAPTURES;
+  case Attribute::DeadOnReturn:
+    return bitc::ATTR_KIND_DEAD_ON_RETURN;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -1799,12 +1807,14 @@ unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
   // location (it's never more expensive than building an array size 1).
   auto Abbv = std::make_shared<BitCodeAbbrev>();
   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // line
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // column
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // scope
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // inlinedAt
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // atomGroup
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
   return Stream.EmitAbbrev(std::move(Abbv));
 }
 
@@ -1820,7 +1830,8 @@ void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
   Record.push_back(VE.getMetadataID(N->getScope()));
   Record.push_back(VE.getMetadataOrNullID(N->getInlinedAt()));
   Record.push_back(N->isImplicitCode());
-
+  Record.push_back(N->getAtomGroup());
+  Record.push_back(N->getAtomRank());
   Stream.EmitRecord(bitc::METADATA_LOCATION, Record, Abbrev);
   Record.clear();
 }
@@ -2148,6 +2159,7 @@ void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
   Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get()));
   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
   Record.push_back(VE.getMetadataOrNullID(N->getRawTargetFuncName()));
+  Record.push_back(N->getKeyInstructionsEnabled());
 
   Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev);
   Record.clear();
@@ -2344,11 +2356,16 @@ void ModuleBitcodeWriter::writeDILocalVariable(
 void ModuleBitcodeWriter::writeDILabel(
     const DILabel *N, SmallVectorImpl<uint64_t> &Record,
     unsigned Abbrev) {
-  Record.push_back((uint64_t)N->isDistinct());
+  uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
+  Record.push_back((uint64_t)N->isDistinct() | IsArtificialFlag);
   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
   Record.push_back(N->getLine());
+  Record.push_back(N->getColumn());
+  Record.push_back(N->getCoroSuspendIdx().has_value()
+                       ? (uint64_t)N->getCoroSuspendIdx().value()
+                       : std::numeric_limits<uint64_t>::max());
 
   Stream.EmitRecord(bitc::METADATA_LABEL, Record, Abbrev);
   Record.clear();
@@ -3195,12 +3212,17 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
   case Instruction::FCmp: {
     // compare returning Int1Ty or vector of Int1Ty
     Code = bitc::FUNC_CODE_INST_CMP2;
-    pushValueAndType(I.getOperand(0), InstID, Vals);
+    AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
+    if (pushValueAndType(I.getOperand(0), InstID, Vals))
+      AbbrevToUse = 0;
     pushValue(I.getOperand(1), InstID, Vals);
     Vals.push_back(cast<CmpInst>(I).getPredicate());
     uint64_t Flags = getOptimizationFlags(&I);
-    if (Flags != 0)
+    if (Flags != 0) {
       Vals.push_back(Flags);
+      if (AbbrevToUse)
+        AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
+    }
     break;
   }
 
@@ -3222,11 +3244,13 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
   case Instruction::Br:
     {
       Code = bitc::FUNC_CODE_INST_BR;
+      AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
       const BranchInst &II = cast<BranchInst>(I);
       Vals.push_back(VE.getValueID(II.getSuccessor(0)));
       if (II.isConditional()) {
         Vals.push_back(VE.getValueID(II.getSuccessor(1)));
         pushValue(II.getCondition(), InstID, Vals);
+        AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
       }
     }
     break;
@@ -3447,12 +3471,16 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
     }
     break;
   case Instruction::Store:
-    if (cast<StoreInst>(I).isAtomic())
+    if (cast<StoreInst>(I).isAtomic()) {
       Code = bitc::FUNC_CODE_INST_STOREATOMIC;
-    else
+    } else {
       Code = bitc::FUNC_CODE_INST_STORE;
-    pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr
-    pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val
+      AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
+    }
+    if (pushValueAndType(I.getOperand(1), InstID, Vals)) // ptrty + ptr
+      AbbrevToUse = 0;
+    if (pushValueAndType(I.getOperand(0), InstID, Vals)) // valty + val
+      AbbrevToUse = 0;
     Vals.push_back(getEncodedAlign(cast<StoreInst>(I).getAlign()));
     Vals.push_back(cast<StoreInst>(I).isVolatile());
     if (cast<StoreInst>(I).isAtomic()) {
@@ -3675,7 +3703,7 @@ void ModuleBitcodeWriter::writeFunction(
   // in the VST.
   FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
 
-  Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4);
+  Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 5);
   VE.incorporateFunction(F);
 
   SmallVector<unsigned, 64> Vals;
@@ -3724,7 +3752,10 @@ void ModuleBitcodeWriter::writeFunction(
           Vals.push_back(VE.getMetadataOrNullID(DL->getScope()));
           Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt()));
           Vals.push_back(DL->isImplicitCode());
-          Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals);
+          Vals.push_back(DL->getAtomGroup());
+          Vals.push_back(DL->getAtomRank());
+          Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals,
+                            FUNCTION_DEBUG_LOC_ABBREV);
           Vals.clear();
           LastDL = DL;
         }
@@ -3851,6 +3882,12 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   // Other blocks can define their abbrevs inline.
   Stream.EnterBlockInfoBlock();
 
+  // Encode type indices using fixed size based on number of types.
+  BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
+                               VE.computeBitsRequiredForTypeIndices());
+  // Encode value indices as 6-bit VBR.
+  BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
+
   { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
@@ -3896,8 +3933,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // SETTYPE abbrev for CONSTANTS_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
-                              VE.computeBitsRequiredForTypeIndices()));
+    Abbv->Add(TypeAbbrevOp);
     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
         CONSTANTS_SETTYPE_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
@@ -3937,19 +3973,29 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_LOAD abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndices()));
+    Abbv->Add(ValAbbrevOp); // Ptr
+    Abbv->Add(TypeAbbrevOp); // dest ty
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_LOAD_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
+    Abbv->Add(ValAbbrevOp);                                // op1
+    Abbv->Add(ValAbbrevOp);                                // op0
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // align
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_STORE_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   { // INST_UNOP abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
+    Abbv->Add(ValAbbrevOp); // LHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_UNOP_ABBREV)
@@ -3958,7 +4004,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
+    Abbv->Add(ValAbbrevOp); // LHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -3968,8 +4014,8 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_BINOP abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
+    Abbv->Add(ValAbbrevOp); // LHS
+    Abbv->Add(ValAbbrevOp); // RHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_BINOP_ABBREV)
@@ -3978,8 +4024,8 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
+    Abbv->Add(ValAbbrevOp); // LHS
+    Abbv->Add(ValAbbrevOp); // RHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -3989,10 +4035,9 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_CAST abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));    // OpVal
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // dest ty
-                              VE.computeBitsRequiredForTypeIndices()));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // opc
+    Abbv->Add(ValAbbrevOp); // OpVal
+    Abbv->Add(TypeAbbrevOp); // dest ty
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_CAST_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
@@ -4000,9 +4045,8 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndices()));
+    Abbv->Add(ValAbbrevOp); // OpVal
+    Abbv->Add(TypeAbbrevOp); // dest ty
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -4020,11 +4064,31 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_RET abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID
+    Abbv->Add(ValAbbrevOp);
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_RET_VAL_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
+    // TODO: Use different abbrev for absolute value reference (succ0)?
+    Abbv->Add(ValAbbrevOp); // succ0
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_BR_UNCOND_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
+    // TODO: Use different abbrev for absolute value references (succ0, succ1)?
+    Abbv->Add(ValAbbrevOp); // succ0
+    Abbv->Add(ValAbbrevOp); // succ1
+    Abbv->Add(ValAbbrevOp); // cond
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_BR_COND_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
@@ -4035,26 +4099,61 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   {
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
-                              Log2_32_Ceil(VE.getTypes().size() + 1)));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
+    Abbv->Add(TypeAbbrevOp); // dest ty
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(ValAbbrevOp);
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_GEP_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
+    Abbv->Add(ValAbbrevOp);                                // op0
+    Abbv->Add(ValAbbrevOp);                                // op1
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_CMP_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
+    Abbv->Add(ValAbbrevOp);                                // op0
+    Abbv->Add(ValAbbrevOp);                                // op1
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_CMP_FLAGS_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   {
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // val
+    Abbv->Add(ValAbbrevOp); // val
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering! 1");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
+    // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_DEBUG_LOC_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   Stream.ExitBlock();
 }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 754dba73673c2..2c949614d9ff4 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1483,16 +1483,13 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
     }
 
     if (!Features.OmitBBEntries) {
-      // TODO: Remove this check when version 1 is deprecated.
-      if (BBAddrMapVersion > 1) {
-        OutStreamer->AddComment("BB id");
-        // Emit the BB ID for this basic block.
-        // We only emit BaseID since CloneID is unset for
-        // -basic-block-adress-map.
-        // TODO: Emit the full BBID when labels and sections can be mixed
-        // together.
-        OutStreamer->emitULEB128IntValue(MBB.getBBID()->BaseID);
-      }
+      OutStreamer->AddComment("BB id");
+      // Emit the BB ID for this basic block.
+      // We only emit BaseID since CloneID is unset for
+      // -basic-block-adress-map.
+      // TODO: Emit the full BBID when labels and sections can be mixed
+      // together.
+      OutStreamer->emitULEB128IntValue(MBB.getBBID()->BaseID);
       // Emit the basic block offset relative to the end of the previous block.
       // This is zero unless the block is padded due to alignment.
       emitLabelDifferenceAsULEB128(MBBSymbol, PrevMBBEndSymbol);
@@ -3139,6 +3136,14 @@ void AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo &MJTI,
     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
     break;
   }
+
+  case MachineJumpTableInfo::EK_CoffImgRel32: {
+    // This generates an unsigned 32-bit offset, which is MBB's address minus
+    // the COFF image base.
+    Value = MCSymbolRefExpr::create(
+        MBB->getSymbol(), MCSymbolRefExpr::VK_COFF_IMGREL32, OutContext);
+    break;
+  }
   }
 
   assert(Value && "Unknown entry kind!");
diff --git a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
index 0fe4b905831ff..34969d7524890 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
+++ b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
@@ -38,6 +38,7 @@ add_llvm_component_library(LLVMAsmPrinter
   Core
   DebugInfoCodeView
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   MC
   MCParser
   Remarks
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index 5e1b313b4d2fa..8f36c6b2d8d85 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -2085,8 +2085,8 @@ TypeIndex CodeViewDebug::lowerTypeFunction(const DISubroutineType *Ty) {
   ArrayRef<TypeIndex> ArgTypeIndices = {};
   if (!ReturnAndArgTypeIndices.empty()) {
     auto ReturnAndArgTypesRef = ArrayRef(ReturnAndArgTypeIndices);
-    ReturnTypeIndex = ReturnAndArgTypesRef.front();
-    ArgTypeIndices = ReturnAndArgTypesRef.drop_front();
+    ReturnTypeIndex = ReturnAndArgTypesRef.consume_front();
+    ArgTypeIndices = ReturnAndArgTypesRef;
   }
 
   ArgListRecord ArgListRec(TypeRecordKind::ArgList, ArgTypeIndices);
@@ -3564,17 +3564,41 @@ void CodeViewDebug::collectDebugInfoForJumpTables(const MachineFunction *MF,
           std::tie(Base, BaseOffset, Branch, EntrySize) =
               Asm->getCodeViewJumpTableInfo(JumpTableIndex, &BranchMI, Branch);
           break;
+        case MachineJumpTableInfo::EK_CoffImgRel32:
+          EntrySize = JumpTableEntrySize::UInt32;
+          Base = nullptr;
+          break;
         }
 
-        CurFn->JumpTables.push_back(
-            {EntrySize, Base, BaseOffset, Branch,
-             MF->getJTISymbol(JumpTableIndex, MMI->getContext()),
-             JTI.getJumpTables()[JumpTableIndex].MBBs.size()});
+        const MachineJumpTableEntry &JTE = JTI.getJumpTables()[JumpTableIndex];
+        JumpTableInfo CVJTI{EntrySize,
+                            Base,
+                            BaseOffset,
+                            Branch,
+                            MF->getJTISymbol(JumpTableIndex, MMI->getContext()),
+                            JTE.MBBs.size(),
+                            {}};
+        for (const auto &MBB : JTE.MBBs)
+          CVJTI.Cases.push_back(MBB->getSymbol());
+        CurFn->JumpTables.push_back(std::move(CVJTI));
       });
 }
 
 void CodeViewDebug::emitDebugInfoForJumpTables(const FunctionInfo &FI) {
-  for (auto JumpTable : FI.JumpTables) {
+  // Emit S_LABEL32 records for each jump target
+  for (const auto &JumpTable : FI.JumpTables) {
+    for (const auto &CaseSym : JumpTable.Cases) {
+      MCSymbol *LabelEnd = beginSymbolRecord(SymbolKind::S_LABEL32);
+      OS.AddComment("Offset and segment");
+      OS.emitCOFFSecRel32(CaseSym, 0);
+      OS.AddComment("Flags");
+      OS.emitInt8(0);
+      emitNullTerminatedSymbolName(OS, CaseSym->getName());
+      endSymbolRecord(LabelEnd);
+    }
+  }
+
+  for (const auto &JumpTable : FI.JumpTables) {
     MCSymbol *JumpTableEnd = beginSymbolRecord(SymbolKind::S_ARMSWITCHTABLE);
     if (JumpTable.Base) {
       OS.AddComment("Base offset");
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index c862802d835d7..c2b878e52e1c3 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -146,6 +146,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
     const MCSymbol *Branch;
     const MCSymbol *Table;
     size_t TableSize;
+    std::vector<const MCSymbol *> Cases;
   };
 
   // For each function, store a vector of labels to its instructions, as well as
diff --git a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
index 4bbf66206bfba..a2252d8f00f00 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -472,7 +472,10 @@ unsigned DIEExpr::sizeOf(const dwarf::FormParams &FormParams,
 }
 
 LLVM_DUMP_METHOD
-void DIEExpr::print(raw_ostream &O) const { O << "Expr: " << *Expr; }
+void DIEExpr::print(raw_ostream &O) const {
+  O << "Expr: ";
+  MCAsmInfo().printExpr(O, *Expr);
+}
 
 //===----------------------------------------------------------------------===//
 // DIELabel Implementation
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index a20c374e08935..8e8cda4eef573 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -429,7 +429,7 @@ DIE *DwarfCompileUnit::getOrCreateCommonBlock(
   addString(NDie, dwarf::DW_AT_name, Name);
   addGlobalName(Name, NDie, CB->getScope());
   if (CB->getFile())
-    addSourceLine(NDie, CB->getLineNo(), CB->getFile());
+    addSourceLine(NDie, CB->getLineNo(), /*Column*/ 0, CB->getFile());
   if (DIGlobalVariable *V = CB->getDecl())
     getCU().addLocationAttribute(&NDie, V, GlobalExprs);
   return &NDie;
@@ -1404,7 +1404,7 @@ DIE *DwarfCompileUnit::constructImportedEntityDIE(
   else
     EntityDie = getDIE(Entity);
   assert(EntityDie);
-  addSourceLine(*IMDie, Module->getLine(), Module->getFile());
+  addSourceLine(*IMDie, Module->getLine(), /*Column*/ 0, Module->getFile());
   addDIEEntry(*IMDie, dwarf::DW_AT_import, *EntityDie);
   StringRef Name = Module->getName();
   if (!Name.empty()) {
@@ -1701,6 +1701,11 @@ void DwarfCompileUnit::applyLabelAttributes(const DbgLabel &Label,
     addString(LabelDie, dwarf::DW_AT_name, Name);
   const auto *DILabel = Label.getLabel();
   addSourceLine(LabelDie, DILabel);
+  if (DILabel->isArtificial())
+    addFlag(LabelDie, dwarf::DW_AT_artificial);
+  if (DILabel->getCoroSuspendIdx())
+    addUInt(LabelDie, dwarf::DW_AT_LLVM_coro_suspend_idx, std::nullopt,
+            *DILabel->getCoroSuspendIdx());
 }
 
 /// Add a Dwarf expression attribute data and value.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 0edfca78b0886..11b85763a3fb1 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -32,7 +32,7 @@
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Function.h"
@@ -169,8 +169,10 @@ static cl::opt<DwarfDebug::MinimizeAddrInV5> MinimizeAddrInV5Option(
                           "Stuff")),
     cl::init(DwarfDebug::MinimizeAddrInV5::Default));
 
-static cl::opt<bool> KeyInstructionsAreStmts("dwarf-use-key-instructions",
-                                             cl::Hidden, cl::init(false));
+/// Set to false to ignore Key Instructions metadata.
+static cl::opt<bool> KeyInstructionsAreStmts(
+    "dwarf-use-key-instructions", cl::Hidden, cl::init(true),
+    cl::desc("Set to false to ignore Key Instructions metadata"));
 
 static constexpr unsigned ULEB128PadSize = 4;
 
@@ -2077,8 +2079,17 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
   unsigned LastAsmLine =
       Asm->OutStreamer->getContext().getCurrentDwarfLoc().getLine();
 
+  // There may be a mixture of scopes using and not using Key Instructions.
+  // Not-Key-Instructions functions inlined into Key Instructions functions
+  // should use not-key is_stmt handling. Key Instructions functions inlined
+  // into Not-Key-Instructions functions should use Key Instructions is_stmt
+  // handling.
+  bool ScopeUsesKeyInstructions =
+      KeyInstructionsAreStmts && DL &&
+      DL->getScope()->getSubprogram()->getKeyInstructionsEnabled();
+
   bool IsKey = false;
-  if (KeyInstructionsAreStmts && DL && DL.getLine())
+  if (ScopeUsesKeyInstructions && DL && DL.getLine())
     IsKey = KeyInstructions.contains(MI);
 
   if (!DL && MI == PrologEndLoc) {
@@ -2158,7 +2169,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     PrologEndLoc = nullptr;
   }
 
-  if (KeyInstructionsAreStmts) {
+  if (ScopeUsesKeyInstructions) {
     if (IsKey)
       Flags |= DWARF2_FLAG_IS_STMT;
   } else {
@@ -2372,6 +2383,8 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
            std::pair<uint8_t, SmallVector<const MachineInstr *, 2>>>
       GroupCandidates;
 
+  const auto &TII = *MF->getSubtarget().getInstrInfo();
+
   // For each instruction:
   //   * Skip insts without DebugLoc, AtomGroup or AtomRank, and line zeros.
   //   * Check if insts in this group have been seen already in GroupCandidates.
@@ -2400,24 +2413,20 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
       if (MI.isMetaInstruction())
         continue;
 
-      if (!MI.getDebugLoc() || !MI.getDebugLoc().getLine())
+      const DILocation *Loc = MI.getDebugLoc().get();
+      if (!Loc || !Loc->getLine())
         continue;
 
       // Reset the Buoy to this instruction if it has a different line number.
-      if (!Buoy ||
-          Buoy->getDebugLoc().getLine() != MI.getDebugLoc().getLine()) {
+      if (!Buoy || Buoy->getDebugLoc().getLine() != Loc->getLine()) {
         Buoy = &MI;
         BuoyAtom = 0; // Set later when we know which atom the buoy is used by.
       }
 
       // Call instructions are handled specially - we always mark them as key
       // regardless of atom info.
-      const auto &TII =
-          *MI.getParent()->getParent()->getSubtarget().getInstrInfo();
       bool IsCallLike = MI.isCall() || TII.isTailCall(MI);
       if (IsCallLike) {
-        assert(MI.getDebugLoc() && "Unexpectedly missing DL");
-
         // Calls are always key. Put the buoy (may not be the call) into
         // KeyInstructions directly rather than the candidate map to avoid it
         // being erased (and we may not have a group number for the call).
@@ -2427,14 +2436,13 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
         Buoy = nullptr;
         BuoyAtom = 0;
 
-        if (!MI.getDebugLoc()->getAtomGroup() ||
-            !MI.getDebugLoc()->getAtomRank())
+        if (!Loc->getAtomGroup() || !Loc->getAtomRank())
           continue;
       }
 
-      auto *InlinedAt = MI.getDebugLoc()->getInlinedAt();
-      uint64_t Group = MI.getDebugLoc()->getAtomGroup();
-      uint8_t Rank = MI.getDebugLoc()->getAtomRank();
+      auto *InlinedAt = Loc->getInlinedAt();
+      uint64_t Group = Loc->getAtomGroup();
+      uint8_t Rank = Loc->getAtomRank();
       if (!Group || !Rank)
         continue;
 
@@ -2476,8 +2484,8 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
         CandidateInsts.push_back(Buoy);
         CandidateRank = Rank;
 
-        assert(!BuoyAtom || BuoyAtom == MI.getDebugLoc()->getAtomGroup());
-        BuoyAtom = MI.getDebugLoc()->getAtomGroup();
+        assert(!BuoyAtom || BuoyAtom == Loc->getAtomGroup());
+        BuoyAtom = Loc->getAtomGroup();
       } else {
         // Don't add calls, because they've been dealt with already. This means
         // CandidateInsts might now be empty - handle that.
@@ -2651,10 +2659,12 @@ void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) {
   PrologEndLoc = emitInitialLocDirective(
       *MF, Asm->OutStreamer->getContext().getDwarfCompileUnitID());
 
+  // Run both `findForceIsStmtInstrs` and `computeKeyInstructions` because
+  // Not-Key-Instructions functions may be inlined into Key Instructions
+  // functions and vice versa.
   if (KeyInstructionsAreStmts)
     computeKeyInstructions(MF);
-  else
-    findForceIsStmtInstrs(MF);
+  findForceIsStmtInstrs(MF);
 }
 
 unsigned
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 9bd337a962b86..b03fac2d22a52 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -442,49 +442,53 @@ void DwarfUnit::addBlock(DIE &Die, dwarf::Attribute Attribute,
   addBlock(Die, Attribute, Block->BestForm(), Block);
 }
 
-void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, const DIFile *File) {
+void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, unsigned Column,
+                              const DIFile *File) {
   if (Line == 0)
     return;
 
   unsigned FileID = getOrCreateSourceID(File);
   addUInt(Die, dwarf::DW_AT_decl_file, std::nullopt, FileID);
   addUInt(Die, dwarf::DW_AT_decl_line, std::nullopt, Line);
+
+  if (Column != 0)
+    addUInt(Die, dwarf::DW_AT_decl_column, std::nullopt, Column);
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DILocalVariable *V) {
   assert(V);
 
-  addSourceLine(Die, V->getLine(), V->getFile());
+  addSourceLine(Die, V->getLine(), /*Column*/ 0, V->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DIGlobalVariable *G) {
   assert(G);
 
-  addSourceLine(Die, G->getLine(), G->getFile());
+  addSourceLine(Die, G->getLine(), /*Column*/ 0, G->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DISubprogram *SP) {
   assert(SP);
 
-  addSourceLine(Die, SP->getLine(), SP->getFile());
+  addSourceLine(Die, SP->getLine(), /*Column*/ 0, SP->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DILabel *L) {
   assert(L);
 
-  addSourceLine(Die, L->getLine(), L->getFile());
+  addSourceLine(Die, L->getLine(), L->getColumn(), L->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DIType *Ty) {
   assert(Ty);
 
-  addSourceLine(Die, Ty->getLine(), Ty->getFile());
+  addSourceLine(Die, Ty->getLine(), /*Column*/ 0, Ty->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DIObjCProperty *Ty) {
   assert(Ty);
 
-  addSourceLine(Die, Ty->getLine(), Ty->getFile());
+  addSourceLine(Die, Ty->getLine(), /*Column*/ 0, Ty->getFile());
 }
 
 void DwarfUnit::addConstantFPValue(DIE &Die, const ConstantFP *CFP) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
index 43bf197563867..fe05766cf36e1 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -216,7 +216,8 @@ class DwarfUnit : public DIEUnit {
                 DIEBlock *Block);
 
   /// Add location information to specified debug information entry.
-  void addSourceLine(DIE &Die, unsigned Line, const DIFile *File);
+  void addSourceLine(DIE &Die, unsigned Line, unsigned Column,
+                     const DIFile *File);
   void addSourceLine(DIE &Die, const DILocalVariable *V);
   void addSourceLine(DIE &Die, const DIGlobalVariable *G);
   void addSourceLine(DIE &Die, const DISubprogram *SP);
diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
index 55d1350e446ab..dccd71fffe053 100644
--- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
@@ -308,10 +308,8 @@ void WinException::endFuncletImpl() {
 const MCExpr *WinException::create32bitRef(const MCSymbol *Value) {
   if (!Value)
     return MCConstantExpr::create(0, Asm->OutContext);
-  return MCSymbolRefExpr::create(Value, useImageRel32
-                                            ? MCSymbolRefExpr::VK_COFF_IMGREL32
-                                            : MCSymbolRefExpr::VK_None,
-                                 Asm->OutContext);
+  auto Spec = useImageRel32 ? uint16_t(MCSymbolRefExpr::VK_COFF_IMGREL32) : 0;
+  return MCSymbolRefExpr::create(Value, Spec, Asm->OutContext);
 }
 
 const MCExpr *WinException::create32bitRef(const GlobalValue *GV) {
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index ff9f0ff5d5bc3..3b3e7a418feb5 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -42,6 +42,7 @@
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Function.h"
@@ -933,7 +934,13 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
 
   // Sort by hash value so that blocks with identical end sequences sort
   // together.
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+  // If origin-tracking is enabled then MergePotentialElt is no longer a POD
+  // type, so we need std::sort instead.
+  std::sort(MergePotentials.begin(), MergePotentials.end());
+#else
   array_pod_sort(MergePotentials.begin(), MergePotentials.end());
+#endif
 
   // Walk through equivalence sets looking for actual exact matches.
   while (MergePotentials.size() > 1) {
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 43574a54c37dd..9bbb89e37865d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6092,6 +6092,13 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
       }
 
       if (!ResultIndex) {
+        auto PtrInst = dyn_cast<Instruction>(ResultPtr);
+        // We know that we have a pointer without any offsets. If this pointer
+        // originates from a different basic block than the current one, we
+        // must be able to recreate it in the current basic block.
+        // We do not support the recreation of any instructions yet.
+        if (PtrInst && PtrInst->getParent() != MemoryInst->getParent())
+          return Modified;
         SunkAddr = ResultPtr;
       } else {
         if (ResultPtr->getType() != I8PtrTy)
diff --git a/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp b/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp
index 125ffd597bf23..e54419758410a 100644
--- a/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp
+++ b/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp
@@ -80,6 +80,10 @@ void CodeGenTargetMachineImpl::initAsmInfo() {
 
   TmpAsmInfo->setFullRegisterNames(Options.MCOptions.PPCUseFullRegisterNames);
 
+  assert(TmpAsmInfo->getExceptionHandlingType() ==
+             getTargetTriple().getDefaultExceptionHandling() &&
+         "MCAsmInfo and Triple disagree on default exception handling type");
+
   if (Options.ExceptionModel != ExceptionHandling::None)
     TmpAsmInfo->setExceptionsType(Options.ExceptionModel);
 
diff --git a/llvm/lib/CodeGen/FaultMaps.cpp b/llvm/lib/CodeGen/FaultMaps.cpp
index 3f8fe2402d653..79025e3ae80d2 100644
--- a/llvm/lib/CodeGen/FaultMaps.cpp
+++ b/llvm/lib/CodeGen/FaultMaps.cpp
@@ -86,16 +86,8 @@ void FaultMaps::emitFunctionInfo(const MCSymbol *FnLabel,
   OS.emitInt32(0); // Reserved
 
   for (const auto &Fault : FFI) {
-    LLVM_DEBUG(dbgs() << WFMP << "    fault type: "
-                      << faultTypeToString(Fault.Kind) << "\n");
     OS.emitInt32(Fault.Kind);
-
-    LLVM_DEBUG(dbgs() << WFMP << "    faulting PC offset: "
-                      << *Fault.FaultingOffsetExpr << "\n");
     OS.emitValue(Fault.FaultingOffsetExpr, 4);
-
-    LLVM_DEBUG(dbgs() << WFMP << "    fault handler PC offset: "
-                      << *Fault.HandlerOffsetExpr << "\n");
     OS.emitValue(Fault.HandlerOffsetExpr, 4);
   }
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 05dd269d48921..3b11d0848d300 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -5295,12 +5295,13 @@ bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
   return false;
 }
 
-MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) const {
-  assert(MI.getOpcode() == TargetOpcode::G_UDIV);
-  auto &UDiv = cast<GenericMachineInstr>(MI);
-  Register Dst = UDiv.getReg(0);
-  Register LHS = UDiv.getReg(1);
-  Register RHS = UDiv.getReg(2);
+MachineInstr *CombinerHelper::buildUDivorURemUsingMul(MachineInstr &MI) const {
+  unsigned Opcode = MI.getOpcode();
+  assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
+  auto &UDivorRem = cast<GenericMachineInstr>(MI);
+  Register Dst = UDivorRem.getReg(0);
+  Register LHS = UDivorRem.getReg(1);
+  Register RHS = UDivorRem.getReg(2);
   LLT Ty = MRI.getType(Dst);
   LLT ScalarTy = Ty.getScalarType();
   const unsigned EltBits = ScalarTy.getScalarSizeInBits();
@@ -5453,11 +5454,18 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) const {
   auto IsOne = MIB.buildICmp(
       CmpInst::Predicate::ICMP_EQ,
       Ty.isScalar() ? LLT::scalar(1) : Ty.changeElementSize(1), RHS, One);
-  return MIB.buildSelect(Ty, IsOne, LHS, Q);
+  auto ret = MIB.buildSelect(Ty, IsOne, LHS, Q);
+
+  if (Opcode == TargetOpcode::G_UREM) {
+    auto Prod = MIB.buildMul(Ty, ret, RHS);
+    return MIB.buildSub(Ty, LHS, Prod);
+  }
+  return ret;
 }
 
-bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
-  assert(MI.getOpcode() == TargetOpcode::G_UDIV);
+bool CombinerHelper::matchUDivorURemByConst(MachineInstr &MI) const {
+  unsigned Opcode = MI.getOpcode();
+  assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
   Register Dst = MI.getOperand(0).getReg();
   Register RHS = MI.getOperand(2).getReg();
   LLT DstTy = MRI.getType(Dst);
@@ -5474,7 +5482,8 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
   if (MF.getFunction().hasMinSize())
     return false;
 
-  if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
+  if (Opcode == TargetOpcode::G_UDIV &&
+      MI.getFlag(MachineInstr::MIFlag::IsExact)) {
     return matchUnaryPredicate(
         MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
   }
@@ -5494,14 +5503,17 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
              {DstTy.isVector() ? DstTy.changeElementSize(1) : LLT::scalar(1),
               DstTy}}))
       return false;
+    if (Opcode == TargetOpcode::G_UREM &&
+        !isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy, DstTy}}))
+      return false;
   }
 
   return matchUnaryPredicate(
       MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
 }
 
-void CombinerHelper::applyUDivByConst(MachineInstr &MI) const {
-  auto *NewMI = buildUDivUsingMul(MI);
+void CombinerHelper::applyUDivorURemByConst(MachineInstr &MI) const {
+  auto *NewMI = buildUDivorURemUsingMul(MI);
   replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 5d7e07003f10b..ef39fc74554c9 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1681,7 +1681,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
     auto OffsetMIB =
         MIRBuilder.buildConstant(OffsetTy, Offset);
 
-    if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
+    if (Offset >= 0 && cast<GEPOperator>(U).isInBounds())
       Flags |= MachineInstr::MIFlag::NoUWrap;
 
     MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index b87b029d01632..cdf192f9e7e3a 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -7695,6 +7695,22 @@ LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
   return Legalized;
 }
 
+/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
+/// convert fpround f64->f16 without double-rounding, so we manually perform the
+/// lowering here where we know it is valid.
+static LegalizerHelper::LegalizeResult
+loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
+                   LLT SrcTy, MachineIRBuilder &MIRBuilder) {
+  auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
+                ? MIRBuilder.buildUITOFP(SrcTy, Src)
+                : MIRBuilder.buildSITOFP(SrcTy, Src);
+  LLT S32Ty = SrcTy.changeElementSize(32);
+  auto M2 = MIRBuilder.buildFPTrunc(S32Ty, M1);
+  MIRBuilder.buildFPTrunc(Dst, M2);
+  MI.eraseFromParent();
+  return LegalizerHelper::Legalized;
+}
+
 LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
   auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
 
@@ -7706,6 +7722,9 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
     return Legalized;
   }
 
+  if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
+    return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+
   if (SrcTy != LLT::scalar(64))
     return UnableToLegalize;
 
@@ -7737,6 +7756,9 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
     return Legalized;
   }
 
+  if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
+    return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+
   if (SrcTy != S64)
     return UnableToLegalize;
 
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 1c4150127a908..b2f84351c9a95 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -458,7 +458,7 @@ bool LoadStoreOpt::processMergeCandidate(StoreMergeCandidate &C) {
     for (auto AliasInfo : reverse(C.PotentialAliases)) {
       MachineInstr *PotentialAliasOp = AliasInfo.first;
       unsigned PreCheckedIdx = AliasInfo.second;
-      if (static_cast<unsigned>(Idx) < PreCheckedIdx) {
+      if (Idx < PreCheckedIdx) {
         // Once our store index is lower than the index associated with the
         // potential alias, we know that we've already checked for this alias
         // and all of the earlier potential aliases too.
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 38ad582ba923c..e3d52f4125d00 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -1328,6 +1328,7 @@ unsigned MachineJumpTableInfo::getEntrySize(const DataLayout &TD) const {
   case MachineJumpTableInfo::EK_GPRel32BlockAddress:
   case MachineJumpTableInfo::EK_LabelDifference32:
   case MachineJumpTableInfo::EK_Custom32:
+  case MachineJumpTableInfo::EK_CoffImgRel32:
     return 4;
   case MachineJumpTableInfo::EK_Inline:
     return 0;
@@ -1348,6 +1349,7 @@ unsigned MachineJumpTableInfo::getEntryAlignment(const DataLayout &TD) const {
     return TD.getABIIntegerTypeAlignment(64).value();
   case MachineJumpTableInfo::EK_GPRel32BlockAddress:
   case MachineJumpTableInfo::EK_LabelDifference32:
+  case MachineJumpTableInfo::EK_CoffImgRel32:
   case MachineJumpTableInfo::EK_Custom32:
     return TD.getABIIntegerTypeAlignment(32).value();
   case MachineJumpTableInfo::EK_Inline:
@@ -1356,6 +1358,25 @@ unsigned MachineJumpTableInfo::getEntryAlignment(const DataLayout &TD) const {
   llvm_unreachable("Unknown jump table encoding!");
 }
 
+/// getEntryIsSigned - Return true if the load for the jump table index
+/// should use signed extension, false if zero extension (unsigned)
+bool MachineJumpTableInfo::getEntryIsSigned() const {
+  switch (getEntryKind()) {
+  case MachineJumpTableInfo::EK_BlockAddress:
+  case MachineJumpTableInfo::EK_GPRel64BlockAddress:
+  case MachineJumpTableInfo::EK_GPRel32BlockAddress:
+  case MachineJumpTableInfo::EK_LabelDifference32:
+  case MachineJumpTableInfo::EK_LabelDifference64:
+  case MachineJumpTableInfo::EK_Inline:
+  case MachineJumpTableInfo::EK_Custom32:
+    return true;
+
+  case MachineJumpTableInfo::EK_CoffImgRel32:
+    return false;
+  }
+  llvm_unreachable("Unknown jump table encoding!");
+}
+
 /// Create a new jump table entry in the jump table info.
 unsigned MachineJumpTableInfo::createJumpTableIndex(
                                const std::vector<MachineBasicBlock*> &DestBBs) {
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index c9079170ca575..f1811c47e5ad4 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -400,7 +400,7 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
     // Estimate register pressure during pre-regalloc pass.
     unsigned NumRPS = TRI->getNumRegPressureSets();
     RegPressure.resize(NumRPS);
-    std::fill(RegPressure.begin(), RegPressure.end(), 0);
+    llvm::fill(RegPressure, 0);
     RegLimit.resize(NumRPS);
     for (unsigned i = 0, e = NumRPS; i != e; ++i)
       RegLimit[i] = TRI->getRegPressureSetLimit(MF, i);
@@ -941,7 +941,7 @@ static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) {
 /// initialize the starting "register pressure". Note this does not count live
 /// through (livein but not used) registers.
 void MachineLICMImpl::InitRegPressure(MachineBasicBlock *BB) {
-  std::fill(RegPressure.begin(), RegPressure.end(), 0);
+  llvm::fill(RegPressure, 0);
 
   // If the preheader has only a single predecessor and it ends with a
   // fallthrough or an unconditional branch, then scan its predecessor for live
@@ -1219,7 +1219,7 @@ bool MachineLICMImpl::HasHighOperandLatency(MachineInstr &MI, unsigned DefIdx,
 /// Return true if the instruction is marked "cheap" or the operand latency
 /// between its def and a use is one or less.
 bool MachineLICMImpl::IsCheapInstruction(MachineInstr &MI) const {
-  if (TII->isAsCheapAsAMove(MI) || MI.isCopyLike())
+  if (TII->isAsCheapAsAMove(MI) || MI.isSubregToReg())
     return true;
 
   bool isCheap = false;
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index e48612369a5db..b0bce2c21a470 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -67,6 +67,7 @@
 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/IRBuilder.h"
@@ -77,6 +78,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/SuffixTree.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <tuple>
 #include <vector>
@@ -104,6 +106,7 @@ STATISTIC(StableHashAttempts,
           "Count of hashing attempts made for outlined functions");
 STATISTIC(StableHashDropped,
           "Count of unsuccessful hashing attempts for outlined functions");
+STATISTIC(NumRemovedLOHs, "Total number of Linker Optimization Hints removed");
 
 // Set to true if the user wants the outliner to run on linkonceodr linkage
 // functions. This is false by default because the linker can dedupe linkonceodr
@@ -426,6 +429,7 @@ struct MachineOutliner : public ModulePass {
   static char ID;
 
   MachineModuleInfo *MMI = nullptr;
+  const TargetMachine *TM = nullptr;
 
   /// Set to true if the outliner should consider functions with
   /// linkonceodr linkage.
@@ -461,6 +465,7 @@ struct MachineOutliner : public ModulePass {
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<MachineModuleInfoWrapperPass>();
+    AU.addRequired<TargetPassConfig>();
     AU.addPreserved<MachineModuleInfoWrapperPass>();
     AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>();
     AU.setPreservesAll();
@@ -1075,6 +1080,17 @@ bool MachineOutliner::outline(
                       << " B) > threshold (" << OutlinerBenefitThreshold
                       << " B)\n");
 
+    // Remove all Linker Optimization Hints from the candidates.
+    // TODO: The intersection of the LOHs from all candidates should be legal in
+    // the outlined function.
+    SmallPtrSet<MachineInstr *, 2> MIs;
+    for (Candidate &C : OF->Candidates) {
+      for (MachineInstr &MI : C)
+        MIs.insert(&MI);
+      NumRemovedLOHs += TM->clearLinkerOptimizationHints(MIs);
+      MIs.clear();
+    }
+
     // It's beneficial. Create the function and outline its sequence's
     // occurrences.
     OF->MF = createOutlinedFunction(M, *OF, Mapper, OutlinedFunctionNum);
@@ -1386,6 +1402,7 @@ bool MachineOutliner::runOnModule(Module &M) {
   initializeOutlinerMode(M);
 
   MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+  TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
 
   // Number to append to the current outlined function.
   unsigned OutlinedFunctionNum = 0;
diff --git a/llvm/lib/CodeGen/MachinePostDominators.cpp b/llvm/lib/CodeGen/MachinePostDominators.cpp
index 1cb7e465881a2..d6b5ee0e46d74 100644
--- a/llvm/lib/CodeGen/MachinePostDominators.cpp
+++ b/llvm/lib/CodeGen/MachinePostDominators.cpp
@@ -99,8 +99,8 @@ MachineBasicBlock *MachinePostDominatorTree::findNearestCommonDominator(
     ArrayRef<MachineBasicBlock *> Blocks) const {
   assert(!Blocks.empty());
 
-  MachineBasicBlock *NCD = Blocks.front();
-  for (MachineBasicBlock *BB : Blocks.drop_front()) {
+  MachineBasicBlock *NCD = Blocks.consume_front();
+  for (MachineBasicBlock *BB : Blocks) {
     NCD = Base::findNearestCommonDominator(NCD, BB);
 
     // Stop when the root is reached.
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 7b4f29e82fe6f..76cba2949af60 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1290,13 +1290,13 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceTopDown() const {
                    SchedModel.getWriteProcResEnd(SC)));
 
     if (MISchedSortResourcesInTrace)
-      llvm::stable_sort(ResourcesIt,
-                        [](const MCWriteProcResEntry &LHS,
-                           const MCWriteProcResEntry &RHS) -> bool {
-                          return LHS.AcquireAtCycle < RHS.AcquireAtCycle ||
-                                 (LHS.AcquireAtCycle == RHS.AcquireAtCycle &&
-                                  LHS.ReleaseAtCycle < RHS.ReleaseAtCycle);
-                        });
+      llvm::stable_sort(
+          ResourcesIt,
+          [](const MCWriteProcResEntry &LHS,
+             const MCWriteProcResEntry &RHS) -> bool {
+            return std::tie(LHS.AcquireAtCycle, LHS.ReleaseAtCycle) <
+                   std::tie(RHS.AcquireAtCycle, RHS.ReleaseAtCycle);
+          });
     for (const MCWriteProcResEntry &PI : ResourcesIt) {
       C = FirstCycle;
       const std::string ResName =
@@ -1371,13 +1371,13 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceBottomUp() const {
                    SchedModel.getWriteProcResEnd(SC)));
 
     if (MISchedSortResourcesInTrace)
-      llvm::stable_sort(ResourcesIt,
-                        [](const MCWriteProcResEntry &LHS,
-                           const MCWriteProcResEntry &RHS) -> bool {
-                          return LHS.AcquireAtCycle < RHS.AcquireAtCycle ||
-                                 (LHS.AcquireAtCycle == RHS.AcquireAtCycle &&
-                                  LHS.ReleaseAtCycle < RHS.ReleaseAtCycle);
-                        });
+      llvm::stable_sort(
+          ResourcesIt,
+          [](const MCWriteProcResEntry &LHS,
+             const MCWriteProcResEntry &RHS) -> bool {
+            return std::tie(LHS.AcquireAtCycle, LHS.ReleaseAtCycle) <
+                   std::tie(RHS.AcquireAtCycle, RHS.ReleaseAtCycle);
+          });
     for (const MCWriteProcResEntry &PI : ResourcesIt) {
       C = FirstCycle;
       const std::string ResName =
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 2d25f097348af..2d7987a2e1988 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -4152,10 +4152,8 @@ void RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
   if (JoinGlobalCopies) {
     SmallVector<MachineInstr *, 2> LocalTerminals;
     SmallVector<MachineInstr *, 2> GlobalTerminals;
-    // Coalesce copies bottom-up to coalesce local defs before local uses. They
-    // are not inherently easier to resolve, but slightly preferable until we
-    // have local live range splitting. In particular this is required by
-    // cmp+jmp macro fusion.
+    // Coalesce copies top-down to propagate coalescing and rematerialization
+    // forward.
     for (MachineInstr &MI : *MBB) {
       if (!MI.isCopyLike())
         continue;
@@ -4177,6 +4175,8 @@ void RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
     WorkList.append(GlobalTerminals.begin(), GlobalTerminals.end());
   } else {
     SmallVector<MachineInstr *, 2> Terminals;
+    // Coalesce copies top-down to propagate coalescing and rematerialization
+    // forward.
     for (MachineInstr &MII : *MBB)
       if (MII.isCopyLike()) {
         if (applyTerminalRule(MII))
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 08dab7c697b99..9ffdda28f7899 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -638,7 +638,6 @@ namespace {
     SDValue mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex);
     SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
     SDValue combineInsertEltToLoad(SDNode *N, unsigned InsIndex);
-    SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
     SDValue BuildSDIV(SDNode *N);
     SDValue BuildSDIVPow2(SDNode *N);
     SDValue BuildUDIV(SDNode *N);
@@ -4281,7 +4280,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     return V;
 
   // (A - B) - 1  ->  add (xor B, -1), A
-  if (sd_match(N, m_Sub(m_OneUse(m_Sub(m_Value(A), m_Value(B))), m_One())))
+  if (sd_match(N, m_Sub(m_OneUse(m_Sub(m_Value(A), m_Value(B))),
+                        m_One(/*AllowUndefs=*/true))))
     return DAG.getNode(ISD::ADD, DL, VT, A, DAG.getNOT(DL, B, VT));
 
   // Look for:
@@ -7518,7 +7518,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     // vector as a scalar and use the splat value.
     APInt Constant = APInt::getZero(1);
     if (const ConstantSDNode *C = isConstOrConstSplat(
-            N1, /*AllowUndef=*/false, /*AllowTruncation=*/true)) {
+            N1, /*AllowUndefs=*/false, /*AllowTruncation=*/true)) {
       Constant = C->getAPIntValue();
     } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
       unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
@@ -9902,11 +9902,14 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
     if (SDValue Combined = visitADDLike(N))
       return Combined;
 
-  // fold !(x cc y) -> (x !cc y)
+  // fold not (setcc x, y, cc) -> setcc x y !cc
+  // Avoid breaking: and (not(setcc x, y, cc), z) -> andn for vec
   unsigned N0Opcode = N0.getOpcode();
   SDValue LHS, RHS, CC;
   if (TLI.isConstTrueVal(N1) &&
-      isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/ true)) {
+      isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/ true) &&
+      !(VT.isVector() && TLI.hasAndNot(SDValue(N, 0)) && N->hasOneUse() &&
+        N->use_begin()->getUser()->getOpcode() == ISD::AND)) {
     ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
                                                LHS.getValueType());
     if (!LegalOperations ||
@@ -11402,18 +11405,25 @@ SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
   SDValue AbsOp0 = N->getOperand(0);
   unsigned Opc0 = Op0.getOpcode();
 
-  // Check if the operands of the sub are (zero|sign)-extended.
-  // TODO: Should we use ValueTracking instead?
+  // Check if the operands of the sub are (zero|sign)-extended, otherwise
+  // fallback to ValueTracking.
   if (Opc0 != Op1.getOpcode() ||
       (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND &&
        Opc0 != ISD::SIGN_EXTEND_INREG)) {
     // fold (abs (sub nsw x, y)) -> abds(x, y)
     // Don't fold this for unsupported types as we lose the NSW handling.
-    if (AbsOp0->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) &&
-        TLI.preferABDSToABSWithNSW(VT)) {
+    if (hasOperation(ISD::ABDS, VT) && TLI.preferABDSToABSWithNSW(VT) &&
+        (AbsOp0->getFlags().hasNoSignedWrap() ||
+         DAG.willNotOverflowSub(/*IsSigned=*/true, Op0, Op1))) {
       SDValue ABD = DAG.getNode(ISD::ABDS, DL, VT, Op0, Op1);
       return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
     }
+    // fold (abs (sub x, y)) -> abdu(x, y)
+    if (hasOperation(ISD::ABDU, VT) && DAG.SignBitIsZero(Op0) &&
+        DAG.SignBitIsZero(Op1)) {
+      SDValue ABD = DAG.getNode(ISD::ABDU, DL, VT, Op0, Op1);
+      return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
+    }
     return SDValue();
   }
 
@@ -13069,6 +13079,107 @@ SDValue DAGCombiner::visitVP_SELECT(SDNode *N) {
   return SDValue();
 }
 
+static SDValue combineVSelectWithAllOnesOrZeros(SDValue Cond, SDValue TVal,
+                                                SDValue FVal,
+                                                const TargetLowering &TLI,
+                                                SelectionDAG &DAG,
+                                                const SDLoc &DL) {
+  EVT VT = TVal.getValueType();
+  if (!TLI.isTypeLegal(VT))
+    return SDValue();
+
+  EVT CondVT = Cond.getValueType();
+  assert(CondVT.isVector() && "Vector select expects a vector selector!");
+
+  bool IsTAllZero = ISD::isBuildVectorAllZeros(TVal.getNode());
+  bool IsTAllOne = ISD::isBuildVectorAllOnes(TVal.getNode());
+  bool IsFAllZero = ISD::isBuildVectorAllZeros(FVal.getNode());
+  bool IsFAllOne = ISD::isBuildVectorAllOnes(FVal.getNode());
+
+  // no vselect(cond, 0/-1, X) or vselect(cond, X, 0/-1), return
+  if (!IsTAllZero && !IsTAllOne && !IsFAllZero && !IsFAllOne)
+    return SDValue();
+
+  // select Cond, 0, 0 → 0
+  if (IsTAllZero && IsFAllZero) {
+    return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, DL, VT)
+                                : DAG.getConstant(0, DL, VT);
+  }
+
+  // check select(setgt lhs, -1), 1, -1 --> or (sra lhs, bitwidth - 1), 1
+  APInt TValAPInt;
+  if (Cond.getOpcode() == ISD::SETCC &&
+      Cond.getOperand(2) == DAG.getCondCode(ISD::SETGT) &&
+      Cond.getOperand(0).getValueType() == VT && VT.isSimple() &&
+      ISD::isConstantSplatVector(TVal.getNode(), TValAPInt) &&
+      TValAPInt.isOne() &&
+      ISD::isConstantSplatVectorAllOnes(Cond.getOperand(1).getNode()) &&
+      ISD::isConstantSplatVectorAllOnes(FVal.getNode())) {
+    return SDValue();
+  }
+
+  // To use the condition operand as a bitwise mask, it must have elements that
+  // are the same size as the select elements. i.e, the condition operand must
+  // have already been promoted from the IR select condition type <N x i1>.
+  // Don't check if the types themselves are equal because that excludes
+  // vector floating-point selects.
+  if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
+    return SDValue();
+
+  // Cond value must be 'sign splat' to be converted to a logical op.
+  if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
+    return SDValue();
+
+  // Try inverting Cond and swapping T/F if it gives all-ones/all-zeros form
+  if (!IsTAllOne && !IsFAllZero && Cond.hasOneUse() &&
+      Cond.getOpcode() == ISD::SETCC &&
+      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
+          CondVT) {
+    if (IsTAllZero || IsFAllOne) {
+      SDValue CC = Cond.getOperand(2);
+      ISD::CondCode InverseCC = ISD::getSetCCInverse(
+          cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
+      Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
+                          InverseCC);
+      std::swap(TVal, FVal);
+      std::swap(IsTAllOne, IsFAllOne);
+      std::swap(IsTAllZero, IsFAllZero);
+    }
+  }
+
+  assert(DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits() &&
+         "Select condition no longer all-sign bits");
+
+  // select Cond, -1, 0 → bitcast Cond
+  if (IsTAllOne && IsFAllZero)
+    return DAG.getBitcast(VT, Cond);
+
+  // select Cond, -1, x → or Cond, x
+  if (IsTAllOne) {
+    SDValue X = DAG.getBitcast(CondVT, FVal);
+    SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, X);
+    return DAG.getBitcast(VT, Or);
+  }
+
+  // select Cond, x, 0 → and Cond, x
+  if (IsFAllZero) {
+    SDValue X = DAG.getBitcast(CondVT, TVal);
+    SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, X);
+    return DAG.getBitcast(VT, And);
+  }
+
+  // select Cond, 0, x -> and not(Cond), x
+  if (IsTAllZero &&
+      (isBitwiseNot(peekThroughBitcasts(Cond)) || TLI.hasAndNot(Cond))) {
+    SDValue X = DAG.getBitcast(CondVT, FVal);
+    SDValue And =
+        DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), X);
+    return DAG.getBitcast(VT, And);
+  }
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitVSELECT(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -13337,6 +13448,9 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
   if (SimplifyDemandedVectorElts(SDValue(N, 0)))
     return SDValue(N, 0);
 
+  if (SDValue V = combineVSelectWithAllOnesOrZeros(N0, N1, N2, TLI, DAG, DL))
+    return V;
+
   return SDValue();
 }
 
@@ -16329,8 +16443,8 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
         TLI.isTypeLegal(VT.getVectorElementType()))) &&
       N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() &&
       cast<BuildVectorSDNode>(N0)->isConstant())
-    return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(),
-                                             VT.getVectorElementType());
+    return DAG.FoldConstantBuildVector(cast<BuildVectorSDNode>(N0), SDLoc(N),
+                                       VT.getVectorElementType());
 
   // If the input is a constant, let getNode fold it.
   if (isIntOrFPConstant(N0)) {
@@ -16609,8 +16723,7 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
   // Fold freeze(op(x, ...)) -> op(freeze(x), ...).
   // Try to push freeze through instructions that propagate but don't produce
   // poison as far as possible. If an operand of freeze follows three
-  // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
-  // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push
+  // conditions 1) one-use, and 2) does not produce poison then push
   // the freeze through to the operands that are not guaranteed non-poison.
   // NOTE: we will strip poison-generating flags, so ignore them here.
   if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false,
@@ -16618,13 +16731,6 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
       N0->getNumValues() != 1 || !N0->hasOneUse())
     return SDValue();
 
-  bool AllowMultipleMaybePoisonOperands =
-      N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC ||
-      N0.getOpcode() == ISD::BUILD_VECTOR ||
-      N0.getOpcode() == ISD::BUILD_PAIR ||
-      N0.getOpcode() == ISD::VECTOR_SHUFFLE ||
-      N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL;
-
   // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all
   // ones" or "constant" into something that depends on FrozenUndef. We can
   // instead pick undef values to keep those properties, while at the same time
@@ -16657,10 +16763,6 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
       MaybePoisonOperandNumbers.push_back(OpNo);
     if (!HadMaybePoisonOperands)
       continue;
-    if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
-      // Multiple maybe-poison ops when not allowed - bail out.
-      return SDValue();
-    }
   }
   // NOTE: the whole op may be not guaranteed to not be undef or poison because
   // it could create undef or poison due to it's poison-generating flags.
@@ -16714,105 +16816,25 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
       Op = DAG.getFreeze(Op);
   }
 
-  SDValue R;
-  if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0)) {
-    // Special case handling for ShuffleVectorSDNode nodes.
-    R = DAG.getVectorShuffle(N0.getValueType(), SDLoc(N0), Ops[0], Ops[1],
-                             SVN->getMask());
-  } else {
-    // NOTE: this strips poison generating flags.
-    // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
-    // ninf, nsz, or fast.
-    // However, contract, reassoc, afn, and arcp should be preserved,
-    // as these fast-math flags do not introduce poison values.
-    SDNodeFlags SrcFlags = N0->getFlags();
-    SDNodeFlags SafeFlags;
-    SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
-    SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
-    SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
-    SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
-    R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops, SafeFlags);
-  }
-  assert(DAG.isGuaranteedNotToBeUndefOrPoison(R, /*PoisonOnly*/ false) &&
-         "Can't create node that may be undef/poison!");
-  return R;
-}
-
-/// We know that BV is a build_vector node with Constant, ConstantFP or Undef
-/// operands. DstEltVT indicates the destination element value type.
-SDValue DAGCombiner::
-ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
-  EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
-
-  // If this is already the right type, we're done.
-  if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
-
-  unsigned SrcBitSize = SrcEltVT.getSizeInBits();
-  unsigned DstBitSize = DstEltVT.getSizeInBits();
-
-  // If this is a conversion of N elements of one type to N elements of another
-  // type, convert each element.  This handles FP<->INT cases.
-  if (SrcBitSize == DstBitSize) {
-    SmallVector<SDValue, 8> Ops;
-    for (SDValue Op : BV->op_values()) {
-      // If the vector element type is not legal, the BUILD_VECTOR operands
-      // are promoted and implicitly truncated.  Make that explicit here.
-      if (Op.getValueType() != SrcEltVT)
-        Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op);
-      Ops.push_back(DAG.getBitcast(DstEltVT, Op));
-      AddToWorklist(Ops.back().getNode());
-    }
-    EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
-                              BV->getValueType(0).getVectorNumElements());
-    return DAG.getBuildVector(VT, SDLoc(BV), Ops);
-  }
-
-  // Otherwise, we're growing or shrinking the elements.  To avoid having to
-  // handle annoying details of growing/shrinking FP values, we convert them to
-  // int first.
-  if (SrcEltVT.isFloatingPoint()) {
-    // Convert the input float vector to a int vector where the elements are the
-    // same sizes.
-    EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
-    BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
-    SrcEltVT = IntVT;
-  }
-
-  // Now we know the input is an integer vector.  If the output is a FP type,
-  // convert to integer first, then to FP of the right size.
-  if (DstEltVT.isFloatingPoint()) {
-    EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
-    SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
-
-    // Next, convert to FP elements of the same size.
-    return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
-  }
-
-  // Okay, we know the src/dst types are both integers of differing types.
-  assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
-
-  // TODO: Should ConstantFoldBITCASTofBUILD_VECTOR always take a
-  // BuildVectorSDNode?
-  auto *BVN = cast<BuildVectorSDNode>(BV);
-
-  // Extract the constant raw bit data.
-  BitVector UndefElements;
-  SmallVector<APInt> RawBits;
-  bool IsLE = DAG.getDataLayout().isLittleEndian();
-  if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
-    return SDValue();
-
-  SDLoc DL(BV);
-  SmallVector<SDValue, 8> Ops;
-  for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
-    if (UndefElements[I])
-      Ops.push_back(DAG.getUNDEF(DstEltVT));
-    else
-      Ops.push_back(DAG.getConstant(RawBits[I], DL, DstEltVT));
-  }
+  SDLoc DL(N0);
 
-  EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
-  return DAG.getBuildVector(VT, DL, Ops);
+  // Special case handling for ShuffleVectorSDNode nodes.
+  if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0))
+    return DAG.getVectorShuffle(N0.getValueType(), DL, Ops[0], Ops[1],
+                                SVN->getMask());
+
+  // NOTE: this strips poison generating flags.
+  // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
+  // ninf, nsz, or fast.
+  // However, contract, reassoc, afn, and arcp should be preserved,
+  // as these fast-math flags do not introduce poison values.
+  SDNodeFlags SrcFlags = N0->getFlags();
+  SDNodeFlags SafeFlags;
+  SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
+  SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
+  SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
+  SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
+  return DAG.getNode(N0.getOpcode(), DL, N0->getVTList(), Ops, SafeFlags);
 }
 
 // Returns true if floating point contraction is allowed on the FMUL-SDValue
@@ -16820,7 +16842,7 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
 static bool isContractableFMUL(const TargetOptions &Options, SDValue N) {
   assert(N.getOpcode() == ISD::FMUL);
 
-  return Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath ||
+  return Options.AllowFPOpFusion == FPOpFusion::Fast ||
          N->getFlags().hasAllowContract();
 }
 
@@ -17093,8 +17115,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     return SDValue();
 
   const SDNodeFlags Flags = N->getFlags();
-  bool AllowFusionGlobally = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
-                              Options.UnsafeFPMath || HasFMAD);
+  bool AllowFusionGlobally =
+      (Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD);
 
   // If the subtraction is not contractable, do not combine.
   if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
@@ -17249,13 +17271,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     }
   }
 
-  auto isReassociable = [&Options](SDNode *N) {
-    return Options.UnsafeFPMath || N->getFlags().hasAllowReassociation();
-  };
-
-  auto isContractableAndReassociableFMUL = [&isContractableFMUL,
-                                            &isReassociable](SDValue N) {
-    return isContractableFMUL(N) && isReassociable(N.getNode());
+  auto isContractableAndReassociableFMUL = [&isContractableFMUL](SDValue N) {
+    return isContractableFMUL(N) && N->getFlags().hasAllowReassociation();
   };
 
   auto isFusedOp = [&](SDValue N) {
@@ -17263,8 +17280,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   };
 
   // More folding opportunities when target permits.
-  if (Aggressive && isReassociable(N)) {
-    bool CanFuse = Options.UnsafeFPMath || N->getFlags().hasAllowContract();
+  if (Aggressive && N->getFlags().hasAllowReassociation()) {
+    bool CanFuse = N->getFlags().hasAllowContract();
     // fold (fsub (fma x, y, (fmul u, v)), z)
     //   -> (fma x, y (fma u, v, (fneg z)))
     if (CanFuse && isFusedOp(N0) &&
@@ -17421,8 +17438,7 @@ SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
 
   // Floating-point multiply-add with intermediate rounding. This can result
   // in a less precise result due to the changed rounding order.
-  bool HasFMAD = Options.UnsafeFPMath &&
-                 (LegalOperations && TLI.isFMADLegal(DAG, N));
+  bool HasFMAD = LegalOperations && TLI.isFMADLegal(DAG, N);
 
   // No valid opcode, do not combine.
   if (!HasFMAD && !HasFMA)
@@ -18096,8 +18112,7 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) {
 
   // FIXME: use fast math flags instead of Options.UnsafeFPMath
   // TODO: Finally migrate away from global TargetOptions.
-  if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
-      (Options.NoNaNsFPMath && Options.NoInfsFPMath) ||
+  if ((Options.NoNaNsFPMath && Options.NoInfsFPMath) ||
       (N->getFlags().hasNoNaNs() && N->getFlags().hasNoInfs())) {
     if (Options.NoSignedZerosFPMath || N->getFlags().hasNoSignedZeros() ||
         (N2CFP && !N2CFP->isExactlyValue(-0.0))) {
@@ -18321,8 +18336,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
     // Only do the transform if the reciprocal is a legal fp immediate that
     // isn't too nasty (eg NaN, denormal, ...).
     if (((st == APFloat::opOK && !Recip.isDenormal()) ||
-         (st == APFloat::opInexact &&
-          (Options.UnsafeFPMath || Flags.hasAllowReciprocal()))) &&
+         (st == APFloat::opInexact && Flags.hasAllowReciprocal())) &&
         (!LegalOperations ||
          // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
          // backend)... we should handle this gracefully after Legalize.
@@ -18333,7 +18347,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
                          DAG.getConstantFP(Recip, DL, VT));
   }
 
-  if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
+  if (Flags.hasAllowReciprocal()) {
     // If this FDIV is part of a reciprocal square root, it may be folded
     // into a target-specific square root estimate instruction.
     if (N1.getOpcode() == ISD::FSQRT) {
@@ -18408,7 +18422,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
 
   // Fold X/Sqrt(X) -> Sqrt(X)
   if ((Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) &&
-      (Options.UnsafeFPMath || Flags.hasAllowReassociation()))
+      Flags.hasAllowReassociation())
     if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0))
       return N1;
 
@@ -23195,13 +23209,16 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
 
     // Ensure all the operands are the same value type, fill any missing
     // operands with UNDEF and create the BUILD_VECTOR.
-    auto CanonicalizeBuildVector = [&](SmallVectorImpl<SDValue> &Ops) {
+    auto CanonicalizeBuildVector = [&](SmallVectorImpl<SDValue> &Ops,
+                                       bool FreezeUndef = false) {
       assert(Ops.size() == NumElts && "Unexpected vector size");
+      SDValue UndefOp = FreezeUndef ? DAG.getFreeze(DAG.getUNDEF(MaxEltVT))
+                                    : DAG.getUNDEF(MaxEltVT);
       for (SDValue &Op : Ops) {
         if (Op)
           Op = VT.isInteger() ? DAG.getAnyExtOrTrunc(Op, DL, MaxEltVT) : Op;
         else
-          Op = DAG.getUNDEF(MaxEltVT);
+          Op = UndefOp;
       }
       return DAG.getBuildVector(VT, DL, Ops);
     };
@@ -23215,6 +23232,10 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
       if (CurVec.isUndef())
         return CanonicalizeBuildVector(Ops);
 
+      // FREEZE(UNDEF) - build new BUILD_VECTOR from already inserted operands.
+      if (ISD::isFreezeUndef(CurVec.getNode()) && CurVec.hasOneUse())
+        return CanonicalizeBuildVector(Ops, /*FreezeUndef=*/true);
+
       // BUILD_VECTOR - insert unused operands and build new BUILD_VECTOR.
       if (CurVec.getOpcode() == ISD::BUILD_VECTOR && CurVec.hasOneUse()) {
         for (unsigned I = 0; I != NumElts; ++I)
@@ -27599,6 +27620,11 @@ SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
     if (DAG.isConstantValueOfAnyType(N1.getOperand(0)) || N1.hasOneUse())
       return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, N1.getOperand(0));
 
+  // insert_subvector (splat X), (splat X), N2 -> splat X
+  if (N0.getOpcode() == ISD::SPLAT_VECTOR && N0.getOpcode() == N1.getOpcode() &&
+      N0.getOperand(0) == N1.getOperand(0))
+    return N0;
+
   // If we are inserting a bitcast value into an undef, with the same
   // number of elements, just use the bitcast input of the extract.
   // i.e. INSERT_SUBVECTOR UNDEF (BITCAST N1) N2 ->
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 4b7a9127b3fc3..03d3e8eab35d0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -631,16 +631,17 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, VRBaseMapType &VRBaseMap,
 void
 InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
                                      VRBaseMapType &VRBaseMap) {
-  Register VReg = getVR(Node->getOperand(0), VRBaseMap);
-
   // Create the new VReg in the destination class and emit a copy.
   unsigned DstRCIdx = Node->getConstantOperandVal(1);
   const TargetRegisterClass *DstRC =
     TRI->getAllocatableClass(TRI->getRegClass(DstRCIdx));
   Register NewVReg = MRI->createVirtualRegister(DstRC);
-  BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
-    NewVReg).addReg(VReg);
+  const MCInstrDesc &II = TII->get(TargetOpcode::COPY);
+  MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II, NewVReg);
+  AddOperand(MIB, Node->getOperand(0), 1, &II, VRBaseMap, /*IsDebug=*/false,
+             /*IsClone=*/false, /*IsCloned*/ false);
 
+  MBB->insert(InsertPos, MIB);
   SDValue Op(Node, 0);
   bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second;
   (void)isNew; // Silence compiler warning.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index f5f4d71236fee..7bb6cadbce378 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3356,6 +3356,32 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Results.push_back(Op);
     break;
   }
+  case ISD::FCANONICALIZE: {
+    // This implements llvm.canonicalize.f* by multiplication with 1.0, as
+    // suggested in
+    // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
+    // It uses strict_fp operations even outside a strict_fp context in order
+    // to guarantee that the canonicalization is not optimized away by later
+    // passes. The result chain introduced by that is intentionally ignored
+    // since no ordering requirement is intended here.
+
+    // Create strict multiplication by 1.0.
+    SDValue Operand = Node->getOperand(0);
+    EVT VT = Operand.getValueType();
+    SDValue One = DAG.getConstantFP(1.0, dl, VT);
+    SDValue Chain = DAG.getEntryNode();
+    SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other},
+                              {Chain, Operand, One});
+
+    // Propagate existing flags on canonicalize, and additionally set
+    // NoFPExcept.
+    SDNodeFlags CanonicalizeFlags = Node->getFlags();
+    CanonicalizeFlags.setNoFPExcept(true);
+    Mul->setFlags(CanonicalizeFlags);
+
+    Results.push_back(Mul);
+    break;
+  }
   case ISD::SIGN_EXTEND_INREG: {
     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
     EVT VT = Node->getValueType(0);
@@ -4107,8 +4133,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     const DataLayout &TD = DAG.getDataLayout();
     EVT PTy = TLI.getPointerTy(TD);
 
-    unsigned EntrySize =
-      DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
+    MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
+    unsigned EntrySize = MJTI->getEntrySize(TD);
 
     // For power-of-two jumptable entry sizes convert multiplication to a shift.
     // This transformation needs to be done here since otherwise the MIPS
@@ -4125,10 +4151,15 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
 
     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
     SDValue LD = DAG.getExtLoad(
-        ISD::SEXTLOAD, dl, PTy, Chain, Addr,
-        MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
+        MJTI->getEntryIsSigned() ? ISD::SEXTLOAD : ISD::ZEXTLOAD, dl, PTy,
+        Chain, Addr, MachinePointerInfo::getJumpTable(DAG.getMachineFunction()),
+        MemVT);
     Addr = LD;
-    if (TLI.isJumpTableRelative()) {
+    if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_CoffImgRel32) {
+      SDValue ImageBase = DAG.getExternalSymbol(
+          "__ImageBase", TLI.getPointerTy(DAG.getDataLayout()));
+      Addr = DAG.getMemBasePlusOffset(ImageBase, Addr, dl);
+    } else if (TLI.isJumpTableRelative()) {
       // For PIC, the sequence is:
       // BRIND(RelocBase + load(Jumptable + index))
       // RelocBase can be JumpTable, GOT or some sort of global base.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 7dfb43164be37..2cad36eff9c88 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -704,22 +704,15 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
-  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
-                                               RTLIB::POW_F32,
-                                               RTLIB::POW_F64,
-                                               RTLIB::POW_F80,
-                                               RTLIB::POW_F128,
-                                               RTLIB::POW_PPCF128));
+  return SoftenFloatRes_Binary(N, RTLIB::getPOW(N->getValueType(0)));
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
   bool IsStrict = N->isStrictFPOpcode();
   unsigned Offset = IsStrict ? 1 : 0;
-  assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
-          N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
-         "Unsupported power type!");
   bool IsPowI =
       N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
 
   RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
                              : RTLIB::getLDEXP(N->getValueType(0));
@@ -727,19 +720,22 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
   if (!TLI.getLibcallName(LC)) {
     // Some targets don't have a powi libcall; use pow instead.
     // FIXME: Implement this if some target needs it.
-    DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
-    return DAG.getUNDEF(N->getValueType(0));
+    DAG.getContext()->emitError("do not know how to soften fpowi to fpow");
+    if (IsStrict)
+      ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
+    return DAG.getPOISON(NVT);
   }
 
   if (DAG.getLibInfo().getIntSize() !=
       N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
     // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
     // would use the wrong type for the argument.
-    DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
-    return DAG.getUNDEF(N->getValueType(0));
+    DAG.getContext()->emitError("powi exponent does not match sizeof(int)");
+    if (IsStrict)
+      ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
+    return DAG.getPOISON(NVT);
   }
 
-  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
                      N->getOperand(1 + Offset) };
   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
@@ -760,20 +756,21 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
   EVT VT0 = N->getValueType(0);
   EVT VT1 = N->getValueType(1);
   RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
+  EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
+  SDLoc DL(N);
 
   if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
     // If the exponent does not match with sizeof(int) a libcall would use the
     // wrong type for the argument.
     // TODO: Should be able to handle mismatches.
     DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
-    return DAG.getUNDEF(N->getValueType(0));
+    SDValue PoisonExp = DAG.getPOISON(VT1);
+    ReplaceValueWith(SDValue(N, 1), PoisonExp);
+    return DAG.getMergeValues({DAG.getPOISON(NVT0), PoisonExp}, DL);
   }
 
-  EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
   SDValue StackSlot = DAG.CreateStackTemporary(VT1);
 
-  SDLoc DL(N);
-
   auto PointerTy = PointerType::getUnqual(*DAG.getContext());
   TargetLowering::MakeLibCallOptions CallOptions;
   SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
@@ -797,7 +794,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
   return ReturnVal;
 }
 
-SDValue DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
+bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
     SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
   assert(!N->isStrictFPOpcode() && "strictfp not implemented");
   EVT VT = N->getValueType(0);
@@ -806,7 +803,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
          "expected both return values to have the same type");
 
   if (!TLI.getLibcallName(LC))
-    return SDValue();
+    return false;
 
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
 
@@ -852,17 +849,46 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
     SetSoftenedFloat(SDValue(N, ResNum), CreateStackLoad(SlackSlot));
   }
 
-  return SDValue();
+  return true;
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
-  return SoftenFloatRes_UnaryWithTwoFPResults(
-      N, RTLIB::getSINCOS(N->getValueType(0)));
+  EVT VT = N->getValueType(0);
+  if (SoftenFloatRes_UnaryWithTwoFPResults(N, RTLIB::getSINCOS(VT)))
+    return SDValue();
+
+  // Fall back on softening the separate sin and cos calls if available.
+  RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
+  RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
+
+  SDValue SoftSin, SoftCos;
+  if (!TLI.getLibcallName(SinLC) || !TLI.getLibcallName(CosLC)) {
+    DAG.getContext()->emitError("do not know how to soften fsincos");
+
+    EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+    SoftSin = SoftCos = DAG.getPOISON(NVT);
+  } else {
+    SoftSin = SoftenFloatRes_Unary(N, SinLC);
+    SoftCos = SoftenFloatRes_Unary(N, CosLC);
+  }
+
+  SetSoftenedFloat(SDValue(N, 0), SoftSin);
+  SetSoftenedFloat(SDValue(N, 1), SoftCos);
+  return SDValue();
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
-  return SoftenFloatRes_UnaryWithTwoFPResults(
-      N, RTLIB::getMODF(N->getValueType(0)), /*CallRetResNo=*/0);
+  EVT VT = N->getValueType(0);
+  if (SoftenFloatRes_UnaryWithTwoFPResults(N, RTLIB::getMODF(VT),
+                                           /*CallRetResNo=*/0))
+    return SDValue();
+
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+  DAG.getContext()->emitError("do not know how to soften fmodf");
+  SDValue Poison = DAG.getPOISON(NVT);
+  SetSoftenedFloat(SDValue(N, 0), Poison);
+  SetSoftenedFloat(SDValue(N, 1), Poison);
+  return SDValue();
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 8643ae9d78159..9b537248d4ab4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -564,7 +564,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   // Convert Float Results to Integer.
   void SoftenFloatResult(SDNode *N, unsigned ResNo);
   SDValue SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC);
-  SDValue SoftenFloatRes_UnaryWithTwoFPResults(
+  bool SoftenFloatRes_UnaryWithTwoFPResults(
       SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo = {});
   SDValue SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC);
   SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
index e0e8d503ca92a..0a449fd011e69 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
@@ -54,8 +54,8 @@ ResourcePriorityQueue::ResourcePriorityQueue(SelectionDAGISel *IS)
   unsigned NumRC = TRI->getNumRegClasses();
   RegLimit.resize(NumRC);
   RegPressure.resize(NumRC);
-  std::fill(RegLimit.begin(), RegLimit.end(), 0);
-  std::fill(RegPressure.begin(), RegPressure.end(), 0);
+  llvm::fill(RegLimit, 0);
+  llvm::fill(RegPressure, 0);
   for (const TargetRegisterClass *RC : TRI->regclasses())
     RegLimit[RC->getID()] = TRI->getRegPressureLimit(RC, *IS->MF);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 2c51a01044cdf..a570b71ecd28d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -1769,8 +1769,8 @@ class RegReductionPQBase : public SchedulingPriorityQueue {
       unsigned NumRC = TRI->getNumRegClasses();
       RegLimit.resize(NumRC);
       RegPressure.resize(NumRC);
-      std::fill(RegLimit.begin(), RegLimit.end(), 0);
-      std::fill(RegPressure.begin(), RegPressure.end(), 0);
+      llvm::fill(RegLimit, 0);
+      llvm::fill(RegPressure, 0);
       for (const TargetRegisterClass *RC : TRI->regclasses())
         RegLimit[RC->getID()] = tri->getRegPressureLimit(RC, MF);
     }
@@ -1793,7 +1793,7 @@ class RegReductionPQBase : public SchedulingPriorityQueue {
   void releaseState() override {
     SUnits = nullptr;
     SethiUllmanNumbers.clear();
-    std::fill(RegPressure.begin(), RegPressure.end(), 0);
+    llvm::fill(RegPressure, 0);
   }
 
   unsigned getNodePriority(const SUnit *SU) const;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index fe9a6ea3e77e6..c1356239ad206 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1370,7 +1370,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
                         const TargetLibraryInfo *LibraryInfo,
                         UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
                         BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
-                        FunctionVarLocs const *VarLocs) {
+                        FunctionVarLocs const *VarLocs, bool HasDivergency) {
   MF = &NewMF;
   SDAGISelPass = PassPtr;
   ORE = &NewORE;
@@ -1383,6 +1383,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
   BFI = BFIin;
   MMI = &MMIin;
   FnVarLocs = VarLocs;
+  DivergentTarget = HasDivergency;
 }
 
 SelectionDAG::~SelectionDAG() {
@@ -1457,8 +1458,8 @@ void SelectionDAG::clear() {
   TargetExternalSymbols.clear();
   MCSymbols.clear();
   SDEI.clear();
-  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), nullptr);
-  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), nullptr);
+  llvm::fill(CondCodeNodes, nullptr);
+  llvm::fill(ValueTypeNodes, nullptr);
 
   EntryNode.UseList = nullptr;
   InsertNode(&EntryNode);
@@ -2329,7 +2330,8 @@ SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
     return SDValue(E, 0);
 
   auto *N = newSDNode<RegisterSDNode>(Reg, VTs);
-  N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
+  N->SDNodeBits.IsDivergent =
+      DivergentTarget && TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
   CSEMap.InsertNode(N, IP);
   InsertNode(N);
   return SDValue(N, 0);
@@ -2519,8 +2521,11 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
   EVT VT = Node->getValueType(0);
   unsigned NumResults = Node->getNumValues();
 
+  if (LC == RTLIB::UNKNOWN_LIBCALL)
+    return false;
+
   const char *LCName = TLI->getLibcallName(LC);
-  if (!LC || !LCName)
+  if (!LCName)
     return false;
 
   auto getVecDesc = [&]() -> VecDesc const * {
@@ -5522,6 +5527,13 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
 
   unsigned Opcode = Op.getOpcode();
   switch (Opcode) {
+  case ISD::AssertSext:
+  case ISD::AssertZext:
+  case ISD::AssertAlign:
+  case ISD::AssertNoFPClass:
+    // Assertion nodes can create poison if the assertion fails.
+    return true;
+
   case ISD::FREEZE:
   case ISD::CONCAT_VECTORS:
   case ISD::INSERT_SUBVECTOR:
@@ -5543,6 +5555,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::FSHL:
   case ISD::FSHR:
   case ISD::BSWAP:
+  case ISD::CTTZ:
+  case ISD::CTLZ:
   case ISD::CTPOP:
   case ISD::BITREVERSE:
   case ISD::PARITY:
@@ -5557,6 +5571,23 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::SPLAT_VECTOR:
     return false;
 
+  case ISD::ADDC:
+  case ISD::SUBC:
+  case ISD::ADDE:
+  case ISD::SUBE:
+  case ISD::SADDO:
+  case ISD::SSUBO:
+  case ISD::SMULO:
+  case ISD::SADDO_CARRY:
+  case ISD::SSUBO_CARRY:
+  case ISD::UADDO:
+  case ISD::USUBO:
+  case ISD::UMULO:
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:
+    // No poison on result or overflow flags.
+    return false;
+
   case ISD::SELECT_CC:
   case ISD::SETCC: {
     // Integer setcc cannot create undef or poison.
@@ -5597,9 +5628,13 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::SRA:
     // If the max shift amount isn't in range, then the shift can
     // create poison.
-    return !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedElts,
-                                             PoisonOnly, Depth + 1) ||
-           !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);
+    return !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);
+
+  case ISD::CTTZ_ZERO_UNDEF:
+  case ISD::CTLZ_ZERO_UNDEF:
+    // If the amount is zero then the result will be poison.
+    // TODO: Add isKnownNeverZero DemandedElts handling.
+    return !isKnownNeverZero(Op.getOperand(0), Depth + 1);
 
   case ISD::SCALAR_TO_VECTOR:
     // Check if we demand any upper (undef) elements.
@@ -5610,12 +5645,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
     // Ensure that the element index is in bounds.
     EVT VecVT = Op.getOperand(0).getValueType();
     SDValue Idx = Op.getOperand(Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
-    if (isGuaranteedNotToBeUndefOrPoison(Idx, DemandedElts, PoisonOnly,
-                                         Depth + 1)) {
-      KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);
-      return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
-    }
-    return true;
+    KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);
+    return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
   }
 
   case ISD::VECTOR_SHUFFLE: {
@@ -7254,6 +7285,78 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
   return SDValue();
 }
 
+SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
+                                              const SDLoc &DL, EVT DstEltVT) {
+  EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
+
+  // If this is already the right type, we're done.
+  if (SrcEltVT == DstEltVT)
+    return SDValue(BV, 0);
+
+  unsigned SrcBitSize = SrcEltVT.getSizeInBits();
+  unsigned DstBitSize = DstEltVT.getSizeInBits();
+
+  // If this is a conversion of N elements of one type to N elements of another
+  // type, convert each element.  This handles FP<->INT cases.
+  if (SrcBitSize == DstBitSize) {
+    SmallVector<SDValue, 8> Ops;
+    for (SDValue Op : BV->op_values()) {
+      // If the vector element type is not legal, the BUILD_VECTOR operands
+      // are promoted and implicitly truncated.  Make that explicit here.
+      if (Op.getValueType() != SrcEltVT)
+        Op = getNode(ISD::TRUNCATE, DL, SrcEltVT, Op);
+      Ops.push_back(getBitcast(DstEltVT, Op));
+    }
+    EVT VT = EVT::getVectorVT(*getContext(), DstEltVT,
+                              BV->getValueType(0).getVectorNumElements());
+    return getBuildVector(VT, DL, Ops);
+  }
+
+  // Otherwise, we're growing or shrinking the elements.  To avoid having to
+  // handle annoying details of growing/shrinking FP values, we convert them to
+  // int first.
+  if (SrcEltVT.isFloatingPoint()) {
+    // Convert the input float vector to a int vector where the elements are the
+    // same sizes.
+    EVT IntEltVT = EVT::getIntegerVT(*getContext(), SrcEltVT.getSizeInBits());
+    if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))
+      return FoldConstantBuildVector(cast<BuildVectorSDNode>(Tmp), DL,
+                                     DstEltVT);
+    return SDValue();
+  }
+
+  // Now we know the input is an integer vector.  If the output is a FP type,
+  // convert to integer first, then to FP of the right size.
+  if (DstEltVT.isFloatingPoint()) {
+    EVT IntEltVT = EVT::getIntegerVT(*getContext(), DstEltVT.getSizeInBits());
+    if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))
+      return FoldConstantBuildVector(cast<BuildVectorSDNode>(Tmp), DL,
+                                     DstEltVT);
+    return SDValue();
+  }
+
+  // Okay, we know the src/dst types are both integers of differing types.
+  assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
+
+  // Extract the constant raw bit data.
+  BitVector UndefElements;
+  SmallVector<APInt> RawBits;
+  bool IsLE = getDataLayout().isLittleEndian();
+  if (!BV->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
+    return SDValue();
+
+  SmallVector<SDValue, 8> Ops;
+  for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
+    if (UndefElements[I])
+      Ops.push_back(getUNDEF(DstEltVT));
+    else
+      Ops.push_back(getConstant(RawBits[I], DL, DstEltVT));
+  }
+
+  EVT VT = EVT::getVectorVT(*getContext(), DstEltVT, Ops.size());
+  return getBuildVector(VT, DL, Ops);
+}
+
 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
   assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
 
@@ -12041,6 +12144,8 @@ static bool gluePropagatesDivergence(const SDNode *Node) {
 }
 
 bool SelectionDAG::calculateDivergence(SDNode *N) {
+  if (!DivergentTarget)
+    return false;
   if (TLI->isSDNodeAlwaysUniform(N)) {
     assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
            "Conflicting divergence information!");
@@ -12060,6 +12165,8 @@ bool SelectionDAG::calculateDivergence(SDNode *N) {
 }
 
 void SelectionDAG::updateDivergence(SDNode *N) {
+  if (!DivergentTarget)
+    return;
   SmallVector<SDNode *, 16> Worklist(1, N);
   do {
     N = Worklist.pop_back_val();
@@ -12543,6 +12650,18 @@ bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
   return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
 }
 
+bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
+  ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
+  return C && APInt::isSameValue(C->getAPIntValue(),
+                                 APInt(C->getAPIntValue().getBitWidth(), 1));
+}
+
+bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
+  N = peekThroughBitcasts(N);
+  ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, true);
+  return C && C->isZero();
+}
+
 HandleSDNode::~HandleSDNode() {
   DropOperands();
 }
@@ -13607,16 +13726,20 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
     Ops[I].setInitial(Vals[I]);
     EVT VT = Ops[I].getValueType();
 
+    // Take care of the Node's operands iff target has divergence
     // Skip Chain. It does not carry divergence.
-    if (VT != MVT::Other &&
+    if (DivergentTarget && VT != MVT::Other &&
         (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
         Ops[I].getNode()->isDivergent()) {
+      // Node is going to be divergent if at least one of its operand is
+      // divergent, unless it belongs to the "AlwaysUniform" exemptions.
       IsDivergent = true;
     }
   }
   Node->NumOperands = Vals.size();
   Node->OperandList = Ops;
-  if (!TLI->isSDNodeAlwaysUniform(Node)) {
+  // Check the divergence of the Node itself.
+  if (DivergentTarget && !TLI->isSDNodeAlwaysUniform(Node)) {
     IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
     Node->SDNodeBits.IsDivergent = IsDivergent;
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 04d6fd5f48cc3..ecd1ff87e7fbc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4336,19 +4336,13 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
   auto &TLI = DAG.getTargetLoweringInfo();
   GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
 
-  // Normalize Vector GEP - all scalar operands should be converted to the
-  // splat vector.
+  // For a vector GEP, keep the prefix scalar as long as possible, then
+  // convert any scalars encountered after the first vector operand to vectors.
   bool IsVectorGEP = I.getType()->isVectorTy();
   ElementCount VectorElementCount =
       IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
                   : ElementCount::getFixed(0);
 
-  if (IsVectorGEP && !N.getValueType().isVector()) {
-    LLVMContext &Context = *DAG.getContext();
-    EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
-    N = DAG.getSplat(VT, dl, N);
-  }
-
   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
        GTI != E; ++GTI) {
     const Value *Idx = GTI.getOperand();
@@ -4396,7 +4390,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
         APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
         LLVMContext &Context = *DAG.getContext();
         SDValue OffsVal;
-        if (IsVectorGEP)
+        if (N.getValueType().isVector())
           OffsVal = DAG.getConstant(
               Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
         else
@@ -4418,10 +4412,16 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
       // N = N + Idx * ElementMul;
       SDValue IdxN = getValue(Idx);
 
-      if (!IdxN.getValueType().isVector() && IsVectorGEP) {
-        EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
-                                  VectorElementCount);
-        IdxN = DAG.getSplat(VT, dl, IdxN);
+      if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
+        if (N.getValueType().isVector()) {
+          EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
+                                    VectorElementCount);
+          IdxN = DAG.getSplat(VT, dl, IdxN);
+        } else {
+          EVT VT =
+              EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
+          N = DAG.getSplat(VT, dl, N);
+        }
       }
 
       // If the index is smaller or larger than intptr_t, truncate or extend
@@ -4442,7 +4442,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
         SDValue VScale = DAG.getNode(
             ISD::VSCALE, dl, VScaleTy,
             DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
-        if (IsVectorGEP)
+        if (N.getValueType().isVector())
           VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
         IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
                            ScaleFlags);
@@ -4475,6 +4475,11 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
     }
   }
 
+  if (IsVectorGEP && !N.getValueType().isVector()) {
+    EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
+    N = DAG.getSplat(VT, dl, N);
+  }
+
   MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
   MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
   if (IsVectorGEP) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index d9b9cf6bcc772..6260ad2f24dea 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -482,7 +482,10 @@ void SelectionDAGISel::initializeAnalysisResults(
   MachineModuleInfo &MMI =
       MAMP.getCachedResult<MachineModuleAnalysis>(*Fn.getParent())->getMMI();
 
-  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
+  TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
+
+  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
+               TTI->hasBranchDivergence(&Fn));
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -500,10 +503,6 @@ void SelectionDAGISel::initializeAnalysisResults(
     BatchAA = std::nullopt;
 
   SP = &FAM.getResult<SSPLayoutAnalysis>(Fn);
-
-#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
-  TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
-#endif
 }
 
 void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
@@ -539,7 +538,10 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
   MachineModuleInfo &MMI =
       MFP.getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
 
-  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
+  TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
+
+  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
+               TTI->hasBranchDivergence(&Fn));
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -558,10 +560,6 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
     BatchAA = std::nullopt;
 
   SP = &MFP.getAnalysis<StackProtector>().getLayoutInfo();
-
-#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
-  TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
-#endif
 }
 
 bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 000f8cc6786a5..45ab7526c3a32 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -4247,7 +4247,7 @@ SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
   // (X | Y) == Y
   // (X | Y) != Y
   SDValue X;
-  if (sd_match(N0, m_Or(m_Value(X), m_Specific(N1))) && hasAndNotCompare(N1)) {
+  if (sd_match(N0, m_Or(m_Value(X), m_Specific(N1))) && hasAndNotCompare(X)) {
     // If the target supports an 'and-not' or 'and-complement' logic operation,
     // try to use that to make a comparison operation more efficient.
 
@@ -7412,17 +7412,6 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
   return Blended;
 }
 
-bool TargetLowering::
-verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
-  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
-    DAG.getContext()->emitError("argument to '__builtin_return_address' must "
-                                "be a constant integer");
-    return true;
-  }
-
-  return false;
-}
-
 SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
                                          const DenormalMode &Mode) const {
   SDLoc DL(Op);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index cda41a91a372f..85b148a466e52 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -396,6 +396,10 @@ RTLIB::Libcall RTLIB::getPOWI(EVT RetVT) {
                       POWI_PPCF128);
 }
 
+RTLIB::Libcall RTLIB::getPOW(EVT RetVT) {
+  return getFPLibCall(RetVT, POW_F32, POW_F64, POW_F80, POW_F128, POW_PPCF128);
+}
+
 RTLIB::Libcall RTLIB::getLDEXP(EVT RetVT) {
   return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,
                       LDEXP_PPCF128);
@@ -406,6 +410,14 @@ RTLIB::Libcall RTLIB::getFREXP(EVT RetVT) {
                       FREXP_PPCF128);
 }
 
+RTLIB::Libcall RTLIB::getSIN(EVT RetVT) {
+  return getFPLibCall(RetVT, SIN_F32, SIN_F64, SIN_F80, SIN_F128, SIN_PPCF128);
+}
+
+RTLIB::Libcall RTLIB::getCOS(EVT RetVT) {
+  return getFPLibCall(RetVT, COS_F32, COS_F64, COS_F80, COS_F128, COS_PPCF128);
+}
+
 RTLIB::Libcall RTLIB::getSINCOS(EVT RetVT) {
   return getFPLibCall(RetVT, SINCOS_F32, SINCOS_F64, SINCOS_F80, SINCOS_F128,
                       SINCOS_PPCF128);
@@ -681,9 +693,8 @@ void TargetLoweringBase::initActions() {
   memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
   memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
   memset(CondCodeActions, 0, sizeof(CondCodeActions));
-  std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
-  std::fill(std::begin(TargetDAGCombineArray),
-            std::end(TargetDAGCombineArray), 0);
+  llvm::fill(RegClassForVT, nullptr);
+  llvm::fill(TargetDAGCombineArray, 0);
 
   // Let extending atomic loads be unsupported by default.
   for (MVT ValVT : MVT::all_valuetypes())
@@ -872,6 +883,10 @@ void TargetLoweringBase::initActions() {
                       ISD::FATAN2},
                      {MVT::f32, MVT::f64, MVT::f128}, Expand);
 
+  // Insert custom handling default for llvm.canonicalize.*.
+  setOperationAction(ISD::FCANONICALIZE,
+                     {MVT::f16, MVT::f32, MVT::f64, MVT::f128}, Expand);
+
   // FIXME: Query RuntimeLibCalls to make the decision.
   setOperationAction({ISD::LRINT, ISD::LLRINT, ISD::LROUND, ISD::LLROUND},
                      {MVT::f32, MVT::f64, MVT::f128}, LibCall);
diff --git a/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
index def325efd5445..5eb86e740ff7c 100644
--- a/llvm/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
@@ -39,12 +39,11 @@ bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
 
 bool TargetOptions::FramePointerIsReserved(const MachineFunction &MF) const {
   const Function &F = MF.getFunction();
-
-  if (!F.hasFnAttribute("frame-pointer"))
+  Attribute FPAttr = F.getFnAttribute("frame-pointer");
+  if (!FPAttr.isValid())
     return false;
 
-  StringRef FP = F.getFnAttribute("frame-pointer").getValueAsString();
-  return StringSwitch<bool>(FP)
+  return StringSwitch<bool>(FPAttr.getValueAsString())
       .Cases("all", "non-leaf", "reserved", true)
       .Case("none", false);
 }
diff --git a/llvm/lib/DWARFLinker/Classic/CMakeLists.txt b/llvm/lib/DWARFLinker/Classic/CMakeLists.txt
index 5d3b259e4a33a..4e5cb3c6dfb51 100644
--- a/llvm/lib/DWARFLinker/Classic/CMakeLists.txt
+++ b/llvm/lib/DWARFLinker/Classic/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_component_library(LLVMDWARFLinkerClassic
   CodeGen
   CodeGenTypes
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   DWARFLinker
   MC
   Object
diff --git a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp
index ae4cc6d85c120..222dc88098102 100644
--- a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp
@@ -23,10 +23,10 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugMacro.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFSection.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Error.h"
diff --git a/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp b/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp
index 66bf158e60f1d..027363aa394e0 100644
--- a/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp
+++ b/llvm/lib/DWARFLinker/Classic/DWARFLinkerCompileUnit.cpp
@@ -10,7 +10,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/DWARFLinker/Classic/DWARFLinkerDeclContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/FormatVariadic.h"
 
 namespace llvm {
diff --git a/llvm/lib/DWARFLinker/Parallel/CMakeLists.txt b/llvm/lib/DWARFLinker/Parallel/CMakeLists.txt
index 1e7ff0703d135..29d8028b33fb4 100644
--- a/llvm/lib/DWARFLinker/Parallel/CMakeLists.txt
+++ b/llvm/lib/DWARFLinker/Parallel/CMakeLists.txt
@@ -25,6 +25,7 @@ add_llvm_component_library(LLVMDWARFLinkerParallel
   BinaryFormat
   CodeGen
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   DWARFLinker
   MC
   Object
diff --git a/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp b/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
index d291be6c64397..8d065bf793f35 100644
--- a/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
+++ b/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
@@ -101,10 +101,8 @@ void CompileUnit::maybeResetToLoadedStage() {
   OutUnitDIE = nullptr;
   DebugAddrIndexMap.clear();
 
-  for (uint64_t &Offset : OutDieOffsetArray)
-    Offset = 0;
-  for (TypeEntry *&Name : TypeEntries)
-    Name = nullptr;
+  llvm::fill(OutDieOffsetArray, 0);
+  llvm::fill(TypeEntries, nullptr);
   eraseSections();
 
   setStage(Stage::CreatedNotLoaded);
diff --git a/llvm/lib/DebugInfo/DWARF/CMakeLists.txt b/llvm/lib/DebugInfo/DWARF/CMakeLists.txt
index 86e74110b15ea..3755be256a305 100644
--- a/llvm/lib/DebugInfo/DWARF/CMakeLists.txt
+++ b/llvm/lib/DebugInfo/DWARF/CMakeLists.txt
@@ -1,9 +1,10 @@
+add_subdirectory(LowLevel)
+
 add_llvm_component_library(LLVMDebugInfoDWARF
   DWARFAbbreviationDeclaration.cpp
   DWARFAddressRange.cpp
   DWARFAcceleratorTable.cpp
   DWARFCFIPrinter.cpp
-  DWARFCFIProgram.cpp
   DWARFCompileUnit.cpp
   DWARFContext.cpp
   DWARFDebugAbbrev.cpp
@@ -19,7 +20,7 @@ add_llvm_component_library(LLVMDebugInfoDWARF
   DWARFDebugRangeList.cpp
   DWARFDebugRnglists.cpp
   DWARFDie.cpp
-  DWARFExpression.cpp
+  DWARFExpressionPrinter.cpp
   DWARFFormValue.cpp
   DWARFGdbIndex.cpp
   DWARFListTable.cpp
@@ -27,6 +28,7 @@ add_llvm_component_library(LLVMDebugInfoDWARF
   DWARFTypeUnit.cpp
   DWARFUnitIndex.cpp
   DWARFUnit.cpp
+  DWARFUnwindTablePrinter.cpp
   DWARFVerifier.cpp
 
   ADDITIONAL_HEADER_DIRS
@@ -35,6 +37,7 @@ add_llvm_component_library(LLVMDebugInfoDWARF
 
   LINK_COMPONENTS
   BinaryFormat
+  DebugInfoDWARFLowLevel
   Object
   Support
   TargetParser
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp b/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp
index e52f671e4fa1c..2abab0277a9f6 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp
@@ -8,8 +8,9 @@
 
 #include "llvm/DebugInfo/DWARF/DWARFCFIPrinter.h"
 #include "llvm/DebugInfo/DIContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Errc.h"
@@ -100,8 +101,7 @@ static void printOperand(raw_ostream &OS, const DIDumpOptions &DumpOpts,
   case CFIProgram::OT_Expression:
     assert(Instr.Expression && "missing DWARFExpression object");
     OS << " ";
-    DWARFExpressionPrinter::print(&Instr.Expression.value(), OS, DumpOpts,
-                                  nullptr);
+    printDwarfExpression(&Instr.Expression.value(), OS, DumpOpts, nullptr);
     break;
   }
 }
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
index 9184502adc7cf..6610eef54801b 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
@@ -13,9 +13,11 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFCFIPrinter.h"
-#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/DWARFUnwindTablePrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Errc.h"
@@ -31,180 +33,6 @@
 using namespace llvm;
 using namespace dwarf;
 
-static void printRegister(raw_ostream &OS, DIDumpOptions DumpOpts,
-                          unsigned RegNum) {
-  if (DumpOpts.GetNameForDWARFReg) {
-    auto RegName = DumpOpts.GetNameForDWARFReg(RegNum, DumpOpts.IsEH);
-    if (!RegName.empty()) {
-      OS << RegName;
-      return;
-    }
-  }
-  OS << "reg" << RegNum;
-}
-
-UnwindLocation UnwindLocation::createUnspecified() { return {Unspecified}; }
-
-UnwindLocation UnwindLocation::createUndefined() { return {Undefined}; }
-
-UnwindLocation UnwindLocation::createSame() { return {Same}; }
-
-UnwindLocation UnwindLocation::createIsConstant(int32_t Value) {
-  return {Constant, InvalidRegisterNumber, Value, std::nullopt, false};
-}
-
-UnwindLocation UnwindLocation::createIsCFAPlusOffset(int32_t Offset) {
-  return {CFAPlusOffset, InvalidRegisterNumber, Offset, std::nullopt, false};
-}
-
-UnwindLocation UnwindLocation::createAtCFAPlusOffset(int32_t Offset) {
-  return {CFAPlusOffset, InvalidRegisterNumber, Offset, std::nullopt, true};
-}
-
-UnwindLocation
-UnwindLocation::createIsRegisterPlusOffset(uint32_t RegNum, int32_t Offset,
-                                           std::optional<uint32_t> AddrSpace) {
-  return {RegPlusOffset, RegNum, Offset, AddrSpace, false};
-}
-
-UnwindLocation
-UnwindLocation::createAtRegisterPlusOffset(uint32_t RegNum, int32_t Offset,
-                                           std::optional<uint32_t> AddrSpace) {
-  return {RegPlusOffset, RegNum, Offset, AddrSpace, true};
-}
-
-UnwindLocation UnwindLocation::createIsDWARFExpression(DWARFExpression Expr) {
-  return {Expr, false};
-}
-
-UnwindLocation UnwindLocation::createAtDWARFExpression(DWARFExpression Expr) {
-  return {Expr, true};
-}
-
-void UnwindLocation::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
-  if (Dereference)
-    OS << '[';
-  switch (Kind) {
-  case Unspecified:
-    OS << "unspecified";
-    break;
-  case Undefined:
-    OS << "undefined";
-    break;
-  case Same:
-    OS << "same";
-    break;
-  case CFAPlusOffset:
-    OS << "CFA";
-    if (Offset == 0)
-      break;
-    if (Offset > 0)
-      OS << "+";
-    OS << Offset;
-    break;
-  case RegPlusOffset:
-    printRegister(OS, DumpOpts, RegNum);
-    if (Offset == 0 && !AddrSpace)
-      break;
-    if (Offset >= 0)
-      OS << "+";
-    OS << Offset;
-    if (AddrSpace)
-      OS << " in addrspace" << *AddrSpace;
-    break;
-  case DWARFExpr: {
-    if (Expr)
-      DWARFExpressionPrinter::print(&(*Expr), OS, DumpOpts, nullptr);
-    break;
-  }
-  case Constant:
-    OS << Offset;
-    break;
-  }
-  if (Dereference)
-    OS << ']';
-}
-
-raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS,
-                                     const UnwindLocation &UL) {
-  auto DumpOpts = DIDumpOptions();
-  UL.dump(OS, DumpOpts);
-  return OS;
-}
-
-bool UnwindLocation::operator==(const UnwindLocation &RHS) const {
-  if (Kind != RHS.Kind)
-    return false;
-  switch (Kind) {
-  case Unspecified:
-  case Undefined:
-  case Same:
-    return true;
-  case CFAPlusOffset:
-    return Offset == RHS.Offset && Dereference == RHS.Dereference;
-  case RegPlusOffset:
-    return RegNum == RHS.RegNum && Offset == RHS.Offset &&
-           Dereference == RHS.Dereference;
-  case DWARFExpr:
-    return *Expr == *RHS.Expr && Dereference == RHS.Dereference;
-  case Constant:
-    return Offset == RHS.Offset;
-  }
-  return false;
-}
-
-void RegisterLocations::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
-  bool First = true;
-  for (const auto &RegLocPair : Locations) {
-    if (First)
-      First = false;
-    else
-      OS << ", ";
-    printRegister(OS, DumpOpts, RegLocPair.first);
-    OS << '=';
-    RegLocPair.second.dump(OS, DumpOpts);
-  }
-}
-
-raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS,
-                                     const RegisterLocations &RL) {
-  auto DumpOpts = DIDumpOptions();
-  RL.dump(OS, DumpOpts);
-  return OS;
-}
-
-void UnwindRow::dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-                     unsigned IndentLevel) const {
-  OS.indent(2 * IndentLevel);
-  if (hasAddress())
-    OS << format("0x%" PRIx64 ": ", *Address);
-  OS << "CFA=";
-  CFAValue.dump(OS, DumpOpts);
-  if (RegLocs.hasLocations()) {
-    OS << ": ";
-    RegLocs.dump(OS, DumpOpts);
-  }
-  OS << "\n";
-}
-
-raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS, const UnwindRow &Row) {
-  auto DumpOpts = DIDumpOptions();
-  Row.dump(OS, DumpOpts, 0);
-  return OS;
-}
-
-void UnwindTable::dump(raw_ostream &OS, DIDumpOptions DumpOpts,
-                       unsigned IndentLevel) const {
-  for (const UnwindRow &Row : Rows)
-    Row.dump(OS, DumpOpts, IndentLevel);
-}
-
-raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS, const UnwindTable &Rows) {
-  auto DumpOpts = DIDumpOptions();
-  Rows.dump(OS, DumpOpts, 0);
-  return OS;
-}
-
 Expected<UnwindTable> llvm::dwarf::createUnwindTable(const FDE *Fde) {
   const CIE *Cie = Fde->getLinkedCIE();
   if (Cie == nullptr)
@@ -258,316 +86,6 @@ Expected<UnwindTable> llvm::dwarf::createUnwindTable(const CIE *Cie) {
   return UnwindTable(std::move(Rows));
 }
 
-Expected<UnwindTable::RowContainer>
-llvm::dwarf::parseRows(const CFIProgram &CFIP, UnwindRow &Row,
-                       const RegisterLocations *InitialLocs) {
-  // All the unwinding rows parsed during processing of the CFI program.
-  UnwindTable::RowContainer Rows;
-
-  // State consists of CFA value and register locations.
-  std::vector<std::pair<UnwindLocation, RegisterLocations>> States;
-  for (const CFIProgram::Instruction &Inst : CFIP) {
-    switch (Inst.Opcode) {
-    case dwarf::DW_CFA_set_loc: {
-      // The DW_CFA_set_loc instruction takes a single operand that
-      // represents a target address. The required action is to create a new
-      // table row using the specified address as the location. All other
-      // values in the new row are initially identical to the current row.
-      // The new location value is always greater than the current one. If
-      // the segment_size field of this FDE's CIE is non- zero, the initial
-      // location is preceded by a segment selector of the given length
-      llvm::Expected<uint64_t> NewAddress = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!NewAddress)
-        return NewAddress.takeError();
-      if (*NewAddress <= Row.getAddress())
-        return createStringError(
-            errc::invalid_argument,
-            "%s with adrress 0x%" PRIx64 " which must be greater than the "
-            "current row address 0x%" PRIx64,
-            CFIP.callFrameString(Inst.Opcode).str().c_str(), *NewAddress,
-            Row.getAddress());
-      Rows.push_back(Row);
-      Row.setAddress(*NewAddress);
-      break;
-    }
-
-    case dwarf::DW_CFA_advance_loc:
-    case dwarf::DW_CFA_advance_loc1:
-    case dwarf::DW_CFA_advance_loc2:
-    case dwarf::DW_CFA_advance_loc4: {
-      // The DW_CFA_advance instruction takes a single operand that
-      // represents a constant delta. The required action is to create a new
-      // table row with a location value that is computed by taking the
-      // current entry’s location value and adding the value of delta *
-      // code_alignment_factor. All other values in the new row are initially
-      // identical to the current row.
-      Rows.push_back(Row);
-      llvm::Expected<uint64_t> Offset = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!Offset)
-        return Offset.takeError();
-      Row.slideAddress(*Offset);
-      break;
-    }
-
-    case dwarf::DW_CFA_restore:
-    case dwarf::DW_CFA_restore_extended: {
-      // The DW_CFA_restore instruction takes a single operand (encoded with
-      // the opcode) that represents a register number. The required action
-      // is to change the rule for the indicated register to the rule
-      // assigned it by the initial_instructions in the CIE.
-      if (InitialLocs == nullptr)
-        return createStringError(
-            errc::invalid_argument, "%s encountered while parsing a CIE",
-            CFIP.callFrameString(Inst.Opcode).str().c_str());
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      if (std::optional<UnwindLocation> O =
-              InitialLocs->getRegisterLocation(*RegNum))
-        Row.getRegisterLocations().setRegisterLocation(*RegNum, *O);
-      else
-        Row.getRegisterLocations().removeRegisterLocation(*RegNum);
-      break;
-    }
-
-    case dwarf::DW_CFA_offset:
-    case dwarf::DW_CFA_offset_extended:
-    case dwarf::DW_CFA_offset_extended_sf: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
-      if (!Offset)
-        return Offset.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createAtCFAPlusOffset(*Offset));
-      break;
-    }
-
-    case dwarf::DW_CFA_nop:
-      break;
-
-    case dwarf::DW_CFA_remember_state:
-      States.push_back(
-          std::make_pair(Row.getCFAValue(), Row.getRegisterLocations()));
-      break;
-
-    case dwarf::DW_CFA_restore_state:
-      if (States.empty())
-        return createStringError(errc::invalid_argument,
-                                 "DW_CFA_restore_state without a matching "
-                                 "previous DW_CFA_remember_state");
-      Row.getCFAValue() = States.back().first;
-      Row.getRegisterLocations() = States.back().second;
-      States.pop_back();
-      break;
-
-    case dwarf::DW_CFA_GNU_window_save:
-      switch (CFIP.triple()) {
-      case Triple::aarch64:
-      case Triple::aarch64_be:
-      case Triple::aarch64_32: {
-        // DW_CFA_GNU_window_save is used for different things on different
-        // architectures. For aarch64 it is known as
-        // DW_CFA_AARCH64_negate_ra_state. The action is to toggle the
-        // value of the return address state between 1 and 0. If there is
-        // no rule for the AARCH64_DWARF_PAUTH_RA_STATE register, then it
-        // should be initially set to 1.
-        constexpr uint32_t AArch64DWARFPAuthRaState = 34;
-        auto LRLoc = Row.getRegisterLocations().getRegisterLocation(
-            AArch64DWARFPAuthRaState);
-        if (LRLoc) {
-          if (LRLoc->getLocation() == UnwindLocation::Constant) {
-            // Toggle the constant value from 0 to 1 or 1 to 0.
-            LRLoc->setConstant(LRLoc->getConstant() ^ 1);
-            Row.getRegisterLocations().setRegisterLocation(
-                AArch64DWARFPAuthRaState, *LRLoc);
-          } else {
-            return createStringError(
-                errc::invalid_argument,
-                "%s encountered when existing rule for this register is not "
-                "a constant",
-                CFIP.callFrameString(Inst.Opcode).str().c_str());
-          }
-        } else {
-          Row.getRegisterLocations().setRegisterLocation(
-              AArch64DWARFPAuthRaState, UnwindLocation::createIsConstant(1));
-        }
-        break;
-      }
-
-      case Triple::sparc:
-      case Triple::sparcv9:
-      case Triple::sparcel:
-        for (uint32_t RegNum = 16; RegNum < 32; ++RegNum) {
-          Row.getRegisterLocations().setRegisterLocation(
-              RegNum, UnwindLocation::createAtCFAPlusOffset((RegNum - 16) * 8));
-        }
-        break;
-
-      default: {
-        return createStringError(
-            errc::not_supported,
-            "DW_CFA opcode %#x is not supported for architecture %s",
-            Inst.Opcode, Triple::getArchTypeName(CFIP.triple()).str().c_str());
-
-        break;
-      }
-      }
-      break;
-
-    case dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc: {
-      constexpr uint32_t AArch64DWARFPAuthRaState = 34;
-      auto LRLoc = Row.getRegisterLocations().getRegisterLocation(
-          AArch64DWARFPAuthRaState);
-      if (LRLoc) {
-        if (LRLoc->getLocation() == UnwindLocation::Constant) {
-          // Toggle the constant value of bits[1:0] from 0 to 1 or 1 to 0.
-          LRLoc->setConstant(LRLoc->getConstant() ^ 0x3);
-        } else {
-          return createStringError(
-              errc::invalid_argument,
-              "%s encountered when existing rule for this register is not "
-              "a constant",
-              CFIP.callFrameString(Inst.Opcode).str().c_str());
-        }
-      } else {
-        Row.getRegisterLocations().setRegisterLocation(
-            AArch64DWARFPAuthRaState, UnwindLocation::createIsConstant(0x3));
-      }
-      break;
-    }
-
-    case dwarf::DW_CFA_undefined: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createUndefined());
-      break;
-    }
-
-    case dwarf::DW_CFA_same_value: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createSame());
-      break;
-    }
-
-    case dwarf::DW_CFA_GNU_args_size:
-      break;
-
-    case dwarf::DW_CFA_register: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      llvm::Expected<uint64_t> NewRegNum = Inst.getOperandAsUnsigned(CFIP, 1);
-      if (!NewRegNum)
-        return NewRegNum.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createIsRegisterPlusOffset(*NewRegNum, 0));
-      break;
-    }
-
-    case dwarf::DW_CFA_val_offset:
-    case dwarf::DW_CFA_val_offset_sf: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
-      if (!Offset)
-        return Offset.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createIsCFAPlusOffset(*Offset));
-      break;
-    }
-
-    case dwarf::DW_CFA_expression: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createAtDWARFExpression(*Inst.Expression));
-      break;
-    }
-
-    case dwarf::DW_CFA_val_expression: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      Row.getRegisterLocations().setRegisterLocation(
-          *RegNum, UnwindLocation::createIsDWARFExpression(*Inst.Expression));
-      break;
-    }
-
-    case dwarf::DW_CFA_def_cfa_register: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      if (Row.getCFAValue().getLocation() != UnwindLocation::RegPlusOffset)
-        Row.getCFAValue() =
-            UnwindLocation::createIsRegisterPlusOffset(*RegNum, 0);
-      else
-        Row.getCFAValue().setRegister(*RegNum);
-      break;
-    }
-
-    case dwarf::DW_CFA_def_cfa_offset:
-    case dwarf::DW_CFA_def_cfa_offset_sf: {
-      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 0);
-      if (!Offset)
-        return Offset.takeError();
-      if (Row.getCFAValue().getLocation() != UnwindLocation::RegPlusOffset) {
-        return createStringError(
-            errc::invalid_argument,
-            "%s found when CFA rule was not RegPlusOffset",
-            CFIP.callFrameString(Inst.Opcode).str().c_str());
-      }
-      Row.getCFAValue().setOffset(*Offset);
-      break;
-    }
-
-    case dwarf::DW_CFA_def_cfa:
-    case dwarf::DW_CFA_def_cfa_sf: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
-      if (!Offset)
-        return Offset.takeError();
-      Row.getCFAValue() =
-          UnwindLocation::createIsRegisterPlusOffset(*RegNum, *Offset);
-      break;
-    }
-
-    case dwarf::DW_CFA_LLVM_def_aspace_cfa:
-    case dwarf::DW_CFA_LLVM_def_aspace_cfa_sf: {
-      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
-      if (!RegNum)
-        return RegNum.takeError();
-      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
-      if (!Offset)
-        return Offset.takeError();
-      llvm::Expected<uint32_t> CFAAddrSpace =
-          Inst.getOperandAsUnsigned(CFIP, 2);
-      if (!CFAAddrSpace)
-        return CFAAddrSpace.takeError();
-      Row.getCFAValue() = UnwindLocation::createIsRegisterPlusOffset(
-          *RegNum, *Offset, *CFAAddrSpace);
-      break;
-    }
-
-    case dwarf::DW_CFA_def_cfa_expression:
-      Row.getCFAValue() =
-          UnwindLocation::createIsDWARFExpression(*Inst.Expression);
-      break;
-    }
-  }
-  return Rows;
-}
-
 // Returns the CIE identifier to be used by the requested format.
 // CIE ids for .debug_frame sections are defined in Section 7.24 of DWARFv5.
 // For CIE ID in .eh_frame sections see
@@ -619,7 +137,7 @@ void CIE::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
   OS << "\n";
 
   if (Expected<UnwindTable> RowsOrErr = createUnwindTable(this))
-    RowsOrErr->dump(OS, DumpOpts, 1);
+    printUnwindTable(*RowsOrErr, OS, DumpOpts, 1);
   else {
     DumpOpts.RecoverableErrorHandler(joinErrors(
         createStringError(errc::invalid_argument,
@@ -647,7 +165,7 @@ void FDE::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
   OS << "\n";
 
   if (Expected<UnwindTable> RowsOrErr = createUnwindTable(this))
-    RowsOrErr->dump(OS, DumpOpts, 1);
+    printUnwindTable(*RowsOrErr, OS, DumpOpts, 1);
   else {
     DumpOpts.RecoverableErrorHandler(joinErrors(
         createStringError(errc::invalid_argument,
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
index fc71be32fdd79..5bd2427da7d73 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
@@ -11,10 +11,11 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -117,7 +118,7 @@ static void dumpExpression(raw_ostream &OS, DIDumpOptions DumpOpts,
   if (U)
     Format = U->getFormat();
   DWARFExpression E(Extractor, AddressSize, Format);
-  DWARFExpressionPrinter::print(&E, OS, DumpOpts, U);
+  printDwarfExpression(&E, OS, DumpOpts, U);
 }
 
 bool DWARFLocationTable::dumpLocationList(
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
index 08dd9d30812d1..edc69a36cdff2 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
@@ -15,11 +15,12 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypePrinter.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Format.h"
@@ -99,7 +100,7 @@ static void dumpLocationExpr(raw_ostream &OS, const DWARFFormValue &FormValue,
   DataExtractor Data(StringRef((const char *)Expr.data(), Expr.size()),
                      Ctx.isLittleEndian(), 0);
   DWARFExpression DE(Data, U->getAddressByteSize(), U->getFormParams().Format);
-  DWARFExpressionPrinter::print(&DE, OS, DumpOpts, U);
+  printDwarfExpression(&DE, OS, DumpOpts, U);
 }
 
 static DWARFDie resolveReferencedType(DWARFDie D, DWARFFormValue F) {
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
deleted file mode 100644
index 8255e013a2d79..0000000000000
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ /dev/null
@@ -1,538 +0,0 @@
-//===-- DWARFExpression.cpp -----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/DebugInfo/DWARF/DWARFUnit.h"
-#include "llvm/Support/Format.h"
-#include <cassert>
-#include <cstdint>
-#include <vector>
-
-using namespace llvm;
-using namespace dwarf;
-
-namespace llvm {
-
-typedef DWARFExpression::Operation Op;
-typedef Op::Description Desc;
-
-static std::vector<Desc> getOpDescriptions() {
-  std::vector<Desc> Descriptions;
-  Descriptions.resize(0xff);
-  Descriptions[DW_OP_addr] = Desc(Op::Dwarf2, Op::SizeAddr);
-  Descriptions[DW_OP_deref] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_const1u] = Desc(Op::Dwarf2, Op::Size1);
-  Descriptions[DW_OP_const1s] = Desc(Op::Dwarf2, Op::SignedSize1);
-  Descriptions[DW_OP_const2u] = Desc(Op::Dwarf2, Op::Size2);
-  Descriptions[DW_OP_const2s] = Desc(Op::Dwarf2, Op::SignedSize2);
-  Descriptions[DW_OP_const4u] = Desc(Op::Dwarf2, Op::Size4);
-  Descriptions[DW_OP_const4s] = Desc(Op::Dwarf2, Op::SignedSize4);
-  Descriptions[DW_OP_const8u] = Desc(Op::Dwarf2, Op::Size8);
-  Descriptions[DW_OP_const8s] = Desc(Op::Dwarf2, Op::SignedSize8);
-  Descriptions[DW_OP_constu] = Desc(Op::Dwarf2, Op::SizeLEB);
-  Descriptions[DW_OP_consts] = Desc(Op::Dwarf2, Op::SignedSizeLEB);
-  Descriptions[DW_OP_dup] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_drop] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_over] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_pick] = Desc(Op::Dwarf2, Op::Size1);
-  Descriptions[DW_OP_swap] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_rot] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_xderef] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_abs] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_and] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_div] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_minus] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_mod] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_mul] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_neg] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_not] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_or] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_plus] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_plus_uconst] = Desc(Op::Dwarf2, Op::SizeLEB);
-  Descriptions[DW_OP_shl] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_shr] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_shra] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_xor] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_bra] = Desc(Op::Dwarf2, Op::SignedSize2);
-  Descriptions[DW_OP_eq] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_ge] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_gt] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_le] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_lt] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_ne] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_skip] = Desc(Op::Dwarf2, Op::SignedSize2);
-  for (uint16_t LA = DW_OP_lit0; LA <= DW_OP_lit31; ++LA)
-    Descriptions[LA] = Desc(Op::Dwarf2);
-  for (uint16_t LA = DW_OP_reg0; LA <= DW_OP_reg31; ++LA)
-    Descriptions[LA] = Desc(Op::Dwarf2);
-  for (uint16_t LA = DW_OP_breg0; LA <= DW_OP_breg31; ++LA)
-    Descriptions[LA] = Desc(Op::Dwarf2, Op::SignedSizeLEB);
-  Descriptions[DW_OP_regx] = Desc(Op::Dwarf2, Op::SizeLEB);
-  Descriptions[DW_OP_fbreg] = Desc(Op::Dwarf2, Op::SignedSizeLEB);
-  Descriptions[DW_OP_bregx] = Desc(Op::Dwarf2, Op::SizeLEB, Op::SignedSizeLEB);
-  Descriptions[DW_OP_piece] = Desc(Op::Dwarf2, Op::SizeLEB);
-  Descriptions[DW_OP_deref_size] = Desc(Op::Dwarf2, Op::Size1);
-  Descriptions[DW_OP_xderef_size] = Desc(Op::Dwarf2, Op::Size1);
-  Descriptions[DW_OP_nop] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_push_object_address] = Desc(Op::Dwarf3);
-  Descriptions[DW_OP_call2] = Desc(Op::Dwarf3, Op::Size2);
-  Descriptions[DW_OP_call4] = Desc(Op::Dwarf3, Op::Size4);
-  Descriptions[DW_OP_call_ref] = Desc(Op::Dwarf3, Op::SizeRefAddr);
-  Descriptions[DW_OP_form_tls_address] = Desc(Op::Dwarf3);
-  Descriptions[DW_OP_call_frame_cfa] = Desc(Op::Dwarf3);
-  Descriptions[DW_OP_bit_piece] = Desc(Op::Dwarf3, Op::SizeLEB, Op::SizeLEB);
-  Descriptions[DW_OP_implicit_value] =
-      Desc(Op::Dwarf4, Op::SizeLEB, Op::SizeBlock);
-  Descriptions[DW_OP_stack_value] = Desc(Op::Dwarf4);
-  Descriptions[DW_OP_implicit_pointer] =
-      Desc(Op::Dwarf5, Op::SizeRefAddr, Op::SignedSizeLEB);
-  Descriptions[DW_OP_addrx] = Desc(Op::Dwarf5, Op::SizeLEB);
-  Descriptions[DW_OP_constx] = Desc(Op::Dwarf5, Op::SizeLEB);
-  Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
-  Descriptions[DW_OP_convert] = Desc(Op::Dwarf5, Op::BaseTypeRef);
-  Descriptions[DW_OP_regval_type] =
-      Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
-  Descriptions[DW_OP_WASM_location] =
-      Desc(Op::Dwarf4, Op::SizeLEB, Op::WasmLocationArg);
-  Descriptions[DW_OP_GNU_push_tls_address] = Desc(Op::Dwarf3);
-  Descriptions[DW_OP_GNU_addr_index] = Desc(Op::Dwarf4, Op::SizeLEB);
-  Descriptions[DW_OP_GNU_const_index] = Desc(Op::Dwarf4, Op::SizeLEB);
-  Descriptions[DW_OP_GNU_entry_value] = Desc(Op::Dwarf4, Op::SizeLEB);
-  Descriptions[DW_OP_GNU_implicit_pointer] =
-      Desc(Op::Dwarf4, Op::SizeRefAddr, Op::SignedSizeLEB);
-  // This Description acts as a marker that getSubOpDesc must be called
-  // to fetch the final Description for the operation. Each such final
-  // Description must share the same first SizeSubOpLEB operand.
-  Descriptions[DW_OP_LLVM_user] = Desc(Op::Dwarf5, Op::SizeSubOpLEB);
-  return Descriptions;
-}
-
-static Desc getDescImpl(ArrayRef<Desc> Descriptions, unsigned Opcode) {
-  // Handle possible corrupted or unsupported operation.
-  if (Opcode >= Descriptions.size())
-    return {};
-  return Descriptions[Opcode];
-}
-
-static Desc getOpDesc(unsigned Opcode) {
-  static std::vector<Desc> Descriptions = getOpDescriptions();
-  return getDescImpl(Descriptions, Opcode);
-}
-
-static std::vector<Desc> getSubOpDescriptions() {
-  static constexpr unsigned LlvmUserDescriptionsSize = 1
-#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME) +1
-#include "llvm/BinaryFormat/Dwarf.def"
-      ;
-  std::vector<Desc> Descriptions;
-  Descriptions.resize(LlvmUserDescriptionsSize);
-  Descriptions[DW_OP_LLVM_nop] = Desc(Op::Dwarf5, Op::SizeSubOpLEB);
-  return Descriptions;
-}
-
-static Desc getSubOpDesc(unsigned Opcode, unsigned SubOpcode) {
-  assert(Opcode == DW_OP_LLVM_user);
-  static std::vector<Desc> Descriptions = getSubOpDescriptions();
-  return getDescImpl(Descriptions, SubOpcode);
-}
-
-bool DWARFExpression::Operation::extract(DataExtractor Data,
-                                         uint8_t AddressSize, uint64_t Offset,
-                                         std::optional<DwarfFormat> Format) {
-  EndOffset = Offset;
-  Opcode = Data.getU8(&Offset);
-
-  Desc = getOpDesc(Opcode);
-  if (Desc.Version == Operation::DwarfNA)
-    return false;
-
-  Operands.resize(Desc.Op.size());
-  OperandEndOffsets.resize(Desc.Op.size());
-  for (unsigned Operand = 0; Operand < Desc.Op.size(); ++Operand) {
-    unsigned Size = Desc.Op[Operand];
-    unsigned Signed = Size & Operation::SignBit;
-
-    switch (Size & ~Operation::SignBit) {
-    case Operation::SizeSubOpLEB:
-      assert(Operand == 0 && "SubOp operand must be the first operand");
-      Operands[Operand] = Data.getULEB128(&Offset);
-      Desc = getSubOpDesc(Opcode, Operands[Operand]);
-      if (Desc.Version == Operation::DwarfNA)
-        return false;
-      assert(Desc.Op[Operand] == Operation::SizeSubOpLEB &&
-             "SizeSubOpLEB Description must begin with SizeSubOpLEB operand");
-      break;
-    case Operation::Size1:
-      Operands[Operand] = Data.getU8(&Offset);
-      if (Signed)
-        Operands[Operand] = (int8_t)Operands[Operand];
-      break;
-    case Operation::Size2:
-      Operands[Operand] = Data.getU16(&Offset);
-      if (Signed)
-        Operands[Operand] = (int16_t)Operands[Operand];
-      break;
-    case Operation::Size4:
-      Operands[Operand] = Data.getU32(&Offset);
-      if (Signed)
-        Operands[Operand] = (int32_t)Operands[Operand];
-      break;
-    case Operation::Size8:
-      Operands[Operand] = Data.getU64(&Offset);
-      break;
-    case Operation::SizeAddr:
-      Operands[Operand] = Data.getUnsigned(&Offset, AddressSize);
-      break;
-    case Operation::SizeRefAddr:
-      if (!Format)
-        return false;
-      Operands[Operand] =
-          Data.getUnsigned(&Offset, dwarf::getDwarfOffsetByteSize(*Format));
-      break;
-    case Operation::SizeLEB:
-      if (Signed)
-        Operands[Operand] = Data.getSLEB128(&Offset);
-      else
-        Operands[Operand] = Data.getULEB128(&Offset);
-      break;
-    case Operation::BaseTypeRef:
-      Operands[Operand] = Data.getULEB128(&Offset);
-      break;
-    case Operation::WasmLocationArg:
-      assert(Operand == 1);
-      switch (Operands[0]) {
-      case 0:
-      case 1:
-      case 2:
-      case 4:
-        Operands[Operand] = Data.getULEB128(&Offset);
-        break;
-      case 3: // global as uint32
-         Operands[Operand] = Data.getU32(&Offset);
-         break;
-      default:
-        return false; // Unknown Wasm location
-      }
-      break;
-    case Operation::SizeBlock:
-      // We need a size, so this cannot be the first operand
-      if (Operand == 0)
-        return false;
-      // Store the offset of the block as the value.
-      Operands[Operand] = Offset;
-      Offset += Operands[Operand - 1];
-      break;
-    default:
-      llvm_unreachable("Unknown DWARFExpression Op size");
-    }
-
-    OperandEndOffsets[Operand] = Offset;
-  }
-
-  EndOffset = Offset;
-  return true;
-}
-
-std::optional<unsigned> DWARFExpression::Operation::getSubCode() const {
-  if (!Desc.Op.size() || Desc.Op[0] != Operation::SizeSubOpLEB)
-    return std::nullopt;
-  return Operands[0];
-}
-
-bool DWARFExpression::operator==(const DWARFExpression &RHS) const {
-  if (AddressSize != RHS.AddressSize || Format != RHS.Format)
-    return false;
-  return Data.getData() == RHS.Data.getData();
-}
-
-void DWARFExpressionPrinter::prettyPrintBaseTypeRef(DWARFUnit *U,
-                                                    raw_ostream &OS,
-                                                    DIDumpOptions DumpOpts,
-                                                    ArrayRef<uint64_t> Operands,
-                                                    unsigned Operand) {
-  assert(Operand < Operands.size() && "operand out of bounds");
-  if (!U) {
-    OS << format(" <base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
-    return;
-  }
-  auto Die = U->getDIEForOffset(U->getOffset() + Operands[Operand]);
-  if (Die && Die.getTag() == dwarf::DW_TAG_base_type) {
-    OS << " (";
-    if (DumpOpts.Verbose)
-      OS << format("0x%08" PRIx64 " -> ", Operands[Operand]);
-    OS << format("0x%08" PRIx64 ")", U->getOffset() + Operands[Operand]);
-    if (auto Name = dwarf::toString(Die.find(dwarf::DW_AT_name)))
-      OS << " \"" << *Name << "\"";
-  } else {
-    OS << format(" <invalid base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
-  }
-}
-
-bool DWARFExpressionPrinter::prettyPrintRegisterOp(
-    DWARFUnit *U, raw_ostream &OS, DIDumpOptions DumpOpts, uint8_t Opcode,
-    ArrayRef<uint64_t> Operands) {
-  if (!DumpOpts.GetNameForDWARFReg)
-    return false;
-
-  uint64_t DwarfRegNum;
-  unsigned OpNum = 0;
-
-  if (Opcode == DW_OP_bregx || Opcode == DW_OP_regx ||
-      Opcode == DW_OP_regval_type)
-    DwarfRegNum = Operands[OpNum++];
-  else if (Opcode >= DW_OP_breg0 && Opcode < DW_OP_bregx)
-    DwarfRegNum = Opcode - DW_OP_breg0;
-  else
-    DwarfRegNum = Opcode - DW_OP_reg0;
-
-  auto RegName = DumpOpts.GetNameForDWARFReg(DwarfRegNum, DumpOpts.IsEH);
-  if (!RegName.empty()) {
-    if ((Opcode >= DW_OP_breg0 && Opcode <= DW_OP_breg31) ||
-        Opcode == DW_OP_bregx)
-      OS << ' ' << RegName << format("%+" PRId64, Operands[OpNum]);
-    else
-      OS << ' ' << RegName.data();
-
-    if (Opcode == DW_OP_regval_type)
-      prettyPrintBaseTypeRef(U, OS, DumpOpts, Operands, 1);
-    return true;
-  }
-
-  return false;
-}
-
-bool DWARFExpressionPrinter::printOp(const DWARFExpression::Operation *Op,
-                                     raw_ostream &OS, DIDumpOptions DumpOpts,
-                                     const DWARFExpression *Expr,
-                                     DWARFUnit *U) {
-  if (Op->Error) {
-    OS << "<decoding error>";
-    return false;
-  }
-
-  StringRef Name = OperationEncodingString(Op->Opcode);
-  assert(!Name.empty() && "DW_OP has no name!");
-  OS << Name;
-
-  if ((Op->Opcode >= DW_OP_breg0 && Op->Opcode <= DW_OP_breg31) ||
-      (Op->Opcode >= DW_OP_reg0 && Op->Opcode <= DW_OP_reg31) ||
-      Op->Opcode == DW_OP_bregx || Op->Opcode == DW_OP_regx ||
-      Op->Opcode == DW_OP_regval_type)
-    if (prettyPrintRegisterOp(U, OS, DumpOpts, Op->Opcode, Op->Operands))
-      return true;
-
-  for (unsigned Operand = 0; Operand < Op->Desc.Op.size(); ++Operand) {
-    unsigned Size = Op->Desc.Op[Operand];
-    unsigned Signed = Size & DWARFExpression::Operation::SignBit;
-
-    if (Size == DWARFExpression::Operation::SizeSubOpLEB) {
-      StringRef SubName =
-          SubOperationEncodingString(Op->Opcode, Op->Operands[Operand]);
-      assert(!SubName.empty() && "DW_OP SubOp has no name!");
-      OS << " " << SubName;
-    } else if (Size == DWARFExpression::Operation::BaseTypeRef && U) {
-      // For DW_OP_convert the operand may be 0 to indicate that conversion to
-      // the generic type should be done. The same holds for DW_OP_reinterpret,
-      // which is currently not supported.
-      if (Op->Opcode == DW_OP_convert && Op->Operands[Operand] == 0)
-        OS << " 0x0";
-      else
-        prettyPrintBaseTypeRef(U, OS, DumpOpts, Op->Operands, Operand);
-    } else if (Size == DWARFExpression::Operation::WasmLocationArg) {
-      assert(Operand == 1);
-      switch (Op->Operands[0]) {
-      case 0:
-      case 1:
-      case 2:
-      case 3: // global as uint32
-      case 4:
-        OS << format(" 0x%" PRIx64, Op->Operands[Operand]);
-        break;
-      default: assert(false);
-      }
-    } else if (Size == DWARFExpression::Operation::SizeBlock) {
-      uint64_t Offset = Op->Operands[Operand];
-      for (unsigned i = 0; i < Op->Operands[Operand - 1]; ++i)
-        OS << format(" 0x%02x", Expr->Data.getU8(&Offset));
-    } else {
-      if (Signed)
-        OS << format(" %+" PRId64, (int64_t)Op->Operands[Operand]);
-      else if (Op->Opcode != DW_OP_entry_value &&
-               Op->Opcode != DW_OP_GNU_entry_value)
-        OS << format(" 0x%" PRIx64, Op->Operands[Operand]);
-    }
-  }
-  return true;
-}
-
-void DWARFExpressionPrinter::print(const DWARFExpression *E, raw_ostream &OS,
-                                   DIDumpOptions DumpOpts, DWARFUnit *U,
-                                   bool IsEH) {
-  uint32_t EntryValExprSize = 0;
-  uint64_t EntryValStartOffset = 0;
-  if (E->Data.getData().empty())
-    OS << "<empty>";
-
-  for (auto &Op : *E) {
-    DumpOpts.IsEH = IsEH;
-    if (!printOp(&Op, OS, DumpOpts, E, U)) {
-      uint64_t FailOffset = Op.getEndOffset();
-      while (FailOffset < E->Data.getData().size())
-        OS << format(" %02x", E->Data.getU8(&FailOffset));
-      return;
-    }
-
-    if (Op.getCode() == DW_OP_entry_value ||
-        Op.getCode() == DW_OP_GNU_entry_value) {
-      OS << "(";
-      EntryValExprSize = Op.getRawOperand(0);
-      EntryValStartOffset = Op.getEndOffset();
-      continue;
-    }
-
-    if (EntryValExprSize) {
-      EntryValExprSize -= Op.getEndOffset() - EntryValStartOffset;
-      if (EntryValExprSize == 0)
-        OS << ")";
-    }
-
-    if (Op.getEndOffset() < E->Data.getData().size())
-      OS << ", ";
-  }
-}
-
-/// A user-facing string representation of a DWARF expression. This might be an
-/// Address expression, in which case it will be implicitly dereferenced, or a
-/// Value expression.
-struct PrintedExpr {
-  enum ExprKind {
-    Address,
-    Value,
-  };
-  ExprKind Kind;
-  SmallString<16> String;
-
-  PrintedExpr(ExprKind K = Address) : Kind(K) {}
-};
-
-static bool printCompactDWARFExpr(
-    raw_ostream &OS, DWARFExpression::iterator I,
-    const DWARFExpression::iterator E,
-    std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg =
-        nullptr) {
-  SmallVector<PrintedExpr, 4> Stack;
-
-  while (I != E) {
-    const DWARFExpression::Operation &Op = *I;
-    uint8_t Opcode = Op.getCode();
-    switch (Opcode) {
-    case dwarf::DW_OP_regx: {
-      // DW_OP_regx: A register, with the register num given as an operand.
-      // Printed as the plain register name.
-      uint64_t DwarfRegNum = Op.getRawOperand(0);
-      auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
-      if (RegName.empty())
-        return false;
-      raw_svector_ostream S(Stack.emplace_back(PrintedExpr::Value).String);
-      S << RegName;
-      break;
-    }
-    case dwarf::DW_OP_bregx: {
-      int DwarfRegNum = Op.getRawOperand(0);
-      int64_t Offset = Op.getRawOperand(1);
-      auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
-      if (RegName.empty())
-        return false;
-      raw_svector_ostream S(Stack.emplace_back().String);
-      S << RegName;
-      if (Offset)
-        S << format("%+" PRId64, Offset);
-      break;
-    }
-    case dwarf::DW_OP_entry_value:
-    case dwarf::DW_OP_GNU_entry_value: {
-      // DW_OP_entry_value contains a sub-expression which must be rendered
-      // separately.
-      uint64_t SubExprLength = Op.getRawOperand(0);
-      DWARFExpression::iterator SubExprEnd = I.skipBytes(SubExprLength);
-      ++I;
-      raw_svector_ostream S(Stack.emplace_back().String);
-      S << "entry(";
-      printCompactDWARFExpr(S, I, SubExprEnd, GetNameForDWARFReg);
-      S << ")";
-      I = SubExprEnd;
-      continue;
-    }
-    case dwarf::DW_OP_stack_value: {
-      // The top stack entry should be treated as the actual value of tne
-      // variable, rather than the address of the variable in memory.
-      assert(!Stack.empty());
-      Stack.back().Kind = PrintedExpr::Value;
-      break;
-    }
-    case dwarf::DW_OP_nop: {
-      break;
-    }
-    case dwarf::DW_OP_LLVM_user: {
-      assert(Op.getSubCode() == dwarf::DW_OP_LLVM_nop);
-      break;
-    }
-    default:
-      if (Opcode >= dwarf::DW_OP_reg0 && Opcode <= dwarf::DW_OP_reg31) {
-        // DW_OP_reg<N>: A register, with the register num implied by the
-        // opcode. Printed as the plain register name.
-        uint64_t DwarfRegNum = Opcode - dwarf::DW_OP_reg0;
-        auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
-        if (RegName.empty())
-          return false;
-        raw_svector_ostream S(Stack.emplace_back(PrintedExpr::Value).String);
-        S << RegName;
-      } else if (Opcode >= dwarf::DW_OP_breg0 &&
-                 Opcode <= dwarf::DW_OP_breg31) {
-        int DwarfRegNum = Opcode - dwarf::DW_OP_breg0;
-        int64_t Offset = Op.getRawOperand(0);
-        auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
-        if (RegName.empty())
-          return false;
-        raw_svector_ostream S(Stack.emplace_back().String);
-        S << RegName;
-        if (Offset)
-          S << format("%+" PRId64, Offset);
-      } else {
-        // If we hit an unknown operand, we don't know its effect on the stack,
-        // so bail out on the whole expression.
-        OS << "<unknown op " << dwarf::OperationEncodingString(Opcode) << " ("
-           << (int)Opcode << ")>";
-        return false;
-      }
-      break;
-    }
-    ++I;
-  }
-
-  if (Stack.size() != 1) {
-    OS << "<stack of size " << Stack.size() << ", expected 1>";
-    return false;
-  }
-
-  if (Stack.front().Kind == PrintedExpr::Address)
-    OS << "[" << Stack.front().String << "]";
-  else
-    OS << Stack.front().String;
-
-  return true;
-}
-
-bool DWARFExpressionPrinter::printCompact(
-    const DWARFExpression *E, raw_ostream &OS,
-    std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg) {
-  return printCompactDWARFExpr(OS, E->begin(), E->end(), GetNameForDWARFReg);
-}
-
-} // namespace llvm
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpressionPrinter.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpressionPrinter.cpp
new file mode 100644
index 0000000000000..ebcd4dda50488
--- /dev/null
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpressionPrinter.cpp
@@ -0,0 +1,310 @@
+//===-- DWARFExpression.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
+#include "llvm/Support/Format.h"
+#include <cassert>
+#include <cstdint>
+
+using namespace llvm;
+using namespace dwarf;
+
+namespace llvm {
+
+typedef DWARFExpression::Operation Op;
+typedef Op::Description Desc;
+
+static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
+                                   DIDumpOptions DumpOpts,
+                                   ArrayRef<uint64_t> Operands,
+                                   unsigned Operand) {
+  assert(Operand < Operands.size() && "operand out of bounds");
+  if (!U) {
+    OS << format(" <base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
+    return;
+  }
+  auto Die = U->getDIEForOffset(U->getOffset() + Operands[Operand]);
+  if (Die && Die.getTag() == dwarf::DW_TAG_base_type) {
+    OS << " (";
+    if (DumpOpts.Verbose)
+      OS << format("0x%08" PRIx64 " -> ", Operands[Operand]);
+    OS << format("0x%08" PRIx64 ")", U->getOffset() + Operands[Operand]);
+    if (auto Name = dwarf::toString(Die.find(dwarf::DW_AT_name)))
+      OS << " \"" << *Name << "\"";
+  } else {
+    OS << format(" <invalid base_type ref: 0x%" PRIx64 ">", Operands[Operand]);
+  }
+}
+
+static bool printOp(const DWARFExpression::Operation *Op, raw_ostream &OS,
+                    DIDumpOptions DumpOpts, const DWARFExpression *Expr,
+                    DWARFUnit *U) {
+  if (Op->isError()) {
+    OS << "<decoding error>";
+    return false;
+  }
+
+  StringRef Name = OperationEncodingString(Op->getCode());
+  assert(!Name.empty() && "DW_OP has no name!");
+  OS << Name;
+
+  if ((Op->getCode() >= DW_OP_breg0 && Op->getCode() <= DW_OP_breg31) ||
+      (Op->getCode() >= DW_OP_reg0 && Op->getCode() <= DW_OP_reg31) ||
+      Op->getCode() == DW_OP_bregx || Op->getCode() == DW_OP_regx ||
+      Op->getCode() == DW_OP_regval_type)
+    if (prettyPrintRegisterOp(U, OS, DumpOpts, Op->getCode(),
+                              Op->getRawOperands()))
+      return true;
+
+  for (unsigned Operand = 0; Operand < Op->getDescription().Op.size();
+       ++Operand) {
+    unsigned Size = Op->getDescription().Op[Operand];
+    unsigned Signed = Size & DWARFExpression::Operation::SignBit;
+
+    if (Size == DWARFExpression::Operation::SizeSubOpLEB) {
+      StringRef SubName =
+          SubOperationEncodingString(Op->getCode(), Op->getRawOperand(Operand));
+      assert(!SubName.empty() && "DW_OP SubOp has no name!");
+      OS << " " << SubName;
+    } else if (Size == DWARFExpression::Operation::BaseTypeRef && U) {
+      // For DW_OP_convert the operand may be 0 to indicate that conversion to
+      // the generic type should be done. The same holds for DW_OP_reinterpret,
+      // which is currently not supported.
+      if (Op->getCode() == DW_OP_convert && Op->getRawOperand(Operand) == 0)
+        OS << " 0x0";
+      else
+        prettyPrintBaseTypeRef(U, OS, DumpOpts, Op->getRawOperands(), Operand);
+    } else if (Size == DWARFExpression::Operation::WasmLocationArg) {
+      assert(Operand == 1);
+      switch (Op->getRawOperand(0)) {
+      case 0:
+      case 1:
+      case 2:
+      case 3: // global as uint32
+      case 4:
+        OS << format(" 0x%" PRIx64, Op->getRawOperand(Operand));
+        break;
+      default:
+        assert(false);
+      }
+    } else if (Size == DWARFExpression::Operation::SizeBlock) {
+      uint64_t Offset = Op->getRawOperand(Operand);
+      for (unsigned i = 0; i < Op->getRawOperand(Operand - 1); ++i)
+        OS << format(" 0x%02x",
+                     static_cast<uint8_t>(Expr->getData()[Offset++]));
+    } else {
+      if (Signed)
+        OS << format(" %+" PRId64, (int64_t)Op->getRawOperand(Operand));
+      else if (Op->getCode() != DW_OP_entry_value &&
+               Op->getCode() != DW_OP_GNU_entry_value)
+        OS << format(" 0x%" PRIx64, Op->getRawOperand(Operand));
+    }
+  }
+  return true;
+}
+
+void printDwarfExpression(const DWARFExpression *E, raw_ostream &OS,
+                          DIDumpOptions DumpOpts, DWARFUnit *U, bool IsEH) {
+  uint32_t EntryValExprSize = 0;
+  uint64_t EntryValStartOffset = 0;
+  if (E->getData().empty())
+    OS << "<empty>";
+
+  for (auto &Op : *E) {
+    DumpOpts.IsEH = IsEH;
+    if (!printOp(&Op, OS, DumpOpts, E, U)) {
+      uint64_t FailOffset = Op.getEndOffset();
+      while (FailOffset < E->getData().size())
+        OS << format(" %02x", static_cast<uint8_t>(E->getData()[FailOffset++]));
+      return;
+    }
+
+    if (Op.getCode() == DW_OP_entry_value ||
+        Op.getCode() == DW_OP_GNU_entry_value) {
+      OS << "(";
+      EntryValExprSize = Op.getRawOperand(0);
+      EntryValStartOffset = Op.getEndOffset();
+      continue;
+    }
+
+    if (EntryValExprSize) {
+      EntryValExprSize -= Op.getEndOffset() - EntryValStartOffset;
+      if (EntryValExprSize == 0)
+        OS << ")";
+    }
+
+    if (Op.getEndOffset() < E->getData().size())
+      OS << ", ";
+  }
+}
+
+/// A user-facing string representation of a DWARF expression. This might be an
+/// Address expression, in which case it will be implicitly dereferenced, or a
+/// Value expression.
+struct PrintedExpr {
+  enum ExprKind {
+    Address,
+    Value,
+  };
+  ExprKind Kind;
+  SmallString<16> String;
+
+  PrintedExpr(ExprKind K = Address) : Kind(K) {}
+};
+
+static bool printCompactDWARFExpr(
+    raw_ostream &OS, DWARFExpression::iterator I,
+    const DWARFExpression::iterator E,
+    std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg =
+        nullptr) {
+  SmallVector<PrintedExpr, 4> Stack;
+
+  while (I != E) {
+    const DWARFExpression::Operation &Op = *I;
+    uint8_t Opcode = Op.getCode();
+    switch (Opcode) {
+    case dwarf::DW_OP_regx: {
+      // DW_OP_regx: A register, with the register num given as an operand.
+      // Printed as the plain register name.
+      uint64_t DwarfRegNum = Op.getRawOperand(0);
+      auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
+      if (RegName.empty())
+        return false;
+      raw_svector_ostream S(Stack.emplace_back(PrintedExpr::Value).String);
+      S << RegName;
+      break;
+    }
+    case dwarf::DW_OP_bregx: {
+      int DwarfRegNum = Op.getRawOperand(0);
+      int64_t Offset = Op.getRawOperand(1);
+      auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
+      if (RegName.empty())
+        return false;
+      raw_svector_ostream S(Stack.emplace_back().String);
+      S << RegName;
+      if (Offset)
+        S << format("%+" PRId64, Offset);
+      break;
+    }
+    case dwarf::DW_OP_entry_value:
+    case dwarf::DW_OP_GNU_entry_value: {
+      // DW_OP_entry_value contains a sub-expression which must be rendered
+      // separately.
+      uint64_t SubExprLength = Op.getRawOperand(0);
+      DWARFExpression::iterator SubExprEnd = I.skipBytes(SubExprLength);
+      ++I;
+      raw_svector_ostream S(Stack.emplace_back().String);
+      S << "entry(";
+      printCompactDWARFExpr(S, I, SubExprEnd, GetNameForDWARFReg);
+      S << ")";
+      I = SubExprEnd;
+      continue;
+    }
+    case dwarf::DW_OP_stack_value: {
+      // The top stack entry should be treated as the actual value of tne
+      // variable, rather than the address of the variable in memory.
+      assert(!Stack.empty());
+      Stack.back().Kind = PrintedExpr::Value;
+      break;
+    }
+    case dwarf::DW_OP_nop: {
+      break;
+    }
+    case dwarf::DW_OP_LLVM_user: {
+      assert(Op.getSubCode() == dwarf::DW_OP_LLVM_nop);
+      break;
+    }
+    default:
+      if (Opcode >= dwarf::DW_OP_reg0 && Opcode <= dwarf::DW_OP_reg31) {
+        // DW_OP_reg<N>: A register, with the register num implied by the
+        // opcode. Printed as the plain register name.
+        uint64_t DwarfRegNum = Opcode - dwarf::DW_OP_reg0;
+        auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
+        if (RegName.empty())
+          return false;
+        raw_svector_ostream S(Stack.emplace_back(PrintedExpr::Value).String);
+        S << RegName;
+      } else if (Opcode >= dwarf::DW_OP_breg0 &&
+                 Opcode <= dwarf::DW_OP_breg31) {
+        int DwarfRegNum = Opcode - dwarf::DW_OP_breg0;
+        int64_t Offset = Op.getRawOperand(0);
+        auto RegName = GetNameForDWARFReg(DwarfRegNum, false);
+        if (RegName.empty())
+          return false;
+        raw_svector_ostream S(Stack.emplace_back().String);
+        S << RegName;
+        if (Offset)
+          S << format("%+" PRId64, Offset);
+      } else {
+        // If we hit an unknown operand, we don't know its effect on the stack,
+        // so bail out on the whole expression.
+        OS << "<unknown op " << dwarf::OperationEncodingString(Opcode) << " ("
+           << (int)Opcode << ")>";
+        return false;
+      }
+      break;
+    }
+    ++I;
+  }
+
+  if (Stack.size() != 1) {
+    OS << "<stack of size " << Stack.size() << ", expected 1>";
+    return false;
+  }
+
+  if (Stack.front().Kind == PrintedExpr::Address)
+    OS << "[" << Stack.front().String << "]";
+  else
+    OS << Stack.front().String;
+
+  return true;
+}
+
+bool printDwarfExpressionCompact(
+    const DWARFExpression *E, raw_ostream &OS,
+    std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg) {
+  return printCompactDWARFExpr(OS, E->begin(), E->end(), GetNameForDWARFReg);
+}
+
+bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
+                           DIDumpOptions DumpOpts, uint8_t Opcode,
+                           ArrayRef<uint64_t> Operands) {
+  if (!DumpOpts.GetNameForDWARFReg)
+    return false;
+
+  uint64_t DwarfRegNum;
+  unsigned OpNum = 0;
+
+  if (Opcode == DW_OP_bregx || Opcode == DW_OP_regx ||
+      Opcode == DW_OP_regval_type)
+    DwarfRegNum = Operands[OpNum++];
+  else if (Opcode >= DW_OP_breg0 && Opcode < DW_OP_bregx)
+    DwarfRegNum = Opcode - DW_OP_breg0;
+  else
+    DwarfRegNum = Opcode - DW_OP_reg0;
+
+  auto RegName = DumpOpts.GetNameForDWARFReg(DwarfRegNum, DumpOpts.IsEH);
+  if (!RegName.empty()) {
+    if ((Opcode >= DW_OP_breg0 && Opcode <= DW_OP_breg31) ||
+        Opcode == DW_OP_bregx)
+      OS << ' ' << RegName << format("%+" PRId64, Operands[OpNum]);
+    else
+      OS << ' ' << RegName.data();
+
+    if (Opcode == DW_OP_regval_type)
+      prettyPrintBaseTypeRef(U, OS, DumpOpts, Operands, 1);
+    return true;
+  }
+
+  return false;
+}
+
+} // namespace llvm
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
index d719a47c84072..ef59c82fc6a01 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -19,12 +19,12 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRnglists.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFListTable.h"
 #include "llvm/DebugInfo/DWARF/DWARFObject.h"
 #include "llvm/DebugInfo/DWARF/DWARFSection.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Errc.h"
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnwindTablePrinter.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnwindTablePrinter.cpp
new file mode 100644
index 0000000000000..a88f4a554bcf0
--- /dev/null
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnwindTablePrinter.cpp
@@ -0,0 +1,186 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/DWARF/DWARFUnwindTablePrinter.h"
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <cinttypes>
+#include <cstdint>
+#include <optional>
+
+using namespace llvm;
+using namespace dwarf;
+
+static void printRegister(raw_ostream &OS, DIDumpOptions DumpOpts,
+                          unsigned RegNum) {
+  if (DumpOpts.GetNameForDWARFReg) {
+    auto RegName = DumpOpts.GetNameForDWARFReg(RegNum, DumpOpts.IsEH);
+    if (!RegName.empty()) {
+      OS << RegName;
+      return;
+    }
+  }
+  OS << "reg" << RegNum;
+}
+
+/// Print an unwind location expression as text and use the register information
+/// if some is provided.
+///
+/// \param R the unwind location to print.
+///
+/// \param OS the stream to use for output.
+///
+/// \param MRI register information that helps emit register names insteead
+/// of raw register numbers.
+///
+/// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
+/// instead of from .debug_frame. This is needed for register number
+/// conversion because some register numbers differ between the two sections
+/// for certain architectures like x86.
+static void printUnwindLocation(const UnwindLocation &UL, raw_ostream &OS,
+                                DIDumpOptions DumpOpts) {
+  if (UL.getDereference())
+    OS << '[';
+  switch (UL.getLocation()) {
+  case UnwindLocation::Unspecified:
+    OS << "unspecified";
+    break;
+  case UnwindLocation::Undefined:
+    OS << "undefined";
+    break;
+  case UnwindLocation::Same:
+    OS << "same";
+    break;
+  case UnwindLocation::CFAPlusOffset:
+    OS << "CFA";
+    if (UL.getOffset() == 0)
+      break;
+    if (UL.getOffset() > 0)
+      OS << "+";
+    OS << UL.getOffset();
+    break;
+  case UnwindLocation::RegPlusOffset:
+    printRegister(OS, DumpOpts, UL.getRegister());
+    if (UL.getOffset() == 0 && !UL.hasAddressSpace())
+      break;
+    if (UL.getOffset() >= 0)
+      OS << "+";
+    OS << UL.getOffset();
+    if (UL.hasAddressSpace())
+      OS << " in addrspace" << UL.getAddressSpace();
+    break;
+  case UnwindLocation::DWARFExpr: {
+    if (UL.getDWARFExpressionBytes()) {
+      auto Expr = *UL.getDWARFExpressionBytes();
+      printDwarfExpression(&Expr, OS, DumpOpts, nullptr);
+    }
+    break;
+  }
+  case UnwindLocation::Constant:
+    OS << UL.getOffset();
+    break;
+  }
+  if (UL.getDereference())
+    OS << ']';
+}
+
+raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS,
+                                     const UnwindLocation &UL) {
+  auto DumpOpts = DIDumpOptions();
+  printUnwindLocation(UL, OS, DumpOpts);
+  return OS;
+}
+
+/// Print all registers + locations that are currently defined in a register
+/// locations.
+///
+/// \param RL the register locations to print.
+///
+/// \param OS the stream to use for output.
+///
+/// \param MRI register information that helps emit register names insteead
+/// of raw register numbers.
+///
+/// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
+/// instead of from .debug_frame. This is needed for register number
+/// conversion because some register numbers differ between the two sections
+/// for certain architectures like x86.
+static void printRegisterLocations(const RegisterLocations &RL, raw_ostream &OS,
+                                   DIDumpOptions DumpOpts) {
+  bool First = true;
+  for (uint32_t Reg : RL.getRegisters()) {
+    auto Loc = *RL.getRegisterLocation(Reg);
+    if (First)
+      First = false;
+    else
+      OS << ", ";
+    printRegister(OS, DumpOpts, Reg);
+    OS << '=';
+    printUnwindLocation(Loc, OS, DumpOpts);
+  }
+}
+
+raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS,
+                                     const RegisterLocations &RL) {
+  auto DumpOpts = DIDumpOptions();
+  printRegisterLocations(RL, OS, DumpOpts);
+  return OS;
+}
+
+/// Print an UnwindRow to the stream.
+///
+/// \param Row the UnwindRow to print.
+///
+/// \param OS the stream to use for output.
+///
+/// \param MRI register information that helps emit register names insteead
+/// of raw register numbers.
+///
+/// \param IsEH true if the DWARF Call Frame Information is from .eh_frame
+/// instead of from .debug_frame. This is needed for register number
+/// conversion because some register numbers differ between the two sections
+/// for certain architectures like x86.
+///
+/// \param IndentLevel specify the indent level as an integer. The UnwindRow
+/// will be output to the stream preceded by 2 * IndentLevel number of spaces.
+static void printUnwindRow(const UnwindRow &Row, raw_ostream &OS,
+                           DIDumpOptions DumpOpts, unsigned IndentLevel) {
+  OS.indent(2 * IndentLevel);
+  if (Row.hasAddress())
+    OS << format("0x%" PRIx64 ": ", Row.getAddress());
+  OS << "CFA=";
+  printUnwindLocation(Row.getCFAValue(), OS, DumpOpts);
+  if (Row.getRegisterLocations().hasLocations()) {
+    OS << ": ";
+    printRegisterLocations(Row.getRegisterLocations(), OS, DumpOpts);
+  }
+  OS << "\n";
+}
+
+raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS, const UnwindRow &Row) {
+  auto DumpOpts = DIDumpOptions();
+  printUnwindRow(Row, OS, DumpOpts, 0);
+  return OS;
+}
+
+void llvm::dwarf::printUnwindTable(const UnwindTable &Rows, raw_ostream &OS,
+                                   DIDumpOptions DumpOpts,
+                                   unsigned IndentLevel) {
+  for (const UnwindRow &Row : Rows)
+    printUnwindRow(Row, OS, DumpOpts, IndentLevel);
+}
+
+raw_ostream &llvm::dwarf::operator<<(raw_ostream &OS, const UnwindTable &Rows) {
+  auto DumpOpts = DIDumpOptions();
+  printUnwindTable(Rows, OS, DumpOpts, 0);
+  return OS;
+}
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index c12786cac8686..8ec3f1729b974 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -19,13 +19,13 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFObject.h"
 #include "llvm/DebugInfo/DWARF/DWARFSection.h"
 #include "llvm/DebugInfo/DWARF/DWARFTypeUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Support/DJB.h"
 #include "llvm/Support/Error.h"
diff --git a/llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt b/llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt
new file mode 100644
index 0000000000000..eda8d2314b7dd
--- /dev/null
+++ b/llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_llvm_component_library(LLVMDebugInfoDWARFLowLevel
+  DWARFCFIProgram.cpp
+  DWARFExpression.cpp
+  DWARFUnwindTable.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/DWARF/LowLevel
+
+  # This code should have almost no dependencies. Anything that needs
+  # more should use the higher-level DebugInfo/DWARF/ library.
+  LINK_COMPONENTS
+  BinaryFormat
+  Support
+  TargetParser
+  )
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFCFIProgram.cpp b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp
similarity index 60%
rename from llvm/lib/DebugInfo/DWARF/DWARFCFIProgram.cpp
rename to llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp
index 365b26b98a1e3..25a0ccb50c8a9 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFCFIProgram.cpp
+++ b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
-#include "llvm/DebugInfo/DIContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFDataExtractorSimple.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Errc.h"
@@ -23,149 +22,6 @@
 using namespace llvm;
 using namespace dwarf;
 
-// See DWARF standard v3, section 7.23
-const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
-const uint8_t DWARF_CFI_PRIMARY_OPERAND_MASK = 0x3f;
-
-Error CFIProgram::parse(DWARFDataExtractor Data, uint64_t *Offset,
-                        uint64_t EndOffset) {
-  DataExtractor::Cursor C(*Offset);
-  while (C && C.tell() < EndOffset) {
-    uint8_t Opcode = Data.getRelocatedValue(C, 1);
-    if (!C)
-      break;
-
-    // Some instructions have a primary opcode encoded in the top bits.
-    if (uint8_t Primary = Opcode & DWARF_CFI_PRIMARY_OPCODE_MASK) {
-      // If it's a primary opcode, the first operand is encoded in the bottom
-      // bits of the opcode itself.
-      uint64_t Op1 = Opcode & DWARF_CFI_PRIMARY_OPERAND_MASK;
-      switch (Primary) {
-      case DW_CFA_advance_loc:
-      case DW_CFA_restore:
-        addInstruction(Primary, Op1);
-        break;
-      case DW_CFA_offset:
-        addInstruction(Primary, Op1, Data.getULEB128(C));
-        break;
-      default:
-        llvm_unreachable("invalid primary CFI opcode");
-      }
-      continue;
-    }
-
-    // Extended opcode - its value is Opcode itself.
-    switch (Opcode) {
-    default:
-      return createStringError(errc::illegal_byte_sequence,
-                               "invalid extended CFI opcode 0x%" PRIx8, Opcode);
-    case DW_CFA_nop:
-    case DW_CFA_remember_state:
-    case DW_CFA_restore_state:
-    case DW_CFA_GNU_window_save:
-    case DW_CFA_AARCH64_negate_ra_state_with_pc:
-      // No operands
-      addInstruction(Opcode);
-      break;
-    case DW_CFA_set_loc:
-      // Operands: Address
-      addInstruction(Opcode, Data.getRelocatedAddress(C));
-      break;
-    case DW_CFA_advance_loc1:
-      // Operands: 1-byte delta
-      addInstruction(Opcode, Data.getRelocatedValue(C, 1));
-      break;
-    case DW_CFA_advance_loc2:
-      // Operands: 2-byte delta
-      addInstruction(Opcode, Data.getRelocatedValue(C, 2));
-      break;
-    case DW_CFA_advance_loc4:
-      // Operands: 4-byte delta
-      addInstruction(Opcode, Data.getRelocatedValue(C, 4));
-      break;
-    case DW_CFA_restore_extended:
-    case DW_CFA_undefined:
-    case DW_CFA_same_value:
-    case DW_CFA_def_cfa_register:
-    case DW_CFA_def_cfa_offset:
-    case DW_CFA_GNU_args_size:
-      // Operands: ULEB128
-      addInstruction(Opcode, Data.getULEB128(C));
-      break;
-    case DW_CFA_def_cfa_offset_sf:
-      // Operands: SLEB128
-      addInstruction(Opcode, Data.getSLEB128(C));
-      break;
-    case DW_CFA_LLVM_def_aspace_cfa:
-    case DW_CFA_LLVM_def_aspace_cfa_sf: {
-      auto RegNum = Data.getULEB128(C);
-      auto CfaOffset = Opcode == DW_CFA_LLVM_def_aspace_cfa
-                           ? Data.getULEB128(C)
-                           : Data.getSLEB128(C);
-      auto AddressSpace = Data.getULEB128(C);
-      addInstruction(Opcode, RegNum, CfaOffset, AddressSpace);
-      break;
-    }
-    case DW_CFA_offset_extended:
-    case DW_CFA_register:
-    case DW_CFA_def_cfa:
-    case DW_CFA_val_offset: {
-      // Operands: ULEB128, ULEB128
-      // Note: We can not embed getULEB128 directly into function
-      // argument list. getULEB128 changes Offset and order of evaluation
-      // for arguments is unspecified.
-      uint64_t op1 = Data.getULEB128(C);
-      uint64_t op2 = Data.getULEB128(C);
-      addInstruction(Opcode, op1, op2);
-      break;
-    }
-    case DW_CFA_offset_extended_sf:
-    case DW_CFA_def_cfa_sf:
-    case DW_CFA_val_offset_sf: {
-      // Operands: ULEB128, SLEB128
-      // Note: see comment for the previous case
-      uint64_t op1 = Data.getULEB128(C);
-      uint64_t op2 = (uint64_t)Data.getSLEB128(C);
-      addInstruction(Opcode, op1, op2);
-      break;
-    }
-    case DW_CFA_def_cfa_expression: {
-      uint64_t ExprLength = Data.getULEB128(C);
-      addInstruction(Opcode, 0);
-      StringRef Expression = Data.getBytes(C, ExprLength);
-
-      DataExtractor Extractor(Expression, Data.isLittleEndian(),
-                              Data.getAddressSize());
-      // Note. We do not pass the DWARF format to DWARFExpression, because
-      // DW_OP_call_ref, the only operation which depends on the format, is
-      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
-      Instructions.back().Expression =
-          DWARFExpression(Extractor, Data.getAddressSize());
-      break;
-    }
-    case DW_CFA_expression:
-    case DW_CFA_val_expression: {
-      uint64_t RegNum = Data.getULEB128(C);
-      addInstruction(Opcode, RegNum, 0);
-
-      uint64_t BlockLength = Data.getULEB128(C);
-      StringRef Expression = Data.getBytes(C, BlockLength);
-      DataExtractor Extractor(Expression, Data.isLittleEndian(),
-                              Data.getAddressSize());
-      // Note. We do not pass the DWARF format to DWARFExpression, because
-      // DW_OP_call_ref, the only operation which depends on the format, is
-      // prohibited in call frame instructions, see sec. 6.4.2 in DWARFv5.
-      Instructions.back().Expression =
-          DWARFExpression(Extractor, Data.getAddressSize());
-      break;
-    }
-    }
-  }
-
-  *Offset = C.tell();
-  return C.takeError();
-}
-
 StringRef CFIProgram::callFrameString(unsigned Opcode) const {
   return dwarf::CallFrameString(Opcode, Arch);
 }
diff --git a/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFExpression.cpp
new file mode 100644
index 0000000000000..70724185463b0
--- /dev/null
+++ b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFExpression.cpp
@@ -0,0 +1,253 @@
+//===-- DWARFExpression.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Format.h"
+#include <cassert>
+#include <cstdint>
+#include <vector>
+
+using namespace llvm;
+using namespace dwarf;
+
+namespace llvm {
+
+typedef DWARFExpression::Operation Op;
+typedef Op::Description Desc;
+
+static std::vector<Desc> getOpDescriptions() {
+  std::vector<Desc> Descriptions;
+  Descriptions.resize(0xff);
+  Descriptions[DW_OP_addr] = Desc(Op::Dwarf2, Op::SizeAddr);
+  Descriptions[DW_OP_deref] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_const1u] = Desc(Op::Dwarf2, Op::Size1);
+  Descriptions[DW_OP_const1s] = Desc(Op::Dwarf2, Op::SignedSize1);
+  Descriptions[DW_OP_const2u] = Desc(Op::Dwarf2, Op::Size2);
+  Descriptions[DW_OP_const2s] = Desc(Op::Dwarf2, Op::SignedSize2);
+  Descriptions[DW_OP_const4u] = Desc(Op::Dwarf2, Op::Size4);
+  Descriptions[DW_OP_const4s] = Desc(Op::Dwarf2, Op::SignedSize4);
+  Descriptions[DW_OP_const8u] = Desc(Op::Dwarf2, Op::Size8);
+  Descriptions[DW_OP_const8s] = Desc(Op::Dwarf2, Op::SignedSize8);
+  Descriptions[DW_OP_constu] = Desc(Op::Dwarf2, Op::SizeLEB);
+  Descriptions[DW_OP_consts] = Desc(Op::Dwarf2, Op::SignedSizeLEB);
+  Descriptions[DW_OP_dup] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_drop] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_over] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_pick] = Desc(Op::Dwarf2, Op::Size1);
+  Descriptions[DW_OP_swap] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_rot] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_xderef] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_abs] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_and] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_div] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_minus] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_mod] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_mul] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_neg] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_not] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_or] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_plus] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_plus_uconst] = Desc(Op::Dwarf2, Op::SizeLEB);
+  Descriptions[DW_OP_shl] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_shr] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_shra] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_xor] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_bra] = Desc(Op::Dwarf2, Op::SignedSize2);
+  Descriptions[DW_OP_eq] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_ge] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_gt] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_le] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_lt] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_ne] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_skip] = Desc(Op::Dwarf2, Op::SignedSize2);
+  for (uint16_t LA = DW_OP_lit0; LA <= DW_OP_lit31; ++LA)
+    Descriptions[LA] = Desc(Op::Dwarf2);
+  for (uint16_t LA = DW_OP_reg0; LA <= DW_OP_reg31; ++LA)
+    Descriptions[LA] = Desc(Op::Dwarf2);
+  for (uint16_t LA = DW_OP_breg0; LA <= DW_OP_breg31; ++LA)
+    Descriptions[LA] = Desc(Op::Dwarf2, Op::SignedSizeLEB);
+  Descriptions[DW_OP_regx] = Desc(Op::Dwarf2, Op::SizeLEB);
+  Descriptions[DW_OP_fbreg] = Desc(Op::Dwarf2, Op::SignedSizeLEB);
+  Descriptions[DW_OP_bregx] = Desc(Op::Dwarf2, Op::SizeLEB, Op::SignedSizeLEB);
+  Descriptions[DW_OP_piece] = Desc(Op::Dwarf2, Op::SizeLEB);
+  Descriptions[DW_OP_deref_size] = Desc(Op::Dwarf2, Op::Size1);
+  Descriptions[DW_OP_xderef_size] = Desc(Op::Dwarf2, Op::Size1);
+  Descriptions[DW_OP_nop] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_push_object_address] = Desc(Op::Dwarf3);
+  Descriptions[DW_OP_call2] = Desc(Op::Dwarf3, Op::Size2);
+  Descriptions[DW_OP_call4] = Desc(Op::Dwarf3, Op::Size4);
+  Descriptions[DW_OP_call_ref] = Desc(Op::Dwarf3, Op::SizeRefAddr);
+  Descriptions[DW_OP_form_tls_address] = Desc(Op::Dwarf3);
+  Descriptions[DW_OP_call_frame_cfa] = Desc(Op::Dwarf3);
+  Descriptions[DW_OP_bit_piece] = Desc(Op::Dwarf3, Op::SizeLEB, Op::SizeLEB);
+  Descriptions[DW_OP_implicit_value] =
+      Desc(Op::Dwarf4, Op::SizeLEB, Op::SizeBlock);
+  Descriptions[DW_OP_stack_value] = Desc(Op::Dwarf4);
+  Descriptions[DW_OP_implicit_pointer] =
+      Desc(Op::Dwarf5, Op::SizeRefAddr, Op::SignedSizeLEB);
+  Descriptions[DW_OP_addrx] = Desc(Op::Dwarf5, Op::SizeLEB);
+  Descriptions[DW_OP_constx] = Desc(Op::Dwarf5, Op::SizeLEB);
+  Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
+  Descriptions[DW_OP_convert] = Desc(Op::Dwarf5, Op::BaseTypeRef);
+  Descriptions[DW_OP_regval_type] =
+      Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
+  Descriptions[DW_OP_WASM_location] =
+      Desc(Op::Dwarf4, Op::SizeLEB, Op::WasmLocationArg);
+  Descriptions[DW_OP_GNU_push_tls_address] = Desc(Op::Dwarf3);
+  Descriptions[DW_OP_GNU_addr_index] = Desc(Op::Dwarf4, Op::SizeLEB);
+  Descriptions[DW_OP_GNU_const_index] = Desc(Op::Dwarf4, Op::SizeLEB);
+  Descriptions[DW_OP_GNU_entry_value] = Desc(Op::Dwarf4, Op::SizeLEB);
+  Descriptions[DW_OP_GNU_implicit_pointer] =
+      Desc(Op::Dwarf4, Op::SizeRefAddr, Op::SignedSizeLEB);
+  // This Description acts as a marker that getSubOpDesc must be called
+  // to fetch the final Description for the operation. Each such final
+  // Description must share the same first SizeSubOpLEB operand.
+  Descriptions[DW_OP_LLVM_user] = Desc(Op::Dwarf5, Op::SizeSubOpLEB);
+  return Descriptions;
+}
+
+static Desc getDescImpl(ArrayRef<Desc> Descriptions, unsigned Opcode) {
+  // Handle possible corrupted or unsupported operation.
+  if (Opcode >= Descriptions.size())
+    return {};
+  return Descriptions[Opcode];
+}
+
+static Desc getOpDesc(unsigned Opcode) {
+  static std::vector<Desc> Descriptions = getOpDescriptions();
+  return getDescImpl(Descriptions, Opcode);
+}
+
+static std::vector<Desc> getSubOpDescriptions() {
+  static constexpr unsigned LlvmUserDescriptionsSize = 1
+#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME) +1
+#include "llvm/BinaryFormat/Dwarf.def"
+      ;
+  std::vector<Desc> Descriptions;
+  Descriptions.resize(LlvmUserDescriptionsSize);
+  Descriptions[DW_OP_LLVM_nop] = Desc(Op::Dwarf5, Op::SizeSubOpLEB);
+  return Descriptions;
+}
+
+static Desc getSubOpDesc(unsigned Opcode, unsigned SubOpcode) {
+  assert(Opcode == DW_OP_LLVM_user);
+  static std::vector<Desc> Descriptions = getSubOpDescriptions();
+  return getDescImpl(Descriptions, SubOpcode);
+}
+
+bool DWARFExpression::Operation::extract(DataExtractor Data,
+                                         uint8_t AddressSize, uint64_t Offset,
+                                         std::optional<DwarfFormat> Format) {
+  EndOffset = Offset;
+  Opcode = Data.getU8(&Offset);
+
+  Desc = getOpDesc(Opcode);
+  if (Desc.Version == Operation::DwarfNA)
+    return false;
+
+  Operands.resize(Desc.Op.size());
+  OperandEndOffsets.resize(Desc.Op.size());
+  for (unsigned Operand = 0; Operand < Desc.Op.size(); ++Operand) {
+    unsigned Size = Desc.Op[Operand];
+    unsigned Signed = Size & Operation::SignBit;
+
+    switch (Size & ~Operation::SignBit) {
+    case Operation::SizeSubOpLEB:
+      assert(Operand == 0 && "SubOp operand must be the first operand");
+      Operands[Operand] = Data.getULEB128(&Offset);
+      Desc = getSubOpDesc(Opcode, Operands[Operand]);
+      if (Desc.Version == Operation::DwarfNA)
+        return false;
+      assert(Desc.Op[Operand] == Operation::SizeSubOpLEB &&
+             "SizeSubOpLEB Description must begin with SizeSubOpLEB operand");
+      break;
+    case Operation::Size1:
+      Operands[Operand] = Data.getU8(&Offset);
+      if (Signed)
+        Operands[Operand] = (int8_t)Operands[Operand];
+      break;
+    case Operation::Size2:
+      Operands[Operand] = Data.getU16(&Offset);
+      if (Signed)
+        Operands[Operand] = (int16_t)Operands[Operand];
+      break;
+    case Operation::Size4:
+      Operands[Operand] = Data.getU32(&Offset);
+      if (Signed)
+        Operands[Operand] = (int32_t)Operands[Operand];
+      break;
+    case Operation::Size8:
+      Operands[Operand] = Data.getU64(&Offset);
+      break;
+    case Operation::SizeAddr:
+      Operands[Operand] = Data.getUnsigned(&Offset, AddressSize);
+      break;
+    case Operation::SizeRefAddr:
+      if (!Format)
+        return false;
+      Operands[Operand] =
+          Data.getUnsigned(&Offset, dwarf::getDwarfOffsetByteSize(*Format));
+      break;
+    case Operation::SizeLEB:
+      if (Signed)
+        Operands[Operand] = Data.getSLEB128(&Offset);
+      else
+        Operands[Operand] = Data.getULEB128(&Offset);
+      break;
+    case Operation::BaseTypeRef:
+      Operands[Operand] = Data.getULEB128(&Offset);
+      break;
+    case Operation::WasmLocationArg:
+      assert(Operand == 1);
+      switch (Operands[0]) {
+      case 0:
+      case 1:
+      case 2:
+      case 4:
+        Operands[Operand] = Data.getULEB128(&Offset);
+        break;
+      case 3: // global as uint32
+        Operands[Operand] = Data.getU32(&Offset);
+        break;
+      default:
+        return false; // Unknown Wasm location
+      }
+      break;
+    case Operation::SizeBlock:
+      // We need a size, so this cannot be the first operand
+      if (Operand == 0)
+        return false;
+      // Store the offset of the block as the value.
+      Operands[Operand] = Offset;
+      Offset += Operands[Operand - 1];
+      break;
+    default:
+      llvm_unreachable("Unknown DWARFExpression Op size");
+    }
+
+    OperandEndOffsets[Operand] = Offset;
+  }
+
+  EndOffset = Offset;
+  return true;
+}
+
+std::optional<unsigned> DWARFExpression::Operation::getSubCode() const {
+  if (!Desc.Op.size() || Desc.Op[0] != Operation::SizeSubOpLEB)
+    return std::nullopt;
+  return Operands[0];
+}
+
+bool DWARFExpression::operator==(const DWARFExpression &RHS) const {
+  if (AddressSize != RHS.AddressSize || Format != RHS.Format)
+    return false;
+  return Data.getData() == RHS.Data.getData();
+}
+
+} // namespace llvm
diff --git a/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp
new file mode 100644
index 0000000000000..20f52673eefe1
--- /dev/null
+++ b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp
@@ -0,0 +1,388 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <cinttypes>
+#include <cstdint>
+#include <optional>
+
+using namespace llvm;
+using namespace dwarf;
+
+UnwindLocation UnwindLocation::createUnspecified() { return {Unspecified}; }
+
+UnwindLocation UnwindLocation::createUndefined() { return {Undefined}; }
+
+UnwindLocation UnwindLocation::createSame() { return {Same}; }
+
+UnwindLocation UnwindLocation::createIsConstant(int32_t Value) {
+  return {Constant, InvalidRegisterNumber, Value, std::nullopt, false};
+}
+
+UnwindLocation UnwindLocation::createIsCFAPlusOffset(int32_t Offset) {
+  return {CFAPlusOffset, InvalidRegisterNumber, Offset, std::nullopt, false};
+}
+
+UnwindLocation UnwindLocation::createAtCFAPlusOffset(int32_t Offset) {
+  return {CFAPlusOffset, InvalidRegisterNumber, Offset, std::nullopt, true};
+}
+
+UnwindLocation
+UnwindLocation::createIsRegisterPlusOffset(uint32_t RegNum, int32_t Offset,
+                                           std::optional<uint32_t> AddrSpace) {
+  return {RegPlusOffset, RegNum, Offset, AddrSpace, false};
+}
+
+UnwindLocation
+UnwindLocation::createAtRegisterPlusOffset(uint32_t RegNum, int32_t Offset,
+                                           std::optional<uint32_t> AddrSpace) {
+  return {RegPlusOffset, RegNum, Offset, AddrSpace, true};
+}
+
+UnwindLocation UnwindLocation::createIsDWARFExpression(DWARFExpression Expr) {
+  return {Expr, false};
+}
+
+UnwindLocation UnwindLocation::createAtDWARFExpression(DWARFExpression Expr) {
+  return {Expr, true};
+}
+
+bool UnwindLocation::operator==(const UnwindLocation &RHS) const {
+  if (Kind != RHS.Kind)
+    return false;
+  switch (Kind) {
+  case Unspecified:
+  case Undefined:
+  case Same:
+    return true;
+  case CFAPlusOffset:
+    return Offset == RHS.Offset && Dereference == RHS.Dereference;
+  case RegPlusOffset:
+    return RegNum == RHS.RegNum && Offset == RHS.Offset &&
+           Dereference == RHS.Dereference;
+  case DWARFExpr:
+    return *Expr == *RHS.Expr && Dereference == RHS.Dereference;
+  case Constant:
+    return Offset == RHS.Offset;
+  }
+  return false;
+}
+
+Expected<UnwindTable::RowContainer>
+llvm::dwarf::parseRows(const CFIProgram &CFIP, UnwindRow &Row,
+                       const RegisterLocations *InitialLocs) {
+  // All the unwinding rows parsed during processing of the CFI program.
+  UnwindTable::RowContainer Rows;
+
+  // State consists of CFA value and register locations.
+  std::vector<std::pair<UnwindLocation, RegisterLocations>> States;
+  for (const CFIProgram::Instruction &Inst : CFIP) {
+    switch (Inst.Opcode) {
+    case dwarf::DW_CFA_set_loc: {
+      // The DW_CFA_set_loc instruction takes a single operand that
+      // represents a target address. The required action is to create a new
+      // table row using the specified address as the location. All other
+      // values in the new row are initially identical to the current row.
+      // The new location value is always greater than the current one. If
+      // the segment_size field of this FDE's CIE is non- zero, the initial
+      // location is preceded by a segment selector of the given length
+      llvm::Expected<uint64_t> NewAddress = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!NewAddress)
+        return NewAddress.takeError();
+      if (*NewAddress <= Row.getAddress())
+        return createStringError(
+            errc::invalid_argument,
+            "%s with adrress 0x%" PRIx64 " which must be greater than the "
+            "current row address 0x%" PRIx64,
+            CFIP.callFrameString(Inst.Opcode).str().c_str(), *NewAddress,
+            Row.getAddress());
+      Rows.push_back(Row);
+      Row.setAddress(*NewAddress);
+      break;
+    }
+
+    case dwarf::DW_CFA_advance_loc:
+    case dwarf::DW_CFA_advance_loc1:
+    case dwarf::DW_CFA_advance_loc2:
+    case dwarf::DW_CFA_advance_loc4: {
+      // The DW_CFA_advance instruction takes a single operand that
+      // represents a constant delta. The required action is to create a new
+      // table row with a location value that is computed by taking the
+      // current entry’s location value and adding the value of delta *
+      // code_alignment_factor. All other values in the new row are initially
+      // identical to the current row.
+      Rows.push_back(Row);
+      llvm::Expected<uint64_t> Offset = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!Offset)
+        return Offset.takeError();
+      Row.slideAddress(*Offset);
+      break;
+    }
+
+    case dwarf::DW_CFA_restore:
+    case dwarf::DW_CFA_restore_extended: {
+      // The DW_CFA_restore instruction takes a single operand (encoded with
+      // the opcode) that represents a register number. The required action
+      // is to change the rule for the indicated register to the rule
+      // assigned it by the initial_instructions in the CIE.
+      if (InitialLocs == nullptr)
+        return createStringError(
+            errc::invalid_argument, "%s encountered while parsing a CIE",
+            CFIP.callFrameString(Inst.Opcode).str().c_str());
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      if (std::optional<UnwindLocation> O =
+              InitialLocs->getRegisterLocation(*RegNum))
+        Row.getRegisterLocations().setRegisterLocation(*RegNum, *O);
+      else
+        Row.getRegisterLocations().removeRegisterLocation(*RegNum);
+      break;
+    }
+
+    case dwarf::DW_CFA_offset:
+    case dwarf::DW_CFA_offset_extended:
+    case dwarf::DW_CFA_offset_extended_sf: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
+      if (!Offset)
+        return Offset.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createAtCFAPlusOffset(*Offset));
+      break;
+    }
+
+    case dwarf::DW_CFA_nop:
+      break;
+
+    case dwarf::DW_CFA_remember_state:
+      States.push_back(
+          std::make_pair(Row.getCFAValue(), Row.getRegisterLocations()));
+      break;
+
+    case dwarf::DW_CFA_restore_state:
+      if (States.empty())
+        return createStringError(errc::invalid_argument,
+                                 "DW_CFA_restore_state without a matching "
+                                 "previous DW_CFA_remember_state");
+      Row.getCFAValue() = States.back().first;
+      Row.getRegisterLocations() = States.back().second;
+      States.pop_back();
+      break;
+
+    case dwarf::DW_CFA_GNU_window_save:
+      switch (CFIP.triple()) {
+      case Triple::aarch64:
+      case Triple::aarch64_be:
+      case Triple::aarch64_32: {
+        // DW_CFA_GNU_window_save is used for different things on different
+        // architectures. For aarch64 it is known as
+        // DW_CFA_AARCH64_negate_ra_state. The action is to toggle the
+        // value of the return address state between 1 and 0. If there is
+        // no rule for the AARCH64_DWARF_PAUTH_RA_STATE register, then it
+        // should be initially set to 1.
+        constexpr uint32_t AArch64DWARFPAuthRaState = 34;
+        auto LRLoc = Row.getRegisterLocations().getRegisterLocation(
+            AArch64DWARFPAuthRaState);
+        if (LRLoc) {
+          if (LRLoc->getLocation() == UnwindLocation::Constant) {
+            // Toggle the constant value from 0 to 1 or 1 to 0.
+            LRLoc->setConstant(LRLoc->getConstant() ^ 1);
+            Row.getRegisterLocations().setRegisterLocation(
+                AArch64DWARFPAuthRaState, *LRLoc);
+          } else {
+            return createStringError(
+                errc::invalid_argument,
+                "%s encountered when existing rule for this register is not "
+                "a constant",
+                CFIP.callFrameString(Inst.Opcode).str().c_str());
+          }
+        } else {
+          Row.getRegisterLocations().setRegisterLocation(
+              AArch64DWARFPAuthRaState, UnwindLocation::createIsConstant(1));
+        }
+        break;
+      }
+
+      case Triple::sparc:
+      case Triple::sparcv9:
+      case Triple::sparcel:
+        for (uint32_t RegNum = 16; RegNum < 32; ++RegNum) {
+          Row.getRegisterLocations().setRegisterLocation(
+              RegNum, UnwindLocation::createAtCFAPlusOffset((RegNum - 16) * 8));
+        }
+        break;
+
+      default: {
+        return createStringError(
+            errc::not_supported,
+            "DW_CFA opcode %#x is not supported for architecture %s",
+            Inst.Opcode, Triple::getArchTypeName(CFIP.triple()).str().c_str());
+
+        break;
+      }
+      }
+      break;
+
+    case dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc: {
+      constexpr uint32_t AArch64DWARFPAuthRaState = 34;
+      auto LRLoc = Row.getRegisterLocations().getRegisterLocation(
+          AArch64DWARFPAuthRaState);
+      if (LRLoc) {
+        if (LRLoc->getLocation() == UnwindLocation::Constant) {
+          // Toggle the constant value of bits[1:0] from 0 to 1 or 1 to 0.
+          LRLoc->setConstant(LRLoc->getConstant() ^ 0x3);
+        } else {
+          return createStringError(
+              errc::invalid_argument,
+              "%s encountered when existing rule for this register is not "
+              "a constant",
+              CFIP.callFrameString(Inst.Opcode).str().c_str());
+        }
+      } else {
+        Row.getRegisterLocations().setRegisterLocation(
+            AArch64DWARFPAuthRaState, UnwindLocation::createIsConstant(0x3));
+      }
+      break;
+    }
+
+    case dwarf::DW_CFA_undefined: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createUndefined());
+      break;
+    }
+
+    case dwarf::DW_CFA_same_value: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createSame());
+      break;
+    }
+
+    case dwarf::DW_CFA_GNU_args_size:
+      break;
+
+    case dwarf::DW_CFA_register: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      llvm::Expected<uint64_t> NewRegNum = Inst.getOperandAsUnsigned(CFIP, 1);
+      if (!NewRegNum)
+        return NewRegNum.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createIsRegisterPlusOffset(*NewRegNum, 0));
+      break;
+    }
+
+    case dwarf::DW_CFA_val_offset:
+    case dwarf::DW_CFA_val_offset_sf: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
+      if (!Offset)
+        return Offset.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createIsCFAPlusOffset(*Offset));
+      break;
+    }
+
+    case dwarf::DW_CFA_expression: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createAtDWARFExpression(*Inst.Expression));
+      break;
+    }
+
+    case dwarf::DW_CFA_val_expression: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      Row.getRegisterLocations().setRegisterLocation(
+          *RegNum, UnwindLocation::createIsDWARFExpression(*Inst.Expression));
+      break;
+    }
+
+    case dwarf::DW_CFA_def_cfa_register: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      if (Row.getCFAValue().getLocation() != UnwindLocation::RegPlusOffset)
+        Row.getCFAValue() =
+            UnwindLocation::createIsRegisterPlusOffset(*RegNum, 0);
+      else
+        Row.getCFAValue().setRegister(*RegNum);
+      break;
+    }
+
+    case dwarf::DW_CFA_def_cfa_offset:
+    case dwarf::DW_CFA_def_cfa_offset_sf: {
+      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 0);
+      if (!Offset)
+        return Offset.takeError();
+      if (Row.getCFAValue().getLocation() != UnwindLocation::RegPlusOffset) {
+        return createStringError(
+            errc::invalid_argument,
+            "%s found when CFA rule was not RegPlusOffset",
+            CFIP.callFrameString(Inst.Opcode).str().c_str());
+      }
+      Row.getCFAValue().setOffset(*Offset);
+      break;
+    }
+
+    case dwarf::DW_CFA_def_cfa:
+    case dwarf::DW_CFA_def_cfa_sf: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
+      if (!Offset)
+        return Offset.takeError();
+      Row.getCFAValue() =
+          UnwindLocation::createIsRegisterPlusOffset(*RegNum, *Offset);
+      break;
+    }
+
+    case dwarf::DW_CFA_LLVM_def_aspace_cfa:
+    case dwarf::DW_CFA_LLVM_def_aspace_cfa_sf: {
+      llvm::Expected<uint64_t> RegNum = Inst.getOperandAsUnsigned(CFIP, 0);
+      if (!RegNum)
+        return RegNum.takeError();
+      llvm::Expected<int64_t> Offset = Inst.getOperandAsSigned(CFIP, 1);
+      if (!Offset)
+        return Offset.takeError();
+      llvm::Expected<uint32_t> CFAAddrSpace =
+          Inst.getOperandAsUnsigned(CFIP, 2);
+      if (!CFAAddrSpace)
+        return CFAAddrSpace.takeError();
+      Row.getCFAValue() = UnwindLocation::createIsRegisterPlusOffset(
+          *RegNum, *Offset, *CFAAddrSpace);
+      break;
+    }
+
+    case dwarf::DW_CFA_def_cfa_expression:
+      Row.getCFAValue() =
+          UnwindLocation::createIsDWARFExpression(*Inst.Expression);
+      break;
+    }
+  }
+  return Rows;
+}
diff --git a/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt b/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt
index 67ee7698a3825..462857ef93063 100644
--- a/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt
+++ b/llvm/lib/DebugInfo/LogicalView/CMakeLists.txt
@@ -47,7 +47,8 @@ add_llvm_component_library(LLVMDebugInfoLogicalView
   MC
   Support
   TargetParser
-  DebugInfoDWARF
   DebugInfoCodeView
+  DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   DebugInfoPDB
   )
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
index 55880fab5e88e..93a567e89f774 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
@@ -2097,7 +2097,7 @@ Error LVScopeRoot::doPrintMatches(bool Split, raw_ostream &OS,
     print(OS);
 
     for (LVScope *Scope : *Scopes) {
-      getReader().setCompileUnit(const_cast<LVScope *>(Scope));
+      getReader().setCompileUnit(Scope);
 
       // If 'Split', we use the scope name (CU name) as the ouput file; the
       // delimiters in the pathname, must be replaced by a normal character.
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
index 95b7007d119bc..696e2bc948a2e 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
@@ -14,7 +14,8 @@
 #include "llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVLine.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVScope.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVSymbol.h"
@@ -594,8 +595,7 @@ std::string LVDWARFReader::getRegisterName(LVSmall Opcode,
     return {};
   };
   DumpOpts.GetNameForDWARFReg = GetRegName;
-  DWARFExpressionPrinter::prettyPrintRegisterOp(/*U=*/nullptr, Stream, DumpOpts,
-                                                Opcode, Operands);
+  prettyPrintRegisterOp(/*U=*/nullptr, Stream, DumpOpts, Opcode, Operands);
   return Stream.str();
 }
 
diff --git a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
index 09cb355ca7319..c9a1eebb8a854 100644
--- a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
@@ -119,7 +119,7 @@ static CVSymbol serializePublic(uint8_t *Mem, const BulkPublic &Pub) {
   memcpy(NameMem, Pub.Name, NameLen);
   // Zero the null terminator and remaining bytes.
   memset(&NameMem[NameLen], 0, Size - sizeof(PublicSym32Layout) - NameLen);
-  return CVSymbol(ArrayRef(reinterpret_cast<uint8_t *>(Mem), Size));
+  return CVSymbol(ArrayRef(Mem, Size));
 }
 
 uint32_t GSIHashStreamBuilder::calculateSerializedLength() const {
diff --git a/llvm/lib/DebugInfo/PDB/UDTLayout.cpp b/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
index 679881054f9da..37d1a28b4f356 100644
--- a/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
+++ b/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
@@ -19,7 +19,6 @@
 #include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h"
-#include "llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypePointer.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h"
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index 9455a3847692b..01456d54465d2 100644
--- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -952,8 +952,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       if (CAZ) {
         GenericValue floatZero;
         floatZero.FloatVal = 0.f;
-        std::fill(Result.AggregateVal.begin(), Result.AggregateVal.end(),
-                  floatZero);
+        llvm::fill(Result.AggregateVal, floatZero);
         break;
       }
       if(CV) {
@@ -974,8 +973,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       if (CAZ) {
         GenericValue doubleZero;
         doubleZero.DoubleVal = 0.0;
-        std::fill(Result.AggregateVal.begin(), Result.AggregateVal.end(),
-                  doubleZero);
+        llvm::fill(Result.AggregateVal, doubleZero);
         break;
       }
       if(CV) {
@@ -996,8 +994,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       if (CAZ) {
         GenericValue intZero;
         intZero.IntVal = APInt(ElemTy->getScalarSizeInBits(), 0ull);
-        std::fill(Result.AggregateVal.begin(), Result.AggregateVal.end(),
-                  intZero);
+        llvm::fill(Result.AggregateVal, intZero);
         break;
       }
       if(CV) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index b617fe222df00..cd2eed748acde 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -523,7 +523,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
     case ELFTLSDescCall:
       return "ELFTLSDescCall";
     default:
-      return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+      return getGenericEdgeKindName(R);
     }
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 3af0c0cdeb7c3..308dac48cbf7a 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -545,7 +545,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
     case MachONegDelta64:
       return "MachONegDelta64";
     default:
-      return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+      return getGenericEdgeKindName(R);
     }
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index fbeb920f4e189..b19b839aab7fb 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -85,7 +85,7 @@ const char *getEdgeKindName(Edge::Kind R) {
   case RequestTLSDescEntryAndTransformToPageOffset12:
     return "RequestTLSDescEntryAndTransformToPageOffset12";
   default:
-    return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+    return getGenericEdgeKindName(R);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
index 27484aaf20590..3280c5eed525b 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
@@ -137,7 +137,7 @@ const char *getEdgeKindName(Edge::Kind K) {
   case RequestTLSDescInGOTAndTransformToDelta34:
     return "RequestTLSDescInGOTAndTransformToDelta34";
   default:
-    return getGenericEdgeKindName(static_cast<Edge::Kind>(K));
+    return getGenericEdgeKindName(K);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
index 6ac991651f082..9d312a281315a 100644
--- a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
@@ -75,7 +75,7 @@ const char *getEdgeKindName(Edge::Kind K) {
   case RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable:
     return "RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable";
   default:
-    return getGenericEdgeKindName(static_cast<Edge::Kind>(K));
+    return getGenericEdgeKindName(K);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 4e3c09e970cbe..67bb7dd8ad08f 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -632,16 +632,19 @@ Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
       int32_t result;
       auto E = ES.callSPSWrapper<SPSDLUpdateSig>(WrapperAddr->getAddress(),
                                                  result, DSOHandles[&JD]);
-      if (result)
+      if (E)
+        return E;
+      else if (result)
         return make_error<StringError>("dlupdate failed",
                                        inconvertibleErrorCode());
-      return E;
-    }
-    return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
-                                           DSOHandles[&JD], JD.getName(),
-                                           int32_t(ORC_RT_RTLD_LAZY));
+    } else
+      return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
+                                             DSOHandles[&JD], JD.getName(),
+                                             int32_t(ORC_RT_RTLD_LAZY));
   } else
     return WrapperAddr.takeError();
+
+  return Error::success();
 }
 
 Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
index 9999e1ff3bf00..fd805fbf01fb7 100644
--- a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
@@ -729,9 +729,9 @@ LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) {
   return wrap(new ThreadSafeContext(std::make_unique<LLVMContext>()));
 }
 
-LLVMContextRef
-LLVMOrcThreadSafeContextGetContext(LLVMOrcThreadSafeContextRef TSCtx) {
-  return wrap(unwrap(TSCtx)->getContext());
+LLVMOrcThreadSafeContextRef
+LLVMOrcCreateNewThreadSafeContextFromLLVMContext(LLVMContextRef Ctx) {
+  return wrap(new ThreadSafeContext(std::unique_ptr<LLVMContext>(unwrap(Ctx))));
 }
 
 void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx) {
diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
index 74b9eb29bdccf..fee94b96a9e8a 100644
--- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
@@ -60,8 +60,6 @@ void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
                               ThreadSafeModule TSM) {
 
   assert(TSM && "Speculation Layer received Null Module ?");
-  assert(TSM.getContext().getContext() != nullptr &&
-         "Module with null LLVMContext?");
 
   // Instrumentation of runtime calls, lock the Module
   TSM.withModuleDo([this, &R](Module &M) {
diff --git a/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
index 2e128dd237443..19c000e2472a8 100644
--- a/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
@@ -1,5 +1,4 @@
-//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities
-//h-===//
+//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -15,49 +14,63 @@
 namespace llvm {
 namespace orc {
 
-ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM,
-                                   GVPredicate ShouldCloneDef,
-                                   GVModifier UpdateClonedDefSource) {
+ThreadSafeModule cloneToContext(const ThreadSafeModule &TSM,
+                                ThreadSafeContext TSCtx,
+                                GVPredicate ShouldCloneDef,
+                                GVModifier UpdateClonedDefSource) {
   assert(TSM && "Can not clone null module");
 
   if (!ShouldCloneDef)
     ShouldCloneDef = [](const GlobalValue &) { return true; };
 
-  return TSM.withModuleDo([&](Module &M) {
-    SmallVector<char, 1> ClonedModuleBuffer;
+  // First copy the source module into a buffer.
+  std::string ModuleName;
+  SmallVector<char, 1> ClonedModuleBuffer;
+  TSM.withModuleDo([&](Module &M) {
+    ModuleName = M.getModuleIdentifier();
+    std::set<GlobalValue *> ClonedDefsInSrc;
+    ValueToValueMapTy VMap;
+    auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
+      if (ShouldCloneDef(*GV)) {
+        ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
+        return true;
+      }
+      return false;
+    });
 
-    {
-      std::set<GlobalValue *> ClonedDefsInSrc;
-      ValueToValueMapTy VMap;
-      auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
-        if (ShouldCloneDef(*GV)) {
-          ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
-          return true;
-        }
-        return false;
-      });
+    if (UpdateClonedDefSource)
+      for (auto *GV : ClonedDefsInSrc)
+        UpdateClonedDefSource(*GV);
 
-      if (UpdateClonedDefSource)
-        for (auto *GV : ClonedDefsInSrc)
-          UpdateClonedDefSource(*GV);
+    BitcodeWriter BCWriter(ClonedModuleBuffer);
+    BCWriter.writeModule(*Tmp);
+    BCWriter.writeSymtab();
+    BCWriter.writeStrtab();
+  });
+
+  MemoryBufferRef ClonedModuleBufferRef(
+      StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
+      "cloned module buffer");
 
-      BitcodeWriter BCWriter(ClonedModuleBuffer);
+  // Then parse the buffer into the new Module.
+  auto M = TSCtx.withContextDo([&](LLVMContext *Ctx) {
+    assert(Ctx && "No LLVMContext provided");
+    auto TmpM = cantFail(parseBitcodeFile(ClonedModuleBufferRef, *Ctx));
+    TmpM->setModuleIdentifier(ModuleName);
+    return TmpM;
+  });
 
-      BCWriter.writeModule(*Tmp);
-      BCWriter.writeSymtab();
-      BCWriter.writeStrtab();
-    }
+  return ThreadSafeModule(std::move(M), std::move(TSCtx));
+}
 
-    MemoryBufferRef ClonedModuleBufferRef(
-        StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
-        "cloned module buffer");
-    ThreadSafeContext NewTSCtx(std::make_unique<LLVMContext>());
+ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM,
+                                   GVPredicate ShouldCloneDef,
+                                   GVModifier UpdateClonedDefSource) {
+  assert(TSM && "Can not clone null module");
 
-    auto ClonedModule = cantFail(
-        parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext()));
-    ClonedModule->setModuleIdentifier(M.getName());
-    return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx));
-  });
+  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
+  return cloneToContext(TSM, std::move(TSCtx), std::move(ShouldCloneDef),
+                        std::move(UpdateClonedDefSource));
 }
 
 } // end namespace orc
diff --git a/llvm/lib/FileCheck/FileCheck.cpp b/llvm/lib/FileCheck/FileCheck.cpp
index bcca499322aee..b79f6ec5b4f04 100644
--- a/llvm/lib/FileCheck/FileCheck.cpp
+++ b/llvm/lib/FileCheck/FileCheck.cpp
@@ -264,7 +264,7 @@ BinaryOperation::getImplicitFormat(const SourceMgr &SM) const {
                                                          : *RightFormat;
 }
 
-Expected<std::string> NumericSubstitution::getResult() const {
+Expected<std::string> NumericSubstitution::getResultRegex() const {
   assert(ExpressionPointer->getAST() != nullptr &&
          "Substituting empty expression");
   Expected<APInt> EvaluatedValue = ExpressionPointer->getAST()->eval();
@@ -274,7 +274,18 @@ Expected<std::string> NumericSubstitution::getResult() const {
   return Format.getMatchingString(*EvaluatedValue);
 }
 
-Expected<std::string> StringSubstitution::getResult() const {
+Expected<std::string> NumericSubstitution::getResultForDiagnostics() const {
+  // The "regex" returned by getResultRegex() is just a numeric value
+  // like '42', '0x2A', '-17', 'DEADBEEF' etc. This is already suitable for use
+  // in diagnostics.
+  Expected<std::string> Literal = getResultRegex();
+  if (!Literal)
+    return Literal;
+
+  return "\"" + std::move(*Literal) + "\"";
+}
+
+Expected<std::string> StringSubstitution::getResultRegex() const {
   // Look up the value and escape it so that we can put it into the regex.
   Expected<StringRef> VarVal = Context->getPatternVarValue(FromStr);
   if (!VarVal)
@@ -282,6 +293,36 @@ Expected<std::string> StringSubstitution::getResult() const {
   return Regex::escape(*VarVal);
 }
 
+Expected<std::string> StringSubstitution::getResultForDiagnostics() const {
+  Expected<StringRef> VarVal = Context->getPatternVarValue(FromStr);
+  if (!VarVal)
+    return VarVal.takeError();
+
+  std::string Result;
+  Result.reserve(VarVal->size() + 2);
+  raw_string_ostream OS(Result);
+
+  OS << '"';
+  // Escape the string if it contains any characters that
+  // make it hard to read, such as non-printable characters (including all
+  // whitespace except space) and double quotes. These are the characters that
+  // are escaped by write_escaped(), except we do not include backslashes,
+  // because they are common in Windows paths and escaping them would make the
+  // output harder to read. However, when we do escape, backslashes are escaped
+  // as well, otherwise the output would be ambiguous.
+  const bool NeedsEscaping =
+      llvm::any_of(*VarVal, [](char C) { return !isPrint(C) || C == '"'; });
+  if (NeedsEscaping)
+    OS.write_escaped(*VarVal);
+  else
+    OS << *VarVal;
+  OS << '"';
+  if (NeedsEscaping)
+    OS << " (escaped value)";
+
+  return Result;
+}
+
 bool Pattern::isValidVarNameStart(char C) { return C == '_' || isAlpha(C); }
 
 Expected<Pattern::VariableProperties>
@@ -1106,7 +1147,7 @@ Pattern::MatchResult Pattern::match(StringRef Buffer,
     Error Errs = Error::success();
     for (const auto &Substitution : Substitutions) {
       // Substitute and check for failure (e.g. use of undefined variable).
-      Expected<std::string> Value = Substitution->getResult();
+      Expected<std::string> Value = Substitution->getResultRegex();
       if (!Value) {
         // Convert to an ErrorDiagnostic to get location information. This is
         // done here rather than printMatch/printNoMatch since now we know which
@@ -1210,7 +1251,8 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
       SmallString<256> Msg;
       raw_svector_ostream OS(Msg);
 
-      Expected<std::string> MatchedValue = Substitution->getResult();
+      Expected<std::string> MatchedValue =
+          Substitution->getResultForDiagnostics();
       // Substitution failures are handled in printNoMatch().
       if (!MatchedValue) {
         consumeError(MatchedValue.takeError());
@@ -1218,8 +1260,8 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
       }
 
       OS << "with \"";
-      OS.write_escaped(Substitution->getFromString()) << "\" equal to \"";
-      OS.write_escaped(*MatchedValue) << "\"";
+      OS.write_escaped(Substitution->getFromString()) << "\" equal to ";
+      OS << *MatchedValue;
 
       // We report only the start of the match/search range to suggest we are
       // reporting the substitutions as set at the start of the match/search.
diff --git a/llvm/lib/FileCheck/FileCheckImpl.h b/llvm/lib/FileCheck/FileCheckImpl.h
index b3cd2af5d57e3..5cf5548cdfde1 100644
--- a/llvm/lib/FileCheck/FileCheckImpl.h
+++ b/llvm/lib/FileCheck/FileCheckImpl.h
@@ -366,9 +366,15 @@ class Substitution {
   /// \returns the index where the substitution is to be performed in RegExStr.
   size_t getIndex() const { return InsertIdx; }
 
+  /// \returns a regular expression string that matches the result of the
+  /// substitution represented by this class instance or an error if
+  /// substitution failed.
+  virtual Expected<std::string> getResultRegex() const = 0;
+
   /// \returns a string containing the result of the substitution represented
-  /// by this class instance or an error if substitution failed.
-  virtual Expected<std::string> getResult() const = 0;
+  /// by this class instance in a form suitable for diagnostics, or an error if
+  /// substitution failed.
+  virtual Expected<std::string> getResultForDiagnostics() const = 0;
 };
 
 class StringSubstitution : public Substitution {
@@ -379,7 +385,12 @@ class StringSubstitution : public Substitution {
 
   /// \returns the text that the string variable in this substitution matched
   /// when defined, or an error if the variable is undefined.
-  Expected<std::string> getResult() const override;
+  Expected<std::string> getResultRegex() const override;
+
+  /// \returns the text that the string variable in this substitution matched
+  /// when defined, in a form suitable for diagnostics, or an error if the
+  /// variable is undefined.
+  Expected<std::string> getResultForDiagnostics() const override;
 };
 
 class NumericSubstitution : public Substitution {
@@ -397,7 +408,12 @@ class NumericSubstitution : public Substitution {
 
   /// \returns a string containing the result of evaluating the expression in
   /// this substitution, or an error if evaluation failed.
-  Expected<std::string> getResult() const override;
+  Expected<std::string> getResultRegex() const override;
+
+  /// \returns a string containing the result of evaluating the expression in
+  /// this substitution, in a form suitable for diagnostics, or an error if
+  /// evaluation failed.
+  Expected<std::string> getResultForDiagnostics() const override;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Frontend/HLSL/CMakeLists.txt b/llvm/lib/Frontend/HLSL/CMakeLists.txt
index dfebe354807bf..534346920ff19 100644
--- a/llvm/lib/Frontend/HLSL/CMakeLists.txt
+++ b/llvm/lib/Frontend/HLSL/CMakeLists.txt
@@ -1,7 +1,9 @@
 add_llvm_component_library(LLVMFrontendHLSL
   CBuffer.cpp
   HLSLResource.cpp
-  HLSLRootSignatureUtils.cpp
+  HLSLRootSignature.cpp
+  RootSignatureMetadata.cpp
+  RootSignatureValidations.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend
@@ -11,6 +13,7 @@ add_llvm_component_library(LLVMFrontendHLSL
   intrinsics_gen
 
   LINK_COMPONENTS
+  BinaryFormat
   Core
   Support
   )
diff --git a/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp b/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
new file mode 100644
index 0000000000000..78c20a6c5c9ff
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
@@ -0,0 +1,244 @@
+//===- HLSLRootSignature.cpp - HLSL Root Signature helpers ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains helpers for working with HLSL Root Signatures.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+namespace llvm {
+namespace hlsl {
+namespace rootsig {
+
+template <typename T>
+static std::optional<StringRef> getEnumName(const T Value,
+                                            ArrayRef<EnumEntry<T>> Enums) {
+  for (const auto &EnumItem : Enums)
+    if (EnumItem.Value == Value)
+      return EnumItem.Name;
+  return std::nullopt;
+}
+
+template <typename T>
+static raw_ostream &printEnum(raw_ostream &OS, const T Value,
+                              ArrayRef<EnumEntry<T>> Enums) {
+  auto MaybeName = getEnumName(Value, Enums);
+  if (MaybeName)
+    OS << *MaybeName;
+  return OS;
+}
+
+template <typename T>
+static raw_ostream &printFlags(raw_ostream &OS, const T Value,
+                               ArrayRef<EnumEntry<T>> Flags) {
+  bool FlagSet = false;
+  unsigned Remaining = llvm::to_underlying(Value);
+  while (Remaining) {
+    unsigned Bit = 1u << llvm::countr_zero(Remaining);
+    if (Remaining & Bit) {
+      if (FlagSet)
+        OS << " | ";
+
+      auto MaybeFlag = getEnumName(T(Bit), Flags);
+      if (MaybeFlag)
+        OS << *MaybeFlag;
+      else
+        OS << "invalid: " << Bit;
+
+      FlagSet = true;
+    }
+    Remaining &= ~Bit;
+  }
+
+  if (!FlagSet)
+    OS << "None";
+  return OS;
+}
+
+static const EnumEntry<RegisterType> RegisterNames[] = {
+    {"b", RegisterType::BReg},
+    {"t", RegisterType::TReg},
+    {"u", RegisterType::UReg},
+    {"s", RegisterType::SReg},
+};
+
+static raw_ostream &operator<<(raw_ostream &OS, const Register &Reg) {
+  printEnum(OS, Reg.ViewType, ArrayRef(RegisterNames));
+  OS << Reg.Number;
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const llvm::dxbc::ShaderVisibility &Visibility) {
+  printEnum(OS, Visibility, dxbc::getShaderVisibility());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const llvm::dxbc::SamplerFilter &Filter) {
+  printEnum(OS, Filter, dxbc::getSamplerFilters());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::TextureAddressMode &Address) {
+  printEnum(OS, Address, dxbc::getTextureAddressModes());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::ComparisonFunc &CompFunc) {
+  printEnum(OS, CompFunc, dxbc::getComparisonFuncs());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::StaticBorderColor &BorderColor) {
+  printEnum(OS, BorderColor, dxbc::getStaticBorderColors());
+
+  return OS;
+}
+
+static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
+    {"CBV", dxil::ResourceClass::CBuffer},
+    {"SRV", dxil::ResourceClass::SRV},
+    {"UAV", dxil::ResourceClass::UAV},
+    {"Sampler", dxil::ResourceClass::Sampler},
+};
+
+static raw_ostream &operator<<(raw_ostream &OS, const ClauseType &Type) {
+  printEnum(OS, dxil::ResourceClass(llvm::to_underlying(Type)),
+            ArrayRef(ResourceClassNames));
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::RootDescriptorFlags &Flags) {
+  printFlags(OS, Flags, dxbc::getRootDescriptorFlags());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const llvm::dxbc::DescriptorRangeFlags &Flags) {
+  printFlags(OS, Flags, dxbc::getDescriptorRangeFlags());
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const dxbc::RootFlags &Flags) {
+  OS << "RootFlags(";
+  printFlags(OS, Flags, dxbc::getRootFlags());
+  OS << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const RootConstants &Constants) {
+  OS << "RootConstants(num32BitConstants = " << Constants.Num32BitConstants
+     << ", " << Constants.Reg << ", space = " << Constants.Space
+     << ", visibility = " << Constants.Visibility << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table) {
+  OS << "DescriptorTable(numClauses = " << Table.NumClauses
+     << ", visibility = " << Table.Visibility << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const DescriptorTableClause &Clause) {
+  OS << Clause.Type << "(" << Clause.Reg << ", numDescriptors = ";
+  if (Clause.NumDescriptors == NumDescriptorsUnbounded)
+    OS << "unbounded";
+  else
+    OS << Clause.NumDescriptors;
+  OS << ", space = " << Clause.Space << ", offset = ";
+  if (Clause.Offset == DescriptorTableOffsetAppend)
+    OS << "DescriptorTableOffsetAppend";
+  else
+    OS << Clause.Offset;
+  OS << ", flags = " << Clause.Flags << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const RootDescriptor &Descriptor) {
+  ClauseType Type = ClauseType(llvm::to_underlying(Descriptor.Type));
+  OS << "Root" << Type << "(" << Descriptor.Reg
+     << ", space = " << Descriptor.Space
+     << ", visibility = " << Descriptor.Visibility
+     << ", flags = " << Descriptor.Flags << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const StaticSampler &Sampler) {
+  OS << "StaticSampler(" << Sampler.Reg << ", filter = " << Sampler.Filter
+     << ", addressU = " << Sampler.AddressU
+     << ", addressV = " << Sampler.AddressV
+     << ", addressW = " << Sampler.AddressW
+     << ", mipLODBias = " << Sampler.MipLODBias
+     << ", maxAnisotropy = " << Sampler.MaxAnisotropy
+     << ", comparisonFunc = " << Sampler.CompFunc
+     << ", borderColor = " << Sampler.BorderColor
+     << ", minLOD = " << Sampler.MinLOD << ", maxLOD = " << Sampler.MaxLOD
+     << ", space = " << Sampler.Space << ", visibility = " << Sampler.Visibility
+     << ")";
+  return OS;
+}
+
+namespace {
+
+// We use the OverloadVisit with std::visit to ensure the compiler catches if a
+// new RootElement variant type is added but it's operator<< isn't handled.
+template <class... Ts> struct OverloadedVisit : Ts... {
+  using Ts::operator()...;
+};
+template <class... Ts> OverloadedVisit(Ts...) -> OverloadedVisit<Ts...>;
+
+} // namespace
+
+raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element) {
+  const auto Visitor = OverloadedVisit{
+      [&OS](const dxbc::RootFlags &Flags) { OS << Flags; },
+      [&OS](const RootConstants &Constants) { OS << Constants; },
+      [&OS](const RootDescriptor &Descriptor) { OS << Descriptor; },
+      [&OS](const DescriptorTableClause &Clause) { OS << Clause; },
+      [&OS](const DescriptorTable &Table) { OS << Table; },
+      [&OS](const StaticSampler &Sampler) { OS << Sampler; },
+  };
+  std::visit(Visitor, Element);
+  return OS;
+}
+
+void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements) {
+  OS << " RootElements{";
+  bool First = true;
+  for (const RootElement &Element : Elements) {
+    if (!First)
+      OS << ",";
+    OS << " " << Element;
+    First = false;
+  }
+  OS << "}";
+}
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
diff --git a/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp b/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
deleted file mode 100644
index 22a761ba3b135..0000000000000
--- a/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
+++ /dev/null
@@ -1,577 +0,0 @@
-//===- HLSLRootSignatureUtils.cpp - HLSL Root Signature helpers -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file This file contains helpers for working with HLSL Root Signatures.
-///
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/bit.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Metadata.h"
-#include "llvm/Support/ScopedPrinter.h"
-
-namespace llvm {
-namespace hlsl {
-namespace rootsig {
-
-template <typename T>
-static std::optional<StringRef> getEnumName(const T Value,
-                                            ArrayRef<EnumEntry<T>> Enums) {
-  for (const auto &EnumItem : Enums)
-    if (EnumItem.Value == Value)
-      return EnumItem.Name;
-  return std::nullopt;
-}
-
-template <typename T>
-static raw_ostream &printEnum(raw_ostream &OS, const T Value,
-                              ArrayRef<EnumEntry<T>> Enums) {
-  auto MaybeName = getEnumName(Value, Enums);
-  if (MaybeName)
-    OS << *MaybeName;
-  return OS;
-}
-
-template <typename T>
-static raw_ostream &printFlags(raw_ostream &OS, const T Value,
-                               ArrayRef<EnumEntry<T>> Flags) {
-  bool FlagSet = false;
-  unsigned Remaining = llvm::to_underlying(Value);
-  while (Remaining) {
-    unsigned Bit = 1u << llvm::countr_zero(Remaining);
-    if (Remaining & Bit) {
-      if (FlagSet)
-        OS << " | ";
-
-      auto MaybeFlag = getEnumName(T(Bit), Flags);
-      if (MaybeFlag)
-        OS << *MaybeFlag;
-      else
-        OS << "invalid: " << Bit;
-
-      FlagSet = true;
-    }
-    Remaining &= ~Bit;
-  }
-
-  if (!FlagSet)
-    OS << "None";
-  return OS;
-}
-
-static const EnumEntry<RegisterType> RegisterNames[] = {
-    {"b", RegisterType::BReg},
-    {"t", RegisterType::TReg},
-    {"u", RegisterType::UReg},
-    {"s", RegisterType::SReg},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const Register &Reg) {
-  printEnum(OS, Reg.ViewType, ArrayRef(RegisterNames));
-  OS << Reg.Number;
-
-  return OS;
-}
-
-static const EnumEntry<ShaderVisibility> VisibilityNames[] = {
-    {"All", ShaderVisibility::All},
-    {"Vertex", ShaderVisibility::Vertex},
-    {"Hull", ShaderVisibility::Hull},
-    {"Domain", ShaderVisibility::Domain},
-    {"Geometry", ShaderVisibility::Geometry},
-    {"Pixel", ShaderVisibility::Pixel},
-    {"Amplification", ShaderVisibility::Amplification},
-    {"Mesh", ShaderVisibility::Mesh},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const ShaderVisibility &Visibility) {
-  printEnum(OS, Visibility, ArrayRef(VisibilityNames));
-
-  return OS;
-}
-
-static const EnumEntry<SamplerFilter> SamplerFilterNames[] = {
-    {"MinMagMipPoint", SamplerFilter::MinMagMipPoint},
-    {"MinMagPointMipLinear", SamplerFilter::MinMagPointMipLinear},
-    {"MinPointMagLinearMipPoint", SamplerFilter::MinPointMagLinearMipPoint},
-    {"MinPointMagMipLinear", SamplerFilter::MinPointMagMipLinear},
-    {"MinLinearMagMipPoint", SamplerFilter::MinLinearMagMipPoint},
-    {"MinLinearMagPointMipLinear", SamplerFilter::MinLinearMagPointMipLinear},
-    {"MinMagLinearMipPoint", SamplerFilter::MinMagLinearMipPoint},
-    {"MinMagMipLinear", SamplerFilter::MinMagMipLinear},
-    {"Anisotropic", SamplerFilter::Anisotropic},
-    {"ComparisonMinMagMipPoint", SamplerFilter::ComparisonMinMagMipPoint},
-    {"ComparisonMinMagPointMipLinear",
-     SamplerFilter::ComparisonMinMagPointMipLinear},
-    {"ComparisonMinPointMagLinearMipPoint",
-     SamplerFilter::ComparisonMinPointMagLinearMipPoint},
-    {"ComparisonMinPointMagMipLinear",
-     SamplerFilter::ComparisonMinPointMagMipLinear},
-    {"ComparisonMinLinearMagMipPoint",
-     SamplerFilter::ComparisonMinLinearMagMipPoint},
-    {"ComparisonMinLinearMagPointMipLinear",
-     SamplerFilter::ComparisonMinLinearMagPointMipLinear},
-    {"ComparisonMinMagLinearMipPoint",
-     SamplerFilter::ComparisonMinMagLinearMipPoint},
-    {"ComparisonMinMagMipLinear", SamplerFilter::ComparisonMinMagMipLinear},
-    {"ComparisonAnisotropic", SamplerFilter::ComparisonAnisotropic},
-    {"MinimumMinMagMipPoint", SamplerFilter::MinimumMinMagMipPoint},
-    {"MinimumMinMagPointMipLinear", SamplerFilter::MinimumMinMagPointMipLinear},
-    {"MinimumMinPointMagLinearMipPoint",
-     SamplerFilter::MinimumMinPointMagLinearMipPoint},
-    {"MinimumMinPointMagMipLinear", SamplerFilter::MinimumMinPointMagMipLinear},
-    {"MinimumMinLinearMagMipPoint", SamplerFilter::MinimumMinLinearMagMipPoint},
-    {"MinimumMinLinearMagPointMipLinear",
-     SamplerFilter::MinimumMinLinearMagPointMipLinear},
-    {"MinimumMinMagLinearMipPoint", SamplerFilter::MinimumMinMagLinearMipPoint},
-    {"MinimumMinMagMipLinear", SamplerFilter::MinimumMinMagMipLinear},
-    {"MinimumAnisotropic", SamplerFilter::MinimumAnisotropic},
-    {"MaximumMinMagMipPoint", SamplerFilter::MaximumMinMagMipPoint},
-    {"MaximumMinMagPointMipLinear", SamplerFilter::MaximumMinMagPointMipLinear},
-    {"MaximumMinPointMagLinearMipPoint",
-     SamplerFilter::MaximumMinPointMagLinearMipPoint},
-    {"MaximumMinPointMagMipLinear", SamplerFilter::MaximumMinPointMagMipLinear},
-    {"MaximumMinLinearMagMipPoint", SamplerFilter::MaximumMinLinearMagMipPoint},
-    {"MaximumMinLinearMagPointMipLinear",
-     SamplerFilter::MaximumMinLinearMagPointMipLinear},
-    {"MaximumMinMagLinearMipPoint", SamplerFilter::MaximumMinMagLinearMipPoint},
-    {"MaximumMinMagMipLinear", SamplerFilter::MaximumMinMagMipLinear},
-    {"MaximumAnisotropic", SamplerFilter::MaximumAnisotropic},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const SamplerFilter &Filter) {
-  printEnum(OS, Filter, ArrayRef(SamplerFilterNames));
-
-  return OS;
-}
-
-static const EnumEntry<TextureAddressMode> TextureAddressModeNames[] = {
-    {"Wrap", TextureAddressMode::Wrap},
-    {"Mirror", TextureAddressMode::Mirror},
-    {"Clamp", TextureAddressMode::Clamp},
-    {"Border", TextureAddressMode::Border},
-    {"MirrorOnce", TextureAddressMode::MirrorOnce},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const TextureAddressMode &Address) {
-  printEnum(OS, Address, ArrayRef(TextureAddressModeNames));
-
-  return OS;
-}
-
-static const EnumEntry<ComparisonFunc> ComparisonFuncNames[] = {
-    {"Never", ComparisonFunc::Never},
-    {"Less", ComparisonFunc::Less},
-    {"Equal", ComparisonFunc::Equal},
-    {"LessEqual", ComparisonFunc::LessEqual},
-    {"Greater", ComparisonFunc::Greater},
-    {"NotEqual", ComparisonFunc::NotEqual},
-    {"GreaterEqual", ComparisonFunc::GreaterEqual},
-    {"Always", ComparisonFunc::Always},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const ComparisonFunc &CompFunc) {
-  printEnum(OS, CompFunc, ArrayRef(ComparisonFuncNames));
-
-  return OS;
-}
-
-static const EnumEntry<StaticBorderColor> StaticBorderColorNames[] = {
-    {"TransparentBlack", StaticBorderColor::TransparentBlack},
-    {"OpaqueBlack", StaticBorderColor::OpaqueBlack},
-    {"OpaqueWhite", StaticBorderColor::OpaqueWhite},
-    {"OpaqueBlackUint", StaticBorderColor::OpaqueBlackUint},
-    {"OpaqueWhiteUint", StaticBorderColor::OpaqueWhiteUint},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const StaticBorderColor &BorderColor) {
-  printEnum(OS, BorderColor, ArrayRef(StaticBorderColorNames));
-
-  return OS;
-}
-
-static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
-    {"CBV", dxil::ResourceClass::CBuffer},
-    {"SRV", dxil::ResourceClass::SRV},
-    {"UAV", dxil::ResourceClass::UAV},
-    {"Sampler", dxil::ResourceClass::Sampler},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const ClauseType &Type) {
-  printEnum(OS, dxil::ResourceClass(llvm::to_underlying(Type)),
-            ArrayRef(ResourceClassNames));
-
-  return OS;
-}
-
-static const EnumEntry<RootDescriptorFlags> RootDescriptorFlagNames[] = {
-    {"DataVolatile", RootDescriptorFlags::DataVolatile},
-    {"DataStaticWhileSetAtExecute",
-     RootDescriptorFlags::DataStaticWhileSetAtExecute},
-    {"DataStatic", RootDescriptorFlags::DataStatic},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const RootDescriptorFlags &Flags) {
-  printFlags(OS, Flags, ArrayRef(RootDescriptorFlagNames));
-
-  return OS;
-}
-
-static const EnumEntry<DescriptorRangeFlags> DescriptorRangeFlagNames[] = {
-    {"DescriptorsVolatile", DescriptorRangeFlags::DescriptorsVolatile},
-    {"DataVolatile", DescriptorRangeFlags::DataVolatile},
-    {"DataStaticWhileSetAtExecute",
-     DescriptorRangeFlags::DataStaticWhileSetAtExecute},
-    {"DataStatic", DescriptorRangeFlags::DataStatic},
-    {"DescriptorsStaticKeepingBufferBoundsChecks",
-     DescriptorRangeFlags::DescriptorsStaticKeepingBufferBoundsChecks},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const DescriptorRangeFlags &Flags) {
-  printFlags(OS, Flags, ArrayRef(DescriptorRangeFlagNames));
-
-  return OS;
-}
-
-static const EnumEntry<RootFlags> RootFlagNames[] = {
-    {"AllowInputAssemblerInputLayout",
-     RootFlags::AllowInputAssemblerInputLayout},
-    {"DenyVertexShaderRootAccess", RootFlags::DenyVertexShaderRootAccess},
-    {"DenyHullShaderRootAccess", RootFlags::DenyHullShaderRootAccess},
-    {"DenyDomainShaderRootAccess", RootFlags::DenyDomainShaderRootAccess},
-    {"DenyGeometryShaderRootAccess", RootFlags::DenyGeometryShaderRootAccess},
-    {"DenyPixelShaderRootAccess", RootFlags::DenyPixelShaderRootAccess},
-    {"AllowStreamOutput", RootFlags::AllowStreamOutput},
-    {"LocalRootSignature", RootFlags::LocalRootSignature},
-    {"DenyAmplificationShaderRootAccess",
-     RootFlags::DenyAmplificationShaderRootAccess},
-    {"DenyMeshShaderRootAccess", RootFlags::DenyMeshShaderRootAccess},
-    {"CBVSRVUAVHeapDirectlyIndexed", RootFlags::CBVSRVUAVHeapDirectlyIndexed},
-    {"SamplerHeapDirectlyIndexed", RootFlags::SamplerHeapDirectlyIndexed},
-};
-
-raw_ostream &operator<<(raw_ostream &OS, const RootFlags &Flags) {
-  OS << "RootFlags(";
-  printFlags(OS, Flags, ArrayRef(RootFlagNames));
-  OS << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const RootConstants &Constants) {
-  OS << "RootConstants(num32BitConstants = " << Constants.Num32BitConstants
-     << ", " << Constants.Reg << ", space = " << Constants.Space
-     << ", visibility = " << Constants.Visibility << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table) {
-  OS << "DescriptorTable(numClauses = " << Table.NumClauses
-     << ", visibility = " << Table.Visibility << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const DescriptorTableClause &Clause) {
-  OS << Clause.Type << "(" << Clause.Reg << ", numDescriptors = ";
-  if (Clause.NumDescriptors == NumDescriptorsUnbounded)
-    OS << "unbounded";
-  else
-    OS << Clause.NumDescriptors;
-  OS << ", space = " << Clause.Space << ", offset = ";
-  if (Clause.Offset == DescriptorTableOffsetAppend)
-    OS << "DescriptorTableOffsetAppend";
-  else
-    OS << Clause.Offset;
-  OS << ", flags = " << Clause.Flags << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const RootDescriptor &Descriptor) {
-  ClauseType Type = ClauseType(llvm::to_underlying(Descriptor.Type));
-  OS << "Root" << Type << "(" << Descriptor.Reg
-     << ", space = " << Descriptor.Space
-     << ", visibility = " << Descriptor.Visibility
-     << ", flags = " << Descriptor.Flags << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const StaticSampler &Sampler) {
-  OS << "StaticSampler(" << Sampler.Reg << ", filter = " << Sampler.Filter
-     << ", addressU = " << Sampler.AddressU
-     << ", addressV = " << Sampler.AddressV
-     << ", addressW = " << Sampler.AddressW
-     << ", mipLODBias = " << Sampler.MipLODBias
-     << ", maxAnisotropy = " << Sampler.MaxAnisotropy
-     << ", comparisonFunc = " << Sampler.CompFunc
-     << ", borderColor = " << Sampler.BorderColor
-     << ", minLOD = " << Sampler.MinLOD << ", maxLOD = " << Sampler.MaxLOD
-     << ", space = " << Sampler.Space << ", visibility = " << Sampler.Visibility
-     << ")";
-  return OS;
-}
-
-namespace {
-
-// We use the OverloadVisit with std::visit to ensure the compiler catches if a
-// new RootElement variant type is added but it's operator<< or metadata
-// generation isn't handled.
-template <class... Ts> struct OverloadedVisit : Ts... {
-  using Ts::operator()...;
-};
-template <class... Ts> OverloadedVisit(Ts...) -> OverloadedVisit<Ts...>;
-
-} // namespace
-
-raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element) {
-  const auto Visitor = OverloadedVisit{
-      [&OS](const RootFlags &Flags) { OS << Flags; },
-      [&OS](const RootConstants &Constants) { OS << Constants; },
-      [&OS](const RootDescriptor &Descriptor) { OS << Descriptor; },
-      [&OS](const DescriptorTableClause &Clause) { OS << Clause; },
-      [&OS](const DescriptorTable &Table) { OS << Table; },
-      [&OS](const StaticSampler &Sampler) { OS << Sampler; },
-  };
-  std::visit(Visitor, Element);
-  return OS;
-}
-
-void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements) {
-  OS << " RootElements{";
-  bool First = true;
-  for (const RootElement &Element : Elements) {
-    if (!First)
-      OS << ",";
-    OS << " " << Element;
-    First = false;
-  }
-  OS << "}";
-}
-
-MDNode *MetadataBuilder::BuildRootSignature() {
-  const auto Visitor = OverloadedVisit{
-      [this](const RootFlags &Flags) -> MDNode * {
-        return BuildRootFlags(Flags);
-      },
-      [this](const RootConstants &Constants) -> MDNode * {
-        return BuildRootConstants(Constants);
-      },
-      [this](const RootDescriptor &Descriptor) -> MDNode * {
-        return BuildRootDescriptor(Descriptor);
-      },
-      [this](const DescriptorTableClause &Clause) -> MDNode * {
-        return BuildDescriptorTableClause(Clause);
-      },
-      [this](const DescriptorTable &Table) -> MDNode * {
-        return BuildDescriptorTable(Table);
-      },
-      [this](const StaticSampler &Sampler) -> MDNode * {
-        return BuildStaticSampler(Sampler);
-      },
-  };
-
-  for (const RootElement &Element : Elements) {
-    MDNode *ElementMD = std::visit(Visitor, Element);
-    assert(ElementMD != nullptr &&
-           "Root Element must be initialized and validated");
-    GeneratedMetadata.push_back(ElementMD);
-  }
-
-  return MDNode::get(Ctx, GeneratedMetadata);
-}
-
-MDNode *MetadataBuilder::BuildRootFlags(const RootFlags &Flags) {
-  IRBuilder<> Builder(Ctx);
-  Metadata *Operands[] = {
-      MDString::get(Ctx, "RootFlags"),
-      ConstantAsMetadata::get(Builder.getInt32(llvm::to_underlying(Flags))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
-  IRBuilder<> Builder(Ctx);
-  Metadata *Operands[] = {
-      MDString::get(Ctx, "RootConstants"),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Constants.Visibility))),
-      ConstantAsMetadata::get(Builder.getInt32(Constants.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Constants.Space)),
-      ConstantAsMetadata::get(Builder.getInt32(Constants.Num32BitConstants)),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
-  IRBuilder<> Builder(Ctx);
-  std::optional<StringRef> TypeName =
-      getEnumName(dxil::ResourceClass(llvm::to_underlying(Descriptor.Type)),
-                  ArrayRef(ResourceClassNames));
-  assert(TypeName && "Provided an invalid Resource Class");
-  llvm::SmallString<7> Name({"Root", *TypeName});
-  Metadata *Operands[] = {
-      MDString::get(Ctx, Name),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Descriptor.Visibility))),
-      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Space)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Descriptor.Flags))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
-  IRBuilder<> Builder(Ctx);
-  SmallVector<Metadata *> TableOperands;
-  // Set the mandatory arguments
-  TableOperands.push_back(MDString::get(Ctx, "DescriptorTable"));
-  TableOperands.push_back(ConstantAsMetadata::get(
-      Builder.getInt32(llvm::to_underlying(Table.Visibility))));
-
-  // Remaining operands are references to the table's clauses. The in-memory
-  // representation of the Root Elements created from parsing will ensure that
-  // the previous N elements are the clauses for this table.
-  assert(Table.NumClauses <= GeneratedMetadata.size() &&
-         "Table expected all owned clauses to be generated already");
-  // So, add a refence to each clause to our operands
-  TableOperands.append(GeneratedMetadata.end() - Table.NumClauses,
-                       GeneratedMetadata.end());
-  // Then, remove those clauses from the general list of Root Elements
-  GeneratedMetadata.pop_back_n(Table.NumClauses);
-
-  return MDNode::get(Ctx, TableOperands);
-}
-
-MDNode *MetadataBuilder::BuildDescriptorTableClause(
-    const DescriptorTableClause &Clause) {
-  IRBuilder<> Builder(Ctx);
-  std::optional<StringRef> Name =
-      getEnumName(dxil::ResourceClass(llvm::to_underlying(Clause.Type)),
-                  ArrayRef(ResourceClassNames));
-  assert(Name && "Provided an invalid Resource Class");
-  Metadata *Operands[] = {
-      MDString::get(Ctx, *Name),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.NumDescriptors)),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.Space)),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.Offset)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Clause.Flags))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildStaticSampler(const StaticSampler &Sampler) {
-  IRBuilder<> Builder(Ctx);
-  Metadata *Operands[] = {
-      MDString::get(Ctx, "StaticSampler"),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.Filter))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.AddressU))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.AddressV))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.AddressW))),
-      ConstantAsMetadata::get(llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx),
-                                                    Sampler.MipLODBias)),
-      ConstantAsMetadata::get(Builder.getInt32(Sampler.MaxAnisotropy)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.CompFunc))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.BorderColor))),
-      ConstantAsMetadata::get(
-          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MinLOD)),
-      ConstantAsMetadata::get(
-          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MaxLOD)),
-      ConstantAsMetadata::get(Builder.getInt32(Sampler.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Sampler.Space)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.Visibility))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-std::optional<const RangeInfo *>
-ResourceRange::getOverlapping(const RangeInfo &Info) const {
-  MapT::const_iterator Interval = Intervals.find(Info.LowerBound);
-  if (!Interval.valid() || Info.UpperBound < Interval.start())
-    return std::nullopt;
-  return Interval.value();
-}
-
-const RangeInfo *ResourceRange::lookup(uint32_t X) const {
-  return Intervals.lookup(X, nullptr);
-}
-
-void ResourceRange::clear() { return Intervals.clear(); }
-
-std::optional<const RangeInfo *> ResourceRange::insert(const RangeInfo &Info) {
-  uint32_t LowerBound = Info.LowerBound;
-  uint32_t UpperBound = Info.UpperBound;
-
-  std::optional<const RangeInfo *> Res = std::nullopt;
-  MapT::iterator Interval = Intervals.begin();
-
-  while (true) {
-    if (UpperBound < LowerBound)
-      break;
-
-    Interval.advanceTo(LowerBound);
-    if (!Interval.valid()) // No interval found
-      break;
-
-    // Let Interval = [x;y] and [LowerBound;UpperBound] = [a;b] and note that
-    // a <= y implicitly from Intervals.find(LowerBound)
-    if (UpperBound < Interval.start())
-      break; // found interval does not overlap with inserted one
-
-    if (!Res.has_value()) // Update to be the first found intersection
-      Res = Interval.value();
-
-    if (Interval.start() <= LowerBound && UpperBound <= Interval.stop()) {
-      // x <= a <= b <= y implies that [a;b] is covered by [x;y]
-      //  -> so we don't need to insert this, report an overlap
-      return Res;
-    } else if (LowerBound <= Interval.start() &&
-               Interval.stop() <= UpperBound) {
-      // a <= x <= y <= b implies that [x;y] is covered by [a;b]
-      //  -> so remove the existing interval that we will cover with the
-      //  overwrite
-      Interval.erase();
-    } else if (LowerBound < Interval.start() && UpperBound <= Interval.stop()) {
-      // a < x <= b <= y implies that [a; x] is not covered but [x;b] is
-      //  -> so set b = x - 1 such that [a;x-1] is now the interval to insert
-      UpperBound = Interval.start() - 1;
-    } else if (Interval.start() <= LowerBound && Interval.stop() < UpperBound) {
-      // a < x <= b <= y implies that [y; b] is not covered but [a;y] is
-      //  -> so set a = y + 1 such that [y+1;b] is now the interval to insert
-      LowerBound = Interval.stop() + 1;
-    }
-  }
-
-  assert(LowerBound <= UpperBound && "Attempting to insert an empty interval");
-  Intervals.insert(LowerBound, UpperBound, &Info);
-  return Res;
-}
-
-} // namespace rootsig
-} // namespace hlsl
-} // namespace llvm
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
new file mode 100644
index 0000000000000..f7669f09dcecc
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
@@ -0,0 +1,194 @@
+//===- RootSignatureMetadata.h - HLSL Root Signature helpers --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file implements a library for working with HLSL Root Signatures
+/// and their metadata representation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+namespace llvm {
+namespace hlsl {
+namespace rootsig {
+
+static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
+    {"CBV", dxil::ResourceClass::CBuffer},
+    {"SRV", dxil::ResourceClass::SRV},
+    {"UAV", dxil::ResourceClass::UAV},
+    {"Sampler", dxil::ResourceClass::Sampler},
+};
+
+static std::optional<StringRef> getResourceName(dxil::ResourceClass Class) {
+  for (const auto &ClassEnum : ResourceClassNames)
+    if (ClassEnum.Value == Class)
+      return ClassEnum.Name;
+  return std::nullopt;
+}
+
+namespace {
+
+// We use the OverloadVisit with std::visit to ensure the compiler catches if a
+// new RootElement variant type is added but it's metadata generation isn't
+// handled.
+template <class... Ts> struct OverloadedVisit : Ts... {
+  using Ts::operator()...;
+};
+template <class... Ts> OverloadedVisit(Ts...) -> OverloadedVisit<Ts...>;
+
+} // namespace
+
+MDNode *MetadataBuilder::BuildRootSignature() {
+  const auto Visitor = OverloadedVisit{
+      [this](const dxbc::RootFlags &Flags) -> MDNode * {
+        return BuildRootFlags(Flags);
+      },
+      [this](const RootConstants &Constants) -> MDNode * {
+        return BuildRootConstants(Constants);
+      },
+      [this](const RootDescriptor &Descriptor) -> MDNode * {
+        return BuildRootDescriptor(Descriptor);
+      },
+      [this](const DescriptorTableClause &Clause) -> MDNode * {
+        return BuildDescriptorTableClause(Clause);
+      },
+      [this](const DescriptorTable &Table) -> MDNode * {
+        return BuildDescriptorTable(Table);
+      },
+      [this](const StaticSampler &Sampler) -> MDNode * {
+        return BuildStaticSampler(Sampler);
+      },
+  };
+
+  for (const RootElement &Element : Elements) {
+    MDNode *ElementMD = std::visit(Visitor, Element);
+    assert(ElementMD != nullptr &&
+           "Root Element must be initialized and validated");
+    GeneratedMetadata.push_back(ElementMD);
+  }
+
+  return MDNode::get(Ctx, GeneratedMetadata);
+}
+
+MDNode *MetadataBuilder::BuildRootFlags(const dxbc::RootFlags &Flags) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "RootFlags"),
+      ConstantAsMetadata::get(Builder.getInt32(llvm::to_underlying(Flags))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "RootConstants"),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Constants.Visibility))),
+      ConstantAsMetadata::get(Builder.getInt32(Constants.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Constants.Space)),
+      ConstantAsMetadata::get(Builder.getInt32(Constants.Num32BitConstants)),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
+  IRBuilder<> Builder(Ctx);
+  std::optional<StringRef> ResName = getResourceName(
+      dxil::ResourceClass(llvm::to_underlying(Descriptor.Type)));
+  assert(ResName && "Provided an invalid Resource Class");
+  llvm::SmallString<7> Name({"Root", *ResName});
+  Metadata *Operands[] = {
+      MDString::get(Ctx, Name),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Descriptor.Visibility))),
+      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Space)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Descriptor.Flags))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
+  IRBuilder<> Builder(Ctx);
+  SmallVector<Metadata *> TableOperands;
+  // Set the mandatory arguments
+  TableOperands.push_back(MDString::get(Ctx, "DescriptorTable"));
+  TableOperands.push_back(ConstantAsMetadata::get(
+      Builder.getInt32(llvm::to_underlying(Table.Visibility))));
+
+  // Remaining operands are references to the table's clauses. The in-memory
+  // representation of the Root Elements created from parsing will ensure that
+  // the previous N elements are the clauses for this table.
+  assert(Table.NumClauses <= GeneratedMetadata.size() &&
+         "Table expected all owned clauses to be generated already");
+  // So, add a refence to each clause to our operands
+  TableOperands.append(GeneratedMetadata.end() - Table.NumClauses,
+                       GeneratedMetadata.end());
+  // Then, remove those clauses from the general list of Root Elements
+  GeneratedMetadata.pop_back_n(Table.NumClauses);
+
+  return MDNode::get(Ctx, TableOperands);
+}
+
+MDNode *MetadataBuilder::BuildDescriptorTableClause(
+    const DescriptorTableClause &Clause) {
+  IRBuilder<> Builder(Ctx);
+  std::optional<StringRef> ResName =
+      getResourceName(dxil::ResourceClass(llvm::to_underlying(Clause.Type)));
+  assert(ResName && "Provided an invalid Resource Class");
+  Metadata *Operands[] = {
+      MDString::get(Ctx, *ResName),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.NumDescriptors)),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.Space)),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.Offset)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Clause.Flags))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildStaticSampler(const StaticSampler &Sampler) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "StaticSampler"),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.Filter))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressU))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressV))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressW))),
+      ConstantAsMetadata::get(llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx),
+                                                    Sampler.MipLODBias)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.MaxAnisotropy)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.CompFunc))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.BorderColor))),
+      ConstantAsMetadata::get(
+          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MinLOD)),
+      ConstantAsMetadata::get(
+          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MaxLOD)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.Space)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.Visibility))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
new file mode 100644
index 0000000000000..b5b5fc0c74d83
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
@@ -0,0 +1,227 @@
+//===- HLSLRootSignatureValidations.cpp - HLSL Root Signature helpers -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains helpers for working with HLSL Root Signatures.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
+
+#include <cmath>
+
+namespace llvm {
+namespace hlsl {
+namespace rootsig {
+
+bool verifyRootFlag(uint32_t Flags) { return (Flags & ~0xfff) == 0; }
+
+bool verifyVersion(uint32_t Version) { return (Version == 1 || Version == 2); }
+
+bool verifyRegisterValue(uint32_t RegisterValue) {
+  return RegisterValue != ~0U;
+}
+
+// This Range is reserverved, therefore invalid, according to the spec
+// https://github.com/llvm/wg-hlsl/blob/main/proposals/0002-root-signature-in-clang.md#all-the-values-should-be-legal
+bool verifyRegisterSpace(uint32_t RegisterSpace) {
+  return !(RegisterSpace >= 0xFFFFFFF0 && RegisterSpace <= 0xFFFFFFFF);
+}
+
+bool verifyDescriptorFlag(uint32_t Flags) { return (Flags & ~0xE) == 0; }
+
+bool verifyRangeType(uint32_t Type) {
+  switch (Type) {
+  case llvm::to_underlying(dxbc::DescriptorRangeType::CBV):
+  case llvm::to_underlying(dxbc::DescriptorRangeType::SRV):
+  case llvm::to_underlying(dxbc::DescriptorRangeType::UAV):
+  case llvm::to_underlying(dxbc::DescriptorRangeType::Sampler):
+    return true;
+  };
+
+  return false;
+}
+
+bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type,
+                               uint32_t FlagsVal) {
+  using FlagT = dxbc::DescriptorRangeFlags;
+  FlagT Flags = FlagT(FlagsVal);
+
+  const bool IsSampler =
+      (Type == llvm::to_underlying(dxbc::DescriptorRangeType::Sampler));
+
+  if (Version == 1) {
+    // Since the metadata is unversioned, we expect to explicitly see the values
+    // that map to the version 1 behaviour here.
+    if (IsSampler)
+      return Flags == FlagT::DescriptorsVolatile;
+    return Flags == (FlagT::DataVolatile | FlagT::DescriptorsVolatile);
+  }
+
+  // The data-specific flags are mutually exclusive.
+  FlagT DataFlags = FlagT::DataVolatile | FlagT::DataStatic |
+                    FlagT::DataStaticWhileSetAtExecute;
+
+  if (popcount(llvm::to_underlying(Flags & DataFlags)) > 1)
+    return false;
+
+  // The descriptor-specific flags are mutually exclusive.
+  FlagT DescriptorFlags = FlagT::DescriptorsStaticKeepingBufferBoundsChecks |
+                          FlagT::DescriptorsVolatile;
+  if (popcount(llvm::to_underlying(Flags & DescriptorFlags)) > 1)
+    return false;
+
+  // For volatile descriptors, DATA_is never valid.
+  if ((Flags & FlagT::DescriptorsVolatile) == FlagT::DescriptorsVolatile) {
+    FlagT Mask = FlagT::DescriptorsVolatile;
+    if (!IsSampler) {
+      Mask |= FlagT::DataVolatile;
+      Mask |= FlagT::DataStaticWhileSetAtExecute;
+    }
+    return (Flags & ~Mask) == FlagT::None;
+  }
+
+  // For "KEEPING_BUFFER_BOUNDS_CHECKS" descriptors,
+  // the other data-specific flags may all be set.
+  if ((Flags & FlagT::DescriptorsStaticKeepingBufferBoundsChecks) ==
+      FlagT::DescriptorsStaticKeepingBufferBoundsChecks) {
+    FlagT Mask = FlagT::DescriptorsStaticKeepingBufferBoundsChecks;
+    if (!IsSampler) {
+      Mask |= FlagT::DataVolatile;
+      Mask |= FlagT::DataStatic;
+      Mask |= FlagT::DataStaticWhileSetAtExecute;
+    }
+    return (Flags & ~Mask) == FlagT::None;
+  }
+
+  // When no descriptor flag is set, any data flag is allowed.
+  FlagT Mask = FlagT::None;
+  if (!IsSampler) {
+    Mask |= FlagT::DataVolatile;
+    Mask |= FlagT::DataStaticWhileSetAtExecute;
+    Mask |= FlagT::DataStatic;
+  }
+  return (Flags & ~Mask) == FlagT::None;
+}
+
+bool verifySamplerFilter(uint32_t Value) {
+  switch (Value) {
+#define FILTER(Num, Val) case llvm::to_underlying(dxbc::SamplerFilter::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+// Values allowed here:
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_texture_address_mode#syntax
+bool verifyAddress(uint32_t Address) {
+  switch (Address) {
+#define TEXTURE_ADDRESS_MODE(Num, Val)                                         \
+  case llvm::to_underlying(dxbc::TextureAddressMode::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+bool verifyMipLODBias(float MipLODBias) {
+  return MipLODBias >= -16.f && MipLODBias <= 15.99f;
+}
+
+bool verifyMaxAnisotropy(uint32_t MaxAnisotropy) {
+  return MaxAnisotropy <= 16u;
+}
+
+bool verifyComparisonFunc(uint32_t ComparisonFunc) {
+  switch (ComparisonFunc) {
+#define COMPARISON_FUNC(Num, Val)                                              \
+  case llvm::to_underlying(dxbc::ComparisonFunc::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+bool verifyBorderColor(uint32_t BorderColor) {
+  switch (BorderColor) {
+#define STATIC_BORDER_COLOR(Num, Val)                                          \
+  case llvm::to_underlying(dxbc::StaticBorderColor::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+bool verifyLOD(float LOD) { return !std::isnan(LOD); }
+
+std::optional<const RangeInfo *>
+ResourceRange::getOverlapping(const RangeInfo &Info) const {
+  MapT::const_iterator Interval = Intervals.find(Info.LowerBound);
+  if (!Interval.valid() || Info.UpperBound < Interval.start())
+    return std::nullopt;
+  return Interval.value();
+}
+
+const RangeInfo *ResourceRange::lookup(uint32_t X) const {
+  return Intervals.lookup(X, nullptr);
+}
+
+void ResourceRange::clear() { return Intervals.clear(); }
+
+std::optional<const RangeInfo *> ResourceRange::insert(const RangeInfo &Info) {
+  uint32_t LowerBound = Info.LowerBound;
+  uint32_t UpperBound = Info.UpperBound;
+
+  std::optional<const RangeInfo *> Res = std::nullopt;
+  MapT::iterator Interval = Intervals.begin();
+
+  while (true) {
+    if (UpperBound < LowerBound)
+      break;
+
+    Interval.advanceTo(LowerBound);
+    if (!Interval.valid()) // No interval found
+      break;
+
+    // Let Interval = [x;y] and [LowerBound;UpperBound] = [a;b] and note that
+    // a <= y implicitly from Intervals.find(LowerBound)
+    if (UpperBound < Interval.start())
+      break; // found interval does not overlap with inserted one
+
+    if (!Res.has_value()) // Update to be the first found intersection
+      Res = Interval.value();
+
+    if (Interval.start() <= LowerBound && UpperBound <= Interval.stop()) {
+      // x <= a <= b <= y implies that [a;b] is covered by [x;y]
+      //  -> so we don't need to insert this, report an overlap
+      return Res;
+    } else if (LowerBound <= Interval.start() &&
+               Interval.stop() <= UpperBound) {
+      // a <= x <= y <= b implies that [x;y] is covered by [a;b]
+      //  -> so remove the existing interval that we will cover with the
+      //  overwrite
+      Interval.erase();
+    } else if (LowerBound < Interval.start() && UpperBound <= Interval.stop()) {
+      // a < x <= b <= y implies that [a; x] is not covered but [x;b] is
+      //  -> so set b = x - 1 such that [a;x-1] is now the interval to insert
+      UpperBound = Interval.start() - 1;
+    } else if (Interval.start() <= LowerBound && Interval.stop() < UpperBound) {
+      // a < x <= b <= y implies that [y; b] is not covered but [a;y] is
+      //  -> so set a = y + 1 such that [y+1;b] is now the interval to insert
+      LowerBound = Interval.stop() + 1;
+    }
+  }
+
+  assert(LowerBound <= UpperBound && "Attempting to insert an empty interval");
+  Intervals.insert(LowerBound, UpperBound, &Info);
+  return Res;
+}
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
diff --git a/llvm/lib/Frontend/OpenMP/CMakeLists.txt b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
index 5bf15ca3a8991..e60b59c1203b9 100644
--- a/llvm/lib/Frontend/OpenMP/CMakeLists.txt
+++ b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMFrontendOpenMP
   OMP.cpp
   OMPContext.cpp
   OMPIRBuilder.cpp
+  DirectiveNameParser.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend
diff --git a/llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp b/llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp
new file mode 100644
index 0000000000000..62e24825111f6
--- /dev/null
+++ b/llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp
@@ -0,0 +1,83 @@
+//===- DirectiveNameParser.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cassert>
+#include <memory>
+
+namespace llvm::omp {
+DirectiveNameParser::DirectiveNameParser(SourceLanguage L) {
+  // Take every directive, get its name in every version, break the name up
+  // into whitespace-separated tokens, and insert each token.
+  for (size_t I : llvm::seq<size_t>(Directive_enumSize)) {
+    auto D = static_cast<Directive>(I);
+    if (D == Directive::OMPD_unknown || !(getDirectiveLanguages(D) & L))
+      continue;
+    for (unsigned Ver : getOpenMPVersions())
+      insertName(getOpenMPDirectiveName(D, Ver), D);
+  }
+}
+
+const DirectiveNameParser::State *
+DirectiveNameParser::consume(const State *Current, StringRef Tok) const {
+  if (!Current)
+    return Current;
+  assert(Current->isValid() && "Invalid input state");
+  if (const State *Next = Current->next(Tok))
+    return Next->isValid() ? Next : nullptr;
+  return nullptr;
+}
+
+SmallVector<StringRef> DirectiveNameParser::tokenize(StringRef Str) {
+  SmallVector<StringRef> Tokens;
+  SplitString(Str, Tokens);
+  return Tokens;
+}
+
+void DirectiveNameParser::insertName(StringRef Name, Directive D) {
+  State *Where = &InitialState;
+
+  for (StringRef Tok : tokenize(Name))
+    Where = insertTransition(Where, Tok);
+
+  Where->Value = D;
+}
+
+DirectiveNameParser::State *
+DirectiveNameParser::insertTransition(State *From, StringRef Tok) {
+  assert(From && "Expecting state");
+  if (!From->Transition)
+    From->Transition = std::make_unique<State::TransitionMapTy>();
+  if (State *Next = From->next(Tok))
+    return Next;
+
+  auto [Where, DidIt] = From->Transition->try_emplace(Tok, State());
+  assert(DidIt && "Map insertion failed");
+  return &Where->second;
+}
+
+const DirectiveNameParser::State *
+DirectiveNameParser::State::next(StringRef Tok) const {
+  if (!Transition)
+    return nullptr;
+  auto F = Transition->find(Tok);
+  return F != Transition->end() ? &F->second : nullptr;
+}
+
+DirectiveNameParser::State *DirectiveNameParser::State::next(StringRef Tok) {
+  if (!Transition)
+    return nullptr;
+  auto F = Transition->find(Tok);
+  return F != Transition->end() ? &F->second : nullptr;
+}
+} // namespace llvm::omp
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 85451b1233f96..db792a3b52d24 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6891,23 +6891,19 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
-    auto NewSP = Func->getSubprogram();
     DILocalVariable *&NewVar = RemappedVariables[OldVar];
     // Only use cached variable if the arg number matches. This is important
     // so that DIVariable created for privatized variables are not discarded.
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getAlignInBits(), OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -6921,7 +6917,8 @@ static void FixupDebugInfoForOutlinedFunction(
         ArgNo = std::get<1>(Iter->second) + 1;
       }
     }
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7000,36 +6997,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
diff --git a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
index 5e793997aee6e..8b034137e4702 100644
--- a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
+++ b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
@@ -108,6 +108,55 @@ Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
   return findOrCreateSource(BB, Insts, {}, anyType());
 }
 
+// Adapts the current pointer for a legal mem operation on the target arch.
+static Value *buildTargetLegalPtr(Module *M, Value *Ptr, InsertPosition IP,
+                                  const Twine &Name,
+                                  SmallVector<Instruction *> *NewInsts) {
+  if (M && M->getTargetTriple().isAMDGCN()) {
+    // Check if we should perform an address space cast
+    PointerType *pointerType = dyn_cast<PointerType>(Ptr->getType());
+    if (pointerType && pointerType->getAddressSpace() == 8) {
+      // Perform address space cast from address space 8 to address space 7
+      auto NewPtr = new AddrSpaceCastInst(
+          Ptr, PointerType::get(M->getContext(), 7), Name + ".ASC", IP);
+      if (NewInsts)
+        NewInsts->push_back(NewPtr);
+      return NewPtr;
+    }
+  }
+
+  return Ptr;
+}
+
+// Stores a value to memory, considering the target triple's restrictions.
+static Instruction *buildTargetLegalStore(Value *Val, Value *Ptr,
+                                          InsertPosition IP, Module *M) {
+  Value *StorePtr = buildTargetLegalPtr(M, Ptr, IP, "", nullptr);
+  Instruction *Store = new StoreInst(Val, StorePtr, IP);
+  return Store;
+}
+
+// Loads a value from memory, considering the target triple's restrictions.
+static std::pair<Instruction *, SmallVector<Instruction *>>
+buildTargetLegalLoad(Type *AccessTy, Value *Ptr, InsertPosition IP, Module *M,
+                     const Twine &LoadName) {
+  SmallVector<Instruction *> NewInsts;
+
+  Value *LoadPtr = buildTargetLegalPtr(M, Ptr, IP, LoadName, &NewInsts);
+
+  Instruction *Load = new LoadInst(AccessTy, LoadPtr, LoadName, IP);
+  NewInsts.push_back(Load);
+
+  return std::make_pair(Load, NewInsts);
+}
+
+static void eraseNewInstructions(SmallVector<Instruction *> &NewInsts) {
+  // Remove in reverse order (uses before defs)
+  for (auto it = NewInsts.rbegin(); it != NewInsts.rend(); ++it) {
+    (*it)->eraseFromParent();
+  }
+}
+
 Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
                                            ArrayRef<Instruction *> Insts,
                                            ArrayRef<Value *> Srcs,
@@ -158,19 +207,20 @@ Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
       Module *M = BB.getParent()->getParent();
       auto [GV, DidCreate] = findOrCreateGlobalVariable(M, Srcs, Pred);
       Type *Ty = GV->getValueType();
-      LoadInst *LoadGV = nullptr;
-      if (BB.getTerminator()) {
-        LoadGV = new LoadInst(Ty, GV, "LGV", BB.getFirstInsertionPt());
-      } else {
-        LoadGV = new LoadInst(Ty, GV, "LGV", &BB);
-      }
+      InsertPosition IP = BB.getTerminator()
+                              ? InsertPosition(BB.getFirstInsertionPt())
+                              : InsertPosition(&BB);
+      // Build a legal load and track new instructions in case a rollback is
+      // needed.
+      auto [LoadGV, NewInsts] = buildTargetLegalLoad(Ty, GV, IP, M, "LGV");
       // Because we might be generating new values, we have to check if it
       // matches again.
       if (DidCreate) {
         if (Pred.matches(Srcs, LoadGV)) {
           return LoadGV;
         }
-        LoadGV->eraseFromParent();
+        // Remove newly inserted instructions
+        eraseNewInstructions(NewInsts);
         // If no one is using this GlobalVariable, delete it too.
         if (GV->use_empty()) {
           GV->eraseFromParent();
@@ -208,13 +258,18 @@ Value *RandomIRBuilder::newSource(BasicBlock &BB, ArrayRef<Instruction *> Insts,
     }
     // Pick the type independently.
     Type *AccessTy = RS.getSelection()->getType();
-    auto *NewLoad = new LoadInst(AccessTy, Ptr, "L", IP);
+    // Build a legal load and track new instructions in case a rollback is
+    // needed.
+    auto [NewLoad, NewInsts] =
+        buildTargetLegalLoad(AccessTy, Ptr, IP, BB.getModule(), "L");
 
     // Only sample this load if it really matches the descriptor
     if (Pred.matches(Srcs, NewLoad))
       RS.sample(NewLoad, RS.totalWeight());
-    else
-      NewLoad->eraseFromParent();
+    else {
+      // Remove newly inserted instructions
+      eraseNewInstructions(NewInsts);
+    }
   }
 
   Value *newSrc = RS.getSelection();
@@ -320,8 +375,10 @@ Instruction *RandomIRBuilder::connectToSink(BasicBlock &BB,
       std::shuffle(Dominators.begin(), Dominators.end(), Rand);
       for (BasicBlock *Dom : Dominators) {
         for (Instruction &I : *Dom) {
-          if (isa<PointerType>(I.getType()))
-            return new StoreInst(V, &I, Insts.back()->getIterator());
+          if (isa<PointerType>(I.getType())) {
+            return buildTargetLegalStore(V, &I, Insts.back()->getIterator(),
+                                         I.getModule());
+          }
         }
       }
       break;
@@ -344,10 +401,10 @@ Instruction *RandomIRBuilder::connectToSink(BasicBlock &BB,
       /// TODO: allocate a new stack memory.
       return newSink(BB, Insts, V);
     case SinkToGlobalVariable: {
-      Module *M = BB.getParent()->getParent();
+      Module *M = BB.getModule();
       auto [GV, DidCreate] =
           findOrCreateGlobalVariable(M, {}, fuzzerop::onlyType(V->getType()));
-      return new StoreInst(V, GV, Insts.back()->getIterator());
+      return buildTargetLegalStore(V, GV, Insts.back()->getIterator(), M);
     }
     case EndOfValueSink:
     default:
@@ -369,7 +426,8 @@ Instruction *RandomIRBuilder::newSink(BasicBlock &BB,
     }
   }
 
-  return new StoreInst(V, Ptr, Insts.back()->getIterator());
+  return buildTargetLegalStore(V, Ptr, Insts.back()->getIterator(),
+                               BB.getModule());
 }
 
 Value *RandomIRBuilder::findPointer(BasicBlock &BB,
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 7828ba45ec27f..145ef10f28f35 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -447,8 +447,7 @@ void llvm::printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name) {
       // in the range 0-255.  This is important when building with MSVC because
       // its implementation will assert.  This situation can arise when dealing
       // with UTF-8 multibyte characters.
-      if (!isalnum(static_cast<unsigned char>(C)) && C != '-' && C != '.' &&
-          C != '_') {
+      if (!isalnum(C) && C != '-' && C != '.' && C != '_') {
         NeedsQuotes = true;
         break;
       }
@@ -1915,6 +1914,8 @@ struct MDFieldPrinter {
                    bool ShouldSkipEmpty = true);
   void printMetadata(StringRef Name, const Metadata *MD,
                      bool ShouldSkipNull = true);
+  void printMetadataOrInt(StringRef Name, const Metadata *MD, bool IsUnsigned,
+                          bool ShouldSkipZero = true);
   template <class IntTy>
   void printInt(StringRef Name, IntTy Int, bool ShouldSkipZero = true);
   void printAPInt(StringRef Name, const APInt &Int, bool IsUnsigned,
@@ -1987,6 +1988,21 @@ void MDFieldPrinter::printMetadata(StringRef Name, const Metadata *MD,
   writeMetadataAsOperand(Out, MD, WriterCtx);
 }
 
+void MDFieldPrinter::printMetadataOrInt(StringRef Name, const Metadata *MD,
+                                        bool IsUnsigned, bool ShouldSkipZero) {
+  if (!MD)
+    return;
+
+  if (auto *CI = dyn_cast<ConstantAsMetadata>(MD)) {
+    auto *CV = cast<ConstantInt>(CI->getValue());
+    if (IsUnsigned)
+      printInt(Name, CV->getZExtValue(), ShouldSkipZero);
+    else
+      printInt(Name, CV->getSExtValue(), ShouldSkipZero);
+  } else
+    printMetadata(Name, MD);
+}
+
 template <class IntTy>
 void MDFieldPrinter::printInt(StringRef Name, IntTy Int, bool ShouldSkipZero) {
   if (ShouldSkipZero && !Int)
@@ -2130,38 +2146,21 @@ static void writeDISubrange(raw_ostream &Out, const DISubrange *N,
   Out << "!DISubrange(";
   MDFieldPrinter Printer(Out, WriterCtx);
 
-  auto *Count = N->getRawCountNode();
-  if (auto *CE = dyn_cast_or_null<ConstantAsMetadata>(Count)) {
-    auto *CV = cast<ConstantInt>(CE->getValue());
-    Printer.printInt("count", CV->getSExtValue(),
-                     /* ShouldSkipZero */ false);
-  } else
-    Printer.printMetadata("count", Count, /*ShouldSkipNull */ true);
+  Printer.printMetadataOrInt("count", N->getRawCountNode(),
+                             /* IsUnsigned */ false,
+                             /* ShouldSkipZero */ false);
 
   // A lowerBound of constant 0 should not be skipped, since it is different
   // from an unspecified lower bound (= nullptr).
-  auto *LBound = N->getRawLowerBound();
-  if (auto *LE = dyn_cast_or_null<ConstantAsMetadata>(LBound)) {
-    auto *LV = cast<ConstantInt>(LE->getValue());
-    Printer.printInt("lowerBound", LV->getSExtValue(),
-                     /* ShouldSkipZero */ false);
-  } else
-    Printer.printMetadata("lowerBound", LBound, /*ShouldSkipNull */ true);
-
-  auto *UBound = N->getRawUpperBound();
-  if (auto *UE = dyn_cast_or_null<ConstantAsMetadata>(UBound)) {
-    auto *UV = cast<ConstantInt>(UE->getValue());
-    Printer.printInt("upperBound", UV->getSExtValue(),
-                     /* ShouldSkipZero */ false);
-  } else
-    Printer.printMetadata("upperBound", UBound, /*ShouldSkipNull */ true);
-
-  auto *Stride = N->getRawStride();
-  if (auto *SE = dyn_cast_or_null<ConstantAsMetadata>(Stride)) {
-    auto *SV = cast<ConstantInt>(SE->getValue());
-    Printer.printInt("stride", SV->getSExtValue(), /* ShouldSkipZero */ false);
-  } else
-    Printer.printMetadata("stride", Stride, /*ShouldSkipNull */ true);
+  Printer.printMetadataOrInt("lowerBound", N->getRawLowerBound(),
+                             /* IsUnsigned */ false,
+                             /* ShouldSkipZero */ false);
+  Printer.printMetadataOrInt("upperBound", N->getRawUpperBound(),
+                             /* IsUnsigned */ false,
+                             /* ShouldSkipZero */ false);
+  Printer.printMetadataOrInt("stride", N->getRawStride(),
+                             /* IsUnsigned */ false,
+                             /* ShouldSkipZero */ false);
 
   Out << ")";
 }
@@ -2227,13 +2226,13 @@ static void writeDIEnumerator(raw_ostream &Out, const DIEnumerator *N,
 }
 
 static void writeDIBasicType(raw_ostream &Out, const DIBasicType *N,
-                             AsmWriterContext &) {
+                             AsmWriterContext &WriterCtx) {
   Out << "!DIBasicType(";
-  MDFieldPrinter Printer(Out);
+  MDFieldPrinter Printer(Out, WriterCtx);
   if (N->getTag() != dwarf::DW_TAG_base_type)
     Printer.printTag(N);
   Printer.printString("name", N->getName());
-  Printer.printInt("size", N->getSizeInBits());
+  Printer.printMetadataOrInt("size", N->getRawSizeInBits(), true);
   Printer.printInt("align", N->getAlignInBits());
   Printer.printDwarfEnum("encoding", N->getEncoding(),
                          dwarf::AttributeEncodingString);
@@ -2243,13 +2242,13 @@ static void writeDIBasicType(raw_ostream &Out, const DIBasicType *N,
 }
 
 static void writeDIFixedPointType(raw_ostream &Out, const DIFixedPointType *N,
-                                  AsmWriterContext &) {
+                                  AsmWriterContext &WriterCtx) {
   Out << "!DIFixedPointType(";
-  MDFieldPrinter Printer(Out);
+  MDFieldPrinter Printer(Out, WriterCtx);
   if (N->getTag() != dwarf::DW_TAG_base_type)
     Printer.printTag(N);
   Printer.printString("name", N->getName());
-  Printer.printInt("size", N->getSizeInBits());
+  Printer.printMetadataOrInt("size", N->getRawSizeInBits(), true);
   Printer.printInt("align", N->getAlignInBits());
   Printer.printDwarfEnum("encoding", N->getEncoding(),
                          dwarf::AttributeEncodingString);
@@ -2276,7 +2275,7 @@ static void writeDIStringType(raw_ostream &Out, const DIStringType *N,
   Printer.printMetadata("stringLengthExpression", N->getRawStringLengthExp());
   Printer.printMetadata("stringLocationExpression",
                         N->getRawStringLocationExp());
-  Printer.printInt("size", N->getSizeInBits());
+  Printer.printMetadataOrInt("size", N->getRawSizeInBits(), true);
   Printer.printInt("align", N->getAlignInBits());
   Printer.printDwarfEnum("encoding", N->getEncoding(),
                          dwarf::AttributeEncodingString);
@@ -2294,9 +2293,9 @@ static void writeDIDerivedType(raw_ostream &Out, const DIDerivedType *N,
   Printer.printInt("line", N->getLine());
   Printer.printMetadata("baseType", N->getRawBaseType(),
                         /* ShouldSkipNull */ false);
-  Printer.printInt("size", N->getSizeInBits());
+  Printer.printMetadataOrInt("size", N->getRawSizeInBits(), true);
   Printer.printInt("align", N->getAlignInBits());
-  Printer.printInt("offset", N->getOffsetInBits());
+  Printer.printMetadataOrInt("offset", N->getRawOffsetInBits(), true);
   Printer.printDIFlags("flags", N->getFlags());
   Printer.printMetadata("extraData", N->getRawExtraData());
   if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
@@ -2324,7 +2323,7 @@ static void writeDISubrangeType(raw_ostream &Out, const DISubrangeType *N,
   Printer.printMetadata("scope", N->getRawScope());
   Printer.printMetadata("file", N->getRawFile());
   Printer.printInt("line", N->getLine());
-  Printer.printInt("size", N->getSizeInBits());
+  Printer.printMetadataOrInt("size", N->getRawSizeInBits(), true);
   Printer.printInt("align", N->getAlignInBits());
   Printer.printDIFlags("flags", N->getFlags());
   Printer.printMetadata("baseType", N->getRawBaseType(),
@@ -2346,9 +2345,9 @@ static void writeDICompositeType(raw_ostream &Out, const DICompositeType *N,
   Printer.printMetadata("file", N->getRawFile());
   Printer.printInt("line", N->getLine());
   Printer.printMetadata("baseType", N->getRawBaseType());
-  Printer.printInt("size", N->getSizeInBits());
+  Printer.printMetadataOrInt("size", N->getRawSizeInBits(), true);
   Printer.printInt("align", N->getAlignInBits());
-  Printer.printInt("offset", N->getOffsetInBits());
+  Printer.printMetadataOrInt("offset", N->getRawOffsetInBits(), true);
   Printer.printInt("num_extra_inhabitants", N->getNumExtraInhabitants());
   Printer.printDIFlags("flags", N->getFlags());
   Printer.printMetadata("elements", N->getRawElements());
@@ -2459,6 +2458,7 @@ static void writeDISubprogram(raw_ostream &Out, const DISubprogram *N,
   Printer.printMetadata("thrownTypes", N->getRawThrownTypes());
   Printer.printMetadata("annotations", N->getRawAnnotations());
   Printer.printString("targetFuncName", N->getTargetFuncName());
+  Printer.printBool("keyInstructions", N->getKeyInstructionsEnabled(), false);
   Out << ")";
 }
 
@@ -2611,6 +2611,11 @@ static void writeDILabel(raw_ostream &Out, const DILabel *N,
   Printer.printString("name", N->getName());
   Printer.printMetadata("file", N->getRawFile());
   Printer.printInt("line", N->getLine());
+  Printer.printInt("column", N->getColumn());
+  Printer.printBool("isArtificial", N->isArtificial(), false);
+  if (N->getCoroSuspendIdx())
+    Printer.printInt("coroSuspendIdx", *N->getCoroSuspendIdx(),
+                     /* ShouldSkipZero */ false);
   Out << ")";
 }
 
@@ -4129,6 +4134,9 @@ void AssemblyWriter::printFunction(const Function *F) {
       Out << "; Function Attrs: " << AttrStr << '\n';
   }
 
+  if (F->isIntrinsic() && F->getIntrinsicID() == Intrinsic::not_intrinsic)
+    Out << "; Unknown intrinsic\n";
+
   Machine.incorporateFunction(F);
 
   if (F->isDeclaration()) {
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index bfb32ff9995d1..d1fbcb9e893a7 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -2424,7 +2424,8 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty, AttributeSet AS,
           .addAttribute(Attribute::Writable)
           .addAttribute(Attribute::DeadOnUnwind)
           .addAttribute(Attribute::Initializes)
-          .addAttribute(Attribute::Captures);
+          .addAttribute(Attribute::Captures)
+          .addAttribute(Attribute::DeadOnReturn);
     if (ASK & ASK_UNSAFE_TO_DROP)
       Incompatible.addAttribute(Attribute::Nest)
           .addAttribute(Attribute::SwiftError)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index e429e2d65c05f..86285a03c66bb 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -719,6 +719,12 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                              .StartsWith("vqaddu.", Intrinsic::uadd_sat)
                              .StartsWith("vqsubs.", Intrinsic::ssub_sat)
                              .StartsWith("vqsubu.", Intrinsic::usub_sat)
+                             .StartsWith("vrinta.", Intrinsic::round)
+                             .StartsWith("vrintn.", Intrinsic::roundeven)
+                             .StartsWith("vrintm.", Intrinsic::floor)
+                             .StartsWith("vrintp.", Intrinsic::ceil)
+                             .StartsWith("vrintx.", Intrinsic::rint)
+                             .StartsWith("vrintz.", Intrinsic::trunc)
                              .Default(Intrinsic::not_intrinsic);
       if (ID != Intrinsic::not_intrinsic) {
         NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 6001ed421183b..1344df9298c80 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -946,9 +946,13 @@ DILocalVariable *DIBuilder::createParameterVariable(
 }
 
 DILabel *DIBuilder::createLabel(DIScope *Context, StringRef Name, DIFile *File,
-                                 unsigned LineNo, bool AlwaysPreserve) {
+                                unsigned LineNo, unsigned Column,
+                                bool IsArtificial,
+                                std::optional<unsigned> CoroSuspendIdx,
+                                bool AlwaysPreserve) {
   auto *Scope = cast<DILocalScope>(Context);
-  auto *Node = DILabel::get(VMContext, Scope, Name, File, LineNo);
+  auto *Node = DILabel::get(VMContext, Scope, Name, File, LineNo, Column,
+                            IsArtificial, CoroSuspendIdx);
 
   if (AlwaysPreserve) {
     /// The optimizer may remove labels. If there is an interest
@@ -975,14 +979,14 @@ DISubprogram *DIBuilder::createFunction(
     unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine,
     DINode::DIFlags Flags, DISubprogram::DISPFlags SPFlags,
     DITemplateParameterArray TParams, DISubprogram *Decl,
-    DITypeArray ThrownTypes, DINodeArray Annotations,
-    StringRef TargetFuncName) {
+    DITypeArray ThrownTypes, DINodeArray Annotations, StringRef TargetFuncName,
+    bool UseKeyInstructions) {
   bool IsDefinition = SPFlags & DISubprogram::SPFlagDefinition;
   auto *Node = getSubprogram(
       /*IsDistinct=*/IsDefinition, VMContext, getNonCompileUnitScope(Context),
       Name, LinkageName, File, LineNo, Ty, ScopeLine, nullptr, 0, 0, Flags,
       SPFlags, IsDefinition ? CUNode : nullptr, TParams, Decl, nullptr,
-      ThrownTypes, Annotations, TargetFuncName);
+      ThrownTypes, Annotations, TargetFuncName, UseKeyInstructions);
 
   AllSubprograms.push_back(Node);
   trackIfUnresolved(Node);
@@ -1009,7 +1013,7 @@ DISubprogram *DIBuilder::createMethod(
     unsigned LineNo, DISubroutineType *Ty, unsigned VIndex, int ThisAdjustment,
     DIType *VTableHolder, DINode::DIFlags Flags,
     DISubprogram::DISPFlags SPFlags, DITemplateParameterArray TParams,
-    DITypeArray ThrownTypes) {
+    DITypeArray ThrownTypes, bool UseKeyInstructions) {
   assert(getNonCompileUnitScope(Context) &&
          "Methods should have both a Context and a context that isn't "
          "the compile unit.");
@@ -1019,7 +1023,7 @@ DISubprogram *DIBuilder::createMethod(
       /*IsDistinct=*/IsDefinition, VMContext, cast<DIScope>(Context), Name,
       LinkageName, F, LineNo, Ty, LineNo, VTableHolder, VIndex, ThisAdjustment,
       Flags, SPFlags, IsDefinition ? CUNode : nullptr, TParams, nullptr,
-      nullptr, ThrownTypes);
+      nullptr, ThrownTypes, nullptr, "", IsDefinition && UseKeyInstructions);
 
   AllSubprograms.push_back(SP);
   trackIfUnresolved(SP);
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index ecb19fd3c82db..4e09f847627af 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -376,7 +376,7 @@ bool DebugInfoFinder::addType(DIType *DT) {
   if (!NodesSeen.insert(DT).second)
     return false;
 
-  TYs.push_back(const_cast<DIType *>(DT));
+  TYs.push_back(DT);
   return true;
 }
 
@@ -1837,7 +1837,8 @@ LLVMMetadataRef LLVMDIBuilderCreateLabel(LLVMDIBuilderRef Builder,
                                          LLVMBool AlwaysPreserve) {
   return wrap(unwrap(Builder)->createLabel(
       unwrapDI<DIScope>(Context), StringRef(Name, NameLen),
-      unwrapDI<DIFile>(File), LineNo, AlwaysPreserve));
+      unwrapDI<DIFile>(File), LineNo, /*Column*/ 0, /*IsArtificial*/ false,
+      /*CoroSuspendIdx*/ std::nullopt, AlwaysPreserve));
 }
 
 LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(LLVMDIBuilderRef Builder,
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 44b0f0d50067c..2270923bd3719 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -1292,11 +1292,12 @@ const char *DICompileUnit::nameTableKindString(DebugNameTableKind NTK) {
 DISubprogram::DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line,
                            unsigned ScopeLine, unsigned VirtualIndex,
                            int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags,
-                           ArrayRef<Metadata *> Ops)
+                           bool UsesKeyInstructions, ArrayRef<Metadata *> Ops)
     : DILocalScope(C, DISubprogramKind, Storage, dwarf::DW_TAG_subprogram, Ops),
       Line(Line), ScopeLine(ScopeLine), VirtualIndex(VirtualIndex),
       ThisAdjustment(ThisAdjustment), Flags(Flags), SPFlags(SPFlags) {
   static_assert(dwarf::DW_VIRTUALITY_max < 4, "Virtuality out of range");
+  SubclassData1 = UsesKeyInstructions;
 }
 DISubprogram::DISPFlags
 DISubprogram::toSPFlags(bool IsLocalToUnit, bool IsDefinition, bool IsOptimized,
@@ -1395,7 +1396,7 @@ DISubprogram *DISubprogram::getImpl(
     int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, Metadata *Unit,
     Metadata *TemplateParams, Metadata *Declaration, Metadata *RetainedNodes,
     Metadata *ThrownTypes, Metadata *Annotations, MDString *TargetFuncName,
-    StorageType Storage, bool ShouldCreate) {
+    bool UsesKeyInstructions, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   assert(isCanonical(LinkageName) && "Expected canonical MDString");
   assert(isCanonical(TargetFuncName) && "Expected canonical MDString");
@@ -1404,7 +1405,7 @@ DISubprogram *DISubprogram::getImpl(
                          ContainingType, VirtualIndex, ThisAdjustment, Flags,
                          SPFlags, Unit, TemplateParams, Declaration,
                          RetainedNodes, ThrownTypes, Annotations,
-                         TargetFuncName));
+                         TargetFuncName, UsesKeyInstructions));
   SmallVector<Metadata *, 13> Ops = {
       File,           Scope,          Name,        LinkageName,
       Type,           Unit,           Declaration, RetainedNodes,
@@ -1424,10 +1425,10 @@ DISubprogram *DISubprogram::getImpl(
       }
     }
   }
-  DEFINE_GETIMPL_STORE_N(
-      DISubprogram,
-      (Line, ScopeLine, VirtualIndex, ThisAdjustment, Flags, SPFlags), Ops,
-      Ops.size());
+  DEFINE_GETIMPL_STORE_N(DISubprogram,
+                         (Line, ScopeLine, VirtualIndex, ThisAdjustment, Flags,
+                          SPFlags, UsesKeyInstructions),
+                         Ops, Ops.size());
 }
 
 bool DISubprogram::describes(const Function *F) const {
@@ -1616,18 +1617,27 @@ std::optional<uint64_t> DIVariable::getSizeInBits() const {
 }
 
 DILabel::DILabel(LLVMContext &C, StorageType Storage, unsigned Line,
+                 unsigned Column, bool IsArtificial,
+                 std::optional<unsigned> CoroSuspendIdx,
                  ArrayRef<Metadata *> Ops)
     : DINode(C, DILabelKind, Storage, dwarf::DW_TAG_label, Ops) {
-  SubclassData32 = Line;
+  this->SubclassData32 = Line;
+  this->Column = Column;
+  this->IsArtificial = IsArtificial;
+  this->CoroSuspendIdx = CoroSuspendIdx;
 }
 DILabel *DILabel::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
-                          Metadata *File, unsigned Line, StorageType Storage,
-                          bool ShouldCreate) {
+                          Metadata *File, unsigned Line, unsigned Column,
+                          bool IsArtificial,
+                          std::optional<unsigned> CoroSuspendIdx,
+                          StorageType Storage, bool ShouldCreate) {
   assert(Scope && "Expected scope");
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(DILabel, (Scope, Name, File, Line));
+  DEFINE_GETIMPL_LOOKUP(
+      DILabel, (Scope, Name, File, Line, Column, IsArtificial, CoroSuspendIdx));
   Metadata *Ops[] = {Scope, Name, File};
-  DEFINE_GETIMPL_STORE(DILabel, (Line), Ops);
+  DEFINE_GETIMPL_STORE(DILabel, (Line, Column, IsArtificial, CoroSuspendIdx),
+                       Ops);
 }
 
 DIExpression *DIExpression::getImpl(LLVMContext &Context,
diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp
index ffeeeb6f1e4b0..79c5b896f8f25 100644
--- a/llvm/lib/IR/DebugLoc.cpp
+++ b/llvm/lib/IR/DebugLoc.cpp
@@ -9,11 +9,35 @@
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DebugInfo.h"
+
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#include "llvm/Support/Signals.h"
+
+namespace llvm {
+DbgLocOrigin::DbgLocOrigin(bool ShouldCollectTrace) {
+  if (!ShouldCollectTrace)
+    return;
+  auto &[Depth, StackTrace] = StackTraces.emplace_back();
+  Depth = sys::getStackTrace(StackTrace);
+}
+void DbgLocOrigin::addTrace() {
+  // We only want to add new stacktraces if we already have one: addTrace exists
+  // to provide more context to how missing DebugLocs have propagated through
+  // the program, but by design if there is no existing stacktrace then we have
+  // decided not to track this DebugLoc as being "missing".
+  if (StackTraces.empty())
+    return;
+  auto &[Depth, StackTrace] = StackTraces.emplace_back();
+  Depth = sys::getStackTrace(StackTrace);
+}
+} // namespace llvm
+#endif
+
 using namespace llvm;
 
 #if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 DILocAndCoverageTracking::DILocAndCoverageTracking(const DILocation *L)
-    : TrackingMDNodeRef(const_cast<DILocation *>(L)),
+    : TrackingMDNodeRef(const_cast<DILocation *>(L)), DbgLocOrigin(!L),
       Kind(DebugLocKind::Normal) {}
 #endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 3e7fcbb983738..7a03663e129dc 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -130,6 +130,12 @@ bool Argument::hasByValAttr() const {
   return hasAttribute(Attribute::ByVal);
 }
 
+bool Argument::hasDeadOnReturnAttr() const {
+  if (!getType()->isPointerTy())
+    return false;
+  return hasAttribute(Attribute::DeadOnReturn);
+}
+
 bool Argument::hasByRefAttr() const {
   if (!getType()->isPointerTy())
     return false;
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index beefd5e7d7eef..28037d7ec5616 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -51,7 +51,7 @@ GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
-  GV->setAlignment(Align(1));
+  GV->setAlignment(M->getDataLayout().getPrefTypeAlign(getInt8Ty()));
   return GV;
 }
 
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 8d9b545d4134f..9fe79366f9b2f 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2582,7 +2582,7 @@ Type *ExtractValueInst::getIndexedType(Type *Agg,
       return nullptr;
     }
   }
-  return const_cast<Type*>(Agg);
+  return Agg;
 }
 
 //===----------------------------------------------------------------------===//
@@ -3488,7 +3488,7 @@ CmpInst::CmpInst(Type *ty, OtherOps op, Predicate predicate, Value *LHS,
     : Instruction(ty, op, AllocMarker, InsertBefore) {
   Op<0>() = LHS;
   Op<1>() = RHS;
-  setPredicate((Predicate)predicate);
+  setPredicate(predicate);
   setName(Name);
   if (FlagsSource)
     copyIRFlags(FlagsSource);
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 4446f47d323d2..aa2a60e5038a7 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -924,8 +924,8 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
   MDString *LinkageName;
   Metadata *File;
   unsigned Line;
-  Metadata *Type;
   unsigned ScopeLine;
+  Metadata *Type;
   Metadata *ContainingType;
   unsigned VirtualIndex;
   int ThisAdjustment;
@@ -938,6 +938,7 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
   Metadata *ThrownTypes;
   Metadata *Annotations;
   MDString *TargetFuncName;
+  bool UsesKeyInstructions;
 
   MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *LinkageName,
                 Metadata *File, unsigned Line, Metadata *Type,
@@ -946,18 +947,19 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
                 unsigned SPFlags, Metadata *Unit, Metadata *TemplateParams,
                 Metadata *Declaration, Metadata *RetainedNodes,
                 Metadata *ThrownTypes, Metadata *Annotations,
-                MDString *TargetFuncName)
+                MDString *TargetFuncName, bool UsesKeyInstructions)
       : Scope(Scope), Name(Name), LinkageName(LinkageName), File(File),
-        Line(Line), Type(Type), ScopeLine(ScopeLine),
+        Line(Line), ScopeLine(ScopeLine), Type(Type),
         ContainingType(ContainingType), VirtualIndex(VirtualIndex),
         ThisAdjustment(ThisAdjustment), Flags(Flags), SPFlags(SPFlags),
         Unit(Unit), TemplateParams(TemplateParams), Declaration(Declaration),
         RetainedNodes(RetainedNodes), ThrownTypes(ThrownTypes),
-        Annotations(Annotations), TargetFuncName(TargetFuncName) {}
+        Annotations(Annotations), TargetFuncName(TargetFuncName),
+        UsesKeyInstructions(UsesKeyInstructions) {}
   MDNodeKeyImpl(const DISubprogram *N)
       : Scope(N->getRawScope()), Name(N->getRawName()),
         LinkageName(N->getRawLinkageName()), File(N->getRawFile()),
-        Line(N->getLine()), Type(N->getRawType()), ScopeLine(N->getScopeLine()),
+        Line(N->getLine()), ScopeLine(N->getScopeLine()), Type(N->getRawType()),
         ContainingType(N->getRawContainingType()),
         VirtualIndex(N->getVirtualIndex()),
         ThisAdjustment(N->getThisAdjustment()), Flags(N->getFlags()),
@@ -967,7 +969,8 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
         RetainedNodes(N->getRawRetainedNodes()),
         ThrownTypes(N->getRawThrownTypes()),
         Annotations(N->getRawAnnotations()),
-        TargetFuncName(N->getRawTargetFuncName()) {}
+        TargetFuncName(N->getRawTargetFuncName()),
+        UsesKeyInstructions(N->getKeyInstructionsEnabled()) {}
 
   bool isKeyOf(const DISubprogram *RHS) const {
     return Scope == RHS->getRawScope() && Name == RHS->getRawName() &&
@@ -984,7 +987,8 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
            RetainedNodes == RHS->getRawRetainedNodes() &&
            ThrownTypes == RHS->getRawThrownTypes() &&
            Annotations == RHS->getRawAnnotations() &&
-           TargetFuncName == RHS->getRawTargetFuncName();
+           TargetFuncName == RHS->getRawTargetFuncName() &&
+           UsesKeyInstructions == RHS->getKeyInstructionsEnabled();
   }
 
   bool isDefinition() const { return SPFlags & DISubprogram::SPFlagDefinition; }
@@ -1329,20 +1333,33 @@ template <> struct MDNodeKeyImpl<DILabel> {
   MDString *Name;
   Metadata *File;
   unsigned Line;
+  unsigned Column;
+  bool IsArtificial;
+  std::optional<unsigned> CoroSuspendIdx;
 
-  MDNodeKeyImpl(Metadata *Scope, MDString *Name, Metadata *File, unsigned Line)
-      : Scope(Scope), Name(Name), File(File), Line(Line) {}
+  MDNodeKeyImpl(Metadata *Scope, MDString *Name, Metadata *File, unsigned Line,
+                unsigned Column, bool IsArtificial,
+                std::optional<unsigned> CoroSuspendIdx)
+      : Scope(Scope), Name(Name), File(File), Line(Line), Column(Column),
+        IsArtificial(IsArtificial), CoroSuspendIdx(CoroSuspendIdx) {}
   MDNodeKeyImpl(const DILabel *N)
       : Scope(N->getRawScope()), Name(N->getRawName()), File(N->getRawFile()),
-        Line(N->getLine()) {}
+        Line(N->getLine()), Column(N->getColumn()),
+        IsArtificial(N->isArtificial()),
+        CoroSuspendIdx(N->getCoroSuspendIdx()) {}
 
   bool isKeyOf(const DILabel *RHS) const {
     return Scope == RHS->getRawScope() && Name == RHS->getRawName() &&
-           File == RHS->getRawFile() && Line == RHS->getLine();
+           File == RHS->getRawFile() && Line == RHS->getLine() &&
+           Column == RHS->getColumn() && IsArtificial == RHS->isArtificial() &&
+           CoroSuspendIdx == RHS->getCoroSuspendIdx();
   }
 
   /// Using name and line to get hash value. It should already be mostly unique.
-  unsigned getHashValue() const { return hash_combine(Scope, Name, Line); }
+  unsigned getHashValue() const {
+    return hash_combine(Scope, Name, Line, Column, IsArtificial,
+                        CoroSuspendIdx);
+  }
 };
 
 template <> struct MDNodeKeyImpl<DIExpression> {
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index c8f1606ea06cb..1d9a61ce4a10b 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -120,7 +120,7 @@ void PMDataManager::emitInstrCountChangedRemark(
   // If no function was passed in, then we're either a module pass or an
   // CGSCC pass.
   if (!CouldOnlyImpactOneFunction)
-    std::for_each(M.begin(), M.end(), UpdateFunctionChanges);
+    llvm::for_each(M, UpdateFunctionChanges);
   else
     UpdateFunctionChanges(*F);
 
@@ -197,9 +197,7 @@ void PMDataManager::emitInstrCountChangedRemark(
   // Are we looking at more than one function? If so, emit remarks for all of
   // the functions in the module. Otherwise, only emit one remark.
   if (!CouldOnlyImpactOneFunction)
-    std::for_each(FunctionToInstrCount.keys().begin(),
-                  FunctionToInstrCount.keys().end(),
-                  EmitFunctionSizeChangedRemark);
+    llvm::for_each(FunctionToInstrCount.keys(), EmitFunctionSizeChangedRemark);
   else
     EmitFunctionSizeChangedRemark(F->getName().str());
 }
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index 86408d9643c75..f0448b06e7e82 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -699,7 +699,7 @@ MDNode::Header::~Header() {
   }
   MDOperand *O = reinterpret_cast<MDOperand *>(this);
   for (MDOperand *E = O - SmallSize; O != E; --O)
-    (void)(O - 1)->~MDOperand();
+    (O - 1)->~MDOperand();
 }
 
 void *MDNode::Header::getSmallPtr() {
diff --git a/llvm/lib/IR/OptBisect.cpp b/llvm/lib/IR/OptBisect.cpp
index 559b199445366..427e8b78fd03f 100644
--- a/llvm/lib/IR/OptBisect.cpp
+++ b/llvm/lib/IR/OptBisect.cpp
@@ -37,15 +37,15 @@ static cl::opt<bool> OptBisectVerbose(
     cl::desc("Show verbose output when opt-bisect-limit is set"), cl::Hidden,
     cl::init(true), cl::Optional);
 
-static void printPassMessage(const StringRef &Name, int PassNum,
-                             StringRef TargetDesc, bool Running) {
+static void printPassMessage(StringRef Name, int PassNum, StringRef TargetDesc,
+                             bool Running) {
   StringRef Status = Running ? "" : "NOT ";
-  errs() << "BISECT: " << Status << "running pass "
-         << "(" << PassNum << ") " << Name << " on " << TargetDesc << "\n";
+  errs() << "BISECT: " << Status << "running pass (" << PassNum << ") " << Name
+         << " on " << TargetDesc << '\n';
 }
 
-bool OptBisect::shouldRunPass(const StringRef PassName,
-                              StringRef IRDescription) {
+bool OptBisect::shouldRunPass(StringRef PassName,
+                              StringRef IRDescription) const {
   assert(isEnabled());
 
   int CurBisectNum = ++LastBisectNum;
@@ -55,6 +55,4 @@ bool OptBisect::shouldRunPass(const StringRef PassName,
   return ShouldRun;
 }
 
-const int OptBisect::Disabled;
-
 OptPassGate &llvm::getGlobalPassGate() { return getOptBisector(); }
diff --git a/llvm/lib/IR/Pass.cpp b/llvm/lib/IR/Pass.cpp
index f3ae2e6314429..2c5ef7193b463 100644
--- a/llvm/lib/IR/Pass.cpp
+++ b/llvm/lib/IR/Pass.cpp
@@ -60,8 +60,8 @@ static std::string getDescription(const Module &M) {
   return "module (" + M.getName().str() + ")";
 }
 
-bool ModulePass::skipModule(Module &M) const {
-  OptPassGate &Gate = M.getContext().getOptPassGate();
+bool ModulePass::skipModule(const Module &M) const {
+  const OptPassGate &Gate = M.getContext().getOptPassGate();
   return Gate.isEnabled() &&
          !Gate.shouldRunPass(this->getPassName(), getDescription(M));
 }
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index 740023ca6d23b..b1b5f67689e6d 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -94,6 +94,7 @@ const char *MDProfLabels::ValueProfile = "VP";
 const char *MDProfLabels::FunctionEntryCount = "function_entry_count";
 const char *MDProfLabels::SyntheticFunctionEntryCount =
     "synthetic_function_entry_count";
+const char *MDProfLabels::UnknownBranchWeightsMarker = "unknown";
 
 bool hasProfMD(const Instruction &I) {
   return I.hasMetadata(LLVMContext::MD_prof);
@@ -103,7 +104,7 @@ bool isBranchWeightMD(const MDNode *ProfileData) {
   return isTargetMD(ProfileData, MDProfLabels::BranchWeights, MinBWOps);
 }
 
-static bool isValueProfileMD(const MDNode *ProfileData) {
+bool isValueProfileMD(const MDNode *ProfileData) {
   return isTargetMD(ProfileData, MDProfLabels::ValueProfile, MinVPOps);
 }
 
@@ -241,6 +242,27 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) {
   return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);
 }
 
+void setExplicitlyUnknownBranchWeights(Instruction &I) {
+  MDBuilder MDB(I.getContext());
+  I.setMetadata(
+      LLVMContext::MD_prof,
+      MDNode::get(I.getContext(),
+                  MDB.createString(MDProfLabels::UnknownBranchWeightsMarker)));
+}
+
+bool isExplicitlyUnknownBranchWeightsMetadata(const MDNode &MD) {
+  if (MD.getNumOperands() != 1)
+    return false;
+  return MD.getOperand(0).equalsStr(MDProfLabels::UnknownBranchWeightsMarker);
+}
+
+bool hasExplicitlyUnknownBranchWeights(const Instruction &I) {
+  auto *MD = I.getMetadata(LLVMContext::MD_prof);
+  if (!MD)
+    return false;
+  return isExplicitlyUnknownBranchWeightsMetadata(*MD);
+}
+
 void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
                       bool IsExpected) {
   MDBuilder MDB(I.getContext());
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 712f1a48d0b7b..376b9888c399a 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -12,51 +12,16 @@
 using namespace llvm;
 using namespace RTLIB;
 
-#define GET_INIT_RUNTIME_LIBCALL_UTILS
 #define GET_INIT_RUNTIME_LIBCALL_NAMES
+#define GET_SET_TARGET_RUNTIME_LIBCALL_SETS
 #include "llvm/IR/RuntimeLibcalls.inc"
-#undef GET_INIT_RUNTIME_LIBCALL_UTILS
 #undef GET_INIT_RUNTIME_LIBCALL_NAMES
+#undef GET_SET_TARGET_RUNTIME_LIBCALL_SETS
 
 static cl::opt<bool>
     HexagonEnableFastMathRuntimeCalls("hexagon-fast-math", cl::Hidden,
                                       cl::desc("Enable Fast Math processing"));
 
-static void setAArch64LibcallNames(RuntimeLibcallsInfo &Info,
-                                   const Triple &TT) {
-#define LCALLNAMES(A, B, N)                                                    \
-  Info.setLibcallImpl(A##N##_RELAX, B##N##_relax);                             \
-  Info.setLibcallImpl(A##N##_ACQ, B##N##_acq);                                 \
-  Info.setLibcallImpl(A##N##_REL, B##N##_rel);                                 \
-  Info.setLibcallImpl(A##N##_ACQ_REL, B##N##_acq_rel);
-#define LCALLNAME4(A, B)                                                       \
-  LCALLNAMES(A, B, 1)                                                          \
-  LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
-#define LCALLNAME5(A, B)                                                       \
-  LCALLNAMES(A, B, 1)                                                          \
-  LCALLNAMES(A, B, 2)                                                          \
-  LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
-
-  if (TT.isWindowsArm64EC()) {
-    LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, RTLIB::arm64ec___aarch64_cas)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, RTLIB::arm64ec___aarch64_swp)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, RTLIB::arm64ec___aarch64_ldadd)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, RTLIB::arm64ec___aarch64_ldset)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, RTLIB::arm64ec___aarch64_ldclr)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, RTLIB::arm64ec___aarch64_ldeor)
-  } else {
-    LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, RTLIB::__aarch64_cas)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, RTLIB::__aarch64_swp)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, RTLIB::__aarch64_ldadd)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, RTLIB::__aarch64_ldset)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, RTLIB::__aarch64_ldclr)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, RTLIB::__aarch64_ldeor)
-  }
-#undef LCALLNAMES
-#undef LCALLNAME4
-#undef LCALLNAME5
-}
-
 static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
                                FloatABI::ABIType FloatABIType,
                                EABI EABIVersion) {
@@ -64,8 +29,8 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
     CallingConv::ID DefaultCC = FloatABIType == FloatABI::Hard
                                     ? CallingConv::ARM_AAPCS_VFP
                                     : CallingConv::ARM_AAPCS;
-    for (RTLIB::Libcall LC : RTLIB::libcalls())
-      Info.setLibcallCallingConv(LC, DefaultCC);
+    for (RTLIB::LibcallImpl LC : RTLIB::libcall_impls())
+      Info.setLibcallImplCallingConv(LC, DefaultCC);
   }
 
   // Register based DivRem for AEABI (RTABI 4.2)
@@ -85,7 +50,7 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
 
       for (const auto &LC : LibraryCalls) {
         Info.setLibcallImpl(LC.Op, LC.Impl);
-        Info.setLibcallCallingConv(LC.Op, LC.CC);
+        Info.setLibcallImplCallingConv(LC.Impl, LC.CC);
       }
     } else {
       const struct {
@@ -101,7 +66,7 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
 
       for (const auto &LC : LibraryCalls) {
         Info.setLibcallImpl(LC.Op, LC.Impl);
-        Info.setLibcallCallingConv(LC.Op, LC.CC);
+        Info.setLibcallImplCallingConv(LC.Impl, LC.CC);
       }
     }
   }
@@ -124,7 +89,7 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
 
     for (const auto &LC : LibraryCalls) {
       Info.setLibcallImpl(LC.Op, LC.Impl);
-      Info.setLibcallCallingConv(LC.Op, LC.CC);
+      Info.setLibcallImplCallingConv(LC.Impl, LC.CC);
     }
   }
 
@@ -135,123 +100,6 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
   }
 }
 
-static void setMSP430Libcalls(RuntimeLibcallsInfo &Info, const Triple &TT) {
-  // EABI Libcalls - EABI Section 6.2
-  const struct {
-    const RTLIB::Libcall Op;
-    const RTLIB::LibcallImpl Impl;
-  } LibraryCalls[] = {
-      // Floating point conversions - EABI Table 6
-      {RTLIB::FPROUND_F64_F32, RTLIB::__mspabi_cvtdf},
-      {RTLIB::FPEXT_F32_F64, RTLIB::__mspabi_cvtfd},
-      // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOSINT_F64_I16,  RTLIB::__mspabi_fixdi },
-      {RTLIB::FPTOSINT_F64_I32, RTLIB::__mspabi_fixdli},
-      {RTLIB::FPTOSINT_F64_I64, RTLIB::__mspabi_fixdlli},
-      // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOUINT_F64_I16,  RTLIB::__mspabi_fixdu },
-      {RTLIB::FPTOUINT_F64_I32, RTLIB::__mspabi_fixdul},
-      {RTLIB::FPTOUINT_F64_I64, RTLIB::__mspabi_fixdull},
-      // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOSINT_F32_I16,  RTLIB::__mspabi_fixfi },
-      {RTLIB::FPTOSINT_F32_I32, RTLIB::__mspabi_fixfli},
-      {RTLIB::FPTOSINT_F32_I64, RTLIB::__mspabi_fixflli},
-      // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOUINT_F32_I16,  RTLIB::__mspabi_fixfu },
-      {RTLIB::FPTOUINT_F32_I32, RTLIB::__mspabi_fixful},
-      {RTLIB::FPTOUINT_F32_I64, RTLIB::__mspabi_fixfull},
-      // TODO The following IS implemented in libgcc
-      //{ RTLIB::SINTTOFP_I16_F64,  RTLIB::__mspabi_fltid },
-      {RTLIB::SINTTOFP_I32_F64, RTLIB::__mspabi_fltlid},
-      // TODO The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::SINTTOFP_I64_F64, RTLIB::__mspabi_fltllid},
-      // TODO The following IS implemented in libgcc
-      //{ RTLIB::UINTTOFP_I16_F64,  RTLIB::__mspabi_fltud },
-      {RTLIB::UINTTOFP_I32_F64, RTLIB::__mspabi_fltuld},
-      // The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::UINTTOFP_I64_F64, RTLIB::__mspabi_fltulld},
-      // TODO The following IS implemented in libgcc
-      //{ RTLIB::SINTTOFP_I16_F32,  RTLIB::__mspabi_fltif },
-      {RTLIB::SINTTOFP_I32_F32, RTLIB::__mspabi_fltlif},
-      // TODO The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::SINTTOFP_I64_F32, RTLIB::__mspabi_fltllif},
-      // TODO The following IS implemented in libgcc
-      //{ RTLIB::UINTTOFP_I16_F32,  RTLIB::__mspabi_fltuf },
-      {RTLIB::UINTTOFP_I32_F32, RTLIB::__mspabi_fltulf},
-      // The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::UINTTOFP_I64_F32, RTLIB::__mspabi_fltullf},
-
-      // Floating point comparisons - EABI Table 7
-      {RTLIB::OEQ_F64, RTLIB::__mspabi_cmpd__oeq},
-      {RTLIB::UNE_F64, RTLIB::__mspabi_cmpd__une},
-      {RTLIB::OGE_F64, RTLIB::__mspabi_cmpd__oge},
-      {RTLIB::OLT_F64, RTLIB::__mspabi_cmpd__olt},
-      {RTLIB::OLE_F64, RTLIB::__mspabi_cmpd__ole},
-      {RTLIB::OGT_F64, RTLIB::__mspabi_cmpd__ogt},
-      {RTLIB::OEQ_F32, RTLIB::__mspabi_cmpf__oeq},
-      {RTLIB::UNE_F32, RTLIB::__mspabi_cmpf__une},
-      {RTLIB::OGE_F32, RTLIB::__mspabi_cmpf__oge},
-      {RTLIB::OLT_F32, RTLIB::__mspabi_cmpf__olt},
-      {RTLIB::OLE_F32, RTLIB::__mspabi_cmpf__ole},
-      {RTLIB::OGT_F32, RTLIB::__mspabi_cmpf__ogt},
-
-      // Floating point arithmetic - EABI Table 8
-      {RTLIB::ADD_F64, RTLIB::__mspabi_addd},
-      {RTLIB::ADD_F32, RTLIB::__mspabi_addf},
-      {RTLIB::DIV_F64, RTLIB::__mspabi_divd},
-      {RTLIB::DIV_F32, RTLIB::__mspabi_divf},
-      {RTLIB::MUL_F64, RTLIB::__mspabi_mpyd},
-      {RTLIB::MUL_F32, RTLIB::__mspabi_mpyf},
-      {RTLIB::SUB_F64, RTLIB::__mspabi_subd},
-      {RTLIB::SUB_F32, RTLIB::__mspabi_subf},
-      // The following are NOT implemented in libgcc
-      // { RTLIB::NEG_F64,  RTLIB::__mspabi_negd },
-      // { RTLIB::NEG_F32,  RTLIB::__mspabi_negf },
-
-      // Universal Integer Operations - EABI Table 9
-      {RTLIB::SDIV_I16, RTLIB::__mspabi_divi},
-      {RTLIB::SDIV_I32, RTLIB::__mspabi_divli},
-      {RTLIB::SDIV_I64, RTLIB::__mspabi_divlli},
-      {RTLIB::UDIV_I16, RTLIB::__mspabi_divu},
-      {RTLIB::UDIV_I32, RTLIB::__mspabi_divul},
-      {RTLIB::UDIV_I64, RTLIB::__mspabi_divull},
-      {RTLIB::SREM_I16, RTLIB::__mspabi_remi},
-      {RTLIB::SREM_I32, RTLIB::__mspabi_remli},
-      {RTLIB::SREM_I64, RTLIB::__mspabi_remlli},
-      {RTLIB::UREM_I16, RTLIB::__mspabi_remu},
-      {RTLIB::UREM_I32, RTLIB::__mspabi_remul},
-      {RTLIB::UREM_I64, RTLIB::__mspabi_remull},
-
-      // Bitwise Operations - EABI Table 10
-      // TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
-      {RTLIB::SRL_I32, RTLIB::__mspabi_srll},
-      {RTLIB::SRA_I32, RTLIB::__mspabi_sral},
-      {RTLIB::SHL_I32, RTLIB::__mspabi_slll},
-      // __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
-  };
-
-  for (const auto &LC : LibraryCalls)
-    Info.setLibcallImpl(LC.Op, LC.Impl);
-
-  // Several of the runtime library functions use a special calling conv
-  Info.setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::ADD_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::SUB_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::MUL_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::DIV_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::OEQ_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::UNE_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::OGE_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::OLT_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::OLE_F64, CallingConv::MSP430_BUILTIN);
-  Info.setLibcallCallingConv(RTLIB::OGT_F64, CallingConv::MSP430_BUILTIN);
-
-  // TODO: __mspabi_srall, __mspabi_srlll, __mspabi_sllll
-}
-
 void RuntimeLibcallsInfo::initSoftFloatCmpLibcallPredicates() {
   SoftFloatCompareLibcallPredicates[RTLIB::OEQ_F32] = CmpInst::ICMP_EQ;
   SoftFloatCompareLibcallPredicates[RTLIB::OEQ_F64] = CmpInst::ICMP_EQ;
@@ -358,37 +206,17 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
                                        ExceptionHandling ExceptionModel,
                                        FloatABI::ABIType FloatABI,
                                        EABI EABIVersion, StringRef ABIName) {
+  setTargetRuntimeLibcallSets(TT);
+
   // Use the f128 variants of math functions on x86
   if (TT.isX86() && TT.isGNUEnvironment())
     setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/true);
 
-  if (TT.isX86() || TT.isVE()) {
+  if (TT.isX86() || TT.isVE() || TT.isARM() || TT.isThumb()) {
     if (ExceptionModel == ExceptionHandling::SjLj)
       setLibcallImpl(RTLIB::UNWIND_RESUME, RTLIB::_Unwind_SjLj_Resume);
   }
 
-  if (TT.isPPC()) {
-    setPPCLibCallNameOverrides();
-
-    // TODO: Do the finite only functions exist?
-    setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/false);
-
-    // TODO: Tablegen predicate support
-    if (TT.isOSAIX()) {
-      if (TT.isPPC64()) {
-        setLibcallImpl(RTLIB::MEMCPY, RTLIB::Unsupported);
-        setLibcallImpl(RTLIB::MEMMOVE, RTLIB::___memmove64);
-        setLibcallImpl(RTLIB::MEMSET, RTLIB::___memset64);
-        setLibcallImpl(RTLIB::BZERO, RTLIB::___bzero64);
-      } else {
-        setLibcallImpl(RTLIB::MEMCPY, RTLIB::Unsupported);
-        setLibcallImpl(RTLIB::MEMMOVE, RTLIB::___memmove);
-        setLibcallImpl(RTLIB::MEMSET, RTLIB::___memset);
-        setLibcallImpl(RTLIB::BZERO, RTLIB::___bzero);
-      }
-    }
-  }
-
   // A few names are different on particular architectures or environments.
   if (TT.isOSDarwin()) {
     // For f16/f32 conversions, Darwin uses the standard naming scheme,
@@ -401,17 +229,16 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
     if (TT.isX86()) {
       if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
         setLibcallImpl(RTLIB::BZERO, RTLIB::__bzero);
-    } else if (TT.isAArch64())
-      setLibcallImpl(RTLIB::BZERO, RTLIB::bzero);
+    }
 
     if (darwinHasSinCosStret(TT)) {
       setLibcallImpl(RTLIB::SINCOS_STRET_F32, RTLIB::__sincosf_stret);
       setLibcallImpl(RTLIB::SINCOS_STRET_F64, RTLIB::__sincos_stret);
       if (TT.isWatchABI()) {
-        setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
-                              CallingConv::ARM_AAPCS_VFP);
-        setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
-                              CallingConv::ARM_AAPCS_VFP);
+        setLibcallImplCallingConv(RTLIB::__sincosf_stret,
+                                  CallingConv::ARM_AAPCS_VFP);
+        setLibcallImplCallingConv(RTLIB::__sincos_stret,
+                                  CallingConv::ARM_AAPCS_VFP);
       }
     }
 
@@ -453,14 +280,6 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
     setLibcallImpl(RTLIB::FREXP_PPCF128, RTLIB::Unsupported);
   }
 
-  // Disable most libcalls on AMDGPU and NVPTX.
-  if (TT.isAMDGPU() || TT.isNVPTX()) {
-    for (RTLIB::Libcall LC : RTLIB::libcalls()) {
-      if (!isAtomicLibCall(LC))
-        setLibcallImpl(LC, RTLIB::Unsupported);
-    }
-  }
-
   if (TT.isOSMSVCRT()) {
     // MSVCRT doesn't have powi; fall back to pow
     setLibcallImpl(RTLIB::POWI_F32, RTLIB::Unsupported);
@@ -484,60 +303,12 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
 
     for (const auto &LC : LibraryCalls) {
       setLibcallImpl(LC.Op, LC.Impl);
-      setLibcallCallingConv(LC.Op, LC.CC);
+      setLibcallImplCallingConv(LC.Impl, LC.CC);
     }
   }
 
-  if (TT.isAArch64()) {
-    if (TT.isWindowsArm64EC()) {
-      setWindowsArm64LibCallNameOverrides();
-      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::arm64ec___arm_sc_memcpy);
-      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::arm64ec___arm_sc_memmove);
-      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::arm64ec___arm_sc_memset);
-    } else {
-      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::__arm_sc_memcpy);
-      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::__arm_sc_memmove);
-      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::__arm_sc_memset);
-    }
-
-    setAArch64LibcallNames(*this, TT);
-  } else if (TT.isARM() || TT.isThumb()) {
+  if (TT.isARM() || TT.isThumb())
     setARMLibcallNames(*this, TT, FloatABI, EABIVersion);
-  } else if (TT.getArch() == Triple::ArchType::avr) {
-    // Division rtlib functions (not supported), use divmod functions instead
-    setLibcallImpl(RTLIB::SDIV_I8, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::SDIV_I16, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::SDIV_I32, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::UDIV_I8, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::UDIV_I16, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::UDIV_I32, RTLIB::Unsupported);
-
-    // Modulus rtlib functions (not supported), use divmod functions instead
-    setLibcallImpl(RTLIB::SREM_I8, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::SREM_I16, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::SREM_I32, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::UREM_I8, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::UREM_I16, RTLIB::Unsupported);
-    setLibcallImpl(RTLIB::UREM_I32, RTLIB::Unsupported);
-
-    // Division and modulus rtlib functions
-    setLibcallImpl(RTLIB::SDIVREM_I8, RTLIB::__divmodqi4);
-    setLibcallImpl(RTLIB::SDIVREM_I16, RTLIB::__divmodhi4);
-    setLibcallImpl(RTLIB::SDIVREM_I32, RTLIB::__divmodsi4);
-    setLibcallImpl(RTLIB::UDIVREM_I8, RTLIB::__udivmodqi4);
-    setLibcallImpl(RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4);
-    setLibcallImpl(RTLIB::UDIVREM_I32, RTLIB::__udivmodsi4);
-
-    // Several of the runtime library functions use a special calling conv
-    setLibcallCallingConv(RTLIB::SDIVREM_I8, CallingConv::AVR_BUILTIN);
-    setLibcallCallingConv(RTLIB::SDIVREM_I16, CallingConv::AVR_BUILTIN);
-    setLibcallCallingConv(RTLIB::UDIVREM_I8, CallingConv::AVR_BUILTIN);
-    setLibcallCallingConv(RTLIB::UDIVREM_I16, CallingConv::AVR_BUILTIN);
-
-    // Trigonometric rtlib functions
-    setLibcallImpl(RTLIB::SIN_F32, RTLIB::avr_sin);
-    setLibcallImpl(RTLIB::COS_F32, RTLIB::avr_cos);
-  }
 
   if (!TT.isWasm()) {
     // These libcalls are only available in compiler-rt, not libgcc.
@@ -550,11 +321,6 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
     }
 
     setLibcallImpl(RTLIB::MULO_I128, RTLIB::Unsupported);
-  } else {
-    // Define the emscripten name for return address helper.
-    // TODO: when implementing other Wasm backends, make this generic or only do
-    // this on emscripten depending on what they end up doing.
-    setLibcallImpl(RTLIB::RETURN_ADDRESS, RTLIB::emscripten_return_address);
   }
 
   if (TT.getArch() == Triple::ArchType::hexagon) {
@@ -599,19 +365,13 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
         RTLIB::__hexagon_memcpy_likely_aligned_min32bytes_mult8bytes);
   }
 
-  if (TT.getArch() == Triple::ArchType::msp430)
-    setMSP430Libcalls(*this, TT);
-
-  if (TT.isSystemZ() && TT.isOSzOS())
-    setZOSLibCallNameOverrides();
-
-  if (TT.getArch() == Triple::ArchType::xcore)
-    setLibcallImpl(RTLIB::MEMCPY_ALIGN_4, RTLIB::__memcpy_4);
+  if (TT.getArch() == Triple::ArchType::msp430) {
+    setLibcallImplCallingConv(RTLIB::__mspabi_mpyll,
+                              CallingConv::MSP430_BUILTIN);
+  }
 }
 
 bool RuntimeLibcallsInfo::darwinHasExp10(const Triple &TT) {
-  assert(TT.isOSDarwin() && "should be called with darwin triple");
-
   switch (TT.getOS()) {
   case Triple::MacOSX:
     return !TT.isMacOSXVersionLT(10, 9);
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 7e64992c2dfe2..5e1bf2863191c 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -984,7 +984,7 @@ struct TargetTypeInfo {
 static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
   LLVMContext &C = Ty->getContext();
   StringRef Name = Ty->getName();
-  if (Name == "spirv.Image")
+  if (Name == "spirv.Image" || Name == "spirv.SignedImage")
     return TargetTypeInfo(PointerType::get(C, 0), TargetExtType::CanBeGlobal,
                           TargetExtType::CanBeLocal);
   if (Name == "spirv.Type") {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 9cab88b09779a..eb747bc48a8a5 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -112,6 +112,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
@@ -2526,6 +2527,12 @@ void Verifier::verifyFunctionMetadata(
   for (const auto &Pair : MDs) {
     if (Pair.first == LLVMContext::MD_prof) {
       MDNode *MD = Pair.second;
+      if (isExplicitlyUnknownBranchWeightsMetadata(*MD)) {
+        CheckFailed("'unknown' !prof metadata should appear only on "
+                    "instructions supporting the 'branch_weights' metadata",
+                    MD);
+        continue;
+      }
       Check(MD->getNumOperands() >= 2,
             "!prof annotations should have no less than 2 operands", MD);
 
@@ -3178,6 +3185,12 @@ void Verifier::visitFunction(const Function &F) {
     CheckDI(SP->describes(&F),
             "!dbg attachment points at wrong subprogram for function", N, &F,
             &I, DL, Scope, SP);
+
+    if (DL->getAtomGroup())
+      CheckDI(DL->getScope()->getSubprogram()->getKeyInstructionsEnabled(),
+              "DbgLoc uses atomGroup but DISubprogram doesn't have Key "
+              "Instructions enabled",
+              DL, DL->getScope()->getSubprogram());
   };
   for (auto &BB : F)
     for (auto &I : BB) {
@@ -4982,9 +4995,24 @@ void Verifier::visitDereferenceableMetadata(Instruction& I, MDNode* MD) {
 }
 
 void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
-  Check(MD->getNumOperands() >= 2,
-        "!prof annotations should have no less than 2 operands", MD);
-
+  auto GetBranchingTerminatorNumOperands = [&]() {
+    unsigned ExpectedNumOperands = 0;
+    if (BranchInst *BI = dyn_cast<BranchInst>(&I))
+      ExpectedNumOperands = BI->getNumSuccessors();
+    else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I))
+      ExpectedNumOperands = SI->getNumSuccessors();
+    else if (isa<CallInst>(&I))
+      ExpectedNumOperands = 1;
+    else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I))
+      ExpectedNumOperands = IBI->getNumDestinations();
+    else if (isa<SelectInst>(&I))
+      ExpectedNumOperands = 2;
+    else if (CallBrInst *CI = dyn_cast<CallBrInst>(&I))
+      ExpectedNumOperands = CI->getNumSuccessors();
+    return ExpectedNumOperands;
+  };
+  Check(MD->getNumOperands() >= 1,
+        "!prof annotations should have at least 1 operand", MD);
   // Check first operand.
   Check(MD->getOperand(0) != nullptr, "first operand should not be null", MD);
   Check(isa<MDString>(MD->getOperand(0)),
@@ -4992,6 +5020,19 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
   MDString *MDS = cast<MDString>(MD->getOperand(0));
   StringRef ProfName = MDS->getString();
 
+  if (ProfName == MDProfLabels::UnknownBranchWeightsMarker) {
+    Check(GetBranchingTerminatorNumOperands() != 0 || isa<InvokeInst>(I),
+          "'unknown' !prof should only appear on instructions on which "
+          "'branch_weights' would",
+          MD);
+    Check(MD->getNumOperands() == 1,
+          "'unknown' !prof should have no additional operands", MD);
+    return;
+  }
+
+  Check(MD->getNumOperands() >= 2,
+        "!prof annotations should have no less than 2 operands", MD);
+
   // Check consistency of !prof branch_weights metadata.
   if (ProfName == MDProfLabels::BranchWeights) {
     unsigned NumBranchWeights = getNumBranchWeights(*MD);
@@ -4999,20 +5040,8 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
       Check(NumBranchWeights == 1 || NumBranchWeights == 2,
             "Wrong number of InvokeInst branch_weights operands", MD);
     } else {
-      unsigned ExpectedNumOperands = 0;
-      if (BranchInst *BI = dyn_cast<BranchInst>(&I))
-        ExpectedNumOperands = BI->getNumSuccessors();
-      else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I))
-        ExpectedNumOperands = SI->getNumSuccessors();
-      else if (isa<CallInst>(&I))
-        ExpectedNumOperands = 1;
-      else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I))
-        ExpectedNumOperands = IBI->getNumDestinations();
-      else if (isa<SelectInst>(&I))
-        ExpectedNumOperands = 2;
-      else if (CallBrInst *CI = dyn_cast<CallBrInst>(&I))
-        ExpectedNumOperands = CI->getNumSuccessors();
-      else
+      const unsigned ExpectedNumOperands = GetBranchingTerminatorNumOperands();
+      if (ExpectedNumOperands == 0)
         CheckFailed("!prof branch_weights are not allowed for this instruction",
                     MD);
 
@@ -5026,9 +5055,27 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
       Check(mdconst::dyn_extract<ConstantInt>(MDO),
             "!prof brunch_weights operand is not a const int");
     }
+  } else if (ProfName == MDProfLabels::ValueProfile) {
+    Check(isValueProfileMD(MD), "invalid value profiling metadata", MD);
+    ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
+    Check(KindInt, "VP !prof missing kind argument", MD);
+
+    auto Kind = KindInt->getZExtValue();
+    Check(Kind >= InstrProfValueKind::IPVK_First &&
+              Kind <= InstrProfValueKind::IPVK_Last,
+          "Invalid VP !prof kind", MD);
+    Check(MD->getNumOperands() % 2 == 1,
+          "VP !prof should have an even number "
+          "of arguments after 'VP'",
+          MD);
+    if (Kind == InstrProfValueKind::IPVK_IndirectCallTarget ||
+        Kind == InstrProfValueKind::IPVK_MemOPSize)
+      Check(isa<CallBase>(I),
+            "VP !prof indirect call or memop size expected to be applied to "
+            "CallBase instructions only",
+            MD);
   } else {
-    Check(ProfName == MDProfLabels::ValueProfile,
-          "expected either branch_weights or VP profile name", MD);
+    CheckFailed("expected either branch_weights or VP profile name", MD);
   }
 }
 
@@ -6892,20 +6939,44 @@ void Verifier::visitVPIntrinsic(VPIntrinsic &VPI) {
       break;
     }
   }
-  if (VPI.getIntrinsicID() == Intrinsic::vp_fcmp) {
+
+  switch (VPI.getIntrinsicID()) {
+  case Intrinsic::vp_fcmp: {
     auto Pred = cast<VPCmpIntrinsic>(&VPI)->getPredicate();
     Check(CmpInst::isFPPredicate(Pred),
           "invalid predicate for VP FP comparison intrinsic", &VPI);
+    break;
   }
-  if (VPI.getIntrinsicID() == Intrinsic::vp_icmp) {
+  case Intrinsic::vp_icmp: {
     auto Pred = cast<VPCmpIntrinsic>(&VPI)->getPredicate();
     Check(CmpInst::isIntPredicate(Pred),
           "invalid predicate for VP integer comparison intrinsic", &VPI);
+    break;
   }
-  if (VPI.getIntrinsicID() == Intrinsic::vp_is_fpclass) {
+  case Intrinsic::vp_is_fpclass: {
     auto TestMask = cast<ConstantInt>(VPI.getOperand(1));
     Check((TestMask->getZExtValue() & ~static_cast<unsigned>(fcAllFlags)) == 0,
           "unsupported bits for llvm.vp.is.fpclass test mask");
+    break;
+  }
+  case Intrinsic::experimental_vp_splice: {
+    VectorType *VecTy = cast<VectorType>(VPI.getType());
+    int64_t Idx = cast<ConstantInt>(VPI.getArgOperand(2))->getSExtValue();
+    int64_t KnownMinNumElements = VecTy->getElementCount().getKnownMinValue();
+    if (VPI.getParent() && VPI.getParent()->getParent()) {
+      AttributeList Attrs = VPI.getParent()->getParent()->getAttributes();
+      if (Attrs.hasFnAttr(Attribute::VScaleRange))
+        KnownMinNumElements *= Attrs.getFnAttrs().getVScaleRangeMin();
+    }
+    Check((Idx < 0 && std::abs(Idx) <= KnownMinNumElements) ||
+              (Idx >= 0 && Idx < KnownMinNumElements),
+          "The splice index exceeds the range [-VL, VL-1] where VL is the "
+          "known minimum number of elements in the vector. For scalable "
+          "vectors the minimum number of elements is determined from "
+          "vscale_range.",
+          &VPI);
+    break;
+  }
   }
 }
 
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 779c98ee4441e..73e79c08a56ca 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -135,7 +135,7 @@ std::string llvm::computeLTOCacheKey(
     Hasher.update(Data);
   };
   auto AddUint8 = [&](const uint8_t I) {
-    Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&I, 1));
+    Hasher.update(ArrayRef<uint8_t>(&I, 1));
   };
   AddString(Conf.CPU);
   // FIXME: Hash more of Options. For now all clients initialize Options from
diff --git a/llvm/lib/MC/ConstantPools.cpp b/llvm/lib/MC/ConstantPools.cpp
index 824d2463f30fc..bf74ad42f91fd 100644
--- a/llvm/lib/MC/ConstantPools.cpp
+++ b/llvm/lib/MC/ConstantPools.cpp
@@ -59,7 +59,7 @@ const MCExpr *ConstantPool::addEntry(const MCExpr *Value, MCContext &Context,
   MCSymbol *CPEntryLabel = Context.createTempSymbol();
 
   Entries.push_back(ConstantPoolEntry(CPEntryLabel, Value, Size, Loc));
-  const auto SymRef = MCSymbolRefExpr::create(CPEntryLabel, Context);
+  const auto SymRef = MCSymbolRefExpr::create(CPEntryLabel, Context, Loc);
   if (C)
     CachedConstantEntries[std::make_pair(C->getValue(), Size)] = SymRef;
   if (S)
diff --git a/llvm/lib/MC/DXContainerRootSignature.cpp b/llvm/lib/MC/DXContainerRootSignature.cpp
index 0d51d96cfc7bf..482280b5ef289 100644
--- a/llvm/lib/MC/DXContainerRootSignature.cpp
+++ b/llvm/lib/MC/DXContainerRootSignature.cpp
@@ -71,12 +71,16 @@ void RootSignatureDesc::write(raw_ostream &OS) const {
   BOS.reserveExtraSpace(getSize());
 
   const uint32_t NumParameters = ParametersContainer.size();
-
+  const uint32_t NumSamplers = StaticSamplers.size();
   support::endian::write(BOS, Version, llvm::endianness::little);
   support::endian::write(BOS, NumParameters, llvm::endianness::little);
   support::endian::write(BOS, RootParameterOffset, llvm::endianness::little);
-  support::endian::write(BOS, NumStaticSamplers, llvm::endianness::little);
-  support::endian::write(BOS, StaticSamplersOffset, llvm::endianness::little);
+  support::endian::write(BOS, NumSamplers, llvm::endianness::little);
+  uint32_t SSO = StaticSamplersOffset;
+  if (NumSamplers > 0)
+    SSO = writePlaceholder(BOS);
+  else
+    support::endian::write(BOS, SSO, llvm::endianness::little);
   support::endian::write(BOS, Flags, llvm::endianness::little);
 
   SmallVector<uint32_t> ParamsOffsets;
@@ -142,20 +146,23 @@ void RootSignatureDesc::write(raw_ostream &OS) const {
     }
     }
   }
-  for (const auto &S : StaticSamplers) {
-    support::endian::write(BOS, S.Filter, llvm::endianness::little);
-    support::endian::write(BOS, S.AddressU, llvm::endianness::little);
-    support::endian::write(BOS, S.AddressV, llvm::endianness::little);
-    support::endian::write(BOS, S.AddressW, llvm::endianness::little);
-    support::endian::write(BOS, S.MipLODBias, llvm::endianness::little);
-    support::endian::write(BOS, S.MaxAnisotropy, llvm::endianness::little);
-    support::endian::write(BOS, S.ComparisonFunc, llvm::endianness::little);
-    support::endian::write(BOS, S.BorderColor, llvm::endianness::little);
-    support::endian::write(BOS, S.MinLOD, llvm::endianness::little);
-    support::endian::write(BOS, S.MaxLOD, llvm::endianness::little);
-    support::endian::write(BOS, S.ShaderRegister, llvm::endianness::little);
-    support::endian::write(BOS, S.RegisterSpace, llvm::endianness::little);
-    support::endian::write(BOS, S.ShaderVisibility, llvm::endianness::little);
+  if (NumSamplers > 0) {
+    rewriteOffsetToCurrentByte(BOS, SSO);
+    for (const auto &S : StaticSamplers) {
+      support::endian::write(BOS, S.Filter, llvm::endianness::little);
+      support::endian::write(BOS, S.AddressU, llvm::endianness::little);
+      support::endian::write(BOS, S.AddressV, llvm::endianness::little);
+      support::endian::write(BOS, S.AddressW, llvm::endianness::little);
+      support::endian::write(BOS, S.MipLODBias, llvm::endianness::little);
+      support::endian::write(BOS, S.MaxAnisotropy, llvm::endianness::little);
+      support::endian::write(BOS, S.ComparisonFunc, llvm::endianness::little);
+      support::endian::write(BOS, S.BorderColor, llvm::endianness::little);
+      support::endian::write(BOS, S.MinLOD, llvm::endianness::little);
+      support::endian::write(BOS, S.MaxLOD, llvm::endianness::little);
+      support::endian::write(BOS, S.ShaderRegister, llvm::endianness::little);
+      support::endian::write(BOS, S.RegisterSpace, llvm::endianness::little);
+      support::endian::write(BOS, S.ShaderVisibility, llvm::endianness::little);
+    }
   }
   assert(Storage.size() == getSize());
   OS.write(Storage.data(), Storage.size());
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 0b67f79c17f7a..9f52b3e3e85c0 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -27,8 +27,6 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
@@ -121,7 +119,7 @@ struct ELFWriter {
   } Mode;
 
   uint64_t symbolValue(const MCSymbol &Sym);
-  bool isInSymtab(const MCSymbolELF &Symbol, bool Used, bool Renamed);
+  bool isInSymtab(const MCSymbolELF &Symbol);
 
   /// Helper struct for containing some precomputed information on symbols.
   struct ELFSymbolData {
@@ -399,8 +397,9 @@ static bool isIFunc(const MCSymbolELF *Symbol) {
     const MCSymbolRefExpr *Value;
     if (!Symbol->isVariable() ||
         !(Value = dyn_cast<MCSymbolRefExpr>(Symbol->getVariableValue())) ||
-        Value->getKind() != MCSymbolRefExpr::VK_None ||
-        mergeTypeForSet(Symbol->getType(), ELF::STT_GNU_IFUNC) != ELF::STT_GNU_IFUNC)
+        Value->getSpecifier() ||
+        mergeTypeForSet(Symbol->getType(), ELF::STT_GNU_IFUNC) !=
+            ELF::STT_GNU_IFUNC)
       return false;
     Symbol = &cast<MCSymbolELF>(Value->getSymbol());
   }
@@ -468,7 +467,13 @@ void ELFWriter::writeSymbol(SymbolTableWriter &Writer, uint32_t StringIndex,
                      IsReserved);
 }
 
-bool ELFWriter::isInSymtab(const MCSymbolELF &Symbol, bool Used, bool Renamed) {
+bool ELFWriter::isInSymtab(const MCSymbolELF &Symbol) {
+  if (Symbol.isUsedInReloc() || Symbol.isSignature())
+    return true;
+
+  if (OWriter.Renames.count(&Symbol))
+    return false;
+
   if (Symbol.isVariable()) {
     const MCExpr *Expr = Symbol.getVariableValue();
     // Target Expressions that are always inlined do not appear in the symtab
@@ -478,27 +483,18 @@ bool ELFWriter::isInSymtab(const MCSymbolELF &Symbol, bool Used, bool Renamed) {
     // The .weakref alias does not appear in the symtab.
     if (Symbol.isWeakref())
       return false;
-  }
-
-  if (Used)
-    return true;
 
-  if (Renamed)
-    return false;
-
-  if (Symbol.isVariable() && Symbol.isUndefined()) {
-    // FIXME: this is here just to diagnose the case of a var = commmon_sym.
-    Asm.getBaseSymbol(Symbol);
-    return false;
+    if (Symbol.isUndefined()) {
+      // FIXME: this is here just to diagnose the case of a var = commmon_sym.
+      Asm.getBaseSymbol(Symbol);
+      return false;
+    }
   }
 
   if (Symbol.isTemporary())
     return false;
 
-  if (Symbol.getType() == ELF::STT_SECTION)
-    return false;
-
-  return true;
+  return Symbol.getType() != ELF::STT_SECTION;
 }
 
 void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
@@ -528,10 +524,7 @@ void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
   bool HasLargeSectionIndex = false;
   for (auto It : llvm::enumerate(Asm.symbols())) {
     const auto &Symbol = cast<MCSymbolELF>(It.value());
-    bool Used = Symbol.isUsedInReloc();
-    bool isSignature = Symbol.isSignature();
-    if (!isInSymtab(Symbol, Used || isSignature,
-                    OWriter.Renames.count(&Symbol)))
+    if (!isInSymtab(Symbol))
       continue;
 
     if (Symbol.isTemporary() && Symbol.isUndefined()) {
@@ -556,7 +549,7 @@ void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
         MSD.SectionIndex = ELF::SHN_COMMON;
       }
     } else if (Symbol.isUndefined()) {
-      if (isSignature && !Used) {
+      if (Symbol.isSignature() && !Symbol.isUsedInReloc()) {
         MSD.SectionIndex = RevGroupMap.lookup(&Symbol);
         if (MSD.SectionIndex >= ELF::SHN_LORESERVE)
           HasLargeSectionIndex = true;
@@ -1332,7 +1325,6 @@ bool ELFObjectWriter::checkRelocation(SMLoc Loc, const MCSectionELF *From,
 void ELFObjectWriter::recordRelocation(const MCFragment &F,
                                        const MCFixup &Fixup, MCValue Target,
                                        uint64_t &FixedValue) {
-  MCAsmBackend &Backend = Asm->getBackend();
   const MCSectionELF &Section = cast<MCSectionELF>(*F.getParent());
   MCContext &Ctx = getContext();
 
@@ -1343,8 +1335,7 @@ void ELFObjectWriter::recordRelocation(const MCFragment &F,
   if (DwoOS && !checkRelocation(Fixup.getLoc(), &Section, SecA))
     return;
 
-  bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags &
-                 MCFixupKindInfo::FKF_IsPCRel;
+  bool IsPCRel = Fixup.isPCRel();
   uint64_t FixupOffset = Asm->getFragmentOffset(F) + Fixup.getOffset();
   uint64_t Addend = Target.getConstant();
   if (auto *RefB = Target.getSubSym()) {
diff --git a/llvm/lib/MC/MCAsmBackend.cpp b/llvm/lib/MC/MCAsmBackend.cpp
index c69e42dfc9fe6..39ef521031069 100644
--- a/llvm/lib/MC/MCAsmBackend.cpp
+++ b/llvm/lib/MC/MCAsmBackend.cpp
@@ -10,7 +10,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCDXContainerWriter.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCGOFFObjectWriter.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -87,6 +86,7 @@ std::optional<MCFixupKind> MCAsmBackend::getFixupKind(StringRef Name) const {
 }
 
 MCFixupKindInfo MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+  // clang-format off
   static const MCFixupKindInfo Builtins[] = {
       {"FK_NONE", 0, 0, 0},
       {"FK_Data_1", 0, 8, 0},
@@ -94,15 +94,12 @@ MCFixupKindInfo MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"FK_Data_4", 0, 32, 0},
       {"FK_Data_8", 0, 64, 0},
       {"FK_Data_leb128", 0, 0, 0},
-      {"FK_PCRel_1", 0, 8, MCFixupKindInfo::FKF_IsPCRel},
-      {"FK_PCRel_2", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"FK_PCRel_4", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"FK_PCRel_8", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
       {"FK_SecRel_1", 0, 8, 0},
       {"FK_SecRel_2", 0, 16, 0},
       {"FK_SecRel_4", 0, 32, 0},
       {"FK_SecRel_8", 0, 64, 0},
   };
+  // clang-format on
 
   assert(size_t(Kind - FK_NONE) < std::size(Builtins) && "Unknown fixup kind");
   return Builtins[Kind - FK_NONE];
@@ -116,14 +113,11 @@ bool MCAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
   return fixupNeedsRelaxation(Fixup, Value);
 }
 
-bool MCAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
-                            const MCValue &Target, uint64_t &FixedValue,
-                            bool IsResolved) {
-  if (IsResolved && shouldForceRelocation(Fixup, Target))
-    IsResolved = false;
+void MCAsmBackend::maybeAddReloc(const MCFragment &F, const MCFixup &Fixup,
+                                 const MCValue &Target, uint64_t &Value,
+                                 bool IsResolved) {
   if (!IsResolved)
-    Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
-  return IsResolved;
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
 }
 
 bool MCAsmBackend::isDarwinCanonicalPersonality(const MCSymbol *Sym) const {
@@ -144,21 +138,9 @@ bool MCAsmBackend::isDarwinCanonicalPersonality(const MCSymbol *Sym) const {
 
 const MCSubtargetInfo *MCAsmBackend::getSubtargetInfo(const MCFragment &F) {
   const MCSubtargetInfo *STI = nullptr;
-  switch (F.getKind()) {
-  case MCFragment::FT_Data: {
-    auto &DF = cast<MCDataFragment>(F);
-    STI = DF.getSubtargetInfo();
-    assert(!DF.hasInstructions() || STI != nullptr);
-    break;
-  }
-  case MCFragment::FT_Relaxable: {
-    auto &RF = cast<MCRelaxableFragment>(F);
-    STI = RF.getSubtargetInfo();
-    assert(!RF.hasInstructions() || STI != nullptr);
-    break;
-  }
-  default:
-    break;
+  if (auto *DF = dyn_cast<MCEncodedFragment>(&F)) {
+    STI = DF->getSubtargetInfo();
+    assert(!DF->hasInstructions() || STI != nullptr);
   }
   return STI;
 }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index c6dbc625642d4..aae02652564d3 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCCodeView.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
@@ -690,22 +689,14 @@ void MCAsmStreamer::emitDarwinTargetVariantBuildVersion(
 }
 
 void MCAsmStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
-  // Do not emit a .set on inlined target assignments.
-  bool EmitSet = true;
-  if (auto *E = dyn_cast<MCTargetExpr>(Value))
-    if (E->inlineAssignedExpr())
-      EmitSet = false;
-  if (EmitSet) {
-    bool UseSet = MAI->usesSetToEquateSymbol();
-    if (UseSet)
-      OS << ".set ";
-    Symbol->print(OS, MAI);
-    OS << (UseSet ? ", " : " = ");
-    MAI->printExpr(OS, *Value);
-
-    EmitEOL();
-  }
+  bool UseSet = MAI->usesSetToEquateSymbol();
+  if (UseSet)
+    OS << ".set ";
+  Symbol->print(OS, MAI);
+  OS << (UseSet ? ", " : " = ");
+  MAI->printExpr(OS, *Value);
 
+  EmitEOL();
   MCStreamer::emitAssignment(Symbol, Value);
 }
 
@@ -1229,7 +1220,7 @@ void MCAsmStreamer::PrintQuotedString(StringRef Data, raw_ostream &OS) const {
         continue;
       }
 
-      if (isPrint((unsigned char)C)) {
+      if (isPrint(C)) {
         OS << (char)C;
         continue;
       }
@@ -2422,9 +2413,14 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
     auto Kind = F.getKind();
     if (mc::isRelocation(Kind))
       OS << ", relocation type: " << Kind;
-    else
-      OS << ", kind: "
-         << getAssembler().getBackend().getFixupKindInfo(Kind).Name;
+    else {
+      OS << ", kind: ";
+      auto Info = getAssembler().getBackend().getFixupKindInfo(Kind);
+      if (F.isPCRel() && StringRef(Info.Name).starts_with("FK_Data_"))
+        OS << "FK_PCRel_" << (Info.TargetSize / 8);
+      else
+        OS << Info.Name;
+    }
     OS << '\n';
   }
 }
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 1866c5b9e0e81..480e6fe027beb 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -21,8 +21,6 @@
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -129,7 +127,7 @@ bool MCAssembler::isThumbFunc(const MCSymbol *Symbol) const {
   if (!Expr->evaluateAsRelocatable(V, nullptr))
     return false;
 
-  if (V.getSubSym() || V.getSpecifier() != MCSymbolRefExpr::VK_None)
+  if (V.getSubSym() || V.getSpecifier())
     return false;
 
   auto *Sym = V.getAddSym();
@@ -143,7 +141,7 @@ bool MCAssembler::isThumbFunc(const MCSymbol *Symbol) const {
   return true;
 }
 
-bool MCAssembler::evaluateFixup(const MCFragment *DF, const MCFixup &Fixup,
+bool MCAssembler::evaluateFixup(const MCFragment &F, MCFixup &Fixup,
                                 MCValue &Target, uint64_t &Value,
                                 bool RecordReloc,
                                 MutableArrayRef<char> Contents) const {
@@ -163,38 +161,25 @@ bool MCAssembler::evaluateFixup(const MCFragment *DF, const MCFixup &Fixup,
   }
 
   bool IsResolved = false;
-  unsigned FixupFlags = getBackend().getFixupKindInfo(Fixup.getKind()).Flags;
-  if (FixupFlags & MCFixupKindInfo::FKF_IsTarget) {
-    IsResolved = getBackend().evaluateTargetFixup(Fixup, Target, Value);
+  if (auto State = getBackend().evaluateFixup(F, Fixup, Target, Value)) {
+    IsResolved = *State;
   } else {
     const MCSymbol *Add = Target.getAddSym();
     const MCSymbol *Sub = Target.getSubSym();
-    Value = Target.getConstant();
+    Value += Target.getConstant();
     if (Add && Add->isDefined())
       Value += getSymbolOffset(*Add);
     if (Sub && Sub->isDefined())
       Value -= getSymbolOffset(*Sub);
 
-    bool IsPCRel = FixupFlags & MCFixupKindInfo::FKF_IsPCRel;
-    bool ShouldAlignPC =
-        FixupFlags & MCFixupKindInfo::FKF_IsAlignedDownTo32Bits;
-    if (IsPCRel) {
-      uint64_t Offset = getFragmentOffset(*DF) + Fixup.getOffset();
-
-      // A number of ARM fixups in Thumb mode require that the effective PC
-      // address be determined as the 32-bit aligned version of the actual
-      // offset.
-      if (ShouldAlignPC)
-        Offset &= ~0x3;
-      Value -= Offset;
-
+    if (Fixup.isPCRel()) {
+      Value -= getFragmentOffset(F) + Fixup.getOffset();
       if (Add && !Sub && !Add->isUndefined() && !Add->isAbsolute()) {
         IsResolved = getWriter().isSymbolRefDifferenceFullyResolvedImpl(
-            *Add, *DF, false, true);
+            *Add, F, false, true);
       }
     } else {
       IsResolved = Target.isAbsolute();
-      assert(!ShouldAlignPC && "FKF_IsAlignedDownTo32Bits must be PC-relative");
     }
   }
 
@@ -203,8 +188,7 @@ bool MCAssembler::evaluateFixup(const MCFragment *DF, const MCFixup &Fixup,
 
   if (IsResolved && mc::isRelocRelocation(Fixup.getKind()))
     IsResolved = false;
-  IsResolved = getBackend().addReloc(*DF, Fixup, Target, Value, IsResolved);
-  getBackend().applyFixup(*DF, Fixup, Target, Contents, Value, IsResolved);
+  getBackend().applyFixup(F, Fixup, Target, Contents, Value, IsResolved);
   return true;
 }
 
@@ -212,9 +196,14 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
   assert(getBackendPtr() && "Requires assembler backend");
   switch (F.getKind()) {
   case MCFragment::FT_Data:
-    return cast<MCDataFragment>(F).getContents().size();
   case MCFragment::FT_Relaxable:
-    return cast<MCRelaxableFragment>(F).getContents().size();
+  case MCFragment::FT_LEB:
+  case MCFragment::FT_Dwarf:
+  case MCFragment::FT_DwarfFrame:
+  case MCFragment::FT_CVInlineLines:
+  case MCFragment::FT_CVDefRange:
+  case MCFragment::FT_PseudoProbe:
+    return cast<MCEncodedFragment>(F).getContents().size();
   case MCFragment::FT_Fill: {
     auto &FF = cast<MCFillFragment>(F);
     int64_t NumValues = 0;
@@ -233,9 +222,6 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
   case MCFragment::FT_Nops:
     return cast<MCNopsFragment>(F).getNumBytes();
 
-  case MCFragment::FT_LEB:
-    return cast<MCLEBFragment>(F).getContents().size();
-
   case MCFragment::FT_BoundaryAlign:
     return cast<MCBoundaryAlignFragment>(F).getSize();
 
@@ -291,17 +277,6 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
     }
     return Size;
   }
-
-  case MCFragment::FT_Dwarf:
-    return cast<MCDwarfLineAddrFragment>(F).getContents().size();
-  case MCFragment::FT_DwarfFrame:
-    return cast<MCDwarfCallFrameFragment>(F).getContents().size();
-  case MCFragment::FT_CVInlineLines:
-    return cast<MCCVInlineLineTableFragment>(F).getContents().size();
-  case MCFragment::FT_CVDefRange:
-    return cast<MCCVDefRangeFragment>(F).getContents().size();
-  case MCFragment::FT_PseudoProbe:
-    return cast<MCPseudoProbeAddrFragment>(F).getContents().size();
   }
 
   llvm_unreachable("invalid fragment kind");
@@ -559,6 +534,22 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
   ++stats::EmittedFragments;
 
   switch (F.getKind()) {
+  case MCFragment::FT_Data:
+  case MCFragment::FT_Relaxable:
+  case MCFragment::FT_LEB:
+  case MCFragment::FT_Dwarf:
+  case MCFragment::FT_DwarfFrame:
+  case MCFragment::FT_CVInlineLines:
+  case MCFragment::FT_CVDefRange:
+  case MCFragment::FT_PseudoProbe: {
+    if (F.getKind() == MCFragment::FT_Data)
+      ++stats::EmittedDataFragments;
+    else if (F.getKind() == MCFragment::FT_Relaxable)
+      ++stats::EmittedRelaxableFragments;
+    const auto &EF = cast<MCEncodedFragment>(F);
+    OS << StringRef(EF.getContents().data(), EF.getContents().size());
+    break;
+  }
   case MCFragment::FT_Align: {
     ++stats::EmittedAlignFragments;
     const MCAlignFragment &AF = cast<MCAlignFragment>(F);
@@ -605,16 +596,6 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
-  case MCFragment::FT_Data:
-    ++stats::EmittedDataFragments;
-    OS << cast<MCDataFragment>(F).getContents();
-    break;
-
-  case MCFragment::FT_Relaxable:
-    ++stats::EmittedRelaxableFragments;
-    OS << cast<MCRelaxableFragment>(F).getContents();
-    break;
-
   case MCFragment::FT_Fill: {
     ++stats::EmittedFillFragments;
     const MCFillFragment &FF = cast<MCFillFragment>(F);
@@ -690,12 +671,6 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
-  case MCFragment::FT_LEB: {
-    const MCLEBFragment &LF = cast<MCLEBFragment>(F);
-    OS << LF.getContents();
-    break;
-  }
-
   case MCFragment::FT_BoundaryAlign: {
     const MCBoundaryAlignFragment &BF = cast<MCBoundaryAlignFragment>(F);
     if (!Asm.getBackend().writeNopData(OS, FragmentSize, BF.getSubtargetInfo()))
@@ -720,31 +695,6 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
-  case MCFragment::FT_Dwarf: {
-    const MCDwarfLineAddrFragment &OF = cast<MCDwarfLineAddrFragment>(F);
-    OS << OF.getContents();
-    break;
-  }
-  case MCFragment::FT_DwarfFrame: {
-    const MCDwarfCallFrameFragment &CF = cast<MCDwarfCallFrameFragment>(F);
-    OS << CF.getContents();
-    break;
-  }
-  case MCFragment::FT_CVInlineLines: {
-    const auto &OF = cast<MCCVInlineLineTableFragment>(F);
-    OS << OF.getContents();
-    break;
-  }
-  case MCFragment::FT_CVDefRange: {
-    const auto &DRF = cast<MCCVDefRangeFragment>(F);
-    OS << DRF.getContents();
-    break;
-  }
-  case MCFragment::FT_PseudoProbe: {
-    const MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(F);
-    OS << PF.getContents();
-    break;
-  }
   }
 
   assert(OS.tell() - Start == FragmentSize &&
@@ -812,9 +762,10 @@ void MCAssembler::writeSectionData(raw_ostream &OS,
 
 void MCAssembler::layout() {
   assert(getBackendPtr() && "Expected assembler backend");
-  DEBUG_WITH_TYPE("mc-dump", {
-      errs() << "assembler backend - pre-layout\n--\n";
-      dump(); });
+  DEBUG_WITH_TYPE("mc-dump-pre", {
+    errs() << "assembler backend - pre-layout\n--\n";
+    dump();
+  });
 
   // Assign section ordinals.
   unsigned SectionIndex = 0;
@@ -844,14 +795,11 @@ void MCAssembler::layout() {
   this->HasLayout = true;
   for (MCSection &Sec : *this)
     layoutSection(Sec);
-  while (relaxOnce())
+  unsigned FirstStable = Sections.size();
+  while ((FirstStable = relaxOnce(FirstStable)) > 0)
     if (getContext().hadError())
       return;
 
-  DEBUG_WITH_TYPE("mc-dump", {
-      errs() << "assembler backend - post-relaxation\n--\n";
-      dump(); });
-
   // Some targets might want to adjust fragment offsets. If so, perform another
   // layout iteration.
   if (getBackend().finishLayout(*this))
@@ -875,69 +823,20 @@ void MCAssembler::layout() {
   // Evaluate and apply the fixups, generating relocation entries as necessary.
   for (MCSection &Sec : *this) {
     for (MCFragment &Frag : Sec) {
-      MutableArrayRef<MCFixup> Fixups;
-      MutableArrayRef<char> Contents;
-
-      // Process MCAlignFragment and MCEncodedFragmentWithFixups here.
-      switch (Frag.getKind()) {
-      default:
-        continue;
-      case MCFragment::FT_Align: {
-        MCAlignFragment &AF = cast<MCAlignFragment>(Frag);
-        // Insert fixup type for code alignment if the target define
-        // shouldInsertFixupForCodeAlign target hook.
-        if (Sec.useCodeAlign() && AF.hasEmitNops())
-          getBackend().shouldInsertFixupForCodeAlign(*this, AF);
-        continue;
-      }
-      case MCFragment::FT_Data: {
-        MCDataFragment &DF = cast<MCDataFragment>(Frag);
-        Fixups = DF.getFixups();
-        Contents = DF.getContents();
-        break;
-      }
-      case MCFragment::FT_Relaxable: {
-        MCRelaxableFragment &RF = cast<MCRelaxableFragment>(Frag);
-        Fixups = RF.getFixups();
-        Contents = RF.getContents();
-        break;
-      }
-      case MCFragment::FT_CVDefRange: {
-        MCCVDefRangeFragment &CF = cast<MCCVDefRangeFragment>(Frag);
-        Fixups = CF.getFixups();
-        Contents = CF.getContents();
-        break;
-      }
-      case MCFragment::FT_Dwarf: {
-        MCDwarfLineAddrFragment &DF = cast<MCDwarfLineAddrFragment>(Frag);
-        Fixups = DF.getFixups();
-        Contents = DF.getContents();
-        break;
-      }
-      case MCFragment::FT_DwarfFrame: {
-        MCDwarfCallFrameFragment &DF = cast<MCDwarfCallFrameFragment>(Frag);
-        Fixups = DF.getFixups();
-        Contents = DF.getContents();
-        break;
-      }
-      case MCFragment::FT_LEB: {
-        auto &LF = cast<MCLEBFragment>(Frag);
-        Fixups = LF.getFixups();
-        Contents = LF.getContents();
-        break;
-      }
-      case MCFragment::FT_PseudoProbe: {
-        MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(Frag);
-        Fixups = PF.getFixups();
-        Contents = PF.getContents();
-        break;
-      }
-      }
-      for (const MCFixup &Fixup : Fixups) {
-        uint64_t FixedValue;
-        MCValue Target;
-        evaluateFixup(&Frag, Fixup, Target, FixedValue,
-                      /*RecordReloc=*/true, Contents);
+      // Process fragments with fixups here.
+      if (auto *F = dyn_cast<MCEncodedFragment>(&Frag)) {
+        auto Contents = F->getContents();
+        for (MCFixup &Fixup : F->getFixups()) {
+          uint64_t FixedValue;
+          MCValue Target;
+          evaluateFixup(Frag, Fixup, Target, FixedValue,
+                        /*RecordReloc=*/true, Contents);
+        }
+      } else if (auto *AF = dyn_cast<MCAlignFragment>(&Frag)) {
+        // For RISC-V linker relaxation, an alignment relocation might be
+        // needed.
+        if (AF->hasEmitNops())
+          getBackend().shouldInsertFixupForCodeAlign(*this, *AF);
       }
     }
   }
@@ -953,54 +852,48 @@ void MCAssembler::Finish() {
   assert(PendingErrors.empty());
 }
 
-bool MCAssembler::fixupNeedsRelaxation(const MCFixup &Fixup,
-                                       const MCRelaxableFragment *DF) const {
+bool MCAssembler::fixupNeedsRelaxation(const MCRelaxableFragment &F,
+                                       const MCFixup &Fixup) const {
   assert(getBackendPtr() && "Expected assembler backend");
   MCValue Target;
   uint64_t Value;
-  bool Resolved = evaluateFixup(DF, const_cast<MCFixup &>(Fixup), Target, Value,
+  bool Resolved = evaluateFixup(F, const_cast<MCFixup &>(Fixup), Target, Value,
                                 /*RecordReloc=*/false, {});
   return getBackend().fixupNeedsRelaxationAdvanced(Fixup, Target, Value,
                                                    Resolved);
 }
 
-bool MCAssembler::fragmentNeedsRelaxation(const MCRelaxableFragment *F) const {
-  assert(getBackendPtr() && "Expected assembler backend");
+bool MCAssembler::relaxInstruction(MCRelaxableFragment &F) {
+  assert(getEmitterPtr() &&
+         "Expected CodeEmitter defined for relaxInstruction");
   // If this inst doesn't ever need relaxation, ignore it. This occurs when we
   // are intentionally pushing out inst fragments, or because we relaxed a
   // previous instruction to one that doesn't need relaxation.
-  if (!getBackend().mayNeedRelaxation(F->getInst(), *F->getSubtargetInfo()))
+  if (!getBackend().mayNeedRelaxation(F.getOpcode(), F.getOperands(),
+                                      *F.getSubtargetInfo()))
     return false;
 
-  for (const MCFixup &Fixup : F->getFixups())
-    if (fixupNeedsRelaxation(Fixup, F))
-      return true;
-
-  return false;
-}
-
-bool MCAssembler::relaxInstruction(MCRelaxableFragment &F) {
-  assert(getEmitterPtr() &&
-         "Expected CodeEmitter defined for relaxInstruction");
-  if (!fragmentNeedsRelaxation(&F))
+  bool DoRelax = false;
+  for (const MCFixup &Fixup : F.getFixups())
+    if ((DoRelax = fixupNeedsRelaxation(F, Fixup)))
+      break;
+  if (!DoRelax)
     return false;
 
   ++stats::RelaxedInstructions;
 
-  // FIXME-PERF: We could immediately lower out instructions if we can tell
-  // they are fully resolved, to avoid retesting on later passes.
-
-  // Relax the fragment.
-
+  // TODO Refactor relaxInstruction to accept MCRelaxableFragment and remove
+  // `setInst`.
   MCInst Relaxed = F.getInst();
   getBackend().relaxInstruction(Relaxed, *F.getSubtargetInfo());
 
   // Encode the new instruction.
   F.setInst(Relaxed);
-  F.getFixups().clear();
-  F.getContents().clear();
-  getEmitter().encodeInstruction(Relaxed, F.getContents(), F.getFixups(),
-                                 *F.getSubtargetInfo());
+  SmallVector<char, 16> Data;
+  SmallVector<MCFixup, 1> Fixups;
+  getEmitter().encodeInstruction(Relaxed, Data, Fixups, *F.getSubtargetInfo());
+  F.setContents(Data);
+  F.setFixups(Fixups);
   return true;
 }
 
@@ -1008,8 +901,7 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) {
   const unsigned OldSize = static_cast<unsigned>(LF.getContents().size());
   unsigned PadTo = OldSize;
   int64_t Value;
-  SmallVectorImpl<char> &Data = LF.getContents();
-  LF.getFixups().clear();
+  LF.clearFixups();
   // Use evaluateKnownAbsolute for Mach-O as a hack: .subsections_via_symbols
   // requires that .uleb128 A-B is foldable where A and B reside in different
   // fragments. This is used by __gcc_except_table.
@@ -1030,17 +922,18 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) {
     if (UseZeroPad)
       Value = 0;
   }
-  Data.clear();
-  raw_svector_ostream OSE(Data);
+  uint8_t Data[16];
+  size_t Size = 0;
   // The compiler can generate EH table assembly that is impossible to assemble
   // without either adding padding to an LEB fragment or adding extra padding
   // to a later alignment fragment. To accommodate such tables, relaxation can
   // only increase an LEB fragment size here, not decrease it. See PR35809.
   if (LF.isSigned())
-    encodeSLEB128(Value, OSE, PadTo);
+    Size = encodeSLEB128(Value, Data, PadTo);
   else
-    encodeULEB128(Value, OSE, PadTo);
-  return OldSize != LF.getContents().size();
+    Size = encodeULEB128(Value, Data, PadTo);
+  LF.setContents({reinterpret_cast<char *>(Data), Size});
+  return OldSize != Size;
 }
 
 /// Check if the branch crosses the boundary.
@@ -1110,19 +1003,19 @@ bool MCAssembler::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF) {
     return WasRelaxed;
 
   MCContext &Context = getContext();
-  uint64_t OldSize = DF.getContents().size();
+  auto OldSize = DF.getContents().size();
   int64_t AddrDelta;
   bool Abs = DF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
   assert(Abs && "We created a line delta with an invalid expression");
   (void)Abs;
   int64_t LineDelta;
   LineDelta = DF.getLineDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  Data.clear();
-  DF.getFixups().clear();
+  SmallVector<char, 8> Data;
 
   MCDwarfLineAddr::encode(Context, getDWARFLinetableParams(), LineDelta,
                           AddrDelta, Data);
+  DF.setContents(Data);
+  DF.clearFixups();
   return OldSize != Data.size();
 }
 
@@ -1141,12 +1034,11 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCDwarfCallFrameFragment &DF) {
     return false;
   }
 
-  SmallVectorImpl<char> &Data = DF.getContents();
-  uint64_t OldSize = Data.size();
-  Data.clear();
-  DF.getFixups().clear();
-
+  auto OldSize = DF.getContents().size();
+  SmallVector<char, 8> Data;
   MCDwarfFrameEmitter::encodeAdvanceLoc(Context, Value, Data);
+  DF.setContents(Data);
+  DF.clearFixups();
   return OldSize != Data.size();
 }
 
@@ -1176,13 +1068,13 @@ bool MCAssembler::relaxPseudoProbeAddr(MCPseudoProbeAddrFragment &PF) {
   bool Abs = PF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
   assert(Abs && "We created a pseudo probe with an invalid expression");
   (void)Abs;
-  SmallVectorImpl<char> &Data = PF.getContents();
-  Data.clear();
+  SmallVector<char, 8> Data;
   raw_svector_ostream OSE(Data);
-  PF.getFixups().clear();
 
   // AddrDelta is a signed integer
   encodeSLEB128(AddrDelta, OSE, OldSize);
+  PF.setContents(Data);
+  PF.clearFixups();
   return OldSize != Data.size();
 }
 
@@ -1229,17 +1121,15 @@ void MCAssembler::layoutSection(MCSection &Sec) {
   }
 }
 
-bool MCAssembler::relaxOnce() {
+unsigned MCAssembler::relaxOnce(unsigned FirstStable) {
   ++stats::RelaxationSteps;
   PendingErrors.clear();
 
-  // Size of fragments in one section can depend on the size of fragments in
-  // another. If any fragment has changed size, we have to re-layout (and
-  // as a result possibly further relax) all sections.
-  bool ChangedAny = false;
-  for (MCSection &Sec : *this) {
+  unsigned Res = 0;
+  for (unsigned I = 0; I != FirstStable; ++I) {
     // Assume each iteration finalizes at least one extra fragment. If the
     // layout does not converge after N+1 iterations, bail out.
+    auto &Sec = *Sections[I];
     auto MaxIter = Sec.curFragList()->Tail->getLayoutOrder() + 1;
     for (;;) {
       bool Changed = false;
@@ -1247,13 +1137,20 @@ bool MCAssembler::relaxOnce() {
         if (relaxFragment(F))
           Changed = true;
 
-      ChangedAny |= Changed;
-      if (!Changed || --MaxIter == 0)
+      if (!Changed)
+        break;
+      // If any fragment changed size, it might impact the layout of subsequent
+      // sections. Therefore, we must re-evaluate all sections.
+      FirstStable = Sections.size();
+      Res = I;
+      if (--MaxIter == 0)
         break;
       layoutSection(Sec);
     }
   }
-  return ChangedAny;
+  // The subsequent relaxOnce call only needs to visit Sections [0,Res) if no
+  // change occurred.
+  return Res;
 }
 
 void MCAssembler::reportError(SMLoc L, const Twine &Msg) const {
@@ -1273,31 +1170,26 @@ void MCAssembler::flushPendingErrors() const {
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void MCAssembler::dump() const{
   raw_ostream &OS = errs();
-
-  OS << "<MCAssembler\n";
-  OS << "  Sections:[\n    ";
-  bool First = true;
+  DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>> FragToSyms;
+  // Scan symbols and build a map of fragments to their corresponding symbols.
+  // For variable symbols, we don't want to call their getFragment, which might
+  // modify `Fragment`.
+  for (const MCSymbol &Sym : symbols())
+    if (!Sym.isVariable())
+      if (auto *F = Sym.getFragment())
+        FragToSyms.try_emplace(F).first->second.push_back(&Sym);
+
+  OS << "Sections:[";
   for (const MCSection &Sec : *this) {
-    if (First)
-      First = false;
-    else
-      OS << ",\n    ";
-    Sec.dump();
-  }
-  OS << "],\n";
-  OS << "  Symbols:[";
-
-  First = true;
-  for (const MCSymbol &Sym : symbols()) {
-    if (First)
-      First = false;
-    else
-      OS << ",\n           ";
-    OS << "(";
-    Sym.dump();
-    OS << ", Index:" << Sym.getIndex() << ", ";
-    OS << ")";
+    OS << '\n';
+    Sec.dump(&FragToSyms);
   }
-  OS << "]>\n";
+  OS << "\n]\n";
 }
 #endif
+
+SMLoc MCFixup::getLoc() const {
+  if (auto *E = getValue())
+    return E->getLoc();
+  return {};
+}
diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp
index 1407bc52e1c46..5d7914396e09f 100644
--- a/llvm/lib/MC/MCCodeView.cpp
+++ b/llvm/lib/MC/MCCodeView.cpp
@@ -442,10 +442,13 @@ MCFragment *CodeViewContext::emitDefRange(
     MCObjectStreamer &OS,
     ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
     StringRef FixedSizePortion) {
+  // Store `Ranges` and `FixedSizePortion` in the context, returning references,
+  // as MCCVDefRangeFragment does not own these objects.
+  FixedSizePortion = MCCtx->allocateString(FixedSizePortion);
+  auto &Saved = DefRangeStorage.emplace_back(Ranges.begin(), Ranges.end());
   // Create and insert a fragment into the current section that will be encoded
   // later.
-  auto *F =
-      MCCtx->allocFragment<MCCVDefRangeFragment>(Ranges, FixedSizePortion);
+  auto *F = MCCtx->allocFragment<MCCVDefRangeFragment>(Saved, FixedSizePortion);
   OS.insert(F);
   return F;
 }
@@ -507,8 +510,7 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm,
 
   MCCVFunctionInfo *SiteInfo = getCVFunctionInfo(Frag.SiteFuncId);
 
-  SmallVectorImpl<char> &Buffer = Frag.getContents();
-  Buffer.clear(); // Clear old contents if we went through relaxation.
+  SmallVector<char, 0> Buffer;
   for (const MCCVLoc &Loc : Locs) {
     // Exit early if our line table would produce an oversized InlineSiteSym
     // record. Account for the ChangeCodeLength annotation emitted after the
@@ -601,15 +603,14 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm,
 
   compressAnnotation(BinaryAnnotationsOpCode::ChangeCodeLength, Buffer);
   compressAnnotation(std::min(EndSymLength, LocAfterLength), Buffer);
+  Frag.setContents(Buffer);
 }
 
 void CodeViewContext::encodeDefRange(const MCAssembler &Asm,
                                      MCCVDefRangeFragment &Frag) {
   MCContext &Ctx = Asm.getContext();
-  SmallVectorImpl<char> &Contents = Frag.getContents();
-  Contents.clear();
-  SmallVectorImpl<MCFixup> &Fixups = Frag.getFixups();
-  Fixups.clear();
+  SmallVector<char, 0> Contents;
+  SmallVector<MCFixup, 0> Fixups;
   raw_svector_ostream OS(Contents);
 
   // Compute all the sizes up front.
@@ -689,4 +690,7 @@ void CodeViewContext::encodeDefRange(const MCAssembler &Asm,
       GapStartOffset += GapSize + RangeSize;
     }
   }
+
+  Frag.setContents(Contents);
+  Frag.setFixups(Fixups);
 }
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 3f2f9a756d33b..b0d1cb41fac1c 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -21,7 +21,6 @@
 #include "llvm/MC/MCCodeView.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCLabel.h"
 #include "llvm/MC/MCSectionCOFF.h"
@@ -75,6 +74,8 @@ MCContext::MCContext(const Triple &TheTriple, const MCAsmInfo *mai,
       CurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_IS_STMT, 0, 0),
       AutoReset(DoAutoReset), TargetOptions(TargetOpts) {
   SaveTempLabels = TargetOptions && TargetOptions->MCSaveTempLabels;
+  if (SaveTempLabels)
+    setUseNamesOnTempLabels(true);
   SecureLogFile = TargetOptions ? TargetOptions->AsSecureLogFile : "";
 
   if (SrcMgr && SrcMgr->getNumBuffers())
diff --git a/llvm/lib/MC/MCDXContainerStreamer.cpp b/llvm/lib/MC/MCDXContainerStreamer.cpp
index a596c9a16d498..88c6c8b7e9784 100644
--- a/llvm/lib/MC/MCDXContainerStreamer.cpp
+++ b/llvm/lib/MC/MCDXContainerStreamer.cpp
@@ -16,9 +16,6 @@
 
 using namespace llvm;
 
-void MCDXContainerStreamer::emitInstToData(const MCInst &,
-                                           const MCSubtargetInfo &) {}
-
 MCStreamer *llvm::createDXContainerStreamer(
     MCContext &Context, std::unique_ptr<MCAsmBackend> &&MAB,
     std::unique_ptr<MCObjectWriter> &&OW, std::unique_ptr<MCCodeEmitter> &&CE) {
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 83c156648fa4b..b1dced7e2cda3 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -1879,7 +1879,7 @@ struct CIEKey {
   unsigned LsdaEncoding = -1;
   bool IsSignalFrame = false;
   bool IsSimple = false;
-  unsigned RAReg = static_cast<unsigned>(UINT_MAX);
+  unsigned RAReg = UINT_MAX;
   bool IsBKeyFrame = false;
   bool IsMTETaggedFrame = false;
 };
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 5a514f12ce549..e84b91c5c41b0 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -21,7 +21,6 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -449,11 +448,14 @@ void MCELFStreamer::emitInstToData(const MCInst &Inst,
   // Emit instruction directly into data fragment.
   size_t FixupStartIndex = DF->getFixups().size();
   size_t CodeOffset = DF->getContents().size();
-  Assembler.getEmitter().encodeInstruction(Inst, DF->getContents(),
-                                           DF->getFixups(), STI);
-
-  auto Fixups = MutableArrayRef(DF->getFixups()).slice(FixupStartIndex);
-  for (auto &Fixup : Fixups) {
+  SmallVector<MCFixup, 1> Fixups;
+  Assembler.getEmitter().encodeInstruction(Inst, DF->getContentsForAppending(),
+                                           Fixups, STI);
+  DF->doneAppending();
+  if (!Fixups.empty())
+    DF->appendFixups(Fixups);
+
+  for (auto &Fixup : MutableArrayRef(DF->getFixups()).slice(FixupStartIndex)) {
     Fixup.setOffset(Fixup.getOffset() + CodeOffset);
     if (Fixup.isLinkerRelaxable()) {
       DF->setLinkerRelaxable();
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index 8919a2627cf6a..79c3ec625cb7a 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -91,7 +91,7 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
     Sym.print(OS, MAI);
 
     const MCSymbolRefExpr::VariantKind Kind = SRE.getKind();
-    if (Kind != MCSymbolRefExpr::VK_None) {
+    if (Kind) {
       if (!MAI) // should only be used by dump()
         OS << "@<variant " << Kind << '>';
       else if (MAI->useParensForSpecifier()) // ARM
@@ -179,7 +179,9 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
       return MAI->printSpecifierExpr(OS, SE);
     // Used by dump features like -show-inst. Regular MCAsmStreamer output must
     // set MAI.
-    OS << "specifier(" << SE.getSpecifier() << ',' << *SE.getSubExpr() << ')';
+    OS << "specifier(" << SE.getSpecifier() << ',';
+    SE.getSubExpr()->print(OS, nullptr);
+    OS << ')';
     return;
   }
   }
@@ -189,7 +191,7 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void MCExpr::dump() const {
-  dbgs() << *this;
+  print(dbgs(), nullptr);
   dbgs() << '\n';
 }
 #endif
@@ -215,16 +217,16 @@ const MCConstantExpr *MCConstantExpr::create(int64_t Value, MCContext &Ctx,
 
 /* *** */
 
-MCSymbolRefExpr::MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind,
+MCSymbolRefExpr::MCSymbolRefExpr(const MCSymbol *Symbol, Spec specifier,
                                  const MCAsmInfo *MAI, SMLoc Loc)
-    : MCExpr(MCExpr::SymbolRef, Loc, Kind), Symbol(Symbol) {
+    : MCExpr(MCExpr::SymbolRef, Loc, specifier), Symbol(Symbol) {
   assert(Symbol);
 }
 
 const MCSymbolRefExpr *MCSymbolRefExpr::create(const MCSymbol *Sym,
-                                               VariantKind Kind,
+                                               uint16_t specifier,
                                                MCContext &Ctx, SMLoc Loc) {
-  return new (Ctx) MCSymbolRefExpr(Sym, Kind, Ctx.getAsmInfo(), Loc);
+  return new (Ctx) MCSymbolRefExpr(Sym, specifier, Ctx.getAsmInfo(), Loc);
 }
 
 /* *** */
@@ -487,8 +489,7 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
       }
       return false;
     }
-    if (Sym.isVariable() && (Kind == MCSymbolRefExpr::VK_None || Layout) &&
-        !Sym.isWeakExternal()) {
+    if (Sym.isVariable() && (Kind == 0 || Layout) && !Sym.isWeakExternal()) {
       Sym.setIsResolving(true);
       auto _ = make_scope_exit([&] { Sym.setIsResolving(false); });
       bool IsMachO =
@@ -502,7 +503,7 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
       auto *A = Res.getAddSym();
       auto *B = Res.getSubSym();
       if (InSet || !(A && !B && A->isInSection())) {
-        if (Kind != MCSymbolRefExpr::VK_None) {
+        if (Kind) {
           if (Res.isAbsolute()) {
             Res = MCValue::get(&Sym, nullptr, 0, Kind);
             return true;
@@ -510,8 +511,8 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
           // If the reference has a variant kind, we can only handle expressions
           // which evaluate exactly to a single unadorned symbol. Attach the
           // original VariantKind to SymA of the result.
-          if (Res.getSpecifier() != MCSymbolRefExpr::VK_None ||
-              !Res.getAddSym() || Res.getSubSym() || Res.getConstant())
+          if (Res.getSpecifier() || !Res.getAddSym() || Res.getSubSym() ||
+              Res.getConstant())
             return false;
           Res.Specifier = Kind;
         }
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index aa4dec0a8e9d9..5cf47e3325ff2 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/MC/MCFragment.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
@@ -20,145 +19,90 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
+#include <type_traits>
 #include <utility>
 
 using namespace llvm;
 
+static_assert(std::is_trivially_destructible_v<MCDataFragment>,
+              "fragment classes must be trivially destructible");
+
 MCFragment::MCFragment(FragmentType Kind, bool HasInstructions)
     : Kind(Kind), HasInstructions(HasInstructions), AlignToBundleEnd(false),
       LinkerRelaxable(false), AllowAutoPadding(false) {}
 
-void MCFragment::destroy() {
-  switch (Kind) {
-    case FT_Align:
-      cast<MCAlignFragment>(this)->~MCAlignFragment();
-      return;
-    case FT_Data:
-      cast<MCDataFragment>(this)->~MCDataFragment();
-      return;
-    case FT_Fill:
-      cast<MCFillFragment>(this)->~MCFillFragment();
-      return;
-    case FT_Nops:
-      cast<MCNopsFragment>(this)->~MCNopsFragment();
-      return;
-    case FT_Relaxable:
-      cast<MCRelaxableFragment>(this)->~MCRelaxableFragment();
-      return;
-    case FT_Org:
-      cast<MCOrgFragment>(this)->~MCOrgFragment();
-      return;
-    case FT_Dwarf:
-      cast<MCDwarfLineAddrFragment>(this)->~MCDwarfLineAddrFragment();
-      return;
-    case FT_DwarfFrame:
-      cast<MCDwarfCallFrameFragment>(this)->~MCDwarfCallFrameFragment();
-      return;
-    case FT_LEB:
-      cast<MCLEBFragment>(this)->~MCLEBFragment();
-      return;
-    case FT_BoundaryAlign:
-      cast<MCBoundaryAlignFragment>(this)->~MCBoundaryAlignFragment();
-      return;
-    case FT_SymbolId:
-      cast<MCSymbolIdFragment>(this)->~MCSymbolIdFragment();
-      return;
-    case FT_CVInlineLines:
-      cast<MCCVInlineLineTableFragment>(this)->~MCCVInlineLineTableFragment();
-      return;
-    case FT_CVDefRange:
-      cast<MCCVDefRangeFragment>(this)->~MCCVDefRangeFragment();
-      return;
-    case FT_PseudoProbe:
-      cast<MCPseudoProbeAddrFragment>(this)->~MCPseudoProbeAddrFragment();
-      return;
-  }
-}
-
 const MCSymbol *MCFragment::getAtom() const {
   return cast<MCSectionMachO>(Parent)->getAtom(LayoutOrder);
 }
 
-// Debugging methods
-
-namespace llvm {
-
-raw_ostream &operator<<(raw_ostream &OS, const MCFixup &AF) {
-  OS << "<MCFixup" << " Offset:" << AF.getOffset()
-     << " Value:" << *AF.getValue()
-     << " Kind:" << AF.getKind() << ">";
-  return OS;
-}
-
-} // end namespace llvm
-
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void MCFragment::dump() const {
   raw_ostream &OS = errs();
 
-  OS << "<";
+  OS << Offset << ' ';
   switch (getKind()) {
-  case MCFragment::FT_Align: OS << "MCAlignFragment"; break;
-  case MCFragment::FT_Data:  OS << "MCDataFragment"; break;
-  case MCFragment::FT_Fill:  OS << "MCFillFragment"; break;
-  case MCFragment::FT_Nops:
-    OS << "MCFNopsFragment";
-    break;
-  case MCFragment::FT_Relaxable:  OS << "MCRelaxableFragment"; break;
-  case MCFragment::FT_Org:   OS << "MCOrgFragment"; break;
-  case MCFragment::FT_Dwarf: OS << "MCDwarfFragment"; break;
-  case MCFragment::FT_DwarfFrame: OS << "MCDwarfCallFrameFragment"; break;
-  case MCFragment::FT_LEB:   OS << "MCLEBFragment"; break;
-  case MCFragment::FT_BoundaryAlign: OS<<"MCBoundaryAlignFragment"; break;
-  case MCFragment::FT_SymbolId:    OS << "MCSymbolIdFragment"; break;
-  case MCFragment::FT_CVInlineLines: OS << "MCCVInlineLineTableFragment"; break;
-  case MCFragment::FT_CVDefRange: OS << "MCCVDefRangeTableFragment"; break;
-  case MCFragment::FT_PseudoProbe:
-    OS << "MCPseudoProbe";
-    break;
+    // clang-format off
+  case MCFragment::FT_Align:         OS << "Align"; break;
+  case MCFragment::FT_Data:          OS << "Data"; break;
+  case MCFragment::FT_Fill:          OS << "Fill"; break;
+  case MCFragment::FT_Nops:          OS << "Nops"; break;
+  case MCFragment::FT_Relaxable:     OS << "Relaxable"; break;
+  case MCFragment::FT_Org:           OS << "Org"; break;
+  case MCFragment::FT_Dwarf:         OS << "Dwarf"; break;
+  case MCFragment::FT_DwarfFrame:    OS << "DwarfCallFrame"; break;
+  case MCFragment::FT_LEB:           OS << "LEB"; break;
+  case MCFragment::FT_BoundaryAlign: OS<<"BoundaryAlign"; break;
+  case MCFragment::FT_SymbolId:      OS << "SymbolId"; break;
+  case MCFragment::FT_CVInlineLines: OS << "CVInlineLineTable"; break;
+  case MCFragment::FT_CVDefRange:    OS << "CVDefRangeTable"; break;
+  case MCFragment::FT_PseudoProbe:   OS << "PseudoProbe"; break;
+    // clang-format on
   }
 
-  OS << "<MCFragment " << (const void *)this << " LayoutOrder:" << LayoutOrder
-     << " Offset:" << Offset << " HasInstructions:" << hasInstructions();
   if (const auto *EF = dyn_cast<MCEncodedFragment>(this))
-    OS << " BundlePadding:" << static_cast<unsigned>(EF->getBundlePadding());
-  OS << ">";
+    if (auto Pad = static_cast<unsigned>(EF->getBundlePadding()))
+      OS << " BundlePadding:" << Pad;
+
+  auto printFixups = [&](llvm::ArrayRef<MCFixup> Fixups) {
+    if (Fixups.empty())
+      return;
+    for (auto [I, F] : llvm::enumerate(Fixups)) {
+      OS << "\n  Fixup @" << F.getOffset() << " Value:";
+      F.getValue()->print(OS, nullptr);
+      OS << " Kind:" << F.getKind();
+    }
+  };
 
   switch (getKind()) {
   case MCFragment::FT_Align: {
     const auto *AF = cast<MCAlignFragment>(this);
+    OS << " Align:" << AF->getAlignment().value() << " Value:" << AF->getValue()
+       << " ValueSize:" << AF->getValueSize()
+       << " MaxBytesToEmit:" << AF->getMaxBytesToEmit();
     if (AF->hasEmitNops())
-      OS << " (emit nops)";
-    OS << "\n       ";
-    OS << " Alignment:" << AF->getAlignment().value()
-       << " Value:" << AF->getValue() << " ValueSize:" << AF->getValueSize()
-       << " MaxBytesToEmit:" << AF->getMaxBytesToEmit() << ">";
+      OS << " Nops";
     break;
   }
   case MCFragment::FT_Data:  {
-    const auto *DF = cast<MCDataFragment>(this);
-    OS << "\n       ";
-    OS << " Contents:[";
-    const SmallVectorImpl<char> &Contents = DF->getContents();
+    const auto *F = cast<MCDataFragment>(this);
+    if (F->isLinkerRelaxable())
+      OS << " LinkerRelaxable";
+    auto Contents = F->getContents();
+    OS << " Size:" << Contents.size() << " [";
     for (unsigned i = 0, e = Contents.size(); i != e; ++i) {
       if (i) OS << ",";
-      OS << hexdigit((Contents[i] >> 4) & 0xF) << hexdigit(Contents[i] & 0xF);
-    }
-    OS << "] (" << Contents.size() << " bytes)";
-
-    if (DF->getFixups().size()) {
-      OS << ",\n       ";
-      OS << " Fixups:[";
-      interleave(DF->getFixups(), OS, ",\n                ");
-      OS << "]";
+      OS << format("%02x", uint8_t(Contents[i]));
     }
+    OS << ']';
+    printFixups(F->getFixups());
     break;
   }
   case MCFragment::FT_Fill:  {
     const auto *FF = cast<MCFillFragment>(this);
     OS << " Value:" << static_cast<unsigned>(FF->getValue())
        << " ValueSize:" << static_cast<unsigned>(FF->getValueSize())
-       << " NumValues:" << FF->getNumValues();
+       << " NumValues:";
+    FF->getNumValues().print(OS, nullptr);
     break;
   }
   case MCFragment::FT_Nops: {
@@ -169,41 +113,40 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   }
   case MCFragment::FT_Relaxable:  {
     const auto *F = cast<MCRelaxableFragment>(this);
-    OS << "\n       ";
-    OS << " Inst:";
+    OS << " Size:" << F->getContents().size() << ' ';
     F->getInst().dump_pretty(OS);
-    OS << " (" << F->getContents().size() << " bytes)";
+    printFixups(F->getFixups());
     break;
   }
   case MCFragment::FT_Org:  {
     const auto *OF = cast<MCOrgFragment>(this);
-    OS << "\n       ";
-    OS << " Offset:" << OF->getOffset()
-       << " Value:" << static_cast<unsigned>(OF->getValue());
+    OS << " Offset:";
+    OF->getOffset().print(OS, nullptr);
+    OS << " Value:" << static_cast<unsigned>(OF->getValue());
     break;
   }
   case MCFragment::FT_Dwarf:  {
     const auto *OF = cast<MCDwarfLineAddrFragment>(this);
-    OS << "\n       ";
-    OS << " AddrDelta:" << OF->getAddrDelta()
-       << " LineDelta:" << OF->getLineDelta();
+    OS << " AddrDelta:";
+    OF->getAddrDelta().print(OS, nullptr);
+    OS << " LineDelta:" << OF->getLineDelta();
     break;
   }
   case MCFragment::FT_DwarfFrame:  {
     const auto *CF = cast<MCDwarfCallFrameFragment>(this);
-    OS << "\n       ";
-    OS << " AddrDelta:" << CF->getAddrDelta();
+    OS << " AddrDelta:";
+    CF->getAddrDelta().print(OS, nullptr);
     break;
   }
   case MCFragment::FT_LEB: {
     const auto *LF = cast<MCLEBFragment>(this);
-    OS << "\n       ";
-    OS << " Value:" << LF->getValue() << " Signed:" << LF->isSigned();
+    OS << " Value:";
+    LF->getValue().print(OS, nullptr);
+    OS << " Signed:" << LF->isSigned();
     break;
   }
   case MCFragment::FT_BoundaryAlign: {
     const auto *BF = cast<MCBoundaryAlignFragment>(this);
-    OS << "\n       ";
     OS << " BoundarySize:" << BF->getAlignment().value()
        << " LastFragment:" << BF->getLastFragment()
        << " Size:" << BF->getSize();
@@ -211,19 +154,17 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   }
   case MCFragment::FT_SymbolId: {
     const auto *F = cast<MCSymbolIdFragment>(this);
-    OS << "\n       ";
     OS << " Sym:" << F->getSymbol();
     break;
   }
   case MCFragment::FT_CVInlineLines: {
     const auto *F = cast<MCCVInlineLineTableFragment>(this);
-    OS << "\n       ";
     OS << " Sym:" << *F->getFnStartSym();
     break;
   }
   case MCFragment::FT_CVDefRange: {
     const auto *F = cast<MCCVDefRangeFragment>(this);
-    OS << "\n       ";
+    OS << "\n   ";
     for (std::pair<const MCSymbol *, const MCSymbol *> RangeStartEnd :
          F->getRanges()) {
       OS << " RangeStart:" << RangeStartEnd.first;
@@ -233,11 +174,10 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   }
   case MCFragment::FT_PseudoProbe: {
     const auto *OF = cast<MCPseudoProbeAddrFragment>(this);
-    OS << "\n       ";
-    OS << " AddrDelta:" << OF->getAddrDelta();
+    OS << " AddrDelta:";
+    OF->getAddrDelta().print(OS, nullptr);
     break;
   }
   }
-  OS << ">";
 }
 #endif
diff --git a/llvm/lib/MC/MCInst.cpp b/llvm/lib/MC/MCInst.cpp
index 832d25060f880..46a6a18e15963 100644
--- a/llvm/lib/MC/MCInst.cpp
+++ b/llvm/lib/MC/MCInst.cpp
@@ -8,6 +8,8 @@
 
 #include "llvm/MC/MCInst.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -18,14 +20,14 @@
 
 using namespace llvm;
 
-void MCOperand::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
+void MCOperand::print(raw_ostream &OS, const MCContext *Ctx) const {
   OS << "<MCOperand ";
   if (!isValid())
     OS << "INVALID";
   else if (isReg()) {
     OS << "Reg:";
-    if (RegInfo)
-      OS << RegInfo->getName(getReg());
+    if (Ctx && Ctx->getRegisterInfo())
+      OS << Ctx->getRegisterInfo()->getName(getReg());
     else
       OS << getReg();
   } else if (isImm())
@@ -35,11 +37,15 @@ void MCOperand::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
   else if (isDFPImm())
     OS << "DFPImm:" << bit_cast<double>(getDFPImm());
   else if (isExpr()) {
-    OS << "Expr:" << *getExpr();
+    OS << "Expr:";
+    if (Ctx)
+      Ctx->getAsmInfo()->printExpr(OS, *getExpr());
+    else
+      getExpr()->print(OS, nullptr);
   } else if (isInst()) {
     OS << "Inst:(";
     if (const auto *Inst = getInst())
-      Inst->print(OS, RegInfo);
+      Inst->print(OS, Ctx);
     else
       OS << "NULL";
     OS << ")";
@@ -62,7 +68,7 @@ bool MCOperand::isBareSymbolRef() const {
   const MCExpr *Expr = getExpr();
   MCExpr::ExprKind Kind = getExpr()->getKind();
   return Kind == MCExpr::SymbolRef &&
-    cast<MCSymbolRefExpr>(Expr)->getKind() == MCSymbolRefExpr::VK_None;
+         cast<MCSymbolRefExpr>(Expr)->getSpecifier() == 0;
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -72,24 +78,23 @@ LLVM_DUMP_METHOD void MCOperand::dump() const {
 }
 #endif
 
-void MCInst::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
+void MCInst::print(raw_ostream &OS, const MCContext *Ctx) const {
   OS << "<MCInst " << getOpcode();
   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
     OS << " ";
-    getOperand(i).print(OS, RegInfo);
+    getOperand(i).print(OS, Ctx);
   }
   OS << ">";
 }
 
 void MCInst::dump_pretty(raw_ostream &OS, const MCInstPrinter *Printer,
-                         StringRef Separator,
-                         const MCRegisterInfo *RegInfo) const {
+                         StringRef Separator, const MCContext *Ctx) const {
   StringRef InstName = Printer ? Printer->getOpcodeName(getOpcode()) : "";
-  dump_pretty(OS, InstName, Separator, RegInfo);
+  dump_pretty(OS, InstName, Separator, Ctx);
 }
 
 void MCInst::dump_pretty(raw_ostream &OS, StringRef Name, StringRef Separator,
-                         const MCRegisterInfo *RegInfo) const {
+                         const MCContext *Ctx) const {
   OS << "<MCInst #" << getOpcode();
 
   // Show the instruction opcode name if we have it.
@@ -98,7 +103,7 @@ void MCInst::dump_pretty(raw_ostream &OS, StringRef Name, StringRef Separator,
 
   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
     OS << Separator;
-    getOperand(i).print(OS, RegInfo);
+    getOperand(i).print(OS, Ctx);
   }
   OS << ">";
 }
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 4cf602b7e4563..a54dfb9cbaea1 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
@@ -59,8 +58,6 @@ class MCMachOStreamer : public MCObjectStreamer {
   /// labels in the middle of the section.
   DenseMap<const MCSection*, bool> HasSectionLabel;
 
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &STI) override;
-
   void emitDataRegion(MachO::DataRegionType Kind);
   void emitDataRegionEnd();
 
@@ -423,23 +420,6 @@ void MCMachOStreamer::emitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
   emitZerofill(Section, Symbol, Size, ByteAlignment);
 }
 
-void MCMachOStreamer::emitInstToData(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI) {
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Add the fixups and data.
-  for (MCFixup &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixup);
-  }
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCMachOStreamer::finishImpl() {
   emitFrames(&getAssembler().getBackend());
 
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 4a179fcdf6d08..5cc9bed2669d4 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -63,28 +63,14 @@ void MCObjectStreamer::resolvePendingFixups() {
     PendingFixup.Fixup.setOffset(PendingFixup.Sym->getOffset() +
                                  PendingFixup.Fixup.getOffset());
 
-    // If the location symbol to relocate is in MCEncodedFragmentWithFixups,
+    // If the location symbol to relocate is in MCEncodedFragment,
     // put the Fixup into location symbol's fragment. Otherwise
     // put into PendingFixup.DF
     MCFragment *SymFragment = PendingFixup.Sym->getFragment();
-    switch (SymFragment->getKind()) {
-    case MCFragment::FT_Relaxable:
-    case MCFragment::FT_Dwarf:
-    case MCFragment::FT_PseudoProbe:
-      cast<MCEncodedFragmentWithFixups<8, 1>>(SymFragment)
-          ->getFixups()
-          .push_back(PendingFixup.Fixup);
-      break;
-    case MCFragment::FT_Data:
-    case MCFragment::FT_CVDefRange:
-      cast<MCEncodedFragmentWithFixups<32, 4>>(SymFragment)
-          ->getFixups()
-          .push_back(PendingFixup.Fixup);
-      break;
-    default:
-      PendingFixup.DF->getFixups().push_back(PendingFixup.Fixup);
-      break;
-    }
+    if (auto *F = dyn_cast<MCEncodedFragment>(SymFragment))
+      F->addFixup(PendingFixup.Fixup);
+    else
+      PendingFixup.DF->addFixup(PendingFixup.Fixup);
   }
   PendingFixups.clear();
 }
@@ -201,8 +187,8 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
     emitIntValue(AbsValue, Size);
     return;
   }
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixup::getDataKindForSize(Size), Loc));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               MCFixup::getDataKindForSize(Size)));
   DF->appendContents(Size, 0);
 }
 
@@ -366,7 +352,7 @@ void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
   // If this instruction doesn't need relaxation, just emit it as data.
   MCAssembler &Assembler = getAssembler();
   MCAsmBackend &Backend = Assembler.getBackend();
-  if (!(Backend.mayNeedRelaxation(Inst, STI) ||
+  if (!(Backend.mayNeedRelaxation(Inst.getOpcode(), Inst.getOperands(), STI) ||
         Backend.allowEnhancedRelaxation())) {
     emitInstToData(Inst, STI);
     return;
@@ -380,7 +366,8 @@ void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
   if (Assembler.getRelaxAll() ||
       (Assembler.isBundlingEnabled() && Sec->isBundleLocked())) {
     MCInst Relaxed = Inst;
-    while (Backend.mayNeedRelaxation(Relaxed, STI))
+    while (Backend.mayNeedRelaxation(Relaxed.getOpcode(), Relaxed.getOperands(),
+                                     STI))
       Backend.relaxInstruction(Relaxed, STI);
     emitInstToData(Relaxed, STI);
     return;
@@ -390,16 +377,36 @@ void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
   emitInstToFragment(Inst, STI);
 }
 
+void MCObjectStreamer::emitInstToData(const MCInst &Inst,
+                                      const MCSubtargetInfo &STI) {
+  MCDataFragment *DF = getOrCreateDataFragment();
+  SmallVector<MCFixup, 1> Fixups;
+  SmallString<256> Code;
+  getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
+
+  auto CodeOffset = DF->getContents().size();
+  for (MCFixup &Fixup : Fixups)
+    Fixup.setOffset(Fixup.getOffset() + CodeOffset);
+  if (!Fixups.empty())
+    DF->appendFixups(Fixups);
+  DF->setHasInstructions(STI);
+  DF->appendContents(Code);
+}
+
 void MCObjectStreamer::emitInstToFragment(const MCInst &Inst,
                                           const MCSubtargetInfo &STI) {
   // Always create a new, separate fragment here, because its size can change
   // during relaxation.
   MCRelaxableFragment *IF =
-      getContext().allocFragment<MCRelaxableFragment>(Inst, STI);
+      getContext().allocFragment<MCRelaxableFragment>(STI);
   insert(IF);
+  IF->setInst(Inst);
 
-  getAssembler().getEmitter().encodeInstruction(Inst, IF->getContents(),
-                                                IF->getFixups(), STI);
+  SmallVector<MCFixup, 1> Fixups;
+  getAssembler().getEmitter().encodeInstruction(
+      Inst, IF->getContentsForAppending(), Fixups, STI);
+  IF->doneAppending();
+  IF->appendFixups(Fixups);
 }
 
 #ifndef NDEBUG
@@ -691,8 +698,7 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
   if (OffsetVal.isAbsolute()) {
     if (OffsetVal.getConstant() < 0)
       return std::make_pair(false, std::string(".reloc offset is negative"));
-    DF->getFixups().push_back(
-        MCFixup::create(OffsetVal.getConstant(), Expr, Kind, Loc));
+    DF->addFixup(MCFixup::create(OffsetVal.getConstant(), Expr, Kind));
     return std::nullopt;
   }
   if (OffsetVal.getSubSym())
@@ -708,14 +714,13 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
     if (Error != std::nullopt)
       return Error;
 
-    DF->getFixups().push_back(
-        MCFixup::create(SymbolOffset + OffsetVal.getConstant(),
-                        Expr, Kind, Loc));
+    DF->addFixup(
+        MCFixup::create(SymbolOffset + OffsetVal.getConstant(), Expr, Kind));
     return std::nullopt;
   }
 
   PendingFixups.emplace_back(
-      &Symbol, DF, MCFixup::create(OffsetVal.getConstant(), Expr, Kind, Loc));
+      &Symbol, DF, MCFixup::create(OffsetVal.getConstant(), Expr, Kind));
   return std::nullopt;
 }
 
diff --git a/llvm/lib/MC/MCObjectWriter.cpp b/llvm/lib/MC/MCObjectWriter.cpp
index 86be52d57ef7a..7cbd0b489d9f6 100644
--- a/llvm/lib/MC/MCObjectWriter.cpp
+++ b/llvm/lib/MC/MCObjectWriter.cpp
@@ -10,7 +10,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 namespace llvm {
diff --git a/llvm/lib/MC/MCParser/AsmLexer.cpp b/llvm/lib/MC/MCParser/AsmLexer.cpp
index 3db9ed3199dd8..968ccf776440b 100644
--- a/llvm/lib/MC/MCParser/AsmLexer.cpp
+++ b/llvm/lib/MC/MCParser/AsmLexer.cpp
@@ -834,8 +834,10 @@ AsmToken AsmLexer::LexToken() {
       return LexLineComment();
   }
 
-  if (isAtStartOfComment(TokStart))
+  if (isAtStartOfComment(TokStart)) {
+    CurPtr += MAI.getCommentString().size() - 1;
     return LexLineComment();
+  }
 
   if (isAtStatementSeparator(TokStart)) {
     CurPtr += strlen(MAI.getSeparatorString()) - 1;
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index bb8c45bd901cd..df4538df4c90b 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -1207,17 +1207,14 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
     if (SymbolName.empty())
       return Error(getLexer().getLoc(), "expected a symbol reference");
 
-    MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-
-    // Lookup the symbol variant if used.
+    // Lookup the @specifier if used.
+    uint16_t Spec = 0;
     if (!Split.second.empty()) {
-      auto MaybeVariant = MAI.getSpecifierForName(Split.second);
-      if (MaybeVariant) {
+      auto MaybeSpecifier = MAI.getSpecifierForName(Split.second);
+      if (MaybeSpecifier) {
         SymbolName = Split.first;
-        Variant = MCSymbolRefExpr::VariantKind(*MaybeVariant);
-      } else if (MAI.doesAllowAtInName()) {
-        Variant = MCSymbolRefExpr::VK_None;
-      } else {
+        Spec = *MaybeSpecifier;
+      } else if (!MAI.doesAllowAtInName()) {
         return Error(SMLoc::getFromPointer(Split.second.begin()),
                      "invalid variant '" + Split.second + "'");
       }
@@ -1232,11 +1229,11 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
     // semantics in the face of reassignment.
     if (Sym->isVariable()) {
       auto V = Sym->getVariableValue();
-      bool DoInline = isa<MCConstantExpr>(V) && !Variant;
+      bool DoInline = isa<MCConstantExpr>(V) && !Spec;
       if (auto TV = dyn_cast<MCTargetExpr>(V))
         DoInline = TV->inlineAssignedExpr();
       if (DoInline) {
-        if (Variant)
+        if (Spec)
           return Error(EndLoc, "unexpected modifier on variable reference");
         Res = Sym->getVariableValue();
         return false;
@@ -1244,7 +1241,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
     }
 
     // Otherwise create a symbol ref.
-    Res = MCSymbolRefExpr::create(Sym, Variant, getContext(), FirstTokenLoc);
+    Res = MCSymbolRefExpr::create(Sym, Spec, getContext(), FirstTokenLoc);
     return false;
   }
   case AsmToken::BigNum:
@@ -1260,18 +1257,18 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
       StringRef IDVal = getTok().getString();
       // Lookup the symbol variant if used.
       std::pair<StringRef, StringRef> Split = IDVal.split('@');
-      MCSymbolRefExpr::VariantKind Spec = MCSymbolRefExpr::VK_None;
+      uint16_t Spec = 0;
       if (Split.first.size() != IDVal.size()) {
         auto MaybeSpec = MAI.getSpecifierForName(Split.second);
         if (!MaybeSpec)
           return TokError("invalid variant '" + Split.second + "'");
         IDVal = Split.first;
-        Spec = MCSymbolRefExpr::VariantKind(*MaybeSpec);
+        Spec = *MaybeSpec;
       }
       if (IDVal == "f" || IDVal == "b") {
         MCSymbol *Sym =
             Ctx.getDirectionalLocalSymbol(IntVal, IDVal == "b");
-        Res = MCSymbolRefExpr::create(Sym, Spec, getContext());
+        Res = MCSymbolRefExpr::create(Sym, Spec, getContext(), Loc);
         if (IDVal == "b" && Sym->isUndefined())
           return Error(Loc, "directional label undefined");
         DirLabels.push_back(std::make_tuple(Loc, CppHashInfo, Sym));
@@ -1353,13 +1350,14 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
   case MCExpr::SymbolRef: {
     const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(E);
 
-    if (SRE->getKind() != MCSymbolRefExpr::VK_None) {
+    if (SRE->getSpecifier()) {
       TokError("invalid variant on expression '" + getTok().getIdentifier() +
                "' (already modified)");
       return E;
     }
 
-    return MCSymbolRefExpr::create(&SRE->getSymbol(), Spec, getContext());
+    return MCSymbolRefExpr::create(&SRE->getSymbol(), Spec, getContext(),
+                                   SRE->getLoc());
   }
 
   case MCExpr::Unary: {
@@ -1367,7 +1365,8 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
     const MCExpr *Sub = applySpecifier(UE->getSubExpr(), Spec);
     if (!Sub)
       return nullptr;
-    return MCUnaryExpr::create(UE->getOpcode(), Sub, getContext());
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, getContext(),
+                               UE->getLoc());
   }
 
   case MCExpr::Binary: {
@@ -1383,7 +1382,8 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
     if (!RHS)
       RHS = BE->getRHS();
 
-    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, getContext());
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, getContext(),
+                                BE->getLoc());
   }
   }
 
@@ -2263,7 +2263,7 @@ bool AsmParser::parseAndMatchAndEmitTargetInstruction(ParseStatementInfo &Info,
     for (unsigned i = 0; i != Info.ParsedOperands.size(); ++i) {
       if (i != 0)
         OS << ", ";
-      Info.ParsedOperands[i]->print(OS);
+      Info.ParsedOperands[i]->print(OS, MAI);
     }
     OS << "]";
 
diff --git a/llvm/lib/MC/MCParser/MCAsmParser.cpp b/llvm/lib/MC/MCParser/MCAsmParser.cpp
index 729ea4f4480db..68b9cab2492f5 100644
--- a/llvm/lib/MC/MCParser/MCAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/MCAsmParser.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
@@ -165,6 +166,7 @@ bool MCAsmParser::parseGNUAttribute(SMLoc L, int64_t &Tag,
 void MCParsedAsmOperand::dump() const {
   // Cannot completely remove virtual function even in release mode.
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  dbgs() << "  " << *this;
+  dbgs() << "  ";
+  print(dbgs(), MCAsmInfo());
 #endif
 }
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index c596006ba9a0c..f4684e64e8628 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -2317,7 +2317,7 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     for (unsigned i = 0; i != Info.ParsedOperands.size(); ++i) {
       if (i != 0)
         OS << ", ";
-      Info.ParsedOperands[i]->print(OS);
+      Info.ParsedOperands[i]->print(OS, MAI);
     }
     OS << "]";
 
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index 8a9906df92975..f87d27f4adcc5 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -13,7 +13,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSymbol.h"
diff --git a/llvm/lib/MC/MCSPIRVStreamer.cpp b/llvm/lib/MC/MCSPIRVStreamer.cpp
index 3b8cbee70d209..d2534375e501e 100644
--- a/llvm/lib/MC/MCSPIRVStreamer.cpp
+++ b/llvm/lib/MC/MCSPIRVStreamer.cpp
@@ -16,21 +16,6 @@
 
 using namespace llvm;
 
-void MCSPIRVStreamer::emitInstToData(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI) {
-  MCAssembler &Assembler = getAssembler();
-  SmallVector<MCFixup, 0> Fixups;
-  SmallString<256> Code;
-  Assembler.getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Append the encoded instruction to the current data fragment (or create a
-  // new such fragment if the current fragment is not a data fragment).
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 MCStreamer *llvm::createSPIRVStreamer(MCContext &Context,
                                       std::unique_ptr<MCAsmBackend> &&MAB,
                                       std::unique_ptr<MCObjectWriter> &&OW,
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index b61738d8837a2..56450033529bc 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -10,8 +10,8 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -36,15 +36,6 @@ MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) {
 
 bool MCSection::hasEnded() const { return End && End->isInSection(); }
 
-MCSection::~MCSection() {
-  for (auto &[_, Chain] : Subsections) {
-    for (MCFragment *X = Chain.Head, *Y; X; X = Y) {
-      Y = X->Next;
-      X->destroy();
-    }
-  }
-}
-
 void MCSection::setBundleLockState(BundleLockStateType NewState) {
   if (NewState == NotBundleLocked) {
     if (BundleLockNestingDepth == 0) {
@@ -67,19 +58,61 @@ void MCSection::setBundleLockState(BundleLockStateType NewState) {
 StringRef MCSection::getVirtualSectionKind() const { return "virtual"; }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-LLVM_DUMP_METHOD void MCSection::dump() const {
+LLVM_DUMP_METHOD void MCSection::dump(
+    DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>> *FragToSyms)
+    const {
   raw_ostream &OS = errs();
 
-  OS << "<MCSection Name:" << getName();
-  OS << " Fragments:[\n      ";
-  bool First = true;
+  OS << "MCSection Name:" << getName();
   for (auto &F : *this) {
-    if (First)
-      First = false;
-    else
-      OS << ",\n      ";
+    OS << '\n';
     F.dump();
+    if (!FragToSyms)
+      continue;
+    auto It = FragToSyms->find(&F);
+    if (It == FragToSyms->end())
+      continue;
+    for (auto *Sym : It->second) {
+      OS << "\n  Symbol @" << Sym->getOffset() << ' ' << Sym->getName();
+      if (Sym->isTemporary())
+        OS << " Temporary";
+    }
   }
-  OS << "]>";
 }
 #endif
+
+void MCEncodedFragment::setContents(ArrayRef<char> Contents) {
+  auto &S = getParent()->ContentStorage;
+  if (ContentStart + Contents.size() > ContentEnd) {
+    ContentStart = S.size();
+    S.resize_for_overwrite(S.size() + Contents.size());
+  }
+  ContentEnd = ContentStart + Contents.size();
+  llvm::copy(Contents, S.begin() + ContentStart);
+}
+
+void MCEncodedFragment::addFixup(MCFixup Fixup) { appendFixups({Fixup}); }
+
+void MCEncodedFragment::appendFixups(ArrayRef<MCFixup> Fixups) {
+  auto &S = getParent()->FixupStorage;
+  if (LLVM_UNLIKELY(FixupEnd != S.size())) {
+    // Move the elements to the end. Reserve space to avoid invalidating
+    // S.begin()+I for `append`.
+    auto Size = FixupEnd - FixupStart;
+    auto I = std::exchange(FixupStart, S.size());
+    S.reserve(S.size() + Size);
+    S.append(S.begin() + I, S.begin() + I + Size);
+  }
+  S.append(Fixups.begin(), Fixups.end());
+  FixupEnd = S.size();
+}
+
+void MCEncodedFragment::setFixups(ArrayRef<MCFixup> Fixups) {
+  auto &S = getParent()->FixupStorage;
+  if (FixupStart + Fixups.size() > FixupEnd) {
+    FixupStart = S.size();
+    S.resize_for_overwrite(S.size() + Fixups.size());
+  }
+  FixupEnd = FixupStart + Fixups.size();
+  llvm::copy(Fixups, S.begin() + FixupStart);
+}
diff --git a/llvm/lib/MC/MCSectionELF.cpp b/llvm/lib/MC/MCSectionELF.cpp
index 181c4ff25d541..cc7cdf2fe4d1a 100644
--- a/llvm/lib/MC/MCSectionELF.cpp
+++ b/llvm/lib/MC/MCSectionELF.cpp
@@ -170,8 +170,6 @@ void MCSectionELF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
     OS << "llvm_sympart";
   else if (Type == ELF::SHT_LLVM_BB_ADDR_MAP)
     OS << "llvm_bb_addr_map";
-  else if (Type == ELF::SHT_LLVM_BB_ADDR_MAP_V0)
-    OS << "llvm_bb_addr_map_v0";
   else if (Type == ELF::SHT_LLVM_OFFLOADING)
     OS << "llvm_offloading";
   else if (Type == ELF::SHT_LLVM_LTO)
diff --git a/llvm/lib/MC/MCSymbol.cpp b/llvm/lib/MC/MCSymbol.cpp
index 5cba901d33d5e..baf1ff89c0188 100644
--- a/llvm/lib/MC/MCSymbol.cpp
+++ b/llvm/lib/MC/MCSymbol.cpp
@@ -11,7 +11,6 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/llvm/lib/MC/MCWasmStreamer.cpp b/llvm/lib/MC/MCWasmStreamer.cpp
index cc731fe02f7d3..602d40df4cc3a 100644
--- a/llvm/lib/MC/MCWasmStreamer.cpp
+++ b/llvm/lib/MC/MCWasmStreamer.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionWasm.h"
@@ -151,26 +150,6 @@ void MCWasmStreamer::emitIdent(StringRef IdentString) {
   // sections in the object format
 }
 
-void MCWasmStreamer::emitInstToData(const MCInst &Inst,
-                                    const MCSubtargetInfo &STI) {
-  MCAssembler &Assembler = getAssembler();
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  Assembler.getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Append the encoded instruction to the current data fragment (or create a
-  // new such fragment if the current fragment is not a data fragment).
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  // Add the fixups and data.
-  for (MCFixup &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixup);
-  }
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCWasmStreamer::finishImpl() {
   emitFrames(nullptr);
 
diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp
index 7fc8919d10453..8cc74db9beea4 100644
--- a/llvm/lib/MC/MCWin64EH.cpp
+++ b/llvm/lib/MC/MCWin64EH.cpp
@@ -338,7 +338,7 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
       auto *MCE = MCUnwindV2EpilogTargetExpr::create(*info, Epilog.second,
                                                      EpilogSize, context);
       MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_2);
-      DF->getFixups().push_back(Fixup);
+      DF->addFixup(Fixup);
       DF->appendContents(2, 0);
     }
   }
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index d95e0cb8f0439..0c0866f9e36a4 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -136,23 +135,6 @@ WinCOFFObjectWriter &MCWinCOFFStreamer::getWriter() {
   return static_cast<WinCOFFObjectWriter &>(getAssembler().getWriter());
 }
 
-void MCWinCOFFStreamer::emitInstToData(const MCInst &Inst,
-                                       const MCSubtargetInfo &STI) {
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Add the fixups and data.
-  for (MCFixup &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixup);
-  }
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCWinCOFFStreamer::initSections(bool NoExecStack,
                                      const MCSubtargetInfo &STI) {
   // FIXME: this is identical to the ELF one.
@@ -299,7 +281,7 @@ void MCWinCOFFStreamer::emitCOFFSectionIndex(const MCSymbol *Symbol) {
   MCDataFragment *DF = getOrCreateDataFragment();
   const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext());
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, FK_SecRel_2);
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   DF->appendContents(2, 0);
 }
 
@@ -316,7 +298,7 @@ void MCWinCOFFStreamer::emitCOFFSecRel32(const MCSymbol *Symbol,
   // Build the secrel32 relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_SecRel_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
@@ -335,7 +317,7 @@ void MCWinCOFFStreamer::emitCOFFImgRel32(const MCSymbol *Symbol,
   // Build the imgrel relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
@@ -349,7 +331,7 @@ void MCWinCOFFStreamer::emitCOFFSecNumber(MCSymbol const *Symbol) {
   // Build the relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
@@ -363,7 +345,7 @@ void MCWinCOFFStreamer::emitCOFFSecOffset(MCSymbol const *Symbol) {
   // Build the relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
diff --git a/llvm/lib/MC/MCXCOFFStreamer.cpp b/llvm/lib/MC/MCXCOFFStreamer.cpp
index 15de5d83bfe10..12e6680bcf074 100644
--- a/llvm/lib/MC/MCXCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCXCOFFStreamer.cpp
@@ -98,7 +98,7 @@ void MCXCOFFStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) {
 
   MCFixupKind Kind = *MaybeKind;
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, Kind);
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
 }
 
 void MCXCOFFStreamer::emitXCOFFRenameDirective(const MCSymbol *Name,
@@ -138,26 +138,6 @@ void MCXCOFFStreamer::emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   emitZeros(Size);
 }
 
-void MCXCOFFStreamer::emitInstToData(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI) {
-  MCAssembler &Assembler = getAssembler();
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  Assembler.getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Add the fixups and data.
-  MCDataFragment *DF = getOrCreateDataFragment(&STI);
-  const size_t ContentsSize = DF->getContents().size();
-  auto &DataFragmentFixups = DF->getFixups();
-  for (auto &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + ContentsSize);
-    DataFragmentFixups.push_back(Fixup);
-  }
-
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCXCOFFStreamer::emitXCOFFLocalCommonSymbol(MCSymbol *LabelSym,
                                                  uint64_t Size,
                                                  MCSymbol *CsectSym,
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 9377a74b377ea..fb8e1fdf62f50 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -15,8 +15,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -86,12 +84,6 @@ MachSymbolData::operator<(const MachSymbolData &RHS) const {
   return Symbol->getName() < RHS.Symbol->getName();
 }
 
-bool MachObjectWriter::isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind) {
-  MCFixupKindInfo FKI = Asm.getBackend().getFixupKindInfo((MCFixupKind)Kind);
-
-  return FKI.Flags & MCFixupKindInfo::FKF_IsPCRel;
-}
-
 uint64_t
 MachObjectWriter::getFragmentAddress(const MCAssembler &Asm,
                                      const MCFragment *Fragment) const {
@@ -808,8 +800,8 @@ uint64_t MachObjectWriter::writeObject() {
     MCSection *CGProfileSection = getContext().getMachOSection(
         "__LLVM", "__cg_profile", 0, SectionKind::getMetadata());
     auto &Frag = cast<MCDataFragment>(*CGProfileSection->begin());
-    Frag.getContents().clear();
-    raw_svector_ostream OS(Frag.getContents());
+    Frag.clearContents();
+    raw_svector_ostream OS(Frag.getContentsForAppending());
     for (const MCObjectWriter::CGProfileEntry &CGPE : CGProfile) {
       uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
       uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
@@ -817,6 +809,7 @@ uint64_t MachObjectWriter::writeObject() {
       support::endian::write(OS, ToIndex, W.Endian);
       support::endian::write(OS, CGPE.Count, W.Endian);
     }
+    Frag.doneAppending();
   }
 
   unsigned NumSections = Asm.end() - Asm.begin();
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index 3b82fb782f888..d148b521ff9fe 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionWasm.h"
 #include "llvm/MC/MCSymbolWasm.h"
@@ -479,8 +478,7 @@ void WasmObjectWriter::recordRelocation(const MCFragment &F,
                                         const MCFixup &Fixup, MCValue Target,
                                         uint64_t &FixedValue) {
   // The WebAssembly backend should never generate FKF_IsPCRel fixups
-  assert(!(Asm->getBackend().getFixupKindInfo(Fixup.getKind()).Flags &
-           MCFixupKindInfo::FKF_IsPCRel));
+  assert(!Fixup.isPCRel());
 
   const auto &FixupSection = cast<MCSectionWasm>(*F.getParent());
   uint64_t C = Target.getConstant();
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index 2ed0aa3d20fc9..c2ef430984ed4 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionCOFF.h"
@@ -1071,7 +1070,7 @@ uint64_t WinCOFFWriter::writeObject() {
     auto *Sec = getContext().getCOFFSection(".llvm_addrsig",
                                             COFF::IMAGE_SCN_LNK_REMOVE);
     auto *Frag = cast<MCDataFragment>(Sec->curFragList()->Head);
-    raw_svector_ostream OS(Frag->getContents());
+    raw_svector_ostream OS(Frag->getContentsForAppending());
     for (const MCSymbol *S : OWriter.AddrsigSyms) {
       if (!S->isRegistered())
         continue;
@@ -1086,6 +1085,7 @@ uint64_t WinCOFFWriter::writeObject() {
              "executePostLayoutBinding!");
       encodeULEB128(SectionMap[TargetSection]->Symbol->getIndex(), OS);
     }
+    Frag->doneAppending();
   }
 
   // Create the contents of the .llvm.call-graph-profile section.
@@ -1093,7 +1093,7 @@ uint64_t WinCOFFWriter::writeObject() {
     auto *Sec = getContext().getCOFFSection(".llvm.call-graph-profile",
                                             COFF::IMAGE_SCN_LNK_REMOVE);
     auto *Frag = cast<MCDataFragment>(Sec->curFragList()->Head);
-    raw_svector_ostream OS(Frag->getContents());
+    raw_svector_ostream OS(Frag->getContentsForAppending());
     for (const auto &CGPE : OWriter.getCGProfile()) {
       uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
       uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
@@ -1101,6 +1101,7 @@ uint64_t WinCOFFWriter::writeObject() {
       support::endian::write(OS, ToIndex, W.Endian);
       support::endian::write(OS, CGPE.Count, W.Endian);
     }
+    Frag->doneAppending();
   }
 
   assignFileOffsets();
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index ddb2ca8a03e2e..2f6785fe7b807 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionXCOFF.h"
 #include "llvm/MC/MCSymbolXCOFF.h"
@@ -686,15 +685,10 @@ void XCOFFWriter::recordRelocation(const MCFragment &F, const MCFixup &Fixup,
   };
 
   const MCSymbol *const SymA = Target.getAddSym();
-
-  MCAsmBackend &Backend = Asm->getBackend();
-  bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags &
-                 MCFixupKindInfo::FKF_IsPCRel;
-
   uint8_t Type;
   uint8_t SignAndSize;
-  std::tie(Type, SignAndSize) =
-      TargetObjectWriter->getRelocTypeAndSignSize(Target, Fixup, IsPCRel);
+  std::tie(Type, SignAndSize) = TargetObjectWriter->getRelocTypeAndSignSize(
+      Target, Fixup, Fixup.isPCRel());
 
   const MCSectionXCOFF *SymASec = getContainingCsect(cast<MCSymbolXCOFF>(SymA));
   assert(SectionMap.contains(SymASec) &&
diff --git a/llvm/lib/MCA/CodeEmitter.cpp b/llvm/lib/MCA/CodeEmitter.cpp
index b3936a95c06a8..5bc48d9227f4c 100644
--- a/llvm/lib/MCA/CodeEmitter.cpp
+++ b/llvm/lib/MCA/CodeEmitter.cpp
@@ -22,12 +22,8 @@ CodeEmitter::EncodingInfo CodeEmitter::getOrCreateEncodingInfo(unsigned MCID) {
 
   SmallVector<llvm::MCFixup, 2> Fixups;
   const MCInst &Inst = Sequence[MCID];
-  MCInst Relaxed(Sequence[MCID]);
-  if (MAB.mayNeedRelaxation(Inst, STI))
-    MAB.relaxInstruction(Relaxed, STI);
-
   EI.first = Code.size();
-  MCE.encodeInstruction(Relaxed, Code, Fixups, STI);
+  MCE.encodeInstruction(Inst, Code, Fixups, STI);
   EI.second = Code.size() - EI.first;
   return EI;
 }
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index 6ee33d94ee861..ff99dc25abe07 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -317,7 +317,6 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) {
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_SYMPART);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_PART_EHDR);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_PART_PHDR);
-    STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_BB_ADDR_MAP_V0);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_BB_ADDR_MAP);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_OFFLOADING);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_LTO);
@@ -833,33 +832,24 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
   BBAddrMap::Features FeatEnable{};
   while (!ULEBSizeErr && !MetadataDecodeErr && Cur &&
          Cur.tell() < Content.size()) {
-    if (Sec.sh_type == ELF::SHT_LLVM_BB_ADDR_MAP) {
-      Version = Data.getU8(Cur);
-      if (!Cur)
-        break;
-      if (Version > 3)
-        return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
-                           Twine(static_cast<int>(Version)));
-      Feature = Data.getU8(Cur); // Feature byte
-      if (!Cur)
-        break;
-      auto FeatEnableOrErr = BBAddrMap::Features::decode(Feature);
-      if (!FeatEnableOrErr)
-        return FeatEnableOrErr.takeError();
-      FeatEnable = *FeatEnableOrErr;
-      if (FeatEnable.hasPGOAnalysis() && Version < 2)
-        return createError(
-            "version should be >= 2 for SHT_LLVM_BB_ADDR_MAP when "
-            "PGO features are enabled: version = " +
-            Twine(static_cast<int>(Version)) +
-            " feature = " + Twine(static_cast<int>(Feature)));
-      if (FeatEnable.CallsiteOffsets && Version < 3)
-        return createError(
-            "version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when "
-            "callsite offsets feature is enabled: version = " +
-            Twine(static_cast<int>(Version)) +
-            " feature = " + Twine(static_cast<int>(Feature)));
-    }
+    Version = Data.getU8(Cur);
+    if (!Cur)
+      break;
+    if (Version < 2 || Version > 3)
+      return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
+                         Twine(static_cast<int>(Version)));
+    Feature = Data.getU8(Cur); // Feature byte
+    if (!Cur)
+      break;
+    auto FeatEnableOrErr = BBAddrMap::Features::decode(Feature);
+    if (!FeatEnableOrErr)
+      return FeatEnableOrErr.takeError();
+    FeatEnable = *FeatEnableOrErr;
+    if (FeatEnable.CallsiteOffsets && Version < 3)
+      return createError("version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when "
+                         "callsite offsets feature is enabled: version = " +
+                         Twine(static_cast<int>(Version)) +
+                         " feature = " + Twine(static_cast<int>(Feature)));
     uint32_t NumBlocksInBBRange = 0;
     uint32_t NumBBRanges = 1;
     typename ELFFile<ELFT>::uintX_t RangeBaseAddress = 0;
@@ -895,9 +885,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
         for (uint32_t BlockIndex = 0; !MetadataDecodeErr && !ULEBSizeErr &&
                                       Cur && (BlockIndex < NumBlocksInBBRange);
              ++BlockIndex) {
-          uint32_t ID = Version >= 2
-                            ? readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr)
-                            : BlockIndex;
+          uint32_t ID = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
           uint32_t Offset = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
           // Read the callsite offsets.
           uint32_t LastCallsiteOffset = 0;
@@ -917,19 +905,15 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
           uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr) +
                           LastCallsiteOffset;
           uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
-          if (Version >= 1) {
-            // Offset is calculated relative to the end of the previous BB.
-            Offset += PrevBBEndOffset;
-            PrevBBEndOffset = Offset + Size;
-          }
           Expected<BBAddrMap::BBEntry::Metadata> MetadataOrErr =
               BBAddrMap::BBEntry::Metadata::decode(MD);
           if (!MetadataOrErr) {
             MetadataDecodeErr = MetadataOrErr.takeError();
             break;
           }
-          BBEntries.push_back(
-              {ID, Offset, Size, *MetadataOrErr, CallsiteOffsets});
+          BBEntries.push_back({ID, Offset + PrevBBEndOffset, Size,
+                               *MetadataOrErr, CallsiteOffsets});
+          PrevBBEndOffset += Offset + Size;
         }
         TotalNumBlocks += BBEntries.size();
       }
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index cce5c74aa4d1d..0e13d32bbe522 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -904,8 +904,7 @@ Expected<std::vector<BBAddrMap>> static readBBAddrMapImpl(
 
   const auto &Sections = cantFail(EF.sections());
   auto IsMatch = [&](const Elf_Shdr &Sec) -> Expected<bool> {
-    if (Sec.sh_type != ELF::SHT_LLVM_BB_ADDR_MAP &&
-        Sec.sh_type != ELF::SHT_LLVM_BB_ADDR_MAP_V0)
+    if (Sec.sh_type != ELF::SHT_LLVM_BB_ADDR_MAP)
       return false;
     if (!TextSectionIndex)
       return true;
diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index f6d0e5227e64a..263f7bdf37bca 100644
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -63,9 +63,9 @@ readDescriptorRanges(DXContainerYAML::RootParameterHeaderYaml &Header,
     NewR.RangeType = R.RangeType;
     if constexpr (std::is_same_v<T, dxbc::RTS0::v2::DescriptorRange>) {
       // Set all flag fields for v2
-#define DESCRIPTOR_RANGE_FLAG(Num, Val)                                        \
-  NewR.Val =                                                                   \
-      (R.Flags & llvm::to_underlying(dxbc::DescriptorRangeFlag::Val)) != 0;
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag)                                 \
+  NewR.Enum =                                                                  \
+      (R.Flags & llvm::to_underlying(dxbc::DescriptorRangeFlags::Enum)) != 0;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
     }
     TableYaml.Ranges.push_back(NewR);
@@ -140,10 +140,10 @@ DXContainerYAML::RootSignatureYamlDesc::create(
       YamlDescriptor.ShaderRegister = Descriptor.ShaderRegister;
       YamlDescriptor.RegisterSpace = Descriptor.RegisterSpace;
       if (Version > 1) {
-#define ROOT_DESCRIPTOR_FLAG(Num, Val)                                         \
-  YamlDescriptor.Val =                                                         \
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag)                                  \
+  YamlDescriptor.Enum =                                                        \
       (Descriptor.Flags &                                                      \
-       llvm::to_underlying(dxbc::RootDescriptorFlag::Val)) > 0;
+       llvm::to_underlying(dxbc::RootDescriptorFlags::Enum)) > 0;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
       }
     } else if (auto *DTV =
@@ -180,38 +180,37 @@ DXContainerYAML::RootSignatureYamlDesc::create(
     RootSigDesc.StaticSamplers.push_back(NewS);
   }
 
-#define ROOT_ELEMENT_FLAG(Num, Val)                                            \
-  RootSigDesc.Val =                                                            \
-      (Flags & llvm::to_underlying(dxbc::RootElementFlag::Val)) > 0;
+#define ROOT_SIGNATURE_FLAG(Num, Val)                                          \
+  RootSigDesc.Val = (Flags & llvm::to_underlying(dxbc::RootFlags::Val)) > 0;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
   return RootSigDesc;
 }
 
 uint32_t DXContainerYAML::RootDescriptorYaml::getEncodedFlags() const {
-  uint64_t Flag = 0;
-#define ROOT_DESCRIPTOR_FLAG(Num, Val)                                         \
-  if (Val)                                                                     \
-    Flag |= (uint32_t)dxbc::RootDescriptorFlag::Val;
+  uint64_t Flags = 0;
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag)                                  \
+  if (Enum)                                                                    \
+    Flags |= (uint32_t)dxbc::RootDescriptorFlags::Enum;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
-  return Flag;
+  return Flags;
 }
 
 uint32_t DXContainerYAML::RootSignatureYamlDesc::getEncodedFlags() {
   uint64_t Flag = 0;
-#define ROOT_ELEMENT_FLAG(Num, Val)                                            \
+#define ROOT_SIGNATURE_FLAG(Num, Val)                                          \
   if (Val)                                                                     \
-    Flag |= (uint32_t)dxbc::RootElementFlag::Val;
+    Flag |= (uint32_t)dxbc::RootFlags::Val;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
   return Flag;
 }
 
 uint32_t DXContainerYAML::DescriptorRangeYaml::getEncodedFlags() const {
-  uint64_t Flag = 0;
-#define DESCRIPTOR_RANGE_FLAG(Num, Val)                                        \
-  if (Val)                                                                     \
-    Flag |= (uint32_t)dxbc::DescriptorRangeFlag::Val;
+  uint64_t Flags = 0;
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag)                                 \
+  if (Enum)                                                                    \
+    Flags |= (uint32_t)dxbc::DescriptorRangeFlags::Enum;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
-  return Flag;
+  return Flags;
 }
 
 uint64_t DXContainerYAML::ShaderFeatureFlags::getEncodedFlags() {
@@ -382,7 +381,7 @@ void MappingTraits<DXContainerYAML::RootSignatureYamlDesc>::mapping(
   IO.mapRequired("StaticSamplersOffset", S.StaticSamplersOffset);
   IO.mapRequired("Parameters", S.Parameters.Locations, S);
   IO.mapOptional("Samplers", S.StaticSamplers);
-#define ROOT_ELEMENT_FLAG(Num, Val) IO.mapOptional(#Val, S.Val, false);
+#define ROOT_SIGNATURE_FLAG(Num, Val) IO.mapOptional(#Val, S.Val, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
@@ -406,7 +405,8 @@ void MappingTraits<llvm::DXContainerYAML::DescriptorRangeYaml>::mapping(
   IO.mapRequired("RegisterSpace", R.RegisterSpace);
   IO.mapRequired("OffsetInDescriptorsFromTableStart",
                  R.OffsetInDescriptorsFromTableStart);
-#define DESCRIPTOR_RANGE_FLAG(Num, Val) IO.mapOptional(#Val, R.Val, false);
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag)                                 \
+  IO.mapOptional(#Flag, R.Enum, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
@@ -459,7 +459,8 @@ void MappingTraits<llvm::DXContainerYAML::RootDescriptorYaml>::mapping(
     IO &IO, llvm::DXContainerYAML::RootDescriptorYaml &D) {
   IO.mapRequired("RegisterSpace", D.RegisterSpace);
   IO.mapRequired("ShaderRegister", D.ShaderRegister);
-#define ROOT_DESCRIPTOR_FLAG(Num, Val) IO.mapOptional(#Val, D.Val, false);
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag)                                  \
+  IO.mapOptional(#Flag, D.Enum, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index b28191730f018..970811f40f918 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -720,7 +720,6 @@ void ScalarEnumerationTraits<ELFYAML::ELF_SHT>::enumeration(
   ECase(SHT_LLVM_SYMPART);
   ECase(SHT_LLVM_PART_EHDR);
   ECase(SHT_LLVM_PART_PHDR);
-  ECase(SHT_LLVM_BB_ADDR_MAP_V0);
   ECase(SHT_LLVM_BB_ADDR_MAP);
   ECase(SHT_LLVM_OFFLOADING);
   ECase(SHT_LLVM_LTO);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f95c32d859d14..67800874b0a3f 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -339,6 +339,7 @@
 #include "llvm/Transforms/Utils/CountVisits.h"
 #include "llvm/Transforms/Utils/DXILUpgrade.h"
 #include "llvm/Transforms/Utils/Debugify.h"
+#include "llvm/Transforms/Utils/DeclareRuntimeLibcalls.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 #include "llvm/Transforms/Utils/FixIrreducible.h"
 #include "llvm/Transforms/Utils/HelloWorld.h"
@@ -938,6 +939,19 @@ parseLowerAllowCheckPassOptions(StringRef Params) {
 
         Result.cutoffs[index] = cutoff;
       }
+    } else if (ParamName.starts_with("runtime_check")) {
+      StringRef ValueString;
+      std::tie(std::ignore, ValueString) = ParamName.split("=");
+      int runtime_check;
+      if (ValueString.getAsInteger(0, runtime_check)) {
+        return make_error<StringError>(
+            formatv("invalid LowerAllowCheck pass runtime_check parameter '{}' "
+                    "({})",
+                    ValueString, Params)
+                .str(),
+            inconvertibleErrorCode());
+      }
+      Result.runtime_check = runtime_check;
     } else {
       return make_error<StringError>(
           formatv("invalid LowerAllowCheck pass parameter '{}'", ParamName)
@@ -1511,6 +1525,41 @@ Expected<bool> parseVirtRegRewriterPassOptions(StringRef Params) {
   return ClearVirtRegs;
 }
 
+struct FatLTOOptions {
+  OptimizationLevel OptLevel;
+  bool ThinLTO = false;
+  bool EmitSummary = false;
+};
+
+Expected<FatLTOOptions> parseFatLTOOptions(StringRef Params) {
+  FatLTOOptions Result;
+  bool HaveOptLevel = false;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+
+    if (ParamName == "thinlto") {
+      Result.ThinLTO = true;
+    } else if (ParamName == "emit-summary") {
+      Result.EmitSummary = true;
+    } else if (std::optional<OptimizationLevel> OptLevel =
+                   parseOptLevel(ParamName)) {
+      Result.OptLevel = *OptLevel;
+      HaveOptLevel = true;
+    } else {
+      return make_error<StringError>(
+          formatv("invalid fatlto-pre-link pass parameter '{}'", ParamName)
+              .str(),
+          inconvertibleErrorCode());
+    }
+  }
+  if (!HaveOptLevel)
+    return make_error<StringError>(
+        "missing optimization level for fatlto-pre-link pipeline",
+        inconvertibleErrorCode());
+  return Result;
+}
+
 } // namespace
 
 /// Tests whether registered callbacks will accept a given pass name.
@@ -2170,9 +2219,18 @@ Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
 Error PassBuilder::parseMachinePass(MachineFunctionPassManager &MFPM,
                                     const PipelineElement &E) {
   StringRef Name = E.Name;
-  if (!E.InnerPipeline.empty())
+  // Handle any nested pass managers.
+  if (!E.InnerPipeline.empty()) {
+    if (E.Name == "machine-function") {
+      MachineFunctionPassManager NestedPM;
+      if (auto Err = parseMachinePassPipeline(NestedPM, E.InnerPipeline))
+        return Err;
+      MFPM.addPass(std::move(NestedPM));
+      return Error::success();
+    }
     return make_error<StringError>("invalid pipeline",
                                    inconvertibleErrorCode());
+  }
 
 #define MACHINE_MODULE_PASS(NAME, CREATE_PASS)                                 \
   if (Name == NAME) {                                                          \
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index c83d2dc1f1514..98821bb1408a7 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -690,9 +690,6 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
 
   LPM2.addPass(LoopDeletionPass());
 
-  if (PTO.LoopInterchange)
-    LPM2.addPass(LoopInterchangePass());
-
   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
   // because it changes IR to makes profile annotation in back compile
   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
@@ -1547,6 +1544,10 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
   //        this may need to be revisited once we run GVN before loop deletion
   //        in the simplification pipeline.
   LPM.addPass(LoopDeletionPass());
+
+  if (PTO.LoopInterchange)
+    LPM.addPass(LoopInterchangePass());
+
   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
       std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false));
 
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 1d924f38a47fb..96250772da4a0 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -63,18 +63,21 @@ MODULE_PASS("coro-early", CoroEarlyPass())
 MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass())
 MODULE_PASS("ctx-instr-gen",
             PGOInstrumentationGen(PGOInstrumentationType::CTXPROF))
-MODULE_PASS("ctx-prof-flatten", PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false))
+MODULE_PASS("ctx-prof-flatten",
+            PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false))
 MODULE_PASS("ctx-prof-flatten-prethinlink",
             PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true))
 MODULE_PASS("noinline-nonprevailing", NoinlineNonPrevailing())
 MODULE_PASS("deadargelim", DeadArgumentEliminationPass())
 MODULE_PASS("debugify", NewPMDebugifyPass())
+MODULE_PASS("declare-runtime-libcalls", DeclareRuntimeLibcallsPass())
 MODULE_PASS("dfsan", DataFlowSanitizerPass())
 MODULE_PASS("dot-callgraph", CallGraphDOTPrinterPass())
 MODULE_PASS("dxil-upgrade", DXILUpgradePass())
 MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass())
 MODULE_PASS("extract-blocks", BlockExtractorPass({}, false))
-MODULE_PASS("expand-variadics", ExpandVariadicsPass(ExpandVariadicsMode::Disable))
+MODULE_PASS("expand-variadics",
+            ExpandVariadicsPass(ExpandVariadicsMode::Disable))
 MODULE_PASS("forceattrs", ForceFunctionAttrsPass())
 MODULE_PASS("function-import", FunctionImportPass())
 MODULE_PASS("global-merge-func", GlobalMergeFuncPass())
@@ -104,7 +107,10 @@ MODULE_PASS("lower-ifunc", LowerIFuncPass())
 MODULE_PASS("simplify-type-tests", SimplifyTypeTestsPass())
 MODULE_PASS("lowertypetests", LowerTypeTestsPass())
 MODULE_PASS("fatlto-cleanup", FatLtoCleanup())
-MODULE_PASS("pgo-force-function-attrs", PGOForceFunctionAttrsPass(PGOOpt ? PGOOpt->ColdOptType : PGOOptions::ColdFuncOpt::Default))
+MODULE_PASS("pgo-force-function-attrs",
+            PGOForceFunctionAttrsPass(PGOOpt
+                                          ? PGOOpt->ColdOptType
+                                          : PGOOptions::ColdFuncOpt::Default))
 MODULE_PASS("memprof-context-disambiguation", MemProfContextDisambiguation())
 MODULE_PASS("memprof-module", ModuleMemProfilerPass())
 MODULE_PASS("mergefunc", MergeFunctionsPass())
@@ -136,6 +142,7 @@ MODULE_PASS("print<dxil-metadata>", DXILMetadataAnalysisPrinterPass(errs()))
 MODULE_PASS("print<dxil-resources>", DXILResourcePrinterPass(errs()))
 MODULE_PASS("print<inline-advisor>", InlineAdvisorAnalysisPrinterPass(errs()))
 MODULE_PASS("print<ir2vec>", IR2VecPrinterPass(errs()))
+MODULE_PASS("print<ir2vec-vocab>", IR2VecVocabPrinterPass(errs()))
 MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(errs()))
 MODULE_PASS("print<reg-usage>", PhysicalRegisterUsageInfoPrinterPass(errs()))
 MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
@@ -175,10 +182,10 @@ MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass())
 MODULE_PASS_WITH_PARAMS(
     "asan", "AddressSanitizerPass",
     [](AddressSanitizerOptions Opts) { return AddressSanitizerPass(Opts); },
-    parseASanPassOptions, "kernel")
+    parseASanPassOptions, "kernel;use-after-scope")
 MODULE_PASS_WITH_PARAMS(
     "cg-profile", "CGProfilePass",
-    [](bool InLTOPostLink) { return CGProfilePass(InLTOPostLink);},
+    [](bool InLTOPostLink) { return CGProfilePass(InLTOPostLink); },
     parseCGProfilePassOptions, "in-lto-post-link")
 MODULE_PASS_WITH_PARAMS(
     "global-merge", "GlobalMergePass",
@@ -187,7 +194,7 @@ MODULE_PASS_WITH_PARAMS(
     "group-by-use;ignore-single-use;max-offset=N;merge-const;"
     "merge-const-aggressive;merge-external;no-group-by-use;"
     "no-ignore-single-use;no-merge-const;no-merge-const-aggressive;"
-    "no-merge-external;size-only")
+    "no-merge-external")
 MODULE_PASS_WITH_PARAMS(
     "embed-bitcode", "EmbedBitcodePass",
     [](EmbedBitcodeOptions Opts) { return EmbedBitcodePass(Opts); },
@@ -195,7 +202,7 @@ MODULE_PASS_WITH_PARAMS(
 MODULE_PASS_WITH_PARAMS(
     "globaldce", "GlobalDCEPass",
     [](bool InLTOPostLink) { return GlobalDCEPass(InLTOPostLink); },
-    parseGlobalDCEPassOptions, "in-lto-post-link")
+    parseGlobalDCEPassOptions, "vfe-linkage-unit-visibility")
 MODULE_PASS_WITH_PARAMS(
     "hwasan", "HWAddressSanitizerPass",
     [](HWAddressSanitizerOptions Opts) { return HWAddressSanitizerPass(Opts); },
@@ -276,6 +283,13 @@ MODULE_PASS_WITH_PARAMS(
       return buildLTODefaultPipeline(L, nullptr);
     },
     parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "fatlto-pre-link", "", [&](FatLTOOptions Opts) {
+      setupOptionsForPipelineAlias(PTO, Opts.OptLevel);
+      return buildFatLTODefaultPipeline(Opts.OptLevel, Opts.ThinLTO,
+                                        Opts.EmitSummary);
+    },
+    parseFatLTOOptions, "O0;O1;O2;O3;Os;Oz;thinlto;emit-summary")
 
 #undef MODULE_PASS_WITH_PARAMS
 
@@ -324,7 +338,8 @@ CGSCC_PASS_WITH_PARAMS(
 FUNCTION_ANALYSIS("aa", AAManager())
 FUNCTION_ANALYSIS("access-info", LoopAccessAnalysis())
 FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis())
-FUNCTION_ANALYSIS("bb-sections-profile-reader", BasicBlockSectionsProfileReaderAnalysis(TM))
+FUNCTION_ANALYSIS("bb-sections-profile-reader",
+                  BasicBlockSectionsProfileReaderAnalysis(TM))
 FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis())
 FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis())
 FUNCTION_ANALYSIS("cycles", CycleAnalysis())
@@ -414,7 +429,7 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM))
 FUNCTION_PASS("expand-fp", ExpandFpPass(TM))
 FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass(TM))
 FUNCTION_PASS("extra-vector-passes",
-                  ExtraFunctionPassManager<ShouldRunExtraVectorPasses>())
+              ExtraFunctionPassManager<ShouldRunExtraVectorPasses>())
 FUNCTION_PASS("fix-irreducible", FixIrreduciblePass())
 FUNCTION_PASS("flatten-cfg", FlattenCFGPass())
 FUNCTION_PASS("float2int", Float2IntPass())
@@ -581,12 +596,10 @@ FUNCTION_PASS_WITH_PARAMS(
     "instcombine", "InstCombinePass",
     [](InstCombineOptions Opts) { return InstCombinePass(Opts); },
     parseInstCombineOptions,
-    "no-use-loop-info;use-loop-info;no-verify-fixpoint;verify-fixpoint;"
-    "max-iterations=N")
+    "no-verify-fixpoint;verify-fixpoint;max-iterations=N")
 FUNCTION_PASS_WITH_PARAMS(
     "lint", "LintPass",
-    [](bool AbortOnError) { return LintPass(AbortOnError); },
-    parseLintOptions,
+    [](bool AbortOnError) { return LintPass(AbortOnError); }, parseLintOptions,
     "abort-on-error")
 FUNCTION_PASS_WITH_PARAMS(
     "loop-unroll", "LoopUnrollPass",
@@ -604,7 +617,7 @@ FUNCTION_PASS_WITH_PARAMS(
 FUNCTION_PASS_WITH_PARAMS(
     "lower-allow-check", "LowerAllowCheckPass",
     [](LowerAllowCheckPass::Options Opts) { return LowerAllowCheckPass(Opts); },
-    parseLowerAllowCheckPassOptions, "")
+    parseLowerAllowCheckPassOptions, "cutoffs[indices]=N")
 FUNCTION_PASS_WITH_PARAMS(
     "lower-matrix-intrinsics", "LowerMatrixIntrinsicsPass",
     [](bool Minimal) { return LowerMatrixIntrinsicsPass(Minimal); },
@@ -613,7 +626,8 @@ FUNCTION_PASS_WITH_PARAMS(
     "normalize", "IRNormalizerPass",
     [](IRNormalizerOptions Options) { return IRNormalizerPass(Options); },
     parseIRNormalizerPassOptions,
-    "no-preserve-order;preserve-order;no-rename-all;rename-all;no-fold-all;fold-all;no-reorder-operands;reorder-operands")
+    "no-preserve-order;preserve-order;no-rename-all;rename-all;no-fold-all;"
+    "fold-all;no-reorder-operands;reorder-operands")
 FUNCTION_PASS_WITH_PARAMS(
     "mldst-motion", "MergedLoadStoreMotionPass",
     [](MergedLoadStoreMotionOptions Opts) {
@@ -627,7 +641,7 @@ FUNCTION_PASS_WITH_PARAMS(
     },
     [](StringRef Params) {
       return PassBuilder::parseSinglePassOption(Params, "allow-partial",
-						"LoopAccessInfoPrinterPass");
+                                                "LoopAccessInfoPrinterPass");
     },
     "allow-partial")
 FUNCTION_PASS_WITH_PARAMS(
@@ -662,10 +676,14 @@ FUNCTION_PASS_WITH_PARAMS(
     "simplifycfg", "SimplifyCFGPass",
     [](SimplifyCFGOptions Opts) { return SimplifyCFGPass(Opts); },
     parseSimplifyCFGOptions,
-    "no-forward-switch-cond;forward-switch-cond;no-switch-range-to-icmp;"
-    "switch-range-to-icmp;no-switch-to-lookup;switch-to-lookup;no-keep-loops;"
-    "keep-loops;no-hoist-common-insts;hoist-common-insts;no-sink-common-insts;"
-    "sink-common-insts;bonus-inst-threshold=N")
+    "no-speculate-blocks;speculate-blocks;no-simplify-cond-branch;"
+    "simplify-cond-branch;no-forward-switch-cond;forward-switch-cond;"
+    "no-switch-range-to-icmp;switch-range-to-icmp;no-switch-to-lookup;"
+    "switch-to-lookup;no-keep-loops;keep-loops;no-hoist-common-insts;"
+    "hoist-common-insts;no-hoist-loads-stores-with-cond-faulting;"
+    "hoist-loads-stores-with-cond-faulting;no-sink-common-insts;"
+    "sink-common-insts;no-speculate-unpredictables;speculate-unpredictables;"
+    "bonus-inst-threshold=N")
 FUNCTION_PASS_WITH_PARAMS(
     "speculative-execution", "SpeculativeExecutionPass",
     [](bool OnlyIfDivergentTarget) {
@@ -677,11 +695,11 @@ FUNCTION_PASS_WITH_PARAMS(
     [](SROAOptions PreserveCFG) { return SROAPass(PreserveCFG); },
     parseSROAOptions, "preserve-cfg;modify-cfg")
 FUNCTION_PASS_WITH_PARAMS(
-  "structurizecfg", "StructurizeCFG",
-  [](bool SkipUniformRegions) {
-    return StructurizeCFGPass(SkipUniformRegions);
-  },
-  parseStructurizeCFGPassOptions, "skip-uniform-regions")
+    "structurizecfg", "StructurizeCFG",
+    [](bool SkipUniformRegions) {
+      return StructurizeCFGPass(SkipUniformRegions);
+    },
+    parseStructurizeCFGPassOptions, "skip-uniform-regions")
 FUNCTION_PASS_WITH_PARAMS(
     "win-eh-prepare", "WinEHPreparePass",
     [](bool DemoteCatchSwitchPHIOnly) {
@@ -693,7 +711,8 @@ FUNCTION_PASS_WITH_PARAMS(
     [](BoundsCheckingPass::Options Options) {
       return BoundsCheckingPass(Options);
     },
-    parseBoundsCheckingOptions, "trap")
+    parseBoundsCheckingOptions,
+    "trap;rt;rt-abort;min-rt;min-rt-abort;merge;guard=N")
 #undef FUNCTION_PASS_WITH_PARAMS
 
 #ifndef LOOPNEST_PASS
@@ -713,7 +732,7 @@ LOOP_ANALYSIS("iv-users", IVUsersAnalysis())
 LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
 LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
 LOOP_ANALYSIS("should-run-extra-simple-loop-unswitch",
-                  ShouldRunExtraSimpleLoopUnswitch())
+              ShouldRunExtraSimpleLoopUnswitch())
 
 #undef LOOP_ANALYSIS
 
@@ -725,7 +744,7 @@ LOOP_PASS("dot-ddg", DDGDotPrinterPass())
 LOOP_PASS("evl-iv-simplify", EVLIndVarSimplifyPass())
 LOOP_PASS("guard-widening", GuardWideningPass())
 LOOP_PASS("extra-simple-loop-unswitch-passes",
-              ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch>())
+          ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch>())
 LOOP_PASS("indvars", IndVarSimplifyPass())
 LOOP_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 LOOP_PASS("loop-bound-split", LoopBoundSplitPass())
@@ -753,10 +772,10 @@ LOOP_PASS("print<loopnest>", LoopNestPrinterPass(errs()))
 #endif
 LOOP_PASS_WITH_PARAMS(
     "licm", "LICMPass", [](LICMOptions Params) { return LICMPass(Params); },
-    parseLICMOptions, "allowspeculation")
+    parseLICMOptions, "allowspeculation;no-allowspeculation")
 LOOP_PASS_WITH_PARAMS(
     "lnicm", "LNICMPass", [](LICMOptions Params) { return LNICMPass(Params); },
-    parseLICMOptions, "allowspeculation")
+    parseLICMOptions, "allowspeculation;no-allowspeculation")
 LOOP_PASS_WITH_PARAMS(
     "loop-rotate", "LoopRotatePass",
     [](std::pair<bool, bool> Params) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 4e8abc43572d7..0623e66772047 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1074,7 +1074,7 @@ bool OptPassGateInstrumentation::shouldRun(StringRef PassName, Any IR) {
 
 void OptPassGateInstrumentation::registerCallbacks(
     PassInstrumentationCallbacks &PIC) {
-  OptPassGate &PassGate = Context.getOptPassGate();
+  const OptPassGate &PassGate = Context.getOptPassGate();
   if (!PassGate.isEnabled())
     return;
 
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index d26af1c3d5547..c2366dc0ae03e 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_component_library(LLVMProfileData
   Demangle
   Symbolize
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   TargetParser
   )
 
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 9a8b4cfc81f6a..bd3964ce127fd 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -617,28 +617,24 @@ Error readAndDecodeStrings(StringRef NameStrings,
 }
 
 Error InstrProfSymtab::create(StringRef NameStrings) {
-  return readAndDecodeStrings(
-      NameStrings,
-      std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
+  return readAndDecodeStrings(NameStrings,
+                              [&](StringRef S) { return addFuncName(S); });
 }
 
 Error InstrProfSymtab::create(StringRef FuncNameStrings,
                               StringRef VTableNameStrings) {
-  if (Error E = readAndDecodeStrings(FuncNameStrings,
-                                     std::bind(&InstrProfSymtab::addFuncName,
-                                               this, std::placeholders::_1)))
+  if (Error E = readAndDecodeStrings(
+          FuncNameStrings, [&](StringRef S) { return addFuncName(S); }))
     return E;
 
-  return readAndDecodeStrings(
-      VTableNameStrings,
-      std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+  return readAndDecodeStrings(VTableNameStrings,
+                              [&](StringRef S) { return addVTableName(S); });
 }
 
 Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
     StringRef CompressedVTableStrings) {
-  return readAndDecodeStrings(
-      CompressedVTableStrings,
-      std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+  return readAndDecodeStrings(CompressedVTableStrings,
+                              [&](StringRef S) { return addVTableName(S); });
 }
 
 StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
index d92107f93dc56..07b69e9fad3ec 100644
--- a/llvm/lib/ProfileData/InstrProfCorrelator.cpp
+++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
@@ -10,10 +10,10 @@
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
diff --git a/llvm/lib/Support/DataExtractor.cpp b/llvm/lib/Support/DataExtractor.cpp
index 3da9511bf2669..7390093af08a1 100644
--- a/llvm/lib/Support/DataExtractor.cpp
+++ b/llvm/lib/Support/DataExtractor.cpp
@@ -142,13 +142,13 @@ int64_t
 DataExtractor::getSigned(uint64_t *offset_ptr, uint32_t byte_size) const {
   switch (byte_size) {
   case 1:
-    return (int8_t)getU8(offset_ptr);
+    return getS8(offset_ptr);
   case 2:
-    return (int16_t)getU16(offset_ptr);
+    return getS16(offset_ptr);
   case 4:
-    return (int32_t)getU32(offset_ptr);
+    return getS32(offset_ptr);
   case 8:
-    return (int64_t)getU64(offset_ptr);
+    return getS64(offset_ptr);
   }
   llvm_unreachable("getSigned unhandled case!");
 }
diff --git a/llvm/lib/Support/JSON.cpp b/llvm/lib/Support/JSON.cpp
index e7b762f550d46..d8662340cb3ed 100644
--- a/llvm/lib/Support/JSON.cpp
+++ b/llvm/lib/Support/JSON.cpp
@@ -516,7 +516,7 @@ bool Parser::parseNumber(char First, Value &Out) {
   errno = 0;
   int64_t I = std::strtoll(S.c_str(), &End, 10);
   if (End == S.end() && errno != ERANGE) {
-    Out = int64_t(I);
+    Out = I;
     return true;
   }
   // strtroull has a special handling for negative numbers, but in this
diff --git a/llvm/lib/Support/Signals.cpp b/llvm/lib/Support/Signals.cpp
index 9f9030e79d104..f8a14a45ddc3e 100644
--- a/llvm/lib/Support/Signals.cpp
+++ b/llvm/lib/Support/Signals.cpp
@@ -31,7 +31,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include <array>
 #include <cmath>
-#include <vector>
 
 //===----------------------------------------------------------------------===//
 //=== WARNING: Implementation here must contain only TRULY operating system
@@ -137,47 +136,28 @@ static FormattedNumber format_ptr(void *PC) {
   return format_hex((uint64_t)PC, PtrWidth);
 }
 
-/// Helper that launches llvm-symbolizer and symbolizes a backtrace.
-LLVM_ATTRIBUTE_USED
-static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
-                                      int Depth, llvm::raw_ostream &OS) {
-  if (DisableSymbolicationFlag || getenv(DisableSymbolizationEnv))
-    return false;
-
-  // Don't recursively invoke the llvm-symbolizer binary.
-  if (Argv0.contains("llvm-symbolizer"))
-    return false;
-
-  // FIXME: Subtract necessary number from StackTrace entries to turn return addresses
-  // into actual instruction addresses.
-  // Use llvm-symbolizer tool to symbolize the stack traces. First look for it
-  // alongside our binary, then in $PATH.
-  ErrorOr<std::string> LLVMSymbolizerPathOrErr = std::error_code();
-  if (const char *Path = getenv(LLVMSymbolizerPathEnv)) {
-    LLVMSymbolizerPathOrErr = sys::findProgramByName(Path);
-  } else if (!Argv0.empty()) {
-    StringRef Parent = llvm::sys::path::parent_path(Argv0);
-    if (!Parent.empty())
-      LLVMSymbolizerPathOrErr = sys::findProgramByName("llvm-symbolizer", Parent);
-  }
-  if (!LLVMSymbolizerPathOrErr)
-    LLVMSymbolizerPathOrErr = sys::findProgramByName("llvm-symbolizer");
-  if (!LLVMSymbolizerPathOrErr)
-    return false;
-  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
-
-  // If we don't know argv0 or the address of main() at this point, try
-  // to guess it anyway (it's possible on some platforms).
-  std::string MainExecutableName =
-      sys::fs::exists(Argv0) ? (std::string)std::string(Argv0)
-                             : sys::fs::getMainExecutable(nullptr, nullptr);
+/// Reads a file \p Filename written by llvm-symbolizer containing function
+/// names and source locations for the addresses in \p AddressList and returns
+/// the strings in a vector of pairs, where the first pair element is the index
+/// of the corresponding entry in AddressList and the second is the symbolized
+/// frame, in a format based on the sanitizer stack trace printer, with the
+/// exception that it does not write out frame numbers (i.e. "#2 " for the
+/// third address), as it is not assumed that \p AddressList corresponds to a
+/// single stack trace.
+/// There may be multiple returned entries for a single \p AddressList entry if
+/// that frame address corresponds to one or more inlined frames; in this case,
+/// all frames for an address will appear contiguously and in-order.
+std::optional<SmallVector<std::pair<unsigned, std::string>, 0>>
+collectAddressSymbols(void **AddressList, unsigned AddressCount,
+                      const char *MainExecutableName,
+                      const std::string &LLVMSymbolizerPath) {
   BumpPtrAllocator Allocator;
   StringSaver StrPool(Allocator);
-  std::vector<const char *> Modules(Depth, nullptr);
-  std::vector<intptr_t> Offsets(Depth, 0);
-  if (!findModulesAndOffsets(StackTrace, Depth, Modules.data(), Offsets.data(),
-                             MainExecutableName.c_str(), StrPool))
-    return false;
+  SmallVector<const char *, 0> Modules(AddressCount, nullptr);
+  SmallVector<intptr_t, 0> Offsets(AddressCount, 0);
+  if (!findModulesAndOffsets(AddressList, AddressCount, Modules.data(),
+                             Offsets.data(), MainExecutableName, StrPool))
+    return {};
   int InputFD;
   SmallString<32> InputFile, OutputFile;
   sys::fs::createTemporaryFile("symbolizer-input", "", InputFD, InputFile);
@@ -187,9 +167,9 @@ static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
 
   {
     raw_fd_ostream Input(InputFD, true);
-    for (int i = 0; i < Depth; i++) {
-      if (Modules[i])
-        Input << Modules[i] << " " << (void*)Offsets[i] << "\n";
+    for (unsigned AddrIdx = 0; AddrIdx < AddressCount; AddrIdx++) {
+      if (Modules[AddrIdx])
+        Input << Modules[AddrIdx] << " " << (void *)Offsets[AddrIdx] << "\n";
     }
   }
 
@@ -206,53 +186,149 @@ static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
   int RunResult =
       sys::ExecuteAndWait(LLVMSymbolizerPath, Args, std::nullopt, Redirects);
   if (RunResult != 0)
-    return false;
+    return {};
 
-  // This report format is based on the sanitizer stack trace printer.  See
-  // sanitizer_stacktrace_printer.cc in compiler-rt.
+  SmallVector<std::pair<unsigned, std::string>, 0> Result;
   auto OutputBuf = MemoryBuffer::getFile(OutputFile.c_str());
   if (!OutputBuf)
-    return false;
+    return {};
   StringRef Output = OutputBuf.get()->getBuffer();
   SmallVector<StringRef, 32> Lines;
   Output.split(Lines, "\n");
-  auto CurLine = Lines.begin();
-  int frame_no = 0;
-  for (int i = 0; i < Depth; i++) {
-    auto PrintLineHeader = [&]() {
-      OS << right_justify(formatv("#{0}", frame_no++).str(),
-                          std::log10(Depth) + 2)
-         << ' ' << format_ptr(StackTrace[i]) << ' ';
-    };
-    if (!Modules[i]) {
-      PrintLineHeader();
-      OS << '\n';
+  auto *CurLine = Lines.begin();
+  // Lines contains the output from llvm-symbolizer, which should contain for
+  // each address with a module in order of appearance, one or more lines
+  // containing the function name and line associated with that address,
+  // followed by an empty line.
+  // For each address, adds an output entry for every real or inlined frame at
+  // that address. For addresses without known modules, we have a single entry
+  // containing just the formatted address; for all other output entries, we
+  // output the function entry if it is known, and either the line number if it
+  // is known or the module+address offset otherwise.
+  for (unsigned AddrIdx = 0; AddrIdx < AddressCount; AddrIdx++) {
+    if (!Modules[AddrIdx]) {
+      auto &SymbolizedFrame = Result.emplace_back(std::make_pair(AddrIdx, ""));
+      raw_string_ostream OS(SymbolizedFrame.second);
+      OS << format_ptr(AddressList[AddrIdx]);
       continue;
     }
     // Read pairs of lines (function name and file/line info) until we
     // encounter empty line.
     for (;;) {
       if (CurLine == Lines.end())
-        return false;
+        return {};
       StringRef FunctionName = *CurLine++;
       if (FunctionName.empty())
         break;
-      PrintLineHeader();
+      auto &SymbolizedFrame = Result.emplace_back(std::make_pair(AddrIdx, ""));
+      raw_string_ostream OS(SymbolizedFrame.second);
+      OS << format_ptr(AddressList[AddrIdx]) << ' ';
       if (!FunctionName.starts_with("??"))
         OS << FunctionName << ' ';
       if (CurLine == Lines.end())
-        return false;
+        return {};
       StringRef FileLineInfo = *CurLine++;
-      if (!FileLineInfo.starts_with("??"))
+      if (!FileLineInfo.starts_with("??")) {
         OS << FileLineInfo;
-      else
-        OS << "(" << Modules[i] << '+' << format_hex(Offsets[i], 0) << ")";
-      OS << "\n";
+      } else {
+        OS << "(" << Modules[AddrIdx] << '+' << format_hex(Offsets[AddrIdx], 0)
+           << ")";
+      }
     }
   }
+  return Result;
+}
+
+ErrorOr<std::string> getLLVMSymbolizerPath(StringRef Argv0 = {}) {
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr = std::error_code();
+  if (const char *Path = getenv(LLVMSymbolizerPathEnv)) {
+    LLVMSymbolizerPathOrErr = sys::findProgramByName(Path);
+  } else if (!Argv0.empty()) {
+    StringRef Parent = llvm::sys::path::parent_path(Argv0);
+    if (!Parent.empty())
+      LLVMSymbolizerPathOrErr =
+          sys::findProgramByName("llvm-symbolizer", Parent);
+  }
+  if (!LLVMSymbolizerPathOrErr)
+    LLVMSymbolizerPathOrErr = sys::findProgramByName("llvm-symbolizer");
+  return LLVMSymbolizerPathOrErr;
+}
+
+/// Helper that launches llvm-symbolizer and symbolizes a backtrace.
+LLVM_ATTRIBUTE_USED
+static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
+                                      int Depth, llvm::raw_ostream &OS) {
+  if (DisableSymbolicationFlag || getenv(DisableSymbolizationEnv))
+    return false;
+
+  // Don't recursively invoke the llvm-symbolizer binary.
+  if (Argv0.contains("llvm-symbolizer"))
+    return false;
+
+  // FIXME: Subtract necessary number from StackTrace entries to turn return
+  // addresses into actual instruction addresses.
+  // Use llvm-symbolizer tool to symbolize the stack traces. First look for it
+  // alongside our binary, then in $PATH.
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr = getLLVMSymbolizerPath(Argv0);
+  if (!LLVMSymbolizerPathOrErr)
+    return false;
+  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
+
+  // If we don't know argv0 or the address of main() at this point, try
+  // to guess it anyway (it's possible on some platforms).
+  std::string MainExecutableName =
+      sys::fs::exists(Argv0) ? std::string(Argv0)
+                             : sys::fs::getMainExecutable(nullptr, nullptr);
+
+  auto SymbolizedAddressesOpt = collectAddressSymbols(
+      StackTrace, Depth, MainExecutableName.c_str(), LLVMSymbolizerPath);
+  if (!SymbolizedAddressesOpt)
+    return false;
+  for (unsigned FrameNo = 0; FrameNo < SymbolizedAddressesOpt->size();
+       ++FrameNo) {
+    OS << right_justify(formatv("#{0}", FrameNo).str(), std::log10(Depth) + 2)
+       << ' ' << (*SymbolizedAddressesOpt)[FrameNo].second << '\n';
+  }
   return true;
 }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+void sys::symbolizeAddresses(AddressSet &Addresses,
+                             SymbolizedAddressMap &SymbolizedAddresses) {
+  assert(!DisableSymbolicationFlag && !getenv(DisableSymbolizationEnv) &&
+         "Debugify origin stacktraces require symbolization to be enabled.");
+
+  // Convert Set of Addresses to ordered list.
+  SmallVector<void *, 0> AddressList(Addresses.begin(), Addresses.end());
+  if (AddressList.empty())
+    return;
+  llvm::sort(AddressList);
+
+  // Use llvm-symbolizer tool to symbolize the stack traces. First look for it
+  // alongside our binary, then in $PATH.
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr = getLLVMSymbolizerPath();
+  if (!LLVMSymbolizerPathOrErr)
+    report_fatal_error("Debugify origin stacktraces require llvm-symbolizer");
+  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
+
+  // Try to guess the main executable name, since we don't have argv0 available
+  // here.
+  std::string MainExecutableName = sys::fs::getMainExecutable(nullptr, nullptr);
+
+  auto SymbolizedAddressesOpt =
+      collectAddressSymbols(AddressList.begin(), AddressList.size(),
+                            MainExecutableName.c_str(), LLVMSymbolizerPath);
+  if (!SymbolizedAddressesOpt)
+    return;
+  for (auto SymbolizedFrame : *SymbolizedAddressesOpt) {
+    SmallVector<std::string, 0> &SymbolizedAddrs =
+        SymbolizedAddresses[AddressList[SymbolizedFrame.first]];
+    SymbolizedAddrs.push_back(SymbolizedFrame.second);
+  }
+  return;
+}
+#endif
+
 static bool printMarkupContext(raw_ostream &OS, const char *MainExecutableName);
 
 LLVM_ATTRIBUTE_USED
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index b675966c8d822..b4ee0f8ee8bfd 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -271,7 +271,7 @@ TextEncodingConverterIconv::convertString(StringRef Source,
     // Setup the input. Use nullptr to reset iconv state if input length is
     // zero.
     size_t InputLength = Source.size();
-    char *Input = InputLength ? const_cast<char *>(Source.data()) : "";
+    char *Input = const_cast<char *>(InputLength ? Source.data() : "");
     Ret = iconv(ConvDesc, &Input, &InputLength, &Output, &OutputLength);
     if (Ret != 0) {
       if (auto EC = HandleError(Ret))
diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc
index 6668a2953b3b2..a4525a5903649 100644
--- a/llvm/lib/Support/Unix/Signals.inc
+++ b/llvm/lib/Support/Unix/Signals.inc
@@ -507,6 +507,21 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
   return 0;
 }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#if !defined(HAVE_BACKTRACE)
+#error DebugLoc origin-tracking currently requires `backtrace()`.
+#endif
+namespace llvm {
+namespace sys {
+template <unsigned long MaxDepth>
+int getStackTrace(std::array<void *, MaxDepth> &StackTrace) {
+  return backtrace(StackTrace.data(), MaxDepth);
+}
+template int getStackTrace<16ul>(std::array<void *, 16ul> &);
+} // namespace sys
+} // namespace llvm
+#endif
+
 /// If this is an ELF platform, we can find all loaded modules and their virtual
 /// addresses with dl_iterate_phdr.
 static bool findModulesAndOffsets(void **StackTrace, int Depth,
diff --git a/llvm/lib/Support/Windows/Signals.inc b/llvm/lib/Support/Windows/Signals.inc
index 959caa22f83ad..dad0fa3066868 100644
--- a/llvm/lib/Support/Windows/Signals.inc
+++ b/llvm/lib/Support/Windows/Signals.inc
@@ -9,6 +9,7 @@
 // This file provides the Win32 specific implementation of the Signals class.
 //
 //===----------------------------------------------------------------------===//
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/ExitCodes.h"
 #include "llvm/Support/FileSystem.h"
@@ -478,6 +479,10 @@ void sys::PrintStackTraceOnErrorSignal(StringRef Argv0,
 }
 } // namespace llvm
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#error DebugLoc origin-tracking currently unimplemented for Windows.
+#endif
+
 static void LocalPrintStackTrace(raw_ostream &OS, PCONTEXT C) {
   STACKFRAME64 StackFrame{};
   CONTEXT Context{};
diff --git a/llvm/lib/Support/Z3Solver.cpp b/llvm/lib/Support/Z3Solver.cpp
index 27027093a0c6f..056a17ad80123 100644
--- a/llvm/lib/Support/Z3Solver.cpp
+++ b/llvm/lib/Support/Z3Solver.cpp
@@ -932,7 +932,8 @@ class Z3Statistics final : public SMTSolverStatistics {
   };
   unsigned getUnsigned(StringRef Key) const override {
     auto It = UnsignedValues.find(Key.str());
-    assert(It != UnsignedValues.end());
+    if (It == UnsignedValues.end())
+      return 0;
     return It->second;
   };
 
diff --git a/llvm/lib/TableGen/DetailedRecordsBackend.cpp b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
index cf697599e53a2..1ed64356b7c62 100644
--- a/llvm/lib/TableGen/DetailedRecordsBackend.cpp
+++ b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
@@ -1,4 +1,4 @@
-//===- DetailedRecordBackend.cpp - Detailed Records Report      -*- C++ -*-===//
+//===- DetailedRecordBackend.cpp - Detailed Records Report ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This Tablegen backend prints a report that includes all the global 
+// This Tablegen backend prints a report that includes all the global
 // variables, classes, and records in complete detail. It includes more
 // detail than the default TableGen printer backend.
 //
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index 91423664a84cc..de0c4c971efeb 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -24,7 +24,7 @@ namespace llvm {
 SourceMgr SrcMgr;
 unsigned ErrorsPrinted = 0;
 
-static void PrintMessage(ArrayRef<SMLoc> Loc, SourceMgr::DiagKind Kind,
+static void PrintMessage(ArrayRef<SMLoc> Locs, SourceMgr::DiagKind Kind,
                          const Twine &Msg) {
   // Count the total number of errors printed.
   // This is used to exit with an error code if there were any errors.
@@ -32,11 +32,11 @@ static void PrintMessage(ArrayRef<SMLoc> Loc, SourceMgr::DiagKind Kind,
     ++ErrorsPrinted;
 
   SMLoc NullLoc;
-  if (Loc.empty())
-    Loc = NullLoc;
-  SrcMgr.PrintMessage(Loc.front(), Kind, Msg);
-  for (unsigned i = 1; i < Loc.size(); ++i)
-    SrcMgr.PrintMessage(Loc[i], SourceMgr::DK_Note,
+  if (Locs.empty())
+    Locs = NullLoc;
+  SrcMgr.PrintMessage(Locs.consume_front(), Kind, Msg);
+  for (SMLoc Loc : Locs)
+    SrcMgr.PrintMessage(Loc, SourceMgr::DK_Note,
                         "instantiated from multiclass");
 }
 
diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp
index ea716215e0679..f545706d6fe30 100644
--- a/llvm/lib/TableGen/Main.cpp
+++ b/llvm/lib/TableGen/Main.cpp
@@ -8,7 +8,7 @@
 //
 // TableGen is a tool which can be used to build up a description of something,
 // then invoke one or more "tablegen backends" to emit information about the
-// description in some predefined format.  In practice, this is used by the LLVM
+// description in some predefined format. In practice, this is used by the LLVM
 // code generators to automate generation of a code generator through a
 // high-level description of the target.
 //
@@ -156,7 +156,7 @@ int llvm::TableGenMain(const char *argv0,
     return 1;
 
   // Always write the depfile, even if the main output hasn't changed.
-  // If it's missing, Ninja considers the output dirty.  If this was below
+  // If it's missing, Ninja considers the output dirty. If this was below
   // the early exit below and someone deleted the .inc.d file but not the .inc
   // file, tablegen would never write the depfile.
   if (!DependFilename.empty()) {
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 7f2ed77a74099..1f3e5dc68f1d6 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -500,7 +500,7 @@ const Init *BitsInit::convertInitializerTo(const RecTy *Ty) const {
   }
 
   if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
-    // If the number of bits is right, return it.  Otherwise we need to expand
+    // If the number of bits is right, return it. Otherwise we need to expand
     // or truncate.
     if (getNumBits() != BRT->getNumBits()) return nullptr;
     return this;
@@ -944,7 +944,7 @@ const Init *UnOpInit::Fold(const Record *CurRec, bool IsFinal) const {
   case TAIL:
     if (const auto *LHSl = dyn_cast<ListInit>(LHS)) {
       assert(!LHSl->empty() && "Empty list in tail");
-      // Note the slice(1).  We can't just pass the result of getElements()
+      // Note the slice(1). We can't just pass the result of getElements()
       // directly.
       return ListInit::get(LHSl->getElements().slice(1),
                            LHSl->getElementType());
@@ -2921,16 +2921,16 @@ void Record::setName(const Init *NewName) {
   Name = NewName;
   checkName();
   // DO NOT resolve record values to the name at this point because
-  // there might be default values for arguments of this def.  Those
+  // there might be default values for arguments of this def. Those
   // arguments might not have been resolved yet so we don't want to
   // prematurely assume values for those arguments were not passed to
   // this def.
   //
   // Nonetheless, it may be that some of this Record's values
-  // reference the record name.  Indeed, the reason for having the
-  // record name be an Init is to provide this flexibility.  The extra
+  // reference the record name. Indeed, the reason for having the
+  // record name be an Init is to provide this flexibility. The extra
   // resolve steps after completely instantiating defs takes care of
-  // this.  See TGParser::ParseDef and TGParser::ParseDefm.
+  // this. See TGParser::ParseDef and TGParser::ParseDefm.
 }
 
 void Record::resolveReferences(Resolver &R, const RecordVal *SkipVal) {
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index 46487cba9453e..c8e020d791e09 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -102,7 +102,7 @@ SMRange TGLexer::getLocRange() const {
 }
 
 /// ReturnError - Set the error to the specified string at the specified
-/// location.  This is defined to always return tgtok::Error.
+/// location. This is defined to always return tgtok::Error.
 tgtok::TokKind TGLexer::ReturnError(SMLoc Loc, const Twine &Msg) {
   PrintError(Loc, Msg);
   return tgtok::Error;
@@ -116,7 +116,7 @@ bool TGLexer::processEOF() {
   SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
   if (ParentIncludeLoc != SMLoc()) {
     // If prepExitInclude() detects a problem with the preprocessing
-    // control stack, it will return false.  Pretend that we reached
+    // control stack, it will return false. Pretend that we reached
     // the final EOF and stop lexing more tokens by returning false
     // to LexToken().
     if (!prepExitInclude(false))
@@ -147,7 +147,7 @@ int TGLexer::getNextChar() {
 
   case 0: {
     // A NUL character in the stream is either the end of the current buffer or
-    // a spurious NUL in the file.  Disambiguate that here.
+    // a spurious NUL in the file. Disambiguate that here.
     if (CurPtr - 1 == CurBuf.end()) {
       --CurPtr; // Arrange for another call to return EOF again.
       return EOF;
@@ -160,7 +160,7 @@ int TGLexer::getNextChar() {
   case '\n':
   case '\r':
     // Handle the newline character by ignoring it and incrementing the line
-    // count.  However, be careful about 'dos style' files with \n\r in them.
+    // count. However, be careful about 'dos style' files with \n\r in them.
     // Only treat a \n\r or \r\n as a single line.
     if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
         *CurPtr != CurChar)
@@ -259,7 +259,7 @@ tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
     int NextChar = 0;
     if (isDigit(CurChar)) {
       // Allow identifiers to start with a number if it is followed by
-      // an identifier.  This can happen with paste operations like
+      // an identifier. This can happen with paste operations like
       // foo#8i.
       int i = 0;
       do {
@@ -422,7 +422,7 @@ tgtok::TokKind TGLexer::LexIdentifier() {
   return Kind;
 }
 
-/// LexInclude - We just read the "include" token.  Get the string token that
+/// LexInclude - We just read the "include" token. Get the string token that
 /// comes next and enter the include.
 bool TGLexer::LexInclude() {
   // The token after the include must be a string.
@@ -461,7 +461,7 @@ void TGLexer::SkipBCPLComment() {
   CurPtr = (EOLPos == StringRef::npos) ? CurBuf.end() : CurBuf.data() + EOLPos;
 }
 
-/// SkipCComment - This skips C-style /**/ comments.  The only difference from C
+/// SkipCComment - This skips C-style /**/ comments. The only difference from C
 /// is that we allow nesting.
 bool TGLexer::SkipCComment() {
   ++CurPtr;  // skip the star.
@@ -554,8 +554,8 @@ tgtok::TokKind TGLexer::LexNumber() {
   return Base == 2 ? tgtok::BinaryIntVal : tgtok::IntVal;
 }
 
-/// LexBracket - We just read '['.  If this is a code block, return it,
-/// otherwise return the bracket.  Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
+/// LexBracket - We just read '['. If this is a code block, return it,
+/// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
 tgtok::TokKind TGLexer::LexBracket() {
   if (CurPtr[0] != '{')
     return tgtok::l_square;
@@ -687,7 +687,7 @@ tgtok::TokKind TGLexer::prepIsDirective() const {
         NextChar == '\n' ||
         // It looks like TableGen does not support '\r' as the actual
         // carriage return, e.g. getNextChar() treats a single '\r'
-        // as '\n'.  So we do the same here.
+        // as '\n'. So we do the same here.
         NextChar == '\r')
       return Kind;
 
@@ -700,7 +700,7 @@ tgtok::TokKind TGLexer::prepIsDirective() const {
     //     #define/**/ AND #define//
     //
     // These cases will be reported as incorrect after calling
-    // prepLexMacroName().  We could have supported C-style comments
+    // prepLexMacroName(). We could have supported C-style comments
     // after #ifdef/#define, but this would complicate the code
     // for little benefit.
     if (NextChar == '/') {
@@ -733,7 +733,7 @@ void TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
 
 tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
                                         bool ReturnNextLiveToken) {
-  // We must be looking at a preprocessing directive.  Eat it!
+  // We must be looking at a preprocessing directive. Eat it!
   prepEatPreprocessorDirective(Kind);
 
   if (Kind == tgtok::Ifdef || Kind == tgtok::Ifndef) {
@@ -879,7 +879,7 @@ bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
     tgtok::TokKind Kind = prepIsDirective();
 
     // If we did not find a preprocessing directive or it is #define,
-    // then just skip to the next line.  We do not have to do anything
+    // then just skip to the next line. We do not have to do anything
     // for #define in the line-skipping mode.
     if (Kind == tgtok::Error || Kind == tgtok::Define)
       continue;
@@ -909,7 +909,7 @@ bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
   } while (CurPtr != CurBuf.end());
 
   // We have reached the end of the file, but never left the lines-skipping
-  // mode.  This means there is no matching #endif.
+  // mode. This means there is no matching #endif.
   prepReportPreprocessorStackError();
   return false;
 }
@@ -939,8 +939,8 @@ bool TGLexer::prepSkipLineBegin() {
         // Skip C-style comment.
         // Note that we do not care about skipping the C++-style comments.
         // If the line contains "//", it may not contain any processable
-        // preprocessing directive.  Just return CurPtr pointing to
-        // the first '/' in this case.  We also do not care about
+        // preprocessing directive. Just return CurPtr pointing to
+        // the first '/' in this case. We also do not care about
         // incorrect symbols after the first '/' - we are in lines-skipping
         // mode, so incorrect code is allowed to some extent.
 
@@ -968,7 +968,7 @@ bool TGLexer::prepSkipLineBegin() {
     ++CurPtr;
   }
 
-  // We have reached the end of the file.  Return to the lines skipping
+  // We have reached the end of the file. Return to the lines skipping
   // code, and allow it to handle the EOF as needed.
   return true;
 }
@@ -994,7 +994,7 @@ bool TGLexer::prepSkipDirectiveEnd() {
         SkipBCPLComment();
       } else if (NextChar == '*') {
         // When we are skipping C-style comment at the end of a preprocessing
-        // directive, we can skip several lines.  If any meaningful TD token
+        // directive, we can skip several lines. If any meaningful TD token
         // follows the end of the C-style comment on the same line, it will
         // be considered as an invalid usage of TD token.
         // For example, we want to forbid usages like this one:
@@ -1003,7 +1003,7 @@ bool TGLexer::prepSkipDirectiveEnd() {
         //     #define MACRO /* This macro is used
         //                      to ... */ class Class {}
         // One can argue that this should be allowed, but it does not seem
-        // to be worth of the complication.  Moreover, this matches
+        // to be worth of the complication. Moreover, this matches
         // the C preprocessor behavior.
 
         // Set TokStart to the beginning of the comment to enable proper
diff --git a/llvm/lib/TableGen/TGLexer.h b/llvm/lib/TableGen/TGLexer.h
index a5b10b37e9886..ed7d8f3baae59 100644
--- a/llvm/lib/TableGen/TGLexer.h
+++ b/llvm/lib/TableGen/TGLexer.h
@@ -61,7 +61,7 @@ enum TokKind {
   // Integer value.
   IntVal,
 
-  // Binary constant.  Note that these are sized according to the number of
+  // Binary constant. Note that these are sized according to the number of
   // bits given.
   BinaryIntVal,
 
@@ -280,7 +280,7 @@ class TGLexer {
   //
   // An ordered list of preprocessing controls defined by #ifdef/#else
   // directives that are in effect currently is called preprocessing
-  // control stack.  It is represented as a vector of PreprocessorControlDesc's.
+  // control stack. It is represented as a vector of PreprocessorControlDesc's.
   //
   // The control stack is updated according to the following rules:
   //
@@ -321,9 +321,9 @@ class TGLexer {
   //     EOF
   //
   // To do this, we clear the preprocessing control stack on entry
-  // to each of the included file.  PrepIncludeStack is used to store
+  // to each of the included file. PrepIncludeStack is used to store
   // preprocessing control stacks for the current file and all its
-  // parent files.  The back() element is the preprocessing control
+  // parent files. The back() element is the preprocessing control
   // stack for the current file.
   SmallVector<SmallVector<PreprocessorControlDesc>> PrepIncludeStack;
 
@@ -332,7 +332,7 @@ class TGLexer {
   //
   // If IncludeStackMustBeEmpty is true, the include stack must be empty
   // after the popping, otherwise, the include stack must not be empty
-  // after the popping.  Basically, the include stack must be empty
+  // after the popping. Basically, the include stack must be empty
   // only if we exit the "top-level" file (i.e. finish lexing).
   //
   // The method returns false, if the current preprocessing control stack
@@ -340,8 +340,8 @@ class TGLexer {
   // true - otherwise.
   bool prepExitInclude(bool IncludeStackMustBeEmpty);
 
-  // Look ahead for a preprocessing directive starting from CurPtr.  The caller
-  // must only call this method, if *(CurPtr - 1) is '#'.  If the method matches
+  // Look ahead for a preprocessing directive starting from CurPtr. The caller
+  // must only call this method, if *(CurPtr - 1) is '#'. If the method matches
   // a preprocessing directive word followed by a whitespace, then it returns
   // one of the internal token kinds, i.e. Ifdef, Else, Endif, Define.
   //
@@ -353,26 +353,26 @@ class TGLexer {
   //
   // We use look-ahead prepIsDirective() and prepEatPreprocessorDirective()
   // to avoid adjusting CurPtr before we are sure that '#' is followed
-  // by a preprocessing directive.  If it is not, then we fall back to
+  // by a preprocessing directive. If it is not, then we fall back to
   // tgtok::paste interpretation of '#'.
   void prepEatPreprocessorDirective(tgtok::TokKind Kind);
 
   // The main "exit" point from the token parsing to preprocessor.
   //
   // The method is called for CurPtr, when prepIsDirective() returns
-  // true.  The first parameter matches the result of prepIsDirective(),
+  // true. The first parameter matches the result of prepIsDirective(),
   // denoting the actual preprocessor directive to be processed.
   //
   // If the preprocessing directive disables the tokens processing, e.g.:
   //     #ifdef NAME // NAME is undefined
   // then lexPreprocessor() enters the lines-skipping mode.
   // In this mode, it does not parse any tokens, because the code under
-  // the #ifdef may not even be a correct tablegen code.  The preprocessor
+  // the #ifdef may not even be a correct tablegen code. The preprocessor
   // looks for lines containing other preprocessing directives, which
-  // may be prepended with whitespaces and C-style comments.  If the line
+  // may be prepended with whitespaces and C-style comments. If the line
   // does not contain a preprocessing directive, it is skipped completely.
   // Otherwise, the preprocessing directive is processed by recursively
-  // calling lexPreprocessor().  The processing of the encountered
+  // calling lexPreprocessor(). The processing of the encountered
   // preprocessing directives includes updating preprocessing control stack
   // and adding new macros into DefinedMacros set.
   //
@@ -390,38 +390,38 @@ class TGLexer {
 
   // Worker method for lexPreprocessor() to skip lines after some
   // preprocessing directive up to the buffer end or to the directive
-  // that re-enables token processing.  The method returns true
+  // that re-enables token processing. The method returns true
   // upon processing the next directive that re-enables tokens
-  // processing.  False is returned if an error was encountered.
+  // processing. False is returned if an error was encountered.
   //
   // Note that prepSkipRegion() calls lexPreprocessor() to process
-  // encountered preprocessing directives.  In this case, the second
-  // parameter to lexPreprocessor() is set to false.  Being passed
+  // encountered preprocessing directives. In this case, the second
+  // parameter to lexPreprocessor() is set to false. Being passed
   // false ReturnNextLiveToken, lexPreprocessor() must never call
-  // prepSkipRegion().  We assert this by passing ReturnNextLiveToken
+  // prepSkipRegion(). We assert this by passing ReturnNextLiveToken
   // to prepSkipRegion() and checking that it is never set to false.
   bool prepSkipRegion(bool MustNeverBeFalse);
 
-  // Lex name of the macro after either #ifdef or #define.  We could have used
+  // Lex name of the macro after either #ifdef or #define. We could have used
   // LexIdentifier(), but it has special handling of "include" word, which
-  // could result in awkward diagnostic errors.  Consider:
+  // could result in awkward diagnostic errors. Consider:
   // ----
   // #ifdef include
   // class ...
   // ----
   // LexIdentifier() will engage LexInclude(), which will complain about
-  // missing file with name "class".  Instead, prepLexMacroName() will treat
+  // missing file with name "class". Instead, prepLexMacroName() will treat
   // "include" as a normal macro name.
   //
   // On entry, CurPtr points to the end of a preprocessing directive word.
   // The method allows for whitespaces between the preprocessing directive
-  // and the macro name.  The allowed whitespaces are ' ' and '\t'.
+  // and the macro name. The allowed whitespaces are ' ' and '\t'.
   //
   // If the first non-whitespace symbol after the preprocessing directive
   // is a valid start symbol for an identifier (i.e. [a-zA-Z_]), then
   // the method updates TokStart to the position of the first non-whitespace
   // symbol, sets CurPtr to the position of the macro name's last symbol,
-  // and returns a string reference to the macro name.  Otherwise,
+  // and returns a string reference to the macro name. Otherwise,
   // TokStart is set to the first non-whitespace symbol after the preprocessing
   // directive, and the method returns an empty string reference.
   //
@@ -429,10 +429,10 @@ class TGLexer {
   // the preprocessing directive.
   StringRef prepLexMacroName();
 
-  // Skip any whitespaces starting from CurPtr.  The method is used
+  // Skip any whitespaces starting from CurPtr. The method is used
   // only in the lines-skipping mode to find the first non-whitespace
-  // symbol after or at CurPtr.  Allowed whitespaces are ' ', '\t', '\n'
-  // and '\r'.  The method skips C-style comments as well, because
+  // symbol after or at CurPtr. Allowed whitespaces are ' ', '\t', '\n'
+  // and '\r'. The method skips C-style comments as well, because
   // it is used to find the beginning of the preprocessing directive.
   // If we do not handle C-style comments the following code would
   // result in incorrect detection of a preprocessing directive:
@@ -445,13 +445,13 @@ class TGLexer {
   //        second line comment */ #ifdef NAME
   //
   // The method returns true upon reaching the first non-whitespace symbol
-  // or EOF, CurPtr is set to point to this symbol.  The method returns false,
+  // or EOF, CurPtr is set to point to this symbol. The method returns false,
   // if an error occurred during skipping of a C-style comment.
   bool prepSkipLineBegin();
 
   // Skip any whitespaces or comments after a preprocessing directive.
   // The method returns true upon reaching either end of the line
-  // or end of the file.  If there is a multiline C-style comment
+  // or end of the file. If there is a multiline C-style comment
   // after the preprocessing directive, the method skips
   // the comment, so the final CurPtr may point to one of the next lines.
   // The method returns false, if an error occurred during skipping
@@ -459,7 +459,7 @@ class TGLexer {
   // after the preprocessing directive.
   //
   // The method maybe called both during lines-skipping and tokens
-  // processing.  It actually verifies that only whitespaces or/and
+  // processing. It actually verifies that only whitespaces or/and
   // comments follow a preprocessing directive.
   //
   // After the execution of this mehod, CurPtr points either to new line
@@ -475,7 +475,7 @@ class TGLexer {
   bool prepIsProcessingEnabled();
 
   // Report an error, if we reach EOF with non-empty preprocessing control
-  // stack.  This means there is no matching #endif for the previous
+  // stack. This means there is no matching #endif for the previous
   // #ifdef/#else.
   void prepReportPreprocessorStackError();
 };
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 87a1fb64943c4..62c5355654149 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -240,17 +240,15 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, const Init *ValName,
     return Error(Loc, "Value '" + ValName->getAsUnquotedString() +
                  "' unknown!");
 
-  // Do not allow assignments like 'X = X'.  This will just cause infinite loops
+  // Do not allow assignments like 'X = X'. This will just cause infinite loops
   // in the resolution machinery.
   if (BitList.empty())
     if (const auto *VI = dyn_cast<VarInit>(V))
       if (VI->getNameInit() == ValName && !AllowSelfAssignment)
         return Error(Loc, "Recursion / self-assignment forbidden");
 
-  // If we are assigning to a subset of the bits in the value... then we must be
-  // assigning to a field of BitsRecTy, which must have a BitsInit
-  // initializer.
-  //
+  // If we are assigning to a subset of the bits in the value we must be
+  // assigning to a field of BitsRecTy, which must have a BitsInit initializer.
   if (!BitList.empty()) {
     const auto *CurVal = dyn_cast<BitsInit>(RV->getValue());
     if (!CurVal)
@@ -427,9 +425,9 @@ bool TGParser::resolve(const ForeachLoop &Loop, SubstStack &Substs,
   const Init *List = Loop.ListValue->resolveReferences(R);
 
   // For if-then-else blocks, we lower to a foreach loop whose list is a
-  // ternary selection between lists of different length.  Since we don't
+  // ternary selection between lists of different length. Since we don't
   // have a means to track variable length record lists, we *must* resolve
-  // the condition here.  We want to defer final resolution of the arms
+  // the condition here. We want to defer final resolution of the arms
   // until the resulting records are finalized.
   // e.g. !if(!exists<SchedWrite>("__does_not_exist__"), [1], [])
   if (const auto *TI = dyn_cast<TernOpInit>(List);
@@ -700,7 +698,7 @@ const Init *TGParser::ParseObjectName(MultiClass *CurMultiClass) {
   return Name;
 }
 
-/// ParseClassID - Parse and resolve a reference to a class name.  This returns
+/// ParseClassID - Parse and resolve a reference to a class name. This returns
 /// null on error.
 ///
 ///    ClassID ::= ID
@@ -788,7 +786,7 @@ ParseSubClassReference(Record *CurRec, bool isDefm) {
 }
 
 /// ParseSubMultiClassReference - Parse a reference to a subclass or to a
-/// templated submulticlass.  This returns a SubMultiClassRefTy with a null
+/// templated submulticlass. This returns a SubMultiClassRefTy with a null
 /// Record* on error.
 ///
 ///  SubMultiClassRef ::= MultiClassID
@@ -1078,7 +1076,7 @@ bool TGParser::ParseOptionalBitList(SmallVectorImpl<unsigned> &Ranges) {
   return false;
 }
 
-/// ParseType - Parse and return a tblgen type.  This returns null on error.
+/// ParseType - Parse and return a tblgen type. This returns null on error.
 ///
 ///   Type ::= STRING                       // string type
 ///   Type ::= CODE                         // code type
@@ -1180,7 +1178,7 @@ const Init *TGParser::ParseIDValue(Record *CurRec, const StringInit *Name,
   return nullptr;
 }
 
-/// ParseOperation - Parse an operator.  This returns null on error.
+/// ParseOperation - Parse an operator. This returns null on error.
 ///
 /// Operation ::= XOperator ['<' Type '>'] '(' Args ')'
 ///
@@ -2253,7 +2251,7 @@ const Init *TGParser::ParseOperation(Record *CurRec, const RecTy *ItemType) {
   }
 }
 
-/// ParseOperatorType - Parse a type for an operator.  This returns
+/// ParseOperatorType - Parse a type for an operator. This returns
 /// null on error.
 ///
 /// OperatorType ::= '<' Type '>'
@@ -2670,7 +2668,7 @@ const Init *TGParser::ParseOperationCond(Record *CurRec,
   return CondOpInit::get(Case, Val, Type)->Fold(CurRec);
 }
 
-/// ParseSimpleValue - Parse a tblgen value.  This returns null on error.
+/// ParseSimpleValue - Parse a tblgen value. This returns null on error.
 ///
 ///   SimpleValue ::= IDValue
 ///   SimpleValue ::= INTVAL
@@ -2803,7 +2801,7 @@ const Init *TGParser::ParseSimpleValue(Record *CurRec, const RecTy *ItemType,
     SmallVector<const Init *, 16> NewBits;
 
     // As we parse { a, b, ... }, 'a' is the highest bit, but we parse it
-    // first.  We'll first read everything in to a vector, then we can reverse
+    // first. We'll first read everything in to a vector, then we can reverse
     // it to get the bits in the correct order for the BitsInit value.
     for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
       // FIXME: The following two loops would not be duplicated
@@ -3319,10 +3317,10 @@ bool TGParser::ParseTemplateArgValueList(
 }
 
 /// ParseDeclaration - Read a declaration, returning the name of field ID, or an
-/// empty string on error.  This can happen in a number of different contexts,
+/// empty string on error. This can happen in a number of different contexts,
 /// including within a def or in the template args for a class (in which case
 /// CurRec will be non-null) and within the template args for a multiclass (in
-/// which case CurRec will be null, but CurMultiClass will be set).  This can
+/// which case CurRec will be null, but CurMultiClass will be set). This can
 /// also happen within a def that is within a multiclass, which will set both
 /// CurRec and CurMultiClass.
 ///
@@ -3384,7 +3382,7 @@ const Init *TGParser::ParseDeclaration(Record *CurRec,
     if (!Val ||
         SetValue(CurRec, ValLoc, DeclName, {}, Val,
                  /*AllowSelfAssignment=*/false, /*OverrideDefLoc=*/false)) {
-      // Return the name, even if an error is thrown.  This is so that we can
+      // Return the name, even if an error is thrown. This is so that we can
       // continue to make some progress, even without the value having been
       // initialized.
       return DeclName;
@@ -3395,7 +3393,7 @@ const Init *TGParser::ParseDeclaration(Record *CurRec,
 }
 
 /// ParseForeachDeclaration - Read a foreach declaration, returning
-/// the name of the declared object or a NULL Init on error.  Return
+/// the name of the declared object or a NULL Init on error. Return
 /// the name of the parsed initializer list through ForeachListName.
 ///
 ///  ForeachDeclaration ::= ID '=' '{' RangeList '}'
@@ -3477,7 +3475,7 @@ TGParser::ParseForeachDeclaration(const Init *&ForeachListValue) {
 }
 
 /// ParseTemplateArgList - Read a template argument list, which is a non-empty
-/// sequence of template-declarations in <>'s.  If CurRec is non-null, these are
+/// sequence of template-declarations in <>'s. If CurRec is non-null, these are
 /// template args for a class. If null, these are the template args for a
 /// multiclass.
 ///
@@ -3578,7 +3576,7 @@ bool TGParser::ParseBodyItem(Record *CurRec) {
   return SetValue(CurRec, IdLoc, FieldName, BitList, Val);
 }
 
-/// ParseBody - Read the body of a class or def.  Return true on error, false on
+/// ParseBody - Read the body of a class or def. Return true on error, false on
 /// success.
 ///
 ///   Body     ::= ';'
@@ -3641,8 +3639,8 @@ bool TGParser::ApplyLetStack(RecordsEntry &Entry) {
   return false;
 }
 
-/// ParseObjectBody - Parse the body of a def or class.  This consists of an
-/// optional ClassList followed by a Body.  CurRec is the current def or class
+/// ParseObjectBody - Parse the body of a def or class. This consists of an
+/// optional ClassList followed by a Body. CurRec is the current def or class
 /// that is being parsed.
 ///
 ///   ObjectBody      ::= BaseClassList Body
@@ -3841,8 +3839,8 @@ bool TGParser::ParseDefvar(Record *CurRec) {
   return false;
 }
 
-/// ParseForeach - Parse a for statement.  Return the record corresponding
-/// to it.  This returns true on error.
+/// ParseForeach - Parse a for statement. Return the record corresponding
+/// to it. This returns true on error.
 ///
 ///   Foreach ::= FOREACH Declaration IN '{ ObjectList '}'
 ///   Foreach ::= FOREACH Declaration IN Object
@@ -4129,7 +4127,7 @@ void TGParser::ParseLetList(SmallVectorImpl<LetRecord> &Result) {
   } while (consume(tgtok::comma));
 }
 
-/// ParseTopLevelLet - Parse a 'let' at top level.  This can be a couple of
+/// ParseTopLevelLet - Parse a 'let' at top level. This can be a couple of
 /// different related productions. This works inside multiclasses too.
 ///
 ///   Object ::= LET LetList IN '{' ObjectList '}'
diff --git a/llvm/lib/TableGen/TGParser.h b/llvm/lib/TableGen/TGParser.h
index 017cc5fff683a..2a5a1925343cf 100644
--- a/llvm/lib/TableGen/TGParser.h
+++ b/llvm/lib/TableGen/TGParser.h
@@ -161,10 +161,10 @@ class TGParser {
   // Record tracker
   RecordKeeper &Records;
 
-  // A "named boolean" indicating how to parse identifiers.  Usually
+  // A "named boolean" indicating how to parse identifiers. Usually
   // identifiers map to some existing object but in special cases
   // (e.g. parsing def names) no such object exists yet because we are
-  // in the middle of creating in.  For those situations, allow the
+  // in the middle of creating in. For those situations, allow the
   // parser to ignore missing object errors.
   enum IDParseMode {
     ParseValueMode,   // We are parsing a value we expect to look up.
@@ -183,7 +183,7 @@ class TGParser {
         NoWarnOnUnusedTemplateArgs(NoWarnOnUnusedTemplateArgs),
         TrackReferenceLocs(TrackReferenceLocs) {}
 
-  /// ParseFile - Main entrypoint for parsing a tblgen file.  These parser
+  /// ParseFile - Main entrypoint for parsing a tblgen file. These parser
   /// routines return true on error, or false on success.
   bool ParseFile();
 
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index aa2a177d88c5b..86f95488e6bb7 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -58,12 +58,12 @@ include "AArch64SystemOperands.td"
 
 class AArch64Unsupported { list<Predicate> F; }
 
-let F = [HasSVE2p1, HasSVE2p1_or_SME2, HasSVE2p1_or_SME2p1] in
+let F = [HasSVE2p1, HasSVE2p1_or_SME2, HasSVE2p1_or_StreamingSME2, HasSVE2p1_or_SME2p1] in
 def SVE2p1Unsupported : AArch64Unsupported;
 
 def SVE2Unsupported : AArch64Unsupported {
-  let F = !listconcat([HasSVE2, HasSVE2_or_SME, HasSVE2_or_SME2, HasSSVE_FP8FMA, HasSMEF8F16,
-                       HasSMEF8F32, HasSVEAES, HasSVESHA3, HasSVE2SM4, HasSVEBitPerm,
+  let F = !listconcat([HasSVE2, HasSVE2_or_SME, HasNonStreamingSVE2_or_SME2, HasSSVE_FP8FMA, HasSMEF8F16,
+                       HasSMEF8F32, HasSVEAES, HasSVESHA3, HasSVESM4, HasSVEBitPerm,
                        HasSVEB16B16],
                        SVE2p1Unsupported.F);
 }
@@ -85,9 +85,9 @@ def SME2p1Unsupported : AArch64Unsupported {
 }
 
 def SME2Unsupported : AArch64Unsupported {
-  let F = !listconcat([HasSME2, HasSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
+  let F = !listconcat([HasSME2, HasNonStreamingSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
                       HasSMEF8F16, HasSMEF8F32, HasSMEF16F16_or_SMEF8F16, HasSMEB16B16,
-                      HasNonStreamingSVE_or_SSVE_AES],
+                      HasNonStreamingSVE_or_SSVE_AES, HasSVE2p1_or_StreamingSME2],
                       SME2p1Unsupported.F);
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index dd10050592190..15a4ae480f632 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -2254,15 +2254,19 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
   uint64_t KeyID = CPA.getKey()->getZExtValue();
   // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
   // AArch64AuthMCExpr::printImpl, so fail fast.
-  if (KeyID > AArch64PACKey::LAST)
-    report_fatal_error("AArch64 PAC Key ID '" + Twine(KeyID) +
-                       "' out of range [0, " +
-                       Twine((unsigned)AArch64PACKey::LAST) + "]");
+  if (KeyID > AArch64PACKey::LAST) {
+    CPA.getContext().emitError("AArch64 PAC Key ID '" + Twine(KeyID) +
+                               "' out of range [0, " +
+                               Twine((unsigned)AArch64PACKey::LAST) + "]");
+    KeyID = 0;
+  }
 
   uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
-  if (!isUInt<16>(Disc))
-    report_fatal_error("AArch64 PAC Discriminator '" + Twine(Disc) +
-                       "' out of range [0, 0xFFFF]");
+  if (!isUInt<16>(Disc)) {
+    CPA.getContext().emitError("AArch64 PAC Discriminator '" + Twine(Disc) +
+                               "' out of range [0, 0xFFFF]");
+    Disc = 0;
+  }
 
   // Finally build the complete @AUTH expr.
   return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID),
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 24fbe207c4969..9973df865ea17 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -370,8 +370,11 @@ def FeatureSVEAES : ExtensionWithMArch<"sve-aes", "SVEAES",
 def FeatureAliasSVE2AES : ExtensionWithMArch<"sve2-aes", "SVE2AES",
   "", "Shorthand for +sve2+sve-aes", [FeatureSVE2, FeatureSVEAES]>;
 
-def FeatureSVE2SM4 : ExtensionWithMArch<"sve2-sm4", "SVE2SM4", "FEAT_SVE_SM4",
-  "Enable SM4 SVE2 instructions", [FeatureSVE2, FeatureSM4]>;
+def FeatureSVESM4 : ExtensionWithMArch<"sve-sm4", "SVESM4", "FEAT_SVE_SM4",
+  "Enable SVE SM4 instructions", [FeatureSM4]>;
+
+def FeatureAliasSVE2SM4 : ExtensionWithMArch<"sve2-sm4", "SVE2SM4",
+  "", "Shorthand for +sve2+sve-sm4", [FeatureSVE2, FeatureSVESM4]>;
 
 def FeatureSVESHA3 : ExtensionWithMArch<"sve-sha3", "SVESHA3", "FEAT_SVE_SHA3",
   "Enable SVE SHA3 instructions", [FeatureSHA3]>;
@@ -612,8 +615,11 @@ def FeatureExperimentalZeroingPseudos
 def FeatureNoSVEFPLD1R : SubtargetFeature<"no-sve-fp-ld1r",
   "NoSVEFPLD1R", "true", "Avoid using LD1RX instructions for FP">;
 
-def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true",
-                                        "Has zero-cycle register moves">;
+def FeatureZCRegMoveGPR64 : SubtargetFeature<"zcm-gpr64", "HasZeroCycleRegMoveGPR64", "true",
+                                        "Has zero-cycle register moves for GPR64 registers">;
+
+def FeatureZCRegMoveGPR32 : SubtargetFeature<"zcm-gpr32", "HasZeroCycleRegMoveGPR32", "true",
+                                        "Has zero-cycle register moves for GPR32 registers">;
 
 def FeatureZCRegMoveFPR64 : SubtargetFeature<"zcm-fpr64", "HasZeroCycleRegMoveFPR64", "true",
                                         "Has zero-cycle register moves for FPR64 registers">;
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 6f1ce5bdbe286..b91e5c55e520d 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -435,6 +435,8 @@ AArch64FrameLowering::getStackIDForScalableVectors() const {
 static unsigned getFixedObjectSize(const MachineFunction &MF,
                                    const AArch64FunctionInfo *AFI, bool IsWin64,
                                    bool IsFunclet) {
+  assert(AFI->getTailCallReservedStack() % 16 == 0 &&
+         "Tail call reserved stack must be aligned to 16 bytes");
   if (!IsWin64 || IsFunclet) {
     return AFI->getTailCallReservedStack();
   } else {
@@ -442,12 +444,31 @@ static unsigned getFixedObjectSize(const MachineFunction &MF,
         !MF.getFunction().getAttributes().hasAttrSomewhere(
             Attribute::SwiftAsync))
       report_fatal_error("cannot generate ABI-changing tail call for Win64");
+    unsigned FixedObjectSize = AFI->getTailCallReservedStack();
+
     // Var args are stored here in the primary function.
-    const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
-    // To support EH funclets we allocate an UnwindHelp object
-    const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
-    return AFI->getTailCallReservedStack() +
-           alignTo(VarArgsArea + UnwindHelpObject, 16);
+    FixedObjectSize += AFI->getVarArgsGPRSize();
+
+    if (MF.hasEHFunclets()) {
+      // Catch objects are stored here in the primary function.
+      const MachineFrameInfo &MFI = MF.getFrameInfo();
+      const WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
+      SmallSetVector<int, 8> CatchObjFrameIndices;
+      for (const WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
+        for (const WinEHHandlerType &H : TBME.HandlerArray) {
+          int FrameIndex = H.CatchObj.FrameIndex;
+          if ((FrameIndex != INT_MAX) &&
+              CatchObjFrameIndices.insert(FrameIndex)) {
+            FixedObjectSize = alignTo(FixedObjectSize,
+                                      MFI.getObjectAlign(FrameIndex).value()) +
+                              MFI.getObjectSize(FrameIndex);
+          }
+        }
+      }
+      // To support EH funclets we allocate an UnwindHelp object
+      FixedObjectSize += 8;
+    }
+    return alignTo(FixedObjectSize, 16);
   }
 }
 
@@ -518,6 +539,27 @@ bool AArch64FrameLowering::hasFPImpl(const MachineFunction &MF) const {
   return false;
 }
 
+/// Should the Frame Pointer be reserved for the current function?
+bool AArch64FrameLowering::isFPReserved(const MachineFunction &MF) const {
+  const TargetMachine &TM = MF.getTarget();
+  const Triple &TT = TM.getTargetTriple();
+
+  // These OSes require the frame chain is valid, even if the current frame does
+  // not use a frame pointer.
+  if (TT.isOSDarwin() || TT.isOSWindows())
+    return true;
+
+  // If the function has a frame pointer, it is reserved.
+  if (hasFP(MF))
+    return true;
+
+  // Frontend has requested to preserve the frame pointer.
+  if (TM.Options.FramePointerIsReserved(MF))
+    return true;
+
+  return false;
+}
+
 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
 /// not required, we reserve argument space for call sites in the function
 /// immediately on entry to the current function.  This eliminates the need for
@@ -2521,9 +2563,9 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
           .buildDefCFA(AArch64::SP, NumBytes);
 
     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
-                    StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
-                    TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
-                    &HasWinCFI, EmitCFI, StackOffset::getFixed(NumBytes));
+                    StackOffset::getFixed(NumBytes + AfterCSRPopSize), TII,
+                    MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI,
+                    EmitCFI, StackOffset::getFixed(NumBytes));
     return;
   }
 
@@ -4641,23 +4683,39 @@ void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
   // anything.
   if (!MF.hasEHFunclets())
     return;
-  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
-  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
 
-  MachineBasicBlock &MBB = MF.front();
-  auto MBBI = MBB.begin();
-  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
-    ++MBBI;
+  // Win64 C++ EH needs to allocate space for the catch objects in the fixed
+  // object area right next to the UnwindHelp object.
+  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
+  int64_t CurrentOffset =
+      AFI->getVarArgsGPRSize() + AFI->getTailCallReservedStack();
+  for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
+    for (WinEHHandlerType &H : TBME.HandlerArray) {
+      int FrameIndex = H.CatchObj.FrameIndex;
+      if ((FrameIndex != INT_MAX) && MFI.getObjectOffset(FrameIndex) == 0) {
+        CurrentOffset =
+            alignTo(CurrentOffset, MFI.getObjectAlign(FrameIndex).value());
+        CurrentOffset += MFI.getObjectSize(FrameIndex);
+        MFI.setObjectOffset(FrameIndex, -CurrentOffset);
+      }
+    }
+  }
 
   // Create an UnwindHelp object.
   // The UnwindHelp object is allocated at the start of the fixed object area
-  int64_t FixedObject =
-      getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
-  int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
-                                           /*SPOffset*/ -FixedObject,
+  int64_t UnwindHelpOffset = alignTo(CurrentOffset + 8, Align(16));
+  assert(UnwindHelpOffset == getFixedObjectSize(MF, AFI, /*IsWin64*/ true,
+                                                /*IsFunclet*/ false) &&
+         "UnwindHelpOffset must be at the start of the fixed object area");
+  int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8, -UnwindHelpOffset,
                                            /*IsImmutable=*/false);
   EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
 
+  MachineBasicBlock &MBB = MF.front();
+  auto MBBI = MBB.begin();
+  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
+    ++MBBI;
+
   // We need to store -2 into the UnwindHelp object at the start of the
   // function.
   DebugLoc DL;
@@ -4665,6 +4723,7 @@ void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
   RS->backward(MBBI);
   Register DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
   assert(DstReg && "There must be a free register after frame setup");
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
   BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
   BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
       .addReg(DstReg, getKillRegState(true))
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
index e7d52bb350f13..ced69c9cd3699 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
@@ -126,6 +126,8 @@ class AArch64FrameLowering : public TargetFrameLowering {
   orderFrameObjects(const MachineFunction &MF,
                     SmallVectorImpl<int> &ObjectsToAllocate) const override;
 
+  bool isFPReserved(const MachineFunction &MF) const;
+
 protected:
   bool hasFPImpl(const MachineFunction &MF) const override;
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c8b1eafd35495..6b7e9357aab5a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -768,6 +768,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(Op, MVT::v8bf16, Expand);
   }
 
+  // Legalize fcanonicalize to circumvent default expansion
+  setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
+  if (Subtarget->hasFullFP16()) {
+    setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
+  }
+
   // fpextend from f16 or bf16 to f32 is legal
   setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
   setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Legal);
@@ -10683,12 +10689,9 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
 
   EVT VT = Op.getValueType();
   if (VT.isScalableVector() ||
-      useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
+      useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
 
-  if (!Subtarget->isNeonAvailable())
-    return SDValue();
-
   bool IsParity = Op.getOpcode() == ISD::PARITY;
   SDValue Val = Op.getOperand(0);
   SDLoc DL(Op);
@@ -10698,6 +10701,36 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
   if (VT == MVT::i32 && IsParity)
     return SDValue();
 
+  if (Subtarget->isSVEorStreamingSVEAvailable()) {
+    if (VT == MVT::i32 || VT == MVT::i64) {
+      EVT ContainerVT = VT == MVT::i32 ? MVT::nxv4i32 : MVT::nxv2i64;
+      Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT,
+                        DAG.getUNDEF(ContainerVT), Val,
+                        DAG.getVectorIdxConstant(0, DL));
+      Val = DAG.getNode(ISD::CTPOP, DL, ContainerVT, Val);
+      Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Val,
+                        DAG.getVectorIdxConstant(0, DL));
+      if (IsParity)
+        Val = DAG.getNode(ISD::AND, DL, VT, Val, DAG.getConstant(1, DL, VT));
+      return Val;
+    }
+
+    if (VT == MVT::i128) {
+      Val = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Val);
+      Val = convertToScalableVector(DAG, MVT::nxv2i64, Val);
+      Val = DAG.getNode(ISD::CTPOP, DL, MVT::nxv2i64, Val);
+      Val = convertFromScalableVector(DAG, MVT::v2i64, Val);
+      Val = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i64, Val);
+      Val = DAG.getZExtOrTrunc(Val, DL, VT);
+      if (IsParity)
+        Val = DAG.getNode(ISD::AND, DL, VT, Val, DAG.getConstant(1, DL, VT));
+      return Val;
+    }
+  }
+
+  if (!Subtarget->isNeonAvailable())
+    return SDValue();
+
   // If there is no CNT instruction available, GPR popcount can
   // be more efficiently lowered to the following sequence that uses
   // AdvSIMD registers/instructions as long as the copies to/from
@@ -18019,6 +18052,15 @@ static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
 //  v16i32 abs(
 //    v16i32 sub(
 //     v16i32 [sign|zero]_extend(v16i8 a), v16i32 [sign|zero]_extend(v16i8 b))))
+//
+//  or
+//
+// i32 vecreduce_add(
+//  v16i32 zext(
+//   v16i16 abs(
+//    v16i16 sub(
+//     v16i16 [sign|zero]_extend(v16i8 a), v16i16 [sign|zero]_extend(v16i8 b))))
+//
 // =================>
 // i32 vecreduce_add(
 //   v4i32 UADDLP(
@@ -18034,23 +18076,35 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
     return SDValue();
 
   SDValue VecReduceOp0 = N->getOperand(0);
+  bool SawTrailingZext = false;
+  // Look through an optional post-ABS ZEXT from v16i16 -> v16i32.
+  if (VecReduceOp0.getOpcode() == ISD::ZERO_EXTEND &&
+      VecReduceOp0->getValueType(0) == MVT::v16i32 &&
+      VecReduceOp0->getOperand(0)->getOpcode() == ISD::ABS &&
+      VecReduceOp0->getOperand(0)->getValueType(0) == MVT::v16i16) {
+    SawTrailingZext = true;
+    VecReduceOp0 = VecReduceOp0.getOperand(0);
+  }
+
+  // Peel off an optional post-ABS extend (v16i16 -> v16i32).
+  MVT AbsInputVT = SawTrailingZext ? MVT::v16i16 : MVT::v16i32;
+  // Assumed v16i16 or v16i32 abs input
   unsigned Opcode = VecReduceOp0.getOpcode();
-  // Assumed v16i32 abs
-  if (Opcode != ISD::ABS || VecReduceOp0->getValueType(0) != MVT::v16i32)
+  if (Opcode != ISD::ABS || VecReduceOp0->getValueType(0) != AbsInputVT)
     return SDValue();
 
   SDValue ABS = VecReduceOp0;
-  // Assumed v16i32 sub
+  // Assumed v16i16 or v16i32 sub
   if (ABS->getOperand(0)->getOpcode() != ISD::SUB ||
-      ABS->getOperand(0)->getValueType(0) != MVT::v16i32)
+      ABS->getOperand(0)->getValueType(0) != AbsInputVT)
     return SDValue();
 
   SDValue SUB = ABS->getOperand(0);
   unsigned Opcode0 = SUB->getOperand(0).getOpcode();
   unsigned Opcode1 = SUB->getOperand(1).getOpcode();
-  // Assumed v16i32 type
-  if (SUB->getOperand(0)->getValueType(0) != MVT::v16i32 ||
-      SUB->getOperand(1)->getValueType(0) != MVT::v16i32)
+  // Assumed v16i16 or v16i32 type
+  if (SUB->getOperand(0)->getValueType(0) != AbsInputVT ||
+      SUB->getOperand(1)->getValueType(0) != AbsInputVT)
     return SDValue();
 
   // Assumed zext or sext
@@ -25496,6 +25550,9 @@ static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
     return SwapResult;
 
   SDValue N0 = N->getOperand(0);
+  SDValue IfTrue = N->getOperand(1);
+  SDValue IfFalse = N->getOperand(2);
+  EVT ResVT = N->getValueType(0);
   EVT CCVT = N0.getValueType();
 
   if (isAllActivePredicate(DAG, N0))
@@ -25504,6 +25561,22 @@ static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
   if (isAllInactivePredicate(N0))
     return N->getOperand(2);
 
+  if (isMergePassthruOpcode(IfTrue.getOpcode()) && IfTrue.hasOneUse()) {
+    // vselect A, (merge_pasthru_op all_active, B,{Bn,} -), C
+    // vselect A, (merge_pasthru_op -, B,{Bn,} undef), C
+    // vselect A, (merge_pasthru_op A, B,{Bn,} -), C
+    //   -> merge_pasthru_op A, B,{Bn,} C
+    if (isAllActivePredicate(DAG, IfTrue->getOperand(0)) ||
+        IfTrue->getOperand(IfTrue.getNumOperands() - 1).isUndef() ||
+        IfTrue->getOperand(0) == N0) {
+      SmallVector<SDValue, 4> Ops(IfTrue->op_values());
+      Ops[0] = N0;
+      Ops[IfTrue.getNumOperands() - 1] = IfFalse;
+
+      return DAG.getNode(IfTrue.getOpcode(), SDLoc(N), ResVT, Ops);
+    }
+  }
+
   // Check for sign pattern (VSELECT setgt, iN lhs, -1, 1, -1) and transform
   // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
   // supported types.
@@ -25543,14 +25616,11 @@ static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
       CmpVT.getVectorElementType().isFloatingPoint())
     return SDValue();
 
-  EVT ResVT = N->getValueType(0);
   // Only combine when the result type is of the same size as the compared
   // operands.
   if (ResVT.getSizeInBits() != CmpVT.getSizeInBits())
     return SDValue();
 
-  SDValue IfTrue = N->getOperand(1);
-  SDValue IfFalse = N->getOperand(2);
   SetCC = DAG.getSetCC(SDLoc(N), CmpVT.changeVectorElementTypeToInteger(),
                        N0.getOperand(0), N0.getOperand(1),
                        cast<CondCodeSDNode>(N0.getOperand(2))->get());
@@ -28486,9 +28556,7 @@ void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
 }
 
 // Unlike X86, we let frame lowering assign offsets to all catch objects.
-bool AArch64TargetLowering::needsFixedCatchObjects() const {
-  return false;
-}
+bool AArch64TargetLowering::needsFixedCatchObjects() const { return true; }
 
 bool AArch64TargetLowering::shouldLocalize(
     const MachineInstr &MI, const TargetTransformInfo *TTI) const {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index c3837cfe73d28..8847c62690714 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -5037,7 +5037,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
 
     if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
       // If either operand is WSP, expand to ADD #0.
-      if (Subtarget.hasZeroCycleRegMove()) {
+      if (Subtarget.hasZeroCycleRegMoveGPR64() &&
+          !Subtarget.hasZeroCycleRegMoveGPR32()) {
         // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
         MCRegister DestRegX = TRI->getMatchingSuperReg(
             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
@@ -5063,7 +5064,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
           .addImm(0)
           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
     } else {
-      if (Subtarget.hasZeroCycleRegMove()) {
+      if (Subtarget.hasZeroCycleRegMoveGPR64() &&
+          !Subtarget.hasZeroCycleRegMoveGPR32()) {
         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
         MCRegister DestRegX = TRI->getMatchingSuperReg(
             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
@@ -9694,9 +9696,6 @@ AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
                                        MachineBasicBlock::iterator &MIT,
                                        unsigned Flags) const {
   MachineInstr &MI = *MIT;
-  MachineBasicBlock *MBB = MI.getParent();
-  MachineFunction *MF = MBB->getParent();
-  AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
 
   // Don't outline anything used for return address signing. The outlined
   // function will get signed later if needed
@@ -9724,10 +9723,6 @@ AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
     return outliner::InstrType::Illegal;
   }
 
-  // Don't outline LOHs.
-  if (FuncInfo->getLOHRelated().count(&MI))
-    return outliner::InstrType::Illegal;
-
   // We can only outline these if we will tail call the outlined function, or
   // fix up the CFI offsets. Currently, CFI instructions are outlined only if
   // in a tail call.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f01b634f44ba6..811877ffacedb 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -151,8 +151,8 @@ def HasSVE2p1        : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasS
                                  AssemblerPredicateWithAll<(all_of FeatureSVE2p1), "sve2p1">;
 def HasSVEAES       : Predicate<"Subtarget->hasSVEAES()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVEAES), "sve-aes">;
-def HasSVE2SM4       : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SM4()">,
-                                 AssemblerPredicateWithAll<(all_of FeatureSVE2SM4), "sve2-sm4">;
+def HasSVESM4       : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVESM4()">,
+                                 AssemblerPredicateWithAll<(all_of FeatureSVESM4), "sve-sm4">;
 def HasSVESHA3      : Predicate<"Subtarget->hasSVESHA3()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVESHA3), "sve-sha3">;
 def HasSVEBitPerm   : Predicate<"Subtarget->hasSVEBitPerm()">,
@@ -254,7 +254,7 @@ def HasNonStreamingSVE_or_SME2p1
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p1),
                 "sve or sme2p1">;
 def HasNonStreamingSVE_or_SME2p2
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
+    : Predicate<"Subtarget->isSVEAvailable() ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p2),
                 "sve or sme2p2">;
@@ -269,32 +269,38 @@ def HasNonStreamingSVE_or_SSVE_BitPerm
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_BitPerm),
                 "sve or ssve-bitperm">;
 def HasNonStreamingSVE_or_SSVE_FEXPA
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
+    : Predicate<"Subtarget->isSVEAvailable() ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_FEXPA())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_FEXPA),
                 "sve or ssve-fexpa">;
 
 def HasSVE2_or_SME
-    : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME),
                 "sve2 or sme">;
-def HasSVE2_or_SME2
-    : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
+def HasNonStreamingSVE2_or_SME2
+    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME2),
                 "sve2 or sme2">;
 
 def HasSVE2p1_or_SME
-    : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME())">,
                 AssemblerPredicateWithAll<(any_of FeatureSME, FeatureSVE2p1),
                 "sme or sve2p1">;
 def HasSVE2p1_or_SME2
-    : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
                 "sme2 or sve2p1">;
 def HasSVE2p1_or_SME2p1
-    : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2p1())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME2p1())">,
                 AssemblerPredicateWithAll<(any_of FeatureSME2p1, FeatureSVE2p1),
                 "sme2p1 or sve2p1">;
+def HasSVE2p1_or_StreamingSME2
+    : Predicate<"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVE2p1()) ||"
+                "(Subtarget->isStreaming() && Subtarget->hasSME2())">,
+                AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
+                "sme2 or sve2p1">;
 
 def HasSVE2p2_or_SME2p2
     : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2())">,
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index f51f0d11ef9d8..dde4e7ab0e890 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -1081,8 +1081,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
           LLVM_DEBUG(dbgs() << "Renamed " << MI);
           return true;
         };
-    forAllMIsUntilDef(MergeForward ? *I : *std::prev(Paired), RegToRename, TRI,
-                      UINT32_MAX, UpdateMIs);
+    forAllMIsUntilDef(MergeForward ? *I : *Paired->getPrevNode(), RegToRename,
+                      TRI, UINT32_MAX, UpdateMIs);
 
 #if !defined(NDEBUG)
     // For forward merging store:
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 361d5ec3f2b22..e61f2280865b4 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -504,6 +504,20 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
     LOHRelated.insert_range(Args);
   }
 
+  size_t
+  clearLinkerOptimizationHints(const SmallPtrSetImpl<MachineInstr *> &MIs) {
+    size_t InitialSize = LOHContainerSet.size();
+    erase_if(LOHContainerSet, [&](const auto &D) {
+      return any_of(D.getArgs(), [&](auto *Arg) { return MIs.contains(Arg); });
+    });
+    // In theory there could be an LOH with one label in MIs and another label
+    // outside MIs, however we don't know if the label outside MIs is used in
+    // any other LOHs, so we can't remove them from LOHRelated. In that case, we
+    // might produce a few extra labels, but it won't break anything.
+    LOHRelated.remove_if([&](auto *MI) { return MIs.contains(MI); });
+    return InitialSize - LOHContainerSet.size();
+  };
+
   SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
     return ForwardedMustTailRegParms;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h b/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
index e9bc6d947b0d9..f7beca1b8b77e 100644
--- a/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
+++ b/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
@@ -6750,6 +6750,21 @@ inline std::optional<unsigned> isDUPQMask(ArrayRef<int> Mask, unsigned Segments,
   return std::nullopt;
 }
 
+/// isDUPFirstSegmentMask - matches a splat of the first 128b segment.
+inline bool isDUPFirstSegmentMask(ArrayRef<int> Mask, unsigned Segments,
+                                  unsigned SegmentSize) {
+  // Make sure there's no size changes.
+  if (SegmentSize * Segments != Mask.size())
+    return false;
+
+  // Check that all lanes refer to the equivalent lane in the first segment.
+  // Undef/poison lanes (<0) are also accepted.
+  return all_of(enumerate(Mask), [&](auto P) {
+    const unsigned IndexWithinSegment = P.index() % SegmentSize;
+    return P.value() < 0 || unsigned(P.value()) == IndexWithinSegment;
+  });
+}
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 4a5682475d107..5379305bc7a7f 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -311,7 +311,7 @@ def TuneAppleA7  : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7",
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES, FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing,
                                     FeatureZCZeroingFPWorkaround]>;
@@ -325,7 +325,7 @@ def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -338,7 +338,7 @@ def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -351,7 +351,7 @@ def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -364,7 +364,7 @@ def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -382,7 +382,7 @@ def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -400,7 +400,7 @@ def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -418,7 +418,7 @@ def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -436,7 +436,7 @@ def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
                                     FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
@@ -453,7 +453,7 @@ def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4",
                                      FeatureFuseCCSelect,
                                      FeatureFuseCryptoEOR,
                                      FeatureFuseLiterals,
-                                     FeatureZCRegMove,
+                                     FeatureZCRegMoveGPR64,
                                      FeatureZCRegMoveFPR64,
                                      FeatureZCZeroing
                                      ]>;
@@ -906,7 +906,7 @@ def ProcessorFeatures {
   list<SubtargetFeature> MONAKA = [HasV9_3aOps, FeaturePerfMon, FeatureCCIDX,
                                    FeatureFPAC, FeatureFP16FML, FeatureRandGen,
                                    FeatureSSBS, FeatureLS64, FeatureCLRBHB,
-                                   FeatureSPECRES2, FeatureSVEAES, FeatureSVE2SM4,
+                                   FeatureSPECRES2, FeatureSVEAES, FeatureSVESM4,
                                    FeatureSVESHA3, FeatureSVE2, FeatureSVEBitPerm, FeatureETE,
                                    FeatureMEC, FeatureFAMINMAX, FeatureFP8DOT2, FeatureFP8DOT4,
                                    FeatureFP8FMA, FeatureLUT];
@@ -921,7 +921,7 @@ def ProcessorFeatures {
                                     FeatureMTE, FeaturePerfMon, FeatureRandGen,
                                     FeatureSPE, FeatureSPE_EEF, FeatureSSBS,
                                     FeatureSVEBitPerm, FeatureSVESHA3,
-                                    FeatureSVE2SM4, FeatureSVEAES];
+                                    FeatureSVESM4, FeatureSVEAES, FeatureSVE2];
   list<SubtargetFeature> AppleA7  = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON,FeaturePerfMon];
   list<SubtargetFeature> AppleA10 = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
@@ -1117,7 +1117,8 @@ def ProcessorFeatures {
                                      FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
                                      FeatureSSBS, FeatureCCIDX,
                                      FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
-  list<SubtargetFeature> Grace = !listconcat(NeoverseV2, [FeatureSVE2SM4, FeatureSVEAES, FeatureSVESHA3]);
+  list<SubtargetFeature> GB10 = !listconcat(X925, [FeatureSVEAES, FeatureSVESHA3, FeatureSVESM4]);
+  list<SubtargetFeature> Grace = !listconcat(NeoverseV2, [FeatureSVESM4, FeatureSVEAES, FeatureSVESHA3]);
 
   // ETE and TRBE are future architecture extensions. We temporarily enable them
   // by default for users targeting generic AArch64. The extensions do not
@@ -1203,6 +1204,8 @@ def : ProcessorModel<"cortex-x4", NeoverseV2Model, ProcessorFeatures.X4,
                      [TuneX4]>;
 def : ProcessorModel<"cortex-x925", NeoverseV2Model, ProcessorFeatures.X925,
                      [TuneX925]>;
+def : ProcessorModel<"gb10", NeoverseV2Model, ProcessorFeatures.GB10,
+                     [TuneX925]>;
 def : ProcessorModel<"grace", NeoverseV2Model, ProcessorFeatures.Grace,
                      [TuneNeoverseV2]>;
 def : ProcessorModel<"neoverse-e1", CortexA53Model,
diff --git a/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp b/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp
index 8edf1d0f9296b..3f45d55063b50 100644
--- a/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp
@@ -345,6 +345,10 @@ static bool shouldConvertImpl(const Constant *Cst) {
   if (Cst->isZeroValue())
     return false;
 
+  // Globals cannot be or contain scalable vectors.
+  if (Cst->getType()->isScalableTy())
+    return false;
+
   if (Stress)
     return true;
 
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index fb472ddc719fc..dd23bf51a98c4 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -441,7 +441,7 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
   markSuperRegs(Reserved, AArch64::WSP);
   markSuperRegs(Reserved, AArch64::WZR);
 
-  if (TFI->hasFP(MF) || TT.isOSDarwin())
+  if (TFI->isFPReserved(MF))
     markSuperRegs(Reserved, AArch64::W29);
 
   if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index e998db8c9cb23..3c0fd785de29b 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2445,14 +2445,23 @@ let Predicates = [HasSVE_or_SME] in {
   defm FCVTZU_ZPmZ_DtoD : sve_fp_2op_p_zd< 0b1111111, "fcvtzu", ZPR64, ZPR64, null_frag,                     AArch64fcvtzu_mt, nxv2i64, nxv2i1, nxv2f64, ElementSizeD>;
 
   //These patterns exist to improve the code quality of conversions on unpacked types.
+  def : Pat<(nxv2f32 (AArch64fcvte_mt nxv2i1:$Pg, nxv2f16:$Zs, nxv2f32:$Zd)),
+            (FCVT_ZPmZ_HtoS ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
   def : Pat<(nxv2f32 (AArch64fcvte_mt (nxv2i1 (SVEAllActive:$Pg)), nxv2f16:$Zs, nxv2f32:$Zd)),
             (FCVT_ZPmZ_HtoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   // FP_ROUND has an additional 'precise' flag which indicates the type of rounding.
   // This is ignored by the pattern below where it is matched by (i64 timm0_1)
+  def : Pat<(nxv2f16 (AArch64fcvtr_mt nxv2i1:$Pg, nxv2f32:$Zs, (i64 timm0_1), nxv2f16:$Zd)),
+            (FCVT_ZPmZ_StoH ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
   def : Pat<(nxv2f16 (AArch64fcvtr_mt (nxv2i1 (SVEAllActive:$Pg)), nxv2f32:$Zs, (i64 timm0_1), nxv2f16:$Zd)),
             (FCVT_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
+  def : Pat<(nxv4f32 (AArch64fcvte_mt nxv4i1:$Pg, nxv4bf16:$Zs, nxv4f32:$Zd)),
+            (SEL_ZPZZ_S $Pg, (LSL_ZZI_S $Zs, (i32 16)), $Zd)>;
+  def : Pat<(nxv2f32 (AArch64fcvte_mt nxv2i1:$Pg, nxv2bf16:$Zs, nxv2f32:$Zd)),
+            (SEL_ZPZZ_D $Pg, (LSL_ZZI_S $Zs, (i32 16)), $Zd)>;
+
   def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), nxv4bf16:$op, undef)),
             (LSL_ZZI_S $op, (i32 16))>;
   def : Pat<(nxv2f32 (AArch64fcvte_mt (SVEAnyPredicate), nxv2bf16:$op, undef)),
@@ -4109,12 +4118,12 @@ let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in {
   defm PMULLT_ZZZ_Q : sve2_wide_int_arith_pmul<0b00, 0b11011, "pmullt", int_aarch64_sve_pmullt_pair>;
 }
 
-let Predicates = [HasSVE2SM4] in {
+let Predicates = [HasSVESM4] in {
   // SVE2 crypto constructive binary operations
   defm SM4EKEY_ZZZ_S : sve2_crypto_cons_bin_op<0b0, "sm4ekey", ZPR32, int_aarch64_sve_sm4ekey, nxv4i32>;
   // SVE2 crypto destructive binary operations
   def SM4E_ZZZ_S : sve2_crypto_des_bin_op<0b10, "sm4e", ZPR32, int_aarch64_sve_sm4e, nxv4i32>;
-} // End HasSVE2SM4
+} // End HasSVESM4
 
 let Predicates = [HasSVESHA3, HasNonStreamingSVE_or_SME2p1] in {
   // SVE2 crypto constructive binary operations
@@ -4183,11 +4192,6 @@ defm UDOT_ZZZ_HtoS  : sve2p1_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2
 defm SDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"sdot", 0b0, int_aarch64_sve_sdot_lane_x2>;
 defm UDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"udot", 0b1, int_aarch64_sve_udot_lane_x2>;
 
-defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
-defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext", int_aarch64_sve_pext>;
-defm PEXT_2PCI : sve2p1_pred_as_ctr_to_mask_pair<"pext">;
-defm PTRUE_C  : sve2p1_ptrue_pn<"ptrue">;
-
 defm SQCVTN_Z2Z_StoH  : sve2p1_multi_vec_extract_narrow<"sqcvtn", 0b00, int_aarch64_sve_sqcvtn_x2>;
 defm UQCVTN_Z2Z_StoH  : sve2p1_multi_vec_extract_narrow<"uqcvtn", 0b01, int_aarch64_sve_uqcvtn_x2>;
 defm SQCVTUN_Z2Z_StoH : sve2p1_multi_vec_extract_narrow<"sqcvtun", 0b10, int_aarch64_sve_sqcvtun_x2>;
@@ -4195,6 +4199,22 @@ defm SQRSHRN_Z2ZI_StoH  : sve2p1_multi_vec_shift_narrow<"sqrshrn", 0b101, int_aa
 defm UQRSHRN_Z2ZI_StoH  : sve2p1_multi_vec_shift_narrow<"uqrshrn", 0b111, int_aarch64_sve_uqrshrn_x2>;
 defm SQRSHRUN_Z2ZI_StoH : sve2p1_multi_vec_shift_narrow<"sqrshrun", 0b001, int_aarch64_sve_sqrshrun_x2>;
 
+defm WHILEGE_2PXX : sve2p1_int_while_rr_pair<"whilege", 0b000>;
+defm WHILEGT_2PXX : sve2p1_int_while_rr_pair<"whilegt", 0b001>;
+defm WHILELT_2PXX : sve2p1_int_while_rr_pair<"whilelt", 0b010>;
+defm WHILELE_2PXX : sve2p1_int_while_rr_pair<"whilele", 0b011>;
+defm WHILEHS_2PXX : sve2p1_int_while_rr_pair<"whilehs", 0b100>;
+defm WHILEHI_2PXX : sve2p1_int_while_rr_pair<"whilehi", 0b101>;
+defm WHILELO_2PXX : sve2p1_int_while_rr_pair<"whilelo", 0b110>;
+defm WHILELS_2PXX : sve2p1_int_while_rr_pair<"whilels", 0b111>;
+} // End HasSVE2p1_or_SME2
+
+let Predicates = [HasSVE2p1_or_StreamingSME2] in {
+defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
+defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext", int_aarch64_sve_pext>;
+defm PEXT_2PCI : sve2p1_pred_as_ctr_to_mask_pair<"pext">;
+defm PTRUE_C  : sve2p1_ptrue_pn<"ptrue">;
+
 // Load to two registers
 defm LD1B_2Z       : sve2p1_mem_cld_ss_2z<"ld1b", 0b00, 0b0, ZZ_b_mul_r, GPR64shifted8, ZZ_b_strided_and_contiguous>;
 defm LD1H_2Z       : sve2p1_mem_cld_ss_2z<"ld1h", 0b01, 0b0, ZZ_h_mul_r, GPR64shifted16, ZZ_h_strided_and_contiguous>;
@@ -4318,14 +4338,6 @@ defm : store_pn_x4<nxv8bf16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM>;
 defm : store_pn_x4<nxv4f32, int_aarch64_sve_stnt1_pn_x4, STNT1W_4Z_IMM>;
 defm : store_pn_x4<nxv2f64, int_aarch64_sve_stnt1_pn_x4, STNT1D_4Z_IMM>;
 
-defm WHILEGE_2PXX : sve2p1_int_while_rr_pair<"whilege", 0b000>;
-defm WHILEGT_2PXX : sve2p1_int_while_rr_pair<"whilegt", 0b001>;
-defm WHILELT_2PXX : sve2p1_int_while_rr_pair<"whilelt", 0b010>;
-defm WHILELE_2PXX : sve2p1_int_while_rr_pair<"whilele", 0b011>;
-defm WHILEHS_2PXX : sve2p1_int_while_rr_pair<"whilehs", 0b100>;
-defm WHILEHI_2PXX : sve2p1_int_while_rr_pair<"whilehi", 0b101>;
-defm WHILELO_2PXX : sve2p1_int_while_rr_pair<"whilelo", 0b110>;
-defm WHILELS_2PXX : sve2p1_int_while_rr_pair<"whilels", 0b111>;
 defm WHILEGE_CXX  : sve2p1_int_while_rr_pn<"whilege", 0b000>;
 defm WHILEGT_CXX  : sve2p1_int_while_rr_pn<"whilegt", 0b001>;
 defm WHILELT_CXX  : sve2p1_int_while_rr_pn<"whilelt", 0b010>;
@@ -4334,7 +4346,7 @@ defm WHILEHS_CXX  : sve2p1_int_while_rr_pn<"whilehs", 0b100>;
 defm WHILEHI_CXX  : sve2p1_int_while_rr_pn<"whilehi", 0b101>;
 defm WHILELO_CXX  : sve2p1_int_while_rr_pn<"whilelo", 0b110>;
 defm WHILELS_CXX  : sve2p1_int_while_rr_pn<"whilels", 0b111>;
-} // End HasSVE2p1_or_SME2
+} // End HasSVE2p1_or_StreamingSME2
 
 let Predicates = [HasSVE_or_SME] in {
 
@@ -4539,7 +4551,7 @@ let Predicates = [HasNonStreamingSVE2p2_or_SME2p2] in {
 //===----------------------------------------------------------------------===//
 // SVE2 FP8 instructions
 //===----------------------------------------------------------------------===//
-let Predicates = [HasSVE2_or_SME2, HasFP8] in {
+let Predicates = [HasNonStreamingSVE2_or_SME2, HasFP8] in {
 // FP8 upconvert
 defm F1CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b00, "f1cvt",    nxv8f16,  int_aarch64_sve_fp8_cvt1>;
 defm F2CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b01, "f2cvt",    nxv8f16,  int_aarch64_sve_fp8_cvt2>;
@@ -4556,15 +4568,15 @@ defm FCVTNB_Z2Z_StoB : sve2_fp8_down_cvt_single<0b01, "fcvtnb", ZZ_s_mul_r, nxv4
 defm BFCVTN_Z2Z_HtoB : sve2_fp8_down_cvt_single<0b10, "bfcvtn", ZZ_h_mul_r, nxv8bf16, int_aarch64_sve_fp8_cvtn>;
 
 defm FCVTNT_Z2Z_StoB : sve2_fp8_down_cvt_single_top<0b11, "fcvtnt", ZZ_s_mul_r, nxv4f32,  int_aarch64_sve_fp8_cvtnt>;
-} // End HasSVE2_or_SME2, HasFP8
+} // End HasNonStreamingSVE2_or_SME2, HasFP8
 
-let Predicates = [HasSVE2_or_SME2, HasFAMINMAX] in {
+let Predicates = [HasNonStreamingSVE2_or_SME2, HasFAMINMAX] in {
 defm FAMIN_ZPmZ : sve_fp_2op_p_zds<0b1111, "famin", "FAMIN_ZPZZ", int_aarch64_sve_famin, DestructiveBinaryComm>;
 defm FAMAX_ZPmZ : sve_fp_2op_p_zds<0b1110, "famax", "FAMAX_ZPZZ", int_aarch64_sve_famax, DestructiveBinaryComm>;
 
 defm FAMAX_ZPZZ : sve_fp_bin_pred_hfd<AArch64famax_p>;
 defm FAMIN_ZPZZ : sve_fp_bin_pred_hfd<AArch64famin_p>;
-} // End HasSVE2_or_SME2, HasFAMINMAX
+} // End HasNonStreamingSVE2_or_SME2, HasFAMINMAX
 
 let Predicates = [HasSSVE_FP8FMA] in {
 // FP8 Widening Multiply-Add Long - Indexed Group
@@ -4608,14 +4620,14 @@ defm FDOT_ZZZI_BtoS : sve2_fp8_dot_indexed_s<"fdot", int_aarch64_sve_fp8_fdot_la
 defm FDOT_ZZZ_BtoS : sve_fp8_dot<0b1, ZPR32, "fdot", nxv4f32, int_aarch64_sve_fp8_fdot>;
 }
 
-let Predicates = [HasSVE2_or_SME2, HasLUT] in {
+let Predicates = [HasNonStreamingSVE2_or_SME2, HasLUT] in {
 // LUTI2
   defm LUTI2_ZZZI : sve2_luti2_vector_index<"luti2">;
 // LUTI4
   defm LUTI4_ZZZI   : sve2_luti4_vector_index<"luti4">;
 // LUTI4 (two contiguous registers)
   defm LUTI4_Z2ZZI  : sve2_luti4_vector_vg2_index<"luti4">;
-} // End HasSVE2_or_SME2, HasLUT
+} // End HasNonStreamingSVE2_or_SME2, HasLUT
 
 //===----------------------------------------------------------------------===//
 // Checked Pointer Arithmetic (FEAT_CPA)
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
index e23daec97bd2d..91a707910a7f3 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 def NeoverseN2Model : SchedMachineModel {
-  let IssueWidth            =  10; // Micro-ops dispatched at a time.
+  let IssueWidth            =   5; // Micro-ops dispatched at a time.
   let MicroOpBufferSize     = 160; // Entries in micro-op re-order buffer.
   let LoadLatency           =   4; // Optimistic load latency.
   let MispredictPenalty     =  10; // Extra cycles for mispredicted branch.
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index d719f234b27f7..bafb8d0773232 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -12,7 +12,6 @@
 
 #include "AArch64SelectionDAGInfo.h"
 #include "AArch64MachineFunctionInfo.h"
-#include "AArch64TargetMachine.h"
 
 #define GET_SDNODE_DESC
 #include "AArch64GenSDNodeInfo.inc"
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 281a1457fe57e..95eab16511e5a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -512,6 +512,15 @@ AArch64TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
   return DAG;
 }
 
+size_t AArch64TargetMachine::clearLinkerOptimizationHints(
+    const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+  if (MIs.empty())
+    return 0;
+  auto *MI = *MIs.begin();
+  auto *FuncInfo = MI->getMF()->getInfo<AArch64FunctionInfo>();
+  return FuncInfo->clearLinkerOptimizationHints(MIs);
+}
+
 void AArch64leTargetMachine::anchor() { }
 
 AArch64leTargetMachine::AArch64leTargetMachine(
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index f8ba41f215430..b9e522dd6f226 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -76,6 +76,9 @@ class AArch64TargetMachine : public CodeGenTargetMachineImpl {
   ScheduleDAGInstrs *
   createPostMachineScheduler(MachineSchedContext *C) const override;
 
+  size_t clearLinkerOptimizationHints(
+      const SmallPtrSetImpl<MachineInstr *> &MIs) const override;
+
 private:
   bool isLittle;
 };
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 79b2dc2b3845e..adc905384bf53 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4967,9 +4967,9 @@ void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
   BaseT::getPeelingPreferences(L, SE, PP);
 }
 
-Value *
-AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                                  Type *ExpectedType) const {
+Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
+                                                         Type *ExpectedType,
+                                                         bool CanCreate) const {
   switch (Inst->getIntrinsicID()) {
   default:
     return nullptr;
@@ -4978,7 +4978,7 @@ AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
   case Intrinsic::aarch64_neon_st4: {
     // Create a struct type
     StructType *ST = dyn_cast<StructType>(ExpectedType);
-    if (!ST)
+    if (!CanCreate || !ST)
       return nullptr;
     unsigned NumElts = Inst->arg_size() - 1;
     if (ST->getNumElements() != NumElts)
@@ -5405,11 +5405,21 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
 
   // Sub opcodes currently only occur in chained cases.
   // Independent partial reduction subtractions are still costed as an add
-  if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
+  if ((Opcode != Instruction::Add && Opcode != Instruction::Sub) ||
+      OpAExtend == TTI::PR_None)
     return Invalid;
 
-  if (InputTypeA != InputTypeB)
+  // We only support multiply binary operations for now, and for muls we
+  // require the types being extended to be the same.
+  // NOTE: For muls AArch64 supports lowering mixed extensions to a usdot but
+  // only if the i8mm or sve/streaming features are available.
+  if (BinOp && (*BinOp != Instruction::Mul || InputTypeA != InputTypeB ||
+                OpBExtend == TTI::PR_None ||
+                (OpAExtend != OpBExtend && !ST->hasMatMulInt8() &&
+                 !ST->isSVEorStreamingSVEAvailable())))
     return Invalid;
+  assert((BinOp || (OpBExtend == TTI::PR_None && !InputTypeB)) &&
+         "Unexpected values for OpBExtend or InputTypeB");
 
   EVT InputEVT = EVT::getEVT(InputTypeA);
   EVT AccumEVT = EVT::getEVT(AccumType);
@@ -5456,15 +5466,6 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
   } else
     return Invalid;
 
-  // AArch64 supports lowering mixed fixed-width extensions to a usdot but only
-  // if the i8mm feature is available.
-  if (OpAExtend == TTI::PR_None || OpBExtend == TTI::PR_None ||
-      (OpAExtend != OpBExtend && !ST->hasMatMulInt8()))
-    return Invalid;
-
-  if (!BinOp || *BinOp != Instruction::Mul)
-    return Invalid;
-
   return Cost;
 }
 
@@ -5600,9 +5601,8 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
   }
 
   // Segmented shuffle matching.
-  if ((ST->hasSVE2p1() || ST->hasSME2p1()) &&
-      ST->isSVEorStreamingSVEAvailable() && Kind == TTI::SK_PermuteSingleSrc &&
-      isa<FixedVectorType>(SrcTy) && !Mask.empty() &&
+  if (Kind == TTI::SK_PermuteSingleSrc && isa<FixedVectorType>(SrcTy) &&
+      !Mask.empty() && SrcTy->getPrimitiveSizeInBits().isNonZero() &&
       SrcTy->getPrimitiveSizeInBits().isKnownMultipleOf(
           AArch64::SVEBitsPerBlock)) {
 
@@ -5612,7 +5612,14 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
     unsigned SegmentElts = VTy->getNumElements() / Segments;
 
     // dupq zd.t, zn.t[idx]
-    if (isDUPQMask(Mask, Segments, SegmentElts))
+    if ((ST->hasSVE2p1() || ST->hasSME2p1()) &&
+        ST->isSVEorStreamingSVEAvailable() &&
+        isDUPQMask(Mask, Segments, SegmentElts))
+      return LT.first;
+
+    // mov zd.q, vn
+    if (ST->isSVEorStreamingSVEAvailable() &&
+        isDUPFirstSegmentMask(Mask, Segments, SegmentElts))
       return LT.first;
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 9ada70bd7086a..ff0ab68a16a88 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -270,8 +270,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                              TTI::PeelingPreferences &PP) const override;
 
-  Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
-                                           Type *ExpectedType) const override;
+  Value *
+  getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType,
+                                    bool CanCreate = true) const override;
 
   bool getTgtMemIntrinsic(IntrinsicInst *Inst,
                           MemIntrinsicInfo &Info) const override;
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 99fabff13319f..1ca61f5c6b349 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -2294,7 +2294,7 @@ class AArch64Operand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createImm((MCE->getValue() - 90) / 180));
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<AArch64Operand>
   CreateToken(StringRef Str, SMLoc S, MCContext &Ctx, bool IsSuffix = false) {
@@ -2584,7 +2584,7 @@ class AArch64Operand : public MCParsedAsmOperand {
 
 } // end anonymous namespace.
 
-void AArch64Operand::print(raw_ostream &OS) const {
+void AArch64Operand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case k_FPImm:
     OS << "<fpimm " << getFPImm().bitcastToAPInt().getZExtValue();
@@ -2601,12 +2601,12 @@ void AArch64Operand::print(raw_ostream &OS) const {
     break;
   }
   case k_Immediate:
-    OS << *getImm();
+    MAI.printExpr(OS, *getImm());
     break;
   case k_ShiftedImm: {
     unsigned Shift = getShiftedImmShift();
     OS << "<shiftedimm ";
-    OS << *getShiftedImmVal();
+    MAI.printExpr(OS, *getShiftedImmVal());
     OS << ", lsl #" << AArch64_AM::getShiftValue(Shift) << ">";
     break;
   }
@@ -3299,7 +3299,8 @@ ParseStatus AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) {
     if (DarwinSpec == AArch64::S_None && ELFSpec == AArch64::S_INVALID) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADRP relocation (unfortunately).
-      Expr = MCSpecifierExpr::create(Expr, AArch64::S_ABS_PAGE, getContext());
+      Expr =
+          MCSpecifierExpr::create(Expr, AArch64::S_ABS_PAGE, getContext(), S);
     } else if ((DarwinSpec == AArch64::S_MACHO_GOTPAGE ||
                 DarwinSpec == AArch64::S_MACHO_TLVPPAGE) &&
                Addend != 0) {
@@ -3351,7 +3352,7 @@ ParseStatus AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) {
     if (DarwinSpec == AArch64::S_None && ELFSpec == AArch64::S_INVALID) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADR relocation (unfortunately).
-      Expr = MCSpecifierExpr::create(Expr, AArch64::S_ABS, getContext());
+      Expr = MCSpecifierExpr::create(Expr, AArch64::S_ABS, getContext(), S);
     } else if (ELFSpec != AArch64::S_GOT_AUTH_PAGE) {
       // For tiny code model, we use :got_auth: operator to fill 21-bit imm of
       // adr. It's not actually GOT entry page address but the GOT address
@@ -3741,7 +3742,8 @@ static const struct Extension {
     {"sve2", {AArch64::FeatureSVE2}},
     {"sve-aes", {AArch64::FeatureSVEAES}},
     {"sve2-aes", {AArch64::FeatureAliasSVE2AES, AArch64::FeatureSVEAES}},
-    {"sve2-sm4", {AArch64::FeatureSVE2SM4}},
+    {"sve-sm4", {AArch64::FeatureSVESM4}},
+    {"sve2-sm4", {AArch64::FeatureAliasSVE2SM4, AArch64::FeatureSVESM4}},
     {"sve-sha3", {AArch64::FeatureSVESHA3}},
     {"sve2-sha3", {AArch64::FeatureAliasSVE2SHA3, AArch64::FeatureSVESHA3}},
     {"sve-bitperm", {AArch64::FeatureSVEBitPerm}},
@@ -4394,7 +4396,7 @@ bool AArch64AsmParser::parseRegister(OperandVector &Operands) {
 bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
   bool HasELFModifier = false;
   AArch64::Specifier RefKind;
-
+  SMLoc Loc = getLexer().getLoc();
   if (parseOptionalToken(AsmToken::Colon)) {
     HasELFModifier = true;
 
@@ -4468,7 +4470,7 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
     return true;
 
   if (HasELFModifier)
-    ImmVal = MCSpecifierExpr::create(ImmVal, RefKind, getContext());
+    ImmVal = MCSpecifierExpr::create(ImmVal, RefKind, getContext(), Loc);
 
   SMLoc EndLoc;
   if (getContext().getAsmInfo()->hasSubsectionsViaSymbols()) {
@@ -7404,7 +7406,7 @@ bool AArch64AsmParser::parseDirectiveLOH(StringRef IDVal, SMLoc Loc) {
   if (parseEOL())
     return true;
 
-  getStreamer().emitLOHDirective((MCLOHType)Kind, Args);
+  getStreamer().emitLOHDirective(Kind, Args);
   return false;
 }
 
@@ -8211,7 +8213,7 @@ bool AArch64AsmParser::parseDataExpr(const MCExpr *&Res) {
     const MCExpr *Term;
     if (getParser().parsePrimaryExpr(Term, EndLoc, nullptr))
       return true;
-    Res = MCBinaryExpr::create(*Opcode, Res, Term, getContext());
+    Res = MCBinaryExpr::create(*Opcode, Res, Term, getContext(), Res->getLoc());
   }
   return false;
 }
@@ -8265,7 +8267,7 @@ bool AArch64AsmParser::parseAuthExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     return true;
 
   Res = AArch64AuthMCExpr::create(Res, Discriminator, *KeyIDOrNone,
-                                  UseAddressDiversity, Ctx);
+                                  UseAddressDiversity, Ctx, Res->getLoc());
   return false;
 }
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 12843e16d0da1..75f9a44f61e6f 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -923,16 +923,28 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .moreElementsToNextPow2(1)
       .widenScalarOrEltToNextPow2OrMinSize(1)
       .minScalar(1, s32)
+      .lowerIf([](const LegalityQuery &Query) {
+        return Query.Types[1].isVector() &&
+               Query.Types[1].getScalarSizeInBits() == 64 &&
+               Query.Types[0].getScalarSizeInBits() == 16;
+      })
       .widenScalarOrEltToNextPow2OrMinSize(0, /*MinSize=*/HasFP16 ? 16 : 32)
+      .scalarizeIf(
+          // v2i64->v2f32 needs to scalarize to avoid double-rounding issues.
+          [](const LegalityQuery &Query) {
+            return Query.Types[0].getScalarSizeInBits() == 32 &&
+                   Query.Types[1].getScalarSizeInBits() == 64;
+          },
+          0)
       .widenScalarIf(
-          [=](const LegalityQuery &Query) {
+          [](const LegalityQuery &Query) {
             return Query.Types[1].getScalarSizeInBits() <= 64 &&
                    Query.Types[0].getScalarSizeInBits() <
                        Query.Types[1].getScalarSizeInBits();
           },
           LegalizeMutations::changeElementSizeTo(0, 1))
       .widenScalarIf(
-          [=](const LegalityQuery &Query) {
+          [](const LegalityQuery &Query) {
             return Query.Types[0].getScalarSizeInBits() <= 64 &&
                    Query.Types[0].getScalarSizeInBits() >
                        Query.Types[1].getScalarSizeInBits();
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 0ff51c7161539..84884d98e6f9c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -15,7 +15,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -30,8 +29,6 @@ using namespace llvm;
 namespace {
 
 class AArch64AsmBackend : public MCAsmBackend {
-  static const unsigned PCRelFlagVal =
-      MCFixupKindInfo::FKF_IsAlignedDownTo32Bits | MCFixupKindInfo::FKF_IsPCRel;
 protected:
   Triple TheTriple;
 
@@ -50,27 +47,27 @@ class AArch64AsmBackend : public MCAsmBackend {
         // in AArch64FixupKinds.h.
         //
         // Name                           Offset (bits) Size (bits)     Flags
-        {"fixup_aarch64_pcrel_adr_imm21", 0, 32, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_adrp_imm21", 0, 32, PCRelFlagVal},
+        {"fixup_aarch64_pcrel_adr_imm21", 0, 32, 0},
+        {"fixup_aarch64_pcrel_adrp_imm21", 0, 32, 0},
         {"fixup_aarch64_add_imm12", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale1", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale2", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale4", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale8", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale16", 10, 12, 0},
-        {"fixup_aarch64_ldr_pcrel_imm19", 5, 19, PCRelFlagVal},
+        {"fixup_aarch64_ldr_pcrel_imm19", 5, 19, 0},
         {"fixup_aarch64_movw", 5, 16, 0},
-        {"fixup_aarch64_pcrel_branch9", 5, 9,  PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch14", 5, 14, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch16", 5, 16, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch19", 5, 19, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch26", 0, 26, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_call26", 0, 26, PCRelFlagVal}};
+        {"fixup_aarch64_pcrel_branch9", 5, 9, 0},
+        {"fixup_aarch64_pcrel_branch14", 5, 14, 0},
+        {"fixup_aarch64_pcrel_branch16", 5, 16, 0},
+        {"fixup_aarch64_pcrel_branch19", 5, 19, 0},
+        {"fixup_aarch64_pcrel_branch26", 0, 26, 0},
+        {"fixup_aarch64_pcrel_call26", 0, 26, 0}};
 
     // Fixup kinds from raw relocation types and .reloc directives force
     // relocations and do not need these fields.
     if (mc::isRelocation(Kind))
-      return MCAsmBackend::getFixupKindInfo(FK_NONE);
+      return {};
 
     if (Kind < FirstTargetFixupKind)
       return MCAsmBackend::getFixupKindInfo(Kind);
@@ -87,15 +84,10 @@ class AArch64AsmBackend : public MCAsmBackend {
 
   bool fixupNeedsRelaxation(const MCFixup &Fixup,
                             uint64_t Value) const override;
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override;
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
 
   unsigned getFixupKindContainereSizeInBytes(unsigned Kind) const;
-
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
 };
 
 } // end anonymous namespace
@@ -412,10 +404,29 @@ unsigned AArch64AsmBackend::getFixupKindContainereSizeInBytes(unsigned Kind) con
   }
 }
 
-void AArch64AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+static bool shouldForceRelocation(const MCFixup &Fixup) {
+  // The ADRP instruction adds some multiple of 0x1000 to the current PC &
+  // ~0xfff. This means that the required offset to reach a symbol can vary by
+  // up to one step depending on where the ADRP is in memory. For example:
+  //
+  //     ADRP x0, there
+  //  there:
+  //
+  // If the ADRP occurs at address 0xffc then "there" will be at 0x1000 and
+  // we'll need that as an offset. At any other address "there" will be in the
+  // same page as the ADRP and the instruction should encode 0x0. Assuming the
+  // section isn't 0x1000-aligned, we therefore need to delegate this decision
+  // to the linker -- a relocation!
+  return Fixup.getTargetKind() == AArch64::fixup_aarch64_pcrel_adrp_imm21;
+}
+
+void AArch64AsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                    const MCValue &Target,
                                    MutableArrayRef<char> Data, uint64_t Value,
                                    bool IsResolved) {
+  if (shouldForceRelocation(Fixup))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
@@ -496,11 +507,6 @@ bool AArch64AsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
   return int64_t(Value) != int64_t(int8_t(Value));
 }
 
-void AArch64AsmBackend::relaxInstruction(MCInst &Inst,
-                                         const MCSubtargetInfo &STI) const {
-  llvm_unreachable("AArch64AsmBackend::relaxInstruction() unimplemented");
-}
-
 bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
                                      const MCSubtargetInfo *STI) const {
   // If the count is not 4-byte aligned, we must be writing data into the text
@@ -515,23 +521,6 @@ bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-bool AArch64AsmBackend::shouldForceRelocation(const MCFixup &Fixup,
-                                              const MCValue &Target) {
-  // The ADRP instruction adds some multiple of 0x1000 to the current PC &
-  // ~0xfff. This means that the required offset to reach a symbol can vary by
-  // up to one step depending on where the ADRP is in memory. For example:
-  //
-  //     ADRP x0, there
-  //  there:
-  //
-  // If the ADRP occurs at address 0xffc then "there" will be at 0x1000 and
-  // we'll need that as an offset. At any other address "there" will be in the
-  // same page as the ADRP and the instruction should encode 0x0. Assuming the
-  // section isn't 0x1000-aligned, we therefore need to delegate this decision
-  // to the linker -- a relocation!
-  return Fixup.getTargetKind() == AArch64::fixup_aarch64_pcrel_adrp_imm21;
-}
-
 namespace {
 
 namespace CU {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
index e5c8564f09849..828c5c5462407 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
@@ -169,7 +169,7 @@ void AArch64AuthMCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
   bool WrapSubExprInParens = !isa<MCSymbolRefExpr>(getSubExpr());
   if (WrapSubExprInParens)
     OS << '(';
-  getSubExpr()->print(OS, MAI);
+  MAI->printExpr(OS, *getSubExpr());
   if (WrapSubExprInParens)
     OS << ')';
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
index f0b9c4dbb21e4..c28e925d77e2b 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
@@ -199,15 +199,17 @@ class AArch64AuthMCExpr final : public MCSpecifierExpr {
   AArch64PACKey::ID Key;
 
   explicit AArch64AuthMCExpr(const MCExpr *Expr, uint16_t Discriminator,
-                             AArch64PACKey::ID Key, bool HasAddressDiversity)
-      : MCSpecifierExpr(Expr, HasAddressDiversity ? AArch64::S_AUTHADDR
-                                                  : AArch64::S_AUTH),
+                             AArch64PACKey::ID Key, bool HasAddressDiversity,
+                             SMLoc Loc)
+      : MCSpecifierExpr(
+            Expr, HasAddressDiversity ? AArch64::S_AUTHADDR : AArch64::S_AUTH,
+            Loc),
         Discriminator(Discriminator), Key(Key) {}
 
 public:
   static const AArch64AuthMCExpr *
   create(const MCExpr *Expr, uint16_t Discriminator, AArch64PACKey::ID Key,
-         bool HasAddressDiversity, MCContext &Ctx);
+         bool HasAddressDiversity, MCContext &Ctx, SMLoc Loc = SMLoc());
 
   AArch64PACKey::ID getKey() const { return Key; }
   uint16_t getDiscriminator() const { return Discriminator; }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index a1034934e646d..6e5a69030dbc6 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -223,6 +223,11 @@ class AArch64MCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 /// getMachineOpValue - Return binary encoding of operand. If the machine
 /// operand requires relocation, record the relocation and return zero.
 unsigned
@@ -248,7 +253,7 @@ AArch64MCCodeEmitter::getLdStUImm12OpValue(const MCInst &MI, unsigned OpIdx,
   else {
     assert(MO.isExpr() && "unable to encode load/store imm operand");
     MCFixupKind Kind = MCFixupKind(FixupKind);
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
     ++MCNumFixups;
   }
 
@@ -269,14 +274,11 @@ AArch64MCCodeEmitter::getAdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
   assert(MO.isExpr() && "Unexpected target type!");
   const MCExpr *Expr = MO.getExpr();
 
-  MCFixupKind Kind = MI.getOpcode() == AArch64::ADR
-                         ? MCFixupKind(AArch64::fixup_aarch64_pcrel_adr_imm21)
-                         : MCFixupKind(AArch64::fixup_aarch64_pcrel_adrp_imm21);
-  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
-
+  unsigned Kind = MI.getOpcode() == AArch64::ADR
+                      ? AArch64::fixup_aarch64_pcrel_adr_imm21
+                      : AArch64::fixup_aarch64_pcrel_adrp_imm21;
+  addFixup(Fixups, 0, Expr, Kind, true);
   MCNumFixups += 1;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -301,8 +303,8 @@ AArch64MCCodeEmitter::getAddSubImmOpValue(const MCInst &MI, unsigned OpIdx,
   const MCExpr *Expr = MO.getExpr();
 
   // Encode the 12 bits of the fixup.
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_add_imm12);
-  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+  MCFixupKind Kind = AArch64::fixup_aarch64_add_imm12;
+  addFixup(Fixups, 0, Expr, Kind);
 
   ++MCNumFixups;
 
@@ -329,12 +331,9 @@ uint32_t AArch64MCCodeEmitter::getCondBranchTargetOpValue(
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch19);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_pcrel_branch19,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -350,12 +349,9 @@ uint32_t AArch64MCCodeEmitter::getCondCompBranchTargetOpValue(
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch9);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
+                                   AArch64::fixup_aarch64_pcrel_branch9, true));
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -373,12 +369,9 @@ AArch64MCCodeEmitter::getPAuthPCRelOpValue(const MCInst &MI, unsigned OpIdx,
     return -(MO.getImm());
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch16);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_pcrel_branch16,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -395,12 +388,9 @@ AArch64MCCodeEmitter::getLoadLiteralOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_ldr_pcrel_imm19);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_ldr_pcrel_imm19,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -423,8 +413,8 @@ AArch64MCCodeEmitter::getMoveWideImmOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected movz/movk immediate");
 
-  Fixups.push_back(MCFixup::create(
-      0, MO.getExpr(), MCFixupKind(AArch64::fixup_aarch64_movw), MI.getLoc()));
+  Fixups.push_back(
+      MCFixup::create(0, MO.getExpr(), AArch64::fixup_aarch64_movw));
 
   ++MCNumFixups;
 
@@ -443,12 +433,9 @@ uint32_t AArch64MCCodeEmitter::getTestBranchTargetOpValue(
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected ADR target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch14);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_pcrel_branch14,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -465,10 +452,10 @@ AArch64MCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected ADR target type!");
 
-  MCFixupKind Kind = MI.getOpcode() == AArch64::BL
-                         ? MCFixupKind(AArch64::fixup_aarch64_pcrel_call26)
-                         : MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
+  unsigned Kind = MI.getOpcode() == AArch64::BL
+                      ? AArch64::fixup_aarch64_pcrel_call26
+                      : AArch64::fixup_aarch64_pcrel_branch26;
+  addFixup(Fixups, 0, MO.getExpr(), Kind, true);
 
   ++MCNumFixups;
 
@@ -748,7 +735,7 @@ void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI,
     auto Reloc = STI.getTargetTriple().getEnvironment() == Triple::GNUILP32
                      ? ELF::R_AARCH64_P32_TLSDESC_CALL
                      : ELF::R_AARCH64_TLSDESC_CALL;
-    Fixups.push_back(MCFixup::create(0, MI.getOperand(0).getExpr(), Reloc));
+    addFixup(Fixups, 0, MI.getOperand(0).getExpr(), Reloc);
     return;
   }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
index a3b1ff5dad312..3d4a14b162bf0 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
@@ -18,7 +18,7 @@ const AArch64AuthMCExpr *AArch64AuthMCExpr::create(const MCExpr *Expr,
                                                    uint16_t Discriminator,
                                                    AArch64PACKey::ID Key,
                                                    bool HasAddressDiversity,
-                                                   MCContext &Ctx) {
+                                                   MCContext &Ctx, SMLoc Loc) {
   return new (Ctx)
-      AArch64AuthMCExpr(Expr, Discriminator, Key, HasAddressDiversity);
+      AArch64AuthMCExpr(Expr, Discriminator, Key, HasAddressDiversity, Loc);
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index 7d849d3cb3153..61458d7c24be0 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -148,7 +147,7 @@ static bool canUseLocalRelocation(const MCSectionMachO &Section,
 void AArch64MachObjectWriter::recordRelocation(
     MachObjectWriter *Writer, MCAssembler &Asm, const MCFragment *Fragment,
     const MCFixup &Fixup, MCValue Target, uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
 
   // See <reloc.h>.
   uint32_t FixupOffset = Asm.getFragmentOffset(*Fragment);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
index eecde7a161214..a53b676142a03 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
@@ -15,7 +15,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCWinCOFFObjectWriter.h"
@@ -49,16 +48,18 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
     MCContext &Ctx, const MCValue &Target, const MCFixup &Fixup,
     bool IsCrossSection, const MCAsmBackend &MAB) const {
   unsigned FixupKind = Fixup.getKind();
+  bool PCRel = Fixup.isPCRel();
   if (IsCrossSection) {
     // IMAGE_REL_ARM64_REL64 does not exist. We treat FK_Data_8 as FK_PCRel_4 so
     // that .xword a-b can lower to IMAGE_REL_ARM64_REL32. This allows generic
     // instrumentation to not bother with the COFF limitation. A negative value
     // needs attention.
-    if (FixupKind != FK_Data_4 && FixupKind != FK_Data_8) {
+    if (PCRel || (FixupKind != FK_Data_4 && FixupKind != FK_Data_8)) {
       Ctx.reportError(Fixup.getLoc(), "Cannot represent this expression");
       return COFF::IMAGE_REL_ARM64_ADDR32;
     }
-    FixupKind = FK_PCRel_4;
+    FixupKind = FK_Data_4;
+    PCRel = true;
   }
 
   auto Spec = Target.getSpecifier();
@@ -93,17 +94,14 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
     return COFF::IMAGE_REL_ARM64_ABSOLUTE; // Dummy return value
   }
 
-  case FK_PCRel_4:
-    return COFF::IMAGE_REL_ARM64_REL32;
-
   case FK_Data_4:
+    if (PCRel)
+      return COFF::IMAGE_REL_ARM64_REL32;
     switch (Spec) {
     default:
       return COFF::IMAGE_REL_ARM64_ADDR32;
     case MCSymbolRefExpr::VK_COFF_IMGREL32:
       return COFF::IMAGE_REL_ARM64_ADDR32NB;
-    case MCSymbolRefExpr::VK_SECREL:
-      return COFF::IMAGE_REL_ARM64_SECREL;
     }
 
   case FK_Data_8:
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 848b1c30bbeb5..d5c12a9658113 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -5368,6 +5368,19 @@ multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm,
   def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;
   def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;
   def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;
+
+  // Allow selecting SVE2 ternary ops with Neon types.
+  foreach VT = [nxv16i8, nxv8i16, nxv4i32, nxv2i64] in {
+    def : Pat<(SVEType<VT>.DSub (op V64:$op1, V64:$op2, V64:$op3)),
+              (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $op1, dsub),
+                                                        (INSERT_SUBREG (IMPLICIT_DEF), $op2, dsub),
+                                                        (INSERT_SUBREG (IMPLICIT_DEF), $op3, dsub)), dsub)>;
+
+    def : Pat<(SVEType<VT>.ZSub (op V128:$op1, V128:$op2, V128:$op3)),
+              (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $op1, zsub),
+                                                        (INSERT_SUBREG (IMPLICIT_DEF), $op2, zsub),
+                                                        (INSERT_SUBREG (IMPLICIT_DEF), $op3, zsub)), zsub)>;
+  }
 }
 
 class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 72d6a78539ada..5797a2ad0b4ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -143,6 +143,12 @@ def FeatureFmaMixInsts : SubtargetFeature<"fma-mix-insts",
   "Has v_fma_mix_f32, v_fma_mixlo_f16, v_fma_mixhi_f16 instructions"
 >;
 
+def FeatureIEEEMinimumMaximumInsts : SubtargetFeature<"ieee-minimum-maximum-insts",
+  "HasIEEEMinimumMaximumInsts",
+  "true",
+  "Has v_minimum/maximum_f16/f32/f64, v_minimummaximum/maximumminimum_f16/f32 and v_pk_minimum/maximum_f16 instructions"
+>;
+
 def FeatureMinimum3Maximum3F32 : SubtargetFeature<"minimum3-maximum3-f32",
   "HasMinimum3Maximum3F32",
   "true",
@@ -819,6 +825,12 @@ def FeatureFP8ConversionInsts : SubtargetFeature<"fp8-conversion-insts",
   "Has fp8 and bf8 conversion instructions"
 >;
 
+def FeatureFP8E5M3Insts : SubtargetFeature<"fp8e5m3-insts",
+  "HasFP8E5M3Insts",
+  "true",
+  "Has fp8 e5m3 format support"
+>;
+
 def FeatureCvtFP8VOP1Bug : SubtargetFeature<"cvt-fp8-vop1-bug",
   "HasCvtFP8Vop1Bug",
   "true",
@@ -1319,6 +1331,12 @@ def FeatureMemToLDSLoad : SubtargetFeature<"vmem-to-lds-load-insts",
   "The platform has memory to lds instructions (global_load w/lds bit set, buffer_load w/lds bit set or global_load_lds. This does not include scratch_load_lds."
 >;
 
+def FeatureLdsBarrierArriveAtomic : SubtargetFeature< "lds-barrier-arrive-atomic",
+  "HasLdsBarrierArriveAtomic",
+  "true",
+  "Has LDS barrier-arrive atomic instructions"
+>;
+
 // Dummy feature used to disable assembler instructions.
 def FeatureDisable : SubtargetFeature<"",
   "FeatureDisable","true",
@@ -1459,8 +1477,8 @@ def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12",
    FeatureUnalignedDSAccess, FeatureTrue16BitInsts,
    FeatureDefaultComponentBroadcast, FeatureMaxHardClauseLength32,
    FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF32FlatInsts,
-   FeatureMinimum3Maximum3F32, FeatureMinimum3Maximum3F16,
-   FeatureAgentScopeFineGrainedRemoteMemoryAtomics
+   FeatureIEEEMinimumMaximumInsts, FeatureMinimum3Maximum3F32,
+   FeatureMinimum3Maximum3F16, FeatureAgentScopeFineGrainedRemoteMemoryAtomics
   ]
 >;
 
@@ -1895,6 +1913,7 @@ def FeatureISAVersion12 : FeatureSet<
    FeatureImageInsts,
    FeatureExtendedImageInsts,
    FeatureFP8ConversionInsts,
+   FeatureIEEEMinimumMaximumInsts,
    FeaturePackedTID,
    FeatureVcmpxPermlaneHazard,
    FeatureSALUFloatInsts,
@@ -1931,6 +1950,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureAtomicBufferPkAddBF16Inst,
    FeatureFlatAtomicFaddF32Inst,
    FeatureFP8ConversionInsts,
+   FeatureFP8E5M3Insts,
    FeaturePackedTID,
    FeatureVcmpxPermlaneHazard,
    FeatureSALUFloatInsts,
@@ -1955,6 +1975,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureMemoryAtomicFAddF32DenormalSupport,
    FeatureKernargPreload,
    FeatureLshlAddU64Inst,
+   FeatureLdsBarrierArriveAtomic,
    FeatureSetPrioIncWgInst,
 ]>;
 
@@ -2284,6 +2305,10 @@ def isNotGFX1250Plus :
   Predicate<"!Subtarget->hasGFX1250Insts()">,
   AssemblerPredicate<(all_of (not FeatureGFX1250Insts))>;
 
+def HasIEEEMinimumMaximumInsts :
+  Predicate<"Subtarget->hasIEEEMinimumMaximumInsts()">,
+  AssemblerPredicate<(all_of FeatureIEEEMinimumMaximumInsts)>;
+
 def HasMinimum3Maximum3F32 :
   Predicate<"Subtarget->hasMinimum3Maximum3F32()">,
   AssemblerPredicate<(all_of FeatureMinimum3Maximum3F32)>;
@@ -2566,6 +2591,12 @@ def HasFP8Insts : Predicate<"Subtarget->hasFP8Insts()">,
 def HasFP8ConversionInsts : Predicate<"Subtarget->hasFP8ConversionInsts()">,
   AssemblerPredicate<(all_of FeatureFP8ConversionInsts)>;
 
+def HasFP8E5M3Insts : Predicate<"Subtarget->hasFP8E5M3Insts()">,
+  AssemblerPredicate<(all_of FeatureFP8E5M3Insts)>;
+
+def NotHasFP8E5M3Insts : Predicate<"!Subtarget->hasFP8E5M3Insts()">,
+  AssemblerPredicate<(all_of (not FeatureFP8E5M3Insts))>;
+
 def HasPkFmacF16Inst : Predicate<"Subtarget->hasPkFmacF16Inst()">,
   AssemblerPredicate<(all_of FeaturePkFmacF16Inst)>;
 
@@ -2687,6 +2718,9 @@ def HasAshrPkInsts : Predicate<"Subtarget->hasAshrPkInsts()">,
 def HasLshlAddU64Inst : Predicate<"Subtarget->hasLshlAddU64Inst()">,
                         AssemblerPredicate<(all_of FeatureLshlAddU64Inst)>;
 
+def HasLdsBarrierArriveAtomic : Predicate<"Subtarget->hasLdsBarrierArriveAtomic()">,
+  AssemblerPredicate<(all_of FeatureLdsBarrierArriveAtomic)>;
+
 def HasSetPrioIncWgInst : Predicate<"Subtarget->hasSetPrioIncWgInst()">,
  AssemblerPredicate<(all_of FeatureSetPrioIncWgInst)>;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index fef22c81c9391..79cf49f88d6dc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -235,6 +235,10 @@ class AMDGPUInformationCache : public InformationCache {
     return ST.getMaxWavesPerEU();
   }
 
+  unsigned getMaxAddrSpace() const override {
+    return AMDGPUAS::MAX_AMDGPU_ADDRESS;
+  }
+
 private:
   /// Check if the ConstantExpr \p CE uses an addrspacecast from private or
   /// local to flat. These casts may require the queue pointer.
@@ -1380,8 +1384,8 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM,
        &AAPotentialValues::ID, &AAAMDFlatWorkGroupSize::ID,
        &AAAMDMaxNumWorkgroups::ID, &AAAMDWavesPerEU::ID, &AAAMDGPUNoAGPR::ID,
        &AACallEdges::ID, &AAPointerInfo::ID, &AAPotentialConstantValues::ID,
-       &AAUnderlyingObjects::ID, &AAAddressSpace::ID, &AAIndirectCallInfo::ID,
-       &AAInstanceInfo::ID});
+       &AAUnderlyingObjects::ID, &AANoAliasAddrSpace::ID, &AAAddressSpace::ID,
+       &AAIndirectCallInfo::ID, &AAInstanceInfo::ID});
 
   AttributorConfig AC(CGUpdater);
   AC.IsClosedWorldModule = Options.IsClosedWorld;
@@ -1420,18 +1424,19 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM,
     }
 
     for (auto &I : instructions(F)) {
-      if (auto *LI = dyn_cast<LoadInst>(&I)) {
-        A.getOrCreateAAFor<AAAddressSpace>(
-            IRPosition::value(*LI->getPointerOperand()));
-      } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
-        A.getOrCreateAAFor<AAAddressSpace>(
-            IRPosition::value(*SI->getPointerOperand()));
-      } else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
-        A.getOrCreateAAFor<AAAddressSpace>(
-            IRPosition::value(*RMW->getPointerOperand()));
-      } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(&I)) {
-        A.getOrCreateAAFor<AAAddressSpace>(
-            IRPosition::value(*CmpX->getPointerOperand()));
+      Value *Ptr = nullptr;
+      if (auto *LI = dyn_cast<LoadInst>(&I))
+        Ptr = LI->getPointerOperand();
+      else if (auto *SI = dyn_cast<StoreInst>(&I))
+        Ptr = SI->getPointerOperand();
+      else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+        Ptr = RMW->getPointerOperand();
+      else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(&I))
+        Ptr = CmpX->getPointerOperand();
+
+      if (Ptr) {
+        A.getOrCreateAAFor<AAAddressSpace>(IRPosition::value(*Ptr));
+        A.getOrCreateAAFor<AANoAliasAddrSpace>(IRPosition::value(*Ptr));
       }
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 6e990cb2e160c..202693b316122 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2032,7 +2032,8 @@ bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
     const SIInstrInfo *TII = Subtarget->getInstrInfo();
 
-    if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
+    if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
+                               SIInstrFlags::FlatScratch)) {
       Addr = LHS;
       ImmOffset = COffsetVal;
     } else if (!LHS->isDivergent() && COffsetVal > 0) {
@@ -2040,8 +2041,8 @@ bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
       // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
       //                         (large_offset & MaxOffset);
       int64_t SplitImmOffset, RemainderOffset;
-      std::tie(SplitImmOffset, RemainderOffset)
-        = TII->splitFlatOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true);
+      std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
+          COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch);
 
       if (isUInt<32>(RemainderOffset)) {
         SDNode *VMov = CurDAG->getMachineNode(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index d75c7a178b4a8..26be37046e2e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -423,6 +423,11 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
   setOperationAction({ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, MVT::f16,
                      Custom);
 
+  setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
+  if (Subtarget->has16BitInsts()) {
+    setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
+  }
+
   // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
   // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
   // default unless marked custom/legal.
@@ -3853,7 +3858,7 @@ bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
   return true;
 }
 
-// Replace load of an illegal type with a store of a bitcast to a friendlier
+// Replace load of an illegal type with a bitcast from a load of a friendlier
 // type.
 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
@@ -4328,7 +4333,8 @@ SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
     Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
   }
 
-  SDValue NewShift = DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt);
+  SDValue NewShift =
+      DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt, N->getFlags());
 
   SDValue Vec;
   if (VT.isVector()) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index 383b5c0e9dcd5..44eaebffb70dc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -156,10 +156,9 @@ class AMDGPUInsertDelayAlu {
       SALUCycles = std::max(SALUCycles, RHS.SALUCycles);
     }
 
-    // Update this DelayInfo after issuing an instruction. IsVALU should be 1
-    // when issuing a (non-TRANS) VALU, else 0. IsTRANS should be 1 when issuing
-    // a TRANS, else 0. Cycles is the number of cycles it takes to issue the
-    // instruction.  Return true if there is no longer any useful delay info.
+    // Update this DelayInfo after issuing an instruction of the specified type.
+    // Cycles is the number of cycles it takes to issue the instruction.  Return
+    // true if there is no longer any useful delay info.
     bool advance(DelayType Type, unsigned Cycles) {
       bool Erase = true;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index b632b16f5c198..ea79c57080faa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4327,60 +4327,581 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
   }};
 }
 
-std::pair<Register, unsigned>
-AMDGPUInstructionSelector::selectVOP3PModsImpl(
-  Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const {
+enum class SrcStatus {
+  IS_SAME,
+  IS_UPPER_HALF,
+  IS_LOWER_HALF,
+  IS_UPPER_HALF_NEG,
+  // This means current op = [op_upper, op_lower] and src = -op_lower.
+  IS_LOWER_HALF_NEG,
+  IS_HI_NEG,
+  // This means current op = [op_upper, op_lower] and src = [op_upper,
+  // -op_lower].
+  IS_LO_NEG,
+  IS_BOTH_NEG,
+  INVALID,
+  NEG_START = IS_UPPER_HALF_NEG,
+  NEG_END = IS_BOTH_NEG,
+  HALF_START = IS_UPPER_HALF,
+  HALF_END = IS_LOWER_HALF_NEG
+};
+/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
+static bool isTruncHalf(const MachineInstr *MI,
+                        const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_TRUNC)
+    return false;
+
+  unsigned DstSize = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits();
+  unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+  return DstSize * 2 == SrcSize;
+}
+
+/// Test if the MI is logic shift right with half bits,
+/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
+static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_LSHR)
+    return false;
+
+  Register ShiftSrc;
+  std::optional<ValueAndVReg> ShiftAmt;
+  if (mi_match(MI->getOperand(0).getReg(), MRI,
+               m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
+    unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+    unsigned Shift = ShiftAmt->Value.getZExtValue();
+    return Shift * 2 == SrcSize;
+  }
+  return false;
+}
+
+/// Test if the MI is shift left with half bits,
+/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
+static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_SHL)
+    return false;
+
+  Register ShiftSrc;
+  std::optional<ValueAndVReg> ShiftAmt;
+  if (mi_match(MI->getOperand(0).getReg(), MRI,
+               m_GShl(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
+    unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
+    unsigned Shift = ShiftAmt->Value.getZExtValue();
+    return Shift * 2 == SrcSize;
+  }
+  return false;
+}
+
+/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
+static bool isUnmergeHalf(const MachineInstr *MI,
+                          const MachineRegisterInfo &MRI) {
+  if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
+    return false;
+  return MI->getNumOperands() == 3 && MI->getOperand(0).isDef() &&
+         MI->getOperand(1).isDef() && !MI->getOperand(2).isDef();
+}
+
+enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
+
+static TypeClass isVectorOfTwoOrScalar(Register Reg,
+                                       const MachineRegisterInfo &MRI) {
+  LLT OpTy = MRI.getType(Reg);
+  if (OpTy.isScalar())
+    return TypeClass::SCALAR;
+  if (OpTy.isVector() && OpTy.getNumElements() == 2)
+    return TypeClass::VECTOR_OF_TWO;
+  return TypeClass::NONE_OF_LISTED;
+}
+
+static SrcStatus getNegStatus(Register Reg, SrcStatus S,
+                              const MachineRegisterInfo &MRI) {
+  TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
+  if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
+    return SrcStatus::INVALID;
+
+  switch (S) {
+  case SrcStatus::IS_SAME:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [-OpHi, -OpLo]
+      return SrcStatus::IS_BOTH_NEG;
+    }
+    if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [-OpHi, OpLo]
+      return SrcStatus::IS_HI_NEG;
+    }
+    break;
+  case SrcStatus::IS_HI_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
+      return SrcStatus::IS_LO_NEG;
+    }
+    if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [-(-OpHi), OpLo] = [OpHi, OpLo]
+      return SrcStatus::IS_SAME;
+    }
+    break;
+  case SrcStatus::IS_LO_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
+      return SrcStatus::IS_HI_NEG;
+    }
+    if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [-OpHi, -OpLo]
+      return SrcStatus::IS_BOTH_NEG;
+    }
+    break;
+  case SrcStatus::IS_BOTH_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // [SrcHi, SrcLo]   = [-CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
+      // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
+      // [SrcHi, SrcLo]   = [OpHi, OpLo]
+      return SrcStatus::IS_SAME;
+    }
+    if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // [SrcHi, SrcLo]   = [-CurrHi, -CurrLo]
+      // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
+      // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
+      // [SrcHi, SrcLo]   = [OpHi, -OpLo]
+      return SrcStatus::IS_LO_NEG;
+    }
+    break;
+  case SrcStatus::IS_UPPER_HALF:
+    // Vector of 2:
+    // Src = CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+    // Src = -OpUpper
+    //
+    // Scalar:
+    // Src = CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+    // Src = -OpUpper
+    return SrcStatus::IS_UPPER_HALF_NEG;
+  case SrcStatus::IS_LOWER_HALF:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // Src = CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+      // Src = -OpLower
+      return SrcStatus::IS_LOWER_HALF_NEG;
+    }
+    if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // Src = CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+      // Src = OpLower
+      return SrcStatus::IS_LOWER_HALF;
+    }
+    break;
+  case SrcStatus::IS_UPPER_HALF_NEG:
+    // Vector of 2:
+    // Src = -CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+    // Src = -(-OpUpper) = OpUpper
+    //
+    // Scalar:
+    // Src = -CurrUpper
+    // Curr = [CurrUpper, CurrLower]
+    // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+    // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+    // Src = -(-OpUpper) = OpUpper
+    return SrcStatus::IS_UPPER_HALF;
+  case SrcStatus::IS_LOWER_HALF_NEG:
+    if (NegType == TypeClass::VECTOR_OF_TWO) {
+      // Vector of 2:
+      // Src = -CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
+      // Src = -(-OpLower) = OpLower
+      return SrcStatus::IS_LOWER_HALF;
+    }
+    if (NegType == TypeClass::SCALAR) {
+      // Scalar:
+      // Src = -CurrLower
+      // Curr = [CurrUpper, CurrLower]
+      // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
+      // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
+      // Src = -OpLower
+      return SrcStatus::IS_LOWER_HALF_NEG;
+    }
+    break;
+  default:
+    break;
+  }
+  llvm_unreachable("unexpected SrcStatus & NegType combination");
+}
+
+static std::optional<std::pair<Register, SrcStatus>>
+calcNextStatus(std::pair<Register, SrcStatus> Curr,
+               const MachineRegisterInfo &MRI) {
+  const MachineInstr *MI = MRI.getVRegDef(Curr.first);
+
+  unsigned Opc = MI->getOpcode();
+
+  // Handle general Opc cases.
+  switch (Opc) {
+  case AMDGPU::G_BITCAST:
+    return std::optional<std::pair<Register, SrcStatus>>(
+        {MI->getOperand(1).getReg(), Curr.second});
+  case AMDGPU::COPY:
+    if (MI->getOperand(1).getReg().isPhysical())
+      return std::nullopt;
+    return std::optional<std::pair<Register, SrcStatus>>(
+        {MI->getOperand(1).getReg(), Curr.second});
+  case AMDGPU::G_FNEG: {
+    SrcStatus Stat = getNegStatus(Curr.first, Curr.second, MRI);
+    if (Stat == SrcStatus::INVALID)
+      return std::nullopt;
+    return std::optional<std::pair<Register, SrcStatus>>(
+        {MI->getOperand(1).getReg(), Stat});
+  }
+  default:
+    break;
+  }
+
+  // Calc next Stat from current Stat.
+  switch (Curr.second) {
+  case SrcStatus::IS_SAME:
+    if (isTruncHalf(MI, MRI))
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF});
+    else if (isUnmergeHalf(MI, MRI)) {
+      if (Curr.first == MI->getOperand(0).getReg())
+        return std::optional<std::pair<Register, SrcStatus>>(
+            {MI->getOperand(2).getReg(), SrcStatus::IS_LOWER_HALF});
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(2).getReg(), SrcStatus::IS_UPPER_HALF});
+    }
+    break;
+  case SrcStatus::IS_HI_NEG:
+    if (isTruncHalf(MI, MRI)) {
+      // [SrcHi, SrcLo]   = [-CurrHi, CurrLo]
+      // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
+      //                  = [OpLowerHi, OpLowerLo]
+      // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
+      //     = [-OpLowerHi, OpLowerLo]
+      //     = -OpLower
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
+    }
+    if (isUnmergeHalf(MI, MRI)) {
+      if (Curr.first == MI->getOperand(0).getReg())
+        return std::optional<std::pair<Register, SrcStatus>>(
+            {MI->getOperand(2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
+    }
+    break;
+  case SrcStatus::IS_UPPER_HALF:
+    if (isShlHalf(MI, MRI))
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF});
+    break;
+  case SrcStatus::IS_LOWER_HALF:
+    if (isLshrHalf(MI, MRI))
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(1).getReg(), SrcStatus::IS_UPPER_HALF});
+    break;
+  case SrcStatus::IS_UPPER_HALF_NEG:
+    if (isShlHalf(MI, MRI))
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
+    break;
+  case SrcStatus::IS_LOWER_HALF_NEG:
+    if (isLshrHalf(MI, MRI))
+      return std::optional<std::pair<Register, SrcStatus>>(
+          {MI->getOperand(1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
+    break;
+  default:
+    break;
+  }
+  return std::nullopt;
+}
+
+/// This is used to control valid status that current MI supports. For example,
+/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
+/// bit on VOP3P.
+/// The class can be further extended to recognize support on SEL, NEG, ABS bit
+/// for different MI on different arch
+class SearchOptions {
+private:
+  bool HasNeg = false;
+  // Assume all complex pattern of VOP3P have opsel.
+  bool HasOpsel = true;
+
+public:
+  SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
+    const MachineInstr *MI = MRI.getVRegDef(Reg);
+    unsigned Opc = MI->getOpcode();
+
+    if (Opc < TargetOpcode::GENERIC_OP_END) {
+      // Keep same for generic op.
+      HasNeg = true;
+    } else if (Opc == TargetOpcode::G_INTRINSIC) {
+      Intrinsic::ID IntrinsicID = cast<GIntrinsic>(*MI).getIntrinsicID();
+      // Only float point intrinsic has neg & neg_hi bits.
+      if (IntrinsicID == Intrinsic::amdgcn_fdot2)
+        HasNeg = true;
+    }
+  }
+  bool checkOptions(SrcStatus Stat) const {
+    if (!HasNeg &&
+        (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
+      return false;
+    }
+    if (!HasOpsel &&
+        (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
+      return false;
+    }
+    return true;
+  }
+};
+
+static SmallVector<std::pair<Register, SrcStatus>>
+getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
+            int MaxDepth = 3) {
+  int Depth = 0;
+  auto Curr = calcNextStatus({Reg, SrcStatus::IS_SAME}, MRI);
+  SmallVector<std::pair<Register, SrcStatus>> Statlist;
+
+  while (Depth <= MaxDepth && Curr.has_value()) {
+    Depth++;
+    if (SO.checkOptions(Curr.value().second))
+      Statlist.push_back(Curr.value());
+    Curr = calcNextStatus(Curr.value(), MRI);
+  }
+
+  return Statlist;
+}
+
+static std::pair<Register, SrcStatus>
+getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
+                 int MaxDepth = 3) {
+  int Depth = 0;
+  std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
+  auto Curr = calcNextStatus(LastSameOrNeg, MRI);
+
+  while (Depth <= MaxDepth && Curr.has_value()) {
+    Depth++;
+    SrcStatus Stat = Curr.value().second;
+    if (SO.checkOptions(Stat)) {
+      if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
+          Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
+        LastSameOrNeg = Curr.value();
+    }
+    Curr = calcNextStatus(Curr.value(), MRI);
+  }
+
+  return LastSameOrNeg;
+}
+
+static bool isSameBitWidth(Register Reg1, Register Reg2,
+                           const MachineRegisterInfo &MRI) {
+  unsigned Width1 = MRI.getType(Reg1).getSizeInBits();
+  unsigned Width2 = MRI.getType(Reg2).getSizeInBits();
+  return Width1 == Width2;
+}
+
+static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
+  // SrcStatus::IS_LOWER_HALF remain 0.
+  if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
+    Mods ^= SISrcMods::NEG_HI;
+    Mods |= SISrcMods::OP_SEL_1;
+  } else if (HiStat == SrcStatus::IS_UPPER_HALF)
+    Mods |= SISrcMods::OP_SEL_1;
+  else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
+    Mods ^= SISrcMods::NEG_HI;
+  else if (HiStat == SrcStatus::IS_HI_NEG)
+    Mods ^= SISrcMods::NEG_HI;
+
+  if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
+    Mods ^= SISrcMods::NEG;
+    Mods |= SISrcMods::OP_SEL_0;
+  } else if (LoStat == SrcStatus::IS_UPPER_HALF)
+    Mods |= SISrcMods::OP_SEL_0;
+  else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
+    Mods |= SISrcMods::NEG;
+  else if (LoStat == SrcStatus::IS_HI_NEG)
+    Mods ^= SISrcMods::NEG;
+
+  return Mods;
+}
+
+static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
+                          Register RootReg, const SIInstrInfo &TII,
+                          const MachineRegisterInfo &MRI) {
+  auto IsHalfState = [](SrcStatus S) {
+    return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
+           S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
+  };
+  return isSameBitWidth(NewReg, RootReg, MRI) && IsHalfState(LoStat) &&
+         IsHalfState(HiStat);
+}
+
+std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
+    Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
   unsigned Mods = 0;
-  MachineInstr *MI = MRI.getVRegDef(Src);
+  // No modification if Root type is not form of <2 x Type>.
+  if (isVectorOfTwoOrScalar(RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {RootReg, Mods};
+  }
+
+  SearchOptions SO(RootReg, MRI);
 
-  if (MI->getOpcode() == AMDGPU::G_FNEG &&
-      // It's possible to see an f32 fneg here, but unlikely.
-      // TODO: Treat f32 fneg as only high bit.
-      MRI.getType(Src) == LLT::fixed_vector(2, 16)) {
+  std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(RootReg, MRI, SO);
+
+  if (Stat.second == SrcStatus::IS_BOTH_NEG)
     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
-    Src = MI->getOperand(1).getReg();
-    MI = MRI.getVRegDef(Src);
+  else if (Stat.second == SrcStatus::IS_HI_NEG)
+    Mods ^= SISrcMods::NEG_HI;
+  else if (Stat.second == SrcStatus::IS_LO_NEG)
+    Mods ^= SISrcMods::NEG;
+
+  MachineInstr *MI = MRI.getVRegDef(Stat.first);
+
+  if (MI->getOpcode() != AMDGPU::G_BUILD_VECTOR || MI->getNumOperands() != 3 ||
+      (IsDOT && Subtarget->hasDOTOpSelHazard())) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Stat.first, Mods};
   }
 
-  // TODO: Handle G_FSUB 0 as fneg
+  SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
+      getSrcStats(MI->getOperand(2).getReg(), MRI, SO);
+
+  if (StatlistHi.empty()) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Stat.first, Mods};
+  }
 
-  // TODO: Match op_sel through g_build_vector_trunc and g_shuffle_vector.
-  (void)IsDOT; // DOTs do not use OPSEL on gfx942+, check ST.hasDOTOpSelHazard()
+  SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
+      getSrcStats(MI->getOperand(1).getReg(), MRI, SO);
 
+  if (StatlistLo.empty()) {
+    Mods |= SISrcMods::OP_SEL_1;
+    return {Stat.first, Mods};
+  }
+
+  for (int I = StatlistHi.size() - 1; I >= 0; I--) {
+    for (int J = StatlistLo.size() - 1; J >= 0; J--) {
+      if (StatlistHi[I].first == StatlistLo[J].first &&
+          isValidToPack(StatlistHi[I].second, StatlistLo[J].second,
+                        StatlistHi[I].first, RootReg, TII, MRI))
+        return {StatlistHi[I].first,
+                updateMods(StatlistHi[I].second, StatlistLo[J].second, Mods)};
+    }
+  }
   // Packed instructions do not have abs modifiers.
   Mods |= SISrcMods::OP_SEL_1;
 
-  return std::pair(Src, Mods);
+  return {Stat.first, Mods};
 }
 
-InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
+// Removed unused function `getAllKindImm` to eliminate dead code.
 
-  Register Src;
+static bool checkRB(Register Reg, unsigned int RBNo,
+                    const AMDGPURegisterBankInfo &RBI,
+                    const MachineRegisterInfo &MRI,
+                    const TargetRegisterInfo &TRI) {
+  const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
+  return RB->getID() == RBNo;
+}
+
+// This function is used to get the correct register bank for returned reg.
+// Assume:
+// 1. VOP3P is always legal for VGPR.
+// 2. RootOp's regbank is legal.
+// Thus
+// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
+// 2. If RootOp is VGPR, then NewOp must be VGPR.
+static Register getLegalRegBank(Register NewReg, Register RootReg,
+                                const AMDGPURegisterBankInfo &RBI,
+                                MachineRegisterInfo &MRI,
+                                const TargetRegisterInfo &TRI,
+                                const SIInstrInfo &TII) {
+  // RootOp can only be VGPR or SGPR (some hand written cases such as.
+  // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
+  if (checkRB(RootReg, AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
+      checkRB(NewReg, AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
+    return NewReg;
+
+  MachineInstr *MI = MRI.getVRegDef(RootReg);
+  if (MI->getOpcode() == AMDGPU::COPY && NewReg == MI->getOperand(1).getReg()) {
+    // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
+    return RootReg;
+  }
+
+  MachineBasicBlock *BB = MI->getParent();
+  Register DstReg = MRI.cloneVirtualRegister(RootReg);
+
+  MachineInstrBuilder MIB =
+      BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
+          .addReg(NewReg);
+
+  // Only accept VGPR.
+  return MIB->getOperand(0).getReg();
+}
+
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
+                                                bool IsDOT) const {
+  MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
+  Register Reg;
   unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI);
+  std::tie(Reg, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, IsDOT);
 
+  Reg = getLegalRegBank(Reg, Root.getReg(), RBI, MRI, TRI, TII);
   return {{
-      [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }  // src_mods
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
   }};
 }
 
 InstructionSelector::ComplexRendererFns
-AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
+AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
 
-  Register Src;
-  unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true);
+  return selectVOP3PRetHelper(Root);
+}
 
-  return {{
-      [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }  // src_mods
-  }};
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
+
+  return selectVOP3PRetHelper(Root, true);
 }
 
 InstructionSelector::ComplexRendererFns
@@ -5070,7 +5591,8 @@ AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
 
   Register OrigAddr = Addr;
   if (ConstOffset != 0 &&
-      TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS, true)) {
+      TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS,
+                            SIInstrFlags::FlatScratch)) {
     Addr = PtrBase;
     ImmOffset = ConstOffset;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 6c3f3026e877a..8e9e573147a86 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -188,8 +188,10 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
   ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
 
   std::pair<Register, unsigned>
-  selectVOP3PModsImpl(Register Src, const MachineRegisterInfo &MRI,
+  selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
                       bool IsDOT = false) const;
+  InstructionSelector::ComplexRendererFns
+  selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
 
   InstructionSelector::ComplexRendererFns
   selectVOP3PMods(MachineOperand &Root) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index f82e6df9bcbfc..aa678df675fb6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2095,7 +2095,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
        G_SADDO, G_SSUBO})
       .lower();
 
-  if (ST.hasIEEEMinMax()) {
+  if (ST.hasIEEEMinimumMaximumInsts()) {
     getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
         .legalFor(FPTypesPK16)
         .clampMaxNumElements(0, S16, 2)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index b20760c356263..1483d97d23fcc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3348,6 +3348,20 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
         MI.eraseFromParent();
       return;
     }
+    case Intrinsic::amdgcn_tensor_load_to_lds:
+    case Intrinsic::amdgcn_tensor_store_from_lds: {
+      constrainOpWithReadfirstlane(B, MI, 1);
+      constrainOpWithReadfirstlane(B, MI, 2);
+      constrainOpWithReadfirstlane(B, MI, 3);
+      constrainOpWithReadfirstlane(B, MI, 4);
+      return;
+    }
+    case Intrinsic::amdgcn_tensor_load_to_lds_d2:
+    case Intrinsic::amdgcn_tensor_store_from_lds_d2: {
+      constrainOpWithReadfirstlane(B, MI, 1);
+      constrainOpWithReadfirstlane(B, MI, 2);
+      return;
+    }
     default: {
       if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
               AMDGPU::lookupRsrcIntrinsic(IntrID)) {
@@ -4586,6 +4600,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_dot4_f32_fp8_fp8:
     case Intrinsic::amdgcn_dot4_f32_bf8_bf8:
     case Intrinsic::amdgcn_cvt_f32_fp8:
+    case Intrinsic::amdgcn_cvt_f32_fp8_e5m3:
     case Intrinsic::amdgcn_cvt_f32_bf8:
     case Intrinsic::amdgcn_cvt_off_f32_i4:
     case Intrinsic::amdgcn_cvt_pk_f32_fp8:
@@ -4596,6 +4611,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_cvt_sr_bf8_f32:
     case Intrinsic::amdgcn_cvt_sr_bf16_f32:
     case Intrinsic::amdgcn_cvt_sr_f16_f32:
+    case Intrinsic::amdgcn_cvt_f16_fp8:
+    case Intrinsic::amdgcn_cvt_f16_bf8:
     case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f16:
     case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f16:
     case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_bf16:
@@ -5105,10 +5122,18 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
     case Intrinsic::amdgcn_global_load_tr_b64:
     case Intrinsic::amdgcn_global_load_tr_b128:
+    case Intrinsic::amdgcn_global_load_tr4_b64:
+    case Intrinsic::amdgcn_global_load_tr6_b96:
+    case Intrinsic::amdgcn_ds_load_tr8_b64:
+    case Intrinsic::amdgcn_ds_load_tr16_b128:
+    case Intrinsic::amdgcn_ds_load_tr4_b64:
+    case Intrinsic::amdgcn_ds_load_tr6_b96:
     case Intrinsic::amdgcn_ds_read_tr4_b64:
     case Intrinsic::amdgcn_ds_read_tr6_b96:
     case Intrinsic::amdgcn_ds_read_tr8_b64:
     case Intrinsic::amdgcn_ds_read_tr16_b64:
+    case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
+    case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_ds_ordered_add:
     case Intrinsic::amdgcn_ds_ordered_swap: {
@@ -5344,6 +5369,22 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     }
     case Intrinsic::amdgcn_pops_exiting_wave_id:
       return getDefaultMappingSOP(MI);
+    case Intrinsic::amdgcn_tensor_load_to_lds_d2:
+    case Intrinsic::amdgcn_tensor_store_from_lds_d2:
+    case Intrinsic::amdgcn_tensor_load_to_lds:
+    case Intrinsic::amdgcn_tensor_store_from_lds: {
+      // Lie and claim everything is legal, even all operands need to be
+      // SGPRs. applyMapping will have to deal with it with readfirstlane.
+      for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
+        if (MI.getOperand(I).isReg()) {
+          Register Reg = MI.getOperand(I).getReg();
+          auto OpBank = getRegBankID(Reg, MRI);
+          unsigned Size = getSizeInBits(Reg, MRI, *TRI);
+          OpdsMapping[I] = AMDGPU::getValueMapping(OpBank, Size);
+        }
+      }
+      break;
+    }
     case Intrinsic::amdgcn_s_prefetch_data: {
       OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 3b62dcf3c92cd..1f6002a3c6a20 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -344,6 +344,12 @@ def : SourceOfDivergence<intr>;
 
 def : SourceOfDivergence<int_amdgcn_global_load_tr_b64>;
 def : SourceOfDivergence<int_amdgcn_global_load_tr_b128>;
+def : SourceOfDivergence<int_amdgcn_global_load_tr4_b64>;
+def : SourceOfDivergence<int_amdgcn_global_load_tr6_b96>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr8_b64>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr16_b128>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr4_b64>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr6_b96>;
 
 def : SourceOfDivergence<int_amdgcn_ds_read_tr4_b64>;
 def : SourceOfDivergence<int_amdgcn_ds_read_tr6_b96>;
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 30dcd6d81f16d..3af140461afdb 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "AMDKernelCodeT.h"
 #include "MCTargetDesc/AMDGPUInstPrinter.h"
+#include "MCTargetDesc/AMDGPUMCAsmInfo.h"
 #include "MCTargetDesc/AMDGPUMCExpr.h"
 #include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -1151,7 +1152,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     // clang-format on
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Register:
       OS << "<register " << AMDGPUInstPrinter::getRegisterName(getReg())
@@ -1168,7 +1169,9 @@ class AMDGPUOperand : public MCParsedAsmOperand {
       OS << '\'' << getToken() << '\'';
       break;
     case Expression:
-      OS << "<expr " << *Expr << '>';
+      OS << "<expr ";
+      MAI.printExpr(OS, *Expr);
+      OS << '>';
       break;
     }
   }
@@ -1781,6 +1784,7 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   bool validateMIMGAddrSize(const MCInst &Inst, const SMLoc &IDLoc);
   bool validateMIMGD16(const MCInst &Inst);
   bool validateMIMGDim(const MCInst &Inst, const OperandVector &Operands);
+  bool validateTensorR128(const MCInst &Inst);
   bool validateMIMGMSAA(const MCInst &Inst);
   bool validateOpSel(const MCInst &Inst);
   bool validateTrue16OpSel(const MCInst &Inst);
@@ -4277,6 +4281,18 @@ bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
   return true;
 }
 
+bool AMDGPUAsmParser::validateTensorR128(const MCInst &Inst) {
+  const unsigned Opc = Inst.getOpcode();
+  const MCInstrDesc &Desc = MII.get(Opc);
+
+  if ((Desc.TSFlags & SIInstrFlags::TENSOR_CNT) == 0)
+    return true;
+
+  int R128Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::r128);
+
+  return R128Idx < 0 || !Inst.getOperand(R128Idx).getImm();
+}
+
 static bool IsRevOpcode(const unsigned Opcode)
 {
   switch (Opcode) {
@@ -5110,14 +5126,11 @@ bool AMDGPUAsmParser::validateTHAndScopeBits(const MCInst &Inst,
       return PrintError("scope and th combination is not valid");
   }
 
-  bool IsStore = TID.mayStore();
-  bool IsAtomic =
-      TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet);
-
-  if (IsAtomic) {
+  unsigned THType = AMDGPU::getTemporalHintType(TID);
+  if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) {
     if (!(CPol & AMDGPU::CPol::TH_TYPE_ATOMIC))
       return PrintError("invalid th value for atomic instructions");
-  } else if (IsStore) {
+  } else if (THType == AMDGPU::CPol::TH_TYPE_STORE) {
     if (!(CPol & AMDGPU::CPol::TH_TYPE_STORE))
       return PrintError("invalid th value for store instructions");
   } else {
@@ -5202,6 +5215,11 @@ bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
     Error(IDLoc, "missing dim operand");
     return false;
   }
+  if (!validateTensorR128(Inst)) {
+    Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
+          "instruction must set modifier r128=0");
+    return false;
+  }
   if (!validateMIMGMSAA(Inst)) {
     Error(getImmLoc(AMDGPUOperand::ImmTyDim, Operands),
           "invalid dim; must be MSAA type");
@@ -6451,26 +6469,26 @@ StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
   setForcedDPP(false);
   setForcedSDWA(false);
 
-  if (Name.ends_with("_e64_dpp")) {
+  if (Name.consume_back("_e64_dpp")) {
     setForcedDPP(true);
     setForcedEncodingSize(64);
-    return Name.substr(0, Name.size() - 8);
+    return Name;
   }
-  if (Name.ends_with("_e64")) {
+  if (Name.consume_back("_e64")) {
     setForcedEncodingSize(64);
-    return Name.substr(0, Name.size() - 4);
+    return Name;
   }
-  if (Name.ends_with("_e32")) {
+  if (Name.consume_back("_e32")) {
     setForcedEncodingSize(32);
-    return Name.substr(0, Name.size() - 4);
+    return Name;
   }
-  if (Name.ends_with("_dpp")) {
+  if (Name.consume_back("_dpp")) {
     setForcedDPP(true);
-    return Name.substr(0, Name.size() - 4);
+    return Name;
   }
-  if (Name.ends_with("_sdwa")) {
+  if (Name.consume_back("_sdwa")) {
     setForcedSDWA(true);
-    return Name.substr(0, Name.size() - 5);
+    return Name;
   }
   return Name;
 }
@@ -9548,13 +9566,13 @@ void AMDGPUAsmParser::cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
     }
   }
 
-  if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::byte_sel))
+  if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::clamp) && !IsVOP3CvtSrDpp)
     addOptionalImmOperand(Inst, Operands, OptionalIdx,
-                          AMDGPUOperand::ImmTyByteSel);
+                          AMDGPUOperand::ImmTyClamp);
 
-  if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::clamp))
+  if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::byte_sel))
     addOptionalImmOperand(Inst, Operands, OptionalIdx,
-                          AMDGPUOperand::ImmTyClamp);
+                          AMDGPUOperand::ImmTyByteSel);
 
   if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::omod))
     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 445ba9a26d336..e219fe05f881b 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -354,6 +354,20 @@ class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,
   let gdsValue = 1;
 }
 
+class DS_1A_Off16_NORET <string opName>
+: DS_Pseudo<opName,
+  (outs),
+  (ins VGPR_32:$addr, Offset:$offset, gds:$gds),
+  " $addr$offset$gds"> {
+
+  let has_vdst = 0;
+  let has_offset = 1;
+  let has_data0 = 0;
+  let has_data1 = 0;
+  let has_m0_read = 0;
+  let IsAtomicNoRet = 1;
+}
+
 class DS_0A_RET <string opName> : DS_Pseudo<opName,
   (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
   (ins Offset:$offset, gds:$gds),
@@ -789,12 +803,30 @@ let WaveSizePredicate = isWave32, mayStore = 0 in {
 let OtherPredicates = [HasTransposeLoadF4F6Insts] in {
 defm DS_LOAD_TR4_B64   : DS_1A_RET_NoM0<"ds_load_tr4_b64",   VReg_64>;
 defm DS_LOAD_TR6_B96   : DS_1A_RET_NoM0<"ds_load_tr6_b96",   VReg_96>;
-} // let OtherPredicates = [HasTransposeLoadF4F6Insts]
+} // End OtherPredicates = [HasTransposeLoadF4F6Insts]
 defm DS_LOAD_TR8_B64   : DS_1A_RET_NoM0<"ds_load_tr8_b64",   VReg_64>;
 defm DS_LOAD_TR16_B128 : DS_1A_RET_NoM0<"ds_load_tr16_b128", VReg_128>;
-} // let WaveSizePredicate = isWave32, mayStore = 0
+} // End WaveSizePredicate = isWave32, mayStore = 0
 
-} // let SubtargetPredicate = isGFX1250Plus
+let OtherPredicates = [HasLdsBarrierArriveAtomic] in {
+let ASYNC_CNT = 1, LGKM_CNT = 0, Uses = [EXEC, ASYNCcnt], Defs = [ASYNCcnt] in {
+def DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_1A_Off16_NORET<"ds_atomic_async_barrier_arrive_b64">;
+}
+
+def : GCNPat <
+  (int_amdgcn_ds_atomic_async_barrier_arrive_b64 (DS1Addr1Offset i32:$ptr, i32:$offset)),
+  (DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 VGPR_32:$ptr, Offset:$offset, (i1 0))
+>;
+
+defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 : DS_1A1D_RET_mc_gfx9<"ds_atomic_barrier_arrive_rtn_b64", VReg_64>;
+
+def : GCNPat<
+  (i64 (int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 (DS1Addr1Offset i32:$ptr, i32:$offset), i64:$data)),
+  (DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 $ptr, $data, Offset:$offset, (i1 0))
+>;
+} // End OtherPredicates = [HasLdsBarrierArriveAtomic]
+
+} // End SubtargetPredicate = isGFX1250Plus
 
 let WaveSizePredicate = isWave64, SubtargetPredicate = HasGFX950Insts, mayStore = 0 in {
   defm DS_READ_B64_TR_B4  : DS_1A_RET_NoM0<"ds_read_b64_tr_b4", VReg_64>;
@@ -1276,6 +1308,17 @@ class DSLoadTrPat <DS_Pseudo inst, ValueType vt, SDPatternOperator node> : GCNPa
   (inst $ptr, Offset:$offset, (i1 0))
 >;
 
+let WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus in {
+let OtherPredicates = [HasTransposeLoadF4F6Insts] in {
+  def : DSLoadTrPat <DS_LOAD_TR4_B64, v2i32, int_amdgcn_ds_load_tr4_b64>;
+  def : DSLoadTrPat <DS_LOAD_TR6_B96, v3i32, int_amdgcn_ds_load_tr6_b96>;
+} // End OtherPredicates = [HasTransposeLoadF4F6Insts]
+
+  def : DSLoadTrPat <DS_LOAD_TR8_B64, v2i32, int_amdgcn_ds_load_tr8_b64>;
+  foreach vt = [v8i16, v8f16, v8bf16] in
+    def : DSLoadTrPat <DS_LOAD_TR16_B128, vt, int_amdgcn_ds_load_tr16_b128>;
+} // End WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus
+
 let SubtargetPredicate = HasGFX950Insts in {
   def : DSLoadTrPat <DS_READ_B64_TR_B4,  v2i32, int_amdgcn_ds_read_tr4_b64>;
   def : DSLoadTrPat <DS_READ_B64_TR_B8,  v2i32, int_amdgcn_ds_read_tr8_b64>;
@@ -1315,14 +1358,14 @@ class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef,
 // GFX12.
 //===----------------------------------------------------------------------===//
 
-multiclass DS_Real_gfx12<bits<8> op, string name = !tolower(NAME), bit needAlias = true> {
+multiclass DS_Real_gfx12<bits<8> op, string name = !tolower(NAME)> {
   defvar ps = !cast<DS_Pseudo>(NAME);
   let AssemblerPredicate = isGFX12Plus in {
     let DecoderNamespace = "GFX12" in
       def _gfx12 :
         Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12,
                                                name, /*hasGDS=*/false>;
-    if !and(needAlias, !ne(ps.Mnemonic, name)) then
+    if !ne(ps.Mnemonic, name) then
       def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;
   } // End AssemblerPredicate
 }
@@ -1351,10 +1394,15 @@ defm DS_LOAD_TR16_B128    : DS_Real_gfx12<0x0fc>;
 defm DS_LOAD_TR8_B64      : DS_Real_gfx12<0x0fd>;
 
 defm DS_BVH_STACK_RTN_B32             : DS_Real_gfx12<0x0e0,
-  "ds_bvh_stack_push4_pop1_rtn_b32", true>;
+  "ds_bvh_stack_push4_pop1_rtn_b32">;
 defm DS_BVH_STACK_PUSH8_POP1_RTN_B32  : DS_Real_gfx12<0x0e1>;
 defm DS_BVH_STACK_PUSH8_POP2_RTN_B64  : DS_Real_gfx12<0x0e2>;
 
+let AssemblerPredicate = HasLdsBarrierArriveAtomic in {
+defm DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_Real_gfx12<0x056>;
+defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64   : DS_Real_gfx12<0x075>;
+}
+
 // New aliases added in GFX12 without renaming the instructions.
 let AssemblerPredicate = isGFX12Plus in {
   def : AMDGPUMnemonicAlias<"ds_subrev_u32", "ds_rsub_u32">;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c4db88b6e5105..3625db9a4791f 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -208,21 +208,26 @@ class GlobalSaddrTable <bit is_saddr, string Name = ""> {
 // TODO: Is exec allowed for saddr? The disabled value 0x7f is the
 // same encoding value as exec_hi, so it isn't possible to use that if
 // saddr is 32-bit (which isn't handled here yet).
-class FLAT_Load_Pseudo <string opName, RegisterClass regClass,
-  bit HasTiedOutput = 0,
-  bit HasSaddr = 0, bit EnableSaddr = 0,
-  RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo<
-  opName,
-  (outs vdata_op:$vdst),
-  !con(
+class FLAT_Load_Pseudo<
+    string opName, RegisterClass regClass, bit HasTiedOutput = 0,
+    bit HasSaddr = 0, bit EnableSaddr = 0>
+    : FLAT_Pseudo<opName, (outs), (ins), ""> {
+
+  defvar vdata_op = getLdStRegisterOperand<regClass>.ret;
+  let OutOperandList = (outs vdata_op:$vdst);
+  let InOperandList = !con(
     !if(EnableSaddr,
-      (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr),
-      (ins VReg_64:$vaddr)),
-      (ins flat_offset:$offset),
-      // FIXME: Operands with default values do not work with following non-optional operands.
-      !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),
-                         (ins CPol_0:$cpol))),
-  " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> {
+        (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr),
+        (ins VReg_64:$vaddr)),
+    (ins flat_offset:$offset),
+    // FIXME: Operands with default values do not work with following
+    // non-optional operands.
+    !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),
+                       (ins CPol_0:$cpol)));
+  let AsmOperands = " $vdst, $vaddr"
+    # !if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")
+    # "$offset$cpol";
+
   let has_data = 0;
   let mayLoad = 1;
   let has_saddr = HasSaddr;
@@ -387,6 +392,23 @@ multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass
     GlobalSaddrTable<1, opName>;
 }
 
+class FLAT_Global_Tensor_Pseudo<string opName, bit EnableSaddr = 0> : FLAT_Pseudo<
+  opName,
+  (outs ),
+  !con(!if(EnableSaddr, (ins SReg_64:$saddr, flat_offset:$offset), (ins )), (ins CPol_0:$cpol)),
+  !if(EnableSaddr, " $saddr$offset", " ")#"$cpol"> {
+
+  let is_flat_global = 1;
+  let has_vdst = 0;
+  let has_data = 0;
+  let has_vaddr = 0;
+  let mayLoad = 0;
+  let mayStore = 1;
+  let has_saddr = 1;
+  let enabled_saddr = EnableSaddr;
+  let has_offset = EnableSaddr;
+}
+
 class FLAT_Global_Invalidate_Writeback<string opName, SDPatternOperator node = null_frag> :
   FLAT_Pseudo<opName, (outs), (ins CPol:$cpol), "$cpol", [(node)]> {
 
@@ -1047,6 +1069,12 @@ let SubtargetPredicate = isGFX12Plus in {
   def GLOBAL_WBINV  : FLAT_Global_Invalidate_Writeback<"global_wbinv">;
 } // End SubtargetPredicate = isGFX12Plus
 
+let SubtargetPredicate = isGFX1250Plus in {
+
+def TENSOR_SAVE : FLAT_Global_Tensor_Pseudo<"tensor_save", 1>;
+def TENSOR_STOP : FLAT_Global_Tensor_Pseudo<"tensor_stop">;
+} // End SubtargetPredicate = isGFX1250Plus
+
 defm SCRATCH_LOAD_UBYTE    : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>;
 defm SCRATCH_LOAD_SBYTE    : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>;
 defm SCRATCH_LOAD_USHORT   : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>;
@@ -1848,6 +1876,11 @@ let WaveSizePredicate = isWave64, OtherPredicates = [isGFX12PlusNot12_50] in {
     defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, vt>;
 }
 
+let WaveSizePredicate = isWave32,  OtherPredicates = [HasTransposeLoadF4F6Insts] in {
+  defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR4_B64, int_amdgcn_global_load_tr4_b64, v2i32>;
+  defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR6_B96, int_amdgcn_global_load_tr6_b96, v3i32>;
+}
+
 let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts, OtherPredicates = [HasFlatGlobalInsts] in {
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>;
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>;
@@ -3051,6 +3084,9 @@ multiclass VFLAT_Real_Atomics_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME
   VFLAT_Real_RTN_gfx1250<op, name>,
   VFLAT_Real_SADDR_RTN_gfx1250<op, name>;
 
+defm TENSOR_SAVE                      : VFLAT_Real_gfx1250<0x06e>;
+defm TENSOR_STOP                      : VFLAT_Real_gfx1250<0x06f>;
+
 defm GLOBAL_LOAD_TR_B128_w32          : VFLAT_Real_AllAddr_gfx1250<0x057, "global_load_tr16_b128">;
 defm GLOBAL_LOAD_TR_B64_w32           : VFLAT_Real_AllAddr_gfx1250<0x058, "global_load_tr8_b64">;
 
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index bc95d3f040e1d..0976fccf78d86 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -44,6 +44,11 @@ static cl::opt<unsigned, false, MFMAPaddingRatioParser>
                      cl::desc("Fill a percentage of the latency between "
                               "neighboring MFMA with s_nops."));
 
+// This is intended for debugging purposes only.
+static cl::opt<unsigned>
+    NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden,
+               cl::desc("Insert a s_nop x before every instruction"));
+
 //===----------------------------------------------------------------------===//
 // Hazard Recognizer Implementation
 //===----------------------------------------------------------------------===//
@@ -300,7 +305,7 @@ unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
   unsigned W = PreEmitNoopsCommon(MI);
   fixHazards(MI);
   CurrCycleInstr = nullptr;
-  return W;
+  return std::max(W, NopPadding.getValue());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 2f79599091faf..fa1209db2fa07 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -165,6 +165,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasMAIInsts = false;
   bool HasFP8Insts = false;
   bool HasFP8ConversionInsts = false;
+  bool HasFP8E5M3Insts = false;
   bool HasCvtFP8Vop1Bug = false;
   bool HasPkFmacF16Inst = false;
   bool HasAtomicFMinFMaxF32GlobalInsts = false;
@@ -258,11 +259,13 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasRequiredExportPriority = false;
   bool HasVmemWriteVgprInOrder = false;
   bool HasAshrPkInsts = false;
+  bool HasIEEEMinimumMaximumInsts = false;
   bool HasMinimum3Maximum3F32 = false;
   bool HasMinimum3Maximum3F16 = false;
   bool HasMinimum3Maximum3PKF16 = false;
   bool HasLshlAddU64Inst = false;
   bool HasPointSampleAccel = false;
+  bool HasLdsBarrierArriveAtomic = false;
   bool HasSetPrioIncWgInst = false;
 
   bool RequiresCOV6 = false;
@@ -860,6 +863,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool hasFP8ConversionInsts() const { return HasFP8ConversionInsts; }
 
+  bool hasFP8E5M3Insts() const { return HasFP8E5M3Insts; }
+
   bool hasPkFmacF16Inst() const {
     return HasPkFmacF16Inst;
   }
@@ -1381,6 +1386,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool hasPointSampleAccel() const { return HasPointSampleAccel; }
 
+  bool hasLdsBarrierArriveAtomic() const { return HasLdsBarrierArriveAtomic; }
+
   /// \returns The maximum number of instructions that can be enclosed in an
   /// S_CLAUSE on the given subtarget, or 0 for targets that do not support that
   /// instruction.
@@ -1460,10 +1467,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool hasIEEEMode() const { return getGeneration() < GFX12; }
 
   // \returns true if the target has IEEE fminimum/fmaximum instructions
-  bool hasIEEEMinMax() const { return getGeneration() >= GFX12; }
-
-  // \returns true if the target has IEEE fminimum3/fmaximum3 instructions
-  bool hasIEEEMinMax3() const { return hasIEEEMinMax(); }
+  bool hasIEEEMinimumMaximumInsts() const { return HasIEEEMinimumMaximumInsts; }
 
   // \returns true if the target has WG_RR_MODE kernel descriptor mode bit
   bool hasRrWGMode() const { return getGeneration() >= GFX12; }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index 78bd8de07306f..8f89168754180 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -15,7 +15,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -41,7 +40,7 @@ class AMDGPUAsmBackend : public MCAsmBackend {
   void relaxInstruction(MCInst &Inst,
                         const MCSubtargetInfo &STI) const override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override;
 
   unsigned getMinimumNopSize() const override;
@@ -50,7 +49,6 @@ class AMDGPUAsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
-  bool shouldForceRelocation(const MCFixup &, const MCValue &) override;
 };
 
 } //End anonymous namespace
@@ -72,12 +70,13 @@ bool AMDGPUAsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
   return (((int64_t(Value)/4)-1) == 0x3f);
 }
 
-bool AMDGPUAsmBackend::mayNeedRelaxation(const MCInst &Inst,
-                       const MCSubtargetInfo &STI) const {
+bool AMDGPUAsmBackend::mayNeedRelaxation(unsigned Opcode,
+                                         ArrayRef<MCOperand> Operands,
+                                         const MCSubtargetInfo &STI) const {
   if (!STI.hasFeature(AMDGPU::FeatureOffset3fBug))
     return false;
 
-  if (AMDGPU::getSOPPWithRelaxation(Inst.getOpcode()) >= 0)
+  if (AMDGPU::getSOPPWithRelaxation(Opcode) >= 0)
     return true;
 
   return false;
@@ -95,7 +94,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
     return 2;
   case FK_SecRel_4:
   case FK_Data_4:
-  case FK_PCRel_4:
     return 4;
   case FK_SecRel_8:
   case FK_Data_8:
@@ -122,7 +120,6 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
   case FK_Data_2:
   case FK_Data_4:
   case FK_Data_8:
-  case FK_PCRel_4:
   case FK_SecRel_4:
     return Value;
   default:
@@ -130,10 +127,13 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
   }
 }
 
-void AMDGPUAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void AMDGPUAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                   const MCValue &Target,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved) {
+  if (Target.getSpecifier())
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   if (mc::isRelocation(Fixup.getKind()))
     return;
 
@@ -173,12 +173,12 @@ AMDGPUAsmBackend::getFixupKind(StringRef Name) const {
 
 MCFixupKindInfo AMDGPUAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[AMDGPU::NumTargetFixupKinds] = {
-    // name                   offset bits  flags
-    { "fixup_si_sopp_br",     0,     16,   MCFixupKindInfo::FKF_IsPCRel },
+      // name                   offset bits  flags
+      {"fixup_si_sopp_br", 0, 16, 0},
   };
 
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -188,11 +188,6 @@ MCFixupKindInfo AMDGPUAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   return Infos[Kind - FirstTargetFixupKind];
 }
 
-bool AMDGPUAsmBackend::shouldForceRelocation(const MCFixup &,
-                                             const MCValue &Target) {
-  return Target.getSpecifier();
-}
-
 unsigned AMDGPUAsmBackend::getMinimumNopSize() const {
   return 4;
 }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
index 9a3f418e0d527..22ae5f4e71915 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
@@ -69,8 +69,6 @@ unsigned AMDGPUELFObjectWriter::getRelocType(const MCFixup &Fixup,
   MCFixupKind Kind = Fixup.getKind();
   switch (Kind) {
   default: break;
-  case FK_PCRel_4:
-    return ELF::R_AMDGPU_REL32;
   case FK_Data_4:
   case FK_SecRel_4:
     return IsPCRel ? ELF::R_AMDGPU_REL32 : ELF::R_AMDGPU_ABS32;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index a6ce42dca92be..fa1474d153834 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -173,13 +173,12 @@ void AMDGPUInstPrinter::printTH(const MCInst *MI, int64_t TH, int64_t Scope,
 
   const unsigned Opcode = MI->getOpcode();
   const MCInstrDesc &TID = MII.get(Opcode);
-  bool IsStore = TID.mayStore();
-  bool IsAtomic =
-      TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet);
+  unsigned THType = AMDGPU::getTemporalHintType(TID);
+  bool IsStore = (THType == AMDGPU::CPol::TH_TYPE_STORE);
 
   O << " th:";
 
-  if (IsAtomic) {
+  if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) {
     O << "TH_ATOMIC_";
     if (TH & AMDGPU::CPol::TH_ATOMIC_CASCADE) {
       if (Scope >= AMDGPU::CPol::SCOPE_DEV)
@@ -196,9 +195,6 @@ void AMDGPUInstPrinter::printTH(const MCInst *MI, int64_t TH, int64_t Scope,
     if (!IsStore && TH == AMDGPU::CPol::TH_RESERVED)
       O << formatHex(TH);
     else {
-      // This will default to printing load variants when neither MayStore nor
-      // MayLoad flag is present which is the case with instructions like
-      // image_get_resinfo.
       O << (IsStore ? "TH_STORE_" : "TH_LOAD_");
       switch (TH) {
       case AMDGPU::CPol::TH_NT:
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index abb5003f3320b..086f980f5fe23 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -103,6 +103,11 @@ MCCodeEmitter *llvm::createAMDGPUMCCodeEmitter(const MCInstrInfo &MCII,
   return new AMDGPUMCCodeEmitter(MCII, *Ctx.getRegisterInfo());
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // Returns the encoding value to use if the given integer is an integer inline
 // immediate value, or 0 if it is not.
 template <typename IntTy>
@@ -445,8 +450,7 @@ void AMDGPUMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
 
   if (MO.isExpr()) {
     const MCExpr *Expr = MO.getExpr();
-    MCFixupKind Kind = (MCFixupKind)AMDGPU::fixup_si_sopp_br;
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+    addFixup(Fixups, 0, Expr, AMDGPU::fixup_si_sopp_br, true);
     Op = APInt::getZero(96);
   } else {
     getMachineOpValue(MI, MO, Op, Fixups, STI);
@@ -652,17 +656,11 @@ void AMDGPUMCCodeEmitter::getMachineOpValueCommon(
     //
     // .Ltmp1:
     //   s_add_u32 s2, s2, (extern_const_addrspace+16)-.Ltmp1
-    MCFixupKind Kind;
-    if (needsPCRel(MO.getExpr()))
-      Kind = FK_PCRel_4;
-    else
-      Kind = FK_Data_4;
-
+    bool PCRel = needsPCRel(MO.getExpr());
     const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
     uint32_t Offset = Desc.getSize();
     assert(Offset == 4 || Offset == 8);
-
-    Fixups.push_back(MCFixup::create(Offset, MO.getExpr(), Kind, MI.getLoc()));
+    addFixup(Fixups, Offset, MO.getExpr(), FK_Data_4, PCRel);
   }
 
   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
index 8195c93d847b0..02a2b06d0951c 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
@@ -166,7 +166,7 @@ uint64_t R600MCCodeEmitter::getMachineOpValue(const MCInst &MI,
     // We can't easily get the order of the current one, so compare against
     // the first one and adjust offset.
     const unsigned offset = (&MO == &MI.getOperand(0)) ? 0 : 4;
-    Fixups.push_back(MCFixup::create(offset, MO.getExpr(), FK_SecRel_4, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(offset, MO.getExpr(), FK_SecRel_4));
     return 0;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 8d94d73bc1aab..531fae3ceff59 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -2019,3 +2019,97 @@ def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O_nortn, IMAGE_SAMPLE_CD_O_G16_nortn>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O_nortn, IMAGE_SAMPLE_CD_CL_O_G16_nortn>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O_nortn, IMAGE_SAMPLE_C_CD_O_G16_nortn>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O_nortn, IMAGE_SAMPLE_C_CD_CL_O_G16_nortn>;
+
+//===----------------------------------------------------------------------===//
+// VIMAGE Tensor Instructions
+//===----------------------------------------------------------------------===//
+
+class VIMAGE_TENSOR_Pseudo<string opName, bit _UpTo2D = 0> :
+  InstSI<(outs ), (ins ), "", []>,
+  SIMCInstr<opName#!if(_UpTo2D, "_D2", ""), SIEncodingFamily.NONE> {
+
+  let isPseudo = 1;
+  let isCodeGenOnly = 1;
+  string Mnemonic = opName;
+
+  let VALU = 1;
+  let maybeAtomic = 0;
+  let TENSOR_CNT = 1;
+  let mayLoad = 1;
+  let mayStore = 1;
+  let Uses = [EXEC, TENSORcnt];
+  let Defs = [TENSORcnt];
+  let SchedRW = [WriteVMEM, WriteLDS];
+  let UseNamedOperandTable = 1;
+  let hasSideEffects = 0;
+
+  bit UpTo2D = _UpTo2D;
+  let InOperandList = !if(UpTo2D, (ins SReg_128:$vaddr0, SReg_256:$vaddr1, R128A16:$r128, CPol:$cpol),
+                                      (ins SReg_128:$vaddr0, SReg_256:$vaddr1, SReg_128:$vaddr2,
+                                       SReg_128:$vaddr3, R128A16:$r128, CPol:$cpol));
+  string AsmOperands = " $vaddr0, $vaddr1"#!if(UpTo2D, "", ", $vaddr2, $vaddr3")#"$r128$cpol";
+}
+
+let SubtargetPredicate = isGFX1250Plus in {
+def TENSOR_LOAD_TO_LDS    : VIMAGE_TENSOR_Pseudo<"tensor_load_to_lds">;
+def TENSOR_STORE_FROM_LDS : VIMAGE_TENSOR_Pseudo<"tensor_store_from_lds">;
+def TENSOR_LOAD_TO_LDS_D2    : VIMAGE_TENSOR_Pseudo<"tensor_load_to_lds", 1>;
+def TENSOR_STORE_FROM_LDS_D2 : VIMAGE_TENSOR_Pseudo<"tensor_store_from_lds", 1>;
+} // End SubtargetPredicate = isGFX1250Plus.
+
+class TensorPat <VIMAGE_TENSOR_Pseudo inst, SDPatternOperator node> : GCNPat <
+  (node v4i32:$vaddr0, v8i32:$vaddr1, v4i32:$vaddr2, v4i32:$vaddr3, (i32 timm:$cpol)),
+  (inst $vaddr0, $vaddr1, $vaddr2, $vaddr3, 0, $cpol)
+>;
+
+class TensorD2Pat <VIMAGE_TENSOR_Pseudo inst, SDPatternOperator node> : GCNPat <
+  (node v4i32:$vaddr0, v8i32:$vaddr1, (i32 timm:$cpol)),
+  (inst $vaddr0, $vaddr1, 0, $cpol)
+>;
+
+let SubtargetPredicate = isGFX1250Plus in {
+def : TensorPat   <TENSOR_LOAD_TO_LDS, int_amdgcn_tensor_load_to_lds>;
+def : TensorPat   <TENSOR_STORE_FROM_LDS, int_amdgcn_tensor_store_from_lds>;
+def : TensorD2Pat <TENSOR_LOAD_TO_LDS_D2, int_amdgcn_tensor_load_to_lds_d2>;
+def : TensorD2Pat <TENSOR_STORE_FROM_LDS_D2, int_amdgcn_tensor_store_from_lds_d2>;
+}
+
+class VIMAGE_TENSOR_Real <bits<8> op, VIMAGE_TENSOR_Pseudo ps, string opName = ps.Mnemonic> :
+  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>,
+  VIMAGEe<op> {
+
+  // copy relevant pseudo op flags
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let TSFlags              = ps.TSFlags;
+  let mayLoad              = ps.mayLoad;
+  let mayStore             = ps.mayStore;
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SchedRW              = ps.SchedRW;
+
+  // D# group 2 and 3 set to NULL for 2D or less.
+  let vaddr2 = !if(ps.UpTo2D, !cast<int>(SGPR_NULL_gfx11plus.HWEncoding), ?);
+  let vaddr3 = !if(ps.UpTo2D, !cast<int>(SGPR_NULL_gfx11plus.HWEncoding), ?);
+
+  // set to 0 based on SPG.
+  let vaddr4 = 0;
+  let rsrc = 0;
+  let vdata = 0;
+  let d16 = 0;
+  let a16 = 0;
+  let tfe = 0;
+  let dmask = 1; // sp3
+  let dim = 1;   // sp3
+}
+
+multiclass VIMAGE_TENSOR_Real_gfx1250<bits<8> op> {
+  let AssemblerPredicate = isGFX1250Plus, DecoderNamespace = "GFX1250" in {
+    foreach DSuffix = ["_D2", ""] in {
+      defvar ps = !cast<VIMAGE_TENSOR_Pseudo>(NAME # DSuffix);
+      def DSuffix # _gfx1250 : VIMAGE_TENSOR_Real<op, ps, ps.Mnemonic>,
+                SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX1250>;
+    }
+  }
+}
+
+defm TENSOR_LOAD_TO_LDS    : VIMAGE_TENSOR_Real_gfx1250<0xc4>;
+defm TENSOR_STORE_FROM_LDS : VIMAGE_TENSOR_Real_gfx1250<0xc5>;
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index baf74dbdde20e..76e29e4393206 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -107,14 +107,12 @@ enum : uint64_t {
   DisableWQM = UINT64_C(1) << 36,
   Gather4 = UINT64_C(1) << 37,
 
-  // Reserved, must be 0.
-  Reserved0 = UINT64_C(1) << 38,
+  TENSOR_CNT = UINT64_C(1) << 38,
 
   SCALAR_STORE = UINT64_C(1) << 39,
   FIXED_SIZE = UINT64_C(1) << 40,
 
-  // Reserved, must be 0.
-  Reserved1 = UINT64_C(1) << 41,
+  ASYNC_CNT = UINT64_C(1) << 41,
 
   VOP3_OPSEL = UINT64_C(1) << 42,
   maybeAtomic = UINT64_C(1) << 43,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8d7dcf8c4a064..0f70792cc03f4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -877,7 +877,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   if (Subtarget->hasPrefetch() && Subtarget->hasSafeSmemPrefetch())
     setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
-  if (Subtarget->hasIEEEMinMax()) {
+  if (Subtarget->hasIEEEMinimumMaximumInsts()) {
     setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM},
                        {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal);
   } else {
@@ -1404,6 +1404,19 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
     return true;
   }
+  case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
+  case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
+    Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
+                   ? ISD::INTRINSIC_W_CHAIN
+                   : ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::getVT(CI.getType());
+    Info.ptrVal = CI.getOperand(0);
+    Info.memVT = MVT::i64;
+    Info.size = 8;
+    Info.align.reset();
+    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+    return true;
+  }
   case Intrinsic::amdgcn_global_atomic_csub: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(CI.getType());
@@ -1444,6 +1457,12 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                   MachineMemOperand::MOVolatile;
     return true;
   }
+  case Intrinsic::amdgcn_ds_load_tr6_b96:
+  case Intrinsic::amdgcn_ds_load_tr4_b64:
+  case Intrinsic::amdgcn_ds_load_tr8_b64:
+  case Intrinsic::amdgcn_ds_load_tr16_b128:
+  case Intrinsic::amdgcn_global_load_tr6_b96:
+  case Intrinsic::amdgcn_global_load_tr4_b64:
   case Intrinsic::amdgcn_global_load_tr_b64:
   case Intrinsic::amdgcn_global_load_tr_b128:
   case Intrinsic::amdgcn_ds_read_tr4_b64:
@@ -1548,12 +1567,18 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
   case Intrinsic::amdgcn_atomic_cond_sub_u32:
   case Intrinsic::amdgcn_ds_append:
   case Intrinsic::amdgcn_ds_consume:
+  case Intrinsic::amdgcn_ds_load_tr8_b64:
+  case Intrinsic::amdgcn_ds_load_tr16_b128:
+  case Intrinsic::amdgcn_ds_load_tr4_b64:
+  case Intrinsic::amdgcn_ds_load_tr6_b96:
   case Intrinsic::amdgcn_ds_read_tr4_b64:
   case Intrinsic::amdgcn_ds_read_tr6_b96:
   case Intrinsic::amdgcn_ds_read_tr8_b64:
   case Intrinsic::amdgcn_ds_read_tr16_b64:
   case Intrinsic::amdgcn_ds_ordered_add:
   case Intrinsic::amdgcn_ds_ordered_swap:
+  case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
+  case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
   case Intrinsic::amdgcn_flat_atomic_fmax_num:
   case Intrinsic::amdgcn_flat_atomic_fmin_num:
   case Intrinsic::amdgcn_global_atomic_csub:
@@ -1562,6 +1587,8 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
   case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
   case Intrinsic::amdgcn_global_load_tr_b64:
   case Intrinsic::amdgcn_global_load_tr_b128:
+  case Intrinsic::amdgcn_global_load_tr4_b64:
+  case Intrinsic::amdgcn_global_load_tr6_b96:
     Ptr = II->getArgOperand(0);
     break;
   case Intrinsic::amdgcn_load_to_lds:
@@ -7102,7 +7129,8 @@ SDValue SITargetLowering::lowerFMINIMUM_FMAXIMUM(SDValue Op,
   if (VT.isVector())
     return splitBinaryVectorOp(Op, DAG);
 
-  assert(!Subtarget->hasIEEEMinMax() && !Subtarget->hasMinimum3Maximum3F16() &&
+  assert(!Subtarget->hasIEEEMinimumMaximumInsts() &&
+         !Subtarget->hasMinimum3Maximum3F16() &&
          Subtarget->hasMinimum3Maximum3PKF16() && VT == MVT::f16 &&
          "should not need to widen f16 minimum/maximum to v2f16");
 
@@ -14015,7 +14043,7 @@ SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
   // operand form.
   const SDNodeFlags Flags = N->getFlags();
   if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) &&
-      !Subtarget->hasIEEEMinMax() && Flags.hasNoNaNs()) {
+      !Subtarget->hasIEEEMinimumMaximumInsts() && Flags.hasNoNaNs()) {
     unsigned NewOpc =
         Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
     return DAG.getNode(NewOpc, SDLoc(N), VT, Op0, Op1, Flags);
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 69ea8aa6122aa..7ce1359f03da6 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -197,18 +197,22 @@ static bool isNormalMode(InstCounterType MaxCounter) {
 
 VmemType getVmemType(const MachineInstr &Inst) {
   assert(updateVMCntOnly(Inst));
-  if (!SIInstrInfo::isMIMG(Inst) && !SIInstrInfo::isVIMAGE(Inst) &&
-      !SIInstrInfo::isVSAMPLE(Inst))
+  if (!SIInstrInfo::isImage(Inst))
     return VMEM_NOSAMPLER;
   const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode());
   const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
       AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
+
+  if (BaseInfo->BVH)
+    return VMEM_BVH;
+
   // We have to make an additional check for isVSAMPLE here since some
   // instructions don't have a sampler, but are still classified as sampler
   // instructions for the purposes of e.g. waitcnt.
-  bool HasSampler =
-      BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(Inst);
-  return BaseInfo->BVH ? VMEM_BVH : HasSampler ? VMEM_SAMPLER : VMEM_NOSAMPLER;
+  if (BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(Inst))
+    return VMEM_SAMPLER;
+
+  return VMEM_NOSAMPLER;
 }
 
 unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) {
@@ -1786,8 +1790,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
                                                  bool FlushVmCnt) {
   setForceEmitWaitcnt();
 
-  if (MI.isMetaInstruction())
-    return false;
+  assert(!MI.isMetaInstruction());
 
   AMDGPU::Waitcnt Wait;
 
@@ -2474,6 +2477,10 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
                                          E = Block.instr_end();
        Iter != E;) {
     MachineInstr &Inst = *Iter;
+    if (Inst.isMetaInstruction()) {
+      ++Iter;
+      continue;
+    }
 
     // Track pre-existing waitcnts that were added in earlier iterations or by
     // the memory legalizer.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrFormats.td b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
index 267c9a94b9096..c27d4e0df6fc5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrFormats.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
@@ -68,6 +68,9 @@ class InstSI <dag outs, dag ins, string asm = "",
 
   field bit Gather4 = 0;
 
+  // wait count to manage tensor loads/stores.
+  field bit TENSOR_CNT = 0;
+
   // This is an s_store_dword* instruction that requires a cache flush
   // on wave termination. It is necessary to distinguish from mayStore
   // SMEM instructions like the cache flush ones.
@@ -77,6 +80,9 @@ class InstSI <dag outs, dag ins, string asm = "",
   // instruction size.
   field bit FixedSize = 0;
 
+  // wait count to manage asynchronous loads/stores.
+  field bit ASYNC_CNT = 0;
+
   // This bit indicates that this is a VOP3 opcode which supports op_sel
   // modifier.
   field bit VOP3_OPSEL = 0;
@@ -198,14 +204,12 @@ class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{36} = DisableWQM;
   let TSFlags{37} = Gather4;
 
-  // Reserved, must be 0.
-  let TSFlags{38} = 0;
+  let TSFlags{38} = TENSOR_CNT;
 
   let TSFlags{39} = ScalarStore;
   let TSFlags{40} = FixedSize;
 
-  // Reserved, must be 0.
-  let TSFlags{41} = 0;
+  let TSFlags{41} = ASYNC_CNT;
 
   let TSFlags{42} = VOP3_OPSEL;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 5962556db62eb..2d37451eb32b9 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -387,7 +387,10 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
       if (DataOpIdx == -1)
         DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
-      Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
+      if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+        Width = LocationSize::precise(64);
+      else
+        Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     } else {
       // The 2 offset instructions use offset0 and offset1 instead. We can treat
       // these as a load with a single offset if the 2 offsets are consecutive.
@@ -7047,6 +7050,19 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
     return nullptr;
   }
 
+  // Legalize TENSOR_LOAD_TO_LDS, TENSOR_LOAD_TO_LDS_D2, TENSOR_STORE_FROM_LDS,
+  // TENSOR_STORE_FROM_LDS_D2. All their operands are scalar.
+  if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS ||
+      MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_D2 ||
+      MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS ||
+      MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_D2) {
+    for (MachineOperand &Src : MI.explicit_operands()) {
+      if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
+        Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
+    }
+    return CreatedBB;
+  }
+
   // Legalize MUBUF instructions.
   bool isSoffsetLegal = true;
   int SoffsetIdx =
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 01dd3c9f4119e..9e84822bfc273 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1448,8 +1448,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
                         Align Alignment = Align(4)) const;
 
   /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
-  /// encoded instruction. If \p Signed, this is for an instruction that
-  /// interprets the offset as signed.
+  /// encoded instruction with the given \p FlatVariant.
   bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
                          uint64_t FlatVariant) const;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 7b45023dd3c77..5694d72751f84 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3956,21 +3956,21 @@ let True16Predicate = UseFakeTrue16Insts in {
 }
 } // End SubtargetPredicate = [isGFX9Plus]
 
-let SubtargetPredicate = isGFX12Plus in {
+let SubtargetPredicate = HasIEEEMinimumMaximumInsts in {
 def : FPMinMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 }
 
-let True16Predicate = UseRealTrue16Insts, SubtargetPredicate = isGFX12Plus in {
+let True16Predicate = UseRealTrue16Insts, SubtargetPredicate = HasIEEEMinimumMaximumInsts in {
 def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_t16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_t16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_t16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_t16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 }
 
-let True16Predicate = UseFakeTrue16Insts, SubtargetPredicate = isGFX12Plus in {
+let True16Predicate = UseFakeTrue16Insts, SubtargetPredicate = HasIEEEMinimumMaximumInsts in {
 def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
 def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
 def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 8c3873d23419f..6754be1a0b619 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -656,6 +656,10 @@ BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE);
   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
 
+  // Reserve async counters pseudo registers
+  reserveRegisterTuples(Reserved, AMDGPU::ASYNCcnt);
+  reserveRegisterTuples(Reserved, AMDGPU::TENSORcnt);
+
   // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen.
   reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID);
 
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index d595163f820cb..a6c7b164c8b2c 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -234,6 +234,10 @@ defm SRC_POPS_EXITING_WAVE_ID : SIRegLoHi16<"src_pops_exiting_wave_id", 239>;
 // Not addressable
 def MODE : SIReg <"mode", 0>;
 
+// Not addressable, used to model dependencies.
+def ASYNCcnt : SIReg <"ASYNCcnt", 0>;
+def TENSORcnt : SIReg <"TENSORcnt", 0>;
+
 def LDS_DIRECT : SIReg <"src_lds_direct", 254> {
   // There is no physical register corresponding to this. This is an
   // encoding value in a source field, which will ultimately trigger a
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 0eb55daf32089..fd39b8a1350c6 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -842,10 +842,7 @@ bool SIShrinkInstructions::run(MachineFunction &MF) {
 
   unsigned VCCReg = ST->isWave32() ? AMDGPU::VCC_LO : AMDGPU::VCC;
 
-  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
-                                                  BI != BE; ++BI) {
-
-    MachineBasicBlock &MBB = *BI;
+  for (MachineBasicBlock &MBB : MF) {
     MachineBasicBlock::iterator I, Next;
     for (I = MBB.begin(); I != MBB.end(); I = Next) {
       Next = std::next(I);
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index de217cc602c98..c7c4276e0e252 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1680,6 +1680,12 @@ def S_SET_GPR_IDX_OFF : SOPP_Pseudo<"s_set_gpr_idx_off", (ins) > {
   let Uses = [MODE];
 }
 }
+
+def S_MONITOR_SLEEP : SOPP_Pseudo <"s_monitor_sleep", (ins i16imm:$simm16), "$simm16",
+  [(int_amdgcn_s_monitor_sleep timm:$simm16)]> {
+  let SubtargetPredicate = isGFX1250Plus;
+}
+
 } // End hasSideEffects
 
 let SubtargetPredicate = HasVGPRIndexMode in {
@@ -2692,6 +2698,12 @@ defm S_ICACHE_INV                 : SOPP_Real_32_gfx11_gfx12<0x03c>;
 
 defm S_BARRIER                    : SOPP_Real_32_gfx11<0x03d>;
 
+//===----------------------------------------------------------------------===//
+// SOPP - GFX1250.
+//===----------------------------------------------------------------------===//
+
+defm S_MONITOR_SLEEP              : SOPP_Real_32_gfx12<0x004>;
+
 //===----------------------------------------------------------------------===//
 // SOPP - GFX6, GFX7, GFX8, GFX9, GFX10
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 13549e5c4e58b..5b21f4a6083a5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -684,6 +684,30 @@ bool isGenericAtomic(unsigned Opc) {
          Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
 }
 
+bool isAsyncStore(unsigned Opc) {
+  return false; // placeholder before async store implementation.
+}
+
+bool isTensorStore(unsigned Opc) {
+  return Opc == TENSOR_STORE_FROM_LDS_gfx1250 ||
+         Opc == TENSOR_STORE_FROM_LDS_D2_gfx1250;
+}
+
+unsigned getTemporalHintType(const MCInstrDesc TID) {
+  if (TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet))
+    return CPol::TH_TYPE_ATOMIC;
+  unsigned Opc = TID.getOpcode();
+  // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
+  if (TID.mayStore() &&
+      (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
+    return CPol::TH_TYPE_STORE;
+
+  // This will default to returning TH_TYPE_LOAD when neither MayStore nor
+  // MayLoad flag is present which is the case with instructions like
+  // image_get_resinfo.
+  return CPol::TH_TYPE_LOAD;
+}
+
 bool isTrue16Inst(unsigned Opc) {
   const VOPTrue16Info *Info = getTrue16OpcodeHelper(Opc);
   return Info && Info->IsTrue16;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index e6078d6918ac2..4f7d18170d586 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -891,6 +891,13 @@ LLVM_READONLY
 VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
                                const MCInstrInfo *InstrInfo);
 
+LLVM_READONLY
+bool isAsyncStore(unsigned Opc);
+LLVM_READONLY
+bool isTensorStore(unsigned Opc);
+LLVM_READONLY
+unsigned getTemporalHintType(const MCInstrDesc TID);
+
 LLVM_READONLY
 bool isTrue16Inst(unsigned Opc);
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index 79152e707a25c..e464470143e52 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -898,7 +898,7 @@ bool AMDGPUPALMetadata::setFromString(StringRef S) {
         errs() << "Unrecognized PAL metadata register key '" << S << "'\n";
         continue;
       }
-      Key = MsgPackDoc.getNode(uint64_t(Val));
+      Key = MsgPackDoc.getNode(Val);
     }
     Registers.getMap()[Key] = I.second;
   }
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index d504c8134202d..5e88684c102ce 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -401,8 +401,8 @@ def VOP_PERMLANE_SWAP : VOPProfile<[i32, i32, untyped, untyped]> {
   let HasExtDPP = 0;
   let HasExtSDWA = 0;
 
-  let Ins32 = (ins Src0RC64:$vdst_in, Src0RC32:$src0);
-  let Ins64 = (ins Src0RC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
+  let Ins32 = (ins DstRC:$vdst_in, Src0RC32:$src0);
+  let Ins64 = (ins DstRC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
   let InsVOP3OpSel = (ins Src0RC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
   let Asm64 = "$vdst, $src0$bound_ctrl$fi";
 }
@@ -676,20 +676,32 @@ let HasClamp = 0, HasOMod = 0, HasExtDPP = 0, HasExtVOP3DPP = 0,
   }
 }
 
-class VOPProfile_Base_CVT_F_F8_ByteSel<ValueType DstVT> : VOPProfile<[DstVT, i32, untyped, untyped]> {
+class VOPProfile_Base_CVT_F_F8_ByteSel<ValueType DstVT, bit _HasClamp = 0> :
+      VOPProfile<[DstVT, i32, untyped, untyped]> {
+  let HasClamp = _HasClamp;
   let HasFP8SrcByteSel = 1;
   let HasOpSel = 0;
   let HasExtDPP = 1;
   let HasExtVOP3DPP = 1;
   let HasExtSDWA = 0;
-  let HasClamp = 0;
   let HasOMod = 0;
   let HasModifiers = 0;
 }
 
+let IsSingle = 0, HasOpSel = 1, HasModifiers = 1 in {
+def V_CVT_F16_F8_Profile : VOPProfile_Base_CVT_F_F8_ByteSel<f16>;
+def V_CVT_F16_F8_True16_Profile : VOP3_Profile_True16<V_CVT_F16_F8_Profile>;
+def V_CVT_F16_F8_Fake16_Profile : VOP3_Profile_Fake16<V_CVT_F16_F8_Profile>;
+}
+
 let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts],
     mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
-  defm V_CVT_F32_FP8_OP_SEL    : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
+  // FIXME: This differs from downstream due to changes that haven't been upstreamed yet.
+  let SubtargetPredicate = isGFX12PlusNot12_50 in
+    defm V_CVT_F32_FP8_OP_SEL    : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
+  let SubtargetPredicate = isGFX125xOnly in
+    defm V_CVT_F32_FP8_gfx1250   : VOP1Inst<"v_cvt_f32_fp8_gfx1250", VOPProfile_Base_CVT_F_F8_ByteSel<f32, 1>>;
+
   defm V_CVT_F32_BF8_OP_SEL    : VOP1Inst<"v_cvt_f32_bf8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
 
   let True16Predicate = UseFakeTrue16Insts in {
@@ -702,14 +714,25 @@ let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts],
   }
 }
 
-class Cvt_F_F8_Pat_ByteSel<SDPatternOperator node, VOP3_Pseudo inst> : GCNPat<
+class Cvt_F_F8_Pat_ByteSel<SDPatternOperator node, VOP3_Pseudo inst, bit HasOpSel = 0> : GCNPat<
   (node i32:$src0, timm:$byte_sel),
-  (inst $src0, (as_i32timm $byte_sel))
+  !if(HasOpSel, (inst 0, $src0, (as_i32timm $byte_sel)),
+                (inst $src0, (as_i32timm $byte_sel)))
 >;
 
-let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts] in {
-  def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_fp8, V_CVT_F32_FP8_OP_SEL_e64>;
-  def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
+let OtherPredicates = [HasFP8ConversionInsts] in {
+  // FIXME: This differs from downstream due to changes that haven't been upstreamed yet.
+  let SubtargetPredicate = isGFX12PlusNot12_50 in
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_fp8, V_CVT_F32_FP8_OP_SEL_e64>;
+  let SubtargetPredicate = isGFX125xOnly in {
+    def : GCNPat<(int_amdgcn_cvt_f32_fp8 i32:$src0, timm:$byte_sel),
+                 (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, (as_i32timm $byte_sel))>;
+    def : GCNPat<(int_amdgcn_cvt_f32_fp8_e5m3 i32:$src0, timm:$byte_sel),
+                 (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.ENABLE, (as_i32timm $byte_sel))>;
+  }
+  // FIXME: This differs from downstream due to changes that haven't been upstreamed yet.
+  let SubtargetPredicate = isGFX12Plus in
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
 }
 
 class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
@@ -738,6 +761,10 @@ def VOPProfile_CVT_PK_F16_F8_fake16 : VOP3_Profile_Fake16<VOPProfile_CVT_PK_F16_
 
 let SubtargetPredicate = isGFX1250Plus in {
   let mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
+    defm V_CVT_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_fp8",
+      V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
+    defm V_CVT_F16_BF8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_bf8",
+      V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
     defm V_CVT_PK_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_pk_f16_fp8",
       VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
       int_amdgcn_cvt_pk_f16_fp8>;
@@ -745,6 +772,15 @@ let SubtargetPredicate = isGFX1250Plus in {
       VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
       int_amdgcn_cvt_pk_f16_bf8>;
   }
+
+  let True16Predicate = UseRealTrue16Insts in {
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_fp8, V_CVT_F16_FP8_t16_e64, 1>;
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_t16_e64, 1>;
+  }
+  let True16Predicate = UseFakeTrue16Insts in {
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_fp8, V_CVT_F16_FP8_fake16_e64, 1>;
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_fake16_e64, 1>;
+  }
 } // End SubtargetPredicate = isGFX1250Plus
 
 let SubtargetPredicate = isGFX10Plus in {
@@ -1018,7 +1054,9 @@ multiclass VOP1_Real_FULL_t16_and_fake16_gfx1250<
        VOP1_Real_FULL_with_name<GFX1250Gen, op, opName#"_fake16", asmName>;
 }
 
-defm V_CVT_F32_FP8      : VOP1_Real_FULL_with_name<GFX12Gen, 0x06c, "V_CVT_F32_FP8_OP_SEL", "v_cvt_f32_fp8">;
+defm V_CVT_F32_FP8      : VOP1_Real_FULL_with_name<GFX12Not12_50Gen, 0x06c, "V_CVT_F32_FP8_OP_SEL", "v_cvt_f32_fp8">;
+defm V_CVT_F32_FP8      : VOP1_Real_FULL_with_name<GFX1250Gen, 0x06c, "V_CVT_F32_FP8_gfx1250", "v_cvt_f32_fp8">;
+
 defm V_CVT_F32_BF8      : VOP1_Real_FULL_with_name<GFX12Gen, 0x06d, "V_CVT_F32_BF8_OP_SEL", "v_cvt_f32_bf8">;
 
 defm V_CVT_PK_F32_FP8_fake16 : VOP1_Real_e32_with_name<GFX12Gen, 0x06e, "V_CVT_PK_F32_FP8_fake16", "v_cvt_pk_f32_fp8">;
@@ -1082,6 +1120,8 @@ defm V_CVT_F32_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00b>;
 defm V_CVT_F32_BF16          : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x072, "v_cvt_f32_bf16", "V_CVT_F32_BF16_gfx1250">;
 defm V_CVT_PK_F16_FP8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x075>;
 defm V_CVT_PK_F16_BF8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x076>;
+defm V_CVT_F16_FP8           : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x077>;
+defm V_CVT_F16_BF8           : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x078>;
 
 //===----------------------------------------------------------------------===//
 // GFX10.
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 65d4189f9949d..859d5bae3d460 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1395,14 +1395,18 @@ let SubtargetPredicate = isGFX12Plus, isReMaterializable = 1 in {
       defm V_ADD_F64_pseudo : VOP2Inst <"v_add_f64_pseudo", VOP_F64_F64_F64, any_fadd>;
       defm V_MUL_F64_pseudo : VOP2Inst <"v_mul_f64_pseudo", VOP_F64_F64_F64, fmul>;
     } // End FPDPRounding = 1
-    defm V_MIN_NUM_F64 : VOP2Inst <"v_min_num_f64", VOP_F64_F64_F64, fminnum_like>;
-    defm V_MAX_NUM_F64 : VOP2Inst <"v_max_num_f64", VOP_F64_F64_F64, fmaxnum_like>;
   } // End SchedRW = [WriteDoubleAdd], isCommutable = 1
   let SchedRW = [Write64Bit] in {
     defm V_LSHLREV_B64_pseudo : VOP2Inst <"v_lshlrev_b64_pseudo", VOP_I64_I32_I64, clshl_rev_64>;
   } // End SchedRW = [Write64Bit]
 } // End SubtargetPredicate = isGFX12Plus, isReMaterializable = 1
 
+let SubtargetPredicate = HasIEEEMinimumMaximumInsts, isReMaterializable = 1,
+    SchedRW = [WriteDoubleAdd], isCommutable = 1 in {
+  defm V_MIN_NUM_F64 : VOP2Inst <"v_min_num_f64", VOP_F64_F64_F64, fminnum_like>;
+  defm V_MAX_NUM_F64 : VOP2Inst <"v_max_num_f64", VOP_F64_F64_F64, fmaxnum_like>;
+}
+
 //===----------------------------------------------------------------------===//
 // DPP Encodings
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 279de32a9cee8..aa7578a29faed 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -171,7 +171,7 @@ defm V_MUL_LO_I32 : VOP3Inst <"v_mul_lo_i32", V_MUL_PROF<VOP_I32_I32_I32>>;
 defm V_MUL_HI_I32 : VOP3Inst <"v_mul_hi_i32", V_MUL_PROF<VOP_I32_I32_I32>, mulhs>;
 } // End SchedRW = [WriteIntMul]
 
-let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0, AddedComplexity = 1 in {
+let SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0, AddedComplexity = 1 in {
 defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, fminimum>;
 defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, fmaximum>;
 defm V_MINIMUM_F16 : VOP3Inst_t16 <"v_minimum_f16", VOP_F16_F16_F16, fminimum>;
@@ -181,7 +181,7 @@ let SchedRW = [WriteDoubleAdd] in {
 defm V_MINIMUM_F64 : VOP3Inst <"v_minimum_f64", VOP3_Profile<VOP_F64_F64_F64>, fminimum>;
 defm V_MAXIMUM_F64 : VOP3Inst <"v_maximum_f64", VOP3_Profile<VOP_F64_F64_F64>, fmaximum>;
 } // End SchedRW = [WriteDoubleAdd]
-} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0, AddedComplexity = 1
+} // End SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0, AddedComplexity = 1
 
 } // End isReMaterializable = 1
 
@@ -1532,12 +1532,12 @@ let SubtargetPredicate = HasF32ToF16BF16ConversionSRInsts in {
   def : Cvt_Scale_Sr_F32ToBF16F16_Pat<int_amdgcn_cvt_sr_f16_f32, V_CVT_SR_F16_F32_e64, v2f16>;
 }
 
-let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+let SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0 in {
   defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
   defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
   defm V_MAXIMUMMINIMUM_F16 : VOP3Inst_t16<"v_maximumminimum_f16", VOP_F16_F16_F16_F16>;
   defm V_MINIMUMMAXIMUM_F16 : VOP3Inst_t16<"v_minimummaximum_f16", VOP_F16_F16_F16_F16>;
-} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
+} // End SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0
 
 let SubtargetPredicate = HasDot9Insts, IsDOT=1 in {
   defm V_DOT2_F16_F16 :   VOP3Inst_t16_with_profiles<"v_dot2_f16_f16", VOP3_DOT_Profile<VOP_F16_V2F16_V2F16_F16>,
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index e8db879ca5077..31997f803dfc6 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -115,10 +115,10 @@ defm V_PK_MIN_U16 : VOP3PInst<"v_pk_min_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I1
 defm V_PK_MAX_I16 : VOP3PInst<"v_pk_max_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, smax>;
 defm V_PK_MAX_U16 : VOP3PInst<"v_pk_max_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, umax>;
 
-let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+let SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0 in {
 defm V_PK_MAXIMUM_F16 : VOP3PInst<"v_pk_maximum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16, VOP3_PACKED>, fmaximum>;
 defm V_PK_MINIMUM_F16 : VOP3PInst<"v_pk_minimum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16, VOP3_PACKED>, fminimum>;
-} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
+} // End SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0
 }
 
 defm V_PK_SUB_U16 : VOP3PInst<"v_pk_sub_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>>;
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index ca5ed5cd24603..9e84f6aed0176 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -846,7 +846,7 @@ defm V_CMP_T_U32 : VOPC_I32 <"v_cmp_t_u32">;
 defm V_CMPX_F_U32 : VOPCX_I32 <"v_cmpx_f_u32">;
 defm V_CMPX_LT_U32 : VOPCX_I32 <"v_cmpx_lt_u32", "v_cmpx_gt_u32">;
 defm V_CMPX_EQ_U32 : VOPCX_I32 <"v_cmpx_eq_u32">;
-defm V_CMPX_LE_U32 : VOPCX_I32 <"v_cmpx_le_u32", "v_cmpx_le_u32">;
+defm V_CMPX_LE_U32 : VOPCX_I32 <"v_cmpx_le_u32", "v_cmpx_ge_u32">;
 defm V_CMPX_GT_U32 : VOPCX_I32 <"v_cmpx_gt_u32">;
 defm V_CMPX_NE_U32 : VOPCX_I32 <"v_cmpx_ne_u32">;
 defm V_CMPX_GE_U32 : VOPCX_I32 <"v_cmpx_ge_u32">;
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 1e47acb5fde4f..3e01f8cd044e2 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -348,16 +348,17 @@ class VOP3OpSel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3OpSel_gfx10<op, p>;
 class VOP3FP8OpSel_src_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
   bits<2> byte_sel;
   let Inst{11-12} = byte_sel; // NB: bit order is intentionally reversed!
-  let Inst{14-13} = 0;  // op_sel2/3
+  let Inst{13} = !if(!and(p.HasOpSel, p.HasSrc2), src2_modifiers{2}, 0);
+  let Inst{14} = !if(!and(p.HasOpSel, p.HasDst), src0_modifiers{3}, 0);
 }
 
- class VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
-   bits<2> byte_sel;
+class VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
+  bits<2> byte_sel;
 
-   let Inst{11} = 0; // op_sel0
-   let Inst{12} = 0; // op_sel1
-   let Inst{14-13} = byte_sel;  // op_sel2/3
- }
+  let Inst{11} = !if(!and(p.HasOpSel, p.HasSrc0Mods), src0_modifiers{2}, 0); // op_sel0
+  let Inst{12} = !if(!and(p.HasOpSel, p.HasSrc1Mods), src1_modifiers{2}, 0); // op_sel1
+  let Inst{14-13} = byte_sel;  // op_sel2/3
+}
 
 class VOP3DotOpSel_gfx11_gfx12<bits<10> op, VOPProfile p> :
     VOP3e_t16_gfx11_gfx12<op, p>{
@@ -526,7 +527,7 @@ class VOP3PXe <bits<7> op, VOPProfile MFMAPfl, bit acc_cd = 0> : Enc128, VOP3Pe_
   bits<9> scale_src1;
 
   //MFMALdScaleModifierOp transforms 2 bit opsel input to 4 bit value
-  //where opsel and opselHi are in 3rd and 4th bit. 
+  //where opsel and opselHi are in 3rd and 4th bit.
   bits<4> src0_modifiers;
   bits<4> src1_modifiers;
 
@@ -869,14 +870,14 @@ class VOP3_DPPe_Common_Base<bits<10> op, VOPProfile P> : Enc96 {
   let Inst{9}     = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
   let Inst{10}    = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
   // 16-bit select fields which can be interpreted as OpSel or hi/lo suffix
-  let Inst{11} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{2}, 0),
-                                 !if(P.HasFP8SrcByteSel, byte_sel{1}, ?));
-  let Inst{12} = !if(P.HasOpSel, !if(P.HasSrc1Mods, src1_modifiers{2}, 0),
-                                 !if(P.HasFP8SrcByteSel, byte_sel{0}, ?));
-  let Inst{13} = !if(P.HasOpSel, !if(P.HasSrc2Mods, src2_modifiers{2}, 0),
-                                 !if(P.HasFP8DstByteSel, byte_sel{0}, ?));
-  let Inst{14} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{3}, 0),
-                                 !if(P.HasFP8DstByteSel, byte_sel{1}, ?));
+  let Inst{11} = !if(P.HasFP8SrcByteSel, byte_sel{1},
+                 !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{2}, 0), ?));
+  let Inst{12} = !if(P.HasFP8SrcByteSel, byte_sel{0},
+                 !if(P.HasOpSel, !if(P.HasSrc1Mods, src1_modifiers{2}, 0), ?));
+  let Inst{13} = !if(P.HasFP8DstByteSel, byte_sel{0},
+                 !if(P.HasOpSel, !if(P.HasSrc2Mods, src2_modifiers{2}, 0), ?));
+  let Inst{14} = !if(P.HasFP8DstByteSel, byte_sel{1},
+                 !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{3}, 0), ?));
   let Inst{15}    = !if(P.HasClamp, clamp, 0);
   let Inst{25-16} = op;
   let Inst{31-26} = 0x35;
diff --git a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
index e669b9479369d..c54f33cbcb999 100644
--- a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
+++ b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
@@ -124,7 +124,7 @@ static void printExpr(const MCExpr *Expr, const MCAsmInfo *MAI,
     SRE = dyn_cast<MCSymbolRefExpr>(Expr);
     assert(SRE && "Unexpected MCExpr type.");
   }
-  assert(SRE->getKind() == MCSymbolRefExpr::VK_None);
+  assert(SRE->getSpecifier() == 0);
 
   // Symbols are prefixed with '@'
   OS << '@';
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 2fd784373f4ad..850b00406f09e 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -859,7 +859,7 @@ static uint8_t getModifierSpecifier(ARMCP::ARMCPModifier Modifier) {
   case ARMCP::GOT_PREL:
     return ARM::S_GOT_PREL;
   case ARMCP::SECREL:
-    return MCSymbolRefExpr::VK_SECREL;
+    return ARM::S_COFF_SECREL;
   }
   llvm_unreachable("Invalid ARMCPModifier!");
 }
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 8f17c62d4b906..50217c3a047df 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -3126,8 +3126,8 @@ bool ARMBaseInstrInfo::optimizeCompareInstr(
   // Modify the condition code of operands in OperandsToUpdate.
   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
-  for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
-    OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
+  for (auto &[MO, Cond] : OperandsToUpdate)
+    MO->setImm(Cond);
 
   MI->clearRegisterDeads(ARM::CPSR);
 
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index ca3dc15ff3ad6..ef690838f0f3b 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -476,8 +476,10 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
 
     LLVM_DEBUG(dbgs() << "Beginning BR iteration #" << NoBRIters << '\n');
     bool BRChange = false;
-    for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i)
+    for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i) {
+      // Note: fixupImmediateBr can append to ImmBranches.
       BRChange |= fixupImmediateBr(ImmBranches[i]);
+    }
     if (BRChange && ++NoBRIters > 30)
       report_fatal_error("Branch Fix Up pass failed to converge!");
     LLVM_DEBUG(dumpBBs());
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 80805ff34e1bd..50d8eee8644c1 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -1701,8 +1701,8 @@ void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
                                     .addReg(ARM::SP)
                                     .setMIFlags(MachineInstr::FrameSetup)
                                     .add(predOps(ARMCC::AL));
-      for (unsigned i = 0, e = Regs.size(); i < e; ++i)
-        MIB.addReg(Regs[i].first, getKillRegState(Regs[i].second));
+      for (const auto &[Reg, Kill] : Regs)
+        MIB.addReg(Reg, getKillRegState(Kill));
     } else if (Regs.size() == 1) {
       BuildMI(MBB, MI, DL, TII.get(StrOpc), ARM::SP)
           .addReg(Regs[0].first, getKillRegState(Regs[0].second))
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index d8fcf9d10d74c..6b85e62d2eb8b 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -684,7 +684,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
 
     for (const auto &LC : LibraryCalls) {
       setLibcallImpl(LC.Op, LC.Impl);
-      setLibcallCallingConv(LC.Op, LC.CC);
+      setLibcallImplCallingConv(LC.Impl, LC.CC);
       if (LC.Cond != CmpInst::BAD_ICMP_PREDICATE)
         setCmpLibcallCC(LC.Op, LC.Cond);
     }
@@ -723,7 +723,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
 
       for (const auto &LC : MemOpsLibraryCalls) {
         setLibcallImpl(LC.Op, LC.Impl);
-        setLibcallCallingConv(LC.Op, LC.CC);
+        setLibcallImplCallingConv(LC.Impl, LC.CC);
       }
     }
   }
@@ -733,13 +733,13 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   // hard-float calling convention by default.
   if (!TT.isWatchABI()) {
     if (TM.isAAPCS_ABI()) {
-      setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
-      setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
-      setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
+      setLibcallImplCallingConv(RTLIB::__truncsfhf2, CallingConv::ARM_AAPCS);
+      setLibcallImplCallingConv(RTLIB::__truncdfhf2, CallingConv::ARM_AAPCS);
+      setLibcallImplCallingConv(RTLIB::__extendhfsf2, CallingConv::ARM_AAPCS);
     } else {
-      setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS);
-      setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS);
-      setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS);
+      setLibcallImplCallingConv(RTLIB::__truncsfhf2, CallingConv::ARM_APCS);
+      setLibcallImplCallingConv(RTLIB::__truncdfhf2, CallingConv::ARM_APCS);
+      setLibcallImplCallingConv(RTLIB::__extendhfsf2, CallingConv::ARM_APCS);
     }
   }
 
@@ -761,7 +761,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
 
     for (const auto &LC : LibraryCalls) {
       setLibcallImpl(LC.Op, LC.Impl);
-      setLibcallCallingConv(LC.Op, LC.CC);
+      setLibcallImplCallingConv(LC.Impl, LC.CC);
     }
   } else if (!TT.isOSBinFormatMachO()) {
     setLibcallImpl(RTLIB::FPROUND_F32_F16, RTLIB::__gnu_f2h_ieee);
@@ -1400,8 +1400,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
-  if (Subtarget->useSjLjEH())
-    setLibcallImpl(RTLIB::UNWIND_RESUME, RTLIB::_Unwind_SjLj_Resume);
 
   setOperationAction(ISD::SETCC,     MVT::i32, Expand);
   setOperationAction(ISD::SETCC,     MVT::f32, Expand);
@@ -1543,6 +1541,21 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
     setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
 
+    if (Subtarget->hasV8Ops()) {
+      setOperationAction(ISD::FFLOOR, MVT::v2f32, Legal);
+      setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
+      setOperationAction(ISD::FROUND, MVT::v2f32, Legal);
+      setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v2f32, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v4f32, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v2f32, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v2f32, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
+      setOperationAction(ISD::FRINT, MVT::v2f32, Legal);
+      setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
+    }
+
     if (Subtarget->hasFullFP16()) {
       setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
       setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);
@@ -1553,6 +1566,19 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
       setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal);
       setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal);
       setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal);
+
+      setOperationAction(ISD::FFLOOR, MVT::v4f16, Legal);
+      setOperationAction(ISD::FFLOOR, MVT::v8f16, Legal);
+      setOperationAction(ISD::FROUND, MVT::v4f16, Legal);
+      setOperationAction(ISD::FROUND, MVT::v8f16, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v4f16, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v8f16, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v4f16, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v8f16, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v4f16, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v8f16, Legal);
+      setOperationAction(ISD::FRINT, MVT::v4f16, Legal);
+      setOperationAction(ISD::FRINT, MVT::v8f16, Legal);
     }
   }
 
@@ -2766,9 +2792,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -2952,9 +2977,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const uint32_t *Mask;
@@ -5500,6 +5524,21 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue FalseVal = Op.getOperand(3);
   ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
   ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
+  ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
+  if (Op.getValueType().isInteger()) {
+    // Check for sign pattern (SELECT_CC setgt, iN lhs, -1, 1, -1) and transform
+    // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
+    // supported types.
+    if (CC == ISD::SETGT && RHSC && RHSC->isAllOnes() && CTVal && CFVal &&
+        CTVal->isOne() && CFVal->isAllOnes() &&
+        LHS.getValueType() == TrueVal.getValueType()) {
+      EVT VT = LHS.getValueType();
+      SDValue Shift =
+          DAG.getNode(ISD::SRA, dl, VT, LHS,
+                      DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
+      return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT));
+    }
+  }
 
   if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
       LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
@@ -6146,9 +6185,6 @@ SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
@@ -12321,6 +12357,11 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     Fn->insert(BBI, RSBBB);
     Fn->insert(BBI, SinkBB);
 
+    // Set the call frame size on entry to the new basic blocks.
+    unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
+    RSBBB->setCallFrameSize(CallFrameSize);
+    SinkBB->setCallFrameSize(CallFrameSize);
+
     Register ABSSrcReg = MI.getOperand(1).getReg();
     Register ABSDstReg = MI.getOperand(0).getReg();
     bool ABSSrcKIll = MI.getOperand(1).isKill();
@@ -21918,14 +21959,16 @@ Register ARMTargetLowering::getExceptionPointerRegister(
     const Constant *PersonalityFn) const {
   // Platforms which do not use SjLj EH may return values in these registers
   // via the personality function.
-  return Subtarget->useSjLjEH() ? Register() : ARM::R0;
+  ExceptionHandling EM = getTargetMachine().getExceptionModel();
+  return EM == ExceptionHandling::SjLj ? Register() : ARM::R0;
 }
 
 Register ARMTargetLowering::getExceptionSelectorRegister(
     const Constant *PersonalityFn) const {
   // Platforms which do not use SjLj EH may return values in these registers
   // via the personality function.
-  return Subtarget->useSjLjEH() ? Register() : ARM::R1;
+  ExceptionHandling EM = getTargetMachine().getExceptionModel();
+  return EM == ExceptionHandling::SjLj ? Register() : ARM::R1;
 }
 
 void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 9c330e60a7d54..604910e04d4cc 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -609,6 +609,11 @@ class VectorType;
 
     bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
 
+    bool hasAndNotCompare(SDValue V) const override {
+      // We can use bics for any scalar.
+      return V.getValueType().isScalarInteger();
+    }
+
     bool
     isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index 3335f52f15555..7485ef569445a 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -7312,12 +7312,12 @@ multiclass VRINT_FPI<string op, bits<3> op9_7, SDPatternOperator Int> {
   }
 }
 
-defm VRINTNN : VRINT_FPI<"n", 0b000, int_arm_neon_vrintn>;
-defm VRINTXN : VRINT_FPI<"x", 0b001, int_arm_neon_vrintx>;
-defm VRINTAN : VRINT_FPI<"a", 0b010, int_arm_neon_vrinta>;
-defm VRINTZN : VRINT_FPI<"z", 0b011, int_arm_neon_vrintz>;
-defm VRINTMN : VRINT_FPI<"m", 0b101, int_arm_neon_vrintm>;
-defm VRINTPN : VRINT_FPI<"p", 0b111, int_arm_neon_vrintp>;
+defm VRINTNN : VRINT_FPI<"n", 0b000, froundeven>;
+defm VRINTXN : VRINT_FPI<"x", 0b001, frint>;
+defm VRINTAN : VRINT_FPI<"a", 0b010, fround>;
+defm VRINTZN : VRINT_FPI<"z", 0b011, ftrunc>;
+defm VRINTMN : VRINT_FPI<"m", 0b101, ffloor>;
+defm VRINTPN : VRINT_FPI<"p", 0b111, fceil>;
 
 // Cryptography instructions
 let PostEncoderMethod = "NEONThumb2DataIPostEncoder",
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 1d5c12fabf093..31650e0137beb 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -1135,13 +1135,8 @@ multiclass vrint_inst_anpm<string opc, bits<2> rm,
         Requires<[HasFPARMv8,HasDPVFP]>;
 }
 
-// Match either froundeven or int_arm_neon_vrintn
-def vrintn_or_froundeven : PatFrags<(ops node:$src),
-                                    [(int_arm_neon_vrintn node:$src),
-                                     (froundeven node:$src)]>;
-
 defm VRINTA : vrint_inst_anpm<"a", 0b00, fround>;
-defm VRINTN : vrint_inst_anpm<"n", 0b01, vrintn_or_froundeven>;
+defm VRINTN : vrint_inst_anpm<"n", 0b01, froundeven>;
 defm VRINTP : vrint_inst_anpm<"p", 0b10, fceil>;
 defm VRINTM : vrint_inst_anpm<"m", 0b11, ffloor>;
 
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index abca4bb947bc4..13185a7d797a3 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -72,7 +72,6 @@ ForceFastISel("arm-force-fast-isel",
 /// so that we can use initializer lists for subtarget initialization.
 ARMSubtarget &ARMSubtarget::initializeSubtargetDependencies(StringRef CPU,
                                                             StringRef FS) {
-  initializeEnvironment();
   initSubtargetFeatures(CPU, FS);
   return *this;
 }
@@ -137,19 +136,6 @@ bool ARMSubtarget::isXRaySupported() const {
   return hasV6Ops() && hasARMOps() && !isTargetWindows();
 }
 
-void ARMSubtarget::initializeEnvironment() {
-  // MCAsmInfo isn't always present (e.g. in opt) so we can't initialize this
-  // directly from it, but we can try to make sure they're consistent when both
-  // available.
-  UseSjLjEH = (isTargetDarwin() && !isTargetWatchABI() &&
-               Options.ExceptionModel == ExceptionHandling::None) ||
-              Options.ExceptionModel == ExceptionHandling::SjLj;
-  assert((!TM.getMCAsmInfo() ||
-          (TM.getMCAsmInfo()->getExceptionHandlingType() ==
-           ExceptionHandling::SjLj) == UseSjLjEH) &&
-         "inconsistent sjlj choice between CodeGen and MC");
-}
-
 void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (CPUString.empty()) {
     CPUString = "generic";
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 3e1314349564c..beb1ff6447148 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -151,9 +151,6 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   ///  blocks.
   bool RestrictIT = false;
 
-  /// UseSjLjEH - If true, the target uses SjLj exception handling (e.g. iOS).
-  bool UseSjLjEH = false;
-
   /// stackAlignment - The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   Align stackAlignment = Align(4);
@@ -270,7 +267,6 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   std::unique_ptr<LegalizerInfo> Legalizer;
   std::unique_ptr<RegisterBankInfo> RegBankInfo;
 
-  void initializeEnvironment();
   void initSubtargetFeatures(StringRef CPU, StringRef FS);
   ARMFrameLowering *initializeFrameLowering(StringRef CPU, StringRef FS);
 
@@ -321,7 +317,6 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   }
   bool useFPVFMx16() const { return useFPVFMx() && hasFullFP16(); }
   bool useFPVFMx64() const { return useFPVFMx() && hasFP64(); }
-  bool useSjLjEH() const { return UseSjLjEH; }
   bool hasBaseDSP() const {
     if (isThumb())
       return hasThumb2() && hasDSP();
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 25f0273013373..ec6f4e2ae216e 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -3629,7 +3629,7 @@ class ARMOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createImm(Imm == 48 ? 1 : 0));
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<ARMOperand> CreateITMask(unsigned Mask, SMLoc S,
                                                   ARMAsmParser &Parser) {
@@ -3979,7 +3979,7 @@ class ARMOperand : public MCParsedAsmOperand {
 
 } // end anonymous namespace.
 
-void ARMOperand::print(raw_ostream &OS) const {
+void ARMOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   auto RegName = [](MCRegister Reg) {
     if (Reg)
       return ARMInstPrinter::getRegisterName(Reg);
@@ -4024,7 +4024,7 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << "<banked reg: " << getBankedReg() << ">";
     break;
   case k_Immediate:
-    OS << *getImm();
+    MAI.printExpr(OS, *getImm());
     break;
   case k_MemBarrierOpt:
     OS << "<ARM_MB::" << MemBOptToString(getMemBarrierOpt(), false) << ">";
@@ -4039,8 +4039,10 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << "<memory";
     if (Memory.BaseRegNum)
       OS << " base:" << RegName(Memory.BaseRegNum);
-    if (Memory.OffsetImm)
-      OS << " offset-imm:" << *Memory.OffsetImm;
+    if (Memory.OffsetImm) {
+      OS << " offset-imm:";
+      MAI.printExpr(OS, *Memory.OffsetImm);
+    }
     if (Memory.OffsetRegNum)
       OS << " offset-reg:" << (Memory.isNegative ? "-" : "")
          << RegName(Memory.OffsetRegNum);
@@ -4094,7 +4096,8 @@ void ARMOperand::print(raw_ostream &OS) const {
        <<  ModImm.Rot << ")>";
     break;
   case k_ConstantPoolImmediate:
-    OS << "<constant_pool_imm #" << *getConstantPoolImm();
+    OS << "<constant_pool_imm #";
+    MAI.printExpr(OS, *getConstantPoolImm());
     break;
   case k_BitfieldDescriptor:
     OS << "<bitfield " << "lsb: " << Bitfield.LSB
@@ -6433,7 +6436,7 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
       return true;
 
     const auto *ExprVal =
-        MCSpecifierExpr::create(SubExprVal, Spec, getContext());
+        MCSpecifierExpr::create(SubExprVal, Spec, getContext(), S);
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
     Operands.push_back(ARMOperand::CreateImm(ExprVal, S, E, *this));
     return false;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index f43fdae554b8b..376bddb120d5f 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -70,44 +69,30 @@ MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       // ARMFixupKinds.h.
       //
       // Name                      Offset (bits) Size (bits)     Flags
-      {"fixup_arm_ldst_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_ldst_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_10_unscaled", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_pcrel_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_10", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_9", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_9", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
+      {"fixup_arm_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_t2_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_arm_pcrel_10_unscaled", 0, 32, 0},
+      {"fixup_arm_pcrel_10", 0, 32, 0},
+      {"fixup_t2_pcrel_10", 0, 32, 0},
+      {"fixup_arm_pcrel_9", 0, 32, 0},
+      {"fixup_t2_pcrel_9", 0, 32, 0},
       {"fixup_arm_ldst_abs_12", 0, 32, 0},
-      {"fixup_thumb_adr_pcrel_10", 0, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_adr_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_adr_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_condbranch", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbranch", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_condbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_uncondbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_br", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbl", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_condbl", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_blx", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_bl", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_blx", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_cb", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_cp", 0, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_bcc", 0, 8, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_thumb_adr_pcrel_10", 0, 8, 0},
+      {"fixup_arm_adr_pcrel_12", 0, 32, 0},
+      {"fixup_t2_adr_pcrel_12", 0, 32, 0},
+      {"fixup_arm_condbranch", 0, 24, 0},
+      {"fixup_arm_uncondbranch", 0, 24, 0},
+      {"fixup_t2_condbranch", 0, 32, 0},
+      {"fixup_t2_uncondbranch", 0, 32, 0},
+      {"fixup_arm_thumb_br", 0, 16, 0},
+      {"fixup_arm_uncondbl", 0, 24, 0},
+      {"fixup_arm_condbl", 0, 24, 0},
+      {"fixup_arm_blx", 0, 24, 0},
+      {"fixup_arm_thumb_bl", 0, 32, 0},
+      {"fixup_arm_thumb_blx", 0, 32, 0},
+      {"fixup_arm_thumb_cb", 0, 16, 0},
+      {"fixup_arm_thumb_cp", 0, 8, 0},
+      {"fixup_arm_thumb_bcc", 0, 8, 0},
       // movw / movt: 16-bits immediate but scattered into two chunks 0 - 12, 16
       // - 19.
       {"fixup_arm_movt_hi16", 0, 20, 0},
@@ -120,56 +105,43 @@ MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_arm_thumb_lower_0_7", 0, 8, 0},
       {"fixup_arm_mod_imm", 0, 12, 0},
       {"fixup_t2_so_imm", 0, 26, 0},
-      {"fixup_bf_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bf_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfl_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfc_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_bf_branch", 0, 32, 0},
+      {"fixup_bf_target", 0, 32, 0},
+      {"fixup_bfl_target", 0, 32, 0},
+      {"fixup_bfc_target", 0, 32, 0},
       {"fixup_bfcsel_else_target", 0, 32, 0},
-      {"fixup_wls", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_le", 0, 32, MCFixupKindInfo::FKF_IsPCRel}};
+      {"fixup_wls", 0, 32, 0},
+      {"fixup_le", 0, 32, 0},
+  };
   const static MCFixupKindInfo InfosBE[ARM::NumTargetFixupKinds] = {
       // This table *must* be in the order that the fixup_* kinds are defined in
       // ARMFixupKinds.h.
       //
       // Name                      Offset (bits) Size (bits)     Flags
-      {"fixup_arm_ldst_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_ldst_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_10_unscaled", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_pcrel_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_10", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_9", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_9", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
+      {"fixup_arm_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_t2_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_arm_pcrel_10_unscaled", 0, 32, 0},
+      {"fixup_arm_pcrel_10", 0, 32, 0},
+      {"fixup_t2_pcrel_10", 0, 32, 0},
+      {"fixup_arm_pcrel_9", 0, 32, 0},
+      {"fixup_t2_pcrel_9", 0, 32, 0},
       {"fixup_arm_ldst_abs_12", 0, 32, 0},
-      {"fixup_thumb_adr_pcrel_10", 8, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_adr_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_adr_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_condbranch", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbranch", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_condbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_uncondbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_br", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbl", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_condbl", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_blx", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_bl", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_blx", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_cb", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_cp", 8, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_bcc", 8, 8, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_thumb_adr_pcrel_10", 8, 8, 0},
+      {"fixup_arm_adr_pcrel_12", 0, 32, 0},
+      {"fixup_t2_adr_pcrel_12", 0, 32, 0},
+      {"fixup_arm_condbranch", 8, 24, 0},
+      {"fixup_arm_uncondbranch", 8, 24, 0},
+      {"fixup_t2_condbranch", 0, 32, 0},
+      {"fixup_t2_uncondbranch", 0, 32, 0},
+      {"fixup_arm_thumb_br", 0, 16, 0},
+      {"fixup_arm_uncondbl", 8, 24, 0},
+      {"fixup_arm_condbl", 8, 24, 0},
+      {"fixup_arm_blx", 8, 24, 0},
+      {"fixup_arm_thumb_bl", 0, 32, 0},
+      {"fixup_arm_thumb_blx", 0, 32, 0},
+      {"fixup_arm_thumb_cb", 0, 16, 0},
+      {"fixup_arm_thumb_cp", 8, 8, 0},
+      {"fixup_arm_thumb_bcc", 8, 8, 0},
       // movw / movt: 16-bits immediate but scattered into two chunks 0 - 12, 16
       // - 19.
       {"fixup_arm_movt_hi16", 12, 20, 0},
@@ -182,18 +154,19 @@ MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_arm_thumb_lower_0_7", 24, 8, 0},
       {"fixup_arm_mod_imm", 20, 12, 0},
       {"fixup_t2_so_imm", 26, 6, 0},
-      {"fixup_bf_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bf_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfl_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfc_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_bf_branch", 0, 32, 0},
+      {"fixup_bf_target", 0, 32, 0},
+      {"fixup_bfl_target", 0, 32, 0},
+      {"fixup_bfc_target", 0, 32, 0},
       {"fixup_bfcsel_else_target", 0, 32, 0},
-      {"fixup_wls", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_le", 0, 32, MCFixupKindInfo::FKF_IsPCRel}};
+      {"fixup_wls", 0, 32, 0},
+      {"fixup_le", 0, 32, 0},
+  };
 
   // Fixup kinds from .reloc directive are like R_ARM_NONE. They do not require
   // any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -228,11 +201,9 @@ unsigned ARMAsmBackend::getRelaxedOpcode(unsigned Op,
   }
 }
 
-bool ARMAsmBackend::mayNeedRelaxation(const MCInst &Inst,
+bool ARMAsmBackend::mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand>,
                                       const MCSubtargetInfo &STI) const {
-  if (getRelaxedOpcode(Inst.getOpcode(), STI) != Inst.getOpcode())
-    return true;
-  return false;
+  return getRelaxedOpcode(Opcode, STI) != Opcode;
 }
 
 static const char *checkPCRelOffset(uint64_t Value, int64_t Min, int64_t Max) {
@@ -356,15 +327,7 @@ bool ARMAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
 void ARMAsmBackend::relaxInstruction(MCInst &Inst,
                                      const MCSubtargetInfo &STI) const {
   unsigned RelaxedOp = getRelaxedOpcode(Inst.getOpcode(), STI);
-
-  // Return a diagnostic if we get here w/ a bogus instruction.
-  if (RelaxedOp == Inst.getOpcode()) {
-    SmallString<256> Tmp;
-    raw_svector_ostream OS(Tmp);
-    Inst.dump_pretty(OS);
-    OS << "\n";
-    report_fatal_error("unexpected instruction to relax: " + OS.str());
-  }
+  assert(RelaxedOp != Inst.getOpcode());
 
   // If we are changing Thumb CBZ or CBNZ instruction to a NOP, aka tHINT, we
   // have to change the operands too.
@@ -1124,10 +1087,32 @@ static unsigned getFixupKindContainerSizeBytes(unsigned Kind) {
   }
 }
 
+std::optional<bool> ARMAsmBackend::evaluateFixup(const MCFragment &F,
+                                                 MCFixup &Fixup, MCValue &,
+                                                 uint64_t &Value) {
+  // For a few PC-relative fixups in Thumb mode, offsets need to be aligned
+  // down. We compensate here because the default handler's `Value` decrement
+  // doesn't account for this alignment.
+  switch (Fixup.getTargetKind()) {
+  case ARM::fixup_t2_ldst_pcrel_12:
+  case ARM::fixup_t2_pcrel_10:
+  case ARM::fixup_t2_pcrel_9:
+  case ARM::fixup_thumb_adr_pcrel_10:
+  case ARM::fixup_t2_adr_pcrel_12:
+  case ARM::fixup_arm_thumb_blx:
+  case ARM::fixup_arm_thumb_cp:
+    Value = (Asm->getFragmentOffset(F) + Fixup.getOffset()) % 4;
+  }
+  return {};
+}
+
 void ARMAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                const MCValue &Target,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   auto Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
index 5efc969641b90..877e3afdb1d57 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
@@ -30,21 +30,22 @@ class ARMAsmBackend : public MCAsmBackend {
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
 
   unsigned adjustFixupValue(const MCAssembler &Asm, const MCFixup &Fixup,
                             const MCValue &Target, uint64_t Value,
                             bool IsResolved, MCContext &Ctx,
                             const MCSubtargetInfo *STI) const;
 
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
 
   unsigned getRelaxedOpcode(unsigned Op, const MCSubtargetInfo &STI) const;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override;
 
   const char *reasonForFixupRelaxation(const MCFixup &Fixup,
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 73ad62ed79532..c61e405bd3a02 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -32,7 +32,6 @@
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
@@ -1210,8 +1209,7 @@ inline void ARMELFStreamer::SwitchToExIdxSection(const MCSymbol &FnStart) {
 
 void ARMELFStreamer::EmitFixup(const MCExpr *Expr, MCFixupKind Kind) {
   MCDataFragment *Frag = getOrCreateDataFragment();
-  Frag->getFixups().push_back(MCFixup::create(Frag->getContents().size(), Expr,
-                                              Kind));
+  Frag->addFixup(MCFixup::create(Frag->getContents().size(), Expr, Kind));
 }
 
 void ARMELFStreamer::EHReset() {
@@ -1299,7 +1297,7 @@ void ARMELFStreamer::EmitPersonalityFixup(StringRef Name) {
 
   visitUsedExpr(*PersonalityRef);
   MCDataFragment *DF = getOrCreateDataFragment();
-  DF->getFixups().push_back(
+  DF->addFixup(
       MCFixup::create(DF->getContents().size(), PersonalityRef, FK_Data_4));
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index ab9954c53571d..e9f4b72f4e52a 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -34,7 +34,7 @@ const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {ARM::S_GOTTPOFF, "GOTTPOFF"},
     {ARM::S_GOTTPOFF_FDPIC, "gottpoff_fdpic"},
     {ARM::S_PLT, "PLT"},
-    {MCSymbolRefExpr::VK_SECREL, "SECREL32"},
+    {ARM::S_COFF_SECREL, "SECREL32"},
     {ARM::S_TLSCALL, "tlscall"},
     {ARM::S_TLSDESC, "tlsdesc"},
     {ARM::S_TLSGD, "TLSGD"},
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
index f3f075e99d961..8b24e7b5d8015 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
@@ -92,6 +92,8 @@ namespace ARM {
 using Specifier = uint16_t;
 enum {
   S_None,
+  S_COFF_SECREL,
+
   S_HI16 =
       MCSymbolRefExpr::FirstTargetSpecifier, // The R_ARM_MOVT_ABS relocation
                                              // (:upper16: in the .s file)
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index fba32eae4dfa8..c56b589519533 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -313,41 +313,12 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
 
   unsigned getModImmOpValue(const MCInst &MI, unsigned Op,
                             SmallVectorImpl<MCFixup> &Fixups,
-                            const MCSubtargetInfo &ST) const {
-    const MCOperand &MO = MI.getOperand(Op);
-
-    // Support for fixups (MCFixup)
-    if (MO.isExpr()) {
-      const MCExpr *Expr = MO.getExpr();
-      // Fixups resolve to plain values that need to be encoded.
-      MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_mod_imm);
-      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
-      return 0;
-    }
-
-    // Immediate is already in its encoded format
-    return MO.getImm();
-  }
+                            const MCSubtargetInfo &ST) const;
 
   /// getT2SOImmOpValue - Return an encoded 12-bit shifted-immediate value.
   unsigned getT2SOImmOpValue(const MCInst &MI, unsigned Op,
-                           SmallVectorImpl<MCFixup> &Fixups,
-                           const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Op);
-
-    // Support for fixups (MCFixup)
-    if (MO.isExpr()) {
-      const MCExpr *Expr = MO.getExpr();
-      // Fixups resolve to plain values that need to be encoded.
-      MCFixupKind Kind = MCFixupKind(ARM::fixup_t2_so_imm);
-      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
-      return 0;
-    }
-    unsigned SoImm = MO.getImm();
-    unsigned Encoded =  ARM_AM::getT2SOImmVal(SoImm);
-    assert(Encoded != ~0U && "Not a Thumb2 so_imm value?");
-    return Encoded;
-  }
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
 
   unsigned getT2AddrModeSORegOpValue(const MCInst &MI, unsigned OpNum,
     SmallVectorImpl<MCFixup> &Fixups,
@@ -462,6 +433,44 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case ARM::fixup_arm_ldst_pcrel_12:
+  case ARM::fixup_t2_ldst_pcrel_12:
+  case ARM::fixup_arm_pcrel_10_unscaled:
+  case ARM::fixup_arm_pcrel_10:
+  case ARM::fixup_t2_pcrel_10:
+  case ARM::fixup_arm_pcrel_9:
+  case ARM::fixup_t2_pcrel_9:
+  case ARM::fixup_thumb_adr_pcrel_10:
+  case ARM::fixup_arm_adr_pcrel_12:
+  case ARM::fixup_t2_adr_pcrel_12:
+  case ARM::fixup_arm_condbranch:
+  case ARM::fixup_arm_uncondbranch:
+  case ARM::fixup_t2_condbranch:
+  case ARM::fixup_t2_uncondbranch:
+  case ARM::fixup_arm_thumb_br:
+  case ARM::fixup_arm_uncondbl:
+  case ARM::fixup_arm_condbl:
+  case ARM::fixup_arm_blx:
+  case ARM::fixup_arm_thumb_bl:
+  case ARM::fixup_arm_thumb_blx:
+  case ARM::fixup_arm_thumb_cb:
+  case ARM::fixup_arm_thumb_cp:
+  case ARM::fixup_arm_thumb_bcc:
+  case ARM::fixup_bf_branch:
+  case ARM::fixup_bf_target:
+  case ARM::fixup_bfl_target:
+  case ARM::fixup_bfc_target:
+  case ARM::fixup_wls:
+  case ARM::fixup_le:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 /// NEONThumb2DataIPostEncoder - Post-process encoded NEON data-processing
 /// instructions, and rewrite them to their Thumb2 form if we are currently in
 /// Thumb2 mode.
@@ -616,7 +625,7 @@ static uint32_t getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
   assert(MO.isExpr() && "Unexpected branch target type!");
   const MCExpr *Expr = MO.getExpr();
   MCFixupKind Kind = MCFixupKind(FixupKind);
-  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Kind);
 
   // All of the information is in the fixup.
   return 0;
@@ -978,18 +987,18 @@ getAddrModeImm12OpValue(const MCInst &MI, unsigned OpIdx,
              "Thumb mode requires different encoding");
       Reg = CTX.getRegisterInfo()->getEncodingValue(MO.getReg());
       isAdd = false; // 'U' bit is set as part of the fixup.
-      MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_ldst_abs_12);
-      Fixups.push_back(MCFixup::create(0, MO1.getExpr(), Kind, MI.getLoc()));
+      MCFixupKind Kind = ARM::fixup_arm_ldst_abs_12;
+      addFixup(Fixups, 0, MO1.getExpr(), Kind);
     }
   } else if (MO.isExpr()) {
     Reg = CTX.getRegisterInfo()->getEncodingValue(ARM::PC); // Rn is PC.
     isAdd = false; // 'U' bit is set as part of the fixup.
     MCFixupKind Kind;
     if (isThumb2(STI))
-      Kind = MCFixupKind(ARM::fixup_t2_ldst_pcrel_12);
+      Kind = ARM::fixup_t2_ldst_pcrel_12;
     else
-      Kind = MCFixupKind(ARM::fixup_arm_ldst_pcrel_12);
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
+      Kind = ARM::fixup_arm_ldst_pcrel_12;
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
 
     ++MCNumCPRelocations;
   } else {
@@ -1113,8 +1122,8 @@ getT2AddrModeImm8s4OpValue(const MCInst &MI, unsigned OpIdx,
 
     assert(MO.isExpr() && "Unexpected machine operand type!");
     const MCExpr *Expr = MO.getExpr();
-    MCFixupKind Kind = MCFixupKind(ARM::fixup_t2_pcrel_10);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+    MCFixupKind Kind = ARM::fixup_t2_pcrel_10;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
   } else
@@ -1232,26 +1241,26 @@ uint32_t ARMMCCodeEmitter::getHiLoImmOpValue(const MCInst &MI, unsigned OpIdx,
     case ARM::S_HI_8_15:
       if (!isThumb(STI))
         llvm_unreachable(":upper_8_15: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_upper_8_15);
+      Kind = ARM::fixup_arm_thumb_upper_8_15;
       break;
     case ARM::S_HI_0_7:
       if (!isThumb(STI))
         llvm_unreachable(":upper_0_7: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_upper_0_7);
+      Kind = ARM::fixup_arm_thumb_upper_0_7;
       break;
     case ARM::S_LO_8_15:
       if (!isThumb(STI))
         llvm_unreachable(":lower_8_15: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_lower_8_15);
+      Kind = ARM::fixup_arm_thumb_lower_8_15;
       break;
     case ARM::S_LO_0_7:
       if (!isThumb(STI))
         llvm_unreachable(":lower_0_7: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_lower_0_7);
+      Kind = ARM::fixup_arm_thumb_lower_0_7;
       break;
     }
 
-    Fixups.push_back(MCFixup::create(0, E, Kind, MI.getLoc()));
+    addFixup(Fixups, 0, E, Kind);
     return 0;
   }
   // If the expression doesn't have :upper16:, :lower16: on it, it's just a
@@ -1372,8 +1381,8 @@ getAddrMode3OpValue(const MCInst &MI, unsigned OpIdx,
 
     assert(MO.isExpr() && "Unexpected machine operand type!");
     const MCExpr *Expr = MO.getExpr();
-    MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_pcrel_10_unscaled);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+    MCFixupKind Kind = ARM::fixup_arm_pcrel_10_unscaled;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
     return (Rn << 9) | (1 << 13);
@@ -1452,10 +1461,10 @@ getAddrMode5OpValue(const MCInst &MI, unsigned OpIdx,
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind;
     if (isThumb2(STI))
-      Kind = MCFixupKind(ARM::fixup_t2_pcrel_10);
+      Kind = ARM::fixup_t2_pcrel_10;
     else
-      Kind = MCFixupKind(ARM::fixup_arm_pcrel_10);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+      Kind = ARM::fixup_arm_pcrel_10;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
   } else {
@@ -1492,10 +1501,10 @@ getAddrMode5FP16OpValue(const MCInst &MI, unsigned OpIdx,
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind;
     if (isThumb2(STI))
-      Kind = MCFixupKind(ARM::fixup_t2_pcrel_9);
+      Kind = ARM::fixup_t2_pcrel_9;
     else
-      Kind = MCFixupKind(ARM::fixup_arm_pcrel_9);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+      Kind = ARM::fixup_arm_pcrel_9;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
   } else {
@@ -1511,6 +1520,43 @@ getAddrMode5FP16OpValue(const MCInst &MI, unsigned OpIdx,
   return Binary;
 }
 
+unsigned ARMMCCodeEmitter::getModImmOpValue(const MCInst &MI, unsigned Op,
+                                            SmallVectorImpl<MCFixup> &Fixups,
+                                            const MCSubtargetInfo &ST) const {
+  const MCOperand &MO = MI.getOperand(Op);
+
+  // Support for fixups (MCFixup)
+  if (MO.isExpr()) {
+    const MCExpr *Expr = MO.getExpr();
+    // Fixups resolve to plain values that need to be encoded.
+    MCFixupKind Kind = ARM::fixup_arm_mod_imm;
+    addFixup(Fixups, 0, Expr, Kind);
+    return 0;
+  }
+
+  // Immediate is already in its encoded format
+  return MO.getImm();
+}
+
+unsigned ARMMCCodeEmitter::getT2SOImmOpValue(const MCInst &MI, unsigned Op,
+                                             SmallVectorImpl<MCFixup> &Fixups,
+                                             const MCSubtargetInfo &STI) const {
+  const MCOperand &MO = MI.getOperand(Op);
+
+  // Support for fixups (MCFixup)
+  if (MO.isExpr()) {
+    const MCExpr *Expr = MO.getExpr();
+    // Fixups resolve to plain values that need to be encoded.
+    MCFixupKind Kind = ARM::fixup_t2_so_imm;
+    addFixup(Fixups, 0, Expr, Kind);
+    return 0;
+  }
+  unsigned SoImm = MO.getImm();
+  unsigned Encoded = ARM_AM::getT2SOImmVal(SoImm);
+  assert(Encoded != ~0U && "Not a Thumb2 so_imm value?");
+  return Encoded;
+}
+
 unsigned ARMMCCodeEmitter::
 getSORegRegOpValue(const MCInst &MI, unsigned OpIdx,
                 SmallVectorImpl<MCFixup> &Fixups,
@@ -1949,8 +1995,8 @@ ARMMCCodeEmitter::getBFAfterTargetOpValue(const MCInst &MI, unsigned OpIdx,
     assert(BranchMO.isExpr());
     const MCExpr *DiffExpr = MCBinaryExpr::createSub(
         MO.getExpr(), BranchMO.getExpr(), CTX);
-    MCFixupKind Kind = MCFixupKind(ARM::fixup_bfcsel_else_target);
-    Fixups.push_back(llvm::MCFixup::create(0, DiffExpr, Kind, MI.getLoc()));
+    MCFixupKind Kind = ARM::fixup_bfcsel_else_target;
+    addFixup(Fixups, 0, DiffExpr, Kind);
     return 0;
   }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index f6108ede42701..c0c40ade5810a 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -144,7 +144,7 @@ void ARMMachObjectWriter::recordARMScatteredHalfRelocation(
     return;
   }
 
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Type = MachO::ARM_RELOC_HALF;
 
   // See <reloc.h>.
@@ -251,7 +251,7 @@ void ARMMachObjectWriter::recordARMScatteredRelocation(
     return;
   }
 
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
 
   // See <reloc.h>.
   const MCSymbol *A = Target.getAddSym();
@@ -358,7 +358,7 @@ void ARMMachObjectWriter::recordRelocation(MachObjectWriter *Writer,
                                            const MCFragment *Fragment,
                                            const MCFixup &Fixup, MCValue Target,
                                            uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Log2Size;
   unsigned RelocType = MachO::ARM_RELOC_VANILLA;
   if (!getARMFixupKindMachOInfo(Fixup.getKind(), RelocType, Log2Size)) {
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
index bfcd5e8a9b3be..5fef2c0a0bdce 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/ARMFixupKinds.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/MC/MCAsmBackend.h"
@@ -46,12 +47,14 @@ unsigned ARMWinCOFFObjectWriter::getRelocType(MCContext &Ctx,
                                               const MCAsmBackend &MAB) const {
   auto Spec = Target.getSpecifier();
   unsigned FixupKind = Fixup.getKind();
+  bool PCRel = false;
   if (IsCrossSection) {
-    if (FixupKind != FK_Data_4) {
+    if (PCRel || FixupKind != FK_Data_4) {
       Ctx.reportError(Fixup.getLoc(), "Cannot represent this expression");
       return COFF::IMAGE_REL_ARM_ADDR32;
     }
-    FixupKind = FK_PCRel_4;
+    FixupKind = FK_Data_4;
+    PCRel = true;
   }
 
 
@@ -61,16 +64,16 @@ unsigned ARMWinCOFFObjectWriter::getRelocType(MCContext &Ctx,
     return COFF::IMAGE_REL_ARM_ABSOLUTE;
   }
   case FK_Data_4:
+    if (PCRel)
+      return COFF::IMAGE_REL_ARM_REL32;
     switch (Spec) {
     case MCSymbolRefExpr::VK_COFF_IMGREL32:
       return COFF::IMAGE_REL_ARM_ADDR32NB;
-    case MCSymbolRefExpr::VK_SECREL:
+    case ARM::S_COFF_SECREL:
       return COFF::IMAGE_REL_ARM_SECREL;
     default:
       return COFF::IMAGE_REL_ARM_ADDR32;
     }
-  case FK_PCRel_4:
-    return COFF::IMAGE_REL_ARM_REL32;
   case FK_SecRel_2:
     return COFF::IMAGE_REL_ARM_SECTION;
   case FK_SecRel_4:
diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
index 012cf2c70e2e5..4e00b192b875e 100644
--- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
+++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
@@ -247,7 +247,7 @@ class AVROperand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return Start; }
   SMLoc getEndLoc() const override { return End; }
 
-  void print(raw_ostream &O) const override {
+  void print(raw_ostream &O, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Token:
       O << "Token: \"" << getToken() << "\"";
@@ -256,12 +256,16 @@ class AVROperand : public MCParsedAsmOperand {
       O << "Register: " << getReg();
       break;
     case k_Immediate:
-      O << "Immediate: \"" << *getImm() << "\"";
+      O << "Immediate: \"";
+      MAI.printExpr(O, *getImm());
+      O << "\"";
       break;
     case k_Memri: {
       // only manually print the size for non-negative values,
       // as the sign is inserted automatically.
-      O << "Memri: \"" << getReg() << '+' << *getImm() << "\"";
+      O << "Memri: \"" << getReg() << '+';
+      MAI.printExpr(O, *getImm());
+      O << "\"";
       break;
     }
     }
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index 41341387b42c2..db09738bfd22d 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -19,7 +19,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -368,26 +367,21 @@ AVRAsmBackend::createObjectTargetWriter() const {
   return createAVRELFObjectWriter(MCELFObjectTargetWriter::getOSABI(OSType));
 }
 
-bool AVRAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
-                             const MCValue &Target, uint64_t &FixedValue,
-                             bool IsResolved) {
+void AVRAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                               const MCValue &Target,
+                               MutableArrayRef<char> Data, uint64_t Value,
+                               bool IsResolved) {
   // AVR sets the fixup value to bypass the assembly time overflow with a
   // relocation.
   if (IsResolved) {
-    auto TargetVal = MCValue::get(Target.getAddSym(), Target.getSubSym(),
-                                  FixedValue, Target.getSpecifier());
+    auto TargetVal = MCValue::get(Target.getAddSym(), Target.getSubSym(), Value,
+                                  Target.getSpecifier());
     if (forceRelocation(F, Fixup, TargetVal))
       IsResolved = false;
   }
   if (!IsResolved)
-    Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
-  return IsResolved;
-}
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
 
-void AVRAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
-                               const MCValue &Target,
-                               MutableArrayRef<char> Data, uint64_t Value,
-                               bool IsResolved) {
   if (mc::isRelocation(Fixup.getKind()))
     return;
   adjustFixupValue(Fixup, Target, Value, &getContext());
@@ -439,8 +433,8 @@ MCFixupKindInfo AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       // name                    offset  bits  flags
       {"fixup_32", 0, 32, 0},
 
-      {"fixup_7_pcrel", 3, 7, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_13_pcrel", 0, 12, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_7_pcrel", 3, 7, 0},
+      {"fixup_13_pcrel", 0, 12, 0},
 
       {"fixup_16", 0, 16, 0},
       {"fixup_16_pm", 0, 16, 0},
@@ -491,7 +485,7 @@ MCFixupKindInfo AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_AVR_NONE. They do not require
   // any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index 283f937e1ad87..68c839ec8432a 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -37,9 +37,6 @@ class AVRAsmBackend : public MCAsmBackend {
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
 
-  bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                uint64_t &FixedValue, bool IsResolved) override;
-
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
index 1e4b2e27a1837..481219164a0f9 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
@@ -166,7 +166,7 @@ void AVRInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo,
     O << Imm;
   } else {
     assert(Op.isExpr() && "Unknown pcrel immediate operand");
-    O << *Op.getExpr();
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -189,7 +189,7 @@ void AVRInstPrinter::printMemri(const MCInst *MI, unsigned OpNo,
 
     O << Offset;
   } else if (OffsetOp.isExpr()) {
-    O << *OffsetOp.getExpr();
+    MAI.printExpr(O, *OffsetOp.getExpr());
   } else {
     llvm_unreachable("unknown type for offset");
   }
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
index cfd7dc5822627..2ba8383e48e80 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
@@ -195,7 +195,7 @@ bool AVRMCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
       return false;
 
     auto Spec = AVR::S_None;
-    if (Value.getSpecifier() != MCSymbolRefExpr::VK_None)
+    if (Value.getSpecifier())
       return false;
     assert(!Value.getSubSym());
     if (E.getSpecifier() == AVR::S_PM)
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
index 4934e1c71bc03..4bb16e237db48 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
@@ -35,6 +35,17 @@
 
 namespace llvm {
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case AVR::fixup_7_pcrel:
+  case AVR::fixup_13_pcrel:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 /// Performs a post-encoding step on a `LD` or `ST` instruction.
 ///
 /// The encoding of the LD/ST family of instructions is inconsistent w.r.t
@@ -110,8 +121,7 @@ AVRMCCodeEmitter::encodeRelCondBrTarget(const MCInst &MI, unsigned OpNo,
   const MCOperand &MO = MI.getOperand(OpNo);
 
   if (MO.isExpr()) {
-    Fixups.push_back(
-        MCFixup::create(0, MO.getExpr(), MCFixupKind(Fixup), MI.getLoc()));
+    addFixup(Fixups, 0, MO.getExpr(), MCFixupKind(Fixup));
     return 0;
   }
 
@@ -156,8 +166,7 @@ unsigned AVRMCCodeEmitter::encodeMemri(const MCInst &MI, unsigned OpNo,
     OffsetBits = OffsetOp.getImm();
   } else if (OffsetOp.isExpr()) {
     OffsetBits = 0;
-    Fixups.push_back(MCFixup::create(0, OffsetOp.getExpr(),
-                                     MCFixupKind(AVR::fixup_6), MI.getLoc()));
+    addFixup(Fixups, 0, OffsetOp.getExpr(), AVR::fixup_6);
   } else {
     llvm_unreachable("Invalid value for offset");
   }
@@ -191,8 +200,7 @@ unsigned AVRMCCodeEmitter::encodeImm(const MCInst &MI, unsigned OpNo,
     }
 
     MCFixupKind FixupKind = static_cast<MCFixupKind>(Fixup);
-    Fixups.push_back(
-        MCFixup::create(Offset, MO.getExpr(), FixupKind, MI.getLoc()));
+    addFixup(Fixups, Offset, MO.getExpr(), FixupKind);
 
     return 0;
   }
@@ -207,8 +215,8 @@ unsigned AVRMCCodeEmitter::encodeCallTarget(const MCInst &MI, unsigned OpNo,
   auto MO = MI.getOperand(OpNo);
 
   if (MO.isExpr()) {
-    MCFixupKind FixupKind = static_cast<MCFixupKind>(AVR::fixup_call);
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), FixupKind, MI.getLoc()));
+    MCFixupKind FixupKind = AVR::fixup_call;
+    addFixup(Fixups, 0, MO.getExpr(), FixupKind);
     return 0;
   }
 
@@ -238,7 +246,7 @@ unsigned AVRMCCodeEmitter::getExprOpValue(const MCExpr *Expr,
     }
 
     MCFixupKind FixupKind = static_cast<MCFixupKind>(AVRExpr->getFixupKind());
-    Fixups.push_back(MCFixup::create(0, AVRExpr, FixupKind));
+    addFixup(Fixups, 0, AVRExpr, FixupKind);
     return 0;
   }
 
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index b49e8fd96c66a..a347794a9a30c 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/BPFMCAsmInfo.h"
 #include "MCTargetDesc/BPFMCTargetDesc.h"
 #include "TargetInfo/BPFTargetInfo.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -164,10 +165,10 @@ struct BPFOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case Register:
       OS << "<register x";
@@ -347,6 +348,9 @@ bool BPFAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidSImm16:
     return Error(Operands[ErrorInfo]->getStartLoc(),
                  "operand is not a 16-bit signed integer");
+  case Match_InvalidTiedOperand:
+    return Error(Operands[ErrorInfo]->getStartLoc(),
+                 "operand is not the same as the dst register");
   }
 
   llvm_unreachable("Unknown match type detected!");
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index 1704fecaff031..958790d49d087 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -12,7 +12,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/Support/EndianStream.h"
 #include <cassert>
@@ -44,7 +43,7 @@ class BPFAsmBackend : public MCAsmBackend {
 
 MCFixupKindInfo BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[BPF::NumTargetFixupKinds] = {
-    { "FK_BPF_PCRel_4",  0, 32, MCFixupKindInfo::FKF_IsPCRel },
+      {"FK_BPF_PCRel_4", 0, 32, 0},
   };
 
   if (Kind < FirstTargetFixupKind)
@@ -66,10 +65,11 @@ bool BPFAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-void BPFAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void BPFAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                const MCValue &Target,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   if (Fixup.getKind() == FK_SecRel_8) {
     // The Value is 0 for global variables, and the in-section offset
     // for static variables. Write to the immediate field of the inst.
@@ -77,11 +77,11 @@ void BPFAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
     support::endian::write<uint32_t>(&Data[Fixup.getOffset() + 4],
                                      static_cast<uint32_t>(Value),
                                      Endian);
-  } else if (Fixup.getKind() == FK_Data_4) {
+  } else if (Fixup.getKind() == FK_Data_4 && !Fixup.isPCRel()) {
     support::endian::write<uint32_t>(&Data[Fixup.getOffset()], Value, Endian);
   } else if (Fixup.getKind() == FK_Data_8) {
     support::endian::write<uint64_t>(&Data[Fixup.getOffset()], Value, Endian);
-  } else if (Fixup.getKind() == FK_PCRel_4) {
+  } else if (Fixup.getKind() == FK_Data_4 && Fixup.isPCRel()) {
     Value = (uint32_t)((Value - 8) / 8);
     if (Endian == llvm::endianness::little) {
       Data[Fixup.getOffset() + 1] = 0x10;
@@ -96,7 +96,7 @@ void BPFAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
     support::endian::write<uint32_t>(&Data[Fixup.getOffset() + 4], Value,
                                      Endian);
   } else {
-    assert(Fixup.getKind() == FK_PCRel_2);
+    assert(Fixup.getKind() == FK_Data_2 && Fixup.isPCRel());
 
     int64_t ByteOff = (int64_t)Value - 8;
     if (ByteOff > INT16_MAX * 8 || ByteOff < INT16_MIN * 8)
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
index 5d653eed1755c..827e928b9b660 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
@@ -45,12 +45,11 @@ unsigned BPFELFObjectWriter::getRelocType(const MCFixup &Fixup,
   case FK_SecRel_8:
     // LD_imm64 instruction.
     return ELF::R_BPF_64_64;
-  case FK_PCRel_4:
-    // CALL instruction.
-    return ELF::R_BPF_64_32;
   case FK_Data_8:
     return ELF::R_BPF_64_ABS64;
   case FK_Data_4:
+    if (Fixup.isPCRel()) // CALL instruction
+      return ELF::R_BPF_64_32;
     if (const auto *A = Target.getAddSym()) {
       const MCSymbol &Sym = *A;
 
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp
index 83ff3ef327de3..a0011e81ddd47 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp
@@ -32,20 +32,6 @@ void BPFInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   printAnnotation(O, Annot);
 }
 
-static void printExpr(const MCExpr *Expr, raw_ostream &O) {
-  const MCSymbolRefExpr *SRE;
-
-  if (const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(Expr))
-    SRE = dyn_cast<MCSymbolRefExpr>(BE->getLHS());
-  else
-    SRE = dyn_cast<MCSymbolRefExpr>(Expr);
-  if (!SRE)
-    report_fatal_error("Unexpected MCExpr type.");
-
-  assert(SRE->getSpecifier() == 0);
-  O << *Expr;
-}
-
 void BPFInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                   raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
@@ -55,7 +41,7 @@ void BPFInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << formatImm((int32_t)Op.getImm());
   } else {
     assert(Op.isExpr() && "Expected an expression");
-    printExpr(Op.getExpr(), O);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -86,7 +72,7 @@ void BPFInstPrinter::printImm64Operand(const MCInst *MI, unsigned OpNo,
   if (Op.isImm())
     O << formatImm(Op.getImm());
   else if (Op.isExpr())
-    printExpr(Op.getExpr(), O);
+    MAI.printExpr(O, *Op.getExpr());
   else
     O << Op;
 }
@@ -103,7 +89,7 @@ void BPFInstPrinter::printBrTargetOperand(const MCInst *MI, unsigned OpNo,
       O << ((Imm >= 0) ? "+" : "") << formatImm(Imm);
     }
   } else if (Op.isExpr()) {
-    printExpr(Op.getExpr(), O);
+    MAI.printExpr(O, *Op.getExpr());
   } else {
     O << Op;
   }
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index bd9d2de58c8b2..cd32e7d16fcac 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -77,6 +77,11 @@ MCCodeEmitter *llvm::createBPFbeMCCodeEmitter(const MCInstrInfo &MCII,
   return new BPFMCCodeEmitter(MCII, *Ctx.getRegisterInfo(), false, Ctx);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
                                              const MCOperand &MO,
                                              SmallVectorImpl<MCFixup> &Fixups,
@@ -102,14 +107,14 @@ unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
 
   if (MI.getOpcode() == BPF::JAL)
     // func call name
-    Fixups.push_back(MCFixup::create(0, Expr, FK_PCRel_4));
+    addFixup(Fixups, 0, Expr, FK_Data_4, true);
   else if (MI.getOpcode() == BPF::LD_imm64)
-    Fixups.push_back(MCFixup::create(0, Expr, FK_SecRel_8));
+    addFixup(Fixups, 0, Expr, FK_SecRel_8);
   else if (MI.getOpcode() == BPF::JMPL)
-    Fixups.push_back(MCFixup::create(0, Expr, (MCFixupKind)BPF::FK_BPF_PCRel_4));
+    addFixup(Fixups, 0, Expr, BPF::FK_BPF_PCRel_4, true);
   else
     // bb label
-    Fixups.push_back(MCFixup::create(0, Expr, FK_PCRel_2));
+    addFixup(Fixups, 0, Expr, FK_Data_2, true);
 
   return 0;
 }
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 9472288229cac..f5f6f5386f899 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -22,7 +22,7 @@ add_llvm_component_library(LLVMTarget
 # When building shared objects for each target there are some internal APIs
 # that are used across shared objects which we can't hide.
 if (NOT BUILD_SHARED_LIBS AND NOT APPLE AND
-    (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
+    (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
     NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
     NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
   # Set default visibility to hidden, so we don't export all the Target classes
diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
index 2ac1212e4bfd6..5f7e1497a1ebf 100644
--- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
+++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/CSKYInstPrinter.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "MCTargetDesc/CSKYMCTargetDesc.h"
 #include "MCTargetDesc/CSKYTargetStreamer.h"
 #include "TargetInfo/CSKYTargetInfo.h"
@@ -430,7 +430,7 @@ struct CSKYOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     auto RegName = [](MCRegister Reg) {
       if (Reg)
         return CSKYInstPrinter::getRegisterName(Reg);
@@ -440,10 +440,10 @@ struct CSKYOperand : public MCParsedAsmOperand {
 
     switch (Kind) {
     case CPOP:
-      OS << *getConstpoolOp();
+      MAI.printExpr(OS, *getConstpoolOp());
       break;
     case Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case KindTy::Register:
       OS << "<register " << RegName(getReg()) << ">";
@@ -848,11 +848,11 @@ bool CSKYAsmParser::processLRW(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out) {
     }
   } else {
     const MCExpr *AdjustExpr = nullptr;
-    if (const CSKYMCExpr *CSKYExpr =
-            dyn_cast<CSKYMCExpr>(Inst.getOperand(1).getExpr())) {
-      if (CSKYExpr->getSpecifier() == CSKYMCExpr::VK_TLSGD ||
-          CSKYExpr->getSpecifier() == CSKYMCExpr::VK_TLSIE ||
-          CSKYExpr->getSpecifier() == CSKYMCExpr::VK_TLSLDM) {
+    if (const auto *CSKYExpr =
+            dyn_cast<MCSpecifierExpr>(Inst.getOperand(1).getExpr())) {
+      if (CSKYExpr->getSpecifier() == CSKY::S_TLSGD ||
+          CSKYExpr->getSpecifier() == CSKY::S_TLSIE ||
+          CSKYExpr->getSpecifier() == CSKY::S_TLSLDM) {
         MCSymbol *Dot = getContext().createNamedTempSymbol();
         Out.emitLabel(Dot);
         AdjustExpr = MCSymbolRefExpr::create(Dot, getContext());
@@ -1172,25 +1172,25 @@ ParseStatus CSKYAsmParser::parseCSKYSymbol(OperandVector &Operands) {
   if (getParser().parseIdentifier(Identifier))
     return Error(getLoc(), "unknown identifier");
 
-  CSKYMCExpr::Specifier Kind = CSKYMCExpr::VK_None;
+  CSKY::Specifier Kind = CSKY::S_None;
   if (Identifier.consume_back("@GOT"))
-    Kind = CSKYMCExpr::VK_GOT;
+    Kind = CSKY::S_GOT;
   else if (Identifier.consume_back("@GOTOFF"))
-    Kind = CSKYMCExpr::VK_GOTOFF;
+    Kind = CSKY::S_GOTOFF;
   else if (Identifier.consume_back("@PLT"))
-    Kind = CSKYMCExpr::VK_PLT;
+    Kind = CSKY::S_PLT;
   else if (Identifier.consume_back("@GOTPC"))
-    Kind = CSKYMCExpr::VK_GOTPC;
+    Kind = CSKY::S_GOTPC;
   else if (Identifier.consume_back("@TLSGD32"))
-    Kind = CSKYMCExpr::VK_TLSGD;
+    Kind = CSKY::S_TLSGD;
   else if (Identifier.consume_back("@GOTTPOFF"))
-    Kind = CSKYMCExpr::VK_TLSIE;
+    Kind = CSKY::S_TLSIE;
   else if (Identifier.consume_back("@TPOFF"))
-    Kind = CSKYMCExpr::VK_TLSLE;
+    Kind = CSKY::S_TLSLE;
   else if (Identifier.consume_back("@TLSLDM32"))
-    Kind = CSKYMCExpr::VK_TLSLDM;
+    Kind = CSKY::S_TLSLDM;
   else if (Identifier.consume_back("@TLSLDO32"))
-    Kind = CSKYMCExpr::VK_TLSLDO;
+    Kind = CSKY::S_TLSLDO;
 
   MCSymbol *Sym = getContext().getInlineAsmLabel(Identifier);
 
@@ -1210,8 +1210,8 @@ ParseStatus CSKYAsmParser::parseCSKYSymbol(OperandVector &Operands) {
   MCBinaryExpr::Opcode Opcode;
   switch (getLexer().getKind()) {
   default:
-    if (Kind != CSKYMCExpr::VK_None)
-      Res = CSKYMCExpr::create(Res, Kind, getContext());
+    if (Kind != CSKY::S_None)
+      Res = MCSpecifierExpr::create(Res, Kind, getContext());
 
     Operands.push_back(CSKYOperand::createImm(Res, S, E));
     return ParseStatus::Success;
@@ -1258,11 +1258,11 @@ ParseStatus CSKYAsmParser::parseDataSymbol(OperandVector &Operands) {
   if (getParser().parseIdentifier(Identifier))
     return Error(getLoc(), "unknown identifier " + Identifier);
 
-  CSKYMCExpr::Specifier Kind = CSKYMCExpr::VK_None;
+  CSKY::Specifier Kind = CSKY::S_None;
   if (Identifier.consume_back("@GOT"))
-    Kind = CSKYMCExpr::VK_GOT_IMM18_BY4;
+    Kind = CSKY::S_GOT_IMM18_BY4;
   else if (Identifier.consume_back("@PLT"))
-    Kind = CSKYMCExpr::VK_PLT_IMM18_BY4;
+    Kind = CSKY::S_PLT_IMM18_BY4;
 
   MCSymbol *Sym = getContext().getInlineAsmLabel(Identifier);
 
@@ -1288,8 +1288,8 @@ ParseStatus CSKYAsmParser::parseDataSymbol(OperandVector &Operands) {
 
     getLexer().Lex(); // Eat ']'.
 
-    if (Kind != CSKYMCExpr::VK_None)
-      Res = CSKYMCExpr::create(Res, Kind, getContext());
+    if (Kind != CSKY::S_None)
+      Res = MCSpecifierExpr::create(Res, Kind, getContext());
 
     Operands.push_back(CSKYOperand::createConstpoolOp(Res, S, E));
     return ParseStatus::Success;
diff --git a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
index 9a259eac52f47..7e1eb9d9ec43a 100644
--- a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
+++ b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
@@ -15,7 +15,7 @@
 #include "CSKYConstantPoolValue.h"
 #include "CSKYTargetMachine.h"
 #include "MCTargetDesc/CSKYInstPrinter.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "MCTargetDesc/CSKYTargetStreamer.h"
 #include "TargetInfo/CSKYTargetInfo.h"
 #include "llvm/ADT/Statistic.h"
@@ -168,25 +168,24 @@ void CSKYAsmPrinter::emitInstruction(const MachineInstr *MI) {
 
 // Convert a CSKY-specific constant pool modifier into the associated
 // MCSymbolRefExpr variant kind.
-static CSKYMCExpr::Specifier
-getModifierVariantKind(CSKYCP::CSKYCPModifier Modifier) {
+static CSKY::Specifier getModifierVariantKind(CSKYCP::CSKYCPModifier Modifier) {
   switch (Modifier) {
   case CSKYCP::NO_MOD:
-    return CSKYMCExpr::VK_None;
+    return CSKY::S_None;
   case CSKYCP::ADDR:
-    return CSKYMCExpr::VK_ADDR;
+    return CSKY::S_ADDR;
   case CSKYCP::GOT:
-    return CSKYMCExpr::VK_GOT;
+    return CSKY::S_GOT;
   case CSKYCP::GOTOFF:
-    return CSKYMCExpr::VK_GOTOFF;
+    return CSKY::S_GOTOFF;
   case CSKYCP::PLT:
-    return CSKYMCExpr::VK_PLT;
+    return CSKY::S_PLT;
   case CSKYCP::TLSGD:
-    return CSKYMCExpr::VK_TLSGD;
+    return CSKY::S_TLSGD;
   case CSKYCP::TLSLE:
-    return CSKYMCExpr::VK_TLSLE;
+    return CSKY::S_TLSLE;
   case CSKYCP::TLSIE:
-    return CSKYMCExpr::VK_TLSIE;
+    return CSKY::S_TLSIE;
   }
   llvm_unreachable("Invalid CSKYCPModifier!");
 }
@@ -240,8 +239,8 @@ void CSKYAsmPrinter::emitMachineConstantPoolValue(
   }
 
   // Create an MCSymbol for the reference.
-  Expr = CSKYMCExpr::create(Expr, getModifierVariantKind(CCPV->getModifier()),
-                            OutContext);
+  Expr = MCSpecifierExpr::create(
+      Expr, getModifierVariantKind(CCPV->getModifier()), OutContext);
 
   OutStreamer->emitValue(Expr, Size);
 }
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index 34221cc699589..707017106e61d 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -1207,9 +1207,6 @@ SDValue CSKYTargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
diff --git a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
index a91f94bb2357a..db38c622dcd2d 100644
--- a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
+++ b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
@@ -13,7 +13,7 @@
 
 #include "CSKYMCInstLower.h"
 #include "MCTargetDesc/CSKYBaseInfo.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/MC/MCExpr.h"
 
@@ -36,38 +36,38 @@ void CSKYMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
 
 MCOperand CSKYMCInstLower::lowerSymbolOperand(const MachineOperand &MO,
                                               MCSymbol *Sym) const {
-  CSKYMCExpr::Specifier Spec;
+  CSKY::Specifier Spec;
   MCContext &Ctx = Printer.OutContext;
 
   switch (MO.getTargetFlags()) {
   default:
     llvm_unreachable("Unknown target flag.");
   case CSKYII::MO_None:
-    Spec = CSKYMCExpr::VK_None;
+    Spec = CSKY::S_None;
     break;
   case CSKYII::MO_GOT32:
-    Spec = CSKYMCExpr::VK_GOT;
+    Spec = CSKY::S_GOT;
     break;
   case CSKYII::MO_GOTOFF:
-    Spec = CSKYMCExpr::VK_GOTOFF;
+    Spec = CSKY::S_GOTOFF;
     break;
   case CSKYII::MO_ADDR32:
-    Spec = CSKYMCExpr::VK_ADDR;
+    Spec = CSKY::S_ADDR;
     break;
   case CSKYII::MO_PLT32:
-    Spec = CSKYMCExpr::VK_PLT;
+    Spec = CSKY::S_PLT;
     break;
   case CSKYII::MO_ADDR_HI16:
-    Spec = CSKYMCExpr::VK_ADDR_HI16;
+    Spec = CSKY::S_ADDR_HI16;
     break;
   case CSKYII::MO_ADDR_LO16:
-    Spec = CSKYMCExpr::VK_ADDR_LO16;
+    Spec = CSKY::S_ADDR_LO16;
     break;
   }
   const MCExpr *ME = MCSymbolRefExpr::create(Sym, Ctx);
 
-  if (Spec != CSKYMCExpr::VK_None)
-    ME = CSKYMCExpr::create(ME, Spec, Ctx);
+  if (Spec != CSKY::S_None)
+    ME = MCSpecifierExpr::create(ME, Spec, Ctx);
 
   return MCOperand::createExpr(ME);
 }
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
index 73b4429349ea8..e0932808bb7af 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMCSKYDesc
   CSKYELFStreamer.cpp
   CSKYInstPrinter.cpp
   CSKYMCAsmInfo.cpp
-  CSKYMCExpr.cpp
   CSKYMCTargetDesc.cpp
   CSKYMCCodeEmitter.cpp
   CSKYTargetStreamer.cpp
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
index 47d2728915707..ce1da6e58b9cd 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
@@ -11,7 +11,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
@@ -32,34 +31,27 @@ MCFixupKindInfo CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {CSKY::Fixups::fixup_csky_addr_hi16, {"fixup_csky_addr_hi16", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_addr_lo16, {"fixup_csky_addr_lo16", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm16_scale2,
-       {"fixup_csky_pcrel_imm16_scale2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm16_scale2", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_uimm16_scale4,
-       {"fixup_csky_pcrel_uimm16_scale4", 0, 32,
-        MCFixupKindInfo::FKF_IsPCRel |
-            MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}},
+       {"fixup_csky_pcrel_uimm16_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_uimm8_scale4,
-       {"fixup_csky_pcrel_uimm8_scale4", 0, 32,
-        MCFixupKindInfo::FKF_IsPCRel |
-            MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}},
+       {"fixup_csky_pcrel_uimm8_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm26_scale2,
-       {"fixup_csky_pcrel_imm26_scale2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm26_scale2", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm18_scale2,
-       {"fixup_csky_pcrel_imm18_scale2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm18_scale2", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_got32, {"fixup_csky_got32", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_got_imm18_scale4,
        {"fixup_csky_got_imm18_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_gotoff, {"fixup_csky_gotoff", 0, 32, 0}},
-      {CSKY::Fixups::fixup_csky_gotpc,
-       {"fixup_csky_gotpc", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+      {CSKY::Fixups::fixup_csky_gotpc, {"fixup_csky_gotpc", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_plt32, {"fixup_csky_plt32", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_plt_imm18_scale4,
        {"fixup_csky_plt_imm18_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm10_scale2,
-       {"fixup_csky_pcrel_imm10_scale2", 0, 16, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm10_scale2", 0, 16, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_uimm7_scale4,
-       {"fixup_csky_pcrel_uimm7_scale4", 0, 16,
-        MCFixupKindInfo::FKF_IsPCRel |
-            MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}},
+       {"fixup_csky_pcrel_uimm7_scale4", 0, 16, 0}},
       {CSKY::Fixups::fixup_csky_doffset_imm18,
        {"fixup_csky_doffset_imm18", 0, 18, 0}},
       {CSKY::Fixups::fixup_csky_doffset_imm18_scale2,
@@ -71,7 +63,7 @@ MCFixupKindInfo CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
          "Not all fixup kinds added to Infos array");
 
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
   return Infos[Kind];
@@ -188,10 +180,29 @@ bool CSKYAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
   }
 }
 
-void CSKYAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+std::optional<bool> CSKYAsmBackend::evaluateFixup(const MCFragment &F,
+                                                  MCFixup &Fixup, MCValue &,
+                                                  uint64_t &Value) {
+  // For a few PC-relative fixups, offsets need to be aligned down. We
+  // compensate here because the default handler's `Value` decrement doesn't
+  // account for this alignment.
+  switch (Fixup.getTargetKind()) {
+  case CSKY::fixup_csky_pcrel_uimm16_scale4:
+  case CSKY::fixup_csky_pcrel_uimm8_scale4:
+  case CSKY::fixup_csky_pcrel_uimm7_scale4:
+    Value = (Asm->getFragmentOffset(F) + Fixup.getOffset()) % 4;
+  }
+  return {};
+}
+
+void CSKYAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                 const MCValue &Target,
                                 MutableArrayRef<char> Data, uint64_t Value,
                                 bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
+
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
@@ -228,9 +239,9 @@ void CSKYAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
   }
 }
 
-bool CSKYAsmBackend::mayNeedRelaxation(const MCInst &Inst,
+bool CSKYAsmBackend::mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand>,
                                        const MCSubtargetInfo &STI) const {
-  switch (Inst.getOpcode()) {
+  switch (Opcode) {
   default:
     return false;
   case CSKY::JBR32:
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
index 099aaa6e67cd7..1d3a22c2bbbb4 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
@@ -22,6 +22,8 @@ class CSKYAsmBackend : public MCAsmBackend {
   CSKYAsmBackend(const MCSubtargetInfo &STI, const MCTargetOptions &OP)
       : MCAsmBackend(llvm::endianness::little) {}
 
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
@@ -31,20 +33,18 @@ class CSKYAsmBackend : public MCAsmBackend {
   bool fixupNeedsRelaxation(const MCFixup &Fixup,
                             uint64_t Value) const override;
 
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
+                         const MCSubtargetInfo &STI) const override;
   void relaxInstruction(MCInst &Inst,
                         const MCSubtargetInfo &STI) const override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override;
-
   bool fixupNeedsRelaxationAdvanced(const MCFixup &, const MCValue &, uint64_t,
                                     bool) const override;
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
 
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
index 0a2087db3fee4..1de82e6cc6ce8 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYFixupKinds.h"
-#include "CSKYMCExpr.h"
+#include "CSKYMCAsmInfo.h"
 #include "CSKYMCTargetDesc.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -43,11 +43,11 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
   uint8_t Modifier = Target.getSpecifier();
 
   switch (Target.getSpecifier()) {
-  case CSKYMCExpr::VK_TLSIE:
-  case CSKYMCExpr::VK_TLSLE:
-  case CSKYMCExpr::VK_TLSGD:
-  case CSKYMCExpr::VK_TLSLDM:
-  case CSKYMCExpr::VK_TLSLDO:
+  case CSKY::S_TLSIE:
+  case CSKY::S_TLSLE:
+  case CSKY::S_TLSGD:
+  case CSKY::S_TLSLDM:
+  case CSKY::S_TLSLDO:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -62,7 +62,6 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
       reportError(Fixup.getLoc(), "Unsupported relocation type");
       return ELF::R_CKCORE_NONE;
     case FK_Data_4:
-    case FK_PCRel_4:
       return ELF::R_CKCORE_PCREL32;
     case CSKY::fixup_csky_pcrel_uimm16_scale4:
       return ELF::R_CKCORE_PCREL_IMM16_4;
@@ -93,29 +92,29 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Fixup.getLoc(), "2-byte data relocations not supported");
     return ELF::R_CKCORE_NONE;
   case FK_Data_4:
-    if (Expr->getKind() == MCExpr::Target) {
-      auto TK = cast<CSKYMCExpr>(Expr)->getSpecifier();
-      if (TK == CSKYMCExpr::VK_ADDR)
+    if (Expr->getKind() == MCExpr::Specifier) {
+      auto TK = cast<MCSpecifierExpr>(Expr)->getSpecifier();
+      if (TK == CSKY::S_ADDR)
         return ELF::R_CKCORE_ADDR32;
-      if (TK == CSKYMCExpr::VK_GOT)
+      if (TK == CSKY::S_GOT)
         return ELF::R_CKCORE_GOT32;
-      if (TK == CSKYMCExpr::VK_GOTOFF)
+      if (TK == CSKY::S_GOTOFF)
         return ELF::R_CKCORE_GOTOFF;
-      if (TK == CSKYMCExpr::VK_PLT)
+      if (TK == CSKY::S_PLT)
         return ELF::R_CKCORE_PLT32;
-      if (TK == CSKYMCExpr::VK_TLSIE)
+      if (TK == CSKY::S_TLSIE)
         return ELF::R_CKCORE_TLS_IE32;
-      if (TK == CSKYMCExpr::VK_TLSLE)
+      if (TK == CSKY::S_TLSLE)
         return ELF::R_CKCORE_TLS_LE32;
-      if (TK == CSKYMCExpr::VK_TLSGD)
+      if (TK == CSKY::S_TLSGD)
         return ELF::R_CKCORE_TLS_GD32;
-      if (TK == CSKYMCExpr::VK_TLSLDM)
+      if (TK == CSKY::S_TLSLDM)
         return ELF::R_CKCORE_TLS_LDM32;
-      if (TK == CSKYMCExpr::VK_TLSLDO)
+      if (TK == CSKY::S_TLSLDO)
         return ELF::R_CKCORE_TLS_LDO32;
-      if (TK == CSKYMCExpr::VK_GOTPC)
+      if (TK == CSKY::S_GOTPC)
         return ELF::R_CKCORE_GOTPC;
-      if (TK == CSKYMCExpr::VK_None)
+      if (TK == CSKY::S_None)
         return ELF::R_CKCORE_ADDR32;
 
       LLVM_DEBUG(dbgs() << "Unknown FK_Data_4 TK  = " << TK);
@@ -125,19 +124,19 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Fixup.getLoc(), "invalid fixup for 4-byte data relocation");
         return ELF::R_CKCORE_NONE;
-      case CSKYMCExpr::VK_GOT:
+      case CSKY::S_GOT:
         return ELF::R_CKCORE_GOT32;
-      case CSKYMCExpr::VK_GOTOFF:
+      case CSKY::S_GOTOFF:
         return ELF::R_CKCORE_GOTOFF;
-      case CSKYMCExpr::VK_PLT:
+      case CSKY::S_PLT:
         return ELF::R_CKCORE_PLT32;
-      case CSKYMCExpr::VK_TLSGD:
+      case CSKY::S_TLSGD:
         return ELF::R_CKCORE_TLS_GD32;
-      case CSKYMCExpr::VK_TLSLDM:
+      case CSKY::S_TLSLDM:
         return ELF::R_CKCORE_TLS_LDM32;
-      case CSKYMCExpr::VK_TPOFF:
+      case CSKY::S_TPOFF:
         return ELF::R_CKCORE_TLS_LE32;
-      case CSKYMCExpr::VK_None:
+      case CSKY::S_None:
         return ELF::R_CKCORE_ADDR32;
       }
     }
@@ -167,8 +166,8 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
 bool CSKYELFObjectWriter::needsRelocateWithSymbol(const MCValue &V,
                                                   unsigned Type) const {
   switch (V.getSpecifier()) {
-  case CSKYMCExpr::VK_PLT:
-  case CSKYMCExpr::VK_GOT:
+  case CSKY::S_PLT:
+  case CSKY::S_GOT:
     return true;
   default:
     return false;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
index 47c05d67296e3..90428e34cefa3 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 #include "CSKYInstPrinter.h"
 #include "MCTargetDesc/CSKYBaseInfo.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -138,7 +138,7 @@ void CSKYInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void CSKYInstPrinter::printDataSymbol(const MCInst *MI, unsigned OpNo,
@@ -150,7 +150,7 @@ void CSKYInstPrinter::printDataSymbol(const MCInst *MI, unsigned OpNo,
   if (MO.isImm())
     O << MO.getImm();
   else
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
   O << "]";
 }
 
@@ -173,7 +173,7 @@ void CSKYInstPrinter::printConstpool(const MCInst *MI, uint64_t Address,
   assert(MO.isExpr() && "Unknown operand kind in printConstpool");
 
   O << "[";
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
   O << "]";
 }
 
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
index 6e10ce4570c06..803b75332f7b0 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYMCAsmInfo.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
@@ -19,9 +19,9 @@
 using namespace llvm;
 
 const MCAsmInfo::AtSpecifier atSpecifiers[] = {
-    {CSKYMCExpr::VK_GOT, "GOT"},       {CSKYMCExpr::VK_GOTOFF, "GOTOFF"},
-    {CSKYMCExpr::VK_PLT, "PLT"},       {CSKYMCExpr::VK_TLSGD, "TLSGD"},
-    {CSKYMCExpr::VK_TLSLDM, "TLSLDM"}, {CSKYMCExpr::VK_TPOFF, "TPOFF"},
+    {CSKY::S_GOT, "GOT"},       {CSKY::S_GOTOFF, "GOTOFF"},
+    {CSKY::S_PLT, "PLT"},       {CSKY::S_TLSGD, "TLSGD"},
+    {CSKY::S_TLSLDM, "TLSLDM"}, {CSKY::S_TPOFF, "TPOFF"},
 };
 
 void CSKYMCAsmInfo::anchor() {}
@@ -38,3 +38,44 @@ CSKYMCAsmInfo::CSKYMCAsmInfo(const Triple &TargetTriple) {
 
   initializeAtSpecifiers(atSpecifiers);
 }
+
+static StringRef getVariantKindName(uint8_t Kind) {
+  using namespace CSKY;
+  switch (Kind) {
+  default:
+    llvm_unreachable("Invalid ELF symbol kind");
+  case S_None:
+  case S_ADDR:
+    return "";
+  case S_ADDR_HI16:
+    return "@HI16";
+  case S_ADDR_LO16:
+    return "@LO16";
+  case S_GOT_IMM18_BY4:
+  case S_GOT:
+    return "@GOT";
+  case S_GOTPC:
+    return "@GOTPC";
+  case S_GOTOFF:
+    return "@GOTOFF";
+  case S_PLT_IMM18_BY4:
+  case S_PLT:
+    return "@PLT";
+  case S_TLSLE:
+    return "@TPOFF";
+  case S_TLSIE:
+    return "@GOTTPOFF";
+  case S_TLSGD:
+    return "@TLSGD32";
+  case S_TLSLDO:
+    return "@TLSLDO32";
+  case S_TLSLDM:
+    return "@TLSLDM32";
+  }
+}
+
+void CSKYMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                       const MCSpecifierExpr &Expr) const {
+  printExpr(OS, *Expr.getSubExpr());
+  OS << getVariantKindName(Expr.getSpecifier());
+}
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
index 3e0609f195311..aa4ba147c00a5 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
@@ -14,6 +14,8 @@
 #define LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCASMINFO_H
 
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCValue.h"
 
 namespace llvm {
 class Triple;
@@ -23,7 +25,33 @@ class CSKYMCAsmInfo : public MCAsmInfoELF {
 
 public:
   explicit CSKYMCAsmInfo(const Triple &TargetTriple);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
+
+namespace CSKY {
+using Specifier = uint8_t;
+enum {
+  S_None,
+  S_ADDR,
+  S_ADDR_HI16,
+  S_ADDR_LO16,
+  S_PCREL,
+  S_GOT,
+  S_GOT_IMM18_BY4,
+  S_GOTPC,
+  S_GOTOFF,
+  S_PLT,
+  S_PLT_IMM18_BY4,
+  S_TLSIE,
+  S_TLSLE,
+  S_TLSGD,
+  S_TLSLDO,
+  S_TLSLDM,
+  S_TPOFF,
+  S_Invalid
+};
+} // namespace CSKY
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCASMINFO_H
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
index 70a797ddd331e..5f9f55eb606c2 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
@@ -10,10 +10,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CSKYMCCodeEmitter.h"
-#include "CSKYMCExpr.h"
+#include "CSKYMCAsmInfo.h"
+#include "MCTargetDesc/CSKYFixupKinds.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "MCTargetDesc/CSKYMCTargetDesc.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -27,6 +30,186 @@ using namespace llvm;
 
 STATISTIC(MCNumEmitted, "Number of MC instructions emitted");
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case CSKY::Fixups::fixup_csky_pcrel_imm16_scale2:
+  case CSKY::Fixups::fixup_csky_pcrel_uimm16_scale4:
+  case CSKY::Fixups::fixup_csky_pcrel_imm26_scale2:
+  case CSKY::Fixups::fixup_csky_pcrel_imm18_scale2:
+  case CSKY::Fixups::fixup_csky_gotpc:
+  case CSKY::Fixups::fixup_csky_pcrel_imm10_scale2:
+  case CSKY::Fixups::fixup_csky_pcrel_uimm7_scale4:
+  case CSKY::Fixups::fixup_csky_pcrel_uimm8_scale4:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
+namespace {
+class CSKYMCCodeEmitter : public MCCodeEmitter {
+  MCContext &Ctx;
+  const MCInstrInfo &MII;
+
+public:
+  CSKYMCCodeEmitter(MCContext &Ctx, const MCInstrInfo &MII)
+      : Ctx(Ctx), MII(MII) {}
+
+  ~CSKYMCCodeEmitter() {}
+
+  void encodeInstruction(const MCInst &Inst, SmallVectorImpl<char> &CB,
+                         SmallVectorImpl<MCFixup> &Fixups,
+                         const MCSubtargetInfo &STI) const override;
+
+  // Generated by tablegen.
+  uint64_t getBinaryCodeForInstr(const MCInst &MI,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const;
+
+  // Default encoding method used by tablegen.
+  unsigned getMachineOpValue(const MCInst &MI, const MCOperand &MO,
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
+
+  template <int shift = 0>
+  unsigned getImmOpValue(const MCInst &MI, unsigned Idx,
+                         SmallVectorImpl<MCFixup> &Fixups,
+                         const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    if (MO.isImm())
+      return (MO.getImm() >> shift);
+
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = getTargetFixup(MO.getExpr());
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  unsigned getRegSeqImmOpValue(const MCInst &MI, unsigned Idx,
+                               SmallVectorImpl<MCFixup> &Fixups,
+                               const MCSubtargetInfo &STI) const;
+
+  unsigned getRegisterSeqOpValue(const MCInst &MI, unsigned Op,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const;
+
+  unsigned getOImmOpValue(const MCInst &MI, unsigned Idx,
+                          SmallVectorImpl<MCFixup> &Fixups,
+                          const MCSubtargetInfo &STI) const;
+
+  unsigned getImmOpValueIDLY(const MCInst &MI, unsigned Idx,
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
+
+  unsigned getImmJMPIX(const MCInst &MI, unsigned Idx,
+                       SmallVectorImpl<MCFixup> &Fixups,
+                       const MCSubtargetInfo &STI) const;
+
+  unsigned getImmOpValueMSBSize(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const;
+
+  unsigned getImmShiftOpValue(const MCInst &MI, unsigned Idx,
+                              SmallVectorImpl<MCFixup> &Fixups,
+                              const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isImm() && "Unexpected MO type.");
+    return 1 << MO.getImm();
+  }
+
+  MCFixupKind getTargetFixup(const MCExpr *Expr) const;
+
+  template <llvm::CSKY::Fixups FIXUP>
+  unsigned getBranchSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                  SmallVectorImpl<MCFixup> &Fixups,
+                                  const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+
+    if (MO.isImm())
+      return MO.getImm() >> 1;
+
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = MCFixupKind(FIXUP);
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  template <llvm::CSKY::Fixups FIXUP>
+  unsigned getConstpoolSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                     SmallVectorImpl<MCFixup> &Fixups,
+                                     const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = MCFixupKind(FIXUP);
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  template <llvm::CSKY::Fixups FIXUP>
+  unsigned getDataSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = MCFixupKind(FIXUP);
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  unsigned getCallSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = CSKY::fixup_csky_pcrel_imm26_scale2;
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  unsigned getBareSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = CSKY::fixup_csky_pcrel_imm18_scale2;
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  void expandJBTF(const MCInst &MI, SmallVectorImpl<char> &CB,
+                  SmallVectorImpl<MCFixup> &Fixups,
+                  const MCSubtargetInfo &STI) const;
+  void expandNEG(const MCInst &MI, SmallVectorImpl<char> &CB,
+                 SmallVectorImpl<MCFixup> &Fixups,
+                 const MCSubtargetInfo &STI) const;
+  void expandRSUBI(const MCInst &MI, SmallVectorImpl<char> &CB,
+                   SmallVectorImpl<MCFixup> &Fixups,
+                   const MCSubtargetInfo &STI) const;
+};
+} // namespace
+
 unsigned CSKYMCCodeEmitter::getOImmOpValue(const MCInst &MI, unsigned Idx,
                                            SmallVectorImpl<MCFixup> &Fixups,
                                            const MCSubtargetInfo &STI) const {
@@ -293,28 +476,28 @@ unsigned CSKYMCCodeEmitter::getImmJMPIX(const MCInst &MI, unsigned Idx,
 }
 
 MCFixupKind CSKYMCCodeEmitter::getTargetFixup(const MCExpr *Expr) const {
-  const CSKYMCExpr *CSKYExpr = cast<CSKYMCExpr>(Expr);
+  const auto *CSKYExpr = cast<MCSpecifierExpr>(Expr);
   switch (CSKYExpr->getSpecifier()) {
   default:
     llvm_unreachable("Unhandled fixup kind!");
-  case CSKYMCExpr::VK_ADDR:
-    return MCFixupKind(CSKY::fixup_csky_addr32);
-  case CSKYMCExpr::VK_ADDR_HI16:
-    return MCFixupKind(CSKY::fixup_csky_addr_hi16);
-  case CSKYMCExpr::VK_ADDR_LO16:
-    return MCFixupKind(CSKY::fixup_csky_addr_lo16);
-  case CSKYMCExpr::VK_GOT:
-    return MCFixupKind(CSKY::fixup_csky_got32);
-  case CSKYMCExpr::VK_GOTPC:
-    return MCFixupKind(CSKY::fixup_csky_gotpc);
-  case CSKYMCExpr::VK_GOTOFF:
-    return MCFixupKind(CSKY::fixup_csky_gotoff);
-  case CSKYMCExpr::VK_PLT:
-    return MCFixupKind(CSKY::fixup_csky_plt32);
-  case CSKYMCExpr::VK_PLT_IMM18_BY4:
-    return MCFixupKind(CSKY::fixup_csky_plt_imm18_scale4);
-  case CSKYMCExpr::VK_GOT_IMM18_BY4:
-    return MCFixupKind(CSKY::fixup_csky_got_imm18_scale4);
+  case CSKY::S_ADDR:
+    return CSKY::fixup_csky_addr32;
+  case CSKY::S_ADDR_HI16:
+    return CSKY::fixup_csky_addr_hi16;
+  case CSKY::S_ADDR_LO16:
+    return CSKY::fixup_csky_addr_lo16;
+  case CSKY::S_GOT:
+    return CSKY::fixup_csky_got32;
+  case CSKY::S_GOTPC:
+    return CSKY::fixup_csky_gotpc;
+  case CSKY::S_GOTOFF:
+    return CSKY::fixup_csky_gotoff;
+  case CSKY::S_PLT:
+    return CSKY::fixup_csky_plt32;
+  case CSKY::S_PLT_IMM18_BY4:
+    return CSKY::fixup_csky_plt_imm18_scale4;
+  case CSKY::S_GOT_IMM18_BY4:
+    return CSKY::fixup_csky_got_imm18_scale4;
   }
 }
 
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h
deleted file mode 100644
index 0e47d259d43f5..0000000000000
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h
+++ /dev/null
@@ -1,188 +0,0 @@
-//===-- CSKYMCCodeEmitter.cpp - CSKY Code Emitter interface ---------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the CSKYMCCodeEmitter class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCCODEEMITTER_H
-#define LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCCODEEMITTER_H
-
-#include "MCTargetDesc/CSKYFixupKinds.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
-#include "llvm/MC/MCCodeEmitter.h"
-#include "llvm/MC/MCContext.h"
-
-namespace llvm {
-
-class MCInstrInfo;
-
-class CSKYMCCodeEmitter : public MCCodeEmitter {
-  MCContext &Ctx;
-  const MCInstrInfo &MII;
-
-public:
-  CSKYMCCodeEmitter(MCContext &Ctx, const MCInstrInfo &MII)
-      : Ctx(Ctx), MII(MII) {}
-
-  ~CSKYMCCodeEmitter() {}
-
-  void encodeInstruction(const MCInst &Inst, SmallVectorImpl<char> &CB,
-                         SmallVectorImpl<MCFixup> &Fixups,
-                         const MCSubtargetInfo &STI) const override;
-
-  // Generated by tablegen.
-  uint64_t getBinaryCodeForInstr(const MCInst &MI,
-                                 SmallVectorImpl<MCFixup> &Fixups,
-                                 const MCSubtargetInfo &STI) const;
-
-  // Default encoding method used by tablegen.
-  unsigned getMachineOpValue(const MCInst &MI, const MCOperand &MO,
-                             SmallVectorImpl<MCFixup> &Fixups,
-                             const MCSubtargetInfo &STI) const;
-
-  template <int shift = 0>
-  unsigned getImmOpValue(const MCInst &MI, unsigned Idx,
-                         SmallVectorImpl<MCFixup> &Fixups,
-                         const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    if (MO.isImm())
-      return (MO.getImm() >> shift);
-
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = getTargetFixup(MO.getExpr());
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  unsigned getRegSeqImmOpValue(const MCInst &MI, unsigned Idx,
-                               SmallVectorImpl<MCFixup> &Fixups,
-                               const MCSubtargetInfo &STI) const;
-
-  unsigned getRegisterSeqOpValue(const MCInst &MI, unsigned Op,
-                                 SmallVectorImpl<MCFixup> &Fixups,
-                                 const MCSubtargetInfo &STI) const;
-
-  unsigned getOImmOpValue(const MCInst &MI, unsigned Idx,
-                          SmallVectorImpl<MCFixup> &Fixups,
-                          const MCSubtargetInfo &STI) const;
-
-  unsigned getImmOpValueIDLY(const MCInst &MI, unsigned Idx,
-                             SmallVectorImpl<MCFixup> &Fixups,
-                             const MCSubtargetInfo &STI) const;
-
-  unsigned getImmJMPIX(const MCInst &MI, unsigned Idx,
-                       SmallVectorImpl<MCFixup> &Fixups,
-                       const MCSubtargetInfo &STI) const;
-
-  unsigned getImmOpValueMSBSize(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const;
-
-  unsigned getImmShiftOpValue(const MCInst &MI, unsigned Idx,
-                              SmallVectorImpl<MCFixup> &Fixups,
-                              const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isImm() && "Unexpected MO type.");
-    return 1 << MO.getImm();
-  }
-
-  MCFixupKind getTargetFixup(const MCExpr *Expr) const;
-
-  template <llvm::CSKY::Fixups FIXUP>
-  unsigned getBranchSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                  SmallVectorImpl<MCFixup> &Fixups,
-                                  const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-
-    if (MO.isImm())
-      return MO.getImm() >> 1;
-
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(FIXUP);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  template <llvm::CSKY::Fixups FIXUP>
-  unsigned getConstpoolSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                     SmallVectorImpl<MCFixup> &Fixups,
-                                     const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(FIXUP);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  template <llvm::CSKY::Fixups FIXUP>
-  unsigned getDataSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(FIXUP);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  unsigned getCallSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(CSKY::fixup_csky_pcrel_imm26_scale2);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  unsigned getBareSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(CSKY::fixup_csky_pcrel_imm18_scale2);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  void expandJBTF(const MCInst &MI, SmallVectorImpl<char> &CB,
-                  SmallVectorImpl<MCFixup> &Fixups,
-                  const MCSubtargetInfo &STI) const;
-  void expandNEG(const MCInst &MI, SmallVectorImpl<char> &CB,
-                 SmallVectorImpl<MCFixup> &Fixups,
-                 const MCSubtargetInfo &STI) const;
-  void expandRSUBI(const MCInst &MI, SmallVectorImpl<char> &CB,
-                   SmallVectorImpl<MCFixup> &Fixups,
-                   const MCSubtargetInfo &STI) const;
-};
-
-} // namespace llvm
-
-#endif // LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCCODEEMITTER_H
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp
deleted file mode 100644
index 696cd1daa035c..0000000000000
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- CSKYMCExpr.cpp - CSKY specific MC expression classes -*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "CSKYMCExpr.h"
-#include "CSKYFixupKinds.h"
-#include "llvm/BinaryFormat/ELF.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbolELF.h"
-#include "llvm/Support/Casting.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "csky-mc-expr"
-
-const CSKYMCExpr *CSKYMCExpr::create(const MCExpr *Expr, Specifier Kind,
-                                     MCContext &Ctx) {
-  return new (Ctx) CSKYMCExpr(Expr, Kind);
-}
-
-StringRef CSKYMCExpr::getVariantKindName(Specifier Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Invalid ELF symbol kind");
-  case VK_None:
-  case VK_ADDR:
-    return "";
-  case VK_ADDR_HI16:
-    return "@HI16";
-  case VK_ADDR_LO16:
-    return "@LO16";
-  case VK_GOT_IMM18_BY4:
-  case VK_GOT:
-    return "@GOT";
-  case VK_GOTPC:
-    return "@GOTPC";
-  case VK_GOTOFF:
-    return "@GOTOFF";
-  case VK_PLT_IMM18_BY4:
-  case VK_PLT:
-    return "@PLT";
-  case VK_TLSLE:
-    return "@TPOFF";
-  case VK_TLSIE:
-    return "@GOTTPOFF";
-  case VK_TLSGD:
-    return "@TLSGD32";
-  case VK_TLSLDO:
-    return "@TLSLDO32";
-  case VK_TLSLDM:
-    return "@TLSLDM32";
-  }
-}
-
-void CSKYMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-void CSKYMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  Expr->print(OS, MAI);
-  OS << getVariantKindName(getSpecifier());
-}
-
-bool CSKYMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                           const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return !Res.getSubSym();
-}
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h
deleted file mode 100644
index 14fe56f496a0b..0000000000000
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h
+++ /dev/null
@@ -1,75 +0,0 @@
-//===-- CSKYMCExpr.h - CSKY specific MC expression classes -*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCEXPR_H
-#define LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCValue.h"
-
-namespace llvm {
-
-class CSKYMCExpr : public MCTargetExpr {
-public:
-  enum Specifier : uint8_t {
-    VK_None,
-    VK_ADDR = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_ADDR_HI16,
-    VK_ADDR_LO16,
-    VK_PCREL,
-    VK_GOT,
-    VK_GOT_IMM18_BY4,
-    VK_GOTPC,
-    VK_GOTOFF,
-    VK_PLT,
-    VK_PLT_IMM18_BY4,
-    VK_TLSIE,
-    VK_TLSLE,
-    VK_TLSGD,
-    VK_TLSLDO,
-    VK_TLSLDM,
-    VK_TPOFF,
-    VK_Invalid
-  };
-
-private:
-  const MCExpr *Expr;
-  const Specifier specifier;
-
-  explicit CSKYMCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
-
-public:
-  static const CSKYMCExpr *create(const MCExpr *Expr, Specifier Kind,
-                                  MCContext &Ctx);
-
-  // Returns the kind of this expression.
-  Specifier getSpecifier() const { return specifier; }
-
-  // Returns the child of this expression.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  static StringRef getVariantKindName(Specifier Kind);
-};
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
index bb122f683644b..faaeeb8ed4dca 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
@@ -15,7 +15,6 @@
 #include "CSKYELFStreamer.h"
 #include "CSKYInstPrinter.h"
 #include "CSKYMCAsmInfo.h"
-#include "CSKYMCCodeEmitter.h"
 #include "CSKYTargetStreamer.h"
 #include "TargetInfo/CSKYTargetInfo.h"
 #include "llvm/MC/MCAssembler.h"
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp
index 42876b3891271..c2a389515b856 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYTargetStreamer.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -57,11 +58,11 @@ const MCExpr *CSKYConstantPool::addEntry(MCStreamer &Streamer,
   const auto SymRef = MCSymbolRefExpr::create(CPEntryLabel, Context);
 
   if (AdjustExpr) {
-    const CSKYMCExpr *CSKYExpr = cast<CSKYMCExpr>(Value);
+    auto *CSKYExpr = cast<MCSpecifierExpr>(Value);
 
     Value = MCBinaryExpr::createSub(AdjustExpr, SymRef, Context);
     Value = MCBinaryExpr::createSub(CSKYExpr->getSubExpr(), Value, Context);
-    Value = CSKYMCExpr::create(Value, CSKYExpr->getSpecifier(), Context);
+    Value = MCSpecifierExpr::create(Value, CSKYExpr->getSpecifier(), Context);
   }
 
   Entries.push_back(ConstantPoolEntry(CPEntryLabel, Value, Size, Loc));
@@ -84,12 +85,12 @@ CSKYTargetStreamer::CSKYTargetStreamer(MCStreamer &S)
 const MCExpr *
 CSKYTargetStreamer::addConstantPoolEntry(const MCExpr *Expr, SMLoc Loc,
                                          const MCExpr *AdjustExpr) {
-  auto ELFRefKind = CSKYMCExpr::VK_Invalid;
+  uint8_t ELFRefKind = CSKY::S_Invalid;
   ConstantCounter++;
 
   const MCExpr *OrigExpr = Expr;
 
-  if (const CSKYMCExpr *CE = dyn_cast<CSKYMCExpr>(Expr)) {
+  if (auto *CE = dyn_cast<MCSpecifierExpr>(Expr)) {
     Expr = CE->getSubExpr();
     ELFRefKind = CE->getSpecifier();
   }
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h
index 03d5a7b0e08cb..7b428502c34f5 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIB_TARGET_CSKY_CSKYTARGETSTREAMER_H
 #define LLVM_LIB_TARGET_CSKY_CSKYTARGETSTREAMER_H
 
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/MC/ConstantPools.h"
 #include "llvm/MC/MCStreamer.h"
 
@@ -46,7 +46,7 @@ class CSKYTargetStreamer : public MCTargetStreamer {
 public:
   typedef struct {
     const MCSymbol *sym;
-    CSKYMCExpr::Specifier kind;
+    CSKY::Specifier kind;
   } SymbolIndex;
 
 protected:
@@ -76,10 +76,10 @@ class CSKYTargetStreamer : public MCTargetStreamer {
 
 template <> struct DenseMapInfo<CSKYTargetStreamer::SymbolIndex> {
   static inline CSKYTargetStreamer::SymbolIndex getEmptyKey() {
-    return {nullptr, CSKYMCExpr::VK_Invalid};
+    return {nullptr, CSKY::S_Invalid};
   }
   static inline CSKYTargetStreamer::SymbolIndex getTombstoneKey() {
-    return {nullptr, CSKYMCExpr::VK_Invalid};
+    return {nullptr, CSKY::S_Invalid};
   }
   static unsigned getHashValue(const CSKYTargetStreamer::SymbolIndex &V) {
     return hash_combine(DenseMapInfo<const MCSymbol *>::getHashValue(V.sym),
diff --git a/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp b/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
index 7559f61b4cfb9..44277971acd60 100644
--- a/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
@@ -11,9 +11,11 @@
 #include "llvm/Frontend/HLSL/CBuffer.h"
 #include "llvm/Frontend/HLSL/HLSLResource.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsDirectX.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 #define DEBUG_TYPE "dxil-cbuffer-access"
@@ -54,71 +56,106 @@ struct CBufferRowIntrin {
     }
   }
 };
-} // namespace
 
-static size_t getOffsetForCBufferGEP(GEPOperator *GEP, GlobalVariable *Global,
-                                     const DataLayout &DL) {
-  // Since we should always have a constant offset, we should only ever have a
-  // single GEP of indirection from the Global.
-  assert(GEP->getPointerOperand() == Global &&
-         "Indirect access to resource handle");
+// Helper for creating CBuffer handles and loading data from them
+struct CBufferResource {
+  GlobalVariable *GVHandle;
+  GlobalVariable *Member;
+  size_t MemberOffset;
 
-  APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
-  bool Success = GEP->accumulateConstantOffset(DL, ConstantOffset);
-  (void)Success;
-  assert(Success && "Offsets into cbuffer globals must be constant");
+  LoadInst *Handle;
 
-  if (auto *ATy = dyn_cast<ArrayType>(Global->getValueType()))
-    ConstantOffset = hlsl::translateCBufArrayOffset(DL, ConstantOffset, ATy);
+  CBufferResource(GlobalVariable *GVHandle, GlobalVariable *Member,
+                  size_t MemberOffset)
+      : GVHandle(GVHandle), Member(Member), MemberOffset(MemberOffset) {}
 
-  return ConstantOffset.getZExtValue();
-}
+  const DataLayout &getDataLayout() { return GVHandle->getDataLayout(); }
+  Type *getValueType() { return Member->getValueType(); }
+  iterator_range<ConstantDataSequential::user_iterator> users() {
+    return Member->users();
+  }
+
+  /// Get the byte offset of a Pointer-typed Value * `Val` relative to Member.
+  /// `Val` can either be Member itself, or a GEP of a constant offset from
+  /// Member
+  size_t getOffsetForCBufferGEP(Value *Val) {
+    assert(isa<PointerType>(Val->getType()) &&
+           "Expected a pointer-typed value");
+
+    if (Val == Member)
+      return 0;
+
+    if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
+      // Since we should always have a constant offset, we should only ever have
+      // a single GEP of indirection from the Global.
+      assert(GEP->getPointerOperand() == Member &&
+             "Indirect access to resource handle");
+
+      const DataLayout &DL = getDataLayout();
+      APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
+      bool Success = GEP->accumulateConstantOffset(DL, ConstantOffset);
+      (void)Success;
+      assert(Success && "Offsets into cbuffer globals must be constant");
+
+      if (auto *ATy = dyn_cast<ArrayType>(Member->getValueType()))
+        ConstantOffset =
+            hlsl::translateCBufArrayOffset(DL, ConstantOffset, ATy);
+
+      return ConstantOffset.getZExtValue();
+    }
 
-/// Replace access via cbuffer global with a load from the cbuffer handle
-/// itself.
-static void replaceAccess(LoadInst *LI, GlobalVariable *Global,
-                          GlobalVariable *HandleGV, size_t BaseOffset,
-                          SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
-  const DataLayout &DL = HandleGV->getDataLayout();
+    llvm_unreachable("Expected Val to be a GlobalVariable or GEP");
+  }
 
-  size_t Offset = BaseOffset;
-  if (auto *GEP = dyn_cast<GEPOperator>(LI->getPointerOperand()))
-    Offset += getOffsetForCBufferGEP(GEP, Global, DL);
-  else if (LI->getPointerOperand() != Global)
-    llvm_unreachable("Load instruction doesn't reference cbuffer global");
+  /// Create a handle for this cbuffer resource using the IRBuilder `Builder`
+  /// and sets the handle as the current one to use for subsequent calls to
+  /// `loadValue`
+  void createAndSetCurrentHandle(IRBuilder<> &Builder) {
+    Handle = Builder.CreateLoad(GVHandle->getValueType(), GVHandle,
+                                GVHandle->getName());
+  }
 
-  IRBuilder<> Builder(LI);
-  auto *Handle = Builder.CreateLoad(HandleGV->getValueType(), HandleGV,
-                                    HandleGV->getName());
-
-  Type *Ty = LI->getType();
-  CBufferRowIntrin Intrin(DL, Ty->getScalarType());
-  // The cbuffer consists of some number of 16-byte rows.
-  unsigned int CurrentRow = Offset / hlsl::CBufferRowSizeInBytes;
-  unsigned int CurrentIndex =
-      (Offset % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
-
-  auto *CBufLoad = Builder.CreateIntrinsic(
-      Intrin.RetTy, Intrin.IID,
-      {Handle, ConstantInt::get(Builder.getInt32Ty(), CurrentRow)}, nullptr,
-      LI->getName());
-  auto *Elt =
-      Builder.CreateExtractValue(CBufLoad, {CurrentIndex++}, LI->getName());
-
-  Value *Result = nullptr;
-  unsigned int Remaining =
-      ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
-  if (Remaining == 0) {
-    // We only have a single element, so we're done.
-    Result = Elt;
-
-    // However, if we loaded a <1 x T>, then we need to adjust the type here.
-    if (auto *VT = dyn_cast<FixedVectorType>(LI->getType())) {
-      assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
-      Result = Builder.CreateInsertElement(PoisonValue::get(VT), Result,
-                                           Builder.getInt32(0));
+  /// Load a value of type `Ty` at offset `Offset` using the handle from the
+  /// last call to `createAndSetCurrentHandle`
+  Value *loadValue(IRBuilder<> &Builder, Type *Ty, size_t Offset,
+                   const Twine &Name = "") {
+    assert(Handle &&
+           "Expected a handle for this cbuffer global resource to be created "
+           "before loading a value from it");
+    const DataLayout &DL = getDataLayout();
+
+    size_t TargetOffset = MemberOffset + Offset;
+    CBufferRowIntrin Intrin(DL, Ty->getScalarType());
+    // The cbuffer consists of some number of 16-byte rows.
+    unsigned int CurrentRow = TargetOffset / hlsl::CBufferRowSizeInBytes;
+    unsigned int CurrentIndex =
+        (TargetOffset % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
+
+    auto *CBufLoad = Builder.CreateIntrinsic(
+        Intrin.RetTy, Intrin.IID,
+        {Handle, ConstantInt::get(Builder.getInt32Ty(), CurrentRow)}, nullptr,
+        Name + ".load");
+    auto *Elt = Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},
+                                           Name + ".extract");
+
+    Value *Result = nullptr;
+    unsigned int Remaining =
+        ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
+
+    if (Remaining == 0) {
+      // We only have a single element, so we're done.
+      Result = Elt;
+
+      // However, if we loaded a <1 x T>, then we need to adjust the type here.
+      if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
+        assert(VT->getNumElements() == 1 &&
+               "Can't have multiple elements here");
+        Result = Builder.CreateInsertElement(PoisonValue::get(VT), Result,
+                                             Builder.getInt32(0), Name);
+      }
+      return Result;
     }
-  } else {
+
     // Walk each element and extract it, wrapping to new rows as needed.
     SmallVector<Value *> Extracts{Elt};
     while (Remaining--) {
@@ -128,40 +165,138 @@ static void replaceAccess(LoadInst *LI, GlobalVariable *Global,
         CBufLoad = Builder.CreateIntrinsic(
             Intrin.RetTy, Intrin.IID,
             {Handle, ConstantInt::get(Builder.getInt32Ty(), ++CurrentRow)},
-            nullptr, LI->getName());
+            nullptr, Name + ".load");
 
       Extracts.push_back(Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},
-                                                    LI->getName()));
+                                                    Name + ".extract"));
     }
 
     // Finally, we build up the original loaded value.
     Result = PoisonValue::get(Ty);
     for (int I = 0, E = Extracts.size(); I < E; ++I)
       Result =
-          Builder.CreateInsertElement(Result, Extracts[I], Builder.getInt32(I));
+          Builder.CreateInsertElement(Result, Extracts[I], Builder.getInt32(I),
+                                      Name + formatv(".upto{}", I));
+    return Result;
   }
+};
 
+} // namespace
+
+/// Replace load via cbuffer global with a load from the cbuffer handle itself.
+static void replaceLoad(LoadInst *LI, CBufferResource &CBR,
+                        SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
+  size_t Offset = CBR.getOffsetForCBufferGEP(LI->getPointerOperand());
+  IRBuilder<> Builder(LI);
+  CBR.createAndSetCurrentHandle(Builder);
+  Value *Result = CBR.loadValue(Builder, LI->getType(), Offset, LI->getName());
   LI->replaceAllUsesWith(Result);
   DeadInsts.push_back(LI);
 }
 
-static void replaceAccessesWithHandle(GlobalVariable *Global,
-                                      GlobalVariable *HandleGV,
-                                      size_t BaseOffset) {
+/// This function recursively copies N array elements from the cbuffer resource
+/// CBR to the MemCpy Destination. Recursion is used to unravel multidimensional
+/// arrays into a sequence of scalar/vector extracts and stores.
+static void copyArrayElemsForMemCpy(IRBuilder<> &Builder, MemCpyInst *MCI,
+                                    CBufferResource &CBR, ArrayType *ArrTy,
+                                    size_t ArrOffset, size_t N,
+                                    const Twine &Name = "") {
+  const DataLayout &DL = MCI->getDataLayout();
+  Type *ElemTy = ArrTy->getElementType();
+  size_t ElemTySize = DL.getTypeAllocSize(ElemTy);
+  for (unsigned I = 0; I < N; ++I) {
+    size_t Offset = ArrOffset + I * ElemTySize;
+
+    // Recursively copy nested arrays
+    if (ArrayType *ElemArrTy = dyn_cast<ArrayType>(ElemTy)) {
+      copyArrayElemsForMemCpy(Builder, MCI, CBR, ElemArrTy, Offset,
+                              ElemArrTy->getNumElements(), Name);
+      continue;
+    }
+
+    // Load CBuffer value and store it in Dest
+    APInt CBufArrayOffset(
+        DL.getIndexTypeSizeInBits(MCI->getSource()->getType()), Offset);
+    CBufArrayOffset =
+        hlsl::translateCBufArrayOffset(DL, CBufArrayOffset, ArrTy);
+    Value *CBufferVal =
+        CBR.loadValue(Builder, ElemTy, CBufArrayOffset.getZExtValue(), Name);
+    Value *GEP =
+        Builder.CreateInBoundsGEP(Builder.getInt8Ty(), MCI->getDest(),
+                                  {Builder.getInt32(Offset)}, Name + ".dest");
+    Builder.CreateStore(CBufferVal, GEP, MCI->isVolatile());
+  }
+}
+
+/// Replace memcpy from a cbuffer global with a memcpy from the cbuffer handle
+/// itself. Assumes the cbuffer global is an array, and the length of bytes to
+/// copy is divisible by array element allocation size.
+/// The memcpy source must also be a direct cbuffer global reference, not a GEP.
+static void replaceMemCpy(MemCpyInst *MCI, CBufferResource &CBR) {
+
+  ArrayType *ArrTy = dyn_cast<ArrayType>(CBR.getValueType());
+  assert(ArrTy && "MemCpy lowering is only supported for array types");
+
+  // This assumption vastly simplifies the implementation
+  if (MCI->getSource() != CBR.Member)
+    reportFatalUsageError(
+        "Expected MemCpy source to be a cbuffer global variable");
+
+  ConstantInt *Length = dyn_cast<ConstantInt>(MCI->getLength());
+  uint64_t ByteLength = Length->getZExtValue();
+
+  // If length to copy is zero, no memcpy is needed
+  if (ByteLength == 0) {
+    MCI->eraseFromParent();
+    return;
+  }
+
+  const DataLayout &DL = CBR.getDataLayout();
+
+  Type *ElemTy = ArrTy->getElementType();
+  size_t ElemSize = DL.getTypeAllocSize(ElemTy);
+  assert(ByteLength % ElemSize == 0 &&
+         "Length of bytes to MemCpy must be divisible by allocation size of "
+         "source/destination array elements");
+  size_t ElemsToCpy = ByteLength / ElemSize;
+
+  IRBuilder<> Builder(MCI);
+  CBR.createAndSetCurrentHandle(Builder);
+
+  copyArrayElemsForMemCpy(Builder, MCI, CBR, ArrTy, 0, ElemsToCpy,
+                          "memcpy." + MCI->getDest()->getName() + "." +
+                              MCI->getSource()->getName());
+
+  MCI->eraseFromParent();
+}
+
+static void replaceAccessesWithHandle(CBufferResource &CBR) {
   SmallVector<WeakTrackingVH> DeadInsts;
 
-  SmallVector<User *> ToProcess{Global->users()};
+  SmallVector<User *> ToProcess{CBR.users()};
   while (!ToProcess.empty()) {
     User *Cur = ToProcess.pop_back_val();
 
     // If we have a load instruction, replace the access.
     if (auto *LI = dyn_cast<LoadInst>(Cur)) {
-      replaceAccess(LI, Global, HandleGV, BaseOffset, DeadInsts);
+      replaceLoad(LI, CBR, DeadInsts);
+      continue;
+    }
+
+    // If we have a memcpy instruction, replace it with multiple accesses and
+    // subsequent stores to the destination
+    if (auto *MCI = dyn_cast<MemCpyInst>(Cur)) {
+      replaceMemCpy(MCI, CBR);
       continue;
     }
 
     // Otherwise, walk users looking for a load...
-    ToProcess.append(Cur->user_begin(), Cur->user_end());
+    if (isa<GetElementPtrInst>(Cur) || isa<GEPOperator>(Cur)) {
+      ToProcess.append(Cur->user_begin(), Cur->user_end());
+      continue;
+    }
+
+    llvm_unreachable("Unexpected user of Global");
   }
   RecursivelyDeleteTriviallyDeadInstructions(DeadInsts);
 }
@@ -173,7 +308,8 @@ static bool replaceCBufferAccesses(Module &M) {
 
   for (const hlsl::CBufferMapping &Mapping : *CBufMD)
     for (const hlsl::CBufferMember &Member : Mapping.Members) {
-      replaceAccessesWithHandle(Member.GV, Mapping.Handle, Member.Offset);
+      CBufferResource CBR(Mapping.Handle, Member.GV, Member.Offset);
+      replaceAccessesWithHandle(CBR);
       Member.GV->removeFromParent();
     }
 
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
index 94b2dbe78c4f7..5f331dbd2d826 100644
--- a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
@@ -18,6 +18,16 @@
 using namespace llvm;
 
 static bool finalizeLinkage(Module &M) {
+  bool MadeChange = false;
+
+  // Convert private global variables to internal linkage.
+  for (GlobalVariable &GV : M.globals()) {
+    if (GV.hasPrivateLinkage()) {
+      GV.setLinkage(GlobalValue::InternalLinkage);
+      MadeChange = true;
+    }
+  }
+
   SmallVector<Function *> Funcs;
 
   // Collect non-entry and non-exported functions to set to internal linkage.
@@ -32,13 +42,17 @@ static bool finalizeLinkage(Module &M) {
   }
 
   for (Function *F : Funcs) {
-    if (F->getLinkage() == GlobalValue::ExternalLinkage)
+    if (F->getLinkage() == GlobalValue::ExternalLinkage) {
       F->setLinkage(GlobalValue::InternalLinkage);
-    if (F->isDefTriviallyDead())
+      MadeChange = true;
+    }
+    if (F->isDefTriviallyDead()) {
       M.getFunctionList().erase(F);
+      MadeChange = true;
+    }
   }
 
-  return false;
+  return MadeChange;
 }
 
 PreservedAnalyses DXILFinalizeLinkage::run(Module &M,
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index f99e8e7ccdc5d..ee1db54446cb8 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -25,6 +25,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 
@@ -41,6 +42,15 @@ class DXILIntrinsicExpansionLegacy : public ModulePass {
   static char ID; // Pass identification.
 };
 
+static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy,
+                                              bool IsRaw) {
+  if (IsRaw && M->getTargetTriple().getDXILVersion() > VersionTuple(1, 2))
+    return false;
+
+  Type *ScalarTy = OverloadTy->getScalarType();
+  return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(64);
+}
+
 static bool isIntrinsicExpansion(Function &F) {
   switch (F.getIntrinsicID()) {
   case Intrinsic::abs:
@@ -70,15 +80,20 @@ static bool isIntrinsicExpansion(Function &F) {
   case Intrinsic::vector_reduce_add:
   case Intrinsic::vector_reduce_fadd:
     return true;
+  case Intrinsic::dx_resource_load_rawbuffer:
+    return resourceAccessNeeds64BitExpansion(
+        F.getParent(), F.getReturnType()->getStructElementType(0),
+        /*IsRaw*/ true);
   case Intrinsic::dx_resource_load_typedbuffer:
-    // We need to handle doubles and vector of doubles.
-    return F.getReturnType()
-        ->getStructElementType(0)
-        ->getScalarType()
-        ->isDoubleTy();
+    return resourceAccessNeeds64BitExpansion(
+        F.getParent(), F.getReturnType()->getStructElementType(0),
+        /*IsRaw*/ false);
+  case Intrinsic::dx_resource_store_rawbuffer:
+    return resourceAccessNeeds64BitExpansion(
+        F.getParent(), F.getFunctionType()->getParamType(3), /*IsRaw*/ true);
   case Intrinsic::dx_resource_store_typedbuffer:
-    // We need to handle doubles and vector of doubles.
-    return F.getFunctionType()->getParamType(2)->getScalarType()->isDoubleTy();
+    return resourceAccessNeeds64BitExpansion(
+        F.getParent(), F.getFunctionType()->getParamType(2), /*IsRaw*/ false);
   }
   return false;
 }
@@ -541,51 +556,92 @@ static Value *expandRadiansIntrinsic(CallInst *Orig) {
   return Builder.CreateFMul(X, PiOver180);
 }
 
-static bool expandTypedBufferLoadIntrinsic(CallInst *Orig) {
+static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw) {
   IRBuilder<> Builder(Orig);
 
   Type *BufferTy = Orig->getType()->getStructElementType(0);
-  assert(BufferTy->getScalarType()->isDoubleTy() &&
-         "Only expand double or double2");
-
+  Type *ScalarTy = BufferTy->getScalarType();
+  bool IsDouble = ScalarTy->isDoubleTy();
+  assert(IsDouble || ScalarTy->isIntegerTy(64) &&
+                         "Only expand double or int64 scalars or vectors");
+  bool IsVector = false;
   unsigned ExtractNum = 2;
   if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {
-    assert(VT->getNumElements() == 2 &&
-           "TypedBufferLoad double vector has wrong size");
-    ExtractNum = 4;
+    ExtractNum = 2 * VT->getNumElements();
+    IsVector = true;
+    assert(IsRaw || ExtractNum == 4 && "TypedBufferLoad vector must be size 2");
   }
 
-  Type *Ty = VectorType::get(Builder.getInt32Ty(), ExtractNum, false);
-
-  Type *LoadType = StructType::get(Ty, Builder.getInt1Ty());
-  CallInst *Load =
-      Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_typedbuffer,
-                              {Orig->getOperand(0), Orig->getOperand(1)});
-
-  // extract the buffer load's result
-  Value *Extract = Builder.CreateExtractValue(Load, {0});
+  SmallVector<Value *, 2> Loads;
+  Value *Result = PoisonValue::get(BufferTy);
+  unsigned Base = 0;
+  // If we need to extract more than 4 i32; we need to break it up into
+  // more than one load. LoadNum tells us how many i32s we are loading in
+  // each load
+  while (ExtractNum > 0) {
+    unsigned LoadNum = std::min(ExtractNum, 4u);
+    Type *Ty = VectorType::get(Builder.getInt32Ty(), LoadNum, false);
+
+    Type *LoadType = StructType::get(Ty, Builder.getInt1Ty());
+    Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer;
+    SmallVector<Value *, 3> Args = {Orig->getOperand(0), Orig->getOperand(1)};
+    if (IsRaw) {
+      LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer;
+      Value *Tmp = Builder.getInt32(4 * Base * 2);
+      Args.push_back(Builder.CreateAdd(Orig->getOperand(2), Tmp));
+    }
 
-  SmallVector<Value *> ExtractElements;
-  for (unsigned I = 0; I < ExtractNum; ++I)
-    ExtractElements.push_back(
-        Builder.CreateExtractElement(Extract, Builder.getInt32(I)));
+    CallInst *Load = Builder.CreateIntrinsic(LoadType, LoadIntrinsic, Args);
+    Loads.push_back(Load);
+
+    // extract the buffer load's result
+    Value *Extract = Builder.CreateExtractValue(Load, {0});
+
+    SmallVector<Value *> ExtractElements;
+    for (unsigned I = 0; I < LoadNum; ++I)
+      ExtractElements.push_back(
+          Builder.CreateExtractElement(Extract, Builder.getInt32(I)));
+
+    // combine into double(s) or int64(s)
+    for (unsigned I = 0; I < LoadNum; I += 2) {
+      Value *Combined = nullptr;
+      if (IsDouble)
+        // For doubles, use dx_asdouble intrinsic
+        Combined = Builder.CreateIntrinsic(
+            Builder.getDoubleTy(), Intrinsic::dx_asdouble,
+            {ExtractElements[I], ExtractElements[I + 1]});
+      else {
+        // For int64, manually combine two int32s
+        // First, zero-extend both values to i64
+        Value *Lo =
+            Builder.CreateZExt(ExtractElements[I], Builder.getInt64Ty());
+        Value *Hi =
+            Builder.CreateZExt(ExtractElements[I + 1], Builder.getInt64Ty());
+        // Shift the high bits left by 32 bits
+        Value *ShiftedHi = Builder.CreateShl(Hi, Builder.getInt64(32));
+        // OR the high and low bits together
+        Combined = Builder.CreateOr(Lo, ShiftedHi);
+      }
+
+      if (IsVector)
+        Result = Builder.CreateInsertElement(Result, Combined,
+                                             Builder.getInt32((I / 2) + Base));
+      else
+        Result = Combined;
+    }
 
-  // combine into double(s)
-  Value *Result = PoisonValue::get(BufferTy);
-  for (unsigned I = 0; I < ExtractNum; I += 2) {
-    Value *Dbl =
-        Builder.CreateIntrinsic(Builder.getDoubleTy(), Intrinsic::dx_asdouble,
-                                {ExtractElements[I], ExtractElements[I + 1]});
-    if (ExtractNum == 4)
-      Result =
-          Builder.CreateInsertElement(Result, Dbl, Builder.getInt32(I / 2));
-    else
-      Result = Dbl;
+    ExtractNum -= LoadNum;
+    Base += LoadNum / 2;
   }
 
   Value *CheckBit = nullptr;
   for (User *U : make_early_inc_range(Orig->users())) {
-    auto *EVI = cast<ExtractValueInst>(U);
+    // If it's not a ExtractValueInst, we don't know how to
+    // handle it
+    auto *EVI = dyn_cast<ExtractValueInst>(U);
+    if (!EVI)
+      llvm_unreachable("Unexpected user of typedbufferload");
+
     ArrayRef<unsigned> Indices = EVI->getIndices();
     assert(Indices.size() == 1);
 
@@ -595,8 +651,14 @@ static bool expandTypedBufferLoadIntrinsic(CallInst *Orig) {
     } else {
       // Use of the check bit
       assert(Indices[0] == 1 && "Unexpected type for typedbufferload");
-      if (!CheckBit)
-        CheckBit = Builder.CreateExtractValue(Load, {1});
+      // Note: This does not always match the historical behaviour of DXC.
+      // See https://github.com/microsoft/DirectXShaderCompiler/issues/7622
+      if (!CheckBit) {
+        SmallVector<Value *, 2> CheckBits;
+        for (Value *L : Loads)
+          CheckBits.push_back(Builder.CreateExtractValue(L, {1}));
+        CheckBit = Builder.CreateAnd(CheckBits);
+      }
       EVI->replaceAllUsesWith(CheckBit);
     }
     EVI->eraseFromParent();
@@ -605,42 +667,102 @@ static bool expandTypedBufferLoadIntrinsic(CallInst *Orig) {
   return true;
 }
 
-static bool expandTypedBufferStoreIntrinsic(CallInst *Orig) {
+static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw) {
   IRBuilder<> Builder(Orig);
 
-  Type *BufferTy = Orig->getFunctionType()->getParamType(2);
-  assert(BufferTy->getScalarType()->isDoubleTy() &&
-         "Only expand double or double2");
+  unsigned ValIndex = IsRaw ? 3 : 2;
+  Type *BufferTy = Orig->getFunctionType()->getParamType(ValIndex);
+  Type *ScalarTy = BufferTy->getScalarType();
+  bool IsDouble = ScalarTy->isDoubleTy();
+  assert((IsDouble || ScalarTy->isIntegerTy(64)) &&
+         "Only expand double or int64 scalars or vectors");
 
+  // Determine if we're dealing with a vector or scalar
+  bool IsVector = false;
   unsigned ExtractNum = 2;
+  unsigned VecLen = 0;
   if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {
-    assert(VT->getNumElements() == 2 &&
-           "TypedBufferStore double vector has wrong size");
-    ExtractNum = 4;
+    VecLen = VT->getNumElements();
+    assert(IsRaw || VecLen == 2 && "TypedBufferStore vector must be size 2");
+    ExtractNum = VecLen * 2;
+    IsVector = true;
+  }
+
+  // Create the appropriate vector type for the result
+  Type *Int32Ty = Builder.getInt32Ty();
+  Type *ResultTy = VectorType::get(Int32Ty, ExtractNum, false);
+  Value *Val = PoisonValue::get(ResultTy);
+
+  Type *SplitElementTy = Int32Ty;
+  if (IsVector)
+    SplitElementTy = VectorType::get(SplitElementTy, VecLen, false);
+
+  Value *LowBits = nullptr;
+  Value *HighBits = nullptr;
+  // Split the 64-bit values into 32-bit components
+  if (IsDouble) {
+    auto *SplitTy = llvm::StructType::get(SplitElementTy, SplitElementTy);
+    Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,
+                                           {Orig->getOperand(ValIndex)});
+    LowBits = Builder.CreateExtractValue(Split, 0);
+    HighBits = Builder.CreateExtractValue(Split, 1);
+  } else {
+    // Handle int64 type(s)
+    Value *InputVal = Orig->getOperand(ValIndex);
+    Constant *ShiftAmt = Builder.getInt64(32);
+    if (IsVector)
+      ShiftAmt =
+          ConstantVector::getSplat(ElementCount::getFixed(VecLen), ShiftAmt);
+
+    // Split into low and high 32-bit parts
+    LowBits = Builder.CreateTrunc(InputVal, SplitElementTy);
+    Value *ShiftedVal = Builder.CreateLShr(InputVal, ShiftAmt);
+    HighBits = Builder.CreateTrunc(ShiftedVal, SplitElementTy);
   }
 
-  Type *SplitElementTy = Builder.getInt32Ty();
-  if (ExtractNum == 4)
-    SplitElementTy = VectorType::get(SplitElementTy, 2, false);
-
-  // split our double(s)
-  auto *SplitTy = llvm::StructType::get(SplitElementTy, SplitElementTy);
-  Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,
-                                         Orig->getOperand(2));
-  // create our vector
-  Value *LowBits = Builder.CreateExtractValue(Split, 0);
-  Value *HighBits = Builder.CreateExtractValue(Split, 1);
-  Value *Val;
-  if (ExtractNum == 2) {
-    Val = PoisonValue::get(VectorType::get(SplitElementTy, 2, false));
+  if (IsVector) {
+    SmallVector<int, 8> Mask;
+    for (unsigned I = 0; I < VecLen; ++I) {
+      Mask.push_back(I);
+      Mask.push_back(I + VecLen);
+    }
+    Val = Builder.CreateShuffleVector(LowBits, HighBits, Mask);
+  } else {
     Val = Builder.CreateInsertElement(Val, LowBits, Builder.getInt32(0));
     Val = Builder.CreateInsertElement(Val, HighBits, Builder.getInt32(1));
-  } else
-    Val = Builder.CreateShuffleVector(LowBits, HighBits, {0, 2, 1, 3});
+  }
+
+  // If we need to extract more than 4 i32; we need to break it up into
+  // more than one store. StoreNum tells us how many i32s we are storing in
+  // each store
+  unsigned Base = 0;
+  while (ExtractNum > 0) {
+    unsigned StoreNum = std::min(ExtractNum, 4u);
+
+    Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer;
+    SmallVector<Value *, 4> Args = {Orig->getOperand(0), Orig->getOperand(1)};
+    if (IsRaw) {
+      StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer;
+      Value *Tmp = Builder.getInt32(4 * Base);
+      Args.push_back(Builder.CreateAdd(Orig->getOperand(2), Tmp));
+    }
+
+    SmallVector<int, 4> Mask;
+    for (unsigned I = 0; I < StoreNum; ++I) {
+      Mask.push_back(Base + I);
+    }
 
-  Builder.CreateIntrinsic(Builder.getVoidTy(),
-                          Intrinsic::dx_resource_store_typedbuffer,
-                          {Orig->getOperand(0), Orig->getOperand(1), Val});
+    Value *SubVal = Val;
+    if (VecLen > 2)
+      SubVal = Builder.CreateShuffleVector(Val, Mask);
+
+    Args.push_back(SubVal);
+    // Create the final intrinsic call
+    Builder.CreateIntrinsic(Builder.getVoidTy(), StoreIntrinsic, Args);
+
+    ExtractNum -= StoreNum;
+    Base += StoreNum;
+  }
   Orig->eraseFromParent();
   return true;
 }
@@ -773,12 +895,20 @@ static bool expandIntrinsic(Function &F, CallInst *Orig) {
   case Intrinsic::dx_radians:
     Result = expandRadiansIntrinsic(Orig);
     break;
+  case Intrinsic::dx_resource_load_rawbuffer:
+    if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ true))
+      return true;
+    break;
+  case Intrinsic::dx_resource_store_rawbuffer:
+    if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ true))
+      return true;
+    break;
   case Intrinsic::dx_resource_load_typedbuffer:
-    if (expandTypedBufferLoadIntrinsic(Orig))
+    if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ false))
       return true;
     break;
   case Intrinsic::dx_resource_store_typedbuffer:
-    if (expandTypedBufferStoreIntrinsic(Orig))
+    if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ false))
       return true;
     break;
   case Intrinsic::usub_sat:
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 995120309a938..0ec15a629d0a2 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -39,11 +39,13 @@ class OpLowerer {
   DXILOpBuilder OpBuilder;
   DXILResourceMap &DRM;
   DXILResourceTypeMap &DRTM;
+  const ModuleMetadataInfo &MMDI;
   SmallVector<CallInst *> CleanupCasts;
 
 public:
-  OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM)
-      : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM) {}
+  OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM,
+            const ModuleMetadataInfo &MMDI)
+      : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM), MMDI(MMDI) {}
 
   /// Replace every call to \c F using \c ReplaceCall, and then erase \c F. If
   /// there is an error replacing a call, we emit a diagnostic and return true.
@@ -316,8 +318,7 @@ class OpLowerer {
   /// model and taking into account binding information from
   /// DXILResourceAnalysis.
   bool lowerHandleFromBinding(Function &F) {
-    const Triple &TT = M.getTargetTriple();
-    if (TT.getDXILVersion() < VersionTuple(1, 6))
+    if (MMDI.DXILVersion < VersionTuple(1, 6))
       return lowerToCreateHandle(F);
     return lowerToBindAndAnnotateHandle(F);
   }
@@ -486,8 +487,6 @@ class OpLowerer {
   }
 
   [[nodiscard]] bool lowerRawBufferLoad(Function &F) {
-    const Triple &TT = M.getTargetTriple();
-    VersionTuple DXILVersion = TT.getDXILVersion();
     const DataLayout &DL = F.getDataLayout();
     IRBuilder<> &IRB = OpBuilder.getIRB();
     Type *Int8Ty = IRB.getInt8Ty();
@@ -511,7 +510,7 @@ class OpLowerer {
           ConstantInt::get(Int32Ty, DL.getPrefTypeAlign(ScalarTy).value());
 
       Expected<CallInst *> OpCall =
-          DXILVersion >= VersionTuple(1, 2)
+          MMDI.DXILVersion >= VersionTuple(1, 2)
               ? OpBuilder.tryCreateOp(OpCode::RawBufferLoad,
                                       {Handle, Index0, Index1, Mask, Align},
                                       CI->getName(), NewRetTy)
@@ -586,8 +585,6 @@ class OpLowerer {
   }
 
   [[nodiscard]] bool lowerBufferStore(Function &F, bool IsRaw) {
-    const Triple &TT = M.getTargetTriple();
-    VersionTuple DXILVersion = TT.getDXILVersion();
     const DataLayout &DL = F.getDataLayout();
     IRBuilder<> &IRB = OpBuilder.getIRB();
     Type *Int8Ty = IRB.getInt8Ty();
@@ -654,7 +651,7 @@ class OpLowerer {
       SmallVector<Value *, 9> Args{
           Handle,          Index0,          Index1,          DataElements[0],
           DataElements[1], DataElements[2], DataElements[3], Mask};
-      if (IsRaw && DXILVersion >= VersionTuple(1, 2)) {
+      if (IsRaw && MMDI.DXILVersion >= VersionTuple(1, 2)) {
         Op = OpCode::RawBufferStore;
         // RawBufferStore requires the alignment
         Args.push_back(
@@ -745,6 +742,37 @@ class OpLowerer {
     });
   }
 
+  [[nodiscard]] bool lowerLifetimeIntrinsic(Function &F) {
+    IRBuilder<> &IRB = OpBuilder.getIRB();
+    return replaceFunction(F, [&](CallInst *CI) -> Error {
+      IRB.SetInsertPoint(CI);
+      Value *Ptr = CI->getArgOperand(1);
+      assert(Ptr->getType()->isPointerTy() &&
+             "Expected operand of lifetime intrinsic to be a pointer");
+
+      auto ZeroOrUndef = [&](Type *Ty) {
+        return MMDI.ValidatorVersion < VersionTuple(1, 6)
+                   ? Constant::getNullValue(Ty)
+                   : UndefValue::get(Ty);
+      };
+
+      Value *Val = nullptr;
+      if (auto *GV = dyn_cast<GlobalVariable>(Ptr)) {
+        if (GV->hasInitializer() || GV->isExternallyInitialized())
+          return Error::success();
+        Val = ZeroOrUndef(GV->getValueType());
+      } else if (auto *AI = dyn_cast<AllocaInst>(Ptr))
+        Val = ZeroOrUndef(AI->getAllocatedType());
+
+      assert(Val && "Expected operand of lifetime intrinsic to be a global "
+                    "variable or alloca instruction");
+      IRB.CreateStore(Val, Ptr, false);
+
+      CI->eraseFromParent();
+      return Error::success();
+    });
+  }
+
   [[nodiscard]] bool lowerIsFPClass(Function &F) {
     IRBuilder<> &IRB = OpBuilder.getIRB();
     Type *RetTy = IRB.getInt1Ty();
@@ -803,8 +831,6 @@ class OpLowerer {
       case Intrinsic::dx_resource_casthandle:
       // NOTE: llvm.dbg.value is supported as is in DXIL.
       case Intrinsic::dbg_value:
-      case Intrinsic::lifetime_start:
-      case Intrinsic::lifetime_end:
       case Intrinsic::not_intrinsic:
         if (F.use_empty())
           F.eraseFromParent();
@@ -855,6 +881,17 @@ class OpLowerer {
       case Intrinsic::ctpop:
         HasErrors |= lowerCtpopToCountBits(F);
         break;
+      case Intrinsic::lifetime_start:
+      case Intrinsic::lifetime_end:
+        if (F.use_empty())
+          F.eraseFromParent();
+        else {
+          if (MMDI.DXILVersion < VersionTuple(1, 6))
+            HasErrors |= lowerLifetimeIntrinsic(F);
+          else
+            continue;
+        }
+        break;
       case Intrinsic::is_fpclass:
         HasErrors |= lowerIsFPClass(F);
         break;
@@ -872,8 +909,9 @@ class OpLowerer {
 PreservedAnalyses DXILOpLowering::run(Module &M, ModuleAnalysisManager &MAM) {
   DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(M);
   DXILResourceTypeMap &DRTM = MAM.getResult<DXILResourceTypeAnalysis>(M);
+  const ModuleMetadataInfo MMDI = MAM.getResult<DXILMetadataAnalysis>(M);
 
-  bool MadeChanges = OpLowerer(M, DRM, DRTM).lowerIntrinsics();
+  const bool MadeChanges = OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
   if (!MadeChanges)
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
@@ -891,8 +929,10 @@ class DXILOpLoweringLegacy : public ModulePass {
         getAnalysis<DXILResourceWrapperPass>().getResourceMap();
     DXILResourceTypeMap &DRTM =
         getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
+    const ModuleMetadataInfo MMDI =
+        getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();
 
-    return OpLowerer(M, DRM, DRTM).lowerIntrinsics();
+    return OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();
   }
   StringRef getPassName() const override { return "DXIL Op Lowering"; }
   DXILOpLoweringLegacy() : ModulePass(ID) {}
@@ -901,6 +941,7 @@ class DXILOpLoweringLegacy : public ModulePass {
   void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
     AU.addRequired<DXILResourceTypeWrapperPass>();
     AU.addRequired<DXILResourceWrapperPass>();
+    AU.addRequired<DXILMetadataAnalysisWrapperPass>();
     AU.addPreserved<DXILResourceWrapperPass>();
     AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
     AU.addPreserved<ShaderFlagsAnalysisWrapper>();
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index 29e78fcce5262..e46b184a353f1 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/DXILMetadataAnalysis.h"
 #include "llvm/BinaryFormat/DXContainer.h"
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Function.h"
@@ -55,6 +56,13 @@ static std::optional<uint32_t> extractMdIntValue(MDNode *Node,
   return std::nullopt;
 }
 
+static std::optional<float> extractMdFloatValue(MDNode *Node,
+                                                unsigned int OpId) {
+  if (auto *CI = mdconst::dyn_extract<ConstantFP>(Node->getOperand(OpId).get()))
+    return CI->getValueAPF().convertToFloat();
+  return std::nullopt;
+}
+
 static std::optional<StringRef> extractMdStringValue(MDNode *Node,
                                                      unsigned int OpId) {
   MDString *NodeText = dyn_cast<MDString>(Node->getOperand(OpId));
@@ -261,6 +269,81 @@ static bool parseDescriptorTable(LLVMContext *Ctx,
   return false;
 }
 
+static bool parseStaticSampler(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD,
+                               MDNode *StaticSamplerNode) {
+  if (StaticSamplerNode->getNumOperands() != 14)
+    return reportError(Ctx, "Invalid format for Static Sampler");
+
+  dxbc::RTS0::v1::StaticSampler Sampler;
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 1))
+    Sampler.Filter = *Val;
+  else
+    return reportError(Ctx, "Invalid value for Filter");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 2))
+    Sampler.AddressU = *Val;
+  else
+    return reportError(Ctx, "Invalid value for AddressU");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 3))
+    Sampler.AddressV = *Val;
+  else
+    return reportError(Ctx, "Invalid value for AddressV");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 4))
+    Sampler.AddressW = *Val;
+  else
+    return reportError(Ctx, "Invalid value for AddressW");
+
+  if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 5))
+    Sampler.MipLODBias = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MipLODBias");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 6))
+    Sampler.MaxAnisotropy = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MaxAnisotropy");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 7))
+    Sampler.ComparisonFunc = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ComparisonFunc ");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 8))
+    Sampler.BorderColor = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ComparisonFunc ");
+
+  if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 9))
+    Sampler.MinLOD = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MinLOD");
+
+  if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 10))
+    Sampler.MaxLOD = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MaxLOD");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 11))
+    Sampler.ShaderRegister = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderRegister");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 12))
+    Sampler.RegisterSpace = *Val;
+  else
+    return reportError(Ctx, "Invalid value for RegisterSpace");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 13))
+    Sampler.ShaderVisibility = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderVisibility");
+
+  RSD.StaticSamplers.push_back(Sampler);
+  return false;
+}
+
 static bool parseRootSignatureElement(LLVMContext *Ctx,
                                       mcdxbc::RootSignatureDesc &RSD,
                                       MDNode *Element) {
@@ -276,6 +359,7 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
           .Case("RootSRV", RootSignatureElementKind::SRV)
           .Case("RootUAV", RootSignatureElementKind::UAV)
           .Case("DescriptorTable", RootSignatureElementKind::DescriptorTable)
+          .Case("StaticSampler", RootSignatureElementKind::StaticSamplers)
           .Default(RootSignatureElementKind::Error);
 
   switch (ElementKind) {
@@ -290,6 +374,8 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
     return parseRootDescriptors(Ctx, RSD, Element, ElementKind);
   case RootSignatureElementKind::DescriptorTable:
     return parseDescriptorTable(Ctx, RSD, Element);
+  case RootSignatureElementKind::StaticSamplers:
+    return parseStaticSampler(Ctx, RSD, Element);
   case RootSignatureElementKind::Error:
     return reportError(Ctx, "Invalid Root Signature Element: " + *ElementText);
   }
@@ -313,106 +399,13 @@ static bool parse(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD,
   return HasError;
 }
 
-static bool verifyRootFlag(uint32_t Flags) { return (Flags & ~0xfff) == 0; }
-
-static bool verifyVersion(uint32_t Version) {
-  return (Version == 1 || Version == 2);
-}
-
-static bool verifyRegisterValue(uint32_t RegisterValue) {
-  return RegisterValue != ~0U;
-}
-
-// This Range is reserverved, therefore invalid, according to the spec
-// https://github.com/llvm/wg-hlsl/blob/main/proposals/0002-root-signature-in-clang.md#all-the-values-should-be-legal
-static bool verifyRegisterSpace(uint32_t RegisterSpace) {
-  return !(RegisterSpace >= 0xFFFFFFF0 && RegisterSpace <= 0xFFFFFFFF);
-}
-
-static bool verifyDescriptorFlag(uint32_t Flags) { return (Flags & ~0xE) == 0; }
-
-static bool verifyRangeType(uint32_t Type) {
-  switch (Type) {
-  case llvm::to_underlying(dxbc::DescriptorRangeType::CBV):
-  case llvm::to_underlying(dxbc::DescriptorRangeType::SRV):
-  case llvm::to_underlying(dxbc::DescriptorRangeType::UAV):
-  case llvm::to_underlying(dxbc::DescriptorRangeType::Sampler):
-    return true;
-  };
-
-  return false;
-}
-
-static bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type,
-                                      uint32_t FlagsVal) {
-  using FlagT = dxbc::DescriptorRangeFlag;
-  FlagT Flags = FlagT(FlagsVal);
-
-  const bool IsSampler =
-      (Type == llvm::to_underlying(dxbc::DescriptorRangeType::Sampler));
-
-  if (Version == 1) {
-    // Since the metadata is unversioned, we expect to explicitly see the values
-    // that map to the version 1 behaviour here.
-    if (IsSampler)
-      return Flags == FlagT::DESCRIPTORS_VOLATILE;
-    return Flags == (FlagT::DATA_VOLATILE | FlagT::DESCRIPTORS_VOLATILE);
-  }
-
-  // The data-specific flags are mutually exclusive.
-  FlagT DataFlags = FlagT::DATA_VOLATILE | FlagT::DATA_STATIC |
-                    FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
-
-  if (popcount(llvm::to_underlying(Flags & DataFlags)) > 1)
-    return false;
-
-  // The descriptor-specific flags are mutually exclusive.
-  FlagT DescriptorFlags =
-      FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS |
-      FlagT::DESCRIPTORS_VOLATILE;
-  if (popcount(llvm::to_underlying(Flags & DescriptorFlags)) > 1)
-    return false;
-
-  // For volatile descriptors, DATA_STATIC is never valid.
-  if ((Flags & FlagT::DESCRIPTORS_VOLATILE) == FlagT::DESCRIPTORS_VOLATILE) {
-    FlagT Mask = FlagT::DESCRIPTORS_VOLATILE;
-    if (!IsSampler) {
-      Mask |= FlagT::DATA_VOLATILE;
-      Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
-    }
-    return (Flags & ~Mask) == FlagT::NONE;
-  }
-
-  // For "STATIC_KEEPING_BUFFER_BOUNDS_CHECKS" descriptors,
-  // the other data-specific flags may all be set.
-  if ((Flags & FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS) ==
-      FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS) {
-    FlagT Mask = FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS;
-    if (!IsSampler) {
-      Mask |= FlagT::DATA_VOLATILE;
-      Mask |= FlagT::DATA_STATIC;
-      Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
-    }
-    return (Flags & ~Mask) == FlagT::NONE;
-  }
-
-  // When no descriptor flag is set, any data flag is allowed.
-  FlagT Mask = FlagT::NONE;
-  if (!IsSampler) {
-    Mask |= FlagT::DATA_VOLATILE;
-    Mask |= FlagT::DATA_STATIC;
-    Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
-  }
-  return (Flags & ~Mask) == FlagT::NONE;
-}
-
 static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
 
-  if (!verifyVersion(RSD.Version)) {
+  if (!llvm::hlsl::rootsig::verifyVersion(RSD.Version)) {
     return reportValueError(Ctx, "Version", RSD.Version);
   }
 
-  if (!verifyRootFlag(RSD.Flags)) {
+  if (!llvm::hlsl::rootsig::verifyRootFlag(RSD.Flags)) {
     return reportValueError(Ctx, "RootFlags", RSD.Flags);
   }
 
@@ -431,15 +424,15 @@ static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
     case llvm::to_underlying(dxbc::RootParameterType::SRV): {
       const dxbc::RTS0::v2::RootDescriptor &Descriptor =
           RSD.ParametersContainer.getRootDescriptor(Info.Location);
-      if (!verifyRegisterValue(Descriptor.ShaderRegister))
+      if (!llvm::hlsl::rootsig::verifyRegisterValue(Descriptor.ShaderRegister))
         return reportValueError(Ctx, "ShaderRegister",
                                 Descriptor.ShaderRegister);
 
-      if (!verifyRegisterSpace(Descriptor.RegisterSpace))
+      if (!llvm::hlsl::rootsig::verifyRegisterSpace(Descriptor.RegisterSpace))
         return reportValueError(Ctx, "RegisterSpace", Descriptor.RegisterSpace);
 
       if (RSD.Version > 1) {
-        if (!verifyDescriptorFlag(Descriptor.Flags))
+        if (!llvm::hlsl::rootsig::verifyDescriptorFlag(Descriptor.Flags))
           return reportValueError(Ctx, "DescriptorRangeFlag", Descriptor.Flags);
       }
       break;
@@ -448,14 +441,14 @@ static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
       const mcdxbc::DescriptorTable &Table =
           RSD.ParametersContainer.getDescriptorTable(Info.Location);
       for (const dxbc::RTS0::v2::DescriptorRange &Range : Table) {
-        if (!verifyRangeType(Range.RangeType))
+        if (!llvm::hlsl::rootsig::verifyRangeType(Range.RangeType))
           return reportValueError(Ctx, "RangeType", Range.RangeType);
 
-        if (!verifyRegisterSpace(Range.RegisterSpace))
+        if (!llvm::hlsl::rootsig::verifyRegisterSpace(Range.RegisterSpace))
           return reportValueError(Ctx, "RegisterSpace", Range.RegisterSpace);
 
-        if (!verifyDescriptorRangeFlag(RSD.Version, Range.RangeType,
-                                       Range.Flags))
+        if (!llvm::hlsl::rootsig::verifyDescriptorRangeFlag(
+                RSD.Version, Range.RangeType, Range.Flags))
           return reportValueError(Ctx, "DescriptorFlag", Range.Flags);
       }
       break;
@@ -463,6 +456,48 @@ static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
     }
   }
 
+  for (const dxbc::RTS0::v1::StaticSampler &Sampler : RSD.StaticSamplers) {
+    if (!llvm::hlsl::rootsig::verifySamplerFilter(Sampler.Filter))
+      return reportValueError(Ctx, "Filter", Sampler.Filter);
+
+    if (!llvm::hlsl::rootsig::verifyAddress(Sampler.AddressU))
+      return reportValueError(Ctx, "AddressU", Sampler.AddressU);
+
+    if (!llvm::hlsl::rootsig::verifyAddress(Sampler.AddressV))
+      return reportValueError(Ctx, "AddressV", Sampler.AddressV);
+
+    if (!llvm::hlsl::rootsig::verifyAddress(Sampler.AddressW))
+      return reportValueError(Ctx, "AddressW", Sampler.AddressW);
+
+    if (!llvm::hlsl::rootsig::verifyMipLODBias(Sampler.MipLODBias))
+      return reportValueError(Ctx, "MipLODBias", Sampler.MipLODBias);
+
+    if (!llvm::hlsl::rootsig::verifyMaxAnisotropy(Sampler.MaxAnisotropy))
+      return reportValueError(Ctx, "MaxAnisotropy", Sampler.MaxAnisotropy);
+
+    if (!llvm::hlsl::rootsig::verifyComparisonFunc(Sampler.ComparisonFunc))
+      return reportValueError(Ctx, "ComparisonFunc", Sampler.ComparisonFunc);
+
+    if (!llvm::hlsl::rootsig::verifyBorderColor(Sampler.BorderColor))
+      return reportValueError(Ctx, "BorderColor", Sampler.BorderColor);
+
+    if (!llvm::hlsl::rootsig::verifyLOD(Sampler.MinLOD))
+      return reportValueError(Ctx, "MinLOD", Sampler.MinLOD);
+
+    if (!llvm::hlsl::rootsig::verifyLOD(Sampler.MaxLOD))
+      return reportValueError(Ctx, "MaxLOD", Sampler.MaxLOD);
+
+    if (!llvm::hlsl::rootsig::verifyRegisterValue(Sampler.ShaderRegister))
+      return reportValueError(Ctx, "ShaderRegister", Sampler.ShaderRegister);
+
+    if (!llvm::hlsl::rootsig::verifyRegisterSpace(Sampler.RegisterSpace))
+      return reportValueError(Ctx, "RegisterSpace", Sampler.RegisterSpace);
+
+    if (!dxbc::isValidShaderVisibility(Sampler.ShaderVisibility))
+      return reportValueError(Ctx, "ShaderVisibility",
+                              Sampler.ShaderVisibility);
+  }
+
   return false;
 }
 
@@ -542,6 +577,9 @@ analyzeModule(Module &M) {
     // offset will always equal to the header size.
     RSD.RootParameterOffset = sizeof(dxbc::RTS0::v1::RootSignatureHeader);
 
+    // static sampler offset is calculated when writting dxcontainer.
+    RSD.StaticSamplersOffset = 0u;
+
     if (parse(Ctx, RSD, RootElementListNode) || validate(Ctx, RSD)) {
       return RSDMap;
     }
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.h b/llvm/lib/Target/DirectX/DXILRootSignature.h
index b45cebc15fd39..be5cc78bc6bdf 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.h
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.h
@@ -32,6 +32,7 @@ enum class RootSignatureElementKind {
   UAV = 4,
   CBV = 5,
   DescriptorTable = 6,
+  StaticSamplers = 7
 };
 class RootSignatureAnalysis : public AnalysisInfoMixin<RootSignatureAnalysis> {
   friend AnalysisInfoMixin<RootSignatureAnalysis>;
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index c423dca90a4ab..b94b1484205ae 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -423,7 +423,7 @@ struct HexagonOperand : public MCParsedAsmOperand {
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<HexagonOperand> CreateToken(MCContext &Context,
                                                      StringRef Str, SMLoc S) {
@@ -456,7 +456,7 @@ struct HexagonOperand : public MCParsedAsmOperand {
 
 } // end anonymous namespace
 
-void HexagonOperand::print(raw_ostream &OS) const {
+void HexagonOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case Immediate:
     HexagonMCAsmInfo(Triple()).printExpr(OS, *getImm());
diff --git a/llvm/lib/Target/Hexagon/HexagonBlockRanges.h b/llvm/lib/Target/Hexagon/HexagonBlockRanges.h
index 5a3b6433fba78..e152dbf764c97 100644
--- a/llvm/lib/Target/Hexagon/HexagonBlockRanges.h
+++ b/llvm/lib/Target/Hexagon/HexagonBlockRanges.h
@@ -37,7 +37,7 @@ struct HexagonBlockRanges {
     unsigned Sub;
 
     bool operator<(RegisterRef R) const {
-      return Reg < R.Reg || (Reg == R.Reg && Sub < R.Sub);
+      return std::tie(Reg, Sub) < std::tie(R.Reg, R.Sub);
     }
   };
   using RegisterSet = std::set<RegisterRef>;
diff --git a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
index 740479174c814..bb7814c5226fd 100644
--- a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
@@ -105,8 +105,7 @@ namespace {
     };
 
     LatticeCell() : Kind(Top), Size(0), IsSpecial(false) {
-      for (const Constant *&Value : Values)
-        Value = nullptr;
+      llvm::fill(Values, nullptr);
     }
 
     bool meet(const LatticeCell &L);
diff --git a/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp b/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
index fcd0429eae8e9..1322c627ecfb7 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
@@ -44,9 +44,6 @@ static cl::opt<unsigned> ExtractCutoff("extract-cutoff", cl::init(~0U),
 static cl::opt<bool> NoSR0("extract-nosr0", cl::init(true), cl::Hidden,
   cl::desc("No extract instruction with offset 0"));
 
-static cl::opt<bool> NeedAnd("extract-needand", cl::init(true), cl::Hidden,
-  cl::desc("Require & in extract patterns"));
-
 namespace {
 
   class HexagonGenExtract : public FunctionPass {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 6534ae938fede..5243f3bb1bf3c 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -102,6 +102,10 @@ static cl::opt<int>
     MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", cl::Hidden, cl::init(4),
                                 cl::desc("Max #stores to inline memset"));
 
+static cl::opt<bool>
+    ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(true),
+                       cl::desc("Convert constant loads to immediate values."));
+
 static cl::opt<bool> AlignLoads("hexagon-align-loads",
   cl::Hidden, cl::init(false),
   cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
@@ -153,6 +157,81 @@ static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
 
 #include "HexagonGenCallingConv.inc"
 
+unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(
+    LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
+    unsigned &NumIntermediates, MVT &RegisterVT) const {
+
+  bool isBoolVector = VT.getVectorElementType() == MVT::i1;
+  bool isPowerOf2 = VT.isPow2VectorType();
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // Split vectors of type vXi1 into (X/8) vectors of type v8i1,
+  // where X is divisible by 8.
+  if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {
+    RegisterVT = MVT::v8i8;
+    IntermediateVT = MVT::v8i1;
+    NumIntermediates = NumElts / 8;
+    return NumIntermediates;
+  }
+
+  // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors
+  // of type v64i1, provided that X is divisible by 64.
+  if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {
+    RegisterVT = MVT::v64i8;
+    IntermediateVT = MVT::v64i1;
+    NumIntermediates = NumElts / 64;
+    return NumIntermediates;
+  }
+
+  // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors
+  // of type v128i1, provided that X is divisible by 128.
+  if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&
+      NumElts >= 128) {
+    RegisterVT = MVT::v128i8;
+    IntermediateVT = MVT::v128i1;
+    NumIntermediates = NumElts / 128;
+    return NumIntermediates;
+  }
+
+  return TargetLowering::getVectorTypeBreakdownForCallingConv(
+      Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
+}
+
+std::pair<MVT, unsigned>
+HexagonTargetLowering::handleMaskRegisterForCallingConv(
+    const HexagonSubtarget &Subtarget, EVT VT) const {
+  assert(VT.getVectorElementType() == MVT::i1);
+
+  const unsigned NumElems = VT.getVectorNumElements();
+
+  if (!VT.isPow2VectorType())
+    return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
+
+  if (!Subtarget.useHVXOps() && NumElems >= 8)
+    return {MVT::v8i8, NumElems / 8};
+
+  if (Subtarget.useHVX64BOps() && NumElems >= 64)
+    return {MVT::v64i8, NumElems / 64};
+
+  if (Subtarget.useHVX128BOps() && NumElems >= 128)
+    return {MVT::v128i8, NumElems / 128};
+
+  return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
+}
+
+MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
+                                                         CallingConv::ID CC,
+                                                         EVT VT) const {
+
+  if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
+    auto [RegisterVT, NumRegisters] =
+        handleMaskRegisterForCallingConv(Subtarget, VT);
+    if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
+      return RegisterVT;
+  }
+
+  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
+}
 
 SDValue
 HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
@@ -1179,9 +1258,6 @@ HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
@@ -3607,6 +3683,18 @@ bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
   return true;
 }
 
+/// Returns true if it is beneficial to convert a load of a constant
+/// to just the constant itself.
+bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
+                                                              Type *Ty) const {
+  if (!ConstantLoadsToImm)
+    return false;
+
+  assert(Ty->isIntegerTy());
+  unsigned BitSize = Ty->getPrimitiveSizeInBits();
+  return (BitSize > 0 && BitSize <= 64);
+}
+
 /// isLegalAddressingMode - Return true if the addressing mode represented by
 /// AM is legal for this target, for a load/store of the specified type.
 bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 1321bee44a295..5e5f3ab78dcd7 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -183,6 +183,9 @@ class HexagonTargetLowering : public TargetLowering {
                           SelectionDAG &DAG) const override;
 
   const char *getTargetNodeName(unsigned Opcode) const override;
+  std::pair<MVT, unsigned>
+  handleMaskRegisterForCallingConv(const HexagonSubtarget &Subtarget,
+                                   EVT VT) const;
 
   SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
@@ -263,6 +266,14 @@ class HexagonTargetLowering : public TargetLowering {
   Register getRegisterByName(const char* RegName, LLT VT,
                              const MachineFunction &MF) const override;
 
+  unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context,
+                                                CallingConv::ID CC, EVT VT,
+                                                EVT &IntermediateVT,
+                                                unsigned &NumIntermediates,
+                                                MVT &RegisterVT) const override;
+
+  MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
+                                    EVT VT) const override;
   /// If a physical register, this returns the register that receives the
   /// exception address on entry to an EH pad.
   Register
@@ -342,6 +353,11 @@ class HexagonTargetLowering : public TargetLowering {
   SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG)
                                    const override;
 
+  /// Returns true if it is beneficial to convert a load of a constant
+  /// to just the constant itself.
+  bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
+                                         Type *Ty) const override;
+
   bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
                              std::optional<unsigned> ByteOffset) const override;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index fbbcacf0d713e..0e13dd3214da6 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -1687,28 +1687,27 @@ HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
     // all operations are expected to be type-legalized, and i16 is not
     // a legal type. If any of the extracted elements is not of a valid
     // type, sign-extend it to a valid one.
-    for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
-      SDValue V = Elems[i];
+    for (SDValue &V : Elems) {
       MVT Ty = ty(V);
       if (!isTypeLegal(Ty)) {
         MVT NTy = typeLegalize(Ty, DAG);
         if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
-          Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
-                                 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
-                                             V.getOperand(0), V.getOperand(1)),
-                                 DAG.getValueType(Ty));
+          V = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
+                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
+                                      V.getOperand(0), V.getOperand(1)),
+                          DAG.getValueType(Ty));
           continue;
         }
         // A few less complicated cases.
         switch (V.getOpcode()) {
           case ISD::Constant:
-            Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
+            V = DAG.getSExtOrTrunc(V, dl, NTy);
             break;
           case ISD::UNDEF:
-            Elems[i] = DAG.getUNDEF(NTy);
+            V = DAG.getUNDEF(NTy);
             break;
           case ISD::TRUNCATE:
-            Elems[i] = V.getOperand(0);
+            V = V.getOperand(0);
             break;
           default:
             llvm_unreachable("Unexpected vector element");
diff --git a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
index 6ea3f96cffd2c..6dd83c1d820f4 100644
--- a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
@@ -448,9 +448,7 @@ bool HexagonOptAddrMode::usedInLoadStore(NodeAddr<StmtNode *> CurrentInstSN,
 
   getAllRealUses(CurrentInstSN, LoadStoreUseList);
   bool FoundLoadStoreUse = false;
-  for (auto I = LoadStoreUseList.begin(), E = LoadStoreUseList.end(); I != E;
-       ++I) {
-    NodeAddr<UseNode *> UN = *I;
+  for (NodeAddr<UseNode *> UN : LoadStoreUseList) {
     NodeAddr<StmtNode *> SN = UN.Addr->getOwner(*DFG);
     MachineInstr *LoadStoreMI = SN.Addr->getCode();
     const MCInstrDesc &MID = LoadStoreMI->getDesc();
@@ -579,8 +577,7 @@ bool HexagonOptAddrMode::processAddBases(NodeAddr<StmtNode *> AddSN,
   // Find all Addi instructions that share the same base register and add them
   // to the AddiList
   getAllRealUses(ReachingDefStmt, AddiUseList);
-  for (auto I = AddiUseList.begin(), E = AddiUseList.end(); I != E; ++I) {
-    NodeAddr<UseNode *> UN = *I;
+  for (NodeAddr<UseNode *> UN : AddiUseList) {
     NodeAddr<StmtNode *> SN = UN.Addr->getOwner(*DFG);
     MachineInstr *MI = SN.Addr->getCode();
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 45bb82c945a7a..de7bd5d4b2c66 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -41,7 +40,7 @@ class HexagonAsmBackend : public MCAsmBackend {
   uint8_t OSABI;
   StringRef CPU;
   mutable uint64_t relaxedCnt;
-  mutable const MCInst *RelaxedMCB = nullptr;
+  mutable MCInst RelaxedMCB;
   std::unique_ptr <MCInstrInfo> MCII;
   std::unique_ptr <MCInst *> RelaxTarget;
   MCInst * Extender;
@@ -84,14 +83,15 @@ class HexagonAsmBackend : public MCAsmBackend {
   }
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
+    // clang-format off
     const static MCFixupKindInfo Infos[Hexagon::NumTargetFixupKinds] = {
       // This table *must* be in same the order of fixup_* kinds in
       // HexagonFixupKinds.h.
       //
       // namei                          offset  bits    flags
-      { "fixup_Hexagon_B22_PCREL",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B15_PCREL",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B7_PCREL",       0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_B22_PCREL",      0,      32,     0 },
+      { "fixup_Hexagon_B15_PCREL",      0,      32,     0 },
+      { "fixup_Hexagon_B7_PCREL",       0,      32,     0 },
       { "fixup_Hexagon_LO16",           0,      32,     0 },
       { "fixup_Hexagon_HI16",           0,      32,     0 },
       { "fixup_Hexagon_32",             0,      32,     0 },
@@ -102,15 +102,15 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_GPREL16_2",      0,      32,     0 },
       { "fixup_Hexagon_GPREL16_3",      0,      32,     0 },
       { "fixup_Hexagon_HL16",           0,      32,     0 },
-      { "fixup_Hexagon_B13_PCREL",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B9_PCREL",       0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B32_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_B13_PCREL",      0,      32,     0 },
+      { "fixup_Hexagon_B9_PCREL",       0,      32,     0 },
+      { "fixup_Hexagon_B32_PCREL_X",    0,      32,     0 },
       { "fixup_Hexagon_32_6_X",         0,      32,     0 },
-      { "fixup_Hexagon_B22_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B15_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B13_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B9_PCREL_X",     0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B7_PCREL_X",     0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_B22_PCREL_X",    0,      32,     0 },
+      { "fixup_Hexagon_B15_PCREL_X",    0,      32,     0 },
+      { "fixup_Hexagon_B13_PCREL_X",    0,      32,     0 },
+      { "fixup_Hexagon_B9_PCREL_X",     0,      32,     0 },
+      { "fixup_Hexagon_B7_PCREL_X",     0,      32,     0 },
       { "fixup_Hexagon_16_X",           0,      32,     0 },
       { "fixup_Hexagon_12_X",           0,      32,     0 },
       { "fixup_Hexagon_11_X",           0,      32,     0 },
@@ -119,12 +119,12 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_8_X",            0,      32,     0 },
       { "fixup_Hexagon_7_X",            0,      32,     0 },
       { "fixup_Hexagon_6_X",            0,      32,     0 },
-      { "fixup_Hexagon_32_PCREL",       0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_32_PCREL",       0,      32,     0 },
       { "fixup_Hexagon_COPY",           0,      32,     0 },
       { "fixup_Hexagon_GLOB_DAT",       0,      32,     0 },
       { "fixup_Hexagon_JMP_SLOT",       0,      32,     0 },
       { "fixup_Hexagon_RELATIVE",       0,      32,     0 },
-      { "fixup_Hexagon_PLT_B22_PCREL",  0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_PLT_B22_PCREL",  0,      32,     0 },
       { "fixup_Hexagon_GOTREL_LO16",    0,      32,     0 },
       { "fixup_Hexagon_GOTREL_HI16",    0,      32,     0 },
       { "fixup_Hexagon_GOTREL_32",      0,      32,     0 },
@@ -137,8 +137,8 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_DTPREL_HI16",    0,      32,     0 },
       { "fixup_Hexagon_DTPREL_32",      0,      32,     0 },
       { "fixup_Hexagon_DTPREL_16",      0,      32,     0 },
-      { "fixup_Hexagon_GD_PLT_B22_PCREL",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_LD_PLT_B22_PCREL",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_GD_PLT_B22_PCREL",0,     32,     0 },
+      { "fixup_Hexagon_LD_PLT_B22_PCREL",0,     32,     0 },
       { "fixup_Hexagon_GD_GOT_LO16",    0,      32,     0 },
       { "fixup_Hexagon_GD_GOT_HI16",    0,      32,     0 },
       { "fixup_Hexagon_GD_GOT_32",      0,      32,     0 },
@@ -159,7 +159,7 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_TPREL_HI16",     0,      32,     0 },
       { "fixup_Hexagon_TPREL_32",       0,      32,     0 },
       { "fixup_Hexagon_TPREL_16",       0,      32,     0 },
-      { "fixup_Hexagon_6_PCREL_X",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_6_PCREL_X",      0,      32,     0 },
       { "fixup_Hexagon_GOTREL_32_6_X",  0,      32,     0 },
       { "fixup_Hexagon_GOTREL_16_X",    0,      32,     0 },
       { "fixup_Hexagon_GOTREL_11_X",    0,      32,     0 },
@@ -183,11 +183,12 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_TPREL_32_6_X",   0,      32,     0 },
       { "fixup_Hexagon_TPREL_16_X",     0,      32,     0 },
       { "fixup_Hexagon_TPREL_11_X",     0,      32,     0 },
-      { "fixup_Hexagon_GD_PLT_B22_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_GD_PLT_B32_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_LD_PLT_B22_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_LD_PLT_B32_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel }
+      { "fixup_Hexagon_GD_PLT_B22_PCREL_X", 0,  32,     0 },
+      { "fixup_Hexagon_GD_PLT_B32_PCREL_X", 0,  32,     0 },
+      { "fixup_Hexagon_LD_PLT_B22_PCREL_X", 0,  32,     0 },
+      { "fixup_Hexagon_LD_PLT_B32_PCREL_X", 0,  32,     0 },
     };
+    // clang-format on
 
     if (Kind < FirstTargetFixupKind)
       return MCAsmBackend::getFixupKindInfo(Kind);
@@ -198,8 +199,7 @@ class HexagonAsmBackend : public MCAsmBackend {
     return Infos[Kind - FirstTargetFixupKind];
   }
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override {
+  bool shouldForceRelocation(const MCFixup &Fixup) {
     switch(Fixup.getTargetKind()) {
       default:
         llvm_unreachable("Unknown Fixup Kind!");
@@ -316,7 +316,6 @@ class HexagonAsmBackend : public MCAsmBackend {
       case FK_Data_1:
       case FK_Data_2:
       case FK_Data_4:
-      case FK_PCRel_4:
       case fixup_Hexagon_32:
         // Leave these relocations alone as they are used for EH.
         return false;
@@ -334,8 +333,7 @@ class HexagonAsmBackend : public MCAsmBackend {
         return 1;
       case FK_Data_2:
         return 2;
-      case FK_Data_4:         // this later gets mapped to R_HEX_32
-      case FK_PCRel_4:        // this later gets mapped to R_HEX_32_PCREL
+      case FK_Data_4: // this later gets mapped to R_HEX_32 or R_HEX_32_PCREL
       case fixup_Hexagon_32:
       case fixup_Hexagon_B32_PCREL_X:
       case fixup_Hexagon_B22_PCREL:
@@ -404,131 +402,9 @@ class HexagonAsmBackend : public MCAsmBackend {
     llvm_unreachable(errStr.str().c_str());
   }
 
-  /// ApplyFixup - Apply the \arg Value for given \arg Fixup into the provided
-  /// data fragment, at the offset specified by the fixup and following the
-  /// fixup kind as appropriate.
-  void applyFixup(const MCFragment &, const MCFixup &Fixup,
-                  const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t FixupValue, bool IsResolved) override {
-
-    // When FixupValue is 0 the relocation is external and there
-    // is nothing for us to do.
-    if (!FixupValue) return;
-
-    MCFixupKind Kind = Fixup.getKind();
-    uint64_t Value;
-    uint32_t InstMask;
-    uint32_t Reloc;
-
-    // LLVM gives us an encoded value, we have to convert it back
-    // to a real offset before we can use it.
-    uint32_t Offset = Fixup.getOffset();
-    unsigned NumBytes = getFixupKindNumBytes(Kind);
-    assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
-    char *InstAddr = Data.data() + Offset;
-
-    Value = adjustFixupValue(Kind, FixupValue);
-    if(!Value)
-      return;
-    int sValue = (int)Value;
-
-    switch((unsigned)Kind) {
-      default:
-        return;
-
-      case fixup_Hexagon_B7_PCREL:
-        if (!(isIntN(7, sValue)))
-          HandleFixupError(7, 2, (int64_t)FixupValue, "B7_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B7_PCREL_X:
-        InstMask = 0x00001f18;  // Word32_B7
-        Reloc = (((Value >> 2) & 0x1f) << 8) |    // Value 6-2 = Target 12-8
-                ((Value & 0x3) << 3);             // Value 1-0 = Target 4-3
-        break;
-
-      case fixup_Hexagon_B9_PCREL:
-        if (!(isIntN(9, sValue)))
-          HandleFixupError(9, 2, (int64_t)FixupValue, "B9_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B9_PCREL_X:
-        InstMask = 0x003000fe;  // Word32_B9
-        Reloc = (((Value >> 7) & 0x3) << 20) |    // Value 8-7 = Target 21-20
-                ((Value & 0x7f) << 1);            // Value 6-0 = Target 7-1
-        break;
-
-        // Since the existing branches that use this relocation cannot be
-        // extended, they should only be fixed up if the target is within range.
-      case fixup_Hexagon_B13_PCREL:
-        if (!(isIntN(13, sValue)))
-          HandleFixupError(13, 2, (int64_t)FixupValue, "B13_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B13_PCREL_X:
-        InstMask = 0x00202ffe;  // Word32_B13
-        Reloc = (((Value >> 12) & 0x1) << 21) |    // Value 12   = Target 21
-                (((Value >> 11) & 0x1) << 13) |    // Value 11   = Target 13
-                ((Value & 0x7ff) << 1);            // Value 10-0 = Target 11-1
-        break;
-
-      case fixup_Hexagon_B15_PCREL:
-        if (!(isIntN(15, sValue)))
-          HandleFixupError(15, 2, (int64_t)FixupValue, "B15_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B15_PCREL_X:
-        InstMask = 0x00df20fe;  // Word32_B15
-        Reloc = (((Value >> 13) & 0x3) << 22) |    // Value 14-13 = Target 23-22
-                (((Value >> 8) & 0x1f) << 16) |    // Value 12-8  = Target 20-16
-                (((Value >> 7) & 0x1)  << 13) |    // Value 7     = Target 13
-                ((Value & 0x7f) << 1);             // Value 6-0   = Target 7-1
-        break;
-
-      case fixup_Hexagon_B22_PCREL:
-        if (!(isIntN(22, sValue)))
-          HandleFixupError(22, 2, (int64_t)FixupValue, "B22_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B22_PCREL_X:
-        InstMask = 0x01ff3ffe;  // Word32_B22
-        Reloc = (((Value >> 13) & 0x1ff) << 16) |  // Value 21-13 = Target 24-16
-                ((Value & 0x1fff) << 1);           // Value 12-0  = Target 13-1
-        break;
-
-      case fixup_Hexagon_B32_PCREL_X:
-        InstMask = 0x0fff3fff;  // Word32_X26
-        Reloc = (((Value >> 14) & 0xfff) << 16) |  // Value 25-14 = Target 27-16
-                (Value & 0x3fff);                  // Value 13-0  = Target 13-0
-        break;
-
-      case FK_Data_1:
-      case FK_Data_2:
-      case FK_Data_4:
-      case fixup_Hexagon_32:
-        InstMask = 0xffffffff;  // Word32
-        Reloc = Value;
-        break;
-    }
-
-    LLVM_DEBUG(dbgs() << "Name=" << getFixupKindInfo(Kind).Name << "("
-                      << (unsigned)Kind << ")\n");
-    LLVM_DEBUG(
-        uint32_t OldData = 0; for (unsigned i = 0; i < NumBytes; i++) OldData |=
-                              (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
-        dbgs() << "\tBValue=0x"; dbgs().write_hex(Value) << ": AValue=0x";
-        dbgs().write_hex(FixupValue)
-        << ": Offset=" << Offset << ": Size=" << Data.size() << ": OInst=0x";
-        dbgs().write_hex(OldData) << ": Reloc=0x"; dbgs().write_hex(Reloc););
-
-    // For each byte of the fragment that the fixup touches, mask in the
-    // bits from the fixup value. The Value has been "split up" into the
-    // appropriate bitfields above.
-    for (unsigned i = 0; i < NumBytes; i++){
-      InstAddr[i] &= uint8_t(~InstMask >> (i * 8)) & 0xff; // Clear reloc bits
-      InstAddr[i] |= uint8_t(Reloc >> (i * 8)) & 0xff;     // Apply new reloc
-    }
-
-    LLVM_DEBUG(uint32_t NewData = 0;
-               for (unsigned i = 0; i < NumBytes; i++) NewData |=
-               (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
-               dbgs() << ": NInst=0x"; dbgs().write_hex(NewData) << "\n";);
-  }
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &,
+                  MutableArrayRef<char> Data, uint64_t FixupValue,
+                  bool IsResolved) override;
 
   bool isInstRelaxable(MCInst const &HMI) const {
     const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(*MCII, HMI);
@@ -552,13 +428,11 @@ class HexagonAsmBackend : public MCAsmBackend {
     return Relaxable;
   }
 
-  /// MayNeedRelaxation - Check whether the given instruction may need
-  /// relaxation.
-  ///
-  /// \param Inst - The instruction to test.
-  bool mayNeedRelaxation(MCInst const &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override {
-    RelaxedMCB = &Inst;
+    RelaxedMCB.clear();
+    RelaxedMCB.setOpcode(Opcode);
+    RelaxedMCB.setOperands(Operands);
     return true;
   }
 
@@ -567,7 +441,7 @@ class HexagonAsmBackend : public MCAsmBackend {
   bool fixupNeedsRelaxationAdvanced(const MCFixup &Fixup, const MCValue &,
                                     uint64_t Value,
                                     bool Resolved) const override {
-    MCInst const &MCB = *RelaxedMCB;
+    MCInst const &MCB = RelaxedMCB;
     assert(HexagonMCInstrInfo::isBundle(MCB));
 
     *RelaxTarget = nullptr;
@@ -722,7 +596,7 @@ class HexagonAsmBackend : public MCAsmBackend {
             case MCFragment::FT_Relaxable: {
               MCContext &Context = getContext();
               auto &RF = cast<MCRelaxableFragment>(*Frags[K]);
-              auto &Inst = const_cast<MCInst &>(RF.getInst());
+              MCInst Inst = RF.getInst();
 
               const bool WouldTraverseLabel = llvm::any_of(
                   Asm.symbols(), [&Asm, &RF, &Inst](MCSymbol const &sym) {
@@ -775,6 +649,133 @@ class HexagonAsmBackend : public MCAsmBackend {
 
 } // namespace
 
+void HexagonAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                                   const MCValue &Target,
+                                   MutableArrayRef<char> Data,
+                                   uint64_t FixupValue, bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, FixupValue, IsResolved);
+  // When FixupValue is 0 the relocation is external and there
+  // is nothing for us to do.
+  if (!FixupValue)
+    return;
+
+  MCFixupKind Kind = Fixup.getKind();
+  uint64_t Value;
+  uint32_t InstMask;
+  uint32_t Reloc;
+
+  // LLVM gives us an encoded value, we have to convert it back
+  // to a real offset before we can use it.
+  uint32_t Offset = Fixup.getOffset();
+  unsigned NumBytes = getFixupKindNumBytes(Kind);
+  assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
+  char *InstAddr = Data.data() + Offset;
+
+  Value = adjustFixupValue(Kind, FixupValue);
+  if (!Value)
+    return;
+  int sValue = (int)Value;
+
+  switch ((unsigned)Kind) {
+  default:
+    return;
+
+  case fixup_Hexagon_B7_PCREL:
+    if (!(isIntN(7, sValue)))
+      HandleFixupError(7, 2, (int64_t)FixupValue, "B7_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B7_PCREL_X:
+    InstMask = 0x00001f18;                 // Word32_B7
+    Reloc = (((Value >> 2) & 0x1f) << 8) | // Value 6-2 = Target 12-8
+            ((Value & 0x3) << 3);          // Value 1-0 = Target 4-3
+    break;
+
+  case fixup_Hexagon_B9_PCREL:
+    if (!(isIntN(9, sValue)))
+      HandleFixupError(9, 2, (int64_t)FixupValue, "B9_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B9_PCREL_X:
+    InstMask = 0x003000fe;                 // Word32_B9
+    Reloc = (((Value >> 7) & 0x3) << 20) | // Value 8-7 = Target 21-20
+            ((Value & 0x7f) << 1);         // Value 6-0 = Target 7-1
+    break;
+
+    // Since the existing branches that use this relocation cannot be
+    // extended, they should only be fixed up if the target is within range.
+  case fixup_Hexagon_B13_PCREL:
+    if (!(isIntN(13, sValue)))
+      HandleFixupError(13, 2, (int64_t)FixupValue, "B13_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B13_PCREL_X:
+    InstMask = 0x00202ffe;                  // Word32_B13
+    Reloc = (((Value >> 12) & 0x1) << 21) | // Value 12   = Target 21
+            (((Value >> 11) & 0x1) << 13) | // Value 11   = Target 13
+            ((Value & 0x7ff) << 1);         // Value 10-0 = Target 11-1
+    break;
+
+  case fixup_Hexagon_B15_PCREL:
+    if (!(isIntN(15, sValue)))
+      HandleFixupError(15, 2, (int64_t)FixupValue, "B15_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B15_PCREL_X:
+    InstMask = 0x00df20fe;                  // Word32_B15
+    Reloc = (((Value >> 13) & 0x3) << 22) | // Value 14-13 = Target 23-22
+            (((Value >> 8) & 0x1f) << 16) | // Value 12-8  = Target 20-16
+            (((Value >> 7) & 0x1) << 13) |  // Value 7     = Target 13
+            ((Value & 0x7f) << 1);          // Value 6-0   = Target 7-1
+    break;
+
+  case fixup_Hexagon_B22_PCREL:
+    if (!(isIntN(22, sValue)))
+      HandleFixupError(22, 2, (int64_t)FixupValue, "B22_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B22_PCREL_X:
+    InstMask = 0x01ff3ffe;                    // Word32_B22
+    Reloc = (((Value >> 13) & 0x1ff) << 16) | // Value 21-13 = Target 24-16
+            ((Value & 0x1fff) << 1);          // Value 12-0  = Target 13-1
+    break;
+
+  case fixup_Hexagon_B32_PCREL_X:
+    InstMask = 0x0fff3fff;                    // Word32_X26
+    Reloc = (((Value >> 14) & 0xfff) << 16) | // Value 25-14 = Target 27-16
+            (Value & 0x3fff);                 // Value 13-0  = Target 13-0
+    break;
+
+  case FK_Data_1:
+  case FK_Data_2:
+  case FK_Data_4:
+  case fixup_Hexagon_32:
+    InstMask = 0xffffffff; // Word32
+    Reloc = Value;
+    break;
+  }
+
+  LLVM_DEBUG(dbgs() << "Name=" << getFixupKindInfo(Kind).Name << "("
+                    << (unsigned)Kind << ")\n");
+  LLVM_DEBUG(
+      uint32_t OldData = 0; for (unsigned i = 0; i < NumBytes; i++) OldData |=
+                            (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
+      dbgs() << "\tBValue=0x"; dbgs().write_hex(Value) << ": AValue=0x";
+      dbgs().write_hex(FixupValue)
+      << ": Offset=" << Offset << ": Size=" << Data.size() << ": OInst=0x";
+      dbgs().write_hex(OldData) << ": Reloc=0x"; dbgs().write_hex(Reloc););
+
+  // For each byte of the fragment that the fixup touches, mask in the
+  // bits from the fixup value. The Value has been "split up" into the
+  // appropriate bitfields above.
+  for (unsigned i = 0; i < NumBytes; i++) {
+    InstAddr[i] &= uint8_t(~InstMask >> (i * 8)) & 0xff; // Clear reloc bits
+    InstAddr[i] |= uint8_t(Reloc >> (i * 8)) & 0xff;     // Apply new reloc
+  }
+
+  LLVM_DEBUG(uint32_t NewData = 0;
+             for (unsigned i = 0; i < NumBytes; i++) NewData |=
+             (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
+             dbgs() << ": NInst=0x"; dbgs().write_hex(NewData) << "\n";);
+}
+
 // MCAsmBackend
 MCAsmBackend *llvm::createHexagonAsmBackend(Target const &T,
                                             const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
index b0d79357557f0..ed381c33225d2 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
@@ -85,8 +85,6 @@ unsigned HexagonELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       report_fatal_error("Unrecognized variant type");
     };
-  case FK_PCRel_4:
-    return ELF::R_HEX_32_PCREL;
   case FK_Data_2:
     switch(Variant) {
     case HexagonMCExpr::VK_DTPREL:
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
index 9211671bb630c..dfe0fa973c9b3 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
@@ -337,6 +337,35 @@ static const std::map<unsigned, std::vector<unsigned>> StdFixups = {
 #undef P
 #undef _
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case Hexagon::fixup_Hexagon_B22_PCREL:
+  case Hexagon::fixup_Hexagon_B15_PCREL:
+  case Hexagon::fixup_Hexagon_B7_PCREL:
+  case Hexagon::fixup_Hexagon_B13_PCREL:
+  case Hexagon::fixup_Hexagon_B9_PCREL:
+  case Hexagon::fixup_Hexagon_B32_PCREL_X:
+  case Hexagon::fixup_Hexagon_B22_PCREL_X:
+  case Hexagon::fixup_Hexagon_B15_PCREL_X:
+  case Hexagon::fixup_Hexagon_B13_PCREL_X:
+  case Hexagon::fixup_Hexagon_B9_PCREL_X:
+  case Hexagon::fixup_Hexagon_B7_PCREL_X:
+  case Hexagon::fixup_Hexagon_32_PCREL:
+  case Hexagon::fixup_Hexagon_PLT_B22_PCREL:
+  case Hexagon::fixup_Hexagon_GD_PLT_B22_PCREL:
+  case Hexagon::fixup_Hexagon_LD_PLT_B22_PCREL:
+  case Hexagon::fixup_Hexagon_6_PCREL_X:
+  case Hexagon::fixup_Hexagon_GD_PLT_B22_PCREL_X:
+  case Hexagon::fixup_Hexagon_GD_PLT_B32_PCREL_X:
+  case Hexagon::fixup_Hexagon_LD_PLT_B22_PCREL_X:
+  case Hexagon::fixup_Hexagon_LD_PLT_B32_PCREL_X:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 uint32_t HexagonMCCodeEmitter::parseBits(size_t Last, MCInst const &MCB,
                                          MCInst const &MCI) const {
   bool Duplex = HexagonMCInstrInfo::isDuplex(MCII, MCI);
@@ -617,7 +646,7 @@ unsigned HexagonMCCodeEmitter::getExprOpValue(const MCInst &MI,
   const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(MCII, MI);
   unsigned FixupWidth = HexagonMCInstrInfo::getExtentBits(MCII, MI) -
                         HexagonMCInstrInfo::getExtentAlignment(MCII, MI);
-  HexagonMCExpr::VariantKind VarKind = getVariantKind(MCSRE);
+  auto VarKind = HexagonMCExpr::VariantKind(MCSRE->getSpecifier());
   unsigned Opc = MCID.getOpcode();
   unsigned IType = HexagonMCInstrInfo::getType(MCII, MI);
 
@@ -698,10 +727,7 @@ unsigned HexagonMCCodeEmitter::getExprOpValue(const MCInst &MI,
     FixupExpr = MCBinaryExpr::createAdd(FixupExpr, C, MCT);
   }
 
-  MCFixup Fixup = MCFixup::create(State.Addend, FixupExpr,
-                                  MCFixupKind(FixupKind), MI.getLoc());
-  Fixups.push_back(Fixup);
-  // All of the information is in the fixup.
+  addFixup(Fixups, State.Addend, FixupExpr, FixupKind);
   return 0;
 }
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
index b22960efc372b..a30d81536941c 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
@@ -59,11 +59,6 @@ class HexagonMCExpr : public MCTargetExpr {
   bool S27_2_reloc;
   bool SignMismatch;
 };
-
-static inline HexagonMCExpr::VariantKind
-getVariantKind(const MCSymbolRefExpr *SRE) {
-  return HexagonMCExpr::VariantKind(SRE->getKind());
-}
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_HEXAGON_HEXAGONMCEXPR_H
diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index 6a74686a239d0..cef77f1c512f6 100644
--- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -344,7 +344,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
       return SymbolRefExpr->getSpecifier() == Lanai::S_None;
     if (const MCSymbolRefExpr *SymbolRefExpr =
             dyn_cast<MCSymbolRefExpr>(Imm.Value)) {
-      return SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None;
+      return SymbolRefExpr->getSpecifier() == 0;
     }
 
     // Binary expression
@@ -354,7 +354,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
         return SymbolRefExpr->getSpecifier() == Lanai::S_None;
       if (const MCSymbolRefExpr *SymbolRefExpr =
               dyn_cast<MCSymbolRefExpr>(BinaryExpr->getLHS()))
-        return SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None;
+        return SymbolRefExpr->getSpecifier() == 0;
     }
 
     return false;
@@ -535,8 +535,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
 #ifndef NDEBUG
       const MCSymbolRefExpr *SymbolRefExpr =
           dyn_cast<MCSymbolRefExpr>(getImm());
-      assert(SymbolRefExpr &&
-             SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None);
+      assert(SymbolRefExpr && SymbolRefExpr->getSpecifier() == 0);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCBinaryExpr>(getImm())) {
@@ -551,7 +550,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
       assert(false && "Operand type not supported.");
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case IMMEDIATE:
       OS << "Imm: " << getImm() << "\n";
@@ -563,10 +562,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
       OS << "Reg: %r" << getReg() << "\n";
       break;
     case MEMORY_IMM:
-      OS << "MemImm: " << *getMemOffset() << "\n";
+      OS << "MemImm: ";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << '\n';
       break;
     case MEMORY_REG_IMM:
-      OS << "MemRegImm: " << getMemBaseReg() << "+" << *getMemOffset() << "\n";
+      OS << "MemRegImm: " << getMemBaseReg() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << '\n';
       break;
     case MEMORY_REG_REG:
       assert(getMemOffset() == nullptr);
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 16ea2c8461fa4..d23c5f43ad4ff 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -150,10 +150,6 @@ LanaiTargetLowering::LanaiTargetLowering(const TargetMachine &TM,
   // statements. Re-evaluate this on new benchmarks.
   setMinimumJumpTableEntries(100);
 
-  // Use fast calling convention for library functions.
-  for (RTLIB::Libcall LC : RTLIB::libcalls())
-    setLibcallCallingConv(LC, CallingConv::Fast);
-
   MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
   MaxStoresPerMemsetOptSize = 8;
   MaxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores
@@ -712,9 +708,8 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
   // Build a sequence of copy-to-reg nodes chained together with token chain and
   // flag operands which copy the outgoing args into registers.  The InGlue in
   // necessary since all emitted instructions must be stuck together.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
-                             RegsToPass[I].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -745,9 +740,8 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
-    Ops.push_back(DAG.getRegister(RegsToPass[I].first,
-                                  RegsToPass[I].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   if (InGlue.getNode())
     Ops.push_back(InGlue);
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
index a0bc4f17ad884..83d16971e35c3 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
@@ -11,9 +11,9 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -71,13 +71,15 @@ bool LanaiAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-void LanaiAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void LanaiAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target,
                                  MutableArrayRef<char> Data, uint64_t Value,
-                                 bool) {
+                                 bool IsResolved) {
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
   MCFixupKind Kind = Fixup.getKind();
   Value = adjustFixupValue(static_cast<unsigned>(Kind), Value);
-
   if (!Value)
     return; // This value doesn't change the encoding
 
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
index d1b2da40446a6..84dc4b8312370 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
@@ -290,8 +290,7 @@ unsigned LanaiMCCodeEmitter::getBranchTargetOpValue(
   if (MCOp.isReg() || MCOp.isImm())
     return getMachineOpValue(Inst, MCOp, Fixups, SubtargetInfo);
 
-  Fixups.push_back(MCFixup::create(
-      0, MCOp.getExpr(), static_cast<MCFixupKind>(Lanai::FIXUP_LANAI_25)));
+  Fixups.push_back(MCFixup::create(0, MCOp.getExpr(), Lanai::FIXUP_LANAI_25));
 
   return 0;
 }
diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index f5223a599737e..5be4713b349ee 100644
--- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -555,7 +555,7 @@ class LoongArchOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     auto RegName = [](MCRegister Reg) {
       if (Reg)
         return LoongArchInstPrinter::getRegisterName(Reg);
@@ -565,7 +565,7 @@ class LoongArchOperand : public MCParsedAsmOperand {
 
     switch (Kind) {
     case KindTy::Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case KindTy::Register:
       OS << "<register " << RegName(getReg()) << ">";
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 7dae4d30d31be..72dbb44815657 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -195,6 +195,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
     setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
     setOperationAction(ISD::IS_FPCLASS, MVT::f32, Legal);
@@ -242,6 +243,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
     setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
     setOperationAction(ISD::IS_FPCLASS, MVT::f64, Legal);
     setOperationAction(ISD::FSIN, MVT::f64, Expand);
@@ -2607,9 +2609,6 @@ SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
 
 SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
                                                  SelectionDAG &DAG) const {
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   // Currently only support lowering return address for current frame.
   if (Op.getConstantOperandVal(0) != 0) {
     DAG.getContext()->emitError(
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index b1491b75ac5bc..1b8893029bb33 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -57,9 +57,9 @@ MCFixupKindInfo LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       // LoongArchFixupKinds.h.
       //
       // {name, offset, bits, flags}
-      {"fixup_loongarch_b16", 10, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_loongarch_b21", 0, 26, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_loongarch_b26", 0, 26, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_loongarch_b16", 10, 16, 0},
+      {"fixup_loongarch_b21", 0, 26, 0},
+      {"fixup_loongarch_b26", 0, 26, 0},
       {"fixup_loongarch_abs_hi20", 5, 20, 0},
       {"fixup_loongarch_abs_lo12", 10, 12, 0},
       {"fixup_loongarch_abs64_lo20", 5, 20, 0},
@@ -72,7 +72,7 @@ MCFixupKindInfo LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_LARCH_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -140,10 +140,13 @@ static void fixupLeb128(MCContext &Ctx, const MCFixup &Fixup,
     Ctx.reportError(Fixup.getLoc(), "Invalid uleb128 value!");
 }
 
-void LoongArchAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void LoongArchAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                      const MCValue &Target,
                                      MutableArrayRef<char> Data, uint64_t Value,
                                      bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  IsResolved = addReloc(F, Fixup, Target, Value, IsResolved);
   if (!Value)
     return; // Doesn't change encoding.
 
@@ -262,23 +265,17 @@ getRelocPairForSize(unsigned Size) {
   default:
     llvm_unreachable("unsupported fixup size");
   case 6:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD6),
-                          MCFixupKind(ELF::R_LARCH_SUB6));
+    return std::make_pair(ELF::R_LARCH_ADD6, ELF::R_LARCH_SUB6);
   case 8:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD8),
-                          MCFixupKind(ELF::R_LARCH_SUB8));
+    return std::make_pair(ELF::R_LARCH_ADD8, ELF::R_LARCH_SUB8);
   case 16:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD16),
-                          MCFixupKind(ELF::R_LARCH_SUB16));
+    return std::make_pair(ELF::R_LARCH_ADD16, ELF::R_LARCH_SUB16);
   case 32:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD32),
-                          MCFixupKind(ELF::R_LARCH_SUB32));
+    return std::make_pair(ELF::R_LARCH_ADD32, ELF::R_LARCH_SUB32);
   case 64:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD64),
-                          MCFixupKind(ELF::R_LARCH_SUB64));
+    return std::make_pair(ELF::R_LARCH_ADD64, ELF::R_LARCH_SUB64);
   case 128:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD_ULEB128),
-                          MCFixupKind(ELF::R_LARCH_SUB_ULEB128));
+    return std::make_pair(ELF::R_LARCH_ADD_ULEB128, ELF::R_LARCH_SUB_ULEB128);
   }
 }
 
@@ -287,8 +284,7 @@ std::pair<bool, bool> LoongArchAsmBackend::relaxLEB128(MCLEBFragment &LF,
   const MCExpr &Expr = LF.getValue();
   if (LF.isSigned() || !Expr.evaluateKnownAbsolute(Value, *Asm))
     return std::make_pair(false, false);
-  LF.getFixups().push_back(
-      MCFixup::create(0, &Expr, FK_Data_leb128, Expr.getLoc()));
+  LF.addFixup(MCFixup::create(0, &Expr, FK_Data_leb128));
   return std::make_pair(true, true);
 }
 
@@ -298,9 +294,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 
   int64_t LineDelta = DF.getLineDelta();
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 1> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -309,8 +304,7 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
   assert(IsAbsolute && "CFA with invalid expression");
   (void)IsAbsolute;
 
-  Data.clear();
-  Fixups.clear();
+  SmallVector<char> Data;
   raw_svector_ostream OS(Data);
 
   // INT64_MAX is a signal that this is actually a DW_LNE_end_sequence.
@@ -355,6 +349,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
     OS << uint8_t(dwarf::DW_LNS_copy);
   }
 
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
   WasRelaxed = OldSize != Data.size();
   return true;
 }
@@ -362,9 +358,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
                                         bool &WasRelaxed) const {
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 2> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -373,14 +368,12 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   assert(IsAbsolute && "CFA with invalid expression");
   (void)IsAbsolute;
 
-  Data.clear();
-  Fixups.clear();
-  raw_svector_ostream OS(Data);
-
   assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 &&
          "expected 1-byte alignment");
   if (Value == 0) {
-    WasRelaxed = OldSize != Data.size();
+    DF.clearContents();
+    DF.clearFixups();
+    WasRelaxed = OldSize != DF.getContents().size();
     return true;
   }
 
@@ -392,6 +385,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
     Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(FK)));
   };
 
+  SmallVector<char, 8> Data;
+  raw_svector_ostream OS(Data);
   if (isUIntN(6, Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc);
     AddFixups(0, getRelocPairForSize(6));
@@ -410,6 +405,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   } else {
     llvm_unreachable("unsupported CFA encoding");
   }
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
 
   WasRelaxed = OldSize != Data.size();
   return true;
@@ -453,7 +450,8 @@ bool LoongArchAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
                                    const MCValue &Target, uint64_t &FixedValue,
                                    bool IsResolved) {
   auto Fallback = [&]() {
-    return MCAsmBackend::addReloc(F, Fixup, Target, FixedValue, IsResolved);
+    MCAsmBackend::maybeAddReloc(F, Fixup, Target, FixedValue, IsResolved);
+    return true;
   };
   uint64_t FixedValueA, FixedValueB;
   if (Target.getSubSym()) {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index 56554c5c664eb..4446cadf11e22 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -18,7 +18,6 @@
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 
@@ -40,7 +39,7 @@ class LoongArchAsmBackend : public MCAsmBackend {
                       const MCTargetOptions &Options);
 
   bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                uint64_t &FixedValue, bool IsResolved) override;
+                uint64_t &FixedValue, bool IsResolved);
 
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
@@ -54,16 +53,12 @@ class LoongArchAsmBackend : public MCAsmBackend {
   bool shouldInsertFixupForCodeAlign(MCAssembler &Asm,
                                      MCAlignFragment &AF) override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override {}
-
   bool relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
                           bool &WasRelaxed) const override;
   bool relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
index ba891ebebdbe3..b7ead5e61ab81 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
@@ -97,6 +97,18 @@ class LoongArchMCCodeEmitter : public MCCodeEmitter {
 };
 } // end namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case LoongArch::fixup_loongarch_b16:
+  case LoongArch::fixup_loongarch_b21:
+  case LoongArch::fixup_loongarch_b26:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 unsigned
 LoongArchMCCodeEmitter::getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                                           SmallVectorImpl<MCFixup> &Fixups,
@@ -167,9 +179,7 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
       RelaxCandidate = true;
       break;
     }
-  } else if (Kind == MCExpr::SymbolRef &&
-             cast<MCSymbolRefExpr>(Expr)->getKind() ==
-                 MCSymbolRefExpr::VK_None) {
+  } else if (Kind == MCExpr::SymbolRef) {
     switch (MI.getOpcode()) {
     default:
       break;
@@ -197,8 +207,7 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
   assert(FixupKind != LoongArch::fixup_loongarch_invalid &&
          "Unhandled expression!");
 
-  Fixups.push_back(
-      MCFixup::create(0, Expr, MCFixupKind(FixupKind), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, FixupKind);
   // If linker relaxation is enabled and supported by this relocation, set
   // a bit so that if fixup is unresolved, a R_LARCH_RELAX relocation will be
   // appended.
@@ -251,8 +260,7 @@ void LoongArchMCCodeEmitter::expandAddTPRel(const MCInst &MI,
          "Expected %le_add_r relocation on TP-relative symbol");
 
   // Emit the correct %le_add_r relocation for the symbol.
-  Fixups.push_back(
-      MCFixup::create(0, Expr, ELF::R_LARCH_TLS_LE_ADD_R, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, ELF::R_LARCH_TLS_LE_ADD_R);
   if (STI.hasFeature(LoongArch::FeatureRelax))
     Fixups.back().setLinkerRelaxable();
 
diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
index e264d4e521cc4..16f97d971840b 100644
--- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
+++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "M68kInstrInfo.h"
 #include "M68kRegisterInfo.h"
+#include "MCTargetDesc/M68kMCAsmInfo.h"
 #include "TargetInfo/M68kTargetInfo.h"
 
 #include "llvm/MC/MCContext.h"
@@ -144,7 +145,7 @@ class M68kOperand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return Start; }
   SMLoc getEndLoc() const override { return End; }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   bool isMem() const override { return false; }
   bool isMemOp() const { return Kind == KindTy::MemOp; }
@@ -1051,7 +1052,7 @@ bool M68kAsmParser::matchAndEmitInstruction(SMLoc Loc, unsigned &Opcode,
   }
 }
 
-void M68kOperand::print(raw_ostream &OS) const {
+void M68kOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case KindTy::Invalid:
     OS << "invalid";
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 568938f1f50b7..21e9319aaf0b3 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -95,8 +95,8 @@ bool M68kInstrInfo::AnalyzeBranchImpl(MachineBasicBlock &MBB,
   // Erase any instructions if allowed at the end of the scope.
   std::vector<std::reference_wrapper<llvm::MachineInstr>> EraseList;
   auto FinalizeOnReturn = llvm::make_scope_exit([&EraseList] {
-    std::for_each(EraseList.begin(), EraseList.end(),
-                  [](auto &ref) { ref.get().eraseFromParent(); });
+    for (auto &Ref : EraseList)
+      Ref.get().eraseFromParent();
   });
 
   // Start from the bottom of the block and work up, examining the
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
index 20a37302a5e1c..5e039033704f9 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
@@ -18,9 +18,9 @@
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -52,25 +52,11 @@ class M68kAsmBackend : public MCAsmBackend {
                               .CasesLower("m68020", "m68030", "m68040", true)
                               .Default(false)) {}
 
-  void applyFixup(const MCFragment &, const MCFixup &Fixup, const MCValue &,
-                  MutableArrayRef<char> Data, uint64_t Value, bool) override {
-    unsigned Size = 1 << getFixupKindLog2Size(Fixup.getKind());
-
-    assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
-    // Check that uppper bits are either all zeros or all ones.
-    // Specifically ignore overflow/underflow as long as the leakage is
-    // limited to the lower bits. This is to remain compatible with
-    // other assemblers.
-    assert(isIntN(Size * 8 + 1, static_cast<int64_t>(Value)) &&
-           "Value does not fit in the Fixup field");
-
-    // Write in Big Endian
-    for (unsigned i = 0; i != Size; ++i)
-      Data[Fixup.getOffset() + i] =
-          uint8_t(static_cast<int64_t>(Value) >> ((Size - i - 1) * 8));
-  }
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &,
+                  MutableArrayRef<char> Data, uint64_t Value,
+                  bool IsResolved) override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override;
 
   bool fixupNeedsRelaxation(const MCFixup &Fixup,
@@ -91,6 +77,28 @@ class M68kAsmBackend : public MCAsmBackend {
 };
 } // end anonymous namespace
 
+void M68kAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                                const MCValue &Target,
+                                MutableArrayRef<char> Data, uint64_t Value,
+                                bool IsResolved) {
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
+  unsigned Size = 1 << getFixupKindLog2Size(Fixup.getKind());
+  assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
+  // Check that uppper bits are either all zeros or all ones.
+  // Specifically ignore overflow/underflow as long as the leakage is
+  // limited to the lower bits. This is to remain compatible with
+  // other assemblers.
+  assert(isIntN(Size * 8 + 1, static_cast<int64_t>(Value)) &&
+         "Value does not fit in the Fixup field");
+
+  // Write in Big Endian
+  for (unsigned i = 0; i != Size; ++i)
+    Data[Fixup.getOffset() + i] =
+        uint8_t(static_cast<int64_t>(Value) >> ((Size - i - 1) * 8));
+}
+
 /// cc—Carry clear      GE—Greater than or equal
 /// LS—Lower or same    PL—Plus
 /// CS—Carry set        GT—Greater than
@@ -99,8 +107,7 @@ class M68kAsmBackend : public MCAsmBackend {
 /// MI—Minus            VC—Overflow clear
 ///                     LE—Less than or equal
 /// NE—Not equal        VS—Overflow set
-static unsigned getRelaxedOpcodeBranch(const MCInst &Inst) {
-  unsigned Op = Inst.getOpcode();
+static unsigned getRelaxedOpcodeBranch(unsigned Op) {
   switch (Op) {
   default:
     return Op;
@@ -171,37 +178,17 @@ static unsigned getRelaxedOpcodeBranch(const MCInst &Inst) {
   }
 }
 
-static unsigned getRelaxedOpcodeArith(const MCInst &Inst) {
-  unsigned Op = Inst.getOpcode();
+static unsigned getRelaxedOpcode(unsigned Opcode) {
   // NOTE there will be some relaxations for PCD and ARD mem for x20
-  return Op;
-}
-
-static unsigned getRelaxedOpcode(const MCInst &Inst) {
-  unsigned R = getRelaxedOpcodeArith(Inst);
-  if (R != Inst.getOpcode())
-    return R;
-  return getRelaxedOpcodeBranch(Inst);
+  return getRelaxedOpcodeBranch(Opcode);
 }
 
-bool M68kAsmBackend::mayNeedRelaxation(const MCInst &Inst,
+bool M68kAsmBackend::mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand>,
                                        const MCSubtargetInfo &STI) const {
   // Branches can always be relaxed in either mode.
-  if (getRelaxedOpcodeBranch(Inst) != Inst.getOpcode())
-    return true;
+  return getRelaxedOpcode(Opcode) != Opcode;
 
-  // Check if this instruction is ever relaxable.
-  if (getRelaxedOpcodeArith(Inst) == Inst.getOpcode())
-    return false;
-
-  // Check if the relaxable operand has an expression. For the current set of
-  // relaxable instructions, the relaxable operand is always the last operand.
   // NOTE will change for x20 mem
-  unsigned RelaxableOp = Inst.getNumOperands() - 1;
-  if (Inst.getOperand(RelaxableOp).isExpr())
-    return true;
-
-  return false;
 }
 
 bool M68kAsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
@@ -234,16 +221,8 @@ bool M68kAsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
 // we can relax?
 void M68kAsmBackend::relaxInstruction(MCInst &Inst,
                                       const MCSubtargetInfo &STI) const {
-  unsigned RelaxedOp = getRelaxedOpcode(Inst);
-
-  if (RelaxedOp == Inst.getOpcode()) {
-    SmallString<256> Tmp;
-    raw_svector_ostream OS(Tmp);
-    Inst.dump_pretty(OS);
-    OS << "\n";
-    report_fatal_error("unexpected instruction to relax: " + OS.str());
-  }
-
+  unsigned RelaxedOp = getRelaxedOpcode(Inst.getOpcode());
+  assert(RelaxedOp != Inst.getOpcode());
   Inst.setOpcode(RelaxedOp);
 }
 
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
index 03416df639cf3..ca94a475f7834 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
@@ -49,12 +49,9 @@ static M68kRelType getType(unsigned Kind, M68k::Specifier &Modifier,
                            bool &IsPCRel) {
   switch (Kind) {
   case FK_Data_4:
-  case FK_PCRel_4:
     return RT_32;
-  case FK_PCRel_2:
   case FK_Data_2:
     return RT_16;
-  case FK_PCRel_1:
   case FK_Data_1:
     return RT_8;
   }
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h b/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
index 54a0e98fea6e1..0245654d554f7 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
@@ -19,36 +19,15 @@
 namespace llvm {
 static inline unsigned getFixupKindLog2Size(unsigned Kind) {
   switch (Kind) {
-  case FK_PCRel_1:
-  case FK_SecRel_1:
   case FK_Data_1:
     return 0;
-  case FK_PCRel_2:
-  case FK_SecRel_2:
   case FK_Data_2:
     return 1;
-  case FK_PCRel_4:
-  case FK_SecRel_4:
   case FK_Data_4:
     return 2;
   }
   llvm_unreachable("invalid fixup kind!");
 }
-
-static inline MCFixupKind getFixupForSize(unsigned Size, bool isPCRel) {
-  switch (Size) {
-  case 8:
-    return isPCRel ? FK_PCRel_1 : FK_Data_1;
-  case 16:
-    return isPCRel ? FK_PCRel_2 : FK_Data_2;
-  case 32:
-    return isPCRel ? FK_PCRel_4 : FK_Data_4;
-  case 64:
-    return isPCRel ? FK_PCRel_8 : FK_Data_8;
-  }
-  llvm_unreachable("Invalid generic fixup size!");
-}
-
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_M68k_MCTARGETDESC_M68kFIXUPKINDS_H
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
index 3bd1d0d7dcaeb..9b755e021c5f2 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
@@ -103,6 +103,11 @@ template <unsigned Size> static unsigned getBytePosition(unsigned BitPos) {
   }
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // We need special handlings for relocatable & pc-relative operands that are
 // larger than a word.
 // A M68k instruction is aligned by word (16 bits). That means, 32-bit
@@ -139,9 +144,7 @@ void M68kMCCodeEmitter::encodeRelocImm(const MCInst &MI, unsigned OpIdx,
 
     // Relocatable address
     unsigned InsertByte = getBytePosition<Size>(InsertPos);
-    Fixups.push_back(MCFixup::create(InsertByte, Expr,
-                                     getFixupForSize(Size, /*IsPCRel=*/false),
-                                     MI.getLoc()));
+    addFixup(Fixups, InsertByte, Expr, MCFixup::getDataKindForSize(Size / 8));
   }
 }
 
@@ -175,9 +178,8 @@ void M68kMCCodeEmitter::encodePCRelImm(const MCInst &MI, unsigned OpIdx,
             Expr, MCConstantExpr::create(LabelOffset, Ctx), Ctx);
     }
 
-    Fixups.push_back(MCFixup::create(InsertByte, Expr,
-                                     getFixupForSize(Size, /*IsPCRel=*/true),
-                                     MI.getLoc()));
+    addFixup(Fixups, InsertByte, Expr, MCFixup::getDataKindForSize(Size / 8),
+             true);
   }
 }
 
diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
index 5a4121f7cafd7..38d22eda5f171 100644
--- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
+++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/MSP430MCAsmInfo.h"
+#include "MCTargetDesc/MSP430MCTargetDesc.h"
 #include "MSP430.h"
 #include "MSP430RegisterInfo.h"
-#include "MCTargetDesc/MSP430MCTargetDesc.h"
 #include "TargetInfo/MSP430TargetInfo.h"
 
 #include "llvm/ADT/APInt.h"
@@ -224,7 +225,7 @@ class MSP430Operand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return Start; }
   SMLoc getEndLoc() const override { return End; }
 
-  void print(raw_ostream &O) const override {
+  void print(raw_ostream &O, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Tok:
       O << "Token " << Tok;
@@ -233,11 +234,12 @@ class MSP430Operand : public MCParsedAsmOperand {
       O << "Register " << Reg;
       break;
     case k_Imm:
-      O << "Immediate " << *Imm;
+      O << "Immediate ";
+      MAI.printExpr(O, *Imm);
       break;
     case k_Mem:
       O << "Memory ";
-      O << *Mem.Offset << "(" << Reg << ")";
+      MAI.printExpr(O, *Mem.Offset);
       break;
     case k_IndReg:
       O << "RegInd " << Reg;
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
index 54fd008790f90..29e5bfabe8c9a 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
@@ -46,21 +45,23 @@ class MSP430AsmBackend : public MCAsmBackend {
   }
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
+    // clang-format off
     const static MCFixupKindInfo Infos[MSP430::NumTargetFixupKinds] = {
       // This table must be in the same order of enum in MSP430FixupKinds.h.
       //
       // name            offset bits flags
       {"fixup_32",            0, 32, 0},
-      {"fixup_10_pcrel",      0, 10, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_10_pcrel",      0, 10, 0},
       {"fixup_16",            0, 16, 0},
-      {"fixup_16_pcrel",      0, 16, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_16_pcrel",      0, 16, 0},
       {"fixup_16_byte",       0, 16, 0},
-      {"fixup_16_pcrel_byte", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_2x_pcrel",      0, 10, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_rl_pcrel",      0, 16, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_16_pcrel_byte", 0, 16, 0},
+      {"fixup_2x_pcrel",      0, 10, 0},
+      {"fixup_rl_pcrel",      0, 16, 0},
       {"fixup_8",             0,  8, 0},
       {"fixup_sym_diff",      0, 32, 0},
     };
+    // clang-format on
     static_assert((std::size(Infos)) == MSP430::NumTargetFixupKinds,
                   "Not all fixup kinds added to Infos array");
 
@@ -103,10 +104,11 @@ uint64_t MSP430AsmBackend::adjustFixupValue(const MCFixup &Fixup,
   }
 }
 
-void MSP430AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void MSP430AsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                   const MCValue &Target,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   Value = adjustFixupValue(Fixup, Value, getContext());
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   if (!Value)
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
index 35a7da4fbe5a2..f018bdbeb0bfb 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
@@ -77,6 +77,20 @@ class MSP430MCCodeEmitter : public MCCodeEmitter {
                          const MCSubtargetInfo &STI) const override;
 };
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case MSP430::fixup_10_pcrel:
+  case MSP430::fixup_16_pcrel:
+  case MSP430::fixup_16_pcrel_byte:
+  case MSP430::fixup_2x_pcrel:
+  case MSP430::fixup_rl_pcrel:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void MSP430MCCodeEmitter::encodeInstruction(const MCInst &MI,
                                             SmallVectorImpl<char> &CB,
                                             SmallVectorImpl<MCFixup> &Fixups,
@@ -111,8 +125,7 @@ unsigned MSP430MCCodeEmitter::getMachineOpValue(const MCInst &MI,
   }
 
   assert(MO.isExpr() && "Expected expr operand");
-  Fixups.push_back(MCFixup::create(Offset, MO.getExpr(),
-      static_cast<MCFixupKind>(MSP430::fixup_16_byte), MI.getLoc()));
+  addFixup(Fixups, Offset, MO.getExpr(), MSP430::fixup_16_byte);
   Offset += 2;
   return 0;
 }
@@ -143,8 +156,7 @@ unsigned MSP430MCCodeEmitter::getMemOpValue(const MCInst &MI, unsigned Op,
     FixupKind = MSP430::fixup_16_byte;
     break;
   }
-  Fixups.push_back(MCFixup::create(Offset, MO2.getExpr(),
-    static_cast<MCFixupKind>(FixupKind), MI.getLoc()));
+  addFixup(Fixups, Offset, MO2.getExpr(), FixupKind);
   Offset += 2;
   return Reg;
 }
@@ -157,8 +169,7 @@ unsigned MSP430MCCodeEmitter::getPCRelImmOpValue(const MCInst &MI, unsigned Op,
     return MO.getImm();
 
   assert(MO.isExpr() && "Expr operand expected");
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-    static_cast<MCFixupKind>(MSP430::fixup_10_pcrel), MI.getLoc()));
+  addFixup(Fixups, 0, MO.getExpr(), MSP430::fixup_10_pcrel);
   return 0;
 }
 
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index f8f688c8fbb14..d23504c203dd3 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -208,7 +208,6 @@ MSP430TargetLowering::MSP430TargetLowering(const TargetMachine &TM,
     for (const auto &LC : LibraryCalls) {
       setLibcallImpl(LC.Op, LC.Impl);
     }
-    setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::MSP430_BUILTIN);
   }
 
   setMinFunctionAlignment(Align(2));
@@ -768,9 +767,8 @@ SDValue MSP430TargetLowering::LowerCCCCallTo(
   // flag operands which copy the outgoing args into registers.  The InGlue in
   // necessary since all emitted instructions must be stuck together.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -790,9 +788,8 @@ SDValue MSP430TargetLowering::LowerCCCCallTo(
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   if (InGlue.getNode())
     Ops.push_back(InGlue);
@@ -1144,9 +1141,6 @@ SDValue MSP430TargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   unsigned Depth = Op.getConstantOperandVal(0);
   SDLoc dl(Op);
   EVT PtrVT = Op.getValueType();
diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index b559a8b896e0f..01e4d17f6236d 100644
--- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -1660,18 +1660,18 @@ class MipsOperand : public MCParsedAsmOperand {
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Immediate:
       OS << "Imm<";
-      OS << *Imm.Val;
+      MAI.printExpr(OS, *Imm.Val);
       OS << ">";
       break;
     case k_Memory:
       OS << "Mem<";
-      Mem.Base->print(OS);
+      Mem.Base->print(OS, MAI);
       OS << ", ";
-      OS << *Mem.Off;
+      MAI.printExpr(OS, *Mem.Off);
       OS << ">";
       break;
     case k_RegisterIndex:
@@ -1774,7 +1774,7 @@ static bool isEvaluated(const MCExpr *Expr) {
   case MCExpr::Constant:
     return true;
   case MCExpr::SymbolRef:
-    return (cast<MCSymbolRefExpr>(Expr)->getKind() != MCSymbolRefExpr::VK_None);
+    return (cast<MCSymbolRefExpr>(Expr)->getSpecifier());
   case MCExpr::Binary: {
     const MCBinaryExpr *BE = cast<MCBinaryExpr>(Expr);
     if (!isEvaluated(BE->getLHS()))
@@ -1817,7 +1817,7 @@ static bool needsExpandMemInst(MCInst &Inst, const MCInstrDesc &MCID) {
 
     // Expand symbol.
     const MCSymbolRefExpr *SR = static_cast<const MCSymbolRefExpr *>(Expr);
-    return SR->getKind() == MCSymbolRefExpr::VK_None;
+    return SR->getSpecifier() == 0;
   }
 
   return false;
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 39d20f0e3c155..c2169beea2753 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -20,7 +20,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptions.h"
@@ -239,13 +238,57 @@ static unsigned calculateMMLEIndex(unsigned i) {
   return (1 - i / 2) * 2 + i % 2;
 }
 
+static bool shouldForceRelocation(const MCFixup &Fixup) {
+  const unsigned FixupKind = Fixup.getKind();
+  switch (FixupKind) {
+  default:
+    return false;
+  // All these relocations require special processing
+  // at linking time. Delegate this work to a linker.
+  case Mips::fixup_Mips_CALL_HI16:
+  case Mips::fixup_Mips_CALL_LO16:
+  case Mips::fixup_Mips_CALL16:
+  case Mips::fixup_Mips_GOT:
+  case Mips::fixup_Mips_GOT_PAGE:
+  case Mips::fixup_Mips_GOT_OFST:
+  case Mips::fixup_Mips_GOT_DISP:
+  case Mips::fixup_Mips_GOT_HI16:
+  case Mips::fixup_Mips_GOT_LO16:
+  case Mips::fixup_Mips_GOTTPREL:
+  case Mips::fixup_Mips_DTPREL_HI:
+  case Mips::fixup_Mips_DTPREL_LO:
+  case Mips::fixup_Mips_TLSGD:
+  case Mips::fixup_Mips_TLSLDM:
+  case Mips::fixup_Mips_TPREL_HI:
+  case Mips::fixup_Mips_TPREL_LO:
+  case Mips::fixup_Mips_JALR:
+  case Mips::fixup_MICROMIPS_CALL16:
+  case Mips::fixup_MICROMIPS_GOT_DISP:
+  case Mips::fixup_MICROMIPS_GOT_PAGE:
+  case Mips::fixup_MICROMIPS_GOT_OFST:
+  case Mips::fixup_MICROMIPS_GOT16:
+  case Mips::fixup_MICROMIPS_GOTTPREL:
+  case Mips::fixup_MICROMIPS_TLS_DTPREL_HI16:
+  case Mips::fixup_MICROMIPS_TLS_DTPREL_LO16:
+  case Mips::fixup_MICROMIPS_TLS_GD:
+  case Mips::fixup_MICROMIPS_TLS_LDM:
+  case Mips::fixup_MICROMIPS_TLS_TPREL_HI16:
+  case Mips::fixup_MICROMIPS_TLS_TPREL_LO16:
+  case Mips::fixup_MICROMIPS_JALR:
+    return true;
+  }
+}
+
 /// ApplyFixup - Apply the \p Value for given \p Fixup into the provided
 /// data fragment, at the offset specified by the fixup and following the
 /// fixup kind as appropriate.
-void MipsAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void MipsAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                 const MCValue &Target,
                                 MutableArrayRef<char> Data, uint64_t Value,
                                 bool IsResolved) {
+  if (shouldForceRelocation(Fixup))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   MCContext &Ctx = getContext();
   Value = adjustFixupValue(Fixup, Value, Ctx);
@@ -314,41 +357,38 @@ std::optional<MCFixupKind> MipsAsmBackend::getFixupKind(StringRef Name) const {
   return StringSwitch<std::optional<MCFixupKind>>(Name)
       .Case("R_MIPS_NONE", FK_NONE)
       .Case("R_MIPS_32", FK_Data_4)
-      .Case("R_MIPS_CALL_HI16", (MCFixupKind)Mips::fixup_Mips_CALL_HI16)
-      .Case("R_MIPS_CALL_LO16", (MCFixupKind)Mips::fixup_Mips_CALL_LO16)
-      .Case("R_MIPS_CALL16", (MCFixupKind)Mips::fixup_Mips_CALL16)
-      .Case("R_MIPS_GOT16", (MCFixupKind)Mips::fixup_Mips_GOT)
-      .Case("R_MIPS_GOT_PAGE", (MCFixupKind)Mips::fixup_Mips_GOT_PAGE)
-      .Case("R_MIPS_GOT_OFST", (MCFixupKind)Mips::fixup_Mips_GOT_OFST)
-      .Case("R_MIPS_GOT_DISP", (MCFixupKind)Mips::fixup_Mips_GOT_DISP)
-      .Case("R_MIPS_GOT_HI16", (MCFixupKind)Mips::fixup_Mips_GOT_HI16)
-      .Case("R_MIPS_GOT_LO16", (MCFixupKind)Mips::fixup_Mips_GOT_LO16)
-      .Case("R_MIPS_TLS_GOTTPREL", (MCFixupKind)Mips::fixup_Mips_GOTTPREL)
-      .Case("R_MIPS_TLS_DTPREL_HI16", (MCFixupKind)Mips::fixup_Mips_DTPREL_HI)
-      .Case("R_MIPS_TLS_DTPREL_LO16", (MCFixupKind)Mips::fixup_Mips_DTPREL_LO)
-      .Case("R_MIPS_TLS_GD", (MCFixupKind)Mips::fixup_Mips_TLSGD)
-      .Case("R_MIPS_TLS_LDM", (MCFixupKind)Mips::fixup_Mips_TLSLDM)
-      .Case("R_MIPS_TLS_TPREL_HI16", (MCFixupKind)Mips::fixup_Mips_TPREL_HI)
-      .Case("R_MIPS_TLS_TPREL_LO16", (MCFixupKind)Mips::fixup_Mips_TPREL_LO)
-      .Case("R_MICROMIPS_CALL16", (MCFixupKind)Mips::fixup_MICROMIPS_CALL16)
-      .Case("R_MICROMIPS_GOT_DISP", (MCFixupKind)Mips::fixup_MICROMIPS_GOT_DISP)
-      .Case("R_MICROMIPS_GOT_PAGE", (MCFixupKind)Mips::fixup_MICROMIPS_GOT_PAGE)
-      .Case("R_MICROMIPS_GOT_OFST", (MCFixupKind)Mips::fixup_MICROMIPS_GOT_OFST)
-      .Case("R_MICROMIPS_GOT16", (MCFixupKind)Mips::fixup_MICROMIPS_GOT16)
-      .Case("R_MICROMIPS_TLS_GOTTPREL",
-            (MCFixupKind)Mips::fixup_MICROMIPS_GOTTPREL)
+      .Case("R_MIPS_CALL_HI16", Mips::fixup_Mips_CALL_HI16)
+      .Case("R_MIPS_CALL_LO16", Mips::fixup_Mips_CALL_LO16)
+      .Case("R_MIPS_CALL16", Mips::fixup_Mips_CALL16)
+      .Case("R_MIPS_GOT16", Mips::fixup_Mips_GOT)
+      .Case("R_MIPS_GOT_PAGE", Mips::fixup_Mips_GOT_PAGE)
+      .Case("R_MIPS_GOT_OFST", Mips::fixup_Mips_GOT_OFST)
+      .Case("R_MIPS_GOT_DISP", Mips::fixup_Mips_GOT_DISP)
+      .Case("R_MIPS_GOT_HI16", Mips::fixup_Mips_GOT_HI16)
+      .Case("R_MIPS_GOT_LO16", Mips::fixup_Mips_GOT_LO16)
+      .Case("R_MIPS_TLS_GOTTPREL", Mips::fixup_Mips_GOTTPREL)
+      .Case("R_MIPS_TLS_DTPREL_HI16", Mips::fixup_Mips_DTPREL_HI)
+      .Case("R_MIPS_TLS_DTPREL_LO16", Mips::fixup_Mips_DTPREL_LO)
+      .Case("R_MIPS_TLS_GD", Mips::fixup_Mips_TLSGD)
+      .Case("R_MIPS_TLS_LDM", Mips::fixup_Mips_TLSLDM)
+      .Case("R_MIPS_TLS_TPREL_HI16", Mips::fixup_Mips_TPREL_HI)
+      .Case("R_MIPS_TLS_TPREL_LO16", Mips::fixup_Mips_TPREL_LO)
+      .Case("R_MICROMIPS_CALL16", Mips::fixup_MICROMIPS_CALL16)
+      .Case("R_MICROMIPS_GOT_DISP", Mips::fixup_MICROMIPS_GOT_DISP)
+      .Case("R_MICROMIPS_GOT_PAGE", Mips::fixup_MICROMIPS_GOT_PAGE)
+      .Case("R_MICROMIPS_GOT_OFST", Mips::fixup_MICROMIPS_GOT_OFST)
+      .Case("R_MICROMIPS_GOT16", Mips::fixup_MICROMIPS_GOT16)
+      .Case("R_MICROMIPS_TLS_GOTTPREL", Mips::fixup_MICROMIPS_GOTTPREL)
       .Case("R_MICROMIPS_TLS_DTPREL_HI16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_DTPREL_HI16)
+            Mips::fixup_MICROMIPS_TLS_DTPREL_HI16)
       .Case("R_MICROMIPS_TLS_DTPREL_LO16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_DTPREL_LO16)
-      .Case("R_MICROMIPS_TLS_GD", (MCFixupKind)Mips::fixup_MICROMIPS_TLS_GD)
-      .Case("R_MICROMIPS_TLS_LDM", (MCFixupKind)Mips::fixup_MICROMIPS_TLS_LDM)
-      .Case("R_MICROMIPS_TLS_TPREL_HI16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_TPREL_HI16)
-      .Case("R_MICROMIPS_TLS_TPREL_LO16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_TPREL_LO16)
-      .Case("R_MIPS_JALR", (MCFixupKind)Mips::fixup_Mips_JALR)
-      .Case("R_MICROMIPS_JALR", (MCFixupKind)Mips::fixup_MICROMIPS_JALR)
+            Mips::fixup_MICROMIPS_TLS_DTPREL_LO16)
+      .Case("R_MICROMIPS_TLS_GD", Mips::fixup_MICROMIPS_TLS_GD)
+      .Case("R_MICROMIPS_TLS_LDM", Mips::fixup_MICROMIPS_TLS_LDM)
+      .Case("R_MICROMIPS_TLS_TPREL_HI16", Mips::fixup_MICROMIPS_TLS_TPREL_HI16)
+      .Case("R_MICROMIPS_TLS_TPREL_LO16", Mips::fixup_MICROMIPS_TLS_TPREL_LO16)
+      .Case("R_MIPS_JALR", Mips::fixup_Mips_JALR)
+      .Case("R_MICROMIPS_JALR", Mips::fixup_MICROMIPS_JALR)
       .Default(MCAsmBackend::getFixupKind(Name));
 }
 
@@ -374,7 +414,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GPREL16",      0,     16,   0 },
     { "fixup_Mips_LITERAL",      0,     16,   0 },
     { "fixup_Mips_GOT",          0,     16,   0 },
-    { "fixup_Mips_PC16",         0,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC16",         0,     16,   0 },
     { "fixup_Mips_CALL16",       0,     16,   0 },
     { "fixup_Mips_SHIFT5",       6,      5,   0 },
     { "fixup_Mips_SHIFT6",       6,      5,   0 },
@@ -386,7 +426,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_TLSLDM",       0,     16,   0 },
     { "fixup_Mips_DTPREL_HI",    0,     16,   0 },
     { "fixup_Mips_DTPREL_LO",    0,     16,   0 },
-    { "fixup_Mips_Branch_PCRel", 0,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_Branch_PCRel", 0,     16,   0 },
     { "fixup_Mips_GPOFF_HI",     0,     16,   0 },
     { "fixup_MICROMIPS_GPOFF_HI",0,     16,   0 },
     { "fixup_Mips_GPOFF_LO",     0,     16,   0 },
@@ -402,23 +442,23 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GOT_LO16",     0,     16,   0 },
     { "fixup_Mips_CALL_HI16",    0,     16,   0 },
     { "fixup_Mips_CALL_LO16",    0,     16,   0 },
-    { "fixup_Mips_PC18_S3",      0,     18,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC19_S2",      0,     19,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC21_S2",      0,     21,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC26_S2",      0,     26,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCHI16",       0,     16,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCLO16",       0,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC18_S3",      0,     18,   0 },
+    { "fixup_MIPS_PC19_S2",      0,     19,   0 },
+    { "fixup_MIPS_PC21_S2",      0,     21,   0 },
+    { "fixup_MIPS_PC26_S2",      0,     26,   0 },
+    { "fixup_MIPS_PCHI16",       0,     16,   0 },
+    { "fixup_MIPS_PCLO16",       0,     16,   0 },
     { "fixup_MICROMIPS_26_S1",   0,     26,   0 },
     { "fixup_MICROMIPS_HI16",    0,     16,   0 },
     { "fixup_MICROMIPS_LO16",    0,     16,   0 },
     { "fixup_MICROMIPS_GOT16",   0,     16,   0 },
-    { "fixup_MICROMIPS_PC7_S1",  0,      7,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC10_S1", 0,     10,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC16_S1", 0,     16,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC26_S1", 0,     26,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC19_S2", 0,     19,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC18_S3", 0,     18,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC21_S1", 0,     21,   MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_MICROMIPS_PC7_S1",  0,      7,   0 },
+    { "fixup_MICROMIPS_PC10_S1", 0,     10,   0 },
+    { "fixup_MICROMIPS_PC16_S1", 0,     16,   0 },
+    { "fixup_MICROMIPS_PC26_S1", 0,     26,   0 },
+    { "fixup_MICROMIPS_PC19_S2", 0,     19,   0 },
+    { "fixup_MICROMIPS_PC18_S3", 0,     18,   0 },
+    { "fixup_MICROMIPS_PC21_S1", 0,     21,   0 },
     { "fixup_MICROMIPS_CALL16",  0,     16,   0 },
     { "fixup_MICROMIPS_GOT_DISP",        0,     16,   0 },
     { "fixup_MICROMIPS_GOT_PAGE",        0,     16,   0 },
@@ -460,7 +500,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GPREL16",     16,     16,   0 },
     { "fixup_Mips_LITERAL",     16,     16,   0 },
     { "fixup_Mips_GOT",         16,     16,   0 },
-    { "fixup_Mips_PC16",        16,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC16",        16,     16,   0 },
     { "fixup_Mips_CALL16",      16,     16,   0 },
     { "fixup_Mips_SHIFT5",      21,      5,   0 },
     { "fixup_Mips_SHIFT6",      21,      5,   0 },
@@ -472,11 +512,11 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_TLSLDM",      16,     16,   0 },
     { "fixup_Mips_DTPREL_HI",   16,     16,   0 },
     { "fixup_Mips_DTPREL_LO",   16,     16,   0 },
-    { "fixup_Mips_Branch_PCRel",16,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_Branch_PCRel",16,     16,   0 },
     { "fixup_Mips_GPOFF_HI",    16,     16,   0 },
-    { "fixup_MICROMIPS_GPOFF_HI", 16,     16,   0 },
+    { "fixup_MICROMIPS_GPOFF_HI", 16,   16,   0 },
     { "fixup_Mips_GPOFF_LO",    16,     16,   0 },
-    { "fixup_MICROMIPS_GPOFF_LO", 16,     16,   0 },
+    { "fixup_MICROMIPS_GPOFF_LO", 16,   16,   0 },
     { "fixup_Mips_GOT_PAGE",    16,     16,   0 },
     { "fixup_Mips_GOT_OFST",    16,     16,   0 },
     { "fixup_Mips_GOT_DISP",    16,     16,   0 },
@@ -488,23 +528,23 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GOT_LO16",    16,     16,   0 },
     { "fixup_Mips_CALL_HI16",   16,     16,   0 },
     { "fixup_Mips_CALL_LO16",   16,     16,   0 },
-    { "fixup_Mips_PC18_S3",     14,     18,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC19_S2",     13,     19,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC21_S2",     11,     21,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC26_S2",      6,     26,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCHI16",      16,     16,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCLO16",      16,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC18_S3",     14,     18,   0 },
+    { "fixup_MIPS_PC19_S2",     13,     19,   0 },
+    { "fixup_MIPS_PC21_S2",     11,     21,   0 },
+    { "fixup_MIPS_PC26_S2",      6,     26,   0 },
+    { "fixup_MIPS_PCHI16",      16,     16,   0 },
+    { "fixup_MIPS_PCLO16",      16,     16,   0 },
     { "fixup_MICROMIPS_26_S1",   6,     26,   0 },
     { "fixup_MICROMIPS_HI16",   16,     16,   0 },
     { "fixup_MICROMIPS_LO16",   16,     16,   0 },
     { "fixup_MICROMIPS_GOT16",  16,     16,   0 },
-    { "fixup_MICROMIPS_PC7_S1",  9,      7,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC10_S1", 6,     10,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC16_S1",16,     16,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC26_S1", 6,     26,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC19_S2",13,     19,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC18_S3",14,     18,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC21_S1",11,     21,   MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_MICROMIPS_PC7_S1",  9,      7,   0 },
+    { "fixup_MICROMIPS_PC10_S1", 6,     10,   0 },
+    { "fixup_MICROMIPS_PC16_S1",16,     16,   0 },
+    { "fixup_MICROMIPS_PC26_S1", 6,     26,   0 },
+    { "fixup_MICROMIPS_PC19_S2",13,     19,   0 },
+    { "fixup_MICROMIPS_PC18_S3",14,     18,   0 },
+    { "fixup_MICROMIPS_PC21_S1",11,     21,   0 },
     { "fixup_MICROMIPS_CALL16", 16,     16,   0 },
     { "fixup_MICROMIPS_GOT_DISP",        16,     16,   0 },
     { "fixup_MICROMIPS_GOT_PAGE",        16,     16,   0 },
@@ -526,7 +566,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
                 "Not all MIPS big endian fixup kinds added!");
 
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
 
@@ -556,48 +596,6 @@ bool MipsAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-bool MipsAsmBackend::shouldForceRelocation(const MCFixup &Fixup,
-                                           const MCValue &Target) {
-  const unsigned FixupKind = Fixup.getKind();
-  switch (FixupKind) {
-  default:
-    return false;
-  // All these relocations require special processing
-  // at linking time. Delegate this work to a linker.
-  case Mips::fixup_Mips_CALL_HI16:
-  case Mips::fixup_Mips_CALL_LO16:
-  case Mips::fixup_Mips_CALL16:
-  case Mips::fixup_Mips_GOT:
-  case Mips::fixup_Mips_GOT_PAGE:
-  case Mips::fixup_Mips_GOT_OFST:
-  case Mips::fixup_Mips_GOT_DISP:
-  case Mips::fixup_Mips_GOT_HI16:
-  case Mips::fixup_Mips_GOT_LO16:
-  case Mips::fixup_Mips_GOTTPREL:
-  case Mips::fixup_Mips_DTPREL_HI:
-  case Mips::fixup_Mips_DTPREL_LO:
-  case Mips::fixup_Mips_TLSGD:
-  case Mips::fixup_Mips_TLSLDM:
-  case Mips::fixup_Mips_TPREL_HI:
-  case Mips::fixup_Mips_TPREL_LO:
-  case Mips::fixup_Mips_JALR:
-  case Mips::fixup_MICROMIPS_CALL16:
-  case Mips::fixup_MICROMIPS_GOT_DISP:
-  case Mips::fixup_MICROMIPS_GOT_PAGE:
-  case Mips::fixup_MICROMIPS_GOT_OFST:
-  case Mips::fixup_MICROMIPS_GOT16:
-  case Mips::fixup_MICROMIPS_GOTTPREL:
-  case Mips::fixup_MICROMIPS_TLS_DTPREL_HI16:
-  case Mips::fixup_MICROMIPS_TLS_DTPREL_LO16:
-  case Mips::fixup_MICROMIPS_TLS_GD:
-  case Mips::fixup_MICROMIPS_TLS_LDM:
-  case Mips::fixup_MICROMIPS_TLS_TPREL_HI16:
-  case Mips::fixup_MICROMIPS_TLS_TPREL_LO16:
-  case Mips::fixup_MICROMIPS_JALR:
-    return true;
-  }
-}
-
 namespace {
 
 class WindowsMipsAsmBackend : public MipsAsmBackend {
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 8661339d3b4ab..816626da723af 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -48,9 +48,6 @@ class MipsAsmBackend : public MCAsmBackend {
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
-
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
 }; // class MipsAsmBackend
 
 } // namespace
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index 35d4e0db35c31..a8369f2b28fb7 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -55,6 +55,30 @@ MCCodeEmitter *createMipsMCCodeEmitterEL(const MCInstrInfo &MCII,
 
 } // end namespace llvm
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case Mips::fixup_Mips_PC16:
+  case Mips::fixup_Mips_Branch_PCRel:
+  case Mips::fixup_MIPS_PC18_S3:
+  case Mips::fixup_MIPS_PC19_S2:
+  case Mips::fixup_MIPS_PC21_S2:
+  case Mips::fixup_MIPS_PC26_S2:
+  case Mips::fixup_MIPS_PCHI16:
+  case Mips::fixup_MIPS_PCLO16:
+  case Mips::fixup_MICROMIPS_PC7_S1:
+  case Mips::fixup_MICROMIPS_PC10_S1:
+  case Mips::fixup_MICROMIPS_PC16_S1:
+  case Mips::fixup_MICROMIPS_PC26_S1:
+  case Mips::fixup_MICROMIPS_PC19_S2:
+  case Mips::fixup_MICROMIPS_PC18_S3:
+  case Mips::fixup_MICROMIPS_PC21_S1:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // If the D<shift> instruction has a shift amount that is greater
 // than 31 (checked in calling routine), lower it to a D<shift>32 instruction
 static void LowerLargeShift(MCInst& Inst) {
@@ -236,8 +260,7 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -258,8 +281,7 @@ getBranchTargetOpValue1SImm16(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -281,8 +303,7 @@ getBranchTargetOpValueMMR6(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-2, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -304,8 +325,7 @@ getBranchTargetOpValueLsl2MMR6(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -325,8 +345,7 @@ getBranchTarget7OpValueMM(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValueMM expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_PC7_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_PC7_S1);
   return 0;
 }
 
@@ -346,8 +365,7 @@ getBranchTargetOpValueMMPC10(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValuePC10 expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                   MCFixupKind(Mips::fixup_MICROMIPS_PC10_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_PC10_S1);
   return 0;
 }
 
@@ -367,9 +385,7 @@ getBranchTargetOpValueMM(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValueMM expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                   MCFixupKind(Mips::
-                               fixup_MICROMIPS_PC16_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_PC16_S1);
   return 0;
 }
 
@@ -390,8 +406,7 @@ getBranchTarget21OpValue(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MIPS_PC21_S2)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MIPS_PC21_S2);
   return 0;
 }
 
@@ -412,8 +427,7 @@ getBranchTarget21OpValueMM(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_PC21_S1)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MICROMIPS_PC21_S1);
   return 0;
 }
 
@@ -434,8 +448,7 @@ getBranchTarget26OpValue(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MIPS_PC26_S2)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MIPS_PC26_S2);
   return 0;
 }
 
@@ -456,8 +469,7 @@ unsigned MipsMCCodeEmitter::getBranchTarget26OpValueMM(
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_PC26_S1)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MICROMIPS_PC26_S1);
   return 0;
 }
 
@@ -478,7 +490,7 @@ getJumpOffset16OpValue(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   Mips::Fixups FixupKind =
       isMicroMips(STI) ? Mips::fixup_MICROMIPS_LO16 : Mips::fixup_Mips_LO16;
-  Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
+  addFixup(Fixups, 0, Expr, MCFixupKind(FixupKind));
   return 0;
 }
 
@@ -497,8 +509,7 @@ getJumpTargetOpValue(const MCInst &MI, unsigned OpNo,
          "getJumpTargetOpValue expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                                   MCFixupKind(Mips::fixup_Mips_26)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_Mips_26);
   return 0;
 }
 
@@ -514,8 +525,7 @@ getJumpTargetOpValueMM(const MCInst &MI, unsigned OpNo,
          "getJumpTargetOpValueMM expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_26_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_26_S1);
   return 0;
 }
 
@@ -693,7 +703,7 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
           isMicroMips(STI) ? Mips::fixup_MICROMIPS_SUB : Mips::fixup_Mips_SUB;
       break;
     }
-    Fixups.push_back(MCFixup::create(0, MipsExpr, MCFixupKind(FixupKind)));
+    addFixup(Fixups, 0, MipsExpr, MCFixupKind(FixupKind));
     return 0;
   }
 
@@ -734,8 +744,7 @@ unsigned MipsMCCodeEmitter::getImmOpValue(const MCInst &MI, const MCOperand &MO,
     return Res;
   unsigned MIFrm = MipsII::getFormat(MCII.get(MI.getOpcode()).TSFlags);
   if (!isa<MCSpecifierExpr>(Expr) && MIFrm == MipsII::FrmI) {
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Mips::fixup_Mips_AnyImm16), Expr->getLoc()));
+    addFixup(Fixups, 0, Expr, Mips::fixup_Mips_AnyImm16);
     return 0;
   }
   return getExprOpValue(Expr, Fixups, STI);
@@ -964,7 +973,7 @@ MipsMCCodeEmitter::getSimm19Lsl2Encoding(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   Mips::Fixups FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_PC19_S2
                                             : Mips::fixup_MIPS_PC19_S2;
-  Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
+  addFixup(Fixups, 0, Expr, MCFixupKind(FixupKind));
   return 0;
 }
 
@@ -986,7 +995,7 @@ MipsMCCodeEmitter::getSimm18Lsl3Encoding(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   Mips::Fixups FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_PC18_S3
                                             : Mips::fixup_MIPS_PC18_S3;
-  Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
+  addFixup(Fixups, 0, Expr, MCFixupKind(FixupKind));
   return 0;
 }
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
index 29f61ed9b2b83..ab1eda0f48e15 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
@@ -174,7 +174,7 @@ static MCAsmInfo *createMipsMCAsmInfo(const MCRegisterInfo &MRI,
                                       const MCTargetOptions &Options) {
   MCAsmInfo *MAI;
 
-  if (TT.isOSWindows())
+  if (TT.isOSBinFormatCOFF())
     MAI = new MipsCOFFMCAsmInfo();
   else
     MAI = new MipsELFMCAsmInfo(TT, Options);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index 6097ad8017846..c69fc68ab5af1 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -1034,43 +1034,43 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() {
 
 void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_GPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_GPREL32));
   DF->appendContents(4, 0);
 }
 
 void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_GPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_GPREL32));
   DF->appendContents(8, 0);
 }
 
 void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_DTPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_DTPREL32));
   DF->appendContents(4, 0);
 }
 
 void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_DTPREL64)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_DTPREL64));
   DF->appendContents(8, 0);
 }
 
 void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_TPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_TPREL32));
   DF->appendContents(4, 0);
 }
 
 void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_TPREL64)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_TPREL64));
   DF->appendContents(8, 0);
 }
 
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 9b4b336727235..74a256e9729b3 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -373,6 +373,8 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
     setOperationAction(ISD::IS_FPCLASS, MVT::f32, Legal);
     setOperationAction(ISD::IS_FPCLASS, MVT::f64, Legal);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
   } else {
     setOperationAction(ISD::FCANONICALIZE, MVT::f32, Custom);
     setOperationAction(ISD::FCANONICALIZE, MVT::f64, Custom);
@@ -2698,9 +2700,6 @@ lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue MipsTargetLowering::lowerRETURNADDR(SDValue Op,
                                             SelectionDAG &DAG) const {
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   // check the depth
   if (Op.getConstantOperandVal(0) != 0) {
     DAG.getContext()->emitError(
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index d9192fbfceff1..bb0aeb493ed48 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -28,6 +28,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
@@ -390,35 +391,27 @@ static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
 /// and promote them to a larger size if they're not.
 ///
 /// The promoted type is placed in \p PromoteVT if the function returns true.
-static std::optional<MVT> PromoteScalarIntegerPTX(const EVT &VT) {
+static EVT promoteScalarIntegerPTX(const EVT VT) {
   if (VT.isScalarInteger()) {
-    MVT PromotedVT;
     switch (PowerOf2Ceil(VT.getFixedSizeInBits())) {
     default:
       llvm_unreachable(
           "Promotion is not suitable for scalars of size larger than 64-bits");
     case 1:
-      PromotedVT = MVT::i1;
-      break;
+      return MVT::i1;
     case 2:
     case 4:
     case 8:
-      PromotedVT = MVT::i8;
-      break;
+      return MVT::i8;
     case 16:
-      PromotedVT = MVT::i16;
-      break;
+      return MVT::i16;
     case 32:
-      PromotedVT = MVT::i32;
-      break;
+      return MVT::i32;
     case 64:
-      PromotedVT = MVT::i64;
-      break;
+      return MVT::i64;
     }
-    if (VT != PromotedVT)
-      return PromotedVT;
   }
-  return std::nullopt;
+  return VT;
 }
 
 // Check whether we can merge loads/stores of some of the pieces of a
@@ -1053,10 +1046,8 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     break;
 
     MAKE_CASE(NVPTXISD::RET_GLUE)
-    MAKE_CASE(NVPTXISD::DeclareParam)
+    MAKE_CASE(NVPTXISD::DeclareArrayParam)
     MAKE_CASE(NVPTXISD::DeclareScalarParam)
-    MAKE_CASE(NVPTXISD::DeclareRet)
-    MAKE_CASE(NVPTXISD::DeclareRetParam)
     MAKE_CASE(NVPTXISD::CALL)
     MAKE_CASE(NVPTXISD::LoadParam)
     MAKE_CASE(NVPTXISD::LoadParamV2)
@@ -1162,8 +1153,8 @@ SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
 }
 
 std::string NVPTXTargetLowering::getPrototype(
-    const DataLayout &DL, Type *retTy, const ArgListTy &Args,
-    const SmallVectorImpl<ISD::OutputArg> &Outs, MaybeAlign RetAlign,
+    const DataLayout &DL, Type *RetTy, const ArgListTy &Args,
+    const SmallVectorImpl<ISD::OutputArg> &Outs,
     std::optional<unsigned> FirstVAArg, const CallBase &CB,
     unsigned UniqueCallSite) const {
   auto PtrVT = getPointerTy(DL);
@@ -1172,22 +1163,22 @@ std::string NVPTXTargetLowering::getPrototype(
   raw_string_ostream O(Prototype);
   O << "prototype_" << UniqueCallSite << " : .callprototype ";
 
-  if (retTy->isVoidTy()) {
+  if (RetTy->isVoidTy()) {
     O << "()";
   } else {
     O << "(";
-    if (shouldPassAsArray(retTy)) {
-      assert(RetAlign && "RetAlign must be set for non-void return types");
-      O << ".param .align " << RetAlign->value() << " .b8 _["
-        << DL.getTypeAllocSize(retTy) << "]";
-    } else if (retTy->isFloatingPointTy() || retTy->isIntegerTy()) {
+    if (shouldPassAsArray(RetTy)) {
+      const Align RetAlign = getArgumentAlignment(&CB, RetTy, 0, DL);
+      O << ".param .align " << RetAlign.value() << " .b8 _["
+        << DL.getTypeAllocSize(RetTy) << "]";
+    } else if (RetTy->isFloatingPointTy() || RetTy->isIntegerTy()) {
       unsigned size = 0;
-      if (auto *ITy = dyn_cast<IntegerType>(retTy)) {
+      if (auto *ITy = dyn_cast<IntegerType>(RetTy)) {
         size = ITy->getBitWidth();
       } else {
-        assert(retTy->isFloatingPointTy() &&
+        assert(RetTy->isFloatingPointTy() &&
                "Floating point type expected here");
-        size = retTy->getPrimitiveSizeInBits();
+        size = RetTy->getPrimitiveSizeInBits();
       }
       // PTX ABI requires all scalar return values to be at least 32
       // bits in size.  fp16 normally uses .b16 as its storage type in
@@ -1195,7 +1186,7 @@ std::string NVPTXTargetLowering::getPrototype(
       size = promoteScalarArgumentSize(size);
 
       O << ".param .b" << size << " _";
-    } else if (isa<PointerType>(retTy)) {
+    } else if (isa<PointerType>(RetTy)) {
       O << ".param .b" << PtrVT.getSizeInBits() << " _";
     } else {
       llvm_unreachable("Unknown return type");
@@ -1256,7 +1247,7 @@ std::string NVPTXTargetLowering::getPrototype(
 
   if (FirstVAArg)
     O << (first ? "" : ",") << " .param .align "
-      << STI.getMaxRequiredAlignment() << " .b8 _[]\n";
+      << STI.getMaxRequiredAlignment() << " .b8 _[]";
   O << ")";
   if (shouldEmitPTXNoReturn(&CB, *nvTM))
     O << " .noreturn";
@@ -1442,6 +1433,21 @@ static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags) {
   return ISD::ANY_EXTEND;
 }
 
+static SDValue correctParamType(SDValue V, EVT ExpectedVT,
+                                ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
+                                SDLoc dl) {
+  const EVT ActualVT = V.getValueType();
+  assert((ActualVT == ExpectedVT ||
+          (ExpectedVT.isInteger() && ActualVT.isInteger())) &&
+         "Non-integer argument type size mismatch");
+  if (ExpectedVT.bitsGT(ActualVT))
+    return DAG.getNode(getExtOpcode(Flags), dl, ExpectedVT, V);
+  if (ExpectedVT.bitsLT(ActualVT))
+    return DAG.getNode(ISD::TRUNCATE, dl, ExpectedVT, V);
+
+  return V;
+}
+
 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                        SmallVectorImpl<SDValue> &InVals) const {
 
@@ -1505,9 +1511,11 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
          "Outs and OutVals must be the same size");
   // Declare the .params or .reg need to pass values
   // to the function
-  for (const auto [ArgI, Arg] : llvm::enumerate(Args)) {
-    const auto ArgOuts = AllOuts.take_while(
-        [ArgI = ArgI](auto O) { return O.OrigArgIndex == ArgI; });
+  for (const auto E : llvm::enumerate(Args)) {
+    const auto ArgI = E.index();
+    const auto Arg = E.value();
+    const auto ArgOuts =
+        AllOuts.take_while([&](auto O) { return O.OrigArgIndex == ArgI; });
     const auto ArgOutVals = AllOutVals.take_front(ArgOuts.size());
     AllOuts = AllOuts.drop_front(ArgOuts.size());
     AllOutVals = AllOutVals.drop_front(ArgOuts.size());
@@ -1515,6 +1523,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     const bool IsVAArg = (ArgI >= FirstVAArg);
     const bool IsByVal = Arg.IsByVal;
 
+    const SDValue ParamSymbol =
+        getCallParamSymbol(DAG, IsVAArg ? FirstVAArg : ArgI, MVT::i32);
+
     SmallVector<EVT, 16> VTs;
     SmallVector<uint64_t, 16> Offsets;
 
@@ -1525,38 +1536,43 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     assert(VTs.size() == Offsets.size() && "Size mismatch");
     assert((IsByVal || VTs.size() == ArgOuts.size()) && "Size mismatch");
 
-    Align ArgAlign;
-    if (IsByVal) {
-      // The ByValAlign in the Outs[OIdx].Flags is always set at this point,
-      // so we don't need to worry whether it's naturally aligned or not.
-      // See TargetLowering::LowerCallTo().
-      Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
-      ArgAlign = getFunctionByValParamAlign(CB->getCalledFunction(), ETy,
-                                            InitialAlign, DL);
-      if (IsVAArg)
-        VAOffset = alignTo(VAOffset, ArgAlign);
-    } else {
-      ArgAlign = getArgumentAlignment(CB, Arg.Ty, ArgI + 1, DL);
-    }
+    const Align ArgAlign = [&]() {
+      if (IsByVal) {
+        // The ByValAlign in the Outs[OIdx].Flags is always set at this point,
+        // so we don't need to worry whether it's naturally aligned or not.
+        // See TargetLowering::LowerCallTo().
+        const Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
+        const Align ByValAlign = getFunctionByValParamAlign(
+            CB->getCalledFunction(), ETy, InitialAlign, DL);
+        if (IsVAArg)
+          VAOffset = alignTo(VAOffset, ByValAlign);
+        return ByValAlign;
+      }
+      return getArgumentAlignment(CB, Arg.Ty, ArgI + 1, DL);
+    }();
 
     const unsigned TypeSize = DL.getTypeAllocSize(ETy);
     assert((!IsByVal || TypeSize == ArgOuts[0].Flags.getByValSize()) &&
            "type size mismatch");
 
-    const bool PassAsArray = IsByVal || shouldPassAsArray(Arg.Ty);
-    if (IsVAArg) {
-      if (ArgI == FirstVAArg) {
-        VADeclareParam = Chain =
-            DAG.getNode(NVPTXISD::DeclareParam, dl, {MVT::Other, MVT::Glue},
-                        {Chain, GetI32(STI.getMaxRequiredAlignment()),
-                         GetI32(ArgI), GetI32(1), InGlue});
+    const std::optional<SDValue> ArgDeclare = [&]() -> std::optional<SDValue> {
+      if (IsVAArg) {
+        if (ArgI == FirstVAArg) {
+          VADeclareParam = DAG.getNode(
+              NVPTXISD::DeclareArrayParam, dl, {MVT::Other, MVT::Glue},
+              {Chain, ParamSymbol, GetI32(STI.getMaxRequiredAlignment()),
+               GetI32(0), InGlue});
+          return VADeclareParam;
+        }
+        return std::nullopt;
+      }
+      if (IsByVal || shouldPassAsArray(Arg.Ty)) {
+        // declare .param .align <align> .b8 .param<n>[<size>];
+        return DAG.getNode(NVPTXISD::DeclareArrayParam, dl,
+                           {MVT::Other, MVT::Glue},
+                           {Chain, ParamSymbol, GetI32(ArgAlign.value()),
+                            GetI32(TypeSize), InGlue});
       }
-    } else if (PassAsArray) {
-      // declare .param .align <align> .b8 .param<n>[<size>];
-      Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, {MVT::Other, MVT::Glue},
-                          {Chain, GetI32(ArgAlign.value()), GetI32(ArgI),
-                           GetI32(TypeSize), InGlue});
-    } else {
       assert(ArgOuts.size() == 1 && "We must pass only one value as non-array");
       // declare .param .b<size> .param<n>;
 
@@ -1568,11 +1584,14 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
               ? promoteScalarArgumentSize(TypeSize * 8)
               : TypeSize * 8;
 
-      Chain =
-          DAG.getNode(NVPTXISD::DeclareScalarParam, dl, {MVT::Other, MVT::Glue},
-                      {Chain, GetI32(ArgI), GetI32(PromotedSize), InGlue});
+      return DAG.getNode(NVPTXISD::DeclareScalarParam, dl,
+                         {MVT::Other, MVT::Glue},
+                         {Chain, ParamSymbol, GetI32(PromotedSize), InGlue});
+    }();
+    if (ArgDeclare) {
+      Chain = ArgDeclare->getValue(0);
+      InGlue = ArgDeclare->getValue(1);
     }
-    InGlue = Chain.getValue(1);
 
     // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
     // than 32-bits are sign extended or zero extended, depending on
@@ -1594,8 +1613,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       } else {
         StVal = ArgOutVals[I];
 
-        if (auto PromotedVT = PromoteScalarIntegerPTX(StVal.getValueType())) {
-          StVal = DAG.getNode(getExtOpcode(ArgOuts[I].Flags), dl, *PromotedVT,
+        auto PromotedVT = promoteScalarIntegerPTX(StVal.getValueType());
+        if (PromotedVT != StVal.getValueType()) {
+          StVal = DAG.getNode(getExtOpcode(ArgOuts[I].Flags), dl, PromotedVT,
                               StVal);
         }
       }
@@ -1619,12 +1639,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     unsigned J = 0;
     for (const unsigned NumElts : VectorInfo) {
       const int CurOffset = Offsets[J];
-      EVT EltVT = VTs[J];
+      EVT EltVT = promoteScalarIntegerPTX(VTs[J]);
       const Align PartAlign = commonAlignment(ArgAlign, CurOffset);
 
-      if (auto PromotedVT = PromoteScalarIntegerPTX(EltVT))
-        EltVT = *PromotedVT;
-
       // If we have a PVF_SCALAR entry, it may not be sufficiently aligned for a
       // scalar store. In such cases, fall back to byte stores.
       if (NumElts == 1 && !IsVAArg && PartAlign < DAG.getEVTAlign(EltVT)) {
@@ -1695,27 +1712,26 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
-  MaybeAlign RetAlign = std::nullopt;
 
   // Handle Result
   if (!Ins.empty()) {
-    RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
-
-    // Declare
-    //  .param .align N .b8 retval0[<size-in-bytes>], or
-    //  .param .b<size-in-bits> retval0
-    const unsigned ResultSize = DL.getTypeAllocSizeInBits(RetTy);
-    if (!shouldPassAsArray(RetTy)) {
-      const unsigned PromotedResultSize = promoteScalarArgumentSize(ResultSize);
-      Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, {MVT::Other, MVT::Glue},
-                          {Chain, GetI32(PromotedResultSize), InGlue});
-      InGlue = Chain.getValue(1);
-    } else {
-      Chain = DAG.getNode(
-          NVPTXISD::DeclareRetParam, dl, {MVT::Other, MVT::Glue},
-          {Chain, GetI32(RetAlign->value()), GetI32(ResultSize / 8), InGlue});
-      InGlue = Chain.getValue(1);
-    }
+    const SDValue RetDeclare = [&]() {
+      const SDValue RetSymbol = DAG.getExternalSymbol("retval0", MVT::i32);
+      const unsigned ResultSize = DL.getTypeAllocSizeInBits(RetTy);
+      if (shouldPassAsArray(RetTy)) {
+        const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
+        return DAG.getNode(NVPTXISD::DeclareArrayParam, dl,
+                           {MVT::Other, MVT::Glue},
+                           {Chain, RetSymbol, GetI32(RetAlign.value()),
+                            GetI32(ResultSize / 8), InGlue});
+      }
+      const auto PromotedResultSize = promoteScalarArgumentSize(ResultSize);
+      return DAG.getNode(
+          NVPTXISD::DeclareScalarParam, dl, {MVT::Other, MVT::Glue},
+          {Chain, RetSymbol, GetI32(PromotedResultSize), InGlue});
+    }();
+    Chain = RetDeclare.getValue(0);
+    InGlue = RetDeclare.getValue(1);
   }
 
   const bool HasVAArgs = CLI.IsVarArg && (CLI.Args.size() > CLI.NumFixedArgs);
@@ -1760,7 +1776,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // The prototype is embedded in a string and put as the operand for a
     // CallPrototype SDNode which will print out to the value of the string.
     std::string Proto =
-        getPrototype(DL, RetTy, Args, CLI.Outs, RetAlign,
+        getPrototype(DL, RetTy, Args, CLI.Outs,
                      HasVAArgs ? std::optional(FirstVAArg) : std::nullopt, *CB,
                      UniqueCallSite);
     const char *ProtoStr = nvTM->getStrPool().save(Proto).data();
@@ -1773,11 +1789,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   if (ConvertToIndirectCall) {
     // Copy the function ptr to a ptx register and use the register to call the
     // function.
-    EVT DestVT = Callee.getValueType();
-    MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
+    const MVT DestVT = Callee.getValueType().getSimpleVT();
+    MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-    unsigned DestReg =
-        RegInfo.createVirtualRegister(TLI.getRegClassFor(DestVT.getSimpleVT()));
+    Register DestReg = MRI.createVirtualRegister(TLI.getRegClassFor(DestVT));
     auto RegCopy = DAG.getCopyToReg(DAG.getEntryNode(), dl, DestReg, Callee);
     Callee = DAG.getCopyFromReg(RegCopy, dl, DestReg, DestVT);
   }
@@ -1810,7 +1825,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets, 0);
     assert(VTs.size() == Ins.size() && "Bad value decomposition");
 
-    assert(RetAlign && "RetAlign is guaranteed to be set");
+    const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
 
     // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
     // 32-bits are sign extended or zero extended, depending on whether
@@ -1818,17 +1833,15 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     const bool ExtendIntegerRetVal =
         RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
 
-    const auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, *RetAlign);
+    const auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign);
     unsigned I = 0;
     for (const unsigned VectorizedSize : VectorInfo) {
-      EVT TheLoadType = VTs[I];
+      EVT TheLoadType = promoteScalarIntegerPTX(VTs[I]);
       EVT EltType = Ins[I].VT;
-      const Align EltAlign = commonAlignment(*RetAlign, Offsets[I]);
+      const Align EltAlign = commonAlignment(RetAlign, Offsets[I]);
 
-      if (auto PromotedVT = PromoteScalarIntegerPTX(TheLoadType)) {
-        TheLoadType = *PromotedVT;
-        EltType = *PromotedVT;
-      }
+      if (TheLoadType != VTs[I])
+        EltType = TheLoadType;
 
       if (ExtendIntegerRetVal) {
         TheLoadType = MVT::i32;
@@ -1898,13 +1911,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       continue;
     }
 
-    SDValue Ret = DAG.getNode(
-        NVPTXISD::ProxyReg, dl,
-        {ProxyRegOps[I].getSimpleValueType(), MVT::Other, MVT::Glue},
-        {Chain, ProxyRegOps[I], InGlue});
-
-    Chain = Ret.getValue(1);
-    InGlue = Ret.getValue(2);
+    SDValue Ret =
+        DAG.getNode(NVPTXISD::ProxyReg, dl, ProxyRegOps[I].getSimpleValueType(),
+                    {Chain, ProxyRegOps[I]});
 
     const EVT ExpectedVT = Ins[I].VT;
     if (!Ret.getValueType().bitsEq(ExpectedVT)) {
@@ -1914,14 +1923,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   for (SDValue &T : TempProxyRegOps) {
-    SDValue Repl = DAG.getNode(NVPTXISD::ProxyReg, dl,
-                               {T.getSimpleValueType(), MVT::Other, MVT::Glue},
-                               {Chain, T.getOperand(0), InGlue});
+    SDValue Repl = DAG.getNode(NVPTXISD::ProxyReg, dl, T.getSimpleValueType(),
+                               {Chain, T.getOperand(0)});
     DAG.ReplaceAllUsesWith(T, Repl);
     DAG.RemoveDeadNode(T.getNode());
-
-    Chain = Repl.getValue(1);
-    InGlue = Repl.getValue(2);
   }
 
   // set isTailCall to false for now, until we figure out how to express
@@ -2786,15 +2791,16 @@ static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG,
   SDValue X = Op->getOperand(0);
   SDValue Y = Op->getOperand(1);
   EVT Ty = Op.getValueType();
+  SDNodeFlags Flags = Op->getFlags();
 
-  SDValue Div = DAG.getNode(ISD::FDIV, DL, Ty, X, Y);
-  SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, Ty, Div);
-  SDValue Mul =
-      DAG.getNode(ISD::FMUL, DL, Ty, Trunc, Y, SDNodeFlags::AllowContract);
-  SDValue Sub =
-      DAG.getNode(ISD::FSUB, DL, Ty, X, Mul, SDNodeFlags::AllowContract);
+  SDValue Div = DAG.getNode(ISD::FDIV, DL, Ty, X, Y, Flags);
+  SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, Ty, Div, Flags);
+  SDValue Mul = DAG.getNode(ISD::FMUL, DL, Ty, Trunc, Y,
+                            Flags | SDNodeFlags::AllowContract);
+  SDValue Sub = DAG.getNode(ISD::FSUB, DL, Ty, X, Mul,
+                            Flags | SDNodeFlags::AllowContract);
 
-  if (AllowUnsafeFPMath || Op->getFlags().hasNoInfs())
+  if (AllowUnsafeFPMath || Flags.hasNoInfs())
     return Sub;
 
   // If Y is infinite, return X
@@ -3292,11 +3298,17 @@ bool NVPTXTargetLowering::splitValueIntoRegisterParts(
 // Name of the symbol is composed from its index and the function name.
 // Negative index corresponds to special parameter (unsized array) used for
 // passing variable arguments.
-SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx,
-                                            EVT v) const {
+SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int I,
+                                            EVT T) const {
   StringRef SavedStr = nvTM->getStrPool().save(
-      getParamName(&DAG.getMachineFunction().getFunction(), idx));
-  return DAG.getExternalSymbol(SavedStr.data(), v);
+      getParamName(&DAG.getMachineFunction().getFunction(), I));
+  return DAG.getExternalSymbol(SavedStr.data(), T);
+}
+
+SDValue NVPTXTargetLowering::getCallParamSymbol(SelectionDAG &DAG, int I,
+                                                EVT T) const {
+  const StringRef SavedStr = nvTM->getStrPool().save("param" + Twine(I));
+  return DAG.getExternalSymbol(SavedStr.data(), T);
 }
 
 SDValue NVPTXTargetLowering::LowerFormalArguments(
@@ -3393,8 +3405,11 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         const unsigned PackingAmt =
             LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
 
-        const EVT VecVT = EVT::getVectorVT(
-            F->getContext(), LoadVT.getScalarType(), NumElts * PackingAmt);
+        const EVT VecVT =
+            NumElts == 1
+                ? LoadVT
+                : EVT::getVectorVT(F->getContext(), LoadVT.getScalarType(),
+                                   NumElts * PackingAmt);
 
         SDValue VecAddr = DAG.getObjectPtrOffset(
             dl, ArgSymbol, TypeSize::getFixed(Offsets[I]));
@@ -3408,22 +3423,16 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         if (P.getNode())
           P.getNode()->setIROrder(Arg.getArgNo() + 1);
         for (const unsigned J : llvm::seq(NumElts)) {
-          SDValue Elt = DAG.getNode(
-              LoadVT.isVector() ? ISD::EXTRACT_SUBVECTOR
-                                : ISD::EXTRACT_VECTOR_ELT,
-              dl, LoadVT, P, DAG.getVectorIdxConstant(J * PackingAmt, dl));
-
-          // Extend or truncate the element if necessary (e.g. an i8 is loaded
-          // into an i16 register)
-          const EVT ExpectedVT = ArgIns[I + J].VT;
-          assert((Elt.getValueType() == ExpectedVT ||
-                  (ExpectedVT.isInteger() && Elt.getValueType().isInteger())) &&
-                 "Non-integer argument type size mismatch");
-          if (ExpectedVT.bitsGT(Elt.getValueType()))
-            Elt = DAG.getNode(getExtOpcode(ArgIns[I + J].Flags), dl, ExpectedVT,
-                              Elt);
-          else if (ExpectedVT.bitsLT(Elt.getValueType()))
-            Elt = DAG.getNode(ISD::TRUNCATE, dl, ExpectedVT, Elt);
+          SDValue Elt =
+              NumElts == 1
+                  ? P
+                  : DAG.getNode(LoadVT.isVector() ? ISD::EXTRACT_SUBVECTOR
+                                                  : ISD::EXTRACT_VECTOR_ELT,
+                                dl, LoadVT, P,
+                                DAG.getVectorIdxConstant(J * PackingAmt, dl));
+
+          Elt = correctParamType(Elt, ArgIns[I + J].VT, ArgIns[I + J].Flags,
+                                 DAG, dl);
           InVals.push_back(Elt);
         }
         I += NumElts;
@@ -3466,25 +3475,14 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
 
   const auto GetRetVal = [&](unsigned I) -> SDValue {
     SDValue RetVal = OutVals[I];
-    assert(!PromoteScalarIntegerPTX(RetVal.getValueType()) &&
+    assert(promoteScalarIntegerPTX(RetVal.getValueType()) ==
+               RetVal.getValueType() &&
            "OutVal type should always be legal");
 
-    EVT VTI = VTs[I];
-    if (const auto PromotedVT = PromoteScalarIntegerPTX(VTI))
-      VTI = *PromotedVT;
-
+    const EVT VTI = promoteScalarIntegerPTX(VTs[I]);
     const EVT StoreVT =
         ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
-
-    assert((RetVal.getValueType() == StoreVT ||
-            (StoreVT.isInteger() && RetVal.getValueType().isInteger())) &&
-           "Non-integer argument type size mismatch");
-    if (StoreVT.bitsGT(RetVal.getValueType())) {
-      RetVal = DAG.getNode(getExtOpcode(Outs[I].Flags), dl, StoreVT, RetVal);
-    } else if (StoreVT.bitsLT(RetVal.getValueType())) {
-      RetVal = DAG.getNode(ISD::TRUNCATE, dl, StoreVT, RetVal);
-    }
-    return RetVal;
+    return correctParamType(RetVal, StoreVT, Outs[I].Flags, DAG, dl);
   };
 
   const auto RetAlign = getFunctionParamOptimizedAlign(&F, RetTy, DL);
@@ -3499,7 +3497,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     if (NumElts == 1) {
       Val = GetRetVal(I);
     } else {
-      SmallVector<SDValue, 6> StoreVals;
+      SmallVector<SDValue, 4> StoreVals;
       for (const unsigned J : llvm::seq(NumElts)) {
         SDValue ValJ = GetRetVal(I + J);
         if (ValJ.getValueType().isVector())
@@ -3513,7 +3511,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
       Val = DAG.getBuildVector(VT, dl, StoreVals);
     }
 
-    SDValue RetSymbol = DAG.getExternalSymbol("func_retval0", MVT::i32);
+    const SDValue RetSymbol = DAG.getExternalSymbol("func_retval0", MVT::i32);
     SDValue Ptr =
         DAG.getObjectPtrOffset(dl, RetSymbol, TypeSize::getFixed(Offsets[I]));
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 3a8091fecfde1..2477e1fb61595 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -25,10 +25,15 @@ enum NodeType : unsigned {
   // Start the numbering from where ISD NodeType finishes.
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   RET_GLUE,
-  DeclareParam,
+
+  /// These nodes represent a parameter declaration. In PTX this will look like:
+  ///   .param .align 16 .b8 param0[1024];
+  ///   .param .b32 retval0;
+  ///
+  /// DeclareArrayParam(Chain, Externalsym, Align, Size, Glue)
+  /// DeclareScalarParam(Chain, Externalsym, Size, Glue)
   DeclareScalarParam,
-  DeclareRetParam,
-  DeclareRet,
+  DeclareArrayParam,
 
   /// This node represents a PTX call instruction. It's operands are as follows:
   ///
@@ -174,7 +179,6 @@ class NVPTXTargetLowering : public TargetLowering {
 
   std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
                            const SmallVectorImpl<ISD::OutputArg> &,
-                           MaybeAlign RetAlign,
                            std::optional<unsigned> FirstVAArg,
                            const CallBase &CB, unsigned UniqueCallSite) const;
 
@@ -272,8 +276,8 @@ class NVPTXTargetLowering : public TargetLowering {
   const NVPTXSubtarget &STI; // cache the subtarget here
   mutable unsigned GlobalUniqueCallSite;
 
-  SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
-
+  SDValue getParamSymbol(SelectionDAG &DAG, int I, EVT T) const;
+  SDValue getCallParamSymbol(SelectionDAG &DAG, int I, EVT T) const;
   SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 1a2515b7f66f3..441ddeeb7d667 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -1990,9 +1990,9 @@ defm FSetNE : FSET_FORMAT<setne, CmpNE, CmpNE_FTZ>;
 defm FSetNUM : FSET_FORMAT<seto, CmpNUM, CmpNUM_FTZ>;
 defm FSetNAN : FSET_FORMAT<setuo, CmpNAN, CmpNAN_FTZ>;
 
-def SDTDeclareParamProfile :
+def SDTDeclareArrayParam :
   SDTypeProfile<0, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i32>]>;
-def SDTDeclareScalarParamProfile :
+def SDTDeclareScalarParam :
   SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
 def SDTLoadParamProfile : SDTypeProfile<1, 2, [SDTCisInt<1>, SDTCisInt<2>]>;
 def SDTLoadParamV2Profile : SDTypeProfile<2, 2, [SDTCisSameAs<0, 1>, SDTCisInt<2>, SDTCisInt<3>]>;
@@ -2001,22 +2001,17 @@ def SDTStoreParamProfile : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>]>;
 def SDTStoreParamV2Profile : SDTypeProfile<0, 4, [SDTCisInt<0>, SDTCisInt<1>]>;
 def SDTStoreParamV4Profile : SDTypeProfile<0, 6, [SDTCisInt<0>, SDTCisInt<1>]>;
 def SDTMoveParamProfile : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisSameAs<0, 1>]>;
-def SDTProxyRegProfile : SDTypeProfile<1, 1, []>;
 
-def DeclareParam :
-  SDNode<"NVPTXISD::DeclareParam", SDTDeclareParamProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def DeclareScalarParam :
-  SDNode<"NVPTXISD::DeclareScalarParam", SDTDeclareScalarParamProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def DeclareRetParam :
-  SDNode<"NVPTXISD::DeclareRetParam",
-         SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>,
+def SDTProxyReg : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>]>;
+
+
+def declare_array_param :
+  SDNode<"NVPTXISD::DeclareArrayParam", SDTDeclareArrayParam,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def DeclareRet :
-  SDNode<"NVPTXISD::DeclareRet",
-         SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>,
+def declare_scalar_param :
+  SDNode<"NVPTXISD::DeclareScalarParam", SDTDeclareScalarParam,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
+
 def LoadParam :
   SDNode<"NVPTXISD::LoadParam", SDTLoadParamProfile,
          [SDNPHasChain, SDNPMayLoad, SDNPOutGlue, SDNPInGlue]>;
@@ -2037,9 +2032,8 @@ def StoreParamV4 :
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
 def MoveParam :
   SDNode<"NVPTXISD::MoveParam", SDTMoveParamProfile, []>;
-def ProxyReg :
-  SDNode<"NVPTXISD::ProxyReg", SDTProxyRegProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
+def proxy_reg :
+  SDNode<"NVPTXISD::ProxyReg", SDTProxyReg, [SDNPHasChain]>;
 
   /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
   ///      NumParams, Callee, Proto, InGlue)
@@ -2188,23 +2182,17 @@ defm StoreParamV2F64  : StoreParamV2Inst<B64, f64imm, ".b64">;
 
 defm StoreParamV4F32  : StoreParamV4Inst<B32, f32imm, ".b32">;
 
-def DeclareRetMemInst :
-  NVPTXInst<(outs), (ins i32imm:$align, i32imm:$size),
-            ".param .align $align .b8 retval0[$size];",
-            [(DeclareRetParam imm:$align, imm:$size)]>;
-def DeclareRetScalarInst :
-  NVPTXInst<(outs), (ins i32imm:$size),
-            ".param .b$size retval0;",
-            [(DeclareRet imm:$size)]>;
-
-def DeclareParamInst :
-  NVPTXInst<(outs), (ins i32imm:$align, i32imm:$a, i32imm:$size),
-            ".param .align $align .b8 param$a[$size];",
-            [(DeclareParam imm:$align, imm:$a, imm:$size)]>;
-def DeclareScalarParamInst :
+def DECLARE_PARAM_array :
+  NVPTXInst<(outs), (ins i32imm:$a, i32imm:$align, i32imm:$size),
+            ".param .align $align .b8 \t$a[$size];", []>;
+def DECLARE_PARAM_scalar :
   NVPTXInst<(outs), (ins i32imm:$a, i32imm:$size),
-            ".param .b$size param$a;",
-            [(DeclareScalarParam imm:$a, imm:$size)]>;
+            ".param .b$size \t$a;", []>;
+
+def : Pat<(declare_array_param externalsym:$a, imm:$align, imm:$size),
+          (DECLARE_PARAM_array (to_texternsym $a), imm:$align, imm:$size)>;
+def : Pat<(declare_scalar_param externalsym:$a, imm:$size),
+          (DECLARE_PARAM_scalar (to_texternsym $a), imm:$size)>;
 
 foreach t = [I32RT, I64RT] in {
   defvar inst_name = "MOV" # t.Size # "_PARAM";
@@ -2217,7 +2205,7 @@ multiclass ProxyRegInst<string SzStr, NVPTXRegClass rc> {
   def NAME : BasicNVPTXInst<(outs rc:$dst), (ins rc:$src),
                  "mov." # SzStr>;
   foreach vt = rc.RegTypes in
-    def : Pat<(vt (ProxyReg vt:$src)), (!cast<NVPTXInst>(NAME) $src)>;
+    def : Pat<(vt (proxy_reg vt:$src)), (!cast<NVPTXInst>(NAME) $src)>;
 }
 
 defm ProxyRegB1  : ProxyRegInst<"pred", B1>;
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 6a4fc64a3f80c..58766b16e86fe 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -666,7 +666,7 @@ struct PPCOperand : public MCParsedAsmOperand {
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<PPCOperand> CreateToken(StringRef Str, SMLoc S,
                                                  bool IsPPC64) {
@@ -781,7 +781,7 @@ struct PPCOperand : public MCParsedAsmOperand {
 
 } // end anonymous namespace.
 
-void PPCOperand::print(raw_ostream &OS) const {
+void PPCOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case Token:
     OS << "'" << getToken() << "'";
@@ -791,10 +791,10 @@ void PPCOperand::print(raw_ostream &OS) const {
     OS << getImm();
     break;
   case Expression:
-    OS << *getExpr();
+    MAI.printExpr(OS, *getExpr());
     break;
   case TLSRegister:
-    OS << *getTLSReg();
+    MAI.printExpr(OS, *getTLSReg());
     break;
   }
 }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 658089f9582d6..5779d4e8ca8cc 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -91,84 +90,13 @@ class PPCAsmBackend : public MCAsmBackend {
                                          : llvm::endianness::big),
         TT(TT) {}
 
-  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
-    const static MCFixupKindInfo InfosBE[PPC::NumTargetFixupKinds] = {
-      // name                    offset  bits  flags
-      { "fixup_ppc_br24",        6,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24_notoc",  6,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_brcond14",    16,     14,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24abs",     6,      24,   0 },
-      { "fixup_ppc_brcond14abs", 16,     14,   0 },
-      { "fixup_ppc_half16",       0,     16,   0 },
-      { "fixup_ppc_half16ds",     0,     14,   0 },
-      { "fixup_ppc_pcrel34",     0,      34,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_imm34",       0,      34,   0 },
-      { "fixup_ppc_nofixup",      0,      0,   0 }
-    };
-    const static MCFixupKindInfo InfosLE[PPC::NumTargetFixupKinds] = {
-      // name                    offset  bits  flags
-      { "fixup_ppc_br24",        2,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24_notoc",  2,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_brcond14",    2,      14,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24abs",     2,      24,   0 },
-      { "fixup_ppc_brcond14abs", 2,      14,   0 },
-      { "fixup_ppc_half16",      0,      16,   0 },
-      { "fixup_ppc_half16ds",    2,      14,   0 },
-      { "fixup_ppc_pcrel34",     0,      34,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_imm34",       0,      34,   0 },
-      { "fixup_ppc_nofixup",     0,       0,   0 }
-    };
-
-    // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They
-    // do not require any extra processing.
-    if (mc::isRelocation(Kind))
-      return MCAsmBackend::getFixupKindInfo(FK_NONE);
-
-    if (Kind < FirstTargetFixupKind)
-      return MCAsmBackend::getFixupKindInfo(Kind);
-
-    assert(Kind - FirstTargetFixupKind < PPC::NumTargetFixupKinds &&
-           "Invalid kind!");
-    return (Endian == llvm::endianness::little
-                ? InfosLE
-                : InfosBE)[Kind - FirstTargetFixupKind];
-  }
-
-  bool addReloc(const MCFragment &F, const MCFixup &Fixup,
-                const MCValue &TargetVal, uint64_t &FixedValue,
-                bool IsResolved) override {
-    // In PPC64 ELFv1, .quad .TOC.@tocbase in the .opd section is expected to
-    // reference the null symbol.
-    auto Target = TargetVal;
-    if (Target.getSpecifier() == PPC::S_TOCBASE)
-      Target.setAddSym(nullptr);
-    return MCAsmBackend::addReloc(F, Fixup, Target, FixedValue, IsResolved);
-  }
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   void applyFixup(const MCFragment &, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t Value, bool IsResolved) override {
-    MCFixupKind Kind = Fixup.getKind();
-    if (mc::isRelocation(Kind))
-      return;
-    Value = adjustFixupValue(Kind, Value);
-    if (!Value) return;           // Doesn't change encoding.
+                  uint64_t Value, bool IsResolved) override;
 
-    unsigned Offset = Fixup.getOffset();
-    unsigned NumBytes = getFixupKindNumBytes(Kind);
-
-    // For each byte of the fragment that the fixup touches, mask in the bits
-    // from the fixup value. The Value has been "split up" into the appropriate
-    // bitfields above.
-    for (unsigned i = 0; i != NumBytes; ++i) {
-      unsigned Idx =
-          Endian == llvm::endianness::little ? i : (NumBytes - 1 - i);
-      Data[Offset + i] |= uint8_t((Value >> (Idx * 8)) & 0xff);
-    }
-  }
-
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override {
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target) {
     // If there is a @ specifier, unless it is optimized out (e.g. constant @l),
     // force a relocation.
     if (Target.getSpecifier())
@@ -200,12 +128,6 @@ class PPCAsmBackend : public MCAsmBackend {
     }
   }
 
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override {
-    // FIXME.
-    llvm_unreachable("relaxInstruction() unimplemented");
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override {
     uint64_t NumNops = Count / 4;
@@ -219,6 +141,81 @@ class PPCAsmBackend : public MCAsmBackend {
 };
 } // end anonymous namespace
 
+MCFixupKindInfo PPCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+  // clang-format off
+  const static MCFixupKindInfo InfosBE[PPC::NumTargetFixupKinds] = {
+      // name                    offset  bits  flags
+      {"fixup_ppc_br24", 6, 24, 0},
+      {"fixup_ppc_br24_notoc", 6, 24, 0},
+      {"fixup_ppc_brcond14", 16, 14, 0},
+      {"fixup_ppc_br24abs", 6, 24, 0},
+      {"fixup_ppc_brcond14abs", 16, 14, 0},
+      {"fixup_ppc_half16", 0, 16, 0},
+      {"fixup_ppc_half16ds", 0, 14, 0},
+      {"fixup_ppc_pcrel34", 0, 34, 0},
+      {"fixup_ppc_imm34", 0, 34, 0},
+      {"fixup_ppc_nofixup", 0, 0, 0}};
+  const static MCFixupKindInfo InfosLE[PPC::NumTargetFixupKinds] = {
+      // name                    offset  bits  flags
+      {"fixup_ppc_br24", 2, 24, 0},
+      {"fixup_ppc_br24_notoc", 2, 24, 0},
+      {"fixup_ppc_brcond14", 2, 14, 0},
+      {"fixup_ppc_br24abs", 2, 24, 0},
+      {"fixup_ppc_brcond14abs", 2, 14, 0},
+      {"fixup_ppc_half16", 0, 16, 0},
+      {"fixup_ppc_half16ds", 2, 14, 0},
+      {"fixup_ppc_pcrel34", 0, 34, 0},
+      {"fixup_ppc_imm34", 0, 34, 0},
+      {"fixup_ppc_nofixup", 0, 0, 0}};
+  // clang-format on
+
+  // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They
+  // do not require any extra processing.
+  if (mc::isRelocation(Kind))
+    return {};
+
+  if (Kind < FirstTargetFixupKind)
+    return MCAsmBackend::getFixupKindInfo(Kind);
+
+  assert(Kind - FirstTargetFixupKind < PPC::NumTargetFixupKinds &&
+         "Invalid kind!");
+  return (Endian == llvm::endianness::little
+              ? InfosLE
+              : InfosBE)[Kind - FirstTargetFixupKind];
+}
+
+void PPCAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                               const MCValue &TargetVal,
+                               MutableArrayRef<char> Data, uint64_t Value,
+                               bool IsResolved) {
+  // In PPC64 ELFv1, .quad .TOC.@tocbase in the .opd section is expected to
+  // reference the null symbol.
+  auto Target = TargetVal;
+  if (Target.getSpecifier() == PPC::S_TOCBASE)
+    Target.setAddSym(nullptr);
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
+  MCFixupKind Kind = Fixup.getKind();
+  if (mc::isRelocation(Kind))
+    return;
+  Value = adjustFixupValue(Kind, Value);
+  if (!Value)
+    return; // Doesn't change encoding.
+
+  unsigned Offset = Fixup.getOffset();
+  unsigned NumBytes = getFixupKindNumBytes(Kind);
+
+  // For each byte of the fragment that the fixup touches, mask in the bits
+  // from the fixup value. The Value has been "split up" into the appropriate
+  // bitfields above.
+  for (unsigned i = 0; i != NumBytes; ++i) {
+    unsigned Idx = Endian == llvm::endianness::little ? i : (NumBytes - 1 - i);
+    Data[Offset + i] |= uint8_t((Value >> (Idx * 8)) & 0xff);
+  }
+}
 
 // FIXME: This should be in a separate file.
 namespace {
@@ -285,7 +282,7 @@ ELFPPCAsmBackend::getFixupKind(StringRef Name) const {
 std::optional<MCFixupKind>
 XCOFFPPCAsmBackend::getFixupKind(StringRef Name) const {
   return StringSwitch<std::optional<MCFixupKind>>(Name)
-      .Case("R_REF", (MCFixupKind)PPC::fixup_ppc_nofixup)
+      .Case("R_REF", PPC::fixup_ppc_nofixup)
       .Default(std::nullopt);
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
index 8ffc92c8dd026..ee99cfc7d655d 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
@@ -159,9 +159,7 @@ void PPCELFStreamer::emitGOTToPCRelReloc(const MCInst &Inst) {
   assert(DF && "Expecting a valid data fragment.");
   MCFixupKind FixupKind = static_cast<MCFixupKind>(FirstLiteralRelocationKind +
                                                    ELF::R_PPC64_PCREL_OPT);
-  DF->getFixups().push_back(
-      MCFixup::create(LabelSym->getOffset() - 8, SubExpr2,
-                      FixupKind, Inst.getLoc()));
+  DF->addFixup(MCFixup::create(LabelSym->getOffset() - 8, SubExpr2, FixupKind));
   emitLabel(CurrentLocation, Inst.getLoc());
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index fc145f37fae95..8ed7c68f54e7f 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -39,6 +39,19 @@ MCCodeEmitter *llvm::createPPCMCCodeEmitter(const MCInstrInfo &MCII,
   return new PPCMCCodeEmitter(MCII, Ctx);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case PPC::fixup_ppc_br24:
+  case PPC::fixup_ppc_br24_notoc:
+  case PPC::fixup_ppc_brcond14:
+  case PPC::fixup_ppc_pcrel34:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 unsigned PPCMCCodeEmitter::
 getDirectBrEncoding(const MCInst &MI, unsigned OpNo,
                     SmallVectorImpl<MCFixup> &Fixups,
@@ -49,10 +62,9 @@ getDirectBrEncoding(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (isNoTOCCallInstr(MI)
-                                        ? (MCFixupKind)PPC::fixup_ppc_br24_notoc
-                                        : (MCFixupKind)PPC::fixup_ppc_br24)));
+  addFixup(
+      Fixups, 0, MO.getExpr(),
+      (isNoTOCCallInstr(MI) ? PPC::fixup_ppc_br24_notoc : PPC::fixup_ppc_br24));
   return 0;
 }
 
@@ -157,8 +169,7 @@ unsigned PPCMCCodeEmitter::getCondBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_brcond14));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_brcond14);
   return 0;
 }
 
@@ -170,8 +181,7 @@ getAbsDirectBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_br24abs));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_br24abs);
   return 0;
 }
 
@@ -183,8 +193,7 @@ getAbsCondBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_brcond14abs));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_brcond14abs);
   return 0;
 }
 
@@ -205,8 +214,7 @@ unsigned PPCMCCodeEmitter::getImm16Encoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the immediate field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(), PPC::fixup_ppc_half16);
   return 0;
 }
 
@@ -220,7 +228,7 @@ uint64_t PPCMCCodeEmitter::getImm34Encoding(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the immediate field.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Fixup));
+  addFixup(Fixups, 0, MO.getExpr(), Fixup);
   return 0;
 }
 
@@ -228,16 +236,14 @@ uint64_t
 PPCMCCodeEmitter::getImm34EncodingNoPCRel(const MCInst &MI, unsigned OpNo,
                                           SmallVectorImpl<MCFixup> &Fixups,
                                           const MCSubtargetInfo &STI) const {
-  return getImm34Encoding(MI, OpNo, Fixups, STI,
-                          (MCFixupKind)PPC::fixup_ppc_imm34);
+  return getImm34Encoding(MI, OpNo, Fixups, STI, PPC::fixup_ppc_imm34);
 }
 
 uint64_t
 PPCMCCodeEmitter::getImm34EncodingPCRel(const MCInst &MI, unsigned OpNo,
                                         SmallVectorImpl<MCFixup> &Fixups,
                                         const MCSubtargetInfo &STI) const {
-  return getImm34Encoding(MI, OpNo, Fixups, STI,
-                          (MCFixupKind)PPC::fixup_ppc_pcrel34);
+  return getImm34Encoding(MI, OpNo, Fixups, STI, PPC::fixup_ppc_pcrel34);
 }
 
 unsigned PPCMCCodeEmitter::getDispRIEncoding(const MCInst &MI, unsigned OpNo,
@@ -248,8 +254,7 @@ unsigned PPCMCCodeEmitter::getDispRIEncoding(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI) & 0xFFFF;
 
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(), PPC::fixup_ppc_half16);
   return 0;
 }
 
@@ -262,8 +267,8 @@ PPCMCCodeEmitter::getDispRIXEncoding(const MCInst &MI, unsigned OpNo,
     return ((getMachineOpValue(MI, MO, Fixups, STI) >> 2) & 0x3FFF);
 
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16ds));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(),
+           PPC::fixup_ppc_half16ds);
   return 0;
 }
 
@@ -279,8 +284,8 @@ PPCMCCodeEmitter::getDispRIX16Encoding(const MCInst &MI, unsigned OpNo,
   }
 
   // Otherwise add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian ? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16dq));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(),
+           PPC::fixup_ppc_half16dq);
   return 0;
 }
 
@@ -335,9 +340,7 @@ PPCMCCodeEmitter::getDispRI34PCRelEncoding(const MCInst &MI, unsigned OpNo,
            "specifier must be S_PCREL, S_GOT_PCREL, S_GOT_TLSGD_PCREL, "
            "S_GOT_TLSLD_PCREL, or S_GOT_TPREL_PCREL");
     // Generate the fixup for the relocation.
-    Fixups.push_back(
-        MCFixup::create(0, Expr,
-                        static_cast<MCFixupKind>(PPC::fixup_ppc_pcrel34)));
+    addFixup(Fixups, 0, Expr, PPC::fixup_ppc_pcrel34);
     // Put zero in the location of the immediate. The linker will fill in the
     // correct value based on the relocation.
     return 0;
@@ -369,9 +372,7 @@ PPCMCCodeEmitter::getDispRI34PCRelEncoding(const MCInst &MI, unsigned OpNo,
             getSpecifier(SRE) == PPC::S_GOT_PCREL) &&
            "VariantKind must be VK_PCREL or VK_GOT_PCREL");
     // Generate the fixup for the relocation.
-    Fixups.push_back(
-        MCFixup::create(0, Expr,
-                        static_cast<MCFixupKind>(PPC::fixup_ppc_pcrel34)));
+    addFixup(Fixups, 0, Expr, PPC::fixup_ppc_pcrel34);
     // Put zero in the location of the immediate. The linker will fill in the
     // correct value based on the relocation.
     return 0;
@@ -431,8 +432,7 @@ unsigned PPCMCCodeEmitter::getTLSRegEncoding(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(Expr);
   bool IsPCRel = getSpecifier(SRE) == PPC::S_TLS_PCREL;
-  Fixups.push_back(MCFixup::create(IsPCRel ? 1 : 0, Expr,
-                                   (MCFixupKind)PPC::fixup_ppc_nofixup));
+  addFixup(Fixups, IsPCRel ? 1 : 0, Expr, PPC::fixup_ppc_nofixup);
   const Triple &TT = STI.getTargetTriple();
   bool isPPC64 = TT.isPPC64();
   return CTX.getRegisterInfo()->getEncodingValue(isPPC64 ? PPC::X13 : PPC::R2);
@@ -445,8 +445,7 @@ unsigned PPCMCCodeEmitter::getTLSCallEncoding(const MCInst &MI, unsigned OpNo,
   // (__tls_get_addr), which we create via getDirectBrEncoding as usual,
   // and one for the TLSGD or TLSLD symbol, which is emitted here.
   const MCOperand &MO = MI.getOperand(OpNo+1);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_nofixup));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_nofixup);
   return getDirectBrEncoding(MI, OpNo, Fixups, STI);
 }
 
diff --git a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
index e6e8a881e610d..6fe7688224d39 100644
--- a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
+++ b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
@@ -45,7 +45,6 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/User.h"
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 15466ba9876cc..c0860fcdaa3b0 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -2170,8 +2170,8 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
   // The Floating-point register save area is right below the back chain word
   // of the previous stack frame.
   if (HasFPSaveArea) {
-    for (unsigned i = 0, e = FPRegs.size(); i != e; ++i) {
-      int FI = FPRegs[i].getFrameIdx();
+    for (const CalleeSavedInfo &FPReg : FPRegs) {
+      int FI = FPReg.getFrameIdx();
 
       MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
     }
@@ -2219,18 +2219,18 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
   if (HasGPSaveArea || HasG8SaveArea) {
     // Move general register save area spill slots down, taking into account
     // the size of the Floating-point register save area.
-    for (unsigned i = 0, e = GPRegs.size(); i != e; ++i) {
-      if (!GPRegs[i].isSpilledToReg()) {
-        int FI = GPRegs[i].getFrameIdx();
+    for (const CalleeSavedInfo &GPReg : GPRegs) {
+      if (!GPReg.isSpilledToReg()) {
+        int FI = GPReg.getFrameIdx();
         MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
       }
     }
 
     // Move general register save area spill slots down, taking into account
     // the size of the Floating-point register save area.
-    for (unsigned i = 0, e = G8Regs.size(); i != e; ++i) {
-      if (!G8Regs[i].isSpilledToReg()) {
-        int FI = G8Regs[i].getFrameIdx();
+    for (const CalleeSavedInfo &G8Reg : G8Regs) {
+      if (!G8Reg.isSpilledToReg()) {
+        int FI = G8Reg.getFrameIdx();
         MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
       }
     }
@@ -2272,8 +2272,8 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
     assert(LowerBound <= 0 && "Expect LowerBound have a non-positive value!");
     LowerBound &= ~(15);
 
-    for (unsigned i = 0, e = VRegs.size(); i != e; ++i) {
-      int FI = VRegs[i].getFrameIdx();
+    for (const CalleeSavedInfo &VReg : VRegs) {
+      int FI = VReg.getFrameIdx();
 
       MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
     }
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 88c6fe632d266..b96505816dee8 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -777,6 +777,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
     setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
     setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
+    setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
   }
 
   if (Subtarget.hasAltivec()) {
@@ -4445,11 +4447,11 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
     // The fixed integer arguments of a variadic function are stored to the
     // VarArgsFrameIndex on the stack so that they may be loaded by
     // dereferencing the result of va_next.
-    for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) {
+    for (MCPhysReg GPArgReg : GPArgRegs) {
       // Get an existing live-in vreg, or add a new one.
-      Register VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]);
+      Register VReg = MF.getRegInfo().getLiveInVirtReg(GPArgReg);
       if (!VReg)
-        VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass);
+        VReg = MF.addLiveIn(GPArgReg, &PPC::GPRCRegClass);
 
       SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
       SDValue Store =
@@ -4549,13 +4551,13 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
   unsigned NumBytes = LinkageSize;
   unsigned AvailableFPRs = Num_FPR_Regs;
   unsigned AvailableVRs = Num_VR_Regs;
-  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
-    if (Ins[i].Flags.isNest())
+  for (const ISD::InputArg &In : Ins) {
+    if (In.Flags.isNest())
       continue;
 
-    if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
-                               PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs))
+    if (CalculateStackSlotUsed(In.VT, In.ArgVT, In.Flags, PtrByteSize,
+                               LinkageSize, ParamAreaSize, NumBytes,
+                               AvailableFPRs, AvailableVRs))
       HasParameterArea = true;
   }
 
@@ -5766,9 +5768,8 @@ buildCallOperands(SmallVectorImpl<SDValue> &Ops,
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
   // no way to mark dependencies as implicit here.
@@ -6191,9 +6192,8 @@ SDValue PPCTargetLowering::LowerCall_32SVR4(
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -6805,9 +6805,8 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -9587,6 +9586,36 @@ static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
   return false;
 }
 
+bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN) {
+  assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
+
+  BitMask.clearAllBits();
+  EVT VT = BVN.getValueType(0);
+  APInt ConstValue(VT.getSizeInBits(), 0);
+
+  unsigned EltWidth = VT.getScalarSizeInBits();
+
+  unsigned BitPos = 0;
+  for (auto OpVal : BVN.op_values()) {
+    auto *CN = dyn_cast<ConstantSDNode>(OpVal);
+
+    if (!CN)
+      return false;
+
+    ConstValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
+    BitPos += EltWidth;
+  }
+
+  for (unsigned J = 0; J < 16; ++J) {
+    APInt ExtractValue = ConstValue.extractBits(8, J * 8);
+    if (ExtractValue != 0x00 && ExtractValue != 0xFF)
+      return false;
+    if (ExtractValue == 0xFF)
+      BitMask.setBit(J);
+  }
+  return true;
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.  If we CAN select this case, and if it
 // selects to a single instruction, return Op.  Otherwise, if we can codegen
@@ -9598,6 +9627,25 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
   assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
 
+  if (Subtarget.hasP10Vector()) {
+    APInt BitMask(32, 0);
+    // If the value of the vector is all zeros or all ones,
+    // we do not convert it to MTVSRBMI.
+    // The xxleqv instruction sets a vector with all ones.
+    // The xxlxor instruction sets a vector with all zeros.
+    if (isValidMtVsrBmi(BitMask, *BVN) && BitMask != 0 && BitMask != 0xffff) {
+      SDValue SDConstant = DAG.getTargetConstant(BitMask, dl, MVT::i32);
+      MachineSDNode *MSDNode =
+          DAG.getMachineNode(PPC::MTVSRBMI, dl, MVT::v16i8, SDConstant);
+      SDValue SDV = SDValue(MSDNode, 0);
+      EVT DVT = BVN->getValueType(0);
+      EVT SVT = SDV.getValueType();
+      if (SVT != DVT) {
+        SDV = DAG.getNode(ISD::BITCAST, dl, DVT, SDV);
+      }
+      return SDV;
+    }
+  }
   // Check if this is a splat of a constant value.
   APInt APSplatBits, APSplatUndef;
   unsigned SplatBitSize;
@@ -17915,9 +17963,6 @@ SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
 
diff --git a/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp b/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
index ec20ad2565e68..0ffd35dfa279c 100644
--- a/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
+++ b/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
@@ -108,6 +108,8 @@ struct BlockSplitInfo {
   MachineInstr *OrigBranch;
   MachineInstr *SplitBefore;
   MachineInstr *SplitCond;
+  unsigned OrigSubreg;
+  unsigned SplitCondSubreg;
   bool InvertNewBranch;
   bool InvertOrigBranch;
   bool BranchToFallThrough;
@@ -220,7 +222,7 @@ static bool splitMBB(BlockSplitInfo &BSI) {
   // Add the branches to ThisMBB.
   BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
           TII->get(NewBROpcode))
-      .addReg(BSI.SplitCond->getOperand(0).getReg())
+      .addReg(BSI.SplitCond->getOperand(0).getReg(), 0, BSI.SplitCondSubreg)
       .addMBB(NewBRTarget);
   BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
           TII->get(PPC::B))
@@ -234,6 +236,7 @@ static bool splitMBB(BlockSplitInfo &BSI) {
     assert(FirstTerminator->getOperand(0).isReg() &&
            "Can't update condition of unconditional branch.");
     FirstTerminator->getOperand(0).setReg(BSI.NewCond->getOperand(0).getReg());
+    FirstTerminator->getOperand(0).setSubReg(BSI.OrigSubreg);
   }
   if (BSI.InvertOrigBranch)
     FirstTerminator->setDesc(TII->get(InvertedOpcode));
@@ -471,6 +474,7 @@ PPCReduceCRLogicals::createCRLogicalOpInfo(MachineInstr &MIParam) {
   } else {
     MachineInstr *Def1 = lookThroughCRCopy(MIParam.getOperand(1).getReg(),
                                            Ret.SubregDef1, Ret.CopyDefs.first);
+    Ret.SubregDef1 = MIParam.getOperand(1).getSubReg();
     assert(Def1 && "Must be able to find a definition of operand 1.");
     Ret.DefsSingleUse &=
       MRI->hasOneNonDBGUse(Def1->getOperand(0).getReg());
@@ -481,6 +485,7 @@ PPCReduceCRLogicals::createCRLogicalOpInfo(MachineInstr &MIParam) {
       MachineInstr *Def2 = lookThroughCRCopy(MIParam.getOperand(2).getReg(),
                                              Ret.SubregDef2,
                                              Ret.CopyDefs.second);
+      Ret.SubregDef2 = MIParam.getOperand(2).getSubReg();
       assert(Def2 && "Must be able to find a definition of operand 2.");
       Ret.DefsSingleUse &=
         MRI->hasOneNonDBGUse(Def2->getOperand(0).getReg());
@@ -535,7 +540,6 @@ PPCReduceCRLogicals::createCRLogicalOpInfo(MachineInstr &MIParam) {
 MachineInstr *PPCReduceCRLogicals::lookThroughCRCopy(unsigned Reg,
                                                      unsigned &Subreg,
                                                      MachineInstr *&CpDef) {
-  Subreg = -1;
   if (!Register::isVirtualRegister(Reg))
     return nullptr;
   MachineInstr *Copy = MRI->getVRegDef(Reg);
@@ -543,18 +547,8 @@ MachineInstr *PPCReduceCRLogicals::lookThroughCRCopy(unsigned Reg,
   if (!Copy->isCopy())
     return Copy;
   Register CopySrc = Copy->getOperand(1).getReg();
-  Subreg = Copy->getOperand(1).getSubReg();
   if (!CopySrc.isVirtual()) {
     const TargetRegisterInfo *TRI = &TII->getRegisterInfo();
-    // Set the Subreg
-    if (CopySrc == PPC::CR0EQ || CopySrc == PPC::CR6EQ)
-      Subreg = PPC::sub_eq;
-    if (CopySrc == PPC::CR0LT || CopySrc == PPC::CR6LT)
-      Subreg = PPC::sub_lt;
-    if (CopySrc == PPC::CR0GT || CopySrc == PPC::CR6GT)
-      Subreg = PPC::sub_gt;
-    if (CopySrc == PPC::CR0UN || CopySrc == PPC::CR6UN)
-      Subreg = PPC::sub_un;
     // Loop backwards and return the first MI that modifies the physical CR Reg.
     MachineBasicBlock::iterator Me = Copy, B = Copy->getParent()->begin();
     while (Me != B)
@@ -682,16 +676,21 @@ bool PPCReduceCRLogicals::splitBlockOnBinaryCROp(CRLogicalOpInfo &CRI) {
   computeBranchTargetAndInversion(Opc, Branch->getOpcode(), UsingDef1,
                                   InvertNewBranch, InvertOrigBranch,
                                   TargetIsFallThrough);
-  MachineInstr *SplitCond =
-    UsingDef1 ? CRI.CopyDefs.second : CRI.CopyDefs.first;
+  MachineInstr *NewCond = CRI.CopyDefs.first;
+  MachineInstr *SplitCond = CRI.CopyDefs.second;
+  if (!UsingDef1) {
+    std::swap(NewCond, SplitCond);
+    std::swap(CRI.SubregDef1, CRI.SubregDef2);
+  }
   LLVM_DEBUG(dbgs() << "We will " << (InvertNewBranch ? "invert" : "copy"));
   LLVM_DEBUG(dbgs() << " the original branch and the target is the "
                     << (TargetIsFallThrough ? "fallthrough block\n"
                                             : "orig. target block\n"));
   LLVM_DEBUG(dbgs() << "Original branch instruction: "; Branch->dump());
-  BlockSplitInfo BSI { Branch, SplitBefore, SplitCond, InvertNewBranch,
-    InvertOrigBranch, TargetIsFallThrough, MBPI, CRI.MI,
-    UsingDef1 ? CRI.CopyDefs.first : CRI.CopyDefs.second };
+  BlockSplitInfo BSI{
+      Branch,         SplitBefore,     SplitCond,        CRI.SubregDef1,
+      CRI.SubregDef2, InvertNewBranch, InvertOrigBranch, TargetIsFallThrough,
+      MBPI,           CRI.MI,          NewCond};
   bool Changed = splitMBB(BSI);
   // If we've split on a CR logical that is fed by a CR logical,
   // recompute the source CR logical as it may be usable for splitting.
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index e5d8ab07891ac..66f4aade380fa 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -732,6 +732,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
   bool isUImm6() const { return isUImm<6>(); }
   bool isUImm7() const { return isUImm<7>(); }
   bool isUImm8() const { return isUImm<8>(); }
+  bool isUImm9() const { return isUImm<9>(); }
   bool isUImm10() const { return isUImm<10>(); }
   bool isUImm11() const { return isUImm<11>(); }
   bool isUImm16() const { return isUImm<16>(); }
@@ -807,6 +808,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
 
   bool isSImm5() const { return isSImm<5>(); }
   bool isSImm6() const { return isSImm<6>(); }
+  bool isSImm10() const { return isSImm<10>(); }
   bool isSImm11() const { return isSImm<11>(); }
   bool isSImm16() const { return isSImm<16>(); }
   bool isSImm26() const { return isSImm<26>(); }
@@ -901,6 +903,8 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            VK == RISCV::S_QC_ABS20;
   }
 
+  bool isSImm10Unsigned() const { return isSImm<10>() || isUImm<10>(); }
+
   bool isUImm20LUI() const {
     if (!isImm())
       return false;
@@ -1016,7 +1020,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     return Fence.Val;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     auto RegName = [](MCRegister Reg) {
       if (Reg)
         return RISCVInstPrinter::getRegisterName(Reg);
@@ -1026,8 +1030,9 @@ struct RISCVOperand final : public MCParsedAsmOperand {
 
     switch (Kind) {
     case KindTy::Immediate:
-      OS << "<imm: " << *Imm.Val << " " << (Imm.IsRV64 ? "rv64" : "rv32")
-         << ">";
+      OS << "<imm: ";
+      MAI.printExpr(OS, *Imm.Val);
+      OS << ' ' << (Imm.IsRV64 ? "rv64" : "rv32") << '>';
       break;
     case KindTy::FPImmediate:
       OS << "<fpimm: " << FPImm.Val << ">";
@@ -1523,6 +1528,9 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 0, (1 << 8) - 8,
         "immediate must be a multiple of 8 bytes in the range");
+  case Match_InvalidUImm9:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 9) - 1,
+                                      "immediate offset must be in the range");
   case Match_InvalidBareSImm9Lsb0:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, -(1 << 8), (1 << 8) - 2,
@@ -1531,6 +1539,12 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 0, (1 << 9) - 8,
         "immediate must be a multiple of 8 bytes in the range");
+  case Match_InvalidSImm10:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 9),
+                                      (1 << 9) - 1);
+  case Match_InvalidSImm10Unsigned:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 9),
+                                      (1 << 10) - 1);
   case Match_InvalidUImm10Lsb00NonZero:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 4, (1 << 10) - 4,
@@ -2078,12 +2092,13 @@ ParseStatus RISCVAsmParser::parseOperandWithSpecifier(OperandVector &Operands) {
 }
 
 bool RISCVAsmParser::parseExprWithSpecifier(const MCExpr *&Res, SMLoc &E) {
+  SMLoc Loc = getLoc();
   if (getLexer().getKind() != AsmToken::Identifier)
-    return Error(getLoc(), "expected '%' relocation specifier");
+    return TokError("expected '%' relocation specifier");
   StringRef Identifier = getParser().getTok().getIdentifier();
   auto Spec = RISCV::parseSpecifierName(Identifier);
   if (!Spec)
-    return Error(getLoc(), "invalid relocation specifier");
+    return TokError("invalid relocation specifier");
 
   getParser().Lex(); // Eat the identifier
   if (parseToken(AsmToken::LParen, "expected '('"))
@@ -2093,7 +2108,7 @@ bool RISCVAsmParser::parseExprWithSpecifier(const MCExpr *&Res, SMLoc &E) {
   if (getParser().parseParenExpression(SubExpr, E))
     return true;
 
-  Res = MCSpecifierExpr::create(SubExpr, Spec, getContext());
+  Res = MCSpecifierExpr::create(SubExpr, Spec, getContext(), Loc);
   return false;
 }
 
@@ -3749,9 +3764,9 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
   if (!(MCID.TSFlags & RISCVII::ConstraintMask))
     return false;
 
-  if (Opcode == RISCV::VC_V_XVW || Opcode == RISCV::VC_V_IVW ||
-      Opcode == RISCV::VC_V_FVW || Opcode == RISCV::VC_V_VVW) {
-    // Operands Opcode, Dst, uimm, Dst, Rs2, Rs1 for VC_V_XVW.
+  if (Opcode == RISCV::SF_VC_V_XVW || Opcode == RISCV::SF_VC_V_IVW ||
+      Opcode == RISCV::SF_VC_V_FVW || Opcode == RISCV::SF_VC_V_VVW) {
+    // Operands Opcode, Dst, uimm, Dst, Rs2, Rs1 for SF_VC_V_XVW.
     MCRegister VCIXDst = Inst.getOperand(0).getReg();
     SMLoc VCIXDstLoc = Operands[2]->getStartLoc();
     if (MCID.TSFlags & RISCVII::VS1Constraint) {
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 27e04c0cb1f8b..b723958a6ff28 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -775,7 +775,8 @@ static constexpr FeatureBitset XTHeadGroup = {
 
 static constexpr FeatureBitset XAndesGroup = {
     RISCV::FeatureVendorXAndesPerf, RISCV::FeatureVendorXAndesVBFHCvt,
-    RISCV::FeatureVendorXAndesVPackFPH, RISCV::FeatureVendorXAndesVDot};
+    RISCV::FeatureVendorXAndesVSIntLoad, RISCV::FeatureVendorXAndesVPackFPH,
+    RISCV::FeatureVendorXAndesVDot};
 
 static constexpr DecoderListEntry DecoderList32[]{
     // Vendor Extensions
@@ -790,6 +791,9 @@ static constexpr DecoderListEntry DecoderList32[]{
     {DecoderTableXmipscmov32,
      {RISCV::FeatureVendorXMIPSCMov},
      "MIPS mips.ccmov"},
+    {DecoderTableXmipscbop32,
+     {RISCV::FeatureVendorXMIPSCBOP},
+     "MIPS mips.pref"},
     {DecoderTableXAndes32, XAndesGroup, "Andes extensions"},
     // Standard Extensions
     {DecoderTableXCV32, XCVFeatureGroup, "CORE-V extensions"},
@@ -811,7 +815,9 @@ DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
   }
   Size = 4;
 
-  uint32_t Insn = support::endian::read32le(Bytes.data());
+  // Use uint64_t to match getInstruction48. decodeInstruction is templated
+  // on the Insn type.
+  uint64_t Insn = support::endian::read32le(Bytes.data());
 
   for (const DecoderListEntry &Entry : DecoderList32) {
     if (!Entry.haveContainedFeatures(STI.getFeatureBits()))
@@ -857,7 +863,9 @@ DecodeStatus RISCVDisassembler::getInstruction16(MCInst &MI, uint64_t &Size,
   }
   Size = 2;
 
-  uint32_t Insn = support::endian::read16le(Bytes.data());
+  // Use uint64_t to match getInstruction48. decodeInstruction is templated
+  // on the Insn type.
+  uint64_t Insn = support::endian::read16le(Bytes.data());
 
   for (const DecoderListEntry &Entry : DecoderList16) {
     if (!Entry.haveContainedFeatures(STI.getFeatureBits()))
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index a6c2d2fc29119..89a87798d71e4 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
@@ -70,25 +69,23 @@ MCFixupKindInfo RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_riscv_lo12_i", 20, 12, 0},
       {"fixup_riscv_12_i", 20, 12, 0},
       {"fixup_riscv_lo12_s", 0, 32, 0},
-      {"fixup_riscv_pcrel_hi20", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_pcrel_lo12_i", 20, 12,
-       MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
-      {"fixup_riscv_pcrel_lo12_s", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
-      {"fixup_riscv_jal", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_rvc_jump", 2, 11, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_rvc_branch", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_call", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_call_plt", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
-
-      {"fixup_riscv_qc_e_branch", 0, 48, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_riscv_pcrel_hi20", 12, 20, 0},
+      {"fixup_riscv_pcrel_lo12_i", 20, 12, 0},
+      {"fixup_riscv_pcrel_lo12_s", 0, 32, 0},
+      {"fixup_riscv_jal", 12, 20, 0},
+      {"fixup_riscv_branch", 0, 32, 0},
+      {"fixup_riscv_rvc_jump", 2, 11, 0},
+      {"fixup_riscv_rvc_branch", 0, 16, 0},
+      {"fixup_riscv_call", 0, 64, 0},
+      {"fixup_riscv_call_plt", 0, 64, 0},
+
+      {"fixup_riscv_qc_e_branch", 0, 48, 0},
       {"fixup_riscv_qc_e_32", 16, 32, 0},
       {"fixup_riscv_qc_abs20_u", 12, 20, 0},
-      {"fixup_riscv_qc_e_call_plt", 0, 48, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_riscv_qc_e_call_plt", 0, 48, 0},
 
       // Andes fixups
-      {"fixup_riscv_nds_branch_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_riscv_nds_branch_10", 0, 32, 0},
   };
   static_assert((std::size(Infos)) == RISCV::NumTargetFixupKinds,
                 "Not all fixup kinds added to Infos array");
@@ -96,7 +93,7 @@ MCFixupKindInfo RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from raw relocation types and .reloc directives force
   // relocations and do not use these fields.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -143,9 +140,9 @@ bool RISCVAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
 // Given a compressed control flow instruction this function returns
 // the expanded instruction, or the original instruction code if no
 // expansion is available.
-static unsigned getRelaxedOpcode(const MCInst &Inst,
+static unsigned getRelaxedOpcode(unsigned Opcode, ArrayRef<MCOperand> Operands,
                                  const MCSubtargetInfo &STI) {
-  switch (Inst.getOpcode()) {
+  switch (Opcode) {
   case RISCV::C_BEQZ:
     return RISCV::BEQ;
   case RISCV::C_BNEZ:
@@ -161,7 +158,7 @@ static unsigned getRelaxedOpcode(const MCInst &Inst,
       break;
 
     // And only if it is using X0 or X1 for rd.
-    MCRegister Reg = Inst.getOperand(0).getReg();
+    MCRegister Reg = Operands[0].getReg();
     if (Reg == RISCV::X0)
       return RISCV::QC_E_J;
     if (Reg == RISCV::X1)
@@ -208,7 +205,7 @@ static unsigned getRelaxedOpcode(const MCInst &Inst,
   }
 
   // Returning the original opcode means we cannot relax the instruction.
-  return Inst.getOpcode();
+  return Opcode;
 }
 
 void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
@@ -226,7 +223,8 @@ void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
   case RISCV::C_JAL: {
     [[maybe_unused]] bool Success = RISCVRVC::uncompress(Res, Inst, STI);
     assert(Success && "Can't uncompress instruction");
-    assert(Res.getOpcode() == getRelaxedOpcode(Inst, STI) &&
+    assert(Res.getOpcode() ==
+               getRelaxedOpcode(Inst.getOpcode(), Inst.getOperands(), STI) &&
            "Branch Relaxation Error");
     break;
   }
@@ -238,7 +236,7 @@ void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
     assert((Inst.getOperand(0).getReg() == RISCV::X0 ||
             Inst.getOperand(0).getReg() == RISCV::X1) &&
            "JAL only relaxable with rd=x0 or rd=x1");
-    Res.setOpcode(getRelaxedOpcode(Inst, STI));
+    Res.setOpcode(getRelaxedOpcode(Inst.getOpcode(), Inst.getOperands(), STI));
     Res.addOperand(Inst.getOperand(1));
     break;
   }
@@ -260,7 +258,7 @@ void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
   case RISCV::QC_E_BGEI:
   case RISCV::QC_E_BLTUI:
   case RISCV::QC_E_BGEUI:
-    Res.setOpcode(getRelaxedOpcode(Inst, STI));
+    Res.setOpcode(getRelaxedOpcode(Inst.getOpcode(), Inst.getOperands(), STI));
     Res.addOperand(Inst.getOperand(0));
     Res.addOperand(Inst.getOperand(1));
     Res.addOperand(Inst.getOperand(2));
@@ -275,17 +273,16 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 
   int64_t LineDelta = DF.getLineDelta();
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 1> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   [[maybe_unused]] bool IsAbsolute =
       AddrDelta.evaluateKnownAbsolute(Value, *Asm);
   assert(IsAbsolute && "CFA with invalid expression");
 
-  Data.clear();
   Fixups.clear();
+  SmallVector<char> Data;
   raw_svector_ostream OS(Data);
 
   // INT64_MAX is a signal that this is actually a DW_LNE_end_sequence.
@@ -330,6 +327,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
     OS << uint8_t(dwarf::DW_LNS_copy);
   }
 
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
   WasRelaxed = OldSize != Data.size();
   return true;
 }
@@ -337,9 +336,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
                                     bool &WasRelaxed) const {
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 2> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -348,14 +346,12 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
       AddrDelta.evaluateKnownAbsolute(Value, *Asm);
   assert(IsAbsolute && "CFA with invalid expression");
 
-  Data.clear();
-  Fixups.clear();
-  raw_svector_ostream OS(Data);
-
   assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 &&
          "expected 1-byte alignment");
   if (Value == 0) {
-    WasRelaxed = OldSize != Data.size();
+    DF.clearContents();
+    DF.clearFixups();
+    WasRelaxed = OldSize != DF.getContents().size();
     return true;
   }
 
@@ -366,6 +362,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
     Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(Fixup)));
   };
 
+  SmallVector<char, 8> Data;
+  raw_svector_ostream OS(Data);
   if (isUIntN(6, Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc);
     AddFixups(0, {ELF::R_RISCV_SET6, ELF::R_RISCV_SUB6});
@@ -384,6 +382,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   } else {
     llvm_unreachable("unsupported CFA encoding");
   }
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
 
   WasRelaxed = OldSize != Data.size();
   return true;
@@ -395,13 +395,13 @@ std::pair<bool, bool> RISCVAsmBackend::relaxLEB128(MCLEBFragment &LF,
     return std::make_pair(false, false);
   const MCExpr &Expr = LF.getValue();
   if (ULEB128Reloc) {
-    LF.getFixups().push_back(
-        MCFixup::create(0, &Expr, FK_Data_leb128, Expr.getLoc()));
+    LF.addFixup(MCFixup::create(0, &Expr, FK_Data_leb128));
   }
   return std::make_pair(Expr.evaluateKnownAbsolute(Value, *Asm), false);
 }
 
-bool RISCVAsmBackend::mayNeedRelaxation(const MCInst &Inst,
+bool RISCVAsmBackend::mayNeedRelaxation(unsigned Opcode,
+                                        ArrayRef<MCOperand> Operands,
                                         const MCSubtargetInfo &STI) const {
   // This function has access to two STIs, the member of the AsmBackend, and the
   // one passed as an argument. The latter is more specific, so we query it for
@@ -409,7 +409,7 @@ bool RISCVAsmBackend::mayNeedRelaxation(const MCInst &Inst,
   if (STI.hasFeature(RISCV::FeatureExactAssembly))
     return false;
 
-  return getRelaxedOpcode(Inst, STI) != Inst.getOpcode();
+  return getRelaxedOpcode(Opcode, Operands, STI) != Opcode;
 }
 
 bool RISCVAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
@@ -657,15 +657,17 @@ static const MCFixup *getPCRelHiFixup(const MCSpecifierExpr &Expr,
   return nullptr;
 }
 
-bool RISCVAsmBackend::evaluateTargetFixup(const MCFixup &Fixup,
-                                          const MCValue &Target,
-                                          uint64_t &Value) {
+std::optional<bool> RISCVAsmBackend::evaluateFixup(const MCFragment &,
+                                                   MCFixup &Fixup,
+                                                   MCValue &Target,
+                                                   uint64_t &Value) {
   const MCFixup *AUIPCFixup;
   const MCFragment *AUIPCDF;
   MCValue AUIPCTarget;
   switch (Fixup.getTargetKind()) {
   default:
-    llvm_unreachable("Unexpected fixup kind!");
+    // Use default handling for `Value` and `IsResolved`.
+    return {};
   case RISCV::fixup_riscv_pcrel_lo12_i:
   case RISCV::fixup_riscv_pcrel_lo12_s: {
     AUIPCFixup =
@@ -794,8 +796,7 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
   // generate a relocation and then append a RELAX.
   if (Fixup.isLinkerRelaxable())
     IsResolved = false;
-  if (IsResolved &&
-      (getFixupKindInfo(Fixup.getKind()).Flags & MCFixupKindInfo::FKF_IsPCRel))
+  if (IsResolved && Fixup.isPCRel())
     IsResolved = isPCRelFixupResolved(Target.getAddSym(), F);
 
   if (!IsResolved) {
@@ -815,10 +816,11 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
   return false;
 }
 
-void RISCVAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void RISCVAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target,
                                  MutableArrayRef<char> Data, uint64_t Value,
                                  bool IsResolved) {
+  IsResolved = addReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
@@ -885,8 +887,7 @@ bool RISCVAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
 
   MCContext &Ctx = getContext();
   const MCExpr *Dummy = MCConstantExpr::create(0, Ctx);
-  // Create fixup_riscv_align fixup.
-  MCFixup Fixup = MCFixup::create(0, Dummy, ELF::R_RISCV_ALIGN, SMLoc());
+  MCFixup Fixup = MCFixup::create(0, Dummy, ELF::R_RISCV_ALIGN);
 
   uint64_t FixedValue = 0;
   MCValue NopBytes = MCValue::get(Count);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 91efd44547509..1f1a6f5fe31a0 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -14,7 +14,6 @@
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/MC/MCAsmBackend.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 
 namespace llvm {
@@ -47,11 +46,10 @@ class RISCVAsmBackend : public MCAsmBackend {
   bool shouldInsertFixupForCodeAlign(MCAssembler &Asm,
                                      MCAlignFragment &AF) override;
 
-  bool evaluateTargetFixup(const MCFixup &Fixup, const MCValue &Target,
-                           uint64_t &Value) override;
-
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                uint64_t &FixedValue, bool IsResolved) override;
+                uint64_t &FixedValue, bool IsResolved);
 
   void maybeAddVendorReloc(const MCFragment &, const MCFixup &);
 
@@ -69,9 +67,8 @@ class RISCVAsmBackend : public MCAsmBackend {
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override;
-
   void relaxInstruction(MCInst &Inst,
                         const MCSubtargetInfo &STI) const override;
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index 3d304842fac13..e123ef45fa503 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -316,6 +316,7 @@ enum OperandType : unsigned {
   OPERAND_UIMM8_LSB000,
   OPERAND_UIMM8_GE32,
   OPERAND_UIMM9_LSB000,
+  OPERAND_UIMM9,
   OPERAND_UIMM10,
   OPERAND_UIMM10_LSB00_NONZERO,
   OPERAND_UIMM11,
@@ -337,6 +338,8 @@ enum OperandType : unsigned {
   OPERAND_SIMM5_PLUS1,
   OPERAND_SIMM6,
   OPERAND_SIMM6_NONZERO,
+  OPERAND_SIMM10,
+  OPERAND_SIMM10_UNSIGNED,
   OPERAND_SIMM10_LSB0000_NONZERO,
   OPERAND_SIMM11,
   OPERAND_SIMM12,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
index b5c23772e6d8c..c1cdf511fae5b 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
@@ -71,21 +71,17 @@ getRelocPairForSize(unsigned Size) {
   default:
     llvm_unreachable("unsupported fixup size");
   case 1:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD8),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB8));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD8,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB8);
   case 2:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD16),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB16));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD16,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB16);
   case 4:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD32),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB32));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD32,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB32);
   case 8:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD64),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB64));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD64,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB64);
   }
 }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index ce0fbc0ac0654..2ed7cd9f008a6 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -120,6 +120,28 @@ MCCodeEmitter *llvm::createRISCVMCCodeEmitter(const MCInstrInfo &MCII,
   return new RISCVMCCodeEmitter(Ctx, MCII);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case ELF::R_RISCV_CALL_PLT:
+  case RISCV::fixup_riscv_pcrel_hi20:
+  case RISCV::fixup_riscv_pcrel_lo12_i:
+  case RISCV::fixup_riscv_pcrel_lo12_s:
+  case RISCV::fixup_riscv_jal:
+  case RISCV::fixup_riscv_branch:
+  case RISCV::fixup_riscv_rvc_jump:
+  case RISCV::fixup_riscv_rvc_branch:
+  case RISCV::fixup_riscv_call:
+  case RISCV::fixup_riscv_call_plt:
+  case RISCV::fixup_riscv_qc_e_branch:
+  case RISCV::fixup_riscv_qc_e_call_plt:
+  case RISCV::fixup_riscv_nds_branch_10:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // Expand PseudoCALL(Reg), PseudoTAIL and PseudoJump to AUIPC and JALR with
 // relocation types. We expand those pseudo-instructions while encoding them,
 // meaning AUIPC and JALR won't go through RISC-V MC to MC compressed
@@ -181,8 +203,7 @@ void RISCVMCCodeEmitter::expandTLSDESCCall(const MCInst &MI,
   MCRegister Link = MI.getOperand(0).getReg();
   MCRegister Dest = MI.getOperand(1).getReg();
   int64_t Imm = MI.getOperand(2).getImm();
-  Fixups.push_back(
-      MCFixup::create(0, Expr, ELF::R_RISCV_TLSDESC_CALL, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, ELF::R_RISCV_TLSDESC_CALL);
   MCInst Call =
       MCInstBuilder(RISCV::JALR).addReg(Link).addReg(Dest).addImm(Imm);
 
@@ -209,8 +230,7 @@ void RISCVMCCodeEmitter::expandAddTPRel(const MCInst &MI,
   assert(Expr && Expr->getSpecifier() == ELF::R_RISCV_TPREL_ADD &&
          "Expected tprel_add relocation on TP-relative symbol");
 
-  Fixups.push_back(
-      MCFixup::create(0, Expr, ELF::R_RISCV_TPREL_ADD, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, ELF::R_RISCV_TPREL_ADD);
   if (STI.hasFeature(RISCV::FeatureRelax))
     Fixups.back().setLinkerRelaxable();
 
@@ -320,11 +340,8 @@ void RISCVMCCodeEmitter::expandLongCondBr(const MCInst &MI,
   // Drop any fixup added so we can add the correct one.
   Fixups.resize(FixupStartIndex);
 
-  if (SrcSymbol.isExpr()) {
-    Fixups.push_back(MCFixup::create(Offset, SrcSymbol.getExpr(),
-                                     MCFixupKind(RISCV::fixup_riscv_jal),
-                                     MI.getLoc()));
-  }
+  if (SrcSymbol.isExpr())
+    addFixup(Fixups, Offset, SrcSymbol.getExpr(), RISCV::fixup_riscv_jal);
 }
 
 // Expand PseudoLongQC_(E_)Bxxx to an inverted conditional branch and an
@@ -371,11 +388,8 @@ void RISCVMCCodeEmitter::expandQCLongCondBrImm(const MCInst &MI,
   support::endian::write(CB, JBinary, llvm::endianness::little);
   // Drop any fixup added so we can add the correct one.
   Fixups.resize(FixupStartIndex);
-  if (SrcSymbol.isExpr()) {
-    Fixups.push_back(MCFixup::create(Offset, SrcSymbol.getExpr(),
-                                     MCFixupKind(RISCV::fixup_riscv_jal),
-                                     MI.getLoc()));
-  }
+  if (SrcSymbol.isExpr())
+    addFixup(Fixups, Offset, SrcSymbol.getExpr(), RISCV::fixup_riscv_jal);
 }
 
 void RISCVMCCodeEmitter::encodeInstruction(const MCInst &MI,
@@ -653,7 +667,7 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
 
   assert(FixupKind != RISCV::fixup_riscv_invalid && "Unhandled expression!");
 
-  Fixups.push_back(MCFixup::create(0, Expr, FixupKind, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, FixupKind);
   // If linker relaxation is enabled and supported by this relocation, set
   // a bit so that if fixup is unresolved, a R_RISCV_RELAX relocation will be
   // appended.
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index b078b9268c984..d4f5d8fcad6bc 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -305,8 +305,7 @@ void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
 }
 
 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
-  RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
-                                        getSubtargetInfo().getFeatureBits());
+  RISCV_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
 
   emitNTLHint(MI);
 
@@ -517,12 +516,9 @@ void RISCVAsmPrinter::emitSled(const MachineInstr *MI, SledKind Kind) {
   // Assuming we're using JAL to jump to .tmpN, then we only need
   // (68 - 4)/2 = 32 NOPs for RV64 and (44 - 4)/2 = 20 for RV32. However, there
   // is a chance that we'll use C.JAL instead, so an additional NOP is needed.
-  const uint8_t NoopsInSledCount =
-      MI->getParent()->getParent()->getSubtarget<RISCVSubtarget>().is64Bit()
-          ? 33
-          : 21;
+  const uint8_t NoopsInSledCount = STI->is64Bit() ? 33 : 21;
 
-  OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
+  OutStreamer->emitCodeAlignment(Align(4), STI);
   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
   OutStreamer->emitLabel(CurSled);
   auto Target = OutContext.createTempSymbol();
@@ -1070,7 +1066,8 @@ bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
 }
 
 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
-                                            MCInst &OutMI) {
+                                            MCInst &OutMI,
+                                            const RISCVSubtarget *STI) {
   const RISCVVPseudosTable::PseudoInfo *RVV =
       RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
   if (!RVV)
@@ -1078,14 +1075,8 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
 
   OutMI.setOpcode(RVV->BaseInstr);
 
-  const MachineBasicBlock *MBB = MI->getParent();
-  assert(MBB && "MI expected to be in a basic block");
-  const MachineFunction *MF = MBB->getParent();
-  assert(MF && "MBB expected to be in a machine function");
-
-  const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
-  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
-  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
+  const TargetInstrInfo *TII = STI->getInstrInfo();
+  const TargetRegisterInfo *TRI = STI->getRegisterInfo();
   assert(TRI && "TargetRegisterInfo expected");
 
   const MCInstrDesc &MCID = MI->getDesc();
@@ -1182,7 +1173,7 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
 }
 
 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
-  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
+  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI, STI))
     return false;
 
   OutMI.setOpcode(MI->getOpcode());
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 6e103dd7d8c44..addeab8d9aba6 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1404,6 +1404,13 @@ def HasVendorXMIPSLSP
     : Predicate<"Subtarget->hasVendorXMIPSLSP()">,
       AssemblerPredicate<(all_of FeatureVendorXMIPSLSP),
                          "'Xmipslsp' (load and store pair instructions)">;
+def FeatureVendorXMIPSCBOP
+    : RISCVExtension<1, 0, "MIPS Software Prefetch">;
+def HasVendorXMIPSCBOP
+    : Predicate<"Subtarget->hasVendorXMIPSCBOP()">,
+      AssemblerPredicate<(all_of FeatureVendorXMIPSCBOP),
+                         "'Xmipscbop' (MIPS hardware prefetch)">;
+def NotHasVendorXMIPSCBOP : Predicate<"!Subtarget->hasVendorXMIPSCBOP()">;
 
 // WCH / Nanjing Qinheng Microelectronics Extension(s)
 
@@ -1600,10 +1607,17 @@ def HasVendorXAndesVBFHCvt
       AssemblerPredicate<(all_of FeatureVendorXAndesVBFHCvt),
                          "'XAndesVBFHCvt' (Andes Vector BFLOAT16 Conversion Extension)">;
 
+def FeatureVendorXAndesVSIntLoad
+    : RISCVExtension<5, 0, "Andes Vector INT4 Load Extension",
+                     [FeatureStdExtZve32x]>;
+def HasVendorXAndesVSIntLoad
+    : Predicate<"Subtarget->hasVendorXAndesVSIntLoad()">,
+      AssemblerPredicate<(all_of FeatureVendorXAndesVSIntLoad),
+                         "'XAndesVSIntLoad' (Andes Vector INT4 Load Extension)">;
 
 def FeatureVendorXAndesVPackFPH
-    : RISCVExtension<5, 0, "Andes Vector Packed FP16 Extension",
-                     [FeatureStdExtZvfhmin]>;
+    : RISCVExtension<5, 0,
+                     "Andes Vector Packed FP16 Extension", [FeatureStdExtF]>;
 def HasVendorXAndesVPackFPH
     : Predicate<"Subtarget->hasVendorXAndesVPackFPH()">,
       AssemblerPredicate<(all_of FeatureVendorXAndesVPackFPH),
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 4539efd591c8b..1a6f65a789a28 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -163,8 +163,6 @@ void RISCVDAGToDAGISel::PostprocessISelDAG() {
 
   CurDAG->setRoot(Dummy.getValue());
 
-  MadeChange |= doPeepholeMergeVVMFold();
-
   // After we're done with everything else, convert IMPLICIT_DEF
   // passthru operands to NoRegister.  This is required to workaround
   // an optimization deficiency in MachineCSE.  This really should
@@ -683,6 +681,43 @@ bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
   return false;
 }
 
+bool RISCVDAGToDAGISel::trySignedBitfieldInsertInMask(SDNode *Node) {
+  // Supported only in Xqcibm for now.
+  if (!Subtarget->hasVendorXqcibm())
+    return false;
+
+  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
+  if (!N1C)
+    return false;
+
+  int32_t C1 = N1C->getSExtValue();
+  if (!isShiftedMask_32(C1) || isInt<12>(C1))
+    return false;
+
+  // If C1 is a shifted mask (but can't be formed as an ORI),
+  // use a bitfield insert of -1.
+  // Transform (or x, C1)
+  //        -> (qc.insbi x, width, shift)
+  const unsigned Leading = llvm::countl_zero((uint32_t)C1);
+  const unsigned Trailing = llvm::countr_zero((uint32_t)C1);
+  const unsigned Width = 32 - Leading - Trailing;
+
+  // If Zbs is enabled and it is a single bit set we can use BSETI which
+  // can be compressed to C_BSETI when Xqcibm in enabled.
+  if (Width == 1 && Subtarget->hasStdExtZbs())
+    return false;
+
+  SDLoc DL(Node);
+  MVT VT = Node->getSimpleValueType(0);
+
+  SDValue Ops[] = {CurDAG->getSignedTargetConstant(-1, DL, VT),
+                   CurDAG->getTargetConstant(Width, DL, VT),
+                   CurDAG->getTargetConstant(Trailing, DL, VT)};
+  SDNode *BitIns = CurDAG->getMachineNode(RISCV::QC_INSBI, DL, VT, Ops);
+  ReplaceNode(Node, BitIns);
+  return true;
+}
+
 bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
   // Only supported with XAndesPerf at the moment.
   if (!Subtarget->hasVendorXAndesPerf())
@@ -879,32 +914,32 @@ void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
   auto *LMulSDNode = cast<ConstantSDNode>(Node->getOperand(7));
   switch (LMulSDNode->getSExtValue()) {
   case 5:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_MF8
-                                                  : RISCV::PseudoVC_I_SE_MF8;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
+                                                  : RISCV::PseudoSF_VC_I_SE_MF8;
     break;
   case 6:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_MF4
-                                                  : RISCV::PseudoVC_I_SE_MF4;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
+                                                  : RISCV::PseudoSF_VC_I_SE_MF4;
     break;
   case 7:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_MF2
-                                                  : RISCV::PseudoVC_I_SE_MF2;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
+                                                  : RISCV::PseudoSF_VC_I_SE_MF2;
     break;
   case 0:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M1
-                                                  : RISCV::PseudoVC_I_SE_M1;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
+                                                  : RISCV::PseudoSF_VC_I_SE_M1;
     break;
   case 1:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M2
-                                                  : RISCV::PseudoVC_I_SE_M2;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
+                                                  : RISCV::PseudoSF_VC_I_SE_M2;
     break;
   case 2:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M4
-                                                  : RISCV::PseudoVC_I_SE_M4;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
+                                                  : RISCV::PseudoSF_VC_I_SE_M4;
     break;
   case 3:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M8
-                                                  : RISCV::PseudoVC_I_SE_M8;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
+                                                  : RISCV::PseudoSF_VC_I_SE_M8;
     break;
   }
 
@@ -1300,7 +1335,15 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     ReplaceNode(Node, SRAI);
     return;
   }
-  case ISD::OR:
+  case ISD::OR: {
+    if (trySignedBitfieldInsertInMask(Node))
+      return;
+
+    if (tryShrinkShlLogicImm(Node))
+      return;
+
+    break;
+  }
   case ISD::XOR:
     if (tryShrinkShlLogicImm(Node))
       return;
@@ -2824,10 +2867,6 @@ bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
       Base = Addr.getOperand(0);
       return true;
     }
-  } else if (UnwrapShl(Addr, Index, Scale)) {
-    EVT VT = Addr.getValueType();
-    Base = CurDAG->getRegister(RISCV::X0, VT);
-    return true;
   }
 
   return false;
@@ -2925,6 +2964,33 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
   return true;
 }
 
+/// Similar to SelectAddrRegImm, except that the offset restricted for
+/// unsinged nine bits.
+bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
+                                          SDValue &Offset) {
+  if (SelectAddrFrameIndex(Addr, Base, Offset))
+    return true;
+
+  SDLoc DL(Addr);
+  MVT VT = Addr.getSimpleValueType();
+
+  if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
+    if (isUInt<9>(CVal)) {
+      Base = Addr.getOperand(0);
+
+      if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
+        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
+      Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
+      return true;
+    }
+  }
+
+  Base = Addr;
+  Offset = CurDAG->getTargetConstant(0, DL, VT);
+  return true;
+}
+
 /// Similar to SelectAddrRegImm, except that the least significant 5 bits of
 /// Offset should be all zeros.
 bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
@@ -4069,218 +4135,6 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
   return true;
 }
 
-static bool IsVMerge(SDNode *N) {
-  return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMERGE_VVM;
-}
-
-// Try to fold away VMERGE_VVM instructions into their true operands:
-//
-// %true = PseudoVADD_VV ...
-// %x = PseudoVMERGE_VVM %false, %false, %true, %mask
-// ->
-// %x = PseudoVADD_VV_MASK %false, ..., %mask
-//
-// We can only fold if vmerge's passthru operand, vmerge's false operand and
-// %true's passthru operand (if it has one) are the same. This is because we
-// have to consolidate them into one passthru operand in the result.
-//
-// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
-// mask is all ones.
-//
-// The resulting VL is the minimum of the two VLs.
-//
-// The resulting policy is the effective policy the vmerge would have had,
-// i.e. whether or not it's passthru operand was implicit-def.
-bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
-  SDValue Passthru, False, True, VL, Mask;
-  assert(IsVMerge(N));
-  Passthru = N->getOperand(0);
-  False = N->getOperand(1);
-  True = N->getOperand(2);
-  Mask = N->getOperand(3);
-  VL = N->getOperand(4);
-
-  // If the EEW of True is different from vmerge's SEW, then we can't fold.
-  if (True.getSimpleValueType() != N->getSimpleValueType(0))
-    return false;
-
-  // We require that either passthru and false are the same, or that passthru
-  // is undefined.
-  if (Passthru != False && !isImplicitDef(Passthru))
-    return false;
-
-  assert(True.getResNo() == 0 &&
-         "Expect True is the first output of an instruction.");
-
-  // Need N is the exactly one using True.
-  if (!True.hasOneUse())
-    return false;
-
-  if (!True.isMachineOpcode())
-    return false;
-
-  unsigned TrueOpc = True.getMachineOpcode();
-  const MCInstrDesc &TrueMCID = TII->get(TrueOpc);
-  uint64_t TrueTSFlags = TrueMCID.TSFlags;
-  bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(TrueMCID);
-
-  const RISCV::RISCVMaskedPseudoInfo *Info =
-      RISCV::lookupMaskedIntrinsicByUnmasked(TrueOpc);
-  if (!Info)
-    return false;
-
-  // If True has a passthru operand then it needs to be the same as vmerge's
-  // False, since False will be used for the result's passthru operand.
-  if (HasTiedDest && !isImplicitDef(True->getOperand(0))) {
-    SDValue PassthruOpTrue = True->getOperand(0);
-    if (False != PassthruOpTrue)
-      return false;
-  }
-
-  // Skip if True has side effect.
-  if (TII->get(TrueOpc).hasUnmodeledSideEffects())
-    return false;
-
-  unsigned TrueChainOpIdx = True.getNumOperands() - 1;
-  bool HasChainOp =
-      True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other;
-
-  if (HasChainOp) {
-    // Avoid creating cycles in the DAG. We must ensure that none of the other
-    // operands depend on True through it's Chain.
-    SmallVector<const SDNode *, 4> LoopWorklist;
-    SmallPtrSet<const SDNode *, 16> Visited;
-    LoopWorklist.push_back(False.getNode());
-    LoopWorklist.push_back(Mask.getNode());
-    LoopWorklist.push_back(VL.getNode());
-    if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
-      return false;
-  }
-
-  // The vector policy operand may be present for masked intrinsics
-  bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags);
-  unsigned TrueVLIndex =
-      True.getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
-  SDValue TrueVL = True.getOperand(TrueVLIndex);
-  SDValue SEW = True.getOperand(TrueVLIndex + 1);
-
-  auto GetMinVL = [](SDValue LHS, SDValue RHS) {
-    if (LHS == RHS)
-      return LHS;
-    if (isAllOnesConstant(LHS))
-      return RHS;
-    if (isAllOnesConstant(RHS))
-      return LHS;
-    auto *CLHS = dyn_cast<ConstantSDNode>(LHS);
-    auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
-    if (!CLHS || !CRHS)
-      return SDValue();
-    return CLHS->getZExtValue() <= CRHS->getZExtValue() ? LHS : RHS;
-  };
-
-  // Because N and True must have the same passthru operand (or True's operand
-  // is implicit_def), the "effective" body is the minimum of their VLs.
-  SDValue OrigVL = VL;
-  VL = GetMinVL(TrueVL, VL);
-  if (!VL)
-    return false;
-
-  // Some operations produce different elementwise results depending on the
-  // active elements, like viota.m or vredsum. This transformation is illegal
-  // for these if we change the active elements (i.e. mask or VL).
-  const MCInstrDesc &TrueBaseMCID = TII->get(RISCV::getRVVMCOpcode(TrueOpc));
-  if (RISCVII::elementsDependOnVL(TrueBaseMCID.TSFlags) && (TrueVL != VL))
-    return false;
-  if (RISCVII::elementsDependOnMask(TrueBaseMCID.TSFlags) &&
-      (Mask && !usesAllOnesMask(Mask)))
-    return false;
-
-  // Make sure it doesn't raise any observable fp exceptions, since changing the
-  // active elements will affect how fflags is set.
-  if (mayRaiseFPException(True.getNode()) && !True->getFlags().hasNoFPExcept())
-    return false;
-
-  SDLoc DL(N);
-
-  unsigned MaskedOpc = Info->MaskedPseudo;
-#ifndef NDEBUG
-  const MCInstrDesc &MaskedMCID = TII->get(MaskedOpc);
-  assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) &&
-         "Expected instructions with mask have policy operand.");
-  assert(MaskedMCID.getOperandConstraint(MaskedMCID.getNumDefs(),
-                                         MCOI::TIED_TO) == 0 &&
-         "Expected instructions with mask have a tied dest.");
-#endif
-
-  // Use a tumu policy, relaxing it to tail agnostic provided that the passthru
-  // operand is undefined.
-  //
-  // However, if the VL became smaller than what the vmerge had originally, then
-  // elements past VL that were previously in the vmerge's body will have moved
-  // to the tail. In that case we always need to use tail undisturbed to
-  // preserve them.
-  bool MergeVLShrunk = VL != OrigVL;
-  uint64_t Policy = (isImplicitDef(Passthru) && !MergeVLShrunk)
-                        ? RISCVVType::TAIL_AGNOSTIC
-                        : /*TUMU*/ 0;
-  SDValue PolicyOp =
-    CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT());
-
-
-  SmallVector<SDValue, 8> Ops;
-  Ops.push_back(False);
-
-  const bool HasRoundingMode = RISCVII::hasRoundModeOp(TrueTSFlags);
-  const unsigned NormalOpsEnd = TrueVLIndex - HasRoundingMode;
-  Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + NormalOpsEnd);
-
-  Ops.push_back(Mask);
-
-  // For unmasked "VOp" with rounding mode operand, that is interfaces like
-  // (..., rm, vl) or (..., rm, vl, policy).
-  // Its masked version is (..., vm, rm, vl, policy).
-  // Check the rounding mode pseudo nodes under RISCVInstrInfoVPseudos.td
-  if (HasRoundingMode)
-    Ops.push_back(True->getOperand(TrueVLIndex - 1));
-
-  Ops.append({VL, SEW, PolicyOp});
-
-  // Result node should have chain operand of True.
-  if (HasChainOp)
-    Ops.push_back(True.getOperand(TrueChainOpIdx));
-
-  MachineSDNode *Result =
-      CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
-  Result->setFlags(True->getFlags());
-
-  if (!cast<MachineSDNode>(True)->memoperands_empty())
-    CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(True)->memoperands());
-
-  // Replace vmerge.vvm node by Result.
-  ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
-
-  // Replace another value of True. E.g. chain and VL.
-  for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
-    ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
-
-  return true;
-}
-
-bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
-  bool MadeChange = false;
-  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
-
-  while (Position != CurDAG->allnodes_begin()) {
-    SDNode *N = &*--Position;
-    if (N->use_empty() || !N->isMachineOpcode())
-      continue;
-
-    if (IsVMerge(N))
-      MadeChange |= performCombineVMergeAndVOps(N);
-  }
-  return MadeChange;
-}
-
 /// If our passthru is an implicit_def, use noreg instead.  This side
 /// steps issues with MachineCSE not being able to CSE expressions with
 /// IMPLICIT_DEF operands while preserving the semantic intent. See
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index cb63c21fd8fc9..29ee3ae31606e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -47,6 +47,7 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
 
   bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset);
   bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset);
+  bool SelectAddrRegImm9(SDValue Addr, SDValue &Base, SDValue &Offset);
   bool SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, SDValue &Offset);
 
   bool SelectAddrRegRegScale(SDValue Addr, unsigned MaxShiftAmount,
@@ -78,6 +79,7 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
   bool tryShrinkShlLogicImm(SDNode *Node);
   bool trySignedBitfieldExtract(SDNode *Node);
   bool trySignedBitfieldInsertInSign(SDNode *Node);
+  bool trySignedBitfieldInsertInMask(SDNode *Node);
   bool tryUnsignedBitfieldExtract(SDNode *Node, const SDLoc &DL, MVT VT,
                                   SDValue X, unsigned Msb, unsigned Lsb);
   bool tryUnsignedBitfieldInsertInZero(SDNode *Node, const SDLoc &DL, MVT VT,
@@ -199,7 +201,6 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
 private:
   bool doPeepholeSExtW(SDNode *Node);
   bool doPeepholeMaskedRVV(MachineSDNode *Node);
-  bool doPeepholeMergeVVMFold();
   bool doPeepholeNoRegPassThru();
   bool performCombineVMergeAndVOps(SDNode *N);
   bool selectImm64IfCheaper(int64_t Imm, int64_t OrigImm, SDValue N,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 6261ae69a4d3e..5126ab6c31c28 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -211,7 +211,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       addRegClassForRVV(VT);
     }
 
-    if (Subtarget.hasVInstructionsF16Minimal())
+    if (Subtarget.hasVInstructionsF16Minimal() ||
+        Subtarget.hasVendorXAndesVPackFPH())
       for (MVT VT : F16VecVTs)
         addRegClassForRVV(VT);
 
@@ -683,9 +684,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   if (Subtarget.is64Bit())
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
 
-  if (Subtarget.hasStdExtZicbop()) {
+  if (Subtarget.hasVendorXMIPSCBOP())
+    setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
+  else if (Subtarget.hasStdExtZicbop())
     setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
-  }
 
   if (Subtarget.hasStdExtA()) {
     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
@@ -1505,7 +1507,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                            VT, Custom);
 
         setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
-                            ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT},
+                            ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
+                            ISD::LLRINT, ISD::FNEARBYINT},
                            VT, Custom);
 
         setCondCodeAction(VFPCCToExpand, VT, Expand);
@@ -2415,12 +2418,15 @@ unsigned RISCVTargetLowering::getVectorTypeBreakdownForCallingConv(
 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
 // with 1/-1.
 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
-                                    ISD::CondCode &CC, SelectionDAG &DAG) {
+                                    ISD::CondCode &CC, SelectionDAG &DAG,
+                                    const RISCVSubtarget &Subtarget) {
   // If this is a single bit test that can't be handled by ANDI, shift the
   // bit to be tested to the MSB and perform a signed compare with 0.
   if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
       LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
-      isa<ConstantSDNode>(LHS.getOperand(1))) {
+      isa<ConstantSDNode>(LHS.getOperand(1)) &&
+      // XAndesPerf supports branch on test bit.
+      !Subtarget.hasVendorXAndesPerf()) {
     uint64_t Mask = LHS.getConstantOperandVal(1);
     if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
       unsigned ShAmt = 0;
@@ -2441,8 +2447,6 @@ static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
 
   if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
     int64_t C = RHSC->getSExtValue();
-    const RISCVSubtarget &Subtarget =
-        DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
     switch (CC) {
     default: break;
     case ISD::SETGT:
@@ -3208,7 +3212,14 @@ static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
   case ISD::VP_FROUND:
     return RISCVFPRndMode::RMM;
   case ISD::FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
+  case ISD::STRICT_FRINT:
+  case ISD::STRICT_LRINT:
+  case ISD::STRICT_LLRINT:
   case ISD::VP_FRINT:
+  case ISD::VP_LRINT:
+  case ISD::VP_LLRINT:
     return RISCVFPRndMode::DYN;
   }
 
@@ -3229,6 +3240,9 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
 
   SDValue Src = Op.getOperand(0);
 
+  // Freeze the source since we are increasing the number of uses.
+  Src = DAG.getFreeze(Src);
+
   MVT ContainerVT = VT;
   if (VT.isFixedLengthVector()) {
     ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
@@ -3246,9 +3260,6 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
   }
 
-  // Freeze the source since we are increasing the number of uses.
-  Src = DAG.getFreeze(Src);
-
   // We do the conversion on the absolute value and fix the sign at the end.
   SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
 
@@ -3458,28 +3469,34 @@ lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
 // Expand vector LRINT and LLRINT by converting to the integer domain.
 static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
-  MVT VT = Op.getSimpleValueType();
-  assert(VT.isVector() && "Unexpected type");
-
   SDLoc DL(Op);
+  MVT DstVT = Op.getSimpleValueType();
   SDValue Src = Op.getOperand(0);
-  MVT ContainerVT = VT;
+  MVT SrcVT = Src.getSimpleValueType();
+  assert(SrcVT.isVector() && DstVT.isVector() &&
+         !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
+         "Unexpected type");
 
-  if (VT.isFixedLengthVector()) {
-    ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
-    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
+  MVT DstContainerVT = DstVT;
+  MVT SrcContainerVT = SrcVT;
+
+  if (DstVT.isFixedLengthVector()) {
+    DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget);
+    SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
+    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
   }
 
-  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
-  SDValue Truncated = DAG.getNode(
-      RISCVISD::VFCVT_RM_X_F_VL, DL, ContainerVT, Src, Mask,
-      DAG.getTargetConstant(RISCVFPRndMode::DYN, DL, Subtarget.getXLenVT()),
-      VL);
+  auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
+  SDValue Res =
+      DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
+                  DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
+                                        Subtarget.getXLenVT()),
+                  VL);
 
-  if (!VT.isFixedLengthVector())
-    return Truncated;
+  if (!DstVT.isFixedLengthVector())
+    return Res;
 
-  return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
+  return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
 }
 
 static SDValue
@@ -4222,8 +4239,8 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) || EltVT == MVT::bf16) {
     MVT IVT = VT.changeVectorElementType(MVT::i16);
     SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
-    for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
-      SDValue Elem = Op.getOperand(I);
+    for (const auto &[I, U] : enumerate(Op->ops())) {
+      SDValue Elem = U.get();
       if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
           (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
         // Called by LegalizeDAG, we need to use XLenVT operations since we
@@ -4362,16 +4379,16 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
     SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
     SubVecAOps.reserve(NumElts);
     SubVecBOps.reserve(NumElts);
-    for (unsigned i = 0; i < NumElts; i++) {
-      SDValue Elem = Op->getOperand(i);
-      if (i < NumElts / 2) {
+    for (const auto &[Idx, U] : enumerate(Op->ops())) {
+      SDValue Elem = U.get();
+      if (Idx < NumElts / 2) {
         SubVecAOps.push_back(Elem);
         SubVecBOps.push_back(UndefElem);
       } else {
         SubVecAOps.push_back(UndefElem);
         SubVecBOps.push_back(Elem);
       }
-      bool SelectMaskVal = (i < NumElts / 2);
+      bool SelectMaskVal = (Idx < NumElts / 2);
       MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
     }
     assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
@@ -4752,16 +4769,15 @@ static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
                           ArrayRef<int> Mask, unsigned Factor,
                           bool RequiredPolarity) {
   int NumElts = Mask.size();
-  for (int i = 0; i != NumElts; ++i) {
-    int M = Mask[i];
+  for (const auto &[Idx, M] : enumerate(Mask)) {
     if (M < 0)
       continue;
     int Src = M >= NumElts;
-    int Diff = (int)i - (M % NumElts);
+    int Diff = (int)Idx - (M % NumElts);
     bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
     assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
            "Must match exactly one of the two slides");
-    if (RequiredPolarity != (C == (i / Factor) % 2))
+    if (RequiredPolarity != (C == (Idx / Factor) % 2))
       return false;
   }
   return true;
@@ -5486,18 +5502,18 @@ static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
 static bool isCompressMask(ArrayRef<int> Mask) {
   int Last = -1;
   bool SawUndef = false;
-  for (unsigned i = 0; i < Mask.size(); i++) {
-    if (Mask[i] == -1) {
+  for (const auto &[Idx, M] : enumerate(Mask)) {
+    if (M == -1) {
       SawUndef = true;
       continue;
     }
     if (SawUndef)
       return false;
-    if (i > (unsigned)Mask[i])
+    if (Idx > (unsigned)M)
       return false;
-    if (Mask[i] <= Last)
+    if (M <= Last)
       return false;
-    Last = Mask[i];
+    Last = M;
   }
   return true;
 }
@@ -5889,8 +5905,8 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     // otherwise reuse the even stream for the undef one.  This improves
     // spread(2) shuffles.
     bool LaneIsUndef[2] = { true, true};
-    for (unsigned i = 0; i < Mask.size(); i++)
-      LaneIsUndef[i % 2] &= (Mask[i] == -1);
+    for (const auto &[Idx, M] : enumerate(Mask))
+      LaneIsUndef[Idx % 2] &= (M == -1);
 
     int Size = Mask.size();
     SDValue EvenV, OddV;
@@ -6002,14 +6018,14 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     // undefined in the mask.  If the mask ends up all true (or undef), it
     // will be folded away by general logic.
     SmallVector<SDValue> MaskVals;
-    for (unsigned i = 0; i != Mask.size(); ++i) {
-      int M = Mask[i];
-      if (M < 0 || (SrcInfo[1].second > 0 && i < (unsigned)SrcInfo[1].second)) {
+    for (const auto &[Idx, M] : enumerate(Mask)) {
+      if (M < 0 ||
+          (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
         MaskVals.push_back(DAG.getUNDEF(XLenVT));
         continue;
       }
       int Src = M >= (int)NumElts;
-      int Diff = (int)i - (M % NumElts);
+      int Diff = (int)Idx - (M % NumElts);
       bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
       assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
              "Must match exactly one of the two slides");
@@ -6596,6 +6612,17 @@ SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
   return DAG.getNode(ISD::FNEG, DL, VT, Const);
 }
 
+static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
+                             SelectionDAG &DAG) {
+
+  unsigned IsData = Op.getConstantOperandVal(4);
+
+  // mips-p8700  we support data prefetch for now.
+  if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
+    return Op.getOperand(0);
+  return Op;
+}
+
 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
   SDLoc dl(Op);
@@ -7044,13 +7071,10 @@ static unsigned getRISCVVLOp(SDValue Op) {
 
 static bool isPromotedOpNeedingSplit(SDValue Op,
                                      const RISCVSubtarget &Subtarget) {
-  if (Op.getValueType() == MVT::nxv32f16 &&
-      (Subtarget.hasVInstructionsF16Minimal() &&
-       !Subtarget.hasVInstructionsF16()))
-    return true;
-  if (Op.getValueType() == MVT::nxv32bf16)
-    return true;
-  return false;
+  return (Op.getValueType() == MVT::nxv32f16 &&
+          (Subtarget.hasVInstructionsF16Minimal() &&
+           !Subtarget.hasVInstructionsF16())) ||
+         Op.getValueType() == MVT::nxv32bf16;
 }
 
 static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
@@ -7164,6 +7188,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   switch (Op.getOpcode()) {
   default:
     report_fatal_error("unimplemented operand");
+  case ISD::PREFETCH:
+    return LowerPREFETCH(Op, Subtarget, DAG);
   case ISD::ATOMIC_FENCE:
     return LowerATOMIC_FENCE(Op, DAG, Subtarget);
   case ISD::GlobalAddress:
@@ -9210,7 +9236,7 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
       return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
   }
 
-  translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
+  translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
   // 1 < x ? x : 1 -> 0 < x ? x : 1
   if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
       RHS == TrueV && LHS == FalseV) {
@@ -9252,7 +9278,7 @@ SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
     SDValue RHS = CondV.getOperand(1);
     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
 
-    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
+    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
 
     SDValue TargetCC = DAG.getCondCode(CCVal);
     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
@@ -9312,9 +9338,6 @@ SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
   MVT XLenVT = Subtarget.getXLenVT();
   int XLenInBytes = Subtarget.getXLen() / 8;
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
@@ -18196,7 +18219,7 @@ static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
 
     RHS = LHS.getOperand(1);
     LHS = LHS.getOperand(0);
-    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
+    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
 
     CC = DAG.getCondCode(CCVal);
     return true;
@@ -18262,6 +18285,14 @@ static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
       uint64_t Mask = LHS0.getConstantOperandVal(1);
       uint64_t ShAmt = LHS.getConstantOperandVal(1);
       if (isPowerOf2_64(Mask) && Log2_64(Mask) == ShAmt) {
+        // XAndesPerf supports branch on test bit.
+        if (Subtarget.hasVendorXAndesPerf()) {
+          LHS =
+              DAG.getNode(ISD::AND, DL, LHS.getValueType(), LHS0.getOperand(0),
+                          DAG.getConstant(Mask, DL, LHS.getValueType()));
+          return true;
+        }
+
         CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
         CC = DAG.getCondCode(CCVal);
 
@@ -18303,6 +18334,8 @@ static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
 // (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
 // (select C, (or Y, X), Y)  -> (or Y, (select C, X, 0)).
 // (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
+// (select C, (rotl Y, X), Y) -> (rotl Y, (select C, X, 0)).
+// (select C, (rotr Y, X), Y) -> (rotr Y, (select C, X, 0)).
 static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
                                    SDValue TrueVal, SDValue FalseVal,
                                    bool Swapped) {
@@ -18315,6 +18348,8 @@ static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
   case ISD::SRA:
   case ISD::SRL:
   case ISD::SUB:
+  case ISD::ROTL:
+  case ISD::ROTR:
     Commutative = false;
     break;
   case ISD::ADD:
@@ -21784,6 +21819,8 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
   case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
   case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
+  case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
+  case RISCV::Select_GPR_Using_CC_UImm7_NDS:
   case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR16INX_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
@@ -21900,22 +21937,21 @@ void RISCVTargetLowering::analyzeInputArgs(
     MachineFunction &MF, CCState &CCInfo,
     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
     RISCVCCAssignFn Fn) const {
-  unsigned NumArgs = Ins.size();
   FunctionType *FType = MF.getFunction().getFunctionType();
 
-  for (unsigned i = 0; i != NumArgs; ++i) {
-    MVT ArgVT = Ins[i].VT;
-    ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
+  for (const auto &[Idx, In] : enumerate(Ins)) {
+    MVT ArgVT = In.VT;
+    ISD::ArgFlagsTy ArgFlags = In.Flags;
 
     Type *ArgTy = nullptr;
     if (IsRet)
       ArgTy = FType->getReturnType();
-    else if (Ins[i].isOrigArg())
-      ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
+    else if (In.isOrigArg())
+      ArgTy = FType->getParamType(In.getOrigArgIndex());
 
-    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo,
+    if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo,
            /*IsFixed=*/true, IsRet, ArgTy)) {
-      LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
+      LLVM_DEBUG(dbgs() << "InputArg #" << Idx << " has unhandled type "
                         << ArgVT << '\n');
       llvm_unreachable(nullptr);
     }
@@ -21926,16 +21962,14 @@ void RISCVTargetLowering::analyzeOutputArgs(
     MachineFunction &MF, CCState &CCInfo,
     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
-  unsigned NumArgs = Outs.size();
-
-  for (unsigned i = 0; i != NumArgs; i++) {
-    MVT ArgVT = Outs[i].VT;
-    ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
-    Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
-
-    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo,
-           Outs[i].IsFixed, IsRet, OrigTy)) {
-      LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
+  for (const auto &[Idx, Out] : enumerate(Outs)) {
+    MVT ArgVT = Out.VT;
+    ISD::ArgFlagsTy ArgFlags = Out.Flags;
+    Type *OrigTy = CLI ? CLI->getArgs()[Out.OrigArgIndex].Ty : nullptr;
+
+    if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, Out.IsFixed,
+           IsRet, OrigTy)) {
+      LLVM_DEBUG(dbgs() << "OutputArg #" << Idx << " has unhandled type "
                         << ArgVT << "\n");
       llvm_unreachable(nullptr);
     }
@@ -24052,6 +24086,12 @@ static const Intrinsic::ID FixedVlsegIntrIds[] = {
     Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,
     Intrinsic::riscv_seg8_load_mask};
 
+static const Intrinsic::ID ScalableVlsegIntrIds[] = {
+    Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
+    Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
+    Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
+    Intrinsic::riscv_vlseg8_mask};
+
 /// Lower an interleaved load into a vlsegN intrinsic.
 ///
 /// E.g. Lower an interleaved load (Factor = 2):
@@ -24121,6 +24161,12 @@ static const Intrinsic::ID FixedVssegIntrIds[] = {
     Intrinsic::riscv_seg6_store_mask, Intrinsic::riscv_seg7_store_mask,
     Intrinsic::riscv_seg8_store_mask};
 
+static const Intrinsic::ID ScalableVssegIntrIds[] = {
+    Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
+    Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
+    Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
+    Intrinsic::riscv_vsseg8_mask};
+
 /// Lower an interleaved store into a vssegN intrinsic.
 ///
 /// E.g. Lower an interleaved store (Factor = 3):
@@ -24431,13 +24477,6 @@ bool RISCVTargetLowering::lowerInterleavedVPLoad(
                                      {FVTy, PtrTy, XLenTy},
                                      {Load->getArgOperand(0), Mask, EVL});
   } else {
-    static const Intrinsic::ID IntrMaskIds[] = {
-        Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
-        Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
-        Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
-        Intrinsic::riscv_vlseg8_mask,
-    };
-
     unsigned SEW = DL.getTypeSizeInBits(VTy->getElementType());
     unsigned NumElts = VTy->getElementCount().getKnownMinValue();
     Type *VecTupTy = TargetExtType::get(
@@ -24449,7 +24488,7 @@ bool RISCVTargetLowering::lowerInterleavedVPLoad(
     Value *PoisonVal = PoisonValue::get(VecTupTy);
 
     Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
-        Load->getModule(), IntrMaskIds[Factor - 2],
+        Load->getModule(), ScalableVlsegIntrIds[Factor - 2],
         {VecTupTy, PtrTy, Mask->getType(), EVL->getType()});
 
     Value *Operands[] = {
@@ -24549,13 +24588,6 @@ bool RISCVTargetLowering::lowerInterleavedVPStore(
     return true;
   }
 
-  static const Intrinsic::ID IntrMaskIds[] = {
-      Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
-      Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
-      Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
-      Intrinsic::riscv_vsseg8_mask,
-  };
-
   unsigned SEW = DL.getTypeSizeInBits(VTy->getElementType());
   unsigned NumElts = VTy->getElementCount().getKnownMinValue();
   Type *VecTupTy = TargetExtType::get(
@@ -24572,7 +24604,7 @@ bool RISCVTargetLowering::lowerInterleavedVPStore(
         VecInsertFunc, {StoredVal, InterleaveOperands[i], Builder.getInt32(i)});
 
   Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
-      Store->getModule(), IntrMaskIds[Factor - 2],
+      Store->getModule(), ScalableVssegIntrIds[Factor - 2],
       {VecTupTy, PtrTy, Mask->getType(), EVL->getType()});
 
   Value *Operands[] = {StoredVal, Store->getArgOperand(1), Mask, EVL,
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 78d64ea67324f..dd002473a06e2 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -1768,8 +1768,9 @@ void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
           SlotIndex NewDefSI =
               LIS->InsertMachineInstrInMaps(*ReadVLMI).getRegSlot();
           LiveInterval &DefLI = LIS->getInterval(VLOutput);
-          VNInfo *DefVNI = DefLI.getVNInfoAt(DefLI.beginIndex());
-          DefLI.removeSegment(DefLI.beginIndex(), NewDefSI);
+          LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(NewDefSI);
+          VNInfo *DefVNI = DefLI.getVNInfoAt(DefSeg->start);
+          DefLI.removeSegment(DefSeg->start, NewDefSI);
           DefVNI->def = NewDefSI;
         }
       }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 97247dc2068bc..2723229859a55 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -513,26 +513,24 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                  Register SrcReg, bool KillSrc,
                                  bool RenamableDest, bool RenamableSrc) const {
   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+  unsigned KillFlag = getKillRegState(KillSrc);
 
   if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
-        .addReg(SrcReg,
-                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc))
+        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc))
         .addImm(0);
     return;
   }
 
   if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR16INX), DstReg)
-        .addReg(SrcReg,
-                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc));
+        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
     return;
   }
 
   if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR32INX), DstReg)
-        .addReg(SrcReg,
-                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc));
+        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
     return;
   }
 
@@ -547,11 +545,11 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     // Emit an ADDI for both parts of GPRPair.
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
             TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
-        .addReg(EvenReg, getKillRegState(KillSrc))
+        .addReg(EvenReg, KillFlag)
         .addImm(0);
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
             TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
-        .addReg(OddReg, getKillRegState(KillSrc))
+        .addReg(OddReg, KillFlag)
         .addImm(0);
     return;
   }
@@ -581,36 +579,36 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       Opc = RISCV::FSGNJ_S;
     }
     BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag)
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag)
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag)
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::FPR32RegClass.contains(DstReg) &&
       RISCV::GPRRegClass.contains(SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::GPRRegClass.contains(DstReg) &&
       RISCV::FPR32RegClass.contains(SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
@@ -618,7 +616,7 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       RISCV::GPRRegClass.contains(SrcReg)) {
     assert(STI.getXLen() == 64 && "Unexpected GPR size");
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
@@ -626,7 +624,7 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       RISCV::FPR64RegClass.contains(SrcReg)) {
     assert(STI.getXLen() == 64 && "Unexpected GPR size");
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
@@ -966,11 +964,15 @@ RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
   case RISCV::CV_BEQIMM:
   case RISCV::QC_BEQI:
   case RISCV::QC_E_BEQI:
+  case RISCV::NDS_BBC:
+  case RISCV::NDS_BEQC:
     return RISCVCC::COND_EQ;
   case RISCV::BNE:
   case RISCV::QC_BNEI:
   case RISCV::QC_E_BNEI:
   case RISCV::CV_BNEIMM:
+  case RISCV::NDS_BBS:
+  case RISCV::NDS_BNEC:
     return RISCVCC::COND_NE;
   case RISCV::BLT:
   case RISCV::QC_BLTI:
@@ -1103,6 +1105,26 @@ unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
       return RISCV::QC_E_BGEUI;
     }
     break;
+  case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::NDS_BBC;
+    case RISCVCC::COND_NE:
+      return RISCV::NDS_BBS;
+    }
+    break;
+  case RISCV::Select_GPR_Using_CC_UImm7_NDS:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::NDS_BEQC;
+    case RISCVCC::COND_NE:
+      return RISCV::NDS_BNEC;
+    }
+    break;
   }
 }
 
@@ -1400,6 +1422,18 @@ bool RISCVInstrInfo::reverseBranchCondition(
   case RISCV::QC_E_BLTUI:
     Cond[0].setImm(RISCV::QC_E_BGEUI);
     break;
+  case RISCV::NDS_BBC:
+    Cond[0].setImm(RISCV::NDS_BBS);
+    break;
+  case RISCV::NDS_BBS:
+    Cond[0].setImm(RISCV::NDS_BBC);
+    break;
+  case RISCV::NDS_BEQC:
+    Cond[0].setImm(RISCV::NDS_BNEC);
+    break;
+  case RISCV::NDS_BNEC:
+    Cond[0].setImm(RISCV::NDS_BEQC);
+    break;
   }
 
   return false;
@@ -1572,6 +1606,11 @@ bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   switch (BranchOp) {
   default:
     llvm_unreachable("Unexpected opcode!");
+  case RISCV::NDS_BBC:
+  case RISCV::NDS_BBS:
+  case RISCV::NDS_BEQC:
+  case RISCV::NDS_BNEC:
+    return isInt<11>(BrOffset);
   case RISCV::BEQ:
   case RISCV::BNE:
   case RISCV::BLT:
@@ -1592,12 +1631,12 @@ bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   case RISCV::QC_E_BLTI:
   case RISCV::QC_E_BLTUI:
   case RISCV::QC_E_BGEUI:
-    return isIntN(13, BrOffset);
+    return isInt<13>(BrOffset);
   case RISCV::JAL:
   case RISCV::PseudoBR:
-    return isIntN(21, BrOffset);
+    return isInt<21>(BrOffset);
   case RISCV::PseudoJump:
-    return isIntN(32, SignExtend64(BrOffset + 0x800, XLen));
+    return isInt<32>(SignExtend64(BrOffset + 0x800, XLen));
   }
 }
 
@@ -2766,7 +2805,8 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         CASE_OPERAND_UIMM(6)
         CASE_OPERAND_UIMM(7)
         CASE_OPERAND_UIMM(8)
-        CASE_OPERAND_UIMM(10)
+        CASE_OPERAND_UIMM(9)
+	CASE_OPERAND_UIMM(10)
         CASE_OPERAND_UIMM(12)
         CASE_OPERAND_UIMM(16)
         CASE_OPERAND_UIMM(20)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index a25d4e67d1d73..8f8fb6eba9a62 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -321,7 +321,7 @@ def uimm6 : RISCVUImmLeafOp<6>;
 def uimm7_opcode : RISCVUImmOp<7> {
   let ParserMatchClass = InsnDirectiveOpcode;
 }
-def uimm7 : RISCVUImmOp<7>;
+def uimm7 : RISCVUImmLeafOp<7>;
 def uimm8 : RISCVUImmOp<8>;
 def uimm16 : RISCVUImmOp<16>;
 def uimm32 : RISCVUImmOp<32>;
@@ -1615,6 +1615,10 @@ def riscv_seteq : ComplexPattern<XLenVT, 1, "selectSETEQ", [setcc]>;
 def : Pat<(riscv_seteq (XLenVT GPR:$rs1)), (SLTIU GPR:$rs1, 1)>;
 def : Pat<(riscv_setne (XLenVT GPR:$rs1)), (SLTU (XLenVT X0), GPR:$rs1)>;
 def : Pat<(XLenVT (setne (XLenVT GPR:$rs1), -1)), (SLTIU GPR:$rs1, -1)>;
+def : Pat<(XLenVT (seteq (XLenVT (and GPR:$rs, immop_oneuse<TrailingOnesMask>:$mask)), 0)),
+          (SLTIU (XLenVT (SLLI GPR:$rs, (XLenSubTrailingOnes imm:$mask))), 1)>;
+def : Pat<(XLenVT (setne (XLenVT (and GPR:$rs, immop_oneuse<TrailingOnesMask>:$mask)), 0)),
+          (SLTU (XLenVT X0), (XLenVT (SLLI GPR:$rs, (XLenSubTrailingOnes imm:$mask))))>;
 
 def IntCCtoRISCVCC : SDNodeXForm<riscv_selectcc, [{
   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
@@ -2273,6 +2277,9 @@ include "RISCVInstrInfoV.td"
 include "RISCVInstrInfoZvk.td"
 include "RISCVInstrInfoZvqdotq.td"
 
+// Packed SIMD
+include "RISCVInstrInfoP.td"
+
 // Compressed
 include "RISCVInstrInfoC.td"
 include "RISCVInstrInfoZc.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
new file mode 100644
index 0000000000000..aa9e7b5635def
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -0,0 +1,163 @@
+//===-- RISCVInstrInfoP.td - RISC-V 'P' instructions -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the RISC-V instructions from the standard 'Base P'
+// Packed SIMD instruction set extension.
+//
+//  This version is still experimental as the 'P' extension hasn't been
+//  ratified yet.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Operand and SDNode transformation definitions.
+//===----------------------------------------------------------------------===//
+
+def simm10 : RISCVSImmLeafOp<10>;
+
+// A 10-bit signed immediate allowing range [-512, 1023]
+// but will decode to [-512, 511].
+def simm10_unsigned : RISCVOp {
+  let ParserMatchClass = SImmAsmOperand<10, "Unsigned">;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeSImmOperand<10>";
+  let OperandType = "OPERAND_SIMM10_UNSIGNED";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return isInt<10>(Imm) || isUInt<10>(Imm);
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Instruction class templates
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnaryImm10<bits<7> funct7, string opcodestr,
+                    DAGOperand TyImm10 = simm10>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins TyImm10:$imm10),
+                  opcodestr, "$rd, $imm10"> {
+  bits<10> imm10;
+
+  let Inst{31-25} = funct7;
+  let Inst{24-16} = imm10{8-0};
+  let Inst{15}    = imm10{9};
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnaryImm8<bits<8> funct8, string opcodestr>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins uimm8:$uimm8),
+                  opcodestr, "$rd, $uimm8"> {
+  bits<8> uimm8;
+
+  let Inst{31-24} = funct8;
+  let Inst{23-16} = uimm8;
+  let Inst{15}    = 0b0;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnary<bits<3> f, string opcodestr, dag operands, string argstr>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), operands, opcodestr, argstr> {
+  bits<5> imm;
+  bits<5> rs1;
+
+  let Inst{31}    = 0b1;
+  let Inst{30-28} = f;
+  let Inst{27}    = 0b0;
+  let Inst{19-15} = rs1;
+}
+
+class RVPUnaryImm5<bits<3> f, string opcodestr>
+    : RVPUnary<f, opcodestr, (ins GPR:$rs1, uimm5:$uimm5), "$rd, $rs1, $uimm5"> {
+  bits<5> uimm5;
+
+  let imm = uimm5;
+  let Inst{26-25} = 0b01;
+  let Inst{24-20} = uimm5;
+}
+
+class RVPUnaryImm4<bits<3> f, string opcodestr>
+    : RVPUnary<f, opcodestr, (ins GPR:$rs1, uimm4:$uimm4), "$rd, $rs1, $uimm4"> {
+  bits<4> uimm4;
+
+  let Inst{26-24} = 0b001;
+  let Inst{23-20} = uimm4;
+}
+
+class RVPUnaryImm3<bits<3> f, string opcodestr>
+    : RVPUnary<f, opcodestr, (ins GPR:$rs1, uimm3:$uimm3), "$rd, $rs1, $uimm3"> {
+  bits<3> uimm3;
+
+  let Inst{26-23} = 0b0001;
+  let Inst{22-20} = uimm3;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnaryWUF<bits<2> w, bits<5> uf, string opcodestr>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins GPR:$rs1),
+                  opcodestr, "$rd, $rs1">  {
+  let Inst{31-27} = 0b11100;
+  let Inst{26-25} = w;
+  let Inst{24-20} = uf;
+}
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtP] in {
+def CLS    : Unary_r<0b011000000011, 0b001, "cls">;
+def ABS    : Unary_r<0b011000000111, 0b001, "abs">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32] in
+def REV_RV32  : Unary_r<0b011010011111, 0b101, "rev">;
+
+let Predicates = [HasStdExtP, IsRV64] in {
+def REV16      : Unary_r<0b011010110000, 0b101, "rev16">;
+def REV_RV64   : Unary_r<0b011010111111, 0b101, "rev">;
+
+def CLSW  : UnaryW_r<0b011000000011, 0b001, "clsw">;
+def ABSW  : UnaryW_r<0b011000000111, 0b001, "absw">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+def PSLLI_B  : RVPUnaryImm3<0b000, "pslli.b">;
+def PSLLI_H  : RVPUnaryImm4<0b000, "pslli.h">;
+def PSSLAI_H : RVPUnaryImm4<0b101, "psslai.h">;
+} // Predicates = [HasStdExtP]
+let DecoderNamespace = "RV32Only",
+    Predicates = [HasStdExtP, IsRV32] in
+def SSLAI    : RVPUnaryImm5<0b101, "sslai">;
+let Predicates = [HasStdExtP, IsRV64] in {
+def PSLLI_W  : RVPUnaryImm5<0b000, "pslli.w">;
+def PSSLAI_W : RVPUnaryImm5<0b101, "psslai.w">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in
+def PLI_H : RVPUnaryImm10<0b1011000, "pli.h">;
+let Predicates = [HasStdExtP, IsRV64] in
+def PLI_W : RVPUnaryImm10<0b1011001, "pli.w">;
+let Predicates = [HasStdExtP] in
+def PLI_B : RVPUnaryImm8<0b10110100, "pli.b">;
+
+let Predicates = [HasStdExtP] in {
+def PSEXT_H_B : RVPUnaryWUF<0b00, 0b00100, "psext.h.b">;
+def PSABS_H   : RVPUnaryWUF<0b00, 0b00111, "psabs.h">;
+def PSABS_B   : RVPUnaryWUF<0b10, 0b00111, "psabs.b">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV64] in {
+def PSEXT_W_B      : RVPUnaryWUF<0b01, 0b00100, "psext.w.b">;
+def PSEXT_W_H      : RVPUnaryWUF<0b01, 0b00101, "psext.w.h">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in
+def PLUI_H : RVPUnaryImm10<0b1111000, "plui.h", simm10_unsigned>;
+let Predicates = [HasStdExtP, IsRV64] in
+def PLUI_W : RVPUnaryImm10<0b1111001, "plui.w", simm10_unsigned>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 9c03c7c83af04..de9e55beb6a5e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -351,10 +351,13 @@ defset list<VTypeInfo> AllVectors = {
         def VF16MF4: VTypeInfo<vfloat16mf4_t, vbool64_t, 16, V_MF4, f16, FPR16>;
         def VF16MF2: VTypeInfo<vfloat16mf2_t, vbool32_t, 16, V_MF2, f16, FPR16>;
         def VF32MF2: VTypeInfo<vfloat32mf2_t, vbool64_t, 32, V_MF2, f32, FPR32>;
+        def VBF16MF4: VTypeInfo<vbfloat16mf4_t, vbool64_t, 16, V_MF4, bf16, FPR16>;
+        def VBF16MF2: VTypeInfo<vbfloat16mf2_t, vbool32_t, 16, V_MF2, bf16, FPR16>;
       }
       def VF16M1: VTypeInfo<vfloat16m1_t, vbool16_t, 16, V_M1, f16, FPR16>;
       def VF32M1: VTypeInfo<vfloat32m1_t, vbool32_t, 32, V_M1, f32, FPR32>;
       def VF64M1: VTypeInfo<vfloat64m1_t, vbool64_t, 64, V_M1, f64, FPR64>;
+      def VBF16M1:  VTypeInfo<vbfloat16m1_t, vbool16_t, 16, V_M1, bf16, FPR16>;
     }
 
     defset list<GroupVTypeInfo> GroupFloatVectors = {
@@ -378,19 +381,7 @@ defset list<VTypeInfo> AllVectors = {
                                  V_M4, f64, FPR64>;
       def VF64M8: GroupVTypeInfo<vfloat64m8_t, vfloat64m1_t, vbool8_t,  64,
                                  V_M8, f64, FPR64>;
-    }
-  }
 
-  defset list<VTypeInfo> AllBFloatVectors = {
-    defset list<VTypeInfo> NoGroupBFloatVectors = {
-      defset list<VTypeInfo> FractionalGroupBFloatVectors = {
-        def VBF16MF4: VTypeInfo<vbfloat16mf4_t, vbool64_t, 16, V_MF4, bf16, FPR16>;
-        def VBF16MF2: VTypeInfo<vbfloat16mf2_t, vbool32_t, 16, V_MF2, bf16, FPR16>;
-      }
-      def VBF16M1:  VTypeInfo<vbfloat16m1_t, vbool16_t, 16, V_M1, bf16, FPR16>;
-    }
-  
-    defset list<GroupVTypeInfo> GroupBFloatVectors = {
       def VBF16M2: GroupVTypeInfo<vbfloat16m2_t, vbfloat16m1_t, vbool8_t, 16,
                                   V_M2, bf16, FPR16>;
       def VBF16M4: GroupVTypeInfo<vbfloat16m4_t, vbfloat16m1_t, vbool4_t, 16,
@@ -401,7 +392,6 @@ defset list<VTypeInfo> AllVectors = {
   }
 }
 
-defvar AllFloatVectorsExceptFP16 = !filter(vti, AllFloatVectors, !ne(vti.Scalar, f16));
 defvar AllFP16Vectors = !filter(vti, AllFloatVectors, !eq(vti.Scalar, f16));
 
 // This functor is used to obtain the int vector type that has the same SEW and
@@ -4577,7 +4567,8 @@ multiclass VPatUnaryS_M<string intrinsic_name,
 multiclass VPatUnaryV_V_AnyMask<string intrinsic, string instruction,
                                 list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     def : VPatUnaryAnyMask<intrinsic, instruction, "VM",
                            vti.Vector, vti.Vector, vti.Mask,
                            vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
@@ -4896,7 +4887,7 @@ multiclass VPatBinaryV_VV_INT<string intrinsic, string instruction,
                               list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
     defvar ivti = GetIntVTypeInfo<vti>.Vti;
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatBinary<intrinsic,
                       instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW,
                       vti.Vector, vti.Vector, ivti.Vector, vti.Mask,
@@ -4913,15 +4904,16 @@ multiclass VPatBinaryV_VV_INT_EEW<string intrinsic, string instruction,
     defvar octuple_lmul = vlmul.octuple;
     defvar octuple_emul = !srl(!mul(octuple_lmul, eew), vti.Log2SEW);
     if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
+      defvar ivti = GetIntVTypeInfo<vti>.Vti;
       defvar emul_str = octuple_to_str<octuple_emul>.ret;
-      defvar ivti = !cast<VTypeInfo>("VI" # eew # emul_str);
       defvar inst = instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str;
-      let Predicates = !listconcat(GetVTypeMinimalPredicates<vti>.Predicates,
-                                   GetVTypeMinimalPredicates<ivti>.Predicates) in
+      defvar eivti = !cast<VTypeInfo>("VI" # eew # emul_str);
+      let Predicates = !listconcat(GetVTypePredicates<ivti>.Predicates,
+                                   GetVTypePredicates<eivti>.Predicates) in
       defm : VPatBinary<intrinsic, inst,
-                        vti.Vector, vti.Vector, ivti.Vector, vti.Mask,
+                        vti.Vector, vti.Vector, eivti.Vector, vti.Mask,
                         vti.Log2SEW, vti.RegClass,
-                        vti.RegClass, ivti.RegClass>;
+                        vti.RegClass, eivti.RegClass>;
     }
   }
 }
@@ -4958,12 +4950,14 @@ multiclass VPatBinaryV_VX_RM<string intrinsic, string instruction,
 
 multiclass VPatBinaryV_VX_INT<string intrinsic, string instruction,
                           list<VTypeInfo> vtilist> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatBinary<intrinsic, instruction # "_VX_" # vti.LMul.MX,
                       vti.Vector, vti.Vector, XLenVT, vti.Mask,
                       vti.Log2SEW, vti.RegClass,
                       vti.RegClass, GPR>;
+  }
 }
 
 multiclass VPatBinaryV_VI<string intrinsic, string instruction,
@@ -4990,12 +4984,14 @@ multiclass VPatBinaryV_VI_RM<string intrinsic, string instruction,
 
 multiclass VPatBinaryV_VI_INT<string intrinsic, string instruction,
                               list<VTypeInfo> vtilist, Operand imm_type> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatBinary<intrinsic, instruction # "_VI_" # vti.LMul.MX,
                       vti.Vector, vti.Vector, XLenVT, vti.Mask,
                       vti.Log2SEW, vti.RegClass,
                       vti.RegClass, imm_type>;
+  }
 }
 
 multiclass VPatBinaryM_MM<string intrinsic, string instruction> {
@@ -5583,12 +5579,14 @@ multiclass VPatTernaryV_VV_AAXA_RM<string intrinsic, string instruction,
 
 multiclass VPatTernaryV_VX<string intrinsic, string instruction,
                            list<VTypeInfo> vtilist> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatTernaryWithPolicy<intrinsic, instruction, "VX",
                                  vti.Vector, vti.Vector, XLenVT, vti.Mask,
                                  vti.Log2SEW, vti.LMul, vti.RegClass,
                                  vti.RegClass, GPR>;
+  }
 }
 
 multiclass VPatTernaryV_VX_AAXA<string intrinsic, string instruction,
@@ -5615,12 +5613,14 @@ multiclass VPatTernaryV_VX_AAXA_RM<string intrinsic, string instruction,
 
 multiclass VPatTernaryV_VI<string intrinsic, string instruction,
                            list<VTypeInfo> vtilist, Operand Imm_type> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatTernaryWithPolicy<intrinsic, instruction, "VI",
                                  vti.Vector, vti.Vector, XLenVT, vti.Mask,
                                  vti.Log2SEW, vti.LMul, vti.RegClass,
                                  vti.RegClass, Imm_type>;
+  }
 }
 
 multiclass VPatTernaryW_VV<string intrinsic, string instruction,
@@ -7228,14 +7228,6 @@ foreach vti = AllFloatVectors in {
                                  vti.RegClass, vti.ScalarRegClass>;
 }
 
-foreach vti = AllBFloatVectors in
-  let Predicates = [HasVInstructionsBF16Minimal] in
-    defm : VPatBinaryCarryInTAIL<"int_riscv_vmerge", "PseudoVMERGE", "VVM",
-                                 vti.Vector,
-                                 vti.Vector, vti.Vector, vti.Mask,
-                                 vti.Log2SEW, vti.LMul, vti.RegClass,
-                                 vti.RegClass, vti.RegClass>;
-
 foreach fvti = AllFloatVectors in {
   defvar instr = !cast<Instruction>("PseudoVMERGE_VIM_"#fvti.LMul.MX);
   let Predicates = GetVTypePredicates<fvti>.Predicates in
@@ -7429,8 +7421,6 @@ defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
 
 defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
                                 AllFloatVectors, uimm5>;
-defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
-                                AllBFloatVectors, uimm5>;
 defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
                               eew=16, vtilist=AllFloatVectors>;
 //===----------------------------------------------------------------------===//
@@ -7438,7 +7428,6 @@ defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
 //===----------------------------------------------------------------------===//
 defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>;
 defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>;
-defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllBFloatVectors>;
 
 // Include the non-intrinsic ISel patterns
 include "RISCVInstrInfoVVLPatterns.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 520959b0896f7..b9b7ee4674ae4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1390,7 +1390,7 @@ defm : VPatFPSetCCSDNode_VV_VF_FV<SETOLE, "PseudoVMFLE", "PseudoVMFGE">;
 // Floating-point vselects:
 // 11.15. Vector Integer Merge Instructions
 // 13.15. Vector Floating-Point Merge Instruction
-foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach fvti = AllFloatVectors in {
   defvar ivti = GetIntVTypeInfo<fvti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     def : Pat<(fvti.Vector (vselect (fvti.Mask VMV0:$vm), fvti.RegClass:$rs1,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 6328e6c860f71..1f598863c8d08 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2580,7 +2580,7 @@ foreach vti = AllFloatVectors in {
   }
 }
 
-foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach fvti = AllFloatVectors in {
   // Floating-point vselects:
   // 11.15. Vector Integer Merge Instructions
   // 13.15. Vector Floating-Point Merge Instruction
@@ -2630,7 +2630,7 @@ foreach fvti = AllFloatVectors in {
   }
 }
 
-foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach fvti = AllFloatVectors in {
   defvar ivti = GetIntVTypeInfo<fvti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     // 13.16. Vector Floating-Point Move Instruction
@@ -2954,7 +2954,7 @@ foreach vti = NoGroupFloatVectors in {
   }
 }
 
-foreach vti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach vti = AllFloatVectors in {
   defvar ivti = GetIntVTypeInfo<vti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     def : Pat<(vti.Vector
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
index 3d2b9c209dc59..e46b0d409eb5d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
@@ -53,12 +53,24 @@ def simm20_lsb000 : Operand<XLenVT> {
   let DecoderMethod = "decodeSImmOperandAndLslN<20, 3>";
 }
 
+// Predicate: True if immediate is a power of 2.
+def PowerOf2 : PatLeaf<(imm), [{
+  return isPowerOf2_64(N->getZExtValue());
+}]>;
+
+// Transformation function: Get log2 of immediate.
+def Log2 : SDNodeXForm<imm, [{
+  uint64_t Imm = Log2_64(N->getZExtValue());
+  return CurDAG->getTargetConstant(Imm, SDLoc(N), N->getValueType(0));
+}]>;
+
 //===----------------------------------------------------------------------===//
 // Instruction Class Templates
 //===----------------------------------------------------------------------===//
 
 class NDSRVInstBB<bit cs, string opcodestr>
-    : RVInst<(outs), (ins GPR:$rs1, uimmlog2xlen:$cimm, bare_simm11_lsb0:$imm10),
+    : RVInst<(outs),
+             (ins GPR:$rs1, uimmlog2xlen:$cimm, bare_simm11_lsb0:$imm10),
              opcodestr, "$rs1, $cimm, $imm10", [], InstFormatNDS_BRANCH_10>,
       Sched<[WriteJmp, ReadIALU]> {
   bits<10> imm10;
@@ -106,7 +118,8 @@ class NDSRVInstBC<bits<3> funct3, string opcodestr>
 }
 
 class NDSRVInstBFO<bits<3> funct3, string opcodestr>
-    : RVInst<(outs GPR:$rd), (ins GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb),
+    : RVInst<(outs GPR:$rd),
+             (ins GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb),
              opcodestr, "$rd, $rs1, $msb, $lsb", [], InstFormatOther>,
       Sched<[WriteIALU, ReadIALU]> {
   bits<5> rd;
@@ -361,13 +374,13 @@ class NDSRVInstVD4DOT<bits<6> funct6, string opcodestr>
   let RVVConstraint = VMConstraint;
 }
 
-class NDSRVInstVBFHCvt<bits<7> funct7, bits<5> vs1, string opcodestr>
+class NDSRVInstVBFHCvt<bits<5> vs1, string opcodestr>
     : RVInst<(outs VR:$vd), (ins VR:$vs2, VMaskOp:$vm),
              opcodestr, "$vd, $vs2", [], InstFormatR> {
   bits<5> vs2;
   bits<5> vd;
 
-  let Inst{31-25} = funct7;
+  let Inst{31-25} = 0b0000000;
   let Inst{24-20} = vs2;
   let Inst{19-15} = vs1;
   let Inst{14-12} = 0b100;
@@ -380,6 +393,29 @@ class NDSRVInstVBFHCvt<bits<7> funct7, bits<5> vs1, string opcodestr>
   let Uses = [VL, VTYPE];
 }
 
+class NDSRVInstVLN<bits<5> funct5, string opcodestr>
+    : RVInst<(outs VR:$vd), (ins GPRMemZeroOffset:$rs1, VMaskOp:$vm),
+             opcodestr, "$vd, ${rs1}$vm", [], InstFormatR>,
+      VLESchedMC {
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = 0b000001;
+  let Inst{25} = vm;
+  let Inst{24-20} = funct5;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b100;
+  let Inst{11-7} = vd;
+  let Inst{6-0} = OPC_CUSTOM_2.Value;
+  let hasSideEffects = 0;
+  let mayLoad = 1;
+  let mayStore = 0;
+
+  let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
+}
+
 //===----------------------------------------------------------------------===//
 // Multiclass
 //===----------------------------------------------------------------------===//
@@ -398,7 +434,8 @@ multiclass VPseudoVNCVT_BF16_S {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
     let VLMul = m.value, SEW=16 in
-    def "_" # m.MX : VPseudoUnaryNoMaskRoundingMode<m.vrclass, m.wvrclass, constraint>,
+    def "_" # m.MX : VPseudoUnaryNoMaskRoundingMode<m.vrclass, m.wvrclass,
+                                                    constraint>,
                      SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, 16,
                                 forcePassthruRead=true>;
   }
@@ -409,8 +446,10 @@ multiclass VPatConversionS_BF16<string intrinsic, string instruction> {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
     let Predicates = [HasVendorXAndesVBFHCvt] in
-    def : VPatUnaryNoMask<intrinsic, instruction, "BF16", fwti.Vector, fvti.Vector,
-                          fvti.Log2SEW, fvti.LMul, fwti.RegClass, fvti.RegClass>;
+    def : VPatUnaryNoMask<intrinsic, instruction, "BF16",
+                          fwti.Vector, fvti.Vector,
+                          fvti.Log2SEW, fvti.LMul,
+                          fwti.RegClass, fvti.RegClass>;
   }
 }
 
@@ -419,8 +458,10 @@ multiclass VPatConversionBF16_S<string intrinsic, string instruction> {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
     let Predicates = [HasVendorXAndesVBFHCvt] in
-    def : VPatUnaryNoMaskRoundingMode<intrinsic, instruction, "S", fvti.Vector, fwti.Vector,
-                                      fvti.Log2SEW, fvti.LMul, fvti.RegClass, fwti.RegClass>;
+    def : VPatUnaryNoMaskRoundingMode<intrinsic, instruction, "S",
+                                      fvti.Vector, fwti.Vector,
+                                      fvti.Log2SEW, fvti.LMul,
+                                      fvti.RegClass, fwti.RegClass>;
   }
 }
 
@@ -440,7 +481,8 @@ multiclass VPatVFPMADBinaryV_VX_RM<string intrinsic, string instruction,
                                    list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
     defvar kind = "V"#vti.ScalarSuffix;
-    defm : VPatBinaryRoundingMode<intrinsic, instruction#"_"#kind#"_"#vti.LMul.MX,
+    defm : VPatBinaryRoundingMode<intrinsic,
+                                  instruction#"_"#kind#"_"#vti.LMul.MX,
                                   vti.Vector, vti.Vector, f32, vti.Mask,
                                   vti.Log2SEW, vti.RegClass,
                                   vti.RegClass, FPR32>;
@@ -526,9 +568,18 @@ def NDS_SDGP  : NDSRVInstSDGP<0b111, "nds.sdgp">;
 let Predicates = [HasVendorXAndesVBFHCvt], Constraints = "@earlyclobber $vd",
     mayRaiseFPException = true in {
 let RVVConstraint = VS2Constraint, DestEEW = EEWSEWx2 in
-def NDS_VFWCVT_S_BF16 : NDSRVInstVBFHCvt<0b0000000, 0b00000, "nds.vfwcvt.s.bf16">;
+def NDS_VFWCVT_S_BF16 : NDSRVInstVBFHCvt<0b00000, "nds.vfwcvt.s.bf16">;
 let Uses = [FRM, VL, VTYPE] in
-def NDS_VFNCVT_BF16_S : NDSRVInstVBFHCvt<0b0000000, 0b00001, "nds.vfncvt.bf16.s">;
+def NDS_VFNCVT_BF16_S : NDSRVInstVBFHCvt<0b00001, "nds.vfncvt.bf16.s">;
+}
+
+//===----------------------------------------------------------------------===//
+// XAndesVSIntLoad
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasVendorXAndesVSIntLoad] in {
+def NDS_VLN8_V  : NDSRVInstVLN<0b00010, "nds.vln8.v">;
+def NDS_VLNU8_V : NDSRVInstVLN<0b00011, "nds.vlnu8.v">;
 }
 
 //===----------------------------------------------------------------------===//
@@ -556,8 +607,44 @@ def NDS_VD4DOTSU_VV : NDSRVInstVD4DOT<0b000101, "nds.vd4dotsu">;
 // Pseudo-instructions and codegen patterns
 //===----------------------------------------------------------------------===//
 
+class NDS_BBPat<CondCode Cond, NDSRVInstBB Inst>
+    : Pat<(riscv_brcc (and(XLenVT GPR:$rs1), PowerOf2:$mask), 0, Cond,
+              bb:$imm10),
+          (Inst GPR:$rs1, (Log2 PowerOf2:$mask), bare_simm11_lsb0:$imm10)>;
+
+class NDS_BCPat<CondCode Cond, NDSRVInstBC Inst>
+    : Pat<(riscv_brcc (XLenVT GPR:$rs1), uimm7:$cimm, Cond, bb:$imm10),
+          (Inst GPR:$rs1, uimm7:$cimm, bare_simm11_lsb0:$imm10)>;
+
+defm CC_UImmLog2XLen_NDS : SelectCC_GPR_riirr<GPR, uimmlog2xlen>;
+defm CC_UImm7_NDS        : SelectCC_GPR_riirr<GPR, uimm7>;
+
+class SelectNDS_BB<CondCode Cond>
+    : Pat<(riscv_selectcc_frag:$cc (and(XLenVT GPR:$lhs), PowerOf2:$mask), 0,
+              Cond, (XLenVT GPR:$truev), GPR:$falsev),
+          (Select_GPR_Using_CC_UImmLog2XLen_NDS GPR:$lhs, (Log2 PowerOf2:$mask),
+              (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
+
+class SelectNDS_BC<CondCode Cond>
+    : Pat<(riscv_selectcc_frag:$cc (XLenVT GPR:$lhs), uimm7:$cimm, Cond,
+              (XLenVT GPR:$truev), GPR:$falsev),
+          (Select_GPR_Using_CC_UImm7_NDS GPR:$lhs, uimm7:$cimm,
+              (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
+
 let Predicates = [HasVendorXAndesPerf] in {
 
+def : NDS_BBPat<SETEQ, NDS_BBC>;
+def : NDS_BBPat<SETNE, NDS_BBS>;
+
+def : SelectNDS_BB<SETEQ>;
+def : SelectNDS_BB<SETNE>;
+
+def : NDS_BCPat<SETEQ, NDS_BEQC>;
+def : NDS_BCPat<SETNE, NDS_BNEC>;
+
+def : SelectNDS_BC<SETEQ>;
+def : SelectNDS_BC<SETNE>;
+
 def : Pat<(sext_inreg (XLenVT GPR:$rs1), i16), (NDS_BFOS GPR:$rs1, 15, 0)>;
 def : Pat<(sext_inreg (XLenVT GPR:$rs1), i8), (NDS_BFOS GPR:$rs1, 7, 0)>;
 def : Pat<(sext_inreg (XLenVT GPR:$rs1), i1), (NDS_BFOS GPR:$rs1, 0, 0)>;
@@ -592,8 +679,10 @@ defm PseudoNDS_VFWCVT_S_BF16 : VPseudoVWCVT_S_BF16;
 defm PseudoNDS_VFNCVT_BF16_S : VPseudoVNCVT_BF16_S;
 } // Predicates = [HasVendorXAndesVBFHCvt]
 
-defm : VPatConversionS_BF16<"int_riscv_nds_vfwcvt_s_bf16", "PseudoNDS_VFWCVT_S">;
-defm : VPatConversionBF16_S<"int_riscv_nds_vfncvt_bf16_s", "PseudoNDS_VFNCVT_BF16">;
+defm : VPatConversionS_BF16<"int_riscv_nds_vfwcvt_s_bf16",
+                            "PseudoNDS_VFWCVT_S">;
+defm : VPatConversionBF16_S<"int_riscv_nds_vfncvt_bf16_s",
+                            "PseudoNDS_VFNCVT_BF16">;
 
 let Predicates = [HasVendorXAndesVPackFPH],
     mayRaiseFPException = true in {
@@ -601,8 +690,10 @@ defm PseudoNDS_VFPMADT : VPseudoVFPMAD_VF_RM;
 defm PseudoNDS_VFPMADB : VPseudoVFPMAD_VF_RM;
 } // Predicates = [HasVendorXAndesVPackFPH]
 
-defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadt", "PseudoNDS_VFPMADT", AllFP16Vectors>;
-defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadb", "PseudoNDS_VFPMADB", AllFP16Vectors>;
+defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadt", "PseudoNDS_VFPMADT",
+                               AllFP16Vectors>;
+defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadb", "PseudoNDS_VFPMADB",
+                               AllFP16Vectors>;
 
 let Predicates = [HasVendorXAndesVDot] in {
 defm PseudoNDS_VD4DOTS  : VPseudoVD4DOT_VV;
@@ -622,9 +713,12 @@ defset list<VTypeInfoToWide> AllQuadWidenableVD4DOTVectors = {
   def : VTypeInfoToWide<VI16M8,  VI64M8>;
 }
 
-defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dots", "PseudoNDS_VD4DOTS", AllQuadWidenableVD4DOTVectors>;
-defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotu", "PseudoNDS_VD4DOTU", AllQuadWidenableVD4DOTVectors>;
-defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotsu", "PseudoNDS_VD4DOTSU", AllQuadWidenableVD4DOTVectors>;
+defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dots", "PseudoNDS_VD4DOTS",
+                            AllQuadWidenableVD4DOTVectors>;
+defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotu", "PseudoNDS_VD4DOTU",
+                            AllQuadWidenableVD4DOTVectors>;
+defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotsu", "PseudoNDS_VD4DOTSU",
+                            AllQuadWidenableVD4DOTVectors>;
 
 //===----------------------------------------------------------------------===//
 // Pseudo-instructions for SFB (Short Forward Branch)
@@ -634,12 +728,14 @@ let Predicates = [HasShortForwardBranchOpt], hasSideEffects = 0,
     mayLoad = 0, mayStore = 0, Size = 8, Constraints = "$dst = $falsev" in {
 def PseudoCCNDS_BFOS : Pseudo<(outs GPR:$dst),
                               (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
-                               GPR:$falsev, GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb), []>,
+                               GPR:$falsev, GPR:$rs1,
+                               uimmlog2xlen:$msb, uimmlog2xlen:$lsb), []>,
                        Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                               ReadSFBALU]>;
 def PseudoCCNDS_BFOZ : Pseudo<(outs GPR:$dst),
                               (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
-                               GPR:$falsev, GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb), []>,
+                               GPR:$falsev, GPR:$rs1,
+                               uimmlog2xlen:$msb, uimmlog2xlen:$lsb), []>,
                        Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
                               ReadSFBALU]>;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
index ff751994b89b9..0c8487c2f5dbe 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
@@ -29,6 +29,12 @@ def uimm7_lsb000 : RISCVOp,
   }];
 }
 
+// A 9-bit unsigned offset
+def uimm9 : RISCVUImmOp<9>;
+
+// Custom prefetch ADDR selector
+def AddrRegImm9 : ComplexPattern<iPTR, 2, "SelectAddrRegImm9">;
+
 //===----------------------------------------------------------------------===//
 // MIPS custom instruction formats
 //===----------------------------------------------------------------------===//
@@ -103,9 +109,41 @@ class SWPFormat<dag outs, dag ins, string opcodestr, string argstr>
   let Inst{6-0} = OPC_CUSTOM_0.Value;
 }
 
+// Prefetch format.
+let hasSideEffects = 0, mayLoad = 1, mayStore = 1 in
+class Mips_prefetch_ri<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<9> imm9;
+  bits<5> rs1;
+  bits<5> hint;
+
+  let Inst{31-29} = 0b000;
+  let Inst{28-20} = imm9;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b000;
+  let Inst{11-7} = hint;
+  let Inst{6-0} = OPC_CUSTOM_0.Value;
+}
+
 //===----------------------------------------------------------------------===//
 // MIPS extensions
 //===----------------------------------------------------------------------===//
+let Predicates = [HasVendorXMIPSCBOP] ,DecoderNamespace = "Xmipscbop" in {
+  def MIPS_PREFETCH : Mips_prefetch_ri<(outs), (ins GPR:$rs1, uimm9:$imm9, uimm5:$hint),
+                                        "mips.pref", "$hint, ${imm9}(${rs1})">,
+                       Sched<[]>;
+}
+
+let Predicates = [HasVendorXMIPSCBOP] in {
+  // Prefetch Data Write.
+  def : Pat<(prefetch (AddrRegImm9  (XLenVT GPR:$rs1), uimm9:$imm9),
+             (i32 1), timm, (i32 1)),
+            (MIPS_PREFETCH GPR:$rs1, uimm9:$imm9, 9)>;
+  // Prefetch Data Read.
+  def : Pat<(prefetch (AddrRegImm9 (XLenVT GPR:$rs1), uimm9:$imm9),
+             (i32 0), timm, (i32 1)),
+            (MIPS_PREFETCH GPR:$rs1, uimm9:$imm9, 8)>;
+}
 
 let Predicates = [HasVendorXMIPSCMov], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
                  DecoderNamespace = "Xmipscmov" in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index affe3a8476753..6fe747800b5ad 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -160,10 +160,10 @@ multiclass CustomSiFiveVCIX<string suffix, VCIXType type,
                             DAGOperand InTyRd, DAGOperand InTyRs2,
                             DAGOperand InTyRs1> {
   let vm = 1 in
-  defm VC_ # NAME   : CustomSiFiveVCIXorVCIF<suffix, type, InTyRd, InTyRs2,
+  defm SF_VC_ # NAME   : CustomSiFiveVCIXorVCIF<suffix, type, InTyRd, InTyRs2,
                                              InTyRs1, 0>;
   let vm = 0 in
-  defm VC_V_ # NAME : CustomSiFiveVCIXorVCIF<suffix, type, VR, InTyRs2,
+  defm SF_VC_V_ # NAME : CustomSiFiveVCIXorVCIF<suffix, type, VR, InTyRs2,
                                              InTyRs1, 1>;
 }
 
@@ -201,29 +201,29 @@ let Predicates = [HasVendorXSfvcp], mayLoad = 0, mayStore = 0,
 
 let Predicates = [HasVendorXSfvqmaccdod], DecoderNamespace = "XSfvector",
     DestEEW = EEWSEWx4, RVVConstraint=VS2Constraint in {
-  def VQMACCU_2x8x2  : CustomSiFiveVMACC<0b101100, OPMVV, "sf.vqmaccu.2x8x2">;
-  def VQMACC_2x8x2   : CustomSiFiveVMACC<0b101101, OPMVV, "sf.vqmacc.2x8x2">;
-  def VQMACCUS_2x8x2 : CustomSiFiveVMACC<0b101110, OPMVV, "sf.vqmaccus.2x8x2">;
-  def VQMACCSU_2x8x2 : CustomSiFiveVMACC<0b101111, OPMVV, "sf.vqmaccsu.2x8x2">;
+  def SF_VQMACCU_2x8x2  : CustomSiFiveVMACC<0b101100, OPMVV, "sf.vqmaccu.2x8x2">;
+  def SF_VQMACC_2x8x2   : CustomSiFiveVMACC<0b101101, OPMVV, "sf.vqmacc.2x8x2">;
+  def SF_VQMACCUS_2x8x2 : CustomSiFiveVMACC<0b101110, OPMVV, "sf.vqmaccus.2x8x2">;
+  def SF_VQMACCSU_2x8x2 : CustomSiFiveVMACC<0b101111, OPMVV, "sf.vqmaccsu.2x8x2">;
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq], DecoderNamespace = "XSfvector",
     DestEEW = EEWSEWx4, RVVConstraint=WidenVNoMask in {
-  def VQMACCU_4x8x4  : CustomSiFiveVMACC<0b111100, OPMVV, "sf.vqmaccu.4x8x4">;
-  def VQMACC_4x8x4   : CustomSiFiveVMACC<0b111101, OPMVV, "sf.vqmacc.4x8x4">;
-  def VQMACCUS_4x8x4 : CustomSiFiveVMACC<0b111110, OPMVV, "sf.vqmaccus.4x8x4">;
-  def VQMACCSU_4x8x4 : CustomSiFiveVMACC<0b111111, OPMVV, "sf.vqmaccsu.4x8x4">;
+  def SF_VQMACCU_4x8x4  : CustomSiFiveVMACC<0b111100, OPMVV, "sf.vqmaccu.4x8x4">;
+  def SF_VQMACC_4x8x4   : CustomSiFiveVMACC<0b111101, OPMVV, "sf.vqmacc.4x8x4">;
+  def SF_VQMACCUS_4x8x4 : CustomSiFiveVMACC<0b111110, OPMVV, "sf.vqmaccus.4x8x4">;
+  def SF_VQMACCSU_4x8x4 : CustomSiFiveVMACC<0b111111, OPMVV, "sf.vqmaccsu.4x8x4">;
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq], DecoderNamespace = "XSfvector",
     DestEEW = EEWSEWx2, RVVConstraint=WidenVNoMask in {
-  def VFWMACC_4x4x4 : CustomSiFiveVMACC<0b111100, OPFVV, "sf.vfwmacc.4x4x4">;
+  def SF_VFWMACC_4x4x4 : CustomSiFiveVMACC<0b111100, OPFVV, "sf.vfwmacc.4x4x4">;
 }
 
 let Predicates = [HasVendorXSfvfnrclipxfqf], DecoderNamespace = "XSfvector",
     Uses = [FRM, VL, VTYPE] in {
-  def VFNRCLIP_XU_F_QF : CustomSiFiveVFNRCLIP<0b100010, OPFVF, "sf.vfnrclip.xu.f.qf">;
-  def VFNRCLIP_X_F_QF : CustomSiFiveVFNRCLIP<0b100011, OPFVF, "sf.vfnrclip.x.f.qf">;
+  def SF_VFNRCLIP_XU_F_QF : CustomSiFiveVFNRCLIP<0b100010, OPFVF, "sf.vfnrclip.xu.f.qf">;
+  def SF_VFNRCLIP_X_F_QF : CustomSiFiveVFNRCLIP<0b100011, OPFVF, "sf.vfnrclip.x.f.qf">;
 }
 
 class VPseudoVC_X<Operand OpClass, DAGOperand RS1Class> :
@@ -306,14 +306,14 @@ multiclass VPseudoVC_X<LMULInfo m, DAGOperand RS1Class,
                        Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
         : VPseudoVC_X<OpClass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX
+    def "PseudoSF_VC_V_" # NAME # "_" # m.MX
       : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
@@ -323,14 +323,14 @@ multiclass VPseudoVC_XV<LMULInfo m, DAGOperand RS1Class,
                         Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
         : VPseudoVC_XV<OpClass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX
+    def "PseudoSF_VC_V_" # NAME # "_" # m.MX
       : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
@@ -340,14 +340,14 @@ multiclass VPseudoVC_XVV<LMULInfo m, DAGOperand RS1Class,
                          Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
         : VPseudoVC_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX
+    def "PseudoSF_VC_V_" # NAME # "_" # m.MX
       : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
@@ -357,15 +357,15 @@ multiclass VPseudoVC_XVW<LMULInfo m, DAGOperand RS1Class,
                          Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in
-    def "PseudoVC_" # NAME # "_SE_" # m.MX
+    def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
       : VPseudoVC_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
     let Constraints = "@earlyclobber $rd, $rd = $rs3" in {
       let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_" # m.MX
         : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
@@ -391,7 +391,7 @@ multiclass VPseudoSiFiveVQMACCQOQ {
 }
 
 multiclass VPseudoSiFiveVFWMACC {
-  foreach m = MxListVF2 in
+  foreach m = MxListFW in
     let VLMul = m.value in
     defm NAME : VPseudoSiFiveVMACC<m.MX, m.wvrclass, m.vrclass>;
 }
@@ -435,26 +435,26 @@ let Predicates = [HasVendorXSfvcp] in {
 }
 
 let Predicates = [HasVendorXSfvqmaccdod] in {
-  defm VQMACCU_2x8x2  : VPseudoSiFiveVQMACCDOD;
-  defm VQMACC_2x8x2   : VPseudoSiFiveVQMACCDOD;
-  defm VQMACCUS_2x8x2 : VPseudoSiFiveVQMACCDOD;
-  defm VQMACCSU_2x8x2 : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACCU_2x8x2  : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACC_2x8x2   : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACCUS_2x8x2 : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACCSU_2x8x2 : VPseudoSiFiveVQMACCDOD;
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq] in {
-  defm VQMACCU_4x8x4  : VPseudoSiFiveVQMACCQOQ;
-  defm VQMACC_4x8x4   : VPseudoSiFiveVQMACCQOQ;
-  defm VQMACCUS_4x8x4 : VPseudoSiFiveVQMACCQOQ;
-  defm VQMACCSU_4x8x4 : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACCU_4x8x4  : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACC_4x8x4   : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACCUS_4x8x4 : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACCSU_4x8x4 : VPseudoSiFiveVQMACCQOQ;
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq] in {
-  defm VFWMACC_4x4x4 : VPseudoSiFiveVFWMACC;
+  defm SF_VFWMACC_4x4x4 : VPseudoSiFiveVFWMACC;
 }
 
 let Predicates = [HasVendorXSfvfnrclipxfqf] in {
-  defm VFNRCLIP_XU_F_QF : VPseudoSiFiveVFNRCLIP;
-  defm VFNRCLIP_X_F_QF : VPseudoSiFiveVFNRCLIP;
+  defm SF_VFNRCLIP_XU_F_QF : VPseudoSiFiveVFNRCLIP;
+  defm SF_VFNRCLIP_X_F_QF : VPseudoSiFiveVFNRCLIP;
 }
 
 // SDNode
@@ -660,11 +660,11 @@ class VPatVC_V_OP3<string intrinsic_name,
 multiclass VPatVC_X<string intrinsic_suffix, string instruction_suffix,
                     VTypeInfo vti, ValueType type, DAGOperand kind> {
   def : VPatVC_V_OP3_ISD<!cast<SDPatternOperator>("sf_vc_v_" # intrinsic_suffix # "_se"),
-                         "PseudoVC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                         "PseudoSF_VC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                          vti.Vector, XLenVT, type, vti.Log2SEW,
                          payload5, kind>;
   def : VPatVC_V_OP3<"int_riscv_sf_vc_v_" # intrinsic_suffix,
-                     "PseudoVC_V_" # instruction_suffix # "_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_" # vti.LMul.MX,
                      vti.Vector, XLenVT, type, vti.Log2SEW,
                      payload5, kind>;
 }
@@ -673,15 +673,15 @@ multiclass VPatVC_XV<string intrinsic_suffix, string instruction_suffix,
                      VTypeInfo vti, ValueType type, DAGOperand kind,
                      Operand op1_kind = payload2> {
   def : VPatVC_OP4_ISD<!cast<SDPatternOperator>("sf_vc_" # intrinsic_suffix # "_se"),
-                   "PseudoVC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                   "PseudoSF_VC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                    XLenVT, vti.Vector, type, vti.Log2SEW,
                    payload5, vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP3_ISD<!cast<SDPatternOperator>("sf_vc_v_" # intrinsic_suffix # "_se"),
-                         "PseudoVC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                         "PseudoSF_VC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                          vti.Vector, vti.Vector, type, vti.Log2SEW,
                          vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP3<"int_riscv_sf_vc_v_" # intrinsic_suffix,
-                     "PseudoVC_V_" # instruction_suffix # "_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_" # vti.LMul.MX,
                      vti.Vector, vti.Vector, type, vti.Log2SEW,
                      vti.RegClass, kind, op1_kind>;
 }
@@ -690,15 +690,15 @@ multiclass VPatVC_XVV<string intrinsic_suffix, string instruction_suffix,
                       VTypeInfo wti, VTypeInfo vti, ValueType type, DAGOperand kind,
                       Operand op1_kind = payload2> {
   def : VPatVC_OP4_ISD<!cast<SDPatternOperator>("sf_vc_" # intrinsic_suffix # "_se"),
-                   "PseudoVC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                   "PseudoSF_VC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                    wti.Vector, vti.Vector, type, vti.Log2SEW,
                    wti.RegClass, vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP4_ISD<!cast<SDPatternOperator>("sf_vc_v_" # intrinsic_suffix # "_se"),
-                     "PseudoVC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                      wti.Vector, wti.Vector, vti.Vector, type, vti.Log2SEW,
                      wti.RegClass, vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP4<"int_riscv_sf_vc_v_" # intrinsic_suffix,
-                     "PseudoVC_V_" # instruction_suffix # "_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_" # vti.LMul.MX,
                      wti.Vector, wti.Vector, vti.Vector, type, vti.Log2SEW,
                      wti.RegClass, vti.RegClass, kind, op1_kind>;
 }
@@ -810,26 +810,26 @@ let Predicates = [HasVendorXSfvcp] in {
 }
 
 let Predicates = [HasVendorXSfvqmaccdod] in {
-  defm : VPatVQMACCDOD<"vqmaccu_2x8x2", "VQMACCU", "2x8x2">;
-  defm : VPatVQMACCDOD<"vqmacc_2x8x2", "VQMACC", "2x8x2">;
-  defm : VPatVQMACCDOD<"vqmaccus_2x8x2", "VQMACCUS", "2x8x2">;
-  defm : VPatVQMACCDOD<"vqmaccsu_2x8x2", "VQMACCSU", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmaccu_2x8x2", "SF_VQMACCU", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmacc_2x8x2", "SF_VQMACC", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmaccus_2x8x2", "SF_VQMACCUS", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmaccsu_2x8x2", "SF_VQMACCSU", "2x8x2">;
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq] in {
-  defm : VPatVQMACCQOQ<"vqmaccu_4x8x4", "VQMACCU", "4x8x4">;
-  defm : VPatVQMACCQOQ<"vqmacc_4x8x4", "VQMACC", "4x8x4">;
-  defm : VPatVQMACCQOQ<"vqmaccus_4x8x4", "VQMACCUS", "4x8x4">;
-  defm : VPatVQMACCQOQ<"vqmaccsu_4x8x4", "VQMACCSU", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmaccu_4x8x4", "SF_VQMACCU", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmacc_4x8x4", "SF_VQMACC", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmaccus_4x8x4", "SF_VQMACCUS", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmaccsu_4x8x4", "SF_VQMACCSU", "4x8x4">;
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq] in {
-  defm : VPatVFWMACC<"vfwmacc_4x4x4", "VFWMACC", "4x4x4">;
+  defm : VPatVFWMACC<"vfwmacc_4x4x4", "SF_VFWMACC", "4x4x4">;
 }
 
 let Predicates = [HasVendorXSfvfnrclipxfqf] in {
-  defm : VPatVFNRCLIP<"vfnrclip_xu_f_qf", "VFNRCLIP_XU_F_QF">;
-  defm : VPatVFNRCLIP<"vfnrclip_x_f_qf", "VFNRCLIP_X_F_QF">;
+  defm : VPatVFNRCLIP<"vfnrclip_xu_f_qf", "SF_VFNRCLIP_XU_F_QF">;
+  defm : VPatVFNRCLIP<"vfnrclip_x_f_qf", "SF_VFNRCLIP_X_F_QF">;
 }
 
 let Predicates = [HasVendorXSiFivecdiscarddlone] in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index c23a58ceccead..c0f7ab127aeb7 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -476,10 +476,10 @@ def TH_FSURD : THStoreIndexed<FPR64, 0b01110, "th.fsurd">,
 } // DecoderNamespace = "XTHead"
 
 let Predicates = [HasVendorXTHeadVdot] in {
-defm THVdotVMAQA      : THVdotVMAQA<"th.vmaqa",     0b100000>;
-defm THVdotVMAQAU     : THVdotVMAQA<"th.vmaqau",    0b100010>;
-defm THVdotVMAQASU    : THVdotVMAQA<"th.vmaqasu",   0b100100>;
-defm THVdotVMAQAUS    : THVdotVMAQA_VX<"th.vmaqaus",0b100110>;
+defm TH_VMAQA      : THVdotVMAQA<"th.vmaqa",     0b100000>;
+defm TH_VMAQAU     : THVdotVMAQA<"th.vmaqau",    0b100010>;
+defm TH_VMAQASU    : THVdotVMAQA<"th.vmaqasu",   0b100100>;
+defm TH_VMAQAUS    : THVdotVMAQA_VX<"th.vmaqaus",0b100110>;
 }
 
 // Associate LMUL with tablegen records of register classes.
@@ -661,20 +661,20 @@ def : Pat<(i32 (sub GPR:$rd, (mul (sexti16 (i32 GPR:$rs1)),
 } // Predicates = [HasVendorXTHeadMac, IsRV32]
 
 let Predicates = [HasVendorXTHeadVdot] in {
-defm PseudoTHVdotVMAQA      : VPseudoVMAQA_VV_VX;
-defm PseudoTHVdotVMAQAU     : VPseudoVMAQA_VV_VX;
-defm PseudoTHVdotVMAQASU    : VPseudoVMAQA_VV_VX;
-defm PseudoTHVdotVMAQAUS    : VPseudoVMAQA_VX;
+defm PseudoTH_VMAQA      : VPseudoVMAQA_VV_VX;
+defm PseudoTH_VMAQAU     : VPseudoVMAQA_VV_VX;
+defm PseudoTH_VMAQASU    : VPseudoVMAQA_VV_VX;
+defm PseudoTH_VMAQAUS    : VPseudoVMAQA_VX;
 }
 
 let Predicates = [HasVendorXTHeadVdot] in {
-defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqa",  "PseudoTHVdotVMAQA",
+defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqa",  "PseudoTH_VMAQA",
                               AllQuadWidenableInt8NoVLMulVectors>;
-defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqau", "PseudoTHVdotVMAQAU",
+defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqau", "PseudoTH_VMAQAU",
                               AllQuadWidenableInt8NoVLMulVectors>;
-defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqasu","PseudoTHVdotVMAQASU",
+defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqasu","PseudoTH_VMAQASU",
                               AllQuadWidenableInt8NoVLMulVectors>;
-defm : VPatTernaryVMAQA_VX<"int_riscv_th_vmaqaus",   "PseudoTHVdotVMAQAUS",
+defm : VPatTernaryVMAQA_VX<"int_riscv_th_vmaqaus",   "PseudoTH_VMAQAUS",
                            AllQuadWidenableInt8NoVLMulVectors>;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index e8dd164714875..b2bf09028bc40 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -1308,9 +1308,9 @@ class QCIMVCCPat<CondCode Cond, QCIMVCC Inst>
     : Pat<(select (XLenVT (setcc (XLenVT GPRNoX0:$rs1), (XLenVT GPRNoX0:$rs2), Cond)), (XLenVT GPRNoX0:$rs3), (XLenVT GPRNoX0:$rd)),
           (Inst GPRNoX0:$rd, GPRNoX0:$rs1, GPRNoX0:$rs2, GPRNoX0:$rs3)>;
 
-class QCIMVCCIPat<CondCode Cond, QCIMVCCI Inst>
-    : Pat<(select (XLenVT (setcc (XLenVT GPRNoX0:$rs1), simm5:$imm, Cond)), (XLenVT GPRNoX0:$rs3), (XLenVT GPRNoX0:$rd)),
-          (Inst GPRNoX0:$rd, GPRNoX0:$rs1, simm5:$imm, GPRNoX0:$rs3)>;
+class QCIMVCCIPat<CondCode Cond, QCIMVCCI Inst, DAGOperand InTyImm>
+    : Pat<(select (XLenVT (setcc (XLenVT GPRNoX0:$rs1), InTyImm:$imm, Cond)), (XLenVT GPRNoX0:$rs3), (XLenVT GPRNoX0:$rd)),
+          (Inst GPRNoX0:$rd, GPRNoX0:$rs1, InTyImm:$imm, GPRNoX0:$rs3)>;
 
 // Match `riscv_brcc` and lower to the appropriate XQCIBI branch instruction.
 class BcciPat<CondCode Cond, QCIBranchInst_rii Inst, DAGOperand InTyImm>
@@ -1455,10 +1455,10 @@ def : QCIMVCCPat <SETNE,  QC_MVNE>;
 def : QCIMVCCPat <SETLT,  QC_MVLT>;
 def : QCIMVCCPat <SETULT, QC_MVLTU>;
 
-def : QCIMVCCIPat <SETEQ,  QC_MVEQI>;
-def : QCIMVCCIPat <SETNE,  QC_MVNEI>;
-def : QCIMVCCIPat <SETLT,  QC_MVLTI>;
-def : QCIMVCCIPat <SETULT, QC_MVLTUI>;
+def : QCIMVCCIPat <SETEQ,  QC_MVEQI, simm5>;
+def : QCIMVCCIPat <SETNE,  QC_MVNEI, simm5>;
+def : QCIMVCCIPat <SETLT,  QC_MVLTI, simm5>;
+def : QCIMVCCIPat <SETULT, QC_MVLTUI, uimm5>;
 }
 
 //===----------------------------------------------------------------------===/i
@@ -1560,12 +1560,18 @@ def : CompressPat<(QC_E_LI GPRNoX0:$rd, simm20_li:$imm),
 } // Predicates = [HasVendorXqcili, IsRV32]
 
 let isCompressOnly = true, Predicates = [HasVendorXqcilia, IsRV32] in {
+def : CompressPat<(QC_E_ADDAI GPRNoX0:$rd, simm6nonzero:$imm),
+                  (C_ADDI GPRNoX0:$rd, simm6nonzero:$imm)>;
 def : CompressPat<(QC_E_ADDI GPRNoX0:$rs1, GPRNoX0:$rs1, simm6nonzero:$imm),
                   (C_ADDI GPRNoX0:$rs1, simm6nonzero:$imm)>;
+def : CompressPat<(QC_E_ANDAI GPRC:$rd, simm6:$imm),
+                  (C_ANDI GPRC:$rd, simm6:$imm)>;
 def : CompressPat<(QC_E_ANDI GPRC:$rs1, GPRC:$rs1, simm6:$imm),
                   (C_ANDI GPRC:$rs1, simm6:$imm)>;
 def : CompressPat<(QC_E_ADDI GPRNoX0:$rs1, GPRNoX0:$rs2, 0),
                   (C_MV GPRNoX0:$rs1, GPRNoX0:$rs2)>;
+def : CompressPat<(QC_E_ADDAI X2, simm10_lsb0000nonzero:$imm),
+                  (C_ADDI16SP X2, simm10_lsb0000nonzero:$imm)>;
 def : CompressPat<(QC_E_ADDI X2, X2, simm10_lsb0000nonzero:$imm),
                   (C_ADDI16SP X2, simm10_lsb0000nonzero:$imm)>;
 } // let isCompressOnly = true, Predicates = [HasVendorXqcilia, IsRV32]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
index e44bdcb4e2f0f..878b85b141578 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
@@ -57,7 +57,7 @@ let Predicates = [HasStdExtZicboz] in {
 def CBO_ZERO : CBO_r<0b000000000100, "cbo.zero">, Sched<[]>;
 } // Predicates = [HasStdExtZicboz]
 
-let Predicates = [HasStdExtZicbop] in {
+let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
 def PREFETCH_I : Prefetch_ri<0b00000, "prefetch.i">, Sched<[]>;
 def PREFETCH_R : Prefetch_ri<0b00001, "prefetch.r">, Sched<[]>;
 def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
@@ -69,7 +69,7 @@ def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
 
 def AddrRegImmLsb00000 : ComplexPattern<iPTR, 2, "SelectAddrRegImmLsb00000">;
 
-let Predicates = [HasStdExtZicbop] in {
+let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
   def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
                       timm, timm, (i32 0)),
             (PREFETCH_I GPR:$rs1, simm12_lsb00000:$imm12)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
index 1057eeee31d65..4abe62f4e874c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
@@ -54,6 +54,8 @@ def isSelectPseudo
                        Select_GPRNoX0_Using_CC_UImm5NonZero_QC,
                        Select_GPRNoX0_Using_CC_SImm16NonZero_QC,
                        Select_GPRNoX0_Using_CC_UImm16NonZero_QC,
+                       Select_GPR_Using_CC_UImmLog2XLen_NDS,
+                       Select_GPR_Using_CC_UImm7_NDS,
                        Select_FPR16_Using_CC_GPR,
                        Select_FPR16INX_Using_CC_GPR,
                        Select_FPR32_Using_CC_GPR,
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index 57b415dc713ac..03214fe8239ce 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -120,7 +120,8 @@ def MIPS_P8700 : RISCVProcessorModel<"mips-p8700",
                                       FeatureStdExtZifencei,
                                       FeatureStdExtZicsr,
                                       FeatureVendorXMIPSCMov,
-                                      FeatureVendorXMIPSLSP],
+                                      FeatureVendorXMIPSLSP,
+                                      FeatureVendorXMIPSCBOP],
                                      [TuneMIPSP8700]>;
 
 def ROCKET_RV32 : RISCVProcessorModel<"rocket-rv32",
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index d7aa85048eeda..2d9f38221d424 100644
--- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -517,6 +517,8 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   case RISCV::VWSUB_VV:
   case RISCV::VWSUB_VX:
   case RISCV::VWSLL_VI:
+  case RISCV::VWSLL_VX:
+  case RISCV::VWSLL_VV:
   // Vector Widening Integer Multiply Instructions
   // Destination EEW=2*SEW. Source EEW=SEW.
   case RISCV::VWMUL_VV:
@@ -1019,6 +1021,8 @@ static bool isSupportedInstr(const MachineInstr &MI) {
 
   // Vector Crypto
   case RISCV::VWSLL_VI:
+  case RISCV::VWSLL_VX:
+  case RISCV::VWSLL_VV:
 
   // Vector Mask Instructions
   // Vector Mask-Register Logical Instructions
@@ -1040,6 +1044,13 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VMSOF_M:
   case RISCV::VIOTA_M:
   case RISCV::VID_V:
+  // Vector Slide Instructions
+  case RISCV::VSLIDEUP_VX:
+  case RISCV::VSLIDEUP_VI:
+  case RISCV::VSLIDEDOWN_VX:
+  case RISCV::VSLIDEDOWN_VI:
+  case RISCV::VSLIDE1UP_VX:
+  case RISCV::VFSLIDE1UP_VF:
   // Vector Single-Width Floating-Point Add/Subtract Instructions
   case RISCV::VFADD_VF:
   case RISCV::VFADD_VV:
@@ -1096,6 +1107,8 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VFSQRT_V:
   // Vector Floating-Point Reciprocal Square-Root Estimate Instruction
   case RISCV::VFRSQRT7_V:
+  // Vector Floating-Point Reciprocal Estimate Instruction
+  case RISCV::VFREC7_V:
   // Vector Floating-Point MIN/MAX Instructions
   case RISCV::VFMIN_VF:
   case RISCV::VFMIN_VV:
@@ -1119,6 +1132,10 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VMFLE_VV:
   case RISCV::VMFGT_VF:
   case RISCV::VMFGE_VF:
+  // Vector Floating-Point Merge Instruction
+  case RISCV::VFMERGE_VFM:
+  // Vector Floating-Point Move Instruction
+  case RISCV::VFMV_V_F:
   // Single-Width Floating-Point/Integer Type-Convert Instructions
   case RISCV::VFCVT_XU_F_V:
   case RISCV::VFCVT_X_F_V:
@@ -1252,6 +1269,9 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
     return false;
   }
 
+  assert(!RISCVII::elementsDependOnVL(RISCV::getRVVMCOpcode(MI.getOpcode())) &&
+         "Instruction shouldn't be supported if elements depend on VL");
+
   assert(MI.getOperand(0).isReg() &&
          isVectorRegClass(MI.getOperand(0).getReg(), MRI) &&
          "All supported instructions produce a vector register result");
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index f7acd676461fb..07907298386c3 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -67,12 +67,13 @@ class RISCVVectorPeephole : public MachineFunctionPass {
   bool convertSameMaskVMergeToVMv(MachineInstr &MI);
   bool foldUndefPassthruVMV_V_V(MachineInstr &MI);
   bool foldVMV_V_V(MachineInstr &MI);
+  bool foldVMergeToMask(MachineInstr &MI) const;
 
   bool hasSameEEW(const MachineInstr &User, const MachineInstr &Src) const;
   bool isAllOnesMask(const MachineInstr *MaskDef) const;
   std::optional<unsigned> getConstant(const MachineOperand &VL) const;
   bool ensureDominates(const MachineOperand &Use, MachineInstr &Src) const;
-  bool isKnownSameDefs(const MachineOperand &A, const MachineOperand &B) const;
+  bool isKnownSameDefs(Register A, Register B) const;
 };
 
 } // namespace
@@ -386,13 +387,23 @@ bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const {
   return true;
 }
 
-bool RISCVVectorPeephole::isKnownSameDefs(const MachineOperand &A,
-                                          const MachineOperand &B) const {
-  if (A.getReg().isPhysical() || B.getReg().isPhysical())
+bool RISCVVectorPeephole::isKnownSameDefs(Register A, Register B) const {
+  if (A.isPhysical() || B.isPhysical())
     return false;
 
-  return TRI->lookThruCopyLike(A.getReg(), MRI) ==
-         TRI->lookThruCopyLike(B.getReg(), MRI);
+  auto LookThruVirtRegCopies = [this](Register Reg) {
+    while (MachineInstr *Def = MRI->getUniqueVRegDef(Reg)) {
+      if (!Def->isFullCopy())
+        break;
+      Register Src = Def->getOperand(1).getReg();
+      if (!Src.isVirtual())
+        break;
+      Reg = Src;
+    }
+    return Reg;
+  };
+
+  return LookThruVirtRegCopies(A) == LookThruVirtRegCopies(B);
 }
 
 /// If a PseudoVMERGE_VVM's true operand is a masked pseudo and both have the
@@ -420,7 +431,7 @@ bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) {
   const MachineOperand &TrueMask =
       True->getOperand(TrueMaskedInfo->MaskOpIdx + True->getNumExplicitDefs());
   const MachineOperand &MIMask = MI.getOperand(4);
-  if (!isKnownSameDefs(TrueMask, MIMask))
+  if (!isKnownSameDefs(TrueMask.getReg(), MIMask.getReg()))
     return false;
 
   // True's passthru needs to be equivalent to False
@@ -669,6 +680,133 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
   return true;
 }
 
+/// Try to fold away VMERGE_VVM instructions into their operands:
+///
+/// %true = PseudoVADD_VV ...
+/// %x = PseudoVMERGE_VVM_M1 %false, %false, %true, %mask
+/// ->
+/// %x = PseudoVADD_VV_M1_MASK %false, ..., %mask
+///
+/// We can only fold if vmerge's passthru operand, vmerge's false operand and
+/// %true's passthru operand (if it has one) are the same. This is because we
+/// have to consolidate them into one passthru operand in the result.
+///
+/// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
+/// mask is all ones.
+///
+/// The resulting VL is the minimum of the two VLs.
+///
+/// The resulting policy is the effective policy the vmerge would have had,
+/// i.e. whether or not it's passthru operand was implicit-def.
+bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
+  if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMERGE_VVM)
+    return false;
+
+  Register PassthruReg = MI.getOperand(1).getReg();
+  Register FalseReg = MI.getOperand(2).getReg();
+  Register TrueReg = MI.getOperand(3).getReg();
+  if (!TrueReg.isVirtual() || !MRI->hasOneUse(TrueReg))
+    return false;
+  MachineInstr &True = *MRI->getUniqueVRegDef(TrueReg);
+  if (True.getParent() != MI.getParent())
+    return false;
+  const MachineOperand &MaskOp = MI.getOperand(4);
+  MachineInstr *Mask = MRI->getUniqueVRegDef(MaskOp.getReg());
+  assert(Mask);
+
+  const RISCV::RISCVMaskedPseudoInfo *Info =
+      RISCV::lookupMaskedIntrinsicByUnmasked(True.getOpcode());
+  if (!Info)
+    return false;
+
+  // If the EEW of True is different from vmerge's SEW, then we can't fold.
+  if (!hasSameEEW(MI, True))
+    return false;
+
+  // We require that either passthru and false are the same, or that passthru
+  // is undefined.
+  if (PassthruReg && !isKnownSameDefs(PassthruReg, FalseReg))
+    return false;
+
+  // If True has a passthru operand then it needs to be the same as vmerge's
+  // False, since False will be used for the result's passthru operand.
+  Register TruePassthru = True.getOperand(True.getNumExplicitDefs()).getReg();
+  if (RISCVII::isFirstDefTiedToFirstUse(True.getDesc()) && TruePassthru &&
+      !isKnownSameDefs(TruePassthru, FalseReg))
+    return false;
+
+  // Make sure it doesn't raise any observable fp exceptions, since changing the
+  // active elements will affect how fflags is set.
+  if (True.hasUnmodeledSideEffects() || True.mayRaiseFPException())
+    return false;
+
+  const MachineOperand &VMergeVL =
+      MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
+  const MachineOperand &TrueVL =
+      True.getOperand(RISCVII::getVLOpNum(True.getDesc()));
+
+  MachineOperand MinVL = MachineOperand::CreateImm(0);
+  if (RISCV::isVLKnownLE(TrueVL, VMergeVL))
+    MinVL = TrueVL;
+  else if (RISCV::isVLKnownLE(VMergeVL, TrueVL))
+    MinVL = VMergeVL;
+  else
+    return false;
+
+  unsigned RVVTSFlags =
+      TII->get(RISCV::getRVVMCOpcode(True.getOpcode())).TSFlags;
+  if (RISCVII::elementsDependOnVL(RVVTSFlags) && !TrueVL.isIdenticalTo(MinVL))
+    return false;
+  if (RISCVII::elementsDependOnMask(RVVTSFlags) && !isAllOnesMask(Mask))
+    return false;
+
+  // Use a tumu policy, relaxing it to tail agnostic provided that the passthru
+  // operand is undefined.
+  //
+  // However, if the VL became smaller than what the vmerge had originally, then
+  // elements past VL that were previously in the vmerge's body will have moved
+  // to the tail. In that case we always need to use tail undisturbed to
+  // preserve them.
+  uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
+  if (!PassthruReg && RISCV::isVLKnownLE(VMergeVL, MinVL))
+    Policy |= RISCVVType::TAIL_AGNOSTIC;
+
+  assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&
+         "Foldable unmasked pseudo should have a policy op already");
+
+  // Make sure the mask dominates True, otherwise move down True so it does.
+  // VL will always dominate since if it's a register they need to be the same.
+  if (!ensureDominates(MaskOp, True))
+    return false;
+
+  True.setDesc(TII->get(Info->MaskedPseudo));
+
+  // Insert the mask operand.
+  // TODO: Increment MaskOpIdx by number of explicit defs?
+  True.insert(True.operands_begin() + Info->MaskOpIdx +
+                  True.getNumExplicitDefs(),
+              MachineOperand::CreateReg(MaskOp.getReg(), false));
+
+  // Update the passthru, AVL and policy.
+  True.getOperand(True.getNumExplicitDefs()).setReg(FalseReg);
+  True.removeOperand(RISCVII::getVLOpNum(True.getDesc()));
+  True.insert(True.operands_begin() + RISCVII::getVLOpNum(True.getDesc()),
+              MinVL);
+  True.getOperand(RISCVII::getVecPolicyOpNum(True.getDesc())).setImm(Policy);
+
+  MRI->replaceRegWith(True.getOperand(0).getReg(), MI.getOperand(0).getReg());
+  // Now that True is masked, constrain its operands from vr -> vrnov0.
+  for (MachineOperand &MO : True.explicit_operands()) {
+    if (!MO.isReg() || !MO.getReg().isVirtual())
+      continue;
+    MRI->constrainRegClass(
+        MO.getReg(), True.getRegClassConstraint(MO.getOperandNo(), TII, TRI));
+  }
+  MI.eraseFromParent();
+
+  return true;
+}
+
 bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
@@ -685,6 +823,9 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   bool Changed = false;
 
   for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : make_early_inc_range(MBB))
+      Changed |= foldVMergeToMask(MI);
+
     for (MachineInstr &MI : make_early_inc_range(MBB)) {
       Changed |= convertToVLMAX(MI);
       Changed |= tryToReduceVL(MI);
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index 3e7d56a9d490b..2a424e673ddf6 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -369,19 +369,6 @@ void SPIRVInstPrinter::printUnknownType(const MCInst *MI, raw_ostream &O) {
   printRemainingVariableOps(MI, NumFixedOps, O, true);
 }
 
-static void printExpr(const MCExpr *Expr, raw_ostream &O) {
-#ifndef NDEBUG
-  const MCSymbolRefExpr *SRE;
-
-  if (const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(Expr))
-    SRE = cast<MCSymbolRefExpr>(BE->getLHS());
-  else
-    SRE = cast<MCSymbolRefExpr>(Expr);
-  assert(SRE->getSpecifier() == 0);
-#endif
-  O << *Expr;
-}
-
 void SPIRVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                     raw_ostream &O) {
   if (OpNo < MI->getNumOperands()) {
@@ -393,7 +380,7 @@ void SPIRVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     else if (Op.isDFPImm())
       O << formatImm((double)Op.getDFPImm());
     else if (Op.isExpr())
-      printExpr(Op.getExpr(), O);
+      MAI.printExpr(O, *Op.getExpr());
     else
       llvm_unreachable("Unexpected operand type");
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index f73a39c6ee9da..6ec7544767c52 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -3086,43 +3086,11 @@ static SPIRVType *getCoopMatrType(const TargetExtType *ExtensionType,
       ExtensionType->getIntParameter(3), true);
 }
 
-static SPIRVType *
-getImageType(const TargetExtType *ExtensionType,
-             const SPIRV::AccessQualifier::AccessQualifier Qualifier,
-             MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
-  assert(ExtensionType->getNumTypeParameters() == 1 &&
-         "SPIR-V image builtin type must have sampled type parameter!");
-  const SPIRVType *SampledType =
-      GR->getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder,
-                               SPIRV::AccessQualifier::ReadWrite, true);
-  assert((ExtensionType->getNumIntParameters() == 7 ||
-          ExtensionType->getNumIntParameters() == 6) &&
-         "Invalid number of parameters for SPIR-V image builtin!");
-
-  SPIRV::AccessQualifier::AccessQualifier accessQualifier =
-      SPIRV::AccessQualifier::None;
-  if (ExtensionType->getNumIntParameters() == 7) {
-    accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
-                          ? SPIRV::AccessQualifier::WriteOnly
-                          : SPIRV::AccessQualifier::AccessQualifier(
-                                ExtensionType->getIntParameter(6));
-  }
-
-  // Create or get an existing type from GlobalRegistry.
-  return GR->getOrCreateOpTypeImage(
-      MIRBuilder, SampledType,
-      SPIRV::Dim::Dim(ExtensionType->getIntParameter(0)),
-      ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),
-      ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),
-      SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),
-      accessQualifier);
-}
-
 static SPIRVType *getSampledImageType(const TargetExtType *OpaqueType,
                                       MachineIRBuilder &MIRBuilder,
                                       SPIRVGlobalRegistry *GR) {
-  SPIRVType *OpaqueImageType = getImageType(
-      OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder, GR);
+  SPIRVType *OpaqueImageType = GR->getImageType(
+      OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder);
   // Create or get an existing type from GlobalRegistry.
   return GR->getOrCreateOpTypeSampledImage(OpaqueImageType, MIRBuilder);
 }
@@ -3293,7 +3261,7 @@ SPIRVType *lowerBuiltinType(const Type *OpaqueType,
 
     switch (TypeRecord->Opcode) {
     case SPIRV::OpTypeImage:
-      TargetType = getImageType(BuiltinType, AccessQual, MIRBuilder, GR);
+      TargetType = GR->getImageType(BuiltinType, AccessQual, MIRBuilder);
       break;
     case SPIRV::OpTypePipe:
       TargetType = getPipeType(BuiltinType, MIRBuilder, GR);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 401a762cd62a3..6897865eb4e15 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -1634,6 +1634,7 @@ def : BuiltinType<"spirv.Event", OpTypeEvent>;
 def : BuiltinType<"spirv.Sampler", OpTypeSampler>;
 def : BuiltinType<"spirv.DeviceEvent", OpTypeDeviceEvent>;
 def : BuiltinType<"spirv.Image", OpTypeImage>;
+def : BuiltinType<"spirv.SignedImage", OpTypeImage>;
 def : BuiltinType<"spirv.SampledImage", OpTypeSampledImage>;
 def : BuiltinType<"spirv.Pipe", OpTypePipe>;
 def : BuiltinType<"spirv.CooperativeMatrixKHR", OpTypeCooperativeMatrixKHR>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index fbaca4e0e4d81..2726203d253ad 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -100,7 +100,9 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
          SPIRV::Extension::Extension::SPV_INTEL_ternary_bitwise_function},
         {"SPV_INTEL_2d_block_io",
          SPIRV::Extension::Extension::SPV_INTEL_2d_block_io},
-        {"SPV_INTEL_int4", SPIRV::Extension::Extension::SPV_INTEL_int4}};
+        {"SPV_INTEL_int4", SPIRV::Extension::Extension::SPV_INTEL_int4},
+        {"SPV_KHR_float_controls2",
+         SPIRV::Extension::Extension::SPV_KHR_float_controls2}};
 
 bool SPIRVExtensionsParser::parse(cl::Option &O, StringRef ArgName,
                                   StringRef ArgValue,
@@ -119,6 +121,13 @@ bool SPIRVExtensionsParser::parse(cl::Option &O, StringRef ArgName,
       continue;
     }
 
+    if (Token.size() == 3 && Token.upper() == "KHR") {
+      for (const auto &[ExtensionName, ExtensionEnum] : SPIRVExtensionMap)
+        if (StringRef(ExtensionName).starts_with("SPV_KHR_"))
+          EnabledExtensions.insert(ExtensionEnum);
+      continue;
+    }
+
     if (Token.empty() || (!Token.starts_with("+") && !Token.starts_with("-")))
       return O.error("Invalid extension list format: " + Token.str());
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index cc95fde6a516d..b90e1aadbb5a1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -663,7 +663,8 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
     auto *II = dyn_cast<IntrinsicInst>(I);
     if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
       auto *HandleType = cast<TargetExtType>(II->getOperand(0)->getType());
-      if (HandleType->getTargetExtName() == "spirv.Image") {
+      if (HandleType->getTargetExtName() == "spirv.Image" ||
+          HandleType->getTargetExtName() == "spirv.SignedImage") {
         if (II->hasOneUse()) {
           auto *U = *II->users().begin();
           Ty = cast<Instruction>(U)->getAccessType();
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 292b83e05b56d..83fccdc2bdba3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -1406,6 +1406,40 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateLayoutType(
   return SPIRVStructType;
 }
 
+SPIRVType *SPIRVGlobalRegistry::getImageType(
+    const TargetExtType *ExtensionType,
+    const SPIRV::AccessQualifier::AccessQualifier Qualifier,
+    MachineIRBuilder &MIRBuilder) {
+  assert(ExtensionType->getNumTypeParameters() == 1 &&
+         "SPIR-V image builtin type must have sampled type parameter!");
+  const SPIRVType *SampledType =
+      getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder,
+                           SPIRV::AccessQualifier::ReadWrite, true);
+  assert((ExtensionType->getNumIntParameters() == 7 ||
+          ExtensionType->getNumIntParameters() == 6) &&
+         "Invalid number of parameters for SPIR-V image builtin!");
+
+  SPIRV::AccessQualifier::AccessQualifier accessQualifier =
+      SPIRV::AccessQualifier::None;
+  if (ExtensionType->getNumIntParameters() == 7) {
+    accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
+                          ? SPIRV::AccessQualifier::WriteOnly
+                          : SPIRV::AccessQualifier::AccessQualifier(
+                                ExtensionType->getIntParameter(6));
+  }
+
+  // Create or get an existing type from GlobalRegistry.
+  SPIRVType *R = getOrCreateOpTypeImage(
+      MIRBuilder, SampledType,
+      SPIRV::Dim::Dim(ExtensionType->getIntParameter(0)),
+      ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),
+      ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),
+      SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),
+      accessQualifier);
+  SPIRVToLLVMType[R] = ExtensionType;
+  return R;
+}
+
 SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeImage(
     MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, SPIRV::Dim::Dim Dim,
     uint32_t Depth, uint32_t Arrayed, uint32_t Multisampled, uint32_t Sampled,
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 35f616a1981d2..7ef812828b7cc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -501,6 +501,13 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
                                  MachineIRBuilder &MIRBuilder);
   bool hasBlockDecoration(SPIRVType *Type) const;
 
+  SPIRVType *
+  getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType,
+                         SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
+                         uint32_t Multisampled, uint32_t Sampled,
+                         SPIRV::ImageFormat::ImageFormat ImageFormat,
+                         SPIRV::AccessQualifier::AccessQualifier AccQual);
+
 public:
   Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder,
                             SPIRVType *SpvType, bool EmitIR,
@@ -607,11 +614,9 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
                                    const TargetExtType *T, bool EmitIr = false);
 
   SPIRVType *
-  getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType,
-                         SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
-                         uint32_t Multisampled, uint32_t Sampled,
-                         SPIRV::ImageFormat::ImageFormat ImageFormat,
-                         SPIRV::AccessQualifier::AccessQualifier AccQual);
+  getImageType(const TargetExtType *ExtensionType,
+               const SPIRV::AccessQualifier::AccessQualifier Qualifier,
+               MachineIRBuilder &MIRBuilder);
 
   SPIRVType *getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder);
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 851e0c6b81fcf..40a0bd97adaf9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -341,6 +341,14 @@ class SPIRVInstructionSelector : public InstructionSelector {
                                 GIntrinsic &HandleDef, MachineInstr &Pos) const;
 };
 
+bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
+  const TargetExtType *TET = cast<TargetExtType>(HandleType);
+  if (TET->getTargetExtName() == "spirv.Image") {
+    return false;
+  }
+  assert(TET->getTargetExtName() == "spirv.SignedImage");
+  return TET->getTypeParameter(0)->isIntegerTy();
+}
 } // end anonymous namespace
 
 #define GET_GLOBALISEL_IMPL
@@ -1195,12 +1203,17 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
 
     Register IdxReg = IntPtrDef->getOperand(3).getReg();
     if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
-      return BuildMI(*I.getParent(), I, I.getDebugLoc(),
-                     TII.get(SPIRV::OpImageWrite))
-          .addUse(NewHandleReg)
-          .addUse(IdxReg)
-          .addUse(StoreVal)
-          .constrainAllUses(TII, TRI, RBI);
+      auto BMI = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                         TII.get(SPIRV::OpImageWrite))
+                     .addUse(NewHandleReg)
+                     .addUse(IdxReg)
+                     .addUse(StoreVal);
+
+      const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(HandleType);
+      if (sampledTypeIsSignedInteger(LLVMHandleType))
+        BMI.addImm(0x1000); // SignExtend
+
+      return BMI.constrainAllUses(TII, TRI, RBI);
     }
   }
 
@@ -3246,25 +3259,35 @@ bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg,
                                                  Register ImageReg,
                                                  Register IdxReg, DebugLoc Loc,
                                                  MachineInstr &Pos) const {
+  SPIRVType *ImageType = GR.getSPIRVTypeForVReg(ImageReg);
+  assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
+         "ImageReg is not an image type.");
+  bool IsSignedInteger =
+      sampledTypeIsSignedInteger(GR.getTypeForSPIRVType(ImageType));
+
   uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
   if (ResultSize == 4) {
-    return BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
-        .addDef(ResVReg)
-        .addUse(GR.getSPIRVTypeID(ResType))
-        .addUse(ImageReg)
-        .addUse(IdxReg)
-        .constrainAllUses(TII, TRI, RBI);
+    auto BMI = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
+                   .addDef(ResVReg)
+                   .addUse(GR.getSPIRVTypeID(ResType))
+                   .addUse(ImageReg)
+                   .addUse(IdxReg);
+
+    if (IsSignedInteger)
+      BMI.addImm(0x1000); // SignExtend
+    return BMI.constrainAllUses(TII, TRI, RBI);
   }
 
   SPIRVType *ReadType = widenTypeToVec4(ResType, Pos);
   Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType));
-  bool Succeed =
-      BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
-          .addDef(ReadReg)
-          .addUse(GR.getSPIRVTypeID(ReadType))
-          .addUse(ImageReg)
-          .addUse(IdxReg)
-          .constrainAllUses(TII, TRI, RBI);
+  auto BMI = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
+                 .addDef(ReadReg)
+                 .addUse(GR.getSPIRVTypeID(ReadType))
+                 .addUse(ImageReg)
+                 .addUse(IdxReg);
+  if (IsSignedInteger)
+    BMI.addImm(0x1000); // SignExtend
+  bool Succeed = BMI.constrainAllUses(TII, TRI, RBI);
   if (!Succeed)
     return false;
 
@@ -4106,6 +4129,7 @@ bool SPIRVInstructionSelector::loadHandleBeforePosition(
   // handle is the image object. So images get an extra load.
   uint32_t LoadOpcode =
       IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
+  GR.assignSPIRVTypeToVReg(ResType, HandleReg, *Pos.getMF());
   return BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(),
                  TII.get(LoadOpcode))
       .addDef(HandleReg)
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index b71a9dd68dd44..ad976e5288927 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -70,6 +70,8 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
   AvoidCapabilitiesSet AvoidCaps;
   if (!ST.isShader())
     AvoidCaps.S.insert(SPIRV::Capability::Shader);
+  else
+    AvoidCaps.S.insert(SPIRV::Capability::Kernel);
 
   VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, i);
   VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, i);
@@ -88,8 +90,11 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
   } else if (MinVerOK && MaxVerOK) {
     if (ReqCaps.size() == 1) {
       auto Cap = ReqCaps[0];
-      if (Reqs.isCapabilityAvailable(Cap))
+      if (Reqs.isCapabilityAvailable(Cap)) {
+        ReqExts.append(getSymbolicOperandExtensions(
+            SPIRV::OperandCategory::CapabilityOperand, Cap));
         return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
+      }
     } else {
       // By SPIR-V specification: "If an instruction, enumerant, or other
       // feature specifies multiple enabling capabilities, only one such
@@ -103,8 +108,11 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
           UseCaps.push_back(Cap);
       for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
         auto Cap = UseCaps[i];
-        if (i == Sz - 1 || !AvoidCaps.S.contains(Cap))
+        if (i == Sz - 1 || !AvoidCaps.S.contains(Cap)) {
+          ReqExts.append(getSymbolicOperandExtensions(
+              SPIRV::OperandCategory::CapabilityOperand, Cap));
           return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
+        }
       }
     }
   }
@@ -1975,6 +1983,14 @@ static unsigned getFastMathFlags(const MachineInstr &I) {
   return Flags;
 }
 
+static bool isFastMathMathModeAvailable(const SPIRVSubtarget &ST) {
+  if (ST.isKernel())
+    return true;
+  if (ST.getSPIRVVersion() < VersionTuple(1, 2))
+    return false;
+  return ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
+}
+
 static void handleMIFlagDecoration(MachineInstr &I, const SPIRVSubtarget &ST,
                                    const SPIRVInstrInfo &TII,
                                    SPIRV::RequirementHandler &Reqs) {
@@ -1998,8 +2014,12 @@ static void handleMIFlagDecoration(MachineInstr &I, const SPIRVSubtarget &ST,
   unsigned FMFlags = getFastMathFlags(I);
   if (FMFlags == SPIRV::FPFastMathMode::None)
     return;
-  Register DstReg = I.getOperand(0).getReg();
-  buildOpDecorate(DstReg, I, TII, SPIRV::Decoration::FPFastMathMode, {FMFlags});
+
+  if (isFastMathMathModeAvailable(ST)) {
+    Register DstReg = I.getOperand(0).getReg();
+    buildOpDecorate(DstReg, I, TII, SPIRV::Decoration::FPFastMathMode,
+                    {FMFlags});
+  }
 }
 
 // Walk all functions and add decorations related to MI flags.
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index f1aae42ea2be0..548e9b717c161 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -319,6 +319,7 @@ defm SPV_INTEL_ternary_bitwise_function : ExtensionOperand<120>;
 defm SPV_INTEL_subgroup_matrix_multiply_accumulate : ExtensionOperand<121>;
 defm SPV_INTEL_2d_block_io : ExtensionOperand<122>;
 defm SPV_INTEL_int4 : ExtensionOperand<123>;
+defm SPV_KHR_float_controls2 : ExtensionOperand<124>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Capabilities enum values and at the same time
@@ -489,6 +490,8 @@ defm DotProductInput4x8Bit : CapabilityOperand<6017, 0x10600, 0, [SPV_KHR_intege
 defm DotProductInput4x8BitPacked : CapabilityOperand<6018, 0x10600, 0, [SPV_KHR_integer_dot_product], []>;
 defm DotProduct : CapabilityOperand<6019, 0x10600, 0, [SPV_KHR_integer_dot_product], []>;
 defm GroupNonUniformRotateKHR : CapabilityOperand<6026, 0, 0, [SPV_KHR_subgroup_rotate], [GroupNonUniform]>;
+defm FloatControls2
+    : CapabilityOperand<6029, 0x10200, 0, [SPV_KHR_float_controls2], []>;
 defm AtomicFloat32AddEXT : CapabilityOperand<6033, 0, 0, [SPV_EXT_shader_atomic_float_add], []>;
 defm AtomicFloat64AddEXT : CapabilityOperand<6034, 0, 0, [SPV_EXT_shader_atomic_float_add], []>;
 defm AtomicFloat16AddEXT : CapabilityOperand<6095, 0, 0, [SPV_EXT_shader_atomic_float16_add], []>;
@@ -1239,7 +1242,7 @@ defm XfbBuffer : DecorationOperand<36, 0, 0, [], [TransformFeedback]>;
 defm XfbStride : DecorationOperand<37, 0, 0, [], [TransformFeedback]>;
 defm FuncParamAttr : DecorationOperand<38, 0, 0, [], [Kernel]>;
 defm FPRoundingMode : DecorationOperand<39, 0, 0, [], []>;
-defm FPFastMathMode : DecorationOperand<40, 0, 0, [], [Kernel]>;
+defm FPFastMathMode : DecorationOperand<40, 0, 0, [], [Kernel, FloatControls2]>;
 defm LinkageAttributes : DecorationOperand<41, 0, 0, [], [Linkage]>;
 defm NoContraction : DecorationOperand<42, 0, 0, [], [Shader]>;
 defm InputAttachmentIndex : DecorationOperand<43, 0, 0, [], [InputAttachment]>;
diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index f1009999dc1b7..7c1db3cfcd6b4 100644
--- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -373,7 +373,7 @@ class SparcOperand : public MCParsedAsmOperand {
     return EndLoc;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Token:     OS << "Token: " << getToken() << "\n"; break;
     case k_Register:  OS << "Reg: #" << getReg() << "\n"; break;
@@ -381,9 +381,10 @@ class SparcOperand : public MCParsedAsmOperand {
     case k_MemoryReg: OS << "Mem: " << getMemBase() << "+"
                          << getMemOffsetReg() << "\n"; break;
     case k_MemoryImm: assert(getMemOff() != nullptr);
-      OS << "Mem: " << getMemBase()
-         << "+" << *getMemOff()
-         << "\n"; break;
+      OS << "Mem: " << getMemBase() << "+";
+      MAI.printExpr(OS, *getMemOff());
+      OS << "\n";
+      break;
     case k_ASITag:
       OS << "ASI tag: " << getASITag() << "\n";
       break;
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index cc1b7e93b31cb..ba023af062924 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -12,7 +12,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -182,15 +181,15 @@ std::optional<MCFixupKind> SparcAsmBackend::getFixupKind(StringRef Name) const {
 MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // clang-format off
   const static MCFixupKindInfo InfosBE[Sparc::NumTargetFixupKinds] = {
-    // name                    offset bits  flags
-    { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_sparc_13",        19,     13,  0 },
+      // name                    offset bits flags
+      {"fixup_sparc_call30",     2,     30,  0},
+      {"fixup_sparc_13",        19,     13,  0},
   };
 
   const static MCFixupKindInfo InfosLE[Sparc::NumTargetFixupKinds] = {
-    // name                    offset bits  flags
-    { "fixup_sparc_call30",     0,     30,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_sparc_13",         0,     13,  0 },
+      // name                    offset bits flags
+      {"fixup_sparc_call30",     0,     30,  0},
+      {"fixup_sparc_13",         0,     13,  0},
   };
   // clang-format on
 
@@ -208,22 +207,22 @@ MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   MCFixupKindInfo Info{};
   switch (uint16_t(Kind)) {
   case ELF::R_SPARC_PC10:
-    Info = {"", 22, 10, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 22, 10, 0};
     break;
   case ELF::R_SPARC_PC22:
-    Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 10, 22, 0};
     break;
   case ELF::R_SPARC_WDISP10:
-    Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 0, 32, 0};
     break;
   case ELF::R_SPARC_WDISP16:
-    Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 0, 32, 0};
     break;
   case ELF::R_SPARC_WDISP19:
-    Info = {"", 13, 19, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 13, 19, 0};
     break;
   case ELF::R_SPARC_WDISP22:
-    Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 10, 22, 0};
     break;
 
   case ELF::R_SPARC_HI22:
@@ -253,10 +252,11 @@ MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   return Info;
 }
 
-void SparcAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void SparcAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target,
                                  MutableArrayRef<char> Data, uint64_t Value,
                                  bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   if (!IsResolved)
     return;
   Value = adjustFixupValue(Fixup.getKind(), Value);
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index 8ba99719946a2..2335853469467 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -91,6 +91,22 @@ class SparcMCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case ELF::R_SPARC_PC10:
+  case ELF::R_SPARC_PC22:
+  case ELF::R_SPARC_WDISP10:
+  case ELF::R_SPARC_WDISP16:
+  case ELF::R_SPARC_WDISP19:
+  case ELF::R_SPARC_WDISP22:
+  case Sparc::fixup_sparc_call30:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void SparcMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                            SmallVectorImpl<char> &CB,
                                            SmallVectorImpl<MCFixup> &Fixups,
@@ -135,7 +151,7 @@ getMachineOpValue(const MCInst &MI, const MCOperand &MO,
   assert(MO.isExpr());
   const MCExpr *Expr = MO.getExpr();
   if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
+    addFixup(Fixups, 0, Expr, SExpr->getSpecifier());
     return 0;
   }
 
@@ -165,10 +181,10 @@ unsigned SparcMCCodeEmitter::getSImm5OpValue(const MCInst &MI, unsigned OpNo,
     return CE->getValue();
 
   if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
+    addFixup(Fixups, 0, Expr, SExpr->getSpecifier());
     return 0;
   }
-  Fixups.push_back(MCFixup::create(0, Expr, ELF::R_SPARC_5));
+  addFixup(Fixups, 0, Expr, ELF::R_SPARC_5);
   return 0;
 }
 
@@ -191,10 +207,10 @@ SparcMCCodeEmitter::getSImm13OpValue(const MCInst &MI, unsigned OpNo,
     return CE->getValue();
 
   if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
+    addFixup(Fixups, 0, Expr, SExpr->getSpecifier());
     return 0;
   }
-  Fixups.push_back(MCFixup::create(0, Expr, Sparc::fixup_sparc_13));
+  addFixup(Fixups, 0, Expr, Sparc::fixup_sparc_13);
   return 0;
 }
 
@@ -209,7 +225,7 @@ getCallTargetOpValue(const MCInst &MI, unsigned OpNo,
   }
 
   const MCOperand &MO = MI.getOperand(OpNo);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Sparc::fixup_sparc_call30));
+  addFixup(Fixups, 0, MO.getExpr(), Sparc::fixup_sparc_call30);
   return 0;
 }
 
@@ -221,7 +237,7 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP22));
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP22);
   return 0;
 }
 
@@ -232,7 +248,7 @@ unsigned SparcMCCodeEmitter::getBranchPredTargetOpValue(
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP19));
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP19);
   return 0;
 }
 
@@ -243,8 +259,7 @@ unsigned SparcMCCodeEmitter::getBranchOnRegTargetOpValue(
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP16));
-
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP16);
   return 0;
 }
 
@@ -255,8 +270,7 @@ unsigned SparcMCCodeEmitter::getCompareAndBranchTargetOpValue(
   if (MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP10));
-
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP10);
   return 0;
 }
 
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 392d16883d990..21dbe8f585b3e 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "SparcISelLowering.h"
-#include "MCTargetDesc/SparcMCAsmInfo.h"
 #include "MCTargetDesc/SparcMCTargetDesc.h"
 #include "SparcMachineFunctionInfo.h"
 #include "SparcRegisterInfo.h"
@@ -1045,11 +1044,9 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   // The InGlue in necessary since all emitted instructions must be
   // stuck together.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Register Reg = RegsToPass[i].first;
-    if (!isTailCall)
-      Reg = toCallerWindow(Reg);
-    Chain = DAG.getCopyToReg(Chain, dl, Reg, RegsToPass[i].second, InGlue);
+  for (const auto &[OrigReg, N] : RegsToPass) {
+    Register Reg = isTailCall ? OrigReg : toCallerWindow(OrigReg);
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -1070,11 +1067,9 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
   if (hasStructRetAttr)
     Ops.push_back(DAG.getTargetConstant(SRetArgSize, dl, MVT::i32));
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Register Reg = RegsToPass[i].first;
-    if (!isTailCall)
-      Reg = toCallerWindow(Reg);
-    Ops.push_back(DAG.getRegister(Reg, RegsToPass[i].second.getValueType()));
+  for (const auto &[OrigReg, N] : RegsToPass) {
+    Register Reg = isTailCall ? OrigReg : toCallerWindow(OrigReg);
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
   }
 
   // Add a register mask operand representing the call-preserved registers.
@@ -1376,9 +1371,8 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   // necessary since all emitted instructions must be stuck together in order
   // to pass the live physical registers.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, DL,
-                             RegsToPass[i].first, RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -1396,9 +1390,8 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(Chain);
   Ops.push_back(Callee);
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const SparcRegisterInfo *TRI = Subtarget->getRegisterInfo();
@@ -2881,9 +2874,6 @@ static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (TLI.verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   uint64_t depth = Op.getConstantOperandVal(0);
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 04a4c36109246..597ce8eabfeb2 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -294,7 +294,7 @@ class SystemZOperand : public MCParsedAsmOperand {
   // Override MCParsedAsmOperand.
   SMLoc getStartLoc() const override { return StartLoc; }
   SMLoc getEndLoc() const override { return EndLoc; }
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   /// getLocRange - Get the range between the first and last token of this
   /// operand.
@@ -729,22 +729,7 @@ static struct InsnMatchEntry InsnMatchTable[] = {
     { MCK_U48Imm, MCK_VR128, MCK_BDAddr64Disp12, MCK_U8Imm } }
 };
 
-static void printMCExpr(const MCExpr *E, raw_ostream &OS) {
-  if (!E)
-    return;
-  if (auto *CE = dyn_cast<MCConstantExpr>(E))
-    OS << *CE;
-  else if (auto *UE = dyn_cast<MCUnaryExpr>(E))
-    OS << *UE;
-  else if (auto *BE = dyn_cast<MCBinaryExpr>(E))
-    OS << *BE;
-  else if (auto *SRE = dyn_cast<MCSymbolRefExpr>(E))
-    OS << *SRE;
-  else
-    OS << *E;
-}
-
-void SystemZOperand::print(raw_ostream &OS) const {
+void SystemZOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case KindToken:
     OS << "Token:" << getToken();
@@ -754,24 +739,26 @@ void SystemZOperand::print(raw_ostream &OS) const {
     break;
   case KindImm:
     OS << "Imm:";
-    printMCExpr(getImm(), OS);
+    MAI.printExpr(OS, *getImm());
     break;
   case KindImmTLS:
     OS << "ImmTLS:";
-    printMCExpr(getImmTLS().Imm, OS);
+    MAI.printExpr(OS, *getImmTLS().Imm);
     if (getImmTLS().Sym) {
       OS << ", ";
-      printMCExpr(getImmTLS().Sym, OS);
+      MAI.printExpr(OS, *getImmTLS().Sym);
     }
     break;
   case KindMem: {
     const MemOp &Op = getMem();
-    OS << "Mem:" << *cast<MCConstantExpr>(Op.Disp);
+    OS << "Mem:";
+    MAI.printExpr(OS, *cast<MCConstantExpr>(Op.Disp));
     if (Op.Base) {
       OS << "(";
-      if (Op.MemKind == BDLMem)
-        OS << *cast<MCConstantExpr>(Op.Length.Imm) << ",";
-      else if (Op.MemKind == BDRMem)
+      if (Op.MemKind == BDLMem) {
+        MAI.printExpr(OS, *cast<MCConstantExpr>(Op.Length.Imm));
+        OS << ',';
+      } else if (Op.MemKind == BDRMem)
         OS << SystemZGNUInstPrinter::getRegisterName(Op.Length.Reg) << ",";
       if (Op.Index)
         OS << SystemZGNUInstPrinter::getRegisterName(Op.Index) << ",";
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
index ec8c810809301..9121f0d44936f 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
@@ -12,7 +12,7 @@
 #include "llvm/Support/Signals.h"
 #include <sstream>
 
-#include <cmath>
+using namespace llvm;
 
 void SystemZHLASMAsmStreamer::EmitEOL() {
   // Comments are emitted on a new line before the instruction.
@@ -280,3 +280,8 @@ void SystemZHLASMAsmStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
   emitHLASMValueImpl(Value, Size, true);
   EmitEOL();
 }
+
+void SystemZHLASMAsmStreamer::emitEnd() {
+  OS << " END";
+  EmitEOL();
+}
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
index edaaf984df651..c5275339ce016 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
@@ -10,6 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZHLASMASMSTREAMER_H
+#define LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZHLASMASMSTREAMER_H
+
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCAsmBackend.h"
@@ -24,7 +27,7 @@
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/FormattedStream.h"
 
-using namespace llvm;
+namespace llvm {
 
 class SystemZHLASMAsmStreamer final : public MCStreamer {
   constexpr static size_t InstLimit = 80;
@@ -119,4 +122,9 @@ class SystemZHLASMAsmStreamer final : public MCStreamer {
   void emitHLASMValueImpl(const MCExpr *Value, unsigned Size,
                           bool Parens = false);
   /// @}
+
+  void emitEnd();
 };
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZHLASMASMSTREAMER_H
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
index 297fdc8325928..af79070db7b96 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
@@ -67,7 +67,7 @@ void SystemZInstPrinterCommon::printUImmOperand(const MCInst *MI, int OpNum,
                                                 raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   if (MO.isExpr()) {
-    O << *MO.getExpr();
+    MAI.printExpr(O, *MO.getExpr());
     return;
   }
   uint64_t Value = static_cast<uint64_t>(MO.getImm());
@@ -80,7 +80,7 @@ void SystemZInstPrinterCommon::printSImmOperand(const MCInst *MI, int OpNum,
                                                 raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   if (MO.isExpr()) {
-    O << *MO.getExpr();
+    MAI.printExpr(O, *MO.getExpr());
     return;
   }
   int64_t Value = MI->getOperand(OpNum).getImm();
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 4ec988338c1ad..d5f849235a841 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -13,7 +13,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -113,7 +112,6 @@ class SystemZMCAsmBackend : public MCAsmBackend {
   // Override MCAsmBackend
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
-  bool shouldForceRelocation(const MCFixup &, const MCValue &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
@@ -143,7 +141,7 @@ MCFixupKindInfo SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_390_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -153,15 +151,13 @@ MCFixupKindInfo SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   return SystemZ::MCFixupKindInfos[Kind - FirstTargetFixupKind];
 }
 
-bool SystemZMCAsmBackend::shouldForceRelocation(const MCFixup &,
-                                                const MCValue &Target) {
-  return Target.getSpecifier();
-}
-
-void SystemZMCAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void SystemZMCAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                      const MCValue &Target,
                                      MutableArrayRef<char> Data, uint64_t Value,
                                      bool IsResolved) {
+  if (Target.getSpecifier())
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
index b161eed95d6e2..59d1db784c688 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
@@ -129,6 +129,11 @@ class SystemZMCCodeEmitter : public MCCodeEmitter {
 };
 
 } // end anonymous namespace
+  //
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
 
 void SystemZMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                              SmallVectorImpl<char> &CB,
@@ -170,8 +175,7 @@ uint64_t SystemZMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNum,
     unsigned OpBitSize =
         SystemZ::MCFixupKindInfos[Kind - FirstTargetFixupKind].TargetSize;
     uint32_t BitOffset = MIBitSize - RawBitOffset - OpBitSize;
-    Fixups.push_back(MCFixup::create(BitOffset >> 3, MO.getExpr(),
-                                     (MCFixupKind)Kind, MI.getLoc()));
+    addFixup(Fixups, BitOffset >> 3, MO.getExpr(), Kind);
     return 0;
   }
   llvm_unreachable("Unexpected operand type!");
@@ -203,16 +207,15 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
       // into MI.  Add Offset to the relocation value to cancel out
       // this difference.
       const MCExpr *OffsetExpr = MCConstantExpr::create(Offset, Ctx);
-      Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx);
+      Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx, Loc);
     }
   }
-  Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind, Loc));
+  addFixup(Fixups, Offset, Expr, Kind, true);
 
   // Output the fixup for the TLS marker if present.
   if (AllowTLS && OpNum + 1 < MI.getNumOperands()) {
     const MCOperand &MOTLS = MI.getOperand(OpNum + 1);
-    Fixups.push_back(MCFixup::create(
-        0, MOTLS.getExpr(), (MCFixupKind)SystemZ::FK_390_TLS_CALL, Loc));
+    addFixup(Fixups, 0, MOTLS.getExpr(), SystemZ::FK_390_TLS_CALL);
   }
   return 0;
 }
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
index 512e51c0f9339..1ff4217018f1f 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZMCFIXUPS_H
 #define LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZMCFIXUPS_H
 
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 
 namespace llvm {
 namespace SystemZ {
@@ -41,26 +41,28 @@ enum FixupKind {
   NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
 };
 
+// clang-format off
 const static MCFixupKindInfo MCFixupKindInfos[SystemZ::NumTargetFixupKinds] = {
-    {"FK_390_PC12DBL", 4, 12, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_PC16DBL", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_PC24DBL", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_PC32DBL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_TLS_CALL", 0, 0, 0},
-    {"FK_390_S8Imm", 0, 8, 0},
-    {"FK_390_S16Imm", 0, 16, 0},
-    {"FK_390_S20Imm", 4, 20, 0},
-    {"FK_390_S32Imm", 0, 32, 0},
-    {"FK_390_U1Imm", 0, 1, 0},
-    {"FK_390_U2Imm", 0, 2, 0},
-    {"FK_390_U3Imm", 0, 3, 0},
-    {"FK_390_U4Imm", 0, 4, 0},
-    {"FK_390_U8Imm", 0, 8, 0},
-    {"FK_390_U12Imm", 4, 12, 0},
-    {"FK_390_U16Imm", 0, 16, 0},
-    {"FK_390_U32Imm", 0, 32, 0},
-    {"FK_390_U48Imm", 0, 48, 0},
+    {"FK_390_PC12DBL", 4, 12, 0},
+    {"FK_390_PC16DBL", 0, 16, 0},
+    {"FK_390_PC24DBL", 0, 24, 0},
+    {"FK_390_PC32DBL", 0, 32, 0},
+    {"FK_390_TLS_CALL",0,  0, 0},
+    {"FK_390_S8Imm",   0,  8, 0},
+    {"FK_390_S16Imm",  0, 16, 0},
+    {"FK_390_S20Imm",  4, 20, 0},
+    {"FK_390_S32Imm",  0, 32, 0},
+    {"FK_390_U1Imm",   0,  1, 0},
+    {"FK_390_U2Imm",   0,  2, 0},
+    {"FK_390_U3Imm",   0,  3, 0},
+    {"FK_390_U4Imm",   0,  4, 0},
+    {"FK_390_U8Imm",   0,  8, 0},
+    {"FK_390_U12Imm",  4, 12, 0},
+    {"FK_390_U16Imm",  0, 16, 0},
+    {"FK_390_U32Imm",  0, 32, 0},
+    {"FK_390_U48Imm",  0, 48, 0},
 };
+// clang-format on
 } // end namespace SystemZ
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 86e340b7ff1bd..faef5bb926780 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -203,7 +203,7 @@ static MCInstPrinter *createSystemZMCInstPrinter(const Triple &T,
 static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
                                                  MCInstPrinter *InstPrint) {
-  if (S.getContext().getTargetTriple().isOSzOS())
+  if (S.getContext().getTargetTriple().isOSzOS() && !GNUAsOnzOSCL)
     return new SystemZTargetHLASMStreamer(S, OS);
   else
     return new SystemZTargetGNUStreamer(S, OS);
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
index 7720678097440..1a3e373f25374 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetStreamer.h"
+#include "SystemZHLASMAsmStreamer.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 
@@ -33,10 +34,16 @@ void SystemZTargetStreamer::emitConstantPools() {
   EXRLTargets2Sym.clear();
 }
 
+SystemZHLASMAsmStreamer &SystemZTargetHLASMStreamer::getHLASMStreamer() {
+  return static_cast<SystemZHLASMAsmStreamer &>(getStreamer());
+}
+
 void SystemZTargetHLASMStreamer::emitExtern(StringRef Sym) {
   getStreamer().emitRawText(Twine(" EXTRN ") + Twine(Sym));
 }
 
+void SystemZTargetHLASMStreamer::emitEnd() { getHLASMStreamer().emitEnd(); }
+
 // HLASM statements can only perform a single operation at a time
 const MCExpr *SystemZTargetHLASMStreamer::createWordDiffExpr(
     MCContext &Ctx, const MCSymbol *Hi, const MCSymbol *Lo) {
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h
index 7e3c5f00525f4..3fc09bc8c683a 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h
@@ -20,6 +20,7 @@
 #include <utility>
 
 namespace llvm {
+class SystemZHLASMAsmStreamer;
 
 class SystemZTargetStreamer : public MCTargetStreamer {
 public:
@@ -57,6 +58,7 @@ class SystemZTargetStreamer : public MCTargetStreamer {
   virtual void emitMachine(StringRef CPUOrCommand) {};
 
   virtual void emitExtern(StringRef Str) {};
+  virtual void emitEnd() {};
 
   virtual const MCExpr *createWordDiffExpr(MCContext &Ctx, const MCSymbol *Hi,
                                            const MCSymbol *Lo) {
@@ -77,7 +79,9 @@ class SystemZTargetHLASMStreamer : public SystemZTargetStreamer {
 public:
   SystemZTargetHLASMStreamer(MCStreamer &S, formatted_raw_ostream &OS)
       : SystemZTargetStreamer(S), OS(OS) {}
+  SystemZHLASMAsmStreamer &getHLASMStreamer();
   void emitExtern(StringRef Sym) override;
+  void emitEnd() override;
   const MCExpr *createWordDiffExpr(MCContext &Ctx, const MCSymbol *Hi,
                                    const MCSymbol *Lo) override;
 };
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index eb0365d1f1893..e31d7c6a86476 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -1114,6 +1114,9 @@ void SystemZAsmPrinter::emitEndOfAsmFile(Module &M) {
     emitIDRLSection(M);
   }
   emitAttributes(M);
+  // Emit the END instruction in case of HLASM output. This must be the last
+  // instruction in the source file.
+  getTargetStreamer()->emitEnd();
 }
 
 void SystemZAsmPrinter::emitADASection() {
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 831618c4eff5f..262b607b953cc 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -2387,9 +2387,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
   }
 
   // Build a sequence of copy-to-reg nodes, chained and glued together.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
-                             RegsToPass[I].second, Glue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, Glue);
     Glue = Chain.getValue(1);
   }
 
@@ -2400,9 +2399,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
-    Ops.push_back(DAG.getRegister(RegsToPass[I].first,
-                                  RegsToPass[I].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
@@ -4254,9 +4252,6 @@ SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   SDLoc DL(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
index 7b206e6b100a8..20f561a8dac34 100644
--- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
+++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
@@ -409,7 +409,7 @@ class VEOperand : public MCParsedAsmOperand {
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Token:
       OS << "Token: " << getToken() << "\n";
@@ -422,29 +422,43 @@ class VEOperand : public MCParsedAsmOperand {
       break;
     case k_MemoryRegRegImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: #" << getMemBase() << "+#" << getMemIndexReg() << "+"
-         << *getMemOffset() << "\n";
+      OS << "Mem: #" << getMemBase() << "+#" << getMemIndexReg() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryRegImmImm:
       assert(getMemIndex() != nullptr && getMemOffset() != nullptr);
-      OS << "Mem: #" << getMemBase() << "+" << *getMemIndex() << "+"
-         << *getMemOffset() << "\n";
+      OS << "Mem: #" << getMemBase() << "+";
+      MAI.printExpr(OS, *getMemIndex());
+      OS << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryZeroRegImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: 0+#" << getMemIndexReg() << "+" << *getMemOffset() << "\n";
+      OS << "Mem: 0+#" << getMemIndexReg() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryZeroImmImm:
       assert(getMemIndex() != nullptr && getMemOffset() != nullptr);
-      OS << "Mem: 0+" << *getMemIndex() << "+" << *getMemOffset() << "\n";
+      OS << "Mem: 0+";
+      MAI.printExpr(OS, *getMemIndex());
+      OS << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryRegImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: #" << getMemBase() << "+" << *getMemOffset() << "\n";
+      OS << "Mem: #" << getMemBase() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryZeroImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: 0+" << *getMemOffset() << "\n";
+      OS << "Mem: 0+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_CCOp:
       OS << "CCOp: " << getCCVal() << "\n";
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
index 9a23a0e09f164..e09a916d48c90 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
@@ -11,7 +11,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -59,7 +58,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
     return 1;
   case FK_Data_2:
     return 2;
-    return 4;
   case FK_Data_4:
   case VE::fixup_ve_reflong:
   case VE::fixup_ve_srel32:
@@ -79,7 +77,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   case VE::fixup_ve_tpoff_lo32:
     return 4;
   case FK_Data_8:
-  case FK_PCRel_8:
     return 8;
   }
 }
@@ -96,22 +93,14 @@ class VEAsmBackend : public MCAsmBackend {
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo Infos[VE::NumTargetFixupKinds] = {
         // name, offset, bits, flags
-        {"fixup_ve_reflong", 0, 32, 0},
-        {"fixup_ve_srel32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-        {"fixup_ve_hi32", 0, 32, 0},
-        {"fixup_ve_lo32", 0, 32, 0},
-        {"fixup_ve_pc_hi32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-        {"fixup_ve_pc_lo32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-        {"fixup_ve_got_hi32", 0, 32, 0},
-        {"fixup_ve_got_lo32", 0, 32, 0},
-        {"fixup_ve_gotoff_hi32", 0, 32, 0},
-        {"fixup_ve_gotoff_lo32", 0, 32, 0},
-        {"fixup_ve_plt_hi32", 0, 32, 0},
-        {"fixup_ve_plt_lo32", 0, 32, 0},
-        {"fixup_ve_tls_gd_hi32", 0, 32, 0},
-        {"fixup_ve_tls_gd_lo32", 0, 32, 0},
-        {"fixup_ve_tpoff_hi32", 0, 32, 0},
-        {"fixup_ve_tpoff_lo32", 0, 32, 0},
+        {"fixup_ve_reflong", 0, 32, 0},     {"fixup_ve_srel32", 0, 32, 0},
+        {"fixup_ve_hi32", 0, 32, 0},        {"fixup_ve_lo32", 0, 32, 0},
+        {"fixup_ve_pc_hi32", 0, 32, 0},     {"fixup_ve_pc_lo32", 0, 32, 0},
+        {"fixup_ve_got_hi32", 0, 32, 0},    {"fixup_ve_got_lo32", 0, 32, 0},
+        {"fixup_ve_gotoff_hi32", 0, 32, 0}, {"fixup_ve_gotoff_lo32", 0, 32, 0},
+        {"fixup_ve_plt_hi32", 0, 32, 0},    {"fixup_ve_plt_lo32", 0, 32, 0},
+        {"fixup_ve_tls_gd_hi32", 0, 32, 0}, {"fixup_ve_tls_gd_lo32", 0, 32, 0},
+        {"fixup_ve_tpoff_hi32", 0, 32, 0},  {"fixup_ve_tpoff_lo32", 0, 32, 0},
     };
 
     if (Kind < FirstTargetFixupKind)
@@ -122,20 +111,11 @@ class VEAsmBackend : public MCAsmBackend {
     return Infos[Kind - FirstTargetFixupKind];
   }
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override {
-    switch ((VE::Fixups)Fixup.getKind()) {
-    default:
-      return false;
-    case VE::fixup_ve_tls_gd_hi32:
-    case VE::fixup_ve_tls_gd_lo32:
-    case VE::fixup_ve_tpoff_hi32:
-    case VE::fixup_ve_tpoff_lo32:
-      return true;
-    }
-  }
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &,
+                  MutableArrayRef<char>, uint64_t Value,
+                  bool IsResolved) override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override {
     // Not implemented yet.  For example, if we have a branch with
     // lager than SIMM32 immediate value, we want to relaxation such
@@ -143,12 +123,6 @@ class VEAsmBackend : public MCAsmBackend {
     return false;
   }
 
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override {
-    // Aurora VE doesn't support relaxInstruction yet.
-    llvm_unreachable("relaxInstruction() should not be called");
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override {
     if ((Count % 8) != 0)
@@ -169,31 +143,6 @@ class ELFVEAsmBackend : public VEAsmBackend {
   ELFVEAsmBackend(const Target &T, Triple::OSType OSType)
       : VEAsmBackend(T), OSType(OSType) {}
 
-  void applyFixup(const MCFragment &, const MCFixup &Fixup,
-                  const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t Value, bool IsResolved) override {
-    Value = adjustFixupValue(Fixup.getKind(), Value);
-    if (!Value)
-      return; // Doesn't change encoding.
-
-    MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
-
-    // Shift the value into position.
-    Value <<= Info.TargetOffset;
-
-    unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
-    unsigned Offset = Fixup.getOffset();
-    assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
-    // For each byte of the fragment that the fixup touches, mask in the bits
-    // from the fixup value. The Value has been "split up" into the
-    // appropriate bitfields above.
-    for (unsigned i = 0; i != NumBytes; ++i) {
-      unsigned Idx =
-          Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
-      Data[Offset + Idx] |= static_cast<uint8_t>((Value >> (i * 8)) & 0xff);
-    }
-  }
-
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override {
     uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(OSType);
@@ -202,6 +151,39 @@ class ELFVEAsmBackend : public VEAsmBackend {
 };
 } // end anonymous namespace
 
+void VEAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                              const MCValue &Target, MutableArrayRef<char> Data,
+                              uint64_t Value, bool IsResolved) {
+  switch (Fixup.getTargetKind()) {
+  case VE::fixup_ve_tls_gd_hi32:
+  case VE::fixup_ve_tls_gd_lo32:
+  case VE::fixup_ve_tpoff_hi32:
+  case VE::fixup_ve_tpoff_lo32:
+    IsResolved = false;
+    break;
+  }
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
+  Value = adjustFixupValue(Fixup.getKind(), Value);
+  if (!Value)
+    return; // Doesn't change encoding.
+
+  MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
+
+  // Shift the value into position.
+  Value <<= Info.TargetOffset;
+
+  unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
+  unsigned Offset = Fixup.getOffset();
+  assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
+  // For each byte of the fragment that the fixup touches, mask in the bits
+  // from the fixup value. The Value has been "split up" into the
+  // appropriate bitfields above.
+  for (unsigned i = 0; i != NumBytes; ++i) {
+    unsigned Idx = Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
+    Data[Offset + Idx] |= static_cast<uint8_t>((Value >> (i * 8)) & 0xff);
+  }
+}
+
 MCAsmBackend *llvm::createVEAsmBackend(const Target &T,
                                        const MCSubtargetInfo &STI,
                                        const MCRegisterInfo &MRI,
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
index 6c856a2a08f27..1597e7d080f03 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
@@ -61,7 +61,6 @@ unsigned VEELFObjectWriter::getRelocType(const MCFixup &Fixup,
       reportError(Fixup.getLoc(), "Unsupported pc-relative fixup kind");
       return ELF::R_VE_NONE;
     case FK_Data_1:
-    case FK_PCRel_1:
       reportError(Fixup.getLoc(),
                   "1-byte pc-relative data relocation is not supported");
       return ELF::R_VE_NONE;
@@ -72,7 +71,6 @@ unsigned VEELFObjectWriter::getRelocType(const MCFixup &Fixup,
     case FK_Data_4:
       return ELF::R_VE_SREL32;
     case FK_Data_8:
-    case FK_PCRel_8:
       reportError(Fixup.getLoc(),
                   "8-byte pc-relative data relocation is not supported");
       return ELF::R_VE_NONE;
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
index 9b114d6e11b51..711937c488275 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
@@ -76,6 +76,18 @@ class VEMCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case VE::fixup_ve_srel32:
+  case VE::fixup_ve_pc_hi32:
+  case VE::fixup_ve_pc_lo32:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void VEMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                         SmallVectorImpl<char> &CB,
                                         SmallVectorImpl<MCFixup> &Fixups,
@@ -100,7 +112,7 @@ unsigned VEMCCodeEmitter::getMachineOpValue(const MCInst &MI,
   const MCExpr *Expr = MO.getExpr();
   if (const auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
     auto Kind = VE::getFixupKind(SExpr->getSpecifier());
-    Fixups.push_back(MCFixup::create(0, Expr, Kind));
+    addFixup(Fixups, 0, Expr, Kind);
     return 0;
   }
 
@@ -120,8 +132,7 @@ VEMCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(
-      MCFixup::create(0, MO.getExpr(), (MCFixupKind)VE::fixup_ve_srel32));
+  addFixup(Fixups, 0, MO.getExpr(), VE::fixup_ve_srel32);
   return 0;
 }
 
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 5909d4bba1400..9e8f400256198 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -738,18 +738,16 @@ SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // necessary since all emitted instructions must be stuck together in order
   // to pass the live physical registers.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
   // Build the operands for the call instruction itself.
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(Chain);
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const VERegisterInfo *TRI = Subtarget->getRegisterInfo();
@@ -1741,9 +1739,6 @@ static SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (TLI.verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   SDValue FrameAddr = lowerFRAMEADDR(Op, DAG, TLI, Subtarget);
 
   SDLoc DL(Op);
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 5bcda8a411270..19c9e9cf4173f 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -180,7 +180,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
     }
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Token:
       OS << "Tok:" << Tok.Tok;
@@ -669,6 +669,10 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       if (parseFunctionTableOperand(&FunctionTable))
         return true;
       ExpectFuncType = true;
+    } else if (Name == "ref.test") {
+      // When we get support for wasm-gc types, this should become
+      // ExpectRefType.
+      ExpectFuncType = true;
     }
 
     // Returns true if the next tokens are a catch clause
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 7bc672c069476..837fd8eeac582 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -16,10 +16,10 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCWasmObjectWriter.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -78,10 +78,14 @@ bool WebAssemblyAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-void WebAssemblyAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void WebAssemblyAsmBackend::applyFixup(const MCFragment &F,
+                                       const MCFixup &Fixup,
                                        const MCValue &Target,
                                        MutableArrayRef<char> Data,
-                                       uint64_t Value, bool) {
+                                       uint64_t Value, bool IsResolved) {
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   assert(Info.Flags == 0 && "WebAssembly does not use MCFixupKindInfo flags");
 
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
index 3987b086195a4..a783e5fd8521a 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
@@ -55,14 +55,7 @@ WebAssemblyMCAsmInfo::WebAssemblyMCAsmInfo(const Triple &T,
   LCOMMDirectiveAlignmentType = LCOMM::Log2Alignment;
 
   SupportsDebugInformation = true;
-
-  // When compilation is done on a cpp file by clang, the exception model info
-  // is stored in LangOptions, which is later used to set the info in
-  // TargetOptions and then MCAsmInfo in CodeGenTargetMachine::initAsmInfo().
-  // But this process does not happen when compiling bitcode directly with
-  // clang, so we make sure this info is set correctly.
-  if (WebAssembly::WasmEnableEH || WebAssembly::WasmEnableSjLj)
-    ExceptionsType = ExceptionHandling::Wasm;
+  ExceptionsType = ExceptionHandling::None;
 
   initializeAtSpecifiers(atSpecifiers);
 }
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index 807fd42b8cf6e..cbaf10fc18b88 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -156,10 +156,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
         const MCOperandInfo &Info = Desc.operands()[I];
         switch (Info.OperandType) {
         case WebAssembly::OPERAND_I32IMM:
-          FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i32);
+          FixupKind = WebAssembly::fixup_sleb128_i32;
           break;
         case WebAssembly::OPERAND_I64IMM:
-          FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i64);
+          FixupKind = WebAssembly::fixup_sleb128_i64;
           PaddedSize = 10;
           break;
         case WebAssembly::OPERAND_FUNCTION32:
@@ -169,10 +169,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
         case WebAssembly::OPERAND_TYPEINDEX:
         case WebAssembly::OPERAND_GLOBAL:
         case WebAssembly::OPERAND_TAG:
-          FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32);
+          FixupKind = WebAssembly::fixup_uleb128_i32;
           break;
         case WebAssembly::OPERAND_OFFSET64:
-          FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i64);
+          FixupKind = WebAssembly::fixup_uleb128_i64;
           PaddedSize = 10;
           break;
         default:
@@ -181,10 +181,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
       } else {
         // Variadic expr operands are try_table's catch/catch_ref clauses' tags.
         assert(Opcode == WebAssembly::TRY_TABLE_S);
-        FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32);
+        FixupKind = WebAssembly::fixup_uleb128_i32;
       }
-      Fixups.push_back(MCFixup::create(OS.tell() - Start, MO.getExpr(),
-                                       FixupKind, MI.getLoc()));
+      Fixups.push_back(
+          MCFixup::create(OS.tell() - Start, MO.getExpr(), FixupKind));
       ++MCNumFixups;
       encodeULEB128(0, OS, PaddedSize);
     } else {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
index 6c0031f429c6d..8019ae165ec8c 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -36,35 +36,6 @@ using namespace llvm;
 #define GET_REGINFO_MC_DESC
 #include "WebAssemblyGenRegisterInfo.inc"
 
-// Exception handling & setjmp-longjmp handling related options.
-
-// Emscripten's asm.js-style exception handling
-cl::opt<bool> WebAssembly::WasmEnableEmEH(
-    "enable-emscripten-cxx-exceptions",
-    cl::desc("WebAssembly Emscripten-style exception handling"),
-    cl::init(false));
-// Emscripten's asm.js-style setjmp/longjmp handling
-cl::opt<bool> WebAssembly::WasmEnableEmSjLj(
-    "enable-emscripten-sjlj",
-    cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"),
-    cl::init(false));
-// Exception handling using wasm EH instructions
-cl::opt<bool>
-    WebAssembly::WasmEnableEH("wasm-enable-eh",
-                              cl::desc("WebAssembly exception handling"));
-// setjmp/longjmp handling using wasm EH instrutions
-cl::opt<bool> WebAssembly::WasmEnableSjLj(
-    "wasm-enable-sjlj", cl::desc("WebAssembly setjmp/longjmp handling"));
-// If true, use the legacy Wasm EH proposal:
-// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/legacy/Exceptions.md
-// And if false, use the standardized Wasm EH proposal:
-// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/Exceptions.md
-// Currently set to true by default because not all major web browsers turn on
-// the new standard proposal by default, but will later change to false.
-cl::opt<bool> WebAssembly::WasmUseLegacyEH(
-    "wasm-use-legacy-eh", cl::desc("WebAssembly exception handling (legacy)"),
-    cl::init(true));
-
 static MCAsmInfo *createMCAsmInfo(const MCRegisterInfo & /*MRI*/,
                                   const Triple &TT,
                                   const MCTargetOptions &Options) {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index d6a2fe4c7839d..fe9a4bada2430 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -39,13 +39,6 @@ createWebAssemblyWasmObjectWriter(bool Is64Bit, bool IsEmscripten);
 
 namespace WebAssembly {
 
-// Exception handling / setjmp-longjmp handling command-line options
-extern cl::opt<bool> WasmEnableEmEH;   // asm.js-style EH
-extern cl::opt<bool> WasmEnableEmSjLj; // asm.js-style SjLJ
-extern cl::opt<bool> WasmEnableEH;     // EH using Wasm EH instructions
-extern cl::opt<bool> WasmEnableSjLj;   // SjLj using Wasm EH instructions
-extern cl::opt<bool> WasmUseLegacyEH;  // Legacy Wasm EH
-
 enum OperandType {
   /// Basic block label in a branch construct.
   OPERAND_BASIC_BLOCK = MCOI::OPERAND_FIRST_TARGET,
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
index 1a9aa1e14f716..c18de5fc1939e 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSectionWasm.h"
@@ -112,7 +113,9 @@ void WebAssemblyTargetAsmStreamer::emitExportName(const MCSymbolWasm *Sym,
 }
 
 void WebAssemblyTargetAsmStreamer::emitIndIdx(const MCExpr *Value) {
-  OS << "\t.indidx  \t" << *Value << '\n';
+  OS << "\t.indidx\t";
+  getContext().getAsmInfo()->printExpr(OS, *Value);
+  OS << '\n';
 }
 
 void WebAssemblyTargetWasmStreamer::emitLocal(ArrayRef<wasm::ValType> Types) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
index 44a19e4baaf62..1bf070e9ec9c8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
@@ -24,6 +24,7 @@
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblyRegisterInfo.h"
 #include "WebAssemblyRuntimeLibcallSignatures.h"
+#include "WebAssemblyTargetMachine.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallSet.h"
@@ -155,9 +156,11 @@ static std::string getEmscriptenInvokeSymbolName(wasm::WasmSignature *Sig) {
 //===----------------------------------------------------------------------===//
 
 MCSymbolWasm *WebAssemblyAsmPrinter::getMCSymbolForFunction(
-    const Function *F, bool EnableEmEH, wasm::WasmSignature *Sig,
-    bool &InvokeDetected) {
+    const Function *F, wasm::WasmSignature *Sig, bool &InvokeDetected) {
   MCSymbolWasm *WasmSym = nullptr;
+
+  const bool EnableEmEH =
+      WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj;
   if (EnableEmEH && isEmscriptenInvokeName(F->getName())) {
     assert(Sig);
     InvokeDetected = true;
@@ -344,9 +347,7 @@ void WebAssemblyAsmPrinter::emitDecls(const Module &M) {
     // will discard it later if it turns out not to be necessary.
     auto Signature = signatureFromMVTs(OutContext, Results, Params);
     bool InvokeDetected = false;
-    auto *Sym = getMCSymbolForFunction(
-        &F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj,
-        Signature, InvokeDetected);
+    auto *Sym = getMCSymbolForFunction(&F, Signature, InvokeDetected);
 
     // Multiple functions can be mapped to the same invoke symbol. For
     // example, two IR functions '__invoke_void_i8*' and '__invoke_void_i32'
@@ -404,7 +405,7 @@ void WebAssemblyAsmPrinter::emitEndOfAsmFile(Module &M) {
     if (!F.isIntrinsic() && F.hasAddressTaken()) {
       MCSymbolWasm *FunctionTable =
           WebAssembly::getOrCreateFunctionTableSymbol(OutContext, Subtarget);
-      OutStreamer->emitSymbolAttribute(FunctionTable, MCSA_NoDeadStrip);    
+      OutStreamer->emitSymbolAttribute(FunctionTable, MCSA_NoDeadStrip);
       break;
     }
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
index 46063bbe0fba1..aef562e1dd684 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
@@ -73,7 +73,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter {
   MVT getRegType(unsigned RegNo) const;
   std::string regToString(const MachineOperand &MO);
   WebAssemblyTargetStreamer *getTargetStreamer();
-  MCSymbolWasm *getMCSymbolForFunction(const Function *F, bool EnableEmEH,
+  MCSymbolWasm *getMCSymbolForFunction(const Function *F,
                                        wasm::WasmSignature *Sig,
                                        bool &InvokeDetected);
   MCSymbol *getOrCreateWasmSymbol(StringRef Name);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 640be5fe8e8c9..acb889f00a48c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -21,13 +21,13 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "Utils/WebAssemblyTypeUtilities.h"
 #include "WebAssembly.h"
 #include "WebAssemblyExceptionInfo.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblySortRegion.h"
 #include "WebAssemblySubtarget.h"
+#include "WebAssemblyTargetMachine.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/BinaryFormat/Wasm.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index 48c0b7e50f080..ac819cf5c1801 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -343,17 +343,29 @@ bool WebAssemblyDAGToDAGISel::SelectAddrAddOperands(MVT OffsetType, SDValue N,
   if (N.getOpcode() == ISD::ADD && !N.getNode()->getFlags().hasNoUnsignedWrap())
     return false;
 
-  // Folds constants in an add into the offset.
   for (size_t i = 0; i < 2; ++i) {
     SDValue Op = N.getOperand(i);
     SDValue OtherOp = N.getOperand(i == 0 ? 1 : 0);
 
+    // Folds constants in an add into the offset.
     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
       Offset =
           CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), OffsetType);
       Addr = OtherOp;
       return true;
     }
+
+    // Fold target global addresses into the offset.
+    if (!TM.isPositionIndependent()) {
+      if (Op.getOpcode() == WebAssemblyISD::Wrapper)
+        Op = Op.getOperand(0);
+
+      if (Op.getOpcode() == ISD::TargetGlobalAddress) {
+        Addr = OtherOp;
+        Offset = Op;
+        return true;
+      }
+    }
   }
   return false;
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index ec77154d17caa..bf2e04caa0a61 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -1909,9 +1910,6 @@ SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
     return SDValue();
   }
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   unsigned Depth = Op.getConstantOperandVal(0);
   MakeLibCallOptions CallOptions;
   return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
@@ -3214,20 +3212,23 @@ static SDValue performTruncateCombine(SDNode *N,
 
 static SDValue performBitcastCombine(SDNode *N,
                                      TargetLowering::DAGCombinerInfo &DCI) {
+  using namespace llvm::SDPatternMatch;
   auto &DAG = DCI.DAG;
   SDLoc DL(N);
   SDValue Src = N->getOperand(0);
   EVT VT = N->getValueType(0);
   EVT SrcVT = Src.getValueType();
 
-  // bitcast <N x i1> to iN
+  if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&
+        SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1))
+    return SDValue();
+
+  unsigned NumElts = SrcVT.getVectorNumElements();
+  EVT Width = MVT::getIntegerVT(128 / NumElts);
+
+  // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)
   //   ==> bitmask
-  if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
-      SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1) {
-    unsigned NumElts = SrcVT.getVectorNumElements();
-    if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
-      return SDValue();
-    EVT Width = MVT::getIntegerVT(128 / NumElts);
+  if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
     return DAG.getZExtOrTrunc(
         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                     {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),
@@ -3236,6 +3237,102 @@ static SDValue performBitcastCombine(SDNode *N,
         DL, VT);
   }
 
+  // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)
+  if (NumElts == 32 || NumElts == 64) {
+    // Strategy: We will setcc them seperately in v16i8 -> v16i1
+    // Bitcast them to i16, extend them to either i32 or i64.
+    // Add them together, shifting left 1 by 1.
+    SDValue Concat, SetCCVector;
+    ISD::CondCode SetCond;
+
+    if (!sd_match(N, m_BitCast(m_c_SetCC(m_Value(Concat), m_Value(SetCCVector),
+                                         m_CondCode(SetCond)))))
+      return SDValue();
+    if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
+      return SDValue();
+
+    uint64_t ElementWidth =
+        SetCCVector.getValueType().getVectorElementType().getFixedSizeInBits();
+
+    SmallVector<SDValue> VectorsToShuffle;
+    for (size_t I = 0; I < Concat->ops().size(); I++) {
+      VectorsToShuffle.push_back(DAG.getBitcast(
+          MVT::i16,
+          DAG.getSetCC(DL, MVT::v16i1, Concat->ops()[I],
+                       extractSubVector(SetCCVector, I * (128 / ElementWidth),
+                                        DAG, DL, 128),
+                       SetCond)));
+    }
+
+    MVT ReturnType = VectorsToShuffle.size() == 2 ? MVT::i32 : MVT::i64;
+    SDValue ReturningInteger = DAG.getConstant(0, DL, ReturnType);
+
+    for (SDValue V : VectorsToShuffle) {
+      ReturningInteger = DAG.getNode(
+          ISD::SHL, DL, ReturnType,
+          {DAG.getShiftAmountConstant(16, ReturnType, DL), ReturningInteger});
+
+      SDValue ExtendedV = DAG.getZExtOrTrunc(V, DL, ReturnType);
+      ReturningInteger =
+          DAG.getNode(ISD::ADD, DL, ReturnType, {ReturningInteger, ExtendedV});
+    }
+
+    return ReturningInteger;
+  }
+
+  return SDValue();
+}
+
+static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG) {
+  // any_true (setcc <X>, 0, eq) => (not (all_true X))
+  // all_true (setcc <X>, 0, eq) => (not (any_true X))
+  // any_true (setcc <X>, 0, ne) => (any_true X)
+  // all_true (setcc <X>, 0, ne) => (all_true X)
+  assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
+  using namespace llvm::SDPatternMatch;
+
+  SDValue LHS;
+  if (!sd_match(N->getOperand(1),
+                m_c_SetCC(m_Value(LHS), m_Zero(), m_CondCode())))
+    return SDValue();
+  EVT LT = LHS.getValueType();
+  if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
+    return SDValue();
+
+  auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,
+                                 ISD::CondCode SetType,
+                                 Intrinsic::WASMIntrinsics InPost) {
+    if (N->getConstantOperandVal(0) != InPre)
+      return SDValue();
+
+    SDValue LHS;
+    if (!sd_match(N->getOperand(1), m_c_SetCC(m_Value(LHS), m_Zero(),
+                                              m_SpecificCondCode(SetType))))
+      return SDValue();
+
+    SDLoc DL(N);
+    SDValue Ret = DAG.getZExtOrTrunc(
+        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
+                    {DAG.getConstant(InPost, DL, MVT::i32), LHS}),
+        DL, MVT::i1);
+    if (SetType == ISD::SETEQ)
+      Ret = DAG.getNOT(DL, Ret, MVT::i1);
+    return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
+  };
+
+  if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,
+                                       Intrinsic::wasm_alltrue))
+    return AnyTrueEQ;
+  if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,
+                                       Intrinsic::wasm_anytrue))
+    return AllTrueEQ;
+  if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,
+                                       Intrinsic::wasm_anytrue))
+    return AnyTrueNE;
+  if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,
+                                       Intrinsic::wasm_alltrue))
+    return AllTrueNE;
+
   return SDValue();
 }
 
@@ -3427,8 +3524,11 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
     return performVectorTruncZeroCombine(N, DCI);
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
-  case ISD::INTRINSIC_WO_CHAIN:
+  case ISD::INTRINSIC_WO_CHAIN: {
+    if (auto AnyAllCombine = performAnyAllCombine(N, DCI.DAG))
+      return AnyAllCombine;
     return performLowerPartialReduction(N, DCI.DAG);
+  }
   case ISD::MUL:
     return performMulCombine(N, DCI.DAG);
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
index 2654a09387fd4..40b87a084c687 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
@@ -36,6 +36,14 @@ multiclass REF_I<WebAssemblyRegClass rc, ValueType vt, string ht> {
         Requires<[HasReferenceTypes]>;
 }
 
+defm REF_TEST_FUNCREF :
+  I<(outs I32: $res),
+    (ins TypeIndex:$type, FUNCREF: $ref),
+    (outs),
+    (ins TypeIndex:$type),
+    [],
+    "ref.test\t$type, $ref", "ref.test $type", 0xfb14>;
+
 defm "" : REF_I<FUNCREF, funcref, "func">;
 defm "" : REF_I<EXTERNREF, externref, "extern">;
 defm "" : REF_I<EXNREF, exnref, "exn">;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
index 254ad5c4f2beb..8ac32f939c5f2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
@@ -11,9 +11,9 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "WebAssembly.h"
 #include "WebAssemblySubtarget.h"
+#include "WebAssemblyTargetMachine.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index d5240959e765e..cc36244e63ff5 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -77,9 +77,7 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
   auto Signature = signatureFromMVTs(Ctx, ResultMVTs, ParamMVTs);
 
   bool InvokeDetected = false;
-  auto *WasmSym = Printer.getMCSymbolForFunction(
-      F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj,
-      Signature, InvokeDetected);
+  auto *WasmSym = Printer.getMCSymbolForFunction(F, Signature, InvokeDetected);
   WasmSym->setSignature(Signature);
   WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
   return WasmSym;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
index 17889dacc868c..31a33c1e7365b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
@@ -64,8 +64,8 @@ def I32 : WebAssemblyRegClass<[i32], 32, (add FP32, SP32, I32_0)>;
 def I64 : WebAssemblyRegClass<[i64], 64, (add FP64, SP64, I64_0)>;
 def F32 : WebAssemblyRegClass<[f32], 32, (add F32_0)>;
 def F64 : WebAssemblyRegClass<[f64], 64, (add F64_0)>;
-def V128 : WebAssemblyRegClass<[v8f16, v4f32, v2f64, v2i64, v4i32, v16i8,
-                                v8i16],
+def V128 : WebAssemblyRegClass<[v2i64, v4i32, v16i8, v8i16,
+                                v8f16, v4f32, v2f64],
                                128, (add V128_0)>;
 def FUNCREF : WebAssemblyRegClass<[funcref], 0, (add FUNCREF_0)>;
 def EXTERNREF : WebAssemblyRegClass<[externref], 0, (add EXTERNREF_0)>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 378af22f4907c..ad47cb8ea2fe1 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -54,6 +54,35 @@ static cl::opt<bool> WasmDisableFixIrreducibleControlFlowPass(
              " irreducible control flow optimization pass"),
     cl::init(false));
 
+// Exception handling & setjmp-longjmp handling related options.
+
+// Emscripten's asm.js-style exception handling
+cl::opt<bool> WebAssembly::WasmEnableEmEH(
+    "enable-emscripten-cxx-exceptions",
+    cl::desc("WebAssembly Emscripten-style exception handling"),
+    cl::init(false));
+// Emscripten's asm.js-style setjmp/longjmp handling
+cl::opt<bool> WebAssembly::WasmEnableEmSjLj(
+    "enable-emscripten-sjlj",
+    cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"),
+    cl::init(false));
+// Exception handling using wasm EH instructions
+cl::opt<bool>
+    WebAssembly::WasmEnableEH("wasm-enable-eh",
+                              cl::desc("WebAssembly exception handling"));
+// setjmp/longjmp handling using wasm EH instrutions
+cl::opt<bool> WebAssembly::WasmEnableSjLj(
+    "wasm-enable-sjlj", cl::desc("WebAssembly setjmp/longjmp handling"));
+// If true, use the legacy Wasm EH proposal:
+// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/legacy/Exceptions.md
+// And if false, use the standardized Wasm EH proposal:
+// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/Exceptions.md
+// Currently set to true by default because not all major web browsers turn on
+// the new standard proposal by default, but will later change to false.
+cl::opt<bool> WebAssembly::WasmUseLegacyEH(
+    "wasm-use-legacy-eh", cl::desc("WebAssembly exception handling (legacy)"),
+    cl::init(true));
+
 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeWebAssemblyTarget() {
   // Register the target.
@@ -111,6 +140,57 @@ static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM,
   return *RM;
 }
 
+using WebAssembly::WasmEnableEH;
+using WebAssembly::WasmEnableEmEH;
+using WebAssembly::WasmEnableEmSjLj;
+using WebAssembly::WasmEnableSjLj;
+
+static void basicCheckForEHAndSjLj(TargetMachine *TM) {
+
+  // You can't enable two modes of EH at the same time
+  if (WasmEnableEmEH && WasmEnableEH)
+    report_fatal_error(
+        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-eh");
+  // You can't enable two modes of SjLj at the same time
+  if (WasmEnableEmSjLj && WasmEnableSjLj)
+    report_fatal_error(
+        "-enable-emscripten-sjlj not allowed with -wasm-enable-sjlj");
+  // You can't mix Emscripten EH with Wasm SjLj.
+  if (WasmEnableEmEH && WasmEnableSjLj)
+    report_fatal_error(
+        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-sjlj");
+
+  if (TM->Options.ExceptionModel == ExceptionHandling::None) {
+    // FIXME: These flags should be removed in favor of directly using the
+    // generically configured ExceptionsType
+    if (WebAssembly::WasmEnableEH || WebAssembly::WasmEnableSjLj)
+      TM->Options.ExceptionModel = ExceptionHandling::Wasm;
+  }
+
+  // Basic Correctness checking related to -exception-model
+  if (TM->Options.ExceptionModel != ExceptionHandling::None &&
+      TM->Options.ExceptionModel != ExceptionHandling::Wasm)
+    report_fatal_error("-exception-model should be either 'none' or 'wasm'");
+  if (WasmEnableEmEH && TM->Options.ExceptionModel == ExceptionHandling::Wasm)
+    report_fatal_error("-exception-model=wasm not allowed with "
+                       "-enable-emscripten-cxx-exceptions");
+  if (WasmEnableEH && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
+    report_fatal_error(
+        "-wasm-enable-eh only allowed with -exception-model=wasm");
+  if (WasmEnableSjLj && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
+    report_fatal_error(
+        "-wasm-enable-sjlj only allowed with -exception-model=wasm");
+  if ((!WasmEnableEH && !WasmEnableSjLj) &&
+      TM->Options.ExceptionModel == ExceptionHandling::Wasm)
+    report_fatal_error(
+        "-exception-model=wasm only allowed with at least one of "
+        "-wasm-enable-eh or -wasm-enable-sjlj");
+
+  // Currently it is allowed to mix Wasm EH with Emscripten SjLj as an interim
+  // measure, but some code will error out at compile time in this combination.
+  // See WebAssemblyLowerEmscriptenEHSjLj pass for details.
+}
+
 /// Create an WebAssembly architecture model.
 ///
 WebAssemblyTargetMachine::WebAssemblyTargetMachine(
@@ -149,7 +229,7 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine(
   this->Options.UniqueSectionNames = true;
 
   initAsmInfo();
-
+  basicCheckForEHAndSjLj(this);
   // Note that we don't use setRequiresStructuredCFG(true). It disables
   // optimizations than we're ok with, and want, such as critical edge
   // splitting and tail merging.
@@ -400,61 +480,6 @@ FunctionPass *WebAssemblyPassConfig::createTargetRegisterAllocator(bool) {
   return nullptr; // No reg alloc
 }
 
-using WebAssembly::WasmEnableEH;
-using WebAssembly::WasmEnableEmEH;
-using WebAssembly::WasmEnableEmSjLj;
-using WebAssembly::WasmEnableSjLj;
-
-static void basicCheckForEHAndSjLj(TargetMachine *TM) {
-
-  // You can't enable two modes of EH at the same time
-  if (WasmEnableEmEH && WasmEnableEH)
-    report_fatal_error(
-        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-eh");
-  // You can't enable two modes of SjLj at the same time
-  if (WasmEnableEmSjLj && WasmEnableSjLj)
-    report_fatal_error(
-        "-enable-emscripten-sjlj not allowed with -wasm-enable-sjlj");
-  // You can't mix Emscripten EH with Wasm SjLj.
-  if (WasmEnableEmEH && WasmEnableSjLj)
-    report_fatal_error(
-        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-sjlj");
-
-  // Here we make sure TargetOptions.ExceptionModel is the same as
-  // MCAsmInfo.ExceptionsType. Normally these have to be the same, because clang
-  // stores the exception model info in LangOptions, which is later transferred
-  // to TargetOptions and MCAsmInfo. But when clang compiles bitcode directly,
-  // clang's LangOptions is not used and thus the exception model info is not
-  // correctly transferred to TargetOptions and MCAsmInfo, so we make sure we
-  // have the correct exception model in WebAssemblyMCAsmInfo constructor. But
-  // in this case TargetOptions is still not updated, so we make sure they are
-  // the same.
-  TM->Options.ExceptionModel = TM->getMCAsmInfo()->getExceptionHandlingType();
-
-  // Basic Correctness checking related to -exception-model
-  if (TM->Options.ExceptionModel != ExceptionHandling::None &&
-      TM->Options.ExceptionModel != ExceptionHandling::Wasm)
-    report_fatal_error("-exception-model should be either 'none' or 'wasm'");
-  if (WasmEnableEmEH && TM->Options.ExceptionModel == ExceptionHandling::Wasm)
-    report_fatal_error("-exception-model=wasm not allowed with "
-                       "-enable-emscripten-cxx-exceptions");
-  if (WasmEnableEH && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
-    report_fatal_error(
-        "-wasm-enable-eh only allowed with -exception-model=wasm");
-  if (WasmEnableSjLj && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
-    report_fatal_error(
-        "-wasm-enable-sjlj only allowed with -exception-model=wasm");
-  if ((!WasmEnableEH && !WasmEnableSjLj) &&
-      TM->Options.ExceptionModel == ExceptionHandling::Wasm)
-    report_fatal_error(
-        "-exception-model=wasm only allowed with at least one of "
-        "-wasm-enable-eh or -wasm-enable-sjlj");
-
-  // Currently it is allowed to mix Wasm EH with Emscripten SjLj as an interim
-  // measure, but some code will error out at compile time in this combination.
-  // See WebAssemblyLowerEmscriptenEHSjLj pass for details.
-}
-
 //===----------------------------------------------------------------------===//
 // The following functions are called from lib/CodeGen/Passes.cpp to modify
 // the CodeGen pass sequence.
@@ -475,8 +500,6 @@ void WebAssemblyPassConfig::addIRPasses() {
   if (getOptLevel() != CodeGenOptLevel::None)
     addPass(createWebAssemblyOptimizeReturned());
 
-  basicCheckForEHAndSjLj(TM);
-
   // If exception handling is not enabled and setjmp/longjmp handling is
   // enabled, we lower invokes into calls and delete unreachable landingpad
   // blocks. Lowering invokes when there is no EH support is done in
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
index 1d8197d02588a..6f97f1302189a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
@@ -21,6 +21,15 @@
 
 namespace llvm {
 
+namespace WebAssembly {
+// Exception handling / setjmp-longjmp handling command-line options
+extern cl::opt<bool> WasmEnableEmEH;   // asm.js-style EH
+extern cl::opt<bool> WasmEnableEmSjLj; // asm.js-style SjLJ
+extern cl::opt<bool> WasmEnableEH;     // EH using Wasm EH instructions
+extern cl::opt<bool> WasmEnableSjLj;   // SjLj using Wasm EH instructions
+extern cl::opt<bool> WasmUseLegacyEH;  // Legacy Wasm EH
+} // namespace WebAssembly
+
 class WebAssemblyTargetMachine final : public CodeGenTargetMachineImpl {
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   mutable StringMap<std::unique_ptr<WebAssemblySubtarget>> SubtargetMap;
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index f26b46f0d56c1..b642c1cfe383b 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -9,11 +9,13 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86EncodingOptimization.h"
 #include "MCTargetDesc/X86IntelInstPrinter.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCExpr.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "MCTargetDesc/X86TargetStreamer.h"
 #include "TargetInfo/X86TargetInfo.h"
 #include "X86Operand.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -2118,7 +2120,7 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) {
         if (IDVal == "f" || IDVal == "b") {
           MCSymbol *Sym =
               getContext().getDirectionalLocalSymbol(IntVal, IDVal == "b");
-          auto Variant = X86MCExpr::VK_None;
+          auto Variant = X86::S_None;
           const MCExpr *Val =
               MCSymbolRefExpr::create(Sym, Variant, getContext());
           if (IDVal == "b" && Sym->isUndefined())
@@ -2265,7 +2267,7 @@ bool X86AsmParser::ParseIntelInlineAsmIdentifier(
     return false;
   // Create the symbol reference.
   MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-  auto Variant = X86MCExpr::VK_None;
+  auto Variant = X86::S_None;
   Val = MCSymbolRefExpr::create(Sym, Variant, getParser().getContext());
   return false;
 }
diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h
index c143ebd6eae50..89ac53e0ecac9 100644
--- a/llvm/lib/Target/X86/AsmParser/X86Operand.h
+++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h
@@ -107,8 +107,7 @@ struct X86Operand final : public MCParsedAsmOperand {
   /// getOffsetOfLoc - Get the location of the offset operator.
   SMLoc getOffsetOfLoc() const override { return OffsetOfLoc; }
 
-  void print(raw_ostream &OS) const override {
-
+  void print(raw_ostream &OS, const MCAsmInfo &) const override {
     auto PrintImmValue = [&](const MCExpr *Val, const char *VName) {
       if (Val->getKind() == MCExpr::Constant) {
         if (auto Imm = cast<MCConstantExpr>(Val)->getValue())
diff --git a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
index 4a1f9a77f6a51..4927b453458ef 100644
--- a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -77,6 +77,7 @@
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "TargetInfo/X86TargetInfo.h"
 #include "X86DisassemblerDecoder.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCExpr.h"
diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index 9b0dd0562cde3..3090ad313b90d 100644
--- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -95,8 +95,6 @@ class X86InstructionSelector : public InstructionSelector {
                  MachineFunction &MF) const;
   bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
                   MachineFunction &MF) const;
-  bool selectFAbs(MachineInstr &I, MachineRegisterInfo &MRI,
-                  MachineFunction &MF) const;
   bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
                      MachineFunction &MF) const;
   bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI) const;
@@ -393,8 +391,6 @@ bool X86InstructionSelector::select(MachineInstr &I) {
   switch (I.getOpcode()) {
   default:
     return false;
-  case TargetOpcode::G_FABS:
-    return selectFAbs(I, MRI, MF);
   case TargetOpcode::G_STORE:
   case TargetOpcode::G_LOAD:
     return selectLoadStoreOp(I, MRI, MF);
@@ -1054,35 +1050,6 @@ bool X86InstructionSelector::selectCmp(MachineInstr &I,
   I.eraseFromParent();
   return true;
 }
-bool X86InstructionSelector::selectFAbs(MachineInstr &I,
-                                        MachineRegisterInfo &MRI,
-                                        MachineFunction &MF) const {
-  assert((I.getOpcode() == TargetOpcode::G_FABS) && "unexpected instruction");
-  Register SrcReg = I.getOperand(1).getReg();
-  Register DstReg = I.getOperand(0).getReg();
-  LLT Ty = MRI.getType(SrcReg);
-  unsigned OpAbs;
-  const TargetRegisterClass *DstRC;
-  switch (Ty.getSizeInBits()) {
-  default:
-    return false;
-  case 32:
-    OpAbs = X86::ABS_Fp32;
-    DstRC = &X86::FR32RegClass;
-    break;
-  case 64:
-    OpAbs = X86::ABS_Fp64;
-    DstRC = &X86::FR64RegClass;
-    break;
-  }
-  MRI.setRegClass(DstReg, DstRC);
-  MachineInstr &FAbsInst =
-      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpAbs), DstReg)
-           .addReg(SrcReg);
-  constrainSelectedInstRegOperands(FAbsInst, TII, TRI, RBI);
-  I.eraseFromParent();
-  return true;
-}
 
 bool X86InstructionSelector::selectFCmp(MachineInstr &I,
                                         MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index 0250ec66c0b99..8e304c07ed5cb 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -419,7 +419,8 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
       .legalFor(UseX87, {s80});
 
   getActionDefinitionsBuilder(G_FABS)
-      .legalFor(UseX87 && !HasSSE2 && !HasSSE1, {s64, s80})
+      .legalFor(UseX87, {s80})
+      .legalFor(UseX87 && !Is64Bit, {s64})
       .lower();
 
   // fp comparison
diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
index 74fe5fcaab6cc..b23d791501729 100644
--- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
@@ -341,6 +341,7 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
                     /* Predicate */ PMI_None, FpRegBank, FpRegBank};
     break;
   }
+  case TargetOpcode::G_FABS:
   case TargetOpcode::G_TRUNC:
   case TargetOpcode::G_ANYEXT: {
     auto &Op0 = MI.getOperand(0);
@@ -354,9 +355,9 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
         Ty0.getSizeInBits() == 128 &&
         (Ty1.getSizeInBits() == 32 || Ty1.getSizeInBits() == 64) &&
         Opc == TargetOpcode::G_ANYEXT;
-
-    getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ isFPTrunc || isFPAnyExt,
-                               OpRegBankIdx);
+    bool isFAbs = (Opc == TargetOpcode::G_FABS);
+    getInstrPartialMappingIdxs(
+        MI, MRI, /* isFP= */ isFPTrunc || isFPAnyExt || isFAbs, OpRegBankIdx);
     break;
   }
   case TargetOpcode::G_LOAD: {
diff --git a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
index a04689f3c7569..817e88d8a0bc0 100644
--- a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
+++ b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
@@ -14,6 +14,7 @@
 #include "X86CustomBehaviour.h"
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "TargetInfo/X86TargetInfo.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/MC/TargetRegistry.h"
 
 namespace llvm {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 2d0f021eac5a8..ff2df3d5b192a 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -9,7 +9,7 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86EncodingOptimization.h"
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -21,7 +21,6 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
@@ -169,13 +168,13 @@ class X86AsmBackend : public MCAsmBackend {
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  bool shouldForceRelocation(const MCFixup &, const MCValue &) override;
-
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
 
-  bool mayNeedRelaxation(const MCInst &Inst,
+  bool mayNeedRelaxation(unsigned Opcode, ArrayRef<MCOperand> Operands,
                          const MCSubtargetInfo &STI) const override;
 
   bool fixupNeedsRelaxationAdvanced(const MCFixup &, const MCValue &, uint64_t,
@@ -359,7 +358,7 @@ static bool hasVariantSymbol(const MCInst &MI) {
       continue;
     const MCExpr &Expr = *Operand.getExpr();
     if (Expr.getKind() == MCExpr::SymbolRef &&
-        getSpecifier(cast<MCSymbolRefExpr>(&Expr)) != X86MCExpr::VK_None)
+        cast<MCSymbolRefExpr>(&Expr)->getSpecifier())
       return true;
   }
   return false;
@@ -423,14 +422,10 @@ static size_t getSizeForInstFragment(const MCFragment *F) {
   if (!F || !F->hasInstructions())
     return 0;
   // MCEncodedFragmentWithContents being templated makes this tricky.
-  switch (F->getKind()) {
-  default:
+  if (auto *DF = dyn_cast<MCEncodedFragment>(F))
+    return DF->getContents().size();
+  else
     llvm_unreachable("Unknown fragment with instructions!");
-  case MCFragment::FT_Data:
-    return cast<MCDataFragment>(*F).getContents().size();
-  case MCFragment::FT_Relaxable:
-    return cast<MCRelaxableFragment>(*F).getContents().size();
-  }
 }
 
 /// Return true if we can insert NOP or prefixes automatically before the
@@ -623,24 +618,24 @@ std::optional<MCFixupKind> X86AsmBackend::getFixupKind(StringRef Name) const {
 MCFixupKindInfo X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[X86::NumTargetFixupKinds] = {
       // clang-format off
-      {"reloc_riprel_4byte", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_movq_load", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_movq_load_rex2", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax_rex", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax_rex2", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax_evex", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"reloc_riprel_4byte", 0, 32, 0},
+      {"reloc_riprel_4byte_movq_load", 0, 32, 0},
+      {"reloc_riprel_4byte_movq_load_rex2", 0, 32, 0},
+      {"reloc_riprel_4byte_relax", 0, 32, 0},
+      {"reloc_riprel_4byte_relax_rex", 0, 32, 0},
+      {"reloc_riprel_4byte_relax_rex2", 0, 32, 0},
+      {"reloc_riprel_4byte_relax_evex", 0, 32, 0},
       {"reloc_signed_4byte", 0, 32, 0},
       {"reloc_signed_4byte_relax", 0, 32, 0},
       {"reloc_global_offset_table", 0, 32, 0},
-      {"reloc_branch_4byte_pcrel", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"reloc_branch_4byte_pcrel", 0, 32, 0},
       // clang-format on
   };
 
   // Fixup kinds from .reloc directive are like R_386_NONE/R_X86_64_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -657,15 +652,12 @@ static unsigned getFixupKindSize(unsigned Kind) {
     llvm_unreachable("invalid fixup kind!");
   case FK_NONE:
     return 0;
-  case FK_PCRel_1:
   case FK_SecRel_1:
   case FK_Data_1:
     return 1;
-  case FK_PCRel_2:
   case FK_SecRel_2:
   case FK_Data_2:
     return 2;
-  case FK_PCRel_4:
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_relax:
   case X86::reloc_riprel_4byte_relax_rex:
@@ -680,23 +672,53 @@ static unsigned getFixupKindSize(unsigned Kind) {
   case FK_SecRel_4:
   case FK_Data_4:
     return 4;
-  case FK_PCRel_8:
   case FK_SecRel_8:
   case FK_Data_8:
     return 8;
   }
 }
 
-// Force relocation when there is a specifier. This might be too conservative -
-// GAS doesn't emit a relocation for call local@plt; local:.
-bool X86AsmBackend::shouldForceRelocation(const MCFixup &,
-                                          const MCValue &Target) {
-  return Target.getSpecifier();
+constexpr char GotSymName[] = "_GLOBAL_OFFSET_TABLE_";
+
+// Adjust PC-relative fixup offsets, which are calculated from the start of the
+// next instruction.
+std::optional<bool> X86AsmBackend::evaluateFixup(const MCFragment &,
+                                                 MCFixup &Fixup,
+                                                 MCValue &Target, uint64_t &) {
+  if (Fixup.isPCRel()) {
+    switch (Fixup.getTargetKind()) {
+    case FK_Data_1:
+      Target.setConstant(Target.getConstant() - 1);
+      break;
+    case FK_Data_2:
+      Target.setConstant(Target.getConstant() - 2);
+      break;
+    default: {
+      Target.setConstant(Target.getConstant() - 4);
+      auto *Add = Target.getAddSym();
+      // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
+      // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
+      // this needs to be a GOTPC32 relocation.
+      if (Add && Add->getName() == GotSymName)
+        Fixup = MCFixup::create(Fixup.getOffset(), Fixup.getValue(),
+                                X86::reloc_global_offset_table);
+    } break;
+    }
+  }
+  // Use default handling for `Value` and `IsResolved`.
+  return {};
 }
 
-void X86AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
-                               const MCValue &, MutableArrayRef<char> Data,
-                               uint64_t Value, bool IsResolved) {
+void X86AsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                               const MCValue &Target,
+                               MutableArrayRef<char> Data, uint64_t Value,
+                               bool IsResolved) {
+  // Force relocation when there is a specifier. This might be too conservative
+  // - GAS doesn't emit a relocation for call local@plt; local:.
+  if (Target.getSpecifier())
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
+
   auto Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
@@ -705,8 +727,7 @@ void X86AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
   assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
 
   int64_t SignedValue = static_cast<int64_t>(Value);
-  if (IsResolved &&
-      getFixupKindInfo(Fixup.getKind()).Flags & MCFixupKindInfo::FKF_IsPCRel) {
+  if (IsResolved && Fixup.isPCRel()) {
     // check that PC relative fixup fits into the fixup size.
     if (Size > 0 && !isIntN(Size * 8, SignedValue))
       getContext().reportError(Fixup.getLoc(),
@@ -726,13 +747,13 @@ void X86AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
     Data[Fixup.getOffset() + i] = uint8_t(Value >> (i * 8));
 }
 
-bool X86AsmBackend::mayNeedRelaxation(const MCInst &MI,
+bool X86AsmBackend::mayNeedRelaxation(unsigned Opcode,
+                                      ArrayRef<MCOperand> Operands,
                                       const MCSubtargetInfo &STI) const {
-  unsigned Opcode = MI.getOpcode();
   unsigned SkipOperands = X86::isCCMPCC(Opcode) ? 2 : 0;
   return isRelaxableBranch(Opcode) ||
          (X86::getOpcodeForLongImmediateForm(Opcode) != Opcode &&
-          MI.getOperand(MI.getNumOperands() - 1 - SkipOperands).isExpr());
+          Operands[Operands.size() - 1 - SkipOperands].isExpr());
 }
 
 bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
@@ -748,7 +769,7 @@ bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
 
   // Otherwise, relax unless there is a @ABS8 specifier.
   if (Fixup.getKind() == FK_Data_1 && Target.getAddSym() &&
-      Target.getSpecifier() == X86MCExpr::VK_ABS8)
+      Target.getSpecifier() == X86::S_ABS8)
     return false;
   return true;
 }
@@ -760,15 +781,7 @@ void X86AsmBackend::relaxInstruction(MCInst &Inst,
   // The only relaxations X86 does is from a 1byte pcrel to a 4byte pcrel.
   bool Is16BitMode = STI.hasFeature(X86::Is16Bit);
   unsigned RelaxedOp = getRelaxedOpcode(Inst, Is16BitMode);
-
-  if (RelaxedOp == Inst.getOpcode()) {
-    SmallString<256> Tmp;
-    raw_svector_ostream OS(Tmp);
-    Inst.dump_pretty(OS);
-    OS << "\n";
-    report_fatal_error("unexpected instruction to relax: " + OS.str());
-  }
-
+  assert(RelaxedOp != Inst.getOpcode());
   Inst.setOpcode(RelaxedOp);
 }
 
@@ -781,7 +794,8 @@ bool X86AsmBackend::padInstructionViaPrefix(MCRelaxableFragment &RF,
   // larger value for one of the fixups then can be encoded.  The outer loop
   // will also catch this before moving to the next instruction, but we need to
   // prevent padding this single instruction as well.
-  if (mayNeedRelaxation(RF.getInst(), *RF.getSubtargetInfo()))
+  if (mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
+                        *RF.getSubtargetInfo()))
     return false;
 
   const unsigned OldSize = RF.getContents().size();
@@ -828,7 +842,8 @@ bool X86AsmBackend::padInstructionViaPrefix(MCRelaxableFragment &RF,
 bool X86AsmBackend::padInstructionViaRelaxation(MCRelaxableFragment &RF,
                                                 MCCodeEmitter &Emitter,
                                                 unsigned &RemainingSize) const {
-  if (!mayNeedRelaxation(RF.getInst(), *RF.getSubtargetInfo()))
+  if (!mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
+                         *RF.getSubtargetInfo()))
     // TODO: There are lots of other tricks we could apply for increasing
     // encoding size without impacting performance.
     return false;
@@ -847,7 +862,7 @@ bool X86AsmBackend::padInstructionViaRelaxation(MCRelaxableFragment &RF,
     return false;
   RF.setInst(Relaxed);
   RF.setContents(Code);
-  RF.getFixups() = Fixups;
+  RF.setFixups(Fixups);
   RemainingSize -= Delta;
   return true;
 }
@@ -937,7 +952,8 @@ bool X86AsmBackend::finishLayout(const MCAssembler &Asm) const {
         // We don't need to worry about larger positive offsets as none of the
         // possible offsets between this and our align are visible, and the
         // ones afterwards aren't changing.
-        if (mayNeedRelaxation(RF.getInst(), *RF.getSubtargetInfo()))
+        if (mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
+                              *RF.getSubtargetInfo()))
           break;
       }
       Relaxable.clear();
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index 569484704a249..8c20550bf5129 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -486,6 +486,9 @@ enum TOF {
   /// reference is actually to the ".refptr.FOO" symbol.  This is used for
   /// stub symbols on windows.
   MO_COFFSTUB,
+  /// MO_COFF_IMGREL32: Indicates that the operand value is unsigned 32-bit
+  /// offset from ImageBase to a symbol (basically .imgrel32).
+  MO_COFF_IMGREL32,
 };
 
 enum : uint64_t {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index 13f2a76051df9..3323b387368d6 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -39,10 +39,10 @@ class X86ELFObjectWriter : public MCELFObjectTargetWriter {
 
   void checkIs32(SMLoc Loc, X86_64RelType Type) const;
   void checkIs64(SMLoc Loc, X86_64RelType Type) const;
-  unsigned getRelocType32(SMLoc Loc, X86MCExpr::Specifier Specifier,
+  unsigned getRelocType32(SMLoc Loc, X86::Specifier Specifier,
                           X86_32RelType Type, bool IsPCRel,
                           MCFixupKind Kind) const;
-  unsigned getRelocType64(SMLoc Loc, X86MCExpr::Specifier Specifier,
+  unsigned getRelocType64(SMLoc Loc, X86::Specifier Specifier,
                           X86_64RelType Type, bool IsPCRel,
                           MCFixupKind Kind) const;
 };
@@ -57,8 +57,8 @@ X86ELFObjectWriter::X86ELFObjectWriter(bool IsELF64, uint8_t OSABI,
                               (EMachine != ELF::EM_386) &&
                                   (EMachine != ELF::EM_IAMCU)) {}
 
-static X86_64RelType getType64(MCFixupKind Kind,
-                               X86MCExpr::Specifier &Specifier, bool &IsPCRel) {
+static X86_64RelType getType64(MCFixupKind Kind, X86::Specifier &Specifier,
+                               bool &IsPCRel) {
   switch (unsigned(Kind)) {
   default:
     llvm_unreachable("Unimplemented");
@@ -68,15 +68,14 @@ static X86_64RelType getType64(MCFixupKind Kind,
     return RT64_64;
   case X86::reloc_signed_4byte:
   case X86::reloc_signed_4byte_relax:
-    if (Specifier == X86MCExpr::VK_None && !IsPCRel)
+    if (Specifier == X86::S_None && !IsPCRel)
       return RT64_32S;
     return RT64_32;
   case X86::reloc_global_offset_table:
-    Specifier = X86MCExpr::VK_GOT;
+    Specifier = X86::S_GOT;
     IsPCRel = true;
     return RT64_32;
   case FK_Data_4:
-  case FK_PCRel_4:
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_relax:
   case X86::reloc_riprel_4byte_relax_rex:
@@ -86,12 +85,10 @@ static X86_64RelType getType64(MCFixupKind Kind,
   case X86::reloc_riprel_4byte_relax_evex:
     return RT64_32;
   case X86::reloc_branch_4byte_pcrel:
-    Specifier = X86MCExpr::VK_PLT;
+    Specifier = X86::S_PLT;
     return RT64_32;
-  case FK_PCRel_2:
   case FK_Data_2:
     return RT64_16;
-  case FK_PCRel_1:
   case FK_Data_1:
     return RT64_8;
   }
@@ -107,18 +104,17 @@ void X86ELFObjectWriter::checkIs64(SMLoc Loc, X86_64RelType Type) const {
     reportError(Loc, "64 bit reloc applied to a field with a different size");
 }
 
-unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
-                                            X86MCExpr::Specifier Specifier,
+unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc, X86::Specifier Specifier,
                                             X86_64RelType Type, bool IsPCRel,
                                             MCFixupKind Kind) const {
   switch (Specifier) {
   default:
     llvm_unreachable("Unimplemented");
-  case X86MCExpr::VK_None:
-  case X86MCExpr::VK_ABS8:
+  case X86::S_None:
+  case X86::S_ABS8:
     switch (Type) {
     case RT64_NONE:
-      if (Specifier == X86MCExpr::VK_None)
+      if (Specifier == X86::S_None)
         return ELF::R_X86_64_NONE;
       llvm_unreachable("Unimplemented");
     case RT64_64:
@@ -133,7 +129,7 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       return IsPCRel ? ELF::R_X86_64_PC8 : ELF::R_X86_64_8;
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOT:
+  case X86::S_GOT:
     switch (Type) {
     case RT64_64:
       return IsPCRel ? ELF::R_X86_64_GOTPC64 : ELF::R_X86_64_GOT64;
@@ -146,12 +142,12 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOTOFF:
+  case X86::S_GOTOFF:
     assert(!IsPCRel);
     if (Type != RT64_64)
       reportError(Loc, "unsupported relocation type");
     return ELF::R_X86_64_GOTOFF64;
-  case X86MCExpr::VK_TPOFF:
+  case X86::S_TPOFF:
     assert(!IsPCRel);
     switch (Type) {
     case RT64_64:
@@ -165,7 +161,7 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_DTPOFF:
+  case X86::S_DTPOFF:
     assert(!IsPCRel);
     switch (Type) {
     case RT64_64:
@@ -179,7 +175,7 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_SIZE:
+  case X86::S_SIZE:
     assert(!IsPCRel);
     switch (Type) {
     case RT64_64:
@@ -193,16 +189,16 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_TLSCALL:
+  case X86::S_TLSCALL:
     return ELF::R_X86_64_TLSDESC_CALL;
-  case X86MCExpr::VK_TLSDESC:
+  case X86::S_TLSDESC:
     return ((unsigned)Kind == X86::reloc_riprel_4byte_relax_rex2)
                ? ELF::R_X86_64_CODE_4_GOTPC32_TLSDESC
                : ELF::R_X86_64_GOTPC32_TLSDESC;
-  case X86MCExpr::VK_TLSGD:
+  case X86::S_TLSGD:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_TLSGD;
-  case X86MCExpr::VK_GOTTPOFF:
+  case X86::S_GOTTPOFF:
     checkIs32(Loc, Type);
     if ((unsigned)Kind == X86::reloc_riprel_4byte_movq_load_rex2 ||
         (unsigned)Kind == X86::reloc_riprel_4byte_relax_rex2)
@@ -210,13 +206,13 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
     else if ((unsigned)Kind == X86::reloc_riprel_4byte_relax_evex)
       return ELF::R_X86_64_CODE_6_GOTTPOFF;
     return ELF::R_X86_64_GOTTPOFF;
-  case X86MCExpr::VK_TLSLD:
+  case X86::S_TLSLD:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_TLSLD;
-  case X86MCExpr::VK_PLT:
+  case X86::S_PLT:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_PLT32;
-  case X86MCExpr::VK_GOTPCREL:
+  case X86::S_GOTPCREL:
     checkIs32(Loc, Type);
     // Older versions of ld.bfd/ld.gold/lld
     // do not support GOTPCRELX/REX_GOTPCRELX/CODE_4_GOTPCRELX,
@@ -236,27 +232,26 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       return ELF::R_X86_64_CODE_4_GOTPCRELX;
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOTPCREL_NORELAX:
+  case X86::S_GOTPCREL_NORELAX:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_GOTPCREL;
-  case X86MCExpr::VK_PLTOFF:
+  case X86::S_PLTOFF:
     checkIs64(Loc, Type);
     return ELF::R_X86_64_PLTOFF64;
   }
 }
 
-unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc,
-                                            X86MCExpr::Specifier Specifier,
+unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc, X86::Specifier Specifier,
                                             X86_32RelType Type, bool IsPCRel,
                                             MCFixupKind Kind) const {
   switch (Specifier) {
   default:
     llvm_unreachable("Unimplemented");
-  case X86MCExpr::VK_None:
-  case X86MCExpr::VK_ABS8:
+  case X86::S_None:
+  case X86::S_ABS8:
     switch (Type) {
     case RT32_NONE:
-      if (Specifier == X86MCExpr::VK_None)
+      if (Specifier == X86::S_None)
         return ELF::R_386_NONE;
       llvm_unreachable("Unimplemented");
     case RT32_32:
@@ -267,7 +262,7 @@ unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc,
       return IsPCRel ? ELF::R_386_PC8 : ELF::R_386_8;
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOT:
+  case X86::S_GOT:
     if (Type != RT32_32)
       break;
     if (IsPCRel)
@@ -277,58 +272,57 @@ unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc,
     if (!getContext().getTargetOptions()->X86RelaxRelocations)
       return ELF::R_386_GOT32;
 
-    return Kind == MCFixupKind(X86::reloc_signed_4byte_relax)
-               ? ELF::R_386_GOT32X
-               : ELF::R_386_GOT32;
-  case X86MCExpr::VK_GOTOFF:
+    return Kind == X86::reloc_signed_4byte_relax ? ELF::R_386_GOT32X
+                                                 : ELF::R_386_GOT32;
+  case X86::S_GOTOFF:
     assert(!IsPCRel);
     if (Type != RT32_32)
       break;
     return ELF::R_386_GOTOFF;
-  case X86MCExpr::VK_TLSCALL:
+  case X86::S_TLSCALL:
     return ELF::R_386_TLS_DESC_CALL;
-  case X86MCExpr::VK_TLSDESC:
+  case X86::S_TLSDESC:
     return ELF::R_386_TLS_GOTDESC;
-  case X86MCExpr::VK_TPOFF:
+  case X86::S_TPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_LE_32;
-  case X86MCExpr::VK_DTPOFF:
+  case X86::S_DTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_LDO_32;
-  case X86MCExpr::VK_TLSGD:
+  case X86::S_TLSGD:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_GD;
-  case X86MCExpr::VK_GOTTPOFF:
+  case X86::S_GOTTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_IE_32;
-  case X86MCExpr::VK_PLT:
+  case X86::S_PLT:
     if (Type != RT32_32)
       break;
     return ELF::R_386_PLT32;
-  case X86MCExpr::VK_INDNTPOFF:
+  case X86::S_INDNTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_IE;
-  case X86MCExpr::VK_NTPOFF:
+  case X86::S_NTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_LE;
-  case X86MCExpr::VK_GOTNTPOFF:
+  case X86::S_GOTNTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_GOTIE;
-  case X86MCExpr::VK_TLSLDM:
+  case X86::S_TLSLDM:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
@@ -342,19 +336,19 @@ unsigned X86ELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                           const MCValue &Target,
                                           bool IsPCRel) const {
   MCFixupKind Kind = Fixup.getKind();
-  auto Specifier = X86MCExpr::Specifier(Target.getSpecifier());
+  auto Specifier = X86::Specifier(Target.getSpecifier());
   switch (Specifier) {
-  case X86MCExpr::VK_GOTTPOFF:
-  case X86MCExpr::VK_INDNTPOFF:
-  case X86MCExpr::VK_NTPOFF:
-  case X86MCExpr::VK_GOTNTPOFF:
-  case X86MCExpr::VK_TLSCALL:
-  case X86MCExpr::VK_TLSDESC:
-  case X86MCExpr::VK_TLSGD:
-  case X86MCExpr::VK_TLSLD:
-  case X86MCExpr::VK_TLSLDM:
-  case X86MCExpr::VK_TPOFF:
-  case X86MCExpr::VK_DTPOFF:
+  case X86::S_GOTTPOFF:
+  case X86::S_INDNTPOFF:
+  case X86::S_NTPOFF:
+  case X86::S_GOTNTPOFF:
+  case X86::S_TLSCALL:
+  case X86::S_TLSDESC:
+  case X86::S_TLSGD:
+  case X86::S_TLSLD:
+  case X86::S_TLSLDM:
+  case X86::S_TPOFF:
+  case X86::S_DTPOFF:
     if (auto *S = Target.getAddSym())
       cast<MCSymbolELF>(S)->setType(ELF::STT_TLS);
     break;
@@ -393,10 +387,10 @@ unsigned X86ELFObjectWriter::getRelocType(const MCFixup &Fixup,
 bool X86ELFObjectWriter::needsRelocateWithSymbol(const MCValue &V,
                                                  unsigned Type) const {
   switch (V.getSpecifier()) {
-  case X86MCExpr::VK_GOT:
-  case X86MCExpr::VK_PLT:
-  case X86MCExpr::VK_GOTPCREL:
-  case X86MCExpr::VK_GOTPCREL_NORELAX:
+  case X86::S_GOT:
+  case X86::S_PLT:
+  case X86::S_GOTPCREL:
+  case X86::S_GOTPCREL_NORELAX:
     return true;
   default:
     return false;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp
index e4fdd85929a4b..58c5e329a38f6 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86EncodingOptimization.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "X86BaseInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
@@ -375,7 +375,7 @@ bool X86::optimizeMOV(MCInst &MI, bool In64BitMode) {
   if (MI.getOperand(AddrOp).isExpr()) {
     const MCExpr *MCE = MI.getOperand(AddrOp).getExpr();
     if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE))
-      if (getSpecifier(SRE) == X86MCExpr::VK_TLVP)
+      if (SRE->getSpecifier() == X86::S_TLVP)
         Absolute = false;
   }
   if (Absolute && (MI.getOperand(AddrBase + X86::AddrBaseReg).getReg() ||
@@ -486,7 +486,7 @@ static bool optimizeToShortImmediateForm(MCInst &MI) {
   MCOperand &LastOp = MI.getOperand(MI.getNumOperands() - 1 - SkipOperands);
   if (LastOp.isExpr()) {
     const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(LastOp.getExpr());
-    if (!SRE || getSpecifier(SRE) != X86MCExpr::VK_ABS8)
+    if (!SRE || SRE->getSpecifier() != X86::S_ABS8)
       return false;
   } else if (LastOp.isImm()) {
     if (!isInt<8>(LastOp.getImm()))
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index 3e48c231ea1d6..a769570c4e694 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -36,34 +36,34 @@ MarkedJTDataRegions("mark-data-regions", cl::init(true),
   cl::Hidden);
 
 const MCAsmInfo::AtSpecifier atSpecifiers[] = {
-    {X86MCExpr::VK_ABS8, "ABS8"},
-    {X86MCExpr::VK_DTPOFF, "DTPOFF"},
-    {X86MCExpr::VK_DTPREL, "DTPREL"},
-    {X86MCExpr::VK_GOT, "GOT"},
-    {X86MCExpr::VK_GOTENT, "GOTENT"},
-    {X86MCExpr::VK_GOTNTPOFF, "GOTNTPOFF"},
-    {X86MCExpr::VK_GOTOFF, "GOTOFF"},
-    {X86MCExpr::VK_GOTPCREL, "GOTPCREL"},
-    {X86MCExpr::VK_GOTPCREL_NORELAX, "GOTPCREL_NORELAX"},
-    {X86MCExpr::VK_GOTREL, "GOTREL"},
-    {X86MCExpr::VK_GOTTPOFF, "GOTTPOFF"},
-    {X86MCExpr::VK_INDNTPOFF, "INDNTPOFF"},
+    {X86::S_ABS8, "ABS8"},
+    {X86::S_DTPOFF, "DTPOFF"},
+    {X86::S_DTPREL, "DTPREL"},
+    {X86::S_GOT, "GOT"},
+    {X86::S_GOTENT, "GOTENT"},
+    {X86::S_GOTNTPOFF, "GOTNTPOFF"},
+    {X86::S_GOTOFF, "GOTOFF"},
+    {X86::S_GOTPCREL, "GOTPCREL"},
+    {X86::S_GOTPCREL_NORELAX, "GOTPCREL_NORELAX"},
+    {X86::S_GOTREL, "GOTREL"},
+    {X86::S_GOTTPOFF, "GOTTPOFF"},
+    {X86::S_INDNTPOFF, "INDNTPOFF"},
     {MCSymbolRefExpr::VK_COFF_IMGREL32, "IMGREL"},
-    {X86MCExpr::VK_NTPOFF, "NTPOFF"},
-    {X86MCExpr::VK_PCREL, "PCREL"},
-    {X86MCExpr::VK_PLT, "PLT"},
-    {X86MCExpr::VK_PLTOFF, "PLTOFF"},
-    {MCSymbolRefExpr::VK_SECREL, "SECREL32"},
-    {X86MCExpr::VK_SIZE, "SIZE"},
-    {X86MCExpr::VK_TLSCALL, "tlscall"},
-    {X86MCExpr::VK_TLSDESC, "tlsdesc"},
-    {X86MCExpr::VK_TLSGD, "TLSGD"},
-    {X86MCExpr::VK_TLSLD, "TLSLD"},
-    {X86MCExpr::VK_TLSLDM, "TLSLDM"},
-    {X86MCExpr::VK_TLVP, "TLVP"},
-    {X86MCExpr::VK_TLVPPAGE, "TLVPPAGE"},
-    {X86MCExpr::VK_TLVPPAGEOFF, "TLVPPAGEOFF"},
-    {X86MCExpr::VK_TPOFF, "TPOFF"},
+    {X86::S_NTPOFF, "NTPOFF"},
+    {X86::S_PCREL, "PCREL"},
+    {X86::S_PLT, "PLT"},
+    {X86::S_PLTOFF, "PLTOFF"},
+    {X86::S_COFF_SECREL, "SECREL32"},
+    {X86::S_SIZE, "SIZE"},
+    {X86::S_TLSCALL, "tlscall"},
+    {X86::S_TLSDESC, "tlsdesc"},
+    {X86::S_TLSGD, "TLSGD"},
+    {X86::S_TLSLD, "TLSLD"},
+    {X86::S_TLSLDM, "TLSLDM"},
+    {X86::S_TLVP, "TLVP"},
+    {X86::S_TLVPPAGE, "TLVPPAGE"},
+    {X86::S_TLVPPAGEOFF, "TLVPPAGEOFF"},
+    {X86::S_TPOFF, "TPOFF"},
 };
 
 void X86MCAsmInfoDarwin::anchor() { }
@@ -139,8 +139,7 @@ X86_64MCAsmInfoDarwin::getExprForPersonalitySymbol(const MCSymbol *Sym,
                                                    unsigned Encoding,
                                                    MCStreamer &Streamer) const {
   MCContext &Context = Streamer.getContext();
-  const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, Context);
+  const MCExpr *Res = MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, Context);
   const MCExpr *Four = MCConstantExpr::create(4, Context);
   return MCBinaryExpr::createAdd(Res, Four, Context);
 }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
index ce8e84fb96b9b..9533bfa31cb1f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
@@ -13,9 +13,11 @@
 #ifndef LLVM_LIB_TARGET_X86_MCTARGETDESC_X86MCASMINFO_H
 #define LLVM_LIB_TARGET_X86_MCTARGETDESC_X86MCASMINFO_H
 
+#include "MCTargetDesc/X86MCExpr.h"
 #include "llvm/MC/MCAsmInfoCOFF.h"
 #include "llvm/MC/MCAsmInfoDarwin.h"
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
 
 namespace llvm {
 class Triple;
@@ -61,6 +63,42 @@ class X86MCAsmInfoGNUCOFF : public MCAsmInfoGNUCOFF {
 public:
   explicit X86MCAsmInfoGNUCOFF(const Triple &Triple);
 };
+
+namespace X86 {
+using Specifier = uint16_t;
+
+enum {
+  S_None,
+  S_COFF_SECREL,
+
+  S_ABS8 = MCSymbolRefExpr::FirstTargetSpecifier,
+  S_DTPOFF,
+  S_DTPREL,
+  S_GOT,
+  S_GOTENT,
+  S_GOTNTPOFF,
+  S_GOTOFF,
+  S_GOTPCREL,
+  S_GOTPCREL_NORELAX,
+  S_GOTREL,
+  S_GOTTPOFF,
+  S_INDNTPOFF,
+  S_NTPOFF,
+  S_PCREL,
+  S_PLT,
+  S_PLTOFF,
+  S_SIZE,
+  S_TLSCALL,
+  S_TLSDESC,
+  S_TLSGD,
+  S_TLSLD,
+  S_TLSLDM,
+  S_TLVP,
+  S_TLVPPAGE,
+  S_TLVPPAGEOFF,
+  S_TPOFF,
+};
+} // namespace X86
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 6e28c058b702d..a15930c1433f7 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -12,7 +12,7 @@
 
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -358,8 +358,8 @@ class X86MCCodeEmitter : public MCCodeEmitter {
 
   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
 
-  void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
-                     MCFixupKind FixupKind, uint64_t StartByte,
+  void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned FixupKind,
+                     bool IsPCRel, uint64_t StartByte,
                      SmallVectorImpl<char> &CB,
                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
 
@@ -441,27 +441,25 @@ static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
 /// instruction with the specified TSFlags.
 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
   unsigned Size = X86II::getSizeOfImm(TSFlags);
-  bool isPCRel = X86II::isImmPCRel(TSFlags);
-
   if (X86II::isImmSigned(TSFlags)) {
     switch (Size) {
     default:
       llvm_unreachable("Unsupported signed fixup size!");
     case 4:
-      return MCFixupKind(X86::reloc_signed_4byte);
+      return X86::reloc_signed_4byte;
     }
   }
   switch (Size) {
   default:
     llvm_unreachable("Invalid generic fixup size!");
   case 1:
-    return isPCRel ? FK_PCRel_1 : FK_Data_1;
+    return FK_Data_1;
   case 2:
-    return isPCRel ? FK_PCRel_2 : FK_Data_2;
+    return FK_Data_2;
   case 4:
-    return isPCRel ? FK_PCRel_4 : FK_Data_4;
+    return FK_Data_4;
   case 8:
-    return isPCRel ? FK_PCRel_8 : FK_Data_8;
+    return FK_Data_8;
   }
 }
 
@@ -472,6 +470,9 @@ enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
 /// binary expression.
+///
+/// TODO: Move this to X86AsmBackend.cpp at relocation decision phase so that we
+/// don't have to mess with MCExpr.
 static GlobalOffsetTableExprKind
 startsWithGlobalOffsetTable(const MCExpr *Expr) {
   const MCExpr *RHS = nullptr;
@@ -496,7 +497,7 @@ startsWithGlobalOffsetTable(const MCExpr *Expr) {
 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
   if (Expr->getKind() == MCExpr::SymbolRef) {
     auto *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
-    return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
+    return Ref->getSpecifier() == X86::S_COFF_SECREL;
   }
   return false;
 }
@@ -506,7 +507,8 @@ static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
   const MCInstrDesc &Desc = MCII.get(Opcode);
   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
        Opcode != X86::JCC_4) ||
-      getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
+      !(getImmFixupKind(Desc.TSFlags) == FK_Data_4 &&
+        X86II::isImmPCRel(Desc.TSFlags)))
     return false;
 
   unsigned CurOp = X86II::getOperandBias(Desc);
@@ -515,7 +517,7 @@ static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
     return false;
 
   auto *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
-  return Ref && getSpecifier(Ref) == X86MCExpr::VK_None;
+  return Ref && Ref->getSpecifier() == X86::S_None;
 }
 
 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
@@ -528,17 +530,29 @@ unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
 }
 
 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
-                                     unsigned Size, MCFixupKind FixupKind,
+                                     unsigned FixupKind, bool PCRel,
                                      uint64_t StartByte,
                                      SmallVectorImpl<char> &CB,
                                      SmallVectorImpl<MCFixup> &Fixups,
                                      int ImmOffset) const {
+  unsigned Size = 4;
+  switch (FixupKind) {
+  case FK_Data_1:
+    Size = 1;
+    break;
+  case FK_Data_2:
+    Size = 2;
+    break;
+  case FK_Data_8:
+    Size = 8;
+    break;
+  }
   const MCExpr *Expr = nullptr;
   if (DispOp.isImm()) {
     // If this is a simple integer displacement that doesn't require a
     // relocation, emit it now.
-    if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
-        FixupKind != FK_PCRel_4) {
+    if (!(is_contained({FK_Data_1, FK_Data_2, FK_Data_4}, FixupKind) &&
+          PCRel)) {
       emitConstant(DispOp.getImm() + ImmOffset, Size, CB);
       return;
     }
@@ -549,65 +563,40 @@ void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
 
   // If we have an immoffset, add it to the expression.
   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
-       FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
+       FixupKind == X86::reloc_signed_4byte)) {
     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
     if (Kind != GOT_None) {
       assert(ImmOffset == 0);
 
       if (Size == 8) {
-        FixupKind =
-            MCFixupKind(FirstLiteralRelocationKind + ELF::R_X86_64_GOTPC64);
+        FixupKind = FirstLiteralRelocationKind + ELF::R_X86_64_GOTPC64;
       } else {
         assert(Size == 4);
-        FixupKind = MCFixupKind(X86::reloc_global_offset_table);
+        FixupKind = X86::reloc_global_offset_table;
       }
 
       if (Kind == GOT_Normal)
         ImmOffset = static_cast<int>(CB.size() - StartByte);
     } else if (Expr->getKind() == MCExpr::SymbolRef) {
       if (hasSecRelSymbolRef(Expr)) {
-        FixupKind = MCFixupKind(FK_SecRel_4);
+        FixupKind = FK_SecRel_4;
       }
     } else if (Expr->getKind() == MCExpr::Binary) {
       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
       if (hasSecRelSymbolRef(Bin->getLHS()) ||
           hasSecRelSymbolRef(Bin->getRHS())) {
-        FixupKind = MCFixupKind(FK_SecRel_4);
+        FixupKind = FK_SecRel_4;
       }
     }
   }
 
-  // If the fixup is pc-relative, we need to bias the value to be relative to
-  // the start of the field, not the end of the field.
-  if (FixupKind == FK_PCRel_4 ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load_rex2) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex2) ||
-      FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_evex)) {
-    ImmOffset -= 4;
-    // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
-    // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
-    // this needs to be a GOTPC32 relocation.
-    if (startsWithGlobalOffsetTable(Expr) != GOT_None)
-      FixupKind = MCFixupKind(X86::reloc_global_offset_table);
-  }
-
-  if (FixupKind == FK_PCRel_2)
-    ImmOffset -= 2;
-  if (FixupKind == FK_PCRel_1)
-    ImmOffset -= 1;
-
   if (ImmOffset)
     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
-                                   Ctx);
+                                   Ctx, Expr->getLoc());
 
-  // Emit a symbolic constant as a fixup and 4 zeros.
+  // Emit a symbolic constant as a fixup and a few zero bytes.
   Fixups.push_back(MCFixup::create(static_cast<uint32_t>(CB.size() - StartByte),
-                                   Expr, FixupKind, Loc));
+                                   Expr, FixupKind, PCRel));
   emitConstant(0, Size, CB);
 }
 
@@ -706,8 +695,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
                       ? X86II::getSizeOfImm(TSFlags)
                       : 0;
 
-    emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB,
-                  Fixups, -ImmSize);
+    emitImmediate(Disp, MI.getLoc(), FixupKind, true, StartByte, CB, Fixups,
+                  -ImmSize);
     return;
   }
 
@@ -761,7 +750,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
         }
         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
         emitByte(modRMByte(1, RegOpcodeField, RMfield), CB);
-        emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups);
+        emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB,
+                      Fixups);
         return;
       }
       // This is the [REG]+disp16 case.
@@ -773,7 +763,7 @@ void X86MCCodeEmitter::emitMemModRMByte(
     }
 
     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
-    emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, CB, Fixups);
+    emitImmediate(Disp, MI.getLoc(), FK_Data_2, false, StartByte, CB, Fixups);
     return;
   }
 
@@ -791,7 +781,7 @@ void X86MCCodeEmitter::emitMemModRMByte(
                                    STI.hasFeature(X86::Is64Bit))) {
     if (!BaseReg) { // [disp32]     in X86-32 mode
       emitByte(modRMByte(0, RegOpcodeField, 5), CB);
-      emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, CB, Fixups);
+      emitImmediate(Disp, MI.getLoc(), FK_Data_4, false, StartByte, CB, Fixups);
       return;
     }
 
@@ -809,10 +799,10 @@ void X86MCCodeEmitter::emitMemModRMByte(
       // If the displacement is @tlscall, treat it as a zero.
       if (Disp.isExpr()) {
         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
-        if (Sym && getSpecifier(Sym) == X86MCExpr::VK_TLSCALL) {
+        if (Sym && Sym->getSpecifier() == X86::S_TLSCALL) {
           // This is exclusively used by call *a@tlscall(base). The relocation
           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
-          Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
+          Fixups.push_back(MCFixup::create(0, Sym, FK_NONE));
           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB);
           return;
         }
@@ -827,8 +817,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
       int ImmOffset = 0;
       if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CB);
-        emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups,
-                      ImmOffset);
+        emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB,
+                      Fixups, ImmOffset);
         return;
       }
     }
@@ -840,8 +830,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
     unsigned Opcode = MI.getOpcode();
     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
                                                 : X86::reloc_signed_4byte;
-    emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB,
-                  Fixups);
+    emitImmediate(Disp, MI.getLoc(), MCFixupKind(FixupKind), false, StartByte,
+                  CB, Fixups);
     return;
   }
 
@@ -889,11 +879,11 @@ void X86MCCodeEmitter::emitMemModRMByte(
 
   // Do we need to output a displacement?
   if (ForceDisp8)
-    emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups,
+    emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB, Fixups,
                   ImmOffset);
   else if (ForceDisp32)
-    emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
-                  StartByte, CB, Fixups);
+    emitImmediate(Disp, MI.getLoc(), X86::reloc_signed_4byte, false, StartByte,
+                  CB, Fixups);
 }
 
 /// Emit all instruction prefixes.
@@ -1385,8 +1375,8 @@ PrefixKind X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
       // handled as a special case here so that it also works for hand-written
       // assembly without the user needing to write REX, as with GNU as.
       const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());
-      if (Ref && (getSpecifier(Ref) == X86MCExpr::VK_GOTTPOFF ||
-                  getSpecifier(Ref) == X86MCExpr::VK_TLSDESC)) {
+      if (Ref && (Ref->getSpecifier() == X86::S_GOTTPOFF ||
+                  Ref->getSpecifier() == X86::S_TLSDESC)) {
         Prefix.setLowerBound(REX);
       }
     }
@@ -1628,33 +1618,29 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
       break;
 
     const MCOperand &Op = MI.getOperand(CurOp++);
-    emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
-                  MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, CB,
-                  Fixups);
+    emitImmediate(Op, MI.getLoc(), X86::reloc_branch_4byte_pcrel, true,
+                  StartByte, CB, Fixups);
     break;
   }
   case X86II::RawFrmMemOffs:
     emitByte(BaseOpcode, CB);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
-                  StartByte, CB, Fixups);
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),
+                  X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
     ++CurOp; // skip segment operand
     break;
   case X86II::RawFrmImm8:
     emitByte(BaseOpcode, CB);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),
+                  X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), FK_Data_1, false,
                   StartByte, CB, Fixups);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
-                  CB, Fixups);
     break;
   case X86II::RawFrmImm16:
     emitByte(BaseOpcode, CB);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),
+                  X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), FK_Data_2, false,
                   StartByte, CB, Fixups);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
-                  CB, Fixups);
     break;
 
   case X86II::AddRegFrm:
@@ -2007,7 +1993,7 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
       assert(Val < 16 && "Immediate operand value out of range");
       I8RegNum |= Val;
     }
-    emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
+    emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), FK_Data_1, false,
                   StartByte, CB, Fixups);
   } else {
     // If there is a remaining operand, it must be a trailing immediate. Emit it
@@ -2015,12 +2001,12 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
     // (SSE4a extrq and insertq) have two trailing immediates.
 
     // Skip two trainling conditional operands encoded in EVEX prefix
-    unsigned RemaningOps = NumOps - CurOp - 2 * HasTwoConditionalOps;
-    while (RemaningOps) {
+    unsigned RemainingOps = NumOps - CurOp - 2 * HasTwoConditionalOps;
+    while (RemainingOps) {
       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                    X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
-                    StartByte, CB, Fixups);
-      --RemaningOps;
+                    getImmFixupKind(Desc.TSFlags),
+                    X86II::isImmPCRel(Desc.TSFlags), StartByte, CB, Fixups);
+      --RemainingOps;
     }
     CurOp += 2 * HasTwoConditionalOps;
   }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
index 1d3adaf867f67..e1ec3fa1205f6 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
@@ -31,54 +31,13 @@ class X86MCExpr : public MCTargetExpr {
   explicit X86MCExpr(MCRegister R) : Reg(R) {}
 
 public:
-  // Relocation specifier. Named VK_ for legacy reasons.
-  enum Specifier {
-    VK_None,
-
-    VK_ABS8 = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_DTPOFF,
-    VK_DTPREL,
-    VK_GOT,
-    VK_GOTENT,
-    VK_GOTNTPOFF,
-    VK_GOTOFF,
-    VK_GOTPCREL,
-    VK_GOTPCREL_NORELAX,
-    VK_GOTREL,
-    VK_GOTTPOFF,
-    VK_INDNTPOFF,
-    VK_NTPOFF,
-    VK_PCREL,
-    VK_PLT,
-    VK_PLTOFF,
-    VK_SIZE,
-    VK_TLSCALL,
-    VK_TLSDESC,
-    VK_TLSGD,
-    VK_TLSLD,
-    VK_TLSLDM,
-    VK_TLVP,
-    VK_TLVPPAGE,
-    VK_TLVPPAGEOFF,
-    VK_TPOFF,
-  };
-
-  /// @name Construction
-  /// @{
-
   static const X86MCExpr *create(MCRegister Reg, MCContext &Ctx) {
     return new (Ctx) X86MCExpr(Reg);
   }
 
-  /// @}
-  /// @name Accessors
-  /// @{
-
   /// getSubExpr - Get the child of this expression.
   MCRegister getReg() const { return Reg; }
 
-  /// @}
-
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override {
     if (!MAI || MAI->getAssemblerDialect() == 0)
       OS << '%';
@@ -103,10 +62,6 @@ class X86MCExpr : public MCTargetExpr {
     return E->getKind() == MCExpr::Target;
   }
 };
-
-static inline X86MCExpr::Specifier getSpecifier(const MCSymbolRefExpr *SRE) {
-  return X86MCExpr::Specifier(SRE->getKind());
-}
 } // end namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 2eb22b239aa48..cc7bcd678cb31 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -17,6 +17,7 @@
 #include "X86IntelInstPrinter.h"
 #include "X86MCAsmInfo.h"
 #include "X86TargetStreamer.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/MC/MCDwarf.h"
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index b61dbfd968bd0..c34425f6661b4 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -78,11 +78,8 @@ static unsigned getFixupKindLog2Size(unsigned Kind) {
   switch (Kind) {
   default:
     llvm_unreachable("invalid fixup kind!");
-  case FK_PCRel_1:
   case FK_Data_1: return 0;
-  case FK_PCRel_2:
   case FK_Data_2: return 1;
-  case FK_PCRel_4:
     // FIXME: Remove these!!!
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_relax:
@@ -102,7 +99,7 @@ static unsigned getFixupKindLog2Size(unsigned Kind) {
 void X86MachObjectWriter::RecordX86_64Relocation(
     MachObjectWriter *Writer, MCAssembler &Asm, const MCFragment *Fragment,
     const MCFixup &Fixup, MCValue Target, uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned IsRIPRel = isFixupKindRIPRel(Fixup.getKind());
   unsigned Log2Size = getFixupKindLog2Size(Fixup.getKind());
 
@@ -257,7 +254,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(
     auto Specifier = Target.getSpecifier();
     if (IsPCRel) {
       if (IsRIPRel) {
-        if (Specifier == X86MCExpr::VK_GOTPCREL) {
+        if (Specifier == X86::S_GOTPCREL) {
           // x86_64 distinguishes movq foo@GOTPCREL so that the linker can
           // rewrite the movq to an leaq at link time if the symbol ends up in
           // the same linkage unit.
@@ -265,7 +262,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(
             Type = MachO::X86_64_RELOC_GOT_LOAD;
           else
             Type = MachO::X86_64_RELOC_GOT;
-        } else if (Specifier == X86MCExpr::VK_TLVP) {
+        } else if (Specifier == X86::S_TLVP) {
           Type = MachO::X86_64_RELOC_TLV;
         } else if (Specifier) {
           reportError(Fixup.getLoc(),
@@ -304,16 +301,16 @@ void X86MachObjectWriter::RecordX86_64Relocation(
         Type = MachO::X86_64_RELOC_BRANCH;
       }
     } else {
-      if (Specifier == X86MCExpr::VK_GOT) {
+      if (Specifier == X86::S_GOT) {
         Type = MachO::X86_64_RELOC_GOT;
-      } else if (Specifier == X86MCExpr::VK_GOTPCREL) {
+      } else if (Specifier == X86::S_GOTPCREL) {
         // GOTPCREL is allowed as a modifier on non-PCrel instructions, in which
         // case all we do is set the PCrel bit in the relocation entry; this is
         // used with exception handling, for example. The source is required to
         // include any necessary offset directly.
         Type = MachO::X86_64_RELOC_GOT;
         IsPCRel = 1;
-      } else if (Specifier == X86MCExpr::VK_TLVP) {
+      } else if (Specifier == X86::S_TLVP) {
         reportError(Fixup.getLoc(),
                     "TLVP symbol modifier should have been rip-rel");
         return;
@@ -353,7 +350,7 @@ bool X86MachObjectWriter::recordScatteredRelocation(MachObjectWriter *Writer,
                                                     uint64_t &FixedValue) {
   uint64_t OriginalFixedValue = FixedValue;
   uint32_t FixupOffset = Asm.getFragmentOffset(*Fragment) + Fixup.getOffset();
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Type = MachO::GENERIC_RELOC_VANILLA;
 
   // See <reloc.h>.
@@ -446,7 +443,7 @@ void X86MachObjectWriter::recordTLVPRelocation(MachObjectWriter *Writer,
                                                MCValue Target,
                                                uint64_t &FixedValue) {
   const MCSymbol *SymA = Target.getAddSym();
-  assert(Target.getSpecifier() == X86MCExpr::VK_TLVP && !is64Bit() &&
+  assert(Target.getSpecifier() == X86::S_TLVP && !is64Bit() &&
          "Should only be called with a 32-bit TLVP relocation!");
 
   unsigned Log2Size = getFixupKindLog2Size(Fixup.getKind());
@@ -483,12 +480,12 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
                                               const MCFixup &Fixup,
                                               MCValue Target,
                                               uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Log2Size = getFixupKindLog2Size(Fixup.getKind());
   const MCSymbol *A = Target.getAddSym();
 
   // If this is a 32-bit TLVP reloc it's handled a bit differently.
-  if (A && Target.getSpecifier() == X86MCExpr::VK_TLVP) {
+  if (A && Target.getSpecifier() == X86::S_TLVP) {
     recordTLVPRelocation(Writer, Asm, Fragment, Fixup, Target, FixedValue);
     return;
   }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
index bc8bf3868fc33..0bfc0e071a66d 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/X86FixupKinds.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/MC/MCContext.h"
@@ -44,14 +45,17 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
                                               const MCAsmBackend &MAB) const {
   const bool Is64Bit = getMachine() == COFF::IMAGE_FILE_MACHINE_AMD64;
   unsigned FixupKind = Fixup.getKind();
+  bool PCRel = Fixup.isPCRel();
   if (IsCrossSection) {
     // IMAGE_REL_AMD64_REL64 does not exist. We treat FK_Data_8 as FK_PCRel_4 so
     // that .quad a-b can lower to IMAGE_REL_AMD64_REL32. This allows generic
     // instrumentation to not bother with the COFF limitation. A negative value
     // needs attention.
-    if (FixupKind == FK_Data_4 || FixupKind == llvm::X86::reloc_signed_4byte ||
-        (FixupKind == FK_Data_8 && Is64Bit)) {
-      FixupKind = FK_PCRel_4;
+    if (!PCRel &&
+        (FixupKind == FK_Data_4 || FixupKind == llvm::X86::reloc_signed_4byte ||
+         (FixupKind == FK_Data_8 && Is64Bit))) {
+      FixupKind = FK_Data_4;
+      PCRel = true;
     } else {
       Ctx.reportError(Fixup.getLoc(), "Cannot represent this expression");
       return COFF::IMAGE_REL_AMD64_ADDR32;
@@ -61,7 +65,6 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
   auto Spec = Target.getSpecifier();
   if (Is64Bit) {
     switch (FixupKind) {
-    case FK_PCRel_4:
     case X86::reloc_riprel_4byte:
     case X86::reloc_riprel_4byte_movq_load:
     case X86::reloc_riprel_4byte_movq_load_rex2:
@@ -72,11 +75,14 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
     case X86::reloc_branch_4byte_pcrel:
       return COFF::IMAGE_REL_AMD64_REL32;
     case FK_Data_4:
+      if (PCRel)
+        return COFF::IMAGE_REL_AMD64_REL32;
+      [[fallthrough]];
     case X86::reloc_signed_4byte:
     case X86::reloc_signed_4byte_relax:
       if (Spec == MCSymbolRefExpr::VK_COFF_IMGREL32)
         return COFF::IMAGE_REL_AMD64_ADDR32NB;
-      if (Spec == MCSymbolRefExpr::VK_SECREL)
+      if (Spec == X86::S_COFF_SECREL)
         return COFF::IMAGE_REL_AMD64_SECREL;
       return COFF::IMAGE_REL_AMD64_ADDR32;
     case FK_Data_8:
@@ -91,16 +97,18 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
     }
   } else if (getMachine() == COFF::IMAGE_FILE_MACHINE_I386) {
     switch (FixupKind) {
-    case FK_PCRel_4:
     case X86::reloc_riprel_4byte:
     case X86::reloc_riprel_4byte_movq_load:
       return COFF::IMAGE_REL_I386_REL32;
     case FK_Data_4:
+      if (PCRel)
+        return COFF::IMAGE_REL_I386_REL32;
+      [[fallthrough]];
     case X86::reloc_signed_4byte:
     case X86::reloc_signed_4byte_relax:
       if (Spec == MCSymbolRefExpr::VK_COFF_IMGREL32)
         return COFF::IMAGE_REL_I386_DIR32NB;
-      if (Spec == MCSymbolRefExpr::VK_SECREL)
+      if (Spec == X86::S_COFF_SECREL)
         return COFF::IMAGE_REL_I386_SECREL;
       return COFF::IMAGE_REL_I386_DIR32;
     case FK_SecRel_2:
diff --git a/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp b/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
index c8df3c29bb046..550050e50b43a 100644
--- a/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
+++ b/llvm/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "TargetInfo/X86TargetInfo.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/MC/TargetRegistry.h"
 using namespace llvm;
 
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index c7238839c26b5..50c20fcde49ce 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -21,6 +21,7 @@
 #include "X86InstrInfo.h"
 #include "X86MachineFunctionInfo.h"
 #include "X86Subtarget.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/Analysis/StaticDataProfileInfo.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -1002,11 +1003,11 @@ static bool usesMSVCFloatingPoint(const Triple &TT, const Module &M) {
 
   for (const Function &F : M) {
     for (const Instruction &I : instructions(F)) {
-      if (I.getType()->isFPOrFPVectorTy())
+      if (I.getType()->isFloatingPointTy())
         return true;
 
       for (const auto &Op : I.operands()) {
-        if (Op->getType()->isFPOrFPVectorTy())
+        if (Op->getType()->isFloatingPointTy())
           return true;
       }
     }
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 307a237e2955c..8c6de28398ce9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -19206,7 +19206,7 @@ SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
   // global base reg.
   unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
 
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
   SDLoc DL(JT);
   Result =
@@ -19234,7 +19234,7 @@ X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
   int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
   SDLoc dl(Op);
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
   Result =
       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result);
@@ -19277,7 +19277,7 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
   bool NeedsLoad = isGlobalStubReference(OpFlags);
 
   CodeModel::Model M = DAG.getTarget().getCodeModel();
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   SDValue Result;
 
   if (GV) {
@@ -19536,7 +19536,7 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
     return LowerToTLSEmulatedModel(GA, DAG);
 
   const GlobalValue *GV = GA->getGlobal();
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   bool PositionIndependent = isPositionIndependent();
 
   if (Subtarget.isTargetELF()) {
@@ -25782,7 +25782,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
 
   bool Is64Bit = Subtarget.is64Bit();
-  MVT SPTy = getPointerTy(DAG.getDataLayout());
+  MVT SPTy = Op.getValueType().getSimpleVT();
 
   SDValue Result;
   if (!Lower) {
@@ -25850,7 +25850,9 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
 
 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
-  auto PtrVT = getPointerTy(MF.getDataLayout());
+  SDValue Ptr = Op.getOperand(1);
+  EVT PtrVT = Ptr.getValueType();
+
   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
 
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
@@ -25861,8 +25863,7 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
     // vastart just stores the address of the VarArgsFrameIndex slot into the
     // memory location argument.
     SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
-    return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
-                        MachinePointerInfo(SV));
+    return DAG.getStore(Op.getOperand(0), DL, FR, Ptr, MachinePointerInfo(SV));
   }
 
   // __va_list_tag:
@@ -25951,7 +25952,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
                        DAG.getTargetConstant(ArgSize, dl, MVT::i32),
                        DAG.getTargetConstant(ArgMode, dl, MVT::i8),
                        DAG.getTargetConstant(Align, dl, MVT::i32)};
-  SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
+  SDVTList VTs = DAG.getVTList(SrcPtr.getValueType(), MVT::Other);
   SDValue VAARG = DAG.getMemIntrinsicNode(
       Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl,
       VTs, InstOps, MVT::i64, MachinePointerInfo(SV),
@@ -26272,9 +26273,6 @@ static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
   MachineFunction &MF = DAG.getMachineFunction();
   SDLoc dl;
 
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
-
   // It's possible that the parent function no longer has a personality function
   // if the exceptional code was optimized away, in which case we just return
   // the incoming EBP.
@@ -26285,6 +26283,7 @@ static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
   // registration, or the .set_setframe offset.
   MCSymbol *OffsetSym = MF.getContext().getOrCreateParentFrameOffsetSymbol(
       GlobalValue::dropLLVMManglingEscape(Fn->getName()));
+  MVT PtrVT = EntryEBP.getValueType().getSimpleVT();
   SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
   SDValue ParentFrameOffset =
       DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
@@ -27345,7 +27344,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::thread_pointer: {
     if (Subtarget.isTargetELF()) {
       SDLoc dl(Op);
-      EVT PtrVT = getPointerTy(DAG.getDataLayout());
+      EVT PtrVT = Op.getValueType();
       // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
       Value *Ptr = Constant::getNullValue(PointerType::get(
           *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS));
@@ -28212,12 +28211,9 @@ SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   unsigned Depth = Op.getConstantOperandVal(0);
   SDLoc dl(Op);
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
 
   if (Depth > 0) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
@@ -29423,16 +29419,22 @@ static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
                    Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN);
 
   // If we did no ordering operands for signed zero handling and we need
-  // to process NaN and we know that the second operand is not NaN then put
-  // it in first operand and we will not need to post handle NaN after max/min.
-  if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
+  // to process NaN and we know that one of the operands is not NaN then:
+  //  - For minimum/maximum, put it in the first operand,
+  //  - For minimumnum/maximumnum, put it in the second operand,
+  // and we will not need to post handle NaN after max/min.
+  if (IgnoreSignedZero && !IgnoreNaN &&
+      DAG.isKnownNeverNaN(IsNum ? NewX : NewY))
     std::swap(NewX, NewY);
 
   SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
 
-  if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
+  if (IgnoreNaN || DAG.isKnownNeverNaN(IsNum ? NewY : NewX))
     return MinMax;
 
+  if (DAG.isKnownNeverNaN(NewX))
+    NewX = NewY;
+
   SDValue IsNaN =
       DAG.getSetCC(DL, SetCCType, NewX, NewX, IsNum ? ISD::SETO : ISD::SETUO);
 
@@ -34151,7 +34153,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     }
 
-    if (VT.isVector() && Subtarget.hasFP16() &&
+    if (VT.isVector() && Subtarget.hasFP16() && Subtarget.hasVLX() &&
         SrcVT.getVectorElementType() == MVT::f16) {
       EVT EleVT = VT.getVectorElementType();
       EVT ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
@@ -34165,10 +34167,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       }
 
       if (IsStrict) {
+        Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
         Res =
             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
         Chain = Res.getValue(1);
       } else {
+        Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
         Res = DAG.getNode(Opc, dl, ResVT, Src);
       }
 
@@ -37613,6 +37617,37 @@ X86TargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
       BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg);
       break;
     }
+    case MachineJumpTableInfo::EK_CoffImgRel32: {
+      Register ImageBaseReg = MRI->createVirtualRegister(&X86::GR64RegClass);
+      Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass);
+      Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass);
+
+      // movl (BReg,IReg64,4), OReg
+      // This implicitly zero-extends from uint32 to uint64.
+      BuildMI(DispContBB, MIMD, TII->get(X86::MOV32rm), OReg64)
+          .addReg(BReg)
+          .addImm(4)
+          .addReg(IReg64)
+          .addImm(0)
+          .addReg(0);
+
+      // leaq (__ImageBase,OReg64), ImageBaseReg
+      BuildMI(DispContBB, MIMD, TII->get(X86::LEA64r), ImageBaseReg)
+          .addReg(X86::RIP)
+          .addImm(0)
+          .addReg(0)
+          .addExternalSymbol("__ImageBase", X86II::MO_COFF_IMGREL32)
+          .addReg(0);
+
+      // addq ImageBaseReg, OReg64
+      BuildMI(DispContBB, MIMD, TII->get(X86::ADD64rr), TReg)
+          .addReg(ImageBaseReg)
+          .addReg(OReg64);
+
+      // jmpq *TReg
+      BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg);
+      break;
+    }
     default:
       llvm_unreachable("Unexpected jump table encoding");
     }
@@ -45043,7 +45078,10 @@ bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
   unsigned NumElts = DemandedElts.getBitWidth();
 
   switch (Op.getOpcode()) {
+  case X86ISD::BLENDI:
   case X86ISD::PSHUFD:
+  case X86ISD::UNPCKL:
+  case X86ISD::UNPCKH:
   case X86ISD::VPERMILPI:
   case X86ISD::VPERMV3: {
     SmallVector<int, 8> Mask;
@@ -45087,15 +45125,22 @@ bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
   case X86ISD::VSRLI:
   case X86ISD::VSRAI:
     return false;
+    // SSE blends.
+  case X86ISD::BLENDI:
+  case X86ISD::BLENDV:
+    return false;
+    // SSE target shuffles.
   case X86ISD::PSHUFD:
+  case X86ISD::UNPCKL:
+  case X86ISD::UNPCKH:
   case X86ISD::VPERMILPI:
   case X86ISD::VPERMV3:
-  case X86ISD::UNPCKH:
-  case X86ISD::UNPCKL:
     return false;
-    // SSE comparisons handle all fcmp cases.
-    // TODO: Add PCMPEQ/GT and CMPM/MM with test coverage.
+    // SSE comparisons handle all icmp/fcmp cases.
+    // TODO: Add CMPM/MM with test coverage.
   case X86ISD::CMPP:
+  case X86ISD::PCMPEQ:
+  case X86ISD::PCMPGT:
     return false;
   case ISD::INTRINSIC_WO_CHAIN:
     switch (Op->getConstantOperandVal(0)) {
@@ -45989,21 +46034,6 @@ static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0,
   return false;
 }
 
-// Given a ABS node, detect the following pattern:
-// (ABS (SUB (ZERO_EXTEND a), (ZERO_EXTEND b))).
-// This is useful as it is the input into a SAD pattern.
-static bool detectZextAbsDiff(SDValue Abs, SDValue &Op0, SDValue &Op1) {
-  using namespace SDPatternMatch;
-
-  // Check if the operands of the sub are zero-extended from vectors of i8.
-  EVT SrcVT0, SrcVT1;
-  return sd_match(Abs,
-                  m_Abs(m_Sub(m_AllOf(m_Value(Op0), m_ZExt(m_VT(SrcVT0))),
-                              m_AllOf(m_Value(Op1), m_ZExt(m_VT(SrcVT1)))))) &&
-         SrcVT0.getVectorElementType() == MVT::i8 &&
-         SrcVT1.getVectorElementType() == MVT::i8;
-}
-
 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
                               unsigned &LogBias, const SDLoc &DL,
                               const X86Subtarget &Subtarget) {
@@ -46366,6 +46396,8 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
 
 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
                                       const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
+
   // PSADBW is only supported on SSE2 and up.
   if (!Subtarget.hasSSE2())
     return SDValue();
@@ -46386,8 +46418,7 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
   if (!Root)
     return SDValue();
 
-  // The operand is expected to be zero extended from i8
-  // (verified in detectZextAbsDiff).
+  // The operand is expected to be zero extended from i8.
   // In order to convert to i64 and above, additional any/zero/sign
   // extend is expected.
   // The zero extend from 32 bit has no mathematical effect on the result.
@@ -46399,9 +46430,15 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
       Root.getOpcode() == ISD::ANY_EXTEND)
     Root = Root.getOperand(0);
 
-  // Check whether we have an abs-diff pattern feeding into the select.
+  // Check whether we have an abdu pattern.
+  // TODO: Add handling for ISD::ABDU.
   SDValue Zext0, Zext1;
-  if (!detectZextAbsDiff(Root, Zext0, Zext1))
+  if (!sd_match(
+          Root,
+          m_Abs(m_Sub(m_AllOf(m_Value(Zext0),
+                              m_ZExt(m_SpecificVectorElementVT(MVT::i8))),
+                      m_AllOf(m_Value(Zext1),
+                              m_ZExt(m_SpecificVectorElementVT(MVT::i8)))))))
     return SDValue();
 
   // Create the SAD instruction.
@@ -47254,111 +47291,6 @@ static SDValue combineToExtendBoolVectorInReg(
                      DAG.getConstant(EltSizeInBits - 1, DL, VT));
 }
 
-/// If a vector select has an operand that is -1 or 0, try to simplify the
-/// select to a bitwise logic operation.
-/// TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
-static SDValue
-combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
-                                 TargetLowering::DAGCombinerInfo &DCI,
-                                 const X86Subtarget &Subtarget) {
-  SDValue Cond = N->getOperand(0);
-  SDValue LHS = N->getOperand(1);
-  SDValue RHS = N->getOperand(2);
-  EVT VT = LHS.getValueType();
-  EVT CondVT = Cond.getValueType();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-
-  if (N->getOpcode() != ISD::VSELECT)
-    return SDValue();
-
-  assert(CondVT.isVector() && "Vector select expects a vector selector!");
-
-  // TODO: Use isNullOrNullSplat() to distinguish constants with undefs?
-  // TODO: Can we assert that both operands are not zeros (because that should
-  //       get simplified at node creation time)?
-  bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
-  bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
-
-  // If both inputs are 0/undef, create a complete zero vector.
-  // FIXME: As noted above this should be handled by DAGCombiner/getNode.
-  if (TValIsAllZeros && FValIsAllZeros) {
-    if (VT.isFloatingPoint())
-      return DAG.getConstantFP(0.0, DL, VT);
-    return DAG.getConstant(0, DL, VT);
-  }
-
-  // To use the condition operand as a bitwise mask, it must have elements that
-  // are the same size as the select elements. Ie, the condition operand must
-  // have already been promoted from the IR select condition type <N x i1>.
-  // Don't check if the types themselves are equal because that excludes
-  // vector floating-point selects.
-  if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
-    return SDValue();
-
-  // Try to invert the condition if true value is not all 1s and false value is
-  // not all 0s. Only do this if the condition has one use.
-  bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
-  if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
-      // Check if the selector will be produced by CMPP*/PCMP*.
-      Cond.getOpcode() == ISD::SETCC &&
-      // Check if SETCC has already been promoted.
-      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
-          CondVT) {
-    bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
-
-    if (TValIsAllZeros || FValIsAllOnes) {
-      SDValue CC = Cond.getOperand(2);
-      ISD::CondCode NewCC = ISD::getSetCCInverse(
-          cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
-      Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
-                          NewCC);
-      std::swap(LHS, RHS);
-      TValIsAllOnes = FValIsAllOnes;
-      FValIsAllZeros = TValIsAllZeros;
-    }
-  }
-
-  // Cond value must be 'sign splat' to be converted to a logical op.
-  if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
-    return SDValue();
-
-  // vselect Cond, 111..., 000... -> Cond
-  if (TValIsAllOnes && FValIsAllZeros)
-    return DAG.getBitcast(VT, Cond);
-
-  if (!TLI.isTypeLegal(CondVT))
-    return SDValue();
-
-  // vselect Cond, 111..., X -> or Cond, X
-  if (TValIsAllOnes) {
-    SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
-    SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
-    return DAG.getBitcast(VT, Or);
-  }
-
-  // vselect Cond, X, 000... -> and Cond, X
-  if (FValIsAllZeros) {
-    SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
-    SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
-    return DAG.getBitcast(VT, And);
-  }
-
-  // vselect Cond, 000..., X -> andn Cond, X
-  if (TValIsAllZeros) {
-    SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
-    SDValue AndN;
-    // The canonical form differs for i1 vectors - x86andnp is not used
-    if (CondVT.getScalarType() == MVT::i1)
-      AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
-                         CastRHS);
-    else
-      AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
-    return DAG.getBitcast(VT, AndN);
-  }
-
-  return SDValue();
-}
-
 /// If both arms of a vector select are concatenated vectors, split the select,
 /// and concatenate the result to eliminate a wide (256-bit) vector instruction:
 ///   vselect Cond, (concat T0, T1), (concat F0, F1) -->
@@ -47614,31 +47546,26 @@ static SDValue combineLogicBlendIntoConditionalNegate(
 
 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
                              const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
   if (!Subtarget.hasAVX512())
     return SDValue();
-  if (N->getOpcode() != ISD::VSELECT)
-    return SDValue();
-
-  SDValue Cond = N->getOperand(0);
-  SDValue LHS = N->getOperand(1);
-  SDValue RHS = N->getOperand(2);
-
-  if (canCombineAsMaskOperation(LHS, Subtarget))
-    return SDValue();
 
-  if (!canCombineAsMaskOperation(RHS, Subtarget))
+  ISD::CondCode CC;
+  SDValue Cond, X, Y, LHS, RHS;
+  if (!sd_match(N, m_VSelect(m_AllOf(m_Value(Cond),
+                                     m_OneUse(m_SetCC(m_Value(X), m_Value(Y),
+                                                      m_CondCode(CC)))),
+                             m_Value(LHS), m_Value(RHS))))
     return SDValue();
 
-  if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
+  if (canCombineAsMaskOperation(LHS, Subtarget) ||
+      !canCombineAsMaskOperation(RHS, Subtarget))
     return SDValue();
 
   // Commute LHS and RHS to create opportunity to select mask instruction.
   // (vselect M, L, R) -> (vselect ~M, R, L)
-  ISD::CondCode NewCC =
-      ISD::getSetCCInverse(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
-                           Cond.getOperand(0).getValueType());
-  Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0),
-                      Cond.getOperand(1), NewCC);
+  ISD::CondCode NewCC = ISD::getSetCCInverse(CC, X.getValueType());
+  Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
   return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
 }
 
@@ -48105,9 +48032,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
   if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget))
     return SDValue();
 
-  if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DL, DCI, Subtarget))
-    return V;
-
   if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget))
     return V;
 
@@ -49889,31 +49813,24 @@ static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
                                    const SDLoc &DL,
                                    const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
   assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
-           "SRL or SRA node is required here!");
+         "SRL or SRA node is required here!");
 
   if (!Subtarget.hasSSE2())
     return SDValue();
 
-  // The operation feeding into the shift must be a multiply.
-  SDValue ShiftOperand = N->getOperand(0);
-  if (ShiftOperand.getOpcode() != ISD::MUL || !ShiftOperand.hasOneUse())
-    return SDValue();
-
   // Input type should be at least vXi32.
   EVT VT = N->getValueType(0);
   if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32)
     return SDValue();
 
-  // Need a shift by 16.
-  APInt ShiftAmt;
-  if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) ||
-      ShiftAmt != 16)
+  // The operation must be a multiply shifted right by 16.
+  SDValue LHS, RHS;
+  if (!sd_match(N->getOperand(1), m_SpecificInt(16)) ||
+      !sd_match(N->getOperand(0), m_OneUse(m_Mul(m_Value(LHS), m_Value(RHS)))))
     return SDValue();
 
-  SDValue LHS = ShiftOperand.getOperand(0);
-  SDValue RHS = ShiftOperand.getOperand(1);
-
   unsigned ExtOpc = LHS.getOpcode();
   if ((ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND) ||
       RHS.getOpcode() != ExtOpc)
@@ -52647,7 +52564,8 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
 ///   XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
 /// into:
 ///   SETGT(X, -1)
-static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
+static SDValue foldXorTruncShiftIntoCmp(SDNode *N, const SDLoc &DL,
+                                        SelectionDAG &DAG) {
   // This is only worth doing if the output type is i8 or i1.
   EVT ResultType = N->getValueType(0);
   if (ResultType != MVT::i8 && ResultType != MVT::i1)
@@ -52682,7 +52600,6 @@ static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
   // Create a greater-than comparison against -1.
   // N.B. Using SETGE against 0 works but we want a canonical looking
   // comparison, using SETGT matches up with what TranslateX86CC.
-  SDLoc DL(N);
   SDValue ShiftOp = Shift.getOperand(0);
   EVT ShiftOpTy = ShiftOp.getValueType();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -54852,11 +54769,7 @@ static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   SDLoc dl(N);
-
-  unsigned IntBits = VT.getScalarSizeInBits();
-  MVT IntSVT = MVT::getIntegerVT(IntBits);
-  MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);
-
+  MVT IntVT = VT.changeVectorElementTypeToInteger();
   SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
   SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
   unsigned IntOpcode;
@@ -54873,9 +54786,8 @@ static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
   return DAG.getBitcast(VT, IntOp);
 }
 
-
 /// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
-static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
+static SDValue foldXor1SetCC(SDNode *N, const SDLoc &DL, SelectionDAG &DAG) {
   if (N->getOpcode() != ISD::XOR)
     return SDValue();
 
@@ -54885,7 +54797,6 @@ static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
 
   X86::CondCode NewCC = X86::GetOppositeBranchCondition(
       X86::CondCode(LHS->getConstantOperandVal(0)));
-  SDLoc DL(N);
   return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
 }
 
@@ -54983,13 +54894,13 @@ static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  if (SDValue SetCC = foldXor1SetCC(N, DAG))
+  if (SDValue SetCC = foldXor1SetCC(N, DL, DAG))
     return SetCC;
 
   if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), DL, VT, N0, N1, DAG))
     return R;
 
-  if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
+  if (SDValue RV = foldXorTruncShiftIntoCmp(N, DL, DAG))
     return RV;
 
   // Fold not(iX bitcast(vXi1)) -> (iX bitcast(not(vec))) for legal boolvecs.
@@ -57498,7 +57409,7 @@ static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
   bool IsSub = X86ISD::SUB == N->getOpcode();
   unsigned GenericOpc = IsSub ? ISD::SUB : ISD::ADD;
 
-  if (IsSub && isOneConstant(N->getOperand(1)) && !N->hasAnyUseOfValue(0))
+  if (IsSub && isOneConstant(RHS) && !N->hasAnyUseOfValue(0))
     if (SDValue CMP = combineX86SubCmpForFlags(N, SDValue(N, 1), DAG, DCI, ST))
       return CMP;
 
@@ -61735,6 +61646,9 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           return std::make_pair(0U, &X86::VR128XRegClass);
         return std::make_pair(0U, &X86::VR128RegClass);
       case MVT::f128:
+        if (!Subtarget.is64Bit())
+          break;
+        [[fallthrough]];
       case MVT::v16i8:
       case MVT::v8i16:
       case MVT::v4i32:
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 1aa00d4f09f75..9c2786ae75ddb 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "X86.h"
 #include "X86CallingConv.h"
 #include "X86FrameLowering.h"
@@ -420,6 +420,11 @@ bool X86TargetLowering::allowsMemoryAccess(LLVMContext &Context,
 /// current function.  The returned value is a member of the
 /// MachineJumpTableInfo::JTEntryKind enum.
 unsigned X86TargetLowering::getJumpTableEncoding() const {
+  // Always use EK_CoffImgRel32 for 64-bit Windows targets.
+  if (Subtarget.isTargetWin64()) {
+    return MachineJumpTableInfo::EK_CoffImgRel32;
+  }
+
   // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
   // symbol.
   if (isPositionIndependent() && Subtarget.isPICStyleGOT())
@@ -472,7 +477,7 @@ X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
   assert(isPositionIndependent() && Subtarget.isPICStyleGOT());
   // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
   // entries.
-  return MCSymbolRefExpr::create(MBB->getSymbol(), X86MCExpr::VK_GOTOFF, Ctx);
+  return MCSymbolRefExpr::create(MBB->getSymbol(), X86::S_GOTOFF, Ctx);
 }
 
 /// Returns relocation base for the given PIC jumptable.
@@ -1929,9 +1934,9 @@ SDValue X86TargetLowering::LowerFormalArguments(
   }
 
   if (CallingConv::PreserveNone == CallConv)
-    for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
-      if (Ins[I].Flags.isSwiftSelf() || Ins[I].Flags.isSwiftAsync() ||
-          Ins[I].Flags.isSwiftError()) {
+    for (const ISD::InputArg &In : Ins) {
+      if (In.Flags.isSwiftSelf() || In.Flags.isSwiftAsync() ||
+          In.Flags.isSwiftError()) {
         errorUnsupported(DAG, dl,
                          "Swift attributes can't be used with preserve_none");
         break;
@@ -2421,9 +2426,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and glue operands which copy the outgoing args into registers.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -2462,9 +2466,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const uint32_t *Mask = [&]() {
@@ -2615,9 +2618,9 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   if (CallingConv::PreserveNone == CallConv)
-    for (unsigned I = 0, E = Outs.size(); I != E; ++I) {
-      if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftAsync() ||
-          Outs[I].Flags.isSwiftError()) {
+    for (const ISD::OutputArg &Out : Outs) {
+      if (Out.Flags.isSwiftSelf() || Out.Flags.isSwiftAsync() ||
+          Out.Flags.isSwiftError()) {
         errorUnsupported(DAG, dl,
                          "Swift attributes can't be used with preserve_none");
         break;
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 55d57d15f8d42..41ae0c2d21cd6 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -15,7 +15,7 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86EncodingOptimization.h"
 #include "MCTargetDesc/X86InstComments.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86ShuffleDecode.h"
 #include "MCTargetDesc/X86TargetStreamer.h"
 #include "X86AsmPrinter.h"
@@ -234,7 +234,7 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   // FIXME: We would like an efficient form for this, so we don't have to do a
   // lot of extra uniquing.
   const MCExpr *Expr = nullptr;
-  auto Specifier = X86MCExpr::VK_None;
+  uint16_t Specifier = X86::S_None;
 
   switch (MO.getTargetFlags()) {
   default:
@@ -247,61 +247,61 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
     break;
 
   case X86II::MO_TLVP:
-    Specifier = X86MCExpr::VK_TLVP;
+    Specifier = X86::S_TLVP;
     break;
   case X86II::MO_TLVP_PIC_BASE:
-    Expr = MCSymbolRefExpr::create(Sym, X86MCExpr::VK_TLVP, Ctx);
+    Expr = MCSymbolRefExpr::create(Sym, X86::S_TLVP, Ctx);
     // Subtract the pic base.
     Expr = MCBinaryExpr::createSub(
         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
     break;
   case X86II::MO_SECREL:
-    Specifier = X86MCExpr::Specifier(MCSymbolRefExpr::VK_SECREL);
+    Specifier = uint16_t(X86::S_COFF_SECREL);
     break;
   case X86II::MO_TLSGD:
-    Specifier = X86MCExpr::VK_TLSGD;
+    Specifier = X86::S_TLSGD;
     break;
   case X86II::MO_TLSLD:
-    Specifier = X86MCExpr::VK_TLSLD;
+    Specifier = X86::S_TLSLD;
     break;
   case X86II::MO_TLSLDM:
-    Specifier = X86MCExpr::VK_TLSLDM;
+    Specifier = X86::S_TLSLDM;
     break;
   case X86II::MO_GOTTPOFF:
-    Specifier = X86MCExpr::VK_GOTTPOFF;
+    Specifier = X86::S_GOTTPOFF;
     break;
   case X86II::MO_INDNTPOFF:
-    Specifier = X86MCExpr::VK_INDNTPOFF;
+    Specifier = X86::S_INDNTPOFF;
     break;
   case X86II::MO_TPOFF:
-    Specifier = X86MCExpr::VK_TPOFF;
+    Specifier = X86::S_TPOFF;
     break;
   case X86II::MO_DTPOFF:
-    Specifier = X86MCExpr::VK_DTPOFF;
+    Specifier = X86::S_DTPOFF;
     break;
   case X86II::MO_NTPOFF:
-    Specifier = X86MCExpr::VK_NTPOFF;
+    Specifier = X86::S_NTPOFF;
     break;
   case X86II::MO_GOTNTPOFF:
-    Specifier = X86MCExpr::VK_GOTNTPOFF;
+    Specifier = X86::S_GOTNTPOFF;
     break;
   case X86II::MO_GOTPCREL:
-    Specifier = X86MCExpr::VK_GOTPCREL;
+    Specifier = X86::S_GOTPCREL;
     break;
   case X86II::MO_GOTPCREL_NORELAX:
-    Specifier = X86MCExpr::VK_GOTPCREL_NORELAX;
+    Specifier = X86::S_GOTPCREL_NORELAX;
     break;
   case X86II::MO_GOT:
-    Specifier = X86MCExpr::VK_GOT;
+    Specifier = X86::S_GOT;
     break;
   case X86II::MO_GOTOFF:
-    Specifier = X86MCExpr::VK_GOTOFF;
+    Specifier = X86::S_GOTOFF;
     break;
   case X86II::MO_PLT:
-    Specifier = X86MCExpr::VK_PLT;
+    Specifier = X86::S_PLT;
     break;
   case X86II::MO_ABS8:
-    Specifier = X86MCExpr::VK_ABS8;
+    Specifier = X86::S_ABS8;
     break;
   case X86II::MO_PIC_BASE_OFFSET:
   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
@@ -320,6 +320,9 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
       Expr = MCSymbolRefExpr::create(Label, Ctx);
     }
     break;
+  case X86II::MO_COFF_IMGREL32:
+    Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_COFF_IMGREL32, Ctx);
+    break;
   }
 
   if (!Expr)
@@ -533,23 +536,23 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();
   MCContext &Ctx = OutStreamer->getContext();
 
-  X86MCExpr::Specifier Specifier;
+  X86::Specifier Specifier;
   switch (MI.getOpcode()) {
   case X86::TLS_addr32:
   case X86::TLS_addr64:
   case X86::TLS_addrX32:
-    Specifier = X86MCExpr::VK_TLSGD;
+    Specifier = X86::S_TLSGD;
     break;
   case X86::TLS_base_addr32:
-    Specifier = X86MCExpr::VK_TLSLDM;
+    Specifier = X86::S_TLSLDM;
     break;
   case X86::TLS_base_addr64:
   case X86::TLS_base_addrX32:
-    Specifier = X86MCExpr::VK_TLSLD;
+    Specifier = X86::S_TLSLD;
     break;
   case X86::TLS_desc32:
   case X86::TLS_desc64:
-    Specifier = X86MCExpr::VK_TLSDESC;
+    Specifier = X86::S_TLSDESC;
     break;
   default:
     llvm_unreachable("unexpected opcode");
@@ -566,10 +569,10 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
                 Ctx.getTargetOptions()->X86RelaxRelocations;
 
-  if (Specifier == X86MCExpr::VK_TLSDESC) {
+  if (Specifier == X86::S_TLSDESC) {
     const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(
-        MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)),
-        X86MCExpr::VK_TLSCALL, Ctx);
+        MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), X86::S_TLSCALL,
+        Ctx);
     EmitAndCountInstruction(
         MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)
             .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX)
@@ -586,7 +589,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
             .addExpr(Expr)
             .addReg(0));
   } else if (Is64Bits) {
-    bool NeedsPadding = Specifier == X86MCExpr::VK_TLSGD;
+    bool NeedsPadding = Specifier == X86::S_TLSGD;
     if (NeedsPadding && Is64BitsLP64)
       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
@@ -605,7 +608,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
     }
     if (UseGot) {
       const MCExpr *Expr =
-          MCSymbolRefExpr::create(TlsGetAddr, X86MCExpr::VK_GOTPCREL, Ctx);
+          MCSymbolRefExpr::create(TlsGetAddr, X86::S_GOTPCREL, Ctx);
       EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
                                   .addReg(X86::RIP)
                                   .addImm(1)
@@ -613,12 +616,12 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
                                   .addExpr(Expr)
                                   .addReg(0));
     } else {
-      EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
-                                  .addExpr(MCSymbolRefExpr::create(
-                                      TlsGetAddr, X86MCExpr::VK_PLT, Ctx)));
+      EmitAndCountInstruction(
+          MCInstBuilder(X86::CALL64pcrel32)
+              .addExpr(MCSymbolRefExpr::create(TlsGetAddr, X86::S_PLT, Ctx)));
     }
   } else {
-    if (Specifier == X86MCExpr::VK_TLSGD && !UseGot) {
+    if (Specifier == X86::S_TLSGD && !UseGot) {
       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
                                   .addReg(X86::EAX)
                                   .addReg(0)
@@ -638,8 +641,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
 
     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
     if (UseGot) {
-      const MCExpr *Expr =
-          MCSymbolRefExpr::create(TlsGetAddr, X86MCExpr::VK_GOT, Ctx);
+      const MCExpr *Expr = MCSymbolRefExpr::create(TlsGetAddr, X86::S_GOT, Ctx);
       EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
                                   .addReg(X86::EBX)
                                   .addImm(1)
@@ -647,9 +649,9 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
                                   .addExpr(Expr)
                                   .addReg(0));
     } else {
-      EmitAndCountInstruction(MCInstBuilder(X86::CALLpcrel32)
-                                  .addExpr(MCSymbolRefExpr::create(
-                                      TlsGetAddr, X86MCExpr::VK_PLT, Ctx)));
+      EmitAndCountInstruction(
+          MCInstBuilder(X86::CALLpcrel32)
+              .addExpr(MCSymbolRefExpr::create(TlsGetAddr, X86::S_PLT, Ctx)));
     }
   }
 }
diff --git a/llvm/lib/Target/X86/X86PfmCounters.td b/llvm/lib/Target/X86/X86PfmCounters.td
index 20d96ffdb482e..b299633446410 100644
--- a/llvm/lib/Target/X86/X86PfmCounters.td
+++ b/llvm/lib/Target/X86/X86PfmCounters.td
@@ -254,7 +254,7 @@ def LunarLakePfmCounters : ProcPfmCounters {
   ];
   let ValidationCounters = DefaultIntelPfmValidationCounters;
 }
-def : PfmCountersBinding<"Lunarlake", LunarLakePfmCounters>;
+def : PfmCountersBinding<"lunarlake", LunarLakePfmCounters>;
 
 // AMD X86 Counters.
 defvar DefaultAMDPfmValidationCounters = [
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index f3800d360fdd9..3b4e531f25388 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -80,12 +80,12 @@ struct MIRef {
   bool operator<(const MIRef &RHS) const {
     // Comparison between different BBs happens when inserting a MIRef into set.
     // So we compare MBB first to make the insertion happy.
-    return MBB < RHS.MBB || (MBB == RHS.MBB && Pos < RHS.Pos);
+    return std::tie(MBB, Pos) < std::tie(RHS.MBB, RHS.Pos);
   }
   bool operator>(const MIRef &RHS) const {
     // Comparison between different BBs happens when inserting a MIRef into set.
     // So we compare MBB first to make the insertion happy.
-    return MBB > RHS.MBB || (MBB == RHS.MBB && Pos > RHS.Pos);
+    return std::tie(MBB, Pos) > std::tie(RHS.MBB, RHS.Pos);
   }
 };
 
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 9adac067aace8..83b11eede829e 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -562,7 +562,7 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     Reserved.set(SubReg);
 
   // Set the frame-pointer register and its aliases as reserved if needed.
-  if (TFI->hasFP(MF)) {
+  if (TFI->hasFP(MF) || MF.getTarget().Options.FramePointerIsReserved(MF)) {
     if (MF.getInfo<X86MachineFunctionInfo>()->getFPClobberedByInvoke())
       MF.getContext().reportError(
           SMLoc(),
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 5770f3e39c40e..85cc5b43d40bf 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -19,6 +19,7 @@
 #include "X86Subtarget.h"
 #include "X86TargetObjectFile.h"
 #include "X86TargetTransformInfo.h"
+#include "llvm-c/Visibility.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
diff --git a/llvm/lib/Target/X86/X86TargetObjectFile.cpp b/llvm/lib/Target/X86/X86TargetObjectFile.cpp
index 4be0111a21955..fc3c06fcc1c9b 100644
--- a/llvm/lib/Target/X86/X86TargetObjectFile.cpp
+++ b/llvm/lib/Target/X86/X86TargetObjectFile.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86TargetObjectFile.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCValue.h"
@@ -25,7 +25,7 @@ const MCExpr *X86_64MachoTargetObjectFile::getTTypeGlobalReference(
   if ((Encoding & DW_EH_PE_indirect) && (Encoding & DW_EH_PE_pcrel)) {
     const MCSymbol *Sym = TM.getSymbol(GV);
     const MCExpr *Res =
-        MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, getContext());
+        MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, getContext());
     const MCExpr *Four = MCConstantExpr::create(4, getContext());
     return MCBinaryExpr::createAdd(Res, Four, getContext());
   }
@@ -48,18 +48,18 @@ const MCExpr *X86_64MachoTargetObjectFile::getIndirectSymViaGOTPCRel(
   // foo@GOTPCREL+4+<offset>.
   unsigned FinalOff = Offset+MV.getConstant()+4;
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, getContext());
+      MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, getContext());
   const MCExpr *Off = MCConstantExpr::create(FinalOff, getContext());
   return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
 
 X86ELFTargetObjectFile::X86ELFTargetObjectFile() {
-  PLTRelativeSpecifier = X86MCExpr::VK_PLT;
+  PLTRelativeSpecifier = X86::S_PLT;
 }
 
 const MCExpr *X86ELFTargetObjectFile::getDebugThreadLocalSymbol(
     const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::create(Sym, X86MCExpr::VK_DTPOFF, getContext());
+  return MCSymbolRefExpr::create(Sym, X86::S_DTPOFF, getContext());
 }
 
 const MCExpr *X86_64ELFTargetObjectFile::getIndirectSymViaGOTPCRel(
@@ -67,7 +67,7 @@ const MCExpr *X86_64ELFTargetObjectFile::getIndirectSymViaGOTPCRel(
     int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const {
   int64_t FinalOffset = Offset + MV.getConstant();
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, getContext());
+      MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, getContext());
   const MCExpr *Off = MCConstantExpr::create(FinalOffset, getContext());
   return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
diff --git a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
index 707c4a7908728..dfd138b99ce0d 100644
--- a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
+++ b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
@@ -62,7 +62,7 @@ static void printExpr(const MCExpr *Expr, const MCAsmInfo *MAI,
     SRE = dyn_cast<MCSymbolRefExpr>(Expr);
     assert(SRE && "Unexpected MCExpr type.");
   }
-  assert(SRE->getKind() == MCSymbolRefExpr::VK_None);
+  assert(SRE->getSpecifier() == 0);
 
   SRE->getSymbol().print(OS, MAI);
 
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index 1c6e294597c34..ef4cfcd69415d 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1064,9 +1064,8 @@ SDValue XCoreTargetLowering::LowerCCCCallTo(
   // The InGlue in necessary since all emitted instructions must be
   // stuck together.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -1089,9 +1088,8 @@ SDValue XCoreTargetLowering::LowerCCCCallTo(
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   if (InGlue.getNode())
     Ops.push_back(InGlue);
diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
index 019eea8e435bb..4001b2cfa2a6d 100644
--- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
+++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
@@ -302,10 +302,10 @@ struct XtensaOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case Register:
       OS << "<register x";
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
index deefb22e47a01..671f1d04daf23 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
@@ -12,10 +12,10 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -34,13 +34,11 @@ class XtensaAsmBackend : public MCAsmBackend {
         IsLittleEndian(isLE) {}
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
-  bool mayNeedRelaxation(const MCInst &Inst,
-                         const MCSubtargetInfo &STI) const override;
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override;
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
 
@@ -53,17 +51,14 @@ class XtensaAsmBackend : public MCAsmBackend {
 MCFixupKindInfo XtensaAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[Xtensa::NumTargetFixupKinds] = {
       // name                     offset bits  flags
-      {"fixup_xtensa_branch_6", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_branch_8", 16, 8, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_branch_12", 12, 12, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_jump_18", 6, 18, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_call_18", 6, 18,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_xtensa_l32r_16", 8, 16,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_xtensa_loop_8", 16, 8, MCFixupKindInfo::FKF_IsPCRel}};
+      {"fixup_xtensa_branch_6", 0, 16, 0},
+      {"fixup_xtensa_branch_8", 16, 8, 0},
+      {"fixup_xtensa_branch_12", 12, 12, 0},
+      {"fixup_xtensa_jump_18", 6, 18, 0},
+      {"fixup_xtensa_call_18", 6, 18, 0},
+      {"fixup_xtensa_l32r_16", 8, 16, 0},
+      {"fixup_xtensa_loop_8", 16, 8, 0},
+  };
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -136,17 +131,32 @@ static unsigned getSize(unsigned Kind) {
   switch (Kind) {
   default:
     return 3;
-  case MCFixupKind::FK_Data_4:
+  case FK_Data_4:
     return 4;
   case Xtensa::fixup_xtensa_branch_6:
     return 2;
   }
 }
 
-void XtensaAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+std::optional<bool> XtensaAsmBackend::evaluateFixup(const MCFragment &F,
+                                                    MCFixup &Fixup, MCValue &,
+                                                    uint64_t &Value) {
+  // For a few PC-relative fixups, offsets need to be aligned down. We
+  // compensate here because the default handler's `Value` decrement doesn't
+  // account for this alignment.
+  switch (Fixup.getTargetKind()) {
+  case Xtensa::fixup_xtensa_call_18:
+  case Xtensa::fixup_xtensa_l32r_16:
+    Value = (Asm->getFragmentOffset(F) + Fixup.getOffset()) % 4;
+  }
+  return {};
+}
+
+void XtensaAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                   const MCValue &Target,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCContext &Ctx = getContext();
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
 
@@ -166,14 +176,6 @@ void XtensaAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
   }
 }
 
-bool XtensaAsmBackend::mayNeedRelaxation(const MCInst &Inst,
-                                         const MCSubtargetInfo &STI) const {
-  return false;
-}
-
-void XtensaAsmBackend::relaxInstruction(MCInst &Inst,
-                                        const MCSubtargetInfo &STI) const {}
-
 bool XtensaAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
                                     const MCSubtargetInfo *STI) const {
   uint64_t NumNops24b = Count / 3;
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp
index f2c391b03bd4e..e2b1c5683d50b 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp
@@ -8,6 +8,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/XtensaMCAsmInfo.h"
 #include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -45,10 +46,12 @@ XtensaObjectWriter::~XtensaObjectWriter() {}
 unsigned XtensaObjectWriter::getRelocType(const MCFixup &Fixup,
                                           const MCValue &Target,
                                           bool IsPCRel) const {
+  uint8_t Specifier = Target.getSpecifier();
 
   switch ((unsigned)Fixup.getKind()) {
   case FK_Data_4:
-    return ELF::R_XTENSA_32;
+    return Specifier == Xtensa::S_TPOFF ? ELF::R_XTENSA_TLS_TPOFF
+                                        : ELF::R_XTENSA_32;
   default:
     return ELF::R_XTENSA_SLOT0_OP;
   }
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
index 02bac7a05982b..bd4d4ebd2a729 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
@@ -150,6 +150,22 @@ MCCodeEmitter *llvm::createXtensaMCCodeEmitter(const MCInstrInfo &MCII,
   return new XtensaMCCodeEmitter(MCII, Ctx, true);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case Xtensa::fixup_xtensa_branch_6:
+  case Xtensa::fixup_xtensa_branch_8:
+  case Xtensa::fixup_xtensa_branch_12:
+  case Xtensa::fixup_xtensa_jump_18:
+  case Xtensa::fixup_xtensa_call_18:
+  case Xtensa::fixup_xtensa_l32r_16:
+  case Xtensa::fixup_xtensa_loop_8:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void XtensaMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                             SmallVectorImpl<char> &CB,
                                             SmallVectorImpl<MCFixup> &Fixups,
@@ -195,8 +211,7 @@ XtensaMCCodeEmitter::getJumpTargetEncoding(const MCInst &MI, unsigned int OpNum,
     return MO.getImm();
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(Xtensa::fixup_xtensa_jump_18), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_jump_18);
   return 0;
 }
 
@@ -213,17 +228,14 @@ uint32_t XtensaMCCodeEmitter::getBranchTargetEncoding(
   case Xtensa::BGEZ:
   case Xtensa::BLTZ:
   case Xtensa::BNEZ:
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Xtensa::fixup_xtensa_branch_12), MI.getLoc()));
+    addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_branch_12);
     return 0;
   case Xtensa::BEQZ_N:
   case Xtensa::BNEZ_N:
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Xtensa::fixup_xtensa_branch_6), MI.getLoc()));
+    addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_branch_6);
     return 0;
   default:
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Xtensa::fixup_xtensa_branch_8), MI.getLoc()));
+    addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_branch_8);
     return 0;
   }
 }
@@ -240,8 +252,7 @@ XtensaMCCodeEmitter::getLoopTargetEncoding(const MCInst &MI, unsigned int OpNum,
 
   const MCExpr *Expr = MO.getExpr();
 
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(Xtensa::fixup_xtensa_loop_8), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_loop_8);
   return 0;
 }
 
@@ -261,8 +272,7 @@ XtensaMCCodeEmitter::getCallEncoding(const MCInst &MI, unsigned int OpNum,
 
   assert((MO.isExpr()) && "Unexpected operand value!");
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(Xtensa::fixup_xtensa_call_18), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_call_18);
   return 0;
 }
 
@@ -281,8 +291,7 @@ XtensaMCCodeEmitter::getL32RTargetEncoding(const MCInst &MI, unsigned OpNum,
 
   assert((MO.isExpr()) && "Unexpected operand value!");
 
-  Fixups.push_back(MCFixup::create(
-      0, MO.getExpr(), MCFixupKind(Xtensa::fixup_xtensa_l32r_16), MI.getLoc()));
+  addFixup(Fixups, 0, MO.getExpr(), Xtensa::fixup_xtensa_l32r_16);
   return 0;
 }
 
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
index 9a55635674d72..821cba0fc25c2 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
@@ -186,11 +186,17 @@ bool Xtensa::checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits,
     return FeatureBits[Xtensa::FeatureMiscSR];
   case Xtensa::PRID:
     return RAType == Xtensa::REGISTER_READ && FeatureBits[Xtensa::FeaturePRID];
+  case Xtensa::THREADPTR:
+    return FeatureBits[FeatureTHREADPTR];
   case Xtensa::VECBASE:
     return FeatureBits[Xtensa::FeatureRelocatableVector];
   case Xtensa::FCR:
   case Xtensa::FSR:
     return FeatureBits[FeatureSingleFloat];
+  case Xtensa::F64R_LO:
+  case Xtensa::F64R_HI:
+  case Xtensa::F64S:
+    return FeatureBits[FeatureDFPAccel];
   case Xtensa::WINDOWBASE:
   case Xtensa::WINDOWSTART:
     return FeatureBits[Xtensa::FeatureWindowed];
@@ -203,12 +209,23 @@ bool Xtensa::checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits,
 
 // Get Xtensa User Register by encoding value.
 MCRegister Xtensa::getUserRegister(unsigned Code, const MCRegisterInfo &MRI) {
+  MCRegister UserReg = Xtensa::NoRegister;
+
   if (MRI.getEncodingValue(Xtensa::FCR) == Code) {
-    return Xtensa::FCR;
+    UserReg = Xtensa::FCR;
   } else if (MRI.getEncodingValue(Xtensa::FSR) == Code) {
-    return Xtensa::FSR;
+    UserReg = Xtensa::FSR;
+  } else if (MRI.getEncodingValue(Xtensa::F64R_LO) == Code) {
+    UserReg = Xtensa::F64R_LO;
+  } else if (MRI.getEncodingValue(Xtensa::F64R_HI) == Code) {
+    UserReg = Xtensa::F64R_HI;
+  } else if (MRI.getEncodingValue(Xtensa::F64S) == Code) {
+    UserReg = Xtensa::F64S;
+  } else if (MRI.getEncodingValue(Xtensa::THREADPTR) == Code) {
+    UserReg = Xtensa::THREADPTR;
   }
-  return Xtensa::NoRegister;
+
+  return UserReg;
 }
 
 static MCAsmInfo *createXtensaMCAsmInfo(const MCRegisterInfo &MRI,
diff --git a/llvm/lib/Target/Xtensa/XtensaFeatures.td b/llvm/lib/Target/Xtensa/XtensaFeatures.td
index d48f494388a23..97d5472f3e96c 100644
--- a/llvm/lib/Target/Xtensa/XtensaFeatures.td
+++ b/llvm/lib/Target/Xtensa/XtensaFeatures.td
@@ -98,6 +98,11 @@ def FeatureDataCache : SubtargetFeature<"dcache", "HasDataCache", "true",
 def HasDataCache : Predicate<"Subtarget->hasDataCache()">,
                    AssemblerPredicate<(all_of FeatureDataCache)>;
 
+def FeatureTHREADPTR : SubtargetFeature<"threadptr", "HasTHREADPTR", "true",
+                                        "Enable Xtensa THREADPTR option">;
+def HasTHREADPTR : Predicate<"Subtarget->hasTHREADPTR()">,
+                   AssemblerPredicate<(all_of FeatureTHREADPTR)>;
+
 // Xtensa Interrupts Options.
 def FeatureHighPriInterrupts : SubtargetFeature<"highpriinterrupts",
                                                 "HasHighPriInterrupts", "true",
@@ -137,3 +142,8 @@ def FeatureCoprocessor : SubtargetFeature<"coprocessor", "HasCoprocessor", "true
                                           "Enable Xtensa Coprocessor option">;
 def HasCoprocessor : Predicate<"Subtarget->hasCoprocessor()">,
                                 AssemblerPredicate<(all_of FeatureCoprocessor)>;
+
+def FeatureDFPAccel : SubtargetFeature<"dfpaccel", "HasDFPAccel", "true",
+                                       "Enable Xtensa Double Precision FP acceleration">;
+def HasDFPAccel : Predicate<"Subtarget->hasDFPAccel()">,
+                  AssemblerPredicate<(all_of FeatureDFPAccel)>;
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
index d51c573282da7..fd42fd2e010ba 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
@@ -101,6 +101,7 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::ConstantPool, PtrVT, Custom);
   setOperationAction(ISD::GlobalAddress, PtrVT, Custom);
+  setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
   setOperationAction(ISD::BlockAddress, PtrVT, Custom);
   setOperationAction(ISD::JumpTable, PtrVT, Custom);
 
@@ -110,12 +111,18 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::BR_CC, MVT::i32, Legal);
   setOperationAction(ISD::BR_CC, MVT::i64, Expand);
-  setOperationAction(ISD::BR_CC, MVT::f32, Expand);
 
   setOperationAction(ISD::SELECT, MVT::i32, Expand);
   setOperationAction(ISD::SELECT, MVT::f32, Expand);
   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
-  setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
+
+  if (Subtarget.hasSingleFloat()) {
+    setOperationAction(ISD::BR_CC, MVT::f32, Legal);
+    setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
+  } else {
+    setOperationAction(ISD::BR_CC, MVT::f32, Expand);
+    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
+  }
 
   setOperationAction(ISD::SETCC, MVT::i32, Expand);
   setOperationAction(ISD::SETCC, MVT::f32, Expand);
@@ -841,21 +848,68 @@ static unsigned getBranchOpcode(ISD::CondCode Cond) {
   }
 }
 
+static std::pair<unsigned, unsigned> getFPBranchKind(ISD::CondCode Cond) {
+  switch (Cond) {
+  case ISD::SETUNE:
+    return std::make_pair(Xtensa::BF, Xtensa::OEQ_S);
+  case ISD::SETUO:
+    return std::make_pair(Xtensa::BT, Xtensa::UN_S);
+  case ISD::SETO:
+    return std::make_pair(Xtensa::BF, Xtensa::UN_S);
+  case ISD::SETUEQ:
+    return std::make_pair(Xtensa::BT, Xtensa::UEQ_S);
+  case ISD::SETULE:
+    return std::make_pair(Xtensa::BT, Xtensa::ULE_S);
+  case ISD::SETULT:
+    return std::make_pair(Xtensa::BT, Xtensa::ULT_S);
+  case ISD::SETEQ:
+  case ISD::SETOEQ:
+    return std::make_pair(Xtensa::BT, Xtensa::OEQ_S);
+  case ISD::SETNE:
+    return std::make_pair(Xtensa::BF, Xtensa::OEQ_S);
+  case ISD::SETLE:
+  case ISD::SETOLE:
+    return std::make_pair(Xtensa::BT, Xtensa::OLE_S);
+  case ISD::SETLT:
+  case ISD::SETOLT:
+    return std::make_pair(Xtensa::BT, Xtensa::OLT_S);
+  case ISD::SETGE:
+    return std::make_pair(Xtensa::BF, Xtensa::OLT_S);
+  case ISD::SETGT:
+    return std::make_pair(Xtensa::BF, Xtensa::OLE_S);
+  default:
+    llvm_unreachable("Invalid condition!");
+  }
+}
+
 SDValue XtensaTargetLowering::LowerSELECT_CC(SDValue Op,
                                              SelectionDAG &DAG) const {
   SDLoc DL(Op);
-  EVT Ty = Op.getOperand(0).getValueType();
+  EVT Ty = Op.getValueType();
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
   SDValue TrueValue = Op.getOperand(2);
   SDValue FalseValue = Op.getOperand(3);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op->getOperand(4))->get();
 
-  unsigned BrOpcode = getBranchOpcode(CC);
-  SDValue TargetCC = DAG.getConstant(BrOpcode, DL, MVT::i32);
+  if (LHS.getValueType() == MVT::i32) {
+    unsigned BrOpcode = getBranchOpcode(CC);
+    SDValue TargetCC = DAG.getConstant(BrOpcode, DL, MVT::i32);
 
-  return DAG.getNode(XtensaISD::SELECT_CC, DL, Ty, LHS, RHS, TrueValue,
-                     FalseValue, TargetCC);
+    SDValue Res = DAG.getNode(XtensaISD::SELECT_CC, DL, Ty, LHS, RHS, TrueValue,
+                              FalseValue, TargetCC, Op->getFlags());
+    return Res;
+  }
+  assert(LHS.getValueType() == MVT::f32 &&
+         "We expect MVT::f32 type of the LHS Operand in SELECT_CC");
+  unsigned BrOpcode;
+  unsigned CmpOpCode;
+  std::tie(BrOpcode, CmpOpCode) = getFPBranchKind(CC);
+  SDValue TargetCC = DAG.getConstant(CmpOpCode, DL, MVT::i32);
+  SDValue TargetBC = DAG.getConstant(BrOpcode, DL, MVT::i32);
+  return DAG.getNode(XtensaISD::SELECT_CC_FP, DL, Ty,
+                     {LHS, RHS, TrueValue, FalseValue, TargetCC, TargetBC},
+                     Op->getFlags());
 }
 
 SDValue XtensaTargetLowering::LowerRETURNADDR(SDValue Op,
@@ -919,6 +973,58 @@ SDValue XtensaTargetLowering::LowerGlobalAddress(SDValue Op,
   return Res;
 }
 
+SDValue XtensaTargetLowering::LowerGlobalTLSAddress(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  const GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
+  SDLoc DL(Op);
+  EVT PtrVT = Op.getValueType();
+  const GlobalValue *GV = G->getGlobal();
+
+  if (DAG.getTarget().useEmulatedTLS())
+    return LowerToTLSEmulatedModel(G, DAG);
+
+  TLSModel::Model model = getTargetMachine().getTLSModel(GV);
+
+  if (!Subtarget.hasTHREADPTR()) {
+    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+        DAG.getMachineFunction().getFunction(), "only emulated TLS supported",
+        DL.getDebugLoc()));
+    return DAG.getPOISON(Op->getValueType(0));
+  }
+
+  if (model == TLSModel::LocalExec || model == TLSModel::InitialExec) {
+    bool Priv = GV->isPrivateLinkage(GV->getLinkage());
+    MachineFunction &MF = DAG.getMachineFunction();
+    XtensaMachineFunctionInfo *XtensaFI =
+        MF.getInfo<XtensaMachineFunctionInfo>();
+    unsigned LabelId = XtensaFI->createCPLabelId();
+
+    // Create a constant pool entry for the callee address
+    XtensaConstantPoolValue *CPV = XtensaConstantPoolSymbol::Create(
+        *DAG.getContext(), GV->getName().str().c_str(), LabelId, Priv,
+        XtensaCP::TPOFF);
+
+    // Get the address of the callee into a register
+    SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
+    SDValue CPWrap = getAddrPCRel(CPAddr, DAG);
+    SDValue Addr = DAG.getLoad(
+        PtrVT, DL, DAG.getEntryNode(), CPWrap,
+        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
+
+    SDValue TPRegister = DAG.getRegister(Xtensa::THREADPTR, MVT::i32);
+    SDValue ThreadPointer =
+        DAG.getNode(XtensaISD::RUR, DL, MVT::i32, TPRegister);
+
+    return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadPointer, Addr);
+  }
+
+  DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+      DAG.getMachineFunction().getFunction(),
+      "only local-exec and initial-exec TLS mode supported", DL.getDebugLoc()));
+
+  return DAG.getPOISON(Op->getValueType(0));
+}
+
 SDValue XtensaTargetLowering::LowerBlockAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
   BlockAddressSDNode *Node = cast<BlockAddressSDNode>(Op);
@@ -1353,6 +1459,8 @@ SDValue XtensaTargetLowering::LowerOperation(SDValue Op,
     return LowerRETURNADDR(Op, DAG);
   case ISD::GlobalAddress:
     return LowerGlobalAddress(Op, DAG);
+  case ISD::GlobalTLSAddress:
+    return LowerGlobalTLSAddress(Op, DAG);
   case ISD::BlockAddress:
     return LowerBlockAddress(Op, DAG);
   case ISD::JumpTable:
@@ -1406,8 +1514,12 @@ const char *XtensaTargetLowering::getTargetNodeName(unsigned Opcode) const {
     return "XtensaISD::RET";
   case XtensaISD::RETW:
     return "XtensaISD::RETW";
+  case XtensaISD::RUR:
+    return "XtensaISD::RUR";
   case XtensaISD::SELECT_CC:
     return "XtensaISD::SELECT_CC";
+  case XtensaISD::SELECT_CC_FP:
+    return "XtensaISD::SELECT_CC_FP";
   case XtensaISD::SRCL:
     return "XtensaISD::SRCL";
   case XtensaISD::SRCR:
@@ -1450,7 +1562,6 @@ XtensaTargetLowering::emitSelectCC(MachineInstr &MI,
   MachineOperand &RHS = MI.getOperand(2);
   MachineOperand &TrueValue = MI.getOperand(3);
   MachineOperand &FalseValue = MI.getOperand(4);
-  unsigned BrKind = MI.getOperand(5).getImm();
 
   // To "insert" a SELECT_CC instruction, we actually have to insert
   // CopyMBB and SinkMBB  blocks and add branch to MBB. We build phi
@@ -1482,10 +1593,25 @@ XtensaTargetLowering::emitSelectCC(MachineInstr &MI,
   MBB->addSuccessor(CopyMBB);
   MBB->addSuccessor(SinkMBB);
 
-  BuildMI(MBB, DL, TII.get(BrKind))
-      .addReg(LHS.getReg())
-      .addReg(RHS.getReg())
-      .addMBB(SinkMBB);
+  if (MI.getOpcode() == Xtensa::SELECT_CC_FP_FP ||
+      MI.getOpcode() == Xtensa::SELECT_CC_FP_INT) {
+    unsigned CmpKind = MI.getOperand(5).getImm();
+    unsigned BrKind = MI.getOperand(6).getImm();
+    MCPhysReg BReg = Xtensa::B0;
+
+    BuildMI(MBB, DL, TII.get(CmpKind), BReg)
+        .addReg(LHS.getReg())
+        .addReg(RHS.getReg());
+    BuildMI(MBB, DL, TII.get(BrKind))
+        .addReg(BReg, RegState::Kill)
+        .addMBB(SinkMBB);
+  } else {
+    unsigned BrKind = MI.getOperand(5).getImm();
+    BuildMI(MBB, DL, TII.get(BrKind))
+        .addReg(LHS.getReg())
+        .addReg(RHS.getReg())
+        .addMBB(SinkMBB);
+  }
 
   CopyMBB->addSuccessor(SinkMBB);
 
@@ -1510,6 +1636,30 @@ MachineBasicBlock *XtensaTargetLowering::EmitInstrWithCustomInserter(
   const XtensaInstrInfo &TII = *Subtarget.getInstrInfo();
 
   switch (MI.getOpcode()) {
+  case Xtensa::BRCC_FP: {
+    MachineOperand &Cond = MI.getOperand(0);
+    MachineOperand &LHS = MI.getOperand(1);
+    MachineOperand &RHS = MI.getOperand(2);
+    MachineBasicBlock *TargetBB = MI.getOperand(3).getMBB();
+    unsigned BrKind = 0;
+    unsigned CmpKind = 0;
+    ISD::CondCode CondCode = (ISD::CondCode)Cond.getImm();
+    MCPhysReg BReg = Xtensa::B0;
+
+    std::tie(BrKind, CmpKind) = getFPBranchKind(CondCode);
+    BuildMI(*MBB, MI, DL, TII.get(CmpKind), BReg)
+        .addReg(LHS.getReg())
+        .addReg(RHS.getReg());
+    BuildMI(*MBB, MI, DL, TII.get(BrKind))
+        .addReg(BReg, RegState::Kill)
+        .addMBB(TargetBB);
+
+    MI.eraseFromParent();
+    return MBB;
+  }
+  case Xtensa::SELECT_CC_FP_FP:
+  case Xtensa::SELECT_CC_FP_INT:
+  case Xtensa::SELECT_CC_INT_FP:
   case Xtensa::SELECT:
     return emitSelectCC(MI, MBB);
   case Xtensa::S8I:
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.h b/llvm/lib/Target/Xtensa/XtensaISelLowering.h
index b6f2ebe21c940..e6ddf9864932a 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.h
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.h
@@ -45,11 +45,18 @@ enum {
   RET,
   RETW,
 
+  RUR,
+
   // Select with condition operator - This selects between a true value and
   // a false value (ops #2 and #3) based on the boolean result of comparing
   // the lhs and rhs (ops #0 and #1) of a conditional expression with the
   // condition code in op #4
   SELECT_CC,
+  // Select with condition operator - This selects between a true value and
+  // a false value (ops #2 and #3) based on the boolean result of comparing
+  // f32 operands lhs and rhs (ops #0 and #1) of a conditional expression
+  // with the condition code in op #4 and boolean branch kind in op #5
+  SELECT_CC_FP,
 
   // SRCL(R) performs shift left(right) of the concatenation of 2 registers
   // and returns high(low) 32-bit part of 64-bit result
@@ -156,6 +163,8 @@ class XtensaTargetLowering : public TargetLowering {
 
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
 
+  SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
+
   SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp b/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
index 896e2f8f1c01c..55c0729a0c9e7 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
@@ -261,6 +261,12 @@ bool XtensaInstrInfo::reverseBranchCondition(
   case Xtensa::BGEZ:
     Cond[0].setImm(Xtensa::BLTZ);
     return false;
+  case Xtensa::BF:
+    Cond[0].setImm(Xtensa::BT);
+    return false;
+  case Xtensa::BT:
+    Cond[0].setImm(Xtensa::BF);
+    return false;
   default:
     report_fatal_error("Invalid branch condition!");
   }
@@ -294,6 +300,9 @@ XtensaInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
   case Xtensa::BLTZ:
   case Xtensa::BGEZ:
     return MI.getOperand(1).getMBB();
+  case Xtensa::BT:
+  case Xtensa::BF:
+    return MI.getOperand(1).getMBB();
   default:
     llvm_unreachable("Unknown branch opcode");
   }
@@ -329,6 +338,10 @@ bool XtensaInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   case Xtensa::BGEZ:
     BrOffset -= 4;
     return isIntN(12, BrOffset);
+  case Xtensa::BT:
+  case Xtensa::BF:
+    BrOffset -= 4;
+    return isIntN(8, BrOffset);
   default:
     llvm_unreachable("Unknown branch opcode");
   }
@@ -581,6 +594,10 @@ unsigned XtensaInstrInfo::insertConstBranchAtInst(
   case Xtensa::BGEZ:
     MI = BuildMI(MBB, I, DL, get(BR_C)).addImm(offset).addReg(Cond[1].getReg());
     break;
+  case Xtensa::BT:
+  case Xtensa::BF:
+    MI = BuildMI(MBB, I, DL, get(BR_C)).addImm(offset).addReg(Cond[1].getReg());
+    break;
   default:
     llvm_unreachable("Invalid branch type!");
   }
@@ -641,6 +658,10 @@ unsigned XtensaInstrInfo::insertBranchAtInst(MachineBasicBlock &MBB,
   case Xtensa::BGEZ:
     MI = BuildMI(MBB, I, DL, get(BR_C)).addReg(Cond[1].getReg()).addMBB(TBB);
     break;
+  case Xtensa::BT:
+  case Xtensa::BF:
+    MI = BuildMI(MBB, I, DL, get(BR_C)).addReg(Cond[1].getReg()).addMBB(TBB);
+    break;
   default:
     report_fatal_error("Invalid branch type!");
   }
@@ -689,6 +710,12 @@ bool XtensaInstrInfo::isBranch(const MachineBasicBlock::iterator &MI,
     Target = &MI->getOperand(1);
     return true;
 
+  case Xtensa::BT:
+  case Xtensa::BF:
+    Cond[0].setImm(OpCode);
+    Target = &MI->getOperand(1);
+    return true;
+
   default:
     assert(!MI->getDesc().isBranch() && "Unknown branch opcode");
     return false;
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index 1335c6faff6b7..31608f4659365 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -416,7 +416,6 @@ def BBSI : RRI8_Inst<0x07, (outs),
 }
 
 def : Pat<(brcond AR:$s, bb:$target), (BNEZ AR:$s, bb:$target)>;
-
 //===----------------------------------------------------------------------===//
 // Call and jump instructions
 //===----------------------------------------------------------------------===//
@@ -576,7 +575,7 @@ def WUR : RRR_Inst<0x00, 0x03, 0x0F, (outs UR:$ur), (ins AR:$t),
 }
 
 def RUR : RRR_Inst<0x00, 0x03, 0x0E, (outs AR:$r), (ins UR:$ur),
-                  "rur\t$r, $ur", []> {
+                  "rur\t$r, $ur", [(set AR:$r, (Xtensa_rur UR:$ur))]> {
   bits<8> ur;
 
   let s = ur{7-4};
@@ -1310,6 +1309,34 @@ let AddedComplexity = 10 in
 def : Pat<(f32 (load (Xtensa_pcrel_wrapper tconstpool:$in))),
           (WFR (L32R tconstpool:$in))>;
 
+//===----------------------------------------------------------------------===//
+// SelectCC and BranchCC instructions with FP operands
+//===----------------------------------------------------------------------===//
+
+let usesCustomInserter = 1, Predicates = [HasSingleFloat] in {
+  def SELECT_CC_INT_FP : Pseudo<(outs FPR:$dst), (ins AR:$lhs, AR:$rhs, FPR:$t, FPR:$f, i32imm:$cond),
+                                "!select_cc_int_fp $dst, $lhs, $rhs, $t, $f, $cond",
+                                [(set FPR:$dst, (Xtensa_select_cc AR:$lhs, AR:$rhs, FPR:$t, FPR:$f, imm:$cond))]>;
+  def SELECT_CC_FP_INT : Pseudo<(outs AR:$dst), (ins FPR:$lhs, FPR:$rhs, AR:$t, AR:$f, i32imm:$cond, i32imm:$brkind),
+                                "!select_cc_fp_int $dst, $lhs, $rhs, $t, $f, $cond, $brkind",
+                                [(set AR:$dst, (Xtensa_select_cc_fp FPR:$lhs, FPR:$rhs, AR:$t, AR:$f, imm:$cond, imm:$brkind))]>;
+  def SELECT_CC_FP_FP : Pseudo<(outs FPR:$dst), (ins FPR:$lhs, FPR:$rhs, FPR:$t, FPR:$f, i32imm:$cond, i32imm:$brkind),
+                               "!select_cc_fp_fp $dst, $lhs, $rhs, $t, $f, $cond, $brkind",
+                               [(set FPR:$dst, (Xtensa_select_cc_fp FPR:$lhs, FPR:$rhs, FPR:$t, FPR:$f, imm:$cond, imm:$brkind))]>;
+}
+
+let usesCustomInserter = 1, isBranch = 1, isTerminator = 1, isBarrier = 1, Predicates = [HasSingleFloat] in {
+  def BRCC_FP : Pseudo<(outs), (ins i32imm:$cond, FPR:$lhs, FPR:$rhs, brtarget:$target),
+                       "!brcc_fp $cond, $lhs, $rhs, $target", []>;
+}
+
+def cond_as_i32imm : SDNodeXForm<cond, [{
+  return CurDAG->getTargetConstant(N->get(), SDLoc(N), MVT::i32);
+}]>;
+
+def : Pat<(brcc cond:$cond, FPR:$s, FPR:$t,  bb:$target),
+          (BRCC_FP (cond_as_i32imm $cond), FPR:$s, FPR:$t, bb:$target)>;
+
 //===----------------------------------------------------------------------===//
 // Region Protection feature instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaOperators.td b/llvm/lib/Target/Xtensa/XtensaOperators.td
index fea13c2298d97..56be3d51505e1 100644
--- a/llvm/lib/Target/Xtensa/XtensaOperators.td
+++ b/llvm/lib/Target/Xtensa/XtensaOperators.td
@@ -21,14 +21,15 @@ def SDT_XtensaBrJT                : SDTypeProfile<0, 2,
                                                  [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>;
 
 def SDT_XtensaSelectCC            : SDTypeProfile<1, 5,
-                                                 [SDTCisSameAs<0, 1>,
-                                                  SDTCisSameAs<2, 3>,
+                                                 [SDTCisSameAs<0, 3>, SDTCisSameAs<1, 2>,
+                                                  SDTCisSameAs<3, 4>,
                                                   SDTCisVT<5, i32>]>;
 
 def SDT_XtensaCmp                 : SDTypeProfile<1, 2, [SDTCisVT<0, v1i1>, SDTCisVT<1, f32>, SDTCisVT<2, f32>]>;
 def SDT_XtensaMADD                : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisVT<0, f32>]>;
 def SDT_XtensaMOVS                : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, SDTCisVT<0, f32>]>;
-def SDT_XtensaSelectCCFP          : SDTypeProfile<1, 5, [SDTCisSameAs<0, 3>, SDTCisSameAs<1, 2>, SDTCisSameAs<3, 4>, SDTCisVT<5, i32>]>;
+def SDT_XtensaSelectCCFP          : SDTypeProfile<1, 6, [SDTCisSameAs<0, 3>, SDTCisSameAs<1, 2>, SDTCisSameAs<3, 4>,
+                                                         SDTCisVT<5, i32>, SDTCisVT<6, i32>]>;
 
 def SDT_XtensaSRC                 : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                                          SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
@@ -38,6 +39,8 @@ def SDT_XtensaEXTUI               : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCi
 
 def SDT_XtensaMOVSP               : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
 
+def SDT_XtensaRUR                : SDTypeProfile<1, 1, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
+
 //===----------------------------------------------------------------------===//
 // Node definitions
 //===----------------------------------------------------------------------===//
@@ -76,6 +79,9 @@ def Xtensa_extui: SDNode<"XtensaISD::EXTUI", SDT_XtensaEXTUI>;
 def Xtensa_movsp: SDNode<"XtensaISD::MOVSP", SDT_XtensaMOVSP,
                         [SDNPHasChain, SDNPSideEffect, SDNPInGlue]>;
 
+def Xtensa_rur: SDNode<"XtensaISD::RUR", SDT_XtensaRUR,
+                      [SDNPInGlue]>;
+
 def Xtensa_cmpoeq     : SDNode<"XtensaISD::CMPOEQ", SDT_XtensaCmp, [SDNPOutGlue]>;
 def Xtensa_cmpolt     : SDNode<"XtensaISD::CMPOLT", SDT_XtensaCmp, [SDNPOutGlue]>;
 def Xtensa_cmpole     : SDNode<"XtensaISD::CMPOLE", SDT_XtensaCmp, [SDNPOutGlue]>;
@@ -87,3 +93,6 @@ def Xtensa_cmpuo      : SDNode<"XtensaISD::CMPUO", SDT_XtensaCmp, [SDNPOutGlue]>
 def Xtensa_madd: SDNode<"XtensaISD::MADD", SDT_XtensaMADD, [SDNPInGlue]>;
 def Xtensa_msub: SDNode<"XtensaISD::MSUB", SDT_XtensaMADD, [SDNPInGlue]>;
 def Xtensa_movs: SDNode<"XtensaISD::MOVS", SDT_XtensaMOVS, [SDNPInGlue]>;
+
+def Xtensa_select_cc_fp: SDNode<"XtensaISD::SELECT_CC_FP", SDT_XtensaSelectCCFP,
+                               [SDNPInGlue]>;
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
index 74633050861c2..ed76431bf5495 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
@@ -53,7 +53,10 @@ BitVector XtensaRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     // Reserve frame pointer.
     Reserved.set(getFrameRegister(MF));
   }
-
+  if (Subtarget.hasTHREADPTR()) {
+    // Reserve frame pointer.
+    Reserved.set(Xtensa::THREADPTR);
+  }
   // Reserve stack pointer.
   Reserved.set(Xtensa::SP);
   return Reserved;
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
index 644faee51f513..596c4105c1118 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
@@ -234,10 +234,19 @@ class URReg<bits<8> num, string n, list<string> alt = []> : XtensaReg<n> {
   let AltNames = alt;
 }
 
+// Thread Pointer register
+def THREADPTR : URReg<231, "threadptr", ["THREADPTR"]>;
+
 def FCR : URReg<232, "fcr", ["FCR"]>;
 def FSR : URReg<233, "fsr", ["FSR"]>;
 
-def UR :  RegisterClass<"Xtensa", [i32], 32, (add FCR, FSR)>;
+// DFPAccel registers
+def F64R_LO : URReg<234, "f64r_lo", ["F64R_LO"]>;
+def F64R_HI : URReg<235, "f64r_hi", ["F64R_HI"]>;
+def F64S : URReg<236, "f64s", ["F64S"]>;
+
+def UR :  RegisterClass<"Xtensa", [i32], 32, (add
+  THREADPTR, FCR, FSR, F64R_LO, F64R_HI, F64S)>;
 
 //===----------------------------------------------------------------------===//
 // Floating-Point registers
diff --git a/llvm/lib/Target/Xtensa/XtensaSubtarget.h b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
index bc32541750ece..fd677a451f3fd 100644
--- a/llvm/lib/Target/Xtensa/XtensaSubtarget.h
+++ b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
@@ -91,7 +91,7 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
   bool hasHighPriInterruptsLevel7() const { return HasHighPriInterruptsLevel7; }
   bool hasInterrupt() const { return HasInterrupt; }
   bool hasException() const { return HasException; }
-
+  bool hasTHREADPTR() const { return HasTHREADPTR; }
   bool isWindowedABI() const { return hasWindowed(); }
 
   // Automatically generated by tblgen.
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index 2c805e2f7e664..9432fc2c4ac8d 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -50,7 +50,7 @@ std::optional<AArch64::ArchInfo> AArch64::ArchInfo::findBySubArch(StringRef SubA
 
 std::optional<AArch64::FMVInfo> lookupFMVByID(AArch64::ArchExtKind ExtID) {
   for (const AArch64::FMVInfo &Info : AArch64::getFMVInfo())
-    if (Info.ID && *Info.ID == ExtID)
+    if (Info.ID == ExtID)
       return Info;
   return {};
 }
@@ -298,6 +298,13 @@ void AArch64::ExtensionSet::disable(ArchExtKind E) {
   if (E == AEK_SVE2AES)
     disable(AEK_SVEAES);
 
+  // sve2-sm4 was historically associated with both FEAT_SVE2 and
+  // FEAT_SVE_SM4, the latter is now associated with sve-sm4 and sve2-sm4 has
+  // become shorthand for +sve2+sve-sm4. For backwards compatibility, when we
+  // disable sve2-sm4 we must also disable sve-sm4.
+  if (E == AEK_SVE2SM4)
+    disable(AEK_SVESM4);
+
   // sve2-sha3 was historically associated with both FEAT_SVE2 and
   // FEAT_SVE_SHA3, the latter is now associated with sve-sha3 and sve2-sha3 has
   // become shorthand for +sve2+sve-sha3. For backwards compatibility, when we
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 5957e1befe2da..3b80c152c1264 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -176,25 +176,43 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
   SmallVector<StringRef, 32> Lines;
   ProcCpuinfoContent.split(Lines, '\n');
 
-  // Look for the CPU implementer line.
+  // Look for the CPU implementer and hardware lines, and store the CPU part
+  // numbers found.
   StringRef Implementer;
   StringRef Hardware;
-  StringRef Part;
-  for (unsigned I = 0, E = Lines.size(); I != E; ++I) {
-    if (Lines[I].starts_with("CPU implementer"))
-      Implementer = Lines[I].substr(15).ltrim("\t :");
-    if (Lines[I].starts_with("Hardware"))
-      Hardware = Lines[I].substr(8).ltrim("\t :");
-    if (Lines[I].starts_with("CPU part"))
-      Part = Lines[I].substr(8).ltrim("\t :");
+  SmallVector<StringRef, 32> Parts;
+  for (StringRef Line : Lines) {
+    if (Line.consume_front("CPU implementer"))
+      Implementer = Line.ltrim("\t :");
+    else if (Line.consume_front("Hardware"))
+      Hardware = Line.ltrim("\t :");
+    else if (Line.consume_front("CPU part"))
+      Parts.emplace_back(Line.ltrim("\t :"));
   }
 
+  // Last `Part' seen, in case we don't analyse all `Parts' parsed.
+  StringRef Part = Parts.empty() ? StringRef() : Parts.back();
+
+  // Remove duplicate `Parts'.
+  llvm::sort(Parts);
+  Parts.erase(llvm::unique(Parts), Parts.end());
+
+  auto MatchBigLittle = [](auto const &Parts, StringRef Big, StringRef Little) {
+    if (Parts.size() == 2)
+      return (Parts[0] == Big && Parts[1] == Little) ||
+             (Parts[1] == Big && Parts[0] == Little);
+    return false;
+  };
+
   if (Implementer == "0x41") { // ARM Ltd.
     // MSM8992/8994 may give cpu part for the core that the kernel is running on,
     // which is undeterministic and wrong. Always return cortex-a53 for these SoC.
     if (Hardware.ends_with("MSM8994") || Hardware.ends_with("MSM8996"))
       return "cortex-a53";
 
+    // Detect big.LITTLE systems.
+    if (MatchBigLittle(Parts, "0xd85", "0xd87"))
+      return "cortex-x925";
 
     // The CPU part is a 3 digit hexadecimal number with a 0x prefix. The
     // values correspond to the "Part number" in the CP15/c0 register. The
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index cae12f9a4ed3e..31123c5eb7ab7 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -445,6 +445,7 @@ void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
       Features["prng-inst"] = true;
       Features["transpose-load-f4f6-insts"] = true;
       Features["fp8-conversion-insts"] = true;
+      Features["fp8e5m3-insts"] = true;
       Features["permlane16-swap"] = true;
       Features["ashr-pk-insts"] = true;
       Features["atomic-buffer-pk-add-bf16-inst"] = true;
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index a348640d75f26..0584c941d2e6e 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/VersionTuple.h"
@@ -2255,6 +2256,56 @@ bool Triple::isValidVersionForOS(OSType OSKind, const VersionTuple &Version) {
   llvm_unreachable("unexpected or invalid os version");
 }
 
+ExceptionHandling Triple::getDefaultExceptionHandling() const {
+  if (isOSBinFormatCOFF()) {
+    if (getArch() == Triple::x86 &&
+        (isOSCygMing() || isWindowsItaniumEnvironment()))
+      return ExceptionHandling::DwarfCFI;
+    return ExceptionHandling::WinEH;
+  }
+
+  if (isOSBinFormatXCOFF())
+    return ExceptionHandling::AIX;
+  if (isOSBinFormatGOFF())
+    return ExceptionHandling::ZOS;
+
+  if (isARM() || isThumb()) {
+    if (isOSBinFormatELF()) {
+      return getOS() == Triple::NetBSD ? ExceptionHandling::DwarfCFI
+                                       : ExceptionHandling::ARM;
+    }
+
+    return isOSDarwin() && !isWatchABI() ? ExceptionHandling::SjLj
+                                         : ExceptionHandling::DwarfCFI;
+  }
+
+  if (isAArch64() || isX86() || isPPC() || isMIPS() || isSPARC() || isBPF() ||
+      isRISCV() || isLoongArch())
+    return ExceptionHandling::DwarfCFI;
+
+  switch (getArch()) {
+  case Triple::arc:
+  case Triple::csky:
+  case Triple::hexagon:
+  case Triple::lanai:
+  case Triple::m68k:
+  case Triple::msp430:
+  case Triple::systemz:
+  case Triple::xcore:
+  case Triple::xtensa:
+    return ExceptionHandling::DwarfCFI;
+  default:
+    break;
+  }
+
+  // Explicitly none targets.
+  if (isWasm() || isAMDGPU() || isNVPTX() || isSPIROrSPIRV())
+    return ExceptionHandling::None;
+
+  // Default to none.
+  return ExceptionHandling::None;
+}
+
 // HLSL triple environment orders are relied on in the front end
 static_assert(Triple::Vertex - Triple::Pixel == 1,
               "incorrect HLSL stage order");
diff --git a/llvm/lib/TargetParser/Unix/Host.inc b/llvm/lib/TargetParser/Unix/Host.inc
index ef9e288ee3a01..aeb2f59218700 100644
--- a/llvm/lib/TargetParser/Unix/Host.inc
+++ b/llvm/lib/TargetParser/Unix/Host.inc
@@ -55,7 +55,7 @@ static std::string updateTripleOSVersion(std::string TargetTripleString) {
   // On AIX, the AIX version and release should be that of the current host
   // unless if the version has already been specified.
   if (Triple(LLVM_HOST_TRIPLE).getOS() == Triple::AIX) {
-    Triple TT(std::move(TargetTripleString));
+    Triple TT{TargetTripleString};
     if (TT.getOS() == Triple::AIX && !TT.getOSMajorVersion()) {
       struct utsname name;
       if (uname(&name) != -1) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
index a0a26827aa09d..c00e9c7bbee06 100644
--- a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
@@ -110,11 +110,11 @@ bool Lowerer::lower(Function &F) {
 
 static bool declaresCoroCleanupIntrinsics(const Module &M) {
   return coro::declaresIntrinsics(
-      M, {"llvm.coro.alloc", "llvm.coro.begin", "llvm.coro.subfn.addr",
-          "llvm.coro.free", "llvm.coro.id", "llvm.coro.id.retcon",
-          "llvm.coro.id.async", "llvm.coro.id.retcon.once",
-          "llvm.coro.async.size.replace", "llvm.coro.async.resume",
-          "llvm.coro.begin.custom.abi"});
+      M, {Intrinsic::coro_alloc, Intrinsic::coro_begin,
+          Intrinsic::coro_subfn_addr, Intrinsic::coro_free, Intrinsic::coro_id,
+          Intrinsic::coro_id_retcon, Intrinsic::coro_id_async,
+          Intrinsic::coro_id_retcon_once, Intrinsic::coro_async_size_replace,
+          Intrinsic::coro_async_resume, Intrinsic::coro_begin_custom_abi});
 }
 
 PreservedAnalyses CoroCleanupPass::run(Module &M,
diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index 9ac145b10fe43..e279fec18bdbc 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -279,12 +279,13 @@ void Lowerer::lowerEarlyIntrinsics(Function &F) {
 }
 
 static bool declaresCoroEarlyIntrinsics(const Module &M) {
+  // coro_suspend omitted as it is overloaded.
   return coro::declaresIntrinsics(
-      M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
-          "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
-          "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
-          "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
-          "llvm.coro.suspend"});
+      M, {Intrinsic::coro_id, Intrinsic::coro_id_retcon,
+          Intrinsic::coro_id_retcon_once, Intrinsic::coro_id_async,
+          Intrinsic::coro_destroy, Intrinsic::coro_done, Intrinsic::coro_end,
+          Intrinsic::coro_end_async, Intrinsic::coro_noop, Intrinsic::coro_free,
+          Intrinsic::coro_promise, Intrinsic::coro_resume});
 }
 
 PreservedAnalyses CoroEarlyPass::run(Module &M, ModuleAnalysisManager &) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
index c5131aa0787ed..1c8d4a8592d60 100644
--- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
@@ -450,7 +450,7 @@ bool CoroIdElider::attemptElide() {
 
 PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &M = *F.getParent();
-  if (!coro::declaresIntrinsics(M, {"llvm.coro.id"}))
+  if (!coro::declaresIntrinsics(M, Intrinsic::coro_id))
     return PreservedAnalyses::all();
 
   FunctionElideInfo FEI{&F};
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 1acd9139fe26e..7224a56cd7b8a 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -688,9 +688,8 @@ static DIType *solveDIType(DIBuilder &Builder, Type *Ty,
 static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
                                 FrameDataInfo &FrameData) {
   DISubprogram *DIS = F.getSubprogram();
-  // If there is no DISubprogram for F, it implies the Function are not compiled
-  // with debug info. So we also don't need to generate debug info for the frame
-  // neither.
+  // If there is no DISubprogram for F, it implies the function is compiled
+  // without debug info. So we also don't generate debug info for the frame.
   if (!DIS || !DIS->getUnit() ||
       !dwarf::isCPlusPlus(
           (dwarf::SourceLanguage)DIS->getUnit()->getSourceLanguage()) ||
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index a0b52063aca10..b53c5a48eb10b 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -24,8 +24,7 @@ namespace coro {
 
 bool isSuspendBlock(BasicBlock *BB);
 bool declaresAnyIntrinsic(const Module &M);
-bool declaresIntrinsics(const Module &M,
-                        const std::initializer_list<StringRef>);
+bool declaresIntrinsics(const Module &M, ArrayRef<Intrinsic::ID> List);
 void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
 
 /// Replaces all @llvm.coro.alloc intrinsics calls associated with a given
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 8813f91e9060c..077dbe697a87e 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -42,6 +42,7 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -302,7 +303,7 @@ static void replaceFallthroughCoroEnd(AnyCoroEndInst *End,
 }
 
 // Mark a coroutine as done, which implies that the coroutine is finished and
-// never get resumed.
+// never gets resumed.
 //
 // In resume-switched ABI, the done state is represented by storing zero in
 // ResumeFnAddr.
@@ -1478,6 +1479,18 @@ struct SwitchCoroutineSplitter {
   static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
     LLVMContext &C = F.getContext();
 
+    DIBuilder DBuilder(*F.getParent(), /*AllowUnresolved*/ false);
+    DISubprogram *DIS = F.getSubprogram();
+    // If there is no DISubprogram for F, it implies the function is compiled
+    // without debug info. So we also don't generate debug info for the
+    // suspension points.
+    bool AddDebugLabels =
+        (DIS && DIS->getUnit() &&
+         (DIS->getUnit()->getEmissionKind() ==
+              DICompileUnit::DebugEmissionKind::FullDebug ||
+          DIS->getUnit()->getEmissionKind() ==
+              DICompileUnit::DebugEmissionKind::LineTablesOnly));
+
     // resume.entry:
     //  %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32
     //  0, i32 2 % index = load i32, i32* %index.addr switch i32 %index, label
@@ -1500,6 +1513,7 @@ struct SwitchCoroutineSplitter {
         Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
     Shape.SwitchLowering.ResumeSwitch = Switch;
 
+    // Split all coro.suspend calls
     size_t SuspendIndex = 0;
     for (auto *AnyS : Shape.CoroSuspends) {
       auto *S = cast<CoroSuspendInst>(AnyS);
@@ -1538,6 +1552,7 @@ struct SwitchCoroutineSplitter {
       //     br label %resume.0.landing
       //
       //  resume.0: ; <--- jump from the switch in the resume.entry
+      //        #dbg_label(...)  ; <--- artificial label for debuggers
       //     %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
       //     br label %resume.0.landing
       //
@@ -1560,11 +1575,27 @@ struct SwitchCoroutineSplitter {
       PN->addIncoming(Builder.getInt8(-1), SuspendBB);
       PN->addIncoming(S, ResumeBB);
 
+      if (AddDebugLabels) {
+        if (DebugLoc SuspendLoc = S->getDebugLoc()) {
+          std::string LabelName =
+              ("__coro_resume_" + Twine(SuspendIndex)).str();
+          DILocation &DILoc = *SuspendLoc.get();
+          DILabel *ResumeLabel =
+              DBuilder.createLabel(DIS, LabelName, DILoc.getFile(),
+                                   SuspendLoc.getLine(), SuspendLoc.getCol(),
+                                   /*IsArtificial=*/true,
+                                   /*CoroSuspendIdx=*/SuspendIndex,
+                                   /*AlwaysPreserve=*/false);
+          DBuilder.insertLabel(ResumeLabel, &DILoc, ResumeBB->begin());
+        }
+      }
+
       ++SuspendIndex;
     }
 
     Builder.SetInsertPoint(UnreachBB);
     Builder.CreateUnreachable();
+    DBuilder.finalize();
 
     Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
   }
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 02500ff778b80..59ae057cae793 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -61,71 +61,56 @@ CallInst *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
   return CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt->getIterator());
 }
 
-// NOTE: Must be sorted!
-static const char *const CoroIntrinsics[] = {
-    "llvm.coro.align",
-    "llvm.coro.alloc",
-    "llvm.coro.async.context.alloc",
-    "llvm.coro.async.context.dealloc",
-    "llvm.coro.async.resume",
-    "llvm.coro.async.size.replace",
-    "llvm.coro.await.suspend.bool",
-    "llvm.coro.await.suspend.handle",
-    "llvm.coro.await.suspend.void",
-    "llvm.coro.begin",
-    "llvm.coro.begin.custom.abi",
-    "llvm.coro.destroy",
-    "llvm.coro.done",
-    "llvm.coro.end",
-    "llvm.coro.end.async",
-    "llvm.coro.frame",
-    "llvm.coro.free",
-    "llvm.coro.id",
-    "llvm.coro.id.async",
-    "llvm.coro.id.retcon",
-    "llvm.coro.id.retcon.once",
-    "llvm.coro.noop",
-    "llvm.coro.prepare.async",
-    "llvm.coro.prepare.retcon",
-    "llvm.coro.promise",
-    "llvm.coro.resume",
-    "llvm.coro.save",
-    "llvm.coro.size",
-    "llvm.coro.subfn.addr",
-    "llvm.coro.suspend",
-    "llvm.coro.suspend.async",
-    "llvm.coro.suspend.retcon",
+// We can only efficiently check for non-overloaded intrinsics.
+// The following intrinsics are absent for that reason:
+// coro_align, coro_size, coro_suspend_async, coro_suspend_retcon
+static Intrinsic::ID NonOverloadedCoroIntrinsics[] = {
+    Intrinsic::coro_alloc,
+    Intrinsic::coro_async_context_alloc,
+    Intrinsic::coro_async_context_dealloc,
+    Intrinsic::coro_async_resume,
+    Intrinsic::coro_async_size_replace,
+    Intrinsic::coro_await_suspend_bool,
+    Intrinsic::coro_await_suspend_handle,
+    Intrinsic::coro_await_suspend_void,
+    Intrinsic::coro_begin,
+    Intrinsic::coro_begin_custom_abi,
+    Intrinsic::coro_destroy,
+    Intrinsic::coro_done,
+    Intrinsic::coro_end,
+    Intrinsic::coro_end_async,
+    Intrinsic::coro_frame,
+    Intrinsic::coro_free,
+    Intrinsic::coro_id,
+    Intrinsic::coro_id_async,
+    Intrinsic::coro_id_retcon,
+    Intrinsic::coro_id_retcon_once,
+    Intrinsic::coro_promise,
+    Intrinsic::coro_resume,
+    Intrinsic::coro_save,
+    Intrinsic::coro_subfn_addr,
+    Intrinsic::coro_suspend,
 };
 
-#ifndef NDEBUG
-static bool isCoroutineIntrinsicName(StringRef Name) {
-  return llvm::binary_search(CoroIntrinsics, Name);
-}
-#endif
-
 bool coro::isSuspendBlock(BasicBlock *BB) {
   return isa<AnyCoroSuspendInst>(BB->front());
 }
 
 bool coro::declaresAnyIntrinsic(const Module &M) {
-  for (StringRef Name : CoroIntrinsics) {
-    if (M.getNamedValue(Name))
-      return true;
-  }
-
-  return false;
+  return declaresIntrinsics(M, NonOverloadedCoroIntrinsics);
 }
 
-// Verifies if a module has named values listed. Also, in debug mode verifies
-// that names are intrinsic names.
-bool coro::declaresIntrinsics(const Module &M,
-                              const std::initializer_list<StringRef> List) {
-  for (StringRef Name : List) {
-    assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
-    if (M.getNamedValue(Name))
-      return true;
-  }
+// Checks whether the module declares any of the listed intrinsics.
+bool coro::declaresIntrinsics(const Module &M, ArrayRef<Intrinsic::ID> List) {
+#ifndef NDEBUG
+  for (Intrinsic::ID ID : List)
+    assert(!Intrinsic::isOverloaded(ID) &&
+           "Only non-overloaded intrinsics supported");
+#endif
 
+  for (Intrinsic::ID ID : List)
+    if (Intrinsic::getDeclarationIfExists(&M, ID))
+      return true;
   return false;
 }
 
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index b3e5b7fa6e0b5..8017db1cfe146 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -552,10 +552,10 @@ void sinkSpillUsesAfterCoroBegin(const DominatorTree &Dom,
         Worklist.push_back(Inst);
     }
   };
-  std::for_each(Spills.begin(), Spills.end(),
-                [&](auto &I) { collectUsers(I.first); });
-  std::for_each(Allocas.begin(), Allocas.end(),
-                [&](auto &I) { collectUsers(I.Alloca); });
+  for (auto &I : Spills)
+    collectUsers(I.first);
+  for (auto &I : Allocas)
+    collectUsers(I.Alloca);
 
   // Recursively collect users before coro.begin.
   while (!Worklist.empty()) {
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index dac1f7a30c370..448e1b4982b74 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -990,6 +990,13 @@ static bool addIfNotExistent(LLVMContext &Ctx, const Attribute &Attr,
     AB.addAttribute(Attr);
     return true;
   }
+  if (Attr.isConstantRangeAttribute()) {
+    Attribute::AttrKind Kind = Attr.getKindAsEnum();
+    if (!ForceReplace && AttrSet.hasAttribute(Kind))
+      return false;
+    AB.addAttribute(Attr);
+    return true;
+  }
 
   llvm_unreachable("Expected enum or string attribute!");
 }
@@ -3612,6 +3619,8 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
       if (SimplifyAllLoads)
         getAssumedSimplified(IRPosition::value(I), nullptr,
                              UsedAssumedInformation, AA::Intraprocedural);
+      getOrCreateAAFor<AAInvariantLoadPointer>(
+          IRPosition::value(*LI->getPointerOperand()));
       getOrCreateAAFor<AAAddressSpace>(
           IRPosition::value(*LI->getPointerOperand()));
     } else {
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 3799a696f67af..ed2ac4dbfeecd 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -191,7 +191,9 @@ PIPE_OPERATOR(AAInterFnReachability)
 PIPE_OPERATOR(AAPointerInfo)
 PIPE_OPERATOR(AAAssumptionInfo)
 PIPE_OPERATOR(AAUnderlyingObjects)
+PIPE_OPERATOR(AAInvariantLoadPointer)
 PIPE_OPERATOR(AAAddressSpace)
+PIPE_OPERATOR(AANoAliasAddrSpace)
 PIPE_OPERATOR(AAAllocationInfo)
 PIPE_OPERATOR(AAIndirectCallInfo)
 PIPE_OPERATOR(AAGlobalValueInfo)
@@ -9171,44 +9173,58 @@ struct AAValueConstantRangeImpl : AAValueConstantRange {
     return MDNode::get(Ctx, LowAndHigh);
   }
 
-  /// Return true if \p Assumed is included in \p KnownRanges.
-  static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
-
+  /// Return true if \p Assumed is included in ranges from instruction \p I.
+  static bool isBetterRange(const ConstantRange &Assumed,
+                            const Instruction &I) {
     if (Assumed.isFullSet())
       return false;
 
-    if (!KnownRanges)
-      return true;
-
-    // If multiple ranges are annotated in IR, we give up to annotate assumed
-    // range for now.
+    std::optional<ConstantRange> Known;
 
-    // TODO:  If there exists a known range which containts assumed range, we
-    // can say assumed range is better.
-    if (KnownRanges->getNumOperands() > 2)
-      return false;
+    if (const auto *CB = dyn_cast<CallBase>(&I)) {
+      Known = CB->getRange();
+    } else if (MDNode *KnownRanges = I.getMetadata(LLVMContext::MD_range)) {
+      // If multiple ranges are annotated in IR, we give up to annotate assumed
+      // range for now.
+
+      // TODO:  If there exists a known range which containts assumed range, we
+      // can say assumed range is better.
+      if (KnownRanges->getNumOperands() > 2)
+        return false;
 
-    ConstantInt *Lower =
-        mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
-    ConstantInt *Upper =
-        mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
+      ConstantInt *Lower =
+          mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
+      ConstantInt *Upper =
+          mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
 
-    ConstantRange Known(Lower->getValue(), Upper->getValue());
-    return Known.contains(Assumed) && Known != Assumed;
+      Known.emplace(Lower->getValue(), Upper->getValue());
+    }
+    return !Known || (*Known != Assumed && Known->contains(Assumed));
   }
 
   /// Helper function to set range metadata.
   static bool
   setRangeMetadataIfisBetterRange(Instruction *I,
                                   const ConstantRange &AssumedConstantRange) {
-    auto *OldRangeMD = I->getMetadata(LLVMContext::MD_range);
-    if (isBetterRange(AssumedConstantRange, OldRangeMD)) {
-      if (!AssumedConstantRange.isEmptySet()) {
-        I->setMetadata(LLVMContext::MD_range,
-                       getMDNodeForConstantRange(I->getType(), I->getContext(),
-                                                 AssumedConstantRange));
-        return true;
-      }
+    if (isBetterRange(AssumedConstantRange, *I)) {
+      I->setMetadata(LLVMContext::MD_range,
+                     getMDNodeForConstantRange(I->getType(), I->getContext(),
+                                               AssumedConstantRange));
+      return true;
+    }
+    return false;
+  }
+  /// Helper function to set range return attribute.
+  static bool
+  setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
+                                 Instruction *I,
+                                 const ConstantRange &AssumedConstantRange) {
+    if (isBetterRange(AssumedConstantRange, *I)) {
+      A.manifestAttrs(IRP,
+                      Attribute::get(I->getContext(), Attribute::Range,
+                                     AssumedConstantRange),
+                      /*ForceReplace*/ true);
+      return true;
     }
     return false;
   }
@@ -9225,9 +9241,13 @@ struct AAValueConstantRangeImpl : AAValueConstantRange {
       if (Instruction *I = dyn_cast<Instruction>(&V)) {
         assert(I == getCtxI() && "Should not annotate an instruction which is "
                                  "not the context instruction");
-        if (isa<CallInst>(I) || isa<LoadInst>(I))
+        if (isa<LoadInst>(I))
           if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
             Changed = ChangeStatus::CHANGED;
+        if (isa<CallInst>(I))
+          if (setRangeRetAttrIfisBetterRange(A, getIRPosition(), I,
+                                             AssumedConstantRange))
+            Changed = ChangeStatus::CHANGED;
       }
     }
 
@@ -9623,10 +9643,11 @@ struct AAValueConstantRangeCallSiteReturned
 
   /// See AbstractAttribute::initialize(...).
   void initialize(Attributor &A) override {
-    // If it is a load instruction with range metadata, use the metadata.
-    if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue()))
-      if (auto *RangeMD = CI->getMetadata(LLVMContext::MD_range))
-        intersectKnown(getConstantRangeFromMetadata(*RangeMD));
+    // If it is a call instruction with range attribute, use the range.
+    if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue())) {
+      if (std::optional<ConstantRange> Range = CI->getRange())
+        intersectKnown(*Range);
+    }
 
     AAValueConstantRangeImpl::initialize(A);
   }
@@ -12533,6 +12554,346 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
 };
 } // namespace
 
+/// --------------------- Invariant Load Pointer -------------------------------
+namespace {
+
+struct AAInvariantLoadPointerImpl
+    : public StateWrapper<BitIntegerState<uint8_t, 15>,
+                          AAInvariantLoadPointer> {
+
+  enum {
+    // pointer does not alias within the bounds of the function
+    IS_NOALIAS = 1 << 0,
+    // pointer is not involved in any effectful instructions within the bounds
+    // of the function
+    IS_NOEFFECT = 1 << 1,
+    // loads are invariant within the bounds of the function
+    IS_LOCALLY_INVARIANT = 1 << 2,
+    // memory lifetime is constrained within the bounds of the function
+    IS_LOCALLY_CONSTRAINED = 1 << 3,
+
+    IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
+                    IS_LOCALLY_CONSTRAINED,
+  };
+  static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
+
+  using Base =
+      StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
+
+  // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
+  // pessimistic about IS_KNOWN_INVARIANT
+  AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
+      : Base(IRP) {}
+
+  bool isKnownInvariant() const final {
+    return isKnownLocallyInvariant() && isKnown(IS_LOCALLY_CONSTRAINED);
+  }
+
+  bool isKnownLocallyInvariant() const final {
+    if (isKnown(IS_LOCALLY_INVARIANT))
+      return true;
+    return isKnown(IS_NOALIAS | IS_NOEFFECT);
+  }
+
+  bool isAssumedInvariant() const final {
+    return isAssumedLocallyInvariant() && isAssumed(IS_LOCALLY_CONSTRAINED);
+  }
+
+  bool isAssumedLocallyInvariant() const final {
+    if (isAssumed(IS_LOCALLY_INVARIANT))
+      return true;
+    return isAssumed(IS_NOALIAS | IS_NOEFFECT);
+  }
+
+  ChangeStatus updateImpl(Attributor &A) override {
+    ChangeStatus Changed = ChangeStatus::UNCHANGED;
+
+    Changed |= updateNoAlias(A);
+    if (requiresNoAlias() && !isAssumed(IS_NOALIAS))
+      return indicatePessimisticFixpoint();
+
+    Changed |= updateNoEffect(A);
+
+    Changed |= updateLocalInvariance(A);
+
+    return Changed;
+  }
+
+  ChangeStatus manifest(Attributor &A) override {
+    if (!isKnownInvariant())
+      return ChangeStatus::UNCHANGED;
+
+    ChangeStatus Changed = ChangeStatus::UNCHANGED;
+    const Value *Ptr = &getAssociatedValue();
+    const auto TagInvariantLoads = [&](const Use &U, bool &) {
+      if (U.get() != Ptr)
+        return true;
+      auto *I = dyn_cast<Instruction>(U.getUser());
+      if (!I)
+        return true;
+
+      // Ensure that we are only changing uses from the corresponding callgraph
+      // SSC in the case that the AA isn't run on the entire module
+      if (!A.isRunOn(I->getFunction()))
+        return true;
+
+      if (I->hasMetadata(LLVMContext::MD_invariant_load))
+        return true;
+
+      if (auto *LI = dyn_cast<LoadInst>(I)) {
+        LI->setMetadata(LLVMContext::MD_invariant_load,
+                        MDNode::get(LI->getContext(), {}));
+        Changed = ChangeStatus::CHANGED;
+      }
+      return true;
+    };
+
+    (void)A.checkForAllUses(TagInvariantLoads, *this, *Ptr);
+    return Changed;
+  }
+
+  /// See AbstractAttribute::getAsStr().
+  const std::string getAsStr(Attributor *) const override {
+    if (isKnownInvariant())
+      return "load-invariant pointer";
+    return "non-invariant pointer";
+  }
+
+  /// See AbstractAttribute::trackStatistics().
+  void trackStatistics() const override {}
+
+private:
+  /// Indicate that noalias is required for the pointer to be invariant.
+  bool requiresNoAlias() const {
+    switch (getPositionKind()) {
+    default:
+      // Conservatively default to require noalias.
+      return true;
+    case IRP_FLOAT:
+    case IRP_RETURNED:
+    case IRP_CALL_SITE:
+      return false;
+    case IRP_CALL_SITE_RETURNED: {
+      const auto &CB = cast<CallBase>(getAnchorValue());
+      return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
+          &CB, /*MustPreserveNullness=*/false);
+    }
+    case IRP_ARGUMENT: {
+      const Function *F = getAssociatedFunction();
+      assert(F && "no associated function for argument");
+      return !isCallableCC(F->getCallingConv());
+    }
+    }
+  }
+
+  bool isExternal() const {
+    const Function *F = getAssociatedFunction();
+    if (!F)
+      return true;
+    return isCallableCC(F->getCallingConv()) &&
+           getPositionKind() != IRP_CALL_SITE_RETURNED;
+  }
+
+  ChangeStatus updateNoAlias(Attributor &A) {
+    if (isKnown(IS_NOALIAS) || !isAssumed(IS_NOALIAS))
+      return ChangeStatus::UNCHANGED;
+
+    // Try to use AANoAlias.
+    if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
+            getIRPosition(), this, DepClassTy::REQUIRED)) {
+      if (ANoAlias->isKnownNoAlias()) {
+        addKnownBits(IS_NOALIAS);
+        return ChangeStatus::CHANGED;
+      }
+
+      if (!ANoAlias->isAssumedNoAlias()) {
+        removeAssumedBits(IS_NOALIAS);
+        return ChangeStatus::CHANGED;
+      }
+
+      return ChangeStatus::UNCHANGED;
+    }
+
+    // Try to infer noalias from argument attribute, since it is applicable for
+    // the duration of the function.
+    if (const Argument *Arg = getAssociatedArgument()) {
+      if (Arg->hasNoAliasAttr()) {
+        addKnownBits(IS_NOALIAS);
+        return ChangeStatus::UNCHANGED;
+      }
+
+      // Noalias information is not provided, and cannot be inferred,
+      // so we conservatively assume the pointer aliases.
+      removeAssumedBits(IS_NOALIAS);
+      return ChangeStatus::CHANGED;
+    }
+
+    return ChangeStatus::UNCHANGED;
+  }
+
+  ChangeStatus updateNoEffect(Attributor &A) {
+    if (isKnown(IS_NOEFFECT) || !isAssumed(IS_NOEFFECT))
+      return ChangeStatus::UNCHANGED;
+
+    if (!getAssociatedFunction())
+      return indicatePessimisticFixpoint();
+
+    if (isa<AllocaInst>(&getAssociatedValue()))
+      return indicatePessimisticFixpoint();
+
+    const auto HasNoEffectLoads = [&](const Use &U, bool &) {
+      const auto *LI = dyn_cast<LoadInst>(U.getUser());
+      return !LI || !LI->mayHaveSideEffects();
+    };
+    if (!A.checkForAllUses(HasNoEffectLoads, *this, getAssociatedValue()))
+      return indicatePessimisticFixpoint();
+
+    if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
+            getIRPosition(), this, DepClassTy::REQUIRED)) {
+      // For non-instructions, try to use AAMemoryBehavior to infer the readonly
+      // attribute
+      if (!AMemoryBehavior->isAssumedReadOnly())
+        return indicatePessimisticFixpoint();
+
+      if (AMemoryBehavior->isKnownReadOnly()) {
+        addKnownBits(IS_NOEFFECT);
+        return ChangeStatus::UNCHANGED;
+      }
+
+      return ChangeStatus::UNCHANGED;
+    }
+
+    if (const Argument *Arg = getAssociatedArgument()) {
+      if (Arg->onlyReadsMemory()) {
+        addKnownBits(IS_NOEFFECT);
+        return ChangeStatus::UNCHANGED;
+      }
+
+      // Readonly information is not provided, and cannot be inferred from
+      // AAMemoryBehavior.
+      return indicatePessimisticFixpoint();
+    }
+
+    return ChangeStatus::UNCHANGED;
+  }
+
+  ChangeStatus updateLocalInvariance(Attributor &A) {
+    if (isKnown(IS_LOCALLY_INVARIANT) || !isAssumed(IS_LOCALLY_INVARIANT))
+      return ChangeStatus::UNCHANGED;
+
+    // try to infer invariance from underlying objects
+    const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
+        getIRPosition(), this, DepClassTy::REQUIRED);
+    if (!AUO)
+      return ChangeStatus::UNCHANGED;
+
+    bool UsedAssumedInformation = false;
+    const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
+      if (!V.getType()->isPointerTy())
+        return true;
+      const auto *IsInvariantLoadPointer =
+          A.getOrCreateAAFor<AAInvariantLoadPointer>(IRPosition::value(V), this,
+                                                     DepClassTy::REQUIRED);
+      // Conservatively fail if invariance cannot be inferred.
+      if (!IsInvariantLoadPointer)
+        return false;
+
+      if (IsInvariantLoadPointer->isKnownLocallyInvariant())
+        return true;
+      if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
+        return false;
+
+      UsedAssumedInformation = true;
+      return true;
+    };
+    if (!AUO->forallUnderlyingObjects(IsLocallyInvariantLoadIfPointer))
+      return indicatePessimisticFixpoint();
+
+    if (const auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {
+      if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
+              CB, /*MustPreserveNullness=*/false)) {
+        for (const Value *Arg : CB->args()) {
+          if (!IsLocallyInvariantLoadIfPointer(*Arg))
+            return indicatePessimisticFixpoint();
+        }
+      }
+    }
+
+    if (!UsedAssumedInformation) {
+      // Pointer is known and not just assumed to be locally invariant.
+      addKnownBits(IS_LOCALLY_INVARIANT);
+      return ChangeStatus::CHANGED;
+    }
+
+    return ChangeStatus::UNCHANGED;
+  }
+};
+
+struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+};
+
+struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+
+  void initialize(Attributor &) override {
+    removeAssumedBits(IS_LOCALLY_CONSTRAINED);
+  }
+};
+
+struct AAInvariantLoadPointerCallSiteReturned final
+    : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+
+  void initialize(Attributor &A) override {
+    const Function *F = getAssociatedFunction();
+    assert(F && "no associated function for return from call");
+
+    if (!F->isDeclaration() && !F->isIntrinsic())
+      return AAInvariantLoadPointerImpl::initialize(A);
+
+    const auto &CB = cast<CallBase>(getAnchorValue());
+    if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
+            &CB, /*MustPreserveNullness=*/false))
+      return AAInvariantLoadPointerImpl::initialize(A);
+
+    if (F->onlyReadsMemory() && F->hasNoSync())
+      return AAInvariantLoadPointerImpl::initialize(A);
+
+    // At this point, the function is opaque, so we conservatively assume
+    // non-invariance.
+    indicatePessimisticFixpoint();
+  }
+};
+
+struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+
+  void initialize(Attributor &) override {
+    const Function *F = getAssociatedFunction();
+    assert(F && "no associated function for argument");
+
+    if (!isCallableCC(F->getCallingConv())) {
+      addKnownBits(IS_LOCALLY_CONSTRAINED);
+      return;
+    }
+
+    if (!F->hasLocalLinkage())
+      removeAssumedBits(IS_LOCALLY_CONSTRAINED);
+  }
+};
+
+struct AAInvariantLoadPointerCallSiteArgument final
+    : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+};
+} // namespace
+
 /// ------------------------ Address Space  ------------------------------------
 namespace {
 
@@ -12786,6 +13147,197 @@ struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
 };
 } // namespace
 
+/// ------------------------ No Alias Address Space  ---------------------------
+// This attribute assumes flat address space can alias all other address space
+
+// TODO: this is similar to AAAddressSpace, most of the code should be merged.
+// But merging it created failing cased on gateway test that cannot be
+// reproduced locally. So should open a seperated PR to hande the merge of
+// AANoAliasAddrSpace and AAAddressSpace attribute
+
+namespace {
+struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
+  AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
+      : AANoAliasAddrSpace(IRP, A) {}
+
+  void initialize(Attributor &A) override {
+    assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
+           "Associated value is not a pointer");
+
+    resetASRanges(A);
+
+    std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
+    if (!FlatAS.has_value()) {
+      indicatePessimisticFixpoint();
+      return;
+    }
+
+    removeAS(*FlatAS);
+
+    unsigned AS = getAssociatedType()->getPointerAddressSpace();
+    if (AS != *FlatAS) {
+      removeAS(AS);
+      indicateOptimisticFixpoint();
+    }
+  }
+
+  ChangeStatus updateImpl(Attributor &A) override {
+    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
+    uint32_t OldAssumed = getAssumed();
+
+    auto CheckAddressSpace = [&](Value &Obj) {
+      if (isa<PoisonValue>(&Obj))
+        return true;
+
+      unsigned AS = Obj.getType()->getPointerAddressSpace();
+      if (AS == FlatAS)
+        return false;
+
+      removeAS(Obj.getType()->getPointerAddressSpace());
+      return true;
+    };
+
+    const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
+        getIRPosition(), this, DepClassTy::REQUIRED);
+    if (!AUO->forallUnderlyingObjects(CheckAddressSpace))
+      return indicatePessimisticFixpoint();
+
+    return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
+                                      : ChangeStatus::CHANGED;
+  }
+
+  /// See AbstractAttribute::manifest(...).
+  ChangeStatus manifest(Attributor &A) override {
+    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
+
+    unsigned AS = getAssociatedType()->getPointerAddressSpace();
+    if (AS != FlatAS || Map.empty())
+      return ChangeStatus::UNCHANGED;
+
+    LLVMContext &Ctx = getAssociatedValue().getContext();
+    MDNode *NoAliasASNode = nullptr;
+    MDBuilder MDB(Ctx);
+    // Has to use iterator to get the range info.
+    for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
+      if (!I.value())
+        continue;
+      unsigned Upper = I.stop();
+      unsigned Lower = I.start();
+      if (!NoAliasASNode) {
+        NoAliasASNode = MDB.createRange(APInt(32, Lower), APInt(32, Upper + 1));
+        continue;
+      }
+      MDNode *ASRange = MDB.createRange(APInt(32, Lower), APInt(32, Upper + 1));
+      NoAliasASNode = MDNode::getMostGenericRange(NoAliasASNode, ASRange);
+    }
+
+    Value *AssociatedValue = &getAssociatedValue();
+    bool Changed = false;
+
+    auto AddNoAliasAttr = [&](const Use &U, bool &) {
+      if (U.get() != AssociatedValue)
+        return true;
+      Instruction *Inst = dyn_cast<Instruction>(U.getUser());
+      if (!Inst || Inst->hasMetadata(LLVMContext::MD_noalias_addrspace))
+        return true;
+      if (!isa<LoadInst>(Inst) && !isa<StoreInst>(Inst) &&
+          !isa<AtomicCmpXchgInst>(Inst) && !isa<AtomicRMWInst>(Inst))
+        return true;
+      if (!A.isRunOn(Inst->getFunction()))
+        return true;
+      Inst->setMetadata(LLVMContext::MD_noalias_addrspace, NoAliasASNode);
+      Changed = true;
+      return true;
+    };
+    (void)A.checkForAllUses(AddNoAliasAttr, *this, *AssociatedValue,
+                            /*CheckBBLivenessOnly=*/true);
+    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+  }
+
+  /// See AbstractAttribute::getAsStr().
+  const std::string getAsStr(Attributor *A) const override {
+    if (!isValidState())
+      return "<invalid>";
+    std::string Str;
+    raw_string_ostream OS(Str);
+    OS << "CanNotBeAddrSpace(";
+    for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
+      unsigned Upper = I.stop();
+      unsigned Lower = I.start();
+      OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
+    }
+    OS << " )";
+    return OS.str();
+  }
+
+private:
+  void removeAS(unsigned AS) {
+    RangeMap::iterator I = Map.find(AS);
+
+    if (I != Map.end()) {
+      unsigned Upper = I.stop();
+      unsigned Lower = I.start();
+      I.erase();
+      if (Upper == Lower)
+        return;
+      if (AS != ~((unsigned)0) && AS + 1 <= Upper)
+        Map.insert(AS + 1, Upper, /*what ever this variable name is=*/true);
+      if (AS != 0 && Lower <= AS - 1)
+        Map.insert(Lower, AS - 1, true);
+    }
+  }
+
+  void resetASRanges(Attributor &A) {
+    Map.clear();
+    Map.insert(0, A.getInfoCache().getMaxAddrSpace(), true);
+  }
+};
+
+struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
+  AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
+      : AANoAliasAddrSpaceImpl(IRP, A) {}
+
+  void trackStatistics() const override {
+    STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
+  }
+};
+
+struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
+  AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
+      : AANoAliasAddrSpaceImpl(IRP, A) {}
+
+  void trackStatistics() const override {
+    STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
+  }
+};
+
+struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
+  AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
+      : AANoAliasAddrSpaceImpl(IRP, A) {}
+
+  void trackStatistics() const override {
+    STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
+  }
+};
+
+struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
+  AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
+      : AANoAliasAddrSpaceImpl(IRP, A) {}
+
+  void trackStatistics() const override {
+    STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
+  }
+};
+
+struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
+  AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
+      : AANoAliasAddrSpaceImpl(IRP, A) {}
+
+  void trackStatistics() const override {
+    STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
+  }
+};
+} // namespace
 /// ----------- Allocation Info ----------
 namespace {
 struct AAAllocationInfoImpl : public AAAllocationInfo {
@@ -13038,7 +13590,9 @@ const char AAInterFnReachability::ID = 0;
 const char AAPointerInfo::ID = 0;
 const char AAAssumptionInfo::ID = 0;
 const char AAUnderlyingObjects::ID = 0;
+const char AAInvariantLoadPointer::ID = 0;
 const char AAAddressSpace::ID = 0;
+const char AANoAliasAddrSpace::ID = 0;
 const char AAAllocationInfo::ID = 0;
 const char AAIndirectCallInfo::ID = 0;
 const char AAGlobalValueInfo::ID = 0;
@@ -13172,7 +13726,9 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
 
 CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
diff --git a/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp b/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp
index 73f567734a91b..f02256f8a83b9 100644
--- a/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp
+++ b/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp
@@ -41,5 +41,5 @@ PreservedAnalyses EmbedBitcodePass::run(Module &M, ModuleAnalysisManager &AM) {
 
   embedBufferInModule(M, MemoryBufferRef(Data, "ModuleData"), ".llvm.lto");
 
-  return PreservedAnalyses::all();
+  return PreservedAnalyses::none();
 }
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index f918d7e059b63..f43202eea6306 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -675,14 +675,24 @@ ArgumentAccessInfo getArgumentAccessInfo(const Instruction *I,
       [](Value *Length,
          std::optional<int64_t> Offset) -> std::optional<ConstantRange> {
     auto *ConstantLength = dyn_cast<ConstantInt>(Length);
-    if (ConstantLength && Offset &&
-        ConstantLength->getValue().isStrictlyPositive()) {
-      return ConstantRange(
-          APInt(64, *Offset, true),
-          APInt(64, *Offset + ConstantLength->getSExtValue(), true));
+    if (ConstantLength && Offset) {
+      int64_t Len = ConstantLength->getSExtValue();
+
+      // Reject zero or negative lengths
+      if (Len <= 0)
+        return std::nullopt;
+
+      APInt Low(64, *Offset, true);
+      bool Overflow;
+      APInt High = Low.sadd_ov(APInt(64, Len, true), Overflow);
+      if (Overflow)
+        return std::nullopt;
+
+      return ConstantRange(Low, High);
     }
     return std::nullopt;
   };
+
   if (auto *SI = dyn_cast<StoreInst>(I)) {
     if (SI->isSimple() && &SI->getOperandUse(1) == ArgUse.U) {
       // Get the fixed type size of "SI". Since the access range of a write
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 7f5a2a982982d..b1c0e9910f378 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -92,8 +92,6 @@ STATISTIC(NumInternalFunc, "Number of internal functions");
 STATISTIC(NumColdCC, "Number of functions marked coldcc");
 STATISTIC(NumIFuncsResolved, "Number of statically resolved IFuncs");
 STATISTIC(NumIFuncsDeleted, "Number of IFuncs removed");
-STATISTIC(NumGlobalArraysPadded,
-          "Number of global arrays padded to alignment boundary");
 
 static cl::opt<bool>
     OptimizeNonFMVCallers("optimize-non-fmv-callers",
@@ -2037,165 +2035,6 @@ OptimizeFunctions(Module &M,
   return Changed;
 }
 
-static bool callInstIsMemcpy(CallInst *CI) {
-  if (!CI)
-    return false;
-
-  Function *F = CI->getCalledFunction();
-  if (!F || !F->isIntrinsic() || F->getIntrinsicID() != Intrinsic::memcpy)
-    return false;
-
-  return true;
-}
-
-static bool destArrayCanBeWidened(CallInst *CI) {
-  auto *IsVolatile = dyn_cast<ConstantInt>(CI->getArgOperand(3));
-  auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
-
-  if (!Alloca || !IsVolatile || IsVolatile->isOne())
-    return false;
-
-  if (!Alloca->isStaticAlloca())
-    return false;
-
-  if (!Alloca->getAllocatedType()->isArrayTy())
-    return false;
-
-  return true;
-}
-
-static GlobalVariable *widenGlobalVariable(GlobalVariable *OldVar,
-                                           unsigned NumBytesToPad,
-                                           unsigned NumBytesToCopy) {
-  if (!OldVar->hasInitializer())
-    return nullptr;
-
-  ConstantDataArray *DataArray =
-      dyn_cast<ConstantDataArray>(OldVar->getInitializer());
-  if (!DataArray)
-    return nullptr;
-
-  // Update to be word aligned (memcpy(...,X,...))
-  // create replacement with padded null bytes.
-  StringRef Data = DataArray->getRawDataValues();
-  std::vector<uint8_t> StrData(Data.begin(), Data.end());
-  for (unsigned int p = 0; p < NumBytesToPad; p++)
-    StrData.push_back('\0');
-  auto Arr = ArrayRef(StrData.data(), NumBytesToCopy + NumBytesToPad);
-  // Create new padded version of global variable.
-  Constant *SourceReplace = ConstantDataArray::get(OldVar->getContext(), Arr);
-  GlobalVariable *NewGV = new GlobalVariable(
-      *(OldVar->getParent()), SourceReplace->getType(), true,
-      OldVar->getLinkage(), SourceReplace, SourceReplace->getName());
-  // Copy any other attributes from original global variable
-  // e.g. unamed_addr
-  NewGV->copyAttributesFrom(OldVar);
-  NewGV->takeName(OldVar);
-  return NewGV;
-}
-
-static void widenDestArray(CallInst *CI, const unsigned NumBytesToPad,
-                           const unsigned NumBytesToCopy,
-                           ConstantDataArray *SourceDataArray) {
-
-  auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
-  if (Alloca) {
-    unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
-    unsigned int TotalBytes = NumBytesToCopy + NumBytesToPad;
-    unsigned NumElementsToCopy = divideCeil(TotalBytes, ElementByteWidth);
-    // Update destination array to be word aligned (memcpy(X,...,...))
-    IRBuilder<> BuildAlloca(Alloca);
-    AllocaInst *NewAlloca = BuildAlloca.CreateAlloca(ArrayType::get(
-        Alloca->getAllocatedType()->getArrayElementType(), NumElementsToCopy));
-    NewAlloca->takeName(Alloca);
-    NewAlloca->setAlignment(Alloca->getAlign());
-    Alloca->replaceAllUsesWith(NewAlloca);
-    Alloca->eraseFromParent();
-  }
-}
-
-static bool tryWidenGlobalArrayAndDests(GlobalVariable *SourceVar,
-                                        const unsigned NumBytesToPad,
-                                        const unsigned NumBytesToCopy,
-                                        ConstantInt *BytesToCopyOp,
-                                        ConstantDataArray *SourceDataArray) {
-  auto *NewSourceGV =
-      widenGlobalVariable(SourceVar, NumBytesToPad, NumBytesToCopy);
-  if (!NewSourceGV)
-    return false;
-
-  // Update arguments of remaining uses  that
-  // are memcpys.
-  for (auto *User : SourceVar->users()) {
-    auto *CI = dyn_cast<CallInst>(User);
-    if (!callInstIsMemcpy(CI) || !destArrayCanBeWidened(CI))
-      continue;
-
-    if (CI->getArgOperand(1) != SourceVar)
-      continue;
-
-    widenDestArray(CI, NumBytesToPad, NumBytesToCopy, SourceDataArray);
-
-    CI->setArgOperand(2, ConstantInt::get(BytesToCopyOp->getType(),
-                                          NumBytesToCopy + NumBytesToPad));
-  }
-  SourceVar->replaceAllUsesWith(NewSourceGV);
-
-  NumGlobalArraysPadded++;
-  return true;
-}
-
-static bool tryWidenGlobalArraysUsedByMemcpy(
-    GlobalVariable *GV,
-    function_ref<TargetTransformInfo &(Function &)> GetTTI) {
-
-  if (!GV->hasInitializer() || !GV->isConstant() || !GV->hasLocalLinkage() ||
-      !GV->hasGlobalUnnamedAddr())
-    return false;
-
-  for (auto *User : GV->users()) {
-    CallInst *CI = dyn_cast<CallInst>(User);
-    if (!callInstIsMemcpy(CI) || !destArrayCanBeWidened(CI))
-      continue;
-
-    auto *BytesToCopyOp = dyn_cast<ConstantInt>(CI->getArgOperand(2));
-    if (!BytesToCopyOp)
-      continue;
-
-    ConstantDataArray *SourceDataArray =
-        dyn_cast<ConstantDataArray>(GV->getInitializer());
-    if (!SourceDataArray)
-      continue;
-
-    unsigned NumBytesToCopy = BytesToCopyOp->getZExtValue();
-
-    auto *Alloca = cast<AllocaInst>(CI->getArgOperand(0));
-    uint64_t DZSize = Alloca->getAllocatedType()->getArrayNumElements();
-    uint64_t SZSize = SourceDataArray->getType()->getNumElements();
-    unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
-    // Calculate the number of elements to copy while avoiding floored
-    // division of integers returning wrong values i.e. copying one byte
-    // from an array of i16 would yield 0 elements to copy as supposed to 1.
-    unsigned NumElementsToCopy = divideCeil(NumBytesToCopy, ElementByteWidth);
-
-    // For safety purposes lets add a constraint and only pad when
-    // NumElementsToCopy == destination array size ==
-    // source which is a constant
-    if (NumElementsToCopy != DZSize || DZSize != SZSize)
-      continue;
-
-    unsigned NumBytesToPad =
-        GetTTI(*CI->getFunction())
-            .getNumBytesToPadGlobalArray(NumBytesToCopy,
-                                         SourceDataArray->getType());
-    if (NumBytesToPad) {
-      return tryWidenGlobalArrayAndDests(GV, NumBytesToPad, NumBytesToCopy,
-                                         BytesToCopyOp, SourceDataArray);
-    }
-  }
-  return false;
-}
-
 static bool
 OptimizeGlobalVars(Module &M,
                    function_ref<TargetTransformInfo &(Function &)> GetTTI,
@@ -2225,10 +2064,6 @@ OptimizeGlobalVars(Module &M,
       continue;
     }
 
-    // For global variable arrays called in a memcpy
-    // we try to pad to nearest valid alignment boundary
-    Changed |= tryWidenGlobalArraysUsedByMemcpy(&GV, GetTTI);
-
     Changed |= processGlobal(GV, GetTTI, GetTLI, LookupDomTree);
   }
   return Changed;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 418302d1edb3f..981c5271fb3f6 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1194,7 +1194,9 @@ Value *InstCombinerImpl::SimplifyAddWithRemainder(BinaryOperator &I) {
   Value *DivOpV;
   APInt DivOpC;
   if (MatchRem(Rem, X, C0, IsSigned) &&
-      MatchDiv(Div, DivOpV, DivOpC, IsSigned) && X == DivOpV && C0 == DivOpC) {
+      MatchDiv(Div, DivOpV, DivOpC, IsSigned) && X == DivOpV && C0 == DivOpC &&
+      // Avoid unprofitable replacement of and with mul.
+      !(C1.isOne() && !IsSigned && DivOpC.isPowerOf2() && DivOpC != 2)) {
     APInt NewC = C1 - C2 * C0;
     if (!NewC.isZero() && !Rem->hasOneUse())
       return nullptr;
@@ -2164,10 +2166,14 @@ Value *InstCombinerImpl::OptimizePointerDifference(Value *LHS, Value *RHS,
 
   // If this is a single inbounds GEP and the original sub was nuw,
   // then the final multiplication is also nuw.
-  if (auto *I = dyn_cast<Instruction>(Result))
+  if (auto *I = dyn_cast<OverflowingBinaryOperator>(Result))
     if (IsNUW && match(Offset2, m_Zero()) && Base.LHSNW.isInBounds() &&
-        I->getOpcode() == Instruction::Mul)
-      I->setHasNoUnsignedWrap();
+        (I->use_empty() || I->hasOneUse()) && I->hasNoSignedWrap() &&
+        !I->hasNoUnsignedWrap() &&
+        ((I->getOpcode() == Instruction::Mul &&
+          match(I->getOperand(1), m_NonNegative())) ||
+         I->getOpcode() == Instruction::Shl))
+      cast<Instruction>(I)->setHasNoUnsignedWrap();
 
   // If we have a 2nd GEP of the same base pointer, subtract the offsets.
   // If both GEPs are inbounds, then the subtract does not have signed overflow.
@@ -3044,6 +3050,17 @@ Instruction *InstCombinerImpl::visitFNeg(UnaryOperator &I) {
     return replaceInstUsesWith(I, NewCopySign);
   }
 
+  // fneg (shuffle x, Mask) --> shuffle (fneg x), Mask
+  ArrayRef<int> Mask;
+  if (match(OneUse, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))
+    return new ShuffleVectorInst(Builder.CreateFNegFMF(X, &I), Mask);
+
+  // fneg (reverse x) --> reverse (fneg x)
+  if (match(OneUse, m_VecReverse(m_Value(X)))) {
+    Value *Reverse = Builder.CreateVectorReverse(Builder.CreateFNegFMF(X, &I));
+    return replaceInstUsesWith(I, Reverse);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index dce695a036006..706cb828acc63 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -681,7 +681,7 @@ static Value *foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd,
         }
         Value *NewAnd = Builder.CreateAnd(A, BD);
         Value *CEVal = ConstantInt::get(A->getType(), CE);
-        return Builder.CreateICmp(CC, CEVal, NewAnd);
+        return Builder.CreateICmp(CC, NewAnd, CEVal);
       };
 
       if (Mask & BMask_Mixed)
@@ -3602,6 +3602,11 @@ struct DecomposedBitMaskMul {
   APInt Mask;
   bool NUW;
   bool NSW;
+
+  bool isCombineableWith(const DecomposedBitMaskMul Other) {
+    return X == Other.X && !Mask.intersects(Other.Mask) &&
+           Factor == Other.Factor;
+  }
 };
 
 static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
@@ -3659,6 +3664,59 @@ static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
   return std::nullopt;
 }
 
+/// (A & N) * C + (A & M) * C -> (A & (N + M)) & C
+/// This also accepts the equivalent select form of (A & N) * C
+/// expressions i.e. !(A & N) ? 0 : N * C)
+static Value *foldBitmaskMul(Value *Op0, Value *Op1,
+                             InstCombiner::BuilderTy &Builder) {
+  auto Decomp1 = matchBitmaskMul(Op1);
+  if (!Decomp1)
+    return nullptr;
+
+  auto Decomp0 = matchBitmaskMul(Op0);
+  if (!Decomp0)
+    return nullptr;
+
+  if (Decomp0->isCombineableWith(*Decomp1)) {
+    Value *NewAnd = Builder.CreateAnd(
+        Decomp0->X,
+        ConstantInt::get(Decomp0->X->getType(), Decomp0->Mask + Decomp1->Mask));
+
+    return Builder.CreateMul(
+        NewAnd, ConstantInt::get(NewAnd->getType(), Decomp1->Factor), "",
+        Decomp0->NUW && Decomp1->NUW, Decomp0->NSW && Decomp1->NSW);
+  }
+
+  return nullptr;
+}
+
+Value *InstCombinerImpl::foldDisjointOr(Value *LHS, Value *RHS) {
+  if (Value *Res = foldBitmaskMul(LHS, RHS, Builder))
+    return Res;
+
+  return nullptr;
+}
+
+Value *InstCombinerImpl::reassociateDisjointOr(Value *LHS, Value *RHS) {
+
+  Value *X, *Y;
+  if (match(RHS, m_OneUse(m_DisjointOr(m_Value(X), m_Value(Y))))) {
+    if (Value *Res = foldDisjointOr(LHS, X))
+      return Builder.CreateOr(Res, Y, "", /*IsDisjoint=*/true);
+    if (Value *Res = foldDisjointOr(LHS, Y))
+      return Builder.CreateOr(Res, X, "", /*IsDisjoint=*/true);
+  }
+
+  if (match(LHS, m_OneUse(m_DisjointOr(m_Value(X), m_Value(Y))))) {
+    if (Value *Res = foldDisjointOr(X, RHS))
+      return Builder.CreateOr(Res, Y, "", /*IsDisjoint=*/true);
+    if (Value *Res = foldDisjointOr(Y, RHS))
+      return Builder.CreateOr(Res, X, "", /*IsDisjoint=*/true);
+  }
+
+  return nullptr;
+}
+
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
@@ -3741,28 +3799,11 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
                                    /*NSW=*/true, /*NUW=*/true))
       return R;
 
-    // (A & N) * C + (A & M) * C -> (A & (N + M)) & C
-    // This also accepts the equivalent select form of (A & N) * C
-    // expressions i.e. !(A & N) ? 0 : N * C)
-    auto Decomp1 = matchBitmaskMul(I.getOperand(1));
-    if (Decomp1) {
-      auto Decomp0 = matchBitmaskMul(I.getOperand(0));
-      if (Decomp0 && Decomp0->X == Decomp1->X &&
-          (Decomp0->Mask & Decomp1->Mask).isZero() &&
-          Decomp0->Factor == Decomp1->Factor) {
-
-        Value *NewAnd = Builder.CreateAnd(
-            Decomp0->X, ConstantInt::get(Decomp0->X->getType(),
-                                         (Decomp0->Mask + Decomp1->Mask)));
-
-        auto *Combined = BinaryOperator::CreateMul(
-            NewAnd, ConstantInt::get(NewAnd->getType(), Decomp1->Factor));
-
-        Combined->setHasNoUnsignedWrap(Decomp0->NUW && Decomp1->NUW);
-        Combined->setHasNoSignedWrap(Decomp0->NSW && Decomp1->NSW);
-        return Combined;
-      }
-    }
+    if (Value *Res = foldBitmaskMul(I.getOperand(0), I.getOperand(1), Builder))
+      return replaceInstUsesWith(I, Res);
+
+    if (Value *Res = reassociateDisjointOr(I.getOperand(0), I.getOperand(1)))
+      return replaceInstUsesWith(I, Res);
   }
 
   Value *X, *Y;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b6ed1dc4331d2..e521c9d7001ac 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1404,11 +1404,6 @@ InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
       !II->getCalledFunction()->isSpeculatable())
     return nullptr;
 
-  // fabs is canonicalized to fabs (shuffle ...) in foldShuffleOfUnaryOps, so
-  // avoid undoing it.
-  if (match(II, m_FAbs(m_Value())))
-    return nullptr;
-
   Value *X;
   Constant *C;
   ArrayRef<int> Mask;
@@ -1461,6 +1456,43 @@ InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
   return new ShuffleVectorInst(NewIntrinsic, Mask);
 }
 
+/// If all arguments of the intrinsic are reverses, try to pull the reverse
+/// after the intrinsic.
+Value *InstCombinerImpl::foldReversedIntrinsicOperands(IntrinsicInst *II) {
+  if (!isTriviallyVectorizable(II->getIntrinsicID()))
+    return nullptr;
+
+  // At least 1 operand must be a reverse with 1 use because we are creating 2
+  // instructions.
+  if (none_of(II->args(), [](Value *V) {
+        return match(V, m_OneUse(m_VecReverse(m_Value())));
+      }))
+    return nullptr;
+
+  Value *X;
+  Constant *C;
+  SmallVector<Value *> NewArgs;
+  for (Use &Arg : II->args()) {
+    if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+                                           Arg.getOperandNo(), nullptr))
+      NewArgs.push_back(Arg);
+    else if (match(&Arg, m_VecReverse(m_Value(X))))
+      NewArgs.push_back(X);
+    else if (isSplatValue(Arg))
+      NewArgs.push_back(Arg);
+    else if (match(&Arg, m_ImmConstant(C)))
+      NewArgs.push_back(Builder.CreateVectorReverse(C));
+    else
+      return nullptr;
+  }
+
+  // intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)
+  Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
+  Instruction *NewIntrinsic = Builder.CreateIntrinsic(
+      II->getType(), II->getIntrinsicID(), NewArgs, FPI);
+  return Builder.CreateVectorReverse(NewIntrinsic);
+}
+
 /// Fold the following cases and accepts bswap and bitreverse intrinsics:
 ///   bswap(logic_op(bswap(x), y)) --> logic_op(x, bswap(y))
 ///   bswap(logic_op(bswap(x), bswap(y))) --> logic_op(x, y) (ignores multiuse)
@@ -1972,6 +2004,21 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       return BinaryOperator::CreateOr(I0, I1);
     }
 
+    // smin(smax(X, -1), 1) -> scmp(X, 0)
+    // smax(smin(X, 1), -1) -> scmp(X, 0)
+    // At this point, smax(smin(X, 1), -1) is changed to smin(smax(X, -1)
+    // And i1's have been changed to and/ors
+    // So we only need to check for smin
+    if (IID == Intrinsic::smin) {
+      if (match(I0, m_OneUse(m_SMax(m_Value(X), m_AllOnes()))) &&
+          match(I1, m_One())) {
+        Value *Zero = ConstantInt::get(X->getType(), 0);
+        return replaceInstUsesWith(
+            CI,
+            Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {X, Zero}));
+      }
+    }
+
     if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
       // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y)
       // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y)
@@ -3554,23 +3601,6 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     }
     break;
   }
-  case Intrinsic::vector_reverse: {
-    Value *Vec = II->getArgOperand(0);
-    // Note: We canonicalize reverse after binops, so we don't need a
-    // corresponding binop case here. TODO: Consider canonicalizing
-    // reverse after fneg?
-
-    // rev(unop rev(X)) --> unop X
-    Value *X;
-    if (match(Vec, m_OneUse(m_UnOp(m_VecReverse(m_Value(X)))))) {
-      auto *OldUnOp = cast<UnaryOperator>(Vec);
-      auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags(
-          OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(),
-          II->getIterator());
-      return replaceInstUsesWith(CI, NewUnOp);
-    }
-    break;
-  }
   case Intrinsic::experimental_vp_reverse: {
     Value *X;
     Value *Vec = II->getArgOperand(0);
@@ -3578,6 +3608,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     if (!match(Mask, m_AllOnes()))
       break;
     Value *EVL = II->getArgOperand(2);
+    // TODO: Canonicalize experimental.vp.reverse after unop/binops?
     // rev(unop rev(X)) --> unop X
     if (match(Vec,
               m_OneUse(m_UnOp(m_Intrinsic<Intrinsic::experimental_vp_reverse>(
@@ -3872,6 +3903,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   if (Instruction *Shuf = foldShuffledIntrinsicOperands(II))
     return Shuf;
 
+  if (Value *Reverse = foldReversedIntrinsicOperands(II))
+    return replaceInstUsesWith(*II, Reverse);
+
   // Some intrinsics (like experimental_gc_statepoint) can be used in invoke
   // context, so it is handled in visitCallBase and we should trigger it.
   return visitCallBase(*II);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 0894ca92086f3..c6f317a668cfe 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -708,11 +708,13 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
     return I;
   };
 
-  Value *PtrBase = GEPLHS->getOperand(0);
-  if (PtrBase == RHS && CanFold(GEPLHS->getNoWrapFlags())) {
+  CommonPointerBase Base = CommonPointerBase::compute(GEPLHS, RHS);
+  if (Base.Ptr == RHS && CanFold(Base.LHSNW)) {
     // ((gep Ptr, OFFSET) cmp Ptr)   ---> (OFFSET cmp 0).
-    Value *Offset = EmitGEPOffset(GEPLHS);
-    return NewICmp(GEPLHS->getNoWrapFlags(), Offset,
+    Type *IdxTy = DL.getIndexType(GEPLHS->getType());
+    Value *Offset =
+        EmitGEPOffsets(Base.LHSGEPs, Base.LHSNW, IdxTy, /*RewriteGEPs=*/true);
+    return NewICmp(Base.LHSNW, Offset,
                    Constant::getNullValue(Offset->getType()));
   }
 
@@ -750,6 +752,7 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
 
     // If the base pointers are different, but the indices are the same, just
     // compare the base pointer.
+    Value *PtrBase = GEPLHS->getOperand(0);
     if (PtrBase != GEPRHS->getOperand(0)) {
       bool IndicesTheSame =
           GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
@@ -5117,6 +5120,18 @@ static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q,
   return nullptr;
 }
 
+/// Return true if X is a multiple of C.
+/// TODO: Handle non-power-of-2 factors.
+static bool isMultipleOf(Value *X, const APInt &C, const SimplifyQuery &Q) {
+  if (C.isOne())
+    return true;
+
+  if (!C.isPowerOf2())
+    return false;
+
+  return MaskedValueIsZero(X, C - 1, Q);
+}
+
 /// Try to fold icmp (binop), X or icmp X, (binop).
 /// TODO: A large part of this logic is duplicated in InstSimplify's
 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
@@ -5275,66 +5290,62 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
     return new ICmpInst(Pred, Y, Z);
   }
 
-  // icmp slt (A + -1), Op1 -> icmp sle A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
-      match(B, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
-
-  // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
-      match(B, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
-
-  // icmp sle (A + 1), Op1 -> icmp slt A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
-
-  // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
+  if (ICmpInst::isRelational(Pred)) {
+    // Return if both X and Y is divisible by Z/-Z.
+    // TODO: Generalize to check if (X - Y) is divisible by Z/-Z.
+    auto ShareCommonDivisor = [&Q](Value *X, Value *Y, Value *Z,
+                                   bool IsNegative) -> bool {
+      const APInt *OffsetC;
+      if (!match(Z, m_APInt(OffsetC)))
+        return false;
 
-  // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
-      match(D, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
+      // Fast path for Z == 1/-1.
+      if (IsNegative ? OffsetC->isAllOnes() : OffsetC->isOne())
+        return true;
 
-  // icmp sle Op0, (C + -1) -> icmp slt Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
-      match(D, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
+      APInt C = *OffsetC;
+      if (IsNegative)
+        C.negate();
+      // Note: -INT_MIN is also negative.
+      if (!C.isStrictlyPositive())
+        return false;
 
-  // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
+      return isMultipleOf(X, C, Q) && isMultipleOf(Y, C, Q);
+    };
 
-  // icmp slt Op0, (C + 1) -> icmp sle Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
+    // TODO: The subtraction-related identities shown below also hold, but
+    // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
+    // wouldn't happen even if they were implemented.
+    //
+    // icmp ult (A - 1), Op1 -> icmp ule A, Op1
+    // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
+    // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
+    // icmp ule Op0, (C - 1) -> icmp ult Op0, C
+
+    // icmp slt (A + -1), Op1 -> icmp sle A, Op1
+    // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
+    // icmp sle (A + 1), Op1 -> icmp slt A, Op1
+    // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
+    // icmp ule (A + 1), Op0 -> icmp ult A, Op1
+    // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
+    if (A && NoOp0WrapProblem &&
+        ShareCommonDivisor(A, Op1, B,
+                           ICmpInst::isLT(Pred) || ICmpInst::isGE(Pred)))
+      return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(Pred), A,
+                          Op1);
 
-  // TODO: The subtraction-related identities shown below also hold, but
-  // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
-  // wouldn't happen even if they were implemented.
-  //
-  // icmp ult (A - 1), Op1 -> icmp ule A, Op1
-  // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
-  // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
-  // icmp ule Op0, (C - 1) -> icmp ult Op0, C
-
-  // icmp ule (A + 1), Op0 -> icmp ult A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
-
-  // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
-
-  // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
-
-  // icmp ult Op0, (C + 1) -> icmp ule Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
+    // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
+    // icmp sle Op0, (C + -1) -> icmp slt Op0, C
+    // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
+    // icmp slt Op0, (C + 1) -> icmp sle Op0, C
+    // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
+    // icmp ult Op0, (C + 1) -> icmp ule Op0, C
+    if (C && NoOp1WrapProblem &&
+        ShareCommonDivisor(Op0, C, D,
+                           ICmpInst::isGT(Pred) || ICmpInst::isLE(Pred)))
+      return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(Pred), Op0,
+                          C);
+  }
 
   // if C1 has greater magnitude than C2:
   //  icmp (A + C1), (C + C2) -> icmp (A + C3), C
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 8c9de862fe8f2..1b963952f1d61 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -148,6 +148,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Instruction *foldItoFPtoI(CastInst &FI);
   Instruction *visitSelectInst(SelectInst &SI);
   Instruction *foldShuffledIntrinsicOperands(IntrinsicInst *II);
+  Value *foldReversedIntrinsicOperands(IntrinsicInst *II);
   Instruction *visitCallInst(CallInst &CI);
   Instruction *visitInvokeInst(InvokeInst &II);
   Instruction *visitCallBrInst(CallBrInst &CBI);
@@ -438,6 +439,10 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Value *reassociateBooleanAndOr(Value *LHS, Value *X, Value *Y, Instruction &I,
                                  bool IsAnd, bool RHSIsLogical);
 
+  Value *foldDisjointOr(Value *LHS, Value *RHS);
+
+  Value *reassociateDisjointOr(Value *LHS, Value *RHS);
+
   Instruction *
   canonicalizeConditionalNegationViaMathToSelect(BinaryOperator &i);
 
@@ -620,6 +625,20 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Instruction *foldOpIntoPhi(Instruction &I, PHINode *PN,
                              bool AllowMultipleUses = false);
 
+  /// Try to fold binary operators whose operands are simple interleaved
+  /// recurrences to a single recurrence. This is a common pattern in reduction
+  /// operations.
+  /// Example:
+  ///   %phi1 = phi [init1, %BB1], [%op1, %BB2]
+  ///   %phi2 = phi [init2, %BB1], [%op2, %BB2]
+  ///   %op1 = binop %phi1, constant1
+  ///   %op2 = binop %phi2, constant2
+  ///   %rdx = binop %op1, %op2
+  /// -->
+  ///   %phi_combined = phi [init_combined, %BB1], [%op_combined, %BB2]
+  ///   %rdx_combined = binop %phi_combined, constant_combined
+  Instruction *foldBinopWithRecurrence(BinaryOperator &BO);
+
   /// For a binary operator with 2 phi operands, try to hoist the binary
   /// operation before the phi. This can result in fewer instructions in
   /// patterns where at least one set of phi operands simplifies.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index f946c3856948b..1a8d4215b5b71 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -542,27 +542,39 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
         }
       }
     } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
-      // If this is extracting an element from a shufflevector, figure out where
-      // it came from and extract from the appropriate input element instead.
-      // Restrict the following transformation to fixed-length vector.
-      if (isa<FixedVectorType>(SVI->getType()) && isa<ConstantInt>(Index)) {
-        int SrcIdx =
-            SVI->getMaskValue(cast<ConstantInt>(Index)->getZExtValue());
-        Value *Src;
-        unsigned LHSWidth = cast<FixedVectorType>(SVI->getOperand(0)->getType())
-                                ->getNumElements();
-
-        if (SrcIdx < 0)
-          return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
-        if (SrcIdx < (int)LHSWidth)
-          Src = SVI->getOperand(0);
-        else {
-          SrcIdx -= LHSWidth;
-          Src = SVI->getOperand(1);
+      int SplatIndex = getSplatIndex(SVI->getShuffleMask());
+      // We know the all-0 splat must be reading from the first operand, even
+      // in the case of scalable vectors (vscale is always > 0).
+      if (SplatIndex == 0)
+        return ExtractElementInst::Create(SVI->getOperand(0),
+                                          Builder.getInt64(0));
+
+      if (isa<FixedVectorType>(SVI->getType())) {
+        std::optional<int> SrcIdx;
+        // getSplatIndex returns -1 to mean not-found.
+        if (SplatIndex != -1)
+          SrcIdx = SplatIndex;
+        else if (ConstantInt *CI = dyn_cast<ConstantInt>(Index))
+          SrcIdx = SVI->getMaskValue(CI->getZExtValue());
+
+        if (SrcIdx) {
+          Value *Src;
+          unsigned LHSWidth =
+              cast<FixedVectorType>(SVI->getOperand(0)->getType())
+                  ->getNumElements();
+
+          if (*SrcIdx < 0)
+            return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
+          if (*SrcIdx < (int)LHSWidth)
+            Src = SVI->getOperand(0);
+          else {
+            *SrcIdx -= LHSWidth;
+            Src = SVI->getOperand(1);
+          }
+          Type *Int64Ty = Type::getInt64Ty(EI.getContext());
+          return ExtractElementInst::Create(
+              Src, ConstantInt::get(Int64Ty, *SrcIdx, false));
         }
-        Type *Int64Ty = Type::getInt64Ty(EI.getContext());
-        return ExtractElementInst::Create(
-            Src, ConstantInt::get(Int64Ty, SrcIdx, false));
       }
     } else if (auto *CI = dyn_cast<CastInst>(I)) {
       // Canonicalize extractelement(cast) -> cast(extractelement).
@@ -2533,20 +2545,6 @@ static Instruction *foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf,
 
   bool IsFNeg = S0->getOpcode() == Instruction::FNeg;
 
-  // Match 1-input (unary) shuffle.
-  // shuffle (fneg/fabs X), Mask --> fneg/fabs (shuffle X, Mask)
-  if (S0->hasOneUse() && match(Shuf.getOperand(1), m_Poison())) {
-    Value *NewShuf = Builder.CreateShuffleVector(X, Shuf.getShuffleMask());
-    if (IsFNeg)
-      return UnaryOperator::CreateFNegFMF(NewShuf, S0);
-
-    Function *FAbs = Intrinsic::getOrInsertDeclaration(
-        Shuf.getModule(), Intrinsic::fabs, Shuf.getType());
-    CallInst *NewF = CallInst::Create(FAbs, {NewShuf});
-    NewF->setFastMathFlags(S0->getFastMathFlags());
-    return NewF;
-  }
-
   // Match 2-input (binary) shuffle.
   auto *S1 = dyn_cast<Instruction>(Shuf.getOperand(1));
   Value *Y;
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 40578e5edc3ab..91a1b61ddc483 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -205,9 +205,8 @@ Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
     Builder.SetInsertPoint(Inst);
 
   Value *Offset = EmitGEPOffset(GEP);
-  // If a non-trivial GEP has other uses, rewrite it to avoid duplicating
-  // the offset arithmetic.
-  if (Inst && !GEP->hasOneUse() && !GEP->hasAllConstantIndices() &&
+  // Rewrite non-trivial GEPs to avoid duplicating the offset arithmetic.
+  if (Inst && !GEP->hasAllConstantIndices() &&
       !GEP->getSourceElementType()->isIntegerTy(8)) {
     replaceInstUsesWith(
         *Inst, Builder.CreateGEP(Builder.getInt8Ty(), GEP->getPointerOperand(),
@@ -1989,7 +1988,114 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN,
   return replaceInstUsesWith(I, NewPN);
 }
 
+Instruction *InstCombinerImpl::foldBinopWithRecurrence(BinaryOperator &BO) {
+  if (!BO.isAssociative())
+    return nullptr;
+
+  // Find the interleaved binary ops.
+  auto Opc = BO.getOpcode();
+  auto *BO0 = dyn_cast<BinaryOperator>(BO.getOperand(0));
+  auto *BO1 = dyn_cast<BinaryOperator>(BO.getOperand(1));
+  if (!BO0 || !BO1 || !BO0->hasNUses(2) || !BO1->hasNUses(2) ||
+      BO0->getOpcode() != Opc || BO1->getOpcode() != Opc ||
+      !BO0->isAssociative() || !BO1->isAssociative() ||
+      BO0->getParent() != BO1->getParent())
+    return nullptr;
+
+  assert(BO.isCommutative() && BO0->isCommutative() && BO1->isCommutative() &&
+         "Expected commutative instructions!");
+
+  // Find the matching phis, forming the recurrences.
+  PHINode *PN0, *PN1;
+  Value *Start0, *Step0, *Start1, *Step1;
+  if (!matchSimpleRecurrence(BO0, PN0, Start0, Step0) || !PN0->hasOneUse() ||
+      !matchSimpleRecurrence(BO1, PN1, Start1, Step1) || !PN1->hasOneUse() ||
+      PN0->getParent() != PN1->getParent())
+    return nullptr;
+
+  assert(PN0->getNumIncomingValues() == 2 && PN1->getNumIncomingValues() == 2 &&
+         "Expected PHIs with two incoming values!");
+
+  // Convert the start and step values to constants.
+  auto *Init0 = dyn_cast<Constant>(Start0);
+  auto *Init1 = dyn_cast<Constant>(Start1);
+  auto *C0 = dyn_cast<Constant>(Step0);
+  auto *C1 = dyn_cast<Constant>(Step1);
+  if (!Init0 || !Init1 || !C0 || !C1)
+    return nullptr;
+
+  // Fold the recurrence constants.
+  auto *Init = ConstantFoldBinaryInstruction(Opc, Init0, Init1);
+  auto *C = ConstantFoldBinaryInstruction(Opc, C0, C1);
+  if (!Init || !C)
+    return nullptr;
+
+  // Create the reduced PHI.
+  auto *NewPN = PHINode::Create(PN0->getType(), PN0->getNumIncomingValues(),
+                                "reduced.phi");
+
+  // Create the new binary op.
+  auto *NewBO = BinaryOperator::Create(Opc, NewPN, C);
+  if (Opc == Instruction::FAdd || Opc == Instruction::FMul) {
+    // Intersect FMF flags for FADD and FMUL.
+    FastMathFlags Intersect = BO0->getFastMathFlags() &
+                              BO1->getFastMathFlags() & BO.getFastMathFlags();
+    NewBO->setFastMathFlags(Intersect);
+  } else {
+    OverflowTracking Flags;
+    Flags.AllKnownNonNegative = false;
+    Flags.AllKnownNonZero = false;
+    Flags.mergeFlags(*BO0);
+    Flags.mergeFlags(*BO1);
+    Flags.mergeFlags(BO);
+    Flags.applyFlags(*NewBO);
+  }
+  NewBO->takeName(&BO);
+
+  for (unsigned I = 0, E = PN0->getNumIncomingValues(); I != E; ++I) {
+    auto *V = PN0->getIncomingValue(I);
+    auto *BB = PN0->getIncomingBlock(I);
+    if (V == Init0) {
+      assert(((PN1->getIncomingValue(0) == Init1 &&
+               PN1->getIncomingBlock(0) == BB) ||
+              (PN1->getIncomingValue(1) == Init1 &&
+               PN1->getIncomingBlock(1) == BB)) &&
+             "Invalid incoming block!");
+      NewPN->addIncoming(Init, BB);
+    } else if (V == BO0) {
+      assert(((PN1->getIncomingValue(0) == BO1 &&
+               PN1->getIncomingBlock(0) == BB) ||
+              (PN1->getIncomingValue(1) == BO1 &&
+               PN1->getIncomingBlock(1) == BB)) &&
+             "Invalid incoming block!");
+      NewPN->addIncoming(NewBO, BB);
+    } else
+      llvm_unreachable("Unexpected incoming value!");
+  }
+
+  LLVM_DEBUG(dbgs() << "  Combined " << *PN0 << "\n           " << *BO0
+                    << "\n      with " << *PN1 << "\n           " << *BO1
+                    << '\n');
+
+  // Insert the new recurrence and remove the old (dead) ones.
+  InsertNewInstWith(NewPN, PN0->getIterator());
+  InsertNewInstWith(NewBO, BO0->getIterator());
+
+  eraseInstFromFunction(
+      *replaceInstUsesWith(*BO0, PoisonValue::get(BO0->getType())));
+  eraseInstFromFunction(
+      *replaceInstUsesWith(*BO1, PoisonValue::get(BO1->getType())));
+  eraseInstFromFunction(*PN0);
+  eraseInstFromFunction(*PN1);
+
+  return replaceInstUsesWith(BO, NewBO);
+}
+
 Instruction *InstCombinerImpl::foldBinopWithPhiOperands(BinaryOperator &BO) {
+  // Attempt to fold binary operators whose operands are simple recurrences.
+  if (auto *NewBO = foldBinopWithRecurrence(BO))
+    return NewBO;
+
   // TODO: This should be similar to the incoming values check in foldOpIntoPhi:
   //       we are guarding against replicating the binop in >1 predecessor.
   //       This could miss matching a phi with 2 constant incoming values.
diff --git a/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp b/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
index 196f58a8f1d1c..e7a6fa48e9004 100644
--- a/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
+++ b/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
@@ -74,7 +74,7 @@ static void emitRemark(IntrinsicInst *II, OptimizationRemarkEmitter &ORE,
 static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
                              const ProfileSummaryInfo *PSI,
                              OptimizationRemarkEmitter &ORE,
-                             const std::vector<unsigned int> &cutoffs) {
+                             const LowerAllowCheckPass::Options &Opts) {
   SmallVector<std::pair<IntrinsicInst *, bool>, 16> ReplaceWithValue;
   std::unique_ptr<RandomNumberGenerator> Rng;
 
@@ -89,8 +89,10 @@ static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
       return HotPercentileCutoff;
     else if (II->getIntrinsicID() == Intrinsic::allow_ubsan_check) {
       auto *Kind = cast<ConstantInt>(II->getArgOperand(0));
-      if (Kind->getZExtValue() < cutoffs.size())
-        return cutoffs[Kind->getZExtValue()];
+      if (Kind->getZExtValue() < Opts.cutoffs.size())
+        return Opts.cutoffs[Kind->getZExtValue()];
+    } else if (II->getIntrinsicID() == Intrinsic::allow_runtime_check) {
+      return Opts.runtime_check;
     }
 
     return 0;
@@ -157,7 +159,7 @@ PreservedAnalyses LowerAllowCheckPass::run(Function &F,
   OptimizationRemarkEmitter &ORE =
       AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
 
-  return removeUbsanTraps(F, BFI, PSI, ORE, Opts.cutoffs)
+  return removeUbsanTraps(F, BFI, PSI, ORE, Opts)
              // We do not change the CFG, we only replace the intrinsics with
              // true or false.
              ? PreservedAnalyses::none().preserveSet<CFGAnalyses>()
@@ -193,5 +195,11 @@ void LowerAllowCheckPass::printPipeline(
 
     i++;
   }
+  if (Opts.runtime_check) {
+    if (printed)
+      OS << ";";
+    OS << "runtime_check=" << Opts.runtime_check;
+  }
+
   OS << '>';
 }
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index ca655b3597671..36fb7d11b488a 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -158,6 +158,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/bit.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -1392,9 +1393,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       std::tie(ShadowPtr, OriginPtr) =
           getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
 
-      StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
+      [[maybe_unused]] StoreInst *NewSI =
+          IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
       LLVM_DEBUG(dbgs() << "  STORE: " << *NewSI << "\n");
-      (void)NewSI;
 
       if (SI->isAtomic())
         SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
@@ -2021,18 +2022,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       Value *Shadow = ShadowMap[V];
       if (!Shadow) {
         LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
-        (void)I;
         assert(Shadow && "No shadow for a value");
       }
       return Shadow;
     }
     // Handle fully undefined values
     // (partially undefined constant vectors are handled later)
-    if (UndefValue *U = dyn_cast<UndefValue>(V)) {
+    if ([[maybe_unused]] UndefValue *U = dyn_cast<UndefValue>(V)) {
       Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
                                                         : getCleanShadow(V);
       LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
-      (void)U;
       return AllOnes;
     }
     if (Argument *A = dyn_cast<Argument>(V)) {
@@ -2081,10 +2080,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
             } else {
               Value *Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
               const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
-              Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
-                                                 CopyAlign, Size);
+              [[maybe_unused]] Value *Cpy = EntryIRB.CreateMemCpy(
+                  CpShadowPtr, CopyAlign, Base, CopyAlign, Size);
               LLVM_DEBUG(dbgs() << "  ByValCpy: " << *Cpy << "\n");
-              (void)Cpy;
 
               if (MS.TrackOrigins) {
                 Value *OriginPtr = getOriginPtrForArgument(EntryIRB, ArgOffset);
@@ -2182,7 +2180,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   ///
   /// This location will be later instrumented with a check that will print a
   /// UMR warning in runtime if the shadow value is not 0.
-  void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
+  void insertCheckShadow(Value *Shadow, Value *Origin, Instruction *OrigIns) {
     assert(Shadow);
     if (!InsertChecks)
       return;
@@ -2203,11 +2201,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
   }
 
-  /// Remember the place where a shadow check should be inserted.
+  /// Get shadow for value, and remember the place where a shadow check should
+  /// be inserted.
   ///
   /// This location will be later instrumented with a check that will print a
   /// UMR warning in runtime if the value is not fully defined.
-  void insertShadowCheck(Value *Val, Instruction *OrigIns) {
+  void insertCheckShadowOf(Value *Val, Instruction *OrigIns) {
     assert(Val);
     Value *Shadow, *Origin;
     if (ClCheckConstantShadow) {
@@ -2221,7 +2220,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         return;
       Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
     }
-    insertShadowCheck(Shadow, Origin, OrigIns);
+    insertCheckShadow(Shadow, Origin, OrigIns);
   }
 
   AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
@@ -2333,7 +2332,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(I.getPointerOperand(), &I);
+      insertCheckShadowOf(I.getPointerOperand(), &I);
 
     if (I.isAtomic())
       I.setOrdering(addAcquireOrdering(I.getOrdering()));
@@ -2356,7 +2355,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   void visitStoreInst(StoreInst &I) {
     StoreList.push_back(&I);
     if (ClCheckAccessAddress)
-      insertShadowCheck(I.getPointerOperand(), &I);
+      insertCheckShadowOf(I.getPointerOperand(), &I);
   }
 
   void handleCASOrRMW(Instruction &I) {
@@ -2370,13 +2369,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                            .first;
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(Addr, &I);
+      insertCheckShadowOf(Addr, &I);
 
     // Only test the conditional argument of cmpxchg instruction.
     // The other argument can potentially be uninitialized, but we can not
     // detect this situation reliably without possible false positives.
     if (isa<AtomicCmpXchgInst>(I))
-      insertShadowCheck(Val, &I);
+      insertCheckShadowOf(Val, &I);
 
     IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
 
@@ -2396,7 +2395,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
   // Vector manipulation.
   void visitExtractElementInst(ExtractElementInst &I) {
-    insertShadowCheck(I.getOperand(1), &I);
+    insertCheckShadowOf(I.getOperand(1), &I);
     IRBuilder<> IRB(&I);
     setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
                                            "_msprop"));
@@ -2404,7 +2403,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   }
 
   void visitInsertElementInst(InsertElementInst &I) {
-    insertShadowCheck(I.getOperand(2), &I);
+    insertCheckShadowOf(I.getOperand(2), &I);
     IRBuilder<> IRB(&I);
     auto *Shadow0 = getShadow(&I, 0);
     auto *Shadow1 = getShadow(&I, 1);
@@ -2515,20 +2514,25 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     //    S = S | (V1 & V2)
     Value *S1 = getShadow(&I, 0);
     Value *S2 = getShadow(&I, 1);
-    Value *V1 = IRB.CreateNot(I.getOperand(0));
-    Value *V2 = IRB.CreateNot(I.getOperand(1));
+    Value *V1 = I.getOperand(0);
+    Value *V2 = I.getOperand(1);
     if (V1->getType() != S1->getType()) {
       V1 = IRB.CreateIntCast(V1, S1->getType(), false);
       V2 = IRB.CreateIntCast(V2, S2->getType(), false);
     }
+
+    Value *NotV1 = IRB.CreateNot(V1);
+    Value *NotV2 = IRB.CreateNot(V2);
+
     Value *S1S2 = IRB.CreateAnd(S1, S2);
-    Value *V1S2 = IRB.CreateAnd(V1, S2);
-    Value *S1V2 = IRB.CreateAnd(S1, V2);
+    Value *S2NotV1 = IRB.CreateAnd(NotV1, S2);
+    Value *S1NotV2 = IRB.CreateAnd(S1, NotV2);
+
+    Value *S = IRB.CreateOr({S1S2, S2NotV1, S1NotV2});
 
-    Value *S = IRB.CreateOr({S1S2, V1S2, S1V2});
     if (ClPreciseDisjointOr && cast<PossiblyDisjointInst>(&I)->isDisjoint()) {
       Value *V1V2 = IRB.CreateAnd(V1, V2);
-      S = IRB.CreateOr({S, V1V2});
+      S = IRB.CreateOr(S, V1V2, "_ms_disjoint");
     }
 
     setShadow(&I, S);
@@ -2881,7 +2885,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   void handleIntegerDiv(Instruction &I) {
     IRBuilder<> IRB(&I);
     // Strict on the second argument.
-    insertShadowCheck(I.getOperand(1), &I);
+    insertCheckShadowOf(I.getOperand(1), &I);
     setShadow(&I, getShadow(&I, 0));
     setOrigin(&I, getOrigin(&I, 0));
   }
@@ -3159,7 +3163,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(Addr, &I);
+      insertCheckShadowOf(Addr, &I);
 
     // FIXME: factor out common code from materializeStores
     if (MS.TrackOrigins)
@@ -3192,7 +3196,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(Addr, &I);
+      insertCheckShadowOf(Addr, &I);
 
     if (MS.TrackOrigins) {
       if (PropagateShadow)
@@ -3386,6 +3390,57 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
+  /// Some instructions have additional zero-elements in the return type
+  /// e.g., <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, ...)
+  ///
+  /// This function will return a vector type with the same number of elements
+  /// as the input, but same per-element width as the return value e.g.,
+  /// <8 x i8>.
+  FixedVectorType *maybeShrinkVectorShadowType(Value *Src, IntrinsicInst &I) {
+    assert(isa<FixedVectorType>(getShadowTy(&I)));
+    FixedVectorType *ShadowType = cast<FixedVectorType>(getShadowTy(&I));
+
+    // TODO: generalize beyond 2x?
+    if (ShadowType->getElementCount() ==
+        cast<VectorType>(Src->getType())->getElementCount() * 2)
+      ShadowType = FixedVectorType::getHalfElementsVectorType(ShadowType);
+
+    assert(ShadowType->getElementCount() ==
+           cast<VectorType>(Src->getType())->getElementCount());
+
+    return ShadowType;
+  }
+
+  /// Doubles the length of a vector shadow (filled with zeros) if necessary to
+  /// match the length of the shadow for the instruction.
+  /// This is more type-safe than CreateShadowCast().
+  Value *maybeExtendVectorShadowWithZeros(Value *Shadow, IntrinsicInst &I) {
+    IRBuilder<> IRB(&I);
+    assert(isa<FixedVectorType>(Shadow->getType()));
+    assert(isa<FixedVectorType>(I.getType()));
+
+    Value *FullShadow = getCleanShadow(&I);
+    assert(cast<FixedVectorType>(Shadow->getType())->getNumElements() <=
+           cast<FixedVectorType>(FullShadow->getType())->getNumElements());
+    assert(cast<FixedVectorType>(Shadow->getType())->getScalarType() ==
+           cast<FixedVectorType>(FullShadow->getType())->getScalarType());
+
+    if (Shadow->getType() == FullShadow->getType()) {
+      FullShadow = Shadow;
+    } else {
+      // TODO: generalize beyond 2x?
+      SmallVector<int, 32> ShadowMask(
+          cast<FixedVectorType>(FullShadow->getType())->getNumElements());
+      std::iota(ShadowMask.begin(), ShadowMask.end(), 0);
+
+      // Append zeros
+      FullShadow =
+          IRB.CreateShuffleVector(Shadow, getCleanShadow(Shadow), ShadowMask);
+    }
+
+    return FullShadow;
+  }
+
   /// Handle x86 SSE vector conversion.
   ///
   /// e.g., single-precision to half-precision conversion:
@@ -3416,13 +3471,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     // The return type might have more elements than the input.
     // Temporarily shrink the return type's number of elements.
-    VectorType *ShadowType = cast<VectorType>(getShadowTy(&I));
-    if (ShadowType->getElementCount() ==
-        cast<VectorType>(Src->getType())->getElementCount() * 2)
-      ShadowType = VectorType::getHalfElementsVectorType(ShadowType);
-
-    assert(ShadowType->getElementCount() ==
-           cast<VectorType>(Src->getType())->getElementCount());
+    VectorType *ShadowType = maybeShrinkVectorShadowType(Src, I);
 
     IRBuilder<> IRB(&I);
     Value *S0 = getShadow(&I, 0);
@@ -3437,19 +3486,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     // The return type might have more elements than the input.
     // Extend the return type back to its original width if necessary.
-    Value *FullShadow = getCleanShadow(&I);
-
-    if (Shadow->getType() == FullShadow->getType()) {
-      FullShadow = Shadow;
-    } else {
-      SmallVector<int, 8> ShadowMask(
-          cast<FixedVectorType>(FullShadow->getType())->getNumElements());
-      std::iota(ShadowMask.begin(), ShadowMask.end(), 0);
-
-      // Append zeros
-      FullShadow =
-          IRB.CreateShuffleVector(Shadow, getCleanShadow(Shadow), ShadowMask);
-    }
+    Value *FullShadow = maybeExtendVectorShadowWithZeros(Shadow, I);
 
     setShadow(&I, FullShadow);
     setOriginForNaryOp(I);
@@ -3516,7 +3553,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       AggShadow = ConvertShadow;
     }
     assert(AggShadow->getType()->isIntegerTy());
-    insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
+    insertCheckShadow(AggShadow, getOrigin(ConvertOp), &I);
 
     // Build result shadow by zero-filling parts of CopyOp shadow that come from
     // ConvertOp.
@@ -3923,7 +3960,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     IRB.CreateStore(getCleanShadow(Ty), ShadowPtr);
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(Addr, &I);
+      insertCheckShadowOf(Addr, &I);
   }
 
   void handleLdmxcsr(IntrinsicInst &I) {
@@ -3939,12 +3976,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(Addr, &I);
+      insertCheckShadowOf(Addr, &I);
 
     Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
     Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
                                     : getCleanOrigin();
-    insertShadowCheck(Shadow, Origin, &I);
+    insertCheckShadow(Shadow, Origin, &I);
   }
 
   void handleMaskedExpandLoad(IntrinsicInst &I) {
@@ -3955,8 +3992,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *PassThru = I.getArgOperand(2);
 
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Ptr, &I);
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Ptr, &I);
+      insertCheckShadowOf(Mask, &I);
     }
 
     if (!PropagateShadow) {
@@ -3988,8 +4025,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *Mask = I.getArgOperand(2);
 
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Ptr, &I);
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Ptr, &I);
+      insertCheckShadowOf(Mask, &I);
     }
 
     Value *Shadow = getShadow(Values);
@@ -4013,11 +4050,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     Type *PtrsShadowTy = getShadowTy(Ptrs);
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Mask, &I);
       Value *MaskedPtrShadow = IRB.CreateSelect(
           Mask, getShadow(Ptrs), Constant::getNullValue((PtrsShadowTy)),
           "_msmaskedptrs");
-      insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &I);
+      insertCheckShadow(MaskedPtrShadow, getOrigin(Ptrs), &I);
     }
 
     if (!PropagateShadow) {
@@ -4051,11 +4088,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     Type *PtrsShadowTy = getShadowTy(Ptrs);
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Mask, &I);
       Value *MaskedPtrShadow = IRB.CreateSelect(
           Mask, getShadow(Ptrs), Constant::getNullValue((PtrsShadowTy)),
           "_msmaskedptrs");
-      insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &I);
+      insertCheckShadow(MaskedPtrShadow, getOrigin(Ptrs), &I);
     }
 
     Value *Shadow = getShadow(Values);
@@ -4083,8 +4120,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *Shadow = getShadow(V);
 
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Ptr, &I);
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Ptr, &I);
+      insertCheckShadowOf(Mask, &I);
     }
 
     Value *ShadowPtr;
@@ -4116,8 +4153,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *PassThru = I.getArgOperand(3);
 
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Ptr, &I);
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Ptr, &I);
+      insertCheckShadowOf(Mask, &I);
     }
 
     if (!PropagateShadow) {
@@ -4180,8 +4217,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *SrcShadow = getShadow(Src);
 
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Dst, &I);
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Dst, &I);
+      insertCheckShadowOf(Mask, &I);
     }
 
     Value *DstShadowPtr;
@@ -4241,7 +4278,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     const Align Alignment = Align(1);
 
     if (ClCheckAccessAddress) {
-      insertShadowCheck(Mask, &I);
+      insertCheckShadowOf(Mask, &I);
     }
 
     Type *SrcShadowTy = getShadowTy(Src);
@@ -4270,12 +4307,31 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, PtrSrcOrigin);
   }
 
+  void maskedCheckAVXIndexShadow(IRBuilder<> &IRB, Value *Idx, Instruction *I) {
+    auto IdxVectorSize =
+        cast<FixedVectorType>(Idx->getType())->getNumElements();
+    assert(isPowerOf2_64(IdxVectorSize));
+    auto *IdxVectorElemType =
+        cast<FixedVectorType>(Idx->getType())->getElementType();
+    Constant *IndexBits =
+        ConstantInt::get(IdxVectorElemType, IdxVectorSize - 1);
+    auto *IdxShadow = getShadow(Idx);
+    // Only the low bits of Idx are used.
+    Value *V = nullptr;
+    for (size_t i = 0; i < IdxVectorSize; ++i) {
+      V = IRB.CreateExtractElement(IdxShadow, i);
+      assert(V->getType() == IndexBits->getType());
+      V = IRB.CreateOr(V, IRB.CreateAnd(V, IndexBits));
+    }
+    insertCheckShadow(V, getOrigin(Idx), I);
+  }
+
   // Instrument AVX permutation intrinsic.
   // We apply the same permutation (argument index 1) to the shadow.
   void handleAVXVpermilvar(IntrinsicInst &I) {
     IRBuilder<> IRB(&I);
     Value *Shadow = getShadow(&I, 0);
-    insertShadowCheck(I.getArgOperand(1), &I);
+    maskedCheckAVXIndexShadow(IRB, I.getArgOperand(1), &I);
 
     // Shadows are integer-ish types but some intrinsics require a
     // different (e.g., floating-point) type.
@@ -4287,6 +4343,94 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
+  // Instrument AVX permutation intrinsic.
+  // We apply the same permutation (argument index 1) to the shadows.
+  void handleAVXVpermi2var(IntrinsicInst &I) {
+    assert(I.arg_size() == 3);
+    assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
+    assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
+    assert(isa<FixedVectorType>(I.getArgOperand(2)->getType()));
+    [[maybe_unused]] auto ArgVectorSize =
+        cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
+    assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
+               ->getNumElements() == ArgVectorSize);
+    assert(cast<FixedVectorType>(I.getArgOperand(2)->getType())
+               ->getNumElements() == ArgVectorSize);
+    assert(I.getArgOperand(0)->getType() == I.getArgOperand(2)->getType());
+    assert(I.getType() == I.getArgOperand(0)->getType());
+    assert(I.getArgOperand(1)->getType()->isIntOrIntVectorTy());
+    IRBuilder<> IRB(&I);
+    Value *AShadow = getShadow(&I, 0);
+    Value *Idx = I.getArgOperand(1);
+    Value *BShadow = getShadow(&I, 2);
+
+    maskedCheckAVXIndexShadow(IRB, Idx, &I);
+
+    // Shadows are integer-ish types but some intrinsics require a
+    // different (e.g., floating-point) type.
+    AShadow = IRB.CreateBitCast(AShadow, I.getArgOperand(0)->getType());
+    BShadow = IRB.CreateBitCast(BShadow, I.getArgOperand(2)->getType());
+    CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+                                       {AShadow, Idx, BShadow});
+    setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
+    setOriginForNaryOp(I);
+  }
+
+  // e.g., call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
+  //                           (<16 x float> a, <16 x i32> writethru, i16 mask,
+  //                           i32 rounding)
+  //
+  // dst[i] = mask[i] ? convert(a[i]) : writethru[i]
+  // dst_shadow[i] = mask[i] ? all_or_nothing(a_shadow[i]) : writethru_shadow[i]
+  //    where all_or_nothing(x) is fully uninitialized if x has any
+  //    uninitialized bits
+  void handleAVX512VectorConvertFPToInt(IntrinsicInst &I) {
+    IRBuilder<> IRB(&I);
+
+    assert(I.arg_size() == 4);
+    Value *A = I.getOperand(0);
+    Value *WriteThrough = I.getOperand(1);
+    Value *Mask = I.getOperand(2);
+    [[maybe_unused]] Value *RoundingMode = I.getOperand(3);
+
+    assert(isa<FixedVectorType>(A->getType()));
+    assert(A->getType()->isFPOrFPVectorTy());
+
+    assert(isa<FixedVectorType>(WriteThrough->getType()));
+    assert(WriteThrough->getType()->isIntOrIntVectorTy());
+
+    unsigned ANumElements =
+        cast<FixedVectorType>(A->getType())->getNumElements();
+    assert(ANumElements ==
+           cast<FixedVectorType>(WriteThrough->getType())->getNumElements());
+
+    assert(Mask->getType()->isIntegerTy());
+    assert(Mask->getType()->getScalarSizeInBits() == ANumElements);
+
+    assert(RoundingMode->getType()->isIntegerTy());
+
+    assert(I.getType() == WriteThrough->getType());
+
+    // Convert i16 mask to <16 x i1>
+    Mask = IRB.CreateBitCast(
+        Mask, FixedVectorType::get(IRB.getInt1Ty(), ANumElements));
+
+    Value *AShadow = getShadow(A);
+    /// For scalars:
+    /// Since they are converting from floating-point, the output is:
+    /// - fully uninitialized if *any* bit of the input is uninitialized
+    /// - fully ininitialized if all bits of the input are ininitialized
+    /// We apply the same principle on a per-element basis for vectors.
+    AShadow = IRB.CreateSExt(IRB.CreateICmpNE(AShadow, getCleanShadow(A)),
+                             getShadowTy(A));
+
+    Value *WriteThroughShadow = getShadow(WriteThrough);
+    Value *Shadow = IRB.CreateSelect(Mask, AShadow, WriteThroughShadow);
+
+    setShadow(&I, Shadow);
+    setOriginForNaryOp(I);
+  }
+
   // Instrument BMI / BMI2 intrinsics.
   // All of these intrinsics are Z = I(X, Y)
   // where the types of all operands and the result match, and are either i32 or
@@ -4468,8 +4612,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // Technically, we could probably just check whether the LSB is
     // initialized, but intuitively it feels like a partly uninitialized mask
     // is unintended, and we should warn the user immediately.
-    insertShadowCheck(Mask, &I);
-    insertShadowCheck(RoundingMode, &I);
+    insertCheckShadowOf(Mask, &I);
+    insertCheckShadowOf(RoundingMode, &I);
 
     assert(isa<FixedVectorType>(A->getType()));
     unsigned NumElements =
@@ -4550,7 +4694,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       ShadowArgs.push_back(LaneNumber);
 
       // TODO: blend shadow of lane number into output shadow?
-      insertShadowCheck(LaneNumber, &I);
+      insertCheckShadowOf(LaneNumber, &I);
     }
 
     Value *Src = I.getArgOperand(numArgs - 1);
@@ -4602,7 +4746,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     int skipTrailingOperands = 1;
 
     if (ClCheckAccessAddress)
-      insertShadowCheck(Addr, &I);
+      insertCheckShadowOf(Addr, &I);
 
     // Second-last operand is the lane number (for vst{2,3,4}lane)
     if (useLane) {
@@ -5242,6 +5386,32 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     }
 
+    case Intrinsic::x86_avx512_vpermi2var_d_128:
+    case Intrinsic::x86_avx512_vpermi2var_d_256:
+    case Intrinsic::x86_avx512_vpermi2var_d_512:
+    case Intrinsic::x86_avx512_vpermi2var_hi_128:
+    case Intrinsic::x86_avx512_vpermi2var_hi_256:
+    case Intrinsic::x86_avx512_vpermi2var_hi_512:
+    case Intrinsic::x86_avx512_vpermi2var_pd_128:
+    case Intrinsic::x86_avx512_vpermi2var_pd_256:
+    case Intrinsic::x86_avx512_vpermi2var_pd_512:
+    case Intrinsic::x86_avx512_vpermi2var_ps_128:
+    case Intrinsic::x86_avx512_vpermi2var_ps_256:
+    case Intrinsic::x86_avx512_vpermi2var_ps_512:
+    case Intrinsic::x86_avx512_vpermi2var_q_128:
+    case Intrinsic::x86_avx512_vpermi2var_q_256:
+    case Intrinsic::x86_avx512_vpermi2var_q_512:
+    case Intrinsic::x86_avx512_vpermi2var_qi_128:
+    case Intrinsic::x86_avx512_vpermi2var_qi_256:
+    case Intrinsic::x86_avx512_vpermi2var_qi_512:
+      handleAVXVpermi2var(I);
+      break;
+
+    case Intrinsic::x86_avx512_mask_cvtps2dq_512: {
+      handleAVX512VectorConvertFPToInt(I);
+      break;
+    }
+
     case Intrinsic::x86_avx512fp16_mask_add_sh_round:
     case Intrinsic::x86_avx512fp16_mask_sub_sh_round:
     case Intrinsic::x86_avx512fp16_mask_mul_sh_round:
@@ -5551,7 +5721,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       if (A->getType()->isScalableTy()) {
         LLVM_DEBUG(dbgs() << "Arg  " << i << " is vscale: " << CB << "\n");
         // Handle as noundef, but don't reserve tls slots.
-        insertShadowCheck(A, &CB);
+        insertCheckShadowOf(A, &CB);
         continue;
       }
 
@@ -5563,10 +5733,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
 
       if (EagerCheck) {
-        insertShadowCheck(A, &CB);
+        insertCheckShadowOf(A, &CB);
         Size = DL.getTypeAllocSize(A->getType());
       } else {
-        Value *Store = nullptr;
+        [[maybe_unused]] Value *Store = nullptr;
         // Compute the Shadow for arg even if it is ByVal, because
         // in that case getShadow() will copy the actual arg shadow to
         // __msan_param_tls.
@@ -5623,7 +5793,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                             getOriginPtrForArgument(IRB, ArgOffset));
           }
         }
-        (void)Store;
         assert(Store != nullptr);
         LLVM_DEBUG(dbgs() << "  Param:" << *Store << "\n");
       }
@@ -5712,7 +5881,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *Shadow = getShadow(RetVal);
     bool StoreOrigin = true;
     if (EagerCheck) {
-      insertShadowCheck(RetVal, &I);
+      insertCheckShadowOf(RetVal, &I);
       Shadow = getCleanShadow(RetVal);
       StoreOrigin = false;
     }
@@ -5945,7 +6114,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // Each such pointer is instrumented with a call to the runtime library.
     Type *OpType = Operand->getType();
     // Check the operand value itself.
-    insertShadowCheck(Operand, &I);
+    insertCheckShadowOf(Operand, &I);
     if (!OpType->isPointerTy() || !isOutput) {
       assert(!isOutput);
       return;
@@ -6055,7 +6224,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
       Value *Operand = I.getOperand(i);
       if (Operand->getType()->isSized())
-        insertShadowCheck(Operand, &I);
+        insertCheckShadowOf(Operand, &I);
     }
     setShadow(&I, getCleanShadow(&I));
     setOrigin(&I, getCleanOrigin());
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 4a2eb9284a6ea..85dd9a1bf7161 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1017,10 +1017,10 @@ struct DSEState {
       }
     }
 
-    // Treat byval or inalloca arguments the same as Allocas, stores to them are
-    // dead at the end of the function.
+    // Treat byval, inalloca or dead on return arguments the same as Allocas,
+    // stores to them are dead at the end of the function.
     for (Argument &AI : F.args())
-      if (AI.hasPassPointeeByValueCopyAttr())
+      if (AI.hasPassPointeeByValueCopyAttr() || AI.hasDeadOnReturnAttr())
         InvisibleToCallerAfterRet.insert({&AI, true});
 
     // Collect whether there is any irreducible control flow in the function.
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 381de60fcface..0f8cc6ca6ed21 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -493,7 +493,7 @@ struct CallValue {
 
   static bool canHandle(Instruction *Inst) {
     CallInst *CI = dyn_cast<CallInst>(Inst);
-    if (!CI || !CI->onlyReadsMemory() ||
+    if (!CI || (!CI->onlyReadsMemory() && !CI->onlyWritesMemory()) ||
         // FIXME: Currently the calls which may access the thread id may
         // be considered as not accessing the memory. But this is
         // problematic for coroutines, since coroutines may resume in a
@@ -958,7 +958,8 @@ class EarlyCSE {
   bool overridingStores(const ParseMemoryInst &Earlier,
                         const ParseMemoryInst &Later);
 
-  Value *getOrCreateResult(Instruction *Inst, Type *ExpectedType) const {
+  Value *getOrCreateResult(Instruction *Inst, Type *ExpectedType,
+                           bool CanCreate) const {
     // TODO: We could insert relevant casts on type mismatch.
     // The load or the store's first operand.
     Value *V;
@@ -971,7 +972,8 @@ class EarlyCSE {
         V = II->getOperand(0);
         break;
       default:
-        return TTI.getOrCreateResultFromMemIntrinsic(II, ExpectedType);
+        return TTI.getOrCreateResultFromMemIntrinsic(II, ExpectedType,
+                                                     CanCreate);
       }
     } else {
       V = isa<LoadInst>(Inst) ? Inst : cast<StoreInst>(Inst)->getValueOperand();
@@ -1255,9 +1257,10 @@ Value *EarlyCSE::getMatchingValue(LoadValue &InVal, ParseMemoryInst &MemInst,
 
   // For stores check the result values before checking memory generation
   // (otherwise isSameMemGeneration may crash).
-  Value *Result = MemInst.isStore()
-                      ? getOrCreateResult(Matching, Other->getType())
-                      : nullptr;
+  Value *Result =
+      MemInst.isStore()
+          ? getOrCreateResult(Matching, Other->getType(), /*CanCreate=*/false)
+          : nullptr;
   if (MemInst.isStore() && InVal.DefInst != Result)
     return nullptr;
 
@@ -1278,7 +1281,7 @@ Value *EarlyCSE::getMatchingValue(LoadValue &InVal, ParseMemoryInst &MemInst,
     return nullptr;
 
   if (!Result)
-    Result = getOrCreateResult(Matching, Other->getType());
+    Result = getOrCreateResult(Matching, Other->getType(), /*CanCreate=*/true);
   return Result;
 }
 
@@ -1626,14 +1629,19 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         !(MemInst.isValid() && !MemInst.mayReadFromMemory()))
       LastStore = nullptr;
 
-    // If this is a read-only call, process it.
-    if (CallValue::canHandle(&Inst)) {
+    // If this is a read-only or write-only call, process it. Skip store
+    // MemInsts, as they will be more precisely handled later on. Also skip
+    // memsets, as DSE may be able to optimize them better by removing the
+    // earlier rather than later store.
+    if (CallValue::canHandle(&Inst) &&
+        (!MemInst.isValid() || !MemInst.isStore()) && !isa<MemSetInst>(&Inst)) {
       // If we have an available version of this call, and if it is the right
       // generation, replace this instruction.
       std::pair<Instruction *, unsigned> InVal = AvailableCalls.lookup(&Inst);
       if (InVal.first != nullptr &&
           isSameMemGeneration(InVal.second, CurrentGeneration, InVal.first,
-                              &Inst)) {
+                              &Inst) &&
+          InVal.first->mayReadFromMemory() == Inst.mayReadFromMemory()) {
         LLVM_DEBUG(dbgs() << "EarlyCSE CSE CALL: " << Inst
                           << "  to: " << *InVal.first << '\n');
         if (!DebugCounter::shouldExecute(CSECounter)) {
@@ -1651,6 +1659,11 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         continue;
       }
 
+      // Increase memory generation for writes. Do this before inserting
+      // the call, so it has the generation after the write occurred.
+      if (Inst.mayWriteToMemory())
+        ++CurrentGeneration;
+
       // Otherwise, remember that we have this instruction.
       AvailableCalls.insert(&Inst, std::make_pair(&Inst, CurrentGeneration));
       continue;
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index cf84366c4200b..68094c354cf46 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -2490,8 +2490,12 @@ static bool hoistMinMax(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo,
   NewCond->takeName(&I);
   I.replaceAllUsesWith(NewCond);
   eraseInstruction(I, SafetyInfo, MSSAU);
-  eraseInstruction(*cast<Instruction>(Cond1), SafetyInfo, MSSAU);
-  eraseInstruction(*cast<Instruction>(Cond2), SafetyInfo, MSSAU);
+  Instruction &CondI1 = *cast<Instruction>(Cond1);
+  Instruction &CondI2 = *cast<Instruction>(Cond2);
+  salvageDebugInfo(CondI1);
+  salvageDebugInfo(CondI2);
+  eraseInstruction(CondI1, SafetyInfo, MSSAU);
+  eraseInstruction(CondI2, SafetyInfo, MSSAU);
   return true;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
index e5355d23b5d61..0664eed072a9a 100644
--- a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
@@ -467,8 +467,7 @@ static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
     SE.forgetLoop(L);
     // Set incoming value to poison for phi nodes in the exit block.
     for (PHINode &P : ExitBlock->phis()) {
-      std::fill(P.incoming_values().begin(), P.incoming_values().end(),
-                PoisonValue::get(P.getType()));
+      llvm::fill(P.incoming_values(), PoisonValue::get(P.getType()));
     }
     ORE.emit([&]() {
       return OptimizationRemark(DEBUG_TYPE, "NeverExecutes", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 7ea6b72c213a6..a5008907b9014 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -407,6 +407,32 @@ class LoopInterchangeLegality {
   SmallVector<PHINode *, 8> InnerLoopInductions;
 };
 
+/// Manages information utilized by the profitability check for cache. The main
+/// purpose of this class is to delay the computation of CacheCost until it is
+/// actually needed.
+class CacheCostManager {
+  Loop *OutermostLoop;
+  LoopStandardAnalysisResults *AR;
+  DependenceInfo *DI;
+
+  /// CacheCost for \ref OutermostLoop. Once it is computed, it is cached. Note
+  /// that the result can be nullptr.
+  std::optional<std::unique_ptr<CacheCost>> CC;
+
+  /// Maps each loop to an index representing the optimal position within the
+  /// loop-nest, as determined by the cache cost analysis.
+  DenseMap<const Loop *, unsigned> CostMap;
+
+  void computeIfUnitinialized();
+
+public:
+  CacheCostManager(Loop *OutermostLoop, LoopStandardAnalysisResults *AR,
+                   DependenceInfo *DI)
+      : OutermostLoop(OutermostLoop), AR(AR), DI(DI) {}
+  CacheCost *getCacheCost();
+  const DenseMap<const Loop *, unsigned> &getCostMap();
+};
+
 /// LoopInterchangeProfitability checks if it is profitable to interchange the
 /// loop.
 class LoopInterchangeProfitability {
@@ -418,15 +444,12 @@ class LoopInterchangeProfitability {
   /// Check if the loop interchange is profitable.
   bool isProfitable(const Loop *InnerLoop, const Loop *OuterLoop,
                     unsigned InnerLoopId, unsigned OuterLoopId,
-                    CharMatrix &DepMatrix,
-                    const DenseMap<const Loop *, unsigned> &CostMap,
-                    std::unique_ptr<CacheCost> &CC);
+                    CharMatrix &DepMatrix, CacheCostManager &CCM);
 
 private:
   int getInstrOrderCost();
   std::optional<bool> isProfitablePerLoopCacheAnalysis(
-      const DenseMap<const Loop *, unsigned> &CostMap,
-      std::unique_ptr<CacheCost> &CC);
+      const DenseMap<const Loop *, unsigned> &CostMap, CacheCost *CC);
   std::optional<bool> isProfitablePerInstrOrderCost();
   std::optional<bool> isProfitableForVectorization(unsigned InnerLoopId,
                                                    unsigned OuterLoopId,
@@ -477,15 +500,15 @@ struct LoopInterchange {
   LoopInfo *LI = nullptr;
   DependenceInfo *DI = nullptr;
   DominatorTree *DT = nullptr;
-  std::unique_ptr<CacheCost> CC = nullptr;
+  LoopStandardAnalysisResults *AR = nullptr;
 
   /// Interface to emit optimization remarks.
   OptimizationRemarkEmitter *ORE;
 
   LoopInterchange(ScalarEvolution *SE, LoopInfo *LI, DependenceInfo *DI,
-                  DominatorTree *DT, std::unique_ptr<CacheCost> &CC,
+                  DominatorTree *DT, LoopStandardAnalysisResults *AR,
                   OptimizationRemarkEmitter *ORE)
-      : SE(SE), LI(LI), DI(DI), DT(DT), CC(std::move(CC)), ORE(ORE) {}
+      : SE(SE), LI(LI), DI(DI), DT(DT), AR(AR), ORE(ORE) {}
 
   bool run(Loop *L) {
     if (L->getParentLoop())
@@ -540,21 +563,7 @@ struct LoopInterchange {
     }
 
     unsigned SelecLoopId = selectLoopForInterchange(LoopList);
-    // Obtain the loop vector returned from loop cache analysis beforehand,
-    // and put each <Loop, index> pair into a map for constant time query
-    // later. Indices in loop vector reprsent the optimal order of the
-    // corresponding loop, e.g., given a loopnest with depth N, index 0
-    // indicates the loop should be placed as the outermost loop and index N
-    // indicates the loop should be placed as the innermost loop.
-    //
-    // For the old pass manager CacheCost would be null.
-    DenseMap<const Loop *, unsigned> CostMap;
-    if (CC != nullptr) {
-      const auto &LoopCosts = CC->getLoopCosts();
-      for (unsigned i = 0; i < LoopCosts.size(); i++) {
-        CostMap[LoopCosts[i].first] = i;
-      }
-    }
+    CacheCostManager CCM(LoopList[0], AR, DI);
     // We try to achieve the globally optimal memory access for the loopnest,
     // and do interchange based on a bubble-sort fasion. We start from
     // the innermost loop, move it outwards to the best possible position
@@ -563,7 +572,7 @@ struct LoopInterchange {
       bool ChangedPerIter = false;
       for (unsigned i = SelecLoopId; i > SelecLoopId - j; i--) {
         bool Interchanged =
-            processLoop(LoopList, i, i - 1, DependencyMatrix, CostMap);
+            processLoop(LoopList, i, i - 1, DependencyMatrix, CCM);
         ChangedPerIter |= Interchanged;
         Changed |= Interchanged;
       }
@@ -578,7 +587,7 @@ struct LoopInterchange {
   bool processLoop(SmallVectorImpl<Loop *> &LoopList, unsigned InnerLoopId,
                    unsigned OuterLoopId,
                    std::vector<std::vector<char>> &DependencyMatrix,
-                   const DenseMap<const Loop *, unsigned> &CostMap) {
+                   CacheCostManager &CCM) {
     Loop *OuterLoop = LoopList[OuterLoopId];
     Loop *InnerLoop = LoopList[InnerLoopId];
     LLVM_DEBUG(dbgs() << "Processing InnerLoopId = " << InnerLoopId
@@ -591,7 +600,7 @@ struct LoopInterchange {
     LLVM_DEBUG(dbgs() << "Loops are legal to interchange\n");
     LoopInterchangeProfitability LIP(OuterLoop, InnerLoop, SE, ORE);
     if (!LIP.isProfitable(InnerLoop, OuterLoop, InnerLoopId, OuterLoopId,
-                          DependencyMatrix, CostMap, CC)) {
+                          DependencyMatrix, CCM)) {
       LLVM_DEBUG(dbgs() << "Interchanging loops not profitable.\n");
       return false;
     }
@@ -970,8 +979,8 @@ areInnerLoopExitPHIsSupported(Loop *InnerL, Loop *OuterL,
 static bool areOuterLoopExitPHIsSupported(Loop *OuterLoop, Loop *InnerLoop) {
   BasicBlock *LoopNestExit = OuterLoop->getUniqueExitBlock();
   for (PHINode &PHI : LoopNestExit->phis()) {
-    for (unsigned i = 0; i < PHI.getNumIncomingValues(); i++) {
-      Instruction *IncomingI = dyn_cast<Instruction>(PHI.getIncomingValue(i));
+    for (Value *Incoming : PHI.incoming_values()) {
+      Instruction *IncomingI = dyn_cast<Instruction>(Incoming);
       if (!IncomingI || IncomingI->getParent() != OuterLoop->getLoopLatch())
         continue;
 
@@ -1124,21 +1133,49 @@ bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
   return true;
 }
 
+void CacheCostManager::computeIfUnitinialized() {
+  if (CC.has_value())
+    return;
+
+  LLVM_DEBUG(dbgs() << "Compute CacheCost.\n");
+  CC = CacheCost::getCacheCost(*OutermostLoop, *AR, *DI);
+  // Obtain the loop vector returned from loop cache analysis beforehand,
+  // and put each <Loop, index> pair into a map for constant time query
+  // later. Indices in loop vector reprsent the optimal order of the
+  // corresponding loop, e.g., given a loopnest with depth N, index 0
+  // indicates the loop should be placed as the outermost loop and index N
+  // indicates the loop should be placed as the innermost loop.
+  //
+  // For the old pass manager CacheCost would be null.
+  if (*CC != nullptr)
+    for (const auto &[Idx, Cost] : enumerate((*CC)->getLoopCosts()))
+      CostMap[Cost.first] = Idx;
+}
+
+CacheCost *CacheCostManager::getCacheCost() {
+  computeIfUnitinialized();
+  return CC->get();
+}
+
+const DenseMap<const Loop *, unsigned> &CacheCostManager::getCostMap() {
+  computeIfUnitinialized();
+  return CostMap;
+}
+
 int LoopInterchangeProfitability::getInstrOrderCost() {
   unsigned GoodOrder, BadOrder;
   BadOrder = GoodOrder = 0;
   for (BasicBlock *BB : InnerLoop->blocks()) {
     for (Instruction &Ins : *BB) {
       if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&Ins)) {
-        unsigned NumOp = GEP->getNumOperands();
         bool FoundInnerInduction = false;
         bool FoundOuterInduction = false;
-        for (unsigned i = 0; i < NumOp; ++i) {
+        for (Value *Op : GEP->operands()) {
           // Skip operands that are not SCEV-able.
-          if (!SE->isSCEVable(GEP->getOperand(i)->getType()))
+          if (!SE->isSCEVable(Op->getType()))
             continue;
 
-          const SCEV *OperandVal = SE->getSCEV(GEP->getOperand(i));
+          const SCEV *OperandVal = SE->getSCEV(Op);
           const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(OperandVal);
           if (!AR)
             continue;
@@ -1180,8 +1217,7 @@ int LoopInterchangeProfitability::getInstrOrderCost() {
 
 std::optional<bool>
 LoopInterchangeProfitability::isProfitablePerLoopCacheAnalysis(
-    const DenseMap<const Loop *, unsigned> &CostMap,
-    std::unique_ptr<CacheCost> &CC) {
+    const DenseMap<const Loop *, unsigned> &CostMap, CacheCost *CC) {
   // This is the new cost model returned from loop cache analysis.
   // A smaller index means the loop should be placed an outer loop, and vice
   // versa.
@@ -1218,8 +1254,8 @@ LoopInterchangeProfitability::isProfitablePerInstrOrderCost() {
 
 /// Return true if we can vectorize the loop specified by \p LoopId.
 static bool canVectorize(const CharMatrix &DepMatrix, unsigned LoopId) {
-  for (unsigned I = 0; I != DepMatrix.size(); I++) {
-    char Dir = DepMatrix[I][LoopId];
+  for (const auto &Dep : DepMatrix) {
+    char Dir = Dep[LoopId];
     if (Dir != 'I' && Dir != '=')
       return false;
   }
@@ -1249,9 +1285,7 @@ std::optional<bool> LoopInterchangeProfitability::isProfitableForVectorization(
 
 bool LoopInterchangeProfitability::isProfitable(
     const Loop *InnerLoop, const Loop *OuterLoop, unsigned InnerLoopId,
-    unsigned OuterLoopId, CharMatrix &DepMatrix,
-    const DenseMap<const Loop *, unsigned> &CostMap,
-    std::unique_ptr<CacheCost> &CC) {
+    unsigned OuterLoopId, CharMatrix &DepMatrix, CacheCostManager &CCM) {
   // isProfitable() is structured to avoid endless loop interchange. If the
   // highest priority rule (isProfitablePerLoopCacheAnalysis by default) could
   // decide the profitability then, profitability check will stop and return the
@@ -1264,9 +1298,12 @@ bool LoopInterchangeProfitability::isProfitable(
   std::optional<bool> shouldInterchange;
   for (RuleTy RT : Profitabilities) {
     switch (RT) {
-    case RuleTy::PerLoopCacheAnalysis:
+    case RuleTy::PerLoopCacheAnalysis: {
+      CacheCost *CC = CCM.getCacheCost();
+      const DenseMap<const Loop *, unsigned> &CostMap = CCM.getCostMap();
       shouldInterchange = isProfitablePerLoopCacheAnalysis(CostMap, CC);
       break;
+    }
     case RuleTy::PerInstrOrderCost:
       shouldInterchange = isProfitablePerInstrOrderCost();
       break;
@@ -1844,10 +1881,7 @@ PreservedAnalyses LoopInterchangePass::run(LoopNest &LN,
   });
 
   DependenceInfo DI(&F, &AR.AA, &AR.SE, &AR.LI);
-  std::unique_ptr<CacheCost> CC =
-      CacheCost::getCacheCost(LN.getOutermostLoop(), AR, DI);
-
-  if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, CC, &ORE).run(LN))
+  if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, &AR, &ORE).run(LN))
     return PreservedAnalyses::all();
   U.markLoopNestChanged(true);
   return getLoopPassPreservedAnalyses();
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 4ba69034d6448..845afa6d4228b 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -923,10 +923,11 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
 /// If S involves the addition of a constant integer value, return that integer
 /// value, and mutate S to point to a new SCEV with that value excluded.
 static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
-  if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
-    if (C->getAPInt().getSignificantBits() <= 64) {
-      S = SE.getConstant(C->getType(), 0);
-      return Immediate::getFixed(C->getValue()->getSExtValue());
+  const APInt *C;
+  if (match(S, m_scev_APInt(C))) {
+    if (C->getSignificantBits() <= 64) {
+      S = SE.getConstant(S->getType(), 0);
+      return Immediate::getFixed(C->getSExtValue());
     }
   } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
     SmallVector<const SCEV *, 8> NewOps(Add->operands());
@@ -942,14 +943,10 @@ static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
                            // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
                            SCEV::FlagAnyWrap);
     return Result;
-  } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
-    if (EnableVScaleImmediates && M->getNumOperands() == 2) {
-      if (const SCEVConstant *C = dyn_cast<SCEVConstant>(M->getOperand(0)))
-        if (isa<SCEVVScale>(M->getOperand(1))) {
-          S = SE.getConstant(M->getType(), 0);
-          return Immediate::getScalable(C->getValue()->getSExtValue());
-        }
-    }
+  } else if (EnableVScaleImmediates &&
+             match(S, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale()))) {
+    S = SE.getConstant(S->getType(), 0);
+    return Immediate::getScalable(C->getSExtValue());
   }
   return Immediate::getZero();
 }
@@ -1133,23 +1130,22 @@ static bool isHighCostExpansion(const SCEV *S,
     return false;
   }
 
-  if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
-    if (Mul->getNumOperands() == 2) {
-      // Multiplication by a constant is ok
-      if (isa<SCEVConstant>(Mul->getOperand(0)))
-        return isHighCostExpansion(Mul->getOperand(1), Processed, SE);
-
-      // If we have the value of one operand, check if an existing
-      // multiplication already generates this expression.
-      if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) {
-        Value *UVal = U->getValue();
-        for (User *UR : UVal->users()) {
-          // If U is a constant, it may be used by a ConstantExpr.
-          Instruction *UI = dyn_cast<Instruction>(UR);
-          if (UI && UI->getOpcode() == Instruction::Mul &&
-              SE.isSCEVable(UI->getType())) {
-            return SE.getSCEV(UI) == Mul;
-          }
+  const SCEV *Op0, *Op1;
+  if (match(S, m_scev_Mul(m_SCEV(Op0), m_SCEV(Op1)))) {
+    // Multiplication by a constant is ok
+    if (isa<SCEVConstant>(Op0))
+      return isHighCostExpansion(Op1, Processed, SE);
+
+    // If we have the value of one operand, check if an existing
+    // multiplication already generates this expression.
+    if (const auto *U = dyn_cast<SCEVUnknown>(Op1)) {
+      Value *UVal = U->getValue();
+      for (User *UR : UVal->users()) {
+        // If U is a constant, it may be used by a ConstantExpr.
+        Instruction *UI = dyn_cast<Instruction>(UR);
+        if (UI && UI->getOpcode() == Instruction::Mul &&
+            SE.isSCEVable(UI->getType())) {
+          return SE.getSCEV(UI) == S;
         }
       }
     }
@@ -3333,14 +3329,11 @@ static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst,
     IncOffset = Immediate::getFixed(IncConst->getValue()->getSExtValue());
   } else {
     // Look for mul(vscale, constant), to detect a scalable offset.
-    auto *IncVScale = dyn_cast<SCEVMulExpr>(IncExpr);
-    if (!IncVScale || IncVScale->getNumOperands() != 2 ||
-        !isa<SCEVVScale>(IncVScale->getOperand(1)))
-      return false;
-    auto *Scale = dyn_cast<SCEVConstant>(IncVScale->getOperand(0));
-    if (!Scale || Scale->getType()->getScalarSizeInBits() > 64)
+    const APInt *C;
+    if (!match(IncExpr, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale())) ||
+        C->getSignificantBits() > 64)
       return false;
-    IncOffset = Immediate::getScalable(Scale->getValue()->getSExtValue());
+    IncOffset = Immediate::getScalable(C->getSExtValue());
   }
 
   if (!isAddressUse(TTI, UserInst, Operand))
@@ -3818,6 +3811,8 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
     return nullptr;
   }
   const SCEV *Start, *Step;
+  const SCEVConstant *Op0;
+  const SCEV *Op1;
   if (match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step)))) {
     // Split a non-zero base out of an addrec.
     if (Start->isZero())
@@ -3839,19 +3834,13 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
                               // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
                               SCEV::FlagAnyWrap);
     }
-  } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
+  } else if (match(S, m_scev_Mul(m_SCEVConstant(Op0), m_SCEV(Op1)))) {
     // Break (C * (a + b + c)) into C*a + C*b + C*c.
-    if (Mul->getNumOperands() != 2)
-      return S;
-    if (const SCEVConstant *Op0 =
-        dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
-      C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
-      const SCEV *Remainder =
-        CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1);
-      if (Remainder)
-        Ops.push_back(SE.getMulExpr(C, Remainder));
-      return nullptr;
-    }
+    C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
+    const SCEV *Remainder = CollectSubexprs(Op1, C, Ops, L, SE, Depth + 1);
+    if (Remainder)
+      Ops.push_back(SE.getMulExpr(C, Remainder));
+    return nullptr;
   }
   return S;
 }
@@ -6479,10 +6468,6 @@ struct SCEVDbgValueBuilder {
   /// Chain (non-affine) SCEVs are not supported.
   bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
     assert(SAR.isAffine() && "Expected affine SCEV");
-    // TODO: Is this check needed?
-    if (isa<SCEVAddRecExpr>(SAR.getStart()))
-      return false;
-
     const SCEV *Start = SAR.getStart();
     const SCEV *Stride = SAR.getStepRecurrence(SE);
 
@@ -6550,11 +6535,6 @@ struct SCEVDbgValueBuilder {
   bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
                            ScalarEvolution &SE) {
     assert(SAR.isAffine() && "Expected affine SCEV");
-    if (isa<SCEVAddRecExpr>(SAR.getStart())) {
-      LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: "
-                        << SAR << '\n');
-      return false;
-    }
     const SCEV *Start = SAR.getStart();
     const SCEV *Stride = SAR.getStepRecurrence(SE);
 
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index a712b4632e9a8..b3bffeb7ea412 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -2382,7 +2382,7 @@ class LowerMatrixIntrinsics {
       llvm::copy(C.vectors(), std::back_inserter(CondV));
     } else {
       CondV.resize(A.getNumVectors());
-      std::fill(CondV.begin(), CondV.end(), Cond);
+      llvm::fill(CondV, Cond);
     }
 
     for (auto [CV, AV, BV] : llvm::zip_equal(CondV, A.vectors(), B.vectors()))
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 42d1d9a437bb2..70b4552190a4e 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -4070,18 +4070,27 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
   //   => select cond, gep(ptr1, idx), gep(ptr2, idx)
   // and  gep ptr, (select cond, idx1, idx2)
   //   => select cond, gep(ptr, idx1), gep(ptr, idx2)
+  // We also allow for i1 zext indices, which are equivalent to selects.
   bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
     // Check whether the GEP has exactly one select operand and all indices
     // will become constant after the transform.
-    SelectInst *Sel = dyn_cast<SelectInst>(GEPI.getPointerOperand());
+    Instruction *Sel = dyn_cast<SelectInst>(GEPI.getPointerOperand());
     for (Value *Op : GEPI.indices()) {
       if (auto *SI = dyn_cast<SelectInst>(Op)) {
         if (Sel)
           return false;
 
         Sel = SI;
-        if (!isa<ConstantInt>(Sel->getTrueValue()) ||
-            !isa<ConstantInt>(Sel->getFalseValue()))
+        if (!isa<ConstantInt>(SI->getTrueValue()) ||
+            !isa<ConstantInt>(SI->getFalseValue()))
+          return false;
+        continue;
+      }
+      if (auto *ZI = dyn_cast<ZExtInst>(Op)) {
+        if (Sel)
+          return false;
+        Sel = ZI;
+        if (!ZI->getSrcTy()->isIntegerTy(1))
           return false;
         continue;
       }
@@ -4107,8 +4116,16 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
       return NewOps;
     };
 
-    Value *True = Sel->getTrueValue();
-    Value *False = Sel->getFalseValue();
+    Value *Cond, *True, *False;
+    if (auto *SI = dyn_cast<SelectInst>(Sel)) {
+      Cond = SI->getCondition();
+      True = SI->getTrueValue();
+      False = SI->getFalseValue();
+    } else {
+      Cond = Sel->getOperand(0);
+      True = ConstantInt::get(Sel->getType(), 1);
+      False = ConstantInt::get(Sel->getType(), 0);
+    }
     SmallVector<Value *> TrueOps = GetNewOps(True);
     SmallVector<Value *> FalseOps = GetNewOps(False);
 
@@ -4123,8 +4140,8 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
         IRB.CreateGEP(Ty, FalseOps[0], ArrayRef(FalseOps).drop_front(),
                       False->getName() + ".sroa.gep", NW);
 
-    Value *NSel = IRB.CreateSelect(Sel->getCondition(), NTrue, NFalse,
-                                   Sel->getName() + ".sroa.sel");
+    Value *NSel =
+        IRB.CreateSelect(Cond, NTrue, NFalse, Sel->getName() + ".sroa.sel");
     Visited.erase(&GEPI);
     GEPI.replaceAllUsesWith(NSel);
     GEPI.eraseFromParent();
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 78cad0d253be8..f7e66eca6aeb3 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_component_library(LLVMTransformUtils
   CtorUtils.cpp
   CountVisits.cpp
   Debugify.cpp
+  DeclareRuntimeLibcalls.cpp
   DemoteRegToStack.cpp
   DXILUpgrade.cpp
   EntryExitInstrumenter.cpp
diff --git a/llvm/lib/Transforms/Utils/CloneModule.cpp b/llvm/lib/Transforms/Utils/CloneModule.cpp
index 55fb0acd39eae..ae6234c1349d6 100644
--- a/llvm/lib/Transforms/Utils/CloneModule.cpp
+++ b/llvm/lib/Transforms/Utils/CloneModule.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm-c/Core.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Utils/Cloning.h"
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 242cf6d811b66..eacaf42e4e8ba 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1020,6 +1020,7 @@ Function *CodeExtractor::constructFunctionDeclaration(
       case Attribute::EndAttrKinds:
       case Attribute::EmptyKey:
       case Attribute::TombstoneKey:
+      case Attribute::DeadOnReturn:
         llvm_unreachable("Not a function attribute");
       }
 
@@ -1347,8 +1348,10 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     if (!NewLabel) {
       DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
           *OldLabel->getScope(), *NewSP, Ctx, Cache);
-      NewLabel = DILabel::get(Ctx, NewScope, OldLabel->getName(),
-                              OldLabel->getFile(), OldLabel->getLine());
+      NewLabel =
+          DILabel::get(Ctx, NewScope, OldLabel->getName(), OldLabel->getFile(),
+                       OldLabel->getLine(), OldLabel->getColumn(),
+                       OldLabel->isArtificial(), OldLabel->getCoroSuspendIdx());
     }
     LabelRecord->setLabel(cast<DILabel>(NewLabel));
   };
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index c2dbdc57eb3b5..a1f030a336c15 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -16,8 +16,10 @@
 #include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -28,6 +30,11 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/JSON.h"
 #include <optional>
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+// We need the Signals header to operate on stacktraces if we're using DebugLoc
+// origin-tracking.
+#include "llvm/Support/Signals.h"
+#endif
 
 #define DEBUG_TYPE "debugify"
 
@@ -59,6 +66,54 @@ cl::opt<Level> DebugifyLevel(
 
 raw_ostream &dbg() { return Quiet ? nulls() : errs(); }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+// These maps refer to addresses in the current LLVM process, so we can reuse
+// them everywhere - therefore, we store them at file scope.
+static SymbolizedAddressMap SymbolizedAddrs;
+static AddressSet UnsymbolizedAddrs;
+
+std::string symbolizeStackTrace(const Instruction *I) {
+  // We flush the set of unsymbolized addresses at the latest possible moment,
+  // i.e. now.
+  if (!UnsymbolizedAddrs.empty()) {
+    sys::symbolizeAddresses(UnsymbolizedAddrs, SymbolizedAddrs);
+    UnsymbolizedAddrs.clear();
+  }
+  const DbgLocOrigin::StackTracesTy &OriginStackTraces =
+      I->getDebugLoc().getOriginStackTraces();
+  std::string Result;
+  raw_string_ostream OS(Result);
+  for (size_t TraceIdx = 0; TraceIdx < OriginStackTraces.size(); ++TraceIdx) {
+    if (TraceIdx != 0)
+      OS << "========================================\n";
+    auto &[Depth, StackTrace] = OriginStackTraces[TraceIdx];
+    unsigned VirtualFrameNo = 0;
+    for (int Frame = 0; Frame < Depth; ++Frame) {
+      assert(SymbolizedAddrs.contains(StackTrace[Frame]) &&
+             "Expected each address to have been symbolized.");
+      for (std::string &SymbolizedFrame : SymbolizedAddrs[StackTrace[Frame]]) {
+        OS << right_justify(formatv("#{0}", VirtualFrameNo++).str(),
+                            std::log10(Depth) + 2)
+           << ' ' << SymbolizedFrame << '\n';
+      }
+    }
+  }
+  return Result;
+}
+void collectStackAddresses(Instruction &I) {
+  auto &OriginStackTraces = I.getDebugLoc().getOriginStackTraces();
+  for (auto &[Depth, StackTrace] : OriginStackTraces) {
+    for (int Frame = 0; Frame < Depth; ++Frame) {
+      void *Addr = StackTrace[Frame];
+      if (!SymbolizedAddrs.contains(Addr))
+        UnsymbolizedAddrs.insert(Addr);
+    }
+  }
+}
+#else
+void collectStackAddresses(Instruction &I) {}
+#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+
 uint64_t getAllocSizeInBits(Module &M, Type *Ty) {
   return Ty->isSized() ? M.getDataLayout().getTypeAllocSizeInBits(Ty) : 0;
 }
@@ -123,7 +178,9 @@ bool llvm::applyDebugifyMetadata(
     if (F.hasPrivateLinkage() || F.hasInternalLinkage())
       SPFlags |= DISubprogram::SPFlagLocalToUnit;
     auto SP = DIB.createFunction(CU, F.getName(), F.getName(), File, NextLine,
-                                 SPType, NextLine, DINode::FlagZero, SPFlags);
+                                 SPType, NextLine, DINode::FlagZero, SPFlags,
+                                 nullptr, nullptr, nullptr, nullptr, "",
+                                 /*UseKeyInstructions*/ ApplyAtomGroups);
     F.setSubprogram(SP);
 
     // Helper that inserts a dbg.value before \p InsertBefore, copying the
@@ -373,6 +430,8 @@ bool llvm::collectDebugInfoMetadata(Module &M,
         LLVM_DEBUG(dbgs() << "  Collecting info for inst: " << I << '\n');
         DebugInfoBeforePass.InstToDelete.insert({&I, &I});
 
+        // Track the addresses to symbolize, if the feature is enabled.
+        collectStackAddresses(I);
         DebugInfoBeforePass.DILocations.insert({&I, hasLoc(I)});
       }
     }
@@ -448,14 +507,23 @@ static bool checkInstructions(const DebugInstMap &DILocsBefore,
     auto BBName = BB->hasName() ? BB->getName() : "no-name";
     auto InstName = Instruction::getOpcodeName(Instr->getOpcode());
 
+    auto CreateJSONBugEntry = [&](const char *Action) {
+      Bugs.push_back(llvm::json::Object({
+          {"metadata", "DILocation"},
+          {"fn-name", FnName.str()},
+          {"bb-name", BBName.str()},
+          {"instr", InstName},
+          {"action", Action},
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+          {"origin", symbolizeStackTrace(Instr)},
+#endif
+      }));
+    };
+
     auto InstrIt = DILocsBefore.find(Instr);
     if (InstrIt == DILocsBefore.end()) {
       if (ShouldWriteIntoJSON)
-        Bugs.push_back(llvm::json::Object({{"metadata", "DILocation"},
-                                           {"fn-name", FnName.str()},
-                                           {"bb-name", BBName.str()},
-                                           {"instr", InstName},
-                                           {"action", "not-generate"}}));
+        CreateJSONBugEntry("not-generate");
       else
         dbg() << "WARNING: " << NameOfWrappedPass
               << " did not generate DILocation for " << *Instr
@@ -468,11 +536,7 @@ static bool checkInstructions(const DebugInstMap &DILocsBefore,
       // If the instr had the !dbg attached before the pass, consider it as
       // a debug info issue.
       if (ShouldWriteIntoJSON)
-        Bugs.push_back(llvm::json::Object({{"metadata", "DILocation"},
-                                           {"fn-name", FnName.str()},
-                                           {"bb-name", BBName.str()},
-                                           {"instr", InstName},
-                                           {"action", "drop"}}));
+        CreateJSONBugEntry("drop");
       else
         dbg() << "WARNING: " << NameOfWrappedPass << " dropped DILocation of "
               << *Instr << " (BB: " << BBName << ", Fn: " << FnName
@@ -610,6 +674,8 @@ bool llvm::checkDebugInfoMetadata(Module &M,
 
         LLVM_DEBUG(dbgs() << "  Collecting info for inst: " << I << '\n');
 
+        // Track the addresses to symbolize, if the feature is enabled.
+        collectStackAddresses(I);
         DebugInfoAfterPass.DILocations.insert({&I, hasLoc(I)});
       }
     }
diff --git a/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp b/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp
new file mode 100644
index 0000000000000..540039b7d2cbd
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp
@@ -0,0 +1,38 @@
+//===- DeclareRuntimeLibcalls.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Insert declarations for all runtime library calls known for the target.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/DeclareRuntimeLibcalls.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/RuntimeLibcalls.h"
+
+using namespace llvm;
+
+PreservedAnalyses DeclareRuntimeLibcallsPass::run(Module &M,
+                                                  ModuleAnalysisManager &MAM) {
+  RTLIB::RuntimeLibcallsInfo RTLCI(M.getTargetTriple());
+  LLVMContext &Ctx = M.getContext();
+
+  for (RTLIB::LibcallImpl Impl : RTLCI.getLibcallImpls()) {
+    if (Impl == RTLIB::Unsupported)
+      continue;
+
+    // TODO: Declare with correct type, calling convention, and attributes.
+
+    FunctionType *FuncTy =
+        FunctionType::get(Type::getVoidTy(Ctx), {}, /*IsVarArgs=*/true);
+
+    const char *FuncName = RTLCI.getLibcallImplName(Impl);
+    M.getOrInsertFunction(FuncName, FuncTy);
+  }
+
+  return PreservedAnalyses::none();
+}
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index c50bb4a497c6a..200d1fb854155 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1171,7 +1171,7 @@ Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
     SmallVector<int, 32> ShuffleMask(VF);
     for (unsigned stride = 1; stride < VF; stride <<= 1) {
       // Initialise the mask with undef.
-      std::fill(ShuffleMask.begin(), ShuffleMask.end(), -1);
+      llvm::fill(ShuffleMask, -1);
       for (unsigned j = 0; j < VF; j += stride << 1) {
         ShuffleMask[j] = j + stride;
       }
@@ -1227,8 +1227,10 @@ Value *llvm::createFindLastIVReduction(IRBuilderBase &Builder, Value *Src,
                                        RecurKind RdxKind, Value *Start,
                                        Value *Sentinel) {
   bool IsSigned = RecurrenceDescriptor::isSignedRecurrenceKind(RdxKind);
+  bool IsMaxRdx = RecurrenceDescriptor::isFindLastIVRecurrenceKind(RdxKind);
   Value *MaxRdx = Src->getType()->isVectorTy()
-                      ? Builder.CreateIntMaxReduce(Src, IsSigned)
+                      ? (IsMaxRdx ? Builder.CreateIntMaxReduce(Src, IsSigned)
+                                  : Builder.CreateIntMinReduce(Src, IsSigned))
                       : Src;
   // Correct the final reduction result back to the start value if the maximum
   // reduction is sentinel value.
@@ -1324,8 +1326,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
 Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
                                    RecurKind Kind, Value *Mask, Value *EVL) {
   assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
-         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
-         "AnyOf or FindLastIV reductions are not supported.");
+         !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
+         "AnyOf and FindIV reductions are not supported.");
   Intrinsic::ID Id = getReductionIntrinsicID(Kind);
   auto VPID = VPIntrinsic::getForIntrinsic(Id);
   assert(VPReductionIntrinsic::isVPReduction(VPID) &&
diff --git a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
index af0f516f7b002..d53a3144bf57d 100644
--- a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
+++ b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
@@ -66,6 +66,20 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
   if (!ElemType->isPointerTy() || DL.getPointerTypeSizeInBits(ElemType) != 64)
     return false;
 
+  SmallVector<GlobalVariable *, 4> GVOps;
+  Triple TT = M.getTargetTriple();
+  // FIXME: This should be removed in the future.
+  bool ShouldDropUnnamedAddr =
+      // Drop unnamed_addr to avoid matching pattern in
+      // `handleIndirectSymViaGOTPCRel`, which generates GOTPCREL relocations
+      // not supported by the GNU linker and LLD versions below 18 on aarch64.
+      TT.isAArch64()
+      // Apple's ld64 (and ld-prime on Xcode 15.2) miscompile something on
+      // x86_64-apple-darwin. See
+      // https://github.com/rust-lang/rust/issues/140686 and
+      // https://github.com/rust-lang/rust/issues/141306.
+      || (TT.isX86() && TT.isOSDarwin());
+
   for (const Use &Op : Array->operands()) {
     Constant *ConstOp = cast<Constant>(&Op);
     GlobalValue *GVOp;
@@ -85,8 +99,15 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
         !GlovalVarOp->isDSOLocal() ||
         !GlovalVarOp->isImplicitDSOLocal())
       return false;
+
+    if (ShouldDropUnnamedAddr)
+      GVOps.push_back(GlovalVarOp);
   }
 
+  if (ShouldDropUnnamedAddr)
+    for (auto *GVOp : GVOps)
+      GVOp->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
+
   return true;
 }
 
@@ -108,24 +129,8 @@ static GlobalVariable *createRelLookupTable(Function &Func,
   uint64_t Idx = 0;
   SmallVector<Constant *, 64> RelLookupTableContents(NumElts);
 
-  Triple TT = M.getTargetTriple();
-  // FIXME: This should be removed in the future.
-  bool ShouldDropUnnamedAddr =
-      // Drop unnamed_addr to avoid matching pattern in
-      // `handleIndirectSymViaGOTPCRel`, which generates GOTPCREL relocations
-      // not supported by the GNU linker and LLD versions below 18 on aarch64.
-      TT.isAArch64()
-      // Apple's ld64 (and ld-prime on Xcode 15.2) miscompile something on
-      // x86_64-apple-darwin. See
-      // https://github.com/rust-lang/rust/issues/140686 and
-      // https://github.com/rust-lang/rust/issues/141306.
-      || (TT.isX86() && TT.isOSDarwin());
-
   for (Use &Operand : LookupTableArr->operands()) {
     Constant *Element = cast<Constant>(Operand);
-    if (ShouldDropUnnamedAddr)
-      if (auto *GlobalElement = dyn_cast<GlobalValue>(Element))
-        GlobalElement->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
     Type *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
     Constant *Base = llvm::ConstantExpr::getPtrToInt(RelLookupTable, IntPtrTy);
     Constant *Target = llvm::ConstantExpr::getPtrToInt(Element, IntPtrTy);
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index cb6aafa279b5f..586874f57ae73 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -426,7 +426,10 @@ void SCCPSolver::inferArgAttributes() const {
 class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   const DataLayout &DL;
   std::function<const TargetLibraryInfo &(Function &)> GetTLI;
-  SmallPtrSet<BasicBlock *, 8> BBExecutable; // The BBs that are executable.
+  /// Basic blocks that are executable (but may not have been visited yet).
+  SmallPtrSet<BasicBlock *, 8> BBExecutable;
+  /// Basic blocks that are executable and have been visited at least once.
+  SmallPtrSet<BasicBlock *, 8> BBVisited;
   DenseMap<Value *, ValueLatticeElement>
       ValueState; // The state each value is in.
 
@@ -466,15 +469,14 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   /// constants.
   SmallPtrSet<Function *, 16> TrackingIncomingArguments;
 
-  /// The reason for two worklists is that overdefined is the lowest state
-  /// on the lattice, and moving things to overdefined as fast as possible
-  /// makes SCCP converge much faster.
-  ///
-  /// By having a separate worklist, we accomplish this because everything
-  /// possibly overdefined will become overdefined at the soonest possible
-  /// point.
-  SmallVector<Value *, 64> OverdefinedInstWorkList;
-  SmallVector<Value *, 64> InstWorkList;
+  /// Worklist of instructions to re-visit. This only includes instructions
+  /// in blocks that have already been visited at least once.
+  SmallSetVector<Instruction *, 16> InstWorkList;
+
+  /// Current instruction while visiting a block for the first time, used to
+  /// avoid unnecessary instruction worklist insertions. Null if an instruction
+  /// is visited outside a whole-block visitation.
+  Instruction *CurI = nullptr;
 
   // The BasicBlock work list
   SmallVector<BasicBlock *, 64> BBWorkList;
@@ -497,12 +499,15 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
     return dyn_cast_or_null<ConstantInt>(getConstant(IV, Ty));
   }
 
-  // pushToWorkList - Helper for markConstant/markOverdefined
-  void pushToWorkList(ValueLatticeElement &IV, Value *V);
+  /// Push instruction \p I to the worklist.
+  void pushToWorkList(Instruction *I);
+
+  /// Push users of value \p V to the worklist.
+  void pushUsersToWorkList(Value *V);
 
-  // Helper to push \p V to the worklist, after updating it to \p IV. Also
-  // prints a debug message with the updated value.
-  void pushToWorkListMsg(ValueLatticeElement &IV, Value *V);
+  /// Like pushUsersToWorkList(), but also prints a debug message with the
+  /// updated value.
+  void pushUsersToWorkListMsg(ValueLatticeElement &IV, Value *V);
 
   // markConstant - Make a value be marked as "constant".  If the value
   // is not already a constant, add it to the instruction work list so that
@@ -661,46 +666,9 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   // successors are reachable from a given terminator instruction.
   void getFeasibleSuccessors(Instruction &TI, SmallVectorImpl<bool> &Succs);
 
-  // OperandChangedState - This method is invoked on all of the users of an
-  // instruction that was just changed state somehow.  Based on this
-  // information, we need to update the specified user of this instruction.
-  void operandChangedState(Instruction *I) {
-    if (BBExecutable.count(I->getParent())) // Inst is executable?
-      visit(*I);
-  }
-
   // Add U as additional user of V.
   void addAdditionalUser(Value *V, User *U) { AdditionalUsers[V].insert(U); }
 
-  // Mark I's users as changed, including AdditionalUsers.
-  void markUsersAsChanged(Value *I) {
-    // Functions include their arguments in the use-list. Changed function
-    // values mean that the result of the function changed. We only need to
-    // update the call sites with the new function result and do not have to
-    // propagate the call arguments.
-    if (isa<Function>(I)) {
-      for (User *U : I->users()) {
-        if (auto *CB = dyn_cast<CallBase>(U))
-          handleCallResult(*CB);
-      }
-    } else {
-      for (User *U : I->users())
-        if (auto *UI = dyn_cast<Instruction>(U))
-          operandChangedState(UI);
-    }
-
-    auto Iter = AdditionalUsers.find(I);
-    if (Iter != AdditionalUsers.end()) {
-      // Copy additional users before notifying them of changes, because new
-      // users may be added, potentially invalidating the iterator.
-      SmallVector<Instruction *, 2> ToNotify;
-      for (User *U : Iter->second)
-        if (auto *UI = dyn_cast<Instruction>(U))
-          ToNotify.push_back(UI);
-      for (Instruction *UI : ToNotify)
-        operandChangedState(UI);
-    }
-  }
   void handleCallOverdefined(CallBase &CB);
   void handleCallResult(CallBase &CB);
   void handleCallArguments(CallBase &CB);
@@ -985,19 +953,41 @@ bool SCCPInstVisitor::markBlockExecutable(BasicBlock *BB) {
   return true;
 }
 
-void SCCPInstVisitor::pushToWorkList(ValueLatticeElement &IV, Value *V) {
-  if (IV.isOverdefined()) {
-    if (OverdefinedInstWorkList.empty() || OverdefinedInstWorkList.back() != V)
-      OverdefinedInstWorkList.push_back(V);
+void SCCPInstVisitor::pushToWorkList(Instruction *I) {
+  // If we're currently visiting a block, do not push any instructions in the
+  // same blocks that are after the current one, as they will be visited
+  // anyway. We do have to push updates to earlier instructions (e.g. phi
+  // nodes or loads of tracked globals).
+  if (CurI && I->getParent() == CurI->getParent() && !I->comesBefore(CurI))
     return;
+  // Only push instructions in already visited blocks. Otherwise we'll handle
+  // it when we visit the block for the first time.
+  if (BBVisited.contains(I->getParent()))
+    InstWorkList.insert(I);
+}
+
+void SCCPInstVisitor::pushUsersToWorkList(Value *V) {
+  for (User *U : V->users())
+    if (auto *UI = dyn_cast<Instruction>(U))
+      pushToWorkList(UI);
+
+  auto Iter = AdditionalUsers.find(V);
+  if (Iter != AdditionalUsers.end()) {
+    // Copy additional users before notifying them of changes, because new
+    // users may be added, potentially invalidating the iterator.
+    SmallVector<Instruction *, 2> ToNotify;
+    for (User *U : Iter->second)
+      if (auto *UI = dyn_cast<Instruction>(U))
+        ToNotify.push_back(UI);
+    for (Instruction *UI : ToNotify)
+      pushToWorkList(UI);
   }
-  if (InstWorkList.empty() || InstWorkList.back() != V)
-    InstWorkList.push_back(V);
 }
 
-void SCCPInstVisitor::pushToWorkListMsg(ValueLatticeElement &IV, Value *V) {
+void SCCPInstVisitor::pushUsersToWorkListMsg(ValueLatticeElement &IV,
+                                             Value *V) {
   LLVM_DEBUG(dbgs() << "updated " << IV << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
 }
 
 bool SCCPInstVisitor::markConstant(ValueLatticeElement &IV, Value *V,
@@ -1005,7 +995,7 @@ bool SCCPInstVisitor::markConstant(ValueLatticeElement &IV, Value *V,
   if (!IV.markConstant(C, MayIncludeUndef))
     return false;
   LLVM_DEBUG(dbgs() << "markConstant: " << *C << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1014,7 +1004,7 @@ bool SCCPInstVisitor::markNotConstant(ValueLatticeElement &IV, Value *V,
   if (!IV.markNotConstant(C))
     return false;
   LLVM_DEBUG(dbgs() << "markNotConstant: " << *C << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1023,7 +1013,7 @@ bool SCCPInstVisitor::markConstantRange(ValueLatticeElement &IV, Value *V,
   if (!IV.markConstantRange(CR))
     return false;
   LLVM_DEBUG(dbgs() << "markConstantRange: " << CR << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1036,7 +1026,7 @@ bool SCCPInstVisitor::markOverdefined(ValueLatticeElement &IV, Value *V) {
              << "Function '" << F->getName() << "'\n";
              else dbgs() << *V << '\n');
   // Only instructions go on the work list
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1144,7 +1134,7 @@ bool SCCPInstVisitor::mergeInValue(ValueLatticeElement &IV, Value *V,
                                    ValueLatticeElement MergeWithV,
                                    ValueLatticeElement::MergeOptions Opts) {
   if (IV.mergeIn(MergeWithV, Opts)) {
-    pushToWorkList(IV, V);
+    pushUsersToWorkList(V);
     LLVM_DEBUG(dbgs() << "Merged " << MergeWithV << " into " << *V << " : "
                       << IV << "\n");
     return true;
@@ -1164,7 +1154,7 @@ bool SCCPInstVisitor::markEdgeExecutable(BasicBlock *Source, BasicBlock *Dest) {
                       << " -> " << Dest->getName() << '\n');
 
     for (PHINode &PN : Dest->phis())
-      visitPHINode(PN);
+      pushToWorkList(&PN);
   }
   return true;
 }
@@ -1540,7 +1530,7 @@ void SCCPInstVisitor::visitSelectInst(SelectInst &I) {
   bool Changed = State.mergeIn(TVal);
   Changed |= State.mergeIn(FVal);
   if (Changed)
-    pushToWorkListMsg(State, &I);
+    pushUsersToWorkListMsg(State, &I);
 }
 
 // Handle Unary Operators.
@@ -2039,53 +2029,28 @@ void SCCPInstVisitor::handleCallResult(CallBase &CB) {
 
 void SCCPInstVisitor::solve() {
   // Process the work lists until they are empty!
-  while (!BBWorkList.empty() || !InstWorkList.empty() ||
-         !OverdefinedInstWorkList.empty()) {
-    // Process the overdefined instruction's work list first, which drives other
-    // things to overdefined more quickly.
-    while (!OverdefinedInstWorkList.empty()) {
-      Value *I = OverdefinedInstWorkList.pop_back_val();
-      Invalidated.erase(I);
-
-      LLVM_DEBUG(dbgs() << "\nPopped off OI-WL: " << *I << '\n');
-
-      // "I" got into the work list because it either made the transition from
-      // bottom to constant, or to overdefined.
-      //
-      // Anything on this worklist that is overdefined need not be visited
-      // since all of its users will have already been marked as overdefined
-      // Update all of the users of this instruction's value.
-      //
-      markUsersAsChanged(I);
-    }
-
+  while (!BBWorkList.empty() || !InstWorkList.empty()) {
     // Process the instruction work list.
     while (!InstWorkList.empty()) {
-      Value *I = InstWorkList.pop_back_val();
+      Instruction *I = InstWorkList.pop_back_val();
       Invalidated.erase(I);
 
       LLVM_DEBUG(dbgs() << "\nPopped off I-WL: " << *I << '\n');
 
-      // "I" got into the work list because it made the transition from undef to
-      // constant.
-      //
-      // Anything on this worklist that is overdefined need not be visited
-      // since all of its users will have already been marked as overdefined.
-      // Update all of the users of this instruction's value.
-      //
-      if (I->getType()->isStructTy() || !getValueState(I).isOverdefined())
-        markUsersAsChanged(I);
+      visit(I);
     }
 
     // Process the basic block work list.
     while (!BBWorkList.empty()) {
       BasicBlock *BB = BBWorkList.pop_back_val();
+      BBVisited.insert(BB);
 
       LLVM_DEBUG(dbgs() << "\nPopped off BBWL: " << *BB << '\n');
-
-      // Notify all instructions in this basic block that they are newly
-      // executable.
-      visit(BB);
+      for (Instruction &I : *BB) {
+        CurI = &I;
+        visit(I);
+      }
+      CurI = nullptr;
     }
   }
 }
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index e205551658aa5..a75f29000ca18 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6198,7 +6198,7 @@ static bool initializeUniqueCases(SwitchInst *SI, PHINode *&PHI,
 // TODO: Handle switches with more than 2 cases that map to the same result.
 static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector,
                                  Constant *DefaultResult, Value *Condition,
-                                 IRBuilder<> &Builder) {
+                                 IRBuilder<> &Builder, const DataLayout &DL) {
   // If we are selecting between only two cases transform into a simple
   // select or a two-way select if default is possible.
   // Example:
@@ -6234,10 +6234,33 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector,
     // case 0,2,8,10 -> Cond & 0b1..0101 == 0 ? result : default
     if (isPowerOf2_32(CaseCount)) {
       ConstantInt *MinCaseVal = CaseValues[0];
-      // Find mininal value.
-      for (auto *Case : CaseValues)
+      // If there are bits that are set exclusively by CaseValues, we
+      // can transform the switch into a select if the conjunction of
+      // all the values uniquely identify CaseValues.
+      APInt AndMask = APInt::getAllOnes(MinCaseVal->getBitWidth());
+
+      // Find the minimum value and compute the and of all the case values.
+      for (auto *Case : CaseValues) {
         if (Case->getValue().slt(MinCaseVal->getValue()))
           MinCaseVal = Case;
+        AndMask &= Case->getValue();
+      }
+      KnownBits Known = computeKnownBits(Condition, DL);
+
+      if (!AndMask.isZero() && Known.getMaxValue().uge(AndMask)) {
+        // Compute the number of bits that are free to vary.
+        unsigned FreeBits = Known.countMaxActiveBits() - AndMask.popcount();
+
+        // Check if the number of values covered by the mask is equal
+        // to the number of cases.
+        if (FreeBits == Log2_32(CaseCount)) {
+          Value *And = Builder.CreateAnd(Condition, AndMask);
+          Value *Cmp = Builder.CreateICmpEQ(
+              And, Constant::getIntegerValue(And->getType(), AndMask));
+          return Builder.CreateSelect(Cmp, ResultVector[0].first,
+                                      DefaultResult);
+        }
+      }
 
       // Mark the bits case number touched.
       APInt BitMask = APInt::getZero(MinCaseVal->getBitWidth());
@@ -6325,7 +6348,7 @@ static bool trySwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
   assert(PHI != nullptr && "PHI for value select not found");
   Builder.SetInsertPoint(SI);
   Value *SelectValue =
-      foldSwitchToSelect(UniqueResults, DefaultResult, Cond, Builder);
+      foldSwitchToSelect(UniqueResults, DefaultResult, Cond, Builder, DL);
   if (!SelectValue)
     return false;
 
@@ -6347,7 +6370,7 @@ class SwitchLookupTable {
 
   /// Build instructions with Builder to retrieve the value at
   /// the position given by Index in the lookup table.
-  Value *buildLookup(Value *Index, IRBuilder<> &Builder);
+  Value *buildLookup(Value *Index, IRBuilder<> &Builder, const DataLayout &DL);
 
   /// Return true if a table with TableSize elements of
   /// type ElementType would fit in a target-legal register.
@@ -6533,7 +6556,8 @@ SwitchLookupTable::SwitchLookupTable(
   Kind = ArrayKind;
 }
 
-Value *SwitchLookupTable::buildLookup(Value *Index, IRBuilder<> &Builder) {
+Value *SwitchLookupTable::buildLookup(Value *Index, IRBuilder<> &Builder,
+                                      const DataLayout &DL) {
   switch (Kind) {
   case SingleValueKind:
     return SingleValue;
@@ -6575,16 +6599,12 @@ Value *SwitchLookupTable::buildLookup(Value *Index, IRBuilder<> &Builder) {
     return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
   }
   case ArrayKind: {
-    // Make sure the table index will not overflow when treated as signed.
-    IntegerType *IT = cast<IntegerType>(Index->getType());
-    uint64_t TableSize =
-        Array->getInitializer()->getType()->getArrayNumElements();
-    if (TableSize > (1ULL << std::min(IT->getBitWidth() - 1, 63u)))
-      Index = Builder.CreateZExt(
-          Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1),
-          "switch.tableidx.zext");
-
-    Value *GEPIndices[] = {Builder.getInt32(0), Index};
+    Type *IndexTy = DL.getIndexType(Array->getType());
+
+    if (Index->getType() != IndexTy)
+      Index = Builder.CreateZExtOrTrunc(Index, IndexTy);
+
+    Value *GEPIndices[] = {ConstantInt::get(IndexTy, 0), Index};
     Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
                                            GEPIndices, "switch.gep");
     return Builder.CreateLoad(
@@ -7064,7 +7084,7 @@ static bool switchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     SwitchLookupTable Table(Mod, TableSize, TableIndexOffset, ResultList, DV,
                             DL, FuncName);
 
-    Value *Result = Table.buildLookup(TableIndex, Builder);
+    Value *Result = Table.buildLookup(TableIndex, Builder, DL);
 
     // Do a small peephole optimization: re-use the switch table compare if
     // possible.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 144f35e10132f..0eaa6d9bf03da 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -64,12 +64,6 @@ class VPBuilder {
     return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
   }
 
-  VPInstruction *createInstruction(unsigned Opcode,
-                                   std::initializer_list<VPValue *> Operands,
-                                   DebugLoc DL, const Twine &Name = "") {
-    return createInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name);
-  }
-
 public:
   VPBuilder() = default;
   VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); }
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index aa16083829625..6bd6ba2d77bbb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1778,9 +1778,6 @@ class GeneratedRTChecks {
   /// If it is nullptr no memory runtime checks have been generated.
   Value *MemRuntimeCheckCond = nullptr;
 
-  /// True if any checks have been added.
-  bool AddedAnyChecks = false;
-
   DominatorTree *DT;
   LoopInfo *LI;
   TargetTransformInfo *TTI;
@@ -2044,7 +2041,6 @@ class GeneratedRTChecks {
     if (AddBranchWeights)
       setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false);
     ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI);
-    AddedAnyChecks = true;
     return SCEVCheckBlock;
   }
 
@@ -2072,12 +2068,15 @@ class GeneratedRTChecks {
     MemCheckBlock->getTerminator()->setDebugLoc(
         Pred->getTerminator()->getDebugLoc());
 
-    AddedAnyChecks = true;
     return MemCheckBlock;
   }
 
   /// Return true if any runtime checks have been added
-  bool hasChecks() const { return AddedAnyChecks; }
+  bool hasChecks() const {
+    using namespace llvm::PatternMatch;
+    return (SCEVCheckCond && !match(SCEVCheckCond, m_ZeroInt())) ||
+           MemRuntimeCheckCond;
+  }
 };
 } // namespace
 
@@ -4677,7 +4676,7 @@ void LoopVectorizationCostModel::collectElementTypesForWidening() {
         if (!Legal->isReductionVariable(PN))
           continue;
         const RecurrenceDescriptor &RdxDesc =
-            Legal->getReductionVars().find(PN)->second;
+            Legal->getRecurrenceDescriptor(PN);
         if (PreferInLoopReductions || useOrderedReductions(RdxDesc) ||
             TTI.preferInLoopReduction(RdxDesc.getRecurrenceKind(),
                                       RdxDesc.getRecurrenceType()))
@@ -4922,7 +4921,7 @@ LoopVectorizationCostModel::selectInterleaveCount(VPlan &Plan, ElementCount VF,
           const RecurrenceDescriptor &RdxDesc = Reduction.second;
           RecurKind RK = RdxDesc.getRecurrenceKind();
           return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
-                 RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK);
+                 RecurrenceDescriptor::isFindIVRecurrenceKind(RK);
         });
     if (HasSelectCmpReductions) {
       LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
@@ -5477,7 +5476,7 @@ LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
     ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
 
   const RecurrenceDescriptor &RdxDesc =
-      Legal->getReductionVars().find(cast<PHINode>(ReductionPhi))->second;
+      Legal->getRecurrenceDescriptor(cast<PHINode>(ReductionPhi));
 
   InstructionCost BaseCost;
   RecurKind RK = RdxDesc.getRecurrenceKind();
@@ -6556,7 +6555,7 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
   }
 
   for (const auto &[_, Ops] : DeadInvariantStoreOps)
-    llvm::append_range(DeadOps, ArrayRef(Ops).drop_back());
+    llvm::append_range(DeadOps, drop_end(Ops));
 
   // Mark ops that would be trivially dead and are only used by ignored
   // instructions as free.
@@ -7240,8 +7239,8 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
 
 static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
   using namespace VPlanPatternMatch;
-  assert(RdxResult->getOpcode() == VPInstruction::ComputeFindLastIVResult &&
-         "RdxResult must be ComputeFindLastIVResult");
+  assert(RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult &&
+         "RdxResult must be ComputeFindIVResult");
   VPValue *StartVPV = RdxResult->getOperand(1);
   match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
   return StartVPV->getLiveInIRValue();
@@ -7255,17 +7254,19 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
   // Get the VPInstruction computing the reduction result in the middle block.
   // The first operand may not be from the middle block if it is not connected
   // to the scalar preheader. In that case, there's nothing to fix.
-  auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
+  VPValue *Incoming = EpiResumePhiR->getOperand(0);
+  match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
+                      VPlanPatternMatch::m_VPValue(Incoming)));
+  auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
   if (!EpiRedResult ||
       (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
        EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
-       EpiRedResult->getOpcode() != VPInstruction::ComputeFindLastIVResult))
+       EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
     return;
 
   auto *EpiRedHeaderPhi =
       cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
-  const RecurrenceDescriptor &RdxDesc =
-      EpiRedHeaderPhi->getRecurrenceDescriptor();
+  RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
   Value *MainResumeValue;
   if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
     assert((VPI->getOpcode() == VPInstruction::Broadcast ||
@@ -7274,8 +7275,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
     MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
   } else
     MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
-  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-          RdxDesc.getRecurrenceKind())) {
+  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
     [[maybe_unused]] Value *StartV =
         EpiRedResult->getOperand(1)->getLiveInIRValue();
     auto *Cmp = cast<ICmpInst>(MainResumeValue);
@@ -7285,8 +7285,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
            "AnyOf expected to start by comparing main resume value to original "
            "start value");
     MainResumeValue = Cmp->getOperand(0);
-  } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
-                 RdxDesc.getRecurrenceKind())) {
+  } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
     Value *StartV = getStartValueFromReductionResult(EpiRedResult);
     Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
     using namespace llvm::PatternMatch;
@@ -7324,9 +7323,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
                            OrigLoop->getHeader()->getContext());
   VPlanTransforms::runPass(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
   VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
-  if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
+  if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
+    std::optional<unsigned> VScale = CM.getVScaleForTuning();
     VPlanTransforms::runPass(VPlanTransforms::addBranchWeightToMiddleTerminator,
-                             BestVPlan, BestVF);
+                             BestVPlan, BestVF, VScale);
+  }
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
   VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
   VPlanTransforms::narrowInterleaveGroups(
@@ -7764,8 +7765,9 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
           (CM.foldTailByMasking() || !GEP || !GEP->isInBounds())
               ? GEPNoWrapFlags::none()
               : GEPNoWrapFlags::inBounds();
-      VectorPtr = new VPVectorEndPointerRecipe(
-          Ptr, &Plan.getVF(), getLoadStoreType(I), Flags, I->getDebugLoc());
+      VectorPtr =
+          new VPVectorEndPointerRecipe(Ptr, &Plan.getVF(), getLoadStoreType(I),
+                                       /*Stride*/ -1, Flags, I->getDebugLoc());
     } else {
       VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
                                             GEP ? GEP->getNoWrapFlags()
@@ -8143,15 +8145,15 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   // something that isn't another partial reduction. This is because the
   // extends are intended to be lowered along with the reduction itself.
 
-  // Build up a set of partial reduction bin ops for efficient use checking.
-  SmallSet<User *, 4> PartialReductionBinOps;
+  // Build up a set of partial reduction ops for efficient use checking.
+  SmallSet<User *, 4> PartialReductionOps;
   for (const auto &[PartialRdx, _] : PartialReductionChains)
-    PartialReductionBinOps.insert(PartialRdx.BinOp);
+    PartialReductionOps.insert(PartialRdx.ExtendUser);
 
   auto ExtendIsOnlyUsedByPartialReductions =
-      [&PartialReductionBinOps](Instruction *Extend) {
+      [&PartialReductionOps](Instruction *Extend) {
         return all_of(Extend->users(), [&](const User *U) {
-          return PartialReductionBinOps.contains(U);
+          return PartialReductionOps.contains(U);
         });
       };
 
@@ -8160,7 +8162,7 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   for (auto Pair : PartialReductionChains) {
     PartialReductionChain Chain = Pair.first;
     if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
-        ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB))
+        (!Chain.ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
       ScaledReductionMap.try_emplace(Chain.Reduction, Pair.second);
   }
 }
@@ -8168,7 +8170,6 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
 bool VPRecipeBuilder::getScaledReductions(
     Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
     SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
-
   if (!CM.TheLoop->contains(RdxExitInstr))
     return false;
 
@@ -8197,43 +8198,71 @@ bool VPRecipeBuilder::getScaledReductions(
   if (PhiOp != PHI)
     return false;
 
-  auto *BinOp = dyn_cast<BinaryOperator>(Op);
-  if (!BinOp || !BinOp->hasOneUse())
-    return false;
-
   using namespace llvm::PatternMatch;
-  // Use the side-effect of match to replace BinOp only if the pattern is
-  // matched, we don't care at this point whether it actually matched.
-  match(BinOp, m_Neg(m_BinOp(BinOp)));
 
-  Value *A, *B;
-  if (!match(BinOp->getOperand(0), m_ZExtOrSExt(m_Value(A))) ||
-      !match(BinOp->getOperand(1), m_ZExtOrSExt(m_Value(B))))
-    return false;
+  // If the update is a binary operator, check both of its operands to see if
+  // they are extends. Otherwise, see if the update comes directly from an
+  // extend.
+  Instruction *Exts[2] = {nullptr};
+  BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
+  std::optional<unsigned> BinOpc;
+  Type *ExtOpTypes[2] = {nullptr};
+
+  auto CollectExtInfo = [&Exts,
+                         &ExtOpTypes](SmallVectorImpl<Value *> &Ops) -> bool {
+    unsigned I = 0;
+    for (Value *OpI : Ops) {
+      Value *ExtOp;
+      if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))))
+        return false;
+      Exts[I] = cast<Instruction>(OpI);
+      ExtOpTypes[I] = ExtOp->getType();
+      I++;
+    }
+    return true;
+  };
+
+  if (ExtendUser) {
+    if (!ExtendUser->hasOneUse())
+      return false;
+
+    // Use the side-effect of match to replace BinOp only if the pattern is
+    // matched, we don't care at this point whether it actually matched.
+    match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
+
+    SmallVector<Value *> Ops(ExtendUser->operands());
+    if (!CollectExtInfo(Ops))
+      return false;
 
-  Instruction *ExtA = cast<Instruction>(BinOp->getOperand(0));
-  Instruction *ExtB = cast<Instruction>(BinOp->getOperand(1));
+    BinOpc = std::make_optional(ExtendUser->getOpcode());
+  } else if (match(Update, m_Add(m_Value(), m_Value()))) {
+    // We already know the operands for Update are Op and PhiOp.
+    SmallVector<Value *> Ops({Op});
+    if (!CollectExtInfo(Ops))
+      return false;
+
+    ExtendUser = Update;
+    BinOpc = std::nullopt;
+  } else
+    return false;
 
   TTI::PartialReductionExtendKind OpAExtend =
-      TargetTransformInfo::getPartialReductionExtendKind(ExtA);
+      TTI::getPartialReductionExtendKind(Exts[0]);
   TTI::PartialReductionExtendKind OpBExtend =
-      TargetTransformInfo::getPartialReductionExtendKind(ExtB);
-
-  PartialReductionChain Chain(RdxExitInstr, ExtA, ExtB, BinOp);
+      Exts[1] ? TTI::getPartialReductionExtendKind(Exts[1]) : TTI::PR_None;
+  PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
 
   TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
-  TypeSize ASize = A->getType()->getPrimitiveSizeInBits();
-
+  TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
   if (!PHISize.hasKnownScalarFactor(ASize))
     return false;
-
   unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
 
   if (LoopVectorizationPlanner::getDecisionAndClampRange(
           [&](ElementCount VF) {
             InstructionCost Cost = TTI->getPartialReductionCost(
-                Update->getOpcode(), A->getType(), B->getType(), PHI->getType(),
-                VF, OpAExtend, OpBExtend, BinOp->getOpcode(), CM.CostKind);
+                Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
+                PHI->getType(), VF, OpAExtend, OpBExtend, BinOpc, CM.CostKind);
             return Cost.isValid();
           },
           Range)) {
@@ -8268,8 +8297,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
            "can only widen reductions and fixed-order recurrences here");
     VPValue *StartV = Operands[0];
     if (Legal->isReductionVariable(Phi)) {
-      const RecurrenceDescriptor &RdxDesc =
-          Legal->getReductionVars().find(Phi)->second;
+      const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
       assert(RdxDesc.getRecurrenceStartValue() ==
              Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
 
@@ -8277,7 +8305,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
       unsigned ScaleFactor =
           getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
       PhiRecipe = new VPReductionPHIRecipe(
-          Phi, RdxDesc, *StartV, CM.isInLoopReduction(Phi),
+          Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
           CM.useOrderedReductions(RdxDesc), ScaleFactor);
     } else {
       // TODO: Currently fixed-order recurrences are modeled as chains of
@@ -9037,12 +9065,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
       continue;
 
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind Kind = RdxDesc.getRecurrenceKind();
+    RecurKind Kind = PhiR->getRecurrenceKind();
     assert(
         !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
-        !RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
-        "AnyOf and FindLast reductions are not allowed for in-loop reductions");
+        !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
+        "AnyOf and FindIV reductions are not allowed for in-loop reductions");
 
     // Collect the chain of "link" recipes for the reduction starting at PhiR.
     SetVector<VPSingleDefRecipe *> Worklist;
@@ -9070,7 +9097,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     // get folded to their non-phi operand, as the reduction recipe handles the
     // condition directly.
     VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
-    for (VPSingleDefRecipe *CurrentLink : Worklist.getArrayRef().drop_front()) {
+    for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
       if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
         assert(Blend->getNumIncomingValues() == 2 &&
                "Blend must have 2 incoming values");
@@ -9144,13 +9171,16 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
         CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
 
+      // TODO: Retrieve FMFs from recipes directly.
+      RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
+          cast<PHINode>(PhiR->getUnderlyingInstr()));
       // Non-FP RdxDescs will have all fast math flags set, so clear them.
       FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
                                ? RdxDesc.getFastMathFlags()
                                : FastMathFlags();
       auto *RedRecipe = new VPReductionRecipe(
           Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
-          CM.useOrderedReductions(RdxDesc), CurrentLinkI->getDebugLoc());
+          PhiR->isOrdered(), CurrentLinkI->getDebugLoc());
       // Append the recipe to the end of the VPBasicBlock because we need to
       // ensure that it comes after all of it's inputs, including CondOp.
       // Delete CurrentLink as it will be invalid if its operand is replaced
@@ -9174,7 +9204,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (!PhiR)
       continue;
 
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+    const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
+        cast<PHINode>(PhiR->getUnderlyingInstr()));
     Type *PhiTy = PhiR->getUnderlyingValue()->getType();
     // If tail is folded by masking, introduce selects between the phi
     // and the users outside the vector region of each reduction, at the
@@ -9200,34 +9231,12 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
                 cast<VPInstruction>(&U)->getOpcode() ==
                     VPInstruction::ComputeReductionResult ||
                 cast<VPInstruction>(&U)->getOpcode() ==
-                    VPInstruction::ComputeFindLastIVResult);
+                    VPInstruction::ComputeFindIVResult);
       });
       if (CM.usePredicatedReductionSelect())
         PhiR->setOperand(1, NewExitingVPV);
     }
 
-    // If the vector reduction can be performed in a smaller type, we truncate
-    // then extend the loop exit value to enable InstCombine to evaluate the
-    // entire expression in the smaller type.
-    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
-        !RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
-      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
-      Type *RdxTy = RdxDesc.getRecurrenceType();
-      auto *Trunc =
-          new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
-      auto *Extnd =
-          RdxDesc.isSigned()
-              ? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
-              : new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
-
-      Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
-      Extnd->insertAfter(Trunc);
-      if (PhiR->getOperand(1) == NewExitingVPV)
-        PhiR->setOperand(1, Extnd->getVPSingleValue());
-      NewExitingVPV = Extnd;
-    }
-
     // We want code in the middle block to appear to execute on the location of
     // the scalar loop's latch terminator because: (a) it is all compiler
     // generated, (b) these instructions are always executed after evaluating
@@ -9244,28 +9253,53 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     VPInstruction *FinalReductionResult;
     VPBuilder::InsertPointGuard Guard(Builder);
     Builder.setInsertPoint(MiddleVPBB, IP);
-    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
+    RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
+    if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind)) {
       VPValue *Start = PhiR->getStartValue();
       VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
       FinalReductionResult =
-          Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
+          Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
                                {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
-    } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-                   RdxDesc.getRecurrenceKind())) {
+    } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
       VPValue *Start = PhiR->getStartValue();
       FinalReductionResult =
           Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
                                {PhiR, Start, NewExitingVPV}, ExitDL);
     } else {
-      VPIRFlags Flags = RecurrenceDescriptor::isFloatingPointRecurrenceKind(
-                            RdxDesc.getRecurrenceKind())
-                            ? VPIRFlags(RdxDesc.getFastMathFlags())
-                            : VPIRFlags();
+      VPIRFlags Flags =
+          RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurrenceKind)
+              ? VPIRFlags(RdxDesc.getFastMathFlags())
+              : VPIRFlags();
       FinalReductionResult =
           Builder.createNaryOp(VPInstruction::ComputeReductionResult,
                                {PhiR, NewExitingVPV}, Flags, ExitDL);
     }
+    // If the vector reduction can be performed in a smaller type, we truncate
+    // then extend the loop exit value to enable InstCombine to evaluate the
+    // entire expression in the smaller type.
+    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
+        !RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
+      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
+      assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
+             "Unexpected truncated min-max recurrence!");
+      Type *RdxTy = RdxDesc.getRecurrenceType();
+      auto *Trunc =
+          new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
+      Instruction::CastOps ExtendOpc =
+          RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
+      auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
+      Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
+      Extnd->insertAfter(Trunc);
+      if (PhiR->getOperand(1) == NewExitingVPV)
+        PhiR->setOperand(1, Extnd->getVPSingleValue());
+
+      // Update ComputeReductionResult with the truncated exiting value and
+      // extend its result.
+      FinalReductionResult->setOperand(1, Trunc);
+      FinalReductionResult =
+          Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
+    }
+
     // Update all users outside the vector region. Also replace redundant
     // ExtractLastElement.
     for (auto *U : to_vector(OrigExitingVPV->users())) {
@@ -9282,8 +9316,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     // with a boolean reduction phi node to check if the condition is true in
     // any iteration. The final value is selected by the final
     // ComputeReductionResult.
-    if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
+    if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
       auto *Select = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
         return isa<VPWidenSelectRecipe>(U) ||
                (isa<VPReplicateRecipe>(U) &&
@@ -9312,16 +9345,16 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       continue;
     }
 
-    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
+    if (RecurrenceDescriptor::isFindIVRecurrenceKind(
             RdxDesc.getRecurrenceKind())) {
-      // Adjust the start value for FindLastIV recurrences to use the sentinel
-      // value after generating the ResumePhi recipe, which uses the original
-      // start value.
+      // Adjust the start value for FindFirstIV/FindLastIV recurrences to use
+      // the sentinel value after generating the ResumePhi recipe, which uses
+      // the original start value.
       PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
     }
     RecurKind RK = RdxDesc.getRecurrenceKind();
     if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
-         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
+         !RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
          !RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))) {
       VPBuilder PHBuilder(Plan->getVectorPreheader());
       VPValue *Iden = Plan->getOrAddLiveIn(
@@ -9704,18 +9737,18 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
   VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan);
 
   using namespace VPlanPatternMatch;
-  // When vectorizing the epilogue, FindLastIV reductions can introduce multiple
-  // uses of undef/poison. If the reduction start value may be undef or poison
-  // it needs to be frozen and the frozen start has to be used when computing
-  // the reduction result. We also need to use the frozen value in the resume
-  // phi generated by the main vector loop, as this is also used to compute the
-  // reduction result after the epilogue vector loop.
+  // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
+  // introduce multiple uses of undef/poison. If the reduction start value may
+  // be undef or poison it needs to be frozen and the frozen start has to be
+  // used when computing the reduction result. We also need to use the frozen
+  // value in the resume phi generated by the main vector loop, as this is also
+  // used to compute the reduction result after the epilogue vector loop.
   auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
                                              bool UpdateResumePhis) {
     VPBuilder Builder(Plan.getEntry());
     for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
       auto *VPI = dyn_cast<VPInstruction>(&R);
-      if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindLastIVResult)
+      if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
         continue;
       VPValue *OrigStart = VPI->getOperand(1);
       if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
@@ -9810,38 +9843,30 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
             return VPI &&
                    (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
                     VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
-                    VPI->getOpcode() == VPInstruction::ComputeFindLastIVResult);
+                    VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
           }));
       ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
                     ->getIncomingValueForBlock(L->getLoopPreheader());
-      const RecurrenceDescriptor &RdxDesc =
-          ReductionPhi->getRecurrenceDescriptor();
-      RecurKind RK = RdxDesc.getRecurrenceKind();
+      RecurKind RK = ReductionPhi->getRecurrenceKind();
       if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
         Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
-        assert(RdxDesc.getRecurrenceStartValue() == StartV &&
-               "start value from ComputeAnyOfResult must match");
-
         // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
         // start value; compare the final value from the main vector loop
         // to the start value.
         BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
         ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
-      } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
+      } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
         Value *StartV = getStartValueFromReductionResult(RdxResult);
-        assert(RdxDesc.getRecurrenceStartValue() == StartV &&
-               "start value from ComputeFindLastIVResult must match");
-
         ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
             EPI.MainLoopIterationCountCheck);
 
-        // VPReductionPHIRecipe for FindLastIV reductions requires an adjustment
-        // to the resume value. The resume value is adjusted to the sentinel
-        // value when the final value from the main vector loop equals the start
-        // value. This ensures correctness when the start value might not be
-        // less than the minimum value of a monotonically increasing induction
-        // variable.
+        // VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
+        // an adjustment to the resume value. The resume value is adjusted to
+        // the sentinel value when the final value from the main vector loop
+        // equals the start value. This ensures correctness when the start value
+        // might not be less than the minimum value of a monotonically
+        // increasing induction variable.
         BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
         Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 27a7538ecd939..c93af749507f8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1511,8 +1511,8 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
 /// \returns true if all of the values in \p VL have the same type or false
 /// otherwise.
 static bool allSameType(ArrayRef<Value *> VL) {
-  Type *Ty = VL.front()->getType();
-  return all_of(VL.drop_front(), [&](Value *V) { return V->getType() == Ty; });
+  Type *Ty = VL.consume_front()->getType();
+  return all_of(VL, [&](Value *V) { return V->getType() == Ty; });
 }
 
 /// \returns True if in-tree use also needs extract. This refers to
@@ -5443,7 +5443,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
       MutableArrayRef<unsigned> Slice = CurrentOrder.slice(I * PartSz, Limit);
       // Shuffle of at least 2 vectors - ignore.
       if (any_of(Slice, [&](unsigned I) { return I != NumScalars; })) {
-        std::fill(Slice.begin(), Slice.end(), NumScalars);
+        llvm::fill(Slice, NumScalars);
         ShuffledSubMasks.set(I);
         continue;
       }
@@ -5471,7 +5471,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
       FirstMin = (FirstMin / PartSz) * PartSz;
       // Shuffle of at least 2 vectors - ignore.
       if (SecondVecFound) {
-        std::fill(Slice.begin(), Slice.end(), NumScalars);
+        llvm::fill(Slice, NumScalars);
         ShuffledSubMasks.set(I);
         continue;
       }
@@ -5492,7 +5492,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
       }
       // Shuffle of at least 2 vectors - ignore.
       if (SecondVecFound) {
-        std::fill(Slice.begin(), Slice.end(), NumScalars);
+        llvm::fill(Slice, NumScalars);
         ShuffledSubMasks.set(I);
         continue;
       }
@@ -5567,8 +5567,8 @@ static bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
 /// Calculates minimal alignment as a common alignment.
 template <typename T>
 static Align computeCommonAlignment(ArrayRef<Value *> VL) {
-  Align CommonAlignment = cast<T>(VL.front())->getAlign();
-  for (Value *V : VL.drop_front())
+  Align CommonAlignment = cast<T>(VL.consume_front())->getAlign();
+  for (Value *V : VL)
     CommonAlignment = std::min(CommonAlignment, cast<T>(V)->getAlign());
   return CommonAlignment;
 }
@@ -9483,8 +9483,8 @@ class PHIHandler {
       ArrayRef<unsigned> IncomingValues = P.second;
       if (IncomingValues.size() <= 1)
         continue;
-      unsigned BasicI = IncomingValues.front();
-      for (unsigned I : IncomingValues.drop_front()) {
+      unsigned BasicI = IncomingValues.consume_front();
+      for (unsigned I : IncomingValues) {
         assert(all_of(enumerate(Operands[I]),
                       [&](const auto &Data) {
                         return !Data.value() ||
@@ -23180,6 +23180,7 @@ class HorizontalReduction {
         case RecurKind::FMul:
         case RecurKind::FMulAdd:
         case RecurKind::AnyOf:
+        case RecurKind::FindFirstIVSMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
         case RecurKind::FMaximumNum:
@@ -23315,6 +23316,7 @@ class HorizontalReduction {
     case RecurKind::FMul:
     case RecurKind::FMulAdd:
     case RecurKind::AnyOf:
+    case RecurKind::FindFirstIVSMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
     case RecurKind::FMaximumNum:
@@ -23415,6 +23417,7 @@ class HorizontalReduction {
     case RecurKind::FMul:
     case RecurKind::FMulAdd:
     case RecurKind::AnyOf:
+    case RecurKind::FindFirstIVSMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
     case RecurKind::FMaximumNum:
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
index f3b62e36e5115..ddb4a1e154a18 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
@@ -24,6 +24,13 @@ static cl::opt<bool>
     AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
                  cl::desc("Allow non-power-of-2 vectorization."));
 
+#define LoadSeedsDef "loads"
+#define StoreSeedsDef "stores"
+cl::opt<std::string> CollectSeeds(
+    "sbvec-collect-seeds", cl::init(StoreSeedsDef), cl::Hidden,
+    cl::desc("Collect these seeds. Use empty for none or a comma-separated "
+             "list of '" StoreSeedsDef "' and '" LoadSeedsDef "'."));
+
 namespace sandboxir {
 SeedCollection::SeedCollection(StringRef Pipeline)
     : FunctionPass("seed-collection"),
@@ -38,10 +45,12 @@ bool SeedCollection::runOnFunction(Function &F, const Analyses &A) {
           : A.getTTI()
                 .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
                 .getFixedValue();
+  bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
+  bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
 
   // TODO: Start from innermost BBs first
   for (auto &BB : F) {
-    SeedCollector SC(&BB, A.getScalarEvolution());
+    SeedCollector SC(&BB, A.getScalarEvolution(), CollectStores, CollectLoads);
     for (SeedBundle &Seeds : SC.getStoreSeeds()) {
       unsigned ElmBits =
           Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index 008976ac9749d..1934326866cdf 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -21,12 +21,7 @@ namespace llvm::sandboxir {
 static cl::opt<unsigned> SeedBundleSizeLimit(
     "sbvec-seed-bundle-size-limit", cl::init(32), cl::Hidden,
     cl::desc("Limit the size of the seed bundle to cap compilation time."));
-#define LoadSeedsDef "loads"
-#define StoreSeedsDef "stores"
-static cl::opt<std::string> CollectSeeds(
-    "sbvec-collect-seeds", cl::init(LoadSeedsDef "," StoreSeedsDef), cl::Hidden,
-    cl::desc("Collect these seeds. Use empty for none or a comma-separated "
-             "list of '" LoadSeedsDef "' and '" StoreSeedsDef "'."));
+
 static cl::opt<unsigned> SeedGroupsLimit(
     "sbvec-seed-groups-limit", cl::init(256), cl::Hidden,
     cl::desc("Limit the number of collected seeds groups in a BB to "
@@ -162,11 +157,10 @@ template <typename LoadOrStoreT> static bool isValidMemSeed(LoadOrStoreT *LSI) {
 template bool isValidMemSeed<LoadInst>(LoadInst *LSI);
 template bool isValidMemSeed<StoreInst>(StoreInst *LSI);
 
-SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE)
+SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
+                             bool CollectStores, bool CollectLoads)
     : StoreSeeds(SE), LoadSeeds(SE), Ctx(BB->getContext()) {
 
-  bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
-  bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
   if (!CollectStores && !CollectLoads)
     return;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 8369c78a2d78f..41878e3c648e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -24,13 +24,14 @@ struct HistogramInfo;
 struct VFRange;
 
 /// A chain of instructions that form a partial reduction.
-/// Designed to match: reduction_bin_op (bin_op (extend (A), (extend (B))),
-/// accumulator).
+/// Designed to match either:
+///   reduction_bin_op (extend (A), accumulator), or
+///   reduction_bin_op (bin_op (extend (A), (extend (B))), accumulator).
 struct PartialReductionChain {
   PartialReductionChain(Instruction *Reduction, Instruction *ExtendA,
-                        Instruction *ExtendB, Instruction *BinOp)
-      : Reduction(Reduction), ExtendA(ExtendA), ExtendB(ExtendB), BinOp(BinOp) {
-  }
+                        Instruction *ExtendB, Instruction *ExtendUser)
+      : Reduction(Reduction), ExtendA(ExtendA), ExtendB(ExtendB),
+        ExtendUser(ExtendUser) {}
   /// The top-level binary operation that forms the reduction to a scalar
   /// after the loop body.
   Instruction *Reduction;
@@ -38,8 +39,8 @@ struct PartialReductionChain {
   Instruction *ExtendA;
   Instruction *ExtendB;
 
-  /// The binary operation using the extends that is then reduced.
-  Instruction *BinOp;
+  /// The user of the extends that is then reduced.
+  Instruction *ExtendUser;
 };
 
 /// Helper class to create VPRecipies from IR instructions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 32e788be76cb7..356af4a0e74e4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -528,11 +528,10 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
     case VPRecipeBase::VPDerivedIVSC:
     case VPRecipeBase::VPEVLBasedIVPHISC:
     case VPRecipeBase::VPExpandSCEVSC:
+    case VPRecipeBase::VPExpressionSC:
     case VPRecipeBase::VPInstructionSC:
     case VPRecipeBase::VPReductionEVLSC:
     case VPRecipeBase::VPReductionSC:
-    case VPRecipeBase::VPMulAccumulateReductionSC:
-    case VPRecipeBase::VPExtendedReductionSC:
     case VPRecipeBase::VPReplicateSC:
     case VPRecipeBase::VPScalarIVStepsSC:
     case VPRecipeBase::VPVectorPointerSC:
@@ -848,13 +847,12 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
            R->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
+           R->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
            R->getVPDefID() == VPRecipeBase::VPReductionSC ||
            R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
            R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
            R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC ||
-           R->getVPDefID() == VPRecipeBase::VPVectorPointerSC ||
-           R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC ||
-           R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC;
+           R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
   }
 
   static inline bool classof(const VPUser *U) {
@@ -946,7 +944,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
     /// Compute the final result of a AnyOf reduction with select(cmp(),x,y),
     /// where one of (x,y) is loop invariant, and both x and y are integer type.
     ComputeAnyOfResult,
-    ComputeFindLastIVResult,
+    ComputeFindIVResult,
     ComputeReductionResult,
     // Extracts the last lane from its operand if it is a vector, or the last
     // part if scalar. In the latter case, the recipe will be removed during
@@ -1706,17 +1704,23 @@ class VPWidenGEPRecipe : public VPRecipeWithIRFlags {
 
 /// A recipe to compute a pointer to the last element of each part of a widened
 /// memory access for widened memory accesses of IndexedTy. Used for
-/// VPWidenMemoryRecipes that are reversed.
+/// VPWidenMemoryRecipes or VPInterleaveRecipes that are reversed.
 class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags,
                                  public VPUnrollPartAccessor<2> {
   Type *IndexedTy;
 
+  /// The constant stride of the pointer computed by this recipe, expressed in
+  /// units of IndexedTy.
+  int64_t Stride;
+
 public:
   VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy,
-                           GEPNoWrapFlags GEPFlags, DebugLoc DL)
+                           int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPVectorEndPointerSC,
                             ArrayRef<VPValue *>({Ptr, VF}), GEPFlags, DL),
-        IndexedTy(IndexedTy) {}
+        IndexedTy(IndexedTy), Stride(Stride) {
+    assert(Stride < 0 && "Stride must be negative");
+  }
 
   VP_CLASSOF_IMPL(VPDef::VPVectorEndPointerSC)
 
@@ -1748,7 +1752,8 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags,
 
   VPVectorEndPointerRecipe *clone() override {
     return new VPVectorEndPointerRecipe(getOperand(0), getVFValue(), IndexedTy,
-                                        getGEPNoWrapFlags(), getDebugLoc());
+                                        Stride, getGEPNoWrapFlags(),
+                                        getDebugLoc());
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2186,8 +2191,8 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
 /// operand.
 class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
                              public VPUnrollPartAccessor<2> {
-  /// Descriptor for the reduction.
-  const RecurrenceDescriptor &RdxDesc;
+  /// The recurrence kind of the reduction.
+  const RecurKind Kind;
 
   /// The phi is part of an in-loop reduction.
   bool IsInLoop;
@@ -2200,23 +2205,21 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   unsigned VFScaleFactor = 1;
 
 public:
-  /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
-  /// RdxDesc.
-  VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
-                       VPValue &Start, bool IsInLoop = false,
-                       bool IsOrdered = false, unsigned VFScaleFactor = 1)
-      : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
-        RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
-        VFScaleFactor(VFScaleFactor) {
+  /// Create a new VPReductionPHIRecipe for the reduction \p Phi.
+  VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
+                       bool IsInLoop = false, bool IsOrdered = false,
+                       unsigned VFScaleFactor = 1)
+      : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
+        IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
     assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
   }
 
   ~VPReductionPHIRecipe() override = default;
 
   VPReductionPHIRecipe *clone() override {
-    auto *R = new VPReductionPHIRecipe(cast<PHINode>(getUnderlyingInstr()),
-                                       RdxDesc, *getOperand(0), IsInLoop,
-                                       IsOrdered, VFScaleFactor);
+    auto *R = new VPReductionPHIRecipe(
+        dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
+        *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
     R->addOperand(getBackedgeValue());
     return R;
   }
@@ -2235,9 +2238,8 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
              VPSlotTracker &SlotTracker) const override;
 #endif
 
-  const RecurrenceDescriptor &getRecurrenceDescriptor() const {
-    return RdxDesc;
-  }
+  /// Returns the recurrence kind of the reduction.
+  RecurKind getRecurrenceKind() const { return Kind; }
 
   /// Returns true, if the phi is part of an ordered reduction.
   bool isOrdered() const { return IsOrdered; }
@@ -2341,6 +2343,9 @@ class VPInterleaveRecipe : public VPRecipeBase {
                      DL),
 
         IG(IG), NeedsMaskForGaps(NeedsMaskForGaps) {
+    // TODO: extend the masked interleaved-group support to reversed access.
+    assert((!Mask || !IG->isReverse()) &&
+           "Reversed masked interleave-group not supported.");
     for (unsigned i = 0; i < IG->getFactor(); ++i)
       if (Instruction *I = IG->getMember(i)) {
         if (I->getType()->isVoidTy())
@@ -2440,28 +2445,6 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
     setUnderlyingValue(I);
   }
 
-  /// For VPExtendedReductionRecipe.
-  /// Note that the debug location is from the extend.
-  VPReductionRecipe(const unsigned char SC, const RecurKind RdxKind,
-                    ArrayRef<VPValue *> Operands, VPValue *CondOp,
-                    bool IsOrdered, DebugLoc DL)
-      : VPRecipeWithIRFlags(SC, Operands, DL), RdxKind(RdxKind),
-        IsOrdered(IsOrdered), IsConditional(CondOp) {
-    if (CondOp)
-      addOperand(CondOp);
-  }
-
-  /// For VPMulAccumulateReductionRecipe.
-  /// Note that the NUW/NSW flags and the debug location are from the Mul.
-  VPReductionRecipe(const unsigned char SC, const RecurKind RdxKind,
-                    ArrayRef<VPValue *> Operands, VPValue *CondOp,
-                    bool IsOrdered, WrapFlagsTy WrapFlags, DebugLoc DL)
-      : VPRecipeWithIRFlags(SC, Operands, WrapFlags, DL), RdxKind(RdxKind),
-        IsOrdered(IsOrdered), IsConditional(CondOp) {
-    if (CondOp)
-      addOperand(CondOp);
-  }
-
 public:
   VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I,
                     VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
@@ -2487,9 +2470,7 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
 
   static inline bool classof(const VPRecipeBase *R) {
     return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
-           R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
-           R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC ||
-           R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC;
+           R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
   }
 
   static inline bool classof(const VPUser *U) {
@@ -2628,190 +2609,6 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
   }
 };
 
-/// A recipe to represent inloop extended reduction operations, performing a
-/// reduction on a extended vector operand into a scalar value, and adding the
-/// result to a chain. This recipe is abstract and needs to be lowered to
-/// concrete recipes before codegen. The operands are {ChainOp, VecOp,
-/// [Condition]}.
-class VPExtendedReductionRecipe : public VPReductionRecipe {
-  /// Opcode of the extend for VecOp.
-  Instruction::CastOps ExtOp;
-
-  /// The scalar type after extending.
-  Type *ResultTy;
-
-  /// For cloning VPExtendedReductionRecipe.
-  VPExtendedReductionRecipe(VPExtendedReductionRecipe *ExtRed)
-      : VPReductionRecipe(
-            VPDef::VPExtendedReductionSC, ExtRed->getRecurrenceKind(),
-            {ExtRed->getChainOp(), ExtRed->getVecOp()}, ExtRed->getCondOp(),
-            ExtRed->isOrdered(), ExtRed->getDebugLoc()),
-        ExtOp(ExtRed->getExtOpcode()), ResultTy(ExtRed->getResultType()) {
-    transferFlags(*ExtRed);
-    setUnderlyingValue(ExtRed->getUnderlyingValue());
-  }
-
-public:
-  VPExtendedReductionRecipe(VPReductionRecipe *R, VPWidenCastRecipe *Ext)
-      : VPReductionRecipe(VPDef::VPExtendedReductionSC, R->getRecurrenceKind(),
-                          {R->getChainOp(), Ext->getOperand(0)}, R->getCondOp(),
-                          R->isOrdered(), Ext->getDebugLoc()),
-        ExtOp(Ext->getOpcode()), ResultTy(Ext->getResultType()) {
-    assert((ExtOp == Instruction::CastOps::ZExt ||
-            ExtOp == Instruction::CastOps::SExt) &&
-           "VPExtendedReductionRecipe only supports zext and sext.");
-
-    transferFlags(*Ext);
-    setUnderlyingValue(R->getUnderlyingValue());
-  }
-
-  ~VPExtendedReductionRecipe() override = default;
-
-  VPExtendedReductionRecipe *clone() override {
-    return new VPExtendedReductionRecipe(this);
-  }
-
-  VP_CLASSOF_IMPL(VPDef::VPExtendedReductionSC);
-
-  void execute(VPTransformState &State) override {
-    llvm_unreachable("VPExtendedReductionRecipe should be transform to "
-                     "VPExtendedRecipe + VPReductionRecipe before execution.");
-  };
-
-  /// Return the cost of VPExtendedReductionRecipe.
-  InstructionCost computeCost(ElementCount VF,
-                              VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  /// Print the recipe.
-  void print(raw_ostream &O, const Twine &Indent,
-             VPSlotTracker &SlotTracker) const override;
-#endif
-
-  /// The scalar type after extending.
-  Type *getResultType() const { return ResultTy; }
-
-  /// Is the extend ZExt?
-  bool isZExt() const { return getExtOpcode() == Instruction::ZExt; }
-
-  /// Get the opcode of the extend for VecOp.
-  Instruction::CastOps getExtOpcode() const { return ExtOp; }
-};
-
-/// A recipe to represent inloop MulAccumulateReduction operations,  multiplying
-/// the vector operands (which may be extended), performing a reduction.add on
-/// the result, and adding the scalar result to a chain. This recipe is abstract
-/// and needs to be lowered to concrete recipes before codegen. The operands are
-/// {ChainOp, VecOp1, VecOp2, [Condition]}.
-class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
-  /// Opcode of the extend for VecOp1 and VecOp2.
-  Instruction::CastOps ExtOp;
-
-  /// Non-neg flag of the extend recipe.
-  bool IsNonNeg = false;
-
-  /// The scalar type after extending.
-  Type *ResultTy = nullptr;
-
-  /// For cloning VPMulAccumulateReductionRecipe.
-  VPMulAccumulateReductionRecipe(VPMulAccumulateReductionRecipe *MulAcc)
-      : VPReductionRecipe(
-            VPDef::VPMulAccumulateReductionSC, MulAcc->getRecurrenceKind(),
-            {MulAcc->getChainOp(), MulAcc->getVecOp0(), MulAcc->getVecOp1()},
-            MulAcc->getCondOp(), MulAcc->isOrdered(),
-            WrapFlagsTy(MulAcc->hasNoUnsignedWrap(), MulAcc->hasNoSignedWrap()),
-            MulAcc->getDebugLoc()),
-        ExtOp(MulAcc->getExtOpcode()), IsNonNeg(MulAcc->isNonNeg()),
-        ResultTy(MulAcc->getResultType()) {
-    transferFlags(*MulAcc);
-    setUnderlyingValue(MulAcc->getUnderlyingValue());
-  }
-
-public:
-  VPMulAccumulateReductionRecipe(VPReductionRecipe *R, VPWidenRecipe *Mul,
-                                 VPWidenCastRecipe *Ext0,
-                                 VPWidenCastRecipe *Ext1, Type *ResultTy)
-      : VPReductionRecipe(
-            VPDef::VPMulAccumulateReductionSC, R->getRecurrenceKind(),
-            {R->getChainOp(), Ext0->getOperand(0), Ext1->getOperand(0)},
-            R->getCondOp(), R->isOrdered(),
-            WrapFlagsTy(Mul->hasNoUnsignedWrap(), Mul->hasNoSignedWrap()),
-            R->getDebugLoc()),
-        ExtOp(Ext0->getOpcode()), ResultTy(ResultTy) {
-    assert(RecurrenceDescriptor::getOpcode(getRecurrenceKind()) ==
-               Instruction::Add &&
-           "The reduction instruction in MulAccumulateteReductionRecipe must "
-           "be Add");
-    assert((ExtOp == Instruction::CastOps::ZExt ||
-            ExtOp == Instruction::CastOps::SExt) &&
-           "VPMulAccumulateReductionRecipe only supports zext and sext.");
-    setUnderlyingValue(R->getUnderlyingValue());
-    // Only set the non-negative flag if the original recipe contains.
-    if (Ext0->hasNonNegFlag())
-      IsNonNeg = Ext0->isNonNeg();
-  }
-
-  VPMulAccumulateReductionRecipe(VPReductionRecipe *R, VPWidenRecipe *Mul,
-                                 Type *ResultTy)
-      : VPReductionRecipe(
-            VPDef::VPMulAccumulateReductionSC, R->getRecurrenceKind(),
-            {R->getChainOp(), Mul->getOperand(0), Mul->getOperand(1)},
-            R->getCondOp(), R->isOrdered(),
-            WrapFlagsTy(Mul->hasNoUnsignedWrap(), Mul->hasNoSignedWrap()),
-            R->getDebugLoc()),
-        ExtOp(Instruction::CastOps::CastOpsEnd), ResultTy(ResultTy) {
-    assert(RecurrenceDescriptor::getOpcode(getRecurrenceKind()) ==
-               Instruction::Add &&
-           "The reduction instruction in MulAccumulateReductionRecipe must be "
-           "Add");
-    setUnderlyingValue(R->getUnderlyingValue());
-  }
-
-  ~VPMulAccumulateReductionRecipe() override = default;
-
-  VPMulAccumulateReductionRecipe *clone() override {
-    return new VPMulAccumulateReductionRecipe(this);
-  }
-
-  VP_CLASSOF_IMPL(VPDef::VPMulAccumulateReductionSC);
-
-  void execute(VPTransformState &State) override {
-    llvm_unreachable("VPMulAccumulateReductionRecipe should transform to "
-                     "VPWidenCastRecipe + "
-                     "VPWidenRecipe + VPReductionRecipe before execution");
-  }
-
-  /// Return the cost of VPMulAccumulateReductionRecipe.
-  InstructionCost computeCost(ElementCount VF,
-                              VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  /// Print the recipe.
-  void print(raw_ostream &O, const Twine &Indent,
-             VPSlotTracker &SlotTracker) const override;
-#endif
-
-  Type *getResultType() const { return ResultTy; }
-
-  /// The first vector value to be extended and reduced.
-  VPValue *getVecOp0() const { return getOperand(1); }
-
-  /// The second vector value to be extended and reduced.
-  VPValue *getVecOp1() const { return getOperand(2); }
-
-  /// Return true if this recipe contains extended operands.
-  bool isExtended() const { return ExtOp != Instruction::CastOps::CastOpsEnd; }
-
-  /// Return the opcode of the extends for the operands.
-  Instruction::CastOps getExtOpcode() const { return ExtOp; }
-
-  /// Return if the operands are zero-extended.
-  bool isZExt() const { return ExtOp == Instruction::CastOps::ZExt; }
-
-  /// Return true if the operand extends have the non-negative flag.
-  bool isNonNeg() const { return IsNonNeg; }
-};
-
 /// VPReplicateRecipe replicates a given instruction producing multiple scalar
 /// copies of the original scalar type, one per lane, instead of producing a
 /// single copy of widened type for all lanes. If the instruction is known to be
@@ -2930,6 +2727,122 @@ class VPBranchOnMaskRecipe : public VPRecipeBase {
   }
 };
 
+/// A recipe to combine multiple recipes into a single 'expression' recipe,
+/// which should be considered a single entity for cost-modeling and transforms.
+/// The recipe needs to be 'decomposed', i.e. replaced by its individual
+/// expression recipes, before execute. The individual expression recipes are
+/// completely disconnected from the def-use graph of other recipes not part of
+/// the expression. Def-use edges between pairs of expression recipes remain
+/// intact, whereas every edge between an expression recipe and a recipe outside
+/// the expression is elevated to connect the non-expression recipe with the
+/// VPExpressionRecipe itself.
+class VPExpressionRecipe : public VPSingleDefRecipe {
+  /// Recipes included in this VPExpressionRecipe.
+  SmallVector<VPSingleDefRecipe *> ExpressionRecipes;
+
+  /// Temporary VPValues used for external operands of the expression, i.e.
+  /// operands not defined by recipes in the expression.
+  SmallVector<VPValue *> LiveInPlaceholders;
+
+  enum class ExpressionTypes {
+    /// Represents an inloop extended reduction operation, performing a
+    /// reduction on an extended vector operand into a scalar value, and adding
+    /// the result to a chain.
+    ExtendedReduction,
+    /// Represent an inloop multiply-accumulate reduction, multiplying the
+    /// extended vector operands, performing a reduction.add on the result, and
+    /// adding the scalar result to a chain.
+    ExtMulAccReduction,
+    /// Represent an inloop multiply-accumulate reduction, multiplying the
+    /// vector operands, performing a reduction.add on the result, and adding
+    /// the scalar result to a chain.
+    MulAccReduction,
+  };
+
+  /// Type of the expression.
+  ExpressionTypes ExpressionType;
+
+  /// Construct a new VPExpressionRecipe by internalizing recipes in \p
+  /// ExpressionRecipes. External operands (i.e. not defined by another recipe
+  /// in the expression) are replaced by temporary VPValues and the original
+  /// operands are transferred to the VPExpressionRecipe itself. Clone recipes
+  /// as needed (excluding last) to ensure they are only used by other recipes
+  /// in the expression.
+  VPExpressionRecipe(ExpressionTypes ExpressionType,
+                     ArrayRef<VPSingleDefRecipe *> ExpressionRecipes);
+
+public:
+  VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
+      : VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
+  VPExpressionRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
+      : VPExpressionRecipe(ExpressionTypes::MulAccReduction, {Mul, Red}) {}
+  VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
+                     VPWidenRecipe *Mul, VPReductionRecipe *Red)
+      : VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
+                           {Ext0, Ext1, Mul, Red}) {}
+
+  ~VPExpressionRecipe() override {
+    for (auto *R : reverse(ExpressionRecipes))
+      delete R;
+    for (VPValue *T : LiveInPlaceholders)
+      delete T;
+  }
+
+  VP_CLASSOF_IMPL(VPDef::VPExpressionSC)
+
+  VPExpressionRecipe *clone() override {
+    assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
+    SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
+    for (auto *R : ExpressionRecipes)
+      NewExpressiondRecipes.push_back(R->clone());
+    for (auto *New : NewExpressiondRecipes) {
+      for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
+        New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
+      // Update placeholder operands in the cloned recipe to use the external
+      // operands, to be internalized when the cloned expression is constructed.
+      for (const auto &[Placeholder, OutsideOp] :
+           zip(LiveInPlaceholders, operands()))
+        New->replaceUsesOfWith(Placeholder, OutsideOp);
+    }
+    return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
+  }
+
+  /// Return the VPValue to use to infer the result type of the recipe.
+  VPValue *getOperandOfResultType() const {
+    unsigned OpIdx =
+        cast<VPReductionRecipe>(ExpressionRecipes.back())->isConditional() ? 2
+                                                                           : 1;
+    return getOperand(getNumOperands() - OpIdx);
+  }
+
+  /// Insert the recipes of the expression back into the VPlan, directly before
+  /// the current recipe. Leaves the expression recipe empty, which must be
+  /// removed before codegen.
+  void decompose();
+
+  /// Method for generating code, must not be called as this recipe is abstract.
+  void execute(VPTransformState &State) override {
+    llvm_unreachable("recipe must be removed before execute");
+  }
+
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print the recipe.
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+#endif
+
+  /// Returns true if this expression contains recipes that may read from or
+  /// write to memory.
+  bool mayReadOrWriteMemory() const;
+
+  /// Returns true if this expression contains recipes that may have side
+  /// effects.
+  bool mayHaveSideEffects() const;
+};
+
 /// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
 /// control converges back from a Branch-on-Mask. The phi nodes are needed in
 /// order to merge values that are set under such a branch and feed their uses.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index da4a52203db3f..92db9674ef42b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -91,11 +91,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
            "different types inferred for different operands");
     return IntegerType::get(Ctx, 1);
   case VPInstruction::ComputeAnyOfResult:
-  case VPInstruction::ComputeFindLastIVResult:
+    return inferScalarType(R->getOperand(1));
+  case VPInstruction::ComputeFindIVResult:
   case VPInstruction::ComputeReductionResult: {
-    auto *PhiR = cast<VPReductionPHIRecipe>(R->getOperand(0));
-    auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
-    return OrigPhi->getType();
+    return inferScalarType(R->getOperand(0));
   }
   case VPInstruction::ExplicitVectorLength:
     return Type::getIntNTy(Ctx, 32);
@@ -298,13 +297,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
             // TODO: Use info from interleave group.
             return V->getUnderlyingValue()->getType();
           })
-          .Case<VPExtendedReductionRecipe, VPMulAccumulateReductionRecipe>(
-              [](const auto *R) { return R->getResultType(); })
           .Case<VPExpandSCEVRecipe>([](const VPExpandSCEVRecipe *R) {
             return R->getSCEV()->getType();
           })
           .Case<VPReductionRecipe>([this](const auto *R) {
             return inferScalarType(R->getChainOp());
+          })
+          .Case<VPExpressionRecipe>([this](const auto *R) {
+            return inferScalarType(R->getOperandOfResultType());
           });
 
   assert(ResultTy && "could not infer type for the given VPValue");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index efea99f22d086..d133610ef4f75 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -366,12 +366,6 @@ m_Freeze(const Op0_t &Op0) {
   return m_VPInstruction<Instruction::Freeze>(Op0);
 }
 
-template <typename Op0_t>
-inline UnaryVPInstruction_match<Op0_t, VPInstruction::Not>
-m_Not(const Op0_t &Op0) {
-  return m_VPInstruction<VPInstruction::Not>(Op0);
-}
-
 template <typename Op0_t>
 inline UnaryVPInstruction_match<Op0_t, VPInstruction::BranchOnCond>
 m_BranchOnCond(const Op0_t &Op0) {
@@ -491,6 +485,15 @@ m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
       {Op0, Op1, Op2});
 }
 
+template <typename Op0_t>
+inline match_combine_or<UnaryVPInstruction_match<Op0_t, VPInstruction::Not>,
+                        AllBinaryRecipe_match<int_pred_ty<is_all_ones>, Op0_t,
+                                              Instruction::Xor, true>>
+m_Not(const Op0_t &Op0) {
+  return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
+                     m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
+}
+
 template <typename Op0_t, typename Op1_t>
 inline match_combine_or<
     BinaryVPInstruction_match<Op0_t, Op1_t, VPInstruction::LogicalAnd>,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7b302407519e7..ccb7512051d77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -49,6 +49,8 @@ using VectorParts = SmallVector<Value *, 2>;
 
 bool VPRecipeBase::mayWriteToMemory() const {
   switch (getVPDefID()) {
+  case VPExpressionSC:
+    return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
   case VPInstructionSC:
     return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
   case VPInterleaveSC:
@@ -73,8 +75,6 @@ bool VPRecipeBase::mayWriteToMemory() const {
   case VPBlendSC:
   case VPReductionEVLSC:
   case VPReductionSC:
-  case VPExtendedReductionSC:
-  case VPMulAccumulateReductionSC:
   case VPVectorPointerSC:
   case VPWidenCanonicalIVSC:
   case VPWidenCastSC:
@@ -99,6 +99,8 @@ bool VPRecipeBase::mayWriteToMemory() const {
 
 bool VPRecipeBase::mayReadFromMemory() const {
   switch (getVPDefID()) {
+  case VPExpressionSC:
+    return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
   case VPInstructionSC:
     return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
   case VPWidenLoadEVLSC:
@@ -123,8 +125,6 @@ bool VPRecipeBase::mayReadFromMemory() const {
   case VPBlendSC:
   case VPReductionEVLSC:
   case VPReductionSC:
-  case VPExtendedReductionSC:
-  case VPMulAccumulateReductionSC:
   case VPVectorPointerSC:
   case VPWidenCanonicalIVSC:
   case VPWidenCastSC:
@@ -147,6 +147,8 @@ bool VPRecipeBase::mayReadFromMemory() const {
 
 bool VPRecipeBase::mayHaveSideEffects() const {
   switch (getVPDefID()) {
+  case VPExpressionSC:
+    return cast<VPExpressionRecipe>(this)->mayHaveSideEffects();
   case VPDerivedIVSC:
   case VPFirstOrderRecurrencePHISC:
   case VPPredInstPHISC:
@@ -163,8 +165,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
   case VPBlendSC:
   case VPReductionEVLSC:
   case VPReductionSC:
-  case VPExtendedReductionSC:
-  case VPMulAccumulateReductionSC:
   case VPScalarIVStepsSC:
   case VPVectorPointerSC:
   case VPWidenCanonicalIVSC:
@@ -296,49 +296,70 @@ bool VPRecipeBase::isScalarCast() const {
 InstructionCost
 VPPartialReductionRecipe::computeCost(ElementCount VF,
                                       VPCostContext &Ctx) const {
-  std::optional<unsigned> Opcode = std::nullopt;
-  VPValue *BinOp = getOperand(1);
+  std::optional<unsigned> Opcode;
+  VPValue *Op = getOperand(0);
+  VPRecipeBase *OpR = Op->getDefiningRecipe();
 
-  // If the partial reduction is predicated, a select will be operand 0 rather
-  // than the binary op
+  // If the partial reduction is predicated, a select will be operand 0
   using namespace llvm::VPlanPatternMatch;
-  if (match(getOperand(1), m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
-    BinOp = BinOp->getDefiningRecipe()->getOperand(1);
-
-  // If BinOp is a negation, use the side effect of match to assign the actual
-  // binary operation to BinOp
-  match(BinOp, m_Binary<Instruction::Sub>(m_SpecificInt(0), m_VPValue(BinOp)));
-  VPRecipeBase *BinOpR = BinOp->getDefiningRecipe();
-
-  if (auto *WidenR = dyn_cast<VPWidenRecipe>(BinOpR))
-    Opcode = std::make_optional(WidenR->getOpcode());
-
-  VPRecipeBase *ExtAR = BinOpR->getOperand(0)->getDefiningRecipe();
-  VPRecipeBase *ExtBR = BinOpR->getOperand(1)->getDefiningRecipe();
+  if (match(getOperand(1), m_Select(m_VPValue(), m_VPValue(Op), m_VPValue()))) {
+    OpR = Op->getDefiningRecipe();
+  }
 
-  auto *PhiType = Ctx.Types.inferScalarType(getOperand(1));
-  auto *InputTypeA = Ctx.Types.inferScalarType(ExtAR ? ExtAR->getOperand(0)
-                                                     : BinOpR->getOperand(0));
-  auto *InputTypeB = Ctx.Types.inferScalarType(ExtBR ? ExtBR->getOperand(0)
-                                                     : BinOpR->getOperand(1));
+  Type *InputTypeA = nullptr, *InputTypeB = nullptr;
+  TTI::PartialReductionExtendKind ExtAType = TTI::PR_None,
+                                  ExtBType = TTI::PR_None;
 
   auto GetExtendKind = [](VPRecipeBase *R) {
-    // The extend could come from outside the plan.
     if (!R)
-      return TargetTransformInfo::PR_None;
+      return TTI::PR_None;
     auto *WidenCastR = dyn_cast<VPWidenCastRecipe>(R);
     if (!WidenCastR)
-      return TargetTransformInfo::PR_None;
+      return TTI::PR_None;
     if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)
-      return TargetTransformInfo::PR_ZeroExtend;
+      return TTI::PR_ZeroExtend;
     if (WidenCastR->getOpcode() == Instruction::CastOps::SExt)
-      return TargetTransformInfo::PR_SignExtend;
-    return TargetTransformInfo::PR_None;
+      return TTI::PR_SignExtend;
+    return TTI::PR_None;
   };
 
-  return Ctx.TTI.getPartialReductionCost(
-      getOpcode(), InputTypeA, InputTypeB, PhiType, VF, GetExtendKind(ExtAR),
-      GetExtendKind(ExtBR), Opcode, Ctx.CostKind);
+  // Pick out opcode, type/ext information and use sub side effects from a widen
+  // recipe.
+  auto HandleWiden = [&](VPWidenRecipe *Widen) {
+    if (match(Widen,
+              m_Binary<Instruction::Sub>(m_SpecificInt(0), m_VPValue(Op)))) {
+      Widen = dyn_cast<VPWidenRecipe>(Op->getDefiningRecipe());
+    }
+    Opcode = Widen->getOpcode();
+    VPRecipeBase *ExtAR = Widen->getOperand(0)->getDefiningRecipe();
+    VPRecipeBase *ExtBR = Widen->getOperand(1)->getDefiningRecipe();
+    InputTypeA = Ctx.Types.inferScalarType(ExtAR ? ExtAR->getOperand(0)
+                                                 : Widen->getOperand(0));
+    InputTypeB = Ctx.Types.inferScalarType(ExtBR ? ExtBR->getOperand(0)
+                                                 : Widen->getOperand(1));
+    ExtAType = GetExtendKind(ExtAR);
+    ExtBType = GetExtendKind(ExtBR);
+  };
+
+  if (isa<VPWidenCastRecipe>(OpR)) {
+    InputTypeA = Ctx.Types.inferScalarType(OpR->getOperand(0));
+    ExtAType = GetExtendKind(OpR);
+  } else if (isa<VPReductionPHIRecipe>(OpR)) {
+    auto RedPhiOp1R = getOperand(1)->getDefiningRecipe();
+    if (isa<VPWidenCastRecipe>(RedPhiOp1R)) {
+      InputTypeA = Ctx.Types.inferScalarType(RedPhiOp1R->getOperand(0));
+      ExtAType = GetExtendKind(RedPhiOp1R);
+    } else if (auto Widen = dyn_cast<VPWidenRecipe>(RedPhiOp1R))
+      HandleWiden(Widen);
+  } else if (auto Widen = dyn_cast<VPWidenRecipe>(OpR)) {
+    HandleWiden(Widen);
+  } else if (auto Reduction = dyn_cast<VPPartialReductionRecipe>(OpR)) {
+    return Reduction->computeCost(VF, Ctx);
+  }
+  auto *PhiType = Ctx.Types.inferScalarType(getOperand(1));
+  return Ctx.TTI.getPartialReductionCost(getOpcode(), InputTypeA, InputTypeB,
+                                         PhiType, VF, ExtAType, ExtBType,
+                                         Opcode, Ctx.CostKind);
 }
 
 void VPPartialReductionRecipe::execute(VPTransformState &State) {
@@ -460,7 +481,7 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
   case VPInstruction::ComputeAnyOfResult:
   case VPInstruction::ReductionStartVector:
     return 3;
-  case VPInstruction::ComputeFindLastIVResult:
+  case VPInstruction::ComputeFindIVResult:
     return 4;
   case Instruction::Call:
   case Instruction::GetElementPtr:
@@ -725,14 +746,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return createAnyOfReduction(Builder, ReducedPartRdx,
                                 State.get(getOperand(1), VPLane(0)), OrigPhi);
   }
-  case VPInstruction::ComputeFindLastIVResult: {
+  case VPInstruction::ComputeFindIVResult: {
     // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
     // and will be removed by breaking up the recipe further.
     auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
     // Get its reduction variable descriptor.
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind RK = RdxDesc.getRecurrenceKind();
-    assert(RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
+    RecurKind RK = PhiR->getRecurrenceKind();
+    assert(RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
            "Unexpected reduction kind");
     assert(!PhiR->isInLoop() &&
            "In-loop FindLastIV reduction is not supported yet");
@@ -741,9 +761,16 @@ Value *VPInstruction::generate(VPTransformState &State) {
     // sentinel value, followed by one operand for each part of the reduction.
     unsigned UF = getNumOperands() - 3;
     Value *ReducedPartRdx = State.get(getOperand(3));
-    RecurKind MinMaxKind = RecurrenceDescriptor::isSignedRecurrenceKind(RK)
-                               ? RecurKind::SMax
-                               : RecurKind::UMax;
+    RecurKind MinMaxKind;
+    bool IsSigned = RecurrenceDescriptor::isSignedRecurrenceKind(RK);
+    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
+      MinMaxKind = IsSigned ? RecurKind::SMax : RecurKind::UMax;
+    } else {
+      assert(RecurrenceDescriptor::isFindFirstIVRecurrenceKind(RK) &&
+             "Kind must either be FindLastIV or FindFirstIV");
+      assert(IsSigned && "Only FindFirstIV with SMax is currently supported");
+      MinMaxKind = RecurKind::SMin;
+    }
     for (unsigned Part = 1; Part < UF; ++Part)
       ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
                                       State.get(getOperand(3 + Part)));
@@ -758,13 +785,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
     // and will be removed by breaking up the recipe further.
     auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
     // Get its reduction variable descriptor.
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
 
-    RecurKind RK = RdxDesc.getRecurrenceKind();
-    assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
-           "should be handled by ComputeFindLastIVResult");
+    RecurKind RK = PhiR->getRecurrenceKind();
+    assert(!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
+           "should be handled by ComputeFindIVResult");
 
-    Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
     // The recipe's operands are the reduction phi, followed by one operand for
     // each part of the reduction.
     unsigned UF = getNumOperands() - 1;
@@ -776,15 +801,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
     if (hasFastMathFlags())
       Builder.setFastMathFlags(getFastMathFlags());
 
-    // If the vector reduction can be performed in a smaller type, we truncate
-    // then extend the loop exit value to enable InstCombine to evaluate the
-    // entire expression in the smaller type.
-    // TODO: Handle this in truncateToMinBW.
-    if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
-      Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
-      for (unsigned Part = 0; Part < UF; ++Part)
-        RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
-    }
     // Reduce all of the unrolled parts into a single vector.
     Value *ReducedPartRdx = RdxParts[0];
     if (PhiR->isOrdered()) {
@@ -796,9 +812,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
         if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))
           ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
         else
-          ReducedPartRdx =
-              Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(),
-                                  RdxPart, ReducedPartRdx, "bin.rdx");
+          ReducedPartRdx = Builder.CreateBinOp(
+              (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),
+              RdxPart, ReducedPartRdx, "bin.rdx");
       }
     }
 
@@ -809,13 +825,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
       // All ops in the reduction inherit fast-math-flags from the recurrence
       // descriptor.
       ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
-
-      // If the reduction can be performed in a smaller type, we need to extend
-      // the reduction to the wider type before we branch to the original loop.
-      if (ResultTy != RdxDesc.getRecurrenceType())
-        ReducedPartRdx = RdxDesc.isSigned()
-                             ? Builder.CreateSExt(ReducedPartRdx, ResultTy)
-                             : Builder.CreateZExt(ReducedPartRdx, ResultTy);
     }
 
     return ReducedPartRdx;
@@ -977,7 +986,7 @@ bool VPInstruction::isVectorToScalar() const {
          getOpcode() == Instruction::ExtractElement ||
          getOpcode() == VPInstruction::FirstActiveLane ||
          getOpcode() == VPInstruction::ComputeAnyOfResult ||
-         getOpcode() == VPInstruction::ComputeFindLastIVResult ||
+         getOpcode() == VPInstruction::ComputeFindIVResult ||
          getOpcode() == VPInstruction::ComputeReductionResult ||
          getOpcode() == VPInstruction::AnyOf;
 }
@@ -1079,7 +1088,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
   case VPInstruction::PtrAdd:
     return Op == getOperand(0) || vputils::onlyFirstLaneUsed(this);
   case VPInstruction::ComputeAnyOfResult:
-  case VPInstruction::ComputeFindLastIVResult:
+  case VPInstruction::ComputeFindIVResult:
     return Op == getOperand(1);
   };
   llvm_unreachable("switch should return");
@@ -1168,8 +1177,8 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ComputeAnyOfResult:
     O << "compute-anyof-result";
     break;
-  case VPInstruction::ComputeFindLastIVResult:
-    O << "compute-find-last-iv-result";
+  case VPInstruction::ComputeFindIVResult:
+    O << "compute-find-iv-result";
     break;
   case VPInstruction::ComputeReductionResult:
     O << "compute-reduction-result";
@@ -2319,12 +2328,12 @@ void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-static Type *getGEPIndexTy(bool IsScalable, bool IsReverse,
+static Type *getGEPIndexTy(bool IsScalable, bool IsReverse, bool IsUnitStride,
                            unsigned CurrentPart, IRBuilderBase &Builder) {
   // Use i32 for the gep index type when the value is constant,
   // or query DataLayout for a more suitable index type otherwise.
   const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();
-  return IsScalable && (IsReverse || CurrentPart > 0)
+  return !IsUnitStride || (IsScalable && (IsReverse || CurrentPart > 0))
              ? DL.getIndexType(Builder.getPtrTy(0))
              : Builder.getInt32Ty();
 }
@@ -2332,18 +2341,21 @@ static Type *getGEPIndexTy(bool IsScalable, bool IsReverse,
 void VPVectorEndPointerRecipe::execute(VPTransformState &State) {
   auto &Builder = State.Builder;
   unsigned CurrentPart = getUnrollPart(*this);
+  bool IsUnitStride = Stride == 1 || Stride == -1;
   Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ true,
-                                CurrentPart, Builder);
+                                IsUnitStride, CurrentPart, Builder);
 
   // The wide store needs to start at the last vector element.
   Value *RunTimeVF = State.get(getVFValue(), VPLane(0));
   if (IndexTy != RunTimeVF->getType())
     RunTimeVF = Builder.CreateZExtOrTrunc(RunTimeVF, IndexTy);
-  // NumElt = -CurrentPart * RunTimeVF
+  // NumElt = Stride * CurrentPart * RunTimeVF
   Value *NumElt = Builder.CreateMul(
-      ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);
-  // LastLane = 1 - RunTimeVF
-  Value *LastLane = Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
+      ConstantInt::get(IndexTy, Stride * (int64_t)CurrentPart), RunTimeVF);
+  // LastLane = Stride * (RunTimeVF - 1)
+  Value *LastLane = Builder.CreateSub(RunTimeVF, ConstantInt::get(IndexTy, 1));
+  if (Stride != 1)
+    LastLane = Builder.CreateMul(ConstantInt::get(IndexTy, Stride), LastLane);
   Value *Ptr = State.get(getOperand(0), VPLane(0));
   Value *ResultPtr =
       Builder.CreateGEP(IndexedTy, Ptr, NumElt, "", getGEPNoWrapFlags());
@@ -2368,7 +2380,7 @@ void VPVectorPointerRecipe::execute(VPTransformState &State) {
   auto &Builder = State.Builder;
   unsigned CurrentPart = getUnrollPart(*this);
   Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ false,
-                                CurrentPart, Builder);
+                                /*IsUnitStride*/ true, CurrentPart, Builder);
   Value *Ptr = State.get(getOperand(0), VPLane(0));
 
   Value *Increment = createStepForVF(Builder, IndexTy, State.VF, CurrentPart);
@@ -2573,30 +2585,183 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
                                             Ctx.CostKind);
 }
 
-InstructionCost
-VPExtendedReductionRecipe::computeCost(ElementCount VF,
-                                       VPCostContext &Ctx) const {
-  unsigned Opcode = RecurrenceDescriptor::getOpcode(getRecurrenceKind());
+VPExpressionRecipe::VPExpressionRecipe(
+    ExpressionTypes ExpressionType,
+    ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
+    : VPSingleDefRecipe(VPDef::VPExpressionSC, {}, {}),
+      ExpressionRecipes(SetVector<VPSingleDefRecipe *>(
+                            ExpressionRecipes.begin(), ExpressionRecipes.end())
+                            .takeVector()),
+      ExpressionType(ExpressionType) {
+  assert(!ExpressionRecipes.empty() && "Nothing to combine?");
+  assert(
+      none_of(ExpressionRecipes,
+              [](VPSingleDefRecipe *R) { return R->mayHaveSideEffects(); }) &&
+      "expression cannot contain recipes with side-effects");
+
+  // Maintain a copy of the expression recipes as a set of users.
+  SmallPtrSet<VPUser *, 4> ExpressionRecipesAsSetOfUsers;
+  for (auto *R : ExpressionRecipes)
+    ExpressionRecipesAsSetOfUsers.insert(R);
+
+  // Recipes in the expression, except the last one, must only be used by
+  // (other) recipes inside the expression. If there are other users, external
+  // to the expression, use a clone of the recipe for external users.
+  for (VPSingleDefRecipe *R : ExpressionRecipes) {
+    if (R != ExpressionRecipes.back() &&
+        any_of(R->users(), [&ExpressionRecipesAsSetOfUsers](VPUser *U) {
+          return !ExpressionRecipesAsSetOfUsers.contains(U);
+        })) {
+      // There are users outside of the expression. Clone the recipe and use the
+      // clone those external users.
+      VPSingleDefRecipe *CopyForExtUsers = R->clone();
+      R->replaceUsesWithIf(CopyForExtUsers, [&ExpressionRecipesAsSetOfUsers](
+                                                VPUser &U, unsigned) {
+        return !ExpressionRecipesAsSetOfUsers.contains(&U);
+      });
+      CopyForExtUsers->insertBefore(R);
+    }
+    if (R->getParent())
+      R->removeFromParent();
+  }
+
+  // Internalize all external operands to the expression recipes. To do so,
+  // create new temporary VPValues for all operands defined by a recipe outside
+  // the expression. The original operands are added as operands of the
+  // VPExpressionRecipe itself.
+  for (auto *R : ExpressionRecipes) {
+    for (const auto &[Idx, Op] : enumerate(R->operands())) {
+      auto *Def = Op->getDefiningRecipe();
+      if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))
+        continue;
+      addOperand(Op);
+      LiveInPlaceholders.push_back(new VPValue());
+      R->setOperand(Idx, LiveInPlaceholders.back());
+    }
+  }
+}
+
+void VPExpressionRecipe::decompose() {
+  for (auto *R : ExpressionRecipes)
+    R->insertBefore(this);
+
+  for (const auto &[Idx, Op] : enumerate(operands()))
+    LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
+
+  replaceAllUsesWith(ExpressionRecipes.back());
+  ExpressionRecipes.clear();
+}
+
+InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
+                                                VPCostContext &Ctx) const {
   Type *RedTy = Ctx.Types.inferScalarType(this);
-  auto *SrcVecTy =
-      cast<VectorType>(toVectorTy(Ctx.Types.inferScalarType(getVecOp()), VF));
+  auto *SrcVecTy = cast<VectorType>(
+      toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF));
   assert(RedTy->isIntegerTy() &&
-         "ExtendedReduction only support integer type currently.");
-  return Ctx.TTI.getExtendedReductionCost(Opcode, isZExt(), RedTy, SrcVecTy,
-                                          std::nullopt, Ctx.CostKind);
+         "VPExpressionRecipe only supports integer types currently.");
+  switch (ExpressionType) {
+  case ExpressionTypes::ExtendedReduction: {
+    unsigned Opcode = RecurrenceDescriptor::getOpcode(
+        cast<VPReductionRecipe>(ExpressionRecipes[1])->getRecurrenceKind());
+    return Ctx.TTI.getExtendedReductionCost(
+        Opcode,
+        cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
+            Instruction::ZExt,
+        RedTy, SrcVecTy, std::nullopt, Ctx.CostKind);
+  }
+  case ExpressionTypes::MulAccReduction:
+    return Ctx.TTI.getMulAccReductionCost(false, RedTy, SrcVecTy, Ctx.CostKind);
+
+  case ExpressionTypes::ExtMulAccReduction:
+    return Ctx.TTI.getMulAccReductionCost(
+        cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
+            Instruction::ZExt,
+        RedTy, SrcVecTy, Ctx.CostKind);
+  }
+  llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
+}
+
+bool VPExpressionRecipe::mayReadOrWriteMemory() const {
+  return any_of(ExpressionRecipes, [](VPSingleDefRecipe *R) {
+    return R->mayReadFromMemory() || R->mayWriteToMemory();
+  });
 }
 
-InstructionCost
-VPMulAccumulateReductionRecipe::computeCost(ElementCount VF,
-                                            VPCostContext &Ctx) const {
-  Type *RedTy = Ctx.Types.inferScalarType(this);
-  auto *SrcVecTy =
-      cast<VectorType>(toVectorTy(Ctx.Types.inferScalarType(getVecOp0()), VF));
-  return Ctx.TTI.getMulAccReductionCost(isZExt(), RedTy, SrcVecTy,
-                                        Ctx.CostKind);
+bool VPExpressionRecipe::mayHaveSideEffects() const {
+  assert(
+      none_of(ExpressionRecipes,
+              [](VPSingleDefRecipe *R) { return R->mayHaveSideEffects(); }) &&
+      "expression cannot contain recipes with side-effects");
+  return false;
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+void VPExpressionRecipe::print(raw_ostream &O, const Twine &Indent,
+                               VPSlotTracker &SlotTracker) const {
+  O << Indent << "EXPRESSION ";
+  printAsOperand(O, SlotTracker);
+  O << " = ";
+  auto *Red = cast<VPReductionRecipe>(ExpressionRecipes.back());
+  unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
+
+  switch (ExpressionType) {
+  case ExpressionTypes::ExtendedReduction: {
+    getOperand(1)->printAsOperand(O, SlotTracker);
+    O << " +";
+    O << " reduce." << Instruction::getOpcodeName(Opcode) << " (";
+    getOperand(0)->printAsOperand(O, SlotTracker);
+    Red->printFlags(O);
+
+    auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]);
+    O << Instruction::getOpcodeName(Ext0->getOpcode()) << " to "
+      << *Ext0->getResultType();
+    if (Red->isConditional()) {
+      O << ", ";
+      Red->getCondOp()->printAsOperand(O, SlotTracker);
+    }
+    O << ")";
+    break;
+  }
+  case ExpressionTypes::MulAccReduction:
+  case ExpressionTypes::ExtMulAccReduction: {
+    getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);
+    O << " + ";
+    O << "reduce."
+      << Instruction::getOpcodeName(
+             RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind()))
+      << " (";
+    O << "mul";
+    bool IsExtended = ExpressionType == ExpressionTypes::ExtMulAccReduction;
+    auto *Mul = cast<VPWidenRecipe>(IsExtended ? ExpressionRecipes[2]
+                                               : ExpressionRecipes[0]);
+    Mul->printFlags(O);
+    if (IsExtended)
+      O << "(";
+    getOperand(0)->printAsOperand(O, SlotTracker);
+    if (IsExtended) {
+      auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]);
+      O << " " << Instruction::getOpcodeName(Ext0->getOpcode()) << " to "
+        << *Ext0->getResultType() << "), (";
+    } else {
+      O << ", ";
+    }
+    getOperand(1)->printAsOperand(O, SlotTracker);
+    if (IsExtended) {
+      auto *Ext1 = cast<VPWidenCastRecipe>(ExpressionRecipes[1]);
+      O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to "
+        << *Ext1->getResultType() << ")";
+    }
+    if (Red->isConditional()) {
+      O << ", ";
+      Red->getCondOp()->printAsOperand(O, SlotTracker);
+    }
+    O << ")";
+    break;
+  }
+  }
+}
+
 void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
                               VPSlotTracker &SlotTracker) const {
   O << Indent << "REDUCE ";
@@ -2639,58 +2804,6 @@ void VPReductionEVLRecipe::print(raw_ostream &O, const Twine &Indent,
   O << ")";
 }
 
-void VPExtendedReductionRecipe::print(raw_ostream &O, const Twine &Indent,
-                                      VPSlotTracker &SlotTracker) const {
-  O << Indent << "EXTENDED-REDUCE ";
-  printAsOperand(O, SlotTracker);
-  O << " = ";
-  getChainOp()->printAsOperand(O, SlotTracker);
-  O << " +";
-  O << " reduce."
-    << Instruction::getOpcodeName(
-           RecurrenceDescriptor::getOpcode(getRecurrenceKind()))
-    << " (";
-  getVecOp()->printAsOperand(O, SlotTracker);
-  printFlags(O);
-  O << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType();
-  if (isConditional()) {
-    O << ", ";
-    getCondOp()->printAsOperand(O, SlotTracker);
-  }
-  O << ")";
-}
-
-void VPMulAccumulateReductionRecipe::print(raw_ostream &O, const Twine &Indent,
-                                           VPSlotTracker &SlotTracker) const {
-  O << Indent << "MULACC-REDUCE ";
-  printAsOperand(O, SlotTracker);
-  O << " = ";
-  getChainOp()->printAsOperand(O, SlotTracker);
-  O << " + ";
-  O << "reduce."
-    << Instruction::getOpcodeName(
-           RecurrenceDescriptor::getOpcode(getRecurrenceKind()))
-    << " (";
-  O << "mul";
-  printFlags(O);
-  if (isExtended())
-    O << "(";
-  getVecOp0()->printAsOperand(O, SlotTracker);
-  if (isExtended())
-    O << " " << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType()
-      << "), (";
-  else
-    O << ", ";
-  getVecOp1()->printAsOperand(O, SlotTracker);
-  if (isExtended())
-    O << " " << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType()
-      << ")";
-  if (isConditional()) {
-    O << ", ";
-    getCondOp()->printAsOperand(O, SlotTracker);
-  }
-  O << ")";
-}
 #endif
 
 /// A helper function to scalarize a single Instruction in the innermost loop.
@@ -3324,36 +3437,9 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
   unsigned InterleaveFactor = Group->getFactor();
   auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);
 
-  // TODO: extend the masked interleaved-group support to reversed access.
   VPValue *BlockInMask = getMask();
-  assert((!BlockInMask || !Group->isReverse()) &&
-         "Reversed masked interleave-group not supported.");
-
   VPValue *Addr = getAddr();
   Value *ResAddr = State.get(Addr, VPLane(0));
-  if (auto *I = dyn_cast<Instruction>(ResAddr))
-    State.setDebugLocFrom(I->getDebugLoc());
-
-  // If the group is reverse, adjust the index to refer to the last vector lane
-  // instead of the first. We adjust the index from the first vector lane,
-  // rather than directly getting the pointer for lane VF - 1, because the
-  // pointer operand of the interleaved access is supposed to be uniform.
-  if (Group->isReverse()) {
-    Value *RuntimeVF =
-        getRuntimeVF(State.Builder, State.Builder.getInt32Ty(), State.VF);
-    Value *Index =
-        State.Builder.CreateSub(RuntimeVF, State.Builder.getInt32(1));
-    Index = State.Builder.CreateMul(Index,
-                                    State.Builder.getInt32(Group->getFactor()));
-    Index = State.Builder.CreateNeg(Index);
-
-    bool InBounds = false;
-    if (auto *Gep = dyn_cast<GetElementPtrInst>(ResAddr->stripPointerCasts()))
-      InBounds = Gep->isInBounds();
-    ResAddr = State.Builder.CreateGEP(ScalarTy, ResAddr, Index, "", InBounds);
-  }
-
-  State.setDebugLocFrom(getDebugLoc());
   Value *PoisonVec = PoisonValue::get(VecTy);
 
   auto CreateGroupMask = [&BlockInMask, &State,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 962c789884b57..701a6e455d09c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1082,6 +1082,15 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
   if (match(Def, m_Select(m_VPValue(), m_VPValue(X), m_Deferred(X))))
     return Def->replaceAllUsesWith(X);
 
+  // select !c, x, y -> select c, y, x
+  VPValue *C;
+  if (match(Def, m_Select(m_Not(m_VPValue(C)), m_VPValue(X), m_VPValue(Y)))) {
+    Def->setOperand(0, C);
+    Def->setOperand(1, Y);
+    Def->setOperand(2, X);
+    return;
+  }
+
   if (match(Def, m_c_Mul(m_VPValue(A), m_SpecificInt(1))))
     return Def->replaceAllUsesWith(A);
 
@@ -1220,9 +1229,9 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
     for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
-      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
-      if (!RepR && !isa<VPWidenRecipe, VPWidenSelectRecipe>(&R))
+      if (!isa<VPWidenRecipe, VPWidenSelectRecipe, VPReplicateRecipe>(&R))
         continue;
+      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
       if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
         continue;
 
@@ -1735,8 +1744,7 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
     auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
     if (!PhiR)
       continue;
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind RK = RdxDesc.getRecurrenceKind();
+    RecurKind RK = PhiR->getRecurrenceKind();
     if (RK != RecurKind::Add && RK != RecurKind::Mul)
       continue;
 
@@ -2114,21 +2122,19 @@ void VPlanTransforms::addActiveLaneMask(
     HeaderMask->replaceAllUsesWith(LaneMask);
 }
 
-/// Try to convert \p CurRecipe to a corresponding EVL-based recipe. Returns
-/// nullptr if no EVL-based recipe could be created.
+/// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
+/// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
+/// recipe could be created.
 /// \p HeaderMask  Header Mask.
 /// \p CurRecipe   Recipe to be transform.
 /// \p TypeInfo    VPlan-based type analysis.
 /// \p AllOneMask  The vector mask parameter of vector-predication intrinsics.
 /// \p EVL         The explicit vector length parameter of vector-predication
 /// intrinsics.
-/// \p PrevEVL     The explicit vector length of the previous iteration. Only
-/// required if \p CurRecipe is a VPInstruction::FirstOrderRecurrenceSplice.
-static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
-                                     VPRecipeBase &CurRecipe,
-                                     VPTypeAnalysis &TypeInfo,
-                                     VPValue &AllOneMask, VPValue &EVL,
-                                     VPValue *PrevEVL) {
+static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
+                                       VPRecipeBase &CurRecipe,
+                                       VPTypeAnalysis &TypeInfo,
+                                       VPValue &AllOneMask, VPValue &EVL) {
   using namespace llvm::VPlanPatternMatch;
   auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
     assert(OrigMask && "Unmasked recipe when folding tail");
@@ -2152,26 +2158,7 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
         VPValue *NewMask = GetNewMask(Red->getCondOp());
         return new VPReductionEVLRecipe(*Red, EVL, NewMask);
       })
-      .Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
-        SmallVector<VPValue *> Ops(Sel->operands());
-        Ops.push_back(&EVL);
-        return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
-                                          TypeInfo.inferScalarType(Sel),
-                                          Sel->getDebugLoc());
-      })
       .Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
-        if (VPI->getOpcode() == VPInstruction::FirstOrderRecurrenceSplice) {
-          assert(PrevEVL && "Fixed-order recurrences require previous EVL");
-          VPValue *MinusOneVPV = VPI->getParent()->getPlan()->getOrAddLiveIn(
-              ConstantInt::getSigned(Type::getInt32Ty(TypeInfo.getContext()),
-                                     -1));
-          SmallVector<VPValue *> Ops(VPI->operands());
-          Ops.append({MinusOneVPV, &AllOneMask, PrevEVL, &EVL});
-          return new VPWidenIntrinsicRecipe(Intrinsic::experimental_vp_splice,
-                                            Ops, TypeInfo.inferScalarType(VPI),
-                                            VPI->getDebugLoc());
-        }
-
         VPValue *LHS, *RHS;
         // Transform select with a header mask condition
         //   select(header_mask, LHS, RHS)
@@ -2198,9 +2185,18 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
 
+  assert(all_of(Plan.getVF().users(),
+                IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+                        VPWidenIntOrFpInductionRecipe>) &&
+         "User of VF that we can't transform to EVL.");
+  Plan.getVF().replaceAllUsesWith(&EVL);
+
+  // Defer erasing recipes till the end so that we don't invalidate the
+  // VPTypeAnalysis cache.
+  SmallVector<VPRecipeBase *> ToErase;
+
   // Create a scalar phi to track the previous EVL if fixed-order recurrence is
   // contained.
-  VPInstruction *PrevEVL = nullptr;
   bool ContainsFORs =
       any_of(Header->phis(), IsaPred<VPFirstOrderRecurrencePHIRecipe>);
   if (ContainsFORs) {
@@ -2213,21 +2209,37 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                                              DebugLoc());
 
     Builder.setInsertPoint(Header, Header->getFirstNonPhi());
-    PrevEVL = Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl");
-  }
-
-  for (VPUser *U : to_vector(Plan.getVF().users())) {
-    if (auto *R = dyn_cast<VPVectorEndPointerRecipe>(U))
-      R->setOperand(1, &EVL);
+    VPValue *PrevEVL =
+        Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl");
+
+    for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+             vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+      for (VPRecipeBase &R : *VPBB) {
+        using namespace VPlanPatternMatch;
+        VPValue *V1, *V2;
+        if (!match(&R,
+                   m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
+                       m_VPValue(V1), m_VPValue(V2))))
+          continue;
+        VPValue *Imm = Plan.getOrAddLiveIn(
+            ConstantInt::getSigned(Type::getInt32Ty(Ctx), -1));
+        VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe(
+            Intrinsic::experimental_vp_splice,
+            {V1, V2, Imm, AllOneMask, PrevEVL, &EVL},
+            TypeInfo.inferScalarType(R.getVPSingleValue()), R.getDebugLoc());
+        VPSplice->insertBefore(&R);
+        R.getVPSingleValue()->replaceAllUsesWith(VPSplice);
+        ToErase.push_back(&R);
+      }
+    }
   }
 
-  SmallVector<VPRecipeBase *> ToErase;
-
+  // Try to optimize header mask recipes away to their EVL variants.
   for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
     for (VPUser *U : collectUsersRecursively(HeaderMask)) {
       auto *CurRecipe = cast<VPRecipeBase>(U);
-      VPRecipeBase *EVLRecipe = createEVLRecipe(
-          HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL, PrevEVL);
+      VPRecipeBase *EVLRecipe =
+          optimizeMaskToEVL(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
       if (!EVLRecipe)
         continue;
 
@@ -2243,8 +2255,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
         VPValue *CurVPV = CurRecipe->getVPSingleValue();
         CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
       }
-      // Defer erasing recipes till the end so that we don't invalidate the
-      // VPTypeAnalysis cache.
       ToErase.push_back(CurRecipe);
     }
   }
@@ -2300,12 +2310,11 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
     VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
   VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
   // The transform updates all users of inductions to work based on EVL, instead
-  // of the VF directly. At the moment, widened inductions cannot be updated, so
-  // bail out if the plan contains any.
-  bool ContainsWidenInductions = any_of(
-      Header->phis(),
-      IsaPred<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>);
-  if (ContainsWidenInductions)
+  // of the VF directly. At the moment, widened pointer inductions cannot be
+  // updated, so bail out if the plan contains any.
+  bool ContainsWidenPointerInductions =
+      any_of(Header->phis(), IsaPred<VPWidenPointerInductionRecipe>);
+  if (ContainsWidenPointerInductions)
     return false;
 
   auto *CanonicalIVPHI = Plan.getCanonicalIV();
@@ -2482,23 +2491,23 @@ void VPlanTransforms::createInterleaveGroups(
     auto *InsertPos =
         cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IRInsertPos));
 
+    bool InBounds = false;
+    if (auto *Gep = dyn_cast<GetElementPtrInst>(
+            getLoadStorePointerOperand(IRInsertPos)->stripPointerCasts()))
+      InBounds = Gep->isInBounds();
+
     // Get or create the start address for the interleave group.
     auto *Start =
         cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IG->getMember(0)));
     VPValue *Addr = Start->getAddr();
     VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
     if (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPos)) {
-      // TODO: Hoist Addr's defining recipe (and any operands as needed) to
-      // InsertPos or sink loads above zero members to join it.
-      bool InBounds = false;
-      if (auto *Gep = dyn_cast<GetElementPtrInst>(
-              getLoadStorePointerOperand(IRInsertPos)->stripPointerCasts()))
-        InBounds = Gep->isInBounds();
-
       // We cannot re-use the address of member zero because it does not
       // dominate the insert position. Instead, use the address of the insert
       // position and create a PtrAdd adjusting it to the address of member
       // zero.
+      // TODO: Hoist Addr's defining recipe (and any operands as needed) to
+      // InsertPos or sink loads above zero members to join it.
       assert(IG->getIndex(IRInsertPos) != 0 &&
              "index of insert position shouldn't be zero");
       auto &DL = IRInsertPos->getDataLayout();
@@ -2512,6 +2521,19 @@ void VPlanTransforms::createInterleaveGroups(
       Addr = InBounds ? B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV)
                       : B.createPtrAdd(InsertPos->getAddr(), OffsetVPV);
     }
+    // If the group is reverse, adjust the index to refer to the last vector
+    // lane instead of the first. We adjust the index from the first vector
+    // lane, rather than directly getting the pointer for lane VF - 1, because
+    // the pointer operand of the interleaved access is supposed to be uniform.
+    if (IG->isReverse()) {
+      auto *ReversePtr = new VPVectorEndPointerRecipe(
+          Addr, &Plan.getVF(), getLoadStoreType(IRInsertPos),
+          -(int64_t)IG->getFactor(),
+          InBounds ? GEPNoWrapFlags::inBounds() : GEPNoWrapFlags::none(),
+          InsertPos->getDebugLoc());
+      ReversePtr->insertBefore(InsertPos);
+      Addr = ReversePtr;
+    }
     auto *VPIG = new VPInterleaveRecipe(IG, Addr, StoredValues,
                                         InsertPos->getMask(), NeedsMaskForGaps, InsertPos->getDebugLoc());
     VPIG->insertBefore(InsertPos);
@@ -2627,6 +2649,8 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
     Inc = SplatVF;
     Prev = WidenIVR->getLastUnrolledPartOperand();
   } else {
+    if (VPRecipeBase *R = VF->getDefiningRecipe())
+      Builder.setInsertPoint(R->getParent(), std::next(R->getIterator()));
     // Multiply the vectorization factor by the step using integer or
     // floating-point arithmetic as appropriate.
     if (StepTy->isFloatingPointTy())
@@ -2663,82 +2687,6 @@ void VPlanTransforms::dissolveLoopRegions(VPlan &Plan) {
     R->dissolveToCFGLoop();
 }
 
-// Expand VPExtendedReductionRecipe to VPWidenCastRecipe + VPReductionRecipe.
-static void expandVPExtendedReduction(VPExtendedReductionRecipe *ExtRed) {
-  VPWidenCastRecipe *Ext;
-  // Only ZExt contains non-neg flags.
-  if (ExtRed->isZExt())
-    Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(),
-                                ExtRed->getResultType(), *ExtRed,
-                                ExtRed->getDebugLoc());
-  else
-    Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(),
-                                ExtRed->getResultType(), {},
-                                ExtRed->getDebugLoc());
-
-  auto *Red = new VPReductionRecipe(
-      ExtRed->getRecurrenceKind(), FastMathFlags(), ExtRed->getChainOp(), Ext,
-      ExtRed->getCondOp(), ExtRed->isOrdered(), ExtRed->getDebugLoc());
-  Ext->insertBefore(ExtRed);
-  Red->insertBefore(ExtRed);
-  ExtRed->replaceAllUsesWith(Red);
-  ExtRed->eraseFromParent();
-}
-
-// Expand VPMulAccumulateReductionRecipe to VPWidenRecipe (mul) +
-// VPReductionRecipe (reduce.add)
-// + VPWidenCastRecipe (optional).
-static void
-expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
-  // Generate inner VPWidenCastRecipes if necessary.
-  // Note that we will drop the extend after mul which transforms
-  // reduce.add(ext(mul(ext, ext))) to reduce.add(mul(ext, ext)).
-  VPValue *Op0, *Op1;
-  if (MulAcc->isExtended()) {
-    Type *RedTy = MulAcc->getResultType();
-    if (MulAcc->isZExt())
-      Op0 = new VPWidenCastRecipe(
-          MulAcc->getExtOpcode(), MulAcc->getVecOp0(), RedTy,
-          VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()), MulAcc->getDebugLoc());
-    else
-      Op0 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp0(),
-                                  RedTy, {}, MulAcc->getDebugLoc());
-    Op0->getDefiningRecipe()->insertBefore(MulAcc);
-    // Prevent reduce.add(mul(ext(A), ext(A))) generate duplicate
-    // VPWidenCastRecipe.
-    if (MulAcc->getVecOp0() == MulAcc->getVecOp1()) {
-      Op1 = Op0;
-    } else {
-      if (MulAcc->isZExt())
-        Op1 = new VPWidenCastRecipe(
-            MulAcc->getExtOpcode(), MulAcc->getVecOp1(), RedTy,
-            VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()),
-            MulAcc->getDebugLoc());
-      else
-        Op1 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp1(),
-                                    RedTy, {}, MulAcc->getDebugLoc());
-      Op1->getDefiningRecipe()->insertBefore(MulAcc);
-    }
-  } else {
-    // No extends in this MulAccRecipe.
-    Op0 = MulAcc->getVecOp0();
-    Op1 = MulAcc->getVecOp1();
-  }
-
-  std::array<VPValue *, 2> MulOps = {Op0, Op1};
-  auto *Mul = new VPWidenRecipe(Instruction::Mul, ArrayRef(MulOps), *MulAcc,
-                                MulAcc->getDebugLoc());
-  Mul->insertBefore(MulAcc);
-
-  auto *Red = new VPReductionRecipe(
-      MulAcc->getRecurrenceKind(), FastMathFlags(), MulAcc->getChainOp(), Mul,
-      MulAcc->getCondOp(), MulAcc->isOrdered(), MulAcc->getDebugLoc());
-  Red->insertBefore(MulAcc);
-
-  MulAcc->replaceAllUsesWith(Red);
-  MulAcc->eraseFromParent();
-}
-
 void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
                                                Type &CanonicalIVTy) {
   using namespace llvm::VPlanPatternMatch;
@@ -2763,6 +2711,11 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
         continue;
       }
 
+      if (auto *Expr = dyn_cast<VPExpressionRecipe>(&R)) {
+        Expr->decompose();
+        ToRemove.push_back(Expr);
+      }
+
       VPValue *VectorStep;
       VPValue *ScalarStep;
       if (!match(&R, m_VPInstruction<VPInstruction::WideIVStep>(
@@ -2803,14 +2756,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
       VPI->replaceAllUsesWith(VectorStep);
       ToRemove.push_back(VPI);
     }
-    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
-      if (auto *ExtRed = dyn_cast<VPExtendedReductionRecipe>(&R)) {
-        expandVPExtendedReduction(ExtRed);
-        continue;
-      }
-      if (auto *MulAcc = dyn_cast<VPMulAccumulateReductionRecipe>(&R))
-        expandVPMulAccumulateReduction(MulAcc);
-    }
   }
 
   for (VPRecipeBase *R : ToRemove)
@@ -2909,10 +2854,10 @@ void VPlanTransforms::handleUncountableEarlyExit(
 }
 
 /// This function tries convert extended in-loop reductions to
-/// VPExtendedReductionRecipe and clamp the \p Range if it is beneficial and
-/// valid. The created recipe must be lowered to concrete
+/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
+/// valid. The created recipe must be decomposed to its constituent
 /// recipes before execution.
-static VPExtendedReductionRecipe *
+static VPExpressionRecipe *
 tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
                                      VFRange &Range) {
   using namespace VPlanPatternMatch;
@@ -2946,19 +2891,20 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
           cast<VPWidenCastRecipe>(VecOp)->getOpcode() ==
               Instruction::CastOps::ZExt,
           Ctx.Types.inferScalarType(A)))
-    return new VPExtendedReductionRecipe(Red, cast<VPWidenCastRecipe>(VecOp));
+    return new VPExpressionRecipe(cast<VPWidenCastRecipe>(VecOp), Red);
 
   return nullptr;
 }
 
 /// This function tries convert extended in-loop reductions to
-/// VPMulAccumulateReductionRecipe and clamp the \p Range if it is beneficial
-/// and valid. The created VPExtendedReductionRecipe must be lower to concrete
-/// recipes before execution. Patterns of MulAccumulateReduction:
+/// VPExpressionRecipe and clamp the \p Range if it is beneficial
+/// and valid. The created VPExpressionRecipe must be decomposed to its
+/// constituent recipes before execution. Patterns of the
+/// VPExpressionRecipe:
 ///   reduce.add(mul(...)),
 ///   reduce.add(mul(ext(A), ext(B))),
 ///   reduce.add(ext(mul(ext(A), ext(B)))).
-static VPMulAccumulateReductionRecipe *
+static VPExpressionRecipe *
 tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
                                           VPCostContext &Ctx, VFRange &Range) {
   using namespace VPlanPatternMatch;
@@ -3014,12 +2960,12 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
         match(RecipeB, m_ZExtOrSExt(m_VPValue())) &&
         IsMulAccValidAndClampRange(RecipeA->getOpcode() ==
                                        Instruction::CastOps::ZExt,
-                                   Mul, RecipeA, RecipeB, nullptr))
-      return new VPMulAccumulateReductionRecipe(Red, Mul, RecipeA, RecipeB,
-                                                RecipeA->getResultType());
+                                   Mul, RecipeA, RecipeB, nullptr)) {
+      return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
+    }
     // Match reduce.add(mul).
     if (IsMulAccValidAndClampRange(true, Mul, nullptr, nullptr, nullptr))
-      return new VPMulAccumulateReductionRecipe(Red, Mul, RedTy);
+      return new VPExpressionRecipe(Mul, Red);
   }
   // Match reduce.add(ext(mul(ext(A), ext(B)))).
   // All extend recipes must have same opcode or A == B
@@ -3036,9 +2982,24 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
         Ext0->getOpcode() == Ext1->getOpcode() &&
         IsMulAccValidAndClampRange(Ext0->getOpcode() ==
                                        Instruction::CastOps::ZExt,
-                                   Mul, Ext0, Ext1, Ext))
-      return new VPMulAccumulateReductionRecipe(Red, Mul, Ext0, Ext1,
-                                                Ext->getResultType());
+                                   Mul, Ext0, Ext1, Ext)) {
+      auto *NewExt0 = new VPWidenCastRecipe(
+          Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
+          Ext0->getDebugLoc());
+      NewExt0->insertBefore(Ext0);
+
+      VPWidenCastRecipe *NewExt1 = NewExt0;
+      if (Ext0 != Ext1) {
+        NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0),
+                                        Ext->getResultType(), *Ext1,
+                                        Ext1->getDebugLoc());
+        NewExt1->insertBefore(Ext1);
+      }
+      Mul->setOperand(0, NewExt0);
+      Mul->setOperand(1, NewExt1);
+      Red->setOperand(1, Mul);
+      return new VPExpressionRecipe(NewExt0, NewExt1, Mul, Red);
+    }
   }
   return nullptr;
 }
@@ -3048,8 +3009,9 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
 static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
                                                VPCostContext &Ctx,
                                                VFRange &Range) {
-  VPReductionRecipe *AbstractR = nullptr;
-
+  VPExpressionRecipe *AbstractR = nullptr;
+  auto IP = std::next(Red->getIterator());
+  auto *VPBB = Red->getParent();
   if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
     AbstractR = MulAcc;
   else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
@@ -3058,7 +3020,7 @@ static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
   if (!AbstractR)
     return;
 
-  AbstractR->insertBefore(Red);
+  AbstractR->insertBefore(*VPBB, IP);
   Red->replaceAllUsesWith(AbstractR);
 }
 
@@ -3066,7 +3028,7 @@ void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
                                                VFRange &Range) {
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
-    for (VPRecipeBase &R : *VPBB) {
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       if (auto *Red = dyn_cast<VPReductionRecipe>(&R))
         tryToCreateAbstractReductionRecipe(Red, Ctx, Range);
     }
@@ -3177,6 +3139,9 @@ static bool isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
          GroupSize == VectorRegWidth;
 }
 
+/// Returns true if \p VPValue is a narrow VPValue.
+static bool isAlreadyNarrow(VPValue *VPV) { return VPV->isLiveIn(); }
+
 void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
                                              unsigned VectorRegWidth) {
   using namespace llvm::VPlanPatternMatch;
@@ -3228,6 +3193,16 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
     if (InterleaveR->getStoredValues().empty())
       continue;
 
+    // Narrow interleave groups, if all operands are already matching narrow
+    // ops.
+    auto *Member0 = InterleaveR->getStoredValues()[0];
+    if (isAlreadyNarrow(Member0) &&
+        all_of(InterleaveR->getStoredValues(),
+               [Member0](VPValue *VPV) { return Member0 == VPV; })) {
+      StoreGroups.push_back(InterleaveR);
+      continue;
+    }
+
     // For now, we only support full interleave groups storing load interleave
     // groups.
     if (all_of(enumerate(InterleaveR->getStoredValues()), [](auto Op) {
@@ -3251,7 +3226,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
     if (!WideMember0)
       return;
     for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
-      auto *R = dyn_cast<VPWidenRecipe>(V->getDefiningRecipe());
+      auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe());
       if (!R || R->getOpcode() != WideMember0->getOpcode() ||
           R->getNumOperands() > 2)
         return;
@@ -3284,6 +3259,12 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
       return L;
     }
 
+    if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
+      assert(RepR->isSingleScalar() &&
+             isa<LoadInst>(RepR->getUnderlyingInstr()) &&
+             "must be a single scalar load");
+      return RepR;
+    }
     auto *WideLoad = cast<VPWidenLoadRecipe>(R);
 
     // Narrow wide load to uniform scalar load, as transformed VPlan will only
@@ -3298,13 +3279,16 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
   // Narrow operation tree rooted at store groups.
   for (auto *StoreGroup : StoreGroups) {
     VPValue *Res = nullptr;
-    if (auto *WideMember0 = dyn_cast<VPWidenRecipe>(
-            StoreGroup->getStoredValues()[0]->getDefiningRecipe())) {
+    VPValue *Member0 = StoreGroup->getStoredValues()[0];
+    if (isAlreadyNarrow(Member0)) {
+      Res = Member0;
+    } else if (auto *WideMember0 =
+                   dyn_cast<VPWidenRecipe>(Member0->getDefiningRecipe())) {
       for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx)
         WideMember0->setOperand(Idx, NarrowOp(WideMember0->getOperand(Idx)));
       Res = WideMember0;
     } else {
-      Res = NarrowOp(StoreGroup->getStoredValues()[0]);
+      Res = NarrowOp(Member0);
     }
 
     auto *S = new VPWidenStoreRecipe(
@@ -3328,8 +3312,8 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
 
 /// Add branch weight metadata, if the \p Plan's middle block is terminated by a
 /// BranchOnCond recipe.
-void VPlanTransforms::addBranchWeightToMiddleTerminator(VPlan &Plan,
-                                                        ElementCount VF) {
+void VPlanTransforms::addBranchWeightToMiddleTerminator(
+    VPlan &Plan, ElementCount VF, std::optional<unsigned> VScaleForTuning) {
   VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
   auto *MiddleTerm =
       dyn_cast_or_null<VPInstruction>(MiddleVPBB->getTerminator());
@@ -3341,6 +3325,8 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(VPlan &Plan,
          "must have a BranchOnCond");
   // Assume that `TripCount % VectorStep ` is equally distributed.
   unsigned VectorStep = Plan.getUF() * VF.getKnownMinValue();
+  if (VF.isScalable() && VScaleForTuning.has_value())
+    VectorStep *= *VScaleForTuning;
   assert(VectorStep > 0 && "trip count should not be zero");
   MDBuilder MDB(Plan.getScalarHeader()->getIRBasicBlock()->getContext());
   MDNode *BranchWeights =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 40885cd52a127..8d2eded45da22 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -238,7 +238,9 @@ struct VPlanTransforms {
 
   /// Add branch weight metadata, if the \p Plan's middle block is terminated by
   /// a BranchOnCond recipe.
-  static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF);
+  static void
+  addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
+                                    std::optional<unsigned> VScaleForTuning);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 4bc9e409069d1..b89cd21595efd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -355,7 +355,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
                       m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
         match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
                       m_VPValue(), m_VPValue(Op1))) ||
-        match(&R, m_VPInstruction<VPInstruction::ComputeFindLastIVResult>(
+        match(&R, m_VPInstruction<VPInstruction::ComputeFindIVResult>(
                       m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
       addUniformForAllParts(cast<VPInstruction>(&R));
       for (unsigned Part = 1; Part != UF; ++Part)
@@ -486,6 +486,7 @@ static VPReplicateRecipe *cloneForLane(VPlan &Plan, VPBuilder &Builder,
   auto *New =
       new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,
                             /*IsSingleScalar=*/true, /*Mask=*/nullptr, *RepR);
+  New->transferFlags(*RepR);
   New->insertBefore(RepR);
   return New;
 }
@@ -493,8 +494,16 @@ static VPReplicateRecipe *cloneForLane(VPlan &Plan, VPBuilder &Builder,
 void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
   Type *IdxTy = IntegerType::get(
       Plan.getScalarHeader()->getIRBasicBlock()->getContext(), 32);
-  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
-           vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+
+  // Visit all VPBBs outside the loop region and directly inside the top-level
+  // loop region.
+  auto VPBBsOutsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
+      vp_depth_first_shallow(Plan.getEntry()));
+  auto VPBBsInsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
+      vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()));
+  auto VPBBsToUnroll =
+      concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion);
+  for (VPBasicBlock *VPBB : VPBBsToUnroll) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
       if (!RepR || RepR->isSingleScalar())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 962f676a07b2f..8dcd57f1b3598 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -69,14 +69,13 @@ inline bool isSingleScalar(const VPValue *VPV) {
     return Rep->isSingleScalar() || (PreservesUniformity(Rep->getOpcode()) &&
                                      all_of(Rep->operands(), isSingleScalar));
   }
-  if (isa<VPWidenGEPRecipe, VPDerivedIVRecipe, VPBlendRecipe>(VPV))
+  if (isa<VPWidenGEPRecipe, VPDerivedIVRecipe, VPBlendRecipe,
+          VPWidenSelectRecipe>(VPV))
     return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
   if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
     return PreservesUniformity(WidenR->getOpcode()) &&
            all_of(WidenR->operands(), isSingleScalar);
   }
-  if (auto *WidenR = dyn_cast<VPWidenSelectRecipe>(VPV))
-    return all_of(WidenR->operands(), isSingleScalar);
   if (auto *VPI = dyn_cast<VPInstruction>(VPV))
     return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
            (PreservesUniformity(VPI->getOpcode()) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index a0d3dc9b934cc..279cdac92d2d1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -49,6 +49,7 @@ class VPValue {
   friend struct VPDoubleValueDef;
   friend class VPInterleaveRecipe;
   friend class VPlan;
+  friend class VPExpressionRecipe;
 
   const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
 
@@ -330,13 +331,12 @@ class VPDef {
     VPBranchOnMaskSC,
     VPDerivedIVSC,
     VPExpandSCEVSC,
+    VPExpressionSC,
     VPIRInstructionSC,
     VPInstructionSC,
     VPInterleaveSC,
     VPReductionEVLSC,
     VPReductionSC,
-    VPMulAccumulateReductionSC,
-    VPExtendedReductionSC,
     VPPartialReductionSC,
     VPReplicateSC,
     VPScalarIVStepsSC,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index fba4a68f4a27b..38ada33d7ee19 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -156,8 +156,16 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
         .Case<VPWidenIntrinsicRecipe>([&](const VPWidenIntrinsicRecipe *S) {
           return VerifyEVLUse(*S, S->getNumOperands() - 1);
         })
-        .Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe>(
+        .Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe,
+              VPWidenIntOrFpInductionRecipe>(
             [&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); })
+        .Case<VPScalarIVStepsRecipe>([&](auto *R) {
+          if (R->getNumOperands() != 3) {
+            errs() << "Unrolling with EVL tail folding not yet supported\n";
+            return false;
+          }
+          return VerifyEVLUse(*R, 2);
+        })
         .Case<VPWidenLoadEVLRecipe, VPVectorEndPointerRecipe>(
             [&](const VPRecipeBase *R) { return VerifyEVLUse(*R, 1); })
         .Case<VPInstructionWithType>(
@@ -165,13 +173,27 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
         .Case<VPInstruction>([&](const VPInstruction *I) {
           if (I->getOpcode() == Instruction::PHI)
             return VerifyEVLUse(*I, 1);
-          if (I->getOpcode() != Instruction::Add) {
-            errs() << "EVL is used as an operand in non-VPInstruction::Add\n";
+          switch (I->getOpcode()) {
+          case Instruction::Add:
+            break;
+          case Instruction::UIToFP:
+          case Instruction::Trunc:
+          case Instruction::ZExt:
+          case Instruction::Mul:
+          case Instruction::FMul:
+            // Opcodes above can only use EVL after wide inductions have been
+            // expanded.
+            if (!VerifyLate) {
+              errs() << "EVL used by unexpected VPInstruction\n";
+              return false;
+            }
+            break;
+          default:
+            errs() << "EVL used by unexpected VPInstruction\n";
             return false;
           }
           if (I->getNumUsers() != 1) {
-            errs() << "EVL is used in VPInstruction:Add with multiple "
-                      "users\n";
+            errs() << "EVL is used in VPInstruction with multiple users\n";
             return false;
           }
           if (!VerifyLate && !isa<VPEVLBasedIVPHIRecipe>(*I->users().begin())) {
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 19e82099e87f0..fe8d74c43dfdc 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -19,9 +19,10 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
-#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/InstSimplifyFolder.h"
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
@@ -72,14 +73,15 @@ class VectorCombine {
                 const DominatorTree &DT, AAResults &AA, AssumptionCache &AC,
                 const DataLayout *DL, TTI::TargetCostKind CostKind,
                 bool TryEarlyFoldsOnly)
-      : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), DL(DL),
-        CostKind(CostKind), TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
+      : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI),
+        DT(DT), AA(AA), AC(AC), DL(DL), CostKind(CostKind),
+        TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
 
   bool run();
 
 private:
   Function &F;
-  IRBuilder<> Builder;
+  IRBuilder<InstSimplifyFolder> Builder;
   const TargetTransformInfo &TTI;
   const DominatorTree &DT;
   AAResults &AA;
@@ -121,6 +123,7 @@ class VectorCombine {
   bool foldBinopOfReductions(Instruction &I);
   bool foldSingleElementStore(Instruction &I);
   bool scalarizeLoadExtract(Instruction &I);
+  bool scalarizeExtExtract(Instruction &I);
   bool foldConcatOfBoolMasks(Instruction &I);
   bool foldPermuteOfBinops(Instruction &I);
   bool foldShuffleOfBinops(Instruction &I);
@@ -529,7 +532,7 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
 /// Create a shuffle that translates (shifts) 1 element from the input vector
 /// to a new element location.
 static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
-                                 unsigned NewIndex, IRBuilder<> &Builder) {
+                                 unsigned NewIndex, IRBuilderBase &Builder) {
   // The shuffle mask is poison except for 1 lane that is being translated
   // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
   // ShufMask = { 2, poison, poison, poison }
@@ -545,7 +548,7 @@ static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
 /// unnecessary instructions.
 static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
                                             unsigned NewIndex,
-                                            IRBuilder<> &Builder) {
+                                            IRBuilderBase &Builder) {
   // Shufflevectors can only be created for fixed-width vectors.
   Value *X = ExtElt->getVectorOperand();
   if (!isa<FixedVectorType>(X->getType()))
@@ -560,7 +563,8 @@ static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
 
   Value *Shuf = createShiftShuffle(X, cast<ConstantInt>(C)->getZExtValue(),
                                    NewIndex, Builder);
-  return cast<ExtractElementInst>(Builder.CreateExtractElement(Shuf, NewIndex));
+  return dyn_cast<ExtractElementInst>(
+      Builder.CreateExtractElement(Shuf, NewIndex));
 }
 
 /// Try to reduce extract element costs by converting scalar compares to vector
@@ -1091,12 +1095,14 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
     return false;
 
   // TODO: Allow intrinsics with different argument types
-  // TODO: Allow intrinsics with scalar arguments
-  if (II && (!isTriviallyVectorizable(II->getIntrinsicID()) ||
-             !all_of(II->args(), [&II](Value *Arg) {
-               return Arg->getType() == II->getType();
-             })))
-    return false;
+  if (II) {
+    if (!isTriviallyVectorizable(II->getIntrinsicID()))
+      return false;
+    for (auto [Idx, Arg] : enumerate(II->args()))
+      if (Arg->getType() != II->getType() &&
+          !isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(), Idx, &TTI))
+        return false;
+  }
 
   // Do not convert the vector condition of a vector select into a scalar
   // condition. That may cause problems for codegen because of differences in
@@ -1109,19 +1115,18 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
 
   // Match constant vectors or scalars being inserted into constant vectors:
   // vec_op [VecC0 | (inselt VecC0, V0, Index)], ...
-  SmallVector<Constant *> VecCs;
-  SmallVector<Value *> ScalarOps;
+  SmallVector<Value *> VecCs, ScalarOps;
   std::optional<uint64_t> Index;
 
   auto Ops = II ? II->args() : I.operands();
-  for (Value *Op : Ops) {
+  for (auto [OpNum, Op] : enumerate(Ops)) {
     Constant *VecC;
     Value *V;
     uint64_t InsIdx = 0;
-    VectorType *OpTy = cast<VectorType>(Op->getType());
-    if (match(Op, m_InsertElt(m_Constant(VecC), m_Value(V),
-                              m_ConstantInt(InsIdx)))) {
+    if (match(Op.get(), m_InsertElt(m_Constant(VecC), m_Value(V),
+                                    m_ConstantInt(InsIdx)))) {
       // Bail if any inserts are out of bounds.
+      VectorType *OpTy = cast<VectorType>(Op->getType());
       if (OpTy->getElementCount().getKnownMinValue() <= InsIdx)
         return false;
       // All inserts must have the same index.
@@ -1132,7 +1137,11 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
         return false;
       VecCs.push_back(VecC);
       ScalarOps.push_back(V);
-    } else if (match(Op, m_Constant(VecC))) {
+    } else if (II && isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+                                                        OpNum, &TTI)) {
+      VecCs.push_back(Op.get());
+      ScalarOps.push_back(Op.get());
+    } else if (match(Op.get(), m_Constant(VecC))) {
       VecCs.push_back(VecC);
       ScalarOps.push_back(nullptr);
     } else {
@@ -1176,16 +1185,17 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   // Fold the vector constants in the original vectors into a new base vector to
   // get more accurate cost modelling.
   Value *NewVecC = nullptr;
+  TargetFolder Folder(*DL);
   if (CI)
-    NewVecC = ConstantFoldCompareInstOperands(CI->getPredicate(), VecCs[0],
-                                              VecCs[1], *DL);
+    NewVecC = Folder.FoldCmp(CI->getPredicate(), VecCs[0], VecCs[1]);
   else if (UO)
-    NewVecC = ConstantFoldUnaryOpOperand(Opcode, VecCs[0], *DL);
+    NewVecC =
+        Folder.FoldUnOpFMF(UO->getOpcode(), VecCs[0], UO->getFastMathFlags());
   else if (BO)
-    NewVecC = ConstantFoldBinaryOpOperands(Opcode, VecCs[0], VecCs[1], *DL);
+    NewVecC = Folder.FoldBinOp(BO->getOpcode(), VecCs[0], VecCs[1]);
   else if (II->arg_size() == 2)
-    NewVecC = ConstantFoldBinaryIntrinsic(II->getIntrinsicID(), VecCs[0],
-                                          VecCs[1], II->getType(), II);
+    NewVecC = Folder.FoldBinaryIntrinsic(II->getIntrinsicID(), VecCs[0],
+                                         VecCs[1], II->getType(), &I);
 
   // Get cost estimate for the insert element. This cost will factor into
   // both sequences.
@@ -1193,8 +1203,9 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   InstructionCost NewCost =
       ScalarOpCost + TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
                                             CostKind, *Index, NewVecC);
-  for (auto [Op, VecC, Scalar] : zip(Ops, VecCs, ScalarOps)) {
-    if (!Scalar)
+  for (auto [Idx, Op, VecC, Scalar] : enumerate(Ops, VecCs, ScalarOps)) {
+    if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
+                              II->getIntrinsicID(), Idx, &TTI)))
       continue;
     InstructionCost InsertCost = TTI.getVectorInstrCost(
         Instruction::InsertElement, VecTy, CostKind, *Index, VecC, Scalar);
@@ -1238,16 +1249,12 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
 
   // Create a new base vector if the constant folding failed.
   if (!NewVecC) {
-    SmallVector<Value *> VecCValues;
-    VecCValues.reserve(VecCs.size());
-    append_range(VecCValues, VecCs);
     if (CI)
       NewVecC = Builder.CreateCmp(CI->getPredicate(), VecCs[0], VecCs[1]);
     else if (UO || BO)
-      NewVecC = Builder.CreateNAryOp(Opcode, VecCValues);
+      NewVecC = Builder.CreateNAryOp(Opcode, VecCs);
     else
-      NewVecC =
-          Builder.CreateIntrinsic(VecTy, II->getIntrinsicID(), VecCValues);
+      NewVecC = Builder.CreateIntrinsic(VecTy, II->getIntrinsicID(), VecCs);
   }
   Value *Insert = Builder.CreateInsertElement(NewVecC, Scalar, *Index);
   replaceValue(I, *Insert);
@@ -1459,10 +1466,12 @@ bool VectorCombine::foldBinopOfReductions(Instruction &I) {
   LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
                     << "\n  OldCost: " << OldCost << " vs NewCost: " << NewCost
                     << "\n");
-  Value *VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
-  if (auto *PDInst = dyn_cast<PossiblyDisjointInst>(&I))
-    if (auto *PDVectorBO = dyn_cast<PossiblyDisjointInst>(VectorBO))
-      PDVectorBO->setIsDisjoint(PDInst->isDisjoint());
+  Value *VectorBO;
+  if (BinOpOpc == Instruction::Or)
+    VectorBO = Builder.CreateOr(V0, V1, "",
+                                cast<PossiblyDisjointInst>(I).isDisjoint());
+  else
+    VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
 
   Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
   replaceValue(I, *Rdx);
@@ -1519,7 +1528,7 @@ class ScalarizationResult {
   }
 
   /// Freeze the ToFreeze and update the use in \p User to use it.
-  void freeze(IRBuilder<> &Builder, Instruction &UserI) {
+  void freeze(IRBuilderBase &Builder, Instruction &UserI) {
     assert(isSafeWithFreeze() &&
            "should only be used when freezing is required");
     assert(is_contained(ToFreeze->users(), &UserI) &&
@@ -1770,6 +1779,73 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
   return true;
 }
 
+bool VectorCombine::scalarizeExtExtract(Instruction &I) {
+  auto *Ext = dyn_cast<ZExtInst>(&I);
+  if (!Ext)
+    return false;
+
+  // Try to convert a vector zext feeding only extracts to a set of scalar
+  //   (Src << ExtIdx *Size) & (Size -1)
+  // if profitable   .
+  auto *SrcTy = dyn_cast<FixedVectorType>(Ext->getOperand(0)->getType());
+  if (!SrcTy)
+    return false;
+  auto *DstTy = cast<FixedVectorType>(Ext->getType());
+
+  Type *ScalarDstTy = DstTy->getElementType();
+  if (DL->getTypeSizeInBits(SrcTy) != DL->getTypeSizeInBits(ScalarDstTy))
+    return false;
+
+  InstructionCost VectorCost =
+      TTI.getCastInstrCost(Instruction::ZExt, DstTy, SrcTy,
+                           TTI::CastContextHint::None, CostKind, Ext);
+  unsigned ExtCnt = 0;
+  bool ExtLane0 = false;
+  for (User *U : Ext->users()) {
+    const APInt *Idx;
+    if (!match(U, m_ExtractElt(m_Value(), m_APInt(Idx))))
+      return false;
+    if (cast<Instruction>(U)->use_empty())
+      continue;
+    ExtCnt += 1;
+    ExtLane0 |= Idx->isZero();
+    VectorCost += TTI.getVectorInstrCost(Instruction::ExtractElement, DstTy,
+                                         CostKind, Idx->getZExtValue(), U);
+  }
+
+  InstructionCost ScalarCost =
+      ExtCnt * TTI.getArithmeticInstrCost(
+                   Instruction::And, ScalarDstTy, CostKind,
+                   {TTI::OK_AnyValue, TTI::OP_None},
+                   {TTI::OK_NonUniformConstantValue, TTI::OP_None}) +
+      (ExtCnt - ExtLane0) *
+          TTI.getArithmeticInstrCost(
+              Instruction::LShr, ScalarDstTy, CostKind,
+              {TTI::OK_AnyValue, TTI::OP_None},
+              {TTI::OK_NonUniformConstantValue, TTI::OP_None});
+  if (ScalarCost > VectorCost)
+    return false;
+
+  Value *ScalarV = Ext->getOperand(0);
+  if (!isGuaranteedNotToBePoison(ScalarV, &AC, dyn_cast<Instruction>(ScalarV),
+                                 &DT))
+    ScalarV = Builder.CreateFreeze(ScalarV);
+  ScalarV = Builder.CreateBitCast(
+      ScalarV,
+      IntegerType::get(SrcTy->getContext(), DL->getTypeSizeInBits(SrcTy)));
+  uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(SrcTy->getElementType());
+  uint64_t EltBitMask = (1ull << SrcEltSizeInBits) - 1;
+  for (User *U : Ext->users()) {
+    auto *Extract = cast<ExtractElementInst>(U);
+    uint64_t Idx =
+        cast<ConstantInt>(Extract->getIndexOperand())->getZExtValue();
+    Value *LShr = Builder.CreateLShr(ScalarV, Idx * SrcEltSizeInBits);
+    Value *And = Builder.CreateAnd(LShr, EltBitMask);
+    U->replaceAllUsesWith(And);
+  }
+  return true;
+}
+
 /// Try to fold "(or (zext (bitcast X)), (shl (zext (bitcast Y)), C))"
 /// to "(bitcast (concat X, Y))"
 /// where X/Y are bitcasted from i1 mask vectors.
@@ -2617,7 +2693,7 @@ static Value *generateNewInstTree(ArrayRef<InstLane> Item, FixedVectorType *Ty,
                                   const SmallPtrSet<Use *, 4> &IdentityLeafs,
                                   const SmallPtrSet<Use *, 4> &SplatLeafs,
                                   const SmallPtrSet<Use *, 4> &ConcatLeafs,
-                                  IRBuilder<> &Builder,
+                                  IRBuilderBase &Builder,
                                   const TargetTransformInfo *TTI) {
   auto [FrontU, FrontLane] = Item.front();
 
@@ -2975,17 +3051,20 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
                     << "\n");
   LLVM_DEBUG(dbgs() << "  OldCost: " << OldCost << " vs NewCost: " << NewCost
                     << "\n");
+  bool MadeChanges = false;
   if (NewCost < OldCost) {
     Builder.SetInsertPoint(Shuffle);
     Value *NewShuffle = Builder.CreateShuffleVector(
         Shuffle->getOperand(0), Shuffle->getOperand(1), ConcatMask);
     LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
     replaceValue(*Shuffle, *NewShuffle);
+    MadeChanges = true;
   }
 
   // See if we can re-use foldSelectShuffle, getting it to reduce the size of
   // the shuffle into a nicer order, as it can ignore the order of the shuffles.
-  return foldSelectShuffle(*Shuffle, true);
+  MadeChanges |= foldSelectShuffle(*Shuffle, true);
+  return MadeChanges;
 }
 
 /// Determine if its more efficient to fold:
@@ -3655,6 +3734,7 @@ bool VectorCombine::run() {
     if (IsVectorType) {
       MadeChange |= scalarizeOpOrCmp(I);
       MadeChange |= scalarizeLoadExtract(I);
+      MadeChange |= scalarizeExtExtract(I);
       MadeChange |= scalarizeVPIntrinsic(I);
       MadeChange |= foldInterleaveIntrinsics(I);
     }
diff --git a/llvm/test/Analysis/BasicAA/cs-cs-arm.ll b/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
index 43e7be2ee20ee..ac5e0db001110 100644
--- a/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
+++ b/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
@@ -24,7 +24,7 @@ entry:
 ; CHECK: Just Ref:  Ptr: i8* %p        <->  %a = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:  Ptr: i8* %q        <->  %a = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
-; CHECK: Both ModRef:  Ptr: i8* %q     <->  call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
+; CHECK: Just Mod:  Ptr: i8* %q     <->  call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
 ; CHECK: Just Ref:  Ptr: i8* %p        <->  %b = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:  Ptr: i8* %q        <->  %b = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:   %a = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16) #{{[0-9]+}} <->   call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
diff --git a/llvm/test/Analysis/BasicAA/intrinsics-arm.ll b/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
index 7772cca2777f0..b7c64f18cdfab 100644
--- a/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
+++ b/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
@@ -27,5 +27,5 @@ declare <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr, i32) nounwind readonly
 declare void @llvm.arm.neon.vst1.p0.v8i16(ptr, <8 x i16>, i32) nounwind
 
 ; CHECK: attributes #0 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
-; CHECK: attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+; CHECK: attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
 ; CHECK: attributes [[ATTR]] = { nounwind }
diff --git a/llvm/test/Analysis/CostModel/AArch64/ldexp.ll b/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
new file mode 100644
index 0000000000000..8cc8657468930
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @ldexp() {
+; CHECK-BASE-LABEL: 'ldexp'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %1 = call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %2 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:46 CodeSize:7 Lat:46 SizeLat:46 for: %3 = call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:92 CodeSize:14 Lat:92 SizeLat:92 for: %4 = call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:184 CodeSize:28 Lat:184 SizeLat:184 for: %5 = call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %6 = call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %7 = call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:44 CodeSize:6 Lat:44 SizeLat:44 for: %8 = call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:88 CodeSize:12 Lat:88 SizeLat:88 for: %9 = call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:176 CodeSize:24 Lat:176 SizeLat:176 for: %10 = call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE-LABEL: 'ldexp'
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %1 = call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %2 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:14 SizeLat:14 for: %3 = call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:22 Lat:28 SizeLat:28 for: %4 = call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:44 Lat:56 SizeLat:56 for: %5 = call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %6 = call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %7 = call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:10 Lat:12 SizeLat:12 for: %8 = call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:20 Lat:24 SizeLat:24 for: %9 = call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:40 Lat:48 SizeLat:48 for: %10 = call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+  call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+  call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+  call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+  call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+  call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+  call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+  call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+  call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+  call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_fp16() {
+; CHECK-BASE-LABEL: 'ldexp_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = call half @llvm.ldexp.f16.i32(half poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %2 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:7 Lat:10 SizeLat:10 for: %3 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:15 Lat:22 SizeLat:22 for: %4 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:44 CodeSize:30 Lat:44 SizeLat:44 for: %5 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE-LABEL: 'ldexp_fp16'
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %1 = call half @llvm.ldexp.f16.i32(half poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %2 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:14 SizeLat:14 for: %3 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:23 Lat:30 SizeLat:30 for: %4 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:60 CodeSize:46 Lat:60 SizeLat:60 for: %5 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call half @llvm.ldexp.f16.i32(half poison, i32 poison)
+  call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> poison, <2 x i32> poison)
+  call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> poison, <4 x i32> poison)
+  call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> poison, <8 x i32> poison)
+  call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> poison, <16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_bf16() {
+; CHECK-BASE-LABEL: 'ldexp_bf16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:46 CodeSize:7 Lat:46 SizeLat:46 for: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:94 CodeSize:15 Lat:94 SizeLat:94 for: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:188 CodeSize:30 Lat:188 SizeLat:188 for: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE-LABEL: 'ldexp_bf16'
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:14 SizeLat:14 for: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:23 Lat:30 SizeLat:30 for: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:60 CodeSize:46 Lat:60 SizeLat:60 for: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+  call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+  call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+  call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+  call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+  ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/AArch64/lrint.ll b/llvm/test/Analysis/CostModel/AArch64/lrint.ll
new file mode 100644
index 0000000000000..70fb60adee89a
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/lrint.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @lrint() {
+; CHECK-LABEL: 'lrint'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lrint.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.lrint.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lrint(float poison)
+  call <2 x i64> @llvm.lrint(<2 x float> poison)
+  call <4 x i64> @llvm.lrint(<4 x float> poison)
+  call <8 x i64> @llvm.lrint(<8 x float> poison)
+  call <16 x i64> @llvm.lrint(<16 x float> poison)
+  call i64 @llvm.lrint(double poison)
+  call <2 x i64> @llvm.lrint(<2 x double> poison)
+  call <4 x i64> @llvm.lrint(<4 x double> poison)
+  call <8 x i64> @llvm.lrint(<8 x double> poison)
+  call <16 x i64> @llvm.lrint(<16 x double> poison)
+  ret void
+}
+
+define void @lrint_fp16() {
+; CHECK-LABEL: 'lrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lrint.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lrint(half poison)
+  call <2 x i64> @llvm.lrint(<2 x half> poison)
+  call <4 x i64> @llvm.lrint(<4 x half> poison)
+  call <8 x i64> @llvm.lrint(<8 x half> poison)
+  call <16 x i64> @llvm.lrint(<16 x half> poison)
+  ret void
+}
+
+define void @lrint_bf16() {
+; CHECK-LABEL: 'lrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lrint.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lrint.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lrint.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lrint.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lrint.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lrint(bfloat poison)
+  call <2 x i64> @llvm.lrint(<2 x bfloat> poison)
+  call <4 x i64> @llvm.lrint(<4 x bfloat> poison)
+  call <8 x i64> @llvm.lrint(<8 x bfloat> poison)
+  call <16 x i64> @llvm.lrint(<16 x bfloat> poison)
+  ret void
+}
+
+define void @llrint() {
+; CHECK-LABEL: 'llrint'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llrint.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.llrint.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llrint(float poison)
+  call <2 x i64> @llvm.llrint(<2 x float> poison)
+  call <4 x i64> @llvm.llrint(<4 x float> poison)
+  call <8 x i64> @llvm.llrint(<8 x float> poison)
+  call <16 x i64> @llvm.llrint(<16 x float> poison)
+  call i64 @llvm.llrint(double poison)
+  call <2 x i64> @llvm.llrint(<2 x double> poison)
+  call <4 x i64> @llvm.llrint(<4 x double> poison)
+  call <8 x i64> @llvm.llrint(<8 x double> poison)
+  call <16 x i64> @llvm.llrint(<16 x double> poison)
+  ret void
+}
+
+define void @llrint_fp16() {
+; CHECK-LABEL: 'llrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llrint.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llrint(half poison)
+  call <2 x i64> @llvm.llrint(<2 x half> poison)
+  call <4 x i64> @llvm.llrint(<4 x half> poison)
+  call <8 x i64> @llvm.llrint(<8 x half> poison)
+  call <16 x i64> @llvm.llrint(<16 x half> poison)
+  ret void
+}
+
+define void @llrint_bf16() {
+; CHECK-LABEL: 'llrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llrint.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llrint.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llrint.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llrint.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llrint.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llrint(bfloat poison)
+  call <2 x i64> @llvm.llrint(<2 x bfloat> poison)
+  call <4 x i64> @llvm.llrint(<4 x bfloat> poison)
+  call <8 x i64> @llvm.llrint(<8 x bfloat> poison)
+  call <16 x i64> @llvm.llrint(<16 x bfloat> poison)
+  ret void
+}
+
+
+define void @lround() {
+; CHECK-LABEL: 'lround'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.lround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.lround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.lround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.lround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lround(float poison)
+  call <2 x i64> @llvm.lround(<2 x float> poison)
+  call <4 x i64> @llvm.lround(<4 x float> poison)
+  call <8 x i64> @llvm.lround(<8 x float> poison)
+  call <16 x i64> @llvm.lround(<16 x float> poison)
+  call i64 @llvm.lround(double poison)
+  call <2 x i64> @llvm.lround(<2 x double> poison)
+  call <4 x i64> @llvm.lround(<4 x double> poison)
+  call <8 x i64> @llvm.lround(<8 x double> poison)
+  call <16 x i64> @llvm.lround(<16 x double> poison)
+  ret void
+}
+
+define void @lround_fp16() {
+; CHECK-LABEL: 'lround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lround.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lround.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lround.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lround.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lround.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lround(half poison)
+  call <2 x i64> @llvm.lround(<2 x half> poison)
+  call <4 x i64> @llvm.lround(<4 x half> poison)
+  call <8 x i64> @llvm.lround(<8 x half> poison)
+  call <16 x i64> @llvm.lround(<16 x half> poison)
+  ret void
+}
+
+define void @lround_bf16() {
+; CHECK-LABEL: 'lround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lround(bfloat poison)
+  call <2 x i64> @llvm.lround(<2 x bfloat> poison)
+  call <4 x i64> @llvm.lround(<4 x bfloat> poison)
+  call <8 x i64> @llvm.lround(<8 x bfloat> poison)
+  call <16 x i64> @llvm.lround(<16 x bfloat> poison)
+  ret void
+}
+
+define void @llround() {
+; CHECK-LABEL: 'llround'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.llround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.llround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llround(float poison)
+  call <2 x i64> @llvm.llround(<2 x float> poison)
+  call <4 x i64> @llvm.llround(<4 x float> poison)
+  call <8 x i64> @llvm.llround(<8 x float> poison)
+  call <16 x i64> @llvm.llround(<16 x float> poison)
+  call i64 @llvm.llround(double poison)
+  call <2 x i64> @llvm.llround(<2 x double> poison)
+  call <4 x i64> @llvm.llround(<4 x double> poison)
+  call <8 x i64> @llvm.llround(<8 x double> poison)
+  call <16 x i64> @llvm.llround(<16 x double> poison)
+  ret void
+}
+
+define void @llround_fp16() {
+; CHECK-LABEL: 'llround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llround.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llround.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llround.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llround.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llround.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llround(half poison)
+  call <2 x i64> @llvm.llround(<2 x half> poison)
+  call <4 x i64> @llvm.llround(<4 x half> poison)
+  call <8 x i64> @llvm.llround(<8 x half> poison)
+  call <16 x i64> @llvm.llround(<16 x half> poison)
+  ret void
+}
+
+define void @llround_bf16() {
+; CHECK-LABEL: 'llround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llround(bfloat poison)
+  call <2 x i64> @llvm.llround(<2 x bfloat> poison)
+  call <4 x i64> @llvm.llround(<4 x bfloat> poison)
+  call <8 x i64> @llvm.llround(<8 x bfloat> poison)
+  call <16 x i64> @llvm.llround(<16 x bfloat> poison)
+  ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-FP16: {{.*}}
+; CHECK-NOFP16: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll b/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll
index 790f49f1d3b82..8b94cefbad63b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll
@@ -49,5 +49,53 @@ define void @dup_within_each_segment_512b() #1 {
   ret void
 }
 
+define void @dup_whole_segment_256b() #0 {
+; CHECK-LABEL: 'dup_whole_segment_256b'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3, i32 poison, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                                                                             i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                               i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                            i32 0, i32 1, i32 2, i32 3>
+  %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1,
+                                                                                       i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3,
+                                                                                            i32 poison, i32 1, i32 2, i32 3>
+  ret void
+}
+
+define void @dup_whole_segment_512b() #1 {
+; CHECK-LABEL: 'dup_whole_segment_512b'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3, i32 poison, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                                                                             i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                               i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                            i32 0, i32 1, i32 2, i32 3>
+  %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1,
+                                                                                       i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3,
+                                                                                            i32 poison, i32 1, i32 2, i32 3>
+  ret void
+}
+
 attributes #0 = { noinline vscale_range(2,2) }
 attributes #1 = { noinline vscale_range(4,4) }
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll b/llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll
new file mode 100644
index 0000000000000..037958cf52e26
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @ldexp() {
+; CHECK-LABEL: 'ldexp'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+  call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_fp16() {
+; CHECK-LABEL: 'ldexp_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x half> @llvm.ldexp.nxv2f16.nxv2i32(<vscale x 2 x half> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x half> @llvm.ldexp.nxv4f16.nxv4i32(<vscale x 4 x half> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x half> @llvm.ldexp.nxv8f16.nxv8i32(<vscale x 8 x half> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x half> @llvm.ldexp.nxv16f16.nxv16i32(<vscale x 16 x half> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x half> @llvm.ldexp.nxv2f16.nxv2i32(<vscale x 2 x half> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x half> @llvm.ldexp.nxv4f16.nxv4i32(<vscale x 4 x half> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x half> @llvm.ldexp.nxv8f16.nxv8i32(<vscale x 8 x half> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x half> @llvm.ldexp.nxv16f16.nxv16i32(<vscale x 16 x half> poison, <vscale x 16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_bf16() {
+; CHECK-LABEL: 'ldexp_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll b/llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll
new file mode 100644
index 0000000000000..540378bbf421a
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll
@@ -0,0 +1,218 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve2 %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @lrint() {
+; CHECK-LABEL: 'lrint'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @lrint_fp16() {
+; CHECK-LABEL: 'lrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @lrint_bf16() {
+; CHECK-LABEL: 'lrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x bfloat> poison)
+  ret void
+}
+
+define void @llrint() {
+; CHECK-LABEL: 'llrint'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @llrint_fp16() {
+; CHECK-LABEL: 'llrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @llrint_bf16() {
+; CHECK-LABEL: 'llrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x bfloat> poison)
+  ret void
+}
+
+
+define void @lround() {
+; CHECK-LABEL: 'lround'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @lround_fp16() {
+; CHECK-LABEL: 'lround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @lround_bf16() {
+; CHECK-LABEL: 'lround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x bfloat> poison)
+  ret void
+}
+
+define void @llround() {
+; CHECK-LABEL: 'llround'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @llround_fp16() {
+; CHECK-LABEL: 'llround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @llround_bf16() {
+; CHECK-LABEL: 'llround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x bfloat> poison)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll b/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll
index 2b1b5906ad017..260cf696270cd 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll
@@ -13,45 +13,45 @@ define void @ldexp_f16_i32() {
 ; GFX7-LABEL: 'ldexp_f16_i32'
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX8PLUS-LABEL: 'ldexp_f16_i32'
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX7-SIZE-LABEL: 'ldexp_f16_i32'
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; GFX8PLUS-SIZE-LABEL: 'ldexp_f16_i32'
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
@@ -69,45 +69,45 @@ define void @ldexp_f16_i16() {
 ; GFX7-LABEL: 'ldexp_f16_i16'
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX8PLUS-LABEL: 'ldexp_f16_i16'
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX7-SIZE-LABEL: 'ldexp_f16_i16'
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; GFX8PLUS-SIZE-LABEL: 'ldexp_f16_i16'
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
@@ -125,12 +125,12 @@ define void @ldexp_bf16() {
 ; GFX7-LABEL: 'ldexp_bf16'
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bf16 = call bfloat @llvm.ldexp.bf16.i32(bfloat undef, i32 undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2bf16 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> undef, <2 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4bf16 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> undef, <4 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8bf16 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> undef, <8 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16bf16 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> undef, <16 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX8PLUS-LABEL: 'ldexp_bf16'
@@ -147,12 +147,12 @@ define void @ldexp_bf16() {
 ; GFX7-SIZE-LABEL: 'ldexp_bf16'
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bf16 = call bfloat @llvm.ldexp.bf16.i32(bfloat undef, i32 undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2bf16 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> undef, <2 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4bf16 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> undef, <4 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8bf16 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> undef, <8 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16bf16 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> undef, <16 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; GFX8PLUS-SIZE-LABEL: 'ldexp_bf16'
diff --git a/llvm/test/Analysis/CostModel/RISCV/abs.ll b/llvm/test/Analysis/CostModel/RISCV/abs.ll
index 7252716af8605..b1f93f3811580 100644
--- a/llvm/test/Analysis/CostModel/RISCV/abs.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/abs.ll
@@ -3,44 +3,6 @@
 ; Check that we don't crash querying costs when vectors are not enabled.
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64
 
-declare <2 x i64>  @llvm.abs.v2i64(<2 x i64>, i1)
-declare <4 x i64>  @llvm.abs.v4i64(<4 x i64>, i1)
-declare <8 x i64>  @llvm.abs.v8i64(<8 x i64>, i1)
-declare <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64>, i1)
-declare <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64>, i1)
-declare <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64>, i1)
-
-declare <2 x i32>  @llvm.abs.v2i32(<2 x i32>, i1)
-declare <4 x i32>  @llvm.abs.v4i32(<4 x i32>, i1)
-declare <8 x i32>  @llvm.abs.v8i32(<8 x i32>, i1)
-declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1)
-declare <vscale x 2 x i32>  @llvm.abs.nxv2i32(<vscale x 2 x i32>, i1)
-declare <vscale x 4 x i32>  @llvm.abs.nxv4i32(<vscale x 4 x i32>, i1)
-declare <vscale x 8 x i32>  @llvm.abs.nxv8i32(<vscale x 8 x i32>, i1)
-declare <vscale x 16 x i32> @llvm.abs.nxv16i32(<vscale x 16 x i32>, i1)
-
-declare <2 x i16>  @llvm.abs.v2i16(<2 x i16>, i1)
-declare <4 x i16>  @llvm.abs.v4i16(<4 x i16>, i1)
-declare <8 x i16>  @llvm.abs.v8i16(<8 x i16>, i1)
-declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1)
-declare <32 x i16> @llvm.abs.v32i16(<32 x i16>, i1)
-declare <vscale x 2 x i16>  @llvm.abs.nxv2i16(<vscale x 2 x i16>, i1)
-declare <vscale x 4 x i16>  @llvm.abs.nxv4i16(<vscale x 4 x i16>, i1)
-declare <vscale x 8 x i16>  @llvm.abs.nxv8i16(<vscale x 8 x i16>, i1)
-declare <vscale x 16 x i16> @llvm.abs.nxv16i16(<vscale x 16 x i16>, i1)
-declare <vscale x 32 x i16> @llvm.abs.nxv32i16(<vscale x 32 x i16>, i1)
-
-declare <2 x i8>   @llvm.abs.v2i8(<2 x i8>, i1)
-declare <4 x i8>   @llvm.abs.v4i8(<4 x i8>, i1)
-declare <8 x i8>   @llvm.abs.v8i8(<8 x i8>, i1)
-declare <16 x i8>  @llvm.abs.v16i8(<16 x i8>, i1)
-declare <32 x i8>  @llvm.abs.v32i8(<32 x i8>, i1)
-declare <64 x i8>  @llvm.abs.v64i8(<64 x i8>, i1)
-declare <vscale x 8 x i8>  @llvm.abs.nxv8i8(<vscale x 8 x i8>, i1)
-declare <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8>, i1)
-declare <vscale x 32 x i8> @llvm.abs.nxv32i8(<vscale x 32 x i8>, i1)
-declare <vscale x 64 x i8> @llvm.abs.nxv64i8(<vscale x 64 x i8>, i1)
-
 define i32 @abs(i32 %arg) {
 ; CHECK-LABEL: 'abs'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
diff --git a/llvm/test/Analysis/CostModel/RISCV/active_lane_mask.ll b/llvm/test/Analysis/CostModel/RISCV/active_lane_mask.ll
index 7ebe14d98b21b..be2bb7374a318 100644
--- a/llvm/test/Analysis/CostModel/RISCV/active_lane_mask.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/active_lane_mask.ll
@@ -57,26 +57,3 @@ define void @get_lane_mask() {
 
   ret void
 }
-
-declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64, i64)
-declare <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64, i64)
-declare <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64, i64)
-declare <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64, i64)
-declare <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i64(i64, i64)
-declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32, i32)
-declare <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32, i32)
-declare <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32, i32)
-declare <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32, i32)
-declare <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i32(i32, i32)
-declare <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64, i64)
-declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16, i16)
-declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64, i64)
-declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64, i64)
-declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64, i64)
-declare <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64, i64)
-declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
-declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
-declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
-declare <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32, i32)
-declare <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64, i64)
-declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16, i16)
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
index 852c237cf2501..6c974afb76031 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
@@ -1577,32 +1577,3 @@ define void @fmuladd_f16() {
   call <vscale x 16 x half> @llvm.fmuladd.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
   ret void
 }
-
-declare <1 x bfloat> @llvm.vp.frem.v1f16(<1 x bfloat>, <1 x bfloat>, <1 x i1>, i32)
-declare <2 x bfloat> @llvm.vp.frem.v2f16(<2 x bfloat>, <2 x bfloat>, <2 x i1>, i32)
-declare <4 x bfloat> @llvm.vp.frem.v4f16(<4 x bfloat>, <4 x bfloat>, <4 x i1>, i32)
-declare <8 x bfloat> @llvm.vp.frem.v8f16(<8 x bfloat>, <8 x bfloat>, <8 x i1>, i32)
-declare <16 x bfloat> @llvm.vp.frem.v16f16(<16 x bfloat>, <16 x bfloat>, <16 x i1>, i32)
-declare <1 x float> @llvm.vp.frem.v1f32(<1 x float>, <1 x float>, <1 x i1>, i32)
-declare <2 x float> @llvm.vp.frem.v2f32(<2 x float>, <2 x float>, <2 x i1>, i32)
-declare <4 x float> @llvm.vp.frem.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32)
-declare <8 x float> @llvm.vp.frem.v8f32(<8 x float>, <8 x float>, <8 x i1>, i32)
-declare <16 x float> @llvm.vp.frem.v16f32(<16 x float>, <16 x float>, <16 x i1>, i32)
-declare <1 x double> @llvm.vp.frem.v1f64(<1 x double>, <1 x double>, <1 x i1>, i32)
-declare <2 x double> @llvm.vp.frem.v2f64(<2 x double>, <2 x double>, <2 x i1>, i32)
-declare <4 x double> @llvm.vp.frem.v4f64(<4 x double>, <4 x double>, <4 x i1>, i32)
-declare <8 x double> @llvm.vp.frem.v8f64(<8 x double>, <8 x double>, <8 x i1>, i32)
-declare <vscale x 1 x bfloat> @llvm.vp.frem.nxv1f16(<vscale x 1 x bfloat>, <vscale x 1 x bfloat>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x bfloat> @llvm.vp.frem.nxv2f16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x bfloat> @llvm.vp.frem.nxv4f16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x bfloat> @llvm.vp.frem.nxv8f16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x bfloat> @llvm.vp.frem.nxv16f16(<vscale x 16 x bfloat>, <vscale x 16 x bfloat>, <vscale x 16 x i1>, i32)
-declare <vscale x 1 x float> @llvm.vp.frem.nxv1f32(<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x float> @llvm.vp.frem.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x float> @llvm.vp.frem.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x float> @llvm.vp.frem.nxv8f32(<vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x float> @llvm.vp.frem.nxv16f32(<vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x i1>, i32)
-declare <vscale x 1 x double> @llvm.vp.frem.nxv1f64(<vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x double> @llvm.vp.frem.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x double> @llvm.vp.frem.nxv4f64(<vscale x 4 x double>, <vscale x 4 x double>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x double> @llvm.vp.frem.nxv8f64(<vscale x 8 x double>, <vscale x 8 x double>, <vscale x 8 x i1>, i32)
diff --git a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
index 6d3dfc3061388..094d73ddd0581 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
@@ -131,19 +131,3 @@ define void @foo_vscale_range_2_16() vscale_range(2,16) {
 
   ret void
 }
-
-declare i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1>, i1)
-declare i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1>, i1)
-declare i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1>, i1)
-declare i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1>, i1)
-declare i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1>, i1)
-declare i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1>, i1)
-declare i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1>, i1)
-
-declare i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1>, i1)
-declare i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1>, i1)
-declare i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1>, i1)
-declare i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1>, i1)
-declare i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1>, i1)
-declare i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1>, i1)
-declare i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1>, i1)
diff --git a/llvm/test/Analysis/CostModel/RISCV/exp.ll b/llvm/test/Analysis/CostModel/RISCV/exp.ll
new file mode 100644
index 0000000000000..55c3f5554eff0
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/exp.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh,+zvfbfmin %s | FileCheck %s
+
+define void @ldexp() {
+; CHECK-LABEL: 'ldexp'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x bfloat> @llvm.ldexp.nxv1bf16.nxv1i32(<vscale x 1 x bfloat> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %11 = call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %12 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %13 = call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %14 = call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %15 = call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %16 = call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i32(<vscale x 1 x float> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %17 = call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %18 = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %19 = call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %20 = call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %21 = call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %22 = call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %23 = call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %24 = call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %25 = call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %26 = call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i32(<vscale x 1 x double> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %27 = call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %28 = call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %29 = call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %30 = call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %31 = call float @llvm.ldexp.f32.i64(float poison, i64 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %32 = call <2 x float> @llvm.ldexp.v2f32.v2i64(<2 x float> poison, <2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %33 = call <4 x float> @llvm.ldexp.v4f32.v4i64(<4 x float> poison, <4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %34 = call <8 x float> @llvm.ldexp.v8f32.v8i64(<8 x float> poison, <8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %35 = call <16 x float> @llvm.ldexp.v16f32.v16i64(<16 x float> poison, <16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %36 = call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i64(<vscale x 1 x float> poison, <vscale x 1 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %37 = call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i64(<vscale x 2 x float> poison, <vscale x 2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %38 = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i64(<vscale x 4 x float> poison, <vscale x 4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %39 = call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i64(<vscale x 8 x float> poison, <vscale x 8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %40 = call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i64(<vscale x 16 x float> poison, <vscale x 16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %41 = call double @llvm.ldexp.f64.i64(double poison, i64 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %42 = call <2 x double> @llvm.ldexp.v2f64.v2i64(<2 x double> poison, <2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %43 = call <4 x double> @llvm.ldexp.v4f64.v4i64(<4 x double> poison, <4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %44 = call <8 x double> @llvm.ldexp.v8f64.v8i64(<8 x double> poison, <8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %45 = call <16 x double> @llvm.ldexp.v16f64.v16i64(<16 x double> poison, <16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %46 = call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i64(<vscale x 1 x double> poison, <vscale x 1 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %47 = call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i64(<vscale x 2 x double> poison, <vscale x 2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %48 = call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i64(<vscale x 4 x double> poison, <vscale x 4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %49 = call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i64(<vscale x 8 x double> poison, <vscale x 8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %50 = call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i64(<vscale x 16 x double> poison, <vscale x 16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+  call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+  call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+  call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+  call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+  call <vscale x 1 x bfloat> @llvm.ldexp.nxv1bf16.nxv1i32(<vscale x 1 x bfloat> poison, <vscale x 1 x i32> poison)
+  call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+  call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+  call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+  call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+  call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+  call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+  call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i32(<vscale x 1 x float> poison, <vscale x 1 x i32> poison)
+  call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+  call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+  call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+  call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+  call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+  call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+  call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i32(<vscale x 1 x double> poison, <vscale x 1 x i32> poison)
+  call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+  call float @llvm.ldexp.f32.i64(float poison, i64 poison)
+  call <2 x float> @llvm.ldexp.v2f32.v2i64(<2 x float> poison, <2 x i64> poison)
+  call <4 x float> @llvm.ldexp.v4f32.v4i64(<4 x float> poison, <4 x i64> poison)
+  call <8 x float> @llvm.ldexp.v8f32.v8i64(<8 x float> poison, <8 x i64> poison)
+  call <16 x float> @llvm.ldexp.v16f32.v16i64(<16 x float> poison, <16 x i64> poison)
+  call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i64(<vscale x 1 x float> poison, <vscale x 1 x i64> poison)
+  call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i64(<vscale x 2 x float> poison, <vscale x 2 x i64> poison)
+  call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i64(<vscale x 4 x float> poison, <vscale x 4 x i64> poison)
+  call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i64(<vscale x 8 x float> poison, <vscale x 8 x i64> poison)
+  call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i64(<vscale x 16 x float> poison, <vscale x 16 x i64> poison)
+  call double @llvm.ldexp.f64.i64(double poison, i64 poison)
+  call <2 x double> @llvm.ldexp.v2f64.v2i64(<2 x double> poison, <2 x i64> poison)
+  call <4 x double> @llvm.ldexp.v4f64.v4i64(<4 x double> poison, <4 x i64> poison)
+  call <8 x double> @llvm.ldexp.v8f64.v8i64(<8 x double> poison, <8 x i64> poison)
+  call <16 x double> @llvm.ldexp.v16f64.v16i64(<16 x double> poison, <16 x i64> poison)
+  call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i64(<vscale x 1 x double> poison, <vscale x 1 x i64> poison)
+  call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i64(<vscale x 2 x double> poison, <vscale x 2 x i64> poison)
+  call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i64(<vscale x 4 x double> poison, <vscale x 4 x i64> poison)
+  call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i64(<vscale x 8 x double> poison, <vscale x 8 x i64> poison)
+  call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i64(<vscale x 16 x double> poison, <vscale x 16 x i64> poison)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll
index ae913456e344c..6eec7ed2f98ec 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll
@@ -185,46 +185,3 @@ define i32 @masked_gather() {
   ret i32 0
 }
 
-declare <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x double>)
-declare <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x double>)
-declare <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x double>)
-declare <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x double>)
-
-declare <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x float>)
-declare <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x float>)
-declare <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x float>)
-declare <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x float>)
-declare <1 x float> @llvm.masked.gather.v1f32.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x float>)
-
-declare <32 x half> @llvm.masked.gather.v32f16.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x half>)
-declare <16 x half> @llvm.masked.gather.v16f16.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x half>)
-declare <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x half>)
-declare <4 x half> @llvm.masked.gather.v4f16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x half>)
-declare <2 x half> @llvm.masked.gather.v2f16.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x half>)
-declare <1 x half> @llvm.masked.gather.v1f16.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x half>)
-
-declare <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i64>)
-declare <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i64>)
-declare <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i64>)
-declare <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x i64>)
-
-declare <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i32>)
-declare <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i32>)
-declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)
-declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)
-declare <1 x i32> @llvm.masked.gather.v1i32.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x i32>)
-
-declare <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x i16>)
-declare <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i16>)
-declare <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i16>)
-declare <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i16>)
-declare <2 x i16> @llvm.masked.gather.v2i16.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i16>)
-declare <1 x i16> @llvm.masked.gather.v1i16.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x i16>)
-
-declare <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr>, i32, <64 x i1>, <64 x i8>)
-declare <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x i8>)
-declare <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i8>)
-declare <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i8>)
-declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i8>)
-declare <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i8>)
-declare <1 x i8> @llvm.masked.gather.v1i8.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x i8>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll
index 9158c2c829135..338683e12654c 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll
@@ -185,46 +185,3 @@ define i32 @masked_scatter() {
   ret i32 0
 }
 
-declare void @llvm.masked.scatter.v8f64.v8p0(<8 x double>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4f64.v4p0(<4 x double>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1f64.v1p0(<1 x double>, <1 x ptr>, i32, <1 x i1>)
-
-declare void @llvm.masked.scatter.v16f32.v16p0(<16 x float>, <16 x ptr>, i32, <16 x i1>)
-declare void @llvm.masked.scatter.v8f32.v8p0(<8 x float>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4f32.v4p0(<4 x float>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2f32.v2p0(<2 x float>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1f32.v1p0(<1 x float>, <1 x ptr>, i32, <1 x i1>)
-
-declare void @llvm.masked.scatter.v32f16.v32p0(<32 x half>, <32 x ptr>, i32, <32 x i1>)
-declare void @llvm.masked.scatter.v16f16.v16p0(<16 x half>, <16 x ptr>, i32, <16 x i1>)
-declare void @llvm.masked.scatter.v8f16.v8p0(<8 x half>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4f16.v4p0(<4 x half>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2f16.v2p0(<2 x half>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1f16.v1p0(<1 x half>, <1 x ptr>, i32, <1 x i1>)
-
-declare void @llvm.masked.scatter.v8i64.v8p0(<8 x i64>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4i64.v4p0(<4 x i64>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2i64.v2p0(<2 x i64>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1i64.v1p0(<1 x i64>, <1 x ptr>, i32, <1 x i1>)
-
-declare void @llvm.masked.scatter.v16i32.v16p0(<16 x i32>, <16 x ptr>, i32, <16 x i1>)
-declare void @llvm.masked.scatter.v8i32.v8p0(<8 x i32>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2i32.v2p0(<2 x i32>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1i32.v1p0(<1 x i32>, <1 x ptr>, i32, <1 x i1>)
-
-declare void @llvm.masked.scatter.v32i16.v32p0(<32 x i16>, <32 x ptr>, i32, <32 x i1>)
-declare void @llvm.masked.scatter.v16i16.v16p0(<16 x i16>, <16 x ptr>, i32, <16 x i1>)
-declare void @llvm.masked.scatter.v8i16.v8p0(<8 x i16>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4i16.v4p0(<4 x i16>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2i16.v2p0(<2 x i16>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1i16.v1p0(<1 x i16>, <1 x ptr>, i32, <1 x i1>)
-
-declare void @llvm.masked.scatter.v64i8.v64p0(<64 x i8>, <64 x ptr>, i32, <64 x i1>)
-declare void @llvm.masked.scatter.v32i8.v32p0(<32 x i8>, <32 x ptr>, i32, <32 x i1>)
-declare void @llvm.masked.scatter.v16i8.v16p0(<16 x i8>, <16 x ptr>, i32, <16 x i1>)
-declare void @llvm.masked.scatter.v8i8.v8p0(<8 x i8>, <8 x ptr>, i32, <8 x i1>)
-declare void @llvm.masked.scatter.v4i8.v4p0(<4 x i8>, <4 x ptr>, i32, <4 x i1>)
-declare void @llvm.masked.scatter.v2i8.v2p0(<2 x i8>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v1i8.v1p0(<1 x i8>, <1 x ptr>, i32, <1 x i1>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fround.ll b/llvm/test/Analysis/CostModel/RISCV/fround.ll
index a0818d487d151..94c09389cadec 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fround.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fround.ll
@@ -606,6 +606,200 @@ define void @llrint_fp16() {
   ret void
 }
 
+define void @lround() {
+; CHECK-LABEL: 'lround'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i32 @llvm.lround.i32.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call <2 x i32> @llvm.lround.v2i32.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = call <4 x i32> @llvm.lround.v4i32.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %4 = call <8 x i32> @llvm.lround.v8i32.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %5 = call <16 x i32> @llvm.lround.v16i32.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1bf16(<vscale x 1 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i32 @llvm.lround.i32.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %13 = call <4 x i32> @llvm.lround.v4i32.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %14 = call <8 x i32> @llvm.lround.v8i32.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %15 = call <16 x i32> @llvm.lround.v16i32.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %16 = call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f32(<vscale x 1 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %17 = call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %18 = call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %19 = call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %20 = call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call i32 @llvm.lround.i32.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = call <2 x i32> @llvm.lround.v2i32.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %23 = call <4 x i32> @llvm.lround.v4i32.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %24 = call <8 x i32> @llvm.lround.v8i32.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %25 = call <16 x i32> @llvm.lround.v16i32.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %26 = call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f64(<vscale x 1 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %27 = call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %28 = call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %29 = call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %30 = call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call i64 @llvm.lround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %32 = call <2 x i64> @llvm.lround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %33 = call <4 x i64> @llvm.lround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %34 = call <8 x i64> @llvm.lround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %35 = call <16 x i64> @llvm.lround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %36 = call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %37 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %38 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %39 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %40 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %41 = call i64 @llvm.lround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %42 = call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %43 = call <4 x i64> @llvm.lround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %44 = call <8 x i64> @llvm.lround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %45 = call <16 x i64> @llvm.lround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %46 = call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %47 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %48 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %49 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %50 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %51 = call i64 @llvm.lround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %52 = call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %53 = call <4 x i64> @llvm.lround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %54 = call <8 x i64> @llvm.lround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %55 = call <16 x i64> @llvm.lround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %56 = call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %57 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %58 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %59 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %60 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i32 @llvm.lround.i32.bf16(bfloat poison)
+  call <2 x i32> @llvm.lround.v2i32.v2bf16(<2 x bfloat> poison)
+  call <4 x i32> @llvm.lround.v4i32.v4bf16(<4 x bfloat> poison)
+  call <8 x i32> @llvm.lround.v8i32.v8bf16(<8 x bfloat> poison)
+  call <16 x i32> @llvm.lround.v16i32.v16bf16(<16 x bfloat> poison)
+  call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1bf16(<vscale x 1 x bfloat> poison)
+  call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2bf16(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4bf16(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8bf16(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16bf16(<vscale x 16 x bfloat> poison)
+  call i32 @llvm.lround.i32.f32(float poison)
+  call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> poison)
+  call <4 x i32> @llvm.lround.v4i32.v4f32(<4 x float> poison)
+  call <8 x i32> @llvm.lround.v8i32.v8f32(<8 x float> poison)
+  call <16 x i32> @llvm.lround.v16i32.v16f32(<16 x float> poison)
+  call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f32(<vscale x 1 x float> poison)
+  call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f32(<vscale x 2 x float> poison)
+  call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f32(<vscale x 4 x float> poison)
+  call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f32(<vscale x 8 x float> poison)
+  call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f32(<vscale x 16 x float> poison)
+  call i32 @llvm.lround.i32.f64(double poison)
+  call <2 x i32> @llvm.lround.v2i32.v2f64(<2 x double> poison)
+  call <4 x i32> @llvm.lround.v4i32.v4f64(<4 x double> poison)
+  call <8 x i32> @llvm.lround.v8i32.v8f64(<8 x double> poison)
+  call <16 x i32> @llvm.lround.v16i32.v16f64(<16 x double> poison)
+  call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f64(<vscale x 1 x double> poison)
+  call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f64(<vscale x 2 x double> poison)
+  call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f64(<vscale x 4 x double> poison)
+  call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f64(<vscale x 8 x double> poison)
+  call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f64(<vscale x 16 x double> poison)
+  call i64 @llvm.lround.i64.bf16(bfloat poison)
+  call <2 x i64> @llvm.lround.v2i64.v2bf16(<2 x bfloat> poison)
+  call <4 x i64> @llvm.lround.v4i64.v4bf16(<4 x bfloat> poison)
+  call <8 x i64> @llvm.lround.v8i64.v8bf16(<8 x bfloat> poison)
+  call <16 x i64> @llvm.lround.v16i64.v16bf16(<16 x bfloat> poison)
+  call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+  call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+  call i64 @llvm.lround.i64.f32(float poison)
+  call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> poison)
+  call <4 x i64> @llvm.lround.v4i64.v4f32(<4 x float> poison)
+  call <8 x i64> @llvm.lround.v8i64.v8f32(<8 x float> poison)
+  call <16 x i64> @llvm.lround.v16i64.v16f32(<16 x float> poison)
+  call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+  call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+  call i64 @llvm.lround.i64.f64(double poison)
+  call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> poison)
+  call <4 x i64> @llvm.lround.v4i64.v4f64(<4 x double> poison)
+  call <8 x i64> @llvm.lround.v8i64.v8f64(<8 x double> poison)
+  call <16 x i64> @llvm.lround.v16i64.v16f64(<16 x double> poison)
+  call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+  call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @llround() {
+; CHECK-LABEL: 'llround'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call <2 x i64> @llvm.llround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %3 = call <4 x i64> @llvm.llround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %4 = call <8 x i64> @llvm.llround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %5 = call <16 x i64> @llvm.llround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.llround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %13 = call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %14 = call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %15 = call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %16 = call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %17 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %18 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %19 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %20 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call i64 @llvm.llround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %23 = call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %24 = call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %25 = call <16 x i64> @llvm.llround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %26 = call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %27 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %28 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %29 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %30 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i64 @llvm.llround.i64.bf16(bfloat poison)
+  call <2 x i64> @llvm.llround.v2i64.v2bf16(<2 x bfloat> poison)
+  call <4 x i64> @llvm.llround.v4i64.v4bf16(<4 x bfloat> poison)
+  call <8 x i64> @llvm.llround.v8i64.v8bf16(<8 x bfloat> poison)
+  call <16 x i64> @llvm.llround.v16i64.v16bf16(<16 x bfloat> poison)
+  call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+  call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+  call i64 @llvm.llround.i64.f32(float poison)
+  call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> poison)
+  call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> poison)
+  call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> poison)
+  call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> poison)
+  call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+  call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+  call i64 @llvm.llround.i64.f64(double poison)
+  call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> poison)
+  call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> poison)
+  call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> poison)
+  call <16 x i64> @llvm.llround.v16i64.v16f64(<16 x double> poison)
+  call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+  call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+  ret void
+}
+
 define void @nearbyint() {
 ; CHECK-LABEL: 'nearbyint'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call bfloat @llvm.nearbyint.bf16(bfloat undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/gep.ll b/llvm/test/Analysis/CostModel/RISCV/gep.ll
index 99c74c4635fc7..f8c370050eaee 100644
--- a/llvm/test/Analysis/CostModel/RISCV/gep.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/gep.ll
@@ -396,14 +396,3 @@ define void @foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
   ret void
 }
 
-declare <2 x i8> @llvm.masked.load.v2i8.p0(ptr, i32, <2 x i1>, <2 x i8>)
-declare <2 x i8> @llvm.masked.gather.v2i8.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i8>)
-declare <2 x i8> @llvm.masked.expandload.v2i8(ptr, <2 x i1>, <2 x i8>)
-declare <2 x i8> @llvm.vp.load.v2i8.p0(ptr, <2 x i1>, i32)
-declare <2 x i8> @llvm.experimental.vp.strided.load.v2i8.i64(ptr, i64, <2 x i1>, i32)
-
-declare void @llvm.masked.store.v2i8.p0(<2 x i8>, ptr, i32, <2 x i1>)
-declare void @llvm.masked.scatter.v2i8.v2p0(<2 x i8>, <2 x ptr>, i32, <2 x i1>)
-declare void @llvm.masked.compressstore.v2i8(<2 x i8>, ptr, <2 x i1>)
-declare void @llvm.vp.store.v2i8.p0(<2 x i8>, ptr, <2 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v2i8.i64(<2 x i8>, ptr, i64, <2 x i1>, i32)
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
index 5f2728f93d551..fe24594d806af 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
@@ -891,340 +891,3 @@ define void @vp_cttz() {
   ret void
 }
 
-declare i16 @llvm.bswap.i16(i16)
-declare <2 x i16> @llvm.bswap.v2i16(<2 x i16>)
-declare <4 x i16> @llvm.bswap.v4i16(<4 x i16>)
-declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
-declare <16 x i16> @llvm.bswap.v16i16(<16 x i16>)
-declare <vscale x 1 x i16> @llvm.bswap.nxv1i16(<vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.bswap.nxv2i16(<vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.bswap.nxv4i16(<vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.bswap.nxv16i16(<vscale x 16 x i16>)
-declare i32 @llvm.bswap.i32(i32)
-declare <2 x i32> @llvm.bswap.v2i32(<2 x i32>)
-declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
-declare <8 x i32> @llvm.bswap.v8i32(<8 x i32>)
-declare <16 x i32> @llvm.bswap.v16i32(<16 x i32>)
-declare <vscale x 1 x i32> @llvm.bswap.nxv1i32(<vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.bswap.nxv2i32(<vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.bswap.nxv8i32(<vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.bswap.nxv16i32(<vscale x 16 x i32>)
-declare i64 @llvm.bswap.i64(i64)
-declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
-declare <4 x i64> @llvm.bswap.v4i64(<4 x i64>)
-declare <8 x i64> @llvm.bswap.v8i64(<8 x i64>)
-declare <16 x i64> @llvm.bswap.v16i64(<16 x i64>)
-declare <vscale x 1 x i64> @llvm.bswap.nxv1i64(<vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.bswap.nxv4i64(<vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.bswap.nxv8i64(<vscale x 8 x i64>)
-declare <vscale x 16 x i64> @llvm.bswap.nxv16i64(<vscale x 16 x i64>)
-
-declare i8 @llvm.bitreverse.i8(i8)
-declare <2 x i8> @llvm.bitreverse.v2i8(<2 x i8>)
-declare <4 x i8> @llvm.bitreverse.v4i8(<4 x i8>)
-declare <8 x i8> @llvm.bitreverse.v8i8(<8 x i8>)
-declare <16 x i8> @llvm.bitreverse.v16i8(<16 x i8>)
-declare <vscale x 1 x i8> @llvm.bitreverse.nxv1i8(<vscale x 1 x i8>)
-declare <vscale x 2 x i8> @llvm.bitreverse.nxv2i8(<vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.bitreverse.nxv4i8(<vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.bitreverse.nxv8i8(<vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.bitreverse.nxv16i8(<vscale x 16 x i8>)
-declare i16 @llvm.bitreverse.i16(i16)
-declare <2 x i16> @llvm.bitreverse.v2i16(<2 x i16>)
-declare <4 x i16> @llvm.bitreverse.v4i16(<4 x i16>)
-declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
-declare <16 x i16> @llvm.bitreverse.v16i16(<16 x i16>)
-declare <vscale x 1 x i16> @llvm.bitreverse.nxv1i16(<vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.bitreverse.nxv2i16(<vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.bitreverse.nxv4i16(<vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.bitreverse.nxv8i16(<vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.bitreverse.nxv16i16(<vscale x 16 x i16>)
-declare i32 @llvm.bitreverse.i32(i32)
-declare <2 x i32> @llvm.bitreverse.v2i32(<2 x i32>)
-declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
-declare <8 x i32> @llvm.bitreverse.v8i32(<8 x i32>)
-declare <16 x i32> @llvm.bitreverse.v16i32(<16 x i32>)
-declare <vscale x 1 x i32> @llvm.bitreverse.nxv1i32(<vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.bitreverse.nxv2i32(<vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.bitreverse.nxv4i32(<vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.bitreverse.nxv8i32(<vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.bitreverse.nxv16i32(<vscale x 16 x i32>)
-declare i64 @llvm.bitreverse.i64(i64)
-declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
-declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>)
-declare <8 x i64> @llvm.bitreverse.v8i64(<8 x i64>)
-declare <16 x i64> @llvm.bitreverse.v16i64(<16 x i64>)
-declare <vscale x 1 x i64> @llvm.bitreverse.nxv1i64(<vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.bitreverse.nxv2i64(<vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.bitreverse.nxv4i64(<vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.bitreverse.nxv8i64(<vscale x 8 x i64>)
-declare <vscale x 16 x i64> @llvm.bitreverse.nxv16i64(<vscale x 16 x i64>)
-
-declare i8 @llvm.ctpop.i8(i8)
-declare <2 x i8> @llvm.ctpop.v2i8(<2 x i8>)
-declare <4 x i8> @llvm.ctpop.v4i8(<4 x i8>)
-declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>)
-declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
-declare <vscale x 1 x i8> @llvm.ctpop.nxv1i8(<vscale x 1 x i8>)
-declare <vscale x 2 x i8> @llvm.ctpop.nxv2i8(<vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.ctpop.nxv4i8(<vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.ctpop.nxv8i8(<vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8>)
-declare i16 @llvm.ctpop.i16(i16)
-declare <2 x i16> @llvm.ctpop.v2i16(<2 x i16>)
-declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
-declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
-declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)
-declare <vscale x 1 x i16> @llvm.ctpop.nxv1i16(<vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.ctpop.nxv2i16(<vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.ctpop.nxv4i16(<vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16>)
-declare i32 @llvm.ctpop.i32(i32)
-declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
-declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
-declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
-declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>)
-declare <vscale x 1 x i32> @llvm.ctpop.nxv1i32(<vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.ctpop.nxv2i32(<vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32>)
-declare i64 @llvm.ctpop.i64(i64)
-declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
-declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
-declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>)
-declare <16 x i64> @llvm.ctpop.v16i64(<16 x i64>)
-declare <vscale x 1 x i64> @llvm.ctpop.nxv1i64(<vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.ctpop.nxv4i64(<vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.ctpop.nxv8i64(<vscale x 8 x i64>)
-declare <vscale x 16 x i64> @llvm.ctpop.nxv16i64(<vscale x 16 x i64>)
-
-declare <2 x i16> @llvm.vp.bswap.v2i16(<2 x i16>, <2 x i1>, i32)
-declare <4 x i16> @llvm.vp.bswap.v4i16(<4 x i16>, <4 x i1>, i32)
-declare <8 x i16> @llvm.vp.bswap.v8i16(<8 x i16>, <8 x i1>, i32)
-declare <16 x i16> @llvm.vp.bswap.v16i16(<16 x i16>, <16 x i1>, i32)
-declare <vscale x 1 x i16> @llvm.vp.bswap.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i16> @llvm.vp.bswap.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i16> @llvm.vp.bswap.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i16> @llvm.vp.bswap.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i16> @llvm.vp.bswap.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i1>, i32)
-declare <2 x i32> @llvm.vp.bswap.v2i32(<2 x i32>, <2 x i1>, i32)
-declare <4 x i32> @llvm.vp.bswap.v4i32(<4 x i32>, <4 x i1>, i32)
-declare <8 x i32> @llvm.vp.bswap.v8i32(<8 x i32>, <8 x i1>, i32)
-declare <16 x i32> @llvm.vp.bswap.v16i32(<16 x i32>, <16 x i1>, i32)
-declare <vscale x 1 x i32> @llvm.vp.bswap.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i32> @llvm.vp.bswap.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i32> @llvm.vp.bswap.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i32> @llvm.vp.bswap.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i32> @llvm.vp.bswap.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i1>, i32)
-declare <2 x i64> @llvm.vp.bswap.v2i64(<2 x i64>, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.bswap.v4i64(<4 x i64>, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.bswap.v8i64(<8 x i64>, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.bswap.v16i64(<16 x i64>, <16 x i1>, i32)
-declare <vscale x 1 x i64> @llvm.vp.bswap.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.bswap.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.bswap.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.bswap.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.bswap.nxv16i64(<vscale x 16 x i64>, <vscale x 16 x i1>, i32)
-
-declare <2 x i8> @llvm.ctlz.v2i8(<2 x i8>, i1)
-declare <4 x i8> @llvm.ctlz.v4i8(<4 x i8>, i1)
-declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>, i1)
-declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1)
-declare <vscale x 1 x i8> @llvm.ctlz.nxv1i8(<vscale x 1 x i8>, i1)
-declare <vscale x 2 x i8> @llvm.ctlz.nxv2i8(<vscale x 2 x i8>, i1)
-declare <vscale x 4 x i8> @llvm.ctlz.nxv4i8(<vscale x 4 x i8>, i1)
-declare <vscale x 8 x i8> @llvm.ctlz.nxv8i8(<vscale x 8 x i8>, i1)
-declare <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8>, i1)
-declare <vscale x 32 x i8> @llvm.ctlz.nxv32i8(<vscale x 32 x i8>, i1)
-declare <vscale x 64 x i8> @llvm.ctlz.nxv64i8(<vscale x 64 x i8>, i1)
-declare <2 x i16> @llvm.ctlz.v2i16(<2 x i16>, i1)
-declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>, i1)
-declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1)
-declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>, i1)
-declare <vscale x 1 x i16> @llvm.ctlz.nxv1i16(<vscale x 1 x i16>, i1)
-declare <vscale x 2 x i16> @llvm.ctlz.nxv2i16(<vscale x 2 x i16>, i1)
-declare <vscale x 4 x i16> @llvm.ctlz.nxv4i16(<vscale x 4 x i16>, i1)
-declare <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16>, i1)
-declare <vscale x 16 x i16> @llvm.ctlz.nxv16i16(<vscale x 16 x i16>, i1)
-declare <vscale x 32 x i16> @llvm.ctlz.nxv32i16(<vscale x 32 x i16>, i1)
-declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
-declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
-declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1)
-declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>, i1)
-declare <vscale x 1 x i32> @llvm.ctlz.nxv1i32(<vscale x 1 x i32>, i1)
-declare <vscale x 2 x i32> @llvm.ctlz.nxv2i32(<vscale x 2 x i32>, i1)
-declare <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32>, i1)
-declare <vscale x 8 x i32> @llvm.ctlz.nxv8i32(<vscale x 8 x i32>, i1)
-declare <vscale x 16 x i32> @llvm.ctlz.nxv16i32(<vscale x 16 x i32>, i1)
-declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
-declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1)
-declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>, i1)
-declare <16 x i64> @llvm.ctlz.v16i64(<16 x i64>, i1)
-declare <vscale x 1 x i64> @llvm.ctlz.nxv1i64(<vscale x 1 x i64>, i1)
-declare <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64>, i1)
-declare <vscale x 4 x i64> @llvm.ctlz.nxv4i64(<vscale x 4 x i64>, i1)
-declare <vscale x 8 x i64> @llvm.ctlz.nxv8i64(<vscale x 8 x i64>, i1)
-declare <vscale x 16 x i64> @llvm.ctlz.nxv16i64(<vscale x 16 x i64>, i1)
-
-declare <2 x i8> @llvm.cttz.v2i8(<2 x i8>, i1)
-declare <4 x i8> @llvm.cttz.v4i8(<4 x i8>, i1)
-declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>, i1)
-declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
-declare <vscale x 1 x i8> @llvm.cttz.nxv1i8(<vscale x 1 x i8>, i1)
-declare <vscale x 2 x i8> @llvm.cttz.nxv2i8(<vscale x 2 x i8>, i1)
-declare <vscale x 4 x i8> @llvm.cttz.nxv4i8(<vscale x 4 x i8>, i1)
-declare <vscale x 8 x i8> @llvm.cttz.nxv8i8(<vscale x 8 x i8>, i1)
-declare <vscale x 16 x i8> @llvm.cttz.nxv16i8(<vscale x 16 x i8>, i1)
-declare <vscale x 32 x i8> @llvm.cttz.nxv32i8(<vscale x 32 x i8>, i1)
-declare <vscale x 64 x i8> @llvm.cttz.nxv64i8(<vscale x 64 x i8>, i1)
-declare <2 x i16> @llvm.cttz.v2i16(<2 x i16>, i1)
-declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>, i1)
-declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
-declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
-declare <vscale x 1 x i16> @llvm.cttz.nxv1i16(<vscale x 1 x i16>, i1)
-declare <vscale x 2 x i16> @llvm.cttz.nxv2i16(<vscale x 2 x i16>, i1)
-declare <vscale x 4 x i16> @llvm.cttz.nxv4i16(<vscale x 4 x i16>, i1)
-declare <vscale x 8 x i16> @llvm.cttz.nxv8i16(<vscale x 8 x i16>, i1)
-declare <vscale x 16 x i16> @llvm.cttz.nxv16i16(<vscale x 16 x i16>, i1)
-declare <vscale x 32 x i16> @llvm.cttz.nxv32i16(<vscale x 32 x i16>, i1)
-declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
-declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
-declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
-declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)
-declare <vscale x 1 x i32> @llvm.cttz.nxv1i32(<vscale x 1 x i32>, i1)
-declare <vscale x 2 x i32> @llvm.cttz.nxv2i32(<vscale x 2 x i32>, i1)
-declare <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32>, i1)
-declare <vscale x 8 x i32> @llvm.cttz.nxv8i32(<vscale x 8 x i32>, i1)
-declare <vscale x 16 x i32> @llvm.cttz.nxv16i32(<vscale x 16 x i32>, i1)
-declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
-declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
-declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)
-declare <16 x i64> @llvm.cttz.v16i64(<16 x i64>, i1)
-declare <vscale x 1 x i64> @llvm.cttz.nxv1i64(<vscale x 1 x i64>, i1)
-declare <vscale x 2 x i64> @llvm.cttz.nxv2i64(<vscale x 2 x i64>, i1)
-declare <vscale x 4 x i64> @llvm.cttz.nxv4i64(<vscale x 4 x i64>, i1)
-declare <vscale x 8 x i64> @llvm.cttz.nxv8i64(<vscale x 8 x i64>, i1)
-declare <vscale x 16 x i64> @llvm.cttz.nxv16i64(<vscale x 16 x i64>, i1)
-
-declare <2 x i8> @llvm.vp.ctpop.v2i8(<2 x i8>, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.ctpop.v4i8(<4 x i8>, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.ctpop.v8i8(<8 x i8>, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.ctpop.v16i8(<16 x i8>, <16 x i1>, i32)
-declare <vscale x 1 x i8> @llvm.vp.ctpop.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.ctpop.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.ctpop.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.ctpop.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.ctpop.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i32)
-declare <2 x i16> @llvm.vp.ctpop.v2i16(<2 x i16>, <2 x i1>, i32)
-declare <4 x i16> @llvm.vp.ctpop.v4i16(<4 x i16>, <4 x i1>, i32)
-declare <8 x i16> @llvm.vp.ctpop.v8i16(<8 x i16>, <8 x i1>, i32)
-declare <16 x i16> @llvm.vp.ctpop.v16i16(<16 x i16>, <16 x i1>, i32)
-declare <vscale x 1 x i16> @llvm.vp.ctpop.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i16> @llvm.vp.ctpop.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i16> @llvm.vp.ctpop.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i16> @llvm.vp.ctpop.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i16> @llvm.vp.ctpop.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i1>, i32)
-declare <2 x i32> @llvm.vp.ctpop.v2i32(<2 x i32>, <2 x i1>, i32)
-declare <4 x i32> @llvm.vp.ctpop.v4i32(<4 x i32>, <4 x i1>, i32)
-declare <8 x i32> @llvm.vp.ctpop.v8i32(<8 x i32>, <8 x i1>, i32)
-declare <16 x i32> @llvm.vp.ctpop.v16i32(<16 x i32>, <16 x i1>, i32)
-declare <vscale x 1 x i32> @llvm.vp.ctpop.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i32> @llvm.vp.ctpop.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i32> @llvm.vp.ctpop.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i32> @llvm.vp.ctpop.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i32> @llvm.vp.ctpop.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i1>, i32)
-declare <2 x i64> @llvm.vp.ctpop.v2i64(<2 x i64>, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.ctpop.v4i64(<4 x i64>, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.ctpop.v8i64(<8 x i64>, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.ctpop.v16i64(<16 x i64>, <16 x i1>, i32)
-declare <vscale x 1 x i64> @llvm.vp.ctpop.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.ctpop.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.ctpop.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.ctpop.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.ctpop.nxv16i64(<vscale x 16 x i64>, <vscale x 16 x i1>, i32)
-
-declare <2 x i8> @llvm.vp.ctlz.v2i8(<2 x i8>, i1 immarg, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.ctlz.v4i8(<4 x i8>, i1 immarg, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.ctlz.v8i8(<8 x i8>, i1 immarg, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.ctlz.v16i8(<16 x i8>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i8> @llvm.vp.ctlz.nxv1i8(<vscale x 1 x i8>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.ctlz.nxv2i8(<vscale x 2 x i8>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.ctlz.nxv4i8(<vscale x 4 x i8>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.ctlz.nxv8i8(<vscale x 8 x i8>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.ctlz.nxv16i8(<vscale x 16 x i8>, i1 immarg, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i8> @llvm.vp.ctlz.nxv32i8(<vscale x 32 x i8>, i1 immarg, <vscale x 32 x i1>, i32)
-declare <vscale x 64 x i8> @llvm.vp.ctlz.nxv64i8(<vscale x 64 x i8>, i1 immarg, <vscale x 64 x i1>, i32)
-declare <2 x i16> @llvm.vp.ctlz.v2i16(<2 x i16>, i1 immarg, <2 x i1>, i32)
-declare <4 x i16> @llvm.vp.ctlz.v4i16(<4 x i16>, i1 immarg, <4 x i1>, i32)
-declare <8 x i16> @llvm.vp.ctlz.v8i16(<8 x i16>, i1 immarg, <8 x i1>, i32)
-declare <16 x i16> @llvm.vp.ctlz.v16i16(<16 x i16>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i16> @llvm.vp.ctlz.nxv1i16(<vscale x 1 x i16>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i16> @llvm.vp.ctlz.nxv2i16(<vscale x 2 x i16>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i16> @llvm.vp.ctlz.nxv4i16(<vscale x 4 x i16>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i16> @llvm.vp.ctlz.nxv8i16(<vscale x 8 x i16>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i16> @llvm.vp.ctlz.nxv16i16(<vscale x 16 x i16>, i1 immarg, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i16> @llvm.vp.ctlz.nxv32i16(<vscale x 32 x i16>, i1 immarg, <vscale x 32 x i1>, i32)
-declare <2 x i32> @llvm.vp.ctlz.v2i32(<2 x i32>, i1 immarg, <2 x i1>, i32)
-declare <4 x i32> @llvm.vp.ctlz.v4i32(<4 x i32>, i1 immarg, <4 x i1>, i32)
-declare <8 x i32> @llvm.vp.ctlz.v8i32(<8 x i32>, i1 immarg, <8 x i1>, i32)
-declare <16 x i32> @llvm.vp.ctlz.v16i32(<16 x i32>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i32> @llvm.vp.ctlz.nxv1i32(<vscale x 1 x i32>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i32> @llvm.vp.ctlz.nxv2i32(<vscale x 2 x i32>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i32> @llvm.vp.ctlz.nxv4i32(<vscale x 4 x i32>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i32> @llvm.vp.ctlz.nxv8i32(<vscale x 8 x i32>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i32> @llvm.vp.ctlz.nxv16i32(<vscale x 16 x i32>, i1 immarg, <vscale x 16 x i1>, i32)
-declare <2 x i64> @llvm.vp.ctlz.v2i64(<2 x i64>, i1 immarg, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.ctlz.v4i64(<4 x i64>, i1 immarg, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.ctlz.v8i64(<8 x i64>, i1 immarg, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.ctlz.v16i64(<16 x i64>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i64> @llvm.vp.ctlz.nxv1i64(<vscale x 1 x i64>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.ctlz.nxv2i64(<vscale x 2 x i64>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.ctlz.nxv4i64(<vscale x 4 x i64>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.ctlz.nxv8i64(<vscale x 8 x i64>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.ctlz.nxv16i64(<vscale x 16 x i64>, i1 immarg, <vscale x 16 x i1>, i32)
-
-declare <2 x i8> @llvm.vp.cttz.v2i8(<2 x i8>, i1 immarg, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.cttz.v4i8(<4 x i8>, i1 immarg, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.cttz.v8i8(<8 x i8>, i1 immarg, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.cttz.v16i8(<16 x i8>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i8> @llvm.vp.cttz.nxv1i8(<vscale x 1 x i8>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.cttz.nxv2i8(<vscale x 2 x i8>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.cttz.nxv4i8(<vscale x 4 x i8>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.cttz.nxv8i8(<vscale x 8 x i8>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.cttz.nxv16i8(<vscale x 16 x i8>, i1 immarg, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i8> @llvm.vp.cttz.nxv32i8(<vscale x 32 x i8>, i1 immarg, <vscale x 32 x i1>, i32)
-declare <vscale x 64 x i8> @llvm.vp.cttz.nxv64i8(<vscale x 64 x i8>, i1 immarg, <vscale x 64 x i1>, i32)
-declare <2 x i16> @llvm.vp.cttz.v2i16(<2 x i16>, i1 immarg, <2 x i1>, i32)
-declare <4 x i16> @llvm.vp.cttz.v4i16(<4 x i16>, i1 immarg, <4 x i1>, i32)
-declare <8 x i16> @llvm.vp.cttz.v8i16(<8 x i16>, i1 immarg, <8 x i1>, i32)
-declare <16 x i16> @llvm.vp.cttz.v16i16(<16 x i16>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i16> @llvm.vp.cttz.nxv1i16(<vscale x 1 x i16>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i16> @llvm.vp.cttz.nxv2i16(<vscale x 2 x i16>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i16> @llvm.vp.cttz.nxv4i16(<vscale x 4 x i16>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i16> @llvm.vp.cttz.nxv8i16(<vscale x 8 x i16>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i16> @llvm.vp.cttz.nxv16i16(<vscale x 16 x i16>, i1 immarg, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i16> @llvm.vp.cttz.nxv32i16(<vscale x 32 x i16>, i1 immarg, <vscale x 32 x i1>, i32)
-declare <2 x i32> @llvm.vp.cttz.v2i32(<2 x i32>, i1 immarg, <2 x i1>, i32)
-declare <4 x i32> @llvm.vp.cttz.v4i32(<4 x i32>, i1 immarg, <4 x i1>, i32)
-declare <8 x i32> @llvm.vp.cttz.v8i32(<8 x i32>, i1 immarg, <8 x i1>, i32)
-declare <16 x i32> @llvm.vp.cttz.v16i32(<16 x i32>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i32> @llvm.vp.cttz.nxv1i32(<vscale x 1 x i32>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i32> @llvm.vp.cttz.nxv2i32(<vscale x 2 x i32>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i32> @llvm.vp.cttz.nxv4i32(<vscale x 4 x i32>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i32> @llvm.vp.cttz.nxv8i32(<vscale x 8 x i32>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i32> @llvm.vp.cttz.nxv16i32(<vscale x 16 x i32>, i1 immarg, <vscale x 16 x i1>, i32)
-declare <2 x i64> @llvm.vp.cttz.v2i64(<2 x i64>, i1 immarg, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.cttz.v4i64(<4 x i64>, i1 immarg, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.cttz.v8i64(<8 x i64>, i1 immarg, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.cttz.v16i64(<16 x i64>, i1 immarg, <16 x i1>, i32)
-declare <vscale x 1 x i64> @llvm.vp.cttz.nxv1i64(<vscale x 1 x i64>, i1 immarg, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.cttz.nxv2i64(<vscale x 2 x i64>, i1 immarg, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.cttz.nxv4i64(<vscale x 4 x i64>, i1 immarg, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.cttz.nxv8i64(<vscale x 8 x i64>, i1 immarg, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.cttz.nxv16i64(<vscale x 16 x i64>, i1 immarg, <vscale x 16 x i1>, i32)
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll b/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll
index 10474d227851f..3be36f13e8731 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-min-max.ll
@@ -341,162 +341,3 @@ define void @umin() {
   ret void
 }
 
-declare i8 @llvm.smax.i8(i8, i8)
-declare <2 x i8> @llvm.smax.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.smax.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.smax.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.smax.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 1 x i8> @llvm.smax.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i8>)
-declare <vscale x 2 x i8> @llvm.smax.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.smax.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.smax.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.smax.i16(i16, i16)
-declare <2 x i16> @llvm.smax.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.smax.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.smax.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 1 x i16> @llvm.smax.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.smax.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.smax.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.smax.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.smax.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.smax.i32(i32, i32)
-declare <2 x i32> @llvm.smax.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.smax.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.smax.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 1 x i32> @llvm.smax.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.smax.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.smax.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.smax.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.smax.i64(i64, i64)
-declare <2 x i64> @llvm.smax.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.smax.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.smax.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.smax.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 1 x i64> @llvm.smax.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.smax.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.smax.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.smax.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.smin.i8(i8, i8)
-declare <2 x i8> @llvm.smin.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.smin.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.smin.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.smin.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 1 x i8> @llvm.smin.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i8>)
-declare <vscale x 2 x i8> @llvm.smin.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.smin.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.smin.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.smin.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.smin.i16(i16, i16)
-declare <2 x i16> @llvm.smin.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.smin.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.smin.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 1 x i16> @llvm.smin.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.smin.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.smin.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.smin.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.smin.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.smin.i32(i32, i32)
-declare <2 x i32> @llvm.smin.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.smin.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.smin.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 1 x i32> @llvm.smin.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.smin.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.smin.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.smin.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.smin.i64(i64, i64)
-declare <2 x i64> @llvm.smin.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.smin.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.smin.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.smin.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 1 x i64> @llvm.smin.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.smin.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.smin.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.smin.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.umax.i8(i8, i8)
-declare <2 x i8> @llvm.umax.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.umax.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.umax.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.umax.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 1 x i8> @llvm.umax.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i8>)
-declare <vscale x 2 x i8> @llvm.umax.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.umax.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.umax.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.umax.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.umax.i16(i16, i16)
-declare <2 x i16> @llvm.umax.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.umax.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.umax.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.umax.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 1 x i16> @llvm.umax.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.umax.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.umax.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.umax.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.umax.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.umax.i32(i32, i32)
-declare <2 x i32> @llvm.umax.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.umax.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.umax.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 1 x i32> @llvm.umax.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.umax.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.umax.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.umax.i64(i64, i64)
-declare <2 x i64> @llvm.umax.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.umax.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.umax.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.umax.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 1 x i64> @llvm.umax.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.umax.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.umax.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.umax.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.umin.i8(i8, i8)
-declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.umin.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.umin.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.umin.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 1 x i8> @llvm.umin.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i8>)
-declare <vscale x 2 x i8> @llvm.umin.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.umin.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.umin.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.umin.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.umin.i16(i16, i16)
-declare <2 x i16> @llvm.umin.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.umin.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.umin.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 1 x i16> @llvm.umin.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i16>)
-declare <vscale x 2 x i16> @llvm.umin.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.umin.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.umin.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.umin.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.umin.i32(i32, i32)
-declare <2 x i32> @llvm.umin.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.umin.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.umin.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 1 x i32> @llvm.umin.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i32>)
-declare <vscale x 2 x i32> @llvm.umin.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.umin.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.umin.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.umin.i64(i64, i64)
-declare <2 x i64> @llvm.umin.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.umin.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.umin.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.umin.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 1 x i64> @llvm.umin.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>)
-declare <vscale x 2 x i64> @llvm.umin.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.umin.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.umin.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll b/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll
index 0758eb204be48..20c6aef3c8e45 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-sat-math.ll
@@ -462,219 +462,3 @@ define void @sshl.sat() {
   call <vscale x 8 x i64> @llvm.sshl.sat.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef)
   ret void
 }
-
-declare i8 @llvm.sadd.sat.i8(i8, i8)
-declare <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 2 x i8> @llvm.sadd.sat.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.sadd.sat.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.sadd.sat.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.sadd.sat.i16(i16, i16)
-declare <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 2 x i16> @llvm.sadd.sat.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.sadd.sat.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.sadd.sat.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.sadd.sat.i32(i32, i32)
-declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 2 x i32> @llvm.sadd.sat.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.sadd.sat.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.sadd.sat.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.sadd.sat.i64(i64, i64)
-declare <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.sadd.sat.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.sadd.sat.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.sadd.sat.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.uadd.sat.i8(i8, i8)
-declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 2 x i8> @llvm.uadd.sat.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.uadd.sat.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.uadd.sat.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.uadd.sat.i16(i16, i16)
-declare <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 2 x i16> @llvm.uadd.sat.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.uadd.sat.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.uadd.sat.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.uadd.sat.i32(i32, i32)
-declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 2 x i32> @llvm.uadd.sat.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.uadd.sat.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.uadd.sat.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.uadd.sat.i64(i64, i64)
-declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.uadd.sat.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.uadd.sat.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.uadd.sat.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.usub.sat.i8(i8, i8)
-declare <2 x i8> @llvm.usub.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.usub.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.usub.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 2 x i8> @llvm.usub.sat.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.usub.sat.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.usub.sat.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.usub.sat.i16(i16, i16)
-declare <2 x i16> @llvm.usub.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.usub.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 2 x i16> @llvm.usub.sat.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.usub.sat.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.usub.sat.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.usub.sat.i32(i32, i32)
-declare <2 x i32> @llvm.usub.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.usub.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 2 x i32> @llvm.usub.sat.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.usub.sat.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.usub.sat.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.usub.sat.i64(i64, i64)
-declare <2 x i64> @llvm.usub.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.usub.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.usub.sat.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.usub.sat.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.usub.sat.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.usub.sat.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.ssub.sat.i8(i8, i8)
-declare <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 2 x i8> @llvm.ssub.sat.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.ssub.sat.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.ssub.sat.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.ssub.sat.i16(i16, i16)
-declare <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 2 x i16> @llvm.ssub.sat.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.ssub.sat.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.ssub.sat.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.ssub.sat.i32(i32, i32)
-declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 2 x i32> @llvm.ssub.sat.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.ssub.sat.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.ssub.sat.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.ssub.sat.i64(i64, i64)
-declare <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.ssub.sat.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.ssub.sat.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.ssub.sat.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.ushl.sat.i8(i8, i8)
-declare <2 x i8> @llvm.ushl.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.ushl.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.ushl.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.ushl.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 2 x i8> @llvm.ushl.sat.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.ushl.sat.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.ushl.sat.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.ushl.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.ushl.sat.i16(i16, i16)
-declare <2 x i16> @llvm.ushl.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.ushl.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.ushl.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.ushl.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 2 x i16> @llvm.ushl.sat.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.ushl.sat.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.ushl.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.ushl.sat.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.ushl.sat.i32(i32, i32)
-declare <2 x i32> @llvm.ushl.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.ushl.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.ushl.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 2 x i32> @llvm.ushl.sat.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.ushl.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.ushl.sat.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.ushl.sat.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.ushl.sat.i64(i64, i64)
-declare <2 x i64> @llvm.ushl.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.ushl.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.ushl.sat.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.ushl.sat.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 2 x i64> @llvm.ushl.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.ushl.sat.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.ushl.sat.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
-
-declare i8 @llvm.sshl.sat.i8(i8, i8)
-declare <2 x i8> @llvm.sshl.sat.v2i8(<2 x i8>, <2 x i8>)
-declare <4 x i8> @llvm.sshl.sat.v4i8(<4 x i8>, <4 x i8>)
-declare <8 x i8> @llvm.sshl.sat.v8i8(<8 x i8>, <8 x i8>)
-declare <16 x i8> @llvm.sshl.sat.v16i8(<16 x i8>, <16 x i8>)
-declare <vscale x 2 x i8> @llvm.sshl.sat.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.sshl.sat.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.sshl.sat.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.sshl.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
-declare i16 @llvm.sshl.sat.i16(i16, i16)
-declare <2 x i16> @llvm.sshl.sat.v2i16(<2 x i16>, <2 x i16>)
-declare <4 x i16> @llvm.sshl.sat.v4i16(<4 x i16>, <4 x i16>)
-declare <8 x i16> @llvm.sshl.sat.v8i16(<8 x i16>, <8 x i16>)
-declare <16 x i16> @llvm.sshl.sat.v16i16(<16 x i16>, <16 x i16>)
-declare <vscale x 2 x i16> @llvm.sshl.sat.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.sshl.sat.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.sshl.sat.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.sshl.sat.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>)
-declare i32 @llvm.sshl.sat.i32(i32, i32)
-declare <2 x i32> @llvm.sshl.sat.v2i32(<2 x i32>, <2 x i32>)
-declare <4 x i32> @llvm.sshl.sat.v4i32(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32>, <8 x i32>)
-declare <16 x i32> @llvm.sshl.sat.v16i32(<16 x i32>, <16 x i32>)
-declare <vscale x 2 x i32> @llvm.sshl.sat.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.sshl.sat.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.sshl.sat.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>)
-declare <vscale x 16 x i32> @llvm.sshl.sat.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>)
-declare i64 @llvm.sshl.sat.i64(i64, i64)
-declare <2 x i64> @llvm.sshl.sat.v2i64(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.sshl.sat.v4i64(<4 x i64>, <4 x i64>)
-declare <8 x i64> @llvm.sshl.sat.v8i64(<8 x i64>, <8 x i64>)
-declare <16 x i64> @llvm.sshl.sat.v16i64(<16 x i64>, <16 x i64>)
-declare <vscale x 2 x i64> @llvm.sshl.sat.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.sshl.sat.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.sshl.sat.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll b/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll
index e6f53d4429c79..892277a2d5740 100644
--- a/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll
@@ -101,43 +101,3 @@ entry:
 
   ret void
 }
-
-declare <2 x i8> @llvm.masked.load.v2i8.p0(ptr, i32, <2 x i1>, <2 x i8>)
-declare <4 x i8> @llvm.masked.load.v4i8.p0(ptr, i32, <4 x i1>, <4 x i8>)
-declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32, <8 x i1>, <8 x i8>)
-declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32, <16 x i1>, <16 x i8>)
-declare <2 x i16> @llvm.masked.load.v2i16.p0(ptr, i32, <2 x i1>, <2 x i16>)
-declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32, <4 x i1>, <4 x i16>)
-declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32, <8 x i1>, <8 x i16>)
-declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>)
-declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)
-declare <2 x i64> @llvm.masked.load.v2i64.p0(ptr, i32, <2 x i1>, <2 x i64>)
-declare <4 x i64> @llvm.masked.load.v4i64.p0(ptr, i32, <4 x i1>, <4 x i64>)
-declare <2 x half> @llvm.masked.load.v2f16.p0(ptr, i32, <2 x i1>, <2 x half>)
-declare <4 x half> @llvm.masked.load.v4f16.p0(ptr, i32, <4 x i1>, <4 x half>)
-declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32, <8 x i1>, <8 x half>)
-declare <32 x half> @llvm.masked.load.v32f16.p0(ptr, i32, <32 x i1>, <32 x half>)
-declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
-declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32, <4 x i1>, <4 x float>)
-declare <2 x double> @llvm.masked.load.v2f64.p0(ptr, i32, <2 x i1>, <2 x double>)
-
-
-declare <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
-declare <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
-declare <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
-declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
-declare <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x i64>)
-declare <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr, i32, <vscale x 1 x i1>, <vscale x 1 x i64>)
-declare <vscale x 2 x half> @llvm.masked.load.nxv2f16.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
-declare <vscale x 4 x half> @llvm.masked.load.nxv4f16.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
-declare <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
-declare <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr, i32, <vscale x 32 x i1>, <vscale x 32 x half>)
-declare <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
-declare <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
-declare <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-add.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-add.ll
index 6032ae01aa718..23ee689937de1 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-add.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-add.ll
@@ -173,44 +173,3 @@ define i32 @reduce_i64(i32 %arg) {
   %V128 = call i64 @llvm.vector.reduce.add.v128i64(<128 x i64> undef)
   ret i32 undef
 }
-
-declare i1 @llvm.vector.reduce.add.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.add.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.add.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.add.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.add.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.add.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.add.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.add.v128i1(<128 x i1>)
-declare i8 @llvm.vector.reduce.add.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.add.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.add.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.add.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.add.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.add.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.add.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.add.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.add.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.add.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.add.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.add.v128i64(<128 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
index dc6a582df133b..eabbe1d43f117 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
@@ -216,47 +216,3 @@ define i32 @reduce_i64(i32 %arg) {
   %V128 = call i64 @llvm.vector.reduce.and.v128i64(<128 x i64> undef)
   ret i32 undef
 }
-
-declare i1 @llvm.vector.reduce.and.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.and.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.and.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.and.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.and.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.and.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.and.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.and.v128i1(<128 x i1>)
-declare i1 @llvm.vector.reduce.and.v256i1(<256 x i1>)
-declare i1 @llvm.vector.reduce.and.v512i1(<512 x i1>)
-declare i1 @llvm.vector.reduce.and.v1024i1(<1024 x i1>)
-declare i8 @llvm.vector.reduce.and.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.and.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.and.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.and.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.and.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.and.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.and.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.and.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.and.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.and.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.and.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.and.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.and.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.and.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.and.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.and.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.and.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.and.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.and.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.and.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.and.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.and.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.and.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.and.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.and.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.and.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.and.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.and.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.and.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.and.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.and.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.and.v128i64(<128 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll
index b14c60012077d..d937d066d6ebe 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll
@@ -55,13 +55,6 @@ call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
 call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
 ret float undef
 }
-declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>)
-declare float @llvm.vector.reduce.fmaximum.v4f32(<4 x float>)
-declare float @llvm.vector.reduce.fmaximum.v8f32(<8 x float>)
-declare float @llvm.vector.reduce.fmaximum.v16f32(<16 x float>)
-declare float @llvm.vector.reduce.fmaximum.v32f32(<32 x float>)
-declare float @llvm.vector.reduce.fmaximum.v64f32(<64 x float>)
-declare float @llvm.vector.reduce.fmaximum.v128f32(<128 x float>)
 
 define double @reduce_fmaximum_f64(double %arg) {
 ; CHECK-LABEL: 'reduce_fmaximum_f64'
@@ -108,9 +101,3 @@ call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
 call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
 ret double undef
 }
-declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>)
-declare double @llvm.vector.reduce.fmaximum.v4f64(<4 x double>)
-declare double @llvm.vector.reduce.fmaximum.v8f64(<8 x double>)
-declare double @llvm.vector.reduce.fmaximum.v16f64(<16 x double>)
-declare double @llvm.vector.reduce.fmaximum.v32f64(<32 x double>)
-declare double @llvm.vector.reduce.fmaximum.v64f64(<64 x double>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll
index 2172a85bc46aa..397f25c2599b1 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll
@@ -34,13 +34,6 @@ define float @reduce_fmaximum_f32(float %arg) {
 %V128   = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef)
 ret float undef
 }
-declare float @llvm.vector.reduce.fminimum.v2f32(<2 x float>)
-declare float @llvm.vector.reduce.fminimum.v4f32(<4 x float>)
-declare float @llvm.vector.reduce.fminimum.v8f32(<8 x float>)
-declare float @llvm.vector.reduce.fminimum.v16f32(<16 x float>)
-declare float @llvm.vector.reduce.fminimum.v32f32(<32 x float>)
-declare float @llvm.vector.reduce.fminimum.v64f32(<64 x float>)
-declare float @llvm.vector.reduce.fminimum.v128f32(<128 x float>)
 
 define double @reduce_fmaximum_f64(double %arg) {
 ; CHECK-LABEL: 'reduce_fmaximum_f64'
@@ -69,9 +62,3 @@ define double @reduce_fmaximum_f64(double %arg) {
 %V64   = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef)
 ret double undef
 }
-declare double @llvm.vector.reduce.fminimum.v2f64(<2 x double>)
-declare double @llvm.vector.reduce.fminimum.v4f64(<4 x double>)
-declare double @llvm.vector.reduce.fminimum.v8f64(<8 x double>)
-declare double @llvm.vector.reduce.fminimum.v16f64(<16 x double>)
-declare double @llvm.vector.reduce.fminimum.v32f64(<32 x double>)
-declare double @llvm.vector.reduce.fminimum.v64f64(<64 x double>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll
index 5c9303af31747..1370beb880d03 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-max.ll
@@ -343,85 +343,3 @@ define i32 @reduce_smax_i64(i32 %arg) {
   %V128 = call i64 @llvm.vector.reduce.smax.v128i64(<128 x i64> undef)
   ret i32 undef
 }
-
-
-declare i1 @llvm.vector.reduce.umax.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.umax.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.umax.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.umax.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.umax.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.umax.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.umax.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.umax.v128i1(<128 x i1>)
-declare i8 @llvm.vector.reduce.umax.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.umax.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.umax.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.umax.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.umax.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.umax.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.umax.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.umax.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.umax.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.umax.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.umax.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.umax.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.umax.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.umax.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.umax.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.umax.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.umax.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.umax.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.umax.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.umax.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.umax.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.umax.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.umax.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.umax.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.umax.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.umax.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.umax.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.umax.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.umax.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.umax.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.umax.v128i64(<128 x i64>)
-declare i1 @llvm.vector.reduce.smax.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.smax.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.smax.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.smax.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.smax.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.smax.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.smax.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.smax.v128i1(<128 x i1>)
-declare i8 @llvm.vector.reduce.smax.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.smax.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.smax.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.smax.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.smax.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.smax.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.smax.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.smax.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.smax.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.smax.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.smax.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.smax.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.smax.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.smax.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.smax.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.smax.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.smax.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.smax.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.smax.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.smax.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.smax.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.smax.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.smax.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.smax.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.smax.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.smax.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.smax.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.smax.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.smax.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.smax.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.smax.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.smax.v128i64(<128 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll
index 9875d3e585811..7acf76b53bb2d 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-min.ll
@@ -343,84 +343,3 @@ define i32 @reduce_smin_i64(i32 %arg) {
   %V128 = call i64 @llvm.vector.reduce.smin.v128i64(<128 x i64> undef)
   ret i32 undef
 }
-
-declare i1 @llvm.vector.reduce.umin.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.umin.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.umin.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.umin.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.umin.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.umin.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.umin.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.umin.v128i1(<128 x i1>)
-declare i8 @llvm.vector.reduce.umin.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.umin.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.umin.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.umin.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.umin.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.umin.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.umin.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.umin.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.umin.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.umin.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.umin.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.umin.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.umin.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.umin.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.umin.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.umin.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.umin.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.umin.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.umin.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.umin.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.umin.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.umin.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.umin.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.umin.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.umin.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.umin.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.umin.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.umin.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.umin.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.umin.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.umin.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.umin.v128i64(<128 x i64>)
-declare i1 @llvm.vector.reduce.smin.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.smin.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.smin.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.smin.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.smin.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.smin.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.smin.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.smin.v128i1(<128 x i1>)
-declare i8 @llvm.vector.reduce.smin.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.smin.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.smin.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.smin.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.smin.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.smin.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.smin.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.smin.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.smin.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.smin.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.smin.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.smin.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.smin.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.smin.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.smin.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.smin.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.smin.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.smin.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.smin.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.smin.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.smin.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.smin.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.smin.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.smin.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.smin.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.smin.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.smin.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.smin.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.smin.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.smin.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.smin.v128i64(<128 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
index b0e90253b2810..db7d4ddf4c5ae 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
@@ -182,47 +182,3 @@ define i32 @reduce_i64(i32 %arg) {
   %V128 = call i64 @llvm.vector.reduce.or.v128i64(<128 x i64> undef)
   ret i32 undef
 }
-
-declare i1 @llvm.vector.reduce.or.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.or.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.or.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.or.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.or.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.or.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.or.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.or.v128i1(<128 x i1>)
-declare i1 @llvm.vector.reduce.or.v256i1(<256 x i1>)
-declare i1 @llvm.vector.reduce.or.v512i1(<512 x i1>)
-declare i1 @llvm.vector.reduce.or.v1024i1(<1024 x i1>)
-declare i8 @llvm.vector.reduce.or.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.or.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.or.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.or.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.or.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.or.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.or.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.or.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.or.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.or.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.or.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.or.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.or.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.or.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.or.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.or.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.or.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.or.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.or.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.or.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.or.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.or.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.or.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.or.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.or.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.or.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.or.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.or.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.or.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.or.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.or.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.or.v128i64(<128 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll
index 8ae387f48ccaa..6f631b2a0aaa9 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-fp.ll
@@ -4,8 +4,6 @@
 ; RUN: opt < %s -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=SIZE
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=SIZE
 
-declare half @llvm.vector.reduce.fadd.nxv1f16(half, <vscale x 1 x half>)
-
 define half @vreduce_fadd_nxv1f16(<vscale x 1 x half> %v, half %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv1f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call reassoc half @llvm.vector.reduce.fadd.nxv1f16(half %s, <vscale x 1 x half> %v)
@@ -32,8 +30,6 @@ define half @vreduce_ord_fadd_nxv1f16(<vscale x 1 x half> %v, half %s) {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fadd.nxv2f16(half, <vscale x 2 x half>)
-
 define half @vreduce_fadd_nxv2f16(<vscale x 2 x half> %v, half %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv2f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call reassoc half @llvm.vector.reduce.fadd.nxv2f16(half %s, <vscale x 2 x half> %v)
@@ -60,8 +56,6 @@ define half @vreduce_ord_fadd_nxv2f16(<vscale x 2 x half> %v, half %s) {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fadd.nxv4f16(half, <vscale x 4 x half>)
-
 define half @vreduce_fadd_nxv4f16(<vscale x 4 x half> %v, half %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv4f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call reassoc half @llvm.vector.reduce.fadd.nxv4f16(half %s, <vscale x 4 x half> %v)
@@ -88,8 +82,6 @@ define half @vreduce_ord_fadd_nxv4f16(<vscale x 4 x half> %v, half %s) {
   ret half %red
 }
 
-declare float @llvm.vector.reduce.fadd.nxv1f32(float, <vscale x 1 x float>)
-
 define float @vreduce_fadd_nxv1f32(<vscale x 1 x float> %v, float %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv1f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call reassoc float @llvm.vector.reduce.fadd.nxv1f32(float %s, <vscale x 1 x float> %v)
@@ -148,8 +140,6 @@ define float @vreduce_ord_fwadd_nxv1f32(<vscale x 1 x half> %v, float %s) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fadd.nxv2f32(float, <vscale x 2 x float>)
-
 define float @vreduce_fadd_nxv2f32(<vscale x 2 x float> %v, float %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv2f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call reassoc float @llvm.vector.reduce.fadd.nxv2f32(float %s, <vscale x 2 x float> %v)
@@ -208,8 +198,6 @@ define float @vreduce_ord_fwadd_nxv2f32(<vscale x 2 x half> %v, float %s) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fadd.nxv4f32(float, <vscale x 4 x float>)
-
 define float @vreduce_fadd_nxv4f32(<vscale x 4 x float> %v, float %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv4f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float %s, <vscale x 4 x float> %v)
@@ -268,8 +256,6 @@ define float @vreduce_ord_fwadd_nxv4f32(<vscale x 4 x half> %v, float %s) {
   ret float %red
 }
 
-declare double @llvm.vector.reduce.fadd.nxv1f64(double, <vscale x 1 x double>)
-
 define double @vreduce_fadd_nxv1f64(<vscale x 1 x double> %v, double %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv1f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call reassoc double @llvm.vector.reduce.fadd.nxv1f64(double %s, <vscale x 1 x double> %v)
@@ -328,8 +314,6 @@ define double @vreduce_ord_fwadd_nxv1f64(<vscale x 1 x float> %v, double %s) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
-
 define double @vreduce_fadd_nxv2f64(<vscale x 2 x double> %v, double %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv2f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call reassoc double @llvm.vector.reduce.fadd.nxv2f64(double %s, <vscale x 2 x double> %v)
@@ -388,8 +372,6 @@ define double @vreduce_ord_fwadd_nxv2f64(<vscale x 2 x float> %v, double %s) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fadd.nxv4f64(double, <vscale x 4 x double>)
-
 define double @vreduce_fadd_nxv4f64(<vscale x 4 x double> %v, double %s) {
 ; CHECK-LABEL: 'vreduce_fadd_nxv4f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call reassoc double @llvm.vector.reduce.fadd.nxv4f64(double %s, <vscale x 4 x double> %v)
@@ -448,8 +430,6 @@ define double @vreduce_ord_fwadd_nxv4f64(<vscale x 4 x float> %v, double %s) {
   ret double %red
 }
 
-declare half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half>)
-
 define half @vreduce_fmin_nxv1f16(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv1f16(<vscale x 1 x half> %v)
@@ -489,8 +469,6 @@ define half @vreduce_fmin_nxv1f16_nonans_noinfs(<vscale x 1 x half> %v) #1 {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half>)
-
 define half @vreduce_fmin_nxv2f16(<vscale x 2 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv2f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %v)
@@ -504,8 +482,6 @@ define half @vreduce_fmin_nxv2f16(<vscale x 2 x half> %v) {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half>)
-
 define half @vreduce_fmin_nxv4f16(<vscale x 4 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv4f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %v)
@@ -519,8 +495,6 @@ define half @vreduce_fmin_nxv4f16(<vscale x 4 x half> %v) {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half>)
-
 define half @vreduce_fmin_nxv64f16(<vscale x 64 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv64f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %red = call half @llvm.vector.reduce.fmin.nxv64f16(<vscale x 64 x half> %v)
@@ -534,8 +508,6 @@ define half @vreduce_fmin_nxv64f16(<vscale x 64 x half> %v) {
   ret half %red
 }
 
-declare float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float>)
-
 define float @vreduce_fmin_nxv1f32(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> %v)
@@ -575,8 +547,6 @@ define float @vreduce_fmin_nxv1f32_nonans_noinfs(<vscale x 1 x float> %v) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float>)
-
 define float @vreduce_fmin_nxv2f32(<vscale x 2 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv2f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %v)
@@ -590,8 +560,6 @@ define float @vreduce_fmin_nxv2f32(<vscale x 2 x float> %v) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float>)
-
 define float @vreduce_fmin_nxv4f32(<vscale x 4 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv4f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v)
@@ -605,8 +573,6 @@ define float @vreduce_fmin_nxv4f32(<vscale x 4 x float> %v) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float>)
-
 define float @vreduce_fmin_nxv32f32(<vscale x 32 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv32f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %red = call float @llvm.vector.reduce.fmin.nxv32f32(<vscale x 32 x float> %v)
@@ -620,8 +586,6 @@ define float @vreduce_fmin_nxv32f32(<vscale x 32 x float> %v) {
   ret float %red
 }
 
-declare double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double>)
-
 define double @vreduce_fmin_nxv1f64(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv1f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv1f64(<vscale x 1 x double> %v)
@@ -661,8 +625,6 @@ define double @vreduce_fmin_nxv1f64_nonans_noinfs(<vscale x 1 x double> %v) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double>)
-
 define double @vreduce_fmin_nxv2f64(<vscale x 2 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv2f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %v)
@@ -676,8 +638,6 @@ define double @vreduce_fmin_nxv2f64(<vscale x 2 x double> %v) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double>)
-
 define double @vreduce_fmin_nxv4f64(<vscale x 4 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv4f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v)
@@ -691,8 +651,6 @@ define double @vreduce_fmin_nxv4f64(<vscale x 4 x double> %v) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double>)
-
 define double @vreduce_fmin_nxv16f64(<vscale x 16 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmin_nxv16f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %red = call double @llvm.vector.reduce.fmin.nxv16f64(<vscale x 16 x double> %v)
@@ -706,8 +664,6 @@ define double @vreduce_fmin_nxv16f64(<vscale x 16 x double> %v) {
   ret double %red
 }
 
-declare half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half>)
-
 define half @vreduce_fmax_nxv1f16(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv1f16(<vscale x 1 x half> %v)
@@ -747,8 +703,6 @@ define half @vreduce_fmax_nxv1f16_nonans_noinfs(<vscale x 1 x half> %v) #1 {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half>)
-
 define half @vreduce_fmax_nxv2f16(<vscale x 2 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv2f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %v)
@@ -762,8 +716,6 @@ define half @vreduce_fmax_nxv2f16(<vscale x 2 x half> %v) {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half>)
-
 define half @vreduce_fmax_nxv4f16(<vscale x 4 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv4f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %v)
@@ -777,8 +729,6 @@ define half @vreduce_fmax_nxv4f16(<vscale x 4 x half> %v) {
   ret half %red
 }
 
-declare half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half>)
-
 define half @vreduce_fmax_nxv64f16(<vscale x 64 x half> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv64f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %red = call half @llvm.vector.reduce.fmax.nxv64f16(<vscale x 64 x half> %v)
@@ -792,8 +742,6 @@ define half @vreduce_fmax_nxv64f16(<vscale x 64 x half> %v) {
   ret half %red
 }
 
-declare float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float>)
-
 define float @vreduce_fmax_nxv1f32(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> %v)
@@ -833,8 +781,6 @@ define float @vreduce_fmax_nxv1f32_nonans_noinfs(<vscale x 1 x float> %v) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float>)
-
 define float @vreduce_fmax_nxv2f32(<vscale x 2 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv2f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %v)
@@ -848,8 +794,6 @@ define float @vreduce_fmax_nxv2f32(<vscale x 2 x float> %v) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float>)
-
 define float @vreduce_fmax_nxv4f32(<vscale x 4 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv4f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v)
@@ -863,8 +807,6 @@ define float @vreduce_fmax_nxv4f32(<vscale x 4 x float> %v) {
   ret float %red
 }
 
-declare float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float>)
-
 define float @vreduce_fmax_nxv32f32(<vscale x 32 x float> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv32f32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %red = call float @llvm.vector.reduce.fmax.nxv32f32(<vscale x 32 x float> %v)
@@ -878,8 +820,6 @@ define float @vreduce_fmax_nxv32f32(<vscale x 32 x float> %v) {
   ret float %red
 }
 
-declare double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double>)
-
 define double @vreduce_fmax_nxv1f64(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv1f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv1f64(<vscale x 1 x double> %v)
@@ -919,8 +859,6 @@ define double @vreduce_fmax_nxv1f64_nonans_noinfs(<vscale x 1 x double> %v) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double>)
-
 define double @vreduce_fmax_nxv2f64(<vscale x 2 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv2f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %v)
@@ -934,8 +872,6 @@ define double @vreduce_fmax_nxv2f64(<vscale x 2 x double> %v) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double>)
-
 define double @vreduce_fmax_nxv4f64(<vscale x 4 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv4f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v)
@@ -949,8 +885,6 @@ define double @vreduce_fmax_nxv4f64(<vscale x 4 x double> %v) {
   ret double %red
 }
 
-declare double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double>)
-
 define double @vreduce_fmax_nxv16f64(<vscale x 16 x double> %v) {
 ; CHECK-LABEL: 'vreduce_fmax_nxv16f64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %red = call double @llvm.vector.reduce.fmax.nxv16f64(<vscale x 16 x double> %v)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
index a1a22b56358a3..131e89f2fc2a3 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
@@ -4,8 +4,6 @@
 ; RUN: opt < %s -mtriple=riscv32 -mattr=+v -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=SIZE
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=SIZE
 
-declare i8 @llvm.vector.reduce.add.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_add_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.add.nxv1i8(<vscale x 1 x i8> %v)
@@ -19,8 +17,6 @@ define signext i8 @vreduce_add_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_umax_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv1i8(<vscale x 1 x i8> %v)
@@ -34,8 +30,6 @@ define signext i8 @vreduce_umax_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_smax_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv1i8(<vscale x 1 x i8> %v)
@@ -49,8 +43,6 @@ define signext i8 @vreduce_smax_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_umin_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv1i8(<vscale x 1 x i8> %v)
@@ -64,8 +56,6 @@ define signext i8 @vreduce_umin_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_smin_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv1i8(<vscale x 1 x i8> %v)
@@ -79,8 +69,6 @@ define signext i8 @vreduce_smin_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_and_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv1i8(<vscale x 1 x i8> %v)
@@ -94,8 +82,6 @@ define signext i8 @vreduce_and_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_or_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv1i8(<vscale x 1 x i8> %v)
@@ -109,8 +95,6 @@ define signext i8 @vreduce_or_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.xor.nxv1i8(<vscale x 1 x i8>)
-
 define signext i8 @vreduce_xor_nxv1i8(<vscale x 1 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv1i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.xor.nxv1i8(<vscale x 1 x i8> %v)
@@ -124,8 +108,6 @@ define signext i8 @vreduce_xor_nxv1i8(<vscale x 1 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_add_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8> %v)
@@ -139,8 +121,6 @@ define signext i8 @vreduce_add_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_umax_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv2i8(<vscale x 2 x i8> %v)
@@ -154,8 +134,6 @@ define signext i8 @vreduce_umax_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_smax_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv2i8(<vscale x 2 x i8> %v)
@@ -169,8 +147,6 @@ define signext i8 @vreduce_smax_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_umin_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv2i8(<vscale x 2 x i8> %v)
@@ -184,8 +160,6 @@ define signext i8 @vreduce_umin_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_smin_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv2i8(<vscale x 2 x i8> %v)
@@ -199,8 +173,6 @@ define signext i8 @vreduce_smin_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_and_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv2i8(<vscale x 2 x i8> %v)
@@ -214,8 +186,6 @@ define signext i8 @vreduce_and_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_or_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv2i8(<vscale x 2 x i8> %v)
@@ -229,8 +199,6 @@ define signext i8 @vreduce_or_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8>)
-
 define signext i8 @vreduce_xor_nxv2i8(<vscale x 2 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv2i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8> %v)
@@ -244,8 +212,6 @@ define signext i8 @vreduce_xor_nxv2i8(<vscale x 2 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_add_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8> %v)
@@ -259,8 +225,6 @@ define signext i8 @vreduce_add_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_umax_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.umax.nxv4i8(<vscale x 4 x i8> %v)
@@ -274,8 +238,6 @@ define signext i8 @vreduce_umax_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_smax_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.smax.nxv4i8(<vscale x 4 x i8> %v)
@@ -289,8 +251,6 @@ define signext i8 @vreduce_smax_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_umin_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.umin.nxv4i8(<vscale x 4 x i8> %v)
@@ -304,8 +264,6 @@ define signext i8 @vreduce_umin_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_smin_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.smin.nxv4i8(<vscale x 4 x i8> %v)
@@ -319,8 +277,6 @@ define signext i8 @vreduce_smin_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_and_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.and.nxv4i8(<vscale x 4 x i8> %v)
@@ -334,8 +290,6 @@ define signext i8 @vreduce_and_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_or_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i8 @llvm.vector.reduce.or.nxv4i8(<vscale x 4 x i8> %v)
@@ -349,8 +303,6 @@ define signext i8 @vreduce_or_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i8 @llvm.vector.reduce.xor.nxv4i8(<vscale x 4 x i8>)
-
 define signext i8 @vreduce_xor_nxv4i8(<vscale x 4 x i8> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv4i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i8 @llvm.vector.reduce.xor.nxv4i8(<vscale x 4 x i8> %v)
@@ -364,8 +316,6 @@ define signext i8 @vreduce_xor_nxv4i8(<vscale x 4 x i8> %v) {
   ret i8 %red
 }
 
-declare i16 @llvm.vector.reduce.add.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_add_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.add.nxv1i16(<vscale x 1 x i16> %v)
@@ -411,8 +361,6 @@ define signext i16 @vwreduce_uadd_nxv1i8(<vscale x 1 x i8> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_umax_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv1i16(<vscale x 1 x i16> %v)
@@ -426,8 +374,6 @@ define signext i16 @vreduce_umax_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_smax_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv1i16(<vscale x 1 x i16> %v)
@@ -441,8 +387,6 @@ define signext i16 @vreduce_smax_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_umin_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv1i16(<vscale x 1 x i16> %v)
@@ -456,8 +400,6 @@ define signext i16 @vreduce_umin_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_smin_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv1i16(<vscale x 1 x i16> %v)
@@ -471,8 +413,6 @@ define signext i16 @vreduce_smin_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_and_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv1i16(<vscale x 1 x i16> %v)
@@ -486,8 +426,6 @@ define signext i16 @vreduce_and_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_or_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv1i16(<vscale x 1 x i16> %v)
@@ -501,8 +439,6 @@ define signext i16 @vreduce_or_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.xor.nxv1i16(<vscale x 1 x i16>)
-
 define signext i16 @vreduce_xor_nxv1i16(<vscale x 1 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv1i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.xor.nxv1i16(<vscale x 1 x i16> %v)
@@ -516,8 +452,6 @@ define signext i16 @vreduce_xor_nxv1i16(<vscale x 1 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.add.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_add_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.add.nxv2i16(<vscale x 2 x i16> %v)
@@ -563,8 +497,6 @@ define signext i16 @vwreduce_uadd_nxv2i8(<vscale x 2 x i8> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_umax_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv2i16(<vscale x 2 x i16> %v)
@@ -578,8 +510,6 @@ define signext i16 @vreduce_umax_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_smax_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv2i16(<vscale x 2 x i16> %v)
@@ -593,8 +523,6 @@ define signext i16 @vreduce_smax_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_umin_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv2i16(<vscale x 2 x i16> %v)
@@ -608,8 +536,6 @@ define signext i16 @vreduce_umin_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_smin_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv2i16(<vscale x 2 x i16> %v)
@@ -623,8 +549,6 @@ define signext i16 @vreduce_smin_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_and_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv2i16(<vscale x 2 x i16> %v)
@@ -638,8 +562,6 @@ define signext i16 @vreduce_and_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_or_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> %v)
@@ -653,8 +575,6 @@ define signext i16 @vreduce_or_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.xor.nxv2i16(<vscale x 2 x i16>)
-
 define signext i16 @vreduce_xor_nxv2i16(<vscale x 2 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv2i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.xor.nxv2i16(<vscale x 2 x i16> %v)
@@ -668,8 +588,6 @@ define signext i16 @vreduce_xor_nxv2i16(<vscale x 2 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.add.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.add.nxv4i16(<vscale x 4 x i16> %v)
@@ -715,8 +633,6 @@ define signext i16 @vwreduce_uadd_nxv4i8(<vscale x 4 x i8> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_umax_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.umax.nxv4i16(<vscale x 4 x i16> %v)
@@ -730,8 +646,6 @@ define signext i16 @vreduce_umax_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_smax_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.smax.nxv4i16(<vscale x 4 x i16> %v)
@@ -745,8 +659,6 @@ define signext i16 @vreduce_smax_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_umin_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.umin.nxv4i16(<vscale x 4 x i16> %v)
@@ -760,8 +672,6 @@ define signext i16 @vreduce_umin_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_smin_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.smin.nxv4i16(<vscale x 4 x i16> %v)
@@ -775,8 +685,6 @@ define signext i16 @vreduce_smin_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_and_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.and.nxv4i16(<vscale x 4 x i16> %v)
@@ -790,8 +698,6 @@ define signext i16 @vreduce_and_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_or_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> %v)
@@ -805,8 +711,6 @@ define signext i16 @vreduce_or_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i16 @llvm.vector.reduce.xor.nxv4i16(<vscale x 4 x i16>)
-
 define signext i16 @vreduce_xor_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv4i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i16 @llvm.vector.reduce.xor.nxv4i16(<vscale x 4 x i16> %v)
@@ -820,8 +724,6 @@ define signext i16 @vreduce_xor_nxv4i16(<vscale x 4 x i16> %v) {
   ret i16 %red
 }
 
-declare i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_add_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> %v)
@@ -867,8 +769,6 @@ define signext i32 @vwreduce_uadd_nxv1i16(<vscale x 1 x i16> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_umax_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv1i32(<vscale x 1 x i32> %v)
@@ -882,8 +782,6 @@ define signext i32 @vreduce_umax_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_smax_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv1i32(<vscale x 1 x i32> %v)
@@ -897,8 +795,6 @@ define signext i32 @vreduce_smax_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_umin_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv1i32(<vscale x 1 x i32> %v)
@@ -912,8 +808,6 @@ define signext i32 @vreduce_umin_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_smin_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv1i32(<vscale x 1 x i32> %v)
@@ -927,8 +821,6 @@ define signext i32 @vreduce_smin_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_and_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> %v)
@@ -942,8 +834,6 @@ define signext i32 @vreduce_and_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_or_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> %v)
@@ -957,8 +847,6 @@ define signext i32 @vreduce_or_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32>)
-
 define signext i32 @vreduce_xor_nxv1i32(<vscale x 1 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv1i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> %v)
@@ -972,8 +860,6 @@ define signext i32 @vreduce_xor_nxv1i32(<vscale x 1 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> %v)
@@ -1019,8 +905,6 @@ define signext i32 @vwreduce_uadd_nxv2i16(<vscale x 2 x i16> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_umax_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv2i32(<vscale x 2 x i32> %v)
@@ -1034,8 +918,6 @@ define signext i32 @vreduce_umax_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_smax_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv2i32(<vscale x 2 x i32> %v)
@@ -1049,8 +931,6 @@ define signext i32 @vreduce_smax_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_umin_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv2i32(<vscale x 2 x i32> %v)
@@ -1064,8 +944,6 @@ define signext i32 @vreduce_umin_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_smin_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv2i32(<vscale x 2 x i32> %v)
@@ -1079,8 +957,6 @@ define signext i32 @vreduce_smin_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_and_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv2i32(<vscale x 2 x i32> %v)
@@ -1094,8 +970,6 @@ define signext i32 @vreduce_and_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_or_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv2i32(<vscale x 2 x i32> %v)
@@ -1109,8 +983,6 @@ define signext i32 @vreduce_or_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.xor.nxv2i32(<vscale x 2 x i32>)
-
 define signext i32 @vreduce_xor_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv2i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.xor.nxv2i32(<vscale x 2 x i32> %v)
@@ -1124,8 +996,6 @@ define signext i32 @vreduce_xor_nxv2i32(<vscale x 2 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v)
@@ -1171,8 +1041,6 @@ define signext i32 @vwreduce_uadd_nxv4i16(<vscale x 4 x i16> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_umax_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v)
@@ -1186,8 +1054,6 @@ define signext i32 @vreduce_umax_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_smax_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v)
@@ -1201,8 +1067,6 @@ define signext i32 @vreduce_smax_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_umin_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v)
@@ -1216,8 +1080,6 @@ define signext i32 @vreduce_umin_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_smin_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v)
@@ -1231,8 +1093,6 @@ define signext i32 @vreduce_smin_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_and_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v)
@@ -1246,8 +1106,6 @@ define signext i32 @vreduce_and_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_or_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v)
@@ -1261,8 +1119,6 @@ define signext i32 @vreduce_or_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32>)
-
 define signext i32 @vreduce_xor_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv4i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v)
@@ -1276,8 +1132,6 @@ define signext i32 @vreduce_xor_nxv4i32(<vscale x 4 x i32> %v) {
   ret i32 %red
 }
 
-declare i64 @llvm.vector.reduce.add.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_add_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv1i64(<vscale x 1 x i64> %v)
@@ -1323,8 +1177,6 @@ define i64 @vwreduce_uadd_nxv1i32(<vscale x 1 x i32> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_umax_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> %v)
@@ -1338,8 +1190,6 @@ define i64 @vreduce_umax_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_smax_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> %v)
@@ -1353,8 +1203,6 @@ define i64 @vreduce_smax_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_umin_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> %v)
@@ -1368,8 +1216,6 @@ define i64 @vreduce_umin_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_smin_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> %v)
@@ -1383,8 +1229,6 @@ define i64 @vreduce_smin_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_and_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv1i64(<vscale x 1 x i64> %v)
@@ -1398,8 +1242,6 @@ define i64 @vreduce_and_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_or_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv1i64(<vscale x 1 x i64> %v)
@@ -1413,8 +1255,6 @@ define i64 @vreduce_or_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.xor.nxv1i64(<vscale x 1 x i64>)
-
 define i64 @vreduce_xor_nxv1i64(<vscale x 1 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv1i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.xor.nxv1i64(<vscale x 1 x i64> %v)
@@ -1428,8 +1268,6 @@ define i64 @vreduce_xor_nxv1i64(<vscale x 1 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_add_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %v)
@@ -1475,8 +1313,6 @@ define i64 @vwreduce_uadd_nxv2i32(<vscale x 2 x i32> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_umax_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv2i64(<vscale x 2 x i64> %v)
@@ -1490,8 +1326,6 @@ define i64 @vreduce_umax_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_smax_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv2i64(<vscale x 2 x i64> %v)
@@ -1505,8 +1339,6 @@ define i64 @vreduce_smax_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_umin_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> %v)
@@ -1520,8 +1352,6 @@ define i64 @vreduce_umin_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_smin_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> %v)
@@ -1535,8 +1365,6 @@ define i64 @vreduce_smin_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_and_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> %v)
@@ -1550,8 +1378,6 @@ define i64 @vreduce_and_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_or_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> %v)
@@ -1565,8 +1391,6 @@ define i64 @vreduce_or_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.xor.nxv2i64(<vscale x 2 x i64>)
-
 define i64 @vreduce_xor_nxv2i64(<vscale x 2 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv2i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.xor.nxv2i64(<vscale x 2 x i64> %v)
@@ -1580,8 +1404,6 @@ define i64 @vreduce_xor_nxv2i64(<vscale x 2 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_add_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_add_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v)
@@ -1627,8 +1449,6 @@ define i64 @vwreduce_uadd_nxv4i32(<vscale x 4 x i32> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_umax_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umax_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v)
@@ -1642,8 +1462,6 @@ define i64 @vreduce_umax_nxv4i64(<vscale x 4 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_smax_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smax_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v)
@@ -1657,8 +1475,6 @@ define i64 @vreduce_smax_nxv4i64(<vscale x 4 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_umin_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_umin_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v)
@@ -1672,8 +1488,6 @@ define i64 @vreduce_umin_nxv4i64(<vscale x 4 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_smin_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_smin_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v)
@@ -1687,8 +1501,6 @@ define i64 @vreduce_smin_nxv4i64(<vscale x 4 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_and_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_and_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v)
@@ -1702,8 +1514,6 @@ define i64 @vreduce_and_nxv4i64(<vscale x 4 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_or_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_or_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v)
@@ -1717,8 +1527,6 @@ define i64 @vreduce_or_nxv4i64(<vscale x 4 x i64> %v) {
   ret i64 %red
 }
 
-declare i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64>)
-
 define i64 @vreduce_xor_nxv4i64(<vscale x 4 x i64> %v) {
 ; CHECK-LABEL: 'vreduce_xor_nxv4i64'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-xor.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-xor.ll
index aa03b02895d5f..abc5335afbc98 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-xor.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-xor.ll
@@ -173,44 +173,3 @@ define i32 @reduce_i64(i32 %arg) {
   %V128 = call i64 @llvm.vector.reduce.xor.v128i64(<128 x i64> undef)
   ret i32 undef
 }
-
-declare i1 @llvm.vector.reduce.xor.v1i1(<1 x i1>)
-declare i1 @llvm.vector.reduce.xor.v2i1(<2 x i1>)
-declare i1 @llvm.vector.reduce.xor.v4i1(<4 x i1>)
-declare i1 @llvm.vector.reduce.xor.v8i1(<8 x i1>)
-declare i1 @llvm.vector.reduce.xor.v16i1(<16 x i1>)
-declare i1 @llvm.vector.reduce.xor.v32i1(<32 x i1>)
-declare i1 @llvm.vector.reduce.xor.v64i1(<64 x i1>)
-declare i1 @llvm.vector.reduce.xor.v128i1(<128 x i1>)
-declare i8 @llvm.vector.reduce.xor.v1i8(<1 x i8>)
-declare i8 @llvm.vector.reduce.xor.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.xor.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.xor.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.xor.v16i8(<16 x i8>)
-declare i8 @llvm.vector.reduce.xor.v32i8(<32 x i8>)
-declare i8 @llvm.vector.reduce.xor.v64i8(<64 x i8>)
-declare i8 @llvm.vector.reduce.xor.v128i8(<128 x i8>)
-declare i16 @llvm.vector.reduce.xor.v1i16(<1 x i16>)
-declare i16 @llvm.vector.reduce.xor.v2i16(<2 x i16>)
-declare i16 @llvm.vector.reduce.xor.v4i16(<4 x i16>)
-declare i16 @llvm.vector.reduce.xor.v8i16(<8 x i16>)
-declare i16 @llvm.vector.reduce.xor.v16i16(<16 x i16>)
-declare i16 @llvm.vector.reduce.xor.v32i16(<32 x i16>)
-declare i16 @llvm.vector.reduce.xor.v64i16(<64 x i16>)
-declare i16 @llvm.vector.reduce.xor.v128i16(<128 x i16>)
-declare i32 @llvm.vector.reduce.xor.v1i32(<1 x i32>)
-declare i32 @llvm.vector.reduce.xor.v2i32(<2 x i32>)
-declare i32 @llvm.vector.reduce.xor.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.xor.v8i32(<8 x i32>)
-declare i32 @llvm.vector.reduce.xor.v16i32(<16 x i32>)
-declare i32 @llvm.vector.reduce.xor.v32i32(<32 x i32>)
-declare i32 @llvm.vector.reduce.xor.v64i32(<64 x i32>)
-declare i32 @llvm.vector.reduce.xor.v128i32(<128 x i32>)
-declare i64 @llvm.vector.reduce.xor.v1i64(<1 x i64>)
-declare i64 @llvm.vector.reduce.xor.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.xor.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.xor.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.xor.v16i64(<16 x i64>)
-declare i64 @llvm.vector.reduce.xor.v32i64(<32 x i64>)
-declare i64 @llvm.vector.reduce.xor.v64i64(<64 x i64>)
-declare i64 @llvm.vector.reduce.xor.v128i64(<128 x i64>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
index e0f80f541bc19..94e2390a12fc1 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
@@ -142,36 +142,3 @@ define void @compress_store() {
   ret void
 }
 
-declare <2 x i8> @llvm.masked.expandload.v2i8(ptr, <2 x i1>, <2 x i8>)
-declare <4 x i8> @llvm.masked.expandload.v4i8(ptr, <4 x i1>, <4 x i8>)
-declare <8 x i8> @llvm.masked.expandload.v8i8(ptr, <8 x i1>, <8 x i8>)
-declare <16 x i8> @llvm.masked.expandload.v16i8(ptr, <16 x i1>, <16 x i8>)
-declare <2 x i64> @llvm.masked.expandload.v2i64(ptr, <2 x i1>, <2 x i64>)
-declare <4 x i64> @llvm.masked.expandload.v4i64(ptr, <4 x i1>, <4 x i64>)
-declare <8 x i64> @llvm.masked.expandload.v8i64(ptr, <8 x i1>, <8 x i64>)
-declare <16 x i64> @llvm.masked.expandload.v16i64(ptr, <16 x i1>, <16 x i64>)
-declare <vscale x 2 x i8> @llvm.masked.expandload.nxv2i8(ptr, <vscale x 2 x i1>, <vscale x 2 x i8>)
-declare <vscale x 4 x i8> @llvm.masked.expandload.nxv4i8(ptr, <vscale x 4 x i1>, <vscale x 4 x i8>)
-declare <vscale x 8 x i8> @llvm.masked.expandload.nxv8i8(ptr, <vscale x 8 x i1>, <vscale x 8 x i8>)
-declare <vscale x 16 x i8> @llvm.masked.expandload.nxv16i8(ptr, <vscale x 16 x i1>, <vscale x 16 x i8>)
-declare <vscale x 2 x i64> @llvm.masked.expandload.nxv2i64(ptr, <vscale x 2 x i1>, <vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.masked.expandload.nxv4i64(ptr, <vscale x 4 x i1>, <vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.masked.expandload.nxv8i64(ptr, <vscale x 8 x i1>, <vscale x 8 x i64>)
-declare <vscale x 16 x i64> @llvm.masked.expandload.nxv16i64(ptr, <vscale x 16 x i1>, <vscale x 16 x i64>)
-
-declare void @llvm.masked.compressstore.v2i8(<2 x i8>, ptr, <2 x i1>)
-declare void @llvm.masked.compressstore.v4i8(<4 x i8>, ptr, <4 x i1>)
-declare void @llvm.masked.compressstore.v8i8(<8 x i8>, ptr, <8 x i1>)
-declare void @llvm.masked.compressstore.v16i8(<16 x i8>, ptr, <16 x i1>)
-declare void @llvm.masked.compressstore.v2i64(<2 x i64>, ptr, <2 x i1>)
-declare void @llvm.masked.compressstore.v4i64(<4 x i64>, ptr, <4 x i1>)
-declare void @llvm.masked.compressstore.v8i64(<8 x i64>, ptr, <8 x i1>)
-declare void @llvm.masked.compressstore.v16i64(<16 x i64>, ptr, <16 x i1>)
-declare void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8>, ptr, <vscale x 2 x i1>)
-declare void @llvm.masked.compressstore.nxv4i8(<vscale x 4 x i8>, ptr, <vscale x 4 x i1>)
-declare void @llvm.masked.compressstore.nxv8i8(<vscale x 8 x i8>, ptr, <vscale x 8 x i1>)
-declare void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8>, ptr, <vscale x 16 x i1>)
-declare void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64>, ptr, <vscale x 2 x i1>)
-declare void @llvm.masked.compressstore.nxv4i64(<vscale x 4 x i64>, ptr, <vscale x 4 x i1>)
-declare void @llvm.masked.compressstore.nxv8i64(<vscale x 8 x i64>, ptr, <vscale x 8 x i1>)
-declare void @llvm.masked.compressstore.nxv16i64(<vscale x 16 x i64>, ptr, <vscale x 16 x i1>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll
index 437a9af8fcc83..ece528d9a6030 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-shuffle.ll
@@ -71,10 +71,6 @@ define void @vector_insert_extract(<vscale x 4 x i32> %v0, <vscale x 16 x i32> %
   %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
   ret void
 }
-declare <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32>, i64)
-declare <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32>, <16 x i32>, i64)
-declare <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32>, i64)
-declare <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32>, <vscale x 4 x i32>, i64)
 
 define void @vector_reverse() {
 ; CHECK-LABEL: 'vector_reverse'
@@ -143,25 +139,6 @@ define void @vector_reverse() {
   ret void
 }
 
-declare <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8>)
-declare <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8>)
-declare <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16>)
-declare <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16>)
-declare <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16>)
-declare <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16>)
-declare <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32>)
-declare <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32>)
-declare <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64>)
-declare <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64>)
-declare <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64>)
-declare <vscale x 16 x i64> @llvm.vector.reverse.nxv16i64(<vscale x 16 x i64>)
-declare <vscale x 32 x i64> @llvm.vector.reverse.nxv32i64(<vscale x 32 x i64>)
-declare <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1>)
-declare <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1>)
-declare <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1>)
-declare <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1>)
-
-
 define void @vector_splice() {
 ; CHECK-LABEL: 'vector_splice'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
@@ -213,19 +190,3 @@ define void @vector_splice() {
   %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
   ret void
 }
-
-declare <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vector.splice.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>, i32)
-declare <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, i32)
-declare <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8>, <vscale x 32 x i8>, i32)
-declare <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>, i32)
-declare <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>, i32)
-declare <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, i32)
-declare <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>, i32)
-declare <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)
-declare <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>, i32)
-declare <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, i32)
-declare <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>, i32)
diff --git a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
index 04b1d18fca8ee..ea3c47dc34201 100644
--- a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
@@ -1648,275 +1648,3 @@ define void @splice() {
   %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1, <vscale x 2 x i1> zeroinitializer, i32 poison, i32 poison)
   ret void
 }
-
-declare <2 x i8> @llvm.vp.add.v2i8(<2 x i8>, <2 x i8>, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.add.v4i8(<4 x i8>, <4 x i8>, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.add.v8i8(<8 x i8>, <8 x i8>, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.add.v16i8(<16 x i8>, <16 x i8>, <16 x i1>, i32)
-declare <2 x i64> @llvm.vp.add.v2i64(<2 x i64>, <2 x i64>, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.add.v4i64(<4 x i64>, <4 x i64>, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.add.v8i64(<8 x i64>, <8 x i64>, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.add.v16i64(<16 x i64>, <16 x i64>, <16 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.add.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.add.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.add.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.add.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.add.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.add.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.add.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.add.nxv16i64(<vscale x 16 x i64>, <vscale x 16 x i64>, <vscale x 16 x i1>, i32)
-
-declare <2 x i8> @llvm.vp.abs.v2i8(<2 x i8>, i1, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.abs.v4i8(<4 x i8>, i1, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.abs.v8i8(<8 x i8>, i1, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.abs.v16i8(<16 x i8>, i1, <16 x i1>, i32)
-declare <2 x i64> @llvm.vp.abs.v2i64(<2 x i64>, i1, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.abs.v4i64(<4 x i64>, i1, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.abs.v8i64(<8 x i64>, i1, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.abs.v16i64(<16 x i64>, i1, <16 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.abs.nxv2i8(<vscale x 2 x i8>, i1, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.abs.nxv4i8(<vscale x 4 x i8>, i1, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.abs.nxv8i8(<vscale x 8 x i8>, i1, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.abs.nxv16i8(<vscale x 16 x i8>, i1, <vscale x 16 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.abs.nxv2i64(<vscale x 2 x i64>, i1, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.abs.nxv4i64(<vscale x 4 x i64>, i1, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.abs.nxv8i64(<vscale x 8 x i64>, i1, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.abs.nxv16i64(<vscale x 16 x i64>, i1, <vscale x 16 x i1>, i32)
-
-declare <2 x i8> @llvm.abs.v2i8(<2 x i8>, i1)
-declare <4 x i8> @llvm.abs.v4i8(<4 x i8>, i1)
-declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
-declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
-declare <2 x i64> @llvm.abs.v2i64(<2 x i64>, i1)
-declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1)
-declare <8 x i64> @llvm.abs.v8i64(<8 x i64>, i1)
-declare <16 x i64> @llvm.abs.v16i64(<16 x i64>, i1)
-declare <vscale x 2 x i8> @llvm.abs.nxv2i8(<vscale x 2 x i8>, i1)
-declare <vscale x 4 x i8> @llvm.abs.nxv4i8(<vscale x 4 x i8>, i1)
-declare <vscale x 8 x i8> @llvm.abs.nxv8i8(<vscale x 8 x i8>, i1)
-declare <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8>, i1)
-declare <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64>, i1)
-declare <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64>, i1)
-declare <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64>, i1)
-declare <vscale x 16 x i64> @llvm.abs.nxv16i64(<vscale x 16 x i64>, i1)
-
-declare <2 x i8> @llvm.vp.load.v2i8(ptr, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.load.v4i8(ptr, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.load.v8i8(ptr, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.load.v16i8(ptr, <16 x i1>, i32)
-declare <2 x i64> @llvm.vp.load.v2i64(ptr, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.load.v4i64(ptr, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.load.v8i64(ptr, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.load.v16i64(ptr, <16 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.load.nxv2i8(ptr, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.load.nxv4i8(ptr, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.load.nxv8i8(ptr, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.load.nxv16i8(ptr, <vscale x 16 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.load.nxv2i64(ptr, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.load.nxv4i64(ptr, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.load.nxv8i64(ptr, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.vp.load.nxv16i64(ptr, <vscale x 16 x i1>, i32)
-
-declare void @llvm.vp.store.v2i8(<2 x i8>, ptr, <2 x i1>, i32)
-declare void @llvm.vp.store.v4i8(<4 x i8>, ptr, <4 x i1>, i32)
-declare void @llvm.vp.store.v8i8(<8 x i8>, ptr, <8 x i1>, i32)
-declare void @llvm.vp.store.v16i8(<16 x i8>, ptr, <16 x i1>, i32)
-declare void @llvm.vp.store.v2i64(<2 x i64>, ptr, <2 x i1>, i32)
-declare void @llvm.vp.store.v4i64(<4 x i64>, ptr, <4 x i1>, i32)
-declare void @llvm.vp.store.v8i64(<8 x i64>, ptr, <8 x i1>, i32)
-declare void @llvm.vp.store.v16i64(<16 x i64>, ptr, <16 x i1>, i32)
-declare void @llvm.vp.store.nxv2i8(<vscale x 2 x i8>, ptr, <vscale x 2 x i1>, i32)
-declare void @llvm.vp.store.nxv4i8(<vscale x 4 x i8>, ptr, <vscale x 4 x i1>, i32)
-declare void @llvm.vp.store.nxv8i8(<vscale x 8 x i8>, ptr, <vscale x 8 x i1>, i32)
-declare void @llvm.vp.store.nxv16i8(<vscale x 16 x i8>, ptr, <vscale x 16 x i1>, i32)
-declare void @llvm.vp.store.nxv2i64(<vscale x 2 x i64>, ptr, <vscale x 2 x i1>, i32)
-declare void @llvm.vp.store.nxv4i64(<vscale x 4 x i64>, ptr, <vscale x 4 x i1>, i32)
-declare void @llvm.vp.store.nxv8i64(<vscale x 8 x i64>, ptr, <vscale x 8 x i1>, i32)
-declare void @llvm.vp.store.nxv16i64(<vscale x 16 x i64>, ptr, <vscale x 16 x i1>, i32)
-
-declare <2 x i8> @llvm.experimental.vp.strided.load.v2i8.i64(ptr, i64, <2 x i1>, i32)
-declare <4 x i8> @llvm.experimental.vp.strided.load.v4i8.i64(ptr, i64, <4 x i1>, i32)
-declare <8 x i8> @llvm.experimental.vp.strided.load.v8i8.i64(ptr, i64, <8 x i1>, i32)
-declare <16 x i8> @llvm.experimental.vp.strided.load.v16i8.i64(ptr, i64, <16 x i1>, i32)
-declare <2 x i64> @llvm.experimental.vp.strided.load.v2i64.i64(ptr, i64, <2 x i1>, i32)
-declare <4 x i64> @llvm.experimental.vp.strided.load.v4i64.i64(ptr, i64, <4 x i1>, i32)
-declare <8 x i64> @llvm.experimental.vp.strided.load.v8i64.i64(ptr, i64, <8 x i1>, i32)
-declare <16 x i64> @llvm.experimental.vp.strided.load.v16i64.i64(ptr, i64, <16 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.experimental.vp.strided.load.nxv2i8.i64(ptr, i64, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.experimental.vp.strided.load.nxv4i8.i64(ptr, i64, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.i64(ptr, i64, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.experimental.vp.strided.load.nxv16i8.i64(ptr, i64, <vscale x 16 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.i64(ptr, i64, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.i64(ptr, i64, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.experimental.vp.strided.load.nxv8i64.i64(ptr, i64, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i64> @llvm.experimental.vp.strided.load.nxv16i64.i64(ptr, i64, <vscale x 16 x i1>, i32)
-
-declare void @llvm.experimental.vp.strided.store.v2i8.i64(<2 x i8>, ptr, i64, <2 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v4i8.i64(<4 x i8>, ptr, i64, <4 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v8i8.i64(<8 x i8>, ptr, i64, <8 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v16i8.i64(<16 x i8>, ptr, i64, <16 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v2i64.i64(<2 x i64>, ptr, i64, <2 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v4i64.i64(<4 x i64>, ptr, i64, <4 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v8i64.i64(<8 x i64>, ptr, i64, <8 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.v16i64.i64(<16 x i64>, ptr, i64, <16 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv2i8.i64(<vscale x 2 x i8>, ptr, i64, <vscale x 2 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv4i8.i64(<vscale x 4 x i8>, ptr, i64, <vscale x 4 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv8i8.i64(<vscale x 8 x i8>, ptr, i64, <vscale x 8 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv16i8.i64(<vscale x 16 x i8>, ptr, i64, <vscale x 16 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv2i64.i64(<vscale x 2 x i64>, ptr, i64, <vscale x 2 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv4i64.i64(<vscale x 4 x i64>, ptr, i64, <vscale x 4 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv8i64.i64(<vscale x 8 x i64>, ptr, i64, <vscale x 8 x i1>, i32)
-declare void @llvm.experimental.vp.strided.store.nxv16i64.i64(<vscale x 16 x i64>, ptr, i64, <vscale x 16 x i1>, i32)
-
-declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
-declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)
-declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
-declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
-declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
-declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
-declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
-declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
-declare i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8>)
-declare i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8>)
-declare i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8>)
-declare i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8>)
-declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)
-declare i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64>)
-declare i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64>)
-declare i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64>)
-
-declare i8 @llvm.vp.reduce.add.v2i8(i8, <2 x i8>, <2 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.v4i8(i8, <4 x i8>, <4 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.v8i8(i8, <8 x i8>, <8 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.v16i8(i8, <16 x i8>, <16 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.v2i64(i64, <2 x i64>, <2 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.v4i64(i64, <4 x i64>, <4 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.v8i64(i64, <8 x i64>, <8 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.v16i64(i64, <16 x i64>, <16 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.nxv2i8(i8, <vscale x 2 x i8>, <vscale x 2 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.nxv4i8(i8, <vscale x 4 x i8>, <vscale x 4 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.nxv8i8(i8, <vscale x 8 x i8>, <vscale x 8 x i1>, i32)
-declare i8 @llvm.vp.reduce.add.nxv16i8(i8, <vscale x 16 x i8>, <vscale x 16 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.nxv2i64(i64, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.nxv4i64(i64, <vscale x 4 x i64>, <vscale x 4 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.nxv8i64(i64, <vscale x 8 x i64>, <vscale x 8 x i1>, i32)
-declare i64 @llvm.vp.reduce.add.nxv16i64(i64, <vscale x 16 x i64>, <vscale x 16 x i1>, i32)
-
-declare float @llvm.vector.reduce.fadd.v2f32(float, <2 x float>)
-declare float @llvm.vector.reduce.fadd.v4f32(float, <4 x float>)
-declare float @llvm.vector.reduce.fadd.v8f32(float, <8 x float>)
-declare float @llvm.vector.reduce.fadd.v16f32(float, <16 x float>)
-declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)
-declare double @llvm.vector.reduce.fadd.v4f64(double, <4 x double>)
-declare double @llvm.vector.reduce.fadd.v8f64(double, <8 x double>)
-declare double @llvm.vector.reduce.fadd.v16f64(double, <16 x double>)
-declare float @llvm.vector.reduce.fadd.nxv2f32(float, <vscale x 2 x float>)
-declare float @llvm.vector.reduce.fadd.nxv4f32(float, <vscale x 4 x float>)
-declare float @llvm.vector.reduce.fadd.nxv8f32(float, <vscale x 8 x float>)
-declare float @llvm.vector.reduce.fadd.nxv16f32(float, <vscale x 16 x float>)
-declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
-declare double @llvm.vector.reduce.fadd.nxv4f64(double, <vscale x 4 x double>)
-declare double @llvm.vector.reduce.fadd.nxv8f64(double, <vscale x 8 x double>)
-declare double @llvm.vector.reduce.fadd.nxv16f64(double, <vscale x 16 x double>)
-
-declare float @llvm.vp.reduce.fadd.v2f32(float, <2 x float>, <2 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.v4f32(float, <4 x float>, <4 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.v8f32(float, <8 x float>, <8 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.v16f32(float, <16 x float>, <16 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.v2f64(double, <2 x double>, <2 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.v4f64(double, <4 x double>, <4 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.v8f64(double, <8 x double>, <8 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.v16f64(double, <16 x double>, <16 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.nxv2f32(float, <vscale x 2 x float>, <vscale x 2 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.nxv4f32(float, <vscale x 4 x float>, <vscale x 4 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.nxv8f32(float, <vscale x 8 x float>, <vscale x 8 x i1>, i32)
-declare float @llvm.vp.reduce.fadd.nxv16f32(float, <vscale x 16 x float>, <vscale x 16 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.nxv2f64(double, <vscale x 2 x double>, <vscale x 2 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.nxv4f64(double, <vscale x 4 x double>, <vscale x 4 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.nxv8f64(double, <vscale x 8 x double>, <vscale x 8 x i1>, i32)
-declare double @llvm.vp.reduce.fadd.nxv16f64(double, <vscale x 16 x double>, <vscale x 16 x i1>, i32)
-
-declare <vscale x 1 x i32> @llvm.fshr.nxv4i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
-declare <vscale x 1 x i32> @llvm.fshl.nxv4i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
-
-declare <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
-declare <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float>, i32)
-declare <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float>)
-
-declare <2 x i8> @llvm.vp.fshr.v2i8(<2 x i8>, <2 x i8>, <2 x i8>, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.fshr.v4i8(<4 x i8>, <4 x i8>, <4 x i8>, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.fshr.v8i8(<8 x i8>, <8 x i8>, <8 x i8>, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.fshr.v16i8(<16 x i8>, <16 x i8>, <16 x i8>, <16 x i1>, i32)
-declare <vscale x 1 x i8> @llvm.vp.fshr.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i8>, <vscale x 1 x i8>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.fshr.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>, <vscale x 2 x i8>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.fshr.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.fshr.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.fshr.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i8> @llvm.vp.fshr.nxv32i8(<vscale x 32 x i8>, <vscale x 32 x i8>, <vscale x 32 x i8>, <vscale x 32 x i1>, i32)
-declare <vscale x 64 x i8> @llvm.vp.fshr.nxv64i8(<vscale x 64 x i8>, <vscale x 64 x i8>, <vscale x 64 x i8>, <vscale x 64 x i1>, i32)
-declare <2 x i16> @llvm.vp.fshr.v2i16(<2 x i16>, <2 x i16>, <2 x i16>, <2 x i1>, i32)
-declare <4 x i16> @llvm.vp.fshr.v4i16(<4 x i16>, <4 x i16>, <4 x i16>, <4 x i1>, i32)
-declare <8 x i16> @llvm.vp.fshr.v8i16(<8 x i16>, <8 x i16>, <8 x i16>, <8 x i1>, i32)
-declare <16 x i16> @llvm.vp.fshr.v16i16(<16 x i16>, <16 x i16>, <16 x i16>, <16 x i1>, i32)
-declare <vscale x 1 x i16> @llvm.vp.fshr.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i16>, <vscale x 1 x i16>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i16> @llvm.vp.fshr.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i16> @llvm.vp.fshr.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i16> @llvm.vp.fshr.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i16> @llvm.vp.fshr.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>, <vscale x 16 x i16>, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i16> @llvm.vp.fshr.nxv32i16(<vscale x 32 x i16>, <vscale x 32 x i16>, <vscale x 32 x i16>, <vscale x 32 x i1>, i32)
-declare <2 x i32> @llvm.vp.fshr.v2i32(<2 x i32>, <2 x i32>, <2 x i32>, <2 x i1>, i32)
-declare <4 x i32> @llvm.vp.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>, <4 x i1>, i32)
-declare <8 x i32> @llvm.vp.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>, <8 x i1>, i32)
-declare <16 x i32> @llvm.vp.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>, <16 x i1>, i32)
-declare <vscale x 1 x i32> @llvm.vp.fshr.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i32> @llvm.vp.fshr.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i32> @llvm.vp.fshr.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i32> @llvm.vp.fshr.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i32> @llvm.vp.fshr.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>, <vscale x 16 x i32>, <vscale x 16 x i1>, i32)
-declare <2 x i64> @llvm.vp.fshr.v2i64(<2 x i64>, <2 x i64>, <2 x i64>, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.fshr.v4i64(<4 x i64>, <4 x i64>, <4 x i64>, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.fshr.v16i64(<16 x i64>, <16 x i64>, <16 x i64>, <16 x i1>, i32)
-declare <vscale x 1 x i64> @llvm.vp.fshr.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.fshr.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.fshr.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>, <vscale x 4 x i64>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.fshr.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>, <vscale x 8 x i64>, <vscale x 8 x i1>, i32)
-
-declare <2 x i8> @llvm.vp.fshl.v2i8(<2 x i8>, <2 x i8>, <2 x i8>, <2 x i1>, i32)
-declare <4 x i8> @llvm.vp.fshl.v4i8(<4 x i8>, <4 x i8>, <4 x i8>, <4 x i1>, i32)
-declare <8 x i8> @llvm.vp.fshl.v8i8(<8 x i8>, <8 x i8>, <8 x i8>, <8 x i1>, i32)
-declare <16 x i8> @llvm.vp.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>, <16 x i1>, i32)
-declare <vscale x 1 x i8> @llvm.vp.fshl.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i8>, <vscale x 1 x i8>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i8> @llvm.vp.fshl.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>, <vscale x 2 x i8>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i8> @llvm.vp.fshl.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i8> @llvm.vp.fshl.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i8> @llvm.vp.fshl.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i8> @llvm.vp.fshl.nxv32i8(<vscale x 32 x i8>, <vscale x 32 x i8>, <vscale x 32 x i8>, <vscale x 32 x i1>, i32)
-declare <vscale x 64 x i8> @llvm.vp.fshl.nxv64i8(<vscale x 64 x i8>, <vscale x 64 x i8>, <vscale x 64 x i8>, <vscale x 64 x i1>, i32)
-declare <2 x i16> @llvm.vp.fshl.v2i16(<2 x i16>, <2 x i16>, <2 x i16>, <2 x i1>, i32)
-declare <4 x i16> @llvm.vp.fshl.v4i16(<4 x i16>, <4 x i16>, <4 x i16>, <4 x i1>, i32)
-declare <8 x i16> @llvm.vp.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>, <8 x i1>, i32)
-declare <16 x i16> @llvm.vp.fshl.v16i16(<16 x i16>, <16 x i16>, <16 x i16>, <16 x i1>, i32)
-declare <vscale x 1 x i16> @llvm.vp.fshl.nxv1i16(<vscale x 1 x i16>, <vscale x 1 x i16>, <vscale x 1 x i16>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i16> @llvm.vp.fshl.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i16> @llvm.vp.fshl.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i16> @llvm.vp.fshl.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i16> @llvm.vp.fshl.nxv16i16(<vscale x 16 x i16>, <vscale x 16 x i16>, <vscale x 16 x i16>, <vscale x 16 x i1>, i32)
-declare <vscale x 32 x i16> @llvm.vp.fshl.nxv32i16(<vscale x 32 x i16>, <vscale x 32 x i16>, <vscale x 32 x i16>, <vscale x 32 x i1>, i32)
-declare <2 x i32> @llvm.vp.fshl.v2i32(<2 x i32>, <2 x i32>, <2 x i32>, <2 x i1>, i32)
-declare <4 x i32> @llvm.vp.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>, <4 x i1>, i32)
-declare <8 x i32> @llvm.vp.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>, <8 x i1>, i32)
-declare <16 x i32> @llvm.vp.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>, <16 x i1>, i32)
-declare <vscale x 1 x i32> @llvm.vp.fshl.nxv1i32(<vscale x 1 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i32> @llvm.vp.fshl.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i32> @llvm.vp.fshl.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i32> @llvm.vp.fshl.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i1>, i32)
-declare <vscale x 16 x i32> @llvm.vp.fshl.nxv16i32(<vscale x 16 x i32>, <vscale x 16 x i32>, <vscale x 16 x i32>, <vscale x 16 x i1>, i32)
-declare <2 x i64> @llvm.vp.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>, <2 x i1>, i32)
-declare <4 x i64> @llvm.vp.fshl.v4i64(<4 x i64>, <4 x i64>, <4 x i64>, <4 x i1>, i32)
-declare <8 x i64> @llvm.vp.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>, <8 x i1>, i32)
-declare <16 x i64> @llvm.vp.fshl.v16i64(<16 x i64>, <16 x i64>, <16 x i64>, <16 x i1>, i32)
-declare <vscale x 1 x i64> @llvm.vp.fshl.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i1>, i32)
-declare <vscale x 2 x i64> @llvm.vp.fshl.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)
-declare <vscale x 4 x i64> @llvm.vp.fshl.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>, <vscale x 4 x i64>, <vscale x 4 x i1>, i32)
-declare <vscale x 8 x i64> @llvm.vp.fshl.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>, <vscale x 8 x i64>, <vscale x 8 x i1>, i32)
diff --git a/llvm/test/Analysis/DependenceAnalysis/DifferentOffsets.ll b/llvm/test/Analysis/DependenceAnalysis/DifferentOffsets.ll
index 1f8fac3087bff..05df105ef79c2 100644
--- a/llvm/test/Analysis/DependenceAnalysis/DifferentOffsets.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/DifferentOffsets.ll
@@ -175,9 +175,9 @@ for.j:
 
 for.k:
   %k = phi i64 [ 0, %for.j ], [ %k.inc, %for.k.inc ]
-  %idx0 = getelementptr inbounds [256 x [256 x [256 x i64]]], ptr %A, i64 %i, i64 %j, i64 %k
+  %idx0 = getelementptr inbounds [256 x [256 x [256 x i64]]], ptr %A, i64 0, i64 %i, i64 %j, i64 %k
   store i32 1, ptr %idx0
-  %idx1 = getelementptr inbounds [256 x [256 x [256 x i32]]], ptr %A, i64 %i, i64 %j, i64 %k
+  %idx1 = getelementptr inbounds [256 x [256 x [256 x i32]]], ptr %A, i64 0, i64 %i, i64 %j, i64 %k
   store i32 1, ptr %idx1
   br label %for.k.inc
 
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index 0366684a13b54..247a105940e6e 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -144,6 +144,54 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
+define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) {
+; CHECK-LABEL: 'crc8.le.tc16'
+; CHECK-NEXT:  Found little-endian CRC-8 loop with trip count 16
+; CHECK-NEXT:    Initial CRC: i8 %checksum
+; CHECK-NEXT:    Generating polynomial: 29
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i8 %crc.lshr, i8 %crc.xor
+; CHECK-NEXT:    Auxiliary data: i16 %msg
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 9 18 27 31 22 13 4 5 12 23 30 26 19 8 1
+; CHECK-NEXT:  10 3 24 17 21 28 7 14 15 6 29 20 16 25 2 11
+; CHECK-NEXT:  20 29 6 15 11 2 25 16 17 24 3 10 14 7 28 21
+; CHECK-NEXT:  30 23 12 5 1 8 19 26 27 18 9 0 4 13 22 31
+; CHECK-NEXT:  19 26 1 8 12 5 30 23 22 31 4 13 9 0 27 18
+; CHECK-NEXT:  25 16 11 2 6 15 20 29 28 21 14 7 3 10 17 24
+; CHECK-NEXT:  7 14 21 28 24 17 10 3 2 11 16 25 29 20 15 6
+; CHECK-NEXT:  13 4 31 22 18 27 0 9 8 1 26 19 23 30 5 12
+; CHECK-NEXT:  29 20 15 6 2 11 16 25 24 17 10 3 7 14 21 28
+; CHECK-NEXT:  23 30 5 12 8 1 26 19 18 27 0 9 13 4 31 22
+; CHECK-NEXT:  9 0 27 18 22 31 4 13 12 5 30 23 19 26 1 8
+; CHECK-NEXT:  3 10 17 24 28 21 14 7 6 15 20 29 25 16 11 2
+; CHECK-NEXT:  14 7 28 21 17 24 3 10 11 2 25 16 20 29 6 15
+; CHECK-NEXT:  4 13 22 31 27 18 9 0 1 8 19 26 30 23 12 5
+; CHECK-NEXT:  26 19 8 1 5 12 23 30 31 22 13 4 0 9 18 27
+; CHECK-NEXT:  16 25 2 11 15 6 29 20 21 28 7 14 10 3 24 17
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i8 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %data.trunc = trunc i16 %data to i8
+  %xor.crc.data = xor i8 %crc, %data.trunc
+  %and.crc.data = and i8 %xor.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i8 %and.crc.data, 0
+  %crc.lshr = lshr i8 %crc, 1
+  %crc.xor = xor i8 %crc.lshr, 29
+  %crc.next = select i1 %check.sb, i8 %crc.lshr, i8 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i8 %crc.next
+}
+
 define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) {
 ; CHECK-LABEL: 'crc16.be.tc8.crc.init.li'
 ; CHECK-NEXT:  Found big-endian CRC-16 loop with trip count 8
@@ -601,7 +649,7 @@ exit:                                              ; preds = %loop
 define i16 @not.crc.wrong.sb.check.const(i8 %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.wrong.sb.check.const'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Bad RHS of significant-bit-check
 ;
 entry:
   br label %loop
@@ -610,9 +658,8 @@ loop:                                              ; preds = %loop, %entry
   %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
   %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
   %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
-  %crc.lshr = lshr i16 %crc, 8
   %data.ext = zext i8 %data to i16
-  %xor.crc.data = xor i16 %crc.lshr, %data.ext
+  %xor.crc.data = xor i16 %crc, %data.ext
   %check.sb = icmp samesign ult i16 %xor.crc.data, 128
   %crc.shl = shl i16 %crc, 1
   %crc.xor = xor i16 %crc.shl, 258
@@ -649,10 +696,10 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
-define i16 @not.crc.excess.tc(i16 %msg, i16 %checksum) {
+define i16 @not.crc.excess.tc(i8 %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.excess.tc'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of result
+; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of data
 ;
 entry:
   br label %loop
@@ -660,16 +707,40 @@ entry:
 loop:                                              ; preds = %loop, %entry
   %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
   %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
-  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
-  %xor.crc.data = xor i16 %crc, %data
-  %and.crc.data = and i16 %xor.crc.data, 1
-  %data.next = lshr i16 %data, 1
-  %check.sb = icmp eq i16 %and.crc.data, 0
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %xor.crc.data = xor i8 %crc.trunc, %data
+  %and.crc.data = and i8 %xor.crc.data, 1
+  %data.next = lshr i8 %data, 1
+  %check.sb = icmp eq i8 %and.crc.data, 0
   %crc.lshr = lshr i16 %crc, 1
   %crc.xor = xor i16 %crc.lshr, -24575
   %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
   %iv.next = add nuw nsw i8 %iv, 1
-  %exit.cond = icmp samesign ult i8 %iv, 31
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.init.arg.excess.tc(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.init.arg.excess.tc'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of data
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 31
   br i1 %exit.cond, label %loop, label %exit
 
 exit:                                              ; preds = %loop
@@ -838,10 +909,37 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
+define i16 @not.crc.bad.cast(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.bad.cast'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Expected bottom 8 bits zero (????????00001011)
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data.ext = zext i8 %data to i16
+  %xor.crc.data = xor i16 %crc, %data.ext
+  %check.sb = icmp slt i16 %xor.crc.data, 0
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 29
+  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+  %data.next = shl i8 %data, 1
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
 define i32 @not.crc.dead.msg.bad.use(i32 %checksum, i32 %msg) {
 ; CHECK-LABEL: 'not.crc.dead.msg.bad.use'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Recurrences not intertwined with XOR
 ;
 entry:
   br label %loop
@@ -869,7 +967,7 @@ exit:                                              ; preds = %loop
 define i16 @not.crc.dead.msg.no.use(i8 %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.dead.msg.no.use'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Recurrences have stray uses
 ;
 entry:
   br label %loop
@@ -898,7 +996,7 @@ exit:                                              ; preds = %loop
 define i32 @not.crc.dead.msg.wrong.op(i32 %checksum, i32 %msg) {
 ; CHECK-LABEL: 'not.crc.dead.msg.wrong.op'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Recurrences not intertwined with XOR
 ;
 entry:
   br label %loop
@@ -922,6 +1020,120 @@ exit:                                              ; preds = %loop
   ret i32 %crc.next
 }
 
+define i16 @not.crc.dead.msg.xor.notin.select.chain(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.dead.msg.xor.notin.select.chain'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Recurrences have stray uses
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %or.crc.data = or i16 %crc, %data
+  %and.crc.data = and i16 %or.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.bad.xor.crc.data(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.bad.xor.crc.data'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Recurrences have stray uses
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %mul.corrupt = mul i16 %xor.crc.data, 0
+  %xor.crc.data.corrupt = xor i16 %mul.corrupt, %crc
+  %and.crc.data = and i16 %xor.crc.data.corrupt, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.dead.msg.or.zero(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.dead.msg.or.zero'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Recurrences have stray uses
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %mul.corrupt = mul i16 %xor.crc.data, 0
+  %or.crc.data.corrupt = or i16 %mul.corrupt, %crc
+  %and.crc.data = and i16 %or.crc.data.corrupt, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.unknown.value(i16 %msg, i16 %checksum, i16 %corrupt) {
+; CHECK-LABEL: 'not.crc.unknown.value'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unknown Value
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %xor.crc.data.corrupt = mul i16 %xor.crc.data, %corrupt
+  %and.crc.data = and i16 %xor.crc.data.corrupt, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
 define i16 @not.crc.float.simple.recurrence(float %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.float.simple.recurrence'
 ; CHECK-NEXT:  Did not find a hash algorithm
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
new file mode 100644
index 0000000000000..0d0fc93b9e7ed
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
@@ -0,0 +1,11 @@
+{
+    "Opcodes": {
+        "dummyOpc": [1, 2]
+    },
+    "Types": {
+        "dummyTy": [1, 2]
+    },
+    "Arguments": {
+        "dummyArg": [1, 2]
+    }
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json
index 0ff5fdd959e9c..3b662ab701681 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json
@@ -1,7 +1,15 @@
 {
-    "alloca": [1, 2, 3],
-    "load": [4, 5, 6],
-    "store": [7, 8, 9],
-    "add": [10, 11, 12],
-    "mul": [13, 14, 15]
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Types": {
+        "dummyTy": [1, 1, 1]
+    },
+    "Arguments": {
+        "dummyArg": [1, 1, 1]
+    }
 }
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json
index d6506f06bb320..3345aaf9c9827 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json
@@ -1,11 +1,17 @@
 {
-    "alloca": [-0.1, -0.2, -0.3, 1, 2],
-    "load": [-0.4, -0.5, -0.6, 4, 5],
-    "store": [-0.7, -0.8, -0.9, 7, 8],
-    "add": [-1.0, -1.1, -1.2, 10, 11],
-    "mul": [12, 13, 14, -1.2, -1.3],
-    "integerTy": [0.2, 0.4, 0.6, 1, 0.5],
-    "pointer": [0, 1, 2, 3, 4],
-    "variable": [2, 4, 6, 8, 10],
-    "ret": [5, 6, 7, 8, 9]
+    "Opcodes": {
+        "alloca": [-0.1, -0.2, -0.3, 1, 2],
+        "load": [-0.4, -0.5, -0.6, 4, 5],
+        "store": [-0.7, -0.8, -0.9, 7, 8],
+        "add": [-1.0, -1.1, -1.2, 10, 11],
+        "mul": [12, 13, 14, -1.2, -1.3],
+        "ret": [5, 6, 7, 8, 9]
+    },
+    "Types": {
+        "integerTy": [0.2, 0.4, 0.6, 1, 0.5]
+    },
+    "Arguments": {
+        "pointer": [0, 1, 2, 3, 4],
+        "variable": [2, 4, 6, 8, 10]
+    }
 }
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json
new file mode 100644
index 0000000000000..d6506f06bb320
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json
@@ -0,0 +1,11 @@
+{
+    "alloca": [-0.1, -0.2, -0.3, 1, 2],
+    "load": [-0.4, -0.5, -0.6, 4, 5],
+    "store": [-0.7, -0.8, -0.9, 7, 8],
+    "add": [-1.0, -1.1, -1.2, 10, 11],
+    "mul": [12, 13, 14, -1.2, -1.3],
+    "integerTy": [0.2, 0.4, 0.6, 1, 0.5],
+    "pointer": [0, 1, 2, 3, 4],
+    "variable": [2, 4, 6, 8, 10],
+    "ret": [5, 6, 7, 8, 9]
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json
new file mode 100644
index 0000000000000..ae5ab0ffcdd03
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json
@@ -0,0 +1,12 @@
+{
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Arguments": {
+        "dummyArg": [1, 1, 1]
+    }
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json
new file mode 100644
index 0000000000000..127d0995eb732
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json
@@ -0,0 +1,12 @@
+{
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Types": {
+        "dummyType": [1, 1, 1]
+    }
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json
new file mode 100644
index 0000000000000..c3e19ec05d6c9
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json
@@ -0,0 +1,16 @@
+{
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Types": {
+        "dummyTy": [1, 1, 1, 1, 1, 1]
+    },
+    "Arguments": {
+        "dummyArg": [1, 1, 1]
+    }
+}
+
diff --git a/llvm/test/Analysis/IR2Vec/basic.ll b/llvm/test/Analysis/IR2Vec/basic.ll
index b10e735a54587..ed65ff46dc953 100644
--- a/llvm/test/Analysis/IR2Vec/basic.ll
+++ b/llvm/test/Analysis/IR2Vec/basic.ll
@@ -1,6 +1,10 @@
 ; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_3D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=3D-CHECK
 ; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_5D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=5D-CHECK
-
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab1.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB1-CHECK
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab2.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB2-CHECK
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab3.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB3-CHECK
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab4.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB4-CHECK
+ 
 define dso_local i32 @abc(i32 %0, i32 %1) {
 entry:
   %3 = alloca i32, align 4
@@ -48,3 +52,11 @@ entry:
 ; 5D-CHECK-NEXT: Instruction:   %7 = mul nsw i32 %5, %6 [ 12.90  14.80  16.70  2.50  2.95 ]
 ; 5D-CHECK-NEXT: Instruction:   %8 = add nsw i32 %4, %7 [ -0.10  0.70  1.50  13.70  15.25 ]
 ; 5D-CHECK-NEXT: Instruction:   ret i32 %8 [ 5.40  6.80  8.20  9.60  11.00 ]
+
+; INCORRECT-VOCAB1-CHECK: error: Error reading vocabulary: Missing 'Opcodes' section in vocabulary file
+
+; INCORRECT-VOCAB2-CHECK: error: Error reading vocabulary: Missing 'Types' section in vocabulary file
+
+; INCORRECT-VOCAB3-CHECK: error: Error reading vocabulary: Missing 'Arguments' section in vocabulary file
+
+; INCORRECT-VOCAB4-CHECK: error: Error reading vocabulary: Vocabulary sections have different dimensions
diff --git a/llvm/test/Analysis/IR2Vec/dbg-inst.ll b/llvm/test/Analysis/IR2Vec/dbg-inst.ll
new file mode 100644
index 0000000000000..0f486b0ba6a52
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/dbg-inst.ll
@@ -0,0 +1,13 @@
+; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_3D_vocab.json %s 2>&1 | FileCheck %s
+
+define void @bar2(ptr %foo)  {
+  store i32 0, ptr %foo, align 4
+  tail call void @llvm.dbg.value(metadata !{}, i64 0, metadata !{}, metadata !{})
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone
+
+; CHECK: Instruction vectors:
+; CHECK-NEXT: Instruction:   store i32 0, ptr %foo, align 4 [ 7.00  8.00  9.00 ]
+; CHECK-NEXT: Instruction:   ret void [ 0.00  0.00  0.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/unreachable.ll b/llvm/test/Analysis/IR2Vec/unreachable.ll
new file mode 100644
index 0000000000000..370fe6881d6ce
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/unreachable.ll
@@ -0,0 +1,42 @@
+; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_3D_vocab.json %s 2>&1 | FileCheck %s
+
+define dso_local i32 @abc(i32 noundef %a, i32 noundef %b) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %cmp = icmp sgt i32 %0, %1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, ptr %b.addr, align 4
+  store i32 %2, ptr %retval, align 4
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %3 = load i32, ptr %a.addr, align 4
+  store i32 %3, ptr %retval, align 4
+  br label %return
+
+unreachable:                                      ; Unreachable
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %4 = load i32, ptr %retval, align 4
+  ret i32 %4
+}
+
+; CHECK: Basic block vectors:
+; CHECK-NEXT: Basic block: entry:
+; CHECK-NEXT:  [ 25.00 32.00 39.00 ]
+; CHECK-NEXT: Basic block: if.then:
+; CHECK-NEXT:  [ 11.00 13.00 15.00 ]
+; CHECK-NEXT: Basic block: if.else:
+; CHECK-NEXT:  [ 11.00 13.00 15.00 ]
+; CHECK-NEXT: Basic block: return:
+; CHECK-NEXT:  [ 4.00 5.00 6.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/vocab-test.ll b/llvm/test/Analysis/IR2Vec/vocab-test.ll
new file mode 100644
index 0000000000000..fbccb504a232f
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/vocab-test.ll
@@ -0,0 +1,20 @@
+; RUN: opt -passes='print<ir2vec-vocab>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_2D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=VOCAB-CHECK
+; RUN: opt -passes='print<ir2vec-vocab>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_2D_vocab.json -ir2vec-opc-weight=0.5 -ir2vec-type-weight=0.5 -ir2vec-arg-weight=0.5 %s 2>&1 | FileCheck %s -check-prefix=WT1-VOCAB-CHECK
+; RUN: opt -passes='print<ir2vec-vocab>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_2D_vocab.json -ir2vec-opc-weight=0.1 -ir2vec-type-weight=0 -ir2vec-arg-weight=0 %s 2>&1 | FileCheck %s -check-prefix=WT2-VOCAB-CHECK
+ 
+define dso_local void @test() {
+  entry:
+    ret void
+}
+
+; VOCAB-CHECK: Key: dummyArg: [ 0.20 0.40 ]
+; VOCAB-CHECK-NEXT: Key: dummyOpc: [ 1.00 2.00 ]
+; VOCAB-CHECK-NEXT: Key: dummyTy: [ 0.50 1.00 ]
+
+; WT1-VOCAB-CHECK: Key: dummyArg: [ 0.50 1.00 ]
+; WT1-VOCAB-CHECK-NEXT: Key: dummyOpc: [ 0.50 1.00 ]
+; WT1-VOCAB-CHECK-NEXT: Key: dummyTy: [ 0.50 1.00 ]
+
+; WT2-VOCAB-CHECK: Key: dummyArg: [ 0.00 0.00 ]
+; WT2-VOCAB-CHECK-NEXT: Key: dummyOpc: [ 0.10 0.20 ]
+; WT2-VOCAB-CHECK-NEXT: Key: dummyTy: [ 0.00 0.00 ]
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll b/llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll
new file mode 100644
index 0000000000000..2df451d5df738
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -disable-output -passes='print<access-info>' %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:128:128:128"
+
+; Tests with 128 bit inductions and 128 bit pointer width and index size.
+
+
+define void @backward_i128(ptr %A, i128 %n) {
+; CHECK-LABEL: 'backward_i128'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Backward loop carried data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Backward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A.1, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  %l = load i32, ptr %gep.A
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  store i32 %l, ptr %gep.A.1
+  %iv.next = add i128 %iv, 1
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @forward_i128(ptr %A, i128 %n) {
+; CHECK-LABEL: 'forward_i128'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A.1, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  %l = load i32, ptr %gep.A.1
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  store i32 %l, ptr %gep.A
+  %iv.next = add i128 %iv, 1
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @forward_negative_step(ptr %A, i128 %n) {
+; CHECK-LABEL: 'forward_negative_step'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A.1, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 36893488147419103232, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  %l = load i32, ptr %gep.A
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  store i32 %l, ptr %gep.A.1
+  %iv.next = add nsw i128 %iv, -1
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; The induction step is i128 2^63 + 1, which does not fit into a 64 bit signed
+; integer.
+define void @forward_i128_step_63bit_plus_one(ptr %A, i128 %n) {
+; CHECK-LABEL: 'forward_i128_step_63bit_plus_one'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unsafe indirect dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        IndirectUnsafe:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A.1, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  %l = load i32, ptr %gep.A.1
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  store i32 %l, ptr %gep.A
+  %iv.next = add i128 %iv, 9223372036854775809
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll
index d4cd61bfde2f4..1dc8d4a7e73f8 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll
@@ -504,3 +504,123 @@ e.0:
 e.1:
   ret i32 1
 }
+
+define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption(ptr %A, ptr %B) {
+; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption'
+; CHECK-NEXT:    loop.header:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %B High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%B,+,4}<nuw><%loop.header>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %A High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 2000) ]
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %B, i64 2000) ]
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+  %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+  %l = load i32, ptr %gep.A, align 4
+  store i32 0, ptr %gep.B, align 4
+  %cntable.c.1 = icmp ult i64 %iv, 1000
+  %iv.next = add nuw nsw i64 %iv, 1
+  br i1 %cntable.c.1, label %b2, label %e.1
+
+b2:
+  %uncntable.c.0 = icmp eq i32 %l, 0
+  br i1 %uncntable.c.0, label %e.2, label %b3
+
+b3:
+  %cntable.c.2 = icmp eq i64 %iv.next, 500
+  br i1 %cntable.c.2, label %cleanup4, label %latch
+
+latch:
+  br label %loop.header
+
+cleanup4:
+  ret void
+
+e.1:
+  ret void
+
+e.2:
+  ret void
+}
+
+define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small(ptr %A, ptr %B) {
+; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small'
+; CHECK-NEXT:    loop.header:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %B High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%B,+,4}<nuw><%loop.header>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %A High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 1999) ]
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %B, i64 1999) ]
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+  %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+  %l = load i32, ptr %gep.A, align 4
+  store i32 0, ptr %gep.B, align 4
+  %cntable.c.1 = icmp ult i64 %iv, 1000
+  %iv.next = add nuw nsw i64 %iv, 1
+  br i1 %cntable.c.1, label %b2, label %e.1
+
+b2:
+  %uncntable.c.0 = icmp eq i32 %l, 0
+  br i1 %uncntable.c.0, label %e.2, label %b3
+
+b3:
+  %cntable.c.2 = icmp eq i64 %iv.next, 500
+  br i1 %cntable.c.2, label %cleanup4, label %latch
+
+latch:
+  br label %loop.header
+
+cleanup4:
+  ret void
+
+e.1:
+  ret void
+
+e.2:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
index 595fb021ec0ab..63abd4ef70d63 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
@@ -320,28 +320,34 @@ exit:
 define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) {
 ; CHECK-LABEL: 'retry_after_dep_check_with_unknown_offset'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
 ; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:        Unknown:
-; CHECK-NEXT:            %l.A = load float, ptr %A, align 4 ->
-; CHECK-NEXT:            store float 0.000000e+00, ptr %A.100.iv.offset.3, align 4
-; CHECK-EMPTY:
-; CHECK-NEXT:        Unknown:
-; CHECK-NEXT:            %l.A = load float, ptr %A, align 4 ->
-; CHECK-NEXT:            store float %l.A, ptr %A.100.iv, align 8
-; CHECK-EMPTY:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT:      Check 1:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT:        Against group GRP2:
+; CHECK-NEXT:        ptr %A
+; CHECK-NEXT:      Check 2:
+; CHECK-NEXT:        Comparing group GRP1:
+; CHECK-NEXT:          %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT:        Against group GRP2:
+; CHECK-NEXT:        ptr %A
 ; CHECK-NEXT:      Grouped accesses:
 ; CHECK-NEXT:        Group GRP0:
-; CHECK-NEXT:          (Low: %A High: (4 + %A))
-; CHECK-NEXT:            Member: %A
+; CHECK-NEXT:          (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
+; CHECK-NEXT:            Member: {(100 + %A),+,8}<%loop>
 ; CHECK-NEXT:        Group GRP1:
 ; CHECK-NEXT:          (Low: (100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A) High: (96 + (16 * (zext i32 %offset to i64))<nuw><nsw> + %A))
 ; CHECK-NEXT:            Member: {(100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A),+,8}<%loop>
 ; CHECK-NEXT:        Group GRP2:
-; CHECK-NEXT:          (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
-; CHECK-NEXT:            Member: {(100 + %A),+,8}<%loop>
+; CHECK-NEXT:          (Low: %A High: (4 + %A))
+; CHECK-NEXT:            Member: %A
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/llvm/test/Analysis/ScalarEvolution/no-wrap-unknown-becount.ll b/llvm/test/Analysis/ScalarEvolution/no-wrap-unknown-becount.ll
index a46063527b7b8..0aa13c5b06ca2 100644
--- a/llvm/test/Analysis/ScalarEvolution/no-wrap-unknown-becount.ll
+++ b/llvm/test/Analysis/ScalarEvolution/no-wrap-unknown-becount.ll
@@ -1,10 +1,28 @@
-; RUN: opt -disable-output "-passes=print<scalar-evolution>" < %s 2>&1 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -disable-output -passes='print<scalar-evolution>' %s 2>&1 | FileCheck %s
+
+; Check that assumes/guards guarding the backedge are taken into account
+; when inferring no-wrap flags.
 
 declare void @llvm.experimental.guard(i1, ...)
 declare void @llvm.assume(i1)
 
 define void @s_0(i32 %n, ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @s_0
+; CHECK-LABEL: 's_0'
+; CHECK-NEXT:  Classifying expressions for: @s_0
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.sext = sext i32 %iv to i64
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%loop> U: [0,-9223372036854775808) S: [0,-9223372036854775808) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @s_0
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
 entry:
   br label %loop
 
@@ -12,8 +30,6 @@ loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
   %iv.inc = add i32 %iv, 1
   %iv.sext = sext i32 %iv to i64
-; CHECK:    %iv.sext = sext i32 %iv to i64
-; CHECK-NEXT:  -->  {0,+,1}<nuw><nsw><%loop>
   %cmp = icmp slt i32 %iv, %n
   call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   %c = load volatile i1, ptr %cond
@@ -24,7 +40,21 @@ leave:
 }
 
 define void @s_1(ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @s_1
+; CHECK-LABEL: 's_1'
+; CHECK-NEXT:  Classifying expressions for: @s_1
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {0,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 3
+; CHECK-NEXT:    --> {3,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.sext = sext i32 %iv to i64
+; CHECK-NEXT:    --> {0,+,3}<nuw><nsw><%loop> U: [0,-9223372036854775808) S: [0,-9223372036854775808) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @s_1
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
 entry:
   br label %loop
 
@@ -32,8 +62,6 @@ loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
   %iv.inc = add i32 %iv, 3
   %iv.sext = sext i32 %iv to i64
-; CHECK:  %iv.sext = sext i32 %iv to i64
-; CHECK-NEXT:  -->  {0,+,3}<nuw><nsw><%loop>
   %cmp = icmp slt i32 %iv, 10000
   call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   %c = load volatile i1, ptr %cond
@@ -44,7 +72,21 @@ leave:
 }
 
 define void @s_2(ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @s_2
+; CHECK-LABEL: 's_2'
+; CHECK-NEXT:  Classifying expressions for: @s_2
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {0,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 3
+; CHECK-NEXT:    --> {3,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.sext = sext i32 %iv to i64
+; CHECK-NEXT:    --> {0,+,3}<nuw><nsw><%loop> U: [0,-9223372036854775808) S: [0,-9223372036854775808) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @s_2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
 entry:
   br label %loop
 
@@ -53,8 +95,6 @@ loop:
   %iv.inc = add i32 %iv, 3
   %iv.sext = sext i32 %iv to i64
   %cmp = icmp slt i32 %iv, 10000
-; CHECK:  %iv.sext = sext i32 %iv to i64
-; CHECK-NEXT:  -->  {0,+,3}<nuw><nsw><%loop>
   call void @llvm.assume(i1 %cmp)
   %c = load volatile i1, ptr %cond
   br i1 %c, label %loop, label %leave
@@ -64,7 +104,25 @@ leave:
 }
 
 define void @s_3(i32 %start, ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @s_3
+; CHECK-LABEL: 's_3'
+; CHECK-NEXT:  Classifying expressions for: @s_3
+; CHECK-NEXT:    %iv = phi i32 [ %start, %entry ], [ %iv.inc, %be ]
+; CHECK-NEXT:    --> {%start,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 3
+; CHECK-NEXT:    --> {(3 + %start),+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc.sext = sext i32 %iv.inc to i64
+; CHECK-NEXT:    --> {(sext i32 (3 + %start) to i64),+,3}<nsw><%loop> U: [-2147483648,4294977296) S: [-2147483648,4294977296) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @s_3
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ((((-1 * (1 umin ((-1 * %start) + (10000 smax %start))))<nuw><nsw> + (-1 * %start) + (10000 smax %start)) /u 3) + (1 umin ((-1 * %start) + (10000 smax %start))))
+; CHECK-NEXT:    exit count for be: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 715831216
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (10000 smax %start))))<nuw><nsw> + (-1 * %start) + (10000 smax %start)) /u 3) + (1 umin ((-1 * %start) + (10000 smax %start))))
+; CHECK-NEXT:    symbolic max exit count for loop: ((((-1 * (1 umin ((-1 * %start) + (10000 smax %start))))<nuw><nsw> + (-1 * %start) + (10000 smax %start)) /u 3) + (1 umin ((-1 * %start) + (10000 smax %start))))
+; CHECK-NEXT:    symbolic max exit count for be: ***COULDNOTCOMPUTE***
+;
 entry:
   br label %loop
 
@@ -76,8 +134,6 @@ loop:
 be:
   %iv.inc = add i32 %iv, 3
   %iv.inc.sext = sext i32 %iv.inc to i64
-; CHECK:  %iv.inc.sext = sext i32 %iv.inc to i64
-; CHECK-NEXT:  -->  {(sext i32 (3 + %start) to i64),+,3}<nsw><%loop>
   %c = load volatile i1, ptr %cond
   br i1 %c, label %loop, label %leave
 
@@ -86,7 +142,25 @@ leave:
 }
 
 define void @s_4(i32 %start, ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @s_4
+; CHECK-LABEL: 's_4'
+; CHECK-NEXT:  Classifying expressions for: @s_4
+; CHECK-NEXT:    %iv = phi i32 [ %start, %entry ], [ %iv.inc, %be ]
+; CHECK-NEXT:    --> {%start,+,-3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, -3
+; CHECK-NEXT:    --> {(-3 + %start),+,-3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc.sext = sext i32 %iv.inc to i64
+; CHECK-NEXT:    --> {(sext i32 (-3 + %start) to i64),+,-3}<nsw><%loop> U: [-4294968296,2147483648) S: [-4294968296,2147483648) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @s_4
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ((2 + (-1 * (-1000 smin %start)) + %start) /u 3)
+; CHECK-NEXT:    exit count for be: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 715828216
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((2 + (-1 * (-1000 smin %start)) + %start) /u 3)
+; CHECK-NEXT:    symbolic max exit count for loop: ((2 + (-1 * (-1000 smin %start)) + %start) /u 3)
+; CHECK-NEXT:    symbolic max exit count for be: ***COULDNOTCOMPUTE***
+;
 entry:
   br label %loop
 
@@ -98,8 +172,6 @@ loop:
 be:
   %iv.inc = add i32 %iv, -3
   %iv.inc.sext = sext i32 %iv.inc to i64
-; CHECK:  %iv.inc.sext = sext i32 %iv.inc to i64
-; CHECK-NEXT:  -->  {(sext i32 (-3 + %start) to i64),+,-3}<nsw><%loop>
   %c = load volatile i1, ptr %cond
   br i1 %c, label %loop, label %leave
 
@@ -108,7 +180,21 @@ leave:
 }
 
 define void @u_0(i32 %n, ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @u_0
+; CHECK-LABEL: 'u_0'
+; CHECK-NEXT:  Classifying expressions for: @u_0
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.zext = zext i32 %iv to i64
+; CHECK-NEXT:    --> {0,+,1}<nuw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @u_0
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
 entry:
   br label %loop
 
@@ -116,8 +202,6 @@ loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
   %iv.inc = add i32 %iv, 1
   %iv.zext = zext i32 %iv to i64
-; CHECK:    %iv.zext = zext i32 %iv to i64
-; CHECK-NEXT:  -->  {0,+,1}<nuw><%loop>
   %cmp = icmp ult i32 %iv, %n
   call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   %c = load volatile i1, ptr %cond
@@ -128,7 +212,21 @@ leave:
 }
 
 define void @u_1(ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @u_1
+; CHECK-LABEL: 'u_1'
+; CHECK-NEXT:  Classifying expressions for: @u_1
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {0,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 3
+; CHECK-NEXT:    --> {3,+,3}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.zext = zext i32 %iv to i64
+; CHECK-NEXT:    --> {0,+,3}<nuw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @u_1
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
 entry:
   br label %loop
 
@@ -136,8 +234,6 @@ loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
   %iv.inc = add i32 %iv, 3
   %iv.zext = zext i32 %iv to i64
-; CHECK:  %iv.zext = zext i32 %iv to i64
-; CHECK-NEXT:  -->  {0,+,3}<nuw><%loop>
   %cmp = icmp ult i32 %iv, 10000
   call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   %c = load volatile i1, ptr %cond
@@ -148,7 +244,21 @@ leave:
 }
 
 define void @u_2(ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @u_2
+; CHECK-LABEL: 'u_2'
+; CHECK-NEXT:  Classifying expressions for: @u_2
+; CHECK-NEXT:    %iv = phi i32 [ 30000, %entry ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {30000,+,-2}<%loop> U: [0,-1) S: [-2147483648,2147483647) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, -2
+; CHECK-NEXT:    --> {29998,+,-2}<%loop> U: [0,-1) S: [-2147483648,2147483647) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.zext = zext i32 %iv to i64
+; CHECK-NEXT:    --> {30000,+,-2}<nw><%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @u_2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
 entry:
   br label %loop
 
@@ -157,8 +267,6 @@ loop:
   %iv.inc = add i32 %iv, -2
   %iv.zext = zext i32 %iv to i64
   %cmp = icmp ugt i32 %iv.inc, -10000
-; CHECK:  %iv.zext = zext i32 %iv to i64
-; CHECK-NEXT:  -->  {30000,+,-2}<nw><%loop>
   call void @llvm.assume(i1 %cmp)
   %c = load volatile i1, ptr %cond
   br i1 %c, label %loop, label %leave
@@ -168,7 +276,25 @@ leave:
 }
 
 define void @u_3(i32 %start, ptr %cond) {
-; CHECK-LABEL: Classifying expressions for: @u_3
+; CHECK-LABEL: 'u_3'
+; CHECK-NEXT:  Classifying expressions for: @u_3
+; CHECK-NEXT:    %iv = phi i32 [ %start, %entry ], [ %iv.inc, %be ]
+; CHECK-NEXT:    --> {%start,+,3}<nw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 3
+; CHECK-NEXT:    --> {(3 + %start),+,3}<nw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc.zext = zext i32 %iv.inc to i64
+; CHECK-NEXT:    --> {(zext i32 (3 + %start) to i64),+,3}<nuw><%loop> U: [0,4294977298) S: [0,4294977298) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %c = load volatile i1, ptr %cond, align 1
+; CHECK-NEXT:    --> %c U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:  Determining loop execution counts for: @u_3
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ((((-1 * (1 umin ((-1 * %start) + (10000 umax %start))))<nuw><nsw> + (-1 * %start) + (10000 umax %start)) /u 3) + (1 umin ((-1 * %start) + (10000 umax %start))))
+; CHECK-NEXT:    exit count for be: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 3334
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (10000 umax %start))))<nuw><nsw> + (-1 * %start) + (10000 umax %start)) /u 3) + (1 umin ((-1 * %start) + (10000 umax %start))))
+; CHECK-NEXT:    symbolic max exit count for loop: ((((-1 * (1 umin ((-1 * %start) + (10000 umax %start))))<nuw><nsw> + (-1 * %start) + (10000 umax %start)) /u 3) + (1 umin ((-1 * %start) + (10000 umax %start))))
+; CHECK-NEXT:    symbolic max exit count for be: ***COULDNOTCOMPUTE***
+;
 entry:
   br label %loop
 
@@ -180,8 +306,6 @@ loop:
 be:
   %iv.inc = add i32 %iv, 3
   %iv.inc.zext = zext i32 %iv.inc to i64
-; CHECK:  %iv.inc.zext = zext i32 %iv.inc to i64
-; CHECK-NEXT:  -->  {(zext i32 (3 + %start) to i64),+,3}<nuw><%loop>
   %c = load volatile i1, ptr %cond
   br i1 %c, label %loop, label %leave
 
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index 1028cc9ebb342..bd7464577b7db 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -261,6 +261,70 @@ bb:
   ret void
 }
 
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr4_b64_v2i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1) %addr)
+  store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr6_b96_v3i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1) %addr)
+  store <3 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr8_b64_v2i32(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3) %addr)
+  store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr4_b64_v2i32(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3) %addr)
+  store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr6_b96_v3i32(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3) %addr)
+  store <3 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr16_b128_v8i16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3) %addr)
+  store <8 x i16> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr16_b128_v8f16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %addr)
+  store <8 x half> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr16_b128_v8bf16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3) %addr)
+  store <8 x bfloat> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
 declare <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3))
 
 ; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3) %gep)
@@ -563,6 +627,14 @@ declare i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1))
 declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1))
 declare <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16(ptr addrspace(1))
 declare <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16(ptr addrspace(1))
+declare <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1))
+declare <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1))
+declare <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3))
+declare <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3))
+declare <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3))
+declare <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3))
+declare <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3))
+declare <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3))
 
 declare i32 @llvm.amdgcn.dead.i32()
 
diff --git a/llvm/test/Assembler/aarch64-intrinsics-attributes.ll b/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
index 31b7101fba116..33f2758a4b18c 100644
--- a/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
+++ b/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
@@ -22,5 +22,15 @@ declare <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32>, <4 x i32>)
 ; CHECK: declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_READNONE_WILLRETURN]]
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
 
+; CHECK: declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr captures(none)) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN:#[0-9]+]]
+declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr)
+
+; CHECK: declare void @llvm.aarch64.neon.st2lane.v4i32.p0(<4 x i32>, <4 x i32>, i64, ptr captures(none)) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN:#[0-9]+]]
+declare void @llvm.aarch64.neon.st2lane.v4i32.p0(<4 x i32>, <4 x i32>, i64, ptr)
+
+; CHECK: declare void @llvm.aarch64.neon.st1x2.v8i16.p0(<8 x i16>, <8 x i16>, ptr captures(none)) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN:#[0-9]+]]
+declare void @llvm.aarch64.neon.st1x2.v8i16.p0(<8 x i16>, <8 x i16>, ptr)
+
 ; CHECK: attributes [[NOFREE_NOUNWIND_WILLRETURN]] = { nofree nounwind willreturn }
 ; CHECK: attributes [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_READNONE_WILLRETURN]] = { nocallback nofree nosync nounwind willreturn memory(none) }
+; CHECK: attributes [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 7dd86a8c0eb16..8c1a76365e1b4 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -567,6 +567,11 @@ define void @captures(ptr captures(address) %p) {
   ret void
 }
 
+; CHECK: define void @dead_on_return(ptr dead_on_return %p)
+define void @dead_on_return(ptr dead_on_return %p) {
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/Bitcode/debug-loc-again.ll b/llvm/test/Bitcode/debug-loc-again.ll
index 3bfbd1ff30b38..d92720caa9262 100644
--- a/llvm/test/Bitcode/debug-loc-again.ll
+++ b/llvm/test/Bitcode/debug-loc-again.ll
@@ -1,9 +1,9 @@
 ; RUN: llvm-as < %s | llvm-bcanalyzer -dump | FileCheck %s -check-prefix=BC
 ; PR23436: Actually emit DEBUG_LOC_AGAIN records.
 
-; BC: <DEBUG_LOC op
+; BC: <DEBUG_LOC abbrevid
 ; BC: <DEBUG_LOC_AGAIN/>
-; BC: <DEBUG_LOC op
+; BC: <DEBUG_LOC abbrevid
 ; BC: <DEBUG_LOC_AGAIN/>
 
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
diff --git a/llvm/test/Bitcode/function-encoding-rel-operands.ll b/llvm/test/Bitcode/function-encoding-rel-operands.ll
index 84e83c11de361..d107a9c5a313d 100644
--- a/llvm/test/Bitcode/function-encoding-rel-operands.ll
+++ b/llvm/test/Bitcode/function-encoding-rel-operands.ll
@@ -48,7 +48,7 @@ define double @test_float_binops(i32 %a) nounwind {
 ; between literals and the formal parameters.
 ; CHECK: INST_GEP {{.*}}
 ; CHECK: INST_LOAD {{.*}}op0=1 {{.*}}
-; CHECK: INST_CMP2 op0=1 {{.*}}
+; CHECK: INST_CMP2 {{.*}}op0=1 {{.*}}
 ; CHECK: INST_RET {{.*}}op0=1
 define i1 @test_load(i32 %a, {i32, i32}* %ptr) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
index a2e20caf43ff9..b21046d6049e8 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
@@ -56,7 +56,7 @@
   }
 
 
-  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
   attributes #2 = { optsize }
   attributes #3 = { minsize }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
index 27d09b2362567..57d031dcd53c0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
@@ -47,7 +47,7 @@
     ret void
   }
 
-  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
index 9ed1e2d9eee3b..f8d2bf387ffc9 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
@@ -50,7 +50,7 @@
 
   declare void @llvm.stackprotector(ptr, ptr) #2
 
-  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/abd-combine.ll b/llvm/test/CodeGen/AArch64/abd-combine.ll
index e48680f4be98b..d0257890d2c43 100644
--- a/llvm/test/CodeGen/AArch64/abd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/abd-combine.ll
@@ -18,13 +18,11 @@ define <8 x i16> @abdu_const(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uabd v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -37,10 +35,10 @@ define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const_lhs:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    usubw v2.4s, v1.4s, v0.4h
-; CHECK-NEXT:    usubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v2.4s
+; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uabd v1.4s, v2.4s, v1.4s
 ; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
@@ -53,13 +51,6 @@ define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
 define <8 x i16> @abdu_const_zero(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-NEXT:    usubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    neg v1.4s, v2.4s
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1
@@ -328,13 +319,11 @@ define <8 x i16> @abds_const(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    sabd v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -347,10 +336,10 @@ define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const_lhs:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    ssubw v2.4s, v1.4s, v0.4h
-; CHECK-NEXT:    ssubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v2.4s
+; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    sabd v1.4s, v2.4s, v1.4s
 ; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
@@ -363,10 +352,8 @@ define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
 define <8 x i16> @abds_const_zero(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-NEXT:    ssubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    neg v1.4s, v2.4s
+; CHECK-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
 ; CHECK-NEXT:    abs v0.4s, v0.4s
 ; CHECK-NEXT:    abs v1.4s, v1.4s
 ; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index 4b1fff642e5f5..c838c08cd5e51 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s
+; RUN: opt -passes="require<profile-summary>,function(codegenprepare)" -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s
 ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck --check-prefix=LLC %s
 
 %struct.X = type { i8, i8, [2 x i8] }
@@ -618,7 +618,7 @@ define void @fct16(ptr nocapture %y, i32 %x) nounwind optsize inlinehint ssp {
 ; LLC-LABEL: fct16:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr w8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and w8, w8, w9
 ; LLC-NEXT:    bfxil w8, w1, #16, #3
@@ -655,7 +655,7 @@ define void @fct16_mask(ptr nocapture %y, i32 %x) nounwind optsize inlinehint ss
 ; LLC-LABEL: fct16_mask:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr w8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and w8, w8, w9
 ; LLC-NEXT:    bfxil w8, w1, #16, #3
@@ -697,7 +697,7 @@ define void @fct17(ptr nocapture %y, i64 %x) nounwind optsize inlinehint ssp {
 ; LLC-LABEL: fct17:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr x8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and x8, x8, x9
 ; LLC-NEXT:    bfxil x8, x1, #16, #3
@@ -734,7 +734,7 @@ define void @fct17_mask(ptr nocapture %y, i64 %x) nounwind optsize inlinehint ss
 ; LLC-LABEL: fct17_mask:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr x8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and x8, x8, x9
 ; LLC-NEXT:    bfxil x8, x1, #16, #3
@@ -819,7 +819,7 @@ define i32 @fct19(i64 %arg1) nounwind readonly ssp  {
 ; LLC-NEXT:    add w0, w8, #32
 ; LLC-NEXT:    ret
 ; LLC-NEXT:  .LBB26_6:
-; LLC-NEXT:    mov w0, #64
+; LLC-NEXT:    mov w0, #64 // =0x40
 ; LLC-NEXT:    ret
 ; OPT-LABEL: @fct19(
 ; OPT-NEXT:  entry:
@@ -916,20 +916,20 @@ return:                                           ; preds = %if.end13, %if.then1
 define i80 @fct20(i128 %a, i128 %b) {
 ; LLC-LABEL: fct20:
 ; LLC:       // %bb.0: // %entry
-; LLC-NEXT:	mov	x12, #11776                     // =0x2e00
-; LLC-NEXT:	lsr	x8, x1, #18
-; LLC-NEXT:	extr	x9, x1, x0, #18
-; LLC-NEXT:	movk	x12, #25856, lsl #16
-; LLC-NEXT:	orr	x10, x2, x3
-; LLC-NEXT:	mov	w11, #26220                     // =0x666c
-; LLC-NEXT:	movk	x12, #11077, lsl #32
-; LLC-NEXT:	and	x11, x8, x11
-; LLC-NEXT:	cmp	x10, #0
-; LLC-NEXT:	movk	x12, #45, lsl #48
-; LLC-NEXT:	csel	x1, x11, x8, eq
-; LLC-NEXT:	and	x12, x9, x12
-; LLC-NEXT:	csel	x0, x12, x9, eq
-; LLC-NEXT:	ret
+; LLC-NEXT:    mov x12, #11776 // =0x2e00
+; LLC-NEXT:    lsr x8, x1, #18
+; LLC-NEXT:    extr x9, x1, x0, #18
+; LLC-NEXT:    movk x12, #25856, lsl #16
+; LLC-NEXT:    orr x10, x2, x3
+; LLC-NEXT:    mov w11, #26220 // =0x666c
+; LLC-NEXT:    movk x12, #11077, lsl #32
+; LLC-NEXT:    and x11, x8, x11
+; LLC-NEXT:    cmp x10, #0
+; LLC-NEXT:    movk x12, #45, lsl #48
+; LLC-NEXT:    csel x1, x11, x8, eq
+; LLC-NEXT:    and x12, x9, x12
+; LLC-NEXT:    csel x0, x12, x9, eq
+; LLC-NEXT:    ret
 ; OPT-LABEL: @fct20(
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    [[SHR:%.*]] = lshr i128 [[A:%.*]], 18
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll
new file mode 100644
index 0000000000000..e14e69b5e6a2a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll
@@ -0,0 +1,54 @@
+; RUN: llc < %s -mtriple=arm64-linux-gnu | FileCheck %s -check-prefixes=NOTCPU-LINUX --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=generic | FileCheck %s -check-prefixes=NOTCPU-APPLE --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm-gpr64 | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm-gpr64 | FileCheck %s -check-prefixes=ATTR --match-full-lines
+
+define void @zero_cycle_regmov_GPR32(i32 %a, i32 %b, i32 %c, i32 %d) {
+entry:
+; CHECK-LABEL: t:
+; NOTCPU-LINUX: mov w0, w2
+; NOTCPU-LINUX: mov w1, w3
+; NOTCPU-LINUX: mov [[REG2:w[0-9]+]], w3
+; NOTCPU-LINUX: mov [[REG1:w[0-9]+]], w2
+; NOTCPU-LINUX-NEXT: bl {{_?foo_i32}}
+; NOTCPU-LINUX: mov w0, [[REG1]]
+; NOTCPU-LINUX: mov w1, [[REG2]]
+
+; NOTCPU-APPLE: mov w0, w2
+; NOTCPU-APPLE: mov w1, w3
+; NOTCPU-APPLE: mov [[REG2:w[0-9]+]], w3
+; NOTCPU-APPLE: mov [[REG1:w[0-9]+]], w2
+; NOTCPU-APPLE-NEXT: bl {{_?foo_i32}}
+; NOTCPU-APPLE: mov w0, [[REG1]]
+; NOTCPU-APPLE: mov w1, [[REG2]]
+
+; CPU: mov [[REG2:x[0-9]+]], x3
+; CPU: mov [[REG1:x[0-9]+]], x2
+; CPU: mov x0, x2
+; CPU: mov x1, x3
+; CPU-NEXT: bl {{_?foo_i32}}
+; CPU: mov x0, [[REG1]]
+; CPU: mov x1, [[REG2]]
+
+; NOTATTR: mov [[REG2:w[0-9]+]], w3
+; NOTATTR: mov [[REG1:w[0-9]+]], w2
+; NOTATTR: mov w0, w2
+; NOTATTR: mov w1, w3
+; NOTATTR-NEXT: bl {{_?foo_i32}}
+; NOTATTR: mov w0, [[REG1]]
+; NOTATTR: mov w1, [[REG2]]
+
+; ATTR: mov x0, x2
+; ATTR: mov x1, x3
+; ATTR: mov [[REG2:x[0-9]+]], x3
+; ATTR: mov [[REG1:x[0-9]+]], x2
+; ATTR-NEXT: bl {{_?foo_i32}}
+; ATTR: mov x0, [[REG1]]
+; ATTR: mov x1, [[REG2]]
+  %call = call i32 @foo_i32(i32 %c, i32 %d)
+  %call1 = call i32 @foo_i32(i32 %c, i32 %d)
+  unreachable
+}
+
+declare i32 @foo_i32(i32, i32)
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
deleted file mode 100644
index 5ef6d3e84805a..0000000000000
--- a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: llc < %s -march=arm64 | FileCheck %s -check-prefixes=NOTCPU --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm | FileCheck %s -check-prefixes=ATTR --match-full-lines
-
-define void @t(i32 %a, i32 %b, i32 %c, i32 %d) {
-entry:
-; CHECK-LABEL: t:
-; NOTCPU: mov w0, w2
-; NOTCPU: mov w1, w3
-; NOTCPU: mov [[REG2:w[0-9]+]], w3
-; NOTCPU: mov [[REG1:w[0-9]+]], w2
-; NOTCPU-NEXT: bl {{_?foo}}
-; NOTCPU: mov w0, [[REG1]]
-; NOTCPU: mov w1, [[REG2]]
-
-; CPU: mov [[REG2:x[0-9]+]], x3
-; CPU: mov [[REG1:x[0-9]+]], x2
-; CPU: mov x0, x2
-; CPU: mov x1, x3
-; CPU-NEXT: bl {{_?foo}}
-; CPU: mov x0, [[REG1]]
-; CPU: mov x1, [[REG2]]
-
-; NOTATTR: mov [[REG2:w[0-9]+]], w3
-; NOTATTR: mov [[REG1:w[0-9]+]], w2
-; NOTATTR: mov w0, w2
-; NOTATTR: mov w1, w3
-; NOTATTR-NEXT: bl {{_?foo}}
-; NOTATTR: mov w0, [[REG1]]
-; NOTATTR: mov w1, [[REG2]]
-
-; ATTR: mov x0, x2
-; ATTR: mov x1, x3
-; ATTR: mov [[REG2:x[0-9]+]], x3
-; ATTR: mov [[REG1:x[0-9]+]], x2
-; ATTR-NEXT: bl {{_?foo}}
-; ATTR: mov x0, [[REG1]]
-; ATTR: mov x1, [[REG2]]
-  %call = call i32 @foo(i32 %c, i32 %d)
-  %call1 = call i32 @foo(i32 %c, i32 %d)
-  unreachable
-}
-
-declare i32 @foo(i32, i32)
diff --git a/llvm/test/CodeGen/AArch64/arm64-zip.ll b/llvm/test/CodeGen/AArch64/arm64-zip.ll
index b24e54a68fb42..20d0c7f1b7085 100644
--- a/llvm/test/CodeGen/AArch64/arm64-zip.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-zip.ll
@@ -413,7 +413,7 @@ define <4 x float> @shuffle_zip1(<4 x float> %arg) {
 ; CHECK-NEXT:    fmov.4s v1, #1.00000000
 ; CHECK-NEXT:    zip1.4h v0, v0, v0
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    and.16b v0, v1, v0
+; CHECK-NEXT:    and.16b v0, v0, v1
 ; CHECK-NEXT:    ret
 bb:
   %inst = fcmp olt <4 x float> zeroinitializer, %arg
diff --git a/llvm/test/CodeGen/AArch64/arm64_32-neon.ll b/llvm/test/CodeGen/AArch64/arm64_32-neon.ll
index b3dfdad191fc3..ede2301f2560a 100644
--- a/llvm/test/CodeGen/AArch64/arm64_32-neon.ll
+++ b/llvm/test/CodeGen/AArch64/arm64_32-neon.ll
@@ -1,28 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=arm64_32-apple-ios7.0 -mcpu=cyclone %s -o - | FileCheck %s
 
 define <2 x double> @test_insert_elt(<2 x double> %vec, double %val) {
 ; CHECK-LABEL: test_insert_elt:
-; CHECK: mov.d v0[0], v1[0]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mov.d v0[0], v1[0]
+; CHECK-NEXT:    ret
   %res = insertelement <2 x double> %vec, double %val, i32 0
   ret <2 x double> %res
 }
 
 define void @test_split_16B(<4 x float> %val, ptr %addr) {
 ; CHECK-LABEL: test_split_16B:
-; CHECK: str q0, [x0]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   store <4 x float> %val, ptr %addr, align 8
   ret void
 }
 
 define void @test_split_16B_splat(<4 x i32>, ptr %addr) {
 ; CHECK-LABEL: test_split_16B_splat:
-; CHECK: str {{q[0-9]+}}
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    movi.4s v0, #42
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %vec.tmp0 = insertelement <4 x i32> undef, i32 42, i32 0
   %vec.tmp1 = insertelement <4 x i32> %vec.tmp0, i32 42, i32 1
   %vec.tmp2 = insertelement <4 x i32> %vec.tmp1, i32 42, i32 2
   %vec = insertelement <4 x i32> %vec.tmp2, i32 42, i32 3
-
   store <4 x i32> %vec, ptr %addr, align 8
   ret void
 }
@@ -33,7 +40,9 @@ define void @test_split_16B_splat(<4 x i32>, ptr %addr) {
 declare {%vec, %vec} @llvm.aarch64.neon.ld2r.v2f64.p0(ptr)
 define {%vec, %vec} @test_neon_load(ptr %addr) {
 ; CHECK-LABEL: test_neon_load:
-; CHECK: ld2r.2d { v0, v1 }, [x0]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ld2r.2d { v0, v1 }, [x0]
+; CHECK-NEXT:    ret
   %res = call {%vec, %vec} @llvm.aarch64.neon.ld2r.v2f64.p0(ptr %addr)
   ret {%vec, %vec} %res
 }
@@ -41,7 +50,11 @@ define {%vec, %vec} @test_neon_load(ptr %addr) {
 declare {%vec, %vec} @llvm.aarch64.neon.ld2lane.v2f64.p0(%vec, %vec, i64, ptr)
 define {%vec, %vec} @test_neon_load_lane(ptr %addr, %vec %in1, %vec %in2) {
 ; CHECK-LABEL: test_neon_load_lane:
-; CHECK: ld2.d { v0, v1 }[0], [x0]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0]
+; CHECK-NEXT:    ret
   %res = call {%vec, %vec} @llvm.aarch64.neon.ld2lane.v2f64.p0(%vec %in1, %vec %in2, i64 0, ptr %addr)
   ret {%vec, %vec} %res
 }
@@ -49,7 +62,11 @@ define {%vec, %vec} @test_neon_load_lane(ptr %addr, %vec %in1, %vec %in2) {
 declare void @llvm.aarch64.neon.st2.v2f64.p0(%vec, %vec, ptr)
 define void @test_neon_store(ptr %addr, %vec %in1, %vec %in2) {
 ; CHECK-LABEL: test_neon_store:
-; CHECK: st2.2d { v0, v1 }, [x0]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    st2.2d { v0, v1 }, [x0]
+; CHECK-NEXT:    ret
   call void @llvm.aarch64.neon.st2.v2f64.p0(%vec %in1, %vec %in2, ptr %addr)
   ret void
 }
@@ -57,7 +74,11 @@ define void @test_neon_store(ptr %addr, %vec %in1, %vec %in2) {
 declare void @llvm.aarch64.neon.st2lane.v2f64.p0(%vec, %vec, i64, ptr)
 define void @test_neon_store_lane(ptr %addr, %vec %in1, %vec %in2) {
 ; CHECK-LABEL: test_neon_store_lane:
-; CHECK: st2.d { v0, v1 }[1], [x0]
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    st2.d { v0, v1 }[1], [x0]
+; CHECK-NEXT:    ret
   call void @llvm.aarch64.neon.st2lane.v2f64.p0(%vec %in1, %vec %in2, i64 1, ptr %addr)
   ret void
 }
@@ -65,13 +86,13 @@ define void @test_neon_store_lane(ptr %addr, %vec %in1, %vec %in2) {
 declare {%vec, %vec} @llvm.aarch64.neon.ld2.v2f64.p0(ptr)
 define {{%vec, %vec}, ptr} @test_neon_load_post(ptr %addr, i32 %offset) {
 ; CHECK-LABEL: test_neon_load_post:
-; CHECK-DAG: sxtw [[OFFSET:x[0-9]+]], w1
-; CHECK: ld2.2d { v0, v1 }, [x0], [[OFFSET]]
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    ld2.2d { v0, v1 }, [x0], x8
+; CHECK-NEXT:    ret
   %vecs = call {%vec, %vec} @llvm.aarch64.neon.ld2.v2f64.p0(ptr %addr)
-
   %addr.new = getelementptr inbounds i8, ptr %addr, i32 %offset
-
   %res.tmp = insertvalue {{%vec, %vec}, ptr} undef, {%vec, %vec} %vecs, 0
   %res = insertvalue {{%vec, %vec}, ptr} %res.tmp, ptr %addr.new, 1
   ret {{%vec, %vec}, ptr} %res
@@ -79,13 +100,15 @@ define {{%vec, %vec}, ptr} @test_neon_load_post(ptr %addr, i32 %offset) {
 
 define {{%vec, %vec}, ptr} @test_neon_load_post_lane(ptr %addr, i32 %offset, %vec %in1, %vec %in2) {
 ; CHECK-LABEL: test_neon_load_post_lane:
-; CHECK-DAG: sxtw [[OFFSET:x[0-9]+]], w1
-; CHECK: ld2.d { v0, v1 }[1], [x0], [[OFFSET]]
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ld2.d { v0, v1 }[1], [x0], x8
+; CHECK-NEXT:    ret
   %vecs = call {%vec, %vec} @llvm.aarch64.neon.ld2lane.v2f64.p0(%vec %in1, %vec %in2, i64 1, ptr %addr)
-
   %addr.new = getelementptr inbounds i8, ptr %addr, i32 %offset
-
   %res.tmp = insertvalue {{%vec, %vec}, ptr} undef, {%vec, %vec} %vecs, 0
   %res = insertvalue {{%vec, %vec}, ptr} %res.tmp, ptr %addr.new, 1
   ret {{%vec, %vec}, ptr} %res
@@ -93,25 +116,29 @@ define {{%vec, %vec}, ptr} @test_neon_load_post_lane(ptr %addr, i32 %offset, %ve
 
 define ptr @test_neon_store_post(ptr %addr, i32 %offset, %vec %in1, %vec %in2) {
 ; CHECK-LABEL: test_neon_store_post:
-; CHECK-DAG: sxtw [[OFFSET:x[0-9]+]], w1
-; CHECK: st2.2d { v0, v1 }, [x0], [[OFFSET]]
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    st2.2d { v0, v1 }, [x0], x8
+; CHECK-NEXT:    ret
   call void @llvm.aarch64.neon.st2.v2f64.p0(%vec %in1, %vec %in2, ptr %addr)
-
   %addr.new = getelementptr inbounds i8, ptr %addr, i32 %offset
-
   ret ptr %addr.new
 }
 
 define ptr @test_neon_store_post_lane(ptr %addr, i32 %offset, %vec %in1, %vec %in2) {
 ; CHECK-LABEL: test_neon_store_post_lane:
-; CHECK: sxtw [[OFFSET:x[0-9]+]], w1
-; CHECK: st2.d { v0, v1 }[0], [x0], [[OFFSET]]
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], x8
+; CHECK-NEXT:    ret
   call void @llvm.aarch64.neon.st2lane.v2f64.p0(%vec %in1, %vec %in2, i64 0, ptr %addr)
-
   %addr.new = getelementptr inbounds i8, ptr %addr, i32 %offset
-
   ret ptr %addr.new
 }
 
@@ -119,28 +146,26 @@ define ptr @test_neon_store_post_lane(ptr %addr, i32 %offset, %vec %in1, %vec %i
 ; rather than an intrinsic.
 define {%vec, ptr} @test_neon_ld1_post_lane(ptr %addr, i32 %offset, %vec %in) {
 ; CHECK-LABEL: test_neon_ld1_post_lane:
-; CHECK: sbfiz [[OFFSET:x[0-9]+]], x1, #3, #32
-; CHECK: ld1.d { v0 }[0], [x0], [[OFFSET]]
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sbfiz x8, x1, #3, #32
+; CHECK-NEXT:    ld1.d { v0 }[0], [x0], x8
+; CHECK-NEXT:    ret
   %loaded = load double, ptr %addr, align 8
   %newvec = insertelement %vec %in, double %loaded, i32 0
-
   %addr.new = getelementptr inbounds double, ptr %addr, i32 %offset
-
   %res.tmp = insertvalue {%vec, ptr} undef, %vec %newvec, 0
   %res = insertvalue {%vec, ptr} %res.tmp, ptr %addr.new, 1
-
   ret {%vec, ptr} %res
 }
 
 define {{%vec, %vec}, ptr} @test_neon_load_post_exact(ptr %addr) {
 ; CHECK-LABEL: test_neon_load_post_exact:
-; CHECK: ld2.2d { v0, v1 }, [x0], #32
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ld2.2d { v0, v1 }, [x0], #32
+; CHECK-NEXT:    ret
   %vecs = call {%vec, %vec} @llvm.aarch64.neon.ld2.v2f64.p0(ptr %addr)
-
   %addr.new = getelementptr inbounds i8, ptr %addr, i32 32
-
   %res.tmp = insertvalue {{%vec, %vec}, ptr} undef, {%vec, %vec} %vecs, 0
   %res = insertvalue {{%vec, %vec}, ptr} %res.tmp, ptr %addr.new, 1
   ret {{%vec, %vec}, ptr} %res
@@ -148,16 +173,14 @@ define {{%vec, %vec}, ptr} @test_neon_load_post_exact(ptr %addr) {
 
 define {%vec, ptr} @test_neon_ld1_post_lane_exact(ptr %addr, %vec %in) {
 ; CHECK-LABEL: test_neon_ld1_post_lane_exact:
-; CHECK: ld1.d { v0 }[0], [x0], #8
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ld1.d { v0 }[0], [x0], #8
+; CHECK-NEXT:    ret
   %loaded = load double, ptr %addr, align 8
   %newvec = insertelement %vec %in, double %loaded, i32 0
-
   %addr.new = getelementptr inbounds double, ptr %addr, i32 1
-
   %res.tmp = insertvalue {%vec, ptr} undef, %vec %newvec, 0
   %res = insertvalue {%vec, ptr} %res.tmp, ptr %addr.new, 1
-
   ret {%vec, ptr} %res
 }
 
@@ -165,34 +188,28 @@ define {%vec, ptr} @test_neon_ld1_post_lane_exact(ptr %addr, %vec %in) {
 ; address wraps. We cannot use post-indexed addressing.
 define {%vec, ptr} @test_neon_ld1_notpost_lane_exact(ptr %addr, %vec %in) {
 ; CHECK-LABEL: test_neon_ld1_notpost_lane_exact:
-; CHECK-NOT: ld1.d { {{v[0-9]+}} }[0], [{{x[0-9]+|sp}}], #8
-; CHECK: add w0, w0, #8
-; CHECK: ret
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ld1.d { v0 }[0], [x0]
+; CHECK-NEXT:    add w0, w0, #8
+; CHECK-NEXT:    ret
   %loaded = load double, ptr %addr, align 8
   %newvec = insertelement %vec %in, double %loaded, i32 0
-
   %addr.new = getelementptr double, ptr %addr, i32 1
-
   %res.tmp = insertvalue {%vec, ptr} undef, %vec %newvec, 0
   %res = insertvalue {%vec, ptr} %res.tmp, ptr %addr.new, 1
-
   ret {%vec, ptr} %res
 }
 
 define {%vec, ptr} @test_neon_ld1_notpost_lane(ptr %addr, i32 %offset, %vec %in) {
 ; CHECK-LABEL: test_neon_ld1_notpost_lane:
-; CHECK-NOT: ld1.d { {{v[0-9]+}} }[0], [{{x[0-9]+|sp}}], {{x[0-9]+|sp}}
-; CHECK: add w0, w0, w1, lsl #3
-; CHECK: ret
-
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ld1.d { v0 }[0], [x0]
+; CHECK-NEXT:    add w0, w0, w1, lsl #3
+; CHECK-NEXT:    ret
   %loaded = load double, ptr %addr, align 8
   %newvec = insertelement %vec %in, double %loaded, i32 0
-
   %addr.new = getelementptr double, ptr %addr, i32 %offset
-
   %res.tmp = insertvalue {%vec, ptr} undef, %vec %newvec, 0
   %res = insertvalue {%vec, ptr} %res.tmp, ptr %addr.new, 1
-
   ret {%vec, ptr} %res
 }
diff --git a/llvm/test/CodeGen/AArch64/bcax.ll b/llvm/test/CodeGen/AArch64/bcax.ll
index e3c73c36e534b..e2274a9ce2f0e 100644
--- a/llvm/test/CodeGen/AArch64/bcax.ll
+++ b/llvm/test/CodeGen/AArch64/bcax.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub
 ; RUN: llc -mtriple=aarch64 -mattr=+sha3 < %s | FileCheck --check-prefix=SHA3 %s
 ; RUN: llc -mtriple=aarch64 -mattr=-sha3 < %s | FileCheck --check-prefix=NOSHA3 %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck --check-prefix=SVE2 %s
+; RUN: llc -mtriple=aarch64 -mattr=+sha3,+sve2 < %s | FileCheck --check-prefix=SHA3 %s
 
 define <2 x i64> @bcax_64x2(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
 ; SHA3-LABEL: bcax_64x2:
@@ -13,6 +15,15 @@ define <2 x i64> @bcax_64x2(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
 ; NOSHA3-NEXT:    bic v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v2.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: bcax_64x2:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    bcax z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <2 x i64> %1, <i64 -1, i64 -1>
   %5 = and <2 x i64> %4, %0
   %6 = xor <2 x i64> %5, %2
@@ -30,6 +41,15 @@ define <4 x i32> @bcax_32x4(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2) {
 ; NOSHA3-NEXT:    bic v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v2.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: bcax_32x4:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    bcax z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
   %5 = and <4 x i32> %4, %0
   %6 = xor <4 x i32> %5, %2
@@ -47,6 +67,15 @@ define <8 x i16> @bcax_16x8(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2) {
 ; NOSHA3-NEXT:    bic v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v2.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: bcax_16x8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    bcax z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <8 x i16> %1, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %5 = and <8 x i16> %4, %0
   %6 = xor <8 x i16> %5, %2
@@ -64,6 +93,15 @@ define <16 x i8> @bcax_8x16(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2) {
 ; NOSHA3-NEXT:    bic v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v2.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: bcax_8x16:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    bcax z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <16 x i8> %1, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %5 = and <16 x i8> %4, %0
   %6 = xor <16 x i8> %5, %2
diff --git a/llvm/test/CodeGen/AArch64/bf16_fast_math.ll b/llvm/test/CodeGen/AArch64/bf16_fast_math.ll
index 871ca12c9de77..e52c76fcc3f20 100644
--- a/llvm/test/CodeGen/AArch64/bf16_fast_math.ll
+++ b/llvm/test/CodeGen/AArch64/bf16_fast_math.ll
@@ -19,13 +19,13 @@ define bfloat @normal_fadd(bfloat %x, bfloat %y) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY3:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[COPY3]], killed [[COPY2]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY4]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY4]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], killed [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr16 = COPY [[COPY5]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY6]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -65,13 +65,13 @@ define bfloat @fast_fadd(bfloat %x, bfloat %y) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY3:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept FADDSrr killed [[COPY3]], killed [[COPY2]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY4]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY4]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], killed [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr16 = COPY [[COPY5]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY6]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -111,13 +111,13 @@ define bfloat @ninf_fadd(bfloat %x, bfloat %y) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY3:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[COPY3]], killed [[COPY2]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY4]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY4]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], killed [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr16 = COPY [[COPY5]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY6]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -161,13 +161,13 @@ define bfloat @normal_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[COPY4]], killed [[COPY3]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY5]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY5]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY7:%[0-9]+]]:fpr16 = COPY [[COPY6]].hsub
   ; CHECK-NOBF16-NEXT:   [[SUBREG_TO_REG2:%[0-9]+]]:fpr64 = SUBREG_TO_REG 0, killed [[COPY7]], %subreg.hsub
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_2:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG2]]
@@ -176,12 +176,12 @@ define bfloat @normal_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_3:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG3]]
   ; CHECK-NOBF16-NEXT:   [[COPY9:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_3]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr1:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[COPY8]], killed [[COPY9]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY [[FADDSrr1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr1]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri2:%[0-9]+]]:gpr32 = UBFMWri [[COPY10]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri2]], [[COPY10]]
   ; CHECK-NOBF16-NEXT:   [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr2]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri3:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr3]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY [[UBFMWri3]]
+  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri3]]
   ; CHECK-NOBF16-NEXT:   [[COPY12:%[0-9]+]]:fpr16 = COPY [[COPY11]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY12]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -232,13 +232,13 @@ define bfloat @nnan_ninf_contract_fadd_sequence(bfloat %x, bfloat %y, bfloat %z)
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nnan ninf contract nofpexcept FADDSrr killed [[COPY4]], killed [[COPY3]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY5]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY5]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY7:%[0-9]+]]:fpr16 = COPY [[COPY6]].hsub
   ; CHECK-NOBF16-NEXT:   [[SUBREG_TO_REG2:%[0-9]+]]:fpr64 = SUBREG_TO_REG 0, killed [[COPY7]], %subreg.hsub
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_2:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG2]]
@@ -247,12 +247,12 @@ define bfloat @nnan_ninf_contract_fadd_sequence(bfloat %x, bfloat %y, bfloat %z)
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_3:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG3]]
   ; CHECK-NOBF16-NEXT:   [[COPY9:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_3]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr1:%[0-9]+]]:fpr32 = nnan ninf contract nofpexcept FADDSrr killed [[COPY8]], killed [[COPY9]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY [[FADDSrr1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr1]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri2:%[0-9]+]]:gpr32 = UBFMWri [[COPY10]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri2]], [[COPY10]]
   ; CHECK-NOBF16-NEXT:   [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr2]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri3:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr3]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY [[UBFMWri3]]
+  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri3]]
   ; CHECK-NOBF16-NEXT:   [[COPY12:%[0-9]+]]:fpr16 = COPY [[COPY11]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY12]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -299,13 +299,13 @@ define bfloat @ninf_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[COPY4]], killed [[COPY3]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY5]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY5]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY7:%[0-9]+]]:fpr16 = COPY [[COPY6]].hsub
   ; CHECK-NOBF16-NEXT:   [[SUBREG_TO_REG2:%[0-9]+]]:fpr64 = SUBREG_TO_REG 0, killed [[COPY7]], %subreg.hsub
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_2:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG2]]
@@ -314,12 +314,12 @@ define bfloat @ninf_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_3:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG3]]
   ; CHECK-NOBF16-NEXT:   [[COPY9:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_3]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr1:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[COPY8]], killed [[COPY9]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY [[FADDSrr1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr1]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri2:%[0-9]+]]:gpr32 = UBFMWri [[COPY10]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri2]], [[COPY10]]
   ; CHECK-NOBF16-NEXT:   [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr2]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri3:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr3]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY [[UBFMWri3]]
+  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri3]]
   ; CHECK-NOBF16-NEXT:   [[COPY12:%[0-9]+]]:fpr16 = COPY [[COPY11]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY12]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/bsl.ll b/llvm/test/CodeGen/AArch64/bsl.ll
new file mode 100644
index 0000000000000..5a270bc71cfc1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bsl.ll
@@ -0,0 +1,433 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=NEON
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 < %s | FileCheck %s --check-prefix=SVE2
+
+; Test SVE2 BSL/NBSL/BSL1N/BSL2N code generation for:
+;   #define BSL(x,y,z)   (  ((x) & (z)) | ( (y) & ~(z)))
+;   #define NBSL(x,y,z)  (~(((x) & (z)) | ( (y) & ~(z))))
+;   #define BSL1N(x,y,z) ( (~(x) & (z)) | ( (y) & ~(z)))
+;   #define BSL2N(x,y,z) (  ((x) & (z)) | (~(y) & ~(z)))
+;
+; See also llvm/test/CodeGen/AArch64/sve2-bsl.ll.
+
+; Test basic codegen.
+
+define <1 x i64> @bsl_v1i64(<1 x i64> %0, <1 x i64> %1, <1 x i64> %2) {
+; NEON-LABEL: bsl_v1i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl_v1i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    bif v0.8b, v1.8b, v2.8b
+; SVE2-NEXT:    ret
+  %4 = and <1 x i64> %2, %0
+  %5 = xor <1 x i64> %2, splat (i64 -1)
+  %6 = and <1 x i64> %1, %5
+  %7 = or <1 x i64> %4, %6
+  ret <1 x i64> %7
+}
+
+define <1 x i64> @nbsl_v1i64(<1 x i64> %0, <1 x i64> %1, <1 x i64> %2) {
+; NEON-LABEL: nbsl_v1i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NEON-NEXT:    mvn v0.8b, v0.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v1i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <1 x i64> %2, %0
+  %5 = xor <1 x i64> %2, splat (i64 -1)
+  %6 = and <1 x i64> %1, %5
+  %7 = or <1 x i64> %4, %6
+  %8 = xor <1 x i64> %7, splat (i64 -1)
+  ret <1 x i64> %8
+}
+
+define <1 x i64> @bsl1n_v1i64(<1 x i64> %0, <1 x i64> %1, <1 x i64> %2) {
+; NEON-LABEL: bsl1n_v1i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    mvn v0.8b, v0.8b
+; NEON-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl1n_v1i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    bsl1n z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = xor <1 x i64> %0, splat (i64 -1)
+  %5 = and <1 x i64> %2, %4
+  %6 = xor <1 x i64> %2, splat (i64 -1)
+  %7 = and <1 x i64> %1, %6
+  %8 = or <1 x i64> %5, %7
+  ret <1 x i64> %8
+}
+
+define <1 x i64> @bsl2n_v1i64(<1 x i64> %0, <1 x i64> %1, <1 x i64> %2) {
+; NEON-LABEL: bsl2n_v1i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    and v0.8b, v2.8b, v0.8b
+; NEON-NEXT:    orr v1.8b, v2.8b, v1.8b
+; NEON-NEXT:    orn v0.8b, v0.8b, v1.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl2n_v1i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    bsl2n z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <1 x i64> %2, %0
+  %5 = or <1 x i64> %2, %1
+  %6 = xor <1 x i64> %5, splat (i64 -1)
+  %7 = or <1 x i64> %4, %6
+  ret <1 x i64> %7
+}
+
+define <2 x i64> @bsl_v2i64(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
+; NEON-LABEL: bsl_v2i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl_v2i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    bif v0.16b, v1.16b, v2.16b
+; SVE2-NEXT:    ret
+  %4 = and <2 x i64> %2, %0
+  %5 = xor <2 x i64> %2, splat (i64 -1)
+  %6 = and <2 x i64> %1, %5
+  %7 = or <2 x i64> %4, %6
+  ret <2 x i64> %7
+}
+
+define <2 x i64> @nbsl_v2i64(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
+; NEON-LABEL: nbsl_v2i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NEON-NEXT:    mvn v0.16b, v0.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v2i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <2 x i64> %2, %0
+  %5 = xor <2 x i64> %2, splat (i64 -1)
+  %6 = and <2 x i64> %1, %5
+  %7 = or <2 x i64> %4, %6
+  %8 = xor <2 x i64> %7, splat (i64 -1)
+  ret <2 x i64> %8
+}
+
+define <2 x i64> @bsl1n_v2i64(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
+; NEON-LABEL: bsl1n_v2i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    mvn v0.16b, v0.16b
+; NEON-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl1n_v2i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    bsl1n z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = xor <2 x i64> %0, splat (i64 -1)
+  %5 = and <2 x i64> %2, %4
+  %6 = xor <2 x i64> %2, splat (i64 -1)
+  %7 = and <2 x i64> %1, %6
+  %8 = or <2 x i64> %5, %7
+  ret <2 x i64> %8
+}
+
+define <2 x i64> @bsl2n_v2i64(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
+; NEON-LABEL: bsl2n_v2i64:
+; NEON:       // %bb.0:
+; NEON-NEXT:    and v0.16b, v2.16b, v0.16b
+; NEON-NEXT:    orr v1.16b, v2.16b, v1.16b
+; NEON-NEXT:    orn v0.16b, v0.16b, v1.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl2n_v2i64:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    bsl2n z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <2 x i64> %2, %0
+  %5 = or <2 x i64> %2, %1
+  %6 = xor <2 x i64> %5, splat (i64 -1)
+  %7 = or <2 x i64> %4, %6
+  ret <2 x i64> %7
+}
+
+; Test other element types.
+
+define <8 x i8> @nbsl_v8i8(<8 x i8> %0, <8 x i8> %1, <8 x i8> %2) {
+; NEON-LABEL: nbsl_v8i8:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NEON-NEXT:    mvn v0.8b, v0.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v8i8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <8 x i8> %2, %0
+  %5 = xor <8 x i8> %2, splat (i8 -1)
+  %6 = and <8 x i8> %1, %5
+  %7 = or <8 x i8> %4, %6
+  %8 = xor <8 x i8> %7, splat (i8 -1)
+  ret <8 x i8> %8
+}
+
+define <4 x i16> @nbsl_v4i16(<4 x i16> %0, <4 x i16> %1, <4 x i16> %2) {
+; NEON-LABEL: nbsl_v4i16:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NEON-NEXT:    mvn v0.8b, v0.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v4i16:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <4 x i16> %2, %0
+  %5 = xor <4 x i16> %2, splat (i16 -1)
+  %6 = and <4 x i16> %1, %5
+  %7 = or <4 x i16> %4, %6
+  %8 = xor <4 x i16> %7, splat (i16 -1)
+  ret <4 x i16> %8
+}
+
+define <2 x i32> @nbsl_v2i32(<2 x i32> %0, <2 x i32> %1, <2 x i32> %2) {
+; NEON-LABEL: nbsl_v2i32:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.8b, v1.8b, v2.8b
+; NEON-NEXT:    mvn v0.8b, v0.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v2i32:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE2-NEXT:    // kill: def $d2 killed $d2 def $z2
+; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <2 x i32> %2, %0
+  %5 = xor <2 x i32> %2, splat (i32 -1)
+  %6 = and <2 x i32> %1, %5
+  %7 = or <2 x i32> %4, %6
+  %8 = xor <2 x i32> %7, splat (i32 -1)
+  ret <2 x i32> %8
+}
+
+define <16 x i8> @nbsl_v16i8(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2) {
+; NEON-LABEL: nbsl_v16i8:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NEON-NEXT:    mvn v0.16b, v0.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v16i8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <16 x i8> %2, %0
+  %5 = xor <16 x i8> %2, splat (i8 -1)
+  %6 = and <16 x i8> %1, %5
+  %7 = or <16 x i8> %4, %6
+  %8 = xor <16 x i8> %7, splat (i8 -1)
+  ret <16 x i8> %8
+}
+
+define <8 x i16> @nbsl_v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2) {
+; NEON-LABEL: nbsl_v8i16:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NEON-NEXT:    mvn v0.16b, v0.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v8i16:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <8 x i16> %2, %0
+  %5 = xor <8 x i16> %2, splat (i16 -1)
+  %6 = and <8 x i16> %1, %5
+  %7 = or <8 x i16> %4, %6
+  %8 = xor <8 x i16> %7, splat (i16 -1)
+  ret <8 x i16> %8
+}
+
+define <4 x i32> @nbsl_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2) {
+; NEON-LABEL: nbsl_v4i32:
+; NEON:       // %bb.0:
+; NEON-NEXT:    bif v0.16b, v1.16b, v2.16b
+; NEON-NEXT:    mvn v0.16b, v0.16b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v4i32:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    nbsl z0.d, z0.d, z1.d, z2.d
+; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT:    ret
+  %4 = and <4 x i32> %2, %0
+  %5 = xor <4 x i32> %2, splat (i32 -1)
+  %6 = and <4 x i32> %1, %5
+  %7 = or <4 x i32> %4, %6
+  %8 = xor <4 x i32> %7, splat (i32 -1)
+  ret <4 x i32> %8
+}
+
+; Test types that need promotion.
+
+define <4 x i8> @bsl_v4i8(<4 x i8> %0, <4 x i8> %1, <4 x i8> %2) {
+; NEON-LABEL: bsl_v4i8:
+; NEON:       // %bb.0:
+; NEON-NEXT:    movi d3, #0xff00ff00ff00ff
+; NEON-NEXT:    and v0.8b, v2.8b, v0.8b
+; NEON-NEXT:    eor v3.8b, v2.8b, v3.8b
+; NEON-NEXT:    and v1.8b, v1.8b, v3.8b
+; NEON-NEXT:    orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl_v4i8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    movi d3, #0xff00ff00ff00ff
+; SVE2-NEXT:    and v0.8b, v2.8b, v0.8b
+; SVE2-NEXT:    eor v3.8b, v2.8b, v3.8b
+; SVE2-NEXT:    and v1.8b, v1.8b, v3.8b
+; SVE2-NEXT:    orr v0.8b, v0.8b, v1.8b
+; SVE2-NEXT:    ret
+  %4 = and <4 x i8> %2, %0
+  %5 = xor <4 x i8> %2, splat (i8 -1)
+  %6 = and <4 x i8> %1, %5
+  %7 = or <4 x i8> %4, %6
+  ret <4 x i8> %7
+}
+
+define <4 x i8> @nbsl_v4i8(<4 x i8> %0, <4 x i8> %1, <4 x i8> %2) {
+; NEON-LABEL: nbsl_v4i8:
+; NEON:       // %bb.0:
+; NEON-NEXT:    movi d3, #0xff00ff00ff00ff
+; NEON-NEXT:    and v0.8b, v2.8b, v0.8b
+; NEON-NEXT:    eor v4.8b, v2.8b, v3.8b
+; NEON-NEXT:    and v1.8b, v1.8b, v4.8b
+; NEON-NEXT:    orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT:    eor v0.8b, v0.8b, v3.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: nbsl_v4i8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    movi d3, #0xff00ff00ff00ff
+; SVE2-NEXT:    and v0.8b, v2.8b, v0.8b
+; SVE2-NEXT:    eor v4.8b, v2.8b, v3.8b
+; SVE2-NEXT:    and v1.8b, v1.8b, v4.8b
+; SVE2-NEXT:    orr v0.8b, v0.8b, v1.8b
+; SVE2-NEXT:    eor v0.8b, v0.8b, v3.8b
+; SVE2-NEXT:    ret
+  %4 = and <4 x i8> %2, %0
+  %5 = xor <4 x i8> %2, splat (i8 -1)
+  %6 = and <4 x i8> %1, %5
+  %7 = or <4 x i8> %4, %6
+  %8 = xor <4 x i8> %7, splat (i8 -1)
+  ret <4 x i8> %8
+}
+
+define <4 x i8> @bsl1n_v4i8(<4 x i8> %0, <4 x i8> %1, <4 x i8> %2) {
+; NEON-LABEL: bsl1n_v4i8:
+; NEON:       // %bb.0:
+; NEON-NEXT:    movi d3, #0xff00ff00ff00ff
+; NEON-NEXT:    eor v0.8b, v0.8b, v3.8b
+; NEON-NEXT:    eor v3.8b, v2.8b, v3.8b
+; NEON-NEXT:    and v0.8b, v2.8b, v0.8b
+; NEON-NEXT:    and v1.8b, v1.8b, v3.8b
+; NEON-NEXT:    orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl1n_v4i8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    movi d3, #0xff00ff00ff00ff
+; SVE2-NEXT:    eor v0.8b, v0.8b, v3.8b
+; SVE2-NEXT:    eor v3.8b, v2.8b, v3.8b
+; SVE2-NEXT:    and v0.8b, v2.8b, v0.8b
+; SVE2-NEXT:    and v1.8b, v1.8b, v3.8b
+; SVE2-NEXT:    orr v0.8b, v0.8b, v1.8b
+; SVE2-NEXT:    ret
+  %4 = xor <4 x i8> %0, splat (i8 -1)
+  %5 = and <4 x i8> %2, %4
+  %6 = xor <4 x i8> %2, splat (i8 -1)
+  %7 = and <4 x i8> %1, %6
+  %8 = or <4 x i8> %5, %7
+  ret <4 x i8> %8
+}
+
+define <4 x i8> @bsl2n_v4i8(<4 x i8> %0, <4 x i8> %1, <4 x i8> %2) {
+; NEON-LABEL: bsl2n_v4i8:
+; NEON:       // %bb.0:
+; NEON-NEXT:    movi d3, #0xff00ff00ff00ff
+; NEON-NEXT:    orr v1.8b, v2.8b, v1.8b
+; NEON-NEXT:    and v0.8b, v2.8b, v0.8b
+; NEON-NEXT:    eor v1.8b, v1.8b, v3.8b
+; NEON-NEXT:    orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT:    ret
+;
+; SVE2-LABEL: bsl2n_v4i8:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    movi d3, #0xff00ff00ff00ff
+; SVE2-NEXT:    orr v1.8b, v2.8b, v1.8b
+; SVE2-NEXT:    and v0.8b, v2.8b, v0.8b
+; SVE2-NEXT:    eor v1.8b, v1.8b, v3.8b
+; SVE2-NEXT:    orr v0.8b, v0.8b, v1.8b
+; SVE2-NEXT:    ret
+  %4 = and <4 x i8> %2, %0
+  %5 = or <4 x i8> %2, %1
+  %6 = xor <4 x i8> %5, splat (i8 -1)
+  %7 = or <4 x i8> %4, %6
+  ret <4 x i8> %7
+}
diff --git a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
index b4f179e992a0d..c2bb032eed78e 100644
--- a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
+++ b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
@@ -197,11 +197,9 @@ define <4 x i32> @not_sign_4xi32(<4 x i32> %a) {
 ; CHECK-LABEL: not_sign_4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI16_0
-; CHECK-NEXT:    movi v2.4s, #1
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
-; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    and v1.16b, v0.16b, v2.16b
-; CHECK-NEXT:    orn v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v0.4s, #1
 ; CHECK-NEXT:    ret
   %c = icmp sgt <4 x i32> %a, <i32 1, i32 -1, i32 -1, i32 -1>
   %res = select <4 x i1> %c, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/AArch64/concatbinop.ll b/llvm/test/CodeGen/AArch64/concatbinop.ll
index 828182d18b38c..062a5a8c35b2c 100644
--- a/llvm/test/CodeGen/AArch64/concatbinop.ll
+++ b/llvm/test/CodeGen/AArch64/concatbinop.ll
@@ -179,7 +179,7 @@ define <16 x i8> @signOf_neon(ptr nocapture noundef readonly %a, ptr nocapture n
 ; CHECK-NEXT:    uzp1 v3.16b, v5.16b, v6.16b
 ; CHECK-NEXT:    uzp1 v1.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <8 x i16>, ptr %a, align 2
diff --git a/llvm/test/CodeGen/AArch64/eor3.ll b/llvm/test/CodeGen/AArch64/eor3.ll
index 06ae6b09d002e..b89d9d608575c 100644
--- a/llvm/test/CodeGen/AArch64/eor3.ll
+++ b/llvm/test/CodeGen/AArch64/eor3.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub
 ; RUN: llc -mtriple=aarch64 -mattr=+sha3 < %s | FileCheck --check-prefix=SHA3 %s
 ; RUN: llc -mtriple=aarch64 -mattr=-sha3 < %s | FileCheck --check-prefix=NOSHA3 %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck --check-prefix=SVE2 %s
+; RUN: llc -mtriple=aarch64 -mattr=+sha3,+sve2 < %s | FileCheck --check-prefix=SHA3 %s
 
 define <16 x i8> @eor3_16x8_left(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2) {
 ; SHA3-LABEL: eor3_16x8_left:
@@ -13,6 +15,15 @@ define <16 x i8> @eor3_16x8_left(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2) {
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v2.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_16x8_left:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <16 x i8> %0, %1
   %5 = xor <16 x i8> %2, %4
   ret <16 x i8> %5
@@ -29,6 +40,15 @@ define <16 x i8> @eor3_16x8_right(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2) {
 ; NOSHA3-NEXT:    eor v1.16b, v1.16b, v2.16b
 ; NOSHA3-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_16x8_right:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z1.d, z1.d, z2.d, z0.d
+; SVE2-NEXT:    mov v0.16b, v1.16b
+; SVE2-NEXT:    ret
   %4 = xor <16 x i8> %1, %2
   %5 = xor <16 x i8> %4, %0
   ret <16 x i8> %5
@@ -45,6 +65,15 @@ define <8 x i16> @eor3_8x16_left(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2) {
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v2.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_8x16_left:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <8 x i16> %0, %1
   %5 = xor <8 x i16> %2, %4
   ret <8 x i16> %5
@@ -61,6 +90,15 @@ define <8 x i16> @eor3_8x16_right(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2) {
 ; NOSHA3-NEXT:    eor v1.16b, v1.16b, v2.16b
 ; NOSHA3-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_8x16_right:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z1.d, z1.d, z2.d, z0.d
+; SVE2-NEXT:    mov v0.16b, v1.16b
+; SVE2-NEXT:    ret
   %4 = xor <8 x i16> %1, %2
   %5 = xor <8 x i16> %4, %0
   ret <8 x i16> %5
@@ -77,6 +115,15 @@ define <4 x i32> @eor3_4x32_left(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2) {
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v2.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_4x32_left:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <4 x i32> %0, %1
   %5 = xor <4 x i32> %2, %4
   ret <4 x i32> %5
@@ -93,6 +140,15 @@ define <4 x i32> @eor3_4x32_right(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2) {
 ; NOSHA3-NEXT:    eor v1.16b, v1.16b, v2.16b
 ; NOSHA3-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_4x32_right:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z1.d, z1.d, z2.d, z0.d
+; SVE2-NEXT:    mov v0.16b, v1.16b
+; SVE2-NEXT:    ret
   %4 = xor <4 x i32> %1, %2
   %5 = xor <4 x i32> %4, %0
   ret <4 x i32> %5
@@ -109,6 +165,15 @@ define <2 x i64> @eor3_2x64_left(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    eor v0.16b, v2.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_2x64_left:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    mov v0.16b, v2.16b
+; SVE2-NEXT:    ret
   %4 = xor <2 x i64> %0, %1
   %5 = xor <2 x i64> %2, %4
   ret <2 x i64> %5
@@ -125,6 +190,15 @@ define <2 x i64> @eor3_2x64_right(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) {
 ; NOSHA3-NEXT:    eor v1.16b, v1.16b, v2.16b
 ; NOSHA3-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_2x64_right:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
+; SVE2-NEXT:    // kill: def $q2 killed $q2 def $z2
+; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE2-NEXT:    eor3 z1.d, z1.d, z2.d, z0.d
+; SVE2-NEXT:    mov v0.16b, v1.16b
+; SVE2-NEXT:    ret
   %4 = xor <2 x i64> %1, %2
   %5 = xor <2 x i64> %4, %0
   ret <2 x i64> %5
@@ -142,6 +216,12 @@ define <2 x i64> @eor3_vnot(<2 x i64> %0, <2 x i64> %1) {
 ; NOSHA3-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; NOSHA3-NEXT:    mvn v0.16b, v0.16b
 ; NOSHA3-NEXT:    ret
+;
+; SVE2-LABEL: eor3_vnot:
+; SVE2:       // %bb.0:
+; SVE2-NEXT:    eor v0.16b, v0.16b, v1.16b
+; SVE2-NEXT:    mvn v0.16b, v0.16b
+; SVE2-NEXT:    ret
   %3 = xor <2 x i64> %0, <i64 -1, i64 -1>
   %4 = xor <2 x i64> %3, %1
   ret <2 x i64> %4
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
index 0189f52bbac06..18b028c9898e1 100644
--- a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
@@ -331,7 +331,7 @@ define float @extract_v4i32_minimum(<4 x float> %a, <4 x float> %b, i32 %c) {
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
 entry:
-  %vector = call <4 x float> @llvm.minimum.v4float(<4 x float> %a, <4 x float> %b)
+  %vector = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> %b)
   %d = extractelement <4 x float> %vector, i32 %c
   ret float %d
 }
@@ -367,7 +367,7 @@ define float @extract_v4i32_minimum_build_vector(<4 x float> %a, <4 x float> %b,
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
 entry:
-  %vector = call <4 x float> @llvm.minimum.v4float(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
+  %vector = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
   %d = extractelement <4 x float> %vector, i32 %c
   ret float %d
 }
@@ -381,7 +381,7 @@ define float @extract_v4i32_minimum_build_vector_const(<4 x float> %a, <4 x floa
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    ret
 entry:
-  %vector = call <4 x float> @llvm.minimum.v4float(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
+  %vector = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
   %d = extractelement <4 x float> %vector, i32 1
   ret float %d
 }
@@ -414,7 +414,7 @@ define float @extract_v4i32_copysign_build_vector(<4 x float> %a, <4 x float> %b
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
 entry:
-  %vector = call <4 x float> @llvm.copysign.v4float(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
+  %vector = call <4 x float> @llvm.copysign.v4f32(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
   %d = extractelement <4 x float> %vector, i32 %c
   ret float %d
 }
@@ -433,7 +433,7 @@ define float @extract_v4i32_copysign_build_vector_const(<4 x float> %a, <4 x flo
 ; CHECK-GI-NEXT:    mov s0, v0.s[2]
 ; CHECK-GI-NEXT:    ret
 entry:
-  %vector = call <4 x float> @llvm.copysign.v4float(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
+  %vector = call <4 x float> @llvm.copysign.v4f32(<4 x float> %a, <4 x float> <float 42.0, float 11.0, float 17.0, float 6.0>)
   %d = extractelement <4 x float> %vector, i32 2
   ret float %d
 }
@@ -508,8 +508,8 @@ entry:
   ret i32 %d
 }
 
-define i32 @extract_v4float_fcmp(<4 x float> %a, <4 x float> %b, i32 %c) {
-; CHECK-SD-LABEL: extract_v4float_fcmp:
+define i32 @extract_v4f32_fcmp(<4 x float> %a, <4 x float> %b, i32 %c) {
+; CHECK-SD-LABEL: extract_v4f32_fcmp:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    sub sp, sp, #16
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
@@ -524,7 +524,7 @@ define i32 @extract_v4float_fcmp(<4 x float> %a, <4 x float> %b, i32 %c) {
 ; CHECK-SD-NEXT:    add sp, sp, #16
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: extract_v4float_fcmp:
+; CHECK-GI-LABEL: extract_v4f32_fcmp:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    sub sp, sp, #16
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
@@ -548,8 +548,8 @@ entry:
   ret i32 %d
 }
 
-define i32 @extract_v4float_fcmp_const(<4 x float> %a, <4 x float> %b, i32 %c) {
-; CHECK-SD-LABEL: extract_v4float_fcmp_const:
+define i32 @extract_v4f32_fcmp_const(<4 x float> %a, <4 x float> %b, i32 %c) {
+; CHECK-SD-LABEL: extract_v4f32_fcmp_const:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    movi v1.4s, #1
 ; CHECK-SD-NEXT:    fcmeq v0.4s, v0.4s, v0.4s
@@ -557,7 +557,7 @@ define i32 @extract_v4float_fcmp_const(<4 x float> %a, <4 x float> %b, i32 %c) {
 ; CHECK-SD-NEXT:    mov w0, v0.s[1]
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: extract_v4float_fcmp_const:
+; CHECK-GI-LABEL: extract_v4f32_fcmp_const:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    fmov v1.4s, #1.00000000
 ; CHECK-GI-NEXT:    fcmge v2.4s, v0.4s, v1.4s
@@ -680,9 +680,9 @@ define i32 @extract_v4i32_abs(<4 x float> %a, i32 %c) {
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
 entry:
-  %ceil = call <4 x float> @llvm.ceil.v4float(<4 x float> %a)
-  %floor = call <4 x float> @llvm.floor.v4float(<4 x float> %ceil)
-  %fabs = call <4 x float> @llvm.fabs.v4float(<4 x float> %floor)
+  %ceil = call <4 x float> @llvm.ceil.v4f32(<4 x float> %a)
+  %floor = call <4 x float> @llvm.floor.v4f32(<4 x float> %ceil)
+  %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %floor)
   %abs = fptosi <4 x float> %fabs to <4 x i32>
   %vector = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %abs, i1 0)
   %d = extractelement <4 x i32> %vector, i32 %c
@@ -708,9 +708,9 @@ define i32 @extract_v4i32_abs_const(<4 x float> %a, i32 %c) {
 ; CHECK-GI-NEXT:    fmov w0, s0
 ; CHECK-GI-NEXT:    ret
 entry:
-  %ceil = call <4 x float> @llvm.ceil.v4float(<4 x float> <float 1.0, float 4.0, float 3.0, float 2.0>)
-  %floor = call <4 x float> @llvm.floor.v4float(<4 x float> %ceil)
-  %fabs = call <4 x float> @llvm.fabs.v4float(<4 x float> %floor)
+  %ceil = call <4 x float> @llvm.ceil.v4f32(<4 x float> <float 1.0, float 4.0, float 3.0, float 2.0>)
+  %floor = call <4 x float> @llvm.floor.v4f32(<4 x float> %ceil)
+  %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %floor)
   %abs = fptosi <4 x float> %fabs to <4 x i32>
   %vector = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %abs, i1 0)
   %d = extractelement <4 x i32> %vector, i32 1
@@ -751,9 +751,9 @@ define i32 @extract_v4i32_abs_half_const(<4 x float> %a, i32 %c) {
 ; CHECK-GI-NEXT:    add sp, sp, #16
 ; CHECK-GI-NEXT:    ret
 entry:
-  %ceil = call <4 x float> @llvm.ceil.v4float(<4 x float> <float 1.0, float 4.0, float 3.0, float 2.0>)
-  %floor = call <4 x float> @llvm.floor.v4float(<4 x float> %ceil)
-  %fabs = call <4 x float> @llvm.fabs.v4float(<4 x float> %floor)
+  %ceil = call <4 x float> @llvm.ceil.v4f32(<4 x float> <float 1.0, float 4.0, float 3.0, float 2.0>)
+  %floor = call <4 x float> @llvm.floor.v4f32(<4 x float> %ceil)
+  %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %floor)
   %abs = fptosi <4 x float> %fabs to <4 x i32>
   %vector = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %abs, i1 0)
   %d = extractelement <4 x i32> %vector, i32 %c
diff --git a/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll b/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
index a1093c28467ab..de9811b92424e 100644
--- a/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
+++ b/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2,+fp8 < %s | FileCheck %s
 ; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-linux"
diff --git a/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll b/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll
index 2ffba10e21100..e42f2b1cfba48 100644
--- a/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll
+++ b/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2,+fp8 < %s | FileCheck %s
 ; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-linux"
diff --git a/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll b/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll
new file mode 100644
index 0000000000000..ee326caa77c0a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mtriple=arm64ec-windows-msvc < %s | FileCheck -check-prefix=ARM64EC %s
+
+; Separate from llvm-frexp.ll test because this errors on half cases
+
+; ARM64EC-LABEL: test_frexp_f32_i32
+; ARM64EC: fcvt d0, s0
+; ARM64EC: bl "#frexp"
+; ARM64EC: fcvt s0, d0
+define { float, i32 } @test_frexp_f32_i32(float %a) {
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+; ARM64EC-LABEL: test_frexp_f64_i32
+; ARM64EC: bl "#frexp"
+define { double, i32 } @test_frexp_f64_i32(double %a) {
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  ret { double, i32 } %result
+}
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index e8194b9bd9b27..217547c31f7f2 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -4309,42 +4309,30 @@ entry:
 }
 
 define <2 x float> @stofp_v2i64_v2f32(<2 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v2i64_v2f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    scvtf s0, x9
-; CHECK-SD-NEXT:    scvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i64_v2f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: stofp_v2i64_v2f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    scvtf s0, x9
+; CHECK-NEXT:    scvtf s1, x8
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %c = sitofp <2 x i64> %a to <2 x float>
   ret <2 x float> %c
 }
 
 define <2 x float> @utofp_v2i64_v2f32(<2 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v2i64_v2f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    ucvtf s0, x9
-; CHECK-SD-NEXT:    ucvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i64_v2f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: utofp_v2i64_v2f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    ucvtf s0, x9
+; CHECK-NEXT:    ucvtf s1, x8
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %c = uitofp <2 x i64> %a to <2 x float>
   ret <2 x float> %c
@@ -4370,16 +4358,13 @@ define <3 x float> @stofp_v3i64_v3f32(<3 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v3i64_v3f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v0.2d
-; CHECK-GI-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    scvtf s0, x8
+; CHECK-GI-NEXT:    scvtf s1, x9
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    scvtf s2, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
 ; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -4407,16 +4392,13 @@ define <3 x float> @utofp_v3i64_v3f32(<3 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v3i64_v3f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v0.2d
-; CHECK-GI-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    ucvtf s0, x8
+; CHECK-GI-NEXT:    ucvtf s1, x9
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    ucvtf s2, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
 ; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -4442,10 +4424,17 @@ define <4 x float> @stofp_v4i64_v4f32(<4 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v4i64_v4f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    scvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    scvtf s2, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    scvtf s1, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    scvtf s2, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i64> %a to <4 x float>
@@ -4470,10 +4459,17 @@ define <4 x float> @utofp_v4i64_v4f32(<4 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v4i64_v4f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    ucvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    ucvtf s2, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    ucvtf s1, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    ucvtf s2, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i64> %a to <4 x float>
@@ -4510,14 +4506,29 @@ define <8 x float> @stofp_v8i64_v8f32(<8 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v8i64_v8f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v4.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    scvtf s0, x10
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    scvtf s5, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    scvtf s4, x8
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    scvtf s2, x10
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    scvtf s1, x9
+; CHECK-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT:    scvtf s3, x10
+; CHECK-GI-NEXT:    scvtf s4, x8
+; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-GI-NEXT:    scvtf s5, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i64> %a to <8 x float>
@@ -4554,14 +4565,29 @@ define <8 x float> @utofp_v8i64_v8f32(<8 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v8i64_v8f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v4.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    ucvtf s0, x10
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    ucvtf s5, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    ucvtf s4, x8
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    ucvtf s2, x10
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    ucvtf s1, x9
+; CHECK-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT:    ucvtf s3, x10
+; CHECK-GI-NEXT:    ucvtf s4, x8
+; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ucvtf s5, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i64> %a to <8 x float>
@@ -4620,22 +4646,50 @@ define <16 x float> @stofp_v16i64_v16f32(<16 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v16i64_v16f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v16.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v7.2d
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x13, d2
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mov x10, v2.d[1]
+; CHECK-GI-NEXT:    scvtf s0, x11
+; CHECK-GI-NEXT:    mov x11, v6.d[1]
+; CHECK-GI-NEXT:    scvtf s16, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    scvtf s1, x13
+; CHECK-GI-NEXT:    fmov x13, d4
+; CHECK-GI-NEXT:    scvtf s4, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    scvtf s17, x10
+; CHECK-GI-NEXT:    mov x10, v3.d[1]
+; CHECK-GI-NEXT:    scvtf s6, x11
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    scvtf s18, x9
+; CHECK-GI-NEXT:    scvtf s2, x13
+; CHECK-GI-NEXT:    fmov x13, d3
+; CHECK-GI-NEXT:    scvtf s3, x12
+; CHECK-GI-NEXT:    mov x9, v5.d[1]
+; CHECK-GI-NEXT:    mov x12, v7.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v16.s[0]
+; CHECK-GI-NEXT:    scvtf s5, x11
+; CHECK-GI-NEXT:    mov v1.s[1], v17.s[0]
+; CHECK-GI-NEXT:    scvtf s19, x13
+; CHECK-GI-NEXT:    fmov x13, d7
+; CHECK-GI-NEXT:    mov v2.s[1], v4.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v6.s[0]
+; CHECK-GI-NEXT:    scvtf s4, x8
+; CHECK-GI-NEXT:    scvtf s6, x10
+; CHECK-GI-NEXT:    scvtf s16, x9
+; CHECK-GI-NEXT:    scvtf s17, x12
+; CHECK-GI-NEXT:    scvtf s7, x13
+; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v6.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i64> %a to <16 x float>
@@ -4694,22 +4748,50 @@ define <16 x float> @utofp_v16i64_v16f32(<16 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v16i64_v16f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v16.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v7.2d
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x13, d2
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mov x10, v2.d[1]
+; CHECK-GI-NEXT:    ucvtf s0, x11
+; CHECK-GI-NEXT:    mov x11, v6.d[1]
+; CHECK-GI-NEXT:    ucvtf s16, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    ucvtf s1, x13
+; CHECK-GI-NEXT:    fmov x13, d4
+; CHECK-GI-NEXT:    ucvtf s4, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    ucvtf s17, x10
+; CHECK-GI-NEXT:    mov x10, v3.d[1]
+; CHECK-GI-NEXT:    ucvtf s6, x11
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    ucvtf s18, x9
+; CHECK-GI-NEXT:    ucvtf s2, x13
+; CHECK-GI-NEXT:    fmov x13, d3
+; CHECK-GI-NEXT:    ucvtf s3, x12
+; CHECK-GI-NEXT:    mov x9, v5.d[1]
+; CHECK-GI-NEXT:    mov x12, v7.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v16.s[0]
+; CHECK-GI-NEXT:    ucvtf s5, x11
+; CHECK-GI-NEXT:    mov v1.s[1], v17.s[0]
+; CHECK-GI-NEXT:    ucvtf s19, x13
+; CHECK-GI-NEXT:    fmov x13, d7
+; CHECK-GI-NEXT:    mov v2.s[1], v4.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v6.s[0]
+; CHECK-GI-NEXT:    ucvtf s4, x8
+; CHECK-GI-NEXT:    ucvtf s6, x10
+; CHECK-GI-NEXT:    ucvtf s16, x9
+; CHECK-GI-NEXT:    ucvtf s17, x12
+; CHECK-GI-NEXT:    ucvtf s7, x13
+; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v6.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i64> %a to <16 x float>
@@ -4816,42 +4898,110 @@ define <32 x float> @stofp_v32i64_v32f32(<32 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v32i64_v32f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    scvtf v24.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    scvtf v25.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v26.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v27.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v18.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v20.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v22.2d
-; CHECK-GI-NEXT:    scvtf v16.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v19.2d
-; CHECK-GI-NEXT:    scvtf v18.2d, v21.2d
-; CHECK-GI-NEXT:    scvtf v19.2d, v23.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v1.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v3.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v5.2s, v5.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn v7.2s, v7.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v24.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v25.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v26.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v27.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v5.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v18.2d
-; CHECK-GI-NEXT:    fcvtn2 v7.4s, v19.2d
+; CHECK-GI-NEXT:    str d12, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset b8, -8
+; CHECK-GI-NEXT:    .cfi_offset b9, -16
+; CHECK-GI-NEXT:    .cfi_offset b10, -24
+; CHECK-GI-NEXT:    .cfi_offset b11, -32
+; CHECK-GI-NEXT:    .cfi_offset b12, -48
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x11, v3.d[1]
+; CHECK-GI-NEXT:    ldp q30, q25, [sp, #112]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    ldp q21, q24, [sp, #48]
+; CHECK-GI-NEXT:    ldp q11, q27, [sp, #80]
+; CHECK-GI-NEXT:    scvtf s28, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    scvtf s16, x10
+; CHECK-GI-NEXT:    mov x10, v6.d[1]
+; CHECK-GI-NEXT:    scvtf s9, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    scvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    scvtf s18, x11
+; CHECK-GI-NEXT:    scvtf s17, x8
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    ldp q31, q26, [sp, #144]
+; CHECK-GI-NEXT:    mov x13, v21.d[1]
+; CHECK-GI-NEXT:    scvtf s10, x10
+; CHECK-GI-NEXT:    fmov x10, d21
+; CHECK-GI-NEXT:    fmov x14, d30
+; CHECK-GI-NEXT:    scvtf s1, x8
+; CHECK-GI-NEXT:    fmov x8, d3
+; CHECK-GI-NEXT:    scvtf s3, x12
+; CHECK-GI-NEXT:    mov x12, v30.d[1]
+; CHECK-GI-NEXT:    scvtf s22, x11
+; CHECK-GI-NEXT:    fmov x11, d11
+; CHECK-GI-NEXT:    scvtf s29, x9
+; CHECK-GI-NEXT:    fmov x9, d4
+; CHECK-GI-NEXT:    scvtf s4, x10
+; CHECK-GI-NEXT:    scvtf s19, x8
+; CHECK-GI-NEXT:    mov x8, v5.d[1]
+; CHECK-GI-NEXT:    mov x10, v11.d[1]
+; CHECK-GI-NEXT:    scvtf s5, x11
+; CHECK-GI-NEXT:    mov x11, v31.d[1]
+; CHECK-GI-NEXT:    scvtf s8, x13
+; CHECK-GI-NEXT:    scvtf s11, x12
+; CHECK-GI-NEXT:    fmov x12, d31
+; CHECK-GI-NEXT:    scvtf s2, x9
+; CHECK-GI-NEXT:    mov x9, v7.d[1]
+; CHECK-GI-NEXT:    scvtf s6, x14
+; CHECK-GI-NEXT:    fmov x13, d27
+; CHECK-GI-NEXT:    scvtf s20, x8
+; CHECK-GI-NEXT:    fmov x8, d7
+; CHECK-GI-NEXT:    scvtf s12, x10
+; CHECK-GI-NEXT:    scvtf s7, x12
+; CHECK-GI-NEXT:    fmov x12, d25
+; CHECK-GI-NEXT:    scvtf s30, x11
+; CHECK-GI-NEXT:    mov x10, v25.d[1]
+; CHECK-GI-NEXT:    mov x11, v26.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v28.s[0]
+; CHECK-GI-NEXT:    scvtf s21, x9
+; CHECK-GI-NEXT:    fmov x9, d24
+; CHECK-GI-NEXT:    scvtf s23, x8
+; CHECK-GI-NEXT:    scvtf s25, x12
+; CHECK-GI-NEXT:    fmov x12, d26
+; CHECK-GI-NEXT:    mov x8, v24.d[1]
+; CHECK-GI-NEXT:    mov v1.s[1], v29.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v9.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v10.s[0]
+; CHECK-GI-NEXT:    scvtf s24, x9
+; CHECK-GI-NEXT:    mov x9, v27.d[1]
+; CHECK-GI-NEXT:    scvtf s27, x13
+; CHECK-GI-NEXT:    scvtf s26, x12
+; CHECK-GI-NEXT:    mov v4.s[1], v8.s[0]
+; CHECK-GI-NEXT:    mov v5.s[1], v12.s[0]
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v6.s[1], v11.s[0]
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v7.s[1], v30.s[0]
+; CHECK-GI-NEXT:    scvtf s28, x8
+; CHECK-GI-NEXT:    scvtf s29, x9
+; CHECK-GI-NEXT:    scvtf s30, x10
+; CHECK-GI-NEXT:    scvtf s31, x11
+; CHECK-GI-NEXT:    mov v0.s[2], v17.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v22.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v23.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v24.s[0]
+; CHECK-GI-NEXT:    mov v5.s[2], v27.s[0]
+; CHECK-GI-NEXT:    mov v6.s[2], v25.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], v26.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v18.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v21.s[0]
+; CHECK-GI-NEXT:    mov v4.s[3], v28.s[0]
+; CHECK-GI-NEXT:    mov v5.s[3], v29.s[0]
+; CHECK-GI-NEXT:    mov v6.s[3], v30.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], v31.s[0]
+; CHECK-GI-NEXT:    ldr d12, [sp], #48 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i64> %a to <32 x float>
@@ -4958,42 +5108,110 @@ define <32 x float> @utofp_v32i64_v32f32(<32 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v32i64_v32f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    ucvtf v24.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    ucvtf v25.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v26.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v27.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v18.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v20.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v22.2d
-; CHECK-GI-NEXT:    ucvtf v16.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v19.2d
-; CHECK-GI-NEXT:    ucvtf v18.2d, v21.2d
-; CHECK-GI-NEXT:    ucvtf v19.2d, v23.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v1.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v3.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v5.2s, v5.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn v7.2s, v7.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v24.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v25.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v26.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v27.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v5.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v18.2d
-; CHECK-GI-NEXT:    fcvtn2 v7.4s, v19.2d
+; CHECK-GI-NEXT:    str d12, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset b8, -8
+; CHECK-GI-NEXT:    .cfi_offset b9, -16
+; CHECK-GI-NEXT:    .cfi_offset b10, -24
+; CHECK-GI-NEXT:    .cfi_offset b11, -32
+; CHECK-GI-NEXT:    .cfi_offset b12, -48
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x11, v3.d[1]
+; CHECK-GI-NEXT:    ldp q30, q25, [sp, #112]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    ldp q21, q24, [sp, #48]
+; CHECK-GI-NEXT:    ldp q11, q27, [sp, #80]
+; CHECK-GI-NEXT:    ucvtf s28, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    ucvtf s16, x10
+; CHECK-GI-NEXT:    mov x10, v6.d[1]
+; CHECK-GI-NEXT:    ucvtf s9, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    ucvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    ucvtf s18, x11
+; CHECK-GI-NEXT:    ucvtf s17, x8
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    ldp q31, q26, [sp, #144]
+; CHECK-GI-NEXT:    mov x13, v21.d[1]
+; CHECK-GI-NEXT:    ucvtf s10, x10
+; CHECK-GI-NEXT:    fmov x10, d21
+; CHECK-GI-NEXT:    fmov x14, d30
+; CHECK-GI-NEXT:    ucvtf s1, x8
+; CHECK-GI-NEXT:    fmov x8, d3
+; CHECK-GI-NEXT:    ucvtf s3, x12
+; CHECK-GI-NEXT:    mov x12, v30.d[1]
+; CHECK-GI-NEXT:    ucvtf s22, x11
+; CHECK-GI-NEXT:    fmov x11, d11
+; CHECK-GI-NEXT:    ucvtf s29, x9
+; CHECK-GI-NEXT:    fmov x9, d4
+; CHECK-GI-NEXT:    ucvtf s4, x10
+; CHECK-GI-NEXT:    ucvtf s19, x8
+; CHECK-GI-NEXT:    mov x8, v5.d[1]
+; CHECK-GI-NEXT:    mov x10, v11.d[1]
+; CHECK-GI-NEXT:    ucvtf s5, x11
+; CHECK-GI-NEXT:    mov x11, v31.d[1]
+; CHECK-GI-NEXT:    ucvtf s8, x13
+; CHECK-GI-NEXT:    ucvtf s11, x12
+; CHECK-GI-NEXT:    fmov x12, d31
+; CHECK-GI-NEXT:    ucvtf s2, x9
+; CHECK-GI-NEXT:    mov x9, v7.d[1]
+; CHECK-GI-NEXT:    ucvtf s6, x14
+; CHECK-GI-NEXT:    fmov x13, d27
+; CHECK-GI-NEXT:    ucvtf s20, x8
+; CHECK-GI-NEXT:    fmov x8, d7
+; CHECK-GI-NEXT:    ucvtf s12, x10
+; CHECK-GI-NEXT:    ucvtf s7, x12
+; CHECK-GI-NEXT:    fmov x12, d25
+; CHECK-GI-NEXT:    ucvtf s30, x11
+; CHECK-GI-NEXT:    mov x10, v25.d[1]
+; CHECK-GI-NEXT:    mov x11, v26.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v28.s[0]
+; CHECK-GI-NEXT:    ucvtf s21, x9
+; CHECK-GI-NEXT:    fmov x9, d24
+; CHECK-GI-NEXT:    ucvtf s23, x8
+; CHECK-GI-NEXT:    ucvtf s25, x12
+; CHECK-GI-NEXT:    fmov x12, d26
+; CHECK-GI-NEXT:    mov x8, v24.d[1]
+; CHECK-GI-NEXT:    mov v1.s[1], v29.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v9.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v10.s[0]
+; CHECK-GI-NEXT:    ucvtf s24, x9
+; CHECK-GI-NEXT:    mov x9, v27.d[1]
+; CHECK-GI-NEXT:    ucvtf s27, x13
+; CHECK-GI-NEXT:    ucvtf s26, x12
+; CHECK-GI-NEXT:    mov v4.s[1], v8.s[0]
+; CHECK-GI-NEXT:    mov v5.s[1], v12.s[0]
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v6.s[1], v11.s[0]
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v7.s[1], v30.s[0]
+; CHECK-GI-NEXT:    ucvtf s28, x8
+; CHECK-GI-NEXT:    ucvtf s29, x9
+; CHECK-GI-NEXT:    ucvtf s30, x10
+; CHECK-GI-NEXT:    ucvtf s31, x11
+; CHECK-GI-NEXT:    mov v0.s[2], v17.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v22.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v23.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v24.s[0]
+; CHECK-GI-NEXT:    mov v5.s[2], v27.s[0]
+; CHECK-GI-NEXT:    mov v6.s[2], v25.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], v26.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v18.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v21.s[0]
+; CHECK-GI-NEXT:    mov v4.s[3], v28.s[0]
+; CHECK-GI-NEXT:    mov v5.s[3], v29.s[0]
+; CHECK-GI-NEXT:    mov v6.s[3], v30.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], v31.s[0]
+; CHECK-GI-NEXT:    ldr d12, [sp], #48 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i64> %a to <32 x float>
@@ -6389,24 +6607,14 @@ define <2 x half> @stofp_v2i64_v2f16(<2 x i64> %a) {
 ; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v2i64_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v2i64_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v2i64_v2f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %c
@@ -6435,198 +6643,78 @@ define <2 x half> @utofp_v2i64_v2f16(<2 x i64> %a) {
 ; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v2i64_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v2i64_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v2i64_v2f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <3 x half> @stofp_v3i64_v3f16(<3 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v3i64_v3f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v3i64_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v3i64_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-FP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: stofp_v3i64_v3f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = sitofp <3 x i64> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <3 x half> @utofp_v3i64_v3f16(<3 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v3i64_v3f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v3i64_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v3i64_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-FP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: utofp_v3i64_v3f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = uitofp <3 x i64> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <4 x half> @stofp_v4i64_v4f16(<4 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v4i64_v4f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v4i64_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v4i64_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: stofp_v4i64_v4f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = sitofp <4 x i64> %a to <4 x half>
   ret <4 x half> %c
 }
 
 define <4 x half> @utofp_v4i64_v4f16(<4 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v4i64_v4f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v4i64_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v4i64_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: utofp_v4i64_v4f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = uitofp <4 x i64> %a to <4 x half>
   ret <4 x half> %c
@@ -6647,46 +6735,19 @@ define <8 x half> @stofp_v8i64_v8f16(<8 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v8i64_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v8i64_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d4, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d5, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d5
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov d1, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v1.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v1.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v8i64_v8f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %c
@@ -6707,46 +6768,19 @@ define <8 x half> @utofp_v8i64_v8f16(<8 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v8i64_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v8i64_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d4, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d5, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d5
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov d1, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v1.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v1.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v8i64_v8f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %c
@@ -6777,79 +6811,29 @@ define <16 x half> @stofp_v16i64_v16f16(<16 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v16i64_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v16i64_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    scvtf v18.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d16, v0.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    fcvt h4, d17
-; CHECK-GI-FP16-NEXT:    mov d17, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v4.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvt h6, d17
-; CHECK-GI-FP16-NEXT:    fcvt h16, d19
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov d5, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v16.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v6.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h7, d17
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov d4, v6.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v7.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v4.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v16i64_v16f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i64> %a to <16 x half>
   ret <16 x half> %c
@@ -6880,79 +6864,29 @@ define <16 x half> @utofp_v16i64_v16f16(<16 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v16i64_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v16i64_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v18.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d16, v0.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    fcvt h4, d17
-; CHECK-GI-FP16-NEXT:    mov d17, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v4.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvt h6, d17
-; CHECK-GI-FP16-NEXT:    fcvt h16, d19
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov d5, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v16.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v6.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h7, d17
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov d4, v6.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v7.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v4.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v16i64_v16f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i64> %a to <16 x half>
   ret <16 x half> %c
@@ -7007,155 +6941,53 @@ define <32 x half> @stofp_v32i64_v32f16(<32 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v3.8h, v7.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v32i64_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NOFP16-NEXT:    scvtf v20.2d, v20.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v18.2d, v18.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v21.2d, v21.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v22.2d, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v16.2s, v16.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v20.2s, v20.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v19.2d, v19.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v23.2d, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v22.2s, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v16.4s, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v20.4s, v21.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v24.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v18.4s, v19.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v22.4s, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v20.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v22.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v32i64_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ldp q16, q18, [sp]
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ldp q17, q19, [sp, #64]
-; CHECK-GI-FP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    scvtf v20.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    scvtf v24.2d, v18.2d
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v16.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    mov d21, v0.d[1]
-; CHECK-GI-FP16-NEXT:    scvtf v25.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    mov d22, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d18, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d1
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v20.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    fcvt h4, d20
-; CHECK-GI-FP16-NEXT:    mov d26, v16.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d5
-; CHECK-GI-FP16-NEXT:    fcvt h5, d16
-; CHECK-GI-FP16-NEXT:    fcvt h28, d21
-; CHECK-GI-FP16-NEXT:    fcvt h29, d22
-; CHECK-GI-FP16-NEXT:    fcvt h22, d24
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h19, d19
-; CHECK-GI-FP16-NEXT:    fcvt h27, d23
-; CHECK-GI-FP16-NEXT:    mov d23, v24.d[1]
-; CHECK-GI-FP16-NEXT:    mov d24, v25.d[1]
-; CHECK-GI-FP16-NEXT:    ldp q25, q16, [sp, #32]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v28.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v29.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[1], v27.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v25.2d, v25.2d
-; CHECK-GI-FP16-NEXT:    fcvt h23, d23
-; CHECK-GI-FP16-NEXT:    mov v5.h[1], v26.h[0]
-; CHECK-GI-FP16-NEXT:    ldp q26, q27, [sp, #96]
-; CHECK-GI-FP16-NEXT:    fcvt h24, d24
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v20.h[0]
-; CHECK-GI-FP16-NEXT:    mov d20, v6.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    scvtf v26.2d, v26.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[2], v22.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h22, d25
-; CHECK-GI-FP16-NEXT:    mov v5.h[2], v21.h[0]
-; CHECK-GI-FP16-NEXT:    mov d21, v25.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v19.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    scvtf v18.2d, v27.2d
-; CHECK-GI-FP16-NEXT:    fcvt h19, d20
-; CHECK-GI-FP16-NEXT:    fcvt h17, d17
-; CHECK-GI-FP16-NEXT:    mov d25, v26.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v4.h[3], v23.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[3], v24.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d21
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    mov v4.h[4], v22.h[0]
-; CHECK-GI-FP16-NEXT:    mov d22, v16.d[1]
-; CHECK-GI-FP16-NEXT:    mov v5.h[4], v26.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    mov d6, v7.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h7, d7
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v19.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h19, d23
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v4.h[5], v20.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d22
-; CHECK-GI-FP16-NEXT:    mov v5.h[5], v21.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v7.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[6], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[6], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[7], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[7], v19.h[0]
-; CHECK-GI-FP16-NEXT:    mov v2.16b, v4.16b
-; CHECK-GI-FP16-NEXT:    mov v3.16b, v5.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v32i64_v32f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    scvtf v16.2d, v16.2d
+; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-GI-NEXT:    scvtf v20.2d, v20.2d
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-GI-NEXT:    scvtf v18.2d, v18.2d
+; CHECK-GI-NEXT:    scvtf v21.2d, v21.2d
+; CHECK-GI-NEXT:    scvtf v22.2d, v22.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v16.2s, v16.2d
+; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v20.2s, v20.2d
+; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    scvtf v19.2d, v19.2d
+; CHECK-GI-NEXT:    scvtf v23.2d, v23.2d
+; CHECK-GI-NEXT:    fcvtn v24.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn v18.2s, v18.2d
+; CHECK-GI-NEXT:    fcvtn v22.2s, v22.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v16.4s, v17.2d
+; CHECK-GI-NEXT:    fcvtn2 v20.4s, v21.2d
+; CHECK-GI-NEXT:    fcvtn2 v24.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn2 v18.4s, v19.2d
+; CHECK-GI-NEXT:    fcvtn2 v22.4s, v23.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn v2.4h, v16.4s
+; CHECK-GI-NEXT:    fcvtn v3.4h, v20.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v24.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    fcvtn2 v2.8h, v18.4s
+; CHECK-GI-NEXT:    fcvtn2 v3.8h, v22.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i64> %a to <32 x half>
   ret <32 x half> %c
@@ -7210,155 +7042,53 @@ define <32 x half> @utofp_v32i64_v32f16(<32 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v3.8h, v7.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v32i64_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v20.2d, v20.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v18.2d, v18.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v21.2d, v21.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v22.2d, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v16.2s, v16.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v20.2s, v20.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v19.2d, v19.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v23.2d, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v22.2s, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v16.4s, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v20.4s, v21.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v24.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v18.4s, v19.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v22.4s, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v20.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v22.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v32i64_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ldp q16, q18, [sp]
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ldp q17, q19, [sp, #64]
-; CHECK-GI-FP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v20.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v24.2d, v18.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v16.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    mov d21, v0.d[1]
-; CHECK-GI-FP16-NEXT:    ucvtf v25.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    mov d22, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d18, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d1
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v20.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    fcvt h4, d20
-; CHECK-GI-FP16-NEXT:    mov d26, v16.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d5
-; CHECK-GI-FP16-NEXT:    fcvt h5, d16
-; CHECK-GI-FP16-NEXT:    fcvt h28, d21
-; CHECK-GI-FP16-NEXT:    fcvt h29, d22
-; CHECK-GI-FP16-NEXT:    fcvt h22, d24
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h19, d19
-; CHECK-GI-FP16-NEXT:    fcvt h27, d23
-; CHECK-GI-FP16-NEXT:    mov d23, v24.d[1]
-; CHECK-GI-FP16-NEXT:    mov d24, v25.d[1]
-; CHECK-GI-FP16-NEXT:    ldp q25, q16, [sp, #32]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v28.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v29.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[1], v27.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v25.2d, v25.2d
-; CHECK-GI-FP16-NEXT:    fcvt h23, d23
-; CHECK-GI-FP16-NEXT:    mov v5.h[1], v26.h[0]
-; CHECK-GI-FP16-NEXT:    ldp q26, q27, [sp, #96]
-; CHECK-GI-FP16-NEXT:    fcvt h24, d24
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v20.h[0]
-; CHECK-GI-FP16-NEXT:    mov d20, v6.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    ucvtf v26.2d, v26.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[2], v22.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h22, d25
-; CHECK-GI-FP16-NEXT:    mov v5.h[2], v21.h[0]
-; CHECK-GI-FP16-NEXT:    mov d21, v25.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v19.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v18.2d, v27.2d
-; CHECK-GI-FP16-NEXT:    fcvt h19, d20
-; CHECK-GI-FP16-NEXT:    fcvt h17, d17
-; CHECK-GI-FP16-NEXT:    mov d25, v26.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v4.h[3], v23.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[3], v24.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d21
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    mov v4.h[4], v22.h[0]
-; CHECK-GI-FP16-NEXT:    mov d22, v16.d[1]
-; CHECK-GI-FP16-NEXT:    mov v5.h[4], v26.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    mov d6, v7.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h7, d7
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v19.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h19, d23
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v4.h[5], v20.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d22
-; CHECK-GI-FP16-NEXT:    mov v5.h[5], v21.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v7.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[6], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[6], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[7], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[7], v19.h[0]
-; CHECK-GI-FP16-NEXT:    mov v2.16b, v4.16b
-; CHECK-GI-FP16-NEXT:    mov v3.16b, v5.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v32i64_v32f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    ucvtf v16.2d, v16.2d
+; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-GI-NEXT:    ucvtf v20.2d, v20.2d
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    ucvtf v17.2d, v17.2d
+; CHECK-GI-NEXT:    ucvtf v18.2d, v18.2d
+; CHECK-GI-NEXT:    ucvtf v21.2d, v21.2d
+; CHECK-GI-NEXT:    ucvtf v22.2d, v22.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v16.2s, v16.2d
+; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v20.2s, v20.2d
+; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    ucvtf v19.2d, v19.2d
+; CHECK-GI-NEXT:    ucvtf v23.2d, v23.2d
+; CHECK-GI-NEXT:    fcvtn v24.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn v18.2s, v18.2d
+; CHECK-GI-NEXT:    fcvtn v22.2s, v22.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v16.4s, v17.2d
+; CHECK-GI-NEXT:    fcvtn2 v20.4s, v21.2d
+; CHECK-GI-NEXT:    fcvtn2 v24.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn2 v18.4s, v19.2d
+; CHECK-GI-NEXT:    fcvtn2 v22.4s, v23.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn v2.4h, v16.4s
+; CHECK-GI-NEXT:    fcvtn v3.4h, v20.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v24.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    fcvtn2 v2.8h, v18.4s
+; CHECK-GI-NEXT:    fcvtn2 v3.8h, v22.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i64> %a to <32 x half>
   ret <32 x half> %c
diff --git a/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll b/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll
new file mode 100644
index 0000000000000..1f8eeccf9c338
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=arm64ec-windows-msvc < %s | FileCheck -check-prefixes=ARM64EC %s
+
+; Separate from ldexp.ll test because this errors on half cases
+
+; ARM64EC-LABEL: ldexp_f32 =
+; ARM64EC: fcvt d0, s0
+; ARM64EC: bl "#ldexp"
+; ARM64EC: fcvt s0, d0
+define float @ldexp_f32(float %val, i32 %a) {
+  %call = call float @llvm.ldexp.f32(float %val, i32 %a)
+  ret float %call
+}
+
+; ARM64EC-LABEL: ldexp_f64 =
+; ARM64EC: b "#ldexp"
+define double @ldexp_f64(double %val, i32 %a) {
+  %call = call double @llvm.ldexp.f64(double %val, i32 %a)
+  ret double %call
+}
diff --git a/llvm/test/CodeGen/AArch64/llvm.frexp.ll b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
index e4cb8ed6eaf90..51400b96e8b77 100644
--- a/llvm/test/CodeGen/AArch64/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
@@ -1,5 +1,112 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -mtriple=aarch64-windows-pc-msvc < %s | FileCheck -check-prefixes=WINDOWS %s
+
+define { half, i32 } @test_frexp_f16_i32(half %a) {
+; CHECK-LABEL: test_frexp_f16_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f16_i32:
+; WINDOWS:       .seh_proc test_frexp_f16_i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  ret { half, i32 } %result
+}
+
+define half @test_frexp_f16_i32_only_use_fract(half %a) {
+; CHECK-LABEL: test_frexp_f16_i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f16_i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_f16_i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  %result.0 = extractvalue { half, i32 } %result, 0
+  ret half %result.0
+}
+
+define i32 @test_frexp_f16_i32_only_use_exp(half %a) {
+; CHECK-LABEL: test_frexp_f16_i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f16_i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_f16_i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  %result.0 = extractvalue { half, i32 } %result, 1
+  ret i32 %result.0
+}
 
 define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; CHECK-LABEL: test_frexp_v2f16_v2i32:
@@ -51,12 +158,242 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f16_v2i32:
+; WINDOWS:       .seh_proc test_frexp_v2f16_v2i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x19, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov h1, v0.h[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #36
+; WINDOWS-NEXT:    add x19, sp, #36
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    add x0, sp, #32
+; WINDOWS-NEXT:    fcvt d1, h1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    mov h1, v1.h[2]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v2.h[1], v1.h[0]
+; WINDOWS-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    mov h1, v1.h[3]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v1.h[2], v2.h[0]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    ldr s1, [sp, #32]
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; WINDOWS-NEXT:    mov v0.h[3], v2.h[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    ldr x19, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
   %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
   ret { <2 x half>, <2 x i32> } %result
 }
 
-define { <3 x float>, <3 x i32> } @test_frexp_v3f16_v3i32(<3 x float> %a) {
-; CHECK-LABEL: test_frexp_v3f16_v3i32:
+define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
+; CHECK-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    mov h1, v1.h[2]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    mov h1, v1.h[3]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    fcvt h1, s0
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.h[3], v1.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v2f16_v2i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov h1, v0.h[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    fcvt d1, h1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #56
+; WINDOWS-NEXT:    mov h1, v1.h[2]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v2.h[1], v1.h[0]
+; WINDOWS-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #60
+; WINDOWS-NEXT:    mov h1, v1.h[3]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v1.h[2], v2.h[0]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt h1, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v0.h[3], v1.h[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
+  %result.0 = extractvalue { <2 x half>, <2 x i32> } %result, 0
+  ret <2 x half> %result.0
+}
+
+define <2 x i32> @test_frexp_v2f16_v2i32_only_use_exp(<2 x half> %a) {
+; CHECK-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x19, sp, #28
+; CHECK-NEXT:    mov h0, v0.h[1]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s0, [sp, #24]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v2f16_v2i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x19, sp, #28
+; WINDOWS-NEXT:    mov h0, v0.h[1]
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #24]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
+  %result.1 = extractvalue { <2 x half>, <2 x i32> } %result, 1
+  ret <2 x i32> %result.1
+}
+
+define { <3 x float>, <3 x i32> } @test_frexp_v3f32_v3i32(<3 x float> %a) {
+; CHECK-LABEL: test_frexp_v3f32_v3i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #80
 ; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
@@ -96,6 +433,1015 @@ define { <3 x float>, <3 x i32> } @test_frexp_v3f16_v3i32(<3 x float> %a) {
 ; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
-  %result = call { <3 x float>, <3 x i32> } @llvm.frexp.v3float.v3i32(<3 x float> %a)
+;
+; WINDOWS-LABEL: test_frexp_v3f32_v3i32:
+; WINDOWS:       .seh_proc test_frexp_v3f32_v3i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    stp x19, x20, [sp, #48] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    stp x21, x30, [sp, #64] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #36
+; WINDOWS-NEXT:    add x19, sp, #36
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #32
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    add x20, sp, #40
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    add x21, sp, #44
+; WINDOWS-NEXT:    mov v1.s[2], v0.s[0]
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s1, [sp, #32]
+; WINDOWS-NEXT:    fcvt s2, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    mov v0.s[3], v2.s[0]
+; WINDOWS-NEXT:    ld1 { v1.s }[2], [x20]
+; WINDOWS-NEXT:    ld1 { v1.s }[3], [x21]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldp x21, x30, [sp, #64] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    ldp x19, x20, [sp, #48] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    add sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <3 x float>, <3 x i32> } @llvm.frexp.v3f32.v3i32(<3 x float> %a)
   ret { <3 x float>, <3 x i32> } %result
 }
+
+define { float, i32 } @test_frexp_f32_i32(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32:
+; WINDOWS:       .seh_proc test_frexp_f32_i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+define { float, i32 } @test_frexp_f32_i32_tailcall(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32_tailcall:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32_tailcall:
+; WINDOWS:       .seh_proc test_frexp_f32_i32_tailcall
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = tail call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+define float @test_frexp_f32_i32_only_use_fract(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_f32_i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  %result.0 = extractvalue { float, i32 } %result, 0
+  ret float %result.0
+}
+
+define i32 @test_frexp_f32_i32_only_use_exp(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_f32_i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  %result.0 = extractvalue { float, i32 } %result, 1
+  ret i32 %result.0
+}
+
+define { <2 x float>, <2 x i32> } @test_frexp_v2f32_v2i32(<2 x float> %a) {
+; CHECK-LABEL: test_frexp_v2f32_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f32_v2i32:
+; WINDOWS:       .seh_proc test_frexp_v2f32_v2i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x19, sp, #28
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d1, s1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr s1, [sp, #24]
+; WINDOWS-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; WINDOWS-NEXT:    mov v0.s[1], v2.s[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a)
+  ret { <2 x float>, <2 x i32> } %result
+}
+
+define <2 x float> @test_frexp_v2f32_v2i32_only_use_fract(<2 x float> %a) {
+; CHECK-LABEL: test_frexp_v2f32_v2i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f32_v2i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v2f32_v2i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #32
+; WINDOWS-NEXT:    .seh_stackalloc 32
+; WINDOWS-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d1, s1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 16
+; WINDOWS-NEXT:    add sp, sp, #32
+; WINDOWS-NEXT:    .seh_stackalloc 32
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a)
+  %result.0 = extractvalue { <2 x float>, <2 x i32> } %result, 0
+  ret <2 x float> %result.0
+}
+
+define <2 x i32> @test_frexp_v2f32_v2i32_only_use_exp(<2 x float> %a) {
+; CHECK-LABEL: test_frexp_v2f32_v2i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x19, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s0, [sp, #28]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f32_v2i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v2f32_v2i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x19, sp, #28
+; WINDOWS-NEXT:    mov s0, v0.s[1]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #24]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a)
+  %result.1 = extractvalue { <2 x float>, <2 x i32> } %result, 1
+  ret <2 x i32> %result.1
+}
+
+define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) {
+; CHECK-LABEL: test_frexp_v4f32_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    stp x30, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #36
+; CHECK-NEXT:    add x20, sp, #36
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #32
+; CHECK-NEXT:    add x21, sp, #32
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    mov v2.s[3], v0.s[0]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ld1 { v1.s }[3], [x21]
+; CHECK-NEXT:    ldp x30, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v4f32_v4i32:
+; WINDOWS:       .seh_proc test_frexp_v4f32_v4i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    stp x19, x20, [sp, #48] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    stp x21, x30, [sp, #64] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #36
+; WINDOWS-NEXT:    add x19, sp, #36
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #32
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    add x20, sp, #40
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    add x21, sp, #44
+; WINDOWS-NEXT:    mov v1.s[2], v0.s[0]
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s1, [sp, #32]
+; WINDOWS-NEXT:    fcvt s2, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    mov v0.s[3], v2.s[0]
+; WINDOWS-NEXT:    ld1 { v1.s }[2], [x20]
+; WINDOWS-NEXT:    ld1 { v1.s }[3], [x21]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldp x21, x30, [sp, #64] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    ldp x19, x20, [sp, #48] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    add sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
+  ret { <4 x float>, <4 x i32> } %result
+}
+
+define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) {
+; CHECK-LABEL: test_frexp_v4f32_v4i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v4f32_v4i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v4f32_v4i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #56
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #60
+; WINDOWS-NEXT:    mov v1.s[2], v0.s[0]
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s1, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v0.s[3], v1.s[0]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
+  %result.0 = extractvalue { <4 x float>, <4 x i32> } %result, 0
+  ret <4 x float> %result.0
+}
+
+define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) {
+; CHECK-LABEL: test_frexp_v4f32_v4i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x21, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x19, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #20
+; CHECK-NEXT:    add x20, sp, #20
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #16
+; CHECK-NEXT:    add x21, sp, #16
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s0, [sp, #28]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ld1 { v0.s }[2], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[3], [x21]
+; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v4f32_v4i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v4f32_v4i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    stp x19, x20, [sp, #32] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_regp x19, 32
+; WINDOWS-NEXT:    stp x21, x30, [sp, #48] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 48
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #16
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #20
+; WINDOWS-NEXT:    add x19, sp, #20
+; WINDOWS-NEXT:    mov s0, v0.s[1]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    add x20, sp, #24
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x21, sp, #28
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #16]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    ld1 { v0.s }[2], [x20]
+; WINDOWS-NEXT:    ld1 { v0.s }[3], [x21]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldp x21, x30, [sp, #48] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 48
+; WINDOWS-NEXT:    ldp x19, x20, [sp, #32] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_regp x19, 32
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
+  %result.1 = extractvalue { <4 x float>, <4 x i32> } %result, 1
+  ret <4 x i32> %result.1
+}
+
+define { double, i32 } @test_frexp_f64_i32(double %a) {
+; CHECK-LABEL: test_frexp_f64_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f64_i32:
+; WINDOWS:       .seh_proc test_frexp_f64_i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  ret { double, i32 } %result
+}
+
+define double @test_frexp_f64_i32_only_use_fract(double %a) {
+; CHECK-LABEL: test_frexp_f64_i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f64_i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_f64_i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  %result.0 = extractvalue { double, i32 } %result, 0
+  ret double %result.0
+}
+
+define i32 @test_frexp_f64_i32_only_use_exp(double %a) {
+; CHECK-LABEL: test_frexp_f64_i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f64_i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_f64_i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  %result.0 = extractvalue { double, i32 } %result, 1
+  ret i32 %result.0
+}
+
+define { <2 x double>, <2 x i32> } @test_frexp_v2f64_v2i32(<2 x double> %a) {
+; CHECK-LABEL: test_frexp_v2f64_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f64_v2i32:
+; WINDOWS:       .seh_proc test_frexp_v2f64_v2i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x19, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    mov d0, v0.d[1]
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    add x19, sp, #40
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s1, [sp, #44]
+; WINDOWS-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    mov v0.d[1], v2.d[0]
+; WINDOWS-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    ldr x19, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a)
+  ret { <2 x double>, <2 x i32> } %result
+}
+
+define <2 x double> @test_frexp_v2f64_v2i32_only_use_fract(<2 x double> %a) {
+; CHECK-LABEL: test_frexp_v2f64_v2i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f64_v2i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v2f64_v2i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 32
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    mov d0, v0.d[1]
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov v0.d[1], v1.d[0]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a)
+  %result.0 = extractvalue { <2 x double>, <2 x i32> } %result, 0
+  ret <2 x double> %result.0
+}
+
+define <2 x i32> @test_frexp_v2f64_v2i32_only_use_exp(<2 x double> %a) {
+; CHECK-LABEL: test_frexp_v2f64_v2i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x19, sp, #24
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr s0, [sp, #28]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f64_v2i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v2f64_v2i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    add x19, sp, #24
+; WINDOWS-NEXT:    mov d0, v0.d[1]
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #28]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a)
+  %result.1 = extractvalue { <2 x double>, <2 x i32> } %result, 1
+  ret <2 x i32> %result.1
+}
+
+declare { float, i32 } @llvm.frexp.f32.i32(float) #0
+declare { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float>) #0
+declare { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float>) #0
+
+declare { half, i32 } @llvm.frexp.f16.i32(half) #0
+declare { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half>) #0
+
+declare { double, i32 } @llvm.frexp.f64.i32(double) #0
+declare { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double>) #0
+
+declare { half, i16 } @llvm.frexp.f16.i16(half) #0
+declare { <2 x half>, <2 x i16> } @llvm.frexp.v2f16.v2i16(<2 x half>) #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll b/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll
index f394bc467af69..29427146e8a43 100644
--- a/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll
+++ b/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll
@@ -53,5 +53,5 @@ bb2:                                              ; preds = %bb1, %bb
   ret i32 %phi
 }
 
-attributes #1 = { minsize nounwind "target-features"="+aes,+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+reserve-lr-for-ra,+sha2,+v8.1a,+v8.2a,+v8.3a,+v8a,+zcm,+zcz" }
+attributes #1 = { minsize nounwind "target-features"="+aes,+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+reserve-lr-for-ra,+sha2,+v8.1a,+v8.2a,+v8.3a,+v8a" }
 
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner-maddimm.mir b/llvm/test/CodeGen/AArch64/machine-combiner-maddimm.mir
new file mode 100644
index 0000000000000..dc75c8c61c53c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-combiner-maddimm.mir
@@ -0,0 +1,147 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass machine-combiner -verify-machineinstrs  -o - %s | FileCheck %s
+
+---
+name:            madd_addwi
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: madd_addwi
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[MOVZWi:%[0-9]+]]:gpr32common = nsw MOVZWi 79, 0
+    ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32common = nsw MADDWrrr [[COPY1]], [[COPY]], [[MOVZWi]]
+    ; CHECK-NEXT: $w0 = COPY [[MADDWrrr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr32 = COPY $w0
+    %1:gpr32 = COPY $w1
+    %2:gpr32common = nsw MADDWrrr %1:gpr32, %0:gpr32, $wzr
+    %3:gpr32sp = nsw ADDWri killed %2:gpr32common, 79, 0
+    $w0 = COPY %3:gpr32sp
+    RET_ReallyLR implicit $w0
+...
+---
+name:            madd_addxi
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: madd_addxi
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[MOVZXi:%[0-9]+]]:gpr64common = nsw MOVZXi 79, 0
+    ; CHECK-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64common = nsw MADDXrrr [[COPY1]], [[COPY]], [[MOVZXi]]
+    ; CHECK-NEXT: $x0 = COPY [[MADDXrrr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = COPY $x1
+    %2:gpr64common = nsw MADDXrrr %1:gpr64, %0:gpr64, $xzr
+    %3:gpr64sp = nsw ADDXri killed %2:gpr64common, 79, 0
+    $x0 = COPY %3:gpr64sp
+    RET_ReallyLR implicit $x0
+...
+---
+name:            madd_subwi
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: madd_subwi
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[MOVNWi:%[0-9]+]]:gpr32common = nsw MOVNWi 0, 0
+    ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY1]], [[COPY]], [[MOVNWi]]
+    ; CHECK-NEXT: $w0 = COPY [[MADDWrrr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr32 = COPY $w0
+    %1:gpr32 = COPY $w1
+    %2:gpr32common = nsw MADDWrrr %1:gpr32, %0:gpr32, $wzr
+    %3:gpr32 = nsw SUBSWri killed %2:gpr32common, 1, 0, implicit-def dead $nzcv
+    $w0 = COPY %3:gpr32
+    RET_ReallyLR implicit $w0
+...
+---
+name:            madd_subxi
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: madd_subxi
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[MOVNXi:%[0-9]+]]:gpr64common = nsw MOVNXi 0, 0
+    ; CHECK-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64 = nsw MADDXrrr [[COPY1]], [[COPY]], [[MOVNXi]]
+    ; CHECK-NEXT: $x0 = COPY [[MADDXrrr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = COPY $x1
+    %2:gpr64common = nsw MADDXrrr %1:gpr64, %0:gpr64, $xzr
+    %3:gpr64 = nsw SUBSXri killed %2:gpr64common, 1, 0, implicit-def dead $nzcv
+    $x0 = COPY %3:gpr64
+    RET_ReallyLR implicit $x0
+...
+---
+name:            madd_addorwi
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: madd_addorwi
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[ORRWri:%[0-9]+]]:gpr32common = nsw ORRWri $wzr, 1291
+    ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32common = nsw MADDWrrr [[COPY1]], [[COPY]], [[ORRWri]]
+    ; CHECK-NEXT: $w0 = COPY [[MADDWrrr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr32 = COPY $w0
+    %1:gpr32 = COPY $w1
+    %2:gpr32common = nsw MADDWrrr %1:gpr32, %0:gpr32, $wzr
+    %3:gpr32sp = nsw ADDWri killed %2:gpr32common, 4095, 12
+    $w0 = COPY %3:gpr32sp
+    RET_ReallyLR implicit $w0
+...
+---
+name:            madd_addorxi
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: madd_addorxi
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[ORRXri:%[0-9]+]]:gpr64common = nsw ORRXri $xzr, 7435
+    ; CHECK-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64common = nsw MADDXrrr [[COPY1]], [[COPY]], [[ORRXri]]
+    ; CHECK-NEXT: $x0 = COPY [[MADDXrrr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = COPY $x1
+    %2:gpr64common = nsw MADDXrrr %1:gpr64, %0:gpr64, $xzr
+    %3:gpr64sp = nsw ADDXri killed %2:gpr64common, 4095, 12
+    $x0 = COPY %3:gpr64sp
+    RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner.ll b/llvm/test/CodeGen/AArch64/machine-combiner.ll
index c8df283aace0b..ec61fee1039ad 100644
--- a/llvm/test/CodeGen/AArch64/machine-combiner.ll
+++ b/llvm/test/CodeGen/AArch64/machine-combiner.ll
@@ -262,8 +262,8 @@ define half @reassociate_adds_half(half %x0, half %x1, half %x2, half %x3) {
 ; CHECK-UNSAFE-LABEL: reassociate_adds_half:
 ; CHECK-UNSAFE:       // %bb.0:
 ; CHECK-UNSAFE-NEXT:    fdiv h0, h0, h1
-; CHECK-UNSAFE-NEXT:    fadd h2, h3, h2
-; CHECK-UNSAFE-NEXT:    fadd h0, h2, h0
+; CHECK-UNSAFE-NEXT:    fadd h1, h3, h2
+; CHECK-UNSAFE-NEXT:    fadd h0, h1, h0
 ; CHECK-UNSAFE-NEXT:    ret
   %t0 = fdiv half %x0, %x1
   %t1 = fadd half %x2, %t0
@@ -284,8 +284,8 @@ define half @reassociate_muls_half(half %x0, half %x1, half %x2, half %x3) {
 ; CHECK-UNSAFE-LABEL: reassociate_muls_half:
 ; CHECK-UNSAFE:       // %bb.0:
 ; CHECK-UNSAFE-NEXT:    fdiv h0, h0, h1
-; CHECK-UNSAFE-NEXT:    fmul h2, h3, h2
-; CHECK-UNSAFE-NEXT:    fmul h0, h2, h0
+; CHECK-UNSAFE-NEXT:    fmul h1, h3, h2
+; CHECK-UNSAFE-NEXT:    fmul h0, h1, h0
 ; CHECK-UNSAFE-NEXT:    ret
   %t0 = fdiv half %x0, %x1
   %t1 = fmul half %x2, %t0
@@ -592,8 +592,10 @@ define <4 x i32> @reassociate_xors_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>
 ; CHECK-LABEL: reassociate_xors_v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    eor v1.16b, v2.16b, v3.16b
-; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT:    eor3 z0.d, z0.d, z2.d, z3.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %t0 = and <4 x i32> %x0, %x1
   %t1 = xor <4 x i32> %t0, %x2
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll b/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
new file mode 100644
index 0000000000000..196aebffce97f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
@@ -0,0 +1,74 @@
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-darwin < %s | FileCheck %s --implicit-check-not=.loh --check-prefixes=CHECK,LOH
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-darwin -enable-machine-outliner < %s | FileCheck %s --implicit-check-not=.loh --check-prefixes=CHECK,OUTLINE
+
+@A = global i32 0, align 4
+@B = global i32 0, align 4
+
+declare void @foo();
+declare void @bar(ptr %a);
+declare void @goo(ptr %a);
+
+; CHECK-LABEL: _a0:
+define void @a0(i32 %a) {
+
+  ; This becomes AdrpAdd when outlining is disabled, otherwise it is outlined
+  ; and there should be no LOH.
+  %addr = getelementptr inbounds i32, ptr @A, i32 0
+  %res = load i32, ptr %addr, align 4
+  ; LOH:      [[L0:Lloh.+]]:
+  ; LOH-NEXT:   adrp x19, _A@PAGE
+  ; LOH-NEXT: [[L1:Lloh.+]]:
+  ; LOH-NEXT:   add x19, x19, _A@PAGEOFF
+
+  call void @foo()
+  ; OUTLINE:      bl _OUTLINED_FUNCTION_0
+  ; OUTLINE-NEXT: mov x0, x19
+  ; OUTLINE-NEXT: bl _bar
+  call void @bar(ptr %addr)
+
+  ; This becomes AdrpAddStr.
+  %addr2 = getelementptr inbounds i32, ptr @B, i32 4
+  store i32 %res, ptr %addr2, align 4
+  ; CHECK:      [[L2:Lloh.+]]:
+  ; CHECK-NEXT:   adrp x8, _B@PAGE
+  ; CHECK-NEXT: [[L3:Lloh.+]]:
+  ; CHECK-NEXT:   add x8, x8, _B@PAGEOFF
+  ; CHECK-NEXT: [[L4:Lloh.+]]:
+  ; CHECK-NEXT:   str w20, [x8, #16]
+  ret void
+
+  ; LOH-DAG:   .loh AdrpAdd [[L0]], [[L1]]
+  ; CHECK-DAG: .loh AdrpAddStr [[L2]], [[L3]], [[L4]]
+  ; CHECK:     .cfi_endproc
+}
+
+; CHECK-LABEL: _a1:
+define i32 @a1(i32 %a) {
+
+  ; This becomes AdrpAdd when outlining is disabled, otherwise it is outlined
+  ; and there should be no LOH.
+  %addr = getelementptr inbounds i32, ptr @A, i32 0
+  %res = load i32, ptr %addr, align 4
+  ; LOH:      [[L5:Lloh.+]]:
+  ; LOH-NEXT:   adrp x19, _A@PAGE
+  ; LOH-NEXT: [[L6:Lloh.+]]:
+  ; LOH-NEXT:   add x19, x19, _A@PAGEOFF
+
+  call void @foo()
+  ; OUTLINE:      bl _OUTLINED_FUNCTION_0
+  ; OUTLINE-NEXT: mov x0, x19
+  ; OUTLINE-NEXT: bl _goo
+  call void @goo(ptr %addr)
+  ret i32 %res
+
+  ; LOH:   .loh AdrpAdd [[L5]], [[L6]]
+  ; CHECK: .cfi_endproc
+}
+
+; Note: it is not safe to add LOHs to this function as outlined functions do not
+; follow calling convention and thus x19 could be live across the call.
+; OUTLINE: _OUTLINED_FUNCTION_0:
+; OUTLINE:   adrp x19, _A@PAGE
+; OUTLINE:   add x19, x19, _A@PAGEOFF
+; OUTLINE:   ldr w20, [x19]
+; OUTLINE:   b _foo
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-loh.mir b/llvm/test/CodeGen/AArch64/machine-outliner-loh.mir
new file mode 100644
index 0000000000000..c613e316f828f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-loh.mir
@@ -0,0 +1,50 @@
+# RUN: llc %s -mtriple=aarch64-apple-ios -run-pass=aarch64-collect-loh -run-pass=machine-outliner -stats -o - 2>%t | FileCheck %s
+# RUN: FileCheck %s --input-file=%t --check-prefix=STATS
+# REQUIRES: asserts
+
+--- |
+  define void  @func0() noredzone minsize { ret void }
+
+  @g0 = external global i32
+...
+---
+# CHECK-LABEL: name: func0
+name: func0
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: bb.0:
+  bb.0:
+    ; CHECK: BL @OUTLINED_FUNCTION_0
+    $x10 = ADRP target-flags(aarch64-page) @g0
+    $x11 = ADDXri $x10, target-flags(aarch64-pageoff) @g0, 0
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+
+  ; CHECK-LABEL: bb.1:
+  bb.1:
+    ; CHECK: BL @OUTLINED_FUNCTION_0
+    ; MCLOH_AdrpAdd is not generated because $x11 is still live. If we want to
+    ; outline these instructions, we must remove the MCLOH_AdrpAdd from bb.0.
+    $x10 = ADRP target-flags(aarch64-page) @g0
+    $x11 = ADDXri $x10, target-flags(aarch64-pageoff) @g0, 0
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    ; CHECK: $x12 = ADDXri $x11, target-flags(aarch64-pageoff) @g0, 0
+    $x12 = ADDXri $x11, target-flags(aarch64-pageoff) @g0, 0
+
+...
+
+# CHECK-LABEL: name: OUTLINED_FUNCTION_0
+# CHECK:           $x10 = ADRP target-flags(aarch64-page) @g0
+# CHECK:           $x11 = ADDXri $x10, target-flags(aarch64-pageoff) @g0, 0
+# CHECK:           $w0 = ORRWri $wzr, 1
+# CHECK:           $w0 = ORRWri $wzr, 1
+# CHECK:           $w0 = ORRWri $wzr, 1
+# CHECK:           $w0 = ORRWri $wzr, 1
+
+# STATS: 1 aarch64-collect-loh - Number of simplifiable ADRP + ADD
+# STATS: 1 machine-outliner    - Total number of Linker Optimization Hints removed
diff --git a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll
index 2271190963d16..7dde168024278 100644
--- a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll
+++ b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll
@@ -134,3 +134,5 @@ declare ptr @mempcpy(ptr, ptr, i64)
 !2 = !{!1}
 !3 = distinct !{!3, !0, !"bax: %q"}
 !4 = !{!3}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MIR: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/memsize-remarks.ll b/llvm/test/CodeGen/AArch64/memsize-remarks.ll
index 93e3d6fb02607..f256389c7349e 100644
--- a/llvm/test/CodeGen/AArch64/memsize-remarks.ll
+++ b/llvm/test/CodeGen/AArch64/memsize-remarks.ll
@@ -315,8 +315,8 @@ bb:
   ret void
 }
 
-attributes #0 = { noinline nounwind ssp uwtable "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" }
-attributes #1 = { nounwind "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" }
+attributes #0 = { noinline nounwind ssp uwtable "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" }
+attributes #1 = { nounwind "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" }
 attributes #2 = { nofree nosync nounwind readnone speculatable willreturn }
 attributes #3 = { argmemonly nofree nosync nounwind willreturn }
 attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll
index a0eba7c747e95..6b2a8c5aaf8b8 100644
--- a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll
+++ b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc %s -mtriple=aarch64 -stop-after=finalize-isel -o - | FileCheck --check-prefix=MIR %s
 
 ; Ensure the scoped AA metadata is still retained after store merging.
@@ -13,9 +14,15 @@
 ; MIR-DAG: ![[SET3:[0-9]+]] = !{![[SCOPE3]]}
 
 define void @blam0(ptr %g0, ptr %g1) {
-; MIR-LABEL: name: blam0
-; MIR: LDRDui %0, 0 :: (load (s64) from %ir.g0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]])
-; MIR: STRDui killed %{{[0-9]*}}, %1, 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope ![[SET1]], !noalias ![[SET0]])
+  ; MIR-LABEL: name: blam0
+  ; MIR: bb.0 (%ir-block.0):
+  ; MIR-NEXT:   liveins: $x0, $x1
+  ; MIR-NEXT: {{  $}}
+  ; MIR-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+  ; MIR-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+  ; MIR-NEXT:   [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY1]], 0 :: (load (s64) from %ir.g0, align 4, !alias.scope !0, !noalias !3)
+  ; MIR-NEXT:   STRDui killed [[LDRDui]], [[COPY]], 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope !3, !noalias !0)
+  ; MIR-NEXT:   RET_ReallyLR
   %tmp4 = getelementptr inbounds <3 x float>, ptr %g1, i64 0, i64 0
   %tmp5 = load <3 x float>, ptr %g0, align 4, !alias.scope !0, !noalias !1
   %tmp6 = extractelement <3 x float> %tmp5, i64 0
@@ -29,13 +36,15 @@ define void @blam0(ptr %g0, ptr %g1) {
 
 ; Ensure new scoped AA metadata are calculated after merging stores.
 define void @blam1(ptr %g0, ptr %g1) {
-; MIR-LABEL: name: blam1
-; MIR: machineMetadataNodes:
-; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE2]], ![[SCOPE1]]}
-; MIR-DAG: ![[MMSET1:[0-9]+]] = !{}
-; MIR: body:
-; MIR: LDRDui %0, 0 :: (load (s64) from %ir.g0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]])
-; MIR: STRDui killed %{{[0-9]*}}, %1, 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET1]])
+  ; MIR-LABEL: name: blam1
+  ; MIR: bb.0 (%ir-block.0):
+  ; MIR-NEXT:   liveins: $x0, $x1
+  ; MIR-NEXT: {{  $}}
+  ; MIR-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+  ; MIR-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+  ; MIR-NEXT:   [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY1]], 0 :: (load (s64) from %ir.g0, align 4, !alias.scope !0, !noalias !3)
+  ; MIR-NEXT:   STRDui killed [[LDRDui]], [[COPY]], 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope !9, !noalias !10)
+  ; MIR-NEXT:   RET_ReallyLR
   %tmp4 = getelementptr inbounds <3 x float>, ptr %g1, i64 0, i64 0
   %tmp5 = load <3 x float>, ptr %g0, align 4, !alias.scope !0, !noalias !1
   %tmp6 = extractelement <3 x float> %tmp5, i64 0
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 0db4dc07fbfe2..2d81a264e02bc 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -1255,4 +1255,3 @@ entry:
     %partial.reduce = tail call <2 x i64> @llvm.experimental.vector.partial.reduce.add(<2 x i64> %acc, <8 x i64> %input.wide)
     ret <2 x i64> %partial.reduce
 }
-
diff --git a/llvm/test/CodeGen/AArch64/neon-sad.ll b/llvm/test/CodeGen/AArch64/neon-sad.ll
index c0cfe8d8ca3cc..f21fafea50451 100644
--- a/llvm/test/CodeGen/AArch64/neon-sad.ll
+++ b/llvm/test/CodeGen/AArch64/neon-sad.ll
@@ -45,3 +45,47 @@ entry:
   %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %5)
   ret i32 %6
 }
+
+define i32 @test_sad_v16i8_two_step_zext(ptr noundef readonly %a, ptr noundef readonly %b) {
+; CHECK-LABEL: test_sad_v16i8_two_step_zext:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    uabdl v2.8h, v1.8b, v0.8b
+; CHECK-NEXT:    uabal2 v2.8h, v1.16b, v0.16b
+; CHECK-NEXT:    uaddlv s0, v2.8h
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i8>, ptr %a
+  %1 = zext <16 x i8> %0 to <16 x i16>
+  %2 = load <16 x i8>, ptr %b
+  %3 = zext <16 x i8> %2 to <16 x i16>
+  %4 = sub nsw <16 x i16> %3, %1
+  %5 = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> %4, i1 false)
+  %6 = zext <16 x i16> %5 to <16 x i32>
+  %7 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %6)
+  ret i32 %7
+}
+
+define i32 @test_sad_v16i8_two_step_sext(ptr noundef readonly %a, ptr noundef readonly %b) {
+; CHECK-LABEL: test_sad_v16i8_two_step_sext:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sabdl v2.8h, v1.8b, v0.8b
+; CHECK-NEXT:    sabal2 v2.8h, v1.16b, v0.16b
+; CHECK-NEXT:    uaddlv s0, v2.8h
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <16 x i8>, ptr %a
+  %1 = sext <16 x i8> %0 to <16 x i16>
+  %2 = load <16 x i8>, ptr %b
+  %3 = sext <16 x i8> %2 to <16 x i16>
+  %4 = sub nsw <16 x i16> %3, %1
+  %5 = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> %4, i1 false)
+  %6 = zext <16 x i16> %5 to <16 x i32>
+  %7 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %6)
+  ret i32 %7
+}
diff --git a/llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll b/llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll
new file mode 100644
index 0000000000000..db7d26b6ed139
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=aarch64 -mattr=+sve -stop-after=aarch64-promote-const < %s | FileCheck %s
+
+; Test that the constant inside the `phi` is not promoted to a global
+; CHECK-NOT: _PromotedConst
+define void @f(i1 %c, ptr %p, ptr %q) {
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %u = load [2 x <vscale x 4 x float> ], ptr %p
+  br label %exit
+
+if.else:
+  br label %exit
+
+exit:
+  %v = phi [2 x <vscale x 4 x float> ]  [ %u, %if.then], [[<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> poison], %if.else]
+  store [2 x <vscale x 4 x float>] %v, ptr %q
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index c158d8ad93b05..3900910c10318 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -23,15 +23,36 @@ define i8 @popcount128(ptr nocapture nonnull readonly %0) {
 ; CHECKO0-NEXT:    fmov w0, s0
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount128:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    add x8, x0, #8
-; CHECK-NEXT:    ld1 { v0.d }[1], [x8]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount128:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    ldr d0, [x0]
+; NEON-NEXT:    add x8, x0, #8
+; NEON-NEXT:    ld1 { v0.d }[1], [x8]
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    addv b0, v0.16b
+; NEON-NEXT:    fmov w0, s0
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount128:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    ldr d0, [x0]
+; DOT-NEXT:    add x8, x0, #8
+; DOT-NEXT:    ld1 { v0.d }[1], [x8]
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    addv b0, v0.16b
+; DOT-NEXT:    fmov w0, s0
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount128:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ldr d0, [x0]
+; SVE-NEXT:    add x8, x0, #8
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    ld1 { v0.d }[1], [x8]
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    addp d0, v0.2d
+; SVE-NEXT:    fmov w0, s0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount128:
 ; BE:       // %bb.0: // %Entry
@@ -107,22 +128,55 @@ define i16 @popcount256(ptr nocapture nonnull readonly %0) {
 ; CHECKO0-NEXT:    mov w0, w8
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount256:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    ldr d0, [x0, #16]
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    add x8, x0, #8
-; CHECK-NEXT:    add x9, x0, #24
-; CHECK-NEXT:    ld1 { v0.d }[1], [x9]
-; CHECK-NEXT:    ld1 { v1.d }[1], [x8]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    addv b1, v1.16b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    fmov w9, s1
-; CHECK-NEXT:    add w0, w9, w8
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount256:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    ldr d0, [x0, #16]
+; NEON-NEXT:    ldr d1, [x0]
+; NEON-NEXT:    add x8, x0, #8
+; NEON-NEXT:    add x9, x0, #24
+; NEON-NEXT:    ld1 { v0.d }[1], [x9]
+; NEON-NEXT:    ld1 { v1.d }[1], [x8]
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    cnt v1.16b, v1.16b
+; NEON-NEXT:    addv b0, v0.16b
+; NEON-NEXT:    addv b1, v1.16b
+; NEON-NEXT:    fmov w8, s0
+; NEON-NEXT:    fmov w9, s1
+; NEON-NEXT:    add w0, w9, w8
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount256:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    ldr d0, [x0, #16]
+; DOT-NEXT:    ldr d1, [x0]
+; DOT-NEXT:    add x8, x0, #8
+; DOT-NEXT:    add x9, x0, #24
+; DOT-NEXT:    ld1 { v0.d }[1], [x9]
+; DOT-NEXT:    ld1 { v1.d }[1], [x8]
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    cnt v1.16b, v1.16b
+; DOT-NEXT:    addv b0, v0.16b
+; DOT-NEXT:    addv b1, v1.16b
+; DOT-NEXT:    fmov w8, s0
+; DOT-NEXT:    fmov w9, s1
+; DOT-NEXT:    add w0, w9, w8
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount256:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ldr d0, [x0, #16]
+; SVE-NEXT:    ldr d1, [x0]
+; SVE-NEXT:    add x8, x0, #8
+; SVE-NEXT:    add x9, x0, #24
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    ld1 { v0.d }[1], [x9]
+; SVE-NEXT:    ld1 { v1.d }[1], [x8]
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    cnt z1.d, p0/m, z1.d
+; SVE-NEXT:    add v0.2d, v1.2d, v0.2d
+; SVE-NEXT:    addp d0, v0.2d
+; SVE-NEXT:    fmov w0, s0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount256:
 ; BE:       // %bb.0: // %Entry
@@ -223,15 +277,36 @@ define <1 x i128> @popcount1x128(<1 x i128> %0) {
 ; CHECKO0-NEXT:    bfi x0, x8, #32, #32
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount1x128:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    fmov d0, x0
-; CHECK-NEXT:    mov v0.d[1], x1
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount1x128:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    fmov d0, x0
+; NEON-NEXT:    mov v0.d[1], x1
+; NEON-NEXT:    mov x1, xzr
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    addv b0, v0.16b
+; NEON-NEXT:    fmov x0, d0
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount1x128:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    fmov d0, x0
+; DOT-NEXT:    mov v0.d[1], x1
+; DOT-NEXT:    mov x1, xzr
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    addv b0, v0.16b
+; DOT-NEXT:    fmov x0, d0
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount1x128:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    fmov d0, x0
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    mov v0.d[1], x1
+; SVE-NEXT:    mov x1, xzr
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    addp d0, v0.2d
+; SVE-NEXT:    fmov x0, d0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount1x128:
 ; BE:       // %bb.0: // %Entry
@@ -305,10 +380,10 @@ define <2 x i64> @popcount2x64(<2 x i64> %0) {
 ;
 ; SVE-LABEL: popcount2x64:
 ; SVE:       // %bb.0: // %Entry
-; SVE-NEXT:    cnt v0.16b, v0.16b
-; SVE-NEXT:    uaddlp v0.8h, v0.16b
-; SVE-NEXT:    uaddlp v0.4s, v0.8h
-; SVE-NEXT:    uaddlp v0.2d, v0.4s
+; SVE-NEXT:    ptrue p0.d, vl2
+; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount2x64:
@@ -374,13 +449,29 @@ define <1 x i64> @popcount1x64(<1 x i64> %0) {
 ; CHECKO0-NEXT:    fmov d0, x8
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount1x64:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount1x64:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    cnt v0.8b, v0.8b
+; NEON-NEXT:    uaddlp v0.4h, v0.8b
+; NEON-NEXT:    uaddlp v0.2s, v0.4h
+; NEON-NEXT:    uaddlp v0.1d, v0.2s
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount1x64:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    cnt v0.8b, v0.8b
+; DOT-NEXT:    uaddlp v0.4h, v0.8b
+; DOT-NEXT:    uaddlp v0.2s, v0.4h
+; DOT-NEXT:    uaddlp v0.1d, v0.2s
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount1x64:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ptrue p0.d, vl1
+; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount1x64:
 ; BE:       // %bb.0: // %Entry
@@ -442,9 +533,10 @@ define <4 x i32> @popcount4x32(<4 x i32> %0) {
 ;
 ; SVE-LABEL: popcount4x32:
 ; SVE:       // %bb.0: // %Entry
-; SVE-NEXT:    cnt v0.16b, v0.16b
-; SVE-NEXT:    uaddlp v0.8h, v0.16b
-; SVE-NEXT:    uaddlp v0.4s, v0.8h
+; SVE-NEXT:    ptrue p0.s, vl4
+; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT:    cnt z0.s, p0/m, z0.s
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount4x32:
@@ -520,9 +612,10 @@ define <2 x i32> @popcount2x32(<2 x i32> %0) {
 ;
 ; SVE-LABEL: popcount2x32:
 ; SVE:       // %bb.0: // %Entry
-; SVE-NEXT:    cnt v0.8b, v0.8b
-; SVE-NEXT:    uaddlp v0.4h, v0.8b
-; SVE-NEXT:    uaddlp v0.2s, v0.4h
+; SVE-NEXT:    ptrue p0.s, vl2
+; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT:    cnt z0.s, p0/m, z0.s
+; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount2x32:
@@ -577,11 +670,25 @@ define <8 x i16> @popcount8x16(<8 x i16> %0) {
 ; CHECKO0-NEXT:    uaddlp v0.8h, v0.16b
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount8x16:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount8x16:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    uaddlp v0.8h, v0.16b
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount8x16:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    uaddlp v0.8h, v0.16b
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount8x16:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ptrue p0.h, vl8
+; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT:    cnt z0.h, p0/m, z0.h
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount8x16:
 ; BE:       // %bb.0: // %Entry
@@ -618,11 +725,25 @@ define <4 x i16> @popcount4x16(<4 x i16> %0) {
 ; CHECKO0-NEXT:    uaddlp v0.4h, v0.8b
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount4x16:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount4x16:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    cnt v0.8b, v0.8b
+; NEON-NEXT:    uaddlp v0.4h, v0.8b
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount4x16:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    cnt v0.8b, v0.8b
+; DOT-NEXT:    uaddlp v0.4h, v0.8b
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount4x16:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ptrue p0.h, vl4
+; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT:    cnt z0.h, p0/m, z0.h
+; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount4x16:
 ; BE:       // %bb.0: // %Entry
@@ -673,20 +794,49 @@ define i32 @ctpop_into_extract(ptr %p) {
 ; CHECKO0-NEXT:    mov w0, wzr
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: ctpop_into_extract:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    movi v2.2d, #0xffffffffffffffff
-; CHECK-NEXT:    mov x8, x0
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    cnt v1.8b, v1.8b
-; CHECK-NEXT:    addv b1, v1.8b
-; CHECK-NEXT:    mov v2.s[1], v1.s[0]
-; CHECK-NEXT:    sub v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    str d0, [x8]
-; CHECK-NEXT:    ret
+; NEON-LABEL: ctpop_into_extract:
+; NEON:       // %bb.0:
+; NEON-NEXT:    ldr d0, [x0]
+; NEON-NEXT:    movi v2.2d, #0xffffffffffffffff
+; NEON-NEXT:    mov x8, x0
+; NEON-NEXT:    mov w0, wzr
+; NEON-NEXT:    fmov w9, s0
+; NEON-NEXT:    fmov s1, w9
+; NEON-NEXT:    cnt v1.8b, v1.8b
+; NEON-NEXT:    addv b1, v1.8b
+; NEON-NEXT:    mov v2.s[1], v1.s[0]
+; NEON-NEXT:    sub v0.2s, v0.2s, v2.2s
+; NEON-NEXT:    str d0, [x8]
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: ctpop_into_extract:
+; DOT:       // %bb.0:
+; DOT-NEXT:    ldr d0, [x0]
+; DOT-NEXT:    movi v2.2d, #0xffffffffffffffff
+; DOT-NEXT:    mov x8, x0
+; DOT-NEXT:    mov w0, wzr
+; DOT-NEXT:    fmov w9, s0
+; DOT-NEXT:    fmov s1, w9
+; DOT-NEXT:    cnt v1.8b, v1.8b
+; DOT-NEXT:    addv b1, v1.8b
+; DOT-NEXT:    mov v2.s[1], v1.s[0]
+; DOT-NEXT:    sub v0.2s, v0.2s, v2.2s
+; DOT-NEXT:    str d0, [x8]
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: ctpop_into_extract:
+; SVE:       // %bb.0:
+; SVE-NEXT:    ldr d0, [x0]
+; SVE-NEXT:    ptrue p0.s
+; SVE-NEXT:    movi v2.2d, #0xffffffffffffffff
+; SVE-NEXT:    mov x8, x0
+; SVE-NEXT:    mov w0, wzr
+; SVE-NEXT:    movprfx z1, z0
+; SVE-NEXT:    cnt z1.s, p0/m, z0.s
+; SVE-NEXT:    mov v2.s[1], v1.s[0]
+; SVE-NEXT:    sub v0.2s, v0.2s, v2.2s
+; SVE-NEXT:    str d0, [x8]
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: ctpop_into_extract:
 ; BE:       // %bb.0:
@@ -752,3 +902,5 @@ define i32 @ctpop_into_extract(ptr %p) {
 }
 
 declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/powi-arm64ec.ll b/llvm/test/CodeGen/AArch64/powi-arm64ec.ll
new file mode 100644
index 0000000000000..707159eb432ec
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/powi-arm64ec.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=arm64ec-windows-msvc < %s | FileCheck -check-prefix=ARM64EC %s
+
+declare double @llvm.powi.f64.i32(double, i32)
+declare float @llvm.powi.f32.i32(float, i32)
+
+; ARM64EC-LABEL: powi_f32
+; ARM64EC: scvtf s1, w0
+; ARM64EC: b "#powf"
+define float @powi_f32(float %x, i32 %n) nounwind {
+  %ret = tail call float @llvm.powi.f32.i32(float %x, i32 %n)
+  ret float %ret
+}
+
+; ARM64EC-LABEL: powi_f64
+; ARM64EC: scvtf d1, w0
+; ARM64EC: b "#pow"
+define double @powi_f64(double %x, i32 %n) nounwind {
+  %ret = tail call double @llvm.powi.f64.i32(double %x, i32 %n)
+  ret double %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/powi.ll b/llvm/test/CodeGen/AArch64/powi.ll
index 64a3d168d1155..8b45f1c13dab4 100644
--- a/llvm/test/CodeGen/AArch64/powi.ll
+++ b/llvm/test/CodeGen/AArch64/powi.ll
@@ -1,25 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+; RUN: llc -mtriple=aarch64-windows-pc-msvc < %s | FileCheck -check-prefix=WINDOWS %s
 
 declare double @llvm.powi.f64.i32(double, i32)
 declare float  @llvm.powi.f32.i32(float,  i32)
 declare float  @pow(double noundef, double noundef)
 
-define float @powi_f32(float %x) nounwind {
-; CHECK-LABEL: powi_f32:
+define float @powi_f32_4(float %x) nounwind {
+; CHECK-LABEL: powi_f32_4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul s0, s0, s0
 ; CHECK-NEXT:    fmul s0, s0, s0
 ; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: powi_f32_4:
+; WINDOWS:       // %bb.0:
+; WINDOWS-NEXT:    fmul s0, s0, s0
+; WINDOWS-NEXT:    fmul s0, s0, s0
+; WINDOWS-NEXT:    ret
   %1 = tail call float @llvm.powi.f32.i32(float %x, i32 4)
   ret float %1
 }
 
-define double @powi_f64(double %x) nounwind {
-; CHECK-LABEL: powi_f64:
+define double @powi_f64_3(double %x) nounwind {
+; CHECK-LABEL: powi_f64_3:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul d1, d0, d0
 ; CHECK-NEXT:    fmul d0, d0, d1
 ; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: powi_f64_3:
+; WINDOWS:       // %bb.0:
+; WINDOWS-NEXT:    fmul d1, d0, d0
+; WINDOWS-NEXT:    fmul d0, d0, d1
+; WINDOWS-NEXT:    ret
   %1 = tail call double @llvm.powi.f64.i32(double %x, i32 3)
   ret double %1
 }
+
+define float @powi_f32(float %x, i32 %n) nounwind {
+; CHECK-LABEL: powi_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    b __powisf2
+;
+; WINDOWS-LABEL: powi_f32:
+; WINDOWS:       // %bb.0:
+; WINDOWS-NEXT:    scvtf s1, w0
+; WINDOWS-NEXT:    b powf
+  %ret = tail call float @llvm.powi.f32.i32(float %x, i32 %n)
+  ret float %ret
+}
+
+define double @powi_f64(double %x, i32 %n) nounwind {
+; CHECK-LABEL: powi_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    b __powidf2
+;
+; WINDOWS-LABEL: powi_f64:
+; WINDOWS:       // %bb.0:
+; WINDOWS-NEXT:    scvtf d1, w0
+; WINDOWS-NEXT:    b pow
+  %ret = tail call double @llvm.powi.f64.i32(double %x, i32 %n)
+  ret double %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/pr58431.ll b/llvm/test/CodeGen/AArch64/pr58431.ll
index 88bab4af95d64..467ceb062f249 100644
--- a/llvm/test/CodeGen/AArch64/pr58431.ll
+++ b/llvm/test/CodeGen/AArch64/pr58431.ll
@@ -4,10 +4,12 @@
 define i32 @f(i64 %0) {
 ; CHECK-LABEL: f:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #10 // =0xa
+; CHECK-NEXT:    mov x8, #-7378697629483820647 // =0x9999999999999999
 ; CHECK-NEXT:    mov w9, w0
-; CHECK-NEXT:    udiv x10, x9, x8
-; CHECK-NEXT:    msub x0, x10, x8, x9
+; CHECK-NEXT:    mov w10, #10 // =0xa
+; CHECK-NEXT:    eor x8, x8, #0x8000000000000003
+; CHECK-NEXT:    umulh x8, x9, x8
+; CHECK-NEXT:    msub x0, x8, x10, x9
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
   %2 = trunc i64 %0 to i32
diff --git a/llvm/test/CodeGen/AArch64/ptradd.ll b/llvm/test/CodeGen/AArch64/ptradd.ll
index 28a8f4303765b..fc364357436e2 100644
--- a/llvm/test/CodeGen/AArch64/ptradd.ll
+++ b/llvm/test/CodeGen/AArch64/ptradd.ll
@@ -285,19 +285,11 @@ entry:
 }
 
 define <1 x ptr> @vector_gep_v1i64_c10(ptr %b) {
-; CHECK-SD-LABEL: vector_gep_v1i64_c10:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    fmov d1, x8
-; CHECK-SD-NEXT:    add d0, d0, d1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vector_gep_v1i64_c10:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    add x8, x0, #10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vector_gep_v1i64_c10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add x8, x0, #10
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
 entry:
   %g = getelementptr i8, ptr %b, <1 x i64> <i64 10>
   ret <1 x ptr> %g
@@ -306,10 +298,8 @@ entry:
 define <2 x ptr> @vector_gep_v2i64_c10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v2i64_c10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    dup v1.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    add x8, x0, #10
+; CHECK-SD-NEXT:    dup v0.2d, x8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v2i64_c10:
@@ -327,15 +317,10 @@ entry:
 define <3 x ptr> @vector_gep_v3i64_c10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v3i64_c10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    fmov d3, x0
-; CHECK-SD-NEXT:    dup v2.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v2.2d
-; CHECK-SD-NEXT:    add d2, d3, d2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    add x8, x0, #10
+; CHECK-SD-NEXT:    fmov d0, x8
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v3i64_c10:
@@ -356,10 +341,8 @@ entry:
 define <4 x ptr> @vector_gep_v4i64_c10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v4i64_c10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    dup v1.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    add x8, x0, #10
+; CHECK-SD-NEXT:    dup v0.2d, x8
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    ret
 ;
@@ -377,19 +360,11 @@ entry:
 }
 
 define <1 x ptr> @vector_gep_v1i64_cm10(ptr %b) {
-; CHECK-SD-LABEL: vector_gep_v1i64_cm10:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    fmov d1, x0
-; CHECK-SD-NEXT:    fmov d0, x8
-; CHECK-SD-NEXT:    add d0, d1, d0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vector_gep_v1i64_cm10:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub x8, x0, #10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vector_gep_v1i64_cm10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub x8, x0, #10
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
 entry:
   %g = getelementptr i8, ptr %b, <1 x i64> <i64 -10>
   ret <1 x ptr> %g
@@ -398,10 +373,8 @@ entry:
 define <2 x ptr> @vector_gep_v2i64_cm10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v2i64_cm10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    dup v1.2d, x0
+; CHECK-SD-NEXT:    sub x8, x0, #10
 ; CHECK-SD-NEXT:    dup v0.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v2i64_cm10:
@@ -419,15 +392,10 @@ entry:
 define <3 x ptr> @vector_gep_v3i64_cm10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v3i64_cm10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    fmov d3, x0
-; CHECK-SD-NEXT:    dup v2.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v2.2d
-; CHECK-SD-NEXT:    add d2, d3, d2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    sub x8, x0, #10
+; CHECK-SD-NEXT:    fmov d0, x8
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v3i64_cm10:
@@ -448,10 +416,8 @@ entry:
 define <4 x ptr> @vector_gep_v4i64_cm10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v4i64_cm10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    dup v1.2d, x0
+; CHECK-SD-NEXT:    sub x8, x0, #10
 ; CHECK-SD-NEXT:    dup v0.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-reloc.ll b/llvm/test/CodeGen/AArch64/ptrauth-reloc.ll
index b7304b957a001..932cc946db0ea 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-reloc.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-reloc.ll
@@ -139,38 +139,39 @@
 
 ;--- err-key.ll
 
-; RUN: not --crash llc < err-key.ll -mtriple arm64e-apple-darwin 2>&1 \
+; RUN: not llc < err-key.ll -mtriple arm64e-apple-darwin 2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-KEY
-; RUN: not --crash llc < err-key.ll -mtriple aarch64-elf -mattr=+pauth 2>&1 \
+; RUN: not llc < err-key.ll -mtriple aarch64-elf -mattr=+pauth 2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-KEY
 
-; RUN: not --crash llc < err-key.ll -mtriple arm64e-apple-darwin \
+; RUN: not llc < err-key.ll -mtriple arm64e-apple-darwin \
 ; RUN:   -global-isel -verify-machineinstrs -global-isel-abort=1  2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-KEY
-; RUN: not --crash llc < err-key.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: not llc < err-key.ll -mtriple aarch64-elf -mattr=+pauth \
 ; RUN:   -global-isel -verify-machineinstrs -global-isel-abort=1 2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-KEY
 
-; CHECK-ERR-KEY: LLVM ERROR: AArch64 PAC Key ID '4' out of range [0, 3]
+; CHECK-ERR-KEY: error: AArch64 PAC Key ID '4' out of range [0, 3]
+
 
 @g = external global i32
 @g.ref.4.0 = constant ptr ptrauth (ptr @g, i32 4, i64 0)
 
 ;--- err-disc.ll
 
-; RUN: not --crash llc < err-disc.ll -mtriple arm64e-apple-darwin 2>&1 \
+; RUN: not llc < err-disc.ll -mtriple arm64e-apple-darwin 2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-DISC
-; RUN: not --crash llc < err-disc.ll -mtriple aarch64-elf -mattr=+pauth 2>&1 \
+; RUN: not llc < err-disc.ll -mtriple aarch64-elf -mattr=+pauth 2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-DISC
 
-; RUN: not --crash llc < err-disc.ll -mtriple arm64e-apple-darwin \
+; RUN: not llc < err-disc.ll -mtriple arm64e-apple-darwin \
 ; RUN:   -global-isel -verify-machineinstrs -global-isel-abort=1  2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-DISC
-; RUN: not --crash llc < err-disc.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: not llc < err-disc.ll -mtriple aarch64-elf -mattr=+pauth \
 ; RUN:   -global-isel -verify-machineinstrs -global-isel-abort=1 2>&1 \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-ERR-DISC
 
-; CHECK-ERR-DISC: LLVM ERROR: AArch64 PAC Discriminator '65536' out of range [0, 0xFFFF]
+; CHECK-ERR-DISC: error: AArch64 PAC Discriminator '65536' out of range [0, 0xFFFF]
 
 @g = external global i32
 @g.ref.ia.65536 = constant ptr ptrauth (ptr @g, i32 0, i64 65536)
diff --git a/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll b/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
index 01943f40d41e8..347f777187af7 100644
--- a/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
+++ b/llvm/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
@@ -1,11 +1,26 @@
-; RUN: llc -mtriple=aarch64-none-linux-gnu -frame-pointer=all < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -frame-pointer=none < %s | \
+; RUN:    FileCheck %s --check-prefixes=CHECK,NONE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -frame-pointer=reserved < %s | \
+; RUN:    FileCheck %s --check-prefixes=CHECK,RESERVED
+; RUN: llc -mtriple=aarch64-none-linux-gnu -frame-pointer=all < %s | \
+; RUN:    FileCheck %s --check-prefixes=CHECK,ALL
+
+; By default, Darwin and Windows will reserve x29
+; RUN: llc -mtriple=aarch64-darwin -frame-pointer=none < %s | \
+; RUN:    FileCheck %s --check-prefixes=CHECK,RESERVED
+; RUN: llc -mtriple=aarch64-darwin -frame-pointer=none < %s | \
+; RUN:    FileCheck %s --check-prefixes=CHECK,RESERVED
 @var = global i32 0
 
 declare void @bar()
 
 define void @test_w29_reserved() {
 ; CHECK-LABEL: test_w29_reserved:
-; CHECK: mov x29, sp
+; ALL: add x29, sp
+; NONE-NOT: add x29
+; NONE-NOT: mov x29
+; RESERVED-NOT: add x29
+; RESERVED-NOT: mov x29
 
   %val1 = load volatile i32, ptr @var
   %val2 = load volatile i32, ptr @var
@@ -16,8 +31,11 @@ define void @test_w29_reserved() {
   %val7 = load volatile i32, ptr @var
   %val8 = load volatile i32, ptr @var
   %val9 = load volatile i32, ptr @var
+  %val10 = load volatile i32, ptr @var
 
-; CHECK-NOT: ldr w29,
+; NONE: ldr w29,
+; ALL-NOT: ldr w29,
+; RESERVED-NOT: ldr w29,
 
   ; Call to prevent fp-elim that occurs regardless in leaf functions.
   call void @bar()
@@ -31,6 +49,7 @@ define void @test_w29_reserved() {
   store volatile i32 %val7,  ptr @var
   store volatile i32 %val8,  ptr @var
   store volatile i32 %val9,  ptr @var
+  store volatile i32 %val10,  ptr @var
 
   ret void
 ; CHECK: ret
diff --git a/llvm/test/CodeGen/AArch64/rem-by-const.ll b/llvm/test/CodeGen/AArch64/rem-by-const.ll
new file mode 100644
index 0000000000000..1376f5d9a380d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/rem-by-const.ll
@@ -0,0 +1,3616 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define i8 @si8_7(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: si8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxtb w8, w0
+; CHECK-SD-NEXT:    mov w9, #-109 // =0xffffff93
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    add w8, w0, w8, lsr #8
+; CHECK-SD-NEXT:    sbfx w9, w8, #2, #6
+; CHECK-SD-NEXT:    and w8, w8, #0x80
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #7
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    mov w9, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i8 %a, 7
+  ret i8 %s
+}
+
+define i8 @si8_100(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: si8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxtb w8, w0
+; CHECK-SD-NEXT:    mov w9, #41 // =0x29
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    asr w9, w8, #12
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i8 %a, 100
+  ret i8 %s
+}
+
+define i8 @ui8_7(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: ui8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #37 // =0x25
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    lsr w8, w8, #8
+; CHECK-SD-NEXT:    sub w9, w0, w8
+; CHECK-SD-NEXT:    and w9, w9, #0xfe
+; CHECK-SD-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #37 // =0x25
+; CHECK-GI-NEXT:    and w9, w0, #0xff
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    lsr w8, w8, #8
+; CHECK-GI-NEXT:    sub w9, w0, w8
+; CHECK-GI-NEXT:    ubfx w9, w9, #1, #7
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    ubfx w8, w8, #2, #6
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i8 %a, 7
+  ret i8 %s
+}
+
+define i8 @ui8_100(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: ui8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #41 // =0x29
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    lsr w8, w8, #12
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #41 // =0x29
+; CHECK-GI-NEXT:    and w9, w0, #0xff
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    lsr w8, w8, #8
+; CHECK-GI-NEXT:    lsr w8, w8, #4
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i8 %a, 100
+  ret i8 %s
+}
+
+define i16 @si16_7(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: si16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxth w8, w0
+; CHECK-SD-NEXT:    mov w9, #18725 // =0x4925
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    asr w9, w8, #17
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxth w8, w0
+; CHECK-GI-NEXT:    mov w9, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i16 %a, 7
+  ret i16 %s
+}
+
+define i16 @si16_100(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: si16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxth w8, w0
+; CHECK-SD-NEXT:    mov w9, #5243 // =0x147b
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    asr w9, w8, #19
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxth w8, w0
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i16 %a, 100
+  ret i16 %s
+}
+
+define i16 @ui16_7(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: ui16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    and w9, w0, #0xffff
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    lsr w8, w8, #16
+; CHECK-SD-NEXT:    sub w9, w0, w8
+; CHECK-SD-NEXT:    and w9, w9, #0xfffe
+; CHECK-SD-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-GI-NEXT:    and w9, w0, #0xffff
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    sub w9, w0, w8
+; CHECK-GI-NEXT:    ubfx w9, w9, #1, #15
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    ubfx w8, w8, #2, #14
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i16 %a, 7
+  ret i16 %s
+}
+
+define i16 @ui16_100(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: ui16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ubfx w8, w0, #2, #14
+; CHECK-SD-NEXT:    mov w9, #5243 // =0x147b
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    lsr w8, w8, #17
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ubfx w8, w0, #2, #14
+; CHECK-GI-NEXT:    mov w9, #5243 // =0x147b
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    lsr w8, w8, #1
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i16 %a, 100
+  ret i16 %s
+}
+
+define i32 @si32_7(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: si32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    smull x8, w0, w8
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    add w8, w8, w0
+; CHECK-SD-NEXT:    asr w9, w8, #2
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w8, w0, w8
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i32 %a, 7
+  ret i32 %s
+}
+
+define i32 @si32_100(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: si32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    smull x8, w0, w8
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w9, w0, w8
+; CHECK-GI-NEXT:    msub w0, w9, w8, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i32 %a, 100
+  ret i32 %s
+}
+
+define i32 @ui32_7(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: ui32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    umull x8, w0, w8
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w9, w0, w8
+; CHECK-SD-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-GI-NEXT:    movk w8, #9362, lsl #16
+; CHECK-GI-NEXT:    umull x8, w0, w8
+; CHECK-GI-NEXT:    lsr x8, x8, #32
+; CHECK-GI-NEXT:    sub w9, w0, w8
+; CHECK-GI-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-GI-NEXT:    lsr w8, w8, #2
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i32 %a, 7
+  ret i32 %s
+}
+
+define i32 @ui32_100(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: ui32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    umull x8, w0, w8
+; CHECK-SD-NEXT:    lsr x8, x8, #37
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    movk w8, #20971, lsl #16
+; CHECK-GI-NEXT:    umull x8, w0, w8
+; CHECK-GI-NEXT:    lsr x8, x8, #32
+; CHECK-GI-NEXT:    lsr w8, w8, #5
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i32 %a, 100
+  ret i32 %s
+}
+
+define i64 @si64_7(i64 %a, i64 %b) {
+; CHECK-SD-LABEL: si64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #16
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #32
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #48
+; CHECK-SD-NEXT:    smulh x8, x0, x8
+; CHECK-SD-NEXT:    asr x9, x8, #1
+; CHECK-SD-NEXT:    add x8, x9, x8, lsr #63
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    add x0, x0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv x8, x0, x8
+; CHECK-GI-NEXT:    lsl x9, x8, #3
+; CHECK-GI-NEXT:    sub x8, x9, x8
+; CHECK-GI-NEXT:    sub x0, x0, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i64 %a, 7
+  ret i64 %s
+}
+
+define i64 @si64_100(i64 %a, i64 %b) {
+; CHECK-SD-LABEL: si64_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #55051 // =0xd70b
+; CHECK-SD-NEXT:    movk x8, #28835, lsl #16
+; CHECK-SD-NEXT:    movk x8, #2621, lsl #32
+; CHECK-SD-NEXT:    movk x8, #41943, lsl #48
+; CHECK-SD-NEXT:    smulh x8, x0, x8
+; CHECK-SD-NEXT:    add x8, x8, x0
+; CHECK-SD-NEXT:    asr x9, x8, #6
+; CHECK-SD-NEXT:    add x8, x9, x8, lsr #63
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    msub x0, x8, x9, x0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si64_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv x9, x0, x8
+; CHECK-GI-NEXT:    msub x0, x9, x8, x0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i64 %a, 100
+  ret i64 %s
+}
+
+define i64 @ui64_7(i64 %a, i64 %b) {
+; CHECK-SD-LABEL: ui64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #32
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #48
+; CHECK-SD-NEXT:    umulh x8, x0, x8
+; CHECK-SD-NEXT:    sub x9, x0, x8
+; CHECK-SD-NEXT:    add x8, x8, x9, lsr #1
+; CHECK-SD-NEXT:    lsr x8, x8, #2
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    add x0, x0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
+; CHECK-GI-NEXT:    umulh x8, x0, x8
+; CHECK-GI-NEXT:    sub x9, x0, x8
+; CHECK-GI-NEXT:    add x8, x8, x9, lsr #1
+; CHECK-GI-NEXT:    lsr x8, x8, #2
+; CHECK-GI-NEXT:    lsl x9, x8, #3
+; CHECK-GI-NEXT:    sub x8, x9, x8
+; CHECK-GI-NEXT:    sub x0, x0, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i64 %a, 7
+  ret i64 %s
+}
+
+define i64 @ui64_100(i64 %a, i64 %b) {
+; CHECK-LABEL: ui64_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x9, #62915 // =0xf5c3
+; CHECK-NEXT:    lsr x8, x0, #2
+; CHECK-NEXT:    movk x9, #23592, lsl #16
+; CHECK-NEXT:    movk x9, #49807, lsl #32
+; CHECK-NEXT:    movk x9, #10485, lsl #48
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    mov w9, #100 // =0x64
+; CHECK-NEXT:    lsr x8, x8, #2
+; CHECK-NEXT:    msub x0, x8, x9, x0
+; CHECK-NEXT:    ret
+entry:
+  %s = urem i64 %a, 100
+  ret i64 %s
+}
+
+define i128 @si128_7(i128 %a, i128 %b) {
+; CHECK-LABEL: si128_7:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov w2, #7 // =0x7
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    bl __modti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %s = srem i128 %a, 7
+  ret i128 %s
+}
+
+define i128 @si128_100(i128 %a, i128 %b) {
+; CHECK-LABEL: si128_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov w2, #100 // =0x64
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    bl __modti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %s = srem i128 %a, 100
+  ret i128 %s
+}
+
+define i128 @ui128_7(i128 %a, i128 %b) {
+; CHECK-SD-LABEL: ui128_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui128_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #18725 // =0x4925
+; CHECK-GI-NEXT:    mov x10, #9362 // =0x2492
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #16
+; CHECK-GI-NEXT:    movk x10, #37449, lsl #16
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #32
+; CHECK-GI-NEXT:    movk x10, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #48
+; CHECK-GI-NEXT:    movk x10, #9362, lsl #48
+; CHECK-GI-NEXT:    mul x9, x1, x8
+; CHECK-GI-NEXT:    mul x11, x0, x10
+; CHECK-GI-NEXT:    umulh x12, x0, x8
+; CHECK-GI-NEXT:    mul x13, x1, x10
+; CHECK-GI-NEXT:    adds x9, x9, x11
+; CHECK-GI-NEXT:    umulh x14, x1, x8
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    cmn x9, x12
+; CHECK-GI-NEXT:    and x9, x11, #0x1
+; CHECK-GI-NEXT:    sub x12, x0, x0
+; CHECK-GI-NEXT:    umulh x15, x0, x10
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    and x11, x11, #0x1
+; CHECK-GI-NEXT:    add x12, x13, x12
+; CHECK-GI-NEXT:    and x13, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x8, xzr, x8
+; CHECK-GI-NEXT:    add x9, x9, x11
+; CHECK-GI-NEXT:    and x11, xzr, #0x1
+; CHECK-GI-NEXT:    adds x12, x12, x14
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    umulh x10, x1, x10
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    adds x12, x12, x15
+; CHECK-GI-NEXT:    and x13, x13, #0x1
+; CHECK-GI-NEXT:    umulh x14, x0, xzr
+; CHECK-GI-NEXT:    cset w15, hs
+; CHECK-GI-NEXT:    adds x9, x12, x9
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    and x12, x15, #0x1
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, x13, #0x1
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    add x10, x11, x12
+; CHECK-GI-NEXT:    add x8, x8, x14
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    subs x10, x0, x9
+; CHECK-GI-NEXT:    sbc x11, x1, x8
+; CHECK-GI-NEXT:    lsl x12, x11, #63
+; CHECK-GI-NEXT:    lsr x11, x11, #1
+; CHECK-GI-NEXT:    orr x10, x12, x10, lsr #1
+; CHECK-GI-NEXT:    adds x9, x10, x9
+; CHECK-GI-NEXT:    adc x8, x11, x8
+; CHECK-GI-NEXT:    lsl x10, x8, #62
+; CHECK-GI-NEXT:    lsr x8, x8, #2
+; CHECK-GI-NEXT:    orr x9, x10, x9, lsr #2
+; CHECK-GI-NEXT:    mov w10, #7 // =0x7
+; CHECK-GI-NEXT:    lsl x12, x8, #3
+; CHECK-GI-NEXT:    umulh x10, x9, x10
+; CHECK-GI-NEXT:    lsl x11, x9, #3
+; CHECK-GI-NEXT:    sub x8, x12, x8
+; CHECK-GI-NEXT:    sub x9, x11, x9
+; CHECK-GI-NEXT:    subs x0, x0, x9
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    sbc x1, x1, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i128 %a, 7
+  ret i128 %s
+}
+
+define i128 @ui128_100(i128 %a, i128 %b) {
+; CHECK-SD-LABEL: ui128_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui128_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #23593 // =0x5c29
+; CHECK-GI-NEXT:    mov x10, #62914 // =0xf5c2
+; CHECK-GI-NEXT:    movk x8, #49807, lsl #16
+; CHECK-GI-NEXT:    movk x10, #23592, lsl #16
+; CHECK-GI-NEXT:    movk x8, #10485, lsl #32
+; CHECK-GI-NEXT:    movk x10, #49807, lsl #32
+; CHECK-GI-NEXT:    movk x8, #36700, lsl #48
+; CHECK-GI-NEXT:    movk x10, #10485, lsl #48
+; CHECK-GI-NEXT:    mul x9, x1, x8
+; CHECK-GI-NEXT:    mul x11, x0, x10
+; CHECK-GI-NEXT:    umulh x12, x0, x8
+; CHECK-GI-NEXT:    mul x13, x1, x10
+; CHECK-GI-NEXT:    adds x9, x9, x11
+; CHECK-GI-NEXT:    umulh x14, x1, x8
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    cmn x9, x12
+; CHECK-GI-NEXT:    and x9, x11, #0x1
+; CHECK-GI-NEXT:    sub x12, x0, x0
+; CHECK-GI-NEXT:    umulh x15, x0, x10
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    and x11, x11, #0x1
+; CHECK-GI-NEXT:    add x12, x13, x12
+; CHECK-GI-NEXT:    and x13, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x8, xzr, x8
+; CHECK-GI-NEXT:    add x9, x9, x11
+; CHECK-GI-NEXT:    and x11, xzr, #0x1
+; CHECK-GI-NEXT:    adds x12, x12, x14
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    umulh x10, x1, x10
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    adds x12, x12, x15
+; CHECK-GI-NEXT:    and x13, x13, #0x1
+; CHECK-GI-NEXT:    umulh x14, x0, xzr
+; CHECK-GI-NEXT:    cset w15, hs
+; CHECK-GI-NEXT:    adds x9, x12, x9
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    and x12, x15, #0x1
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, x13, #0x1
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    add x10, x11, x12
+; CHECK-GI-NEXT:    add x8, x8, x14
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    lsl x10, x8, #60
+; CHECK-GI-NEXT:    lsr x8, x8, #4
+; CHECK-GI-NEXT:    orr x9, x10, x9, lsr #4
+; CHECK-GI-NEXT:    mov w10, #100 // =0x64
+; CHECK-GI-NEXT:    umulh x11, x9, x10
+; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    madd x8, x8, x10, x11
+; CHECK-GI-NEXT:    subs x0, x0, x9
+; CHECK-GI-NEXT:    sbc x1, x1, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i128 %a, 100
+  ret i128 %s
+}
+
+define <2 x i8> @sv2i8_7(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: sv2i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v1.2s, v0.2s, #24
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v1.2s, #24
+; CHECK-SD-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #32
+; CHECK-SD-NEXT:    ssra v2.2s, v1.2s, #24
+; CHECK-SD-NEXT:    sshr v1.2s, v2.2s, #2
+; CHECK-SD-NEXT:    usra v1.2s, v2.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v3.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    smov w11, v1.h[1]
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    smov w10, v1.h[0]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i8> %d, <i8 7, i8 7>
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @sv2i8_100(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: sv2i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    smov w11, v1.h[1]
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    smov w10, v1.h[0]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i8> %d, <i8 100, i8 100>
+  ret <2 x i8> %s
+}
+
+define <3 x i8> @sv3i8_7(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: sv3i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxtb x8, w0
+; CHECK-SD-NEXT:    mov x9, #-56173 // =0xffffffffffff2493
+; CHECK-SD-NEXT:    sxtb x10, w1
+; CHECK-SD-NEXT:    sxtb x11, w2
+; CHECK-SD-NEXT:    movk x9, #37449, lsl #16
+; CHECK-SD-NEXT:    sxtb w12, w1
+; CHECK-SD-NEXT:    smull x8, w8, w9
+; CHECK-SD-NEXT:    sxtb w13, w0
+; CHECK-SD-NEXT:    smull x10, w10, w9
+; CHECK-SD-NEXT:    smull x9, w11, w9
+; CHECK-SD-NEXT:    sxtb w11, w2
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    lsr x10, x10, #32
+; CHECK-SD-NEXT:    lsr x9, x9, #32
+; CHECK-SD-NEXT:    add w8, w8, w13
+; CHECK-SD-NEXT:    add w10, w10, w12
+; CHECK-SD-NEXT:    asr w14, w8, #2
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    asr w15, w10, #2
+; CHECK-SD-NEXT:    asr w16, w9, #2
+; CHECK-SD-NEXT:    add w8, w14, w8, lsr #31
+; CHECK-SD-NEXT:    add w10, w15, w10, lsr #31
+; CHECK-SD-NEXT:    add w9, w16, w9, lsr #31
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    sub w10, w10, w10, lsl #3
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    add w0, w13, w8
+; CHECK-SD-NEXT:    add w1, w12, w10
+; CHECK-SD-NEXT:    add w2, w11, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w11, w1
+; CHECK-GI-NEXT:    sxtb w13, w2
+; CHECK-GI-NEXT:    mov w9, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    sdiv w12, w11, w9
+; CHECK-GI-NEXT:    lsl w14, w10, #3
+; CHECK-GI-NEXT:    sub w10, w14, w10
+; CHECK-GI-NEXT:    sub w0, w8, w10
+; CHECK-GI-NEXT:    sdiv w9, w13, w9
+; CHECK-GI-NEXT:    lsl w15, w12, #3
+; CHECK-GI-NEXT:    sub w12, w15, w12
+; CHECK-GI-NEXT:    sub w1, w11, w12
+; CHECK-GI-NEXT:    lsl w16, w9, #3
+; CHECK-GI-NEXT:    sub w9, w16, w9
+; CHECK-GI-NEXT:    sub w2, w13, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i8> %d, <i8 7, i8 7, i8 7>
+  ret <3 x i8> %s
+}
+
+define <3 x i8> @sv3i8_100(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: sv3i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxtb x8, w0
+; CHECK-SD-NEXT:    mov w9, #34079 // =0x851f
+; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    sxtb x10, w1
+; CHECK-SD-NEXT:    movk w9, #20971, lsl #16
+; CHECK-SD-NEXT:    sxtb x11, w2
+; CHECK-SD-NEXT:    sxtb w12, w0
+; CHECK-SD-NEXT:    smull x8, w8, w9
+; CHECK-SD-NEXT:    smull x10, w10, w9
+; CHECK-SD-NEXT:    smull x9, w11, w9
+; CHECK-SD-NEXT:    mov w11, #100 // =0x64
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    asr x10, x10, #37
+; CHECK-SD-NEXT:    asr x9, x9, #37
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    add w10, w10, w10, lsr #31
+; CHECK-SD-NEXT:    add w9, w9, w9, lsr #31
+; CHECK-SD-NEXT:    msub w0, w8, w11, w12
+; CHECK-SD-NEXT:    sxtb w8, w1
+; CHECK-SD-NEXT:    msub w1, w10, w11, w8
+; CHECK-SD-NEXT:    sxtb w8, w2
+; CHECK-SD-NEXT:    msub w2, w9, w11, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w11, w1
+; CHECK-GI-NEXT:    sxtb w13, w2
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    sdiv w12, w11, w9
+; CHECK-GI-NEXT:    msub w0, w10, w9, w8
+; CHECK-GI-NEXT:    sdiv w14, w13, w9
+; CHECK-GI-NEXT:    msub w1, w12, w9, w11
+; CHECK-GI-NEXT:    msub w2, w14, w9, w13
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i8> %d, <i8 100, i8 100, i8 100>
+  ret <3 x i8> %s
+}
+
+define <4 x i8> @sv4i8_7(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: sv4i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    mov x8, #-56173 // =0xffffffffffff2493
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    smov x9, v0.h[0]
+; CHECK-SD-NEXT:    smov x10, v0.h[1]
+; CHECK-SD-NEXT:    smov w11, v0.h[0]
+; CHECK-SD-NEXT:    smov x12, v0.h[2]
+; CHECK-SD-NEXT:    smov w13, v0.h[1]
+; CHECK-SD-NEXT:    smov x14, v0.h[3]
+; CHECK-SD-NEXT:    smov w16, v0.h[2]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x12, w12, w8
+; CHECK-SD-NEXT:    lsr x9, x9, #32
+; CHECK-SD-NEXT:    smull x8, w14, w8
+; CHECK-SD-NEXT:    smov w14, v0.h[3]
+; CHECK-SD-NEXT:    lsr x10, x10, #32
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    lsr x12, x12, #32
+; CHECK-SD-NEXT:    asr w15, w9, #2
+; CHECK-SD-NEXT:    add w10, w10, w13
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    asr w17, w10, #2
+; CHECK-SD-NEXT:    add w12, w12, w16
+; CHECK-SD-NEXT:    add w9, w15, w9, lsr #31
+; CHECK-SD-NEXT:    asr w15, w12, #2
+; CHECK-SD-NEXT:    add w8, w8, w14
+; CHECK-SD-NEXT:    add w10, w17, w10, lsr #31
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    sub w10, w10, w10, lsl #3
+; CHECK-SD-NEXT:    add w9, w11, w9
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w13, w10
+; CHECK-SD-NEXT:    add w9, w15, w12, lsr #31
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    asr w10, w8, #2
+; CHECK-SD-NEXT:    add w9, w16, w9
+; CHECK-SD-NEXT:    add w8, w10, w8, lsr #31
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w8, w14, w8
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v3.4h, #7
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    mov v3.d[1], v2.d[0]
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w9, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w9
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i8> %d, <i8 7, i8 7, i8 7, i8 7>
+  ret <4 x i8> %s
+}
+
+define <4 x i8> @sv4i8_100(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: sv4i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    sshr v1.4h, v0.4h, #8
+; CHECK-SD-NEXT:    smov x9, v1.h[0]
+; CHECK-SD-NEXT:    smov x10, v1.h[1]
+; CHECK-SD-NEXT:    smov x11, v1.h[2]
+; CHECK-SD-NEXT:    smov w12, v1.h[0]
+; CHECK-SD-NEXT:    smov x13, v1.h[3]
+; CHECK-SD-NEXT:    smov w15, v1.h[1]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x11, w11, w8
+; CHECK-SD-NEXT:    asr x9, x9, #37
+; CHECK-SD-NEXT:    smull x8, w13, w8
+; CHECK-SD-NEXT:    asr x10, x10, #37
+; CHECK-SD-NEXT:    add w9, w9, w9, lsr #31
+; CHECK-SD-NEXT:    asr x11, x11, #37
+; CHECK-SD-NEXT:    add w10, w10, w10, lsr #31
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    msub w9, w9, w14, w12
+; CHECK-SD-NEXT:    msub w10, w10, w14, w15
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w9, w11, w11, lsr #31
+; CHECK-SD-NEXT:    smov w11, v1.h[2]
+; CHECK-SD-NEXT:    msub w9, w9, w14, w11
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    smov w10, v1.h[3]
+; CHECK-SD-NEXT:    msub w8, w8, w14, w10
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v3.4h, #100
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    mov v3.d[1], v2.d[0]
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w9, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w9
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i8> %d, <i8 100, i8 100, i8 100, i8 100>
+  ret <4 x i8> %s
+}
+
+define <8 x i8> @sv8i8_7(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-SD-LABEL: sv8i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #147
+; CHECK-SD-NEXT:    movi v2.8b, #7
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-SD-NEXT:    add v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    sshr v1.8b, v1.8b, #2
+; CHECK-SD-NEXT:    usra v1.8b, v1.8b, #7
+; CHECK-SD-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v4.8b, #7
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v5.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <8 x i8> %s
+}
+
+define <8 x i8> @sv8i8_100(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-SD-LABEL: sv8i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #41
+; CHECK-SD-NEXT:    movi v2.8b, #100
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-SD-NEXT:    sshr v1.8b, v1.8b, #4
+; CHECK-SD-NEXT:    usra v1.8b, v1.8b, #7
+; CHECK-SD-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v4.8b, #100
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v5.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @sv16i8_7(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-SD-LABEL: sv16i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #147
+; CHECK-SD-NEXT:    smull2 v2.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    movi v2.16b, #7
+; CHECK-SD-NEXT:    add v1.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    sshr v1.16b, v1.16b, #2
+; CHECK-SD-NEXT:    usra v1.16b, v1.16b, #7
+; CHECK-SD-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv16i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v16.8b, #7
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s2
+; CHECK-GI-NEXT:    fmov w17, s0
+; CHECK-GI-NEXT:    fmov w2, s3
+; CHECK-GI-NEXT:    mov w14, v2.s[1]
+; CHECK-GI-NEXT:    mov w18, v0.s[1]
+; CHECK-GI-NEXT:    mov w3, v3.s[1]
+; CHECK-GI-NEXT:    mov w15, v2.s[2]
+; CHECK-GI-NEXT:    mov w0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w11, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[1]
+; CHECK-GI-NEXT:    mov w4, v3.s[2]
+; CHECK-GI-NEXT:    mov w16, v2.s[3]
+; CHECK-GI-NEXT:    mov w1, v0.s[3]
+; CHECK-GI-NEXT:    mov w5, v3.s[3]
+; CHECK-GI-NEXT:    sshll v17.4s, v16.4h, #0
+; CHECK-GI-NEXT:    sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s4, w11
+; CHECK-GI-NEXT:    sdiv w17, w17, w8
+; CHECK-GI-NEXT:    fmov s5, w13
+; CHECK-GI-NEXT:    sdiv w2, w2, w8
+; CHECK-GI-NEXT:    fmov s6, w17
+; CHECK-GI-NEXT:    sdiv w12, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[2]
+; CHECK-GI-NEXT:    fmov s7, w2
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v4.s[1], w12
+; CHECK-GI-NEXT:    sdiv w18, w18, w8
+; CHECK-GI-NEXT:    mov v5.s[1], w14
+; CHECK-GI-NEXT:    sdiv w3, w3, w8
+; CHECK-GI-NEXT:    mov v6.s[1], w18
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[3]
+; CHECK-GI-NEXT:    mov v7.s[1], w3
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    sdiv w0, w0, w8
+; CHECK-GI-NEXT:    mov v5.s[2], w15
+; CHECK-GI-NEXT:    sdiv w4, w4, w8
+; CHECK-GI-NEXT:    mov v6.s[2], w0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    mov v7.s[2], w4
+; CHECK-GI-NEXT:    sdiv w16, w16, w8
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mls v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    sdiv w1, w1, w8
+; CHECK-GI-NEXT:    mov v5.s[3], w16
+; CHECK-GI-NEXT:    mls v2.4s, v5.4s, v16.4s
+; CHECK-GI-NEXT:    sdiv w8, w5, w8
+; CHECK-GI-NEXT:    mov v6.s[3], w1
+; CHECK-GI-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    mls v0.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT:    mov v7.s[3], w8
+; CHECK-GI-NEXT:    mls v3.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-GI-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <16 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @sv16i8_100(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-SD-LABEL: sv16i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #41
+; CHECK-SD-NEXT:    smull2 v2.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    movi v2.16b, #100
+; CHECK-SD-NEXT:    sshr v1.16b, v1.16b, #4
+; CHECK-SD-NEXT:    usra v1.16b, v1.16b, #7
+; CHECK-SD-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv16i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v16.8b, #100
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s2
+; CHECK-GI-NEXT:    fmov w17, s0
+; CHECK-GI-NEXT:    fmov w2, s3
+; CHECK-GI-NEXT:    mov w14, v2.s[1]
+; CHECK-GI-NEXT:    mov w18, v0.s[1]
+; CHECK-GI-NEXT:    mov w3, v3.s[1]
+; CHECK-GI-NEXT:    mov w15, v2.s[2]
+; CHECK-GI-NEXT:    mov w0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w11, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[1]
+; CHECK-GI-NEXT:    mov w4, v3.s[2]
+; CHECK-GI-NEXT:    mov w16, v2.s[3]
+; CHECK-GI-NEXT:    mov w1, v0.s[3]
+; CHECK-GI-NEXT:    mov w5, v3.s[3]
+; CHECK-GI-NEXT:    sshll v17.4s, v16.4h, #0
+; CHECK-GI-NEXT:    sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s4, w11
+; CHECK-GI-NEXT:    sdiv w17, w17, w8
+; CHECK-GI-NEXT:    fmov s5, w13
+; CHECK-GI-NEXT:    sdiv w2, w2, w8
+; CHECK-GI-NEXT:    fmov s6, w17
+; CHECK-GI-NEXT:    sdiv w12, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[2]
+; CHECK-GI-NEXT:    fmov s7, w2
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v4.s[1], w12
+; CHECK-GI-NEXT:    sdiv w18, w18, w8
+; CHECK-GI-NEXT:    mov v5.s[1], w14
+; CHECK-GI-NEXT:    sdiv w3, w3, w8
+; CHECK-GI-NEXT:    mov v6.s[1], w18
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[3]
+; CHECK-GI-NEXT:    mov v7.s[1], w3
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    sdiv w0, w0, w8
+; CHECK-GI-NEXT:    mov v5.s[2], w15
+; CHECK-GI-NEXT:    sdiv w4, w4, w8
+; CHECK-GI-NEXT:    mov v6.s[2], w0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    mov v7.s[2], w4
+; CHECK-GI-NEXT:    sdiv w16, w16, w8
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mls v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    sdiv w1, w1, w8
+; CHECK-GI-NEXT:    mov v5.s[3], w16
+; CHECK-GI-NEXT:    mls v2.4s, v5.4s, v16.4s
+; CHECK-GI-NEXT:    sdiv w8, w5, w8
+; CHECK-GI-NEXT:    mov v6.s[3], w1
+; CHECK-GI-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    mls v0.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT:    mov v7.s[3], w8
+; CHECK-GI-NEXT:    mls v3.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-GI-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <16 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <16 x i8> %s
+}
+
+define <2 x i8> @uv2i8_7(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: uv2i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-GI-NEXT:    movi v2.2s, #37
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    and v1.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    neg v3.8b, v3.8b
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    mov w9, v2.s[1]
+; CHECK-GI-NEXT:    mov v2.b[1], w9
+; CHECK-GI-NEXT:    ushl v2.8b, v2.8b, v3.8b
+; CHECK-GI-NEXT:    umov w8, v2.b[0]
+; CHECK-GI-NEXT:    umov w9, v2.b[1]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    add v1.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[1]
+; CHECK-GI-NEXT:    mov v2.b[1], w8
+; CHECK-GI-NEXT:    mov v1.b[1], w9
+; CHECK-GI-NEXT:    neg v2.8b, v2.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    movi v2.2s, #7
+; CHECK-GI-NEXT:    umov w8, v1.b[0]
+; CHECK-GI-NEXT:    umov w9, v1.b[1]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i8> %d, <i8 7, i8 7>
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @uv2i8_100(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: uv2i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-GI-NEXT:    movi v2.2s, #41
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    and v1.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #4 // =0x4
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    neg v2.8b, v3.8b
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    movi v2.2s, #100
+; CHECK-GI-NEXT:    umov w8, v1.b[0]
+; CHECK-GI-NEXT:    umov w9, v1.b[1]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i8> %d, <i8 100, i8 100>
+  ret <2 x i8> %s
+}
+
+define <3 x i8> @uv3i8_7(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: uv3i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    and w10, w1, #0xff
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    and w12, w2, #0xff
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w11, w11, w11, lsl #3
+; CHECK-SD-NEXT:    sub w13, w13, w13, lsl #3
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w9, w11
+; CHECK-SD-NEXT:    add w1, w10, w13
+; CHECK-SD-NEXT:    add w2, w12, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    and w8, w0, #0xff
+; CHECK-GI-NEXT:    mov w10, #37 // =0x25
+; CHECK-GI-NEXT:    and w9, w1, #0xff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    mov v1.h[1], w9
+; CHECK-GI-NEXT:    mov v2.h[1], w10
+; CHECK-GI-NEXT:    and w9, w2, #0xff
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov v0.h[1], w1
+; CHECK-GI-NEXT:    mov v1.h[2], w9
+; CHECK-GI-NEXT:    mov v2.h[2], w10
+; CHECK-GI-NEXT:    mov v3.h[2], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov v0.h[2], w2
+; CHECK-GI-NEXT:    mul v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    neg v2.4h, v3.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    neg v3.8b, v3.8b
+; CHECK-GI-NEXT:    ushl v2.8b, v2.8b, v3.8b
+; CHECK-GI-NEXT:    mov b3, v2.b[1]
+; CHECK-GI-NEXT:    mov b4, v2.b[2]
+; CHECK-GI-NEXT:    fmov w8, s3
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    add v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    neg v2.8b, v3.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    mov b2, v1.b[1]
+; CHECK-GI-NEXT:    mov b3, v1.b[2]
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v1.h[1], w9
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s3
+; CHECK-GI-NEXT:    mov v1.h[2], w9
+; CHECK-GI-NEXT:    mov v2.h[2], w8
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    umov w0, v0.h[0]
+; CHECK-GI-NEXT:    umov w1, v0.h[1]
+; CHECK-GI-NEXT:    umov w2, v0.h[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i8> %d, <i8 7, i8 7, i8 7>
+  ret <3 x i8> %s
+}
+
+define <3 x i8> @uv3i8_100(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: uv3i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    and w10, w1, #0xff
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    and w12, w2, #0xff
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    msub w0, w11, w14, w9
+; CHECK-SD-NEXT:    msub w1, w13, w14, w10
+; CHECK-SD-NEXT:    msub w2, w8, w14, w12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    and w8, w0, #0xff
+; CHECK-GI-NEXT:    mov w10, #41 // =0x29
+; CHECK-GI-NEXT:    and w9, w1, #0xff
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v0.h[1], w9
+; CHECK-GI-NEXT:    mov v1.h[1], w10
+; CHECK-GI-NEXT:    and w9, w2, #0xff
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov v0.h[2], w9
+; CHECK-GI-NEXT:    mov v1.h[2], w10
+; CHECK-GI-NEXT:    mov v2.h[2], w8
+; CHECK-GI-NEXT:    mov w8, #4 // =0x4
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    neg v1.4h, v2.4h
+; CHECK-GI-NEXT:    ushl v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    neg v1.8b, v3.8b
+; CHECK-GI-NEXT:    fmov s3, w0
+; CHECK-GI-NEXT:    ushl v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov v3.h[1], w1
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov v3.h[2], w2
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.h[1], w9
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    mov v0.h[2], w9
+; CHECK-GI-NEXT:    mov v1.h[2], w8
+; CHECK-GI-NEXT:    mls v3.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umov w0, v3.h[0]
+; CHECK-GI-NEXT:    umov w1, v3.h[1]
+; CHECK-GI-NEXT:    umov w2, v3.h[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i8> %d, <i8 100, i8 100, i8 100>
+  ret <3 x i8> %s
+}
+
+define <4 x i8> @uv4i8_7(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: uv4i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    umov w13, v0.h[2]
+; CHECK-SD-NEXT:    umov w15, v0.h[3]
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x12, w10, w8
+; CHECK-SD-NEXT:    umull x14, w13, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    umull x8, w15, w8
+; CHECK-SD-NEXT:    lsr x12, x12, #32
+; CHECK-SD-NEXT:    sub w11, w11, w11, lsl #3
+; CHECK-SD-NEXT:    sub w12, w12, w12, lsl #3
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w10, w12
+; CHECK-SD-NEXT:    lsr x9, x14, #32
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    add w8, w15, w8
+; CHECK-SD-NEXT:    add w9, w13, w9
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #37 // =0x25
+; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.b[1], w8
+; CHECK-GI-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    mul v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    ushr v2.4h, v1.4h, #8
+; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v2.4h
+; CHECK-GI-NEXT:    neg v3.8b, v3.8b
+; CHECK-GI-NEXT:    mov v4.b[1], w8
+; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    mov v4.b[2], w8
+; CHECK-GI-NEXT:    ushl v2.8b, v2.8b, v3.8b
+; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mov v4.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    usra v2.4h, v1.4h, #8
+; CHECK-GI-NEXT:    uzp1 v1.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    neg v2.8b, v4.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    dup v2.4h, w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i8> %d, <i8 7, i8 7, i8 7, i8 7>
+  ret <4 x i8> %s
+}
+
+define <4 x i8> @uv4i8_100(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: uv4i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    umov w12, v0.h[2]
+; CHECK-SD-NEXT:    umov w15, v0.h[3]
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    msub w9, w11, w14, w9
+; CHECK-SD-NEXT:    umull x11, w12, w8
+; CHECK-SD-NEXT:    msub w10, w13, w14, w10
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    umull x8, w15, w8
+; CHECK-SD-NEXT:    lsr x9, x11, #32
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    msub w9, w9, w14, w12
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    msub w8, w8, w14, w15
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #41 // =0x29
+; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.b[1], w8
+; CHECK-GI-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #4 // =0x4
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mul v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    neg v2.8b, v3.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    dup v2.4h, w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i8> %d, <i8 100, i8 100, i8 100, i8 100>
+  ret <4 x i8> %s
+}
+
+define <8 x i8> @uv8i8_7(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-SD-LABEL: uv8i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #37
+; CHECK-SD-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-SD-NEXT:    sub v2.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-SD-NEXT:    shrn v2.8b, v2.8h, #1
+; CHECK-SD-NEXT:    add v1.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT:    movi v2.8b, #7
+; CHECK-SD-NEXT:    ushr v1.8b, v1.8b, #2
+; CHECK-SD-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv8i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.8b, #37
+; CHECK-GI-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-GI-NEXT:    sub v2.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    usra v1.8b, v2.8b, #1
+; CHECK-GI-NEXT:    movi v2.8b, #7
+; CHECK-GI-NEXT:    ushr v1.8b, v1.8b, #2
+; CHECK-GI-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <8 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <8 x i8> %s
+}
+
+define <8 x i8> @uv8i8_100(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-LABEL: uv8i8_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.8b, #41
+; CHECK-NEXT:    movi v2.8b, #100
+; CHECK-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-NEXT:    ushr v1.8b, v1.8b, #4
+; CHECK-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-NEXT:    ret
+entry:
+  %s = urem <8 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @uv16i8_7(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-LABEL: uv16i8_7:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.16b, #37
+; CHECK-NEXT:    umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    sub v2.16b, v0.16b, v1.16b
+; CHECK-NEXT:    usra v1.16b, v2.16b, #1
+; CHECK-NEXT:    movi v2.16b, #7
+; CHECK-NEXT:    ushr v1.16b, v1.16b, #2
+; CHECK-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = urem <16 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @uv16i8_100(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-LABEL: uv16i8_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.16b, #41
+; CHECK-NEXT:    umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    movi v2.16b, #100
+; CHECK-NEXT:    ushr v1.16b, v1.16b, #4
+; CHECK-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = urem <16 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <16 x i8> %s
+}
+
+define <2 x i16> @sv2i16_7(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: sv2i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v1.2s, v0.2s, #16
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v1.2s, #16
+; CHECK-SD-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #32
+; CHECK-SD-NEXT:    ssra v2.2s, v1.2s, #16
+; CHECK-SD-NEXT:    sshr v1.2s, v2.2s, #2
+; CHECK-SD-NEXT:    usra v1.2s, v2.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v3.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i16> %d, <i16 7, i16 7>
+  ret <2 x i16> %s
+}
+
+define <2 x i16> @sv2i16_100(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: sv2i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i16> %d, <i16 100, i16 100>
+  ret <2 x i16> %s
+}
+
+define <3 x i16> @sv3i16_7(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: sv3i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    smov x9, v0.h[0]
+; CHECK-SD-NEXT:    mov x8, #-56173 // =0xffffffffffff2493
+; CHECK-SD-NEXT:    smov x10, v0.h[1]
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    smov w12, v0.h[0]
+; CHECK-SD-NEXT:    smov x11, v0.h[2]
+; CHECK-SD-NEXT:    smov w13, v0.h[1]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x8, w11, w8
+; CHECK-SD-NEXT:    smov w11, v0.h[2]
+; CHECK-SD-NEXT:    lsr x9, x9, #32
+; CHECK-SD-NEXT:    lsr x10, x10, #32
+; CHECK-SD-NEXT:    add w9, w9, w12
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    asr w14, w9, #2
+; CHECK-SD-NEXT:    add w10, w10, w13
+; CHECK-SD-NEXT:    asr w15, w10, #2
+; CHECK-SD-NEXT:    add w8, w8, w11
+; CHECK-SD-NEXT:    add w9, w14, w9, lsr #31
+; CHECK-SD-NEXT:    asr w14, w8, #2
+; CHECK-SD-NEXT:    add w10, w15, w10, lsr #31
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    add w8, w14, w8, lsr #31
+; CHECK-SD-NEXT:    sub w10, w10, w10, lsl #3
+; CHECK-SD-NEXT:    add w9, w12, w9
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w13, w10
+; CHECK-SD-NEXT:    add w8, w11, w8
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    smov w9, v0.h[0]
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    smov w11, v0.h[1]
+; CHECK-GI-NEXT:    smov w13, v0.h[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    lsl w14, w10, #3
+; CHECK-GI-NEXT:    sub w10, w14, w10
+; CHECK-GI-NEXT:    sub w9, w9, w10
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w8, w13, w8
+; CHECK-GI-NEXT:    lsl w15, w12, #3
+; CHECK-GI-NEXT:    sub w10, w15, w12
+; CHECK-GI-NEXT:    sub w10, w11, w10
+; CHECK-GI-NEXT:    mov v0.h[1], w10
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w8, w13, w8
+; CHECK-GI-NEXT:    mov v0.h[2], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i16> %d, <i16 7, i16 7, i16 7>
+  ret <3 x i16> %s
+}
+
+define <3 x i16> @sv3i16_100(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: sv3i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    smov x9, v0.h[0]
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    smov x10, v0.h[1]
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    smov x11, v0.h[2]
+; CHECK-SD-NEXT:    mov w12, #100 // =0x64
+; CHECK-SD-NEXT:    smov w13, v0.h[1]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x8, w11, w8
+; CHECK-SD-NEXT:    smov w11, v0.h[0]
+; CHECK-SD-NEXT:    asr x9, x9, #37
+; CHECK-SD-NEXT:    asr x10, x10, #37
+; CHECK-SD-NEXT:    add w9, w9, w9, lsr #31
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    add w10, w10, w10, lsr #31
+; CHECK-SD-NEXT:    msub w9, w9, w12, w11
+; CHECK-SD-NEXT:    smov w11, v0.h[2]
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    msub w10, w10, w12, w13
+; CHECK-SD-NEXT:    msub w8, w8, w12, w11
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    smov w9, v0.h[0]
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    smov w11, v0.h[1]
+; CHECK-GI-NEXT:    smov w13, v0.h[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    msub w9, w10, w8, w9
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w14, w13, w8
+; CHECK-GI-NEXT:    msub w10, w12, w8, w11
+; CHECK-GI-NEXT:    mov v0.h[1], w10
+; CHECK-GI-NEXT:    msub w8, w14, w8, w13
+; CHECK-GI-NEXT:    mov v0.h[2], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i16> %d, <i16 100, i16 100, i16 100>
+  ret <3 x i16> %s
+}
+
+define <4 x i16> @sv4i16_7(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: sv4i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movi v2.4h, #7
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #17
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    usra v1.4h, v1.4h, #15
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v2.4h, #7
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i16> %d, <i16 7, i16 7, i16 7, i16 7>
+  ret <4 x i16> %s
+}
+
+define <4 x i16> @sv4i16_100(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: sv4i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    movi v2.4h, #100
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #19
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    usra v1.4h, v1.4h, #15
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v2.4h, #100
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i16> %d, <i16 100, i16 100, i16 100, i16 100>
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @sv8i16_7(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: sv8i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    movi v2.8h, #7
+; CHECK-SD-NEXT:    sshr v1.8h, v1.8h, #1
+; CHECK-SD-NEXT:    usra v1.8h, v1.8h, #15
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v4.4h, #7
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i16> %d, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @sv8i16_100(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: sv8i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    movi v2.8h, #100
+; CHECK-SD-NEXT:    sshr v1.8h, v1.8h, #3
+; CHECK-SD-NEXT:    usra v1.8h, v1.8h, #15
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v4.4h, #100
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i16> %d, <i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100>
+  ret <8 x i16> %s
+}
+
+define <2 x i16> @uv2i16_7(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: uv2i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    mul v1.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    neg v3.4h, v3.4h
+; CHECK-GI-NEXT:    ushr v2.2s, v1.2s, #16
+; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT:    uzp1 v2.4h, v2.4h, v0.4h
+; CHECK-GI-NEXT:    ushl v2.4h, v2.4h, v3.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    usra v2.2s, v1.2s, #16
+; CHECK-GI-NEXT:    uzp1 v1.4h, v2.4h, v0.4h
+; CHECK-GI-NEXT:    neg v2.4h, v3.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    dup v2.2s, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i16> %d, <i16 7, i16 7>
+  ret <2 x i16> %s
+}
+
+define <2 x i16> @uv2i16_100(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: uv2i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    uzp1 v2.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    neg v1.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    ushl v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    ushll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #16
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    dup v2.2s, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i16> %d, <i16 100, i16 100>
+  ret <2 x i16> %s
+}
+
+define <3 x i16> @uv3i16_7(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: uv3i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    umov w12, v0.h[2]
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    sub w11, w11, w11, lsl #3
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w13, w13, w13, lsl #3
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w10, w13
+; CHECK-SD-NEXT:    add w8, w12, w8
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    umov w9, v0.h[0]
+; CHECK-GI-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-GI-NEXT:    umov w10, v0.h[1]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    umov w11, v0.h[2]
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov w9, #16 // =0x10
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    mov v3.s[1], w9
+; CHECK-GI-NEXT:    mov v2.s[2], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v3.s[2], w9
+; CHECK-GI-NEXT:    mov w9, #2 // =0x2
+; CHECK-GI-NEXT:    mul v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    neg v2.4s, v3.4s
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    mov v2.h[1], w9
+; CHECK-GI-NEXT:    mov v3.h[2], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    sub v4.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    neg v3.4h, v3.4h
+; CHECK-GI-NEXT:    ushl v3.4h, v4.4h, v3.4h
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    mov v4.h[1], w8
+; CHECK-GI-NEXT:    add v1.4h, v3.4h, v1.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    mov v4.h[2], w8
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v4.4h
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i16> %d, <i16 7, i16 7, i16 7>
+  ret <3 x i16> %s
+}
+
+define <3 x i16> @uv3i16_100(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: uv3i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    umov w12, v0.h[2]
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    msub w9, w11, w14, w9
+; CHECK-SD-NEXT:    lsr x11, x13, #32
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    msub w10, w11, w14, w10
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    msub w8, w8, w14, w12
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    mov w11, #5243 // =0x147b
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov s2, w11
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    mov v2.s[1], w11
+; CHECK-GI-NEXT:    mov v1.h[2], w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    neg v1.4h, v1.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umov w8, v1.h[0]
+; CHECK-GI-NEXT:    umov w9, v1.h[1]
+; CHECK-GI-NEXT:    umov w10, v1.h[2]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov w8, #16 // =0x10
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v1.s[2], w10
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    mul v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    mov v4.h[1], w8
+; CHECK-GI-NEXT:    neg v2.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v4.h[2], w8
+; CHECK-GI-NEXT:    mov v2.h[1], w9
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    neg v3.4h, v4.4h
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v3.4h
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i16> %d, <i16 100, i16 100, i16 100>
+  ret <3 x i16> %s
+}
+
+define <4 x i16> @uv4i16_7(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: uv4i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    shrn v1.4h, v1.4s, #16
+; CHECK-SD-NEXT:    sub v2.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    shrn v2.4h, v2.4s, #1
+; CHECK-SD-NEXT:    add v1.4h, v2.4h, v1.4h
+; CHECK-SD-NEXT:    movi v2.4h, #7
+; CHECK-SD-NEXT:    ushr v1.4h, v1.4h, #2
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI52_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI52_0]
+; CHECK-GI-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    shrn v1.4h, v1.4s, #16
+; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    usra v1.4h, v2.4h, #1
+; CHECK-GI-NEXT:    movi v2.4h, #7
+; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #2
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i16> %d, <i16 7, i16 7, i16 7, i16 7>
+  ret <4 x i16> %s
+}
+
+define <4 x i16> @uv4i16_100(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: uv4i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    ushr v2.4h, v0.4h, #2
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    umull v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    movi v2.4h, #100
+; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #17
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI53_0
+; CHECK-GI-NEXT:    ushr v1.4h, v0.4h, #2
+; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI53_0]
+; CHECK-GI-NEXT:    umull v1.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    movi v2.4h, #100
+; CHECK-GI-NEXT:    shrn v1.4h, v1.4s, #16
+; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #1
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i16> %d, <i16 100, i16 100, i16 100, i16 100>
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @uv8i16_7(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: uv8i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    sub v2.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT:    usra v1.8h, v2.8h, #1
+; CHECK-SD-NEXT:    movi v2.8h, #7
+; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #2
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv8i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI54_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI54_0]
+; CHECK-GI-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    sub v2.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    usra v1.8h, v2.8h, #1
+; CHECK-GI-NEXT:    movi v2.8h, #7
+; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #2
+; CHECK-GI-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <8 x i16> %d, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @uv8i16_100(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: uv8i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    ushr v2.8h, v0.8h, #2
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    umull2 v3.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    umull v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    movi v2.8h, #100
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v3.8h
+; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #1
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv8i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI55_0
+; CHECK-GI-NEXT:    ushr v1.8h, v0.8h, #2
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI55_0]
+; CHECK-GI-NEXT:    umull2 v3.4s, v1.8h, v2.8h
+; CHECK-GI-NEXT:    umull v1.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    movi v2.8h, #100
+; CHECK-GI-NEXT:    uzp2 v1.8h, v1.8h, v3.8h
+; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #1
+; CHECK-GI-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <8 x i16> %d, <i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100>
+  ret <8 x i16> %s
+}
+
+define <2 x i32> @sv2i32_7(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: sv2i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    add v1.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    sshr v2.2s, v1.2s, #2
+; CHECK-SD-NEXT:    usra v2.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v2.2s, v3.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    movi v2.2s, #7
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i32> %d, <i32 7, i32 7>
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @sv2i32_100(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: sv2i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    movi v2.2s, #100
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i32> %d, <i32 100, i32 100>
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @sv3i32_7(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: sv3i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull x8, w9, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    add w8, w8, w9
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    asr w10, w8, #2
+; CHECK-SD-NEXT:    add w8, w10, w8, lsr #31
+; CHECK-SD-NEXT:    add v1.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    sshr v2.2s, v1.2s, #2
+; CHECK-SD-NEXT:    add w8, w9, w8
+; CHECK-SD-NEXT:    usra v2.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v2.2s, v3.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov s0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    fmov w11, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    lsl w14, w10, #3
+; CHECK-GI-NEXT:    sub w10, w14, w10
+; CHECK-GI-NEXT:    sub w9, w9, w10
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w8, w13, w8
+; CHECK-GI-NEXT:    lsl w15, w12, #3
+; CHECK-GI-NEXT:    sub w10, w15, w12
+; CHECK-GI-NEXT:    sub w10, w11, w10
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w8, w13, w8
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i32> %d, <i32 7, i32 7, i32 7>
+  ret <3 x i32> %s
+}
+
+define <3 x i32> @sv3i32_100(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: sv3i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    mov w10, #100 // =0x64
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull x8, w9, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    msub w8, w8, w10, w9
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov s0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    fmov w11, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    msub w9, w10, w8, w9
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w14, w13, w8
+; CHECK-GI-NEXT:    msub w10, w12, w8, w11
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    msub w8, w14, w8, w13
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i32> %d, <i32 100, i32 100, i32 100>
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @sv4i32_7(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: sv4i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.4s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    smull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    sshr v2.4s, v1.4s, #2
+; CHECK-SD-NEXT:    usra v2.4s, v1.4s, #31
+; CHECK-SD-NEXT:    mls v0.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    movi v2.4s, #7
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i32> %d, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @sv4i32_100(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: sv4i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v3.4s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    smull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    sshr v2.4s, v1.4s, #5
+; CHECK-SD-NEXT:    usra v2.4s, v1.4s, #31
+; CHECK-SD-NEXT:    mls v0.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    movi v2.4s, #100
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i32> %d, <i32 100, i32 100, i32 100, i32 100>
+  ret <4 x i32> %s
+}
+
+define <2 x i32> @uv2i32_7(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: uv2i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #1
+; CHECK-SD-NEXT:    add v1.2s, v2.2s, v1.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    ushr v1.2s, v1.2s, #2
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI62_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI62_0]
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    usra v1.2s, v2.2s, #1
+; CHECK-GI-NEXT:    movi v2.2s, #7
+; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #2
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i32> %d, <i32 7, i32 7>
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @uv2i32_100(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: uv2i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ushr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI63_0
+; CHECK-GI-NEXT:    movi v2.2s, #100
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI63_0]
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #5
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i32> %d, <i32 100, i32 100>
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @uv3i32_7(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: uv3i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull x8, w9, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w10, w9, w8
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    add w8, w8, w10, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    add w8, w9, w8
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #1
+; CHECK-SD-NEXT:    add v1.2s, v2.2s, v1.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    ushr v1.2s, v1.2s, #2
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    adrp x8, .LCPI64_0
+; CHECK-GI-NEXT:    mov w9, #18725 // =0x4925
+; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI64_0]
+; CHECK-GI-NEXT:    mov w8, v0.s[2]
+; CHECK-GI-NEXT:    movk w9, #9362, lsl #16
+; CHECK-GI-NEXT:    mov w10, #1 // =0x1
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    umull x8, w8, w9
+; CHECK-GI-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    lsr x8, x8, #32
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #32
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov w9, #2 // =0x2
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mov v3.s[1], w9
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v2.s[2], w10
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov v3.s[2], w9
+; CHECK-GI-NEXT:    sub v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    neg v2.4s, v2.4s
+; CHECK-GI-NEXT:    ushl v2.4s, v4.4s, v2.4s
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    neg v2.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    mov v4.s[2], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v4.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i32> %d, <i32 7, i32 7, i32 7>
+  ret <3 x i32> %s
+}
+
+define <3 x i32> @uv3i32_100(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: uv3i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    mov w10, #100 // =0x64
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull x8, w9, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    lsr x8, x8, #37
+; CHECK-SD-NEXT:    msub w8, w8, w10, w9
+; CHECK-SD-NEXT:    ushr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    adrp x8, .LCPI65_0
+; CHECK-GI-NEXT:    mov w9, v0.s[2]
+; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI65_0]
+; CHECK-GI-NEXT:    mov w8, #5 // =0x5
+; CHECK-GI-NEXT:    mov w10, #34079 // =0x851f
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    movk w10, #20971, lsl #16
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    umull x9, w9, w10
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    lsr x8, x9, #32
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #32
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    fmov s1, w11
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    mov w10, #100 // =0x64
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v2.s[2], w10
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i32> %d, <i32 100, i32 100, i32 100>
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @uv4i32_7(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: uv4i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    sub v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    usra v1.4s, v2.4s, #1
+; CHECK-SD-NEXT:    movi v2.4s, #7
+; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #2
+; CHECK-SD-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI66_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI66_0]
+; CHECK-GI-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    sub v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    usra v1.4s, v2.4s, #1
+; CHECK-GI-NEXT:    movi v2.4s, #7
+; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #2
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i32> %d, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @uv4i32_100(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: uv4i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    movi v2.4s, #100
+; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #5
+; CHECK-SD-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI67_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI67_0]
+; CHECK-GI-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    movi v2.4s, #100
+; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #5
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i32> %d, <i32 100, i32 100, i32 100, i32 100>
+  ret <4 x i32> %s
+}
+
+define <2 x i64> @sv2i64_7(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: sv2i64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #18725 // =0x4925
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #16
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #32
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #48
+; CHECK-SD-NEXT:    smulh x11, x10, x8
+; CHECK-SD-NEXT:    smulh x8, x9, x8
+; CHECK-SD-NEXT:    asr x12, x11, #1
+; CHECK-SD-NEXT:    add x11, x12, x11, lsr #63
+; CHECK-SD-NEXT:    asr x13, x8, #1
+; CHECK-SD-NEXT:    sub x11, x11, x11, lsl #3
+; CHECK-SD-NEXT:    add x8, x13, x8, lsr #63
+; CHECK-SD-NEXT:    add x10, x10, x11
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    add x8, x9, x8
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    sdiv x9, x9, x8
+; CHECK-GI-NEXT:    sdiv x8, x10, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI68_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI68_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i64> %d, <i64 7, i64 7>
+  ret <2 x i64> %s
+}
+
+define <2 x i64> @sv2i64_100(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: sv2i64_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #55051 // =0xd70b
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #28835, lsl #16
+; CHECK-SD-NEXT:    movk x8, #2621, lsl #32
+; CHECK-SD-NEXT:    movk x8, #41943, lsl #48
+; CHECK-SD-NEXT:    smulh x11, x10, x8
+; CHECK-SD-NEXT:    smulh x8, x9, x8
+; CHECK-SD-NEXT:    add x11, x11, x10
+; CHECK-SD-NEXT:    asr x12, x11, #6
+; CHECK-SD-NEXT:    add x8, x8, x9
+; CHECK-SD-NEXT:    add x11, x12, x11, lsr #63
+; CHECK-SD-NEXT:    asr x13, x8, #6
+; CHECK-SD-NEXT:    mov w12, #100 // =0x64
+; CHECK-SD-NEXT:    msub x10, x11, x12, x10
+; CHECK-SD-NEXT:    add x8, x13, x8, lsr #63
+; CHECK-SD-NEXT:    msub x8, x8, x12, x9
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i64_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    sdiv x9, x9, x8
+; CHECK-GI-NEXT:    sdiv x8, x10, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI69_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI69_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i64> %d, <i64 100, i64 100>
+  ret <2 x i64> %s
+}
+
+define <2 x i64> @uv2i64_7(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: uv2i64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #32
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #48
+; CHECK-SD-NEXT:    umulh x11, x10, x8
+; CHECK-SD-NEXT:    umulh x8, x9, x8
+; CHECK-SD-NEXT:    sub x12, x10, x11
+; CHECK-SD-NEXT:    add x11, x11, x12, lsr #1
+; CHECK-SD-NEXT:    sub x12, x9, x8
+; CHECK-SD-NEXT:    lsr x11, x11, #2
+; CHECK-SD-NEXT:    add x8, x8, x12, lsr #1
+; CHECK-SD-NEXT:    sub x11, x11, x11, lsl #3
+; CHECK-SD-NEXT:    lsr x8, x8, #2
+; CHECK-SD-NEXT:    add x10, x10, x11
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    add x8, x9, x8
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
+; CHECK-GI-NEXT:    umulh x10, x10, x8
+; CHECK-GI-NEXT:    umulh x8, x9, x8
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI70_0
+; CHECK-GI-NEXT:    sub v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    usra v1.2d, v2.2d, #1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI70_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #2
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i64> %d, <i64 7, i64 7>
+  ret <2 x i64> %s
+}
+
+define <2 x i64> @uv2i64_100(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: uv2i64_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x8, #62915 // =0xf5c3
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #23592, lsl #16
+; CHECK-SD-NEXT:    movk x8, #49807, lsl #32
+; CHECK-SD-NEXT:    lsr x11, x10, #2
+; CHECK-SD-NEXT:    movk x8, #10485, lsl #48
+; CHECK-SD-NEXT:    lsr x12, x9, #2
+; CHECK-SD-NEXT:    umulh x11, x11, x8
+; CHECK-SD-NEXT:    umulh x8, x12, x8
+; CHECK-SD-NEXT:    mov w12, #100 // =0x64
+; CHECK-SD-NEXT:    lsr x11, x11, #2
+; CHECK-SD-NEXT:    msub x10, x11, x12, x10
+; CHECK-SD-NEXT:    lsr x8, x8, #2
+; CHECK-SD-NEXT:    msub x8, x8, x12, x9
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i64_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushr v1.2d, v0.2d, #2
+; CHECK-GI-NEXT:    mov x8, #62915 // =0xf5c3
+; CHECK-GI-NEXT:    movk x8, #23592, lsl #16
+; CHECK-GI-NEXT:    movk x8, #49807, lsl #32
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    movk x8, #10485, lsl #48
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    umulh x10, x10, x8
+; CHECK-GI-NEXT:    umulh x8, x9, x8
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI71_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI71_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #2
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i64> %d, <i64 100, i64 100>
+  ret <2 x i64> %s
+}
+
+define <2 x i128> @sv2i128_7(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: sv2i128_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i128_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-GI-NEXT:    .cfi_offset w30, -48
+; CHECK-GI-NEXT:    mov x19, x2
+; CHECK-GI-NEXT:    mov x20, x3
+; CHECK-GI-NEXT:    mov w2, #7 // =0x7
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x21, x0
+; CHECK-GI-NEXT:    mov x22, x1
+; CHECK-GI-NEXT:    mov x0, x19
+; CHECK-GI-NEXT:    mov x1, x20
+; CHECK-GI-NEXT:    mov w2, #7 // =0x7
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x2, x0
+; CHECK-GI-NEXT:    mov x3, x1
+; CHECK-GI-NEXT:    mov x0, x21
+; CHECK-GI-NEXT:    mov x1, x22
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i128> %d, <i128 7, i128 7>
+  ret <2 x i128> %s
+}
+
+define <2 x i128> @sv2i128_100(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: sv2i128_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i128_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-GI-NEXT:    .cfi_offset w30, -48
+; CHECK-GI-NEXT:    mov x19, x2
+; CHECK-GI-NEXT:    mov x20, x3
+; CHECK-GI-NEXT:    mov w2, #100 // =0x64
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x21, x0
+; CHECK-GI-NEXT:    mov x22, x1
+; CHECK-GI-NEXT:    mov x0, x19
+; CHECK-GI-NEXT:    mov x1, x20
+; CHECK-GI-NEXT:    mov w2, #100 // =0x64
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x2, x0
+; CHECK-GI-NEXT:    mov x3, x1
+; CHECK-GI-NEXT:    mov x0, x21
+; CHECK-GI-NEXT:    mov x1, x22
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i128> %d, <i128 100, i128 100>
+  ret <2 x i128> %s
+}
+
+define <2 x i128> @uv2i128_7(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: uv2i128_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i128_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x10, #18725 // =0x4925
+; CHECK-GI-NEXT:    mov x8, #9362 // =0x2492
+; CHECK-GI-NEXT:    sub x4, x0, x0
+; CHECK-GI-NEXT:    movk x10, #9362, lsl #16
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
+; CHECK-GI-NEXT:    umulh x18, x0, xzr
+; CHECK-GI-NEXT:    movk x10, #37449, lsl #32
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x10, #18724, lsl #48
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
+; CHECK-GI-NEXT:    mul x11, x1, x10
+; CHECK-GI-NEXT:    mul x12, x0, x8
+; CHECK-GI-NEXT:    umulh x13, x0, x10
+; CHECK-GI-NEXT:    mul x14, x1, x8
+; CHECK-GI-NEXT:    adds x11, x11, x12
+; CHECK-GI-NEXT:    umulh x15, x1, x10
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    cmn x11, x13
+; CHECK-GI-NEXT:    and x11, x12, #0x1
+; CHECK-GI-NEXT:    umulh x16, x0, x8
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    add x14, x14, x4
+; CHECK-GI-NEXT:    and x12, x12, #0x1
+; CHECK-GI-NEXT:    and x4, xzr, #0x1
+; CHECK-GI-NEXT:    mul x13, x3, x10
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, xzr, #0x1
+; CHECK-GI-NEXT:    adds x14, x14, x15
+; CHECK-GI-NEXT:    add x12, x12, x4
+; CHECK-GI-NEXT:    mul x5, x2, x8
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    adds x14, x14, x16
+; CHECK-GI-NEXT:    and x16, x4, #0x1
+; CHECK-GI-NEXT:    umulh x9, xzr, x10
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    adds x11, x14, x11
+; CHECK-GI-NEXT:    add x12, x12, x16
+; CHECK-GI-NEXT:    and x16, x4, #0x1
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    umulh x17, x1, x8
+; CHECK-GI-NEXT:    add x12, x12, x16
+; CHECK-GI-NEXT:    adds x13, x13, x5
+; CHECK-GI-NEXT:    umulh x15, x2, x10
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    and x16, x4, #0x1
+; CHECK-GI-NEXT:    mul x6, x3, x8
+; CHECK-GI-NEXT:    umulh x10, x3, x10
+; CHECK-GI-NEXT:    cmn x13, x15
+; CHECK-GI-NEXT:    and x13, x14, #0x1
+; CHECK-GI-NEXT:    add x14, x9, x17
+; CHECK-GI-NEXT:    umulh x15, x2, x8
+; CHECK-GI-NEXT:    add x12, x12, x13
+; CHECK-GI-NEXT:    add x13, x14, x18
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    sub x17, x2, x2
+; CHECK-GI-NEXT:    and x18, xzr, #0x1
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    umulh x8, x3, x8
+; CHECK-GI-NEXT:    add x12, x13, x12
+; CHECK-GI-NEXT:    add x14, x16, x14
+; CHECK-GI-NEXT:    add x16, x6, x17
+; CHECK-GI-NEXT:    and x17, xzr, #0x1
+; CHECK-GI-NEXT:    adds x10, x16, x10
+; CHECK-GI-NEXT:    add x17, x17, x18
+; CHECK-GI-NEXT:    cset w16, hs
+; CHECK-GI-NEXT:    adds x10, x10, x15
+; CHECK-GI-NEXT:    umulh x15, x2, xzr
+; CHECK-GI-NEXT:    and x16, x16, #0x1
+; CHECK-GI-NEXT:    cset w18, hs
+; CHECK-GI-NEXT:    adds x10, x10, x14
+; CHECK-GI-NEXT:    add x16, x17, x16
+; CHECK-GI-NEXT:    and x17, x18, #0x1
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    add x13, x16, x17
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    add x8, x9, x8
+; CHECK-GI-NEXT:    subs x9, x0, x11
+; CHECK-GI-NEXT:    add x13, x13, x14
+; CHECK-GI-NEXT:    add x8, x8, x15
+; CHECK-GI-NEXT:    sbc x14, x1, x12
+; CHECK-GI-NEXT:    add x8, x8, x13
+; CHECK-GI-NEXT:    subs x13, x2, x10
+; CHECK-GI-NEXT:    lsl x15, x14, #63
+; CHECK-GI-NEXT:    sbc x16, x3, x8
+; CHECK-GI-NEXT:    lsr x14, x14, #1
+; CHECK-GI-NEXT:    orr x9, x15, x9, lsr #1
+; CHECK-GI-NEXT:    lsl x15, x16, #63
+; CHECK-GI-NEXT:    orr x13, x15, x13, lsr #1
+; CHECK-GI-NEXT:    adds x9, x9, x11
+; CHECK-GI-NEXT:    lsr x11, x16, #1
+; CHECK-GI-NEXT:    adc x12, x14, x12
+; CHECK-GI-NEXT:    adds x10, x13, x10
+; CHECK-GI-NEXT:    lsl x13, x12, #62
+; CHECK-GI-NEXT:    lsr x12, x12, #2
+; CHECK-GI-NEXT:    adc x8, x11, x8
+; CHECK-GI-NEXT:    lsl x11, x8, #62
+; CHECK-GI-NEXT:    orr x9, x13, x9, lsr #2
+; CHECK-GI-NEXT:    mov w13, #7 // =0x7
+; CHECK-GI-NEXT:    lsr x8, x8, #2
+; CHECK-GI-NEXT:    lsl x14, x12, #3
+; CHECK-GI-NEXT:    orr x10, x11, x10, lsr #2
+; CHECK-GI-NEXT:    umulh x11, x9, x13
+; CHECK-GI-NEXT:    lsl x15, x9, #3
+; CHECK-GI-NEXT:    sub x12, x14, x12
+; CHECK-GI-NEXT:    lsl x16, x8, #3
+; CHECK-GI-NEXT:    umulh x13, x10, x13
+; CHECK-GI-NEXT:    lsl x14, x10, #3
+; CHECK-GI-NEXT:    sub x9, x15, x9
+; CHECK-GI-NEXT:    sub x8, x16, x8
+; CHECK-GI-NEXT:    subs x0, x0, x9
+; CHECK-GI-NEXT:    sub x10, x14, x10
+; CHECK-GI-NEXT:    add x11, x12, x11
+; CHECK-GI-NEXT:    sbc x1, x1, x11
+; CHECK-GI-NEXT:    subs x2, x2, x10
+; CHECK-GI-NEXT:    add x8, x8, x13
+; CHECK-GI-NEXT:    sbc x3, x3, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i128> %d, <i128 7, i128 7>
+  ret <2 x i128> %s
+}
+
+define <2 x i128> @uv2i128_100(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: uv2i128_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i128_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x10, #23593 // =0x5c29
+; CHECK-GI-NEXT:    mov x8, #62914 // =0xf5c2
+; CHECK-GI-NEXT:    sub x18, x0, x0
+; CHECK-GI-NEXT:    movk x10, #49807, lsl #16
+; CHECK-GI-NEXT:    movk x8, #23592, lsl #16
+; CHECK-GI-NEXT:    movk x10, #10485, lsl #32
+; CHECK-GI-NEXT:    movk x8, #49807, lsl #32
+; CHECK-GI-NEXT:    movk x10, #36700, lsl #48
+; CHECK-GI-NEXT:    movk x8, #10485, lsl #48
+; CHECK-GI-NEXT:    mul x11, x1, x10
+; CHECK-GI-NEXT:    mul x12, x0, x8
+; CHECK-GI-NEXT:    umulh x13, x0, x10
+; CHECK-GI-NEXT:    mul x14, x1, x8
+; CHECK-GI-NEXT:    adds x11, x11, x12
+; CHECK-GI-NEXT:    umulh x15, x1, x10
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    cmn x11, x13
+; CHECK-GI-NEXT:    and x11, x12, #0x1
+; CHECK-GI-NEXT:    umulh x16, x0, x8
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    and x12, x12, #0x1
+; CHECK-GI-NEXT:    add x14, x14, x18
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x9, xzr, x10
+; CHECK-GI-NEXT:    adds x14, x14, x15
+; CHECK-GI-NEXT:    and x15, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x17, x1, x8
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    add x15, x12, x15
+; CHECK-GI-NEXT:    adds x12, x14, x16
+; CHECK-GI-NEXT:    and x4, x4, #0x1
+; CHECK-GI-NEXT:    mul x18, x3, x10
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    adds x12, x12, x11
+; CHECK-GI-NEXT:    add x11, x15, x4
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    cset w15, hs
+; CHECK-GI-NEXT:    mul x5, x2, x8
+; CHECK-GI-NEXT:    add x11, x11, x14
+; CHECK-GI-NEXT:    and x14, x15, #0x1
+; CHECK-GI-NEXT:    add x17, x9, x17
+; CHECK-GI-NEXT:    add x14, x11, x14
+; CHECK-GI-NEXT:    mov w11, #100 // =0x64
+; CHECK-GI-NEXT:    umulh x13, x0, xzr
+; CHECK-GI-NEXT:    umulh x16, x2, x10
+; CHECK-GI-NEXT:    adds x18, x18, x5
+; CHECK-GI-NEXT:    mul x15, x3, x8
+; CHECK-GI-NEXT:    add x13, x17, x13
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    umulh x10, x3, x10
+; CHECK-GI-NEXT:    add x13, x13, x14
+; CHECK-GI-NEXT:    and x17, x17, #0x1
+; CHECK-GI-NEXT:    cmn x18, x16
+; CHECK-GI-NEXT:    sub x18, x2, x2
+; CHECK-GI-NEXT:    umulh x16, x2, x8
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    add x15, x15, x18
+; CHECK-GI-NEXT:    and x18, xzr, #0x1
+; CHECK-GI-NEXT:    add x14, x17, x14
+; CHECK-GI-NEXT:    umulh x8, x3, x8
+; CHECK-GI-NEXT:    and x17, xzr, #0x1
+; CHECK-GI-NEXT:    adds x10, x15, x10
+; CHECK-GI-NEXT:    add x15, x17, x18
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    umulh x18, x2, xzr
+; CHECK-GI-NEXT:    and x17, x17, #0x1
+; CHECK-GI-NEXT:    adds x10, x10, x16
+; CHECK-GI-NEXT:    lsl x16, x13, #60
+; CHECK-GI-NEXT:    add x15, x15, x17
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    adds x10, x10, x14
+; CHECK-GI-NEXT:    and x14, x17, #0x1
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    add x8, x9, x8
+; CHECK-GI-NEXT:    add x14, x15, x14
+; CHECK-GI-NEXT:    and x15, x17, #0x1
+; CHECK-GI-NEXT:    orr x12, x16, x12, lsr #4
+; CHECK-GI-NEXT:    add x9, x14, x15
+; CHECK-GI-NEXT:    add x8, x8, x18
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    lsr x9, x13, #4
+; CHECK-GI-NEXT:    umulh x14, x12, x11
+; CHECK-GI-NEXT:    lsl x13, x8, #60
+; CHECK-GI-NEXT:    lsr x8, x8, #4
+; CHECK-GI-NEXT:    mul x12, x12, x11
+; CHECK-GI-NEXT:    orr x10, x13, x10, lsr #4
+; CHECK-GI-NEXT:    madd x9, x9, x11, x14
+; CHECK-GI-NEXT:    umulh x13, x10, x11
+; CHECK-GI-NEXT:    subs x0, x0, x12
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    sbc x1, x1, x9
+; CHECK-GI-NEXT:    madd x8, x8, x11, x13
+; CHECK-GI-NEXT:    subs x2, x2, x10
+; CHECK-GI-NEXT:    sbc x3, x3, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i128> %d, <i128 100, i128 100>
+  ret <2 x i128> %s
+}
diff --git a/llvm/test/CodeGen/AArch64/sat-add.ll b/llvm/test/CodeGen/AArch64/sat-add.ll
index 2deb19be24821..ecd48d6b7c65b 100644
--- a/llvm/test/CodeGen/AArch64/sat-add.ll
+++ b/llvm/test/CodeGen/AArch64/sat-add.ll
@@ -530,7 +530,7 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %a = add <16 x i8> %x, %y
@@ -570,7 +570,7 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %a = add <8 x i16> %x, %y
@@ -610,7 +610,7 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = add <4 x i32> %x, %y
@@ -651,7 +651,7 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
   %a = add <2 x i64> %x, %y
diff --git a/llvm/test/CodeGen/AArch64/select_cc.ll b/llvm/test/CodeGen/AArch64/select_cc.ll
index 73e4d4c7f0aeb..483f6c26af8c1 100644
--- a/llvm/test/CodeGen/AArch64/select_cc.ll
+++ b/llvm/test/CodeGen/AArch64/select_cc.ll
@@ -88,7 +88,7 @@ define <2 x double> @select_olt_load_cmp(<2 x double> %a, ptr %src) {
 ; CHECK-SD-NEXT:    ldr d1, [x0]
 ; CHECK-SD-NEXT:    fcmgt v1.2s, v1.2s, #0.0
 ; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: select_olt_load_cmp:
diff --git a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
index 32fc9c1377704..0d4a636446164 100644
--- a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
@@ -249,9 +249,6 @@ define <16 x i8> @sel_shift_bool_v16i8(<16 x i1> %t) {
 ; CHECK-SD-LABEL: sel_shift_bool_v16i8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-SD-NEXT:    movi v1.16b, #128
-; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: sel_shift_bool_v16i8:
diff --git a/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll b/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll
index 6a95859c7692d..ef9309959c513 100644
--- a/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll
+++ b/llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll
@@ -1,41 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s
 
 define i128 @ldp_single_csdb(ptr %p) speculative_load_hardening {
+; CHECK-LABEL: ldp_single_csdb:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp x8, x1, [x0]
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    and x8, x8, x16
+; CHECK-NEXT:    and x1, x1, x16
+; CHECK-NEXT:    csdb
+; CHECK-NEXT:    mov x2, sp
+; CHECK-NEXT:    mov x0, x8
+; CHECK-NEXT:    and x2, x2, x16
+; CHECK-NEXT:    mov sp, x2
+; CHECK-NEXT:    ret
 entry:
   %0 = load i128, ptr %p, align 16
   ret i128 %0
-; CHECK-LABEL: ldp_single_csdb
-; CHECK:      ldp   x8, x1, [x0]
-; CHECK-NEXT: cmp sp, #0
-; CHECK-NEXT: csetm x16, ne
-; CHECK-NEXT: and   x8, x8, x16
-; CHECK-NEXT: and   x1, x1, x16
-; CHECK-NEXT: csdb
-; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT: mov x0, x8
-; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT: mov sp, [[TMPREG]]
-; CHECK-NEXT: ret
 }
 
+; Checking that the address loaded from is masked for a floating point load.
 define double @ld_double(ptr %p) speculative_load_hardening {
+; CHECK-LABEL: ld_double:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    and x0, x0, x16
+; CHECK-NEXT:    csdb
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    and x0, x0, x16
+; CHECK-NEXT:    mov sp, x0
+; CHECK-NEXT:    ret
 entry:
   %0 = load double, ptr %p, align 8
   ret double %0
-; Checking that the address loaded from is masked for a floating point load.
-; CHECK-LABEL: ld_double
-; CHECK:      cmp sp, #0
-; CHECK-NEXT: csetm x16, ne
-; CHECK-NEXT: and   x0, x0, x16
-; CHECK-NEXT: csdb
-; CHECK-NEXT: ldr   d0, [x0]
-; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT: mov sp, [[TMPREG]]
-; CHECK-NEXT: ret
 }
 
+; Checking that the address loaded from is masked for a floating point load.
+; csdb instruction must occur before the add instruction with w8 as operand.
 define i32 @csdb_emitted_for_subreg_use(ptr %p, i32 %b) speculative_load_hardening {
+; CHECK-LABEL: csdb_emitted_for_subreg_use:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    and x8, x8, x16
+; CHECK-NEXT:    csdb
+; CHECK-NEXT:    add w9, w1, w8
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    csel w0, w1, w9, eq
+; CHECK-NEXT:    mov x1, sp
+; CHECK-NEXT:    and x1, x1, x16
+; CHECK-NEXT:    mov sp, x1
+; CHECK-NEXT:    ret
 entry:
   %X = load i64, ptr %p, align 8
   %X_trunc = trunc i64 %X to i32
@@ -43,24 +62,25 @@ entry:
   %iszero = icmp eq i64 %X, 0
   %ret = select i1 %iszero, i32 %b, i32 %add
   ret i32 %ret
-; Checking that the address loaded from is masked for a floating point load.
-; CHECK-LABEL: csdb_emitted_for_subreg_use
-; CHECK:      cmp sp, #0
-; CHECK-NEXT: ldr x8, [x0]
-; CHECK-NEXT: csetm x16, ne
-; CHECK-NEXT: and x8, x8, x16
-; csdb instruction must occur before the add instruction with w8 as operand.
-; CHECK-NEXT: csdb
-; CHECK-NEXT: add w9, w1, w8
-; CHECK-NEXT: cmp x8, #0
-; CHECK-NEXT: csel w0, w1, w9, eq
-; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT: mov sp, [[TMPREG]]
-; CHECK-NEXT: ret
 }
 
+; Checking that the address loaded from is masked for a floating point load.
+; csdb instruction must occur before the add instruction with x8 as operand.
 define i64 @csdb_emitted_for_superreg_use(ptr %p, i64 %b) speculative_load_hardening {
+; CHECK-LABEL: csdb_emitted_for_superreg_use:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    and w8, w8, w16
+; CHECK-NEXT:    csdb
+; CHECK-NEXT:    add x9, x1, x8
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    csel x0, x1, x9, eq
+; CHECK-NEXT:    mov x1, sp
+; CHECK-NEXT:    and x1, x1, x16
+; CHECK-NEXT:    mov sp, x1
+; CHECK-NEXT:    ret
 entry:
   %X = load i32, ptr %p, align 4
   %X_ext = zext i32 %X to i64
@@ -68,89 +88,82 @@ entry:
   %iszero = icmp eq i32 %X, 0
   %ret = select i1 %iszero, i64 %b, i64 %add
   ret i64 %ret
-; Checking that the address loaded from is masked for a floating point load.
-; CHECK-LABEL: csdb_emitted_for_superreg_use
-; CHECK:      cmp sp, #0
-; CHECK-NEXT: ldr w8, [x0]
-; CHECK-NEXT: csetm x16, ne
-; CHECK-NEXT: and w8, w8, w16
-; csdb instruction must occur before the add instruction with x8 as operand.
-; CHECK-NEXT: csdb
-; CHECK-NEXT: add x9, x1, x8
-; CHECK-NEXT: cmp w8, #0
-; CHECK-NEXT: csel x0, x1, x9, eq
-; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT: mov sp, [[TMPREG]]
-; CHECK-NEXT: ret
 }
 
 define i64 @no_masking_with_full_control_flow_barriers(i64 %a, i64 %b, ptr %p) speculative_load_hardening {
-; CHECK-LABEL: no_masking_with_full_control_flow_barriers
-; CHECK: dsb sy
-; CHECK: isb
+; CHECK-LABEL: no_masking_with_full_control_flow_barriers:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dsb sy
+; CHECK-NEXT:    isb
+; CHECK-NEXT:    ldr x8, [x2]
+; CHECK-NEXT:    mov x17, x0
+; CHECK-NEXT:    mov x16, x1
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    hint #12
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    add x0, x8, x17
+; CHECK-NEXT:    ret
 entry:
   %0 = tail call i64 asm "hint #12", "={x17},{x16},0"(i64 %b, i64 %a)
   %X = load i64, ptr %p, align 8
   %ret = add i64 %X, %0
-; CHECK-NOT: csdb
-; CHECK-NOT: and
-; CHECK: ret
   ret i64 %ret
 }
 
-define void @f_implicitdef_vector_load(ptr %dst, ptr %src) speculative_load_hardening
-{
+define void @f_implicitdef_vector_load(ptr %dst, ptr %src) speculative_load_hardening {
+; CHECK-LABEL: f_implicitdef_vector_load:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    and x1, x1, x16
+; CHECK-NEXT:    csdb
+; CHECK-NEXT:    ldr d0, [x1]
+; CHECK-NEXT:    mov v0.d[1], v0.d[0]
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    and x0, x0, x16
+; CHECK-NEXT:    mov sp, x0
+; CHECK-NEXT:    ret
 entry:
   %0 = load <2 x i32>, ptr %src, align 8
   %shuffle = shufflevector <2 x i32> %0, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   store <4 x i32> %shuffle, ptr %dst, align 4
   ret void
-; CHECK-LABEL: f_implicitdef_vector_load
-; CHECK:       cmp     sp, #0
-; CHECK-NEXT:  csetm   x16, ne
-; CHECK-NEXT:  and     x1, x1, x16
-; CHECK-NEXT:  csdb
-; CHECK-NEXT:  ldr     d0, [x1]
-; CHECK-NEXT:  mov     v0.d[1], v0.d[0]
-; CHECK-NEXT:  str     q0, [x0]
-; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT:  mov     sp, [[TMPREG]]
-; CHECK-NEXT:  ret
 }
 
 define <2 x double> @f_usedefvectorload(ptr %a, ptr %b) speculative_load_hardening {
+; CHECK-LABEL: f_usedefvectorload:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    and x1, x1, x16
+; CHECK-NEXT:    csdb
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    ldr d0, [x1]
+; CHECK-NEXT:    and x0, x0, x16
+; CHECK-NEXT:    mov sp, x0
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: f_usedefvectorload
-; CHECK:       cmp     sp, #0
-; CHECK-NEXT:  csetm   x16, ne
-; CHECK-NEXT:  and     x1, x1, x16
-; CHECK-NEXT:  csdb
-; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT:  ldr     d0, [x1]
-; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT:  mov     sp, [[TMPREG]]
-; CHECK-NEXT:  ret
   %0 = load double, ptr %b, align 16
   %vld1_lane = insertelement <2 x double> <double undef, double 0.000000e+00>, double %0, i32 0
   ret <2 x double> %vld1_lane
 }
 
 define i32 @deadload() speculative_load_hardening uwtable {
+; CHECK-LABEL: deadload:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cmp sp, #0
+; CHECK-NEXT:    csetm x16, ne
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    ldr w8, [sp, #12]
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    and x0, x0, x16
+; CHECK-NEXT:    mov sp, x0
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: deadload
-; CHECK:       cmp     sp, #0
-; CHECK-NEXT:  csetm   x16, ne
-; CHECK-NEXT:  sub     sp, sp, #16
-; CHECK-NEXT:  .cfi_def_cfa_offset 16
-; CHECK-NEXT:  ldr     w8, [sp, #12]
-; CHECK-NEXT:  add     sp, sp, #16
-; CHECK-NEXT:  .cfi_def_cfa_offset 0
-; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
-; CHECK-NEXT:  mov     sp, [[TMPREG]]
-; CHECK-NEXT:  ret
   %a = alloca i32, align 4
   %val = load volatile i32, ptr %a, align 4
   ret i32 undef
diff --git a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
index cbb7f09513aa1..02fef348592cb 100644
--- a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
+++ b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
@@ -735,6 +735,35 @@ body:             |
     RET undef $lr
 ...
 ---
+name:            bundled_load_rename
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:     1
+  maxCallFrameSize: 0
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0.entry:
+    liveins: $z3, $z19, $p0, $z16
+    ; CHECK-LABEL: name: bundled_load_rename
+    ; CHECK: liveins: $z3, $z19, $p0, $z16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $q16, $q20 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+    ; CHECK-NEXT: BUNDLE implicit-def $z3, implicit-def $q3, implicit-def $d3, implicit-def $s3, implicit-def $h3, implicit-def $b3, implicit $z19, implicit $p0, implicit $z20 {
+    ; CHECK-NEXT:   $z3 = MOVPRFX_ZZ $z19
+    ; CHECK-NEXT:   $z3 = FMUL_ZPmZ_S renamable $p0, killed $z3, $z20
+    ; CHECK-NEXT: }
+    ; CHECK-NEXT: RET undef $lr
+    renamable $q16 = LDRQui $sp, 1 :: (load 16)
+    BUNDLE implicit-def $z3, implicit-def $q3, implicit-def $d3, implicit-def $s3, implicit-def $h3, implicit-def $b3, implicit $z19, implicit $p0, implicit $z16 {
+      $z3 = MOVPRFX_ZZ $z19
+      $z3 = FMUL_ZPmZ_S renamable $p0, killed $z3, renamable $z16
+    }
+    renamable $q16 = LDRQui $sp, 0 :: (load 16, align 32)
+    RET undef $lr
+...
+---
 #
 # CHECK-LABEL: name: bundled_use
 # CHECK: renamable $q0 = LDRQui $sp, 1
diff --git a/llvm/test/CodeGen/AArch64/sve-abd.ll b/llvm/test/CodeGen/AArch64/sve-abd.ll
index 72790155d046f..92bbd4e7f2759 100644
--- a/llvm/test/CodeGen/AArch64/sve-abd.ll
+++ b/llvm/test/CodeGen/AArch64/sve-abd.ll
@@ -283,8 +283,7 @@ define <vscale x 4 x i32> @uabd_non_matching_promotion(<vscale x 4 x i8> %a, <vs
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    and z0.s, z0.s, #0xff
 ; CHECK-NEXT:    sxtb z1.s, p0/m, z1.s
-; CHECK-NEXT:    sub z0.s, z0.s, z1.s
-; CHECK-NEXT:    abs z0.s, p0/m, z0.s
+; CHECK-NEXT:    sabd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
   %b.zext = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll
index 1e71c4b66156c..b62b850434469 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll
@@ -457,23 +457,25 @@ define void @ctpop_v256i8(ptr %a) vscale_range(16,0) #0 {
   ret void
 }
 
-; Don't use SVE for 64-bit vectors.
 define <4 x i16> @ctpop_v4i16(<4 x i16> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
 
-; Don't use SVE for 128-bit vectors.
 define <8 x i16> @ctpop_v8i16(<8 x i16> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
@@ -547,25 +549,25 @@ define void @ctpop_v128i16(ptr %a) vscale_range(16,0) #0 {
   ret void
 }
 
-; Don't use SVE for 64-bit vectors.
 define <2 x i32> @ctpop_v2i32(<2 x i32> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
 
-; Don't use SVE for 128-bit vectors.
 define <4 x i32> @ctpop_v4i32(<4 x i32> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
@@ -639,27 +641,25 @@ define void @ctpop_v64i32(ptr %a) vscale_range(16,0) #0 {
   ret void
 }
 
-; Don't use SVE for 64-bit vectors.
 define <1 x i64> @ctpop_v1i64(<1 x i64> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v1i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
 
-; Don't use SVE for 128-bit vectors.
 define <2 x i64> @ctpop_v2i64(<2 x i64> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
index d916f26f9b26b..c48ee3939bd2e 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -30,7 +30,9 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
 ; CHECK-NEXT:  // %bb.1: // %vector.body
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ldr z4, [x0]
+; CHECK-NEXT:    ldr z5, [x0, #2, mul vl]
+; CHECK-NEXT:    ldr z6, [x0, #3, mul vl]
 ; CHECK-NEXT:    umov w8, v0.b[8]
 ; CHECK-NEXT:    mov v1.b[1], v0.b[1]
 ; CHECK-NEXT:    fmov s2, w8
@@ -60,31 +62,20 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
 ; CHECK-NEXT:    asr z1.s, z1.s, #31
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    lsl z0.s, z0.s, #31
-; CHECK-NEXT:    and z1.s, z1.s, #0x1
+; CHECK-NEXT:    bic z1.d, z4.d, z1.d
 ; CHECK-NEXT:    lsl z2.s, z2.s, #31
+; CHECK-NEXT:    ldr z4, [x0, #1, mul vl]
 ; CHECK-NEXT:    asr z0.s, z0.s, #31
-; CHECK-NEXT:    cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT:    ldr z1, [x0]
+; CHECK-NEXT:    str z1, [x0]
 ; CHECK-NEXT:    lsl z3.s, z3.s, #31
 ; CHECK-NEXT:    asr z2.s, z2.s, #31
-; CHECK-NEXT:    and z0.s, z0.s, #0x1
+; CHECK-NEXT:    bic z0.d, z5.d, z0.d
 ; CHECK-NEXT:    asr z3.s, z3.s, #31
-; CHECK-NEXT:    and z2.s, z2.s, #0x1
-; CHECK-NEXT:    mov z1.s, p1/m, #0 // =0x0
-; CHECK-NEXT:    cmpne p2.s, p0/z, z0.s, #0
-; CHECK-NEXT:    ldr z0, [x0, #2, mul vl]
-; CHECK-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-NEXT:    str z1, [x0]
-; CHECK-NEXT:    cmpne p3.s, p0/z, z3.s, #0
-; CHECK-NEXT:    cmpne p0.s, p0/z, z2.s, #0
-; CHECK-NEXT:    ldr z3, [x0, #3, mul vl]
-; CHECK-NEXT:    ldr z2, [x0, #1, mul vl]
-; CHECK-NEXT:    mov z0.s, p2/m, #0 // =0x0
-; CHECK-NEXT:    mov z3.s, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.s, p0/m, #0 // =0x0
+; CHECK-NEXT:    bic z1.d, z4.d, z2.d
 ; CHECK-NEXT:    str z0, [x0, #2, mul vl]
+; CHECK-NEXT:    bic z3.d, z6.d, z3.d
+; CHECK-NEXT:    str z1, [x0, #1, mul vl]
 ; CHECK-NEXT:    str z3, [x0, #3, mul vl]
-; CHECK-NEXT:    str z2, [x0, #1, mul vl]
 ; CHECK-NEXT:  .LBB1_2: // %exit
 ; CHECK-NEXT:    ret
   %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
index 021d4855905e7..fb837c4279f6e 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -force-streaming -mattr=+ssve-fexpa < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+ssve-fexpa -force-streaming < %s | FileCheck %s
 
 define <vscale x 8 x half> @fexpa_h(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: fexpa_h:
@@ -27,4 +27,4 @@ define <vscale x 2 x double> @fexpa_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fexpa.x.nxv2f64(<vscale x 2 x i64> %a)
   ret <vscale x 2 x double> %out
-}
\ No newline at end of file
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-merging-unary.ll b/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
new file mode 100644
index 0000000000000..07f75c866d3d0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
@@ -0,0 +1,1575 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s
+; RUN: llc -mattr=+sme2 -force-streaming < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: abs_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %b, i1 0)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @abs_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: abs_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %b, i1 0)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @abs_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: abs_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %b, i1 0)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @abs_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: abs_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %b, i1 0)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @clz_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clz_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    clz z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @clz_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clz_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    clz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @clz_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clz_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    clz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @clz_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clz_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    clz z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @cnt_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cnt_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cnt z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @cnt_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cnt_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cnt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @cnt_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cnt_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cnt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @cnt_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cnt_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cnt z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x half> @fabs_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fabs_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.fabs.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @fabs_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fabs_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @fabs_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fabs_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fabs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @fabs_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fabs_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @fabs_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fabs_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fabs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fabs_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fabs_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fabs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x bfloat> @fabs_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fabs_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x7fff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @fabs_nxv4bf16(<vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fabs_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x7fff
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %b.op, <vscale x 4 x bfloat> %a
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @fabs_nxv8bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fabs_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x7fff
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %b.op, <vscale x 8 x bfloat> %a
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 2 x float> @fcvt_nxv2f16_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvt_nxv2f16_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x half> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @fcvt_nxv2f16_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvt_nxv2f16_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x half> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 4 x float> @fcvt_nxv4f16_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fcvt_nxv4f16_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 4 x half> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x half> @fcvt_nxv2f32_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f32_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x float> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x double> @fcvt_nxv2f32_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f32_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x float> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x bfloat> @fcvt_nxv2f32_to_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f32_to_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x float> %b to <vscale x 2 x bfloat>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x half> @fcvt_nxv4f32_to_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvt_nxv4f32_to_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 4 x float> %b to <vscale x 4 x half>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x bfloat> @fcvt_nxv4f32_to_nxv4bf16(<vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvt_nxv4f32_to_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 4 x float> %b to <vscale x 4 x bfloat>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %b.op, <vscale x 4 x bfloat> %a
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 2 x half> @fcvt_nxv2f64_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvt_nxv2f64_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x double> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x half> @fcvt_nxv2f64_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f64_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x float> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x bfloat> @fcvt_nxv2f64_to_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x double> %b) "target-features"="+sve2" {
+; CHECK-LABEL: fcvt_nxv2f64_to_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtx z1.s, p1/m, z1.d
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x double> %b to <vscale x 2 x bfloat>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 2 x float> @fcvt_nxv2bf16_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcvt_nxv2bf16_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @fcvt_nxv2bf16_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcvt_nxv2bf16_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    fcvt z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 4 x float> @fcvt_nxv4bf16_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcvt_nxv4bf16_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 4 x bfloat> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x i64> @fcvtsu_nxv2f16_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvtsu_nxv2f16_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 2 x half> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtsu_nxv4f16_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fcvtsu_nxv4f16_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 4 x half> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @fcvtsu_nxv8f16_to_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtsu_nxv8f16_to_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 8 x half> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 2 x i64> @fcvtsu_nxv2f32_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvtsu_nxv2f32_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 2 x float> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtsu_nxv4f32_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtsu_nxv4f32_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 4 x float> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @fcvtsu_nxv2f64_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtsu_nxv2f64_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 2 x double> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @fcvtzu_nxv2f16_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvtzu_nxv2f16_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 2 x half> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtzu_nxv4f16_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fcvtzu_nxv4f16_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 4 x half> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @fcvtzu_nxv8f16_to_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzu_nxv8f16_to_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 8 x half> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 2 x i64> @fcvtzu_nxv2f32_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvtzu_nxv2f32_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 2 x float> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtzu_nxv4f32_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtzu_nxv4f32_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 4 x float> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @fcvtzu_nxv2f64_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtzu_nxv2f64_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 2 x double> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x half> @fneg_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fneg_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x half> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @fneg_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fneg_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 4 x half> %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @fneg_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fneg_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fneg z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 8 x half> %b
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @fneg_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fneg_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x float> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @fneg_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fneg_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fneg z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 4 x float> %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fneg_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fneg_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fneg z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x double> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x bfloat> @fneg_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fneg_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z1.h, z1.h, #0x8000
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x bfloat> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @fneg_nxv4bf16(<vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fneg_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z1.h, z1.h, #0x8000
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 4 x bfloat> %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %b.op, <vscale x 4 x bfloat> %a
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @fneg_nxv8bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fneg_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z1.h, z1.h, #0x8000
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 8 x bfloat> %b
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %b.op, <vscale x 8 x bfloat> %a
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 2 x half> @frinta_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frinta_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.round.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frinta_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frinta_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.round.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frinta_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frinta_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinta z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.round.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frinta_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frinta_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frinta_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frinta_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinta z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frinta_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frinta_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinta z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frinti_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frinti_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.nearbyint.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frinti_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frinti_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.nearbyint.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frinti_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frinti_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinti z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.nearbyint.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frinti_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frinti_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frinti_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frinti_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinti z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frinti_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frinti_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frinti z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintm_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintm_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.floor.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintm_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintm_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.floor.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintm_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintm_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintm z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.floor.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintm_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintm_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintm_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintm_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintm z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintm_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintm_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintm z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintn_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintn_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.roundeven.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintn_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintn_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.roundeven.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintn_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintn_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.roundeven.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintn_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintn_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintn_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintn_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintn_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintn_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintp_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintp_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.ceil.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintp_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintp_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintp_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintp_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintp z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.ceil.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintp_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintp_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintp_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintp_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintp z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintp_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintp_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintp z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintx_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintx_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.rint.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintx_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintx_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.rint.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintx_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintx_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.rint.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintx_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintx_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintx_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintx_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintx_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintx_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintz_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintz_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.trunc.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintz_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintz_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.trunc.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintz_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintz_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintz z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.trunc.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintz_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintz_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintz_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintz_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintz z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintz_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintz_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintz z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @fsqrt_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fsqrt_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.sqrt.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @fsqrt_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fsqrt_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.sqrt.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @fsqrt_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsqrt_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsqrt z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.sqrt.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @fsqrt_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fsqrt_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.sqrt.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @fsqrt_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsqrt_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsqrt z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fsqrt_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsqrt_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fsqrt z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 16 x i8> @neg_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: neg_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 16 x i8> zeroinitializer, %b
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @neg_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: neg_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 8 x i16> zeroinitializer, %b
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @neg_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: neg_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 4 x i32> zeroinitializer, %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @neg_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: neg_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 2 x i64> zeroinitializer, %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @rbit_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: rbit_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rbit z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.bitreverse.nxv16i8(<vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @rbit_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: rbit_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rbit z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.bitreverse.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @rbit_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: rbit_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rbit z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.bitreverse.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @rbit_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: rbit_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rbit z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.bitreverse.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x i16> @revb_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: revb_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    revb z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @revb_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: revb_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    revb z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @revb_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: revb_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    revb z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @scvtf_nxv8i16_to_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: scvtf_nxv8i16_to_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 8 x i16> %b to <vscale x 8 x half>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x half> @scvtf_nxv4i32_to_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_nxv4i32_to_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 4 x i32> %b to <vscale x 4 x half>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @scvtf_nxv4i32_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_nxv4i32_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 4 x i32> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x half> @scvtf_nxv2i64_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_nxv2i64_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf z0.h, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 2 x i64> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x float> @scvtf_nxv2i64_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_nxv2i64_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 2 x i64> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @scvtf_nxv2i64_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_nxv2i64_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 2 x i64> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x i16> @sxtb_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: sxtb_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    sxtb z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 8 x i8> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sxtb_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i8> %b) {
+; CHECK-LABEL: sxtb_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    sxtb z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sxtb_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i8> %b) {
+; CHECK-LABEL: sxtb_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    sxtb z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 2 x i8> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @sxth_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: sxth_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    sxth z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sxth_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i16> %b) {
+; CHECK-LABEL: sxth_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    sxth z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 2 x i16> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @sxtw_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: sxtw_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 2 x i32> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @ucvtf_nxv8i16_to_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: ucvtf_nxv8i16_to_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 8 x i16> %b to <vscale x 8 x half>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x half> @ucvtf_nxv4i32_to_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_nxv4i32_to_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 4 x i32> %b to <vscale x 4 x half>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @ucvtf_nxv4i32_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_nxv4i32_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 4 x i32> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x half> @ucvtf_nxv2i64_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_nxv2i64_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf z0.h, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 2 x i64> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x float> @ucvtf_nxv2i64_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_nxv2i64_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf z0.s, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 2 x i64> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @ucvtf_nxv2i64_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_nxv2i64_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 2 x i64> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x i16> @uxtb_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: uxtb_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0xff
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 8 x i8> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @uxtb_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i8> %b) {
+; CHECK-LABEL: uxtb_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.s, z1.s, #0xff
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 4 x i8> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uxtb_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i8> %b) {
+; CHECK-LABEL: uxtb_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.d, z1.d, #0xff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 2 x i8> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @uxth_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: uxth_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.s, z1.s, #0xffff
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uxth_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i16> %b) {
+; CHECK-LABEL: uxth_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.d, z1.d, #0xffff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 2 x i16> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @uxtw_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: uxtw_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 2 x i32> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @abs_nxv16i8_all_active_predicate(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: abs_nxv16i8_all_active_predicate:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> %c, <vscale x 16 x i1> splat(i1 true), <vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @abs_nxv16i8_same_predicate(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: abs_nxv16i8_same_predicate:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> %c, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 16 x i8> @abs_nxv16i8_inactive_lanes_are_not_defined(<vscale x 16 x i1> %sel_pg, <vscale x 16 x i1> %op_pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: abs_nxv16i8_inactive_lanes_are_not_defined:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %op_pg, <vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %sel_pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+; Merging op has multiple users.
+declare void @use(<vscale x 16 x i8>,<vscale x 16 x i8>)
+define void @abs_nxv16i8_multi_use(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: abs_nxv16i8_multi_use:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    abs z1.b, p1/m, z1.b
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    b use
+  %b.op = tail call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %b, i1 0)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  tail call void @use(<vscale x 16 x i8> %res, <vscale x 16 x i8> %b.op)
+  ret void
+}
+
+; Inactive lanes of the merging op remain live after the select.
+define <vscale x 16 x i8> @abs_nxv16i8_predicate_mismatch(<vscale x 16 x i1> %sel_pg, <vscale x 16 x i1> %op_pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+; CHECK-LABEL: abs_nxv16i8_predicate_mismatch:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    abs z2.b, p1/m, z1.b
+; CHECK-NEXT:    mov z0.b, p0/m, z2.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.aarch64.sve.abs.nxv16i8(<vscale x 16 x i8> %c, <vscale x 16 x i1> %op_pg, <vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %sel_pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+declare <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8>, i1)
+declare <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16>, i1)
+declare <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32>, i1)
+declare <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64>, i1)
+
+declare <vscale x 16 x i8> @llvm.bitreverse.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.bitreverse.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.bitreverse.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.bitreverse.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 2 x half> @llvm.ceil.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.ceil.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 2 x half> @llvm.fabs.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat>)
+declare <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat>)
+declare <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat>)
+
+declare <vscale x 2 x half> @llvm.floor.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.floor.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.floor.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 4 x half> @llvm.nearbyint.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.nearbyint.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.rint.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.rint.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.rint.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.round.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.round.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.round.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 2 x half> @llvm.nearbyint.nxv2f16(<vscale x 2 x half>)
+
+declare <vscale x 2 x half> @llvm.roundeven.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.roundeven.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.roundeven.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.sqrt.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.sqrt.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.sqrt.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.sqrt.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.trunc.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.trunc.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.trunc.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-vscale.ll b/llvm/test/CodeGen/AArch64/sve-vscale.ll
index 7214c98cc3186..173652630bf1b 100644
--- a/llvm/test/CodeGen/AArch64/sve-vscale.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vscale.ll
@@ -1,50 +1,50 @@
 ; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
-; RUN: opt -mtriple=aarch64 -passes='require<profile-summary>,function(codegenprepare)' -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s
+; RUN: opt -mtriple=aarch64 -passes="require<profile-summary>,function(codegenprepare)" -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s
 
 ;
 ; RDVL
 ;
 
+define i8 @rdvl_i8() nounwind {
 ; CHECK-LABEL: rdvl_i8:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i8 @rdvl_i8() nounwind {
   %vscale = call i8 @llvm.vscale.i8()
   %1 = mul nsw i8 %vscale, 16
   ret i8 %1
 }
 
+define i16 @rdvl_i16() nounwind {
 ; CHECK-LABEL: rdvl_i16:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i16 @rdvl_i16() nounwind {
   %vscale = call i16 @llvm.vscale.i16()
   %1 = mul nsw i16 %vscale, 16
   ret i16 %1
 }
 
+define i32 @rdvl_i32() nounwind {
 ; CHECK-LABEL: rdvl_i32:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i32 @rdvl_i32() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 16
   ret i32 %1
 }
 
+define i64 @rdvl_i64() nounwind {
 ; CHECK-LABEL: rdvl_i64:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i64 @rdvl_i64() nounwind {
   %vscale = call i64 @llvm.vscale.i64()
   %1 = mul nsw i64 %vscale, 16
   ret i64 %1
 }
 
+define i32 @rdvl_const() nounwind {
 ; CHECK-LABEL: rdvl_const:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i32 @rdvl_const() nounwind {
   %vscale.ptr = getelementptr <vscale x 1 x i8>, ptr null, i64 1
   %vscale.int = ptrtoint ptr %vscale.ptr to i32
   %vscale.scaled = mul nsw i32 %vscale.int, 16
@@ -70,32 +70,31 @@ define i32 @vscale_neg1() nounwind {
   ret i32 %neg
 }
 
+define i32 @rdvl_3() nounwind {
 ; CHECK-LABEL: rdvl_3:
 ; CHECK:       rdvl [[VL_B:x[0-9]+]], #1
 ; CHECK-NEXT:  mov  w[[MUL:[0-9]+]], #3
 ; CHECK-NEXT:  lsr  [[VL_Q:x[0-9]+]], [[VL_B]], #4
 ; CHECK-NEXT:  mul  x0, [[VL_Q]], x[[MUL]]
 ; CHECK-NEXT:  ret
-define i32 @rdvl_3() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 3
   ret i32 %1
 }
 
-
+define i32 @rdvl_min() nounwind {
 ; CHECK-LABEL: rdvl_min:
 ; CHECK:       rdvl x0, #-32
 ; CHECK-NEXT:  ret
-define i32 @rdvl_min() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -512
   ret i32 %1
 }
 
+define i32 @rdvl_max() nounwind {
 ; CHECK-LABEL: rdvl_max:
 ; CHECK:       rdvl x0, #31
 ; CHECK-NEXT:  ret
-define i32 @rdvl_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 496
   ret i32 %1
@@ -116,29 +115,29 @@ define i1 @rdvl_i1() {
 ; CNTH
 ;
 
+define i32 @cnth() nounwind {
 ; CHECK-LABEL: cnth:
 ; CHECK:       cnth x0{{$}}
 ; CHECK-NEXT:  ret
-define i32 @cnth() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = shl nsw i32 %vscale, 3
   ret i32 %1
 }
 
+define i32 @cnth_max() nounwind {
 ; CHECK-LABEL: cnth_max:
 ; CHECK:       cnth x0, all, mul #15
 ; CHECK-NEXT:  ret
-define i32 @cnth_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 120
   ret i32 %1
 }
 
+define i32 @cnth_neg() nounwind {
 ; CHECK-LABEL: cnth_neg:
 ; CHECK:       cnth [[CNT:x[0-9]+]]
 ; CHECK:       neg x0, [[CNT]]
 ; CHECK-NEXT:  ret
-define i32 @cnth_neg() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -8
   ret i32 %1
@@ -148,29 +147,29 @@ define i32 @cnth_neg() nounwind {
 ; CNTW
 ;
 
+define i32 @cntw() nounwind {
 ; CHECK-LABEL: cntw:
 ; CHECK:       cntw x0{{$}}
 ; CHECK-NEXT:  ret
-define i32 @cntw() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = shl nsw i32 %vscale, 2
   ret i32 %1
 }
 
+define i32 @cntw_max() nounwind {
 ; CHECK-LABEL: cntw_max:
 ; CHECK:       cntw x0, all, mul #15
 ; CHECK-NEXT:  ret
-define i32 @cntw_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 60
   ret i32 %1
 }
 
+define i32 @cntw_neg() nounwind {
 ; CHECK-LABEL: cntw_neg:
 ; CHECK:       cntw [[CNT:x[0-9]+]]
 ; CHECK:       neg x0, [[CNT]]
 ; CHECK-NEXT:  ret
-define i32 @cntw_neg() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -4
   ret i32 %1
@@ -180,29 +179,29 @@ define i32 @cntw_neg() nounwind {
 ; CNTD
 ;
 
+define i32 @cntd() nounwind {
 ; CHECK-LABEL: cntd:
 ; CHECK:       cntd x0{{$}}
 ; CHECK-NEXT:  ret
-define i32 @cntd() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = shl nsw i32 %vscale, 1
   ret i32 %1
 }
 
+define i32 @cntd_max() nounwind {
 ; CHECK-LABEL: cntd_max:
 ; CHECK:       cntd x0, all, mul #15
 ; CHECK-NEXT:  ret
-define i32 @cntd_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 30
   ret i32 %1
 }
 
+define i32 @cntd_neg() nounwind {
 ; CHECK-LABEL: cntd_neg:
 ; CHECK:       cntd [[CNT:x[0-9]+]]
 ; CHECK:       neg x0, [[CNT]]
 ; CHECK-NEXT:  ret
-define i32 @cntd_neg() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -2
   ret i32 %1
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
index e2483cff3d186..1ccb5264aa837 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ; ADDHNB
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
index 2f7b82751cdcf..2b2c9da1a3063 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll
index 8d395adda0799..3a2a02f80a58b 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
index 6005fb69ae1ba..27416375ad6af 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll
index 7d16f8383d968..9be3a67f88b09 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2 < %s | FileCheck %s
 ; RUN: llc -mattr=+sme2 -force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-linux"
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
index 16041766605e9..1ce7564167992 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
index 52c04d614b4e1..d9a3a28ff9aa8 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
index 6dc2c67b5fd9e..70824f88caef5 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll
index c46016e0c40de..c78a872fcfbf5 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll
index 5cea7536e1f3c..8e53a82401e0b 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sve2,+lut,+bf16 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sve2,+lut | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sve,+sme2,+lut | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sme2,+lut --force-streaming | FileCheck %s
 
 define <vscale x 16 x i8> @test_luti2_lane_i8(<vscale x 16 x i8> %table, <vscale x 16 x i8> %indices){
 ; CHECK-LABEL: test_luti2_lane_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
index ab4945f5178d8..ab6c02b22db7d 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll
index 08afb88a2c576..400b2890f4974 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll
index ccc7a4d14a770..01f7142831914 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll
@@ -1,4 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,sme -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 16 x i1> @psel_b(<vscale x 16 x i1> %p1, <vscale x 16 x i1> %p2, i32 %idx) {
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll
index 16ebd594ca21c..5e837da8bb9c3 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll
@@ -1,4 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 16 x i8> @test_revd_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll
index de5af6ce0ae99..fb7dcb3485c68 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2-sm4 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sve-sm4 < %s | FileCheck %s
 
 ;
 ; SM4E
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
index 6492f094b707a..77503a62041bf 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll
index 8c0ef68dda5aa..63889df5336ef 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll
index e5750099258d6..c702038be31fe 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
index 610333f0ff34b..5f09ab2458f53 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll
index 1786f5c514fba..8ad983d92e840 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll
index 3d2a6930ca976..c88549e7769e9 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
index 6854ad51989a2..7e852f7f1c287 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll
index 2b96d452fba0d..579bbf52dd7a8 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -mattr=+b16b16 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 4 x float> @bfmlslb_f32(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {
 ; CHECK-LABEL: bfmlslb_f32:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll
index 45446a4127ee4..b7c47628a76f8 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define i64 @test_svcntp_c8_vlx2(target("aarch64.svcount") %pn) nounwind {
 ; CHECK-LABEL: test_svcntp_c8_vlx2:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
index f4cb0a3f70ee5..b280ee69bb85e 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 4 x i32> @sdot_x2(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {
 ; CHECK-LABEL: sdot_x2:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
index f1a423b02ac2a..53e7d684e1f90 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i8> @test_dupq_i8 (<vscale x 16 x i8> %zn) {
 ; CHECK-LABEL: test_dupq_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll
index bb4c67fca5dc8..ac52e23133b62 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i8> @test_extq_i8 (<vscale x 16 x i8> %zn, <vscale x 16 x i8> %zm) {
 ; CHECK-LABEL: test_extq_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll
index 726a8b4cfc666..82b272e8d8146 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme,+sve2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
index 41b4c77178b5e..08b62840ed022 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; FMAXNMQV
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
index 94545a3c64429..feb2126214936 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; ORQV
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll
index c0e0a5834eecd..0eacac2ca63f5 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+bf16 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming  < %s | FileCheck %s
 
 ; == Normal Multi-Vector Consecutive Loads ==
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
index f15aa2ba02238..393cdeb6f1b37 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming  < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;;LD2Q
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
index 8fe0694808c8e..67e92d59a42a4 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; ST2Q
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll
index a592dcd4b8ce9..a6f6960507b71 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i1> @test_pmov_to_pred_i8(<vscale x 16 x i8> %zn) {
 ; CHECK-LABEL: test_pmov_to_pred_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll
index b7f36c6510233..6b1ec35f7c00a 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ; Merge
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll
index 9e033a978aab6..ffcc412b50e21 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64 -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i1> @pext_b(target("aarch64.svcount") %x) nounwind {
 ; CHECK-LABEL: pext_b:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll
index 8b40ac4c3c947..0b5469bacc4f1 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py$
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; SQCVTN
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll
index eb6e5a07558a7..b85b4799d1cce 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; S/UQRSHRN x2
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll
index 0e798d3b89a19..18e2769949644 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
index e7d9c3e6839e4..0ff2b53a818e8 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+bf16 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ; == Normal Multi-Vector Consecutive Stores ==
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll
index dd278f4038b8d..2c2539dba1893 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
index 7a3ffb020ea56..efe1e161c50f1 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
-
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; WHILEGE
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll
index ab70f57b48874..db7a3fa46946e 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve,+sme2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ; == WHILEGE ==
 
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index b5d64112db727..aa0a163b96ac8 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -31,12 +31,12 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    add x13, x13, #32
 ; CHECK-NEXT:    fcmgt v3.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    fcmgt v4.4s, v2.4s, v0.4s
-; CHECK-NEXT:    fcmlt v5.4s, v1.4s, #0.0
-; CHECK-NEXT:    fcmlt v6.4s, v2.4s, #0.0
-; CHECK-NEXT:    bit v1.16b, v0.16b, v3.16b
-; CHECK-NEXT:    bit v2.16b, v0.16b, v4.16b
-; CHECK-NEXT:    bic v1.16b, v1.16b, v5.16b
-; CHECK-NEXT:    bic v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
+; CHECK-NEXT:    bsl v4.16b, v0.16b, v2.16b
+; CHECK-NEXT:    fcmlt v1.4s, v1.4s, #0.0
+; CHECK-NEXT:    fcmlt v2.4s, v2.4s, #0.0
+; CHECK-NEXT:    bic v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bic v2.16b, v4.16b, v2.16b
 ; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
 ; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
 ; CHECK-NEXT:    xtn v1.4h, v1.4s
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll b/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
index 830f1efbdf67f..5bd9ec2afdad1 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
@@ -12,7 +12,7 @@
 ; CHECK: Legally typed node: [[VTWOA]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legalizing node: [[VTWOB:t.*]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legally typed node: [[VTWOB]]: v2f64 = BUILD_VECTOR
-; CHECK: Legalizing node: t30: v2f64 = fmaxnum nnan reassoc [[VTWOB]], [[VTWOA]]
+; CHECK: Legalizing node: t31: v2f64 = fmaxnum nnan reassoc [[VTWOB]], [[VTWOA]]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
diff --git a/llvm/test/CodeGen/AArch64/vselect-constants.ll b/llvm/test/CodeGen/AArch64/vselect-constants.ll
index a7cf5ece5d270..3c1f06e0e4ed1 100644
--- a/llvm/test/CodeGen/AArch64/vselect-constants.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-constants.ll
@@ -146,10 +146,8 @@ define <4 x i32> @cmp_sel_0_or_minus1_vec(<4 x i32> %x, <4 x i32> %y) {
 define <4 x i32> @sel_1_or_0_vec(<4 x i1> %cond) {
 ; CHECK-LABEL: sel_1_or_0_vec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    shl v0.4s, v0.4s, #31
-; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %add = select <4 x i1> %cond, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
@@ -171,11 +169,9 @@ define <4 x i32> @cmp_sel_1_or_0_vec(<4 x i32> %x, <4 x i32> %y) {
 define <4 x i32> @sel_0_or_1_vec(<4 x i1> %cond) {
 ; CHECK-LABEL: sel_0_or_1_vec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    shl v0.4s, v0.4s, #31
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
   %add = select <4 x i1> %cond, <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   ret <4 x i32> %add
diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll
index 76b7f3d9dfc0e..4f2b9c5a62669 100644
--- a/llvm/test/CodeGen/AArch64/vselect-ext.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll
@@ -12,10 +12,10 @@ define <16 x i32> @no_existing_zext(<16 x i8> %a, <16 x i32> %op) {
 ; CHECK-NEXT:    sshll.4s v6, v5, #0
 ; CHECK-NEXT:    sshll.4s v7, v0, #0
 ; CHECK-NEXT:    sshll2.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v4, v4, v16
-; CHECK-NEXT:    and.16b v0, v1, v6
-; CHECK-NEXT:    and.16b v1, v2, v5
-; CHECK-NEXT:    and.16b v2, v3, v7
+; CHECK-NEXT:    and.16b v4, v16, v4
+; CHECK-NEXT:    and.16b v0, v6, v1
+; CHECK-NEXT:    and.16b v1, v5, v2
+; CHECK-NEXT:    and.16b v2, v7, v3
 ; CHECK-NEXT:    mov.16b v3, v4
 ; CHECK-NEXT:    ret
 entry:
@@ -40,10 +40,10 @@ define <16 x i32> @second_compare_operand_not_splat(<16 x i8> %a, <16 x i8> %b)
 ; CHECK-NEXT:    sshll.4s v7, v1, #0
 ; CHECK-NEXT:    sshll2.4s v16, v3, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v0, v4, v0
-; CHECK-NEXT:    and.16b v3, v6, v1
-; CHECK-NEXT:    and.16b v1, v2, v16
-; CHECK-NEXT:    and.16b v2, v5, v7
+; CHECK-NEXT:    and.16b v0, v0, v4
+; CHECK-NEXT:    and.16b v3, v1, v6
+; CHECK-NEXT:    and.16b v1, v16, v2
+; CHECK-NEXT:    and.16b v2, v7, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i32>
@@ -69,10 +69,10 @@ define <16 x i32> @same_zext_used_in_cmp_signed_pred_and_select(<16 x i8> %a) {
 ; CHECK-NEXT:    sshll.4s v7, v1, #0
 ; CHECK-NEXT:    sshll2.4s v16, v3, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v0, v4, v0
-; CHECK-NEXT:    and.16b v3, v6, v1
-; CHECK-NEXT:    and.16b v1, v2, v16
-; CHECK-NEXT:    and.16b v2, v5, v7
+; CHECK-NEXT:    and.16b v0, v0, v4
+; CHECK-NEXT:    and.16b v3, v1, v6
+; CHECK-NEXT:    and.16b v1, v16, v2
+; CHECK-NEXT:    and.16b v2, v7, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i32>
@@ -97,10 +97,10 @@ define <8 x i64> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i64(<8 x i8>
 ; CHECK-NEXT:    cmhi.2d v7, v1, v2
 ; CHECK-NEXT:    cmhi.2d v6, v5, v2
 ; CHECK-NEXT:    cmhi.2d v2, v4, v2
-; CHECK-NEXT:    and.16b v0, v3, v0
-; CHECK-NEXT:    and.16b v1, v1, v7
-; CHECK-NEXT:    and.16b v3, v4, v2
-; CHECK-NEXT:    and.16b v2, v5, v6
+; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v7, v1
+; CHECK-NEXT:    and.16b v3, v2, v4
+; CHECK-NEXT:    and.16b v2, v6, v5
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i8> %a to <8 x i64>
   %cmp = icmp ugt <8 x i8> %a, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
@@ -123,10 +123,10 @@ define <16 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v16i32(<16 x i
 ; CHECK-NEXT:    cmhi.4s v7, v2, v1
 ; CHECK-NEXT:    cmhi.4s v6, v5, v1
 ; CHECK-NEXT:    cmhi.4s v1, v4, v1
-; CHECK-NEXT:    and.16b v0, v3, v0
-; CHECK-NEXT:    and.16b v3, v4, v1
-; CHECK-NEXT:    and.16b v1, v2, v7
-; CHECK-NEXT:    and.16b v2, v5, v6
+; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v3, v1, v4
+; CHECK-NEXT:    and.16b v1, v7, v2
+; CHECK-NEXT:    and.16b v2, v6, v5
 ; CHECK-NEXT:    ret
   %ext = zext <16 x i8> %a to <16 x i32>
   %cmp = icmp ugt <16 x i8> %a, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
@@ -143,8 +143,8 @@ define <8 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i32(<8 x i8>
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v3, v0, v1
 ; CHECK-NEXT:    cmhi.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i8> %a to <8 x i32>
   %cmp = icmp ugt <8 x i8> %a, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
@@ -160,8 +160,8 @@ define <8 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i32_2(<8 x i1
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v3, v0, v1
 ; CHECK-NEXT:    cmhi.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i16> %a to <8 x i32>
   %cmp = icmp ugt <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
@@ -179,8 +179,8 @@ define <8 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i32_from_v8i1
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v3, v0, v1
 ; CHECK-NEXT:    cmhi.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i15> %a to <8 x i32>
   %cmp = icmp ugt <8 x i15> %a, <i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10>
@@ -197,8 +197,8 @@ define <7 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v7i32(<7 x i16>
 ; CHECK-NEXT:    ushll2.4s v0, v0, #0
 ; CHECK-NEXT:    sshll.4s v3, v1, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v2, v2, v3
-; CHECK-NEXT:    and.16b v0, v0, v1
+; CHECK-NEXT:    and.16b v2, v3, v2
+; CHECK-NEXT:    and.16b v0, v1, v0
 ; CHECK-NEXT:    mov.s w1, v2[1]
 ; CHECK-NEXT:    mov.s w2, v2[2]
 ; CHECK-NEXT:    mov.s w3, v2[3]
@@ -244,7 +244,7 @@ define <4 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v4i32(<4 x i16>
 ; CHECK-NEXT:    movi.4s v1, #10
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v1, v0, v1
-; CHECK-NEXT:    and.16b v0, v0, v1
+; CHECK-NEXT:    and.16b v0, v1, v0
 ; CHECK-NEXT:    ret
   %ext = zext <4 x i16> %a to <4 x i32>
   %cmp = icmp ugt <4 x i16> %a, <i16 10, i16 10, i16 10, i16 10>
@@ -259,7 +259,7 @@ define <2 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v2i32(<2 x i16>
 ; CHECK-NEXT:    movi.2s v2, #10
 ; CHECK-NEXT:    and.8b v0, v0, v1
 ; CHECK-NEXT:    cmhi.2s v1, v0, v2
-; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    and.8b v0, v1, v0
 ; CHECK-NEXT:    ret
   %ext = zext <2 x i16> %a to <2 x i32>
   %cmp = icmp ugt <2 x i16> %a, <i16 10, i16 10>
@@ -275,8 +275,8 @@ define <8 x i32> @same_zext_used_in_cmp_eq_and_select_v8i32(<8 x i16> %a) {
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmeq.4s v3, v0, v1
 ; CHECK-NEXT:    cmeq.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i16> %a to <8 x i32>
   %cmp = icmp eq <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
@@ -293,8 +293,8 @@ define <8 x i32> @same_zext_used_in_cmp_eq_and_select_v8i32_from_v8i13(<8 x i13>
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmeq.4s v3, v0, v1
 ; CHECK-NEXT:    cmeq.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i13> %a to <8 x i32>
   %cmp = icmp eq <8 x i13> %a, <i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10>
@@ -358,16 +358,16 @@ define <16 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_other_use(<16
 ; CHECK-NEXT:    and.16b v6, v6, v26
 ; CHECK-NEXT:    sshll2.2d v26, v16, #0
 ; CHECK-NEXT:    and.16b v27, v4, v27
-; CHECK-NEXT:    and.16b v4, v0, v18
-; CHECK-NEXT:    and.16b v0, v24, v16
+; CHECK-NEXT:    and.16b v4, v18, v0
+; CHECK-NEXT:    and.16b v0, v16, v24
 ; CHECK-NEXT:    stp q7, q21, [x0, #96]
 ; CHECK-NEXT:    sshll.2d v21, v16, #0
 ; CHECK-NEXT:    and.16b v5, v5, v22
 ; CHECK-NEXT:    and.16b v7, v3, v23
-; CHECK-NEXT:    and.16b v3, v19, v20
+; CHECK-NEXT:    and.16b v3, v20, v19
 ; CHECK-NEXT:    stp q5, q6, [x0, #64]
 ; CHECK-NEXT:    and.16b v6, v2, v26
-; CHECK-NEXT:    and.16b v2, v25, v17
+; CHECK-NEXT:    and.16b v2, v17, v25
 ; CHECK-NEXT:    and.16b v5, v1, v21
 ; CHECK-NEXT:    mov.16b v1, v3
 ; CHECK-NEXT:    mov.16b v3, v4
@@ -397,10 +397,10 @@ define <16 x i32> @same_sext_used_in_cmp_signed_pred_and_select_v16i32(<16 x i8>
 ; CHECK-NEXT:    cmgt.4s v7, v2, v1
 ; CHECK-NEXT:    cmgt.4s v6, v5, v1
 ; CHECK-NEXT:    cmgt.4s v1, v4, v1
-; CHECK-NEXT:    and.16b v0, v3, v0
-; CHECK-NEXT:    and.16b v3, v4, v1
-; CHECK-NEXT:    and.16b v1, v2, v7
-; CHECK-NEXT:    and.16b v2, v5, v6
+; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v3, v1, v4
+; CHECK-NEXT:    and.16b v1, v7, v2
+; CHECK-NEXT:    and.16b v2, v6, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = sext <16 x i8> %a to <16 x i32>
@@ -417,8 +417,8 @@ define <8 x i32> @same_sext_used_in_cmp_eq_and_select_v8i32(<8 x i16> %a) {
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    cmeq.4s v3, v0, v1
 ; CHECK-NEXT:    cmeq.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = sext <8 x i16> %a to <8 x i32>
   %cmp = icmp eq <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
@@ -438,8 +438,8 @@ define <8 x i32> @same_sext_used_in_cmp_eq_and_select_v8i32_from_v8i13(<8 x i13>
 ; CHECK-NEXT:    sshr.4s v2, v2, #19
 ; CHECK-NEXT:    cmeq.4s v3, v2, v1
 ; CHECK-NEXT:    cmeq.4s v1, v0, v1
-; CHECK-NEXT:    and.16b v1, v0, v1
-; CHECK-NEXT:    and.16b v0, v2, v3
+; CHECK-NEXT:    and.16b v1, v1, v0
+; CHECK-NEXT:    and.16b v0, v3, v2
 ; CHECK-NEXT:    ret
   %ext = sext <8 x i13> %a to <8 x i32>
   %cmp = icmp eq <8 x i13> %a, <i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10>
@@ -480,8 +480,8 @@ define <8 x i32> @same_sext_used_in_cmp_signed_pred_and_select_v8i32(<8 x i16> %
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    cmgt.4s v3, v0, v1
 ; CHECK-NEXT:    cmgt.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
 entry:
   %ext = sext <8 x i16> %a to <8 x i32>
@@ -502,8 +502,8 @@ define <8 x i32> @same_sext_used_in_cmp_unsigned_pred_and_select_v8i32_from_v8i1
 ; CHECK-NEXT:    sshr.4s v2, v2, #17
 ; CHECK-NEXT:    cmge.4s v3, v2, v1
 ; CHECK-NEXT:    cmge.4s v1, v0, v1
-; CHECK-NEXT:    and.16b v1, v0, v1
-; CHECK-NEXT:    and.16b v0, v2, v3
+; CHECK-NEXT:    and.16b v1, v1, v0
+; CHECK-NEXT:    and.16b v0, v3, v2
 ; CHECK-NEXT:    ret
   %ext = sext <8 x i15> %a to <8 x i32>
   %cmp = icmp sge <8 x i15> %a, <i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10>
@@ -524,11 +524,11 @@ define <16 x i32> @same_sext_used_in_cmp_unsigned_pred_and_select(<16 x i8> %a)
 ; CHECK-NEXT:    ext.16b v5, v0, v0, #8
 ; CHECK-NEXT:    ext.16b v6, v3, v3, #8
 ; CHECK-NEXT:    ext.16b v7, v1, v1, #8
-; CHECK-NEXT:    and.8b v2, v2, v3
-; CHECK-NEXT:    and.8b v1, v0, v1
+; CHECK-NEXT:    and.8b v2, v3, v2
+; CHECK-NEXT:    and.8b v1, v1, v0
 ; CHECK-NEXT:    sshll.4s v0, v2, #0
-; CHECK-NEXT:    and.8b v3, v5, v7
-; CHECK-NEXT:    and.8b v4, v4, v6
+; CHECK-NEXT:    and.8b v3, v7, v5
+; CHECK-NEXT:    and.8b v4, v6, v4
 ; CHECK-NEXT:    sshll.4s v2, v1, #0
 ; CHECK-NEXT:    sshll.4s v3, v3, #0
 ; CHECK-NEXT:    sshll.4s v1, v4, #0
@@ -556,10 +556,10 @@ define <16 x i32> @same_zext_used_in_cmp_signed_pred_and_select_can_convert_to_u
 ; CHECK-NEXT:    sshll.4s v7, v1, #0
 ; CHECK-NEXT:    sshll2.4s v16, v3, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v0, v4, v0
-; CHECK-NEXT:    and.16b v3, v6, v1
-; CHECK-NEXT:    and.16b v1, v2, v16
-; CHECK-NEXT:    and.16b v2, v5, v7
+; CHECK-NEXT:    and.16b v0, v0, v4
+; CHECK-NEXT:    and.16b v3, v1, v6
+; CHECK-NEXT:    and.16b v1, v16, v2
+; CHECK-NEXT:    and.16b v2, v7, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i32>
@@ -604,10 +604,10 @@ define void @extension_in_loop_v16i8_to_v16i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    sshll.4s v6, v6, #0
 ; CHECK-NEXT:    sshll2.4s v19, v5, #0
 ; CHECK-NEXT:    sshll.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v7, v7, v17
-; CHECK-NEXT:    and.16b v6, v16, v6
-; CHECK-NEXT:    and.16b v16, v18, v19
-; CHECK-NEXT:    and.16b v4, v4, v5
+; CHECK-NEXT:    and.16b v7, v17, v7
+; CHECK-NEXT:    and.16b v6, v6, v16
+; CHECK-NEXT:    and.16b v16, v19, v18
+; CHECK-NEXT:    and.16b v4, v5, v4
 ; CHECK-NEXT:    stp q6, q7, [x1, #32]
 ; CHECK-NEXT:    stp q4, q16, [x1], #64
 ; CHECK-NEXT:    b.ne LBB24_1
@@ -674,10 +674,10 @@ define void @extension_in_loop_as_shuffle_v16i8_to_v16i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    sshll.4s v6, v6, #0
 ; CHECK-NEXT:    sshll2.4s v19, v5, #0
 ; CHECK-NEXT:    sshll.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v7, v7, v17
-; CHECK-NEXT:    and.16b v6, v16, v6
-; CHECK-NEXT:    and.16b v16, v18, v19
-; CHECK-NEXT:    and.16b v4, v4, v5
+; CHECK-NEXT:    and.16b v7, v17, v7
+; CHECK-NEXT:    and.16b v6, v6, v16
+; CHECK-NEXT:    and.16b v16, v19, v18
+; CHECK-NEXT:    and.16b v4, v5, v4
 ; CHECK-NEXT:    stp q6, q7, [x1, #32]
 ; CHECK-NEXT:    stp q4, q16, [x1], #64
 ; CHECK-NEXT:    b.ne LBB25_1
@@ -745,10 +745,10 @@ define void @shuffle_in_loop_is_no_extend_v16i8_to_v16i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    sshll.4s v6, v6, #0
 ; CHECK-NEXT:    sshll2.4s v19, v5, #0
 ; CHECK-NEXT:    sshll.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v7, v7, v17
-; CHECK-NEXT:    and.16b v6, v16, v6
-; CHECK-NEXT:    and.16b v16, v18, v19
-; CHECK-NEXT:    and.16b v4, v4, v5
+; CHECK-NEXT:    and.16b v7, v17, v7
+; CHECK-NEXT:    and.16b v6, v6, v16
+; CHECK-NEXT:    and.16b v16, v19, v18
+; CHECK-NEXT:    and.16b v4, v5, v4
 ; CHECK-NEXT:    stp q6, q7, [x1, #32]
 ; CHECK-NEXT:    stp q4, q16, [x1], #64
 ; CHECK-NEXT:    b.ne LBB26_1
diff --git a/llvm/test/CodeGen/AArch64/win-sve.ll b/llvm/test/CodeGen/AArch64/win-sve.ll
index 691230af3e67d..53ac9344175a3 100644
--- a/llvm/test/CodeGen/AArch64/win-sve.ll
+++ b/llvm/test/CodeGen/AArch64/win-sve.ll
@@ -65,14 +65,18 @@ define i32 @f(<vscale x 2 x i64> %x) {
 ; CHECK-NEXT:    .seh_save_zreg z22, 16
 ; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    .seh_save_zreg z23, 17
-; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    str x28, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 16
+; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x30, 8
 ; CHECK-NEXT:    .seh_endprologue
 ; CHECK-NEXT:    bl g
 ; CHECK-NEXT:    mov w0, #3 // =0x3
 ; CHECK-NEXT:    .seh_startepilogue
-; CHECK-NEXT:    ldp x29, x30, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    .seh_save_fplr 0
+; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
 ; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
@@ -365,8 +369,10 @@ define void @f3(i64 %n, <vscale x 2 x i64> %x) {
 ; CHECK-NEXT:    .seh_save_zreg z22, 16
 ; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    .seh_save_zreg z23, 17
-; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    str x28, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 16
+; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x30, 8
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
 ; CHECK-NEXT:    .seh_endprologue
@@ -376,8 +382,10 @@ define void @f3(i64 %n, <vscale x 2 x i64> %x) {
 ; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
-; CHECK-NEXT:    ldp x29, x30, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    .seh_save_fplr 0
+; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
 ; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
@@ -511,8 +519,10 @@ define void @f4(i64 %n, <vscale x 2 x i64> %x) {
 ; CHECK-NEXT:    .seh_save_zreg z22, 16
 ; CHECK-NEXT:    str z23, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    .seh_save_zreg z23, 17
-; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    str x28, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 16
+; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x30, 8
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
 ; CHECK-NEXT:    addvl sp, sp, #-1
@@ -526,8 +536,10 @@ define void @f4(i64 %n, <vscale x 2 x i64> %x) {
 ; CHECK-NEXT:    .seh_allocz 1
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
-; CHECK-NEXT:    ldp x29, x30, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    .seh_save_fplr 0
+; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 8
+; CHECK-NEXT:    ldr x28, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x28, 0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    .seh_stackalloc 16
 ; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
@@ -1093,8 +1105,10 @@ define void @f7(i64 %n) {
 ; CHECK-LABEL: f7:
 ; CHECK:       .seh_proc f7
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    str x28, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg_x x28, 16
+; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NEXT:    .seh_save_reg x30, 8
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    .seh_allocz 1
 ; CHECK-NEXT:    .seh_endprologue
@@ -1103,8 +1117,10 @@ define void @f7(i64 %n) {
 ; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    .seh_allocz 1
-; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    .seh_save_fplr_x 16
+; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg x30, 8
+; CHECK-NEXT:    ldr x28, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    .seh_save_reg_x x28, 16
 ; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:    .seh_endfunclet
diff --git a/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll b/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll
index 2002c37cb2528..6d14abdc2ed75 100644
--- a/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll
+++ b/llvm/test/CodeGen/AArch64/wincfi-missing-seh-directives.ll
@@ -5,25 +5,23 @@
 ; prologue has a corresponding seh directive.
 ;
 ; CHECK-NOT: error: Incorrect size for
-; CHECK: foo:
-; CHECK: .seh_proc foo
-; CHECK: sub     sp, sp, #288
-; CHECK: .seh_stackalloc 288
-; CHECK: str     x19, [sp]                       // 8-byte Folded Spill
-; CHECK: .seh_save_reg   x19, 0
-; CHECK: str     x21, [sp, #8]                   // 8-byte Folded Spill
-; CHECK: .seh_save_reg   x21, 8
-; CHECK: stp     x23, x24, [sp, #16]             // 16-byte Folded Spill
-; CHECK: .seh_save_regp  x23, 16
-; CHECK: stp     x25, x26, [sp, #32]             // 16-byte Folded Spill
-; CHECK: .seh_save_regp  x25, 32
-; CHECK: stp     x27, x28, [sp, #48]             // 16-byte Folded Spill
-; CHECK: .seh_save_regp  x27, 48
-; CHECK: stp     x29, x30, [sp, #64]             // 16-byte Folded Spill
-; CHECK: .seh_save_fplr  64
-; CHECK: sub     sp, sp, #224
-; CHECK: .seh_stackalloc 224
-; CHECK: .seh_endprologue
+; CHECK-LABEL:  foo:
+; CHECK-NEXT:   .seh_proc foo
+; CHECK:        sub     sp, sp, #496
+; CHECK-NEXT:   .seh_stackalloc 496
+; CHECK-NEXT:   str     x19, [sp, #208]                 // 8-byte Folded Spill
+; CHECK-NEXT:   .seh_save_reg   x19, 208
+; CHECK-NEXT:   str     x21, [sp, #216]                 // 8-byte Folded Spill
+; CHECK-NEXT:   .seh_save_reg   x21, 216
+; CHECK-NEXT:   stp     x23, x24, [sp, #224]            // 16-byte Folded Spill
+; CHECK-NEXT:   .seh_save_regp  x23, 224
+; CHECK-NEXT:   stp     x25, x26, [sp, #240]            // 16-byte Folded Spill
+; CHECK-NEXT:   .seh_save_regp  x25, 240
+; CHECK-NEXT:   stp     x27, x28, [sp, #256]            // 16-byte Folded Spill
+; CHECK-NEXT:   .seh_save_regp  x27, 256
+; CHECK-NEXT:   str     x30, [sp, #272]                 // 8-byte Folded Spill
+; CHECK-NEXT:   .seh_save_reg   x30, 272
+; CHECK-NEXT:   .seh_endprologue
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32"
 target triple = "aarch64-unknown-windows-msvc19.42.34436"
diff --git a/llvm/test/CodeGen/AArch64/wineh-aligned-stack-obj.ll b/llvm/test/CodeGen/AArch64/wineh-aligned-stack-obj.ll
new file mode 100644
index 0000000000000..ed633ce5e8f17
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/wineh-aligned-stack-obj.ll
@@ -0,0 +1,145 @@
+; RUN: llc %s --mtriple=aarch64-pc-windows-msvc -o - | FileCheck %s
+
+; Regression test for handling MSVC C++ exceptions when there's an aligned
+; object on the stack.
+
+; Generated from this C++ code:
+; https://godbolt.org/z/cGzGfqq34
+; > clang --target=aarch64-pc-windows-msvc test.cpp
+; ```
+; // Large object: alignment seems to be important?
+; struct alignas(128) BigObj {
+;     int value;
+;     // Destructor so it's kept alive.
+;     ~BigObj() { }
+; };
+;
+; // Exception type need to be large enough to not fit in a register.
+; struct Error {
+;     int value;
+;     int padding[3];
+; };
+;
+; int main() {
+;     BigObj bo{};
+;
+;     try {
+;         throw Error { 42, {0, 0, 0} };
+;     } catch (const Error& e) {
+;         return e.value;
+;     }
+;     return 0;
+; }
+; ```
+
+; CHECK-LABEL:  main:
+; CHECK:        sub x[[SPTMP:[0-9]+]], sp, #336
+; CHECK:        and sp, x[[SPTMP]], #0xffffffffffffff80
+; CHECK:        mov x[[FP:[0-9]+]], sp
+; CHECK:        str wzr, [x[[FP]], #332]
+
+; CHECK-LABEL:  "?catch$3@?0?main@4HA":
+; CHECK:        str	w8, [x[[FP]], #332]
+; CHECK-NEXT:   .seh_startepilogue
+; CHECK:        ret
+
+; CHECK-LABEL:  $cppxdata$main:
+; CHECK:        .word   -16                             // UnwindHelp
+; CHECK-LABEL:  $handlerMap$0$main:
+; CHECK-NEXT:   .word   8                               // Adjectives
+; CHECK-NEXT:   .word   "??_R0?AUError@@@8"@IMGREL      // Type
+; CHECK-NEXT:   .word   -8                              // CatchObjOffset
+; CHECK-NEXT:   .word   "?catch$3@?0?main@4HA"@IMGREL   // Handler
+
+%rtti.TypeDescriptor11 = type { ptr, ptr, [12 x i8] }
+%eh.CatchableType = type { i32, i32, i32, i32, i32, i32, i32 }
+%eh.CatchableTypeArray.1 = type { i32, [1 x i32] }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+%struct.BigObj = type { i32, [124 x i8] }
+%struct.Error = type { i32, [3 x i32] }
+
+$"??1BigObj@@QEAA@XZ" = comdat any
+
+$"??_R0?AUError@@@8" = comdat any
+
+$"_CT??_R0?AUError@@@816" = comdat any
+
+$"_CTA1?AUError@@" = comdat any
+
+$"_TI1?AUError@@" = comdat any
+
+@"??_7type_info@@6B@" = external constant ptr
+@"??_R0?AUError@@@8" = linkonce_odr global %rtti.TypeDescriptor11 { ptr @"??_7type_info@@6B@", ptr null, [12 x i8] c".?AUError@@\00" }, comdat
+@__ImageBase = external dso_local constant i8
+@"_CT??_R0?AUError@@@816" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (ptr @"??_R0?AUError@@@8" to i64), i64 ptrtoint (ptr @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 16, i32 0 }, section ".xdata", comdat
+@"_CTA1?AUError@@" = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (ptr @"_CT??_R0?AUError@@@816" to i64), i64 ptrtoint (ptr @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@"_TI1?AUError@@" = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (ptr @"_CTA1?AUError@@" to i64), i64 ptrtoint (ptr @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+
+define dso_local noundef i32 @main() personality ptr @__CxxFrameHandler3 {
+entry:
+  %retval = alloca i32, align 4
+  %bo = alloca %struct.BigObj, align 128
+  %tmp = alloca %struct.Error, align 4
+  %e = alloca ptr, align 8
+  %cleanup.dest.slot = alloca i32, align 4
+  store i32 0, ptr %retval, align 4
+  call void @llvm.memset.p0.i64(ptr align 128 %bo, i8 0, i64 128, i1 false)
+  %value = getelementptr inbounds nuw %struct.BigObj, ptr %bo, i32 0, i32 0
+  %value1 = getelementptr inbounds nuw %struct.Error, ptr %tmp, i32 0, i32 0
+  store i32 42, ptr %value1, align 4
+  %padding = getelementptr inbounds nuw %struct.Error, ptr %tmp, i32 0, i32 1
+  store i32 0, ptr %padding, align 4
+  %arrayinit.element = getelementptr inbounds i32, ptr %padding, i64 1
+  store i32 0, ptr %arrayinit.element, align 4
+  %arrayinit.element2 = getelementptr inbounds i32, ptr %padding, i64 2
+  store i32 0, ptr %arrayinit.element2, align 4
+  invoke void @_CxxThrowException(ptr %tmp, ptr @"_TI1?AUError@@") #3
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:
+  %0 = catchswitch within none [label %catch] unwind label %ehcleanup
+
+catch:
+  %1 = catchpad within %0 [ptr @"??_R0?AUError@@@8", i32 8, ptr %e]
+  %2 = load ptr, ptr %e, align 8
+  %value3 = getelementptr inbounds nuw %struct.Error, ptr %2, i32 0, i32 0
+  %3 = load i32, ptr %value3, align 4
+  store i32 %3, ptr %retval, align 4
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  catchret from %1 to label %catchret.dest
+
+catchret.dest:
+  br label %cleanup
+
+try.cont:
+  store i32 0, ptr %retval, align 4
+  store i32 1, ptr %cleanup.dest.slot, align 4
+  br label %cleanup
+
+cleanup:
+  call void @"??1BigObj@@QEAA@XZ"(ptr noundef nonnull align 128 dereferenceable(4) %bo) #4
+  %4 = load i32, ptr %retval, align 4
+  ret i32 %4
+
+ehcleanup:
+  %5 = cleanuppad within none []
+  call void @"??1BigObj@@QEAA@XZ"(ptr noundef nonnull align 128 dereferenceable(4) %bo) [ "funclet"(token %5) ]
+  cleanupret from %5 unwind to caller
+
+unreachable:
+  unreachable
+}
+
+declare void @llvm.memset.p0.i64(ptr writeonly captures(none), i8, i64, i1 immarg) #1
+
+declare dso_local void @_CxxThrowException(ptr, ptr)
+
+declare dso_local i32 @__CxxFrameHandler3(...)
+
+define linkonce_odr dso_local void @"??1BigObj@@QEAA@XZ"(ptr noundef nonnull align 128 dereferenceable(4) %this) unnamed_addr comdat {
+entry:
+  %this.addr = alloca ptr, align 8
+  store ptr %this, ptr %this.addr, align 8
+  %this1 = load ptr, ptr %this.addr, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame5.mir b/llvm/test/CodeGen/AArch64/wineh-frame5.mir
index 180c20f0148f5..0589d97ca64a2 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame5.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame5.mir
@@ -5,8 +5,10 @@
 # CHECK-LABEL:   bb.0.entry:
 # CHECK:         early-clobber $sp = frame-setup STRXpre killed $x19, $sp, -32
 # CHECK-NEXT:    frame-setup SEH_SaveReg_X 19, -32
-# CHECK-NEXT:    frame-setup STPXi killed $fp, killed $lr, $sp, 1
-# CHECK-NEXT:    frame-setup SEH_SaveFPLR 8
+# CHECK-NEXT:    frame-setup STRXui killed $x28, $sp, 1
+# CHECK-NEXT:    frame-setup SEH_SaveReg 28, 8
+# CHECK-NEXT:    frame-setup STRXui killed $lr, $sp, 2
+# CHECK-NEXT:    frame-setup SEH_SaveReg 30, 16
 # CHECK-NEXT:    $sp = frame-setup SUBXri $sp, 496, 0
 # CHECK-NEXT:    frame-setup SEH_StackAlloc 496
 # CHECK-NEXT:    frame-setup SEH_PrologEnd
@@ -15,8 +17,10 @@
 # CHECK:         frame-destroy SEH_EpilogStart
 # CHECK-NEXT:    $sp = frame-destroy ADDXri $sp, 496, 0
 # CHECK-NEXT:    frame-destroy SEH_StackAlloc 496
-# CHECK-NEXT:    $fp, $lr = frame-destroy LDPXi $sp, 1
-# CHECK-NEXT:    frame-destroy SEH_SaveFPLR 8
+# CHECK-NEXT:    $lr = frame-destroy LDRXui $sp, 2
+# CHECK-NEXT:    frame-destroy SEH_SaveReg 30, 16
+# CHECK-NEXT:    $x28 = frame-destroy LDRXui $sp, 1
+# CHECK-NEXT:    frame-destroy SEH_SaveReg 28, 8
 # CHECK-NEXT:    early-clobber $sp, $x19 = frame-destroy LDRXpost $sp, 32
 # CHECK-NEXT:    frame-destroy SEH_SaveReg_X 19, -32
 # CHECK-NEXT:    frame-destroy SEH_EpilogEnd
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame7.mir b/llvm/test/CodeGen/AArch64/wineh-frame7.mir
index 6d44ad3716111..b30afd2b57153 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame7.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame7.mir
@@ -3,13 +3,15 @@
 # Test that stack probe results in Nop unwind codes in the prologue.  Test
 # save_fplr, save_reg_x and stack_alloc with multiple updates.
 
-# CHECK:      early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2
-# CHECK-NEXT: frame-setup SEH_SaveFPLR_X -16
+# CHECK:      early-clobber $sp = frame-setup STRXpre killed $x28, $sp, -32
+# CHECK-NEXT: frame-setup SEH_SaveReg_X 28, -32
+# CHECK-NEXT: frame-setup STPXi killed $fp, killed $lr, $sp, 1
+# CHECK-NEXT: frame-setup SEH_SaveFPLR 8
 # CHECK-NEXT: $x15 = frame-setup MOVZXi 56009, 0
 # CHECK-NEXT: frame-setup SEH_Nop
 # CHECK-NEXT: $x15 = frame-setup MOVKXi $x15, 2, 16
 # CHECK-NEXT: frame-setup SEH_Nop
-# CHECK-NEXT: frame-setup BL &__chkstk, implicit-def $lr, implicit $sp, implicit $x15
+# CHECK-NEXT: frame-setup BL &__chkstk, implicit-def $lr, implicit $sp, implicit $x15, implicit-def dead $x16, implicit-def dead $x17, implicit-def dead $nzcv
 # CHECK-NEXT: frame-setup SEH_Nop
 # CHECK-NEXT: $sp = frame-setup SUBXrx64 killed $sp, killed $x15, 28
 # CHECK-NEXT: frame-setup SEH_StackAlloc 2993296
@@ -19,8 +21,10 @@
 # CHECK-NEXT: frame-destroy SEH_StackAlloc 2990080
 # CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 3216, 0
 # CHECK-NEXT: frame-destroy SEH_StackAlloc 3216
-# CHECK-NEXT: early-clobber $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2
-# CHECK-NEXT: frame-destroy SEH_SaveFPLR_X -16
+# CHECK-NEXT: $fp, $lr = frame-destroy LDPXi $sp, 1
+# CHECK-NEXT: frame-destroy SEH_SaveFPLR 8
+# CHECK-NEXT: early-clobber $sp, $x28 = frame-destroy LDRXpost $sp, 32
+# CHECK-NEXT: frame-destroy SEH_SaveReg_X 28, -32
 # CHECK-NEXT: frame-destroy SEH_EpilogEnd
 # CHECK-NEXT: RET_ReallyLR implicit killed $w0
 --- |
diff --git a/llvm/test/CodeGen/AArch64/wineh-reuse-catch-alloca.ll b/llvm/test/CodeGen/AArch64/wineh-reuse-catch-alloca.ll
new file mode 100644
index 0000000000000..18b8aab0c70e3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/wineh-reuse-catch-alloca.ll
@@ -0,0 +1,100 @@
+; RUN: llc %s --mtriple=aarch64-pc-windows-msvc -o - | FileCheck %s
+
+; Tests the fixed object layouts when two catchpads re-use the same stack
+; allocation for this catch objects.
+
+; Generated from this C++ code, with modifications to the IR (see comments in
+; IR):
+; https://godbolt.org/z/9qv5Yn68j
+; > clang --target=aarch64-pc-windows-msvc test.cpp
+; ```
+; extern "C" void boom();
+; extern "C" int calls_boom();
+; {
+;     try { boom(); }
+;     catch (int& i) { return i; }
+;     catch (long& l) { return l; }
+;     return 0;
+; }
+; ```
+
+; Only need 48 bytes on the stack, not 64.
+; CHECK-LABEL:  calls_boom:
+; CHECK:        sub     sp, sp, #48
+; CHECK:        .seh_stackalloc 48
+
+; Both the catch blocks load from the same address.
+; CHECK-LABEL:  "?catch$3@?0?calls_boom@4HA":
+; CHECK:        ldr     x8, [x29, #24]
+; CHECK-LABEL:  "?catch$4@?0?calls_boom@4HA":
+; CHECK:        ldr     x8, [x29, #24]
+
+; There's enough space for the UnwindHelp to be at -16 instead of -32
+; CHECK-LABEL:  $cppxdata$calls_boom:
+; CHECK:        .word   -16                             // UnwindHelp
+
+; Both catches have the same object offset.
+; CHECK-LABEL:  $handlerMap$0$calls_boom:
+; CHECK:        .word   -8                              // CatchObjOffset
+; CHECK-NEXT:   .word   "?catch$3@?0?calls_boom@4HA"@IMGREL // Handler
+; CHECK:        .word   -8                              // CatchObjOffset
+; CHECK-NEXT:   .word   "?catch$4@?0?calls_boom@4HA"@IMGREL // Handler
+
+%rtti.TypeDescriptor2 = type { ptr, ptr, [3 x i8] }
+
+$"??_R0H@8" = comdat any
+
+$"??_R0J@8" = comdat any
+
+@"??_7type_info@@6B@" = external constant ptr
+@"??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { ptr @"??_7type_info@@6B@", ptr null, [3 x i8] c".H\00" }, comdat
+@"??_R0J@8" = linkonce_odr global %rtti.TypeDescriptor2 { ptr @"??_7type_info@@6B@", ptr null, [3 x i8] c".J\00" }, comdat
+
+define dso_local i32 @calls_boom() personality ptr @__CxxFrameHandler3 {
+entry:
+  %retval = alloca i32, align 4
+; MODIFICATION: Remove unusued alloca
+;  %l = alloca ptr, align 8
+  %i = alloca ptr, align 8
+  invoke void @boom()
+          to label %invoke.cont unwind label %catch.dispatch
+
+catch.dispatch:
+  %0 = catchswitch within none [label %catch1, label %catch] unwind to caller
+
+catch1:
+  %1 = catchpad within %0 [ptr @"??_R0H@8", i32 8, ptr %i]
+  %2 = load ptr, ptr %i, align 8
+  %3 = load i32, ptr %2, align 4
+  store i32 %3, ptr %retval, align 4
+  catchret from %1 to label %catchret.dest2
+
+catch:
+; MODIFICATION: Use %i instead of %l
+  %4 = catchpad within %0 [ptr @"??_R0J@8", i32 8, ptr %i]
+  %5 = load ptr, ptr %i, align 8
+  %6 = load i32, ptr %5, align 4
+  store i32 %6, ptr %retval, align 4
+  catchret from %4 to label %catchret.dest
+
+invoke.cont:
+  br label %try.cont
+
+catchret.dest:
+  br label %return
+
+catchret.dest2:
+  br label %return
+
+try.cont:
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:
+  %7 = load i32, ptr %retval, align 4
+  ret i32 %7
+}
+
+declare dso_local void @boom() #1
+
+declare dso_local i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
index e10b05e2488fd..6bb22fca24591 100644
--- a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
@@ -20,7 +20,7 @@
 ; CHECK:       str     x28, [sp, #24]
 ; CHECK:       stp     x29, x30, [sp, #32]
 ; CHECK:       add     x29, sp, #32
-; CHECK:       sub     sp, sp, #624
+; CHECK:       sub     sp, sp, #608
 ; CHECK:       mov     x19, sp
 ; CHECK:       mov     x0, #-2
 ; CHECK:       stur    x0, [x29, #16]
@@ -51,7 +51,7 @@
 ; CHECK:       str     x21, [sp, #16]
 ; CHECK:       str     x28, [sp, #24]
 ; CHECK:       stp     x29, x30, [sp, #32]
-; CHECK:       add     x20, x19, #12
+; CHECK:       add     x20, x19, #0
 
 ; Check that there are no further stack updates.
 ; CHECK-NOT:   sub     sp, sp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
index 543f8e413abd8..e03aa18d3147f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
@@ -106,6 +106,104 @@ define <2 x half> @v_fmul_v2f16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %mul
 }
 
+define <2 x half> @v_fmul_v2f16_partial_neg(<2 x half> %a, <2 x half> %b) {
+; GFX9-LABEL: v_fmul_v2f16_partial_neg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_hi:[1,0]
+; GFX9-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_lo:[1,0]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fmul_v2f16_partial_neg:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80008000, v1
+; GFX8-NEXT:    v_mul_f16_e32 v3, v1, v0
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_mul_f16_e32 v1, v2, v3
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fmul_v2f16_partial_neg:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_hi:[1,0]
+; GFX10-NEXT:    v_pk_mul_f16 v0, v1, v0 neg_lo:[1,0]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = fneg <2 x half> %b3
+  %mul1 = fmul <2 x half> %b3, %a
+  %mul2 = fmul <2 x half> %b4, %mul1
+  ret <2 x half> %mul2
+}
+
+define <2 x half> @fmul_v2_half_neg_hi(<2 x half> %a, <2 x half> %b) #0 {
+; GFX9-LABEL: fmul_v2_half_neg_hi:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: fmul_v2_half_neg_hi:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fmul_v2_half_neg_hi:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = extractelement <2 x half> %b3, i64 1
+  %tmp = insertelement <2 x half> poison, half %b4, i64 0
+  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
+  %mul = fmul <2 x half> %a, %k
+  ret <2 x half> %mul
+}
+
+
+define <2 x half> @fmul_v2_half_neg_lo1(<2 x half> %a, <2 x half> %b) #0 {
+; GFX9-LABEL: fmul_v2_half_neg_lo1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: fmul_v2_half_neg_lo1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT:    v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fmul_v2_half_neg_lo1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_pk_mul_f16 v0, v0, v1 op_sel_hi:[1,0]
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %b1 = bitcast <2 x half> %b to float
+  %b2 = fneg float %b1
+  %b3 = bitcast float %b2 to <2 x half>
+  %b4 = extractelement <2 x half> %b3, i64 0
+  %tmp = insertelement <2 x half> poison, half %b4, i64 0
+  %k = shufflevector <2 x half> %tmp, <2 x half> %b, <2 x i32> <i32 2, i32 0>
+  %mul = fmul <2 x half> %a, %k
+  ret <2 x half> %mul
+}
+
 define <3 x half> @v_fmul_v3f16(<3 x half> %a, <3 x half> %b) {
 ; GFX9-LABEL: v_fmul_v3f16:
 ; GFX9:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index 744a5b7feb48d..8f0ae8c47098a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -304,8 +304,7 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_shuffle10_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_shuffle10_a:
@@ -319,8 +318,7 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-LABEL: v_sdot2_shuffle10_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
@@ -331,8 +329,7 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_sdot2_shuffle10_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_sdot2_shuffle10_b:
@@ -346,8 +343,7 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-LABEL: v_sdot2_shuffle10_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index 9e623494a5a04..287a009ca1405 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -289,22 +289,19 @@ define i32 @v_udot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_shuffle10_a:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_shuffle10_a:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_shuffle10_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[0,1,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
@@ -315,22 +312,19 @@ define i32 @v_udot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX906-LABEL: v_udot2_shuffle10_b:
 ; GFX906:       ; %bb.0:
 ; GFX906-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX906-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX906-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX908-LABEL: v_udot2_shuffle10_b:
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX908-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX908-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_udot2_shuffle10_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
+; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 op_sel:[0,1,0] op_sel_hi:[1,0,1]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
index 24ec4fa48f778..6ae2f56f6ae6d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
@@ -211,91 +211,41 @@ define i32 @v_urem_i32_oddk_denom(i32 %num) {
 ; CHECK-LABEL: v_urem_i32_oddk_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, 0x4996c7d8
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0xffed2705
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x12d8fb
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v2
-; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0xb2a50881
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v3
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 20, v1
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 0xffed2705, v0
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 0xffed2705, v0
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i32 %num, 1235195
   ret i32 %result
 }
 
 define <2 x i32> @v_urem_v2i32_oddk_denom(<2 x i32> %num) {
-; GISEL-LABEL: v_urem_v2i32_oddk_denom:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v3, 0x12d8fb
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffed2705
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_lo_u32 v5, v3, v4
-; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GISEL-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
-; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v0, v4
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 0xffed2705, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v0, v4
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 0xffed2705, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; CGP-LABEL: v_urem_v2i32_oddk_denom:
-; CGP:       ; %bb.0:
-; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v2, 0x4996c7d8
-; CGP-NEXT:    v_mov_b32_e32 v3, 0xffed2705
-; CGP-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
-; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v3
-; CGP-NEXT:    v_mul_hi_u32 v5, v2, v5
-; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, v2
-; CGP-NEXT:    v_mul_hi_u32 v2, v1, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v2, v2, v4
-; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
-; CGP-NEXT:    v_add_i32_e32 v2, vcc, v0, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, 0xffed2705, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; CGP-NEXT:    v_add_i32_e32 v2, vcc, v0, v3
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, 0xffed2705, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CGP-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-LABEL: v_urem_v2i32_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xb2a50881
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0x12d8fb
+; CHECK-NEXT:    v_mul_hi_u32 v4, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v4
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v1, v2
+; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 1, v5
+; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 1, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 20, v4
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 20, v2
+; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v3
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i32> %num, <i32 1235195, i32 1235195>
   ret <2 x i32> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index f6a228614a27e..2a1bf4bf068f0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -968,523 +968,106 @@ define i64 @v_urem_i64_oddk_denom(i64 %num) {
 ; CHECK-LABEL: v_urem_i64_oddk_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_ubyte0_e32 v4, 0
-; CHECK-NEXT:    v_mov_b32_e32 v5, 0xffed2705
-; CHECK-NEXT:    v_mac_f32_e32 v3, 0x4f800000, v4
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x5f7ffffc, v3
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x2f800000, v3
-; CHECK-NEXT:    v_trunc_f32_e32 v4, v4
-; CHECK-NEXT:    v_mac_f32_e32 v3, 0xcf800000, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_lo_u32 v6, v4, v5
-; CHECK-NEXT:    v_mul_lo_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v8, v3, v5
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v6, v3
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v7
-; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v7
-; CHECK-NEXT:    v_mul_hi_u32 v7, v4, v7
-; CHECK-NEXT:    v_mul_lo_u32 v10, v3, v6
-; CHECK-NEXT:    v_mul_lo_u32 v11, v4, v6
-; CHECK-NEXT:    v_mul_hi_u32 v12, v3, v6
-; CHECK-NEXT:    v_mul_hi_u32 v6, v4, v6
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
-; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
-; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CHECK-NEXT:    v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_lo_u32 v5, v4, v5
-; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v6
-; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v6
-; CHECK-NEXT:    v_mul_hi_u32 v6, v4, v6
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v5, v3
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT:    v_mul_lo_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_lo_u32 v10, v4, v5
-; CHECK-NEXT:    v_mul_hi_u32 v11, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v5, v4, v5
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
-; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
-; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
-; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v9
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT:    v_addc_u32_e32 v4, vcc, v4, v5, vcc
-; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v3
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0x1fb03c31
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0xd9528440
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
+; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v9, v0, v3
 ; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
-; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v4
-; CHECK-NEXT:    v_mul_lo_u32 v8, v1, v4
-; CHECK-NEXT:    v_mul_hi_u32 v9, v0, v4
-; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v8, v3
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v9
 ; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v8, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v2
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v2
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v6
-; CHECK-NEXT:    v_subb_u32_e64 v4, vcc, v1, v3, s[4:5]
-; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, -1, v3, s[6:7]
-; CHECK-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v2
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
-; CHECK-NEXT:    s_mov_b64 s[4:5], vcc
-; CHECK-NEXT:    v_subrev_i32_e32 v6, vcc, 0x12d8fb, v5
-; CHECK-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, -1, v2, s[4:5]
-; CHECK-NEXT:    v_subbrev_u32_e32 v7, vcc, 0, v1, vcc
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_lshr_b64 v[2:3], v[2:3], 20
+; CHECK-NEXT:    v_mul_lo_u32 v5, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i64 %num, 1235195
   ret i64 %result
 }
 
 define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
-; GISEL-LABEL: v_urem_v2i64_oddk_denom:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, 0x12d8fb
-; GISEL-NEXT:    v_cvt_f32_ubyte0_e32 v7, 0
-; GISEL-NEXT:    s_mov_b32 s4, 1
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0xffed2705
-; GISEL-NEXT:    s_mov_b32 s5, 1
-; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v7
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
-; GISEL-NEXT:    s_subb_u32 s6, 0, 0
-; GISEL-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
-; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v6
-; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
-; GISEL-NEXT:    s_subb_u32 s7, 0, 0
-; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
-; GISEL-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; GISEL-NEXT:    v_mul_lo_u32 v8, v7, v5
-; GISEL-NEXT:    v_mul_lo_u32 v9, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v10, s6, v6
-; GISEL-NEXT:    v_mul_hi_u32 v11, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v12, s7, v6
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v8
-; GISEL-NEXT:    v_mul_lo_u32 v13, v7, v9
-; GISEL-NEXT:    v_mul_hi_u32 v14, v6, v9
-; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
-; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v10
-; GISEL-NEXT:    v_mul_lo_u32 v12, v7, v10
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v10
-; GISEL-NEXT:    v_mul_hi_u32 v10, v7, v10
-; GISEL-NEXT:    v_mul_lo_u32 v16, v6, v8
-; GISEL-NEXT:    v_mul_lo_u32 v17, v7, v8
-; GISEL-NEXT:    v_mul_hi_u32 v18, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
-; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v17, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
-; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v18
-; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v19, v11
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
-; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v15
-; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v17, v18
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v15, v14
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v6, v11
-; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, v7, v10, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v5
-; GISEL-NEXT:    v_mul_lo_u32 v13, s6, v11
-; GISEL-NEXT:    v_mul_hi_u32 v14, v11, v5
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
-; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, v7, v8, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v9, s7, v6
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v16, v10, v5
-; GISEL-NEXT:    v_mul_lo_u32 v17, v10, v12
-; GISEL-NEXT:    v_mul_hi_u32 v18, v11, v12
-; GISEL-NEXT:    v_mul_hi_u32 v12, v10, v12
-; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v5
-; GISEL-NEXT:    v_mul_lo_u32 v19, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
-; GISEL-NEXT:    v_mul_hi_u32 v16, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v13, v14
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v15
-; GISEL-NEXT:    v_mul_lo_u32 v13, v11, v9
-; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
-; GISEL-NEXT:    v_mul_hi_u32 v15, v11, v9
-; GISEL-NEXT:    v_mul_hi_u32 v9, v10, v9
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v17, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v18
-; GISEL-NEXT:    v_mul_lo_u32 v13, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v18, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v19, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
-; GISEL-NEXT:    v_mul_hi_u32 v13, v6, v5
-; GISEL-NEXT:    v_mul_hi_u32 v5, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v18, v8
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
-; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
-; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v16, v13
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v17
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v10, v9, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v10, v1, v11
-; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v11
-; GISEL-NEXT:    v_mul_hi_u32 v11, v1, v11
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v7, v3, v6
-; GISEL-NEXT:    v_mul_hi_u32 v8, v2, v6
-; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GISEL-NEXT:    v_mul_lo_u32 v13, v0, v9
-; GISEL-NEXT:    v_mul_lo_u32 v14, v1, v9
-; GISEL-NEXT:    v_mul_hi_u32 v15, v0, v9
-; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
-; GISEL-NEXT:    v_mul_lo_u32 v16, v2, v5
-; GISEL-NEXT:    v_mul_lo_u32 v17, v3, v5
-; GISEL-NEXT:    v_mul_hi_u32 v18, v2, v5
-; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v14, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v16
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v17, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v18
-; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v16, v7
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v17, v8
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v4
-; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v4
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT:    v_mul_lo_u32 v7, v9, v4
-; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v12
-; GISEL-NEXT:    v_subb_u32_e64 v6, vcc, v1, v7, s[4:5]
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[6:7], v2, v8
-; GISEL-NEXT:    v_subb_u32_e64 v8, vcc, v3, v5, s[6:7]
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v2, v4
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[8:9], 0, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, -1, v7, s[8:9]
-; GISEL-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[4:5]
-; GISEL-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[6:7]
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT:    s_mov_b64 s[4:5], vcc
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 0x12d8fb, v9
-; GISEL-NEXT:    v_sub_i32_e64 v12, s[6:7], v0, v4
-; GISEL-NEXT:    v_subbrev_u32_e64 v1, s[6:7], 0, v1, s[6:7]
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; GISEL-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, -1, v13, s[4:5]
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[4:5], v12, v4
-; GISEL-NEXT:    v_subbrev_u32_e64 v14, s[4:5], 0, v1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GISEL-NEXT:    v_cndmask_b32_e64 v10, -1, v10, s[4:5]
-; GISEL-NEXT:    v_subbrev_u32_e32 v15, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v14, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
-; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; CGP-LABEL: v_urem_v2i64_oddk_denom:
-; CGP:       ; %bb.0:
-; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
-; CGP-NEXT:    v_cvt_f32_u32_e32 v5, 0x12d8fb
-; CGP-NEXT:    v_cvt_f32_ubyte0_e32 v6, 0
-; CGP-NEXT:    v_mov_b32_e32 v7, 0xffed2705
-; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v6
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v5
-; CGP-NEXT:    v_trunc_f32_e32 v6, v6
-; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; CGP-NEXT:    v_mul_lo_u32 v8, v6, v7
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v10, v5, v7
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v8, v5
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v9
-; CGP-NEXT:    v_mul_hi_u32 v11, v5, v9
-; CGP-NEXT:    v_mul_hi_u32 v9, v6, v9
-; CGP-NEXT:    v_mul_lo_u32 v12, v5, v8
-; CGP-NEXT:    v_mul_lo_u32 v13, v6, v8
-; CGP-NEXT:    v_mul_hi_u32 v14, v5, v8
-; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
-; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_lo_u32 v7, v6, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v8
-; CGP-NEXT:    v_mul_hi_u32 v11, v5, v8
-; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v7, v5
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_lo_u32 v12, v6, v7
-; CGP-NEXT:    v_mul_hi_u32 v13, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v7, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
-; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v11
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
-; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v7, vcc
-; CGP-NEXT:    v_mul_lo_u32 v7, v1, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v0, v5
-; CGP-NEXT:    v_mul_hi_u32 v9, v1, v5
-; CGP-NEXT:    v_mul_lo_u32 v10, v3, v5
-; CGP-NEXT:    v_mul_hi_u32 v11, v2, v5
-; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
-; CGP-NEXT:    v_mul_lo_u32 v12, v0, v6
-; CGP-NEXT:    v_mul_lo_u32 v13, v1, v6
-; CGP-NEXT:    v_mul_hi_u32 v14, v0, v6
-; CGP-NEXT:    v_mul_hi_u32 v15, v1, v6
-; CGP-NEXT:    v_mul_lo_u32 v16, v2, v6
-; CGP-NEXT:    v_mul_lo_u32 v17, v3, v6
-; CGP-NEXT:    v_mul_hi_u32 v18, v2, v6
-; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
-; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v16
-; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v17, v5
-; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v14
-; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v18
-; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v16, v10
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v17, v11
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT:    v_mul_lo_u32 v9, v7, v4
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v4
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT:    v_mul_lo_u32 v11, v5, v4
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v15, v8
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
-; CGP-NEXT:    v_mul_lo_u32 v8, v8, v4
-; CGP-NEXT:    v_mul_lo_u32 v6, v6, v4
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v9
-; CGP-NEXT:    v_subb_u32_e64 v6, vcc, v1, v7, s[4:5]
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; CGP-NEXT:    v_sub_i32_e64 v2, s[6:7], v2, v11
-; CGP-NEXT:    v_subb_u32_e64 v8, vcc, v3, v5, s[6:7]
-; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v2, v4
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[8:9], 0, v6
-; CGP-NEXT:    v_cndmask_b32_e64 v7, -1, v7, s[8:9]
-; CGP-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[4:5]
-; CGP-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[6:7]
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; CGP-NEXT:    s_mov_b64 s[4:5], vcc
-; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, 0x12d8fb, v9
-; CGP-NEXT:    v_sub_i32_e64 v12, s[6:7], v0, v4
-; CGP-NEXT:    v_subbrev_u32_e64 v1, s[6:7], 0, v1, s[6:7]
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; CGP-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CGP-NEXT:    v_cndmask_b32_e64 v13, -1, v13, s[4:5]
-; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v12, v4
-; CGP-NEXT:    v_subbrev_u32_e64 v14, s[4:5], 0, v1, s[4:5]
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
-; CGP-NEXT:    v_cndmask_b32_e64 v10, -1, v10, s[4:5]
-; CGP-NEXT:    v_subbrev_u32_e32 v15, vcc, 0, v3, vcc
-; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v14, vcc
-; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
-; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
-; CGP-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; CGP-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-LABEL: v_urem_v2i64_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x1fb03c31
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0xd9528440
+; CHECK-NEXT:    v_mov_b32_e32 v8, 0x12d8fb
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v0, v4
+; CHECK-NEXT:    v_mul_lo_u32 v10, v1, v5
+; CHECK-NEXT:    v_mul_hi_u32 v11, v1, v4
+; CHECK-NEXT:    v_mul_hi_u32 v12, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v13, v1, v5
+; CHECK-NEXT:    v_mul_lo_u32 v14, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v15, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v16, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v17, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v18, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v19, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v14, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v17, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v4, v18
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v14, v11
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v15, v4
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v7, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v13, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v19, v7
+; CHECK-NEXT:    v_lshr_b64 v[4:5], v[4:5], 20
+; CHECK-NEXT:    v_lshr_b64 v[6:7], v[6:7], 20
+; CHECK-NEXT:    v_mul_lo_u32 v9, v4, v8
+; CHECK-NEXT:    v_mul_lo_u32 v5, v5, v8
+; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v6, v8
+; CHECK-NEXT:    v_mul_lo_u32 v7, v7, v8
+; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v8
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v6
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v9
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; CHECK-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i64> %num, <i64 1235195, i64 1235195>
   ret <2 x i64> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/add_i1.ll b/llvm/test/CodeGen/AMDGPU/add_i1.ll
index ff1a3ee38be1d..c0d73fc14eb14 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i1.ll
@@ -1,11 +1,56 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
 
-; GCN-LABEL: {{^}}add_var_var_i1:
-; GFX9:  v_xor_b32_e32
-; GFX10: v_xor_b32_e32
 define amdgpu_kernel void @add_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; GFX9-LABEL: add_var_var_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v2, v0, s[6:7] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_var_var_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v2, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: add_var_var_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_u8 v2, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
   %add = add i1 %a, %b
@@ -13,21 +58,175 @@ define amdgpu_kernel void @add_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; GCN-LABEL: {{^}}add_var_imm_i1:
-; GFX9:  s_xor_b64
-; GFX10: s_xor_b32
 define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; GFX9-LABEL: add_var_imm_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_xor_b64 s[2:3], vcc, -1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_var_imm_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: add_var_imm_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in
   %add = add i1 %a, 1
   store i1 %add, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}add_i1_cf:
-; GCN: ; %endif
-; GFX9: s_xor_b64
-; GFX10: s_xor_b32
 define amdgpu_kernel void @add_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; GFX9-LABEL: add_i1_cf:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, 15, v0
+; GFX9-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %else
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[8:9] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
+; GFX9-NEXT:  .LBB2_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_4
+; GFX9-NEXT:  ; %bb.3: ; %if
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_andn2_b64 s[2:3], s[4:5], exec
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT:    s_or_b64 s[4:5], s[2:3], s[4:5]
+; GFX9-NEXT:  .LBB2_4: ; %endif
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b64 s[2:3], s[4:5], -1
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_i1_cf:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 15, v0
+; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_2
+; GFX10-NEXT:  ; %bb.1: ; %else
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 1, v0
+; GFX10-NEXT:  .LBB2_2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s5, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_4
+; GFX10-NEXT:  ; %bb.3: ; %if
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 s2, s4, exec_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s4, s2, s3
+; GFX10-NEXT:  .LBB2_4: ; %endif
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_xor_b32 s2, s4, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: add_i1_cf:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s7, exec_lo
+; GFX11-NEXT:    ; implicit-def: $sgpr6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmpx_lt_u32_e32 15, v0
+; GFX11-NEXT:    s_xor_b32 s7, exec_lo, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %else
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v0, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 1, v0
+; GFX11-NEXT:  .LBB2_2: ; %Flow
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s4, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_4
+; GFX11-NEXT:  ; %bb.3: ; %if
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_u8 v0, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_and_not1_b32 s2, s6, exec_lo
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-NEXT:    s_or_b32 s6, s2, s3
+; GFX11-NEXT:  .LBB2_4: ; %endif
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_xor_b32 s2, s6, -1
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %d_cmp = icmp ult i32 %tid, 16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir
new file mode 100644
index 0000000000000..22c913496b734
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir
@@ -0,0 +1,132 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -amdgpu-snop-padding=8 -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN8 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -amdgpu-snop-padding=16 -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN16 %s
+
+---
+name:            test_snop_padding
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: false
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+body:             |
+  ; GCN8-LABEL: name: test_snop_padding
+  ; GCN8: bb.0:
+  ; GCN8-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN8-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_BRANCH %bb.1
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT: bb.1:
+  ; GCN8-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; GCN8-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 10, implicit $exec
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_CBRANCH_EXECZ %bb.3, implicit $exec
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT: bb.2:
+  ; GCN8-NEXT:   successors: %bb.3(0x80000000)
+  ; GCN8-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   SI_SPILL_S32_SAVE killed $sgpr6, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_NOP 0
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   renamable $sgpr6 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 20, implicit $exec
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_BRANCH %bb.3
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT: bb.3:
+  ; GCN8-NEXT:   liveins: $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   $sgpr5 = V_READFIRSTLANE_B32 [[V_MOV_B32_e32_]], implicit $exec
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_STORE_DWORD_IMM $sgpr5, $sgpr10_sgpr11, 0, 0
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   SI_RETURN
+  ;
+  ; GCN16-LABEL: name: test_snop_padding
+  ; GCN16: bb.0:
+  ; GCN16-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN16-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_BRANCH %bb.1
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT: bb.1:
+  ; GCN16-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; GCN16-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 10, implicit $exec
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_CBRANCH_EXECZ %bb.3, implicit $exec
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT: bb.2:
+  ; GCN16-NEXT:   successors: %bb.3(0x80000000)
+  ; GCN16-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   SI_SPILL_S32_SAVE killed $sgpr6, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 0
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   renamable $sgpr6 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 20, implicit $exec
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_BRANCH %bb.3
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT: bb.3:
+  ; GCN16-NEXT:   liveins: $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   $sgpr5 = V_READFIRSTLANE_B32 [[V_MOV_B32_e32_]], implicit $exec
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_STORE_DWORD_IMM $sgpr5, $sgpr10_sgpr11, 0, 0
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   SI_RETURN
+  bb.0:
+    liveins: $sgpr6, $sgpr10_sgpr11
+    S_BRANCH %bb.1
+  bb.1:
+    liveins: $sgpr6, $sgpr10_sgpr11
+    %0:vgpr_32 = V_MOV_B32_e32 10, implicit $exec
+    S_CBRANCH_EXECZ %bb.3, implicit $exec
+  bb.2:
+    liveins: $sgpr6, $sgpr10_sgpr11
+    SI_SPILL_S32_SAVE killed $sgpr6, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_NOP 0
+    renamable $sgpr6 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    %0:vgpr_32 = V_MOV_B32_e32 20, implicit $exec
+    S_BRANCH %bb.3
+  bb.3:
+    liveins: $sgpr10_sgpr11
+    $sgpr5 = V_READFIRSTLANE_B32 %0:vgpr_32, implicit $exec
+    S_STORE_DWORD_IMM $sgpr5, $sgpr10_sgpr11, 0, 0
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
new file mode 100644
index 0000000000000..7ce5a00c0bf36
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -0,0 +1,639 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-attributor %s | FileCheck %s
+
+@gptr = addrspace(1) externally_initialized global i32 0, align 4
+@gptr2 = addrspace(4) externally_initialized global i32 0, align 4
+@gptr3 = addrspace(3) externally_initialized global i32 0, align 4
+
+define amdgpu_kernel void @no_alias_addr_space_select(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_select(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    [[ADD_A:%.*]] = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), i32 [[OFFSET]]
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[COND1]], ptr [[ADD_A]], ptr [[B]]
+; CHECK-NEXT:    [[PTR2:%.*]] = select i1 [[COND2]], ptr [[PTR]], ptr [[C]]
+; CHECK-NEXT:    store i32 [[VAL]], ptr [[PTR2]], align 4, !noalias.addrspace [[META0:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  %add_a = getelementptr inbounds i8, ptr %a, i32 %offset
+  %ptr = select i1 %cond1, ptr %add_a, ptr %b
+  %ptr2 = select i1 %cond2, ptr %ptr, ptr %c
+  store i32 %val, ptr %ptr2
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_branch(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_branch(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
+; CHECK:       [[BB_1_TRUE]]:
+; CHECK-NEXT:    [[A:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT:    br label %[[BB_1_END:.*]]
+; CHECK:       [[BB_1_FALSE]]:
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_1_END]]
+; CHECK:       [[BB_1_END]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[A]], %[[BB_1_TRUE]] ], [ [[B]], %[[BB_1_FALSE]] ]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BB_2_TRUE:.*]], label %[[BB_2_END:.*]]
+; CHECK:       [[BB_2_TRUE]]:
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_2_END]]
+; CHECK:       [[BB_2_END]]:
+; CHECK-NEXT:    [[PTR2:%.*]] = phi ptr [ [[PTR1]], %[[BB_1_END]] ], [ [[C]], %[[BB_2_TRUE]] ]
+; CHECK-NEXT:    store i32 [[VAL]], ptr [[PTR2]], align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond1, label %bb.1.true, label %bb.1.false
+bb.1.true:
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  br label %bb.1.end
+
+bb.1.false:
+  %lptr = alloca i32, align 4, addrspace(5)
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  br label %bb.1.end
+
+bb.1.end:
+  %ptr1 = phi ptr [ %a, %bb.1.true ], [ %b, %bb.1.false ]
+  br i1 %cond2, label %bb.2.true, label %bb.2.end
+
+bb.2.true:
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  br label %bb.2.end
+
+bb.2.end:
+  %ptr2 = phi ptr [ %ptr1, %bb.1.end ], [ %c, %bb.2.true ]
+  store i32 %val, ptr %ptr2
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_select_cmpxchg(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_select_cmpxchg(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    [[ADD_A:%.*]] = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), i32 [[OFFSET]]
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[COND1]], ptr [[ADD_A]], ptr [[B]]
+; CHECK-NEXT:    [[PTR2:%.*]] = select i1 [[COND2]], ptr [[PTR]], ptr [[C]]
+; CHECK-NEXT:    [[CMPXCHG_0:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_1:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_2:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_3:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_4:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_5:%.*]] = cmpxchg weak ptr [[PTR2]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[PTR2]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[PTR2]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  %add_a = getelementptr inbounds i8, ptr %a, i32 %offset
+  %ptr = select i1 %cond1, ptr %add_a, ptr %b
+  %ptr2 = select i1 %cond2, ptr %ptr, ptr %c
+  %cmpxchg.0 = cmpxchg ptr %ptr2, i32 0, i32 4 monotonic monotonic, align 4
+  %cmpxchg.1 = cmpxchg ptr %ptr2, i32 0, i32 5 acq_rel monotonic, align 4
+  %cmpxchg.2 = cmpxchg ptr %ptr2, i32 0, i32 6 acquire monotonic, align 4
+  %cmpxchg.3 = cmpxchg ptr %ptr2, i32 0, i32 7 release monotonic, align 4
+  %cmpxchg.4 = cmpxchg ptr %ptr2, i32 0, i32 8 seq_cst monotonic, align 4
+  %cmpxchg.5 = cmpxchg weak ptr %ptr2, i32 0, i32 9 seq_cst monotonic, align 4
+  %cmpxchg.6 = cmpxchg volatile ptr %ptr2, i32 0, i32 10 seq_cst monotonic, align 4
+  %cmpxchg.7 = cmpxchg weak volatile ptr %ptr2, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_branch_cmpxchg(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_branch_cmpxchg(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
+; CHECK:       [[BB_1_TRUE]]:
+; CHECK-NEXT:    [[A:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT:    br label %[[BB_1_END:.*]]
+; CHECK:       [[BB_1_FALSE]]:
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_1_END]]
+; CHECK:       [[BB_1_END]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[A]], %[[BB_1_TRUE]] ], [ [[B]], %[[BB_1_FALSE]] ]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BB_2_TRUE:.*]], label %[[BB_2_END:.*]]
+; CHECK:       [[BB_2_TRUE]]:
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_2_END]]
+; CHECK:       [[BB_2_END]]:
+; CHECK-NEXT:    [[PTR2:%.*]] = phi ptr [ [[PTR1]], %[[BB_1_END]] ], [ [[C]], %[[BB_2_TRUE]] ]
+; CHECK-NEXT:    [[CMPXCHG_0:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_1:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_2:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_3:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_4:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_5:%.*]] = cmpxchg weak ptr [[PTR2]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[PTR2]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[PTR2]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond1, label %bb.1.true, label %bb.1.false
+bb.1.true:
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  br label %bb.1.end
+
+bb.1.false:
+  %lptr = alloca i32, align 4, addrspace(5)
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  br label %bb.1.end
+
+bb.1.end:
+  %ptr1 = phi ptr [ %a, %bb.1.true ], [ %b, %bb.1.false ]
+  br i1 %cond2, label %bb.2.true, label %bb.2.end
+
+bb.2.true:
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  br label %bb.2.end
+
+bb.2.end:
+  %ptr2 = phi ptr [ %ptr1, %bb.1.end ], [ %c, %bb.2.true ]
+  %cmpxchg.0 = cmpxchg ptr %ptr2, i32 0, i32 4 monotonic monotonic, align 4
+  %cmpxchg.1 = cmpxchg ptr %ptr2, i32 0, i32 5 acq_rel monotonic, align 4
+  %cmpxchg.2 = cmpxchg ptr %ptr2, i32 0, i32 6 acquire monotonic, align 4
+  %cmpxchg.3 = cmpxchg ptr %ptr2, i32 0, i32 7 release monotonic, align 4
+  %cmpxchg.4 = cmpxchg ptr %ptr2, i32 0, i32 8 seq_cst monotonic, align 4
+  %cmpxchg.5 = cmpxchg weak ptr %ptr2, i32 0, i32 9 seq_cst monotonic, align 4
+  %cmpxchg.6 = cmpxchg volatile ptr %ptr2, i32 0, i32 10 seq_cst monotonic, align 4
+  %cmpxchg.7 = cmpxchg weak volatile ptr %ptr2, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_select_atomicrmw(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_select_atomicrmw(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    [[ADD_A:%.*]] = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), i32 [[OFFSET]]
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[COND1]], ptr [[ADD_A]], ptr [[B]]
+; CHECK-NEXT:    [[PTR2:%.*]] = select i1 [[COND2]], ptr [[PTR]], ptr [[C]]
+; CHECK-NEXT:    [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr [[PTR2]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr [[PTR2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr [[PTR2]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr [[PTR2]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr [[PTR2]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr [[PTR2]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr [[PTR2]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr [[PTR2]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[PTR2]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr [[PTR2]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[PTR2]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  %add_a = getelementptr inbounds i8, ptr %a, i32 %offset
+  %ptr = select i1 %cond1, ptr %add_a, ptr %b
+  %ptr2 = select i1 %cond2, ptr %ptr, ptr %c
+  %atomicrmw.xchg = atomicrmw xchg ptr %ptr2, i32 12 monotonic, align 4
+  %atomicrmw.add = atomicrmw add ptr %ptr2, i32 13 monotonic, align 4
+  %atomicrmw.sub = atomicrmw sub ptr %ptr2, i32 14 monotonic, align 4
+  %atomicrmw.and = atomicrmw and ptr %ptr2, i32 15 monotonic, align 4
+  %atomicrmw.nand = atomicrmw nand ptr %ptr2, i32 16 monotonic, align 4
+  %atomicrmw.or = atomicrmw or ptr %ptr2, i32 17 monotonic, align 4
+  %atomicrmw.xor = atomicrmw xor ptr %ptr2, i32 18 monotonic, align 4
+  %atomicrmw.max = atomicrmw max ptr %ptr2, i32 19 monotonic, align 4
+  %atomicrmw.min = atomicrmw volatile min ptr %ptr2, i32 20 monotonic, align 4
+  %atomicrmw.umax = atomicrmw umax ptr %ptr2, i32 21 syncscope("singlethread") monotonic, align 4
+  %atomicrmw.umin = atomicrmw volatile umin ptr %ptr2, i32 22 syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_branch_atomicrmw(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_branch_atomicrmw(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
+; CHECK:       [[BB_1_TRUE]]:
+; CHECK-NEXT:    [[A:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT:    br label %[[BB_1_END:.*]]
+; CHECK:       [[BB_1_FALSE]]:
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_1_END]]
+; CHECK:       [[BB_1_END]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[A]], %[[BB_1_TRUE]] ], [ [[B]], %[[BB_1_FALSE]] ]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BB_2_TRUE:.*]], label %[[BB_2_END:.*]]
+; CHECK:       [[BB_2_TRUE]]:
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_2_END]]
+; CHECK:       [[BB_2_END]]:
+; CHECK-NEXT:    [[PTR2:%.*]] = phi ptr [ [[PTR1]], %[[BB_1_END]] ], [ [[C]], %[[BB_2_TRUE]] ]
+; CHECK-NEXT:    [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr [[PTR2]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr [[PTR2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr [[PTR2]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr [[PTR2]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr [[PTR2]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr [[PTR2]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr [[PTR2]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr [[PTR2]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[PTR2]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr [[PTR2]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[PTR2]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond1, label %bb.1.true, label %bb.1.false
+bb.1.true:
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  br label %bb.1.end
+
+bb.1.false:
+  %lptr = alloca i32, align 4, addrspace(5)
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  br label %bb.1.end
+
+bb.1.end:
+  %ptr1 = phi ptr [ %a, %bb.1.true ], [ %b, %bb.1.false ]
+  br i1 %cond2, label %bb.2.true, label %bb.2.end
+
+bb.2.true:
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  br label %bb.2.end
+
+bb.2.end:
+  %ptr2 = phi ptr [ %ptr1, %bb.1.end ], [ %c, %bb.2.true ]
+  %atomicrmw.xchg = atomicrmw xchg ptr %ptr2, i32 12 monotonic, align 4
+  %atomicrmw.add = atomicrmw add ptr %ptr2, i32 13 monotonic, align 4
+  %atomicrmw.sub = atomicrmw sub ptr %ptr2, i32 14 monotonic, align 4
+  %atomicrmw.and = atomicrmw and ptr %ptr2, i32 15 monotonic, align 4
+  %atomicrmw.nand = atomicrmw nand ptr %ptr2, i32 16 monotonic, align 4
+  %atomicrmw.or = atomicrmw or ptr %ptr2, i32 17 monotonic, align 4
+  %atomicrmw.xor = atomicrmw xor ptr %ptr2, i32 18 monotonic, align 4
+  %atomicrmw.max = atomicrmw max ptr %ptr2, i32 19 monotonic, align 4
+  %atomicrmw.min = atomicrmw volatile min ptr %ptr2, i32 20 monotonic, align 4
+  %atomicrmw.umax = atomicrmw umax ptr %ptr2, i32 21 syncscope("singlethread") monotonic, align 4
+  %atomicrmw.umin = atomicrmw volatile umin ptr %ptr2, i32 22 syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_select_cmpxchg_flat(ptr %c, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_select_cmpxchg_flat(
+; CHECK-SAME: ptr [[C:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    [[ADD_A:%.*]] = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), i32 [[OFFSET]]
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[COND1]], ptr [[ADD_A]], ptr [[B]]
+; CHECK-NEXT:    [[PTR2:%.*]] = select i1 [[COND2]], ptr [[PTR]], ptr [[C]]
+; CHECK-NEXT:    [[CMPXCHG_0:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 4 monotonic monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_1:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 5 acq_rel monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_2:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 6 acquire monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_3:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 7 release monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_4:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 8 seq_cst monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_5:%.*]] = cmpxchg weak ptr [[PTR2]], i32 0, i32 9 seq_cst monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[PTR2]], i32 0, i32 10 seq_cst monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[PTR2]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  %add_a = getelementptr inbounds i8, ptr %a, i32 %offset
+  %ptr = select i1 %cond1, ptr %add_a, ptr %b
+  %ptr2 = select i1 %cond2, ptr %ptr, ptr %c
+  %cmpxchg.0 = cmpxchg ptr %ptr2, i32 0, i32 4 monotonic monotonic, align 4
+  %cmpxchg.1 = cmpxchg ptr %ptr2, i32 0, i32 5 acq_rel monotonic, align 4
+  %cmpxchg.2 = cmpxchg ptr %ptr2, i32 0, i32 6 acquire monotonic, align 4
+  %cmpxchg.3 = cmpxchg ptr %ptr2, i32 0, i32 7 release monotonic, align 4
+  %cmpxchg.4 = cmpxchg ptr %ptr2, i32 0, i32 8 seq_cst monotonic, align 4
+  %cmpxchg.5 = cmpxchg weak ptr %ptr2, i32 0, i32 9 seq_cst monotonic, align 4
+  %cmpxchg.6 = cmpxchg volatile ptr %ptr2, i32 0, i32 10 seq_cst monotonic, align 4
+  %cmpxchg.7 = cmpxchg weak volatile ptr %ptr2, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_branch_cmpxchg_flat(ptr %c, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_branch_cmpxchg_flat(
+; CHECK-SAME: ptr [[C:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
+; CHECK:       [[BB_1_TRUE]]:
+; CHECK-NEXT:    [[A:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT:    br label %[[BB_1_END:.*]]
+; CHECK:       [[BB_1_FALSE]]:
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_1_END]]
+; CHECK:       [[BB_1_END]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[A]], %[[BB_1_TRUE]] ], [ [[B]], %[[BB_1_FALSE]] ]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BB_2_TRUE:.*]], label %[[BB_2_END:.*]]
+; CHECK:       [[BB_2_TRUE]]:
+; CHECK-NEXT:    br label %[[BB_2_END]]
+; CHECK:       [[BB_2_END]]:
+; CHECK-NEXT:    [[PTR2:%.*]] = phi ptr [ [[PTR1]], %[[BB_1_END]] ], [ [[C]], %[[BB_2_TRUE]] ]
+; CHECK-NEXT:    [[CMPXCHG_0:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 4 monotonic monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_1:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 5 acq_rel monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_2:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 6 acquire monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_3:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 7 release monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_4:%.*]] = cmpxchg ptr [[PTR2]], i32 0, i32 8 seq_cst monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_5:%.*]] = cmpxchg weak ptr [[PTR2]], i32 0, i32 9 seq_cst monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[PTR2]], i32 0, i32 10 seq_cst monotonic, align 4
+; CHECK-NEXT:    [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[PTR2]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond1, label %bb.1.true, label %bb.1.false
+bb.1.true:
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  br label %bb.1.end
+
+bb.1.false:
+  %lptr = alloca i32, align 4, addrspace(5)
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  br label %bb.1.end
+
+bb.1.end:
+  %ptr1 = phi ptr [ %a, %bb.1.true ], [ %b, %bb.1.false ]
+  br i1 %cond2, label %bb.2.true, label %bb.2.end
+
+bb.2.true:
+  br label %bb.2.end
+
+bb.2.end:
+  %ptr2 = phi ptr [ %ptr1, %bb.1.end ], [ %c, %bb.2.true ]
+  %cmpxchg.0 = cmpxchg ptr %ptr2, i32 0, i32 4 monotonic monotonic, align 4
+  %cmpxchg.1 = cmpxchg ptr %ptr2, i32 0, i32 5 acq_rel monotonic, align 4
+  %cmpxchg.2 = cmpxchg ptr %ptr2, i32 0, i32 6 acquire monotonic, align 4
+  %cmpxchg.3 = cmpxchg ptr %ptr2, i32 0, i32 7 release monotonic, align 4
+  %cmpxchg.4 = cmpxchg ptr %ptr2, i32 0, i32 8 seq_cst monotonic, align 4
+  %cmpxchg.5 = cmpxchg weak ptr %ptr2, i32 0, i32 9 seq_cst monotonic, align 4
+  %cmpxchg.6 = cmpxchg volatile ptr %ptr2, i32 0, i32 10 seq_cst monotonic, align 4
+  %cmpxchg.7 = cmpxchg weak volatile ptr %ptr2, i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_select_atomicrmw_flat(ptr %c, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_select_atomicrmw_flat(
+; CHECK-SAME: ptr [[C:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    [[ADD_A:%.*]] = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), i32 [[OFFSET]]
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[COND1]], ptr [[ADD_A]], ptr [[B]]
+; CHECK-NEXT:    [[PTR2:%.*]] = select i1 [[COND2]], ptr [[PTR]], ptr [[C]]
+; CHECK-NEXT:    [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr [[PTR2]], i32 12 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr [[PTR2]], i32 13 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr [[PTR2]], i32 14 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr [[PTR2]], i32 15 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr [[PTR2]], i32 16 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr [[PTR2]], i32 17 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr [[PTR2]], i32 18 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr [[PTR2]], i32 19 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[PTR2]], i32 20 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr [[PTR2]], i32 21 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[PTR2]], i32 22 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  %add_a = getelementptr inbounds i8, ptr %a, i32 %offset
+  %ptr = select i1 %cond1, ptr %add_a, ptr %b
+  %ptr2 = select i1 %cond2, ptr %ptr, ptr %c
+  %atomicrmw.xchg = atomicrmw xchg ptr %ptr2, i32 12 monotonic, align 4
+  %atomicrmw.add = atomicrmw add ptr %ptr2, i32 13 monotonic, align 4
+  %atomicrmw.sub = atomicrmw sub ptr %ptr2, i32 14 monotonic, align 4
+  %atomicrmw.and = atomicrmw and ptr %ptr2, i32 15 monotonic, align 4
+  %atomicrmw.nand = atomicrmw nand ptr %ptr2, i32 16 monotonic, align 4
+  %atomicrmw.or = atomicrmw or ptr %ptr2, i32 17 monotonic, align 4
+  %atomicrmw.xor = atomicrmw xor ptr %ptr2, i32 18 monotonic, align 4
+  %atomicrmw.max = atomicrmw max ptr %ptr2, i32 19 monotonic, align 4
+  %atomicrmw.min = atomicrmw volatile min ptr %ptr2, i32 20 monotonic, align 4
+  %atomicrmw.umax = atomicrmw umax ptr %ptr2, i32 21 syncscope("singlethread") monotonic, align 4
+  %atomicrmw.umin = atomicrmw volatile umin ptr %ptr2, i32 22 syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_branch_atomicrmw_flat(ptr %c, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_branch_atomicrmw_flat(
+; CHECK-SAME: ptr [[C:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
+; CHECK:       [[BB_1_TRUE]]:
+; CHECK-NEXT:    [[A:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT:    br label %[[BB_1_END:.*]]
+; CHECK:       [[BB_1_FALSE]]:
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    br label %[[BB_1_END]]
+; CHECK:       [[BB_1_END]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[A]], %[[BB_1_TRUE]] ], [ [[B]], %[[BB_1_FALSE]] ]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BB_2_TRUE:.*]], label %[[BB_2_END:.*]]
+; CHECK:       [[BB_2_TRUE]]:
+; CHECK-NEXT:    br label %[[BB_2_END]]
+; CHECK:       [[BB_2_END]]:
+; CHECK-NEXT:    [[PTR2:%.*]] = phi ptr [ [[PTR1]], %[[BB_1_END]] ], [ [[C]], %[[BB_2_TRUE]] ]
+; CHECK-NEXT:    [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr [[PTR2]], i32 12 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr [[PTR2]], i32 13 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr [[PTR2]], i32 14 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr [[PTR2]], i32 15 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr [[PTR2]], i32 16 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr [[PTR2]], i32 17 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr [[PTR2]], i32 18 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr [[PTR2]], i32 19 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[PTR2]], i32 20 monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr [[PTR2]], i32 21 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT:    [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[PTR2]], i32 22 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond1, label %bb.1.true, label %bb.1.false
+bb.1.true:
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  br label %bb.1.end
+
+bb.1.false:
+  %lptr = alloca i32, align 4, addrspace(5)
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  br label %bb.1.end
+
+bb.1.end:
+  %ptr1 = phi ptr [ %a, %bb.1.true ], [ %b, %bb.1.false ]
+  br i1 %cond2, label %bb.2.true, label %bb.2.end
+
+bb.2.true:
+  br label %bb.2.end
+
+bb.2.end:
+  %ptr2 = phi ptr [ %ptr1, %bb.1.end ], [ %c, %bb.2.true ]
+  %atomicrmw.xchg = atomicrmw xchg ptr %ptr2, i32 12 monotonic, align 4
+  %atomicrmw.add = atomicrmw add ptr %ptr2, i32 13 monotonic, align 4
+  %atomicrmw.sub = atomicrmw sub ptr %ptr2, i32 14 monotonic, align 4
+  %atomicrmw.and = atomicrmw and ptr %ptr2, i32 15 monotonic, align 4
+  %atomicrmw.nand = atomicrmw nand ptr %ptr2, i32 16 monotonic, align 4
+  %atomicrmw.or = atomicrmw or ptr %ptr2, i32 17 monotonic, align 4
+  %atomicrmw.xor = atomicrmw xor ptr %ptr2, i32 18 monotonic, align 4
+  %atomicrmw.max = atomicrmw max ptr %ptr2, i32 19 monotonic, align 4
+  %atomicrmw.min = atomicrmw volatile min ptr %ptr2, i32 20 monotonic, align 4
+  %atomicrmw.umax = atomicrmw umax ptr %ptr2, i32 21 syncscope("singlethread") monotonic, align 4
+  %atomicrmw.umin = atomicrmw volatile umin ptr %ptr2, i32 22 syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
+; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
+; CHECK-NEXT:    [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
+; CHECK-NEXT:    store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
+; CHECK-NEXT:    [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr [[PTR5]], i32 12 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr [[PTR5]], i32 13 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr [[PTR5]], i32 14 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr [[PTR5]], i32 15 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr [[PTR5]], i32 16 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr [[PTR5]], i32 17 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr [[PTR5]], i32 18 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr [[PTR5]], i32 19 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[PTR5]], i32 20 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr [[PTR5]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[PTR5]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    ret void
+;
+  %ptr4 = select i1 %cond1, ptr %ptr1, ptr %ptr2
+  %ptr5 = select i1 %cond2, ptr %ptr4, ptr %ptr3
+  store i32 %val, ptr %ptr5
+  %atomicrmw.xchg = atomicrmw xchg ptr %ptr5, i32 12 monotonic, align 4
+  %atomicrmw.add = atomicrmw add ptr %ptr5, i32 13 monotonic, align 4
+  %atomicrmw.sub = atomicrmw sub ptr %ptr5, i32 14 monotonic, align 4
+  %atomicrmw.and = atomicrmw and ptr %ptr5, i32 15 monotonic, align 4
+  %atomicrmw.nand = atomicrmw nand ptr %ptr5, i32 16 monotonic, align 4
+  %atomicrmw.or = atomicrmw or ptr %ptr5, i32 17 monotonic, align 4
+  %atomicrmw.xor = atomicrmw xor ptr %ptr5, i32 18 monotonic, align 4
+  %atomicrmw.max = atomicrmw max ptr %ptr5, i32 19 monotonic, align 4
+  %atomicrmw.min = atomicrmw volatile min ptr %ptr5, i32 20 monotonic, align 4
+  %atomicrmw.umax = atomicrmw umax ptr %ptr5, i32 21 syncscope("singlethread") monotonic, align 4
+  %atomicrmw.umin = atomicrmw volatile umin ptr %ptr5, i32 22 syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
+; CHECK:       [[BB_1_TRUE]]:
+; CHECK-NEXT:    br label %[[BB_1_END:.*]]
+; CHECK:       [[BB_1_FALSE]]:
+; CHECK-NEXT:    br label %[[BB_1_END]]
+; CHECK:       [[BB_1_END]]:
+; CHECK-NEXT:    [[PTR4:%.*]] = phi ptr [ addrspacecast (ptr addrspace(1) @gptr to ptr), %[[BB_1_TRUE]] ], [ addrspacecast (ptr addrspace(4) @gptr2 to ptr), %[[BB_1_FALSE]] ]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BB_2_TRUE:.*]], label %[[BB_2_END:.*]]
+; CHECK:       [[BB_2_TRUE]]:
+; CHECK-NEXT:    br label %[[BB_2_END]]
+; CHECK:       [[BB_2_END]]:
+; CHECK-NEXT:    [[PTR5:%.*]] = phi ptr [ [[PTR4]], %[[BB_1_END]] ], [ addrspacecast (ptr addrspace(3) @gptr3 to ptr), %[[BB_2_TRUE]] ]
+; CHECK-NEXT:    store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr [[PTR5]], i32 12 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr [[PTR5]], i32 13 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr [[PTR5]], i32 14 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr [[PTR5]], i32 15 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr [[PTR5]], i32 16 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr [[PTR5]], i32 17 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr [[PTR5]], i32 18 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr [[PTR5]], i32 19 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[PTR5]], i32 20 monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr [[PTR5]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[PTR5]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META1]]
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond1, label %bb.1.true, label %bb.1.false
+bb.1.true:
+  br label %bb.1.end
+
+bb.1.false:
+  br label %bb.1.end
+
+bb.1.end:
+  %ptr4 = phi ptr [ %ptr1, %bb.1.true ], [ %ptr2, %bb.1.false ]
+  br i1 %cond2, label %bb.2.true, label %bb.2.end
+
+bb.2.true:
+  br label %bb.2.end
+
+bb.2.end:
+  %ptr5 = phi ptr [ %ptr4, %bb.1.end ], [ %ptr3, %bb.2.true ]
+  store i32 %val, ptr %ptr5
+  %atomicrmw.xchg = atomicrmw xchg ptr %ptr5, i32 12 monotonic, align 4
+  %atomicrmw.add = atomicrmw add ptr %ptr5, i32 13 monotonic, align 4
+  %atomicrmw.sub = atomicrmw sub ptr %ptr5, i32 14 monotonic, align 4
+  %atomicrmw.and = atomicrmw and ptr %ptr5, i32 15 monotonic, align 4
+  %atomicrmw.nand = atomicrmw nand ptr %ptr5, i32 16 monotonic, align 4
+  %atomicrmw.or = atomicrmw or ptr %ptr5, i32 17 monotonic, align 4
+  %atomicrmw.xor = atomicrmw xor ptr %ptr5, i32 18 monotonic, align 4
+  %atomicrmw.max = atomicrmw max ptr %ptr5, i32 19 monotonic, align 4
+  %atomicrmw.min = atomicrmw volatile min ptr %ptr5, i32 20 monotonic, align 4
+  %atomicrmw.umax = atomicrmw umax ptr %ptr5, i32 21 syncscope("singlethread") monotonic, align 4
+  %atomicrmw.umin = atomicrmw volatile umin ptr %ptr5, i32 22 syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
+; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT:    call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(4) @gptr2 to ptr
+  %c = addrspacecast ptr addrspace(3) @gptr3 to ptr
+  call void @callee_no_alias_addr_space_select(ptr %a, ptr %b, ptr %c, i1 %cond1, i1 %cond2, i32 %val)
+  call void @callee_alias_addr_space_branch(ptr %a, ptr %b, ptr %c, i1 %cond1, i1 %cond2, i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_unhandled(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_unhandled(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(3) [[SPTR]], align 4
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(5) [[LPTR]], align 4
+; CHECK-NEXT:    store i32 [[VAL]], ptr addrspace(1) @gptr, align 4
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  store i32 %val, ptr addrspace(3) %sptr
+  store i32 %val, ptr addrspace(5) %lptr
+  store i32 %val, ptr addrspace(1) @gptr
+  ret void
+}
+
+define amdgpu_kernel void @no_alias_addr_space_has_meta(ptr addrspace(3) %sptr, i1 %cond1, i1 %cond2, i32 %val, i32 %offset) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_alias_addr_space_has_meta(
+; CHECK-SAME: ptr addrspace(3) [[SPTR:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]], i32 [[OFFSET:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr addrspace(5) [[LPTR]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = addrspacecast ptr addrspace(3) [[SPTR]] to ptr
+; CHECK-NEXT:    [[ADD_A:%.*]] = getelementptr inbounds i8, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), i32 [[OFFSET]]
+; CHECK-NEXT:    [[PTR:%.*]] = select i1 [[COND1]], ptr [[ADD_A]], ptr [[B]]
+; CHECK-NEXT:    [[PTR2:%.*]] = select i1 [[COND2]], ptr [[PTR]], ptr [[C]]
+; CHECK-NEXT:    store i32 [[VAL]], ptr [[PTR2]], align 4, !noalias.addrspace [[META2:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %lptr = alloca i32, align 4, addrspace(5)
+  %a = addrspacecast ptr addrspace(1) @gptr to ptr
+  %b = addrspacecast ptr addrspace(5) %lptr to ptr
+  %c = addrspacecast ptr addrspace(3) %sptr to ptr
+  %add_a = getelementptr inbounds i8, ptr %a, i32 %offset
+  %ptr = select i1 %cond1, ptr %add_a, ptr %b
+  %ptr2 = select i1 %cond2, ptr %ptr, ptr %c
+  store i32 %val, ptr %ptr2, !noalias.addrspace !0
+  ret void
+}
+
+!0 = !{i32 2, i32 3, i32 4, i32 10}
+
+;.
+; CHECK: [[META0]] = !{i32 2, i32 3, i32 4, i32 5, i32 6, i32 10}
+; CHECK: [[META1]] = !{i32 2, i32 3, i32 5, i32 10}
+; CHECK: [[META2]] = !{i32 2, i32 3, i32 4, i32 10}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir b/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir
index ae3fa153f381a..9ff904cc31de8 100644
--- a/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir
+++ b/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir
@@ -10,3 +10,16 @@ body:             |
     ; GCN: S_CMP_LT_I32 65, 1, implicit-def $scc
     S_CMP_GT_I32 1, 65, implicit-def $scc
 ...
+
+---
+name:             shrink_icmp
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: shrink_icmp
+    ; GCN: %op0:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: %cond:sreg_64 = V_CMPX_GE_U32_e64 5, %op0, implicit-def $exec, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    %op0:sgpr_32 = COPY $sgpr0
+    %cond:sreg_64 = V_CMPX_LE_U32_e64 %op0, 5, implicit-def $exec, implicit $exec
+    S_NOP 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll b/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll
index b98c81db5da99..4b0fc9380b293 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll
@@ -1,13 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}and_i1_sext_bool:
-; GCN: v_cmp_{{gt|le}}_u32_e{{32|64}} [[CC:[^,]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: v_cndmask_b32_e{{32|64}} [[VAL:v[0-9]+]], 0, v{{[0-9]+}}, [[CC]]
-; GCN: store_dword {{.*}}[[VAL]]
-; GCN-NOT: v_cndmask_b32_e64 v{{[0-9]+}}, {{0|-1}}, {{0|-1}}
-; GCN-NOT: v_and_b32_e32
-
 define amdgpu_kernel void @and_i1_sext_bool(ptr addrspace(1) nocapture %arg) {
+; GCN-LABEL: and_i1_sext_bool:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, v0, v1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc
+; GCN-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT:    s_endpgm
 bb:
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
   %y = tail call i32 @llvm.amdgcn.workitem.id.y()
@@ -20,37 +28,40 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_fcmp:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_fcmp(float %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_fcmp:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp oeq float %x, 0.0
   %sext = sext i1 %cmp to i32
   %and = and i32 %sext, %y
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_fpclass:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 [[K:v[0-9]+]], 0x7b
-; GCN-NEXT: v_cmp_class_f32_e32 vcc, v0, [[K]]
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_fpclass(float %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_fpclass:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x7b
+; GCN-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %class = call i1 @llvm.is.fpclass(float %x, i32 123)
   %sext = sext i1 %class to i32
   %and = and i32 %sext, %y
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_uadd_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_uadd_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_uadd_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -58,12 +69,13 @@ define i32 @and_sext_bool_uadd_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_usub_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_usub_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_usub_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -71,15 +83,16 @@ define i32 @and_sext_bool_usub_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_sadd_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
-; GCN-NEXT: v_add_i32_e64 v2, s[4:5], v0, v1
-; GCN-NEXT: v_cmp_lt_i32_e64 s[4:5], v2, v0
-; GCN-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_sadd_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_sadd_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; GCN-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
+; GCN-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GCN-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -87,15 +100,16 @@ define i32 @and_sext_bool_sadd_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_ssub_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
-; GCN-NEXT: v_add_i32_e64 v2, s[4:5], v0, v1
-; GCN-NEXT: v_cmp_lt_i32_e64 s[4:5], v2, v0
-; GCN-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_ssub_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_ssub_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; GCN-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
+; GCN-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GCN-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -103,15 +117,16 @@ define i32 @and_sext_bool_ssub_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_smul_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_hi_i32 v2, v0, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v1
-; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, v2, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_smul_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_smul_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_i32 v2, v0, v1
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, v2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -119,13 +134,14 @@ define i32 @and_sext_bool_smul_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_umul_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_hi_u32 v0, v0, v1
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_umul_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_umul_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 51398a45055eb..f8e13fcdd2273 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -475,21 +475,28 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -500,6 +507,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -1035,10 +1043,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
@@ -2656,21 +2664,28 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -2681,6 +2696,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -3216,10 +3232,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll
index 4a3a44d09210e..ea28af4508382 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @s_ctpop_i32(ptr addrspace(1) noalias %out, i32 %val)
 
 ; GCN-LABEL: name:            s_ctpop_i64
 ; GCN: %[[BCNT:[0-9]+]]:sreg_32 = S_BCNT1_I32_B64
-; GCN: %[[SREG1:[0-9]+]]:sreg_32 = COPY %[[BCNT]]
+; GCN: %[[SREG1:[0-9]+]]:sreg_32 = COPY killed %[[BCNT]]
 ; GCN: %[[SREG2:[0-9]+]]:sreg_32 = S_MOV_B32 0
 ; GCN: REG_SEQUENCE killed %[[SREG1]], %subreg.sub0, killed %[[SREG2]], %subreg.sub1
 define amdgpu_kernel void @s_ctpop_i64(ptr addrspace(1) noalias %out, i64 %val) nounwind {
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 9ae7c4aaa1e95..b0439b1f7968f 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -2250,7 +2250,6 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
 ; GFX9-SDAG-NEXT:    s_mov_b32 s32, s34
 ; GFX9-SDAG-NEXT:    s_mov_b32 s34, s12
 ; GFX9-SDAG-NEXT:    s_mov_b32 s33, s11
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_control_flow:
@@ -2317,7 +2316,6 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
 ; GFX9-GISEL-NEXT:    s_mov_b32 s32, s34
 ; GFX9-GISEL-NEXT:    s_mov_b32 s34, s12
 ; GFX9-GISEL-NEXT:    s_mov_b32 s33, s11
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-LABEL: test_dynamic_stackalloc_device_control_flow:
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index bb66bb319d481..a07f1d8a02941 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -731,7 +731,6 @@ define <4 x i16> @vec_16xi16_extract_4xi16(ptr addrspace(1) %p0, ptr addrspace(1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
 ; GFX9-NEXT:  .LBB3_3: ; %exit
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v5 op_sel_hi:[0,0]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
@@ -973,7 +972,6 @@ define <4 x i16> @vec_16xi16_extract_4xi16_2(ptr addrspace(1) %p0, ptr addrspace
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
 ; GFX9-NEXT:  .LBB4_3: ; %exit
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v7 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
@@ -1217,7 +1215,6 @@ define <4 x half> @vec_16xf16_extract_4xf16(ptr addrspace(1) %p0, ptr addrspace(
 ; GFX9-NEXT:  .LBB5_3: ; %exit
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3d00
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cmp_ge_f16_e32 vcc, 0.5, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3800
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
@@ -1595,7 +1592,6 @@ define amdgpu_gfx <8 x i16> @vec_16xi16_extract_8xi16_0(i1 inreg %cond, ptr addr
 ; GFX9-NEXT:    s_movk_i32 s34, 0x3800
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3d00
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cmp_gt_u16_e32 vcc, s35, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; GFX9-NEXT:    v_cmp_gt_u16_sdwa vcc, v7, s34 src0_sel:WORD_1 src1_sel:DWORD
@@ -1933,7 +1929,6 @@ define amdgpu_gfx <8 x half> @vec_16xf16_extract_8xf16_0(i1 inreg %cond, ptr add
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3800
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x3d00
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cmp_ge_f16_e32 vcc, 0.5, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v2, vcc
 ; GFX9-NEXT:    v_cmp_nle_f16_sdwa vcc, v7, v0 src0_sel:WORD_1 src1_sel:DWORD
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
index 41082821bafe3..a8d94146195f4 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
@@ -127,7 +127,6 @@ define <2 x i64> @extract_2xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -138,7 +137,6 @@ define <2 x i64> @extract_2xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB1_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0xffff8000
 ; GCN-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, -1, v1, vcc
@@ -197,7 +195,6 @@ define <4 x i64> @extract_4xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -208,7 +205,6 @@ define <4 x i64> @extract_4xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB2_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0xffff8000
 ; GCN-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
@@ -305,7 +301,6 @@ define <8 x i64> @extract_8xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[6:7], 0, v[10:11]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[8:9], 0, v[12:13]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[10:11], 0, v[14:15]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[12:13], 0, v[16:17]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[14:15], 0, v[18:19]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[16:17], 0, v[4:5]
@@ -376,7 +371,6 @@ define <2 x double> @extract_2xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -387,7 +381,6 @@ define <2 x double> @extract_2xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB4_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0xbff00000
 ; GCN-NEXT:    v_cmp_lt_f64_e32 vcc, -1.0, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e64 v1, v0, -2.0, vcc
@@ -446,7 +439,6 @@ define <4 x double> @extract_4xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -457,7 +449,6 @@ define <4 x double> @extract_4xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB5_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0xbff00000
 ; GCN-NEXT:    v_cmp_nlt_f64_e32 vcc, -1.0, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, -2.0, v0, vcc
@@ -554,7 +545,6 @@ define <8 x double> @extract_8xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[6:7], -1.0, v[10:11]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[8:9], -1.0, v[12:13]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[10:11], -1.0, v[14:15]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[12:13], -1.0, v[16:17]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[14:15], -1.0, v[18:19]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[16:17], -1.0, v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/fma-combine.ll b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
index a96d022b66f12..c79cf87712dc0 100644
--- a/llvm/test/CodeGen/AMDGPU/fma-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
@@ -1,11 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-FMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-NOFMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FMA %s
-
-; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs  -enable-no-infs-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-FMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs  < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=ieee -verify-machineinstrs -enable-no-infs-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FMA %s
 
 ; Note: The SI-FMA conversions of type x * (y + 1) --> x * y + x would be
 ; beneficial even without fp32 denormals, but they do require no-infs-fp-math
@@ -65,8 +63,8 @@ define amdgpu_kernel void @combine_to_fma_f64_0(ptr addrspace(1) noalias %out, p
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fadd double %mul, %c
+  %mul = fmul contract double %a, %b
+  %fma = fadd contract double %mul, %c
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -134,9 +132,9 @@ define amdgpu_kernel void @combine_to_fma_f64_0_2use(ptr addrspace(1) noalias %o
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %fma0 = fadd double %mul, %c
-  %fma1 = fadd double %mul, %d
+  %mul = fmul contract double %a, %b
+  %fma0 = fadd contract double %mul, %c
+  %fma1 = fadd contract double %mul, %d
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -190,8 +188,8 @@ define amdgpu_kernel void @combine_to_fma_f64_1(ptr addrspace(1) noalias %out, p
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fadd double %c, %mul
+  %mul = fmul contract double %a, %b
+  %fma = fadd contract double %c, %mul
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -244,8 +242,8 @@ define amdgpu_kernel void @combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %o
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fsub double %mul, %c
+  %mul = fmul contract double %a, %b
+  %fma = fsub contract double %mul, %c
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -313,9 +311,9 @@ define amdgpu_kernel void @combine_to_fma_fsub_f64_0_2use(ptr addrspace(1) noali
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %fma0 = fsub double %mul, %c
-  %fma1 = fsub double %mul, %d
+  %mul = fmul contract double %a, %b
+  %fma0 = fsub contract double %mul, %c
+  %fma1 = fsub contract double %mul, %d
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -369,8 +367,8 @@ define amdgpu_kernel void @combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %o
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fsub double %c, %mul
+  %mul = fmul contract double %a, %b
+  %fma = fsub contract double %c, %mul
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -438,9 +436,9 @@ define amdgpu_kernel void @combine_to_fma_fsub_1_f64_2use(ptr addrspace(1) noali
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %fma0 = fsub double %c, %mul
-  %fma1 = fsub double %d, %mul
+  %mul = fmul contract double %a, %b
+  %fma0 = fsub contract double %c, %mul
+  %fma1 = fsub contract double %d, %mul
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -494,9 +492,9 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64(ptr addrspace(1) noalias %o
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %mul.neg = fsub double -0.0, %mul
-  %fma = fsub double %mul.neg, %c
+  %mul = fmul contract double %a, %b
+  %mul.neg = fsub contract double -0.0, %mul
+  %fma = fsub contract double %mul.neg, %c
 
   store double %fma, ptr addrspace(1) %gep.out
   ret void
@@ -565,10 +563,10 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_neg(ptr addrspace(1)
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %mul.neg = fsub double -0.0, %mul
-  %fma0 = fsub double %mul.neg, %c
-  %fma1 = fsub double %mul.neg, %d
+  %mul = fmul contract double %a, %b
+  %mul.neg = fsub contract double -0.0, %mul
+  %fma0 = fsub contract double %mul.neg, %c
+  %fma1 = fsub contract double %mul.neg, %d
 
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
@@ -638,10 +636,10 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_mul(ptr addrspace(1)
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %mul.neg = fsub double -0.0, %mul
-  %fma0 = fsub double %mul.neg, %c
-  %fma1 = fsub double %mul, %d
+  %mul = fmul contract double %a, %b
+  %mul.neg = fsub contract double -0.0, %mul
+  %fma0 = fsub contract double %mul.neg, %c
+  %fma1 = fsub contract double %mul, %d
 
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
@@ -650,32 +648,6 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_mul(ptr addrspace(1)
 
 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
 define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
-; SI-NOFMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
-; SI-NOFMA:       ; %bb.0:
-; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NOFMA-NEXT:    s_mov_b32 s6, 0
-; SI-NOFMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; SI-NOFMA-NEXT:    v_mov_b32_e32 v1, 0
-; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NOFMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
-; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
-; SI-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[8:9]
-; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
-; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
-; SI-NOFMA-NEXT:    s_endpgm
-;
 ; SI-FMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
 ; SI-FMA:       ; %bb.0:
 ; SI-FMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
@@ -701,30 +673,6 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1)
 ; SI-FMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
 ; SI-FMA-NEXT:    s_endpgm
 ;
-; GFX11-NOFMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
-; GFX11-NOFMA:       ; %bb.0:
-; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
-; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[6:7]
-; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
-; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
-; GFX11-NOFMA-NEXT:    s_endpgm
-;
 ; GFX11-FMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
 ; GFX11-FMA:       ; %bb.0:
 ; GFX11-FMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -761,18 +709,16 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1)
   %u = load volatile double, ptr addrspace(1) %gep.3
   %v = load volatile double, ptr addrspace(1) %gep.4
 
-  %tmp0 = fmul double %u, %v
-  %tmp1 = call double @llvm.fma.f64(double %x, double %y, double %tmp0) #0
-  %tmp2 = fsub double %tmp1, %z
+  %tmp0 = fmul contract fast double %u, %v
+  %tmp1 = call contract fast double @llvm.fma.f64(double %x, double %y, double %tmp0) #0
+  %tmp2 = fsub contract fast double %tmp1, %z
 
   store double %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
 
-; fold (fsub x, (fma y, z, (fmul u, v)))
-;   -> (fma (fneg y), z, (fma (fneg u), v, x))
-define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
-; SI-NOFMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
+define amdgpu_kernel void @no_aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_0_f64:
 ; SI-NOFMA:       ; %bb.0:
 ; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
@@ -793,11 +739,59 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
 ; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
 ; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
-; SI-NOFMA-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; SI-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
 ; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
 ; SI-NOFMA-NEXT:    s_endpgm
 ;
+; GFX11-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_0_f64:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[6:7]
+; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_endpgm
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr double, ptr addrspace(1) %gep.0, i32 2
+  %gep.3 = getelementptr double, ptr addrspace(1) %gep.0, i32 3
+  %gep.4 = getelementptr double, ptr addrspace(1) %gep.0, i32 4
+  %gep.out = getelementptr double, ptr addrspace(1) %out, i32 %tid
+
+  %x = load volatile double, ptr addrspace(1) %gep.0
+  %y = load volatile double, ptr addrspace(1) %gep.1
+  %z = load volatile double, ptr addrspace(1) %gep.2
+  %u = load volatile double, ptr addrspace(1) %gep.3
+  %v = load volatile double, ptr addrspace(1) %gep.4
+
+  %tmp0 = fmul double %u, %v
+  %tmp1 = call double @llvm.fma.f64(double %x, double %y, double %tmp0) #0
+  %tmp2 = fsub double %tmp1, %z
+
+  store double %tmp2, ptr addrspace(1) %gep.out
+  ret void
+}
+
+; fold (fsub x, (fma y, z, (fmul u, v)))
+;   -> (fma (fneg y), z, (fma (fneg u), v, x))
+define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
 ; SI-FMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
 ; SI-FMA:       ; %bb.0:
 ; SI-FMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
@@ -823,30 +817,6 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
 ; SI-FMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
 ; SI-FMA-NEXT:    s_endpgm
 ;
-; GFX11-NOFMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
-; GFX11-NOFMA:       ; %bb.0:
-; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
-; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
-; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
-; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
-; GFX11-NOFMA-NEXT:    s_endpgm
-;
 ; GFX11-FMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
 ; GFX11-FMA:       ; %bb.0:
 ; GFX11-FMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -883,6 +853,78 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
   %u = load volatile double, ptr addrspace(1) %gep.3
   %v = load volatile double, ptr addrspace(1) %gep.4
 
+  ; nsz flag is needed since this combine may change sign of zero
+  %tmp0 = fmul contract reassoc nsz double %u, %v
+  %tmp1 = call contract reassoc nsz double @llvm.fma.f64(double %y, double %z, double %tmp0) #0
+  %tmp2 = fsub contract reassoc nsz double %x, %tmp1
+
+  store double %tmp2, ptr addrspace(1) %gep.out
+  ret void
+}
+define amdgpu_kernel void @no_aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_1_f64:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s6, 0
+; SI-NOFMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NOFMA-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
+; SI-NOFMA-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_1_f64:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_endpgm
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr double, ptr addrspace(1) %gep.0, i32 2
+  %gep.3 = getelementptr double, ptr addrspace(1) %gep.0, i32 3
+  %gep.4 = getelementptr double, ptr addrspace(1) %gep.0, i32 4
+  %gep.out = getelementptr double, ptr addrspace(1) %out, i32 %tid
+
+  %x = load volatile double, ptr addrspace(1) %gep.0
+  %y = load volatile double, ptr addrspace(1) %gep.1
+  %z = load volatile double, ptr addrspace(1) %gep.2
+  %u = load volatile double, ptr addrspace(1) %gep.3
+  %v = load volatile double, ptr addrspace(1) %gep.4
+
   ; nsz flag is needed since this combine may change sign of zero
   %tmp0 = fmul nsz double %u, %v
   %tmp1 = call nsz double @llvm.fma.f64(double %y, double %z, double %tmp0) #0
@@ -979,8 +1021,8 @@ define amdgpu_kernel void @test_f32_mul_add_x_one_y(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load volatile float, ptr addrspace(1) %in1
   %y = load volatile float, ptr addrspace(1) %in2
-  %a = fadd float %x, 1.0
-  %m = fmul float %a, %y
+  %a = fadd contract float %x, 1.0
+  %m = fmul contract float %a, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1068,8 +1110,8 @@ define amdgpu_kernel void @test_f32_mul_y_add_x_one(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load volatile float, ptr addrspace(1) %in1
   %y = load volatile float, ptr addrspace(1) %in2
-  %a = fadd float %x, 1.0
-  %m = fmul float %y, %a
+  %a = fadd contract float %x, 1.0
+  %m = fmul contract float %y, %a
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1157,8 +1199,8 @@ define amdgpu_kernel void @test_f32_mul_add_x_negone_y(ptr addrspace(1) %out,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %a = fadd float %x, -1.0
-  %m = fmul float %a, %y
+  %a = fadd contract float %x, -1.0
+  %m = fmul contract float %a, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1246,8 +1288,8 @@ define amdgpu_kernel void @test_f32_mul_y_add_x_negone(ptr addrspace(1) %out,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %a = fadd float %x, -1.0
-  %m = fmul float %y, %a
+  %a = fadd contract float %x, -1.0
+  %m = fmul contract float %y, %a
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1335,8 +1377,8 @@ define amdgpu_kernel void @test_f32_mul_sub_one_x_y(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float 1.0, %x
-  %m = fmul float %s, %y
+  %s = fsub contract float 1.0, %x
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1424,8 +1466,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_one_x(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float 1.0, %x
-  %m = fmul float %y, %s
+  %s = fsub contract float 1.0, %x
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1513,8 +1555,8 @@ define amdgpu_kernel void @test_f32_mul_sub_negone_x_y(ptr addrspace(1) %out,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float -1.0, %x
-  %m = fmul float %s, %y
+  %s = fsub contract float -1.0, %x
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1602,8 +1644,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_negone_x(ptr addrspace(1) %out,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float -1.0, %x
-  %m = fmul float %y, %s
+  %s = fsub contract float -1.0, %x
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1691,8 +1733,8 @@ define amdgpu_kernel void @test_f32_mul_sub_x_one_y(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, 1.0
-  %m = fmul float %s, %y
+  %s = fsub contract float %x, 1.0
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1780,8 +1822,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_x_one(ptr addrspace(1) %out,
                                       ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, 1.0
-  %m = fmul float %y, %s
+  %s = fsub contract float %x, 1.0
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1869,8 +1911,8 @@ define amdgpu_kernel void @test_f32_mul_sub_x_negone_y(ptr addrspace(1) %out,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, -1.0
-  %m = fmul float %s, %y
+  %s = fsub contract float %x, -1.0
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1958,8 +2000,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_x_negone(ptr addrspace(1) %out,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, -1.0
-  %m = fmul float %y, %s
+  %s = fsub contract float %x, -1.0
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -2072,10 +2114,10 @@ define amdgpu_kernel void @test_f32_interp(ptr addrspace(1) %out,
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
   %t = load float, ptr addrspace(1) %in3
-  %t1 = fsub float 1.0, %t
-  %tx = fmul float %x, %t
-  %ty = fmul float %y, %t1
-  %r = fadd float %tx, %ty
+  %t1 = fsub contract float 1.0, %t
+  %tx = fmul contract float %x, %t
+  %ty = fmul contract float %y, %t1
+  %r = fadd contract float %tx, %ty
   store float %r, ptr addrspace(1) %out
   ret void
 }
@@ -2152,10 +2194,10 @@ define amdgpu_kernel void @test_f64_interp(ptr addrspace(1) %out,
   %x = load double, ptr addrspace(1) %in1
   %y = load double, ptr addrspace(1) %in2
   %t = load double, ptr addrspace(1) %in3
-  %t1 = fsub double 1.0, %t
-  %tx = fmul double %x, %t
-  %ty = fmul double %y, %t1
-  %r = fadd double %tx, %ty
+  %t1 = fsub contract double 1.0, %t
+  %tx = fmul contract double %x, %t
+  %ty = fmul contract double %y, %t1
+  %r = fadd contract double %tx, %ty
   store double %r, ptr addrspace(1) %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index 7c0d3692242a4..d4471c85c467c 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -73,8 +73,8 @@ define amdgpu_kernel void @multiple_fadd_use_test_f32(ptr addrspace(1) %out, flo
   %a16 = select i1 %a15, float %a12, float %a14
   %a17 = fmul float %a16, 2.0
   %a18 = fmul float %a17, %a17
-  %a19 = fmul float %a18, %a17
-  %a20 = fsub float 1.0, %a19
+  %a19 = fmul contract float %a18, %a17
+  %a20 = fsub contract float 1.0, %a19
   store float %a20, ptr addrspace(1) %out
   ret void
 }
@@ -540,8 +540,8 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
   %a16 = select i1 %a15, half %a12, half %a14
   %a17 = fmul half %a16, 2.0
   %a18 = fmul half %a17, %a17
-  %a19 = fmul half %a18, %a17
-  %a20 = fsub half 1.0, %a19
+  %a19 = fmul contract half %a18, %a17
+  %a20 = fsub contract half 1.0, %a19
   store half %a20, ptr addrspace(1) %out
   ret void
 }
@@ -1166,7 +1166,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f16(ptr addrspace(1) %out, i16 zeroext %x
   ret void
 }
 
-attributes #0 = { nounwind "unsafe-fp-math"="true" }
+attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GFX11-DENORM: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll
index d431503643d63..230b73a037221 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll
@@ -398,7 +398,7 @@ define amdgpu_kernel void @uniform_fneg_f64(ptr addrspace(1) %out, ptr addrspace
 ; GCN: %[[HI32:[0-9]+]]:sreg_32 = COPY %[[VREG64]].sub1
 ; GCN: %[[SREG_MASK:[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
 ; GCN: %[[XOR:[0-9]+]]:sreg_32 = S_XOR_B32 killed %[[HI32]], killed %[[SREG_MASK]]
-; GCN: %[[XOR_COPY:[0-9]+]]:sreg_32 = COPY %[[XOR]]
+; GCN: %[[XOR_COPY:[0-9]+]]:sreg_32 = COPY killed %[[XOR]]
 ; GCN: REG_SEQUENCE killed %[[LO32]], %subreg.sub0, killed %[[XOR_COPY]], %subreg.sub1
 
   %in.gep = getelementptr inbounds double, ptr addrspace(1) %in, i64 %idx
@@ -440,7 +440,7 @@ define amdgpu_kernel void @uniform_fabs_f64(ptr addrspace(1) %out, ptr addrspace
 ; GCN: %[[HI32:[0-9]+]]:sreg_32 = COPY %[[VREG64]].sub1
 ; GCN: %[[SREG_MASK:[0-9]+]]:sreg_32 = S_MOV_B32 2147483647
 ; GCN: %[[AND:[0-9]+]]:sreg_32 = S_AND_B32 killed %[[HI32]], killed %[[SREG_MASK]]
-; GCN: %[[AND_COPY:[0-9]+]]:sreg_32 = COPY %[[AND]]
+; GCN: %[[AND_COPY:[0-9]+]]:sreg_32 = COPY killed %[[AND]]
 ; GCN: REG_SEQUENCE killed %[[LO32]], %subreg.sub0, killed %[[AND_COPY]], %subreg.sub1
 
 
@@ -484,7 +484,7 @@ define amdgpu_kernel void @uniform_fneg_fabs_f64(ptr addrspace(1) %out, ptr addr
 ; GCN: %[[HI32:[0-9]+]]:sreg_32 = COPY %[[VREG64]].sub1
 ; GCN: %[[SREG_MASK:[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
 ; GCN: %[[OR:[0-9]+]]:sreg_32 = S_OR_B32 killed %[[HI32]], killed %[[SREG_MASK]]
-; GCN: %[[OR_COPY:[0-9]+]]:sreg_32 = COPY %[[OR]]
+; GCN: %[[OR_COPY:[0-9]+]]:sreg_32 = COPY killed %[[OR]]
 ; GCN: REG_SEQUENCE killed %[[LO32]], %subreg.sub0, killed %[[OR_COPY]], %subreg.sub1
 
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 82c58394c03bb..37756d15861be 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -5736,966 +5736,469 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-NEXT:  .LBB9_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[0:1], v[3:4], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-NEXT:  .LBB9_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-NEXT:  .LBB9_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-NEXT:  .LBB9_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-NEXT:  .LBB9_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_mov_b32 s2, 0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-NEXT:  .LBB9_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-DPP-NEXT:  .LBB9_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[3:4], v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-DPP-NEXT:  .LBB9_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-DPP-NEXT:  .LBB9_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-DPP-NEXT:  .LBB9_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
-; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-DPP-NEXT:  .LBB9_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-DPP-NEXT:  .LBB9_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -6705,7 +6208,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB10_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -6715,102 +6218,70 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX7LESS-NEXT:  .LBB10_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -6823,99 +6294,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX9-NEXT:  .LBB10_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -6928,93 +6364,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1064-NEXT:  .LBB10_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7023,61 +6423,32 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1032-NEXT:  .LBB10_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -7085,31 +6456,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7118,7 +6482,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -7133,81 +6497,48 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1164-NEXT:  .LBB10_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7218,522 +6549,358 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
-; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
-; GFX1132-NEXT:  .LBB10_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
+; GFX1132-NEXT:  .LBB10_5:
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -7741,238 +6908,175 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value()
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
@@ -11760,998 +10864,555 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-NEXT:  .LBB16_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-NEXT:  .LBB16_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s14, -1
+; GFX1064-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-NEXT:  .LBB16_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s14, -1
+; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-NEXT:  .LBB16_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    s_clause 0x1
-; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT:    s_clause 0x1
+; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-NEXT:  .LBB16_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_clause 0x1
-; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT:    s_clause 0x1
+; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-NEXT:  .LBB16_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-DPP-NEXT:  .LBB16_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-DPP-NEXT:  .LBB16_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-DPP-NEXT:  .LBB16_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-DPP-NEXT:  .LBB16_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    s_clause 0x1
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_clause 0x1
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-DPP-NEXT:  .LBB16_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_clause 0x1
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT:    s_clause 0x1
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-DPP-NEXT:  .LBB16_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -12761,7 +11422,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB17_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -12771,102 +11432,70 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX7LESS-NEXT:  .LBB17_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -12879,99 +11508,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX9-NEXT:  .LBB17_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -12984,93 +11578,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1064-NEXT:  .LBB17_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13079,61 +11637,32 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1032-NEXT:  .LBB17_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -13141,31 +11670,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13174,7 +11696,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -13189,81 +11711,48 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1164-NEXT:  .LBB17_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13274,522 +11763,358 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1132-NEXT:  .LBB17_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7LESS-DPP-NEXT:  .LBB17_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX9-DPP-NEXT:  .LBB17_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1064-DPP-NEXT:  .LBB17_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1032-DPP-NEXT:  .LBB17_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -13797,238 +12122,175 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1164-DPP-NEXT:  .LBB17_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1132-DPP-NEXT:  .LBB17_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value() strictfp
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
@@ -14374,7 +12636,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
 ; GFX1132-DPP-NEXT:    global_atomic_add_f32 v1, v0, s[2:3]
 ; GFX1132-DPP-NEXT:  .LBB18_2:
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0  monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0  monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index f8f911b693e09..c8b24f71d5735 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -3134,932 +3134,377 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-NEXT:  .LBB6_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-NEXT:  .LBB6_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-NEXT:  .LBB6_3:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB6_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-NEXT:  .LBB6_3:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB6_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-NEXT:  .LBB6_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-NEXT:  .LBB6_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-DPP-NEXT:  .LBB6_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-DPP-NEXT:  .LBB6_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-DPP-NEXT:  .LBB6_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB6_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-DPP-NEXT:  .LBB6_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB6_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-DPP-NEXT:  .LBB6_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-DPP-NEXT:  .LBB6_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -4067,7 +3512,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB7_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -4077,109 +3522,77 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX7LESS-NEXT:  .LBB7_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4189,90 +3602,55 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX9-NEXT:  .LBB7_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -4296,93 +3674,42 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1064-NEXT:  .LBB7_5:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB7_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -4402,96 +3729,45 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1032-NEXT:  .LBB7_5:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB7_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -4500,7 +3776,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4511,89 +3787,57 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1164-NEXT:  .LBB7_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -4601,811 +3845,557 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1132-NEXT:  .LBB7_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB7_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    s_nop 1
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX9-DPP-NEXT:  .LBB7_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1064-DPP-NEXT:  .LBB7_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB7_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1032-DPP-NEXT:  .LBB7_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB7_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1164-DPP-NEXT:  .LBB7_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1132-DPP-NEXT:  .LBB7_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
@@ -6680,932 +5670,377 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-NEXT:  .LBB10_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-NEXT:  .LBB10_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-NEXT:  .LBB10_3:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB10_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-NEXT:  .LBB10_3:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB10_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-NEXT:  .LBB10_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-NEXT:  .LBB10_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-DPP-NEXT:  .LBB10_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-DPP-NEXT:  .LBB10_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB10_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-DPP-NEXT:  .LBB10_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB10_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -7613,7 +6048,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB11_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -7623,109 +6058,77 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX7LESS-NEXT:  .LBB11_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -7735,90 +6138,55 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX9-NEXT:  .LBB11_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -7842,93 +6210,42 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1064-NEXT:  .LBB11_5:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB11_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -7948,96 +6265,45 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1032-NEXT:  .LBB11_5:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB11_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -8046,7 +6312,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8057,89 +6323,57 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1164-NEXT:  .LBB11_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -8147,811 +6381,557 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1132-NEXT:  .LBB11_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX9-DPP-NEXT:  .LBB11_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1064-DPP-NEXT:  .LBB11_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB11_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1032-DPP-NEXT:  .LBB11_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB11_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1164-DPP-NEXT:  .LBB11_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1132-DPP-NEXT:  .LBB11_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 1f76a476107a3..4fccfc051d56d 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -3134,932 +3134,377 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-NEXT:  .LBB6_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-NEXT:  .LBB6_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-NEXT:  .LBB6_3:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB6_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-NEXT:  .LBB6_3:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB6_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-NEXT:  .LBB6_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-NEXT:  .LBB6_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-DPP-NEXT:  .LBB6_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-DPP-NEXT:  .LBB6_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-DPP-NEXT:  .LBB6_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB6_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-DPP-NEXT:  .LBB6_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB6_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-DPP-NEXT:  .LBB6_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-DPP-NEXT:  .LBB6_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -4067,7 +3512,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB7_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -4077,109 +3522,77 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX7LESS-NEXT:  .LBB7_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4189,90 +3602,55 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX9-NEXT:  .LBB7_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -4296,93 +3674,42 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1064-NEXT:  .LBB7_5:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB7_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -4402,96 +3729,45 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1032-NEXT:  .LBB7_5:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB7_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -4500,7 +3776,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4511,89 +3787,57 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1164-NEXT:  .LBB7_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -4601,811 +3845,557 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1132-NEXT:  .LBB7_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB7_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    s_nop 1
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_min_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX9-DPP-NEXT:  .LBB7_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1064-DPP-NEXT:  .LBB7_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB7_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1032-DPP-NEXT:  .LBB7_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB7_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1164-DPP-NEXT:  .LBB7_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1132-DPP-NEXT:  .LBB7_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
@@ -6680,932 +5670,377 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-NEXT:  .LBB10_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-NEXT:  .LBB10_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-NEXT:  .LBB10_3:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB10_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-NEXT:  .LBB10_3:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB10_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-NEXT:  .LBB10_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-NEXT:  .LBB10_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-DPP-NEXT:  .LBB10_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-DPP-NEXT:  .LBB10_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB10_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-DPP-NEXT:  .LBB10_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB10_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -7613,7 +6048,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB11_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -7623,109 +6058,77 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX7LESS-NEXT:  .LBB11_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -7735,90 +6138,55 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX9-NEXT:  .LBB11_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -7842,93 +6210,42 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1064-NEXT:  .LBB11_5:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB11_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -7948,96 +6265,45 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1032-NEXT:  .LBB11_5:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB11_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -8046,7 +6312,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8057,89 +6323,57 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1164-NEXT:  .LBB11_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -8147,811 +6381,557 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1132-NEXT:  .LBB11_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_min_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX9-DPP-NEXT:  .LBB11_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1064-DPP-NEXT:  .LBB11_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB11_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1032-DPP-NEXT:  .LBB11_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB11_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1164-DPP-NEXT:  .LBB11_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1132-DPP-NEXT:  .LBB11_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 9db3c37045ccf..bb119ebd11daa 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -6064,966 +6064,469 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-NEXT:  .LBB9_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[0:1], v[3:4], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-NEXT:  .LBB9_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-NEXT:  .LBB9_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-NEXT:  .LBB9_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-NEXT:  .LBB9_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_mov_b32 s2, 0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-NEXT:  .LBB9_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-DPP-NEXT:  .LBB9_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[3:4], -v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-DPP-NEXT:  .LBB9_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-DPP-NEXT:  .LBB9_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-DPP-NEXT:  .LBB9_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-DPP-NEXT:  .LBB9_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-DPP-NEXT:  .LBB9_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -7033,7 +6536,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB10_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -7043,102 +6546,70 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX7LESS-NEXT:  .LBB10_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -7151,99 +6622,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX9-NEXT:  .LBB10_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -7256,93 +6692,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1064-NEXT:  .LBB10_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7351,61 +6751,32 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1032-NEXT:  .LBB10_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -7413,31 +6784,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7446,7 +6810,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -7461,81 +6825,48 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1164-NEXT:  .LBB10_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7546,522 +6877,358 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1132-NEXT:  .LBB10_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
-; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
+; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], -v[0:1]
+; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], -s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -8069,238 +7236,175 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value()
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8
   ret void
 }
 
@@ -12086,998 +11190,555 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-NEXT:  .LBB16_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-NEXT:  .LBB16_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s14, -1
+; GFX1064-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-NEXT:  .LBB16_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s14, -1
+; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-NEXT:  .LBB16_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    s_clause 0x1
-; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT:    s_clause 0x1
+; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-NEXT:  .LBB16_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_clause 0x1
-; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT:    s_clause 0x1
+; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-NEXT:  .LBB16_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-DPP-NEXT:  .LBB16_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-DPP-NEXT:  .LBB16_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-DPP-NEXT:  .LBB16_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-DPP-NEXT:  .LBB16_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    s_clause 0x1
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_clause 0x1
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-DPP-NEXT:  .LBB16_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_clause 0x1
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT:    s_clause 0x1
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-DPP-NEXT:  .LBB16_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 8
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -13087,7 +11748,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB17_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -13097,102 +11758,70 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX7LESS-NEXT:  .LBB17_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -13205,99 +11834,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX9-NEXT:  .LBB17_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -13310,93 +11904,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1064-NEXT:  .LBB17_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13405,61 +11963,32 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1032-NEXT:  .LBB17_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -13467,31 +11996,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13500,7 +12022,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -13515,81 +12037,48 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1164-NEXT:  .LBB17_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13600,522 +12089,358 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1132-NEXT:  .LBB17_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7LESS-DPP-NEXT:  .LBB17_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], -v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], -s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX9-DPP-NEXT:  .LBB17_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1064-DPP-NEXT:  .LBB17_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1032-DPP-NEXT:  .LBB17_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -14123,238 +12448,175 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1164-DPP-NEXT:  .LBB17_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1132-DPP-NEXT:  .LBB17_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value() strictfp
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 8
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/icmp.i16.ll b/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
index 6a4ae7f4e0d78..f9dcd92a3e519 100644
--- a/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
@@ -1,18 +1,95 @@
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s| FileCheck -check-prefixes=GCN,GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s| FileCheck -check-prefixes=GCN,GFX11-TRUE16 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s| FileCheck -check-prefix=GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s| FileCheck -check-prefix=GFX11-TRUE16 %s
 
 ;;;==========================================================================;;;
 ;; 16-bit integer comparisons
 ;;;==========================================================================;;;
-
-; GCN-LABEL: {{^}}i16_eq:
-; VI: v_cmp_eq_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_eq_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_eq(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_eq:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_eq:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_eq:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_eq:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -27,12 +104,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ne:
-; VI: v_cmp_ne_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ne_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ne(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ne:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ne_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ne:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ne:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ne:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -47,12 +201,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ugt:
-; VI: v_cmp_gt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ugt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ugt:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ugt:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ugt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ugt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -67,12 +298,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_uge:
-; VI: v_cmp_ge_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_uge(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_uge:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_uge:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_uge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_uge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -87,12 +395,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ult:
-; VI: v_cmp_lt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ult(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ult:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ult:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ult:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ult:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -107,12 +492,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ule:
-; VI: v_cmp_le_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ule(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ule:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ule:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ule:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ule:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -128,12 +590,89 @@ entry:
 
 }
 
-; GCN-LABEL: {{^}}i16_sgt:
-; VI: v_cmp_gt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sgt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_sgt:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sgt:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sgt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sgt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -148,12 +687,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sge:
-; VI: v_cmp_ge_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sge(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_sge:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sge:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -168,12 +784,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_slt:
-; VI: v_cmp_lt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_slt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_slt:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_slt:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_slt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_slt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -188,12 +881,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sle:
-; VI: v_cmp_le_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sle(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_sle:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sle:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sle:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sle:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -209,12 +979,78 @@ entry:
 }
 
 ; These should be commuted to reduce code size
-; GCN-LABEL: {{^}}i16_eq_v_s:
-; VI: v_cmp_eq_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_eq_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_eq_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_eq_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_eq_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_eq_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_eq_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_eq_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_eq_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -227,12 +1063,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ne_v_s:
-; VI: v_cmp_ne_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ne_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ne_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ne_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ne_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ne_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ne_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ne_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ne_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ne_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -245,12 +1147,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ugt_v_s:
-; VI: v_cmp_lt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ugt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ugt_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ugt_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ugt_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ugt_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -263,12 +1231,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_uge_v_s:
-; VI: v_cmp_le_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_uge_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_uge_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_uge_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_uge_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_uge_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -281,12 +1315,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ult_v_s:
-; VI: v_cmp_gt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ult_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ult_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ult_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ult_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ult_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -299,12 +1399,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ule_v_s:
-; VI: v_cmp_ge_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ule_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ule_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ule_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ule_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ule_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -317,12 +1483,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sgt_v_s:
-; VI: v_cmp_lt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sgt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_sgt_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sgt_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sgt_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sgt_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -335,12 +1567,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sge_v_s:
-; VI: v_cmp_le_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sge_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_sge_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sge_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sge_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sge_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -353,12 +1651,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_slt_v_s:
-; VI: v_cmp_gt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_slt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_slt_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_slt_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_slt_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_slt_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -371,12 +1735,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sle_v_s:
-; VI: v_cmp_ge_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sle_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_sle_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sle_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sle_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sle_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index b5665835eaf7a..17a5f520ff41e 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -6178,13 +6178,12 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
 ; NOOPT-NEXT:    v_mov_b32_e32 v11, v14
 ; NOOPT-NEXT:    v_mov_b32_e32 v12, v13
 ; NOOPT-NEXT:    buffer_store_dwordx4 v[9:12], off, s[4:7], 0 offset:32
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
+; NOOPT-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT:    s_waitcnt expcnt(0)
 ; NOOPT-NEXT:    v_mov_b32_e32 v9, v4
 ; NOOPT-NEXT:    v_mov_b32_e32 v10, v3
 ; NOOPT-NEXT:    v_mov_b32_e32 v11, v2
@@ -7297,7 +7296,6 @@ define amdgpu_kernel void @extract_adjacent_blocks(i32 %arg) {
 ; NOOPT-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 glc
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
@@ -7320,7 +7318,6 @@ define amdgpu_kernel void @extract_adjacent_blocks(i32 %arg) {
 ; NOOPT-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 glc
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
@@ -7541,7 +7538,6 @@ define amdgpu_kernel void @insert_adjacent_blocks(i32 %arg, float %val0) {
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3
 ; NOOPT-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
@@ -7565,7 +7561,6 @@ define amdgpu_kernel void @insert_adjacent_blocks(i32 %arg, float %val0) {
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3
 ; NOOPT-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
index 243f6c4d23732..25889ded91681 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
@@ -4,6 +4,177 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-REAL16 %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-FAKE16 %s
 
+declare half @llvm.amdgcn.cvt.f16.bf8(i32, i32)
+declare half @llvm.amdgcn.cvt.f16.fp8(i32, i32)
+declare <2 x half> @llvm.amdgcn.cvt.pk.f16.bf8(i16)
+declare <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16)
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 0)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:1
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:1
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:1
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:1
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 1)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:2
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:2
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:2
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:2
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 2)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 3)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.h, v0 byte_sel:3
+; GFX1250-SDAG-REAL16-NEXT:    v_mov_b16_e32 v0.l, 0
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_perm_b32 v0, v0, 0, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_lshl_or_b32 v0, v0, 16, 0
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_lshl_or_b32 v0, v0, 16, 0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 3)
+  %ins.0 = insertelement <2 x half> undef, half 0.0, i32 0
+  %ins.1 = insertelement <2 x half> %ins.0, half %cvt, i32 1
+  %ret = bitcast <2 x half> %ins.1 to float
+  ret float %ret
+}
+
 define amdgpu_ps float @test_cvt_pk_f16_bf8_v(i16 %a) {
 ; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_bf8_v:
 ; GFX1250-SDAG-REAL16:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
new file mode 100644
index 0000000000000..43c8d8318df1d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
+
+declare float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32, i32)
+
+define float @test_cvt_f32_fp8_e5m3_byte0(i32 %a) {
+; GFX1250-LABEL: test_cvt_f32_fp8_e5m3_byte0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 clamp
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %ret = tail call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 %a, i32 0)
+  ret float %ret
+}
+
+define float @test_cvt_f32_fp8_e5m3_byte1(i32 %a) {
+; GFX1250-LABEL: test_cvt_f32_fp8_e5m3_byte1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:1 clamp
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %ret = tail call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 %a, i32 1)
+  ret float %ret
+}
+
+define float @test_cvt_f32_fp8_e5m3_byte2(i32 %a) {
+; GFX1250-LABEL: test_cvt_f32_fp8_e5m3_byte2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:2 clamp
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %ret = tail call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 %a, i32 2)
+  ret float %ret
+}
+
+define float @test_cvt_f32_fp8_e5m3_byte3(i32 %a) {
+; GFX1250-LABEL: test_cvt_f32_fp8_e5m3_byte3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:3 clamp
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %ret = tail call float @llvm.amdgcn.cvt.f32.fp8.e5m3(i32 %a, i32 3)
+  ret float %ret
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
index 16d32b73b9b0d..09b1ea7c55afe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
@@ -6,6 +6,9 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
 
 declare float @llvm.amdgcn.cvt.f32.bf8(i32, i32)
 declare float @llvm.amdgcn.cvt.f32.fp8(i32, i32)
@@ -38,6 +41,13 @@ define float @test_cvt_f32_bf8_byte0(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_bf8_e32 v0, v0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_bf8_byte0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_bf8_e32 v0, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.bf8(i32 %a, i32 0)
   ret float %ret
 }
@@ -58,6 +68,13 @@ define float @test_cvt_f32_bf8_byte1(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_bf8_byte1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.bf8(i32 %a, i32 1)
   ret float %ret
 }
@@ -78,6 +95,13 @@ define float @test_cvt_f32_bf8_byte2(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_bf8_byte2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.bf8(i32 %a, i32 2)
   ret float %ret
 }
@@ -98,6 +122,13 @@ define float @test_cvt_f32_bf8_byte3(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_bf8_byte3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:3
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.bf8(i32 %a, i32 3)
   ret float %ret
 }
@@ -124,6 +155,13 @@ define float @test_cvt_f32_fp8_byte0(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_fp8_e32 v0, v0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_fp8_byte0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e32 v0, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.fp8(i32 %a, i32 0)
   ret float %ret
 }
@@ -144,6 +182,13 @@ define float @test_cvt_f32_fp8_byte1(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_fp8_byte1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.fp8(i32 %a, i32 1)
   ret float %ret
 }
@@ -164,6 +209,13 @@ define float @test_cvt_f32_fp8_byte2(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_fp8_byte2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.fp8(i32 %a, i32 2)
   ret float %ret
 }
@@ -184,6 +236,13 @@ define float @test_cvt_f32_fp8_byte3(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_f32_fp8_byte3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call float @llvm.amdgcn.cvt.f32.fp8(i32 %a, i32 3)
   ret float %ret
 }
@@ -204,6 +263,13 @@ define <2 x float> @test_cvt_pk_f32_bf8_word0(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_pk_f32_bf8_e32 v[0:1], v0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_pk_f32_bf8_word0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.bf8(i32 %a, i1 false)
   ret <2 x float> %ret
 }
@@ -224,6 +290,13 @@ define <2 x float> @test_cvt_pk_f32_bf8_word1(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_pk_f32_bf8_e64 v[0:1], v0 op_sel:[1,0]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_pk_f32_bf8_word1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_pk_f32_bf8_e64 v[0:1], v0 op_sel:[1,0]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.bf8(i32 %a, i1 true)
   ret <2 x float> %ret
 }
@@ -244,6 +317,13 @@ define <2 x float> @test_cvt_pk_f32_fp8_word0(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_pk_f32_fp8_e32 v[0:1], v0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_pk_f32_fp8_word0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.fp8(i32 %a, i1 false)
   ret <2 x float> %ret
 }
@@ -264,6 +344,13 @@ define <2 x float> @test_cvt_pk_f32_fp8_word1(i32 %a) {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_cvt_pk_f32_fp8_e64 v[0:1], v0 op_sel:[1,0]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_pk_f32_fp8_word1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_pk_f32_fp8_e64 v[0:1], v0 op_sel:[1,0]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.fp8(i32 %a, i1 true)
   ret <2 x float> %ret
 }
@@ -299,6 +386,24 @@ define i32 @test_cvt_pk_bf8_f32_word0(float %x, float %y, i32 %old) {
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX1250-TRUE16:       ; %bb.0:
+; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT:    v_cvt_pk_bf8_f32 v2.l, v0, v1
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0:
+; GFX1250-FAKE16:       ; %bb.0:
+; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT:    v_cvt_pk_bf8_f32 v2, v0, v1
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %x, float %y, i32 %old, i1 false)
   ret i32 %ret
 }
@@ -335,6 +440,24 @@ define i32 @test_cvt_pk_bf8_f32_word1(float %x, float %y, i32 %old) {
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: test_cvt_pk_bf8_f32_word1:
+; GFX1250-TRUE16:       ; %bb.0:
+; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT:    v_cvt_pk_bf8_f32 v2.h, v0, v1 op_sel:[0,0,1]
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word1:
+; GFX1250-FAKE16:       ; %bb.0:
+; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT:    v_cvt_pk_bf8_f32 v2, v0, v1 op_sel:[0,0,1]
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %x, float %y, i32 %old, i1 true)
   ret i32 %ret
 }
@@ -370,6 +493,24 @@ define i32 @test_cvt_pk_fp8_f32_word0(float %x, float %y, i32 %old) {
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word0:
+; GFX1250-TRUE16:       ; %bb.0:
+; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT:    v_cvt_pk_fp8_f32 v2.l, v0, v1
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word0:
+; GFX1250-FAKE16:       ; %bb.0:
+; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT:    v_cvt_pk_fp8_f32 v2, v0, v1
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %x, float %y, i32 %old, i1 false)
   ret i32 %ret
 }
@@ -406,6 +547,24 @@ define i32 @test_cvt_pk_fp8_f32_word1(float %x, float %y, i32 %old) {
 ; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX1250-TRUE16:       ; %bb.0:
+; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT:    v_cvt_pk_fp8_f32 v2.h, v0, v1 op_sel:[0,0,1]
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1:
+; GFX1250-FAKE16:       ; %bb.0:
+; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT:    v_cvt_pk_fp8_f32 v2, v0, v1 op_sel:[0,0,1]
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %x, float %y, i32 %old, i1 true)
   ret i32 %ret
 }
@@ -429,6 +588,15 @@ define i32 @test_cvt_sr_bf8_f32_byte0(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_bf8_f32_byte0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_bf8_f32 v2, v0, v1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %x, i32 %r, i32 %old, i32 0)
   ret i32 %ret
 }
@@ -453,6 +621,15 @@ define i32 @test_cvt_sr_bf8_f32_byte1(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_bf8_f32_byte1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_bf8_f32 v2, v0, v1 byte_sel:1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %x, i32 %r, i32 %old, i32 1)
   ret i32 %ret
 }
@@ -477,6 +654,15 @@ define i32 @test_cvt_sr_bf8_f32_byte2(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_bf8_f32_byte2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_bf8_f32 v2, v0, v1 byte_sel:2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %x, i32 %r, i32 %old, i32 2)
   ret i32 %ret
 }
@@ -501,6 +687,15 @@ define i32 @test_cvt_sr_bf8_f32_byte3(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_bf8_f32_byte3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_bf8_f32 v2, v0, v1 byte_sel:3
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %x, i32 %r, i32 %old, i32 3)
   ret i32 %ret
 }
@@ -524,6 +719,15 @@ define i32 @test_cvt_sr_fp8_f32_byte0(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_fp8_f32 v2, v0, v1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %x, i32 %r, i32 %old, i32 0)
   ret i32 %ret
 }
@@ -548,6 +752,15 @@ define i32 @test_cvt_sr_fp8_f32_byte1(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_fp8_f32 v2, v0, v1 byte_sel:1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %x, i32 %r, i32 %old, i32 1)
   ret i32 %ret
 }
@@ -572,6 +785,15 @@ define i32 @test_cvt_sr_fp8_f32_byte2(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_fp8_f32 v2, v0, v1 byte_sel:2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %x, i32 %r, i32 %old, i32 2)
   ret i32 %ret
 }
@@ -596,6 +818,15 @@ define i32 @test_cvt_sr_fp8_f32_byte3(float %x, i32 %r, i32 %old) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_cvt_sr_fp8_f32 v2, v0, v1 byte_sel:3
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %x, i32 %r, i32 %old, i32 3)
   ret i32 %ret
 }
@@ -619,6 +850,15 @@ define float @test_sext_cvt_f32_fp8(i16 %a) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_sext_cvt_f32_fp8:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_f32_fp8_e64 v0, v0 byte_sel:1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %a.sext = sext i16 %a to i32
   %ret = tail call float @llvm.amdgcn.cvt.f32.fp8(i32 %a.sext, i32 1)
   ret float %ret
@@ -643,6 +883,15 @@ define float @test_sext_cvt_f32_bf8(i16 %a) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_sext_cvt_f32_bf8:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_f32_bf8_e64 v0, v0 byte_sel:1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %a.sext = sext i16 %a to i32
   %ret = tail call float @llvm.amdgcn.cvt.f32.bf8(i32 %a.sext, i32 1)
   ret float %ret
@@ -667,6 +916,15 @@ define <2 x float> @test_sext_cvt_pk_f32_bf8_word1(i16 %a) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cvt_pk_f32_bf8_e64 v[0:1], v0 op_sel:[1,0]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_sext_cvt_pk_f32_bf8_word1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_pk_f32_bf8_e64 v[0:1], v0 op_sel:[1,0]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %a.sext = sext i16 %a to i32
   %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.bf8(i32 %a.sext, i1 true)
   ret <2 x float> %ret
@@ -691,6 +949,15 @@ define <2 x float> @test_sext_cvt_pk_f32_fp8_word0(i16 %a) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cvt_pk_f32_fp8_e32 v[0:1], v0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: test_sext_cvt_pk_f32_fp8_word0:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %a.sext = sext i16 %a to i32
   %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.fp8(i32 %a.sext, i1 false)
   ret <2 x float> %ret
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
new file mode 100644
index 0000000000000..9e0bd5e24e7bc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
@@ -0,0 +1,21 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+
+declare void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3))
+
+; GCN-LABEL: {{^}}test_ds_atomic_async_barrier_arrive_b64:
+; GCN: ds_atomic_async_barrier_arrive_b64 v0{{$}}
+define void @test_ds_atomic_async_barrier_arrive_b64(ptr addrspace(3) %bar) {
+entry:
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_ds_atomic_async_barrier_arrive_b64_off:
+; GCN: ds_atomic_async_barrier_arrive_b64 v0 offset:8184{{$}}
+define void @test_ds_atomic_async_barrier_arrive_b64_off(ptr addrspace(3) %in) {
+entry:
+  %bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
new file mode 100644
index 0000000000000..8c7c760aa5010
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
@@ -0,0 +1,27 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+
+declare i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3), i64)
+
+; GCN-LABEL: {{^}}test_ds_atomic_barrier_arrive_rtn_b64:
+; GCN: ds_atomic_barrier_arrive_rtn_b64 v[{{[0-9:]+}}], v2, v[0:1]{{$}}
+; GCN: s_wait_dscnt 0x0
+; GCN: flat_store_b64
+define void @test_ds_atomic_barrier_arrive_rtn_b64(i64 %data, ptr addrspace(3) %bar, ptr %out) {
+entry:
+  %ret = call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) %bar, i64 %data)
+  store i64 %ret, ptr %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_ds_atomic_barrier_arrive_rtn_b64_off:
+; GCN: ds_atomic_barrier_arrive_rtn_b64 v[{{[0-9:]+}}], v0, v[{{[0-9:]+}}] offset:8184{{$}}
+; GCN: s_wait_dscnt 0x0
+; GCN: flat_store_b64
+define void @test_ds_atomic_barrier_arrive_rtn_b64_off(ptr addrspace(3) %in, ptr %out) {
+entry:
+  %bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
+  %ret = call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) %bar, i64 512)
+  store i64 %ret, ptr %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
index a319f1260d870..85dd275207902 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
@@ -5,18 +5,19 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) {
 ; GCN-LABEL: test_iglp_opt_rev_mfma_gemm:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    v_mov_b32_e32 v32, 0
+; GCN-NEXT:    ds_read_b128 v[0:3], v32
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GCN-NEXT:    ds_read_b128 v[28:31], v32 offset:112
 ; GCN-NEXT:    ds_read_b128 v[24:27], v32 offset:96
 ; GCN-NEXT:    ds_read_b128 v[20:23], v32 offset:80
 ; GCN-NEXT:    ds_read_b128 v[16:19], v32 offset:64
-; GCN-NEXT:    ds_read_b128 v[0:3], v32
 ; GCN-NEXT:    ds_read_b128 v[4:7], v32 offset:16
 ; GCN-NEXT:    ds_read_b128 v[8:11], v32 offset:32
 ; GCN-NEXT:    ds_read_b128 v[12:15], v32 offset:48
-; GCN-NEXT:    v_mov_b32_e32 v34, 0
-; GCN-NEXT:    v_mov_b32_e32 v35, v34
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_write_b128 v32, v[0:3]
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
 ; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GCN-NEXT:    ; iglp_opt mask(0x00000001)
 ; GCN-NEXT:    ds_write_b128 v32, v[28:31] offset:112
@@ -26,8 +27,7 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) {
 ; GCN-NEXT:    ds_write_b128 v32, v[12:15] offset:48
 ; GCN-NEXT:    ds_write_b128 v32, v[8:11] offset:32
 ; GCN-NEXT:    ds_write_b128 v32, v[4:7] offset:16
-; GCN-NEXT:    ds_write_b128 v32, v[0:3]
-; GCN-NEXT:    ds_write_b64 v32, v[34:35]
+; GCN-NEXT:    ds_write_b64 v32, v[0:1]
 ; GCN-NEXT:    s_endpgm
 entry:
   call void @llvm.amdgcn.iglp.opt(i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
index 38c1f10f2c011..4a2c1fe2cf91c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
@@ -1,18 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -early-live-intervals < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
-; GCN-LABEL: {{^}}load_1d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_lwe:
-; GCN: image_load v[0:4], v{{[0-9]+}}, s[0:7] dmask:0xf unorm lwe{{$}}
 define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s) {
+; GCN-LABEL: load_1d_lwe:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v6, v0
+; GCN-NEXT:    v_mov_b32_e32 v9, v8
+; GCN-NEXT:    v_mov_b32_e32 v10, v8
+; GCN-NEXT:    v_mov_b32_e32 v11, v8
+; GCN-NEXT:    v_mov_b32_e32 v12, v8
+; GCN-NEXT:    v_mov_b32_e32 v0, v8
+; GCN-NEXT:    v_mov_b32_e32 v1, v9
+; GCN-NEXT:    v_mov_b32_e32 v2, v10
+; GCN-NEXT:    v_mov_b32_e32 v3, v11
+; GCN-NEXT:    v_mov_b32_e32 v4, v12
+; GCN-NEXT:    image_load v[0:4], v6, s[0:7] dmask:0xf unorm lwe
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v8, v4, s[8:9]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
@@ -21,32 +41,39 @@ main_body:
   ret <4 x float> %v.vec
 }
 
-; GCN-LABEL: {{^}}load_2d:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+; GCN-LABEL: load_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_3d:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) {
+; GCN-LABEL: load_3d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_cube:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: load_cube:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.cube.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_cube_lwe:
-; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
 define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
@@ -56,24 +83,28 @@ main_body:
   ret <4 x float> %v.vec
 }
 
-; GCN-LABEL: {{^}}load_1darray:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, i32 %s, i32 %slice) {
+; GCN-LABEL: load_1darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1darray.v4f32.i32(i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_2darray:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: load_2darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2darray.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_2darray_lwe:
-; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
 define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
@@ -83,184 +114,235 @@ main_body:
   ret <4 x float> %v.vec
 }
 
-; GCN-LABEL: {{^}}load_2dmsaa:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
+; GCN-LABEL: load_2dmsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_2darraymsaa:
-; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GCN-LABEL: load_2darraymsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_addr_align:
-; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
-; GCN: image_load v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_1d_addr_align(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+; GCN-LABEL: load_1d_addr_align:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %s1 = extractelement <2 x i32> %s, i32 1
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}store_1d:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2d:
-; GCN: image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t) {
+; GCN-LABEL: store_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_3d:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %r) {
+; GCN-LABEL: store_3d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.3d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_cube:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: store_cube:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.cube.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1darray:
-; GCN: image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %slice) {
+; GCN-LABEL: store_1darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1darray.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2darray:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: store_2darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2darray.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2dmsaa:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %fragid) {
+; GCN-LABEL: store_2dmsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2dmsaa.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2darraymsaa:
-; GCN: image_store v[0:3], v[4:7], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GCN-LABEL: store_2darraymsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:7], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2darraymsaa.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}load_1d_V1:
-; GCN: image_load v0, v0, s[0:7] dmask:0x8 unorm{{$}}
 define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_V1:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 8, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret float %v
 }
 
-; GCN-LABEL: {{^}}load_1d_V2:
-; GCN: image_load v[0:1], v0, s[0:7] dmask:0x9 unorm{{$}}
 define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_V2:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 9, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret <2 x float> %v
 }
 
-; GCN-LABEL: {{^}}store_1d_V1:
-; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
-; GCN: image_store v0, [[VADDR]], s[0:7] dmask:0x2 unorm{{$}}
 define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, i32 %s) {
+; GCN-LABEL: store_1d_V1:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    image_store v0, v2, s[0:7] dmask:0x2 unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.f32.i32(float %vdata, i32 2, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1d_V2:
-; GCN: image_store v[0:1], v2, s[0:7] dmask:0xc unorm{{$}}
 define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_V2:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:1], v2, s[0:7] dmask:0xc unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v2f32.i32(<2 x float> %vdata, i32 12, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}load_1d_glc:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc{{$}}
 define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_glc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_slc:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm slc{{$}}
 define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm slc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_glc_slc:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc slc{{$}}
 define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_glc_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc slc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 3)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}store_1d_glc:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc{{$}}
 define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_glc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1d_slc:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm slc{{$}}
 define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm slc
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1d_glc_slc:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc slc{{$}}
 define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_glc_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc slc
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 3)
   ret void
 }
 
-; GCN-LABEL: {{^}}image_store_wait:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf
-; SI: s_waitcnt expcnt(0)
-; GCN: image_load v[0:3], v4, s[8:15] dmask:0xf
-; GCN: s_waitcnt vmcnt(0)
-; GCN: image_store v[0:3], v4, s[16:23] dmask:0xf
 define amdgpu_ps void @image_store_wait(<8 x i32> inreg %arg, <8 x i32> inreg %arg1, <8 x i32> inreg %arg2, <4 x float> %arg3, i32 %arg4) #0 {
+; GCN-LABEL: image_store_wait:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    image_load v[0:3], v4, s[8:15] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    image_store v[0:3], v4, s[16:23] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %arg3, i32 15, i32 %arg4, <8 x i32> %arg, i32 0, i32 0)
   %data = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %arg4, <8 x i32> %arg1, i32 0, i32 0)
@@ -268,8 +350,6 @@ main_body:
   ret void
 }
 
-; GCN-LABEL: image_load_mmo
-; GCN: image_load v1, v[{{[0-9:]+}}], s[0:7] dmask:0x1 unorm
 define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, ptr addrspace(3) %lds, <2 x i32> %c) #0 {
   store float 0.000000e+00, ptr addrspace(3) %lds
   %c0 = extractelement <2 x i32> %c, i32 0
@@ -280,9 +360,13 @@ define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, ptr addrspace(3) %
   ret float %tex
 }
 
-; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
-; GCN: image_get_resinfo v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm
 define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+; GCN-LABEL: getresinfo_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %s1 = extractelement <2 x i32> %s, i32 1
   %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
new file mode 100644
index 0000000000000..d91b03ca4461d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
@@ -0,0 +1,322 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+declare <2 x i32>    @llvm.amdgcn.global.load.tr4.b64.v2i32.p1(ptr addrspace(1))
+declare <2 x i32>    @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1))
+declare <3 x i32>    @llvm.amdgcn.global.load.tr6.b96.v3i32.p1(ptr addrspace(1))
+declare <8 x i16>    @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1))
+declare <8 x half>   @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1))
+declare <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1))
+
+declare <2 x i32>    @llvm.amdgcn.ds.load.tr4.b64.v2i32.p3(ptr addrspace(3))
+declare <2 x i32>    @llvm.amdgcn.ds.load.tr8.b64.v2i32.p3(ptr addrspace(3))
+declare <3 x i32>    @llvm.amdgcn.ds.load.tr6.b96.v3i32.p3(ptr addrspace(3))
+declare <8 x i16>    @llvm.amdgcn.ds.load.tr16.b128.v8i16.p3(ptr addrspace(3))
+declare <8 x half>   @llvm.amdgcn.ds.load.tr16.b128.v8f16.p3(ptr addrspace(3))
+declare <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16.p3(ptr addrspace(3))
+
+
+define amdgpu_ps void @global_load_tr4_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr4_b64_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr4_b64 v[0:1], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr4_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr4_b64_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr4_b64 v[2:3], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr8_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr8_b64_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr8_b64 v[0:1], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr8_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr8_b64_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr8_b64 v[2:3], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr6_b96_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr6_b96_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr6_b96 v[4:6], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b96 v[2:3], v[4:6], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32.p1(ptr addrspace(1) %gep)
+  store <3 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr6_b96_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr6_b96_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr6_b96 v[2:4], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b96 v[0:1], v[2:4], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32.p1(ptr addrspace(1) %gep)
+  store <3 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8i16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8i16_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr16_b128 v[4:7], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1) %gep)
+  store <8 x i16> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8i16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8i16_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1) %gep)
+  store <8 x i16> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8f16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8f16_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr16_b128 v[4:7], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8f16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8f16_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8b16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8b16_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr16_b128 v[4:7], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8bf16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8bf16_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr4_b64(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr4_b64:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr4_b64 v[0:1], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr4_b64:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr4_b64 v[0:1], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b64 v[4:5], v[0:1], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32.p3(ptr addrspace(3) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr8_b64(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr8_b64:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr8_b64 v[0:1], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr8_b64:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr8_b64 v[0:1], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b64 v[4:5], v[0:1], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32.p3(ptr addrspace(3) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr6_b96(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr6_b96:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr6_b96 v[0:2], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b96 v[4:5], v[0:2], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr6_b96:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr6_b96 v[0:2], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b96 v[4:5], v[0:2], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32.p3(ptr addrspace(3) %gep)
+  store <3 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr16_b128_v8i16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr16_b128_v8i16:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr16_b128_v8i16:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16.p3(ptr addrspace(3) %gep)
+  store <8 x i16> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr16_b128_v8f16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr16_b128_v8f16:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr16_b128_v8f16:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16.p3(ptr addrspace(3) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr16_b128_v8bf16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: ds_load_tr16_b128_v8bf16:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16.p3(ptr addrspace(3) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll
new file mode 100644
index 0000000000000..706f470a6285a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll
@@ -0,0 +1,20 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+
+declare void @llvm.amdgcn.s.monitor.sleep(i16)
+
+; GCN-LABEL: {{^}}test_monitor_sleep_1:
+; GCN: s_monitor_sleep 1
+define amdgpu_ps void @test_monitor_sleep_1() {
+  call void @llvm.amdgcn.s.monitor.sleep(i16 1)
+  ret void
+}
+
+; FIXME: 0x8000 would look better
+
+; GCN-LABEL: {{^}}test_monitor_sleep_forever:
+; GCN: s_monitor_sleep 0xffff8000
+define amdgpu_ps void @test_monitor_sleep_forever() {
+  call void @llvm.amdgcn.s.monitor.sleep(i16 32768)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
new file mode 100644
index 0000000000000..d2712ac8e08a3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -0,0 +1,245 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 %cpol)
+declare void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 %cpol)
+declare void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 %cpol)
+declare void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 %cpol)
+
+define amdgpu_ps void @tensor_load_to_lds(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
+; GFX1250-LABEL: tensor_load_to_lds:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19]
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @tensor_load_to_lds_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
+; GFX1250-SDAG-LABEL: tensor_load_to_lds_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_load_to_lds_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 0)
+  ret void
+}
+
+
+define amdgpu_ps void @tensor_load_to_lds_d2(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
+; GFX1250-LABEL: tensor_load_to_lds_d2:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 27)
+  ret void
+}
+
+define amdgpu_ps void @tensor_load_to_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
+; GFX1250-SDAG-LABEL: tensor_load_to_lds_d2_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_load_to_lds_d2_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 27)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
+; GFX1250-LABEL: tensor_store_from_lds:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 22)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
+; GFX1250-SDAG-LABEL: tensor_store_from_lds_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_store_from_lds_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 22)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds_d2(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
+; GFX1250-LABEL: tensor_store_from_lds_d2:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_store_from_lds s[0:3], s[4:11]
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
+; GFX1250-SDAG-LABEL: tensor_store_from_lds_d2_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_store_from_lds s[8:11], s[0:7]
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_store_from_lds_d2_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_store_from_lds s[8:11], s[0:7]
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll b/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
index 13ea8b08d2ade..91a8446915f66 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
@@ -18,9 +18,10 @@ define amdgpu_cs void @test_uniform_load_b96(ptr addrspace(1) %ptr, i32 %arg) "a
 ; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x8
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v2, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_or3_b32 v2, s2, v2, s0
+; GFX11-NEXT:    s_or_b32 s1, s2, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_or_b32 s0, s0, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -33,12 +34,14 @@ define amdgpu_cs void @test_uniform_load_b96(ptr addrspace(1) %ptr, i32 %arg) "a
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, v1, v3, vcc_lo
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX12-NEXT:    s_load_b96 s[0:2], s[0:1], 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_or_b32 s0, s0, s1
+; GFX12-NEXT:    s_or_b32 s0, s2, s0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v2, s0
-; GFX12-NEXT:    v_or3_b32 v2, v2, s1, s2
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-NEXT:    s_endpgm
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
index 896e60900c745..5bc02c4d63181 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -1,179 +1,1351 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GCN-NOHSA,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GCN-HSA,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GCN-NOHSA,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mcpu=cayman < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=GCN-NOHSA-SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=GCN-HSA %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=GCN-NOHSA-VI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=EG %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mcpu=cayman < %s | FileCheck -allow-deprecated-dag-overlap --check-prefix=CM %s
 
-
-; FUNC-LABEL: {{^}}global_load_i8:
-; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}}
-; GCN-HSA: flat_load_ubyte
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; TODO: NOT AND
 define amdgpu_kernel void @global_load_i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_load_i8:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_load_i8:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_byte v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_load_i8:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_load_i8:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.X, literal.y,
+; EG-NEXT:    3(4.203895e-45), 255(3.573311e-43)
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.X, T1.W, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_load_i8:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T0.Z, T0.X, literal.x,
+; CM-NEXT:     LSHL * T0.W, PV.W, literal.y,
+; CM-NEXT:    255(3.573311e-43), 3(4.203895e-45)
+; CM-NEXT:     LSHL T0.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     MOV T0.Y, 0.0,
+; CM-NEXT:     MOV * T0.Z, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load i8, ptr addrspace(1) %in
   store i8 %ld, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_load_v2i8:
-; GCN-NOHSA: buffer_load_ushort v
-; GCN-HSA: flat_load_ushort v
-
-; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_load_v2i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_load_v2i8:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_load_v2i8:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_short v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_load_v2i8:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_load_v2i8:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.X, literal.y,
+; EG-NEXT:    3(4.203895e-45), 65535(9.183409e-41)
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.X, T1.W, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_load_v2i8:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T0.Z, T0.X, literal.x,
+; CM-NEXT:     LSHL * T0.W, PV.W, literal.y,
+; CM-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; CM-NEXT:     LSHL T0.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; CM-NEXT:     MOV T0.Y, 0.0,
+; CM-NEXT:     MOV * T0.Z, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <2 x i8>, ptr addrspace(1) %in
   store <2 x i8> %ld, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_load_v3i8:
-; GCN-NOHSA: buffer_load_dword v
-; GCN-HSA: flat_load_dword v
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_load_v3i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_load_v3i8:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GCN-NOHSA-SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    buffer_store_byte v1, off, s[4:7], 0 offset:2
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_load_v3i8:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_short v[0:1], v2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_byte v[0:1], v3
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_load_v3i8:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    buffer_store_byte v1, off, s[4:7], 0 offset:2
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_load_v3i8:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 27, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T6.XW, T8.X
+; EG-NEXT:    MEM_RAT MSKOR T5.XW, T7.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T5.X, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T2.X, T5.X,
+; EG-NEXT:     MOV T0.Y, PV.X,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
+; EG-NEXT:     MOV * T2.W, literal.y,
+; EG-NEXT:    3(4.203895e-45), 8(1.121039e-44)
+; EG-NEXT:     BFE_UINT T2.W, T0.Y, literal.x, PS,
+; EG-NEXT:     LSHL * T1.W, PV.W, literal.y,
+; EG-NEXT:    16(2.242078e-44), 3(4.203895e-45)
+; EG-NEXT:     LSHL T6.X, PV.W, PS,
+; EG-NEXT:     LSHL * T6.W, literal.x, PS,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T6.Y, 0.0,
+; EG-NEXT:     AND_INT T1.W, KC0[2].Y, literal.x,
+; EG-NEXT:     AND_INT * T2.W, T5.X, literal.y,
+; EG-NEXT:    3(4.203895e-45), 65535(9.183409e-41)
+; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T5.X, T2.W, PV.W,
+; EG-NEXT:     LSHL * T5.W, literal.x, PV.W,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T5.Y, 0.0,
+; EG-NEXT:     MOV T6.Z, 0.0,
+; EG-NEXT:     MOV * T5.Z, 0.0,
+; EG-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:     LSHR * T8.X, T0.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_load_v3i8:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 28, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T6.XW, T8.X
+; CM-NEXT:    MEM_RAT MSKOR T5.XW, T7.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T5.X, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T2.X, T5.X,
+; CM-NEXT:     MOV T0.Y, PV.X,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T0.Z, PV.W, literal.x,
+; CM-NEXT:     MOV * T1.W, literal.y,
+; CM-NEXT:    3(4.203895e-45), 8(1.121039e-44)
+; CM-NEXT:     BFE_UINT T1.Z, T0.Y, literal.x, PV.W,
+; CM-NEXT:     LSHL * T1.W, PV.Z, literal.y,
+; CM-NEXT:    16(2.242078e-44), 3(4.203895e-45)
+; CM-NEXT:     LSHL T6.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T6.W, literal.x, PV.W,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     MOV T6.Y, 0.0,
+; CM-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T0.Z, T5.X, literal.x,
+; CM-NEXT:     LSHL * T1.W, PV.W, literal.y,
+; CM-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; CM-NEXT:     LSHL T5.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T5.W, literal.x, PV.W,
+; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; CM-NEXT:     MOV T5.Y, 0.0,
+; CM-NEXT:     MOV * T6.Z, 0.0,
+; CM-NEXT:     MOV * T5.Z, 0.0,
+; CM-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T8.X, T0.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <3 x i8>, ptr addrspace(1) %in
   store <3 x i8> %ld, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_load_v4i8:
-; GCN-NOHSA: buffer_load_dword v
-; GCN-HSA: flat_load_dword v
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_load_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_load_v4i8:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_load_v4i8:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_load_v4i8:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_load_v4i8:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_load_v4i8:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <4 x i8>, ptr addrspace(1) %in
   store <4 x i8> %ld, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_load_v8i8:
-; GCN-NOHSA: buffer_load_dwordx2
-; GCN-HSA: flat_load_dwordx2
-
-; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_load_v8i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_load_v8i8:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_load_v8i8:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_load_v8i8:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_load_v8i8:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_load_v8i8:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <8 x i8>, ptr addrspace(1) %in
   store <8 x i8> %ld, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_load_v16i8:
-; GCN-NOHSA: buffer_load_dwordx4
-
-; GCN-HSA: flat_load_dwordx4
-
-; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_load_v16i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_load_v16i8:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_load_v16i8:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_load_v16i8:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_load_v16i8:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_load_v16i8:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <16 x i8>, ptr addrspace(1) %in
   store <16 x i8> %ld, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_i8_to_i32:
-; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}},
-; GCN-HSA: flat_load_ubyte
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_i8_to_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_i8_to_i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_i8_to_i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_i8_to_i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_i8_to_i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_i8_to_i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i8, ptr addrspace(1) %in
   %ext = zext i8 %a to i32
   store i32 %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_i8_to_i32:
-; GCN-NOHSA: buffer_load_sbyte
-; GCN-HSA: flat_load_sbyte
-
-; EG: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1
-; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
-; EG: 8
 define amdgpu_kernel void @global_sextload_i8_to_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_i8_to_i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_i8_to_i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_sbyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_i8_to_i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_i8_to_i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.X, T0.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+;
+; CM-LABEL: global_sextload_i8_to_i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 3, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT * T0.X, T0.X, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %ld = load i8, ptr addrspace(1) %in
   %ext = sext i8 %ld to i32
   store i32 %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v1i8_to_v1i32:
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v1i8_to_v1i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v1i8_to_v1i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v1i8_to_v1i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v1i8_to_v1i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v1i8_to_v1i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v1i8_to_v1i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 1, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <1 x i8>, ptr addrspace(1) %in
   %ext = zext <1 x i8> %load to <1 x i32>
   store <1 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v1i8_to_v1i32:
-
-; EG: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1
-; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
-; EG: 8
 define amdgpu_kernel void @global_sextload_v1i8_to_v1i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v1i8_to_v1i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v1i8_to_v1i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_sbyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v1i8_to_v1i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v1i8_to_v1i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.X, T0.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+;
+; CM-LABEL: global_sextload_v1i8_to_v1i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 3, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0.X, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT * T0.X, T0.X, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <1 x i8>, ptr addrspace(1) %in
   %ext = sext <1 x i8> %load to <1 x i32>
   store <1 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v2i8_to_v2i32:
-; GCN-NOHSA: buffer_load_ushort
-; GCN-HSA: flat_load_ushort
-
-; EG: VTX_READ_16 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal
-; EG-DAG: 8
+; TODO: EG/CM  should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_zextload_v2i8_to_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v2i8_to_v2i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v2i8_to_v2i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GCN-HSA-NEXT:    v_and_b32_e32 v2, 0xff, v2
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v2i8_to_v2i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 8
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v2i8_to_v2i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 12, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XY, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.Y, T2.X,
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     AND_INT T0.W, T4.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), -65536(nan)
+; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
+; EG-NEXT:     MOV * T2.X, PV.W,
+; EG-NEXT:     MOV T0.Y, PV.X,
+; EG-NEXT:     MOV * T1.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T4.Y, PV.Y, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T4.X, T0.W, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
+;
+; CM-LABEL: global_zextload_v2i8_to_v2i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 13, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.Y, T2.X,
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     AND_INT T0.Z, T4.X, literal.x,
+; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
+; CM-NEXT:    65535(9.183409e-41), -65536(nan)
+; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
+; CM-NEXT:     MOV * T2.X, PV.W,
+; CM-NEXT:     MOV T0.Y, PV.X,
+; CM-NEXT:     MOV * T1.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T4.Y, PV.Y, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT * T4.X, T0.W, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <2 x i8>, ptr addrspace(1) %in
   %ext = zext <2 x i8> %load to <2 x i32>
   store <2 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v2i8_to_v2i32:
-; GCN-NOHSA: buffer_load_ushort
-; GCN-HSA: flat_load_ushort
-
-; EG: VTX_READ_16 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_sextload_v2i8_to_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v2i8_to_v2i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v2i8_to_v2i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_bfe_i32 v3, v2, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v2i8_to_v2i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v2i8_to_v2i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 11, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XY, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.Y, T2.X,
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     AND_INT T0.W, T4.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), -65536(nan)
+; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
+; EG-NEXT:     MOV * T2.X, PV.W,
+; EG-NEXT:     MOV * T0.Y, PV.X,
+; EG-NEXT:     BFE_INT T4.X, T0.W, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.W, PV.Y, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+; EG-NEXT:     BFE_INT * T4.Y, PV.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_v2i8_to_v2i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 11, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.Y, T2.X,
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     AND_INT T0.Z, T4.X, literal.x,
+; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
+; CM-NEXT:    65535(9.183409e-41), -65536(nan)
+; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
+; CM-NEXT:     MOV * T2.X, PV.W,
+; CM-NEXT:     MOV * T0.Y, PV.X,
+; CM-NEXT:     BFE_INT T4.X, T0.W, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, PV.Y, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T4.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <2 x i8>, ptr addrspace(1) %in
   %ext = sext <2 x i8> %load to <2 x i32>
   store <2 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v3i8_to_v3i32:
-; GCN-NOHSA: buffer_load_dword v
-; GCN-HSA: flat_load_dword v
-
-; GCN-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8
-; GCN-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 16, 8
-; GCN-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xff,
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_zextload_v3i8_to_v3i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v3i8_to_v3i32:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v1, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v2, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v3i8_to_v3i32:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_bfe_u32 v1, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v0, 0xff, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v2, v2, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v3i8_to_v3i32:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v2, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v1, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v3i8_to_v3i32:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.X, T7.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XY, T6.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T5.Y, T4.X, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T5.X, T4.X, literal.x,
+; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
+; EG-NEXT:     BFE_UINT T4.X, T4.X, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; EG-NEXT:     LSHR * T7.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v3i8_to_v3i32:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T7.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5.X, T6.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T5.X, T4.X, literal.x, PV.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; CM-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; CM-NEXT:     LSHR T6.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT * T4.Y, T4.X, literal.y, T0.W,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     AND_INT * T4.X, T4.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 entry:
   %ld = load <3 x i8>, ptr addrspace(1) %in
   %ext = zext <3 x i8> %ld to <3 x i32>
@@ -181,28 +1353,120 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v3i8_to_v3i32:
-; GCN-NOHSA: buffer_load_dword v
-; GCN-HSA: flat_load_dword v
-
-; t23: i16 = truncate t18
-; t49: i16 = srl t23, Constant:i32<8>
-; t57: i32 = any_extend t49
-; t58: i32 = sign_extend_inreg t57, ValueType:ch:i8
-
-; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8
-; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8
-; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 16, 8
-
-; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_sextload_v3i8_to_v3i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v3i8_to_v3i32:
+; GCN-NOHSA-SI:       ; %bb.0: ; %entry
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v2, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0 offset:8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v3i8_to_v3i32:
+; GCN-HSA:       ; %bb.0: ; %entry
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v0, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v3i8_to_v3i32:
+; GCN-NOHSA-VI:       ; %bb.0: ; %entry
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx3 v[0:2], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v3i8_to_v3i32:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.X, T4.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XY, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; EG-NEXT:     LSHR * T0.W, T4.X, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT * T6.X, PV.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T7.X, T4.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.W, T4.X, literal.x,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T4.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T7.Y, PV.W, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v3i8_to_v3i32:
+; CM:       ; %bb.0: ; %entry
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T4.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5.X, T6.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHR * T0.W, T4.X, literal.x,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T5.X, PV.W, 0.0, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T6.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T7.X, T4.X, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, T4.X, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T4.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T7.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
 entry:
   %ld = load <3 x i8>, ptr addrspace(1) %in
   %ext = sext <3 x i8> %ld to <3 x i32>
@@ -210,513 +1474,7647 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v4i8_to_v4i32:
-; GCN-NOHSA: buffer_load_dword
-; GCN-HSA: flat_load_dword
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_zextload_v4i8_to_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v4i8_to_v4i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v1, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v2, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v4i8_to_v4i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v1, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v0, 0xff, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v2, v2, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v4i8_to_v4i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v2, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v1, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v4i8_to_v4i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 9, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T4.Z, T4.X, literal.x, PV.W,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T4.Y, T4.X, literal.x, T0.W,
+; EG-NEXT:     LSHR * T4.W, T4.X, literal.y,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:     AND_INT T4.X, T4.X, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
+;
+; CM-LABEL: global_zextload_v4i8_to_v4i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T4.Z, T4.X, literal.x, PV.W,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T4.Y, T4.X, literal.x, T0.W,
+; CM-NEXT:     LSHR * T4.W, T4.X, literal.y,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:     AND_INT * T4.X, T4.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <4 x i8>, ptr addrspace(1) %in
   %ext = zext <4 x i8> %load to <4 x i32>
   store <4 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v4i8_to_v4i32:
-; GCN-NOHSA: buffer_load_dword
-; GCN-HSA: flat_load_dword
-
-; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_sextload_v4i8_to_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v4i8_to_v4i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v4i8_to_v4i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 24, v0
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v0, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v4i8_to_v4i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v4i8_to_v4i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XYZW, T4.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T4.X, literal.y,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:     BFE_INT T5.W, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T4.X, literal.y,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T5.Z, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T4.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T4.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_INT * T5.Y, PV.W, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v4i8_to_v4i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T4.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, T4.X, literal.y,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:     LSHR T0.Z, T4.X, literal.x,
+; CM-NEXT:     BFE_INT * T5.W, PV.W, 0.0, literal.y,
+; CM-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; CM-NEXT:     BFE_INT T5.Z, PV.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, T4.X, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T4.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T5.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <4 x i8>, ptr addrspace(1) %in
   %ext = sext <4 x i8> %load to <4 x i32>
   store <4 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v8i8_to_v8i32:
-
-; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_zextload_v8i8_to_v8i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v8i8_to_v8i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[8:9], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v1, v8, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 24, v9
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v5, v9, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v2, v8, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff, v9
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v6, v9, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v8i8_to_v8i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[8:9], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v7, 24, v9
+; GCN-HSA-NEXT:    v_bfe_u32 v5, v9, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v4, 0xff, v9
+; GCN-HSA-NEXT:    v_bfe_u32 v6, v9, 16, 8
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
+; GCN-HSA-NEXT:    v_bfe_u32 v1, v8, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v0, 0xff, v8
+; GCN-HSA-NEXT:    v_bfe_u32 v2, v8, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[4:7]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v8i8_to_v8i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[8:9], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 24, v9
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v5, v9, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xff, v9
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v6, v9, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v1, v8, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v8, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v8i8_to_v8i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 20, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T8.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T6.Z, T5.X, literal.x, PV.W,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T6.Y, T5.X, literal.x, T0.W,
+; EG-NEXT:     BFE_UINT T7.Z, T5.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR * T6.W, T5.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T6.X, T5.X, literal.x,
+; EG-NEXT:     BFE_UINT T7.Y, T5.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T7.W, T5.Y, literal.x,
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T7.X, T5.Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 16(2.242078e-44)
+; EG-NEXT:     LSHR * T8.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v8i8_to_v8i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 20, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T8.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6, T7.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T6.Z, T5.Y, literal.x, PV.W,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T6.Y, T5.Y, literal.x, T0.W,
+; CM-NEXT:     BFE_UINT T5.Z, T5.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T6.W, T5.Y, literal.z,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T6.X, T5.Y, literal.x,
+; CM-NEXT:     BFE_UINT T5.Y, T5.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T7.X, PV.W, literal.x,
+; CM-NEXT:     LSHR * T5.W, T5.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:     AND_INT * T5.X, T5.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T8.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <8 x i8>, ptr addrspace(1) %in
   %ext = zext <8 x i8> %load to <8 x i32>
   store <8 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v8i8_to_v8i32:
-
-; EG: VTX_READ_64 [[DST:T[0-9]+\.XY]], T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_sextload_v8i8_to_v8i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v8i8_to_v8i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[7:8], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 24, v7
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v7, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v7, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v7, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v7, 24, v8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v6, v8, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v5, v8, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v4, v8, 0, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v8i8_to_v8i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[7:8], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 24, v7
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v7, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v1, v7, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v0, v7, 0, 8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 24, v8
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v8, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v5, v8, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v8, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[11:12], v[4:7]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[9:10], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v8i8_to_v8i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[7:8], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 24, v7
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v7, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v7, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v7, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 24, v8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v8, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v5, v8, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v8, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v8i8_to_v8i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 23, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T8.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T6.X, T5.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T5.X, literal.y,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:     BFE_INT T7.X, T5.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Z, T5.Y, literal.y,
+; EG-NEXT:     BFE_INT T6.W, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T5.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T0.Y, T5.Y, literal.x,
+; EG-NEXT:     BFE_INT T6.Z, PS, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T7.W, PV.Z, 0.0, literal.y,
+; EG-NEXT:     LSHR * T0.W, T5.X, literal.y,
+; EG-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; EG-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_INT T6.Y, PS, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T7.Z, PV.Y, 0.0, literal.y,
+; EG-NEXT:     LSHR T0.W, T5.Y, literal.y,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T8.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T7.Y, PV.W, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v8i8_to_v8i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 23, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T5.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6, T9.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT T6.X, T5.Y, 0.0, literal.x,
+; CM-NEXT:     LSHR T0.Z, T5.X, literal.y,
+; CM-NEXT:     LSHR * T0.W, T5.Y, literal.z,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T7.X, T5.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T0.Y, T5.X, literal.y,
+; CM-NEXT:     LSHR T1.Z, T5.Y, literal.z,
+; CM-NEXT:     BFE_INT * T6.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T8.X, T5.Y, literal.x,
+; CM-NEXT:     ADD_INT T1.Y, KC0[2].Y, literal.y,
+; CM-NEXT:     BFE_INT T6.Z, PV.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T7.W, PV.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     LSHR T9.X, PV.Y, literal.x,
+; CM-NEXT:     BFE_INT T6.Y, PV.X, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T7.Z, T0.Z, 0.0, literal.y,
+; CM-NEXT:     LSHR * T0.W, T5.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T7.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <8 x i8>, ptr addrspace(1) %in
   %ext = sext <8 x i8> %load to <8 x i32>
   store <8 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v16i8_to_v16i32:
-
-; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_zextload_v16i8_to_v16i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v16i8_to_v16i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v5, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v9, v1, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v15, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v13, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v19, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v17, v3, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v6, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v10, v1, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v14, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v16, 0xff, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v18, v3, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v16i8_to_v16i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s1
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s3
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v10, 24, v3
+; GCN-HSA-NEXT:    v_bfe_u32 v8, v3, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v3
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v3, 16, 8
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v6, 24, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v4, v0, 8, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[11:12], v[7:10]
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v14, 24, v2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v10, 24, v1
+; GCN-HSA-NEXT:    v_bfe_u32 v8, v1, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v5, v0, 16, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v1
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v1, 16, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v11, 0xff, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v2, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[17:18], v[11:14]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[3:6]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v16i8_to_v16i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v19, 24, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v17, v3, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v16, 0xff, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v18, v3, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v5, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v11, 24, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v9, v1, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v13, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xff, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v6, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v8, 0xff, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v10, v1, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xff, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v14, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v16i8_to_v16i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 39, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XYZW, T13.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T11.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T7.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T8.Z, T7.X, literal.x, PV.W,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T8.Y, T7.X, literal.x, T0.W,
+; EG-NEXT:     BFE_UINT T9.Z, T7.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR * T8.W, T7.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T8.X, T7.X, literal.x,
+; EG-NEXT:     BFE_UINT T9.Y, T7.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T10.Z, T7.Z, literal.x, T0.W,
+; EG-NEXT:     LSHR * T9.W, T7.Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     AND_INT T9.X, T7.Y, literal.x,
+; EG-NEXT:     BFE_UINT T10.Y, T7.Z, literal.y, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T11.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T12.Z, T7.W, literal.y, T0.W,
+; EG-NEXT:     LSHR T10.W, T7.Z, literal.z,
+; EG-NEXT:     AND_INT * T10.X, T7.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T12.Y, T7.W, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T13.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T12.W, T7.W, literal.y,
+; EG-NEXT:     AND_INT * T12.X, T7.W, literal.z,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T14.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v16i8_to_v16i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @8
+; CM-NEXT:    ALU 40, @11, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T14.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T13.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T9, T12.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T8, T10.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     MOV * T7.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 11:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T8.Z, T7.W, literal.x, PV.W,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T8.Y, T7.W, literal.x, T0.W,
+; CM-NEXT:     BFE_UINT T9.Z, T7.Z, literal.y, T0.W,
+; CM-NEXT:     LSHR * T8.W, T7.W, literal.z,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T8.X, T7.W, literal.x,
+; CM-NEXT:     BFE_UINT T9.Y, T7.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T10.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T11.Z, T7.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T9.W, T7.Z, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T9.X, T7.Z, literal.x,
+; CM-NEXT:     BFE_UINT T11.Y, T7.Y, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T12.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T7.Z, T7.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T11.W, T7.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T11.X, T7.Y, literal.x,
+; CM-NEXT:     BFE_UINT T7.Y, T7.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T13.X, PV.W, literal.x,
+; CM-NEXT:     LSHR * T7.W, T7.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:     AND_INT * T7.X, T7.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T14.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <16 x i8>, ptr addrspace(1) %in
   %ext = zext <16 x i8> %load to <16 x i32>
   store <16 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v16i8_to_v16i32:
-
-; EG: VTX_READ_128 [[DST:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_sextload_v16i8_to_v16i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v16i8_to_v16i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v7, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v6, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v5, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v4, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v11, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v10, v1, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v9, v1, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v8, v1, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v15, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v14, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v13, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v12, v2, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v3, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v3, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v16, v3, 0, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v16i8_to_v16i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s1
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 24, v3
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v3, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v3, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v3, 0, 8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 24, v0
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v0, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v5, v0, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v0, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[8:11]
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v15, 24, v2
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 24, v1
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v1, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v1, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v1, 0, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v14, v2, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v2, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v2, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[12:15]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v16i8_to_v16i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 24, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v3, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v3, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v16, v3, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v5, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v11, 24, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v10, v1, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v9, v1, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v1, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v12, v2, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v16i8_to_v16i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 47, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T13.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XYZW, T7.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T8.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHR T8.X, KC0[2].Y, literal.x,
+; EG-NEXT:     LSHR T0.W, T7.W, literal.y,
+; EG-NEXT:     LSHR * T1.W, T7.Z, literal.z,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T9.X, T7.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Y, T7.W, literal.y,
+; EG-NEXT:     LSHR T0.Z, T7.Z, literal.z,
+; EG-NEXT:     LSHR T2.W, T7.Y, literal.x,
+; EG-NEXT:     LSHR * T3.W, T7.X, literal.y,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T10.X, T7.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR T1.Y, T7.Z, literal.y,
+; EG-NEXT:     LSHR T1.Z, T7.Y, literal.y,
+; EG-NEXT:     BFE_INT T9.W, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR * T3.W, T7.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T11.X, T7.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T2.Y, T7.Y, literal.y,
+; EG-NEXT:     BFE_INT T9.Z, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T10.W, PV.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T3.W, T7.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T12.X, T7.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T9.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T10.Z, PV.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T11.W, T1.Y, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T3.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     LSHR T7.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T10.Y, T2.W, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T11.Z, T0.Z, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T12.W, T0.Y, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T13.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T11.Y, T1.W, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T12.Z, T0.W, 0.0, literal.y, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T0.W, T7.W, literal.y, BS:VEC_201
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T14.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T12.Y, PV.W, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v16i8_to_v16i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @8
+; CM-NEXT:    ALU 48, @11, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T13, T7.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T15.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T14.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T10, T9.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     MOV * T7.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 11:
+; CM-NEXT:     LSHR * T0.W, T7.X, literal.x,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T8.X, T7.Y, literal.x,
+; CM-NEXT:     ADD_INT T0.Y, KC0[2].Y, literal.y,
+; CM-NEXT:     LSHR T0.Z, T7.X, literal.z,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 48(6.726233e-44)
+; CM-NEXT:     LSHR T9.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T1.Y, T7.Y, literal.y,
+; CM-NEXT:     LSHR T1.Z, T7.Z, literal.z,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; CM-NEXT:     BFE_INT T10.X, T7.W, 0.0, literal.x,
+; CM-NEXT:     LSHR T2.Y, T7.Y, literal.y,
+; CM-NEXT:     LSHR T2.Z, T7.Z, literal.z,
+; CM-NEXT:     LSHR * T2.W, T7.W, literal.y,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T11.X, T7.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR T3.Y, T7.Z, literal.y,
+; CM-NEXT:     LSHR T3.Z, T7.W, literal.z,
+; CM-NEXT:     BFE_INT * T10.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T12.X, T7.Y, 0.0, literal.x,
+; CM-NEXT:     LSHR T4.Y, T7.W, literal.x,
+; CM-NEXT:     BFE_INT T10.Z, PV.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T11.W, PV.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T13.X, T7.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T10.Y, PV.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T11.Z, T2.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T12.W, T2.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T14.X, T1.W, literal.x,
+; CM-NEXT:     BFE_INT T11.Y, T1.Z, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T12.Z, T1.Y, 0.0, literal.y,
+; CM-NEXT:     BFE_INT * T13.W, T0.Z, 0.0, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T15.X, T0.Y, literal.x,
+; CM-NEXT:     BFE_INT T12.Y, T8.X, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T13.Z, T0.W, 0.0, literal.y,
+; CM-NEXT:     LSHR * T0.W, T7.X, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T13.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <16 x i8>, ptr addrspace(1) %in
   %ext = sext <16 x i8> %load to <16 x i32>
   store <16 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v32i8_to_v32i32:
-
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_zextload_v32i8_to_v32i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v32i8_to_v32i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v9, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v15, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v13, v1, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v19, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v17, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v23, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v21, v3, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v10, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v14, v1, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v16, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v18, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v20, 0xff, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v22, v3, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 24, v4
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v1, v4, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v27, 24, v5
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v25, v5, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v31, 24, v6
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v29, v6, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v35, 24, v7
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v33, v7, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v4
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v2, v4, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v24, 0xff, v5
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v26, v5, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v28, 0xff, v6
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v30, v6, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v32, 0xff, v7
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v34, v7, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v32i8_to_v32i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x70
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v11, 24, v7
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v7, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v7
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v7, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[8:11]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x60
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v10, 24, v6
+; GCN-HSA-NEXT:    v_bfe_u32 v8, v6, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v6
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v6, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[7:10]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x50
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 24, v5
+; GCN-HSA-NEXT:    v_bfe_u32 v7, v5, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v6, 0xff, v5
+; GCN-HSA-NEXT:    v_bfe_u32 v8, v5, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[6:9]
+; GCN-HSA-NEXT:    v_and_b32_e32 v5, 0xff, v4
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v8, 24, v4
+; GCN-HSA-NEXT:    v_bfe_u32 v6, v4, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v7, v4, 16, 8
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 64
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[5:8]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(4)
+; GCN-HSA-NEXT:    v_and_b32_e32 v4, 0xff, v3
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v7, 24, v3
+; GCN-HSA-NEXT:    v_bfe_u32 v5, v3, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v6, v3, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v11, 24, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v0, 8, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[4:7]
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v15, 24, v2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v6, 24, v1
+; GCN-HSA-NEXT:    v_bfe_u32 v4, v1, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v0, 16, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff, v1
+; GCN-HSA-NEXT:    v_bfe_u32 v5, v1, 16, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v12, 0xff, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v14, v2, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[3:6]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v32i8_to_v32i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v11, 24, v0
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v35, 24, v7
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v33, v7, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v32, 0xff, v7
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v34, v7, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v9, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 24, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v13, v1, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v19, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v17, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v23, 24, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v21, v3, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v8, 0xff, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v10, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xff, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v14, v1, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v16, 0xff, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v18, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v20, 0xff, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v22, v3, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v4
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v1, v4, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v27, 24, v5
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v25, v5, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v31, 24, v6
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v29, v6, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v4
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v4, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v24, 0xff, v5
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v26, v5, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v28, 0xff, v6
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v30, v6, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v32i8_to_v32i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @16, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @12
+; EG-NEXT:    ALU 75, @17, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T24.XYZW, T26.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T22.XYZW, T25.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T21.XYZW, T23.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T20.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T15.XYZW, T18.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T14.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T13.XYZW, T11.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 12:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; EG-NEXT:    ALU clause starting at 16:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 17:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T13.Z, T11.X, literal.x, PV.W,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T13.Y, T11.X, literal.x, T0.W,
+; EG-NEXT:     BFE_UINT T14.Z, T11.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR * T13.W, T11.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T13.X, T11.X, literal.x,
+; EG-NEXT:     BFE_UINT T14.Y, T11.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR * T11.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T15.Z, T11.Z, literal.x, T0.W,
+; EG-NEXT:     LSHR * T14.W, T11.Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     AND_INT T14.X, T11.Y, literal.x,
+; EG-NEXT:     BFE_UINT T15.Y, T11.Z, literal.y, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T16.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T17.Z, T11.W, literal.y, T0.W,
+; EG-NEXT:     LSHR T15.W, T11.Z, literal.z,
+; EG-NEXT:     AND_INT * T15.X, T11.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T17.Y, T11.W, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T18.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T19.Z, T12.X, literal.y, T0.W, BS:VEC_021/SCL_122
+; EG-NEXT:     LSHR T17.W, T11.W, literal.z,
+; EG-NEXT:     AND_INT * T17.X, T11.W, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T19.Y, T12.X, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 48(6.726233e-44)
+; EG-NEXT:     LSHR T20.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T21.Z, T12.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR T19.W, T12.X, literal.z,
+; EG-NEXT:     AND_INT * T19.X, T12.X, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T21.Y, T12.Y, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 64(8.968310e-44)
+; EG-NEXT:     LSHR T12.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T22.Z, T12.Z, literal.y, T0.W,
+; EG-NEXT:     LSHR T21.W, T12.Y, literal.z,
+; EG-NEXT:     AND_INT * T21.X, T12.Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T22.Y, T12.Z, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 80(1.121039e-43)
+; EG-NEXT:     LSHR T23.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T24.Z, T12.W, literal.y, T0.W,
+; EG-NEXT:     LSHR T22.W, T12.Z, literal.z,
+; EG-NEXT:     AND_INT * T22.X, T12.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T24.Y, T12.W, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 96(1.345247e-43)
+; EG-NEXT:     LSHR T25.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T24.W, T12.W, literal.y,
+; EG-NEXT:     AND_INT * T24.X, T12.W, literal.z,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T26.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v32i8_to_v32i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @16, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @12
+; CM-NEXT:    ALU 80, @17, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T26.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T23, T25.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T21, T24.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T19, T22.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T20.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T16, T18.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T14, T17.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T13, T15.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 12:
+; CM-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 0, #1
+; CM-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 16, #1
+; CM-NEXT:    ALU clause starting at 16:
+; CM-NEXT:     MOV * T11.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 17:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T13.Z, T11.W, literal.x, PV.W,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T13.Y, T11.W, literal.x, T0.W,
+; CM-NEXT:     BFE_UINT T14.Z, T11.Z, literal.y, T0.W,
+; CM-NEXT:     LSHR * T13.W, T11.W, literal.z,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T13.X, T11.W, literal.x,
+; CM-NEXT:     BFE_UINT T14.Y, T11.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T15.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T16.Z, T11.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T14.W, T11.Z, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T14.X, T11.Z, literal.x,
+; CM-NEXT:     BFE_UINT T16.Y, T11.Y, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    96(1.345247e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T17.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T11.Z, T11.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T16.W, T11.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T16.X, T11.Y, literal.x,
+; CM-NEXT:     BFE_UINT T11.Y, T11.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    80(1.121039e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T18.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T19.Z, T12.W, literal.y, T0.W,
+; CM-NEXT:     LSHR * T11.W, T11.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T11.X, T11.X, literal.x,
+; CM-NEXT:     BFE_UINT T19.Y, T12.W, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    64(8.968310e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T20.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T21.Z, T12.Z, literal.y, T0.W,
+; CM-NEXT:     LSHR * T19.W, T12.W, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T19.X, T12.W, literal.x,
+; CM-NEXT:     BFE_UINT T21.Y, T12.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T22.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T23.Z, T12.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T21.W, T12.Z, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T21.X, T12.Z, literal.x,
+; CM-NEXT:     BFE_UINT T23.Y, T12.Y, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T24.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T12.Z, T12.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T23.W, T12.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T23.X, T12.Y, literal.x,
+; CM-NEXT:     BFE_UINT T12.Y, T12.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T25.X, PV.W, literal.x,
+; CM-NEXT:     LSHR * T12.W, T12.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:     AND_INT * T12.X, T12.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T26.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = zext <32 x i8> %load to <32 x i32>
   store <32 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v32i8_to_v32i32:
-
-; EG-DAG: VTX_READ_128 [[DST_LO:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 [[DST_HI:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 16, #1
-; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
-; EG-DAG: 8
+; TODO: EG/CM should use DST, but for some there are redundant MOVs
 define amdgpu_kernel void @global_sextload_v32i8_to_v32i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v32i8_to_v32i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v11, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v10, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v9, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v8, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v15, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v14, v1, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v13, v1, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v12, v1, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v16, v2, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v3, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v21, v3, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v20, v3, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 24, v4
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v4, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v4, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v4, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v27, 24, v5
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v26, v5, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v25, v5, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v24, v5, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v31, 24, v6
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v30, v6, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v29, v6, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v28, v6, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v35, 24, v7
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v34, v7, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v33, v7, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v32, v7, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v32i8_to_v32i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x70
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v10, 24, v7
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v7, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v7, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v7, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[13:14], v[7:10]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x60
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 24, v6
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v6, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v6, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v6, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s3
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[6:9]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x50
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v8, 24, v5
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v5, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v5, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v5, v5, 0, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[17:18], v[5:8]
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 64
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 24, v4
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v4, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v5, v4, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[11:12], v[4:7]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(4)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 24, v3
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v3, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v3, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v3, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 24, v0
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v0, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v5, v0, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v0, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[13:14], v[8:11]
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v15, 24, v2
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 24, v1
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v1, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v1, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v1, 0, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v14, v2, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v2, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v2, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[19:20], v[12:15]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[8:11]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v32i8_to_v32i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v11, 24, v0
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v35, 24, v7
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v34, v7, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v33, v7, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v32, v7, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v10, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v9, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 24, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v1, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v1, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v12, v1, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v16, v2, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v23, 24, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v22, v3, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v3, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v3, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 24, v4
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v4, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v4, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v4, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v27, 24, v5
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v26, v5, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v25, v5, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v24, v5, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v31, 24, v6
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v30, v6, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v29, v6, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v28, v6, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v32i8_to_v32i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @18, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @14
+; EG-NEXT:    ALU 18, @19, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @16
+; EG-NEXT:    ALU 75, @38, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T24.XYZW, T26.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T23.XYZW, T25.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T22.XYZW, T17.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T21.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T15.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T18.XYZW, T13.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 14:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; EG-NEXT:    Fetch clause starting at 16:
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; EG-NEXT:    ALU clause starting at 18:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 19:
+; EG-NEXT:     LSHR T13.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T14.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T15.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T0.Z, T12.W, literal.y,
+; EG-NEXT:     LSHR T0.W, T12.Z, literal.z,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    8(1.121039e-44), 48(6.726233e-44)
+; EG-NEXT:     LSHR T16.X, PS, literal.x,
+; EG-NEXT:     LSHR T0.Y, T12.W, literal.y,
+; EG-NEXT:     LSHR T1.Z, T12.Z, literal.z,
+; EG-NEXT:     LSHR T1.W, T12.Y, literal.w,
+; EG-NEXT:     LSHR * T2.W, T12.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; EG-NEXT:    ALU clause starting at 38:
+; EG-NEXT:     ADD_INT * T3.W, KC0[2].Y, literal.x,
+; EG-NEXT:    64(8.968310e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T17.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T1.Y, T12.Y, literal.y,
+; EG-NEXT:     LSHR T2.Z, T12.Y, literal.z,
+; EG-NEXT:     LSHR T3.W, T12.X, literal.y,
+; EG-NEXT:     LSHR * T4.W, T12.X, literal.z,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T18.X, T11.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T2.Y, T11.W, literal.y,
+; EG-NEXT:     LSHR T3.Z, T11.W, literal.z,
+; EG-NEXT:     LSHR T5.W, T11.Z, literal.y,
+; EG-NEXT:     LSHR * T6.W, T11.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T19.X, T11.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR T3.Y, T11.Z, literal.y,
+; EG-NEXT:     LSHR T4.Z, T11.Y, literal.y,
+; EG-NEXT:     BFE_INT T18.W, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR * T6.W, T11.X, literal.z,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T20.X, T11.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T4.Y, T11.Y, literal.y,
+; EG-NEXT:     BFE_INT T18.Z, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T19.W, PV.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T6.W, T11.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T21.X, T11.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T18.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T19.Z, PV.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T20.W, T3.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR * T6.W, T11.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T22.X, T12.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T19.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T20.Z, T5.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T21.W, T3.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T5.W, T11.Z, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T11.X, T12.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T20.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T21.Z, T2.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     BFE_INT T22.W, T4.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T4.W, T11.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T23.X, T12.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T21.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T22.Z, T3.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T11.W, T2.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR * T3.W, T12.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T24.X, T12.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T22.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T11.Z, T1.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T23.W, T2.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 80(1.121039e-43)
+; EG-NEXT:     LSHR T12.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T11.Y, T1.W, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T23.Z, T1.Z, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T24.W, T0.Y, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    96(1.345247e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T25.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T23.Y, T0.W, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T24.Z, T0.Z, 0.0, literal.y,
+; EG-NEXT:     LSHR T0.W, T12.W, literal.y, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T26.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T24.Y, PV.W, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v32i8_to_v32i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @18, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @14
+; CM-NEXT:    ALU 19, @19, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @16
+; CM-NEXT:    ALU 78, @39, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T26, T12.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T25, T14.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T24, T27.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T23, T19.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T18.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T22, T17.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T21, T16.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T20, T15.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 14:
+; CM-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 0, #1
+; CM-NEXT:    Fetch clause starting at 16:
+; CM-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 16, #1
+; CM-NEXT:    ALU clause starting at 18:
+; CM-NEXT:     MOV * T11.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 19:
+; CM-NEXT:     LSHR * T0.W, T12.X, literal.x,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T13.X, T12.Y, literal.x,
+; CM-NEXT:     ADD_INT T0.Y, KC0[2].Y, literal.y,
+; CM-NEXT:     LSHR T0.Z, T12.X, literal.z,
+; CM-NEXT:     LSHR * T1.W, T12.Y, literal.y,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T14.X, T12.Z, literal.x,
+; CM-NEXT:     ADD_INT T1.Y, KC0[2].Y, literal.y,
+; CM-NEXT:     LSHR T1.Z, T12.Y, literal.z,
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.w,
+; CM-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; CM-NEXT:    24(3.363116e-44), 112(1.569454e-43)
+; CM-NEXT:     LSHR T15.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T2.Y, T12.Z, literal.y,
+; CM-NEXT:     LSHR T2.Z, T12.W, literal.z,
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 96(1.345247e-43)
+; CM-NEXT:    ALU clause starting at 39:
+; CM-NEXT:     LSHR T16.X, T2.W, literal.x,
+; CM-NEXT:     LSHR T3.Y, T12.Z, literal.y,
+; CM-NEXT:     LSHR T3.Z, T12.W, literal.z, BS:VEC_120/SCL_212
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 80(1.121039e-43)
+; CM-NEXT:     LSHR T17.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T4.Y, T11.X, literal.y,
+; CM-NEXT:     LSHR T4.Z, T12.W, literal.z,
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    24(3.363116e-44), 64(8.968310e-44)
+; CM-NEXT:     LSHR T18.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T5.Y, T11.X, literal.y,
+; CM-NEXT:     LSHR T5.Z, T11.Y, literal.z,
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 48(6.726233e-44)
+; CM-NEXT:     LSHR T19.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T6.Y, T11.X, literal.y,
+; CM-NEXT:     LSHR T6.Z, T11.Y, literal.z,
+; CM-NEXT:     LSHR * T2.W, T11.Z, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; CM-NEXT:     BFE_INT T20.X, T11.W, 0.0, literal.x,
+; CM-NEXT:     LSHR T7.Y, T11.Y, literal.y,
+; CM-NEXT:     LSHR T7.Z, T11.Z, literal.z,
+; CM-NEXT:     LSHR * T3.W, T11.W, literal.y,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T21.X, T11.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR T8.Y, T11.Z, literal.y,
+; CM-NEXT:     LSHR T8.Z, T11.W, literal.z,
+; CM-NEXT:     BFE_INT * T20.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T22.X, T11.Y, 0.0, literal.x,
+; CM-NEXT:     LSHR T9.Y, T11.W, literal.x,
+; CM-NEXT:     BFE_INT T20.Z, PV.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T21.W, PV.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T11.X, T11.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T20.Y, PV.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T21.Z, T7.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T22.W, T7.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T23.X, T12.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T21.Y, T2.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT T22.Z, T6.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T11.W, T6.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T24.X, T12.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T22.Y, T5.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT T11.Z, T5.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T23.W, T4.Z, 0.0, literal.x, BS:VEC_201
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T25.X, T12.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T11.Y, T4.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT T23.Z, T3.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T24.W, T3.Y, 0.0, literal.x, BS:VEC_201
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T26.X, T12.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T23.Y, T2.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T24.Z, T2.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T25.W, T1.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T27.X, T1.Y, literal.x,
+; CM-NEXT:     BFE_INT T24.Y, T14.X, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T25.Z, T1.W, 0.0, literal.y,
+; CM-NEXT:     BFE_INT * T26.W, T0.Z, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T14.X, T0.Y, literal.x,
+; CM-NEXT:     BFE_INT T25.Y, T13.X, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T26.Z, T0.W, 0.0, literal.y,
+; CM-NEXT:     LSHR * T0.W, T12.X, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T12.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T26.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = sext <32 x i8> %load to <32 x i32>
   store <32 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v64i8_to_v64i32:
-
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 16, #1
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 32, #1
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 48, #1
 define amdgpu_kernel void @global_zextload_v64i8_to_v64i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v64i8_to_v64i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s14, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s15, 0xe8f000
+; GCN-NOHSA-SI-NEXT:    s_add_u32 s12, s12, s11
+; GCN-NOHSA-SI-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[12:15], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 24, v13
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v5, v13, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 24, v12
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v9, v12, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v19, 24, v15
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v17, v15, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v23, 24, v14
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v21, v14, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff, v13
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v6, v13, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v4, off, s[12:15], 0 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v5, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v6, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v7, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xff, v12
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v10, v12, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v16, 0xff, v15
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v18, v15, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v20, 0xff, v14
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v22, v14, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v15, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v13, v1, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v35, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v33, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v39, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v37, v3, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v43, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v41, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v14, v1, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v32, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v34, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v36, 0xff, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v38, v3, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v40, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v42, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(5) expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 24, v25
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v5, v25, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v47, 24, v24
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v45, v24, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v51, 24, v27
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v49, v27, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v55, 24, v26
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v53, v26, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff, v25
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v6, v25, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v44, 0xff, v24
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v46, v24, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v48, 0xff, v27
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v50, v27, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v52, 0xff, v26
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v54, v26, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v27, 24, v29
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v25, v29, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v59, 24, v28
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v57, v28, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v63, 24, v31
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v61, v31, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 24, v30
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v1, v30, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v24, 0xff, v29
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v26, v29, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v56, 0xff, v28
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v58, v28, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v60, 0xff, v31
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v62, v31, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v30
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v2, v30, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:240
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:192
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:160
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:128
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:144
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v64i8_to_v64i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    s_add_u32 s4, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s5, s3, 0
+; GCN-HSA-NEXT:    s_add_u32 s6, s2, 32
+; GCN-HSA-NEXT:    s_addc_u32 s7, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[4:5]
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[4:5]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s5
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s8, s0, 0xe0
+; GCN-HSA-NEXT:    s_addc_u32 s9, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s10, s0, 0xf0
+; GCN-HSA-NEXT:    s_addc_u32 s11, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s12, s0, 0xc0
+; GCN-HSA-NEXT:    s_addc_u32 s13, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s14, s0, 0xd0
+; GCN-HSA-NEXT:    s_addc_u32 s15, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s16, s0, 0xa0
+; GCN-HSA-NEXT:    s_addc_u32 s17, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s17
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s16
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(3)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v19, 24, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v17, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v16, 0xff, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v18, v2, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s3
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(3)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v19, 24, v14
+; GCN-HSA-NEXT:    v_bfe_u32 v17, v14, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v16, 0xff, v14
+; GCN-HSA-NEXT:    v_bfe_u32 v18, v14, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[16:19]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xb0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v19, 24, v15
+; GCN-HSA-NEXT:    v_bfe_u32 v17, v15, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v16, 0xff, v15
+; GCN-HSA-NEXT:    v_bfe_u32 v18, v15, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[16:19]
+; GCN-HSA-NEXT:    v_bfe_u32 v15, v12, 8, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v17, 24, v12
+; GCN-HSA-NEXT:    v_and_b32_e32 v14, 0xff, v12
+; GCN-HSA-NEXT:    v_bfe_u32 v16, v12, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[14:17]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s9
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v17, 24, v13
+; GCN-HSA-NEXT:    v_bfe_u32 v15, v13, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v14, 0xff, v13
+; GCN-HSA-NEXT:    v_bfe_u32 v16, v13, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s6
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[14:17]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(6)
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v10, 8, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s11
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v15, 24, v10
+; GCN-HSA-NEXT:    v_and_b32_e32 v12, 0xff, v10
+; GCN-HSA-NEXT:    v_bfe_u32 v14, v10, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s10
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[12:15]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s13
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v15, 24, v11
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v11, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v12, 0xff, v11
+; GCN-HSA-NEXT:    v_bfe_u32 v14, v11, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s12
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v13, 24, v8
+; GCN-HSA-NEXT:    v_bfe_u32 v11, v8, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v8
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v8, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s15
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s14
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v13, 24, v9
+; GCN-HSA-NEXT:    v_bfe_u32 v11, v9, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v9
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v9, 16, 8
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x80
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[10:13]
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v3, 8, 8
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v11, 24, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v3
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v3, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x90
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[17:18], v[8:11]
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v20, 24, v1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v11, 24, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v0, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v0, 16, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v18, v1, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v17, 0xff, v1
+; GCN-HSA-NEXT:    v_bfe_u32 v19, v1, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x60
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(10)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v16, 24, v5
+; GCN-HSA-NEXT:    v_bfe_u32 v14, v5, 8, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[2:3], v[8:11]
+; GCN-HSA-NEXT:    v_and_b32_e32 v13, 0xff, v5
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v11, 24, v4
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v4, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v15, v5, 16, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v4
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v4, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x70
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[17:20]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v20, 24, v6
+; GCN-HSA-NEXT:    v_bfe_u32 v18, v6, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v17, 0xff, v6
+; GCN-HSA-NEXT:    v_bfe_u32 v19, v6, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[17:20]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 24, v7
+; GCN-HSA-NEXT:    v_bfe_u32 v1, v7, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v0, 0xff, v7
+; GCN-HSA-NEXT:    v_bfe_u32 v2, v7, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 64
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 0x50
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[13:16]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v64i8_to_v64i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s90, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s91, 0xe80000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_add_u32 s88, s88, s11
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[32:35], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[36:39], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    s_addc_u32 s89, s89, 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v17
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v1, v17, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v17
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v17, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[88:91], 0 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v2, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v3, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v38
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v1, v38, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v38
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v38, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 24, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v5, v16, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v11, 24, v19
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v9, v19, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 24, v18
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v13, v18, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xff, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v6, v16, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v8, 0xff, v19
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v10, v19, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xff, v18
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v14, v18, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v19, 24, v29
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v17, v29, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v23, 24, v28
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v21, v28, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v27, 24, v31
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v25, v31, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v43, 24, v30
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v41, v30, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v16, 0xff, v29
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v18, v29, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v20, 0xff, v28
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v22, v28, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v24, 0xff, v31
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v26, v31, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v40, 0xff, v30
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v42, v30, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v31, 24, v33
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v29, v33, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v47, 24, v32
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v45, v32, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v51, 24, v35
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v49, v35, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v55, 24, v34
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v53, v34, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v28, 0xff, v33
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v30, v33, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v44, 0xff, v32
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v46, v32, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v48, 0xff, v35
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v50, v35, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v52, 0xff, v34
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v54, v34, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v35, 24, v37
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v33, v37, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v59, 24, v36
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v57, v36, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v63, 24, v39
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v61, v39, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v32, 0xff, v37
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v34, v37, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v56, 0xff, v36
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v58, v36, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v60, 0xff, v39
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v62, v39, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:192
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[88:91], 0 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v2, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v3, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v64i8_to_v64i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @30, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @22
+; EG-NEXT:    ALU 59, @31, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @26
+; EG-NEXT:    ALU 88, @91, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T48.XYZW, T50.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T47.XYZW, T49.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T45.XYZW, T32.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T43.XYZW, T46.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T41.XYZW, T44.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T40.XYZW, T42.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T38.XYZW, T33.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T36.XYZW, T39.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T34.XYZW, T37.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T21.XYZW, T35.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T30.XYZW, T22.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T28.XYZW, T31.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T26.XYZW, T29.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T25.XYZW, T27.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T23.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T24.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 22:
+; EG-NEXT:     VTX_READ_128 T22.XYZW, T21.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T23.XYZW, T21.X, 0, #1
+; EG-NEXT:    Fetch clause starting at 26:
+; EG-NEXT:     VTX_READ_128 T32.XYZW, T21.X, 48, #1
+; EG-NEXT:     VTX_READ_128 T33.XYZW, T21.X, 32, #1
+; EG-NEXT:    ALU clause starting at 30:
+; EG-NEXT:     MOV * T21.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 31:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T19.Z, T23.Y, literal.x, PV.W,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T19.Y, T23.Y, literal.x, T0.W,
+; EG-NEXT:     BFE_UINT T20.Z, T23.X, literal.y, T0.W,
+; EG-NEXT:     LSHR * T19.W, T23.Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T19.X, T23.Y, literal.x,
+; EG-NEXT:     BFE_UINT T20.Y, T23.X, literal.y, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T24.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T25.Z, T23.W, literal.y, T0.W,
+; EG-NEXT:     LSHR T20.W, T23.X, literal.z,
+; EG-NEXT:     AND_INT * T20.X, T23.X, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT * T25.Y, T23.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T23.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_UINT T26.Z, T23.Z, literal.y, T0.W, BS:VEC_021/SCL_122
+; EG-NEXT:     LSHR T25.W, T23.W, literal.z,
+; EG-NEXT:     AND_INT * T25.X, T23.W, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T26.Y, T23.Z, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 48(6.726233e-44)
+; EG-NEXT:     LSHR T27.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T28.Z, T22.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR T26.W, T23.Z, literal.z,
+; EG-NEXT:     AND_INT * T26.X, T23.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T28.Y, T22.Y, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T29.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T30.Z, T22.X, literal.y, T0.W,
+; EG-NEXT:     LSHR T28.W, T22.Y, literal.z,
+; EG-NEXT:     AND_INT * T28.X, T22.Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T30.Y, T22.X, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 80(1.121039e-43)
+; EG-NEXT:     LSHR T31.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T30.W, T22.X, literal.y,
+; EG-NEXT:     AND_INT * T30.X, T22.X, literal.z,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T21.Z, T22.W, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 64(8.968310e-44)
+; EG-NEXT:     LSHR T22.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT * T21.Y, T22.W, literal.y, T0.W,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    ALU clause starting at 91:
+; EG-NEXT:     BFE_UINT T34.Z, T22.Z, literal.x, T0.W,
+; EG-NEXT:     LSHR * T21.W, T22.W, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     AND_INT T21.X, T22.W, literal.x,
+; EG-NEXT:     BFE_UINT T34.Y, T22.Z, literal.y, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T35.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T36.Z, T33.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR T34.W, T22.Z, literal.z,
+; EG-NEXT:     AND_INT * T34.X, T22.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T36.Y, T33.Y, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 96(1.345247e-43)
+; EG-NEXT:     LSHR T37.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T38.Z, T33.X, literal.y, T0.W,
+; EG-NEXT:     LSHR T36.W, T33.Y, literal.z,
+; EG-NEXT:     AND_INT * T36.X, T33.Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T38.Y, T33.X, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 144(2.017870e-43)
+; EG-NEXT:     LSHR T39.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T40.Z, T33.W, literal.y, T0.W,
+; EG-NEXT:     LSHR T38.W, T33.X, literal.z,
+; EG-NEXT:     AND_INT * T38.X, T33.X, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T40.Y, T33.W, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 128(1.793662e-43)
+; EG-NEXT:     LSHR T33.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T41.Z, T33.Z, literal.y, T0.W, BS:VEC_021/SCL_122
+; EG-NEXT:     LSHR T40.W, T33.W, literal.z,
+; EG-NEXT:     AND_INT * T40.X, T33.W, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T41.Y, T33.Z, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 176(2.466285e-43)
+; EG-NEXT:     LSHR T42.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T43.Z, T32.Y, literal.y, T0.W,
+; EG-NEXT:     LSHR T41.W, T33.Z, literal.z,
+; EG-NEXT:     AND_INT * T41.X, T33.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T43.Y, T32.Y, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 160(2.242078e-43)
+; EG-NEXT:     LSHR T44.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T45.Z, T32.X, literal.y, T0.W,
+; EG-NEXT:     LSHR T43.W, T32.Y, literal.z,
+; EG-NEXT:     AND_INT * T43.X, T32.Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T45.Y, T32.X, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 208(2.914701e-43)
+; EG-NEXT:     LSHR T46.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T47.Z, T32.W, literal.y, T0.W,
+; EG-NEXT:     LSHR T45.W, T32.X, literal.z,
+; EG-NEXT:     AND_INT * T45.X, T32.X, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T47.Y, T32.W, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 192(2.690493e-43)
+; EG-NEXT:     LSHR T32.X, PV.W, literal.x,
+; EG-NEXT:     BFE_UINT T48.Z, T32.Z, literal.y, T0.W, BS:VEC_021/SCL_122
+; EG-NEXT:     LSHR T47.W, T32.W, literal.z,
+; EG-NEXT:     AND_INT * T47.X, T32.W, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 255(3.573311e-43)
+; EG-NEXT:     BFE_UINT T48.Y, T32.Z, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 240(3.363116e-43)
+; EG-NEXT:     LSHR T49.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T48.W, T32.Z, literal.y,
+; EG-NEXT:     AND_INT * T48.X, T32.Z, literal.z,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    224(3.138909e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T50.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v64i8_to_v64i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @30, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @22
+; CM-NEXT:    ALU 63, @31, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @26
+; CM-NEXT:    ALU 95, @95, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T49, T50.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T47, T32.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T45, T48.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T43, T46.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T42, T44.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T40, T33.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T38, T41.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T36, T39.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T35, T37.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T19, T20.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T30, T34.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T28, T31.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T27, T29.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T25, T21.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T23, T26.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T22, T24.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 22:
+; CM-NEXT:     VTX_READ_128 T20.XYZW, T19.X, 32, #1
+; CM-NEXT:     VTX_READ_128 T21.XYZW, T19.X, 48, #1
+; CM-NEXT:    Fetch clause starting at 26:
+; CM-NEXT:     VTX_READ_128 T32.XYZW, T19.X, 0, #1
+; CM-NEXT:     VTX_READ_128 T33.XYZW, T19.X, 16, #1
+; CM-NEXT:    ALU clause starting at 30:
+; CM-NEXT:     MOV * T19.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 31:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T22.Z, T21.Z, literal.x, PV.W,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T22.Y, T21.Z, literal.x, T0.W,
+; CM-NEXT:     BFE_UINT T23.Z, T21.W, literal.y, T0.W,
+; CM-NEXT:     LSHR * T22.W, T21.Z, literal.z,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T22.X, T21.Z, literal.x,
+; CM-NEXT:     BFE_UINT T23.Y, T21.W, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    224(3.138909e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T24.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T25.Z, T21.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T23.W, T21.W, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T23.X, T21.W, literal.x,
+; CM-NEXT:     BFE_UINT T25.Y, T21.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    240(3.363116e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T26.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T27.Z, T21.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T25.W, T21.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T25.X, T21.X, literal.x,
+; CM-NEXT:     BFE_UINT T27.Y, T21.Y, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    192(2.690493e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T21.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T28.Z, T20.Z, literal.y, T0.W,
+; CM-NEXT:     LSHR * T27.W, T21.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T27.X, T21.Y, literal.x,
+; CM-NEXT:     BFE_UINT T28.Y, T20.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    208(2.914701e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T29.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T30.Z, T20.W, literal.y, T0.W,
+; CM-NEXT:     LSHR * T28.W, T20.Z, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T28.X, T20.Z, literal.x,
+; CM-NEXT:     BFE_UINT T30.Y, T20.W, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    160(2.242078e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T31.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T19.Z, T20.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T30.W, T20.W, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T30.X, T20.W, literal.x,
+; CM-NEXT:     BFE_UINT T19.Y, T20.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    176(2.466285e-43), 0(0.000000e+00)
+; CM-NEXT:    ALU clause starting at 95:
+; CM-NEXT:     LSHR T34.X, T1.W, literal.x,
+; CM-NEXT:     BFE_UINT T35.Z, T20.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T19.W, T20.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T19.X, T20.X, literal.x,
+; CM-NEXT:     BFE_UINT T35.Y, T20.Y, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    128(1.793662e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T20.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T36.Z, T33.Z, literal.y, T0.W,
+; CM-NEXT:     LSHR * T35.W, T20.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T35.X, T20.Y, literal.x,
+; CM-NEXT:     BFE_UINT T36.Y, T33.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    144(2.017870e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T37.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T38.Z, T33.W, literal.y, T0.W,
+; CM-NEXT:     LSHR * T36.W, T33.Z, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T36.X, T33.Z, literal.x,
+; CM-NEXT:     BFE_UINT T38.Y, T33.W, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    96(1.345247e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T39.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T40.Z, T33.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T38.W, T33.W, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T38.X, T33.W, literal.x,
+; CM-NEXT:     BFE_UINT T40.Y, T33.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T41.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T42.Z, T33.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T40.W, T33.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T40.X, T33.X, literal.x,
+; CM-NEXT:     BFE_UINT T42.Y, T33.Y, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    64(8.968310e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T33.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T43.Z, T32.Z, literal.y, T0.W,
+; CM-NEXT:     LSHR * T42.W, T33.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T42.X, T33.Y, literal.x,
+; CM-NEXT:     BFE_UINT T43.Y, T32.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    80(1.121039e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T44.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T45.Z, T32.W, literal.y, T0.W,
+; CM-NEXT:     LSHR * T43.W, T32.Z, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T43.X, T32.Z, literal.x,
+; CM-NEXT:     BFE_UINT T45.Y, T32.W, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T46.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T47.Z, T32.X, literal.y, T0.W,
+; CM-NEXT:     LSHR * T45.W, T32.W, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T45.X, T32.W, literal.x,
+; CM-NEXT:     BFE_UINT T47.Y, T32.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T48.X, PV.W, literal.x,
+; CM-NEXT:     BFE_UINT T49.Z, T32.Y, literal.y, T0.W,
+; CM-NEXT:     LSHR * T47.W, T32.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T47.X, T32.X, literal.x,
+; CM-NEXT:     BFE_UINT * T49.Y, T32.Y, literal.y, T0.W,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:     LSHR T32.X, KC0[2].Y, literal.x,
+; CM-NEXT:     LSHR * T49.W, T32.Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:     AND_INT T49.X, T32.Y, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    255(3.573311e-43), 16(2.242078e-44)
+; CM-NEXT:     LSHR * T50.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <64 x i8>, ptr addrspace(1) %in
   %ext = zext <64 x i8> %load to <64 x i32>
   store <64 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v64i8_to_v64i32:
-
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 16, #1
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 32, #1
-; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 48, #1
 define amdgpu_kernel void @global_sextload_v64i8_to_v64i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v64i8_to_v64i32:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s14, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s15, 0xe8f000
+; GCN-NOHSA-SI-NEXT:    s_add_u32 s12, s12, s11
+; GCN-NOHSA-SI-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s7
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[10:13], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[16:19], off, s[4:7], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[20:23], off, s[4:7], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[24:27], off, s[4:7], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 24, v11
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v11, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v11, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v11, 0, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v7, 24, v10
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v6, v10, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v5, v10, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v4, v10, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v11, 24, v13
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v10, v13, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v9, v13, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v8, v13, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v15, 24, v12
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v14, v12, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v13, v12, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v12, v12, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v31, 24, v17
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v30, v17, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v29, v17, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v28, v17, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v35, 24, v16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v34, v16, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v33, v16, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v32, v16, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v39, 24, v19
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v38, v19, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v37, v19, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v36, v19, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v43, 24, v18
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v42, v18, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v41, v18, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v40, v18, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(5)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 24, v21
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v21, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v21, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v16, v21, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v47, 24, v20
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v46, v20, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v45, v20, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v44, v20, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v51, 24, v23
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v50, v23, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v49, v23, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v48, v23, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v55, 24, v22
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v54, v22, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v53, v22, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v52, v22, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 24, v25
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v25, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v21, v25, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v20, v25, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v59, 24, v24
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v58, v24, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v57, v24, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v56, v24, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v63, 24, v27
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v62, v27, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v61, v27, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v60, v27, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 24, v26
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v26, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v26, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v26, 0, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:240
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:192
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:160
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:128
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:144
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v64i8_to_v64i32:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s4, s2, 48
+; GCN-HSA-NEXT:    s_addc_u32 s5, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s4, s2, 32
+; GCN-HSA-NEXT:    s_addc_u32 s5, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(3)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v19, 24, v14
+; GCN-HSA-NEXT:    v_bfe_i32 v18, v14, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v17, v14, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v14, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s3
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v18, 24, v15
+; GCN-HSA-NEXT:    v_bfe_i32 v17, v15, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v15, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v15, v15, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[19:20], v[15:18]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s1
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 24, v12
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v12, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v15, v12, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v14, v12, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[14:17]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xe0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xf0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v16, 24, v13
+; GCN-HSA-NEXT:    v_bfe_i32 v15, v13, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v14, v13, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v13, 0, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[17:18], v[13:16]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xc0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(6)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v15, 24, v10
+; GCN-HSA-NEXT:    v_bfe_i32 v14, v10, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v10, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v10, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[19:20], v[12:15]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xd0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v14, 24, v11
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v11, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v11, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v11, v11, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[11:14]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xa0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v13, 24, v8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v8, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v11, v8, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v8, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s3
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s2
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v12, 24, v9
+; GCN-HSA-NEXT:    v_bfe_i32 v11, v9, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v9, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v9, 0, 8
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xb0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[9:12]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(9)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v14, 24, v6
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v6, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v6, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v11, v6, 0, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[11:14]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x80
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v10, 24, v7
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v7, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v7, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v7, 0, 8
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[7:10]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s3
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 24, v4
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v4, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v4, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v4, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x90
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(10)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v14, 24, v1
+; GCN-HSA-NEXT:    v_bfe_i32 v13, v1, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v1, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v11, v1, 0, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[6:9]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 24, v0
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v0, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v0, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v0, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v18, 24, v5
+; GCN-HSA-NEXT:    v_bfe_i32 v17, v5, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v5, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v15, v5, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x60
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[15:18]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v18, 24, v3
+; GCN-HSA-NEXT:    v_bfe_i32 v17, v3, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v3, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v15, v3, 0, 8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v5, 24, v2
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v2, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v3, v2, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x70
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 64
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[15:18]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 0x50
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[6:9]
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v64i8_to_v64i32:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s90, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s91, 0xe80000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_add_u32 s88, s88, s11
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[10:13], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[26:29], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[30:33], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[34:37], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    s_addc_u32 s89, s89, 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(3)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 24, v11
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v11, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v11, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v11, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[88:91], 0 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v2, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v3, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 24, v36
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v36, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v1, v36, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v36, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 24, v10
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v10, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v5, v10, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v10, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v11, 24, v13
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v10, v13, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v9, v13, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v13, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 24, v12
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v12, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v12, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v12, v12, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 24, v27
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v27, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v27, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v16, v27, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v23, 24, v26
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v22, v26, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v26, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v26, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v27, 24, v29
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v26, v29, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v25, v29, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v24, v29, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v41, 24, v28
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v40, v28, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v39, v28, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v38, v28, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v45, 24, v31
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v44, v31, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v43, v31, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v42, v31, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v49, 24, v30
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v48, v30, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v47, v30, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v46, v30, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v31, 24, v33
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v30, v33, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v29, v33, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v28, v33, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v53, 24, v32
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v52, v32, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v51, v32, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v50, v32, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v57, 24, v35
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v56, v35, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v55, v35, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v54, v35, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v61, 24, v34
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v60, v34, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v59, v34, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v58, v34, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v35, 24, v37
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v34, v37, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v33, v37, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v32, v37, 0, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[58:61], off, s[0:3], 0 offset:192
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[54:57], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[50:53], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[46:49], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[42:45], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[38:41], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[88:91], 0 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v2, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v3, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v64i8_to_v64i32:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @32, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @24
+; EG-NEXT:    ALU 41, @33, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @28
+; EG-NEXT:    ALU 76, @75, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 72, @152, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T48.XYZW, T50.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T47.XYZW, T49.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T46.XYZW, T19.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T35.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T45.XYZW, T34.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T44.XYZW, T33.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T43.XYZW, T32.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T21.XYZW, T30.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T42.XYZW, T29.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T41.XYZW, T28.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T40.XYZW, T27.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T31.XYZW, T26.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T39.XYZW, T25.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T38.XYZW, T24.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T37.XYZW, T23.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T36.XYZW, T22.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 24:
+; EG-NEXT:     VTX_READ_128 T20.XYZW, T21.X, 32, #1
+; EG-NEXT:     VTX_READ_128 T19.XYZW, T21.X, 48, #1
+; EG-NEXT:    Fetch clause starting at 28:
+; EG-NEXT:     VTX_READ_128 T31.XYZW, T21.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T21.XYZW, T21.X, 16, #1
+; EG-NEXT:    ALU clause starting at 32:
+; EG-NEXT:     MOV * T21.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 33:
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T22.X, PV.W, literal.x,
+; EG-NEXT:     LSHR * T23.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T24.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T25.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+; EG-NEXT:     LSHR T26.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; EG-NEXT:     LSHR T27.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 112(1.569454e-43)
+; EG-NEXT:     LSHR T28.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T0.Y, T19.Z, literal.y,
+; EG-NEXT:     LSHR T0.Z, T19.W, literal.z,
+; EG-NEXT:     LSHR * T0.W, T19.Z, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT:    96(1.345247e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T29.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T1.Y, T19.W, literal.y,
+; EG-NEXT:     LSHR T1.Z, T19.X, literal.z,
+; EG-NEXT:     LSHR * T1.W, T19.W, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.x,
+; EG-NEXT:    144(2.017870e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T30.X, PV.W, literal.x,
+; EG-NEXT:     LSHR T2.Y, T19.X, literal.y,
+; EG-NEXT:     LSHR T2.Z, T19.X, literal.z,
+; EG-NEXT:     LSHR T2.W, T19.Y, literal.y,
+; EG-NEXT:     LSHR * T3.W, T19.Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:    ALU clause starting at 75:
+; EG-NEXT:     LSHR T3.Y, T20.Z, literal.x,
+; EG-NEXT:     LSHR T3.Z, T20.Z, literal.y,
+; EG-NEXT:     LSHR T4.W, T20.W, literal.x,
+; EG-NEXT:     ADD_INT * T5.W, KC0[2].Y, literal.z,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:    128(1.793662e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T32.X, PS, literal.x,
+; EG-NEXT:     LSHR T4.Y, T20.W, literal.y,
+; EG-NEXT:     LSHR T4.Z, T20.X, literal.z,
+; EG-NEXT:     LSHR T5.W, T20.X, literal.y,
+; EG-NEXT:     ADD_INT * T6.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 176(2.466285e-43)
+; EG-NEXT:     LSHR T33.X, PS, literal.x,
+; EG-NEXT:     LSHR T5.Y, T20.Y, literal.y,
+; EG-NEXT:     LSHR T5.Z, T20.Y, literal.z,
+; EG-NEXT:     LSHR T6.W, T21.Z, literal.y,
+; EG-NEXT:     ADD_INT * T7.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 160(2.242078e-43)
+; EG-NEXT:     LSHR T34.X, PS, literal.x,
+; EG-NEXT:     LSHR T6.Y, T21.Z, literal.y,
+; EG-NEXT:     LSHR T6.Z, T21.W, literal.z,
+; EG-NEXT:     LSHR T7.W, T21.W, literal.y,
+; EG-NEXT:     ADD_INT * T8.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 208(2.914701e-43)
+; EG-NEXT:     LSHR T35.X, PS, literal.x,
+; EG-NEXT:     LSHR T7.Y, T21.X, literal.y,
+; EG-NEXT:     LSHR T7.Z, T21.X, literal.z,
+; EG-NEXT:     LSHR T8.W, T21.Y, literal.y,
+; EG-NEXT:     LSHR * T9.W, T21.Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T36.X, T31.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR T8.Y, T31.Z, literal.y,
+; EG-NEXT:     LSHR T8.Z, T31.Z, literal.z,
+; EG-NEXT:     LSHR T10.W, T31.W, literal.y,
+; EG-NEXT:     LSHR * T11.W, T31.Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T37.X, T31.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T9.Y, T31.W, literal.y,
+; EG-NEXT:     LSHR T9.Z, T31.X, literal.y,
+; EG-NEXT:     BFE_INT T36.W, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR * T11.W, T31.Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T38.X, T31.W, 0.0, literal.x,
+; EG-NEXT:     LSHR T10.Y, T31.X, literal.y,
+; EG-NEXT:     BFE_INT T36.Z, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T37.W, PV.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T11.W, T31.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T39.X, T31.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T36.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T37.Z, PV.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T38.W, T9.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR * T11.W, T31.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T31.X, T21.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T37.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T38.Z, T10.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T39.W, T8.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T10.W, T31.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T40.X, T21.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T38.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T39.Z, T8.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T31.W, T9.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T9.W, T31.Z, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T41.X, T21.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T39.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T31.Z, T8.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     BFE_INT * T40.W, T7.Z, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:    ALU clause starting at 152:
+; EG-NEXT:     LSHR * T8.W, T21.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T42.X, T21.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T31.Y, PV.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T40.Z, T7.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T41.W, T7.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T7.W, T21.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T21.X, T20.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T40.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T41.Z, T6.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T42.W, T6.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR * T7.W, T21.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T43.X, T20.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T41.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T42.Z, T6.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T21.W, T5.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T6.W, T21.Z, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T44.X, T20.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T42.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T21.Z, T5.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T43.W, T5.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR * T5.W, T20.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T45.X, T20.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T21.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T43.Z, T4.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     BFE_INT T44.W, T4.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR * T5.W, T20.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T20.X, T19.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T43.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T44.Z, T4.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T45.W, T3.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T4.W, T20.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T46.X, T19.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T44.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T45.Z, T3.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T20.W, T3.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T3.W, T20.Z, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T47.X, T19.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T45.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T20.Z, T2.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     BFE_INT T46.W, T2.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR * T2.W, T19.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T48.X, T19.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T20.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T46.Z, T2.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T47.W, T1.W, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 192(2.690493e-43)
+; EG-NEXT:     LSHR T19.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T46.Y, T1.Z, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T47.Z, T1.Y, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T48.W, T0.W, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    240(3.363116e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T49.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T47.Y, T0.Z, 0.0, literal.y,
+; EG-NEXT:     BFE_INT T48.Z, T0.Y, 0.0, literal.y,
+; EG-NEXT:     LSHR T0.W, T19.Z, literal.y, BS:VEC_120/SCL_212
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    224(3.138909e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T50.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T48.Y, PV.W, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v64i8_to_v64i32:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @32, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @24
+; CM-NEXT:    ALU 39, @33, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @28
+; CM-NEXT:    ALU 84, @73, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    ALU 73, @158, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T24, T50.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T19, T21.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T49, T23.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T48, T38.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T47, T37.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T20, T36.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T46, T35.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T45, T34.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T44, T33.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T22, T32.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T43, T31.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T42, T30.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T41, T29.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T28, T27.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T40, T26.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T39, T25.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 24:
+; CM-NEXT:     VTX_READ_128 T19.XYZW, T22.X, 0, #1
+; CM-NEXT:     VTX_READ_128 T20.XYZW, T22.X, 16, #1
+; CM-NEXT:    Fetch clause starting at 28:
+; CM-NEXT:     VTX_READ_128 T28.XYZW, T22.X, 48, #1
+; CM-NEXT:     VTX_READ_128 T22.XYZW, T22.X, 32, #1
+; CM-NEXT:    ALU clause starting at 32:
+; CM-NEXT:     MOV * T22.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 33:
+; CM-NEXT:     LSHR T0.Y, T19.Y, literal.x,
+; CM-NEXT:     LSHR T0.Z, T19.Y, literal.y,
+; CM-NEXT:     LSHR * T0.W, T19.X, literal.x,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     LSHR T21.X, T19.Y, literal.x,
+; CM-NEXT:     LSHR T1.Y, T19.X, literal.y,
+; CM-NEXT:     LSHR T1.Z, T19.W, literal.z,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; CM-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 48(6.726233e-44)
+; CM-NEXT:     LSHR T23.X, T19.X, literal.x,
+; CM-NEXT:     LSHR T2.Y, T19.W, literal.y,
+; CM-NEXT:     LSHR T2.Z, T19.Z, literal.z,
+; CM-NEXT:     LSHR * T2.W, T19.W, literal.x,
+; CM-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T24.X, T19.Z, literal.x,
+; CM-NEXT:     LSHR T3.Y, T20.Y, literal.y,
+; CM-NEXT:     LSHR T3.Z, T19.Z, literal.z,
+; CM-NEXT:     ADD_INT * T3.W, KC0[2].Y, literal.w,
+; CM-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; CM-NEXT:    24(3.363116e-44), 224(3.138909e-43)
+; CM-NEXT:     LSHR T25.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T4.Y, T20.Y, literal.y,
+; CM-NEXT:     LSHR T4.Z, T20.X, literal.z,
+; CM-NEXT:     ADD_INT * T3.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 240(3.363116e-43)
+; CM-NEXT:     LSHR T26.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T5.Y, T20.Y, literal.y,
+; CM-NEXT:     LSHR T5.Z, T20.X, literal.z,
+; CM-NEXT:     ADD_INT * T3.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 192(2.690493e-43)
+; CM-NEXT:     LSHR T27.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T6.Y, T20.W, literal.y,
+; CM-NEXT:     LSHR T6.Z, T20.X, literal.z,
+; CM-NEXT:     LSHR * T3.W, T20.W, literal.w,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; CM-NEXT:    ALU clause starting at 73:
+; CM-NEXT:     LSHR T7.Z, T20.Z, literal.x,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.y,
+; CM-NEXT:    8(1.121039e-44), 208(2.914701e-43)
+; CM-NEXT:     LSHR T29.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T7.Y, T20.W, literal.y,
+; CM-NEXT:     LSHR T8.Z, T20.Z, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 160(2.242078e-43)
+; CM-NEXT:     LSHR T30.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T8.Y, T22.Y, literal.y,
+; CM-NEXT:     LSHR T9.Z, T20.Z, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    24(3.363116e-44), 176(2.466285e-43)
+; CM-NEXT:     LSHR T31.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T9.Y, T22.Y, literal.y,
+; CM-NEXT:     LSHR T10.Z, T22.X, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 128(1.793662e-43)
+; CM-NEXT:     LSHR T32.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T10.Y, T22.Y, literal.y,
+; CM-NEXT:     LSHR T11.Z, T22.X, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 144(2.017870e-43)
+; CM-NEXT:     LSHR T33.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T11.Y, T22.W, literal.y,
+; CM-NEXT:     LSHR T12.Z, T22.X, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    24(3.363116e-44), 96(1.345247e-43)
+; CM-NEXT:     LSHR T34.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T12.Y, T22.W, literal.y,
+; CM-NEXT:     LSHR T13.Z, T22.Z, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 112(1.569454e-43)
+; CM-NEXT:     LSHR T35.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T13.Y, T22.W, literal.y,
+; CM-NEXT:     LSHR T14.Z, T22.Z, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 64(8.968310e-44)
+; CM-NEXT:     LSHR T36.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T14.Y, T28.Y, literal.y,
+; CM-NEXT:     LSHR T15.Z, T22.Z, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    24(3.363116e-44), 80(1.121039e-43)
+; CM-NEXT:     LSHR T37.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T15.Y, T28.Y, literal.y,
+; CM-NEXT:     LSHR T16.Z, T28.X, literal.z,
+; CM-NEXT:     ADD_INT * T4.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; CM-NEXT:     LSHR T38.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T16.Y, T28.Y, literal.y,
+; CM-NEXT:     LSHR T17.Z, T28.X, literal.z,
+; CM-NEXT:     LSHR * T4.W, T28.W, literal.w,
+; CM-NEXT:    2(2.802597e-45), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 8(1.121039e-44)
+; CM-NEXT:     BFE_INT T39.X, T28.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR T17.Y, T28.X, literal.y,
+; CM-NEXT:     LSHR T18.Z, T28.W, literal.z,
+; CM-NEXT:     LSHR * T5.W, T28.Z, literal.y,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T40.X, T28.W, 0.0, literal.x,
+; CM-NEXT:     LSHR T18.Y, T28.W, literal.y,
+; CM-NEXT:     LSHR T21.Z, T28.Z, literal.z,
+; CM-NEXT:     BFE_INT * T39.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T28.X, T28.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T21.Y, T28.Z, literal.x,
+; CM-NEXT:     BFE_INT T39.Z, PV.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T40.W, PV.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T41.X, T28.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T39.Y, PV.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T40.Z, T18.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T28.W, T17.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:    ALU clause starting at 158:
+; CM-NEXT:     BFE_INT T42.X, T22.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T40.Y, T4.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T28.Z, T17.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT * T41.W, T16.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T43.X, T22.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T28.Y, T16.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T41.Z, T15.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T42.W, T15.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T22.X, T22.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T41.Y, T14.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T42.Z, T14.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T43.W, T13.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T44.X, T22.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T42.Y, T13.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T43.Z, T12.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT * T22.W, T12.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T45.X, T20.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T43.Y, T11.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T22.Z, T11.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT * T44.W, T10.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T46.X, T20.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T22.Y, T10.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T44.Z, T9.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T45.W, T9.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T20.X, T20.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T44.Y, T8.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T45.Z, T8.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T46.W, T7.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T47.X, T20.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T45.Y, T7.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T46.Z, T3.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T20.W, T6.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T48.X, T19.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T46.Y, T6.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T20.Z, T5.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT * T47.W, T5.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T49.X, T19.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T20.Y, T4.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T47.Z, T4.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T48.W, T3.Z, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T19.X, T19.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T47.Y, T3.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T48.Z, T24.X, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT * T49.W, T2.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T24.X, T19.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T48.Y, T2.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T49.Z, T2.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:     BFE_INT * T19.W, T23.X, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T23.X, T1.W, literal.x,
+; CM-NEXT:     BFE_INT T49.Y, T1.Z, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T19.Z, T1.Y, 0.0, literal.y,
+; CM-NEXT:     BFE_INT * T24.W, T21.X, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T21.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT T19.Y, T0.W, 0.0, literal.y,
+; CM-NEXT:     BFE_INT T24.Z, T0.Z, 0.0, literal.y,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T50.X, PV.W, literal.x,
+; CM-NEXT:     BFE_INT * T24.Y, T0.Y, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <64 x i8>, ptr addrspace(1) %in
   %ext = sext <64 x i8> %load to <64 x i32>
   store <64 x i32> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_i8_to_i64:
-; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
-
-; GCN-NOHSA-DAG: buffer_load_ubyte v[[LO:[0-9]+]],
-; GCN-NOHSA: buffer_store_dwordx2 v[[[LO]]:[[HI]]]
-
-; GCN-HSA-DAG: flat_load_ubyte v[[LO:[0-9]+]],
-; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v[[[LO]]:[[HI]]]
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG: MOV {{.*}}, 0.0
 define amdgpu_kernel void @global_zextload_i8_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_i8_to_i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_i8_to_i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_i8_to_i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_i8_to_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.Y, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_i8_to_i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.Y, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i8, ptr addrspace(1) %in
   %ext = zext i8 %a to i64
   store i64 %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_i8_to_i64:
-; GCN-NOHSA: buffer_load_sbyte v[[LO:[0-9]+]],
-; GCN-HSA: flat_load_sbyte v[[LO:[0-9]+]],
-; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
-
-; GCN-NOHSA: buffer_store_dwordx2 v[[[LO]]:[[HI]]]
-; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v[[[LO]]:[[HI]]]
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal
-; TODO: Why not 7 ?
-; EG: 31
+; TODO: EG Why 31 not 7 ?
 define amdgpu_kernel void @global_sextload_i8_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_i8_to_i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_i8_to_i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_sbyte v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_i8_to_i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_i8_to_i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.X, T0.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+; EG-NEXT:     ASHR * T0.Y, PV.X, literal.x,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_i8_to_i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT * T0.X, T0.X, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR * T0.Y, PV.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %a = load i8, ptr addrspace(1) %in
   %ext = sext i8 %a to i64
   store i64 %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v1i8_to_v1i64:
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG: MOV {{.*}}, 0.0
 define amdgpu_kernel void @global_zextload_v1i8_to_v1i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v1i8_to_v1i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v1i8_to_v1i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v1i8_to_v1i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v1i8_to_v1i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.Y, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v1i8_to_v1i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 2, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.Y, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <1 x i8>, ptr addrspace(1) %in
   %ext = zext <1 x i8> %load to <1 x i64>
   store <1 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v1i8_to_v1i64:
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal
-; TODO: Why not 7 ?
-; EG: 31
+; TODO: EG Why 31 not 7 ?
 define amdgpu_kernel void @global_sextload_v1i8_to_v1i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v1i8_to_v1i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v1i8_to_v1i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_sbyte v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v1i8_to_v1i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v1i8_to_v1i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.X, T0.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+; EG-NEXT:     ASHR * T0.Y, PV.X, literal.x,
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_v1i8_to_v1i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 4, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T0, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT * T0.X, T0.X, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR * T0.Y, PV.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %load = load <1 x i8>, ptr addrspace(1) %in
   %ext = sext <1 x i8> %load to <1 x i64>
   store <1 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v2i8_to_v2i64:
-
-; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v2i8_to_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v2i8_to_v2i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v2i8_to_v2i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-HSA-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v2i8_to_v2i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v2i8_to_v2i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 14, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.Y, T2.X,
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     AND_INT T0.W, T4.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), -65536(nan)
+; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
+; EG-NEXT:     MOV * T2.X, PV.W,
+; EG-NEXT:     MOV T0.Y, PV.X,
+; EG-NEXT:     MOV * T1.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T4.Z, PV.Y, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T4.X, T0.W, literal.x,
+; EG-NEXT:     MOV T4.Y, 0.0,
+; EG-NEXT:     MOV T4.W, 0.0,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
+;
+; CM-LABEL: global_zextload_v2i8_to_v2i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 15, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.Y, T2.X,
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     AND_INT T0.Z, T4.X, literal.x,
+; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
+; CM-NEXT:    65535(9.183409e-41), -65536(nan)
+; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
+; CM-NEXT:     MOV * T2.X, PV.W,
+; CM-NEXT:     MOV T0.Y, PV.X,
+; CM-NEXT:     MOV * T1.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T4.Z, PV.Y, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T4.X, T0.W, literal.x,
+; CM-NEXT:     MOV T4.Y, 0.0,
+; CM-NEXT:     MOV * T4.W, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <2 x i8>, ptr addrspace(1) %in
   %ext = zext <2 x i8> %load to <2 x i64>
   store <2 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v2i8_to_v2i64:
-
-; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_sextload_v2i8_to_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v2i8_to_v2i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v2i8_to_v2i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v0, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-HSA-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v2i8_to_v2i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v2i8_to_v2i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 15, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.Y, T2.X,
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     AND_INT T0.W, T4.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), -65536(nan)
+; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
+; EG-NEXT:     MOV * T2.X, PV.W,
+; EG-NEXT:     MOV * T0.Y, PV.X,
+; EG-NEXT:     BFE_INT * T4.X, T0.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     ASHR T4.Y, PV.X, literal.x,
+; EG-NEXT:     LSHR * T0.W, T0.Y, literal.y,
+; EG-NEXT:    31(4.344025e-44), 8(1.121039e-44)
+; EG-NEXT:     BFE_INT * T4.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ASHR * T4.W, PV.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+;
+; CM-LABEL: global_sextload_v2i8_to_v2i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 15, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.Y, T2.X,
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     AND_INT T0.Z, T4.X, literal.x,
+; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
+; CM-NEXT:    65535(9.183409e-41), -65536(nan)
+; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
+; CM-NEXT:     MOV * T2.X, PV.W,
+; CM-NEXT:     MOV * T0.Y, PV.X,
+; CM-NEXT:     BFE_INT * T4.X, T0.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     ASHR T4.Y, PV.X, literal.x,
+; CM-NEXT:     LSHR * T0.W, T0.Y, literal.y,
+; CM-NEXT:    31(4.344025e-44), 8(1.121039e-44)
+; CM-NEXT:     BFE_INT * T4.Z, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR * T4.W, PV.Z, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %load = load <2 x i8>, ptr addrspace(1) %in
   %ext = sext <2 x i8> %load to <2 x i64>
   store <2 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v4i8_to_v4i64:
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v4i8_to_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v4i8_to_v4i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, v1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v6, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v0, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v4i8_to_v4i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v0, v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_bfe_u32 v6, v0, 8, 8
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v2, 24, v0
+; GCN-HSA-NEXT:    v_and_b32_e32 v4, 0xff, v0
+; GCN-HSA-NEXT:    v_bfe_u32 v0, v0, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[0:3]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[4:7]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v4i8_to_v4i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, v1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v6, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xff, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v0, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v4i8_to_v4i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 17, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XYZW, T7.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XYZW, T6.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T5.X, T4.X, literal.x, PV.W,
+; EG-NEXT:     LSHR * T5.Z, T4.X, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T5.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T4.Z, T4.X, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T4.X, T4.X, literal.x,
+; EG-NEXT:     MOV T4.Y, 0.0,
+; EG-NEXT:     MOV T5.W, 0.0,
+; EG-NEXT:     MOV * T4.W, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T6.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR * T7.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v4i8_to_v4i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 18, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T7.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6, T4.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T5.Z, T4.X, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T5.X, T4.X, literal.x,
+; CM-NEXT:     MOV * T5.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T6.X, T4.X, literal.x, T0.W,
+; CM-NEXT:     LSHR * T6.Z, T4.X, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T6.Y, 0.0,
+; CM-NEXT:     MOV * T5.W, 0.0,
+; CM-NEXT:     MOV * T6.W, 0.0,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T4.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <4 x i8>, ptr addrspace(1) %in
   %ext = zext <4 x i8> %load to <4 x i64>
   store <4 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v4i8_to_v4i64:
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_sextload_v4i8_to_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v4i8_to_v4i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v6, v4, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v4, v3, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v4i8_to_v4i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v0, v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v4, 0, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v3, 0, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v4i8_to_v4i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v4, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v3, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v4i8_to_v4i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 18, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XYZW, T7.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XYZW, T6.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; EG-NEXT:     ASHR T4.W, T4.X, literal.y,
+; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     ASHR T5.Y, PV.X, literal.x,
+; EG-NEXT:     ASHR T4.Z, T4.X, literal.y,
+; EG-NEXT:     LSHR T0.W, T4.X, literal.z,
+; EG-NEXT:     LSHR * T1.W, T4.X, literal.w,
+; EG-NEXT:    31(4.344025e-44), 24(3.363116e-44)
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T4.X, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T5.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     LSHR T7.X, PV.W, literal.x,
+; EG-NEXT:     ASHR T4.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR * T5.W, PV.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+;
+; CM-LABEL: global_sextload_v4i8_to_v4i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 18, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T7.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T6.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T0.Y, T4.X, literal.x,
+; CM-NEXT:     ADD_INT T0.Z, KC0[2].Y, literal.y,
+; CM-NEXT:     ASHR * T4.W, T4.X, literal.z,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T6.X, PV.Z, literal.x,
+; CM-NEXT:     ASHR T5.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T4.Z, T4.X, literal.z,
+; CM-NEXT:     LSHR * T0.W, T4.X, literal.w,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; CM-NEXT:     BFE_INT T4.X, PV.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T5.Z, T0.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR T4.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR * T5.W, PV.Z, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %load = load <4 x i8>, ptr addrspace(1) %in
   %ext = sext <4 x i8> %load to <4 x i64>
   store <4 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v8i8_to_v8i64:
-
-; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v8i8_to_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v8i8_to_v8i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[16:17], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v13, v1
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v15, v1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v6, 24, v16
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 24, v17
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v10, v17, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v14, v16, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xff, v16
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xff, v17
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v4, v16, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v0, v17, 16, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v8i8_to_v8i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[15:16], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v1
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v2, 24, v16
+; GCN-HSA-NEXT:    v_bfe_u32 v0, v16, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[17:18], v[0:3]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 32
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v5, 24, v15
+; GCN-HSA-NEXT:    v_bfe_u32 v3, v15, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s0
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v16, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v15, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v11, 0xff, v15
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[17:18], v[3:6]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[19:20], v[7:10]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v8i8_to_v8i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[16:17], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, v1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v9, v1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, v1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v13, v1
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, v1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v17
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v0, v17, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v6, v17, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v10, 24, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v14, v16, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xff, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v8, v16, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xff, v17
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v8i8_to_v8i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 34, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T11.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T10.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XYZW, T9.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T6.X, T5.Y, literal.x, PV.W,
+; EG-NEXT:     LSHR * T6.Z, T5.Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T6.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T7.Z, T5.Y, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T7.X, T5.Y, literal.x,
+; EG-NEXT:     MOV * T7.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T8.X, T5.X, literal.x, T0.W,
+; EG-NEXT:     LSHR * T8.Z, T5.X, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T8.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T5.Z, T5.X, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T5.X, T5.X, literal.x,
+; EG-NEXT:     MOV T5.Y, 0.0,
+; EG-NEXT:     MOV T6.W, 0.0,
+; EG-NEXT:     MOV * T7.W, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T8.W, 0.0,
+; EG-NEXT:     MOV * T5.W, 0.0,
+; EG-NEXT:     LSHR T9.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T10.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T11.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; EG-NEXT:     LSHR * T12.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v8i8_to_v8i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @8
+; CM-NEXT:    ALU 35, @11, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6, T12.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T11.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T8, T10.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T9.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 11:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T6.Z, T5.X, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T6.X, T5.X, literal.x,
+; CM-NEXT:     MOV * T6.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T7.X, T5.X, literal.x, T0.W,
+; CM-NEXT:     LSHR * T7.Z, T5.X, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T7.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T8.Z, T5.Y, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T8.X, T5.Y, literal.x,
+; CM-NEXT:     MOV * T8.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T5.X, T5.Y, literal.x, T0.W,
+; CM-NEXT:     LSHR * T5.Z, T5.Y, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T5.Y, 0.0,
+; CM-NEXT:     MOV * T6.W, 0.0,
+; CM-NEXT:     MOV * T7.W, 0.0,
+; CM-NEXT:     MOV * T8.W, 0.0,
+; CM-NEXT:     MOV * T5.W, 0.0,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T9.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; CM-NEXT:     LSHR T10.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:     LSHR * T11.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T12.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <8 x i8>, ptr addrspace(1) %in
   %ext = zext <8 x i8> %load to <8 x i64>
   store <8 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v8i8_to_v8i64:
-
-; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_sextload_v8i8_to_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v8i8_to_v8i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s4, v1
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s5, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s6, s4, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s8, s4, 8
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s10, s5, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s12, s5, 24
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s14, s5, 8
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[16:17], s[4:5], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s15, s4, 31
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s18, s4, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[4:5], s[14:15], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s18
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s15
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, s16
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, s17
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v8, s8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, s9
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s10
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s12
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s13
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v8i8_to_v8i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s2, v1
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s3, v0
+; GCN-HSA-NEXT:    s_lshr_b32 s4, s2, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s6, s2, 8
+; GCN-HSA-NEXT:    s_lshr_b32 s8, s3, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s10, s3, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s12, s3, 8
+; GCN-HSA-NEXT:    s_ashr_i32 s13, s2, 31
+; GCN-HSA-NEXT:    s_bfe_i64 s[14:15], s[2:3], 0x80000
+; GCN-HSA-NEXT:    s_ashr_i32 s16, s2, 24
+; GCN-HSA-NEXT:    s_bfe_i64 s[2:3], s[12:13], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x80000
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s5
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s5
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, s16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, s13
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s4
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[4:7]
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s14
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s15
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s6
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[8:11]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s10
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s11
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[12:15]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v8i8_to_v8i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v1
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v0
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s6, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s8, s4, 8
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s10, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s12, s5, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s14, s5, 8
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s15, s4, 31
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s18, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[16:17], s[4:5], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[4:5], s[14:15], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, s18
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, s15
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v4, s6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, s7
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v8, s16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v9, s17
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v10, s8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, s9
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v12, s10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v13, s11
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v14, s12
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, s13
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v8i8_to_v8i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @8
+; EG-NEXT:    ALU 39, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T9.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XYZW, T8.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T6.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T7.X, T5.Y, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; EG-NEXT:     LSHR T8.X, PV.W, literal.x,
+; EG-NEXT:     ASHR T7.Y, PV.X, literal.y,
+; EG-NEXT:     LSHR T0.W, T5.Y, literal.z,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T9.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T7.Z, PV.W, 0.0, literal.y,
+; EG-NEXT:     ASHR * T10.W, T5.X, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T11.X, T5.X, 0.0, literal.x,
+; EG-NEXT:     ASHR T10.Z, T5.X, literal.y,
+; EG-NEXT:     LSHR T0.W, T5.X, literal.z,
+; EG-NEXT:     ASHR * T5.W, T5.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T10.X, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T11.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T5.Z, T5.Y, literal.z,
+; EG-NEXT:     LSHR T0.W, T5.X, literal.x,
+; EG-NEXT:     LSHR * T1.W, T5.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T5.X, PS, 0.0, literal.x,
+; EG-NEXT:     ASHR T10.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T11.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T12.X, PV.W, literal.x,
+; EG-NEXT:     ASHR T5.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T11.W, PV.Z, literal.y,
+; EG-NEXT:     ASHR * T7.W, T7.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+;
+; CM-LABEL: global_sextload_v8i8_to_v8i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @8
+; CM-NEXT:    ALU 39, @11, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T12.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T9.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T8.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T10, T6.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 11:
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T6.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T7.X, T5.Y, 0.0, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; CM-NEXT:     LSHR T8.X, PV.W, literal.x,
+; CM-NEXT:     ASHR T7.Y, PV.X, literal.y,
+; CM-NEXT:     LSHR T0.Z, T5.Y, literal.z,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     LSHR T9.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T0.Y, T5.X, literal.y,
+; CM-NEXT:     BFE_INT T7.Z, PV.Z, 0.0, literal.y,
+; CM-NEXT:     ASHR * T10.W, T5.Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T11.X, T5.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T1.Y, T5.Y, literal.y,
+; CM-NEXT:     ASHR T10.Z, T5.Y, literal.z,
+; CM-NEXT:     ASHR * T5.W, T5.X, literal.w,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T10.X, PV.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T11.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T5.Z, T5.X, literal.z,
+; CM-NEXT:     LSHR * T0.W, T5.X, literal.w,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; CM-NEXT:     BFE_INT T5.X, PV.W, 0.0, literal.x,
+; CM-NEXT:     ASHR T10.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T11.Z, T0.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR * T7.W, T7.Z, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     LSHR T12.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR T5.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR * T11.W, PV.Z, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %load = load <8 x i8>, ptr addrspace(1) %in
   %ext = sext <8 x i8> %load to <8 x i64>
   store <8 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v16i8_to_v16i64:
-
-; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v16i8_to_v16i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v16i8_to_v16i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v9, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v13, v1, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v11, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v7, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v17, v3, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v15, 0xff, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v21, v2, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v19, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v25, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v23, v1, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v29, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v27, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v33, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v31, v3, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v3, v2, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v32, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v34, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v28, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v30, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v24, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v26, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v20, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v22, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v18, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v8, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v12, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v14, v4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[31:34], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[27:30], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[23:26], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v16i8_to_v16i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x50
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x70
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, v5
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, v5
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v10, 24, v3
+; GCN-HSA-NEXT:    v_bfe_u32 v8, v3, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[8:11]
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v13, 24, v0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 48
+; GCN-HSA-NEXT:    v_bfe_u32 v11, v0, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[11:14]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 64
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v16, 24, v1
+; GCN-HSA-NEXT:    v_bfe_u32 v14, v1, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[14:17]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s4
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[8:11]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v4, v2, 16, 8
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x60
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, v5
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[15:16], v[4:7]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, v5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s3
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v3, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v0, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v17, v1, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v15, 0xff, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v0
+; GCN-HSA-NEXT:    v_and_b32_e32 v11, 0xff, v3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[19:20], v[11:14]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[7:10]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[15:18]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v16i8_to_v16i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v29, 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v31, v29
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v25, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v27, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v21, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v23, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v17, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v19, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v9, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v13, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, v29
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v30, 24, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v28, v3, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v6, v1, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v10, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v14, v0, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xff, v0
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v8, v0, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v4, 0xff, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v18, v3, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v16, 0xff, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v22, v2, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v20, 0xff, v2
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v26, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v24, v2, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v0, v1, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, v29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, v29
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v16i8_to_v16i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @12
+; EG-NEXT:    ALU 68, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T22.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T21.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T10.XYZW, T20.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T19.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T18.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T13.XYZW, T17.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T14.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T15.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 12:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 14:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T8.X, T7.W, literal.x, PV.W,
+; EG-NEXT:     LSHR * T8.Z, T7.W, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T8.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T9.Z, T7.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T9.X, T7.W, literal.x,
+; EG-NEXT:     MOV * T9.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T10.X, T7.Z, literal.x, T0.W,
+; EG-NEXT:     LSHR * T10.Z, T7.Z, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T10.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T11.Z, T7.Z, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T11.X, T7.Z, literal.x,
+; EG-NEXT:     MOV * T11.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T12.X, T7.Y, literal.x, T0.W,
+; EG-NEXT:     LSHR * T12.Z, T7.Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T12.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T13.Z, T7.Y, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T13.X, T7.Y, literal.x,
+; EG-NEXT:     MOV * T13.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T14.X, T7.X, literal.x, T0.W,
+; EG-NEXT:     LSHR * T14.Z, T7.X, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T14.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T7.Z, T7.X, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T7.X, T7.X, literal.x,
+; EG-NEXT:     MOV T7.Y, 0.0,
+; EG-NEXT:     MOV T8.W, 0.0,
+; EG-NEXT:     MOV * T9.W, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T10.W, 0.0,
+; EG-NEXT:     MOV * T11.W, 0.0,
+; EG-NEXT:     MOV T12.W, 0.0,
+; EG-NEXT:     MOV * T13.W, 0.0,
+; EG-NEXT:     MOV T14.W, 0.0,
+; EG-NEXT:     MOV * T7.W, 0.0,
+; EG-NEXT:     LSHR T15.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T16.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T17.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; EG-NEXT:     LSHR T18.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; EG-NEXT:     LSHR T19.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+; EG-NEXT:     LSHR T20.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 96(1.345247e-43)
+; EG-NEXT:     LSHR T21.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 112(1.569454e-43)
+; EG-NEXT:     LSHR * T22.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v16i8_to_v16i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @12
+; CM-NEXT:    ALU 69, @15, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T8, T22.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T9, T21.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T10, T20.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T19.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T18.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T17.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T13, T16.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T14, T15.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 12:
+; CM-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; CM-NEXT:    ALU clause starting at 14:
+; CM-NEXT:     MOV * T7.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 15:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T8.Z, T7.X, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T8.X, T7.X, literal.x,
+; CM-NEXT:     MOV * T8.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T9.X, T7.X, literal.x, T0.W,
+; CM-NEXT:     LSHR * T9.Z, T7.X, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T9.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T10.Z, T7.Y, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T10.X, T7.Y, literal.x,
+; CM-NEXT:     MOV * T10.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T11.X, T7.Y, literal.x, T0.W,
+; CM-NEXT:     LSHR * T11.Z, T7.Y, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T11.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T12.Z, T7.Z, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T12.X, T7.Z, literal.x,
+; CM-NEXT:     MOV * T12.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T7.X, T7.Z, literal.x, T0.W,
+; CM-NEXT:     LSHR * T7.Z, T7.Z, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T7.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T13.Z, T7.W, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T13.X, T7.W, literal.x,
+; CM-NEXT:     MOV * T13.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T14.X, T7.W, literal.x, T0.W,
+; CM-NEXT:     LSHR * T14.Z, T7.W, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T14.Y, 0.0,
+; CM-NEXT:     MOV * T8.W, 0.0,
+; CM-NEXT:     MOV * T9.W, 0.0,
+; CM-NEXT:     MOV * T10.W, 0.0,
+; CM-NEXT:     MOV * T11.W, 0.0,
+; CM-NEXT:     MOV * T12.W, 0.0,
+; CM-NEXT:     MOV * T7.W, 0.0,
+; CM-NEXT:     MOV * T13.W, 0.0,
+; CM-NEXT:     MOV * T14.W, 0.0,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T15.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 96(1.345247e-43)
+; CM-NEXT:     LSHR T16.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+; CM-NEXT:     LSHR T17.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; CM-NEXT:     LSHR T18.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; CM-NEXT:     LSHR T19.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; CM-NEXT:     LSHR T20.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:     LSHR * T21.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T22.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <16 x i8>, ptr addrspace(1) %in
   %ext = zext <16 x i8> %load to <16 x i64>
   store <16 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v16i8_to_v16i64:
-
-; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_sextload_v16i8_to_v16i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v16i8_to_v16i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s5, v3
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s4, v2
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s9, v1
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s8, v0
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s10, s5, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s12, s5, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s14, s5
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s16, s4, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s18, s4, 24
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s20, s4, 8
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s22, s9, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s24, s9, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s26, s9
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s28, s8, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s30, s8, 24
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s34, s8, 8
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[6:7], s[8:9], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[36:37], s[4:5], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s31, s9, 31
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s33, s9, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s35, s5, 31
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s38, s5, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[4:5], s[34:35], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[8:9], s[30:31], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s38
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s35
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s14
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s15
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v8, s36
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, s37
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v12, s33
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v13, s31
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v14, s26
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v15, s27
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, s12
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, s13
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s16
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s17
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s18
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s19
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, s20
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, s21
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, s22
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, s23
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, s24
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v17, s25
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s28
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s29
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s9
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v16i8_to_v16i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s3, v3
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s2, v2
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s5, v1
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s4, v0
+; GCN-HSA-NEXT:    s_lshr_b32 s6, s3, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s8, s3, 8
+; GCN-HSA-NEXT:    s_mov_b32 s10, s3
+; GCN-HSA-NEXT:    s_lshr_b32 s12, s2, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s14, s2, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s16, s2, 8
+; GCN-HSA-NEXT:    s_ashr_i32 s7, s3, 31
+; GCN-HSA-NEXT:    s_ashr_i32 s9, s3, 24
+; GCN-HSA-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[24:25], s[4:5], 0x80000
+; GCN-HSA-NEXT:    s_lshr_b32 s18, s5, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s20, s5, 8
+; GCN-HSA-NEXT:    s_mov_b32 s22, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_lshr_b32 s2, s4, 16
+; GCN-HSA-NEXT:    s_ashr_i32 s3, s5, 31
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s7
+; GCN-HSA-NEXT:    s_ashr_i32 s7, s5, 24
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s24
+; GCN-HSA-NEXT:    s_lshr_b32 s24, s4, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s4, s4, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s25
+; GCN-HSA-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x80000
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 0x70
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s6
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 0x60
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[2:5]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s6
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 0x50
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s10
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s6
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 64
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[2:5]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s12
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s13
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s14
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s15
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s17
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s6
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[0:3]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s18
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s19
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[4:7]
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s22
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s23
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s20
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s21
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[12:15]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s24
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s25
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s5
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[8:11]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v16i8_to_v16i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v3
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v2
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s16, s5, 16
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s9, v1
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s8, v0
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s18, s5, 8
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s20, s5
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s22, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s24, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s26, s4, 8
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s35, s5, 31
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s38, s5, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s28, s9, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s12, s9, 8
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s14, s9
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s10, s8, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s30, s8, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s34, s8, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[36:37], s[4:5], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s31, s9, 31
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s38
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s35
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s17
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[6:7], s[8:9], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s33, s9, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[4:5], s[34:35], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[8:9], s[30:31], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v4, s36
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, s37
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v14, s20
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, s21
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v16, s18
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v17, s19
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, s26
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s22
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s23
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s24
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s25
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, s27
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v10, s33
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, s31
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v12, s6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v13, s7
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v8, s28
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v9, s29
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v18, s12
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v16, s14
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v17, s15
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v19, s13
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s9
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v16i8_to_v16i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @12
+; EG-NEXT:    ALU 78, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T21.XYZW, T22.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T13.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T15.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T7.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T11.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T14.XYZW, T10.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T9.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T18.XYZW, T8.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 12:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 14:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 15:
+; EG-NEXT:     LSHR T8.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T9.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T10.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; EG-NEXT:     LSHR T11.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; EG-NEXT:     LSHR * T12.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT * T13.X, T7.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T14.X, T7.Y, 0.0, literal.x,
+; EG-NEXT:     ASHR T13.Y, PV.X, literal.y,
+; EG-NEXT:     LSHR T0.W, T7.W, literal.x,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    80(1.121039e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T15.X, PS, literal.x,
+; EG-NEXT:     ASHR T14.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T13.Z, PV.W, 0.0, literal.z,
+; EG-NEXT:     LSHR T0.W, T7.Y, literal.z,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; EG-NEXT:    8(1.121039e-44), 96(1.345247e-43)
+; EG-NEXT:     LSHR T16.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T14.Z, PV.W, 0.0, literal.y,
+; EG-NEXT:     ASHR * T17.W, T7.X, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T18.X, T7.X, 0.0, literal.x,
+; EG-NEXT:     ASHR T17.Z, T7.X, literal.y,
+; EG-NEXT:     LSHR T0.W, T7.X, literal.z,
+; EG-NEXT:     ASHR * T19.W, T7.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T17.X, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T18.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T19.Z, T7.Y, literal.z,
+; EG-NEXT:     LSHR T0.W, T7.X, literal.x,
+; EG-NEXT:     LSHR * T1.W, T7.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T19.X, PS, 0.0, literal.x,
+; EG-NEXT:     ASHR T17.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T18.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     ADD_INT T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:     ASHR * T20.W, T7.Z, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T7.X, T7.Z, 0.0, literal.x,
+; EG-NEXT:     ASHR T19.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T20.Z, T7.Z, literal.z,
+; EG-NEXT:     LSHR T1.W, T7.Z, literal.w,
+; EG-NEXT:     ASHR * T21.W, T7.W, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T20.X, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T7.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T21.Z, T7.W, literal.z,
+; EG-NEXT:     LSHR T1.W, T7.Z, literal.x,
+; EG-NEXT:     LSHR * T2.W, T7.W, literal.w,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T21.X, PS, 0.0, literal.x,
+; EG-NEXT:     ASHR T20.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T7.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T18.W, T18.Z, literal.y,
+; EG-NEXT:     ASHR * T14.W, T14.Z, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:     LSHR T22.X, T0.W, literal.x,
+; EG-NEXT:     ASHR T21.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T7.W, PV.Z, literal.y,
+; EG-NEXT:     ASHR * T13.W, T13.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+;
+; CM-LABEL: global_sextload_v16i8_to_v16i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @12
+; CM-NEXT:    ALU 79, @15, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T21, T22.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T7, T16.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T13, T15.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T20, T12.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T18, T11.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T19, T10.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T14, T9.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T17, T8.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 12:
+; CM-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; CM-NEXT:    ALU clause starting at 14:
+; CM-NEXT:     MOV * T7.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 15:
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    112(1.569454e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T8.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 96(1.345247e-43)
+; CM-NEXT:     LSHR T9.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+; CM-NEXT:     LSHR T10.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; CM-NEXT:     LSHR T11.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; CM-NEXT:     LSHR T12.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T0.Z, T7.X, literal.y,
+; CM-NEXT:     LSHR * T0.W, T7.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T13.X, T7.Y, 0.0, literal.x,
+; CM-NEXT:     LSHR T0.Y, T7.Y, literal.y,
+; CM-NEXT:     LSHR T1.Z, T7.Z, literal.x,
+; CM-NEXT:     LSHR * T1.W, T7.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     BFE_INT T14.X, T7.W, 0.0, literal.x,
+; CM-NEXT:     ASHR T13.Y, PV.X, literal.y,
+; CM-NEXT:     LSHR T2.Z, T7.Y, literal.x,
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.z,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T15.X, PV.W, literal.x,
+; CM-NEXT:     ASHR T14.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T13.Z, PV.Z, 0.0, literal.z,
+; CM-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     LSHR T16.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T1.Y, T7.Z, literal.y,
+; CM-NEXT:     BFE_INT T14.Z, T1.W, 0.0, literal.z,
+; CM-NEXT:     ASHR * T17.W, T7.W, literal.w, BS:VEC_120/SCL_212
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T18.X, T7.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR T2.Y, T7.W, literal.y,
+; CM-NEXT:     ASHR T17.Z, T7.W, literal.z,
+; CM-NEXT:     ASHR * T19.W, T7.Z, literal.w,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T17.X, PV.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T18.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T19.Z, T7.Z, literal.z,
+; CM-NEXT:     ASHR * T14.W, T14.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T19.X, T1.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T17.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T18.Z, T1.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR * T20.W, T7.Y, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T21.X, T7.X, 0.0, literal.x,
+; CM-NEXT:     ASHR T19.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T20.Z, T7.Y, literal.z,
+; CM-NEXT:     ASHR * T7.W, T7.X, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T20.X, T0.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T21.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T7.Z, T7.X, literal.z,
+; CM-NEXT:     ASHR * T18.W, T18.Z, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T7.X, T0.W, 0.0, literal.x,
+; CM-NEXT:     ASHR T20.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T21.Z, T0.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR * T13.W, T13.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     LSHR T22.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR T7.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR * T21.W, PV.Z, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %load = load <16 x i8>, ptr addrspace(1) %in
   %ext = sext <16 x i8> %load to <16 x i64>
   store <16 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v32i8_to_v32i64:
-
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1
 define amdgpu_kernel void @global_zextload_v32i8_to_v32i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v32i8_to_v32i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s14, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s15, 0xe8f000
+; GCN-NOHSA-SI-NEXT:    s_add_u32 s12, s12, s11
+; GCN-NOHSA-SI-NEXT:    s_addc_u32 s13, s13, 0
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[10:13], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[17:20], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v2, v12, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v3, v11, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v6, v10, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff, v10
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v4, off, s[12:15], 0 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v5, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v6, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v7, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff, v11
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v1, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v2, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v4, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v14, 0xff, v12
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, v2
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v55, v13, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v53, 0xff, v13
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v27, 24, v10
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v25, v10, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v31, 24, v11
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v29, v11, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v35, 24, v12
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v33, v12, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v39, 24, v13
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v37, v13, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v10, 24, v20
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v43, v17, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v41, 0xff, v17
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v47, v18, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v45, 0xff, v18
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v51, v19, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v49, 0xff, v19
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v23, v20, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v21, 0xff, v20
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v59, 24, v17
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v57, v17, 16, 8
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(5)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v6, 24, v18
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v4, v18, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 24, v19
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v0, v19, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_u32 v8, v20, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v58, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v60, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v38, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v40, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v34, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v36, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v30, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v32, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v26, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v28, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v22, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v24, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v50, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v52, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v46, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v48, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v42, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v44, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v54, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v56, v9
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v53, off, s[12:15], 0 offset:32 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v54, off, s[12:15], 0 offset:36 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v55, off, s[12:15], 0 offset:40 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v56, off, s[12:15], 0 offset:44 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v15, v9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v17, v9
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v14, off, s[12:15], 0 offset:48 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v15, off, s[12:15], 0 offset:52 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v16, off, s[12:15], 0 offset:56 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v17, off, s[12:15], 0 offset:60 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v12, off, s[12:15], 0 offset:16 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v13, off, s[12:15], 0 offset:20 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(3)
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v14, off, s[12:15], 0 offset:24 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(2)
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v15, off, s[12:15], 0 offset:28 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1) expcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, v14
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v14, v12
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v15, v9
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v53, off, s[12:15], 0 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v54, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v55, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v56, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(2)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v54, v9
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v56, v9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:240
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[57:60], off, s[0:3], 0 offset:144
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[37:40], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[33:36], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[29:32], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:224
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[49:52], off, s[0:3], 0 offset:192
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[45:48], off, s[0:3], 0 offset:160
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[41:44], off, s[0:3], 0 offset:128
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0 offset:32 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:36 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:40 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:44 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0 offset:48 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:52 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:56 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:60 ; 4-byte Folded Reload
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[53:56], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v32i8_to_v32i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[2:5], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[6:9], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x70
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x50
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, v1
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, v1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v4
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v4, 16, 8
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 16
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v3
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v3, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0xf0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v2
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v2, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0xd0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(3)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v9
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v9, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0xb0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v8
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v8, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x90
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v7
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v7, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x60
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v6
+; GCN-HSA-NEXT:    v_bfe_u32 v10, v6, 16, 8
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 64
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v5, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v5
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 32
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v4, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v4
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v3, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v3
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[10:13]
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0xe0
+; GCN-HSA-NEXT:    v_bfe_u32 v12, v2, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[2:3], v[10:13]
+; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-HSA-NEXT:    v_bfe_u32 v11, v9, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[2:3], v[9:12]
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v2, 24, v5
+; GCN-HSA-NEXT:    v_bfe_u32 v0, v5, 16, 8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xc0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xa0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_bfe_u32 v16, v8, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v14, 0xff, v8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, v1
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 0x80
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[2:3], v[14:17]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, v1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, v1
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_bfe_u32 v9, v7, 8, 8
+; GCN-HSA-NEXT:    v_bfe_u32 v13, v6, 8, 8
+; GCN-HSA-NEXT:    v_and_b32_e32 v11, 0xff, v6
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[2:3], v[7:10]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v32i8_to_v32i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[10:13], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[14:17], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v53, 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v49, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v51, v53
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v46, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v43, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v40, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v37, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v30, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v32, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v34, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v27, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v23, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v25, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v55, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v19, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v21, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v8, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v4, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, v53
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v9, v11, 8, 8
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v50, 24, v15
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v48, v15, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v47, 24, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v45, v16, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v44, v16, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v48, v53
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v42, 0xff, v16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[45:48], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v41, v15, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v45, v53
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v39, 0xff, v15
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[42:45], off, s[0:3], 0 offset:192
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v38, 24, v14
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v42, v53
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v36, v14, 16, 8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[39:42], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v5, 24, v10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v39, v53
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v2, v10, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xff, v10
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v3, v10, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v7, 0xff, v11
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v20, v12, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v18, 0xff, v12
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v24, 24, v12
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v22, v12, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v28, v13, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v26, 0xff, v13
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v12, 24, v11
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v10, v11, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v31, 24, v13
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v29, v13, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v35, v14, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v33, 0xff, v14
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v16, v17, 8, 8
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v14, 0xff, v17
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v54, 24, v17
+; GCN-NOHSA-VI-NEXT:    v_bfe_u32 v52, v17, 16, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v13, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v17, v53
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[29:32], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v36, v53
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v29, v53
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v10, v53
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[33:36], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[26:29], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    s_nop 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, v53
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v32i8_to_v32i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @22
+; EG-NEXT:    ALU 103, @27, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 33, @131, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T13.XYZW, T42.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T14.XYZW, T41.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T15.XYZW, T40.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T16.XYZW, T39.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T38.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T18.XYZW, T37.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T19.XYZW, T36.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T35.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T34.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T21.XYZW, T33.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T22.XYZW, T32.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T23.XYZW, T31.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T24.XYZW, T30.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T25.XYZW, T29.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T26.XYZW, T28.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T27.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 22:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 16, #1
+; EG-NEXT:    ALU clause starting at 26:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 27:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T13.X, T11.W, literal.x, PV.W,
+; EG-NEXT:     LSHR * T13.Z, T11.W, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T13.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T14.Z, T11.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T14.X, T11.W, literal.x,
+; EG-NEXT:     MOV * T14.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T15.X, T11.Z, literal.x, T0.W,
+; EG-NEXT:     LSHR * T15.Z, T11.Z, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T15.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T16.Z, T11.Z, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T16.X, T11.Z, literal.x,
+; EG-NEXT:     MOV * T16.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T17.X, T11.Y, literal.x, T0.W,
+; EG-NEXT:     LSHR * T17.Z, T11.Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T17.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T18.Z, T11.Y, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T18.X, T11.Y, literal.x,
+; EG-NEXT:     MOV * T18.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T19.X, T11.X, literal.x, T0.W,
+; EG-NEXT:     LSHR * T19.Z, T11.X, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T19.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T11.Z, T11.X, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T11.X, T11.X, literal.x,
+; EG-NEXT:     MOV * T11.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T20.X, T12.W, literal.x, T0.W,
+; EG-NEXT:     LSHR * T20.Z, T12.W, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T20.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T21.Z, T12.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T21.X, T12.W, literal.x,
+; EG-NEXT:     MOV * T21.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T22.X, T12.Z, literal.x, T0.W,
+; EG-NEXT:     LSHR * T22.Z, T12.Z, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T22.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T23.Z, T12.Z, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T23.X, T12.Z, literal.x,
+; EG-NEXT:     MOV * T23.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T24.X, T12.Y, literal.x, T0.W,
+; EG-NEXT:     LSHR * T24.Z, T12.Y, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T24.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T25.Z, T12.Y, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T25.X, T12.Y, literal.x,
+; EG-NEXT:     MOV * T25.Y, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T26.X, T12.X, literal.x, T0.W,
+; EG-NEXT:     LSHR * T26.Z, T12.X, literal.y,
+; EG-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; EG-NEXT:     MOV T26.Y, 0.0,
+; EG-NEXT:     BFE_UINT * T12.Z, T12.X, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T12.X, T12.X, literal.x,
+; EG-NEXT:     MOV T12.Y, 0.0,
+; EG-NEXT:     MOV T13.W, 0.0,
+; EG-NEXT:     MOV * T14.W, 0.0,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T15.W, 0.0,
+; EG-NEXT:     MOV * T16.W, 0.0,
+; EG-NEXT:     MOV T17.W, 0.0,
+; EG-NEXT:     MOV * T18.W, 0.0,
+; EG-NEXT:     MOV T19.W, 0.0,
+; EG-NEXT:     MOV * T11.W, 0.0,
+; EG-NEXT:     MOV T20.W, 0.0,
+; EG-NEXT:     MOV * T21.W, 0.0,
+; EG-NEXT:     MOV T22.W, 0.0,
+; EG-NEXT:     MOV * T23.W, 0.0,
+; EG-NEXT:     MOV T24.W, 0.0,
+; EG-NEXT:     MOV * T25.W, 0.0,
+; EG-NEXT:     MOV T26.W, 0.0,
+; EG-NEXT:     MOV * T12.W, 0.0,
+; EG-NEXT:     LSHR T27.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T28.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T29.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; EG-NEXT:     LSHR T30.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; EG-NEXT:     LSHR * T31.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:    ALU clause starting at 131:
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    80(1.121039e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T32.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 96(1.345247e-43)
+; EG-NEXT:     LSHR T33.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 112(1.569454e-43)
+; EG-NEXT:     LSHR T34.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 128(1.793662e-43)
+; EG-NEXT:     LSHR T35.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 144(2.017870e-43)
+; EG-NEXT:     LSHR T36.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 160(2.242078e-43)
+; EG-NEXT:     LSHR T37.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 176(2.466285e-43)
+; EG-NEXT:     LSHR T38.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 192(2.690493e-43)
+; EG-NEXT:     LSHR T39.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 208(2.914701e-43)
+; EG-NEXT:     LSHR T40.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 224(3.138909e-43)
+; EG-NEXT:     LSHR T41.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 240(3.363116e-43)
+; EG-NEXT:     LSHR * T42.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v32i8_to_v32i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @22
+; CM-NEXT:    ALU 103, @27, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    ALU 33, @131, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T13, T42.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T14, T41.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T15, T40.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T16, T39.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T17, T38.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T37.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T18, T36.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T19, T35.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T20, T34.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T21, T33.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T22, T32.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T23, T31.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T24, T30.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T29.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T25, T28.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T26, T27.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 22:
+; CM-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; CM-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; CM-NEXT:    ALU clause starting at 26:
+; CM-NEXT:     MOV * T11.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 27:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T13.Z, T11.X, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T13.X, T11.X, literal.x,
+; CM-NEXT:     MOV * T13.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T14.X, T11.X, literal.x, T0.W,
+; CM-NEXT:     LSHR * T14.Z, T11.X, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T14.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T15.Z, T11.Y, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T15.X, T11.Y, literal.x,
+; CM-NEXT:     MOV * T15.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T16.X, T11.Y, literal.x, T0.W,
+; CM-NEXT:     LSHR * T16.Z, T11.Y, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T16.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T17.Z, T11.Z, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T17.X, T11.Z, literal.x,
+; CM-NEXT:     MOV * T17.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T11.X, T11.Z, literal.x, T0.W,
+; CM-NEXT:     LSHR * T11.Z, T11.Z, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T11.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T18.Z, T11.W, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T18.X, T11.W, literal.x,
+; CM-NEXT:     MOV * T18.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T19.X, T11.W, literal.x, T0.W,
+; CM-NEXT:     LSHR * T19.Z, T11.W, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T19.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T20.Z, T12.X, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T20.X, T12.X, literal.x,
+; CM-NEXT:     MOV * T20.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T21.X, T12.X, literal.x, T0.W,
+; CM-NEXT:     LSHR * T21.Z, T12.X, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T21.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T22.Z, T12.Y, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T22.X, T12.Y, literal.x,
+; CM-NEXT:     MOV * T22.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T23.X, T12.Y, literal.x, T0.W,
+; CM-NEXT:     LSHR * T23.Z, T12.Y, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T23.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T24.Z, T12.Z, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T24.X, T12.Z, literal.x,
+; CM-NEXT:     MOV * T24.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T12.X, T12.Z, literal.x, T0.W,
+; CM-NEXT:     LSHR * T12.Z, T12.Z, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T12.Y, 0.0,
+; CM-NEXT:     BFE_UINT * T25.Z, T12.W, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T25.X, T12.W, literal.x,
+; CM-NEXT:     MOV * T25.Y, 0.0,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T26.X, T12.W, literal.x, T0.W,
+; CM-NEXT:     LSHR * T26.Z, T12.W, literal.y,
+; CM-NEXT:    16(2.242078e-44), 24(3.363116e-44)
+; CM-NEXT:     MOV T26.Y, 0.0,
+; CM-NEXT:     MOV * T13.W, 0.0,
+; CM-NEXT:     MOV * T14.W, 0.0,
+; CM-NEXT:     MOV * T15.W, 0.0,
+; CM-NEXT:     MOV * T16.W, 0.0,
+; CM-NEXT:     MOV * T17.W, 0.0,
+; CM-NEXT:     MOV * T11.W, 0.0,
+; CM-NEXT:     MOV * T18.W, 0.0,
+; CM-NEXT:     MOV * T19.W, 0.0,
+; CM-NEXT:     MOV * T20.W, 0.0,
+; CM-NEXT:     MOV * T21.W, 0.0,
+; CM-NEXT:     MOV * T22.W, 0.0,
+; CM-NEXT:     MOV * T23.W, 0.0,
+; CM-NEXT:     MOV * T24.W, 0.0,
+; CM-NEXT:     MOV * T12.W, 0.0,
+; CM-NEXT:     MOV * T25.W, 0.0,
+; CM-NEXT:     MOV * T26.W, 0.0,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    240(3.363116e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T27.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 224(3.138909e-43)
+; CM-NEXT:     LSHR T28.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 208(2.914701e-43)
+; CM-NEXT:     LSHR T29.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 192(2.690493e-43)
+; CM-NEXT:     LSHR T30.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 176(2.466285e-43)
+; CM-NEXT:    ALU clause starting at 131:
+; CM-NEXT:     LSHR T31.X, T0.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 160(2.242078e-43)
+; CM-NEXT:     LSHR T32.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 144(2.017870e-43)
+; CM-NEXT:     LSHR T33.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 128(1.793662e-43)
+; CM-NEXT:     LSHR T34.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 112(1.569454e-43)
+; CM-NEXT:     LSHR T35.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 96(1.345247e-43)
+; CM-NEXT:     LSHR T36.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+; CM-NEXT:     LSHR T37.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; CM-NEXT:     LSHR T38.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; CM-NEXT:     LSHR T39.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; CM-NEXT:     LSHR T40.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:     LSHR * T41.X, PV.W, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T42.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = zext <32 x i8> %load to <32 x i64>
   store <32 x i64> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v32i8_to_v32i64:
-
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1
 define amdgpu_kernel void @global_sextload_v32i8_to_v32i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v32i8_to_v32i64:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s39, v3
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s38, v2
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s37, v1
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s36, v0
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s7, v7
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s6, v6
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s11, v5
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s10, v4
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s30, s39, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s34, s39, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s42, s39
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s28, s38, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s26, s38, 24
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s24, s38, 8
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s4, s37, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s8, s37, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s40, s37
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s12, s36, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s14, s36, 24
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s16, s36, 8
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s18, s7, 16
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s20, s7, 8
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s22, s7
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[42:43], s[42:43], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s5, s39, 31
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s9, s39, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[44:45], s[38:39], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s42
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s43
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s38, s6, 16
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s44
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s45
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s42, s6, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[40:41], s[40:41], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v8, s9
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s9, s37, 31
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, s5
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s5, s37, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[36:37], s[36:37], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, s40
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, s41
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s40, s6, 8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v14, s36
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v15, s37
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s36, s11, 16
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[30:31], s[30:31], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[34:35], s[34:35], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, s30
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, s31
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s44, s11, 8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s34
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s35
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s34, s11
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x80000
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:240
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, s28
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v17, s29
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s28, s10, 16
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v18, s26
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v19, s27
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s26, s10, 24
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, s24
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, s25
+; GCN-NOHSA-SI-NEXT:    s_lshr_b32 s24, s10, 8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s5, s7, 31
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s9
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s9, s7, 24
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v8, s9
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s33, s11, 31
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, s5
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s41, s11, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[30:31], s[10:11], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[46:47], s[6:7], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[34:35], s[34:35], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[48:49], s[22:23], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[6:7], s[24:25], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[10:11], s[26:27], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[22:23], s[28:29], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[24:25], s[44:45], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[26:27], s[36:37], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[28:29], s[40:41], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[36:37], s[42:43], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[38:39], s[38:39], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x80000
+; GCN-NOHSA-SI-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x80000
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:192
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s48
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s49
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s46
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s47
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v12, s8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v13, s9
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:160
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, s12
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, s13
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v12, s14
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v13, s15
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:144
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, s41
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v11, s33
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, s16
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v17, s17
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:128
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v12, s34
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v13, s35
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, s18
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, s19
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v16, s30
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v17, s31
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v6, s20
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v7, s21
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s38
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s39
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v4, s36
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v5, s37
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[2:5], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s28
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s29
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v8, s26
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v9, s27
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v14, s24
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v15, s25
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(2)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s22
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s23
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s10
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v18, s6
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v19, s7
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v32i8_to_v32i64:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[6:9], v[0:1]
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[2:5], v[0:1]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s7, v9
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s6, v8
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s9, v7
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s8, v6
+; GCN-HSA-NEXT:    s_lshr_b32 s16, s7, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s18, s7, 8
+; GCN-HSA-NEXT:    s_mov_b32 s24, s7
+; GCN-HSA-NEXT:    s_lshr_b32 s22, s6, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s14, s6, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s2, s6, 8
+; GCN-HSA-NEXT:    s_lshr_b32 s4, s9, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s10, s9, 8
+; GCN-HSA-NEXT:    s_mov_b32 s12, s9
+; GCN-HSA-NEXT:    s_bfe_i64 s[26:27], s[6:7], 0x80000
+; GCN-HSA-NEXT:    s_lshr_b32 s6, s8, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s28, s8, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s30, s8, 8
+; GCN-HSA-NEXT:    s_ashr_i32 s43, s9, 31
+; GCN-HSA-NEXT:    s_ashr_i32 s52, s9, 24
+; GCN-HSA-NEXT:    s_bfe_i64 s[20:21], s[8:9], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[8:9], s[16:17], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[16:17], s[24:25], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s25, v5
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s24, v4
+; GCN-HSA-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x80000
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s37, v3
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s36, v2
+; GCN-HSA-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x80000
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s26
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s27
+; GCN-HSA-NEXT:    s_bfe_i64 s[26:27], s[30:31], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[30:31], s[28:29], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[34:35], s[6:7], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[44:45], s[12:13], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[40:41], s[10:11], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[46:47], s[4:5], 0x80000
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s18
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s22
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, s14
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-HSA-NEXT:    s_lshr_b32 s14, s37, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s10, s37, 8
+; GCN-HSA-NEXT:    s_mov_b32 s12, s37
+; GCN-HSA-NEXT:    s_lshr_b32 s8, s36, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s6, s36, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s4, s36, 8
+; GCN-HSA-NEXT:    s_bfe_i64 s[2:3], s[36:37], 0x80000
+; GCN-HSA-NEXT:    s_lshr_b32 s38, s25, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s28, s25, 8
+; GCN-HSA-NEXT:    s_mov_b32 s36, s25
+; GCN-HSA-NEXT:    s_lshr_b32 s48, s24, 16
+; GCN-HSA-NEXT:    s_lshr_b32 s22, s24, 24
+; GCN-HSA-NEXT:    s_lshr_b32 s18, s24, 8
+; GCN-HSA-NEXT:    s_ashr_i32 s50, s7, 31
+; GCN-HSA-NEXT:    s_ashr_i32 s51, s7, 24
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s17
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s19
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s23
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, s15
+; GCN-HSA-NEXT:    s_ashr_i32 s33, s37, 31
+; GCN-HSA-NEXT:    s_ashr_i32 s42, s37, 24
+; GCN-HSA-NEXT:    s_ashr_i32 s53, s25, 31
+; GCN-HSA-NEXT:    s_ashr_i32 s54, s25, 24
+; GCN-HSA-NEXT:    s_bfe_i64 s[16:17], s[24:25], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[24:25], s[48:49], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[36:37], s[36:37], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x80000
+; GCN-HSA-NEXT:    s_bfe_i64 s[38:39], s[38:39], 0x80000
+; GCN-HSA-NEXT:    s_add_u32 s48, s0, 0x70
+; GCN-HSA-NEXT:    s_addc_u32 s49, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s49
+; GCN-HSA-NEXT:    s_add_u32 s48, s0, 0x60
+; GCN-HSA-NEXT:    s_addc_u32 s49, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v24, s48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v25, s49
+; GCN-HSA-NEXT:    s_add_u32 s48, s0, 0x50
+; GCN-HSA-NEXT:    s_addc_u32 s49, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v26, s48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v27, s49
+; GCN-HSA-NEXT:    s_add_u32 s48, s0, 64
+; GCN-HSA-NEXT:    s_addc_u32 s49, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s44
+; GCN-HSA-NEXT:    s_add_u32 s44, s0, 48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s45
+; GCN-HSA-NEXT:    s_addc_u32 s45, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v28, s48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s51
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s50
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[26:27], v[4:7]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v29, s49
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s34
+; GCN-HSA-NEXT:    s_add_u32 s34, s0, 32
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[8:11]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s35
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s44
+; GCN-HSA-NEXT:    s_addc_u32 s35, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s46
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s47
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s45
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s52
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s43
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[28:29], v[0:3]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s20
+; GCN-HSA-NEXT:    s_add_u32 s20, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s21
+; GCN-HSA-NEXT:    s_addc_u32 s21, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s20
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s21
+; GCN-HSA-NEXT:    s_add_u32 s20, s0, 0xf0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[24:25], v[12:15]
+; GCN-HSA-NEXT:    s_addc_u32 s21, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s34
+; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s20
+; GCN-HSA-NEXT:    v_mov_b32_e32 v22, s40
+; GCN-HSA-NEXT:    v_mov_b32_e32 v23, s41
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s35
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, s30
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, s31
+; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s21
+; GCN-HSA-NEXT:    s_add_u32 s20, s0, 0xe0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[20:23]
+; GCN-HSA-NEXT:    s_addc_u32 s21, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s20
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[4:7]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s26
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s27
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s21
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    s_add_u32 s20, s0, 0xd0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s38
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s39
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s54
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s53
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s36
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s37
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s28
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s29
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[8:11]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[12:15]
+; GCN-HSA-NEXT:    s_addc_u32 s21, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s20
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s24
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s25
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s22
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s23
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s21
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_nop 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-HSA-NEXT:    s_add_u32 s16, s0, 0xc0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s17
+; GCN-HSA-NEXT:    s_addc_u32 s17, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s18
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s19
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s17
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_nop 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s14
+; GCN-HSA-NEXT:    s_add_u32 s14, s0, 0xb0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-HSA-NEXT:    s_addc_u32 s15, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s14
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s42
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s33
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s15
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_nop 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s10
+; GCN-HSA-NEXT:    s_add_u32 s10, s0, 0xa0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-HSA-NEXT:    s_addc_u32 s11, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s10
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s11
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_nop 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 0x90
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s6
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 0x80
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s7
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v32i8_to_v32i64:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s9, v3
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s8, v2
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s30, s9, 16
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s11, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s38, s9, 8
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s40, s9
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s46, s8, 8
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s71, s9, 31
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s72, s9, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[30:31], s[30:31], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s10, v0
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s50, s11, 16
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[28:29], s[8:9], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[46:47], s[46:47], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[40:41], s[40:41], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[38:39], s[38:39], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s72
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s71
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s30
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s31
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s13, v7
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s12, v6
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s15, v5
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s14, v4
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s52, s11, 8
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s54, s11
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s69, s11, 31
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s70, s11, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[50:51], s[50:51], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v4, s40
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, s41
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, s38
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, s39
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s28
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s29
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s46
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s47
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s56, s10, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s58, s10, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[54:55], s[54:55], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[52:53], s[52:53], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:192
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s60, s10, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s50
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s51
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s70
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s69
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[58:59], s[58:59], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[56:57], s[56:57], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s48, s13, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s54
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s55
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s52
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s53
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[18:19], s[10:11], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[60:61], s[60:61], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s34, s13, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s56
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s57
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s58
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s59
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s36, s13
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s65, s13, 31
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s67, s13, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[48:49], s[48:49], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s26, s12, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s18
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s19
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s60
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s61
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s24, s12, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[36:37], s[36:37], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[34:35], s[34:35], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s22, s12, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s48
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s49
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s67
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s65
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s20, s15, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s36
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s37
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s34
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s35
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[6:7], s[12:13], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s62, s15, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s26
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s27
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s24
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s25
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s16, s15
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s33, s15, 31
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s63, s15, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s64, s14, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s22
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s23
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s66, s14, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s68, s14, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[4:5], s[14:15], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[14:15], s[62:63], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s42, s8, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s20
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s21
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s63
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s33
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s44, s8, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[10:11], s[66:67], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[12:13], s[64:65], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[8:9], s[68:69], 0x80000
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s17
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s14
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s15
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[44:45], s[44:45], 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i64 s[42:43], s[42:43], 0x80000
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v8, s42
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s12
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v9, s43
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v10, s44
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, s45
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s9
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v32i8_to_v32i64:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @22
+; EG-NEXT:    ALU 84, @27, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    ALU 71, @112, KC0[], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T41.XYZW, T42.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T26.XYZW, T31.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T40.XYZW, T30.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T11.XYZW, T25.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T39.XYZW, T24.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T27.XYZW, T23.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T12.XYZW, T22.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T28.XYZW, T21.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T37.XYZW, T20.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T29.XYZW, T19.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T35.XYZW, T18.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T33.XYZW, T17.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T34.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T38.XYZW, T15.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T32.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T36.XYZW, T13.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 22:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 0, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 16, #1
+; EG-NEXT:    ALU clause starting at 26:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 27:
+; EG-NEXT:     LSHR T13.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T14.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
+; EG-NEXT:     LSHR T15.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 48(6.726233e-44)
+; EG-NEXT:     LSHR T16.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 64(8.968310e-44)
+; EG-NEXT:     LSHR T17.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 80(1.121039e-43)
+; EG-NEXT:     LSHR T18.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 96(1.345247e-43)
+; EG-NEXT:     LSHR T19.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 112(1.569454e-43)
+; EG-NEXT:     LSHR T20.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 128(1.793662e-43)
+; EG-NEXT:     LSHR T21.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 144(2.017870e-43)
+; EG-NEXT:     LSHR T22.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 160(2.242078e-43)
+; EG-NEXT:     LSHR T23.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 176(2.466285e-43)
+; EG-NEXT:     LSHR T24.X, PV.W, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 192(2.690493e-43)
+; EG-NEXT:     LSHR * T25.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT * T26.X, T11.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T27.X, T11.Y, 0.0, literal.x,
+; EG-NEXT:     ASHR T26.Y, PV.X, literal.y,
+; EG-NEXT:     LSHR * T0.W, T11.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T28.X, T11.X, 0.0, literal.x,
+; EG-NEXT:     ASHR T27.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T26.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T11.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T29.X, T12.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T28.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T27.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.W, T11.X, literal.x,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    208(2.914701e-43), 0(0.000000e+00)
+; EG-NEXT:     LSHR T30.X, PS, literal.x,
+; EG-NEXT:     ASHR T29.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T28.Z, PV.W, 0.0, literal.z,
+; EG-NEXT:     LSHR T0.W, T12.W, literal.z,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.w,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; EG-NEXT:    8(1.121039e-44), 224(3.138909e-43)
+; EG-NEXT:     LSHR T31.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T29.Z, PV.W, 0.0, literal.y,
+; EG-NEXT:     ADD_INT T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:     ASHR * T32.W, T12.X, literal.w,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    240(3.363116e-43), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T33.X, T12.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Y, T11.Z, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     ASHR T32.Z, T12.X, literal.y,
+; EG-NEXT:     LSHR T1.W, T12.X, literal.z,
+; EG-NEXT:     ASHR * T34.W, T12.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 24(3.363116e-44)
+; EG-NEXT:    16(2.242078e-44), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T32.X, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T33.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T34.Z, T12.Y, literal.z,
+; EG-NEXT:     LSHR T1.W, T12.Z, literal.x,
+; EG-NEXT:     LSHR * T2.W, T12.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT * T34.X, PS, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:    ALU clause starting at 112:
+; EG-NEXT:     ASHR T32.Y, T32.X, literal.x,
+; EG-NEXT:     BFE_INT T33.Z, T1.W, 0.0, literal.y,
+; EG-NEXT:     LSHR T1.W, T11.W, literal.z, BS:VEC_120/SCL_212
+; EG-NEXT:     ASHR * T35.W, T12.Z, literal.x,
+; EG-NEXT:    31(4.344025e-44), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T36.X, T12.X, 0.0, literal.x,
+; EG-NEXT:     ASHR T34.Y, T34.X, literal.y, BS:VEC_120/SCL_212
+; EG-NEXT:     ASHR T35.Z, T12.Z, literal.z,
+; EG-NEXT:     LSHR T2.W, T12.Z, literal.w,
+; EG-NEXT:     ASHR * T37.W, T12.W, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T35.X, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T36.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T37.Z, T12.W, literal.z,
+; EG-NEXT:     LSHR T2.W, T12.X, literal.x,
+; EG-NEXT:     LSHR * T3.W, T12.W, literal.w,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T37.X, PS, 0.0, literal.x,
+; EG-NEXT:     ASHR T35.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T36.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR T2.W, T11.Z, literal.z,
+; EG-NEXT:     ASHR * T12.W, T11.X, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T38.X, T12.Y, 0.0, literal.x,
+; EG-NEXT:     ASHR T37.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T12.Z, T11.X, literal.z,
+; EG-NEXT:     LSHR T3.W, T11.X, literal.w,
+; EG-NEXT:     ASHR * T39.W, T11.Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T12.X, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T38.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T39.Z, T11.Y, literal.z,
+; EG-NEXT:     LSHR T3.W, T12.Y, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR * T4.W, T11.Y, literal.w,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 16(2.242078e-44)
+; EG-NEXT:     BFE_INT T39.X, PS, 0.0, literal.x,
+; EG-NEXT:     ASHR T12.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T38.Z, PV.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T36.W, T36.Z, literal.y,
+; EG-NEXT:     ASHR * T40.W, T11.Z, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:     BFE_INT T11.X, T11.Z, 0.0, literal.x,
+; EG-NEXT:     ASHR T39.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T40.Z, T11.Z, literal.z,
+; EG-NEXT:     ASHR T38.W, PV.Z, literal.y,
+; EG-NEXT:     ASHR * T41.W, T11.W, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T40.X, T2.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T11.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T41.Z, T11.W, literal.z, BS:VEC_120/SCL_212
+; EG-NEXT:     ASHR T33.W, T33.Z, literal.y,
+; EG-NEXT:     ASHR * T29.W, T29.Z, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T41.X, T1.W, 0.0, literal.x,
+; EG-NEXT:     ASHR T40.Y, PV.X, literal.y,
+; EG-NEXT:     BFE_INT T11.Z, T0.Y, 0.0, literal.x,
+; EG-NEXT:     ASHR T28.W, T28.Z, literal.y,
+; EG-NEXT:     ASHR * T27.W, T27.Z, literal.y,
+; EG-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; EG-NEXT:     LSHR T42.X, T0.W, literal.x,
+; EG-NEXT:     ASHR T41.Y, PV.X, literal.y,
+; EG-NEXT:     ASHR T11.W, PV.Z, literal.y,
+; EG-NEXT:     ASHR * T26.W, T26.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+;
+; CM-LABEL: global_sextload_v32i8_to_v32i64:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @26, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @22
+; CM-NEXT:    ALU 84, @27, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    ALU 74, @112, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T41, T42.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T31.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T28, T30.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T40, T25.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T38, T24.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T39, T23.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T36, T22.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T37, T21.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T33, T20.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T19.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T29, T18.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T35, T17.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T27, T16.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T34, T15.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T26, T14.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T32, T13.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 22:
+; CM-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; CM-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; CM-NEXT:    ALU clause starting at 26:
+; CM-NEXT:     MOV * T11.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 27:
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    240(3.363116e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T13.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 224(3.138909e-43)
+; CM-NEXT:     LSHR T14.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 208(2.914701e-43)
+; CM-NEXT:     LSHR T15.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 192(2.690493e-43)
+; CM-NEXT:     LSHR T16.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 176(2.466285e-43)
+; CM-NEXT:     LSHR T17.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 160(2.242078e-43)
+; CM-NEXT:     LSHR T18.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 144(2.017870e-43)
+; CM-NEXT:     LSHR T19.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 128(1.793662e-43)
+; CM-NEXT:     LSHR T20.X, PV.W, literal.x,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 112(1.569454e-43)
+; CM-NEXT:     LSHR T21.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T0.Z, T11.X, literal.y,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    96(1.345247e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T22.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T0.Y, T11.X, literal.y,
+; CM-NEXT:     LSHR T1.Z, T11.Y, literal.y,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    80(1.121039e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR T23.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T1.Y, T11.Z, literal.y,
+; CM-NEXT:     LSHR T2.Z, T11.Z, literal.z,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 64(8.968310e-44)
+; CM-NEXT:     LSHR T24.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T2.Y, T11.W, literal.y,
+; CM-NEXT:     LSHR T3.Z, T11.W, literal.z,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.w,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 48(6.726233e-44)
+; CM-NEXT:     LSHR T25.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T3.Y, T12.X, literal.y,
+; CM-NEXT:     LSHR T4.Z, T12.Y, literal.y,
+; CM-NEXT:     LSHR * T0.W, T12.X, literal.z,
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T26.X, T12.W, 0.0, literal.x,
+; CM-NEXT:     LSHR T4.Y, T12.Y, literal.x,
+; CM-NEXT:     ADD_INT T5.Z, KC0[2].Y, literal.y,
+; CM-NEXT:     LSHR * T1.W, T11.Y, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     BFE_INT T27.X, T12.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR T26.Y, PV.X, literal.y,
+; CM-NEXT:     ADD_INT T6.Z, KC0[2].Y, literal.z,
+; CM-NEXT:     LSHR * T2.W, T12.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T28.X, T11.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T27.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T26.Z, PV.W, 0.0, literal.x,
+; CM-NEXT:     LSHR * T2.W, T12.Z, literal.x,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T29.X, T12.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T28.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T27.Z, PV.W, 0.0, literal.x,
+; CM-NEXT:     ASHR * T26.W, PV.Z, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     LSHR T30.X, T6.Z, literal.x,
+; CM-NEXT:     ASHR T29.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T28.Z, T1.W, 0.0, literal.z,
+; CM-NEXT:     ASHR * T27.W, PV.Z, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T31.X, T5.Z, literal.x,
+; CM-NEXT:     LSHR * T5.Y, T12.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; CM-NEXT:    ALU clause starting at 112:
+; CM-NEXT:     BFE_INT T29.Z, T4.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR * T32.W, T12.W, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T33.X, T12.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T4.Y, T12.W, literal.y,
+; CM-NEXT:     ASHR T32.Z, T12.W, literal.z,
+; CM-NEXT:     ASHR * T34.W, T12.Z, literal.w,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:    24(3.363116e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T32.X, PV.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T33.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T34.Z, T12.Z, literal.z,
+; CM-NEXT:     ASHR * T29.W, T29.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T34.X, T5.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T32.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T33.Z, T0.W, 0.0, literal.x,
+; CM-NEXT:     ASHR * T35.W, T12.Y, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T36.X, T11.W, 0.0, literal.x,
+; CM-NEXT:     ASHR T34.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T35.Z, T12.Y, literal.z,
+; CM-NEXT:     ASHR * T12.W, T12.X, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T35.X, T4.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR T36.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T12.Z, T12.X, literal.z,
+; CM-NEXT:     ASHR * T33.W, T33.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T12.X, T3.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T35.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T36.Z, T3.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR * T37.W, T11.W, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T38.X, T11.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR T12.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T37.Z, T11.W, literal.z,
+; CM-NEXT:     ASHR * T39.W, T11.Z, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T37.X, T2.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T38.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T39.Z, T11.Z, literal.z,
+; CM-NEXT:     ASHR * T36.W, T36.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T39.X, T2.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR T37.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T38.Z, T1.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR * T40.W, T11.Y, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     BFE_INT T41.X, T11.X, 0.0, literal.x,
+; CM-NEXT:     ASHR T39.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T40.Z, T11.Y, literal.z,
+; CM-NEXT:     ASHR * T11.W, T11.X, literal.y,
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T40.X, T1.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR T41.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR T11.Z, T11.X, literal.z,
+; CM-NEXT:     ASHR * T38.W, T38.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:    24(3.363116e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T11.X, T0.Y, 0.0, literal.x,
+; CM-NEXT:     ASHR T40.Y, PV.X, literal.y,
+; CM-NEXT:     BFE_INT T41.Z, T0.Z, 0.0, literal.x,
+; CM-NEXT:     ASHR * T28.W, T28.Z, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 31(4.344025e-44)
+; CM-NEXT:     LSHR T42.X, KC0[2].Y, literal.x,
+; CM-NEXT:     ASHR T11.Y, PV.X, literal.y,
+; CM-NEXT:     ASHR * T41.W, PV.Z, literal.y,
+; CM-NEXT:    2(2.802597e-45), 31(4.344025e-44)
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = sext <32 x i8> %load to <32 x i64>
   store <32 x i64> %ext, ptr addrspace(1) %out
@@ -739,200 +9137,2698 @@ define amdgpu_kernel void @global_sextload_v32i8_to_v32i64(ptr addrspace(1) %out
 ;   ret void
 ; }
 
-; FUNC-LABEL: {{^}}global_zextload_i8_to_i16:
-; GCN-NOHSA: buffer_load_ubyte v[[VAL:[0-9]+]],
-; GCN-NOHSA: buffer_store_short v[[VAL]]
-
-; GCN-HSA: flat_load_ubyte v[[VAL:[0-9]+]],
-; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]]
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_i8_to_i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_i8_to_i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_i8_to_i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_short v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_i8_to_i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_i8_to_i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.X, T0.X, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_i8_to_i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHL T0.X, T0.X, PV.W,
+; CM-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; CM-NEXT:     MOV T0.Y, 0.0,
+; CM-NEXT:     MOV * T0.Z, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i8, ptr addrspace(1) %in
   %ext = zext i8 %a to i16
   store i16 %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_i8_to_i16:
-; GCN-NOHSA: buffer_load_sbyte v[[VAL:[0-9]+]],
-; GCN-HSA: flat_load_sbyte v[[VAL:[0-9]+]],
-
-; GCN-NOHSA: buffer_store_short v[[VAL]]
-; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]]
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
 define amdgpu_kernel void @global_sextload_i8_to_i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_i8_to_i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_i8_to_i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_sbyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_short v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_i8_to_i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_i8_to_i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.W, T0.X, 0.0, literal.x,
+; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 3(4.203895e-45)
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     LSHL * T1.W, PS, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_i8_to_i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     AND_INT T0.Z, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T0.W, T0.X, 0.0, literal.y,
+; CM-NEXT:    3(4.203895e-45), 8(1.121039e-44)
+; CM-NEXT:     AND_INT T1.Z, PV.W, literal.x,
+; CM-NEXT:     LSHL * T0.W, PV.Z, literal.y,
+; CM-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; CM-NEXT:     LSHL T0.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; CM-NEXT:     MOV T0.Y, 0.0,
+; CM-NEXT:     MOV * T0.Z, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %a = load i8, ptr addrspace(1) %in
   %ext = sext i8 %a to i16
   store i16 %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v1i8_to_v1i16:
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v1i8_to_v1i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v1i8_to_v1i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v1i8_to_v1i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ubyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_short v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v1i8_to_v1i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v1i8_to_v1i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.X, T0.X, PV.W,
+; EG-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v1i8_to_v1i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; CM-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; CM-NEXT:     LSHL T0.X, T0.X, PV.W,
+; CM-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; CM-NEXT:     MOV T0.Y, 0.0,
+; CM-NEXT:     MOV * T0.Z, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <1 x i8>, ptr addrspace(1) %in
   %ext = zext <1 x i8> %load to <1 x i16>
   store <1 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v1i8_to_v1i16:
-
-; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
 define amdgpu_kernel void @global_sextload_v1i8_to_v1i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v1i8_to_v1i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v1i8_to_v1i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_sbyte v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    flat_store_short v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v1i8_to_v1i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_sbyte v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v1i8_to_v1i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T0.W, T0.X, 0.0, literal.x,
+; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 3(4.203895e-45)
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     LSHL * T1.W, PS, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_v1i8_to_v1i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     AND_INT T0.Z, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T0.W, T0.X, 0.0, literal.y,
+; CM-NEXT:    3(4.203895e-45), 8(1.121039e-44)
+; CM-NEXT:     AND_INT T1.Z, PV.W, literal.x,
+; CM-NEXT:     LSHL * T0.W, PV.Z, literal.y,
+; CM-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; CM-NEXT:     LSHL T0.X, PV.Z, PV.W,
+; CM-NEXT:     LSHL * T0.W, literal.x, PV.W,
+; CM-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; CM-NEXT:     MOV T0.Y, 0.0,
+; CM-NEXT:     MOV * T0.Z, 0.0,
+; CM-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <1 x i8>, ptr addrspace(1) %in
   %ext = sext <1 x i8> %load to <1 x i16>
   store <1 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v2i8_to_v2i16:
-
-; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v2i8_to_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v2i8_to_v2i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xff00ff, v0
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v2i8_to_v2i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v3, 8, v2
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v2, 0xff00ff, v2
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v2i8_to_v2i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v1, 0xff0000, v1
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v2i8_to_v2i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 7, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.X, T6.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T5.X, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     LSHL * T0.W, T5.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T5.X, literal.y,
+; EG-NEXT:    16711680(2.341805e-38), 255(3.573311e-43)
+; EG-NEXT:     OR_INT T5.X, PS, PV.W,
+; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v2i8_to_v2i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 7, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5.X, T6.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T5.X, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     LSHL * T0.W, T5.X, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T0.Z, PV.W, literal.x,
+; CM-NEXT:     AND_INT * T0.W, T5.X, literal.y,
+; CM-NEXT:    16711680(2.341805e-38), 255(3.573311e-43)
+; CM-NEXT:     OR_INT * T5.X, PV.W, PV.Z,
+; CM-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <2 x i8>, ptr addrspace(1) %in
   %ext = zext <2 x i8> %load to <2 x i16>
   store <2 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v2i8_to_v2i16:
-
-; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
 define amdgpu_kernel void @global_sextload_v2i8_to_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v2i8_to_v2i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NOHSA-SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v2i8_to_v2i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_ushort v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_bfe_i32 v3, v2, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-HSA-NEXT:    flat_store_dword v[0:1], v2
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v2i8_to_v2i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, 8
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_and_b32_sdwa v1, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_sdwa v0, v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NOHSA-VI-NEXT:    v_or_b32_e32 v0, v1, v0
+; GCN-NOHSA-VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v2i8_to_v2i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 16, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.X, T6.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_16 T5.X, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.Y, T2.X,
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     AND_INT T0.W, T5.X, literal.x,
+; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
+; EG-NEXT:    65535(9.183409e-41), -65536(nan)
+; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
+; EG-NEXT:     MOV * T2.X, PV.W,
+; EG-NEXT:     MOV * T0.Y, PV.X,
+; EG-NEXT:     LSHR * T1.W, PV.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T0.Z, T0.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT * T0.W, PV.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
+; EG-NEXT:     AND_INT * T1.W, PV.Z, literal.y,
+; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
+; EG-NEXT:     OR_INT T5.X, PS, PV.W,
+; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_v2i8_to_v2i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 16, @10, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5.X, T6.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_16 T5.X, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T0.Y, T2.X,
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 10:
+; CM-NEXT:     AND_INT T0.Z, T5.X, literal.x,
+; CM-NEXT:     AND_INT * T0.W, T0.Y, literal.y,
+; CM-NEXT:    65535(9.183409e-41), -65536(nan)
+; CM-NEXT:     OR_INT * T0.W, PV.W, PV.Z,
+; CM-NEXT:     MOV * T2.X, PV.W,
+; CM-NEXT:     MOV * T0.Y, PV.X,
+; CM-NEXT:     LSHR * T1.W, PV.Y, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T0.Z, T0.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T0.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHL T1.Z, PV.W, literal.x,
+; CM-NEXT:     AND_INT * T0.W, PV.Z, literal.y,
+; CM-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
+; CM-NEXT:     OR_INT * T5.X, PV.W, PV.Z,
+; CM-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <2 x i8>, ptr addrspace(1) %in
   %ext = sext <2 x i8> %load to <2 x i16>
   store <2 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v4i8_to_v4i16:
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v4i8_to_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v4i8_to_v4i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff00, v0
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v3, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v0, v2, v0, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v2, 8, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v3, v2
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v4i8_to_v4i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff00, v2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v4, 24, v2
+; GCN-HSA-NEXT:    v_and_b32_e32 v5, 0xff, v2
+; GCN-HSA-NEXT:    v_alignbit_b32 v2, v4, v2, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v4, 8, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff00ff, v2
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v5, v4
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v4i8_to_v4i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v2, 8, v0
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v1, v1, v0, 16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v2, 0xff0000, v2
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v1
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v4i8_to_v4i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 6, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T4.XY, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T4.Y, T4.X, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T4.X, T4.X, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
+;
+; CM-LABEL: global_zextload_v4i8_to_v4i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 7, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T4, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T4.Y, T4.X, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT * T4.X, T4.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <4 x i8>, ptr addrspace(1) %in
   %ext = zext <4 x i8> %load to <4 x i16>
   store <4 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v4i8_to_v4i16:
-
-; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
 define amdgpu_kernel void @global_sextload_v4i8_to_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v4i8_to_v4i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v0, 16, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v3, v0, 8, 8
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v1, v2, v1
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v0, v3
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v4i8_to_v4i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dword v2, v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v3, 24, v2
+; GCN-HSA-NEXT:    v_bfe_i32 v4, v2, 16, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v5, v2, 8, 8
+; GCN-HSA-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GCN-HSA-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GCN-HSA-NEXT:    v_or_b32_e32 v3, v4, v3
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v2, v5
+; GCN-HSA-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v4i8_to_v4i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, 8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_sdwa v1, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_sdwa v4, v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_sdwa v2, v2, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GCN-NOHSA-VI-NEXT:    v_or_b32_e32 v0, v4, v1
+; GCN-NOHSA-VI-NEXT:    v_or_b32_e32 v1, v2, v5
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v4i8_to_v4i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 5, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T5.XY, T4.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T4.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.W, T4.X, literal.x,
+; EG-NEXT:     LSHR * T4.X, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 2(2.802597e-45)
+; EG-NEXT:     BFE_INT * T5.Y, PV.W, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+;
+; CM-LABEL: global_sextload_v4i8_to_v4i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 5, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T5, T4.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_32 T4.X, T4.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T4.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT T5.X, T4.X, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, T4.X, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T4.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T5.Y, PV.W, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <4 x i8>, ptr addrspace(1) %in
   %ext = sext <4 x i8> %load to <4 x i16>
   store <4 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v8i8_to_v8i16:
-
-; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v8i8_to_v8i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v8i8_to_v8i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s6, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s7
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s3
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s4, s0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s5, s1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v2, 0xff00, v0
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff00, v1
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v7, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v1, v5, v1, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v0, v3, v0, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v2
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v3, 0xff00ff, v1
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v2, v6, v4
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v7, v5
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v8i8_to_v8i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_and_b32_e32 v2, 0xff00, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GCN-HSA-NEXT:    v_and_b32_e32 v6, 0xff00, v1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v9, 0xff, v0
+; GCN-HSA-NEXT:    v_alignbit_b32 v1, v7, v1, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GCN-HSA-NEXT:    v_alignbit_b32 v0, v3, v0, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v7, 8, v2
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff00ff, v1
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v8, v6
+; GCN-HSA-NEXT:    v_and_b32_e32 v1, 0xff00ff, v0
+; GCN-HSA-NEXT:    v_or_b32_e32 v0, v9, v7
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v8i8_to_v8i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s5, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s6, s4, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s7, s4, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v0
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s4, s4, 0xff0000
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v2, v2, v0, 16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v3, 0xff0000, v1
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s5, s6, s5
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s4, s7, s4
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v2
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v8i8_to_v8i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 9, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T6.W, T5.Y, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T6.Y, T5.X, literal.x, T0.W,
+; EG-NEXT:     AND_INT * T6.Z, T5.Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; EG-NEXT:     AND_INT T6.X, T5.X, literal.x,
+; EG-NEXT:     LSHR * T5.X, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 2(2.802597e-45)
+;
+; CM-LABEL: global_zextload_v8i8_to_v8i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T6.W, T5.Y, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T6.Y, T5.X, literal.x, T0.W,
+; CM-NEXT:     AND_INT * T6.Z, T5.Y, literal.y,
+; CM-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; CM-NEXT:     AND_INT * T6.X, T5.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <8 x i8>, ptr addrspace(1) %in
   %ext = zext <8 x i8> %load to <8 x i16>
   store <8 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v8i8_to_v8i16:
-
-; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
 define amdgpu_kernel void @global_sextload_v8i8_to_v8i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v8i8_to_v8i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s1, v1
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s6, s1, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s7, s1, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s8, s1, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s1, s1
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s9, s0, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s10, s0, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s11, s0, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s0, s0
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s6, s6, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s7, s7, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s1, s1, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s9, s9, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s10, s10, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s11, s11, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s0, s0, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s6, s7, s6
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s7, s1, s8
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s8, s10, s9
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s9, s0, s11
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s7
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s6
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v8i8_to_v8i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s1, v1
+; GCN-HSA-NEXT:    s_ashr_i32 s2, s1, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s3, s1, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s4, s1, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s1, s1
+; GCN-HSA-NEXT:    s_ashr_i32 s5, s0, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s6, s0, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s7, s0, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s0, s0
+; GCN-HSA-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-HSA-NEXT:    s_and_b32 s3, s3, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-HSA-NEXT:    s_and_b32 s1, s1, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-HSA-NEXT:    s_and_b32 s6, s6, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s7, s7, 16
+; GCN-HSA-NEXT:    s_and_b32 s0, s0, 0xffff
+; GCN-HSA-NEXT:    s_or_b32 s2, s3, s2
+; GCN-HSA-NEXT:    s_or_b32 s1, s1, s4
+; GCN-HSA-NEXT:    s_or_b32 s3, s6, s5
+; GCN-HSA-NEXT:    s_or_b32 s0, s0, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v8i8_to_v8i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v0
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s6, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s7, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s8, s5
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s9, s5, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s10, s4
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s5, s5, 24
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s4, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s8, s8, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s7, s7, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s6, s6, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s9, 0xffff, s9
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s10, s10, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s8, s8, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s7, 0xffff, s7
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s6, 0xffff, s6
+; GCN-NOHSA-VI-NEXT:    v_and_b32_sdwa v0, v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s10, s10, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s8, s9, s8
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s5, s7, s5
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s4, s6, s4
+; GCN-NOHSA-VI-NEXT:    v_or_b32_e32 v0, s10, v0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v8i8_to_v8i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T6.XYZW, T5.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T5.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT * T6.Z, T5.Y, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T6.X, T5.X, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T5.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T6.W, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T5.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_INT * T6.Y, PS, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v8i8_to_v8i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 10, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T6, T5.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    PAD
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_64 T5.XY, T5.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T5.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT * T6.Z, T5.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T6.X, T5.X, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, T5.Y, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T0.Z, T5.X, literal.x,
+; CM-NEXT:     BFE_INT * T6.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T5.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T6.Y, PV.Z, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <8 x i8>, ptr addrspace(1) %in
   %ext = sext <8 x i8> %load to <8 x i16>
   store <8 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v16i8_to_v16i16:
-
-; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @global_zextload_v16i8_to_v16i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v16i8_to_v16i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff00, v2
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v6, 0xff00, v3
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xff00, v0
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v9, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v10, 0xff00, v1
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v12, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v13, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v14, 0xff, v3
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v15, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v1, v11, v1, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v10, 8, v10
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v0, v9, v0, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v7, v7, v3, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v5, v5, v2, 16
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v3, 0xff00ff, v1
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v2, v12, v10
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v13, v8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v7, 0xff00ff, v7
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v6, v14, v6
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v5, 0xff00ff, v5
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v4, v15, v4
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v16i8_to_v16i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_and_b32_e32 v4, 0xff00, v2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
+; GCN-HSA-NEXT:    v_and_b32_e32 v6, 0xff00, v3
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v7, 24, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v12, 0xff00, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v13, 24, v0
+; GCN-HSA-NEXT:    v_and_b32_e32 v14, 0xff00, v1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v15, 24, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v18, 0xff, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v19, 0xff, v2
+; GCN-HSA-NEXT:    v_alignbit_b32 v7, v7, v3, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GCN-HSA-NEXT:    v_alignbit_b32 v5, v5, v2, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GCN-HSA-NEXT:    v_and_b32_e32 v16, 0xff, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v17, 0xff, v0
+; GCN-HSA-NEXT:    v_alignbit_b32 v1, v15, v1, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v14, 8, v14
+; GCN-HSA-NEXT:    v_alignbit_b32 v0, v13, v0, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v12, 8, v12
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff00ff, v7
+; GCN-HSA-NEXT:    v_or_b32_e32 v6, v18, v6
+; GCN-HSA-NEXT:    v_and_b32_e32 v5, 0xff00ff, v5
+; GCN-HSA-NEXT:    v_or_b32_e32 v4, v19, v4
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff00ff, v1
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v16, v14
+; GCN-HSA-NEXT:    v_and_b32_e32 v1, 0xff00ff, v0
+; GCN-HSA-NEXT:    v_or_b32_e32 v0, v17, v12
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v16i8_to_v16i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v3
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s6, s5, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s9, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s10, s4, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s11, s4, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v4, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v1, 8, v0
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v5, 8, v2
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s7, s5, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s8, s5, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s6, s6, 16
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s9, s9, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s4, s4, 0xff0000
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v6, 0xff0000, v1
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v4, v4, v2, 16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v7, 0xff0000, v5
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s5, s5, 0xff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s6, s7, s6
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s7, s10, s9
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s4, s11, s4
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v3, v3, v0, 16
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v5, 0xff00ff, v4
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v4, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s5, s8, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, s7
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v3
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s6
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v16i8_to_v16i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 19, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T10.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T7.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T8.W, T7.Y, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T8.Y, T7.X, literal.x, T0.W,
+; EG-NEXT:     AND_INT T8.Z, T7.Y, literal.y,
+; EG-NEXT:     BFE_UINT * T9.W, T7.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; EG-NEXT:     AND_INT T8.X, T7.X, literal.x,
+; EG-NEXT:     BFE_UINT T9.Y, T7.Z, literal.y, T0.W,
+; EG-NEXT:     LSHR * T7.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT * T9.Z, T7.W, literal.x,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T9.X, T7.Z, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    255(3.573311e-43), 16(2.242078e-44)
+; EG-NEXT:     LSHR * T10.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v16i8_to_v16i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 19, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T9, T7.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T8, T10.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T7.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T8.W, T7.W, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T8.Y, T7.Z, literal.x, T0.W,
+; CM-NEXT:     AND_INT T8.Z, T7.W, literal.y,
+; CM-NEXT:     BFE_UINT * T9.W, T7.Y, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; CM-NEXT:     AND_INT T8.X, T7.Z, literal.x,
+; CM-NEXT:     BFE_UINT T9.Y, T7.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T10.X, PV.W, literal.x,
+; CM-NEXT:     AND_INT * T9.Z, T7.Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 255(3.573311e-43)
+; CM-NEXT:     AND_INT * T9.X, T7.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <16 x i8>, ptr addrspace(1) %in
   %ext = zext <16 x i8> %load to <16 x i16>
   store <16 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v16i8_to_v16i16:
-
-; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
 define amdgpu_kernel void @global_sextload_v16i8_to_v16i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v16i8_to_v16i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s4, v2
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s5, v3
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s6, v0
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s7, v1
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s8, s7, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s9, s7, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s10, s7, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s7, s7
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s11, s6, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s12, s6, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s13, s6, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s6, s6
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s14, s5, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s15, s5, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s16, s5, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s5, s5
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s17, s4, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s18, s4, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s19, s4, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s4, s4
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s9, s9, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s10, s10, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s7, s7, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s11, s11, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s12, s12, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s13, s13, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s6, s6, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s14, s14, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s15, s15, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s16, s16, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s5, s5, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s17, s17, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s18, s18, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s19, s19, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s4, s4, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s8, s9, s8
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s7, s7, s10
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s9, s12, s11
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s10, s15, s14
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s5, s5, s16
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s11, s18, s17
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s4, s4, s19
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s6, s6, s13
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s11
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s10
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s7
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v16i8_to_v16i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v8, s0
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s2, v2
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s3, v3
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s4, v0
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s5, v1
+; GCN-HSA-NEXT:    s_ashr_i32 s6, s5, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s7, s5, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s8, s5, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s5, s5
+; GCN-HSA-NEXT:    s_ashr_i32 s9, s4, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s10, s4, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s11, s4, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s4, s4
+; GCN-HSA-NEXT:    s_ashr_i32 s12, s3, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s13, s3, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s14, s3, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s3, s3
+; GCN-HSA-NEXT:    s_ashr_i32 s15, s2, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s16, s2, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s17, s2, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s2, s2
+; GCN-HSA-NEXT:    s_lshl_b32 s6, s6, 16
+; GCN-HSA-NEXT:    s_and_b32 s7, s7, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-HSA-NEXT:    s_and_b32 s5, s5, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s9, s9, 16
+; GCN-HSA-NEXT:    s_and_b32 s10, s10, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s11, s11, 16
+; GCN-HSA-NEXT:    s_and_b32 s4, s4, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s12, s12, 16
+; GCN-HSA-NEXT:    s_and_b32 s13, s13, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s14, s14, 16
+; GCN-HSA-NEXT:    s_and_b32 s3, s3, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s15, s15, 16
+; GCN-HSA-NEXT:    s_and_b32 s16, s16, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s17, s17, 16
+; GCN-HSA-NEXT:    s_and_b32 s2, s2, 0xffff
+; GCN-HSA-NEXT:    s_or_b32 s6, s7, s6
+; GCN-HSA-NEXT:    s_or_b32 s5, s5, s8
+; GCN-HSA-NEXT:    s_or_b32 s7, s10, s9
+; GCN-HSA-NEXT:    s_or_b32 s4, s4, s11
+; GCN-HSA-NEXT:    s_or_b32 s8, s13, s12
+; GCN-HSA-NEXT:    s_or_b32 s3, s3, s14
+; GCN-HSA-NEXT:    s_or_b32 s9, s16, s15
+; GCN-HSA-NEXT:    s_or_b32 s2, s2, s17
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v6, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v7, s6
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[0:3]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[8:9], v[4:7]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v16i8_to_v16i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v2
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v3
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s6, v0
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s7, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s8, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s9, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s10, s6, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s11, s7, 16
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s12, s7
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s13, s7, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s7, s7, 16
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s16, s5
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s18, s4
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s14, s6
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s15, s6, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s6, s6, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s17, s5, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s19, s4, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s5, s5, 24
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s4, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s7, s7, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s11, s11, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s10, s10, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s16, s16, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s18, s18, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s9, s9, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s8, s8, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s12, s12, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s14, s14, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s6, s6, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s17, 0xffff, s17
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s19, 0xffff, s19
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s5, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s7, s7, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s11, 0xffff, s11
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s10, 0xffff, s10
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s16, s16, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s18, s18, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s9, 0xffff, s9
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s8, 0xffff, s8
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s13, 0xffff, s13
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s15, 0xffff, s15
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s12, s12, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s14, s14, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s7, s11, s7
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s6, s10, s6
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s10, s17, s16
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s11, s19, s18
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s5, s9, s5
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s4, s8, s4
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s12, s13, s12
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s13, s15, s14
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s11
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v4, s13
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, s6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, s12
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, s7
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v16i8_to_v16i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 0 @6
+; EG-NEXT:    ALU 20, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T9.XYZW, T10.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XYZW, T7.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T7.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 9:
+; EG-NEXT:     BFE_INT * T8.Z, T7.Y, 0.0, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T8.X, T7.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T9.Z, T7.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T7.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T9.X, T7.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Z, T7.W, literal.x,
+; EG-NEXT:     BFE_INT T8.W, PV.W, 0.0, literal.x,
+; EG-NEXT:     LSHR * T0.W, T7.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
+; EG-NEXT:     BFE_INT T8.Y, PS, 0.0, literal.y,
+; EG-NEXT:     LSHR T1.Z, T7.Z, literal.y,
+; EG-NEXT:     BFE_INT T9.W, PV.Z, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T10.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T9.Y, PV.Z, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v16i8_to_v16i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 0 @6
+; CM-NEXT:    ALU 19, @9, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T9, T7.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T8, T10.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 6:
+; CM-NEXT:     VTX_READ_128 T7.XYZW, T7.X, 0, #1
+; CM-NEXT:    ALU clause starting at 8:
+; CM-NEXT:     MOV * T7.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 9:
+; CM-NEXT:     BFE_INT * T8.Z, T7.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T8.X, T7.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR T0.Y, T7.Y, literal.x,
+; CM-NEXT:     BFE_INT T9.Z, T7.Y, 0.0, literal.x,
+; CM-NEXT:     LSHR * T0.W, T7.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T9.X, T7.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T1.Y, T7.Z, literal.x,
+; CM-NEXT:     ADD_INT T0.Z, KC0[2].Y, literal.y,
+; CM-NEXT:     BFE_INT * T8.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     LSHR T10.X, PV.Z, literal.x,
+; CM-NEXT:     BFE_INT T8.Y, PV.Y, 0.0, literal.y,
+; CM-NEXT:     LSHR T0.Z, T7.X, literal.y,
+; CM-NEXT:     BFE_INT * T9.W, T0.Y, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T7.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T9.Y, PV.Z, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <16 x i8>, ptr addrspace(1) %in
   %ext = sext <16 x i8> %load to <16 x i16>
   store <16 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_zextload_v32i8_to_v32i16:
-
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1
 define amdgpu_kernel void @global_zextload_v32i8_to_v32i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_zextload_v32i8_to_v32i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff00, v2
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v5, 0xff00, v3
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v6, 0xff, v3
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v7, 0xff, v2
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v6, v6, v5
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v4, v7, v4
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[8:11], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 24, v3
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v3, v5, v3, 16
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v7, 0xff00ff, v3
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v2, v3, v2, 16
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v5, 0xff00ff, v2
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v3, 0xff00, v10
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 24, v10
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v4, 24, v11
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v5, 24, v8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v6, 24, v9
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v7, v6, v9, 16
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v5, v5, v8, 16
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v12, v4, v11, 16
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v13, v2, v10, 16
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v4, 0xff00, v11
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v2, 0xff00, v8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v6, 0xff00, v9
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v8, 0xff, v8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v11, 0xff, v11
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v10, 0xff, v10
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v14, 24, v1
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v14, v14, v1, 16
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v15, 24, v0
+; GCN-NOHSA-SI-NEXT:    v_alignbit_b32 v15, v15, v0, 16
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v16, 0xff00, v0
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v17, 0xff00, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v18, 0xff, v0
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v6
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v2
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v2, v9, v0
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v0, v8, v6
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v6, v11, v4
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v4, v10, v3
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v3, 8, v17
+; GCN-NOHSA-SI-NEXT:    v_lshlrev_b32_e32 v8, 8, v16
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v10, v1, v3
+; GCN-NOHSA-SI-NEXT:    v_or_b32_e32 v8, v18, v8
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v3, 0xff00ff, v7
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v5
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v7, 0xff00ff, v12
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v5, 0xff00ff, v13
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v11, 0xff00ff, v14
+; GCN-NOHSA-SI-NEXT:    v_and_b32_e32 v9, 0xff00ff, v15
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_zextload_v32i8_to_v32i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v13, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v12, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v11, s1
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s3
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s2
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 24, v7
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff00, v7
+; GCN-HSA-NEXT:    v_and_b32_e32 v17, 0xff, v7
+; GCN-HSA-NEXT:    v_alignbit_b32 v7, v9, v7, 16
+; GCN-HSA-NEXT:    v_and_b32_e32 v16, 0xff00, v6
+; GCN-HSA-NEXT:    v_and_b32_e32 v9, 0xff00ff, v7
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v7, 24, v6
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GCN-HSA-NEXT:    v_alignbit_b32 v7, v7, v6, 16
+; GCN-HSA-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v16, 8, v16
+; GCN-HSA-NEXT:    v_or_b32_e32 v8, v17, v8
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff00ff, v7
+; GCN-HSA-NEXT:    v_or_b32_e32 v6, v6, v16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[6:9]
+; GCN-HSA-NEXT:    v_and_b32_e32 v12, 0xff, v5
+; GCN-HSA-NEXT:    v_and_b32_e32 v6, 0xff00, v4
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v7, 24, v4
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff00, v5
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 24, v5
+; GCN-HSA-NEXT:    v_and_b32_e32 v13, 0xff, v4
+; GCN-HSA-NEXT:    v_alignbit_b32 v5, v9, v5, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GCN-HSA-NEXT:    v_alignbit_b32 v9, v7, v4, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v4, 8, v6
+; GCN-HSA-NEXT:    v_or_b32_e32 v6, v12, v8
+; GCN-HSA-NEXT:    v_or_b32_e32 v4, v13, v4
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff00ff, v5
+; GCN-HSA-NEXT:    v_and_b32_e32 v5, 0xff00ff, v9
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_and_b32_e32 v18, 0xff00, v2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v19, 24, v2
+; GCN-HSA-NEXT:    v_and_b32_e32 v8, 0xff00, v3
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v12, 24, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v13, 0xff00, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 24, v0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GCN-HSA-NEXT:    v_and_b32_e32 v10, 0xff, v3
+; GCN-HSA-NEXT:    v_and_b32_e32 v4, 0xff00, v1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v5, 24, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff, v0
+; GCN-HSA-NEXT:    v_and_b32_e32 v11, 0xff, v2
+; GCN-HSA-NEXT:    v_alignbit_b32 v1, v5, v1, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GCN-HSA-NEXT:    v_alignbit_b32 v0, v9, v0, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v5, 8, v13
+; GCN-HSA-NEXT:    v_alignbit_b32 v9, v12, v3, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GCN-HSA-NEXT:    v_alignbit_b32 v12, v19, v2, 16
+; GCN-HSA-NEXT:    v_lshlrev_b32_e32 v13, 8, v18
+; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s1
+; GCN-HSA-NEXT:    v_and_b32_e32 v3, 0xff00ff, v1
+; GCN-HSA-NEXT:    v_or_b32_e32 v2, v6, v4
+; GCN-HSA-NEXT:    v_and_b32_e32 v1, 0xff00ff, v0
+; GCN-HSA-NEXT:    v_or_b32_e32 v0, v7, v5
+; GCN-HSA-NEXT:    v_and_b32_e32 v7, 0xff00ff, v9
+; GCN-HSA-NEXT:    v_or_b32_e32 v6, v10, v8
+; GCN-HSA-NEXT:    v_and_b32_e32 v5, 0xff00ff, v12
+; GCN-HSA-NEXT:    v_or_b32_e32 v4, v11, v13
+; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[4:7]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_zextload_v32i8_to_v32i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s6, v7
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s7, v5
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s8, s7, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s11, s6, 24
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s17, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s18, s4, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s19, s4, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s4, s4, 8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v8, 24, v2
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v5, 8, v4
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v11, 8, v2
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s9, s7, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s10, s7, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s7, s7, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s12, s6, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s13, s6, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s6, s6, 8
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s14, s5, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_u32 s15, s5, 0x80010
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s16, s5, 0xff
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s11, s11, 16
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s17, s17, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s4, s4, 0xff0000
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 24, v6
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 24, v4
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v9, 8, v6
+; GCN-NOHSA-VI-NEXT:    v_lshlrev_b32_e32 v10, 8, v0
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v12, 0xff0000, v5
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v13, v8, v2, 16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v11, 0xff0000, v11
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s7, s7, 0xff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s6, s6, 0xff0000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s14, s14, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s5, s5, 0xff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s8, s9, s8
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s9, s12, s11
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s11, s18, s17
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s4, s19, s4
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v7, v7, v4, 16
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v1, v1, v6, 16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v9, 0xff0000, v9
+; GCN-NOHSA-VI-NEXT:    v_alignbit_b32 v3, v3, v0, 16
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v10, 0xff0000, v10
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v12, v2, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v13, 0xff00ff, v13
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s7, s10, s7
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s6, s13, s6
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s10, s15, s14
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s5, s16, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v14, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v15, s11
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v5, 0xff00ff, v7
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v8, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v9, 0xff00ff, v1
+; GCN-NOHSA-VI-NEXT:    v_and_b32_e32 v1, 0xff00ff, v3
+; GCN-NOHSA-VI-NEXT:    v_or_b32_sdwa v0, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v10, s6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v11, s9
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, s7
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v7, s8
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_zextload_v32i8_to_v32i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @8
+; EG-NEXT:    ALU 37, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T18.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T15.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T14.XYZW, T16.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T13.XYZW, T11.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     MOV * T0.W, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT * T13.W, T11.Y, literal.x, PV.W,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T13.Y, T11.X, literal.x, T0.W,
+; EG-NEXT:     AND_INT T13.Z, T11.Y, literal.y,
+; EG-NEXT:     BFE_UINT * T14.W, T11.W, literal.x, T0.W,
+; EG-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; EG-NEXT:     AND_INT T13.X, T11.X, literal.x,
+; EG-NEXT:     BFE_UINT T14.Y, T11.Z, literal.y, T0.W,
+; EG-NEXT:     LSHR * T11.X, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T14.Z, T11.W, literal.x,
+; EG-NEXT:     BFE_UINT * T15.W, T12.Y, literal.y, T0.W,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:     AND_INT T14.X, T11.Z, literal.x,
+; EG-NEXT:     BFE_UINT T15.Y, T12.X, literal.y, T0.W,
+; EG-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; EG-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T16.X, PV.W, literal.x,
+; EG-NEXT:     AND_INT T15.Z, T12.Y, literal.y,
+; EG-NEXT:     BFE_UINT T17.W, T12.W, literal.z, T0.W,
+; EG-NEXT:     AND_INT * T15.X, T12.X, literal.y,
+; EG-NEXT:    2(2.802597e-45), 255(3.573311e-43)
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_UINT T17.Y, T12.Z, literal.x, T0.W,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T12.X, PV.W, literal.x,
+; EG-NEXT:     AND_INT T17.Z, T12.W, literal.y,
+; EG-NEXT:     AND_INT * T17.X, T12.Z, literal.y,
+; EG-NEXT:    2(2.802597e-45), 255(3.573311e-43)
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR * T18.X, PV.W, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; CM-LABEL: global_zextload_v32i8_to_v32i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @8
+; CM-NEXT:    ALU 39, @13, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T17, T12.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T11, T18.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T14, T16.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T13, T15.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 0, #1
+; CM-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 16, #1
+; CM-NEXT:    ALU clause starting at 12:
+; CM-NEXT:     MOV * T11.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 13:
+; CM-NEXT:     MOV * T0.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT * T13.W, T11.W, literal.x, PV.W,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_UINT T13.Y, T11.Z, literal.x, T0.W,
+; CM-NEXT:     AND_INT T13.Z, T11.W, literal.y,
+; CM-NEXT:     BFE_UINT * T14.W, T11.Y, literal.x, T0.W,
+; CM-NEXT:    8(1.121039e-44), 255(3.573311e-43)
+; CM-NEXT:     AND_INT T13.X, T11.Z, literal.x,
+; CM-NEXT:     BFE_UINT T14.Y, T11.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T15.X, PV.W, literal.x,
+; CM-NEXT:     AND_INT T14.Z, T11.Y, literal.y,
+; CM-NEXT:     BFE_UINT * T11.W, T12.W, literal.z, T0.W,
+; CM-NEXT:    2(2.802597e-45), 255(3.573311e-43)
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T14.X, T11.X, literal.x,
+; CM-NEXT:     BFE_UINT T11.Y, T12.Z, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T1.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T16.X, PV.W, literal.x,
+; CM-NEXT:     AND_INT T11.Z, T12.W, literal.y,
+; CM-NEXT:     BFE_UINT * T17.W, T12.Y, literal.z, T0.W,
+; CM-NEXT:    2(2.802597e-45), 255(3.573311e-43)
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     AND_INT T11.X, T12.Z, literal.x,
+; CM-NEXT:     BFE_UINT T17.Y, T12.X, literal.y, T0.W,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    255(3.573311e-43), 8(1.121039e-44)
+; CM-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T18.X, PV.W, literal.x,
+; CM-NEXT:     AND_INT * T17.Z, T12.Y, literal.y,
+; CM-NEXT:    2(2.802597e-45), 255(3.573311e-43)
+; CM-NEXT:     AND_INT * T17.X, T12.X, literal.x,
+; CM-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; CM-NEXT:     LSHR * T12.X, KC0[2].Y, literal.x,
+; CM-NEXT:    2(2.802597e-45), 0(0.000000e+00)
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = zext <32 x i8> %load to <32 x i16>
   store <32 x i16> %ext, ptr addrspace(1) %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_sextload_v32i8_to_v32i16:
-
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1
-; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
-; EG: BFE_{{U?}}INT
 define amdgpu_kernel void @global_sextload_v32i8_to_v32i16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; GCN-NOHSA-SI-LABEL: global_sextload_v32i8_to_v32i16:
+; GCN-NOHSA-SI:       ; %bb.0:
+; GCN-NOHSA-SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-SI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s4, v2
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s5, v3
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s6, v0
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s7, v1
+; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s8, v6
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s9, v7
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s10, v4
+; GCN-NOHSA-SI-NEXT:    v_readfirstlane_b32 s11, v5
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s12, s11, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s13, s11, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s14, s11, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s11, s11
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s15, s10, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s16, s10, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s17, s10, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s10, s10
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s18, s9, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s19, s9, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s20, s9, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s9, s9
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s21, s8, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s22, s8, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s23, s8, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s8, s8
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s24, s7, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s25, s7, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s26, s7, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s7, s7
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s27, s6, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s28, s6, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s29, s6, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s6, s6
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s30, s5, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s31, s5, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s33, s5, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s5, s5
+; GCN-NOHSA-SI-NEXT:    s_ashr_i32 s34, s4, 24
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s35, s4, 0x80010
+; GCN-NOHSA-SI-NEXT:    s_bfe_i32 s36, s4, 0x80008
+; GCN-NOHSA-SI-NEXT:    s_sext_i32_i8 s4, s4
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s12, s12, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s13, s13, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s14, s14, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s11, s11, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s15, s15, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s16, s16, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s17, s17, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s10, s10, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s18, s18, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s19, s19, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s20, s20, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s9, s9, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s21, s21, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s22, s22, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s23, s23, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s8, s8, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s24, s24, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s25, s25, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s26, s26, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s7, s7, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s27, s27, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s28, s28, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s29, s29, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s6, s6, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s30, s30, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s31, s31, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s33, s33, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s5, s5, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s34, s34, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s35, s35, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_lshl_b32 s36, s36, 16
+; GCN-NOHSA-SI-NEXT:    s_and_b32 s4, s4, 0xffff
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s12, s13, s12
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s11, s11, s14
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s13, s16, s15
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s10, s10, s17
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s14, s19, s18
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s9, s9, s20
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s15, s22, s21
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s8, s8, s23
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s16, s25, s24
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s7, s7, s26
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s17, s28, s27
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s18, s31, s30
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s5, s5, s33
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s19, s35, s34
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s4, s4, s36
+; GCN-NOHSA-SI-NEXT:    s_or_b32 s6, s6, s29
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s19
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s18
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s17
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s7
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s9
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s14
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v2, s11
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v3, s12
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-SI-NEXT:    s_endpgm
+;
+; GCN-HSA-LABEL: global_sextload_v32i8_to_v32i16:
+; GCN-HSA:       ; %bb.0:
+; GCN-HSA-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-HSA-NEXT:    s_add_i32 s12, s12, s17
+; GCN-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GCN-HSA-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-HSA-NEXT:    s_add_u32 s2, s2, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GCN-HSA-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(1)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s2, v2
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s3, v3
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s4, v0
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s5, v1
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s21, v5
+; GCN-HSA-NEXT:    s_ashr_i32 s6, s5, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s7, s5, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s8, s5, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s5, s5
+; GCN-HSA-NEXT:    s_ashr_i32 s9, s4, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s10, s4, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s11, s4, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s4, s4
+; GCN-HSA-NEXT:    s_ashr_i32 s12, s3, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s13, s3, 0x80010
+; GCN-HSA-NEXT:    s_bfe_i32 s14, s3, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s3, s3
+; GCN-HSA-NEXT:    s_ashr_i32 s15, s2, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s16, s2, 0x80010
+; GCN-HSA-NEXT:    s_ashr_i32 s22, s21, 24
+; GCN-HSA-NEXT:    s_bfe_i32 s23, s21, 0x80010
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s18, v6
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s19, v7
+; GCN-HSA-NEXT:    v_readfirstlane_b32 s20, v4
+; GCN-HSA-NEXT:    s_lshl_b32 s6, s6, 16
+; GCN-HSA-NEXT:    s_and_b32 s7, s7, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-HSA-NEXT:    s_and_b32 s5, s5, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s9, s9, 16
+; GCN-HSA-NEXT:    s_and_b32 s10, s10, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s11, s11, 16
+; GCN-HSA-NEXT:    s_and_b32 s4, s4, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s12, s12, 16
+; GCN-HSA-NEXT:    s_and_b32 s13, s13, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s14, s14, 16
+; GCN-HSA-NEXT:    s_and_b32 s3, s3, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s15, s15, 16
+; GCN-HSA-NEXT:    s_and_b32 s16, s16, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s22, s22, 16
+; GCN-HSA-NEXT:    s_and_b32 s23, s23, 0xffff
+; GCN-HSA-NEXT:    s_bfe_i32 s17, s2, 0x80008
+; GCN-HSA-NEXT:    s_bfe_i32 s24, s21, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s21, s21
+; GCN-HSA-NEXT:    s_ashr_i32 s25, s20, 24
+; GCN-HSA-NEXT:    s_or_b32 s6, s7, s6
+; GCN-HSA-NEXT:    s_bfe_i32 s7, s20, 0x80010
+; GCN-HSA-NEXT:    s_or_b32 s5, s5, s8
+; GCN-HSA-NEXT:    s_bfe_i32 s8, s20, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s20, s20
+; GCN-HSA-NEXT:    s_or_b32 s9, s10, s9
+; GCN-HSA-NEXT:    s_ashr_i32 s10, s19, 24
+; GCN-HSA-NEXT:    s_or_b32 s4, s4, s11
+; GCN-HSA-NEXT:    s_bfe_i32 s11, s19, 0x80010
+; GCN-HSA-NEXT:    s_or_b32 s12, s13, s12
+; GCN-HSA-NEXT:    s_bfe_i32 s13, s19, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s19, s19
+; GCN-HSA-NEXT:    s_or_b32 s3, s3, s14
+; GCN-HSA-NEXT:    s_ashr_i32 s14, s18, 24
+; GCN-HSA-NEXT:    s_or_b32 s15, s16, s15
+; GCN-HSA-NEXT:    s_bfe_i32 s16, s18, 0x80010
+; GCN-HSA-NEXT:    s_or_b32 s22, s23, s22
+; GCN-HSA-NEXT:    s_bfe_i32 s23, s18, 0x80008
+; GCN-HSA-NEXT:    s_sext_i32_i8 s18, s18
+; GCN-HSA-NEXT:    s_sext_i32_i8 s2, s2
+; GCN-HSA-NEXT:    s_lshl_b32 s17, s17, 16
+; GCN-HSA-NEXT:    s_lshl_b32 s24, s24, 16
+; GCN-HSA-NEXT:    s_and_b32 s21, s21, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s25, s25, 16
+; GCN-HSA-NEXT:    s_and_b32 s7, s7, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-HSA-NEXT:    s_and_b32 s20, s20, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s10, s10, 16
+; GCN-HSA-NEXT:    s_and_b32 s11, s11, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s13, s13, 16
+; GCN-HSA-NEXT:    s_and_b32 s19, s19, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s14, s14, 16
+; GCN-HSA-NEXT:    s_and_b32 s16, s16, 0xffff
+; GCN-HSA-NEXT:    s_lshl_b32 s23, s23, 16
+; GCN-HSA-NEXT:    s_and_b32 s18, s18, 0xffff
+; GCN-HSA-NEXT:    s_and_b32 s2, s2, 0xffff
+; GCN-HSA-NEXT:    s_or_b32 s21, s21, s24
+; GCN-HSA-NEXT:    s_or_b32 s7, s7, s25
+; GCN-HSA-NEXT:    s_or_b32 s8, s20, s8
+; GCN-HSA-NEXT:    s_or_b32 s10, s11, s10
+; GCN-HSA-NEXT:    s_or_b32 s11, s19, s13
+; GCN-HSA-NEXT:    s_or_b32 s13, s16, s14
+; GCN-HSA-NEXT:    s_or_b32 s14, s18, s23
+; GCN-HSA-NEXT:    s_or_b32 s2, s2, s17
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s15
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s12
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s9
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s6
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-HSA-NEXT:    s_add_u32 s0, s0, 32
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s14
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s13
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s11
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s10
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s2
+; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s8
+; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v2, s21
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, s22
+; GCN-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-HSA-NEXT:    s_endpgm
+;
+; GCN-NOHSA-VI-LABEL: global_sextload_v32i8_to_v32i16:
+; GCN-NOHSA-VI:       ; %bb.0:
+; GCN-NOHSA-VI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s10, s2
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
+; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s6, v0
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s7, v1
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s14, s6, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s15, s7, 16
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s30, s6
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s31, s6, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s6, s6, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s14, s14, 0x80000
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s5, v3
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s28, s7
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s29, s7, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s7, s7, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s15, s15, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s6, s6, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s14, 0xffff, s14
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s10, v4
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s11, v5
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s7, s7, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s15, 0xffff, s15
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s6, s14, s6
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s14, s5
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s20, s11
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s22, s10
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s7, s15, s7
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s14, s14, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s15, s5, 0x80000
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s4, v2
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s21, s11, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s23, s10, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s20, s20, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s22, s22, 8
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s14, s14, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s15, 0xffff, s15
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s8, v6
+; GCN-NOHSA-VI-NEXT:    v_readfirstlane_b32 s9, v7
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s12, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s13, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s21, 0xffff, s21
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s23, 0xffff, s23
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s20, s20, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s22, s22, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s14, s15, s14
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s15, s4
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s5, s5, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s16, s8, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s17, s9, 16
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s24, s9
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s25, s9, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s9, s9, 16
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s20, s21, s20
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s21, s23, s22
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s15, s15, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s22, s4, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s5, s5, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s13, s13, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s4, s4, 24
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s12, s12, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s18, s10, 16
+; GCN-NOHSA-VI-NEXT:    s_lshr_b32 s19, s11, 16
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s11, s11, 16
+; GCN-NOHSA-VI-NEXT:    s_sext_i32_i16 s26, s8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s27, s8, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s8, s8, 24
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s9, s9, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s17, s17, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s16, s16, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s28, s28, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s30, s30, 8
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s15, s15, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s22, 0xffff, s22
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s5, s5, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s13, 0xffff, s13
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s4, s4, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s12, 0xffff, s12
+; GCN-NOHSA-VI-NEXT:    s_ashr_i32 s10, s10, 24
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s11, s11, 8
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s19, s19, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_bfe_i32 s18, s18, 0x80000
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s24, s24, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s26, s26, 8
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s8, s8, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s29, 0xffff, s29
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s31, 0xffff, s31
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s9, s9, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s17, 0xffff, s17
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s16, 0xffff, s16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s28, s28, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s30, s30, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s15, s22, s15
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s5, s13, s5
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s4, s12, s4
+; GCN-NOHSA-VI-NEXT:    s_lshl_b32 s10, s10, 16
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s25, 0xffff, s25
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s27, 0xffff, s27
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s11, s11, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s19, 0xffff, s19
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s18, 0xffff, s18
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s24, s24, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_and_b32 s26, s26, 0xffff0000
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s9, s17, s9
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s8, s16, s8
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s16, s29, s28
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s17, s31, s30
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s15
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s14
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s11, s19, s11
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s10, s18, s10
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s18, s25, s24
+; GCN-NOHSA-VI-NEXT:    s_or_b32 s19, s27, s26
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    s_nop 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s17
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    s_nop 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s19
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s8
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s18
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s9
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    s_nop 0
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v0, s21
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, s10
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, s20
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    s_endpgm
+;
+; EG-LABEL: global_sextload_v32i8_to_v32i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @8
+; EG-NEXT:    ALU 39, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T18.XYZW, T12.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T17.XYZW, T11.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T16.XYZW, T14.X, 0
+; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T15.XYZW, T13.X, 1
+; EG-NEXT:    CF_END
+; EG-NEXT:    Fetch clause starting at 8:
+; EG-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; EG-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; EG-NEXT:    ALU clause starting at 12:
+; EG-NEXT:     MOV * T11.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 13:
+; EG-NEXT:     LSHR T13.X, KC0[2].Y, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    2(2.802597e-45), 16(2.242078e-44)
+; EG-NEXT:     LSHR T14.X, PV.W, literal.x,
+; EG-NEXT:     BFE_INT * T15.Z, T11.Y, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:     BFE_INT T15.X, T11.X, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Y, T12.W, literal.x,
+; EG-NEXT:     BFE_INT T16.Z, T11.W, 0.0, literal.x, BS:VEC_120/SCL_212
+; EG-NEXT:     LSHR T0.W, T12.Y, literal.x,
+; EG-NEXT:     LSHR * T1.W, T11.Y, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T16.X, T11.Z, 0.0, literal.x,
+; EG-NEXT:     LSHR T1.Y, T11.W, literal.x,
+; EG-NEXT:     BFE_INT T17.Z, T12.Y, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T15.W, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.W, T11.X, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T17.X, T12.X, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T15.Y, PS, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T18.Z, T12.W, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T16.W, PV.Y, 0.0, literal.x,
+; EG-NEXT:     LSHR * T1.W, T11.Z, literal.x,
+; EG-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT:     BFE_INT T18.X, T12.Z, 0.0, literal.x,
+; EG-NEXT:     BFE_INT T16.Y, PS, 0.0, literal.x,
+; EG-NEXT:     LSHR T0.Z, T12.X, literal.x,
+; EG-NEXT:     BFE_INT T17.W, T0.W, 0.0, literal.x,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.y,
+; EG-NEXT:    8(1.121039e-44), 32(4.484155e-44)
+; EG-NEXT:     LSHR T11.X, PS, literal.x,
+; EG-NEXT:     BFE_INT T17.Y, PV.Z, 0.0, literal.y,
+; EG-NEXT:     LSHR T0.Z, T12.Z, literal.y,
+; EG-NEXT:     BFE_INT T18.W, T0.Y, 0.0, literal.y,
+; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; EG-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; EG-NEXT:     LSHR T12.X, PS, literal.x,
+; EG-NEXT:     BFE_INT * T18.Y, PV.Z, 0.0, literal.y,
+; EG-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+;
+; CM-LABEL: global_sextload_v32i8_to_v32i16:
+; CM:       ; %bb.0:
+; CM-NEXT:    ALU 0, @12, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    TEX 1 @8
+; CM-NEXT:    ALU 40, @13, KC0[CB0:0-32], KC1[]
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T17, T11.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T12, T18.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T16, T14.X
+; CM-NEXT:    MEM_RAT_CACHELESS STORE_DWORD T15, T13.X
+; CM-NEXT:    CF_END
+; CM-NEXT:    Fetch clause starting at 8:
+; CM-NEXT:     VTX_READ_128 T12.XYZW, T11.X, 16, #1
+; CM-NEXT:     VTX_READ_128 T11.XYZW, T11.X, 0, #1
+; CM-NEXT:    ALU clause starting at 12:
+; CM-NEXT:     MOV * T11.X, KC0[2].Z,
+; CM-NEXT:    ALU clause starting at 13:
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.x,
+; CM-NEXT:    48(6.726233e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T13.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T0.Y, T11.Y, literal.y,
+; CM-NEXT:     LSHR T0.Z, T11.Z, literal.y,
+; CM-NEXT:     ADD_INT * T0.W, KC0[2].Y, literal.z,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:    32(4.484155e-44), 0(0.000000e+00)
+; CM-NEXT:     LSHR T14.X, PV.W, literal.x,
+; CM-NEXT:     LSHR T1.Y, T11.W, literal.y,
+; CM-NEXT:     BFE_INT T15.Z, T12.W, 0.0, literal.y, BS:VEC_120/SCL_212
+; CM-NEXT:     LSHR * T0.W, T12.X, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     BFE_INT T15.X, T12.Z, 0.0, literal.x,
+; CM-NEXT:     LSHR T2.Y, T12.Y, literal.x,
+; CM-NEXT:     BFE_INT T16.Z, T12.Y, 0.0, literal.x,
+; CM-NEXT:     LSHR * T1.W, T12.W, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T16.X, T12.X, 0.0, literal.x,
+; CM-NEXT:     LSHR T3.Y, T12.Z, literal.x,
+; CM-NEXT:     BFE_INT T12.Z, T11.W, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T15.W, PV.W, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T12.X, T11.Z, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T15.Y, PV.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T17.Z, T11.Y, 0.0, literal.x,
+; CM-NEXT:     BFE_INT * T16.W, T2.Y, 0.0, literal.x, BS:VEC_120/SCL_212
+; CM-NEXT:    8(1.121039e-44), 0(0.000000e+00)
+; CM-NEXT:     BFE_INT T17.X, T11.X, 0.0, literal.x,
+; CM-NEXT:     BFE_INT T16.Y, T0.W, 0.0, literal.x,
+; CM-NEXT:     ADD_INT T1.Z, KC0[2].Y, literal.y,
+; CM-NEXT:     BFE_INT * T12.W, T1.Y, 0.0, literal.x,
+; CM-NEXT:    8(1.121039e-44), 16(2.242078e-44)
+; CM-NEXT:     LSHR T18.X, PV.Z, literal.x,
+; CM-NEXT:     BFE_INT T12.Y, T0.Z, 0.0, literal.y,
+; CM-NEXT:     LSHR T0.Z, T11.X, literal.y,
+; CM-NEXT:     BFE_INT * T17.W, T0.Y, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
+; CM-NEXT:     LSHR T11.X, KC0[2].Y, literal.x,
+; CM-NEXT:     BFE_INT * T17.Y, PV.Z, 0.0, literal.y,
+; CM-NEXT:    2(2.802597e-45), 8(1.121039e-44)
   %load = load <32 x i8>, ptr addrspace(1) %in
   %ext = sext <32 x i8> %load to <32 x i16>
   store <32 x i16> %ext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/mad-combine.ll b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
index e94aa4b8ce3d1..7b8aa86bee2cc 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
@@ -1,14 +1,11 @@
 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
 
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD  -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
-
-; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
 
 ; Make sure we don't form mad with denormals
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare float @llvm.fabs.f32(float) #0
@@ -25,15 +22,41 @@ declare float @llvm.fmuladd.f32(float, float, float) #0
 
 ; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
 
+; SI-DENORM-FASTFMAF: buffer_store_dword [[RESULT]]
+; SI-STD: buffer_store_dword [[C]]
+define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr float, ptr addrspace(1) %gep.0, i32 2
+  %gep.out = getelementptr float, ptr addrspace(1) %out, i32 %tid
+
+  %a = load volatile float, ptr addrspace(1) %gep.0
+  %b = load volatile float, ptr addrspace(1) %gep.1
+  %c = load volatile float, ptr addrspace(1) %gep.2
+
+
+  %mul = fmul contract float %a, %b
+  %fma = fadd contract float %mul, %c
+  store float %fma, ptr addrspace(1) %gep.out
+  ret void
+}
+; FUNC-LABEL: {{^}}no_combine_to_mad_f32_0:
+; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
+; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
+; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
+
+; SI-STD: v_mac_f32_e32 [[C]], [[A]], [[B]]
+
 ; SI-DENORM-SLOWFMAF-NOT: v_fma
 ; SI-DENORM-SLOWFMAF-NOT: v_mad
 
 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]],  [[TMP]], [[C]]
 
-; SI-DENORM: buffer_store_dword [[RESULT]]
+; SI-DENORM-SLOWFMAF: buffer_store_dword [[RESULT]]
 ; SI-STD: buffer_store_dword [[C]]
-define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @no_combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -72,7 +95,46 @@ define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, p
 ; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI: s_endpgm
-define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_fast) #1 {
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr float, ptr addrspace(1) %gep.0, i32 2
+  %gep.3 = getelementptr float, ptr addrspace(1) %gep.0, i32 3
+  %gep.out.0 = getelementptr float, ptr addrspace(1) %out, i32 %tid
+  %gep.out.1 = getelementptr float, ptr addrspace(1) %gep.out.0, i32 1
+
+  %a = load volatile float, ptr addrspace(1) %gep.0
+  %b = load volatile float, ptr addrspace(1) %gep.1
+  %c = load volatile float, ptr addrspace(1) %gep.2
+  %d = load volatile float, ptr addrspace(1) %gep.3
+
+  %mul = fmul contract fast float %a, %b
+  %fma0 = fadd contract fast float %mul, %c
+  %fma1 = fadd contract fast float %mul, %d
+  store volatile float %fma0, ptr addrspace(1) %gep.out.0
+  store volatile float %fma1, ptr addrspace(1) %gep.out.1
+  ret void
+}
+; FUNC-LABEL: {{^}}no_combine_to_mad_f32_0_2use:
+; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
+; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
+; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
+; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
+
+; SI-STD-DAG: v_mac_f32_e32 [[C]], [[A]], [[B]]
+; SI-STD-DAG: v_mac_f32_e32 [[D]], [[A]], [[B]]
+
+; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
+; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
+; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
+
+; SI-DENORM-SLOWFMAF-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-DENORM-SLOWFMAF-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
+; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
+; SI: s_endpgm
+define amdgpu_kernel void @no_combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_fast) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -89,7 +151,6 @@ define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %o
   %mul = fmul float %a, %b
   %fma0 = fadd float %mul, %c
   %fma1 = fadd float %mul, %d
-
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -120,8 +181,8 @@ define amdgpu_kernel void @combine_to_mad_f32_1(ptr addrspace(1) noalias %out, p
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %fma = fadd float %c, %mul
+  %mul = fmul contract float %a, %b
+  %fma = fadd contract float %c, %mul
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -150,8 +211,8 @@ define amdgpu_kernel void @combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %o
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %fma = fsub float %mul, %c
+  %mul = fmul contract float %a, %b
+  %fma = fsub contract float %mul, %c
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -190,9 +251,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_0_f32_2use(ptr addrspace(1) noali
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %fma0 = fsub float %mul, %c
-  %fma1 = fsub float %mul, %d
+  %mul = fmul contract float %a, %b
+  %fma0 = fsub contract float %mul, %c
+  %fma1 = fsub contract float %mul, %d
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -222,8 +283,8 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32(ptr addrspace(1) noalias %o
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %fma = fsub float %c, %mul
+  %mul = fmul contract float %a, %b
+  %fma = fsub contract float %c, %mul
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -262,9 +323,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32_2use(ptr addrspace(1) noali
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %fma0 = fsub float %c, %mul
-  %fma1 = fsub float %d, %mul
+  %mul = fmul contract float %a, %b
+  %fma0 = fsub contract float %c, %mul
+  %fma1 = fsub contract float %d, %mul
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -295,9 +356,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %o
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %mul.neg = fneg float %mul
-  %fma = fsub float %mul.neg, %c
+  %mul = fmul contract float %a, %b
+  %mul.neg = fneg contract float %mul
+  %fma = fsub contract float %mul.neg, %c
 
   store float %fma, ptr addrspace(1) %gep.out
   ret void
@@ -337,10 +398,10 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(ptr addrspace(1)
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %mul.neg = fneg float %mul
-  %fma0 = fsub float %mul.neg, %c
-  %fma1 = fsub float %mul.neg, %d
+  %mul = fmul contract float %a, %b
+  %mul.neg = fneg contract float %mul
+  %fma0 = fsub contract float %mul.neg, %c
+  %fma1 = fsub contract float %mul.neg, %d
 
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
@@ -381,10 +442,10 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(ptr addrspace(1)
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %mul.neg = fneg float %mul
-  %fma0 = fsub float %mul.neg, %c
-  %fma1 = fsub float %mul, %d
+  %mul = fmul contract float %a, %b
+  %mul.neg = fneg contract float %mul
+  %fma0 = fsub contract float %mul.neg, %c
+  %fma1 = fsub contract float %mul, %d
 
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
@@ -412,7 +473,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(ptr addrspace(1)
 ; SI-DENORM: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[C]]
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -427,10 +488,22 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
   %u = load volatile float, ptr addrspace(1) %gep.3
   %v = load volatile float, ptr addrspace(1) %gep.4
 
-  %tmp0 = fmul float %u, %v
-  %tmp1 = call float @llvm.fma.f32(float %x, float %y, float %tmp0) #0
-  %tmp2 = fsub float %tmp1, %z
+  br i1 %is_aggressive, label %aggressive, label %normal
+
+normal:
+  %tmp0_normal = fmul float %u, %v
+  %tmp1_normal = call float @llvm.fma.f32(float %x, float %y, float %tmp0_normal) #0
+  %tmp2_normal = fsub float %tmp1_normal, %z
+  br label %exit
 
+aggressive:
+  %tmp0_aggressive = fmul contract reassoc float %u, %v
+  %tmp1_aggressive = call contract reassoc float @llvm.fma.f32(float %x, float %y, float %tmp0_aggressive) #0
+  %tmp2_aggressive = fsub contract reassoc float %tmp1_aggressive, %z
+  br label %exit
+
+exit:
+  %tmp2 = phi float [%tmp2_normal, %normal], [%tmp2_aggressive, %aggressive]
   store float %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
@@ -505,7 +578,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_1_f32(ptr addrspace(1)
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: s_endpgm
-define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -520,10 +593,22 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
   %u = load volatile float, ptr addrspace(1) %gep.3
   %v = load volatile float, ptr addrspace(1) %gep.4
 
-  %tmp0 = fmul float %u, %v
-  %tmp1 = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0) #0
-  %tmp2 = fsub float %tmp1, %z
+  br i1 %is_aggressive, label %aggressive, label %normal
 
+normal:
+  %tmp0_normal = fmul float %u, %v
+  %tmp1_normal = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0_normal) #0
+  %tmp2_normal = fsub float %tmp1_normal, %z
+  br label %exit
+
+aggressive:
+  %tmp0_aggressive = fmul contract reassoc float %u, %v
+  %tmp1_aggressive = call contract reassoc float @llvm.fmuladd.f32(float %x, float %y, float %tmp0_aggressive) #0
+  %tmp2_aggressive = fsub contract reassoc float %tmp1_aggressive, %z
+  br label %exit
+
+exit:
+  %tmp2 = phi float [%tmp2_normal, %normal], [%tmp2_aggressive, %aggressive]
   store float %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
@@ -556,7 +641,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: s_endpgm
-define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -571,11 +656,23 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1)
   %u = load volatile float, ptr addrspace(1) %gep.3
   %v = load volatile float, ptr addrspace(1) %gep.4
 
-  ; nsz flag is needed since this combine may change sign of zero
-  %tmp0 = fmul nsz float %u, %v
-  %tmp1 = call nsz float @llvm.fmuladd.f32(float %y, float %z, float %tmp0) #0
-  %tmp2 = fsub nsz float %x, %tmp1
+  br i1 %is_aggressive, label %aggressive, label %normal
 
+normal:
+  ; nsz flag is needed since this combine may change sign of zero
+  %tmp0_normal = fmul nsz float %u, %v
+  %tmp1_normal = call nsz float @llvm.fmuladd.f32(float %y, float %z, float %tmp0_normal) #0
+  %tmp2_normal = fsub nsz float %x, %tmp1_normal
+  br label %exit
+
+aggressive:
+  %tmp0_aggressive = fmul contract reassoc nsz float %u, %v
+  %tmp1_aggressive = call contract reassoc nsz float @llvm.fmuladd.f32(float %y, float %z, float %tmp0_aggressive) #0
+  %tmp2_aggressive = fsub contract reassoc nsz float %x, %tmp1_aggressive
+  br label %exit
+
+exit:
+  %tmp2 = phi float [%tmp2_normal, %normal], [%tmp2_aggressive, %aggressive]
   store float %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
index 8b600c835a160..74a72e04fa4ae 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -7367,7 +7367,6 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7918,7 +7917,6 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8215,7 +8213,6 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8505,7 +8502,6 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8777,7 +8773,6 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9052,7 +9047,6 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9349,7 +9343,6 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9646,7 +9639,6 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9943,7 +9935,6 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -10240,7 +10231,6 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -10533,7 +10523,6 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -10830,7 +10819,6 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -11127,7 +11115,6 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18653,7 +18640,6 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18928,7 +18914,6 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19225,7 +19210,6 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19515,7 +19499,6 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19787,7 +19770,6 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20062,7 +20044,6 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20359,7 +20340,6 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20656,7 +20636,6 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20953,7 +20932,6 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21250,7 +21228,6 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21543,7 +21520,6 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21840,7 +21816,6 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -22137,7 +22112,6 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
index 9c11781da56f2..be148464c156e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -6365,7 +6365,6 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -6642,7 +6641,6 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -6945,7 +6943,6 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7241,7 +7238,6 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7515,7 +7511,6 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7792,7 +7787,6 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8095,7 +8089,6 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8398,7 +8391,6 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8701,7 +8693,6 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9004,7 +8995,6 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9303,7 +9293,6 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9606,7 +9595,6 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9909,7 +9897,6 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -17555,7 +17542,6 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18112,7 +18098,6 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18415,7 +18400,6 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18711,7 +18695,6 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18985,7 +18968,6 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19262,7 +19244,6 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19565,7 +19546,6 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19868,7 +19848,6 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20171,7 +20150,6 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20474,7 +20452,6 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20773,7 +20750,6 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21076,7 +21052,6 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21379,7 +21354,6 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local.mir
index adaee7ebaddd3..56dd95e373dc6 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local.mir
@@ -1,18 +1,23 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=amdgcn -mcpu=gfx803 -run-pass si-memory-legalizer %s -o - | FileCheck -check-prefix=GCN %s
 
 ---
 
-# GCN-LABEL: name: load_singlethread_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") unordered (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -26,17 +31,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_singlethread_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") monotonic (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -50,17 +59,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_singlethread_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") acquire (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -74,17 +87,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_singlethread_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") seq_cst (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -98,17 +115,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") unordered (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -122,17 +143,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") monotonic (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -146,17 +171,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") acquire (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -170,17 +199,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") seq_cst (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -194,17 +227,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") unordered (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -218,17 +255,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") monotonic (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -242,17 +283,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") acquire (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -266,17 +311,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") seq_cst (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -290,17 +339,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") unordered (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -314,17 +367,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") monotonic (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -338,17 +395,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") acquire (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -362,17 +423,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") seq_cst (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -386,17 +451,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") unordered (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -410,17 +479,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") monotonic (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -434,17 +507,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") acquire (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -458,17 +535,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") seq_cst (s32) from `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -482,17 +563,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") unordered (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -504,17 +587,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") monotonic (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -526,17 +611,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") release (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -548,17 +635,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") seq_cst (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -570,17 +659,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") unordered (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -592,17 +683,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") monotonic (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -614,17 +707,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") release (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -636,17 +731,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") seq_cst (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -658,17 +755,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") unordered (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -680,17 +779,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") monotonic (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -702,17 +803,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") release (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -724,17 +827,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") seq_cst (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -746,17 +851,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") unordered (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -768,17 +875,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") monotonic (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -790,17 +899,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") release (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -812,17 +923,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") seq_cst (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -834,17 +947,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("one-as") unordered (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -856,17 +971,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("one-as") monotonic (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -878,17 +995,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("one-as") release (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -900,17 +1019,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("one-as") seq_cst (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -922,17 +1043,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") unordered (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -944,17 +1067,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") monotonic (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -966,17 +1091,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") acquire (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -988,17 +1115,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") release (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -1010,17 +1139,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_acq_rel
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_acq_rel
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_acq_rel
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") acq_rel (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -1032,17 +1163,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") seq_cst (s32) into `ptr addrspace(3) poison`, addrspace 3)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-region.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-region.mir
index 9405c8a946627..36a244f6250db 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-region.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-region.mir
@@ -1,18 +1,23 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=amdgcn -mcpu=gfx803 -run-pass si-memory-legalizer %s -o - | FileCheck -check-prefix=GCN %s
 
 ---
 
-# GCN-LABEL: name: load_singlethread_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 1, 0, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") unordered (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -26,17 +31,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_singlethread_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") monotonic (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -50,17 +59,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_singlethread_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") acquire (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -74,17 +87,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_singlethread_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_singlethread_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_singlethread_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("singlethread-one-as") seq_cst (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -98,17 +115,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") unordered (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -122,17 +143,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") monotonic (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -146,17 +171,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") acquire (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -170,17 +199,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_wavefront_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_wavefront_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_wavefront_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("wavefront-one-as") seq_cst (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -194,17 +227,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") unordered (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -218,17 +255,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") monotonic (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -242,17 +283,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") acquire (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -266,17 +311,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_workgroup_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_workgroup_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_workgroup_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("workgroup-one-as") seq_cst (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -290,17 +339,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") unordered (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -314,17 +367,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") monotonic (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -338,17 +395,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") acquire (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -362,17 +423,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_agent_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_agent_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_agent_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("agent-one-as") seq_cst (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -386,17 +451,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") unordered (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -410,17 +479,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") monotonic (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -434,17 +507,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") acquire (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -458,17 +535,21 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: load_system_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_READ_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       FLAT_STORE_DWORD
 
 name:            load_system_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: load_system_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: renamable $vgpr2 = DS_READ_B32 killed renamable $vgpr0, 0, 1, implicit $m0, implicit $exec :: (volatile load syncscope("one-as") seq_cst (s32) from `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+    ; GCN-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 44, 0 :: (dereferenceable invariant load (s64) from `ptr addrspace(4) poison`, align 4, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -482,17 +563,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") unordered (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -504,17 +587,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") monotonic (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -526,17 +611,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") release (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -548,17 +635,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_singlethread_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_singlethread_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_singlethread_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") seq_cst (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -570,17 +659,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") unordered (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -592,17 +683,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") monotonic (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -614,17 +707,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") release (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -636,17 +731,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_wavefront_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_wavefront_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_wavefront_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("wavefront-one-as") seq_cst (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -658,17 +755,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") unordered (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -680,17 +779,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") monotonic (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -702,17 +803,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") release (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -724,17 +827,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_workgroup_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_workgroup_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_workgroup_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("workgroup-one-as") seq_cst (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -746,17 +851,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") unordered (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -768,17 +875,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") monotonic (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -790,17 +899,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") release (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -812,17 +923,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_agent_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_agent_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_agent_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("agent-one-as") seq_cst (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -834,17 +947,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store unordered (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -856,17 +971,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store monotonic (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -878,17 +995,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("one-as") release (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -900,17 +1019,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: store_system_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRITE_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            store_system_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: store_system_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: DS_WRITE_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("one-as") seq_cst (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -922,17 +1043,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_unordered
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_unordered
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_unordered
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") unordered (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -944,17 +1067,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_monotonic
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_monotonic
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_monotonic
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") monotonic (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -966,17 +1091,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_acquire
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_acquire
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_acquire
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") acquire (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -988,17 +1115,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_release
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_release
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_release
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") release (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -1010,17 +1139,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_acq_rel
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_acq_rel
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_acq_rel
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") acq_rel (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
@@ -1032,17 +1163,19 @@ body:             |
 ...
 ---
 
-# GCN-LABEL: name: atomicrmw_singlethread_seq_cst
 
-# GCN-LABEL: bb.0:
-# GCN-NOT:   S_WAITCNT
-# GCN:       DS_WRXCHG_RTN_B32
-# GCN-NOT:   S_WAITCNT
-# GCN:       S_ENDPGM 0
 
 name:            atomicrmw_singlethread_seq_cst
 body:             |
   bb.0:
+    ; GCN-LABEL: name: atomicrmw_singlethread_seq_cst
+    ; GCN: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
+    ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
+    ; GCN-NEXT: $m0 = S_MOV_B32 -1
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr2 = DS_WRXCHG_RTN_B32 killed renamable $vgpr0, killed renamable $vgpr1, 0, 1, implicit $m0, implicit $exec :: (volatile store syncscope("singlethread-one-as") seq_cst (s32) into `ptr addrspace(2) poison`, addrspace 2)
+    ; GCN-NEXT: S_ENDPGM 0
     $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, addrspace 4)
     $sgpr0 = S_LOAD_DWORD_IMM killed $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s32) from `ptr addrspace(4) poison`, align 8, addrspace 4)
     $m0 = S_MOV_B32 -1
diff --git a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
index 34260c49ff92c..944951d3a536a 100644
--- a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
@@ -31,7 +31,6 @@ define amdgpu_kernel void @simple_barrier(ptr addrspace(1) %arg) {
 ; GCN-NEXT:    s_load_dword s2, s[0:1], 0x0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_barrier
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    ; wave barrier
 ; GCN-NEXT:    s_load_dword s3, s[0:1], 0x4
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index bef38c1a65ef8..0e1e5e4c4987c 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -1982,9 +1982,7 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
 ; PACKED-GISEL-NEXT:    ds_read_b32 v5, v5 offset:8
 ; PACKED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[4:5], 1.0, v[4:5] op_sel_hi:[0,1]
-; PACKED-GISEL-NEXT:    v_xor_b32_e32 v4, 0x80000000, v4
-; PACKED-GISEL-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
-; PACKED-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5]
+; PACKED-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1] neg_hi:[0,0,1]
 ; PACKED-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; PACKED-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-GISEL-NEXT:    s_endpgm
@@ -2046,9 +2044,7 @@ define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(
 ; PACKED-GISEL-NEXT:    ds_read_b64 v[2:3], v2 offset:8
 ; PACKED-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; PACKED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; PACKED-GISEL-NEXT:    v_mov_b32_e32 v4, v3
-; PACKED-GISEL-NEXT:    v_mov_b32_e32 v5, v2
-; PACKED-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; PACKED-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
 ; PACKED-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; PACKED-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-GISEL-NEXT:    s_endpgm
@@ -2110,12 +2106,8 @@ define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrsp
 ; PACKED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; PACKED-GISEL-NEXT:    ds_read_b64 v[2:3], v2 offset:8
 ; PACKED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; PACKED-GISEL-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; PACKED-GISEL-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[2:3], 1.0, v[2:3] op_sel_hi:[0,1]
-; PACKED-GISEL-NEXT:    v_mov_b32_e32 v4, v3
-; PACKED-GISEL-NEXT:    v_mov_b32_e32 v5, v2
-; PACKED-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[2:3], 1.0, v[2:3] op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
+; PACKED-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
 ; PACKED-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; PACKED-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-GISEL-NEXT:    s_endpgm
@@ -2351,9 +2343,7 @@ define amdgpu_kernel void @fneg_v2f32_vec(ptr addrspace(1) %a) {
 ; PACKED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; PACKED-GISEL-NEXT:    global_load_dwordx2 v[0:1], v2, s[0:1]
 ; PACKED-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; PACKED-GISEL-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; PACKED-GISEL-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[0:1], 1.0, v[0:1] op_sel_hi:[0,1]
+; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[0:1], 1.0, v[0:1] op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
 ; PACKED-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -2394,9 +2384,7 @@ define amdgpu_kernel void @fneg_v2f32_scalar(ptr addrspace(1) %a, <2 x float> %x
 ; PACKED-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; PACKED-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; PACKED-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; PACKED-GISEL-NEXT:    s_xor_b32 s2, s2, 0x80000000
-; PACKED-GISEL-NEXT:    s_xor_b32 s3, s3, 0x80000000
-; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[0:1], 1.0, s[2:3] op_sel_hi:[0,1]
+; PACKED-GISEL-NEXT:    v_pk_mul_f32 v[0:1], 1.0, s[2:3] op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
 ; PACKED-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-GISEL-NEXT:    s_endpgm
   %fneg = fsub <2 x float> <float -0.0, float -0.0>, %x
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index e7343c150ea70..b1e304e79348e 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -773,7 +773,6 @@ define amdgpu_kernel void @mix_elt_types_op_sel(ptr addrspace(1) %out, ptr addrs
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; kill: killed $vgpr0_vgpr1
 ; GCN-NEXT:    v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0]
 ; GCN-NEXT:    global_store_dword v3, v0, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 24677b60be6c2..663fd98b46bf7 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -6,6 +6,7 @@
 
 ; Partial reg copy and spill missed during regalloc handled later at frame lowering.
 define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
+
   ; REGALLOC-GFX908-LABEL: name: partial_copy
   ; REGALLOC-GFX908: bb.0 (%ir-block.0):
   ; REGALLOC-GFX908-NEXT:   liveins: $sgpr4_sgpr5
@@ -13,7 +14,6 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, undef %6:agpr_32
   ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %7
   ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3538954 /* regdef:VReg_64 */, def %8
-
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
   ; REGALLOC-GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
@@ -21,8 +21,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
   ; REGALLOC-GFX908-NEXT:   [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
   ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
-  ; REGALLOC-GFX908-NEXT:   [[COPY3:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
-  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY3]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+  ; REGALLOC-GFX908-NEXT:   [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
+  ; REGALLOC-GFX908-NEXT:   GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
   ; REGALLOC-GFX908-NEXT:   S_ENDPGM 0
   ;
   ; PEI-GFX908-LABEL: name: partial_copy
diff --git a/llvm/test/CodeGen/AMDGPU/reduction.ll b/llvm/test/CodeGen/AMDGPU/reduction.ll
index cd4b3c1d1b546..7f9044ae164d5 100644
--- a/llvm/test/CodeGen/AMDGPU/reduction.ll
+++ b/llvm/test/CodeGen/AMDGPU/reduction.ll
@@ -1,14 +1,22 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
-; RUN: llc -mtriple=amdgcn -mcpu=fiji  -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=fiji  -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
 
-; GCN-LABEL: {{^}}reduction_fadd_v4f16:
-; GFX9:      v_pk_add_f16 [[ADD:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_add_f16_sdwa v{{[0-9]+}}, [[ADD]], [[ADD]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_add_f16_sdwa
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
 define half @reduction_fadd_v4f16(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_fadd_v4f16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_fadd_v4f16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_e32 v0, v0, v1
+; VI-NEXT:    v_add_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fadd <4 x half> %vec4, %rdx.shuf
@@ -18,17 +26,21 @@ entry:
   ret half %res
 }
 
-; GCN-LABEL: {{^}}reduction_fsub_v4f16:
-; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_add_f16 [[ADD:v[0-9]+]], v0, v1 neg_lo:[0,1] neg_hi:[0,1]{{$}}
-; GFX9-NEXT: v_sub_f16_sdwa v0, [[ADD]], [[ADD]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_setpc_b64
-
-; VI:      v_sub_f16_sdwa
-; VI-NEXT: v_sub_f16_e32
-; VI-NEXT: v_sub_f16_e32
-; VI-NEXT: s_setpc_b64
 define half @reduction_fsub_v4f16(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_fsub_v4f16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-NEXT:    v_sub_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_fsub_v4f16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_sub_f16_e32 v0, v0, v1
+; VI-NEXT:    v_sub_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fsub <4 x half> %vec4, %rdx.shuf
@@ -39,18 +51,21 @@ entry:
 }
 
 ; Make sure nsz is preserved when the operations are split.
-; GCN-LABEL: {{^}}reduction_fsub_v4f16_preserve_fmf:
-; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]{{$}}
-; GFX9-NEXT: v_sub_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_setpc_b64
-
-; VI: s_waitcnt
-; VI-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-NEXT: v_sub_f16_e32 v0, v1, v0
-; VI-NEXT: v_add_f16_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64
 define half @reduction_fsub_v4f16_preserve_fmf(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_fsub_v4f16_preserve_fmf:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX9-NEXT:    v_sub_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_fsub_v4f16_preserve_fmf:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_sub_f16_e32 v0, v1, v0
+; VI-NEXT:    v_add_f16_e32 v0, v2, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fsub nsz <4 x half> %vec4, %rdx.shuf
@@ -61,14 +76,21 @@ entry:
   ret half %neg.res
 }
 
-; GCN-LABEL: {{^}}reduction_fmul_half4:
-; GFX9:      v_pk_mul_f16 [[MUL:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_mul_f16_sdwa v{{[0-9]+}}, [[MUL]], [[MUL]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_mul_f16_sdwa
-; VI-NEXT: v_mul_f16_e32
-; VI-NEXT: v_mul_f16_e32
 define half @reduction_fmul_half4(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_fmul_half4:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_mul_f16 v0, v0, v1
+; GFX9-NEXT:    v_mul_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_fmul_half4:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_mul_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_mul_f16_e32 v0, v0, v1
+; VI-NEXT:    v_mul_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fmul <4 x half> %vec4, %rdx.shuf
@@ -78,14 +100,21 @@ entry:
   ret half %res
 }
 
-; GCN-LABEL: {{^}}reduction_v4i16:
-; GFX9:      v_pk_add_u16 [[ADD:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_add_u16_sdwa v{{[0-9]+}}, [[ADD]], [[ADD]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_add_u16_sdwa
-; VI-NEXT: v_add_u16_e32
-; VI-NEXT: v_add_u16_e32
 define i16 @reduction_v4i16(<4 x i16> %vec4) {
+; GFX9-LABEL: reduction_v4i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_v4i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_u16_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = add <4 x i16> %vec4, %rdx.shuf
@@ -95,21 +124,27 @@ entry:
   ret i16 %res
 }
 
-; GCN-LABEL: {{^}}reduction_half8:
-; GFX9:      v_pk_add_f16 [[ADD1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 [[ADD2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 [[ADD3:v[0-9]+]], [[ADD2]], [[ADD1]]{{$}}
-; GFX9-NEXT: v_add_f16_sdwa v{{[0-9]+}}, [[ADD3]], [[ADD3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_add_f16_sdwa
-; VI-NEXT: v_add_f16_sdwa
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-
 define half @reduction_half8(<8 x half> %vec8) {
+; GFX9-LABEL: reduction_half8:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, v3
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v2
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_half8:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_f16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_e32 v1, v1, v3
+; VI-NEXT:    v_add_f16_e32 v0, v0, v2
+; VI-NEXT:    v_add_f16_e32 v2, v5, v4
+; VI-NEXT:    v_add_f16_e32 v0, v0, v1
+; VI-NEXT:    v_add_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <8 x half> %vec8, <8 x half> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx = fadd <8 x half> %vec8, %rdx.shuf
@@ -121,21 +156,27 @@ entry:
   ret half %res
 }
 
-; GCN-LABEL: {{^}}reduction_v8i16:
-; GFX9:      v_pk_add_u16 [[ADD1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_u16 [[ADD2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_u16 [[ADD3:v[0-9]+]], [[ADD2]], [[ADD1]]{{$}}
-; GFX9-NEXT: v_add_u16_sdwa v{{[0-9]+}}, [[ADD3]], [[ADD3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_add_u16_sdwa
-; VI-NEXT: v_add_u16_sdwa
-; VI-NEXT: v_add_u16_e32
-; VI-NEXT: v_add_u16_e32
-; VI-NEXT: v_add_u16_e32
-; VI-NEXT: v_add_u16_e32
-; VI-NEXT: v_add_u16_e32
-
 define i16 @reduction_v8i16(<8 x i16> %vec8) {
+; GFX9-LABEL: reduction_v8i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_v8i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_u16_e32 v1, v1, v3
+; VI-NEXT:    v_add_u16_e32 v0, v0, v2
+; VI-NEXT:    v_add_u16_e32 v2, v5, v4
+; VI-NEXT:    v_add_u16_e32 v0, v0, v1
+; VI-NEXT:    v_add_u16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx = add <8 x i16> %vec8, %rdx.shuf
@@ -147,33 +188,39 @@ entry:
   ret i16 %res
 }
 
-; GCN-LABEL: {{^}}reduction_half16:
-; GFX9:      v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9:      v_pk_add_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 [[ADD1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 [[ADD2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_add_f16 [[ADD3:v[0-9]+]], [[ADD2]], [[ADD1]]{{$}}
-; GFX9-NEXT: v_add_f16_sdwa v{{[0-9]+}}, [[ADD3]], [[ADD3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_add_f16_sdwa
-; VI-NEXT: v_add_f16_sdwa
-; VI-NEXT: v_add_f16_sdwa
-; VI-NEXT: v_add_f16_sdwa
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-; VI-NEXT: v_add_f16_e32
-
 define half @reduction_half16(<16 x half> %vec16) {
+; GFX9-LABEL: reduction_half16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_add_f16 v2, v2, v6
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v4
+; GFX9-NEXT:    v_pk_add_f16 v3, v3, v7
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, v5
+; GFX9-NEXT:    v_pk_add_f16 v1, v1, v3
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v2
+; GFX9-NEXT:    v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_half16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_f16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_add_f16_e32 v2, v2, v6
+; VI-NEXT:    v_add_f16_e32 v0, v0, v4
+; VI-NEXT:    v_add_f16_e32 v3, v3, v7
+; VI-NEXT:    v_add_f16_e32 v1, v1, v5
+; VI-NEXT:    v_add_f16_e32 v4, v11, v10
+; VI-NEXT:    v_add_f16_e32 v5, v9, v8
+; VI-NEXT:    v_add_f16_e32 v1, v1, v3
+; VI-NEXT:    v_add_f16_e32 v0, v0, v2
+; VI-NEXT:    v_add_f16_e32 v2, v5, v4
+; VI-NEXT:    v_add_f16_e32 v0, v0, v1
+; VI-NEXT:    v_add_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <16 x half> %vec16, <16 x half> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx = fadd <16 x half> %vec16, %rdx.shuf
@@ -187,14 +234,21 @@ entry:
   ret half %res
 }
 
-; GCN-LABEL: {{^}}reduction_min_v4i16:
-; GFX9:      v_pk_min_u16 [[MIN:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_min_u16_sdwa v{{[0-9]+}}, [[MIN]], [[MIN]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_min_u16_sdwa
-; VI-NEXT: v_min_u16_e32
-; VI-NEXT: v_min_u16_e32
 define i16 @reduction_min_v4i16(<4 x i16> %vec4) {
+; GFX9-LABEL: reduction_min_v4i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_u16 v0, v0, v1
+; GFX9-NEXT:    v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_min_v4i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_min_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_u16_e32 v0, v0, v1
+; VI-NEXT:    v_min_u16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp ult <4 x i16> %vec4, %rdx.shuf
@@ -206,20 +260,27 @@ entry:
   ret i16 %res
 }
 
-; GCN-LABEL: {{^}}reduction_umin_v8i16:
-; GFX9:      v_pk_min_u16 [[MIN1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_min_u16 [[MIN2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_pk_min_u16 [[MIN3:v[0-9]+]], [[MIN2]], [[MIN1]]{{$}}
-; GFX9-NEXT: v_min_u16_sdwa v{{[0-9]+}}, [[MIN3]], [[MIN3]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_min_u16_sdwa
-; VI-NEXT: v_min_u16_sdwa
-; VI-NEXT: v_min_u16_e32
-; VI-NEXT: v_min_u16_e32
-; VI-NEXT: v_min_u16_e32
-; VI-NEXT: v_min_u16_e32
-; VI-NEXT: v_min_u16_e32
 define i16 @reduction_umin_v8i16(<8 x i16> %vec8) {
+; GFX9-LABEL: reduction_umin_v8i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_u16 v1, v1, v3
+; GFX9-NEXT:    v_pk_min_u16 v0, v0, v2
+; GFX9-NEXT:    v_pk_min_u16 v0, v0, v1
+; GFX9-NEXT:    v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_umin_v8i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_min_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_u16_e32 v1, v1, v3
+; VI-NEXT:    v_min_u16_e32 v0, v0, v2
+; VI-NEXT:    v_min_u16_e32 v2, v5, v4
+; VI-NEXT:    v_min_u16_e32 v0, v0, v1
+; VI-NEXT:    v_min_u16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp ult <8 x i16> %vec8, %rdx.shuf
@@ -235,15 +296,30 @@ entry:
 }
 
 ; Tests to make sure without slp the number of instructions are more.
-; GCN-LABEL: {{^}}reduction_umin_v8i16_woslp:
-; GFX9:      v_lshrrev_b32_e32
-; GFX9-NEXT: v_min_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_u16
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_u16
-; GFX9-NEXT: v_min3_u16
 define i16 @reduction_umin_v8i16_woslp(<8 x i16> %vec8) {
+; GFX9-LABEL: reduction_umin_v8i16_woslp:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT:    v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT:    v_min3_u16 v0, v4, v1, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v3
+; GFX9-NEXT:    v_min3_u16 v0, v5, v2, v0
+; GFX9-NEXT:    v_min3_u16 v0, v6, v3, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_umin_v8i16_woslp:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_u16_e32 v0, v1, v0
+; VI-NEXT:    v_min_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_u16_e32 v0, v2, v0
+; VI-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_u16_e32 v0, v3, v0
+; VI-NEXT:    v_min_u16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %elt0 = extractelement <8 x i16> %vec8, i64 0
   %elt1 = extractelement <8 x i16> %vec8, i64 1
@@ -274,32 +350,39 @@ entry:
   ret i16 %min7
 }
 
-; GCN-LABEL: {{^}}reduction_smin_v16i16:
-; GFX9:        v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_pk_min_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT:   v_min_i16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_min_i16_sdwa
-; VI-NEXT: v_min_i16_sdwa
-; VI-NEXT: v_min_i16_sdwa
-; VI-NEXT: v_min_i16_sdwa
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
-; VI-NEXT: v_min_i16_e32
 define i16 @reduction_smin_v16i16(<16 x i16> %vec16) {
+; GFX9-LABEL: reduction_smin_v16i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_i16 v2, v2, v6
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, v4
+; GFX9-NEXT:    v_pk_min_i16 v3, v3, v7
+; GFX9-NEXT:    v_pk_min_i16 v1, v1, v5
+; GFX9-NEXT:    v_pk_min_i16 v1, v1, v3
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, v2
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, v1
+; GFX9-NEXT:    v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_smin_v16i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_min_i16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_i16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_i16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_i16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_min_i16_e32 v2, v2, v6
+; VI-NEXT:    v_min_i16_e32 v0, v0, v4
+; VI-NEXT:    v_min_i16_e32 v3, v3, v7
+; VI-NEXT:    v_min_i16_e32 v1, v1, v5
+; VI-NEXT:    v_min_i16_e32 v4, v11, v10
+; VI-NEXT:    v_min_i16_e32 v5, v9, v8
+; VI-NEXT:    v_min_i16_e32 v1, v1, v3
+; VI-NEXT:    v_min_i16_e32 v0, v0, v2
+; VI-NEXT:    v_min_i16_e32 v2, v5, v4
+; VI-NEXT:    v_min_i16_e32 v0, v0, v1
+; VI-NEXT:    v_min_i16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <16 x i16> %vec16, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp slt <16 x i16> %vec16, %rdx.shuf
@@ -318,23 +401,46 @@ entry:
 }
 
 ; Tests to make sure without slp the number of instructions are more.
-; GCN-LABEL: {{^}}reduction_smin_v16i16_woslp:
-; GFX9:      v_lshrrev_b32_e32
-; GFX9-NEXT: v_min_i16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_i16
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_i16
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_i16
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_i16
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_i16
-; GFX9-NEXT: v_lshrrev_b32_e32
-; GFX9-NEXT: v_min3_i16
-; GFX9-NEXT: v_min3_i16
 define i16 @reduction_smin_v16i16_woslp(<16 x i16> %vec16) {
+; GFX9-LABEL: reduction_smin_v16i16_woslp:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
+; GFX9-NEXT:    v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
+; GFX9-NEXT:    v_min3_i16 v0, v8, v1, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v3
+; GFX9-NEXT:    v_min3_i16 v0, v9, v2, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v4
+; GFX9-NEXT:    v_min3_i16 v0, v10, v3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v5
+; GFX9-NEXT:    v_min3_i16 v0, v11, v4, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
+; GFX9-NEXT:    v_min3_i16 v0, v12, v5, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v7
+; GFX9-NEXT:    v_min3_i16 v0, v13, v6, v0
+; GFX9-NEXT:    v_min3_i16 v0, v14, v7, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_smin_v16i16_woslp:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v1, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v2, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v3, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v4, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v5, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v6, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_min_i16_e32 v0, v7, v0
+; VI-NEXT:    v_min_i16_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %elt0 = extractelement <16 x i16> %vec16, i64 0
   %elt1 = extractelement <16 x i16> %vec16, i64 1
@@ -395,14 +501,21 @@ entry:
   ret i16 %min15
 }
 
-; GCN-LABEL: {{^}}reduction_umax_v4i16:
-; GFX9:      v_pk_max_u16 [[MAX:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_max_u16_sdwa v{{[0-9]+}}, [[MAX]], [[MAX]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_max_u16_sdwa
-; VI-NEXT: v_max_u16_e32
-; VI-NEXT: v_max_u16_e32
 define i16 @reduction_umax_v4i16(<4 x i16> %vec4) {
+; GFX9-LABEL: reduction_umax_v4i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_max_u16 v0, v0, v1
+; GFX9-NEXT:    v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_umax_v4i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_max_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_u16_e32 v0, v0, v1
+; VI-NEXT:    v_max_u16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp ugt <4 x i16> %vec4, %rdx.shuf
@@ -414,14 +527,21 @@ entry:
   ret i16 %res
 }
 
-; GCN-LABEL: {{^}}reduction_smax_v4i16:
-; GFX9:      v_pk_max_i16 [[MAX:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX9-NEXT: v_max_i16_sdwa v{{[0-9]+}}, [[MAX]], [[MAX]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI:      v_max_i16_sdwa
-; VI-NEXT: v_max_i16_e32
-; VI-NEXT: v_max_i16_e32
 define i16 @reduction_smax_v4i16(<4 x i16> %vec4) #0 {
+; GFX9-LABEL: reduction_smax_v4i16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_max_i16 v0, v0, v1
+; GFX9-NEXT:    v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_smax_v4i16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_max_i16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_i16_e32 v0, v0, v1
+; VI-NEXT:    v_max_i16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp sgt <4 x i16> %vec4, %rdx.shuf
@@ -433,23 +553,27 @@ entry:
   ret i16 %res
 }
 
-; GCN-LABEL: {{^}}reduction_maxnum_v4f16:
-; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_max_f16 [[CANON1:v[0-9]+]], v1, v1
-; GFX9-NEXT: v_pk_max_f16 [[CANON0:v[0-9]+]], v0, v0
-; GFX9-NEXT: v_pk_max_f16 [[MAX:v[0-9]+]], [[CANON0]], [[CANON1]]{{$}}
-; GFX9-NEXT: v_max_f16_sdwa v{{[0-9]+}}, [[MAX]], [[MAX]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-
-; VI-DAG: v_max_f16_sdwa [[CANON1:v[0-9]+]], v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_sdwa [[CANON3:v[0-9]+]], v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_e32 [[CANON0:v[0-9]+]], v0, v0
-; VI-DAG: v_max_f16_e32 [[CANON2:v[0-9]+]], v1, v1
-
-; VI-DAG: v_max_f16_e32 [[MAX0:v[0-9]+]], [[CANON1]], [[CANON3]]
-; VI-DAG: v_max_f16_e32 [[MAX1:v[0-9]+]], [[CANON0]], [[CANON2]]
-; VI: v_max_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_maxnum_v4f16(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_maxnum_v4f16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_max_f16 v1, v1, v1
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v0
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v1
+; GFX9-NEXT:    v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_maxnum_v4f16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_e32 v1, v1, v1
+; VI-NEXT:    v_max_f16_e32 v0, v0, v0
+; VI-NEXT:    v_max_f16_e32 v2, v3, v2
+; VI-NEXT:    v_max_f16_e32 v0, v0, v1
+; VI-NEXT:    v_max_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %vec4, <4 x half> %rdx.shuf)
@@ -459,22 +583,27 @@ entry:
   ret half %res
 }
 
-; GCN-LABEL: {{^}}reduction_minnum_v4f16:
-; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_max_f16 [[CANON1:v[0-9]+]], v1, v1
-; GFX9-NEXT: v_pk_max_f16 [[CANON0:v[0-9]+]], v0, v0
-; GFX9-NEXT: v_pk_min_f16 [[MIN:v[0-9]+]], [[CANON0]], [[CANON1]]{{$}}
-; GFX9-NEXT: v_min_f16_sdwa v{{[0-9]+}}, [[MIN]], [[MIN]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI-DAG: v_max_f16_sdwa [[CANON1:v[0-9]+]], v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_sdwa [[CANON3:v[0-9]+]], v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_e32 [[CANON0:v[0-9]+]], v0, v0
-; VI-DAG: v_max_f16_e32 [[CANON2:v[0-9]+]], v1, v1
-
-; VI-DAG: v_min_f16_e32 [[MAX0:v[0-9]+]], [[CANON1]], [[CANON3]]
-; VI-DAG: v_min_f16_e32 [[MAX1:v[0-9]+]], [[CANON0]], [[CANON2]]
-; VI: v_min_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_minnum_v4f16(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_minnum_v4f16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_max_f16 v1, v1, v1
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v0
+; GFX9-NEXT:    v_pk_min_f16 v0, v0, v1
+; GFX9-NEXT:    v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_minnum_v4f16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_e32 v1, v1, v1
+; VI-NEXT:    v_max_f16_e32 v0, v0, v0
+; VI-NEXT:    v_min_f16_e32 v2, v3, v2
+; VI-NEXT:    v_min_f16_e32 v0, v0, v1
+; VI-NEXT:    v_min_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax = call <4 x half> @llvm.minnum.v4f16(<4 x half> %vec4, <4 x half> %rdx.shuf)
@@ -486,32 +615,27 @@ entry:
 
 ; FIXME: Need to preserve fast math flags when fmaxnum matched
 ; directly from the IR to avoid unnecessary quieting.
-
-; GCN-LABEL: {{^}}reduction_fast_max_pattern_v4f16:
-; XGFX9:      v_pk_max_f16 [[MAX:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; XGFX9-NEXT: v_max_f16_sdwa v{{[0-9]+}}, [[MAX]], [[MAX]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; XVI: s_waitcnt
-; XVI-NEXT: v_max_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; XVI-NEXT: v_max_f16_e32 v0, v0, v1
-; XVI-NEXT: v_max_f16_e32 v0, v0, v2
-; XVI-NEXT: s_setpc_b64
-
-; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_max_f16 [[CANON1:v[0-9]+]], v1, v1
-; GFX9-NEXT: v_pk_max_f16 [[CANON0:v[0-9]+]], v0, v0
-; GFX9-NEXT: v_pk_max_f16 [[MAX:v[0-9]+]], [[CANON0]], [[CANON1]]{{$}}
-; GFX9-NEXT: v_max_f16_sdwa v{{[0-9]+}}, [[MAX]], [[MAX]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI-DAG: v_max_f16_sdwa [[CANON1:v[0-9]+]], v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_sdwa [[CANON3:v[0-9]+]], v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_e32 [[CANON0:v[0-9]+]], v0, v0
-; VI-DAG: v_max_f16_e32 [[CANON2:v[0-9]+]], v1, v1
-
-; VI-DAG: v_max_f16_e32 [[MAX0:v[0-9]+]], [[CANON1]], [[CANON3]]
-; VI-DAG: v_max_f16_e32 [[MAX1:v[0-9]+]], [[CANON0]], [[CANON2]]
-; VI: v_max_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_fast_max_pattern_v4f16(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_fast_max_pattern_v4f16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_max_f16 v1, v1, v1
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v0
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v1
+; GFX9-NEXT:    v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_fast_max_pattern_v4f16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_e32 v1, v1, v1
+; VI-NEXT:    v_max_f16_e32 v0, v0, v0
+; VI-NEXT:    v_max_f16_e32 v2, v3, v2
+; VI-NEXT:    v_max_f16_e32 v0, v0, v1
+; VI-NEXT:    v_max_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = fcmp nnan nsz ogt <4 x half> %vec4, %rdx.shuf
@@ -525,32 +649,27 @@ entry:
 
 ; FIXME: Need to preserve fast math flags when fmaxnum matched
 ; directly from the IR to avoid unnecessary quieting.
-
-; GCN-LABEL: {{^}}reduction_fast_min_pattern_v4f16:
-; XGFX9:      v_pk_min_f16 [[MIN:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; XGFX9-NEXT: v_min_f16_sdwa v{{[0-9]+}}, [[MIN]], [[MIN]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; XVI: s_waitcnt
-; XVI-NEXT: v_min_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; XVI-NEXT: v_min_f16_e32 v0, v0, v1
-; XVI-NEXT: v_min_f16_e32 v0, v0, v2
-; XVI-NEXT: s_setpc_b64
-
-; GFX9: s_waitcnt
-; GFX9-NEXT: v_pk_max_f16 [[CANON1:v[0-9]+]], v1, v1
-; GFX9-NEXT: v_pk_max_f16 [[CANON0:v[0-9]+]], v0, v0
-; GFX9-NEXT: v_pk_min_f16 [[MIN:v[0-9]+]], [[CANON0]], [[CANON1]]{{$}}
-; GFX9-NEXT: v_min_f16_sdwa v{{[0-9]+}}, [[MIN]], [[MIN]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-
-; VI-DAG: v_max_f16_sdwa [[CANON1:v[0-9]+]], v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_sdwa [[CANON3:v[0-9]+]], v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_max_f16_e32 [[CANON0:v[0-9]+]], v0, v0
-; VI-DAG: v_max_f16_e32 [[CANON2:v[0-9]+]], v1, v1
-
-; VI-DAG: v_min_f16_e32 [[MAX0:v[0-9]+]], [[CANON1]], [[CANON3]]
-; VI-DAG: v_min_f16_e32 [[MAX1:v[0-9]+]], [[CANON0]], [[CANON2]]
-; VI: v_min_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_fast_min_pattern_v4f16(<4 x half> %vec4) {
+; GFX9-LABEL: reduction_fast_min_pattern_v4f16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_max_f16 v1, v1, v1
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v0
+; GFX9-NEXT:    v_pk_min_f16 v0, v0, v1
+; GFX9-NEXT:    v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: reduction_fast_min_pattern_v4f16:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT:    v_max_f16_e32 v1, v1, v1
+; VI-NEXT:    v_max_f16_e32 v0, v0, v0
+; VI-NEXT:    v_min_f16_e32 v2, v3, v2
+; VI-NEXT:    v_min_f16_e32 v0, v0, v1
+; VI-NEXT:    v_min_f16_e32 v0, v0, v2
+; VI-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = fcmp nnan nsz olt <4 x half> %vec4, %rdx.shuf
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 6512bee36e88b..ba9dd8f7c2468 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -513,21 +513,28 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -538,6 +545,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -1073,10 +1081,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
@@ -1889,21 +1897,28 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -1914,6 +1929,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -2449,10 +2465,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
new file mode 100644
index 0000000000000..7f822c135ffb4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
@@ -0,0 +1,1956 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,SI-DAZ-SAFE %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,SI-IEEE-SAFE %s
+
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,CI-DAZ-SAFE %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,CI-IEEE-SAFE %s
+
+
+declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+declare float @llvm.sqrt.f32(float) nounwind readnone
+declare <2 x float> @llvm.sqrt.v2f32(<2 x float>) nounwind readnone
+
+define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
+; GCN-DAZ-SAFE-LABEL: rsq_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: rsq_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: rsq_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %val = load float, ptr addrspace(1) %in, align 4
+  %sqrt = call contract float @llvm.sqrt.f32(float %val) nounwind readnone
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %val) {
+; GCN-DAZ-SAFE-LABEL: rsq_f32_sgpr:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, -1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v3, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %sqrt = call contract float @llvm.sqrt.f32(float %val) nounwind readnone
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; Recognize that this is rsqrt(a) * rcp(b) * c,
+; not 1 / ( 1 / sqrt(a)) * rcp(b) * c.
+
+; NOTE: c * rcp( sqrt(a) * b ) is generated when we move rcp generation to AMGGPUCogenPrepare.
+define amdgpu_kernel void @rsqrt_fmul(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; GCN-DAZ-SAFE-LABEL: rsqrt_fmul:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0
+; GCN-DAZ-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v2, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v5, v7, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v7, v8, v7
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v8, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v7, v7, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v8, v5, v7
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v3, s[4:5], v2, v2, v4
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
+; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
+; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
+; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GCN-DAZ-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
+; GCN-DAZ-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; GCN-IEEE-SAFE-LABEL: rsqrt_fmul:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0
+; GCN-IEEE-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v5, v2
+; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[0:1], -1, v5
+; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v8, s[0:1], 1, v5
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v9, -v7, v5, v2
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v10, -v8, v5, v2
+; GCN-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[0:1], 0, v9
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[0:1]
+; GCN-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], 0, v10
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[0:1]
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v3, s[0:1], v2, v2, v4
+; GCN-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
+; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
+; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
+; GCN-IEEE-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
+; GCN-IEEE-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
+; GCN-IEEE-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-IEEE-SAFE-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+  %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
+  %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr float, ptr addrspace(1) %gep.0, i32 2
+
+  %a = load volatile float, ptr addrspace(1) %gep.0
+  %b = load volatile float, ptr addrspace(1) %gep.1
+  %c = load volatile float, ptr addrspace(1) %gep.2
+
+  %x = call contract float @llvm.sqrt.f32(float %a)
+  %y = fmul contract float %x, %b
+  %z = fdiv contract float %c, %y
+  store float %z, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @neg_rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
+; GCN-DAZ-SAFE-LABEL: neg_rsq_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: neg_rsq_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: neg_rsq_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %val = load float, ptr addrspace(1) %in, align 4
+  %sqrt = call contract float @llvm.sqrt.f32(float %val)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @neg_rsq_neg_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
+; GCN-DAZ-SAFE-LABEL: neg_rsq_neg_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %val = load float, ptr addrspace(1) %in, align 4
+  %val.fneg = fneg float %val
+  %sqrt = call contract float @llvm.sqrt.f32(float %val.fneg)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define float @v_neg_rsq_neg_f32(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val.fneg = fneg float %val
+  %sqrt = call contract float @llvm.sqrt.f32(float %val.fneg)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  ret float %div
+}
+
+define <2 x float> @v_neg_rsq_neg_v2f32(<2 x float> %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v3, -v0, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val.fneg = fneg <2 x float> %val
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val.fneg)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  ret <2 x float> %div
+}
+
+define float @v_neg_rsq_neg_f32_foldable_user(float %val0, float %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val0.neg = fneg float %val0
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0.neg)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  %user = fmul contract float %div, %val1
+  ret float %user
+}
+
+define <2 x float> @v_neg_rsq_neg_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0.5, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v4, v5, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v6, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v6, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v7, v4, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v5, -v0, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, v0, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v4, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v7, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v4, v4, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v7, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v8, v5, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v0, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v1, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val0.fneg = fneg <2 x float> %val0
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0.fneg)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  %user = fmul contract <2 x float> %div, %val1
+  ret <2 x float> %user
+}
+
+define float @v_neg_rsq_f32(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  ret float %div
+}
+
+define <2 x float> @v_neg_rsq_v2f32(<2 x float> %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  ret <2 x float> %div
+}
+
+define float @v_neg_rsq_f32_foldable_user(float %val0, float %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  %user = fmul contract float %div, %val1
+  ret float %user
+}
+
+define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0.5, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v4, v5, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v6, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v6, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v7, v4, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, v0, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v4, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v7, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v4, v4, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v7, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v8, v5, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v0, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v1, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  %user = fmul contract <2 x float> %div, %val1
+  ret <2 x float> %user
+}
+
+define float @v_rsq_f32(float %val) {
+; GCN-DAZ-LABEL: v_rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+define { float, float } @v_rsq_f32_multi_use(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_rsq_f32_multi_use:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v2, vcc, 0, v2
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v2, vcc, 0, v2
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %insert.0 = insertvalue { float, float } poison, float %sqrt, 0
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %insert.1 = insertvalue { float, float } %insert.0, float %div, 1
+  ret { float, float } %insert.1
+}
+
+define float @v_rsq_f32_missing_contract0(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract0:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+define float @v_rsq_f32_missing_contract1(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract1:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+; Test that we contract into FMA for an fadd user after introducing
+; the fmul.
+define float @v_rsq_f32_contractable_user(float %val0, float %val1) {
+; GCN-DAZ-LABEL: v_rsq_f32_contractable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %add = fadd contract float %div, %val1
+  ret float %add
+}
+
+; Missing contract on the fdiv
+define float @v_rsq_f32_contractable_user_missing_contract0(float %val0, float %val1) {
+; GCN-DAZ-LABEL: v_rsq_f32_contractable_user_missing_contract0:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %add = fadd contract float %div, %val1
+  ret float %add
+}
+
+; Missing contract on the fadd
+define float @v_rsq_f32_contractable_user_missing_contract1(float %val0, float %val1) {
+; GCN-DAZ-LABEL: v_rsq_f32_contractable_user_missing_contract1:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %add = fadd float %div, %val1
+  ret float %add
+}
+
+define float @v_rsq_f32_known_never_denormal(float nofpclass(sub) %val) {
+; GCN-DAZ-LABEL: v_rsq_f32_known_never_denormal:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-LABEL: v_rsq_f32_known_never_denormal:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
+; GCN-DAZ-LABEL: v_rsq_f32_known_never_posdenormal:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_known_never_posdenormal:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+!0 = !{float 2.500000e+00}
+!1 = !{float 1.000000e+00}
+
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CI-DAZ-SAFE: {{.*}}
+; SI-DAZ-SAFE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
index f4b947ade8dac..f7e0388561104 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
@@ -2,175 +2,51 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-UNSAFE,SI-DAZ-UNSAFE %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -enable-unsafe-fp-math < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-UNSAFE,SI-IEEE-UNSAFE %s
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,SI-DAZ-SAFE %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,SI-IEEE-SAFE %s
-
 
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-UNSAFE,CI-DAZ-UNSAFE %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=ieee -enable-unsafe-fp-math < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-UNSAFE,CI-IEEE-UNSAFE %s
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,CI-DAZ-SAFE %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,CI-IEEE-SAFE %s
-
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare float @llvm.sqrt.f32(float) nounwind readnone
 declare <2 x float> @llvm.sqrt.v2f32(<2 x float>) nounwind readnone
 
 define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
-; GCN-DAZ-UNSAFE-LABEL: rsq_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: rsq_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: rsq_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-NEXT:    s_endpgm
 ;
-; CI-IEEE-SAFE-LABEL: rsq_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-IEEE-LABEL: rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, -1
+; GCN-IEEE-NEXT:    s_mov_b32 s10, s6
+; GCN-IEEE-NEXT:    s_mov_b32 s11, s7
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s8, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s9, s3
+; GCN-IEEE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-IEEE-NEXT:    s_mov_b32 s4, s0
+; GCN-IEEE-NEXT:    s_mov_b32 s5, s1
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: rsq_f32:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
@@ -196,131 +72,27 @@ define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(
 }
 
 define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %val) {
-; GCN-DAZ-UNSAFE-LABEL: rsq_f32_sgpr:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dword s2, s[4:5], 0xb
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s2, -1
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: rsq_f32_sgpr:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dword s2, s[4:5], 0xb
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s2, -1
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: rsq_f32_sgpr:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, -1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, s0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v3, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
-;
-; CI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: rsq_f32_sgpr:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dword s2, s[4:5], 0xb
+; GCN-DAZ-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s2, -1
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-DAZ-NEXT:    s_endpgm
+;
+; GCN-IEEE-LABEL: rsq_f32_sgpr:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dword s2, s[4:5], 0xb
+; GCN-IEEE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s2, -1
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: rsq_f32_sgpr:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dword s2, s[0:1], 0xb
@@ -365,154 +137,53 @@ define amdgpu_kernel void @rsqrt_fmul(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-UNSAFE-NEXT:    v_mul_f32_e32 v2, v4, v2
 ; GCN-UNSAFE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
 ; GCN-UNSAFE-NEXT:    s_endpgm
-; GCN-DAZ-UNSAFE-LABEL: rsqrt_fmul:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, 0
-; GCN-DAZ-UNSAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b64 s[8:9], s[2:3]
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b64 s[10:11], s[6:7]
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b64 s[4:5], s[0:1]
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v2, v2
-; GCN-DAZ-UNSAFE-NEXT:    v_rcp_f32_e32 v3, v3
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e32 v2, v4, v2
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: rsqrt_fmul:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, 0
-; GCN-IEEE-UNSAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b64 s[8:9], s[2:3]
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b64 s[10:11], s[6:7]
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b64 s[4:5], s[0:1]
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v2, v2
-; GCN-IEEE-UNSAFE-NEXT:    v_rcp_f32_e32 v3, v3
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e32 v2, v4, v2
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: rsqrt_fmul:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0
-; GCN-DAZ-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v2, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v5, v7, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v7, v8, v7
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v8, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v7, v7, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v8, v5, v7
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v3, s[4:5], v2, v2, v4
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
-; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
-; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
-; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GCN-DAZ-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
-; GCN-DAZ-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-SAFE-LABEL: rsqrt_fmul:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0
-; GCN-IEEE-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
-; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GCN-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v5, v2
-; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[0:1], -1, v5
-; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v8, s[0:1], 1, v5
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v9, -v7, v5, v2
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v10, -v8, v5, v2
-; GCN-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[0:1], 0, v9
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[0:1]
-; GCN-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], 0, v10
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[0:1]
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v3, s[0:1], v2, v2, v4
-; GCN-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
-; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
-; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
-; GCN-IEEE-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
-; GCN-IEEE-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
-; GCN-IEEE-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; GCN-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: rsqrt_fmul:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, 0
+; GCN-DAZ-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-DAZ-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; GCN-DAZ-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GCN-DAZ-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v2, v2
+; GCN-DAZ-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-DAZ-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-DAZ-NEXT:    v_mul_f32_e32 v2, v4, v2
+; GCN-DAZ-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GCN-DAZ-NEXT:    s_endpgm
+;
+; GCN-IEEE-LABEL: rsqrt_fmul:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, 0
+; GCN-IEEE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; GCN-IEEE-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GCN-IEEE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v2, v2
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v2, v4, v2
+; GCN-IEEE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GCN-IEEE-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
@@ -525,167 +196,49 @@ define amdgpu_kernel void @rsqrt_fmul(ptr addrspace(1) %out, ptr addrspace(1) %i
 
   %x = call contract float @llvm.sqrt.f32(float %a)
   %y = fmul contract float %x, %b
-  %z = fdiv contract float %c, %y
+  %z = fdiv arcp contract float %c, %y
   store float %z, ptr addrspace(1) %out.gep
   ret void
 }
 
 define amdgpu_kernel void @neg_rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
-; GCN-DAZ-UNSAFE-LABEL: neg_rsq_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: neg_rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: neg_rsq_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: neg_rsq_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: neg_rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-NEXT:    s_endpgm
 ;
-; CI-IEEE-SAFE-LABEL: neg_rsq_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-IEEE-LABEL: neg_rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, -1
+; GCN-IEEE-NEXT:    s_mov_b32 s10, s6
+; GCN-IEEE-NEXT:    s_mov_b32 s11, s7
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s8, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s9, s3
+; GCN-IEEE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-IEEE-NEXT:    s_mov_b32 s4, s0
+; GCN-IEEE-NEXT:    s_mov_b32 s5, s1
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: neg_rsq_f32:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
@@ -712,161 +265,43 @@ define amdgpu_kernel void @neg_rsq_f32(ptr addrspace(1) noalias %out, ptr addrsp
 }
 
 define amdgpu_kernel void @neg_rsq_neg_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
-; GCN-DAZ-UNSAFE-LABEL: neg_rsq_neg_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: neg_rsq_neg_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: neg_rsq_neg_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
-;
-; CI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: neg_rsq_neg_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-NEXT:    s_endpgm
+;
+; GCN-IEEE-LABEL: neg_rsq_neg_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, -1
+; GCN-IEEE-NEXT:    s_mov_b32 s10, s6
+; GCN-IEEE-NEXT:    s_mov_b32 s11, s7
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s8, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s9, s3
+; GCN-IEEE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-IEEE-NEXT:    s_mov_b32 s4, s0
+; GCN-IEEE-NEXT:    s_mov_b32 s5, s1
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: neg_rsq_neg_f32:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
@@ -894,101 +329,19 @@ define amdgpu_kernel void @neg_rsq_neg_f32(ptr addrspace(1) noalias %out, ptr ad
 }
 
 define float @v_neg_rsq_neg_f32(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_neg_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %val.fneg = fneg float %val
   %sqrt = call contract float @llvm.sqrt.f32(float %val.fneg)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
@@ -996,168 +349,23 @@ define float @v_neg_rsq_neg_f32(float %val) {
 }
 
 define <2 x float> @v_neg_rsq_neg_v2f32(<2 x float> %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_v2f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_v2f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v3, -v0, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_v2f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_neg_v2f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %val.fneg = fneg <2 x float> %val
   %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val.fneg)
   %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
@@ -1165,104 +373,19 @@ define <2 x float> @v_neg_rsq_neg_v2f32(<2 x float> %val) {
 }
 
 define float @v_neg_rsq_neg_f32_foldable_user(float %val0, float %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %val0.neg = fneg float %val0
   %sqrt = call contract float @llvm.sqrt.f32(float %val0.neg)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
@@ -1271,546 +394,86 @@ define float @v_neg_rsq_neg_f32_foldable_user(float %val0, float %val1) {
 }
 
 define <2 x float> @v_neg_rsq_neg_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, v1, v4
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0.5, v4
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v4, v5, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v6, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v6, v4
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v7, v4, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v5, -v0, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v6
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, v0, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v4, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v7, v4
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v4, v4, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v7, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v8, v5, v4
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v6
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v0, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v1, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-  %val0.fneg = fneg <2 x float> %val0
-  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0.fneg)
-  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
-  %user = fmul contract <2 x float> %div, %val1
-  ret <2 x float> %user
-}
-
-define float @v_neg_rsq_f32(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %val0.fneg = fneg <2 x float> %val0
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0.fneg)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  %user = fmul contract <2 x float> %div, %val1
+  ret <2 x float> %user
+}
+
+define float @v_neg_rsq_f32(float %val) {
+; GCN-DAZ-LABEL: v_neg_rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-LABEL: v_neg_rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
   ret float %div
 }
 
 define <2 x float> @v_neg_rsq_v2f32(<2 x float> %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_v2f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_v2f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_v2f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_v2f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val)
   %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
   ret <2 x float> %div
 }
 
 define float @v_neg_rsq_f32_foldable_user(float %val0, float %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
   %user = fmul contract float %div, %val1
@@ -1818,24 +481,23 @@ define float @v_neg_rsq_f32_foldable_user(float %val0, float %val1) {
 }
 
 define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_v2f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GCN-IEEE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
 ; GCN-DAZ-SAFE:       ; %bb.0:
 ; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1876,7 +538,6 @@ define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float>
 ; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
 ; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
 ; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
 ; SI-IEEE-SAFE:       ; %bb.0:
 ; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1932,7 +593,6 @@ define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float>
 ; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
 ; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
 ; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
 ; CI-IEEE-SAFE:       ; %bb.0:
 ; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1996,12 +656,11 @@ define float @v_rsq_f32(float %val) {
 ; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
+; GCN-IEEE-LABEL: v_rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-IEEE-SAFE-LABEL: v_rsq_f32:
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2019,29 +678,27 @@ define float @v_rsq_f32(float %val) {
 }
 
 define { float, float } @v_rsq_f32_multi_use(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_rsq_f32_multi_use:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_multi_use:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_rsq_f32_multi_use:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_sqrt_f32_e32 v2, v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GCN-IEEE-LABEL: v_rsq_f32_multi_use:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_sqrt_f32_e32 v2, v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-DAZ-SAFE-LABEL: v_rsq_f32_multi_use:
 ; GCN-DAZ-SAFE:       ; %bb.0:
 ; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
 ; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v1, v0
 ; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; SI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
 ; SI-IEEE-SAFE:       ; %bb.0:
 ; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2072,7 +729,6 @@ define { float, float } @v_rsq_f32_multi_use(float %val) {
 ; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v2, vcc, 0, v2
 ; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
 ; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; CI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
 ; CI-IEEE-SAFE:       ; %bb.0:
 ; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2102,31 +758,29 @@ define { float, float } @v_rsq_f32_multi_use(float %val) {
 ; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
   %insert.0 = insertvalue { float, float } poison, float %sqrt, 0
-  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %div = fdiv arcp contract float 1.0, %sqrt, !fpmath !1
   %insert.1 = insertvalue { float, float } %insert.0, float %div, 1
   ret { float, float } %insert.1
 }
 
 define float @v_rsq_f32_missing_contract0(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_rsq_f32_missing_contract0:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_missing_contract0:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_rsq_f32_missing_contract0:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GCN-IEEE-LABEL: v_rsq_f32_missing_contract0:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract0:
 ; GCN-DAZ-SAFE:       ; %bb.0:
 ; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
 ; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
 ; SI-IEEE-SAFE:       ; %bb.0:
 ; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2157,7 +811,6 @@ define float @v_rsq_f32_missing_contract0(float %val) {
 ; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
 ; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
 ; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
 ; CI-IEEE-SAFE:       ; %bb.0:
 ; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2186,90 +839,24 @@ define float @v_rsq_f32_missing_contract0(float %val) {
 ; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
 ; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call float @llvm.sqrt.f32(float %val), !fpmath !1
-  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %div = fdiv arcp contract float 1.0, %sqrt, !fpmath !1
   ret float %div
 }
 
 define float @v_rsq_f32_missing_contract1(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_rsq_f32_missing_contract1:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_missing_contract1:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract1:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_rsq_f32_missing_contract1:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_missing_contract1:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
-  %div = fdiv float 1.0, %sqrt, !fpmath !1
+  %div = fdiv arcp float 1.0, %sqrt, !fpmath !1
   ret float %div
 }
 
@@ -2283,25 +870,12 @@ define float @v_rsq_f32_contractable_user(float %val0, float %val1) {
 ; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_contractable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_contractable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   %add = fadd contract float %div, %val1
@@ -2317,25 +891,12 @@ define float @v_rsq_f32_contractable_user_missing_contract0(float %val0, float %
 ; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   %add = fadd contract float %div, %val1
@@ -2351,25 +912,12 @@ define float @v_rsq_f32_contractable_user_missing_contract1(float %val0, float %
 ; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   %add = fadd float %div, %val1
@@ -2400,23 +948,11 @@ define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
 ; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_known_never_posdenormal:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_known_never_posdenormal:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_known_never_posdenormal:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   ret float %div
@@ -2427,9 +963,9 @@ define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CI-DAZ-SAFE: {{.*}}
 ; CI-DAZ-UNSAFE: {{.*}}
 ; CI-IEEE-UNSAFE: {{.*}}
-; SI-DAZ-SAFE: {{.*}}
+; GCN-DAZ-UNSAFE: {{.*}}
+; GCN-IEEE-UNSAFE: {{.*}}
 ; SI-DAZ-UNSAFE: {{.*}}
 ; SI-IEEE-UNSAFE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll b/llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll
new file mode 100644
index 0000000000000..ca4b72872a03b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=finalize-isel -o - %s | FileCheck %s
+
+;; Test that reduction of:
+;;
+;;   DST = lshr i64 X, Y
+;;
+;; where Y is in the range [63-32] to:
+;;
+;;   DST = [srl i32 X, (Y & 0x1F), 0]
+;;
+;; preserves flags
+
+define i64 @srl_exact(i64 %arg0, i64 %shift_amt) {
+  ; CHECK-LABEL: name: srl_exact
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; CHECK-NEXT:   [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[DEF3]]
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+  ; CHECK-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = exact V_LSHRREV_B32_e64 killed [[COPY5]], killed [[COPY3]], implicit $exec
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY [[V_LSHRREV_B32_e64_]]
+  ; CHECK-NEXT:   $vgpr1 = COPY [[V_MOV_B32_e32_]]
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %or = or i64 %shift_amt, 32
+  %srl = lshr exact i64 %arg0, %or
+  ret i64 %srl
+}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 2f1dfa11fd34d..141b86a24c1c4 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -742,9 +742,8 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX9-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX9-GISEL:       ; %bb.0:
-; GFX9-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-GISEL-NEXT:    v_pk_add_f16 v0, s2, v0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT:    v_pk_add_f16 v0, s2, v0 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX9-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
@@ -784,8 +783,7 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX10-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX10-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
+; GFX10-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX10-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-SDAG-TRUE16-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
@@ -810,8 +808,7 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ;
 ; GFX11-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX11-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
+; GFX11-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX11-GISEL-NEXT:    ; return to shader part epilog
 ; GFX10PLUS-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10PLUS-SDAG:       ; %bb.0:
@@ -824,8 +821,7 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
 ; GFX10PLUS-SDAG-NEXT:    ; return to shader part epilog
 ; GFX10PLUS-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
 ; GFX10PLUS-GISEL:       ; %bb.0:
-; GFX10PLUS-GISEL-NEXT:    s_xor_b32 s0, s3, 0x80008000
-; GFX10PLUS-GISEL-NEXT:    v_pk_add_f16 v0, s2, s0
+; GFX10PLUS-GISEL-NEXT:    v_pk_add_f16 v0, s2, s3 neg_lo:[0,1] neg_hi:[0,1]
 ; GFX10PLUS-GISEL-NEXT:    ; return to shader part epilog
   %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
   ret <2 x half> %val
diff --git a/llvm/test/CodeGen/AMDGPU/sub_i1.ll b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
index 19d012fc074f8..8e65e6450ec45 100644
--- a/llvm/test/CodeGen/AMDGPU/sub_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
@@ -1,12 +1,56 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,WAVE64 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,WAVE32 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,WAVE32 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
 
-
-; GCN-LABEL: {{^}}sub_var_var_i1:
-; WAVE32: v_xor_b32_e32
-; WAVE64: v_xor_b32_e32
 define amdgpu_kernel void @sub_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; GFX9-LABEL: sub_var_var_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v2, v0, s[6:7] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_var_var_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v2, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: sub_var_var_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_u8 v2, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
   %sub = sub i1 %a, %b
@@ -14,21 +58,175 @@ define amdgpu_kernel void @sub_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; GCN-LABEL: {{^}}sub_var_imm_i1:
-; WAVE32: s_xor_b32
-; WAVE64: s_xor_b64
 define amdgpu_kernel void @sub_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; GFX9-LABEL: sub_var_imm_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_xor_b64 s[2:3], vcc, -1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_var_imm_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: sub_var_imm_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in
   %sub = sub i1 %a, 1
   store i1 %sub, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}sub_i1_cf:
-; GCN: ; %endif
-; WAVE32: s_xor_b32
-; WAVE64: s_xor_b64
 define amdgpu_kernel void @sub_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; GFX9-LABEL: sub_i1_cf:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, 15, v0
+; GFX9-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %else
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[8:9] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
+; GFX9-NEXT:  .LBB2_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_4
+; GFX9-NEXT:  ; %bb.3: ; %if
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_andn2_b64 s[2:3], s[4:5], exec
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT:    s_or_b64 s[4:5], s[2:3], s[4:5]
+; GFX9-NEXT:  .LBB2_4: ; %endif
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b64 s[2:3], s[4:5], -1
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_i1_cf:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 15, v0
+; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_2
+; GFX10-NEXT:  ; %bb.1: ; %else
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 1, v0
+; GFX10-NEXT:  .LBB2_2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s5, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_4
+; GFX10-NEXT:  ; %bb.3: ; %if
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 s2, s4, exec_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s4, s2, s3
+; GFX10-NEXT:  .LBB2_4: ; %endif
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_xor_b32 s2, s4, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: sub_i1_cf:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s7, exec_lo
+; GFX11-NEXT:    ; implicit-def: $sgpr6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmpx_lt_u32_e32 15, v0
+; GFX11-NEXT:    s_xor_b32 s7, exec_lo, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %else
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v0, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 1, v0
+; GFX11-NEXT:  .LBB2_2: ; %Flow
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s4, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_4
+; GFX11-NEXT:  ; %bb.3: ; %if
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_u8 v0, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_and_not1_b32 s2, s6, exec_lo
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-NEXT:    s_or_b32 s6, s2, s3
+; GFX11-NEXT:  .LBB2_4: ; %endif
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_xor_b32 s2, s6, -1
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %d_cmp = icmp ult i32 %tid, 16
diff --git a/llvm/test/CodeGen/AMDGPU/swizzle-export.ll b/llvm/test/CodeGen/AMDGPU/swizzle-export.ll
index 8d5b6568340db..d35e05e003952 100644
--- a/llvm/test/CodeGen/AMDGPU/swizzle-export.ll
+++ b/llvm/test/CodeGen/AMDGPU/swizzle-export.ll
@@ -1,12 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=r600 -mcpu=redwood | FileCheck --check-prefix=EG %s
 
-;EG: {{^}}main:
-;EG: EXPORT T{{[0-9]+}}.XYXX
-;EG: EXPORT T{{[0-9]+}}.ZXXX
-;EG: EXPORT T{{[0-9]+}}.XXWX
-;EG: EXPORT T{{[0-9]+}}.XXXW
-
 define amdgpu_vs void @main(<4 x float> inreg %reg0, <4 x float> inreg %reg1) {
+; EG-LABEL: main:
+; EG:       ; %bb.0: ; %main_body
+; EG-NEXT:    CALL_FS
+; EG-NEXT:    ALU 3, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    EXPORT T1.XYZW
+; EG-NEXT:    EXPORT T0.XYXX
+; EG-NEXT:    EXPORT T0.ZXXX
+; EG-NEXT:    EXPORT T0.XXWX
+; EG-NEXT:    EXPORT T0.XXXW
+; EG-NEXT:    CF_END
+; EG-NEXT:    ALU clause starting at 8:
+; EG-NEXT:     MOV * T0.W, KC0[0].X,
+; EG-NEXT:     MUL_IEEE T0.X, KC0[0].X, 0.0,
+; EG-NEXT:     MUL_IEEE T0.Y, KC0[0].Y, KC0[0].X,
+; EG-NEXT:     MUL_IEEE * T0.Z, KC0[0].Z, KC0[0].X,
 main_body:
   %0 = extractelement <4 x float> %reg1, i32 0
   %1 = extractelement <4 x float> %reg1, i32 1
@@ -92,11 +102,28 @@ main_body:
   ret void
 }
 
-; EG: {{^}}main2:
-; EG: T{{[0-9]+}}.XY__
-; EG: T{{[0-9]+}}.ZXY0
-
 define amdgpu_vs void @main2(<4 x float> inreg %reg0, <4 x float> inreg %reg1) {
+; EG-LABEL: main2:
+; EG:       ; %bb.0: ; %main_body
+; EG-NEXT:    CALL_FS
+; EG-NEXT:    ALU 11, @6, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    EXPORT T1.XY__
+; EG-NEXT:    EXPORT T0.ZXY0
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    ALU clause starting at 6:
+; EG-NEXT:     MULADD_IEEE * T0.W, KC0[1].X, literal.x, 0.5,
+; EG-NEXT:    1042479491(1.591549e-01), 0(0.000000e+00)
+; EG-NEXT:     FRACT * T0.W, PV.W,
+; EG-NEXT:     ADD * T0.W, PV.W, literal.x,
+; EG-NEXT:    -1090519040(-5.000000e-01), 0(0.000000e+00)
+; EG-NEXT:     COS * T0.Z, PV.W,
+; EG-NEXT:     ADD T1.X, T1.X, literal.x,
+; EG-NEXT:     MOV T0.Y, KC0[0].Y,
+; EG-NEXT:     MOV * T0.X, KC0[0].X,
+; EG-NEXT:    1075838976(2.500000e+00), 0(0.000000e+00)
+; EG-NEXT:     MUL_IEEE * T1.Y, T1.Y, literal.x,
+; EG-NEXT:    1080033280(3.500000e+00), 0(0.000000e+00)
 main_body:
   %0 = extractelement <4 x float> %reg1, i32 0
   %1 = extractelement <4 x float> %reg1, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
new file mode 100644
index 0000000000000..726e35d6651d0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GCN %s
+
+declare i32 @llvm.amdgcn.atomic.cond.sub.u32.p1(ptr addrspace(1), i32)
+
+
+define amdgpu_kernel void @test_isel_single_lane(ptr addrspace(1) %in, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test_isel_single_lane:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_wait_kmcnt 0x0
+; GCN-NEXT:    s_load_b32 s4, s[0:1], 0x58
+; GCN-NEXT:    s_wait_kmcnt 0x0
+; GCN-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GCN-NEXT:    global_atomic_cond_sub_u32 v1, v0, v1, s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GCN-NEXT:    s_wait_loadcnt 0x0
+; GCN-NEXT:    v_readfirstlane_b32 s0, v1
+; GCN-NEXT:    s_addk_co_i32 s0, 0xf4
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GCN-NEXT:    s_lshl_b32 s1, s0, 4
+; GCN-NEXT:    s_mul_i32 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GCN-NEXT:    s_lshl_b32 s0, s0, 12
+; GCN-NEXT:    s_sub_co_i32 s0, s1, s0
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GCN-NEXT:    s_endpgm
+        %gep0 = getelementptr i32, ptr addrspace(1) %in, i32 22
+        %val0 = load i32, ptr addrspace(1) %gep0, align 4
+        %gep1 = getelementptr i32, ptr addrspace(1) %in, i32 4
+        %val1 = call i32 @llvm.amdgcn.atomic.cond.sub.u32.p0(ptr addrspace(1) %gep1, i32 %val0)
+        %res0 = add i32 %val1, 244
+        %res1 = shl i32 %res0, 4
+        %res2 = mul i32 %res0, %res1
+        %res3 = shl i32 %res2, 12
+        %res4 = sub i32 %res1, %res3
+        store i32 %res4, ptr addrspace(1) %out
+        ret void
+}
+
+
+attributes #0 = {
+  "amdgpu-flat-work-group-size"="1,1"
+  "amdgpu-waves-per-eu"="1,1"
+  "uniform-work-group-size"="true"
+}
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll b/llvm/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll
index 7d4ec547328a2..def8d7e45ef2b 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll
@@ -1,10 +1,41 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
 
-; GCN-LABEL: {{^}}trunc_i64_bitcast_v2i32:
-; GCN: buffer_load_dword v
-; GCN: buffer_store_dword v
 define amdgpu_kernel void @trunc_i64_bitcast_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i64_bitcast_v2i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i64_bitcast_v2i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <2 x i32>, ptr addrspace(1) %in
   %bc = bitcast <2 x i32> %ld to i64
   %trunc = trunc i64 %bc to i32
@@ -12,10 +43,40 @@ define amdgpu_kernel void @trunc_i64_bitcast_v2i32(ptr addrspace(1) %out, ptr ad
   ret void
 }
 
-; GCN-LABEL: {{^}}trunc_i96_bitcast_v3i32:
-; GCN: buffer_load_dword v
-; GCN: buffer_store_dword v
 define amdgpu_kernel void @trunc_i96_bitcast_v3i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i96_bitcast_v3i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i96_bitcast_v3i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <3 x i32>, ptr addrspace(1) %in
   %bc = bitcast <3 x i32> %ld to i96
   %trunc = trunc i96 %bc to i32
@@ -23,10 +84,40 @@ define amdgpu_kernel void @trunc_i96_bitcast_v3i32(ptr addrspace(1) %out, ptr ad
   ret void
 }
 
-; GCN-LABEL: {{^}}trunc_i128_bitcast_v4i32:
-; GCN: buffer_load_dword v
-; GCN: buffer_store_dword v
 define amdgpu_kernel void @trunc_i128_bitcast_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i128_bitcast_v4i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i128_bitcast_v4i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <4 x i32>, ptr addrspace(1) %in
   %bc = bitcast <4 x i32> %ld to i128
   %trunc = trunc i128 %bc to i32
@@ -35,10 +126,40 @@ define amdgpu_kernel void @trunc_i128_bitcast_v4i32(ptr addrspace(1) %out, ptr a
 }
 
 ; Don't want load width reduced in this case.
-; GCN-LABEL: {{^}}trunc_i16_bitcast_v2i16:
-; GCN: buffer_load_dword [[VAL:v[0-9]+]]
-; GCN: buffer_store_short [[VAL]]
 define amdgpu_kernel void @trunc_i16_bitcast_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i16_bitcast_v2i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i16_bitcast_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <2 x i16>, ptr addrspace(1) %in
   %bc = bitcast <2 x i16> %ld to i32
   %trunc = trunc i32 %bc to i16
@@ -46,14 +167,44 @@ define amdgpu_kernel void @trunc_i16_bitcast_v2i16(ptr addrspace(1) %out, ptr ad
   ret void
 }
 
-; GCN-LABEL: {{^}}trunc_i16_bitcast_v4i16:
 ; FIXME We need to teach the dagcombiner to reduce load width for:
 ;   t21: v2i32,ch = load<LD8[%in(addrspace=1)]> t12, t10, undef:i64
 ;        t23: i64 = bitcast t21
 ;      t30: i16 = truncate t23
-; GCN: buffer_load_dword v[[VAL:[0-9]+]]
-; GCN: buffer_store_short v[[VAL]], off
 define amdgpu_kernel void @trunc_i16_bitcast_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i16_bitcast_v4i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i16_bitcast_v4i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <4 x i16>, ptr addrspace(1) %in
   %bc = bitcast <4 x i16> %ld to i64
   %trunc = trunc i64 %bc to i16
@@ -62,11 +213,40 @@ define amdgpu_kernel void @trunc_i16_bitcast_v4i16(ptr addrspace(1) %out, ptr ad
 }
 
 ; FIXME: Consistently shrink or not here
-; GCN-LABEL: {{^}}trunc_i8_bitcast_v2i8:
-; SI: buffer_load_ubyte [[VAL:v[0-9]+]]
-; VI: buffer_load_ushort [[VAL:v[0-9]+]]
-; GCN: buffer_store_byte [[VAL]]
 define amdgpu_kernel void @trunc_i8_bitcast_v2i8(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i8_bitcast_v2i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i8_bitcast_v2i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <2 x i8>, ptr addrspace(1) %in
   %bc = bitcast <2 x i8> %ld to i16
   %trunc = trunc i16 %bc to i8
@@ -74,10 +254,40 @@ define amdgpu_kernel void @trunc_i8_bitcast_v2i8(ptr addrspace(1) %out, ptr addr
   ret void
 }
 
-; GCN-LABEL: {{^}}trunc_i32_bitcast_v4i8:
-; GCN: buffer_load_dword [[VAL:v[0-9]+]]
-; GCN: buffer_store_byte [[VAL]]
 define amdgpu_kernel void @trunc_i32_bitcast_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i32_bitcast_v4i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i32_bitcast_v4i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <4 x i8>, ptr addrspace(1) %in
   %bc = bitcast <4 x i8> %ld to i32
   %trunc = trunc i32 %bc to i8
@@ -85,10 +295,40 @@ define amdgpu_kernel void @trunc_i32_bitcast_v4i8(ptr addrspace(1) %out, ptr add
   ret void
 }
 
-; GCN-LABEL: {{^}}trunc_i24_bitcast_v3i8:
-; GCN: buffer_load_dword [[VAL:v[0-9]+]]
-; GCN: buffer_store_byte [[VAL]]
 define amdgpu_kernel void @trunc_i24_bitcast_v3i8(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: trunc_i24_bitcast_v3i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: trunc_i24_bitcast_v3i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %ld = load <3 x i8>, ptr addrspace(1) %in
   %bc = bitcast <3 x i8> %ld to i24
   %trunc = trunc i24 %bc to i8
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll b/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
index e5bc4c5721b90..da5ec09e195b8 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
@@ -1,15 +1,49 @@
-; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
-; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_eq_0:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
-; SI: v_cmp_eq_u32_e32 vcc, 0, [[TMP]]{{$}}
-; SI: v_cndmask_b32_e64
-; SI: buffer_store_byte
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_0(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: sextload_i1_to_i32_trunc_cmp_eq_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: sextload_i1_to_i32_trunc_cmp_eq_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = sext i1 %load to i32
   %cmp = icmp eq i32 %ext, 0
@@ -18,14 +52,48 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_0(ptr addrspace(1) %o
 }
 
 ; FIXME: The negate should be inverting the compare.
-; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_eq_0:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
-; SI: v_cmp_eq_u32_e32 vcc, 1, [[TMP]]{{$}}
-; SI-NEXT: s_xor_b64 [[NEG:s\[[0-9]+:[0-9]+\]]], vcc, -1
-; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[NEG]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_0(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: zextload_i1_to_i32_trunc_cmp_eq_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, -1
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: zextload_i1_to_i32_trunc_cmp_eq_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, -1
+; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = zext i1 %load to i32
   %cmp = icmp eq i32 %ext, 0
@@ -33,10 +101,26 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_0(ptr addrspace(1) %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_eq_1:
-; SI: v_mov_b32_e32 [[RESULT:v[0-9]+]], 0{{$}}
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: sextload_i1_to_i32_trunc_cmp_eq_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: sextload_i1_to_i32_trunc_cmp_eq_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = sext i1 %load to i32
   %cmp = icmp eq i32 %ext, 1
@@ -44,11 +128,42 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_1(ptr addrspace(1) %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_eq_1:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[LOAD]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: zextload_i1_to_i32_trunc_cmp_eq_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: zextload_i1_to_i32_trunc_cmp_eq_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = zext i1 %load to i32
   %cmp = icmp eq i32 %ext, 1
@@ -56,11 +171,42 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_1(ptr addrspace(1) %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_eq_neg1:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[LOAD]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_neg1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: sextload_i1_to_i32_trunc_cmp_eq_neg1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: sextload_i1_to_i32_trunc_cmp_eq_neg1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = sext i1 %load to i32
   %cmp = icmp eq i32 %ext, -1
@@ -68,10 +214,26 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_neg1(ptr addrspace(1)
   ret void
 }
 
-; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_eq_neg1:
-; SI: v_mov_b32_e32 [[RESULT:v[0-9]+]], 0{{$}}
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_neg1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: zextload_i1_to_i32_trunc_cmp_eq_neg1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: zextload_i1_to_i32_trunc_cmp_eq_neg1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = zext i1 %load to i32
   %cmp = icmp eq i32 %ext, -1
@@ -80,11 +242,42 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_neg1(ptr addrspace(1)
 }
 
 
-; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_ne_0:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[LOAD]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_0(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: sextload_i1_to_i32_trunc_cmp_ne_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: sextload_i1_to_i32_trunc_cmp_ne_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = sext i1 %load to i32
   %cmp = icmp ne i32 %ext, 0
@@ -92,11 +285,42 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_0(ptr addrspace(1) %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_ne_0:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[LOAD]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_0(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: zextload_i1_to_i32_trunc_cmp_ne_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: zextload_i1_to_i32_trunc_cmp_ne_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = zext i1 %load to i32
   %cmp = icmp ne i32 %ext, 0
@@ -104,10 +328,26 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_0(ptr addrspace(1) %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_ne_1:
-; SI: v_mov_b32_e32 [[RESULT:v[0-9]+]], 1{{$}}
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: sextload_i1_to_i32_trunc_cmp_ne_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, 1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: sextload_i1_to_i32_trunc_cmp_ne_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, 1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = sext i1 %load to i32
   %cmp = icmp ne i32 %ext, 1
@@ -115,14 +355,48 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_1(ptr addrspace(1) %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_ne_1:
-; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
-; SI: v_cmp_eq_u32_e32 vcc, 1, [[TMP]]{{$}}
-; SI-NEXT: s_xor_b64 [[NEG:s\[[0-9]+:[0-9]+\]]], vcc, -1
-; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[NEG]]
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: zextload_i1_to_i32_trunc_cmp_ne_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, -1
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: zextload_i1_to_i32_trunc_cmp_ne_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, -1
+; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = zext i1 %load to i32
   %cmp = icmp ne i32 %ext, 1
@@ -131,13 +405,46 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_1(ptr addrspace(1) %o
 }
 
 ; FIXME: This should be one compare.
-; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_ne_neg1:
-; XSI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; XSI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
-; XSI: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]+\]]], [[TMP]], 0{{$}}
-; XSI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CMP0]]
-; XSI-NEXT: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: sextload_i1_to_i32_trunc_cmp_ne_neg1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: sextload_i1_to_i32_trunc_cmp_ne_neg1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = sext i1 %load to i32
   %cmp = icmp ne i32 %ext, -1
@@ -145,10 +452,26 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1)
   ret void
 }
 
-; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_ne_neg1:
-; SI: v_mov_b32_e32 [[RESULT:v[0-9]+]], 1{{$}}
-; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: zextload_i1_to_i32_trunc_cmp_ne_neg1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, 1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: zextload_i1_to_i32_trunc_cmp_ne_neg1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    v_mov_b32_e32 v0, 1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %load = load i1, ptr addrspace(1) %in
   %ext = zext i1 %load to i32
   %cmp = icmp ne i32 %ext, -1
@@ -157,12 +480,41 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1)
 }
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
-; FUNC-LABEL: {{^}}masked_load_i1_to_i32_trunc_cmp_ne_neg1:
-; SI: {{buffer|flat}}_load_sbyte [[LOAD:v[0-9]+]]
-; SI: v_cmp_ne_u32_e32 vcc, -1, [[LOAD]]{{$}}
-; SI-NEXT: v_cndmask_b32_e64
-; SI: {{buffer|flat}}_store_byte
 define amdgpu_kernel void @masked_load_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; SI-LABEL: masked_load_i1_to_i32_trunc_cmp_ne_neg1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sbyte v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: masked_load_i1_to_i32_trunc_cmp_ne_neg1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_sbyte v0, v[0:1]
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v0
+; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %in.ptr = getelementptr i8, ptr addrspace(1) %in, i32 %tid.x
   %load = load i8, ptr addrspace(1) %in.ptr
diff --git a/llvm/test/CodeGen/AMDGPU/valu-i1.ll b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
index 35cd2663f523f..c500565d36b0c 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
@@ -1,30 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs -enable-misched -asm-verbose -disable-block-placement -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=SI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
-; SI-LABEL: {{^}}test_if:
 ; Make sure the i1 values created by the cfg structurizer pass are
 ; moved using VALU instructions
 
 
 ; waitcnt should be inserted after exec modification
-; SI:      v_cmp_lt_i32_e32 vcc, 1,
-; SI-NEXT: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0
-; SI-NEXT: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0
-; SI-NEXT: s_and_saveexec_b64 [[SAVE1:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_xor_b64 [[SAVE2:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE1]]
-; SI-NEXT: s_cbranch_execz [[FLOW_BB:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}: ; %LeafBlock3
-; SI:      s_mov_b64 s[{{[0-9]:[0-9]}}], -1
-; SI:      s_and_saveexec_b64
-; SI-NEXT: s_cbranch_execnz
-
 ; v_mov should be after exec modification
-; SI: [[FLOW_BB]]:
-; SI-NEXT: s_andn2_saveexec_b64 [[SAVE2]], [[SAVE2]]
-;
 define amdgpu_kernel void @test_if(i32 %b, ptr addrspace(1) %src, ptr addrspace(1) %dst) #1 {
+; SI-LABEL: test_if:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s8, s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 1, v0
+; SI-NEXT:    s_mov_b64 s[10:11], 0
+; SI-NEXT:    s_mov_b64 s[2:3], 0
+; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz .LBB0_3
+; SI-NEXT:  ; %bb.1: ; %LeafBlock3
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], -1
+; SI-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SI-NEXT:    s_cbranch_execnz .LBB0_9
+; SI-NEXT:  .LBB0_2: ; %Flow7
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT:  .LBB0_3: ; %Flow6
+; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT:    s_cbranch_execz .LBB0_5
+; SI-NEXT:  ; %bb.4: ; %LeafBlock
+; SI-NEXT:    s_mov_b64 s[10:11], exec
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 1, v0
+; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; SI-NEXT:    s_and_b64 s[6:7], vcc, exec
+; SI-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; SI-NEXT:  .LBB0_5: ; %Flow8
+; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; SI-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execnz .LBB0_10
+; SI-NEXT:  .LBB0_6: ; %Flow9
+; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_and_saveexec_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_cbranch_execz .LBB0_8
+; SI-NEXT:  ; %bb.7: ; %case1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_ashr_i32 s9, s8, 31
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_lshl_b64 s[4:5], s[8:9], 2
+; SI-NEXT:    v_mov_b32_e32 v2, 13
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB0_8: ; %end
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB0_9: ; %case2
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_ashr_i32 s9, s8, 31
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_lshl_b64 s[12:13], s[8:9], 2
+; SI-NEXT:    v_mov_b32_e32 v3, 17
+; SI-NEXT:    v_mov_b32_e32 v1, s12
+; SI-NEXT:    v_mov_b32_e32 v2, s13
+; SI-NEXT:    buffer_store_dword v3, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_xor_b64 s[2:3], exec, -1
+; SI-NEXT:    s_branch .LBB0_2
+; SI-NEXT:  .LBB0_10: ; %default
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_ashr_i32 s9, s8, 31
+; SI-NEXT:    s_lshl_b64 s[4:5], s[8:9], 2
+; SI-NEXT:    s_add_u32 s4, s0, s4
+; SI-NEXT:    s_addc_u32 s5, s1, s5
+; SI-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SI-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB0_14
+; SI-NEXT:  .LBB0_11: ; %Flow
+; SI-NEXT:    s_andn2_saveexec_b64 s[12:13], s[12:13]
+; SI-NEXT:    s_cbranch_execz .LBB0_13
+; SI-NEXT:  ; %bb.12: ; %if
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, 19
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:  .LBB0_13: ; %Flow5
+; SI-NEXT:    s_or_b64 exec, exec, s[12:13]
+; SI-NEXT:    s_andn2_b64 s[10:11], s[10:11], exec
+; SI-NEXT:    s_branch .LBB0_6
+; SI-NEXT:  .LBB0_14: ; %else
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, 21
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_branch .LBB0_11
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   switch i32 %tid, label %default [
@@ -59,17 +134,23 @@ end:
   ret void
 }
 
-; SI-LABEL: {{^}}simple_test_v_if:
-; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}:
-; SI: buffer_store_dword
-
-; SI-NEXT: {{^}}[[EXIT]]:
-; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_if:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB1_2
+; SI-NEXT:  ; %bb.1: ; %then
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0x3e7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB1_2: ; %exit
+; SI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %is.0 = icmp ne i32 %tid, 0
   br i1 %is.0, label %then, label %exit
@@ -84,18 +165,23 @@ exit:
 }
 
 ; FIXME: It would be better to endpgm in the then block.
-
-; SI-LABEL: {{^}}simple_test_v_if_ret_else_ret:
-; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}:
-; SI: buffer_store_dword
-
-; SI-NEXT: {{^}}[[EXIT]]:
-; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if_ret_else_ret(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_if_ret_else_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB2_2
+; SI-NEXT:  ; %bb.1: ; %then
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0x3e7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB2_2: ; %UnifiedReturnBlock
+; SI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %is.0 = icmp ne i32 %tid, 0
   br i1 %is.0, label %then, label %exit
@@ -112,27 +198,33 @@ exit:
 ; Final block has more than a ret to execute. This was miscompiled
 ; before function exit blocks were unified since the endpgm would
 ; terminate the then wavefront before reaching the store.
-
-; SI-LABEL: {{^}}simple_test_v_if_ret_else_code_ret:
-; SI: v_cmp_eq_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]]
-; SI: s_cbranch_execnz [[EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: {{^.LBB[0-9]+_[0-9]+}}: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 [[BR_SREG]], [[BR_SREG]]
-; SI-NEXT: s_cbranch_execz [[UNIFIED_RETURN:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}: ; %then
-; SI: s_waitcnt
-; SI-NEXT: buffer_store_dword
-
-; SI-NEXT: {{^}}[[UNIFIED_RETURN]]: ; %UnifiedReturnBlock
-; SI: s_endpgm
-
-; SI-NEXT: {{^}}[[EXIT]]:
-; SI: ds_write_b32
 define amdgpu_kernel void @simple_test_v_if_ret_else_code_ret(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_if_ret_else_code_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
+; SI-NEXT:    s_cbranch_execnz .LBB3_4
+; SI-NEXT:  .LBB3_1: ; %Flow
+; SI-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; SI-NEXT:    s_cbranch_execz .LBB3_3
+; SI-NEXT:  ; %bb.2: ; %then
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0x3e7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB3_3: ; %UnifiedReturnBlock
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB3_4: ; %exit
+; SI-NEXT:    v_mov_b32_e32 v0, 7
+; SI-NEXT:    s_mov_b32 m0, -1
+; SI-NEXT:    ds_write_b32 v0, v0
+; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    s_branch .LBB3_1
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %is.0 = icmp ne i32 %tid, 0
   br i1 %is.0, label %then, label %exit
@@ -147,21 +239,38 @@ exit:
   ret void
 }
 
-; SI-LABEL: {{^}}simple_test_v_loop:
-; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[LABEL_EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0{{$}}
-
-; SI: [[LABEL_LOOP:.LBB[0-9]+_[0-9]+]]:
-; SI: buffer_load_dword
-; SI-DAG: buffer_store_dword
-; SI-DAG: s_cmpk_lg_i32 s{{[0-9]+}}, 0x100
-; SI: s_cbranch_scc1 [[LABEL_LOOP]]
-; SI: [[LABEL_EXIT]]:
-; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_loop(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_loop:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB4_3
+; SI-NEXT:  ; %bb.1: ; %loop.preheader
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    s_mov_b64 s[0:1], 0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s8, v0
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s10
+; SI-NEXT:    s_mov_b32 s5, s11
+; SI-NEXT:    s_mov_b32 s7, s3
+; SI-NEXT:  .LBB4_2: ; %loop
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_add_u32 s0, s0, 4
+; SI-NEXT:    s_addc_u32 s1, s1, 0
+; SI-NEXT:    s_cmpk_lg_i32 s0, 0x100
+; SI-NEXT:    s_cbranch_scc1 .LBB4_2
+; SI-NEXT:  .LBB4_3: ; %exit
+; SI-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %is.0 = icmp ne i32 %tid, 0
@@ -182,45 +291,81 @@ exit:
   ret void
 }
 
-; SI-LABEL: {{^}}multi_vcond_loop:
-
 ; Load loop limit from buffer
 ; Branch to exit if uniformly not taken
-; SI: ; %bb.0:
-; SI: buffer_load_dword [[VBOUND:v[0-9]+]]
-; SI: v_cmp_lt_i32_e32 vcc
-; SI: s_and_saveexec_b64 [[OUTER_CMP_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[LABEL_EXIT:.LBB[0-9]+_[0-9]+]]
-
 ; Initialize inner condition to false
-; SI: ; %bb.{{[0-9]+}}: ; %bb10.preheader
-; SI: s_mov_b64 [[COND_STATE:s\[[0-9]+:[0-9]+\]]], 0{{$}}
-
 ; Clear exec bits for workitems that load -1s
-; SI: .L[[LABEL_LOOP:BB[0-9]+_[0-9]+]]:
-; SI: buffer_load_dword [[B:v[0-9]+]]
-; SI: buffer_load_dword [[A:v[0-9]+]]
-; SI-DAG: v_cmp_ne_u32_e64 [[NEG1_CHECK_0:s\[[0-9]+:[0-9]+\]]], -1, [[A]]
-; SI-DAG: v_cmp_ne_u32_e32 [[NEG1_CHECK_1:vcc]], -1, [[B]]
-; SI: s_and_b64 [[ORNEG1:s\[[0-9]+:[0-9]+\]]], [[NEG1_CHECK_1]], [[NEG1_CHECK_0]]
-; SI: s_and_saveexec_b64 [[ORNEG2:s\[[0-9]+:[0-9]+\]]], [[ORNEG1]]
-; SI: s_cbranch_execz [[LABEL_FLOW:.LBB[0-9]+_[0-9]+]]
-
-; SI: ; %bb.{{[0-9]+}}: ; %bb20
-; SI: buffer_store_dword
-
-; SI: [[LABEL_FLOW]]:
-; SI-NEXT: ; in Loop: Header=[[LABEL_LOOP]]
-; SI-NEXT: s_or_b64 exec, exec, [[ORNEG2]]
-; SI-NEXT: s_and_b64 [[TMP1:s\[[0-9]+:[0-9]+\]]],
-; SI-NEXT: s_or_b64 [[COND_STATE]], [[TMP1]], [[COND_STATE]]
-; SI-NEXT: s_andn2_b64 exec, exec, [[COND_STATE]]
-; SI-NEXT: s_cbranch_execnz .L[[LABEL_LOOP]]
-
-; SI: [[LABEL_EXIT]]:
-; SI-NOT: [[COND_STATE]]
-; SI: s_endpgm
 define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %arg, ptr addrspace(1) noalias nocapture readonly %arg1, ptr addrspace(1) noalias nocapture readonly %arg2, ptr addrspace(1) noalias nocapture readonly %arg3) #1 {
+; SI-LABEL: multi_vcond_loop:
+; SI:       ; %bb.0: ; %bb
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_mov_b32_e32 v7, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v0, v[6:7], s[8:11], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB5_5
+; SI-NEXT:  ; %bb.1: ; %bb10.preheader
+; SI-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; SI-NEXT:    s_mov_b64 s[2:3], 0
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, s13
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s12, v6
+; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s1
+; SI-NEXT:    v_add_i32_e32 v4, vcc, s0, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v7, s15
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s14, v6
+; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB5_2: ; %bb10
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, v[6:7], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_dword v9, v[4:5], s[8:11], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], -1, v9
+; SI-NEXT:    s_and_b64 s[12:13], vcc, s[0:1]
+; SI-NEXT:    s_or_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], s[12:13]
+; SI-NEXT:    s_cbranch_execz .LBB5_4
+; SI-NEXT:  ; %bb.3: ; %bb20
+; SI-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; SI-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; SI-NEXT:    s_add_u32 s6, s6, 1
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 4, v4
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 4, v6
+; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; SI-NEXT:    buffer_store_dword v8, v[2:3], s[8:11], 0 addr64
+; SI-NEXT:    s_addc_u32 s7, s7, 0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 4, v2
+; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; SI-NEXT:    v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
+; SI-NEXT:    s_andn2_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_and_b64 s[12:13], vcc, exec
+; SI-NEXT:    s_or_b64 s[4:5], s[4:5], s[12:13]
+; SI-NEXT:  .LBB5_4: ; %Flow
+; SI-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
+; SI-NEXT:    s_and_b64 s[0:1], exec, s[4:5]
+; SI-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_cbranch_execnz .LBB5_2
+; SI-NEXT:  .LBB5_5: ; %bb26
+; SI-NEXT:    s_endpgm
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %tmp4 = sext i32 %tmp to i64
diff --git a/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll b/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll
index 2c66d38a1be62..33ca7180325af 100644
--- a/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll
+++ b/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll
@@ -1,12 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX1010 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX1030 %s
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_lt:
-; GFX1010: v_cmp_lt_i32_e32 vcc_lo, 15, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_lt_i32_e32 15, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_lt(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_lt:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 15, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_lt:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 15, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp slt i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -19,12 +36,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_gt:
-; GFX1010: v_cmp_gt_i32_e32 vcc_lo, 17, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_gt_i32_e32 17, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_gt(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_gt:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 17, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_gt:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_gt_i32_e32 17, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp sgt i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -37,12 +70,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_eq:
-; GFX1010: v_cmp_ne_u32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_ne_u32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_eq(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_eq:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_eq:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_ne_u32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp eq i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -55,12 +104,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_ne:
-; GFX1010: v_cmp_eq_u32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_eq_u32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_ne(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_ne:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_ne:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_eq_u32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp ne i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -73,12 +138,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_le:
-; GFX1010: v_cmp_lt_i32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_lt_i32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_le(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_le:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_le:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp sle i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -91,12 +172,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_ge:
-; GFX1010: v_cmp_gt_i32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_gt_i32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_ge(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_ge:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_ge:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_gt_i32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp sge i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -113,13 +210,32 @@ declare amdgpu_gfx void @check_live_outs_helper(i64) #0
 
 ; In cases where the output operand cannot be safely removed,
 ; don't apply the v_cmpx transformation.
-
-; GCN-LABEL: {{^}}check_live_outs:
-; GFX1010: v_cmp_eq_u32_e64 s{{.*}}, v{{.*}}, v{{.*}}
-; GFX1010: s_and_saveexec_b32 s{{.*}}, s{{.*}}
-; GFX1030: v_cmp_eq_u32_e64 s{{.*}}, v{{.*}}, v{{.*}}
-; GFX1030: s_and_saveexec_b32 s{{.*}}, s{{.*}}
 define amdgpu_cs void @check_live_outs(i32 %a, i32 %b) {
+; GCN-LABEL: check_live_outs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GCN-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s11, 0x31c16000
+; GCN-NEXT:    s_add_u32 s8, s8, s0
+; GCN-NEXT:    v_cmp_eq_u32_e64 s0, v0, v1
+; GCN-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-NEXT:    s_mov_b32 s32, 0
+; GCN-NEXT:    s_and_saveexec_b32 s1, s0
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %l1
+; GCN-NEXT:    s_getpc_b64 s[2:3]
+; GCN-NEXT:    s_add_u32 s2, s2, check_live_outs_helper@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s3, s3, check_live_outs_helper@gotpcrel32@hi+12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b64 s[0:1], s[8:9]
+; GCN-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:  .LBB6_2: ; %l2
+; GCN-NEXT:    s_endpgm
   %cond = icmp eq i32 %a, %b
   %result = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 32)
   br i1 %cond, label %l1, label %l2
@@ -132,14 +248,27 @@ l2:
 
 ; Omit the transformation if the s_and_saveexec instruction overwrites
 ; any of the v_cmp source operands.
-
-; GCN-LABEL: check_saveexec_overwrites_vcmp_source:
-; GCN:  .LBB7_2: ; %then
-; GFX1010:          v_cmp_eq_u32_e64 s[[C:[0-9]+]], s[[A:[0-9]+]], s[[B:[0-9]+]]
-; GFX1010-NEXT:     s_cmp_ge_i32 s[[C]], s[[B]]
-; GFX1030:          v_cmp_eq_u32_e64 s[[C:[0-9]+]], s[[A:[0-9]+]], s[[B:[0-9]+]]
-; GFX1030-NEXT:     s_cmp_ge_i32 s[[C]], s[[B]]
 define i32 @check_saveexec_overwrites_vcmp_source(i32 inreg %a, i32 inreg %b) {
+; GCN-LABEL: check_saveexec_overwrites_vcmp_source:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_cmp_lt_i32 s16, 0
+; GCN-NEXT:    s_cbranch_scc1 .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %if
+; GCN-NEXT:    s_lshl_b32 s4, s16, 2
+; GCN-NEXT:    s_or_b32 s4, s4, s17
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_2: ; %then
+; GCN-NEXT:    v_cmp_eq_u32_e64 s4, s16, s17
+; GCN-NEXT:    s_cmp_ge_i32 s4, s17
+; GCN-NEXT:    s_cbranch_scc1 .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %after
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %0 = icmp sge i32 %a, 0
   br i1 %0, label %if, label %then
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir
new file mode 100644
index 0000000000000..90faebd6967bd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir
@@ -0,0 +1,20 @@
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck %s
+
+# Check that a trivial soft waitcnt at the end of a block is deleted even if it
+# is followed by a meta instruction.
+
+# CHECK-LABEL: name: waitcnt-no-redundant
+# CHECK: S_WAITCNT 0
+# CHECK: S_MOV_B32
+# CHECK-NOT: S_WAITCNT
+
+---
+name: waitcnt-no-redundant
+body: |
+  bb.1:
+    S_WAITCNT_soft 53119
+    $sgpr2 = S_MOV_B32 42
+    S_WAITCNT_soft 53119
+    $vgpr2 = IMPLICIT_DEF
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
index 26be45e45697f..ce01a9d669558 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
@@ -1,15 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; Check that DAGTypeLegalizer::WidenVSELECTAndMask doesn't try to
 ; create vselects with i64 condition masks.
 
 ; FIXME: Should be able to avoid intermediate vselect
-; GCN-LABEL: {{^}}widen_vselect_and_mask_v4f64:
-; GCN: v_cmp_u_f64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]],
-; GCN: v_cndmask_b32_e64 v[[VSEL:[0-9]+]], 0, -1, [[CMP]]
-; GCN: v_mov_b32_e32 v[[VSEL_EXT:[0-9]+]], v[[VSEL]]
-; GCN: v_cmp_lt_i64_e32 vcc, -1, v[[[VSEL]]:[[VSEL_EXT]]]
 define amdgpu_kernel void @widen_vselect_and_mask_v4f64(<4 x double> %arg) #0 {
+; GCN-LABEL: widen_vselect_and_mask_v4f64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], 16
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, v0
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    v_cmp_u_f64_e64 s[2:3], s[0:1], s[0:1]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, -1, s[2:3]
+; GCN-NEXT:    v_cmp_neq_f64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[1:2]
+; GCN-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GCN-NEXT:    s_cselect_b32 s0, 0x3ff00000, 0
+; GCN-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = extractelement <4 x double> %arg, i64 0
   %tmp1 = fcmp uno double %tmp, 0.000000e+00
@@ -26,12 +48,34 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_vselect_and_mask_v4i64:
-; GCN: v_cmp_eq_u64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]],
-; GCN: v_cndmask_b32_e64 v[[VSEL:[0-9]+]], 0, -1, [[CMP]]
-; GCN: v_mov_b32_e32 v[[VSEL_EXT:[0-9]+]], v[[VSEL]]
-; GCN: v_cmp_lt_i64_e32 vcc, -1, v[[[VSEL]]:[[VSEL_EXT]]]
 define amdgpu_kernel void @widen_vselect_and_mask_v4i64(<4 x i64> %arg) #0 {
+; GCN-LABEL: widen_vselect_and_mask_v4i64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_mov_b64 s[8:9], 16
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, v0
+; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[0:1], 0
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[2:3]
+; GCN-NEXT:    v_cmp_ne_u64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:    v_mov_b32_e32 v5, v4
+; GCN-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
+; GCN-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    v_mov_b32_e32 v5, v0
+; GCN-NEXT:    v_mov_b32_e32 v6, v0
+; GCN-NEXT:    v_mov_b32_e32 v7, v0
+; GCN-NEXT:    s_mov_b32 s6, s10
+; GCN-NEXT:    s_mov_b32 s7, s11
+; GCN-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = extractelement <4 x i64> %arg, i64 0
   %tmp1 = icmp eq i64 %tmp, 0
diff --git a/llvm/test/CodeGen/ARM/Windows/frexp.ll b/llvm/test/CodeGen/ARM/Windows/frexp.ll
new file mode 100644
index 0000000000000..5646c75a75979
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/Windows/frexp.ll
@@ -0,0 +1,28 @@
+; RUN: llc -mtriple thumbv7--windows-itanium < %s | FileCheck %s
+
+; CHECK-LABEL: test_frexp_f16_i32:
+; CHECK: bl __gnu_h2f_ieee
+; CHECK: vcvt.f64.f32 d0, s0
+; CHECK: bl frexp
+; CHECK: vcvt.f32.f64 s0, d0
+; CHECK: bl __gnu_f2h_ieee
+define { half, i32 } @test_frexp_f16_i32(half %a) {
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  ret { half, i32 } %result
+}
+
+; CHECK-LABEL: test_frexp_f32_i32:
+; CHECK: vcvt.f64.f32
+; CHECK: bl frexp
+; CHECK: vcvt.f32.f64	s0, d0
+define { float, i32 } @test_frexp_f32_i32(float %a) {
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+; CHECK-LABEL: test_frexp_f64_i32:
+; CHECK: bl frexp
+define { double, i32 } @test_frexp_f64_i32(double %a) {
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  ret { double, i32 } %result
+}
diff --git a/llvm/test/CodeGen/ARM/cmp-select-sign.ll b/llvm/test/CodeGen/ARM/cmp-select-sign.ll
new file mode 100644
index 0000000000000..298a623ed1d87
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cmp-select-sign.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv7a < %s | FileCheck %s --check-prefix=ARM
+; RUN: llc -mtriple=armv6m < %s | FileCheck %s --check-prefix=THUMB
+; RUN: llc -mtriple=armv7m < %s | FileCheck %s --check-prefix=THUMB2
+; RUN: llc -mtriple=thumbv8.1m.main < %s | FileCheck %s --check-prefix=THUMBV8
+
+define i3 @sign_i3(i3 %a) {
+; ARM-LABEL: sign_i3:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #29
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i3:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #29
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i3:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #29
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i3:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #29
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i3 %a, -1
+  %res = select i1 %c, i3 1, i3 -1
+  ret i3 %res
+}
+
+define i4 @sign_i4(i4 %a) {
+; ARM-LABEL: sign_i4:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #28
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i4:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #28
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i4:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #28
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i4:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #28
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i4 %a, -1
+  %res = select i1 %c, i4 1, i4 -1
+  ret i4 %res
+}
+
+define i8 @sign_i8(i8 %a) {
+; ARM-LABEL: sign_i8:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #24
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i8:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #24
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i8:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #24
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i8:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #24
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i8 %a, -1
+  %res = select i1 %c, i8 1, i8 -1
+  ret i8 %res
+}
+
+define i16 @sign_i16(i16 %a) {
+; ARM-LABEL: sign_i16:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #16
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i16:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #16
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i16:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #16
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i16:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #16
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i16 %a, -1
+  %res = select i1 %c, i16 1, i16 -1
+  ret i16 %res
+}
+
+define i32 @sign_i32(i32 %a) {
+; ARM-LABEL: sign_i32:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i32:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i32:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i32:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i32 %a, -1
+  %res = select i1 %c, i32 1, i32 -1
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/ARM/fp16_fast_math.ll b/llvm/test/CodeGen/ARM/fp16_fast_math.ll
index 4c2e3ce4efcd5..165eb4b8af43e 100644
--- a/llvm/test/CodeGen/ARM/fp16_fast_math.ll
+++ b/llvm/test/CodeGen/ARM/fp16_fast_math.ll
@@ -21,7 +21,7 @@ define half @normal_fadd(half %x, half %y) {
   ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -55,7 +55,7 @@ define half @fast_fadd(half %x, half %y) {
   ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = nnan ninf nsz arcp contract afn reassoc VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -89,7 +89,7 @@ define half @ninf_fadd(half %x, half %y) {
   ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = ninf VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -129,13 +129,13 @@ define half @normal_fadd_sequence(half %x, half %y, half %z) {
   ; CHECK-CVT-NEXT:   [[VCVTBHS2:%[0-9]+]]:spr = VCVTBHS killed [[COPY5]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
-  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY [[COPY6]]
+  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY killed [[COPY6]]
   ; CHECK-CVT-NEXT:   [[VCVTBHS3:%[0-9]+]]:spr = VCVTBHS killed [[COPY7]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[VADDS1:%[0-9]+]]:spr = VADDS killed [[VCVTBHS3]], killed [[VCVTBHS2]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF1:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH1:%[0-9]+]]:spr = VCVTBSH [[DEF1]], killed [[VADDS1]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY [[VCVTBSH1]]
+  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY killed [[VCVTBSH1]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY8]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -177,7 +177,7 @@ define half @nnan_ninf_contract_fadd_sequence(half %x, half %y, half %z) {
   ; CHECK-CVT-NEXT:   [[VADDS1:%[0-9]+]]:spr = nnan ninf contract VADDS killed [[VADDS]], killed [[VCVTBHS2]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS1]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY6]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -218,13 +218,13 @@ define half @ninf_fadd_sequence(half %x, half %y, half %z) {
   ; CHECK-CVT-NEXT:   [[VCVTBHS2:%[0-9]+]]:spr = ninf VCVTBHS killed [[COPY5]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
-  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY [[COPY6]]
+  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY killed [[COPY6]]
   ; CHECK-CVT-NEXT:   [[VCVTBHS3:%[0-9]+]]:spr = ninf VCVTBHS killed [[COPY7]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[VADDS1:%[0-9]+]]:spr = ninf VADDS killed [[VCVTBHS3]], killed [[VCVTBHS2]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF1:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH1:%[0-9]+]]:spr = VCVTBSH [[DEF1]], killed [[VADDS1]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY [[VCVTBSH1]]
+  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY killed [[VCVTBSH1]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY8]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
diff --git a/llvm/test/CodeGen/ARM/fpclamptosat.ll b/llvm/test/CodeGen/ARM/fpclamptosat.ll
index 478b98dfac80f..8ab56b228d2a7 100644
--- a/llvm/test/CodeGen/ARM/fpclamptosat.ll
+++ b/llvm/test/CodeGen/ARM/fpclamptosat.ll
@@ -613,14 +613,21 @@ define i16 @ustest_f64i16(double %x) {
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    vmov r0, r1, d0
 ; VFP2-NEXT:    bl __aeabi_d2iz
-; VFP2-NEXT:    usat r0, #16, r0
+; VFP2-NEXT:    movw r1, #65535
+; VFP2-NEXT:    cmp r0, r1
+; VFP2-NEXT:    it lt
+; VFP2-NEXT:    movlt r1, r0
+; VFP2-NEXT:    bic.w r0, r1, r1, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f64i16:
 ; FULL:       @ %bb.0: @ %entry
 ; FULL-NEXT:    vcvt.s32.f64 s0, d0
+; FULL-NEXT:    movw r1, #65535
 ; FULL-NEXT:    vmov r0, s0
-; FULL-NEXT:    usat r0, #16, r0
+; FULL-NEXT:    cmp r0, r1
+; FULL-NEXT:    csel r0, r0, r1, lt
+; FULL-NEXT:    bic.w r0, r0, r0, asr #31
 ; FULL-NEXT:    bx lr
 entry:
   %conv = fptosi double %x to i32
@@ -738,12 +745,26 @@ define i16 @ustest_f32i16(float %x) {
 ; SOFT-NEXT:  .LCPI14_0:
 ; SOFT-NEXT:    .long 65535 @ 0xffff
 ;
-; VFP-LABEL: ustest_f32i16:
-; VFP:       @ %bb.0: @ %entry
-; VFP-NEXT:    vcvt.s32.f32 s0, s0
-; VFP-NEXT:    vmov r0, s0
-; VFP-NEXT:    usat r0, #16, r0
-; VFP-NEXT:    bx lr
+; VFP2-LABEL: ustest_f32i16:
+; VFP2:       @ %bb.0: @ %entry
+; VFP2-NEXT:    vcvt.s32.f32 s0, s0
+; VFP2-NEXT:    movw r1, #65535
+; VFP2-NEXT:    vmov r0, s0
+; VFP2-NEXT:    cmp r0, r1
+; VFP2-NEXT:    it lt
+; VFP2-NEXT:    movlt r1, r0
+; VFP2-NEXT:    bic.w r0, r1, r1, asr #31
+; VFP2-NEXT:    bx lr
+;
+; FULL-LABEL: ustest_f32i16:
+; FULL:       @ %bb.0: @ %entry
+; FULL-NEXT:    vcvt.s32.f32 s0, s0
+; FULL-NEXT:    movw r1, #65535
+; FULL-NEXT:    vmov r0, s0
+; FULL-NEXT:    cmp r0, r1
+; FULL-NEXT:    csel r0, r0, r1, lt
+; FULL-NEXT:    bic.w r0, r0, r0, asr #31
+; FULL-NEXT:    bx lr
 entry:
   %conv = fptosi float %x to i32
   %0 = icmp slt i32 %conv, 65535
@@ -890,16 +911,23 @@ define i16 @ustest_f16i16(half %x) {
 ; VFP2-NEXT:    vmov r0, s0
 ; VFP2-NEXT:    bl __aeabi_h2f
 ; VFP2-NEXT:    vmov s0, r0
+; VFP2-NEXT:    movw r1, #65535
 ; VFP2-NEXT:    vcvt.s32.f32 s0, s0
 ; VFP2-NEXT:    vmov r0, s0
-; VFP2-NEXT:    usat r0, #16, r0
+; VFP2-NEXT:    cmp r0, r1
+; VFP2-NEXT:    it lt
+; VFP2-NEXT:    movlt r1, r0
+; VFP2-NEXT:    bic.w r0, r1, r1, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f16i16:
 ; FULL:       @ %bb.0: @ %entry
 ; FULL-NEXT:    vcvt.s32.f16 s0, s0
+; FULL-NEXT:    movw r1, #65535
 ; FULL-NEXT:    vmov r0, s0
-; FULL-NEXT:    usat r0, #16, r0
+; FULL-NEXT:    cmp r0, r1
+; FULL-NEXT:    csel r0, r0, r1, lt
+; FULL-NEXT:    bic.w r0, r0, r0, asr #31
 ; FULL-NEXT:    bx lr
 entry:
   %conv = fptosi half %x to i32
@@ -1101,83 +1129,48 @@ entry:
 define i64 @ustest_f64i64(double %x) {
 ; SOFT-LABEL: ustest_f64i64:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r6, lr}
-; SOFT-NEXT:    push {r4, r5, r6, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    subs r6, r2, #1
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    bge .LBB20_9
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    subs r2, r2, #1
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    sbcs r2, r4
+; SOFT-NEXT:    bge .LBB20_5
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    bge .LBB20_10
+; SOFT-NEXT:    bge .LBB20_6
 ; SOFT-NEXT:  .LBB20_2: @ %entry
-; SOFT-NEXT:    bge .LBB20_11
+; SOFT-NEXT:    blt .LBB20_4
 ; SOFT-NEXT:  .LBB20_3: @ %entry
-; SOFT-NEXT:    blt .LBB20_5
+; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:  .LBB20_4: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    asrs r2, r3, #31
+; SOFT-NEXT:    bics r0, r2
+; SOFT-NEXT:    bics r1, r2
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB20_5: @ %entry
-; SOFT-NEXT:    rsbs r6, r0, #0
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r1
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r2
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB20_12
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB20_13
-; SOFT-NEXT:  .LBB20_7: @ %entry
-; SOFT-NEXT:    beq .LBB20_14
-; SOFT-NEXT:  .LBB20_8: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB20_9: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    blt .LBB20_2
-; SOFT-NEXT:  .LBB20_10: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    blt .LBB20_3
-; SOFT-NEXT:  .LBB20_11: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bge .LBB20_4
-; SOFT-NEXT:    b .LBB20_5
-; SOFT-NEXT:  .LBB20_12: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB20_7
-; SOFT-NEXT:  .LBB20_13: @ %entry
+; SOFT-NEXT:  .LBB20_6: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB20_8
-; SOFT-NEXT:  .LBB20_14: @ %entry
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
+; SOFT-NEXT:    bge .LBB20_3
+; SOFT-NEXT:    b .LBB20_4
 ;
 ; VFP2-LABEL: ustest_f64i64:
 ; VFP2:       @ %bb.0: @ %entry
 ; VFP2-NEXT:    .save {r7, lr}
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    bl __fixdfti
-; VFP2-NEXT:    subs.w lr, r2, #1
+; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
-; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    sbcs r2, r3, #0
+; VFP2-NEXT:    itt ge
 ; VFP2-NEXT:    movge r3, r12
-; VFP2-NEXT:    movge r2, #1
-; VFP2-NEXT:    movge r1, r12
 ; VFP2-NEXT:    movge r0, r12
-; VFP2-NEXT:    rsbs.w lr, r0, #0
-; VFP2-NEXT:    sbcs.w lr, r12, r1
-; VFP2-NEXT:    sbcs.w r2, r12, r2
-; VFP2-NEXT:    sbcs.w r2, r12, r3
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w r12, #1
-; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    bic.w r0, r0, r3, asr #31
+; VFP2-NEXT:    bic.w r1, r1, r3, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f64i64:
@@ -1185,22 +1178,14 @@ define i64 @ustest_f64i64(double %x) {
 ; FULL-NEXT:    .save {r7, lr}
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixdfti
-; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    subs r2, #1
 ; FULL-NEXT:    mov.w r12, #0
-; FULL-NEXT:    sbcs lr, r3, #0
-; FULL-NEXT:    it ge
-; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    sbcs r2, r3, #0
+; FULL-NEXT:    csel r2, r3, r12, lt
 ; FULL-NEXT:    csel r0, r0, r12, lt
-; FULL-NEXT:    csel lr, r3, r12, lt
 ; FULL-NEXT:    csel r1, r1, r12, lt
-; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, r12, r1
-; FULL-NEXT:    sbcs.w r2, r12, r2
-; FULL-NEXT:    sbcs.w r2, r12, lr
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    csel r1, r1, r2, ne
+; FULL-NEXT:    bic.w r0, r0, r2, asr #31
+; FULL-NEXT:    bic.w r1, r1, r2, asr #31
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptosi double %x to i128
@@ -1400,83 +1385,48 @@ entry:
 define i64 @ustest_f32i64(float %x) {
 ; SOFT-LABEL: ustest_f32i64:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r6, lr}
-; SOFT-NEXT:    push {r4, r5, r6, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    subs r6, r2, #1
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    bge .LBB23_9
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    subs r2, r2, #1
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    sbcs r2, r4
+; SOFT-NEXT:    bge .LBB23_5
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    bge .LBB23_10
+; SOFT-NEXT:    bge .LBB23_6
 ; SOFT-NEXT:  .LBB23_2: @ %entry
-; SOFT-NEXT:    bge .LBB23_11
+; SOFT-NEXT:    blt .LBB23_4
 ; SOFT-NEXT:  .LBB23_3: @ %entry
-; SOFT-NEXT:    blt .LBB23_5
+; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:  .LBB23_4: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    asrs r2, r3, #31
+; SOFT-NEXT:    bics r0, r2
+; SOFT-NEXT:    bics r1, r2
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB23_5: @ %entry
-; SOFT-NEXT:    rsbs r6, r0, #0
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r1
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r2
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB23_12
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB23_13
-; SOFT-NEXT:  .LBB23_7: @ %entry
-; SOFT-NEXT:    beq .LBB23_14
-; SOFT-NEXT:  .LBB23_8: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB23_9: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    blt .LBB23_2
-; SOFT-NEXT:  .LBB23_10: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    blt .LBB23_3
-; SOFT-NEXT:  .LBB23_11: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bge .LBB23_4
-; SOFT-NEXT:    b .LBB23_5
-; SOFT-NEXT:  .LBB23_12: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB23_7
-; SOFT-NEXT:  .LBB23_13: @ %entry
+; SOFT-NEXT:  .LBB23_6: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB23_8
-; SOFT-NEXT:  .LBB23_14: @ %entry
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
+; SOFT-NEXT:    bge .LBB23_3
+; SOFT-NEXT:    b .LBB23_4
 ;
 ; VFP2-LABEL: ustest_f32i64:
 ; VFP2:       @ %bb.0: @ %entry
 ; VFP2-NEXT:    .save {r7, lr}
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    bl __fixsfti
-; VFP2-NEXT:    subs.w lr, r2, #1
+; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
-; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    sbcs r2, r3, #0
+; VFP2-NEXT:    itt ge
 ; VFP2-NEXT:    movge r3, r12
-; VFP2-NEXT:    movge r2, #1
-; VFP2-NEXT:    movge r1, r12
 ; VFP2-NEXT:    movge r0, r12
-; VFP2-NEXT:    rsbs.w lr, r0, #0
-; VFP2-NEXT:    sbcs.w lr, r12, r1
-; VFP2-NEXT:    sbcs.w r2, r12, r2
-; VFP2-NEXT:    sbcs.w r2, r12, r3
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w r12, #1
-; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    bic.w r0, r0, r3, asr #31
+; VFP2-NEXT:    bic.w r1, r1, r3, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f32i64:
@@ -1484,22 +1434,14 @@ define i64 @ustest_f32i64(float %x) {
 ; FULL-NEXT:    .save {r7, lr}
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixsfti
-; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    subs r2, #1
 ; FULL-NEXT:    mov.w r12, #0
-; FULL-NEXT:    sbcs lr, r3, #0
-; FULL-NEXT:    it ge
-; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    sbcs r2, r3, #0
+; FULL-NEXT:    csel r2, r3, r12, lt
 ; FULL-NEXT:    csel r0, r0, r12, lt
-; FULL-NEXT:    csel lr, r3, r12, lt
 ; FULL-NEXT:    csel r1, r1, r12, lt
-; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, r12, r1
-; FULL-NEXT:    sbcs.w r2, r12, r2
-; FULL-NEXT:    sbcs.w r2, r12, lr
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    csel r1, r1, r2, ne
+; FULL-NEXT:    bic.w r0, r0, r2, asr #31
+; FULL-NEXT:    bic.w r1, r1, r2, asr #31
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptosi float %x to i128
@@ -1713,61 +1655,34 @@ entry:
 define i64 @ustest_f16i64(half %x) {
 ; SOFT-LABEL: ustest_f16i64:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r6, lr}
-; SOFT-NEXT:    push {r4, r5, r6, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    subs r6, r2, #1
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    bge .LBB26_9
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    subs r2, r2, #1
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    sbcs r2, r4
+; SOFT-NEXT:    bge .LBB26_5
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    bge .LBB26_10
+; SOFT-NEXT:    bge .LBB26_6
 ; SOFT-NEXT:  .LBB26_2: @ %entry
-; SOFT-NEXT:    bge .LBB26_11
+; SOFT-NEXT:    blt .LBB26_4
 ; SOFT-NEXT:  .LBB26_3: @ %entry
-; SOFT-NEXT:    blt .LBB26_5
+; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:  .LBB26_4: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    asrs r2, r3, #31
+; SOFT-NEXT:    bics r0, r2
+; SOFT-NEXT:    bics r1, r2
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB26_5: @ %entry
-; SOFT-NEXT:    rsbs r6, r0, #0
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r1
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r2
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB26_12
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_13
-; SOFT-NEXT:  .LBB26_7: @ %entry
-; SOFT-NEXT:    beq .LBB26_14
-; SOFT-NEXT:  .LBB26_8: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB26_9: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    blt .LBB26_2
-; SOFT-NEXT:  .LBB26_10: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    blt .LBB26_3
-; SOFT-NEXT:  .LBB26_11: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bge .LBB26_4
-; SOFT-NEXT:    b .LBB26_5
-; SOFT-NEXT:  .LBB26_12: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB26_7
-; SOFT-NEXT:  .LBB26_13: @ %entry
+; SOFT-NEXT:  .LBB26_6: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB26_8
-; SOFT-NEXT:  .LBB26_14: @ %entry
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
+; SOFT-NEXT:    bge .LBB26_3
+; SOFT-NEXT:    b .LBB26_4
 ;
 ; VFP2-LABEL: ustest_f16i64:
 ; VFP2:       @ %bb.0: @ %entry
@@ -1777,24 +1692,16 @@ define i64 @ustest_f16i64(half %x) {
 ; VFP2-NEXT:    bl __aeabi_h2f
 ; VFP2-NEXT:    vmov s0, r0
 ; VFP2-NEXT:    bl __fixsfti
-; VFP2-NEXT:    subs.w lr, r2, #1
+; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
-; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    sbcs r2, r3, #0
+; VFP2-NEXT:    itt ge
 ; VFP2-NEXT:    movge r3, r12
-; VFP2-NEXT:    movge r2, #1
-; VFP2-NEXT:    movge r1, r12
 ; VFP2-NEXT:    movge r0, r12
-; VFP2-NEXT:    rsbs.w lr, r0, #0
-; VFP2-NEXT:    sbcs.w lr, r12, r1
-; VFP2-NEXT:    sbcs.w r2, r12, r2
-; VFP2-NEXT:    sbcs.w r2, r12, r3
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w r12, #1
-; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    bic.w r0, r0, r3, asr #31
+; VFP2-NEXT:    bic.w r1, r1, r3, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f16i64:
@@ -1804,22 +1711,14 @@ define i64 @ustest_f16i64(half %x) {
 ; FULL-NEXT:    vmov.f16 r0, s0
 ; FULL-NEXT:    vmov s0, r0
 ; FULL-NEXT:    bl __fixhfti
-; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    subs r2, #1
 ; FULL-NEXT:    mov.w r12, #0
-; FULL-NEXT:    sbcs lr, r3, #0
-; FULL-NEXT:    it ge
-; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    sbcs r2, r3, #0
+; FULL-NEXT:    csel r2, r3, r12, lt
 ; FULL-NEXT:    csel r0, r0, r12, lt
-; FULL-NEXT:    csel lr, r3, r12, lt
 ; FULL-NEXT:    csel r1, r1, r12, lt
-; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, r12, r1
-; FULL-NEXT:    sbcs.w r2, r12, r2
-; FULL-NEXT:    sbcs.w r2, r12, lr
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    csel r1, r1, r2, ne
+; FULL-NEXT:    bic.w r0, r0, r2, asr #31
+; FULL-NEXT:    bic.w r1, r1, r2, asr #31
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptosi half %x to i128
diff --git a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
index 4d091c2302658..96f009a4da02d 100644
--- a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
@@ -174,7 +174,7 @@ define <2 x i32> @ustest_f64i32(<2 x double> %x) {
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    mvnne r2, #0
 ; CHECK-NEXT:    vmov.32 d18[0], r2
-; CHECK-NEXT:    vand q8, q8, q9
+; CHECK-NEXT:    vand q8, q9, q8
 ; CHECK-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r11, pc}
@@ -483,8 +483,8 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) {
 ; CHECK-NEXT:    vmov.32 d20[0], r7
 ; CHECK-NEXT:    mvnne r4, #0
 ; CHECK-NEXT:    vmov.32 d18[0], r4
-; CHECK-NEXT:    vand q10, q4, q10
-; CHECK-NEXT:    vand q8, q8, q9
+; CHECK-NEXT:    vand q10, q10, q4
+; CHECK-NEXT:    vand q8, q9, q8
 ; CHECK-NEXT:    vmovn.i64 d1, q10
 ; CHECK-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
@@ -995,8 +995,8 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    vmov.32 d20[0], r6
 ; CHECK-NEON-NEXT:    mvnne r7, #0
 ; CHECK-NEON-NEXT:    vmov.32 d18[0], r7
-; CHECK-NEON-NEXT:    vand q10, q4, q10
-; CHECK-NEON-NEXT:    vand q8, q8, q9
+; CHECK-NEON-NEXT:    vand q10, q10, q4
+; CHECK-NEON-NEXT:    vand q8, q9, q8
 ; CHECK-NEON-NEXT:    vmovn.i64 d1, q10
 ; CHECK-NEON-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEON-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
@@ -1097,8 +1097,8 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-FP16-NEXT:    vmov.32 d20[0], r7
 ; CHECK-FP16-NEXT:    mvnne r6, #0
 ; CHECK-FP16-NEXT:    vmov.32 d18[0], r6
-; CHECK-FP16-NEXT:    vand q10, q4, q10
-; CHECK-FP16-NEXT:    vand q8, q8, q9
+; CHECK-FP16-NEXT:    vand q10, q10, q4
+; CHECK-FP16-NEXT:    vand q8, q9, q8
 ; CHECK-FP16-NEXT:    vmovn.i64 d1, q10
 ; CHECK-FP16-NEXT:    vmovn.i64 d0, q8
 ; CHECK-FP16-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
diff --git a/llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll b/llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll
new file mode 100644
index 0000000000000..7a02dbd9d0d11
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll
@@ -0,0 +1,8 @@
+; RUN: not llc -mtriple arm-linux-gnu -float-abi=soft -filetype=null %s 2>&1 | FileCheck %s
+
+; FIXME: This should not fail but isn't implemented
+; CHECK: error: ffrexp exponent does not match sizeof(int)
+define { float, i64 } @soften_frexp_error(float %x) {
+  %frexp = call { float, i64 } @llvm.frexp.f32.i64(float %x)
+  ret { float, i64 } %frexp
+}
diff --git a/llvm/test/CodeGen/ARM/iabs.ll b/llvm/test/CodeGen/ARM/iabs.ll
index fffa9555b2966..758fe7507c0b2 100644
--- a/llvm/test/CodeGen/ARM/iabs.ll
+++ b/llvm/test/CodeGen/ARM/iabs.ll
@@ -50,3 +50,28 @@ define i64 @test3(i64 %a) {
   %abs = select i1 %b, i64 %a, i64 %tmp1neg
   ret i64 %abs
 }
+
+declare void @callee(...)
+
+define void @testcallframe(i32 %a) {
+; CHECK-LABEL: testcallframe:
+; CHECK:       @ %bb.0: @ %bb
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, sp, #8
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    rsbmi r0, r0, #0
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    mov r3, #0
+; CHECK-NEXT:    str r1, [sp]
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    add sp, sp, #8
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    bx lr
+bb:
+  %i = tail call i32 @llvm.abs.i32(i32 %a, i1 false)
+  tail call void @callee(i32 %i, i32 0, i32 0, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/ldexp.ll b/llvm/test/CodeGen/ARM/ldexp.ll
index 941390ee94c1e..cdf91eb902e05 100644
--- a/llvm/test/CodeGen/ARM/ldexp.ll
+++ b/llvm/test/CodeGen/ARM/ldexp.ll
@@ -5,7 +5,7 @@ define double @testExp(double %val, i32 %a) {
 ; LINUX:    b ldexp{{$}}
 ; WINDOWS:  b.w ldexp{{$}}
 entry:
-  %call = tail call fast double @ldexp(double %val, i32 %a)
+  %call = tail call double @ldexp(double %val, i32 %a)
   ret double %call
 }
 
@@ -15,7 +15,7 @@ define double @testExpIntrinsic(double %val, i32 %a) {
 ; LINUX:    b ldexp{{$}}
 ; WINDOWS:  b.w ldexp{{$}}
 entry:
-  %call = tail call fast double @llvm.ldexp.f64(double %val, i32 %a)
+  %call = tail call double @llvm.ldexp.f64(double %val, i32 %a)
   ret double %call
 }
 
@@ -23,7 +23,7 @@ define float @testExpf(float %val, i32 %a) {
 ; LINUX:    b ldexpf
 ; WINDOWS:  b.w ldexpf
 entry:
-  %call = tail call fast float @ldexpf(float %val, i32 %a)
+  %call = tail call float @ldexpf(float %val, i32 %a)
   ret float %call
 }
 
@@ -31,7 +31,7 @@ define float @testExpfIntrinsic(float %val, i32 %a) {
 ; LINUX:    b ldexpf
 ; WINDOWS:  bl ldexp{{$}}
 entry:
-  %call = tail call fast float @llvm.ldexp.f32(float %val, i32 %a)
+  %call = tail call float @llvm.ldexp.f32(float %val, i32 %a)
   ret float %call
 }
 
@@ -41,7 +41,7 @@ define fp128 @testExpl(fp128 %val, i32 %a) {
 ; LINUX:    bl ldexpl
 ; WINDOWS:    b.w ldexpl
 entry:
-  %call = tail call fast fp128 @ldexpl(fp128 %val, i32 %a)
+  %call = tail call fp128 @ldexpl(fp128 %val, i32 %a)
   ret fp128 %call
 }
 
@@ -51,7 +51,7 @@ define half @testExpf16(half %val, i32 %a) {
 ; LINUX: bl ldexpf
 ; WINDOWS: bl ldexp{{$}}
 entry:
-  %0 = tail call fast half @llvm.ldexp.f16.i32(half %val, i32 %a)
+  %0 = tail call half @llvm.ldexp.f16.i32(half %val, i32 %a)
   ret half %0
 }
 
diff --git a/llvm/test/CodeGen/ARM/llvm.frexp.ll b/llvm/test/CodeGen/ARM/llvm.frexp.ll
index e79ddbe93336e..43edb17fe1081 100644
--- a/llvm/test/CodeGen/ARM/llvm.frexp.ll
+++ b/llvm/test/CodeGen/ARM/llvm.frexp.ll
@@ -141,6 +141,43 @@ define <2 x i32> @test_frexp_v2f16_v2i32_only_use_exp(<2 x half> %a) {
   ret <2 x i32> %result.1
 }
 
+define { <3 x float>, <3 x i32> } @test_frexp_v3f32_v3i32(<3 x float> %a) {
+; CHECK-LABEL: test_frexp_v3f32_v3i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    mov r8, sp
+; CHECK-NEXT:    vldr d9, [sp, #48]
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r5, r3
+; CHECK-NEXT:    vmov d8, r2, r3
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    add r6, sp, #4
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vld1.32 {d16[0]}, [r8:32]
+; CHECK-NEXT:    add.w r1, r4, #16
+; CHECK-NEXT:    vld1.32 {d16[1]}, [r6:32]
+; CHECK-NEXT:    vst1.32 {d16}, [r1:64]!
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    vmov s1, r5
+; CHECK-NEXT:    vmov s0, r7
+; CHECK-NEXT:    vst1.32 {d0}, [r4:64]!
+; CHECK-NEXT:    str r0, [r4]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+  %result = call { <3 x float>, <3 x i32> } @llvm.frexp.v3f32.v3i32(<3 x float> %a)
+  ret { <3 x float>, <3 x i32> } %result
+}
+
 define { float, i32 } @test_frexp_f32_i32(float %a) {
 ; CHECK-LABEL: test_frexp_f32_i32:
 ; CHECK:       @ %bb.0:
diff --git a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
index ec7516524ee67..6706d25ae01d2 100644
--- a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
@@ -1362,7 +1362,7 @@ define void @pr65820(ptr %y, <4 x float> %splat) {
 ; ARMV7-NEXT:    vmov d16, r2, r3
 ; ARMV7-NEXT:    vdup.32 q8, d16[0]
 ; ARMV7-NEXT:    vcgt.f32 q9, q8, #0
-; ARMV7-NEXT:    vand q8, q8, q9
+; ARMV7-NEXT:    vand q8, q9, q8
 ; ARMV7-NEXT:    vst1.32 {d16, d17}, [r0]
 ; ARMV7-NEXT:    bx lr
 ;
diff --git a/llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll b/llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll
new file mode 100644
index 0000000000000..1feb544d42e74
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll
@@ -0,0 +1,8 @@
+; RUN: not llc -mtriple=arm-linux-gnu -float-abi=soft -filetype=null %s 2>&1 | FileCheck %s
+
+; FIXME: This should not fail but isn't implemented
+; CHECK: error: powi exponent does not match sizeof(int)
+define float @soften_powi_error(float %x, i64 %n) {
+  %powi = call float @llvm.powi.f32.i64(float %x, i64 %n)
+  ret float %powi
+}
diff --git a/llvm/test/CodeGen/ARM/usat-with-shift.ll b/llvm/test/CodeGen/ARM/usat-with-shift.ll
index cc3de9d6d3407..b9c083e498c0c 100644
--- a/llvm/test/CodeGen/ARM/usat-with-shift.ll
+++ b/llvm/test/CodeGen/ARM/usat-with-shift.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=armv6-eabi %s -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+dsp %s -o - | FileCheck %s
+; RUN: llc -mtriple=armv6-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,ARMV6
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+dsp %s -o - | FileCheck %s --check-prefixes=CHECK,THUMB
 
 define arm_aapcs_vfpcc i32 @usat_lsl(i32 %num){
-; CHECK-LABEL:  usat_lsl
-; CHECK:        @ %bb.0: @ %entry
-; CHECK-NEXT: 	usat	r0, #7, r0, lsl #2
-; CHECK-NEXT:   bx	lr
+; CHECK-LABEL: usat_lsl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    usat r0, #7, r0, lsl #2
+; CHECK-NEXT:    bx lr
 entry:
   %shl = shl i32 %num, 2
   %0 = tail call i32 @llvm.arm.usat(i32 %shl, i32 7)
@@ -14,10 +14,10 @@ entry:
 }
 
 define arm_aapcs_vfpcc i32 @usat_asr(i32 %num){
-; CHECK-LABEL:  usat_asr
-; CHECK:        @ %bb.0: @ %entry
-; CHECK-NEXT:   usat	r0, #7, r0, asr #2
-; CHECK-NEXT:   bx	lr
+; CHECK-LABEL: usat_asr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    usat r0, #7, r0, asr #2
+; CHECK-NEXT:    bx lr
 entry:
   %shr = ashr i32 %num, 2
   %0 = tail call i32 @llvm.arm.usat(i32 %shr, i32 7)
@@ -25,10 +25,24 @@ entry:
 }
 
 define arm_aapcs_vfpcc i32 @usat_lsl2(i32 %num){
-; CHECK-LABEL: usat_lsl2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    usat r0, #15, r0, lsl #15
-; CHECK-NEXT:    bx lr
+; ARMV6-LABEL: usat_lsl2:
+; ARMV6:       @ %bb.0: @ %entry
+; ARMV6-NEXT:    lsl r0, r0, #15
+; ARMV6-NEXT:    bic r1, r0, r0, asr #31
+; ARMV6-NEXT:    mov r0, #255
+; ARMV6-NEXT:    orr r0, r0, #32512
+; ARMV6-NEXT:    cmp r1, r0
+; ARMV6-NEXT:    movlt r0, r1
+; ARMV6-NEXT:    bx lr
+;
+; THUMB-LABEL: usat_lsl2:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    lsls r0, r0, #15
+; THUMB-NEXT:    movw r1, #32767
+; THUMB-NEXT:    bic.w r0, r0, r0, asr #31
+; THUMB-NEXT:    cmp r0, r1
+; THUMB-NEXT:    csel r0, r0, r1, lt
+; THUMB-NEXT:    bx lr
 entry:
   %shl = shl nsw i32 %num, 15
   %0 = icmp sgt i32 %shl, 0
@@ -39,10 +53,24 @@ entry:
 }
 
 define arm_aapcs_vfpcc i32 @usat_asr2(i32 %num){
-; CHECK-LABEL: usat_asr2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    usat r0, #15, r0, asr #15
-; CHECK-NEXT:    bx lr
+; ARMV6-LABEL: usat_asr2:
+; ARMV6:       @ %bb.0: @ %entry
+; ARMV6-NEXT:    asr r1, r0, #15
+; ARMV6-NEXT:    bic r1, r1, r0, asr #31
+; ARMV6-NEXT:    mov r0, #255
+; ARMV6-NEXT:    orr r0, r0, #32512
+; ARMV6-NEXT:    cmp r1, r0
+; ARMV6-NEXT:    movlt r0, r1
+; ARMV6-NEXT:    bx lr
+;
+; THUMB-LABEL: usat_asr2:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    asrs r1, r0, #15
+; THUMB-NEXT:    bic.w r0, r1, r0, asr #31
+; THUMB-NEXT:    movw r1, #32767
+; THUMB-NEXT:    cmp r0, r1
+; THUMB-NEXT:    csel r0, r0, r1, lt
+; THUMB-NEXT:    bx lr
 entry:
   %shr = ashr i32 %num, 15
   %0 = icmp sgt i32 %shr, 0
diff --git a/llvm/test/CodeGen/ARM/usat.ll b/llvm/test/CodeGen/ARM/usat.ll
index d01aa1520b326..2e1d0283ebde2 100644
--- a/llvm/test/CodeGen/ARM/usat.ll
+++ b/llvm/test/CodeGen/ARM/usat.ll
@@ -32,12 +32,23 @@ define i32 @unsigned_sat_base_32bit(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_base_32bit:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    ldr r1, .LCPI0_0
+; V6-NEXT:    cmp r0, r1
+; V6-NEXT:    movlt r1, r0
+; V6-NEXT:    bic r0, r1, r1, asr #31
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI0_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_base_32bit:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    movw r1, #65535
+; V6T2-NEXT:    movt r1, #127
+; V6T2-NEXT:    cmp r0, r1
+; V6T2-NEXT:    movlt r1, r0
+; V6T2-NEXT:    bic r0, r1, r1, asr #31
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i32 %x, 8388607
@@ -57,11 +68,9 @@ define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; V4T-NEXT:    orr r2, r2, #1792
 ; V4T-NEXT:    asr r1, r1, #16
 ; V4T-NEXT:    cmp r1, r2
-; V4T-NEXT:    movge r0, r2
-; V4T-NEXT:    lsl r1, r0, #16
-; V4T-NEXT:    asr r1, r1, #16
-; V4T-NEXT:    cmp r1, #0
-; V4T-NEXT:    movle r0, #0
+; V4T-NEXT:    movlt r2, r0
+; V4T-NEXT:    lsl r0, r2, #16
+; V4T-NEXT:    bic r0, r2, r0, asr #31
 ; V4T-NEXT:    bx lr
 ;
 ; V6-LABEL: unsigned_sat_base_16bit:
@@ -70,10 +79,9 @@ define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; V6-NEXT:    sxth r1, r0
 ; V6-NEXT:    orr r2, r2, #1792
 ; V6-NEXT:    cmp r1, r2
-; V6-NEXT:    movge r0, r2
-; V6-NEXT:    sxth r1, r0
-; V6-NEXT:    cmp r1, #0
-; V6-NEXT:    movle r0, #0
+; V6-NEXT:    movlt r2, r0
+; V6-NEXT:    sxth r0, r2
+; V6-NEXT:    bic r0, r2, r0, asr #15
 ; V6-NEXT:    bx lr
 ;
 ; V6T2-LABEL: unsigned_sat_base_16bit:
@@ -81,10 +89,9 @@ define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; V6T2-NEXT:    sxth r1, r0
 ; V6T2-NEXT:    movw r2, #2047
 ; V6T2-NEXT:    cmp r1, r2
-; V6T2-NEXT:    movge r0, r2
-; V6T2-NEXT:    sxth r1, r0
-; V6T2-NEXT:    cmp r1, #0
-; V6T2-NEXT:    movle r0, #0
+; V6T2-NEXT:    movlt r2, r0
+; V6T2-NEXT:    sxth r0, r2
+; V6T2-NEXT:    bic r0, r2, r0, asr #15
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i16 %x, 2047
@@ -104,9 +111,7 @@ define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; V4T-NEXT:    cmp r1, #31
 ; V4T-NEXT:    movge r0, #31
 ; V4T-NEXT:    lsl r1, r0, #24
-; V4T-NEXT:    asr r1, r1, #24
-; V4T-NEXT:    cmp r1, #0
-; V4T-NEXT:    movle r0, #0
+; V4T-NEXT:    bic r0, r0, r1, asr #31
 ; V4T-NEXT:    bx lr
 ;
 ; V6-LABEL: unsigned_sat_base_8bit:
@@ -115,8 +120,7 @@ define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; V6-NEXT:    cmp r1, #31
 ; V6-NEXT:    movge r0, #31
 ; V6-NEXT:    sxtb r1, r0
-; V6-NEXT:    cmp r1, #0
-; V6-NEXT:    movle r0, #0
+; V6-NEXT:    bic r0, r0, r1, asr #7
 ; V6-NEXT:    bx lr
 ;
 ; V6T2-LABEL: unsigned_sat_base_8bit:
@@ -125,8 +129,7 @@ define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; V6T2-NEXT:    cmp r1, #31
 ; V6T2-NEXT:    movge r0, #31
 ; V6T2-NEXT:    sxtb r1, r0
-; V6T2-NEXT:    cmp r1, #0
-; V6T2-NEXT:    movle r0, #0
+; V6T2-NEXT:    bic r0, r0, r1, asr #7
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i8 %x, 31
@@ -157,12 +160,23 @@ define i32 @unsigned_sat_lower_upper_1(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_lower_upper_1:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    ldr r1, .LCPI3_0
+; V6-NEXT:    cmp r0, r1
+; V6-NEXT:    movlt r1, r0
+; V6-NEXT:    bic r0, r1, r1, asr #31
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI3_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_lower_upper_1:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    movw r1, #65535
+; V6T2-NEXT:    movt r1, #127
+; V6T2-NEXT:    cmp r0, r1
+; V6T2-NEXT:    movlt r1, r0
+; V6T2-NEXT:    bic r0, r1, r1, asr #31
 ; V6T2-NEXT:    bx lr
 entry:
   %cmpUp = icmp slt i32 %x, 8388607
@@ -188,12 +202,23 @@ define i32 @unsigned_sat_lower_upper_2(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_lower_upper_2:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    ldr r1, .LCPI4_0
+; V6-NEXT:    cmp r0, r1
+; V6-NEXT:    movlt r1, r0
+; V6-NEXT:    bic r0, r1, r1, asr #31
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI4_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_lower_upper_2:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    movw r1, #65535
+; V6T2-NEXT:    movt r1, #127
+; V6T2-NEXT:    cmp r0, r1
+; V6T2-NEXT:    movlt r1, r0
+; V6T2-NEXT:    bic r0, r1, r1, asr #31
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i32 %x, 8388607
@@ -219,12 +244,23 @@ define i32 @unsigned_sat_upper_lower_1(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_upper_lower_1:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    bic r1, r0, r0, asr #31
+; V6-NEXT:    ldr r0, .LCPI5_0
+; V6-NEXT:    cmp r1, r0
+; V6-NEXT:    movlt r0, r1
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI5_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_upper_lower_1:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    bic r1, r0, r0, asr #31
+; V6T2-NEXT:    movw r0, #65535
+; V6T2-NEXT:    movt r0, #127
+; V6T2-NEXT:    cmp r1, r0
+; V6T2-NEXT:    movlt r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp sgt i32 %x, 0
@@ -250,12 +286,23 @@ define i32 @unsigned_sat_upper_lower_2(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_upper_lower_2:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    bic r1, r0, r0, asr #31
+; V6-NEXT:    ldr r0, .LCPI6_0
+; V6-NEXT:    cmp r1, r0
+; V6-NEXT:    movlt r0, r1
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI6_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_upper_lower_2:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    bic r1, r0, r0, asr #31
+; V6T2-NEXT:    movw r0, #65535
+; V6T2-NEXT:    movt r0, #127
+; V6T2-NEXT:    cmp r1, r0
+; V6T2-NEXT:    movlt r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp sgt i32 %x, 0
@@ -281,12 +328,23 @@ define i32 @unsigned_sat_upper_lower_3(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_upper_lower_3:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    bic r1, r0, r0, asr #31
+; V6-NEXT:    ldr r0, .LCPI7_0
+; V6-NEXT:    cmp r1, r0
+; V6-NEXT:    movlt r0, r1
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI7_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_upper_lower_3:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    bic r1, r0, r0, asr #31
+; V6T2-NEXT:    movw r0, #65535
+; V6T2-NEXT:    movt r0, #127
+; V6T2-NEXT:    cmp r1, r0
+; V6T2-NEXT:    movlt r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %cmpLow = icmp sgt i32 %x, 0
diff --git a/llvm/test/CodeGen/ARM/vrint.ll b/llvm/test/CodeGen/ARM/vrint.ll
index 2391e9499479b..52107aac65187 100644
--- a/llvm/test/CodeGen/ARM/vrint.ll
+++ b/llvm/test/CodeGen/ARM/vrint.ll
@@ -79,21 +79,7 @@ define <4 x half> @frinta_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frinta_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrinta.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrinta.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrinta.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrinta.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrinta.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.round.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -243,35 +229,7 @@ define <8 x half> @frinta_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frinta_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s1
-; CHECK-FP16-NEXT:    vrinta.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrinta.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.round.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -297,9 +255,7 @@ define <2 x float> @frinta_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frinta_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrinta.f32 s3, s1
-; CHECK-NEXT:    vrinta.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrinta.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.round.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -331,11 +287,7 @@ define <4 x float> @frinta_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frinta_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrinta.f32 s7, s3
-; CHECK-NEXT:    vrinta.f32 s6, s2
-; CHECK-NEXT:    vrinta.f32 s5, s1
-; CHECK-NEXT:    vrinta.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrinta.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.round.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -813,21 +765,7 @@ define <4 x half> @frintm_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintm_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintm.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintm.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintm.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintm.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintm.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.floor.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -977,35 +915,7 @@ define <8 x half> @frintm_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintm_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintm.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintm.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.floor.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -1031,9 +941,7 @@ define <2 x float> @frintm_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintm_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintm.f32 s3, s1
-; CHECK-NEXT:    vrintm.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintm.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.floor.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -1065,11 +973,7 @@ define <4 x float> @frintm_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintm_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintm.f32 s7, s3
-; CHECK-NEXT:    vrintm.f32 s6, s2
-; CHECK-NEXT:    vrintm.f32 s5, s1
-; CHECK-NEXT:    vrintm.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintm.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.floor.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -1180,21 +1084,7 @@ define <4 x half> @frintn_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintn_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintn.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintn.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintn.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintn.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintn.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.roundeven.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -1344,35 +1234,7 @@ define <8 x half> @frintn_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintn_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintn.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintn.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.roundeven.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -1398,9 +1260,7 @@ define <2 x float> @frintn_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintn_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintn.f32 s3, s1
-; CHECK-NEXT:    vrintn.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintn.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -1432,11 +1292,7 @@ define <4 x float> @frintn_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintn_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintn.f32 s7, s3
-; CHECK-NEXT:    vrintn.f32 s6, s2
-; CHECK-NEXT:    vrintn.f32 s5, s1
-; CHECK-NEXT:    vrintn.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintn.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -1547,21 +1403,7 @@ define <4 x half> @frintp_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintp_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintp.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintp.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintp.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintp.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintp.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.ceil.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -1711,35 +1553,7 @@ define <8 x half> @frintp_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintp_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintp.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintp.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.ceil.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -1765,9 +1579,7 @@ define <2 x float> @frintp_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintp_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintp.f32 s3, s1
-; CHECK-NEXT:    vrintp.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintp.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.ceil.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -1799,11 +1611,7 @@ define <4 x float> @frintp_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintp_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintp.f32 s7, s3
-; CHECK-NEXT:    vrintp.f32 s6, s2
-; CHECK-NEXT:    vrintp.f32 s5, s1
-; CHECK-NEXT:    vrintp.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintp.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.ceil.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -1914,21 +1722,7 @@ define <4 x half> @frintx_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintx_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintx.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintx.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintx.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintx.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintx.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.rint.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -2078,35 +1872,7 @@ define <8 x half> @frintx_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintx_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintx.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintx.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.rint.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -2132,9 +1898,7 @@ define <2 x float> @frintx_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintx_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintx.f32 s3, s1
-; CHECK-NEXT:    vrintx.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintx.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.rint.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -2166,11 +1930,7 @@ define <4 x float> @frintx_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintx_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintx.f32 s7, s3
-; CHECK-NEXT:    vrintx.f32 s6, s2
-; CHECK-NEXT:    vrintx.f32 s5, s1
-; CHECK-NEXT:    vrintx.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintx.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.rint.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -2281,21 +2041,7 @@ define <4 x half> @frintz_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintz_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintz.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintz.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintz.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintz.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintz.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.trunc.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -2445,35 +2191,7 @@ define <8 x half> @frintz_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintz_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintz.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintz.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.trunc.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -2499,9 +2217,7 @@ define <2 x float> @frintz_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintz_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintz.f32 s3, s1
-; CHECK-NEXT:    vrintz.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintz.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.trunc.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -2533,11 +2249,7 @@ define <4 x float> @frintz_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintz_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintz.f32 s7, s3
-; CHECK-NEXT:    vrintz.f32 s6, s2
-; CHECK-NEXT:    vrintz.f32 s5, s1
-; CHECK-NEXT:    vrintz.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintz.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.trunc.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
diff --git a/llvm/test/CodeGen/ARM/vrintn.ll b/llvm/test/CodeGen/ARM/vrintn.ll
index 40f806ba55d36..077007b22fc4b 100644
--- a/llvm/test/CodeGen/ARM/vrintn.ll
+++ b/llvm/test/CodeGen/ARM/vrintn.ll
@@ -1,12 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=armv8 -mattr=+neon %s -o - | FileCheck %s
 
 declare float @llvm.arm.neon.vrintn.f32(float) nounwind readnone
 declare <2 x float> @llvm.arm.neon.vrintn.v2f32(<2 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrintn.v4f32(<4 x float>) nounwind readnone
 
-; CHECK-LABEL: vrintn_f32:
-; CHECK: vrintn.f32
 define float @vrintn_f32(ptr %A) nounwind {
+; CHECK-LABEL: vrintn_f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr s0, [r0]
+; CHECK-NEXT:    vrintn.f32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
   %tmp1 = load float, ptr %A
   %tmp2 = call float @llvm.arm.neon.vrintn.f32(float %tmp1)
   ret float %tmp2
@@ -74,10 +79,9 @@ define <4 x half> @roundeven_4h(<4 x half> %A) nounwind {
 define <2 x float> @roundeven_2s(<2 x float> %A) nounwind {
 ; CHECK-LABEL: roundeven_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vrintn.f32 s3, s1
-; CHECK-NEXT:    vrintn.f32 s2, s0
-; CHECK-NEXT:    vmov r0, r1, d1
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vrintn.f32 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    bx lr
 	%tmp3 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %A)
 	ret <2 x float> %tmp3
@@ -86,14 +90,11 @@ define <2 x float> @roundeven_2s(<2 x float> %A) nounwind {
 define <4 x float> @roundeven_4s(<4 x float> %A) nounwind {
 ; CHECK-LABEL: roundeven_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov d1, r2, r3
-; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vrintn.f32 s7, s3
-; CHECK-NEXT:    vrintn.f32 s6, s2
-; CHECK-NEXT:    vrintn.f32 s5, s1
-; CHECK-NEXT:    vrintn.f32 s4, s0
-; CHECK-NEXT:    vmov r2, r3, d3
-; CHECK-NEXT:    vmov r0, r1, d2
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vrintn.f32 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    bx lr
 	%tmp3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %A)
 	ret <4 x float> %tmp3
diff --git a/llvm/test/CodeGen/AVR/sincos-soften-error.ll b/llvm/test/CodeGen/AVR/sincos-soften-error.ll
new file mode 100644
index 0000000000000..85f5aabb4e966
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/sincos-soften-error.ll
@@ -0,0 +1,9 @@
+; RUN: not llc -mtriple=avr -filetype=null %s 2>&1 | FileCheck %s
+
+; CHECK: error: do not know how to soften fsincos
+define { double, double } @test_sincos_f64(double %a) #0 {
+  %result = call { double, double } @llvm.sincos.f64(double %a)
+  ret { double, double } %result
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll b/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll
index 80a071a66364b..25abf2111060c 100644
--- a/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll
+++ b/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll
@@ -88,4 +88,4 @@ define void @loadf64WithCheckBit() {
   ; CHECK-NOT: extractvalue { double, i1 }
   %cb = extractvalue {double, i1} %load0, 1
   ret void
-}
\ No newline at end of file
+}
diff --git a/llvm/test/CodeGen/DirectX/BufferLoadInt64.ll b/llvm/test/CodeGen/DirectX/BufferLoadInt64.ll
new file mode 100644
index 0000000000000..42c0012ff3475
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/BufferLoadInt64.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-intrinsic-expansion %s | FileCheck %s
+
+target triple = "dxil-pc-shadermodel6.2-compute"
+
+define void @loadi64() {
+; CHECK-LABEL: define void @loadi64() {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, i1 } @llvm.dx.resource.load.typedbuffer.v2i32.tdx.TypedBuffer_i64_1_0_0t(target("dx.TypedBuffer", i64, 1, 0, 0) [[BUFFER]], i32 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %result = call { i64, i1 } @llvm.dx.resource.load.typedbuffer.tdx.TypedBuffer_i64_1_0_0t(target("dx.TypedBuffer", i64, 1, 0, 0) %buffer, i32 0)
+  ret void
+}
+
+define void @loadv2i64() {
+; CHECK-LABEL: define void @loadv2i64() {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, i1 } @llvm.dx.resource.load.typedbuffer.v4i32.tdx.TypedBuffer_v2i64_1_0_0t(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) [[BUFFER]], i32 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 32
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = shl i64 [[TMP13]], 32
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP12]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP15]], i32 1
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %result = call { <2 x i64>, i1 } @llvm.dx.resource.load.typedbuffer.tdx.TypedBuffer_v2i64_1_0_0t(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) %buffer, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/BufferStoreInt64.ll b/llvm/test/CodeGen/DirectX/BufferStoreInt64.ll
new file mode 100644
index 0000000000000..c97a02d1873a0
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/BufferStoreInt64.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-intrinsic-expansion %s | FileCheck %s
+
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+define void @storei64(i64 %0) {
+; CHECK-LABEL: define void @storei64(
+; CHECK-SAME: i64 [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1
+; CHECK-NEXT:    call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_i64_1_0_0t.v2i32(target("dx.TypedBuffer", i64, 1, 0, 0) [[BUFFER]], i32 0, <2 x i32> [[TMP6]])
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_i64_1_0_0t(target("dx.TypedBuffer", i64, 1, 0, 0) %buffer, i32 0,i64 %0)
+  ret void
+}
+
+
+define void @storev2i64(<2 x i64> %0) {
+; CHECK-LABEL: define void @storev2i64(
+; CHECK-SAME: <2 x i64> [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i64> [[TMP0]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <2 x i64> [[TMP0]], splat (i64 32)
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP4]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_v2i64_1_0_0t.v4i32(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) [[BUFFER]], i32 0, <4 x i32> [[TMP13]])
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_v2i64_1_0_0t(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) %buffer, i32 0, <2 x i64> %0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll
new file mode 100644
index 0000000000000..001f3320137a6
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll
@@ -0,0 +1,216 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB : register(b0) {
+;   float a1[3];
+;   double3 a2[2];
+;   float16_t a3[2][2];
+;   uint64_t a4[3];
+;   int2 a5[3][2];
+;   uint16_t a6[1];
+;   int64_t a7[2];
+;   bool a8[4];
+; }
+%__cblayout_CB = type <{ [3 x float], [2 x <3 x double>], [2 x [2 x half]], [3 x i64], [3 x [2 x <2 x i32>]], [1 x i16], [2 x i64], [4 x i32] }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 272, 288, 320)) poison
+@a1 = external local_unnamed_addr addrspace(2) global [3 x float], align 4
+@a2 = external local_unnamed_addr addrspace(2) global [2 x <3 x double>], align 32
+@a3 = external local_unnamed_addr addrspace(2) global [2 x [2 x half]], align 2
+@a4 = external local_unnamed_addr addrspace(2) global [3 x i64], align 8
+@a5 = external local_unnamed_addr addrspace(2) global [3 x [2 x <2 x i32>]], align 16
+@a6 = external local_unnamed_addr addrspace(2) global [1 x i16], align 2
+@a7 = external local_unnamed_addr addrspace(2) global [2 x i64], align 8
+@a8 = external local_unnamed_addr addrspace(2) global [4 x i32], align 4
+
+; CHECK: define void @f(
+define void @f(ptr %dst) {
+entry:
+  %CB.cb_h.i.i = tail call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 272, 288, 320)) @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 272, 288, 320)) %CB.cb_h.i.i, ptr @CB.cb, align 4
+
+  %a1.copy = alloca [3 x float], align 4
+  %a2.copy = alloca [2 x <3 x double>], align 32
+  %a3.copy = alloca [2 x [2 x half]], align 2
+  %a4.copy = alloca [3 x i64], align 8
+  %a5.copy = alloca [3 x [2 x <2 x i32>]], align 16
+  %a6.copy = alloca [1 x i16], align 2
+  %a7.copy = alloca [2 x i64], align 8
+  %a8.copy = alloca [4 x i32], align 4
+
+  ; Try copying no elements
+; CHECK-NOT: memcpy
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a1.copy, ptr addrspace(2) align 4 @a1, i32 0, i1 false)
+
+  ; Try copying only the first element
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+; CHECK:    [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY:%.*]], i32 0
+; CHECK:    store float [[X]], ptr [[DEST]], align 4
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a1.copy, ptr addrspace(2) align 4 @a1, i32 4, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+; CHECK:    [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY:%.*]], i32 0
+; CHECK:    store float [[X]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+; CHECK:    [[Y:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY]], i32 4
+; CHECK:    store float [[Y]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 2)
+; CHECK:    [[Z:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY]], i32 8
+; CHECK:    store float [[Z]], ptr [[DEST]], align 4
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a1.copy, ptr addrspace(2) align 4 @a1, i32 12, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 3)
+; CHECK:    [[X:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { double, double } [[LOAD]], 1
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 4)
+; CHECK:    [[Z:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[UPTO0:%.*]] = insertelement <3 x double> poison, double [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <3 x double> [[UPTO0]], double [[Y]], i32 1
+; CHECK:    [[UPTO2:%.*]] = insertelement <3 x double> [[UPTO1]], double [[Z]], i32 2
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A2_COPY:%.*]], i32 0
+; CHECK:    store <3 x double> [[UPTO2]], ptr [[DEST]], align 32
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 5)
+; CHECK:    [[X:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { double, double } [[LOAD]], 1
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 6)
+; CHECK:    [[Z:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[UPTO0:%.*]] = insertelement <3 x double> poison, double [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <3 x double> [[UPTO0]], double [[Y]], i32 1
+; CHECK:    [[UPTO2:%.*]] = insertelement <3 x double> [[UPTO1]], double [[Z]], i32 2
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A2_COPY]], i32 32
+; CHECK:    store <3 x double> [[UPTO2]], ptr [[DEST]], align 32
+  call void @llvm.memcpy.p0.p2.i32(ptr align 32 %a2.copy, ptr addrspace(2) align 32 @a2, i32 64, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 7)
+; CHECK:    [[X:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY:%.*]], i32 0
+; CHECK:    store half [[X]], ptr [[DEST]], align 2
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 8)
+; CHECK:    [[Y:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY]], i32 2
+; CHECK:    store half [[Y]], ptr [[DEST]], align 2
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 9)
+; CHECK:    [[X:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY]], i32 4
+; CHECK:    store half [[X]], ptr [[DEST]], align 2
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 10)
+; CHECK:    [[Y:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY]], i32 6
+; CHECK:    store half [[Y]], ptr [[DEST]], align 2
+  call void @llvm.memcpy.p0.p2.i32(ptr align 2 %a3.copy, ptr addrspace(2) align 2 @a3, i32 8, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 11)
+; CHECK:    [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A4_COPY:%.*]], i32 0
+; CHECK:    store i64 [[X]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 12)
+; CHECK:    [[Y:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A4_COPY]], i32 8
+; CHECK:    store i64 [[Y]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 13)
+; CHECK:    [[Z:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A4_COPY]], i32 16
+; CHECK:    store i64 [[Z]], ptr [[DEST]], align 8
+  call void @llvm.memcpy.p0.p2.i32(ptr align 8 %a4.copy, ptr addrspace(2) align 8 @a4, i32 24, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 14)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY:%.*]], i32 0
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 15)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 8
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 16)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 16
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 17)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 24
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 18)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 32
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 19)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 40
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+  call void @llvm.memcpy.p0.p2.i32(ptr align 16 %a5.copy, ptr addrspace(2) align 16 @a5, i32 48, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i16, i16, i16, i16, i16, i16, i16, i16 } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 17)
+; CHECK:    [[X:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A6_COPY:%.*]], i32 0
+; CHECK:    store i16 [[X]], ptr [[DEST]], align 2
+  call void @llvm.memcpy.p0.p2.i32(ptr align 2 %a6.copy, ptr addrspace(2) align 2 @a6, i32 2, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 18)
+; CHECK:    [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A7_COPY:%.*]], i32 0
+; CHECK:    store i64 [[X]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 19)
+; CHECK:    [[Y:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A7_COPY]], i32 8
+; CHECK:    store i64 [[Y]], ptr [[DEST]], align 8
+  call void @llvm.memcpy.p0.p2.i32(ptr align 8 %a7.copy, ptr addrspace(2) align 8 @a7, i32 16, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 20)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY:%.*]], i32 0
+; CHECK:    store i32 [[X]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 21)
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY]], i32 4
+; CHECK:    store i32 [[Y]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 22)
+; CHECK:    [[Z:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY]], i32 8
+; CHECK:    store i32 [[Z]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 23)
+; CHECK:    [[W:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY]], i32 12
+; CHECK:    store i32 [[W]], ptr [[DEST]], align 4
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a8.copy, ptr addrspace(2) align 4 @a8, i32 16, i1 false)
+
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p2.i32(ptr noalias writeonly captures(none), ptr addrspace(2) noalias readonly captures(none), i32, i1 immarg)
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1, ptr addrspace(2) @a2, ptr addrspace(2) @a3, ptr addrspace(2) @a4, ptr addrspace(2) @a5, ptr addrspace(2) @a6, ptr addrspace(2) @a7, ptr addrspace(2) @a8}
+!1 = !{i32 0, i32 2}
+!2 = !{}
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll
new file mode 100644
index 0000000000000..288dea00b6e55
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for AddressU: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 666, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll
new file mode 100644
index 0000000000000..e9abcf9669999
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for AddressV: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 666, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll
new file mode 100644
index 0000000000000..238f488ee78d6
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for AddressW: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 666, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll
new file mode 100644
index 0000000000000..8dc69eb1f9d7c
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for BorderColor: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 666, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll
new file mode 100644
index 0000000000000..b2c8faf8d4a0a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for ComparisonFunc: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 666, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll
new file mode 100644
index 0000000000000..758d2629ed78e
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for Filter: 45 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 45, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll
new file mode 100644
index 0000000000000..47d4b52d72e8e
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MaxAnisotropy: 666 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 666, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll
new file mode 100644
index 0000000000000..7e7d56eff309c
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MaxLOD: 0
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 0x7FF8000000000000, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll
new file mode 100644
index 0000000000000..d958f10d3c1af
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MinLOD: 0
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float 0x7FF8000000000000, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll
new file mode 100644
index 0000000000000..34b27eb40f5fb
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MipLODBias: 666 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 6.660000e+02, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll
new file mode 100644
index 0000000000000..dc6ee4290b532
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for RegisterSpace: 4294967280
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 4294967280, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll
new file mode 100644
index 0000000000000..6cee1dd95fd81
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for ShaderRegister: 4294967295
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 4294967295, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll
new file mode 100644
index 0000000000000..fa5bf12e2b8cd
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for ShaderVisibility: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 666 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll
new file mode 100644
index 0000000000000..d9ee39dbb7287
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll
@@ -0,0 +1,42 @@
+; RUN: opt %s -dxil-embed -dxil-globals -S -o - | FileCheck %s
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: @dx.rts0 = private constant [76 x i8]  c"{{.*}}", section "RTS0", align 4
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
+
+; DXC: - Name:            RTS0
+; DXC-NEXT:     Size:            76
+; DXC-NEXT:     RootSignature:
+; DXC-NEXT:       Version:         2
+; DXC-NEXT:       NumRootParameters: 0
+; DXC-NEXT:       RootParametersOffset: 24
+; DXC-NEXT:       NumStaticSamplers: 1
+; DXC-NEXT:       StaticSamplersOffset: 24
+; DXC-NEXT:       Parameters:      []
+; DXC-NEXT:       Samplers:
+; DXC-NEXT:         - Filter:          4
+; DXC-NEXT:           AddressU:        2
+; DXC-NEXT:           AddressV:        3
+; DXC-NEXT:           AddressW:        5
+; DXC-NEXT:           MipLODBias:      1.425
+; DXC-NEXT:           MaxAnisotropy:   9
+; DXC-NEXT:           ComparisonFunc:  3
+; DXC-NEXT:           BorderColor:     2
+; DXC-NEXT:           MinLOD:          -128
+; DXC-NEXT:           MaxLOD:          128
+; DXC-NEXT:           ShaderRegister:  42
+; DXC-NEXT:           RegisterSpace:   0
+; DXC-NEXT:           ShaderVisibility: 0
diff --git a/llvm/test/CodeGen/DirectX/RawBufferLoad-error64.ll b/llvm/test/CodeGen/DirectX/RawBufferLoad-error64.ll
deleted file mode 100644
index d8b6311c8ff2e..0000000000000
--- a/llvm/test/CodeGen/DirectX/RawBufferLoad-error64.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; We use llc for this test so that we don't abort after the first error.
-; RUN: not llc %s -o /dev/null 2>&1 | FileCheck %s
-
-target triple = "dxil-pc-shadermodel6.2-compute"
-
-declare void @v4f64_user(<4 x double>)
-
-; Can't load 64 bit types directly until SM6.3 (byteaddressbuf.Load<int64_t4>)
-; CHECK: error:
-; CHECK-SAME: in function loadv4f64_byte
-; CHECK-SAME: Cannot create RawBufferLoad operation: Invalid overload type
-define void @loadv4f64_byte(i32 %offset) "hlsl.export" {
-  %buffer = call target("dx.RawBuffer", i8, 0, 0, 0)
-      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i8_0_0_0(
-          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
-
-  %load = call {<4 x double>, i1} @llvm.dx.resource.load.rawbuffer.v4i64(
-      target("dx.RawBuffer", i8, 0, 0, 0) %buffer, i32 %offset, i32 0)
-  %data = extractvalue {<4 x double>, i1} %load, 0
-
-  call void @v4f64_user(<4 x double> %data)
-
-  ret void
-}
diff --git a/llvm/test/CodeGen/DirectX/RawBufferLoadDouble.ll b/llvm/test/CodeGen/DirectX/RawBufferLoadDouble.ll
new file mode 100644
index 0000000000000..9213d60c9b496
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/RawBufferLoadDouble.ll
@@ -0,0 +1,229 @@
+; RUN: opt -mtriple=dxil-pc-shadermodel6.2-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK62
+; RUN: opt -mtriple=dxil-pc-shadermodel6.3-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK63
+
+define void @loadf64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", double, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_f64_0_0t(
+  ; CHECK-SAME: i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", double, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_f64_1_0_0t(
+          i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { double, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", double, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we load an <2 x i32> instead of a double
+  ; CHECK62-NOT: call {double, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <2 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v2i32.tdx.Rawbuffer_f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", double, 0, 0) [[B]], i32 %index, i32 0)	
+  %load0 = call {double, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", double, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { double, i1 } [[L0]], 0
+
+  ; check we extract the two i32 and construct a double
+  ; CHECK62: [[D0:%.*]] = extractvalue { <2 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo:%.*]] = extractelement <2 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi:%.*]] = extractelement <2 x i32> [[D0]], i32 1
+  ; CHECK62: [[DBL:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo]], i32 [[Hi]])
+  ; CHECK62-NOT: extractvalue { double, i1 }
+  %data0 = extractvalue {double, i1} %load0, 0
+  ret void
+}
+
+define void @loadv2f64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", <2 x double>, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v2f64_0_0t(
+  ; CHECK-SAME: i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", <2 x double>, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v2f64_1_0_0t(
+          i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { <2 x double>, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", <2 x double>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we load an <4 x i32> instead of a double2
+  ; CHECK62: [[L0:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v2f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <2 x double>, 0, 0) [[B]], i32 %index, i32 0)
+  %load0 = call { <2 x double>, i1 } @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", <2 x double>, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { <2 x double>, i1 } [[L0]], 0
+
+  ; check we extract the 4 i32 and construct a <2 x double>
+  ; CHECK62: [[D0:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo1:%.*]] = extractelement <4 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi1:%.*]] = extractelement <4 x i32> [[D0]], i32 1
+  ; CHECK62: [[Lo2:%.*]] = extractelement <4 x i32> [[D0]], i32 2
+  ; CHECK62: [[Hi2:%.*]] = extractelement <4 x i32> [[D0]], i32 3
+  ; CHECK62: [[Dbl1:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo1]], i32 [[Hi1]])
+  ; CHECK62: [[Vec:%.*]] = insertelement <2 x double> poison, double [[Dbl1]], i32 0
+  ; CHECK62: [[Dbl2:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo2]], i32 [[Hi2]])
+  ; CHECK62: [[Vec2:%.*]] = insertelement <2 x double> [[Vec]], double [[Dbl2]], i32 1
+  ; CHECK62-NOT: extractvalue { <2 x double>, i1 }
+  %data0 = extractvalue { <2 x double>, i1 } %load0, 0
+  ret void
+}
+
+; show we properly handle extracting the check bit
+define void @loadf64WithCheckBit(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", double, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_f64_0_0t(
+  ; CHECK-SAME: i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", double, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_f64_1_0_0t(
+          i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { double, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", double, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we load an <2 x i32> instead of a double
+  ; CHECK62-NOT: call {double, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <2 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v2i32.tdx.Rawbuffer_f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", double, 0, 0) [[B]], i32 %index, i32 0)	
+  %load0 = call {double, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", double, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { double, i1 } [[L0]], 0
+  ; CHECK63: extractvalue { double, i1 } [[L0]], 1
+
+  ; check we extract the two i32 and construct a double
+  ; CHECK62: [[D0:%.*]] = extractvalue { <2 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo:%.*]] = extractelement <2 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi:%.*]] = extractelement <2 x i32> [[D0]], i32 1
+  ; CHECK62: [[DBL:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo]], i32 [[Hi]])
+  %data0 = extractvalue {double, i1} %load0, 0
+  ; CHECK62: extractvalue { <2 x i32>, i1 } [[L0]], 1
+  ; CHECK62-NOT: extractvalue { double, i1 }
+  %cb = extractvalue {double, i1} %load0, 1
+  ret void
+}
+
+; Raw Buffer Load allows for double3 and double4 to be loaded
+; In SM6.2 and below, two loads will be performed.
+; Show we and the checkbits together
+  
+define void @loadv3f64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", <3 x double>, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v3f64_0_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", <3 x double>, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v3f64_0_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { <3 x double>, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", <3 x double>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we perform two loads
+  ; and do 6 extracts and construct 3 doubles
+  ; CHECK62-NOT: call {<3 x double>, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v3f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <3 x double>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; CHECK62: [[D0:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo1:%.*]] = extractelement <4 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi1:%.*]] = extractelement <4 x i32> [[D0]], i32 1
+  ; CHECK62: [[Lo2:%.*]] = extractelement <4 x i32> [[D0]], i32 2
+  ; CHECK62: [[Hi2:%.*]] = extractelement <4 x i32> [[D0]], i32 3
+  ; CHECK62: [[DBL1:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo1]], i32 [[Hi1]])
+  ; CHECK62: [[Vec1:%.*]] = insertelement <3 x double> poison, double [[DBL1]], i32 0
+  ; CHECK62: [[DBL2:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo2]], i32 [[Hi2]])
+  ; CHECK62: [[Vec2:%.*]] = insertelement <3 x double> [[Vec1]], double [[DBL2]], i32 1
+
+  ; 2nd load
+  ; CHECK62: [[L2:%.*]] = call { <2 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v2i32.tdx.Rawbuffer_v3f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <3 x double>, 0, 0) [[B]], i32 %index, i32 16)
+
+  ; CHECK62: [[D2:%.*]] = extractvalue { <2 x i32>, i1 } [[L2]], 0
+  ; CHECK62: [[Lo3:%.*]] = extractelement <2 x i32> [[D2]], i32 0
+  ; CHECK62: [[Hi3:%.*]] = extractelement <2 x i32> [[D2]], i32 1
+  ; CHECK62: [[DBL3:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo3]], i32 [[Hi3]])
+  ; CHECK62: [[Vec3:%.*]] = insertelement <3 x double> [[Vec2]], double [[DBL3]], i32 2
+  %load0 = call {<3 x double>, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", <3 x double>, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { <3 x double>, i1 } [[L0]], 0
+  ; CHECK63: extractvalue { <3 x double>, i1 } [[L0]], 1
+
+  ; CHECK62-NOT: extractvalue {<3 x double>, i1 }
+  %data0 = extractvalue {<3 x double>, i1} %load0, 0
+  ; check we extract checkbit from both loads and and them together
+  ; CHECK62: [[B1:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 1
+  ; CHECK62: [[B2:%.*]] = extractvalue { <2 x i32>, i1 } [[L2]], 1
+  ; CHECK62: and i1 [[B1]], [[B2]]
+  %cb = extractvalue {<3 x double>, i1} %load0, 1
+  ret void
+}
+
+define void @loadv4f64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", <4 x double>, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v4f64_0_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", <4 x double>, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v4f64_0_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { <4 x double>, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", <4 x double>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we perform two loads
+  ; and do 8 extracts and construct 4 doubles
+  ; CHECK62-NOT: call {<4 x double>, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v4f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <4 x double>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; CHECK62: [[D0:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo1:%.*]] = extractelement <4 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi1:%.*]] = extractelement <4 x i32> [[D0]], i32 1
+  ; CHECK62: [[Lo2:%.*]] = extractelement <4 x i32> [[D0]], i32 2
+  ; CHECK62: [[Hi2:%.*]] = extractelement <4 x i32> [[D0]], i32 3
+  ; CHECK62: [[DBL1:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo1]], i32 [[Hi1]])
+  ; CHECK62: [[Vec1:%.*]] = insertelement <4 x double> poison, double [[DBL1]], i32 0
+  ; CHECK62: [[DBL2:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo2]], i32 [[Hi2]])
+  ; CHECK62: [[Vec2:%.*]] = insertelement <4 x double> [[Vec1]], double [[DBL2]], i32 1
+
+  ; 2nd load
+  ; CHECK62: [[L2:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v4f64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <4 x double>, 0, 0) [[B]], i32 %index, i32 16)
+
+  ; CHECK62: [[D2:%.*]] = extractvalue { <4 x i32>, i1 } [[L2]], 0
+  ; CHECK62: [[Lo3:%.*]] = extractelement <4 x i32> [[D2]], i32 0
+  ; CHECK62: [[Hi3:%.*]] = extractelement <4 x i32> [[D2]], i32 1
+  ; CHECK62: [[Lo4:%.*]] = extractelement <4 x i32> [[D2]], i32 2
+  ; CHECK62: [[Hi4:%.*]] = extractelement <4 x i32> [[D2]], i32 3
+  ; CHECK62: [[DBL3:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo3]], i32 [[Hi3]])
+  ; CHECK62: [[Vec3:%.*]] = insertelement <4 x double> [[Vec2]], double [[DBL3]], i32 2
+  ; CHECK62: [[DBL4:%.*]] = call double @llvm.dx.asdouble.i32(i32 [[Lo4]], i32 [[Hi4]])
+  ; CHECK62: [[Vec4:%.*]] = insertelement <4 x double> [[Vec3]], double [[DBL4]], i32 3
+  %load0 = call {<4 x double>, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", <4 x double>, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { <4 x double>, i1 } [[L0]], 0
+  ; CHECK63: extractvalue { <4 x double>, i1 } [[L0]], 1
+
+  ; CHECK62-NOT: extractvalue {<4 x double>, i1 }
+  %data0 = extractvalue {<4 x double>, i1} %load0, 0
+  ; check we extract checkbit from both loads and and them together
+  ; CHECK62: [[B1:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 1
+  ; CHECK62: [[B2:%.*]] = extractvalue { <4 x i32>, i1 } [[L2]], 1
+  ; CHECK62: and i1 [[B1]], [[B2]]
+  %cb = extractvalue {<4 x double>, i1} %load0, 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/RawBufferLoadInt64.ll b/llvm/test/CodeGen/DirectX/RawBufferLoadInt64.ll
new file mode 100644
index 0000000000000..a1c153f2c0c84
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/RawBufferLoadInt64.ll
@@ -0,0 +1,262 @@
+; RUN: opt -mtriple=dxil-pc-shadermodel6.2-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK62
+; RUN: opt -mtriple=dxil-pc-shadermodel6.3-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK63
+
+define void @loadi64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", i64, 1, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_i64_1_0_0t(
+  ; CHECK-SAME: i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", i64, 1, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_i64_1_0_0t(
+          i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { i64, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", i64, 1, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we load an <2 x i32> instead of a i64
+  ; CHECK62-NOT: call {i64, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <2 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v2i32.tdx.Rawbuffer_i64_1_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", i64, 1, 0, 0) [[B]], i32 %index, i32 0)	
+  %load0 = call {i64, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", i64, 1, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { i64, i1 } [[L0]], 0
+
+  ; check we extract the two i32 and construct a i64
+  ; CHECK62: [[D0:%.*]] = extractvalue { <2 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo:%.*]] = extractelement <2 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi:%.*]] = extractelement <2 x i32> [[D0]], i32 1
+  ; CHECK62: [[ZLo1:%.*]] = zext i32 [[Lo]] to i64
+  ; CHECK62: [[ZHi1:%.*]] = zext i32 [[Hi]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi1]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo1]], [[A]]
+  ; CHECK62-NOT: extractvalue { i64, i1 }
+  %data0 = extractvalue {i64, i1} %load0, 0
+  ret void
+}
+
+define void @loadv2i64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", <2 x i64>, 1, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v2i64_1_0_0t(
+  ; CHECK-SAME: i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", <2 x i64>, 1, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v2i64_1_0_0t(
+          i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { <2 x i64>, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", <2 x i64>, 1, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we load an <4 x i32> instead of a i642
+  ; CHECK62: [[L0:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v2i64_1_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <2 x i64>, 1, 0, 0) [[B]], i32 %index, i32 0)
+  %load0 = call { <2 x i64>, i1 } @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", <2 x i64>, 1, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { <2 x i64>, i1 } [[L0]], 0
+
+  ; check we extract the 4 i32 and construct a <2 x i64>
+  ; CHECK62: [[D0:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo1:%.*]] = extractelement <4 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi1:%.*]] = extractelement <4 x i32> [[D0]], i32 1
+  ; CHECK62: [[Lo2:%.*]] = extractelement <4 x i32> [[D0]], i32 2
+  ; CHECK62: [[Hi2:%.*]] = extractelement <4 x i32> [[D0]], i32 3
+  ; CHECK62: [[ZLo1:%.*]] = zext i32 [[Lo1]] to i64
+  ; CHECK62: [[ZHi1:%.*]] = zext i32 [[Hi1]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi1]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo1]], [[A]]
+  ; CHECK62: [[Vec:%.*]] = insertelement <2 x i64> poison, i64 [[B]], i32 0
+  ; CHECK62: [[ZLo2:%.*]] = zext i32 [[Lo2]] to i64
+  ; CHECK62: [[ZHi2:%.*]] = zext i32 [[Hi2]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi2]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo2]], [[A]]
+  ; CHECK62: [[Vec2:%.*]] = insertelement <2 x i64> [[Vec]], i64 [[B]], i32 1
+  ; CHECK62-NOT: extractvalue { <2 x i64>, i1 }
+  %data0 = extractvalue { <2 x i64>, i1 } %load0, 0
+  ret void
+}
+
+; show we properly handle extracting the check bit
+define void @loadi64WithCheckBit(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[B:%.*]] = call target("dx.Rawbuffer", i64, 1, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_i64_1_0_0t(
+  ; CHECK-SAME: i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", i64, 1, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_i64_1_0_0t(
+          i32 0, i32 1, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { i64, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", i64, 1, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we load an <2 x i32> instead of a i64
+  ; CHECK62-NOT: call {i64, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <2 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v2i32.tdx.Rawbuffer_i64_1_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", i64, 1, 0, 0) [[B]], i32 %index, i32 0)	
+  %load0 = call {i64, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", i64, 1, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { i64, i1 } [[L0]], 0
+  ; CHECK63: extractvalue { i64, i1 } [[L0]], 1
+
+  ; check we extract the two i32 and construct a i64
+  ; CHECK62: [[D0:%.*]] = extractvalue { <2 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo:%.*]] = extractelement <2 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi:%.*]] = extractelement <2 x i32> [[D0]], i32 1
+  ; CHECK62: [[ZLo1:%.*]] = zext i32 [[Lo]] to i64
+  ; CHECK62: [[ZHi1:%.*]] = zext i32 [[Hi]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi1]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo1]], [[A]]
+  %data0 = extractvalue {i64, i1} %load0, 0
+  ; CHECK62: extractvalue { <2 x i32>, i1 } [[L0]], 1
+  ; CHECK62-NOT: extractvalue { i64, i1 }
+  %cb = extractvalue {i64, i1} %load0, 1
+  ret void
+}
+
+; Raw Buffer Load allows for i64_t3 and i64_t4 to be loaded
+; In SM6.2 and below, two loads will be performed.
+; Show we and the checkbits together
+
+define void @loadv3i64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK: [[Buf:%.*]] = call target("dx.Rawbuffer", <3 x i64>, 0, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v3i64_0_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", <3 x i64>, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v3i64_0_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { <3 x i64>, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", <3 x i64>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we perform two loads
+  ; and do 6 extracts and construct 3 i64s
+  ; CHECK62-NOT: call {<3 x i64>, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v3i64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <3 x i64>, 0, 0) [[Buf]], i32 %index, i32 0)
+
+  ; CHECK62: [[D0:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo1:%.*]] = extractelement <4 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi1:%.*]] = extractelement <4 x i32> [[D0]], i32 1
+  ; CHECK62: [[Lo2:%.*]] = extractelement <4 x i32> [[D0]], i32 2
+  ; CHECK62: [[Hi2:%.*]] = extractelement <4 x i32> [[D0]], i32 3
+  ; CHECK62: [[ZLo1:%.*]] = zext i32 [[Lo1]] to i64
+  ; CHECK62: [[ZHi1:%.*]] = zext i32 [[Hi1]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi1]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo1]], [[A]]
+  ; CHECK62: [[Vec1:%.*]] = insertelement <3 x i64> poison, i64 [[B]], i32 0
+  ; CHECK62: [[ZLo2:%.*]] = zext i32 [[Lo2]] to i64
+  ; CHECK62: [[ZHi2:%.*]] = zext i32 [[Hi2]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi2]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo2]], [[A]]
+  ; CHECK62: [[Vec2:%.*]] = insertelement <3 x i64> [[Vec1]], i64 [[B]], i32 1
+
+  ; 2nd load
+  ; CHECK62: [[L2:%.*]] = call { <2 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v2i32.tdx.Rawbuffer_v3i64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <3 x i64>, 0, 0) [[Buf]], i32 %index, i32 16)
+
+  ; CHECK62: [[D2:%.*]] = extractvalue { <2 x i32>, i1 } [[L2]], 0
+  ; CHECK62: [[Lo3:%.*]] = extractelement <2 x i32> [[D2]], i32 0
+  ; CHECK62: [[Hi3:%.*]] = extractelement <2 x i32> [[D2]], i32 1
+  ; CHECK62: [[ZLo3:%.*]] = zext i32 [[Lo3]] to i64
+  ; CHECK62: [[ZHi3:%.*]] = zext i32 [[Hi3]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi3]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo3]], [[A]]
+  ; CHECK62: [[Vec3:%.*]] = insertelement <3 x i64> [[Vec2]], i64 [[B]], i32 2
+  %load0 = call {<3 x i64>, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", <3 x i64>, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { <3 x i64>, i1 } [[L0]], 0
+  ; CHECK63: extractvalue { <3 x i64>, i1 } [[L0]], 1
+
+  ; CHECK62-NOT: extractvalue {<3 x i64>, i1 }
+  %data0 = extractvalue {<3 x i64>, i1} %load0, 0
+  ; check we extract checkbit from both loads and and them together
+  ; CHECK62: [[B1:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 1
+  ; CHECK62: [[B2:%.*]] = extractvalue { <2 x i32>, i1 } [[L2]], 1
+  ; CHECK62: and i1 [[B1]], [[B2]]
+  %cb = extractvalue {<3 x i64>, i1} %load0, 1
+  ret void
+}
+
+define void @loadv4i64(i32 %index) {
+  ; check the handle from binding is unchanged
+  ; CHECK62: [[Buf:%.*]] = call target("dx.Rawbuffer", <4 x i64>, 0, 0)
+  ; CHECK62-SAME: @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v4i64_0_0t(
+  ; CHECK62-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = call target("dx.Rawbuffer", <4 x i64>, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.Rawbuffer_v4i64_0_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: [[L0:%.*]] = call { <4 x i64>, i1 } @llvm.dx.resource.load.rawbuffer
+  ; CHECK63-SAME: target("dx.Rawbuffer", <4 x i64>, 0, 0) [[B]], i32 %index, i32 0)
+
+  ; check we perform two loads
+  ; and do 8 extracts and construct 4 i64s
+  ; CHECK62-NOT: call {<4 x i64>, i1} @llvm.dx.resource.load.rawbuffer
+  ; CHECK62: [[L0:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v4i64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <4 x i64>, 0, 0) [[Buf]], i32 %index, i32 0)
+
+  ; CHECK62: [[D0:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 0
+  ; CHECK62: [[Lo1:%.*]] = extractelement <4 x i32> [[D0]], i32 0
+  ; CHECK62: [[Hi1:%.*]] = extractelement <4 x i32> [[D0]], i32 1
+  ; CHECK62: [[Lo2:%.*]] = extractelement <4 x i32> [[D0]], i32 2
+  ; CHECK62: [[Hi2:%.*]] = extractelement <4 x i32> [[D0]], i32 3
+  ; CHECK62: [[ZLo1:%.*]] = zext i32 [[Lo1]] to i64
+  ; CHECK62: [[ZHi1:%.*]] = zext i32 [[Hi1]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi1]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo1]], [[A]]
+  ; CHECK62: [[Vec1:%.*]] = insertelement <4 x i64> poison, i64 [[B]], i32 0
+  ; CHECK62: [[ZLo2:%.*]] = zext i32 [[Lo2]] to i64
+  ; CHECK62: [[ZHi2:%.*]] = zext i32 [[Hi2]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi2]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo2]], [[A]]
+  ; CHECK62: [[Vec2:%.*]] = insertelement <4 x i64> [[Vec1]], i64 [[B]], i32 1
+
+  ; 2nd load
+  ; CHECK62: [[L2:%.*]] = call { <4 x i32>, i1 }
+  ; CHECK62-SAME: @llvm.dx.resource.load.rawbuffer.v4i32.tdx.Rawbuffer_v4i64_0_0t(
+  ; CHECK62-SAME: target("dx.Rawbuffer", <4 x i64>, 0, 0) [[Buf]], i32 %index, i32 16)
+
+  ; CHECK62: [[D2:%.*]] = extractvalue { <4 x i32>, i1 } [[L2]], 0
+  ; CHECK62: [[Lo3:%.*]] = extractelement <4 x i32> [[D2]], i32 0
+  ; CHECK62: [[Hi3:%.*]] = extractelement <4 x i32> [[D2]], i32 1
+  ; CHECK62: [[Lo4:%.*]] = extractelement <4 x i32> [[D2]], i32 2
+  ; CHECK62: [[Hi4:%.*]] = extractelement <4 x i32> [[D2]], i32 3
+  ; CHECK62: [[ZLo3:%.*]] = zext i32 [[Lo3]] to i64
+  ; CHECK62: [[ZHi3:%.*]] = zext i32 [[Hi3]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi3]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo3]], [[A]]
+  ; CHECK62: [[Vec3:%.*]] = insertelement <4 x i64> [[Vec2]], i64 [[B]], i32 2
+  ; CHECK62: [[ZLo4:%.*]] = zext i32 [[Lo4]] to i64
+  ; CHECK62: [[ZHi4:%.*]] = zext i32 [[Hi4]] to i64
+  ; CHECK62: [[A:%.*]] = shl i64 [[ZHi4]], 32
+  ; CHECK62: [[B:%.*]] = or i64 [[ZLo4]], [[A]]
+  ; CHECK62: [[Vec4:%.*]] = insertelement <4 x i64> [[Vec3]], i64 [[B]], i32 3
+  %load0 = call {<4 x i64>, i1} @llvm.dx.resource.load.rawbuffer(
+      target("dx.Rawbuffer", <4 x i64>, 0, 0) %buffer, i32 %index, i32 0)
+
+  ; CHECK63: extractvalue { <4 x i64>, i1 } [[L0]], 0
+  ; CHECK63: extractvalue { <4 x i64>, i1 } [[L0]], 1
+
+  ; CHECK62-NOT: extractvalue {<4 x i64>, i1 }
+  %data0 = extractvalue {<4 x i64>, i1} %load0, 0
+  ; check we extract checkbit from both loads and and them together
+  ; CHECK62: [[B1:%.*]] = extractvalue { <4 x i32>, i1 } [[L0]], 1
+  ; CHECK62: [[B2:%.*]] = extractvalue { <4 x i32>, i1 } [[L2]], 1
+  ; CHECK62: and i1 [[B1]], [[B2]]
+  %cb = extractvalue {<4 x i64>, i1} %load0, 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/RawBufferStore-error64.ll b/llvm/test/CodeGen/DirectX/RawBufferStore-error64.ll
deleted file mode 100644
index 895c4c3b9f5fe..0000000000000
--- a/llvm/test/CodeGen/DirectX/RawBufferStore-error64.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; We use llc for this test so that we don't abort after the first error.
-; RUN: not llc %s -o /dev/null 2>&1 | FileCheck %s
-
-target triple = "dxil-pc-shadermodel6.2-compute"
-
-; Can't store 64 bit types directly until SM6.3 (byteaddressbuf.Store<int64_t4>)
-; CHECK: error:
-; CHECK-SAME: in function storev4f64_byte
-; CHECK-SAME: Cannot create RawBufferStore operation: Invalid overload type
-define void @storev4f64_byte(i32 %offset, <4 x double> %data) "hlsl.export" {
-  %buffer = call target("dx.RawBuffer", i8, 1, 0, 0)
-      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i8_0_0_0(
-          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
-
-  call void @llvm.dx.resource.store.rawbuffer.v4i64(
-      target("dx.RawBuffer", i8, 1, 0, 0) %buffer,
-      i32 %offset, i32 0, <4 x double> %data)
-
-  ret void
-}
diff --git a/llvm/test/CodeGen/DirectX/RawBufferStoreDouble.ll b/llvm/test/CodeGen/DirectX/RawBufferStoreDouble.ll
new file mode 100644
index 0000000000000..ddcd761d812fa
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/RawBufferStoreDouble.ll
@@ -0,0 +1,105 @@
+; RUN: opt -mtriple=dxil-pc-shadermodel6.2-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK62
+; RUN: opt -mtriple=dxil-pc-shadermodel6.3-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK63
+
+define void @storef64(double %0, i32 %index) {
+  ; CHECK: [[B:%.*]] = tail call target("dx.RawBuffer", double, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_f64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", double, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_f64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", double, 1, 0) [[B]], i32 %index, i32 0, double %0)
+
+  ; check we split the double and store the lo and hi bits
+  ; CHECK62: [[SD:%.*]] = call { i32, i32 } @llvm.dx.splitdouble.i32(double %0)
+  ; CHECK62: [[Lo:%.*]] = extractvalue { i32, i32 } [[SD]], 0
+  ; CHECK62: [[Hi:%.*]] = extractvalue { i32, i32 } [[SD]], 1
+  ; CHECK62: [[Vec1:%.*]] = insertelement <2 x i32> poison, i32 [[Lo]], i32 0
+  ; CHECK62: [[Vec2:%.*]] = insertelement <2 x i32> [[Vec1]], i32 [[Hi]], i32 1
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_f64_1_0t.v2i32(
+  ; CHECK62-SAME: target("dx.RawBuffer", double, 1, 0) [[B]], i32 %index, i32 0, <2 x i32> [[Vec2]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", double, 1, 0) %buffer, i32 %index, i32 0,
+      double %0)
+  ret void
+}
+
+define void @storev2f64(<2 x double> %0, i32 %index) {
+  ; CHECK: [[B:%.*]] = tail call target("dx.RawBuffer", <2 x double>, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v2f64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", <2 x double>, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v2f64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", <2 x double>, 1, 0) [[B]], i32 %index, i32 0, <2 x double> %0)
+
+  ; CHECK62: [[SD:%.*]] = call { <2 x i32>, <2 x i32> }
+  ; CHECK62-SAME: @llvm.dx.splitdouble.v2i32(<2 x double> %0)
+  ; CHECK62: [[Lo:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[SD]], 0
+  ; CHECK62: [[Hi:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[SD]], 1
+  ; CHECK62: [[Vec:%.*]] = shufflevector <2 x i32> [[Lo]], <2 x i32> [[Hi]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v2f64_1_0t.v4i32(
+  ; CHECK62-SAME: target("dx.RawBuffer", <2 x double>, 1, 0) [[B]], i32 %index, i32 0, <4 x i32> [[Vec]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", <2 x double>, 1, 0) %buffer, i32 %index, i32 0,
+      <2 x double> %0)
+  ret void
+}
+
+define void @storev3f64(<3 x double> %0, i32 %index) {
+  ; CHECK: [[Buf:%.*]] = tail call target("dx.RawBuffer", <3 x double>, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v3f64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", <3 x double>, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v3f64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", <3 x double>, 1, 0) [[B]], i32 %index, i32 0, <3 x double> %0)
+
+  ; CHECK62: [[A:%.*]] = call { <3 x i32>, <3 x i32> } @llvm.dx.splitdouble.v3i32(<3 x double> %0)
+  ; CHECK62: [[B:%.*]] = extractvalue { <3 x i32>, <3 x i32> } [[A]], 0
+  ; CHECK62: [[C:%.*]] = extractvalue { <3 x i32>, <3 x i32> } [[A]], 1
+  ; CHECK62: [[D:%.*]] = shufflevector <3 x i32> [[B]], <3 x i32> [[C]], <6 x i32> <i32 0, i32 3, i32 1, i32 4, i32 2, i32 5>
+  ; CHECK62: [[E:%.*]] = shufflevector <6 x i32> [[D]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v3f64_1_0t.v4i32(target("dx.RawBuffer", <3 x double>, 1, 0) [[Buf]], i32 %index, i32 0, <4 x i32> [[E]])
+  ; CHECK62: [[F:%.*]] = shufflevector <6 x i32> [[D]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v3f64_1_0t.v2i32(target("dx.RawBuffer", <3 x double>, 1, 0) [[Buf]], i32 %index, i32 16, <2 x i32> [[F]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", <3 x double>, 1, 0) %buffer, i32 %index, i32 0,
+      <3 x double> %0)
+  ret void
+}
+
+define void @storev4f64(<4 x double> %0, i32 %index) {
+  ; CHECK: [[Buf:%.*]] = tail call target("dx.RawBuffer", <4 x double>, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v4f64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", <4 x double>, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v4f64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", <4 x double>, 1, 0) [[B]], i32 %index, i32 0, <4 x double> %0)
+
+  ; CHECK62: [[A:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.dx.splitdouble.v4i32(<4 x double> %0)
+  ; CHECK62: [[B:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[A]], 0
+  ; CHECK62: [[C:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[A]], 1
+  ; CHECK62: [[D:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[C]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  ; CHECK62: [[E:%.*]] = shufflevector <8 x i32> [[D]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v4f64_1_0t.v4i32(target("dx.RawBuffer", <4 x double>, 1, 0) [[Buf]], i32 %index, i32 0, <4 x i32> [[E]])
+  ; CHECK62: [[F:%.*]] = shufflevector <8 x i32> [[D]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v4f64_1_0t.v4i32(target("dx.RawBuffer", <4 x double>, 1, 0) [[Buf]], i32 %index, i32 16, <4 x i32> [[F]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", <4 x double>, 1, 0) %buffer, i32 %index, i32 0,
+      <4 x double> %0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/RawBufferStoreInt64.ll b/llvm/test/CodeGen/DirectX/RawBufferStoreInt64.ll
new file mode 100644
index 0000000000000..54ec4d2cd2fb7
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/RawBufferStoreInt64.ll
@@ -0,0 +1,103 @@
+; RUN: opt -mtriple=dxil-pc-shadermodel6.2-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK62
+; RUN: opt -mtriple=dxil-pc-shadermodel6.3-compute -S -dxil-intrinsic-expansion %s | FileCheck %s --check-prefixes=CHECK,CHECK63
+
+define void @storei64(i64 %0, i32 %index) {
+  ; CHECK: [[Buf:%.*]] = tail call target("dx.RawBuffer", i64, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", i64, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", i64, 1, 0) [[Buf]], i32 %index, i32 0, i64 %0) 
+
+  ; check we split the i64 and store the lo and hi bits
+  ; CHECK62: [[A:%.*]] = trunc i64 %0 to i32
+  ; CHECK62: [[B:%.*]] = lshr i64 %0, 32
+  ; CHECK62: [[C:%.*]] = trunc i64 [[B]] to i32
+  ; CHECK62: [[Vec1:%.*]] = insertelement <2 x i32> poison, i32 [[A]], i32 0
+  ; CHECK62: [[Vec2:%.*]] = insertelement <2 x i32> [[Vec1]], i32 [[C]], i32 1
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_i64_1_0t.v2i32(
+  ; CHECK62-SAME: target("dx.RawBuffer", i64, 1, 0) [[Buf]], i32 %index, i32 0, <2 x i32> [[Vec2]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", i64, 1, 0) %buffer, i32 %index, i32 0,
+      i64 %0)
+  ret void
+}
+
+define void @storev2i64(<2 x i64> %0, i32 %index) {
+  ; CHECK: [[Buf:%.*]] = tail call target("dx.RawBuffer", <2 x i64>, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v2i64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", <2 x i64>, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v2i64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", <2 x i64>, 1, 0) [[Buf]], i32 %index, i32 0, <2 x i64> %0)
+
+  ; CHECK62: [[A:%.*]] = trunc <2 x i64> %0 to <2 x i32>
+  ; CHECK62: [[B:%.*]] = lshr <2 x i64> %0, splat (i64 32)
+  ; CHECK62: [[C:%.*]] = trunc <2 x i64> [[B]] to <2 x i32>
+  ; CHECK62: [[Vec:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> [[C]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v2i64_1_0t.v4i32(
+  ; CHECK62-SAME: target("dx.RawBuffer", <2 x i64>, 1, 0) [[Buf]], i32 %index, i32 0, <4 x i32> [[Vec]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", <2 x i64>, 1, 0) %buffer, i32 %index, i32 0,
+      <2 x i64> %0)
+  ret void
+}
+
+define void @storev3i64(<3 x i64> %0, i32 %index) {
+  ; CHECK: [[Buf:%.*]] = tail call target("dx.RawBuffer", <3 x i64>, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v3i64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", <3 x i64>, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v3i64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", <3 x i64>, 1, 0) [[Buf]], i32 %index, i32 0, <3 x i64> %0)
+
+  ; CHECK62: [[A:%.*]] = trunc <3 x i64> %0 to <3 x i32>
+  ; CHECK62: [[B:%.*]] = lshr <3 x i64> %0, splat (i64 32)
+  ; CHECK62: [[C:%.*]] = trunc <3 x i64> [[B]] to <3 x i32>
+  ; CHECK62: [[D:%.*]] = shufflevector <3 x i32> [[A]], <3 x i32> [[C]], <6 x i32> <i32 0, i32 3, i32 1, i32 4, i32 2, i32 5>
+  ; CHECK62: [[E:%.*]] = shufflevector <6 x i32> [[D]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v3i64_1_0t.v4i32(target("dx.RawBuffer", <3 x i64>, 1, 0) [[Buf]], i32 %index, i32 0, <4 x i32> [[E]])
+  ; CHECK62: [[F:%.*]] = shufflevector <6 x i32> [[D]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v3i64_1_0t.v2i32(target("dx.RawBuffer", <3 x i64>, 1, 0) [[Buf]], i32 %index, i32 16, <2 x i32> [[F]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", <3 x i64>, 1, 0) %buffer, i32 %index, i32 0,
+      <3 x i64> %0)
+  ret void
+}
+
+define void @storev4i64(<4 x i64> %0, i32 %index) {
+  ; CHECK: [[Buf:%.*]] = tail call target("dx.RawBuffer", <4 x i64>, 1, 0)
+  ; CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v4i64_1_0t(
+  ; CHECK-SAME: i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %buffer = tail call target("dx.RawBuffer", <4 x i64>, 1, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_v4i64_1_0t(
+          i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  ; check we don't modify the code in sm6.3 or later
+  ; CHECK63: call void @llvm.dx.resource.store.rawbuffer
+  ; CHECK63-SAME: target("dx.RawBuffer", <4 x i64>, 1, 0) [[Buf]], i32 %index, i32 0, <4 x i64> %0)
+
+  ; CHECK62: [[A:%.*]] = trunc <4 x i64> %0 to <4 x i32>
+  ; CHECK62: [[B:%.*]] = lshr <4 x i64> %0, splat (i64 32)
+  ; CHECK62: [[C:%.*]] = trunc <4 x i64> [[B]] to <4 x i32>
+  ; CHECK62: [[D:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> [[C]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  ; CHECK62: [[E:%.*]] = shufflevector <8 x i32> [[D]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v4i64_1_0t.v4i32(target("dx.RawBuffer", <4 x i64>, 1, 0) [[Buf]], i32 %index, i32 0, <4 x i32> [[E]])
+  ; CHECK62: [[F:%.*]] = shufflevector <8 x i32> [[D]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  ; CHECK62: call void @llvm.dx.resource.store.rawbuffer.tdx.RawBuffer_v4i64_1_0t.v4i32(target("dx.RawBuffer", <4 x i64>, 1, 0) [[Buf]], i32 %index, i32 16, <4 x i32> [[F]])
+  call void @llvm.dx.resource.store.rawbuffer(
+      target("dx.RawBuffer", <4 x i64>, 1, 0) %buffer, i32 %index, i32 0,
+      <4 x i64> %0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/finalize_linkage.ll b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
index df691db5cff36..dc1140f1c9160 100644
--- a/llvm/test/CodeGen/DirectX/finalize_linkage.ll
+++ b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
@@ -4,7 +4,28 @@
 target triple = "dxilv1.5-pc-shadermodel6.5-compute"
 
 ; DXILFinalizeLinkage changes linkage of all functions that are hidden to
-; internal.
+; internal, and converts private global variables to internal linkage.
+
+; CHECK: @switch.table = internal unnamed_addr constant [4 x i32]
+@switch.table = private unnamed_addr constant [4 x i32] [i32 1, i32 257, i32 65793, i32 16843009], align 4
+
+; CHECK: @private_array = internal constant [3 x float]
+@private_array = private constant [3 x float] [float 1.0, float 2.0, float 3.0], align 4
+
+; CHECK: @private_var = internal global i32
+@private_var = private global i32 1, align 4
+
+; Internal global should remain internal
+; CHECK: @internal_var = internal global i32
+@internal_var = internal global i32 1, align 4
+
+; External global should remain external
+; CHECK: @external_var = external global i32
+@external_var = external global i32, align 4
+
+; Hidden global should remain hidden
+; CHECK: @hidden_var = hidden global i32
+@hidden_var = hidden global i32 1, align 4
 
 ; CHECK-NOT: define internal void @"?f1@@YAXXZ"()
 define void @"?f1@@YAXXZ"() #0 {
diff --git a/llvm/test/CodeGen/DirectX/legalize-lifetimes.ll b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll
similarity index 74%
rename from llvm/test/CodeGen/DirectX/legalize-lifetimes.ll
rename to llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll
index 44b2419a05d64..e485fa20ddfce 100644
--- a/llvm/test/CodeGen/DirectX/legalize-lifetimes.ll
+++ b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll
@@ -3,9 +3,9 @@
 ; CHECK-LABEL: define void @test_legal_lifetime() {
 ; CHECK-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [1 x i32], align 4
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[ACCUM_I_FLAT]], i32 0
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
+; CHECK-NEXT:    store [1 x i32] zeroinitializer, ptr [[ACCUM_I_FLAT]], align 4
 ; CHECK-NEXT:    store i32 0, ptr [[GEP]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
+; CHECK-NEXT:    store [1 x i32] zeroinitializer, ptr [[ACCUM_I_FLAT]], align 4
 ; CHECK-NEXT:    ret void
 ;
 define void @test_legal_lifetime()  {
@@ -16,3 +16,7 @@ define void @test_legal_lifetime()  {
   call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
   ret void
 }
+
+; Set the validator version to 1.5
+!dx.valver = !{!0}
+!0 = !{i32 1, i32 5}
diff --git a/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll
new file mode 100644
index 0000000000000..6552ccddddab4
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -passes='dxil-op-lower' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,CHECK-SM63
+; RUN: opt -S -passes='dxil-op-lower' -mtriple=dxil-pc-shadermodel6.6-library %s | FileCheck %s --check-prefixes=CHECK,CHECK-SM66
+
+; CHECK-LABEL: define void @test_legal_lifetime() {
+; 
+; CHECK-SM63-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [1 x i32], align 4
+; CHECK-SM63-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[ACCUM_I_FLAT]], i32 0
+; CHECK-SM63-NEXT:    store [1 x i32] undef, ptr [[ACCUM_I_FLAT]], align 4
+; CHECK-SM63-NEXT:    store i32 0, ptr [[GEP]], align 4
+; CHECK-SM63-NEXT:    store [1 x i32] undef, ptr [[ACCUM_I_FLAT]], align 4
+; 
+; CHECK-SM66-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [1 x i32], align 4
+; CHECK-SM66-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[ACCUM_I_FLAT]], i32 0
+; CHECK-SM66-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
+; CHECK-SM66-NEXT:    store i32 0, ptr [[GEP]], align 4
+; CHECK-SM66-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
+; 
+; CHECK-NEXT:    ret void
+;
+define void @test_legal_lifetime()  {
+  %accum.i.flat = alloca [1 x i32], align 4
+  %gep = getelementptr i32, ptr %accum.i.flat, i32 0
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %accum.i.flat)
+  store i32 0, ptr %gep, align 4
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
+  ret void
+}
+
+; Set the validator version to 1.6
+!dx.valver = !{!0}
+!0 = !{i32 1, i32 6}
diff --git a/llvm/test/CodeGen/Hexagon/aggr-copy-order.ll b/llvm/test/CodeGen/Hexagon/aggr-copy-order.ll
index 80467b6c4492f..2ebfc993ef2a3 100644
--- a/llvm/test/CodeGen/Hexagon/aggr-copy-order.ll
+++ b/llvm/test/CodeGen/Hexagon/aggr-copy-order.ll
@@ -1,7 +1,5 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=hexagon -mattr=-packets -hexagon-check-bank-conflict=0 < %s | FileCheck %s
-; Do not check stores. They undergo some optimizations in the DAG combiner
-; resulting in getting out of order. There is likely little that can be
-; done to keep the original order.
 
 target triple = "hexagon"
 
@@ -9,10 +7,42 @@ target triple = "hexagon"
 
 ; Function Attrs: nounwind
 define void @f0(ptr %a0, ptr %a1) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = memw(r1+#0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#0) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r29+#0) = r0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = memw(r1+#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#4) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = memw(r1+#8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#8) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
 b0:
-; CHECK: = memw({{.*}}+#0)
-; CHECK: = memw({{.*}}+#4)
-; CHECK: = memw({{.*}}+#8)
   %v0 = alloca ptr, align 4
   %v1 = alloca ptr, align 4
   store ptr %a0, ptr %v0, align 4
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll b/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll
index d56520da78ae2..965f3c684963a 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=hexagon < %s | FileCheck %s
 
 ; Check that this code does compile.
@@ -6,9 +7,248 @@ target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i
 target triple = "hexagon"
 
 ; Function Attrs: nounwind
-; CHECK-LABEL: f0:
-; CHECK: vinsert
 define <32 x float> @f0(ptr %a0, ptr %a1) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vxor(v0,v0)
+; CHECK-NEXT:     r2 = memw(r0+#0)
+; CHECK-NEXT:     r6 = memw(r0+#8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = v0
+; CHECK-NEXT:     r3 = memw(r0+#64)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r4 = memw(r0+#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#68)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r7 = memw(r1+r4<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#72)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#12)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#76)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#80)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#20)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#84)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#24)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#88)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#28)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#92)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#32)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#96)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#36)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#100)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#40)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#104)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#44)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#108)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#48)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#112)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#52)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#116)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#56)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#120)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#60)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r2 = #68
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r5 = memw(r0+#124)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r4 = memw(r1+r6<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r0 = memw(r1+r5<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vror(v1,r2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vor(v1,v0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
 b0:
   %v1 = load i32, ptr %a0, align 4
   %v2 = getelementptr float, ptr %a1, i32 %v1
@@ -173,9 +413,380 @@ b0:
 }
 
 ; Function Attrs: nounwind
-; CHECK-LABEL: f1:
-; CHECK: vinsert
 define <64 x half> @f1(ptr %a0, ptr %a1) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vxor(v0,v0)
+; CHECK-NEXT:     r6 = memw(r0+#0)
+; CHECK-NEXT:     r2 = memw(r0+#16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = v0
+; CHECK-NEXT:     r8 = memw(r0+#4)
+; CHECK-NEXT:     r4 = memw(r0+#24)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = memw(r0+#8)
+; CHECK-NEXT:     r3 = memw(r0+#20)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memw(r0+#28)
+; CHECK-NEXT:     r9 = memw(r0+#12)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r8 = memuh(r1+r8<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r8 = combine(r8.l,r6.l)
+; CHECK-NEXT:     r14 = memw(r0+#128)
+; CHECK-NEXT:     r6 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r8)
+; CHECK-NEXT:     r5 = memw(r0+#136)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r2.l)
+; CHECK-NEXT:     r2 = combine(r6.l,r4.l)
+; CHECK-NEXT:     r12 = memw(r0+#132)
+; CHECK-NEXT:     r6 = memw(r0+#140)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r9 = memuh(r1+r9<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = combine(r9.l,r7.l)
+; CHECK-NEXT:     r9 = memuh(r1+r14<<#1)
+; CHECK-NEXT:     r12 = memuh(r1+r12<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     r15 = combine(r12.l,r9.l)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r15)
+; CHECK-NEXT:     r5 = combine(r6.l,r5.l)
+; CHECK-NEXT:     r6 = memw(r0+#144)
+; CHECK-NEXT:     r4 = memw(r0+#32)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memw(r0+#148)
+; CHECK-NEXT:     r28 = memw(r0+#36)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r10 = memw(r0+#40)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r5)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r6 = combine(r7.l,r6.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memw(r0+#152)
+; CHECK-NEXT:     r9 = memuh(r1+r28<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memw(r0+#156)
+; CHECK-NEXT:     r5 = memw(r0+#44)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r6)
+; CHECK-NEXT:     r4 = combine(r9.l,r4.l)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r11 = memw(r0+#48)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r6 = memw(r0+#52)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     r3 = combine(r3.l,r7.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r0+#160)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r0+#164)
+; CHECK-NEXT:     r13 = memw(r0+#168)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r8 = memuh(r1+r10<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r7.l)
+; CHECK-NEXT:     r5 = combine(r5.l,r8.l)
+; CHECK-NEXT:     r7 = memw(r0+#172)
+; CHECK-NEXT:     r9 = memuh(r1+r11<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     v1.w = vinsert(r5)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r14 = memw(r0+#176)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r6 = combine(r6.l,r9.l)
+; CHECK-NEXT:     r9 = memuh(r1+r13<<#1)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = combine(r7.l,r9.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r12 = memw(r0+#56)
+; CHECK-NEXT:     r15 = memw(r0+#180)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r7)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memw(r0+#60)
+; CHECK-NEXT:     r4 = memw(r0+#64)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r6)
+; CHECK-NEXT:     r7 = memuh(r1+r14<<#1)
+; CHECK-NEXT:     r28 = memuh(r1+r15<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = combine(r28.l,r7.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r2 = memw(r0+#68)
+; CHECK-NEXT:     r6 = memw(r0+#184)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r7)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r8 = memuh(r1+r12<<#1)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r8.l)
+; CHECK-NEXT:     r7 = memw(r0+#188)
+; CHECK-NEXT:     r5 = memw(r0+#72)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r4.l)
+; CHECK-NEXT:     r4 = memw(r0+#76)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r6 = combine(r7.l,r6.l)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:     r7 = memw(r0+#88)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r6)
+; CHECK-NEXT:     r4 = combine(r4.l,r5.l)
+; CHECK-NEXT:     r6 = memw(r0+#192)
+; CHECK-NEXT:     r10 = memw(r0+#80)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r5 = memw(r0+#92)
+; CHECK-NEXT:     r2 = memw(r0+#196)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memw(r0+#84)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r4 = combine(r5.l,r7.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:     r11 = memw(r0+#200)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r6.l)
+; CHECK-NEXT:     r5 = memw(r0+#204)
+; CHECK-NEXT:     r8 = memuh(r1+r10<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r7 = memuh(r1+r11<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r8.l)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r6 = memw(r0+#96)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     r5 = combine(r5.l,r7.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memw(r0+#208)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r5)
+; CHECK-NEXT:     r7 = memw(r0+#212)
+; CHECK-NEXT:     r2 = memw(r0+#100)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r5 = memw(r0+#104)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r4 = memw(r0+#108)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r7.l,r3.l)
+; CHECK-NEXT:     r7 = memw(r0+#216)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:     r3 = memw(r0+#220)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r6.l)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r4 = combine(r4.l,r5.l)
+; CHECK-NEXT:     r2 = memw(r0+#224)
+; CHECK-NEXT:     r6 = memuh(r1+r7<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r5 = memw(r0+#228)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r6.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r6 = memw(r0+#112)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     r3 = memw(r0+#116)
+; CHECK-NEXT:     r4 = memw(r0+#232)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r7 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r5.l,r2.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r5 = memw(r0+#236)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r2 = combine(r3.l,r7.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memw(r0+#240)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r4 = combine(r5.l,r4.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r2 = memw(r0+#120)
+; CHECK-NEXT:     r6 = memw(r0+#244)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r4)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memw(r0+#124)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r6 = memw(r0+#248)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r5.l,r3.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r0 = memw(r0+#252)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r7.l,r2.l)
+; CHECK-NEXT:     r2 = #68
+; CHECK-NEXT:     r0 = memuh(r1+r0<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     r0 = combine(r0.l,r6.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vror(v1,r2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vor(v1,v0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
 b0:
   %v1 = load i32, ptr %a0, align 4
   %v2 = getelementptr half, ptr %a1, i32 %v1
diff --git a/llvm/test/CodeGen/Hexagon/calloperand-v128i1.ll b/llvm/test/CodeGen/Hexagon/calloperand-v128i1.ll
index ddac8c1cd8279..9d323b455449e 100644
--- a/llvm/test/CodeGen/Hexagon/calloperand-v128i1.ll
+++ b/llvm/test/CodeGen/Hexagon/calloperand-v128i1.ll
@@ -1,10 +1,20 @@
-;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length128b < %s -o - | FileCheck %s
+;RUN: llc -mtriple=hexagon  < %s -o - | FileCheck %s --check-prefix=CHECK
+;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length64b < %s -o - | FileCheck %s --check-prefixes=CHECK-64,CHECK-64-128
+;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length128b < %s -o - | FileCheck %s --check-prefixes=CHECK-128,CHECK-64-128
 
 ; CHECK-LABEL: compare_vectors
-; CHECK: [[REG1:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.b,v{{[0-9]+}}.b)
-; CHECK: [[REG2:(r[0-9]+)]] = #-1
-; CHECK: v0 = vand([[REG1]],[[REG2]])
-
+; CHECK: [[REG8:(r[0-9]+):[0-9]]] = CONST64(#72340172838076673)
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
+; CHECK-128: [[REG1:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.b,v{{[0-9]+}}.b)
+; CHECK-128: [[REG2:(r[0-9]+)]] = #-1
+; CHECK-128: v0 = vand([[REG1]],[[REG2]])
+; CHECK-64: [[REG5:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.b,v{{[0-9]+}}.b)
+; CHECK-64: [[REG6:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.b,v{{[0-9]+}}.b)
+; CHECK-64: [[REG7:(r[0-9]+)]] = #-1
+; CHECK-64: v0 = vand([[REG5]],[[REG7]])
+; CHECK-64: v1 = vand([[REG6]],[[REG7]])
 define void @compare_vectors(<128 x i8> %a, <128 x i8> %b) {
 entry:
   %result = icmp eq <128 x i8> %a, %b
@@ -13,11 +23,13 @@ entry:
 }
 
 ; CHECK-LABEL: f.1:
-; CHECK: [[REG3:(q[0-9]+)]] = vand(v0,r{{[0-9]+}})
-; CHECK: [[REG4:(v[0-9]+)]] = vand([[REG3]],r{{[0-9]+}})
-; CHECK: r{{[0-9]+}} = vextract([[REG4]],r{{[0-9]+}})
+; CHECK: [[REG9:(r[0-9]+)]] = and([[REG9]],##16843009)
+; CHECK: [[REG10:(r[0-9]+)]] = and([[REG10]],##16843009)
+; CHECK-64-128: [[REG3:(q[0-9]+)]] = vand(v0,r{{[0-9]+}})
+; CHECK-64-128: [[REG4:(v[0-9]+)]] = vand([[REG3]],r{{[0-9]+}})
+; CHECK-64-128: r{{[0-9]+}} = vextract([[REG4]],r{{[0-9]+}})
 
-define i32 @f.1(<128 x i1> %vec) {
+define i32 @f.1(<128 x i1> %vec){
   %element = extractelement <128 x i1> %vec, i32 6
   %is_true = icmp eq i1 %element, true
   br i1 %is_true, label %if_true, label %if_false
diff --git a/llvm/test/CodeGen/Hexagon/calloperand-v16i1.ll b/llvm/test/CodeGen/Hexagon/calloperand-v16i1.ll
index bbb2697246df1..6cf1a3fca70b7 100644
--- a/llvm/test/CodeGen/Hexagon/calloperand-v16i1.ll
+++ b/llvm/test/CodeGen/Hexagon/calloperand-v16i1.ll
@@ -1,10 +1,15 @@
-;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length64b < %s -o - | FileCheck %s --check-prefix=CHECK
-;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length128b < %s -o - | FileCheck %s --check-prefix=CHECK
+;RUN: llc -mtriple=hexagon  < %s -o - | FileCheck %s --check-prefix=CHECK
+;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length64b < %s -o - | FileCheck %s --check-prefix=CHECK-HVX
+;RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length128b < %s -o - | FileCheck %s --check-prefix=CHECK-HVX
 
 ; CHECK-LABEL: compare_vectors
-; CHECK: [[REG1:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.w,v{{[0-9]+}}.w)
-; CHECK: [[REG2:(r[0-9]+)]] = #-1
-; CHECK: v0  = vand([[REG1]],[[REG2]])
+; CHECK: [[REG5:(r[0-9]+):[0-9]]] = CONST64(#72340172838076673)
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG5]])
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG5]])
+
+; CHECK-HVX: [[REG1:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.w,v{{[0-9]+}}.w)
+; CHECK-HVX: [[REG2:(r[0-9]+)]] = #-1
+; CHECK-HVX: v0  = vand([[REG1]],[[REG2]])
 
 define void @compare_vectors(<16 x i32> %a, <16 x i32> %b) {
 entry:
@@ -14,9 +19,11 @@ entry:
 }
 
 ; CHECK-LABEL: f.1:
-; CHECK: [[REG3:(q[0-9]+)]] = vand(v0,r{{[0-9]+}})
-; CHECK: [[REG4:(v[0-9]+)]] = vand([[REG3]],r{{[0-9]+}})
-; CHECK: r{{[0-9]+}} = vextract([[REG4]],r{{[0-9]+}})
+; CHECK: [[REG3:(r[0-9]+)]] = and([[REG3]],##16843009)
+; CHECK: [[REG4:(r[0-9]+)]] = and([[REG4]],##16843009)
+; CHECK-HVX: [[REG3:(q[0-9]+)]] = vand(v0,r{{[0-9]+}})
+; CHECK-HVX: [[REG4:(v[0-9]+)]] = vand([[REG3]],r{{[0-9]+}})
+; CHECK-HVX: r{{[0-9]+}} = vextract([[REG4]],r{{[0-9]+}})
 
 define i32 @f.1(<16 x i1> %vec) {
   %element = extractelement <16 x i1> %vec, i32 6
diff --git a/llvm/test/CodeGen/Hexagon/calloperand-v32i1.ll b/llvm/test/CodeGen/Hexagon/calloperand-v32i1.ll
index a73478728d910..c43ad70b94100 100644
--- a/llvm/test/CodeGen/Hexagon/calloperand-v32i1.ll
+++ b/llvm/test/CodeGen/Hexagon/calloperand-v32i1.ll
@@ -1,7 +1,11 @@
+; RUN: llc -mtriple=hexagon  < %s -o - | FileCheck %s --check-prefix=CHECK
 ; RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length64b < %s -o - | FileCheck %s --check-prefix=CHECK-64
 ; RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length128b < %s -o - | FileCheck %s --check-prefix=CHECK-128
 
 ; CHECK-LABEL: compare_vectors
+; CHECK: [[REG8:(r[0-9]+):[0-9]]] = CONST64(#72340172838076673)
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
 ; CHECK-64: [[REG1:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.h,v{{[0-9]+}}.h)
 ; CHECK-64: [[REG2:(r[0-9]+)]] = #-1
 ; CHECK-64: v0 = vand([[REG1]],[[REG2]])
@@ -21,6 +25,8 @@ entry:
 }
 
 ; CHECK-LABEL: f.1:
+; CHECK: [[REG9:(r[0-9]+)]] = and([[REG9]],##16843009)
+; CHECK: [[REG10:(r[0-9]+)]] = and([[REG10]],##16843009)
 ; CHECK-64: [[REG3:(q[0-9]+)]] = vand(v0,r{{[0-9]+}})
 ; CHECK-64: [[REG4:(v[0-9]+)]] = vand([[REG3]],r{{[0-9]+}})
 ; CHECK-64: r{{[0-9]+}} = vextract([[REG4]],r{{[0-9]+}})
diff --git a/llvm/test/CodeGen/Hexagon/calloperand-v64i1.ll b/llvm/test/CodeGen/Hexagon/calloperand-v64i1.ll
index 7cc562085a7e6..90a2da14f1971 100644
--- a/llvm/test/CodeGen/Hexagon/calloperand-v64i1.ll
+++ b/llvm/test/CodeGen/Hexagon/calloperand-v64i1.ll
@@ -1,7 +1,12 @@
+; RUN: llc -mtriple=hexagon  < %s -o - | FileCheck %s --check-prefix=CHECK
 ; RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length64b < %s -o - | FileCheck %s --check-prefix=CHECK-64
 ; RUN: llc -mtriple=hexagon -mattr=+hvxv79,+hvx-length128b < %s -o - | FileCheck %s --check-prefix=CHECK-128
 
 ; CHECK-LABEL: compare_vectors
+; CHECK: [[REG8:(r[0-9]+):[0-9]]] = CONST64(#72340172838076673)
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
+; CHECK: r{{[0-9]+}}:{{[0-9]+}} = and(r{{[0-9]+}}:{{[0-9]+}},[[REG8]])
 ; CHECK-64: [[REG1:(q[0-9]+)]] = vcmp.eq(v{{[0-9]+}}.b,v{{[0-9]+}}.b)
 ; CHECK-64: [[REG2:(r[0-9]+)]] = #-1
 ; CHECK-64: v0 = vand([[REG1]],[[REG2]])
@@ -21,6 +26,8 @@ entry:
 }
 
 ; CHECK-LABEL: f.1:
+; CHECK: [[REG9:(r[0-9]+)]] = and([[REG9]],##16843009)
+; CHECK: [[REG10:(r[0-9]+)]] = and([[REG10]],##16843009)
 ; CHECK-64: [[REG3:(q[0-9]+)]] = vand(v0,r{{[0-9]+}})
 ; CHECK-64: [[REG4:(v[0-9]+)]] = vand([[REG3]],r{{[0-9]+}})
 ; CHECK-64: r{{[0-9]+}} = vextract([[REG4]],r{{[0-9]+}})
diff --git a/llvm/test/CodeGen/Hexagon/clr_set_toggle.ll b/llvm/test/CodeGen/Hexagon/clr_set_toggle.ll
index 99e90165bd074..6bfed9e7df46e 100644
--- a/llvm/test/CodeGen/Hexagon/clr_set_toggle.ll
+++ b/llvm/test/CodeGen/Hexagon/clr_set_toggle.ll
@@ -1,10 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=hexagon -hexagon-bit=0 < %s | FileCheck %s
 ; Optimized bitwise operations.
 
 define i32 @my_clrbit(i32 %x) nounwind {
+; CHECK-LABEL: my_clrbit:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = clrbit(r0,#31)
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_clrbit
-; CHECK: r{{[0-9]+}} = clrbit(r{{[0-9]+}},#31)
   %x.addr = alloca i32, align 4
   store i32 %x, ptr %x.addr, align 4
   %0 = load i32, ptr %x.addr, align 4
@@ -13,9 +24,18 @@ entry:
 }
 
 define i64 @my_clrbit2(i64 %x) nounwind {
+; CHECK-LABEL: my_clrbit2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = clrbit(r0,#31)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_clrbit2
-; CHECK: r{{[0-9]+}} = clrbit(r{{[0-9]+}},#31)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -24,9 +44,18 @@ entry:
 }
 
 define i64 @my_clrbit3(i64 %x) nounwind {
+; CHECK-LABEL: my_clrbit3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = clrbit(r1,#31)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_clrbit3
-; CHECK: r{{[0-9]+}} = clrbit(r{{[0-9]+}},#31)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -35,9 +64,19 @@ entry:
 }
 
 define i32 @my_clrbit4(i32 %x) nounwind {
+; CHECK-LABEL: my_clrbit4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = clrbit(r0,#13)
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_clrbit4
-; CHECK: r{{[0-9]+}} = clrbit(r{{[0-9]+}},#13)
   %x.addr = alloca i32, align 4
   store i32 %x, ptr %x.addr, align 4
   %0 = load i32, ptr %x.addr, align 4
@@ -46,9 +85,18 @@ entry:
 }
 
 define i64 @my_clrbit5(i64 %x) nounwind {
+; CHECK-LABEL: my_clrbit5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = clrbit(r0,#13)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_clrbit5
-; CHECK: r{{[0-9]+}} = clrbit(r{{[0-9]+}},#13)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -57,9 +105,18 @@ entry:
 }
 
 define i64 @my_clrbit6(i64 %x) nounwind {
+; CHECK-LABEL: my_clrbit6:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = clrbit(r1,#27)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_clrbit6
-; CHECK: r{{[0-9]+}} = clrbit(r{{[0-9]+}},#27)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -68,9 +125,18 @@ entry:
 }
 
 define zeroext i16 @my_setbit(i16 zeroext %crc) nounwind {
+; CHECK-LABEL: my_setbit:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = setbit(r0,#15)
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memh(r29+#6) = r0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_setbit
-; CHECK: r{{[0-9]+}} = setbit(r{{[0-9]+}},#15)
   %crc.addr = alloca i16, align 2
   store i16 %crc, ptr %crc.addr, align 2
   %0 = load i16, ptr %crc.addr, align 2
@@ -83,9 +149,19 @@ entry:
 }
 
 define i32 @my_setbit2(i32 %x) nounwind {
+; CHECK-LABEL: my_setbit2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = setbit(r0,#15)
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_setbit2
-; CHECK: r{{[0-9]+}} = setbit(r{{[0-9]+}},#15)
   %x.addr = alloca i32, align 4
   store i32 %x, ptr %x.addr, align 4
   %0 = load i32, ptr %x.addr, align 4
@@ -94,9 +170,18 @@ entry:
 }
 
 define i64 @my_setbit3(i64 %x) nounwind {
+; CHECK-LABEL: my_setbit3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = setbit(r0,#15)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_setbit3
-; CHECK: r{{[0-9]+}} = setbit(r{{[0-9]+}},#15)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -105,9 +190,19 @@ entry:
 }
 
 define i32 @my_setbit4(i32 %x) nounwind {
+; CHECK-LABEL: my_setbit4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = setbit(r0,#31)
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_setbit4
-; CHECK: r{{[0-9]+}} = setbit(r{{[0-9]+}},#31)
   %x.addr = alloca i32, align 4
   store i32 %x, ptr %x.addr, align 4
   %0 = load i32, ptr %x.addr, align 4
@@ -116,9 +211,18 @@ entry:
 }
 
 define i64 @my_setbit5(i64 %x) nounwind {
+; CHECK-LABEL: my_setbit5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = setbit(r1,#13)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_setbit5
-; CHECK: r{{[0-9]+}} = setbit(r{{[0-9]+}},#13)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -127,9 +231,18 @@ entry:
 }
 
 define zeroext i16 @my_togglebit(i16 zeroext %crc) nounwind {
+; CHECK-LABEL: my_togglebit:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = togglebit(r0,#15)
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memh(r29+#6) = r0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_togglebit
-; CHECK: r{{[0-9]+}} = togglebit(r{{[0-9]+}},#15)
   %crc.addr = alloca i16, align 2
   store i16 %crc, ptr %crc.addr, align 2
   %0 = load i16, ptr %crc.addr, align 2
@@ -142,9 +255,19 @@ entry:
 }
 
 define i32 @my_togglebit2(i32 %x) nounwind {
+; CHECK-LABEL: my_togglebit2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = togglebit(r0,#15)
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_togglebit2
-; CHECK: r{{[0-9]+}} = togglebit(r{{[0-9]+}},#15)
   %x.addr = alloca i32, align 4
   store i32 %x, ptr %x.addr, align 4
   %0 = load i32, ptr %x.addr, align 4
@@ -153,9 +276,18 @@ entry:
 }
 
 define i64 @my_togglebit3(i64 %x) nounwind {
+; CHECK-LABEL: my_togglebit3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = togglebit(r0,#15)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_togglebit3
-; CHECK: r{{[0-9]+}} = togglebit(r{{[0-9]+}},#15)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
@@ -164,9 +296,18 @@ entry:
 }
 
 define i64 @my_togglebit4(i64 %x) nounwind {
+; CHECK-LABEL: my_togglebit4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = togglebit(r1,#20)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     memd(r29+#0) = r1:0
+; CHECK-NEXT:    }
 entry:
-; CHECK-LABEL: my_togglebit4
-; CHECK: r{{[0-9]+}} = togglebit(r{{[0-9]+}},#20)
   %x.addr = alloca i64, align 8
   store i64 %x, ptr %x.addr, align 8
   %0 = load i64, ptr %x.addr, align 8
diff --git a/llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll b/llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll
new file mode 100644
index 0000000000000..b23366bc11aca
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=hexagon -verify-machineinstrs  < %s | FileCheck %s
+
+@.str = private unnamed_addr constant [31 x i8] c"DHRYSTONE PROGRAM, 3'RD STRING\00", align 1
+@.str1 = private unnamed_addr constant [3 x i8] c"%s\00", align 1
+
+; Function Attrs: nounwind
+declare i32 @printf(i8* nocapture readonly, ...)
+
+; Function Attrs: nounwind
+define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    .cfi_def_cfa r30, 8
+; CHECK-NEXT:    .cfi_offset r31, -4
+; CHECK-NEXT:    .cfi_offset r30, -8
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = ##.L.str1
+; CHECK-NEXT:     r3:2 = CONST64(#2325073635944967245)
+; CHECK-NEXT:     allocframe(r29,#40):raw
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = add(r29,#8)
+; CHECK-NEXT:     r7:6 = CONST64(#4706902966564560965)
+; CHECK-NEXT:     r5:4 = CONST64(#5642821575076104260)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memb(r29+#38) = #0
+; CHECK-NEXT:     memw(r29+#0) = r1
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r29+#24) = r3:2
+; CHECK-NEXT:     memd(r29+#16) = r7:6
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r29+#8) = r5:4
+; CHECK-NEXT:     memh(r29+#36) = ##18254
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call printf
+; CHECK-NEXT:     memw(r29+#32) = ##1230132307
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = #0
+; CHECK-NEXT:     dealloc_return
+; CHECK-NEXT:    }
+entry:
+  %blah = alloca [30 x i8], align 8
+  %arraydecay = getelementptr inbounds [30 x i8], [30 x i8]* %blah, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %arraydecay, i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str, i32 0, i32 0), i32 31, i32 1, i1 false)
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str1, i32 0, i32 0), i8* %arraydecay)
+  ret i32 0
+}
+
+; Function Attrs: nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
index 26bbd71ad44de..e7bf5a02b3640 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
@@ -13,14 +13,14 @@
 ; CHECK: memb(r{{[0-9]+}}+#0) =
 
 ; Function Attrs: nounwind optsize
-define void @f0(i1 %x) #0 {
+define void @f0(i1 %x, i32 %y, ptr %p1, ptr %p2, ptr %p3) #0 {
 b0:
   br label %b1
 
 b1:                                               ; preds = %b5, %b0
-  %v0 = load i16, ptr undef, align 2, !tbaa !0
+  %v0 = load i16, ptr %p1, align 2, !tbaa !0
   %v1 = sext i16 %v0 to i32
-  %v2 = load i16, ptr undef, align 2, !tbaa !0
+  %v2 = load i16, ptr %p2, align 2, !tbaa !0
   %v3 = sext i16 %v2 to i32
   %v4 = and i32 %v1, 7
   %v5 = and i32 %v3, 7
@@ -31,8 +31,8 @@ b2:                                               ; preds = %b4, %b1
 
 b3:                                               ; preds = %b3, %b2
   %v6 = phi i32 [ 0, %b2 ], [ %v22, %b3 ]
-  %v7 = add i32 %v6, undef
-  %v8 = icmp slt i32 undef, %v7
+  %v7 = add i32 %v6, %y
+  %v8 = icmp slt i32 %y, %v7
   %v9 = add nsw i32 %v7, 1
   %v10 = select i1 %x, i32 1, i32 %v9
   %v11 = add i32 %v10, 0
@@ -46,16 +46,16 @@ b3:                                               ; preds = %b3, %b2
   %v19 = add i32 %v18, 0
   %v20 = lshr i32 %v19, 6
   %v21 = trunc i32 %v20 to i8
-  store i8 %v21, ptr undef, align 1, !tbaa !4
+  store i8 %v21, ptr %p3, align 1, !tbaa !4
   %v22 = add i32 %v6, 1
   %v23 = icmp eq i32 %v22, 2
   br i1 %v23, label %b4, label %b3
 
 b4:                                               ; preds = %b3
-  br i1 undef, label %b5, label %b2
+  br i1 true, label %b5, label %b2
 
 b5:                                               ; preds = %b4
-  br i1 undef, label %b1, label %b6
+  br i1 false, label %b1, label %b6
 
 b6:                                               ; preds = %b5
   ret void
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll b/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
index 568d596266b0c..f9092a83a628c 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
@@ -1,12 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
 
-; Check that store is post-incremented.
-; CHECK: r{{[0-9]+:[0-9]+}} = vasrw(r{{[0-9]+:[0-9]+}},r{{[0-9]+}})
-; CHECK: r{{[0-9]+:[0-9]+}} = vaslw(r{{[0-9]+:[0-9]+}},r{{[0-9]+}})
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
 target triple = "hexagon"
 
+; Check that store is post-incremented.
 define void @foo(ptr nocapture %buf, ptr nocapture %dest, i32 %offset, i32 %oddBlock, i32 %gb) #0 {
+; CHECK-LABEL: foo:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = sub(r2,r3)
+; CHECK-NEXT:     r3 = sub(#31,r4)
+; CHECK-NEXT:     p0 = cmp.eq(r3,#0)
+; CHECK-NEXT:     r5 = memw(r0+#0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = asr(r5,r4)
+; CHECK-NEXT:     r29 = add(r29,#-24)
+; CHECK-NEXT:     r9:8 = combine(#-1,#-1)
+; CHECK-NEXT:     memw(r0+#0) = r5.new
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = and(r2,#7)
+; CHECK-NEXT:     r2 = r3
+; CHECK-NEXT:     memd(r29+#8) = r19:18
+; CHECK-NEXT:     memd(r29+#16) = r17:16
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7:6 = vaslw(r3:2,#1)
+; CHECK-NEXT:     if (p0) r5 = add(r5,#1)
+; CHECK-NEXT:     memd(r29+#0) = r21:20
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = addasl(r1,r5,#2)
+; CHECK-NEXT:     r7:6 = vaddw(r7:6,r9:8)
+; CHECK-NEXT:     r14 = memw(r1+r5<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15 = memw(r0+#256)
+; CHECK-NEXT:     r10 = memw(r0+#512)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vasrw(r15:14,#31)
+; CHECK-NEXT:     r19:18 = vasrw(r15:14,r3)
+; CHECK-NEXT:     r12 = memw(r0+#1024)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r9:8,r19:18)
+; CHECK-NEXT:     r17:16 = xor(r9:8,r7:6)
+; CHECK-NEXT:     r11 = memw(r0+#768)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15:14 = vmux(p0,r15:14,r17:16)
+; CHECK-NEXT:     r19:18 = vasrw(r11:10,#31)
+; CHECK-NEXT:     r13 = memw(r0+#1280)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r11:10,r3)
+; CHECK-NEXT:     r15:14 = vaslw(r15:14,r4)
+; CHECK-NEXT:     r8 = memw(r0+#1536)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p1 = vcmpw.eq(r19:18,r21:20)
+; CHECK-NEXT:     r19:18 = xor(r19:18,r7:6)
+; CHECK-NEXT:     r9 = memw(r0+#1792)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r13:12,#31)
+; CHECK-NEXT:     r11:10 = vmux(p1,r11:10,r19:18)
+; CHECK-NEXT:     memw(r0+#32) = r14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r17:16 = vasrw(r13:12,r3)
+; CHECK-NEXT:     r11:10 = vaslw(r11:10,r4)
+; CHECK-NEXT:     r14 = memw(r0+#2048)
+; CHECK-NEXT:     memw(r1+r5<<#2) = r14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r21:20,r17:16)
+; CHECK-NEXT:     r19:18 = xor(r21:20,r7:6)
+; CHECK-NEXT:     memw(r0+#288) = r15
+; CHECK-NEXT:     memw(r0+#256) = r15
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r17:16 = vasrw(r9:8,#31)
+; CHECK-NEXT:     r19:18 = vmux(p0,r13:12,r19:18)
+; CHECK-NEXT:     r15 = memw(r0+#2304)
+; CHECK-NEXT:     memw(r0+#544) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r9:8,r3)
+; CHECK-NEXT:     memw(r0+#800) = r11
+; CHECK-NEXT:     memw(r0+#512) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r13:12 = vasrw(r15:14,#31)
+; CHECK-NEXT:     r11:10 = vasrw(r15:14,r3)
+; CHECK-NEXT:     memw(r0+#768) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r17:16,r21:20)
+; CHECK-NEXT:     p1 = vcmpw.eq(r13:12,r11:10)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = xor(r13:12,r7:6)
+; CHECK-NEXT:     r17:16 = xor(r17:16,r7:6)
+; CHECK-NEXT:     r12 = memw(r0+#2560)
+; CHECK-NEXT:     r13 = memw(r0+#2816)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vaslw(r19:18,r4)
+; CHECK-NEXT:     r17:16 = vmux(p0,r9:8,r17:16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vmux(p1,r15:14,r21:20)
+; CHECK-NEXT:     r19:18 = vasrw(r13:12,#31)
+; CHECK-NEXT:     r14 = memw(r0+#3072)
+; CHECK-NEXT:     memw(r0+#1056) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r13:12,r3)
+; CHECK-NEXT:     r9:8 = vaslw(r9:8,r4)
+; CHECK-NEXT:     memw(r0+#1312) = r11
+; CHECK-NEXT:     memw(r0+#1024) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vaslw(r17:16,r4)
+; CHECK-NEXT:     p0 = vcmpw.eq(r19:18,r21:20)
+; CHECK-NEXT:     r15 = memw(r0+#3328)
+; CHECK-NEXT:     memw(r0+#1280) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r17:16 = xor(r19:18,r7:6)
+; CHECK-NEXT:     r21:20 = vasrw(r15:14,#31)
+; CHECK-NEXT:     memw(r0+#1568) = r10
+; CHECK-NEXT:     memw(r0+#1824) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vasrw(r15:14,r3)
+; CHECK-NEXT:     r13:12 = vmux(p0,r13:12,r17:16)
+; CHECK-NEXT:     memw(r0+#1536) = r10
+; CHECK-NEXT:     memw(r0+#1792) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r21:20,r11:10)
+; CHECK-NEXT:     r19:18 = xor(r21:20,r7:6)
+; CHECK-NEXT:     r10 = memw(r0+#3584)
+; CHECK-NEXT:     memw(r0+#2080) = r8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r19:18 = vmux(p0,r15:14,r19:18)
+; CHECK-NEXT:     r13:12 = vaslw(r13:12,r4)
+; CHECK-NEXT:     r11 = memw(r0+#3840)
+; CHECK-NEXT:     memw(r0+#2336) = r9
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r11:10,#31)
+; CHECK-NEXT:     r3:2 = vasrw(r11:10,r3)
+; CHECK-NEXT:     memw(r0+#2048) = r8
+; CHECK-NEXT:     memw(r0+#2304) = r9
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7:6 = xor(r21:20,r7:6)
+; CHECK-NEXT:     p0 = vcmpw.eq(r21:20,r3:2)
+; CHECK-NEXT:     r17:16 = memd(r29+#16)
+; CHECK-NEXT:     memw(r0+#2592) = r12
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3:2 = vmux(p0,r11:10,r7:6)
+; CHECK-NEXT:     r9:8 = vaslw(r19:18,r4)
+; CHECK-NEXT:     r19:18 = memd(r29+#8)
+; CHECK-NEXT:     memw(r0+#2848) = r13
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3:2 = vaslw(r3:2,r4)
+; CHECK-NEXT:     r29 = add(r29,#24)
+; CHECK-NEXT:     r21:20 = memd(r29+#0)
+; CHECK-NEXT:     memw(r0+#2560) = r12
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#2816) = r13
+; CHECK-NEXT:     memw(r0+#3104) = r8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#3360) = r9
+; CHECK-NEXT:     memw(r0+#3072) = r8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#3328) = r9
+; CHECK-NEXT:     memw(r0+#3616) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#3872) = r3
+; CHECK-NEXT:     memw(r0+#3584) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r0+#3840) = r3
+; CHECK-NEXT:    }
 entry:
   %0 = load i32, ptr %buf, align 4, !tbaa !0
   %shr = ashr i32 %0, %gb
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll
new file mode 100644
index 0000000000000..86808c7a8f014
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define i32 @bitcast_extract_v8f32(<8 x float> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    movgr2fr.w $fa0, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x float> %a, i32 7
+  %c = bitcast float %b to i32
+  ret i32 %c
+}
+
+define i64 @bitcast_extract_v4f64(<4 x double> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.d $fa0, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x double> %a, i32 3
+  %c = bitcast double %b to i64
+  ret i64 %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll
new file mode 100644
index 0000000000000..7b2461b11f12d
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define <8 x float> @insert_bitcast_v8f32(<8 x float> %a, i32 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i32 %b to float
+  %d = insertelement <8 x float> %a, float %c, i32 1
+  ret <8 x float> %d
+}
+
+define <4 x double> @insert_bitcast_v4f64(<4 x double> %a, i64 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i64 %b to double
+  %d = insertelement <4 x double> %a, double %c, i32 1
+  ret <4 x double> %d
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
index a5d3a0d395b3c..3fdc439e68679 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
@@ -1,6 +1,62 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
 
+define <32 x i8> @insert_extract_v32i8(<32 x i8> %a) nounwind {
+; CHECK-LABEL: insert_extract_v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.b $a0, $sp, 31
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <32 x i8> %a, i32 31
+  %c = insertelement <32 x i8> %a, i8 %b, i32 1
+  ret <32 x i8> %c
+}
+
+define <16 x i16> @insert_extract_v16i16(<16 x i16> %a) nounwind {
+; CHECK-LABEL: insert_extract_v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.h $a0, $sp, 30
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <16 x i16> %a, i32 15
+  %c = insertelement <16 x i16> %a, i16 %b, i32 1
+  ret <16 x i16> %c
+}
+
+define <8 x i32> @insert_extract_v8i32(<8 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x i32> %a, i32 7
+  %c = insertelement <8 x i32> %a, i32 %b, i32 1
+  ret <8 x i32> %c
+}
+
 define <8 x float> @insert_extract_v8f32(<8 x float> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v8f32:
 ; CHECK:       # %bb.0: # %entry
@@ -15,6 +71,18 @@ entry:
   ret <8 x float> %c
 }
 
+define <4 x i64> @insert_extract_v4i64(<4 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x i64> %a, i32 3
+  %c = insertelement <4 x i64> %a, i64 %b, i32 1
+  ret <4 x i64> %c
+}
+
 define <4 x double> @insert_extract_v4f64(<4 x double> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v4f64:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll
new file mode 100644
index 0000000000000..88c3e4367ffa7
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define <32 x i8> @insert_extract_v32i8(<32 x i8> %a) nounwind {
+; CHECK-LABEL: insert_extract_v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    vpickve2gr.b $a0, $vr0, 15
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.b $a1, $sp, 31
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 1
+; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <32 x i8> %a, i32 15
+  %b_hi = extractelement <32 x i8> %a, i32 31
+  %c = insertelement <32 x i8> %a, i8 %b_lo, i32 1
+  %d = insertelement <32 x i8> %c, i8 %b_hi, i32 17
+  ret <32 x i8> %d
+}
+
+define <16 x i16> @insert_extract_v16i16(<16 x i16> %a) nounwind {
+; CHECK-LABEL: insert_extract_v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    vpickve2gr.h $a0, $vr0, 7
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.h $a1, $sp, 30
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT:    vinsgr2vr.h $vr1, $a1, 1
+; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <16 x i16> %a, i32 7
+  %b_hi = extractelement <16 x i16> %a, i32 15
+  %c = insertelement <16 x i16> %a, i16 %b_lo, i32 1
+  %d = insertelement <16 x i16> %c, i16 %b_hi, i32 9
+  ret <16 x i16> %d
+}
+
+define <8 x i32> @insert_extract_v8i32(<8 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
+; CHECK-NEXT:    xvpickve2gr.w $a1, $xr0, 7
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a1, 5
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <8 x i32> %a, i32 3
+  %b_hi = extractelement <8 x i32> %a, i32 7
+  %c = insertelement <8 x i32> %a, i32 %b_lo, i32 1
+  %d = insertelement <8 x i32> %c, i32 %b_hi, i32 5
+  ret <8 x i32> %d
+}
+
+define <8 x float> @insert_extract_v8f32(<8 x float> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    movgr2fr.w $fa2, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    movfr2gr.s $a0, $fa2
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 5
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <8 x float> %a, i32 3
+  %b_hi = extractelement <8 x float> %a, i32 7
+  %c = insertelement <8 x float> %a, float %b_lo, i32 1
+  %d = insertelement <8 x float> %c, float %b_hi, i32 5
+  ret <8 x float> %d
+}
+
+define <4 x i64> @insert_extract_v4i64(<4 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a1, $xr0, 3
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a1, 2
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <4 x i64> %a, i32 1
+  %b_hi = extractelement <4 x i64> %a, i32 3
+  %c = insertelement <4 x i64> %a, i64 %b_lo, i32 0
+  %d = insertelement <4 x i64> %c, i64 %b_hi, i32 2
+  ret <4 x i64> %d
+}
+
+define <4 x double> @insert_extract_v4f64(<4 x double> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.d $fa2, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa2
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 2
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <4 x double> %a, i32 1
+  %b_hi = extractelement <4 x double> %a, i32 3
+  %c = insertelement <4 x double> %a, double %b_lo, i32 0
+  %d = insertelement <4 x double> %c, double %b_hi, i32 2
+  ret <4 x double> %d
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll b/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
index 5a861be95977d..c61b7841b95ac 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
@@ -383,8 +383,7 @@ define i8 @xvmsk_eq_vsel_slt_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2)
 ; CHECK-LABEL: xvmsk_eq_vsel_slt_v8i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xvseq.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvrepli.b $xr1, -1
-; CHECK-NEXT:    xvbitsel.v $xr0, $xr2, $xr1, $xr0
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr2
 ; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
 ; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
 ; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
diff --git a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
new file mode 100644
index 0000000000000..ffedd7f9e9438
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
@@ -0,0 +1,866 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch32 --mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 --mattr=+d < %s | FileCheck %s --check-prefix=LA64
+
+define { half, half } @test_sincos_f16(half %a) #0 {
+; LA32-LABEL: test_sincos_f16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    movfr2gr.s $fp, $fs1
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    or $a0, $fp, $a1
+; LA32-NEXT:    movgr2fr.w $fa1, $a0
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_f16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $fp, $fs1
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    or $a0, $fp, $a1
+; LA64-NEXT:    movgr2fr.w $fa1, $a0
+; LA64-NEXT:    fld.d $fs1, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  ret { half, half } %result
+}
+
+define half @test_sincos_f16_only_use_sin(half %a) #0 {
+; LA32-LABEL: test_sincos_f16_only_use_sin:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_f16_only_use_sin:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.0 = extractvalue { half, half } %result, 0
+  ret half %result.0
+}
+
+define half @test_sincos_f16_only_use_cos(half %a) #0 {
+; LA32-LABEL: test_sincos_f16_only_use_cos:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    lu12i.w $a1, -16
+; LA32-NEXT:    or $a0, $a0, $a1
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_f16_only_use_cos:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    lu12i.w $a1, -16
+; LA64-NEXT:    or $a0, $a0, $a1
+; LA64-NEXT:    movgr2fr.w $fa0, $a0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.1 = extractvalue { half, half } %result, 1
+  ret half %result.1
+}
+
+define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) #0 {
+; LA32-LABEL: test_sincos_v2f16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -64
+; LA32-NEXT:    st.w $ra, $sp, 60 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 56 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 52 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 48 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 44 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 32 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 24 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs2, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs3, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    movgr2fr.w $fs0, $a2
+; LA32-NEXT:    movgr2fr.w $fa0, $a1
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    fmov.s $fs2, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl __extendhfsf2
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    fmov.s $fs3, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs1
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    movfr2gr.s $s0, $fs1
+; LA32-NEXT:    movfr2gr.s $s1, $fs3
+; LA32-NEXT:    movfr2gr.s $s2, $fs2
+; LA32-NEXT:    bl __truncsfhf2
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    st.h $a0, $fp, 6
+; LA32-NEXT:    st.h $s2, $fp, 4
+; LA32-NEXT:    st.h $s1, $fp, 2
+; LA32-NEXT:    st.h $s0, $fp, 0
+; LA32-NEXT:    fld.d $fs3, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs2, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs1, $sp, 24 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 32 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $s2, $sp, 44 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 48 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 52 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 56 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 60 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 64
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_v2f16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs2, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs3, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    move $s0, $a2
+; LA64-NEXT:    move $fp, $a0
+; LA64-NEXT:    movgr2fr.w $fa0, $a1
+; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    movgr2fr.w $fa0, $s0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__extendhfsf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs2, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs3, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs2
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $s0, $fs0
+; LA64-NEXT:    movfr2gr.s $s1, $fs3
+; LA64-NEXT:    movfr2gr.s $s2, $fs1
+; LA64-NEXT:    pcaddu18i $ra, %call36(__truncsfhf2)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.s $a0, $fa0
+; LA64-NEXT:    st.h $a0, $fp, 6
+; LA64-NEXT:    st.h $s2, $fp, 4
+; LA64-NEXT:    st.h $s1, $fp, 2
+; LA64-NEXT:    st.h $s0, $fp, 0
+; LA64-NEXT:    fld.d $fs3, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs2, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs1, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
+; LA64-NEXT:    ret
+  %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_sincos_f32(float %a) #0 {
+; LA32-LABEL: test_sincos_f32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.s $fs0, $fa0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    fmov.s $fa1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs1
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_f32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fmov.s $fs0, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fs1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.s $fa1, $fa0
+; LA64-NEXT:    fmov.s $fa0, $fs1
+; LA64-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %result = call { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 {
+; LA32-LABEL: test_sincos_v2f32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -48
+; LA32-NEXT:    st.w $ra, $sp, 44 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 40 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 32 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 24 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs2, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs3, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.s $fs0, $fa1
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    fmov.s $fs2, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    fmov.s $fs3, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs1
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    fst.s $fa0, $fp, 12
+; LA32-NEXT:    fst.s $fs1, $fp, 8
+; LA32-NEXT:    fst.s $fs3, $fp, 4
+; LA32-NEXT:    fst.s $fs2, $fp, 0
+; LA32-NEXT:    fld.d $fs3, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs2, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs1, $sp, 24 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 32 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 40 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 44 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 48
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_v2f32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -64
+; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
+; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
+; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vpackev.w $vr0, $vr0, $vr1
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    # kill: def $f0 killed $f0 def $vr0
+; LA64-NEXT:    vld $vr1, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vpackev.w $vr1, $vr0, $vr1
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    ret
+  %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 {
+; LA32-LABEL: test_sincos_v3f32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -64
+; LA32-NEXT:    st.w $ra, $sp, 60 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 56 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 48 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 40 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs2, $sp, 32 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs3, $sp, 24 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs4, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs5, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.s $fs0, $fa2
+; LA32-NEXT:    fmov.s $fs1, $fa1
+; LA32-NEXT:    fmov.s $fs2, $fa0
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    fmov.s $fs3, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs1
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    fmov.s $fs4, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl sinf
+; LA32-NEXT:    fmov.s $fs5, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs2
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    fmov.s $fs2, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs1
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    fmov.s $fs1, $fa0
+; LA32-NEXT:    fmov.s $fa0, $fs0
+; LA32-NEXT:    bl cosf
+; LA32-NEXT:    fst.s $fa0, $fp, 24
+; LA32-NEXT:    fst.s $fs1, $fp, 20
+; LA32-NEXT:    fst.s $fs2, $fp, 16
+; LA32-NEXT:    fst.s $fs5, $fp, 8
+; LA32-NEXT:    fst.s $fs4, $fp, 4
+; LA32-NEXT:    fst.s $fs3, $fp, 0
+; LA32-NEXT:    fld.d $fs5, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs4, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs3, $sp, 24 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs2, $sp, 32 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs1, $sp, 40 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 48 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 56 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 60 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 64
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_v3f32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -96
+; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vreplvei.w $vr0, $vr0, 2
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fst.s $fa0, $sp, 72
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vreplvei.w $vr0, $vr0, 1
+; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fst.s $fa0, $sp, 68
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vreplvei.w $vr0, $vr0, 0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fst.s $fa0, $sp, 64
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fst.s $fa0, $sp, 56
+; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fst.s $fa0, $sp, 52
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0 killed $f0 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosf)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fst.s $fa0, $sp, 48
+; LA64-NEXT:    vld $vr0, $sp, 64
+; LA64-NEXT:    vld $vr1, $sp, 48
+; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 96
+; LA64-NEXT:    ret
+  %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
+  ret { <3 x float>, <3 x float> } %result
+}
+
+define { double, double } @test_sincos_f64(double %a) #0 {
+; LA32-LABEL: test_sincos_f64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.d $fs0, $fa0
+; LA32-NEXT:    bl sin
+; LA32-NEXT:    fmov.d $fs1, $fa0
+; LA32-NEXT:    fmov.d $fa0, $fs0
+; LA32-NEXT:    bl cos
+; LA32-NEXT:    fmov.d $fa1, $fa0
+; LA32-NEXT:    fmov.d $fa0, $fs1
+; LA32-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_f64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -32
+; LA64-NEXT:    st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    fmov.d $fs0, $fa0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.d $fs1, $fa0
+; LA64-NEXT:    fmov.d $fa0, $fs0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    fmov.d $fa1, $fa0
+; LA64-NEXT:    fmov.d $fa0, $fs1
+; LA64-NEXT:    fld.d $fs1, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 32
+; LA64-NEXT:    ret
+  %result = call { double, double } @llvm.sincos.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 {
+; LA32-LABEL: test_sincos_v2f64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -48
+; LA32-NEXT:    st.w $ra, $sp, 44 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 40 # 4-byte Folded Spill
+; LA32-NEXT:    fst.d $fs0, $sp, 32 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs1, $sp, 24 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs2, $sp, 16 # 8-byte Folded Spill
+; LA32-NEXT:    fst.d $fs3, $sp, 8 # 8-byte Folded Spill
+; LA32-NEXT:    fmov.d $fs0, $fa1
+; LA32-NEXT:    fmov.d $fs1, $fa0
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    bl sin
+; LA32-NEXT:    fmov.d $fs2, $fa0
+; LA32-NEXT:    fmov.d $fa0, $fs0
+; LA32-NEXT:    bl sin
+; LA32-NEXT:    fmov.d $fs3, $fa0
+; LA32-NEXT:    fmov.d $fa0, $fs1
+; LA32-NEXT:    bl cos
+; LA32-NEXT:    fmov.d $fs1, $fa0
+; LA32-NEXT:    fmov.d $fa0, $fs0
+; LA32-NEXT:    bl cos
+; LA32-NEXT:    fst.d $fa0, $fp, 24
+; LA32-NEXT:    fst.d $fs1, $fp, 16
+; LA32-NEXT:    fst.d $fs3, $fp, 8
+; LA32-NEXT:    fst.d $fs2, $fp, 0
+; LA32-NEXT:    fld.d $fs3, $sp, 8 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs2, $sp, 16 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs1, $sp, 24 # 8-byte Folded Reload
+; LA32-NEXT:    fld.d $fs0, $sp, 32 # 8-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 40 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 44 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 48
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_v2f64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -64
+; LA64-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
+; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    vreplvei.d $vr0, $vr0, 1
+; LA64-NEXT:    vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sin)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 1
+; LA64-NEXT:    vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
+; LA64-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; LA64-NEXT:    vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT:    vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT:    # kill: def $f0_64 killed $f0_64 killed $vr0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cos)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    movfr2gr.d $a0, $fa0
+; LA64-NEXT:    vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT:    vinsgr2vr.d $vr1, $a0, 1
+; LA64-NEXT:    vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 64
+; LA64-NEXT:    ret
+  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
+
+define { fp128, fp128 } @test_sincos_f128(fp128 %a) #0 {
+; LA32-LABEL: test_sincos_f128:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -96
+; LA32-NEXT:    st.w $ra, $sp, 92 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 88 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 84 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 80 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 76 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s3, $sp, 72 # 4-byte Folded Spill
+; LA32-NEXT:    ld.w $s0, $a1, 0
+; LA32-NEXT:    ld.w $s1, $a1, 4
+; LA32-NEXT:    ld.w $s2, $a1, 8
+; LA32-NEXT:    ld.w $s3, $a1, 12
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    st.w $s3, $sp, 52
+; LA32-NEXT:    st.w $s2, $sp, 48
+; LA32-NEXT:    st.w $s1, $sp, 44
+; LA32-NEXT:    addi.w $a0, $sp, 56
+; LA32-NEXT:    addi.w $a1, $sp, 40
+; LA32-NEXT:    st.w $s0, $sp, 40
+; LA32-NEXT:    bl sinl
+; LA32-NEXT:    st.w $s3, $sp, 20
+; LA32-NEXT:    st.w $s2, $sp, 16
+; LA32-NEXT:    st.w $s1, $sp, 12
+; LA32-NEXT:    addi.w $a0, $sp, 24
+; LA32-NEXT:    addi.w $a1, $sp, 8
+; LA32-NEXT:    st.w $s0, $sp, 8
+; LA32-NEXT:    bl cosl
+; LA32-NEXT:    ld.w $a0, $sp, 56
+; LA32-NEXT:    ld.w $a1, $sp, 60
+; LA32-NEXT:    ld.w $a2, $sp, 64
+; LA32-NEXT:    ld.w $a3, $sp, 68
+; LA32-NEXT:    ld.w $a4, $sp, 36
+; LA32-NEXT:    ld.w $a5, $sp, 32
+; LA32-NEXT:    ld.w $a6, $sp, 28
+; LA32-NEXT:    ld.w $a7, $sp, 24
+; LA32-NEXT:    st.w $a4, $fp, 28
+; LA32-NEXT:    st.w $a5, $fp, 24
+; LA32-NEXT:    st.w $a6, $fp, 20
+; LA32-NEXT:    st.w $a7, $fp, 16
+; LA32-NEXT:    st.w $a3, $fp, 12
+; LA32-NEXT:    st.w $a2, $fp, 8
+; LA32-NEXT:    st.w $a1, $fp, 4
+; LA32-NEXT:    st.w $a0, $fp, 0
+; LA32-NEXT:    ld.w $s3, $sp, 72 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s2, $sp, 76 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 80 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 84 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 88 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 92 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 96
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_f128:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -48
+; LA64-NEXT:    st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s3, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    move $fp, $a2
+; LA64-NEXT:    move $s0, $a1
+; LA64-NEXT:    move $s1, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    move $a1, $a2
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinl)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    move $s2, $a0
+; LA64-NEXT:    move $s3, $a1
+; LA64-NEXT:    move $a0, $s0
+; LA64-NEXT:    move $a1, $fp
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosl)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    st.d $a1, $s1, 24
+; LA64-NEXT:    st.d $a0, $s1, 16
+; LA64-NEXT:    st.d $s3, $s1, 8
+; LA64-NEXT:    st.d $s2, $s1, 0
+; LA64-NEXT:    ld.d $s3, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 48
+; LA64-NEXT:    ret
+  %result = call { fp128, fp128 } @llvm.sincos.f128(fp128 %a)
+  ret { fp128, fp128 } %result
+}
+
+define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
+; LA32-LABEL: test_sincos_v2f128:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -176
+; LA32-NEXT:    st.w $ra, $sp, 172 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 168 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 164 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 160 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 156 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s3, $sp, 152 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s4, $sp, 148 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s5, $sp, 144 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s6, $sp, 140 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s7, $sp, 136 # 4-byte Folded Spill
+; LA32-NEXT:    ld.w $s0, $a1, 16
+; LA32-NEXT:    ld.w $s1, $a1, 20
+; LA32-NEXT:    ld.w $s2, $a1, 24
+; LA32-NEXT:    ld.w $s3, $a1, 28
+; LA32-NEXT:    ld.w $s4, $a1, 0
+; LA32-NEXT:    ld.w $s5, $a1, 4
+; LA32-NEXT:    ld.w $s6, $a1, 8
+; LA32-NEXT:    ld.w $s7, $a1, 12
+; LA32-NEXT:    move $fp, $a0
+; LA32-NEXT:    st.w $s7, $sp, 52
+; LA32-NEXT:    st.w $s6, $sp, 48
+; LA32-NEXT:    st.w $s5, $sp, 44
+; LA32-NEXT:    addi.w $a0, $sp, 56
+; LA32-NEXT:    addi.w $a1, $sp, 40
+; LA32-NEXT:    st.w $s4, $sp, 40
+; LA32-NEXT:    bl sinl
+; LA32-NEXT:    st.w $s3, $sp, 116
+; LA32-NEXT:    st.w $s2, $sp, 112
+; LA32-NEXT:    st.w $s1, $sp, 108
+; LA32-NEXT:    addi.w $a0, $sp, 120
+; LA32-NEXT:    addi.w $a1, $sp, 104
+; LA32-NEXT:    st.w $s0, $sp, 104
+; LA32-NEXT:    bl sinl
+; LA32-NEXT:    st.w $s7, $sp, 20
+; LA32-NEXT:    st.w $s6, $sp, 16
+; LA32-NEXT:    st.w $s5, $sp, 12
+; LA32-NEXT:    addi.w $a0, $sp, 24
+; LA32-NEXT:    addi.w $a1, $sp, 8
+; LA32-NEXT:    st.w $s4, $sp, 8
+; LA32-NEXT:    bl cosl
+; LA32-NEXT:    st.w $s3, $sp, 84
+; LA32-NEXT:    st.w $s2, $sp, 80
+; LA32-NEXT:    st.w $s1, $sp, 76
+; LA32-NEXT:    addi.w $a0, $sp, 88
+; LA32-NEXT:    addi.w $a1, $sp, 72
+; LA32-NEXT:    st.w $s0, $sp, 72
+; LA32-NEXT:    bl cosl
+; LA32-NEXT:    ld.w $a0, $sp, 56
+; LA32-NEXT:    ld.w $a1, $sp, 60
+; LA32-NEXT:    ld.w $a2, $sp, 64
+; LA32-NEXT:    ld.w $a3, $sp, 68
+; LA32-NEXT:    ld.w $a4, $sp, 120
+; LA32-NEXT:    ld.w $a5, $sp, 124
+; LA32-NEXT:    ld.w $a6, $sp, 128
+; LA32-NEXT:    ld.w $a7, $sp, 132
+; LA32-NEXT:    ld.w $t0, $sp, 24
+; LA32-NEXT:    ld.w $t1, $sp, 28
+; LA32-NEXT:    ld.w $t2, $sp, 32
+; LA32-NEXT:    ld.w $t3, $sp, 36
+; LA32-NEXT:    ld.w $t4, $sp, 100
+; LA32-NEXT:    ld.w $t5, $sp, 96
+; LA32-NEXT:    ld.w $t6, $sp, 92
+; LA32-NEXT:    ld.w $t7, $sp, 88
+; LA32-NEXT:    st.w $t4, $fp, 60
+; LA32-NEXT:    st.w $t5, $fp, 56
+; LA32-NEXT:    st.w $t6, $fp, 52
+; LA32-NEXT:    st.w $t7, $fp, 48
+; LA32-NEXT:    st.w $t3, $fp, 44
+; LA32-NEXT:    st.w $t2, $fp, 40
+; LA32-NEXT:    st.w $t1, $fp, 36
+; LA32-NEXT:    st.w $t0, $fp, 32
+; LA32-NEXT:    st.w $a7, $fp, 28
+; LA32-NEXT:    st.w $a6, $fp, 24
+; LA32-NEXT:    st.w $a5, $fp, 20
+; LA32-NEXT:    st.w $a4, $fp, 16
+; LA32-NEXT:    st.w $a3, $fp, 12
+; LA32-NEXT:    st.w $a2, $fp, 8
+; LA32-NEXT:    st.w $a1, $fp, 4
+; LA32-NEXT:    st.w $a0, $fp, 0
+; LA32-NEXT:    ld.w $s7, $sp, 136 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s6, $sp, 140 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s5, $sp, 144 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s4, $sp, 148 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s3, $sp, 152 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s2, $sp, 156 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 160 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 164 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 168 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 172 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 176
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: test_sincos_v2f128:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s6, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s7, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT:    ld.d $fp, $a1, 16
+; LA64-NEXT:    ld.d $s0, $a1, 24
+; LA64-NEXT:    ld.d $s1, $a1, 0
+; LA64-NEXT:    ld.d $s2, $a1, 8
+; LA64-NEXT:    move $s3, $a0
+; LA64-NEXT:    move $a0, $s1
+; LA64-NEXT:    move $a1, $s2
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinl)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    move $s4, $a0
+; LA64-NEXT:    move $s5, $a1
+; LA64-NEXT:    move $a0, $fp
+; LA64-NEXT:    move $a1, $s0
+; LA64-NEXT:    pcaddu18i $ra, %call36(sinl)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    move $s6, $a0
+; LA64-NEXT:    move $s7, $a1
+; LA64-NEXT:    move $a0, $s1
+; LA64-NEXT:    move $a1, $s2
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosl)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    move $s1, $a0
+; LA64-NEXT:    move $s2, $a1
+; LA64-NEXT:    move $a0, $fp
+; LA64-NEXT:    move $a1, $s0
+; LA64-NEXT:    pcaddu18i $ra, %call36(cosl)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    st.d $a1, $s3, 56
+; LA64-NEXT:    st.d $a0, $s3, 48
+; LA64-NEXT:    st.d $s2, $s3, 40
+; LA64-NEXT:    st.d $s1, $s3, 32
+; LA64-NEXT:    st.d $s7, $s3, 24
+; LA64-NEXT:    st.d $s6, $s3, 16
+; LA64-NEXT:    st.d $s5, $s3, 8
+; LA64-NEXT:    st.d $s4, $s3, 0
+; LA64-NEXT:    ld.d $s7, $sp, 0 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s6, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
+; LA64-NEXT:    ret
+  %result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a)
+  ret { <2 x fp128>, <2 x fp128> } %result
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll
new file mode 100644
index 0000000000000..df4896d7ec936
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s
+
+define i32 @bitcast_extract_v4f32(<4 x float> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vreplvei.w $vr0, $vr0, 3
+; CHECK-NEXT:    movfr2gr.s $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x float> %a, i32 3
+  %c = bitcast float %b to i32
+  ret i32 %c
+}
+
+define i64 @bitcast_extract_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vreplvei.d $vr0, $vr0, 1
+; CHECK-NEXT:    movfr2gr.d $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <2 x double> %a, i32 1
+  %c = bitcast double %b to i64
+  ret i64 %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll
new file mode 100644
index 0000000000000..a20d17efdfb11
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s
+
+define <4 x float> @insert_bitcast_v4f32(<4 x float> %a, i32 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i32 %b to float
+  %d = insertelement <4 x float> %a, float %c, i32 1
+  ret <4 x float> %d
+}
+
+define <2 x double> @insert_bitcast_v2f64(<2 x double> %a, i64 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i64 %b to double
+  %d = insertelement <2 x double> %a, double %c, i32 1
+  ret <2 x double> %d
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll
index dcf23f0240712..c7dd1454c7e33 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll
@@ -1,6 +1,42 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s
 
+define <16 x i8> @insert_extract_v16i8(<16 x i8> %a) nounwind {
+; CHECK-LABEL: insert_extract_v16i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.b $a0, $vr0, 15
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <16 x i8> %a, i32 15
+  %c = insertelement <16 x i8> %a, i8 %b, i32 1
+  ret <16 x i8> %c
+}
+
+define <8 x i16> @insert_extract_v8i16(<8 x i16> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.h $a0, $vr0, 7
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x i16> %a, i32 7
+  %c = insertelement <8 x i16> %a, i16 %b, i32 1
+  ret <8 x i16> %c
+}
+
+define <4 x i32> @insert_extract_v4i32(<4 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 3
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x i32> %a, i32 3
+  %c = insertelement <4 x i32> %a, i32 %b, i32 1
+  ret <4 x i32> %c
+}
+
 define <4 x float> @insert_extract_v4f32(<4 x float> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v4f32:
 ; CHECK:       # %bb.0: # %entry
@@ -14,6 +50,18 @@ entry:
   ret <4 x float> %c
 }
 
+define <2 x i64> @insert_extract_v2i64(<2 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <2 x i64> %a, i32 1
+  %c = insertelement <2 x i64> %a, i64 %b, i32 0
+  ret <2 x i64> %c
+}
+
 define <2 x double> @insert_extract_v2f64(<2 x double> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v2f64:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
index 1c10e6c3087ad..4d2ddeb2889bb 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
@@ -6,8 +6,8 @@ define void @select_v16i8_imm(ptr %res, ptr %a0) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vld $vr0, $a1, 0
 ; CHECK-NEXT:    vrepli.h $vr1, -256
-; CHECK-NEXT:    vbitseli.b $vr1, $vr0, 255
-; CHECK-NEXT:    vst $vr1, $a0, 0
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %v0 = load <16 x i8>, ptr %a0
   %sel = select <16 x i1> <i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %v0
diff --git a/llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll b/llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll
new file mode 100644
index 0000000000000..b0fbdc6717612
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll
@@ -0,0 +1,16 @@
+; RUN: not llc -mtriple=msp430 -filetype=null %s 2>&1 | FileCheck %s
+
+; FIXME: This should not fail but isn't implemented
+; CHECK: error: powi exponent does not match sizeof(int)
+define float @soften_powi_error(float %x, i32 %n) {
+  %powi = call float @llvm.powi.f32.i32(float %x, i32 %n)
+  ret float %powi
+}
+
+; CHECK: error: powi exponent does not match sizeof(int)
+define float @soften_powi_error_strictfp(float %x, i32 %n) strictfp {
+ %powi = call float @llvm.experimental.constrained.powi.f32.i32(float %x, i32 %n, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %powi
+}
+
+
diff --git a/llvm/test/CodeGen/Mips/cconv/vector.ll b/llvm/test/CodeGen/Mips/cconv/vector.ll
index 383e5ef19cebf..c64c60ddef73d 100644
--- a/llvm/test/CodeGen/Mips/cconv/vector.ll
+++ b/llvm/test/CodeGen/Mips/cconv/vector.ll
@@ -72,6 +72,23 @@ define <2 x i8> @i8_2(<2 x i8> %a, <2 x i8> %b) {
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
 ;
+; MIPS64R5EB-LABEL: i8_2:
+; MIPS64R5EB:       # %bb.0:
+; MIPS64R5EB-NEXT:    daddiu $sp, $sp, -48
+; MIPS64R5EB-NEXT:    .cfi_def_cfa_offset 48
+; MIPS64R5EB-NEXT:    sh $5, 16($sp)
+; MIPS64R5EB-NEXT:    sh $4, 0($sp)
+; MIPS64R5EB-NEXT:    ld.b $w0, 16($sp)
+; MIPS64R5EB-NEXT:    ld.b $w1, 0($sp)
+; MIPS64R5EB-NEXT:    addv.b $w0, $w1, $w0
+; MIPS64R5EB-NEXT:    shf.b $w0, $w0, 177
+; MIPS64R5EB-NEXT:    copy_s.h $1, $w0[0]
+; MIPS64R5EB-NEXT:    sh $1, 44($sp)
+; MIPS64R5EB-NEXT:    lh $2, 44($sp)
+; MIPS64R5EB-NEXT:    daddiu $sp, $sp, 48
+; MIPS64R5EB-NEXT:    jr $ra
+; MIPS64R5EB-NEXT:    nop
+;
 ; MIPS32R5EL-LABEL: i8_2:
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
@@ -97,22 +114,6 @@ define <2 x i8> @i8_2(<2 x i8> %a, <2 x i8> %b) {
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
 ;
-; MIPS64R5EB-LABEL: i8_2:
-; MIPS64R5EB:       # %bb.0:
-; MIPS64R5EB-NEXT:    daddiu $sp, $sp, -48
-; MIPS64R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS64R5EB-NEXT:    sh $5, 16($sp)
-; MIPS64R5EB-NEXT:    sh $4, 0($sp)
-; MIPS64R5EB-NEXT:    ld.b $w0, 16($sp)
-; MIPS64R5EB-NEXT:    ld.b $w1, 0($sp)
-; MIPS64R5EB-NEXT:    addv.b $w0, $w1, $w0
-; MIPS64R5EB-NEXT:    shf.b $w0, $w0, 177
-; MIPS64R5EB-NEXT:    copy_s.h $1, $w0[0]
-; MIPS64R5EB-NEXT:    sh $1, 44($sp)
-; MIPS64R5EB-NEXT:    lh $2, 44($sp)
-; MIPS64R5EB-NEXT:    daddiu $sp, $sp, 48
-; MIPS64R5EB-NEXT:    jr $ra
-;
 ; MIPS64R5EL-LABEL: i8_2:
 ; MIPS64R5EL:       # %bb.0:
 ; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -48
@@ -287,37 +288,6 @@ define <2 x i8> @i8x2_7(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x
 ; MIPS64R5EB-NEXT:    jr $ra
 ; MIPS64R5EB-NEXT:    nop
 ;
-; MIPS64R5EL-LABEL: i8x2_7:
-; MIPS64R5EL:       # %bb.0: # %entry
-; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -128
-; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 128
-; MIPS64R5EL-NEXT:    sh $5, 16($sp)
-; MIPS64R5EL-NEXT:    sh $4, 0($sp)
-; MIPS64R5EL-NEXT:    ld.b $w0, 16($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 0($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w1, $w0
-; MIPS64R5EL-NEXT:    sh $6, 32($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 32($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
-; MIPS64R5EL-NEXT:    sh $7, 48($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 48($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
-; MIPS64R5EL-NEXT:    sh $8, 64($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 64($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
-; MIPS64R5EL-NEXT:    sh $9, 80($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 80($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
-; MIPS64R5EL-NEXT:    sh $10, 96($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 96($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
-; MIPS64R5EL-NEXT:    copy_s.h $1, $w0[0]
-; MIPS64R5EL-NEXT:    sh $1, 124($sp)
-; MIPS64R5EL-NEXT:    lh $2, 124($sp)
-; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 128
-; MIPS64R5EL-NEXT:    jr $ra
-; MIPS64R5EL-NEXT:    nop
-;
 ; MIPS32EL-LABEL: i8x2_7:
 ; MIPS32EL:       # %bb.0: # %entry
 ; MIPS32EL-NEXT:    addu $1, $4, $5
@@ -394,6 +364,37 @@ define <2 x i8> @i8x2_7(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d, <2 x
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 128
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
+;
+; MIPS64R5EL-LABEL: i8x2_7:
+; MIPS64R5EL:       # %bb.0: # %entry
+; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -128
+; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 128
+; MIPS64R5EL-NEXT:    sh $5, 16($sp)
+; MIPS64R5EL-NEXT:    sh $4, 0($sp)
+; MIPS64R5EL-NEXT:    ld.b $w0, 16($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 0($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w1, $w0
+; MIPS64R5EL-NEXT:    sh $6, 32($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 32($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    sh $7, 48($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 48($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    sh $8, 64($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 64($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    sh $9, 80($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 80($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    sh $10, 96($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 96($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w0, $w1
+; MIPS64R5EL-NEXT:    copy_s.h $1, $w0[0]
+; MIPS64R5EL-NEXT:    sh $1, 124($sp)
+; MIPS64R5EL-NEXT:    lh $2, 124($sp)
+; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 128
+; MIPS64R5EL-NEXT:    jr $ra
+; MIPS64R5EL-NEXT:    nop
 entry:
   %0 = add <2 x i8> %a, %b
   %1 = add <2 x i8> %0, %c
@@ -498,6 +499,31 @@ define <4 x i8> @i8_4(<4 x i8> %a, <4 x i8> %b) {
 ; MIPS64R5EB-NEXT:    jr $ra
 ; MIPS64R5EB-NEXT:    nop
 ;
+; MIPS32R5EL-LABEL: i8_4:
+; MIPS32R5EL:       # %bb.0:
+; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
+; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
+; MIPS32R5EL-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
+; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
+; MIPS32R5EL-NEXT:    move $fp, $sp
+; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
+; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
+; MIPS32R5EL-NEXT:    and $sp, $sp, $1
+; MIPS32R5EL-NEXT:    sw $5, 16($sp)
+; MIPS32R5EL-NEXT:    sw $4, 0($sp)
+; MIPS32R5EL-NEXT:    ld.b $w0, 16($sp)
+; MIPS32R5EL-NEXT:    ld.b $w1, 0($sp)
+; MIPS32R5EL-NEXT:    addv.b $w0, $w1, $w0
+; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
+; MIPS32R5EL-NEXT:    move $sp, $fp
+; MIPS32R5EL-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
+; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
+; MIPS32R5EL-NEXT:    jr $ra
+; MIPS32R5EL-NEXT:    nop
+;
 ; MIPS64R5EL-LABEL: i8_4:
 ; MIPS64R5EL:       # %bb.0:
 ; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
@@ -665,20 +691,6 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS64R5EB-NEXT:    jr $ra
 ; MIPS64R5EB-NEXT:    nop
 ;
-; MIPS64R5EL-LABEL: i8_8:
-; MIPS64R5EL:       # %bb.0:
-; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
-; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 32
-; MIPS64R5EL-NEXT:    sd $5, 16($sp)
-; MIPS64R5EL-NEXT:    sd $4, 0($sp)
-; MIPS64R5EL-NEXT:    ld.b $w0, 16($sp)
-; MIPS64R5EL-NEXT:    ld.b $w1, 0($sp)
-; MIPS64R5EL-NEXT:    addv.b $w0, $w1, $w0
-; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
-; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 32
-; MIPS64R5EL-NEXT:    jr $ra
-; MIPS64R5EL-NEXT:    nop
-;
 ; MIPS32R5EL-LABEL: i8_8:
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
@@ -706,6 +718,20 @@ define <8 x i8> @i8_8(<8 x i8> %a, <8 x i8> %b) {
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
+;
+; MIPS64R5EL-LABEL: i8_8:
+; MIPS64R5EL:       # %bb.0:
+; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
+; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 32
+; MIPS64R5EL-NEXT:    sd $5, 16($sp)
+; MIPS64R5EL-NEXT:    sd $4, 0($sp)
+; MIPS64R5EL-NEXT:    ld.b $w0, 16($sp)
+; MIPS64R5EL-NEXT:    ld.b $w1, 0($sp)
+; MIPS64R5EL-NEXT:    addv.b $w0, $w1, $w0
+; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
+; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 32
+; MIPS64R5EL-NEXT:    jr $ra
+; MIPS64R5EL-NEXT:    nop
   %1 = add <8 x i8> %a, %b
   ret <8 x i8> %1
 }
@@ -1042,6 +1068,23 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ; MIPS32R5EB-NEXT:    jr $ra
 ; MIPS32R5EB-NEXT:    nop
 ;
+; MIPS64R5EB-LABEL: i16_2:
+; MIPS64R5EB:       # %bb.0:
+; MIPS64R5EB-NEXT:    daddiu $sp, $sp, -32
+; MIPS64R5EB-NEXT:    .cfi_def_cfa_offset 32
+; MIPS64R5EB-NEXT:    sll $1, $5, 0
+; MIPS64R5EB-NEXT:    sw $1, 16($sp)
+; MIPS64R5EB-NEXT:    sll $1, $4, 0
+; MIPS64R5EB-NEXT:    sw $1, 0($sp)
+; MIPS64R5EB-NEXT:    ld.h $w0, 16($sp)
+; MIPS64R5EB-NEXT:    ld.h $w1, 0($sp)
+; MIPS64R5EB-NEXT:    addv.h $w0, $w1, $w0
+; MIPS64R5EB-NEXT:    shf.h $w0, $w0, 177
+; MIPS64R5EB-NEXT:    copy_s.w $2, $w0[0]
+; MIPS64R5EB-NEXT:    daddiu $sp, $sp, 32
+; MIPS64R5EB-NEXT:    jr $ra
+; MIPS64R5EB-NEXT:    nop
+;
 ; MIPS32R5EL-LABEL: i16_2:
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
@@ -1067,23 +1110,6 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
 ;
-; MIPS64R5EB-LABEL: i16_2:
-; MIPS64R5EB:       # %bb.0:
-; MIPS64R5EB-NEXT:    daddiu $sp, $sp, -32
-; MIPS64R5EB-NEXT:    .cfi_def_cfa_offset 32
-; MIPS64R5EB-NEXT:    sll $1, $5, 0
-; MIPS64R5EB-NEXT:    sw $1, 16($sp)
-; MIPS64R5EB-NEXT:    sll $1, $4, 0
-; MIPS64R5EB-NEXT:    sw $1, 0($sp)
-; MIPS64R5EB-NEXT:    ld.h $w0, 16($sp)
-; MIPS64R5EB-NEXT:    ld.h $w1, 0($sp)
-; MIPS64R5EB-NEXT:    addv.h $w0, $w1, $w0
-; MIPS64R5EB-NEXT:    shf.h $w0, $w0, 177
-; MIPS64R5EB-NEXT:    copy_s.w $2, $w0[0]
-; MIPS64R5EB-NEXT:    daddiu $sp, $sp, 32
-; MIPS64R5EB-NEXT:    jr $ra
-; MIPS64R5EB-NEXT:    nop
-;
 ; MIPS64R5EL-LABEL: i16_2:
 ; MIPS64R5EL:       # %bb.0:
 ; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
@@ -1098,6 +1124,7 @@ define <2 x i16> @i16_2(<2 x i16> %a, <2 x i16> %b) {
 ; MIPS64R5EL-NEXT:    copy_s.w $2, $w0[0]
 ; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 32
 ; MIPS64R5EL-NEXT:    jr $ra
+; MIPS64R5EL-NEXT:    nop
   %1 = add <2 x i16> %a, %b
   ret <2 x i16> %1
 }
@@ -1180,6 +1207,7 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS32R5EB-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
 ; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EB-NEXT:    jr $ra
+; MIPS32R5EB-NEXT:    nop
 ;
 ; MIPS64R5EB-LABEL: i16_4:
 ; MIPS64R5EB:       # %bb.0:
@@ -1196,20 +1224,6 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS64R5EB-NEXT:    jr $ra
 ; MIPS64R5EB-NEXT:    nop
 ;
-; MIPS64R5EL-LABEL: i16_4:
-; MIPS64R5EL:       # %bb.0:
-; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
-; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 32
-; MIPS64R5EL-NEXT:    sd $5, 16($sp)
-; MIPS64R5EL-NEXT:    sd $4, 0($sp)
-; MIPS64R5EL-NEXT:    ld.h $w0, 16($sp)
-; MIPS64R5EL-NEXT:    ld.h $w1, 0($sp)
-; MIPS64R5EL-NEXT:    addv.h $w0, $w1, $w0
-; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
-; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 32
-; MIPS64R5EL-NEXT:    jr $ra
-; MIPS64R5EL-NEXT:    nop
-;
 ; MIPS32R5EL-LABEL: i16_4:
 ; MIPS32R5EL:       # %bb.0:
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
@@ -1237,6 +1251,20 @@ define <4 x i16> @i16_4(<4 x i16> %a, <4 x i16> %b) {
 ; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
 ; MIPS32R5EL-NEXT:    jr $ra
 ; MIPS32R5EL-NEXT:    nop
+;
+; MIPS64R5EL-LABEL: i16_4:
+; MIPS64R5EL:       # %bb.0:
+; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
+; MIPS64R5EL-NEXT:    .cfi_def_cfa_offset 32
+; MIPS64R5EL-NEXT:    sd $5, 16($sp)
+; MIPS64R5EL-NEXT:    sd $4, 0($sp)
+; MIPS64R5EL-NEXT:    ld.h $w0, 16($sp)
+; MIPS64R5EL-NEXT:    ld.h $w1, 0($sp)
+; MIPS64R5EL-NEXT:    addv.h $w0, $w1, $w0
+; MIPS64R5EL-NEXT:    copy_s.d $2, $w0[0]
+; MIPS64R5EL-NEXT:    daddiu $sp, $sp, 32
+; MIPS64R5EL-NEXT:    jr $ra
+; MIPS64R5EL-NEXT:    nop
   %1 = add <4 x i16> %a, %b
   ret <4 x i16> %1
 }
@@ -1438,33 +1466,33 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS64-NEXT:    jr $ra
 ; MIPS64-NEXT:    nop
 ;
-; MIPS32R5EB-LABEL: i32_2:
-; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    addiu $sp, $sp, -48
-; MIPS32R5EB-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EB-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
-; MIPS32R5EB-NEXT:    .cfi_offset 31, -4
-; MIPS32R5EB-NEXT:    .cfi_offset 30, -8
-; MIPS32R5EB-NEXT:    move $fp, $sp
-; MIPS32R5EB-NEXT:    .cfi_def_cfa_register 30
-; MIPS32R5EB-NEXT:    addiu $1, $zero, -16
-; MIPS32R5EB-NEXT:    and $sp, $sp, $1
-; MIPS32R5EB-NEXT:    sw $7, 20($sp)
-; MIPS32R5EB-NEXT:    sw $6, 16($sp)
-; MIPS32R5EB-NEXT:    sw $5, 4($sp)
-; MIPS32R5EB-NEXT:    sw $4, 0($sp)
-; MIPS32R5EB-NEXT:    ld.w $w0, 16($sp)
-; MIPS32R5EB-NEXT:    ld.w $w1, 0($sp)
-; MIPS32R5EB-NEXT:    addv.w $w0, $w1, $w0
-; MIPS32R5EB-NEXT:    copy_s.w $2, $w0[0]
-; MIPS32R5EB-NEXT:    copy_s.w $3, $w0[1]
-; MIPS32R5EB-NEXT:    move $sp, $fp
-; MIPS32R5EB-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
-; MIPS32R5EB-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
-; MIPS32R5EB-NEXT:    addiu $sp, $sp, 48
-; MIPS32R5EB-NEXT:    jr $ra
-; MIPS32R5EB-NEXT:    nop
+; MIPS32R5-LABEL: i32_2:
+; MIPS32R5:       # %bb.0:
+; MIPS32R5-NEXT:    addiu $sp, $sp, -48
+; MIPS32R5-NEXT:    .cfi_def_cfa_offset 48
+; MIPS32R5-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
+; MIPS32R5-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
+; MIPS32R5-NEXT:    .cfi_offset 31, -4
+; MIPS32R5-NEXT:    .cfi_offset 30, -8
+; MIPS32R5-NEXT:    move $fp, $sp
+; MIPS32R5-NEXT:    .cfi_def_cfa_register 30
+; MIPS32R5-NEXT:    addiu $1, $zero, -16
+; MIPS32R5-NEXT:    and $sp, $sp, $1
+; MIPS32R5-NEXT:    sw $7, 20($sp)
+; MIPS32R5-NEXT:    sw $6, 16($sp)
+; MIPS32R5-NEXT:    sw $5, 4($sp)
+; MIPS32R5-NEXT:    sw $4, 0($sp)
+; MIPS32R5-NEXT:    ld.w $w0, 16($sp)
+; MIPS32R5-NEXT:    ld.w $w1, 0($sp)
+; MIPS32R5-NEXT:    addv.w $w0, $w1, $w0
+; MIPS32R5-NEXT:    copy_s.w $2, $w0[0]
+; MIPS32R5-NEXT:    copy_s.w $3, $w0[1]
+; MIPS32R5-NEXT:    move $sp, $fp
+; MIPS32R5-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
+; MIPS32R5-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
+; MIPS32R5-NEXT:    addiu $sp, $sp, 48
+; MIPS32R5-NEXT:    jr $ra
+; MIPS32R5-NEXT:    nop
 ;
 ; MIPS64R5EB-LABEL: i32_2:
 ; MIPS64R5EB:       # %bb.0:
@@ -1481,34 +1509,6 @@ define <2 x i32> @i32_2(<2 x i32> %a, <2 x i32> %b) {
 ; MIPS64R5EB-NEXT:    jr $ra
 ; MIPS64R5EB-NEXT:    nop
 ;
-; MIPS32R5EL-LABEL: i32_2:
-; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    addiu $sp, $sp, -48
-; MIPS32R5EL-NEXT:    .cfi_def_cfa_offset 48
-; MIPS32R5EL-NEXT:    sw $ra, 44($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    sw $fp, 40($sp) # 4-byte Folded Spill
-; MIPS32R5EL-NEXT:    .cfi_offset 31, -4
-; MIPS32R5EL-NEXT:    .cfi_offset 30, -8
-; MIPS32R5EL-NEXT:    move $fp, $sp
-; MIPS32R5EL-NEXT:    .cfi_def_cfa_register 30
-; MIPS32R5EL-NEXT:    addiu $1, $zero, -16
-; MIPS32R5EL-NEXT:    and $sp, $sp, $1
-; MIPS32R5EL-NEXT:    sw $7, 20($sp)
-; MIPS32R5EL-NEXT:    sw $6, 16($sp)
-; MIPS32R5EL-NEXT:    sw $5, 4($sp)
-; MIPS32R5EL-NEXT:    sw $4, 0($sp)
-; MIPS32R5EL-NEXT:    ld.w $w0, 16($sp)
-; MIPS32R5EL-NEXT:    ld.w $w1, 0($sp)
-; MIPS32R5EL-NEXT:    addv.w $w0, $w1, $w0
-; MIPS32R5EL-NEXT:    copy_s.w $2, $w0[0]
-; MIPS32R5EL-NEXT:    copy_s.w $3, $w0[1]
-; MIPS32R5EL-NEXT:    move $sp, $fp
-; MIPS32R5EL-NEXT:    lw $fp, 40($sp) # 4-byte Folded Reload
-; MIPS32R5EL-NEXT:    lw $ra, 44($sp) # 4-byte Folded Reload
-; MIPS32R5EL-NEXT:    addiu $sp, $sp, 48
-; MIPS32R5EL-NEXT:    jr $ra
-; MIPS32R5EL-NEXT:    nop
-;
 ; MIPS64R5EL-LABEL: i32_2:
 ; MIPS64R5EL:       # %bb.0:
 ; MIPS64R5EL-NEXT:    daddiu $sp, $sp, -32
@@ -2265,13 +2265,14 @@ define <8 x i8> @ret_8_i8() {
 ; MIPS64-NEXT:    jr $ra
 ; MIPS64-NEXT:    nop
 ;
-; MIPS32R5EB-LABEL: ret_8_i8:
-; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    lui $1, %hi(gv8i8)
-; MIPS32R5EB-NEXT:    lw $2, %lo(gv8i8)($1)
-; MIPS32R5EB-NEXT:    addiu $1, $1, %lo(gv8i8)
-; MIPS32R5EB-NEXT:    lw $3, 4($1)
-; MIPS32R5EB-NEXT:    jr $ra
+; MIPS32R5-LABEL: ret_8_i8:
+; MIPS32R5:       # %bb.0:
+; MIPS32R5-NEXT:    lui $1, %hi(gv8i8)
+; MIPS32R5-NEXT:    lw $2, %lo(gv8i8)($1)
+; MIPS32R5-NEXT:    addiu $1, $1, %lo(gv8i8)
+; MIPS32R5-NEXT:    lw $3, 4($1)
+; MIPS32R5-NEXT:    jr $ra
+; MIPS32R5-NEXT:    nop
 ;
 ; MIPS64R5-LABEL: ret_8_i8:
 ; MIPS64R5:       # %bb.0:
@@ -2282,15 +2283,6 @@ define <8 x i8> @ret_8_i8() {
 ; MIPS64R5-NEXT:    ld $2, 0($1)
 ; MIPS64R5-NEXT:    jr $ra
 ; MIPS64R5-NEXT:    nop
-;
-; MIPS32R5EL-LABEL: ret_8_i8:
-; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    lui $1, %hi(gv8i8)
-; MIPS32R5EL-NEXT:    lw $2, %lo(gv8i8)($1)
-; MIPS32R5EL-NEXT:    addiu $1, $1, %lo(gv8i8)
-; MIPS32R5EL-NEXT:    lw $3, 4($1)
-; MIPS32R5EL-NEXT:    jr $ra
-; MIPS32R5EL-NEXT:    nop
   %1 = load <8 x i8>, ptr @gv8i8
   ret <8 x i8> %1
 }
@@ -2403,14 +2395,14 @@ define <4 x i16> @ret_4_i16() {
 ; MIPS64-NEXT:    jr $ra
 ; MIPS64-NEXT:    nop
 ;
-; MIPS32R5EB-LABEL: ret_4_i16:
-; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    lui $1, %hi(gv4i16)
-; MIPS32R5EB-NEXT:    lw $2, %lo(gv4i16)($1)
-; MIPS32R5EB-NEXT:    addiu $1, $1, %lo(gv4i16)
-; MIPS32R5EB-NEXT:    lw $3, 4($1)
-; MIPS32R5EB-NEXT:    jr $ra
-; MIPS32R5EB-NEXT:    nop
+; MIPS32R5-LABEL: ret_4_i16:
+; MIPS32R5:       # %bb.0:
+; MIPS32R5-NEXT:    lui $1, %hi(gv4i16)
+; MIPS32R5-NEXT:    lw $2, %lo(gv4i16)($1)
+; MIPS32R5-NEXT:    addiu $1, $1, %lo(gv4i16)
+; MIPS32R5-NEXT:    lw $3, 4($1)
+; MIPS32R5-NEXT:    jr $ra
+; MIPS32R5-NEXT:    nop
 ;
 ; MIPS64R5-LABEL: ret_4_i16:
 ; MIPS64R5:       # %bb.0:
@@ -2421,15 +2413,6 @@ define <4 x i16> @ret_4_i16() {
 ; MIPS64R5-NEXT:    ld $2, 0($1)
 ; MIPS64R5-NEXT:    jr $ra
 ; MIPS64R5-NEXT:    nop
-;
-; MIPS32R5EL-LABEL: ret_4_i16:
-; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    lui $1, %hi(gv4i16)
-; MIPS32R5EL-NEXT:    lw $2, %lo(gv4i16)($1)
-; MIPS32R5EL-NEXT:    addiu $1, $1, %lo(gv4i16)
-; MIPS32R5EL-NEXT:    lw $3, 4($1)
-; MIPS32R5EL-NEXT:    jr $ra
-; MIPS32R5EL-NEXT:    nop
   %1 = load <4 x i16>, ptr @gv4i16
   ret <4 x i16> %1
 }
@@ -2504,14 +2487,14 @@ define <2 x i32> @ret_2_i32() {
 ; MIPS64-NEXT:    jr $ra
 ; MIPS64-NEXT:    nop
 ;
-; MIPS32R5EB-LABEL: ret_2_i32:
-; MIPS32R5EB:       # %bb.0:
-; MIPS32R5EB-NEXT:    lui $1, %hi(gv2i32)
-; MIPS32R5EB-NEXT:    lw $2, %lo(gv2i32)($1)
-; MIPS32R5EB-NEXT:    addiu $1, $1, %lo(gv2i32)
-; MIPS32R5EB-NEXT:    lw $3, 4($1)
-; MIPS32R5EB-NEXT:    jr $ra
-; MIPS32R5EB-NEXT:    nop
+; MIPS32R5-LABEL: ret_2_i32:
+; MIPS32R5:       # %bb.0:
+; MIPS32R5-NEXT:    lui $1, %hi(gv2i32)
+; MIPS32R5-NEXT:    lw $2, %lo(gv2i32)($1)
+; MIPS32R5-NEXT:    addiu $1, $1, %lo(gv2i32)
+; MIPS32R5-NEXT:    lw $3, 4($1)
+; MIPS32R5-NEXT:    jr $ra
+; MIPS32R5-NEXT:    nop
 ;
 ; MIPS64R5-LABEL: ret_2_i32:
 ; MIPS64R5:       # %bb.0:
@@ -2522,15 +2505,6 @@ define <2 x i32> @ret_2_i32() {
 ; MIPS64R5-NEXT:    ld $2, 0($1)
 ; MIPS64R5-NEXT:    jr $ra
 ; MIPS64R5-NEXT:    nop
-;
-; MIPS32R5EL-LABEL: ret_2_i32:
-; MIPS32R5EL:       # %bb.0:
-; MIPS32R5EL-NEXT:    lui $1, %hi(gv2i32)
-; MIPS32R5EL-NEXT:    lw $2, %lo(gv2i32)($1)
-; MIPS32R5EL-NEXT:    addiu $1, $1, %lo(gv2i32)
-; MIPS32R5EL-NEXT:    lw $3, 4($1)
-; MIPS32R5EL-NEXT:    jr $ra
-; MIPS32R5EL-NEXT:    nop
   %1 = load <2 x i32>, ptr @gv2i32
   ret <2 x i32> %1
 }
diff --git a/llvm/test/CodeGen/Mips/ins.ll b/llvm/test/CodeGen/Mips/ins.ll
index 06e493bcd759c..0492a5d1a9e75 100644
--- a/llvm/test/CodeGen/Mips/ins.ll
+++ b/llvm/test/CodeGen/Mips/ins.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -O3 -mcpu=mips32r2 -mtriple=mipsel-linux-gnu < %s -o - \
 ; RUN:   | FileCheck %s --check-prefixes=MIPS32R2
 ; RUN: llc -O3 -mcpu=mips64r2 -mtriple=mips64el  < %s \
@@ -9,7 +10,14 @@ define i32 @or_and_shl(i32 %a, i32 %b) {
 ; MIPS32R2-NEXT:    ins $4, $5, 31, 1
 ; MIPS32R2-NEXT:    jr $ra
 ; MIPS32R2-NEXT:    move $2, $4
-
+;
+; MIPS64R2-LABEL: or_and_shl:
+; MIPS64R2:       # %bb.0: # %entry
+; MIPS64R2-NEXT:    sll $2, $4, 0
+; MIPS64R2-NEXT:    sll $1, $5, 0
+; MIPS64R2-NEXT:    ins $2, $1, 31, 1
+; MIPS64R2-NEXT:    jr $ra
+; MIPS64R2-NEXT:    nop
 entry:
   %shl = shl i32 %b, 31
   %and = and i32 %a, 2147483647
@@ -23,7 +31,14 @@ define i32 @or_shl_and(i32 %a, i32 %b) {
 ; MIPS32R2-NEXT:    ins $4, $5, 31, 1
 ; MIPS32R2-NEXT:    jr $ra
 ; MIPS32R2-NEXT:    move $2, $4
-
+;
+; MIPS64R2-LABEL: or_shl_and:
+; MIPS64R2:       # %bb.0: # %entry
+; MIPS64R2-NEXT:    sll $2, $4, 0
+; MIPS64R2-NEXT:    sll $1, $5, 0
+; MIPS64R2-NEXT:    ins $2, $1, 31, 1
+; MIPS64R2-NEXT:    jr $ra
+; MIPS64R2-NEXT:    nop
 entry:
   %shl = shl i32 %b, 31
   %and = and i32 %a, 2147483647
@@ -32,12 +47,20 @@ entry:
 }
 
 define i64 @dinsm(i64 %a, i64 %b) {
+; MIPS32R2-LABEL: dinsm:
+; MIPS32R2:       # %bb.0: # %entry
+; MIPS32R2-NEXT:    ins $4, $6, 17, 15
+; MIPS32R2-NEXT:    srl $1, $6, 15
+; MIPS32R2-NEXT:    sll $2, $7, 17
+; MIPS32R2-NEXT:    or $3, $2, $1
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    move $2, $4
+;
 ; MIPS64R2-LABEL: dinsm:
 ; MIPS64R2:       # %bb.0: # %entry
 ; MIPS64R2-NEXT:    dinsm $4, $5, 17, 47
 ; MIPS64R2-NEXT:    jr $ra
 ; MIPS64R2-NEXT:    move $2, $4
-
 entry:
   %shl = shl i64 %b, 17
   %and = and i64 %a, 131071
@@ -46,12 +69,18 @@ entry:
 }
 
 define i64 @dinsu(i64 %a, i64 %b) {
+; MIPS32R2-LABEL: dinsu:
+; MIPS32R2:       # %bb.0: # %entry
+; MIPS32R2-NEXT:    ins $5, $6, 3, 29
+; MIPS32R2-NEXT:    move $2, $4
+; MIPS32R2-NEXT:    jr $ra
+; MIPS32R2-NEXT:    move $3, $5
+;
 ; MIPS64R2-LABEL: dinsu:
 ; MIPS64R2:       # %bb.0: # %entry
 ; MIPS64R2-NEXT:    dinsu $4, $5, 35, 29
 ; MIPS64R2-NEXT:    jr $ra
 ; MIPS64R2-NEXT:    move $2, $4
-
 entry:
   %shl = shl i64 %b, 35
   %and = and i64 %a, 34359738367
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/load.ll b/llvm/test/CodeGen/Mips/llvm-ir/load.ll
index ee858ac94aed6..4f29fd848aed5 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/load.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/load.ll
@@ -27,48 +27,48 @@ define i8 @f1() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR3-LABEL: f1:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R6-LABEL: f1:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7:]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR6-LABEL: f1:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ;
@@ -76,11 +76,11 @@ define i8 @f1() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -88,7 +88,7 @@ define i8 @f1() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -98,17 +98,17 @@ define i8 @f1() {
 ; MIPS3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64-LABEL: f1:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -116,7 +116,7 @@ define i8 @f1() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -126,17 +126,17 @@ define i8 @f1() {
 ; MIPS64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64R6-LABEL: f1:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -144,7 +144,7 @@ define i8 @f1() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -155,31 +155,31 @@ define i8 @f1() {
 ; MIPS64R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR5FP64-LABEL: f1:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR5FP64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f1:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R5FP643-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 entry:
   %0 = load i8, ptr @a
   ret i8 %0
@@ -190,48 +190,48 @@ define i32 @f2() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR3-LABEL: f2:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R6-LABEL: f2:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR6-LABEL: f2:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -239,11 +239,11 @@ define i32 @f2() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -251,7 +251,7 @@ define i32 @f2() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -261,17 +261,17 @@ define i32 @f2() {
 ; MIPS3-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64-LABEL: f2:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -279,7 +279,7 @@ define i32 @f2() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -289,17 +289,17 @@ define i32 @f2() {
 ; MIPS64-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64R6-LABEL: f2:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -307,7 +307,7 @@ define i32 @f2() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -318,31 +318,31 @@ define i32 @f2() {
 ; MIPS64R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR5FP64-LABEL: f2:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f2:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 entry:
   %0 = load i8, ptr @a
   %1 = sext i8 %0 to i32
@@ -354,48 +354,48 @@ define i16 @f3() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR3-LABEL: f3:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R6-LABEL: f3:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR6-LABEL: f3:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -403,11 +403,11 @@ define i16 @f3() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -415,7 +415,7 @@ define i16 @f3() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -425,17 +425,17 @@ define i16 @f3() {
 ; MIPS3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64-LABEL: f3:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -443,7 +443,7 @@ define i16 @f3() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -453,17 +453,17 @@ define i16 @f3() {
 ; MIPS64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64R6-LABEL: f3:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -471,7 +471,7 @@ define i16 @f3() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -482,31 +482,31 @@ define i16 @f3() {
 ; MIPS64R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR5FP64-LABEL: f3:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f3:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 entry:
   %0 = load i16, ptr @b
   ret i16 %0
@@ -517,48 +517,48 @@ define i32 @f4() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR3-LABEL: f4:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R6-LABEL: f4:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR6-LABEL: f4:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -566,11 +566,11 @@ define i32 @f4() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -578,7 +578,7 @@ define i32 @f4() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -588,17 +588,17 @@ define i32 @f4() {
 ; MIPS3-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64-LABEL: f4:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -606,7 +606,7 @@ define i32 @f4() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -616,17 +616,17 @@ define i32 @f4() {
 ; MIPS64-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64R6-LABEL: f4:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -634,7 +634,7 @@ define i32 @f4() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -645,31 +645,31 @@ define i32 @f4() {
 ; MIPS64R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR5FP64-LABEL: f4:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f4:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 entry:
   %0 = load i16, ptr @b
   %1 = sext i16 %0 to i32
@@ -681,48 +681,48 @@ define i32 @f5() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR3-LABEL: f5:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R6-LABEL: f5:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR6-LABEL: f5:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -730,11 +730,11 @@ define i32 @f5() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -742,7 +742,7 @@ define i32 @f5() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -752,17 +752,17 @@ define i32 @f5() {
 ; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64-LABEL: f5:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -770,7 +770,7 @@ define i32 @f5() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -780,17 +780,17 @@ define i32 @f5() {
 ; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f5:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -798,7 +798,7 @@ define i32 @f5() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -809,31 +809,31 @@ define i32 @f5() {
 ; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f5:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f5:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 entry:
   %0 = load i32, ptr @c
   ret i32 %0
@@ -844,11 +844,11 @@ define i64 @f6() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
@@ -860,7 +860,7 @@ define i64 @f6() {
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Imm:0>>
@@ -869,17 +869,17 @@ define i64 @f6() {
 ; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R6-LABEL: f6:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
@@ -892,11 +892,11 @@ define i64 @f6() {
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Imm:0>>
@@ -907,11 +907,11 @@ define i64 @f6() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -919,7 +919,7 @@ define i64 @f6() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -929,17 +929,17 @@ define i64 @f6() {
 ; MIPS3-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64-LABEL: f6:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -947,7 +947,7 @@ define i64 @f6() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -957,17 +957,17 @@ define i64 @f6() {
 ; MIPS64-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f6:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -975,7 +975,7 @@ define i64 @f6() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -986,13 +986,13 @@ define i64 @f6() {
 ; MIPS64R6-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f6:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Imm:0>>
@@ -1001,17 +1001,17 @@ define i64 @f6() {
 ; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f6:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
@@ -1029,11 +1029,11 @@ define i64 @f7() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
@@ -1045,11 +1045,11 @@ define i64 @f7() {
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
@@ -1061,11 +1061,11 @@ define i64 @f7() {
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
@@ -1078,11 +1078,11 @@ define i64 @f7() {
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
@@ -1094,11 +1094,11 @@ define i64 @f7() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1106,7 +1106,7 @@ define i64 @f7() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1116,17 +1116,17 @@ define i64 @f7() {
 ; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64-LABEL: f7:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1134,7 +1134,7 @@ define i64 @f7() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1144,17 +1144,17 @@ define i64 @f7() {
 ; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f7:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1162,7 +1162,7 @@ define i64 @f7() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1173,17 +1173,17 @@ define i64 @f7() {
 ; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f7:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
@@ -1195,11 +1195,11 @@ define i64 @f7() {
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
@@ -1217,48 +1217,48 @@ define float @f8() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR3-LABEL: f8:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R6-LABEL: f8:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR6-LABEL: f8:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -1266,11 +1266,11 @@ define float @f8() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1278,7 +1278,7 @@ define float @f8() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1288,17 +1288,17 @@ define float @f8() {
 ; MIPS3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS64-LABEL: f8:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1306,7 +1306,7 @@ define float @f8() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1316,17 +1316,17 @@ define float @f8() {
 ; MIPS64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS64R6-LABEL: f8:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1334,7 +1334,7 @@ define float @f8() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1345,31 +1345,31 @@ define float @f8() {
 ; MIPS64R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR5FP64-LABEL: f8:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R5FP643-LABEL: f8:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 entry:
   %0 = load float, ptr @e
   ret float %0
@@ -1380,48 +1380,48 @@ define double @f9() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST29:]] LDC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR3-LABEL: f9:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST30:]] LDC1_MM_D32
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R6-LABEL: f9:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR6-LABEL: f9:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST32:]] LDC1_D64_MMR6
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -1429,11 +1429,11 @@ define double @f9() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1441,7 +1441,7 @@ define double @f9() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1451,17 +1451,17 @@ define double @f9() {
 ; MIPS3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS64-LABEL: f9:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1469,7 +1469,7 @@ define double @f9() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1479,17 +1479,17 @@ define double @f9() {
 ; MIPS64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS64R6-LABEL: f9:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1497,7 +1497,7 @@ define double @f9() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1508,31 +1508,31 @@ define double @f9() {
 ; MIPS64R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR5FP64-LABEL: f9:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST33:]] LDC1_MM_D64
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R5FP643-LABEL: f9:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 entry:
   %0 = load double, ptr @f
   ret double %0
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/store.ll b/llvm/test/CodeGen/Mips/llvm-ir/store.ll
index 880a0f522574b..8b51c0939b8d4 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/store.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/store.ll
@@ -26,48 +26,48 @@ define void @f1(i8 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR3-LABEL: f1:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR3-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R6-LABEL: f1:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7:]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR6-LABEL: f1:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ;
@@ -75,11 +75,11 @@ define void @f1(i8 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -87,7 +87,7 @@ define void @f1(i8 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -97,17 +97,17 @@ define void @f1(i8 %a) {
 ; MIPS4-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST12:]] SB64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64R6-LABEL: f1:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -115,7 +115,7 @@ define void @f1(i8 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -126,31 +126,31 @@ define void @f1(i8 %a) {
 ; MIPS64R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST12:]] SB64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR5FP64-LABEL: f1:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR5FP64-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f1:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R5FP643-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
   store i8 %a, ptr @a
   ret void
 }
@@ -160,48 +160,48 @@ define void @f2(i16 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR3-LABEL: f2:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R6-LABEL: f2:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR6-LABEL: f2:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -209,11 +209,11 @@ define void @f2(i16 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -221,7 +221,7 @@ define void @f2(i16 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -231,17 +231,17 @@ define void @f2(i16 %a) {
 ; MIPS4-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST16:]] SH64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64R6-LABEL: f2:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -249,7 +249,7 @@ define void @f2(i16 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -260,31 +260,31 @@ define void @f2(i16 %a) {
 ; MIPS64R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST16:]] SH64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR5FP64-LABEL: f2:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f2:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
   store i16 %a, ptr @b
   ret void
 }
@@ -294,48 +294,48 @@ define void @f3(i32 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR3-LABEL: f3:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R6-LABEL: f3:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR6-LABEL: f3:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -343,11 +343,11 @@ define void @f3(i32 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -355,7 +355,7 @@ define void @f3(i32 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -365,17 +365,17 @@ define void @f3(i32 %a) {
 ; MIPS4-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST19:]] SW64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f3:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -383,7 +383,7 @@ define void @f3(i32 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -394,31 +394,31 @@ define void @f3(i32 %a) {
 ; MIPS64R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST19:]] SW64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f3:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f3:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
   store i32 %a, ptr @c
   ret void
 }
@@ -428,15 +428,15 @@ define void @f4(i64 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS32-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
@@ -448,15 +448,15 @@ define void @f4(i64 %a) {
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MMR3-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR3-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR3-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
@@ -468,15 +468,15 @@ define void @f4(i64 %a) {
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS32R6-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R6-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
@@ -489,15 +489,15 @@ define void @f4(i64 %a) {
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MMR6-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR6-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR6-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
@@ -509,11 +509,11 @@ define void @f4(i64 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(d) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,d)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,d)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -521,7 +521,7 @@ define void @f4(i64 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -531,17 +531,17 @@ define void @f4(i64 %a) {
 ; MIPS4-NEXT:    sd $4, %lo(d)($1) # <MCInst #[[#MCINST23:]] SD
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ;
 ; MIPS64R6-LABEL: f4:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(d) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,d)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,d)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -549,7 +549,7 @@ define void @f4(i64 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -560,21 +560,21 @@ define void @f4(i64 %a) {
 ; MIPS64R6-NEXT:    sd $4, %lo(d)($1) # <MCInst #[[#MCINST23:]] SD
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ;
 ; MMR5FP64-LABEL: f4:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MMR5FP64-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR5FP64-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR5FP64-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
@@ -586,15 +586,15 @@ define void @f4(i64 %a) {
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS32R5FP643-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R5FP643-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
@@ -610,48 +610,48 @@ define void @f5(float %e) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR3-LABEL: f5:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R6-LABEL: f5:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR6-LABEL: f5:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -659,11 +659,11 @@ define void @f5(float %e) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -671,7 +671,7 @@ define void @f5(float %e) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -681,17 +681,17 @@ define void @f5(float %e) {
 ; MIPS4-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS64R6-LABEL: f5:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -699,7 +699,7 @@ define void @f5(float %e) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -710,31 +710,31 @@ define void @f5(float %e) {
 ; MIPS64R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR5FP64-LABEL: f5:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R5FP643-LABEL: f5:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
   store float %e, ptr @e
   ret void
 }
@@ -744,48 +744,48 @@ define void @f6(double %f) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST26:]] SDC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR3-LABEL: f6:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST27:]] SDC1_MM_D32
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R6-LABEL: f6:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR6-LABEL: f6:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST29:]] SDC1_D64_MMR6
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -793,11 +793,11 @@ define void @f6(double %f) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -805,7 +805,7 @@ define void @f6(double %f) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -815,17 +815,17 @@ define void @f6(double %f) {
 ; MIPS4-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS64R6-LABEL: f6:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -833,7 +833,7 @@ define void @f6(double %f) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -844,31 +844,31 @@ define void @f6(double %f) {
 ; MIPS64R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR5FP64-LABEL: f6:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST30:]] SDC1_MM_D64
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R5FP643-LABEL: f6:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
   store double %f, ptr @f
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/msa/compare_float.ll b/llvm/test/CodeGen/Mips/msa/compare_float.ll
index 178264581ea19..d06131bd09627 100644
--- a/llvm/test/CodeGen/Mips/msa/compare_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/compare_float.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=mips-elf -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,MIPS
-; RUN: llc -mtriple=mipsel-elf -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s --check-prefixes=CHECK,MIPSEL
+; RUN: llc -mtriple=mips-elf -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel-elf -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
 declare <4 x float> @llvm.mips.fmax.w(<4 x float>, <4 x float>) nounwind
 declare <2 x double> @llvm.mips.fmax.d(<2 x double>, <2 x double>) nounwind
@@ -525,26 +525,14 @@ define void @bsel_v2f64(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
 
 ; Note that IfSet and IfClr are swapped since the condition is inverted
 define void @bseli_v4f32(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
-; MIPS-LABEL: bseli_v4f32:
-; MIPS:       # %bb.0:
-; MIPS-NEXT:    ld.w $w0, 0($5)
-; MIPS-NEXT:    ld.w $w1, 0($6)
-; MIPS-NEXT:    fclt.w $w1, $w1, $w0
-; MIPS-NEXT:    ldi.b $w2, 0
-; MIPS-NEXT:    shf.b $w2, $w2, 27
-; MIPS-NEXT:    bsel.v $w1, $w2, $w0
-; MIPS-NEXT:    jr $ra
-; MIPS-NEXT:    st.w $w1, 0($4)
-;
-; MIPSEL-LABEL: bseli_v4f32:
-; MIPSEL:       # %bb.0:
-; MIPSEL-NEXT:    ld.w $w0, 0($5)
-; MIPSEL-NEXT:    ld.w $w1, 0($6)
-; MIPSEL-NEXT:    fclt.w $w1, $w1, $w0
-; MIPSEL-NEXT:    ldi.b $w2, 0
-; MIPSEL-NEXT:    bsel.v $w1, $w2, $w0
-; MIPSEL-NEXT:    jr $ra
-; MIPSEL-NEXT:    st.w $w1, 0($4)
+; CHECK-LABEL: bseli_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fclt.w $w1, $w1, $w0
+; CHECK-NEXT:    and.v $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
   %2 = load <4 x float>, ptr %b
   %3 = fcmp ogt <4 x float> %1, %2
@@ -555,27 +543,14 @@ define void @bseli_v4f32(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
 
 ; Note that IfSet and IfClr are swapped since the condition is inverted
 define void @bseli_v2f64(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
-; MIPS-LABEL: bseli_v2f64:
-; MIPS:       # %bb.0:
-; MIPS-NEXT:    ld.d $w0, 0($5)
-; MIPS-NEXT:    ld.d $w1, 0($6)
-; MIPS-NEXT:    fclt.d $w1, $w1, $w0
-; MIPS-NEXT:    ldi.b $w2, 0
-; MIPS-NEXT:    shf.b $w2, $w2, 27
-; MIPS-NEXT:    shf.w $w2, $w2, 177
-; MIPS-NEXT:    bsel.v $w1, $w2, $w0
-; MIPS-NEXT:    jr $ra
-; MIPS-NEXT:    st.d $w1, 0($4)
-;
-; MIPSEL-LABEL: bseli_v2f64:
-; MIPSEL:       # %bb.0:
-; MIPSEL-NEXT:    ld.d $w0, 0($5)
-; MIPSEL-NEXT:    ld.d $w1, 0($6)
-; MIPSEL-NEXT:    fclt.d $w1, $w1, $w0
-; MIPSEL-NEXT:    ldi.b $w2, 0
-; MIPSEL-NEXT:    bsel.v $w1, $w2, $w0
-; MIPSEL-NEXT:    jr $ra
-; MIPSEL-NEXT:    st.d $w1, 0($4)
+; CHECK-LABEL: bseli_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fclt.d $w1, $w1, $w0
+; CHECK-NEXT:    and.v $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
   %2 = load <2 x double>, ptr %b
   %3 = fcmp ogt <2 x double> %1, %2
diff --git a/llvm/test/CodeGen/Mips/setcc-se.ll b/llvm/test/CodeGen/Mips/setcc-se.ll
index 26fd2845f0817..c1054607d5e91 100644
--- a/llvm/test/CodeGen/Mips/setcc-se.ll
+++ b/llvm/test/CodeGen/Mips/setcc-se.ll
@@ -1,39 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=mipsel < %s | FileCheck %s
-; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic < %s -asm-show-inst | FileCheck %s -check-prefix=MMR6
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic < %s -asm-show-inst | FileCheck %s -check-prefix=MMR6
 
 @g1 = external global i32
 
-; CHECK-LABEL: seteq0:
-; CHECK:  sltiu ${{[0-9]+}}, $4, 1
-; MMR6:   sltiu ${{[0-9]+}}, $4, 1
-; MMR6:   <MCInst #{{[0-9]+}} SLTiu_MM
-
 define i32 @seteq0(i32 %a) {
+; CHECK-LABEL: seteq0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    sltiu $2, $4, 1
+;
+; MMR6-LABEL: seteq0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sltiu $2, $4, 1 # <MCInst #[[#MCINST1:]] SLTiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>
+; MMR6-NEXT:    # <MCOperand Imm:1>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2:]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
 entry:
   %cmp = icmp eq i32 %a, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
-; CHECK-LABEL: setne0:
-; CHECK:  sltu ${{[0-9]+}}, $zero, $4
-; MMR6:   sltu ${{[0-9]+}}, $zero, $4
-; MMR6:   <MCInst #{{[0-9]+}} SLTu_MM
-
 define i32 @setne0(i32 %a) {
+; CHECK-LABEL: setne0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    sltu $2, $zero, $4
+;
+; MMR6-LABEL: setne0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sltu $2, $zero, $4 # <MCInst #[[#MCINST3:]] SLTu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ne i32 %a, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
-; CHECK-LABEL: slti_beq0:
-; CHECK:  slti $[[R0:[0-9]+]], $4, -32768
-; MMR6:   slti $[[R0:[0-9]+]], $4, -32768
-; MMR6:   <MCInst #{{[0-9]+}} SLTi_MM
-; CHECK:  beqz $[[R0]]
-
 define void @slti_beq0(i32 %a) {
+; CHECK-LABEL: slti_beq0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slti $1, $4, -32768
+; CHECK-NEXT:    beqz $1, $BB2_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB2_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4:]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5:]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6:]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>>
+; MMR6-NEXT:    slti $1, $4, -32768 # <MCInst #[[#MCINST7:]] SLTi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:-32768>>
+; MMR6-NEXT:    beqzc $1, $BB2_2 # <MCInst #[[#MCINST8:]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB2_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9:]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10:]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB2_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, -32768
   br i1 %cmp, label %if.then, label %if.end
@@ -46,12 +101,60 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: slti_beq1:
-; CHECK:  slt ${{[0-9]+}}
-; MMR6:   slt ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLT_MM
-
 define void @slti_beq1(i32 %a) {
+; CHECK-LABEL: slti_beq1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui $1, 65535
+; CHECK-NEXT:    ori $1, $1, 32766
+; CHECK-NEXT:    slt $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB3_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB3_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq1:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    lui $1, 65535 # <MCInst #[[#MCINST11:]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:65535>>
+; MMR6-NEXT:    ori $1, $1, 32766 # <MCInst #[[#MCINST12:]] ORi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:32766>>
+; MMR6-NEXT:    slt $1, $1, $4 # <MCInst #[[#MCINST13:]] SLT_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB3_2 # <MCInst #[[#MCINST14:]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB3_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB3_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, -32769
   br i1 %cmp, label %if.then, label %if.end
@@ -64,13 +167,51 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: slti_beq2:
-; CHECK:  slti $[[R0:[0-9]+]], $4, 32767
-; MMR6:   slti $[[R0:[0-9]+]], $4, 32767
-; MMR6:   <MCInst #{{[0-9]+}} SLTi_MM
-; CHECK:  beqz $[[R0]]
-
 define void @slti_beq2(i32 %a) {
+; CHECK-LABEL: slti_beq2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slti $1, $4, 32767
+; CHECK-NEXT:    beqz $1, $BB4_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB4_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq2:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    slti $1, $4, 32767 # <MCInst #[[#MCINST7]] SLTi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    beqzc $1, $BB4_2 # <MCInst #[[#MCINST8]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB4_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB4_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, 32767
   br i1 %cmp, label %if.then, label %if.end
@@ -83,12 +224,56 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: slti_beq3:
-; CHECK:  slt ${{[0-9]+}}
-; MMR6:   slt ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLT_MM
-
 define void @slti_beq3(i32 %a) {
+; CHECK-LABEL: slti_beq3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $1, $zero, 32767
+; CHECK-NEXT:    slt $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB5_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB5_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq3:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    addiu $1, $zero, 32767 # <MCInst #[[#MCINST15:]] ADDiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    slt $1, $1, $4 # <MCInst #[[#MCINST13]] SLT_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB5_2 # <MCInst #[[#MCINST14]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB5_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB5_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, 32768
   br i1 %cmp, label %if.then, label %if.end
@@ -101,13 +286,51 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq0:
-; CHECK:  sltiu $[[R0:[0-9]+]], $4, 32767
-; MMR6:   sltiu $[[R0:[0-9]+]], $4, 32767
-; MMR6:   <MCInst #{{[0-9]+}} SLTiu_MM
-; CHECK:  beqz $[[R0]]
-
 define void @sltiu_beq0(i32 %a) {
+; CHECK-LABEL: sltiu_beq0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sltiu $1, $4, 32767
+; CHECK-NEXT:    beqz $1, $BB6_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB6_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    sltiu $1, $4, 32767 # <MCInst #[[#MCINST1]] SLTiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    beqzc $1, $BB6_2 # <MCInst #[[#MCINST8]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB6_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB6_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, 32767
   br i1 %cmp, label %if.then, label %if.end
@@ -120,12 +343,56 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq1:
-; CHECK:  sltu ${{[0-9]+}}
-; MMR6:   sltu ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLTu_MM
-
 define void @sltiu_beq1(i32 %a) {
+; CHECK-LABEL: sltiu_beq1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $1, $zero, 32767
+; CHECK-NEXT:    sltu $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB7_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB7_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq1:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    addiu $1, $zero, 32767 # <MCInst #[[#MCINST15]] ADDiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    sltu $1, $1, $4 # <MCInst #[[#MCINST3]] SLTu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB7_2 # <MCInst #[[#MCINST14]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB7_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB7_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, 32768
   br i1 %cmp, label %if.then, label %if.end
@@ -138,13 +405,51 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq2:
-; CHECK:  sltiu $[[R0:[0-9]+]], $4, -32768
-; MMR6:   sltiu $[[R0:[0-9]+]], $4, -32768
-; MMR6:   <MCInst #{{[0-9]+}} SLTiu_MM
-; CHECK:  beqz $[[R0]]
-
 define void @sltiu_beq2(i32 %a) {
+; CHECK-LABEL: sltiu_beq2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sltiu $1, $4, -32768
+; CHECK-NEXT:    beqz $1, $BB8_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB8_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq2:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    sltiu $1, $4, -32768 # <MCInst #[[#MCINST1]] SLTiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:-32768>>
+; MMR6-NEXT:    beqzc $1, $BB8_2 # <MCInst #[[#MCINST8]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB8_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB8_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, -32768
   br i1 %cmp, label %if.then, label %if.end
@@ -157,12 +462,60 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq3:
-; CHECK:  sltu ${{[0-9]+}}
-; MMR6:   sltu ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLTu_MM
-
 define void @sltiu_beq3(i32 %a) {
+; CHECK-LABEL: sltiu_beq3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui $1, 65535
+; CHECK-NEXT:    ori $1, $1, 32766
+; CHECK-NEXT:    sltu $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB9_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB9_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq3:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    lui $1, 65535 # <MCInst #[[#MCINST11]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:65535>>
+; MMR6-NEXT:    ori $1, $1, 32766 # <MCInst #[[#MCINST12]] ORi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:32766>>
+; MMR6-NEXT:    sltu $1, $1, $4 # <MCInst #[[#MCINST3]] SLTu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB9_2 # <MCInst #[[#MCINST14]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB9_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB9_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, -32769
   br i1 %cmp, label %if.then, label %if.end
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
index 65a077d67e4ba..c99860cc5cc1b 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
@@ -14,17 +14,17 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -41,7 +41,7 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB0_1;
 ; SM60-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -59,17 +59,17 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -86,7 +86,7 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB1_1;
 ; SM60-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -104,17 +104,17 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -131,7 +131,7 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB2_1;
 ; SM60-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -149,17 +149,17 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -177,7 +177,7 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB3_1;
 ; SM60-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -195,17 +195,17 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -223,7 +223,7 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB4_1;
 ; SM60-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -241,17 +241,17 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
@@ -269,7 +269,7 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB5_1;
 ; SM60-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -287,18 +287,18 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
@@ -316,7 +316,7 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB6_1;
 ; SM60-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -334,18 +334,18 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
@@ -363,7 +363,7 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB7_1;
 ; SM60-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -381,18 +381,18 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
@@ -410,7 +410,7 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB8_1;
 ; SM60-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -428,17 +428,17 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
@@ -456,7 +456,7 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB9_1;
 ; SM60-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -474,17 +474,17 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
@@ -502,7 +502,7 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB10_1;
 ; SM60-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -520,17 +520,17 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
@@ -548,7 +548,7 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB11_1;
 ; SM60-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -566,17 +566,17 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
@@ -594,7 +594,7 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB12_1;
 ; SM60-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -612,17 +612,17 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
@@ -640,7 +640,7 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB13_1;
 ; SM60-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -658,17 +658,17 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
@@ -686,7 +686,7 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB14_1;
 ; SM60-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -704,18 +704,18 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
@@ -733,7 +733,7 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB15_1;
 ; SM60-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -751,18 +751,18 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
@@ -780,7 +780,7 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB16_1;
 ; SM60-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -798,18 +798,18 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
@@ -827,7 +827,7 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB17_1;
 ; SM60-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -845,18 +845,18 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
@@ -873,7 +873,7 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB18_1;
 ; SM60-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -891,18 +891,18 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
@@ -919,7 +919,7 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB19_1;
 ; SM60-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -937,18 +937,18 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
@@ -965,7 +965,7 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB20_1;
 ; SM60-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -983,18 +983,18 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
@@ -1012,7 +1012,7 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB21_1;
 ; SM60-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1030,18 +1030,18 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
@@ -1059,7 +1059,7 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB22_1;
 ; SM60-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1077,18 +1077,18 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
@@ -1106,7 +1106,7 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB23_1;
 ; SM60-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1124,18 +1124,18 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
@@ -1153,7 +1153,7 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB24_1;
 ; SM60-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1171,18 +1171,18 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
@@ -1200,7 +1200,7 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB25_1;
 ; SM60-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1218,18 +1218,18 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
@@ -1247,7 +1247,7 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB26_1;
 ; SM60-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1265,18 +1265,18 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
@@ -1294,7 +1294,7 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB27_1;
 ; SM60-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1312,18 +1312,18 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
@@ -1341,7 +1341,7 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB28_1;
 ; SM60-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1359,18 +1359,18 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
@@ -1388,7 +1388,7 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB29_1;
 ; SM60-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1406,18 +1406,18 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB30_1: // %partword.cmpxchg.loop
@@ -1435,7 +1435,7 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB30_1;
 ; SM60-NEXT:  $L__BB30_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1453,18 +1453,18 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB31_1: // %partword.cmpxchg.loop
@@ -1482,7 +1482,7 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB31_1;
 ; SM60-NEXT:  $L__BB31_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1500,18 +1500,18 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB32_1: // %partword.cmpxchg.loop
@@ -1529,7 +1529,7 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB32_1;
 ; SM60-NEXT:  $L__BB32_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1547,18 +1547,18 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB33_1: // %partword.cmpxchg.loop
@@ -1576,7 +1576,7 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB33_1;
 ; SM60-NEXT:  $L__BB33_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1594,18 +1594,18 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB34_1: // %partword.cmpxchg.loop
@@ -1623,7 +1623,7 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB34_1;
 ; SM60-NEXT:  $L__BB34_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1641,18 +1641,18 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB35_1: // %partword.cmpxchg.loop
@@ -1670,7 +1670,7 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB35_1;
 ; SM60-NEXT:  $L__BB35_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1688,18 +1688,18 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB36_1: // %partword.cmpxchg.loop
@@ -1717,7 +1717,7 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB36_1;
 ; SM60-NEXT:  $L__BB36_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1735,18 +1735,18 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB37_1: // %partword.cmpxchg.loop
@@ -1764,7 +1764,7 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB37_1;
 ; SM60-NEXT:  $L__BB37_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1782,18 +1782,18 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB38_1: // %partword.cmpxchg.loop
@@ -1811,7 +1811,7 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB38_1;
 ; SM60-NEXT:  $L__BB38_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1829,18 +1829,18 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB39_1: // %partword.cmpxchg.loop
@@ -1858,7 +1858,7 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB39_1;
 ; SM60-NEXT:  $L__BB39_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1876,18 +1876,18 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB40_1: // %partword.cmpxchg.loop
@@ -1905,7 +1905,7 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB40_1;
 ; SM60-NEXT:  $L__BB40_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1923,18 +1923,18 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB41_1: // %partword.cmpxchg.loop
@@ -1952,7 +1952,7 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB41_1;
 ; SM60-NEXT:  $L__BB41_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1970,18 +1970,18 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB42_1: // %partword.cmpxchg.loop
@@ -1999,7 +1999,7 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB42_1;
 ; SM60-NEXT:  $L__BB42_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2017,18 +2017,18 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB43_1: // %partword.cmpxchg.loop
@@ -2046,7 +2046,7 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB43_1;
 ; SM60-NEXT:  $L__BB43_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2064,18 +2064,18 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB44_1: // %partword.cmpxchg.loop
@@ -2093,7 +2093,7 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB44_1;
 ; SM60-NEXT:  $L__BB44_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
index 7107fbcf6eb54..68de517f65bb9 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
@@ -14,17 +14,17 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -41,7 +41,7 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB0_1;
 ; SM70-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -59,17 +59,17 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -86,7 +86,7 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB1_1;
 ; SM70-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -104,17 +104,17 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -131,7 +131,7 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB2_1;
 ; SM70-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -149,17 +149,17 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -177,7 +177,7 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB3_1;
 ; SM70-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -195,17 +195,17 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -223,7 +223,7 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB4_1;
 ; SM70-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -241,17 +241,17 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
@@ -269,7 +269,7 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB5_1;
 ; SM70-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -287,18 +287,18 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
@@ -316,7 +316,7 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB6_1;
 ; SM70-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -334,18 +334,18 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
@@ -363,7 +363,7 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB7_1;
 ; SM70-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -381,18 +381,18 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
@@ -410,7 +410,7 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB8_1;
 ; SM70-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -428,17 +428,17 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
@@ -456,7 +456,7 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB9_1;
 ; SM70-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -474,17 +474,17 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
@@ -502,7 +502,7 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB10_1;
 ; SM70-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -520,17 +520,17 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
@@ -548,7 +548,7 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB11_1;
 ; SM70-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -566,17 +566,17 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
@@ -594,7 +594,7 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB12_1;
 ; SM70-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -612,17 +612,17 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
@@ -640,7 +640,7 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB13_1;
 ; SM70-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -658,17 +658,17 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
@@ -686,7 +686,7 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB14_1;
 ; SM70-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -704,18 +704,18 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
@@ -733,7 +733,7 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB15_1;
 ; SM70-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -751,18 +751,18 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
@@ -780,7 +780,7 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB16_1;
 ; SM70-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -798,18 +798,18 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
@@ -827,7 +827,7 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB17_1;
 ; SM70-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -845,18 +845,18 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
@@ -873,7 +873,7 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB18_1;
 ; SM70-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -891,18 +891,18 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
@@ -919,7 +919,7 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB19_1;
 ; SM70-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -937,18 +937,18 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
@@ -965,7 +965,7 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB20_1;
 ; SM70-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -983,18 +983,18 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
@@ -1012,7 +1012,7 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB21_1;
 ; SM70-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1030,18 +1030,18 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
@@ -1059,7 +1059,7 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB22_1;
 ; SM70-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1077,18 +1077,18 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
@@ -1106,7 +1106,7 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB23_1;
 ; SM70-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1124,18 +1124,18 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
@@ -1153,7 +1153,7 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB24_1;
 ; SM70-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1171,18 +1171,18 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
@@ -1200,7 +1200,7 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB25_1;
 ; SM70-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1218,18 +1218,18 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
@@ -1247,7 +1247,7 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB26_1;
 ; SM70-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1265,18 +1265,18 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
@@ -1294,7 +1294,7 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB27_1;
 ; SM70-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1312,18 +1312,18 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
@@ -1341,7 +1341,7 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB28_1;
 ; SM70-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1359,18 +1359,18 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
@@ -1388,7 +1388,7 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB29_1;
 ; SM70-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1406,18 +1406,18 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB30_1: // %partword.cmpxchg.loop
@@ -1435,7 +1435,7 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB30_1;
 ; SM70-NEXT:  $L__BB30_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1453,18 +1453,18 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB31_1: // %partword.cmpxchg.loop
@@ -1482,7 +1482,7 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB31_1;
 ; SM70-NEXT:  $L__BB31_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1500,18 +1500,18 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB32_1: // %partword.cmpxchg.loop
@@ -1529,7 +1529,7 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB32_1;
 ; SM70-NEXT:  $L__BB32_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1547,18 +1547,18 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB33_1: // %partword.cmpxchg.loop
@@ -1576,7 +1576,7 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB33_1;
 ; SM70-NEXT:  $L__BB33_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1594,18 +1594,18 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB34_1: // %partword.cmpxchg.loop
@@ -1623,7 +1623,7 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB34_1;
 ; SM70-NEXT:  $L__BB34_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1641,18 +1641,18 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB35_1: // %partword.cmpxchg.loop
@@ -1670,7 +1670,7 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB35_1;
 ; SM70-NEXT:  $L__BB35_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1688,18 +1688,18 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB36_1: // %partword.cmpxchg.loop
@@ -1717,7 +1717,7 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB36_1;
 ; SM70-NEXT:  $L__BB36_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1735,18 +1735,18 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB37_1: // %partword.cmpxchg.loop
@@ -1764,7 +1764,7 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB37_1;
 ; SM70-NEXT:  $L__BB37_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1782,18 +1782,18 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB38_1: // %partword.cmpxchg.loop
@@ -1811,7 +1811,7 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB38_1;
 ; SM70-NEXT:  $L__BB38_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1829,18 +1829,18 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB39_1: // %partword.cmpxchg.loop
@@ -1858,7 +1858,7 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB39_1;
 ; SM70-NEXT:  $L__BB39_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1876,18 +1876,18 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB40_1: // %partword.cmpxchg.loop
@@ -1905,7 +1905,7 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB40_1;
 ; SM70-NEXT:  $L__BB40_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1923,18 +1923,18 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB41_1: // %partword.cmpxchg.loop
@@ -1952,7 +1952,7 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB41_1;
 ; SM70-NEXT:  $L__BB41_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1970,18 +1970,18 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB42_1: // %partword.cmpxchg.loop
@@ -1999,7 +1999,7 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB42_1;
 ; SM70-NEXT:  $L__BB42_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2017,18 +2017,18 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB43_1: // %partword.cmpxchg.loop
@@ -2046,7 +2046,7 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB43_1;
 ; SM70-NEXT:  $L__BB43_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2064,18 +2064,18 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB44_1: // %partword.cmpxchg.loop
@@ -2093,7 +2093,7 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB44_1;
 ; SM70-NEXT:  $L__BB44_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
index f289c3cf3d509..e20f988577282 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
@@ -14,17 +14,17 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -41,7 +41,7 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB0_1;
 ; SM90-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -59,17 +59,17 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -86,7 +86,7 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB1_1;
 ; SM90-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -104,17 +104,17 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -131,7 +131,7 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB2_1;
 ; SM90-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -149,17 +149,17 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -177,7 +177,7 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB3_1;
 ; SM90-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -195,17 +195,17 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -223,7 +223,7 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB4_1;
 ; SM90-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -241,17 +241,17 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
@@ -269,7 +269,7 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB5_1;
 ; SM90-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -287,18 +287,18 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
@@ -316,7 +316,7 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB6_1;
 ; SM90-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -334,18 +334,18 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
@@ -363,7 +363,7 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB7_1;
 ; SM90-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -381,18 +381,18 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
@@ -410,7 +410,7 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB8_1;
 ; SM90-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -428,17 +428,17 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
@@ -456,7 +456,7 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB9_1;
 ; SM90-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -474,17 +474,17 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
@@ -502,7 +502,7 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB10_1;
 ; SM90-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -520,17 +520,17 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
@@ -548,7 +548,7 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB11_1;
 ; SM90-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -566,17 +566,17 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
@@ -594,7 +594,7 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB12_1;
 ; SM90-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -612,17 +612,17 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
@@ -640,7 +640,7 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB13_1;
 ; SM90-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -658,17 +658,17 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
@@ -686,7 +686,7 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB14_1;
 ; SM90-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -704,18 +704,18 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
@@ -733,7 +733,7 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB15_1;
 ; SM90-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -751,18 +751,18 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
@@ -780,7 +780,7 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB16_1;
 ; SM90-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -798,18 +798,18 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
@@ -827,7 +827,7 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB17_1;
 ; SM90-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -845,18 +845,18 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
@@ -873,7 +873,7 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB18_1;
 ; SM90-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -891,18 +891,18 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
@@ -919,7 +919,7 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB19_1;
 ; SM90-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -937,18 +937,18 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
@@ -965,7 +965,7 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB20_1;
 ; SM90-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -983,18 +983,18 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
@@ -1012,7 +1012,7 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB21_1;
 ; SM90-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1030,18 +1030,18 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
@@ -1059,7 +1059,7 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB22_1;
 ; SM90-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1077,18 +1077,18 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
@@ -1106,7 +1106,7 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB23_1;
 ; SM90-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1124,18 +1124,18 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
@@ -1153,7 +1153,7 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB24_1;
 ; SM90-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1171,18 +1171,18 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
@@ -1200,7 +1200,7 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB25_1;
 ; SM90-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1218,18 +1218,18 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
@@ -1247,7 +1247,7 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB26_1;
 ; SM90-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1265,18 +1265,18 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
@@ -1294,7 +1294,7 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB27_1;
 ; SM90-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1312,18 +1312,18 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
@@ -1341,7 +1341,7 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB28_1;
 ; SM90-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1359,18 +1359,18 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
@@ -1388,7 +1388,7 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB29_1;
 ; SM90-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1406,18 +1406,18 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB30_1: // %partword.cmpxchg.loop
@@ -1435,7 +1435,7 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB30_1;
 ; SM90-NEXT:  $L__BB30_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1453,18 +1453,18 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB31_1: // %partword.cmpxchg.loop
@@ -1482,7 +1482,7 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB31_1;
 ; SM90-NEXT:  $L__BB31_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1500,18 +1500,18 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB32_1: // %partword.cmpxchg.loop
@@ -1529,7 +1529,7 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB32_1;
 ; SM90-NEXT:  $L__BB32_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1547,18 +1547,18 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB33_1: // %partword.cmpxchg.loop
@@ -1576,7 +1576,7 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB33_1;
 ; SM90-NEXT:  $L__BB33_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1594,18 +1594,18 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB34_1: // %partword.cmpxchg.loop
@@ -1623,7 +1623,7 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB34_1;
 ; SM90-NEXT:  $L__BB34_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1641,18 +1641,18 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB35_1: // %partword.cmpxchg.loop
@@ -1670,7 +1670,7 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB35_1;
 ; SM90-NEXT:  $L__BB35_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1688,18 +1688,18 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB36_1: // %partword.cmpxchg.loop
@@ -1717,7 +1717,7 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB36_1;
 ; SM90-NEXT:  $L__BB36_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1735,18 +1735,18 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB37_1: // %partword.cmpxchg.loop
@@ -1764,7 +1764,7 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB37_1;
 ; SM90-NEXT:  $L__BB37_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1782,18 +1782,18 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB38_1: // %partword.cmpxchg.loop
@@ -1811,7 +1811,7 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB38_1;
 ; SM90-NEXT:  $L__BB38_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1829,18 +1829,18 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB39_1: // %partword.cmpxchg.loop
@@ -1858,7 +1858,7 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB39_1;
 ; SM90-NEXT:  $L__BB39_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1876,18 +1876,18 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB40_1: // %partword.cmpxchg.loop
@@ -1905,7 +1905,7 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB40_1;
 ; SM90-NEXT:  $L__BB40_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1923,18 +1923,18 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB41_1: // %partword.cmpxchg.loop
@@ -1952,7 +1952,7 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB41_1;
 ; SM90-NEXT:  $L__BB41_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1970,18 +1970,18 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB42_1: // %partword.cmpxchg.loop
@@ -1999,7 +1999,7 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB42_1;
 ; SM90-NEXT:  $L__BB42_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2017,18 +2017,18 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB43_1: // %partword.cmpxchg.loop
@@ -2046,7 +2046,7 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB43_1;
 ; SM90-NEXT:  $L__BB43_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2064,18 +2064,18 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB44_1: // %partword.cmpxchg.loop
@@ -2093,7 +2093,7 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB44_1;
 ; SM90-NEXT:  $L__BB44_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg.ll b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
index 9eeff9d7c2b75..85414a2ab04e8 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
@@ -21,17 +21,17 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [relaxed_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i8_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [relaxed_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    ld.param.b8 %r9, [relaxed_sys_i8_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -48,7 +48,7 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    mov.b32 %r20, %r8;
 ; SM30-NEXT:    @%p2 bra $L__BB0_1;
 ; SM30-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: relaxed_sys_i8(
@@ -62,17 +62,17 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [relaxed_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i8_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [relaxed_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [relaxed_sys_i8_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -89,7 +89,7 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB0_1;
 ; SM70-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: relaxed_sys_i8(
 ; SM90:       {
@@ -147,17 +147,17 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [acquire_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acquire_sys_i8_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [acquire_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    ld.param.b8 %r9, [acquire_sys_i8_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -175,7 +175,7 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    @%p2 bra $L__BB1_1;
 ; SM30-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acquire_sys_i8(
@@ -189,17 +189,17 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_sys_i8_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_sys_i8_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -217,7 +217,7 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB1_1;
 ; SM70-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acquire_sys_i8(
 ; SM90:       {
@@ -276,18 +276,18 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [release_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [release_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ld.param.b8 %r9, [release_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [release_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -304,7 +304,7 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    mov.b32 %r20, %r8;
 ; SM30-NEXT:    @%p2 bra $L__BB2_1;
 ; SM30-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: release_sys_i8(
@@ -318,18 +318,18 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_sys_i8_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -346,7 +346,7 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB2_1;
 ; SM70-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: release_sys_i8(
 ; SM90:       {
@@ -405,18 +405,18 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [acq_rel_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ld.param.b8 %r9, [acq_rel_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [acq_rel_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -434,7 +434,7 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    @%p2 bra $L__BB3_1;
 ; SM30-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acq_rel_sys_i8(
@@ -448,18 +448,18 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i8_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -477,7 +477,7 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB3_1;
 ; SM70-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acq_rel_sys_i8(
 ; SM90:       {
@@ -537,18 +537,18 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [seq_cst_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ld.param.b8 %r9, [seq_cst_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [seq_cst_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -566,7 +566,7 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    @%p2 bra $L__BB4_1;
 ; SM30-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: seq_cst_sys_i8(
@@ -580,18 +580,18 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i8_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -609,7 +609,7 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB4_1;
 ; SM70-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: seq_cst_sys_i8(
 ; SM90:       {
diff --git a/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll b/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll
index 8a0c0f8c3b452..a2fc8da3f1e61 100644
--- a/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll
@@ -24,11 +24,11 @@ define i16 @cvt_i16_i32(i32 %x) {
 define i16 @cvt_i16_i64(i64 %x) {
 ; CHECK-LABEL: cvt_i16_i64(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [cvt_i16_i64_param_0];
-; CHECK-NEXT:    st.param.b32 [func_retval0], %rd1;
+; CHECK-NEXT:    ld.param.b16 %r1, [cvt_i16_i64_param_0];
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   %a = trunc i64 %x to i16
   ret i16 %a
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s.ll
index b27f3078300b1..e0aceaf0901c9 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-g2s.ll
@@ -55,11 +55,11 @@ define void @cp_async_bulk_tensor_g2s_tile_1d(ptr addrspace(7) %d, ptr addrspace
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3}], [%r2], %rs1;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3}], [%r2], %rs1, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 0, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 undef, i1 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 0)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.1d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i16 %mc, i64 %ch, i1 1, i1 1)
   ret void
@@ -106,11 +106,11 @@ define void @cp_async_bulk_tensor_g2s_tile_2d(ptr addrspace(7) %d, ptr addrspace
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4}], [%r2], %rs1;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4}], [%r2], %rs1, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 0, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 undef, i1 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 1, i1 0)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.2d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i16 %mc, i64 %ch, i1 1, i1 1)
   ret void
@@ -159,9 +159,9 @@ define void @cp_async_bulk_tensor_g2s_tile_3d(ptr addrspace(7) %d, ptr addrspace
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rs1;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], %rs1, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 0, i1 1)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %mc, i64 %ch, i1 1, i1 0)
 
@@ -214,9 +214,9 @@ define void @cp_async_bulk_tensor_g2s_tile_4d(ptr addrspace(7) %d, ptr addrspace
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rs1;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], %rs1, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 0, i1 1)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %mc, i64 %ch, i1 1, i1 0)
 
@@ -271,9 +271,9 @@ define void @cp_async_bulk_tensor_g2s_tile_5d(ptr addrspace(7) %d, ptr addrspace
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rs1;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], %rs1, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 0, i1 1)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.tile.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %mc, i64 %ch, i1 1, i1 0)
 
@@ -326,9 +326,9 @@ define void @cp_async_bulk_tensor_g2s_im2col_3d(ptr addrspace(7) %d, ptr addrspa
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rs2;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4, %r5}], [%r2], {%rs1}, %rs2, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 0, i1 1)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i16 %mc, i64 %ch, i1 1, i1 0)
 
@@ -385,9 +385,9 @@ define void @cp_async_bulk_tensor_g2s_im2col_4d(ptr addrspace(7) %d, ptr addrspa
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rs3;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4, %r5, %r6}], [%r2], {%rs1, %rs2}, %rs3, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 0, i1 1)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 1, i1 0)
 
@@ -448,9 +448,9 @@ define void @cp_async_bulk_tensor_g2s_im2col_5d(ptr addrspace(7) %d, ptr addrspa
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rs4;
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.shared::cluster.global.im2col.mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint [%r1], [%rd1, {%r3, %r4, %r5, %r6, %r7}], [%r2], {%rs1, %rs2, %rs3}, %rs4, %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 undef, i64 undef, i1 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 0, i1 0)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 undef, i64 %ch, i1 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 0, i1 1)
 
   tail call void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(7) %d, ptr addrspace(3) %bar, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 1, i1 0)
 
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch.ll
index c32c5559b1591..09dbe91d07513 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-prefetch.ll
@@ -28,7 +28,7 @@ define void @cp_async_bulk_tensor_prefetch_tile_1d(ptr %tmap, i32 %d0, i64 %ch)
 ; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_prefetch_tile_1d_param_2];
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.1d.L2.global.tile.L2::cache_hint [%rd1, {%r1}], %rd2;
 ; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %tmap, i32 %d0, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %tmap, i32 %d0, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %tmap, i32 %d0, i64 %ch, i1 1)
   ret void
 }
@@ -48,7 +48,7 @@ define void @cp_async_bulk_tensor_prefetch_tile_2d(i32 %flag, ptr %tmap, i32 %d0
 ; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_prefetch_tile_2d_param_4];
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.2d.L2.global.tile.L2::cache_hint [%rd1, {%r1, %r2}], %rd2;
 ; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
   ret void
 }
@@ -73,10 +73,10 @@ define void @cp_async_bulk_tensor_prefetch_3d(i32 %flag, ptr %tmap, i32 %d0, i32
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.3d.L2.global.im2col [%rd1, {%r1, %r2, %r3}], {%rs1};
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.3d.L2.global.im2col.L2::cache_hint [%rd1, {%r1, %r2, %r3}], {%rs1}, %rd2;
 ; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i16 %im2col0, i64 %ch, i1 1)
   ret void
 }
@@ -103,10 +103,10 @@ define void @cp_async_bulk_tensor_prefetch_4d(i32 %flag, ptr %tmap, i32 %d0, i32
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.4d.L2.global.im2col [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2};
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.4d.L2.global.im2col.L2::cache_hint [%rd1, {%r1, %r2, %r3, %r4}], {%rs1, %rs2}, %rd2;
 ; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i64 %ch, i1 1)
   ret void
 }
@@ -135,10 +135,10 @@ define void @cp_async_bulk_tensor_prefetch_5d(i32 %flag, ptr %tmap, i32 %d0, i32
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.5d.L2.global.im2col [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3};
 ; CHECK-PTX-NEXT:    cp.async.bulk.prefetch.tensor.5d.L2.global.im2col.L2::cache_hint [%rd1, {%r1, %r2, %r3, %r4, %r5}], {%rs1, %rs2, %rs3}, %rd2;
 ; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i64 %ch, i1 1)
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
index b73631d219ba1..4045b8b2792ee 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
@@ -52,14 +52,14 @@ define void @cp_async_bulk_tensor_reduce_tile_1d(ptr addrspace(3) %src, ptr %tma
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
   ret void
 }
 
@@ -102,14 +102,14 @@ define void @cp_async_bulk_tensor_reduce_tile_2d(ptr addrspace(3) %src, ptr %tma
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
   ret void
 }
 
@@ -153,14 +153,14 @@ define void @cp_async_bulk_tensor_reduce_tile_3d(ptr addrspace(3) %src, ptr %tma
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
   ret void
 }
 
@@ -205,14 +205,14 @@ define void @cp_async_bulk_tensor_reduce_tile_4d(ptr addrspace(3) %src, ptr %tma
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
   ret void
 }
 
@@ -258,14 +258,14 @@ define void @cp_async_bulk_tensor_reduce_tile_5d(ptr addrspace(3) %src, ptr %tma
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
   ret void
 }
 
@@ -309,14 +309,14 @@ define void @cp_async_bulk_tensor_reduce_im2col_3d(ptr addrspace(3) %src, ptr %t
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
   ret void
 }
 
@@ -361,14 +361,14 @@ define void @cp_async_bulk_tensor_reduce_im2col_4d(ptr addrspace(3) %src, ptr %t
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
   ret void
 }
 
@@ -414,13 +414,13 @@ define void @cp_async_bulk_tensor_reduce_im2col_5d(ptr addrspace(3) %src, ptr %t
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g.ll
index 6a366f658c777..5998883f77ac1 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-s2g.ll
@@ -45,7 +45,7 @@ define void @cp_async_bulk_tensor_s2g_tile_1d(ptr addrspace(3) %src, ptr %tmap,
 ; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_1d_param_3];
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group.L2::cache_hint [%rd1, {%r2}], [%r1], %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
   ret void
 }
@@ -81,7 +81,7 @@ define void @cp_async_bulk_tensor_s2g_tile_2d(i32 %flag, ptr addrspace(3) %src,
 ; CHECK-PTX-SHARED32-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_s2g_tile_2d_param_5];
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group.L2::cache_hint [%rd1, {%r2, %r3}], [%r1], %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
   ret void
 }
@@ -123,10 +123,10 @@ define void @cp_async_bulk_tensor_s2g_3d(i32 %flag, ptr addrspace(3) %src, ptr %
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs.bulk_group [%rd1, {%r2, %r3, %r4}], [%r1];
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.3d.global.shared::cta.im2col_no_offs.bulk_group.L2::cache_hint [%rd1, {%r2, %r3, %r4}], [%r1], %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
   ret void
 }
@@ -170,10 +170,10 @@ define void @cp_async_bulk_tensor_s2g_4d(i32 %flag, ptr addrspace(3) %src, ptr %
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs.bulk_group [%rd1, {%r2, %r3, %r4, %r5}], [%r1];
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.4d.global.shared::cta.im2col_no_offs.bulk_group.L2::cache_hint [%rd1, {%r2, %r3, %r4, %r5}], [%r1], %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
   ret void
 }
@@ -219,10 +219,10 @@ define void @cp_async_bulk_tensor_s2g_5d(i32 %flag, ptr addrspace(3) %src, ptr %
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs.bulk_group [%rd1, {%r2, %r3, %r4, %r5, %r6}], [%r1];
 ; CHECK-PTX-SHARED32-NEXT:    cp.async.bulk.tensor.5d.global.shared::cta.im2col_no_offs.bulk_group.L2::cache_hint [%rd1, {%r2, %r3, %r4, %r5, %r6}], [%r1], %rd2;
 ; CHECK-PTX-SHARED32-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 undef, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
   tail call void @llvm.nvvm.cp.async.bulk.tensor.s2g.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/extractelement.ll b/llvm/test/CodeGen/NVPTX/extractelement.ll
index b1eadf381d3b4..f37777ab954e2 100644
--- a/llvm/test/CodeGen/NVPTX/extractelement.ll
+++ b/llvm/test/CodeGen/NVPTX/extractelement.ll
@@ -158,27 +158,24 @@ define i16  @test_v8i8(i64 %a) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<16>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_v8i8_param_0];
-; CHECK-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r1}, %rd1; }
-; CHECK-NEXT:    cvt.u32.u64 %r2, %rd1;
-; CHECK-NEXT:    bfe.s32 %r3, %r2, 0, 8;
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_v8i8_param_0];
+; CHECK-NEXT:    bfe.s32 %r3, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs1, %r3;
-; CHECK-NEXT:    bfe.s32 %r4, %r2, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs2, %r4;
-; CHECK-NEXT:    bfe.s32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r5, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs3, %r5;
-; CHECK-NEXT:    bfe.s32 %r6, %r2, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 24, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs4, %r6;
-; CHECK-NEXT:    bfe.s32 %r7, %r1, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r7, %r2, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs5, %r7;
-; CHECK-NEXT:    bfe.s32 %r8, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r8, %r2, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs6, %r8;
-; CHECK-NEXT:    bfe.s32 %r9, %r1, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r9, %r2, 16, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs7, %r9;
-; CHECK-NEXT:    bfe.s32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r2, 24, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs8, %r10;
 ; CHECK-NEXT:    add.s16 %rs9, %rs1, %rs2;
 ; CHECK-NEXT:    add.s16 %rs10, %rs3, %rs4;
diff --git a/llvm/test/CodeGen/NVPTX/frem.ll b/llvm/test/CodeGen/NVPTX/frem.ll
index 909f2534f8219..5805aed1bebe6 100644
--- a/llvm/test/CodeGen/NVPTX/frem.ll
+++ b/llvm/test/CodeGen/NVPTX/frem.ll
@@ -222,25 +222,52 @@ define double @frem_f64_ninf(double %a, double %b) {
   ret double %r
 }
 
-define float @frem_f32_imm1(float %a) {
-; FAST-LABEL: frem_f32_imm1(
+define float @frem_f32_imm1_fast(float %a) {
+; FAST-LABEL: frem_f32_imm1_fast(
 ; FAST:       {
 ; FAST-NEXT:    .reg .b32 %r<5>;
 ; FAST-EMPTY:
 ; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_param_0];
+; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
 ; FAST-NEXT:    mul.f32 %r2, %r1, 0f3E124925;
 ; FAST-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
 ; FAST-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
 ; FAST-NEXT:    st.param.b32 [func_retval0], %r4;
 ; FAST-NEXT:    ret;
 ;
-; NORMAL-LABEL: frem_f32_imm1(
+; NORMAL-LABEL: frem_f32_imm1_fast(
 ; NORMAL:       {
 ; NORMAL-NEXT:    .reg .b32 %r<5>;
 ; NORMAL-EMPTY:
 ; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_param_0];
+; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
+; NORMAL-NEXT:    mul.rn.f32 %r2, %r1, 0f3E124925;
+; NORMAL-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; NORMAL-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
+; NORMAL-NEXT:    st.param.b32 [func_retval0], %r4;
+; NORMAL-NEXT:    ret;
+  %r = frem arcp float %a, 7.0
+  ret float %r
+}
+define float @frem_f32_imm1_normal(float %a) {
+; FAST-LABEL: frem_f32_imm1_normal(
+; FAST:       {
+; FAST-NEXT:    .reg .b32 %r<5>;
+; FAST-EMPTY:
+; FAST-NEXT:  // %bb.0:
+; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
+; FAST-NEXT:    div.approx.f32 %r2, %r1, 0f40E00000;
+; FAST-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; FAST-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
+; FAST-NEXT:    st.param.b32 [func_retval0], %r4;
+; FAST-NEXT:    ret;
+;
+; NORMAL-LABEL: frem_f32_imm1_normal(
+; NORMAL:       {
+; NORMAL-NEXT:    .reg .b32 %r<5>;
+; NORMAL-EMPTY:
+; NORMAL-NEXT:  // %bb.0:
+; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
 ; NORMAL-NEXT:    div.rn.f32 %r2, %r1, 0f40E00000;
 ; NORMAL-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
 ; NORMAL-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
diff --git a/llvm/test/CodeGen/NVPTX/i1-select.ll b/llvm/test/CodeGen/NVPTX/i1-select.ll
index 6fb5aad4b1eb9..562c746200d87 100644
--- a/llvm/test/CodeGen/NVPTX/i1-select.ll
+++ b/llvm/test/CodeGen/NVPTX/i1-select.ll
@@ -94,27 +94,27 @@ define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %fals
 define i32 @test_select_i1_basic_folding(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %false) {
 ; CHECK-LABEL: test_select_i1_basic_folding(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<13>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .pred %p<12>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_select_i1_basic_folding_param_0];
 ; CHECK-NEXT:    setp.eq.s32 %p1, %r1, 0;
-; CHECK-NEXT:    ld.param.b32 %r2, [test_select_i1_basic_folding_param_1];
-; CHECK-NEXT:    setp.ne.s32 %p2, %r2, 0;
-; CHECK-NEXT:    setp.eq.s32 %p3, %r2, 0;
-; CHECK-NEXT:    ld.param.b32 %r3, [test_select_i1_basic_folding_param_2];
-; CHECK-NEXT:    setp.eq.s32 %p4, %r3, 0;
-; CHECK-NEXT:    ld.param.b32 %r4, [test_select_i1_basic_folding_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [test_select_i1_basic_folding_param_1];
+; CHECK-NEXT:    setp.ne.s32 %p2, %r3, 0;
+; CHECK-NEXT:    setp.eq.s32 %p3, %r3, 0;
+; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_folding_param_2];
+; CHECK-NEXT:    setp.eq.s32 %p4, %r5, 0;
+; CHECK-NEXT:    ld.param.b32 %r6, [test_select_i1_basic_folding_param_3];
 ; CHECK-NEXT:    xor.pred %p6, %p1, %p3;
-; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_folding_param_4];
+; CHECK-NEXT:    ld.param.b32 %r7, [test_select_i1_basic_folding_param_4];
 ; CHECK-NEXT:    and.pred %p7, %p6, %p4;
-; CHECK-NEXT:    and.pred %p9, %p2, %p4;
-; CHECK-NEXT:    and.pred %p10, %p3, %p7;
-; CHECK-NEXT:    or.pred %p11, %p10, %p9;
-; CHECK-NEXT:    xor.pred %p12, %p11, %p3;
-; CHECK-NEXT:    selp.b32 %r6, %r4, %r5, %p12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    and.pred %p8, %p2, %p4;
+; CHECK-NEXT:    and.pred %p9, %p3, %p7;
+; CHECK-NEXT:    or.pred %p10, %p9, %p8;
+; CHECK-NEXT:    xor.pred %p11, %p10, %p3;
+; CHECK-NEXT:    selp.b32 %r8, %r6, %r7, %p11;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-NEXT:    ret;
   %b1 = icmp eq i32 %v1, 0
   %b2 = icmp eq i32 %v2, 0
diff --git a/llvm/test/CodeGen/NVPTX/i128.ll b/llvm/test/CodeGen/NVPTX/i128.ll
index 29408a24213cc..49dbc5c385dc0 100644
--- a/llvm/test/CodeGen/NVPTX/i128.ll
+++ b/llvm/test/CodeGen/NVPTX/i128.ll
@@ -5,9 +5,9 @@
 define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: srem_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<20>;
+; CHECK-NEXT:    .reg .pred %p<22>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<127>;
+; CHECK-NEXT:    .reg .b64 %rd<126>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd45, %rd46}, [srem_i128_param_0];
@@ -42,102 +42,103 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd62, %r4;
 ; CHECK-NEXT:    add.s64 %rd63, %rd62, 64;
 ; CHECK-NEXT:    selp.b64 %rd64, %rd61, %rd63, %p7;
-; CHECK-NEXT:    mov.b64 %rd117, 0;
+; CHECK-NEXT:    mov.b64 %rd116, 0;
 ; CHECK-NEXT:    sub.cc.s64 %rd66, %rd60, %rd64;
-; CHECK-NEXT:    subc.cc.s64 %rd67, %rd117, 0;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd66, 127;
-; CHECK-NEXT:    setp.eq.s64 %p9, %rd67, 0;
-; CHECK-NEXT:    and.pred %p10, %p9, %p8;
-; CHECK-NEXT:    setp.ne.s64 %p11, %rd67, 0;
-; CHECK-NEXT:    or.pred %p12, %p10, %p11;
-; CHECK-NEXT:    or.pred %p13, %p5, %p12;
-; CHECK-NEXT:    xor.b64 %rd68, %rd66, 127;
-; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd67;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd69, 0;
-; CHECK-NEXT:    selp.b64 %rd126, 0, %rd4, %p13;
-; CHECK-NEXT:    selp.b64 %rd125, 0, %rd3, %p13;
-; CHECK-NEXT:    or.pred %p15, %p13, %p14;
-; CHECK-NEXT:    @%p15 bra $L__BB0_5;
+; CHECK-NEXT:    subc.cc.s64 %rd8, %rd116, 0;
+; CHECK-NEXT:    setp.ne.s64 %p8, %rd8, 0;
+; CHECK-NEXT:    and.pred %p10, %p8, %p8;
+; CHECK-NEXT:    setp.eq.s64 %p11, %rd8, 0;
+; CHECK-NEXT:    setp.gt.u64 %p12, %rd66, 127;
+; CHECK-NEXT:    and.pred %p13, %p11, %p12;
+; CHECK-NEXT:    or.pred %p14, %p13, %p10;
+; CHECK-NEXT:    or.pred %p15, %p5, %p14;
+; CHECK-NEXT:    xor.b64 %rd67, %rd66, 127;
+; CHECK-NEXT:    or.b64 %rd68, %rd67, %rd8;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd68, 0;
+; CHECK-NEXT:    selp.b64 %rd125, 0, %rd4, %p15;
+; CHECK-NEXT:    selp.b64 %rd124, 0, %rd3, %p15;
+; CHECK-NEXT:    or.pred %p17, %p15, %p16;
+; CHECK-NEXT:    @%p17 bra $L__BB0_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd66, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd67, 0;
-; CHECK-NEXT:    or.b64 %rd72, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd72, 0;
+; CHECK-NEXT:    add.cc.s64 %rd118, %rd66, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd119, %rd8, 0;
+; CHECK-NEXT:    or.b64 %rd71, %rd118, %rd119;
+; CHECK-NEXT:    setp.eq.s64 %p18, %rd71, 0;
 ; CHECK-NEXT:    cvt.u32.u64 %r5, %rd66;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd73, %rd4, %r6;
+; CHECK-NEXT:    shl.b64 %rd72, %rd4, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r7;
-; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
+; CHECK-NEXT:    shr.u64 %rd73, %rd3, %r7;
+; CHECK-NEXT:    or.b64 %rd74, %rd72, %rd73;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd76, %rd3, %r8;
-; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd124, %rd76, %rd75, %p17;
-; CHECK-NEXT:    shl.b64 %rd123, %rd3, %r6;
-; CHECK-NEXT:    mov.b64 %rd114, %rd117;
-; CHECK-NEXT:    @%p16 bra $L__BB0_4;
+; CHECK-NEXT:    shl.b64 %rd75, %rd3, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p19, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd123, %rd75, %rd74, %p19;
+; CHECK-NEXT:    shl.b64 %rd122, %rd3, %r6;
+; CHECK-NEXT:    mov.b64 %rd113, %rd116;
+; CHECK-NEXT:    @%p18 bra $L__BB0_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd119;
-; CHECK-NEXT:    shr.u64 %rd79, %rd3, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd118;
+; CHECK-NEXT:    shr.u64 %rd78, %rd3, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd80, %rd4, %r10;
-; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
+; CHECK-NEXT:    shl.b64 %rd79, %rd4, %r10;
+; CHECK-NEXT:    or.b64 %rd80, %rd78, %rd79;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd82, %rd4, %r11;
-; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd121, %rd82, %rd81, %p18;
-; CHECK-NEXT:    shr.u64 %rd122, %rd4, %r9;
+; CHECK-NEXT:    shr.u64 %rd81, %rd4, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p20, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd120, %rd81, %rd80, %p20;
+; CHECK-NEXT:    shr.u64 %rd121, %rd4, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd5, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd6, -1;
-; CHECK-NEXT:    mov.b64 %rd114, 0;
-; CHECK-NEXT:    mov.b64 %rd117, %rd114;
+; CHECK-NEXT:    mov.b64 %rd113, 0;
+; CHECK-NEXT:    mov.b64 %rd116, %rd113;
 ; CHECK-NEXT:  $L__BB0_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd83, %rd121, 63;
-; CHECK-NEXT:    shl.b64 %rd84, %rd122, 1;
-; CHECK-NEXT:    or.b64 %rd85, %rd84, %rd83;
-; CHECK-NEXT:    shl.b64 %rd86, %rd121, 1;
-; CHECK-NEXT:    shr.u64 %rd87, %rd124, 63;
-; CHECK-NEXT:    or.b64 %rd88, %rd86, %rd87;
-; CHECK-NEXT:    shr.u64 %rd89, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd123, %rd117, %rd92;
-; CHECK-NEXT:    or.b64 %rd124, %rd114, %rd91;
-; CHECK-NEXT:    sub.cc.s64 %rd93, %rd35, %rd88;
-; CHECK-NEXT:    subc.cc.s64 %rd94, %rd36, %rd85;
-; CHECK-NEXT:    shr.s64 %rd95, %rd94, 63;
-; CHECK-NEXT:    and.b64 %rd117, %rd95, 1;
-; CHECK-NEXT:    and.b64 %rd96, %rd95, %rd5;
-; CHECK-NEXT:    and.b64 %rd97, %rd95, %rd6;
-; CHECK-NEXT:    sub.cc.s64 %rd121, %rd88, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd122, %rd85, %rd97;
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd119, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd120, -1;
-; CHECK-NEXT:    or.b64 %rd98, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p19, %rd98, 0;
-; CHECK-NEXT:    @%p19 bra $L__BB0_4;
+; CHECK-NEXT:    shr.u64 %rd82, %rd120, 63;
+; CHECK-NEXT:    shl.b64 %rd83, %rd121, 1;
+; CHECK-NEXT:    or.b64 %rd84, %rd83, %rd82;
+; CHECK-NEXT:    shl.b64 %rd85, %rd120, 1;
+; CHECK-NEXT:    shr.u64 %rd86, %rd123, 63;
+; CHECK-NEXT:    or.b64 %rd87, %rd85, %rd86;
+; CHECK-NEXT:    shr.u64 %rd88, %rd122, 63;
+; CHECK-NEXT:    shl.b64 %rd89, %rd123, 1;
+; CHECK-NEXT:    or.b64 %rd90, %rd89, %rd88;
+; CHECK-NEXT:    shl.b64 %rd91, %rd122, 1;
+; CHECK-NEXT:    or.b64 %rd122, %rd116, %rd91;
+; CHECK-NEXT:    or.b64 %rd123, %rd113, %rd90;
+; CHECK-NEXT:    sub.cc.s64 %rd92, %rd35, %rd87;
+; CHECK-NEXT:    subc.cc.s64 %rd93, %rd36, %rd84;
+; CHECK-NEXT:    shr.s64 %rd94, %rd93, 63;
+; CHECK-NEXT:    and.b64 %rd116, %rd94, 1;
+; CHECK-NEXT:    and.b64 %rd95, %rd94, %rd5;
+; CHECK-NEXT:    and.b64 %rd96, %rd94, %rd6;
+; CHECK-NEXT:    sub.cc.s64 %rd120, %rd87, %rd95;
+; CHECK-NEXT:    subc.cc.s64 %rd121, %rd84, %rd96;
+; CHECK-NEXT:    add.cc.s64 %rd118, %rd118, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd119, %rd119, -1;
+; CHECK-NEXT:    or.b64 %rd97, %rd118, %rd119;
+; CHECK-NEXT:    setp.eq.s64 %p21, %rd97, 0;
+; CHECK-NEXT:    @%p21 bra $L__BB0_4;
 ; CHECK-NEXT:    bra.uni $L__BB0_2;
 ; CHECK-NEXT:  $L__BB0_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd99, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd100, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd101, %rd100, %rd99;
-; CHECK-NEXT:    shl.b64 %rd102, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd125, %rd117, %rd102;
-; CHECK-NEXT:    or.b64 %rd126, %rd114, %rd101;
+; CHECK-NEXT:    shr.u64 %rd98, %rd122, 63;
+; CHECK-NEXT:    shl.b64 %rd99, %rd123, 1;
+; CHECK-NEXT:    or.b64 %rd100, %rd99, %rd98;
+; CHECK-NEXT:    shl.b64 %rd101, %rd122, 1;
+; CHECK-NEXT:    or.b64 %rd124, %rd116, %rd101;
+; CHECK-NEXT:    or.b64 %rd125, %rd113, %rd100;
 ; CHECK-NEXT:  $L__BB0_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd103, %rd5, %rd125;
-; CHECK-NEXT:    mad.lo.s64 %rd104, %rd5, %rd126, %rd103;
-; CHECK-NEXT:    mad.lo.s64 %rd105, %rd6, %rd125, %rd104;
-; CHECK-NEXT:    mul.lo.s64 %rd106, %rd5, %rd125;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd3, %rd106;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd4, %rd105;
+; CHECK-NEXT:    mul.hi.u64 %rd102, %rd5, %rd124;
+; CHECK-NEXT:    mad.lo.s64 %rd103, %rd5, %rd125, %rd102;
+; CHECK-NEXT:    mad.lo.s64 %rd104, %rd6, %rd124, %rd103;
+; CHECK-NEXT:    mul.lo.s64 %rd105, %rd5, %rd124;
+; CHECK-NEXT:    sub.cc.s64 %rd106, %rd3, %rd105;
+; CHECK-NEXT:    subc.cc.s64 %rd107, %rd4, %rd104;
+; CHECK-NEXT:    xor.b64 %rd108, %rd106, %rd2;
 ; CHECK-NEXT:    xor.b64 %rd109, %rd107, %rd2;
-; CHECK-NEXT:    xor.b64 %rd110, %rd108, %rd2;
-; CHECK-NEXT:    sub.cc.s64 %rd111, %rd109, %rd2;
-; CHECK-NEXT:    subc.cc.s64 %rd112, %rd110, %rd2;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd111, %rd112};
+; CHECK-NEXT:    sub.cc.s64 %rd110, %rd108, %rd2;
+; CHECK-NEXT:    subc.cc.s64 %rd111, %rd109, %rd2;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd110, %rd111};
 ; CHECK-NEXT:    ret;
   %div = srem i128 %lhs, %rhs
   ret i128 %div
@@ -148,7 +149,7 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<18>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<113>;
+; CHECK-NEXT:    .reg .b64 %rd<111>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd41, %rd42}, [urem_i128_param_0];
@@ -172,98 +173,98 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
 ; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
 ; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.b64 %rd103, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd57, %rd103, 0;
-; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
-; CHECK-NEXT:    setp.eq.s64 %p7, %rd57, 0;
+; CHECK-NEXT:    mov.b64 %rd101, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd5, %rd50, %rd54;
+; CHECK-NEXT:    subc.cc.s64 %rd6, %rd101, 0;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd5, 127;
+; CHECK-NEXT:    setp.eq.s64 %p7, %rd6, 0;
 ; CHECK-NEXT:    and.pred %p8, %p7, %p6;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd57, 0;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd6, 0;
 ; CHECK-NEXT:    or.pred %p10, %p8, %p9;
 ; CHECK-NEXT:    or.pred %p11, %p3, %p10;
-; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
-; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.s64 %p12, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd112, 0, %rd42, %p11;
-; CHECK-NEXT:    selp.b64 %rd111, 0, %rd41, %p11;
+; CHECK-NEXT:    xor.b64 %rd56, %rd5, 127;
+; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd6;
+; CHECK-NEXT:    setp.eq.s64 %p12, %rd57, 0;
+; CHECK-NEXT:    selp.b64 %rd110, 0, %rd42, %p11;
+; CHECK-NEXT:    selp.b64 %rd109, 0, %rd41, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p11, %p12;
 ; CHECK-NEXT:    @%p13 bra $L__BB1_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd56, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd57, 0;
-; CHECK-NEXT:    or.b64 %rd62, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    add.cc.s64 %rd103, %rd5, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd104, %rd6, 0;
+; CHECK-NEXT:    or.b64 %rd60, %rd103, %rd104;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd60, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd5;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    shl.b64 %rd61, %rd42, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
-; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
+; CHECK-NEXT:    shr.u64 %rd62, %rd41, %r7;
+; CHECK-NEXT:    or.b64 %rd63, %rd61, %rd62;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    shl.b64 %rd64, %rd41, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd110, %rd66, %rd65, %p15;
-; CHECK-NEXT:    shl.b64 %rd109, %rd41, %r6;
-; CHECK-NEXT:    mov.b64 %rd100, %rd103;
+; CHECK-NEXT:    selp.b64 %rd108, %rd64, %rd63, %p15;
+; CHECK-NEXT:    shl.b64 %rd107, %rd41, %r6;
+; CHECK-NEXT:    mov.b64 %rd98, %rd101;
 ; CHECK-NEXT:    @%p14 bra $L__BB1_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd105;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd103;
+; CHECK-NEXT:    shr.u64 %rd67, %rd41, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
-; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
+; CHECK-NEXT:    shl.b64 %rd68, %rd42, %r10;
+; CHECK-NEXT:    or.b64 %rd69, %rd67, %rd68;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    shr.u64 %rd70, %rd42, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd107, %rd72, %rd71, %p16;
-; CHECK-NEXT:    shr.u64 %rd108, %rd42, %r9;
+; CHECK-NEXT:    selp.b64 %rd105, %rd70, %rd69, %p16;
+; CHECK-NEXT:    shr.u64 %rd106, %rd42, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd4, -1;
-; CHECK-NEXT:    mov.b64 %rd100, 0;
-; CHECK-NEXT:    mov.b64 %rd103, %rd100;
+; CHECK-NEXT:    mov.b64 %rd98, 0;
+; CHECK-NEXT:    mov.b64 %rd101, %rd98;
 ; CHECK-NEXT:  $L__BB1_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd73, %rd107, 63;
-; CHECK-NEXT:    shl.b64 %rd74, %rd108, 1;
-; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
-; CHECK-NEXT:    shl.b64 %rd76, %rd107, 1;
-; CHECK-NEXT:    shr.u64 %rd77, %rd110, 63;
-; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
-; CHECK-NEXT:    shr.u64 %rd79, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd80, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
-; CHECK-NEXT:    shl.b64 %rd82, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd109, %rd103, %rd82;
-; CHECK-NEXT:    or.b64 %rd110, %rd100, %rd81;
-; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
-; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
-; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
-; CHECK-NEXT:    and.b64 %rd103, %rd85, 1;
-; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd3;
-; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd78, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd75, %rd87;
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd105, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd106, -1;
-; CHECK-NEXT:    or.b64 %rd88, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p17, %rd88, 0;
+; CHECK-NEXT:    shr.u64 %rd71, %rd105, 63;
+; CHECK-NEXT:    shl.b64 %rd72, %rd106, 1;
+; CHECK-NEXT:    or.b64 %rd73, %rd72, %rd71;
+; CHECK-NEXT:    shl.b64 %rd74, %rd105, 1;
+; CHECK-NEXT:    shr.u64 %rd75, %rd108, 63;
+; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd77, %rd107, 63;
+; CHECK-NEXT:    shl.b64 %rd78, %rd108, 1;
+; CHECK-NEXT:    or.b64 %rd79, %rd78, %rd77;
+; CHECK-NEXT:    shl.b64 %rd80, %rd107, 1;
+; CHECK-NEXT:    or.b64 %rd107, %rd101, %rd80;
+; CHECK-NEXT:    or.b64 %rd108, %rd98, %rd79;
+; CHECK-NEXT:    sub.cc.s64 %rd81, %rd33, %rd76;
+; CHECK-NEXT:    subc.cc.s64 %rd82, %rd34, %rd73;
+; CHECK-NEXT:    shr.s64 %rd83, %rd82, 63;
+; CHECK-NEXT:    and.b64 %rd101, %rd83, 1;
+; CHECK-NEXT:    and.b64 %rd84, %rd83, %rd3;
+; CHECK-NEXT:    and.b64 %rd85, %rd83, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd105, %rd76, %rd84;
+; CHECK-NEXT:    subc.cc.s64 %rd106, %rd73, %rd85;
+; CHECK-NEXT:    add.cc.s64 %rd103, %rd103, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd104, %rd104, -1;
+; CHECK-NEXT:    or.b64 %rd86, %rd103, %rd104;
+; CHECK-NEXT:    setp.eq.s64 %p17, %rd86, 0;
 ; CHECK-NEXT:    @%p17 bra $L__BB1_4;
 ; CHECK-NEXT:    bra.uni $L__BB1_2;
 ; CHECK-NEXT:  $L__BB1_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd89, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd111, %rd103, %rd92;
-; CHECK-NEXT:    or.b64 %rd112, %rd100, %rd91;
+; CHECK-NEXT:    shr.u64 %rd87, %rd107, 63;
+; CHECK-NEXT:    shl.b64 %rd88, %rd108, 1;
+; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
+; CHECK-NEXT:    shl.b64 %rd90, %rd107, 1;
+; CHECK-NEXT:    or.b64 %rd109, %rd101, %rd90;
+; CHECK-NEXT:    or.b64 %rd110, %rd98, %rd89;
 ; CHECK-NEXT:  $L__BB1_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd93, %rd3, %rd111;
-; CHECK-NEXT:    mad.lo.s64 %rd94, %rd3, %rd112, %rd93;
-; CHECK-NEXT:    mad.lo.s64 %rd95, %rd4, %rd111, %rd94;
-; CHECK-NEXT:    mul.lo.s64 %rd96, %rd3, %rd111;
-; CHECK-NEXT:    sub.cc.s64 %rd97, %rd41, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd98, %rd42, %rd95;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd97, %rd98};
+; CHECK-NEXT:    mul.hi.u64 %rd91, %rd3, %rd109;
+; CHECK-NEXT:    mad.lo.s64 %rd92, %rd3, %rd110, %rd91;
+; CHECK-NEXT:    mad.lo.s64 %rd93, %rd4, %rd109, %rd92;
+; CHECK-NEXT:    mul.lo.s64 %rd94, %rd3, %rd109;
+; CHECK-NEXT:    sub.cc.s64 %rd95, %rd41, %rd94;
+; CHECK-NEXT:    subc.cc.s64 %rd96, %rd42, %rd93;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd95, %rd96};
 ; CHECK-NEXT:    ret;
   %div = urem i128 %lhs, %rhs
   ret i128 %div
@@ -306,9 +307,9 @@ define i128 @urem_i128_pow2k(i128 %lhs) {
 define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: sdiv_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<20>;
+; CHECK-NEXT:    .reg .pred %p<22>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<122>;
+; CHECK-NEXT:    .reg .b64 %rd<121>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd45, %rd46}, [sdiv_i128_param_0];
@@ -344,96 +345,97 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd63, %r4;
 ; CHECK-NEXT:    add.s64 %rd64, %rd63, 64;
 ; CHECK-NEXT:    selp.b64 %rd65, %rd62, %rd64, %p7;
-; CHECK-NEXT:    mov.b64 %rd112, 0;
+; CHECK-NEXT:    mov.b64 %rd111, 0;
 ; CHECK-NEXT:    sub.cc.s64 %rd67, %rd61, %rd65;
-; CHECK-NEXT:    subc.cc.s64 %rd68, %rd112, 0;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd67, 127;
-; CHECK-NEXT:    setp.eq.s64 %p9, %rd68, 0;
-; CHECK-NEXT:    and.pred %p10, %p9, %p8;
-; CHECK-NEXT:    setp.ne.s64 %p11, %rd68, 0;
-; CHECK-NEXT:    or.pred %p12, %p10, %p11;
-; CHECK-NEXT:    or.pred %p13, %p5, %p12;
-; CHECK-NEXT:    xor.b64 %rd69, %rd67, 127;
-; CHECK-NEXT:    or.b64 %rd70, %rd69, %rd68;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd70, 0;
-; CHECK-NEXT:    selp.b64 %rd121, 0, %rd2, %p13;
-; CHECK-NEXT:    selp.b64 %rd120, 0, %rd1, %p13;
-; CHECK-NEXT:    or.pred %p15, %p13, %p14;
-; CHECK-NEXT:    @%p15 bra $L__BB4_5;
+; CHECK-NEXT:    subc.cc.s64 %rd8, %rd111, 0;
+; CHECK-NEXT:    setp.ne.s64 %p8, %rd8, 0;
+; CHECK-NEXT:    and.pred %p10, %p8, %p8;
+; CHECK-NEXT:    setp.eq.s64 %p11, %rd8, 0;
+; CHECK-NEXT:    setp.gt.u64 %p12, %rd67, 127;
+; CHECK-NEXT:    and.pred %p13, %p11, %p12;
+; CHECK-NEXT:    or.pred %p14, %p13, %p10;
+; CHECK-NEXT:    or.pred %p15, %p5, %p14;
+; CHECK-NEXT:    xor.b64 %rd68, %rd67, 127;
+; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd8;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd69, 0;
+; CHECK-NEXT:    selp.b64 %rd120, 0, %rd2, %p15;
+; CHECK-NEXT:    selp.b64 %rd119, 0, %rd1, %p15;
+; CHECK-NEXT:    or.pred %p17, %p15, %p16;
+; CHECK-NEXT:    @%p17 bra $L__BB4_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd114, %rd67, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd115, %rd68, 0;
-; CHECK-NEXT:    or.b64 %rd73, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd73, 0;
+; CHECK-NEXT:    add.cc.s64 %rd113, %rd67, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd114, %rd8, 0;
+; CHECK-NEXT:    or.b64 %rd72, %rd113, %rd114;
+; CHECK-NEXT:    setp.eq.s64 %p18, %rd72, 0;
 ; CHECK-NEXT:    cvt.u32.u64 %r5, %rd67;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd74, %rd2, %r6;
+; CHECK-NEXT:    shl.b64 %rd73, %rd2, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd75, %rd1, %r7;
-; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd74, %rd1, %r7;
+; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd77, %rd1, %r8;
-; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd119, %rd77, %rd76, %p17;
-; CHECK-NEXT:    shl.b64 %rd118, %rd1, %r6;
-; CHECK-NEXT:    mov.b64 %rd109, %rd112;
-; CHECK-NEXT:    @%p16 bra $L__BB4_4;
+; CHECK-NEXT:    shl.b64 %rd76, %rd1, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p19, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd118, %rd76, %rd75, %p19;
+; CHECK-NEXT:    shl.b64 %rd117, %rd1, %r6;
+; CHECK-NEXT:    mov.b64 %rd108, %rd111;
+; CHECK-NEXT:    @%p18 bra $L__BB4_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd114;
-; CHECK-NEXT:    shr.u64 %rd80, %rd1, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd113;
+; CHECK-NEXT:    shr.u64 %rd79, %rd1, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd81, %rd2, %r10;
-; CHECK-NEXT:    or.b64 %rd82, %rd80, %rd81;
+; CHECK-NEXT:    shl.b64 %rd80, %rd2, %r10;
+; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd83, %rd2, %r11;
-; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd116, %rd83, %rd82, %p18;
-; CHECK-NEXT:    shr.u64 %rd117, %rd2, %r9;
+; CHECK-NEXT:    shr.u64 %rd82, %rd2, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p20, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd115, %rd82, %rd81, %p20;
+; CHECK-NEXT:    shr.u64 %rd116, %rd2, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd4, -1;
-; CHECK-NEXT:    mov.b64 %rd109, 0;
-; CHECK-NEXT:    mov.b64 %rd112, %rd109;
+; CHECK-NEXT:    mov.b64 %rd108, 0;
+; CHECK-NEXT:    mov.b64 %rd111, %rd108;
 ; CHECK-NEXT:  $L__BB4_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd84, %rd116, 63;
-; CHECK-NEXT:    shl.b64 %rd85, %rd117, 1;
-; CHECK-NEXT:    or.b64 %rd86, %rd85, %rd84;
-; CHECK-NEXT:    shl.b64 %rd87, %rd116, 1;
-; CHECK-NEXT:    shr.u64 %rd88, %rd119, 63;
-; CHECK-NEXT:    or.b64 %rd89, %rd87, %rd88;
-; CHECK-NEXT:    shr.u64 %rd90, %rd118, 63;
-; CHECK-NEXT:    shl.b64 %rd91, %rd119, 1;
-; CHECK-NEXT:    or.b64 %rd92, %rd91, %rd90;
-; CHECK-NEXT:    shl.b64 %rd93, %rd118, 1;
-; CHECK-NEXT:    or.b64 %rd118, %rd112, %rd93;
-; CHECK-NEXT:    or.b64 %rd119, %rd109, %rd92;
-; CHECK-NEXT:    sub.cc.s64 %rd94, %rd35, %rd89;
-; CHECK-NEXT:    subc.cc.s64 %rd95, %rd36, %rd86;
-; CHECK-NEXT:    shr.s64 %rd96, %rd95, 63;
-; CHECK-NEXT:    and.b64 %rd112, %rd96, 1;
-; CHECK-NEXT:    and.b64 %rd97, %rd96, %rd3;
-; CHECK-NEXT:    and.b64 %rd98, %rd96, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd116, %rd89, %rd97;
-; CHECK-NEXT:    subc.cc.s64 %rd117, %rd86, %rd98;
-; CHECK-NEXT:    add.cc.s64 %rd114, %rd114, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd115, %rd115, -1;
-; CHECK-NEXT:    or.b64 %rd99, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.s64 %p19, %rd99, 0;
-; CHECK-NEXT:    @%p19 bra $L__BB4_4;
+; CHECK-NEXT:    shr.u64 %rd83, %rd115, 63;
+; CHECK-NEXT:    shl.b64 %rd84, %rd116, 1;
+; CHECK-NEXT:    or.b64 %rd85, %rd84, %rd83;
+; CHECK-NEXT:    shl.b64 %rd86, %rd115, 1;
+; CHECK-NEXT:    shr.u64 %rd87, %rd118, 63;
+; CHECK-NEXT:    or.b64 %rd88, %rd86, %rd87;
+; CHECK-NEXT:    shr.u64 %rd89, %rd117, 63;
+; CHECK-NEXT:    shl.b64 %rd90, %rd118, 1;
+; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
+; CHECK-NEXT:    shl.b64 %rd92, %rd117, 1;
+; CHECK-NEXT:    or.b64 %rd117, %rd111, %rd92;
+; CHECK-NEXT:    or.b64 %rd118, %rd108, %rd91;
+; CHECK-NEXT:    sub.cc.s64 %rd93, %rd35, %rd88;
+; CHECK-NEXT:    subc.cc.s64 %rd94, %rd36, %rd85;
+; CHECK-NEXT:    shr.s64 %rd95, %rd94, 63;
+; CHECK-NEXT:    and.b64 %rd111, %rd95, 1;
+; CHECK-NEXT:    and.b64 %rd96, %rd95, %rd3;
+; CHECK-NEXT:    and.b64 %rd97, %rd95, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd115, %rd88, %rd96;
+; CHECK-NEXT:    subc.cc.s64 %rd116, %rd85, %rd97;
+; CHECK-NEXT:    add.cc.s64 %rd113, %rd113, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd114, %rd114, -1;
+; CHECK-NEXT:    or.b64 %rd98, %rd113, %rd114;
+; CHECK-NEXT:    setp.eq.s64 %p21, %rd98, 0;
+; CHECK-NEXT:    @%p21 bra $L__BB4_4;
 ; CHECK-NEXT:    bra.uni $L__BB4_2;
 ; CHECK-NEXT:  $L__BB4_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd100, %rd118, 63;
-; CHECK-NEXT:    shl.b64 %rd101, %rd119, 1;
-; CHECK-NEXT:    or.b64 %rd102, %rd101, %rd100;
-; CHECK-NEXT:    shl.b64 %rd103, %rd118, 1;
-; CHECK-NEXT:    or.b64 %rd120, %rd112, %rd103;
-; CHECK-NEXT:    or.b64 %rd121, %rd109, %rd102;
+; CHECK-NEXT:    shr.u64 %rd99, %rd117, 63;
+; CHECK-NEXT:    shl.b64 %rd100, %rd118, 1;
+; CHECK-NEXT:    or.b64 %rd101, %rd100, %rd99;
+; CHECK-NEXT:    shl.b64 %rd102, %rd117, 1;
+; CHECK-NEXT:    or.b64 %rd119, %rd111, %rd102;
+; CHECK-NEXT:    or.b64 %rd120, %rd108, %rd101;
 ; CHECK-NEXT:  $L__BB4_5: // %udiv-end
+; CHECK-NEXT:    xor.b64 %rd103, %rd119, %rd5;
 ; CHECK-NEXT:    xor.b64 %rd104, %rd120, %rd5;
-; CHECK-NEXT:    xor.b64 %rd105, %rd121, %rd5;
-; CHECK-NEXT:    sub.cc.s64 %rd106, %rd104, %rd5;
-; CHECK-NEXT:    subc.cc.s64 %rd107, %rd105, %rd5;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd106, %rd107};
+; CHECK-NEXT:    sub.cc.s64 %rd105, %rd103, %rd5;
+; CHECK-NEXT:    subc.cc.s64 %rd106, %rd104, %rd5;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd105, %rd106};
 ; CHECK-NEXT:    ret;
   %div = sdiv i128 %lhs, %rhs
   ret i128 %div
@@ -444,7 +446,7 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<18>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<107>;
+; CHECK-NEXT:    .reg .b64 %rd<105>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd41, %rd42}, [udiv_i128_param_0];
@@ -468,92 +470,92 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
 ; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
 ; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.b64 %rd97, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd57, %rd97, 0;
-; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
-; CHECK-NEXT:    setp.eq.s64 %p7, %rd57, 0;
+; CHECK-NEXT:    mov.b64 %rd95, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd5, %rd50, %rd54;
+; CHECK-NEXT:    subc.cc.s64 %rd6, %rd95, 0;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd5, 127;
+; CHECK-NEXT:    setp.eq.s64 %p7, %rd6, 0;
 ; CHECK-NEXT:    and.pred %p8, %p7, %p6;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd57, 0;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd6, 0;
 ; CHECK-NEXT:    or.pred %p10, %p8, %p9;
 ; CHECK-NEXT:    or.pred %p11, %p3, %p10;
-; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
-; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.s64 %p12, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd106, 0, %rd42, %p11;
-; CHECK-NEXT:    selp.b64 %rd105, 0, %rd41, %p11;
+; CHECK-NEXT:    xor.b64 %rd56, %rd5, 127;
+; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd6;
+; CHECK-NEXT:    setp.eq.s64 %p12, %rd57, 0;
+; CHECK-NEXT:    selp.b64 %rd104, 0, %rd42, %p11;
+; CHECK-NEXT:    selp.b64 %rd103, 0, %rd41, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p11, %p12;
 ; CHECK-NEXT:    @%p13 bra $L__BB5_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd99, %rd56, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd100, %rd57, 0;
-; CHECK-NEXT:    or.b64 %rd62, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    add.cc.s64 %rd97, %rd5, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd98, %rd6, 0;
+; CHECK-NEXT:    or.b64 %rd60, %rd97, %rd98;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd60, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd5;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    shl.b64 %rd61, %rd42, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
-; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
+; CHECK-NEXT:    shr.u64 %rd62, %rd41, %r7;
+; CHECK-NEXT:    or.b64 %rd63, %rd61, %rd62;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    shl.b64 %rd64, %rd41, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd104, %rd66, %rd65, %p15;
-; CHECK-NEXT:    shl.b64 %rd103, %rd41, %r6;
-; CHECK-NEXT:    mov.b64 %rd94, %rd97;
+; CHECK-NEXT:    selp.b64 %rd102, %rd64, %rd63, %p15;
+; CHECK-NEXT:    shl.b64 %rd101, %rd41, %r6;
+; CHECK-NEXT:    mov.b64 %rd92, %rd95;
 ; CHECK-NEXT:    @%p14 bra $L__BB5_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd99;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd97;
+; CHECK-NEXT:    shr.u64 %rd67, %rd41, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
-; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
+; CHECK-NEXT:    shl.b64 %rd68, %rd42, %r10;
+; CHECK-NEXT:    or.b64 %rd69, %rd67, %rd68;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    shr.u64 %rd70, %rd42, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd101, %rd72, %rd71, %p16;
-; CHECK-NEXT:    shr.u64 %rd102, %rd42, %r9;
+; CHECK-NEXT:    selp.b64 %rd99, %rd70, %rd69, %p16;
+; CHECK-NEXT:    shr.u64 %rd100, %rd42, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd43, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd44, -1;
-; CHECK-NEXT:    mov.b64 %rd94, 0;
-; CHECK-NEXT:    mov.b64 %rd97, %rd94;
+; CHECK-NEXT:    mov.b64 %rd92, 0;
+; CHECK-NEXT:    mov.b64 %rd95, %rd92;
 ; CHECK-NEXT:  $L__BB5_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd73, %rd101, 63;
-; CHECK-NEXT:    shl.b64 %rd74, %rd102, 1;
-; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
-; CHECK-NEXT:    shl.b64 %rd76, %rd101, 1;
-; CHECK-NEXT:    shr.u64 %rd77, %rd104, 63;
-; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
-; CHECK-NEXT:    shr.u64 %rd79, %rd103, 63;
-; CHECK-NEXT:    shl.b64 %rd80, %rd104, 1;
-; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
-; CHECK-NEXT:    shl.b64 %rd82, %rd103, 1;
-; CHECK-NEXT:    or.b64 %rd103, %rd97, %rd82;
-; CHECK-NEXT:    or.b64 %rd104, %rd94, %rd81;
-; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
-; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
-; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
-; CHECK-NEXT:    and.b64 %rd97, %rd85, 1;
-; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd43;
-; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd44;
-; CHECK-NEXT:    sub.cc.s64 %rd101, %rd78, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd102, %rd75, %rd87;
-; CHECK-NEXT:    add.cc.s64 %rd99, %rd99, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd100, %rd100, -1;
-; CHECK-NEXT:    or.b64 %rd88, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.s64 %p17, %rd88, 0;
+; CHECK-NEXT:    shr.u64 %rd71, %rd99, 63;
+; CHECK-NEXT:    shl.b64 %rd72, %rd100, 1;
+; CHECK-NEXT:    or.b64 %rd73, %rd72, %rd71;
+; CHECK-NEXT:    shl.b64 %rd74, %rd99, 1;
+; CHECK-NEXT:    shr.u64 %rd75, %rd102, 63;
+; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd77, %rd101, 63;
+; CHECK-NEXT:    shl.b64 %rd78, %rd102, 1;
+; CHECK-NEXT:    or.b64 %rd79, %rd78, %rd77;
+; CHECK-NEXT:    shl.b64 %rd80, %rd101, 1;
+; CHECK-NEXT:    or.b64 %rd101, %rd95, %rd80;
+; CHECK-NEXT:    or.b64 %rd102, %rd92, %rd79;
+; CHECK-NEXT:    sub.cc.s64 %rd81, %rd33, %rd76;
+; CHECK-NEXT:    subc.cc.s64 %rd82, %rd34, %rd73;
+; CHECK-NEXT:    shr.s64 %rd83, %rd82, 63;
+; CHECK-NEXT:    and.b64 %rd95, %rd83, 1;
+; CHECK-NEXT:    and.b64 %rd84, %rd83, %rd43;
+; CHECK-NEXT:    and.b64 %rd85, %rd83, %rd44;
+; CHECK-NEXT:    sub.cc.s64 %rd99, %rd76, %rd84;
+; CHECK-NEXT:    subc.cc.s64 %rd100, %rd73, %rd85;
+; CHECK-NEXT:    add.cc.s64 %rd97, %rd97, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd98, %rd98, -1;
+; CHECK-NEXT:    or.b64 %rd86, %rd97, %rd98;
+; CHECK-NEXT:    setp.eq.s64 %p17, %rd86, 0;
 ; CHECK-NEXT:    @%p17 bra $L__BB5_4;
 ; CHECK-NEXT:    bra.uni $L__BB5_2;
 ; CHECK-NEXT:  $L__BB5_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd89, %rd103, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd104, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd103, 1;
-; CHECK-NEXT:    or.b64 %rd105, %rd97, %rd92;
-; CHECK-NEXT:    or.b64 %rd106, %rd94, %rd91;
+; CHECK-NEXT:    shr.u64 %rd87, %rd101, 63;
+; CHECK-NEXT:    shl.b64 %rd88, %rd102, 1;
+; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
+; CHECK-NEXT:    shl.b64 %rd90, %rd101, 1;
+; CHECK-NEXT:    or.b64 %rd103, %rd95, %rd90;
+; CHECK-NEXT:    or.b64 %rd104, %rd92, %rd89;
 ; CHECK-NEXT:  $L__BB5_5: // %udiv-end
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd105, %rd106};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd103, %rd104};
 ; CHECK-NEXT:    ret;
   %div = udiv i128 %lhs, %rhs
   ret i128 %div
diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
index 0a2cd81ac904c..321a6240df098 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
@@ -121,7 +121,7 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 %input, p
 define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; PTX-LABEL: grid_const_escape(
 ; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<3>;
+; PTX-NEXT:    .reg .b32 %r<2>;
 ; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -153,7 +153,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; PTX-NEXT:    .local .align 4 .b8 __local_depot4[4];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
-; PTX-NEXT:    .reg .b32 %r<4>;
+; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<8>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -255,7 +255,7 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4
 define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %output) {
 ; PTX-LABEL: grid_const_partial_escape(
 ; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<5>;
+; PTX-NEXT:    .reg .b32 %r<4>;
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -295,7 +295,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %ou
 define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) %input, ptr %output) {
 ; PTX-LABEL: grid_const_partial_escapemem(
 ; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<6>;
+; PTX-NEXT:    .reg .b32 %r<5>;
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
diff --git a/llvm/test/CodeGen/NVPTX/lower-args.ll b/llvm/test/CodeGen/NVPTX/lower-args.ll
index 6f334b075241b..c165de7ffff03 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args.ll
@@ -31,7 +31,7 @@ define void @load_alignment(ptr nocapture readonly byval(%class.outer) align 8 %
 ; PTX-LABEL: load_alignment(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<4>;
-; PTX-NEXT:    .reg .b64 %rd<8>;
+; PTX-NEXT:    .reg .b64 %rd<7>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
 ; PTX-NEXT:    mov.b64 %rd1, load_alignment_param_0;
@@ -76,7 +76,7 @@ define void @load_padding(ptr nocapture readonly byval(%class.padded) %arg) {
 ;
 ; PTX-LABEL: load_padding(
 ; PTX:       {
-; PTX-NEXT:    .reg .b64 %rd<5>;
+; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd1, load_padding_param_0;
diff --git a/llvm/test/CodeGen/NVPTX/misched_func_call.ll b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
index 2e9eb6913ac0e..8401f457418d1 100644
--- a/llvm/test/CodeGen/NVPTX/misched_func_call.ll
+++ b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
@@ -8,7 +8,7 @@ define ptx_kernel void @wombat(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-LABEL: wombat(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<11>;
-; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %bb
 ; CHECK-NEXT:    ld.param.b32 %r4, [wombat_param_2];
@@ -27,11 +27,11 @@ define ptx_kernel void @wombat(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-NEXT:    mul.lo.s32 %r7, %r10, %r3;
 ; CHECK-NEXT:    or.b32 %r8, %r4, %r7;
 ; CHECK-NEXT:    mul.lo.s32 %r9, %r2, %r8;
-; CHECK-NEXT:    cvt.rn.f64.s32 %rd3, %r9;
-; CHECK-NEXT:    cvt.rn.f64.u32 %rd4, %r10;
-; CHECK-NEXT:    add.rn.f64 %rd5, %rd4, %rd3;
-; CHECK-NEXT:    mov.b64 %rd6, 0;
-; CHECK-NEXT:    st.global.b64 [%rd6], %rd5;
+; CHECK-NEXT:    cvt.rn.f64.s32 %rd2, %r9;
+; CHECK-NEXT:    cvt.rn.f64.u32 %rd3, %r10;
+; CHECK-NEXT:    add.rn.f64 %rd4, %rd3, %rd2;
+; CHECK-NEXT:    mov.b64 %rd5, 0;
+; CHECK-NEXT:    st.global.b64 [%rd5], %rd4;
 ; CHECK-NEXT:    mov.b32 %r10, 1;
 ; CHECK-NEXT:    bra.uni $L__BB0_1;
 bb:
diff --git a/llvm/test/CodeGen/NVPTX/mulwide.ll b/llvm/test/CodeGen/NVPTX/mulwide.ll
index 7db7dec561992..666c7a160e1f0 100644
--- a/llvm/test/CodeGen/NVPTX/mulwide.ll
+++ b/llvm/test/CodeGen/NVPTX/mulwide.ll
@@ -1,111 +1,325 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O3 | FileCheck %s --check-prefix=OPT
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O0 | FileCheck %s --check-prefix=NOOPT
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O3 | %ptxas-verify %}
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O0 | %ptxas-verify %}
 
-; OPT-LABEL: @mulwide16
-; NOOPT-LABEL: @mulwide16
 define i32 @mulwide16(i16 %a, i16 %b) {
-; OPT: mul.wide.s16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwide16(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b16 %rs1, [mulwide16_param_0];
+; OPT-NEXT:    ld.param.b16 %rs2, [mulwide16_param_1];
+; OPT-NEXT:    mul.wide.s16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwide16(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b16 %rs2, [mulwide16_param_1];
+; NOOPT-NEXT:    ld.param.b16 %rs1, [mulwide16_param_0];
+; NOOPT-NEXT:    cvt.s32.s16 %r1, %rs1;
+; NOOPT-NEXT:    cvt.s32.s16 %r2, %rs2;
+; NOOPT-NEXT:    mul.lo.s32 %r3, %r1, %r2;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r3;
+; NOOPT-NEXT:    ret;
   %val0 = sext i16 %a to i32
   %val1 = sext i16 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwideu16
-; NOOPT-LABEL: @mulwideu16
 define i32 @mulwideu16(i16 %a, i16 %b) {
-; OPT: mul.wide.u16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwideu16(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b16 %rs1, [mulwideu16_param_0];
+; OPT-NEXT:    ld.param.b16 %rs2, [mulwideu16_param_1];
+; OPT-NEXT:    mul.wide.u16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu16(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b16 %rs2, [mulwideu16_param_1];
+; NOOPT-NEXT:    ld.param.b16 %rs1, [mulwideu16_param_0];
+; NOOPT-NEXT:    cvt.u32.u16 %r1, %rs1;
+; NOOPT-NEXT:    cvt.u32.u16 %r2, %rs2;
+; NOOPT-NEXT:    mul.lo.s32 %r3, %r1, %r2;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r3;
+; NOOPT-NEXT:    ret;
   %val0 = zext i16 %a to i32
   %val1 = zext i16 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwide8
-; NOOPT-LABEL: @mulwide8
 define i32 @mulwide8(i8 %a, i8 %b) {
-; OPT: mul.wide.s16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwide8(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.s8 %rs1, [mulwide8_param_0];
+; OPT-NEXT:    ld.param.s8 %rs2, [mulwide8_param_1];
+; OPT-NEXT:    mul.wide.s16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwide8(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwide8_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwide8_param_0];
+; NOOPT-NEXT:    cvt.u32.u16 %r1, %rs1;
+; NOOPT-NEXT:    cvt.s32.s8 %r2, %r1;
+; NOOPT-NEXT:    cvt.u32.u16 %r3, %rs2;
+; NOOPT-NEXT:    cvt.s32.s8 %r4, %r3;
+; NOOPT-NEXT:    mul.lo.s32 %r5, %r2, %r4;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r5;
+; NOOPT-NEXT:    ret;
   %val0 = sext i8 %a to i32
   %val1 = sext i8 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwideu8
-; NOOPT-LABEL: @mulwideu8
 define i32 @mulwideu8(i8 %a, i8 %b) {
-; OPT: mul.wide.u16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwideu8(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b8 %rs1, [mulwideu8_param_0];
+; OPT-NEXT:    ld.param.b8 %rs2, [mulwideu8_param_1];
+; OPT-NEXT:    mul.wide.u16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu8(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwideu8_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwideu8_param_0];
+; NOOPT-NEXT:    cvt.u32.u16 %r1, %rs1;
+; NOOPT-NEXT:    and.b32 %r2, %r1, 255;
+; NOOPT-NEXT:    cvt.u32.u16 %r3, %rs2;
+; NOOPT-NEXT:    and.b32 %r4, %r3, 255;
+; NOOPT-NEXT:    mul.lo.s32 %r5, %r2, %r4;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r5;
+; NOOPT-NEXT:    ret;
   %val0 = zext i8 %a to i32
   %val1 = zext i8 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwide32
-; NOOPT-LABEL: @mulwide32
 define i64 @mulwide32(i32 %a, i32 %b) {
-; OPT: mul.wide.s32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwide32(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b32 %r1, [mulwide32_param_0];
+; OPT-NEXT:    ld.param.b32 %r2, [mulwide32_param_1];
+; OPT-NEXT:    mul.wide.s32 %rd1, %r1, %r2;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwide32(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r2, [mulwide32_param_1];
+; NOOPT-NEXT:    ld.param.b32 %r1, [mulwide32_param_0];
+; NOOPT-NEXT:    cvt.s64.s32 %rd1, %r1;
+; NOOPT-NEXT:    cvt.s64.s32 %rd2, %r2;
+; NOOPT-NEXT:    mul.lo.s64 %rd3, %rd1, %rd2;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd3;
+; NOOPT-NEXT:    ret;
   %val0 = sext i32 %a to i64
   %val1 = sext i32 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @mulwideu32
-; NOOPT-LABEL: @mulwideu32
 define i64 @mulwideu32(i32 %a, i32 %b) {
-; OPT: mul.wide.u32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwideu32(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b32 %r1, [mulwideu32_param_0];
+; OPT-NEXT:    ld.param.b32 %r2, [mulwideu32_param_1];
+; OPT-NEXT:    mul.wide.u32 %rd1, %r1, %r2;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu32(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r2, [mulwideu32_param_1];
+; NOOPT-NEXT:    ld.param.b32 %r1, [mulwideu32_param_0];
+; NOOPT-NEXT:    cvt.u64.u32 %rd1, %r1;
+; NOOPT-NEXT:    cvt.u64.u32 %rd2, %r2;
+; NOOPT-NEXT:    mul.lo.s64 %rd3, %rd1, %rd2;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd3;
+; NOOPT-NEXT:    ret;
   %val0 = zext i32 %a to i64
   %val1 = zext i32 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @mulwideu7
-; NOOPT-LABEL: @mulwideu7
 define i64 @mulwideu7(i7 %a, i7 %b) {
-; OPT: mul.wide.u32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwideu7(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b8 %r1, [mulwideu7_param_0];
+; OPT-NEXT:    ld.param.b8 %r2, [mulwideu7_param_1];
+; OPT-NEXT:    mul.wide.u32 %rd1, %r1, %r2;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu7(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwideu7_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwideu7_param_0];
+; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs1;
+; NOOPT-NEXT:    and.b64 %rd2, %rd1, 127;
+; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs2;
+; NOOPT-NEXT:    and.b64 %rd4, %rd3, 127;
+; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd2, %rd4;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd5;
+; NOOPT-NEXT:    ret;
   %val0 = zext i7 %a to i64
   %val1 = zext i7 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @mulwides7
-; NOOPT-LABEL: @mulwides7
 define i64 @mulwides7(i7 %a, i7 %b) {
-; OPT: mul.wide.s32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwides7(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<5>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b8 %r1, [mulwides7_param_0];
+; OPT-NEXT:    bfe.s32 %r2, %r1, 0, 7;
+; OPT-NEXT:    ld.param.b8 %r3, [mulwides7_param_1];
+; OPT-NEXT:    bfe.s32 %r4, %r3, 0, 7;
+; OPT-NEXT:    mul.wide.s32 %rd1, %r2, %r4;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwides7(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwides7_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwides7_param_0];
+; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs1;
+; NOOPT-NEXT:    bfe.s64 %rd2, %rd1, 0, 7;
+; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs2;
+; NOOPT-NEXT:    bfe.s64 %rd4, %rd3, 0, 7;
+; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd2, %rd4;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd5;
+; NOOPT-NEXT:    ret;
   %val0 = sext i7 %a to i64
   %val1 = sext i7 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @shl30
-; NOOPT-LABEL: @shl30
 define i64 @shl30(i32 %a) {
-; OPT: mul.wide
-; NOOPT: shl.b64
+; OPT-LABEL: shl30(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b32 %r1, [shl30_param_0];
+; OPT-NEXT:    mul.wide.s32 %rd1, %r1, 1073741824;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: shl30(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<2>;
+; NOOPT-NEXT:    .reg .b64 %rd<3>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r1, [shl30_param_0];
+; NOOPT-NEXT:    cvt.s64.s32 %rd1, %r1;
+; NOOPT-NEXT:    shl.b64 %rd2, %rd1, 30;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd2;
+; NOOPT-NEXT:    ret;
   %conv = sext i32 %a to i64
   %shl = shl i64 %conv, 30
   ret i64 %shl
 }
 
-; OPT-LABEL: @shl31
-; NOOPT-LABEL: @shl31
 define i64 @shl31(i32 %a) {
-; OPT-NOT: mul.wide
-; NOOPT-NOT: mul.wide
+; OPT-LABEL: shl31(
+; OPT:       {
+; OPT-NEXT:    .reg .b64 %rd<3>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.s32 %rd1, [shl31_param_0];
+; OPT-NEXT:    shl.b64 %rd2, %rd1, 31;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd2;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: shl31(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<2>;
+; NOOPT-NEXT:    .reg .b64 %rd<3>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r1, [shl31_param_0];
+; NOOPT-NEXT:    cvt.s64.s32 %rd1, %r1;
+; NOOPT-NEXT:    shl.b64 %rd2, %rd1, 31;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd2;
+; NOOPT-NEXT:    ret;
   %conv = sext i32 %a to i64
   %shl = shl i64 %conv, 31
   ret i64 %shl
diff --git a/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll b/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll
index 3096b953e8d3a..160511387652c 100644
--- a/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll
+++ b/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=nvptx64 -verify-machineinstrs < %s | FileCheck %s
 ; RUN: %if ptxas %{ llc -mtriple=nvptx64 -verify-machineinstrs < %s | %ptxas-verify %}
 
@@ -10,9 +11,19 @@
 ; value will be identical regardless of the boolean representation.
 ; Check that the optimization triggers in this case.
 
-; CHECK-LABEL: @pow2_mask_cmp
-; CHECK: bfe.u32 {{%r[0-9]+}}, {{%r[0-9]+}}, 3, 1
 define i32 @pow2_mask_cmp(i32 %x) {
+; CHECK-LABEL: pow2_mask_cmp(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [pow2_mask_cmp_param_0];
+; CHECK-NEXT:    shr.u16 %rs2, %rs1, 3;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT:    and.b32 %r2, %r1, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %a = and i32 %x, 8
   %cmp = icmp ne i32 %a, 0
   %r = zext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
index a28d264cd8ec0..3989c8e32e458 100644
--- a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
@@ -43,7 +43,7 @@ define float @test_rsqrt_ftz(float %a) #0 #1 {
   ret float %ret
 }
 
-define double @test_rsqrt64(double %a) #0 {
+define double @test_rsqrt64(double %a) {
 ; CHECK-LABEL: test_rsqrt64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -54,12 +54,12 @@ define double @test_rsqrt64(double %a) #0 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
 ; There's no rsqrt.approx.ftz.f64 instruction; we just use the non-ftz version.
-define double @test_rsqrt64_ftz(double %a) #0 #1 {
+define double @test_rsqrt64_ftz(double %a) #1 {
 ; CHECK-LABEL: test_rsqrt64_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -70,7 +70,7 @@ define double @test_rsqrt64_ftz(double %a) #0 #1 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
@@ -229,7 +229,7 @@ define float @test_rsqrt32_refined(float %a) #0 #2 {
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv float 1.0, %val
+  %ret = fdiv arcp float 1.0, %val
   ret float %ret
 }
 
@@ -284,7 +284,7 @@ define double @test_rsqrt64_refined(double %a) #0 #2 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
@@ -341,7 +341,7 @@ define float @test_rsqrt32_refined_ftz(float %a) #0 #1 #2 {
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv float 1.0, %val
+  %ret = fdiv arcp float 1.0, %val
   ret float %ret
 }
 
@@ -396,7 +396,7 @@ define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
diff --git a/llvm/test/CodeGen/NVPTX/st-param-imm.ll b/llvm/test/CodeGen/NVPTX/st-param-imm.ll
index 50d3e8049a947..6aa111932a4a5 100644
--- a/llvm/test/CodeGen/NVPTX/st-param-imm.ll
+++ b/llvm/test/CodeGen/NVPTX/st-param-imm.ll
@@ -445,12 +445,12 @@ define void @st_param_v4_i8_irrr(i8 %b, i8 %c, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irrr_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irrr_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irrr_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_irrr_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_irrr_param_0];
 ; CHECK-NEXT:    { // callseq 24, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, %rs2, %rs3};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs3, %rs2, %rs1};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 24
 ; CHECK-NEXT:    ret;
@@ -467,12 +467,12 @@ define void @st_param_v4_i8_rirr(i8 %a, i8 %c, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rirr_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rirr_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rirr_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rirr_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rirr_param_0];
 ; CHECK-NEXT:    { // callseq 25, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, %rs2, %rs3};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs3, 2, %rs2, %rs1};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 25
 ; CHECK-NEXT:    ret;
@@ -489,12 +489,12 @@ define void @st_param_v4_i8_rrir(i8 %a, i8 %b, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrir_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrir_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrir_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrir_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrir_param_0];
 ; CHECK-NEXT:    { // callseq 26, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, %rs2, 3, %rs3};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs3, %rs2, 3, %rs1};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 26
 ; CHECK-NEXT:    ret;
@@ -511,12 +511,12 @@ define void @st_param_v4_i8_rrri(i8 %a, i8 %b, i8 %c) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrri_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrri_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrri_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrri_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrri_param_0];
 ; CHECK-NEXT:    { // callseq 27, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, %rs2, %rs3, 4};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs3, %rs2, %rs1, 4};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 27
 ; CHECK-NEXT:    ret;
@@ -533,11 +533,11 @@ define void @st_param_v4_i8_iirr(i8 %c, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_iirr_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_iirr_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_iirr_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_iirr_param_0];
 ; CHECK-NEXT:    { // callseq 28, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, %rs1, %rs2};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, %rs2, %rs1};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 28
 ; CHECK-NEXT:    ret;
@@ -554,11 +554,11 @@ define void @st_param_v4_i8_irir(i8 %b, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irir_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irir_param_0];
 ; CHECK-NEXT:    { // callseq 29, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, 3, %rs2};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs2, 3, %rs1};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 29
 ; CHECK-NEXT:    ret;
@@ -575,11 +575,11 @@ define void @st_param_v4_i8_irri(i8 %b, i8 %c) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irri_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irri_param_0];
 ; CHECK-NEXT:    { // callseq 30, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, %rs2, 4};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs2, %rs1, 4};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 30
 ; CHECK-NEXT:    ret;
@@ -596,11 +596,11 @@ define void @st_param_v4_i8_riir(i8 %a, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riir_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riir_param_0];
 ; CHECK-NEXT:    { // callseq 31, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, 3, %rs2};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs2, 2, 3, %rs1};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 31
 ; CHECK-NEXT:    ret;
@@ -617,11 +617,11 @@ define void @st_param_v4_i8_riri(i8 %a, i8 %c) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riri_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riri_param_0];
 ; CHECK-NEXT:    { // callseq 32, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, %rs2, 4};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs2, 2, %rs1, 4};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 32
 ; CHECK-NEXT:    ret;
@@ -638,11 +638,11 @@ define void @st_param_v4_i8_rrii(i8 %a, i8 %b) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrii_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrii_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrii_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrii_param_0];
 ; CHECK-NEXT:    { // callseq 33, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, %rs2, 3, 4};
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs2, %rs1, 3, 4};
 ; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 33
 ; CHECK-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll b/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
index e4e668018d872..87e46b1505e31 100644
--- a/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
@@ -103,16 +103,16 @@ define %s_i8i32p @test_s_i8i32p(%s_i8i32p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+8];
 ; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+16];
 ; CHECK-NEXT:    } // callseq 1
-; CHECK-NEXT:    cvt.u32.u16 %r16, %rs3;
-; CHECK-NEXT:    cvt.u32.u16 %r17, %rs4;
-; CHECK-NEXT:    cvt.u32.u16 %r18, %rs5;
-; CHECK-NEXT:    cvt.u32.u16 %r19, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r17, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r18, %rs6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r19;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r18;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r17;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r18;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r15;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8i32p @test_s_i8i32p(%s_i8i32p %a)
@@ -185,48 +185,48 @@ define %s_i8i64p @test_s_i8i64p(%s_i8i64p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs10, [retval0+16];
 ; CHECK-NEXT:    ld.param.b64 %rd32, [retval0+24];
 ; CHECK-NEXT:    } // callseq 2
-; CHECK-NEXT:    cvt.u64.u16 %rd35, %rs3;
+; CHECK-NEXT:    cvt.u64.u16 %rd33, %rs3;
+; CHECK-NEXT:    and.b64 %rd34, %rd33, 255;
+; CHECK-NEXT:    cvt.u64.u16 %rd35, %rs4;
 ; CHECK-NEXT:    and.b64 %rd36, %rd35, 255;
-; CHECK-NEXT:    cvt.u64.u16 %rd37, %rs4;
-; CHECK-NEXT:    and.b64 %rd38, %rd37, 255;
-; CHECK-NEXT:    shl.b64 %rd39, %rd38, 8;
-; CHECK-NEXT:    or.b64 %rd40, %rd36, %rd39;
-; CHECK-NEXT:    cvt.u64.u16 %rd41, %rs5;
-; CHECK-NEXT:    and.b64 %rd42, %rd41, 255;
-; CHECK-NEXT:    shl.b64 %rd43, %rd42, 16;
-; CHECK-NEXT:    or.b64 %rd44, %rd40, %rd43;
-; CHECK-NEXT:    cvt.u64.u16 %rd45, %rs6;
-; CHECK-NEXT:    and.b64 %rd46, %rd45, 255;
-; CHECK-NEXT:    shl.b64 %rd47, %rd46, 24;
-; CHECK-NEXT:    or.b64 %rd48, %rd44, %rd47;
-; CHECK-NEXT:    cvt.u64.u16 %rd49, %rs7;
-; CHECK-NEXT:    and.b64 %rd50, %rd49, 255;
-; CHECK-NEXT:    shl.b64 %rd51, %rd50, 32;
-; CHECK-NEXT:    or.b64 %rd52, %rd48, %rd51;
-; CHECK-NEXT:    cvt.u64.u16 %rd53, %rs8;
-; CHECK-NEXT:    and.b64 %rd54, %rd53, 255;
-; CHECK-NEXT:    shl.b64 %rd55, %rd54, 40;
-; CHECK-NEXT:    or.b64 %rd56, %rd52, %rd55;
-; CHECK-NEXT:    cvt.u64.u16 %rd57, %rs9;
-; CHECK-NEXT:    and.b64 %rd58, %rd57, 255;
-; CHECK-NEXT:    shl.b64 %rd59, %rd58, 48;
-; CHECK-NEXT:    or.b64 %rd60, %rd56, %rd59;
-; CHECK-NEXT:    cvt.u64.u16 %rd61, %rs10;
-; CHECK-NEXT:    shl.b64 %rd62, %rd61, 56;
-; CHECK-NEXT:    or.b64 %rd63, %rd60, %rd62;
+; CHECK-NEXT:    shl.b64 %rd37, %rd36, 8;
+; CHECK-NEXT:    or.b64 %rd38, %rd34, %rd37;
+; CHECK-NEXT:    cvt.u64.u16 %rd39, %rs5;
+; CHECK-NEXT:    and.b64 %rd40, %rd39, 255;
+; CHECK-NEXT:    shl.b64 %rd41, %rd40, 16;
+; CHECK-NEXT:    or.b64 %rd42, %rd38, %rd41;
+; CHECK-NEXT:    cvt.u64.u16 %rd43, %rs6;
+; CHECK-NEXT:    and.b64 %rd44, %rd43, 255;
+; CHECK-NEXT:    shl.b64 %rd45, %rd44, 24;
+; CHECK-NEXT:    or.b64 %rd46, %rd42, %rd45;
+; CHECK-NEXT:    cvt.u64.u16 %rd47, %rs7;
+; CHECK-NEXT:    and.b64 %rd48, %rd47, 255;
+; CHECK-NEXT:    shl.b64 %rd49, %rd48, 32;
+; CHECK-NEXT:    or.b64 %rd50, %rd46, %rd49;
+; CHECK-NEXT:    cvt.u64.u16 %rd51, %rs8;
+; CHECK-NEXT:    and.b64 %rd52, %rd51, 255;
+; CHECK-NEXT:    shl.b64 %rd53, %rd52, 40;
+; CHECK-NEXT:    or.b64 %rd54, %rd50, %rd53;
+; CHECK-NEXT:    cvt.u64.u16 %rd55, %rs9;
+; CHECK-NEXT:    and.b64 %rd56, %rd55, 255;
+; CHECK-NEXT:    shl.b64 %rd57, %rd56, 48;
+; CHECK-NEXT:    or.b64 %rd58, %rd54, %rd57;
+; CHECK-NEXT:    cvt.u64.u16 %rd59, %rs10;
+; CHECK-NEXT:    shl.b64 %rd60, %rd59, 56;
+; CHECK-NEXT:    or.b64 %rd61, %rd58, %rd60;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd31;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs2;
-; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd45;
-; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd41;
-; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd37;
-; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd35;
-; CHECK-NEXT:    shr.u64 %rd64, %rd52, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd43;
+; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd39;
+; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd35;
+; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd33;
+; CHECK-NEXT:    shr.u64 %rd64, %rd50, 32;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd64;
-; CHECK-NEXT:    shr.u64 %rd65, %rd56, 40;
+; CHECK-NEXT:    shr.u64 %rd65, %rd54, 40;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd65;
-; CHECK-NEXT:    shr.u64 %rd66, %rd60, 48;
+; CHECK-NEXT:    shr.u64 %rd66, %rd58, 48;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd66;
-; CHECK-NEXT:    shr.u64 %rd67, %rd63, 56;
+; CHECK-NEXT:    shr.u64 %rd67, %rd61, 56;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd67;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+24], %rd32;
 ; CHECK-NEXT:    ret;
@@ -317,16 +317,16 @@ define %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+8];
 ; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+16];
 ; CHECK-NEXT:    } // callseq 4
-; CHECK-NEXT:    cvt.u32.u16 %r16, %rs3;
-; CHECK-NEXT:    cvt.u32.u16 %r17, %rs4;
-; CHECK-NEXT:    cvt.u32.u16 %r18, %rs5;
-; CHECK-NEXT:    cvt.u32.u16 %r19, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r17, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r18, %rs6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r19;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r18;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r17;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r18;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r15;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a)
@@ -376,16 +376,16 @@ define %s_i8f32p @test_s_i8f32p(%s_i8f32p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+8];
 ; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+16];
 ; CHECK-NEXT:    } // callseq 5
-; CHECK-NEXT:    cvt.u32.u16 %r16, %rs3;
-; CHECK-NEXT:    cvt.u32.u16 %r17, %rs4;
-; CHECK-NEXT:    cvt.u32.u16 %r18, %rs5;
-; CHECK-NEXT:    cvt.u32.u16 %r19, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r17, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r18, %rs6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r19;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r18;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r17;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r18;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r15;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8f32p @test_s_i8f32p(%s_i8f32p %a)
@@ -458,48 +458,48 @@ define %s_i8f64p @test_s_i8f64p(%s_i8f64p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs10, [retval0+16];
 ; CHECK-NEXT:    ld.param.b64 %rd32, [retval0+24];
 ; CHECK-NEXT:    } // callseq 6
-; CHECK-NEXT:    cvt.u64.u16 %rd35, %rs3;
+; CHECK-NEXT:    cvt.u64.u16 %rd33, %rs3;
+; CHECK-NEXT:    and.b64 %rd34, %rd33, 255;
+; CHECK-NEXT:    cvt.u64.u16 %rd35, %rs4;
 ; CHECK-NEXT:    and.b64 %rd36, %rd35, 255;
-; CHECK-NEXT:    cvt.u64.u16 %rd37, %rs4;
-; CHECK-NEXT:    and.b64 %rd38, %rd37, 255;
-; CHECK-NEXT:    shl.b64 %rd39, %rd38, 8;
-; CHECK-NEXT:    or.b64 %rd40, %rd36, %rd39;
-; CHECK-NEXT:    cvt.u64.u16 %rd41, %rs5;
-; CHECK-NEXT:    and.b64 %rd42, %rd41, 255;
-; CHECK-NEXT:    shl.b64 %rd43, %rd42, 16;
-; CHECK-NEXT:    or.b64 %rd44, %rd40, %rd43;
-; CHECK-NEXT:    cvt.u64.u16 %rd45, %rs6;
-; CHECK-NEXT:    and.b64 %rd46, %rd45, 255;
-; CHECK-NEXT:    shl.b64 %rd47, %rd46, 24;
-; CHECK-NEXT:    or.b64 %rd48, %rd44, %rd47;
-; CHECK-NEXT:    cvt.u64.u16 %rd49, %rs7;
-; CHECK-NEXT:    and.b64 %rd50, %rd49, 255;
-; CHECK-NEXT:    shl.b64 %rd51, %rd50, 32;
-; CHECK-NEXT:    or.b64 %rd52, %rd48, %rd51;
-; CHECK-NEXT:    cvt.u64.u16 %rd53, %rs8;
-; CHECK-NEXT:    and.b64 %rd54, %rd53, 255;
-; CHECK-NEXT:    shl.b64 %rd55, %rd54, 40;
-; CHECK-NEXT:    or.b64 %rd56, %rd52, %rd55;
-; CHECK-NEXT:    cvt.u64.u16 %rd57, %rs9;
-; CHECK-NEXT:    and.b64 %rd58, %rd57, 255;
-; CHECK-NEXT:    shl.b64 %rd59, %rd58, 48;
-; CHECK-NEXT:    or.b64 %rd60, %rd56, %rd59;
-; CHECK-NEXT:    cvt.u64.u16 %rd61, %rs10;
-; CHECK-NEXT:    shl.b64 %rd62, %rd61, 56;
-; CHECK-NEXT:    or.b64 %rd63, %rd60, %rd62;
+; CHECK-NEXT:    shl.b64 %rd37, %rd36, 8;
+; CHECK-NEXT:    or.b64 %rd38, %rd34, %rd37;
+; CHECK-NEXT:    cvt.u64.u16 %rd39, %rs5;
+; CHECK-NEXT:    and.b64 %rd40, %rd39, 255;
+; CHECK-NEXT:    shl.b64 %rd41, %rd40, 16;
+; CHECK-NEXT:    or.b64 %rd42, %rd38, %rd41;
+; CHECK-NEXT:    cvt.u64.u16 %rd43, %rs6;
+; CHECK-NEXT:    and.b64 %rd44, %rd43, 255;
+; CHECK-NEXT:    shl.b64 %rd45, %rd44, 24;
+; CHECK-NEXT:    or.b64 %rd46, %rd42, %rd45;
+; CHECK-NEXT:    cvt.u64.u16 %rd47, %rs7;
+; CHECK-NEXT:    and.b64 %rd48, %rd47, 255;
+; CHECK-NEXT:    shl.b64 %rd49, %rd48, 32;
+; CHECK-NEXT:    or.b64 %rd50, %rd46, %rd49;
+; CHECK-NEXT:    cvt.u64.u16 %rd51, %rs8;
+; CHECK-NEXT:    and.b64 %rd52, %rd51, 255;
+; CHECK-NEXT:    shl.b64 %rd53, %rd52, 40;
+; CHECK-NEXT:    or.b64 %rd54, %rd50, %rd53;
+; CHECK-NEXT:    cvt.u64.u16 %rd55, %rs9;
+; CHECK-NEXT:    and.b64 %rd56, %rd55, 255;
+; CHECK-NEXT:    shl.b64 %rd57, %rd56, 48;
+; CHECK-NEXT:    or.b64 %rd58, %rd54, %rd57;
+; CHECK-NEXT:    cvt.u64.u16 %rd59, %rs10;
+; CHECK-NEXT:    shl.b64 %rd60, %rd59, 56;
+; CHECK-NEXT:    or.b64 %rd61, %rd58, %rd60;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd31;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs2;
-; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd45;
-; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd41;
-; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd37;
-; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd35;
-; CHECK-NEXT:    shr.u64 %rd64, %rd52, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd43;
+; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd39;
+; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd35;
+; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd33;
+; CHECK-NEXT:    shr.u64 %rd64, %rd50, 32;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd64;
-; CHECK-NEXT:    shr.u64 %rd65, %rd56, 40;
+; CHECK-NEXT:    shr.u64 %rd65, %rd54, 40;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd65;
-; CHECK-NEXT:    shr.u64 %rd66, %rd60, 48;
+; CHECK-NEXT:    shr.u64 %rd66, %rd58, 48;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd66;
-; CHECK-NEXT:    shr.u64 %rd67, %rd63, 56;
+; CHECK-NEXT:    shr.u64 %rd67, %rd61, 56;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd67;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+24], %rd32;
 ; CHECK-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/variadics-backend.ll b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
index 167d7faafe5b3..ad2e7044e93bc 100644
--- a/llvm/test/CodeGen/NVPTX/variadics-backend.ll
+++ b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
@@ -348,7 +348,7 @@ define dso_local void @qux() {
 ; CHECK-PTX-NEXT:    .local .align 8 .b8 __local_depot7[24];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
 ; CHECK-PTX-NEXT:    .reg .b64 %rd<8>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
diff --git a/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll b/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll
index 04f81124b6d8c..9e50455a0b42a 100644
--- a/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll
+++ b/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll
@@ -18,13 +18,13 @@ declare <4 x i32> @split_spill(double, double, double, ...)
 ; CHECK-DAG: [[ELEMENT1:%[0-9]+]]:gprc = LWZ 48, $r1 :: (load (s32))
 ; CHECK-DAG: [[ELEMENT2:%[0-9]+]]:gprc = LWZ 52, $r1 :: (load (s32))
 ; CHECK:     [[FLOAT1SPLAT:%[0-9]+]]:vrrc = VSPLTISW 1
-; CHECK:     [[FLOAT1COPY:%[0-9]+]]:vsrc = COPY [[FLOAT1SPLAT]]
+; CHECK:     [[FLOAT1COPY:%[0-9]+]]:vsrc = COPY killed [[FLOAT1SPLAT]]
 ; CHECK:     [[DOUBLE1:%[0-9]+]]:vsrc = XVCVSXWDP killed [[FLOAT1COPY]], implicit $rm
-; CHECK:     [[DOUBLE1COPY:%[0-9]+]]:vsfrc = COPY [[DOUBLE1]]
+; CHECK:     [[DOUBLE1COPY:%[0-9]+]]:vsfrc = COPY killed [[DOUBLE1]]
 ; CHECK:     [[FLOAT2SPLAT:%[0-9]+]]:vrrc = VSPLTISW 2
-; CHECK:     [[FLOAT2COPY:%[0-9]+]]:vsrc = COPY [[FLOAT2SPLAT]]
+; CHECK:     [[FLOAT2COPY:%[0-9]+]]:vsrc = COPY killed [[FLOAT2SPLAT]]
 ; CHECK:     [[DOUBLE2:%[0-9]+]]:vsrc = XVCVSXWDP killed [[FLOAT2COPY]], implicit $rm
-; CHECK:     [[DOUBLE2COPY:%[0-9]+]]:vsfrc = COPY [[DOUBLE2]]
+; CHECK:     [[DOUBLE2COPY:%[0-9]+]]:vsfrc = COPY killed [[DOUBLE2]]
 
 ; CHECK:     [[DZERO:%[0-9]+]]:vsfrc = XXLXORdpz
 ; CHECK:     [[DTOI1:%[0-9]+]]:gprc = LIS 16368
diff --git a/llvm/test/CodeGen/PowerPC/anyext_srl.ll b/llvm/test/CodeGen/PowerPC/anyext_srl.ll
index 49b13d70150d3..f92e622601b04 100644
--- a/llvm/test/CodeGen/PowerPC/anyext_srl.ll
+++ b/llvm/test/CodeGen/PowerPC/anyext_srl.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:     -mcpu=pwr8 < %s | FileCheck %s
 
@@ -6,6 +7,13 @@
 
 ; Function Attrs: norecurse nounwind readonly
 define zeroext i1 @foo(ptr %s_a, ptr %s_b) local_unnamed_addr {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lbz 3, 0(3)
+; CHECK-NEXT:    lbz 4, 0(4)
+; CHECK-NEXT:    xor 3, 3, 4
+; CHECK-NEXT:    rldicl 3, 3, 61, 63
+; CHECK-NEXT:    blr
 entry:
   %0 = load i32, ptr %s_a, align 8, !tbaa !1
   %and.i = and i32 %0, 8
@@ -15,9 +23,6 @@ entry:
   %cmp.i5 = icmp ne i32 %and.i4, 0
   %cmp = xor i1 %cmp.i, %cmp.i5
   ret i1 %cmp
-; CHECK-LABEL: @foo
-; CHECK: rldicl  {{[0-9]+}}, {{[0-9]+}}, 61, 63
-
 }
 
 !1 = !{!2, !2, i64 0}
diff --git a/llvm/test/CodeGen/PowerPC/crreduce-reg.mir b/llvm/test/CodeGen/PowerPC/crreduce-reg.mir
new file mode 100644
index 0000000000000..1b6045d82f9dd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/crreduce-reg.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=powerpc64-unknown-linux-gnu -run-pass=ppc-reduce-cr-ops \
+# RUN:   -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name:            subreg_folding_regression
+tracksRegLiveness: true
+isSSA:           true
+body:             |
+  ; CHECK-LABEL: name: subreg_folding_regression
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $x3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
+  ; CHECK-NEXT:   [[LD:%[0-9]+]]:g8rc = LD 0, [[COPY]] :: (load (s64))
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x20000000), %bb.3(0x60000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:g8rc_and_g8rc_nox0 = PHI [[LD]], %bb.0, %3, %bb.3, %4, %bb.2
+  ; CHECK-NEXT:   [[LBZ:%[0-9]+]]:gprc = LBZ 0, [[PHI]] :: (load (s8))
+  ; CHECK-NEXT:   [[CMPWI:%[0-9]+]]:crrc = CMPWI killed [[LBZ]], 0
+  ; CHECK-NEXT:   [[ADDI8_:%[0-9]+]]:g8rc = nuw ADDI8 [[PHI]], 1
+  ; CHECK-NEXT:   STD [[ADDI8_]], 0, [[COPY]] :: (store (s64))
+  ; CHECK-NEXT:   [[LBZ1:%[0-9]+]]:gprc = LBZ 1, [[PHI]] :: (load (s8))
+  ; CHECK-NEXT:   BCn [[CMPWI]].sub_lt, %bb.2
+  ; CHECK-NEXT:   B %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.1(0x55555555), %bb.2(0x2aaaaaab)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[CMPWI1:%[0-9]+]]:crrc = CMPWI killed [[LBZ1]], 0
+  ; CHECK-NEXT:   BC killed [[CMPWI1]].sub_eq, %bb.1
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ADDI8_1:%[0-9]+]]:g8rc = nuw ADDI8 [[PHI]], 2
+  ; CHECK-NEXT:   STD [[ADDI8_1]], 0, [[COPY]] :: (store (s64))
+  ; CHECK-NEXT:   B %bb.1
+  bb.0:
+    liveins: $x3
+
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %1:g8rc = LD 0, %0 :: (load (s64))
+
+  bb.1:
+    %2:g8rc_and_g8rc_nox0 = PHI %1, %bb.0, %3, %bb.1, %4, %bb.2
+    %5:gprc = LBZ 0, %2 :: (load (s8))
+    %6:crrc = CMPWI killed %5, 0
+    %3:g8rc = nuw ADDI8 %2, 1
+    STD %3, 0, %0 :: (store (s64))
+    %7:gprc = LBZ 1, %2 :: (load (s8))
+    %8:crrc = CMPWI killed %7, 0
+    %9:crbitrc = CRAND %8.sub_eq, %6.sub_lt
+    BC killed %9, %bb.1
+    B %bb.2
+
+  bb.2:
+    %4:g8rc = nuw ADDI8 %2, 2
+    STD %4, 0, %0 :: (store (s64))
+    B %bb.1
+
+...
diff --git a/llvm/test/CodeGen/PowerPC/fma-combine.ll b/llvm/test/CodeGen/PowerPC/fma-combine.ll
index 3d45e9a3a509c..456f85ad3eefd 100644
--- a/llvm/test/CodeGen/PowerPC/fma-combine.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-combine.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -enable-no-signed-zeros-fp-math \
-; RUN:     -enable-unsafe-fp-math  < %s | FileCheck -check-prefix=CHECK-FAST %s
+; RUN:       < %s | FileCheck -check-prefix=CHECK-FAST %s
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -enable-no-signed-zeros-fp-math \
-; RUN:     -enable-unsafe-fp-math -mattr=-vsx < %s | FileCheck -check-prefix=CHECK-FAST-NOVSX %s
+; RUN:      -mattr=-vsx < %s | FileCheck -check-prefix=CHECK-FAST-NOVSX %s
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
 
 define dso_local double @fma_combine1(double %a, double %b, double %c) {
@@ -19,13 +19,12 @@ define dso_local double @fma_combine1(double %a, double %b, double %c) {
 ; CHECK-LABEL: fma_combine1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xsnegdp 0, 3
-; CHECK-NEXT:    xsmuldp 0, 0, 2
-; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    xsmsubadp 1, 0, 2
 ; CHECK-NEXT:    blr
 entry:
   %fneg1 = fneg double %c
-  %mul = fmul double %fneg1, %b
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %fneg1, %b
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -43,13 +42,12 @@ define dso_local double @fma_combine2(double %a, double %b, double %c) {
 ; CHECK-LABEL: fma_combine2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xsnegdp 0, 3
-; CHECK-NEXT:    xsmuldp 0, 2, 0
-; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    xsmsubadp 1, 2, 0
 ; CHECK-NEXT:    blr
 entry:
   %fneg1 = fneg double %c
-  %mul = fmul double %b, %fneg1
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %b, %fneg1
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -85,17 +83,16 @@ define dso_local double @fma_combine_two_uses(double %a, double %b, double %c) {
 ; CHECK-NEXT:    stfd 0, v@toc@l(3)
 ; CHECK-NEXT:    xsnegdp 0, 3
 ; CHECK-NEXT:    addis 3, 2, z@toc@ha
+; CHECK-NEXT:    xsmsubadp 1, 0, 2
 ; CHECK-NEXT:    stfd 0, z@toc@l(3)
-; CHECK-NEXT:    xsmuldp 0, 0, 2
-; CHECK-NEXT:    xssubdp 1, 0, 1
 ; CHECK-NEXT:    blr
 entry:
   %fneg = fneg double %a
   store double %fneg, ptr @v, align 8
   %fneg1 = fneg double %c
   store double %fneg1, ptr @z, align 8
-  %mul = fmul double %fneg1, %b
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %fneg1, %b
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -122,15 +119,14 @@ define dso_local double @fma_combine_one_use(double %a, double %b, double %c) {
 ; CHECK-NEXT:    addis 3, 2, v@toc@ha
 ; CHECK-NEXT:    stfd 0, v@toc@l(3)
 ; CHECK-NEXT:    xsnegdp 0, 3
-; CHECK-NEXT:    xsmuldp 0, 0, 2
-; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    xsmsubadp 1, 0, 2
 ; CHECK-NEXT:    blr
 entry:
   %fneg = fneg double %a
   store double %fneg, ptr @v, align 8
   %fneg1 = fneg double %c
-  %mul = fmul double %fneg1, %b
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %fneg1, %b
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -327,15 +323,12 @@ define dso_local double @fma_combine_const(double %a, double %b) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis 3, 2, .LCPI9_0@toc@ha
 ; CHECK-NEXT:    lfd 0, .LCPI9_0@toc@l(3)
-; CHECK-NEXT:    addis 3, 2, .LCPI9_1@toc@ha
-; CHECK-NEXT:    xsmuldp 0, 1, 0
-; CHECK-NEXT:    lfd 1, .LCPI9_1@toc@l(3)
-; CHECK-NEXT:    xsmaddadp 2, 0, 1
+; CHECK-NEXT:    xsmaddadp 2, 1, 0
 ; CHECK-NEXT:    fmr 1, 2
 ; CHECK-NEXT:    blr
 entry:
-  %0 = fmul double %a, 1.1
-  %1 = call contract double @llvm.fma.f64(double %0, double 2.1, double %b)
+  %0 = fmul reassoc double %a, 1.1
+  %1 = call contract reassoc double @llvm.fma.f64(double %0, double 2.1, double %b)
   ret double %1
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/fp-branch.ll b/llvm/test/CodeGen/PowerPC/fp-branch.ll
index ba8052d93cb98..4b67c4d319913 100644
--- a/llvm/test/CodeGen/PowerPC/fp-branch.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-branch.ll
@@ -1,8 +1,21 @@
-; RUN: llc -verify-machineinstrs < %s -mattr=-vsx -mtriple=ppc32-- | grep fcmp | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs < %s -mattr=-vsx -mtriple=ppc32-- | FileCheck %s
 
 declare i1 @llvm.isunordered.f64(double, double)
 
 define i1 @intcoord_cond_next55(double %tmp48.reload) {
+; CHECK-LABEL: intcoord_cond_next55:
+; CHECK:       # %bb.0: # %newFuncRoot
+; CHECK-NEXT:    lis 3, .LCPI0_0@ha
+; CHECK-NEXT:    lfs 0, .LCPI0_0@l(3)
+; CHECK-NEXT:    fcmpu 0, 1, 0
+; CHECK-NEXT:    blt 0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %bb72.exitStub
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB0_2: # %cond_next62.exitStub
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    blr
 newFuncRoot:
         br label %cond_next55
 
diff --git a/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll b/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll
index c6e6737211c2e..55482a0c5ff2c 100644
--- a/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll
+++ b/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ;; Test hoisting `xxspltib` out of the loop.
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff --mcpu=pwr10 \
@@ -10,27 +11,166 @@
 ; RUN:   %s -o - 2>&1 | FileCheck --check-prefix=LINUX64LE %s
 
 define void @_Z3fooPfS_Pi(ptr noalias nocapture noundef %_a, ptr noalias nocapture %In_a, ptr noalias nocapture %n) {
+; AIX64-LABEL: _Z3fooPfS_Pi:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    lwz 5, 0(5)
+; AIX64-NEXT:    cmpwi 5, 1
+; AIX64-NEXT:    bltlr 0
+; AIX64-NEXT:  # %bb.1: # %for.body.preheader
+; AIX64-NEXT:    li 6, 0
+; AIX64-NEXT:    cmplwi 5, 1
+; AIX64-NEXT:    beq 0, L..BB0_4
+; AIX64-NEXT:  # %bb.2: # %for.body.preheader.new
+; AIX64-NEXT:    rlwinm 6, 5, 0, 1, 30
+; AIX64-NEXT:    xxspltib 0, 6
+; AIX64-NEXT:    addi 9, 4, -8
+; AIX64-NEXT:    addi 7, 3, -8
+; AIX64-NEXT:    li 8, 8
+; AIX64-NEXT:    li 10, 12
+; AIX64-NEXT:    li 11, 4
+; AIX64-NEXT:    addi 6, 6, -2
+; AIX64-NEXT:    rldicl 6, 6, 63, 1
+; AIX64-NEXT:    addi 6, 6, 1
+; AIX64-NEXT:    mtctr 6
+; AIX64-NEXT:    li 6, 0
+; AIX64-NEXT:    .align 4
+; AIX64-NEXT:  L..BB0_3: # %for.body
+; AIX64-NEXT:    #
+; AIX64-NEXT:    lxvwsx 1, 9, 8
+; AIX64-NEXT:    addi 6, 6, 2
+; AIX64-NEXT:    xxland 1, 1, 0
+; AIX64-NEXT:    xscvspdpn 1, 1
+; AIX64-NEXT:    stfsu 1, 8(7)
+; AIX64-NEXT:    lxvwsx 1, 9, 10
+; AIX64-NEXT:    addi 9, 9, 8
+; AIX64-NEXT:    xxland 1, 1, 0
+; AIX64-NEXT:    xxsldwi 1, 1, 1, 3
+; AIX64-NEXT:    stfiwx 1, 7, 11
+; AIX64-NEXT:    bdnz L..BB0_3
+; AIX64-NEXT:  L..BB0_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; AIX64-NEXT:    andi. 5, 5, 1
+; AIX64-NEXT:    bclr 4, 1, 0
+; AIX64-NEXT:  # %bb.5: # %for.body.epil
+; AIX64-NEXT:    sldi 5, 6, 2
+; AIX64-NEXT:    xxspltib 1, 6
+; AIX64-NEXT:    lxvwsx 0, 4, 5
+; AIX64-NEXT:    xxland 0, 0, 1
+; AIX64-NEXT:    xxsldwi 0, 0, 0, 3
+; AIX64-NEXT:    stfiwx 0, 3, 5
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: _Z3fooPfS_Pi:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    lwz 5, 0(5)
+; AIX32-NEXT:    cmpwi 5, 1
+; AIX32-NEXT:    bltlr 0
+; AIX32-NEXT:  # %bb.1: # %for.body.preheader
+; AIX32-NEXT:    li 6, 0
+; AIX32-NEXT:    beq 0, L..BB0_4
+; AIX32-NEXT:  # %bb.2: # %for.body.preheader.new
+; AIX32-NEXT:    xxspltib 0, 6
+; AIX32-NEXT:    addi 12, 4, -8
+; AIX32-NEXT:    addi 9, 3, -8
+; AIX32-NEXT:    rlwinm 7, 5, 0, 1, 30
+; AIX32-NEXT:    li 8, 0
+; AIX32-NEXT:    li 10, 8
+; AIX32-NEXT:    li 11, 12
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB0_3: # %for.body
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lxvwsx 1, 12, 10
+; AIX32-NEXT:    addic 6, 6, 2
+; AIX32-NEXT:    addze 8, 8
+; AIX32-NEXT:    xor 0, 6, 7
+; AIX32-NEXT:    or. 0, 0, 8
+; AIX32-NEXT:    xxland 1, 1, 0
+; AIX32-NEXT:    xscvspdpn 1, 1
+; AIX32-NEXT:    stfsu 1, 8(9)
+; AIX32-NEXT:    lxvwsx 1, 12, 11
+; AIX32-NEXT:    addi 12, 12, 8
+; AIX32-NEXT:    xxland 1, 1, 0
+; AIX32-NEXT:    xscvspdpn 1, 1
+; AIX32-NEXT:    stfs 1, 4(9)
+; AIX32-NEXT:    bne 0, L..BB0_3
+; AIX32-NEXT:  L..BB0_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; AIX32-NEXT:    andi. 5, 5, 1
+; AIX32-NEXT:    bclr 4, 1, 0
+; AIX32-NEXT:  # %bb.5: # %for.body.epil
+; AIX32-NEXT:    slwi 5, 6, 2
+; AIX32-NEXT:    xxspltib 1, 6
+; AIX32-NEXT:    lxvwsx 0, 4, 5
+; AIX32-NEXT:    xxland 0, 0, 1
+; AIX32-NEXT:    xscvspdpn 0, 0
+; AIX32-NEXT:    stfsx 0, 3, 5
+; AIX32-NEXT:    blr
+;
+; LINUX64LE-LABEL: _Z3fooPfS_Pi:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    lwz 5, 0(5)
+; LINUX64LE-NEXT:    cmpwi 5, 1
+; LINUX64LE-NEXT:    bltlr 0
+; LINUX64LE-NEXT:  # %bb.1: # %for.body.preheader
+; LINUX64LE-NEXT:    li 6, 0
+; LINUX64LE-NEXT:    cmplwi 5, 1
+; LINUX64LE-NEXT:    beq 0, .LBB0_4
+; LINUX64LE-NEXT:  # %bb.2: # %for.body.preheader.new
+; LINUX64LE-NEXT:    rlwinm 6, 5, 0, 1, 30
+; LINUX64LE-NEXT:    xxspltib 0, 6
+; LINUX64LE-NEXT:    addi 8, 4, -8
+; LINUX64LE-NEXT:    addi 7, 3, -8
+; LINUX64LE-NEXT:    li 9, 8
+; LINUX64LE-NEXT:    li 10, 12
+; LINUX64LE-NEXT:    li 11, 4
+; LINUX64LE-NEXT:    addi 6, 6, -2
+; LINUX64LE-NEXT:    rldicl 6, 6, 63, 1
+; LINUX64LE-NEXT:    addi 6, 6, 1
+; LINUX64LE-NEXT:    mtctr 6
+; LINUX64LE-NEXT:    li 6, 0
+; LINUX64LE-NEXT:    .p2align 4
+; LINUX64LE-NEXT:  .LBB0_3: # %for.body
+; LINUX64LE-NEXT:    #
+; LINUX64LE-NEXT:    lxvwsx 1, 8, 9
+; LINUX64LE-NEXT:    addi 6, 6, 2
+; LINUX64LE-NEXT:    xxland 1, 1, 0
+; LINUX64LE-NEXT:    xxsldwi 1, 1, 1, 3
+; LINUX64LE-NEXT:    xscvspdpn 1, 1
+; LINUX64LE-NEXT:    stfsu 1, 8(7)
+; LINUX64LE-NEXT:    lxvwsx 1, 8, 10
+; LINUX64LE-NEXT:    addi 8, 8, 8
+; LINUX64LE-NEXT:    xxland 1, 1, 0
+; LINUX64LE-NEXT:    stxvrwx 1, 7, 11
+; LINUX64LE-NEXT:    bdnz .LBB0_3
+; LINUX64LE-NEXT:  .LBB0_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; LINUX64LE-NEXT:    andi. 5, 5, 1
+; LINUX64LE-NEXT:    bclr 4, 1, 0
+; LINUX64LE-NEXT:  # %bb.5: # %for.body.epil
+; LINUX64LE-NEXT:    sldi 5, 6, 2
+; LINUX64LE-NEXT:    xxspltib 1, 6
+; LINUX64LE-NEXT:    lxvwsx 0, 4, 5
+; LINUX64LE-NEXT:    xxland 0, 0, 1
+; LINUX64LE-NEXT:    stxvrwx 0, 3, 5
+; LINUX64LE-NEXT:    blr
 entry:
   %0 = load i32, ptr %n, align 4
   %cmp9 = icmp sgt i32 %0, 0
   br i1 %cmp9, label %for.body.preheader, label %for.cond.cleanup
 
-for.body.preheader:                               
+for.body.preheader:
   %wide.trip.count = zext nneg i32 %0 to i64
   %xtraiter = and i64 %wide.trip.count, 1
   %1 = icmp eq i32 %0, 1
   br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
 
-for.body.preheader.new:                           
+for.body.preheader.new:
   %unroll_iter = and i64 %wide.trip.count, 2147483646
   br label %for.body
 
-for.cond.cleanup.loopexit.unr-lcssa:              
+for.cond.cleanup.loopexit.unr-lcssa:
   %indvars.iv.unr = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next.1, %for.body ]
   %lcmp.mod.not = icmp eq i64 %xtraiter, 0
   br i1 %lcmp.mod.not, label %for.cond.cleanup, label %for.body.epil
 
-for.body.epil:                                    
+for.body.epil:
   %arrayidx.epil = getelementptr inbounds nuw float, ptr %In_a, i64 %indvars.iv.unr
   %2 = load float, ptr %arrayidx.epil, align 4
   %vecins.i.epil = insertelement <4 x float> poison, float %2, i64 0
@@ -42,10 +182,10 @@ for.body.epil:
   store float %vecext.i.epil, ptr %arrayidx5.epil, align 4
   br label %for.cond.cleanup
 
-for.cond.cleanup:                                 
+for.cond.cleanup:
   ret void
 
-for.body:                                         
+for.body:
   %indvars.iv = phi i64 [ 0, %for.body.preheader.new ], [ %indvars.iv.next.1, %for.body ]
   %niter = phi i64 [ 0, %for.body.preheader.new ], [ %niter.next.1, %for.body ]
   %arrayidx = getelementptr inbounds nuw float, ptr %In_a, i64 %indvars.iv
@@ -72,113 +212,3 @@ for.body:
   %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter
   br i1 %niter.ncmp.1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
 }
-
-; AIX32:      ._Z3fooPfS_Pi:
-; AIX32-NEXT: # %bb.0:                                # %entry
-; AIX32-NEXT:   lwz 5, 0(5)
-; AIX32-NEXT:   cmpwi   5, 1
-; AIX32-NEXT:   bltlr   0
-; AIX32-NEXT: # %bb.1:                                # %for.body.preheader
-; AIX32-NEXT:   li 6, 0
-; AIX32-NEXT:   beq     0, L..BB0_4
-; AIX32-NEXT: # %bb.2:                                # %for.body.preheader.new
-; AIX32-NEXT:   xxspltib 0, 6
-; AIX32-NEXT:   addi 12, 4, -8
-; AIX32-NEXT:   addi 9, 3, -8
-; AIX32-NEXT:   rlwinm 7, 5, 0, 1, 30
-; AIX32-NEXT:   li 8, 0
-; AIX32-NEXT:   li 10, 8
-; AIX32-NEXT:   li 11, 12
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT: L..BB0_3:                               # %for.body
-; AIX32-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   lxvwsx 1, 12, 10
-; AIX32-NEXT:   addic 6, 6, 2
-; AIX32-NEXT:   addze 8, 8
-; AIX32-NEXT:   xor 0, 6, 7
-; AIX32-NEXT:   or. 0, 0, 8
-; AIX32-NEXT:   xxland 1, 1, 0
-; AIX32-NEXT:   xscvspdpn 1, 1
-; AIX32-NEXT:   stfsu 1, 8(9)
-; AIX32-NEXT:   lxvwsx 1, 12, 11
-; AIX32-NEXT:   addi 12, 12, 8
-; AIX32-NEXT:   xxland 1, 1, 0
-; AIX32-NEXT:   xscvspdpn 1, 1
-; AIX32-NEXT:   stfs 1, 4(9)
-; AIX32-NEXT:   bne     0, L..BB0_3
-
-; AIX64:      ._Z3fooPfS_Pi:
-; AIX64-NEXT: # %bb.0:                                # %entry
-; AIX64-NEXT:   lwz 5, 0(5)
-; AIX64-NEXT:   cmpwi   5, 1
-; AIX64-NEXT:   bltlr   0
-; AIX64-NEXT: # %bb.1:                                # %for.body.preheader
-; AIX64-NEXT:   li 6, 0
-; AIX64-NEXT:   cmplwi  5, 1
-; AIX64-NEXT:   beq     0, L..BB0_4
-; AIX64-NEXT: # %bb.2:                                # %for.body.preheader.new
-; AIX64-NEXT:   rlwinm 6, 5, 0, 1, 30
-; AIX64-NEXT:   xxspltib 0, 6
-; AIX64-NEXT:   addi 9, 4, -8
-; AIX64-NEXT:   addi 7, 3, -8
-; AIX64-NEXT:   li 8, 8
-; AIX64-NEXT:   li 10, 12
-; AIX64-NEXT:   li 11, 4
-; AIX64-NEXT:   addi 6, 6, -2
-; AIX64-NEXT:   rldicl 6, 6, 63, 1
-; AIX64-NEXT:   addi 6, 6, 1
-; AIX64-NEXT:   mtctr 6
-; AIX64-NEXT:   li 6, 0
-; AIX64-NEXT:   .align  4
-; AIX64-NEXT: L..BB0_3:                               # %for.body
-; AIX64-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; AIX64-NEXT:   lxvwsx 1, 9, 8
-; AIX64-NEXT:   addi 6, 6, 2
-; AIX64-NEXT:   xxland 1, 1, 0
-; AIX64-NEXT:   xscvspdpn 1, 1
-; AIX64-NEXT:   stfsu 1, 8(7)
-; AIX64-NEXT:   lxvwsx 1, 9, 10
-; AIX64-NEXT:   addi 9, 9, 8
-; AIX64-NEXT:   xxland 1, 1, 0
-; AIX64-NEXT:   xxsldwi 1, 1, 1, 3
-; AIX64-NEXT:   stfiwx 1, 7, 11
-; AIX64-NEXT:   bdnz L..BB0_3
-
-; LINUX64LE:      _Z3fooPfS_Pi:                           # @_Z3fooPfS_Pi
-; LINUX64LE-NEXT: .Lfunc_begin0:
-; LINUX64LE-NEXT:       .cfi_startproc
-; LINUX64LE-NEXT: # %bb.0:                                # %entry
-; LINUX64LE-NEXT:       lwz 5, 0(5)
-; LINUX64LE-NEXT:       cmpwi   5, 1
-; LINUX64LE-NEXT:       bltlr   0
-; LINUX64LE-NEXT: # %bb.1:                                # %for.body.preheader
-; LINUX64LE-NEXT:       li 6, 0
-; LINUX64LE-NEXT:       cmplwi  5, 1
-; LINUX64LE-NEXT:       beq     0, .LBB0_4
-; LINUX64LE-NEXT: # %bb.2:                                # %for.body.preheader.new
-; LINUX64LE-NEXT:       rlwinm 6, 5, 0, 1, 30
-; LINUX64LE-NEXT:       xxspltib 0, 6
-; LINUX64LE-NEXT:       addi 8, 4, -8
-; LINUX64LE-NEXT:       addi 7, 3, -8
-; LINUX64LE-NEXT:       li 9, 8
-; LINUX64LE-NEXT:       li 10, 12
-; LINUX64LE-NEXT:       li 11, 4
-; LINUX64LE-NEXT:       addi 6, 6, -2
-; LINUX64LE-NEXT:       rldicl 6, 6, 63, 1
-; LINUX64LE-NEXT:       addi 6, 6, 1
-; LINUX64LE-NEXT:       mtctr 6
-; LINUX64LE-NEXT:       li 6, 0
-; LINUX64LE-NEXT:       .p2align        4
-; LINUX64LE-NEXT: .LBB0_3:                                # %for.body
-; LINUX64LE-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; LINUX64LE-NEXT:       lxvwsx 1, 8, 9
-; LINUX64LE-NEXT:       addi 6, 6, 2
-; LINUX64LE-NEXT:       xxland 1, 1, 0
-; LINUX64LE-NEXT:       xxsldwi 1, 1, 1, 3
-; LINUX64LE-NEXT:       xscvspdpn 1, 1
-; LINUX64LE-NEXT:       stfsu 1, 8(7)
-; LINUX64LE-NEXT:       lxvwsx 1, 8, 10
-; LINUX64LE-NEXT:       addi 8, 8, 8
-; LINUX64LE-NEXT:       xxland 1, 1, 0
-; LINUX64LE-NEXT:       stxvrwx 1, 7, 11
-; LINUX64LE-NEXT:       bdnz .LBB0_3
diff --git a/llvm/test/CodeGen/PowerPC/memintr32.ll b/llvm/test/CodeGen/PowerPC/memintr32.ll
new file mode 100644
index 0000000000000..c07a5af17e48a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/memintr32.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names -mtriple=powerpc-ibm-aix < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-AIX-32-P9
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names -mtriple=powerpc-unknown-linux-gnu < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-LINUX32-P9
+
+define i32 @memcmp_test(ptr nocapture noundef readonly %ptr1, ptr nocapture noundef readonly %ptr2, i32 noundef %num) nounwind {
+; CHECK-AIX-32-P9-LABEL: memcmp_test:
+; CHECK-AIX-32-P9:       # %bb.0: # %entry
+; CHECK-AIX-32-P9-NEXT:    mflr r0
+; CHECK-AIX-32-P9-NEXT:    stwu r1, -64(r1)
+; CHECK-AIX-32-P9-NEXT:    stw r0, 72(r1)
+; CHECK-AIX-32-P9-NEXT:    bl .memcmp[PR]
+; CHECK-AIX-32-P9-NEXT:    nop
+; CHECK-AIX-32-P9-NEXT:    addi r1, r1, 64
+; CHECK-AIX-32-P9-NEXT:    lwz r0, 8(r1)
+; CHECK-AIX-32-P9-NEXT:    mtlr r0
+; CHECK-AIX-32-P9-NEXT:    blr
+;
+; CHECK-LINUX32-P9-LABEL: memcmp_test:
+; CHECK-LINUX32-P9:       # %bb.0: # %entry
+; CHECK-LINUX32-P9-NEXT:    mflr r0
+; CHECK-LINUX32-P9-NEXT:    stwu r1, -16(r1)
+; CHECK-LINUX32-P9-NEXT:    stw r0, 20(r1)
+; CHECK-LINUX32-P9-NEXT:    bl memcmp
+; CHECK-LINUX32-P9-NEXT:    lwz r0, 20(r1)
+; CHECK-LINUX32-P9-NEXT:    addi r1, r1, 16
+; CHECK-LINUX32-P9-NEXT:    mtlr r0
+; CHECK-LINUX32-P9-NEXT:    blr
+entry:
+  %call = tail call i32 @memcmp(ptr noundef %ptr1, ptr noundef %ptr2, i32 noundef %num)
+    ret i32 %call
+}
+
+declare i32 @memcmp(ptr noundef captures(none), ptr noundef captures(none), i32 noundef) nounwind
+
+
+
diff --git a/llvm/test/CodeGen/PowerPC/memintr64.ll b/llvm/test/CodeGen/PowerPC/memintr64.ll
new file mode 100644
index 0000000000000..b3a6650b8f6e6
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/memintr64.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-LE-P9
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc64-unknown-linux-gnu < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-BE-P9
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64-ibm-aix < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-AIX-64-P9
+
+define noundef i32 @_Z11memcmp_testPKvS0_m(ptr noundef readonly captures(none) %ptr1, ptr noundef readonly captures(none) %ptr2, i64 noundef %num) nounwind {
+; CHECK-LE-P9-LABEL: _Z11memcmp_testPKvS0_m:
+; CHECK-LE-P9:       # %bb.0: # %entry
+; CHECK-LE-P9-NEXT:    mflr r0
+; CHECK-LE-P9-NEXT:    stdu r1, -32(r1)
+; CHECK-LE-P9-NEXT:    std r0, 48(r1)
+; CHECK-LE-P9-NEXT:    bl memcmp
+; CHECK-LE-P9-NEXT:    nop
+; CHECK-LE-P9-NEXT:    addi r1, r1, 32
+; CHECK-LE-P9-NEXT:    ld r0, 16(r1)
+; CHECK-LE-P9-NEXT:    mtlr r0
+; CHECK-LE-P9-NEXT:    blr
+;
+; CHECK-BE-P9-LABEL: _Z11memcmp_testPKvS0_m:
+; CHECK-BE-P9:       # %bb.0: # %entry
+; CHECK-BE-P9-NEXT:    mflr r0
+; CHECK-BE-P9-NEXT:    stdu r1, -112(r1)
+; CHECK-BE-P9-NEXT:    std r0, 128(r1)
+; CHECK-BE-P9-NEXT:    bl memcmp
+; CHECK-BE-P9-NEXT:    nop
+; CHECK-BE-P9-NEXT:    addi r1, r1, 112
+; CHECK-BE-P9-NEXT:    ld r0, 16(r1)
+; CHECK-BE-P9-NEXT:    mtlr r0
+; CHECK-BE-P9-NEXT:    blr
+;
+; CHECK-AIX-64-P9-LABEL: _Z11memcmp_testPKvS0_m:
+; CHECK-AIX-64-P9:       # %bb.0: # %entry
+; CHECK-AIX-64-P9-NEXT:    mflr r0
+; CHECK-AIX-64-P9-NEXT:    stdu r1, -112(r1)
+; CHECK-AIX-64-P9-NEXT:    std r0, 128(r1)
+; CHECK-AIX-64-P9-NEXT:    bl .memcmp[PR]
+; CHECK-AIX-64-P9-NEXT:    nop
+; CHECK-AIX-64-P9-NEXT:    addi r1, r1, 112
+; CHECK-AIX-64-P9-NEXT:    ld r0, 16(r1)
+; CHECK-AIX-64-P9-NEXT:    mtlr r0
+; CHECK-AIX-64-P9-NEXT:    blr
+entry:
+  %call = tail call i32 @memcmp(ptr noundef %ptr1, ptr noundef %ptr2, i64 noundef %num)
+  ret i32 %call
+}
+
+declare i32 @memcmp(ptr noundef captures(none), ptr noundef captures(none), i64 noundef) nounwind
+
+
diff --git a/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll b/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
index 5486dc02faf90..232014db9a012 100644
--- a/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
+++ b/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
@@ -10,28 +10,13 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK
 
 define dso_local noundef range(i8 -1, 1) <16 x i8> @_Z5v00FFv() {
-; CHECK:      L..CPI0_0:
-; CHECK-NEXT:   .byte   255                             # 0xff
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NOT:      L..CPI0_0:
+; CHECK-NOT:   .byte   255                             # 0xff
+; CHECK-NOT:   .byte   0                               # 0x0
 
 ; CHECK-LABEL: _Z5v00FFv:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lwz r3, L..C0(r2) # %const.0
-; CHECK-NEXT:    lxv vs34, 0(r3)
+; CHECK-NEXT:    mtvsrbmi v2, 1
 ; CHECK-NEXT:    blr
 entry:
   ret <16 x i8> <i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
diff --git a/llvm/test/CodeGen/PowerPC/nofpexcept.ll b/llvm/test/CodeGen/PowerPC/nofpexcept.ll
index f764dfc2d2498..14c6e68fb9226 100644
--- a/llvm/test/CodeGen/PowerPC/nofpexcept.ll
+++ b/llvm/test/CodeGen/PowerPC/nofpexcept.ll
@@ -81,35 +81,35 @@ define void @fptoint_nofpexcept(ppc_fp128 %p, fp128 %m, ptr %addr1, ptr %addr2)
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:f8rc = COPY $f2
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:f8rc = COPY $f1
   ; CHECK-NEXT:   [[XSCVQPSWZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPSWZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vslrc = COPY [[XSCVQPSWZ]]
-  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vsfrc = COPY [[COPY5]].sub_64
-  ; CHECK-NEXT:   STIWX killed [[COPY6]], $zero8, [[COPY1]]
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vslrc = COPY killed [[XSCVQPSWZ]]
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vsfrc = COPY killed [[COPY5]].sub_64
+  ; CHECK-NEXT:   STIWX killed [[COPY6]], $zero8, [[COPY1]] :: (volatile store (s32) into %ir.addr1)
   ; CHECK-NEXT:   [[XSCVQPUWZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPUWZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vslrc = COPY [[XSCVQPUWZ]]
-  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vsfrc = COPY [[COPY7]].sub_64
-  ; CHECK-NEXT:   STIWX killed [[COPY8]], $zero8, [[COPY1]]
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vslrc = COPY killed [[XSCVQPUWZ]]
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vsfrc = COPY killed [[COPY7]].sub_64
+  ; CHECK-NEXT:   STIWX killed [[COPY8]], $zero8, [[COPY1]] :: (volatile store (s32) into %ir.addr1)
   ; CHECK-NEXT:   [[XSCVQPSDZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPSDZ [[COPY2]]
   ; CHECK-NEXT:   [[MFVRD:%[0-9]+]]:g8rc = nofpexcept MFVRD killed [[XSCVQPSDZ]]
   ; CHECK-NEXT:   [[XSCVQPSDZ1:%[0-9]+]]:vrrc = nofpexcept XSCVQPSDZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vslrc = COPY [[XSCVQPSDZ1]]
-  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vfrc = COPY [[COPY9]].sub_64
-  ; CHECK-NEXT:   STXSD killed [[COPY10]], 0, [[COPY]]
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vslrc = COPY killed [[XSCVQPSDZ1]]
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vfrc = COPY killed [[COPY9]].sub_64
+  ; CHECK-NEXT:   STXSD killed [[COPY10]], 0, [[COPY]] :: (volatile store (s64) into %ir.addr2)
   ; CHECK-NEXT:   [[XSCVQPUDZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPUDZ [[COPY2]]
   ; CHECK-NEXT:   [[MFVRD1:%[0-9]+]]:g8rc = nofpexcept MFVRD killed [[XSCVQPUDZ]]
   ; CHECK-NEXT:   [[XSCVQPUDZ1:%[0-9]+]]:vrrc = nofpexcept XSCVQPUDZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vslrc = COPY [[XSCVQPUDZ1]]
-  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vfrc = COPY [[COPY11]].sub_64
-  ; CHECK-NEXT:   STXSD killed [[COPY12]], 0, [[COPY]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vslrc = COPY killed [[XSCVQPUDZ1]]
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vfrc = COPY killed [[COPY11]].sub_64
+  ; CHECK-NEXT:   STXSD killed [[COPY12]], 0, [[COPY]] :: (volatile store (s64) into %ir.addr2)
   ; CHECK-NEXT:   [[MFFS:%[0-9]+]]:f8rc = MFFS implicit $rm
   ; CHECK-NEXT:   MTFSB1 31, implicit-def $rm, implicit-def $rm
   ; CHECK-NEXT:   MTFSB0 30, implicit-def $rm, implicit-def $rm
   ; CHECK-NEXT:   [[FADD:%[0-9]+]]:f8rc = nofpexcept FADD [[COPY3]], [[COPY4]], implicit $rm
   ; CHECK-NEXT:   MTFSFb 1, [[MFFS]], implicit-def $rm
   ; CHECK-NEXT:   [[XSCVDPSXWS:%[0-9]+]]:vsfrc = nofpexcept XSCVDPSXWS killed [[FADD]], implicit $rm
-  ; CHECK-NEXT:   STIWX killed [[XSCVDPSXWS]], $zero8, [[COPY1]]
+  ; CHECK-NEXT:   STIWX killed [[XSCVDPSXWS]], $zero8, [[COPY1]] :: (volatile store (s32) into %ir.addr1)
   ; CHECK-NEXT:   [[ADDIStocHA8_:%[0-9]+]]:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
   ; CHECK-NEXT:   [[DFLOADf32_:%[0-9]+]]:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0, killed [[ADDIStocHA8_]] :: (load (s32) from constant-pool)
-  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:f8rc = COPY [[DFLOADf32_]]
+  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:f8rc = COPY killed [[DFLOADf32_]]
   ; CHECK-NEXT:   [[FCMPOD:%[0-9]+]]:crrc = FCMPOD [[COPY4]], [[COPY13]]
   ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:crbitrc = COPY [[FCMPOD]].sub_eq
   ; CHECK-NEXT:   [[XXLXORdpz:%[0-9]+]]:f8rc = XXLXORdpz
@@ -174,8 +174,8 @@ define signext i32 @q_to_i32(fp128 %m) #0 {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vrrc = COPY $v2
   ; CHECK-NEXT:   [[XSCVQPSWZ:%[0-9]+]]:vrrc = XSCVQPSWZ [[COPY]]
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY [[XSCVQPSWZ]]
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY [[COPY1]].sub_64
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY killed [[XSCVQPSWZ]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY killed [[COPY1]].sub_64
   ; CHECK-NEXT:   [[MFVSRWZ:%[0-9]+]]:gprc = MFVSRWZ killed [[COPY2]]
   ; CHECK-NEXT:   [[EXTSW_32_64_:%[0-9]+]]:g8rc = EXTSW_32_64 killed [[MFVSRWZ]]
   ; CHECK-NEXT:   $x3 = COPY [[EXTSW_32_64_]]
@@ -222,8 +222,8 @@ define zeroext i32 @q_to_u32(fp128 %m) #0 {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vrrc = COPY $v2
   ; CHECK-NEXT:   [[XSCVQPUWZ:%[0-9]+]]:vrrc = XSCVQPUWZ [[COPY]]
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY [[XSCVQPUWZ]]
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY [[COPY1]].sub_64
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY killed [[XSCVQPUWZ]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY killed [[COPY1]].sub_64
   ; CHECK-NEXT:   [[MFVSRWZ:%[0-9]+]]:gprc = MFVSRWZ killed [[COPY2]]
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:g8rc = IMPLICIT_DEF
   ; CHECK-NEXT:   [[INSERT_SUBREG:%[0-9]+]]:g8rc = INSERT_SUBREG [[DEF]], killed [[MFVSRWZ]], %subreg.sub_32
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
index b540948b20f75..821cfd00dcd07 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
@@ -764,13 +764,8 @@ define <16 x i8> @sub_absv_8_ext(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr
 ;
 ; CHECK-PWR7-LABEL: sub_absv_8_ext:
 ; CHECK-PWR7:       # %bb.0: # %entry
-; CHECK-PWR7-NEXT:    stdu r1, -512(r1)
-; CHECK-PWR7-NEXT:    .cfi_def_cfa_offset 512
-; CHECK-PWR7-NEXT:    .cfi_offset r14, -144
-; CHECK-PWR7-NEXT:    .cfi_offset r15, -136
-; CHECK-PWR7-NEXT:    .cfi_offset r16, -128
-; CHECK-PWR7-NEXT:    .cfi_offset r17, -120
-; CHECK-PWR7-NEXT:    .cfi_offset r18, -112
+; CHECK-PWR7-NEXT:    stdu r1, -448(r1)
+; CHECK-PWR7-NEXT:    .cfi_def_cfa_offset 448
 ; CHECK-PWR7-NEXT:    .cfi_offset r19, -104
 ; CHECK-PWR7-NEXT:    .cfi_offset r20, -96
 ; CHECK-PWR7-NEXT:    .cfi_offset r21, -88
@@ -783,244 +778,258 @@ define <16 x i8> @sub_absv_8_ext(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr
 ; CHECK-PWR7-NEXT:    .cfi_offset r28, -32
 ; CHECK-PWR7-NEXT:    .cfi_offset r29, -24
 ; CHECK-PWR7-NEXT:    .cfi_offset r30, -16
-; CHECK-PWR7-NEXT:    .cfi_offset r31, -8
-; CHECK-PWR7-NEXT:    .cfi_offset r2, -152
-; CHECK-PWR7-NEXT:    addi r3, r1, 320
-; CHECK-PWR7-NEXT:    std r14, 368(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r15, 376(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r16, 384(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r17, 392(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r18, 400(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r19, 408(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r20, 416(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r21, 424(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r22, 432(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r23, 440(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r24, 448(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r25, 456(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r26, 464(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r27, 472(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r28, 480(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r29, 488(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r30, 496(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r31, 504(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r2, 360(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    addi r3, r1, 304
+; CHECK-PWR7-NEXT:    std r19, 344(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r20, 352(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r21, 360(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r22, 368(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r23, 376(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r24, 384(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r25, 392(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r26, 400(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r27, 408(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r28, 416(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r29, 424(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r30, 432(r1) # 8-byte Folded Spill
 ; CHECK-PWR7-NEXT:    stxvw4x v2, 0, r3
-; CHECK-PWR7-NEXT:    lbz r3, 320(r1)
-; CHECK-PWR7-NEXT:    addi r4, r1, 336
-; CHECK-PWR7-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
-; CHECK-PWR7-NEXT:    stxvw4x v3, 0, r4
-; CHECK-PWR7-NEXT:    lbz r15, 334(r1)
-; CHECK-PWR7-NEXT:    lbz r14, 350(r1)
-; CHECK-PWR7-NEXT:    lbz r31, 335(r1)
-; CHECK-PWR7-NEXT:    lbz r2, 351(r1)
-; CHECK-PWR7-NEXT:    sub r15, r15, r14
-; CHECK-PWR7-NEXT:    sub r14, r31, r2
-; CHECK-PWR7-NEXT:    srawi r2, r14, 31
-; CHECK-PWR7-NEXT:    xor r14, r14, r2
-; CHECK-PWR7-NEXT:    lbz r3, 333(r1)
-; CHECK-PWR7-NEXT:    lbz r19, 331(r1)
-; CHECK-PWR7-NEXT:    lbz r18, 347(r1)
-; CHECK-PWR7-NEXT:    sub r19, r19, r18
-; CHECK-PWR7-NEXT:    lbz r17, 332(r1)
-; CHECK-PWR7-NEXT:    lbz r16, 348(r1)
-; CHECK-PWR7-NEXT:    sub r17, r17, r16
-; CHECK-PWR7-NEXT:    lbz r23, 329(r1)
-; CHECK-PWR7-NEXT:    sub r14, r14, r2
-; CHECK-PWR7-NEXT:    lbz r2, 349(r1)
-; CHECK-PWR7-NEXT:    lbz r22, 345(r1)
-; CHECK-PWR7-NEXT:    lbz r4, 336(r1)
-; CHECK-PWR7-NEXT:    lbz r5, 321(r1)
-; CHECK-PWR7-NEXT:    lbz r6, 337(r1)
-; CHECK-PWR7-NEXT:    lbz r7, 322(r1)
-; CHECK-PWR7-NEXT:    lbz r8, 338(r1)
-; CHECK-PWR7-NEXT:    lbz r9, 323(r1)
-; CHECK-PWR7-NEXT:    lbz r10, 339(r1)
-; CHECK-PWR7-NEXT:    lbz r11, 324(r1)
-; CHECK-PWR7-NEXT:    lbz r12, 340(r1)
-; CHECK-PWR7-NEXT:    lbz r0, 325(r1)
-; CHECK-PWR7-NEXT:    lbz r30, 341(r1)
-; CHECK-PWR7-NEXT:    lbz r29, 326(r1)
-; CHECK-PWR7-NEXT:    lbz r28, 342(r1)
-; CHECK-PWR7-NEXT:    lbz r27, 327(r1)
-; CHECK-PWR7-NEXT:    lbz r26, 343(r1)
-; CHECK-PWR7-NEXT:    sub r3, r3, r2
-; CHECK-PWR7-NEXT:    lbz r25, 328(r1)
-; CHECK-PWR7-NEXT:    lbz r24, 344(r1)
-; CHECK-PWR7-NEXT:    lbz r21, 330(r1)
-; CHECK-PWR7-NEXT:    lbz r20, 346(r1)
+; CHECK-PWR7-NEXT:    addi r3, r1, 320
+; CHECK-PWR7-NEXT:    lbz r7, 304(r1)
+; CHECK-PWR7-NEXT:    stxvw4x v3, 0, r3
+; CHECK-PWR7-NEXT:    lbz r8, 320(r1)
+; CHECK-PWR7-NEXT:    lbz r9, 305(r1)
+; CHECK-PWR7-NEXT:    lbz r10, 321(r1)
+; CHECK-PWR7-NEXT:    lbz r26, 325(r1)
+; CHECK-PWR7-NEXT:    clrlwi r7, r7, 24
+; CHECK-PWR7-NEXT:    clrlwi r8, r8, 24
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r10, r10, 24
+; CHECK-PWR7-NEXT:    lbz r11, 306(r1)
+; CHECK-PWR7-NEXT:    lbz r12, 322(r1)
+; CHECK-PWR7-NEXT:    lbz r23, 314(r1)
+; CHECK-PWR7-NEXT:    clrlwi r22, r26, 24
+; CHECK-PWR7-NEXT:    lbz r26, 330(r1)
+; CHECK-PWR7-NEXT:    sub r8, r7, r8
+; CHECK-PWR7-NEXT:    lbz r7, 315(r1)
+; CHECK-PWR7-NEXT:    sub r20, r9, r10
+; CHECK-PWR7-NEXT:    lbz r9, 331(r1)
+; CHECK-PWR7-NEXT:    lbz r0, 307(r1)
+; CHECK-PWR7-NEXT:    lbz r30, 323(r1)
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    clrlwi r12, r12, 24
+; CHECK-PWR7-NEXT:    clrlwi r23, r23, 24
+; CHECK-PWR7-NEXT:    clrlwi r21, r26, 24
+; CHECK-PWR7-NEXT:    clrlwi r7, r7, 24
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r0, r0, 24
+; CHECK-PWR7-NEXT:    clrlwi r30, r30, 24
+; CHECK-PWR7-NEXT:    lbz r29, 308(r1)
+; CHECK-PWR7-NEXT:    lbz r28, 324(r1)
+; CHECK-PWR7-NEXT:    lbz r27, 309(r1)
+; CHECK-PWR7-NEXT:    lbz r25, 310(r1)
+; CHECK-PWR7-NEXT:    lbz r24, 326(r1)
+; CHECK-PWR7-NEXT:    sub r19, r11, r12
+; CHECK-PWR7-NEXT:    sub r11, r23, r21
+; CHECK-PWR7-NEXT:    sub r9, r7, r9
+; CHECK-PWR7-NEXT:    sub r26, r0, r30
+; CHECK-PWR7-NEXT:    srawi r12, r11, 31
+; CHECK-PWR7-NEXT:    srawi r0, r9, 31
+; CHECK-PWR7-NEXT:    lbz r3, 312(r1)
+; CHECK-PWR7-NEXT:    clrlwi r29, r29, 24
+; CHECK-PWR7-NEXT:    clrlwi r28, r28, 24
+; CHECK-PWR7-NEXT:    clrlwi r27, r27, 24
+; CHECK-PWR7-NEXT:    clrlwi r25, r25, 24
+; CHECK-PWR7-NEXT:    clrlwi r24, r24, 24
+; CHECK-PWR7-NEXT:    xor r11, r11, r12
+; CHECK-PWR7-NEXT:    xor r9, r9, r0
+; CHECK-PWR7-NEXT:    sub r28, r29, r28
+; CHECK-PWR7-NEXT:    sub r30, r27, r22
+; CHECK-PWR7-NEXT:    sub r29, r25, r24
+; CHECK-PWR7-NEXT:    sub r27, r11, r12
+; CHECK-PWR7-NEXT:    sub r24, r9, r0
+; CHECK-PWR7-NEXT:    lbz r9, 316(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 332(r1)
+; CHECK-PWR7-NEXT:    lbz r4, 328(r1)
+; CHECK-PWR7-NEXT:    lbz r5, 311(r1)
+; CHECK-PWR7-NEXT:    lbz r6, 327(r1)
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    clrlwi r3, r3, 24
+; CHECK-PWR7-NEXT:    clrlwi r4, r4, 24
+; CHECK-PWR7-NEXT:    clrlwi r5, r5, 24
+; CHECK-PWR7-NEXT:    clrlwi r6, r6, 24
+; CHECK-PWR7-NEXT:    sub r3, r3, r4
 ; CHECK-PWR7-NEXT:    sub r5, r5, r6
-; CHECK-PWR7-NEXT:    srawi r18, r3, 31
-; CHECK-PWR7-NEXT:    sub r7, r7, r8
-; CHECK-PWR7-NEXT:    sub r9, r9, r10
-; CHECK-PWR7-NEXT:    sub r11, r11, r12
-; CHECK-PWR7-NEXT:    sub r0, r0, r30
-; CHECK-PWR7-NEXT:    sub r29, r29, r28
-; CHECK-PWR7-NEXT:    sub r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r25, r25, r24
-; CHECK-PWR7-NEXT:    srawi r31, r15, 31
-; CHECK-PWR7-NEXT:    ld r2, 360(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    xor r3, r3, r18
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    srawi r4, r3, 31
 ; CHECK-PWR7-NEXT:    srawi r6, r5, 31
-; CHECK-PWR7-NEXT:    srawi r8, r7, 31
-; CHECK-PWR7-NEXT:    srawi r10, r9, 31
-; CHECK-PWR7-NEXT:    srawi r12, r11, 31
-; CHECK-PWR7-NEXT:    srawi r30, r0, 31
-; CHECK-PWR7-NEXT:    sub r3, r3, r18
-; CHECK-PWR7-NEXT:    srawi r18, r19, 31
-; CHECK-PWR7-NEXT:    srawi r28, r29, 31
-; CHECK-PWR7-NEXT:    ld r16, 384(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    srawi r26, r27, 31
-; CHECK-PWR7-NEXT:    srawi r24, r25, 31
-; CHECK-PWR7-NEXT:    xor r19, r19, r18
-; CHECK-PWR7-NEXT:    xor r15, r15, r31
+; CHECK-PWR7-NEXT:    xor r3, r3, r4
+; CHECK-PWR7-NEXT:    sldi r27, r27, 56
 ; CHECK-PWR7-NEXT:    xor r5, r5, r6
-; CHECK-PWR7-NEXT:    std r3, 272(r1)
-; CHECK-PWR7-NEXT:    std r3, 280(r1)
-; CHECK-PWR7-NEXT:    srawi r3, r17, 31
-; CHECK-PWR7-NEXT:    sub r19, r19, r18
-; CHECK-PWR7-NEXT:    xor r7, r7, r8
-; CHECK-PWR7-NEXT:    sub r15, r15, r31
-; CHECK-PWR7-NEXT:    xor r17, r17, r3
-; CHECK-PWR7-NEXT:    xor r9, r9, r10
-; CHECK-PWR7-NEXT:    xor r11, r11, r12
-; CHECK-PWR7-NEXT:    xor r0, r0, r30
-; CHECK-PWR7-NEXT:    xor r29, r29, r28
-; CHECK-PWR7-NEXT:    xor r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r3, r17, r3
-; CHECK-PWR7-NEXT:    xor r25, r25, r24
-; CHECK-PWR7-NEXT:    sub r25, r25, r24
-; CHECK-PWR7-NEXT:    sub r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r29, r29, r28
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    sub r3, r3, r4
+; CHECK-PWR7-NEXT:    sldi r24, r24, 56
 ; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    sub r0, r0, r30
-; CHECK-PWR7-NEXT:    sub r11, r11, r12
-; CHECK-PWR7-NEXT:    sub r9, r9, r10
-; CHECK-PWR7-NEXT:    sub r7, r7, r8
-; CHECK-PWR7-NEXT:    sub r5, r5, r6
-; CHECK-PWR7-NEXT:    sldi r14, r14, 56
-; CHECK-PWR7-NEXT:    sldi r15, r15, 56
-; CHECK-PWR7-NEXT:    ld r31, 504(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r3, 256(r1)
-; CHECK-PWR7-NEXT:    std r3, 264(r1)
-; CHECK-PWR7-NEXT:    sldi r3, r19, 56
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r27, 208(r1)
+; CHECK-PWR7-NEXT:    sub r4, r5, r6
+; CHECK-PWR7-NEXT:    std r27, 216(r1)
+; CHECK-PWR7-NEXT:    srawi r27, r29, 31
+; CHECK-PWR7-NEXT:    lbz r10, 313(r1)
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
+; CHECK-PWR7-NEXT:    std r24, 224(r1)
+; CHECK-PWR7-NEXT:    lbz r22, 329(r1)
+; CHECK-PWR7-NEXT:    std r24, 232(r1)
+; CHECK-PWR7-NEXT:    srawi r24, r30, 31
+; CHECK-PWR7-NEXT:    ld r21, 360(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r23, r9, r11
+; CHECK-PWR7-NEXT:    lbz r9, 317(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 333(r1)
+; CHECK-PWR7-NEXT:    xor r29, r29, r27
+; CHECK-PWR7-NEXT:    std r3, 176(r1)
+; CHECK-PWR7-NEXT:    std r3, 184(r1)
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
+; CHECK-PWR7-NEXT:    sldi r23, r23, 56
+; CHECK-PWR7-NEXT:    xor r30, r30, r24
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    sub r4, r30, r24
+; CHECK-PWR7-NEXT:    ld r30, 432(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 160(r1)
+; CHECK-PWR7-NEXT:    std r3, 168(r1)
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    sub r3, r29, r27
+; CHECK-PWR7-NEXT:    std r23, 240(r1)
+; CHECK-PWR7-NEXT:    ld r29, 424(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r23, 248(r1)
+; CHECK-PWR7-NEXT:    ld r27, 408(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r23, r28, 31
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    xor r28, r28, r23
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
+; CHECK-PWR7-NEXT:    std r3, 144(r1)
+; CHECK-PWR7-NEXT:    ld r24, 384(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 152(r1)
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
+; CHECK-PWR7-NEXT:    sub r25, r9, r11
+; CHECK-PWR7-NEXT:    lbz r9, 318(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 334(r1)
+; CHECK-PWR7-NEXT:    std r3, 128(r1)
 ; CHECK-PWR7-NEXT:    sldi r25, r25, 56
-; CHECK-PWR7-NEXT:    sldi r27, r27, 56
-; CHECK-PWR7-NEXT:    std r3, 240(r1)
-; CHECK-PWR7-NEXT:    std r3, 248(r1)
-; CHECK-PWR7-NEXT:    sub r3, r23, r22
-; CHECK-PWR7-NEXT:    srawi r23, r3, 31
-; CHECK-PWR7-NEXT:    sub r22, r21, r20
-; CHECK-PWR7-NEXT:    srawi r21, r22, 31
-; CHECK-PWR7-NEXT:    sldi r29, r29, 56
-; CHECK-PWR7-NEXT:    sldi r0, r0, 56
-; CHECK-PWR7-NEXT:    sldi r11, r11, 56
-; CHECK-PWR7-NEXT:    xor r3, r3, r23
-; CHECK-PWR7-NEXT:    xor r22, r22, r21
-; CHECK-PWR7-NEXT:    sldi r9, r9, 56
-; CHECK-PWR7-NEXT:    sldi r7, r7, 56
-; CHECK-PWR7-NEXT:    sldi r5, r5, 56
-; CHECK-PWR7-NEXT:    ld r30, 496(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r28, 480(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    sub r3, r3, r23
-; CHECK-PWR7-NEXT:    sub r22, r22, r21
-; CHECK-PWR7-NEXT:    std r14, 304(r1)
-; CHECK-PWR7-NEXT:    ld r26, 464(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 136(r1)
+; CHECK-PWR7-NEXT:    sub r3, r28, r23
 ; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    sldi r22, r22, 56
-; CHECK-PWR7-NEXT:    ld r24, 448(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r23, 440(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r14, 312(r1)
-; CHECK-PWR7-NEXT:    std r15, 288(r1)
-; CHECK-PWR7-NEXT:    std r3, 208(r1)
-; CHECK-PWR7-NEXT:    std r3, 216(r1)
-; CHECK-PWR7-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r15, 296(r1)
-; CHECK-PWR7-NEXT:    ld r21, 424(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r20, 416(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r22, 224(r1)
-; CHECK-PWR7-NEXT:    std r22, 232(r1)
-; CHECK-PWR7-NEXT:    sub r4, r3, r4
-; CHECK-PWR7-NEXT:    std r25, 192(r1)
-; CHECK-PWR7-NEXT:    ld r22, 432(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r19, 408(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    srawi r3, r4, 31
-; CHECK-PWR7-NEXT:    std r25, 200(r1)
-; CHECK-PWR7-NEXT:    ld r25, 456(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r27, 176(r1)
-; CHECK-PWR7-NEXT:    std r27, 184(r1)
-; CHECK-PWR7-NEXT:    xor r4, r4, r3
-; CHECK-PWR7-NEXT:    std r29, 160(r1)
-; CHECK-PWR7-NEXT:    ld r27, 472(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r29, 168(r1)
-; CHECK-PWR7-NEXT:    std r0, 144(r1)
-; CHECK-PWR7-NEXT:    sub r3, r4, r3
-; CHECK-PWR7-NEXT:    std r0, 152(r1)
-; CHECK-PWR7-NEXT:    ld r29, 488(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r18, 400(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 112(r1)
+; CHECK-PWR7-NEXT:    ld r28, 416(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    clrlwi r10, r10, 24
+; CHECK-PWR7-NEXT:    std r25, 256(r1)
+; CHECK-PWR7-NEXT:    std r25, 264(r1)
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    srawi r25, r26, 31
+; CHECK-PWR7-NEXT:    xor r26, r26, r25
+; CHECK-PWR7-NEXT:    ld r23, 376(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r3, 120(r1)
+; CHECK-PWR7-NEXT:    sub r4, r26, r25
+; CHECK-PWR7-NEXT:    clrlwi r22, r22, 24
+; CHECK-PWR7-NEXT:    srawi r7, r8, 31
+; CHECK-PWR7-NEXT:    sub r10, r10, r22
+; CHECK-PWR7-NEXT:    ld r26, 400(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
+; CHECK-PWR7-NEXT:    srawi r22, r10, 31
+; CHECK-PWR7-NEXT:    xor r8, r8, r7
+; CHECK-PWR7-NEXT:    xor r10, r10, r22
+; CHECK-PWR7-NEXT:    sub r10, r10, r22
+; CHECK-PWR7-NEXT:    ld r25, 392(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r12, r9, r11
+; CHECK-PWR7-NEXT:    lbz r9, 319(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 335(r1)
+; CHECK-PWR7-NEXT:    std r3, 96(r1)
+; CHECK-PWR7-NEXT:    sldi r12, r12, 56
+; CHECK-PWR7-NEXT:    std r3, 104(r1)
+; CHECK-PWR7-NEXT:    ld r22, 368(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sldi r10, r10, 56
+; CHECK-PWR7-NEXT:    std r10, 192(r1)
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    std r12, 272(r1)
+; CHECK-PWR7-NEXT:    std r12, 280(r1)
+; CHECK-PWR7-NEXT:    srawi r12, r19, 31
+; CHECK-PWR7-NEXT:    xor r0, r19, r12
+; CHECK-PWR7-NEXT:    ld r19, 344(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r3, r0, r12
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r10, 200(r1)
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
 ; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    std r11, 128(r1)
-; CHECK-PWR7-NEXT:    ld r17, 392(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r11, 136(r1)
-; CHECK-PWR7-NEXT:    std r9, 112(r1)
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    std r3, 80(r1)
+; CHECK-PWR7-NEXT:    std r3, 88(r1)
+; CHECK-PWR7-NEXT:    sldi r9, r9, 56
+; CHECK-PWR7-NEXT:    std r9, 288(r1)
+; CHECK-PWR7-NEXT:    std r9, 296(r1)
+; CHECK-PWR7-NEXT:    srawi r9, r20, 31
+; CHECK-PWR7-NEXT:    xor r11, r20, r9
+; CHECK-PWR7-NEXT:    ld r20, 352(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r4, r11, r9
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
 ; CHECK-PWR7-NEXT:    std r3, 64(r1)
 ; CHECK-PWR7-NEXT:    std r3, 72(r1)
-; CHECK-PWR7-NEXT:    addi r3, r1, 304
-; CHECK-PWR7-NEXT:    std r9, 120(r1)
-; CHECK-PWR7-NEXT:    ld r15, 376(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r7, 96(r1)
-; CHECK-PWR7-NEXT:    std r7, 104(r1)
-; CHECK-PWR7-NEXT:    std r5, 80(r1)
-; CHECK-PWR7-NEXT:    std r5, 88(r1)
-; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
+; CHECK-PWR7-NEXT:    sub r3, r8, r7
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    std r3, 48(r1)
+; CHECK-PWR7-NEXT:    std r3, 56(r1)
 ; CHECK-PWR7-NEXT:    addi r3, r1, 288
-; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
+; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 272
-; CHECK-PWR7-NEXT:    ld r14, 368(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 256
-; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
+; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 240
+; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 224
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    vmrghh v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 224
-; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 208
-; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 192
-; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
+; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
+; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 176
+; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 160
 ; CHECK-PWR7-NEXT:    vmrghb v4, v5, v4
 ; CHECK-PWR7-NEXT:    vmrghh v3, v4, v3
 ; CHECK-PWR7-NEXT:    xxmrghw vs0, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 160
-; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 144
-; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 128
+; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
+; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 112
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 96
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
-; CHECK-PWR7-NEXT:    addi r3, r1, 112
 ; CHECK-PWR7-NEXT:    vmrghh v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 96
-; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 80
-; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 64
+; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
+; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 48
 ; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
 ; CHECK-PWR7-NEXT:    vmrghb v4, v5, v4
 ; CHECK-PWR7-NEXT:    vmrghh v3, v4, v3
 ; CHECK-PWR7-NEXT:    xxmrghw vs1, v3, v2
 ; CHECK-PWR7-NEXT:    xxmrghd v2, vs1, vs0
-; CHECK-PWR7-NEXT:    addi r1, r1, 512
+; CHECK-PWR7-NEXT:    addi r1, r1, 448
 ; CHECK-PWR7-NEXT:    blr
 entry:
   %vecext = extractelement <16 x i8> %a, i32 0
diff --git a/llvm/test/CodeGen/PowerPC/recipest.ll b/llvm/test/CodeGen/PowerPC/recipest.ll
index 96e2c6c3e081a..2598a410b8761 100644
--- a/llvm/test/CodeGen/PowerPC/recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/recipest.ll
@@ -1024,15 +1024,16 @@ define <4 x float> @hoo3_fmf(<4 x float> %a) #1 {
 ; CHECK-P7-NEXT:    vslw 3, 3, 3
 ; CHECK-P7-NEXT:    lvx 0, 0, 3
 ; CHECK-P7-NEXT:    addis 3, 2, .LCPI25_1@toc@ha
-; CHECK-P7-NEXT:    addi 3, 3, .LCPI25_1@toc@l
 ; CHECK-P7-NEXT:    vmaddfp 5, 2, 4, 3
-; CHECK-P7-NEXT:    lvx 1, 0, 3
+; CHECK-P7-NEXT:    addi 3, 3, .LCPI25_1@toc@l
 ; CHECK-P7-NEXT:    vmaddfp 4, 5, 4, 0
-; CHECK-P7-NEXT:    vmaddfp 5, 5, 1, 3
+; CHECK-P7-NEXT:    lvx 0, 0, 3
+; CHECK-P7-NEXT:    vmaddfp 5, 5, 0, 3
 ; CHECK-P7-NEXT:    vmaddfp 3, 5, 4, 3
 ; CHECK-P7-NEXT:    vxor 4, 4, 4
 ; CHECK-P7-NEXT:    vcmpeqfp 2, 2, 4
-; CHECK-P7-NEXT:    vsel 2, 3, 4, 2
+; CHECK-P7-NEXT:    vnot 2, 2
+; CHECK-P7-NEXT:    vand 2, 2, 3
 ; CHECK-P7-NEXT:    blr
 ;
 ; CHECK-P8-LABEL: hoo3_fmf:
diff --git a/llvm/test/CodeGen/PowerPC/sat-add.ll b/llvm/test/CodeGen/PowerPC/sat-add.ll
index d9b22bda85e44..34b703a981105 100644
--- a/llvm/test/CodeGen/PowerPC/sat-add.ll
+++ b/llvm/test/CodeGen/PowerPC/sat-add.ll
@@ -536,12 +536,11 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI34_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI34_0@toc@l
 ; CHECK-NEXT:    lxvd2x 35, 0, 3
 ; CHECK-NEXT:    vaddudm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtud 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, %a
@@ -553,7 +552,6 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI35_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI35_0@toc@l
 ; CHECK-NEXT:    lxvd2x 35, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI35_1@toc@ha
@@ -561,7 +559,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-NEXT:    lxvd2x 36, 0, 3
 ; CHECK-NEXT:    vaddudm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtud 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
@@ -599,9 +597,8 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vaddubm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtub 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %a = add <16 x i8> %x, %y
@@ -640,9 +637,8 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vadduhm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuh 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %a = add <8 x i16> %x, %y
@@ -681,9 +677,8 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vadduwm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuw 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = add <4 x i32> %x, %y
@@ -710,9 +705,8 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddudm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtud 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %a
@@ -725,9 +719,8 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vaddudm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtud 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
   %a = add <2 x i64> %x, %y
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
index 2868669c52ce6..57d4c48a1aaa2 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -47,6 +47,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, xor(B, C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_xor_BC_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_xor_BC_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %xor, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, xor(B, C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_xor_BC_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_xor_BC_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %xor, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_and_BC_4x32:
@@ -86,6 +122,44 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, nor(B, C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector NOR operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector NOR operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, eqv(B, C), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_eqv_BC_and_BC_4x32:
@@ -125,6 +199,44 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, eqv(B, C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, eqv(B, C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(C), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_C_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_C_and_BC_4x32:
@@ -162,6 +274,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, not(C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_C_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_C_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %C, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_C_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_C_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %C, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(B), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_B_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_B_and_BC_4x32:
@@ -198,3 +346,39 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %and
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, not(B), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_B_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_B_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %B, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(B), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_B_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_B_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %B, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %and
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
index 37a0edb14b78f..c366fd5f0a8c2 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -43,6 +43,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_B_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %B
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_B_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %B
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), B) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_B_4x32:
@@ -78,6 +110,40 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, nor(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_B_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector NOR operation
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %B
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_B_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector NOR operation
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %B
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, eqv(B, C), B) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_eqv_BC_B_4x32:
@@ -113,37 +179,38 @@ entry:
   ret <2 x i64> %res
 }
 
-; Function to test ternary(A, not(C), B) for <4 x i32>
-define <4 x i32> @ternary_A_not_C_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
-; CHECK-LABEL: ternary_A_not_C_B_4x32:
+; Function to test ternary(A, eqv(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_B_16x8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxleqv v5, v5, v5
-; CHECK-NEXT:    xxlnor vs0, v4, v4
-; CHECK-NEXT:    vslw v2, v2, v5
-; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
 ; CHECK-NEXT:    xxsel v2, v3, vs0, v2
 ; CHECK-NEXT:    blr
 entry:
-  %not = xor <4 x i32> %C, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
-  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %B
-  ret <4 x i32> %res
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %B
+  ret <16 x i8> %res
 }
 
-; Function to test ternary(A, not(C), B) for <2 x i64>
-define <2 x i64> @ternary_A_not_C_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
-; CHECK-LABEL: ternary_A_not_C_B_2x64:
+; Function to test ternary(A, eqv(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_B_8x16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxlxor v5, v5, v5
-; CHECK-NEXT:    xxlnor vs0, v4, v4
-; CHECK-NEXT:    xxsplti32dx v5, 1, 63
-; CHECK-NEXT:    vsld v2, v2, v5
-; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
 ; CHECK-NEXT:    xxsel v2, v3, vs0, v2
 ; CHECK-NEXT:    blr
 entry:
-  %not = xor <2 x i64> %C, <i64 -1, i64 -1>  ; Vector not operation
-  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %B
-  ret <2 x i64> %res
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %B
+  ret <8 x i16> %res
 }
 
 ; Function to test ternary(A, nand(B, C), B) for <4 x i32>
@@ -180,3 +247,37 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %B
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nand(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_nand_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_B_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %nand = xor <16 x i8> %and, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector nand operation
+  %res = select <16 x i1> %A, <16 x i8> %nand, <16 x i8> %B
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nand(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_nand_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_B_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %nand = xor <8 x i16> %and, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector nand operation
+  %res = select <8 x i1> %A, <8 x i16> %nand, <8 x i16> %B
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
index 411aa27a61861..f70f1d093f069 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -43,6 +43,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %C
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), C) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_C_4x32:
@@ -78,6 +110,40 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, nor(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector NOR operation
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector NOR operation
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %C
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, eqv(B, C), C) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_eqv_BC_C_4x32:
@@ -113,6 +179,40 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, eqv(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, eqv(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %C
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nand(B, C), C) for <4 x i32>
 define <4 x i32> @ternary_A_nand_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nand_BC_C_4x32:
@@ -147,3 +247,37 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %C
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nand(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_nand_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %nand = xor <16 x i8> %and, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector nand operation
+  %res = select <16 x i1> %A, <16 x i8> %nand, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nand(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_nand_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %nand = xor <8 x i16> %and, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector nand operation
+  %res = select <8 x i1> %A, <8 x i16> %nand, <8 x i16> %C
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
index 1ad7e95e3682e..861789f4d3261 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
@@ -47,6 +47,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, B, or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_B_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_B_or_BC_4x32:
@@ -80,6 +116,37 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, B, or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_B_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_B_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %B, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, B, or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_B_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_B_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %B, <8 x i16> %or
+  ret <8 x i16> %res
+}
 
 ; Function to test ternary(A, C, or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_C_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
@@ -114,6 +181,37 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, C, or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_C_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_C_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %C, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, C, or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_C_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_C_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %C, <8 x i16> %or
+  ret <8 x i16> %res
+}
 
 ; Function to test ternary(A, eqv(B,C), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
@@ -154,6 +252,44 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, eqv(B,C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, eqv(B,C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(C), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_C_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_C_or_BC_4x32:
@@ -191,6 +327,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, not(C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_C_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_C_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %C, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_C_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_C_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %C, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(B), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_B_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_B_or_BC_4x32:
@@ -228,6 +400,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, not(B), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_B_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_B_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %B, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(B), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_B_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_B_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %B, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nand(B,C), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_nand_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nand_BC_or_BC_4x32:
@@ -266,3 +474,41 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %or
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nand(B,C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_nand_BC_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %nand = xor <16 x i8> %and, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector nand operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %nand, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nand(B,C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_nand_BC_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %nand = xor <8 x i16> %and, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector nand operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %nand, <8 x i16> %or
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
index f8a8b6e9a0486..a3fdc905cb52c 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -47,6 +47,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, B, xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_B_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_B_xor_BC_4x32:
@@ -80,6 +116,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, B, xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_B_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_B_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %B, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, B, xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_B_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_B_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %B, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, C, xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_C_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_C_xor_BC_4x32:
@@ -113,6 +181,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, C, xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_C_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_C_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %C, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, C, xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_C_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_C_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %C, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, or(B, C), xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_or_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_or_BC_xor_BC_4x32:
@@ -150,6 +250,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, or(B, C), xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_or_BC_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_or_BC_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %or, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, or(B, C), xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_or_BC_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_or_BC_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %or, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_xor_BC_4x32:
@@ -188,3 +324,41 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nor, <2 x i64> %xor
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nor(B, C), xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>    ; vector nor operation
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>    ; vector nor operation
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %xor
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
index 88413291c26cd..7f387a763b6da 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
@@ -972,19 +972,19 @@ define i1 @fpclass(float %x) {
 ; RV32I-NEXT:    slli a2, a0, 1
 ; RV32I-NEXT:    lui a3, 2048
 ; RV32I-NEXT:    lui a4, 1046528
-; RV32I-NEXT:    srli a2, a2, 1
+; RV32I-NEXT:    srli a5, a2, 1
 ; RV32I-NEXT:    addi a3, a3, -1
-; RV32I-NEXT:    addi a5, a2, -1
+; RV32I-NEXT:    xor a0, a0, a5
+; RV32I-NEXT:    xor a6, a5, a1
+; RV32I-NEXT:    sltu a1, a1, a5
+; RV32I-NEXT:    add a4, a5, a4
+; RV32I-NEXT:    addi a5, a5, -1
 ; RV32I-NEXT:    sltu a3, a5, a3
 ; RV32I-NEXT:    lui a5, 520192
-; RV32I-NEXT:    xor a0, a0, a2
-; RV32I-NEXT:    add a4, a2, a4
 ; RV32I-NEXT:    sltu a4, a4, a5
-; RV32I-NEXT:    xor a5, a2, a1
-; RV32I-NEXT:    sltu a1, a1, a2
 ; RV32I-NEXT:    seqz a2, a2
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    seqz a5, a5
+; RV32I-NEXT:    seqz a5, a6
 ; RV32I-NEXT:    and a3, a3, a0
 ; RV32I-NEXT:    or a2, a2, a5
 ; RV32I-NEXT:    and a0, a4, a0
@@ -1000,19 +1000,19 @@ define i1 @fpclass(float %x) {
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    li a3, 1
 ; RV64I-NEXT:    lui a4, 2048
-; RV64I-NEXT:    lui a5, 520192
-; RV64I-NEXT:    srli a2, a2, 33
-; RV64I-NEXT:    addi a6, a4, -1
-; RV64I-NEXT:    xor a0, a0, a2
-; RV64I-NEXT:    subw a3, a2, a3
-; RV64I-NEXT:    sltu a3, a3, a6
-; RV64I-NEXT:    xor a6, a2, a1
-; RV64I-NEXT:    sltu a1, a1, a2
-; RV64I-NEXT:    subw a4, a2, a4
+; RV64I-NEXT:    srli a5, a2, 33
+; RV64I-NEXT:    xor a0, a0, a5
+; RV64I-NEXT:    subw a3, a5, a3
+; RV64I-NEXT:    xor a6, a5, a1
+; RV64I-NEXT:    sltu a1, a1, a5
+; RV64I-NEXT:    subw a5, a5, a4
+; RV64I-NEXT:    addi a4, a4, -1
+; RV64I-NEXT:    sltu a3, a3, a4
+; RV64I-NEXT:    lui a4, 520192
 ; RV64I-NEXT:    seqz a2, a2
 ; RV64I-NEXT:    snez a0, a0
 ; RV64I-NEXT:    seqz a6, a6
-; RV64I-NEXT:    sltu a4, a4, a5
+; RV64I-NEXT:    sltu a4, a5, a4
 ; RV64I-NEXT:    and a3, a3, a0
 ; RV64I-NEXT:    or a2, a2, a6
 ; RV64I-NEXT:    or a1, a2, a1
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index c5188aa1918bf..43e16606aae73 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -107,6 +107,7 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcisync %s -o - | FileCheck --check-prefix=RV32XQCISYNC %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesperf %s -o - | FileCheck --check-prefix=RV32XANDESPERF %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesvbfhcvt %s -o - | FileCheck --check-prefix=RV32XANDESVBFHCVT %s
+; RUN: llc -mtriple=riscv32 -mattr=+xandesvsintload %s -o - | FileCheck --check-prefix=RV32XANDESVSINTLOAD %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesvdot %s -o - | FileCheck --check-prefix=RV32XANDESVDOT %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesvpackfph %s -o - | FileCheck --check-prefix=RV32XANDESVPACKFPH %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zaamo %s -o - | FileCheck --check-prefix=RV32ZAAMO %s
@@ -262,6 +263,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+xtheadvdot %s -o - | FileCheck --check-prefixes=CHECK,RV64XTHEADVDOT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesperf %s -o - | FileCheck --check-prefix=RV64XANDESPERF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesvbfhcvt %s -o - | FileCheck --check-prefix=RV64XANDESVBFHCVT %s
+; RUN: llc -mtriple=riscv64 -mattr=+xandesvsintload %s -o - | FileCheck --check-prefix=RV64XANDESVSINTLOAD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesvdot %s -o - | FileCheck --check-prefix=RV64XANDESVDOT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesvpackfph %s -o - | FileCheck --check-prefix=RV64XANDESVPACKFPH %s
 ; RUN: llc -mtriple=riscv64 -mattr=+za64rs %s -o - | FileCheck --check-prefixes=CHECK,RV64ZA64RS %s
@@ -337,6 +339,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+sdext  %s -o - | FileCheck --check-prefix=RV64SDEXT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+sdtrig  %s -o - | FileCheck --check-prefix=RV64SDTRIG %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-xqccmp %s -o - | FileCheck --check-prefix=RV64XQCCMP %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-p %s -o - | FileCheck --check-prefix=RV64P %s
 
 
 ; Tests for profile features.
@@ -460,8 +463,9 @@
 ; RV32XQCISYNC: attribute 5, "rv32i2p1_zca1p0_xqcisync0p3"
 ; RV32XANDESPERF: .attribute 5, "rv32i2p1_xandesperf5p0"
 ; RV32XANDESVBFHCVT: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvl32b1p0_xandesvbfhcvt5p0"
+; RV32XANDESVSINTLOAD: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvsintload5p0"
 ; RV32XANDESVDOT: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvdot5p0"
-; RV32XANDESVPACKFPH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfhmin1p0_zvl32b1p0_xandesvpackfph5p0"
+; RV32XANDESVPACKFPH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_xandesvpackfph5p0"
 ; RV32ZAAMO: .attribute 5, "rv32i2p1_zaamo1p0"
 ; RV32ZALRSC: .attribute 5, "rv32i2p1_zalrsc1p0"
 ; RV32ZCA: .attribute 5, "rv32i2p1_zca1p0"
@@ -526,6 +530,7 @@
 ; RV32SUPM: .attribute 5, "rv32i2p1_supm1p0"
 ; RV32SMCTR: .attribute 5, "rv32i2p1_smctr1p0_sscsrind1p0"
 ; RV32SSCTR: .attribute 5, "rv32i2p1_sscsrind1p0_ssctr1p0"
+; RV32P: .attribute 5, "rv32i2p1_p0p14"
 
 ; RV64M: .attribute 5, "rv64i2p1_m2p0_zmmul1p0"
 ; RV64ZMMUL: .attribute 5, "rv64i2p1_zmmul1p0"
@@ -616,8 +621,9 @@
 ; RV64XTHEADVDOT: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_xtheadvdot1p0"
 ; RV64XANDESPERF: .attribute 5, "rv64i2p1_xandesperf5p0"
 ; RV64XANDESVBFHCVT: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvl32b1p0_xandesvbfhcvt5p0"
+; RV64XANDESVSINTLOAD: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvsintload5p0"
 ; RV64XANDESVDOT: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvdot5p0"
-; RV64XANDESVPACKFPH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfhmin1p0_zvl32b1p0_xandesvpackfph5p0"
+; RV64XANDESVPACKFPH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_xandesvpackfph5p0"
 ; RV64ZTSO: .attribute 5, "rv64i2p1_ztso1p0"
 ; RV64ZAAMO: .attribute 5, "rv64i2p1_zaamo1p0"
 ; RV64ZALRSC: .attribute 5, "rv64i2p1_zalrsc1p0"
@@ -688,6 +694,7 @@
 ; RV64SDEXT: .attribute 5, "rv64i2p1_sdext1p0"
 ; RV64SDTRIG: .attribute 5, "rv64i2p1_sdtrig1p0"
 ; RV64XQCCMP: .attribute 5, "rv64i2p1_zca1p0_xqccmp0p3"
+; RV64P: .attribute 5, "rv64i2p1_p0p14"
 
 ; RVI20U32: .attribute 5, "rv32i2p1"
 ; RVI20U64: .attribute 5, "rv64i2p1"
diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll
index af2e31e5d5331..95d330889d2c0 100644
--- a/llvm/test/CodeGen/RISCV/features-info.ll
+++ b/llvm/test/CodeGen/RISCV/features-info.ll
@@ -176,6 +176,7 @@
 ; CHECK-NEXT:   xandesvbfhcvt                    - 'XAndesVBFHCvt' (Andes Vector BFLOAT16 Conversion Extension).
 ; CHECK-NEXT:   xandesvdot                       - 'XAndesVDot' (Andes Vector Dot Product Extension).
 ; CHECK-NEXT:   xandesvpackfph                   - 'XAndesVPackFPH' (Andes Vector Packed FP16 Extension).
+; CHECK-NEXT:   xandesvsintload                  - 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension).
 ; CHECK-NEXT:   xcvalu                           - 'XCValu' (CORE-V ALU Operations).
 ; CHECK-NEXT:   xcvbi                            - 'XCVbi' (CORE-V Immediate Branching).
 ; CHECK-NEXT:   xcvbitmanip                      - 'XCVbitmanip' (CORE-V Bit Manipulation).
@@ -183,6 +184,7 @@
 ; CHECK-NEXT:   xcvmac                           - 'XCVmac' (CORE-V Multiply-Accumulate).
 ; CHECK-NEXT:   xcvmem                           - 'XCVmem' (CORE-V Post-incrementing Load & Store).
 ; CHECK-NEXT:   xcvsimd                          - 'XCVsimd' (CORE-V SIMD ALU).
+; CHECK-NEXT:   xmipscbop                        - 'XMIPSCBOP' (MIPS Software Prefetch).
 ; CHECK-NEXT:   xmipscmov                        - 'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov)).
 ; CHECK-NEXT:   xmipslsp                         - 'XMIPSLSP' (MIPS optimization for hardware load-store bonding).
 ; CHECK-NEXT:   xsfcease                         - 'XSfcease' (SiFive sf.cease Instruction).
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index be9ddc68ce667..ed50042f54ab5 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -1625,18 +1625,18 @@ define i1 @fpclass(float %x) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    slli a1, a0, 1
 ; RV32I-NEXT:    lui a2, 2048
-; RV32I-NEXT:    slti a0, a0, 0
 ; RV32I-NEXT:    lui a3, 522240
 ; RV32I-NEXT:    lui a4, 1046528
-; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    srli a5, a1, 1
 ; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    addi a5, a1, -1
+; RV32I-NEXT:    xor a6, a5, a3
+; RV32I-NEXT:    slt a3, a3, a5
+; RV32I-NEXT:    add a4, a5, a4
+; RV32I-NEXT:    addi a5, a5, -1
 ; RV32I-NEXT:    sltu a2, a5, a2
-; RV32I-NEXT:    xor a5, a1, a3
-; RV32I-NEXT:    slt a3, a3, a1
-; RV32I-NEXT:    add a4, a1, a4
+; RV32I-NEXT:    slti a0, a0, 0
 ; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    seqz a5, a5
+; RV32I-NEXT:    seqz a5, a6
 ; RV32I-NEXT:    srli a4, a4, 24
 ; RV32I-NEXT:    and a2, a2, a0
 ; RV32I-NEXT:    or a1, a1, a5
@@ -1649,29 +1649,29 @@ define i1 @fpclass(float %x) {
 ;
 ; RV64I-LABEL: fpclass:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    slli a1, a0, 33
 ; RV64I-NEXT:    lui a2, 2048
 ; RV64I-NEXT:    lui a3, 522240
 ; RV64I-NEXT:    lui a4, 1046528
-; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    srli a5, a1, 33
 ; RV64I-NEXT:    addi a2, a2, -1
-; RV64I-NEXT:    slti a1, a1, 0
-; RV64I-NEXT:    addi a5, a0, -1
+; RV64I-NEXT:    xor a6, a5, a3
+; RV64I-NEXT:    slt a3, a3, a5
+; RV64I-NEXT:    add a4, a5, a4
+; RV64I-NEXT:    addi a5, a5, -1
 ; RV64I-NEXT:    sltu a2, a5, a2
-; RV64I-NEXT:    xor a5, a0, a3
-; RV64I-NEXT:    slt a3, a3, a0
-; RV64I-NEXT:    add a4, a0, a4
-; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    seqz a5, a5
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    seqz a5, a6
 ; RV64I-NEXT:    srliw a4, a4, 24
-; RV64I-NEXT:    and a2, a2, a1
-; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    and a2, a2, a0
+; RV64I-NEXT:    or a1, a1, a5
 ; RV64I-NEXT:    sltiu a4, a4, 127
-; RV64I-NEXT:    or a0, a0, a2
-; RV64I-NEXT:    or a0, a0, a3
-; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    or a1, a1, a3
+; RV64I-NEXT:    and a0, a4, a0
+; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
   %cmp = call i1 @llvm.is.fpclass.f32(float %x, i32 639)
   ret i1 %cmp
diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index 117e3e4aac45d..246e6a614d6aa 100644
--- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -3292,30 +3292,30 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    mv a1, a0
 ; RV32IF-NEXT:    addi a0, sp, 8
 ; RV32IF-NEXT:    call __fixdfti
-; RV32IF-NEXT:    lw a0, 20(sp)
-; RV32IF-NEXT:    lw a1, 8(sp)
-; RV32IF-NEXT:    lw a2, 12(sp)
+; RV32IF-NEXT:    lw a0, 8(sp)
+; RV32IF-NEXT:    lw a1, 12(sp)
+; RV32IF-NEXT:    lw a2, 20(sp)
 ; RV32IF-NEXT:    lw a3, 16(sp)
-; RV32IF-NEXT:    beqz a0, .LBB47_2
+; RV32IF-NEXT:    beqz a2, .LBB47_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a4, a0, 0
+; RV32IF-NEXT:    slti a4, a2, 0
 ; RV32IF-NEXT:    j .LBB47_3
 ; RV32IF-NEXT:  .LBB47_2:
 ; RV32IF-NEXT:    seqz a4, a3
 ; RV32IF-NEXT:  .LBB47_3: # %entry
 ; RV32IF-NEXT:    xori a3, a3, 1
-; RV32IF-NEXT:    or a3, a3, a0
+; RV32IF-NEXT:    or a3, a3, a2
 ; RV32IF-NEXT:    seqz a3, a3
 ; RV32IF-NEXT:    addi a3, a3, -1
 ; RV32IF-NEXT:    and a3, a3, a4
 ; RV32IF-NEXT:    neg a3, a3
-; RV32IF-NEXT:    and a2, a3, a2
 ; RV32IF-NEXT:    and a1, a3, a1
 ; RV32IF-NEXT:    and a0, a3, a0
-; RV32IF-NEXT:    slti a0, a0, 0
-; RV32IF-NEXT:    addi a3, a0, -1
-; RV32IF-NEXT:    and a0, a3, a1
-; RV32IF-NEXT:    and a1, a3, a2
+; RV32IF-NEXT:    and a2, a3, a2
+; RV32IF-NEXT:    slti a2, a2, 0
+; RV32IF-NEXT:    addi a2, a2, -1
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    and a1, a2, a1
 ; RV32IF-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    .cfi_restore ra
 ; RV32IF-NEXT:    addi sp, sp, 32
@@ -3354,30 +3354,30 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    .cfi_offset ra, -4
 ; RV32IFD-NEXT:    addi a0, sp, 8
 ; RV32IFD-NEXT:    call __fixdfti
-; RV32IFD-NEXT:    lw a0, 20(sp)
-; RV32IFD-NEXT:    lw a1, 8(sp)
-; RV32IFD-NEXT:    lw a2, 12(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    lw a2, 20(sp)
 ; RV32IFD-NEXT:    lw a3, 16(sp)
-; RV32IFD-NEXT:    beqz a0, .LBB47_2
+; RV32IFD-NEXT:    beqz a2, .LBB47_2
 ; RV32IFD-NEXT:  # %bb.1: # %entry
-; RV32IFD-NEXT:    slti a4, a0, 0
+; RV32IFD-NEXT:    slti a4, a2, 0
 ; RV32IFD-NEXT:    j .LBB47_3
 ; RV32IFD-NEXT:  .LBB47_2:
 ; RV32IFD-NEXT:    seqz a4, a3
 ; RV32IFD-NEXT:  .LBB47_3: # %entry
 ; RV32IFD-NEXT:    xori a3, a3, 1
-; RV32IFD-NEXT:    or a3, a3, a0
+; RV32IFD-NEXT:    or a3, a3, a2
 ; RV32IFD-NEXT:    seqz a3, a3
 ; RV32IFD-NEXT:    addi a3, a3, -1
 ; RV32IFD-NEXT:    and a3, a3, a4
 ; RV32IFD-NEXT:    neg a3, a3
-; RV32IFD-NEXT:    and a2, a3, a2
 ; RV32IFD-NEXT:    and a1, a3, a1
 ; RV32IFD-NEXT:    and a0, a3, a0
-; RV32IFD-NEXT:    slti a0, a0, 0
-; RV32IFD-NEXT:    addi a3, a0, -1
-; RV32IFD-NEXT:    and a0, a3, a1
-; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    and a2, a3, a2
+; RV32IFD-NEXT:    slti a2, a2, 0
+; RV32IFD-NEXT:    addi a2, a2, -1
+; RV32IFD-NEXT:    and a0, a2, a0
+; RV32IFD-NEXT:    and a1, a2, a1
 ; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    .cfi_restore ra
 ; RV32IFD-NEXT:    addi sp, sp, 32
@@ -3530,30 +3530,30 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    call __fixsfti
-; RV32-NEXT:    lw a0, 20(sp)
-; RV32-NEXT:    lw a1, 8(sp)
-; RV32-NEXT:    lw a2, 12(sp)
+; RV32-NEXT:    lw a0, 8(sp)
+; RV32-NEXT:    lw a1, 12(sp)
+; RV32-NEXT:    lw a2, 20(sp)
 ; RV32-NEXT:    lw a3, 16(sp)
-; RV32-NEXT:    beqz a0, .LBB50_2
+; RV32-NEXT:    beqz a2, .LBB50_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a0, 0
+; RV32-NEXT:    slti a4, a2, 0
 ; RV32-NEXT:    j .LBB50_3
 ; RV32-NEXT:  .LBB50_2:
 ; RV32-NEXT:    seqz a4, a3
 ; RV32-NEXT:  .LBB50_3: # %entry
 ; RV32-NEXT:    xori a3, a3, 1
-; RV32-NEXT:    or a3, a3, a0
+; RV32-NEXT:    or a3, a3, a2
 ; RV32-NEXT:    seqz a3, a3
 ; RV32-NEXT:    addi a3, a3, -1
 ; RV32-NEXT:    and a3, a3, a4
 ; RV32-NEXT:    neg a3, a3
-; RV32-NEXT:    and a2, a3, a2
 ; RV32-NEXT:    and a1, a3, a1
 ; RV32-NEXT:    and a0, a3, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    addi a3, a0, -1
-; RV32-NEXT:    and a0, a3, a1
-; RV32-NEXT:    and a1, a3, a2
+; RV32-NEXT:    and a2, a3, a2
+; RV32-NEXT:    slti a2, a2, 0
+; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    and a0, a2, a0
+; RV32-NEXT:    and a1, a2, a1
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore ra
 ; RV32-NEXT:    addi sp, sp, 32
@@ -3767,30 +3767,30 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; RV32-NEXT:    call __extendhfsf2
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    call __fixsfti
-; RV32-NEXT:    lw a0, 20(sp)
-; RV32-NEXT:    lw a1, 8(sp)
-; RV32-NEXT:    lw a2, 12(sp)
+; RV32-NEXT:    lw a0, 8(sp)
+; RV32-NEXT:    lw a1, 12(sp)
+; RV32-NEXT:    lw a2, 20(sp)
 ; RV32-NEXT:    lw a3, 16(sp)
-; RV32-NEXT:    beqz a0, .LBB53_2
+; RV32-NEXT:    beqz a2, .LBB53_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a0, 0
+; RV32-NEXT:    slti a4, a2, 0
 ; RV32-NEXT:    j .LBB53_3
 ; RV32-NEXT:  .LBB53_2:
 ; RV32-NEXT:    seqz a4, a3
 ; RV32-NEXT:  .LBB53_3: # %entry
 ; RV32-NEXT:    xori a3, a3, 1
-; RV32-NEXT:    or a3, a3, a0
+; RV32-NEXT:    or a3, a3, a2
 ; RV32-NEXT:    seqz a3, a3
 ; RV32-NEXT:    addi a3, a3, -1
 ; RV32-NEXT:    and a3, a3, a4
 ; RV32-NEXT:    neg a3, a3
-; RV32-NEXT:    and a2, a3, a2
 ; RV32-NEXT:    and a1, a3, a1
 ; RV32-NEXT:    and a0, a3, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    addi a3, a0, -1
-; RV32-NEXT:    and a0, a3, a1
-; RV32-NEXT:    and a1, a3, a2
+; RV32-NEXT:    and a2, a3, a2
+; RV32-NEXT:    slti a2, a2, 0
+; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    and a0, a2, a0
+; RV32-NEXT:    and a1, a2, a1
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore ra
 ; RV32-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/i32-icmp.ll b/llvm/test/CodeGen/RISCV/i32-icmp.ll
index 6e3e0fe39cca7..53892f9497bba 100644
--- a/llvm/test/CodeGen/RISCV/i32-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/i32-icmp.ll
@@ -1136,3 +1136,57 @@ define i32 @icmp_sle_constant_neg_2050(i32 %a) nounwind {
   %2 = zext i1 %1 to i32
   ret i32 %2
 }
+
+define i32 @mask_test_eq(i32 %x) nounwind {
+; RV32I-LABEL: mask_test_eq:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 12
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: mask_test_eq:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    slli a0, a0, 12
+; RV32XQCILIA-NEXT:    seqz a0, a0
+; RV32XQCILIA-NEXT:    ret
+  %y = and i32 %x, 1048575
+  %cmp = icmp eq i32 %y, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @mask_test_ne(i32 %x) nounwind {
+; RV32I-LABEL: mask_test_ne:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 12
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: mask_test_ne:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    slli a0, a0, 12
+; RV32XQCILIA-NEXT:    snez a0, a0
+; RV32XQCILIA-NEXT:    ret
+  %y = and i32 %x, 1048575
+  %cmp = icmp ne i32 %y, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @mask_test_eq_simm12(i32 %x) nounwind {
+; RV32I-LABEL: mask_test_eq_simm12:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, 3
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: mask_test_eq_simm12:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    andi a0, a0, 3
+; RV32XQCILIA-NEXT:    seqz a0, a0
+; RV32XQCILIA-NEXT:    ret
+  %y = and i32 %x, 3
+  %cmp = icmp eq i32 %y, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
diff --git a/llvm/test/CodeGen/RISCV/i64-icmp.ll b/llvm/test/CodeGen/RISCV/i64-icmp.ll
index 49103231a075f..837987d8b9162 100644
--- a/llvm/test/CodeGen/RISCV/i64-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/i64-icmp.ll
@@ -767,4 +767,56 @@ define i64 @icmp_ne_zext_inreg_umin(i64 %a) nounwind {
   %4 = zext i1 %3 to i64
   ret i64 %4
 }
+
+define i64 @mask_test_eq(i64 %x) nounwind {
+; RV64I-LABEL: mask_test_eq:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 4611686018427387903
+  %cmp = icmp eq i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
+define i64 @mask_test_ne(i64 %x) nounwind {
+; RV64I-LABEL: mask_test_ne:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 4611686018427387903
+  %cmp = icmp ne i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
+define i64 @mask_test_eq_simm12(i64 %x) nounwind {
+; RV64I-LABEL: mask_test_eq_simm12:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a0, a0, 3
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 3
+  %cmp = icmp eq i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
+define i64 @mask_test_eq_multiuse(i64 %x, ptr %p) nounwind {
+; RV64I-LABEL: mask_test_eq_multiuse:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    sd a2, 0(a1)
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 4611686018427387903
+  store i64 %y, ptr %p, align 8
+  %cmp = icmp eq i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
 declare i64 @llvm.umin.i64(i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
index 97d102561129d..b1a6d163664e5 100644
--- a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
+++ b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
@@ -7,18 +7,18 @@
 define i32 @ctz_nxv4i32(<vscale x 4 x i32> %a) #0 {
 ; RV32-LABEL: ctz_nxv4i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; RV32-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV32-NEXT:    vid.v v10
-; RV32-NEXT:    li a1, -1
+; RV32-NEXT:    vmv.v.i v11, -1
+; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v8, 0
 ; RV32-NEXT:    srli a0, a0, 1
 ; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vmadd.vx v10, a1, v8
-; RV32-NEXT:    vmv.v.i v8, 0
-; RV32-NEXT:    vmerge.vvm v8, v8, v10, v0
+; RV32-NEXT:    vmacc.vv v8, v10, v11
+; RV32-NEXT:    vmv.v.i v9, 0
+; RV32-NEXT:    vmerge.vvm v8, v9, v8, v0
 ; RV32-NEXT:    vredmaxu.vs v8, v8, v8
 ; RV32-NEXT:    vmv.x.s a1, v8
 ; RV32-NEXT:    sub a0, a0, a1
@@ -28,18 +28,18 @@ define i32 @ctz_nxv4i32(<vscale x 4 x i32> %a) #0 {
 ;
 ; RV64-LABEL: ctz_nxv4i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; RV64-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV64-NEXT:    vid.v v10
-; RV64-NEXT:    li a1, -1
+; RV64-NEXT:    vmv.v.i v11, -1
+; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vmsne.vi v0, v8, 0
 ; RV64-NEXT:    srli a0, a0, 1
 ; RV64-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vmadd.vx v10, a1, v8
-; RV64-NEXT:    vmv.v.i v8, 0
-; RV64-NEXT:    vmerge.vvm v8, v8, v10, v0
+; RV64-NEXT:    vmacc.vv v8, v10, v11
+; RV64-NEXT:    vmv.v.i v9, 0
+; RV64-NEXT:    vmerge.vvm v8, v9, v8, v0
 ; RV64-NEXT:    vredmaxu.vs v8, v8, v8
 ; RV64-NEXT:    vmv.x.s a1, v8
 ; RV64-NEXT:    subw a0, a0, a1
@@ -109,17 +109,17 @@ define i64 @ctz_nxv8i1_no_range(<vscale x 8 x i16> %a) {
 ;
 ; RV64-LABEL: ctz_nxv8i1_no_range:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vid.v v16
-; RV64-NEXT:    li a1, -1
+; RV64-NEXT:    vmv.v.i v24, -1
+; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; RV64-NEXT:    vmsne.vi v0, v8, 0
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vmadd.vx v16, a1, v8
-; RV64-NEXT:    vmv.v.i v8, 0
-; RV64-NEXT:    vmerge.vvm v8, v8, v16, v0
+; RV64-NEXT:    vmacc.vv v8, v16, v24
+; RV64-NEXT:    vmv.v.i v16, 0
+; RV64-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; RV64-NEXT:    vredmaxu.vs v8, v8, v8
 ; RV64-NEXT:    vmv.x.s a1, v8
 ; RV64-NEXT:    sub a0, a0, a1
diff --git a/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll b/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
index 98c897084ab49..47b90a006a249 100644
--- a/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
@@ -594,7 +594,6 @@ define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, ptr %p) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi a2, a0, 1
 ; RV32-NEXT:    slli a0, a2, 16
-; RV32-NEXT:    srli a0, a0, 16
 ; RV32-NEXT:    seqz a0, a0
 ; RV32-NEXT:    sh a2, 0(a1)
 ; RV32-NEXT:    ret
@@ -603,7 +602,6 @@ define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, ptr %p) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi a2, a0, 1
 ; RV64-NEXT:    slli a0, a2, 48
-; RV64-NEXT:    srli a0, a0, 48
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    sh a2, 0(a1)
 ; RV64-NEXT:    ret
@@ -759,10 +757,9 @@ define i1 @uaddo_i42_increment_illegal_type(i42 %x, ptr %p) {
 ; RV64-LABEL: uaddo_i42_increment_illegal_type:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi a2, a0, 1
-; RV64-NEXT:    slli a0, a2, 22
-; RV64-NEXT:    srli a3, a0, 22
+; RV64-NEXT:    slli a3, a2, 22
 ; RV64-NEXT:    seqz a0, a3
-; RV64-NEXT:    srli a3, a3, 32
+; RV64-NEXT:    srli a3, a3, 54
 ; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    sh a3, 4(a1)
 ; RV64-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rv32xandesperf.ll b/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
index 3e7f09f3d6c22..5cabb8c53e261 100644
--- a/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
@@ -2,6 +2,170 @@
 ; RUN: llc -O0 -mtriple=riscv32 -mattr=+xandesperf -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
+; NDS.BBC
+
+define i32 @bbc(i32 %a) nounwind {
+; CHECK-LABEL: bbc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bbc a0, 16, .LBB0_2
+; CHECK-NEXT:    j .LBB0_1
+; CHECK-NEXT:  .LBB0_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp eq i32 %and, 0
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_bbc(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_bbc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.bbc a0, 16, .LBB1_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp eq i32 %and, 0
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
+; NDS.BBS
+
+define i32 @bbs(i32 %a) nounwind {
+; CHECK-LABEL: bbs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bbs a0, 16, .LBB2_2
+; CHECK-NEXT:    j .LBB2_1
+; CHECK-NEXT:  .LBB2_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp ne i32 %and, 0
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_bbs(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_bbs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.bbs a0, 16, .LBB3_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB3_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp ne i32 %and, 0
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
+; NDS.BEQC
+
+define i32 @beqc(i32 %a) nounwind {
+; CHECK-LABEL: beqc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.beqc a0, 5, .LBB4_2
+; CHECK-NEXT:    j .LBB4_1
+; CHECK-NEXT:  .LBB4_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB4_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %tst = icmp eq i32 %a, 5
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_beqc(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_beqc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.beqc a0, 5, .LBB5_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB5_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %tst = icmp eq i32 %a, 5
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
+; NDS.BNEC
+
+define i32 @bnec(i32 %a) nounwind {
+; CHECK-LABEL: bnec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bnec a0, 5, .LBB6_2
+; CHECK-NEXT:    j .LBB6_1
+; CHECK-NEXT:  .LBB6_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB6_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %tst = icmp ne i32 %a, 5
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_bnec(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_bnec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.bnec a0, 5, .LBB7_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB7_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %tst = icmp ne i32 %a, 5
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
 ; NDS.BFOZ
 
 ; MSB >= LSB
diff --git a/llvm/test/CodeGen/RISCV/rvv/abd.ll b/llvm/test/CodeGen/RISCV/rvv/abd.ll
index 583d872238df7..be4292c9902eb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/abd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/abd.ll
@@ -316,12 +316,10 @@ define <vscale x 4 x i32> @uabd_non_matching_promotion(<vscale x 4 x i8> %a, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vsext.vf2 v8, v9
-; CHECK-NEXT:    vwsub.wv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsext.vf4 v12, v9
+; CHECK-NEXT:    vmin.vv v8, v10, v12
+; CHECK-NEXT:    vmax.vv v10, v10, v12
+; CHECK-NEXT:    vsub.vv v8, v10, v8
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
   %b.zext = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll b/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
index 5dc532273b770..0d8aff306252e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
@@ -313,12 +313,12 @@ define i32 @test_nxv128i1(<vscale x 128 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v0, v6, a0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v7, a1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v7, a0
 ; CHECK-NEXT:    vslidedown.vx v5, v6, a0
+; CHECK-NEXT:    vslidedown.vx v4, v7, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v4
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v5
 ; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
@@ -364,9 +364,9 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vmv1r.v v7, v9
 ; CHECK-NEXT:    vmv1r.v v5, v8
 ; CHECK-NEXT:    vmv1r.v v4, v0
-; CHECK-NEXT:    vmv.v.i v24, 0
+; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    mv a2, a0
@@ -376,7 +376,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v0, v5
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
@@ -389,7 +389,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v2, v5, a0
 ; CHECK-NEXT:    vmv.v.v v0, v3
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    mv a3, a2
@@ -399,41 +399,41 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a2, a2, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v0, v2
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmv8r.v v8, v16
+; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 4
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v3, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v2, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v16, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v24, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v4, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v5, a1
-; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v24, v24, 1, v0
-; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v6, a1
 ; CHECK-NEXT:    vslidedown.vx v5, v7, a1
+; CHECK-NEXT:    vslidedown.vx v4, v6, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v5
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v4
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vadd.vv v8, v8, v24
+; CHECK-NEXT:    vmv1r.v v0, v5
+; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
+; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
@@ -443,7 +443,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v0, v4, a1
 ; CHECK-NEXT:    vslidedown.vx v3, v5, a1
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
+; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v3
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -451,7 +451,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vadd.vv v8, v8, v24
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
@@ -492,16 +492,16 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
-; CHECK-NEXT:    vadd.vv v0, v24, v8
+; CHECK-NEXT:    vadd.vv v24, v24, v8
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vadd.vv v8, v8, v24
-; CHECK-NEXT:    vadd.vv v16, v0, v16
+; CHECK-NEXT:    vl8r.v v0, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vadd.vv v8, v8, v0
+; CHECK-NEXT:    vadd.vv v16, v24, v16
 ; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    vmv.s.x v16, zero
 ; CHECK-NEXT:    vredsum.vs v8, v8, v16
@@ -537,17 +537,18 @@ entry:
 define i16 @test_narrow_nxv64i1(<vscale x 64 x i1> %x) {
 ; CHECK-LABEL: test_narrow_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v0, a0
+; CHECK-NEXT:    vslidedown.vx v8, v0, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vmv.s.x v16, zero
-; CHECK-NEXT:    vredsum.vs v8, v8, v16
+; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
+; CHECK-NEXT:    vmv.s.x v8, zero
+; CHECK-NEXT:    vredsum.vs v8, v16, v8
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
index be73d4808937a..31e79e58f44c5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
@@ -47,7 +47,7 @@ body:             |
     %22:vr = PseudoVMSNE_VI_M1 %3, 0, 1, 6 /* e64 */
     %23:vmv0 = COPY %22
     %25:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, %17, -1, %23, 1, 6 /* e64 */
-    %29:vr = PseudoVC_V_X_SE_M1 3, 31, %2, 1, 6 /* e64 */, implicit-def dead $sf_vcix_state, implicit $sf_vcix_state
+    %29:vr = PseudoSF_VC_V_X_SE_M1 3, 31, %2, 1, 6 /* e64 */, implicit-def dead $sf_vcix_state, implicit $sf_vcix_state
     %30:vr = PseudoVMV_V_I_M1 undef $noreg, 0, 1, 6 /* e64 */, 0
     BGEU %1, $x0, %bb.2
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index 564e95c43f68a..3c3e08d387faa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -1756,16 +1756,16 @@ define <8 x float> @buildvec_v8f32_zvl256(float %e0, float %e1, float %e2, float
 ; CHECK-LABEL: buildvec_v8f32_zvl256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
-; CHECK-NEXT:    vfmv.v.f v8, fa0
-; CHECK-NEXT:    vfmv.v.f v9, fa4
+; CHECK-NEXT:    vfmv.v.f v8, fa4
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa1
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa5
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa2
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa6
-; CHECK-NEXT:    vfslide1down.vf v10, v8, fa3
-; CHECK-NEXT:    vfslide1down.vf v8, v9, fa7
-; CHECK-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa5
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa1
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa6
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa2
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa7
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa3
+; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; CHECK-NEXT:    ret
   %v0 = insertelement <8 x float> poison, float %e0, i64 0
   %v1 = insertelement <8 x float> %v0, float %e1, i64 1
@@ -1807,16 +1807,16 @@ define <8 x double> @buildvec_v8f64_zvl512(double %e0, double %e1, double %e2, d
 ; CHECK-LABEL: buildvec_v8f64_zvl512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m1, ta, mu
-; CHECK-NEXT:    vfmv.v.f v8, fa0
-; CHECK-NEXT:    vfmv.v.f v9, fa4
+; CHECK-NEXT:    vfmv.v.f v8, fa4
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa1
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa5
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa2
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa6
-; CHECK-NEXT:    vfslide1down.vf v10, v8, fa3
-; CHECK-NEXT:    vfslide1down.vf v8, v9, fa7
-; CHECK-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa5
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa1
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa6
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa2
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa7
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa3
+; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; CHECK-NEXT:    ret
   %v0 = insertelement <8 x double> poison, double %e0, i64 0
   %v1 = insertelement <8 x double> %v0, double %e1, i64 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index f235540cc8ffb..d9bb007a10f71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -1359,23 +1359,23 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu s0, 14(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 15(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a1
-; RV32-ONLY-NEXT:    vmv.v.x v9, t1
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t2
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vmv.v.x v8, t1
+; RV32-ONLY-NEXT:    vmv.v.x v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32-ONLY-NEXT:    .cfi_restore s0
 ; RV32-ONLY-NEXT:    addi sp, sp, 16
@@ -1494,23 +1494,23 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu s0, 14(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 15(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a1
-; RV64V-ONLY-NEXT:    vmv.v.x v9, t1
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t2
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vmv.v.x v8, t1
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64V-ONLY-NEXT:    .cfi_restore s0
 ; RV64V-ONLY-NEXT:    addi sp, sp, 16
@@ -1631,23 +1631,23 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu s0, 14(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 15(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32-NEXT:    vmv.v.x v9, t1
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t2
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t3
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t4
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, s0
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vmv.v.x v8, t1
+; RV64ZVE32-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t2
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t3
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t4
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t5
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t6
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, s0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64ZVE32-NEXT:    .cfi_restore s0
 ; RV64ZVE32-NEXT:    addi sp, sp, 16
@@ -1733,23 +1733,23 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu s0, 124(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 144(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a1
-; RV32-ONLY-NEXT:    vmv.v.x v9, t1
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, t2
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vmv.v.x v8, t1
+; RV32-ONLY-NEXT:    vmv.v.x v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32-ONLY-NEXT:    .cfi_restore s0
 ; RV32-ONLY-NEXT:    addi sp, sp, 16
@@ -1868,23 +1868,23 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu s0, 124(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 144(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a1
-; RV64V-ONLY-NEXT:    vmv.v.x v9, t1
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, t2
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vmv.v.x v8, t1
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64V-ONLY-NEXT:    .cfi_restore s0
 ; RV64V-ONLY-NEXT:    addi sp, sp, 16
@@ -2013,23 +2013,23 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu s0, 124(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 144(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32-NEXT:    vmv.v.x v9, t1
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t3
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, s0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t4
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, t2
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vmv.v.x v8, t1
+; RV64ZVE32-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t5
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t6
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t3
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, s0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t4
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t2
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64ZVE32-NEXT:    .cfi_restore s0
 ; RV64ZVE32-NEXT:    addi sp, sp, 16
@@ -2505,17 +2505,17 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu t0, 105(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 161(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a2
-; RV32-ONLY-NEXT:    vmv.v.x v9, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 4
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vmv.v.x v8, a6
+; RV32-ONLY-NEXT:    vmv.v.x v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    ret
 ;
 ; RV32VB-LABEL: buildvec_v16i8_undef_edges:
@@ -2592,17 +2592,17 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu t0, 105(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 161(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a2
-; RV64V-ONLY-NEXT:    vmv.v.x v9, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 4
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vmv.v.x v8, a6
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v8, 4
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ret
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_undef_edges:
@@ -2679,17 +2679,17 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu t0, 105(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 161(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a2
-; RV64ZVE32-NEXT:    vmv.v.x v9, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, a5
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 4
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vmv.v.x v8, a6
+; RV64ZVE32-NEXT:    vmv.v.x v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v8, 4
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ret
   %p4 = getelementptr i8, ptr %p, i32 31
   %p5 = getelementptr i8, ptr %p, i32 44
@@ -2740,21 +2740,21 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu t1, 144(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 154(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a1
-; RV32-ONLY-NEXT:    vmv.v.x v9, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV32-ONLY-NEXT:    vmv.v.x v8, a6
+; RV32-ONLY-NEXT:    vmv.v.x v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
 ; RV32-ONLY-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-ONLY-NEXT:    vslidedown.vi v9, v9, 2
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
 ; RV32-ONLY-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t1
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t1
+; RV32-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    ret
 ;
 ; RV32VB-LABEL: buildvec_v16i8_loads_undef_scattered:
@@ -2834,21 +2834,21 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu t1, 144(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 154(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a1
-; RV64V-ONLY-NEXT:    vmv.v.x v9, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV64V-ONLY-NEXT:    vmv.v.x v8, a6
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
 ; RV64V-ONLY-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64V-ONLY-NEXT:    vslidedown.vi v9, v9, 2
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
 ; RV64V-ONLY-NEXT:    vslidedown.vi v8, v8, 1
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t1
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t1
+; RV64V-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ret
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_loads_undef_scattered:
@@ -2930,21 +2930,21 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu t1, 144(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 154(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32-NEXT:    vmv.v.x v9, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32-NEXT:    vmv.v.x v8, a6
+; RV64ZVE32-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
 ; RV64ZVE32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64ZVE32-NEXT:    vslidedown.vi v9, v9, 2
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslidedown.vi v9, v9, 1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
 ; RV64ZVE32-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t1
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t1
+; RV64ZVE32-NEXT:    vslidedown.vi v9, v9, 1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ret
   %p2 = getelementptr i8, ptr %p, i32 1
   %p3 = getelementptr i8, ptr %p, i32 22
@@ -3002,16 +3002,16 @@ define <8 x i8> @buildvec_v8i8_pack(i8 %e1, i8 %e2, i8 %e3, i8 %e4, i8 %e5, i8 %
 ; RV32-ONLY-LABEL: buildvec_v8i8_pack:
 ; RV32-ONLY:       # %bb.0:
 ; RV32-ONLY-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a0
-; RV32-ONLY-NEXT:    vmv.v.x v9, a4
+; RV32-ONLY-NEXT:    vmv.v.x v8, a4
+; RV32-ONLY-NEXT:    vmv.v.x v9, a0
 ; RV32-ONLY-NEXT:    vmv.v.i v0, 15
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, a7
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a5
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a6
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV32-ONLY-NEXT:    ret
 ;
 ; RV32VB-LABEL: buildvec_v8i8_pack:
@@ -3055,16 +3055,16 @@ define <8 x i8> @buildvec_v8i8_pack(i8 %e1, i8 %e2, i8 %e3, i8 %e4, i8 %e5, i8 %
 ; RV64V-ONLY-LABEL: buildvec_v8i8_pack:
 ; RV64V-ONLY:       # %bb.0:
 ; RV64V-ONLY-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a0
-; RV64V-ONLY-NEXT:    vmv.v.x v9, a4
+; RV64V-ONLY-NEXT:    vmv.v.x v8, a4
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a0
 ; RV64V-ONLY-NEXT:    vmv.v.i v0, 15
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, a7
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a5
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a6
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64V-ONLY-NEXT:    ret
 ;
 ; RVA22U64-LABEL: buildvec_v8i8_pack:
@@ -3110,16 +3110,16 @@ define <8 x i8> @buildvec_v8i8_pack(i8 %e1, i8 %e2, i8 %e3, i8 %e4, i8 %e5, i8 %
 ; RV64ZVE32-LABEL: buildvec_v8i8_pack:
 ; RV64ZVE32:       # %bb.0:
 ; RV64ZVE32-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a0
-; RV64ZVE32-NEXT:    vmv.v.x v9, a4
+; RV64ZVE32-NEXT:    vmv.v.x v8, a4
+; RV64ZVE32-NEXT:    vmv.v.x v9, a0
 ; RV64ZVE32-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, a7
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <8 x i8> poison, i8 %e1, i32 0
   %v2 = insertelement <8 x i8> %v1, i8 %e2, i32 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
index c1252902ede69..0a6f9f5ba0928 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
@@ -7,31 +7,16 @@
 define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 ; RV32-LABEL: llrint_v1i64_v1f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a0), zero
-; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vfwcvt.x.f.v v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v1i64_v1f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-NEXT:    vmv1r.v v8, v9
 ; RV64-NEXT:    ret
   %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
   ret <1 x i64> %a
@@ -41,60 +26,16 @@ declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; RV32-LABEL: llrint_v2i64_v2f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 2 * vlenb
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 16
-; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vfwcvt.x.f.v v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v2i64_v2f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-NEXT:    vmv1r.v v8, v9
 ; RV64-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
   ret <2 x i64> %a
@@ -104,106 +45,16 @@ declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
 define <3 x i64> @llrint_v3i64_v3f32(<3 x float> %x) {
 ; RV32-LABEL: llrint_v3i64_v3f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x03, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 3 * vlenb
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v3i64_v3f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v11
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-NEXT:    ret
   %a = call <3 x i64> @llvm.llrint.v3i64.v3f32(<3 x float> %x)
   ret <3 x i64> %a
@@ -213,106 +64,16 @@ declare <3 x i64> @llvm.llrint.v3i64.v3f32(<3 x float>)
 define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 ; RV32-LABEL: llrint_v4i64_v4f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x03, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 3 * vlenb
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v4i64_v4f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v11
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
   ret <4 x i64> %a
@@ -322,149 +83,16 @@ declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
 define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; RV32-LABEL: llrint_v8i64_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -208
-; RV32-NEXT:    .cfi_def_cfa_offset 208
-; RV32-NEXT:    sw ra, 204(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 200(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 208
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 64(sp)
-; RV32-NEXT:    sw a1, 68(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 7
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 120(sp)
-; RV32-NEXT:    sw a1, 124(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 6
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 112(sp)
-; RV32-NEXT:    sw a1, 116(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 5
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 104(sp)
-; RV32-NEXT:    sw a1, 108(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 4
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 96(sp)
-; RV32-NEXT:    sw a1, 100(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 88(sp)
-; RV32-NEXT:    sw a1, 92(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 80(sp)
-; RV32-NEXT:    sw a1, 84(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 72(sp)
-; RV32-NEXT:    sw a1, 76(sp)
-; RV32-NEXT:    addi a0, sp, 64
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    addi sp, s0, -208
-; RV32-NEXT:    .cfi_def_cfa sp, 208
-; RV32-NEXT:    lw ra, 204(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 200(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 208
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv2r.v v12, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v12
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v8i64_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -128
-; RV64-NEXT:    .cfi_def_cfa_offset 128
-; RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 128
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -64
-; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 6
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-NEXT:    fcvt.l.s a2, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 4
-; RV64-NEXT:    fcvt.l.s a3, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 3
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v8, v8, 1
-; RV64-NEXT:    fcvt.l.s a4, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a5, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a6, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    sd a4, 32(sp)
-; RV64-NEXT:    sd a3, 40(sp)
-; RV64-NEXT:    sd a2, 48(sp)
-; RV64-NEXT:    sd a1, 56(sp)
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    sd a0, 0(sp)
-; RV64-NEXT:    sd a1, 8(sp)
-; RV64-NEXT:    sd a6, 16(sp)
-; RV64-NEXT:    sd a5, 24(sp)
-; RV64-NEXT:    mv a0, sp
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    addi sp, s0, -128
-; RV64-NEXT:    .cfi_def_cfa sp, 128
-; RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
-; RV64-NEXT:    addi sp, sp, 128
-; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vmv2r.v v12, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v12
 ; RV64-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
   ret <8 x i64> %a
@@ -474,214 +102,16 @@ declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; RV32-LABEL: llrint_v16i64_v16f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -400
-; RV32-NEXT:    .cfi_def_cfa_offset 400
-; RV32-NEXT:    sw ra, 396(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 392(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 400
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    andi sp, sp, -128
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 64
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v8, (a0)
-; RV32-NEXT:    flw fa0, 124(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 248(sp)
-; RV32-NEXT:    sw a1, 252(sp)
-; RV32-NEXT:    flw fa0, 120(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 240(sp)
-; RV32-NEXT:    sw a1, 244(sp)
-; RV32-NEXT:    flw fa0, 116(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 232(sp)
-; RV32-NEXT:    sw a1, 236(sp)
-; RV32-NEXT:    flw fa0, 112(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 224(sp)
-; RV32-NEXT:    sw a1, 228(sp)
-; RV32-NEXT:    flw fa0, 108(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 216(sp)
-; RV32-NEXT:    sw a1, 220(sp)
-; RV32-NEXT:    flw fa0, 104(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 208(sp)
-; RV32-NEXT:    sw a1, 212(sp)
-; RV32-NEXT:    flw fa0, 100(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 200(sp)
-; RV32-NEXT:    sw a1, 204(sp)
-; RV32-NEXT:    flw fa0, 96(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 192(sp)
-; RV32-NEXT:    sw a1, 196(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 128(sp)
-; RV32-NEXT:    sw a1, 132(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 152(sp)
-; RV32-NEXT:    sw a1, 156(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 144(sp)
-; RV32-NEXT:    sw a1, 148(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 136(sp)
-; RV32-NEXT:    sw a1, 140(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 7
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 184(sp)
-; RV32-NEXT:    sw a1, 188(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 6
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 176(sp)
-; RV32-NEXT:    sw a1, 180(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 5
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 168(sp)
-; RV32-NEXT:    sw a1, 172(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 4
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 160(sp)
-; RV32-NEXT:    sw a1, 164(sp)
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    addi a1, sp, 128
-; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-NEXT:    vle32.v v8, (a1)
-; RV32-NEXT:    addi sp, s0, -400
-; RV32-NEXT:    .cfi_def_cfa sp, 400
-; RV32-NEXT:    lw ra, 396(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 392(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 400
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vmv4r.v v16, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v16i64_v16f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -384
-; RV64-NEXT:    .cfi_def_cfa_offset 384
-; RV64-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 368(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 384
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -128
-; RV64-NEXT:    addi a0, sp, 64
 ; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-NEXT:    vse32.v v8, (a0)
-; RV64-NEXT:    flw fa5, 124(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v8
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 248(sp)
-; RV64-NEXT:    flw fa5, 120(sp)
-; RV64-NEXT:    vslidedown.vi v13, v8, 1
-; RV64-NEXT:    fcvt.l.s a0, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    sd a1, 240(sp)
-; RV64-NEXT:    flw fa5, 116(sp)
-; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-NEXT:    fcvt.l.s a1, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v12
-; RV64-NEXT:    fcvt.l.s a2, fa5
-; RV64-NEXT:    sd a2, 232(sp)
-; RV64-NEXT:    flw fa5, 112(sp)
-; RV64-NEXT:    fcvt.l.s a2, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v13
-; RV64-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-NEXT:    fcvt.l.s a3, fa5
-; RV64-NEXT:    sd a3, 224(sp)
-; RV64-NEXT:    flw fa5, 108(sp)
-; RV64-NEXT:    fcvt.l.s a3, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-NEXT:    fcvt.l.s a4, fa5
-; RV64-NEXT:    sd a4, 216(sp)
-; RV64-NEXT:    flw fa5, 104(sp)
-; RV64-NEXT:    fcvt.l.s a4, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v12
-; RV64-NEXT:    fcvt.l.s a5, fa4
-; RV64-NEXT:    fcvt.l.s a6, fa5
-; RV64-NEXT:    sd a6, 208(sp)
-; RV64-NEXT:    flw fa5, 100(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    fcvt.l.s a6, fa4
-; RV64-NEXT:    vslidedown.vi v8, v8, 4
-; RV64-NEXT:    fcvt.l.s a7, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    sd a7, 200(sp)
-; RV64-NEXT:    fcvt.l.s a7, fa5
-; RV64-NEXT:    flw fa5, 96(sp)
-; RV64-NEXT:    sd a0, 128(sp)
-; RV64-NEXT:    sd a3, 136(sp)
-; RV64-NEXT:    sd a2, 144(sp)
-; RV64-NEXT:    sd a1, 152(sp)
-; RV64-NEXT:    sd a7, 160(sp)
-; RV64-NEXT:    sd a6, 168(sp)
-; RV64-NEXT:    sd a5, 176(sp)
-; RV64-NEXT:    sd a4, 184(sp)
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 192(sp)
-; RV64-NEXT:    addi a0, sp, 128
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    addi sp, s0, -384
-; RV64-NEXT:    .cfi_def_cfa sp, 384
-; RV64-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 368(sp) # 8-byte Folded Reload
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
-; RV64-NEXT:    addi sp, sp, 384
-; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    vmv4r.v v16, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v16
 ; RV64-NEXT:    ret
   %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
   ret <16 x i64> %a
@@ -691,30 +121,14 @@ declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 ; RV32-LABEL: llrint_v1i64_v1f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a0), zero
-; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v1i64_v1f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
   ret <1 x i64> %a
@@ -724,59 +138,14 @@ declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 ; RV32-LABEL: llrint_v2i64_v2f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 2 * vlenb
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 16
-; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v2i64_v2f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.d a1, fa5
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
   ret <2 x i64> %a
@@ -786,103 +155,14 @@ declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 ; RV32-LABEL: llrint_v4i64_v4f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 4 * vlenb
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v4i64_v4f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v12, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v12
-; RV64-NEXT:    fcvt.l.d a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
 ; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v10, a0
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslide1down.vx v8, v10, a1
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
   ret <4 x i64> %a
@@ -892,137 +172,14 @@ declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; RV32-LABEL: llrint_v8i64_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -272
-; RV32-NEXT:    .cfi_def_cfa_offset 272
-; RV32-NEXT:    sw ra, 268(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 264(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 272
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 64
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vse64.v v8, (a0)
-; RV32-NEXT:    fld fa0, 120(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 184(sp)
-; RV32-NEXT:    sw a1, 188(sp)
-; RV32-NEXT:    fld fa0, 112(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 176(sp)
-; RV32-NEXT:    sw a1, 180(sp)
-; RV32-NEXT:    fld fa0, 104(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 168(sp)
-; RV32-NEXT:    sw a1, 172(sp)
-; RV32-NEXT:    fld fa0, 96(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 160(sp)
-; RV32-NEXT:    sw a1, 164(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 128(sp)
-; RV32-NEXT:    sw a1, 132(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 136(sp)
-; RV32-NEXT:    sw a1, 140(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 152(sp)
-; RV32-NEXT:    sw a1, 156(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 144(sp)
-; RV32-NEXT:    sw a1, 148(sp)
-; RV32-NEXT:    addi a0, sp, 128
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    addi sp, s0, -272
-; RV32-NEXT:    .cfi_def_cfa sp, 272
-; RV32-NEXT:    lw ra, 268(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 264(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 272
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v8i64_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -192
-; RV64-NEXT:    .cfi_def_cfa_offset 192
-; RV64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 192
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -64
-; RV64-NEXT:    mv a0, sp
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    fld fa5, 56(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v8
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-NEXT:    fcvt.l.d a0, fa4
-; RV64-NEXT:    fcvt.l.d a1, fa5
-; RV64-NEXT:    sd a1, 120(sp)
-; RV64-NEXT:    fld fa5, 48(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-NEXT:    fcvt.l.d a1, fa4
-; RV64-NEXT:    fcvt.l.d a2, fa5
-; RV64-NEXT:    sd a2, 112(sp)
-; RV64-NEXT:    fld fa5, 40(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    fcvt.l.d a2, fa4
-; RV64-NEXT:    vslidedown.vi v8, v8, 2
-; RV64-NEXT:    fcvt.l.d a3, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    sd a3, 104(sp)
-; RV64-NEXT:    fcvt.l.d a3, fa5
-; RV64-NEXT:    fld fa5, 32(sp)
-; RV64-NEXT:    sd a0, 64(sp)
-; RV64-NEXT:    sd a1, 72(sp)
-; RV64-NEXT:    sd a3, 80(sp)
-; RV64-NEXT:    sd a2, 88(sp)
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    sd a0, 96(sp)
-; RV64-NEXT:    addi a0, sp, 64
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    addi sp, s0, -192
-; RV64-NEXT:    .cfi_def_cfa sp, 192
-; RV64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
-; RV64-NEXT:    addi sp, sp, 192
-; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
   ret <8 x i64> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
index 27abd624f7765..ef2208666e0b4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
@@ -9,27 +9,21 @@
 define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
 ; RV32-LABEL: lrint_v1f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v1f32:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v1f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-i64-NEXT:    vmv1r.v v8, v9
 ; RV64-i64-NEXT:    ret
   %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float> %x)
   ret <1 x iXLen> %a
@@ -40,38 +34,20 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 ; RV32-LABEL: lrint_v2f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v2f32:
 ; RV64-i32:       # %bb.0:
 ; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vmv.v.x v8, a0
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v2f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-i64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-i64-NEXT:    vmv1r.v v8, v9
 ; RV64-i64-NEXT:    ret
   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
   ret <2 x iXLen> %a
@@ -82,64 +58,20 @@ define <3 x iXLen> @lrint_v3f32(<3 x float> %x) {
 ; RV32-LABEL: lrint_v3f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslide1down.vx v8, v9, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v3f32:
 ; RV64-i32:       # %bb.0:
 ; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    vmv.v.x v9, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v3f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v11
-; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i64-NEXT:    vmv1r.v v10, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-i64-NEXT:    ret
   %a = call <3 x iXLen> @llvm.lrint.v3iXLen.v3f32(<3 x float> %x)
   ret <3 x iXLen> %a
@@ -150,64 +82,20 @@ define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
 ; RV32-LABEL: lrint_v4f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslide1down.vx v8, v9, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v4f32:
 ; RV64-i32:       # %bb.0:
 ; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    vmv.v.x v9, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v4f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v11
-; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i64-NEXT:    vmv1r.v v10, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-i64-NEXT:    ret
   %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
   ret <4 x iXLen> %a
@@ -217,133 +105,21 @@ declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
 define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 ; RV32-LABEL: lrint_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslidedown.vi v11, v8, 2
-; RV32-NEXT:    vslidedown.vi v12, v8, 3
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v11
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v10, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslidedown.vi v12, v8, 4
-; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslidedown.vi v12, v8, 5
-; RV32-NEXT:    vslide1down.vx v10, v10, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslidedown.vi v12, v8, 6
-; RV32-NEXT:    vslidedown.vi v8, v8, 7
-; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslide1down.vx v10, v10, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslide1down.vx v8, v10, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v8f32:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslidedown.vi v11, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 3
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v11
 ; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v10, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 4
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 5
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 7
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslide1down.vx v8, v10, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v8f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    addi sp, sp, -128
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 128
-; RV64-i64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    .cfi_offset ra, -8
-; RV64-i64-NEXT:    .cfi_offset s0, -16
-; RV64-i64-NEXT:    addi s0, sp, 128
-; RV64-i64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i64-NEXT:    andi sp, sp, -64
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 6
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-i64-NEXT:    fcvt.l.s a2, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 4
-; RV64-i64-NEXT:    fcvt.l.s a3, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 3
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 1
-; RV64-i64-NEXT:    fcvt.l.s a4, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a5, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a6, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    sd a4, 32(sp)
-; RV64-i64-NEXT:    sd a3, 40(sp)
-; RV64-i64-NEXT:    sd a2, 48(sp)
-; RV64-i64-NEXT:    sd a1, 56(sp)
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    sd a0, 0(sp)
-; RV64-i64-NEXT:    sd a1, 8(sp)
-; RV64-i64-NEXT:    sd a6, 16(sp)
-; RV64-i64-NEXT:    sd a5, 24(sp)
-; RV64-i64-NEXT:    mv a0, sp
-; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i64-NEXT:    vle64.v v8, (a0)
-; RV64-i64-NEXT:    addi sp, s0, -128
-; RV64-i64-NEXT:    .cfi_def_cfa sp, 128
-; RV64-i64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    .cfi_restore ra
-; RV64-i64-NEXT:    .cfi_restore s0
-; RV64-i64-NEXT:    addi sp, sp, 128
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-i64-NEXT:    vmv2r.v v12, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v12
 ; RV64-i64-NEXT:    ret
   %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
   ret <8 x iXLen> %a
@@ -353,254 +129,21 @@ declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
 define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 ; RV32-LABEL: lrint_v16f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -192
-; RV32-NEXT:    .cfi_def_cfa_offset 192
-; RV32-NEXT:    sw ra, 188(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 184(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 192
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v8, (a0)
-; RV32-NEXT:    flw fa5, 60(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v8
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 3
-; RV32-NEXT:    vslidedown.vi v11, v8, 2
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 124(sp)
-; RV32-NEXT:    flw fa5, 56(sp)
-; RV32-NEXT:    fcvt.w.s a0, fa4
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vslidedown.vi v10, v8, 1
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    sw a1, 120(sp)
-; RV32-NEXT:    flw fa5, 52(sp)
-; RV32-NEXT:    fcvt.w.s a1, fa4
-; RV32-NEXT:    vfmv.f.s fa4, v11
-; RV32-NEXT:    fcvt.w.s a2, fa4
-; RV32-NEXT:    fcvt.w.s a3, fa5
-; RV32-NEXT:    sw a3, 116(sp)
-; RV32-NEXT:    flw fa5, 48(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 7
-; RV32-NEXT:    fcvt.w.s a3, fa4
-; RV32-NEXT:    fcvt.w.s a4, fa5
-; RV32-NEXT:    sw a4, 112(sp)
-; RV32-NEXT:    flw fa5, 44(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vslidedown.vi v10, v8, 6
-; RV32-NEXT:    fcvt.w.s a4, fa4
-; RV32-NEXT:    fcvt.w.s a5, fa5
-; RV32-NEXT:    sw a5, 108(sp)
-; RV32-NEXT:    flw fa5, 40(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vslidedown.vi v10, v8, 5
-; RV32-NEXT:    fcvt.w.s a5, fa4
-; RV32-NEXT:    fcvt.w.s a6, fa5
-; RV32-NEXT:    sw a6, 104(sp)
-; RV32-NEXT:    flw fa5, 36(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    fcvt.w.s a6, fa4
-; RV32-NEXT:    vslidedown.vi v8, v8, 4
-; RV32-NEXT:    fcvt.w.s a7, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    sw a7, 100(sp)
-; RV32-NEXT:    fcvt.w.s a7, fa5
-; RV32-NEXT:    flw fa5, 32(sp)
-; RV32-NEXT:    sw a0, 64(sp)
-; RV32-NEXT:    sw a3, 68(sp)
-; RV32-NEXT:    sw a2, 72(sp)
-; RV32-NEXT:    sw a1, 76(sp)
-; RV32-NEXT:    sw a7, 80(sp)
-; RV32-NEXT:    sw a6, 84(sp)
-; RV32-NEXT:    sw a5, 88(sp)
-; RV32-NEXT:    sw a4, 92(sp)
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 96(sp)
-; RV32-NEXT:    addi a0, sp, 64
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    addi sp, s0, -192
-; RV32-NEXT:    .cfi_def_cfa sp, 192
-; RV32-NEXT:    lw ra, 188(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 184(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 192
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v16f32:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    addi sp, sp, -192
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 192
-; RV64-i32-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    .cfi_offset ra, -8
-; RV64-i32-NEXT:    .cfi_offset s0, -16
-; RV64-i32-NEXT:    addi s0, sp, 192
-; RV64-i32-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i32-NEXT:    andi sp, sp, -64
-; RV64-i32-NEXT:    mv a0, sp
-; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-i32-NEXT:    vse32.v v8, (a0)
-; RV64-i32-NEXT:    flw fa5, 60(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v8
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-i32-NEXT:    vslidedown.vi v11, v8, 2
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 124(sp)
-; RV64-i32-NEXT:    flw fa5, 56(sp)
-; RV64-i32-NEXT:    fcvt.l.s a0, fa4
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    sw a1, 120(sp)
-; RV64-i32-NEXT:    flw fa5, 52(sp)
-; RV64-i32-NEXT:    fcvt.l.s a1, fa4
-; RV64-i32-NEXT:    vfmv.f.s fa4, v11
-; RV64-i32-NEXT:    fcvt.l.s a2, fa4
-; RV64-i32-NEXT:    fcvt.l.s a3, fa5
-; RV64-i32-NEXT:    sw a3, 116(sp)
-; RV64-i32-NEXT:    flw fa5, 48(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-i32-NEXT:    fcvt.l.s a3, fa4
-; RV64-i32-NEXT:    fcvt.l.s a4, fa5
-; RV64-i32-NEXT:    sw a4, 112(sp)
-; RV64-i32-NEXT:    flw fa5, 44(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 6
-; RV64-i32-NEXT:    fcvt.l.s a4, fa4
-; RV64-i32-NEXT:    fcvt.l.s a5, fa5
-; RV64-i32-NEXT:    sw a5, 108(sp)
-; RV64-i32-NEXT:    flw fa5, 40(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-i32-NEXT:    fcvt.l.s a5, fa4
-; RV64-i32-NEXT:    fcvt.l.s a6, fa5
-; RV64-i32-NEXT:    sw a6, 104(sp)
-; RV64-i32-NEXT:    flw fa5, 36(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    fcvt.l.s a6, fa4
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 4
-; RV64-i32-NEXT:    fcvt.l.s a7, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    sw a7, 100(sp)
-; RV64-i32-NEXT:    fcvt.l.s a7, fa5
-; RV64-i32-NEXT:    flw fa5, 32(sp)
-; RV64-i32-NEXT:    sw a0, 64(sp)
-; RV64-i32-NEXT:    sw a3, 68(sp)
-; RV64-i32-NEXT:    sw a2, 72(sp)
-; RV64-i32-NEXT:    sw a1, 76(sp)
-; RV64-i32-NEXT:    sw a7, 80(sp)
-; RV64-i32-NEXT:    sw a6, 84(sp)
-; RV64-i32-NEXT:    sw a5, 88(sp)
-; RV64-i32-NEXT:    sw a4, 92(sp)
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 96(sp)
-; RV64-i32-NEXT:    addi a0, sp, 64
 ; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-i32-NEXT:    vle32.v v8, (a0)
-; RV64-i32-NEXT:    addi sp, s0, -192
-; RV64-i32-NEXT:    .cfi_def_cfa sp, 192
-; RV64-i32-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    .cfi_restore ra
-; RV64-i32-NEXT:    .cfi_restore s0
-; RV64-i32-NEXT:    addi sp, sp, 192
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v16f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    addi sp, sp, -384
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 384
-; RV64-i64-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    sd s0, 368(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    .cfi_offset ra, -8
-; RV64-i64-NEXT:    .cfi_offset s0, -16
-; RV64-i64-NEXT:    addi s0, sp, 384
-; RV64-i64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i64-NEXT:    andi sp, sp, -128
-; RV64-i64-NEXT:    addi a0, sp, 64
 ; RV64-i64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-i64-NEXT:    vse32.v v8, (a0)
-; RV64-i64-NEXT:    flw fa5, 124(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v8
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-i64-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 248(sp)
-; RV64-i64-NEXT:    flw fa5, 120(sp)
-; RV64-i64-NEXT:    vslidedown.vi v13, v8, 1
-; RV64-i64-NEXT:    fcvt.l.s a0, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    sd a1, 240(sp)
-; RV64-i64-NEXT:    flw fa5, 116(sp)
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-i64-NEXT:    fcvt.l.s a1, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v12
-; RV64-i64-NEXT:    fcvt.l.s a2, fa5
-; RV64-i64-NEXT:    sd a2, 232(sp)
-; RV64-i64-NEXT:    flw fa5, 112(sp)
-; RV64-i64-NEXT:    fcvt.l.s a2, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v13
-; RV64-i64-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-i64-NEXT:    fcvt.l.s a3, fa5
-; RV64-i64-NEXT:    sd a3, 224(sp)
-; RV64-i64-NEXT:    flw fa5, 108(sp)
-; RV64-i64-NEXT:    fcvt.l.s a3, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-i64-NEXT:    fcvt.l.s a4, fa5
-; RV64-i64-NEXT:    sd a4, 216(sp)
-; RV64-i64-NEXT:    flw fa5, 104(sp)
-; RV64-i64-NEXT:    fcvt.l.s a4, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v12
-; RV64-i64-NEXT:    fcvt.l.s a5, fa4
-; RV64-i64-NEXT:    fcvt.l.s a6, fa5
-; RV64-i64-NEXT:    sd a6, 208(sp)
-; RV64-i64-NEXT:    flw fa5, 100(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    fcvt.l.s a6, fa4
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 4
-; RV64-i64-NEXT:    fcvt.l.s a7, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    sd a7, 200(sp)
-; RV64-i64-NEXT:    fcvt.l.s a7, fa5
-; RV64-i64-NEXT:    flw fa5, 96(sp)
-; RV64-i64-NEXT:    sd a0, 128(sp)
-; RV64-i64-NEXT:    sd a3, 136(sp)
-; RV64-i64-NEXT:    sd a2, 144(sp)
-; RV64-i64-NEXT:    sd a1, 152(sp)
-; RV64-i64-NEXT:    sd a7, 160(sp)
-; RV64-i64-NEXT:    sd a6, 168(sp)
-; RV64-i64-NEXT:    sd a5, 176(sp)
-; RV64-i64-NEXT:    sd a4, 184(sp)
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 192(sp)
-; RV64-i64-NEXT:    addi a0, sp, 128
-; RV64-i64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-i64-NEXT:    vle64.v v8, (a0)
-; RV64-i64-NEXT:    addi sp, s0, -384
-; RV64-i64-NEXT:    .cfi_def_cfa sp, 384
-; RV64-i64-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    ld s0, 368(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    .cfi_restore ra
-; RV64-i64-NEXT:    .cfi_restore s0
-; RV64-i64-NEXT:    addi sp, sp, 384
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i64-NEXT:    vmv4r.v v16, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v16
 ; RV64-i64-NEXT:    ret
   %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
   ret <16 x iXLen> %a
@@ -610,26 +153,22 @@ declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
 define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
 ; RV32-LABEL: lrint_v1f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vfncvt.x.f.w v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v1f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vfncvt.x.f.w v9, v8
+; RV64-i32-NEXT:    vmv1r.v v8, v9
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v1f64:
 ; RV64-i64:       # %bb.0:
 ; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double> %x)
   ret <1 x iXLen> %a
@@ -639,40 +178,22 @@ declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
 define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
 ; RV32-LABEL: lrint_v2f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.d a1, fa5
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vfncvt.x.f.w v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v2f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.d a1, fa5
 ; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v8, a0
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    vfncvt.x.f.w v9, v8
+; RV64-i32-NEXT:    vmv1r.v v8, v9
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v2f64:
 ; RV64-i64:       # %bb.0:
 ; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.d a1, fa5
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double> %x)
   ret <2 x iXLen> %a
@@ -682,72 +203,22 @@ declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
 define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
 ; RV32-LABEL: lrint_v4f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v12, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    fcvt.w.d a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v9, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfncvt.x.f.w v10, v8
+; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v4f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    fcvt.l.d a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
 ; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v9, a0
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfncvt.x.f.w v10, v8
+; RV64-i32-NEXT:    vmv.v.v v8, v10
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v4f64:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v12, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v12
-; RV64-i64-NEXT:    fcvt.l.d a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
 ; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v10, a0
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslide1down.vx v8, v10, a1
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double> %x)
   ret <4 x iXLen> %a
@@ -757,168 +228,22 @@ declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
 define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; RV32-LABEL: lrint_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -128
-; RV32-NEXT:    .cfi_def_cfa_offset 128
-; RV32-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 128
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v14, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v12, v8, 2
-; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vse64.v v8, (a0)
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa4, v14
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vfmv.f.s fa3, v8
-; RV32-NEXT:    fcvt.w.d a1, fa4
-; RV32-NEXT:    fcvt.w.d a2, fa5
-; RV32-NEXT:    fcvt.w.d a3, fa3
-; RV32-NEXT:    fld fa5, 32(sp)
-; RV32-NEXT:    fld fa4, 40(sp)
-; RV32-NEXT:    fld fa3, 48(sp)
-; RV32-NEXT:    fld fa2, 56(sp)
-; RV32-NEXT:    fcvt.w.d a4, fa5
-; RV32-NEXT:    fcvt.w.d a5, fa4
-; RV32-NEXT:    fcvt.w.d a6, fa3
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-NEXT:    vslide1down.vx v8, v8, a6
-; RV32-NEXT:    fcvt.w.d a0, fa2
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    addi sp, s0, -128
-; RV32-NEXT:    .cfi_def_cfa sp, 128
-; RV32-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 128
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfncvt.x.f.w v12, v8
+; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v8f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    addi sp, sp, -128
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 128
-; RV64-i32-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    .cfi_offset ra, -8
-; RV64-i32-NEXT:    .cfi_offset s0, -16
-; RV64-i32-NEXT:    addi s0, sp, 128
-; RV64-i32-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i32-NEXT:    andi sp, sp, -64
-; RV64-i32-NEXT:    mv a0, sp
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v14, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-i32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i32-NEXT:    vse64.v v8, (a0)
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    vfmv.f.s fa4, v14
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vfmv.f.s fa3, v8
-; RV64-i32-NEXT:    fcvt.l.d a1, fa4
-; RV64-i32-NEXT:    fcvt.l.d a2, fa5
-; RV64-i32-NEXT:    fcvt.l.d a3, fa3
-; RV64-i32-NEXT:    fld fa5, 32(sp)
-; RV64-i32-NEXT:    fld fa4, 40(sp)
-; RV64-i32-NEXT:    fld fa3, 48(sp)
-; RV64-i32-NEXT:    fld fa2, 56(sp)
-; RV64-i32-NEXT:    fcvt.l.d a4, fa5
-; RV64-i32-NEXT:    fcvt.l.d a5, fa4
-; RV64-i32-NEXT:    fcvt.l.d a6, fa3
 ; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v8, a0
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a6
-; RV64-i32-NEXT:    fcvt.l.d a0, fa2
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    addi sp, s0, -128
-; RV64-i32-NEXT:    .cfi_def_cfa sp, 128
-; RV64-i32-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    .cfi_restore ra
-; RV64-i32-NEXT:    .cfi_restore s0
-; RV64-i32-NEXT:    addi sp, sp, 128
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i32-NEXT:    vfncvt.x.f.w v12, v8
+; RV64-i32-NEXT:    vmv.v.v v8, v12
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v8f64:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    addi sp, sp, -192
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 192
-; RV64-i64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    .cfi_offset ra, -8
-; RV64-i64-NEXT:    .cfi_offset s0, -16
-; RV64-i64-NEXT:    addi s0, sp, 192
-; RV64-i64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i64-NEXT:    andi sp, sp, -64
-; RV64-i64-NEXT:    mv a0, sp
-; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i64-NEXT:    vse64.v v8, (a0)
-; RV64-i64-NEXT:    fld fa5, 56(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v8
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-i64-NEXT:    fcvt.l.d a0, fa4
-; RV64-i64-NEXT:    fcvt.l.d a1, fa5
-; RV64-i64-NEXT:    sd a1, 120(sp)
-; RV64-i64-NEXT:    fld fa5, 48(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-i64-NEXT:    fcvt.l.d a1, fa4
-; RV64-i64-NEXT:    fcvt.l.d a2, fa5
-; RV64-i64-NEXT:    sd a2, 112(sp)
-; RV64-i64-NEXT:    fld fa5, 40(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    fcvt.l.d a2, fa4
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 2
-; RV64-i64-NEXT:    fcvt.l.d a3, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    sd a3, 104(sp)
-; RV64-i64-NEXT:    fcvt.l.d a3, fa5
-; RV64-i64-NEXT:    fld fa5, 32(sp)
-; RV64-i64-NEXT:    sd a0, 64(sp)
-; RV64-i64-NEXT:    sd a1, 72(sp)
-; RV64-i64-NEXT:    sd a3, 80(sp)
-; RV64-i64-NEXT:    sd a2, 88(sp)
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    sd a0, 96(sp)
-; RV64-i64-NEXT:    addi a0, sp, 64
 ; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i64-NEXT:    vle64.v v8, (a0)
-; RV64-i64-NEXT:    addi sp, s0, -192
-; RV64-i64-NEXT:    .cfi_def_cfa sp, 192
-; RV64-i64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    .cfi_restore ra
-; RV64-i64-NEXT:    .cfi_restore s0
-; RV64-i64-NEXT:    addi sp, sp, 192
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double> %x)
   ret <8 x iXLen> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
index c29ccd45528b8..3bfe41337a110 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
@@ -280,16 +280,16 @@ define <8 x i1> @buildvec_mask_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vslide1down.vx v9, v8, a0
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    vslide1down.vx v9, v8, a3
+; CHECK-NEXT:    li a3, 1
 ; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v9, v9, zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
-; CHECK-NEXT:    vslide1down.vx v9, v9, a0
-; CHECK-NEXT:    vslide1down.vx v8, v8, zero
-; CHECK-NEXT:    vslide1down.vx v9, v9, a1
-; CHECK-NEXT:    vslide1down.vx v8, v8, a2
-; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vslide1down.vx v9, v9, a2
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v9, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
 ;
@@ -297,16 +297,16 @@ define <8 x i1> @buildvec_mask_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) {
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; ZVE32F-NEXT:    vmv.v.x v8, a0
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a0
-; ZVE32F-NEXT:    li a0, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a3
+; ZVE32F-NEXT:    li a3, 1
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, zero
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; ZVE32F-NEXT:    vand.vi v8, v8, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vand.vi v8, v9, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
@@ -325,16 +325,16 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vslide1down.vx v9, v8, a0
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    vslide1down.vx v9, v8, a3
+; CHECK-NEXT:    li a3, 1
 ; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v9, v9, zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
-; CHECK-NEXT:    vslide1down.vx v9, v9, a0
-; CHECK-NEXT:    vslide1down.vx v8, v8, zero
-; CHECK-NEXT:    vslide1down.vx v9, v9, a1
-; CHECK-NEXT:    vslide1down.vx v8, v8, a2
-; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vslide1down.vx v9, v9, a2
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v9, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
 ;
@@ -342,16 +342,16 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; ZVE32F-NEXT:    vmv.v.x v8, a0
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a0
-; ZVE32F-NEXT:    li a0, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a3
+; ZVE32F-NEXT:    li a3, 1
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, zero
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; ZVE32F-NEXT:    vand.vi v8, v8, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vand.vi v8, v9, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
@@ -371,14 +371,14 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1(i1 %x, i1 %y) optsize {
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.v.x v8, a0
 ; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vslide1down.vx v9, v8, a0
-; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v9, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    vslide1down.vx v9, v9, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vslide1down.vx v9, v9, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v9, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
 ;
@@ -387,14 +387,14 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1(i1 %x, i1 %y) optsize {
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; ZVE32F-NEXT:    vmv.v.x v8, a0
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a0
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; ZVE32F-NEXT:    vand.vi v8, v8, 1
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vand.vi v8, v9, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 533b8b6864ebc..67d55366674f3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -14055,45 +14055,45 @@ define <8 x i16> @mgather_strided_unaligned(ptr %base) {
 ; RV32-NEXT:    vmv.x.s t0, v8
 ; RV32-NEXT:    lbu t1, 0(a1)
 ; RV32-NEXT:    lbu a1, 1(a1)
-; RV32-NEXT:    lbu t2, 0(a2)
-; RV32-NEXT:    lbu a2, 1(a2)
 ; RV32-NEXT:    slli a0, a0, 8
 ; RV32-NEXT:    or a0, a0, a6
-; RV32-NEXT:    lbu a6, 0(a3)
-; RV32-NEXT:    lbu a3, 1(a3)
+; RV32-NEXT:    lbu a6, 0(a2)
+; RV32-NEXT:    lbu a2, 1(a2)
 ; RV32-NEXT:    slli a1, a1, 8
 ; RV32-NEXT:    or a1, a1, t1
-; RV32-NEXT:    lbu t1, 0(a4)
-; RV32-NEXT:    lbu a4, 1(a4)
+; RV32-NEXT:    lbu t1, 0(a3)
+; RV32-NEXT:    lbu a3, 1(a3)
 ; RV32-NEXT:    slli a2, a2, 8
-; RV32-NEXT:    or a2, a2, t2
-; RV32-NEXT:    lbu t2, 0(a5)
-; RV32-NEXT:    lbu a5, 1(a5)
+; RV32-NEXT:    or a2, a2, a6
+; RV32-NEXT:    lbu a6, 0(a4)
+; RV32-NEXT:    lbu a4, 1(a4)
 ; RV32-NEXT:    slli a3, a3, 8
-; RV32-NEXT:    or a3, a3, a6
+; RV32-NEXT:    or a3, a3, t1
+; RV32-NEXT:    lbu t1, 0(a5)
+; RV32-NEXT:    lbu a5, 1(a5)
+; RV32-NEXT:    slli a4, a4, 8
+; RV32-NEXT:    or a4, a4, a6
 ; RV32-NEXT:    lbu a6, 0(a7)
 ; RV32-NEXT:    lbu a7, 1(a7)
-; RV32-NEXT:    slli a4, a4, 8
-; RV32-NEXT:    or a4, a4, t1
+; RV32-NEXT:    slli a5, a5, 8
+; RV32-NEXT:    or a5, a5, t1
 ; RV32-NEXT:    lbu t1, 0(t0)
 ; RV32-NEXT:    lbu t0, 1(t0)
-; RV32-NEXT:    slli a5, a5, 8
-; RV32-NEXT:    or a5, a5, t2
 ; RV32-NEXT:    slli a7, a7, 8
 ; RV32-NEXT:    or a6, a7, a6
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; RV32-NEXT:    vmv.v.i v0, 15
 ; RV32-NEXT:    slli t0, t0, 8
 ; RV32-NEXT:    or a7, t0, t1
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32-NEXT:    vmv.v.x v9, a4
-; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32-NEXT:    vslide1down.vx v9, v9, a5
-; RV32-NEXT:    vslide1down.vx v10, v8, a3
+; RV32-NEXT:    vslide1down.vx v10, v8, a2
 ; RV32-NEXT:    vslide1down.vx v8, v9, a6
-; RV32-NEXT:    vmv.v.i v0, 15
 ; RV32-NEXT:    vslide1down.vx v8, v8, a7
-; RV32-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV32-NEXT:    vslide1down.vx v9, v10, a3
+; RV32-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_strided_unaligned:
@@ -14215,15 +14215,15 @@ define <8 x i16> @mgather_strided_unaligned(ptr %base) {
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    slli a0, a0, 8
 ; RV64ZVE32F-NEXT:    or a0, a0, t1
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a2
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 1, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14258,15 +14258,15 @@ define <8 x i16> @mgather_strided_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 26(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14303,15 +14303,15 @@ define <8 x i16> @mgather_strided_2xSEW_with_offset(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 30(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 2, i64 3, i64 6, i64 7, i64 10, i64 11, i64 14, i64 15>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14342,21 +14342,21 @@ define <8 x i16> @mgather_reverse_unit_strided_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a2, 26(a0)
 ; RV64ZVE32F-NEXT:    lh a3, 28(a0)
 ; RV64ZVE32F-NEXT:    lh a4, 30(a0)
-; RV64ZVE32F-NEXT:    lh a5, 16(a0)
-; RV64ZVE32F-NEXT:    lh a6, 18(a0)
-; RV64ZVE32F-NEXT:    lh a7, 20(a0)
+; RV64ZVE32F-NEXT:    lh a5, 20(a0)
+; RV64ZVE32F-NEXT:    lh a6, 16(a0)
+; RV64ZVE32F-NEXT:    lh a7, 18(a0)
 ; RV64ZVE32F-NEXT:    lh a0, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a3
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 14, i64 15, i64 12, i64 13, i64 10, i64 11, i64 8, i64 9>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14387,21 +14387,21 @@ define <8 x i16> @mgather_reverse_strided_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a2, 22(a0)
 ; RV64ZVE32F-NEXT:    lh a3, 28(a0)
 ; RV64ZVE32F-NEXT:    lh a4, 30(a0)
-; RV64ZVE32F-NEXT:    lh a5, 4(a0)
-; RV64ZVE32F-NEXT:    lh a6, 6(a0)
-; RV64ZVE32F-NEXT:    lh a7, 12(a0)
+; RV64ZVE32F-NEXT:    lh a5, 12(a0)
+; RV64ZVE32F-NEXT:    lh a6, 4(a0)
+; RV64ZVE32F-NEXT:    lh a7, 6(a0)
 ; RV64ZVE32F-NEXT:    lh a0, 14(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a3
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 14, i64 15, i64 10, i64 11, i64 6, i64 7, i64 2, i64 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14437,15 +14437,15 @@ define <8 x i16> @mgather_gather_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 6(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 2, i32 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14484,15 +14484,15 @@ define <8 x i16> @mgather_gather_2xSEW_unaligned(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 6(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 9, i32 10, i32 4, i32 5, i32 2, i32 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14531,15 +14531,15 @@ define <8 x i16> @mgather_gather_2xSEW_unaligned2(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 20(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a4
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a0
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a3
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 1, i32 2, i32 9, i32 10, i32 4, i32 5, i32 2, i32 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14582,15 +14582,15 @@ define <8 x i16> @mgather_gather_4xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 8, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14630,15 +14630,15 @@ define <8 x i16> @mgather_gather_4xSEW_partial_align(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14687,15 +14687,15 @@ define <8 x i16> @mgather_shuffle_rotate(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 6(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 4, i64 5, i64 6, i64 7, i64 0, i64 1, i64 2, i64 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14735,15 +14735,15 @@ define <8 x i16> @mgather_shuffle_vrgather(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 14(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 0, i64 2, i64 3, i64 1, i64 4, i64 5, i64 6, i64 7>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
index b6267bf481c85..0909c11078ff4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
@@ -1340,26 +1340,26 @@ define <4 x i64> @unzip2a_dual_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v4i64:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 0(a2)
-; ZVE32F-NEXT:    ld a2, 16(a2)
-; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a3, 0(a1)
 ; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    ld a4, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2a_dual_v4i64:
@@ -1378,9 +1378,9 @@ entry:
 define <16 x i64> @unzip2a_dual_v16i64(<16 x i64> %a, <16 x i64> %b) {
 ; V-LABEL: unzip2a_dual_v16i64:
 ; V:       # %bb.0: # %entry
-; V-NEXT:    lui a0, 5
 ; V-NEXT:    vsetivli zero, 16, e16, m1, ta, ma
 ; V-NEXT:    vid.v v16
+; V-NEXT:    lui a0, 5
 ; V-NEXT:    addi a0, a0, 1365
 ; V-NEXT:    vmv.s.x v20, a0
 ; V-NEXT:    li a0, -256
@@ -1526,26 +1526,26 @@ define <4 x i64> @unzip2a_dual_v4i64_exact(<4 x i64> %a, <4 x i64> %b) vscale_ra
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v4i64_exact:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 0(a2)
-; ZVE32F-NEXT:    ld a2, 16(a2)
-; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a3, 0(a1)
 ; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    ld a4, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vs1r.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vs1r.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2a_dual_v4i64_exact:
@@ -1574,26 +1574,26 @@ define <4 x i64> @unzip2a_dual_v4i64_exact_nf2(<4 x i64> %a, <4 x i64> %b) vscal
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v4i64_exact_nf2:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 0(a2)
-; ZVE32F-NEXT:    ld a2, 16(a2)
-; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a3, 0(a1)
 ; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    ld a4, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2a_dual_v4i64_exact_nf2:
@@ -1651,76 +1651,76 @@ define <16 x i64> @unzip2a_dual_v16i64_exact(<16 x i64> %a, <16 x i64> %b) vscal
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v16i64_exact:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a6, 0(a1)
+; ZVE32F-NEXT:    ld a5, 96(a2)
+; ZVE32F-NEXT:    ld a7, 0(a1)
 ; ZVE32F-NEXT:    ld a4, 16(a1)
-; ZVE32F-NEXT:    ld a7, 32(a1)
+; ZVE32F-NEXT:    ld t0, 32(a1)
 ; ZVE32F-NEXT:    ld a3, 48(a1)
-; ZVE32F-NEXT:    ld a5, 80(a1)
-; ZVE32F-NEXT:    ld t0, 96(a1)
 ; ZVE32F-NEXT:    ld t1, 64(a1)
+; ZVE32F-NEXT:    ld a6, 80(a1)
+; ZVE32F-NEXT:    ld t2, 96(a1)
 ; ZVE32F-NEXT:    ld a1, 112(a1)
-; ZVE32F-NEXT:    srli t2, a6, 32
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
-; ZVE32F-NEXT:    vmv.v.x v8, t0
+; ZVE32F-NEXT:    vmv.v.x v8, a7
+; ZVE32F-NEXT:    srli a7, a7, 32
+; ZVE32F-NEXT:    vmv.v.x v9, t0
 ; ZVE32F-NEXT:    srli t0, t0, 32
-; ZVE32F-NEXT:    vmv.v.x v9, t1
-; ZVE32F-NEXT:    srli t1, t1, 32
-; ZVE32F-NEXT:    vmv.v.x v10, a6
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, t1
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
-; ZVE32F-NEXT:    vslide1down.vx v10, v10, t2
-; ZVE32F-NEXT:    ld t1, 32(a2)
-; ZVE32F-NEXT:    ld t0, 16(a2)
-; ZVE32F-NEXT:    ld t2, 0(a2)
-; ZVE32F-NEXT:    ld a6, 48(a2)
-; ZVE32F-NEXT:    vmv.v.x v11, t1
+; ZVE32F-NEXT:    vmv.v.x v10, t1
 ; ZVE32F-NEXT:    srli t1, t1, 32
-; ZVE32F-NEXT:    vmv.v.x v12, t2
+; ZVE32F-NEXT:    vmv.v.x v11, t2
 ; ZVE32F-NEXT:    srli t2, t2, 32
-; ZVE32F-NEXT:    vmv.v.x v13, a7
-; ZVE32F-NEXT:    srli a7, a7, 32
-; ZVE32F-NEXT:    vslide1down.vx v13, v13, a7
-; ZVE32F-NEXT:    vslide1down.vx v12, v12, t2
-; ZVE32F-NEXT:    vslide1down.vx v11, v11, t1
-; ZVE32F-NEXT:    ld a7, 64(a2)
-; ZVE32F-NEXT:    ld t1, 80(a2)
-; ZVE32F-NEXT:    ld t2, 96(a2)
-; ZVE32F-NEXT:    ld a2, 112(a2)
-; ZVE32F-NEXT:    vmv.v.x v14, a7
-; ZVE32F-NEXT:    srli a7, a7, 32
-; ZVE32F-NEXT:    vslide1down.vx v14, v14, a7
-; ZVE32F-NEXT:    vmv.v.x v15, t2
-; ZVE32F-NEXT:    srli a7, t2, 32
-; ZVE32F-NEXT:    vslide1down.vx v15, v15, a7
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslide1down.vx v11, v11, t2
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, t1
+; ZVE32F-NEXT:    vslide1down.vx v12, v9, t0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    ld t0, 0(a2)
+; ZVE32F-NEXT:    ld t1, 16(a2)
+; ZVE32F-NEXT:    ld t2, 32(a2)
+; ZVE32F-NEXT:    ld a7, 48(a2)
+; ZVE32F-NEXT:    vmv.v.x v9, t0
+; ZVE32F-NEXT:    srli t0, t0, 32
+; ZVE32F-NEXT:    vmv.v.x v13, t2
+; ZVE32F-NEXT:    srli t2, t2, 32
+; ZVE32F-NEXT:    vslide1down.vx v13, v13, t2
+; ZVE32F-NEXT:    vslide1down.vx v14, v9, t0
+; ZVE32F-NEXT:    ld t0, 64(a2)
+; ZVE32F-NEXT:    ld t2, 112(a2)
+; ZVE32F-NEXT:    vmv.v.x v9, a5
 ; ZVE32F-NEXT:    srli a5, a5, 32
+; ZVE32F-NEXT:    vslide1down.vx v15, v9, a5
+; ZVE32F-NEXT:    ld a2, 80(a2)
+; ZVE32F-NEXT:    vmv.v.x v9, t0
+; ZVE32F-NEXT:    srli a5, t0, 32
 ; ZVE32F-NEXT:    vslide1down.vx v16, v9, a5
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v11, a1
 ; ZVE32F-NEXT:    srli a1, a1, 32
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v8, v10, a4
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, a6
+; ZVE32F-NEXT:    srli a1, a6, 32
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
 ; ZVE32F-NEXT:    srli a4, a4, 32
-; ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; ZVE32F-NEXT:    vslide1down.vx v8, v12, t0
-; ZVE32F-NEXT:    srli a1, t0, 32
-; ZVE32F-NEXT:    vslide1down.vx v12, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v11, v8, a4
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
 ; ZVE32F-NEXT:    vslide1down.vx v8, v14, t1
 ; ZVE32F-NEXT:    srli a1, t1, 32
 ; ZVE32F-NEXT:    vslide1down.vx v14, v8, a1
-; ZVE32F-NEXT:    vslidedown.vi v9, v16, 4, v0.t
-; ZVE32F-NEXT:    vslide1down.vx v8, v13, a3
+; ZVE32F-NEXT:    vslidedown.vi v9, v10, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v8, v12, a3
 ; ZVE32F-NEXT:    srli a3, a3, 32
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
-; ZVE32F-NEXT:    vslide1down.vx v10, v11, a6
-; ZVE32F-NEXT:    srli a1, a6, 32
+; ZVE32F-NEXT:    vslidedown.vi v8, v11, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v10, v13, a7
+; ZVE32F-NEXT:    srli a1, a7, 32
 ; ZVE32F-NEXT:    vslide1down.vx v10, v10, a1
-; ZVE32F-NEXT:    vslidedown.vi v10, v12, 4, v0.t
-; ZVE32F-NEXT:    vslide1down.vx v11, v15, a2
+; ZVE32F-NEXT:    vslidedown.vi v10, v14, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v11, v15, t2
+; ZVE32F-NEXT:    srli a1, t2, 32
+; ZVE32F-NEXT:    vslide1down.vx v11, v11, a1
+; ZVE32F-NEXT:    vslide1down.vx v12, v16, a2
 ; ZVE32F-NEXT:    srli a2, a2, 32
-; ZVE32F-NEXT:    vslide1down.vx v11, v11, a2
-; ZVE32F-NEXT:    vslidedown.vi v11, v14, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v12, v12, a2
+; ZVE32F-NEXT:    vslidedown.vi v11, v12, 4, v0.t
 ; ZVE32F-NEXT:    vs4r.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
@@ -1751,26 +1751,26 @@ define <4 x i64> @unzip2b_dual_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; ZVE32F-LABEL: unzip2b_dual_v4i64:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 8(a2)
-; ZVE32F-NEXT:    ld a2, 24(a2)
-; ZVE32F-NEXT:    ld a4, 8(a1)
+; ZVE32F-NEXT:    ld a3, 8(a1)
 ; ZVE32F-NEXT:    ld a1, 24(a1)
+; ZVE32F-NEXT:    ld a4, 8(a2)
+; ZVE32F-NEXT:    ld a2, 24(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2b_dual_v4i64:
@@ -1802,26 +1802,26 @@ define <4 x i64> @unzip2b_dual_v4i64_exact(<4 x i64> %a, <4 x i64> %b) vscale_ra
 ;
 ; ZVE32F-LABEL: unzip2b_dual_v4i64_exact:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 8(a2)
-; ZVE32F-NEXT:    ld a2, 24(a2)
-; ZVE32F-NEXT:    ld a4, 8(a1)
+; ZVE32F-NEXT:    ld a3, 8(a1)
 ; ZVE32F-NEXT:    ld a1, 24(a1)
+; ZVE32F-NEXT:    ld a4, 8(a2)
+; ZVE32F-NEXT:    ld a2, 24(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vs1r.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vs1r.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2b_dual_v4i64_exact:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll
index 917613d5c786f..3718156971919 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll
@@ -598,43 +598,54 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; ZIP-NEXT:    addi sp, sp, -16
 ; ZIP-NEXT:    .cfi_def_cfa_offset 16
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    slli a0, a0, 5
+; ZIP-NEXT:    li a1, 40
+; ZIP-NEXT:    mul a0, a0, a1
 ; ZIP-NEXT:    sub sp, sp, a0
-; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    li a1, 24
-; ZIP-NEXT:    mul a0, a0, a1
+; ZIP-NEXT:    slli a0, a0, 5
 ; ZIP-NEXT:    add a0, sp, a0
 ; ZIP-NEXT:    addi a0, a0, 16
 ; ZIP-NEXT:    vs8r.v v16, (a0) # vscale x 64-byte Folded Spill
 ; ZIP-NEXT:    addi a0, sp, 16
 ; ZIP-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; ZIP-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; ZIP-NEXT:    vslidedown.vi v24, v8, 16
 ; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; ZIP-NEXT:    vslidedown.vi v16, v8, 16
 ; ZIP-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; ZIP-NEXT:    ri.vzip2a.vv v16, v24, v0
+; ZIP-NEXT:    ri.vzip2a.vv v8, v16, v0
 ; ZIP-NEXT:    csrr a1, vlenb
-; ZIP-NEXT:    li a2, 24
-; ZIP-NEXT:    mul a1, a1, a2
+; ZIP-NEXT:    slli a1, a1, 3
 ; ZIP-NEXT:    add a1, sp, a1
 ; ZIP-NEXT:    addi a1, a1, 16
-; ZIP-NEXT:    vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; ZIP-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    slli a1, a1, 5
+; ZIP-NEXT:    add a1, sp, a1
+; ZIP-NEXT:    addi a1, a1, 16
+; ZIP-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; ZIP-NEXT:    vslidedown.vi v24, v24, 16
+; ZIP-NEXT:    vslidedown.vi v16, v16, 16
 ; ZIP-NEXT:    csrr a1, vlenb
-; ZIP-NEXT:    slli a1, a1, 4
+; ZIP-NEXT:    li a2, 24
+; ZIP-NEXT:    mul a1, a1, a2
 ; ZIP-NEXT:    add a1, sp, a1
 ; ZIP-NEXT:    addi a1, a1, 16
-; ZIP-NEXT:    vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
 ; ZIP-NEXT:    lui a1, 699051
 ; ZIP-NEXT:    addi a1, a1, -1366
 ; ZIP-NEXT:    vmv.s.x v0, a1
 ; ZIP-NEXT:    csrr a1, vlenb
-; ZIP-NEXT:    slli a1, a1, 3
+; ZIP-NEXT:    slli a1, a1, 4
 ; ZIP-NEXT:    add a1, sp, a1
 ; ZIP-NEXT:    addi a1, a1, 16
-; ZIP-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    li a2, 24
+; ZIP-NEXT:    mul a1, a1, a2
+; ZIP-NEXT:    add a1, sp, a1
+; ZIP-NEXT:    addi a1, a1, 16
+; ZIP-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    csrr a1, vlenb
 ; ZIP-NEXT:    slli a1, a1, 4
 ; ZIP-NEXT:    add a1, sp, a1
@@ -646,19 +657,21 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; ZIP-NEXT:    addi a1, a1, 16
 ; ZIP-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; ZIP-NEXT:    ri.vzip2a.vv v16, v8, v24, v0.t
+; ZIP-NEXT:    ri.vzip2a.vv v8, v24, v16, v0.t
+; ZIP-NEXT:    vmv.v.v v24, v8
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    li a1, 24
-; ZIP-NEXT:    mul a0, a0, a1
+; ZIP-NEXT:    slli a0, a0, 5
 ; ZIP-NEXT:    add a0, sp, a0
 ; ZIP-NEXT:    addi a0, a0, 16
-; ZIP-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; ZIP-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    addi a0, sp, 16
 ; ZIP-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; ZIP-NEXT:    ri.vzip2a.vv v0, v8, v24
+; ZIP-NEXT:    ri.vzip2a.vv v0, v8, v16
 ; ZIP-NEXT:    vmv.v.v v8, v0
+; ZIP-NEXT:    vmv.v.v v16, v24
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    slli a0, a0, 5
+; ZIP-NEXT:    li a1, 40
+; ZIP-NEXT:    mul a0, a0, a1
 ; ZIP-NEXT:    add sp, sp, a0
 ; ZIP-NEXT:    .cfi_def_cfa sp, 16
 ; ZIP-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
index 8676803e20e3b..f5216d82c81ba 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
@@ -195,15 +195,15 @@ define <8 x i64> @vrgather_shuffle_xv_v8i64(<8 x i64> %x) {
 ; RV32-NEXT:    lui a0, %hi(.LCPI12_0)
 ; RV32-NEXT:    addi a0, a0, %lo(.LCPI12_0)
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV32-NEXT:    vmv.v.i v16, -1
 ; RV32-NEXT:    vle16.v v20, (a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI12_1)
 ; RV32-NEXT:    addi a0, a0, %lo(.LCPI12_1)
 ; RV32-NEXT:    vle16.v v21, (a0)
+; RV32-NEXT:    vmv.v.i v16, -1
 ; RV32-NEXT:    li a0, 113
 ; RV32-NEXT:    vmv.s.x v0, a0
-; RV32-NEXT:    vrgatherei16.vv v12, v16, v20
-; RV32-NEXT:    vrgatherei16.vv v12, v8, v21, v0.t
+; RV32-NEXT:    vrgatherei16.vv v12, v16, v21
+; RV32-NEXT:    vrgatherei16.vv v12, v8, v20, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
@@ -227,12 +227,12 @@ define <8 x i64> @vrgather_shuffle_xv_v8i64(<8 x i64> %x) {
 define <8 x i64> @vrgather_shuffle_vx_v8i64(<8 x i64> %x) {
 ; RV32-LABEL: vrgather_shuffle_vx_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lui a0, %hi(.LCPI13_0)
-; RV32-NEXT:    addi a0, a0, %lo(.LCPI13_0)
-; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV32-NEXT:    vle16.v v16, (a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI13_1)
 ; RV32-NEXT:    addi a0, a0, %lo(.LCPI13_1)
+; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
+; RV32-NEXT:    vle16.v v16, (a0)
+; RV32-NEXT:    lui a0, %hi(.LCPI13_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI13_0)
 ; RV32-NEXT:    vle16.v v17, (a0)
 ; RV32-NEXT:    li a0, 140
 ; RV32-NEXT:    vmv.s.x v0, a0
@@ -388,14 +388,13 @@ define <8 x i8> @splat_ve2_we0_ins_i0ve4(<8 x i8> %v, <8 x i8> %w) {
 define <8 x i8> @splat_ve2_we0_ins_i0we4(<8 x i8> %v, <8 x i8> %w) {
 ; CHECK-LABEL: splat_ve2_we0_ins_i0we4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vrgather.vi v10, v8, 2
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v11, 4
 ; CHECK-NEXT:    li a0, 67
 ; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 4
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vrgather.vi v10, v8, 2
+; CHECK-NEXT:    vrgather.vv v10, v9, v11, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
   %shuff = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 12, i32 8, i32 2, i32 2, i32 2, i32 2, i32 8, i32 2>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll
index 8160e62a43106..fb22e5eaca2cc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll
@@ -7,28 +7,28 @@
 define <2 x i64> @test_vp_splice_v2i64(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i64> %v
 }
 
 define <2 x i64> @test_vp_splice_v2i64_negative_offset(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 -5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 -1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i64> %v
 }
 
@@ -46,54 +46,54 @@ define <2 x i64> @test_vp_splice_v2i64_zero_offset(<2 x i64> %va, <2 x i64> %vb,
 define <2 x i64> @test_vp_splice_v2i64_masked(<2 x i64> %va, <2 x i64> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 1, <2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <2 x i64> %v
 }
 
 define <4 x i32> @test_vp_splice_v4i32(<4 x i32> %va, <4 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i32> %v
 }
 
 define <4 x i32> @test_vp_splice_v4i32_negative_offset(<4 x i32> %va, <4 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i32_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e32, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 -5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 -3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i32> %v
 }
 
 define <4 x i32> @test_vp_splice_v4i32_masked(<4 x i32> %va, <4 x i32> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i32_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 3, <4 x i1> %mask, i32 %evla, i32 %evlb)
   ret <4 x i32> %v
 }
 
@@ -182,82 +182,82 @@ define <16 x i8> @test_vp_splice_v16i8_masked(<16 x i8> %va, <16 x i8> %vb, <16
 define <2 x double> @test_vp_splice_v2f64(<2 x double> %va, <2 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x double> %v
 }
 
 define <2 x double> @test_vp_splice_v2f64_negative_offset(<2 x double> %va, <2 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2f64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 -5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 -1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x double> %v
 }
 
 define <2 x double> @test_vp_splice_v2f64_masked(<2 x double> %va, <2 x double> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2f64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 1, <2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <2 x double> %v
 }
 
 define <4 x float> @test_vp_splice_v4f32(<4 x float> %va, <4 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x float> %v
 }
 
 define <4 x float> @test_vp_splice_v4f32_negative_offset(<4 x float> %va, <4 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4f32_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e32, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 -5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 -3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x float> %v
 }
 
 define <4 x float> @test_vp_splice_v4f32_masked(<4 x float> %va, <4 x float> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4f32_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 3, <4 x i1> %mask, i32 %evla, i32 %evlb)
   ret <4 x float> %v
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
index 984bc5b2c7352..b97fa1d3a51ec 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
@@ -25,8 +25,8 @@ define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
 ; RV32-NEXT:    vslide1down.vx v10, v10, a4
 ; RV32-NEXT:    vslide1down.vx v10, v10, a2
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -56,8 +56,8 @@ define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
 ; RV64-NEXT:    vslide1down.vx v10, v10, a4
 ; RV64-NEXT:    vslide1down.vx v10, v10, a2
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -95,8 +95,8 @@ define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
 ; RV32-NEXT:    vslide1down.vx v10, v10, a4
 ; RV32-NEXT:    vslide1down.vx v10, v10, a2
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -126,8 +126,8 @@ define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
 ; RV64-NEXT:    vslide1down.vx v10, v10, a4
 ; RV64-NEXT:    vslide1down.vx v10, v10, a2
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -166,8 +166,8 @@ define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vslide1down.vx v10, v10, a3
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -197,8 +197,8 @@ define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-NEXT:    vslide1down.vx v10, v10, a3
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -236,8 +236,8 @@ define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
 ; RV32-NEXT:    vslide1down.vx v10, v10, a4
 ; RV32-NEXT:    vslide1down.vx v10, v10, a2
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -267,8 +267,8 @@ define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
 ; RV64-NEXT:    vslide1down.vx v10, v10, a4
 ; RV64-NEXT:    vslide1down.vx v10, v10, a2
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -306,8 +306,8 @@ define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vslide1down.vx v10, v10, a3
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -337,8 +337,8 @@ define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-NEXT:    vslide1down.vx v10, v10, a3
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -377,8 +377,8 @@ define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vslide1down.vx v10, v10, a3
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -408,8 +408,8 @@ define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-NEXT:    vslide1down.vx v10, v10, a3
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
index 22ddd4f8a95d2..7b2d82a414644 100644
--- a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
@@ -607,6 +607,39 @@ define <vscale x 32 x bfloat> @insert_nxv32bf16_undef_nxv1bf16_26(<vscale x 1 x
   ret <vscale x 32 x bfloat> %v
 }
 
+define <vscale x 8 x i32> @insert_splat_to_undef() {
+; CHECK-LABEL: insert_splat_to_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 1
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv4i32.nxv8i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> splat (i32 1), i64 0)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @insert_splat_to_splat() {
+; CHECK-LABEL: insert_splat_to_splat:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 1
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv4i32.nxv8i32(<vscale x 8 x i32> splat (i32 1), <vscale x 4 x i32> splat (i32 1), i64 0)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @insert_splat_to_splat2() {
+; CHECK-LABEL: insert_splat_to_splat2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i32> @llvm.vector.insert.nxv4i32.nxv8i32(<vscale x 8 x i32> splat (i32 0), <vscale x 4 x i32> splat (i32 1), i64 0)
+  ret <vscale x 8 x i32> %v
+}
+
 attributes #0 = { vscale_range(2,1024) }
 
 declare <vscale x 4 x i1> @llvm.vector.insert.nxv1i1.nxv4i1(<vscale x 4 x i1>, <vscale x 1 x i1>, i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
index 2757e140ecde5..3633885bfa7d2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -2373,3 +2373,219 @@ define <vscale x 1 x i8> @mgather_baseidx_zext_nxv1i1_nxv1i8(ptr %base, <vscale
   %v = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8.nxv1p0(<vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %m, <vscale x 1 x i8> %passthru)
   ret <vscale x 1 x i8> %v
 }
+
+define <4 x i32> @scalar_prefix(ptr %base, i32 signext %index, <4 x i32> %vecidx) {
+; RV32-LABEL: scalar_prefix:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 10
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vluxei32.v v8, (a0), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: scalar_prefix:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    slli a1, a1, 10
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vwmulsu.vx v10, v8, a2
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    vluxei64.v v8, (a0), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, i32 %index, <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_prefix_with_splat(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: scalar_prefix_with_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vsll.vi v9, v9, 10
+; RV32-NEXT:    vadd.vx v9, v9, a0
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vadd.vv v8, v9, v8
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: scalar_prefix_with_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a2, 1024
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vmv.v.x v9, a2
+; RV64-NEXT:    vwmaccsu.vx v10, a1, v9
+; RV64-NEXT:    li a0, 4
+; RV64-NEXT:    vwmaccus.vx v10, a0, v8
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %broadcast.splat, <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_prefix_with_constant_splat(ptr %base, <4 x i32> %vecidx) {
+; RV32-LABEL: scalar_prefix_with_constant_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, 5
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vluxei32.v v8, (a0), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: scalar_prefix_with_constant_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 4
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vwmulsu.vx v10, v8, a1
+; RV64-NEXT:    lui a1, 5
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    vluxei64.v v8, (a0), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> splat (i32 20), <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: reassociate:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 10
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vadd.vx v8, v8, a0
+; RV32-NEXT:    vsll.vi v9, v9, 2
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reassociate:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    li a0, 1024
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vwmaccus.vx v10, a0, v8
+; RV64-NEXT:    vmv.v.i v8, 4
+; RV64-NEXT:    vwmaccsu.vx v10, a1, v8
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %vecidx, i32 %index
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate_with_splat(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: reassociate_with_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vsll.vi v8, v8, 10
+; RV32-NEXT:    vadd.vx v8, v8, a0
+; RV32-NEXT:    vsll.vi v9, v9, 2
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reassociate_with_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    li a0, 1024
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vwmaccus.vx v10, a0, v8
+; RV64-NEXT:    vmv.v.i v8, 4
+; RV64-NEXT:    vwmaccsu.vx v10, a1, v8
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %vecidx, <4 x i32> %broadcast.splat
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate_with_constant_splat(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: reassociate_with_constant_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 10
+; RV32-NEXT:    addi a0, a0, 80
+; RV32-NEXT:    vluxei32.v v8, (a0), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reassociate_with_constant_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 1024
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vwmulsu.vx v10, v8, a1
+; RV64-NEXT:    addi a0, a0, 80
+; RV64-NEXT:    vluxei64.v v8, (a0), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %vecidx, <4 x i32> splat (i32 20)
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @diagonal(ptr %base, <4 x i64> %vecidx) {
+; RV32-LABEL: diagonal:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wi v10, v8, 0
+; RV32-NEXT:    vsll.vi v8, v10, 10
+; RV32-NEXT:    vadd.vx v8, v8, a0
+; RV32-NEXT:    vsll.vi v9, v10, 2
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: diagonal:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vsll.vi v10, v8, 10
+; RV64-NEXT:    vadd.vx v10, v10, a0
+; RV64-NEXT:    vsll.vi v8, v8, 2
+; RV64-NEXT:    vadd.vv v10, v10, v8
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr inbounds nuw [256 x [256 x float]], ptr %base, i64 0, <4 x i64> %vecidx, <4 x i64> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @diagonal_i32(ptr %base, <4 x i32> %vecidx) {
+; RV32-LABEL: diagonal_i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v9, v8, 10
+; RV32-NEXT:    vadd.vx v9, v9, a0
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vadd.vv v8, v9, v8
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: diagonal_i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vsext.vf2 v10, v8
+; RV64-NEXT:    vsll.vi v8, v10, 10
+; RV64-NEXT:    vadd.vx v8, v8, a0
+; RV64-NEXT:    vsll.vi v10, v10, 2
+; RV64-NEXT:    vadd.vv v10, v8, v10
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr inbounds nuw [256 x [256 x float]], ptr %base, i64 0, <4 x i32> %vecidx, <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
deleted file mode 100644
index 0c058b562f53d..0000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v -stop-after=finalize-isel | FileCheck %s
-
-declare <vscale x 2 x i32> @llvm.vp.merge.nxv2i32(<vscale x 2 x i1>, <vscale x 2 x i32>, <vscale x 2 x i32>, i32)
-declare <vscale x 2 x i32> @llvm.vp.select.nxv2i32(<vscale x 2 x i1>, <vscale x 2 x i32>, <vscale x 2 x i32>, i32)
-declare <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr, <vscale x 2 x i1>, i32)
-
-; Test result has chain output of true operand of merge.vvm.
-define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale x 2 x i1> %m, i32 zeroext %vl) {
-  ; CHECK-LABEL: name: vpmerge_vpload_store
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $v8, $x10, $v0, $x11
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v0
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
-  ; CHECK-NEXT:   PseudoVSE32_V_M1 killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]], -1, 5 /* e32 */ :: (store (<vscale x 1 x s64>) into %ir.p)
-  ; CHECK-NEXT:   PseudoRET
-  %a = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr %p, <vscale x 2 x i1> splat (i1 -1), i32 %vl)
-  %b = call <vscale x 2 x i32> @llvm.vp.merge.nxv2i32(<vscale x 2 x i1> %m, <vscale x 2 x i32> %a, <vscale x 2 x i32> %passthru, i32 %vl)
-  store <vscale x 2 x i32> %b, ptr %p
-  ret void
-}
-
-define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale x 2 x i1> %m, i32 zeroext %vl) {
-  ; CHECK-LABEL: name: vpselect_vpload_store
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $v8, $x10, $v0, $x11
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v0
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
-  ; CHECK-NEXT:   PseudoVSE32_V_M1 killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]], -1, 5 /* e32 */ :: (store (<vscale x 1 x s64>) into %ir.p)
-  ; CHECK-NEXT:   PseudoRET
-  %a = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr %p, <vscale x 2 x i1> splat (i1 -1), i32 %vl)
-  %b = call <vscale x 2 x i32> @llvm.vp.select.nxv2i32(<vscale x 2 x i1> %m, <vscale x 2 x i32> %a, <vscale x 2 x i32> %passthru, i32 %vl)
-  store <vscale x 2 x i32> %b, ptr %p
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
index f8581d8e21b39..eaa59eee7dc05 100644
--- a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
@@ -3,9 +3,9 @@
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @select_nxv1f16(i1 zeroext %c, <vscale x 1 x half> %a, <vscale x 1 x half> %b) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 9972df97ad9f4..8495dfe350729 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -299,9 +299,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
@@ -318,9 +317,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, b
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
@@ -628,9 +626,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
@@ -647,9 +644,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, b
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v8, v9
@@ -978,9 +974,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 ; CHECK-LABEL: fcmp_ord_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -998,9 +993,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, b
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -1314,9 +1308,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 ; CHECK-LABEL: fcmp_uno_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -1334,9 +1327,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, b
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -1968,9 +1960,8 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
 define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v8, v9
@@ -1981,9 +1972,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmand.mm v0, v9, v8
@@ -1997,9 +1987,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_swap_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v9, v8
@@ -2010,9 +1999,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmand.mm v0, v8, v9
@@ -2451,9 +2439,8 @@ define <vscale x 1 x i1> @fcmp_uno_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
 define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v8, v9
@@ -2464,9 +2451,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
@@ -2480,9 +2466,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_swap_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v9, v8
@@ -2493,9 +2478,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v8, v9
@@ -2966,9 +2950,8 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfeq.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v10, v12
@@ -2977,9 +2960,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ; ZVFHMIN-LABEL: fcmp_ord_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -2996,9 +2978,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_swap_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfeq.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v12, v10
@@ -3007,9 +2988,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ; ZVFHMIN-LABEL: fcmp_ord_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -3459,9 +3439,8 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfne.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v10, v12
@@ -3470,9 +3449,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ; ZVFHMIN-LABEL: fcmp_uno_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -3489,9 +3467,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_swap_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfne.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v12, v10
@@ -3500,9 +3477,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ; ZVFHMIN-LABEL: fcmp_uno_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -4001,9 +3977,8 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f64(<vscale x 1 x double> %va, <vscale
 define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
@@ -4017,9 +3992,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double
 define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
@@ -4267,9 +4241,8 @@ define <vscale x 1 x i1> @fcmp_uno_vv_nxv1f64(<vscale x 1 x double> %va, <vscale
 define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v8, v9
@@ -4283,9 +4256,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f64(<vscale x 1 x double> %va, double
 define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
@@ -4551,9 +4523,8 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8f64(<vscale x 8 x double> %va, <vscale
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfeq.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v16, v24
@@ -4567,9 +4538,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f64(<vscale x 8 x double> %va, double
 define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfeq.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v24, v16
@@ -4820,9 +4790,8 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8f64(<vscale x 8 x double> %va, <vscale
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfne.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v16, v24
@@ -4836,9 +4805,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f64(<vscale x 8 x double> %va, double
 define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfne.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v24, v16
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index 4753ab915bdf3..5c04a09c9953b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -141,28 +141,26 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
 define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) {
 ; V-LABEL: vector_deinterleave_v8i64_v16i64:
 ; V:       # %bb.0:
-; V-NEXT:    li a0, 85
 ; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; V-NEXT:    vmv.v.i v0, -16
 ; V-NEXT:    vid.v v16
 ; V-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
 ; V-NEXT:    vslidedown.vi v24, v8, 8
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; V-NEXT:    vmv.v.i v0, -16
 ; V-NEXT:    vmv.s.x v12, a0
 ; V-NEXT:    li a0, 170
-; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; V-NEXT:    vadd.vv v20, v16, v16
-; V-NEXT:    vmv.s.x v21, a0
+; V-NEXT:    vadd.vv v13, v16, v16
+; V-NEXT:    vmv.s.x v20, a0
+; V-NEXT:    vadd.vi v21, v13, -8
 ; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; V-NEXT:    vcompress.vm v16, v8, v12
 ; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; V-NEXT:    vadd.vi v22, v20, -8
-; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; V-NEXT:    vcompress.vm v12, v8, v21
-; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; V-NEXT:    vadd.vi v8, v20, -7
+; V-NEXT:    vadd.vi v22, v13, -7
 ; V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; V-NEXT:    vrgatherei16.vv v16, v24, v22, v0.t
-; V-NEXT:    vrgatherei16.vv v12, v24, v8, v0.t
+; V-NEXT:    vcompress.vm v12, v8, v20
+; V-NEXT:    vrgatherei16.vv v16, v24, v21, v0.t
+; V-NEXT:    vrgatherei16.vv v12, v24, v22, v0.t
 ; V-NEXT:    vmv.v.v v8, v16
 ; V-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index b11db3d61f693..a353e8e4398fa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -49,6 +49,32 @@ define {<vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_load_nxv16i
   ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %res1
 }
 
+define <vscale x 16 x i8> @vector_deinterleave_load_nxv16i8_nxv32i8_oneactive(ptr %p) {
+; CHECK-LABEL: vector_deinterleave_load_nxv16i8_nxv32i8_oneactive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4r.v v12, (a0)
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %vec = load <vscale x 32 x i8>, ptr %p
+  %deinterleaved.results = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)
+  %t0 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.results, 0
+  ret <vscale x 16 x i8> %t0
+}
+
+define <vscale x 16 x i8> @vector_deinterleave_load_nxv16i8_nxv32i8_oneactive2(ptr %p) {
+; CHECK-LABEL: vector_deinterleave_load_nxv16i8_nxv32i8_oneactive2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4r.v v12, (a0)
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 8
+; CHECK-NEXT:    ret
+  %vec = load <vscale x 32 x i8>, ptr %p
+  %deinterleaved.results = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)
+  %t1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.results, 1
+  ret <vscale x 16 x i8> %t1
+}
+
 ; Shouldn't be lowered to vlseg because it's unaligned
 define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_load_nxv8i16_nxv16i16_align1(ptr %p) {
 ; CHECK-LABEL: vector_deinterleave_load_nxv8i16_nxv16i16_align1:
@@ -380,6 +406,90 @@ define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
   ret { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res3
 }
 
+define <vscale x 8 x i8> @vector_deinterleave_load_factor4_oneactive(ptr %p) {
+; CHECK-LABEL: vector_deinterleave_load_factor4_oneactive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 4 * vlenb
+; CHECK-NEXT:    vl4r.v v8, (a0)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vlseg4e8.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+  %vec = load <vscale x 32 x i8>, ptr %p
+  %d0 = call { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave4(<vscale x 32 x i8> %vec)
+  %t0 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %d0, 0
+  ret <vscale x 8 x i8> %t0
+}
+
+define <vscale x 8 x i8> @vector_deinterleave_load_factor4_oneactive2(ptr %p) {
+; CHECK-LABEL: vector_deinterleave_load_factor4_oneactive2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 4 * vlenb
+; CHECK-NEXT:    vl4r.v v8, (a0)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vlseg4e8.v v5, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+  %vec = load <vscale x 32 x i8>, ptr %p
+  %d0 = call { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave4(<vscale x 32 x i8> %vec)
+  %t0 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %d0, 3
+  ret <vscale x 8 x i8> %t0
+}
+
+define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @vector_deinterleave_load_factor4_twoactive(ptr %p) {
+; CHECK-LABEL: vector_deinterleave_load_factor4_twoactive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 4 * vlenb
+; CHECK-NEXT:    vl4r.v v8, (a0)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vlseg4e8.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+  %vec = load <vscale x 32 x i8>, ptr %p
+  %d0 = call { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave4(<vscale x 32 x i8> %vec)
+  %t0 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %d0, 0
+  %t1 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %d0, 1
+  %res0 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } poison, <vscale x 8 x i8> %t0, 0
+  %res1 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res0, <vscale x 8 x i8> %t1, 1
+  ret { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res1
+}
+
 define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @vector_deinterleave_load_factor5(ptr %p) {
 ; CHECK-LABEL: vector_deinterleave_load_factor5:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index 6144f916ea52b..0a96e4ff7fba7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -3683,3 +3683,64 @@ define {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vs
   %res = call {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave8.nxv16f64(<vscale x 16 x double> %arg)
   ret {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} %res
 }
+
+define <vscale x 16 x i8> @vector_deinterleave_nxv16i8_nxv32i8_oneactive(<vscale x 32 x i8> %vec) {
+; V-LABEL: vector_deinterleave_nxv16i8_nxv32i8_oneactive:
+; V:       # %bb.0:
+; V-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; V-NEXT:    vnsrl.wi v12, v8, 0
+; V-NEXT:    vmv.v.v v8, v12
+; V-NEXT:    ret
+;
+; ZIP-LABEL: vector_deinterleave_nxv16i8_nxv32i8_oneactive:
+; ZIP:       # %bb.0:
+; ZIP-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; ZIP-NEXT:    ri.vunzip2a.vv v12, v8, v10
+; ZIP-NEXT:    vmv.v.v v8, v12
+; ZIP-NEXT:    ret
+  %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)
+  %ext = extractvalue {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval, 0
+  ret <vscale x 16 x i8> %ext
+}
+
+define <vscale x 1 x float> @vector_deinterleave_nxv1f32_nxv8f32_oneactive(<vscale x 8 x float> %arg) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv1f32_nxv8f32_oneactive:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vlseg8e32.v v8, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %res = call {<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>} @llvm.vector.deinterleave8.nxv8f32(<vscale x 8 x float> %arg)
+  %ext = extractvalue {<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>} %res, 0
+  ret <vscale x 1 x float> %ext
+}
+
+define <vscale x 1 x float> @vector_deinterleave_nxv1f32_nxv8f32_oneactive2(<vscale x 8 x float> %arg) nounwind {
+; CHECK-LABEL: vector_deinterleave_nxv1f32_nxv8f32_oneactive2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vlseg8e32.v v3, (a0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %res = call {<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>} @llvm.vector.deinterleave8.nxv8f32(<vscale x 8 x float> %arg)
+  %ext = extractvalue {<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>} %res, 5
+  ret <vscale x 1 x float> %ext
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll b/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll
index 4a24e2342fabc..3a461c8721440 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll
@@ -11,7 +11,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv8i8(pt
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E8_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E8_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 1)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E8_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E8_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
@@ -27,7 +27,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv4i16(p
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E16_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E16_V_M1 $noreg, [[COPY1]], [[COPY]], 4 /* e16 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 2)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E16_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E16_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
@@ -43,7 +43,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv2i32(p
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E32_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E32_V_M1 $noreg, [[COPY1]], [[COPY]], 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 4)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E32_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E32_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
@@ -59,7 +59,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv1i64(p
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E64_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E64_V_M1 $noreg, [[COPY1]], [[COPY]], 6 /* e64 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 8)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E64_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E64_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll b/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
index 35bed86d61176..03e6e6b7a624d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
@@ -30,9 +30,8 @@ define <vscale x 1 x double> @test2(<vscale x 1 x double> %a, <vscale x 1 x i1>
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI1_0)(a1)
 ; CHECK-NEXT:    lui a1, %hi(.LCPI1_1)
 ; CHECK-NEXT:    fld fa4, %lo(.LCPI1_1)(a1)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa5
 ; CHECK-NEXT:    vfadd.vf v9, v9, fa4, v0.t
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
@@ -49,9 +48,8 @@ define <vscale x 1 x double> @test3(<vscale x 1 x double> %a, <vscale x 1 x doub
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI2_0)(a1)
 ; CHECK-NEXT:    lui a1, %hi(.LCPI2_1)
 ; CHECK-NEXT:    fld fa4, %lo(.LCPI2_1)(a1)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v10, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v10, fa5
 ; CHECK-NEXT:    vfmul.vf v10, v10, fa4, v0.t
 ; CHECK-NEXT:    vfmadd.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v10
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
index 1cbb980aebffc..0aa0cbceefc76 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -3081,24 +3081,66 @@ define <vscale x 1 x i8> @vmv_v_v(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <v
   ret <vscale x 1 x i8> %3
 }
 
-define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, iXLen %vl) {
 ; NOVLOPT-LABEL: vwsll_vi:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; NOVLOPT-NEXT:    vwsll.vi v12, v8, 1
+; NOVLOPT-NEXT:    vwsll.vi v10, v8, 1
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vwsll_vi:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; VLOPT-NEXT:    vwsll.vi v12, v8, 1
+; VLOPT-NEXT:    vwsll.vi v10, v8, 1
 ; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a,iXLen 1, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, iXLen 1, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwsll_vx(<vscale x 4 x i16> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsll_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwsll.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsll_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; VLOPT-NEXT:    vwsll.vx v10, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwsll_vv(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsll_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwsll.vv v10, v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsll_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; VLOPT-NEXT:    vwsll.vv v10, v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, <vscale x 4 x i16> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -3434,6 +3476,172 @@ define <vscale x 4 x i32> @vid.v(<vscale x 4 x i32> %c, iXLen %vl) {
   ret <vscale x 4 x i32> %2
 }
 
+define <vscale x 4 x i32> @vslideup_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslideup_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslideup.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslideup_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vslideup.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslideup(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslideup_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vslideup_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslideup.vi v10, v8, 2
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslideup_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vslideup.vi v10, v8, 2
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslideup(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 2, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslidedown_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslidedown_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslidedown.vx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslidedown_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vslidedown.vx v8, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslidedown(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslidedown_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vslidedown_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslidedown.vi v8, v8, 2
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslidedown_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vslidedown.vi v8, v8, 2
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslidedown(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 2, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslide1up_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslide1up_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslide1up.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslide1up_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vslide1up.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslide1up(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x float> @vfslide1up_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfslide1up_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfslide1up.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfslide1up_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfslide1up.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfslide1up(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+; Negative test – not safe to reduce vl
+
+define <vscale x 4 x i32> @vslide1down_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslide1down_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslide1down.vx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslide1down_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vslide1down.vx v8, v8, a0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslide1down(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+; Negative test – not safe to reduce vl
+
+define <vscale x 4 x float> @vfslide1down_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfslide1down_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfslide1down.vf v8, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfslide1down_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vfslide1down.vf v8, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfslide1down(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
 define <vscale x 4 x float> @vfadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
 ; NOVLOPT-LABEL: vfadd_vv:
 ; NOVLOPT:       # %bb.0:
@@ -4540,6 +4748,46 @@ define <vscale x 4 x float> @vfsgnjx_vf(<vscale x 4 x float> %a, float %b, iXLen
   ret <vscale x 4 x float> %2
 }
 
+define <vscale x 4 x float> @vfmerge_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmerge_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmerge.vfm v10, v8, fa0, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfmerge_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmerge.vfm v10, v8, fa0, v0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmerge(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfmv_v_f(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmv_v_f:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmv.v.f v10, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfmv_v_f:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmv.v.f v10, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmv.v.f(<vscale x 4 x float> poison, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
 define <vscale x 4 x float> @vfmacc_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
 ; NOVLOPT-LABEL: vfmacc_vv:
 ; NOVLOPT:       # %bb.0:
@@ -5117,3 +5365,29 @@ define <vscale x 4 x double> @vfrsqrt7(<vscale x 4 x float> %a) {
   %2 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %1, iXLen 7, iXLen 6, iXLen 0)
   ret <vscale x 4 x double> %2
 }
+
+define <vscale x 4 x double> @vfrec7(<vscale x 4 x float> %a) {
+; NOVLOPT-LABEL: vfrec7:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetivli zero, 7, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmv2r.v v12, v8
+; NOVLOPT-NEXT:    fsrmi a0, 0
+; NOVLOPT-NEXT:    vfrec7.v v14, v8
+; NOVLOPT-NEXT:    fsrm a0
+; NOVLOPT-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwmacc.vv v8, v12, v14
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfrec7:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
+; VLOPT-NEXT:    vmv2r.v v12, v8
+; VLOPT-NEXT:    fsrmi a0, 0
+; VLOPT-NEXT:    vfrec7.v v14, v8
+; VLOPT-NEXT:    fsrm a0
+; VLOPT-NEXT:    vfwmacc.vv v8, v12, v14
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfrec7.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, iXLen 0, iXLen 7)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %1, iXLen 7, iXLen 6, iXLen 0)
+  ret <vscale x 4 x double> %2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
new file mode 100644
index 0000000000000..03204468dc14c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc %s -o - -mtriple=riscv64 -mattr=+v -run-pass=riscv-vector-peephole -verify-machineinstrs | FileCheck %s
+
+---
+name: vle32
+body: |
+  bb.0:
+    liveins: $x8, $v0, $v8
+    ; CHECK-LABEL: name: vle32
+    ; CHECK: liveins: $x8, $v0, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %avl:gprnox0 = COPY $x8
+    ; CHECK-NEXT: %passthru:vrnov0 = COPY $v8
+    ; CHECK-NEXT: %mask:vmv0 = COPY $v0
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVLE32_V_M1_MASK %passthru, $noreg, %mask, %avl, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size, align 1)
+    %avl:gprnox0 = COPY $x8
+    %passthru:vrnov0 = COPY $v8
+    %x:vr = PseudoVLE32_V_M1 $noreg, $noreg, %avl, 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size)
+    %mask:vmv0 = COPY $v0
+    %y:vrnov0 = PseudoVMERGE_VVM_M1 %passthru, %passthru, %x, %mask, %avl, 5 /* e32 */
+...
+---
+name: vle32_no_passthru
+body: |
+  bb.0:
+    liveins: $x8, $v0, $v8
+    ; CHECK-LABEL: name: vle32_no_passthru
+    ; CHECK: liveins: $x8, $v0, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %avl:gprnox0 = COPY $x8
+    ; CHECK-NEXT: %false:vrnov0 = COPY $v8
+    ; CHECK-NEXT: %mask:vmv0 = COPY $v0
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVLE32_V_M1_MASK %false, $noreg, %mask, %avl, 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size, align 1)
+    %avl:gprnox0 = COPY $x8
+    %false:vr = COPY $v8
+    %x:vr = PseudoVLE32_V_M1 $noreg, $noreg, %avl, 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size)
+    %mask:vmv0 = COPY $v0
+    %y:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %x, %mask, %avl, 5 /* e32 */
+...
+---
+name: vle32_move_past_passthru
+body: |
+  bb.0:
+    liveins: $x8, $v0, $v8
+    ; CHECK-LABEL: name: vle32_move_past_passthru
+    ; CHECK: liveins: $x8, $v0, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %avl:gprnox0 = COPY $x8
+    ; CHECK-NEXT: %passthru:vrnov0 = COPY $v8
+    ; CHECK-NEXT: %mask:vmv0 = COPY $v0
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVLE32_V_M1_MASK %passthru, $noreg, %mask, %avl, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size, align 1)
+    %avl:gprnox0 = COPY $x8
+    %x:vr = PseudoVLE32_V_M1 $noreg, $noreg, %avl, 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size)
+    %passthru:vrnov0 = COPY $v8
+    %mask:vmv0 = COPY $v0
+    %y:vrnov0 = PseudoVMERGE_VVM_M1 %passthru, %passthru, %x, %mask, %avl, 5 /* e32 */
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir b/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir
index 2786f58826b91..ccabd5099071b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir
@@ -290,10 +290,10 @@ body:             |
     ; CHECK: liveins: $x2, $x10, $v8, $v13, $v4m4, $v16m4
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x0 = PseudoVSETVLI $x10, 66 /* e8, m4, ta, mu */, implicit-def $vl, implicit-def $vtype
-    ; CHECK-NEXT: early-clobber $v4m4 = PseudoVQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
+    ; CHECK-NEXT: early-clobber $v4m4 = PseudoSF_VQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: $v16m4 = PseudoVMV_V_V_M4 undef $v16m4, $v4m4, $noreg, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     $x0 = PseudoVSETVLI  $x10, 66, implicit-def $vl, implicit-def $vtype
-    early-clobber $v4m4 = PseudoVQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3, 1, implicit $vl, implicit $vtype
+    early-clobber $v4m4 = PseudoSF_VQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3, 1, implicit $vl, implicit $vtype
     $v16m4 = COPY renamable $v4m4
 ...
 ---
@@ -306,10 +306,10 @@ body:             |
     ; CHECK: liveins: $x2, $x10, $v8, $v13, $v4m4, $v16m2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x0 = PseudoVSETVLI $x10, 65 /* e8, m2, ta, mu */, implicit-def $vl, implicit-def $vtype
-    ; CHECK-NEXT: early-clobber $v4m4 = PseudoVQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
+    ; CHECK-NEXT: early-clobber $v4m4 = PseudoSF_VQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: $v16m4 = VMV4R_V $v4m4, implicit $vtype
     $x0 = PseudoVSETVLI  $x10, 65, implicit-def $vl, implicit-def $vtype
-    early-clobber $v4m4 = PseudoVQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3, 1, implicit $vl, implicit $vtype
+    early-clobber $v4m4 = PseudoSF_VQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3, 1, implicit $vl, implicit $vtype
     $v16m4 = COPY renamable $v4m4
 ...
 ---
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
index 745cec4e7c4f6..477f1daf10c24 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
@@ -19,15 +19,15 @@ define <2 x i1> @test_vp_splice_v2i1(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %ev
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 5
+; CHECK-NEXT:    vslidedown.vi v9, v9, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i1> %v
 }
 
@@ -43,15 +43,15 @@ define <2 x i1> @test_vp_splice_v2i1_negative_offset(<2 x i1> %va, <2 x i1> %vb,
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e8, mf8, ta, ma
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v9, v9, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
-; CHECK-NEXT:    vslideup.vi v9, v8, 5
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 -5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 -1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i1> %v
 }
 
@@ -67,16 +67,16 @@ define <2 x i1> @test_vp_splice_v2i1_masked(<2 x i1> %va, <2 x i1> %vb, <2 x i1>
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v10, v11, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v10, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v10, v10, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vslideup.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 1, <2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <2 x i1> %v
 }
 
@@ -92,15 +92,15 @@ define <4 x i1> @test_vp_splice_v4i1(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %ev
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 5
+; CHECK-NEXT:    vslidedown.vi v9, v9, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i1> %v
 }
 
@@ -116,15 +116,15 @@ define <4 x i1> @test_vp_splice_v4i1_negative_offset(<4 x i1> %va, <4 x i1> %vb,
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e8, mf4, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v9, v9, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
-; CHECK-NEXT:    vslideup.vi v9, v8, 5
+; CHECK-NEXT:    vslideup.vi v9, v8, 3
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 -5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 -3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i1> %v
 }
 
@@ -140,16 +140,16 @@ define <4 x i1> @test_vp_splice_v4i1_masked(<4 x i1> %va, <4 x i1> %vb, <4 x i1>
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v10, v11, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v10, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v10, v10, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vslideup.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 3, <4 x i1> %mask, i32 %evla, i32 %evlb)
   ret <4 x i1> %v
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll
index 3b0b183537468..709269904dbd8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-vectors.ll
@@ -10,7 +10,7 @@ declare <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i
 declare <vscale x 32 x i1> @llvm.experimental.vp.splice.nxv32i1(<vscale x 32 x i1>, <vscale x 32 x i1>, i32, <vscale x 32 x i1>, i32, i32)
 declare <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i1>, <vscale x 64 x i1>, i32, <vscale x 64 x i1>, i32, i32)
 
-define <vscale x 1 x i1> @test_vp_splice_nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x i1> @test_vp_splice_nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
@@ -22,19 +22,19 @@ define <vscale x 1 x i1> @test_vp_splice_nxv1i1(<vscale x 1 x i1> %va, <vscale x
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 5
+; CHECK-NEXT:    vslidedown.vi v9, v9, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 5, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 1, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 1 x i1> %v
 }
 
-define <vscale x 1 x i1> @test_vp_splice_nxv1i1_negative_offset(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x i1> @test_vp_splice_nxv1i1_negative_offset(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
@@ -46,19 +46,19 @@ define <vscale x 1 x i1> @test_vp_splice_nxv1i1_negative_offset(<vscale x 1 x i1
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e8, mf8, ta, ma
+; CHECK-NEXT:    addi a0, a0, -2
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v9, v9, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
-; CHECK-NEXT:    vslideup.vi v9, v8, 5
+; CHECK-NEXT:    vslideup.vi v9, v8, 2
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 -5, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 -2, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 1 x i1> %v
 }
 
-define <vscale x 1 x i1> @test_vp_splice_nxv1i1_masked(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x i1> @test_vp_splice_nxv1i1_masked(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
@@ -70,20 +70,20 @@ define <vscale x 1 x i1> @test_vp_splice_nxv1i1_masked(<vscale x 1 x i1> %va, <v
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v10, v11, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v10, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v10, v10, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vslideup.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 5, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 1, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 1 x i1> %v
 }
 
-define <vscale x 2 x i1> @test_vp_splice_nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i1> @test_vp_splice_nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
@@ -95,19 +95,19 @@ define <vscale x 2 x i1> @test_vp_splice_nxv2i1(<vscale x 2 x i1> %va, <vscale x
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 5
+; CHECK-NEXT:    vslidedown.vi v9, v9, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x i1> %v
 }
 
-define <vscale x 2 x i1> @test_vp_splice_nxv2i1_negative_offset(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i1> @test_vp_splice_nxv2i1_negative_offset(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
@@ -119,19 +119,19 @@ define <vscale x 2 x i1> @test_vp_splice_nxv2i1_negative_offset(<vscale x 2 x i1
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e8, mf4, ta, ma
+; CHECK-NEXT:    addi a0, a0, -4
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v9, v9, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
-; CHECK-NEXT:    vslideup.vi v9, v8, 5
+; CHECK-NEXT:    vslideup.vi v9, v8, 4
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 -4, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x i1> %v
 }
 
-define <vscale x 2 x i1> @test_vp_splice_nxv2i1_masked(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i1> @test_vp_splice_nxv2i1_masked(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
@@ -143,20 +143,20 @@ define <vscale x 2 x i1> @test_vp_splice_nxv2i1_masked(<vscale x 2 x i1> %va, <v
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v10, v11, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v10, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v10, v10, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vslideup.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 3, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 2 x i1> %v
 }
 
-define <vscale x 4 x i1> @test_vp_splice_nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 4 x i1> @test_vp_splice_nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv4i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
@@ -180,7 +180,7 @@ define <vscale x 4 x i1> @test_vp_splice_nxv4i1(<vscale x 4 x i1> %va, <vscale x
   ret <vscale x 4 x i1> %v
 }
 
-define <vscale x 4 x i1> @test_vp_splice_nxv4i1_negative_offset(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 4 x i1> @test_vp_splice_nxv4i1_negative_offset(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv4i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
@@ -204,7 +204,7 @@ define <vscale x 4 x i1> @test_vp_splice_nxv4i1_negative_offset(<vscale x 4 x i1
   ret <vscale x 4 x i1> %v
 }
 
-define <vscale x 4 x i1> @test_vp_splice_nxv4i1_masked(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 4 x i1> @test_vp_splice_nxv4i1_masked(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv4i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
@@ -229,7 +229,7 @@ define <vscale x 4 x i1> @test_vp_splice_nxv4i1_masked(<vscale x 4 x i1> %va, <v
   ret <vscale x 4 x i1> %v
 }
 
-define <vscale x 8 x i1> @test_vp_splice_nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 8 x i1> @test_vp_splice_nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv8i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
@@ -253,7 +253,7 @@ define <vscale x 8 x i1> @test_vp_splice_nxv8i1(<vscale x 8 x i1> %va, <vscale x
   ret <vscale x 8 x i1> %v
 }
 
-define <vscale x 8 x i1> @test_vp_splice_nxv8i1_negative_offset(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 8 x i1> @test_vp_splice_nxv8i1_negative_offset(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv8i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
@@ -277,7 +277,7 @@ define <vscale x 8 x i1> @test_vp_splice_nxv8i1_negative_offset(<vscale x 8 x i1
   ret <vscale x 8 x i1> %v
 }
 
-define <vscale x 8 x i1> @test_vp_splice_nxv8i1_masked(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 8 x i1> @test_vp_splice_nxv8i1_masked(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv8i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
@@ -302,7 +302,7 @@ define <vscale x 8 x i1> @test_vp_splice_nxv8i1_masked(<vscale x 8 x i1> %va, <v
   ret <vscale x 8 x i1> %v
 }
 
-define <vscale x 16 x i1> @test_vp_splice_nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 16 x i1> @test_vp_splice_nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv16i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
@@ -326,7 +326,7 @@ define <vscale x 16 x i1> @test_vp_splice_nxv16i1(<vscale x 16 x i1> %va, <vscal
   ret <vscale x 16 x i1> %v
 }
 
-define <vscale x 16 x i1> @test_vp_splice_nxv16i1_negative_offset(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 16 x i1> @test_vp_splice_nxv16i1_negative_offset(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv16i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
@@ -350,7 +350,7 @@ define <vscale x 16 x i1> @test_vp_splice_nxv16i1_negative_offset(<vscale x 16 x
   ret <vscale x 16 x i1> %v
 }
 
-define <vscale x 16 x i1> @test_vp_splice_nxv16i1_masked(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, <vscale x 16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 16 x i1> @test_vp_splice_nxv16i1_masked(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, <vscale x 16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv16i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
@@ -376,7 +376,7 @@ define <vscale x 16 x i1> @test_vp_splice_nxv16i1_masked(<vscale x 16 x i1> %va,
   ret <vscale x 16 x i1> %v
 }
 
-define <vscale x 32 x i1> @test_vp_splice_nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 32 x i1> @test_vp_splice_nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv32i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
@@ -400,7 +400,7 @@ define <vscale x 32 x i1> @test_vp_splice_nxv32i1(<vscale x 32 x i1> %va, <vscal
   ret <vscale x 32 x i1> %v
 }
 
-define <vscale x 32 x i1> @test_vp_splice_nxv32i1_negative_offset(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 32 x i1> @test_vp_splice_nxv32i1_negative_offset(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv32i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
@@ -424,7 +424,7 @@ define <vscale x 32 x i1> @test_vp_splice_nxv32i1_negative_offset(<vscale x 32 x
   ret <vscale x 32 x i1> %v
 }
 
-define <vscale x 32 x i1> @test_vp_splice_nxv32i1_masked(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, <vscale x 32 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 32 x i1> @test_vp_splice_nxv32i1_masked(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, <vscale x 32 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv32i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
@@ -450,7 +450,7 @@ define <vscale x 32 x i1> @test_vp_splice_nxv32i1_masked(<vscale x 32 x i1> %va,
   ret <vscale x 32 x i1> %v
 }
 
-define <vscale x 64 x i1> @test_vp_splice_nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 64 x i1> @test_vp_splice_nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv64i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
@@ -474,7 +474,7 @@ define <vscale x 64 x i1> @test_vp_splice_nxv64i1(<vscale x 64 x i1> %va, <vscal
   ret <vscale x 64 x i1> %v
 }
 
-define <vscale x 64 x i1> @test_vp_splice_nxv64i1_negative_offset(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 64 x i1> @test_vp_splice_nxv64i1_negative_offset(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv64i1_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
@@ -498,7 +498,7 @@ define <vscale x 64 x i1> @test_vp_splice_nxv64i1_negative_offset(<vscale x 64 x
   ret <vscale x 64 x i1> %v
 }
 
-define <vscale x 64 x i1> @test_vp_splice_nxv64i1_masked(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, <vscale x 64 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 64 x i1> @test_vp_splice_nxv64i1_masked(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, <vscale x 64 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv64i1_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
@@ -523,3 +523,5 @@ define <vscale x 64 x i1> @test_vp_splice_nxv64i1_masked(<vscale x 64 x i1> %va,
   %v = call <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 5, <vscale x 64 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 64 x i1> %v
 }
+
+attributes #0 = { vscale_range(2,0) }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll
index 9c8c5da75ff7c..e6a57ae6b1ea5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll
@@ -4,33 +4,33 @@
 ; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v,+zfh,+zfbfmin,+zvfhmin,+zvfbfmin -verify-machineinstrs \
 ; RUN:   < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
-define <vscale x 2 x i64> @test_vp_splice_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i64> @test_vp_splice_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x i64> %v
 }
 
-define <vscale x 2 x i64> @test_vp_splice_nxv2i64_negative_offset(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i64> @test_vp_splice_nxv2i64_negative_offset(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m2, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e64, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 5
+; CHECK-NEXT:    vslideup.vi v8, v10, 3
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 -3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x i64> %v
 }
 
-define <vscale x 2 x i64> @test_vp_splice_nxv2i64_zero_offset(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i64> @test_vp_splice_nxv2i64_zero_offset(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i64_zero_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
@@ -40,98 +40,98 @@ define <vscale x 2 x i64> @test_vp_splice_nxv2i64_zero_offset(<vscale x 2 x i64>
   ret <vscale x 2 x i64> %v
 }
 
-define <vscale x 2 x i64> @test_vp_splice_nxv2i64_masked(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i64> @test_vp_splice_nxv2i64_masked(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 3, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 2 x i64> %v
 }
 
-define <vscale x 1 x i64> @test_vp_splice_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x i64> @test_vp_splice_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 5, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 1, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 1 x i64> %v
 }
 
-define <vscale x 1 x i64> @test_vp_splice_nxv1i64_negative_offset(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x i64> @test_vp_splice_nxv1i64_negative_offset(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1i64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -2
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 -5, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 -2, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 1 x i64> %v
 }
 
-define <vscale x 1 x i64> @test_vp_splice_nxv1i64_masked(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x i64> @test_vp_splice_nxv1i64_masked(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1i64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 5, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 1, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 1 x i64> %v
 }
 
-define <vscale x 2 x i32> @test_vp_splice_nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x i32> %v
 }
 
-define <vscale x 2 x i32> @test_vp_splice_nxv2i32_negative_offset(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32_negative_offset(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i32_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e32, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -4
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 4
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 -4, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x i32> %v
 }
 
-define <vscale x 2 x i32> @test_vp_splice_nxv2i32_masked(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32_masked(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i32_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 3, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 2 x i32> %v
 }
 
-define <vscale x 4 x i16> @test_vp_splice_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 4 x i16> @test_vp_splice_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv4i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a0, a0, -5
@@ -144,7 +144,7 @@ define <vscale x 4 x i16> @test_vp_splice_nxv4i16(<vscale x 4 x i16> %va, <vscal
   ret <vscale x 4 x i16> %v
 }
 
-define <vscale x 4 x i16> @test_vp_splice_nxv4i16_negative_offset(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 4 x i16> @test_vp_splice_nxv4i16_negative_offset(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv4i16_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a0, a0, -5
@@ -157,7 +157,7 @@ define <vscale x 4 x i16> @test_vp_splice_nxv4i16_negative_offset(<vscale x 4 x
   ret <vscale x 4 x i16> %v
 }
 
-define <vscale x 4 x i16> @test_vp_splice_nxv4i16_masked(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 4 x i16> @test_vp_splice_nxv4i16_masked(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv4i16_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a0, a0, -5
@@ -170,7 +170,7 @@ define <vscale x 4 x i16> @test_vp_splice_nxv4i16_masked(<vscale x 4 x i16> %va,
   ret <vscale x 4 x i16> %v
 }
 
-define <vscale x 8 x i8> @test_vp_splice_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 8 x i8> @test_vp_splice_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a0, a0, -5
@@ -183,7 +183,7 @@ define <vscale x 8 x i8> @test_vp_splice_nxv8i8(<vscale x 8 x i8> %va, <vscale x
   ret <vscale x 8 x i8> %v
 }
 
-define <vscale x 8 x i8> @test_vp_splice_nxv8i8_negative_offset(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 8 x i8> @test_vp_splice_nxv8i8_negative_offset(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv8i8_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a0, a0, -5
@@ -196,7 +196,7 @@ define <vscale x 8 x i8> @test_vp_splice_nxv8i8_negative_offset(<vscale x 8 x i8
   ret <vscale x 8 x i8> %v
 }
 
-define <vscale x 8 x i8> @test_vp_splice_nxv8i8_masked(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 8 x i8> @test_vp_splice_nxv8i8_masked(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv8i8_masked:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a0, a0, -5
@@ -209,85 +209,85 @@ define <vscale x 8 x i8> @test_vp_splice_nxv8i8_masked(<vscale x 8 x i8> %va, <v
   ret <vscale x 8 x i8> %v
 }
 
-define <vscale x 1 x double> @test_vp_splice_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x double> @test_vp_splice_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 5, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 1, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 1 x double> %v
 }
 
-define <vscale x 1 x double> @test_vp_splice_nxv1f64_negative_offset(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x double> @test_vp_splice_nxv1f64_negative_offset(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1f64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -2
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 -5, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 -2, <vscale x 1 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 1 x double> %v
 }
 
-define <vscale x 1 x double> @test_vp_splice_nxv1f64_masked(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 1 x double> @test_vp_splice_nxv1f64_masked(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv1f64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 5, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 1, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 1 x double> %v
 }
 
-define <vscale x 2 x float> @test_vp_splice_nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x float> @test_vp_splice_nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x float> %v
 }
 
-define <vscale x 2 x float> @test_vp_splice_nxv2f32_negative_offset(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x float> @test_vp_splice_nxv2f32_negative_offset(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f32_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e32, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 -3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x float> %v
 }
 
-define <vscale x 2 x float> @test_vp_splice_nxv2f32_masked(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x float> @test_vp_splice_nxv2f32_masked(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f32_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 3, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 2 x float> %v
 }
 
-define <vscale x 16 x i64> @test_vp_splice_nxv16i64(<vscale x 16 x i64> %va, <vscale x 16 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) nounwind {
+define <vscale x 16 x i64> @test_vp_splice_nxv16i64(<vscale x 16 x i64> %va, <vscale x 16 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv16i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a4, vlenb
@@ -355,7 +355,7 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64(<vscale x 16 x i64> %va, <vs
   ret <vscale x 16 x i64> %v
 }
 
-define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16 x i64> %va, <vscale x 16 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) nounwind {
+define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16 x i64> %va, <vscale x 16 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv16i64_negative_offset:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a5, vlenb
@@ -428,85 +428,85 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16
   ret <vscale x 16 x i64> %v
 }
 
-define <vscale x 2 x half> @test_vp_splice_nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x half> @test_vp_splice_nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x half> %v
 }
 
-define <vscale x 2 x half> @test_vp_splice_nxv2f16_negative_offset(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x half> @test_vp_splice_nxv2f16_negative_offset(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f16_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e16, mf2, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 -3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x half> %v
 }
 
-define <vscale x 2 x half> @test_vp_splice_nxv2f16_masked(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x half> @test_vp_splice_nxv2f16_masked(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f16_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 3, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 2 x half> %v
 }
 
-define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x bfloat> %v
 }
 
-define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_negative_offset(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_negative_offset(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2bf16_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e16, mf2, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 -3, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 2 x bfloat> %v
 }
 
-define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_masked(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_masked(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2bf16_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 3, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <vscale x 2 x bfloat> %v
 }
 
-define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_firstelt(i32 %first, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_firstelt(i32 %first, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i32_with_firstelt:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
@@ -518,7 +518,7 @@ define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_firstelt(i32 %first, <vsc
   ret <vscale x 2 x i32> %v
 }
 
-define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_splat_firstelt(i32 %first, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_splat_firstelt(i32 %first, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2i32_with_splat_firstelt:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
@@ -531,7 +531,7 @@ define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_splat_firstelt(i32 %first
   ret <vscale x 2 x i32> %v
 }
 
-define <vscale x 2 x float> @test_vp_splice_nxv2f32_with_firstelt(float %first, <vscale x 2 x float> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+define <vscale x 2 x float> @test_vp_splice_nxv2f32_with_firstelt(float %first, <vscale x 2 x float> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2f32_with_firstelt:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
@@ -543,7 +543,7 @@ define <vscale x 2 x float> @test_vp_splice_nxv2f32_with_firstelt(float %first,
   ret <vscale x 2 x float> %v
 }
 
-define <vscale x 2 x half> @test_vp_splice_nxv2f16_with_firstelt(half %first, <vscale x 2 x half> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+define <vscale x 2 x half> @test_vp_splice_nxv2f16_with_firstelt(half %first, <vscale x 2 x half> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) #0 {
 ; ZVFH-LABEL: test_vp_splice_nxv2f16_with_firstelt:
 ; ZVFH:       # %bb.0:
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
@@ -563,7 +563,7 @@ define <vscale x 2 x half> @test_vp_splice_nxv2f16_with_firstelt(half %first, <v
   ret <vscale x 2 x half> %v
 }
 
-define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_with_firstelt(bfloat %first, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_with_firstelt(bfloat %first, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) #0 {
 ; CHECK-LABEL: test_vp_splice_nxv2bf16_with_firstelt:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fmv.x.h a1, fa0
@@ -575,3 +575,5 @@ define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_with_firstelt(bfloat %firs
   %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 0, <vscale x 2 x i1> %mask, i32 1, i32 %evl)
   ret <vscale x 2 x bfloat> %v
 }
+
+attributes #0 = { nounwind vscale_range(2,0) }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
index 35f01f608b56e..7fb822d20f892 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
@@ -536,37 +536,37 @@ define i32 @masked_load_store_factor2_v2_shared_mask_extract(<vscale x 2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v0
+; RV32-NEXT:    slli a2, a1, 1
 ; RV32-NEXT:    vmv.v.i v9, 0
-; RV32-NEXT:    li a2, -1
-; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    li a1, -1
+; RV32-NEXT:    vmerge.vim v10, v9, 1, v0
+; RV32-NEXT:    vwaddu.vv v11, v10, v10
+; RV32-NEXT:    vwmaccu.vx v11, a1, v10
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
 ; RV32-NEXT:    vmv.v.i v10, 0
+; RV32-NEXT:    srli a1, a1, 2
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmerge.vim v11, v9, 1, v0
-; RV32-NEXT:    vwaddu.vv v12, v11, v11
-; RV32-NEXT:    vwmaccu.vx v12, a2, v11
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    srli a2, a2, 2
-; RV32-NEXT:    vmsne.vi v0, v12, 0
+; RV32-NEXT:    vmsne.vi v0, v11, 0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vx v11, v12, a2
+; RV32-NEXT:    vslidedown.vx v11, v11, a1
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
 ; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v11, 0
-; RV32-NEXT:    slli a3, a1, 1
 ; RV32-NEXT:    vmerge.vim v9, v9, 1, v0
-; RV32-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vx v10, v9, a2
-; RV32-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v10, v9, a1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vle32.v v10, (a0), v0.t
 ; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vnsrl.wx v13, v10, a1
 ; RV32-NEXT:    vmv.x.s a1, v10
 ; RV32-NEXT:    vnsrl.wi v12, v10, 0
-; RV32-NEXT:    srli a3, a3, 1
+; RV32-NEXT:    srli a2, a2, 1
 ; RV32-NEXT:    vmv1r.v v0, v8
-; RV32-NEXT:    vsetvli zero, a3, e32, m1, ta, ma
+; RV32-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
 ; RV32-NEXT:    vsseg2e32.v v12, (a0), v0.t
 ; RV32-NEXT:    mv a0, a1
 ; RV32-NEXT:    ret
@@ -657,30 +657,30 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>} @not_same_mask(<vscale x 2 x i1>
 ; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v9, v0
 ; RV32-NEXT:    vmv1r.v v0, v8
+; RV32-NEXT:    slli a1, a1, 1
 ; RV32-NEXT:    vmv.v.i v8, 0
 ; RV32-NEXT:    li a2, -1
-; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vmv.v.i v10, 0
-; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmerge.vim v11, v8, 1, v0
+; RV32-NEXT:    vmerge.vim v10, v8, 1, v0
 ; RV32-NEXT:    vmv1r.v v0, v9
 ; RV32-NEXT:    vmerge.vim v9, v8, 1, v0
-; RV32-NEXT:    vwaddu.vv v12, v9, v11
-; RV32-NEXT:    vwmaccu.vx v12, a2, v11
+; RV32-NEXT:    vwaddu.vv v11, v9, v10
+; RV32-NEXT:    vwmaccu.vx v11, a2, v10
 ; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV32-NEXT:    vmv.v.i v9, 0
 ; RV32-NEXT:    srli a2, a2, 2
-; RV32-NEXT:    vmsne.vi v0, v12, 0
-; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vx v9, v12, a2
-; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmsne.vi v0, v9, 0
-; RV32-NEXT:    slli a1, a1, 1
-; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    vmsne.vi v0, v11, 0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vx v10, v8, a2
+; RV32-NEXT:    vslidedown.vx v10, v11, a2
 ; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV32-NEXT:    vmerge.vim v9, v9, 1, v0
+; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v9, v8, a2
+; RV32-NEXT:    vmsne.vi v0, v9, 0
 ; RV32-NEXT:    vle32.v v10, (a0), v0.t
 ; RV32-NEXT:    li a0, 32
 ; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
index 75c00e406b4f1..809276c751af8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vrgatherei16.vv.nxv1i8(
@@ -1449,3 +1449,208 @@ entry:
 
   ret <vscale x 8 x double> %a
 }
+
+define <vscale x 1 x bfloat> @intrinsic_vrgatherei16_vv_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v9
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv1bf16(
+    <vscale x 1 x bfloat> undef,
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    <vscale x 1 x i16> %2,
+    <vscale x 1 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vrgatherei16_vv_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v9
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv2bf16(
+    <vscale x 2 x bfloat> undef,
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    <vscale x 2 x i16> %2,
+    <vscale x 2 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vrgatherei16_vv_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v9
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv4bf16(
+    <vscale x 4 x bfloat> undef,
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    <vscale x 4 x i16> %2,
+    <vscale x 4 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vrgatherei16_vv_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v10
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv8bf16(
+    <vscale x 8 x bfloat> undef,
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v10, v12, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    <vscale x 8 x i16> %2,
+    <vscale x 8 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vrgatherei16_vv_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v12
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv16bf16(
+    <vscale x 16 x bfloat> undef,
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    <vscale x 16 x i16> %2,
+    <vscale x 16 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 32 x bfloat> @intrinsic_vrgatherei16_vv_nxv32bf16_nxv32bf16(<vscale x 32 x bfloat> %0, <vscale x 32 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv32bf16_nxv32bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v24, v8, v16
+; CHECK-NEXT:    vmv.v.v v8, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv32bf16(
+    <vscale x 32 x bfloat> undef,
+    <vscale x 32 x bfloat> %0,
+    <vscale x 32 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 32 x bfloat> %a
+}
+
+define <vscale x 32 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv32bf16_nxv32bf16(<vscale x 32 x bfloat> %0, <vscale x 32 x bfloat> %1, <vscale x 32 x i16> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv32bf16_nxv32bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8re16.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v16, v24, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16(
+    <vscale x 32 x bfloat> %0,
+    <vscale x 32 x bfloat> %1,
+    <vscale x 32 x i16> %2,
+    <vscale x 32 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 32 x bfloat> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 8b48dc43eca29..439974d56e389 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -307,9 +307,8 @@ define <vscale x 1 x double> @test16(i64 %avl, double %a, <vscale x 1 x double>
 ; CHECK-LABEL: test16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e64, mf2, ta, ma
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v8
 ; CHECK-NEXT:    ret
 entry:
@@ -722,3 +721,35 @@ define i64 @avl_undef2() {
   %1 = tail call i64 @llvm.riscv.vsetvli(i64 poison, i64 2, i64 7)
   ret i64 %1
 }
+
+define i64 @vsetvli_vleff(ptr %s, i64 %evl) {
+; CHECK-LABEL: vsetvli_vleff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:  .LBB37_1: # %while.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vle16ff.v v9, (a0)
+; CHECK-NEXT:    csrr a2, vl
+; CHECK-NEXT:    beqz a2, .LBB37_1
+; CHECK-NEXT:  # %bb.2: # %while.end
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+entry:
+  br label %while.cond
+
+while.cond:
+  %new_vl.0 = phi i64 [ 0, %entry ], [ %1, %while.body ]
+  %cmp = icmp eq i64 %new_vl.0, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:
+  %0 = tail call { <vscale x 4 x i16>, i64 } @llvm.riscv.vleff.nxv4i16.i64(<vscale x 4 x i16> zeroinitializer, ptr %s, i64 %evl)
+  %1 = extractvalue { <vscale x 4 x i16>, i64 } %0, 1
+  br label %while.cond
+
+while.end:
+  ret i64 0
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
index 6bd03eb0c2226..fdd30c9a2c772 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
@@ -100,6 +100,10 @@
     ret void
   }
 
+  define void @vsetvli_vleff() {
+    ret void
+  }
+
   declare <vscale x 1 x i64> @llvm.riscv.vadd.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, i64) #1
 
   declare <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i64(<vscale x 1 x i64>, ptr nocapture, i64) #4
@@ -622,3 +626,41 @@ body: |
     dead $x0 = PseudoVSETIVLI 1, 208, implicit-def $vl, implicit-def $vtype
     %v:vr = COPY $v8, implicit $vtype
     %x = PseudoVSETVLI %x, 208, implicit-def $vl, implicit-def $vtype
+...
+---
+name:            vsetvli_vleff
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: vsetvli_vleff
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %vl:gpr = COPY $x0
+  ; CHECK-NEXT:   BNE $x0, $x0, %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead $x0 = PseudoVSETIVLI 0, 200 /* e16, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
+  ; CHECK-NEXT:   $noreg, $x0 = PseudoVLE16FF_V_M1 $noreg, $noreg, 0, 4 /* e16 */, 2 /* tu, ma */, implicit $vl, implicit $vtype, implicit-def $vl
+  ; CHECK-NEXT:   %vl:gpr = PseudoReadVL implicit $vl
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   $x10 = COPY %vl
+  ; CHECK-NEXT:   PseudoRET implicit killed $x10
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+
+    %vl:gpr = COPY $x0
+    BNE $x0, $x0, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    $noreg, %vl:gpr = PseudoVLE16FF_V_M1 $noreg, $noreg, 0, 4 /* e16 */, 2 /* tu, ma */
+
+  bb.2:
+    $x10 = COPY %vl
+    PseudoRET implicit killed $x10
diff --git a/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll b/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
index f0f78c211c4a9..22a90cc2c94ac 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslidedown.nxv1i8(
@@ -2463,3 +2463,343 @@ entry:
 
   ret <vscale x 4 x double> %a
 }
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_mask_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    <vscale x 1 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_mask_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    <vscale x 1 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_mask_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    <vscale x 2 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_mask_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    <vscale x 2 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_mask_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    <vscale x 4 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_mask_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    <vscale x 4 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v10, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_mask_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v10, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    <vscale x 8 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v10, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_mask_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v10, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    <vscale x 8 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v12, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_mask_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v12, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    <vscale x 16 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v12, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_mask_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v12, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    <vscale x 16 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vslideup.ll b/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
index 8e3c05611bc71..0291207ccdb49 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslideup.nxv1i8(
@@ -2463,3 +2463,343 @@ entry:
 
   ret <vscale x 4 x double> %a
 }
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_mask_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    <vscale x 1 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_mask_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    <vscale x 1 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_mask_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    <vscale x 2 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_mask_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    <vscale x 2 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_mask_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    <vscale x 4 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_mask_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    <vscale x 4 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_mask_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v10, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    <vscale x 8 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_mask_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v10, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    <vscale x 8 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v12, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_mask_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v12, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    <vscale x 16 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_mask_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v12, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    <vscale x 16 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
index f94e46771f49c..c8b882b92b934 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
@@ -1540,12 +1540,11 @@ define <vscale x 8 x i32> @vwadd_vx_splat_zext_i1(<vscale x 8 x i1> %va, i16 %b)
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    slli a0, a0, 48
 ; RV64-NEXT:    srli a0, a0, 48
-; RV64-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v12, a0
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; RV64-NEXT:    vmv.v.x v12, a0
+; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vwaddu.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
   %zb = zext i16 %b to i32
@@ -1615,12 +1614,11 @@ define <vscale x 8 x i32> @vwadd_vx_splat_sext_i1(<vscale x 8 x i1> %va, i16 %b)
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    slli a0, a0, 48
 ; RV64-NEXT:    srai a0, a0, 48
-; RV64-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v12, a0
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; RV64-NEXT:    vmv.v.x v12, a0
+; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vwsub.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
   %sb = sext i16 %b to i32
diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index 990039d765df8..061435c45ad0e 100644
--- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -2,13 +2,13 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+c,+zbb -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=NOSFB %s
 ; RUN: llc -mtriple=riscv64 -mcpu=sifive-u74 -mattr=+zbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=SFB,NOZICOND,RV64SFB,RV64SFBSIFIVEU74 %s
+; RUN:   | FileCheck -check-prefixes=RV64SFB,RV64SFBSIFIVEU74 %s
 ; RUN: llc -mtriple=riscv64 -mcpu=andes-ax45 -mattr=+zbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=SFB,NOZICOND,RV64SFB,RV64SFBANDESAX45 %s
+; RUN:   | FileCheck -check-prefixes=RV64SFB,RV64SFBANDESAX45 %s
 ; RUN: llc -mtriple=riscv64 -mcpu=sifive-u74 -mattr=+zicond,+zbb \
-; RUN:   -verify-machineinstrs < %s | FileCheck -check-prefixes=SFB,ZICOND %s
+; RUN:   -verify-machineinstrs < %s | FileCheck -check-prefixes=ZICOND %s
 ; RUN: llc -mtriple=riscv32 -mcpu=sifive-e76 -mattr=+zbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=SFB,NOZICOND,RV32SFB %s
+; RUN:   | FileCheck -check-prefixes=RV32SFB %s
 
 ; The sifive-7-series can predicate a mv.
 
@@ -21,13 +21,37 @@ define signext i32 @test1(i32 signext %x, i32 signext %y, i32 signext %z) {
 ; NOSFB-NEXT:  .LBB0_2:
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: test1:
-; SFB:       # %bb.0:
-; SFB-NEXT:    beqz a2, .LBB0_2
-; SFB-NEXT:  # %bb.1:
-; SFB-NEXT:    mv a0, a1
-; SFB-NEXT:  .LBB0_2:
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test1:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB0_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB0_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test1:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a2, 0, .LBB0_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB0_2:
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: test1:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    beqz a2, .LBB0_2
+; ZICOND-NEXT:  # %bb.1:
+; ZICOND-NEXT:    mv a0, a1
+; ZICOND-NEXT:  .LBB0_2:
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test1:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    beqz a2, .LBB0_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a1
+; RV32SFB-NEXT:  .LBB0_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 %y
   ret i32 %b
@@ -44,13 +68,38 @@ define signext i32 @test2(i32 signext %x, i32 signext %y, i32 signext %z) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: test2:
-; SFB:       # %bb.0:
-; SFB-NEXT:    bnez a2, .LBB1_2
-; SFB-NEXT:  # %bb.1:
-; SFB-NEXT:    mv a0, a1
-; SFB-NEXT:  .LBB1_2:
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test2:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB1_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB1_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test2:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a2, 0, .LBB1_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    mv a1, a0
+; RV64SFBANDESAX45-NEXT:  .LBB1_2:
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: test2:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    bnez a2, .LBB1_2
+; ZICOND-NEXT:  # %bb.1:
+; ZICOND-NEXT:    mv a0, a1
+; ZICOND-NEXT:  .LBB1_2:
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test2:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    bnez a2, .LBB1_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a1
+; RV32SFB-NEXT:  .LBB1_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %y, i32 %x
   ret i32 %b
@@ -84,15 +133,17 @@ define signext i32 @test3(i32 signext %v, i32 signext %w, i32 signext %x, i32 si
 ;
 ; RV64SFBANDESAX45-LABEL: test3:
 ; RV64SFBANDESAX45:       # %bb.0:
-; RV64SFBANDESAX45-NEXT:    bnez a4, .LBB2_2
+; RV64SFBANDESAX45-NEXT:    nds.bnec a4, 0, .LBB2_3
 ; RV64SFBANDESAX45-NEXT:  # %bb.1:
-; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    nds.bnec a4, 0, .LBB2_4
 ; RV64SFBANDESAX45-NEXT:  .LBB2_2:
-; RV64SFBANDESAX45-NEXT:    beqz a4, .LBB2_4
-; RV64SFBANDESAX45-NEXT:  # %bb.3:
-; RV64SFBANDESAX45-NEXT:    mv a2, a3
+; RV64SFBANDESAX45-NEXT:    addw a0, a1, a2
+; RV64SFBANDESAX45-NEXT:    ret
+; RV64SFBANDESAX45-NEXT:  .LBB2_3:
+; RV64SFBANDESAX45-NEXT:    mv a1, a0
+; RV64SFBANDESAX45-NEXT:    nds.beqc a4, 0, .LBB2_2
 ; RV64SFBANDESAX45-NEXT:  .LBB2_4:
-; RV64SFBANDESAX45-NEXT:    addw a0, a0, a2
+; RV64SFBANDESAX45-NEXT:    addw a0, a1, a3
 ; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test3:
@@ -136,18 +187,34 @@ define signext i32 @test4(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    and a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test4:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    beqz a1, .LBB3_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    li a0, 0
-; NOZICOND-NEXT:  .LBB3_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test4:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB3_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    li a0, 0
+; RV64SFBSIFIVEU74-NEXT:  .LBB3_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test4:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a1, 0, .LBB3_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, 0
+; RV64SFBANDESAX45-NEXT:  .LBB3_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test4:
 ; ZICOND:       # %bb.0:
 ; ZICOND-NEXT:    czero.nez a0, a0, a1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test4:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    beqz a1, .LBB3_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    li a0, 0
+; RV32SFB-NEXT:  .LBB3_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 0
   ret i32 %b
@@ -162,18 +229,34 @@ define signext i32 @test5(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    and a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test5:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    bnez a1, .LBB4_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    li a0, 0
-; NOZICOND-NEXT:  .LBB4_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test5:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    bnez a1, .LBB4_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    li a0, 0
+; RV64SFBSIFIVEU74-NEXT:  .LBB4_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test5:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.bnec a1, 0, .LBB4_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, 0
+; RV64SFBANDESAX45-NEXT:  .LBB4_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test5:
 ; ZICOND:       # %bb.0:
 ; ZICOND-NEXT:    czero.eqz a0, a0, a1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test5:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    bnez a1, .LBB4_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    li a0, 0
+; RV32SFB-NEXT:  .LBB4_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 0, i32 %x
   ret i32 %b
@@ -188,14 +271,22 @@ define signext i32 @test6(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test6:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    li a2, -1
-; NOZICOND-NEXT:    beqz a1, .LBB5_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    mv a0, a2
-; NOZICOND-NEXT:  .LBB5_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test6:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    li a2, -1
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB5_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:  .LBB5_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test6:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a1, 0, .LBB5_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, -1
+; RV64SFBANDESAX45-NEXT:  .LBB5_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test6:
 ; ZICOND:       # %bb.0:
@@ -203,6 +294,15 @@ define signext i32 @test6(i32 signext %x, i32 signext %z) {
 ; ZICOND-NEXT:    czero.nez a0, a0, a1
 ; ZICOND-NEXT:    addi a0, a0, -1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test6:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    li a2, -1
+; RV32SFB-NEXT:    beqz a1, .LBB5_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a2
+; RV32SFB-NEXT:  .LBB5_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 -1
   ret i32 %b
@@ -217,14 +317,22 @@ define signext i32 @test7(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test7:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    li a2, -1
-; NOZICOND-NEXT:    bnez a1, .LBB6_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    mv a0, a2
-; NOZICOND-NEXT:  .LBB6_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test7:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    li a2, -1
+; RV64SFBSIFIVEU74-NEXT:    bnez a1, .LBB6_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:  .LBB6_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test7:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.bnec a1, 0, .LBB6_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, -1
+; RV64SFBANDESAX45-NEXT:  .LBB6_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test7:
 ; ZICOND:       # %bb.0:
@@ -232,6 +340,15 @@ define signext i32 @test7(i32 signext %x, i32 signext %z) {
 ; ZICOND-NEXT:    czero.eqz a0, a0, a1
 ; ZICOND-NEXT:    addi a0, a0, -1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test7:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    li a2, -1
+; RV32SFB-NEXT:    bnez a1, .LBB6_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a2
+; RV32SFB-NEXT:  .LBB6_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 -1, i32 %x
   ret i32 %b
@@ -246,14 +363,40 @@ define i16 @select_xor_1(i16 %A, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_1:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a1, a1, 1
-; SFB-NEXT:    beqz a1, .LBB7_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xori a0, a0, 43
-; SFB-NEXT:  .LBB7_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_1:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a1, a1, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB7_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xori a0, a0, 43
+; RV64SFBSIFIVEU74-NEXT:  .LBB7_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_1:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a1, 0, .LBB7_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xori a0, a0, 43
+; RV64SFBANDESAX45-NEXT:  .LBB7_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_1:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a1, a1, 1
+; ZICOND-NEXT:    beqz a1, .LBB7_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xori a0, a0, 43
+; ZICOND-NEXT:  .LBB7_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_1:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a1, a1, 1
+; RV32SFB-NEXT:    beqz a1, .LBB7_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xori a0, a0, 43
+; RV32SFB-NEXT:  .LBB7_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp eq i8 %and, 0
@@ -273,14 +416,40 @@ define i16 @select_xor_1b(i16 %A, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_1b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a1, a1, 1
-; SFB-NEXT:    beqz a1, .LBB8_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xori a0, a0, 43
-; SFB-NEXT:  .LBB8_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_1b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a1, a1, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB8_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xori a0, a0, 43
+; RV64SFBSIFIVEU74-NEXT:  .LBB8_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_1b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a1, 0, .LBB8_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    xori a0, a0, 43
+; RV64SFBANDESAX45-NEXT:  .LBB8_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_1b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a1, a1, 1
+; ZICOND-NEXT:    beqz a1, .LBB8_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xori a0, a0, 43
+; ZICOND-NEXT:  .LBB8_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_1b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a1, a1, 1
+; RV32SFB-NEXT:    beqz a1, .LBB8_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xori a0, a0, 43
+; RV32SFB-NEXT:  .LBB8_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp ne i8 %and, 1
@@ -298,14 +467,40 @@ define i32 @select_xor_2(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_2:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB9_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xor a0, a0, a1
-; SFB-NEXT:  .LBB9_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_2:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB9_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xor a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB9_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_2:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB9_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xor a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB9_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_2:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB9_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xor a0, a0, a1
+; ZICOND-NEXT:  .LBB9_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_2:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB9_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xor a0, a0, a1
+; RV32SFB-NEXT:  .LBB9_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp eq i8 %and, 0
@@ -325,14 +520,40 @@ define i32 @select_xor_2b(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_2b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB10_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xor a0, a0, a1
-; SFB-NEXT:  .LBB10_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_2b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB10_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xor a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB10_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_2b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB10_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    xor a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB10_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_2b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB10_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xor a0, a0, a1
+; ZICOND-NEXT:  .LBB10_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_2b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB10_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xor a0, a0, a1
+; RV32SFB-NEXT:  .LBB10_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp ne i8 %and, 1
@@ -350,14 +571,40 @@ define i32 @select_or(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB11_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB11_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB11_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB11_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB11_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB11_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB11_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB11_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB11_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB11_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp eq i8 %and, 0
@@ -377,14 +624,40 @@ define i32 @select_or_b(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or_b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB12_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB12_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or_b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB12_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB12_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or_b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB12_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB12_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or_b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB12_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB12_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or_b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB12_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB12_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp ne i8 %and, 1
@@ -402,14 +675,40 @@ define i32 @select_or_1(i32 %A, i32 %B, i32 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or_1:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB13_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB13_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or_1:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB13_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB13_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or_1:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB13_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB13_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or_1:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB13_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB13_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or_1:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB13_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB13_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i32 %cond, 1
  %cmp10 = icmp eq i32 %and, 0
@@ -429,14 +728,40 @@ define i32 @select_or_1b(i32 %A, i32 %B, i32 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or_1b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB14_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB14_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or_1b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB14_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB14_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or_1b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB14_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB14_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or_1b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB14_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB14_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or_1b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB14_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB14_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i32 %cond, 1
  %cmp10 = icmp ne i32 %and, 1
@@ -470,31 +795,55 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; NOSFB-NEXT:    addi sp, sp, 32
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: sextw_removal_ccor:
-; RV64SFB:       # %bb.0: # %bb
-; RV64SFB-NEXT:    addi sp, sp, -32
-; RV64SFB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    mv s0, a3
-; RV64SFB-NEXT:    andi a0, a0, 1
-; RV64SFB-NEXT:    mv s1, a2
-; RV64SFB-NEXT:    beqz a0, .LBB15_4
-; RV64SFB-NEXT:  # %bb.3: # %bb
-; RV64SFB-NEXT:    or s0, a3, a1
-; RV64SFB-NEXT:  .LBB15_4: # %bb
-; RV64SFB-NEXT:  .LBB15_1: # %bb2
-; RV64SFB-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64SFB-NEXT:    mv a0, s0
-; RV64SFB-NEXT:    call bar
-; RV64SFB-NEXT:    sllw s0, s0, s1
-; RV64SFB-NEXT:    bnez a0, .LBB15_1
-; RV64SFB-NEXT:  # %bb.2: # %bb7
-; RV64SFB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    addi sp, sp, 32
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccor:
+; RV64SFBSIFIVEU74:       # %bb.0: # %bb
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    mv s0, a3
+; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    mv s1, a2
+; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB15_4
+; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
+; RV64SFBSIFIVEU74-NEXT:    or s0, a3, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB15_4: # %bb
+; RV64SFBSIFIVEU74-NEXT:  .LBB15_1: # %bb2
+; RV64SFBSIFIVEU74-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBSIFIVEU74-NEXT:    mv a0, s0
+; RV64SFBSIFIVEU74-NEXT:    call bar
+; RV64SFBSIFIVEU74-NEXT:    sllw s0, s0, s1
+; RV64SFBSIFIVEU74-NEXT:    bnez a0, .LBB15_1
+; RV64SFBSIFIVEU74-NEXT:  # %bb.2: # %bb7
+; RV64SFBSIFIVEU74-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, 32
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: sextw_removal_ccor:
+; RV64SFBANDESAX45:       # %bb.0: # %bb
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    mv s0, a3
+; RV64SFBANDESAX45-NEXT:    mv s1, a2
+; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB15_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    or s0, s0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB15_2: # %bb2
+; RV64SFBANDESAX45-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBANDESAX45-NEXT:    mv a0, s0
+; RV64SFBANDESAX45-NEXT:    call bar
+; RV64SFBANDESAX45-NEXT:    sllw s0, s0, s1
+; RV64SFBANDESAX45-NEXT:    bnez a0, .LBB15_2
+; RV64SFBANDESAX45-NEXT:  # %bb.3: # %bb7
+; RV64SFBANDESAX45-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, 32
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: sextw_removal_ccor:
 ; ZICOND:       # %bb.0: # %bb
@@ -589,31 +938,55 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; NOSFB-NEXT:    addi sp, sp, 32
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: sextw_removal_ccaddw:
-; RV64SFB:       # %bb.0: # %bb
-; RV64SFB-NEXT:    addi sp, sp, -32
-; RV64SFB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    mv s1, a1
-; RV64SFB-NEXT:    andi a0, a0, 1
-; RV64SFB-NEXT:    mv s0, a2
-; RV64SFB-NEXT:    beqz a0, .LBB16_4
-; RV64SFB-NEXT:  # %bb.3: # %bb
-; RV64SFB-NEXT:    addw s1, a1, a3
-; RV64SFB-NEXT:  .LBB16_4: # %bb
-; RV64SFB-NEXT:  .LBB16_1: # %bb2
-; RV64SFB-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64SFB-NEXT:    mv a0, s1
-; RV64SFB-NEXT:    call bar
-; RV64SFB-NEXT:    sllw s1, s1, s0
-; RV64SFB-NEXT:    bnez a0, .LBB16_1
-; RV64SFB-NEXT:  # %bb.2: # %bb7
-; RV64SFB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    addi sp, sp, 32
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccaddw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %bb
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    mv s1, a1
+; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    mv s0, a2
+; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB16_4
+; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
+; RV64SFBSIFIVEU74-NEXT:    addw s1, a1, a3
+; RV64SFBSIFIVEU74-NEXT:  .LBB16_4: # %bb
+; RV64SFBSIFIVEU74-NEXT:  .LBB16_1: # %bb2
+; RV64SFBSIFIVEU74-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBSIFIVEU74-NEXT:    mv a0, s1
+; RV64SFBSIFIVEU74-NEXT:    call bar
+; RV64SFBSIFIVEU74-NEXT:    sllw s1, s1, s0
+; RV64SFBSIFIVEU74-NEXT:    bnez a0, .LBB16_1
+; RV64SFBSIFIVEU74-NEXT:  # %bb.2: # %bb7
+; RV64SFBSIFIVEU74-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, 32
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: sextw_removal_ccaddw:
+; RV64SFBANDESAX45:       # %bb.0: # %bb
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    mv s0, a2
+; RV64SFBANDESAX45-NEXT:    mv s1, a1
+; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB16_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    addw s1, s1, a3
+; RV64SFBANDESAX45-NEXT:  .LBB16_2: # %bb2
+; RV64SFBANDESAX45-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBANDESAX45-NEXT:    mv a0, s1
+; RV64SFBANDESAX45-NEXT:    call bar
+; RV64SFBANDESAX45-NEXT:    sllw s1, s1, s0
+; RV64SFBANDESAX45-NEXT:    bnez a0, .LBB16_2
+; RV64SFBANDESAX45-NEXT:  # %bb.3: # %bb7
+; RV64SFBANDESAX45-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, 32
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: sextw_removal_ccaddw:
 ; ZICOND:       # %bb.0: # %bb
@@ -692,14 +1065,23 @@ define i32 @select_sllw(i32 %A, i32 %B, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sllw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB17_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sllw a2, a0, a1
-; RV64SFB-NEXT:  .LBB17_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sllw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB17_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sllw a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB17_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sllw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB17_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sllw a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB17_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sllw:
 ; ZICOND:       # %bb.0: # %entry
@@ -734,14 +1116,23 @@ define i32 @select_srlw(i32 %A, i32 %B, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srlw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB18_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srlw a2, a0, a1
-; RV64SFB-NEXT:  .LBB18_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srlw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB18_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srlw a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB18_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srlw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB18_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srlw a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB18_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srlw:
 ; ZICOND:       # %bb.0: # %entry
@@ -776,14 +1167,23 @@ define i32 @select_sraw(i32 %A, i32 %B, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sraw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB19_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sraw a2, a0, a1
-; RV64SFB-NEXT:  .LBB19_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sraw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB19_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sraw a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB19_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sraw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB19_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sraw a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB19_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sraw:
 ; ZICOND:       # %bb.0: # %entry
@@ -818,14 +1218,23 @@ define i64 @select_sll(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sll:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB20_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sll a2, a0, a1
-; RV64SFB-NEXT:  .LBB20_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sll:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB20_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sll a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB20_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sll:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB20_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sll a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB20_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sll:
 ; ZICOND:       # %bb.0: # %entry
@@ -879,14 +1288,23 @@ define i64 @select_srl(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srl:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB21_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srl a2, a0, a1
-; RV64SFB-NEXT:  .LBB21_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srl:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB21_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srl a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB21_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srl:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB21_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srl a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB21_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srl:
 ; ZICOND:       # %bb.0: # %entry
@@ -940,14 +1358,23 @@ define i64 @select_sra(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sra:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB22_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sra a2, a0, a1
-; RV64SFB-NEXT:  .LBB22_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sra:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB22_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sra a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB22_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sra:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB22_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sra a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB22_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sra:
 ; ZICOND:       # %bb.0: # %entry
@@ -1001,14 +1428,23 @@ define i32 @select_addiw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_addiw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB23_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    addiw a1, a0, 1234
-; RV64SFB-NEXT:  .LBB23_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_addiw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB23_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    addiw a1, a0, 1234
+; RV64SFBSIFIVEU74-NEXT:  .LBB23_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_addiw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB23_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    addiw a1, a0, 1234
+; RV64SFBANDESAX45-NEXT:  .LBB23_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_addiw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1043,14 +1479,23 @@ define i64 @select_addi(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_addi:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB24_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    addi a1, a0, 1234
-; RV64SFB-NEXT:  .LBB24_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_addi:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB24_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    addi a1, a0, 1234
+; RV64SFBSIFIVEU74-NEXT:  .LBB24_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_addi:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB24_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    addi a1, a0, 1234
+; RV64SFBANDESAX45-NEXT:  .LBB24_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_addi:
 ; ZICOND:       # %bb.0: # %entry
@@ -1092,14 +1537,23 @@ define i64 @select_andi(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_andi:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB25_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    andi a1, a0, 567
-; RV64SFB-NEXT:  .LBB25_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_andi:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB25_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a1, a0, 567
+; RV64SFBSIFIVEU74-NEXT:  .LBB25_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_andi:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB25_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    andi a1, a0, 567
+; RV64SFBANDESAX45-NEXT:  .LBB25_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_andi:
 ; ZICOND:       # %bb.0: # %entry
@@ -1139,14 +1593,23 @@ define i64 @select_ori(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_ori:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB26_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    ori a1, a0, 890
-; RV64SFB-NEXT:  .LBB26_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_ori:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB26_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ori a1, a0, 890
+; RV64SFBSIFIVEU74-NEXT:  .LBB26_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_ori:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB26_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    ori a1, a0, 890
+; RV64SFBANDESAX45-NEXT:  .LBB26_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_ori:
 ; ZICOND:       # %bb.0: # %entry
@@ -1185,14 +1648,23 @@ define i64 @select_xori(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_xori:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB27_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    xori a1, a0, 321
-; RV64SFB-NEXT:  .LBB27_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xori:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB27_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xori a1, a0, 321
+; RV64SFBSIFIVEU74-NEXT:  .LBB27_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xori:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB27_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xori a1, a0, 321
+; RV64SFBANDESAX45-NEXT:  .LBB27_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_xori:
 ; ZICOND:       # %bb.0: # %entry
@@ -1231,14 +1703,23 @@ define i64 @select_slli(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_slli:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB28_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    slli a1, a0, 32
-; RV64SFB-NEXT:  .LBB28_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_slli:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB28_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    slli a1, a0, 32
+; RV64SFBSIFIVEU74-NEXT:  .LBB28_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_slli:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB28_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    slli a1, a0, 32
+; RV64SFBANDESAX45-NEXT:  .LBB28_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_slli:
 ; ZICOND:       # %bb.0: # %entry
@@ -1278,14 +1759,23 @@ define i64 @select_srli(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srli:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB29_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srli a1, a0, 35
-; RV64SFB-NEXT:  .LBB29_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srli:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB29_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srli a1, a0, 35
+; RV64SFBSIFIVEU74-NEXT:  .LBB29_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srli:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB29_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srli a1, a0, 35
+; RV64SFBANDESAX45-NEXT:  .LBB29_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srli:
 ; ZICOND:       # %bb.0: # %entry
@@ -1325,14 +1815,23 @@ define i64 @select_srai(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srai:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB30_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srai a1, a0, 63
-; RV64SFB-NEXT:  .LBB30_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srai:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB30_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srai a1, a0, 63
+; RV64SFBSIFIVEU74-NEXT:  .LBB30_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srai:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB30_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srai a1, a0, 63
+; RV64SFBANDESAX45-NEXT:  .LBB30_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srai:
 ; ZICOND:       # %bb.0: # %entry
@@ -1372,14 +1871,23 @@ define i32 @select_slliw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_slliw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB31_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    slliw a1, a0, 3
-; RV64SFB-NEXT:  .LBB31_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_slliw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB31_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    slliw a1, a0, 3
+; RV64SFBSIFIVEU74-NEXT:  .LBB31_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_slliw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB31_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    slliw a1, a0, 3
+; RV64SFBANDESAX45-NEXT:  .LBB31_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_slliw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1414,14 +1922,23 @@ define i32 @select_srliw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srliw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB32_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srliw a1, a0, 17
-; RV64SFB-NEXT:  .LBB32_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srliw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB32_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srliw a1, a0, 17
+; RV64SFBSIFIVEU74-NEXT:  .LBB32_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srliw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB32_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srliw a1, a0, 17
+; RV64SFBANDESAX45-NEXT:  .LBB32_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srliw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1456,14 +1973,23 @@ define i32 @select_sraiw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sraiw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB33_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sraiw a1, a0, 31
-; RV64SFB-NEXT:  .LBB33_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sraiw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB33_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sraiw a1, a0, 31
+; RV64SFBSIFIVEU74-NEXT:  .LBB33_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sraiw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB33_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sraiw a1, a0, 31
+; RV64SFBANDESAX45-NEXT:  .LBB33_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sraiw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1574,14 +2100,23 @@ define i64 @select_andn(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_andn:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB36_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    andn a2, a0, a1
-; RV64SFB-NEXT:  .LBB36_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_andn:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB36_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andn a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB36_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_andn:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB36_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    andn a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB36_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_andn:
 ; ZICOND:       # %bb.0: # %entry
@@ -1622,14 +2157,23 @@ define i64 @select_orn(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_orn:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB37_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    orn a2, a0, a1
-; RV64SFB-NEXT:  .LBB37_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_orn:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB37_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    orn a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB37_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_orn:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB37_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    orn a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB37_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_orn:
 ; ZICOND:       # %bb.0: # %entry
@@ -1670,14 +2214,23 @@ define i64 @select_xnor(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_xnor:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB38_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    xnor a2, a0, a1
-; RV64SFB-NEXT:  .LBB38_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xnor:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB38_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xnor a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB38_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xnor:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB38_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xnor a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB38_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_xnor:
 ; ZICOND:       # %bb.0: # %entry
@@ -1731,7 +2284,7 @@ define i64 @select_bfoz(i64 %A, i64 %B, i1 zeroext %cond) {
 ;
 ; RV64SFBANDESAX45-LABEL: select_bfoz:
 ; RV64SFBANDESAX45:       # %bb.0: # %entry
-; RV64SFBANDESAX45-NEXT:    bnez a2, .LBB39_2
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB39_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
 ; RV64SFBANDESAX45-NEXT:    nds.bfoz a1, a0, 25, 23
 ; RV64SFBANDESAX45-NEXT:  .LBB39_2: # %entry
@@ -1792,7 +2345,7 @@ define i64 @select_bfos(i64 %A, i64 %B, i1 zeroext %cond) {
 ;
 ; RV64SFBANDESAX45-LABEL: select_bfos:
 ; RV64SFBANDESAX45:       # %bb.0: # %entry
-; RV64SFBANDESAX45-NEXT:    bnez a2, .LBB40_2
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB40_2
 ; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
 ; RV64SFBANDESAX45-NEXT:    nds.bfos a1, a0, 14, 46
 ; RV64SFBANDESAX45-NEXT:  .LBB40_2: # %entry
diff --git a/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
index cd7f30d8f5898..32753ca382fc7 100644
--- a/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
@@ -716,101 +716,92 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, a6, a5
-; RV32I-NEXT:    lbu a5, 8(a0)
-; RV32I-NEXT:    lbu a6, 9(a0)
-; RV32I-NEXT:    lbu t3, 10(a0)
-; RV32I-NEXT:    lbu t4, 11(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    or a5, a6, a5
+; RV32I-NEXT:    or a3, t0, a7
+; RV32I-NEXT:    lbu a6, 8(a0)
+; RV32I-NEXT:    lbu a7, 9(a0)
+; RV32I-NEXT:    lbu t0, 10(a0)
+; RV32I-NEXT:    lbu t3, 11(a0)
 ; RV32I-NEXT:    slli t1, t1, 16
 ; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a7, t0, a7
-; RV32I-NEXT:    or t0, t2, t1
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 12(a0)
-; RV32I-NEXT:    lbu t1, 13(a0)
-; RV32I-NEXT:    lbu t2, 14(a0)
-; RV32I-NEXT:    lbu a0, 15(a0)
-; RV32I-NEXT:    slli t3, t3, 16
-; RV32I-NEXT:    slli t4, t4, 24
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    slli t2, t2, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t3, t4, t3
-; RV32I-NEXT:    or a6, t1, a6
-; RV32I-NEXT:    or a0, a0, t2
-; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    lbu t2, 0(a1)
-; RV32I-NEXT:    lbu t4, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t1, t1, t2
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    slli t3, t3, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a7, t3, t0
+; RV32I-NEXT:    lbu t0, 12(a0)
+; RV32I-NEXT:    lbu t2, 13(a0)
+; RV32I-NEXT:    lbu t3, 14(a0)
+; RV32I-NEXT:    lbu t4, 15(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 16(sp)
 ; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 24(sp)
 ; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t4
-; RV32I-NEXT:    mv t2, sp
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, t0, a7
-; RV32I-NEXT:    or a5, t3, a5
-; RV32I-NEXT:    or a0, a0, a6
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    sw a3, 0(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    srli a0, a1, 3
-; RV32I-NEXT:    andi a3, a1, 31
-; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    xori a3, a3, 31
-; RV32I-NEXT:    add a0, t2, a0
-; RV32I-NEXT:    lw a4, 4(a0)
-; RV32I-NEXT:    lw a5, 8(a0)
-; RV32I-NEXT:    lw a6, 0(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    srl a7, a4, a1
-; RV32I-NEXT:    slli t0, a5, 1
-; RV32I-NEXT:    srl a6, a6, a1
-; RV32I-NEXT:    slli a4, a4, 1
-; RV32I-NEXT:    srl a5, a5, a1
-; RV32I-NEXT:    slli t1, a0, 1
-; RV32I-NEXT:    srl a0, a0, a1
-; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a4, a4, a3
-; RV32I-NEXT:    sll a3, t1, a3
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a1, t2, t0
+; RV32I-NEXT:    mv t0, sp
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    srli t3, a0, 3
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    andi a5, a0, 31
+; RV32I-NEXT:    andi t3, t3, 12
+; RV32I-NEXT:    xori a5, a5, 31
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a1, t2, a1
+; RV32I-NEXT:    add t0, t0, t3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a3, 4(sp)
+; RV32I-NEXT:    sw a6, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    lw a1, 4(t0)
+; RV32I-NEXT:    lw a3, 8(t0)
+; RV32I-NEXT:    lw a4, 0(t0)
+; RV32I-NEXT:    lw a6, 12(t0)
+; RV32I-NEXT:    srl a7, a1, a0
+; RV32I-NEXT:    slli t0, a3, 1
+; RV32I-NEXT:    srl a4, a4, a0
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srl a3, a3, a0
+; RV32I-NEXT:    slli t1, a6, 1
+; RV32I-NEXT:    srl a0, a6, a0
+; RV32I-NEXT:    sll a6, t0, a5
+; RV32I-NEXT:    sll a1, a1, a5
+; RV32I-NEXT:    sll a5, t1, a5
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
-; RV32I-NEXT:    or a1, a7, a1
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a1, a4, a1
+; RV32I-NEXT:    or a3, a3, a5
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
 ; RV32I-NEXT:    sb t1, 15(a2)
 ; RV32I-NEXT:    srli a0, a3, 16
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    srli a6, a3, 8
-; RV32I-NEXT:    srli a7, a4, 16
-; RV32I-NEXT:    srli t0, a4, 24
-; RV32I-NEXT:    srli t1, a4, 8
-; RV32I-NEXT:    srli t2, a1, 16
-; RV32I-NEXT:    srli t3, a1, 24
+; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 8
+; RV32I-NEXT:    srli a7, a1, 16
+; RV32I-NEXT:    srli t0, a1, 24
+; RV32I-NEXT:    srli t1, a1, 8
+; RV32I-NEXT:    srli t2, a6, 16
+; RV32I-NEXT:    srli t3, a6, 24
 ; RV32I-NEXT:    sb a3, 8(a2)
-; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a0, a1, 8
-; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a0, a6, 8
+; RV32I-NEXT:    sb a1, 0(a2)
 ; RV32I-NEXT:    sb t1, 1(a2)
 ; RV32I-NEXT:    sb a7, 2(a2)
 ; RV32I-NEXT:    sb t0, 3(a2)
-; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a6, 4(a2)
 ; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    sb t2, 6(a2)
 ; RV32I-NEXT:    sb t3, 7(a2)
@@ -952,102 +943,93 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, a6, a5
-; RV32I-NEXT:    lbu a5, 8(a0)
-; RV32I-NEXT:    lbu a6, 9(a0)
-; RV32I-NEXT:    lbu t3, 10(a0)
-; RV32I-NEXT:    lbu t4, 11(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    or a5, a6, a5
+; RV32I-NEXT:    or a3, t0, a7
+; RV32I-NEXT:    lbu a6, 8(a0)
+; RV32I-NEXT:    lbu a7, 9(a0)
+; RV32I-NEXT:    lbu t0, 10(a0)
+; RV32I-NEXT:    lbu t3, 11(a0)
 ; RV32I-NEXT:    slli t1, t1, 16
 ; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a7, t0, a7
-; RV32I-NEXT:    or t0, t2, t1
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 12(a0)
-; RV32I-NEXT:    lbu t1, 13(a0)
-; RV32I-NEXT:    lbu t2, 14(a0)
-; RV32I-NEXT:    lbu a0, 15(a0)
-; RV32I-NEXT:    slli t3, t3, 16
-; RV32I-NEXT:    slli t4, t4, 24
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    slli t2, t2, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t3, t4, t3
-; RV32I-NEXT:    or a6, t1, a6
-; RV32I-NEXT:    or a0, a0, t2
-; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    lbu t2, 0(a1)
-; RV32I-NEXT:    lbu t4, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t1, t1, t2
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    slli t3, t3, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a7, t3, t0
+; RV32I-NEXT:    lbu t0, 12(a0)
+; RV32I-NEXT:    lbu t2, 13(a0)
+; RV32I-NEXT:    lbu t3, 14(a0)
+; RV32I-NEXT:    lbu t4, 15(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 0(sp)
 ; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 8(sp)
 ; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t4
-; RV32I-NEXT:    addi t2, sp, 16
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a1, t2, t0
+; RV32I-NEXT:    addi t0, sp, 16
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    srli t3, a0, 3
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    andi a5, a0, 31
+; RV32I-NEXT:    andi t3, t3, 12
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a1, t2, a1
+; RV32I-NEXT:    sub a7, t0, t3
+; RV32I-NEXT:    sw a4, 16(sp)
+; RV32I-NEXT:    sw a3, 20(sp)
+; RV32I-NEXT:    sw a6, 24(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    lw a1, 0(a7)
+; RV32I-NEXT:    lw a3, 4(a7)
+; RV32I-NEXT:    lw a4, 8(a7)
+; RV32I-NEXT:    lw a6, 12(a7)
+; RV32I-NEXT:    xori a5, a5, 31
+; RV32I-NEXT:    sll a7, a3, a0
+; RV32I-NEXT:    srli t0, a1, 1
+; RV32I-NEXT:    sll a6, a6, a0
+; RV32I-NEXT:    srli t1, a4, 1
+; RV32I-NEXT:    sll a4, a4, a0
+; RV32I-NEXT:    srli a3, a3, 1
+; RV32I-NEXT:    sll a0, a1, a0
+; RV32I-NEXT:    srl a1, t0, a5
+; RV32I-NEXT:    srl t0, t1, a5
+; RV32I-NEXT:    srl a3, a3, a5
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t1, a0, 24
+; RV32I-NEXT:    srli t2, a0, 8
+; RV32I-NEXT:    or a1, a7, a1
+; RV32I-NEXT:    or a6, a6, t0
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, t0, a7
-; RV32I-NEXT:    or a5, t3, a5
-; RV32I-NEXT:    or a0, a0, a6
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    sw a3, 16(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    srli a0, a1, 3
-; RV32I-NEXT:    andi a3, a1, 31
-; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    sub a0, t2, a0
-; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a5, 4(a0)
-; RV32I-NEXT:    lw a6, 8(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    xori a3, a3, 31
-; RV32I-NEXT:    sll a7, a5, a1
-; RV32I-NEXT:    srli t0, a4, 1
-; RV32I-NEXT:    sll a0, a0, a1
-; RV32I-NEXT:    srli t1, a6, 1
-; RV32I-NEXT:    sll a6, a6, a1
-; RV32I-NEXT:    srli a5, a5, 1
-; RV32I-NEXT:    sll a1, a4, a1
-; RV32I-NEXT:    srl a4, t0, a3
-; RV32I-NEXT:    srl t0, t1, a3
-; RV32I-NEXT:    srl a3, a5, a3
-; RV32I-NEXT:    srli a5, a1, 16
-; RV32I-NEXT:    srli t1, a1, 24
-; RV32I-NEXT:    srli t2, a1, 8
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    or a0, a0, t0
-; RV32I-NEXT:    or a3, a6, a3
-; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    sb t2, 1(a2)
 ; RV32I-NEXT:    sb a5, 2(a2)
 ; RV32I-NEXT:    sb t1, 3(a2)
-; RV32I-NEXT:    srli a1, a3, 16
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    srli a6, a3, 8
-; RV32I-NEXT:    srli a7, a0, 16
-; RV32I-NEXT:    srli t0, a0, 24
-; RV32I-NEXT:    srli t1, a0, 8
-; RV32I-NEXT:    srli t2, a4, 16
-; RV32I-NEXT:    srli t3, a4, 24
+; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 8
+; RV32I-NEXT:    srli a7, a6, 16
+; RV32I-NEXT:    srli t0, a6, 24
+; RV32I-NEXT:    srli t1, a6, 8
+; RV32I-NEXT:    srli t2, a1, 16
+; RV32I-NEXT:    srli t3, a1, 24
 ; RV32I-NEXT:    sb a3, 8(a2)
-; RV32I-NEXT:    sb a6, 9(a2)
-; RV32I-NEXT:    sb a1, 10(a2)
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a1, a4, 8
-; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
+; RV32I-NEXT:    sb a0, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a0, a1, 8
+; RV32I-NEXT:    sb a6, 12(a2)
 ; RV32I-NEXT:    sb t1, 13(a2)
 ; RV32I-NEXT:    sb a7, 14(a2)
 ; RV32I-NEXT:    sb t0, 15(a2)
-; RV32I-NEXT:    sb a4, 4(a2)
-; RV32I-NEXT:    sb a1, 5(a2)
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    sb t2, 6(a2)
 ; RV32I-NEXT:    sb t3, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
@@ -1186,82 +1168,73 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu t1, 6(a0)
 ; RV32I-NEXT:    lbu t2, 7(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    lbu a4, 8(a0)
-; RV32I-NEXT:    lbu t3, 9(a0)
-; RV32I-NEXT:    lbu t4, 10(a0)
-; RV32I-NEXT:    lbu t5, 11(a0)
 ; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a6, a6, 24
 ; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 8(a0)
+; RV32I-NEXT:    lbu a7, 9(a0)
+; RV32I-NEXT:    lbu t0, 10(a0)
+; RV32I-NEXT:    lbu t3, 11(a0)
 ; RV32I-NEXT:    slli t1, t1, 16
 ; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    slli t3, t3, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a7, t3, t0
 ; RV32I-NEXT:    lbu t0, 12(a0)
-; RV32I-NEXT:    lbu t1, 13(a0)
-; RV32I-NEXT:    lbu t2, 14(a0)
-; RV32I-NEXT:    lbu a0, 15(a0)
-; RV32I-NEXT:    slli t3, t3, 8
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or a4, t3, a4
-; RV32I-NEXT:    or t3, t5, t4
-; RV32I-NEXT:    or t0, t1, t0
-; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    lbu t4, 0(a1)
-; RV32I-NEXT:    lbu t5, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t1, t1, t4
-; RV32I-NEXT:    slli t5, t5, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t5
-; RV32I-NEXT:    or a3, a5, a3
-; RV32I-NEXT:    mv a5, sp
-; RV32I-NEXT:    slli t2, t2, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t2, a0, t2
-; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    lbu t2, 13(a0)
+; RV32I-NEXT:    lbu t3, 14(a0)
+; RV32I-NEXT:    lbu t4, 15(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a1, t2, t0
+; RV32I-NEXT:    mv t0, sp
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    srli a4, a0, 3
+; RV32I-NEXT:    or a5, t1, a5
+; RV32I-NEXT:    andi t1, a0, 31
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    srai t3, t4, 31
+; RV32I-NEXT:    andi a4, a4, 12
+; RV32I-NEXT:    xori t1, t1, 31
 ; RV32I-NEXT:    or a6, a7, a6
-; RV32I-NEXT:    or a4, t3, a4
-; RV32I-NEXT:    or a7, t2, t0
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
+; RV32I-NEXT:    or a1, t2, a1
+; RV32I-NEXT:    sw t3, 16(sp)
+; RV32I-NEXT:    sw t3, 20(sp)
+; RV32I-NEXT:    sw t3, 24(sp)
+; RV32I-NEXT:    sw t3, 28(sp)
+; RV32I-NEXT:    add a4, t0, a4
 ; RV32I-NEXT:    sw a3, 0(sp)
-; RV32I-NEXT:    sw a6, 4(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a7, 12(sp)
-; RV32I-NEXT:    srli a0, a1, 3
-; RV32I-NEXT:    andi a3, a1, 31
-; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    xori a3, a3, 31
-; RV32I-NEXT:    add a0, a5, a0
-; RV32I-NEXT:    lw a4, 4(a0)
-; RV32I-NEXT:    lw a5, 8(a0)
-; RV32I-NEXT:    lw a6, 0(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    srl a7, a4, a1
-; RV32I-NEXT:    slli t0, a5, 1
-; RV32I-NEXT:    srl a6, a6, a1
-; RV32I-NEXT:    slli a4, a4, 1
-; RV32I-NEXT:    srl a5, a5, a1
-; RV32I-NEXT:    slli t1, a0, 1
-; RV32I-NEXT:    sra a0, a0, a1
-; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a4, a4, a3
-; RV32I-NEXT:    sll a3, t1, a3
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    sw a6, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    lw a1, 4(a4)
+; RV32I-NEXT:    lw a3, 8(a4)
+; RV32I-NEXT:    lw a5, 0(a4)
+; RV32I-NEXT:    lw a4, 12(a4)
+; RV32I-NEXT:    srl a6, a1, a0
+; RV32I-NEXT:    slli a7, a3, 1
+; RV32I-NEXT:    srl a5, a5, a0
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srl a3, a3, a0
+; RV32I-NEXT:    slli t0, a4, 1
+; RV32I-NEXT:    sra a0, a4, a0
+; RV32I-NEXT:    sll a4, a7, t1
+; RV32I-NEXT:    sll a1, a1, t1
+; RV32I-NEXT:    sll a7, t0, t1
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
-; RV32I-NEXT:    or a1, a7, a1
 ; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a3, a3, a7
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
@@ -1269,21 +1242,21 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srli a0, a3, 16
 ; RV32I-NEXT:    srli a5, a3, 24
 ; RV32I-NEXT:    srli a6, a3, 8
-; RV32I-NEXT:    srli a7, a4, 16
-; RV32I-NEXT:    srli t0, a4, 24
-; RV32I-NEXT:    srli t1, a4, 8
-; RV32I-NEXT:    srli t2, a1, 16
-; RV32I-NEXT:    srli t3, a1, 24
+; RV32I-NEXT:    srli a7, a1, 16
+; RV32I-NEXT:    srli t0, a1, 24
+; RV32I-NEXT:    srli t1, a1, 8
+; RV32I-NEXT:    srli t2, a4, 16
+; RV32I-NEXT:    srli t3, a4, 24
 ; RV32I-NEXT:    sb a3, 8(a2)
 ; RV32I-NEXT:    sb a6, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
 ; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a0, a1, 8
-; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    srli a0, a4, 8
+; RV32I-NEXT:    sb a1, 0(a2)
 ; RV32I-NEXT:    sb t1, 1(a2)
 ; RV32I-NEXT:    sb a7, 2(a2)
 ; RV32I-NEXT:    sb t0, 3(a2)
-; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a4, 4(a2)
 ; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    sb t2, 6(a2)
 ; RV32I-NEXT:    sb t3, 7(a2)
@@ -1299,19 +1272,17 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-LABEL: lshr_32bytes:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd s0, 152(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 144(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 136(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 128(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s4, 120(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s5, 112(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s6, 104(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s7, 96(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s8, 88(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s9, 80(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s10, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s11, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -144
+; RV64I-NEXT:    sd s0, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    lbu a4, 1(a0)
 ; RV64I-NEXT:    lbu a5, 2(a0)
@@ -1328,143 +1299,122 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    lbu s1, 13(a0)
 ; RV64I-NEXT:    lbu s2, 14(a0)
 ; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a5, a5, 16
+; RV64I-NEXT:    slli a6, a6, 24
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    lbu s4, 16(a0)
 ; RV64I-NEXT:    lbu s5, 17(a0)
 ; RV64I-NEXT:    lbu s6, 18(a0)
 ; RV64I-NEXT:    lbu s7, 19(a0)
-; RV64I-NEXT:    slli a4, a4, 8
-; RV64I-NEXT:    slli s8, a5, 16
-; RV64I-NEXT:    slli a6, a6, 24
 ; RV64I-NEXT:    slli t0, t0, 8
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or a5, a4, a3
-; RV64I-NEXT:    or a6, a6, s8
-; RV64I-NEXT:    or a3, t0, a7
-; RV64I-NEXT:    or a4, t2, t1
-; RV64I-NEXT:    lbu s8, 20(a0)
-; RV64I-NEXT:    lbu s9, 21(a0)
-; RV64I-NEXT:    lbu s10, 22(a0)
-; RV64I-NEXT:    lbu s11, 23(a0)
 ; RV64I-NEXT:    slli t4, t4, 8
 ; RV64I-NEXT:    slli t5, t5, 16
 ; RV64I-NEXT:    slli t6, t6, 24
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    lbu t5, 20(a0)
+; RV64I-NEXT:    lbu t6, 21(a0)
+; RV64I-NEXT:    lbu s8, 22(a0)
+; RV64I-NEXT:    lbu s9, 23(a0)
 ; RV64I-NEXT:    slli s1, s1, 8
 ; RV64I-NEXT:    slli s2, s2, 16
 ; RV64I-NEXT:    slli s3, s3, 24
-; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or t0, t6, t5
-; RV64I-NEXT:    or t1, s1, s0
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    lbu t6, 24(a0)
-; RV64I-NEXT:    lbu s0, 25(a0)
-; RV64I-NEXT:    lbu s1, 26(a0)
-; RV64I-NEXT:    lbu s2, 27(a0)
 ; RV64I-NEXT:    slli s5, s5, 8
 ; RV64I-NEXT:    slli s6, s6, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    or t1, s1, s0
+; RV64I-NEXT:    or t2, s3, s2
 ; RV64I-NEXT:    or t3, s5, s4
 ; RV64I-NEXT:    or t4, s7, s6
-; RV64I-NEXT:    or t5, s9, s8
-; RV64I-NEXT:    lbu s3, 28(a0)
+; RV64I-NEXT:    lbu s0, 24(a0)
+; RV64I-NEXT:    lbu s1, 25(a0)
+; RV64I-NEXT:    lbu s2, 26(a0)
+; RV64I-NEXT:    lbu s3, 27(a0)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s8, s8, 16
+; RV64I-NEXT:    slli s9, s9, 24
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or t6, s9, s8
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 28(a0)
 ; RV64I-NEXT:    lbu s4, 29(a0)
 ; RV64I-NEXT:    lbu s5, 30(a0)
 ; RV64I-NEXT:    lbu s6, 31(a0)
-; RV64I-NEXT:    slli s10, s10, 16
-; RV64I-NEXT:    slli s11, s11, 24
-; RV64I-NEXT:    slli s0, s0, 8
-; RV64I-NEXT:    slli s1, s1, 16
-; RV64I-NEXT:    slli s2, s2, 24
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or a0, s11, s10
-; RV64I-NEXT:    or t6, s0, t6
-; RV64I-NEXT:    or s0, s2, s1
-; RV64I-NEXT:    or s1, s4, s3
-; RV64I-NEXT:    lbu s2, 0(a1)
-; RV64I-NEXT:    lbu s3, 1(a1)
-; RV64I-NEXT:    lbu s4, 2(a1)
-; RV64I-NEXT:    lbu s7, 3(a1)
-; RV64I-NEXT:    slli s5, s5, 16
-; RV64I-NEXT:    slli s6, s6, 24
-; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    slli s4, s4, 16
-; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    or s3, s7, s4
-; RV64I-NEXT:    lbu s4, 5(a1)
-; RV64I-NEXT:    lbu s6, 4(a1)
-; RV64I-NEXT:    lbu s7, 6(a1)
-; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or s4, s4, s6
-; RV64I-NEXT:    slli s7, s7, 16
-; RV64I-NEXT:    slli a1, a1, 24
-; RV64I-NEXT:    or a1, a1, s7
+; RV64I-NEXT:    lbu a0, 0(a1)
 ; RV64I-NEXT:    sd zero, 32(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    or a5, a6, a5
-; RV64I-NEXT:    mv a6, sp
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    slli s3, s3, 24
+; RV64I-NEXT:    or a1, s3, s2
+; RV64I-NEXT:    mv s2, sp
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    slli s5, s5, 16
+; RV64I-NEXT:    slli s6, s6, 24
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    srli s3, a0, 3
+; RV64I-NEXT:    or s4, s6, s5
+; RV64I-NEXT:    andi s5, a0, 63
+; RV64I-NEXT:    andi s3, s3, 24
+; RV64I-NEXT:    xori s5, s5, 63
 ; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    or a4, t0, a7
-; RV64I-NEXT:    or a7, t2, t1
-; RV64I-NEXT:    or t0, t4, t3
-; RV64I-NEXT:    or a0, a0, t5
-; RV64I-NEXT:    or t1, s0, t6
-; RV64I-NEXT:    or t2, s5, s1
-; RV64I-NEXT:    or t3, s3, s2
-; RV64I-NEXT:    or a1, a1, s4
-; RV64I-NEXT:    slli a3, a3, 32
-; RV64I-NEXT:    slli a7, a7, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    slli t2, t2, 32
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a3, a3, a5
-; RV64I-NEXT:    or a4, a7, a4
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, t2, t1
-; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    or t1, s4, s1
+; RV64I-NEXT:    add s2, s2, s3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a6, 32
+; RV64I-NEXT:    slli t0, t0, 32
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a1, t1, a1
 ; RV64I-NEXT:    sd a3, 0(sp)
 ; RV64I-NEXT:    sd a4, 8(sp)
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    sd a5, 24(sp)
-; RV64I-NEXT:    srli a0, a1, 3
-; RV64I-NEXT:    andi a3, a1, 63
-; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    xori a3, a3, 63
-; RV64I-NEXT:    add a0, a6, a0
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 0(a0)
-; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a1, 24(sp)
+; RV64I-NEXT:    ld a1, 8(s2)
+; RV64I-NEXT:    ld a3, 16(s2)
+; RV64I-NEXT:    ld a4, 0(s2)
+; RV64I-NEXT:    ld a5, 24(s2)
+; RV64I-NEXT:    srl a6, a1, a0
+; RV64I-NEXT:    slli a7, a3, 1
+; RV64I-NEXT:    srl a4, a4, a0
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    srl a3, a3, a0
 ; RV64I-NEXT:    slli t0, a5, 1
-; RV64I-NEXT:    srl a6, a6, a1
-; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a0, 1
-; RV64I-NEXT:    srl t2, a0, a1
-; RV64I-NEXT:    sll a0, t0, a3
-; RV64I-NEXT:    sll a1, a4, a3
-; RV64I-NEXT:    sll a3, t1, a3
-; RV64I-NEXT:    srli a4, t2, 56
-; RV64I-NEXT:    srli t0, t2, 48
-; RV64I-NEXT:    srli t1, t2, 40
-; RV64I-NEXT:    srli t3, t2, 32
-; RV64I-NEXT:    srli t4, t2, 24
-; RV64I-NEXT:    srli t5, t2, 16
-; RV64I-NEXT:    srli t6, t2, 8
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    or a1, a6, a1
-; RV64I-NEXT:    or a3, a5, a3
+; RV64I-NEXT:    srl a5, a5, a0
+; RV64I-NEXT:    sll a0, a7, s5
+; RV64I-NEXT:    sll a1, a1, s5
+; RV64I-NEXT:    sll a7, t0, s5
+; RV64I-NEXT:    srli t0, a5, 56
+; RV64I-NEXT:    srli t1, a5, 48
+; RV64I-NEXT:    srli t2, a5, 40
+; RV64I-NEXT:    srli t3, a5, 32
+; RV64I-NEXT:    srli t4, a5, 24
+; RV64I-NEXT:    srli t5, a5, 16
+; RV64I-NEXT:    srli t6, a5, 8
+; RV64I-NEXT:    or a0, a6, a0
+; RV64I-NEXT:    or a1, a4, a1
+; RV64I-NEXT:    or a3, a3, a7
 ; RV64I-NEXT:    sb t3, 28(a2)
-; RV64I-NEXT:    sb t1, 29(a2)
-; RV64I-NEXT:    sb t0, 30(a2)
-; RV64I-NEXT:    sb a4, 31(a2)
-; RV64I-NEXT:    sb t2, 24(a2)
+; RV64I-NEXT:    sb t2, 29(a2)
+; RV64I-NEXT:    sb t1, 30(a2)
+; RV64I-NEXT:    sb t0, 31(a2)
+; RV64I-NEXT:    sb a5, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
 ; RV64I-NEXT:    sb t4, 27(a2)
@@ -1513,19 +1463,17 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sb a1, 9(a2)
 ; RV64I-NEXT:    sb a5, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 136(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 128(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s4, 120(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s5, 112(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s6, 104(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s7, 96(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s8, 88(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s9, 80(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s10, 72(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s11, 64(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    ld s0, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 144
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_32bytes:
@@ -1550,67 +1498,55 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a7, 3(a0)
 ; RV32I-NEXT:    lbu a5, 4(a0)
 ; RV32I-NEXT:    lbu t0, 5(a0)
-; RV32I-NEXT:    lbu t1, 6(a0)
-; RV32I-NEXT:    lbu t2, 7(a0)
-; RV32I-NEXT:    lbu t3, 8(a0)
-; RV32I-NEXT:    lbu t4, 9(a0)
-; RV32I-NEXT:    lbu t5, 10(a0)
-; RV32I-NEXT:    lbu t6, 11(a0)
-; RV32I-NEXT:    lbu s0, 12(a0)
-; RV32I-NEXT:    lbu s2, 13(a0)
-; RV32I-NEXT:    lbu s4, 14(a0)
-; RV32I-NEXT:    lbu s5, 15(a0)
-; RV32I-NEXT:    lbu s6, 16(a0)
-; RV32I-NEXT:    lbu s7, 17(a0)
-; RV32I-NEXT:    lbu s8, 18(a0)
-; RV32I-NEXT:    lbu s9, 19(a0)
+; RV32I-NEXT:    lbu t3, 6(a0)
+; RV32I-NEXT:    lbu t6, 7(a0)
+; RV32I-NEXT:    lbu s2, 8(a0)
+; RV32I-NEXT:    lbu s3, 9(a0)
+; RV32I-NEXT:    lbu s4, 10(a0)
+; RV32I-NEXT:    lbu s5, 11(a0)
+; RV32I-NEXT:    lbu s7, 12(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu s9, 14(a0)
+; RV32I-NEXT:    lbu s10, 15(a0)
+; RV32I-NEXT:    lbu s11, 16(a0)
+; RV32I-NEXT:    lbu ra, 17(a0)
+; RV32I-NEXT:    lbu t4, 18(a0)
+; RV32I-NEXT:    lbu s0, 19(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a7, a7, 24
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a4, a7, a6
-; RV32I-NEXT:    lbu s10, 20(a0)
-; RV32I-NEXT:    lbu s11, 21(a0)
-; RV32I-NEXT:    lbu ra, 22(a0)
-; RV32I-NEXT:    lbu a3, 23(a0)
+; RV32I-NEXT:    lbu t1, 20(a0)
+; RV32I-NEXT:    lbu t2, 21(a0)
+; RV32I-NEXT:    lbu t5, 22(a0)
+; RV32I-NEXT:    lbu s1, 23(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    slli t1, t1, 16
-; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli t3, t3, 16
 ; RV32I-NEXT:    slli t6, t6, 24
-; RV32I-NEXT:    or a5, t0, a5
-; RV32I-NEXT:    or a6, t2, t1
-; RV32I-NEXT:    or a7, t4, t3
-; RV32I-NEXT:    or t0, t6, t5
-; RV32I-NEXT:    lbu s1, 24(a0)
-; RV32I-NEXT:    lbu s3, 25(a0)
-; RV32I-NEXT:    lbu t4, 26(a0)
-; RV32I-NEXT:    lbu t5, 27(a0)
-; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s3, s3, 8
 ; RV32I-NEXT:    slli s4, s4, 16
 ; RV32I-NEXT:    slli s5, s5, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, s5, s4
-; RV32I-NEXT:    or t3, s7, s6
-; RV32I-NEXT:    lbu t6, 28(a0)
-; RV32I-NEXT:    lbu s4, 29(a0)
-; RV32I-NEXT:    lbu s5, 30(a0)
-; RV32I-NEXT:    lbu s6, 31(a0)
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli s9, s9, 24
-; RV32I-NEXT:    slli s11, s11, 8
-; RV32I-NEXT:    slli ra, ra, 16
-; RV32I-NEXT:    slli a3, a3, 24
-; RV32I-NEXT:    or a0, s9, s8
-; RV32I-NEXT:    or s0, s11, s10
-; RV32I-NEXT:    or s2, a3, ra
-; RV32I-NEXT:    lbu a3, 0(a1)
-; RV32I-NEXT:    lbu s7, 1(a1)
-; RV32I-NEXT:    lbu s8, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    or a5, t0, a5
+; RV32I-NEXT:    or a6, t6, t3
+; RV32I-NEXT:    or a7, s3, s2
+; RV32I-NEXT:    or t0, s5, s4
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu s5, 25(a0)
+; RV32I-NEXT:    lbu s6, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    slli ra, ra, 8
+; RV32I-NEXT:    or s7, s8, s7
+; RV32I-NEXT:    or s2, s10, s9
+; RV32I-NEXT:    or s3, ra, s11
+; RV32I-NEXT:    lbu s4, 28(a0)
+; RV32I-NEXT:    lbu s8, 29(a0)
+; RV32I-NEXT:    lbu s9, 30(a0)
+; RV32I-NEXT:    lbu s10, 31(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 56(sp)
 ; RV32I-NEXT:    sw zero, 60(sp)
 ; RV32I-NEXT:    sw zero, 64(sp)
@@ -1619,89 +1555,90 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw zero, 44(sp)
 ; RV32I-NEXT:    sw zero, 48(sp)
 ; RV32I-NEXT:    sw zero, 52(sp)
-; RV32I-NEXT:    slli s3, s3, 8
-; RV32I-NEXT:    or s1, s3, s1
-; RV32I-NEXT:    addi s3, sp, 8
 ; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli s4, s4, 8
-; RV32I-NEXT:    slli s5, s5, 16
-; RV32I-NEXT:    slli s6, s6, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    or t5, s4, t6
-; RV32I-NEXT:    or t6, s6, s5
-; RV32I-NEXT:    or a3, s7, a3
-; RV32I-NEXT:    or a1, a1, s8
-; RV32I-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    or a4, a4, s4
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
-; RV32I-NEXT:    or t0, a0, t3
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, t4, s1
-; RV32I-NEXT:    or t3, t6, t5
-; RV32I-NEXT:    or a0, a1, a3
-; RV32I-NEXT:    sw t0, 24(sp)
-; RV32I-NEXT:    sw t1, 28(sp)
-; RV32I-NEXT:    sw t2, 32(sp)
-; RV32I-NEXT:    sw t3, 36(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a5, 12(sp)
-; RV32I-NEXT:    sw a6, 16(sp)
-; RV32I-NEXT:    sw a7, 20(sp)
+; RV32I-NEXT:    slli s0, s0, 24
+; RV32I-NEXT:    or t4, s0, t4
+; RV32I-NEXT:    addi s0, sp, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 24
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    slli s6, s6, 16
+; RV32I-NEXT:    slli t6, t6, 24
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    or t1, t2, t1
 ; RV32I-NEXT:    srli a1, a0, 3
-; RV32I-NEXT:    andi a3, a0, 31
-; RV32I-NEXT:    andi a4, a1, 28
-; RV32I-NEXT:    xori a1, a3, 31
-; RV32I-NEXT:    add a4, s3, a4
-; RV32I-NEXT:    lw a3, 0(a4)
-; RV32I-NEXT:    lw a5, 4(a4)
-; RV32I-NEXT:    lw a6, 8(a4)
-; RV32I-NEXT:    lw a7, 12(a4)
-; RV32I-NEXT:    lw t0, 16(a4)
-; RV32I-NEXT:    lw t1, 20(a4)
-; RV32I-NEXT:    lw t2, 24(a4)
-; RV32I-NEXT:    lw a4, 28(a4)
-; RV32I-NEXT:    srl t3, a5, a0
-; RV32I-NEXT:    slli t4, a6, 1
+; RV32I-NEXT:    or t2, s1, t5
+; RV32I-NEXT:    andi t5, a0, 31
+; RV32I-NEXT:    or t3, s5, t3
+; RV32I-NEXT:    or t6, t6, s6
+; RV32I-NEXT:    or s1, s8, s4
+; RV32I-NEXT:    or s4, s10, s9
+; RV32I-NEXT:    andi s5, a1, 28
+; RV32I-NEXT:    xori a1, t5, 31
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a6, s2, s7
+; RV32I-NEXT:    or a7, t4, s3
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    or t1, t6, t3
+; RV32I-NEXT:    or t2, s4, s1
+; RV32I-NEXT:    add s0, s0, s5
+; RV32I-NEXT:    sw a7, 24(sp)
+; RV32I-NEXT:    sw t0, 28(sp)
+; RV32I-NEXT:    sw t1, 32(sp)
+; RV32I-NEXT:    sw t2, 36(sp)
+; RV32I-NEXT:    sw a3, 8(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a6, 20(sp)
+; RV32I-NEXT:    lw a3, 0(s0)
+; RV32I-NEXT:    lw a4, 4(s0)
+; RV32I-NEXT:    lw a5, 8(s0)
+; RV32I-NEXT:    lw a6, 12(s0)
+; RV32I-NEXT:    lw a7, 16(s0)
+; RV32I-NEXT:    lw t0, 20(s0)
+; RV32I-NEXT:    lw t1, 24(s0)
+; RV32I-NEXT:    lw t2, 28(s0)
+; RV32I-NEXT:    srl t3, a4, a0
+; RV32I-NEXT:    slli t4, a5, 1
 ; RV32I-NEXT:    srl a3, a3, a0
-; RV32I-NEXT:    slli a5, a5, 1
-; RV32I-NEXT:    srl t5, a7, a0
-; RV32I-NEXT:    slli t6, t0, 1
-; RV32I-NEXT:    srl a6, a6, a0
-; RV32I-NEXT:    slli a7, a7, 1
-; RV32I-NEXT:    srl s0, t1, a0
-; RV32I-NEXT:    slli s1, t2, 1
-; RV32I-NEXT:    srl t0, t0, a0
-; RV32I-NEXT:    slli t1, t1, 1
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl t5, a6, a0
+; RV32I-NEXT:    slli t6, a7, 1
+; RV32I-NEXT:    srl a5, a5, a0
+; RV32I-NEXT:    slli a6, a6, 1
+; RV32I-NEXT:    srl s0, t0, a0
+; RV32I-NEXT:    slli s1, t1, 1
+; RV32I-NEXT:    srl a7, a7, a0
+; RV32I-NEXT:    slli t0, t0, 1
+; RV32I-NEXT:    srl t1, t1, a0
+; RV32I-NEXT:    slli s2, t2, 1
 ; RV32I-NEXT:    srl t2, t2, a0
-; RV32I-NEXT:    slli s2, a4, 1
-; RV32I-NEXT:    srl s3, a4, a0
 ; RV32I-NEXT:    sll a0, t4, a1
-; RV32I-NEXT:    sll a4, a5, a1
-; RV32I-NEXT:    sll a5, t6, a1
-; RV32I-NEXT:    sll a7, a7, a1
-; RV32I-NEXT:    sll t4, s1, a1
-; RV32I-NEXT:    sll t1, t1, a1
-; RV32I-NEXT:    sll t6, s2, a1
-; RV32I-NEXT:    srli s1, s3, 24
-; RV32I-NEXT:    srli s2, s3, 16
-; RV32I-NEXT:    srli s4, s3, 8
+; RV32I-NEXT:    sll a4, a4, a1
+; RV32I-NEXT:    sll t4, t6, a1
+; RV32I-NEXT:    sll a6, a6, a1
+; RV32I-NEXT:    sll t6, s1, a1
+; RV32I-NEXT:    sll t0, t0, a1
+; RV32I-NEXT:    sll s1, s2, a1
+; RV32I-NEXT:    srli s2, t2, 24
+; RV32I-NEXT:    srli s3, t2, 16
+; RV32I-NEXT:    srli s4, t2, 8
 ; RV32I-NEXT:    or a0, t3, a0
 ; RV32I-NEXT:    or a1, a3, a4
-; RV32I-NEXT:    or a3, t5, a5
-; RV32I-NEXT:    or a4, a6, a7
-; RV32I-NEXT:    or a5, s0, t4
-; RV32I-NEXT:    or a6, t0, t1
-; RV32I-NEXT:    or a7, t2, t6
-; RV32I-NEXT:    sb s3, 28(a2)
+; RV32I-NEXT:    or a3, t5, t4
+; RV32I-NEXT:    or a4, a5, a6
+; RV32I-NEXT:    or a5, s0, t6
+; RV32I-NEXT:    or a6, a7, t0
+; RV32I-NEXT:    or a7, t1, s1
+; RV32I-NEXT:    sb t2, 28(a2)
 ; RV32I-NEXT:    sb s4, 29(a2)
-; RV32I-NEXT:    sb s2, 30(a2)
-; RV32I-NEXT:    sb s1, 31(a2)
+; RV32I-NEXT:    sb s3, 30(a2)
+; RV32I-NEXT:    sb s2, 31(a2)
 ; RV32I-NEXT:    srli t0, a7, 24
 ; RV32I-NEXT:    srli t1, a7, 16
 ; RV32I-NEXT:    srli t2, a7, 8
@@ -1775,19 +1712,17 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-LABEL: shl_32bytes:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd s0, 152(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 144(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 136(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 128(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s4, 120(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s5, 112(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s6, 104(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s7, 96(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s8, 88(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s9, 80(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s10, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s11, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -144
+; RV64I-NEXT:    sd s0, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    lbu a4, 1(a0)
 ; RV64I-NEXT:    lbu a5, 2(a0)
@@ -1804,146 +1739,125 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    lbu s1, 13(a0)
 ; RV64I-NEXT:    lbu s2, 14(a0)
 ; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a5, a5, 16
+; RV64I-NEXT:    slli a6, a6, 24
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    lbu s4, 16(a0)
 ; RV64I-NEXT:    lbu s5, 17(a0)
 ; RV64I-NEXT:    lbu s6, 18(a0)
 ; RV64I-NEXT:    lbu s7, 19(a0)
-; RV64I-NEXT:    slli a4, a4, 8
-; RV64I-NEXT:    slli s8, a5, 16
-; RV64I-NEXT:    slli a6, a6, 24
 ; RV64I-NEXT:    slli t0, t0, 8
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or a5, a4, a3
-; RV64I-NEXT:    or a6, a6, s8
-; RV64I-NEXT:    or a3, t0, a7
-; RV64I-NEXT:    or a4, t2, t1
-; RV64I-NEXT:    lbu s8, 20(a0)
-; RV64I-NEXT:    lbu s9, 21(a0)
-; RV64I-NEXT:    lbu s10, 22(a0)
-; RV64I-NEXT:    lbu s11, 23(a0)
 ; RV64I-NEXT:    slli t4, t4, 8
 ; RV64I-NEXT:    slli t5, t5, 16
 ; RV64I-NEXT:    slli t6, t6, 24
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    lbu t5, 20(a0)
+; RV64I-NEXT:    lbu t6, 21(a0)
+; RV64I-NEXT:    lbu s8, 22(a0)
+; RV64I-NEXT:    lbu s9, 23(a0)
 ; RV64I-NEXT:    slli s1, s1, 8
 ; RV64I-NEXT:    slli s2, s2, 16
 ; RV64I-NEXT:    slli s3, s3, 24
-; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or t0, t6, t5
-; RV64I-NEXT:    or t1, s1, s0
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    lbu t6, 24(a0)
-; RV64I-NEXT:    lbu s0, 25(a0)
-; RV64I-NEXT:    lbu s1, 26(a0)
-; RV64I-NEXT:    lbu s2, 27(a0)
 ; RV64I-NEXT:    slli s5, s5, 8
 ; RV64I-NEXT:    slli s6, s6, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    or t1, s1, s0
+; RV64I-NEXT:    or t2, s3, s2
 ; RV64I-NEXT:    or t3, s5, s4
 ; RV64I-NEXT:    or t4, s7, s6
-; RV64I-NEXT:    or t5, s9, s8
-; RV64I-NEXT:    lbu s3, 28(a0)
+; RV64I-NEXT:    lbu s0, 24(a0)
+; RV64I-NEXT:    lbu s1, 25(a0)
+; RV64I-NEXT:    lbu s2, 26(a0)
+; RV64I-NEXT:    lbu s3, 27(a0)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s8, s8, 16
+; RV64I-NEXT:    slli s9, s9, 24
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or t6, s9, s8
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 28(a0)
 ; RV64I-NEXT:    lbu s4, 29(a0)
 ; RV64I-NEXT:    lbu s5, 30(a0)
 ; RV64I-NEXT:    lbu s6, 31(a0)
-; RV64I-NEXT:    slli s10, s10, 16
-; RV64I-NEXT:    slli s11, s11, 24
-; RV64I-NEXT:    slli s0, s0, 8
-; RV64I-NEXT:    slli s1, s1, 16
-; RV64I-NEXT:    slli s2, s2, 24
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or a0, s11, s10
-; RV64I-NEXT:    or t6, s0, t6
-; RV64I-NEXT:    or s0, s2, s1
-; RV64I-NEXT:    or s1, s4, s3
-; RV64I-NEXT:    lbu s2, 0(a1)
-; RV64I-NEXT:    lbu s3, 1(a1)
-; RV64I-NEXT:    lbu s4, 2(a1)
-; RV64I-NEXT:    lbu s7, 3(a1)
-; RV64I-NEXT:    slli s5, s5, 16
-; RV64I-NEXT:    slli s6, s6, 24
-; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    slli s4, s4, 16
-; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    or s3, s7, s4
-; RV64I-NEXT:    lbu s4, 5(a1)
-; RV64I-NEXT:    lbu s6, 4(a1)
-; RV64I-NEXT:    lbu s7, 6(a1)
-; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or s4, s4, s6
-; RV64I-NEXT:    slli s7, s7, 16
-; RV64I-NEXT:    slli a1, a1, 24
-; RV64I-NEXT:    or a1, a1, s7
+; RV64I-NEXT:    lbu a0, 0(a1)
 ; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    or a5, a6, a5
-; RV64I-NEXT:    addi a6, sp, 32
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    slli s3, s3, 24
+; RV64I-NEXT:    or a1, s3, s2
+; RV64I-NEXT:    addi s2, sp, 32
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    slli s5, s5, 16
+; RV64I-NEXT:    slli s6, s6, 24
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    srli s3, a0, 3
+; RV64I-NEXT:    or s4, s6, s5
+; RV64I-NEXT:    andi s5, a0, 63
+; RV64I-NEXT:    andi s3, s3, 24
 ; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    or a4, t0, a7
-; RV64I-NEXT:    or a7, t2, t1
-; RV64I-NEXT:    or t0, t4, t3
-; RV64I-NEXT:    or a0, a0, t5
-; RV64I-NEXT:    or t1, s0, t6
-; RV64I-NEXT:    or t2, s5, s1
-; RV64I-NEXT:    or t3, s3, s2
-; RV64I-NEXT:    or a1, a1, s4
-; RV64I-NEXT:    slli a3, a3, 32
-; RV64I-NEXT:    slli a7, a7, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    slli t2, t2, 32
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a3, a3, a5
-; RV64I-NEXT:    or a4, a7, a4
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, t2, t1
-; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    or t1, s4, s1
+; RV64I-NEXT:    sub t2, s2, s3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a6, 32
+; RV64I-NEXT:    slli t0, t0, 32
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a1, t1, a1
 ; RV64I-NEXT:    sd a3, 32(sp)
 ; RV64I-NEXT:    sd a4, 40(sp)
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a5, 56(sp)
-; RV64I-NEXT:    srli a0, a1, 3
-; RV64I-NEXT:    andi a3, a1, 63
-; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    sub a0, a6, a0
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a5, 8(a0)
-; RV64I-NEXT:    ld a6, 16(a0)
-; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    xori a3, a3, 63
-; RV64I-NEXT:    sll a7, a5, a1
-; RV64I-NEXT:    srli t0, a4, 1
-; RV64I-NEXT:    sll t1, a0, a1
-; RV64I-NEXT:    srli a0, a6, 1
-; RV64I-NEXT:    sll a6, a6, a1
-; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    sll a4, a4, a1
-; RV64I-NEXT:    srl a1, t0, a3
-; RV64I-NEXT:    srl t0, a0, a3
-; RV64I-NEXT:    srl a3, a5, a3
-; RV64I-NEXT:    srli a5, a4, 56
-; RV64I-NEXT:    srli t2, a4, 48
-; RV64I-NEXT:    srli t3, a4, 40
-; RV64I-NEXT:    srli t4, a4, 32
-; RV64I-NEXT:    srli t5, a4, 24
-; RV64I-NEXT:    srli t6, a4, 16
-; RV64I-NEXT:    srli s0, a4, 8
-; RV64I-NEXT:    or a0, a7, a1
-; RV64I-NEXT:    or a1, t1, t0
-; RV64I-NEXT:    or a3, a6, a3
-; RV64I-NEXT:    sb t4, 4(a2)
-; RV64I-NEXT:    sb t3, 5(a2)
-; RV64I-NEXT:    sb t2, 6(a2)
-; RV64I-NEXT:    sb a5, 7(a2)
-; RV64I-NEXT:    sb a4, 0(a2)
-; RV64I-NEXT:    sb s0, 1(a2)
-; RV64I-NEXT:    sb t6, 2(a2)
-; RV64I-NEXT:    sb t5, 3(a2)
+; RV64I-NEXT:    sd a5, 48(sp)
+; RV64I-NEXT:    sd a1, 56(sp)
+; RV64I-NEXT:    ld a1, 0(t2)
+; RV64I-NEXT:    ld a3, 8(t2)
+; RV64I-NEXT:    ld a4, 16(t2)
+; RV64I-NEXT:    ld a5, 24(t2)
+; RV64I-NEXT:    xori a6, s5, 63
+; RV64I-NEXT:    sll a7, a3, a0
+; RV64I-NEXT:    srli t0, a1, 1
+; RV64I-NEXT:    sll a5, a5, a0
+; RV64I-NEXT:    srli t1, a4, 1
+; RV64I-NEXT:    sll a4, a4, a0
+; RV64I-NEXT:    srli a3, a3, 1
+; RV64I-NEXT:    sll t2, a1, a0
+; RV64I-NEXT:    srl a0, t0, a6
+; RV64I-NEXT:    srl a1, t1, a6
+; RV64I-NEXT:    srl a3, a3, a6
+; RV64I-NEXT:    srli a6, t2, 56
+; RV64I-NEXT:    srli t0, t2, 48
+; RV64I-NEXT:    srli t1, t2, 40
+; RV64I-NEXT:    srli t3, t2, 32
+; RV64I-NEXT:    srli t4, t2, 24
+; RV64I-NEXT:    srli t5, t2, 16
+; RV64I-NEXT:    srli t6, t2, 8
+; RV64I-NEXT:    or a0, a7, a0
+; RV64I-NEXT:    or a1, a5, a1
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    sb t3, 4(a2)
+; RV64I-NEXT:    sb t1, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb a6, 7(a2)
+; RV64I-NEXT:    sb t2, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb t5, 2(a2)
+; RV64I-NEXT:    sb t4, 3(a2)
 ; RV64I-NEXT:    srli a4, a3, 56
 ; RV64I-NEXT:    srli a5, a3, 48
 ; RV64I-NEXT:    srli a6, a3, 40
@@ -1989,19 +1903,17 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sb a1, 9(a2)
 ; RV64I-NEXT:    sb a5, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 136(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 128(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s4, 120(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s5, 112(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s6, 104(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s7, 96(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s8, 88(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s9, 80(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s10, 72(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s11, 64(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    ld s0, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 144
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_32bytes:
@@ -2026,67 +1938,55 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a7, 3(a0)
 ; RV32I-NEXT:    lbu a5, 4(a0)
 ; RV32I-NEXT:    lbu t0, 5(a0)
-; RV32I-NEXT:    lbu t1, 6(a0)
-; RV32I-NEXT:    lbu t2, 7(a0)
-; RV32I-NEXT:    lbu t3, 8(a0)
-; RV32I-NEXT:    lbu t4, 9(a0)
-; RV32I-NEXT:    lbu t5, 10(a0)
-; RV32I-NEXT:    lbu t6, 11(a0)
-; RV32I-NEXT:    lbu s0, 12(a0)
-; RV32I-NEXT:    lbu s2, 13(a0)
-; RV32I-NEXT:    lbu s4, 14(a0)
-; RV32I-NEXT:    lbu s5, 15(a0)
-; RV32I-NEXT:    lbu s6, 16(a0)
-; RV32I-NEXT:    lbu s7, 17(a0)
-; RV32I-NEXT:    lbu s8, 18(a0)
-; RV32I-NEXT:    lbu s9, 19(a0)
+; RV32I-NEXT:    lbu t3, 6(a0)
+; RV32I-NEXT:    lbu t6, 7(a0)
+; RV32I-NEXT:    lbu s2, 8(a0)
+; RV32I-NEXT:    lbu s3, 9(a0)
+; RV32I-NEXT:    lbu s4, 10(a0)
+; RV32I-NEXT:    lbu s5, 11(a0)
+; RV32I-NEXT:    lbu s7, 12(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu s9, 14(a0)
+; RV32I-NEXT:    lbu s10, 15(a0)
+; RV32I-NEXT:    lbu s11, 16(a0)
+; RV32I-NEXT:    lbu ra, 17(a0)
+; RV32I-NEXT:    lbu t4, 18(a0)
+; RV32I-NEXT:    lbu s0, 19(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a7, a7, 24
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a4, a7, a6
-; RV32I-NEXT:    lbu s10, 20(a0)
-; RV32I-NEXT:    lbu s11, 21(a0)
-; RV32I-NEXT:    lbu ra, 22(a0)
-; RV32I-NEXT:    lbu a3, 23(a0)
+; RV32I-NEXT:    lbu t1, 20(a0)
+; RV32I-NEXT:    lbu t2, 21(a0)
+; RV32I-NEXT:    lbu t5, 22(a0)
+; RV32I-NEXT:    lbu s1, 23(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    slli t1, t1, 16
-; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli t3, t3, 16
 ; RV32I-NEXT:    slli t6, t6, 24
-; RV32I-NEXT:    or a5, t0, a5
-; RV32I-NEXT:    or a6, t2, t1
-; RV32I-NEXT:    or a7, t4, t3
-; RV32I-NEXT:    or t0, t6, t5
-; RV32I-NEXT:    lbu s1, 24(a0)
-; RV32I-NEXT:    lbu s3, 25(a0)
-; RV32I-NEXT:    lbu t4, 26(a0)
-; RV32I-NEXT:    lbu t5, 27(a0)
-; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s3, s3, 8
 ; RV32I-NEXT:    slli s4, s4, 16
 ; RV32I-NEXT:    slli s5, s5, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, s5, s4
-; RV32I-NEXT:    or t3, s7, s6
-; RV32I-NEXT:    lbu t6, 28(a0)
-; RV32I-NEXT:    lbu s4, 29(a0)
-; RV32I-NEXT:    lbu s5, 30(a0)
-; RV32I-NEXT:    lbu s6, 31(a0)
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli s9, s9, 24
-; RV32I-NEXT:    slli s11, s11, 8
-; RV32I-NEXT:    slli ra, ra, 16
-; RV32I-NEXT:    slli a3, a3, 24
-; RV32I-NEXT:    or a0, s9, s8
-; RV32I-NEXT:    or s0, s11, s10
-; RV32I-NEXT:    or s2, a3, ra
-; RV32I-NEXT:    lbu a3, 0(a1)
-; RV32I-NEXT:    lbu s7, 1(a1)
-; RV32I-NEXT:    lbu s8, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    or a5, t0, a5
+; RV32I-NEXT:    or a6, t6, t3
+; RV32I-NEXT:    or a7, s3, s2
+; RV32I-NEXT:    or t0, s5, s4
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu s5, 25(a0)
+; RV32I-NEXT:    lbu s6, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    slli ra, ra, 8
+; RV32I-NEXT:    or s7, s8, s7
+; RV32I-NEXT:    or s2, s10, s9
+; RV32I-NEXT:    or s3, ra, s11
+; RV32I-NEXT:    lbu s4, 28(a0)
+; RV32I-NEXT:    lbu s8, 29(a0)
+; RV32I-NEXT:    lbu s9, 30(a0)
+; RV32I-NEXT:    lbu s10, 31(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 24(sp)
 ; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw zero, 32(sp)
@@ -2095,88 +1995,89 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
 ; RV32I-NEXT:    sw zero, 20(sp)
-; RV32I-NEXT:    slli s3, s3, 8
-; RV32I-NEXT:    or s1, s3, s1
-; RV32I-NEXT:    addi s3, sp, 40
 ; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli s4, s4, 8
-; RV32I-NEXT:    slli s5, s5, 16
-; RV32I-NEXT:    slli s6, s6, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    or t5, s4, t6
-; RV32I-NEXT:    or t6, s6, s5
-; RV32I-NEXT:    or a3, s7, a3
-; RV32I-NEXT:    or a1, a1, s8
-; RV32I-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    or a4, a4, s4
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
-; RV32I-NEXT:    or t0, a0, t3
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, t4, s1
-; RV32I-NEXT:    or t3, t6, t5
-; RV32I-NEXT:    or a0, a1, a3
-; RV32I-NEXT:    sw t0, 56(sp)
-; RV32I-NEXT:    sw t1, 60(sp)
-; RV32I-NEXT:    sw t2, 64(sp)
-; RV32I-NEXT:    sw t3, 68(sp)
-; RV32I-NEXT:    sw a4, 40(sp)
-; RV32I-NEXT:    sw a5, 44(sp)
-; RV32I-NEXT:    sw a6, 48(sp)
-; RV32I-NEXT:    sw a7, 52(sp)
+; RV32I-NEXT:    slli s0, s0, 24
+; RV32I-NEXT:    or t4, s0, t4
+; RV32I-NEXT:    addi s0, sp, 40
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 24
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    slli s6, s6, 16
+; RV32I-NEXT:    slli t6, t6, 24
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    or t1, t2, t1
 ; RV32I-NEXT:    srli a1, a0, 3
-; RV32I-NEXT:    andi a3, a0, 31
-; RV32I-NEXT:    andi a4, a1, 28
-; RV32I-NEXT:    xori a1, a3, 31
-; RV32I-NEXT:    sub a3, s3, a4
-; RV32I-NEXT:    lw a4, 0(a3)
-; RV32I-NEXT:    lw a5, 4(a3)
-; RV32I-NEXT:    lw a6, 8(a3)
-; RV32I-NEXT:    lw a7, 12(a3)
-; RV32I-NEXT:    lw t0, 16(a3)
-; RV32I-NEXT:    lw t1, 20(a3)
-; RV32I-NEXT:    lw t2, 24(a3)
-; RV32I-NEXT:    lw a3, 28(a3)
-; RV32I-NEXT:    sll t3, a5, a0
-; RV32I-NEXT:    srli t4, a4, 1
-; RV32I-NEXT:    sll t5, a7, a0
-; RV32I-NEXT:    srli t6, a6, 1
-; RV32I-NEXT:    sll a6, a6, a0
-; RV32I-NEXT:    srli a5, a5, 1
-; RV32I-NEXT:    sll s0, t1, a0
-; RV32I-NEXT:    srli s1, t0, 1
-; RV32I-NEXT:    sll t0, t0, a0
-; RV32I-NEXT:    srli a7, a7, 1
-; RV32I-NEXT:    sll s2, a3, a0
-; RV32I-NEXT:    srli a3, t2, 1
+; RV32I-NEXT:    or t2, s1, t5
+; RV32I-NEXT:    andi t5, a0, 31
+; RV32I-NEXT:    or t3, s5, t3
+; RV32I-NEXT:    or t6, t6, s6
+; RV32I-NEXT:    or s1, s8, s4
+; RV32I-NEXT:    or s4, s10, s9
+; RV32I-NEXT:    andi s5, a1, 28
+; RV32I-NEXT:    xori a1, t5, 31
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a6, s2, s7
+; RV32I-NEXT:    or a7, t4, s3
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    or t1, t6, t3
+; RV32I-NEXT:    or t2, s4, s1
+; RV32I-NEXT:    sub t3, s0, s5
+; RV32I-NEXT:    sw a7, 56(sp)
+; RV32I-NEXT:    sw t0, 60(sp)
+; RV32I-NEXT:    sw t1, 64(sp)
+; RV32I-NEXT:    sw t2, 68(sp)
+; RV32I-NEXT:    sw a3, 40(sp)
+; RV32I-NEXT:    sw a4, 44(sp)
+; RV32I-NEXT:    sw a5, 48(sp)
+; RV32I-NEXT:    sw a6, 52(sp)
+; RV32I-NEXT:    lw a3, 0(t3)
+; RV32I-NEXT:    lw a4, 4(t3)
+; RV32I-NEXT:    lw a5, 8(t3)
+; RV32I-NEXT:    lw a6, 12(t3)
+; RV32I-NEXT:    lw a7, 16(t3)
+; RV32I-NEXT:    lw t0, 20(t3)
+; RV32I-NEXT:    lw t1, 24(t3)
+; RV32I-NEXT:    lw t2, 28(t3)
+; RV32I-NEXT:    sll t3, a4, a0
+; RV32I-NEXT:    srli t4, a3, 1
+; RV32I-NEXT:    sll t5, a6, a0
+; RV32I-NEXT:    srli t6, a5, 1
+; RV32I-NEXT:    sll a5, a5, a0
+; RV32I-NEXT:    srli a4, a4, 1
+; RV32I-NEXT:    sll s0, t0, a0
+; RV32I-NEXT:    srli s1, a7, 1
+; RV32I-NEXT:    sll a7, a7, a0
+; RV32I-NEXT:    srli a6, a6, 1
 ; RV32I-NEXT:    sll t2, t2, a0
-; RV32I-NEXT:    srli t1, t1, 1
-; RV32I-NEXT:    sll s3, a4, a0
+; RV32I-NEXT:    srli s2, t1, 1
+; RV32I-NEXT:    sll t1, t1, a0
+; RV32I-NEXT:    srli t0, t0, 1
+; RV32I-NEXT:    sll s3, a3, a0
 ; RV32I-NEXT:    srl a0, t4, a1
-; RV32I-NEXT:    srl a4, t6, a1
-; RV32I-NEXT:    srl a5, a5, a1
+; RV32I-NEXT:    srl a3, t6, a1
+; RV32I-NEXT:    srl a4, a4, a1
 ; RV32I-NEXT:    srl t4, s1, a1
-; RV32I-NEXT:    srl a7, a7, a1
-; RV32I-NEXT:    srl t6, a3, a1
-; RV32I-NEXT:    srl t1, t1, a1
+; RV32I-NEXT:    srl a6, a6, a1
+; RV32I-NEXT:    srl t6, s2, a1
+; RV32I-NEXT:    srl t0, t0, a1
 ; RV32I-NEXT:    srli s1, s3, 24
-; RV32I-NEXT:    srli s4, s3, 16
-; RV32I-NEXT:    srli s5, s3, 8
+; RV32I-NEXT:    srli s2, s3, 16
+; RV32I-NEXT:    srli s4, s3, 8
 ; RV32I-NEXT:    or a0, t3, a0
-; RV32I-NEXT:    or a1, t5, a4
-; RV32I-NEXT:    or a3, a6, a5
+; RV32I-NEXT:    or a1, t5, a3
+; RV32I-NEXT:    or a3, a5, a4
 ; RV32I-NEXT:    or a4, s0, t4
-; RV32I-NEXT:    or a5, t0, a7
-; RV32I-NEXT:    or a6, s2, t6
-; RV32I-NEXT:    or a7, t2, t1
+; RV32I-NEXT:    or a5, a7, a6
+; RV32I-NEXT:    or a6, t2, t6
+; RV32I-NEXT:    or a7, t1, t0
 ; RV32I-NEXT:    sb s3, 0(a2)
-; RV32I-NEXT:    sb s5, 1(a2)
-; RV32I-NEXT:    sb s4, 2(a2)
+; RV32I-NEXT:    sb s4, 1(a2)
+; RV32I-NEXT:    sb s2, 2(a2)
 ; RV32I-NEXT:    sb s1, 3(a2)
 ; RV32I-NEXT:    srli t0, a7, 24
 ; RV32I-NEXT:    srli t1, a7, 16
@@ -2251,19 +2152,17 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-LABEL: ashr_32bytes:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd s0, 152(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 144(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 136(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 128(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s4, 120(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s5, 112(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s6, 104(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s7, 96(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s8, 88(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s9, 80(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s10, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s11, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -144
+; RV64I-NEXT:    sd s0, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    lbu a4, 1(a0)
 ; RV64I-NEXT:    lbu a5, 2(a0)
@@ -2280,144 +2179,123 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    lbu s1, 13(a0)
 ; RV64I-NEXT:    lbu s2, 14(a0)
 ; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a5, a5, 16
+; RV64I-NEXT:    slli a6, a6, 24
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    lbu s4, 16(a0)
 ; RV64I-NEXT:    lbu s5, 17(a0)
 ; RV64I-NEXT:    lbu s6, 18(a0)
 ; RV64I-NEXT:    lbu s7, 19(a0)
-; RV64I-NEXT:    slli a4, a4, 8
-; RV64I-NEXT:    slli a5, a5, 16
-; RV64I-NEXT:    slli a6, a6, 24
 ; RV64I-NEXT:    slli t0, t0, 8
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    or a4, a6, a5
-; RV64I-NEXT:    or a5, t0, a7
-; RV64I-NEXT:    or a6, t2, t1
-; RV64I-NEXT:    lbu s8, 20(a0)
-; RV64I-NEXT:    lbu s9, 21(a0)
-; RV64I-NEXT:    lbu s10, 22(a0)
-; RV64I-NEXT:    lbu s11, 23(a0)
 ; RV64I-NEXT:    slli t4, t4, 8
 ; RV64I-NEXT:    slli t5, t5, 16
 ; RV64I-NEXT:    slli t6, t6, 24
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    lbu t5, 20(a0)
+; RV64I-NEXT:    lbu t6, 21(a0)
+; RV64I-NEXT:    lbu s8, 22(a0)
+; RV64I-NEXT:    lbu s9, 23(a0)
 ; RV64I-NEXT:    slli s1, s1, 8
 ; RV64I-NEXT:    slli s2, s2, 16
 ; RV64I-NEXT:    slli s3, s3, 24
-; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or t0, t6, t5
-; RV64I-NEXT:    or t1, s1, s0
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    lbu t6, 24(a0)
-; RV64I-NEXT:    lbu s0, 25(a0)
-; RV64I-NEXT:    lbu s1, 26(a0)
-; RV64I-NEXT:    lbu s2, 27(a0)
 ; RV64I-NEXT:    slli s5, s5, 8
 ; RV64I-NEXT:    slli s6, s6, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    or t1, s1, s0
+; RV64I-NEXT:    or t2, s3, s2
 ; RV64I-NEXT:    or t3, s5, s4
 ; RV64I-NEXT:    or t4, s7, s6
-; RV64I-NEXT:    or t5, s9, s8
-; RV64I-NEXT:    lbu s3, 28(a0)
+; RV64I-NEXT:    lbu s0, 24(a0)
+; RV64I-NEXT:    lbu s1, 25(a0)
+; RV64I-NEXT:    lbu s2, 26(a0)
+; RV64I-NEXT:    lbu s3, 27(a0)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s8, s8, 16
+; RV64I-NEXT:    slli s9, s9, 24
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or t6, s9, s8
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 28(a0)
 ; RV64I-NEXT:    lbu s4, 29(a0)
 ; RV64I-NEXT:    lbu s5, 30(a0)
 ; RV64I-NEXT:    lbu s6, 31(a0)
-; RV64I-NEXT:    slli s10, s10, 16
-; RV64I-NEXT:    slli s11, s11, 24
-; RV64I-NEXT:    slli s0, s0, 8
-; RV64I-NEXT:    slli s1, s1, 16
-; RV64I-NEXT:    slli s2, s2, 24
+; RV64I-NEXT:    lbu a0, 0(a1)
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    slli s3, s3, 24
+; RV64I-NEXT:    or a1, s3, s2
+; RV64I-NEXT:    mv s2, sp
 ; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or a0, s11, s10
-; RV64I-NEXT:    or t6, s0, t6
-; RV64I-NEXT:    or s0, s2, s1
-; RV64I-NEXT:    or s1, s4, s3
-; RV64I-NEXT:    lbu s2, 0(a1)
-; RV64I-NEXT:    lbu s3, 1(a1)
-; RV64I-NEXT:    lbu s4, 2(a1)
-; RV64I-NEXT:    lbu s7, 3(a1)
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
-; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    slli s4, s4, 16
-; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    or s3, s7, s4
-; RV64I-NEXT:    lbu s4, 5(a1)
-; RV64I-NEXT:    lbu s6, 4(a1)
-; RV64I-NEXT:    lbu s7, 6(a1)
-; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or s4, s4, s6
-; RV64I-NEXT:    slli s7, s7, 16
-; RV64I-NEXT:    slli a1, a1, 24
-; RV64I-NEXT:    or a1, a1, s7
-; RV64I-NEXT:    mv s6, sp
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    srli s3, a0, 3
+; RV64I-NEXT:    or s4, s6, s5
+; RV64I-NEXT:    andi s5, a0, 63
+; RV64I-NEXT:    andi s3, s3, 24
+; RV64I-NEXT:    xori s5, s5, 63
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    or a5, t0, a7
 ; RV64I-NEXT:    or a6, t2, t1
 ; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or a0, a0, t5
-; RV64I-NEXT:    or t0, s0, t6
-; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    or a1, a1, s4
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    or t1, s4, s1
+; RV64I-NEXT:    add s2, s2, s3
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    slli t3, t1, 32
-; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    slli t0, t0, 32
+; RV64I-NEXT:    slli t2, t1, 32
 ; RV64I-NEXT:    sraiw t1, t1, 31
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, a6, a5
-; RV64I-NEXT:    or a0, a0, a7
-; RV64I-NEXT:    or a5, t3, t0
-; RV64I-NEXT:    or a1, a1, t2
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a1, t2, a1
 ; RV64I-NEXT:    sd t1, 32(sp)
 ; RV64I-NEXT:    sd t1, 40(sp)
 ; RV64I-NEXT:    sd t1, 48(sp)
 ; RV64I-NEXT:    sd t1, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
 ; RV64I-NEXT:    sd a4, 8(sp)
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    sd a5, 24(sp)
-; RV64I-NEXT:    srli a0, a1, 3
-; RV64I-NEXT:    andi a3, a1, 63
-; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    xori a3, a3, 63
-; RV64I-NEXT:    add a0, s6, a0
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 0(a0)
-; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a1, 24(sp)
+; RV64I-NEXT:    ld a1, 8(s2)
+; RV64I-NEXT:    ld a3, 16(s2)
+; RV64I-NEXT:    ld a4, 0(s2)
+; RV64I-NEXT:    ld a5, 24(s2)
+; RV64I-NEXT:    srl a6, a1, a0
+; RV64I-NEXT:    slli a7, a3, 1
+; RV64I-NEXT:    srl a4, a4, a0
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    srl a3, a3, a0
 ; RV64I-NEXT:    slli t0, a5, 1
-; RV64I-NEXT:    srl a6, a6, a1
-; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a0, 1
-; RV64I-NEXT:    sra t2, a0, a1
-; RV64I-NEXT:    sll a0, t0, a3
-; RV64I-NEXT:    sll a1, a4, a3
-; RV64I-NEXT:    sll a3, t1, a3
-; RV64I-NEXT:    srli a4, t2, 56
-; RV64I-NEXT:    srli t0, t2, 48
-; RV64I-NEXT:    srli t1, t2, 40
-; RV64I-NEXT:    srli t3, t2, 32
-; RV64I-NEXT:    srli t4, t2, 24
-; RV64I-NEXT:    srli t5, t2, 16
-; RV64I-NEXT:    srli t6, t2, 8
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    or a1, a6, a1
-; RV64I-NEXT:    or a3, a5, a3
+; RV64I-NEXT:    sra a5, a5, a0
+; RV64I-NEXT:    sll a0, a7, s5
+; RV64I-NEXT:    sll a1, a1, s5
+; RV64I-NEXT:    sll a7, t0, s5
+; RV64I-NEXT:    srli t0, a5, 56
+; RV64I-NEXT:    srli t1, a5, 48
+; RV64I-NEXT:    srli t2, a5, 40
+; RV64I-NEXT:    srli t3, a5, 32
+; RV64I-NEXT:    srli t4, a5, 24
+; RV64I-NEXT:    srli t5, a5, 16
+; RV64I-NEXT:    srli t6, a5, 8
+; RV64I-NEXT:    or a0, a6, a0
+; RV64I-NEXT:    or a1, a4, a1
+; RV64I-NEXT:    or a3, a3, a7
 ; RV64I-NEXT:    sb t3, 28(a2)
-; RV64I-NEXT:    sb t1, 29(a2)
-; RV64I-NEXT:    sb t0, 30(a2)
-; RV64I-NEXT:    sb a4, 31(a2)
-; RV64I-NEXT:    sb t2, 24(a2)
+; RV64I-NEXT:    sb t2, 29(a2)
+; RV64I-NEXT:    sb t1, 30(a2)
+; RV64I-NEXT:    sb t0, 31(a2)
+; RV64I-NEXT:    sb a5, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
 ; RV64I-NEXT:    sb t4, 27(a2)
@@ -2438,47 +2316,45 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli s3, a0, 56
 ; RV64I-NEXT:    srli s4, a0, 48
 ; RV64I-NEXT:    srli s5, a0, 40
-; RV64I-NEXT:    srli s6, a0, 32
 ; RV64I-NEXT:    sb a7, 20(a2)
 ; RV64I-NEXT:    sb a6, 21(a2)
 ; RV64I-NEXT:    sb a5, 22(a2)
 ; RV64I-NEXT:    sb a4, 23(a2)
-; RV64I-NEXT:    srli a4, a0, 24
+; RV64I-NEXT:    srli a4, a0, 32
 ; RV64I-NEXT:    sb a3, 16(a2)
 ; RV64I-NEXT:    sb t2, 17(a2)
 ; RV64I-NEXT:    sb t1, 18(a2)
 ; RV64I-NEXT:    sb t0, 19(a2)
-; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
 ; RV64I-NEXT:    sb t6, 4(a2)
 ; RV64I-NEXT:    sb t5, 5(a2)
 ; RV64I-NEXT:    sb t4, 6(a2)
 ; RV64I-NEXT:    sb t3, 7(a2)
-; RV64I-NEXT:    srli a5, a0, 8
+; RV64I-NEXT:    srli a5, a0, 16
 ; RV64I-NEXT:    sb a1, 0(a2)
 ; RV64I-NEXT:    sb s2, 1(a2)
 ; RV64I-NEXT:    sb s1, 2(a2)
 ; RV64I-NEXT:    sb s0, 3(a2)
-; RV64I-NEXT:    sb s6, 12(a2)
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    sb a4, 12(a2)
 ; RV64I-NEXT:    sb s5, 13(a2)
 ; RV64I-NEXT:    sb s4, 14(a2)
 ; RV64I-NEXT:    sb s3, 15(a2)
 ; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    sb a5, 9(a2)
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    sb a4, 11(a2)
-; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 136(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 128(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s4, 120(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s5, 112(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s6, 104(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s7, 96(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s8, 88(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s9, 80(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s10, 72(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s11, 64(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb a5, 10(a2)
+; RV64I-NEXT:    sb a3, 11(a2)
+; RV64I-NEXT:    ld s0, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 144
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_32bytes:
@@ -2503,159 +2379,148 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a7, 3(a0)
 ; RV32I-NEXT:    lbu a5, 4(a0)
 ; RV32I-NEXT:    lbu t0, 5(a0)
-; RV32I-NEXT:    lbu t1, 6(a0)
-; RV32I-NEXT:    lbu t2, 7(a0)
-; RV32I-NEXT:    lbu t3, 8(a0)
-; RV32I-NEXT:    lbu t4, 9(a0)
-; RV32I-NEXT:    lbu t5, 10(a0)
-; RV32I-NEXT:    lbu t6, 11(a0)
-; RV32I-NEXT:    lbu s0, 12(a0)
-; RV32I-NEXT:    lbu s1, 13(a0)
-; RV32I-NEXT:    lbu s2, 14(a0)
-; RV32I-NEXT:    lbu s3, 15(a0)
-; RV32I-NEXT:    lbu s4, 16(a0)
-; RV32I-NEXT:    lbu s5, 17(a0)
-; RV32I-NEXT:    lbu s6, 18(a0)
-; RV32I-NEXT:    lbu s7, 19(a0)
+; RV32I-NEXT:    lbu t3, 6(a0)
+; RV32I-NEXT:    lbu t4, 7(a0)
+; RV32I-NEXT:    lbu t6, 8(a0)
+; RV32I-NEXT:    lbu s0, 9(a0)
+; RV32I-NEXT:    lbu s4, 10(a0)
+; RV32I-NEXT:    lbu s5, 11(a0)
+; RV32I-NEXT:    lbu s6, 12(a0)
+; RV32I-NEXT:    lbu s7, 13(a0)
+; RV32I-NEXT:    lbu s8, 14(a0)
+; RV32I-NEXT:    lbu s9, 15(a0)
+; RV32I-NEXT:    lbu s10, 16(a0)
+; RV32I-NEXT:    lbu s11, 17(a0)
+; RV32I-NEXT:    lbu s2, 18(a0)
+; RV32I-NEXT:    lbu s3, 19(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a7, a7, 24
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a4, a7, a6
-; RV32I-NEXT:    lbu s8, 20(a0)
-; RV32I-NEXT:    lbu s9, 21(a0)
-; RV32I-NEXT:    lbu s10, 22(a0)
-; RV32I-NEXT:    lbu s11, 23(a0)
+; RV32I-NEXT:    lbu t1, 20(a0)
+; RV32I-NEXT:    lbu t2, 21(a0)
+; RV32I-NEXT:    lbu t5, 22(a0)
+; RV32I-NEXT:    lbu s1, 23(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    slli t1, t1, 16
-; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    slli t5, t5, 16
-; RV32I-NEXT:    slli t6, t6, 24
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    slli s0, s0, 8
+; RV32I-NEXT:    slli s4, s4, 16
+; RV32I-NEXT:    slli s5, s5, 24
 ; RV32I-NEXT:    or a5, t0, a5
-; RV32I-NEXT:    or a6, t2, t1
-; RV32I-NEXT:    or a7, t4, t3
-; RV32I-NEXT:    or t0, t6, t5
-; RV32I-NEXT:    lbu ra, 24(a0)
-; RV32I-NEXT:    lbu a3, 25(a0)
-; RV32I-NEXT:    lbu t4, 26(a0)
-; RV32I-NEXT:    lbu t5, 27(a0)
-; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    or a6, t4, t3
+; RV32I-NEXT:    or a7, s0, t6
+; RV32I-NEXT:    or t0, s5, s4
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu s4, 25(a0)
+; RV32I-NEXT:    lbu s5, 26(a0)
+; RV32I-NEXT:    lbu ra, 27(a0)
+; RV32I-NEXT:    slli s7, s7, 8
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    slli s9, s9, 24
+; RV32I-NEXT:    slli s11, s11, 8
+; RV32I-NEXT:    or t4, s7, s6
+; RV32I-NEXT:    or t6, s9, s8
+; RV32I-NEXT:    or s0, s11, s10
+; RV32I-NEXT:    lbu s6, 28(a0)
+; RV32I-NEXT:    lbu s7, 29(a0)
+; RV32I-NEXT:    lbu s8, 30(a0)
+; RV32I-NEXT:    lbu s9, 31(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    slli s2, s2, 16
 ; RV32I-NEXT:    slli s3, s3, 24
-; RV32I-NEXT:    slli s5, s5, 8
-; RV32I-NEXT:    or t1, s1, s0
-; RV32I-NEXT:    or t2, s3, s2
-; RV32I-NEXT:    or t3, s5, s4
-; RV32I-NEXT:    lbu t6, 28(a0)
-; RV32I-NEXT:    lbu s0, 29(a0)
-; RV32I-NEXT:    lbu s1, 30(a0)
-; RV32I-NEXT:    lbu a0, 31(a0)
-; RV32I-NEXT:    slli s6, s6, 16
-; RV32I-NEXT:    slli s7, s7, 24
-; RV32I-NEXT:    slli s9, s9, 8
-; RV32I-NEXT:    slli s10, s10, 16
-; RV32I-NEXT:    slli s11, s11, 24
-; RV32I-NEXT:    or s2, s7, s6
-; RV32I-NEXT:    or s3, s9, s8
-; RV32I-NEXT:    or s4, s11, s10
-; RV32I-NEXT:    lbu s5, 0(a1)
-; RV32I-NEXT:    lbu s6, 1(a1)
-; RV32I-NEXT:    lbu s7, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, ra
-; RV32I-NEXT:    addi s8, sp, 8
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli s0, s0, 8
-; RV32I-NEXT:    slli s1, s1, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    slli s6, s6, 8
-; RV32I-NEXT:    slli s7, s7, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    or t5, s0, t6
-; RV32I-NEXT:    or s1, a0, s1
-; RV32I-NEXT:    or t6, s6, s5
-; RV32I-NEXT:    or a1, a1, s7
-; RV32I-NEXT:    srai s0, a0, 31
-; RV32I-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    or a4, a4, a0
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
-; RV32I-NEXT:    or t0, s2, t3
-; RV32I-NEXT:    or t1, s4, s3
-; RV32I-NEXT:    or a3, t4, a3
+; RV32I-NEXT:    or s2, s3, s2
+; RV32I-NEXT:    addi s3, sp, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 24
+; RV32I-NEXT:    slli s4, s4, 8
+; RV32I-NEXT:    slli s5, s5, 16
+; RV32I-NEXT:    slli ra, ra, 24
+; RV32I-NEXT:    slli s7, s7, 8
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    slli s9, s9, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    srli a1, a0, 3
 ; RV32I-NEXT:    or t2, s1, t5
-; RV32I-NEXT:    or a0, a1, t6
-; RV32I-NEXT:    sw s0, 56(sp)
-; RV32I-NEXT:    sw s0, 60(sp)
-; RV32I-NEXT:    sw s0, 64(sp)
-; RV32I-NEXT:    sw s0, 68(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    sw s0, 44(sp)
-; RV32I-NEXT:    sw s0, 48(sp)
-; RV32I-NEXT:    sw s0, 52(sp)
-; RV32I-NEXT:    sw t0, 24(sp)
-; RV32I-NEXT:    sw t1, 28(sp)
-; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    andi t5, a0, 31
+; RV32I-NEXT:    or t3, s4, t3
+; RV32I-NEXT:    or s1, ra, s5
+; RV32I-NEXT:    or s4, s7, s6
+; RV32I-NEXT:    or s5, s9, s8
+; RV32I-NEXT:    srai s6, s9, 31
+; RV32I-NEXT:    andi s7, a1, 28
+; RV32I-NEXT:    xori a1, t5, 31
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a6, t6, t4
+; RV32I-NEXT:    or a7, s2, s0
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    or t1, s1, t3
+; RV32I-NEXT:    or t2, s5, s4
+; RV32I-NEXT:    sw s6, 56(sp)
+; RV32I-NEXT:    sw s6, 60(sp)
+; RV32I-NEXT:    sw s6, 64(sp)
+; RV32I-NEXT:    sw s6, 68(sp)
+; RV32I-NEXT:    sw s6, 40(sp)
+; RV32I-NEXT:    sw s6, 44(sp)
+; RV32I-NEXT:    sw s6, 48(sp)
+; RV32I-NEXT:    sw s6, 52(sp)
+; RV32I-NEXT:    add s3, s3, s7
+; RV32I-NEXT:    sw a7, 24(sp)
+; RV32I-NEXT:    sw t0, 28(sp)
+; RV32I-NEXT:    sw t1, 32(sp)
 ; RV32I-NEXT:    sw t2, 36(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a5, 12(sp)
-; RV32I-NEXT:    sw a6, 16(sp)
-; RV32I-NEXT:    sw a7, 20(sp)
-; RV32I-NEXT:    srli a1, a0, 3
-; RV32I-NEXT:    andi a3, a0, 31
-; RV32I-NEXT:    andi a4, a1, 28
-; RV32I-NEXT:    xori a1, a3, 31
-; RV32I-NEXT:    add a4, s8, a4
-; RV32I-NEXT:    lw a3, 0(a4)
-; RV32I-NEXT:    lw a5, 4(a4)
-; RV32I-NEXT:    lw a6, 8(a4)
-; RV32I-NEXT:    lw a7, 12(a4)
-; RV32I-NEXT:    lw t0, 16(a4)
-; RV32I-NEXT:    lw t1, 20(a4)
-; RV32I-NEXT:    lw t2, 24(a4)
-; RV32I-NEXT:    lw a4, 28(a4)
-; RV32I-NEXT:    srl t3, a5, a0
-; RV32I-NEXT:    slli t4, a6, 1
+; RV32I-NEXT:    sw a3, 8(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a6, 20(sp)
+; RV32I-NEXT:    lw a3, 0(s3)
+; RV32I-NEXT:    lw a4, 4(s3)
+; RV32I-NEXT:    lw a5, 8(s3)
+; RV32I-NEXT:    lw a6, 12(s3)
+; RV32I-NEXT:    lw a7, 16(s3)
+; RV32I-NEXT:    lw t0, 20(s3)
+; RV32I-NEXT:    lw t1, 24(s3)
+; RV32I-NEXT:    lw t2, 28(s3)
+; RV32I-NEXT:    srl t3, a4, a0
+; RV32I-NEXT:    slli t4, a5, 1
 ; RV32I-NEXT:    srl a3, a3, a0
-; RV32I-NEXT:    slli a5, a5, 1
-; RV32I-NEXT:    srl t5, a7, a0
-; RV32I-NEXT:    slli t6, t0, 1
-; RV32I-NEXT:    srl a6, a6, a0
-; RV32I-NEXT:    slli a7, a7, 1
-; RV32I-NEXT:    srl s0, t1, a0
-; RV32I-NEXT:    slli s1, t2, 1
-; RV32I-NEXT:    srl t0, t0, a0
-; RV32I-NEXT:    slli t1, t1, 1
-; RV32I-NEXT:    srl t2, t2, a0
-; RV32I-NEXT:    slli s2, a4, 1
-; RV32I-NEXT:    sra s3, a4, a0
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl t5, a6, a0
+; RV32I-NEXT:    slli t6, a7, 1
+; RV32I-NEXT:    srl a5, a5, a0
+; RV32I-NEXT:    slli a6, a6, 1
+; RV32I-NEXT:    srl s0, t0, a0
+; RV32I-NEXT:    slli s1, t1, 1
+; RV32I-NEXT:    srl a7, a7, a0
+; RV32I-NEXT:    slli t0, t0, 1
+; RV32I-NEXT:    srl t1, t1, a0
+; RV32I-NEXT:    slli s2, t2, 1
+; RV32I-NEXT:    sra t2, t2, a0
 ; RV32I-NEXT:    sll a0, t4, a1
-; RV32I-NEXT:    sll a4, a5, a1
-; RV32I-NEXT:    sll a5, t6, a1
-; RV32I-NEXT:    sll a7, a7, a1
-; RV32I-NEXT:    sll t4, s1, a1
-; RV32I-NEXT:    sll t1, t1, a1
-; RV32I-NEXT:    sll t6, s2, a1
-; RV32I-NEXT:    srli s1, s3, 24
-; RV32I-NEXT:    srli s2, s3, 16
-; RV32I-NEXT:    srli s4, s3, 8
+; RV32I-NEXT:    sll a4, a4, a1
+; RV32I-NEXT:    sll t4, t6, a1
+; RV32I-NEXT:    sll a6, a6, a1
+; RV32I-NEXT:    sll t6, s1, a1
+; RV32I-NEXT:    sll t0, t0, a1
+; RV32I-NEXT:    sll s1, s2, a1
+; RV32I-NEXT:    srli s2, t2, 24
+; RV32I-NEXT:    srli s3, t2, 16
+; RV32I-NEXT:    srli s4, t2, 8
 ; RV32I-NEXT:    or a0, t3, a0
 ; RV32I-NEXT:    or a1, a3, a4
-; RV32I-NEXT:    or a3, t5, a5
-; RV32I-NEXT:    or a4, a6, a7
-; RV32I-NEXT:    or a5, s0, t4
-; RV32I-NEXT:    or a6, t0, t1
-; RV32I-NEXT:    or a7, t2, t6
-; RV32I-NEXT:    sb s3, 28(a2)
+; RV32I-NEXT:    or a3, t5, t4
+; RV32I-NEXT:    or a4, a5, a6
+; RV32I-NEXT:    or a5, s0, t6
+; RV32I-NEXT:    or a6, a7, t0
+; RV32I-NEXT:    or a7, t1, s1
+; RV32I-NEXT:    sb t2, 28(a2)
 ; RV32I-NEXT:    sb s4, 29(a2)
-; RV32I-NEXT:    sb s2, 30(a2)
-; RV32I-NEXT:    sb s1, 31(a2)
+; RV32I-NEXT:    sb s3, 30(a2)
+; RV32I-NEXT:    sb s2, 31(a2)
 ; RV32I-NEXT:    srli t0, a7, 24
 ; RV32I-NEXT:    srli t1, a7, 16
 ; RV32I-NEXT:    srli t2, a7, 8
diff --git a/llvm/test/CodeGen/RISCV/xmips-cbop.ll b/llvm/test/CodeGen/RISCV/xmips-cbop.ll
new file mode 100644
index 0000000000000..cbbd1de13192c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xmips-cbop.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -mattr=+xmipscbop -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32XMIPSPREFETCH
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscbop -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64XMIPSPREFETCH
+
+define void @prefetch_data_read(ptr noundef %ptr) nounwind {
+; RV32XMIPSPREFETCH-LABEL: prefetch_data_read:
+; RV32XMIPSPREFETCH:       # %bb.0: # %entry
+; RV32XMIPSPREFETCH-NEXT:    mips.pref 8, 1(a0)
+; RV32XMIPSPREFETCH-NEXT:    ret
+;
+; RV64XMIPSPREFETCH-LABEL: prefetch_data_read:
+; RV64XMIPSPREFETCH:       # %bb.0: # %entry
+; RV64XMIPSPREFETCH-NEXT:    mips.pref 8, 1(a0)
+; RV64XMIPSPREFETCH-NEXT:    ret
+entry:
+  %arrayidx = getelementptr inbounds nuw i8, ptr %ptr, i64 1
+  tail call void @llvm.prefetch.p0(ptr nonnull %arrayidx, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @prefetch_data_write(ptr noundef %ptr) nounwind  {
+; RV32XMIPSPREFETCH-LABEL: prefetch_data_write:
+; RV32XMIPSPREFETCH:       # %bb.0:
+; RV32XMIPSPREFETCH-NEXT:    addi a0, a0, 512
+; RV32XMIPSPREFETCH-NEXT:    mips.pref 9, 0(a0)
+; RV32XMIPSPREFETCH-NEXT:    ret
+;
+; RV64XMIPSPREFETCH-LABEL: prefetch_data_write:
+; RV64XMIPSPREFETCH:       # %bb.0:
+; RV64XMIPSPREFETCH-NEXT:    addi a0, a0, 512
+; RV64XMIPSPREFETCH-NEXT:    mips.pref 9, 0(a0)
+; RV64XMIPSPREFETCH-NEXT:    ret
+  %arrayidx = getelementptr inbounds nuw i8, ptr %ptr, i64 512
+  tail call void @llvm.prefetch.p0(ptr nonnull %arrayidx, i32 1, i32 0, i32 1)
+  ret void
+}
+
+define void @prefetch_inst_read(ptr noundef %ptr) nounwind  {
+; RV32XMIPSPREFETCH-LABEL: prefetch_inst_read:
+; RV32XMIPSPREFETCH:       # %bb.0:
+; RV32XMIPSPREFETCH-NEXT:    ret
+;
+; RV64XMIPSPREFETCH-LABEL: prefetch_inst_read:
+; RV64XMIPSPREFETCH:       # %bb.0:
+; RV64XMIPSPREFETCH-NEXT:    ret
+  %arrayidx = getelementptr inbounds nuw i8, ptr %ptr, i64 512
+  tail call void @llvm.prefetch.p0(ptr nonnull %arrayidx, i32 0, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/xqcibm-cto-clo-brev.ll b/llvm/test/CodeGen/RISCV/xqcibm-cto-clo-brev.ll
index 691c5bec7fb51..f227fa9aa423d 100644
--- a/llvm/test/CodeGen/RISCV/xqcibm-cto-clo-brev.ll
+++ b/llvm/test/CodeGen/RISCV/xqcibm-cto-clo-brev.ll
@@ -105,8 +105,7 @@ define i16 @test_cttz_i16(i16 %a) nounwind {
 ;
 ; RV32ZBBXQCIBM-LABEL: test_cttz_i16:
 ; RV32ZBBXQCIBM:       # %bb.0:
-; RV32ZBBXQCIBM-NEXT:    lui a1, 16
-; RV32ZBBXQCIBM-NEXT:    orn a0, a1, a0
+; RV32ZBBXQCIBM-NEXT:    qc.insbi a0, -1, 1, 16
 ; RV32ZBBXQCIBM-NEXT:    ctz a0, a0
 ; RV32ZBBXQCIBM-NEXT:    ret
   %1 = xor i16 %a, -1
diff --git a/llvm/test/CodeGen/RISCV/xqcibm-insert.ll b/llvm/test/CodeGen/RISCV/xqcibm-insert.ll
new file mode 100644
index 0000000000000..6b7f9ae856625
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcibm-insert.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32IXQCIBM
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm,+zbs -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32IXQCIBMZBS
+
+
+define i32 @test_ori(i32 %a) nounwind {
+; RV32I-LABEL: test_ori:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ori a0, a0, 511
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test_ori:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    ori a0, a0, 511
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test_ori:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    ori a0, a0, 511
+; RV32IXQCIBMZBS-NEXT:    ret
+  %or = or i32 %a, 511
+  ret i32 %or
+}
+
+define i32 @test_insbi_mask(i32 %a) nounwind {
+; RV32I-LABEL: test_insbi_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test_insbi_mask:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, -1, 16, 0
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test_insbi_mask:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, -1, 16, 0
+; RV32IXQCIBMZBS-NEXT:    ret
+  %or = or i32 %a, 65535
+  ret i32 %or
+}
+
+define i32 @test_insbi_shifted_mask(i32 %a) nounwind {
+; RV32I-LABEL: test_insbi_shifted_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 15
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test_insbi_shifted_mask:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, -1, 4, 12
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test_insbi_shifted_mask:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, -1, 4, 12
+; RV32IXQCIBMZBS-NEXT:    ret
+  %or = or i32 %a, 61440
+  ret i32 %or
+}
+
+define i32 @test_single_bit_set(i32 %a) nounwind {
+; RV32I-LABEL: test_single_bit_set:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test_single_bit_set:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, -1, 1, 12
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test_single_bit_set:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    bseti a0, a0, 12
+; RV32IXQCIBMZBS-NEXT:    ret
+  %or = or i32 %a, 4096
+  ret i32 %or
+}
diff --git a/llvm/test/CodeGen/RISCV/xqcicm.ll b/llvm/test/CodeGen/RISCV/xqcicm.ll
index 4adb3dce265d1..56fc864f8ccdc 100644
--- a/llvm/test/CodeGen/RISCV/xqcicm.ll
+++ b/llvm/test/CodeGen/RISCV/xqcicm.ll
@@ -677,3 +677,26 @@ entry:
   %sel = select i1 %cmp, i32 %x, i32 %y
   ret i32 %sel
 }
+
+define i32 @select_cc_example_ule_neg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ule_neg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, -10
+; RV32I-NEXT:    bltu a0, a1, .LBB31_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB31_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ule_neg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    li a1, -10
+; RV32IXQCICM-NEXT:    qc.mvltu a3, a0, a1, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ule i32 %a, -11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
diff --git a/llvm/test/CodeGen/RISCV/zicond-opts.ll b/llvm/test/CodeGen/RISCV/zicond-opts.ll
index 2512ba803cf48..35b06c4f4fb41 100644
--- a/llvm/test/CodeGen/RISCV/zicond-opts.ll
+++ b/llvm/test/CodeGen/RISCV/zicond-opts.ll
@@ -62,37 +62,30 @@ define i64 @rotate_l_nez(i64 %x, i64 %rot.amt, i1 %cond) {
 ; RV32ZICOND-LABEL: rotate_l_nez:
 ; RV32ZICOND:       # %bb.0:
 ; RV32ZICOND-NEXT:    andi a4, a4, 1
-; RV32ZICOND-NEXT:    bexti a3, a2, 5
-; RV32ZICOND-NEXT:    not a5, a2
-; RV32ZICOND-NEXT:    czero.nez a6, a1, a3
-; RV32ZICOND-NEXT:    czero.eqz a7, a0, a3
-; RV32ZICOND-NEXT:    czero.nez t0, a0, a3
-; RV32ZICOND-NEXT:    czero.eqz a3, a1, a3
-; RV32ZICOND-NEXT:    czero.eqz a0, a0, a4
-; RV32ZICOND-NEXT:    czero.eqz a1, a1, a4
-; RV32ZICOND-NEXT:    or a6, a7, a6
-; RV32ZICOND-NEXT:    or a3, a3, t0
-; RV32ZICOND-NEXT:    sll a7, a6, a2
-; RV32ZICOND-NEXT:    srli t0, a3, 1
-; RV32ZICOND-NEXT:    sll a2, a3, a2
-; RV32ZICOND-NEXT:    srli a3, a6, 1
-; RV32ZICOND-NEXT:    srl a6, t0, a5
-; RV32ZICOND-NEXT:    srl a3, a3, a5
-; RV32ZICOND-NEXT:    or a5, a7, a6
-; RV32ZICOND-NEXT:    or a2, a2, a3
 ; RV32ZICOND-NEXT:    czero.nez a2, a2, a4
-; RV32ZICOND-NEXT:    czero.nez a3, a5, a4
-; RV32ZICOND-NEXT:    or a0, a0, a2
-; RV32ZICOND-NEXT:    or a1, a1, a3
+; RV32ZICOND-NEXT:    bexti a3, a2, 5
+; RV32ZICOND-NEXT:    czero.nez a4, a0, a3
+; RV32ZICOND-NEXT:    czero.eqz a5, a1, a3
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a0, a0, a3
+; RV32ZICOND-NEXT:    not a3, a2
+; RV32ZICOND-NEXT:    or a4, a5, a4
+; RV32ZICOND-NEXT:    or a0, a0, a1
+; RV32ZICOND-NEXT:    sll a1, a4, a2
+; RV32ZICOND-NEXT:    srli a5, a0, 1
+; RV32ZICOND-NEXT:    sll a2, a0, a2
+; RV32ZICOND-NEXT:    srli a4, a4, 1
+; RV32ZICOND-NEXT:    srl a0, a5, a3
+; RV32ZICOND-NEXT:    srl a3, a4, a3
+; RV32ZICOND-NEXT:    or a0, a1, a0
+; RV32ZICOND-NEXT:    or a1, a2, a3
 ; RV32ZICOND-NEXT:    ret
 ;
 ; RV64ZICOND-LABEL: rotate_l_nez:
 ; RV64ZICOND:       # %bb.0:
 ; RV64ZICOND-NEXT:    andi a2, a2, 1
-; RV64ZICOND-NEXT:    rol a1, a0, a1
 ; RV64ZICOND-NEXT:    czero.nez a1, a1, a2
-; RV64ZICOND-NEXT:    czero.eqz a0, a0, a2
-; RV64ZICOND-NEXT:    or a0, a0, a1
+; RV64ZICOND-NEXT:    rol a0, a0, a1
 ; RV64ZICOND-NEXT:    ret
   %6 = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %rot.amt)
   %7 = select i1 %cond, i64 %x, i64 %6
@@ -104,37 +97,30 @@ define i64 @rotate_l_eqz(i64 %x, i64 %rot.amt, i1 %cond) {
 ; RV32ZICOND-LABEL: rotate_l_eqz:
 ; RV32ZICOND:       # %bb.0:
 ; RV32ZICOND-NEXT:    andi a4, a4, 1
-; RV32ZICOND-NEXT:    bexti a3, a2, 5
-; RV32ZICOND-NEXT:    not a5, a2
-; RV32ZICOND-NEXT:    czero.nez a6, a1, a3
-; RV32ZICOND-NEXT:    czero.eqz a7, a0, a3
-; RV32ZICOND-NEXT:    czero.nez t0, a0, a3
-; RV32ZICOND-NEXT:    czero.eqz a3, a1, a3
-; RV32ZICOND-NEXT:    czero.nez a0, a0, a4
-; RV32ZICOND-NEXT:    czero.nez a1, a1, a4
-; RV32ZICOND-NEXT:    or a6, a7, a6
-; RV32ZICOND-NEXT:    or a3, a3, t0
-; RV32ZICOND-NEXT:    sll a7, a6, a2
-; RV32ZICOND-NEXT:    srli t0, a3, 1
-; RV32ZICOND-NEXT:    sll a2, a3, a2
-; RV32ZICOND-NEXT:    srli a3, a6, 1
-; RV32ZICOND-NEXT:    srl a6, t0, a5
-; RV32ZICOND-NEXT:    srl a3, a3, a5
-; RV32ZICOND-NEXT:    or a5, a7, a6
-; RV32ZICOND-NEXT:    or a2, a2, a3
 ; RV32ZICOND-NEXT:    czero.eqz a2, a2, a4
-; RV32ZICOND-NEXT:    czero.eqz a3, a5, a4
-; RV32ZICOND-NEXT:    or a0, a2, a0
-; RV32ZICOND-NEXT:    or a1, a3, a1
+; RV32ZICOND-NEXT:    bexti a3, a2, 5
+; RV32ZICOND-NEXT:    czero.nez a4, a0, a3
+; RV32ZICOND-NEXT:    czero.eqz a5, a1, a3
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a0, a0, a3
+; RV32ZICOND-NEXT:    not a3, a2
+; RV32ZICOND-NEXT:    or a4, a5, a4
+; RV32ZICOND-NEXT:    or a0, a0, a1
+; RV32ZICOND-NEXT:    sll a1, a4, a2
+; RV32ZICOND-NEXT:    srli a5, a0, 1
+; RV32ZICOND-NEXT:    sll a2, a0, a2
+; RV32ZICOND-NEXT:    srli a4, a4, 1
+; RV32ZICOND-NEXT:    srl a0, a5, a3
+; RV32ZICOND-NEXT:    srl a3, a4, a3
+; RV32ZICOND-NEXT:    or a0, a1, a0
+; RV32ZICOND-NEXT:    or a1, a2, a3
 ; RV32ZICOND-NEXT:    ret
 ;
 ; RV64ZICOND-LABEL: rotate_l_eqz:
 ; RV64ZICOND:       # %bb.0:
 ; RV64ZICOND-NEXT:    andi a2, a2, 1
-; RV64ZICOND-NEXT:    rol a1, a0, a1
-; RV64ZICOND-NEXT:    czero.nez a0, a0, a2
 ; RV64ZICOND-NEXT:    czero.eqz a1, a1, a2
-; RV64ZICOND-NEXT:    or a0, a1, a0
+; RV64ZICOND-NEXT:    rol a0, a0, a1
 ; RV64ZICOND-NEXT:    ret
   %6 = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %rot.amt)
   %7 = select i1 %cond, i64 %6, i64 %x
diff --git a/llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll b/llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll
new file mode 100644
index 0000000000000..aa60e13232b46
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll
@@ -0,0 +1,22 @@
+; RUN: llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - | FileCheck %s --check-prefix=CHECK-NOEXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; RUN: llc -O0 -mtriple=spirv1.6-vulkan1.3-compute -spirv-ext=+SPV_KHR_float_controls2 %s -o - | FileCheck %s --check-prefix=CHECK-EXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute -spirv-ext=+SPV_KHR_float_controls2 %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-NOEXT-NOT: OpDecorate FPFastMathMode
+
+; CHECK-EXT: OpCapability FloatControls2
+; CHECK-EXT: OpExtension "SPV_KHR_float_controls2"
+; CHECK-EXT: OpDecorate {{%[0-9]+}} FPFastMathMode NotNaN|NotInf|NSZ|AllowRecip|Fast
+
+define hidden spir_func float @foo(float  %0) local_unnamed_addr {
+  %2 = fmul reassoc nnan ninf nsz arcp afn float %0, 2.000000e+00
+  ret float %2
+}
+
+define void @main() local_unnamed_addr #1 {
+  ret void
+}
+
+attributes #1 = { "hlsl.numthreads"="8,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
index 02d21a1abafa5..4db0ba33d52c9 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
@@ -1,4 +1,6 @@
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=all,-SPV_INTEL_arbitrary_precision_integers %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=KHR %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=khr %s -o - | FileCheck %s
 
 define i6 @foo() {
   %call = tail call i32 @llvm.bitreverse.i32(i32 42)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll
new file mode 100644
index 0000000000000..66fda60b5ac4e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll
@@ -0,0 +1,36 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str.signed = private unnamed_addr constant [7 x i8] c"signed\00", align 1
+@.str.unsigned = private unnamed_addr constant [9 x i8] c"unsigned\00", align 1
+
+; CHECK: {{.*}} OpFunction {{.*}}
+define void @main_unsigned() local_unnamed_addr #0 {
+entry:
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.unsigned)
+
+; CHECK: {{%[0-9]+}} = OpImageRead {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}}
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+; CHECK: OpImageWrite {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}}
+  store i32 %1, ptr %2, align 4
+  ret void
+}
+
+; CHECK: {{.*}} OpFunction {{.*}}
+define void @main_signed() local_unnamed_addr #0 {
+entry:
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.signed)
+
+; CHECK: {{%[0-9]+}} = OpImageRead {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}} SignExtend
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+; CHECK: OpImageWrite {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}} SignExtend
+  store i32 %1, ptr %2, align 4
+  ret void
+}
+
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll
new file mode 100644
index 0000000000000..354796ce9c1eb
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll
@@ -0,0 +1,137 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
+; CHECK-DAG: [[uint:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[v2int:%[0-9]+]] = OpTypeVector [[uint]] 2
+; CHECK-DAG: [[v4int:%[0-9]+]] = OpTypeVector [[uint]] 4
+; CHECK-DAG: [[zero:%[0-9]+]] = OpConstant [[uint]] 0
+; CHECK-DAG: [[one:%[0-9]+]] = OpConstant [[uint]] 1
+; CHECK-DAG: [[twenty:%[0-9]+]] = OpConstant [[uint]] 20
+; CHECK-DAG: [[twenty_three:%[0-9]+]] = OpConstant [[uint]] 23
+; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[uint]] Buffer 2 0 0 2 Unknown 
+; CHECK-DAG: [[ImagePtr:%[0-9]+]] = OpTypePointer UniformConstant [[ImageType]]
+; CHECK-DAG: [[Var:%[0-9]+]] = OpVariable [[ImagePtr]] UniformConstant
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_scalar() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]] SignExtend
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]] SignExtend
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store i32 %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]] SignExtend
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load i32, ptr %3, align 4
+  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]] SignExtend
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store i32 %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector2() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]] SignExtend
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <2 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]] SignExtend
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <2 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]] SignExtend
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <2 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]] SignExtend
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <2 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector4() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]] SignExtend
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <4 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[R]] SignExtend
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <4 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]] SignExtend
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <4 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[R]] SignExtend
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <4 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none) "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(none) }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll
new file mode 100644
index 0000000000000..e22724f02fa75
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll
@@ -0,0 +1,137 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
+; CHECK-DAG: [[uint:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[v2int:%[0-9]+]] = OpTypeVector [[uint]] 2
+; CHECK-DAG: [[v4int:%[0-9]+]] = OpTypeVector [[uint]] 4
+; CHECK-DAG: [[zero:%[0-9]+]] = OpConstant [[uint]] 0
+; CHECK-DAG: [[one:%[0-9]+]] = OpConstant [[uint]] 1
+; CHECK-DAG: [[twenty:%[0-9]+]] = OpConstant [[uint]] 20
+; CHECK-DAG: [[twenty_three:%[0-9]+]] = OpConstant [[uint]] 23
+; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[uint]] Buffer 2 0 0 2 Unknown 
+; CHECK-DAG: [[ImagePtr:%[0-9]+]] = OpTypePointer UniformConstant [[ImageType]]
+; CHECK-DAG: [[Var:%[0-9]+]] = OpVariable [[ImagePtr]] UniformConstant
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_scalar() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]]
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store i32 %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]]
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load i32, ptr %3, align 4
+  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store i32 %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector2() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]]
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <2 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <2 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]]
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <2 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <2 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector4() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <4 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[R]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <4 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <4 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[R]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <4 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none) "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(none) }
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ptr-annotation.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ptr-annotation.ll
index 556062a6fef4b..05071dd43f74c 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ptr-annotation.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/ptr-annotation.ll
@@ -7,13 +7,13 @@
 ; CHECK-DAG: OpName %[[#Ptr3:]] "_arg3"
 ; CHECK-DAG: OpName %[[#Ptr4:]] "_arg4"
 ; CHECK-DAG: OpName %[[#Ptr5:]] "_arg5"
-; CHECK-DAG: OpDecorate %[[#Ptr1]] NonReadable
+; CHECK-DAG: OpDecorate %[[#Ptr1]] Restrict
 ; CHECK-DAG: OpDecorate %[[#Ptr2]] Alignment 128
-; CHECK-DAG: OpDecorate %[[#Ptr2]] NonReadable
+; CHECK-DAG: OpDecorate %[[#Ptr2]] Restrict
 ; CHECK-DAG: OpDecorate %[[#Ptr3]] Alignment 128
-; CHECK-DAG: OpDecorate %[[#Ptr3]] NonReadable
+; CHECK-DAG: OpDecorate %[[#Ptr3]] Restrict
 ; CHECK-DAG: OpDecorate %[[#Ptr4]] Alignment 128
-; CHECK-DAG: OpDecorate %[[#Ptr4]] NonReadable
+; CHECK-DAG: OpDecorate %[[#Ptr4]] Restrict
 ; CHECK-DAG: OpDecorate %[[#Ptr5]] UserSemantic "Unknown format"
 ; CHECK: %[[#Foo]] = OpFunction
 ; CHECK-NEXT: %[[#Ptr1]] = OpFunctionParameter
@@ -24,10 +24,10 @@
 ; CHECK: OpFunctionEnd
 
 @.str.0 = private unnamed_addr addrspace(1) constant [16 x i8] c"../prefetch.hpp\00", section "llvm.metadata"
-@.str.1 = private unnamed_addr addrspace(1) constant [5 x i8] c"{25}\00", section "llvm.metadata"
-@.str.2 = private unnamed_addr addrspace(1) constant [13 x i8] c"{44:128}{25}\00", section "llvm.metadata"
-@.str.3 = private unnamed_addr addrspace(1) constant [15 x i8] c"{44:\22128\22}{25}\00", section "llvm.metadata"
-@.str.4 = private unnamed_addr addrspace(1) constant [13 x i8] c"{44,128}{25}\00", section "llvm.metadata"
+@.str.1 = private unnamed_addr addrspace(1) constant [5 x i8] c"{19}\00", section "llvm.metadata"
+@.str.2 = private unnamed_addr addrspace(1) constant [13 x i8] c"{44:128}{19}\00", section "llvm.metadata"
+@.str.3 = private unnamed_addr addrspace(1) constant [15 x i8] c"{44:\22128\22}{19}\00", section "llvm.metadata"
+@.str.4 = private unnamed_addr addrspace(1) constant [13 x i8] c"{44,128}{19}\00", section "llvm.metadata"
 @.str.5 = private unnamed_addr addrspace(1) constant [15 x i8] c"Unknown format\00", section "llvm.metadata"
 
 define spir_kernel void @foo(ptr addrspace(1) %_arg1, ptr addrspace(1) %_arg2, ptr addrspace(1) %_arg3, ptr addrspace(1) %_arg4, ptr addrspace(1) %_arg5) {
diff --git a/llvm/test/CodeGen/SystemZ/canonicalize-vars.ll b/llvm/test/CodeGen/SystemZ/canonicalize-vars.ll
new file mode 100644
index 0000000000000..e02f931c4d31e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/canonicalize-vars.ll
@@ -0,0 +1,405 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --default-march s390x-unknown-linux-gnu --version 5
+; RUN: llc -mtriple=s390x-linux-gnu -mcpu=z16 < %s | FileCheck %s -check-prefixes=Z16
+
+define half @canonicalize_fp16(half %a) nounwind {
+; Z16-LABEL: canonicalize_fp16:
+; Z16:       # %bb.0:
+; Z16-NEXT:    stmg %r14, %r15, 112(%r15)
+; Z16-NEXT:    aghi %r15, -160
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    lmg %r14, %r15, 272(%r15)
+; Z16-NEXT:    br %r14
+  %canonicalized = call half @llvm.canonicalize.f16(half %a)
+  ret half %canonicalized
+}
+define float @canonicalize_fp32(float %a) {
+; Z16-LABEL: canonicalize_fp32:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    br %r14
+  %canonicalized = call float @llvm.canonicalize.f32(float %a)
+  ret float %canonicalized
+}
+
+define double @canonicalize_fp64(double %a) {
+; Z16-LABEL: canonicalize_fp64:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vgmg %v1, 2, 11
+; Z16-NEXT:    mdbr %f0, %f1
+; Z16-NEXT:    br %r14
+  %canonicalized = call double @llvm.canonicalize.f64(double %a)
+  ret double %canonicalized
+}
+
+define fp128 @canonicalize_fp128(fp128 %a) {
+; Z16-LABEL: canonicalize_fp128:
+; Z16:       # %bb.0:
+; Z16-NEXT:    larl %r1, .LCPI3_0
+; Z16-NEXT:    vl %v0, 0(%r3), 3
+; Z16-NEXT:    vl %v1, 0(%r1), 3
+; Z16-NEXT:    wfmxb %v0, %v0, %v1
+; Z16-NEXT:    vst %v0, 0(%r2), 3
+; Z16-NEXT:    br %r14
+  %canonicalized = call fp128 @llvm.canonicalize.f128(fp128 %a)
+  ret fp128 %canonicalized
+}
+
+define void @canonicalize_ptr_f16(ptr %out) nounwind {
+; Z16-LABEL: canonicalize_ptr_f16:
+; Z16:       # %bb.0:
+; Z16-NEXT:    stmg %r13, %r15, 104(%r15)
+; Z16-NEXT:    aghi %r15, -160
+; Z16-NEXT:    vlreph %v0, 0(%r2)
+; Z16-NEXT:    lgr %r13, %r2
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    vsteh %v0, 0(%r13), 0
+; Z16-NEXT:    lmg %r13, %r15, 264(%r15)
+; Z16-NEXT:    br %r14
+  %val = load half, ptr %out
+  %canonicalized = call half @llvm.canonicalize.f16(half %val)
+  store half %canonicalized, ptr %out
+  ret void
+}
+
+define void @canonicalize_ptr_f32(ptr %out) {
+; Z16-LABEL: canonicalize_ptr_f32:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vgmf %v0, 2, 8
+; Z16-NEXT:    meeb %f0, 0(%r2)
+; Z16-NEXT:    ste %f0, 0(%r2)
+; Z16-NEXT:    br %r14
+  %val = load float, ptr %out
+  %canonicalized = call float @llvm.canonicalize.f32(float %val)
+  store float %canonicalized, ptr %out
+  ret void
+}
+
+define void @canonicalize_ptr_f64(ptr %out) {
+; Z16-LABEL: canonicalize_ptr_f64:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vgmg %v0, 2, 11
+; Z16-NEXT:    mdb %f0, 0(%r2)
+; Z16-NEXT:    std %f0, 0(%r2)
+; Z16-NEXT:    br %r14
+  %val = load double, ptr %out
+  %canonicalized = call double @llvm.canonicalize.f64(double %val)
+  store double %canonicalized, ptr %out
+  ret void
+}
+
+define void @canonicalize_ptr_f128(ptr %out) {
+; Z16-LABEL: canonicalize_ptr_f128:
+; Z16:       # %bb.0:
+; Z16-NEXT:    larl %r1, .LCPI7_0
+; Z16-NEXT:    vl %v0, 0(%r2), 3
+; Z16-NEXT:    vl %v1, 0(%r1), 3
+; Z16-NEXT:    wfmxb %v0, %v0, %v1
+; Z16-NEXT:    vst %v0, 0(%r2), 3
+; Z16-NEXT:    br %r14
+  %val = load fp128, ptr %out
+  %canonicalized = call fp128 @llvm.canonicalize.f128(fp128 %val)
+  store fp128 %canonicalized, ptr %out
+  ret void
+}
+define <8 x half> @canonicalize_v8f16(<8 x half> %a) nounwind {
+; Z16-LABEL: canonicalize_v8f16:
+; Z16:       # %bb.0:
+; Z16-NEXT:    stmg %r13, %r15, 104(%r15)
+; Z16-NEXT:    aghi %r15, -224
+; Z16-NEXT:    std %f8, 216(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f9, 208(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f10, 200(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f11, 192(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f12, 184(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f13, 176(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f14, 168(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f15, 160(%r15) # 8-byte Spill
+; Z16-NEXT:    vlreph %v11, 414(%r15)
+; Z16-NEXT:    vlreph %v12, 406(%r15)
+; Z16-NEXT:    vlreph %v13, 398(%r15)
+; Z16-NEXT:    vlreph %v14, 390(%r15)
+; Z16-NEXT:    ldr %f8, %f6
+; Z16-NEXT:    ldr %f9, %f4
+; Z16-NEXT:    ldr %f10, %f2
+; Z16-NEXT:    lgr %r13, %r2
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f15, %f0
+; Z16-NEXT:    ldr %f0, %f10
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f10, %f0
+; Z16-NEXT:    ldr %f0, %f9
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f9, %f0
+; Z16-NEXT:    ldr %f0, %f8
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f8, %f0
+; Z16-NEXT:    ldr %f0, %f14
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f14, %f0
+; Z16-NEXT:    ldr %f0, %f13
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f13, %f0
+; Z16-NEXT:    ldr %f0, %f12
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f12, %f0
+; Z16-NEXT:    ldr %f0, %f11
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    vsteh %v0, 14(%r13), 0
+; Z16-NEXT:    vsteh %v12, 12(%r13), 0
+; Z16-NEXT:    vsteh %v13, 10(%r13), 0
+; Z16-NEXT:    vsteh %v14, 8(%r13), 0
+; Z16-NEXT:    vsteh %v8, 6(%r13), 0
+; Z16-NEXT:    vsteh %v9, 4(%r13), 0
+; Z16-NEXT:    vsteh %v10, 2(%r13), 0
+; Z16-NEXT:    vsteh %v15, 0(%r13), 0
+; Z16-NEXT:    ld %f8, 216(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f9, 208(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f10, 200(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f11, 192(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f12, 184(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f13, 176(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f14, 168(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f15, 160(%r15) # 8-byte Reload
+; Z16-NEXT:    lmg %r13, %r15, 328(%r15)
+; Z16-NEXT:    br %r14
+  %canonicalized = call <8 x half> @llvm.canonicalize.v8f16(<8 x half> %a)
+  ret <8 x half> %canonicalized
+}
+define <4 x float> @canonicalize_v4f32(<4 x float> %a) {
+; Z16-LABEL: canonicalize_v4f32:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vrepf %v0, %v24, 3
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    vrepf %v2, %v24, 2
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    meebr %f2, %f1
+; Z16-NEXT:    vrepf %v3, %v24, 1
+; Z16-NEXT:    vmrhf %v0, %v2, %v0
+; Z16-NEXT:    wfmsb %f2, %v24, %f1
+; Z16-NEXT:    wfmsb %f1, %f3, %f1
+; Z16-NEXT:    vmrhf %v1, %v2, %v1
+; Z16-NEXT:    vmrhg %v24, %v1, %v0
+; Z16-NEXT:    br %r14
+  %canonicalized = call <4 x float> @llvm.canonicalize.v4f32(<4 x float> %a)
+  ret <4 x float> %canonicalized
+}
+
+define <4 x double> @canonicalize_v4f64(<4 x double> %a) {
+; Z16-LABEL: canonicalize_v4f64:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vgmg %v0, 2, 11
+; Z16-NEXT:    vrepg %v2, %v24, 1
+; Z16-NEXT:    wfmdb %f1, %v24, %f0
+; Z16-NEXT:    mdbr %f2, %f0
+; Z16-NEXT:    vmrhg %v24, %v1, %v2
+; Z16-NEXT:    vrepg %v2, %v26, 1
+; Z16-NEXT:    wfmdb %f1, %v26, %f0
+; Z16-NEXT:    wfmdb %f0, %f2, %f0
+; Z16-NEXT:    vmrhg %v26, %v1, %v0
+; Z16-NEXT:    br %r14
+  %canonicalized = call <4 x double> @llvm.canonicalize.v4f64(<4 x double> %a)
+  ret <4 x double> %canonicalized
+}
+
+define <2 x fp128> @canonicalize_v2f128(<2 x fp128> %a) {
+; Z16-LABEL: canonicalize_v2f128:
+; Z16:       # %bb.0:
+; Z16-NEXT:    larl %r1, .LCPI11_0
+; Z16-NEXT:    vl %v0, 16(%r3), 3
+; Z16-NEXT:    vl %v1, 0(%r3), 3
+; Z16-NEXT:    vl %v2, 0(%r1), 3
+; Z16-NEXT:    wfmxb %v1, %v1, %v2
+; Z16-NEXT:    wfmxb %v0, %v0, %v2
+; Z16-NEXT:    vst %v0, 16(%r2), 4
+; Z16-NEXT:    vst %v1, 0(%r2), 4
+; Z16-NEXT:    br %r14
+  %canonicalized = call <2 x fp128> @llvm.canonicalize.v2f128(<2 x fp128> %a)
+  ret <2 x fp128> %canonicalized
+}
+
+define void @canonicalize_ptr_v8f16(ptr %out) nounwind {
+; Z16-LABEL: canonicalize_ptr_v8f16:
+; Z16:       # %bb.0:
+; Z16-NEXT:    stmg %r13, %r15, 104(%r15)
+; Z16-NEXT:    aghi %r15, -224
+; Z16-NEXT:    std %f8, 216(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f9, 208(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f10, 200(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f11, 192(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f12, 184(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f13, 176(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f14, 168(%r15) # 8-byte Spill
+; Z16-NEXT:    std %f15, 160(%r15) # 8-byte Spill
+; Z16-NEXT:    vlreph %v0, 0(%r2)
+; Z16-NEXT:    vlreph %v8, 14(%r2)
+; Z16-NEXT:    vlreph %v9, 12(%r2)
+; Z16-NEXT:    vlreph %v10, 10(%r2)
+; Z16-NEXT:    lgr %r13, %r2
+; Z16-NEXT:    vlreph %v11, 8(%r2)
+; Z16-NEXT:    vlreph %v12, 6(%r2)
+; Z16-NEXT:    vlreph %v13, 4(%r2)
+; Z16-NEXT:    vlreph %v14, 2(%r2)
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f15, %f0
+; Z16-NEXT:    ldr %f0, %f14
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f14, %f0
+; Z16-NEXT:    ldr %f0, %f13
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f13, %f0
+; Z16-NEXT:    ldr %f0, %f12
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f12, %f0
+; Z16-NEXT:    ldr %f0, %f11
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f11, %f0
+; Z16-NEXT:    ldr %f0, %f10
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f10, %f0
+; Z16-NEXT:    ldr %f0, %f9
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    ldr %f9, %f0
+; Z16-NEXT:    ldr %f0, %f8
+; Z16-NEXT:    brasl %r14, __extendhfsf2@PLT
+; Z16-NEXT:    vgmf %v1, 2, 8
+; Z16-NEXT:    meebr %f0, %f1
+; Z16-NEXT:    brasl %r14, __truncsfhf2@PLT
+; Z16-NEXT:    vsteh %v9, 12(%r13), 0
+; Z16-NEXT:    vsteh %v10, 10(%r13), 0
+; Z16-NEXT:    vsteh %v11, 8(%r13), 0
+; Z16-NEXT:    vsteh %v12, 6(%r13), 0
+; Z16-NEXT:    vsteh %v13, 4(%r13), 0
+; Z16-NEXT:    vsteh %v14, 2(%r13), 0
+; Z16-NEXT:    vsteh %v15, 0(%r13), 0
+; Z16-NEXT:    ld %f8, 216(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f9, 208(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f10, 200(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f11, 192(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f12, 184(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f13, 176(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f14, 168(%r15) # 8-byte Reload
+; Z16-NEXT:    ld %f15, 160(%r15) # 8-byte Reload
+; Z16-NEXT:    vsteh %v0, 14(%r13), 0
+; Z16-NEXT:    lmg %r13, %r15, 328(%r15)
+; Z16-NEXT:    br %r14
+  %val = load <8 x half>, ptr %out
+  %canonicalized = call <8 x half> @llvm.canonicalize.v8f16(<8 x half> %val)
+  store <8 x half> %canonicalized, ptr %out
+  ret void
+}
+
+define void @canonicalize_ptr_v4f32(ptr %out) {
+; Z16-LABEL: canonicalize_ptr_v4f32:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vl %v0, 0(%r2), 3
+; Z16-NEXT:    vrepf %v1, %v0, 3
+; Z16-NEXT:    vgmf %v2, 2, 8
+; Z16-NEXT:    vrepf %v3, %v0, 2
+; Z16-NEXT:    meebr %f1, %f2
+; Z16-NEXT:    meebr %f3, %f2
+; Z16-NEXT:    vmrhf %v1, %v3, %v1
+; Z16-NEXT:    wfmsb %f3, %f0, %f2
+; Z16-NEXT:    vrepf %v0, %v0, 1
+; Z16-NEXT:    meebr %f0, %f2
+; Z16-NEXT:    vmrhf %v0, %v3, %v0
+; Z16-NEXT:    vmrhg %v0, %v0, %v1
+; Z16-NEXT:    vst %v0, 0(%r2), 3
+; Z16-NEXT:    br %r14
+  %val = load <4 x float>, ptr %out
+  %canonicalized = call <4 x float> @llvm.canonicalize.v4f32(<4 x float> %val)
+  store <4 x float> %canonicalized, ptr %out
+  ret void
+}
+
+define void @canonicalize_ptr_v4f64(ptr %out) {
+; Z16-LABEL: canonicalize_ptr_v4f64:
+; Z16:       # %bb.0:
+; Z16-NEXT:    vl %v1, 16(%r2), 4
+; Z16-NEXT:    vgmg %v2, 2, 11
+; Z16-NEXT:    wfmdb %f3, %f1, %f2
+; Z16-NEXT:    vrepg %v1, %v1, 1
+; Z16-NEXT:    mdbr %f1, %f2
+; Z16-NEXT:    vl %v0, 0(%r2), 4
+; Z16-NEXT:    vmrhg %v1, %v3, %v1
+; Z16-NEXT:    wfmdb %f3, %f0, %f2
+; Z16-NEXT:    vrepg %v0, %v0, 1
+; Z16-NEXT:    mdbr %f0, %f2
+; Z16-NEXT:    vmrhg %v0, %v3, %v0
+; Z16-NEXT:    vst %v0, 0(%r2), 4
+; Z16-NEXT:    vst %v1, 16(%r2), 4
+; Z16-NEXT:    br %r14
+  %val = load <4 x double>, ptr %out
+  %canonicalized = call <4 x double> @llvm.canonicalize.v4f64(<4 x double> %val)
+  store <4 x double> %canonicalized, ptr %out
+  ret void
+}
+
+define void @canonicalize_ptr_v2f128(ptr %out) {
+; Z16-LABEL: canonicalize_ptr_v2f128:
+; Z16:       # %bb.0:
+; Z16-NEXT:    larl %r1, .LCPI15_0
+; Z16-NEXT:    vl %v0, 16(%r2), 4
+; Z16-NEXT:    vl %v1, 0(%r2), 4
+; Z16-NEXT:    vl %v2, 0(%r1), 3
+; Z16-NEXT:    wfmxb %v1, %v1, %v2
+; Z16-NEXT:    wfmxb %v0, %v0, %v2
+; Z16-NEXT:    vst %v0, 16(%r2), 4
+; Z16-NEXT:    vst %v1, 0(%r2), 4
+; Z16-NEXT:    br %r14
+  %val = load <2 x fp128>, ptr %out
+  %canonicalized = call <2 x fp128> @llvm.canonicalize.v2f128(<2 x fp128> %val)
+  store <2 x fp128> %canonicalized, ptr %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/pr60413.ll b/llvm/test/CodeGen/SystemZ/pr60413.ll
index 6dee3d303a6f2..bbf4d50bd716d 100644
--- a/llvm/test/CodeGen/SystemZ/pr60413.ll
+++ b/llvm/test/CodeGen/SystemZ/pr60413.ll
@@ -13,73 +13,67 @@ declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
 define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-LABEL: m:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
 ; CHECK-NEXT:    aghi %r15, -168
 ; CHECK-NEXT:    lhrl %r1, f+4
-; CHECK-NEXT:    sll %r1, 8
 ; CHECK-NEXT:    larl %r2, f
-; CHECK-NEXT:    ic %r1, 6(%r2)
-; CHECK-NEXT:    larl %r2, e
-; CHECK-NEXT:    lb %r0, 3(%r2)
-; CHECK-NEXT:    vlvgf %v1, %r1, 0
-; CHECK-NEXT:    vlvgf %v1, %r1, 1
-; CHECK-NEXT:    larl %r2, .LCPI0_0
-; CHECK-NEXT:    vl %v2, 0(%r2), 3
-; CHECK-NEXT:    vlvgf %v1, %r1, 3
-; CHECK-NEXT:    vlvgf %v3, %r1, 3
-; CHECK-NEXT:    vlvgf %v0, %r1, 1
-; CHECK-NEXT:    vperm %v4, %v1, %v0, %v2
-; CHECK-NEXT:    vlvgf %v0, %r1, 3
+; CHECK-NEXT:    llc %r2, 6(%r2)
+; CHECK-NEXT:    larl %r3, e
+; CHECK-NEXT:    lb %r0, 3(%r3)
+; CHECK-NEXT:    rosbg %r2, %r1, 32, 55, 8
+; CHECK-NEXT:    vlvgp %v0, %r2, %r0
+; CHECK-NEXT:    vlvgf %v0, %r2, 0
+; CHECK-NEXT:    vlvgf %v0, %r2, 2
+; CHECK-NEXT:    vlvgp %v1, %r0, %r2
+; CHECK-NEXT:    vlvgp %v2, %r2, %r2
+; CHECK-NEXT:    lr %r1, %r2
 ; CHECK-NEXT:    nilh %r1, 255
 ; CHECK-NEXT:    chi %r1, 128
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    risbg %r1, %r1, 63, 191, 36
-; CHECK-NEXT:    vperm %v0, %v3, %v0, %v2
-; CHECK-NEXT:    larl %r2, .LCPI0_1
-; CHECK-NEXT:    vl %v5, 0(%r2), 3
-; CHECK-NEXT:    vgbm %v6, 30583
-; CHECK-NEXT:    vn %v0, %v0, %v6
-; CHECK-NEXT:    vn %v4, %v4, %v6
-; CHECK-NEXT:    vperm %v1, %v1, %v1, %v5
-; CHECK-NEXT:    vn %v1, %v1, %v6
-; CHECK-NEXT:    vperm %v2, %v0, %v3, %v2
-; CHECK-NEXT:    vn %v2, %v2, %v6
+; CHECK-NEXT:    vgbm %v3, 30583
+; CHECK-NEXT:    vn %v0, %v0, %v3
+; CHECK-NEXT:    vlvgf %v1, %r0, 0
+; CHECK-NEXT:    vlvgf %v1, %r0, 2
+; CHECK-NEXT:    vn %v1, %v1, %v3
+; CHECK-NEXT:    vrepf %v2, %v2, 1
+; CHECK-NEXT:    vn %v2, %v2, %v3
 ; CHECK-NEXT:    vrepif %v3, 127
-; CHECK-NEXT:    vchlf %v1, %v1, %v3
-; CHECK-NEXT:    vlgvf %r3, %v1, 1
-; CHECK-NEXT:    vlgvf %r2, %v1, 0
-; CHECK-NEXT:    risbg %r2, %r2, 48, 176, 15
-; CHECK-NEXT:    rosbg %r2, %r3, 49, 49, 14
-; CHECK-NEXT:    vlgvf %r3, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 50, 50, 13
-; CHECK-NEXT:    vlgvf %r3, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 51, 51, 12
-; CHECK-NEXT:    vchlf %v1, %v4, %v3
-; CHECK-NEXT:    vlgvf %r3, %v1, 0
-; CHECK-NEXT:    rosbg %r2, %r3, 52, 52, 11
-; CHECK-NEXT:    vlgvf %r3, %v1, 1
-; CHECK-NEXT:    rosbg %r2, %r3, 53, 53, 10
-; CHECK-NEXT:    vlgvf %r3, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 54, 54, 9
-; CHECK-NEXT:    vlgvf %r3, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 55, 55, 8
-; CHECK-NEXT:    vchlf %v1, %v2, %v3
-; CHECK-NEXT:    vlgvf %r3, %v1, 0
-; CHECK-NEXT:    rosbg %r2, %r3, 56, 56, 7
-; CHECK-NEXT:    vlgvf %r3, %v1, 1
-; CHECK-NEXT:    rosbg %r2, %r3, 57, 57, 6
-; CHECK-NEXT:    vlgvf %r3, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 58, 58, 5
-; CHECK-NEXT:    vlgvf %r3, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 59, 59, 4
 ; CHECK-NEXT:    vchlf %v0, %v0, %v3
-; CHECK-NEXT:    vlgvf %r3, %v0, 0
-; CHECK-NEXT:    rosbg %r2, %r3, 60, 60, 3
-; CHECK-NEXT:    vlgvf %r3, %v0, 1
-; CHECK-NEXT:    rosbg %r2, %r3, 61, 61, 2
-; CHECK-NEXT:    vlgvf %r3, %v0, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 62, 62, 1
-; CHECK-NEXT:    vlgvf %r3, %v0, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 63, 63, 0
+; CHECK-NEXT:    vlgvf %r13, %v0, 0
+; CHECK-NEXT:    vchlf %v2, %v2, %v3
+; CHECK-NEXT:    vlgvf %r3, %v2, 1
+; CHECK-NEXT:    nilf %r3, 1
+; CHECK-NEXT:    vlgvf %r4, %v2, 0
+; CHECK-NEXT:    risbg %r2, %r4, 48, 176, 15
+; CHECK-NEXT:    rosbg %r2, %r3, 32, 49, 14
+; CHECK-NEXT:    vlgvf %r5, %v2, 2
+; CHECK-NEXT:    nilf %r5, 1
+; CHECK-NEXT:    rosbg %r2, %r5, 32, 50, 13
+; CHECK-NEXT:    vlgvf %r14, %v2, 3
+; CHECK-NEXT:    nilf %r14, 1
+; CHECK-NEXT:    rosbg %r2, %r14, 32, 51, 12
+; CHECK-NEXT:    rosbg %r2, %r13, 52, 52, 11
+; CHECK-NEXT:    vlgvf %r13, %v0, 1
+; CHECK-NEXT:    rosbg %r2, %r13, 53, 53, 10
+; CHECK-NEXT:    vlgvf %r13, %v0, 2
+; CHECK-NEXT:    rosbg %r2, %r13, 54, 54, 9
+; CHECK-NEXT:    vlgvf %r13, %v0, 3
+; CHECK-NEXT:    rosbg %r2, %r13, 55, 55, 8
+; CHECK-NEXT:    vchlf %v0, %v1, %v3
+; CHECK-NEXT:    vlgvf %r13, %v0, 0
+; CHECK-NEXT:    rosbg %r2, %r13, 56, 56, 7
+; CHECK-NEXT:    vlgvf %r13, %v0, 1
+; CHECK-NEXT:    rosbg %r2, %r13, 57, 57, 6
+; CHECK-NEXT:    vlgvf %r13, %v0, 2
+; CHECK-NEXT:    rosbg %r2, %r13, 58, 58, 5
+; CHECK-NEXT:    vlgvf %r13, %v0, 3
+; CHECK-NEXT:    rosbg %r2, %r13, 59, 59, 4
+; CHECK-NEXT:    nilf %r4, 1
+; CHECK-NEXT:    rosbg %r2, %r4, 32, 60, 3
+; CHECK-NEXT:    rosbg %r2, %r3, 32, 61, 2
+; CHECK-NEXT:    rosbg %r2, %r5, 32, 62, 1
+; CHECK-NEXT:    or %r2, %r14
 ; CHECK-NEXT:    vlgvb %r4, %v0, 1
 ; CHECK-NEXT:    vlgvb %r3, %v0, 0
 ; CHECK-NEXT:    risbg %r3, %r3, 48, 176, 15
@@ -122,7 +116,7 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    nr %r2, %r0
 ; CHECK-NEXT:    larl %r1, g
 ; CHECK-NEXT:    stc %r2, 0(%r1)
-; CHECK-NEXT:    aghi %r15, 168
+; CHECK-NEXT:    lmg %r13, %r15, 272(%r15)
 ; CHECK-NEXT:    br %r14
 entry:
   %n = alloca i32, align 4
diff --git a/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll b/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll
index 2125a0b8912b1..10f9f28e599f8 100644
--- a/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll
@@ -52,8 +52,8 @@ define <2 x double> @f5(<2 x double> %val) {
 ; CHECK-LABEL: f5:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgbm %v0, 0
-; CHECK-NEXT:    vfchedb %v1, %v0, %v24
-; CHECK-NEXT:    vsel %v24, %v0, %v24, %v1
+; CHECK-NEXT:    vfchedb %v0, %v0, %v24
+; CHECK-NEXT:    vnc %v24, %v24, %v0
 ; CHECK-NEXT:    br %r14
   %cmp = fcmp ugt <2 x double> %val,  zeroinitializer
   %ret = select <2 x i1> %cmp, <2 x double> %val, <2 x double> zeroinitializer
@@ -64,8 +64,8 @@ define <2 x double> @f6(<2 x double> %val) {
 ; CHECK-LABEL: f6:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgbm %v0, 0
-; CHECK-NEXT:    vfchedb %v1, %v24, %v0
-; CHECK-NEXT:    vsel %v24, %v0, %v24, %v1
+; CHECK-NEXT:    vfchedb %v0, %v24, %v0
+; CHECK-NEXT:    vnc %v24, %v24, %v0
 ; CHECK-NEXT:    br %r14
   %cmp = fcmp ult <2 x double> %val,  zeroinitializer
   %ret = select <2 x i1> %cmp, <2 x double> %val, <2 x double> zeroinitializer
@@ -76,8 +76,8 @@ define <4 x float> @f7(<4 x float> %val) {
 ; CHECK-LABEL: f7:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgbm %v0, 0
-; CHECK-NEXT:    vfchesb %v1, %v0, %v24
-; CHECK-NEXT:    vsel %v24, %v0, %v24, %v1
+; CHECK-NEXT:    vfchesb %v0, %v0, %v24
+; CHECK-NEXT:    vnc %v24, %v24, %v0
 ; CHECK-NEXT:    br %r14
   %cmp = fcmp ugt <4 x float> %val,  zeroinitializer
   %ret = select <4 x i1> %cmp, <4 x float> %val, <4 x float> zeroinitializer
@@ -88,8 +88,8 @@ define <4 x float> @f8(<4 x float> %val) {
 ; CHECK-LABEL: f8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgbm %v0, 0
-; CHECK-NEXT:    vfchesb %v1, %v24, %v0
-; CHECK-NEXT:    vsel %v24, %v0, %v24, %v1
+; CHECK-NEXT:    vfchesb %v0, %v24, %v0
+; CHECK-NEXT:    vnc %v24, %v24, %v0
 ; CHECK-NEXT:    br %r14
   %cmp = fcmp ult <4 x float> %val,  zeroinitializer
   %ret = select <4 x i1> %cmp, <4 x float> %val, <4 x float> zeroinitializer
diff --git a/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll b/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll
index fc52de6c387d1..a29646b8bcc61 100644
--- a/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll
@@ -42,6 +42,7 @@ define void @foo() {
 ; CHECK: DS 0B
 ; CHECK-LABEL: L#.str.1 DS 0H
 ; CHECK: DC XL6'576F726C6400'
+; CHECK: END
 entry:
   %0 = load ptr, ptr @Greeting, align 8
   call void (ptr, ...) @outs(ptr noundef %0, ptr noundef @.str.1)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll b/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll
index 726237eb27f2d..024de2b36667b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll
@@ -282,12 +282,12 @@ define arm_aapcs_vfpcc <2 x i64> @not_signbit_mask_v2i64(<2 x i64> %a, <2 x i64>
 ; CHECK-NEXT:    vmov r1, s1
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    vmov.i32 q2, #0x0
-; CHECK-NEXT:    cmp.w r1, #-1
-; CHECK-NEXT:    csetm r1, gt
+; CHECK-NEXT:    mvns r1, r1
+; CHECK-NEXT:    asrs r1, r1, #31
 ; CHECK-NEXT:    bfi r0, r1, #0, #8
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    cmp.w r1, #-1
-; CHECK-NEXT:    csetm r1, gt
+; CHECK-NEXT:    mvns r1, r1
+; CHECK-NEXT:    asrs r1, r1, #31
 ; CHECK-NEXT:    bfi r0, r1, #8, #8
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q1, q2
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir
index 5a4dc11cd328b..d5fbf41d39031 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
+# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
 
 --- |
   target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/CodeGen/WebAssembly/eh-lsda.ll b/llvm/test/CodeGen/WebAssembly/eh-lsda.ll
index ce55f7620e14a..d5b28b279d419 100644
--- a/llvm/test/CodeGen/WebAssembly/eh-lsda.ll
+++ b/llvm/test/CodeGen/WebAssembly/eh-lsda.ll
@@ -66,9 +66,9 @@ try.cont:                                         ; preds = %entry, %catch.start
 
 ; CHECK-LABEL: test1:
 ; In static linking, we load GCC_except_table as a constant directly.
-; NOPIC:      i[[PTR]].const  $push[[CONTEXT:.*]]=, __wasm_lpad_context
+; NOPIC:      i[[PTR]].const  $push[[CONTEXT:.*]]=, {{[48]}}
 ; NOPIC-NEXT: i[[PTR]].const  $push[[EXCEPT_TABLE:.*]]=, GCC_except_table1
-; NOPIC-NEXT: i[[PTR]].store  {{[48]}}($pop[[CONTEXT]]), $pop[[EXCEPT_TABLE]]
+; NOPIC-NEXT: i[[PTR]].store  __wasm_lpad_context($pop[[CONTEXT]]), $pop[[EXCEPT_TABLE]]
 
 ; In case of PIC, we make GCC_except_table symbols a relative on based on
 ; __memory_base.
diff --git a/llvm/test/CodeGen/WebAssembly/exception-legacy.ll b/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
index b4ffd185e3ca8..149e443903a88 100644
--- a/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
@@ -34,7 +34,7 @@ define void @throw(ptr %p) {
 ; CHECK:       call      foo
 ; CHECK:     catch     $[[EXN:[0-9]+]]=, __cpp_exception
 ; CHECK:       global.set  __stack_pointer
-; CHECK:       i32.{{store|const}} {{.*}} __wasm_lpad_context
+; CHECK:       i32.store __wasm_lpad_context
 ; CHECK:       call       $drop=, _Unwind_CallPersonality, $[[EXN]]
 ; CHECK:       block
 ; CHECK:         br_if     0
diff --git a/llvm/test/CodeGen/WebAssembly/exception-legacy.mir b/llvm/test/CodeGen/WebAssembly/exception-legacy.mir
index 9273ceeadd0e7..95929af4f4cea 100644
--- a/llvm/test/CodeGen/WebAssembly/exception-legacy.mir
+++ b/llvm/test/CodeGen/WebAssembly/exception-legacy.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-late-eh-prepare -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
+# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-late-eh-prepare -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
 
 --- |
   target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/CodeGen/WebAssembly/function-info.mir b/llvm/test/CodeGen/WebAssembly/function-info.mir
index d241d59b67a39..ae7c15efe872f 100644
--- a/llvm/test/CodeGen/WebAssembly/function-info.mir
+++ b/llvm/test/CodeGen/WebAssembly/function-info.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -exception-model=wasm -mattr=+exception-handling,+multivalue,+simd128 -run-pass wasm-cfg-sort -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
+# RUN: llc -mtriple=wasm32-unknown-unknown -exception-model=wasm -wasm-enable-sjlj -mattr=+exception-handling,+multivalue,+simd128 -run-pass wasm-cfg-sort -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
 
 --- |
   target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/CodeGen/WebAssembly/inline-asm.ll b/llvm/test/CodeGen/WebAssembly/inline-asm.ll
index 4462cfb7aa0c4..c378fd953a555 100644
--- a/llvm/test/CodeGen/WebAssembly/inline-asm.ll
+++ b/llvm/test/CodeGen/WebAssembly/inline-asm.ll
@@ -129,7 +129,18 @@ entry:
   ret i32 %ret
 }
 
+; CHECK-LABEL: v128_load
+; CHECK: local.get 0
+; CHECK-NEXT: v128.load 0
+; CHECK-NEXT: local.set 1
+define <4 x i32> @v128_load(ptr %v) #1 {
+entry:
+  %0 = tail call <4 x i32> asm "local.get $1\0Av128.load 0\0Alocal.set $0", "=r,r"(ptr %v)
+  ret <4 x i32> %0
+}
+
 attributes #0 = { nounwind }
+attributes #1 = { "target-features"="+simd128" }
 
 !0 = !{i32 47}
 !1 = !{i32 145}
diff --git a/llvm/test/CodeGen/WebAssembly/llvm.sincos.ll b/llvm/test/CodeGen/WebAssembly/llvm.sincos.ll
new file mode 100644
index 0000000000000..3c10b09525573
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/llvm.sincos.ll
@@ -0,0 +1,609 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=wasm32-unknown-unknown < %s | FileCheck -check-prefixes=CHECK,WASM32 %s
+; RUN: llc -mtriple=wasm64-unknown-unknown < %s | FileCheck -check-prefixes=CHECK,WASM64 %s
+
+define { half, half } @test_sincos_f16(half %a) #0 {
+; WASM32-LABEL: test_sincos_f16:
+; WASM32:         .functype test_sincos_f16 (i32, f32) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    call __extendhfsf2
+; WASM32-NEXT:    local.tee 1
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    i32.store16 2
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    i32.store16 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_f16:
+; WASM64:         .functype test_sincos_f16 (i64, f32) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    call __extendhfsf2
+; WASM64-NEXT:    local.tee 1
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    i32.store16 2
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    i32.store16 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  ret { half, half } %result
+}
+
+define half @test_sincos_f16_only_use_sin(half %a) #0 {
+; CHECK-LABEL: test_sincos_f16_only_use_sin:
+; CHECK:         .functype test_sincos_f16_only_use_sin (f32) -> (f32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    call __truncsfhf2
+; CHECK-NEXT:    call __extendhfsf2
+; CHECK-NEXT:    call sinf
+; CHECK-NEXT:    # fallthrough-return
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.0 = extractvalue { half, half } %result, 0
+  ret half %result.0
+}
+
+define half @test_sincos_f16_only_use_cos(half %a) #0 {
+; CHECK-LABEL: test_sincos_f16_only_use_cos:
+; CHECK:         .functype test_sincos_f16_only_use_cos (f32) -> (f32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    call __truncsfhf2
+; CHECK-NEXT:    call __extendhfsf2
+; CHECK-NEXT:    call cosf
+; CHECK-NEXT:    # fallthrough-return
+  %result = call { half, half } @llvm.sincos.f16(half %a)
+  %result.1 = extractvalue { half, half } %result, 1
+  ret half %result.1
+}
+
+define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) #0 {
+; WASM32-LABEL: test_sincos_v2f16:
+; WASM32:         .functype test_sincos_v2f16 (i32, f32, f32) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    call __extendhfsf2
+; WASM32-NEXT:    local.tee 2
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    i32.store16 6
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    call __extendhfsf2
+; WASM32-NEXT:    local.tee 1
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    i32.store16 4
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    i32.store16 2
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    call __truncsfhf2
+; WASM32-NEXT:    i32.store16 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_v2f16:
+; WASM64:         .functype test_sincos_v2f16 (i64, f32, f32) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    call __extendhfsf2
+; WASM64-NEXT:    local.tee 2
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    i32.store16 6
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    call __extendhfsf2
+; WASM64-NEXT:    local.tee 1
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    i32.store16 4
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    i32.store16 2
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    call __truncsfhf2
+; WASM64-NEXT:    i32.store16 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a)
+  ret { <2 x half>, <2 x half> } %result
+}
+
+define { float, float } @test_sincos_f32(float %a) #0 {
+; WASM32-LABEL: test_sincos_f32:
+; WASM32:         .functype test_sincos_f32 (i32, f32) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    f32.store 4
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    f32.store 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_f32:
+; WASM64:         .functype test_sincos_f32 (i64, f32) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    f32.store 4
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    f32.store 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { float, float } @llvm.sincos.f32(float %a)
+  ret { float, float } %result
+}
+
+define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 {
+; WASM32-LABEL: test_sincos_v2f32:
+; WASM32:         .functype test_sincos_v2f32 (i32, f32, f32) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    f32.store 12
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    f32.store 8
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    f32.store 4
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    f32.store 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_v2f32:
+; WASM64:         .functype test_sincos_v2f32 (i64, f32, f32) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    f32.store 12
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    f32.store 8
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    f32.store 4
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    f32.store 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
+  ret { <2 x float>, <2 x float> } %result
+}
+
+define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 {
+; WASM32-LABEL: test_sincos_v3f32:
+; WASM32:         .functype test_sincos_v3f32 (i32, f32, f32, f32) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    f32.store 24
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    f32.store 8
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    local.set 3
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call cosf
+; WASM32-NEXT:    i32.reinterpret_f32
+; WASM32-NEXT:    i64.extend_i32_u
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i32.reinterpret_f32
+; WASM32-NEXT:    i64.extend_i32_u
+; WASM32-NEXT:    i64.const 32
+; WASM32-NEXT:    i64.shl
+; WASM32-NEXT:    i64.or
+; WASM32-NEXT:    i64.store 16
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    local.set 3
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sinf
+; WASM32-NEXT:    i32.reinterpret_f32
+; WASM32-NEXT:    i64.extend_i32_u
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i32.reinterpret_f32
+; WASM32-NEXT:    i64.extend_i32_u
+; WASM32-NEXT:    i64.const 32
+; WASM32-NEXT:    i64.shl
+; WASM32-NEXT:    i64.or
+; WASM32-NEXT:    i64.store 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_v3f32:
+; WASM64:         .functype test_sincos_v3f32 (i64, f32, f32, f32) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    f32.store 24
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    f32.store 8
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    local.set 3
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call cosf
+; WASM64-NEXT:    i32.reinterpret_f32
+; WASM64-NEXT:    i64.extend_i32_u
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i32.reinterpret_f32
+; WASM64-NEXT:    i64.extend_i32_u
+; WASM64-NEXT:    i64.const 32
+; WASM64-NEXT:    i64.shl
+; WASM64-NEXT:    i64.or
+; WASM64-NEXT:    i64.store 16
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    local.set 3
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sinf
+; WASM64-NEXT:    i32.reinterpret_f32
+; WASM64-NEXT:    i64.extend_i32_u
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i32.reinterpret_f32
+; WASM64-NEXT:    i64.extend_i32_u
+; WASM64-NEXT:    i64.const 32
+; WASM64-NEXT:    i64.shl
+; WASM64-NEXT:    i64.or
+; WASM64-NEXT:    i64.store 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
+  ret { <3 x float>, <3 x float> } %result
+}
+
+define { double, double } @test_sincos_f64(double %a) #0 {
+; WASM32-LABEL: test_sincos_f64:
+; WASM32:         .functype test_sincos_f64 (i32, f64) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call cos
+; WASM32-NEXT:    f64.store 8
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sin
+; WASM32-NEXT:    f64.store 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_f64:
+; WASM64:         .functype test_sincos_f64 (i64, f64) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call cos
+; WASM64-NEXT:    f64.store 8
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sin
+; WASM64-NEXT:    f64.store 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { double, double } @llvm.sincos.f64(double %a)
+  ret { double, double } %result
+}
+
+define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 {
+; WASM32-LABEL: test_sincos_v2f64:
+; WASM32:         .functype test_sincos_v2f64 (i32, f64, f64) -> ()
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call cos
+; WASM32-NEXT:    f64.store 24
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call cos
+; WASM32-NEXT:    f64.store 16
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call sin
+; WASM32-NEXT:    f64.store 8
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    call sin
+; WASM32-NEXT:    f64.store 0
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_v2f64:
+; WASM64:         .functype test_sincos_v2f64 (i64, f64, f64) -> ()
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call cos
+; WASM64-NEXT:    f64.store 24
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call cos
+; WASM64-NEXT:    f64.store 16
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call sin
+; WASM64-NEXT:    f64.store 8
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    call sin
+; WASM64-NEXT:    f64.store 0
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
+  ret { <2 x double>, <2 x double> } %result
+}
+
+define { fp128, fp128 } @test_sincos_f128(fp128 %a) #0 {
+; WASM32-LABEL: test_sincos_f128:
+; WASM32:         .functype test_sincos_f128 (i32, i64, i64) -> ()
+; WASM32-NEXT:    .local i32
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    global.get __stack_pointer
+; WASM32-NEXT:    i32.const 32
+; WASM32-NEXT:    i32.sub
+; WASM32-NEXT:    local.tee 3
+; WASM32-NEXT:    global.set __stack_pointer
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call cosl
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i32.const 16
+; WASM32-NEXT:    i32.add
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call sinl
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i64.load 8
+; WASM32-NEXT:    i64.store 24
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i64.load 0
+; WASM32-NEXT:    i64.store 16
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i64.load 24
+; WASM32-NEXT:    i64.store 8
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i64.load 16
+; WASM32-NEXT:    i64.store 0
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    i32.const 32
+; WASM32-NEXT:    i32.add
+; WASM32-NEXT:    global.set __stack_pointer
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_f128:
+; WASM64:         .functype test_sincos_f128 (i64, i64, i64) -> ()
+; WASM64-NEXT:    .local i64
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    global.get __stack_pointer
+; WASM64-NEXT:    i64.const 32
+; WASM64-NEXT:    i64.sub
+; WASM64-NEXT:    local.tee 3
+; WASM64-NEXT:    global.set __stack_pointer
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call cosl
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i64.const 16
+; WASM64-NEXT:    i64.add
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call sinl
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i64.load 8
+; WASM64-NEXT:    i64.store 24
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i64.load 0
+; WASM64-NEXT:    i64.store 16
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i64.load 24
+; WASM64-NEXT:    i64.store 8
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i64.load 16
+; WASM64-NEXT:    i64.store 0
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    i64.const 32
+; WASM64-NEXT:    i64.add
+; WASM64-NEXT:    global.set __stack_pointer
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { fp128, fp128 } @llvm.sincos.f128(fp128 %a)
+  ret { fp128, fp128 } %result
+}
+
+define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
+; WASM32-LABEL: test_sincos_v2f128:
+; WASM32:         .functype test_sincos_v2f128 (i32, i64, i64, i64, i64) -> ()
+; WASM32-NEXT:    .local i32
+; WASM32-NEXT:  # %bb.0:
+; WASM32-NEXT:    global.get __stack_pointer
+; WASM32-NEXT:    i32.const 64
+; WASM32-NEXT:    i32.sub
+; WASM32-NEXT:    local.tee 5
+; WASM32-NEXT:    global.set __stack_pointer
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i32.const 32
+; WASM32-NEXT:    i32.add
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    local.get 4
+; WASM32-NEXT:    call cosl
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call cosl
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i32.const 48
+; WASM32-NEXT:    i32.add
+; WASM32-NEXT:    local.get 3
+; WASM32-NEXT:    local.get 4
+; WASM32-NEXT:    call sinl
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i32.const 16
+; WASM32-NEXT:    i32.add
+; WASM32-NEXT:    local.get 1
+; WASM32-NEXT:    local.get 2
+; WASM32-NEXT:    call sinl
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 40
+; WASM32-NEXT:    i64.store 56
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 32
+; WASM32-NEXT:    i64.store 48
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 8
+; WASM32-NEXT:    i64.store 40
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 0
+; WASM32-NEXT:    i64.store 32
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 56
+; WASM32-NEXT:    i64.store 24
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 48
+; WASM32-NEXT:    i64.store 16
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 24
+; WASM32-NEXT:    i64.store 8
+; WASM32-NEXT:    local.get 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i64.load 16
+; WASM32-NEXT:    i64.store 0
+; WASM32-NEXT:    local.get 5
+; WASM32-NEXT:    i32.const 64
+; WASM32-NEXT:    i32.add
+; WASM32-NEXT:    global.set __stack_pointer
+; WASM32-NEXT:    # fallthrough-return
+;
+; WASM64-LABEL: test_sincos_v2f128:
+; WASM64:         .functype test_sincos_v2f128 (i64, i64, i64, i64, i64) -> ()
+; WASM64-NEXT:    .local i64
+; WASM64-NEXT:  # %bb.0:
+; WASM64-NEXT:    global.get __stack_pointer
+; WASM64-NEXT:    i64.const 64
+; WASM64-NEXT:    i64.sub
+; WASM64-NEXT:    local.tee 5
+; WASM64-NEXT:    global.set __stack_pointer
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.const 32
+; WASM64-NEXT:    i64.add
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    local.get 4
+; WASM64-NEXT:    call cosl
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call cosl
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.const 48
+; WASM64-NEXT:    i64.add
+; WASM64-NEXT:    local.get 3
+; WASM64-NEXT:    local.get 4
+; WASM64-NEXT:    call sinl
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.const 16
+; WASM64-NEXT:    i64.add
+; WASM64-NEXT:    local.get 1
+; WASM64-NEXT:    local.get 2
+; WASM64-NEXT:    call sinl
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 40
+; WASM64-NEXT:    i64.store 56
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 32
+; WASM64-NEXT:    i64.store 48
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 8
+; WASM64-NEXT:    i64.store 40
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 0
+; WASM64-NEXT:    i64.store 32
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 56
+; WASM64-NEXT:    i64.store 24
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 48
+; WASM64-NEXT:    i64.store 16
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 24
+; WASM64-NEXT:    i64.store 8
+; WASM64-NEXT:    local.get 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.load 16
+; WASM64-NEXT:    i64.store 0
+; WASM64-NEXT:    local.get 5
+; WASM64-NEXT:    i64.const 64
+; WASM64-NEXT:    i64.add
+; WASM64-NEXT:    global.set __stack_pointer
+; WASM64-NEXT:    # fallthrough-return
+  %result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a)
+  ret { <2 x fp128>, <2 x fp128> } %result
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/WebAssembly/offset.ll b/llvm/test/CodeGen/WebAssembly/offset.ll
index 130508424f630..109607c914271 100644
--- a/llvm/test/CodeGen/WebAssembly/offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/offset.ll
@@ -118,6 +118,16 @@ define i32 @load_i32_from_global_address() {
   ret i32 %t
 }
 
+define i32 @load_i32_global_with_folded_gep_offset_nonconst_nuw(i32 %idx) {
+; CHECK-LABEL: load_i32_global_with_folded_gep_offset_nonconst_nuw:
+; CHECK: i32.const $push0=, 2
+; CHECK: i32.shl $push1=, $0, $pop0
+; CHECK: i32.load $push2=, gv($pop1)
+  %s = getelementptr nuw i32, ptr @gv, i32 %idx
+  %t = load i32, ptr %s
+  ret i32 %t
+}
+
 ;===----------------------------------------------------------------------------
 ; Loads: 64-bit
 ;===----------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll b/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll
index ca160c091b229..c2a43a839b1e5 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll
@@ -177,236 +177,20 @@ define i32 @bitmask_v32i8(<32 x i8> %v) {
 ; CHECK:         .functype bitmask_v32i8 (v128, v128) -> (i32)
 ; CHECK-NEXT:    .local v128
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get __stack_pointer
 ; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.sub
-; CHECK-NEXT:    drop
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.const 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 ; CHECK-NEXT:    local.tee 2
 ; CHECK-NEXT:    i8x16.eq
-; CHECK-NEXT:    local.tee 0
-; CHECK-NEXT:    i8x16.extract_lane_u 0
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 1
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 2
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 2
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 3
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 3
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 4
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 4
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 5
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 5
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 6
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 7
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 7
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 8
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 9
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 9
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 10
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 10
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 11
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 11
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 12
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 12
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 13
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 13
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 14
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 14
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 15
-; CHECK-NEXT:    i32.const 15
+; CHECK-NEXT:    i8x16.bitmask
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.const 65535
-; CHECK-NEXT:    i32.and
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i8x16.eq
-; CHECK-NEXT:    local.tee 0
-; CHECK-NEXT:    i8x16.extract_lane_u 15
-; CHECK-NEXT:    i32.const 31
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 14
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 30
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 13
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 29
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 12
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 28
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 11
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 27
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 10
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 26
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 9
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 25
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 8
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 7
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 23
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 6
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 22
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 5
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 21
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 4
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 20
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 3
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 19
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 2
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 18
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 1
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 17
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 0
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
+; CHECK-NEXT:    i8x16.bitmask
+; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    # fallthrough-return
   %cmp = icmp eq <32 x i8> %v, zeroinitializer
   %bitmask = bitcast <32 x i1> %cmp to i32
diff --git a/llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll b/llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll
new file mode 100644
index 0000000000000..e497edc08c4eb
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll
@@ -0,0 +1,596 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -O3 -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+
+target triple = "wasm64"
+
+
+define i16 @legal_bitcast_v16i8(<16 x i8> %x) {
+; CHECK-LABEL: legal_bitcast_v16i8:
+; CHECK:         .functype legal_bitcast_v16i8 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.const $push0=, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16
+; CHECK-NEXT:    i8x16.eq $push1=, $0, $pop0
+; CHECK-NEXT:    i8x16.bitmask $push2=, $pop1
+; CHECK-NEXT:    return $pop2
+    %z = icmp eq <16 x i8> %x, splat (i8 16)
+    %res = bitcast <16 x i1> %z to i16
+    ret i16 %res
+}
+
+define i32 @optimize_illegal_bitcast_v32i8(<32 x i8> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v32i8:
+; CHECK:         .functype optimize_illegal_bitcast_v32i8 (v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const $push2=, 16
+; CHECK-NEXT:    v128.const $push10=, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32
+; CHECK-NEXT:    local.tee $push9=, $2=, $pop10
+; CHECK-NEXT:    i8x16.eq $push0=, $0, $pop9
+; CHECK-NEXT:    i8x16.bitmask $push1=, $pop0
+; CHECK-NEXT:    i32.const $push8=, 16
+; CHECK-NEXT:    i32.add $push3=, $pop1, $pop8
+; CHECK-NEXT:    i32.shl $push4=, $pop2, $pop3
+; CHECK-NEXT:    i8x16.eq $push5=, $1, $2
+; CHECK-NEXT:    i8x16.bitmask $push6=, $pop5
+; CHECK-NEXT:    i32.add $push7=, $pop4, $pop6
+; CHECK-NEXT:    return $pop7
+    %z = icmp eq <32 x i8> %x, splat (i8 32)
+    %res = bitcast <32 x i1> %z to i32
+    ret i32 %res
+}
+
+
+define i32 @optimize_illegal_bitcast_v32i8_const_step_vec(<32 x i8> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v32i8_const_step_vec:
+; CHECK:         .functype optimize_illegal_bitcast_v32i8_const_step_vec (v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    v128.const $push0=, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+; CHECK-NEXT:    i8x16.eq $push1=, $0, $pop0
+; CHECK-NEXT:    i8x16.bitmask $push2=, $pop1
+; CHECK-NEXT:    i32.const $push10=, 16
+; CHECK-NEXT:    i32.add $push4=, $pop2, $pop10
+; CHECK-NEXT:    i32.shl $push5=, $pop3, $pop4
+; CHECK-NEXT:    v128.const $push6=, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32
+; CHECK-NEXT:    i8x16.eq $push7=, $1, $pop6
+; CHECK-NEXT:    i8x16.bitmask $push8=, $pop7
+; CHECK-NEXT:    i32.add $push9=, $pop5, $pop8
+; CHECK-NEXT:    return $pop9
+    %const_step_vec =  add <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8,
+                              i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16,
+                              i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24,
+                              i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32>, zeroinitializer
+    %z = icmp eq <32 x i8> %x, %const_step_vec
+    %res = bitcast <32 x i1> %z to i32
+    ret i32 %res
+}
+
+
+define i32 @optimize_illegal_bitcast_v32i8_non_const_vec(<32 x i8> %x, <32 x i8> %y) {
+; CHECK-LABEL: optimize_illegal_bitcast_v32i8_non_const_vec:
+; CHECK:         .functype optimize_illegal_bitcast_v32i8_non_const_vec (v128, v128, v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const $push2=, 16
+; CHECK-NEXT:    i8x16.eq $push0=, $0, $2
+; CHECK-NEXT:    i8x16.bitmask $push1=, $pop0
+; CHECK-NEXT:    i32.const $push8=, 16
+; CHECK-NEXT:    i32.add $push3=, $pop1, $pop8
+; CHECK-NEXT:    i32.shl $push4=, $pop2, $pop3
+; CHECK-NEXT:    i8x16.eq $push5=, $1, $3
+; CHECK-NEXT:    i8x16.bitmask $push6=, $pop5
+; CHECK-NEXT:    i32.add $push7=, $pop4, $pop6
+; CHECK-NEXT:    return $pop7
+    %z = icmp eq <32 x i8> %x, %y
+    %res = bitcast <32 x i1> %z to i32
+    ret i32 %res
+}
+
+
+
+
+define i64 @optimize_illegal_bitcast_v64i8(<64 x i8> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v64i8:
+; CHECK:         .functype optimize_illegal_bitcast_v64i8 (v128, v128, v128, v128) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i64.const $push3=, 16
+; CHECK-NEXT:    i64.const $push24=, 16
+; CHECK-NEXT:    i64.const $push23=, 16
+; CHECK-NEXT:    v128.const $push22=, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64
+; CHECK-NEXT:    local.tee $push21=, $4=, $pop22
+; CHECK-NEXT:    i8x16.eq $push0=, $0, $pop21
+; CHECK-NEXT:    i8x16.bitmask $push1=, $pop0
+; CHECK-NEXT:    i64.extend_i32_u $push2=, $pop1
+; CHECK-NEXT:    i64.const $push20=, 16
+; CHECK-NEXT:    i64.add $push4=, $pop2, $pop20
+; CHECK-NEXT:    i64.shl $push5=, $pop23, $pop4
+; CHECK-NEXT:    i8x16.eq $push6=, $1, $4
+; CHECK-NEXT:    i8x16.bitmask $push7=, $pop6
+; CHECK-NEXT:    i64.extend_i32_u $push8=, $pop7
+; CHECK-NEXT:    i64.add $push9=, $pop5, $pop8
+; CHECK-NEXT:    i64.shl $push10=, $pop24, $pop9
+; CHECK-NEXT:    i8x16.eq $push11=, $2, $4
+; CHECK-NEXT:    i8x16.bitmask $push12=, $pop11
+; CHECK-NEXT:    i64.extend_i32_u $push13=, $pop12
+; CHECK-NEXT:    i64.add $push14=, $pop10, $pop13
+; CHECK-NEXT:    i64.shl $push15=, $pop3, $pop14
+; CHECK-NEXT:    i8x16.eq $push16=, $3, $4
+; CHECK-NEXT:    i8x16.bitmask $push17=, $pop16
+; CHECK-NEXT:    i64.extend_i32_u $push18=, $pop17
+; CHECK-NEXT:    i64.add $push19=, $pop15, $pop18
+; CHECK-NEXT:    return $pop19
+    %z = icmp eq <64 x i8> %x, splat (i8 64)
+    %res = bitcast <64 x i1> %z to i64
+    ret i64 %res
+}
+
+define i64 @optimize_illegal_bitcast_v64i4(<64 x i4> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v64i4:
+; CHECK:         .functype optimize_illegal_bitcast_v64i4 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push355=, __stack_pointer
+; CHECK-NEXT:    i64.const $push356=, 16
+; CHECK-NEXT:    i64.sub $drop=, $pop355, $pop356
+; CHECK-NEXT:    i8x16.splat $push273=, $0
+; CHECK-NEXT:    i8x16.replace_lane $push274=, $pop273, 1, $1
+; CHECK-NEXT:    i8x16.replace_lane $push275=, $pop274, 2, $2
+; CHECK-NEXT:    i8x16.replace_lane $push276=, $pop275, 3, $3
+; CHECK-NEXT:    i8x16.replace_lane $push277=, $pop276, 4, $4
+; CHECK-NEXT:    i8x16.replace_lane $push278=, $pop277, 5, $5
+; CHECK-NEXT:    i8x16.replace_lane $push279=, $pop278, 6, $6
+; CHECK-NEXT:    i8x16.replace_lane $push280=, $pop279, 7, $7
+; CHECK-NEXT:    i8x16.replace_lane $push281=, $pop280, 8, $8
+; CHECK-NEXT:    i8x16.replace_lane $push282=, $pop281, 9, $9
+; CHECK-NEXT:    i8x16.replace_lane $push283=, $pop282, 10, $10
+; CHECK-NEXT:    i8x16.replace_lane $push284=, $pop283, 11, $11
+; CHECK-NEXT:    i8x16.replace_lane $push285=, $pop284, 12, $12
+; CHECK-NEXT:    i8x16.replace_lane $push286=, $pop285, 13, $13
+; CHECK-NEXT:    i8x16.replace_lane $push287=, $pop286, 14, $14
+; CHECK-NEXT:    i8x16.replace_lane $push288=, $pop287, 15, $15
+; CHECK-NEXT:    v128.const $push460=, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15
+; CHECK-NEXT:    local.tee $push459=, $64=, $pop460
+; CHECK-NEXT:    v128.and $push289=, $pop288, $pop459
+; CHECK-NEXT:    v128.const $push458=, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT:    local.tee $push457=, $65=, $pop458
+; CHECK-NEXT:    i8x16.eq $push456=, $pop289, $pop457
+; CHECK-NEXT:    local.tee $push455=, $66=, $pop456
+; CHECK-NEXT:    i8x16.extract_lane_u $push290=, $pop455, 0
+; CHECK-NEXT:    i32.const $push18=, 1
+; CHECK-NEXT:    i32.and $push291=, $pop290, $pop18
+; CHECK-NEXT:    i8x16.extract_lane_u $push292=, $66, 1
+; CHECK-NEXT:    i32.const $push454=, 1
+; CHECK-NEXT:    i32.and $push293=, $pop292, $pop454
+; CHECK-NEXT:    i32.const $push453=, 1
+; CHECK-NEXT:    i32.shl $push294=, $pop293, $pop453
+; CHECK-NEXT:    i32.or $push295=, $pop291, $pop294
+; CHECK-NEXT:    i8x16.extract_lane_u $push296=, $66, 2
+; CHECK-NEXT:    i32.const $push452=, 1
+; CHECK-NEXT:    i32.and $push297=, $pop296, $pop452
+; CHECK-NEXT:    i32.const $push121=, 2
+; CHECK-NEXT:    i32.shl $push298=, $pop297, $pop121
+; CHECK-NEXT:    i32.or $push299=, $pop295, $pop298
+; CHECK-NEXT:    i8x16.extract_lane_u $push300=, $66, 3
+; CHECK-NEXT:    i32.const $push451=, 1
+; CHECK-NEXT:    i32.and $push301=, $pop300, $pop451
+; CHECK-NEXT:    i32.const $push126=, 3
+; CHECK-NEXT:    i32.shl $push302=, $pop301, $pop126
+; CHECK-NEXT:    i32.or $push303=, $pop299, $pop302
+; CHECK-NEXT:    i8x16.extract_lane_u $push304=, $66, 4
+; CHECK-NEXT:    i32.const $push450=, 1
+; CHECK-NEXT:    i32.and $push305=, $pop304, $pop450
+; CHECK-NEXT:    i32.const $push131=, 4
+; CHECK-NEXT:    i32.shl $push306=, $pop305, $pop131
+; CHECK-NEXT:    i32.or $push307=, $pop303, $pop306
+; CHECK-NEXT:    i8x16.extract_lane_u $push308=, $66, 5
+; CHECK-NEXT:    i32.const $push449=, 1
+; CHECK-NEXT:    i32.and $push309=, $pop308, $pop449
+; CHECK-NEXT:    i32.const $push136=, 5
+; CHECK-NEXT:    i32.shl $push310=, $pop309, $pop136
+; CHECK-NEXT:    i32.or $push311=, $pop307, $pop310
+; CHECK-NEXT:    i8x16.extract_lane_u $push312=, $66, 6
+; CHECK-NEXT:    i32.const $push448=, 1
+; CHECK-NEXT:    i32.and $push313=, $pop312, $pop448
+; CHECK-NEXT:    i32.const $push141=, 6
+; CHECK-NEXT:    i32.shl $push314=, $pop313, $pop141
+; CHECK-NEXT:    i32.or $push315=, $pop311, $pop314
+; CHECK-NEXT:    i8x16.extract_lane_u $push316=, $66, 7
+; CHECK-NEXT:    i32.const $push447=, 1
+; CHECK-NEXT:    i32.and $push317=, $pop316, $pop447
+; CHECK-NEXT:    i32.const $push146=, 7
+; CHECK-NEXT:    i32.shl $push318=, $pop317, $pop146
+; CHECK-NEXT:    i32.or $push319=, $pop315, $pop318
+; CHECK-NEXT:    i8x16.extract_lane_u $push320=, $66, 8
+; CHECK-NEXT:    i32.const $push446=, 1
+; CHECK-NEXT:    i32.and $push321=, $pop320, $pop446
+; CHECK-NEXT:    i32.const $push151=, 8
+; CHECK-NEXT:    i32.shl $push322=, $pop321, $pop151
+; CHECK-NEXT:    i32.or $push323=, $pop319, $pop322
+; CHECK-NEXT:    i8x16.extract_lane_u $push324=, $66, 9
+; CHECK-NEXT:    i32.const $push445=, 1
+; CHECK-NEXT:    i32.and $push325=, $pop324, $pop445
+; CHECK-NEXT:    i32.const $push156=, 9
+; CHECK-NEXT:    i32.shl $push326=, $pop325, $pop156
+; CHECK-NEXT:    i32.or $push327=, $pop323, $pop326
+; CHECK-NEXT:    i8x16.extract_lane_u $push328=, $66, 10
+; CHECK-NEXT:    i32.const $push444=, 1
+; CHECK-NEXT:    i32.and $push329=, $pop328, $pop444
+; CHECK-NEXT:    i32.const $push161=, 10
+; CHECK-NEXT:    i32.shl $push330=, $pop329, $pop161
+; CHECK-NEXT:    i32.or $push331=, $pop327, $pop330
+; CHECK-NEXT:    i8x16.extract_lane_u $push332=, $66, 11
+; CHECK-NEXT:    i32.const $push443=, 1
+; CHECK-NEXT:    i32.and $push333=, $pop332, $pop443
+; CHECK-NEXT:    i32.const $push166=, 11
+; CHECK-NEXT:    i32.shl $push334=, $pop333, $pop166
+; CHECK-NEXT:    i32.or $push335=, $pop331, $pop334
+; CHECK-NEXT:    i8x16.extract_lane_u $push336=, $66, 12
+; CHECK-NEXT:    i32.const $push442=, 1
+; CHECK-NEXT:    i32.and $push337=, $pop336, $pop442
+; CHECK-NEXT:    i32.const $push171=, 12
+; CHECK-NEXT:    i32.shl $push338=, $pop337, $pop171
+; CHECK-NEXT:    i32.or $push339=, $pop335, $pop338
+; CHECK-NEXT:    i8x16.extract_lane_u $push340=, $66, 13
+; CHECK-NEXT:    i32.const $push441=, 1
+; CHECK-NEXT:    i32.and $push341=, $pop340, $pop441
+; CHECK-NEXT:    i32.const $push176=, 13
+; CHECK-NEXT:    i32.shl $push342=, $pop341, $pop176
+; CHECK-NEXT:    i32.or $push343=, $pop339, $pop342
+; CHECK-NEXT:    i8x16.extract_lane_u $push344=, $66, 14
+; CHECK-NEXT:    i32.const $push440=, 1
+; CHECK-NEXT:    i32.and $push345=, $pop344, $pop440
+; CHECK-NEXT:    i32.const $push181=, 14
+; CHECK-NEXT:    i32.shl $push346=, $pop345, $pop181
+; CHECK-NEXT:    i32.or $push347=, $pop343, $pop346
+; CHECK-NEXT:    i8x16.extract_lane_u $push348=, $66, 15
+; CHECK-NEXT:    i32.const $push185=, 15
+; CHECK-NEXT:    i32.shl $push349=, $pop348, $pop185
+; CHECK-NEXT:    i32.or $push350=, $pop347, $pop349
+; CHECK-NEXT:    i32.const $push188=, 65535
+; CHECK-NEXT:    i32.and $push351=, $pop350, $pop188
+; CHECK-NEXT:    i8x16.splat $push194=, $16
+; CHECK-NEXT:    i8x16.replace_lane $push195=, $pop194, 1, $17
+; CHECK-NEXT:    i8x16.replace_lane $push196=, $pop195, 2, $18
+; CHECK-NEXT:    i8x16.replace_lane $push197=, $pop196, 3, $19
+; CHECK-NEXT:    i8x16.replace_lane $push198=, $pop197, 4, $20
+; CHECK-NEXT:    i8x16.replace_lane $push199=, $pop198, 5, $21
+; CHECK-NEXT:    i8x16.replace_lane $push200=, $pop199, 6, $22
+; CHECK-NEXT:    i8x16.replace_lane $push201=, $pop200, 7, $23
+; CHECK-NEXT:    i8x16.replace_lane $push202=, $pop201, 8, $24
+; CHECK-NEXT:    i8x16.replace_lane $push203=, $pop202, 9, $25
+; CHECK-NEXT:    i8x16.replace_lane $push204=, $pop203, 10, $26
+; CHECK-NEXT:    i8x16.replace_lane $push205=, $pop204, 11, $27
+; CHECK-NEXT:    i8x16.replace_lane $push206=, $pop205, 12, $28
+; CHECK-NEXT:    i8x16.replace_lane $push207=, $pop206, 13, $29
+; CHECK-NEXT:    i8x16.replace_lane $push208=, $pop207, 14, $30
+; CHECK-NEXT:    i8x16.replace_lane $push209=, $pop208, 15, $31
+; CHECK-NEXT:    v128.and $push210=, $pop209, $64
+; CHECK-NEXT:    i8x16.eq $push439=, $pop210, $65
+; CHECK-NEXT:    local.tee $push438=, $66=, $pop439
+; CHECK-NEXT:    i8x16.extract_lane_u $push270=, $pop438, 15
+; CHECK-NEXT:    i32.const $push93=, 31
+; CHECK-NEXT:    i32.shl $push271=, $pop270, $pop93
+; CHECK-NEXT:    i8x16.extract_lane_u $push266=, $66, 14
+; CHECK-NEXT:    i32.const $push437=, 1
+; CHECK-NEXT:    i32.and $push267=, $pop266, $pop437
+; CHECK-NEXT:    i32.const $push89=, 30
+; CHECK-NEXT:    i32.shl $push268=, $pop267, $pop89
+; CHECK-NEXT:    i8x16.extract_lane_u $push262=, $66, 13
+; CHECK-NEXT:    i32.const $push436=, 1
+; CHECK-NEXT:    i32.and $push263=, $pop262, $pop436
+; CHECK-NEXT:    i32.const $push84=, 29
+; CHECK-NEXT:    i32.shl $push264=, $pop263, $pop84
+; CHECK-NEXT:    i8x16.extract_lane_u $push258=, $66, 12
+; CHECK-NEXT:    i32.const $push435=, 1
+; CHECK-NEXT:    i32.and $push259=, $pop258, $pop435
+; CHECK-NEXT:    i32.const $push79=, 28
+; CHECK-NEXT:    i32.shl $push260=, $pop259, $pop79
+; CHECK-NEXT:    i8x16.extract_lane_u $push254=, $66, 11
+; CHECK-NEXT:    i32.const $push434=, 1
+; CHECK-NEXT:    i32.and $push255=, $pop254, $pop434
+; CHECK-NEXT:    i32.const $push74=, 27
+; CHECK-NEXT:    i32.shl $push256=, $pop255, $pop74
+; CHECK-NEXT:    i8x16.extract_lane_u $push250=, $66, 10
+; CHECK-NEXT:    i32.const $push433=, 1
+; CHECK-NEXT:    i32.and $push251=, $pop250, $pop433
+; CHECK-NEXT:    i32.const $push69=, 26
+; CHECK-NEXT:    i32.shl $push252=, $pop251, $pop69
+; CHECK-NEXT:    i8x16.extract_lane_u $push246=, $66, 9
+; CHECK-NEXT:    i32.const $push432=, 1
+; CHECK-NEXT:    i32.and $push247=, $pop246, $pop432
+; CHECK-NEXT:    i32.const $push64=, 25
+; CHECK-NEXT:    i32.shl $push248=, $pop247, $pop64
+; CHECK-NEXT:    i8x16.extract_lane_u $push242=, $66, 8
+; CHECK-NEXT:    i32.const $push431=, 1
+; CHECK-NEXT:    i32.and $push243=, $pop242, $pop431
+; CHECK-NEXT:    i32.const $push59=, 24
+; CHECK-NEXT:    i32.shl $push244=, $pop243, $pop59
+; CHECK-NEXT:    i8x16.extract_lane_u $push238=, $66, 7
+; CHECK-NEXT:    i32.const $push430=, 1
+; CHECK-NEXT:    i32.and $push239=, $pop238, $pop430
+; CHECK-NEXT:    i32.const $push54=, 23
+; CHECK-NEXT:    i32.shl $push240=, $pop239, $pop54
+; CHECK-NEXT:    i8x16.extract_lane_u $push234=, $66, 6
+; CHECK-NEXT:    i32.const $push429=, 1
+; CHECK-NEXT:    i32.and $push235=, $pop234, $pop429
+; CHECK-NEXT:    i32.const $push49=, 22
+; CHECK-NEXT:    i32.shl $push236=, $pop235, $pop49
+; CHECK-NEXT:    i8x16.extract_lane_u $push230=, $66, 5
+; CHECK-NEXT:    i32.const $push428=, 1
+; CHECK-NEXT:    i32.and $push231=, $pop230, $pop428
+; CHECK-NEXT:    i32.const $push44=, 21
+; CHECK-NEXT:    i32.shl $push232=, $pop231, $pop44
+; CHECK-NEXT:    i8x16.extract_lane_u $push226=, $66, 4
+; CHECK-NEXT:    i32.const $push427=, 1
+; CHECK-NEXT:    i32.and $push227=, $pop226, $pop427
+; CHECK-NEXT:    i32.const $push39=, 20
+; CHECK-NEXT:    i32.shl $push228=, $pop227, $pop39
+; CHECK-NEXT:    i8x16.extract_lane_u $push222=, $66, 3
+; CHECK-NEXT:    i32.const $push426=, 1
+; CHECK-NEXT:    i32.and $push223=, $pop222, $pop426
+; CHECK-NEXT:    i32.const $push34=, 19
+; CHECK-NEXT:    i32.shl $push224=, $pop223, $pop34
+; CHECK-NEXT:    i8x16.extract_lane_u $push218=, $66, 2
+; CHECK-NEXT:    i32.const $push425=, 1
+; CHECK-NEXT:    i32.and $push219=, $pop218, $pop425
+; CHECK-NEXT:    i32.const $push29=, 18
+; CHECK-NEXT:    i32.shl $push220=, $pop219, $pop29
+; CHECK-NEXT:    i8x16.extract_lane_u $push214=, $66, 1
+; CHECK-NEXT:    i32.const $push424=, 1
+; CHECK-NEXT:    i32.and $push215=, $pop214, $pop424
+; CHECK-NEXT:    i32.const $push24=, 17
+; CHECK-NEXT:    i32.shl $push216=, $pop215, $pop24
+; CHECK-NEXT:    i8x16.extract_lane_u $push211=, $66, 0
+; CHECK-NEXT:    i32.const $push423=, 1
+; CHECK-NEXT:    i32.and $push212=, $pop211, $pop423
+; CHECK-NEXT:    i32.const $push20=, 16
+; CHECK-NEXT:    i32.shl $push213=, $pop212, $pop20
+; CHECK-NEXT:    i32.or $push217=, $pop216, $pop213
+; CHECK-NEXT:    i32.or $push221=, $pop220, $pop217
+; CHECK-NEXT:    i32.or $push225=, $pop224, $pop221
+; CHECK-NEXT:    i32.or $push229=, $pop228, $pop225
+; CHECK-NEXT:    i32.or $push233=, $pop232, $pop229
+; CHECK-NEXT:    i32.or $push237=, $pop236, $pop233
+; CHECK-NEXT:    i32.or $push241=, $pop240, $pop237
+; CHECK-NEXT:    i32.or $push245=, $pop244, $pop241
+; CHECK-NEXT:    i32.or $push249=, $pop248, $pop245
+; CHECK-NEXT:    i32.or $push253=, $pop252, $pop249
+; CHECK-NEXT:    i32.or $push257=, $pop256, $pop253
+; CHECK-NEXT:    i32.or $push261=, $pop260, $pop257
+; CHECK-NEXT:    i32.or $push265=, $pop264, $pop261
+; CHECK-NEXT:    i32.or $push269=, $pop268, $pop265
+; CHECK-NEXT:    i32.or $push272=, $pop271, $pop269
+; CHECK-NEXT:    i32.or $push352=, $pop351, $pop272
+; CHECK-NEXT:    i64.extend_i32_u $push353=, $pop352
+; CHECK-NEXT:    i8x16.splat $push96=, $32
+; CHECK-NEXT:    i8x16.replace_lane $push97=, $pop96, 1, $33
+; CHECK-NEXT:    i8x16.replace_lane $push98=, $pop97, 2, $34
+; CHECK-NEXT:    i8x16.replace_lane $push99=, $pop98, 3, $35
+; CHECK-NEXT:    i8x16.replace_lane $push100=, $pop99, 4, $36
+; CHECK-NEXT:    i8x16.replace_lane $push101=, $pop100, 5, $37
+; CHECK-NEXT:    i8x16.replace_lane $push102=, $pop101, 6, $38
+; CHECK-NEXT:    i8x16.replace_lane $push103=, $pop102, 7, $39
+; CHECK-NEXT:    i8x16.replace_lane $push104=, $pop103, 8, $40
+; CHECK-NEXT:    i8x16.replace_lane $push105=, $pop104, 9, $41
+; CHECK-NEXT:    i8x16.replace_lane $push106=, $pop105, 10, $42
+; CHECK-NEXT:    i8x16.replace_lane $push107=, $pop106, 11, $43
+; CHECK-NEXT:    i8x16.replace_lane $push108=, $pop107, 12, $44
+; CHECK-NEXT:    i8x16.replace_lane $push109=, $pop108, 13, $45
+; CHECK-NEXT:    i8x16.replace_lane $push110=, $pop109, 14, $46
+; CHECK-NEXT:    i8x16.replace_lane $push111=, $pop110, 15, $47
+; CHECK-NEXT:    v128.and $push112=, $pop111, $64
+; CHECK-NEXT:    i8x16.eq $push422=, $pop112, $65
+; CHECK-NEXT:    local.tee $push421=, $66=, $pop422
+; CHECK-NEXT:    i8x16.extract_lane_u $push113=, $pop421, 0
+; CHECK-NEXT:    i32.const $push420=, 1
+; CHECK-NEXT:    i32.and $push114=, $pop113, $pop420
+; CHECK-NEXT:    i8x16.extract_lane_u $push115=, $66, 1
+; CHECK-NEXT:    i32.const $push419=, 1
+; CHECK-NEXT:    i32.and $push116=, $pop115, $pop419
+; CHECK-NEXT:    i32.const $push418=, 1
+; CHECK-NEXT:    i32.shl $push117=, $pop116, $pop418
+; CHECK-NEXT:    i32.or $push118=, $pop114, $pop117
+; CHECK-NEXT:    i8x16.extract_lane_u $push119=, $66, 2
+; CHECK-NEXT:    i32.const $push417=, 1
+; CHECK-NEXT:    i32.and $push120=, $pop119, $pop417
+; CHECK-NEXT:    i32.const $push416=, 2
+; CHECK-NEXT:    i32.shl $push122=, $pop120, $pop416
+; CHECK-NEXT:    i32.or $push123=, $pop118, $pop122
+; CHECK-NEXT:    i8x16.extract_lane_u $push124=, $66, 3
+; CHECK-NEXT:    i32.const $push415=, 1
+; CHECK-NEXT:    i32.and $push125=, $pop124, $pop415
+; CHECK-NEXT:    i32.const $push414=, 3
+; CHECK-NEXT:    i32.shl $push127=, $pop125, $pop414
+; CHECK-NEXT:    i32.or $push128=, $pop123, $pop127
+; CHECK-NEXT:    i8x16.extract_lane_u $push129=, $66, 4
+; CHECK-NEXT:    i32.const $push413=, 1
+; CHECK-NEXT:    i32.and $push130=, $pop129, $pop413
+; CHECK-NEXT:    i32.const $push412=, 4
+; CHECK-NEXT:    i32.shl $push132=, $pop130, $pop412
+; CHECK-NEXT:    i32.or $push133=, $pop128, $pop132
+; CHECK-NEXT:    i8x16.extract_lane_u $push134=, $66, 5
+; CHECK-NEXT:    i32.const $push411=, 1
+; CHECK-NEXT:    i32.and $push135=, $pop134, $pop411
+; CHECK-NEXT:    i32.const $push410=, 5
+; CHECK-NEXT:    i32.shl $push137=, $pop135, $pop410
+; CHECK-NEXT:    i32.or $push138=, $pop133, $pop137
+; CHECK-NEXT:    i8x16.extract_lane_u $push139=, $66, 6
+; CHECK-NEXT:    i32.const $push409=, 1
+; CHECK-NEXT:    i32.and $push140=, $pop139, $pop409
+; CHECK-NEXT:    i32.const $push408=, 6
+; CHECK-NEXT:    i32.shl $push142=, $pop140, $pop408
+; CHECK-NEXT:    i32.or $push143=, $pop138, $pop142
+; CHECK-NEXT:    i8x16.extract_lane_u $push144=, $66, 7
+; CHECK-NEXT:    i32.const $push407=, 1
+; CHECK-NEXT:    i32.and $push145=, $pop144, $pop407
+; CHECK-NEXT:    i32.const $push406=, 7
+; CHECK-NEXT:    i32.shl $push147=, $pop145, $pop406
+; CHECK-NEXT:    i32.or $push148=, $pop143, $pop147
+; CHECK-NEXT:    i8x16.extract_lane_u $push149=, $66, 8
+; CHECK-NEXT:    i32.const $push405=, 1
+; CHECK-NEXT:    i32.and $push150=, $pop149, $pop405
+; CHECK-NEXT:    i32.const $push404=, 8
+; CHECK-NEXT:    i32.shl $push152=, $pop150, $pop404
+; CHECK-NEXT:    i32.or $push153=, $pop148, $pop152
+; CHECK-NEXT:    i8x16.extract_lane_u $push154=, $66, 9
+; CHECK-NEXT:    i32.const $push403=, 1
+; CHECK-NEXT:    i32.and $push155=, $pop154, $pop403
+; CHECK-NEXT:    i32.const $push402=, 9
+; CHECK-NEXT:    i32.shl $push157=, $pop155, $pop402
+; CHECK-NEXT:    i32.or $push158=, $pop153, $pop157
+; CHECK-NEXT:    i8x16.extract_lane_u $push159=, $66, 10
+; CHECK-NEXT:    i32.const $push401=, 1
+; CHECK-NEXT:    i32.and $push160=, $pop159, $pop401
+; CHECK-NEXT:    i32.const $push400=, 10
+; CHECK-NEXT:    i32.shl $push162=, $pop160, $pop400
+; CHECK-NEXT:    i32.or $push163=, $pop158, $pop162
+; CHECK-NEXT:    i8x16.extract_lane_u $push164=, $66, 11
+; CHECK-NEXT:    i32.const $push399=, 1
+; CHECK-NEXT:    i32.and $push165=, $pop164, $pop399
+; CHECK-NEXT:    i32.const $push398=, 11
+; CHECK-NEXT:    i32.shl $push167=, $pop165, $pop398
+; CHECK-NEXT:    i32.or $push168=, $pop163, $pop167
+; CHECK-NEXT:    i8x16.extract_lane_u $push169=, $66, 12
+; CHECK-NEXT:    i32.const $push397=, 1
+; CHECK-NEXT:    i32.and $push170=, $pop169, $pop397
+; CHECK-NEXT:    i32.const $push396=, 12
+; CHECK-NEXT:    i32.shl $push172=, $pop170, $pop396
+; CHECK-NEXT:    i32.or $push173=, $pop168, $pop172
+; CHECK-NEXT:    i8x16.extract_lane_u $push174=, $66, 13
+; CHECK-NEXT:    i32.const $push395=, 1
+; CHECK-NEXT:    i32.and $push175=, $pop174, $pop395
+; CHECK-NEXT:    i32.const $push394=, 13
+; CHECK-NEXT:    i32.shl $push177=, $pop175, $pop394
+; CHECK-NEXT:    i32.or $push178=, $pop173, $pop177
+; CHECK-NEXT:    i8x16.extract_lane_u $push179=, $66, 14
+; CHECK-NEXT:    i32.const $push393=, 1
+; CHECK-NEXT:    i32.and $push180=, $pop179, $pop393
+; CHECK-NEXT:    i32.const $push392=, 14
+; CHECK-NEXT:    i32.shl $push182=, $pop180, $pop392
+; CHECK-NEXT:    i32.or $push183=, $pop178, $pop182
+; CHECK-NEXT:    i8x16.extract_lane_u $push184=, $66, 15
+; CHECK-NEXT:    i32.const $push391=, 15
+; CHECK-NEXT:    i32.shl $push186=, $pop184, $pop391
+; CHECK-NEXT:    i32.or $push187=, $pop183, $pop186
+; CHECK-NEXT:    i32.const $push390=, 65535
+; CHECK-NEXT:    i32.and $push189=, $pop187, $pop390
+; CHECK-NEXT:    i8x16.splat $push0=, $48
+; CHECK-NEXT:    i8x16.replace_lane $push1=, $pop0, 1, $49
+; CHECK-NEXT:    i8x16.replace_lane $push2=, $pop1, 2, $50
+; CHECK-NEXT:    i8x16.replace_lane $push3=, $pop2, 3, $51
+; CHECK-NEXT:    i8x16.replace_lane $push4=, $pop3, 4, $52
+; CHECK-NEXT:    i8x16.replace_lane $push5=, $pop4, 5, $53
+; CHECK-NEXT:    i8x16.replace_lane $push6=, $pop5, 6, $54
+; CHECK-NEXT:    i8x16.replace_lane $push7=, $pop6, 7, $55
+; CHECK-NEXT:    i8x16.replace_lane $push8=, $pop7, 8, $56
+; CHECK-NEXT:    i8x16.replace_lane $push9=, $pop8, 9, $57
+; CHECK-NEXT:    i8x16.replace_lane $push10=, $pop9, 10, $58
+; CHECK-NEXT:    i8x16.replace_lane $push11=, $pop10, 11, $59
+; CHECK-NEXT:    i8x16.replace_lane $push12=, $pop11, 12, $60
+; CHECK-NEXT:    i8x16.replace_lane $push13=, $pop12, 13, $61
+; CHECK-NEXT:    i8x16.replace_lane $push14=, $pop13, 14, $62
+; CHECK-NEXT:    i8x16.replace_lane $push15=, $pop14, 15, $63
+; CHECK-NEXT:    v128.and $push16=, $pop15, $64
+; CHECK-NEXT:    i8x16.eq $push389=, $pop16, $65
+; CHECK-NEXT:    local.tee $push388=, $66=, $pop389
+; CHECK-NEXT:    i8x16.extract_lane_u $push92=, $pop388, 15
+; CHECK-NEXT:    i32.const $push387=, 31
+; CHECK-NEXT:    i32.shl $push94=, $pop92, $pop387
+; CHECK-NEXT:    i8x16.extract_lane_u $push87=, $66, 14
+; CHECK-NEXT:    i32.const $push386=, 1
+; CHECK-NEXT:    i32.and $push88=, $pop87, $pop386
+; CHECK-NEXT:    i32.const $push385=, 30
+; CHECK-NEXT:    i32.shl $push90=, $pop88, $pop385
+; CHECK-NEXT:    i8x16.extract_lane_u $push82=, $66, 13
+; CHECK-NEXT:    i32.const $push384=, 1
+; CHECK-NEXT:    i32.and $push83=, $pop82, $pop384
+; CHECK-NEXT:    i32.const $push383=, 29
+; CHECK-NEXT:    i32.shl $push85=, $pop83, $pop383
+; CHECK-NEXT:    i8x16.extract_lane_u $push77=, $66, 12
+; CHECK-NEXT:    i32.const $push382=, 1
+; CHECK-NEXT:    i32.and $push78=, $pop77, $pop382
+; CHECK-NEXT:    i32.const $push381=, 28
+; CHECK-NEXT:    i32.shl $push80=, $pop78, $pop381
+; CHECK-NEXT:    i8x16.extract_lane_u $push72=, $66, 11
+; CHECK-NEXT:    i32.const $push380=, 1
+; CHECK-NEXT:    i32.and $push73=, $pop72, $pop380
+; CHECK-NEXT:    i32.const $push379=, 27
+; CHECK-NEXT:    i32.shl $push75=, $pop73, $pop379
+; CHECK-NEXT:    i8x16.extract_lane_u $push67=, $66, 10
+; CHECK-NEXT:    i32.const $push378=, 1
+; CHECK-NEXT:    i32.and $push68=, $pop67, $pop378
+; CHECK-NEXT:    i32.const $push377=, 26
+; CHECK-NEXT:    i32.shl $push70=, $pop68, $pop377
+; CHECK-NEXT:    i8x16.extract_lane_u $push62=, $66, 9
+; CHECK-NEXT:    i32.const $push376=, 1
+; CHECK-NEXT:    i32.and $push63=, $pop62, $pop376
+; CHECK-NEXT:    i32.const $push375=, 25
+; CHECK-NEXT:    i32.shl $push65=, $pop63, $pop375
+; CHECK-NEXT:    i8x16.extract_lane_u $push57=, $66, 8
+; CHECK-NEXT:    i32.const $push374=, 1
+; CHECK-NEXT:    i32.and $push58=, $pop57, $pop374
+; CHECK-NEXT:    i32.const $push373=, 24
+; CHECK-NEXT:    i32.shl $push60=, $pop58, $pop373
+; CHECK-NEXT:    i8x16.extract_lane_u $push52=, $66, 7
+; CHECK-NEXT:    i32.const $push372=, 1
+; CHECK-NEXT:    i32.and $push53=, $pop52, $pop372
+; CHECK-NEXT:    i32.const $push371=, 23
+; CHECK-NEXT:    i32.shl $push55=, $pop53, $pop371
+; CHECK-NEXT:    i8x16.extract_lane_u $push47=, $66, 6
+; CHECK-NEXT:    i32.const $push370=, 1
+; CHECK-NEXT:    i32.and $push48=, $pop47, $pop370
+; CHECK-NEXT:    i32.const $push369=, 22
+; CHECK-NEXT:    i32.shl $push50=, $pop48, $pop369
+; CHECK-NEXT:    i8x16.extract_lane_u $push42=, $66, 5
+; CHECK-NEXT:    i32.const $push368=, 1
+; CHECK-NEXT:    i32.and $push43=, $pop42, $pop368
+; CHECK-NEXT:    i32.const $push367=, 21
+; CHECK-NEXT:    i32.shl $push45=, $pop43, $pop367
+; CHECK-NEXT:    i8x16.extract_lane_u $push37=, $66, 4
+; CHECK-NEXT:    i32.const $push366=, 1
+; CHECK-NEXT:    i32.and $push38=, $pop37, $pop366
+; CHECK-NEXT:    i32.const $push365=, 20
+; CHECK-NEXT:    i32.shl $push40=, $pop38, $pop365
+; CHECK-NEXT:    i8x16.extract_lane_u $push32=, $66, 3
+; CHECK-NEXT:    i32.const $push364=, 1
+; CHECK-NEXT:    i32.and $push33=, $pop32, $pop364
+; CHECK-NEXT:    i32.const $push363=, 19
+; CHECK-NEXT:    i32.shl $push35=, $pop33, $pop363
+; CHECK-NEXT:    i8x16.extract_lane_u $push27=, $66, 2
+; CHECK-NEXT:    i32.const $push362=, 1
+; CHECK-NEXT:    i32.and $push28=, $pop27, $pop362
+; CHECK-NEXT:    i32.const $push361=, 18
+; CHECK-NEXT:    i32.shl $push30=, $pop28, $pop361
+; CHECK-NEXT:    i8x16.extract_lane_u $push22=, $66, 1
+; CHECK-NEXT:    i32.const $push360=, 1
+; CHECK-NEXT:    i32.and $push23=, $pop22, $pop360
+; CHECK-NEXT:    i32.const $push359=, 17
+; CHECK-NEXT:    i32.shl $push25=, $pop23, $pop359
+; CHECK-NEXT:    i8x16.extract_lane_u $push17=, $66, 0
+; CHECK-NEXT:    i32.const $push358=, 1
+; CHECK-NEXT:    i32.and $push19=, $pop17, $pop358
+; CHECK-NEXT:    i32.const $push357=, 16
+; CHECK-NEXT:    i32.shl $push21=, $pop19, $pop357
+; CHECK-NEXT:    i32.or $push26=, $pop25, $pop21
+; CHECK-NEXT:    i32.or $push31=, $pop30, $pop26
+; CHECK-NEXT:    i32.or $push36=, $pop35, $pop31
+; CHECK-NEXT:    i32.or $push41=, $pop40, $pop36
+; CHECK-NEXT:    i32.or $push46=, $pop45, $pop41
+; CHECK-NEXT:    i32.or $push51=, $pop50, $pop46
+; CHECK-NEXT:    i32.or $push56=, $pop55, $pop51
+; CHECK-NEXT:    i32.or $push61=, $pop60, $pop56
+; CHECK-NEXT:    i32.or $push66=, $pop65, $pop61
+; CHECK-NEXT:    i32.or $push71=, $pop70, $pop66
+; CHECK-NEXT:    i32.or $push76=, $pop75, $pop71
+; CHECK-NEXT:    i32.or $push81=, $pop80, $pop76
+; CHECK-NEXT:    i32.or $push86=, $pop85, $pop81
+; CHECK-NEXT:    i32.or $push91=, $pop90, $pop86
+; CHECK-NEXT:    i32.or $push95=, $pop94, $pop91
+; CHECK-NEXT:    i32.or $push190=, $pop189, $pop95
+; CHECK-NEXT:    i64.extend_i32_u $push191=, $pop190
+; CHECK-NEXT:    i64.const $push192=, 32
+; CHECK-NEXT:    i64.shl $push193=, $pop191, $pop192
+; CHECK-NEXT:    i64.or $push354=, $pop353, $pop193
+; CHECK-NEXT:    return $pop354
+    %z = icmp eq <64 x i4> %x, splat (i4 64)
+    %res = bitcast <64 x i1> %z to i64
+    ret i64 %res
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll b/llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll
new file mode 100644
index 0000000000000..172ff53bfb458
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+target triple = "wasm64"
+
+define i32 @all_true_16_i8(<16 x i8> %v) {
+; CHECK-LABEL: all_true_16_i8:
+; CHECK:         .functype all_true_16_i8 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i8x16.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <16 x i8> %v, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  %3 = icmp eq i16 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: all_true_4_i32:
+; CHECK:         .functype all_true_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32x4.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp eq i4 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_8_i16(<8 x i16> %v) {
+; CHECK-LABEL: all_true_8_i16:
+; CHECK:         .functype all_true_8_i16 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i16x8.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <8 x i16> %v, zeroinitializer
+  %2 = bitcast <8 x i1> %1 to i8
+  %3 = icmp eq i8 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_2_i64(<2 x i64> %v) {
+; CHECK-LABEL: all_true_2_i64:
+; CHECK:         .functype all_true_2_i64 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i64x2.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <2 x i64> %v, zeroinitializer
+  %2 = bitcast <2 x i1> %1 to i2
+  %3 = icmp eq i2 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_4_i64(<4 x i64> %v) {
+; CHECK-LABEL: all_true_4_i64:
+; CHECK:         .functype all_true_4_i64 (v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.const $push9=, 0, 0
+; CHECK-NEXT:    local.tee $push8=, $2=, $pop9
+; CHECK-NEXT:    i64x2.eq $push1=, $0, $pop8
+; CHECK-NEXT:    i64x2.eq $push0=, $1, $2
+; CHECK-NEXT:    i8x16.shuffle $push2=, $pop1, $pop0, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK-NEXT:    v128.any_true $push3=, $pop2
+; CHECK-NEXT:    i32.const $push4=, -1
+; CHECK-NEXT:    i32.xor $push5=, $pop3, $pop4
+; CHECK-NEXT:    i32.const $push6=, 1
+; CHECK-NEXT:    i32.and $push7=, $pop5, $pop6
+; CHECK-NEXT:    return $pop7
+  %1 = icmp eq <4 x i64> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp eq i4 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+; setcc (iN (bitcast (set_cc (vNi1 X), 0, ne)), 0, ne
+;   => any_true (set_cc (X), 0, ne)
+;   => any_true (X)
+define i32 @any_true_1_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: any_true_1_4_i32:
+; CHECK:         .functype any_true_1_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.any_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp ne <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp ne i4 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+; setcc (iN (bitcast (set_cc (vNi1 X), 0, eq)), -1, ne
+;   => not all_true (set_cc (X), 0, eq)
+;   => not not any_true (X)
+;   => any_true (X)
+define i32 @any_true_2_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: any_true_2_4_i32:
+; CHECK:         .functype any_true_2_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.any_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp ne i4 %2, -1
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+; setcc (iN (bitcast (set_cc (vNi1 X), 0, ne)), -1, eq
+;   => all_true (set_cc (X), 0, ne)
+;   => all_true (X)
+define i32 @all_true_2_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: all_true_2_4_i32:
+; CHECK:         .functype all_true_2_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32x4.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp ne <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp eq i4 %2, -1
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
diff --git a/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll b/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
index e6497bca98dc2..f7143711394fa 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
@@ -1086,9 +1086,9 @@ define i1 @test_cmp_v16i8(<16 x i8> %x) {
 ; CHECK-LABEL: test_cmp_v16i8:
 ; CHECK:         .functype test_cmp_v16i8 (v128) -> (i32)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    v128.const $push0=, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT:    i8x16.eq $push1=, $0, $pop0
-; CHECK-NEXT:    v128.any_true $push2=, $pop1
+; CHECK-NEXT:    i8x16.all_true $push0=, $0
+; CHECK-NEXT:    i32.const $push1=, 1
+; CHECK-NEXT:    i32.xor $push2=, $pop0, $pop1
 ; CHECK-NEXT:    return $pop2
   %zero = icmp eq <16 x i8> %x, zeroinitializer
   %ret = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %zero)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll b/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
index 503b39919b550..a2b13a27774c0 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
@@ -87,7 +87,7 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; EGPR-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; EGPR-NEXT:    testb %al, %al # encoding: [0x84,0xc0]
 ; EGPR-NEXT:    jne .LBB0_3 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_3-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_3, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1: # %loop.header.preheader
 ; EGPR-NEXT:    movq %rdi, %rbx # encoding: [0x48,0x89,0xfb]
 ; EGPR-NEXT:    xorl %r14d, %r14d # encoding: [0x45,0x31,0xf6]
@@ -100,7 +100,7 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; EGPR-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7a,0x4b,0x9c,0x04,0xd0,0x0b,0x00,0x00]
 ; EGPR-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; EGPR-NEXT:    callq foo # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: foo-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: foo, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:    ldtilecfg {{[0-9]+}}(%rsp) # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x78,0x49,0x84,0x24,0xc0,0x03,0x00,0x00]
 ; EGPR-NEXT:    movabsq $64, %rax # encoding: [0x48,0xb8,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 ; EGPR-NEXT:    tileloadd 3024(%rsp,%rax), %tmm3 # 1024-byte Folded Reload
@@ -116,7 +116,7 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; EGPR-NEXT:    incl %r14d # encoding: [0x41,0xff,0xc6]
 ; EGPR-NEXT:    cmpw $100, %r14w # encoding: [0x66,0x41,0x83,0xfe,0x64]
 ; EGPR-NEXT:    jl .LBB0_2 # encoding: [0x7c,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  .LBB0_3: # %exit
 ; EGPR-NEXT:    addq $4056, %rsp # encoding: [0x48,0x81,0xc4,0xd8,0x0f,0x00,0x00]
 ; EGPR-NEXT:    # imm = 0xFD8
@@ -226,7 +226,7 @@ define dso_local void @test2(ptr%buf) nounwind {
 ; EGPR-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; EGPR-NEXT:    testb %al, %al # encoding: [0x84,0xc0]
 ; EGPR-NEXT:    jne .LBB1_3 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_3-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_3, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1: # %loop.header.preheader
 ; EGPR-NEXT:    movq %rdi, %rbx # encoding: [0x48,0x89,0xfb]
 ; EGPR-NEXT:    xorl %r14d, %r14d # encoding: [0x45,0x31,0xf6]
@@ -237,7 +237,7 @@ define dso_local void @test2(ptr%buf) nounwind {
 ; EGPR-NEXT:    tilezero %tmm0 # encoding: [0xc4,0xe2,0x7b,0x49,0xc0]
 ; EGPR-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; EGPR-NEXT:    callq foo # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: foo-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: foo, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:    ldtilecfg {{[0-9]+}}(%rsp) # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x78,0x49,0x44,0x24,0x08]
 ; EGPR-NEXT:    tilezero %tmm2 # encoding: [0xc4,0xe2,0x7b,0x49,0xd0]
 ; EGPR-NEXT:    tileloadd (%rbx,%r15), %tmm0 # EVEX TO VEX Compression encoding: [0xc4,0xa2,0x7b,0x4b,0x04,0x3b]
@@ -247,7 +247,7 @@ define dso_local void @test2(ptr%buf) nounwind {
 ; EGPR-NEXT:    incl %r14d # encoding: [0x41,0xff,0xc6]
 ; EGPR-NEXT:    cmpw $100, %r14w # encoding: [0x66,0x41,0x83,0xfe,0x64]
 ; EGPR-NEXT:    jl .LBB1_2 # encoding: [0x7c,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  .LBB1_3: # %exit
 ; EGPR-NEXT:    addq $72, %rsp # encoding: [0x48,0x83,0xc4,0x48]
 ; EGPR-NEXT:    popq %rbx # encoding: [0x5b]
diff --git a/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll b/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll
index 4fb0a4445862f..254f824379d56 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll
@@ -125,7 +125,7 @@ define dso_local void @test_api(i16 signext %0, i16 signext %1) nounwind {
 ; EGPR-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; EGPR-NEXT:    testb %al, %al # encoding: [0x84,0xc0]
 ; EGPR-NEXT:    jne .LBB0_2 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1: # %if.true
 ; EGPR-NEXT:    movl $buf, %eax # encoding: [0xb8,A,A,A,A]
 ; EGPR-NEXT:    # fixup A - offset: 1, value: buf, kind: FK_Data_4
@@ -143,13 +143,13 @@ define dso_local void @test_api(i16 signext %0, i16 signext %1) nounwind {
 ; EGPR-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; EGPR-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; EGPR-NEXT:    callq foo # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: foo-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: foo, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:    ldtilecfg (%rsp) # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x78,0x49,0x04,0x24]
 ; EGPR-NEXT:    movabsq $64, %rax # encoding: [0x48,0xb8,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 ; EGPR-NEXT:    tileloadd 64(%rsp,%rax), %tmm6 # 1024-byte Folded Reload
 ; EGPR-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7b,0x4b,0x74,0x04,0x40]
 ; EGPR-NEXT:    jmp .LBB0_3 # encoding: [0xeb,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_3-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB0_3, kind: FK_PCRel_1
 ; EGPR-NEXT:  .LBB0_2: # %if.false
 ; EGPR-NEXT:    movl $buf, %eax # encoding: [0xb8,A,A,A,A]
 ; EGPR-NEXT:    # fixup A - offset: 1, value: buf, kind: FK_Data_4
@@ -167,7 +167,7 @@ define dso_local void @test_api(i16 signext %0, i16 signext %1) nounwind {
 ; EGPR-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; EGPR-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; EGPR-NEXT:    callq foo # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: foo-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: foo, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:    ldtilecfg (%rsp) # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x78,0x49,0x04,0x24]
 ; EGPR-NEXT:    movabsq $64, %rax # encoding: [0x48,0xb8,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 ; EGPR-NEXT:    tileloadd 64(%rsp,%rax), %tmm6 # 1024-byte Folded Reload
@@ -294,7 +294,7 @@ define dso_local void @test3(ptr%buf) nounwind {
 ; EGPR-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; EGPR-NEXT:    testb %al, %al # encoding: [0x84,0xc0]
 ; EGPR-NEXT:    jne .LBB1_3 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_3-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_3, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1: # %loop.header.preheader
 ; EGPR-NEXT:    movq %rdi, %rbx # encoding: [0x48,0x89,0xfb]
 ; EGPR-NEXT:    movl $32, %r14d # encoding: [0x41,0xbe,0x20,0x00,0x00,0x00]
@@ -307,7 +307,7 @@ define dso_local void @test3(ptr%buf) nounwind {
 ; EGPR-NEXT:    tilezero %tmm0 # encoding: [0xc4,0xe2,0x7b,0x49,0xc0]
 ; EGPR-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; EGPR-NEXT:    callq foo # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: foo-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: foo, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:    ldtilecfg {{[0-9]+}}(%rsp) # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x78,0x49,0x44,0x24,0x08]
 ; EGPR-NEXT:    tilezero %tmm0 # encoding: [0xc4,0xe2,0x7b,0x49,0xc0]
 ; EGPR-NEXT:    tileloadd (%rbx,%r14), %tmm1 # EVEX TO VEX Compression encoding: [0xc4,0xa2,0x7b,0x4b,0x0c,0x33]
@@ -318,7 +318,7 @@ define dso_local void @test3(ptr%buf) nounwind {
 ; EGPR-NEXT:    incl %r15d # encoding: [0x41,0xff,0xc7]
 ; EGPR-NEXT:    cmpw $100, %r15w # encoding: [0x66,0x41,0x83,0xff,0x64]
 ; EGPR-NEXT:    jl .LBB1_2 # encoding: [0x7c,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  .LBB1_3: # %exit
 ; EGPR-NEXT:    addq $72, %rsp # encoding: [0x48,0x83,0xc4,0x48]
 ; EGPR-NEXT:    popq %rbx # encoding: [0x5b]
diff --git a/llvm/test/CodeGen/X86/abds-neg.ll b/llvm/test/CodeGen/X86/abds-neg.ll
index 6e22d855dc831..f6d66ab47ce05 100644
--- a/llvm/test/CodeGen/X86/abds-neg.ll
+++ b/llvm/test/CodeGen/X86/abds-neg.ll
@@ -1058,15 +1058,15 @@ define i64 @abd_subnsw_i64(i64 %a, i64 %b) nounwind {
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    sarl $31, %edx
-; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    xorl %edx, %ecx
+; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    sbbl %esi, %edx
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %ecx, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
@@ -1089,15 +1089,15 @@ define i64 @abd_subnsw_i64_undef(i64 %a, i64 %b) nounwind {
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    sarl $31, %edx
-; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    xorl %edx, %ecx
+; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    sbbl %esi, %edx
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %ecx, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
@@ -1121,11 +1121,11 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
@@ -1178,11 +1178,11 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
diff --git a/llvm/test/CodeGen/X86/absolute-cmp.ll b/llvm/test/CodeGen/X86/absolute-cmp.ll
index bb0fe021b9524..6e6615e7bddf1 100644
--- a/llvm/test/CodeGen/X86/absolute-cmp.ll
+++ b/llvm/test/CodeGen/X86/absolute-cmp.ll
@@ -16,12 +16,12 @@ define void @foo8(i64 %val) {
 ; NOPIC-NEXT:    cmpq $cmp8@ABS8, %rdi # encoding: [0x48,0x83,0xff,A]
 ; NOPIC-NEXT:    # fixup A - offset: 3, value: cmp8@ABS8, kind: FK_Data_1
 ; NOPIC-NEXT:    ja .LBB0_2 # encoding: [0x77,A]
-; NOPIC-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
+; NOPIC-NEXT:    # fixup A - offset: 1, value: .LBB0_2, kind: FK_PCRel_1
 ; NOPIC-NEXT:  # %bb.1: # %t
 ; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 16
 ; NOPIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
-; NOPIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
+; NOPIC-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_4
 ; NOPIC-NEXT:    popq %rax # encoding: [0x58]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 8
 ; NOPIC-NEXT:  .LBB0_2: # %f
@@ -32,12 +32,12 @@ define void @foo8(i64 %val) {
 ; PIC-NEXT:    cmpq $cmp8@ABS8, %rdi # encoding: [0x48,0x83,0xff,A]
 ; PIC-NEXT:    # fixup A - offset: 3, value: cmp8@ABS8, kind: FK_Data_1
 ; PIC-NEXT:    ja .LBB0_2 # encoding: [0x77,A]
-; PIC-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
+; PIC-NEXT:    # fixup A - offset: 1, value: .LBB0_2, kind: FK_PCRel_1
 ; PIC-NEXT:  # %bb.1: # %t
 ; PIC-NEXT:    pushq %rax # encoding: [0x50]
 ; PIC-NEXT:    .cfi_def_cfa_offset 16
 ; PIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
-; PIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
+; PIC-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_4
 ; PIC-NEXT:    popq %rax # encoding: [0x58]
 ; PIC-NEXT:    .cfi_def_cfa_offset 8
 ; PIC-NEXT:  .LBB0_2: # %f
@@ -59,12 +59,12 @@ define void @foo32(i64 %val) {
 ; NOPIC-NEXT:    cmpq $cmp32, %rdi # encoding: [0x48,0x81,0xff,A,A,A,A]
 ; NOPIC-NEXT:    # fixup A - offset: 3, value: cmp32, kind: reloc_signed_4byte
 ; NOPIC-NEXT:    ja .LBB1_2 # encoding: [0x77,A]
-; NOPIC-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; NOPIC-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; NOPIC-NEXT:  # %bb.1: # %t
 ; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 16
 ; NOPIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
-; NOPIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
+; NOPIC-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_4
 ; NOPIC-NEXT:    popq %rax # encoding: [0x58]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 8
 ; NOPIC-NEXT:  .LBB1_2: # %f
@@ -75,12 +75,12 @@ define void @foo32(i64 %val) {
 ; PIC-NEXT:    cmpq $cmp32, %rdi # encoding: [0x48,0x81,0xff,A,A,A,A]
 ; PIC-NEXT:    # fixup A - offset: 3, value: cmp32, kind: reloc_signed_4byte
 ; PIC-NEXT:    ja .LBB1_2 # encoding: [0x77,A]
-; PIC-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; PIC-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; PIC-NEXT:  # %bb.1: # %t
 ; PIC-NEXT:    pushq %rax # encoding: [0x50]
 ; PIC-NEXT:    .cfi_def_cfa_offset 16
 ; PIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
-; PIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
+; PIC-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_4
 ; PIC-NEXT:    popq %rax # encoding: [0x58]
 ; PIC-NEXT:    .cfi_def_cfa_offset 8
 ; PIC-NEXT:  .LBB1_2: # %f
diff --git a/llvm/test/CodeGen/X86/apx/and.ll b/llvm/test/CodeGen/X86/apx/and.ll
index 3379ac9dec893..b442260f503b2 100644
--- a/llvm/test/CodeGen/X86/apx/and.ll
+++ b/llvm/test/CodeGen/X86/apx/and.ll
@@ -437,7 +437,7 @@ define i1 @andflag8rr(i8 %a, i8 %b) {
 ; CHECK-NEXT:    andb %al, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x20,0xc7]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag8rr:
@@ -446,7 +446,7 @@ define i1 @andflag8rr(i8 %a, i8 %b) {
 ; NF-NEXT:    andb %al, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x20,0xc7]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i8 %b, -1
   %v0 = and i8 %a, %xor  ; 0xff << 50
@@ -462,7 +462,7 @@ define i1 @andflag16rr(i16 %a, i16 %b) {
 ; CHECK-NEXT:    andw %ax, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x21,0xc7]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag16rr:
@@ -471,7 +471,7 @@ define i1 @andflag16rr(i16 %a, i16 %b) {
 ; NF-NEXT:    andw %ax, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x21,0xc7]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 %b, -1
   %v0 = and i16 %a, %xor  ; 0xff << 50
@@ -486,7 +486,7 @@ define i1 @andflag32rr(i32 %a, i32 %b) {
 ; CHECK-NEXT:    andl %edi, %esi # EVEX TO LEGACY Compression encoding: [0x21,0xfe]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag32rr:
@@ -494,7 +494,7 @@ define i1 @andflag32rr(i32 %a, i32 %b) {
 ; NF-NEXT:    andl %edi, %esi # EVEX TO LEGACY Compression encoding: [0x21,0xfe]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = and i32 %a, %b  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -508,7 +508,7 @@ define i1 @andflag64rr(i64 %a, i64 %b) {
 ; CHECK-NEXT:    andq %rdi, %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x21,0xfe]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag64rr:
@@ -516,7 +516,7 @@ define i1 @andflag64rr(i64 %a, i64 %b) {
 ; NF-NEXT:    andq %rdi, %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x21,0xfe]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, %b  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
@@ -531,7 +531,7 @@ define i1 @andflag8rm(ptr %ptr, i8 %b) {
 ; CHECK-NEXT:    andb (%rdi), %al, %cl # encoding: [0x62,0xf4,0x74,0x18,0x22,0x07]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag8rm:
@@ -540,7 +540,7 @@ define i1 @andflag8rm(ptr %ptr, i8 %b) {
 ; NF-NEXT:    andb (%rdi), %al, %cl # encoding: [0x62,0xf4,0x74,0x18,0x22,0x07]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i8, ptr %ptr
   %xor = xor i8 %b, -1
@@ -557,7 +557,7 @@ define i1 @andflag16rm(ptr %ptr, i16 %b) {
 ; CHECK-NEXT:    andw (%rdi), %ax, %cx # encoding: [0x62,0xf4,0x75,0x18,0x23,0x07]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag16rm:
@@ -566,7 +566,7 @@ define i1 @andflag16rm(ptr %ptr, i16 %b) {
 ; NF-NEXT:    andw (%rdi), %ax, %cx # encoding: [0x62,0xf4,0x75,0x18,0x23,0x07]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i16, ptr %ptr
   %xor = xor i16 %b, -1
@@ -582,7 +582,7 @@ define i1 @andflag32rm(ptr %ptr, i32 %b) {
 ; CHECK-NEXT:    andl (%rdi), %esi # EVEX TO LEGACY Compression encoding: [0x23,0x37]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag32rm:
@@ -590,7 +590,7 @@ define i1 @andflag32rm(ptr %ptr, i32 %b) {
 ; NF-NEXT:    andl (%rdi), %esi # EVEX TO LEGACY Compression encoding: [0x23,0x37]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i32, ptr %ptr
   %v0 = and i32 %a, %b  ; 0xff << 50
@@ -605,7 +605,7 @@ define i1 @andflag64rm(ptr %ptr, i64 %b) {
 ; CHECK-NEXT:    andq (%rdi), %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x23,0x37]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag64rm:
@@ -613,7 +613,7 @@ define i1 @andflag64rm(ptr %ptr, i64 %b) {
 ; NF-NEXT:    andq (%rdi), %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x23,0x37]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i64, ptr %ptr
   %v0 = and i64 %a, %b  ; 0xff << 50
@@ -628,7 +628,7 @@ define i1 @andflag8ri(i8 %a) {
 ; CHECK-NEXT:    andb $-124, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xe7,0x84]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag8ri:
@@ -636,7 +636,7 @@ define i1 @andflag8ri(i8 %a) {
 ; NF-NEXT:    andb $-124, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xe7,0x84]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i8 123, -1
   %v0 = and i8 %a, %xor  ; 0xff << 50
@@ -652,7 +652,7 @@ define i1 @andflag16ri(i16 %a) {
 ; CHECK-NEXT:    # imm = 0xFB2D
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag16ri:
@@ -661,7 +661,7 @@ define i1 @andflag16ri(i16 %a) {
 ; NF-NEXT:    # imm = 0xFB2D
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 1234, -1
   %v0 = and i16 %a, %xor  ; 0xff << 50
@@ -677,7 +677,7 @@ define i1 @andflag32ri(i32 %a) {
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag32ri:
@@ -686,7 +686,7 @@ define i1 @andflag32ri(i32 %a) {
 ; NF-NEXT:    # imm = 0x1E240
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = and i32 %a, 123456  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -701,7 +701,7 @@ define i1 @andflag64ri(i64 %a) {
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag64ri:
@@ -710,7 +710,7 @@ define i1 @andflag64ri(i64 %a) {
 ; NF-NEXT:    # imm = 0x1E240
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 123456  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
@@ -724,7 +724,7 @@ define i1 @andflag16ri8(i16 %a) {
 ; CHECK-NEXT:    andw $-124, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xe7,0x84]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag16ri8:
@@ -732,7 +732,7 @@ define i1 @andflag16ri8(i16 %a) {
 ; NF-NEXT:    andw $-124, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xe7,0x84]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 123, -1
   %v0 = and i16 %a, %xor  ; 0xff << 50
@@ -747,7 +747,7 @@ define i1 @andflag32ri8(i32 %a) {
 ; CHECK-NEXT:    andl $123, %edi # EVEX TO LEGACY Compression encoding: [0x83,0xe7,0x7b]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag32ri8:
@@ -755,7 +755,7 @@ define i1 @andflag32ri8(i32 %a) {
 ; NF-NEXT:    andl $123, %edi # EVEX TO LEGACY Compression encoding: [0x83,0xe7,0x7b]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = and i32 %a, 123  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -769,7 +769,7 @@ define i1 @andflag64ri8(i64 %a) {
 ; CHECK-NEXT:    andq $123, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x83,0xe7,0x7b]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: andflag64ri8:
@@ -777,7 +777,7 @@ define i1 @andflag64ri8(i64 %a) {
 ; NF-NEXT:    andq $123, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x83,0xe7,0x7b]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 123  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
diff --git a/llvm/test/CodeGen/X86/apx/ccmp.ll b/llvm/test/CodeGen/X86/apx/ccmp.ll
index 7bd8aeea8863b..4c58c8a980314 100644
--- a/llvm/test/CodeGen/X86/apx/ccmp.ll
+++ b/llvm/test/CodeGen/X86/apx/ccmp.ll
@@ -8,12 +8,12 @@ define void @ccmp8rr_zf(i8 noundef %a, i8 noundef %b, i8 noundef %c) {
 ; CHECK-NEXT:    cmpb %dl, %dil # encoding: [0x40,0x38,0xd7]
 ; CHECK-NEXT:    ccmpneb {dfv=zf} %dl, %sil # encoding: [0x62,0xf4,0x14,0x05,0x38,0xd6]
 ; CHECK-NEXT:    jne .LBB0_1 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB0_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB0_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -22,12 +22,12 @@ define void @ccmp8rr_zf(i8 noundef %a, i8 noundef %b, i8 noundef %c) {
 ; NDD-NEXT:    cmpb %dl, %dil # encoding: [0x40,0x38,0xd7]
 ; NDD-NEXT:    ccmpneb {dfv=zf} %dl, %sil # encoding: [0x62,0xf4,0x14,0x05,0x38,0xd6]
 ; NDD-NEXT:    jne .LBB0_1 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB0_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB0_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -50,12 +50,12 @@ define void @ccmp8rr_cf(i8 noundef %a, i8 noundef %b) {
 ; CHECK-NEXT:    cmpb $2, %dil # encoding: [0x40,0x80,0xff,0x02]
 ; CHECK-NEXT:    ccmpgeb {dfv=cf} $2, %sil # encoding: [0x62,0xf4,0x0c,0x0d,0x80,0xfe,0x02]
 ; CHECK-NEXT:    jb .LBB1_1 # encoding: [0x72,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB1_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB1_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB1_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -64,12 +64,12 @@ define void @ccmp8rr_cf(i8 noundef %a, i8 noundef %b) {
 ; NDD-NEXT:    cmpb $2, %dil # encoding: [0x40,0x80,0xff,0x02]
 ; NDD-NEXT:    ccmpgeb {dfv=cf} $2, %sil # encoding: [0x62,0xf4,0x0c,0x0d,0x80,0xfe,0x02]
 ; NDD-NEXT:    jb .LBB1_1 # encoding: [0x72,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB1_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB1_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB1_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -92,7 +92,7 @@ define i8 @ccmp8rr_sf(i8 %a, i8 %b, i8* nocapture %c)  {
 ; CHECK-NEXT:    testb %dil, %dil # encoding: [0x40,0x84,0xff]
 ; CHECK-NEXT:    ccmpneb {dfv=sf} $2, %sil # encoding: [0x62,0xf4,0x24,0x05,0x80,0xfe,0x02]
 ; CHECK-NEXT:    jl .LBB2_2 # encoding: [0x7c,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    movb %dil, (%rdx) # encoding: [0x40,0x88,0x3a]
 ; CHECK-NEXT:  .LBB2_2: # %if.end
@@ -104,7 +104,7 @@ define i8 @ccmp8rr_sf(i8 %a, i8 %b, i8* nocapture %c)  {
 ; NDD-NEXT:    testb %dil, %dil # encoding: [0x40,0x84,0xff]
 ; NDD-NEXT:    ccmpneb {dfv=sf} $2, %sil # encoding: [0x62,0xf4,0x24,0x05,0x80,0xfe,0x02]
 ; NDD-NEXT:    jl .LBB2_2 # encoding: [0x7c,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.1: # %if.then
 ; NDD-NEXT:    movb %dil, (%rdx) # encoding: [0x40,0x88,0x3a]
 ; NDD-NEXT:  .LBB2_2: # %if.end
@@ -130,7 +130,7 @@ define i8 @ccmp8rr_none(i8 %a, i8 %b, i8* nocapture %c)  {
 ; CHECK-NEXT:    testb %dil, %dil # encoding: [0x40,0x84,0xff]
 ; CHECK-NEXT:    ccmpeb {dfv=} $2, %sil # encoding: [0x62,0xf4,0x04,0x04,0x80,0xfe,0x02]
 ; CHECK-NEXT:    jl .LBB3_2 # encoding: [0x7c,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB3_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB3_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    movb %dil, (%rdx) # encoding: [0x40,0x88,0x3a]
 ; CHECK-NEXT:  .LBB3_2: # %if.end
@@ -142,7 +142,7 @@ define i8 @ccmp8rr_none(i8 %a, i8 %b, i8* nocapture %c)  {
 ; NDD-NEXT:    testb %dil, %dil # encoding: [0x40,0x84,0xff]
 ; NDD-NEXT:    ccmpeb {dfv=} $2, %sil # encoding: [0x62,0xf4,0x04,0x04,0x80,0xfe,0x02]
 ; NDD-NEXT:    jl .LBB3_2 # encoding: [0x7c,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB3_2-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB3_2, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.1: # %if.then
 ; NDD-NEXT:    movb %dil, (%rdx) # encoding: [0x40,0x88,0x3a]
 ; NDD-NEXT:  .LBB3_2: # %if.end
@@ -168,12 +168,12 @@ define void @ccmp16rr_sf(i16 noundef %a, i16 noundef %b, i16 noundef %c) {
 ; CHECK-NEXT:    cmpw %dx, %di # encoding: [0x66,0x39,0xd7]
 ; CHECK-NEXT:    ccmplew {dfv=sf} %dx, %si # encoding: [0x62,0xf4,0x25,0x0e,0x39,0xd6]
 ; CHECK-NEXT:    jge .LBB4_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB4_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB4_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB4_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -182,12 +182,12 @@ define void @ccmp16rr_sf(i16 noundef %a, i16 noundef %b, i16 noundef %c) {
 ; NDD-NEXT:    cmpw %dx, %di # encoding: [0x66,0x39,0xd7]
 ; NDD-NEXT:    ccmplew {dfv=sf} %dx, %si # encoding: [0x62,0xf4,0x25,0x0e,0x39,0xd6]
 ; NDD-NEXT:    jge .LBB4_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB4_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB4_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB4_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -210,12 +210,12 @@ define void @ccmp32rr_cf(i32 noundef %a, i32 noundef %b, i32 noundef %c) {
 ; CHECK-NEXT:    cmpl %edx, %edi # encoding: [0x39,0xd7]
 ; CHECK-NEXT:    ccmpbl {dfv=cf} %edx, %esi # encoding: [0x62,0xf4,0x0c,0x02,0x39,0xd6]
 ; CHECK-NEXT:    ja .LBB5_1 # encoding: [0x77,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB5_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -224,12 +224,12 @@ define void @ccmp32rr_cf(i32 noundef %a, i32 noundef %b, i32 noundef %c) {
 ; NDD-NEXT:    cmpl %edx, %edi # encoding: [0x39,0xd7]
 ; NDD-NEXT:    ccmpbl {dfv=cf} %edx, %esi # encoding: [0x62,0xf4,0x0c,0x02,0x39,0xd6]
 ; NDD-NEXT:    ja .LBB5_1 # encoding: [0x77,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB5_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB5_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB5_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -252,12 +252,12 @@ define void @ccmp64rr_of(i64 %a, i64 %b, i64 %c) {
 ; CHECK-NEXT:    cmpq %rdx, %rdi # encoding: [0x48,0x39,0xd7]
 ; CHECK-NEXT:    ccmpbq {dfv=of} %rsi, %rdi # encoding: [0x62,0xf4,0xc4,0x02,0x39,0xf7]
 ; CHECK-NEXT:    jno .LBB6_1 # encoding: [0x71,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB6_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB6_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB6_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -266,12 +266,12 @@ define void @ccmp64rr_of(i64 %a, i64 %b, i64 %c) {
 ; NDD-NEXT:    cmpq %rdx, %rdi # encoding: [0x48,0x39,0xd7]
 ; NDD-NEXT:    ccmpbq {dfv=of} %rsi, %rdi # encoding: [0x62,0xf4,0xc4,0x02,0x39,0xf7]
 ; NDD-NEXT:    jno .LBB6_1 # encoding: [0x71,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB6_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB6_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB6_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 bb:
@@ -294,7 +294,7 @@ define void @ccmp64rr_of_crossbb(i64 %a, i64 %b) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    testq %rdi, %rdi # encoding: [0x48,0x85,0xff]
 ; CHECK-NEXT:    je .LBB7_2 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB7_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB7_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %bb1
 ; CHECK-NEXT:    cmpq %rsi, %rdi # encoding: [0x48,0x39,0xf7]
 ; CHECK-NEXT:  .LBB7_2: # %bb3
@@ -304,7 +304,7 @@ define void @ccmp64rr_of_crossbb(i64 %a, i64 %b) {
 ; NDD:       # %bb.0: # %bb
 ; NDD-NEXT:    testq %rdi, %rdi # encoding: [0x48,0x85,0xff]
 ; NDD-NEXT:    je .LBB7_2 # encoding: [0x74,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB7_2-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB7_2, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.1: # %bb1
 ; NDD-NEXT:    cmpq %rsi, %rdi # encoding: [0x48,0x39,0xf7]
 ; NDD-NEXT:  .LBB7_2: # %bb3
@@ -332,12 +332,12 @@ define void @ccmp8ri_zf(i8 noundef %a, i8 noundef %b, i8 noundef %c) {
 ; CHECK-NEXT:    cmpb %dl, %dil # encoding: [0x40,0x38,0xd7]
 ; CHECK-NEXT:    ccmpleb {dfv=zf} $123, %sil # encoding: [0x62,0xf4,0x14,0x0e,0x80,0xfe,0x7b]
 ; CHECK-NEXT:    jne .LBB8_1 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB8_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB8_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB8_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -346,12 +346,12 @@ define void @ccmp8ri_zf(i8 noundef %a, i8 noundef %b, i8 noundef %c) {
 ; NDD-NEXT:    cmpb %dl, %dil # encoding: [0x40,0x38,0xd7]
 ; NDD-NEXT:    ccmpleb {dfv=zf} $123, %sil # encoding: [0x62,0xf4,0x14,0x0e,0x80,0xfe,0x7b]
 ; NDD-NEXT:    jne .LBB8_1 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB8_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB8_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB8_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -375,7 +375,7 @@ define i8 @ccmp8ri_zf_double(i8 %a, double %b, i8* nocapture %c)  {
 ; CHECK-NEXT:    ucomisd %xmm1, %xmm0 # encoding: [0x66,0x0f,0x2e,0xc1]
 ; CHECK-NEXT:    ccmpeb {dfv=zf} $123, %dil # encoding: [0x62,0xf4,0x14,0x04,0x80,0xff,0x7b]
 ; CHECK-NEXT:    je .LBB9_2 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB9_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB9_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    movb %dil, (%rsi) # encoding: [0x40,0x88,0x3e]
 ; CHECK-NEXT:  .LBB9_2: # %if.end
@@ -388,7 +388,7 @@ define i8 @ccmp8ri_zf_double(i8 %a, double %b, i8* nocapture %c)  {
 ; NDD-NEXT:    ucomisd %xmm1, %xmm0 # encoding: [0x66,0x0f,0x2e,0xc1]
 ; NDD-NEXT:    ccmpeb {dfv=zf} $123, %dil # encoding: [0x62,0xf4,0x14,0x04,0x80,0xff,0x7b]
 ; NDD-NEXT:    je .LBB9_2 # encoding: [0x74,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB9_2-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB9_2, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.1: # %if.then
 ; NDD-NEXT:    movb %dil, (%rsi) # encoding: [0x40,0x88,0x3e]
 ; NDD-NEXT:  .LBB9_2: # %if.end
@@ -418,7 +418,7 @@ define i8 @ccmp8ri_zf_double_p(i8 %a, double %b, i8* nocapture %c)  {
 ; CHECK-NEXT:    andb %al, %cl # encoding: [0x20,0xc1]
 ; CHECK-NEXT:    cmpb $1, %cl # encoding: [0x80,0xf9,0x01]
 ; CHECK-NEXT:    jne .LBB10_2 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB10_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB10_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    movb %dil, (%rsi) # encoding: [0x40,0x88,0x3e]
 ; CHECK-NEXT:  .LBB10_2: # %if.end
@@ -434,7 +434,7 @@ define i8 @ccmp8ri_zf_double_p(i8 %a, double %b, i8* nocapture %c)  {
 ; NDD-NEXT:    andb %cl, %al # EVEX TO LEGACY Compression encoding: [0x20,0xc8]
 ; NDD-NEXT:    cmpb $1, %al # encoding: [0x3c,0x01]
 ; NDD-NEXT:    jne .LBB10_2 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB10_2-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB10_2, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.1: # %if.then
 ; NDD-NEXT:    movb %dil, (%rsi) # encoding: [0x40,0x88,0x3e]
 ; NDD-NEXT:  .LBB10_2: # %if.end
@@ -464,7 +464,7 @@ define i8 @ccmp8ri_zf_double_np(i8 %a, double %b, i8* nocapture %c)  {
 ; CHECK-NEXT:    andb %al, %cl # encoding: [0x20,0xc1]
 ; CHECK-NEXT:    cmpb $1, %cl # encoding: [0x80,0xf9,0x01]
 ; CHECK-NEXT:    jne .LBB11_2 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB11_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB11_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    movb %dil, (%rsi) # encoding: [0x40,0x88,0x3e]
 ; CHECK-NEXT:  .LBB11_2: # %if.end
@@ -480,7 +480,7 @@ define i8 @ccmp8ri_zf_double_np(i8 %a, double %b, i8* nocapture %c)  {
 ; NDD-NEXT:    andb %cl, %al # EVEX TO LEGACY Compression encoding: [0x20,0xc8]
 ; NDD-NEXT:    cmpb $1, %al # encoding: [0x3c,0x01]
 ; NDD-NEXT:    jne .LBB11_2 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB11_2-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB11_2, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.1: # %if.then
 ; NDD-NEXT:    movb %dil, (%rsi) # encoding: [0x40,0x88,0x3e]
 ; NDD-NEXT:  .LBB11_2: # %if.end
@@ -508,12 +508,12 @@ define void @ccmp16ri_zf(i16 noundef %a, i16 noundef %b, i16 noundef %c) {
 ; CHECK-NEXT:    ccmpael {dfv=sf} $1234, %eax # encoding: [0x62,0xf4,0x24,0x03,0x81,0xf8,0xd2,0x04,0x00,0x00]
 ; CHECK-NEXT:    # imm = 0x4D2
 ; CHECK-NEXT:    jge .LBB12_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB12_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB12_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB12_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -524,12 +524,12 @@ define void @ccmp16ri_zf(i16 noundef %a, i16 noundef %b, i16 noundef %c) {
 ; NDD-NEXT:    ccmpael {dfv=sf} $1234, %eax # encoding: [0x62,0xf4,0x24,0x03,0x81,0xf8,0xd2,0x04,0x00,0x00]
 ; NDD-NEXT:    # imm = 0x4D2
 ; NDD-NEXT:    jge .LBB12_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB12_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB12_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB12_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -553,12 +553,12 @@ define void @ccmp32ri_cf(i32 noundef %a, i32 noundef %b, i32 noundef %c) {
 ; CHECK-NEXT:    ccmpbl {dfv=cf} $1048577, %esi # encoding: [0x62,0xf4,0x0c,0x02,0x81,0xfe,0x01,0x00,0x10,0x00]
 ; CHECK-NEXT:    # imm = 0x100001
 ; CHECK-NEXT:    jae .LBB13_1 # encoding: [0x73,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB13_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB13_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB13_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -568,12 +568,12 @@ define void @ccmp32ri_cf(i32 noundef %a, i32 noundef %b, i32 noundef %c) {
 ; NDD-NEXT:    ccmpbl {dfv=cf} $1048577, %esi # encoding: [0x62,0xf4,0x0c,0x02,0x81,0xfe,0x01,0x00,0x10,0x00]
 ; NDD-NEXT:    # imm = 0x100001
 ; NDD-NEXT:    jae .LBB13_1 # encoding: [0x73,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB13_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB13_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB13_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -597,12 +597,12 @@ define void @ccmp64ri32_zf(i64 noundef %a, i64 noundef %b, i64 noundef %c) {
 ; CHECK-NEXT:    ccmpbeq {dfv=sf} $123456, %rsi # encoding: [0x62,0xf4,0xa4,0x06,0x81,0xfe,0x40,0xe2,0x01,0x00]
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    jge .LBB14_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB14_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB14_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB14_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -612,12 +612,12 @@ define void @ccmp64ri32_zf(i64 noundef %a, i64 noundef %b, i64 noundef %c) {
 ; NDD-NEXT:    ccmpbeq {dfv=sf} $123456, %rsi # encoding: [0x62,0xf4,0xa4,0x06,0x81,0xfe,0x40,0xe2,0x01,0x00]
 ; NDD-NEXT:    # imm = 0x1E240
 ; NDD-NEXT:    jge .LBB14_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB14_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB14_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB14_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -640,12 +640,12 @@ define void @ccmp8rm_zf(i8 noundef %a, i8 noundef %b, i8 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    cmpb %dl, %dil # encoding: [0x40,0x38,0xd7]
 ; CHECK-NEXT:    ccmpneb {dfv=zf} (%rcx), %sil # encoding: [0x62,0xf4,0x14,0x05,0x3a,0x31]
 ; CHECK-NEXT:    jne .LBB15_1 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB15_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB15_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB15_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -654,12 +654,12 @@ define void @ccmp8rm_zf(i8 noundef %a, i8 noundef %b, i8 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    cmpb %dl, %dil # encoding: [0x40,0x38,0xd7]
 ; NDD-NEXT:    ccmpneb {dfv=zf} (%rcx), %sil # encoding: [0x62,0xf4,0x14,0x05,0x3a,0x31]
 ; NDD-NEXT:    jne .LBB15_1 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB15_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB15_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB15_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -683,12 +683,12 @@ define void @ccmp16rm_sf(i16 noundef %a, i16 noundef %b, i16 noundef %c, ptr %pt
 ; CHECK-NEXT:    cmpw %dx, %di # encoding: [0x66,0x39,0xd7]
 ; CHECK-NEXT:    ccmplew {dfv=sf} (%rcx), %si # encoding: [0x62,0xf4,0x25,0x0e,0x3b,0x31]
 ; CHECK-NEXT:    jge .LBB16_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB16_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB16_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB16_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -697,12 +697,12 @@ define void @ccmp16rm_sf(i16 noundef %a, i16 noundef %b, i16 noundef %c, ptr %pt
 ; NDD-NEXT:    cmpw %dx, %di # encoding: [0x66,0x39,0xd7]
 ; NDD-NEXT:    ccmplew {dfv=sf} (%rcx), %si # encoding: [0x62,0xf4,0x25,0x0e,0x3b,0x31]
 ; NDD-NEXT:    jge .LBB16_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB16_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB16_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB16_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -726,12 +726,12 @@ define void @ccmp32rm_cf(i32 noundef %a, i32 noundef %b, i32 noundef %c, ptr %pt
 ; CHECK-NEXT:    cmpl %edx, %edi # encoding: [0x39,0xd7]
 ; CHECK-NEXT:    ccmpgl {dfv=cf} (%rcx), %esi # encoding: [0x62,0xf4,0x0c,0x0f,0x3b,0x31]
 ; CHECK-NEXT:    ja .LBB17_1 # encoding: [0x77,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB17_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB17_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB17_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -740,12 +740,12 @@ define void @ccmp32rm_cf(i32 noundef %a, i32 noundef %b, i32 noundef %c, ptr %pt
 ; NDD-NEXT:    cmpl %edx, %edi # encoding: [0x39,0xd7]
 ; NDD-NEXT:    ccmpgl {dfv=cf} (%rcx), %esi # encoding: [0x62,0xf4,0x0c,0x0f,0x3b,0x31]
 ; NDD-NEXT:    ja .LBB17_1 # encoding: [0x77,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB17_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB17_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB17_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -769,12 +769,12 @@ define void @ccmp64rm_sf(i64 noundef %a, i64 noundef %b, i64 noundef %c, ptr %pt
 ; CHECK-NEXT:    cmpq %rdx, %rdi # encoding: [0x48,0x39,0xd7]
 ; CHECK-NEXT:    ccmpleq {dfv=sf} (%rcx), %rsi # encoding: [0x62,0xf4,0xa4,0x0e,0x3b,0x31]
 ; CHECK-NEXT:    jge .LBB18_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB18_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB18_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB18_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -783,12 +783,12 @@ define void @ccmp64rm_sf(i64 noundef %a, i64 noundef %b, i64 noundef %c, ptr %pt
 ; NDD-NEXT:    cmpq %rdx, %rdi # encoding: [0x48,0x39,0xd7]
 ; NDD-NEXT:    ccmpleq {dfv=sf} (%rcx), %rsi # encoding: [0x62,0xf4,0xa4,0x0e,0x3b,0x31]
 ; NDD-NEXT:    jge .LBB18_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB18_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB18_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB18_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -812,12 +812,12 @@ define void @ccmp8mr_zf(i8 noundef %a, i8 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    cmpb %sil, %dil # encoding: [0x40,0x38,0xf7]
 ; CHECK-NEXT:    ccmpgeb {dfv=zf} %sil, (%rdx) # encoding: [0x62,0xf4,0x14,0x0d,0x38,0x32]
 ; CHECK-NEXT:    jne .LBB19_1 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB19_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB19_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB19_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -826,12 +826,12 @@ define void @ccmp8mr_zf(i8 noundef %a, i8 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    cmpb %sil, %dil # encoding: [0x40,0x38,0xf7]
 ; NDD-NEXT:    ccmpgeb {dfv=zf} %sil, (%rdx) # encoding: [0x62,0xf4,0x14,0x0d,0x38,0x32]
 ; NDD-NEXT:    jne .LBB19_1 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB19_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB19_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB19_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -855,12 +855,12 @@ define void @ccmp16mr_sf(i16 noundef %a, i16 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    cmpw %si, %di # encoding: [0x66,0x39,0xf7]
 ; CHECK-NEXT:    ccmplew {dfv=sf} %si, (%rdx) # encoding: [0x62,0xf4,0x25,0x0e,0x39,0x32]
 ; CHECK-NEXT:    jge .LBB20_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB20_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB20_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB20_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -869,12 +869,12 @@ define void @ccmp16mr_sf(i16 noundef %a, i16 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    cmpw %si, %di # encoding: [0x66,0x39,0xf7]
 ; NDD-NEXT:    ccmplew {dfv=sf} %si, (%rdx) # encoding: [0x62,0xf4,0x25,0x0e,0x39,0x32]
 ; NDD-NEXT:    jge .LBB20_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB20_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB20_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB20_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -898,12 +898,12 @@ define void @ccmp32mr_cf(i32 noundef %a, i32 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
 ; CHECK-NEXT:    ccmpll {dfv=cf} %esi, (%rdx) # encoding: [0x62,0xf4,0x0c,0x0c,0x39,0x32]
 ; CHECK-NEXT:    ja .LBB21_1 # encoding: [0x77,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB21_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB21_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB21_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -912,12 +912,12 @@ define void @ccmp32mr_cf(i32 noundef %a, i32 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
 ; NDD-NEXT:    ccmpll {dfv=cf} %esi, (%rdx) # encoding: [0x62,0xf4,0x0c,0x0c,0x39,0x32]
 ; NDD-NEXT:    ja .LBB21_1 # encoding: [0x77,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB21_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB21_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB21_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -941,12 +941,12 @@ define void @ccmp64mr_sf(i64 noundef %a, i64 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    cmpq %rsi, %rdi # encoding: [0x48,0x39,0xf7]
 ; CHECK-NEXT:    ccmpleq {dfv=sf} %rsi, (%rdx) # encoding: [0x62,0xf4,0xa4,0x0e,0x39,0x32]
 ; CHECK-NEXT:    jge .LBB22_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB22_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB22_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB22_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -955,12 +955,12 @@ define void @ccmp64mr_sf(i64 noundef %a, i64 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    cmpq %rsi, %rdi # encoding: [0x48,0x39,0xf7]
 ; NDD-NEXT:    ccmpleq {dfv=sf} %rsi, (%rdx) # encoding: [0x62,0xf4,0xa4,0x0e,0x39,0x32]
 ; NDD-NEXT:    jge .LBB22_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB22_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB22_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB22_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -984,12 +984,12 @@ define void @ccmp8mi_zf(i8 noundef %a, i8 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    cmpb %sil, %dil # encoding: [0x40,0x38,0xf7]
 ; CHECK-NEXT:    ccmpneb {dfv=zf} $123, (%rdx) # encoding: [0x62,0xf4,0x14,0x05,0x80,0x3a,0x7b]
 ; CHECK-NEXT:    jne .LBB23_1 # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB23_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB23_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB23_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -998,12 +998,12 @@ define void @ccmp8mi_zf(i8 noundef %a, i8 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    cmpb %sil, %dil # encoding: [0x40,0x38,0xf7]
 ; NDD-NEXT:    ccmpneb {dfv=zf} $123, (%rdx) # encoding: [0x62,0xf4,0x14,0x05,0x80,0x3a,0x7b]
 ; NDD-NEXT:    jne .LBB23_1 # encoding: [0x75,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB23_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB23_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB23_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -1028,12 +1028,12 @@ define void @ccmp16mi_zf(i16 noundef %a, i16 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    ccmplew {dfv=sf} $1234, (%rdx) # encoding: [0x62,0xf4,0x25,0x0e,0x81,0x3a,0xd2,0x04]
 ; CHECK-NEXT:    # imm = 0x4D2
 ; CHECK-NEXT:    jge .LBB24_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB24_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB24_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB24_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1043,12 +1043,12 @@ define void @ccmp16mi_zf(i16 noundef %a, i16 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    ccmplew {dfv=sf} $1234, (%rdx) # encoding: [0x62,0xf4,0x25,0x0e,0x81,0x3a,0xd2,0x04]
 ; NDD-NEXT:    # imm = 0x4D2
 ; NDD-NEXT:    jge .LBB24_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB24_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB24_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB24_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -1073,12 +1073,12 @@ define void @ccmp32mi_cf(i32 noundef %a, i32 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    ccmpnel {dfv=cf} $123457, (%rdx) # encoding: [0x62,0xf4,0x0c,0x05,0x81,0x3a,0x41,0xe2,0x01,0x00]
 ; CHECK-NEXT:    # imm = 0x1E241
 ; CHECK-NEXT:    jae .LBB25_1 # encoding: [0x73,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB25_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB25_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB25_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1088,12 +1088,12 @@ define void @ccmp32mi_cf(i32 noundef %a, i32 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    ccmpnel {dfv=cf} $123457, (%rdx) # encoding: [0x62,0xf4,0x0c,0x05,0x81,0x3a,0x41,0xe2,0x01,0x00]
 ; NDD-NEXT:    # imm = 0x1E241
 ; NDD-NEXT:    jae .LBB25_1 # encoding: [0x73,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB25_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB25_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB25_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -1118,12 +1118,12 @@ define void @ccmp64mi32_zf(i64 noundef %a, i64 noundef %c, ptr %ptr) {
 ; CHECK-NEXT:    ccmpleq {dfv=sf} $123456, (%rdx) # encoding: [0x62,0xf4,0xa4,0x0e,0x81,0x3a,0x40,0xe2,0x01,0x00]
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    jge .LBB26_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB26_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB26_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB26_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1133,12 +1133,12 @@ define void @ccmp64mi32_zf(i64 noundef %a, i64 noundef %c, ptr %ptr) {
 ; NDD-NEXT:    ccmpleq {dfv=sf} $123456, (%rdx) # encoding: [0x62,0xf4,0xa4,0x0e,0x81,0x3a,0x40,0xe2,0x01,0x00]
 ; NDD-NEXT:    # imm = 0x1E240
 ; NDD-NEXT:    jge .LBB26_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB26_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB26_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB26_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -1163,12 +1163,12 @@ define void @ccmp_continous(i32 noundef %a, i32 noundef %b, i32 noundef %c) {
 ; CHECK-NEXT:    ccmplel {dfv=} $2, %esi # encoding: [0x62,0xf4,0x04,0x0e,0x83,0xfe,0x02]
 ; CHECK-NEXT:    ccmpll {dfv=} $3, %edx # encoding: [0x62,0xf4,0x04,0x0c,0x83,0xfa,0x03]
 ; CHECK-NEXT:    jge .LBB27_1 # encoding: [0x7d,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB27_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB27_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB27_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1178,12 +1178,12 @@ define void @ccmp_continous(i32 noundef %a, i32 noundef %b, i32 noundef %c) {
 ; NDD-NEXT:    ccmplel {dfv=} $2, %esi # encoding: [0x62,0xf4,0x04,0x0e,0x83,0xfe,0x02]
 ; NDD-NEXT:    ccmpll {dfv=} $3, %edx # encoding: [0x62,0xf4,0x04,0x0c,0x83,0xfa,0x03]
 ; NDD-NEXT:    jge .LBB27_1 # encoding: [0x7d,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB27_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB27_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB27_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -1262,12 +1262,12 @@ define void @ccmp64ri64(i64 noundef %a, i64 noundef %b, i64 noundef %c) {
 ; CHECK-NEXT:    # imm = 0x9167A66BBFE
 ; CHECK-NEXT:    ccmpbeq {dfv=zf} %rax, %rsi # encoding: [0x62,0xf4,0x94,0x06,0x39,0xc6]
 ; CHECK-NEXT:    jg .LBB30_1 # encoding: [0x7f,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB30_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB30_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK-NEXT:  .LBB30_1: # %if.end
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1278,12 +1278,12 @@ define void @ccmp64ri64(i64 noundef %a, i64 noundef %b, i64 noundef %c) {
 ; NDD-NEXT:    # imm = 0x9167A66BBFE
 ; NDD-NEXT:    ccmpbeq {dfv=zf} %rax, %rsi # encoding: [0x62,0xf4,0x94,0x06,0x39,0xc6]
 ; NDD-NEXT:    jg .LBB30_1 # encoding: [0x7f,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: .LBB30_1-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB30_1, kind: FK_PCRel_1
 ; NDD-NEXT:  # %bb.2: # %if.then
 ; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; NDD-NEXT:    jmp foo # TAILCALL
 ; NDD-NEXT:    # encoding: [0xeb,A]
-; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; NDD-NEXT:  .LBB30_1: # %if.end
 ; NDD-NEXT:    retq # encoding: [0xc3]
 entry:
diff --git a/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll b/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll
index e70e5ff80d959..a7fe8983f2f29 100644
--- a/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll
+++ b/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll
@@ -22,8 +22,8 @@ define void @test_fcmp_storei1(i1 %cond, ptr %fptr, ptr %iptr, float %f1, float
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[VCMPSSZrri:%[0-9]+]]:vk1 = nofpexcept VCMPSSZrri [[COPY3]], [[COPY2]], 0, implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vk16 = COPY [[VCMPSSZrri]]
-  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vk32 = COPY [[COPY7]]
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vk16 = COPY killed [[VCMPSSZrri]]
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vk32 = COPY killed [[COPY7]]
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vk8 = COPY [[COPY8]]
   ; CHECK-NEXT:   JMP_1 %bb.3
   ; CHECK-NEXT: {{  $}}
@@ -31,8 +31,8 @@ define void @test_fcmp_storei1(i1 %cond, ptr %fptr, ptr %iptr, float %f1, float
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[VCMPSSZrri1:%[0-9]+]]:vk1 = nofpexcept VCMPSSZrri [[COPY1]], [[COPY]], 0, implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vk16 = COPY [[VCMPSSZrri1]]
-  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vk32 = COPY [[COPY10]]
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vk16 = COPY killed [[VCMPSSZrri1]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vk32 = COPY killed [[COPY10]]
   ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vk8 = COPY [[COPY11]]
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.exit:
diff --git a/llvm/test/CodeGen/X86/apx/kmov-postrapseudos.ll b/llvm/test/CodeGen/X86/apx/kmov-postrapseudos.ll
index b2cb2c3e04b3f..fd811b0bbfc4e 100644
--- a/llvm/test/CodeGen/X86/apx/kmov-postrapseudos.ll
+++ b/llvm/test/CodeGen/X86/apx/kmov-postrapseudos.ll
@@ -8,7 +8,7 @@ define void @kmovkr_1(i1 %cmp23.not) {
 ; AVX512-NEXT:    kmovw %edi, %k1 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x92,0xcf]
 ; AVX512-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} {z} = [1.0E+0,0.0E+0]
 ; AVX512-NEXT:    # encoding: [0x62,0xf1,0xff,0x89,0x10,0x05,A,A,A,A]
-; AVX512-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    vmovsd %xmm0, 0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x11,0x04,0x25,0x00,0x00,0x00,0x00]
 ; AVX512-NEXT:    retq # encoding: [0xc3]
 ;
@@ -17,7 +17,7 @@ define void @kmovkr_1(i1 %cmp23.not) {
 ; AVX512BW-NEXT:    kmovd %edi, %k1 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x92,0xcf]
 ; AVX512BW-NEXT:    vmovsd {{.*#+}} xmm0 {%k1} {z} = [1.0E+0,0.0E+0]
 ; AVX512BW-NEXT:    # encoding: [0x62,0xf1,0xff,0x89,0x10,0x05,A,A,A,A]
-; AVX512BW-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512BW-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512BW-NEXT:    vmovsd %xmm0, 0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x11,0x04,0x25,0x00,0x00,0x00,0x00]
 ; AVX512BW-NEXT:    retq # encoding: [0xc3]
 entry:
@@ -54,7 +54,7 @@ define i32 @kmovrk_1(<4 x ptr> %arg) {
 ; AVX512:       # %bb.0: # %bb
 ; AVX512-NEXT:    vptest %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x17,0xc0]
 ; AVX512-NEXT:    jne .LBB2_1 # encoding: [0x75,A]
-; AVX512-NEXT:    # fixup A - offset: 1, value: .LBB2_1-1, kind: FK_PCRel_1
+; AVX512-NEXT:    # fixup A - offset: 1, value: .LBB2_1, kind: FK_PCRel_1
 ; AVX512-NEXT:  # %bb.2: # %bb3
 ; AVX512-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; AVX512-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
@@ -65,7 +65,7 @@ define i32 @kmovrk_1(<4 x ptr> %arg) {
 ; AVX512BW:       # %bb.0: # %bb
 ; AVX512BW-NEXT:    vptest %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x17,0xc0]
 ; AVX512BW-NEXT:    jne .LBB2_1 # encoding: [0x75,A]
-; AVX512BW-NEXT:    # fixup A - offset: 1, value: .LBB2_1-1, kind: FK_PCRel_1
+; AVX512BW-NEXT:    # fixup A - offset: 1, value: .LBB2_1, kind: FK_PCRel_1
 ; AVX512BW-NEXT:  # %bb.2: # %bb3
 ; AVX512BW-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; AVX512BW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
diff --git a/llvm/test/CodeGen/X86/apx/or.ll b/llvm/test/CodeGen/X86/apx/or.ll
index 12ee5fc3404db..514a7d83b78b0 100644
--- a/llvm/test/CodeGen/X86/apx/or.ll
+++ b/llvm/test/CodeGen/X86/apx/or.ll
@@ -433,7 +433,7 @@ define i1 @orflag8rr(i8 %a, i8 %b) {
 ; CHECK-NEXT:    orb %al, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x08,0xc7]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag8rr:
@@ -442,7 +442,7 @@ define i1 @orflag8rr(i8 %a, i8 %b) {
 ; NF-NEXT:    orb %al, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x08,0xc7]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i8 %b, -1
   %v0 = or i8 %a, %xor  ; 0xff << 50
@@ -458,7 +458,7 @@ define i1 @orflag16rr(i16 %a, i16 %b) {
 ; CHECK-NEXT:    orw %ax, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x09,0xc7]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag16rr:
@@ -467,7 +467,7 @@ define i1 @orflag16rr(i16 %a, i16 %b) {
 ; NF-NEXT:    orw %ax, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x09,0xc7]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 %b, -1
   %v0 = or i16 %a, %xor  ; 0xff << 50
@@ -482,7 +482,7 @@ define i1 @orflag32rr(i32 %a, i32 %b) {
 ; CHECK-NEXT:    orl %edi, %esi # EVEX TO LEGACY Compression encoding: [0x09,0xfe]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag32rr:
@@ -490,7 +490,7 @@ define i1 @orflag32rr(i32 %a, i32 %b) {
 ; NF-NEXT:    orl %edi, %esi # EVEX TO LEGACY Compression encoding: [0x09,0xfe]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = or i32 %a, %b  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -504,7 +504,7 @@ define i1 @orflag64rr(i64 %a, i64 %b) {
 ; CHECK-NEXT:    orq %rdi, %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x09,0xfe]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag64rr:
@@ -512,7 +512,7 @@ define i1 @orflag64rr(i64 %a, i64 %b) {
 ; NF-NEXT:    orq %rdi, %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x09,0xfe]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = or i64 %a, %b  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
@@ -527,7 +527,7 @@ define i1 @orflag8rm(ptr %ptr, i8 %b) {
 ; CHECK-NEXT:    orb (%rdi), %al, %cl # encoding: [0x62,0xf4,0x74,0x18,0x0a,0x07]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag8rm:
@@ -536,7 +536,7 @@ define i1 @orflag8rm(ptr %ptr, i8 %b) {
 ; NF-NEXT:    orb (%rdi), %al, %cl # encoding: [0x62,0xf4,0x74,0x18,0x0a,0x07]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i8, ptr %ptr
   %xor = xor i8 %b, -1
@@ -553,7 +553,7 @@ define i1 @orflag16rm(ptr %ptr, i16 %b) {
 ; CHECK-NEXT:    orw (%rdi), %ax, %cx # encoding: [0x62,0xf4,0x75,0x18,0x0b,0x07]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag16rm:
@@ -562,7 +562,7 @@ define i1 @orflag16rm(ptr %ptr, i16 %b) {
 ; NF-NEXT:    orw (%rdi), %ax, %cx # encoding: [0x62,0xf4,0x75,0x18,0x0b,0x07]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i16, ptr %ptr
   %xor = xor i16 %b, -1
@@ -578,7 +578,7 @@ define i1 @orflag32rm(ptr %ptr, i32 %b) {
 ; CHECK-NEXT:    orl (%rdi), %esi # EVEX TO LEGACY Compression encoding: [0x0b,0x37]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag32rm:
@@ -586,7 +586,7 @@ define i1 @orflag32rm(ptr %ptr, i32 %b) {
 ; NF-NEXT:    orl (%rdi), %esi # EVEX TO LEGACY Compression encoding: [0x0b,0x37]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i32, ptr %ptr
   %v0 = or i32 %a, %b  ; 0xff << 50
@@ -601,7 +601,7 @@ define i1 @orflag64rm(ptr %ptr, i64 %b) {
 ; CHECK-NEXT:    orq (%rdi), %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x0b,0x37]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag64rm:
@@ -609,7 +609,7 @@ define i1 @orflag64rm(ptr %ptr, i64 %b) {
 ; NF-NEXT:    orq (%rdi), %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x0b,0x37]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i64, ptr %ptr
   %v0 = or i64 %a, %b  ; 0xff << 50
@@ -624,7 +624,7 @@ define i1 @orflag8ri(i8 %a) {
 ; CHECK-NEXT:    orb $-124, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xcf,0x84]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag8ri:
@@ -632,7 +632,7 @@ define i1 @orflag8ri(i8 %a) {
 ; NF-NEXT:    orb $-124, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xcf,0x84]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i8 123, -1
   %v0 = or i8 %a, %xor  ; 0xff << 50
@@ -648,7 +648,7 @@ define i1 @orflag16ri(i16 %a) {
 ; CHECK-NEXT:    # imm = 0xFB2D
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag16ri:
@@ -657,7 +657,7 @@ define i1 @orflag16ri(i16 %a) {
 ; NF-NEXT:    # imm = 0xFB2D
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 1234, -1
   %v0 = or i16 %a, %xor  ; 0xff << 50
@@ -673,7 +673,7 @@ define i1 @orflag32ri(i32 %a) {
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag32ri:
@@ -682,7 +682,7 @@ define i1 @orflag32ri(i32 %a) {
 ; NF-NEXT:    # imm = 0x1E240
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = or i32 %a, 123456  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -697,7 +697,7 @@ define i1 @orflag64ri(i64 %a) {
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag64ri:
@@ -706,7 +706,7 @@ define i1 @orflag64ri(i64 %a) {
 ; NF-NEXT:    # imm = 0x1E240
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = or i64 %a, 123456  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
@@ -720,7 +720,7 @@ define i1 @orflag16ri8(i16 %a) {
 ; CHECK-NEXT:    orw $-124, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xcf,0x84]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag16ri8:
@@ -728,7 +728,7 @@ define i1 @orflag16ri8(i16 %a) {
 ; NF-NEXT:    orw $-124, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xcf,0x84]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 123, -1
   %v0 = or i16 %a, %xor  ; 0xff << 50
@@ -743,7 +743,7 @@ define i1 @orflag32ri8(i32 %a) {
 ; CHECK-NEXT:    orl $123, %edi # EVEX TO LEGACY Compression encoding: [0x83,0xcf,0x7b]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag32ri8:
@@ -751,7 +751,7 @@ define i1 @orflag32ri8(i32 %a) {
 ; NF-NEXT:    orl $123, %edi # EVEX TO LEGACY Compression encoding: [0x83,0xcf,0x7b]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = or i32 %a, 123  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -765,7 +765,7 @@ define i1 @orflag64ri8(i64 %a) {
 ; CHECK-NEXT:    orq $123, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x83,0xcf,0x7b]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: orflag64ri8:
@@ -773,7 +773,7 @@ define i1 @orflag64ri8(i64 %a) {
 ; NF-NEXT:    orq $123, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x83,0xcf,0x7b]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = or i64 %a, 123  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
diff --git a/llvm/test/CodeGen/X86/apx/sub.ll b/llvm/test/CodeGen/X86/apx/sub.ll
index 1864ca8279954..d7914577634e7 100644
--- a/llvm/test/CodeGen/X86/apx/sub.ll
+++ b/llvm/test/CodeGen/X86/apx/sub.ll
@@ -750,12 +750,12 @@ define void @sub64ri_reloc(i64 %val) {
 ; CHECK-NEXT:    cmpq $val, %rdi # encoding: [0x48,0x81,0xff,A,A,A,A]
 ; CHECK-NEXT:    # fixup A - offset: 3, value: val, kind: reloc_signed_4byte
 ; CHECK-NEXT:    jbe .LBB41_2 # encoding: [0x76,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB41_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB41_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %t
 ; CHECK-NEXT:    pushq %rax # encoding: [0x50]
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
+; CHECK-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_4
 ; CHECK-NEXT:    popq %rax # encoding: [0x58]
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:  .LBB41_2: # %f
@@ -766,12 +766,12 @@ define void @sub64ri_reloc(i64 %val) {
 ; NF-NEXT:    cmpq $val, %rdi # encoding: [0x48,0x81,0xff,A,A,A,A]
 ; NF-NEXT:    # fixup A - offset: 3, value: val, kind: reloc_signed_4byte
 ; NF-NEXT:    jbe .LBB41_2 # encoding: [0x76,A]
-; NF-NEXT:    # fixup A - offset: 1, value: .LBB41_2-1, kind: FK_PCRel_1
+; NF-NEXT:    # fixup A - offset: 1, value: .LBB41_2, kind: FK_PCRel_1
 ; NF-NEXT:  # %bb.1: # %t
 ; NF-NEXT:    pushq %rax # encoding: [0x50]
 ; NF-NEXT:    .cfi_def_cfa_offset 16
 ; NF-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
+; NF-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_4
 ; NF-NEXT:    popq %rax # encoding: [0x58]
 ; NF-NEXT:    .cfi_def_cfa_offset 8
 ; NF-NEXT:  .LBB41_2: # %f
diff --git a/llvm/test/CodeGen/X86/apx/xor.ll b/llvm/test/CodeGen/X86/apx/xor.ll
index e083c6580398c..e953902aff7ea 100644
--- a/llvm/test/CodeGen/X86/apx/xor.ll
+++ b/llvm/test/CodeGen/X86/apx/xor.ll
@@ -433,7 +433,7 @@ define i1 @xorflag8rr(i8 %a, i8 %b) {
 ; CHECK-NEXT:    xorb $-1, %sil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xf6,0xff]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag8rr:
@@ -442,7 +442,7 @@ define i1 @xorflag8rr(i8 %a, i8 %b) {
 ; NF-NEXT:    xorb $-1, %sil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xf6,0xff]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i8 %b, -1
   %v0 = xor i8 %a, %xor  ; 0xff << 50
@@ -458,7 +458,7 @@ define i1 @xorflag16rr(i16 %a, i16 %b) {
 ; CHECK-NEXT:    xorw $-1, %si, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xf6,0xff]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag16rr:
@@ -467,7 +467,7 @@ define i1 @xorflag16rr(i16 %a, i16 %b) {
 ; NF-NEXT:    xorw $-1, %si, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xf6,0xff]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 %b, -1
   %v0 = xor i16 %a, %xor  ; 0xff << 50
@@ -482,7 +482,7 @@ define i1 @xorflag32rr(i32 %a, i32 %b) {
 ; CHECK-NEXT:    xorl %edi, %esi # EVEX TO LEGACY Compression encoding: [0x31,0xfe]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag32rr:
@@ -490,7 +490,7 @@ define i1 @xorflag32rr(i32 %a, i32 %b) {
 ; NF-NEXT:    xorl %edi, %esi # EVEX TO LEGACY Compression encoding: [0x31,0xfe]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = xor i32 %a, %b  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -504,7 +504,7 @@ define i1 @xorflag64rr(i64 %a, i64 %b) {
 ; CHECK-NEXT:    xorq %rdi, %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x31,0xfe]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag64rr:
@@ -512,7 +512,7 @@ define i1 @xorflag64rr(i64 %a, i64 %b) {
 ; NF-NEXT:    xorq %rdi, %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x31,0xfe]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = xor i64 %a, %b  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
@@ -527,7 +527,7 @@ define i1 @xorflag8rm(ptr %ptr, i8 %b) {
 ; CHECK-NEXT:    xorb $-1, %al, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xf0,0xff]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag8rm:
@@ -536,7 +536,7 @@ define i1 @xorflag8rm(ptr %ptr, i8 %b) {
 ; NF-NEXT:    xorb $-1, %al, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xf0,0xff]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i8, ptr %ptr
   %xor = xor i8 %b, -1
@@ -553,7 +553,7 @@ define i1 @xorflag16rm(ptr %ptr, i16 %b) {
 ; CHECK-NEXT:    xorw $-1, %ax, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xf0,0xff]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag16rm:
@@ -562,7 +562,7 @@ define i1 @xorflag16rm(ptr %ptr, i16 %b) {
 ; NF-NEXT:    xorw $-1, %ax, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xf0,0xff]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i16, ptr %ptr
   %xor = xor i16 %b, -1
@@ -578,7 +578,7 @@ define i1 @xorflag32rm(ptr %ptr, i32 %b) {
 ; CHECK-NEXT:    xorl (%rdi), %esi # EVEX TO LEGACY Compression encoding: [0x33,0x37]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag32rm:
@@ -586,7 +586,7 @@ define i1 @xorflag32rm(ptr %ptr, i32 %b) {
 ; NF-NEXT:    xorl (%rdi), %esi # EVEX TO LEGACY Compression encoding: [0x33,0x37]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %esi, d64(%rip) # encoding: [0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i32, ptr %ptr
   %v0 = xor i32 %a, %b  ; 0xff << 50
@@ -601,7 +601,7 @@ define i1 @xorflag64rm(ptr %ptr, i64 %b) {
 ; CHECK-NEXT:    xorq (%rdi), %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x33,0x37]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag64rm:
@@ -609,7 +609,7 @@ define i1 @xorflag64rm(ptr %ptr, i64 %b) {
 ; NF-NEXT:    xorq (%rdi), %rsi # EVEX TO LEGACY Compression encoding: [0x48,0x33,0x37]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rsi, d64(%rip) # encoding: [0x48,0x89,0x35,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %a = load i64, ptr %ptr
   %v0 = xor i64 %a, %b  ; 0xff << 50
@@ -624,7 +624,7 @@ define i1 @xorflag8ri(i8 %a) {
 ; CHECK-NEXT:    xorb $-124, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xf7,0x84]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag8ri:
@@ -632,7 +632,7 @@ define i1 @xorflag8ri(i8 %a) {
 ; NF-NEXT:    xorb $-124, %dil, %cl # encoding: [0x62,0xf4,0x74,0x18,0x80,0xf7,0x84]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movb %cl, d64(%rip) # encoding: [0x88,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i8 123, -1
   %v0 = xor i8 %a, %xor  ; 0xff << 50
@@ -648,7 +648,7 @@ define i1 @xorflag16ri(i16 %a) {
 ; CHECK-NEXT:    # imm = 0xFB2D
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag16ri:
@@ -657,7 +657,7 @@ define i1 @xorflag16ri(i16 %a) {
 ; NF-NEXT:    # imm = 0xFB2D
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 1234, -1
   %v0 = xor i16 %a, %xor  ; 0xff << 50
@@ -673,7 +673,7 @@ define i1 @xorflag32ri(i32 %a) {
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag32ri:
@@ -682,7 +682,7 @@ define i1 @xorflag32ri(i32 %a) {
 ; NF-NEXT:    # imm = 0x1E240
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = xor i32 %a, 123456  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -697,7 +697,7 @@ define i1 @xorflag64ri(i64 %a) {
 ; CHECK-NEXT:    # imm = 0x1E240
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag64ri:
@@ -706,7 +706,7 @@ define i1 @xorflag64ri(i64 %a) {
 ; NF-NEXT:    # imm = 0x1E240
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = xor i64 %a, 123456  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
@@ -720,7 +720,7 @@ define i1 @xorflag16ri8(i16 %a) {
 ; CHECK-NEXT:    xorw $-124, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xf7,0x84]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag16ri8:
@@ -728,7 +728,7 @@ define i1 @xorflag16ri8(i16 %a) {
 ; NF-NEXT:    xorw $-124, %di, %cx # encoding: [0x62,0xf4,0x75,0x18,0x83,0xf7,0x84]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movw %cx, d64(%rip) # encoding: [0x66,0x89,0x0d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %xor = xor i16 123, -1
   %v0 = xor i16 %a, %xor  ; 0xff << 50
@@ -743,7 +743,7 @@ define i1 @xorflag32ri8(i32 %a) {
 ; CHECK-NEXT:    xorl $123, %edi # EVEX TO LEGACY Compression encoding: [0x83,0xf7,0x7b]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag32ri8:
@@ -751,7 +751,7 @@ define i1 @xorflag32ri8(i32 %a) {
 ; NF-NEXT:    xorl $123, %edi # EVEX TO LEGACY Compression encoding: [0x83,0xf7,0x7b]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movl %edi, d64(%rip) # encoding: [0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 2, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 2, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = xor i32 %a, 123  ; 0xff << 50
   %v1 = icmp eq i32 %v0, 0
@@ -765,7 +765,7 @@ define i1 @xorflag64ri8(i64 %a) {
 ; CHECK-NEXT:    xorq $123, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x83,0xf7,0x7b]
 ; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 ;
 ; NF-LABEL: xorflag64ri8:
@@ -773,7 +773,7 @@ define i1 @xorflag64ri8(i64 %a) {
 ; NF-NEXT:    xorq $123, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x83,0xf7,0x7b]
 ; NF-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; NF-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NF-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NF-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NF-NEXT:    retq # encoding: [0xc3]
   %v0 = xor i64 %a, 123  ; 0xff << 50
   %v1 = icmp eq i64 %v0, 0
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index 0de308a9e0738..217ccebdfb77f 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -1734,20 +1734,20 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-LABEL: not_avg_v16i8_wide_constants:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps (%rdi), %xmm1
-; SSE2-NEXT:    movdqa (%rsi), %xmm0
+; SSE2-NEXT:    movdqa (%rsi), %xmm2
 ; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    movd %eax, %xmm4
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movd %eax, %xmm3
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm5
@@ -1762,9 +1762,6 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-NEXT:    movd %eax, %xmm8
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm10
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm9
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
@@ -1774,6 +1771,9 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-NEXT:    movd %eax, %xmm12
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
+; SSE2-NEXT:    movd %eax, %xmm10
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm13
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
@@ -1783,45 +1783,43 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-NEXT:    movd %eax, %xmm15
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,0,0]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,0,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,0,0,0]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,0,0,0]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm1[0],xmm4[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1]
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
 ; SSE2-NEXT:    movapd %xmm4, %xmm5
 ; SSE2-NEXT:    andpd %xmm1, %xmm5
 ; SSE2-NEXT:    xorpd %xmm4, %xmm1
 ; SSE2-NEXT:    psrlw $1, %xmm1
 ; SSE2-NEXT:    paddw %xmm5, %xmm1
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,0,0,0]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm14[0,0,0,0]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm9[0],xmm2[1]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
-; SSE2-NEXT:    movapd %xmm2, %xmm3
-; SSE2-NEXT:    andpd %xmm0, %xmm3
-; SSE2-NEXT:    xorpd %xmm2, %xmm0
-; SSE2-NEXT:    psrlw $1, %xmm0
-; SSE2-NEXT:    paddw %xmm3, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm10[0],xmm0[1]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; SSE2-NEXT:    movapd %xmm0, %xmm3
+; SSE2-NEXT:    andpd %xmm2, %xmm3
+; SSE2-NEXT:    xorpd %xmm0, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm2
+; SSE2-NEXT:    paddw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
 ; SSE2-NEXT:    movdqu %xmm1, (%rax)
 ; SSE2-NEXT:    retq
 ;
@@ -1831,75 +1829,74 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpextrw $7, %xmm3, %edx
-; AVX1-NEXT:    vpextrw $6, %xmm3, %ecx
-; AVX1-NEXT:    vpextrw $5, %xmm3, %eax
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX1-NEXT:    vpextrd $2, %xmm5, %ecx
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; AVX1-NEXT:    vpextrd $2, %xmm4, %eax
+; AVX1-NEXT:    vpextrw $3, %xmm3, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm4
-; AVX1-NEXT:    vpextrw $4, %xmm3, %edx
-; AVX1-NEXT:    decl %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm5
-; AVX1-NEXT:    vpextrw $1, %xmm3, %ecx
-; AVX1-NEXT:    decl %eax
-; AVX1-NEXT:    vmovd %eax, %xmm6
-; AVX1-NEXT:    vpextrw $0, %xmm3, %eax
+; AVX1-NEXT:    vpextrw $2, %xmm3, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm5
+; AVX1-NEXT:    vpextrw $1, %xmm3, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm6
+; AVX1-NEXT:    vpextrw $0, %xmm3, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm7
-; AVX1-NEXT:    vpextrw $3, %xmm3, %edx
-; AVX1-NEXT:    decq %rcx
-; AVX1-NEXT:    vmovq %rcx, %xmm8
-; AVX1-NEXT:    vpextrw $2, %xmm3, %ecx
-; AVX1-NEXT:    decq %rax
-; AVX1-NEXT:    vmovq %rax, %xmm3
-; AVX1-NEXT:    vpextrw $7, %xmm2, %eax
+; AVX1-NEXT:    vpextrw $3, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm8
+; AVX1-NEXT:    vpextrw $2, %xmm2, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm9
-; AVX1-NEXT:    vpextrw $6, %xmm2, %edx
-; AVX1-NEXT:    decl %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm10
-; AVX1-NEXT:    vpextrw $5, %xmm2, %ecx
-; AVX1-NEXT:    decl %eax
-; AVX1-NEXT:    vmovd %eax, %xmm11
-; AVX1-NEXT:    vpextrw $4, %xmm2, %eax
+; AVX1-NEXT:    vpextrw $1, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm10
+; AVX1-NEXT:    vpextrw $0, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm11
+; AVX1-NEXT:    vpextrw $5, %xmm3, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm12
-; AVX1-NEXT:    vpextrw $1, %xmm2, %edx
+; AVX1-NEXT:    vpextrw $4, %xmm3, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm13
+; AVX1-NEXT:    vpextrw $5, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm14
+; AVX1-NEXT:    vpextrw $4, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm15
+; AVX1-NEXT:    vpextrw $7, %xmm3, %edx
 ; AVX1-NEXT:    decl %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm13
-; AVX1-NEXT:    vpextrw $0, %xmm2, %ecx
-; AVX1-NEXT:    decl %eax
-; AVX1-NEXT:    vmovd %eax, %xmm14
-; AVX1-NEXT:    vpextrw $3, %xmm2, %eax
-; AVX1-NEXT:    decq %rdx
-; AVX1-NEXT:    vmovq %rdx, %xmm15
-; AVX1-NEXT:    vpextrw $2, %xmm2, %edx
-; AVX1-NEXT:    decq %rcx
-; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    vmovd %ecx, %xmm3
+; AVX1-NEXT:    vpextrw $7, %xmm2, %ecx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
 ; AVX1-NEXT:    decl %eax
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX1-NEXT:    vmovd %eax, %xmm5
-; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    decl %ecx
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX1-NEXT:    vmovd %edx, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[0,1,0,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3,4,5],xmm4[6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[0,0,1,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[0,1,0,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3,4,5],xmm4[6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,0,1,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vmovd %ecx, %xmm7
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm4
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm8, %ymm6
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX1-NEXT:    vmovddup {{.*#+}} ymm3 = ymm6[0,0,2,2]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
+; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,0,0,0,4,4,4,4]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3],ymm4[4,5],ymm2[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vandps %ymm0, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm0, %ymm2, %ymm0
diff --git a/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll b/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
index f0203b3b889e4..46f67a8912d4a 100644
--- a/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
@@ -667,7 +667,7 @@ define void @test_x86_sse2_storeu_pd(ptr %a0, <2 x double> %a1) {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x57,0xc9]
 ; X64-AVX-NEXT:    vmovhpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; X64-AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x58,0xc1]
 ; X64-AVX-NEXT:    vmovupd %xmm0, (%rdi) # encoding: [0xc5,0xf9,0x11,0x07]
@@ -677,7 +677,7 @@ define void @test_x86_sse2_storeu_pd(ptr %a0, <2 x double> %a1) {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x57,0xc9]
 ; X64-AVX512VL-NEXT:    vmovhpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; X64-AVX512VL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0xc1]
 ; X64-AVX512VL-NEXT:    vmovupd %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x11,0x07]
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
index c5243a5c18a2d..6b994b0782e6f 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
@@ -39,14 +39,14 @@ define <16 x i16> @test_x86_avx2_packssdw_fold() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packssdw_fold:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
   ret <16 x i16> %res
@@ -90,7 +90,7 @@ define <32 x i8> @test_x86_avx2_packsswb_fold() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x1a,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    # ymm0 = mem[0,1,0,1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -98,7 +98,7 @@ define <32 x i8> @test_x86_avx2_packsswb_fold() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1a,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    # ymm0 = mem[0,1,0,1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
@@ -143,7 +143,7 @@ define <32 x i8> @test_x86_avx2_packuswb_fold() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x1a,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    # ymm0 = mem[0,1,0,1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -151,7 +151,7 @@ define <32 x i8> @test_x86_avx2_packuswb_fold() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1a,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    # ymm0 = mem[0,1,0,1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
@@ -806,14 +806,14 @@ define <16 x i16> @test_x86_avx2_packusdw_fold() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packusdw_fold:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
   ret <16 x i16> %res
@@ -1092,12 +1092,12 @@ define <4 x i32> @test_x86_avx2_psllv_d_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [1,1,1,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x21,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x47,0xc9]
 ; X64-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
@@ -1106,12 +1106,12 @@ define <4 x i32> @test_x86_avx2_psllv_d_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [1,1,1,4294967295]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x21,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x47,0xc9]
 ; X64-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
@@ -1172,14 +1172,14 @@ define <8 x i32> @test_x86_avx2_psllv_d_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x21,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1187,14 +1187,14 @@ define <8 x i32> @test_x86_avx2_psllv_d_256_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res0 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
@@ -1241,18 +1241,18 @@ define <2 x i64> @test_x86_avx2_psllv_q_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbq {{.*#+}} xmm0 = [4,18446744073709551615]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x22,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psllv_q_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbq {{.*#+}} xmm0 = [4,18446744073709551615]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x22,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> <i64 4, i64 -1>, <2 x i64> <i64 1, i64 -1>)
   ret <2 x i64> %res
@@ -1297,18 +1297,18 @@ define <4 x i64> @test_x86_avx2_psllv_q_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbq {{.*#+}} ymm0 = [4,4,4,18446744073709551615]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x22,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psllv_q_256_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbq {{.*#+}} ymm0 = [4,4,4,18446744073709551615]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x22,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
   ret <4 x i64> %res
@@ -1365,14 +1365,14 @@ define <4 x i32> @test_x86_avx2_psrlv_d_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [4,4,4,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x21,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1380,14 +1380,14 @@ define <4 x i32> @test_x86_avx2_psrlv_d_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [4,4,4,4294967295]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x21,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res0 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
@@ -1447,14 +1447,14 @@ define <8 x i32> @test_x86_avx2_psrlv_d_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x21,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1462,14 +1462,14 @@ define <8 x i32> @test_x86_avx2_psrlv_d_256_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res0 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
@@ -1517,18 +1517,18 @@ define <2 x i64> @test_x86_avx2_psrlv_q_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbq {{.*#+}} xmm0 = [4,4]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x22,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_q_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [4,4]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> <i64 4, i64 4>, <2 x i64> <i64 1, i64 -1>)
   ret <2 x i64> %res
@@ -1574,18 +1574,18 @@ define <4 x i64> @test_x86_avx2_psrlv_q_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [4,4,4,4]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x59,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_q_256_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [4,4,4,4]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x59,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 4>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
   ret <4 x i64> %res
@@ -1630,18 +1630,18 @@ define <4 x i32> @test_x86_avx2_psrav_d_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [2,9,4294967284,23]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrav_d_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [2,9,4294967284,23]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> <i32 2, i32 9, i32 -12, i32 23>, <4 x i32> <i32 1, i32 18, i32 35, i32 52>)
   ret <4 x i32> %res
@@ -1685,18 +1685,18 @@ define <8 x i32> @test_x86_avx2_psrav_d_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrav_d_256_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>)
   ret <8 x i32> %res
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index aae48aba93be6..6755ee0b174df 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -5289,14 +5289,14 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[2,3,0,1,7,6,5,4,9,8,11,10,12,13,14,15]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x49,0x0c,0x15,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpermilps {{.*#+}} zmm1 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0xc9,0x0c,0x0d,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vaddps %zmm1, %zmm2, %zmm1 ## encoding: [0x62,0xf1,0x6c,0x48,0x58,0xc9]
 ; X64-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x48,0x0c,0x05,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>, <16 x float> %x2, i16 %x3)
diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
index 210513fe31783..78b0199f2ed38 100644
--- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -1196,7 +1196,7 @@ define <2 x i64> @test45(<2 x i16> %x, <2 x i16> %y) #0 {
 ; AVX512-NEXT:    vpmovzxwq %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x34,0xc0]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test45:
@@ -1217,7 +1217,7 @@ define <2 x i64> @test46(<2 x float> %x, <2 x float> %y) #0 {
 ; AVX512-NEXT:    vshufps $212, %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0xc6,0xc0,0xd4]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0,1,1,3]
 ; AVX512-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0x54,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test46:
@@ -1461,7 +1461,7 @@ define void @half_vec_compare(ptr %x, ptr %y) {
 ; KNL-NEXT:    vpinsrw $1, %edx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc2,0x01]
 ; KNL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0]
 ; KNL-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
-; KNL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; KNL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; KNL-NEXT:    vpextrw $0, %xmm0, (%rsi) ## encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
 ; KNL-NEXT:    retq ## encoding: [0xc3]
 ;
@@ -1487,7 +1487,7 @@ define void @half_vec_compare(ptr %x, ptr %y) {
 ; AVX512BW-NEXT:    vpinsrw $1, %edx, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc2,0x01]
 ; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0]
 ; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
-; AVX512BW-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512BW-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi) ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
 ; AVX512BW-NEXT:    retq ## encoding: [0xc3]
 ;
@@ -1500,7 +1500,7 @@ define void @half_vec_compare(ptr %x, ptr %y) {
 ; SKX-NEXT:    vcmpneqps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x04]
 ; SKX-NEXT:    vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; SKX-NEXT:    ## encoding: [0x62,0xf1,0x7f,0x89,0x6f,0x05,A,A,A,A]
-; SKX-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; SKX-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; SKX-NEXT:    vpextrw $0, %xmm0, (%rsi) ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00]
 ; SKX-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; SKX-NEXT:    retq ## encoding: [0xc3]
@@ -1519,7 +1519,7 @@ define <8 x i64> @cmp_swap_bug(ptr %x, <8 x i64> %y, <8 x i64> %z) {
 ; KNL-NEXT:    vmovdqa (%rdi), %xmm2 ## encoding: [0xc5,0xf9,0x6f,0x17]
 ; KNL-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; KNL-NEXT:    ## encoding: [0xc4,0xe2,0x69,0x00,0x15,A,A,A,A]
-; KNL-NEXT:    ## fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; KNL-NEXT:    ## fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; KNL-NEXT:    vpxor %xmm3, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0xef,0xdb]
 ; KNL-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm2 ## encoding: [0xc5,0xe1,0x64,0xd2]
 ; KNL-NEXT:    vpmovsxbd %xmm2, %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x21,0xd2]
@@ -1532,7 +1532,7 @@ define <8 x i64> @cmp_swap_bug(ptr %x, <8 x i64> %y, <8 x i64> %z) {
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm2 ## encoding: [0xc5,0xf9,0x6f,0x17]
 ; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
 ; AVX512BW-NEXT:    ## encoding: [0xc4,0xe2,0x69,0x00,0x15,A,A,A,A]
-; AVX512BW-NEXT:    ## fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512BW-NEXT:    ## fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512BW-NEXT:    vpmovb2m %zmm2, %k1 ## encoding: [0x62,0xf2,0x7e,0x48,0x29,0xca]
 ; AVX512BW-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x64,0xc0]
 ; AVX512BW-NEXT:    retq ## encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512bf16-vl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bf16-vl-intrinsics.ll
index 482713e12d15c..b3c261147c773 100644
--- a/llvm/test/CodeGen/X86/avx512bf16-vl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bf16-vl-intrinsics.ll
@@ -394,7 +394,7 @@ define <16 x i32> @pr83358() {
 ; X64-LABEL: pr83358:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vcvtneps2bf16y {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x72,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vshufi64x2 $0, %zmm0, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x43,0xc0,0x00]
 ; X64-NEXT:    # zmm0 = zmm0[0,1,0,1,0,1,0,1]
 ; X64-NEXT:    retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
index 41e2aa003ce7a..5b58233ac51aa 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
@@ -1305,9 +1305,9 @@ define <32 x i16> @test_x86_avx512_psrlv_w_512_const() optsize {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0x62,0xf2,0x7d,0x48,0x20,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <32 x i16> %res1
@@ -1424,10 +1424,10 @@ define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi_const(<32 x i16> %x0, <32
 ; X64:       # %bb.0:
 ; X64-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51]
 ; X64-NEXT:    # encoding: [0x62,0xf2,0x7d,0x48,0x5a,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; X64-NEXT:    vpsravw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> <i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51>, <32 x i16> <i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49>)
   ret <32 x i16> %1
@@ -1590,9 +1590,9 @@ define <32 x i16> @test_x86_avx512_psllv_w_512_const() optsize {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0x62,0xf2,0x7d,0x48,0x20,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4,  i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 -1>)
   ret <32 x i16> %res1
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
index f76b96eda7540..01b153b4fbc61 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
@@ -2167,9 +2167,9 @@ define <8 x i16> @test_int_x86_avx512_psrlv_w_128_const() optsize {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbw {{.*#+}} xmm0 = [4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x20,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <8 x i16> %res
@@ -2191,9 +2191,9 @@ define <16 x i16> @test_int_x86_avx512_psrlv_w_256_const() optsize {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbw {{.*#+}} ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x20,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <16 x i16> %res
@@ -2411,9 +2411,9 @@ define <8 x i16> @test_int_x86_avx512_psllv_w_128_const() optsize {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbw {{.*#+}} xmm0 = [4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x20,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <8 x i16> %res
@@ -2436,9 +2436,9 @@ define <16 x i16> @test_int_x86_avx512_psllv_w_256_const() optsize {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbw {{.*#+}} ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x20,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <16 x i16> %res
diff --git a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
index 3f6ddc6ecfd70..f66f0c0ceabc4 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
@@ -1054,4 +1054,22 @@ define <8 x half> @u64tof16(<8 x i64> %a) #0 {
   ret <8 x half> %1
 }
 
+define void @f16tou32(ptr %ptr) {
+; X64-LABEL: f16tou32:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvttph2udq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0
+; X64-NEXT:    vmovlps %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X86-LABEL: f16tou32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vcvttph2udq {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0
+; X86-NEXT:    vmovlps %xmm0, (%eax)
+; X86-NEXT:    retl
+  %1 = fptoui <2 x half> splat (half 0xH7E00) to <2 x i32>
+  store <2 x i32> %1, ptr %ptr, align 8
+  ret void
+}
+
 attributes #0 = { "min-legal-vector-width"="256" "prefer-vector-width"="256" }
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll b/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
index eac803f83e863..2d3b28d255b22 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fmaxnum.ll
@@ -497,7 +497,7 @@ define half @test_maxnum_const_op1(half %x) {
 ; CHECK-LABEL: test_maxnum_const_op1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7e,0x08,0x5f,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %r = call half @llvm.maxnum.f16(half 1.0, half %x)
   ret half %r
@@ -507,7 +507,7 @@ define half @test_maxnum_const_op2(half %x) {
 ; CHECK-LABEL: test_maxnum_const_op2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7e,0x08,0x5f,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %r = call half @llvm.maxnum.f16(half %x, half 1.0)
   ret half %r
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
index 05509e0099c3c..768e0977d88e0 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-fminnum.ll
@@ -497,7 +497,7 @@ define half @test_minnum_const_op1(half %x) {
 ; CHECK-LABEL: test_minnum_const_op1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vminsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7e,0x08,0x5d,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %r = call half @llvm.minnum.f16(half 1.0, half %x)
   ret half %r
@@ -507,7 +507,7 @@ define half @test_minnum_const_op2(half %x) {
 ; CHECK-LABEL: test_minnum_const_op2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vminsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf5,0x7e,0x08,0x5d,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %r = call half @llvm.minnum.f16(half %x, half 1.0)
   ret half %r
diff --git a/llvm/test/CodeGen/X86/avx512vl-arith.ll b/llvm/test/CodeGen/X86/avx512vl-arith.ll
index 1006c5625e26c..b03a8c1a4f51b 100644
--- a/llvm/test/CodeGen/X86/avx512vl-arith.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-arith.ll
@@ -27,7 +27,7 @@ define <4 x i64> @vpaddq256_broadcast_test(<4 x i64> %i) nounwind {
 ; CHECK-LABEL: vpaddq256_broadcast_test:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x38,0xd4,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <4 x i64> %i, <i64 2, i64 2, i64 2, i64 2>
   ret <4 x i64> %x
@@ -68,7 +68,7 @@ define <8 x i32> @vpaddd256_broadcast_test(<8 x i32> %i) nounwind {
 ; CHECK-LABEL: vpaddd256_broadcast_test:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x38,0xfe,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <8 x i32> %i, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <8 x i32> %x
@@ -116,7 +116,7 @@ define <8 x i32> @vpaddd256_mask_broadcast_test(<8 x i32> %i, <8 x i32> %mask1)
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vptestmd %ymm1, %ymm1, %k1 ## encoding: [0x62,0xf2,0x75,0x28,0x27,0xc9]
 ; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x39,0xfe,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = add <8 x i32> %i, <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
@@ -142,7 +142,7 @@ define <8 x i32> @vpaddd256_maskz_broadcast_test(<8 x i32> %i, <8 x i32> %mask1)
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vptestmd %ymm1, %ymm1, %k1 ## encoding: [0x62,0xf2,0x75,0x28,0x27,0xc9]
 ; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xb9,0xfe,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <8 x i32> %mask1, zeroinitializer
   %x = add <8 x i32> %i, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
@@ -191,7 +191,7 @@ define <4 x double> @test_fold_vaddpd_256(<4 x double> %y) {
 ; CHECK-LABEL: test_fold_vaddpd_256:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    vaddpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x58,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %add.i = fadd <4 x double> %y, <double 4.500000e+00, double 3.400000e+00, double 4.500000e+00, double 5.600000e+00>
@@ -202,7 +202,7 @@ define <8 x float> @test_broadcast_vaddpd_256(<8 x float> %a) nounwind {
 ; CHECK-LABEL: test_broadcast_vaddpd_256:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7c,0x38,0x58,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = fadd <8 x float> %a, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
   ret <8 x float> %b
@@ -493,7 +493,7 @@ define <4 x i32> @vpaddd128_broadcast_test(<4 x i32> %i) nounwind {
 ; CHECK-LABEL: vpaddd128_broadcast_test:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x18,0xfe,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %x = add <4 x i32> %i, <i32 6, i32 6, i32 6, i32 6>
   ret <4 x i32> %x
@@ -541,7 +541,7 @@ define <4 x i32> @vpaddd128_mask_broadcast_test(<4 x i32> %i, <4 x i32> %mask1)
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vptestmd %xmm1, %xmm1, %k1 ## encoding: [0x62,0xf2,0x75,0x08,0x27,0xc9]
 ; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x19,0xfe,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = add <4 x i32> %i, <i32 7, i32 7, i32 7, i32 7>
@@ -567,7 +567,7 @@ define <4 x i32> @vpaddd128_maskz_broadcast_test(<4 x i32> %i, <4 x i32> %mask1)
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vptestmd %xmm1, %xmm1, %k1 ## encoding: [0x62,0xf2,0x75,0x08,0x27,0xc9]
 ; CHECK-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xfe,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = icmp ne <4 x i32> %mask1, zeroinitializer
   %x = add <4 x i32> %i, <i32 8, i32 8, i32 8, i32 8>
@@ -616,7 +616,7 @@ define <2 x double> @test_fold_vaddpd_128(<2 x double> %y) {
 ; CHECK-LABEL: test_fold_vaddpd_128:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    vaddpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %add.i = fadd <2 x double> %y, <double 4.500000e+00, double 3.400000e+00>
@@ -627,7 +627,7 @@ define <4 x float> @test_broadcast_vaddpd_128(<4 x float> %a) nounwind {
 ; CHECK-LABEL: test_broadcast_vaddpd_128:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7c,0x18,0x58,0x05,A,A,A,A]
-; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = fadd <4 x float> %a, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
   ret <4 x float> %b
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index e8e22bae23c92..99f666072fbbc 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -7334,9 +7334,9 @@ define <8 x i32>@test_int_x86_avx512_mask_psrav8_si_const() {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>, <8 x i32> zeroinitializer, i8 -1)
   ret <8 x i32> %res
@@ -8647,9 +8647,9 @@ define <2 x i64>@test_int_x86_avx512_mask_psrav_q_128_const(i8 %x3) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbq {{.*#+}} xmm0 = [2,18446744073709551607]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x22,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsravq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> <i64 2, i64 -9>, <2 x i64> <i64 1, i64 90>, <2 x i64> zeroinitializer, i8 -1)
   ret <2 x i64> %res
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
index b408aac218108..7fe32c293420d 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -7042,7 +7042,7 @@ define <8 x i32> @combine_vpermi2d_vpermps(<16 x i32> noundef %a) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [14,13,6,3,5,15,0,1]
 ; X64-NEXT:    # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x21,0x0d,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x16,0xc0]
 ; X64-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; X64-NEXT:    retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/bmi.ll b/llvm/test/CodeGen/X86/bmi.ll
index 2683fab59ad1b..4041934d404e2 100644
--- a/llvm/test/CodeGen/X86/bmi.ll
+++ b/llvm/test/CodeGen/X86/bmi.ll
@@ -1584,7 +1584,7 @@ define void @pr40060(i32, i32) {
 ; EGPR-NEXT:    testl %eax, %eax # encoding: [0x85,0xc0]
 ; EGPR-NEXT:    jns bar # TAILCALL
 ; EGPR-NEXT:    # encoding: [0x79,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    retq # encoding: [0xc3]
   %3 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %0, i32 %1)
@@ -1635,10 +1635,10 @@ define i32 @blsr32_branch(i32 %x) {
 ; EGPR-NEXT:    .cfi_offset %rbx, -16
 ; EGPR-NEXT:    blsrl %edi, %ebx # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x60,0xf3,0xcf]
 ; EGPR-NEXT:    jne .LBB53_2 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB53_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB53_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    callq bar # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:  .LBB53_2:
 ; EGPR-NEXT:    movl %ebx, %eax # encoding: [0x89,0xd8]
 ; EGPR-NEXT:    popq %rbx # encoding: [0x5b]
@@ -1707,10 +1707,10 @@ define i64 @blsr64_branch(i64 %x) {
 ; EGPR-NEXT:    .cfi_offset %rbx, -16
 ; EGPR-NEXT:    blsrq %rdi, %rbx # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xe0,0xf3,0xcf]
 ; EGPR-NEXT:    jne .LBB54_2 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB54_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB54_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    callq bar # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:  .LBB54_2:
 ; EGPR-NEXT:    movq %rbx, %rax # encoding: [0x48,0x89,0xd8]
 ; EGPR-NEXT:    popq %rbx # encoding: [0x5b]
@@ -1764,10 +1764,10 @@ define i32 @blsi32_branch(i32 %x) {
 ; EGPR-NEXT:    .cfi_offset %rbx, -16
 ; EGPR-NEXT:    blsil %edi, %ebx # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x60,0xf3,0xdf]
 ; EGPR-NEXT:    jne .LBB55_2 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB55_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB55_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    callq bar # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:  .LBB55_2:
 ; EGPR-NEXT:    movl %ebx, %eax # encoding: [0x89,0xd8]
 ; EGPR-NEXT:    popq %rbx # encoding: [0x5b]
@@ -1836,10 +1836,10 @@ define i64 @blsi64_branch(i64 %x) {
 ; EGPR-NEXT:    .cfi_offset %rbx, -16
 ; EGPR-NEXT:    blsiq %rdi, %rbx # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xe0,0xf3,0xdf]
 ; EGPR-NEXT:    jne .LBB56_2 # encoding: [0x75,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB56_2-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: .LBB56_2, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    callq bar # encoding: [0xe8,A,A,A,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-4, kind: reloc_branch_4byte_pcrel
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: reloc_branch_4byte_pcrel
 ; EGPR-NEXT:  .LBB56_2:
 ; EGPR-NEXT:    movq %rbx, %rax # encoding: [0x48,0x89,0xd8]
 ; EGPR-NEXT:    popq %rbx # encoding: [0x5b]
@@ -1878,7 +1878,7 @@ define void @pr42118_i32(i32 %x) {
 ; EGPR-NEXT:    testl %eax, %eax # encoding: [0x85,0xc0]
 ; EGPR-NEXT:    je bar # TAILCALL
 ; EGPR-NEXT:    # encoding: [0x74,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    retq # encoding: [0xc3]
   %tmp = sub i32 0, %x
@@ -1931,7 +1931,7 @@ define void @pr42118_i64(i64 %x) {
 ; EGPR-NEXT:    testq %rax, %rax # encoding: [0x48,0x85,0xc0]
 ; EGPR-NEXT:    je bar # TAILCALL
 ; EGPR-NEXT:    # encoding: [0x74,A]
-; EGPR-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; EGPR-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; EGPR-NEXT:  # %bb.1:
 ; EGPR-NEXT:    retq # encoding: [0xc3]
   %tmp = sub i64 0, %x
diff --git a/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll b/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
index ba09ba8b6402b..2aa79fafe59a5 100644
--- a/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
+++ b/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
@@ -4,7 +4,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown \
 ; RUN:   | FileCheck %s --check-prefix=CHECK --check-prefix=SAFE
 
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -enable-unsafe-fp-math \
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown \
 ; RUN:   | FileCheck %s --check-prefix=CHECK --check-prefix=UNSAFE
 
 ; The div in these functions should be converted to a mul when unsafe-fp-math
@@ -12,14 +12,19 @@
 
 ; CHECK-LABEL: unsafe_fp_math_default0:
 define double @unsafe_fp_math_default0(double %x) {
-; SAFE:      divsd
 ; UNSAFE:    mulsd
+  %div = fdiv arcp double %x, 3.0
+  ret double %div
+}
+; CHECK-LABEL: safe_fp_math_default0:
+define double @safe_fp_math_default0(double %x) {
+; SAFE:      divsd
   %div = fdiv double %x, 3.0
   ret double %div
 }
 
 ; CHECK-LABEL: unsafe_fp_math_off:
-define double @unsafe_fp_math_off(double %x) #0 {
+define double @unsafe_fp_math_off(double %x) {
 ; SAFE:      divsd
 ; UNSAFE:    divsd
   %div = fdiv double %x, 3.0
@@ -29,28 +34,37 @@ define double @unsafe_fp_math_off(double %x) #0 {
 ; CHECK-LABEL: unsafe_fp_math_default1:
 define double @unsafe_fp_math_default1(double %x) {
 ; With unsafe math enabled, can change this div to a mul.
-; SAFE:      divsd
 ; UNSAFE:    mulsd
+  %div = fdiv arcp double %x, 3.0
+  ret double %div
+}
+; CHECK-LABEL: safe_fp_math_default1:
+define double @safe_fp_math_default1(double %x) {
+; With unsafe math enabled, can change this div to a mul.
+; SAFE:      divsd
   %div = fdiv double %x, 3.0
   ret double %div
 }
 
 ; CHECK-LABEL: unsafe_fp_math_on:
-define double @unsafe_fp_math_on(double %x) #1 {
+define double @unsafe_fp_math_on(double %x) {
 ; SAFE:      mulsd
 ; UNSAFE:    mulsd
-  %div = fdiv double %x, 3.0
+  %div = fdiv arcp double %x, 3.0
   ret double %div
 }
 
 ; CHECK-LABEL: unsafe_fp_math_default2:
 define double @unsafe_fp_math_default2(double %x) {
 ; With unsafe math enabled, can change this div to a mul.
-; SAFE:      divsd
 ; UNSAFE:    mulsd
+  %div = fdiv arcp double %x, 3.0
+  ret double %div
+}
+; CHECK-LABEL: safe_fp_math_default2:
+define double @safe_fp_math_default2(double %x) {
+; With unsafe math enabled, can change this div to a mul.
+; SAFE:      divsd
   %div = fdiv double %x, 3.0
   ret double %div
 }
-
-attributes #0 = { "unsafe-fp-math"="false" }
-attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/cmp.ll b/llvm/test/CodeGen/X86/cmp.ll
index 0965b1c7208f6..f3e141740b287 100644
--- a/llvm/test/CodeGen/X86/cmp.ll
+++ b/llvm/test/CodeGen/X86/cmp.ll
@@ -10,7 +10,7 @@ define i32 @test1(i32 %X, ptr %y) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    cmpl $0, (%rsi) # encoding: [0x83,0x3e,0x00]
 ; CHECK-NEXT:    je .LBB0_2 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB0_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %cond_true
 ; CHECK-NEXT:    movl $1, %eax # encoding: [0xb8,0x01,0x00,0x00,0x00]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
@@ -35,7 +35,7 @@ define i32 @test2(i32 %X, ptr %y) nounwind {
 ; CHECK-NEXT:    testl $536870911, (%rsi) # encoding: [0xf7,0x06,0xff,0xff,0xff,0x1f]
 ; CHECK-NEXT:    # imm = 0x1FFFFFFF
 ; CHECK-NEXT:    je .LBB1_2 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %cond_true
 ; CHECK-NEXT:    movl $1, %eax # encoding: [0xb8,0x01,0x00,0x00,0x00]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
@@ -60,7 +60,7 @@ define i8 @test2b(i8 %X, ptr %y) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    testb $31, (%rsi) # encoding: [0xf6,0x06,0x1f]
 ; CHECK-NEXT:    je .LBB2_2 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %cond_true
 ; CHECK-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
@@ -109,14 +109,14 @@ define i32 @test5(double %A) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    ucomisd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # encoding: [0x66,0x0f,0x2e,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    ja .LBB5_3 # encoding: [0x77,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_3-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_3, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %entry
 ; CHECK-NEXT:    ucomisd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # encoding: [0x66,0x0f,0x2e,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jb .LBB5_3 # encoding: [0x72,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_3-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_3, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %bb12
 ; CHECK-NEXT:    movl $32, %eax # encoding: [0xb8,0x20,0x00,0x00,0x00]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
@@ -124,7 +124,7 @@ define i32 @test5(double %A) nounwind {
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo@PLT, kind: FK_PCRel_1
 entry:
   %tmp2 = fcmp ogt double %A, 1.500000e+02
   %tmp5 = fcmp ult double %A, 7.500000e+01
@@ -146,7 +146,7 @@ define i32 @test6() nounwind align 2 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    cmpq $0, -{{[0-9]+}}(%rsp) # encoding: [0x48,0x83,0x7c,0x24,0xf8,0x00]
 ; CHECK-NEXT:    je .LBB6_1 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB6_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB6_1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %F
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
@@ -274,10 +274,10 @@ define i32 @test12() ssp uwtable {
 ; CHECK-NEXT:    pushq %rax # encoding: [0x50]
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq test12b@PLT # encoding: [0xe8,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: test12b@PLT-4, kind: FK_PCRel_4
+; CHECK-NEXT:    # fixup A - offset: 1, value: test12b@PLT, kind: FK_PCRel_4
 ; CHECK-NEXT:    testb %al, %al # encoding: [0x84,0xc0]
 ; CHECK-NEXT:    je .LBB12_2 # encoding: [0x74,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB12_2-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB12_2, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %T
 ; CHECK-NEXT:    movl $1, %eax # encoding: [0xb8,0x01,0x00,0x00,0x00]
 ; CHECK-NEXT:    popq %rcx # encoding: [0x59]
@@ -428,7 +428,7 @@ define void @test20(i32 %bf.load, i8 %x1, ptr %b_addr) {
 ; NO-NDD-NEXT:    testl $16777215, %edi # encoding: [0xf7,0xc7,0xff,0xff,0xff,0x00]
 ; NO-NDD-NEXT:    # imm = 0xFFFFFF
 ; NO-NDD-NEXT:    setne d(%rip) # encoding: [0x0f,0x95,0x05,A,A,A,A]
-; NO-NDD-NEXT:    # fixup A - offset: 3, value: d-4, kind: reloc_riprel_4byte
+; NO-NDD-NEXT:    # fixup A - offset: 3, value: d, kind: reloc_riprel_4byte
 ; NO-NDD-NEXT:    retq # encoding: [0xc3]
 ;
 ; NDD-LABEL: test20:
@@ -443,7 +443,7 @@ define void @test20(i32 %bf.load, i8 %x1, ptr %b_addr) {
 ; NDD-NEXT:    testl $16777215, %edi # encoding: [0xf7,0xc7,0xff,0xff,0xff,0x00]
 ; NDD-NEXT:    # imm = 0xFFFFFF
 ; NDD-NEXT:    setne d(%rip) # encoding: [0x0f,0x95,0x05,A,A,A,A]
-; NDD-NEXT:    # fixup A - offset: 3, value: d-4, kind: reloc_riprel_4byte
+; NDD-NEXT:    # fixup A - offset: 3, value: d, kind: reloc_riprel_4byte
 ; NDD-NEXT:    retq # encoding: [0xc3]
   %bf.shl = shl i32 %bf.load, 8
   %bf.ashr = ashr exact i32 %bf.shl, 8
@@ -794,7 +794,7 @@ define i1 @shifted_mask64_extra_use_const(i64 %a) {
 ; CHECK-NEXT:    testq %rcx, %rdi # encoding: [0x48,0x85,0xcf]
 ; CHECK-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
 ; CHECK-NEXT:    movq %rcx, d64(%rip) # encoding: [0x48,0x89,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 287104476244869120  ; 0xff << 50
   %v1 = icmp ne i64 %v0, 0
@@ -810,7 +810,7 @@ define i1 @shifted_mask64_extra_use_and(i64 %a) {
 ; NO-NDD-NEXT:    andq %rdi, %rcx # encoding: [0x48,0x21,0xf9]
 ; NO-NDD-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
 ; NO-NDD-NEXT:    movq %rcx, d64(%rip) # encoding: [0x48,0x89,0x0d,A,A,A,A]
-; NO-NDD-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NO-NDD-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NO-NDD-NEXT:    retq # encoding: [0xc3]
 ;
 ; NDD-LABEL: shifted_mask64_extra_use_and:
@@ -820,7 +820,7 @@ define i1 @shifted_mask64_extra_use_and(i64 %a) {
 ; NDD-NEXT:    andq %rax, %rdi # EVEX TO LEGACY Compression encoding: [0x48,0x21,0xc7]
 ; NDD-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
 ; NDD-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NDD-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NDD-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NDD-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 287104476244869120  ; 0xff << 50
   %v1 = icmp ne i64 %v0, 0
@@ -847,7 +847,7 @@ define i1 @shifted_mask32_extra_use_const(i64 %a) {
 ; CHECK-NEXT:    # imm = 0x3FC0000
 ; CHECK-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
 ; CHECK-NEXT:    movq $66846720, d64(%rip) # encoding: [0x48,0xc7,0x05,A,A,A,A,0x00,0x00,0xfc,0x03]
-; CHECK-NEXT:    # fixup A - offset: 3, value: d64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x3FC0000
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 66846720  ; 0xff << 18
@@ -863,7 +863,7 @@ define i1 @shifted_mask32_extra_use_and(i64 %a) {
 ; NO-NDD-NEXT:    # imm = 0x3FC0000
 ; NO-NDD-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
 ; NO-NDD-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NO-NDD-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NO-NDD-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NO-NDD-NEXT:    retq # encoding: [0xc3]
 ;
 ; NDD-LABEL: shifted_mask32_extra_use_and:
@@ -872,7 +872,7 @@ define i1 @shifted_mask32_extra_use_and(i64 %a) {
 ; NDD-NEXT:    # imm = 0x3FC0000
 ; NDD-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
 ; NDD-NEXT:    movq %rdi, d64(%rip) # encoding: [0x48,0x89,0x3d,A,A,A,A]
-; NDD-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; NDD-NEXT:    # fixup A - offset: 3, value: d64, kind: reloc_riprel_4byte
 ; NDD-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 66846720  ; 0xff << 50
   %v1 = icmp ne i64 %v0, 0
@@ -915,11 +915,11 @@ define i32 @pr42189(i16 signext %c) {
 ; CHECK-NEXT:    # imm = 0x7FFF
 ; CHECK-NEXT:    jne f@PLT # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: f@PLT-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: f@PLT, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    jmp g@PLT # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: g@PLT-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: g@PLT, kind: FK_PCRel_1
 entry:
   %cmp = icmp eq i16 %c, 32767
   br i1 %cmp, label %if.then, label %if.end
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
index b3d5d836874cb..c4c194e139828 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
@@ -13,33 +13,33 @@ define void @f(i32 %x, i32 %y) !prof !14 {
 ; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x08]
 ; CHECK32-NEXT:    jne bar@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0x75,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar@PLT, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %bb1
 ; CHECK32-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo@PLT, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: f:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
 ; CHECK64-NEXT:    jne bar@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x75,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar@PLT, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %bb1
 ; CHECK64-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo@PLT, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: f:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    cmpl %edx, %ecx # encoding: [0x39,0xd1]
 ; WIN64-NEXT:    jne bar # TAILCALL
 ; WIN64-NEXT:    # encoding: [0x75,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %bb1
 ; WIN64-NEXT:    jmp foo # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 entry:
 	%p = icmp eq i32 %x, %y
   br i1 %p, label %bb1, label %bb2
@@ -64,20 +64,20 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; CHECK32-NEXT:    #NO_APP
 ; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x0c]
 ; CHECK32-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %bb1
 ; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK32-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo@PLT, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB1_2: # %bb2
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK32-NEXT:    jmp bar@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar@PLT, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: f_non_leaf:
 ; CHECK64:       # %bb.0: # %entry
@@ -88,20 +88,20 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; CHECK64-NEXT:    #NO_APP
 ; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
 ; CHECK64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %bb1
 ; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK64-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo@PLT, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB1_2: # %bb2
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK64-NEXT:    jmp bar@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar@PLT, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: f_non_leaf:
 ; WIN64:       # %bb.0: # %entry
@@ -112,14 +112,14 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; WIN64-NEXT:    #NO_APP
 ; WIN64-NEXT:    cmpl %edx, %ecx # encoding: [0x39,0xd1]
 ; WIN64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %bb1
 ; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
 ; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp foo # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB1_2: # %bb2
 ; WIN64-NEXT:    nop # encoding: [0x90]
 ; WIN64-NEXT:    .seh_startepilogue
@@ -127,7 +127,7 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp bar # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; WIN64-NEXT:    .seh_endproc
 entry:
   ; Force %ebx to be spilled on the stack, turning this into
@@ -152,58 +152,58 @@ define x86_thiscallcc zeroext i1 @BlockPlacementTest(ptr %this, i32 %x) !prof !1
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x04]
 ; CHECK32-NEXT:    testb $42, %dl # encoding: [0xf6,0xc2,0x2a]
 ; CHECK32-NEXT:    je .LBB2_3 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_3, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %land.rhs
 ; CHECK32-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK32-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
 ; CHECK32-NEXT:    je baz@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: baz@PLT-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: baz@PLT, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB2_2: # %land.end
 ; CHECK32-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK32-NEXT:    retl $4 # encoding: [0xc2,0x04,0x00]
 ; CHECK32-NEXT:  .LBB2_3:
 ; CHECK32-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK32-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: BlockPlacementTest:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    testb $42, %sil # encoding: [0x40,0xf6,0xc6,0x2a]
 ; CHECK64-NEXT:    je .LBB2_3 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_3, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %land.rhs
 ; CHECK64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK64-NEXT:    testb $44, %sil # encoding: [0x40,0xf6,0xc6,0x2c]
 ; CHECK64-NEXT:    je baz@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: baz@PLT-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: baz@PLT, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB2_2: # %land.end
 ; CHECK64-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK64-NEXT:    retq # encoding: [0xc3]
 ; CHECK64-NEXT:  .LBB2_3:
 ; CHECK64-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK64-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: BlockPlacementTest:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    testb $42, %dl # encoding: [0xf6,0xc2,0x2a]
 ; WIN64-NEXT:    je .LBB2_3 # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_3, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %land.rhs
 ; WIN64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; WIN64-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
 ; WIN64-NEXT:    je baz # TAILCALL
 ; WIN64-NEXT:    # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: baz, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB2_2: # %land.end
 ; WIN64-NEXT:    # kill: def $al killed $al killed $eax
 ; WIN64-NEXT:    retq # encoding: [0xc3]
 ; WIN64-NEXT:  .LBB2_3:
 ; WIN64-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; WIN64-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 entry:
   %and = and i32 %x, 42
   %tobool = icmp eq i32 %and, 0
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall.ll b/llvm/test/CodeGen/X86/conditional-tailcall.ll
index 4c990d81810be..9c1d83037d1f5 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall.ll
@@ -13,33 +13,33 @@ define void @f(i32 %x, i32 %y) optsize {
 ; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x08]
 ; CHECK32-NEXT:    jne bar # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0x75,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %bb1
 ; CHECK32-NEXT:    jmp foo # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: f:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
 ; CHECK64-NEXT:    jne bar # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x75,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %bb1
 ; CHECK64-NEXT:    jmp foo # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: f:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    cmpl %edx, %ecx # encoding: [0x39,0xd1]
 ; WIN64-NEXT:    jne bar # TAILCALL
 ; WIN64-NEXT:    # encoding: [0x75,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %bb1
 ; WIN64-NEXT:    jmp foo # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 entry:
 	%p = icmp eq i32 %x, %y
   br i1 %p, label %bb1, label %bb2
@@ -64,20 +64,20 @@ define void @f_non_leaf(i32 %x, i32 %y) optsize {
 ; CHECK32-NEXT:    #NO_APP
 ; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x0c]
 ; CHECK32-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %bb1
 ; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK32-NEXT:    jmp foo # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB1_2: # %bb2
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK32-NEXT:    jmp bar # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: f_non_leaf:
 ; CHECK64:       # %bb.0: # %entry
@@ -88,20 +88,20 @@ define void @f_non_leaf(i32 %x, i32 %y) optsize {
 ; CHECK64-NEXT:    #NO_APP
 ; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
 ; CHECK64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %bb1
 ; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK64-NEXT:    jmp foo # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB1_2: # %bb2
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK64-NEXT:    jmp bar # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: f_non_leaf:
 ; WIN64:       # %bb.0: # %entry
@@ -112,14 +112,14 @@ define void @f_non_leaf(i32 %x, i32 %y) optsize {
 ; WIN64-NEXT:    #NO_APP
 ; WIN64-NEXT:    cmpl %edx, %ecx # encoding: [0x39,0xd1]
 ; WIN64-NEXT:    jne .LBB1_2 # encoding: [0x75,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB1_2, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %bb1
 ; WIN64-NEXT:    .seh_startepilogue
 ; WIN64-NEXT:    popq %rbx # encoding: [0x5b]
 ; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp foo # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: foo, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB1_2: # %bb2
 ; WIN64-NEXT:    nop # encoding: [0x90]
 ; WIN64-NEXT:    .seh_startepilogue
@@ -127,7 +127,7 @@ define void @f_non_leaf(i32 %x, i32 %y) optsize {
 ; WIN64-NEXT:    .seh_endepilogue
 ; WIN64-NEXT:    jmp bar # TAILCALL
 ; WIN64-NEXT:    # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: bar, kind: FK_PCRel_1
 ; WIN64-NEXT:    .seh_endproc
 entry:
   ; Force %ebx to be spilled on the stack, turning this into
@@ -152,58 +152,58 @@ define x86_thiscallcc zeroext i1 @BlockPlacementTest(ptr %this, i32 %x) optsize
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx # encoding: [0x8b,0x54,0x24,0x04]
 ; CHECK32-NEXT:    testb $42, %dl # encoding: [0xf6,0xc2,0x2a]
 ; CHECK32-NEXT:    je .LBB2_3 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_3, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %land.rhs
 ; CHECK32-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK32-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
 ; CHECK32-NEXT:    je baz # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: baz, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB2_2: # %land.end
 ; CHECK32-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK32-NEXT:    retl $4 # encoding: [0xc2,0x04,0x00]
 ; CHECK32-NEXT:  .LBB2_3:
 ; CHECK32-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK32-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: BlockPlacementTest:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    testb $42, %sil # encoding: [0x40,0xf6,0xc6,0x2a]
 ; CHECK64-NEXT:    je .LBB2_3 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_3, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %land.rhs
 ; CHECK64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK64-NEXT:    testb $44, %sil # encoding: [0x40,0xf6,0xc6,0x2c]
 ; CHECK64-NEXT:    je baz # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: baz, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB2_2: # %land.end
 ; CHECK64-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK64-NEXT:    retq # encoding: [0xc3]
 ; CHECK64-NEXT:  .LBB2_3:
 ; CHECK64-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK64-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: BlockPlacementTest:
 ; WIN64:       # %bb.0: # %entry
 ; WIN64-NEXT:    testb $42, %dl # encoding: [0xf6,0xc2,0x2a]
 ; WIN64-NEXT:    je .LBB2_3 # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_3-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_3, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.1: # %land.rhs
 ; WIN64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; WIN64-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
 ; WIN64-NEXT:    je baz # TAILCALL
 ; WIN64-NEXT:    # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: baz, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB2_2: # %land.end
 ; WIN64-NEXT:    # kill: def $al killed $al killed $eax
 ; WIN64-NEXT:    retq # encoding: [0xc3]
 ; WIN64-NEXT:  .LBB2_3:
 ; WIN64-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; WIN64-NEXT:    jmp .LBB2_2 # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_2-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB2_2, kind: FK_PCRel_1
 entry:
   %and = and i32 %x, 42
   %tobool = icmp eq i32 %and, 0
@@ -264,41 +264,41 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK32-NEXT:    testl %edx, %edx # encoding: [0x85,0xd2]
 ; CHECK32-NEXT:    je .LBB3_14 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_14-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_14, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.2: # %for.body
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    cmpl $2, %ebx # encoding: [0x83,0xfb,0x02]
 ; CHECK32-NEXT:    je .LBB3_12 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_12-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_12, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.3: # %for.body
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    cmpl $1, %ebx # encoding: [0x83,0xfb,0x01]
 ; CHECK32-NEXT:    je .LBB3_10 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_10-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_10, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.4: # %for.body
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    testl %ebx, %ebx # encoding: [0x85,0xdb]
 ; CHECK32-NEXT:    jne .LBB3_11 # encoding: [0x75,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.5: # %sw.bb
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    movzbl (%eax), %ebp # encoding: [0x0f,0xb6,0x28]
 ; CHECK32-NEXT:    cmpl $43, %ebp # encoding: [0x83,0xfd,0x2b]
 ; CHECK32-NEXT:    movl %edi, %ebx # encoding: [0x89,0xfb]
 ; CHECK32-NEXT:    je .LBB3_11 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.6: # %sw.bb
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    cmpl $45, %ebp # encoding: [0x83,0xfd,0x2d]
 ; CHECK32-NEXT:    movl %edi, %ebx # encoding: [0x89,0xfb]
 ; CHECK32-NEXT:    je .LBB3_11 # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.7: # %if.else
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    addl $-48, %ebp # encoding: [0x83,0xc5,0xd0]
 ; CHECK32-NEXT:    cmpl $10, %ebp # encoding: [0x83,0xfd,0x0a]
 ; CHECK32-NEXT:    jmp .LBB3_8 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_8-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_8, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB3_10: # %sw.bb14
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    movzbl (%eax), %ebx # encoding: [0x0f,0xb6,0x18]
@@ -308,9 +308,9 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    movl %esi, %ebx # encoding: [0x89,0xf3]
 ; CHECK32-NEXT:    jae .LBB3_9 # encoding: [0x73,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_9-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_9, kind: FK_PCRel_1
 ; CHECK32-NEXT:    jmp .LBB3_11 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB3_12: # %sw.bb22
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    movzbl (%eax), %ebx # encoding: [0x0f,0xb6,0x18]
@@ -318,18 +318,18 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK32-NEXT:    cmpl $10, %ebx # encoding: [0x83,0xfb,0x0a]
 ; CHECK32-NEXT:    movl %esi, %ebx # encoding: [0x89,0xf3]
 ; CHECK32-NEXT:    jae .LBB3_13 # encoding: [0x73,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_13-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_13, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB3_11: # %for.inc
 ; CHECK32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK32-NEXT:    incl %eax # encoding: [0x40]
 ; CHECK32-NEXT:    decl %edx # encoding: [0x4a]
 ; CHECK32-NEXT:    jmp .LBB3_1 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_1, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB3_14:
 ; CHECK32-NEXT:    cmpl $2, %ebx # encoding: [0x83,0xfb,0x02]
 ; CHECK32-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
 ; CHECK32-NEXT:    jmp .LBB3_15 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_15-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_15, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB3_9:
 ; CHECK32-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK32-NEXT:  .LBB3_15: # %cleanup.thread
@@ -355,11 +355,11 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK32-NEXT:    pushl %eax # encoding: [0x50]
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset 4
 ; CHECK32-NEXT:    calll _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_ # encoding: [0xe8,A,A,A,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_-4, kind: FK_PCRel_4
+; CHECK32-NEXT:    # fixup A - offset: 1, value: _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_, kind: FK_PCRel_4
 ; CHECK32-NEXT:    addl $28, %esp # encoding: [0x83,0xc4,0x1c]
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
 ; CHECK32-NEXT:    jmp .LBB3_16 # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_16-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: .LBB3_16, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: pr31257:
 ; CHECK64:       # %bb.0: # %entry
@@ -379,41 +379,41 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK64-NEXT:    testq %rax, %rax # encoding: [0x48,0x85,0xc0]
 ; CHECK64-NEXT:    je .LBB3_12 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_12-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_12, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.2: # %for.body
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    cmpl $2, %r8d # encoding: [0x41,0x83,0xf8,0x02]
 ; CHECK64-NEXT:    je .LBB3_10 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_10-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_10, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.3: # %for.body
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    cmpl $1, %r8d # encoding: [0x41,0x83,0xf8,0x01]
 ; CHECK64-NEXT:    je .LBB3_8 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_8-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_8, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.4: # %for.body
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    testl %r8d, %r8d # encoding: [0x45,0x85,0xc0]
 ; CHECK64-NEXT:    jne .LBB3_11 # encoding: [0x75,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.5: # %sw.bb
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    movzbl (%rdi), %r9d # encoding: [0x44,0x0f,0xb6,0x0f]
 ; CHECK64-NEXT:    cmpl $43, %r9d # encoding: [0x41,0x83,0xf9,0x2b]
 ; CHECK64-NEXT:    movl %edx, %r8d # encoding: [0x41,0x89,0xd0]
 ; CHECK64-NEXT:    je .LBB3_11 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.6: # %sw.bb
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    cmpl $45, %r9d # encoding: [0x41,0x83,0xf9,0x2d]
 ; CHECK64-NEXT:    movl %edx, %r8d # encoding: [0x41,0x89,0xd0]
 ; CHECK64-NEXT:    je .LBB3_11 # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.7: # %if.else
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    addl $-48, %r9d # encoding: [0x41,0x83,0xc1,0xd0]
 ; CHECK64-NEXT:    cmpl $10, %r9d # encoding: [0x41,0x83,0xf9,0x0a]
 ; CHECK64-NEXT:    jmp .LBB3_9 # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_9-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_9, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB3_8: # %sw.bb14
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    movzbl (%rdi), %r8d # encoding: [0x44,0x0f,0xb6,0x07]
@@ -423,9 +423,9 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    movl %ecx, %r8d # encoding: [0x41,0x89,0xc8]
 ; CHECK64-NEXT:    jb .LBB3_11 # encoding: [0x72,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; CHECK64-NEXT:    jmp .LBB3_13 # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_13-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_13, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB3_10: # %sw.bb22
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    movzbl (%rdi), %r8d # encoding: [0x44,0x0f,0xb6,0x07]
@@ -434,13 +434,13 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; CHECK64-NEXT:    movl %ecx, %r8d # encoding: [0x41,0x89,0xc8]
 ; CHECK64-NEXT:    jae _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_ # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x73,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB3_11: # %for.inc
 ; CHECK64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK64-NEXT:    incq %rdi # encoding: [0x48,0xff,0xc7]
 ; CHECK64-NEXT:    decq %rax # encoding: [0x48,0xff,0xc8]
 ; CHECK64-NEXT:    jmp .LBB3_1 # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: .LBB3_1, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB3_12:
 ; CHECK64-NEXT:    cmpl $2, %r8d # encoding: [0x41,0x83,0xf8,0x02]
 ; CHECK64-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
@@ -461,36 +461,36 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; WIN64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; WIN64-NEXT:    testq %rax, %rax # encoding: [0x48,0x85,0xc0]
 ; WIN64-NEXT:    je .LBB3_11 # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_11-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_11, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.2: # %for.body
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    cmpl $2, %r8d # encoding: [0x41,0x83,0xf8,0x02]
 ; WIN64-NEXT:    je .LBB3_9 # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_9-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_9, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.3: # %for.body
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    cmpl $1, %r8d # encoding: [0x41,0x83,0xf8,0x01]
 ; WIN64-NEXT:    je .LBB3_7 # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_7-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_7, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.4: # %for.body
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    testl %r8d, %r8d # encoding: [0x45,0x85,0xc0]
 ; WIN64-NEXT:    jne .LBB3_10 # encoding: [0x75,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.5: # %sw.bb
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    movzbl (%rcx), %r9d # encoding: [0x44,0x0f,0xb6,0x09]
 ; WIN64-NEXT:    cmpl $43, %r9d # encoding: [0x41,0x83,0xf9,0x2b]
 ; WIN64-NEXT:    movl $1, %r8d # encoding: [0x41,0xb8,0x01,0x00,0x00,0x00]
 ; WIN64-NEXT:    je .LBB3_10 # encoding: [0x74,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10, kind: FK_PCRel_1
 ; WIN64-NEXT:  # %bb.6: # %sw.bb
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    cmpl $45, %r9d # encoding: [0x41,0x83,0xf9,0x2d]
 ; WIN64-NEXT:    jne .LBB3_8 # encoding: [0x75,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_8-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_8, kind: FK_PCRel_1
 ; WIN64-NEXT:    jmp .LBB3_10 # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB3_7: # %sw.bb14
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    movzbl (%rcx), %r9d # encoding: [0x44,0x0f,0xb6,0x09]
@@ -500,9 +500,9 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; WIN64-NEXT:    movl $2, %r8d # encoding: [0x41,0xb8,0x02,0x00,0x00,0x00]
 ; WIN64-NEXT:    cmpl $10, %r9d # encoding: [0x41,0x83,0xf9,0x0a]
 ; WIN64-NEXT:    jb .LBB3_10 # encoding: [0x72,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_10, kind: FK_PCRel_1
 ; WIN64-NEXT:    jmp .LBB3_12 # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_12-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_12, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB3_9: # %sw.bb22
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    movzbl (%rcx), %r9d # encoding: [0x44,0x0f,0xb6,0x09]
@@ -511,13 +511,13 @@ define zeroext i1 @pr31257(ptr nocapture readonly dereferenceable(8) %s) minsize
 ; WIN64-NEXT:    cmpl $10, %r9d # encoding: [0x41,0x83,0xf9,0x0a]
 ; WIN64-NEXT:    jae _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_ # TAILCALL
 ; WIN64-NEXT:    # encoding: [0x73,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: _Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB3_10: # %for.inc
 ; WIN64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; WIN64-NEXT:    incq %rcx # encoding: [0x48,0xff,0xc1]
 ; WIN64-NEXT:    decq %rax # encoding: [0x48,0xff,0xc8]
 ; WIN64-NEXT:    jmp .LBB3_1 # encoding: [0xeb,A]
-; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
+; WIN64-NEXT:    # fixup A - offset: 1, value: .LBB3_1, kind: FK_PCRel_1
 ; WIN64-NEXT:  .LBB3_11:
 ; WIN64-NEXT:    cmpl $2, %r8d # encoding: [0x41,0x83,0xf8,0x02]
 ; WIN64-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
diff --git a/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll b/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll
index 2bd7dc445a02b..6291100f42c3d 100644
--- a/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll
+++ b/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=x86_64-- --start-before=x86-isel -mattr=+avx,+fma %s -o - | FileCheck %s
+; RUN: llc -mtriple=x86_64-- --start-before=x86-isel -mattr=+avx,+fma %s -o - -fp-contract=fast | FileCheck %s
 
 define double @fma_folding(double %x) {
 ; CHECK-LABEL: fma_folding:
@@ -20,3 +21,14 @@ define double @fma_no_folding(double %x) {
   %fused = call contract nnan ninf double @llvm.fma.f64(double %x, double 0.0, double -0.0)
   ret double %fused
 }
+
+define double @fma_no_fold_potential_nan(double %x) {
+; CHECK-LABEL: fma_no_fold_potential_nan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; CHECK-NEXT:    retq
+ %fused = call contract double @llvm.fma.f64(double %x, double 0.0, double 1.0)
+ ret double %fused
+}
+
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 8f82a5bc6554e..d869f8ec01a5a 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -172,9 +172,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl 40(%ebp), %esi
-; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    movl 40(%ebp), %ecx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl 36(%ebp), %ecx
 ; X86-NEXT:    xorl %edx, %ecx
@@ -203,45 +204,45 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    orb %cl, %al
 ; X86-NEXT:    movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT:    bsrl %esi, %edx
-; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    bsrl %eax, %ecx
+; X86-NEXT:    bsrl %eax, %edx
+; X86-NEXT:    xorl $31, %edx
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:    bsrl %esi, %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
 ; X86-NEXT:    testl %esi, %esi
-; X86-NEXT:    cmovnel %edx, %ecx
+; X86-NEXT:    cmovel %edx, %ecx
 ; X86-NEXT:    bsrl %ebx, %edx
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    bsrl %edi, %edi
 ; X86-NEXT:    xorl $31, %edi
-; X86-NEXT:    orl $32, %edi
+; X86-NEXT:    addl $32, %edi
 ; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %edx, %edi
-; X86-NEXT:    orl $64, %edi
+; X86-NEXT:    addl $64, %edi
 ; X86-NEXT:    movl %eax, %edx
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    orl %esi, %edx
 ; X86-NEXT:    cmovnel %ecx, %edi
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    bsrl %eax, %edx
+; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    bsrl %eax, %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    cmovnel %edx, %ecx
+; X86-NEXT:    cmovel %edx, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    bsrl %ebx, %esi
 ; X86-NEXT:    xorl $31, %esi
 ; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    orl $32, %edx
+; X86-NEXT:    addl $32, %edx
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    orl $64, %edx
+; X86-NEXT:    addl $64, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    orl %eax, %esi
 ; X86-NEXT:    cmovnel %ecx, %edx
@@ -379,9 +380,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    adcl $-1, %ecx
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    adcl $-1, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 0bef9ee50bd54..7bbddefd82721 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -173,17 +173,17 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    bsrl 36(%ebp), %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
+; X86-NEXT:    addl $32, %ecx
 ; X86-NEXT:    testl %esi, %esi
 ; X86-NEXT:    cmovnel %edx, %ecx
 ; X86-NEXT:    bsrl %edi, %edx
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    bsrl %ebx, %eax
 ; X86-NEXT:    xorl $31, %eax
-; X86-NEXT:    orl $32, %eax
+; X86-NEXT:    addl $32, %eax
 ; X86-NEXT:    testl %edi, %edi
 ; X86-NEXT:    cmovnel %edx, %eax
-; X86-NEXT:    orl $64, %eax
+; X86-NEXT:    addl $64, %eax
 ; X86-NEXT:    movl 36(%ebp), %edx
 ; X86-NEXT:    orl %esi, %edx
 ; X86-NEXT:    cmovnel %ecx, %eax
@@ -193,7 +193,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl 20(%ebp), %ecx
 ; X86-NEXT:    bsrl %ecx, %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
+; X86-NEXT:    addl $32, %ecx
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %edx, %ecx
 ; X86-NEXT:    movl 16(%ebp), %edi
@@ -201,10 +201,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    xorl $31, %esi
 ; X86-NEXT:    bsrl 12(%ebp), %edx
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    orl $32, %edx
+; X86-NEXT:    addl $32, %edx
 ; X86-NEXT:    testl %edi, %edi
 ; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    orl $64, %edx
+; X86-NEXT:    addl $64, %edx
 ; X86-NEXT:    movl 20(%ebp), %edi
 ; X86-NEXT:    movl %edi, %esi
 ; X86-NEXT:    orl %ebx, %esi
diff --git a/llvm/test/CodeGen/X86/fdiv.ll b/llvm/test/CodeGen/X86/fdiv.ll
index 6d2db80a87bdf..67bad09d5dd7f 100644
--- a/llvm/test/CodeGen/X86/fdiv.ll
+++ b/llvm/test/CodeGen/X86/fdiv.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 define double @exact(double %x) {
 ; Exact division by a constant converted to multiplication.
@@ -17,7 +17,7 @@ define double @inexact(double %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %div = fdiv double %x, 0x41DFFFFFFFC00000
+  %div = fdiv arcp double %x, 0x41DFFFFFFFC00000
   ret double %div
 }
 
diff --git a/llvm/test/CodeGen/X86/fltused_vec.ll b/llvm/test/CodeGen/X86/fltused_vec.ll
new file mode 100644
index 0000000000000..6e63e2e7ff891
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fltused_vec.ll
@@ -0,0 +1,29 @@
+; The purpose of this test to verify that the fltused symbol is
+; not emitted when purely vector floating point operations are used on Windows.
+
+; RUN: llc < %s -mtriple i686-pc-win32 | FileCheck %s
+; RUN: llc < %s -mtriple x86_64-pc-win32 | FileCheck %s
+
+@foo = external dso_local global [4 x float], align 16
+
+; Function Attrs: noinline nounwind optnone sspstrong uwtable
+define dso_local <4 x float> @func() #0 {
+entry:
+  %__p.addr.i = alloca ptr, align 8
+  %vector1 = alloca <4 x float>, align 16
+  store ptr @foo, ptr %__p.addr.i, align 8
+  %0 = load ptr, ptr %__p.addr.i, align 8
+  %1 = load <4 x float>, ptr %0, align 16
+  store <4 x float> %1, ptr %vector1, align 16
+  %2 = load <4 x float>, ptr %vector1, align 16
+  ret <4 x float> %2
+}
+
+define <4 x float> @mul_vectors(<4 x float> %a, <4 x float> %b) {
+entry:
+  %result = fmul <4 x float> %a, %b
+  ret <4 x float> %result
+}
+
+; _fltused is determined at a module level
+; CHECK-NOT: .globl {{_?}}_fltused
diff --git a/llvm/test/CodeGen/X86/fma-scalar-combine.ll b/llvm/test/CodeGen/X86/fma-scalar-combine.ll
index 7f7e21bf81d51..540fd24ea4280 100644
--- a/llvm/test/CodeGen/X86/fma-scalar-combine.ll
+++ b/llvm/test/CodeGen/X86/fma-scalar-combine.ll
@@ -548,9 +548,9 @@ define float @fma_const_fmul(float %x) {
 ; CHECK-LABEL: fma_const_fmul:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x59,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    vfmadd132ss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x99,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # xmm0 = (xmm0 * mem) + xmm1
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %mul1 = fmul contract float %x, 10.0
diff --git a/llvm/test/CodeGen/X86/fma.ll b/llvm/test/CodeGen/X86/fma.ll
index f26960b069b0e..6d865628ba715 100644
--- a/llvm/test/CodeGen/X86/fma.ll
+++ b/llvm/test/CodeGen/X86/fma.ll
@@ -27,7 +27,7 @@ define float @test_f32(float %a, float %b, float %c) #0 {
 ; FMACALL32:       ## %bb.0:
 ; FMACALL32-NEXT:    jmp _fmaf ## TAILCALL
 ; FMACALL32-NEXT:    ## encoding: [0xeb,A]
-; FMACALL32-NEXT:    ## fixup A - offset: 1, value: _fmaf-1, kind: FK_PCRel_1
+; FMACALL32-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_1
 ;
 ; FMA64-LABEL: test_f32:
 ; FMA64:       ## %bb.0:
@@ -39,7 +39,7 @@ define float @test_f32(float %a, float %b, float %c) #0 {
 ; FMACALL64:       ## %bb.0:
 ; FMACALL64-NEXT:    jmp _fmaf ## TAILCALL
 ; FMACALL64-NEXT:    ## encoding: [0xeb,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-1, kind: FK_PCRel_1
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_1
 ;
 ; AVX512-LABEL: test_f32:
 ; AVX512:       ## %bb.0:
@@ -129,7 +129,7 @@ define double @test_f64(double %a, double %b, double %c) #0 {
 ; FMACALL32:       ## %bb.0: ## %entry
 ; FMACALL32-NEXT:    jmp _fma ## TAILCALL
 ; FMACALL32-NEXT:    ## encoding: [0xeb,A]
-; FMACALL32-NEXT:    ## fixup A - offset: 1, value: _fma-1, kind: FK_PCRel_1
+; FMACALL32-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_1
 ;
 ; FMA64-LABEL: test_f64:
 ; FMA64:       ## %bb.0: ## %entry
@@ -141,7 +141,7 @@ define double @test_f64(double %a, double %b, double %c) #0 {
 ; FMACALL64:       ## %bb.0: ## %entry
 ; FMACALL64-NEXT:    jmp _fma ## TAILCALL
 ; FMACALL64-NEXT:    ## encoding: [0xeb,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-1, kind: FK_PCRel_1
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_1
 ;
 ; AVX512-LABEL: test_f64:
 ; AVX512:       ## %bb.0: ## %entry
@@ -170,7 +170,7 @@ define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 ; FMA32-NEXT:    fstpt {{[0-9]+}}(%esp) ## encoding: [0xdb,0x7c,0x24,0x10]
 ; FMA32-NEXT:    fstpt (%esp) ## encoding: [0xdb,0x3c,0x24]
 ; FMA32-NEXT:    calll _fmal ## encoding: [0xe8,A,A,A,A]
-; FMA32-NEXT:    ## fixup A - offset: 1, value: _fmal-4, kind: FK_PCRel_4
+; FMA32-NEXT:    ## fixup A - offset: 1, value: _fmal, kind: FK_PCRel_4
 ; FMA32-NEXT:    addl $60, %esp ## encoding: [0x83,0xc4,0x3c]
 ; FMA32-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -184,7 +184,7 @@ define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 ; FMACALL32-NEXT:    fstpt {{[0-9]+}}(%esp) ## encoding: [0xdb,0x7c,0x24,0x10]
 ; FMACALL32-NEXT:    fstpt (%esp) ## encoding: [0xdb,0x3c,0x24]
 ; FMACALL32-NEXT:    calll _fmal ## encoding: [0xe8,A,A,A,A]
-; FMACALL32-NEXT:    ## fixup A - offset: 1, value: _fmal-4, kind: FK_PCRel_4
+; FMACALL32-NEXT:    ## fixup A - offset: 1, value: _fmal, kind: FK_PCRel_4
 ; FMACALL32-NEXT:    addl $60, %esp ## encoding: [0x83,0xc4,0x3c]
 ; FMACALL32-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -198,7 +198,7 @@ define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 ; FMA64-NEXT:    fstpt {{[0-9]+}}(%rsp) ## encoding: [0xdb,0x7c,0x24,0x10]
 ; FMA64-NEXT:    fstpt (%rsp) ## encoding: [0xdb,0x3c,0x24]
 ; FMA64-NEXT:    callq _fmal ## encoding: [0xe8,A,A,A,A]
-; FMA64-NEXT:    ## fixup A - offset: 1, value: _fmal-4, kind: reloc_branch_4byte_pcrel
+; FMA64-NEXT:    ## fixup A - offset: 1, value: _fmal, kind: reloc_branch_4byte_pcrel
 ; FMA64-NEXT:    addq $56, %rsp ## encoding: [0x48,0x83,0xc4,0x38]
 ; FMA64-NEXT:    retq ## encoding: [0xc3]
 ;
@@ -212,7 +212,7 @@ define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 ; FMACALL64-NEXT:    fstpt {{[0-9]+}}(%rsp) ## encoding: [0xdb,0x7c,0x24,0x10]
 ; FMACALL64-NEXT:    fstpt (%rsp) ## encoding: [0xdb,0x3c,0x24]
 ; FMACALL64-NEXT:    callq _fmal ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmal-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmal, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    addq $56, %rsp ## encoding: [0x48,0x83,0xc4,0x38]
 ; FMACALL64-NEXT:    retq ## encoding: [0xc3]
 ;
@@ -226,7 +226,7 @@ define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 ; AVX512-NEXT:    fstpt {{[0-9]+}}(%rsp) ## encoding: [0xdb,0x7c,0x24,0x10]
 ; AVX512-NEXT:    fstpt (%rsp) ## encoding: [0xdb,0x3c,0x24]
 ; AVX512-NEXT:    callq _fmal ## encoding: [0xe8,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 1, value: _fmal-4, kind: reloc_branch_4byte_pcrel
+; AVX512-NEXT:    ## fixup A - offset: 1, value: _fmal, kind: reloc_branch_4byte_pcrel
 ; AVX512-NEXT:    addq $56, %rsp ## encoding: [0x48,0x83,0xc4,0x38]
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
@@ -240,7 +240,7 @@ define x86_fp80 @test_f80(x86_fp80 %a, x86_fp80 %b, x86_fp80 %c) #0 {
 ; AVX512VL-NEXT:    fstpt {{[0-9]+}}(%rsp) ## encoding: [0xdb,0x7c,0x24,0x10]
 ; AVX512VL-NEXT:    fstpt (%rsp) ## encoding: [0xdb,0x3c,0x24]
 ; AVX512VL-NEXT:    callq _fmal ## encoding: [0xe8,A,A,A,A]
-; AVX512VL-NEXT:    ## fixup A - offset: 1, value: _fmal-4, kind: reloc_branch_4byte_pcrel
+; AVX512VL-NEXT:    ## fixup A - offset: 1, value: _fmal, kind: reloc_branch_4byte_pcrel
 ; AVX512VL-NEXT:    addq $56, %rsp ## encoding: [0x48,0x83,0xc4,0x38]
 ; AVX512VL-NEXT:    retq ## encoding: [0xc3]
 entry:
@@ -265,28 +265,28 @@ define float @test_f32_cst() #0 {
 ; FMA64:       ## %bb.0: ## %entry
 ; FMA64-NEXT:    vmovss {{.*#+}} xmm0 = [1.2E+1,0.0E+0,0.0E+0,0.0E+0]
 ; FMA64-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMA64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; FMACALL64-LABEL: test_f32_cst:
 ; FMACALL64:       ## %bb.0: ## %entry
 ; FMACALL64-NEXT:    movss {{.*#+}} xmm0 = [1.2E+1,0.0E+0,0.0E+0,0.0E+0]
 ; FMACALL64-NEXT:    ## encoding: [0xf3,0x0f,0x10,0x05,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMACALL64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMACALL64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512-LABEL: test_f32_cst:
 ; AVX512:       ## %bb.0: ## %entry
 ; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = [1.2E+1,0.0E+0,0.0E+0,0.0E+0]
 ; AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_f32_cst:
 ; AVX512VL:       ## %bb.0: ## %entry
 ; AVX512VL-NEXT:    vmovss {{.*#+}} xmm0 = [1.2E+1,0.0E+0,0.0E+0,0.0E+0]
 ; AVX512VL-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512VL-NEXT:    retq ## encoding: [0xc3]
 entry:
   %call = call float @llvm.fma.f32(float 3.0, float 3.0, float 3.0)
@@ -322,7 +322,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -338,7 +338,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps (%rsp), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x04,0x24]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -351,7 +351,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x54,0x24,0x30]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x40]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -367,7 +367,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x40]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm1 ## encoding: [0x0f,0x14,0xc8]
@@ -404,7 +404,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm1, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x4c,0x24,0x04,0x02]
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x02]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x60]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -417,7 +417,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm2, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x54,0x24,0x04,0x01]
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm1, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x0c,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x54]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -430,7 +430,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x20]
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x40]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 ## 16-byte Reload
@@ -448,7 +448,7 @@ define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x6c,0x24,0x60]
 ; FMACALL32_BDVER2-NEXT:    fstps {{[0-9]+}}(%esp) ## encoding: [0xd9,0x5c,0x24,0x14]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstps {{[0-9]+}}(%esp) ## encoding: [0xd9,0x5c,0x24,0x10]
 ; FMACALL32_BDVER2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x44,0x24,0x1c]
@@ -502,7 +502,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    shufps $255, %xmm4, %xmm2 ## encoding: [0x0f,0xc6,0xd4,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3],xmm4[3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -518,7 +518,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -531,7 +531,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x14,0x24]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -547,7 +547,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x20]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm1 ## encoding: [0x0f,0x14,0xc8]
@@ -570,7 +570,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -586,7 +586,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps (%rsp), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x04,0x24]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -599,7 +599,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x54,0x24,0x50]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -615,7 +615,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm1 ## encoding: [0x0f,0x14,0xc8]
@@ -664,7 +664,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm1, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x0c,0x24,0x02]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0xa4,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -677,7 +677,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm2, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x54,0x24,0x04,0x01]
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm1, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x0c,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x98,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -690,7 +690,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x30]
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x8c,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %ymm0 ## 32-byte Reload
@@ -704,7 +704,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $3, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x03]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x80,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %ymm0 ## 32-byte Reload
@@ -718,7 +718,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x02]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x74]
 ; FMACALL32_BDVER2-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %ymm0 ## 32-byte Reload
@@ -732,7 +732,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x68]
 ; FMACALL32_BDVER2-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %ymm0 ## 32-byte Reload
@@ -746,7 +746,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x50]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 ## 16-byte Reload
@@ -776,7 +776,7 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) #
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xac,0x24,0xa4,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    fstps {{[0-9]+}}(%esp) ## encoding: [0xd9,0x5c,0x24,0x14]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstps {{[0-9]+}}(%esp) ## encoding: [0xd9,0x5c,0x24,0x10]
 ; FMACALL32_BDVER2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x44,0x24,0x2c]
@@ -854,7 +854,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x60]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -869,7 +869,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x44,0x24,0x60]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -881,7 +881,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xb0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x60]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -896,7 +896,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x60]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm1 ## encoding: [0x0f,0x14,0xc8]
@@ -918,7 +918,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -933,7 +933,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -945,7 +945,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xc0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -960,7 +960,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm1 ## encoding: [0x0f,0x14,0xc8]
@@ -982,7 +982,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -997,7 +997,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps (%rsp), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x04,0x24]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -1009,7 +1009,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xd0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1024,7 +1024,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm1 ## encoding: [0x0f,0x14,0xc8]
@@ -1046,7 +1046,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $255, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0xff]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[3,3,3,3]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1061,7 +1061,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Folded Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x14,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    ## xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -1073,7 +1073,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x8c,0x24,0x80,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xe0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x40]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1088,7 +1088,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL64-NEXT:    shufps $85, %xmm2, %xmm2 ## encoding: [0x0f,0xc6,0xd2,0x55]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1,1,1]
 ; FMACALL64-NEXT:    callq _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x5c,0x24,0x40]
 ; FMACALL64-NEXT:    unpcklps %xmm0, %xmm3 ## encoding: [0x0f,0x14,0xd8]
@@ -1144,7 +1144,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm4, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x64,0x24,0x08,0x02]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x54,0x01,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovaps 56(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x38]
@@ -1156,7 +1156,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm2, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x54,0x24,0x04,0x01]
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm1, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x0c,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 56(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x38]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x48,0x01,0x00,0x00]
@@ -1168,7 +1168,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x84,0x24,0xb0,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 40(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x28]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x3c,0x01,0x00,0x00]
@@ -1181,7 +1181,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $3, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x03]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 40(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x28]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x30,0x01,0x00,0x00]
@@ -1194,7 +1194,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x02]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 40(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x28]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x24,0x01,0x00,0x00]
@@ -1207,7 +1207,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 40(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x28]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x18,0x01,0x00,0x00]
@@ -1220,7 +1220,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 24(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x18]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x0c,0x01,0x00,0x00]
@@ -1239,7 +1239,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $3, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x03]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x00,0x01,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovaps 24(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x18]
@@ -1251,7 +1251,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm2, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x54,0x24,0x04,0x02]
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm1, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x0c,0x24,0x02]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0xf4,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    vmovaps 24(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x18]
@@ -1263,7 +1263,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm2, {{[0-9]+}}(%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x54,0x24,0x04,0x01]
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm1, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x0c,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 24(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x18]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0xe8,0x00,0x00,0x00]
@@ -1275,7 +1275,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x60]
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 8(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x08]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0x80,0x00,0x00,0x00]
@@ -1288,7 +1288,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $3, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x03]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 8(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x08]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x60]
@@ -1301,7 +1301,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $2, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x02]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 8(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x08]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0xdc,0x00,0x00,0x00]
@@ -1314,7 +1314,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vextractps $1, %xmm0, (%esp) ## encoding: [0xc4,0xe3,0x79,0x17,0x04,0x24,0x01]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 8(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x08]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xbc,0x24,0xd0,0x00,0x00,0x00]
@@ -1327,7 +1327,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    vmovss %xmm0, (%esp) ## encoding: [0xc5,0xfa,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps 56(%ebp), %xmm0 ## encoding: [0xc5,0xf8,0x28,0x45,0x38]
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x94,0x24,0xc0,0x00,0x00,0x00]
@@ -1380,7 +1380,7 @@ define <16 x float> @test_v16f32(<16 x float> %a, <16 x float> %b, <16 x float>
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xac,0x24,0x54,0x01,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    fstps {{[0-9]+}}(%esp) ## encoding: [0xd9,0x5c,0x24,0x44]
 ; FMACALL32_BDVER2-NEXT:    calll _fmaf ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fmaf, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstps {{[0-9]+}}(%esp) ## encoding: [0xd9,0x5c,0x24,0x40]
 ; FMACALL32_BDVER2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x44,0x24,0x3c]
@@ -1447,7 +1447,7 @@ define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x30]
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm0 ## 16-byte Reload
@@ -1463,7 +1463,7 @@ define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x30]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm1 ## encoding: [0x0f,0x16,0xc8]
@@ -1498,7 +1498,7 @@ define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovlps %xmm2, {{[0-9]+}}(%esp) ## encoding: [0xc5,0xf8,0x13,0x54,0x24,0x10]
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x30]
 ; FMACALL32_BDVER2-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp) ## encoding: [0xc5,0xf8,0x17,0x44,0x24,0x10]
@@ -1510,7 +1510,7 @@ define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    fstpl {{[0-9]+}}(%esp) ## encoding: [0xdd,0x5c,0x24,0x28]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpl {{[0-9]+}}(%esp) ## encoding: [0xdd,0x5c,0x24,0x20]
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x44,0x24,0x28]
@@ -1593,7 +1593,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL64-NEXT:    movaps %xmm2, %xmm1 ## encoding: [0x0f,0x28,0xca]
 ; FMACALL64-NEXT:    movaps %xmm4, %xmm2 ## encoding: [0x0f,0x28,0xd4]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1609,7 +1609,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm1 ## encoding: [0x0f,0x16,0xc8]
@@ -1623,7 +1623,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x54,0x24,0x40]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1639,7 +1639,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm1 ## encoding: [0x0f,0x16,0xc8]
@@ -1685,7 +1685,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x44]
 ; FMACALL32_BDVER2-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %ymm0 ## 32-byte Reload
@@ -1699,7 +1699,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x7c,0x24,0x38]
 ; FMACALL32_BDVER2-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %ymm0 ## 32-byte Reload
@@ -1713,7 +1713,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x44,0x24,0x70]
 ; FMACALL32_BDVER2-NEXT:    vmovhps %xmm0, {{[0-9]+}}(%esp) ## encoding: [0xc5,0xf8,0x17,0x44,0x24,0x10]
@@ -1731,7 +1731,7 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0x6c,0x24,0x44]
 ; FMACALL32_BDVER2-NEXT:    fstpl {{[0-9]+}}(%esp) ## encoding: [0xdd,0x5c,0x24,0x20]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpl {{[0-9]+}}(%esp) ## encoding: [0xdd,0x5c,0x24,0x18]
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x44,0x24,0x30]
@@ -1794,7 +1794,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xa0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    movaps %xmm4, %xmm1 ## encoding: [0x0f,0x28,0xcc]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x30]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1809,7 +1809,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x30]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm1 ## encoding: [0x0f,0x16,0xc8]
@@ -1822,7 +1822,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xb0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x10]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1837,7 +1837,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x10]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm1 ## encoding: [0x0f,0x16,0xc8]
@@ -1850,7 +1850,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xc0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, (%rsp) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x04,0x24]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1865,7 +1865,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps (%rsp), %xmm1 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x0c,0x24]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm1 ## encoding: [0x0f,0x16,0xc8]
@@ -1878,7 +1878,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x4c,0x24,0x70]
 ; FMACALL64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2 ## encoding: [0x0f,0x28,0x94,0x24,0xd0,0x00,0x00,0x00]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x29,0x44,0x24,0x20]
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload
@@ -1893,7 +1893,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL64-NEXT:    movhlps %xmm2, %xmm2 ## encoding: [0x0f,0x12,0xd2]
 ; FMACALL64-NEXT:    ## xmm2 = xmm2[1,1]
 ; FMACALL64-NEXT:    callq _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: reloc_branch_4byte_pcrel
+; FMACALL64-NEXT:    ## fixup A - offset: 1, value: _fma, kind: reloc_branch_4byte_pcrel
 ; FMACALL64-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 ## 16-byte Reload
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x5c,0x24,0x20]
 ; FMACALL64-NEXT:    movlhps %xmm0, %xmm3 ## encoding: [0x0f,0x16,0xd8]
@@ -1948,7 +1948,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x30]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
@@ -1962,7 +1962,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x28]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
@@ -1976,7 +1976,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x20]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
@@ -1997,7 +1997,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x18]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
@@ -2010,7 +2010,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    ## xmm0 = xmm0[0],mem[0]
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x10]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
@@ -2024,7 +2024,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x08]
 ; FMACALL32_BDVER2-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
@@ -2038,7 +2038,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    vmovups %xmm0, (%esp) ## encoding: [0xc5,0xf8,0x11,0x04,0x24]
 ; FMACALL32_BDVER2-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x45,0x40]
 ; FMACALL32_BDVER2-NEXT:    vmovsd %xmm0, {{[0-9]+}}(%esp) ## encoding: [0xc5,0xfb,0x11,0x44,0x24,0x10]
@@ -2068,7 +2068,7 @@ define <8 x double> @test_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xdb,0xac,0x24,0x94,0x00,0x00,0x00]
 ; FMACALL32_BDVER2-NEXT:    fstpl {{[0-9]+}}(%esp) ## encoding: [0xdd,0x5c,0x24,0x70]
 ; FMACALL32_BDVER2-NEXT:    calll _fma ## encoding: [0xe8,A,A,A,A]
-; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma-4, kind: FK_PCRel_4
+; FMACALL32_BDVER2-NEXT:    ## fixup A - offset: 1, value: _fma, kind: FK_PCRel_4
 ; FMACALL32_BDVER2-NEXT:    fstpl {{[0-9]+}}(%esp) ## encoding: [0xdd,0x5c,0x24,0x68]
 ; FMACALL32_BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; FMACALL32_BDVER2-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x44,0x24,0x60]
@@ -2113,28 +2113,28 @@ define float @constant_fold_f32() {
 ; FMA64:       ## %bb.0:
 ; FMA64-NEXT:    vmovss {{.*#+}} xmm0 = [1.02E+3,0.0E+0,0.0E+0,0.0E+0]
 ; FMA64-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMA64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; FMACALL64-LABEL: constant_fold_f32:
 ; FMACALL64:       ## %bb.0:
 ; FMACALL64-NEXT:    movss {{.*#+}} xmm0 = [1.02E+3,0.0E+0,0.0E+0,0.0E+0]
 ; FMACALL64-NEXT:    ## encoding: [0xf3,0x0f,0x10,0x05,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMACALL64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMACALL64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512-LABEL: constant_fold_f32:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vmovss {{.*#+}} xmm0 = [1.02E+3,0.0E+0,0.0E+0,0.0E+0]
 ; AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: constant_fold_f32:
 ; AVX512VL:       ## %bb.0:
 ; AVX512VL-NEXT:    vmovss {{.*#+}} xmm0 = [1.02E+3,0.0E+0,0.0E+0,0.0E+0]
 ; AVX512VL-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512VL-NEXT:    retq ## encoding: [0xc3]
   %r  = call float @llvm.fma.f32(float 5.000000e+01, float 2.000000e+01, float 2.000000e+01)
   ret float %r
@@ -2159,28 +2159,28 @@ define <4 x float> @constant_fold_v4f32() {
 ; FMA64:       ## %bb.0:
 ; FMA64-NEXT:    vmovaps {{.*#+}} xmm0 = [0.0E+0,4.9E+2,1.18E+3,2.07E+3]
 ; FMA64-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMA64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; FMACALL64-LABEL: constant_fold_v4f32:
 ; FMACALL64:       ## %bb.0:
 ; FMACALL64-NEXT:    movaps {{.*#+}} xmm0 = [0.0E+0,4.9E+2,1.18E+3,2.07E+3]
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x05,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMACALL64-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMACALL64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512-LABEL: constant_fold_v4f32:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [0.0E+0,4.9E+2,1.18E+3,2.07E+3]
 ; AVX512-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: constant_fold_v4f32:
 ; AVX512VL:       ## %bb.0:
 ; AVX512VL-NEXT:    vmovaps {{.*#+}} xmm0 = [0.0E+0,4.9E+2,1.18E+3,2.07E+3]
 ; AVX512VL-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512VL-NEXT:    retq ## encoding: [0xc3]
   %r  = call <4 x float> @llvm.fma.v4f32(<4 x float> <float 0.000000e+01, float 1.000000e+01, float 2.000000e+01, float 3.000000e+01>, <4 x float> <float 4.000000e+01, float 5.000000e+01, float 6.000000e+01, float 7.000000e+01>, <4 x float> <float 0.000000e+01, float -1.000000e+01, float -2.000000e+01, float -3.000000e+01>)
   ret <4 x float> %r
@@ -2205,28 +2205,28 @@ define <2 x double> @constant_fold_v2f64() {
 ; FMA64:       ## %bb.0:
 ; FMA64-NEXT:    vmovaps {{.*#+}} xmm0 = [4.1E+2,1.4E+3]
 ; FMA64-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMA64-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMA64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; FMACALL64-LABEL: constant_fold_v2f64:
 ; FMACALL64:       ## %bb.0:
 ; FMACALL64-NEXT:    movaps {{.*#+}} xmm0 = [4.1E+2,1.4E+3]
 ; FMACALL64-NEXT:    ## encoding: [0x0f,0x28,0x05,A,A,A,A]
-; FMACALL64-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; FMACALL64-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; FMACALL64-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512-LABEL: constant_fold_v2f64:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [4.1E+2,1.4E+3]
 ; AVX512-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: constant_fold_v2f64:
 ; AVX512VL:       ## %bb.0:
 ; AVX512VL-NEXT:    vmovaps {{.*#+}} xmm0 = [4.1E+2,1.4E+3]
 ; AVX512VL-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; AVX512VL-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; AVX512VL-NEXT:    retq ## encoding: [0xc3]
   %r  = call <2 x double> @llvm.fma.v2f64(<2 x double> <double 1.000000e+01, double 2.000000e+01>, <2 x double> <double 4.000000e+01, double 7.000000e+01>, <2 x double> <double 1.000000e+01, double 0.000000e+01>)
   ret <2 x double> %r
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index 95d350d45d901..eef87b5a9f85a 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -272,30 +272,15 @@ define double @test_fmaximumnum_zero0(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fmaximumnum_zero0:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movapd %xmm1, %xmm0
-; SSE2-NEXT:    cmpordsd %xmm1, %xmm0
-; SSE2-NEXT:    movapd %xmm0, %xmm2
-; SSE2-NEXT:    andpd %xmm1, %xmm2
-; SSE2-NEXT:    xorpd %xmm3, %xmm3
-; SSE2-NEXT:    maxsd %xmm3, %xmm1
-; SSE2-NEXT:    andnpd %xmm1, %xmm0
-; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    maxsd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fmaximumnum_zero0:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vcmpordsd %xmm1, %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fmaximumnum_zero0:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vcmpordsd %xmm1, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fmaximumnum_zero0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_zero0:
 ; AVX10_2:       # %bb.0:
@@ -311,10 +296,8 @@ define double @test_fmaximumnum_zero0(double %x, double %y) nounwind {
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -326,33 +309,15 @@ define double @test_fmaximumnum_zero0(double %x, double %y) nounwind {
 define double @test_fmaximumnum_zero1(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fmaximumnum_zero1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    cmpordsd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm2
-; SSE2-NEXT:    andpd %xmm0, %xmm2
-; SSE2-NEXT:    xorpd %xmm3, %xmm3
-; SSE2-NEXT:    maxsd %xmm3, %xmm0
-; SSE2-NEXT:    andnpd %xmm0, %xmm1
-; SSE2-NEXT:    orpd %xmm2, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    maxsd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fmaximumnum_zero1:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fmaximumnum_zero1:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vcmpordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
-; AVX512-NEXT:    vmovapd %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fmaximumnum_zero1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_zero1:
 ; AVX10_2:       # %bb.0:
@@ -368,10 +333,8 @@ define double @test_fmaximumnum_zero1(double %x, double %y) nounwind {
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -802,27 +765,13 @@ define double @test_fminimumnum_zero0(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fminimumnum_zero0:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movapd %xmm1, %xmm0
-; SSE2-NEXT:    cmpordsd %xmm1, %xmm0
-; SSE2-NEXT:    movapd %xmm0, %xmm2
-; SSE2-NEXT:    andpd %xmm1, %xmm2
-; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    andnpd %xmm1, %xmm0
-; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fminimumnum_zero0:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vcmpordsd %xmm1, %xmm1, %xmm0
-; AVX1-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fminimumnum_zero0:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcmpordsd %xmm1, %xmm1, %k1
-; AVX512-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fminimumnum_zero0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_zero0:
 ; AVX10_2:       # %bb.0:
@@ -836,10 +785,8 @@ define double @test_fminimumnum_zero0(double %x, double %y) nounwind {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm1
-; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -851,30 +798,13 @@ define double @test_fminimumnum_zero0(double %x, double %y) nounwind {
 define double @test_fminimumnum_zero1(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fminimumnum_zero1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    cmpordsd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm2
-; SSE2-NEXT:    andpd %xmm0, %xmm2
 ; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    andnpd %xmm0, %xmm1
-; SSE2-NEXT:    orpd %xmm2, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fminimumnum_zero1:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm1
-; AVX1-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fminimumnum_zero1:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcmpordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
-; AVX512-NEXT:    vmovapd %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fminimumnum_zero1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_zero1:
 ; AVX10_2:       # %bb.0:
@@ -888,10 +818,8 @@ define double @test_fminimumnum_zero1(double %x, double %y) nounwind {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm1
-; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -1156,13 +1084,19 @@ define <2 x double> @test_fminimumnum_vector_zero(<2 x double> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorpd %xmm1, %xmm1
 ; SSE2-NEXT:    minpd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpordpd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm2, %xmm0
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_zero:
@@ -1174,7 +1108,9 @@ define <2 x double> @test_fminimumnum_vector_zero(<2 x double> %x) {
 ; X86-LABEL: test_fminimumnum_vector_zero:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double 0., double 0.>)
   ret <2 x double> %r
@@ -1185,13 +1121,19 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero(<4 x float> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; SSE2-NEXT:    maxps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpordps %xmm0, %xmm2
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_signed_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_signed_zero:
@@ -1202,7 +1144,9 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero(<4 x float> %x) {
 ; X86-LABEL: test_fmaximumnum_vector_signed_zero:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> <float -0., float -0., float -0., float -0.>)
   ret <4 x float> %r
@@ -1211,17 +1155,23 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero(<4 x float> %x) {
 define <2 x double> @test_fminimumnum_vector_partially_zero(<2 x double> %x) {
 ; SSE2-LABEL: test_fminimumnum_vector_partially_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorpd %xmm1, %xmm1
-; SSE2-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; SSE2-NEXT:    minpd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    cmpordpd %xmm0, %xmm1
+; SSE2-NEXT:    xorpd %xmm2, %xmm2
+; SSE2-NEXT:    movhpd {{.*#+}} xmm2 = xmm2[0],mem[0]
+; SSE2-NEXT:    minpd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm1, %xmm0
+; SSE2-NEXT:    andnpd %xmm2, %xmm1
+; SSE2-NEXT:    orpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_partially_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_partially_zero:
@@ -1235,7 +1185,9 @@ define <2 x double> @test_fminimumnum_vector_partially_zero(<2 x double> %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; X86-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double 0., double 5.>)
   ret <2 x double> %r
@@ -1302,15 +1254,12 @@ define <2 x double> @test_fminimumnum_vector_different_zeros(<2 x double> %x) {
 define <4 x float> @test_fmaximumnum_vector_non_zero(<4 x float> %x) {
 ; SSE2-LABEL: test_fmaximumnum_vector_non_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0]
-; SSE2-NEXT:    maxps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    maxps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_non_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0]
-; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_non_zero:
@@ -1320,8 +1269,7 @@ define <4 x float> @test_fmaximumnum_vector_non_zero(<4 x float> %x) {
 ;
 ; X86-LABEL: test_fmaximumnum_vector_non_zero:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0]
-; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> <float 5., float 4., float 3., float 2.>)
   ret <4 x float> %r
@@ -1370,13 +1318,19 @@ define <2 x double> @test_fminimumnum_vector_zero_first(<2 x double> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorpd %xmm1, %xmm1
 ; SSE2-NEXT:    minpd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpordpd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm2, %xmm0
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_zero_first:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_zero_first:
@@ -1388,7 +1342,9 @@ define <2 x double> @test_fminimumnum_vector_zero_first(<2 x double> %x) {
 ; X86-LABEL: test_fminimumnum_vector_zero_first:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> <double 0., double 0.>, <2 x double> %x)
   ret <2 x double> %r
@@ -1397,20 +1353,12 @@ define <2 x double> @test_fminimumnum_vector_zero_first(<2 x double> %x) {
 define <2 x double> @test_fminimumnum_vector_signed_zero(<2 x double> %x) {
 ; SSE2-LABEL: test_fminimumnum_vector_signed_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    minpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    movapd %xmm0, %xmm2
-; SSE2-NEXT:    cmpordpd %xmm0, %xmm2
-; SSE2-NEXT:    andpd %xmm2, %xmm0
-; SSE2-NEXT:    andnpd %xmm1, %xmm2
-; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    minpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_signed_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm1
-; AVX-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
-; AVX-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_signed_zero:
@@ -1420,9 +1368,7 @@ define <2 x double> @test_fminimumnum_vector_signed_zero(<2 x double> %x) {
 ;
 ; X86-LABEL: test_fminimumnum_vector_signed_zero:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm1
-; X86-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double -0., double -0.>)
   ret <2 x double> %r
@@ -1433,13 +1379,19 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero_first(<4 x float> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; SSE2-NEXT:    maxps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpordps %xmm0, %xmm2
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_signed_zero_first:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_signed_zero_first:
@@ -1450,7 +1402,9 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero_first(<4 x float> %x) {
 ; X86-LABEL: test_fmaximumnum_vector_signed_zero_first:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> <float -0., float -0., float -0., float -0.>, <4 x float> %x)
   ret <4 x float> %r
@@ -1460,21 +1414,13 @@ define <4 x float> @test_fmaximumnum_vector_zero(<4 x float> %x) {
 ; SSE2-LABEL: test_fmaximumnum_vector_zero:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    maxps %xmm1, %xmm2
-; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    cmpordps %xmm0, %xmm1
-; SSE2-NEXT:    andps %xmm1, %xmm0
-; SSE2-NEXT:    andnps %xmm2, %xmm1
-; SSE2-NEXT:    orps %xmm1, %xmm0
+; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmaxps %xmm1, %xmm0, %xmm1
-; AVX-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_zero:
@@ -1486,9 +1432,7 @@ define <4 x float> @test_fmaximumnum_vector_zero(<4 x float> %x) {
 ; X86-LABEL: test_fmaximumnum_vector_zero:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm1
-; X86-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> <float 0., float 0., float 0., float 0.>)
   ret <4 x float> %r
diff --git a/llvm/test/CodeGen/X86/fold-rmw-ops.ll b/llvm/test/CodeGen/X86/fold-rmw-ops.ll
index 22cec09c90599..39a05857fcba2 100644
--- a/llvm/test/CodeGen/X86/fold-rmw-ops.ll
+++ b/llvm/test/CodeGen/X86/fold-rmw-ops.ll
@@ -15,15 +15,15 @@ define void @add64_imm32_br() nounwind {
 ; CHECK-LABEL: add64_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $16777214, g64(%rip) # encoding: [0x48,0x81,0x05,A,A,A,A,0xfe,0xff,0xff,0x00]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0xFFFFFE
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Add 0x00FFFFFE, a positive immediate requiring 24-bits.
@@ -45,15 +45,15 @@ define void @add64_sext_imm32_br() nounwind {
 ; CHECK-LABEL: add64_sext_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-2147483648, g64(%rip) # encoding: [0x48,0x81,0x05,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Add -0x80000000, which requires sign-extended 32 bits.
@@ -75,15 +75,15 @@ define void @add64_imm32_via_sub_br() nounwind {
 ; CHECK-LABEL: add64_imm32_via_sub_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $-2147483648, g64(%rip) # encoding: [0x48,0x81,0x2d,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Add 0x80000000, which cannot fit in a sign extended 32-bit immediate. This
@@ -108,14 +108,14 @@ define void @add64_no_imm32_via_sub_due_to_cf_br() nounwind {
 ; CHECK-NEXT:    movl $2147483648, %eax # encoding: [0xb8,0x00,0x00,0x00,0x80]
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    addq %rax, g64(%rip) # encoding: [0x48,0x01,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jae b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x73,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Add 0x80000000, which cannot fit in a sign extended 32-bit immediate, but
@@ -141,14 +141,14 @@ define void @add64_too_large_imm32_br() nounwind {
 ; CHECK-NEXT:    movl $2147483649, %eax # encoding: [0xb8,0x01,0x00,0x00,0x80]
 ; CHECK-NEXT:    # imm = 0x80000001
 ; CHECK-NEXT:    addq %rax, g64(%rip) # encoding: [0x48,0x01,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Add 0x80000001, which cannot fit in a sign extended 32-bit immediate. This
@@ -171,14 +171,14 @@ define void @add64_imm8_via_sub_br() nounwind {
 ; CHECK-LABEL: add64_imm8_via_sub_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $-128, g64(%rip) # encoding: [0x48,0x83,0x2d,A,A,A,A,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Add 0x80 which can't quite fit into an imm8 because it would be sign
@@ -201,14 +201,14 @@ define void @add64_imm8_br() nounwind {
 ; CHECK-LABEL: add64_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $42, g64(%rip) # encoding: [0x48,0x83,0x05,A,A,A,A,0x2a]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %add = add i64 %load1, 42
@@ -229,14 +229,14 @@ define void @add64_imm8_neg_br() nounwind {
 ; CHECK-LABEL: add64_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-42, g64(%rip) # encoding: [0x48,0x83,0x05,A,A,A,A,0xd6]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %add = add i64 %load1, -42
@@ -257,15 +257,15 @@ define void @add32_imm_br() nounwind {
 ; CHECK-LABEL: add32_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addl $-2147483648, g32(%rip) # encoding: [0x81,0x05,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   ; Add 0x80000000, a positive number requiring 32 bits of immediate.
@@ -287,14 +287,14 @@ define void @add32_imm8_br() nounwind {
 ; CHECK-LABEL: add32_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addl $42, g32(%rip) # encoding: [0x83,0x05,A,A,A,A,0x2a]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %add = add i32 %load1, 42
@@ -315,14 +315,14 @@ define void @add32_imm8_neg_br() nounwind {
 ; CHECK-LABEL: add32_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addl $-42, g32(%rip) # encoding: [0x83,0x05,A,A,A,A,0xd6]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %add = add i32 %load1, -42
@@ -343,15 +343,15 @@ define void @add16_imm_br() nounwind {
 ; CHECK-LABEL: add16_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addw $-32768, g16(%rip) # encoding: [0x66,0x81,0x05,A,A,A,A,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-6, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-2, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x8000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   ; Add 0x8000, a positive number requiring 16 bits of immediate.
@@ -373,14 +373,14 @@ define void @add16_imm8_br() nounwind {
 ; CHECK-LABEL: add16_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addw $42, g16(%rip) # encoding: [0x66,0x83,0x05,A,A,A,A,0x2a]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %add = add i16 %load1, 42
@@ -401,14 +401,14 @@ define void @add16_imm8_neg_br() nounwind {
 ; CHECK-LABEL: add16_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addw $-42, g16(%rip) # encoding: [0x66,0x83,0x05,A,A,A,A,0xd6]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %add = add i16 %load1, -42
@@ -429,14 +429,14 @@ define void @add8_imm_br() nounwind {
 ; CHECK-LABEL: add8_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addb $-2, g8(%rip) # encoding: [0x80,0x05,A,A,A,A,0xfe]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g8-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g8-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %add = add i8 %load1, -2
@@ -457,14 +457,14 @@ define void @add64_reg_br(i64 %arg) nounwind {
 ; CHECK-LABEL: add64_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq %rdi, g64(%rip) # encoding: [0x48,0x01,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %add = add i64 %load1, %arg
@@ -485,14 +485,14 @@ define void @add32_reg_br(i32 %arg) nounwind {
 ; CHECK-LABEL: add32_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addl %edi, g32(%rip) # encoding: [0x01,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %add = add i32 %load1, %arg
@@ -513,14 +513,14 @@ define void @add16_reg_br(i16 %arg) nounwind {
 ; CHECK-LABEL: add16_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addw %di, g16(%rip) # encoding: [0x66,0x01,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %add = add i16 %load1, %arg
@@ -541,14 +541,14 @@ define void @add8_reg_br(i8 %arg) nounwind {
 ; CHECK-LABEL: add8_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addb %dil, g8(%rip) # encoding: [0x40,0x00,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g8-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g8, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %add = add i8 %load1, %arg
@@ -569,15 +569,15 @@ define void @sub64_imm32_br() nounwind {
 ; CHECK-LABEL: sub64_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $-2147483648, g64(%rip) # encoding: [0x48,0x81,0x2d,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Subtract -0x80000000, which can't be negated into a sign-extended 32-bit
@@ -602,14 +602,14 @@ define void @sub64_too_large_imm32_br() nounwind {
 ; CHECK-NEXT:    movabsq $-4294967295, %rax # encoding: [0x48,0xb8,0x01,0x00,0x00,0x00,0xff,0xff,0xff,0xff]
 ; CHECK-NEXT:    # imm = 0xFFFFFFFF00000001
 ; CHECK-NEXT:    addq %rax, g64(%rip) # encoding: [0x48,0x01,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Subtract 0xFFFFFFFF, which cannot fit in a sign extended 32-bit immediate,
@@ -632,14 +632,14 @@ define void @sub64_imm8_br() nounwind {
 ; CHECK-LABEL: sub64_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq $-128, g64(%rip) # encoding: [0x48,0x83,0x2d,A,A,A,A,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Subtract -0x80, which can be done with an 8-bit immediate but only as
@@ -662,15 +662,15 @@ define void @sub32_imm_br() nounwind {
 ; CHECK-LABEL: sub32_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addl $-2147483648, g32(%rip) # encoding: [0x81,0x05,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   ; Subtract -0x80000000, which requires 32 bits of immediate but still gets
@@ -693,14 +693,14 @@ define void @sub32_imm8_br() nounwind {
 ; CHECK-LABEL: sub32_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subl $-128, g32(%rip) # encoding: [0x83,0x2d,A,A,A,A,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   ; Subtract -0x80, which can be done with an 8-bit immediate but only as
@@ -723,15 +723,15 @@ define void @sub16_imm_br() nounwind {
 ; CHECK-LABEL: sub16_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addw $-32768, g16(%rip) # encoding: [0x66,0x81,0x05,A,A,A,A,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-6, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-2, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x8000
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   ; Subtract -0x8000, which requires a 16 bits of immediate but still gets
@@ -754,14 +754,14 @@ define void @sub16_imm8_br() nounwind {
 ; CHECK-LABEL: sub16_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subw $-128, g16(%rip) # encoding: [0x66,0x83,0x2d,A,A,A,A,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   ; Subtract -0x80, which can be done with an 8-bit immediate but only as
@@ -784,14 +784,14 @@ define void @sub8_imm_br() nounwind {
 ; CHECK-LABEL: sub8_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addb $-128, g8(%rip) # encoding: [0x80,0x05,A,A,A,A,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g8-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g8-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   ; Subtract -0x80, which requires an 8-bit immediate but still gets lowered as
@@ -814,14 +814,14 @@ define void @sub64_reg_br(i64 %arg) nounwind {
 ; CHECK-LABEL: sub64_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subq %rdi, g64(%rip) # encoding: [0x48,0x29,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %sub = sub i64 %load1, %arg
@@ -842,14 +842,14 @@ define void @sub32_reg_br(i32 %arg) nounwind {
 ; CHECK-LABEL: sub32_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subl %edi, g32(%rip) # encoding: [0x29,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %sub = sub i32 %load1, %arg
@@ -870,14 +870,14 @@ define void @sub16_reg_br(i16 %arg) nounwind {
 ; CHECK-LABEL: sub16_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subw %di, g16(%rip) # encoding: [0x66,0x29,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %sub = sub i16 %load1, %arg
@@ -898,14 +898,14 @@ define void @sub8_reg_br(i8 %arg) nounwind {
 ; CHECK-LABEL: sub8_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subb %dil, g8(%rip) # encoding: [0x40,0x28,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g8-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g8, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %sub = sub i8 %load1, %arg
@@ -926,15 +926,15 @@ define void @and64_imm32_br() nounwind {
 ; CHECK-LABEL: and64_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andq $16777215, g64(%rip) # encoding: [0x48,0x81,0x25,A,A,A,A,0xff,0xff,0xff,0x00]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0xFFFFFF
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; And 0x00FFFFFF, a positive immediate requiring 24-bits.
@@ -956,15 +956,15 @@ define void @and64_sext_imm32_br() nounwind {
 ; CHECK-LABEL: and64_sext_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andq $-2147483648, g64(%rip) # encoding: [0x48,0x81,0x25,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; And -0x80000000, which requires sign-extended 32 bits.
@@ -986,14 +986,14 @@ define void @and64_imm8_br() nounwind {
 ; CHECK-LABEL: and64_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andq $15, g64(%rip) # encoding: [0x48,0x83,0x25,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %and = and i64 %load1, 15
@@ -1014,14 +1014,14 @@ define void @and64_imm8_neg_br() nounwind {
 ; CHECK-LABEL: and64_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andq $-4, g64(%rip) # encoding: [0x48,0x83,0x25,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %and = and i64 %load1, -4
@@ -1042,15 +1042,15 @@ define void @and32_imm_br() nounwind {
 ; CHECK-LABEL: and32_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andl $-2147483648, g32(%rip) # encoding: [0x81,0x25,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   ; And 0x80000000, a positive number requiring 32 bits of immediate.
@@ -1072,14 +1072,14 @@ define void @and32_imm8_br() nounwind {
 ; CHECK-LABEL: and32_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andl $15, g32(%rip) # encoding: [0x83,0x25,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %and = and i32 %load1, 15
@@ -1100,14 +1100,14 @@ define void @and32_imm8_neg_br() nounwind {
 ; CHECK-LABEL: and32_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andl $-4, g32(%rip) # encoding: [0x83,0x25,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %and = and i32 %load1, -4
@@ -1128,15 +1128,15 @@ define void @and16_imm_br() nounwind {
 ; CHECK-LABEL: and16_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andw $-32768, g16(%rip) # encoding: [0x66,0x81,0x25,A,A,A,A,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-6, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-2, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x8000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %and = and i16 %load1, 32768
@@ -1157,14 +1157,14 @@ define void @and16_imm8_br() nounwind {
 ; CHECK-LABEL: and16_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andw $15, g16(%rip) # encoding: [0x66,0x83,0x25,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %and = and i16 %load1, 15
@@ -1185,14 +1185,14 @@ define void @and16_imm8_neg_br() nounwind {
 ; CHECK-LABEL: and16_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andw $-4, g16(%rip) # encoding: [0x66,0x83,0x25,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %and = and i16 %load1, -4
@@ -1213,14 +1213,14 @@ define void @and8_imm_br() nounwind {
 ; CHECK-LABEL: and8_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andb $-4, g8(%rip) # encoding: [0x80,0x25,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g8-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g8-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %and = and i8 %load1, -4
@@ -1241,14 +1241,14 @@ define void @and64_reg_br(i64 %arg) nounwind {
 ; CHECK-LABEL: and64_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andq %rdi, g64(%rip) # encoding: [0x48,0x21,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %and = and i64 %load1, %arg
@@ -1269,14 +1269,14 @@ define void @and32_reg_br(i32 %arg) nounwind {
 ; CHECK-LABEL: and32_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andl %edi, g32(%rip) # encoding: [0x21,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %and = and i32 %load1, %arg
@@ -1297,14 +1297,14 @@ define void @and16_reg_br(i16 %arg) nounwind {
 ; CHECK-LABEL: and16_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andw %di, g16(%rip) # encoding: [0x66,0x21,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %and = and i16 %load1, %arg
@@ -1325,14 +1325,14 @@ define void @and8_reg_br(i8 %arg) nounwind {
 ; CHECK-LABEL: and8_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andb %dil, g8(%rip) # encoding: [0x40,0x20,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g8-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g8, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %and = and i8 %load1, %arg
@@ -1353,15 +1353,15 @@ define void @or64_imm32_br() nounwind {
 ; CHECK-LABEL: or64_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orq $16777215, g64(%rip) # encoding: [0x48,0x81,0x0d,A,A,A,A,0xff,0xff,0xff,0x00]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0xFFFFFF
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Or 0x00FFFFFF, a positive immediate requiring 24-bits.
@@ -1383,15 +1383,15 @@ define void @or64_sext_imm32_br() nounwind {
 ; CHECK-LABEL: or64_sext_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orq $-2147483648, g64(%rip) # encoding: [0x48,0x81,0x0d,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Or -0x80000000, which requires sign-extended 32 bits.
@@ -1413,14 +1413,14 @@ define void @or64_imm8_br() nounwind {
 ; CHECK-LABEL: or64_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orq $15, g64(%rip) # encoding: [0x48,0x83,0x0d,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %or = or i64 %load1, 15
@@ -1441,14 +1441,14 @@ define void @or64_imm8_neg_br() nounwind {
 ; CHECK-LABEL: or64_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orq $-4, g64(%rip) # encoding: [0x48,0x83,0x0d,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %or = or i64 %load1, -4
@@ -1469,15 +1469,15 @@ define void @or32_imm_br() nounwind {
 ; CHECK-LABEL: or32_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orl $-2147483648, g32(%rip) # encoding: [0x81,0x0d,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   ; Or 0x80000000, a positive number requiring 32 bits of immediate.
@@ -1499,14 +1499,14 @@ define void @or32_imm8_br() nounwind {
 ; CHECK-LABEL: or32_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orl $15, g32(%rip) # encoding: [0x83,0x0d,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %or = or i32 %load1, 15
@@ -1527,14 +1527,14 @@ define void @or32_imm8_neg_br() nounwind {
 ; CHECK-LABEL: or32_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orl $-4, g32(%rip) # encoding: [0x83,0x0d,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %or = or i32 %load1, -4
@@ -1555,15 +1555,15 @@ define void @or16_imm_br() nounwind {
 ; CHECK-LABEL: or16_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orw $-32768, g16(%rip) # encoding: [0x66,0x81,0x0d,A,A,A,A,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-6, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-2, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x8000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %or = or i16 %load1, 32768
@@ -1584,14 +1584,14 @@ define void @or16_imm8_br() nounwind {
 ; CHECK-LABEL: or16_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orw $15, g16(%rip) # encoding: [0x66,0x83,0x0d,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %or = or i16 %load1, 15
@@ -1612,14 +1612,14 @@ define void @or16_imm8_neg_br() nounwind {
 ; CHECK-LABEL: or16_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orw $-4, g16(%rip) # encoding: [0x66,0x83,0x0d,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %or = or i16 %load1, -4
@@ -1640,14 +1640,14 @@ define void @or8_imm_br() nounwind {
 ; CHECK-LABEL: or8_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orb $-4, g8(%rip) # encoding: [0x80,0x0d,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g8-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g8-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %or = or i8 %load1, -4
@@ -1668,14 +1668,14 @@ define void @or64_reg_br(i64 %arg) nounwind {
 ; CHECK-LABEL: or64_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orq %rdi, g64(%rip) # encoding: [0x48,0x09,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %or = or i64 %load1, %arg
@@ -1696,14 +1696,14 @@ define void @or32_reg_br(i32 %arg) nounwind {
 ; CHECK-LABEL: or32_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orl %edi, g32(%rip) # encoding: [0x09,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %or = or i32 %load1, %arg
@@ -1724,14 +1724,14 @@ define void @or16_reg_br(i16 %arg) nounwind {
 ; CHECK-LABEL: or16_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orw %di, g16(%rip) # encoding: [0x66,0x09,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %or = or i16 %load1, %arg
@@ -1752,14 +1752,14 @@ define void @or8_reg_br(i8 %arg) nounwind {
 ; CHECK-LABEL: or8_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    orb %dil, g8(%rip) # encoding: [0x40,0x08,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g8-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g8, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %or = or i8 %load1, %arg
@@ -1780,15 +1780,15 @@ define void @xor64_imm32_br() nounwind {
 ; CHECK-LABEL: xor64_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorq $16777215, g64(%rip) # encoding: [0x48,0x81,0x35,A,A,A,A,0xff,0xff,0xff,0x00]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0xFFFFFF
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Xor 0x00FFFFFF, a positive immediate requiring 24-bits.
@@ -1810,15 +1810,15 @@ define void @xor64_sext_imm32_br() nounwind {
 ; CHECK-LABEL: xor64_sext_imm32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorq $-2147483648, g64(%rip) # encoding: [0x48,0x81,0x35,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   ; Xor -0x80000000, which requires sign-extended 32 bits.
@@ -1840,14 +1840,14 @@ define void @xor64_imm8_br() nounwind {
 ; CHECK-LABEL: xor64_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorq $15, g64(%rip) # encoding: [0x48,0x83,0x35,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %xor = xor i64 %load1, 15
@@ -1868,14 +1868,14 @@ define void @xor64_imm8_neg_br() nounwind {
 ; CHECK-LABEL: xor64_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorq $-4, g64(%rip) # encoding: [0x48,0x83,0x35,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %xor = xor i64 %load1, -4
@@ -1896,15 +1896,15 @@ define void @xor32_imm_br() nounwind {
 ; CHECK-LABEL: xor32_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl $-2147483648, g32(%rip) # encoding: [0x81,0x35,A,A,A,A,0x00,0x00,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-8, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x80000000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   ; Xor 0x80000000, a positive number requiring 32 bits of immediate.
@@ -1926,14 +1926,14 @@ define void @xor32_imm8_br() nounwind {
 ; CHECK-LABEL: xor32_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl $15, g32(%rip) # encoding: [0x83,0x35,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %xor = xor i32 %load1, 15
@@ -1954,14 +1954,14 @@ define void @xor32_imm8_neg_br() nounwind {
 ; CHECK-LABEL: xor32_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl $-4, g32(%rip) # encoding: [0x83,0x35,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %xor = xor i32 %load1, -4
@@ -1982,15 +1982,15 @@ define void @xor16_imm_br() nounwind {
 ; CHECK-LABEL: xor16_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorw $-32768, g16(%rip) # encoding: [0x66,0x81,0x35,A,A,A,A,0x00,0x80]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-6, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-2, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # imm = 0x8000
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %xor = xor i16 %load1, 32768
@@ -2011,14 +2011,14 @@ define void @xor16_imm8_br() nounwind {
 ; CHECK-LABEL: xor16_imm8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorw $15, g16(%rip) # encoding: [0x66,0x83,0x35,A,A,A,A,0x0f]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %xor = xor i16 %load1, 15
@@ -2039,14 +2039,14 @@ define void @xor16_imm8_neg_br() nounwind {
 ; CHECK-LABEL: xor16_imm8_neg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorw $-4, g16(%rip) # encoding: [0x66,0x83,0x35,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %xor = xor i16 %load1, -4
@@ -2067,14 +2067,14 @@ define void @xor8_imm_br() nounwind {
 ; CHECK-LABEL: xor8_imm_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorb $-4, g8(%rip) # encoding: [0x80,0x35,A,A,A,A,0xfc]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g8-5, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g8-1, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %xor = xor i8 %load1, -4
@@ -2095,14 +2095,14 @@ define void @xor64_reg_br(i64 %arg) nounwind {
 ; CHECK-LABEL: xor64_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorq %rdi, g64(%rip) # encoding: [0x48,0x31,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %xor = xor i64 %load1, %arg
@@ -2123,14 +2123,14 @@ define void @xor32_reg_br(i32 %arg) nounwind {
 ; CHECK-LABEL: xor32_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %edi, g32(%rip) # encoding: [0x31,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %xor = xor i32 %load1, %arg
@@ -2151,14 +2151,14 @@ define void @xor16_reg_br(i16 %arg) nounwind {
 ; CHECK-LABEL: xor16_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorw %di, g16(%rip) # encoding: [0x66,0x31,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %xor = xor i16 %load1, %arg
@@ -2179,14 +2179,14 @@ define void @xor8_reg_br(i8 %arg) nounwind {
 ; CHECK-LABEL: xor8_reg_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorb %dil, g8(%rip) # encoding: [0x40,0x30,0x3d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g8-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g8, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jne b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x75,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %xor = xor i8 %load1, %arg
@@ -2207,14 +2207,14 @@ define void @neg64_br() nounwind {
 ; CHECK-LABEL: neg64_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    negq g64(%rip) # encoding: [0x48,0xf7,0x1d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g64, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i64, ptr @g64
   %sub = sub i64 0, %load1
@@ -2235,14 +2235,14 @@ define void @neg32_br() nounwind {
 ; CHECK-LABEL: neg32_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    negl g32(%rip) # encoding: [0xf7,0x1d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g32-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g32, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i32, ptr @g32
   %sub = sub i32 0, %load1
@@ -2263,14 +2263,14 @@ define void @neg16_br() nounwind {
 ; CHECK-LABEL: neg16_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    negw g16(%rip) # encoding: [0x66,0xf7,0x1d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 3, value: g16-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 3, value: g16, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i16, ptr @g16
   %sub = sub i16 0, %load1
@@ -2291,14 +2291,14 @@ define void @neg8_br() nounwind {
 ; CHECK-LABEL: neg8_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    negb g8(%rip) # encoding: [0xf6,0x1d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 2, value: g8-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 2, value: g8, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jns b # TAILCALL
 ; CHECK-NEXT:    # encoding: [0x79,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: b-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: b, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %a
 ; CHECK-NEXT:    jmp a # TAILCALL
 ; CHECK-NEXT:    # encoding: [0xeb,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: a-1, kind: FK_PCRel_1
+; CHECK-NEXT:    # fixup A - offset: 1, value: a, kind: FK_PCRel_1
 entry:
   %load1 = load i8, ptr @g8
   %sub = sub i8 0, %load1
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll b/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
index 4e36d5a31c3d9..bd32430fcedbd 100644
--- a/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
+++ b/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
@@ -60,7 +60,7 @@ define i8 @f20s8(double %x) #0 {
 entry:
 ; CHECK-LABEL: name: f20s8
 ; CHECK: [[CVTTSD2SIrm:%[0-9]+]]:gr32 = CVTTSD2SIrm %fixed-stack.0, 1, $noreg, 0, $noreg, implicit $mxcsr :: (load (s64) from %fixed-stack.0, align 16)
-; CHECK: [[COPY:%[0-9]+]]:gr32_abcd = COPY [[CVTTSD2SIrm]]
+; CHECK: [[COPY:%[0-9]+]]:gr32_abcd = COPY killed [[CVTTSD2SIrm]]
 ; CHECK: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
 ; CHECK: $al = COPY [[COPY1]]
 ; CHECK: RET 0, $al
diff --git a/llvm/test/CodeGen/X86/fp128-select.ll b/llvm/test/CodeGen/X86/fp128-select.ll
index 0486c1c4d28e9..659e4ddedc646 100644
--- a/llvm/test/CodeGen/X86/fp128-select.ll
+++ b/llvm/test/CodeGen/X86/fp128-select.ll
@@ -8,7 +8,7 @@
 ; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=-sse \
 ; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=NOSSE
 
-define void @test_select(ptr %p, ptr %q, i1 zeroext %c) {
+define void @test_select(ptr %p, ptr %q, i1 zeroext %c) nounwind {
 ; SSE-LABEL: test_select:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    testl %edx, %edx
@@ -40,14 +40,11 @@ define void @test_select(ptr %p, ptr %q, i1 zeroext %c) {
 
 ; The uitofp will become a select_cc. This used to crash during type
 ; legalization because we didn't expect the operands to need to be softened.
-define fp128 @test_select_cc(fp128, fp128) {
+define fp128 @test_select_cc(fp128, fp128) nounwind {
 ; SSE-LABEL: test_select_cc:
 ; SSE:       # %bb.0: # %BB0
 ; SSE-NEXT:    pushq %rbx
-; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    subq $32, %rsp
-; SSE-NEXT:    .cfi_def_cfa_offset 48
-; SSE-NEXT:    .cfi_offset %rbx, -16
 ; SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; SSE-NEXT:    callq __netf2@PLT
@@ -70,28 +67,16 @@ define fp128 @test_select_cc(fp128, fp128) {
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:  .LBB1_5: # %BB2
 ; SSE-NEXT:    addq $32, %rsp
-; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    popq %rbx
-; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
 ;
 ; NOSSE-LABEL: test_select_cc:
 ; NOSSE:       # %bb.0: # %BB0
 ; NOSSE-NEXT:    pushq %rbp
-; NOSSE-NEXT:    .cfi_def_cfa_offset 16
 ; NOSSE-NEXT:    pushq %r15
-; NOSSE-NEXT:    .cfi_def_cfa_offset 24
 ; NOSSE-NEXT:    pushq %r14
-; NOSSE-NEXT:    .cfi_def_cfa_offset 32
 ; NOSSE-NEXT:    pushq %r12
-; NOSSE-NEXT:    .cfi_def_cfa_offset 40
 ; NOSSE-NEXT:    pushq %rbx
-; NOSSE-NEXT:    .cfi_def_cfa_offset 48
-; NOSSE-NEXT:    .cfi_offset %rbx, -48
-; NOSSE-NEXT:    .cfi_offset %r12, -40
-; NOSSE-NEXT:    .cfi_offset %r14, -32
-; NOSSE-NEXT:    .cfi_offset %r15, -24
-; NOSSE-NEXT:    .cfi_offset %rbp, -16
 ; NOSSE-NEXT:    movq %rcx, %r15
 ; NOSSE-NEXT:    movq %rdx, %r12
 ; NOSSE-NEXT:    movq %rsi, %rbx
@@ -115,15 +100,10 @@ define fp128 @test_select_cc(fp128, fp128) {
 ; NOSSE-NEXT:    movq %rbx, %rdx
 ; NOSSE-NEXT:  .LBB1_2: # %BB2
 ; NOSSE-NEXT:    popq %rbx
-; NOSSE-NEXT:    .cfi_def_cfa_offset 40
 ; NOSSE-NEXT:    popq %r12
-; NOSSE-NEXT:    .cfi_def_cfa_offset 32
 ; NOSSE-NEXT:    popq %r14
-; NOSSE-NEXT:    .cfi_def_cfa_offset 24
 ; NOSSE-NEXT:    popq %r15
-; NOSSE-NEXT:    .cfi_def_cfa_offset 16
 ; NOSSE-NEXT:    popq %rbp
-; NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; NOSSE-NEXT:    retq
 BB0:
   %a = fcmp oeq fp128 %0, %1
diff --git a/llvm/test/CodeGen/X86/frame-pointer-reserved.ll b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
new file mode 100644
index 0000000000000..bf5887fe50508
--- /dev/null
+++ b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare void @bar(i32, i32)
+
+define void @foo(i32 %0, i32 %1) nounwind "frame-pointer"="reserved" {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    callq bar@PLT
+; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    callq bar@PLT
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+  call void @bar(i32 %0, i32 %1)
+  call void @bar(i32 %0, i32 %1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index 506a08808ff91..189de051011d2 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -802,6 +802,126 @@ define i32 @freeze_fshr(i32 %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
+define i32 @freeze_saddo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_saddo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    adcl $0, %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_saddo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    adcl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = add i32 %v, %a1
+  ret i32 %r
+}
+
+define i32 @freeze_uaddo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_uaddo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    adcl $0, %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_uaddo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    adcl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = add i32 %v, %a1
+  ret i32 %r
+}
+
+define i32 @freeze_ssubo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_ssubo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    setb %cl
+; X86-NEXT:    andl $1, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ssubo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = sub i32 %v, %a1
+  ret i32 %r
+}
+
+define i32 @freeze_usubo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_usubo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    setb %cl
+; X86-NEXT:    andl $1, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_usubo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = sub i32 %v, %a1
+  ret i32 %r
+}
+
 define void @pr59676_frozen(ptr %dst, i32 %x.orig) {
 ; X86-LABEL: pr59676_frozen:
 ; X86:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/freeze-unary.ll b/llvm/test/CodeGen/X86/freeze-unary.ll
index f2c02de74c07c..8602c385af834 100644
--- a/llvm/test/CodeGen/X86/freeze-unary.ll
+++ b/llvm/test/CodeGen/X86/freeze-unary.ll
@@ -126,6 +126,140 @@ define <4 x i32> @freeze_bitreverse_vec(<4 x i32> %a0) nounwind {
 }
 declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
 
+define i32 @freeze_ctlz(i32 %a0) nounwind {
+; X86-LABEL: freeze_ctlz:
+; X86:       # %bb.0:
+; X86-NEXT:    bsrl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $63, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ctlz:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $63, %eax
+; X64-NEXT:    bsrl %edi, %eax
+; X64-NEXT:    xorl $31, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.ctlz.i32(i32 %a0, i1 0)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_ctlz_undef(i32 %a0) nounwind {
+; X86-LABEL: freeze_ctlz_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    bsrl %eax, %ecx
+; X86-NEXT:    xorl $31, %ecx
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ctlz_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsrl %edi, %ecx
+; X64-NEXT:    xorl $31, %ecx
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.ctlz.i32(i32 %a0, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_ctlz_undef_nonzero(i32 %a0) nounwind {
+; X86-LABEL: freeze_ctlz_undef_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    bsrl %eax, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ctlz_undef_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    bsrl %edi, %eax
+; X64-NEXT:    xorl $31, %eax
+; X64-NEXT:    retq
+  %y = or i32 %a0, 1
+  %x = call i32 @llvm.ctlz.i32(i32 %y, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %y, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_cttz(i32 %a0) nounwind {
+; X86-LABEL: freeze_cttz:
+; X86:       # %bb.0:
+; X86-NEXT:    bsfl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_cttz:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.cttz.i32(i32 %a0, i1 0)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_cttz_undef(i32 %a0) nounwind {
+; X86-LABEL: freeze_cttz_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    bsfl %eax, %ecx
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_cttz_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsfl %edi, %ecx
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.cttz.i32(i32 %a0, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_cttz_undef_nonzero(i32 %a0) nounwind {
+; X86-LABEL: freeze_cttz_undef_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_cttz_undef_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+  %y = or i32 %a0, 1
+  %x = call i32 @llvm.cttz.i32(i32 %y, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %y, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
 ; split parity pattern
 define i8 @freeze_ctpop(i8 %a0) nounwind {
 ; X86-LABEL: freeze_ctpop:
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 362b3b945f962..0f66d42697d97 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -171,15 +171,15 @@ define void @freeze_extractelement(ptr %origin0, ptr %origin1, ptr %dst) nounwin
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    vmovdqa (%edx), %xmm0
-; X86-NEXT:    vpand (%ecx), %xmm0, %xmm0
+; X86-NEXT:    vmovdqa (%ecx), %xmm0
+; X86-NEXT:    vpand (%edx), %xmm0, %xmm0
 ; X86-NEXT:    vpextrb $6, %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_extractelement:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa (%rdi), %xmm0
-; X64-NEXT:    vpand (%rsi), %xmm0, %xmm0
+; X64-NEXT:    vmovdqa (%rsi), %xmm0
+; X64-NEXT:    vpand (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    vpextrb $6, %xmm0, (%rdx)
 ; X64-NEXT:    retq
   %i0 = load <16 x i8>, ptr %origin0
@@ -198,8 +198,8 @@ define void @freeze_extractelement_escape(ptr %origin0, ptr %origin1, ptr %dst,
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    vmovdqa (%esi), %xmm0
-; X86-NEXT:    vpand (%edx), %xmm0, %xmm0
+; X86-NEXT:    vmovdqa (%edx), %xmm0
+; X86-NEXT:    vpand (%esi), %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
 ; X86-NEXT:    vpextrb $6, %xmm0, (%eax)
 ; X86-NEXT:    popl %esi
@@ -207,8 +207,8 @@ define void @freeze_extractelement_escape(ptr %origin0, ptr %origin1, ptr %dst,
 ;
 ; X64-LABEL: freeze_extractelement_escape:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa (%rdi), %xmm0
-; X64-NEXT:    vpand (%rsi), %xmm0, %xmm0
+; X64-NEXT:    vmovdqa (%rsi), %xmm0
+; X64-NEXT:    vpand (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    vpextrb $6, %xmm0, (%rdx)
 ; X64-NEXT:    retq
@@ -239,8 +239,8 @@ define void @freeze_extractelement_extra_use(ptr %origin0, ptr %origin1, i64 %id
 ; X86-NEXT:    movl 32(%ebp), %edx
 ; X86-NEXT:    movl 12(%ebp), %esi
 ; X86-NEXT:    movl 8(%ebp), %edi
-; X86-NEXT:    vmovaps (%edi), %xmm0
-; X86-NEXT:    vandps (%esi), %xmm0, %xmm0
+; X86-NEXT:    vmovaps (%esi), %xmm0
+; X86-NEXT:    vandps (%edi), %xmm0, %xmm0
 ; X86-NEXT:    vmovaps %xmm0, (%esp)
 ; X86-NEXT:    movzbl (%esp,%ecx), %ecx
 ; X86-NEXT:    cmpb (%esp,%eax), %cl
@@ -255,8 +255,8 @@ define void @freeze_extractelement_extra_use(ptr %origin0, ptr %origin1, i64 %id
 ; X64:       # %bb.0:
 ; X64-NEXT:    andl $15, %ecx
 ; X64-NEXT:    andl $15, %edx
-; X64-NEXT:    vmovaps (%rdi), %xmm0
-; X64-NEXT:    vandps (%rsi), %xmm0, %xmm0
+; X64-NEXT:    vmovaps (%rsi), %xmm0
+; X64-NEXT:    vandps (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movzbl -24(%rsp,%rdx), %eax
 ; X64-NEXT:    cmpb -24(%rsp,%rcx), %al
@@ -342,42 +342,50 @@ define void @freeze_buildvector_single_repeated_maybe_poison_operand(ptr %origin
 define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst0, ptr %dst1) nounwind {
 ; X86-LABEL: freeze_two_frozen_buildvectors:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %esi
+; X86-NEXT:    andl $15, %esi
 ; X86-NEXT:    movl (%edx), %edx
 ; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
-; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [7,7,7,7]
-; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovd %esi, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; X86-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
+; X86-NEXT:    vbroadcastss {{.*#+}} xmm2 = [7,7,7,7]
+; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
 ; X86-NEXT:    vmovd %edx, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
-; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_two_frozen_buildvectors:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    andl $15, %eax
-; X64-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
-; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    movl (%rsi), %eax
+; X64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
+; X64-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
+; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rdx)
 ; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
-; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
+; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i32, ptr %origin0
   %i0 = and i32 %i0.src, 15
   %i1.src = load i32, ptr %origin1
-  %i1 = and i32 %i0.src, 15
+  %i1 = and i32 %i1.src, 15
   %i2 = insertelement <4 x i32> poison, i32 %i0, i64 1
   %i3 = and <4 x i32> %i2, <i32 7, i32 7, i32 7, i32 7>
   %i4 = freeze <4 x i32> %i3
@@ -392,41 +400,43 @@ define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst
 define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1, ptr %dst0, ptr %dst1) nounwind {
 ; X86-LABEL: freeze_two_buildvectors_only_one_frozen:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl (%edx), %edx
-; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X86-NEXT:    vmovd %edx, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]
-; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6,7]
-; X86-NEXT:    vbroadcastss {{.*#+}} xmm2 = [7,7,7,7]
-; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %esi
+; X86-NEXT:    andl $15, %esi
+; X86-NEXT:    vmovd %esi, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; X86-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
+; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [7,7,7,7]
+; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vbroadcastss (%edx), %xmm2
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,1,0,1]
-; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vpand %xmm1, %xmm2, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_two_buildvectors_only_one_frozen:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    andl $15, %eax
-; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X64-NEXT:    vmovd %eax, %xmm1
-; X64-NEXT:    vpbroadcastd %xmm1, %xmm1
-; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; X64-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
-; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vmovdqa %xmm0, (%rdx)
-; X64-NEXT:    vpand %xmm2, %xmm1, %xmm0
-; X64-NEXT:    vmovdqa %xmm0, (%rcx)
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vbroadcastss %xmm0, %xmm0
+; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
+; X64-NEXT:    vbroadcastss {{.*#+}} xmm1 = [7,7,7,7]
+; X64-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vbroadcastss (%rsi), %xmm2
+; X64-NEXT:    vmovaps %xmm0, (%rdx)
+; X64-NEXT:    vandps %xmm1, %xmm2, %xmm0
+; X64-NEXT:    vmovaps %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i32, ptr %origin0
   %i0 = and i32 %i0.src, 15
   %i1.src = load i32, ptr %origin1
-  %i1 = and i32 %i0.src, 15
+  %i1 = and i32 %i1.src, 15
   %i2 = insertelement <4 x i32> poison, i32 %i0, i64 1
   %i3 = and <4 x i32> %i2, <i32 7, i32 7, i32 7, i32 7>
   %i4 = freeze <4 x i32> %i3
@@ -440,34 +450,40 @@ define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1,
 define void @freeze_two_buildvectors_one_undef_elt(ptr %origin0, ptr %origin1, ptr %dst0, ptr %dst1) nounwind {
 ; X86-LABEL: freeze_two_buildvectors_one_undef_elt:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl (%edx), %edx
-; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vmovddup {{.*#+}} xmm0 = [7,0,7,0]
-; X86-NEXT:    # xmm0 = mem[0,0]
-; X86-NEXT:    vmovd %edx, %xmm1
-; X86-NEXT:    vpand %xmm0, %xmm1, %xmm2
-; X86-NEXT:    vmovdqa %xmm2, (%ecx)
-; X86-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %esi
+; X86-NEXT:    andl $15, %esi
+; X86-NEXT:    vmovd %esi, %xmm0
+; X86-NEXT:    vmovddup {{.*#+}} xmm1 = [7,0,7,0]
+; X86-NEXT:    # xmm1 = mem[0,0]
+; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovddup {{.*#+}} xmm2 = mem[0,0]
+; X86-NEXT:    vmovdqa %xmm0, (%ecx)
+; X86-NEXT:    vpand %xmm1, %xmm2, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_two_buildvectors_one_undef_elt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    andl $15, %eax
 ; X64-NEXT:    vmovd %eax, %xmm0
-; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,7]
+; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpbroadcastq (%rsi), %xmm2
 ; X64-NEXT:    vmovdqa %xmm0, (%rdx)
+; X64-NEXT:    vpand %xmm1, %xmm2, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i64, ptr %origin0
   %i0 = and i64 %i0.src, 15
   %i1.src = load i64, ptr %origin1
-  %i1 = and i64 %i0.src, 15
+  %i1 = and i64 %i1.src, 15
   %i2 = insertelement <2 x i64> poison, i64 %i0, i64 0
   %i3 = and <2 x i64> %i2, <i64 7, i64 7>
   %i4 = freeze <2 x i64> %i3
diff --git a/llvm/test/CodeGen/X86/isel-fabs-x87.ll b/llvm/test/CodeGen/X86/isel-fabs-x87.ll
index a0534e6a1a82e..5579754a54b46 100644
--- a/llvm/test/CodeGen/X86/isel-fabs-x87.ll
+++ b/llvm/test/CodeGen/X86/isel-fabs-x87.ll
@@ -1,29 +1,44 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -fast-isel | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X86,SDAG-ISEL
-; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -fast-isel | FileCheck %s --check-prefixes=X86,Fast-ISEL
-; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -global-isel -global-isel-abort=0 | FileCheck %s --check-prefixes=X86,GISEL-ISEL
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X64,SDAG-X64-ISEL
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -fast-isel | FileCheck %s --check-prefixes=X64,FAST-X64-ISEL
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=X64,GISEL-X64-ISEL
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X86,SDAG-X86-ISEL
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -fast-isel | FileCheck %s --check-prefixes=X86,FAST-X86-ISEL
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -global-isel -global-isel-abort=0 | FileCheck %s --check-prefixes=X86,GISEL-X86-ISEL
 
 define void @test_float_abs(ptr %argptr)   {
-; SDAG-ISEL-LABEL: test_float_abs:
-; SDAG-ISEL:       # %bb.0:
-; SDAG-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SDAG-ISEL-NEXT:    andb $127, 3(%eax)
-; SDAG-ISEL-NEXT:    retl
-;
-; Fast-ISEL-LABEL: test_float_abs:
-; Fast-ISEL:       # %bb.0:
-; Fast-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; Fast-ISEL-NEXT:    andb $127, 3(%eax)
-; Fast-ISEL-NEXT:    retl
-;
-; GISEL-ISEL-LABEL: test_float_abs:
-; GISEL-ISEL:       # %bb.0:
-; GISEL-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GISEL-ISEL-NEXT:    andl $2147483647, (%eax) # imm = 0x7FFFFFFF
-; GISEL-ISEL-NEXT:    retl
+; SDAG-X64-ISEL-LABEL: test_float_abs:
+; SDAG-X64-ISEL:       # %bb.0:
+; SDAG-X64-ISEL-NEXT:    andb $127, 3(%rdi)
+; SDAG-X64-ISEL-NEXT:    retq
+;
+; FAST-X64-ISEL-LABEL: test_float_abs:
+; FAST-X64-ISEL:       # %bb.0:
+; FAST-X64-ISEL-NEXT:    andb $127, 3(%rdi)
+; FAST-X64-ISEL-NEXT:    retq
+;
+; GISEL-X64-ISEL-LABEL: test_float_abs:
+; GISEL-X64-ISEL:       # %bb.0:
+; GISEL-X64-ISEL-NEXT:    andl $2147483647, (%rdi) # imm = 0x7FFFFFFF
+; GISEL-X64-ISEL-NEXT:    retq
+;
+; SDAG-X86-ISEL-LABEL: test_float_abs:
+; SDAG-X86-ISEL:       # %bb.0:
+; SDAG-X86-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-ISEL-NEXT:    andb $127, 3(%eax)
+; SDAG-X86-ISEL-NEXT:    retl
+;
+; FAST-X86-ISEL-LABEL: test_float_abs:
+; FAST-X86-ISEL:       # %bb.0:
+; FAST-X86-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FAST-X86-ISEL-NEXT:    andb $127, 3(%eax)
+; FAST-X86-ISEL-NEXT:    retl
+;
+; GISEL-X86-ISEL-LABEL: test_float_abs:
+; GISEL-X86-ISEL:       # %bb.0:
+; GISEL-X86-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-ISEL-NEXT:    andl $2147483647, (%eax) # imm = 0x7FFFFFFF
+; GISEL-X86-ISEL-NEXT:    retl
      %arg = load  float, float* %argptr
      %abs = tail call float @llvm.fabs.f32(float %arg)
      store float %abs, ptr %argptr
@@ -31,6 +46,22 @@ define void @test_float_abs(ptr %argptr)   {
  }
 
 define void @test_double_abs(ptr %argptr)  {
+; SDAG-X64-ISEL-LABEL: test_double_abs:
+; SDAG-X64-ISEL:       # %bb.0:
+; SDAG-X64-ISEL-NEXT:    andb $127, 7(%rdi)
+; SDAG-X64-ISEL-NEXT:    retq
+;
+; FAST-X64-ISEL-LABEL: test_double_abs:
+; FAST-X64-ISEL:       # %bb.0:
+; FAST-X64-ISEL-NEXT:    andb $127, 7(%rdi)
+; FAST-X64-ISEL-NEXT:    retq
+;
+; GISEL-X64-ISEL-LABEL: test_double_abs:
+; GISEL-X64-ISEL:       # %bb.0:
+; GISEL-X64-ISEL-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; GISEL-X64-ISEL-NEXT:    andq %rax, (%rdi)
+; GISEL-X64-ISEL-NEXT:    retq
+;
 ; X86-LABEL: test_double_abs:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/legalize-sub-zero.ll b/llvm/test/CodeGen/X86/legalize-sub-zero.ll
index 51f17512cae69..296ae3f9e8823 100644
--- a/llvm/test/CodeGen/X86/legalize-sub-zero.ll
+++ b/llvm/test/CodeGen/X86/legalize-sub-zero.ll
@@ -1,13 +1,17 @@
-; RUN: llc < %s -mtriple=i686-pc-win32
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
 
 ;target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 ;target triple = "i686-pc-win32"
 
 define void @test() nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $0, (%eax)
+; CHECK-NEXT:    retl
   %1 = fdiv <3 x double> zeroinitializer, undef
   %2 = fdiv <2 x double> zeroinitializer, undef
-  %3 = shufflevector <2 x double> %2, <2 x double> undef, <3 x i32> <i32 0, i32
-1, i32 undef>
+  %3 = shufflevector <2 x double> %2, <2 x double> undef, <3 x i32> <i32 0, i32 1, i32 undef>
   %4 = insertelement <3 x double> %3, double undef, i32 2
   %5 = bitcast <3 x double> %1 to <3 x i64>
   %6 = bitcast <3 x double> %4 to <3 x i64>
@@ -15,20 +19,16 @@ define void @test() nounwind {
   %8 = shufflevector <3 x i64> %7, <3 x i64> undef, <2 x i32> <i32 0, i32 1>
   %9 = xor <2 x i64> %8, zeroinitializer
   %10 = add nsw <2 x i64> %9, zeroinitializer
-  %11 = shufflevector <2 x i64> %10, <2 x i64> undef, <3 x i32> <i32 0, i32 1,
-i32 undef>
+  %11 = shufflevector <2 x i64> %10, <2 x i64> undef, <3 x i32> <i32 0, i32 1, i32 undef>
   %12 = insertelement <3 x i64> %11, i64 0, i32 2
-  %13 = shufflevector <3 x i64> %12, <3 x i64> undef, <4 x i32> <i32 0, i32 1,
-i32 2, i32 3>
+  %13 = shufflevector <3 x i64> %12, <3 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %14 = shufflevector <4 x i64> %13, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
   %15 = bitcast <2 x i64> %14 to <4 x i32>
-  %16 = shufflevector <4 x i32> %15, <4 x i32> undef, <4 x i32> <i32 0, i32 2,
-i32 0, i32 2>
+  %16 = shufflevector <4 x i32> %15, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 0, i32 2>
   %17 = bitcast <4 x i32> %16 to <2 x i64>
   %18 = shufflevector <2 x i64> %17, <2 x i64> undef, <2 x i32> <i32 0, i32 2>
   %19 = bitcast <2 x i64> %18 to <4 x i32>
-  %20 = shufflevector <4 x i32> %19, <4 x i32> undef, <3 x i32> <i32 0, i32 1,
-i32 2>
+  %20 = shufflevector <4 x i32> %19, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
   %21 = or <3 x i32> %20, zeroinitializer
   store <3 x i32> %21, ptr addrspace(1) undef, align 16
   ret void
diff --git a/llvm/test/CodeGen/X86/pr134607.ll b/llvm/test/CodeGen/X86/pr134607.ll
index 5e824c22e5a22..2b383fc964298 100644
--- a/llvm/test/CodeGen/X86/pr134607.ll
+++ b/llvm/test/CodeGen/X86/pr134607.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse -O3 | FileCheck %s --check-prefixes=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2,+sse -O3 | FileCheck %s --check-prefixes=X64-SSE1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2,+sse -O3 | FileCheck %s --check-prefixes=X64-SSE2
@@ -5,16 +6,22 @@
 define void @store_v2f32_constant(ptr %v) {
 ; X86-LABEL: store_v2f32_constant:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl 4(%esp), %eax
-; X86-NEXT:    movaps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movaps {{.*#+}} xmm0 = [2.56E+2,5.12E+2,u,u]
+; X86-NEXT:    movlps %xmm0, (%eax)
+; X86-NEXT:    retl
+;
 ; X64-SSE1-LABEL: store_v2f32_constant:
 ; X64-SSE1:       # %bb.0:
-; X64-SSE1-NEXT:    movaps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-
+; X64-SSE1-NEXT:    movaps {{.*#+}} xmm0 = [2.56E+2,5.12E+2,u,u]
+; X64-SSE1-NEXT:    movlps %xmm0, (%rdi)
+; X64-SSE1-NEXT:    retq
+;
 ; X64-SSE2-LABEL: store_v2f32_constant:
 ; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    movsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT:    movsd {{.*#+}} xmm0 = [2.56E+2,5.12E+2,0.0E+0,0.0E+0]
+; X64-SSE2-NEXT:    movsd %xmm0, (%rdi)
+; X64-SSE2-NEXT:    retq
   store <2 x float> <float 2.560000e+02, float 5.120000e+02>, ptr %v, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/pr38865.ll b/llvm/test/CodeGen/X86/pr38865.ll
index a09ef52c63d0e..221ec22ecd0b3 100644
--- a/llvm/test/CodeGen/X86/pr38865.ll
+++ b/llvm/test/CodeGen/X86/pr38865.ll
@@ -21,7 +21,7 @@ define dso_local void @e() nounwind {
 ; CHECK-NEXT:    movl $260, %edx # encoding: [0xba,0x04,0x01,0x00,0x00]
 ; CHECK-NEXT:    # imm = 0x104
 ; CHECK-NEXT:    callq memcpy@PLT # encoding: [0xe8,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: memcpy@PLT-4, kind: FK_PCRel_4
+; CHECK-NEXT:    # fixup A - offset: 1, value: memcpy@PLT, kind: FK_PCRel_4
 ; CHECK-NEXT:    movl $32, %ecx # encoding: [0xb9,0x20,0x00,0x00,0x00]
 ; CHECK-NEXT:    movl %esp, %edi # encoding: [0x89,0xe7]
 ; CHECK-NEXT:    movl %ebx, %esi # encoding: [0x89,0xde]
@@ -29,7 +29,7 @@ define dso_local void @e() nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x67,0x8b,0x84,0x24,0x08,0x02,0x00,0x00]
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp) # encoding: [0x67,0x89,0x84,0x24,0x00,0x01,0x00,0x00]
 ; CHECK-NEXT:    callq d # encoding: [0xe8,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: d-4, kind: FK_PCRel_4
+; CHECK-NEXT:    # fixup A - offset: 1, value: d, kind: FK_PCRel_4
 ; CHECK-NEXT:    addl $528, %esp # encoding: [0x81,0xc4,0x10,0x02,0x00,0x00]
 ; CHECK-NEXT:    # imm = 0x210
 ; CHECK-NEXT:    popq %rbx # encoding: [0x5b]
diff --git a/llvm/test/CodeGen/X86/pr43157.ll b/llvm/test/CodeGen/X86/pr43157.ll
index 2b333782f43a0..308a8b91d859e 100644
--- a/llvm/test/CodeGen/X86/pr43157.ll
+++ b/llvm/test/CodeGen/X86/pr43157.ll
@@ -1,6 +1,7 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: not llc < %s -mtriple=i686-pc-linux -o - -mattr=+sse2 2>&1 | FileCheck %s --check-prefix=ERR
 ; RUN: llc < %s -mtriple=x86_64-pc-linux -o - -mattr=+mmx | FileCheck %s
 
+; ERR: error: couldn't allocate input reg for constraint 'x'
 define void @foo(fp128 %x) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/pr61384.ll b/llvm/test/CodeGen/X86/pr61384.ll
index e5178b4904b7e..1bdb13ca71e69 100644
--- a/llvm/test/CodeGen/X86/pr61384.ll
+++ b/llvm/test/CodeGen/X86/pr61384.ll
@@ -8,7 +8,7 @@ define i32 @atomic_global() nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    lock btsq %rax, a(%rip) # encoding: [0xf0,0x48,0x0f,0xab,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 5, value: a-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 5, value: a, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
 entry:
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index fe71a16039c19..084bd797df945 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -1177,7 +1177,51 @@ bb:
   %tmp28 = extractelement <16 x i32> %tmp27, i64 0
   ret i32 %tmp28
 }
+declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1 immarg)
+
+define i32 @PR143456(ptr %p0, ptr %p1) {
+; SSE2-LABEL: PR143456:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pminub %xmm1, %xmm2
+; SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; SSE2-NEXT:    psubb %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: PR143456:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    retq
+  %v0 = load <8 x i8>, ptr %p0, align 1
+  %v1 = load <8 x i8>, ptr %p1, align 1
+  %max = tail call <8 x i8> @llvm.umax.v8i8(<8 x i8> %v0, <8 x i8> %v1)
+  %min = tail call <8 x i8> @llvm.umin.v8i8(<8 x i8> %v0, <8 x i8> %v1)
+  %abd = sub nuw <8 x i8> %max, %min
+  %rdx.shuf = shufflevector <8 x i8> %abd, <8 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %bin.rdx = add <8 x i8> %abd, %rdx.shuf
+  %rdx.shuf15 = shufflevector <8 x i8> %bin.rdx, <8 x i8> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %bin.rdx16 = add <8 x i8> %bin.rdx, %rdx.shuf15
+  %rdx.shuf17 = shufflevector <8 x i8> %bin.rdx16, <8 x i8> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %bin.rdx18 = add <8 x i8> %bin.rdx16, %rdx.shuf17
+  %elt = extractelement <8 x i8> %bin.rdx18, i32 0
+  %res = zext i8 %elt to i32
+  ret i32 %res
+}
+declare <8 x i8> @llvm.umax.v8i8(<8 x i8>, <8 x i8>)
+declare <8 x i8> @llvm.umin.v8i8(<8 x i8>, <8 x i8>)
+declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
 
-; Function Attrs: nofree nosync nounwind readnone speculatable willreturn
-declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1 immarg) #0
 attributes #0 = { nofree nosync nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 2ac2be5545dfd..d2b292f1a7996 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -119,8 +119,8 @@ define void @failing(ptr %0, ptr %1) nounwind {
 ; CHECK-AVX2-NEXT:  .LBB0_2: # %vector.body
 ; CHECK-AVX2-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-AVX2-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-AVX2-NEXT:    vmovdqu 1024(%rdx,%rsi), %xmm5
-; CHECK-AVX2-NEXT:    vmovdqu 1040(%rdx,%rsi), %xmm6
+; CHECK-AVX2-NEXT:    vmovdqu 1024(%rdx,%rsi), %ymm5
+; CHECK-AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; CHECK-AVX2-NEXT:    vpextrq $1, %xmm5, %rdi
 ; CHECK-AVX2-NEXT:    vpextrq $1, %xmm6, %r8
 ; CHECK-AVX2-NEXT:    vmovq %xmm5, %r9
diff --git a/llvm/test/CodeGen/X86/sjlj-eh.ll b/llvm/test/CodeGen/X86/sjlj-eh.ll
index d2dcb35a4908e..4bc24c574d752 100644
--- a/llvm/test/CodeGen/X86/sjlj-eh.ll
+++ b/llvm/test/CodeGen/X86/sjlj-eh.ll
@@ -117,10 +117,10 @@ try.cont:
 ; CHECK-X64: [[CONT]]:
 ;     *Handlers[UFC.__callsite]
 ; CHECK-X64: leaq .[[TABLE:LJTI[0-9]+_[0-9]+]](%rip), %rcx
-; CHECK-X64: movl (%rcx,%rax,4), %eax
-; CHECK-X64: cltq
-; CHECK-X64: addq %rcx, %rax
-; CHECK-X64: jmpq *%rax
+; CHECK-X64: movl (%rcx,%rax,4), %rax
+; CHECK-X64: leaq __ImageBase@IMGREL(%rip), %rcx
+; CHECK-X64: addq %rax, %rcx
+; CHECK-X64: jmpq *%rcx
 
 ; CHECK-X64-LINUX: .[[RESUME:LBB[0-9]+_[0-9]+]]:
 ;     assert(UFC.__callsite < 1);
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
index 78df4f685f6e9..42617c1573be5 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
@@ -35,15 +35,15 @@ define float @sqrt_ieee_ninf(float %f) #0 {
   ; CHECK-NEXT:   [[VFMADD213SSr1:%[0-9]+]]:fr32 = ninf afn nofpexcept VFMADD213SSr [[VMULSSrr2]], [[VMULSSrr3]], [[VMOVSSrm_alt]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr4:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr [[VMULSSrr3]], [[VMOVSSrm_alt1]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr5:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr killed [[VMULSSrr4]], killed [[VFMADD213SSr1]], implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY [[VMULSSrr5]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY killed [[VMULSSrr5]]
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr128 = COPY [[COPY]]
   ; CHECK-NEXT:   [[VPBROADCASTDrm:%[0-9]+]]:vr128 = VPBROADCASTDrm $rip, 1, $noreg, %const.2, $noreg :: (load (s32) from constant-pool)
   ; CHECK-NEXT:   [[VPANDrr:%[0-9]+]]:vr128 = VPANDrr killed [[COPY2]], killed [[VPBROADCASTDrm]]
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY [[VPANDrr]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY killed [[VPANDrr]]
   ; CHECK-NEXT:   [[VCMPSSrmi:%[0-9]+]]:fr32 = nofpexcept VCMPSSrmi killed [[COPY3]], $rip, 1, $noreg, %const.3, $noreg, 1, implicit $mxcsr :: (load (s32) from constant-pool)
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vr128 = COPY [[VCMPSSrmi]]
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vr128 = COPY killed [[VCMPSSrmi]]
   ; CHECK-NEXT:   [[VPANDNrr:%[0-9]+]]:vr128 = VPANDNrr killed [[COPY4]], killed [[COPY1]]
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:fr32 = COPY [[VPANDNrr]]
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:fr32 = COPY killed [[VPANDNrr]]
   ; CHECK-NEXT:   $xmm0 = COPY [[COPY5]]
   ; CHECK-NEXT:   RET 0, $xmm0
   %call = tail call ninf afn float @llvm.sqrt.f32(float %f)
@@ -82,12 +82,12 @@ define float @sqrt_daz_ninf(float %f) #1 {
   ; CHECK-NEXT:   [[VFMADD213SSr1:%[0-9]+]]:fr32 = ninf afn nofpexcept VFMADD213SSr [[VMULSSrr2]], [[VMULSSrr3]], [[VMOVSSrm_alt]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr4:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr [[VMULSSrr3]], [[VMOVSSrm_alt1]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr5:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr killed [[VMULSSrr4]], killed [[VFMADD213SSr1]], implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY [[VMULSSrr5]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY killed [[VMULSSrr5]]
   ; CHECK-NEXT:   [[FsFLD0SS:%[0-9]+]]:fr32 = FsFLD0SS
   ; CHECK-NEXT:   [[VCMPSSrri:%[0-9]+]]:fr32 = nofpexcept VCMPSSrri [[COPY]], killed [[FsFLD0SS]], 0, implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr128 = COPY [[VCMPSSrri]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr128 = COPY killed [[VCMPSSrri]]
   ; CHECK-NEXT:   [[VPANDNrr:%[0-9]+]]:vr128 = VPANDNrr killed [[COPY2]], killed [[COPY1]]
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY [[VPANDNrr]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY killed [[VPANDNrr]]
   ; CHECK-NEXT:   $xmm0 = COPY [[COPY3]]
   ; CHECK-NEXT:   RET 0, $xmm0
   %call = tail call ninf afn float @llvm.sqrt.f32(float %f)
diff --git a/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll b/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
index 413b4e79257a0..6c94f82ac40d9 100644
--- a/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
+++ b/llvm/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
@@ -362,7 +362,7 @@ define void @test_x86_sse2_storeu_pd(ptr %a0, <2 x double> %a1) {
 ; X64-SSE:       ## %bb.0:
 ; X64-SSE-NEXT:    xorpd %xmm1, %xmm1 ## encoding: [0x66,0x0f,0x57,0xc9]
 ; X64-SSE-NEXT:    movhpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 ## encoding: [0x66,0x0f,0x16,0x0d,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; X64-SSE-NEXT:    addpd %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x58,0xc8]
 ; X64-SSE-NEXT:    movupd %xmm1, (%rdi) ## encoding: [0x66,0x0f,0x11,0x0f]
@@ -372,7 +372,7 @@ define void @test_x86_sse2_storeu_pd(ptr %a0, <2 x double> %a1) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0x57,0xc9]
 ; X64-AVX1-NEXT:    vmovhpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; X64-AVX1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x58,0xc1]
 ; X64-AVX1-NEXT:    vmovupd %xmm0, (%rdi) ## encoding: [0xc5,0xf9,0x11,0x07]
@@ -382,7 +382,7 @@ define void @test_x86_sse2_storeu_pd(ptr %a0, <2 x double> %a1) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0x57,0xc9]
 ; X64-AVX512-NEXT:    vmovhpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    ## xmm1 = xmm1[0],mem[0]
 ; X64-AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0xc1]
 ; X64-AVX512-NEXT:    vmovupd %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x11,0x07]
diff --git a/llvm/test/CodeGen/X86/sse2-intrinsics-x86.ll b/llvm/test/CodeGen/X86/sse2-intrinsics-x86.ll
index 777beea886f56..4d5725e2504fd 100644
--- a/llvm/test/CodeGen/X86/sse2-intrinsics-x86.ll
+++ b/llvm/test/CodeGen/X86/sse2-intrinsics-x86.ll
@@ -789,21 +789,21 @@ define <8 x i16> @test_x86_sse2_packssdw_128_fold() {
 ; X64-SSE:       ## %bb.0:
 ; X64-SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,0,0,0,32767,32767,65535,32768]
 ; X64-SSE-NEXT:    ## encoding: [0x0f,0x28,0x05,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: test_x86_sse2_packssdw_128_fold:
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,0,0,32767,32767,65535,32768]
 ; X64-AVX1-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_x86_sse2_packssdw_128_fold:
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,0,0,32767,32767,65535,32768]
 ; X64-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> zeroinitializer, <4 x i32> <i32 65535, i32 65536, i32 -1, i32 -131072>)
   ret <8 x i16> %res
@@ -857,21 +857,21 @@ define <16 x i8> @test_x86_sse2_packsswb_128_fold() {
 ; X64-SSE:       ## %bb.0:
 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-SSE-NEXT:    ## encoding: [0xf2,0x0f,0x10,0x05,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: test_x86_sse2_packsswb_128_fold:
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-AVX1-NEXT:    ## encoding: [0xc5,0xfb,0x10,0x05,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_x86_sse2_packsswb_128_fold:
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vmovsd {{.*#+}} xmm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfb,0x10,0x05,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <8 x i16> zeroinitializer)
   ret <16 x i8> %res
@@ -925,21 +925,21 @@ define <16 x i8> @test_x86_sse2_packuswb_128_fold() {
 ; X64-SSE:       ## %bb.0:
 ; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-SSE-NEXT:    ## encoding: [0xf3,0x0f,0x10,0x05,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: test_x86_sse2_packuswb_128_fold:
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vmovss {{.*#+}} xmm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-AVX1-NEXT:    ## encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_x86_sse2_packuswb_128_fold:
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vmovss {{.*#+}} xmm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x05,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <8 x i16> zeroinitializer)
   ret <16 x i8> %res
diff --git a/llvm/test/CodeGen/X86/sse41-intrinsics-x86.ll b/llvm/test/CodeGen/X86/sse41-intrinsics-x86.ll
index 7c22330d7804b..86f4fbb09e9d9 100644
--- a/llvm/test/CodeGen/X86/sse41-intrinsics-x86.ll
+++ b/llvm/test/CodeGen/X86/sse41-intrinsics-x86.ll
@@ -210,21 +210,21 @@ define <8 x i16> @test_x86_sse41_packusdw_fold() {
 ; X64-SSE:       ## %bb.0:
 ; X64-SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,0,0,0,65535,65535,0,0]
 ; X64-SSE-NEXT:    ## encoding: [0x0f,0x28,0x05,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    ## fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: test_x86_sse41_packusdw_fold:
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,0,0,65535,65535,0,0]
 ; X64-AVX1-NEXT:    ## encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_x86_sse41_packusdw_fold:
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,0,4294967295,0]
 ; X64-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x21,0x05,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512-NEXT:    ## fixup A - offset: 5, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> zeroinitializer, <4 x i32> <i32 65535, i32 65536, i32 -1, i32 -131072>)
   ret <8 x i16> %res
diff --git a/llvm/test/CodeGen/X86/sse41.ll b/llvm/test/CodeGen/X86/sse41.ll
index 4f5b7ee0eaea0..3960583c46fef 100644
--- a/llvm/test/CodeGen/X86/sse41.ll
+++ b/llvm/test/CodeGen/X86/sse41.ll
@@ -104,7 +104,7 @@ define <2 x i64> @pmovzxbq_1() nounwind {
 ; X64-SSE-LABEL: pmovzxbq_1:
 ; X64-SSE:       ## %bb.0: ## %entry
 ; X64-SSE-NEXT:    movq _g16@GOTPCREL(%rip), %rax ## encoding: [0x48,0x8b,0x05,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 3, value: _g16@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
+; X64-SSE-NEXT:    ## fixup A - offset: 3, value: _g16@GOTPCREL, kind: reloc_riprel_4byte_movq_load
 ; X64-SSE-NEXT:    pmovzxbq (%rax), %xmm0 ## xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; X64-SSE-NEXT:    ## encoding: [0x66,0x0f,0x38,0x32,0x00]
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
@@ -112,7 +112,7 @@ define <2 x i64> @pmovzxbq_1() nounwind {
 ; X64-AVX1-LABEL: pmovzxbq_1:
 ; X64-AVX1:       ## %bb.0: ## %entry
 ; X64-AVX1-NEXT:    movq _g16@GOTPCREL(%rip), %rax ## encoding: [0x48,0x8b,0x05,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 3, value: _g16@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
+; X64-AVX1-NEXT:    ## fixup A - offset: 3, value: _g16@GOTPCREL, kind: reloc_riprel_4byte_movq_load
 ; X64-AVX1-NEXT:    vpmovzxbq (%rax), %xmm0 ## xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; X64-AVX1-NEXT:    ## encoding: [0xc4,0xe2,0x79,0x32,0x00]
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
@@ -120,7 +120,7 @@ define <2 x i64> @pmovzxbq_1() nounwind {
 ; X64-AVX512-LABEL: pmovzxbq_1:
 ; X64-AVX512:       ## %bb.0: ## %entry
 ; X64-AVX512-NEXT:    movq _g16@GOTPCREL(%rip), %rax ## encoding: [0x48,0x8b,0x05,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 3, value: _g16@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
+; X64-AVX512-NEXT:    ## fixup A - offset: 3, value: _g16@GOTPCREL, kind: reloc_riprel_4byte_movq_load
 ; X64-AVX512-NEXT:    vpmovzxbq (%rax), %xmm0 ## xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; X64-AVX512-NEXT:    ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x32,0x00]
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -220,7 +220,7 @@ define float @ext_1(<4 x float> %v) nounwind {
 ; X64-SSE-NEXT:    shufps $255, %xmm0, %xmm0 ## encoding: [0x0f,0xc6,0xc0,0xff]
 ; X64-SSE-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-SSE-NEXT:    addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ## encoding: [0xf3,0x0f,0x58,0x05,A,A,A,A]
-; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX1-LABEL: ext_1:
@@ -228,7 +228,7 @@ define float @ext_1(<4 x float> %v) nounwind {
 ; X64-AVX1-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0xc6,0xc0,0xff]
 ; X64-AVX1-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-AVX1-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xfa,0x58,0x05,A,A,A,A]
-; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX1-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX1-NEXT:    retq ## encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: ext_1:
@@ -236,7 +236,7 @@ define float @ext_1(<4 x float> %v) nounwind {
 ; X64-AVX512-NEXT:    vshufps $255, %xmm0, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0xc6,0xc0,0xff]
 ; X64-AVX512-NEXT:    ## xmm0 = xmm0[3,3,3,3]
 ; X64-AVX512-NEXT:    vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x58,0x05,A,A,A,A]
-; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512-NEXT:    ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512-NEXT:    retq ## encoding: [0xc3]
   %s = extractelement <4 x float> %v, i32 3
   %t = fadd float %s, 1.0
diff --git a/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll b/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll
index 391963de79703..8d589c519eff2 100644
--- a/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll
+++ b/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll
@@ -1,5 +1,6 @@
-; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction < %s | FileCheck --check-prefixes CHECK,CHECK-LV %s
-; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction -early-live-intervals < %s | FileCheck --check-prefixes CHECK,CHECK-LIS %s
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction < %s | FileCheck --check-prefix=CHECK-LV %s
+; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction -early-live-intervals < %s | FileCheck --check-prefix=CHECK-LIS %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2"
 target triple = "x86_64-unknown-linux-gnu"
@@ -10,26 +11,54 @@ declare void @bar(ptr addrspace(1)) gc "statepoint-example"
 declare ptr @fake_personality_function()
 
 ; Simplest possible test demonstrating the problem
-
-; CHECK-LABEL: name: test
-; CHECK:  bb.0
-; CHECK-LV:     %0:gr64 = COPY killed $rdi
-; CHECK-LIS:    %0:gr64 = COPY $rdi
-; CHECK:        %1:gr64 = COPY %0
-; CHECK:        %1:gr64 = STATEPOINT 2, 5, 0, undef %2:gr64, 2, 0, 2, 0, 2, 0, 2, 1, %1(tied-def 0), 2, 0, 2, 1, 0, 0
-; CHECK-LV:     TEST64rr killed %0, %0, implicit-def $eflags
-; CHECK-LIS:    TEST64rr %0, %0, implicit-def $eflags
-; CHECK:        JCC_1 %bb.2, 4, implicit killed $eflags
-; CHECK:        JMP_1 %bb.1
-; CHECK:      bb.1
-; CHECK-LV:     $rdi = COPY killed %1
-; CHECK-LV:     STATEPOINT 2, 5, 1, undef %3:gr64, killed $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0
-; CHECK-LIS:    $rdi = COPY %1
-; CHECK-LIS:    STATEPOINT 2, 5, 1, undef %3:gr64, $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0
-; CHECK:        RET 0
-; CHECK:      bb.2
-; CHECK:        RET 0
 define void @test(ptr addrspace(1) %a)  gc "statepoint-example" {
+  ; CHECK-LV-LABEL: name: test
+  ; CHECK-LV: bb.0.entry:
+  ; CHECK-LV-NEXT:   successors: %bb.2(0x30000000), %bb.1(0x50000000)
+  ; CHECK-LV-NEXT:   liveins: $rdi
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY killed $rdi
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-LV-NEXT:   [[COPY1:%[0-9]+]]:gr64 = STATEPOINT 2, 5, 0, undef %2:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY1]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   TEST64rr killed [[COPY]], [[COPY]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   JCC_1 %bb.2, 4, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.1
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.1.not_zero:
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   $rdi = COPY killed [[COPY1]]
+  ; CHECK-LV-NEXT:   STATEPOINT 2, 5, 1, undef %3:gr64, killed $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   RET 0
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.2.zero:
+  ; CHECK-LV-NEXT:   RET 0
+  ;
+  ; CHECK-LIS-LABEL: name: test
+  ; CHECK-LIS: bb.0.entry:
+  ; CHECK-LIS-NEXT:   successors: %bb.2(0x30000000), %bb.1(0x50000000)
+  ; CHECK-LIS-NEXT:   liveins: $rdi
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-LIS-NEXT:   [[COPY1:%[0-9]+]]:gr64 = STATEPOINT 2, 5, 0, undef %2:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY1]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   TEST64rr [[COPY]], [[COPY]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.2, 4, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.1
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.1.not_zero:
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   $rdi = COPY [[COPY1]]
+  ; CHECK-LIS-NEXT:   STATEPOINT 2, 5, 1, undef %3:gr64, $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   RET 0
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.2.zero:
+  ; CHECK-LIS-NEXT:   RET 0
 entry:
   %not7 = icmp eq ptr addrspace(1) %a, null
   %statepoint_token1745 = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2, i32 5, ptr nonnull elementtype(void ()) @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(ptr addrspace(1) %a) ]
@@ -45,28 +74,138 @@ zero:
 }
 
 ; A bit more complex test, where both registers are used in same successor BB
-
-; CHECK-LABEL: name: test2
-; CHECK:  bb.2
-; CHECK:        %1:gr64 = STATEPOINT 2882400000, 0, 0, undef %11:gr64, 2, 0, 2, 0, 2, 0, 2, 1, %1(tied-def 0), 2, 0, 2, 1, 0, 0, csr_64
-; CHECK:        %10:gr64 = COPY %1
-; CHECK:        %10:gr64 = STATEPOINT 2882400000, 0, 0, undef %13:gr64, 2, 0, 2, 0, 2, 0, 2, 1, %10(tied-def 0), 2, 0, 2, 1, 0, 0, csr_64
-; CHECK:        JMP_1 %bb.3
-; CHECK:      bb.3
-; CHECK:        %18:gr8 = COPY %17.sub_8bit
-; CHECK-LV:     TEST8rr killed %18, %18, implicit-def $eflags
-; CHECK-LIS:    TEST8rr %18, %18, implicit-def $eflags
-; CHECK:        JCC_1 %bb.5, 5, implicit killed $eflags
-; CHECK:        JMP_1 %bb.4
-; CHECK:      bb.4
-; CHECK:      bb.5
-; CHECK:        %3:gr64 = COPY %10
-; CHECK:        %4:gr64 = COPY killed %10
-; CHECK:        %4:gr64 = nuw ADD64ri32 %4, 8, implicit-def dead $eflags
-; CHECK:        TEST64rr killed %1, %1, implicit-def $eflags
-; CHECK:        JCC_1 %bb.1, 5, implicit killed $eflags
-; CHECK:        JMP_1 %bb.6
 define void @test2(ptr addrspace(1) %this, i32 %0, ptr addrspace(1) %p0, ptr addrspace(1) %p1) gc "statepoint-example" personality ptr @fake_personality_function {
+  ; CHECK-LV-LABEL: name: test2
+  ; CHECK-LV: bb.0.preheader:
+  ; CHECK-LV-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-LV-NEXT:   liveins: $rdx, $rcx
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY killed $rcx
+  ; CHECK-LV-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY killed $rdx
+  ; CHECK-LV-NEXT:   [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags
+  ; CHECK-LV-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY killed [[COPY1]]
+  ; CHECK-LV-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY killed [[COPY]]
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.1.loop.head:
+  ; CHECK-LV-NEXT:   successors: %bb.6(0x04000000), %bb.2(0x7c000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY4:%[0-9]+]]:gr64 = COPY killed [[COPY3]]
+  ; CHECK-LV-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY killed [[COPY2]]
+  ; CHECK-LV-NEXT:   TEST64rr killed [[COPY5]], [[COPY5]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   JCC_1 %bb.6, 5, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.2
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.2.BB3:
+  ; CHECK-LV-NEXT:   successors: %bb.3(0x7ffff800), %bb.7(0x00000800)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   [[COPY4:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %11:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY4]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY [[COPY4]]
+  ; CHECK-LV-NEXT:   [[COPY6:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %13:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY6]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LV-NEXT:   JMP_1 %bb.3
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.3.BB2:
+  ; CHECK-LV-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY7:%[0-9]+]]:gr8 = COPY [[MOV32r0_]].sub_8bit
+  ; CHECK-LV-NEXT:   TEST8rr killed [[COPY7]], [[COPY7]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   JCC_1 %bb.5, 5, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.4
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.4.BB4:
+  ; CHECK-LV-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.5.tail:
+  ; CHECK-LV-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY8:%[0-9]+]]:gr64 = COPY [[COPY6]]
+  ; CHECK-LV-NEXT:   [[COPY9:%[0-9]+]]:gr64 = COPY killed [[COPY6]]
+  ; CHECK-LV-NEXT:   [[COPY9:%[0-9]+]]:gr64 = nuw ADD64ri32 [[COPY9]], 8, implicit-def dead $eflags
+  ; CHECK-LV-NEXT:   TEST64rr killed [[COPY4]], [[COPY4]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY killed [[COPY9]]
+  ; CHECK-LV-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY killed [[COPY8]]
+  ; CHECK-LV-NEXT:   JCC_1 %bb.1, 5, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.6
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.6.BB1:
+  ; CHECK-LV-NEXT:   RET 0
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.7.BB6 (landing-pad):
+  ; CHECK-LV-NEXT:   liveins: $rax, $rdx
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LV-NEXT:   RET 0
+  ;
+  ; CHECK-LIS-LABEL: name: test2
+  ; CHECK-LIS: bb.0.preheader:
+  ; CHECK-LIS-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-LIS-NEXT:   liveins: $rdx, $rcx
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rcx
+  ; CHECK-LIS-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY $rdx
+  ; CHECK-LIS-NEXT:   [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags
+  ; CHECK-LIS-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY [[COPY1]]
+  ; CHECK-LIS-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.1.loop.head:
+  ; CHECK-LIS-NEXT:   successors: %bb.6(0x04000000), %bb.2(0x7c000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY4:%[0-9]+]]:gr64 = COPY [[COPY3]]
+  ; CHECK-LIS-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY2]]
+  ; CHECK-LIS-NEXT:   TEST64rr [[COPY5]], [[COPY5]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.6, 5, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.2
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.2.BB3:
+  ; CHECK-LIS-NEXT:   successors: %bb.3(0x7ffff800), %bb.7(0x00000800)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   [[COPY4:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %11:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY4]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY [[COPY4]]
+  ; CHECK-LIS-NEXT:   [[COPY6:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %13:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY6]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.3
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.3.BB2:
+  ; CHECK-LIS-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY7:%[0-9]+]]:gr8 = COPY [[MOV32r0_]].sub_8bit
+  ; CHECK-LIS-NEXT:   TEST8rr [[COPY7]], [[COPY7]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.5, 5, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.4
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.4.BB4:
+  ; CHECK-LIS-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.5.tail:
+  ; CHECK-LIS-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY8:%[0-9]+]]:gr64 = COPY [[COPY6]]
+  ; CHECK-LIS-NEXT:   [[COPY9:%[0-9]+]]:gr64 = COPY killed [[COPY6]]
+  ; CHECK-LIS-NEXT:   [[COPY9:%[0-9]+]]:gr64 = nuw ADD64ri32 [[COPY9]], 8, implicit-def dead $eflags
+  ; CHECK-LIS-NEXT:   TEST64rr killed [[COPY4]], [[COPY4]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY [[COPY9]]
+  ; CHECK-LIS-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY8]]
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.1, 5, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.6
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.6.BB1:
+  ; CHECK-LIS-NEXT:   RET 0
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.7.BB6 (landing-pad):
+  ; CHECK-LIS-NEXT:   liveins: $rax, $rdx
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LIS-NEXT:   RET 0
 preheader:
   br label %loop.head
 
diff --git a/llvm/test/CodeGen/X86/unpredictable-brcond.ll b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
index 12411f1c49f2d..bb44171d20d65 100644
--- a/llvm/test/CodeGen/X86/unpredictable-brcond.ll
+++ b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
@@ -87,7 +87,7 @@ define void @isint_branch(double %d) nounwind {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY [[COPY]]
   ; CHECK-NEXT:   [[CVTTPD2DQrr:%[0-9]+]]:vr128 = nofpexcept CVTTPD2DQrr killed [[COPY1]], implicit $mxcsr
   ; CHECK-NEXT:   [[CVTDQ2PDrr:%[0-9]+]]:vr128 = CVTDQ2PDrr killed [[CVTTPD2DQrr]]
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:fr64 = COPY [[CVTDQ2PDrr]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:fr64 = COPY killed [[CVTDQ2PDrr]]
   ; CHECK-NEXT:   nofpexcept UCOMISDrr [[COPY]], killed [[COPY2]], implicit-def $eflags, implicit $mxcsr
   ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 5, implicit $eflags
   ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 10, implicit $eflags
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
index 36094fe56d577..84856aab85079 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
@@ -77,11 +77,9 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: t1_all_odd_ne:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-SSE41-LABEL: t1_all_odd_ne:
@@ -92,7 +90,7 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm0
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
 ; CHECK-AVX1-LABEL: t1_all_odd_ne:
@@ -102,7 +100,7 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    retq
 ;
 ; CHECK-AVX2-LABEL: t1_all_odd_ne:
@@ -113,17 +111,16 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    retq
 ;
 ; CHECK-AVX512VL-LABEL: t1_all_odd_ne:
 ; CHECK-AVX512VL:       # %bb.0:
 ; CHECK-AVX512VL-NEXT:    vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = ~xmm0
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = m64bcst | (xmm0 ^ xmm1)
 ; CHECK-AVX512VL-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
   %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
index bde14e75dfc04..6aad9a6d82c73 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
@@ -82,13 +82,11 @@ define <2 x i32> @strict_vector_fptosi_v2f16_to_v2i32(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i32:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -105,13 +103,11 @@ define <2 x i32> @strict_vector_fptoui_v2f16_to_v2i32(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i32:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; NOVL-NEXT:    vcvttph2udq %ymm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2usi %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -128,13 +124,12 @@ define <2 x i16> @strict_vector_fptosi_v2f16_to_v2i16(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -151,13 +146,12 @@ define <2 x i16> @strict_vector_fptoui_v2f16_to_v2i16(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -175,13 +169,13 @@ define <2 x i8> @strict_vector_fptosi_v2f16_to_v2i8(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -199,13 +193,13 @@ define <2 x i8> @strict_vector_fptoui_v2f16_to_v2i8(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -335,12 +329,18 @@ define <4 x i16> @strict_vector_fptosi_v4f16_to_v4i16(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i16> @llvm.experimental.constrained.fptosi.v4i16.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -356,12 +356,18 @@ define <4 x i16> @strict_vector_fptoui_v4f16_to_v4i16(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i16> @llvm.experimental.constrained.fptoui.v4i16.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -378,12 +384,19 @@ define <4 x i8> @strict_vector_fptosi_v4f16_to_v4i8(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -400,12 +413,19 @@ define <4 x i8> @strict_vector_fptoui_v4f16_to_v4i8(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -554,7 +574,7 @@ define <8 x i8> @strict_vector_fptosi_v8f16_to_v8i8(<8 x half> %a) #0 {
 ; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
 ; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    vzeroupper
 ; NOVL-NEXT:    retq
   %ret = call <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f16(<8 x half> %a,
@@ -573,8 +593,8 @@ define <8 x i8> @strict_vector_fptoui_v8f16_to_v8i8(<8 x half> %a) #0 {
 ; NOVL:       # %bb.0:
 ; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    vzeroupper
 ; NOVL-NEXT:    retq
   %ret = call <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f16(<8 x half> %a,
diff --git a/llvm/test/CodeGen/X86/vec_fpext.ll b/llvm/test/CodeGen/X86/vec_fpext.ll
index ddec397325d7f..29c77bf6ae312 100644
--- a/llvm/test/CodeGen/X86/vec_fpext.ll
+++ b/llvm/test/CodeGen/X86/vec_fpext.ll
@@ -276,21 +276,21 @@ define <2 x double> @fpext_fromconst() {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X64-SSE-NEXT:    # encoding: [0x0f,0x28,0x05,A,A,A,A]
-; X64-SSE-NEXT:    # fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    # fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: fpext_fromconst:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: fpext_fromconst:
 ; X64-AVX512VL:       # %bb.0: # %entry
 ; X64-AVX512VL-NEXT:    vmovaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X64-AVX512VL-NEXT:    # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
 entry:
   %0  = insertelement <2 x float> undef, float 1.0, i32 0
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index e88387a8b7c69..894186f9b343b 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -138,22 +138,21 @@ define <2 x i64> @test_compress_v2i64(<2 x i64> %vec, <2 x i1> %mask, <2 x i64>
 ; AVX2-NEXT:    vmovaps %xmm2, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX2-NEXT:    vmovq %xmm1, %rcx
+; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    movl %ecx, %edx
-; AVX2-NEXT:    subl %eax, %edx
-; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    subq %rax, %rcx
+; AVX2-NEXT:    movl %ecx, %eax
 ; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rcx, %rax
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT:    cmpq $2, %rax
-; AVX2-NEXT:    cmovbq -24(%rsp,%rdx,8), %rsi
+; AVX2-NEXT:    cmpq $2, %rcx
+; AVX2-NEXT:    cmovbq -24(%rsp,%rax,8), %rsi
 ; AVX2-NEXT:    vmovq %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movl %ecx, %ecx
-; AVX2-NEXT:    vpextrq $1, %xmm0, -24(%rsp,%rcx,8)
-; AVX2-NEXT:    cmpq $1, %rax
-; AVX2-NEXT:    movl $1, %ecx
-; AVX2-NEXT:    cmovbq %rax, %rcx
-; AVX2-NEXT:    movq %rsi, -24(%rsp,%rcx,8)
+; AVX2-NEXT:    vpextrq $1, %xmm0, -24(%rsp,%rdx,8)
+; AVX2-NEXT:    cmpq $1, %rcx
+; AVX2-NEXT:    movl $1, %eax
+; AVX2-NEXT:    cmovbq %rcx, %rax
+; AVX2-NEXT:    movl %eax, %eax
+; AVX2-NEXT:    movq %rsi, -24(%rsp,%rax,8)
 ; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    retq
 ;
@@ -188,18 +187,16 @@ define <2 x double> @test_compress_v2f64(<2 x double> %vec, <2 x i1> %mask, <2 x
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vmovq %xmm1, %rcx
-; AVX2-NEXT:    movl %ecx, %edx
-; AVX2-NEXT:    subl %eax, %edx
-; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    movl %eax, %edx
+; AVX2-NEXT:    subq %rcx, %rax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vmovlpd %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    movl %ecx, %edx
 ; AVX2-NEXT:    vmovhpd %xmm0, -24(%rsp,%rdx,8)
-; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    addq %rcx, %rax
 ; AVX2-NEXT:    cmpq $2, %rax
 ; AVX2-NEXT:    jb .LBB3_2
 ; AVX2-NEXT:  # %bb.1:
@@ -208,7 +205,8 @@ define <2 x double> @test_compress_v2f64(<2 x double> %vec, <2 x i1> %mask, <2 x
 ; AVX2-NEXT:    cmpq $1, %rax
 ; AVX2-NEXT:    movl $1, %ecx
 ; AVX2-NEXT:    cmovbq %rax, %rcx
-; AVX2-NEXT:    vmovsd %xmm1, -24(%rsp,%rcx,8)
+; AVX2-NEXT:    movl %ecx, %eax
+; AVX2-NEXT:    vmovsd %xmm1, -24(%rsp,%rax,8)
 ; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
index 9c80720ae921a..d77934adf4cd1 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
@@ -18,10 +18,10 @@ define <3 x float> @constrained_vector_fadd_v3f32() #0 {
 ; CHECK: [[ADDSSrr:%[0-9]+]]:fr32 = ADDSSrr [[MOVSSrm_alt]], killed [[FsFLD0SS]], implicit $mxcsr
 ; CHECK: [[ADDSSrm:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.1, $noreg, implicit $mxcsr :: (load (s32) from constant-pool)
 ; CHECK: [[ADDSSrm1:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.2, $noreg, implicit $mxcsr :: (load (s32) from constant-pool)
-; CHECK: [[COPY:%[0-9]+]]:vr128 = COPY [[ADDSSrm1]]
-; CHECK: [[COPY1:%[0-9]+]]:vr128 = COPY [[ADDSSrm]]
+; CHECK: [[COPY:%[0-9]+]]:vr128 = COPY killed [[ADDSSrm1]]
+; CHECK: [[COPY1:%[0-9]+]]:vr128 = COPY killed [[ADDSSrm]]
 ; CHECK: [[UNPCKLPSrr:%[0-9]+]]:vr128 = UNPCKLPSrr [[COPY1]], killed [[COPY]]
-; CHECK: [[COPY2:%[0-9]+]]:vr128 = COPY [[ADDSSrr]]
+; CHECK: [[COPY2:%[0-9]+]]:vr128 = COPY killed [[ADDSSrr]]
 ; CHECK: [[UNPCKLPDrr:%[0-9]+]]:vr128 = UNPCKLPDrr [[UNPCKLPSrr]], killed [[COPY2]]
 ; CHECK: $xmm0 = COPY [[UNPCKLPDrr]]
 ; CHECK: RET 0, $xmm0
diff --git a/llvm/test/CodeGen/X86/win-import-call-optimization-jumptable.ll b/llvm/test/CodeGen/X86/win-import-call-optimization-jumptable.ll
index fe22b251685e6..d7f3bc20f2881 100644
--- a/llvm/test/CodeGen/X86/win-import-call-optimization-jumptable.ll
+++ b/llvm/test/CodeGen/X86/win-import-call-optimization-jumptable.ll
@@ -2,7 +2,7 @@
 
 ; CHECK-LABEL:  uses_rax:
 ; CHECK:        .Limpcall0:
-; CHECK-NEXT:     jmpq    *%rax
+; CHECK-NEXT:     jmpq    *%rcx
 
 define void @uses_rax(i32 %x) {
 entry:
@@ -74,7 +74,7 @@ declare void @g(i32)
 ; CHECK-NEXT:   .asciz  "RetpolineV1"
 ; CHECK-NEXT:   .long   24
 ; CHECK-NEXT:   .secnum .text
-; CHECK-NEXT:   .long   16
+; CHECK-NEXT:   .long   17
 ; CHECK-NEXT:   .secoffset      .Limpcall0
 ; CHECK-NEXT:   .long   17
 ; CHECK-NEXT:   .secoffset      .Limpcall1
diff --git a/llvm/test/CodeGen/X86/win64-jumptable.ll b/llvm/test/CodeGen/X86/win64-jumptable.ll
index 17ef0d333a727..507b924d20a07 100644
--- a/llvm/test/CodeGen/X86/win64-jumptable.ll
+++ b/llvm/test/CodeGen/X86/win64-jumptable.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -relocation-model=static | FileCheck %s
-; RUN: llc < %s -relocation-model=pic | FileCheck %s --check-prefix=PIC
-; RUN: llc < %s -relocation-model=pic -code-model=large | FileCheck %s --check-prefix=PIC
+; RUN: llc < %s -relocation-model=pic | FileCheck %s --check-prefixes=CHECK,PIC
+; RUN: llc < %s -relocation-model=pic -code-model=large | FileCheck %s --check-prefixes=CHECK,LARGE
 
 ; FIXME: Remove '-relocation-model=static' when it is no longer necessary to
 ; trigger the separate .rdata section.
@@ -43,25 +43,39 @@ declare void @g(i32)
 ; CHECK: .text
 ; CHECK: f:
 ; CHECK: .seh_proc f
-; CHECK: jmpq    *.LJTI0_0
+; CHECK: .seh_endprologue
+
+; STATIC: movl .LJTI0_0(,%rax,4), %eax
+; STATIC: leaq __ImageBase(%rax), %rax
+; STATIC: jmpq *%rax
+
+; PIC: movl %ecx, %eax
+; PIC: leaq .LJTI0_0(%rip), %rcx
+; PIC: movl (%rcx,%rax,4), %eax
+; PIC: leaq __ImageBase(%rip), %rcx
+; PIC: addq %rax, %rcx
+; PIC: jmpq *%rcx
+
+; LARGE: movl %ecx, %eax
+; LARGE-NEXT: movabsq $.LJTI0_0, %rcx
+; LARGE-NEXT: movl (%rcx,%rax,4), %eax
+; LARGE-NEXT: movabsq $__ImageBase, %rcx
+; LARGE-NEXT: addq %rax, %rcx
+; LARGE-NEXT: jmpq *%rcx
+
 ; CHECK: .LBB0_{{.*}}: # %sw.bb
 ; CHECK: .LBB0_{{.*}}: # %sw.bb2
 ; CHECK: .LBB0_{{.*}}: # %sw.bb3
 ; CHECK: .LBB0_{{.*}}: # %sw.bb1
-; CHECK: callq g
-; CHECK: jmp g # TAILCALL
+; STATIC: callq g
+; STATIC: jmp g # TAILCALL
 ; CHECK: .section        .rdata,"dr"
-; CHECK: .quad .LBB0_
-; CHECK: .quad .LBB0_
-; CHECK: .quad .LBB0_
-; CHECK: .quad .LBB0_
+; CHECK: .LJTI0_0:
+; CHECK: .long .LBB0_{{[0-9]+}}@IMGREL
+; CHECK: .long .LBB0_{{[0-9]+}}@IMGREL
+; CHECK: .long .LBB0_{{[0-9]+}}@IMGREL
+; CHECK: .long .LBB0_{{[0-9]+}}@IMGREL
 
 ; It's important that we switch back to .text here, not .rdata.
 ; CHECK: .text
 ; CHECK: .seh_endproc
-
-; Windows PIC code should use 32-bit entries
-; PIC: .long .LBB0_2-.LJTI0_0
-; PIC: .long .LBB0_3-.LJTI0_0
-; PIC: .long .LBB0_4-.LJTI0_0
-; PIC: .long .LBB0_5-.LJTI0_0
diff --git a/llvm/test/CodeGen/X86/x86-access-to-global.ll b/llvm/test/CodeGen/X86/x86-access-to-global.ll
new file mode 100644
index 0000000000000..9e09a035ac519
--- /dev/null
+++ b/llvm/test/CodeGen/X86/x86-access-to-global.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -relocation-model=pic < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+@.str = external dso_local global i32
+
+define i1 @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    calll .L0$pb
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-NEXT:  .L0$pb:
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -4
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
+; CHECK-NEXT:    movl $.str@GOTOFF, %ecx
+; CHECK-NEXT:    addb %al, %cl
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retl
+  %i = ptrtoint ptr @.str to i8
+  %p = zext i8 %i to i32
+  %c = icmp eq i32 %p, 0
+  ret i1 %c
+}
diff --git a/llvm/test/CodeGen/X86/x86-interrupt_cc.ll b/llvm/test/CodeGen/X86/x86-interrupt_cc.ll
index cf8b7096816af..ff40a3404d5ef 100644
--- a/llvm/test/CodeGen/X86/x86-interrupt_cc.ll
+++ b/llvm/test/CodeGen/X86/x86-interrupt_cc.ll
@@ -161,7 +161,7 @@ define x86_intrcc void @foo(ptr byval(i8) %frame) {
 ; CHECK64-KNL-NEXT:    .cfi_offset %k7, -82
 ; CHECK64-KNL-NEXT:    cld ## encoding: [0xfc]
 ; CHECK64-KNL-NEXT:    callq _bar ## encoding: [0xe8,A,A,A,A]
-; CHECK64-KNL-NEXT:    ## fixup A - offset: 1, value: _bar-4, kind: reloc_branch_4byte_pcrel
+; CHECK64-KNL-NEXT:    ## fixup A - offset: 1, value: _bar, kind: reloc_branch_4byte_pcrel
 ; CHECK64-KNL-NEXT:    vmovups (%rsp), %zmm0 ## 64-byte Reload
 ; CHECK64-KNL-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0x04,0x24]
 ; CHECK64-KNL-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 ## 64-byte Reload
@@ -410,7 +410,7 @@ define x86_intrcc void @foo(ptr byval(i8) %frame) {
 ; CHECK64-SKX-NEXT:    cld ## encoding: [0xfc]
 ; CHECK64-SKX-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; CHECK64-SKX-NEXT:    callq _bar ## encoding: [0xe8,A,A,A,A]
-; CHECK64-SKX-NEXT:    ## fixup A - offset: 1, value: _bar-4, kind: reloc_branch_4byte_pcrel
+; CHECK64-SKX-NEXT:    ## fixup A - offset: 1, value: _bar, kind: reloc_branch_4byte_pcrel
 ; CHECK64-SKX-NEXT:    vmovups (%rsp), %zmm0 ## 64-byte Reload
 ; CHECK64-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0x04,0x24]
 ; CHECK64-SKX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 ## 64-byte Reload
@@ -570,7 +570,7 @@ define x86_intrcc void @foo(ptr byval(i8) %frame) {
 ; CHECK32-KNL-NEXT:    .cfi_offset %k7, -22
 ; CHECK32-KNL-NEXT:    cld ## encoding: [0xfc]
 ; CHECK32-KNL-NEXT:    calll _bar ## encoding: [0xe8,A,A,A,A]
-; CHECK32-KNL-NEXT:    ## fixup A - offset: 1, value: _bar-4, kind: FK_PCRel_4
+; CHECK32-KNL-NEXT:    ## fixup A - offset: 1, value: _bar, kind: FK_PCRel_4
 ; CHECK32-KNL-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 ## 64-byte Reload
 ; CHECK32-KNL-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0x85,0xc8,0xfd,0xff,0xff]
 ; CHECK32-KNL-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm1 ## 64-byte Reload
@@ -677,7 +677,7 @@ define x86_intrcc void @foo(ptr byval(i8) %frame) {
 ; CHECK32-SKX-NEXT:    cld ## encoding: [0xfc]
 ; CHECK32-SKX-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
 ; CHECK32-SKX-NEXT:    calll _bar ## encoding: [0xe8,A,A,A,A]
-; CHECK32-SKX-NEXT:    ## fixup A - offset: 1, value: _bar-4, kind: FK_PCRel_4
+; CHECK32-SKX-NEXT:    ## fixup A - offset: 1, value: _bar, kind: FK_PCRel_4
 ; CHECK32-SKX-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 ## 64-byte Reload
 ; CHECK32-SKX-NEXT:    ## encoding: [0x62,0xf1,0x7c,0x48,0x10,0x85,0x88,0xfd,0xff,0xff]
 ; CHECK32-SKX-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm1 ## 64-byte Reload
diff --git a/llvm/test/CodeGen/Xtensa/float-arith.ll b/llvm/test/CodeGen/Xtensa/float-arith.ll
index 2b7186c44aebf..d9369a50c856f 100644
--- a/llvm/test/CodeGen/Xtensa/float-arith.ll
+++ b/llvm/test/CodeGen/Xtensa/float-arith.ll
@@ -568,7 +568,6 @@ define float @round_f32(float %a) nounwind {
   ret float %res
 }
 
-
 define float @fneg_s(float %a) nounwind {
 ; XTENSA-LABEL: fneg_s:
 ; XTENSA:       # %bb.0:
@@ -601,3 +600,23 @@ define i32 @fptoui(float %f) {
   ret i32 %conv
 }
 
+define float @copysign_f32(float %a, float %b) {
+; XTENSA-LABEL: copysign_f32:
+; XTENSA:         .cfi_startproc
+; XTENSA-NEXT:  # %bb.0: # %entry
+; XTENSA-NEXT:    l32r a8, .LCPI35_0
+; XTENSA-NEXT:    and a8, a3, a8
+; XTENSA-NEXT:    l32r a9, .LCPI35_1
+; XTENSA-NEXT:    and a9, a2, a9
+; XTENSA-NEXT:    wfr f8, a9
+; XTENSA-NEXT:    movi a9, 0
+; XTENSA-NEXT:    beq a8, a9, .LBB35_2
+; XTENSA-NEXT:  # %bb.1:
+; XTENSA-NEXT:    neg.s f8, f8
+; XTENSA-NEXT:  .LBB35_2: # %entry
+; XTENSA-NEXT:    rfr a2, f8
+; XTENSA-NEXT:    ret
+entry:
+  %c = call float @llvm.copysign.f32(float %a, float %b)
+  ret float %c
+}
diff --git a/llvm/test/CodeGen/Xtensa/invalid-tls.ll b/llvm/test/CodeGen/Xtensa/invalid-tls.ll
new file mode 100644
index 0000000000000..6aebca30b83da
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/invalid-tls.ll
@@ -0,0 +1,17 @@
+; RUN: not llc -mtriple=xtensa -mattr=+threadptr -relocation-model=pic -filetype=null < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=XTENSA-PIC %s
+; RUN: not llc -mtriple=xtensa -filetype=null < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=XTENSA-NO-THREADPTR %s
+
+; XTENSA-PIC: error: <unknown>:0:0: in function f i32 (): only local-exec and initial-exec TLS mode supported
+; XTENSA-PIC: error: <unknown>:0:0: in function f i32 (): PIC relocations are not supported
+
+; XTENSA-NO-THREADPTR: error: <unknown>:0:0: in function f i32 (): only emulated TLS supported
+
+@i = external thread_local global i32
+
+define i32 @f() {
+entry:
+  %tmp1 = load i32, ptr @i
+  ret i32 %tmp1
+}
diff --git a/llvm/test/CodeGen/Xtensa/select-cc-fp.ll b/llvm/test/CodeGen/Xtensa/select-cc-fp.ll
new file mode 100644
index 0000000000000..ee45ef006123c
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/select-cc-fp.ll
@@ -0,0 +1,457 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=xtensa -mattr=+fp -disable-block-placement -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s
+
+define float @brcc_oeq(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_oeq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    oeq.s b0, f9, f8
+; CHECK-NEXT:    bf b0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI0_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI0_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp oeq float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ogt(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ogt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ule.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI1_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI1_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ogt float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_oge(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_oge:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ult.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI2_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI2_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp oge float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_olt(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_olt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ule.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI3_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB3_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI3_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp olt float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ole(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ole:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ult.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB4_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI4_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB4_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI4_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ole float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_one(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ueq.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI5_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB5_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI5_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp one float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ord(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ord:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    un.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB6_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI6_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB6_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI6_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ord float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ueq(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ueq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ueq.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB7_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI7_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB7_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI7_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ueq float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ugt(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ugt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ole.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI8_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB8_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI8_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ugt float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_uge(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_uge:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    olt.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB9_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI9_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB9_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI9_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp uge float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ult(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ult:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ole.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB10_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI10_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB10_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI10_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ult float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ule(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ule:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    olt.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB11_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI11_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB11_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI11_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ule float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_une(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_une:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    oeq.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB12_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI12_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB12_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI12_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp une float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_uno(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_uno:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    un.s b0, f9, f8
+; CHECK-NEXT:    bf b0, .LBB13_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI13_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB13_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI13_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp uno float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @copysign_f32(float %a, float %b) {
+entry:
+  %c = call float @llvm.copysign.f32(float %a, float %b)
+  ret float %c
+}
diff --git a/llvm/test/CodeGen/Xtensa/threadptr.ll b/llvm/test/CodeGen/Xtensa/threadptr.ll
new file mode 100644
index 0000000000000..5e08addd840d4
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/threadptr.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=xtensa -mattr=+threadptr < %s | FileCheck %s
+
+@i = external thread_local global i32
+
+define i32 @f() {
+; CHECK-LABEL: f:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    l32r a8, .LCPI0_0
+; CHECK-NEXT:    rur a9, threadptr
+; CHECK-NEXT:    add a8, a9, a8
+; CHECK-NEXT:    l32i a2, a8, 0
+; CHECK-NEXT:    ret
+entry:
+  %tmp1 = load i32, ptr @i
+  ret i32 %tmp1
+}
diff --git a/llvm/test/DebugInfo/COFF/jump-table.ll b/llvm/test/DebugInfo/COFF/jump-table.ll
index 3eda2438ea88a..be1de2be55788 100644
--- a/llvm/test/DebugInfo/COFF/jump-table.ll
+++ b/llvm/test/DebugInfo/COFF/jump-table.ll
@@ -118,6 +118,15 @@
 ; CV:         GlobalProcIdSym {
 ; CV:           DisplayName: func
 ; CV-NOT:     GlobalProcIdSym
+; CV:           LabelSym {
+; CV-NEXT:        Kind: S_LABEL32 (0x1105)
+; CV-NEXT:        CodeOffset: 0xC0
+; CV-NEXT:        Segment: 0x0
+; CV-NEXT:        Flags: 0x0
+; CV-NEXT:        Flags [ (0x0)
+; CV-NEXT:        ]
+; CV-NEXT:        DisplayName:
+; CV-NEXT:      }
 ; CV:           JumpTableSym {
 ; CV-NEXT:        Kind: S_ARMSWITCHTABLE (0x1159)
 ; CV-NEXT:        BaseOffset: 0x0
diff --git a/llvm/test/DebugInfo/Generic/debug-label.ll b/llvm/test/DebugInfo/Generic/debug-label.ll
index eff482a25ee0f..0adb0608fafb6 100644
--- a/llvm/test/DebugInfo/Generic/debug-label.ll
+++ b/llvm/test/DebugInfo/Generic/debug-label.ll
@@ -5,6 +5,7 @@
 ; CHECK-NEXT: DW_AT_name {{.*}}"top"
 ; CHECK-NEXT: DW_AT_decl_file [DW_FORM_data1] {{.*}}debug-label.c
 ; CHECK-NEXT: DW_AT_decl_line [DW_FORM_data1] {{.*}}4
+; CHECK-NEXT: DW_AT_decl_column [DW_FORM_data1] {{.*}}9
 ; CHECK-NEXT: DW_AT_low_pc [DW_FORM_addr] {{.*}}{{0x[0-9a-f]+}}
 ; CHECK: DW_TAG_label
 ; CHECK-NEXT: DW_AT_name {{.*}}"done"
@@ -22,6 +23,7 @@
 ; ASM-NEXT: DW_AT_name
 ; ASM: 1 {{.*}} DW_AT_decl_file
 ; ASM-NEXT: 4 {{.*}} DW_AT_decl_line
+; ASM-NEXT: 9 {{.*}} DW_AT_decl_column
 ; ASM-NEXT: [[TOP_LOW_PC]]{{.*}} DW_AT_low_pc
 ; ASM: DW_TAG_label
 ; ASM-NEXT: DW_AT_name
@@ -68,7 +70,7 @@ declare void @llvm.dbg.label(metadata)
 !7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9, !9}
 !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!10 = !DILabel(scope: !6, name: "top", file: !1, line: 4)
+!10 = !DILabel(scope: !6, name: "top", file: !1, line: 4, column: 9)
 !11 = !DILocation(line: 4, column: 1, scope: !6)
 !12 = !DILabel(scope: !15, name: "done", file: !1, line: 7)
 !13 = !DILocation(line: 7, column: 1, scope: !6)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll
index 33f7f673d91c3..125ce16927582 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll
@@ -11,6 +11,8 @@
 
 ; CHECK: _Z1bv()
 ; CHECK: store i32 1, ptr @g, align 4, !dbg [[DBG:!.*]]
+
+; CHECK: distinct !DISubprogram(name: "b", {{.*}}keyInstructions: true)
 ; CHECK: [[DBG]] = !DILocation(line: 3, scope: ![[#]])
 
 @g = hidden global i32 0, align 4
@@ -36,7 +38,7 @@ entry:
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!15 = distinct !DISubprogram(name: "b", scope: !1, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!15 = distinct !DISubprogram(name: "b", scope: !1, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !16 = !DISubroutineType(types: !17)
 !17 = !{}
 !18 = !DILocation(line: 3, scope: !15, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll
index b9b6abdd1e9c0..0d5cc6d4a44de 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll
@@ -8,6 +8,7 @@
 ; CHECK: %add.i = add nsw i32 %mul.i, 1, !dbg [[G1R2:!.*]]
 ; CHECK-NEXT: store i32 %add.i, ptr %x.i, align 4, !dbg [[G1R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "g", {{.*}}keyInstructions: true)
 ; CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 ; CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 
@@ -41,11 +42,10 @@ entry:
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
 !19 = !DILocation(line: 2, scope: !11, atomGroup: 1, atomRank: 2)
 !20 = !DILocation(line: 2, scope: !11, atomGroup: 1, atomRank: 1)
-!23 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!23 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !24 = !DILocation(line: 5, scope: !23)
-
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll
index 612e74792fac2..175272e4e8bb6 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll
@@ -27,6 +27,7 @@
 ; CHECK: bb.f2:
 ; CHECK-NEXT: store ptr @a, ptr %p, align 4, !dbg [[G4R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "foo", {{.*}}keyInstructions: true)
 ; CHECK: [[G1R2]] = !DILocation(line: 1, column: 1, scope: ![[#]], atomGroup: 1, atomRank: 2)
 ; CHECK: [[G1R1]] = !DILocation(line: 1, column: 1, scope: ![[#]], atomGroup: 1, atomRank: 1)
 ; CHECK: [[G3R2]] = !DILocation(line: 1, column: 1, scope: ![[#]], atomGroup: 3, atomRank: 2)
@@ -76,7 +77,7 @@ declare void @f2()
 !2 = !{i32 16}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !9 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll
index 1541df3469087..f285ebbdfda31 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll
@@ -26,6 +26,7 @@
 ; CHECK-NEXT: %C = add i32 %v2, 1, !dbg [[G1R2:!.*]]
 ; CHECK-NEXT: store i32 %C, ptr %p, align 4, !dbg [[G1R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "test1", {{.*}}keyInstructions: true)
 ; CHECK: [[G2R2]] = !DILocation(line: 8, column: 1, scope: ![[#]], atomGroup: 2, atomRank: 2)
 ; CHECK: [[G2R1]] = !DILocation(line: 8, column: 1, scope: ![[#]], atomGroup: 2, atomRank: 1)
 ; CHECK: [[G1R2]] = !DILocation(line: 8, column: 1, scope: ![[#]], atomGroup: 1, atomRank: 2)
@@ -70,7 +71,7 @@ declare void @f3()
 !2 = !{i32 12}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 8, column: 1, scope: !5, atomGroup: 1, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll
index 1ab1c1ac2e9f6..a8a34d0698944 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll
@@ -30,6 +30,7 @@
 ; CHECK-NEXT: %C = add i32 %v2, 1, !dbg [[G2R2:!.*]]
 ; CHECK-NEXT: store i32 %C, ptr %p, align 4, !dbg [[G2R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "test5", {{.*}}keyInstructions: true)
 ; CHECK: [[G3R2]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
 ; CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
 ; CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
@@ -79,7 +80,7 @@ declare void @f3()
 !2 = !{i32 12}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test5", linkageName: "test5", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test5", linkageName: "test5", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll
new file mode 100644
index 0000000000000..104d75d8648b6
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll
@@ -0,0 +1,67 @@
+; RUN: split-file %s %t
+; RUN: llvm-as %t/key-instr-enabled.ll -o %t/key-instr-enabled.bc
+; RUN: llvm-as %t/key-instr-disabled.ll -o %t/key-instr-disabled.bc
+; RUN: llvm-link %t/key-instr-enabled.bc %t/key-instr-disabled.bc -o - | llvm-dis | FileCheck %s
+
+;; Check the Key Instructions metadata is preserved correctly when linking a
+;; modules with Key Instructions enabled/disabled.
+
+;; Key Instructions enabled.
+; CHECK: void @f() !dbg [[f:!.*]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret void, !dbg [[enabled:!.*]]
+; CHECK-NEXT: }
+
+;; Key Instructions disabled.
+; CHECK: void @g() !dbg [[g:!.*]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret void, !dbg [[disabled:!.*]]
+; CHECK-NEXT: }
+
+; CHECK: !llvm.dbg.cu = !{!0, !2}
+; CHECK-NEXT: !llvm.module.flags = !{!4}
+
+; CHECK: [[file1:!.*]] = !DIFile(filename: "key-instr-enabled.cpp", directory: "/")
+; CHECK: [[file2:!.*]] = !DIFile(filename: "key-instr-disabled.cpp", directory: "/")
+; CHECK: [[f]] = distinct !DISubprogram(name: "f", scope: [[file1]]{{.*}},  keyInstructions: true)
+; CHECK: [[enabled]] = !DILocation(line: 1, column: 11, scope: [[f]], atomGroup: 1, atomRank: 1)
+; CHECK: [[g]] = distinct !DISubprogram(name: "g", scope: [[file2]]
+; CHECK-NOT:                            keyInstructions
+; CHECK-SAME:                           )
+; CHECK: [[disabled]] = !DILocation(line: 1, column: 11, scope: [[g]])
+
+;--- key-instr-enabled.ll
+define dso_local void @f() !dbg !10 {
+entry:
+  ret void, !dbg !13
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "key-instr-enabled.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: true)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, column: 11, scope: !10, atomGroup: 1, atomRank: 1)
+
+;--- key-instr-disabled.ll
+define dso_local void @g() !dbg !10 {
+entry:
+  ret void, !dbg !13
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "key-instr-disabled.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, column: 11, scope: !10)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll
index 6da7146b5b07a..0ea0453c4a6c4 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll
@@ -25,6 +25,7 @@
 ; CHECK:    [[CMP:%.*]] = icmp slt i32 {{.*}}, 100, !dbg [[G2R2:![0-9]+]]
 ; CHECK:    br i1 [[CMP]], label %for.body, label %for.end, !dbg [[G2R1:![0-9]+]]
 ;
+; CHECK: distinct !DISubprogram(name: "test1", {{.*}}keyInstructions: true)
 ; CHECK: [[G3R1]] = !DILocation(line: 4{{.*}}, atomGroup: 3, atomRank: 1)
 ; CHECK: [[G4R1]] = !DILocation(line: 6{{.*}}, atomGroup: 4, atomRank: 1)
 ; CHECK: [[G1R1]] = !DILocation(line: 4{{.*}}, atomGroup: 1, atomRank: 1)
@@ -69,6 +70,6 @@ attributes #1 = { noduplicate }
 !2 = !{i32 12}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll
index 6deb04cc00f8d..d23afaed9a047 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll
@@ -25,6 +25,7 @@
 ; CHECK-NEXT: store i64 %indvars.iv.next.2, ptr %p, align 4, !dbg [[G7R1:!.*]]
 ; CHECK-NEXT: %indvars.iv.next.3 = add nuw nsw i64 %indvars.iv, 4
 
+; CHECK: distinct !DISubprogram(name: "unroll", {{.*}}keyInstructions: true)
 ; CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
 ; CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
 ; CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
@@ -63,7 +64,7 @@ for.body:                                         ; preds = %for.body, %for.body
 !2 = !{i32 17}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "unroll", linkageName: "unroll", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "unroll", linkageName: "unroll", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll
index 557e8387e5bc5..3b13bde8bfb3f 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll
@@ -41,7 +41,7 @@ attributes #0 = { nounwind ssp uwtable }
 !2 = !{i32 8}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "f", linkageName: "f", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "f", linkageName: "f", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !11 = !DILocation(line: 4, column: 1, scope: !5, atomGroup: 1, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
index 557950439ef3b..fee7e845d4df9 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
@@ -125,7 +125,7 @@ declare void @clobber()
 !2 = !{i32 13}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "partial_unswitch_true_successor_hoist_invariant", linkageName: "partial_unswitch_true_successor_hoist_invariant", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "partial_unswitch_true_successor_hoist_invariant", linkageName: "partial_unswitch_true_successor_hoist_invariant", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, scope: !5, atomGroup: 1, atomRank: 1)
@@ -142,6 +142,7 @@ declare void @clobber()
 !19 = !DILocation(line: 12, scope: !5, atomGroup: 12, atomRank: 1)
 !20 = !DILocation(line: 13, scope: !5, atomGroup: 13, atomRank: 1)
 ;.
+; CHECK: distinct !DISubprogram(name: "partial_unswitch_true_successor_hoist_invariant", {{.*}}keyInstructions: true)
 ; CHECK: [[DBG8]] = !DILocation(line: 3{{.*}}, atomGroup: 24, atomRank: 1)
 ; CHECK: [[DBG9]] = !DILocation(line: 4{{.*}}, atomGroup: 25, atomRank: 1)
 ; CHECK: [[DBG10]] = !DILocation(line: 5{{.*}}, atomGroup: 26, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll
index 33219a582fee8..79f7a7d79610c 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll
@@ -1,5 +1,6 @@
 ; RUN: opt %s -o - -S| FileCheck %s
 
+; CHECK: distinct !DISubprogram(name: "f", {{.*}}keyInstructions: true)
 ; CHECK: !DILocation(line: 1, column: 11, scope: ![[#]], atomGroup: 1, atomRank: 1)
 
 define dso_local void @f() !dbg !10 {
@@ -13,7 +14,7 @@ entry:
 !1 = !DIFile(filename: "test.cpp", directory: "/")
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 21.0.0git"}
-!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: true)
 !11 = !DISubroutineType(types: !12)
 !12 = !{null}
 !13 = !DILocation(line: 1, column: 11, scope: !10, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll
new file mode 100644
index 0000000000000..79a06e51b264f
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -o - -S  | llvm-as - | llvm-dis - | FileCheck %s
+
+; Key Instructions enabled.
+define dso_local void @f() !dbg !10 {
+; CHECK-LABEL: define dso_local void @f(
+; CHECK-SAME: ) !dbg [[DBG3:![0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    ret void, !dbg [[DBG6:![0-9]+]]
+;
+entry:
+  ret void, !dbg !13
+}
+
+; Key Instructions disabled.
+define dso_local void @g() !dbg !14 {
+; CHECK-LABEL: define dso_local void @g(
+; CHECK-SAME: ) !dbg [[DBG7:![0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    ret void, !dbg [[DBG8:![0-9]+]]
+;
+entry:
+  ret void, !dbg !15
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "key-instr.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: true)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, scope: !10, atomGroup: 1, atomRank: 1)
+!14 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 2, type: !11, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: false)
+!15 = !DILocation(line: 2, scope: !14)
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}key-instr.cpp", directory: {{.*}})
+; CHECK: [[DBG3]] = distinct !DISubprogram(name: "f", scope: [[META1]], file: [[META1]], line: 1, type: [[META4:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META0]], keyInstructions: true)
+; CHECK: [[META4]] = !DISubroutineType(types: [[META5:![0-9]+]])
+; CHECK: [[META5]] = !{null}
+; CHECK: [[DBG6]] = !DILocation(line: 1, scope: [[DBG3]], atomGroup: 1, atomRank: 1)
+; CHECK: [[DBG7]] = distinct !DISubprogram(name: "g", scope: [[META1]], file: [[META1]], line: 2, type: [[META4]], scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META0]])
+; CHECK: [[DBG8]] = !DILocation(line: 2, scope: [[DBG7]])
+;.
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
index 75a05ad931707..3ad956cabdb2d 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
@@ -70,7 +70,7 @@ f:
 !4 = !{i32 0}
 !5 = !{i32 2, !"Debug Info Version", i32 3}
 !7 = !DISubroutineType(types: !2)
-!17 = distinct !DISubprogram(name: "f", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!17 = distinct !DISubprogram(name: "f", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2, keyInstructions: true)
 !18 = !DILocation(line: 10, column: 10, scope: !17)
 !19 = !DILocation(line: 10, column: 10, scope: !17, atomGroup: 1, atomRank: 2)
 !20 = !DILocation(line: 10, column: 10, scope: !17, atomGroup: 2, atomRank: 2)
@@ -79,7 +79,7 @@ f:
 ; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]])
 ; CHECK: [[META1]] = !DIFile(filename: "a.ll", directory: {{.*}})
 ; CHECK: [[META2]] = !{}
-; CHECK: [[DBG6]] = distinct !DISubprogram(name: "f", scope: null, file: [[META1]], line: 1, type: [[META7:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
+; CHECK: [[DBG6]] = distinct !DISubprogram(name: "f", scope: null, file: [[META1]], line: 1, type: [[META7:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]], keyInstructions: true)
 ; CHECK: [[META7]] = !DISubroutineType(types: [[META2]])
 ; CHECK: [[DBG8]] = !DILocation(line: 10, column: 10, scope: [[DBG6]], atomGroup: 5, atomRank: 2)
 ; CHECK: [[DBG9]] = !DILocation(line: 10, column: 10, scope: [[DBG6]], atomGroup: 6, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll
index f8477600c6418..774c637e3b831 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll
@@ -13,6 +13,8 @@
 ; CHECK:   store i32 1{{.*}}, !dbg [[DBG1:!.*]]
 ; CHECK: if.end.1.critedge:
 ; CHECK:   store i32 1{{.*}}, !dbg [[DBG2:!.*]]
+;
+; CHECK: distinct !DISubprogram(name: "bar", {{.*}}keyInstructions: true)
 ; CHECK: [[DBG1]] = !DILocation(line: 1{{.*}}, atomGroup: 1
 ; CHECK: [[DBG2]] = !DILocation(line: 1{{.*}}, atomGroup: 2
 
@@ -56,7 +58,7 @@ declare void @foo()
 !2 = !{i32 15}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll
new file mode 100644
index 0000000000000..0f8f505c51a58
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll
@@ -0,0 +1,22 @@
+; RUN: opt %s -o - -S --passes=verify 2>&1 | FileCheck %s
+
+; CHECK: DbgLoc uses atomGroup but DISubprogram doesn't have Key Instructions enabled
+; CHECK-NEXT: ![[#]] = !DILocation(line: 1, column: 11, scope: ![[f:.*]], atomGroup: 1, atomRank: 1)
+; CHECK-NEXT: ![[f]] = distinct !DISubprogram(name: "f"
+; CHECK-NEXT: warning: ignoring invalid debug info
+
+define dso_local void @f() !dbg !10 {
+entry:
+  ret void, !dbg !13
+}
+
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, column: 11, scope: !10, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll b/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll
index a21b1da15a45d..dcacd96e69a17 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll
@@ -46,7 +46,7 @@ if.else1:                                         ; preds = %if.else1, %preheade
 !2 = !{i32 11}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test", linkageName: "test", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test", linkageName: "test", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !12 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll
index 6988a6fb2ec3f..154bb872684f0 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll
@@ -64,6 +64,6 @@ declare void @_Z12prologue_endv() local_unnamed_addr #1
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll
index ba38bb2ffe903..b97b436ffc573 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll
@@ -55,8 +55,8 @@ entry:
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
-!16 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!16 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !18 = distinct !DILocation(line: 6, scope: !16)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir
index 3b7d5850e7e83..abe27e9870cda 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir
@@ -48,7 +48,7 @@
   !2 = !{i32 7, !"Dwarf Version", i32 5}
   !3 = !{i32 2, !"Debug Info Version", i32 3}
   !4 = !{!"clang version 19.0.0"}
-  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
   !6 = !DISubroutineType(types: !7)
   !7 = !{}
   !8 = !DILocalVariable(name: "x", scope: !5, file: !1, line: 1, type: !7)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir
index 3aff5e22326ec..222604493cc8f 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir
@@ -44,7 +44,7 @@
   !2 = !{i32 7, !"Dwarf Version", i32 5}
   !3 = !{i32 2, !"Debug Info Version", i32 3}
   !4 = !{!"clang version 19.0.0"}
-  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
   !6 = !DISubroutineType(types: !7)
   !7 = !{}
   !8 = !DILocalVariable(name: "x", scope: !5, file: !1, line: 1, type: !7)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll
index 14dfa8a6a9ccb..72fc4db6402a0 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll
@@ -109,6 +109,6 @@ declare i32  @g() local_unnamed_addr
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir
new file mode 100644
index 0000000000000..273d5cbdfda24
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir
@@ -0,0 +1,93 @@
+# RUN: llc %s --start-after=livedebugvalues --dwarf-use-key-instructions --filetype=obj -o - \
+# RUN: | llvm-objdump -d - --no-show-raw-insn \
+# RUN: | FileCheck %s --check-prefix=OBJ
+
+# RUN: llc %s --start-after=livedebugvalues --dwarf-use-key-instructions --filetype=obj -o - \
+# RUN: | llvm-dwarfdump - --debug-line \
+# RUN: | FileCheck %s --check-prefix=DBG
+
+# Test is_stmt emission for a key-instructions function inlined into a
+# not-key-instructions function, and vice versa.
+
+--- |
+  target triple = "x86_64-unknown-linux-gnu"
+
+  define hidden noundef i32 @key() local_unnamed_addr !dbg !5 {
+  entry:
+    ret i32 0
+  }
+
+  define hidden noundef i32 @not_key() local_unnamed_addr !dbg !9 {
+  entry:
+    ret i32 0
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2, !3}
+  !llvm.ident = !{!4}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_17, file: !1, producer: "clang version 21.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "test.cpp", directory: "/")
+  !2 = !{i32 7, !"Dwarf Version", i32 5}
+  !3 = !{i32 2, !"Debug Info Version", i32 3}
+  !4 = !{!"clang version 21.0.0"}
+  !5 = distinct !DISubprogram(name: "key", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{}
+  !9 = distinct !DISubprogram(name: "not_key", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: false)
+  !10 = distinct !DILocation(line: 5, scope: !5)
+  !11 = distinct !DILocation(line: 9, scope: !9)
+...
+---
+name:            key
+alignment:       16
+body:             |
+  bb.0.entry:
+
+    ; OBJ: 0000000000000000 <key>:
+    ; OBJ-NEXT:  0: movl $0x1, %eax
+    ; OBJ-NEXT:  5: movl $0x2, %eax
+    ; OBJ-NEXT:  a: movl $0x3, %eax
+    ; OBJ-NEXT:  f: movl $0x4, %eax
+    ; OBJ-NEXT: 14: movl $0x5, %eax
+    ; OBJ-NEXT: 19: retq
+    ;
+    ; DBG:      Address            Line   Column File   ISA Discriminator OpIndex Flags
+    ; DBG-NEXT: ------------------ ------ ------ ------ --- ------------- ------- -------------
+    ; DBG-NEXT: 0x0000000000000000      1      0      0   0             0       0  is_stmt prologue_end
+    ; DBG-NEXT: 0x0000000000000005      2      0      0   0             0       0  is_stmt
+    ; DBG-NEXT: 0x000000000000000a      2      0      0   0             0       0
+    ; DBG-NEXT: 0x000000000000000f      3      0      0   0             0       0  is_stmt
+    ; DBG-NEXT: 0x0000000000000014      3      0      0   0             0       0
+    ;
+    $eax = MOV32ri 1,  debug-location !DILocation(line: 1, scope: !5)                            ; is_stmt (prologue_end)
+    $eax = MOV32ri 2,  debug-location !DILocation(line: 2, scope: !5, atomGroup: 1, atomRank: 1) ; is_stmt (key)
+    $eax = MOV32ri 3,  debug-location !DILocation(line: 2, scope: !9, inlinedAt: !10)
+    $eax = MOV32ri 4,  debug-location !DILocation(line: 3, scope: !9, inlinedAt: !10)            ; is_stmt (not_key)
+    $eax = MOV32ri 5,  debug-location !DILocation(line: 3, scope: !5, atomGroup: 1, atomRank: 2) ; is_stmt (key)
+    RET64 $eax,        debug-location !DILocation(line: 3, scope: !5, atomGroup: 1, atomRank: 1)
+...
+---
+name:            not_key
+alignment:       16
+body:             |
+  bb.0.entry:
+
+    ; OBJ: 0000000000000020 <not_key>:
+    ; OBJ-NEXT: 20: movl $0x1, %eax
+    ; OBJ-NEXT: 25: movl $0x2, %eax
+    ; OBJ-NEXT: 2a: movl $0x3, %eax
+    ; OBJ-NEXT: 2f: retq
+    ;
+    ;           Address            Line   Column File   ISA Discriminator OpIndex Flags
+    ;           ------------------ ------ ------ ------ --- ------------- ------- -------------
+    ; DBG-NEXT: 0x0000000000000020      1      0      0   0             0       0  is_stmt prologue_end
+    ; DBG-NEXT: 0x0000000000000025      2      0      0   0             0       0
+    ; DBG-NEXT: 0x000000000000002a      3      0      0   0             0       0  is_stmt
+    ; DBG-NEXT: 0x000000000000002f      3      0      0   0             0       0
+    ;
+    $eax = MOV32ri 1,  debug-location !DILocation(line: 1, scope: !9)                                            ; is_stmt (prologue_end)
+    $eax = MOV32ri 2,  debug-location !DILocation(line: 2, scope: !5, inlinedAt: !11, atomGroup: 1, atomRank: 2)
+    $eax = MOV32ri 3,  debug-location !DILocation(line: 3, scope: !5, inlinedAt: !11, atomGroup: 1, atomRank: 1) ; is_stmt (key)
+    RET64 $eax,        debug-location !DILocation(line: 3, scope: !9)
+...
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll
index cd840ecce07f5..e9067e4489426 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll
@@ -60,6 +60,6 @@ declare void @_Z12prologue_endv() local_unnamed_addr #1
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/debugify.ll b/llvm/test/DebugInfo/KeyInstructions/debugify.ll
index 881375873f324..551ae2794c2f3 100644
--- a/llvm/test/DebugInfo/KeyInstructions/debugify.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/debugify.ll
@@ -38,9 +38,12 @@ define i32 @boom() {
   ret i32 %retval
 }
 
+; CHECK: distinct !DISubprogram(name: "foo", {{.*}}keyInstructions: true)
 ; CHECK-DAG: ![[RET1]] = !DILocation(line: 1, {{.*}}, atomGroup: 1, atomRank: 1
+; CHECK: distinct !DISubprogram(name: "bar", {{.*}}keyInstructions: true)
 ; CHECK-DAG: ![[CALL1]] = !DILocation(line: 2, {{.*}}, atomGroup: 2, atomRank: 1
 ; CHECK-DAG: ![[ADD1]] = !DILocation(line: 3, {{.*}}, atomGroup: 3, atomRank: 1
 ; CHECK-DAG: ![[RET2]] = !DILocation(line: 4, {{.*}}, atomGroup: 4, atomRank: 1
+; CHECK: distinct !DISubprogram(name: "boom", {{.*}}keyInstructions: true)
 ; CHECK-DAG: ![[musttail]] = !DILocation(line: 5, {{.*}}, atomGroup: 5, atomRank: 1
 ; CHECK-DAG: ![[musttailRes]] = !DILocation(line: 6, {{.*}}, atomGroup: 6, atomRank: 1
diff --git a/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll b/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll
index 16c9e12accbc0..776674d53533d 100644
--- a/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll
+++ b/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll
@@ -43,15 +43,13 @@
 ;; single data fragment. (Check that we see one data fragment, then no more
 ;; fragments until the next section).
 ;
-; FRAGMENTS:       <MCSection Name:.debug_line Fragments:[
-; FRAGMENTS-NEXT:    <MCDataFragment<MCFragment
-; FRAGMENTS-NEXT:      Contents:[
-; FRAGMENTS-NEXT:      Fixups:[
+; FRAGMENTS:       Section Name:.debug_line
+; FRAGMENTS-NEXT:    Data
+; FRAGMENTS-NEXT:      Fixup
 ;
-; FRAGMENTS-NOT: MCDataFragment
-; FRAGMENTS-NOT: MCFragment
+; FRAGMENTS-NOT:     Data
 ;
-; FRAGMENTS:     <MCSection Name:.debug_line_str Fragments:[
+; FRAGMENTS:       Section Name:.debug_line_str
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/DebugInfo/X86/dynamic-bitfield.ll b/llvm/test/DebugInfo/X86/dynamic-bitfield.ll
index eda4ff0f8f6d1..c9148ca4582f6 100644
--- a/llvm/test/DebugInfo/X86/dynamic-bitfield.ll
+++ b/llvm/test/DebugInfo/X86/dynamic-bitfield.ll
@@ -1,4 +1,6 @@
-; RUN: llc -mtriple=x86_64 -O0 -filetype=obj -o - %s | llvm-dwarfdump -v -debug-info - | FileCheck %s
+; The use of llvm-dis here tests that round-tripping the IR works
+; correctly for the expression case.
+; RUN: llvm-as < %s | llvm-dis | llc -mtriple=x86_64 -O0 -filetype=obj -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s
 
 ; A basic test of using a DIExpression for DW_AT_data_bit_offset and
 ; DW_AT_bit_size.
diff --git a/llvm/test/DebugInfo/debug_frame_symbol.ll b/llvm/test/DebugInfo/debug_frame_symbol.ll
index 56ac55e46a825..0789ee371fdac 100644
--- a/llvm/test/DebugInfo/debug_frame_symbol.ll
+++ b/llvm/test/DebugInfo/debug_frame_symbol.ll
@@ -22,7 +22,7 @@ entry:
   ret void, !dbg !12
 }
 
-attributes #0 = { noinline nounwind optnone ssp "frame-pointer"="non-leaf" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { noinline nounwind optnone ssp "frame-pointer"="non-leaf" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3, !4, !5, !6}
diff --git a/llvm/test/Feature/intrinsics.ll b/llvm/test/Feature/intrinsics.ll
index 49fed0981bd76..b6abc0fff6db7 100644
--- a/llvm/test/Feature/intrinsics.ll
+++ b/llvm/test/Feature/intrinsics.ll
@@ -3,8 +3,12 @@
 ; RUN: diff %t1.ll %t2.ll
 ; RUN: FileCheck %s < %t1.ll
 
+; CHECK: Unknown intrinsic
+; CHECK-NEXT: declare i1 @llvm.isunordered.f32(float, float)
 declare i1 @llvm.isunordered.f32(float, float)
 
+; CHECK: Unknown intrinsic
+; CHECK-NEXT: declare i1 @llvm.isunordered.f64(double, double)
 declare i1 @llvm.isunordered.f64(double, double)
 
 
@@ -60,7 +64,6 @@ define void @libm() {
 
 ; FIXME: test ALL the intrinsics in this file.
 
-; rdar://11542750
 ; CHECK: declare void @llvm.trap() #1
 declare void @llvm.trap()
 
diff --git a/llvm/test/FileCheck/var-escape.txt b/llvm/test/FileCheck/var-escape.txt
index c3dc0fc7591d7..b290b6671be3d 100644
--- a/llvm/test/FileCheck/var-escape.txt
+++ b/llvm/test/FileCheck/var-escape.txt
@@ -14,9 +14,9 @@ VARS-NEXT: [[WINPATH]] [[NOT_ESCAPED]] [[ESCAPED]] [[#$NUMERIC + 0]]
 ; RUN:   -dump-input=never --strict-whitespace --check-prefix=VARS --input-file=%t.1 %s 2>&1 \
 ; RUN: | FileCheck %s
 
-CHECK: with "WINPATH" equal to "A:\\\\windows\\\\style\\\\path"
-CHECK: with "NOT_ESCAPED" equal to "shouldn't be escaped \\[a-Z\\]\\\\\\+\\$"
-CHECK: with "ESCAPED" equal to "\\\\ \014\013 needs\to \"be\" escaped\\\000"
+CHECK: with "WINPATH" equal to "A:\windows\style\path"
+CHECK: with "NOT_ESCAPED" equal to "shouldn't be escaped [a-Z]\+$"
+CHECK: with "ESCAPED" equal to "\\ \014\013 needs\to \"be\" escaped\000" (escaped value)
 CHECK: with "$NUMERIC + 0" equal to "DEADBEEF"
 
 ; Test escaping of the name of a numeric substitution, which might contain
diff --git a/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll b/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll
index 64fcfdcc6127e..0859a7ebb4b08 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll
@@ -24,5 +24,5 @@ entry:
   store i8 2, ptr %arrayidx1, align 1
   ret void
 }
-attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll b/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll
index 90e1ab35d9c36..5f122ad3bdb84 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll
@@ -51,5 +51,5 @@ entry:
 ; CHECK-NOT:  store i64 -723401728380766731, ptr %126, align 1
   ret void
 }
-attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
index c4016e749e70c..6542e56b3fefe 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
@@ -950,16 +950,21 @@ define <2 x double> @test_x86_avx_vpermilvar_pd(<2 x double> %a0, <2 x i64> %a1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP3]], [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <2 x i64> [[TMP1]] to <2 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0]], <2 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <2 x double> [[RES]] to <2 x i64>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       13:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A2:%.*]], <2 x i64> [[A1]])
 ; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[RES1]]
@@ -975,16 +980,27 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64>
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP3]], [[TMP15]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0]], <4 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       18:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       19:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A2:%.*]], <4 x i64> [[A1]])
 ; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[RES1]]
@@ -1014,16 +1030,27 @@ define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) #
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP3]], [[TMP15]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP1]] to <4 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP18:%.*]], label [[TMP19:%.*]], !prof [[PROF1]]
+; CHECK:       18:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       19:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A2:%.*]], <4 x i32> [[A1]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES1]]
@@ -1047,16 +1074,27 @@ define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, ptr %a1) #0
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP11]], 3
+; CHECK-NEXT:    [[TMP13:%.*]] = or i32 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 3
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP17]], [[TMP18]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP11]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
-; CHECK:       12:
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP19]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP23:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK:       23:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       13:
+; CHECK:       24:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A3:%.*]], <4 x i32> [[A2]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP10]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES1]]
@@ -1073,16 +1111,39 @@ define <8 x float> @test_x86_avx_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP3]], 7
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP3]], [[TMP27]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP9]], 7
+; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP24]], [[TMP25]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x float> [[RES]] to <8 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP26]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       31:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A2:%.*]], <8 x i32> [[A1]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES1]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
index 5aeaa1221cd21..e7854538aa3e8 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
@@ -8143,16 +8143,39 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP3]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP3]], [[TMP27]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP29]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 7
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP24]], [[TMP25]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x double> [[TMP7]] to <8 x i64>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP26]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       31:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP5]]
@@ -8169,16 +8192,39 @@ define <8 x double>@test_int_x86_avx512_mask_vpermilvar_pd_512(<8 x double> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP5]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i64 [[TMP5]], [[TMP29]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP31:%.*]] = and i64 [[TMP8]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP8]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i64 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP39:%.*]] = and i64 [[TMP38]], 7
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP42:%.*]] = and i64 [[TMP41]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP26]], [[TMP27]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
-; CHECK:       9:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP28]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP32:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       32:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       10:
+; CHECK:       33:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X4:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
@@ -8204,16 +8250,39 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermilvar_pd_512(<8 x double> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i64 [[TMP4]], [[TMP28]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP7]], [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP38:%.*]] = and i64 [[TMP37]], 7
+; CHECK-NEXT:    [[TMP39:%.*]] = or i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP40]], 7
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP40]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i64 [[TMP25]], [[TMP26]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x double> [[TMP8]] to <8 x i64>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP31:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       31:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       32:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
@@ -8238,16 +8307,63 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP3]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP3]], [[TMP51]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP53:%.*]] = and i32 [[TMP6]], 15
+; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP6]], [[TMP53]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP9]], 15
+; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 15
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP40:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP46:%.*]] = and i32 [[TMP45]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP48]], [[TMP49]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x float> [[TMP7]] to <16 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP9:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP50]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP54:%.*]], label [[TMP55:%.*]], !prof [[PROF1]]
+; CHECK:       54:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       55:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    store <16 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP5]]
@@ -8264,16 +8380,63 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512(<16 x float> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP53:%.*]] = and i32 [[TMP5]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP5]], [[TMP53]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP55:%.*]] = and i32 [[TMP8]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP8]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP60:%.*]] = and i32 [[TMP59]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP63:%.*]] = and i32 [[TMP62]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP66:%.*]] = and i32 [[TMP65]], 15
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP34:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP41]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP44]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i32 [[TMP47]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP50]], [[TMP51]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
-; CHECK:       9:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP52]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP56:%.*]], label [[TMP57:%.*]], !prof [[PROF1]]
+; CHECK:       56:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       10:
+; CHECK:       57:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X4:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
@@ -8300,16 +8463,63 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermilvar_ps_512(<16 x float> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP52:%.*]] = and i32 [[TMP4]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP4]], [[TMP52]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP54:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP7]], [[TMP54]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP59:%.*]] = and i32 [[TMP58]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP62:%.*]] = and i32 [[TMP61]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP64]], 15
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP64]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 15
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP39:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP43]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i32 [[TMP49]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP49]], [[TMP50]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[TMP8]] to <16 x i32>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP17:%.*]], label [[TMP18:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP51]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP55:%.*]], label [[TMP56:%.*]], !prof [[PROF1]]
+; CHECK:       55:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       56:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
@@ -13700,8 +13910,8 @@ define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
@@ -13714,9 +13924,62 @@ define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP4]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X4:%.*]])
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP42]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <16 x i32> [[TMP14]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP45]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP45]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP14]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP14]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP14]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP14]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP14]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP14]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP14]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP14]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP14]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP14]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP14]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP14]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP59:%.*]], label [[TMP60:%.*]], !prof [[PROF1]]
+; CHECK:       59:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       60:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X4:%.*]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP10]]
 ;
@@ -13744,9 +14007,62 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP59:%.*]], label [[TMP60:%.*]], !prof [[PROF1]]
+; CHECK:       59:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       60:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
@@ -13768,25 +14084,46 @@ declare <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double>, <8 x
 define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i64 [[TMP6]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i64> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i64 [[TMP23]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i64> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP4]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP36]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP32:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       32:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       33:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP9]]
 ;
   %res = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1)
@@ -13797,32 +14134,53 @@ define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0,
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP38:%.*]] = or i64 [[TMP7]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP39:%.*]] = or i64 [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 7
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP31:%.*]] = and i64 [[TMP30]], 7
+; CHECK-NEXT:    [[TMP41:%.*]] = or i64 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP32]], 7
+; CHECK-NEXT:    [[TMP42:%.*]] = or i64 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 7
+; CHECK-NEXT:    [[TMP43:%.*]] = or i64 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP5]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP43]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP44:%.*]], label [[TMP45:%.*]], !prof [[PROF1]]
+; CHECK:       33:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       34:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP8]], <8 x i64> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x double> [[TMP10]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP19]], <8 x i64> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <8 x i1> [[TMP13]], <8 x double> [[TMP10]], <8 x double> [[TMP11]]
@@ -13838,25 +14196,70 @@ declare <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float>, <16 x
 define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP6]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP6]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP39]], [[TMP42]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x i32> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP43]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i32> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP8]], i64 8
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP8]], i64 9
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP8]], i64 10
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP8]], i64 11
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP8]], i64 12
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP8]], i64 13
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP8]], i64 14
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP8]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP34]], [[TMP50]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP4]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[TMP35]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP28]], label [[TMP56:%.*]], label [[TMP57:%.*]], !prof [[PROF1]]
+; CHECK:       56:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       57:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <16 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP9]]
 ;
   %res = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1)
@@ -13867,32 +14270,77 @@ define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0,
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP7]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP40]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP44]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP53:%.*]] = and i32 [[TMP52]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP68:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP69:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP69]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP57:%.*]], label [[TMP58:%.*]], !prof [[PROF1]]
+; CHECK:       57:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       58:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> zeroinitializer, <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP8]], <16 x i32> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP19]], <16 x i32> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP13]], <16 x float> [[TMP10]], <16 x float> [[TMP11]]
@@ -13908,12 +14356,41 @@ declare <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64>, <8 x i64>, <
 define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i64> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP23]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP29:%.*]], label [[TMP30:%.*]], !prof [[PROF1]]
+; CHECK:       29:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       30:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
 ;
@@ -13925,13 +14402,42 @@ define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP38:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP2]]
@@ -13968,9 +14474,62 @@ define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X0:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP59:%.*]], label [[TMP60:%.*]], !prof [[PROF1]]
+; CHECK:       59:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       60:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
@@ -13999,7 +14558,7 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
 ; CHECK:       6:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
@@ -14013,26 +14572,47 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[X2INS:%.*]] = insertelement <8 x double> [[EXTRA_PARAM:%.*]], double [[X2S]], i32 0
 ; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP5]], <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[X2:%.*]] = shufflevector <8 x double> [[X2INS]], <8 x double> [[EXTRA_PARAM]], <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP14:%.*]], label [[TMP24:%.*]], !prof [[PROF1]]
-; CHECK:       14:
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP27]], [[TMP12]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <8 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 7
+; CHECK-NEXT:    [[TMP42:%.*]] = or i64 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <8 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP30]], 7
+; CHECK-NEXT:    [[TMP43:%.*]] = or i64 [[TMP30]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <8 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 7
+; CHECK-NEXT:    [[TMP44:%.*]] = or i64 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <8 x i64> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 7
+; CHECK-NEXT:    [[TMP45:%.*]] = or i64 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <8 x i64> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP39:%.*]] = and i64 [[TMP38]], 7
+; CHECK-NEXT:    [[TMP46:%.*]] = or i64 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <8 x i64> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP41:%.*]] = and i64 [[TMP40]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP47:%.*]] = or i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP2]] to <8 x double>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP11]], <8 x i64> [[X0:%.*]], <8 x double> [[TMP24]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <8 x double> [[TMP13]] to <8 x i64>
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP47]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP48:%.*]], label [[TMP49:%.*]], !prof [[PROF1]]
+; CHECK:       39:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       15:
-; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x double> [[X2]])
+; CHECK:       40:
+; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0]], <8 x double> [[X2]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP14]], <8 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x double> [[TMP15]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i64> [[TMP22]], <8 x i64> [[TMP18]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP17]], <8 x double> [[TMP15]], <8 x double> zeroinitializer
@@ -14052,30 +14632,75 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0,
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP7]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP40]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <16 x i32> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP44]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <16 x i32> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP52]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP52]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP68:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X0:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP68]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP57:%.*]], label [[TMP58:%.*]], !prof [[PROF1]]
+; CHECK:       57:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       58:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP14]], zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP17]], <16 x i32> [[TMP13]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP12]], <16 x float> [[TMP10]], <16 x float> zeroinitializer
@@ -14093,13 +14718,42 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i64> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i64> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <8 x i64> [[TMP13]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP31]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP13]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i64> [[TMP13]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i64> [[TMP13]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X0:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
@@ -14120,12 +14774,65 @@ declare <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32>, <16 x i32>
 define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)  #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP36]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 15
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP39]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i32> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP8]], i64 8
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP8]], i64 9
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP8]], i64 10
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP8]], i64 11
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP8]], i64 12
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP8]], i64 13
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP8]], i64 14
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP8]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP34]], [[TMP50]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[TMP35]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP28]], label [[TMP53:%.*]], label [[TMP54:%.*]], !prof [[PROF1]]
+; CHECK:       53:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       54:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
 ;
@@ -14137,13 +14844,66 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16
 ;
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP40:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP37]], [[TMP40]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x i32> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP41]], 15
+; CHECK-NEXT:    [[TMP39:%.*]] = or i32 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x i32> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP43]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <16 x i32> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP46:%.*]] = and i32 [[TMP45]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <16 x i32> [[TMP13]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP47]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP47]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i32> [[TMP13]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i32> [[TMP13]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i32> [[TMP13]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP13]], i64 8
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP13]], i64 9
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP13]], i64 10
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP13]], i64 11
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP13]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP13]], i64 13
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP13]], i64 14
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP13]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP35]], [[TMP51]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[TMP36]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP28]], label [[TMP54:%.*]], label [[TMP55:%.*]], !prof [[PROF1]]
+; CHECK:       54:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       55:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
index 1644a5e3a045c..d9ac1b43924bf 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
@@ -5467,9 +5467,62 @@ define <16 x i32>@test_int_x86_avx512_vpermi2var_d_512(<16 x i32> %x0, <16 x i32
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
 ; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP41]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP41]], [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP11]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP13]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP40:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP56]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP58:%.*]], label [[TMP59:%.*]], !prof [[PROF1]]
+; CHECK:       58:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       59:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP9]]
 ;
@@ -5496,9 +5549,62 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X1:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP59:%.*]], label [[TMP60:%.*]], !prof [[PROF1]]
+; CHECK:       59:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       60:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0:%.*]], <16 x i32> [[X1]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP3]]
@@ -5522,24 +5628,45 @@ declare <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double>, <8 x i64>,
 define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP6]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i64> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i64 [[TMP23]], [[TMP26]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i64 [[TMP10]], [[TMP27]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP4]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP35]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP32:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       32:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
-; CHECK-NEXT:    store <8 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK:       33:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    store <8 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[TMP9]]
 ;
   %1 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)
@@ -5549,32 +5676,53 @@ define <8 x double>@test_int_x86_avx512_vpermi2var_pd_512(<8 x double> %x0, <8 x
 define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x double> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP38:%.*]] = or i64 [[TMP7]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP39:%.*]] = or i64 [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 7
+; CHECK-NEXT:    [[TMP40:%.*]] = or i64 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP31:%.*]] = and i64 [[TMP30]], 7
+; CHECK-NEXT:    [[TMP41:%.*]] = or i64 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP33:%.*]] = and i64 [[TMP32]], 7
+; CHECK-NEXT:    [[TMP42:%.*]] = or i64 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 7
+; CHECK-NEXT:    [[TMP43:%.*]] = or i64 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP3]] to <8 x double>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP5]], <8 x i64> [[X1:%.*]], <8 x double> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x double> [[TMP9]] to <8 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP43]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP44:%.*]], label [[TMP45:%.*]], !prof [[PROF1]]
+; CHECK:       33:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       34:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X0:%.*]], <8 x i64> [[X1]], <8 x double> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[X1]] to <8 x double>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i64> [[TMP8]], <8 x i64> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x double> [[TMP10]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x double> [[TMP11]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <8 x i64> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i64> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i64> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i64> [[TMP19]], <8 x i64> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <8 x i1> [[TMP13]], <8 x double> [[TMP10]], <8 x double> [[TMP11]]
@@ -5593,24 +5741,69 @@ declare <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float>, <16 x i32>
 define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP6]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP6]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP39]], [[TMP42]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP10]], [[TMP43]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i32> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP8]], i64 8
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP8]], i64 9
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP8]], i64 10
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP8]], i64 11
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP8]], i64 12
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP8]], i64 13
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP8]], i64 14
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP8]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP34]], [[TMP50]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP4]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[TMP35]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP28]], label [[TMP56:%.*]], label [[TMP57:%.*]], !prof [[PROF1]]
+; CHECK:       56:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       57:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    store <16 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[TMP9]]
 ;
   %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)
@@ -5620,32 +5813,77 @@ define <16 x float>@test_int_x86_avx512_vpermi2var_ps_512(<16 x float> %x0, <16
 define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP7]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP40]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP44]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP53:%.*]] = and i32 [[TMP52]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP68:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP69:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X1:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP9]] to <16 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP69]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP57:%.*]], label [[TMP58:%.*]], !prof [[PROF1]]
+; CHECK:       57:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       58:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[X1]] to <16 x float>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> zeroinitializer, <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP8]], <16 x i32> [[TMP2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <16 x float> [[TMP11]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <16 x i32> [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i32> [[TMP17]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i32> [[TMP18]], [[TMP2]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP19]], <16 x i32> [[TMP14]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP13]], <16 x float> [[TMP10]], <16 x float> [[TMP11]]
@@ -5664,12 +5902,41 @@ declare <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i
 define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i64> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP23]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP29:%.*]], label [[TMP30:%.*]], !prof [[PROF1]]
+; CHECK:       29:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       30:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[TMP4]]
 ;
@@ -5680,13 +5947,42 @@ define <8 x i64>@test_int_x86_avx512_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %
 define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X1:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP38:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0:%.*]], <8 x i64> [[X1]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> [[TMP2]]
@@ -5722,9 +6018,62 @@ define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP2]], <16 x i32> [[X0:%.*]], <16 x i32> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP59:%.*]], label [[TMP60:%.*]], !prof [[PROF1]]
+; CHECK:       59:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       60:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
@@ -5753,7 +6102,7 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
 ; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
@@ -5767,26 +6116,47 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 ; CHECK-NEXT:    [[X2INS:%.*]] = insertelement <8 x double> [[EXTRA_PARAM:%.*]], double [[X2S]], i32 0
 ; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <8 x i64> [[_MSPROP]], <8 x i64> [[TMP6]], <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[X2:%.*]] = shufflevector <8 x double> [[X2INS]], <8 x double> [[EXTRA_PARAM2:%.*]], <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i64> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSOR]], [[_MSCMP4]]
-; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
-; CHECK:       15:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       16:
-; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x double> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP12]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP43:%.*]] = or i64 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <8 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP34:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NEXT:    [[TMP44:%.*]] = or i64 [[TMP31]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <8 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 7
+; CHECK-NEXT:    [[TMP45:%.*]] = or i64 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <8 x i64> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP38:%.*]] = and i64 [[TMP37]], 7
+; CHECK-NEXT:    [[TMP46:%.*]] = or i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <8 x i64> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP40:%.*]] = and i64 [[TMP39]], 7
+; CHECK-NEXT:    [[TMP47:%.*]] = or i64 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <8 x i64> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP42:%.*]] = and i64 [[TMP41]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i64> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 7
+; CHECK-NEXT:    [[TMP48:%.*]] = or i64 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i64> [[TMP2]] to <8 x double>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i64> [[_MSPROP1]] to <8 x double>
+; CHECK-NEXT:    [[TMP14:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[TMP24]], <8 x i64> [[X0:%.*]], <8 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x double> [[TMP14]] to <8 x i64>
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP48]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP49:%.*]], label [[TMP50:%.*]], !prof [[PROF1]]
+; CHECK:       40:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       41:
+; CHECK-NEXT:    [[TMP15:%.*]] = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> [[X1:%.*]], <8 x i64> [[X0]], <8 x double> [[X2]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> zeroinitializer, <8 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i64> [[TMP25]], <8 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x double> [[TMP15]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor <8 x i64> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i64> [[TMP20]], [[TMP25]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i64> [[TMP21]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP16]], <8 x i64> [[TMP22]], <8 x i64> [[TMP18]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP17]], <8 x double> [[TMP15]], <8 x double> zeroinitializer
@@ -5805,30 +6175,75 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to i512
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       9:
-; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x float> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP7]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP40]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <16 x i32> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP44]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <16 x i32> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP52]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP52]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP64:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP65:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP66:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP67:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP68:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP3]] to <16 x float>
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[TMP5]], <16 x i32> [[X0:%.*]], <16 x float> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x float> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP68]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP57:%.*]], label [[TMP58:%.*]], !prof [[PROF1]]
+; CHECK:       57:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       58:
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> [[X1:%.*]], <16 x i32> [[X0]], <16 x float> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <16 x float> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP14]], zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], zeroinitializer
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP17]], <16 x i32> [[TMP13]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP12]], <16 x float> [[TMP10]], <16 x float> zeroinitializer
@@ -5844,13 +6259,42 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0,
 define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0:%.*]], <8 x i64> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i64> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i64> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i64 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <8 x i64> [[TMP13]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP31]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP13]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i64> [[TMP13]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i64> [[TMP13]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> [[X0:%.*]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; CHECK:       30:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X1:%.*]], <8 x i64> [[X0]], <8 x i64> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[X3:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i64> [[_MSPROP1]], <8 x i64> zeroinitializer
@@ -5871,12 +6315,65 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x
 define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
-; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP36]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 15
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP8]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP39]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP8]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP8]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i32> [[TMP8]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP8]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP8]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP8]], i64 8
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP8]], i64 9
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP8]], i64 10
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP8]], i64 11
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP8]], i64 12
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP8]], i64 13
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP8]], i64 14
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP8]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP34]], [[TMP50]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[TMP35]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP28]], label [[TMP53:%.*]], label [[TMP54:%.*]], !prof [[PROF1]]
+; CHECK:       53:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       54:
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
 ;
@@ -5887,13 +6384,66 @@ define <16 x i32>@test_int_x86_avx512_vpermt2var_d_512(<16 x i32> %x0, <16 x i32
 define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) #0 {
 ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0:%.*]], <16 x i32> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP40:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP37]], [[TMP40]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x i32> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP41]], 15
+; CHECK-NEXT:    [[TMP39:%.*]] = or i32 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x i32> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP43]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <16 x i32> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP46:%.*]] = and i32 [[TMP45]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <16 x i32> [[TMP13]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP47]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP47]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i32> [[TMP13]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i32> [[TMP13]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i32> [[TMP13]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP13]], i64 8
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP58:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP13]], i64 9
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP13]], i64 10
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP13]], i64 11
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP61:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP13]], i64 12
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP62:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP13]], i64 13
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP63:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP13]], i64 14
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP13]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP35]], [[TMP51]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> [[X0:%.*]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP28:%.*]] = icmp ne i32 [[TMP36]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP28]], label [[TMP54:%.*]], label [[TMP55:%.*]], !prof [[PROF1]]
+; CHECK:       54:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
+; CHECK-NEXT:    unreachable
+; CHECK:       55:
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X1:%.*]], <16 x i32> [[X0]], <16 x i32> [[X2:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
@@ -7602,34 +8152,19 @@ define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x
 ; CHECK-LABEL: @test_int_x86_avx512_mask_cvt_ps2dq_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP3]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2:%.*]], i32 10)
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i512 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
-; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
-; CHECK:       10:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       11:
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[X2:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <16 x i1> [[TMP4]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP5]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2]], i32 10)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <16 x i1> [[TMP7]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <16 x i1> splat (i1 true), <16 x i32> [[TMP8]], <16 x i32> [[TMP2]]
 ; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> [[X0]], <16 x i32> [[X1]], i16 -1, i32 8)
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[RES2:%.*]] = add <16 x i32> [[RES]], [[RES1]]
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[RES2]]
 ;
   %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 10)
@@ -8657,16 +9192,39 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP3]], 7
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP3]], [[TMP27]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 7
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP24]], [[TMP25]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP26]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP30:%.*]], label [[TMP31:%.*]], !prof [[PROF1]]
+; CHECK:       30:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       31:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x double> [[RES1]]
@@ -8682,16 +9240,39 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_mask(<8 x double> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP5]], 7
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP29]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP8]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP8]], [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i64 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP38:%.*]] = and i64 [[TMP37]], 7
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP26]], [[TMP27]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
-; CHECK:       9:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP28]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP32:%.*]], label [[TMP33:%.*]], !prof [[PROF1]]
+; CHECK:       32:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       10:
+; CHECK:       33:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X3:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
@@ -8718,16 +9299,39 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_maskz(<8 x double> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = and i64 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP28]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i64 [[TMP7]], [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i64 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP36]], 7
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP36]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i64 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i64 [[TMP25]], [[TMP26]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <8 x i64> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X0]], <8 x i64> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x double> [[RES]] to <8 x i64>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP31:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       31:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       32:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> [[X2:%.*]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[TMP3]] to <8 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
@@ -8754,16 +9358,63 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP3]], 15
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP3]], [[TMP51]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 15
+; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP9]], 15
+; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 15
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 15
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 15
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 15
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP40:%.*]] = and i32 [[TMP39]], 15
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP42]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP46:%.*]] = and i32 [[TMP45]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP49:%.*]] = and i32 [[TMP48]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP48]], [[TMP49]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP50]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP54:%.*]], label [[TMP55:%.*]], !prof [[PROF1]]
+; CHECK:       54:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       8:
+; CHECK:       55:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    store <16 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x float> [[RES1]]
@@ -8779,16 +9430,63 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_mask(<16 x float> %x0,
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP53:%.*]] = and i32 [[TMP5]], 15
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP5]], [[TMP53]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP54:%.*]] = and i32 [[TMP8]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP8]], [[TMP54]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP59:%.*]] = and i32 [[TMP58]], 15
+; CHECK-NEXT:    [[TMP60:%.*]] = or i32 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP62:%.*]] = and i32 [[TMP61]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 15
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 15
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i32 [[TMP29]], 15
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP34:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP35]], 15
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP41]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i32 [[TMP44]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i32 [[TMP47]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i32 [[TMP50]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP50]], [[TMP51]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
-; CHECK:       9:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP52]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP56:%.*]], label [[TMP57:%.*]], !prof [[PROF1]]
+; CHECK:       56:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       10:
+; CHECK:       57:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X3:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
@@ -8815,16 +9513,63 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_maskz(<16 x float> %x0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP52:%.*]] = and i32 [[TMP4]], 15
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP4]], [[TMP52]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP53:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP7]], [[TMP53]]
+; CHECK-NEXT:    [[TMP57:%.*]] = extractelement <16 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP58:%.*]] = and i32 [[TMP57]], 15
+; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <16 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP60]], 15
+; CHECK-NEXT:    [[TMP15:%.*]] = or i32 [[TMP60]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 15
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 15
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP2]], i64 8
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i32> [[TMP2]], i64 9
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP31]], 15
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP2]], i64 10
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i32> [[TMP2]], i64 11
+; CHECK-NEXT:    [[TMP38:%.*]] = and i32 [[TMP37]], 15
+; CHECK-NEXT:    [[TMP39:%.*]] = or i32 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP2]], i64 12
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <16 x i32> [[TMP2]], i64 13
+; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP43]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <16 x i32> [[TMP2]], i64 14
+; CHECK-NEXT:    [[TMP47:%.*]] = and i32 [[TMP46]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <16 x i32> [[TMP2]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i32 [[TMP49]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP49]], [[TMP50]]
 ; CHECK-NEXT:    [[X0:%.*]] = bitcast <16 x i32> [[TMP1]] to <16 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X0]], <16 x i32> [[X1:%.*]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x float> [[RES]] to <16 x i32>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP51]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP55:%.*]], label [[TMP56:%.*]], !prof [[PROF1]]
+; CHECK:       55:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       56:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[X2:%.*]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll
new file mode 100644
index 0000000000000..db4ad6b8fc28b
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll
@@ -0,0 +1,7652 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+;
+; Strictly handled:
+; - llvm.x86.avx512.dbpsadbw.512
+; - llvm.x86.avx512.packssdw.512, llvm.x86.avx512.packsswb.512
+; - llvm.x86.avx512.packusdw.512, llvm.x86.avx512.packuswb.512
+; - llvm.x86.avx512.pmaddubs.w.512
+; - llvm.x86.avx512.pmaddw.d.512
+;
+; Heuristically handled:
+; - llvm.sadd.sat.v32i16, llvm.sadd.sat.v64i8
+; - llvm.smax.v32i16, llvm.smax.v64i8
+; - llvm.smin.v32i16, llvm.smin.v64i8
+; - llvm.ssub.sat.v32i16, llvm.ssub.sat.v64i8
+; - llvm.uadd.sat.v32i16, llvm.uadd.sat.v64i8
+; - llvm.umax.v32i16, llvm.umax.v64i8
+; - llvm.umin.v32i16, llvm.umin.v64i8
+; - llvm.usub.sat.v32i16, llvm.usub.sat.v64i8
+; - llvm.x86.avx512.pavg.b.512, llvm.x86.avx512.pavg.w.512
+; - llvm.x86.avx512.permvar.hi.512
+; - llvm.x86.avx512.pmul.hr.sw.512, llvm.x86.avx512.pmulhu.w.512, llvm.x86.avx512.pmulh.w.512
+; - llvm.x86.avx512.pshuf.b.512
+; - llvm.x86.avx512.psllv.w.512, llvm.x86.avx512.psrav.w.512,  llvm.x86.avx512.psrlv.w.512
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @llvm.x86.avx512.kunpck.wd(i32, i32)
+
+define i32 @test_int_x86_avx512_kunpck_wd(i32 %x0, i32 %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_kunpck_wd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[X0:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X1:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i1> [[TMP3]], <32 x i1> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i1> [[TMP4]], <32 x i1> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <32 x i1> [[TMP5]], <32 x i1> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i1> [[TMP6]], <32 x i1> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <16 x i1> [[_MSPROP1]], <16 x i1> [[_MSPROP]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i1> [[TMP8]], <16 x i1> [[TMP7]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i1> [[_MSPROP2]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i1> [[TMP9]] to i32
+; CHECK-NEXT:    store i32 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  %res = call i32 @llvm.x86.avx512.kunpck.wd(i32 %x0, i32 %x1)
+  ret i32 %res
+}
+
+declare i64 @llvm.x86.avx512.kunpck.dq(i64, i64)
+
+define i64 @test_int_x86_avx512_kunpck_qd(i64 %x0, i64 %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_kunpck_qd(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[TMP1]] to <64 x i1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64 [[X0:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP2]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[X1:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i1> [[TMP3]], <64 x i1> [[TMP3]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <64 x i1> [[TMP4]], <64 x i1> [[TMP4]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <64 x i1> [[TMP5]], <64 x i1> [[TMP5]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <64 x i1> [[TMP6]], <64 x i1> [[TMP6]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <32 x i1> [[_MSPROP1]], <32 x i1> [[_MSPROP]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <32 x i1> [[TMP8]], <32 x i1> [[TMP7]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i1> [[_MSPROP2]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i1> [[TMP9]] to i64
+; CHECK-NEXT:    store i64 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP11]]
+;
+  %res = call i64 @llvm.x86.avx512.kunpck.dq(i64 %x0, i64 %x1)
+  ret i64 %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8, <64 x i8>, i64)
+
+define { <64 x i8>, <64 x i8>, <64 x i8> } @test_int_x86_avx512_mask_pbroadcast_b_gpr_512(i8 %x0, <64 x i8> %x1, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pbroadcast_b_gpr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <64 x i8> splat (i8 -1), i8 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <64 x i8> poison, i8 [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <64 x i8> [[_MSPROP]], <64 x i8> splat (i8 -1), <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <64 x i8> [[DOTSPLATINSERT3]], <64 x i8> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <64 x i8> splat (i8 -1), i8 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <64 x i8> poison, i8 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = shufflevector <64 x i8> [[_MSPROP6]], <64 x i8> splat (i8 -1), <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <64 x i8> [[DOTSPLATINSERT1]], <64 x i8> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64 [[TMP2]] to <64 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[_MSPROP7]], <64 x i8> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[DOTSPLAT2]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP7]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> [[TMP9]], <64 x i8> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[DOTSPLAT2]], <64 x i8> [[X1]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <64 x i8> splat (i8 -1), i8 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = shufflevector <64 x i8> [[_MSPROP8]], <64 x i8> splat (i8 -1), <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <64 x i8> [[DOTSPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP2]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[_MSPROP9]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <64 x i8> [[DOTSPLAT]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i8> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT10:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP16]], <64 x i8> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[DOTSPLAT]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } { <64 x i8> splat (i8 -1), <64 x i8> splat (i8 -1), <64 x i8> splat (i8 -1) }, <64 x i8> [[_MSPROP5]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } poison, <64 x i8> [[DOTSPLAT4]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[TMP18]], <64 x i8> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[RES3]], <64 x i8> [[TMP10]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[TMP19]], <64 x i8> [[_MSPROP_SELECT10]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[RES4]], <64 x i8> [[TMP17]], 2
+; CHECK-NEXT:    store { <64 x i8>, <64 x i8>, <64 x i8> } [[TMP20]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <64 x i8>, <64 x i8>, <64 x i8> } [[RES5]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8 %x0, <64 x i8> %x1, i64 -1)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8 %x0, <64 x i8> %x1, i64 %mask)
+  %res2 = call <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8 %x0, <64 x i8> zeroinitializer, i64 %mask)
+  %res3 = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } poison, <64 x i8>  %res, 0
+  %res4 = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> }  %res3, <64 x i8> %res1, 1
+  %res5 = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> }  %res4, <64 x i8> %res2, 2
+  ret { <64 x i8>, <64 x i8>, <64 x i8> } %res5
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.512(i16, <32 x i16>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_pbroadcast_w_gpr_512(i16 %x0, <32 x i16> %x1, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pbroadcast_w_gpr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <32 x i16> splat (i16 -1), i16 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <32 x i16> poison, i16 [[X0:%.*]], i64 0
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <32 x i16> [[_MSPROP]], <32 x i16> splat (i16 -1), <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <32 x i16> [[DOTSPLATINSERT3]], <32 x i16> poison, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = insertelement <32 x i16> splat (i16 -1), i16 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <32 x i16> poison, i16 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = shufflevector <32 x i16> [[_MSPROP6]], <32 x i16> splat (i16 -1), <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <32 x i16> [[DOTSPLATINSERT1]], <32 x i16> poison, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[_MSPROP7]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[DOTSPLAT2]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[_MSPROP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP9]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[DOTSPLAT2]], <32 x i16> [[X1]]
+; CHECK-NEXT:    [[_MSPROP8:%.*]] = insertelement <32 x i16> splat (i16 -1), i16 [[TMP1]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[X0]], i64 0
+; CHECK-NEXT:    [[_MSPROP9:%.*]] = shufflevector <32 x i16> [[_MSPROP8]], <32 x i16> splat (i16 -1), <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <32 x i16> [[DOTSPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[DOTSPLAT]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT10:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[DOTSPLAT]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP5]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[DOTSPLAT4]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP18]], <32 x i16> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]], <32 x i16> [[TMP10]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP19]], <32 x i16> [[_MSPROP_SELECT10]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES4]], <32 x i16> [[TMP17]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP20]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES5]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.512(i16 %x0, <32 x i16> %x1, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.512(i16 %x0, <32 x i16> %x1, i32 %mask)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.512(i16 %x0, <32 x i16> zeroinitializer, i32 %mask)
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16>  %res, 0
+  %res4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res3, <32 x i16> %res1, 1
+  %res5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res4, <32 x i16> %res2, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res5
+  }
+
+declare void @llvm.x86.avx512.mask.storeu.b.512(ptr, <64 x i8>, i64)
+
+define void @test_int_x86_avx512_mask_storeu_b_512(ptr %ptr1, ptr %ptr2, <64 x i8> %x1, i64 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_storeu_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP1]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[X2:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR1:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP2]], ptr [[TMP9]], i32 1, <64 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i1> [[TMP5]] to i64
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i64 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[X1:%.*]], ptr [[PTR1]], i32 1, <64 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <64 x i8> [[TMP2]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    store <64 x i8> [[X1]], ptr [[PTR2]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.storeu.b.512(ptr %ptr1, <64 x i8> %x1, i64 %x2)
+  call void @llvm.x86.avx512.mask.storeu.b.512(ptr %ptr2, <64 x i8> %x1, i64 -1)
+  ret void
+}
+
+declare void @llvm.x86.avx512.mask.storeu.w.512(ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_storeu_w_512(ptr %ptr1, ptr %ptr2, <32 x i16> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_storeu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR1:%.*]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    call void @llvm.masked.store.v32i16.p0(<32 x i16> [[TMP2]], ptr [[TMP9]], i32 1, <32 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i1> [[TMP5]] to i32
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    call void @llvm.masked.store.v32i16.p0(<32 x i16> [[X1:%.*]], ptr [[PTR1]], i32 1, <32 x i1> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       14:
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP15]], 87960930222080
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    store <32 x i16> [[TMP2]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    store <32 x i16> [[X1]], ptr [[PTR2]], align 1
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.storeu.w.512(ptr %ptr1, <32 x i16> %x1, i32 %x2)
+  call void @llvm.x86.avx512.mask.storeu.w.512(ptr %ptr2, <32 x i16> %x1, i32 -1)
+  ret void
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.loadu.w.512(ptr, <32 x i16>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_loadu_w_512(ptr %ptr, ptr %ptr2, <32 x i16> %x1, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_loadu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load <32 x i16>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <32 x i16> @llvm.masked.load.v32i16.p0(ptr [[TMP14]], i32 1, <32 x i1> [[TMP11]], <32 x i16> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <32 x i1> [[TMP10]] to i32
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i32 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    [[TMP18:%.*]] = call <32 x i16> @llvm.masked.load.v32i16.p0(ptr [[PTR2]], i32 1, <32 x i1> [[TMP11]], <32 x i16> [[TMP6]])
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <32 x i16> @llvm.masked.load.v32i16.p0(ptr [[TMP23]], i32 1, <32 x i1> [[TMP20]], <32 x i16> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <32 x i1> [[TMP19]] to i32
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i32 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP25:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       25:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       26:
+; CHECK-NEXT:    [[TMP27:%.*]] = call <32 x i16> @llvm.masked.load.v32i16.p0(ptr [[PTR]], i32 1, <32 x i1> [[TMP20]], <32 x i16> zeroinitializer)
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSLD]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[TMP6]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP28]], <32 x i16> [[_MSMASKEDLD]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]], <32 x i16> [[TMP18]], 1
+; CHECK-NEXT:    [[TMP30:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP29]], <32 x i16> [[_MSMASKEDLD1]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES4]], <32 x i16> [[TMP27]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP30]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES5]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.loadu.w.512(ptr %ptr, <32 x i16> %x1, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.loadu.w.512(ptr %ptr2, <32 x i16> %res, i32 %mask)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.loadu.w.512(ptr %ptr, <32 x i16> zeroinitializer, i32 %mask)
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16>  %res, 0
+  %res4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res3, <32 x i16> %res1, 1
+  %res5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res4, <32 x i16> %res2, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res5
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.loadu.b.512(ptr, <64 x i8>, i64)
+
+define { <64 x i8>, <64 x i8>, <64 x i8> } @test_int_x86_avx512_mask_loadu_b_512(ptr %ptr, ptr %ptr2, <64 x i8> %x1, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_loadu_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load <64 x i8>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP2]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[PTR2:%.*]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], 87960930222080
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr [[TMP14]], i32 1, <64 x i1> [[TMP11]], <64 x i8> [[_MSLD]])
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <64 x i1> [[TMP10]] to i64
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP16:%.*]], label [[TMP17:%.*]], !prof [[PROF1]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    [[TMP18:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr [[PTR2]], i32 1, <64 x i1> [[TMP11]], <64 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast i64 [[TMP2]] to <64 x i1>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP21]], 87960930222080
+; CHECK-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[_MSMASKEDLD1:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr [[TMP23]], i32 1, <64 x i1> [[TMP20]], <64 x i8> zeroinitializer)
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <64 x i1> [[TMP19]] to i64
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i64 [[TMP24]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSCMP4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP25:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; CHECK:       25:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       26:
+; CHECK-NEXT:    [[TMP27:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr [[PTR]], i32 1, <64 x i1> [[TMP20]], <64 x i8> zeroinitializer)
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } { <64 x i8> splat (i8 -1), <64 x i8> splat (i8 -1), <64 x i8> splat (i8 -1) }, <64 x i8> [[_MSLD]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } poison, <64 x i8> [[TMP6]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[TMP28]], <64 x i8> [[_MSMASKEDLD]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[RES3]], <64 x i8> [[TMP18]], 1
+; CHECK-NEXT:    [[TMP30:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[TMP29]], <64 x i8> [[_MSMASKEDLD1]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } [[RES4]], <64 x i8> [[TMP27]], 2
+; CHECK-NEXT:    store { <64 x i8>, <64 x i8>, <64 x i8> } [[TMP30]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <64 x i8>, <64 x i8>, <64 x i8> } [[RES5]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.loadu.b.512(ptr %ptr, <64 x i8> %x1, i64 -1)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.loadu.b.512(ptr %ptr2, <64 x i8> %res, i64 %mask)
+  %res2 = call <64 x i8> @llvm.x86.avx512.mask.loadu.b.512(ptr %ptr, <64 x i8> zeroinitializer, i64 %mask)
+  %res3 = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> } poison, <64 x i8>  %res, 0
+  %res4 = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> }  %res3, <64 x i8> %res1, 1
+  %res5 = insertvalue { <64 x i8>, <64 x i8>, <64 x i8> }  %res4, <64 x i8> %res2, 2
+  ret { <64 x i8>, <64 x i8>, <64 x i8> } %res5
+}
+
+declare <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64>, i32)
+
+define { <8 x i64>, <8 x i64> } @test_int_x86_avx512_psll_dq_512(<8 x i64> %x0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psll_dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to <64 x i8>
+; CHECK-NEXT:    [[CAST2:%.*]] = bitcast <8 x i64> [[X0:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> zeroinitializer, <64 x i8> [[TMP2]], <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> zeroinitializer, <64 x i8> [[CAST2]], <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <64 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[CAST3:%.*]] = bitcast <64 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <64 x i8>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <8 x i64> [[X0]] to <64 x i8>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <64 x i8> zeroinitializer, <64 x i8> [[TMP5]], <64 x i32> <i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <64 x i8> zeroinitializer, <64 x i8> [[CAST]], <64 x i32> <i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <64 x i8> [[_MSPROP4]] to <8 x i64>
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast <64 x i8> [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[TMP4]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i64>, <8 x i64> } poison, <8 x i64> [[CAST3]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertvalue { <8 x i64>, <8 x i64> } [[TMP8]], <8 x i64> [[TMP7]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64> } [[RES2]], <8 x i64> [[CAST1]], 1
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64> } [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64> } [[RES3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64> %x0, i32 8)
+  %res1 = call <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64> %x0, i32 4)
+  %res2 = insertvalue { <8 x i64>, <8 x i64> } poison, <8 x i64>  %res, 0
+  %res3 = insertvalue { <8 x i64>, <8 x i64> }  %res2, <8 x i64> %res1, 1
+  ret { <8 x i64>, <8 x i64> } %res3
+}
+
+define <8 x i64> @test_int_x86_avx512_psll_load_dq_512(ptr %p0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psll_load_dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[X0:%.*]] = load <8 x i64>, ptr [[P0:%.*]], align 64
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP6]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <8 x i64> [[X0]] to <64 x i8>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> zeroinitializer, <64 x i8> [[TMP7]], <64 x i32> <i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <64 x i8> zeroinitializer, <64 x i8> [[CAST]], <64 x i32> <i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <64 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast <64 x i8> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[CAST1]]
+;
+  %x0 = load <8 x i64>, ptr%p0
+  %res = call <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64> %x0, i32 4)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64>, i32)
+
+define { <8 x i64>, <8 x i64> } @test_int_x86_avx512_psrl_dq_512(<8 x i64> %x0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrl_dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[TMP1]] to <64 x i8>
+; CHECK-NEXT:    [[CAST2:%.*]] = bitcast <8 x i64> [[X0:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP2]], <64 x i8> zeroinitializer, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[CAST2]], <64 x i8> zeroinitializer, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <64 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[CAST3:%.*]] = bitcast <64 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i64> [[TMP1]] to <64 x i8>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <8 x i64> [[X0]] to <64 x i8>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = shufflevector <64 x i8> [[TMP5]], <64 x i8> zeroinitializer, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <64 x i8> [[CAST]], <64 x i8> zeroinitializer, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <64 x i8> [[_MSPROP4]] to <8 x i64>
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast <64 x i8> [[TMP6]] to <8 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { <8 x i64>, <8 x i64> } { <8 x i64> splat (i64 -1), <8 x i64> splat (i64 -1) }, <8 x i64> [[TMP4]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i64>, <8 x i64> } poison, <8 x i64> [[CAST3]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertvalue { <8 x i64>, <8 x i64> } [[TMP8]], <8 x i64> [[TMP7]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64> } [[RES2]], <8 x i64> [[CAST1]], 1
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64> } [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64> } [[RES3]]
+;
+  %res = call <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64> %x0, i32 8)
+  %res1 = call <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64> %x0, i32 4)
+  %res2 = insertvalue { <8 x i64>, <8 x i64> } poison, <8 x i64>  %res, 0
+  %res3 = insertvalue { <8 x i64>, <8 x i64> }  %res2, <8 x i64> %res1, 1
+  ret { <8 x i64>, <8 x i64> } %res3
+}
+
+define <8 x i64> @test_int_x86_avx512_psrl_load_dq_512(ptr %p0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrl_load_dq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP2:%.*]], label [[TMP3:%.*]], !prof [[PROF1]]
+; CHECK:       2:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       3:
+; CHECK-NEXT:    [[X0:%.*]] = load <8 x i64>, ptr [[P0:%.*]], align 64
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i64>, ptr [[TMP6]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i64> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <8 x i64> [[X0]] to <64 x i8>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP7]], <64 x i8> zeroinitializer, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <64 x i8> [[CAST]], <64 x i8> zeroinitializer, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <64 x i8> [[_MSPROP]] to <8 x i64>
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast <64 x i8> [[TMP8]] to <8 x i64>
+; CHECK-NEXT:    store <8 x i64> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i64> [[CAST1]]
+;
+  %x0 = load <8 x i64>, ptr%p0
+  %res = call <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64> %x0, i32 4)
+  ret <8 x i64> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8>, <64 x i8>, i32, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_palignr_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_palignr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
+; CHECK-NEXT:    [[PALIGNR:%.*]] = shufflevector <64 x i8> [[X1:%.*]], <64 x i8> [[X0:%.*]], <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[PALIGNR]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <64 x i8> %x3, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_palignr_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x3, i64 %x4) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_palignr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
+; CHECK-NEXT:    [[PALIGNR:%.*]] = shufflevector <64 x i8> [[X1:%.*]], <64 x i8> [[X0:%.*]], <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[X4:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[PALIGNR]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP10]], <64 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[PALIGNR]], <64 x i8> [[X3]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP11]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <64 x i8> %x3, i64 %x4)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_maskz_palignr_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x4) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_palignr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
+; CHECK-NEXT:    [[PALIGNR:%.*]] = shufflevector <64 x i8> [[X1:%.*]], <64 x i8> [[X0:%.*]], <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[X4:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[PALIGNR]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> [[TMP9]], <64 x i8> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[PALIGNR]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP10]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <64 x i8> zeroinitializer, i64 %x4)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pshufh.w.512(<32 x i16>, i32, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pshufh_w_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshufh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP1]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 28, i32 28>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X0]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 28, i32 28>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pshufh.w.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pshufh_w_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshufh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP1]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 28, i32 28>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X0]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 28, i32 28>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pshufh.w.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_pshufh_w_512(<32 x i16> %x0, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pshufh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP1]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 28, i32 28>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X0]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 28, i32 28>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP9]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP3]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP10]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pshufh.w.512(<32 x i16> %x0, i32 3, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pshufl.w.512(<32 x i16>, i32, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pshufl_w_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshufl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP1]], <32 x i32> <i32 3, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X0]], <32 x i32> <i32 3, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pshufl.w.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pshufl_w_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshufl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP1]], <32 x i32> <i32 3, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X0]], <32 x i32> <i32 3, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pshufl.w.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_pshufl_w_512(<32 x i16> %x0, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pshufl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP1]], <32 x i32> <i32 3, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X0]], <32 x i32> <i32 3, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i16> [[TMP9]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP3]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP10]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pshufl.w.512(<32 x i16> %x0, i32 3, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define i64 @test_pcmpeq_b(<64 x i8> %a, <64 x i8> %b) nounwind #0 {
+; CHECK-LABEL: @test_pcmpeq_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <64 x i8> [[TMP4]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <64 x i8> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <64 x i8> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i1> [[TMP9]] to i64
+; CHECK-NEXT:    store i64 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP11]]
+;
+  %res = call i64 @llvm.x86.avx512.mask.pcmpeq.b.512(<64 x i8> %a, <64 x i8> %b, i64 -1)
+  ret i64 %res
+}
+
+define i64 @test_mask_pcmpeq_b(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_pcmpeq_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <64 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[TMP5]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <64 x i8> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <64 x i8> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <64 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <64 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <64 x i1> [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <64 x i1> [[TMP18]] to i64
+; CHECK-NEXT:    store i64 [[TMP19]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP20]]
+;
+  %res = call i64 @llvm.x86.avx512.mask.pcmpeq.b.512(<64 x i8> %a, <64 x i8> %b, i64 %mask)
+  ret i64 %res
+}
+
+declare i64 @llvm.x86.avx512.mask.pcmpeq.b.512(<64 x i8>, <64 x i8>, i64)
+
+define i32 @test_pcmpeq_w(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_pcmpeq_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <32 x i16> [[TMP4]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i16> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i16> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i1> [[TMP9]] to i32
+; CHECK-NEXT:    store i32 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  %res = call i32 @llvm.x86.avx512.mask.pcmpeq.w.512(<32 x i16> %a, <32 x i16> %b, i32 -1)
+  ret i32 %res
+}
+
+define i32 @test_mask_pcmpeq_w(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_pcmpeq_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <32 x i16> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP5]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <32 x i16> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <32 x i16> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <32 x i1> [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i1> [[TMP18]] to i32
+; CHECK-NEXT:    store i32 [[TMP19]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP20]]
+;
+  %res = call i32 @llvm.x86.avx512.mask.pcmpeq.w.512(<32 x i16> %a, <32 x i16> %b, i32 %mask)
+  ret i32 %res
+}
+
+declare i32 @llvm.x86.avx512.mask.pcmpeq.w.512(<32 x i16>, <32 x i16>, i32)
+
+define i64 @test_pcmpgt_b(<64 x i8> %a, <64 x i8> %b) nounwind #0 {
+; CHECK-LABEL: @test_pcmpgt_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[A:%.*]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <64 x i8> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <64 x i8> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[B:%.*]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP9:%.*]] = and <64 x i8> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt <64 x i8> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <64 x i8> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i1> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt <64 x i8> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <64 x i1> [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <64 x i1> [[TMP14]] to i64
+; CHECK-NEXT:    store i64 [[TMP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP16]]
+;
+  %res = call i64 @llvm.x86.avx512.mask.pcmpgt.b.512(<64 x i8> %a, <64 x i8> %b, i64 -1)
+  ret i64 %res
+}
+
+define i64 @test_mask_pcmpgt_b(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_pcmpgt_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <64 x i8> [[A:%.*]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <64 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <64 x i8> [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[B:%.*]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP10:%.*]] = and <64 x i8> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <64 x i8> [[TMP6]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt <64 x i8> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <64 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt <64 x i8> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = and <64 x i1> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <64 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <64 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <64 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <64 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = and <64 x i1> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <64 x i1> [[TMP22]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <64 x i1> [[TMP23]] to i64
+; CHECK-NEXT:    store i64 [[TMP24]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP25]]
+;
+  %res = call i64 @llvm.x86.avx512.mask.pcmpgt.b.512(<64 x i8> %a, <64 x i8> %b, i64 %mask)
+  ret i64 %res
+}
+
+declare i64 @llvm.x86.avx512.mask.pcmpgt.b.512(<64 x i8>, <64 x i8>, i64)
+
+define i32 @test_pcmpgt_w(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_pcmpgt_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[A:%.*]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP5:%.*]] = and <32 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or <32 x i16> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[B:%.*]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP9:%.*]] = and <32 x i16> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt <32 x i16> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <32 x i16> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i1> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt <32 x i16> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <32 x i1> [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <32 x i1> [[TMP14]] to i32
+; CHECK-NEXT:    store i32 [[TMP15]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP16]]
+;
+  %res = call i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16> %a, <32 x i16> %b, i32 -1)
+  ret i32 %res
+}
+
+define i32 @test_mask_pcmpgt_w(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_pcmpgt_w(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <32 x i16> [[A:%.*]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <32 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <32 x i16> [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[B:%.*]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP10:%.*]] = and <32 x i16> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ugt <32 x i16> [[TMP6]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt <32 x i16> [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt <32 x i16> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i1> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <32 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = and <32 x i1> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <32 x i1> [[TMP22]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x i1> [[TMP23]] to i32
+; CHECK-NEXT:    store i32 [[TMP24]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP25]]
+;
+  %res = call i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16> %a, <32 x i16> %b, i32 %mask)
+  ret i32 %res
+}
+
+declare i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16>, <32 x i16>, i32)
+
+declare <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_punpckhb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpckhb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_punpckhb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpckhb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_punpcklb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpcklb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_punpcklb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpcklb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_punpckhw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpckhw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP2]], <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]], <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_punpckhw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpckhw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP2]], <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]], <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_punpcklw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_punpcklw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP2]], <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]], <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_punpcklw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_punpcklw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> [[TMP2]], <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]], <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_pmaxs_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxs_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.smax.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pmaxs_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.smax.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmaxs_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.smax.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmaxs_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.smax.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_pmaxu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxu_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.umax.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pmaxu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.umax.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmaxu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaxu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.umax.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmaxu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.umax.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_pmins_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmins_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.smin.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pmins_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.smin.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmins_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmins_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.smin.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmins_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.smin.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_pminu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pminu_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.umin.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pminu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.umin.v64i8(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pminu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pminu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.umin.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pminu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.umin.v32i16(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmovzxb.w.512(<32 x i8>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmovzxb_w_512(<32 x i8> %x0, <32 x i16> %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovzxb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> splat (i8 -1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[X0:%.*]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <32 x i8> [[_MSPROP]] to <32 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <32 x i8> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmovzxb.w.512(<32 x i8> %x0, <32 x i16> %x1, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmovzxb_w_512(<32 x i8> %x0, <32 x i16> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovzxb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> splat (i8 -1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[X0:%.*]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <32 x i8> [[_MSPROP]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <32 x i8> [[TMP4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmovzxb.w.512(<32 x i8> %x0, <32 x i16> %x1, i32 %x2)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_pmovzxb_w_512(<32 x i8> %x0, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovzxb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> splat (i8 -1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[X0:%.*]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = zext <32 x i8> [[_MSPROP]] to <32 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <32 x i8> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmovzxb.w.512(<32 x i8> %x0, <32 x i16> zeroinitializer, i32 %x2)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmovsxb.w.512(<32 x i8>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmovsxb_w_512(<32 x i8> %x0, <32 x i16> %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovsxb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> splat (i8 -1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[X0:%.*]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <32 x i8> [[_MSPROP]] to <32 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <32 x i8> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmovsxb.w.512(<32 x i8> %x0, <32 x i16> %x1, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmovsxb_w_512(<32 x i8> %x0, <32 x i16> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovsxb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> splat (i8 -1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[X0:%.*]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <32 x i8> [[_MSPROP]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <32 x i8> [[TMP4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP1]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmovsxb.w.512(<32 x i8> %x0, <32 x i16> %x1, i32 %x2)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_pmovsxb_w_512(<32 x i8> %x0, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovsxb_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> splat (i8 -1), <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[X0:%.*]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = sext <32 x i8> [[_MSPROP]] to <32 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <32 x i8> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP1]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmovsxb.w.512(<32 x i8> %x0, <32 x i16> zeroinitializer, i32 %x2)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP10]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP11]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <32 x i16> [[TMP12]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP18]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP12]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP19]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psrl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP10]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP17]], <32 x i16> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP11]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP18]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16>, i32, <32 x i16>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_psrl_wi_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrl_wi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP5]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP12]], <32 x i16> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP6]], <32 x i16> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i32 [[X3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <32 x i1> [[TMP21]], <32 x i16> [[TMP18]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <32 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <32 x i16> [[TMP23]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <32 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP20]], <32 x i16> [[TMP25]], <32 x i16> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <32 x i1> [[TMP21]], <32 x i16> [[TMP19]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP27]], <32 x i16> [[TMP15]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]], <32 x i16> [[TMP16]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP28]], <32 x i16> [[_MSPROP_SELECT1]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES4]], <32 x i16> [[TMP26]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES5]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 4, <32 x i16> %x2, i32 -1)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 5, <32 x i16> zeroinitializer, i32 %x3)
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16>  %res, 0
+  %res4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res3, <32 x i16> %res1, 1
+  %res5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res4, <32 x i16> %res2, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res5
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_psra_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psra_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP10]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_psra_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psra_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP11]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <32 x i16> [[TMP12]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP18]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP12]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP19]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_psra_w_512(<32 x i16> %x0, <8 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psra_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP10]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP17]], <32 x i16> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP11]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP18]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16>, i32, <32 x i16>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_psra_wi_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psra_wi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP5]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP12]], <32 x i16> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP6]], <32 x i16> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32 [[X3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <32 x i1> [[TMP18]], <32 x i16> [[TMP15]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <32 x i16> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i16> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP17]], <32 x i16> [[TMP22]], <32 x i16> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <32 x i1> [[TMP18]], <32 x i16> [[TMP16]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <32 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP27]], <32 x i16> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]], <32 x i16> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP28]], <32 x i16> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES4]], <32 x i16> [[TMP26]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES5]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 4, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 5, <32 x i16> %x2, i32 -1)
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16>  %res, 0
+  %res4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res3, <32 x i16> %res1, 1
+  %res5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res4, <32 x i16> %res2, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res5
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_psll_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psll_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP10]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_psll_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psll_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP11]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <32 x i16> [[TMP12]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP18]], <32 x i16> [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP12]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP19]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_psll_w_512(<32 x i16> %x0, <8 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psll_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[TMP1]], <8 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[X0:%.*]], <8 x i16> [[X1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP10]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP17]], <32 x i16> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <32 x i1> [[TMP13]], <32 x i16> [[TMP11]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP18]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16>, i32, <32 x i16>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_psll_wi_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psll_wi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[TMP1]], i32 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[X0:%.*]], i32 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP5]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[TMP6]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or <32 x i16> [[TMP11]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP12]], <32 x i16> [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP8]], <32 x i16> [[TMP6]], <32 x i16> [[X2]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[TMP1]], i32 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[X0]], i32 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32 [[X3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <32 x i1> [[TMP18]], <32 x i16> [[TMP15]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <32 x i16> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <32 x i16> [[TMP20]], [[TMP15]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i16> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP17]], <32 x i16> [[TMP22]], <32 x i16> [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <32 x i1> [[TMP18]], <32 x i16> [[TMP16]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[TMP1]], i32 5)
+; CHECK-NEXT:    [[TMP25:%.*]] = or <32 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[X0]], i32 5)
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[TMP13]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP27]], <32 x i16> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]], <32 x i16> [[TMP23]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP28]], <32 x i16> [[TMP25]], 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES4]], <32 x i16> [[TMP26]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP29]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES5]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 4, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 5, <32 x i16> %x2, i32 -1)
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16>  %res, 0
+  %res4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res3, <32 x i16> %res1, 1
+  %res5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res4, <32 x i16> %res2, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res5
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshuf_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pshuf_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+
+declare <64 x i8> @llvm.x86.avx512.cvtmask2b.512(i64)
+
+define <64 x i8> @test_int_x86_avx512_cvtmask2b_512(i64 %x0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_cvtmask2b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[TMP1]] to <64 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[X0:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <64 x i1> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[VPMOVM2:%.*]] = sext <64 x i1> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[VPMOVM2]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.cvtmask2b.512(i64 %x0)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.cvtmask2w.512(i32)
+
+define <32 x i16> @test_int_x86_avx512_cvtmask2w_512(i32 %x0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_cvtmask2w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[X0:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = sext <32 x i1> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    [[VPMOVM2:%.*]] = sext <32 x i1> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[VPMOVM2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.cvtmask2w.512(i32 %x0)
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_mask_packs_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP9]], <32 x i16> [[TMP14]], <32 x i16> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP8]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP15]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rm_512(<16 x i32> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmb_512(<16 x i32> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmbk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmbk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmbkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmbkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32>, <16 x i32>, <32 x i16>, i32)
+
+define <64 x i8> @test_mask_packs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP7]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <64 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP9]], <64 x i8> [[TMP14]], <64 x i8> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP8]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP15]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <64 x i8> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <64 x i8> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP20]], <64 x i8> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> [[TMP14]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP21]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packs_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <64 x i8> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP14]], <64 x i8> [[TMP19]], <64 x i8> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP13]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP20]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16>, <32 x i16>, <64 x i8>, i64)
+
+
+define <32 x i16> @test_mask_packus_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP9]], <32 x i16> [[TMP14]], <32 x i16> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP8]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP15]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rm_512(<16 x i32> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmb_512(<16 x i32> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmbk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmbk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmbkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmbkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32>, <16 x i32>, <32 x i16>, i32)
+
+define <64 x i8> @test_mask_packus_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP7]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <64 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP9]], <64 x i8> [[TMP14]], <64 x i8> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP8]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP15]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <64 x i8> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <64 x i8> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP20]], <64 x i8> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> [[TMP14]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP21]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_packus_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <64 x i8> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP14]], <64 x i8> [[TMP19]], <64 x i8> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP13]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP20]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16>, <32 x i16>, <64 x i8>, i64)
+
+define i64 @test_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1) nounwind #0 {
+; CHECK-LABEL: @test_cmp_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <64 x i8> [[TMP4]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <64 x i8> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i1> [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i8> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP12]], [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP18:%.*]] = and <64 x i8> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult <64 x i8> [[TMP14]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <64 x i8> [[TMP15]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <64 x i1> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <64 x i1> [[TMP22]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <64 x i1> [[TMP23]] to i64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP10]], [[TMP24]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i64 [[TMP11]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <64 x i8> [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <64 x i8> [[TMP26]], [[TMP1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP32:%.*]] = and <64 x i8> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <64 x i8> [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ule <64 x i8> [[TMP28]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ule <64 x i8> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <64 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp sle <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <64 x i1> [[TMP36]] to i64
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <64 x i1> [[TMP37]] to i64
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[_MSPROP]], [[TMP38]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i64 [[RET1]], [[TMP39]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i64 [[_MSPROP1]], 0
+; CHECK-NEXT:    [[RET3:%.*]] = add i64 [[RET2]], 0
+; CHECK-NEXT:    [[TMP40:%.*]] = xor <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <64 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <64 x i8> [[TMP41]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP44:%.*]] = and <64 x i8> [[TMP43]], [[TMP40]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp eq <64 x i8> [[TMP44]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <64 x i1> [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP3]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <64 x i1> [[TMP46]] to i64
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP2]], [[TMP47]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i64 [[RET3]], [[TMP48]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP50:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP51:%.*]] = and <64 x i8> [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = or <64 x i8> [[TMP49]], [[TMP1]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP54:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP55:%.*]] = and <64 x i8> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <64 x i8> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp uge <64 x i8> [[TMP51]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = icmp uge <64 x i8> [[TMP52]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = xor <64 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = icmp sge <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast <64 x i1> [[TMP59]] to i64
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast <64 x i1> [[TMP60]] to i64
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i64 [[_MSPROP4]], [[TMP61]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i64 [[RET4]], [[TMP62]]
+; CHECK-NEXT:    [[TMP63:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP64:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP65:%.*]] = and <64 x i8> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <64 x i8> [[TMP63]], [[TMP1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP68:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP69:%.*]] = and <64 x i8> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <64 x i8> [[TMP67]], [[TMP2]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp ugt <64 x i8> [[TMP65]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = icmp ugt <64 x i8> [[TMP66]], [[TMP69]]
+; CHECK-NEXT:    [[TMP73:%.*]] = xor <64 x i1> [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp sgt <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast <64 x i1> [[TMP73]] to i64
+; CHECK-NEXT:    [[TMP76:%.*]] = bitcast <64 x i1> [[TMP74]] to i64
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i64 [[_MSPROP5]], [[TMP75]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i64 [[RET5]], [[TMP76]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i64 [[_MSPROP6]], 0
+; CHECK-NEXT:    [[RET7:%.*]] = add i64 [[RET6]], -1
+; CHECK-NEXT:    store i64 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RET7]]
+;
+  %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 -1)
+  %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 -1)
+  %ret1 = add i64 %res0, %res1
+  %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 -1)
+  %ret2 = add i64 %ret1, %res2
+  %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 -1)
+  %ret3 = add i64 %ret2, %res3
+  %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 -1)
+  %ret4 = add i64 %ret3, %res4
+  %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 -1)
+  %ret5 = add i64 %ret4, %res5
+  %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 -1)
+  %ret6 = add i64 %ret5, %res6
+  %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 -1)
+  %ret7 = add i64 %ret6, %res7
+  ret i64 %ret7
+}
+
+define i64 @test_mask_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_cmp_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <64 x i8> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[TMP5]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <64 x i8> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <64 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <64 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <64 x i1> [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <64 x i1> [[TMP18]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP23:%.*]] = and <64 x i8> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <64 x i8> [[TMP21]], [[TMP1]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP27:%.*]] = and <64 x i8> [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = or <64 x i8> [[TMP25]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ult <64 x i8> [[TMP23]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <64 x i8> [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <64 x i1> [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp slt <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP35:%.*]] = and <64 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <64 x i1> [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <64 x i1> [[TMP31]], [[TMP34]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <64 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <64 x i1> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <64 x i1> [[TMP32]], [[TMP34]]
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <64 x i1> [[TMP39]] to i64
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <64 x i1> [[TMP40]] to i64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP19]], [[TMP41]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i64 [[TMP20]], [[TMP42]]
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <64 x i8> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <64 x i8> [[TMP43]], [[TMP1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <64 x i8> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <64 x i8> [[TMP47]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp ule <64 x i8> [[TMP45]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ule <64 x i8> [[TMP46]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <64 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp sle <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP57:%.*]] = and <64 x i1> [[TMP53]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <64 x i1> [[TMP53]], [[TMP56]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <64 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <64 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = and <64 x i1> [[TMP54]], [[TMP56]]
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <64 x i1> [[TMP61]] to i64
+; CHECK-NEXT:    [[TMP64:%.*]] = bitcast <64 x i1> [[TMP62]] to i64
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[_MSPROP]], [[TMP63]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i64 [[RET1]], [[TMP64]]
+; CHECK-NEXT:    [[TMP65:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP67:%.*]] = and <64 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <64 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP69:%.*]] = and <64 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <64 x i1> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP71:%.*]] = or <64 x i1> [[TMP70]], [[TMP69]]
+; CHECK-NEXT:    [[TMP72:%.*]] = and <64 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast <64 x i1> [[TMP71]] to i64
+; CHECK-NEXT:    [[TMP74:%.*]] = bitcast <64 x i1> [[TMP72]] to i64
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i64 [[_MSPROP1]], [[TMP73]]
+; CHECK-NEXT:    [[RET3:%.*]] = add i64 [[RET2]], [[TMP74]]
+; CHECK-NEXT:    [[TMP75:%.*]] = xor <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP76:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <64 x i8> [[TMP76]], zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <64 x i8> [[TMP76]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP79:%.*]] = and <64 x i8> [[TMP78]], [[TMP75]]
+; CHECK-NEXT:    [[TMP80:%.*]] = icmp eq <64 x i8> [[TMP79]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <64 x i1> [[TMP77]], [[TMP80]]
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp ne <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP82:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP83:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP84:%.*]] = and <64 x i1> [[_MSPROP_ICMP3]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <64 x i1> [[TMP81]], [[TMP82]]
+; CHECK-NEXT:    [[TMP86:%.*]] = and <64 x i1> [[_MSPROP_ICMP3]], [[TMP83]]
+; CHECK-NEXT:    [[TMP87:%.*]] = or <64 x i1> [[TMP84]], [[TMP85]]
+; CHECK-NEXT:    [[TMP88:%.*]] = or <64 x i1> [[TMP87]], [[TMP86]]
+; CHECK-NEXT:    [[TMP89:%.*]] = and <64 x i1> [[TMP81]], [[TMP83]]
+; CHECK-NEXT:    [[TMP90:%.*]] = bitcast <64 x i1> [[TMP88]] to i64
+; CHECK-NEXT:    [[TMP91:%.*]] = bitcast <64 x i1> [[TMP89]] to i64
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP2]], [[TMP90]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i64 [[RET3]], [[TMP91]]
+; CHECK-NEXT:    [[TMP92:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP93:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP94:%.*]] = and <64 x i8> [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = or <64 x i8> [[TMP92]], [[TMP1]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP97:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP98:%.*]] = and <64 x i8> [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP99:%.*]] = or <64 x i8> [[TMP96]], [[TMP2]]
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp uge <64 x i8> [[TMP94]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = icmp uge <64 x i8> [[TMP95]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = xor <64 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP103:%.*]] = icmp sge <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP104:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = and <64 x i1> [[TMP102]], [[TMP104]]
+; CHECK-NEXT:    [[TMP107:%.*]] = and <64 x i1> [[TMP103]], [[TMP104]]
+; CHECK-NEXT:    [[TMP108:%.*]] = and <64 x i1> [[TMP102]], [[TMP105]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <64 x i1> [[TMP106]], [[TMP107]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <64 x i1> [[TMP109]], [[TMP108]]
+; CHECK-NEXT:    [[TMP111:%.*]] = and <64 x i1> [[TMP103]], [[TMP105]]
+; CHECK-NEXT:    [[TMP112:%.*]] = bitcast <64 x i1> [[TMP110]] to i64
+; CHECK-NEXT:    [[TMP113:%.*]] = bitcast <64 x i1> [[TMP111]] to i64
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i64 [[_MSPROP4]], [[TMP112]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i64 [[RET4]], [[TMP113]]
+; CHECK-NEXT:    [[TMP114:%.*]] = xor <64 x i8> [[A0]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP115:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP116:%.*]] = and <64 x i8> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = or <64 x i8> [[TMP114]], [[TMP1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = xor <64 x i8> [[A1]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP119:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP120:%.*]] = and <64 x i8> [[TMP118]], [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = or <64 x i8> [[TMP118]], [[TMP2]]
+; CHECK-NEXT:    [[TMP122:%.*]] = icmp ugt <64 x i8> [[TMP116]], [[TMP121]]
+; CHECK-NEXT:    [[TMP123:%.*]] = icmp ugt <64 x i8> [[TMP117]], [[TMP120]]
+; CHECK-NEXT:    [[TMP124:%.*]] = xor <64 x i1> [[TMP122]], [[TMP123]]
+; CHECK-NEXT:    [[TMP125:%.*]] = icmp sgt <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP128:%.*]] = and <64 x i1> [[TMP124]], [[TMP126]]
+; CHECK-NEXT:    [[TMP129:%.*]] = and <64 x i1> [[TMP125]], [[TMP126]]
+; CHECK-NEXT:    [[TMP130:%.*]] = and <64 x i1> [[TMP124]], [[TMP127]]
+; CHECK-NEXT:    [[TMP131:%.*]] = or <64 x i1> [[TMP128]], [[TMP129]]
+; CHECK-NEXT:    [[TMP132:%.*]] = or <64 x i1> [[TMP131]], [[TMP130]]
+; CHECK-NEXT:    [[TMP133:%.*]] = and <64 x i1> [[TMP125]], [[TMP127]]
+; CHECK-NEXT:    [[TMP134:%.*]] = bitcast <64 x i1> [[TMP132]] to i64
+; CHECK-NEXT:    [[TMP135:%.*]] = bitcast <64 x i1> [[TMP133]] to i64
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i64 [[_MSPROP5]], [[TMP134]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i64 [[RET5]], [[TMP135]]
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP138:%.*]] = and <64 x i1> zeroinitializer, [[TMP136]]
+; CHECK-NEXT:    [[TMP139:%.*]] = and <64 x i1> splat (i1 true), [[TMP136]]
+; CHECK-NEXT:    [[TMP140:%.*]] = and <64 x i1> zeroinitializer, [[TMP137]]
+; CHECK-NEXT:    [[TMP141:%.*]] = or <64 x i1> [[TMP138]], [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = or <64 x i1> [[TMP141]], [[TMP140]]
+; CHECK-NEXT:    [[TMP143:%.*]] = and <64 x i1> splat (i1 true), [[TMP137]]
+; CHECK-NEXT:    [[TMP144:%.*]] = bitcast <64 x i1> [[TMP142]] to i64
+; CHECK-NEXT:    [[TMP145:%.*]] = bitcast <64 x i1> [[TMP143]] to i64
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i64 [[_MSPROP6]], [[TMP144]]
+; CHECK-NEXT:    [[RET7:%.*]] = add i64 [[RET6]], [[TMP145]]
+; CHECK-NEXT:    store i64 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RET7]]
+;
+  %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 %mask)
+  %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 %mask)
+  %ret1 = add i64 %res0, %res1
+  %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 %mask)
+  %ret2 = add i64 %ret1, %res2
+  %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 %mask)
+  %ret3 = add i64 %ret2, %res3
+  %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 %mask)
+  %ret4 = add i64 %ret3, %res4
+  %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 %mask)
+  %ret5 = add i64 %ret4, %res5
+  %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 %mask)
+  %ret6 = add i64 %ret5, %res6
+  %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 %mask)
+  %ret7 = add i64 %ret6, %res7
+  ret i64 %ret7
+}
+
+declare i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8>, <64 x i8>, i32, i64) nounwind readnone
+
+define i64 @test_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1) nounwind #0 {
+; CHECK-LABEL: @test_ucmp_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <64 x i8> [[TMP4]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <64 x i8> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i1> [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i8> [[A0]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP16:%.*]] = and <64 x i8> [[A1]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ult <64 x i8> [[TMP13]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult <64 x i8> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <64 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <64 x i1> [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <64 x i1> [[TMP21]] to i64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP10]], [[TMP22]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i64 [[TMP11]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <64 x i8> [[A0]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <64 x i8> [[A1]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ule <64 x i8> [[TMP25]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ule <64 x i8> [[TMP26]], [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = xor <64 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ule <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <64 x i1> [[TMP32]] to i64
+; CHECK-NEXT:    [[TMP35:%.*]] = bitcast <64 x i1> [[TMP33]] to i64
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[_MSPROP]], [[TMP34]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i64 [[RET1]], [[TMP35]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i64 [[_MSPROP1]], 0
+; CHECK-NEXT:    [[RET3:%.*]] = add i64 [[RET2]], 0
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <64 x i8> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = xor <64 x i8> [[TMP37]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP40:%.*]] = and <64 x i8> [[TMP39]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp eq <64 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <64 x i1> [[TMP38]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP3]] to i64
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast <64 x i1> [[TMP42]] to i64
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP2]], [[TMP43]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i64 [[RET3]], [[TMP44]]
+; CHECK-NEXT:    [[TMP45:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP46:%.*]] = and <64 x i8> [[A0]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <64 x i8> [[A1]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp uge <64 x i8> [[TMP46]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp uge <64 x i8> [[TMP47]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <64 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp uge <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast <64 x i1> [[TMP53]] to i64
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <64 x i1> [[TMP54]] to i64
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i64 [[_MSPROP4]], [[TMP55]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i64 [[RET4]], [[TMP56]]
+; CHECK-NEXT:    [[TMP57:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i8> [[A0]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP60:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP61:%.*]] = and <64 x i8> [[A1]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP63:%.*]] = icmp ugt <64 x i8> [[TMP58]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ugt <64 x i8> [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = xor <64 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = icmp ugt <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = bitcast <64 x i1> [[TMP65]] to i64
+; CHECK-NEXT:    [[TMP68:%.*]] = bitcast <64 x i1> [[TMP66]] to i64
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i64 [[_MSPROP5]], [[TMP67]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i64 [[RET5]], [[TMP68]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i64 [[_MSPROP6]], 0
+; CHECK-NEXT:    [[RET7:%.*]] = add i64 [[RET6]], -1
+; CHECK-NEXT:    store i64 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RET7]]
+;
+  %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 -1)
+  %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 -1)
+  %ret1 = add i64 %res0, %res1
+  %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 -1)
+  %ret2 = add i64 %ret1, %res2
+  %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 -1)
+  %ret3 = add i64 %ret2, %res3
+  %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 -1)
+  %ret4 = add i64 %ret3, %res4
+  %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 -1)
+  %ret5 = add i64 %ret4, %res5
+  %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 -1)
+  %ret6 = add i64 %ret5, %res6
+  %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 -1)
+  %ret7 = add i64 %ret6, %res7
+  ret i64 %ret7
+}
+
+define i64 @test_mask_x86_avx512_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_x86_avx512_ucmp_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <64 x i8> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[TMP5]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <64 x i8> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <64 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <64 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <64 x i1> [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <64 x i1> [[TMP18]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP22:%.*]] = and <64 x i8> [[A0]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <64 x i8> [[A1]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ult <64 x i8> [[TMP22]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ult <64 x i8> [[TMP23]], [[TMP25]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <64 x i1> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = and <64 x i1> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP34:%.*]] = and <64 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <64 x i1> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = or <64 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <64 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <64 x i1> [[TMP30]], [[TMP32]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <64 x i1> [[TMP37]] to i64
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <64 x i1> [[TMP38]] to i64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP19]], [[TMP39]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i64 [[TMP20]], [[TMP40]]
+; CHECK-NEXT:    [[TMP41:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP42:%.*]] = and <64 x i8> [[A0]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <64 x i8> [[A1]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp ule <64 x i8> [[TMP42]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp ule <64 x i8> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <64 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp ule <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP52:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP53:%.*]] = and <64 x i1> [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[TMP54:%.*]] = and <64 x i1> [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <64 x i1> [[TMP49]], [[TMP52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <64 x i1> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <64 x i1> [[TMP56]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i1> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP59:%.*]] = bitcast <64 x i1> [[TMP57]] to i64
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <64 x i1> [[TMP58]] to i64
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i64 [[_MSPROP]], [[TMP59]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i64 [[RET1]], [[TMP60]]
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP63:%.*]] = and <64 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = and <64 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = and <64 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <64 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP67:%.*]] = or <64 x i1> [[TMP66]], [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <64 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast <64 x i1> [[TMP67]] to i64
+; CHECK-NEXT:    [[TMP70:%.*]] = bitcast <64 x i1> [[TMP68]] to i64
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i64 [[_MSPROP1]], [[TMP69]]
+; CHECK-NEXT:    [[RET3:%.*]] = add i64 [[RET2]], [[TMP70]]
+; CHECK-NEXT:    [[TMP71:%.*]] = xor <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP72:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp ne <64 x i8> [[TMP72]], zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = xor <64 x i8> [[TMP72]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP75:%.*]] = and <64 x i8> [[TMP74]], [[TMP71]]
+; CHECK-NEXT:    [[TMP76:%.*]] = icmp eq <64 x i8> [[TMP75]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <64 x i1> [[TMP73]], [[TMP76]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP78:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP79:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP80:%.*]] = and <64 x i1> [[_MSPROP_ICMP3]], [[TMP78]]
+; CHECK-NEXT:    [[TMP81:%.*]] = and <64 x i1> [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP82:%.*]] = and <64 x i1> [[_MSPROP_ICMP3]], [[TMP79]]
+; CHECK-NEXT:    [[TMP83:%.*]] = or <64 x i1> [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP84:%.*]] = or <64 x i1> [[TMP83]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <64 x i1> [[TMP77]], [[TMP79]]
+; CHECK-NEXT:    [[TMP86:%.*]] = bitcast <64 x i1> [[TMP84]] to i64
+; CHECK-NEXT:    [[TMP87:%.*]] = bitcast <64 x i1> [[TMP85]] to i64
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i64 [[_MSPROP2]], [[TMP86]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i64 [[RET3]], [[TMP87]]
+; CHECK-NEXT:    [[TMP88:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP89:%.*]] = and <64 x i8> [[A0]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP91:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP92:%.*]] = and <64 x i8> [[A1]], [[TMP91]]
+; CHECK-NEXT:    [[TMP93:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp uge <64 x i8> [[TMP89]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = icmp uge <64 x i8> [[TMP90]], [[TMP92]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <64 x i1> [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp uge <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP98:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP99:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP100:%.*]] = and <64 x i1> [[TMP96]], [[TMP98]]
+; CHECK-NEXT:    [[TMP101:%.*]] = and <64 x i1> [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = and <64 x i1> [[TMP96]], [[TMP99]]
+; CHECK-NEXT:    [[TMP103:%.*]] = or <64 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP104:%.*]] = or <64 x i1> [[TMP103]], [[TMP102]]
+; CHECK-NEXT:    [[TMP105:%.*]] = and <64 x i1> [[TMP97]], [[TMP99]]
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast <64 x i1> [[TMP104]] to i64
+; CHECK-NEXT:    [[TMP107:%.*]] = bitcast <64 x i1> [[TMP105]] to i64
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i64 [[_MSPROP4]], [[TMP106]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i64 [[RET4]], [[TMP107]]
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP109:%.*]] = and <64 x i8> [[A0]], [[TMP108]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <64 x i8> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP111:%.*]] = xor <64 x i8> [[TMP2]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP112:%.*]] = and <64 x i8> [[A1]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = or <64 x i8> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp ugt <64 x i8> [[TMP109]], [[TMP113]]
+; CHECK-NEXT:    [[TMP115:%.*]] = icmp ugt <64 x i8> [[TMP110]], [[TMP112]]
+; CHECK-NEXT:    [[TMP116:%.*]] = xor <64 x i1> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = icmp ugt <64 x i8> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP119:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP120:%.*]] = and <64 x i1> [[TMP116]], [[TMP118]]
+; CHECK-NEXT:    [[TMP121:%.*]] = and <64 x i1> [[TMP117]], [[TMP118]]
+; CHECK-NEXT:    [[TMP122:%.*]] = and <64 x i1> [[TMP116]], [[TMP119]]
+; CHECK-NEXT:    [[TMP123:%.*]] = or <64 x i1> [[TMP120]], [[TMP121]]
+; CHECK-NEXT:    [[TMP124:%.*]] = or <64 x i1> [[TMP123]], [[TMP122]]
+; CHECK-NEXT:    [[TMP125:%.*]] = and <64 x i1> [[TMP117]], [[TMP119]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast <64 x i1> [[TMP124]] to i64
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast <64 x i1> [[TMP125]] to i64
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i64 [[_MSPROP5]], [[TMP126]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i64 [[RET5]], [[TMP127]]
+; CHECK-NEXT:    [[TMP128:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP129:%.*]] = bitcast i64 [[MASK]] to <64 x i1>
+; CHECK-NEXT:    [[TMP130:%.*]] = and <64 x i1> zeroinitializer, [[TMP128]]
+; CHECK-NEXT:    [[TMP131:%.*]] = and <64 x i1> splat (i1 true), [[TMP128]]
+; CHECK-NEXT:    [[TMP132:%.*]] = and <64 x i1> zeroinitializer, [[TMP129]]
+; CHECK-NEXT:    [[TMP133:%.*]] = or <64 x i1> [[TMP130]], [[TMP131]]
+; CHECK-NEXT:    [[TMP134:%.*]] = or <64 x i1> [[TMP133]], [[TMP132]]
+; CHECK-NEXT:    [[TMP135:%.*]] = and <64 x i1> splat (i1 true), [[TMP129]]
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast <64 x i1> [[TMP134]] to i64
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast <64 x i1> [[TMP135]] to i64
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i64 [[_MSPROP6]], [[TMP136]]
+; CHECK-NEXT:    [[RET7:%.*]] = add i64 [[RET6]], [[TMP137]]
+; CHECK-NEXT:    store i64 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RET7]]
+;
+  %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 %mask)
+  %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 %mask)
+  %ret1 = add i64 %res0, %res1
+  %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 %mask)
+  %ret2 = add i64 %ret1, %res2
+  %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 %mask)
+  %ret3 = add i64 %ret2, %res3
+  %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 %mask)
+  %ret4 = add i64 %ret3, %res4
+  %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 %mask)
+  %ret5 = add i64 %ret4, %res5
+  %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 %mask)
+  %ret6 = add i64 %ret5, %res6
+  %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 %mask)
+  %ret7 = add i64 %ret6, %res7
+  ret i64 %ret7
+}
+
+declare i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8>, <64 x i8>, i32, i64) nounwind readnone
+
+define i32 @test_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1) nounwind #0 {
+; CHECK-LABEL: @test_cmp_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <32 x i16> [[TMP4]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i16> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i1> [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i16> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP12]], [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i16> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult <32 x i16> [[TMP14]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <32 x i16> [[TMP15]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <32 x i1> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <32 x i1> [[TMP22]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x i1> [[TMP23]] to i32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP10]], [[TMP24]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i32 [[TMP11]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <32 x i16> [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <32 x i16> [[TMP26]], [[TMP1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP32:%.*]] = and <32 x i16> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = or <32 x i16> [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ule <32 x i16> [[TMP28]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ule <32 x i16> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <32 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp sle <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <32 x i1> [[TMP36]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <32 x i1> [[TMP37]] to i32
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[_MSPROP]], [[TMP38]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i32 [[RET1]], [[TMP39]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[_MSPROP1]], 0
+; CHECK-NEXT:    [[RET3:%.*]] = add i32 [[RET2]], 0
+; CHECK-NEXT:    [[TMP40:%.*]] = xor <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <32 x i16> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <32 x i16> [[TMP41]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP44:%.*]] = and <32 x i16> [[TMP43]], [[TMP40]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp eq <32 x i16> [[TMP44]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <32 x i1> [[TMP42]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP3]] to i32
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <32 x i1> [[TMP46]] to i32
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP2]], [[TMP47]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i32 [[RET3]], [[TMP48]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP50:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP51:%.*]] = and <32 x i16> [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = or <32 x i16> [[TMP49]], [[TMP1]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP54:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP55:%.*]] = and <32 x i16> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <32 x i16> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp uge <32 x i16> [[TMP51]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = icmp uge <32 x i16> [[TMP52]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = xor <32 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = icmp sge <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast <32 x i1> [[TMP59]] to i32
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast <32 x i1> [[TMP60]] to i32
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[TMP61]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i32 [[RET4]], [[TMP62]]
+; CHECK-NEXT:    [[TMP63:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP64:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP65:%.*]] = and <32 x i16> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <32 x i16> [[TMP63]], [[TMP1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP68:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP69:%.*]] = and <32 x i16> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <32 x i16> [[TMP67]], [[TMP2]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp ugt <32 x i16> [[TMP65]], [[TMP70]]
+; CHECK-NEXT:    [[TMP72:%.*]] = icmp ugt <32 x i16> [[TMP66]], [[TMP69]]
+; CHECK-NEXT:    [[TMP73:%.*]] = xor <32 x i1> [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp sgt <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast <32 x i1> [[TMP73]] to i32
+; CHECK-NEXT:    [[TMP76:%.*]] = bitcast <32 x i1> [[TMP74]] to i32
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i32 [[_MSPROP5]], [[TMP75]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i32 [[RET5]], [[TMP76]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i32 [[_MSPROP6]], 0
+; CHECK-NEXT:    [[RET7:%.*]] = add i32 [[RET6]], -1
+; CHECK-NEXT:    store i32 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RET7]]
+;
+  %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 -1)
+  %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 -1)
+  %ret1 = add i32 %res0, %res1
+  %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 -1)
+  %ret2 = add i32 %ret1, %res2
+  %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 -1)
+  %ret3 = add i32 %ret2, %res3
+  %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 -1)
+  %ret4 = add i32 %ret3, %res4
+  %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 -1)
+  %ret5 = add i32 %ret4, %res5
+  %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 -1)
+  %ret6 = add i32 %ret5, %res6
+  %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 -1)
+  %ret7 = add i32 %ret6, %res7
+  ret i32 %ret7
+}
+
+define i32 @test_mask_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_cmp_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <32 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP5]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <32 x i16> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <32 x i1> [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i1> [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP22:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP23:%.*]] = and <32 x i16> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or <32 x i16> [[TMP21]], [[TMP1]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP26:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP27:%.*]] = and <32 x i16> [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = or <32 x i16> [[TMP25]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ult <32 x i16> [[TMP23]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <32 x i16> [[TMP24]], [[TMP27]]
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <32 x i1> [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp slt <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP35:%.*]] = and <32 x i1> [[TMP31]], [[TMP33]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <32 x i1> [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <32 x i1> [[TMP31]], [[TMP34]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <32 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <32 x i1> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <32 x i1> [[TMP32]], [[TMP34]]
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <32 x i1> [[TMP39]] to i32
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <32 x i1> [[TMP40]] to i32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP19]], [[TMP41]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i32 [[TMP20]], [[TMP42]]
+; CHECK-NEXT:    [[TMP43:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <32 x i16> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <32 x i16> [[TMP43]], [[TMP1]]
+; CHECK-NEXT:    [[TMP47:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <32 x i16> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <32 x i16> [[TMP47]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp ule <32 x i16> [[TMP45]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ule <32 x i16> [[TMP46]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <32 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp sle <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP57:%.*]] = and <32 x i1> [[TMP53]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <32 x i1> [[TMP53]], [[TMP56]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <32 x i1> [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <32 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = and <32 x i1> [[TMP54]], [[TMP56]]
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <32 x i1> [[TMP61]] to i32
+; CHECK-NEXT:    [[TMP64:%.*]] = bitcast <32 x i1> [[TMP62]] to i32
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[_MSPROP]], [[TMP63]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i32 [[RET1]], [[TMP64]]
+; CHECK-NEXT:    [[TMP65:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP67:%.*]] = and <32 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <32 x i1> zeroinitializer, [[TMP65]]
+; CHECK-NEXT:    [[TMP69:%.*]] = and <32 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP70:%.*]] = or <32 x i1> [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP71:%.*]] = or <32 x i1> [[TMP70]], [[TMP69]]
+; CHECK-NEXT:    [[TMP72:%.*]] = and <32 x i1> zeroinitializer, [[TMP66]]
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast <32 x i1> [[TMP71]] to i32
+; CHECK-NEXT:    [[TMP74:%.*]] = bitcast <32 x i1> [[TMP72]] to i32
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[_MSPROP1]], [[TMP73]]
+; CHECK-NEXT:    [[RET3:%.*]] = add i32 [[RET2]], [[TMP74]]
+; CHECK-NEXT:    [[TMP75:%.*]] = xor <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP76:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <32 x i16> [[TMP76]], zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <32 x i16> [[TMP76]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP79:%.*]] = and <32 x i16> [[TMP78]], [[TMP75]]
+; CHECK-NEXT:    [[TMP80:%.*]] = icmp eq <32 x i16> [[TMP79]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <32 x i1> [[TMP77]], [[TMP80]]
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp ne <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP82:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP83:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP84:%.*]] = and <32 x i1> [[_MSPROP_ICMP3]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <32 x i1> [[TMP81]], [[TMP82]]
+; CHECK-NEXT:    [[TMP86:%.*]] = and <32 x i1> [[_MSPROP_ICMP3]], [[TMP83]]
+; CHECK-NEXT:    [[TMP87:%.*]] = or <32 x i1> [[TMP84]], [[TMP85]]
+; CHECK-NEXT:    [[TMP88:%.*]] = or <32 x i1> [[TMP87]], [[TMP86]]
+; CHECK-NEXT:    [[TMP89:%.*]] = and <32 x i1> [[TMP81]], [[TMP83]]
+; CHECK-NEXT:    [[TMP90:%.*]] = bitcast <32 x i1> [[TMP88]] to i32
+; CHECK-NEXT:    [[TMP91:%.*]] = bitcast <32 x i1> [[TMP89]] to i32
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP2]], [[TMP90]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i32 [[RET3]], [[TMP91]]
+; CHECK-NEXT:    [[TMP92:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP93:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP94:%.*]] = and <32 x i16> [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = or <32 x i16> [[TMP92]], [[TMP1]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP97:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP98:%.*]] = and <32 x i16> [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP99:%.*]] = or <32 x i16> [[TMP96]], [[TMP2]]
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp uge <32 x i16> [[TMP94]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = icmp uge <32 x i16> [[TMP95]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = xor <32 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP103:%.*]] = icmp sge <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP104:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = and <32 x i1> [[TMP102]], [[TMP104]]
+; CHECK-NEXT:    [[TMP107:%.*]] = and <32 x i1> [[TMP103]], [[TMP104]]
+; CHECK-NEXT:    [[TMP108:%.*]] = and <32 x i1> [[TMP102]], [[TMP105]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i1> [[TMP106]], [[TMP107]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i1> [[TMP109]], [[TMP108]]
+; CHECK-NEXT:    [[TMP111:%.*]] = and <32 x i1> [[TMP103]], [[TMP105]]
+; CHECK-NEXT:    [[TMP112:%.*]] = bitcast <32 x i1> [[TMP110]] to i32
+; CHECK-NEXT:    [[TMP113:%.*]] = bitcast <32 x i1> [[TMP111]] to i32
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[TMP112]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i32 [[RET4]], [[TMP113]]
+; CHECK-NEXT:    [[TMP114:%.*]] = xor <32 x i16> [[A0]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP115:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP116:%.*]] = and <32 x i16> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = or <32 x i16> [[TMP114]], [[TMP1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = xor <32 x i16> [[A1]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP119:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP120:%.*]] = and <32 x i16> [[TMP118]], [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = or <32 x i16> [[TMP118]], [[TMP2]]
+; CHECK-NEXT:    [[TMP122:%.*]] = icmp ugt <32 x i16> [[TMP116]], [[TMP121]]
+; CHECK-NEXT:    [[TMP123:%.*]] = icmp ugt <32 x i16> [[TMP117]], [[TMP120]]
+; CHECK-NEXT:    [[TMP124:%.*]] = xor <32 x i1> [[TMP122]], [[TMP123]]
+; CHECK-NEXT:    [[TMP125:%.*]] = icmp sgt <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP128:%.*]] = and <32 x i1> [[TMP124]], [[TMP126]]
+; CHECK-NEXT:    [[TMP129:%.*]] = and <32 x i1> [[TMP125]], [[TMP126]]
+; CHECK-NEXT:    [[TMP130:%.*]] = and <32 x i1> [[TMP124]], [[TMP127]]
+; CHECK-NEXT:    [[TMP131:%.*]] = or <32 x i1> [[TMP128]], [[TMP129]]
+; CHECK-NEXT:    [[TMP132:%.*]] = or <32 x i1> [[TMP131]], [[TMP130]]
+; CHECK-NEXT:    [[TMP133:%.*]] = and <32 x i1> [[TMP125]], [[TMP127]]
+; CHECK-NEXT:    [[TMP134:%.*]] = bitcast <32 x i1> [[TMP132]] to i32
+; CHECK-NEXT:    [[TMP135:%.*]] = bitcast <32 x i1> [[TMP133]] to i32
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i32 [[_MSPROP5]], [[TMP134]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i32 [[RET5]], [[TMP135]]
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP138:%.*]] = and <32 x i1> zeroinitializer, [[TMP136]]
+; CHECK-NEXT:    [[TMP139:%.*]] = and <32 x i1> splat (i1 true), [[TMP136]]
+; CHECK-NEXT:    [[TMP140:%.*]] = and <32 x i1> zeroinitializer, [[TMP137]]
+; CHECK-NEXT:    [[TMP141:%.*]] = or <32 x i1> [[TMP138]], [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = or <32 x i1> [[TMP141]], [[TMP140]]
+; CHECK-NEXT:    [[TMP143:%.*]] = and <32 x i1> splat (i1 true), [[TMP137]]
+; CHECK-NEXT:    [[TMP144:%.*]] = bitcast <32 x i1> [[TMP142]] to i32
+; CHECK-NEXT:    [[TMP145:%.*]] = bitcast <32 x i1> [[TMP143]] to i32
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i32 [[_MSPROP6]], [[TMP144]]
+; CHECK-NEXT:    [[RET7:%.*]] = add i32 [[RET6]], [[TMP145]]
+; CHECK-NEXT:    store i32 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RET7]]
+;
+  %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 %mask)
+  %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 %mask)
+  %ret1 = add i32 %res0, %res1
+  %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 %mask)
+  %ret2 = add i32 %ret1, %res2
+  %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 %mask)
+  %ret3 = add i32 %ret2, %res3
+  %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 %mask)
+  %ret4 = add i32 %ret3, %res4
+  %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 %mask)
+  %ret5 = add i32 %ret4, %res5
+  %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 %mask)
+  %ret6 = add i32 %ret5, %res6
+  %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 %mask)
+  %ret7 = add i32 %ret6, %res7
+  ret i32 %ret7
+}
+
+declare i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16>, <32 x i16>, i32, i32) nounwind readnone
+
+define i32 @test_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1) nounwind #0 {
+; CHECK-LABEL: @test_ucmp_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <32 x i16> [[TMP4]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i16> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i1> [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i16> [[A0]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP16:%.*]] = and <32 x i16> [[A1]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ult <32 x i16> [[TMP13]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult <32 x i16> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = xor <32 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ult <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <32 x i1> [[TMP20]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <32 x i1> [[TMP21]] to i32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP10]], [[TMP22]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i32 [[TMP11]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <32 x i16> [[A0]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP28:%.*]] = and <32 x i16> [[A1]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ule <32 x i16> [[TMP25]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ule <32 x i16> [[TMP26]], [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = xor <32 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ule <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <32 x i1> [[TMP32]] to i32
+; CHECK-NEXT:    [[TMP35:%.*]] = bitcast <32 x i1> [[TMP33]] to i32
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[_MSPROP]], [[TMP34]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i32 [[RET1]], [[TMP35]]
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[_MSPROP1]], 0
+; CHECK-NEXT:    [[RET3:%.*]] = add i32 [[RET2]], 0
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = xor <32 x i16> [[TMP37]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP40:%.*]] = and <32 x i16> [[TMP39]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp eq <32 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <32 x i1> [[TMP38]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP43:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP3]] to i32
+; CHECK-NEXT:    [[TMP44:%.*]] = bitcast <32 x i1> [[TMP42]] to i32
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP2]], [[TMP43]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i32 [[RET3]], [[TMP44]]
+; CHECK-NEXT:    [[TMP45:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP46:%.*]] = and <32 x i16> [[A0]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP48:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP49:%.*]] = and <32 x i16> [[A1]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp uge <32 x i16> [[TMP46]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp uge <32 x i16> [[TMP47]], [[TMP49]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor <32 x i1> [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp uge <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP55:%.*]] = bitcast <32 x i1> [[TMP53]] to i32
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <32 x i1> [[TMP54]] to i32
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[TMP55]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i32 [[RET4]], [[TMP56]]
+; CHECK-NEXT:    [[TMP57:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i16> [[A0]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP60:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP61:%.*]] = and <32 x i16> [[A1]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP63:%.*]] = icmp ugt <32 x i16> [[TMP58]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ugt <32 x i16> [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = xor <32 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = icmp ugt <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP67:%.*]] = bitcast <32 x i1> [[TMP65]] to i32
+; CHECK-NEXT:    [[TMP68:%.*]] = bitcast <32 x i1> [[TMP66]] to i32
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i32 [[_MSPROP5]], [[TMP67]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i32 [[RET5]], [[TMP68]]
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i32 [[_MSPROP6]], 0
+; CHECK-NEXT:    [[RET7:%.*]] = add i32 [[RET6]], -1
+; CHECK-NEXT:    store i32 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RET7]]
+;
+  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 -1)
+  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 -1)
+  %ret1 = add i32 %res0, %res1
+  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 -1)
+  %ret2 = add i32 %ret1, %res2
+  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 -1)
+  %ret3 = add i32 %ret2, %res3
+  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 -1)
+  %ret4 = add i32 %ret3, %res4
+  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 -1)
+  %ret5 = add i32 %ret4, %res5
+  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 -1)
+  %ret6 = add i32 %ret5, %res6
+  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 -1)
+  %ret7 = add i32 %ret6, %res7
+  ret i32 %ret7
+}
+
+define i32 @test_mask_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_ucmp_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <32 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[TMP5]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = and <32 x i16> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <32 x i1> [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i1> [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP22:%.*]] = and <32 x i16> [[A0]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP25:%.*]] = and <32 x i16> [[A1]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ult <32 x i16> [[TMP22]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ult <32 x i16> [[TMP23]], [[TMP25]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <32 x i1> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP33:%.*]] = and <32 x i1> [[TMP29]], [[TMP31]]
+; CHECK-NEXT:    [[TMP34:%.*]] = and <32 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <32 x i1> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = or <32 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <32 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <32 x i1> [[TMP30]], [[TMP32]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <32 x i1> [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <32 x i1> [[TMP38]] to i32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP19]], [[TMP39]]
+; CHECK-NEXT:    [[RET1:%.*]] = add i32 [[TMP20]], [[TMP40]]
+; CHECK-NEXT:    [[TMP41:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP42:%.*]] = and <32 x i16> [[A0]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP44:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP45:%.*]] = and <32 x i16> [[A1]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp ule <32 x i16> [[TMP42]], [[TMP46]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp ule <32 x i16> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor <32 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = icmp ule <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP52:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP53:%.*]] = and <32 x i1> [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[TMP54:%.*]] = and <32 x i1> [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <32 x i1> [[TMP49]], [[TMP52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = or <32 x i1> [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <32 x i1> [[TMP56]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i1> [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP59:%.*]] = bitcast <32 x i1> [[TMP57]] to i32
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <32 x i1> [[TMP58]] to i32
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or i32 [[_MSPROP]], [[TMP59]]
+; CHECK-NEXT:    [[RET2:%.*]] = add i32 [[RET1]], [[TMP60]]
+; CHECK-NEXT:    [[TMP61:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP62:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP63:%.*]] = and <32 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = and <32 x i1> zeroinitializer, [[TMP61]]
+; CHECK-NEXT:    [[TMP65:%.*]] = and <32 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or <32 x i1> [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP67:%.*]] = or <32 x i1> [[TMP66]], [[TMP65]]
+; CHECK-NEXT:    [[TMP68:%.*]] = and <32 x i1> zeroinitializer, [[TMP62]]
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast <32 x i1> [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP70:%.*]] = bitcast <32 x i1> [[TMP68]] to i32
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or i32 [[_MSPROP1]], [[TMP69]]
+; CHECK-NEXT:    [[RET3:%.*]] = add i32 [[RET2]], [[TMP70]]
+; CHECK-NEXT:    [[TMP71:%.*]] = xor <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP72:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp ne <32 x i16> [[TMP72]], zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = xor <32 x i16> [[TMP72]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP75:%.*]] = and <32 x i16> [[TMP74]], [[TMP71]]
+; CHECK-NEXT:    [[TMP76:%.*]] = icmp eq <32 x i16> [[TMP75]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP3:%.*]] = and <32 x i1> [[TMP73]], [[TMP76]]
+; CHECK-NEXT:    [[TMP77:%.*]] = icmp ne <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP78:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP79:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP80:%.*]] = and <32 x i1> [[_MSPROP_ICMP3]], [[TMP78]]
+; CHECK-NEXT:    [[TMP81:%.*]] = and <32 x i1> [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP82:%.*]] = and <32 x i1> [[_MSPROP_ICMP3]], [[TMP79]]
+; CHECK-NEXT:    [[TMP83:%.*]] = or <32 x i1> [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP84:%.*]] = or <32 x i1> [[TMP83]], [[TMP82]]
+; CHECK-NEXT:    [[TMP85:%.*]] = and <32 x i1> [[TMP77]], [[TMP79]]
+; CHECK-NEXT:    [[TMP86:%.*]] = bitcast <32 x i1> [[TMP84]] to i32
+; CHECK-NEXT:    [[TMP87:%.*]] = bitcast <32 x i1> [[TMP85]] to i32
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or i32 [[_MSPROP2]], [[TMP86]]
+; CHECK-NEXT:    [[RET4:%.*]] = add i32 [[RET3]], [[TMP87]]
+; CHECK-NEXT:    [[TMP88:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP89:%.*]] = and <32 x i16> [[A0]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP91:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP92:%.*]] = and <32 x i16> [[A1]], [[TMP91]]
+; CHECK-NEXT:    [[TMP93:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp uge <32 x i16> [[TMP89]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = icmp uge <32 x i16> [[TMP90]], [[TMP92]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor <32 x i1> [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp uge <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP98:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP99:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP100:%.*]] = and <32 x i1> [[TMP96]], [[TMP98]]
+; CHECK-NEXT:    [[TMP101:%.*]] = and <32 x i1> [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP102:%.*]] = and <32 x i1> [[TMP96]], [[TMP99]]
+; CHECK-NEXT:    [[TMP103:%.*]] = or <32 x i1> [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP104:%.*]] = or <32 x i1> [[TMP103]], [[TMP102]]
+; CHECK-NEXT:    [[TMP105:%.*]] = and <32 x i1> [[TMP97]], [[TMP99]]
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast <32 x i1> [[TMP104]] to i32
+; CHECK-NEXT:    [[TMP107:%.*]] = bitcast <32 x i1> [[TMP105]] to i32
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or i32 [[_MSPROP4]], [[TMP106]]
+; CHECK-NEXT:    [[RET5:%.*]] = add i32 [[RET4]], [[TMP107]]
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP109:%.*]] = and <32 x i16> [[A0]], [[TMP108]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP111:%.*]] = xor <32 x i16> [[TMP2]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP112:%.*]] = and <32 x i16> [[A1]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = or <32 x i16> [[A1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp ugt <32 x i16> [[TMP109]], [[TMP113]]
+; CHECK-NEXT:    [[TMP115:%.*]] = icmp ugt <32 x i16> [[TMP110]], [[TMP112]]
+; CHECK-NEXT:    [[TMP116:%.*]] = xor <32 x i1> [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP117:%.*]] = icmp ugt <32 x i16> [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP118:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP119:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP120:%.*]] = and <32 x i1> [[TMP116]], [[TMP118]]
+; CHECK-NEXT:    [[TMP121:%.*]] = and <32 x i1> [[TMP117]], [[TMP118]]
+; CHECK-NEXT:    [[TMP122:%.*]] = and <32 x i1> [[TMP116]], [[TMP119]]
+; CHECK-NEXT:    [[TMP123:%.*]] = or <32 x i1> [[TMP120]], [[TMP121]]
+; CHECK-NEXT:    [[TMP124:%.*]] = or <32 x i1> [[TMP123]], [[TMP122]]
+; CHECK-NEXT:    [[TMP125:%.*]] = and <32 x i1> [[TMP117]], [[TMP119]]
+; CHECK-NEXT:    [[TMP126:%.*]] = bitcast <32 x i1> [[TMP124]] to i32
+; CHECK-NEXT:    [[TMP127:%.*]] = bitcast <32 x i1> [[TMP125]] to i32
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = or i32 [[_MSPROP5]], [[TMP126]]
+; CHECK-NEXT:    [[RET6:%.*]] = add i32 [[RET5]], [[TMP127]]
+; CHECK-NEXT:    [[TMP128:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP129:%.*]] = bitcast i32 [[MASK]] to <32 x i1>
+; CHECK-NEXT:    [[TMP130:%.*]] = and <32 x i1> zeroinitializer, [[TMP128]]
+; CHECK-NEXT:    [[TMP131:%.*]] = and <32 x i1> splat (i1 true), [[TMP128]]
+; CHECK-NEXT:    [[TMP132:%.*]] = and <32 x i1> zeroinitializer, [[TMP129]]
+; CHECK-NEXT:    [[TMP133:%.*]] = or <32 x i1> [[TMP130]], [[TMP131]]
+; CHECK-NEXT:    [[TMP134:%.*]] = or <32 x i1> [[TMP133]], [[TMP132]]
+; CHECK-NEXT:    [[TMP135:%.*]] = and <32 x i1> splat (i1 true), [[TMP129]]
+; CHECK-NEXT:    [[TMP136:%.*]] = bitcast <32 x i1> [[TMP134]] to i32
+; CHECK-NEXT:    [[TMP137:%.*]] = bitcast <32 x i1> [[TMP135]] to i32
+; CHECK-NEXT:    [[_MSPROP7:%.*]] = or i32 [[_MSPROP6]], [[TMP136]]
+; CHECK-NEXT:    [[RET7:%.*]] = add i32 [[RET6]], [[TMP137]]
+; CHECK-NEXT:    store i32 [[_MSPROP7]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RET7]]
+;
+  %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 %mask)
+  %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 %mask)
+  %ret1 = add i32 %res0, %res1
+  %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 %mask)
+  %ret2 = add i32 %ret1, %res2
+  %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 %mask)
+  %ret3 = add i32 %ret2, %res3
+  %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 %mask)
+  %ret4 = add i32 %ret3, %res4
+  %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 %mask)
+  %ret5 = add i32 %ret4, %res5
+  %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 %mask)
+  %ret6 = add i32 %ret5, %res6
+  %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 %mask)
+  %ret7 = add i32 %ret6, %res7
+  ret i32 %ret7
+}
+
+declare i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16>, <32 x i16>, i32, i32) nounwind readnone
+
+
+declare <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @mm512_avg_epu8(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @mm512_avg_epu8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @mm512_mask_avg_epu8(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) nounwind #0 {
+; CHECK-LABEL: @mm512_mask_avg_epu8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  ret <64 x i8> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @mm512_avg_epu16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @mm512_avg_epu16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @mm512_mask_avg_epu16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @mm512_mask_avg_epu16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pabs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.abs.v32i16(<32 x i16> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    store <32 x i16> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.abs.v32i16(<32 x i16> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP1]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
+  ret <32 x i16> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_int_x86_avx512_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pabs_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <64 x i8> @llvm.abs.v64i8(<64 x i8> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    store <64 x i8> [[TMP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP2]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <64 x i8> @llvm.abs.v64i8(<64 x i8> [[X0:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP2]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[X2:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP1]], <64 x i8> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP10]], <64 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP4]], <64 x i8> [[X1]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP11]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
+  ret <64 x i8> %res
+}
+
+declare i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8>, <64 x i8>, i64)
+
+define i64 @test_int_x86_avx512_ptestm_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_ptestm_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <64 x i8> [[X0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <64 x i8> [[TMP1]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <64 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <64 x i8> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <64 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP11]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i8> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i64 [[X2:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <64 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <64 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <64 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <64 x i1> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <64 x i1> [[TMP23]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <64 x i1> [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = and <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <64 x i8> [[X0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and <64 x i8> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <64 x i8> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <64 x i8> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <64 x i8> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = or <64 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <64 x i8> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <64 x i8> [[TMP34]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP37:%.*]] = and <64 x i8> [[TMP36]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq <64 x i8> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <64 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp ne <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP1]] to i64
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <64 x i1> [[TMP39]] to i64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP25]], [[TMP40]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i64 [[TMP26]], [[TMP41]]
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES2]]
+;
+  %res = call i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
+  %res1 = call i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64-1)
+  %res2 = add i64 %res, %res1
+  ret i64 %res2
+}
+
+declare i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16>, <32 x i16>, i32)
+
+define i32 @test_int_x86_avx512_ptestm_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_ptestm_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <32 x i16> [[X0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <32 x i16> [[TMP1]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <32 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <32 x i16> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <32 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <32 x i1> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x i1> [[TMP23]] to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <32 x i1> [[TMP24]] to i32
+; CHECK-NEXT:    [[TMP27:%.*]] = and <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <32 x i16> [[X0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and <32 x i16> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <32 x i16> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <32 x i16> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <32 x i16> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = or <32 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i16> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <32 x i16> [[TMP34]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP37:%.*]] = and <32 x i16> [[TMP36]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <32 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp ne <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP1]] to i32
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <32 x i1> [[TMP39]] to i32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP25]], [[TMP40]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[TMP26]], [[TMP41]]
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %res = call i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
+  %res1 = call i32 @llvm.x86.avx512.ptestm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32-1)
+  %res2 = add i32 %res, %res1
+  ret i32 %res2
+}
+
+declare i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8>, <64 x i8>, i64 %x2)
+
+define i64 @test_int_x86_avx512_ptestnm_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_ptestnm_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <64 x i8> [[X0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <64 x i8> [[TMP1]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <64 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <64 x i8> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <64 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP11]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i8> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i64 [[X2:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <64 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <64 x i1> [[_MSPROP_ICMP]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <64 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <64 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <64 x i1> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <64 x i1> [[TMP23]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <64 x i1> [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = and <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <64 x i8> [[X0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and <64 x i8> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <64 x i8> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <64 x i8> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <64 x i8> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = or <64 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <64 x i8> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <64 x i8> [[TMP34]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP37:%.*]] = and <64 x i8> [[TMP36]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq <64 x i8> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <64 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP1]] to i64
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <64 x i1> [[TMP39]] to i64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i64 [[TMP25]], [[TMP40]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i64 [[TMP26]], [[TMP41]]
+; CHECK-NEXT:    store i64 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[RES2]]
+;
+  %res = call i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
+  %res1 = call i64 @llvm.x86.avx512.ptestnm.b.512(<64 x i8> %x0, <64 x i8> %x1, i64-1)
+  %res2 = add i64 %res, %res1
+  ret i64 %res2
+}
+
+declare i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16>, <32 x i16>, i32 %x2)
+
+define i32 @test_int_x86_avx512_ptestnm_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_ptestnm_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = and <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <32 x i16> [[X0:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and <32 x i16> [[TMP1]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <32 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <32 x i16> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <32 x i1> [[_MSPROP_ICMP]], [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <32 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = and <32 x i1> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <32 x i1> [[TMP23]] to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <32 x i1> [[TMP24]] to i32
+; CHECK-NEXT:    [[TMP27:%.*]] = and <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = and <32 x i16> [[X0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP29:%.*]] = and <32 x i16> [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[TMP30:%.*]] = or <32 x i16> [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = or <32 x i16> [[TMP30]], [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = and <32 x i16> [[X0]], [[X1]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = or <32 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i16> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = xor <32 x i16> [[TMP34]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP37:%.*]] = and <32 x i16> [[TMP36]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <32 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP1]] to i32
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <32 x i1> [[TMP39]] to i32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or i32 [[TMP25]], [[TMP40]]
+; CHECK-NEXT:    [[RES2:%.*]] = add i32 [[TMP26]], [[TMP41]]
+; CHECK-NEXT:    store i32 [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %res = call i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
+  %res1 = call i32 @llvm.x86.avx512.ptestnm.w.512(<32 x i16> %x0, <32 x i16> %x1, i32-1)
+  %res2 = add i32 %res, %res1
+  ret i32 %res2
+}
+
+declare i64 @llvm.x86.avx512.cvtb2mask.512(<64 x i8>)
+
+define i64 @test_int_x86_avx512_cvtb2mask_512(<64 x i8> %x0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_cvtb2mask_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <64 x i8> [[X0:%.*]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[TMP1]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP4:%.*]] = and <64 x i8> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <64 x i8> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult <64 x i8> [[TMP4]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult <64 x i8> [[TMP5]], splat (i8 -128)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i1> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt <64 x i8> [[X0]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i1> [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i1> [[TMP9]] to i64
+; CHECK-NEXT:    store i64 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP11]]
+;
+  %res = call i64 @llvm.x86.avx512.cvtb2mask.512(<64 x i8> %x0)
+  ret i64 %res
+}
+
+declare i32 @llvm.x86.avx512.cvtw2mask.512(<32 x i16>)
+
+define i32 @test_int_x86_avx512_cvtw2mask_512(<32 x i16> %x0) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_cvtw2mask_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <32 x i16> [[X0:%.*]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[TMP1]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP4:%.*]] = and <32 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult <32 x i16> [[TMP4]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult <32 x i16> [[TMP5]], splat (i16 -32768)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i1> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt <32 x i16> [[X0]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i1> [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i1> [[TMP9]] to i32
+; CHECK-NEXT:    store i32 [[TMP10]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  %res = call i32 @llvm.x86.avx512.cvtw2mask.512(<32 x i16> %x0)
+  ret i32 %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmulhu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmulh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmulhr_sw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhr_sw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8>, <64 x i8>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaddubs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaddubs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16>, <32 x i16>, <16 x i32>, i16)
+
+define <16 x i32> @test_int_x86_avx512_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaddw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaddw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3)
+  ret <16 x i32> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_permvar_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_permvar_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_permvar_hi_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i16 [[TMP4]], 31
+; CHECK-NEXT:    [[TMP6:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i16 [[TMP7]], 31
+; CHECK-NEXT:    [[TMP9:%.*]] = or i16 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 31
+; CHECK-NEXT:    [[TMP12:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i16 [[TMP13]], 31
+; CHECK-NEXT:    [[TMP15:%.*]] = or i16 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 31
+; CHECK-NEXT:    [[TMP18:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i16 [[TMP19]], 31
+; CHECK-NEXT:    [[TMP21:%.*]] = or i16 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 31
+; CHECK-NEXT:    [[TMP24:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i16 [[TMP25]], 31
+; CHECK-NEXT:    [[TMP27:%.*]] = or i16 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP29:%.*]] = and i16 [[TMP28]], 31
+; CHECK-NEXT:    [[TMP30:%.*]] = or i16 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP32:%.*]] = and i16 [[TMP31]], 31
+; CHECK-NEXT:    [[TMP33:%.*]] = or i16 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP35:%.*]] = and i16 [[TMP34]], 31
+; CHECK-NEXT:    [[TMP36:%.*]] = or i16 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP38:%.*]] = and i16 [[TMP37]], 31
+; CHECK-NEXT:    [[TMP39:%.*]] = or i16 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP41:%.*]] = and i16 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP42:%.*]] = or i16 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP44:%.*]] = and i16 [[TMP43]], 31
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP47:%.*]] = and i16 [[TMP46]], 31
+; CHECK-NEXT:    [[TMP48:%.*]] = or i16 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i16 [[TMP49]], 31
+; CHECK-NEXT:    [[TMP51:%.*]] = or i16 [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP53:%.*]] = and i16 [[TMP52]], 31
+; CHECK-NEXT:    [[TMP54:%.*]] = or i16 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP56:%.*]] = and i16 [[TMP55]], 31
+; CHECK-NEXT:    [[TMP57:%.*]] = or i16 [[TMP55]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP59:%.*]] = and i16 [[TMP58]], 31
+; CHECK-NEXT:    [[TMP60:%.*]] = or i16 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP62:%.*]] = and i16 [[TMP61]], 31
+; CHECK-NEXT:    [[TMP63:%.*]] = or i16 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP65:%.*]] = and i16 [[TMP64]], 31
+; CHECK-NEXT:    [[TMP66:%.*]] = or i16 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP68:%.*]] = and i16 [[TMP67]], 31
+; CHECK-NEXT:    [[TMP69:%.*]] = or i16 [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP71:%.*]] = and i16 [[TMP70]], 31
+; CHECK-NEXT:    [[TMP72:%.*]] = or i16 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP74:%.*]] = and i16 [[TMP73]], 31
+; CHECK-NEXT:    [[TMP75:%.*]] = or i16 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP77:%.*]] = and i16 [[TMP76]], 31
+; CHECK-NEXT:    [[TMP78:%.*]] = or i16 [[TMP76]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP80:%.*]] = and i16 [[TMP79]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = or i16 [[TMP79]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP83:%.*]] = and i16 [[TMP82]], 31
+; CHECK-NEXT:    [[TMP84:%.*]] = or i16 [[TMP82]], [[TMP83]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP86:%.*]] = and i16 [[TMP85]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = or i16 [[TMP85]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP89:%.*]] = and i16 [[TMP88]], 31
+; CHECK-NEXT:    [[TMP90:%.*]] = or i16 [[TMP88]], [[TMP89]]
+; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP92:%.*]] = and i16 [[TMP91]], 31
+; CHECK-NEXT:    [[TMP93:%.*]] = or i16 [[TMP91]], [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP95:%.*]] = and i16 [[TMP94]], 31
+; CHECK-NEXT:    [[TMP96:%.*]] = or i16 [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP98:%.*]] = and i16 [[TMP97]], 31
+; CHECK-NEXT:    [[TMP99:%.*]] = or i16 [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP100:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X0:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP99]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP101:%.*]], label [[TMP102:%.*]], !prof [[PROF1]]
+; CHECK:       101:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       102:
+; CHECK-NEXT:    [[TMP103:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X1:%.*]], <32 x i16> [[X0]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[TMP100]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP103]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i16 [[TMP5]], 31
+; CHECK-NEXT:    [[TMP7:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i16 [[TMP8]], 31
+; CHECK-NEXT:    [[TMP10:%.*]] = or i16 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 31
+; CHECK-NEXT:    [[TMP13:%.*]] = or i16 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP16:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i16 [[TMP17]], 31
+; CHECK-NEXT:    [[TMP19:%.*]] = or i16 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP22:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i16 [[TMP23]], 31
+; CHECK-NEXT:    [[TMP25:%.*]] = or i16 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i16 [[TMP29]], 31
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP34:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i16 [[TMP35]], 31
+; CHECK-NEXT:    [[TMP37:%.*]] = or i16 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP40:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i16 [[TMP41]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], 31
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP54:%.*]] = and i16 [[TMP53]], 31
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP58:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP60:%.*]] = and i16 [[TMP59]], 31
+; CHECK-NEXT:    [[TMP61:%.*]] = or i16 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP64:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP66:%.*]] = and i16 [[TMP65]], 31
+; CHECK-NEXT:    [[TMP67:%.*]] = or i16 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP70:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP72:%.*]] = and i16 [[TMP71]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP75:%.*]] = and i16 [[TMP74]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP78:%.*]] = and i16 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP81:%.*]] = and i16 [[TMP80]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP84:%.*]] = and i16 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP83]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP87:%.*]] = and i16 [[TMP86]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP90:%.*]] = and i16 [[TMP89]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP93:%.*]] = and i16 [[TMP92]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP96:%.*]] = and i16 [[TMP95]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP95]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP99:%.*]] = and i16 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X0:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP100]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP102:%.*]], label [[TMP103:%.*]], !prof [[PROF1]]
+; CHECK:       102:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       103:
+; CHECK-NEXT:    [[TMP104:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X1:%.*]], <32 x i16> [[X0]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP107:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP101]], <32 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP104]], [[X1]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i16> [[TMP108]], [[TMP101]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[TMP109]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP105]], <32 x i16> [[TMP110]], <32 x i16> [[TMP107]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP104]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP111]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_maskz_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i16 [[TMP5]], 31
+; CHECK-NEXT:    [[TMP7:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i16 [[TMP8]], 31
+; CHECK-NEXT:    [[TMP10:%.*]] = or i16 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 31
+; CHECK-NEXT:    [[TMP13:%.*]] = or i16 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP16:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i16 [[TMP17]], 31
+; CHECK-NEXT:    [[TMP19:%.*]] = or i16 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP22:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i16 [[TMP23]], 31
+; CHECK-NEXT:    [[TMP25:%.*]] = or i16 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i16 [[TMP29]], 31
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP34:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i16 [[TMP35]], 31
+; CHECK-NEXT:    [[TMP37:%.*]] = or i16 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP40:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i16 [[TMP41]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], 31
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP54:%.*]] = and i16 [[TMP53]], 31
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP58:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP60:%.*]] = and i16 [[TMP59]], 31
+; CHECK-NEXT:    [[TMP61:%.*]] = or i16 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP64:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP66:%.*]] = and i16 [[TMP65]], 31
+; CHECK-NEXT:    [[TMP67:%.*]] = or i16 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP70:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP72:%.*]] = and i16 [[TMP71]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP75:%.*]] = and i16 [[TMP74]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP78:%.*]] = and i16 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP81:%.*]] = and i16 [[TMP80]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP84:%.*]] = and i16 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP83]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP87:%.*]] = and i16 [[TMP86]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP90:%.*]] = and i16 [[TMP89]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP93:%.*]] = and i16 [[TMP92]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP96:%.*]] = and i16 [[TMP95]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP95]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP99:%.*]] = and i16 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X0:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP100]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP102:%.*]], label [[TMP103:%.*]], !prof [[PROF1]]
+; CHECK:       102:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       103:
+; CHECK-NEXT:    [[TMP104:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X1:%.*]], <32 x i16> [[X0]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP107:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP101]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP104]], zeroinitializer
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i16> [[TMP108]], [[TMP101]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[TMP109]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP105]], <32 x i16> [[TMP110]], <32 x i16> [[TMP107]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP104]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP111]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_vpermi2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i16 [[TMP4]], 31
+; CHECK-NEXT:    [[TMP6:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i16 [[TMP7]], 31
+; CHECK-NEXT:    [[TMP9:%.*]] = or i16 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 31
+; CHECK-NEXT:    [[TMP12:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i16 [[TMP13]], 31
+; CHECK-NEXT:    [[TMP15:%.*]] = or i16 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 31
+; CHECK-NEXT:    [[TMP18:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i16 [[TMP19]], 31
+; CHECK-NEXT:    [[TMP21:%.*]] = or i16 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 31
+; CHECK-NEXT:    [[TMP24:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i16 [[TMP25]], 31
+; CHECK-NEXT:    [[TMP27:%.*]] = or i16 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP29:%.*]] = and i16 [[TMP28]], 31
+; CHECK-NEXT:    [[TMP30:%.*]] = or i16 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP32:%.*]] = and i16 [[TMP31]], 31
+; CHECK-NEXT:    [[TMP33:%.*]] = or i16 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP35:%.*]] = and i16 [[TMP34]], 31
+; CHECK-NEXT:    [[TMP36:%.*]] = or i16 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP38:%.*]] = and i16 [[TMP37]], 31
+; CHECK-NEXT:    [[TMP39:%.*]] = or i16 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP41:%.*]] = and i16 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP42:%.*]] = or i16 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP44:%.*]] = and i16 [[TMP43]], 31
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP47:%.*]] = and i16 [[TMP46]], 31
+; CHECK-NEXT:    [[TMP48:%.*]] = or i16 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i16 [[TMP49]], 31
+; CHECK-NEXT:    [[TMP51:%.*]] = or i16 [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP53:%.*]] = and i16 [[TMP52]], 31
+; CHECK-NEXT:    [[TMP54:%.*]] = or i16 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP56:%.*]] = and i16 [[TMP55]], 31
+; CHECK-NEXT:    [[TMP57:%.*]] = or i16 [[TMP55]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP59:%.*]] = and i16 [[TMP58]], 31
+; CHECK-NEXT:    [[TMP60:%.*]] = or i16 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP62:%.*]] = and i16 [[TMP61]], 31
+; CHECK-NEXT:    [[TMP63:%.*]] = or i16 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP65:%.*]] = and i16 [[TMP64]], 31
+; CHECK-NEXT:    [[TMP66:%.*]] = or i16 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP68:%.*]] = and i16 [[TMP67]], 31
+; CHECK-NEXT:    [[TMP69:%.*]] = or i16 [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP71:%.*]] = and i16 [[TMP70]], 31
+; CHECK-NEXT:    [[TMP72:%.*]] = or i16 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP74:%.*]] = and i16 [[TMP73]], 31
+; CHECK-NEXT:    [[TMP75:%.*]] = or i16 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP77:%.*]] = and i16 [[TMP76]], 31
+; CHECK-NEXT:    [[TMP78:%.*]] = or i16 [[TMP76]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP80:%.*]] = and i16 [[TMP79]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = or i16 [[TMP79]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP83:%.*]] = and i16 [[TMP82]], 31
+; CHECK-NEXT:    [[TMP84:%.*]] = or i16 [[TMP82]], [[TMP83]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP86:%.*]] = and i16 [[TMP85]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = or i16 [[TMP85]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP89:%.*]] = and i16 [[TMP88]], 31
+; CHECK-NEXT:    [[TMP90:%.*]] = or i16 [[TMP88]], [[TMP89]]
+; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP92:%.*]] = and i16 [[TMP91]], 31
+; CHECK-NEXT:    [[TMP93:%.*]] = or i16 [[TMP91]], [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP95:%.*]] = and i16 [[TMP94]], 31
+; CHECK-NEXT:    [[TMP96:%.*]] = or i16 [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP98:%.*]] = and i16 [[TMP97]], 31
+; CHECK-NEXT:    [[TMP99:%.*]] = or i16 [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP100:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X1:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP99]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP101:%.*]], label [[TMP102:%.*]], !prof [[PROF1]]
+; CHECK:       101:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       102:
+; CHECK-NEXT:    [[TMP103:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[TMP100]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP103]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_vpermi2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i16 [[TMP5]], 31
+; CHECK-NEXT:    [[TMP7:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i16 [[TMP8]], 31
+; CHECK-NEXT:    [[TMP10:%.*]] = or i16 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 31
+; CHECK-NEXT:    [[TMP13:%.*]] = or i16 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP16:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i16 [[TMP17]], 31
+; CHECK-NEXT:    [[TMP19:%.*]] = or i16 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP22:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i16 [[TMP23]], 31
+; CHECK-NEXT:    [[TMP25:%.*]] = or i16 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i16 [[TMP29]], 31
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP34:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i16 [[TMP35]], 31
+; CHECK-NEXT:    [[TMP37:%.*]] = or i16 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP40:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i16 [[TMP41]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], 31
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP54:%.*]] = and i16 [[TMP53]], 31
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP58:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP60:%.*]] = and i16 [[TMP59]], 31
+; CHECK-NEXT:    [[TMP61:%.*]] = or i16 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP64:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP66:%.*]] = and i16 [[TMP65]], 31
+; CHECK-NEXT:    [[TMP67:%.*]] = or i16 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP70:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP72:%.*]] = and i16 [[TMP71]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP75:%.*]] = and i16 [[TMP74]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP78:%.*]] = and i16 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP81:%.*]] = and i16 [[TMP80]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP84:%.*]] = and i16 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP83]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP87:%.*]] = and i16 [[TMP86]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP90:%.*]] = and i16 [[TMP89]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP93:%.*]] = and i16 [[TMP92]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP96:%.*]] = and i16 [[TMP95]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP95]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP99:%.*]] = and i16 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X1:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP100]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP102:%.*]], label [[TMP103:%.*]], !prof [[PROF1]]
+; CHECK:       102:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       103:
+; CHECK-NEXT:    [[TMP104:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP107:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP101]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP104]], [[X1]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i16> [[TMP108]], [[TMP101]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[TMP109]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP105]], <32 x i16> [[TMP110]], <32 x i16> [[TMP107]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP104]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP111]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8>, <64 x i8>, i32, <32 x i16>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_dbpsadbw_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x3, i32 %x4) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_dbpsadbw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], i32 2)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[X4:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[X3]]
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       20:
+; CHECK-NEXT:    [[TMP21:%.*]] = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i32 3)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i32 [[X4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[TMP23]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <32 x i16> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <32 x i16> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP22]], <32 x i16> [[TMP27]], <32 x i16> [[TMP24]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select <32 x i1> [[TMP23]], <32 x i16> [[TMP21]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP29]], 0
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP30]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    br i1 [[_MSOR8]], label [[TMP31:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       31:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       32:
+; CHECK-NEXT:    [[TMP33:%.*]] = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i32 4)
+; CHECK-NEXT:    [[TMP34:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[TMP16]], 0
+; CHECK-NEXT:    [[TMP35:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP34]], <32 x i16> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]], <32 x i16> [[TMP28]], 1
+; CHECK-NEXT:    [[TMP36:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP35]], <32 x i16> zeroinitializer, 2
+; CHECK-NEXT:    [[RES5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES4]], <32 x i16> [[TMP33]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP36]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES5]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> %x3, i32 %x4)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 3, <32 x i16> zeroinitializer, i32 %x4)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 4, <32 x i16> %x3, i32 -1)
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16>  %res, 0
+  %res4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res3, <32 x i16> %res1, 1
+  %res5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res4, <32 x i16> %res2, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res5
+}
+
+define <32 x i16> @test_mask_adds_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP8]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rmk_512(<32 x i16> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP10]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP17]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epu16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_mask_subs_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP8]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rmk_512(<32 x i16> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP10]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP17]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epu16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <64 x i8> @test_mask_adds_epu8_rr_512(<64 x i8> %a, <64 x i8> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu8_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epu8_rrk_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu8_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epu8_rrkz_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu8_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP10]], <64 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP4]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP11]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epu8_rm_512(<64 x i8> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu8_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP8]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epu8_rmk_512(<64 x i8> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu8_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <64 x i8> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i8> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP16]], <64 x i8> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[TMP10]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP17]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epu8_rmkz_512(<64 x i8> %a, ptr %ptr_b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epu8_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_mask_subs_epu8_rr_512(<64 x i8> %a, <64 x i8> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu8_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epu8_rrk_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu8_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epu8_rrkz_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu8_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP10]], <64 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP4]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP11]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epu8_rm_512(<64 x i8> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu8_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP8]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epu8_rmk_512(<64 x i8> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu8_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <64 x i8> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i8> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP16]], <64 x i8> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[TMP10]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP17]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epu8_rmkz_512(<64 x i8> %a, ptr %ptr_b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epu8_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <32 x i16> @test_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_adds_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_adds_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_adds_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_adds_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_adds_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP8]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_adds_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_adds_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP10]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP17]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_adds_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_adds_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_mask_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP8]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP10]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP17]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_subs_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_subs_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_subs_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_subs_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_subs_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP8]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_subs_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_subs_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP10]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP17]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_subs_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_subs_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_mask_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP8]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP10]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP17]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i32 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <64 x i8> @test_mask_adds_epi8_rr_512(<64 x i8> %a, <64 x i8> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi8_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epi8_rrk_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi8_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epi8_rrkz_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi8_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP10]], <64 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP4]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP11]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epi8_rm_512(<64 x i8> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi8_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP8]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epi8_rmk_512(<64 x i8> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi8_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <64 x i8> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i8> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP16]], <64 x i8> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[TMP10]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP17]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_adds_epi8_rmkz_512(<64 x i8> %a, ptr %ptr_b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_adds_epi8_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8> @test_mask_subs_epi8_rr_512(<64 x i8> %a, <64 x i8> %b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi8_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epi8_rrk_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi8_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epi8_rrkz_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi8_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP10]], <64 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP4]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP11]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epi8_rm_512(<64 x i8> %a, ptr %ptr_b) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi8_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP8]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epi8_rmk_512(<64 x i8> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi8_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <64 x i8> [[TMP10]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i8> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP16]], <64 x i8> [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP12]], <64 x i8> [[TMP10]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP17]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_subs_epi8_rmkz_512(<64 x i8> %a, ptr %ptr_b, i64 %mask) nounwind #0 {
+; CHECK-LABEL: @test_mask_subs_epi8_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <64 x i8>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <64 x i8>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> [[A:%.*]], <64 x i8> [[B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %b = load <64 x i8>, ptr %ptr_b
+  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+declare <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrlv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrlv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psrlv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrav32_hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrav32_hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psrav32_hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16> @test_int_x86_avx512_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psllv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psllv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_int_x86_avx512_maskz_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psllv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8> @test_int_x86_avx512_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmov_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <32 x i16> [[TMP1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <32 x i16> [[X0:%.*]] to <32 x i8>
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[TMP2]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_int_x86_avx512_mask_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <32 x i16> [[TMP1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <32 x i16> [[X0:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i8> [[_MSPROP]], <32 x i8> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i8> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i8> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP10]], <32 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i8> [[TMP4]], <32 x i8> [[X1]]
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[TMP11]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_int_x86_avx512_maskz_pmov_wb_512(<32 x i16> %x0, i32 %x2) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmov_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <32 x i16> [[TMP1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <32 x i16> [[X0:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[_MSPROP]], <32 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i8> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> [[TMP9]], <32 x i8> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP3]], <32 x i8> zeroinitializer
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[TMP10]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+  ret <32 x i8> %res
+}
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll
new file mode 100644
index 0000000000000..3f09e54a0224d
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll
@@ -0,0 +1,3687 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -mattr=+avx512f -passes=msan 2>&1 | FileCheck %s
+;
+; Forked from llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
+;
+; Strictly handled:
+; - llvm.x86.avx512.dbpsadbw.512
+; - llvm.x86.avx512.ktestc.d, llvm.x86.avx512.ktestc.q, llvm.x86.avx512.ktestz.d, llvm.x86.avx512.ktestz.q
+; - llvm.x86.avx512.mask.pmovs.wb.512, llvm.x86.avx512.mask.pmovs.wb.mem.512
+; - llvm.x86.avx512.mask.pmovus.wb.512, llvm.x86.avx512.mask.pmovus.wb.mem.512
+; - llvm.x86.avx512.mask.pmov.wb.mem.512
+; - llvm.x86.avx512.packssdw.512, llvm.x86.avx512.packsswb.512
+; - llvm.x86.avx512.packusdw.512, llvm.x86.avx512.packuswb.512
+; - llvm.x86.avx512.pmaddubs.w.512, llvm.x86.avx512.pmaddw.d.512
+; - llvm.x86.avx512.psad.bw.512
+;
+; Heuristically handled:
+; - llvm.x86.avx512.kadd.d, llvm.x86.avx512.kadd.q
+; - llvm.x86.avx512.pavg.b.512, llvm.x86.avx512.pavg.w.512
+; - llvm.x86.avx512.permvar.hi.512
+; - llvm.x86.avx512.pmul.hr.sw.512, llvm.x86.avx512.pmulhu.w.512, llvm.x86.avx512.pmulh.w.512
+; - llvm.x86.avx512.pshuf.b.512
+; - llvm.x86.avx512.psllv.w.512
+; - llvm.x86.avx512.psrav.w.512, llvm.x86.avx512.psrlv.w.512
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @test_int_x86_avx512_kadd_d(<32 x i16> %A, <32 x i16> %B) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_kadd_d(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <32 x i16> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i16> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <32 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <32 x i16> [[TMP3]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <32 x i16> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <32 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[TMP10]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i16> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <32 x i1> [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <32 x i16> [[B]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i1> [[_MSPROP_ICMP]], [[_MSPROP_ICMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call <32 x i1> @llvm.x86.avx512.kadd.d(<32 x i1> [[TMP8]], <32 x i1> [[TMP15]])
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <32 x i1> [[_MSPROP]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i1> [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i32 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i32 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i32 [[TMP20]], -1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], [[TMP19]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP2:%.*]] = and i1 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i32 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext i1 [[_MSPROP_ICMP2]] to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i1 [[TMP25]] to i32
+; CHECK-NEXT:    store i32 [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP26]]
+;
+entry:
+  %0 = icmp ne <32 x i16> %A, zeroinitializer
+  %1 = icmp ne <32 x i16> %B, zeroinitializer
+  %2 = call <32 x i1> @llvm.x86.avx512.kadd.d(<32 x i1> %0, <32 x i1> %1)
+  %3 = bitcast <32 x i1> %2 to i32
+  %4 = icmp eq i32 %3, 0
+  %5 = zext i1 %4 to i32
+  ret i32 %5
+}
+declare <32 x i1> @llvm.x86.avx512.kadd.d(<32 x i1>, <32 x i1>)
+
+define i32 @test_int_x86_avx512_kadd_q(<64 x i8> %A, <64 x i8> %B) nounwind #0 {
+; CHECK-LABEL: @test_int_x86_avx512_kadd_q(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <64 x i8> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = or <64 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <64 x i8> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <64 x i8> [[TMP3]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = and <64 x i8> [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <64 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[TMP10]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i8> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <64 x i1> [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <64 x i8> [[B]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i1> [[_MSPROP_ICMP]], [[_MSPROP_ICMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call <64 x i1> @llvm.x86.avx512.kadd.q(<64 x i1> [[TMP8]], <64 x i1> [[TMP15]])
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i1> [[_MSPROP]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i1> [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i64 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i64 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i64 [[TMP20]], -1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], [[TMP19]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[TMP23]], 0
+; CHECK-NEXT:    [[_MSPROP_ICMP2:%.*]] = and i1 [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = zext i1 [[_MSPROP_ICMP2]] to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i1 [[TMP25]] to i32
+; CHECK-NEXT:    store i32 [[_MSPROP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP26]]
+;
+entry:
+  %0 = icmp ne <64 x i8> %A, zeroinitializer
+  %1 = icmp ne <64 x i8> %B, zeroinitializer
+  %2 = call <64 x i1> @llvm.x86.avx512.kadd.q(<64 x i1> %0, <64 x i1> %1)
+  %3 = bitcast <64 x i1> %2 to i64
+  %4 = icmp eq i64 %3, 0
+  %5 = zext i1 %4 to i32
+  ret i32 %5
+}
+declare <64 x i1> @llvm.x86.avx512.kadd.q(<64 x i1>, <64 x i1>)
+
+define i32 @test_x86_avx512_ktestc_d(<32 x i16> %A, <32 x i16> %B) #0 {
+; CHECK-LABEL: @test_x86_avx512_ktestc_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <32 x i16> [[TMP4]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i16> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i16> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <32 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <32 x i16> [[B]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP]] to i32
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP1]] to i32
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       19:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       20:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.d(<32 x i1> [[TMP9]], <32 x i1> [[TMP16]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = icmp ne <32 x i16> %A, zeroinitializer
+  %2 = icmp ne <32 x i16> %B, zeroinitializer
+  %res = call i32 @llvm.x86.avx512.ktestc.d(<32 x i1> %1, <32 x i1> %2) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx512.ktestc.d(<32 x i1>, <32 x i1>) nounwind readnone
+
+define i32 @test_x86_avx512_ktestz_d(<32 x i16> %A, <32 x i16> %B) #0 {
+; CHECK-LABEL: @test_x86_avx512_ktestz_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <32 x i16> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <32 x i16> [[TMP4]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <32 x i16> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i16> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <32 x i16> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP11]], splat (i16 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <32 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <32 x i16> [[B]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP]] to i32
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i1> [[_MSPROP_ICMP1]] to i32
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       20:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.d(<32 x i1> [[TMP9]], <32 x i1> [[TMP16]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = icmp ne <32 x i16> %A, zeroinitializer
+  %2 = icmp ne <32 x i16> %B, zeroinitializer
+  %res = call i32 @llvm.x86.avx512.ktestz.d(<32 x i1> %1, <32 x i1> %2) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx512.ktestz.d(<32 x i1>, <32 x i1>) nounwind readnone
+
+define i32 @test_x86_avx512_ktestc_q(<64 x i8> %A, <64 x i8> %B) #0 {
+; CHECK-LABEL: @test_x86_avx512_ktestc_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = or <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <64 x i8> [[TMP4]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <64 x i8> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <64 x i8> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <64 x i8> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <64 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP11]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i8> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <64 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <64 x i8> [[B]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       20:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.q(<64 x i1> [[TMP9]], <64 x i1> [[TMP16]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = icmp ne <64 x i8> %A, zeroinitializer
+  %2 = icmp ne <64 x i8> %B, zeroinitializer
+  %res = call i32 @llvm.x86.avx512.ktestc.q(<64 x i1> %1, <64 x i1> %2) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx512.ktestc.q(<64 x i1>, <64 x i1>) nounwind readnone
+
+define i32 @test_x86_avx512_ktestz_q(<64 x i8> %A, <64 x i8> %B) #0 {
+; CHECK-LABEL: @test_x86_avx512_ktestz_q(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <64 x i8> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = or <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <64 x i8> [[TMP4]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP7:%.*]] = and <64 x i8> [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <64 x i8> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <64 x i8> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <64 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP11]], splat (i8 -1)
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i8> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq <64 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_ICMP1:%.*]] = and <64 x i1> [[TMP12]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <64 x i8> [[B]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP]] to i64
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i1> [[_MSPROP_ICMP1]] to i64
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       20:
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.q(<64 x i1> [[TMP9]], <64 x i1> [[TMP16]])
+; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = icmp ne <64 x i8> %A, zeroinitializer
+  %2 = icmp ne <64 x i8> %B, zeroinitializer
+  %res = call i32 @llvm.x86.avx512.ktestz.q(<64 x i1> %1, <64 x i1> %2) ; <i32> [#uses=1]
+  ret i32 %res
+}
+declare i32 @llvm.x86.avx512.ktestz.q(<64 x i1>, <64 x i1>) nounwind readnone
+
+define <32 x i16> @test_mask_packs_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_packs_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packs_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP9]], <32 x i16> [[TMP14]], <32 x i16> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP8]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP15]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packs_epi32_rm_512(<16 x i32> %a, ptr %ptr_b) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmb_512(<16 x i32> %a, ptr %ptr_b) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmbk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmbk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packs_epi32_rmbkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi32_rmbkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32>, <16 x i32>)
+
+define <64 x i8> @test_mask_packs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP7]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a, <32 x i16> %b)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @test_mask_packs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passThru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @test_mask_packs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <64 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP9]], <64 x i8> [[TMP14]], <64 x i8> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP8]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP15]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> zeroinitializer
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @test_mask_packs_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a, <32 x i16> %b)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @test_mask_packs_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <64 x i8> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <64 x i8> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP20]], <64 x i8> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> [[TMP14]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP21]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passThru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @test_mask_packs_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packs_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <64 x i8> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP14]], <64 x i8> [[TMP19]], <64 x i8> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP13]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP20]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> zeroinitializer
+  ret <64 x i8> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16>, <32 x i16>)
+
+
+define <32 x i16> @test_mask_packus_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_packus_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packus_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <32 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP9]], <32 x i16> [[TMP14]], <32 x i16> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP8]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP15]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packus_epi32_rm_512(<16 x i32> %a, ptr %ptr_b) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <16 x i32>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %b = load <16 x i32>, ptr %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmb_512(<16 x i32> %a, ptr %ptr_b) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmbk_512(<16 x i32> %a, ptr %ptr_b, <32 x i16> %passThru, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmbk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <32 x i16> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <32 x i16> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP20]], <32 x i16> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <32 x i1> [[TMP16]], <32 x i16> [[TMP14]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP21]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_packus_epi32_rmbkz_512(<16 x i32> %a, ptr %ptr_b, i32 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi32_rmbkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr [[PTR_B:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i32> splat (i32 -1), i32 [[_MSLD]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <16 x i32> poison, i32 [[Q]], i32 0
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[_MSPROP]], <16 x i32> splat (i32 -1), <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <16 x i32> [[VECINIT_I]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[_MSPROP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> [[A:%.*]], <16 x i32> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP14]], <32 x i16> [[TMP19]], <32 x i16> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <32 x i1> [[TMP15]], <32 x i16> [[TMP13]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP20]]
+;
+  %q = load i32, ptr %ptr_b
+  %vecinit.i = insertelement <16 x i32> poison, i32 %q, i32 0
+  %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> poison, <16 x i32> zeroinitializer
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a, <16 x i32> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32>, <16 x i32>)
+
+define <64 x i8> @test_mask_packus_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rr_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP7]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a, <32 x i16> %b)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @test_mask_packus_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rrk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <64 x i8> [[TMP9]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i8> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP15]], <64 x i8> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP11]], <64 x i8> [[TMP9]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP16]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passThru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @test_mask_packus_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rrkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <64 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <64 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP9]], <64 x i8> [[TMP14]], <64 x i8> [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = select <64 x i1> [[TMP10]], <64 x i8> [[TMP8]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP15]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> zeroinitializer
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @test_mask_packus_epi16_rm_512(<32 x i16> %a, ptr %ptr_b) #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rm_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    store <64 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a, <32 x i16> %b)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @test_mask_packus_epi16_rmk_512(<32 x i16> %a, ptr %ptr_b, <64 x i8> %passThru, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rmk_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP9]], align 64
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP12:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
+; CHECK:       12:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> zeroinitializer, <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = xor <64 x i8> [[TMP14]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <64 x i8> [[TMP19]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP20]], <64 x i8> [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select <64 x i1> [[TMP16]], <64 x i8> [[TMP14]], <64 x i8> [[PASSTHRU]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP21]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passThru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @test_mask_packus_epi16_rmkz_512(<32 x i16> %a, ptr %ptr_b, i64 %mask) #0 {
+; CHECK-LABEL: @test_mask_packus_epi16_rmkz_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[B:%.*]] = load <32 x i16>, ptr [[PTR_B:%.*]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PTR_B]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i16>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i16> [[_MSLD]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[TMP13:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[A:%.*]], <32 x i16> [[B]])
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP16:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> zeroinitializer, <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <64 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = or <64 x i8> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <64 x i8> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP14]], <64 x i8> [[TMP19]], <64 x i8> [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select <64 x i1> [[TMP15]], <64 x i8> [[TMP13]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP20]]
+;
+  %b = load <32 x i16>, ptr %ptr_b
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> zeroinitializer
+  ret <64 x i8> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16>@test_int_x86_avx512_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermt2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i16 [[TMP4]], 31
+; CHECK-NEXT:    [[TMP6:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i16 [[TMP7]], 31
+; CHECK-NEXT:    [[TMP9:%.*]] = or i16 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 31
+; CHECK-NEXT:    [[TMP12:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i16 [[TMP13]], 31
+; CHECK-NEXT:    [[TMP15:%.*]] = or i16 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 31
+; CHECK-NEXT:    [[TMP18:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i16 [[TMP19]], 31
+; CHECK-NEXT:    [[TMP21:%.*]] = or i16 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 31
+; CHECK-NEXT:    [[TMP24:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i16 [[TMP25]], 31
+; CHECK-NEXT:    [[TMP27:%.*]] = or i16 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP29:%.*]] = and i16 [[TMP28]], 31
+; CHECK-NEXT:    [[TMP30:%.*]] = or i16 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP32:%.*]] = and i16 [[TMP31]], 31
+; CHECK-NEXT:    [[TMP33:%.*]] = or i16 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP35:%.*]] = and i16 [[TMP34]], 31
+; CHECK-NEXT:    [[TMP36:%.*]] = or i16 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP38:%.*]] = and i16 [[TMP37]], 31
+; CHECK-NEXT:    [[TMP39:%.*]] = or i16 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP41:%.*]] = and i16 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP42:%.*]] = or i16 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP44:%.*]] = and i16 [[TMP43]], 31
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP47:%.*]] = and i16 [[TMP46]], 31
+; CHECK-NEXT:    [[TMP48:%.*]] = or i16 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i16 [[TMP49]], 31
+; CHECK-NEXT:    [[TMP51:%.*]] = or i16 [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP53:%.*]] = and i16 [[TMP52]], 31
+; CHECK-NEXT:    [[TMP54:%.*]] = or i16 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP56:%.*]] = and i16 [[TMP55]], 31
+; CHECK-NEXT:    [[TMP57:%.*]] = or i16 [[TMP55]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP59:%.*]] = and i16 [[TMP58]], 31
+; CHECK-NEXT:    [[TMP60:%.*]] = or i16 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP62:%.*]] = and i16 [[TMP61]], 31
+; CHECK-NEXT:    [[TMP63:%.*]] = or i16 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP65:%.*]] = and i16 [[TMP64]], 31
+; CHECK-NEXT:    [[TMP66:%.*]] = or i16 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP68:%.*]] = and i16 [[TMP67]], 31
+; CHECK-NEXT:    [[TMP69:%.*]] = or i16 [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP71:%.*]] = and i16 [[TMP70]], 31
+; CHECK-NEXT:    [[TMP72:%.*]] = or i16 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP74:%.*]] = and i16 [[TMP73]], 31
+; CHECK-NEXT:    [[TMP75:%.*]] = or i16 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP77:%.*]] = and i16 [[TMP76]], 31
+; CHECK-NEXT:    [[TMP78:%.*]] = or i16 [[TMP76]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP80:%.*]] = and i16 [[TMP79]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = or i16 [[TMP79]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP83:%.*]] = and i16 [[TMP82]], 31
+; CHECK-NEXT:    [[TMP84:%.*]] = or i16 [[TMP82]], [[TMP83]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP86:%.*]] = and i16 [[TMP85]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = or i16 [[TMP85]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP89:%.*]] = and i16 [[TMP88]], 31
+; CHECK-NEXT:    [[TMP90:%.*]] = or i16 [[TMP88]], [[TMP89]]
+; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP92:%.*]] = and i16 [[TMP91]], 31
+; CHECK-NEXT:    [[TMP93:%.*]] = or i16 [[TMP91]], [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP95:%.*]] = and i16 [[TMP94]], 31
+; CHECK-NEXT:    [[TMP96:%.*]] = or i16 [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP98:%.*]] = and i16 [[TMP97]], 31
+; CHECK-NEXT:    [[TMP99:%.*]] = or i16 [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP100:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X0:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP99]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP101:%.*]], label [[TMP102:%.*]], !prof [[PROF1]]
+; CHECK:       101:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       102:
+; CHECK-NEXT:    [[TMP103:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X1:%.*]], <32 x i16> [[X0]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[TMP100]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP103]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x2)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i16 [[TMP5]], 31
+; CHECK-NEXT:    [[TMP7:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i16 [[TMP8]], 31
+; CHECK-NEXT:    [[TMP10:%.*]] = or i16 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 31
+; CHECK-NEXT:    [[TMP13:%.*]] = or i16 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP16:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i16 [[TMP17]], 31
+; CHECK-NEXT:    [[TMP19:%.*]] = or i16 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP22:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i16 [[TMP23]], 31
+; CHECK-NEXT:    [[TMP25:%.*]] = or i16 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i16 [[TMP29]], 31
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP34:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i16 [[TMP35]], 31
+; CHECK-NEXT:    [[TMP37:%.*]] = or i16 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP40:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i16 [[TMP41]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], 31
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP54:%.*]] = and i16 [[TMP53]], 31
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP58:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP60:%.*]] = and i16 [[TMP59]], 31
+; CHECK-NEXT:    [[TMP61:%.*]] = or i16 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP64:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP66:%.*]] = and i16 [[TMP65]], 31
+; CHECK-NEXT:    [[TMP67:%.*]] = or i16 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP70:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP72:%.*]] = and i16 [[TMP71]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP75:%.*]] = and i16 [[TMP74]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP78:%.*]] = and i16 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP81:%.*]] = and i16 [[TMP80]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP84:%.*]] = and i16 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP83]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP87:%.*]] = and i16 [[TMP86]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP90:%.*]] = and i16 [[TMP89]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP93:%.*]] = and i16 [[TMP92]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP96:%.*]] = and i16 [[TMP95]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP95]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP99:%.*]] = and i16 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X0:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP100]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP102:%.*]], label [[TMP103:%.*]], !prof [[PROF1]]
+; CHECK:       102:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       103:
+; CHECK-NEXT:    [[TMP104:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X1:%.*]], <32 x i16> [[X0]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP107:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP101]], <32 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP104]], [[X1]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i16> [[TMP108]], [[TMP101]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[TMP109]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP105]], <32 x i16> [[TMP110]], <32 x i16> [[TMP107]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP104]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP111]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x2)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x1
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i16 [[TMP5]], 31
+; CHECK-NEXT:    [[TMP7:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i16 [[TMP8]], 31
+; CHECK-NEXT:    [[TMP10:%.*]] = or i16 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 31
+; CHECK-NEXT:    [[TMP13:%.*]] = or i16 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP16:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i16 [[TMP17]], 31
+; CHECK-NEXT:    [[TMP19:%.*]] = or i16 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP22:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i16 [[TMP23]], 31
+; CHECK-NEXT:    [[TMP25:%.*]] = or i16 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i16 [[TMP29]], 31
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP34:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i16 [[TMP35]], 31
+; CHECK-NEXT:    [[TMP37:%.*]] = or i16 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP40:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i16 [[TMP41]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], 31
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP54:%.*]] = and i16 [[TMP53]], 31
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP58:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP60:%.*]] = and i16 [[TMP59]], 31
+; CHECK-NEXT:    [[TMP61:%.*]] = or i16 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP64:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP66:%.*]] = and i16 [[TMP65]], 31
+; CHECK-NEXT:    [[TMP67:%.*]] = or i16 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP70:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP72:%.*]] = and i16 [[TMP71]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP75:%.*]] = and i16 [[TMP74]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP78:%.*]] = and i16 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP81:%.*]] = and i16 [[TMP80]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP84:%.*]] = and i16 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP83]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP87:%.*]] = and i16 [[TMP86]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP90:%.*]] = and i16 [[TMP89]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP93:%.*]] = and i16 [[TMP92]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP96:%.*]] = and i16 [[TMP95]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP95]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP99:%.*]] = and i16 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X0:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP100]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP102:%.*]], label [[TMP103:%.*]], !prof [[PROF1]]
+; CHECK:       102:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       103:
+; CHECK-NEXT:    [[TMP104:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X1:%.*]], <32 x i16> [[X0]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP107:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP101]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP104]], zeroinitializer
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i16> [[TMP108]], [[TMP101]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[TMP109]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP105]], <32 x i16> [[TMP110]], <32 x i16> [[TMP107]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP104]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP111]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x2)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>)
+
+define <32 x i16>@test_int_x86_avx512_vpermi2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_vpermi2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i16 [[TMP4]], 31
+; CHECK-NEXT:    [[TMP6:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i16 [[TMP7]], 31
+; CHECK-NEXT:    [[TMP9:%.*]] = or i16 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 31
+; CHECK-NEXT:    [[TMP12:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i16 [[TMP13]], 31
+; CHECK-NEXT:    [[TMP15:%.*]] = or i16 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 31
+; CHECK-NEXT:    [[TMP18:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i16 [[TMP19]], 31
+; CHECK-NEXT:    [[TMP21:%.*]] = or i16 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 31
+; CHECK-NEXT:    [[TMP24:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i16 [[TMP25]], 31
+; CHECK-NEXT:    [[TMP27:%.*]] = or i16 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP29:%.*]] = and i16 [[TMP28]], 31
+; CHECK-NEXT:    [[TMP30:%.*]] = or i16 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP32:%.*]] = and i16 [[TMP31]], 31
+; CHECK-NEXT:    [[TMP33:%.*]] = or i16 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP35:%.*]] = and i16 [[TMP34]], 31
+; CHECK-NEXT:    [[TMP36:%.*]] = or i16 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP38:%.*]] = and i16 [[TMP37]], 31
+; CHECK-NEXT:    [[TMP39:%.*]] = or i16 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP41:%.*]] = and i16 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP42:%.*]] = or i16 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP44:%.*]] = and i16 [[TMP43]], 31
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP47:%.*]] = and i16 [[TMP46]], 31
+; CHECK-NEXT:    [[TMP48:%.*]] = or i16 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP50:%.*]] = and i16 [[TMP49]], 31
+; CHECK-NEXT:    [[TMP51:%.*]] = or i16 [[TMP49]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP53:%.*]] = and i16 [[TMP52]], 31
+; CHECK-NEXT:    [[TMP54:%.*]] = or i16 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP56:%.*]] = and i16 [[TMP55]], 31
+; CHECK-NEXT:    [[TMP57:%.*]] = or i16 [[TMP55]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP59:%.*]] = and i16 [[TMP58]], 31
+; CHECK-NEXT:    [[TMP60:%.*]] = or i16 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP61:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP62:%.*]] = and i16 [[TMP61]], 31
+; CHECK-NEXT:    [[TMP63:%.*]] = or i16 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP65:%.*]] = and i16 [[TMP64]], 31
+; CHECK-NEXT:    [[TMP66:%.*]] = or i16 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP68:%.*]] = and i16 [[TMP67]], 31
+; CHECK-NEXT:    [[TMP69:%.*]] = or i16 [[TMP67]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP71:%.*]] = and i16 [[TMP70]], 31
+; CHECK-NEXT:    [[TMP72:%.*]] = or i16 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP74:%.*]] = and i16 [[TMP73]], 31
+; CHECK-NEXT:    [[TMP75:%.*]] = or i16 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP77:%.*]] = and i16 [[TMP76]], 31
+; CHECK-NEXT:    [[TMP78:%.*]] = or i16 [[TMP76]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP80:%.*]] = and i16 [[TMP79]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = or i16 [[TMP79]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP83:%.*]] = and i16 [[TMP82]], 31
+; CHECK-NEXT:    [[TMP84:%.*]] = or i16 [[TMP82]], [[TMP83]]
+; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP86:%.*]] = and i16 [[TMP85]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = or i16 [[TMP85]], [[TMP86]]
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP89:%.*]] = and i16 [[TMP88]], 31
+; CHECK-NEXT:    [[TMP90:%.*]] = or i16 [[TMP88]], [[TMP89]]
+; CHECK-NEXT:    [[TMP91:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP92:%.*]] = and i16 [[TMP91]], 31
+; CHECK-NEXT:    [[TMP93:%.*]] = or i16 [[TMP91]], [[TMP92]]
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP95:%.*]] = and i16 [[TMP94]], 31
+; CHECK-NEXT:    [[TMP96:%.*]] = or i16 [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP98:%.*]] = and i16 [[TMP97]], 31
+; CHECK-NEXT:    [[TMP99:%.*]] = or i16 [[TMP97]], [[TMP98]]
+; CHECK-NEXT:    [[TMP100:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X1:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP99]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP101:%.*]], label [[TMP102:%.*]], !prof [[PROF1]]
+; CHECK:       101:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       102:
+; CHECK-NEXT:    [[TMP103:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[TMP100]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP103]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <32 x i16> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = and i16 [[TMP5]], 31
+; CHECK-NEXT:    [[TMP7:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i16> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i16 [[TMP8]], 31
+; CHECK-NEXT:    [[TMP10:%.*]] = or i16 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <32 x i16> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = and i16 [[TMP11]], 31
+; CHECK-NEXT:    [[TMP13:%.*]] = or i16 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP16:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <32 x i16> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = and i16 [[TMP17]], 31
+; CHECK-NEXT:    [[TMP19:%.*]] = or i16 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP22:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <32 x i16> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP24:%.*]] = and i16 [[TMP23]], 31
+; CHECK-NEXT:    [[TMP25:%.*]] = or i16 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <32 x i16> [[TMP3]], i64 8
+; CHECK-NEXT:    [[TMP30:%.*]] = and i16 [[TMP29]], 31
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP3]], i64 9
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP34:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <32 x i16> [[TMP3]], i64 10
+; CHECK-NEXT:    [[TMP36:%.*]] = and i16 [[TMP35]], 31
+; CHECK-NEXT:    [[TMP37:%.*]] = or i16 [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP3]], i64 11
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP40:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <32 x i16> [[TMP3]], i64 12
+; CHECK-NEXT:    [[TMP42:%.*]] = and i16 [[TMP41]], 31
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP3]], i64 13
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <32 x i16> [[TMP3]], i64 14
+; CHECK-NEXT:    [[TMP48:%.*]] = and i16 [[TMP47]], 31
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP3]], i64 15
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <32 x i16> [[TMP3]], i64 16
+; CHECK-NEXT:    [[TMP54:%.*]] = and i16 [[TMP53]], 31
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP53]], [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP3]], i64 17
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP58:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <32 x i16> [[TMP3]], i64 18
+; CHECK-NEXT:    [[TMP60:%.*]] = and i16 [[TMP59]], 31
+; CHECK-NEXT:    [[TMP61:%.*]] = or i16 [[TMP59]], [[TMP60]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP3]], i64 19
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP64:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <32 x i16> [[TMP3]], i64 20
+; CHECK-NEXT:    [[TMP66:%.*]] = and i16 [[TMP65]], 31
+; CHECK-NEXT:    [[TMP67:%.*]] = or i16 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP3]], i64 21
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP70:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <32 x i16> [[TMP3]], i64 22
+; CHECK-NEXT:    [[TMP72:%.*]] = and i16 [[TMP71]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP71]], [[TMP72]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <32 x i16> [[TMP3]], i64 23
+; CHECK-NEXT:    [[TMP75:%.*]] = and i16 [[TMP74]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <32 x i16> [[TMP3]], i64 24
+; CHECK-NEXT:    [[TMP78:%.*]] = and i16 [[TMP77]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP77]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <32 x i16> [[TMP3]], i64 25
+; CHECK-NEXT:    [[TMP81:%.*]] = and i16 [[TMP80]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <32 x i16> [[TMP3]], i64 26
+; CHECK-NEXT:    [[TMP84:%.*]] = and i16 [[TMP83]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP83]], [[TMP84]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <32 x i16> [[TMP3]], i64 27
+; CHECK-NEXT:    [[TMP87:%.*]] = and i16 [[TMP86]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <32 x i16> [[TMP3]], i64 28
+; CHECK-NEXT:    [[TMP90:%.*]] = and i16 [[TMP89]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <32 x i16> [[TMP3]], i64 29
+; CHECK-NEXT:    [[TMP93:%.*]] = and i16 [[TMP92]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <32 x i16> [[TMP3]], i64 30
+; CHECK-NEXT:    [[TMP96:%.*]] = and i16 [[TMP95]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP95]], [[TMP96]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <32 x i16> [[TMP3]], i64 31
+; CHECK-NEXT:    [[TMP99:%.*]] = and i16 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP101:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> [[X1:%.*]], <32 x i16> [[TMP2]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP100]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP102:%.*]], label [[TMP103:%.*]], !prof [[PROF1]]
+; CHECK:       102:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       103:
+; CHECK-NEXT:    [[TMP104:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1]], <32 x i16> [[X2:%.*]])
+; CHECK-NEXT:    [[TMP105:%.*]] = bitcast i32 [[TMP4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP106:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP107:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP101]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP108:%.*]] = xor <32 x i16> [[TMP104]], [[X1]]
+; CHECK-NEXT:    [[TMP109:%.*]] = or <32 x i16> [[TMP108]], [[TMP101]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or <32 x i16> [[TMP109]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP105]], <32 x i16> [[TMP110]], <32 x i16> [[TMP107]]
+; CHECK-NEXT:    [[TMP111:%.*]] = select <32 x i1> [[TMP106]], <32 x i16> [[TMP104]], <32 x i16> [[X1]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP111]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x1
+  ret <32 x i16> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8>, <64 x i8>)
+
+define <64 x i8> @test_int_x86_avx512_pavg_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pavg_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @test_int_x86_avx512_mask_pavg_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <64 x i8> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i8> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i8> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP6]], <64 x i8> [[TMP11]], <64 x i8> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[TMP5]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[TMP12]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1)
+  %2 = bitcast i64 %x3 to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %x2
+  ret <64 x i8> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_int_x86_avx512_pavg_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pavg_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pavg_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
+
+define <64 x i8>@test_int_x86_avx512_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshuf_b_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[RES]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8>@test_int_x86_avx512_pshuf_b_512_mask(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshuf_b_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <64 x i1> [[MASK_CAST]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <64 x i8> [[RES]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> [[TMP9]], <64 x i8> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <64 x i1> [[MASK_CAST]], <64 x i8> [[RES]], <64 x i8> [[X2]]
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[RES2]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1)
+  %mask.cast = bitcast i64 %mask to <64 x i1>
+  %res2 = select <64 x i1> %mask.cast, <64 x i8> %res, <64 x i8> %x2
+  ret <64 x i8> %res2
+}
+
+define <64 x i8>@test_int_x86_avx512_pshuf_b_512_maskz(<64 x i8> %x0, <64 x i8> %x1, i64 %mask) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pshuf_b_512_maskz(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = select <64 x i1> [[MASK_CAST]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <64 x i8> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = or <64 x i8> [[TMP6]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> [[TMP8]], <64 x i8> [[TMP5]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <64 x i1> [[MASK_CAST]], <64 x i8> [[RES]], <64 x i8> zeroinitializer
+; CHECK-NEXT:    store <64 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <64 x i8> [[RES2]]
+;
+  %res = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1)
+  %mask.cast = bitcast i64 %mask to <64 x i1>
+  %res2 = select <64 x i1> %mask.cast, <64 x i8> %res, <64 x i8> zeroinitializer
+  ret <64 x i8> %res2
+}
+
+declare <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_int_x86_avx512_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmulhu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_int_x86_avx512_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmulh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulh_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_int_x86_avx512_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmulhr_sw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhr_sw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+define <32 x i8>@test_int_x86_avx512_pmov_wb_512(<32 x i16> %x0) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmov_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <32 x i16> [[TMP1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <32 x i16> [[X0:%.*]] to <32 x i8>
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[TMP2]]
+;
+  %1 = trunc <32 x i16> %x0 to <32 x i8>
+  ret <32 x i8> %1
+}
+
+define <32 x i8>@test_int_x86_avx512_mask_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <32 x i16> [[TMP1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <32 x i16> [[X0:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i8> [[_MSPROP]], <32 x i8> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i8> [[TMP4]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i8> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i8> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP10]], <32 x i8> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i8> [[TMP4]], <32 x i8> [[X1]]
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[TMP11]]
+;
+  %1 = trunc <32 x i16> %x0 to <32 x i8>
+  %2 = bitcast i32 %x2 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %x1
+  ret <32 x i8> %3
+}
+
+define <32 x i8>@test_int_x86_avx512_maskz_pmov_wb_512(<32 x i16> %x0, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmov_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc <32 x i16> [[TMP1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <32 x i16> [[X0:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[X2:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[_MSPROP]], <32 x i8> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i8> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[TMP7]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i8> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> [[TMP9]], <32 x i8> [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP3]], <32 x i8> zeroinitializer
+; CHECK-NEXT:    store <32 x i8> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[TMP10]]
+;
+  %1 = trunc <32 x i16> %x0 to <32 x i8>
+  %2 = bitcast i32 %x2 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
+}
+
+declare void @llvm.x86.avx512.mask.pmov.wb.mem.512(ptr %ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_pmov_wb_mem_512(ptr %ptr, <32 x i16> %x1, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmov_wb_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.wb.mem.512(ptr [[PTR:%.*]], <32 x i16> [[X1:%.*]], i32 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmov.wb.mem.512(ptr [[PTR]], <32 x i16> [[X1]], i32 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmov.wb.mem.512(ptr %ptr, <32 x i16> %x1, i32 -1)
+  call void @llvm.x86.avx512.mask.pmov.wb.mem.512(ptr %ptr, <32 x i16> %x1, i32 %x2)
+  ret void
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_pmovs_wb_512(<32 x i16> %x0, <32 x i8> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovs_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> [[X0:%.*]], <32 x i8> [[X1:%.*]], i32 -1)
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8>@test_int_x86_avx512_mask_pmovs_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> [[X0:%.*]], <32 x i8> [[X1:%.*]], i32 [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+  ret <32 x i8> %res
+}
+
+define <32 x i8>@test_int_x86_avx512_maskz_pmovs_wb_512(<32 x i16> %x0, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovs_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> [[X0:%.*]], <32 x i8> zeroinitializer, i32 [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+  ret <32 x i8> %res
+}
+
+declare void @llvm.x86.avx512.mask.pmovs.wb.mem.512(ptr %ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_pmovs_wb_mem_512(ptr %ptr, <32 x i16> %x1, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovs_wb_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(ptr [[PTR:%.*]], <32 x i16> [[X1:%.*]], i32 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(ptr [[PTR]], <32 x i16> [[X1]], i32 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(ptr %ptr, <32 x i16> %x1, i32 -1)
+  call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(ptr %ptr, <32 x i16> %x1, i32 %x2)
+  ret void
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_pmovus_wb_512(<32 x i16> %x0, <32 x i8> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmovus_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> [[X0:%.*]], <32 x i8> [[X1:%.*]], i32 -1)
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8>@test_int_x86_avx512_mask_pmovus_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
+; CHECK:       6:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       7:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> [[X0:%.*]], <32 x i8> [[X1:%.*]], i32 [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+  ret <32 x i8> %res
+}
+
+define <32 x i8>@test_int_x86_avx512_maskz_pmovus_wb_512(<32 x i16> %x0, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_pmovus_wb_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP4:%.*]], label [[TMP5:%.*]], !prof [[PROF1]]
+; CHECK:       4:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       5:
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> [[X0:%.*]], <32 x i8> zeroinitializer, i32 [[X2:%.*]])
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i8> [[RES]]
+;
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+  ret <32 x i8> %res
+}
+
+declare void @llvm.x86.avx512.mask.pmovus.wb.mem.512(ptr %ptr, <32 x i16>, i32)
+
+define void @test_int_x86_avx512_mask_pmovus_wb_mem_512(ptr %ptr, <32 x i16> %x1, i32 %x2) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmovus_wb_mem_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(ptr [[PTR:%.*]], <32 x i16> [[X1:%.*]], i32 -1)
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR4:%.*]] = or i1 [[_MSCMP2]], [[_MSCMP3]]
+; CHECK-NEXT:    [[_MSCMP5:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[_MSOR6:%.*]] = or i1 [[_MSOR4]], [[_MSCMP5]]
+; CHECK-NEXT:    br i1 [[_MSOR6]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
+; CHECK:       8:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       9:
+; CHECK-NEXT:    call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(ptr [[PTR]], <32 x i16> [[X1]], i32 [[X2:%.*]])
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(ptr %ptr, <32 x i16> %x1, i32 -1)
+  call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(ptr %ptr, <32 x i16> %x1, i32 %x2)
+  ret void
+}
+
+declare <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
+
+define <32 x i16> @test_int_x86_avx512_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaddubs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaddubs_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP16]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+declare <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
+
+define <16 x i32> @test_int_x86_avx512_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_pmaddw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
+; CHECK:       5:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_pmaddw_d_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP16]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  ret <16 x i32> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8>, <64 x i8>, i32)
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_int_x86_avx512_mask_dbpsadbw_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x3, i32 %x4) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_dbpsadbw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
+; CHECK:       7:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       8:
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], i32 2)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[X4:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[X3:%.*]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[X3]]
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       20:
+; CHECK-NEXT:    [[TMP21:%.*]] = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i32 3)
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i32 [[X4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP24:%.*]] = select <32 x i1> [[TMP23]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <32 x i16> [[TMP21]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = or <32 x i16> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = or <32 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP22]], <32 x i16> [[TMP27]], <32 x i16> [[TMP24]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select <32 x i1> [[TMP23]], <32 x i16> [[TMP21]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP29]], 0
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP30]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    br i1 [[_MSOR8]], label [[TMP31:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       31:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       32:
+; CHECK-NEXT:    [[TMP33:%.*]] = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i32 4)
+; CHECK-NEXT:    [[TMP34:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[TMP16]], 0
+; CHECK-NEXT:    [[TMP35:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP34]], <32 x i16> [[_MSPROP_SELECT1]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES1]], <32 x i16> [[TMP28]], 1
+; CHECK-NEXT:    [[TMP36:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP35]], <32 x i16> zeroinitializer, 2
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RES2]], <32 x i16> [[TMP33]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP36]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RES3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2)
+  %2 = bitcast i32 %x4 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
+  %4 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 3)
+  %5 = bitcast i32 %x4 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
+  %7 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 4)
+  %res1 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> %3, 0
+  %res2 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res1, <32 x i16> %6, 1
+  %res3 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> }  %res2, <32 x i16> %7, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %res3
+}
+
+declare  <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8>, <64 x i8>)
+
+define { <8 x i64>, <8 x i64> } @test_int_x86_avx512_mask_psadb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2){
+; CHECK-LABEL: @test_int_x86_avx512_mask_psadb_w_512(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES0:%.*]] = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> [[X0]], <64 x i8> [[X2:%.*]])
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i64>, <8 x i64> } poison, <8 x i64> [[RES0]], 0
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i64>, <8 x i64> } [[RES2]], <8 x i64> [[RES1]], 1
+; CHECK-NEXT:    store { <8 x i64>, <8 x i64> } zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i64>, <8 x i64> } [[RES3]]
+;
+  %res0 = call  <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %x0, <64 x i8> %x1)
+  %res1 = call  <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %x0, <64 x i8> %x2)
+  %res2 = insertvalue { <8 x i64>, <8 x i64> } poison, <8 x i64> %res0, 0
+  %res3 = insertvalue { <8 x i64>, <8 x i64> }  %res2, <8 x i64> %res1, 1
+  ret { <8 x i64>, <8 x i64> } %res3
+}
+
+declare <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16>, <32 x i16>) nounwind readnone
+
+define <32 x i16> @test_x86_avx512_psrlv_w_512_const() optsize #0 {
+; CHECK-LABEL: @test_x86_avx512_psrlv_w_512_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES1]]
+;
+  %res1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+  ret <32 x i16> %res1
+}
+
+define <32 x i16>@test_int_x86_avx512_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrlv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrlv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psrlv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16>@test_int_x86_avx512_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psrav32_hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrav32_hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psrav32_hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi_const(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psrav32_hi_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> <i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51>, <32 x i16> <i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49>)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> <i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51>, <32 x i16> <i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_psllv32hi(<32 x i16> %x0, <32 x i16> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_psllv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_psllv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_psllv32hi(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16>@test_int_x86_avx512_permvar_hi_512(<32 x i16> %x0, <32 x i16> %x1) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
+define <32 x i16>@test_int_x86_avx512_mask_permvar_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_mask_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[_MSPROP]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <32 x i16> [[TMP5]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP11]], <32 x i16> [[TMP8]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP7]], <32 x i16> [[TMP5]], <32 x i16> [[X2]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP12]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_permvar_hi_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) #0 {
+; CHECK-LABEL: @test_int_x86_avx512_maskz_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[_MSPROP]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[_MSPROP]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[TMP11]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_x86_avx512_psll_w_512(<32 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psll_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_x86_avx512_mask_psll_w_512(<32 x i16> %a0, <8 x i16> %a1, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psll_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP11]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> %passthru
+  ret <32 x i16> %res2
+}
+define <32 x i16> @test_x86_avx512_maskz_psll_w_512(<32 x i16> %a0, <8 x i16> %a1, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psll_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP10]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> zeroinitializer
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16>, <8 x i16>) nounwind readnone
+
+
+define <32 x i16> @test_x86_avx512_psllv_w_512_const() optsize #0 {
+; CHECK-LABEL: @test_x86_avx512_psllv_w_512_const(
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[RES1:%.*]] = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES1]]
+;
+  %res1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4,  i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 -1>)
+  ret <32 x i16> %res1
+}
+declare <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16>, <32 x i16>) nounwind readnone
+
+define <32 x i16> @test_x86_avx512_pslli_w_512(<32 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_pslli_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <32 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_x86_avx512_mask_pslli_w_512(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_pslli_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP5]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> %passthru
+  ret <32 x i16> %res2
+}
+define <32 x i16> @test_x86_avx512_maskz_pslli_w_512(<32 x i16> %a0, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_pslli_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP9]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> zeroinitializer
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16>, i32) nounwind readnone
+
+
+define <32 x i16> @test_x86_avx512_psra_w_512(<32 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psra_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_x86_avx512_mask_psra_w_512(<32 x i16> %a0, <8 x i16> %a1, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psra_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP11]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> %passthru
+  ret <32 x i16> %res2
+}
+define <32 x i16> @test_x86_avx512_maskz_psra_w_512(<32 x i16> %a0, <8 x i16> %a1, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psra_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP10]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> zeroinitializer
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16>, <8 x i16>) nounwind readnone
+
+
+define <32 x i16> @test_x86_avx512_psrai_w_512(<32 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrai_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <32 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_x86_avx512_mask_psrai_w_512(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrai_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP5]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> %passthru
+  ret <32 x i16> %res2
+}
+define <32 x i16> @test_x86_avx512_maskz_psrai_w_512(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrai_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP9]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> zeroinitializer
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16>, i32) nounwind readnone
+
+
+define <32 x i16> @test_x86_avx512_psrl_w_512(<32 x i16> %a0, <8 x i16> %a1) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i1 [[TMP5]] to i512
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i512 [[TMP6]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_x86_avx512_mask_psrl_w_512(<32 x i16> %a0, <8 x i16> %a1, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 144) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i1 [[TMP7]] to i512
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i512 [[TMP8]] to <32 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i16> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP11]], <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <32 x i16> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP12]], <32 x i16> [[TMP16]], <32 x i16> [[TMP13]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> %passthru
+  ret <32 x i16> %res2
+}
+define <32 x i16> @test_x86_avx512_maskz_psrl_w_512(<32 x i16> %a0, <8 x i16> %a1, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrl_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 80) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i512
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i512 [[TMP7]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP1]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP10]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> zeroinitializer
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16>, <8 x i16>) nounwind readnone
+
+define <32 x i16> @test_x86_avx512_psrl_w_512_load(<32 x i16> %a0, ptr %p) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrl_w_512_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF1]]
+; CHECK:       3:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
+; CHECK-NEXT:    unreachable
+; CHECK:       4:
+; CHECK-NEXT:    [[A1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i16>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i16> [[_MSLD]] to i128
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i128 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i1 [[TMP10]] to i512
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i512 [[TMP11]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[TMP2]], <8 x i16> [[A1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[A0:%.*]], <8 x i16> [[A1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP14]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %a1 = load <8 x i16>, ptr %p
+  %res = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %a0, <8 x i16> %a1) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_x86_avx512_psrli_w_512(<32 x i16> %a0) #0 {
+; CHECK-LABEL: @test_x86_avx512_psrli_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    store <32 x i16> [[TMP3]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  ret <32 x i16> %res
+}
+define <32 x i16> @test_x86_avx512_mask_psrli_w_512(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_mask_psrli_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <32 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP5]], <32 x i16> [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <32 x i16> [[RES]], [[PASSTHRU:%.*]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i16> [[TMP9]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP6]], <32 x i16> [[TMP10]], <32 x i16> [[TMP7]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> [[PASSTHRU]]
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> %passthru
+  ret <32 x i16> %res2
+}
+define <32 x i16> @test_x86_avx512_maskz_psrli_w_512(<32 x i16> %a0, i32 %mask) #0 {
+; CHECK-LABEL: @test_x86_avx512_maskz_psrli_w_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[TMP1]], i32 7)
+; CHECK-NEXT:    [[TMP4:%.*]] = or <32 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> [[A0:%.*]], i32 7)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP2]] to <32 x i1>
+; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[TMP4]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <32 x i16> [[RES]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP9]], <32 x i16> [[TMP6]]
+; CHECK-NEXT:    [[RES2:%.*]] = select <32 x i1> [[MASK_CAST]], <32 x i16> [[RES]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    store <32 x i16> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <32 x i16> [[RES2]]
+;
+  %res = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %a0, i32 7) ; <<32 x i16>> [#uses=1]
+  %mask.cast = bitcast i32 %mask to <32 x i1>
+  %res2 = select <32 x i1> %mask.cast, <32 x i16> %res, <32 x i16> zeroinitializer
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16>, i32) nounwind readnone
+
+attributes #0 = { sanitize_memory }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
index 14d68b449a7b6..7d45cec721246 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
@@ -1901,11 +1901,28 @@ define <4 x i32>@test_int_x86_avx512_vpermi2var_d_128(<4 x i32> %x0, <4 x i32> %
 ; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpermi2var_d_128(
 ; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP14]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP6]], <4 x i32> [[X1]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
@@ -1919,12 +1936,29 @@ define <4 x i32>@test_int_x86_avx512_mask_vpermi2var_d_128(<4 x i32> %x0, <4 x i
 ; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_vpermi2var_d_128(
 ; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[_MSPROP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP9]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP8]], <4 x i32> [[X1]], <4 x i32> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -1950,11 +1984,28 @@ define <4 x i32>@test_int_x86_avx512_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %
 ; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpermt2var_d_128(
 ; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP14]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP6]], <4 x i32> [[X0]], <4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X1]], <4 x i32> [[X0]], <4 x i32> [[X2]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
@@ -1968,12 +2019,29 @@ define <4 x i32>@test_int_x86_avx512_mask_vpermt2var_d_128(<4 x i32> %x0, <4 x i
 ; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_mask_vpermt2var_d_128(
 ; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[_MSPROP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP9]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP8]], <4 x i32> [[X0]], <4 x i32> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X1]], <4 x i32> [[X0]], <4 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2000,12 +2068,29 @@ define <4 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_128(<4 x i32> %x0, <4 x
 ; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_maskz_vpermt2var_d_128(
 ; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[_MSPROP1]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 3
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP8]], <4 x i32> [[X0]], <4 x i32> [[TMP9]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X1]], <4 x i32> [[X0]], <4 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2033,11 +2118,40 @@ define <8 x i32>@test_int_x86_avx512_vpermi2var_d_256(<8 x i32> %x0, <8 x i32> %
 ; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpermi2var_d_256(
 ; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP22]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP6]], <8 x i32> [[X1]], <8 x i32> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB29:.*]], label %[[BB30:.*]], !prof [[PROF1]]
+; CHECK:       [[BB29]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB30]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
@@ -2051,12 +2165,41 @@ define <8 x i32>@test_int_x86_avx512_mask_vpermi2var_d_256(<8 x i32> %x0, <8 x i
 ; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_vpermi2var_d_256(
 ; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP9]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP9]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i32 [[TMP13]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP29]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP8]], <8 x i32> [[X1]], <8 x i32> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB30:.*]], label %[[BB31:.*]], !prof [[PROF1]]
+; CHECK:       [[BB30]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB31]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2079,11 +2222,40 @@ define <8 x i32>@test_int_x86_avx512_ask_vpermt2var_d_256(<8 x i32> %x0, <8 x i3
 ; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_ask_vpermt2var_d_256(
 ; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[TMP22]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP6]], <8 x i32> [[X0]], <8 x i32> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB29:.*]], label %[[BB30:.*]], !prof [[PROF1]]
+; CHECK:       [[BB29]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB30]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X1]], <8 x i32> [[X0]], <8 x i32> [[X2]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
@@ -2097,12 +2269,41 @@ define <8 x i32>@test_int_x86_avx512_mask_vpermt2var_d_256(<8 x i32> %x0, <8 x i
 ; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_mask_vpermt2var_d_256(
 ; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP9]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP9]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i32 [[TMP13]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP29]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP8]], <8 x i32> [[X0]], <8 x i32> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB30:.*]], label %[[BB31:.*]], !prof [[PROF1]]
+; CHECK:       [[BB30]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB31]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X1]], <8 x i32> [[X0]], <8 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2126,12 +2327,41 @@ define <8 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_256(<8 x i32> %x0, <8 x
 ; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_maskz_vpermt2var_d_256(
 ; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP9]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP34:%.*]] = or i32 [[TMP29]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP35:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP17]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP8]], <8 x i32> [[X0]], <8 x i32> [[TMP9]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP37]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB30:.*]], label %[[BB31:.*]], !prof [[PROF1]]
+; CHECK:       [[BB30]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB31]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X1]], <8 x i32> [[X0]], <8 x i32> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2156,24 +2386,27 @@ define <2 x double>@test_int_x86_avx512_vpermi2var_pd_128(<2 x double> %x0, <2 x
 ; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_vpermi2var_pd_128(
 ; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x double> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP9]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x i64> [[TMP4]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP6]], [[TMP13]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <2 x i64> [[TMP9]] to <2 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[TMP4]] to <2 x double>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[TMP8]], <2 x i64> [[X1]], <2 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x double> [[TMP10]] to <2 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB14:.*]], label %[[BB15:.*]], !prof [[PROF1]]
+; CHECK:       [[BB14]]:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB15]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[X0]], <2 x i64> [[X1]], <2 x double> [[X2]])
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <2 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2)
@@ -2185,34 +2418,37 @@ define <2 x double>@test_int_x86_avx512_mask_vpermi2var_pd_128(<2 x double> %x0,
 ; CHECK-LABEL: define <2 x double> @test_int_x86_avx512_mask_vpermi2var_pd_128(
 ; CHECK-SAME: <2 x double> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP11]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP13]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <2 x i64> [[TMP8]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP15]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i64> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], 1
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <2 x i64> [[TMP11]] to <2 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i64> [[TMP8]] to <2 x double>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[TMP9]], <2 x i64> [[X1]], <2 x double> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <2 x double> [[TMP17]] to <2 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB15:.*]], label %[[BB16:.*]], !prof [[PROF1]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB16]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> [[X0]], <2 x i64> [[X1]], <2 x double> [[X2]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[X1]] to <2 x double>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> zeroinitializer, <2 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[EXTRACT]], <2 x i64> [[TMP18]], <2 x i64> [[TMP13]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x double> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <2 x i64> [[TMP7]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = or <2 x i64> [[TMP20]], [[TMP13]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <2 x i1> [[_MSPROP]], <2 x i64> [[TMP21]], <2 x i64> [[TMP16]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[EXTRACT]], <2 x double> [[TMP1]], <2 x double> [[TMP2]]
@@ -2233,24 +2469,33 @@ define <4 x double>@test_int_x86_avx512_vpermi2var_pd_256(<4 x double> %x0, <4 x
 ; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_vpermi2var_pd_256(
 ; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x double> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP9]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP3]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i64> [[TMP4]] to i256
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP6]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP17]], [[TMP11]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i64> [[TMP9]] to <4 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i64> [[TMP4]] to <4 x double>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[TMP8]], <4 x i64> [[X1]], <4 x double> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x double> [[TMP10]] to <4 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP21]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[X0]], <4 x i64> [[X1]], <4 x double> [[X2]])
-; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <4 x i64> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %1 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2)
@@ -2262,34 +2507,43 @@ define <4 x double>@test_int_x86_avx512_mask_vpermi2var_pd_256(<4 x double> %x0,
 ; CHECK-LABEL: define <4 x double> @test_int_x86_avx512_mask_vpermi2var_pd_256(
 ; CHECK-SAME: <4 x double> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x double> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP11]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP13]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i64> [[TMP8]] to i256
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP15]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i64> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP15]], 3
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP15]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], 3
+; CHECK-NEXT:    [[TMP29:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 3
+; CHECK-NEXT:    [[TMP30:%.*]] = or i64 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i64> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 3
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i64> [[TMP11]] to <4 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i64> [[TMP8]] to <4 x double>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[TMP9]], <4 x i64> [[X1]], <4 x double> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x double> [[TMP17]] to <4 x i64>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP31]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> [[X0]], <4 x i64> [[X1]], <4 x double> [[X2]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[X1]] to <4 x double>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> zeroinitializer, <4 x i64> [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP18]], <4 x i64> [[TMP13]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double> [[TMP1]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[TMP2]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i64> [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i64> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i64> [[TMP7]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i64> [[TMP20]], [[TMP13]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i64> [[TMP21]], <4 x i64> [[TMP16]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[TMP2]]
@@ -2310,24 +2564,33 @@ define <4 x float>@test_int_x86_avx512_vpermi2var_ps_128(<4 x float> %x0, <4 x i
 ; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_vpermi2var_ps_128(
 ; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x float> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP9]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP4]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP6]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP6]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP17]], [[TMP11]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP9]] to <4 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[TMP8]], <4 x i32> [[X1]], <4 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x float> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP21]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[X0]], <4 x i32> [[X1]], <4 x float> [[X2]])
-; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2)
@@ -2339,34 +2602,43 @@ define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128(<4 x float> %x0, <
 ; CHECK-LABEL: define <4 x float> @test_int_x86_avx512_mask_vpermi2var_ps_128(
 ; CHECK-SAME: <4 x float> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP11]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP13]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i32> [[TMP8]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP15]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP15]], 3
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP15]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i32> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 3
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i32> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 3
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i32> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 3
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP11]] to <4 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP8]] to <4 x float>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[TMP9]], <4 x i32> [[X1]], <4 x float> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x float> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP31]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB21:.*]], label %[[BB22:.*]], !prof [[PROF1]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB22]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[X0]], <4 x i32> [[X1]], <4 x float> [[X2]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[X1]] to <4 x float>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> zeroinitializer, <4 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP18]], <4 x i32> [[TMP13]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP7]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i32> [[TMP20]], [[TMP13]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP21]], <4 x i32> [[TMP16]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[TMP2]]
@@ -2392,30 +2664,39 @@ define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128_cast(<4 x float> %
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <2 x i64> [[TMP11]] to <4 x i32>
 ; CHECK-NEXT:    [[X1CAST:%.*]] = bitcast <2 x i64> [[X1]] to <4 x i32>
-; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i32> [[TMP12]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP19]], 0
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x i32> [[TMP14]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP9]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> [[TMP13]] to i128
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP8]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
-; CHECK:       [[BB9]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB10]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP20]], [[TMP23]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP8]], 3
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP8]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i32> [[TMP14]], i64 2
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 3
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i32> [[TMP14]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP27]], 3
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> [[TMP12]] to <4 x float>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <4 x i32> [[TMP13]] to <4 x float>
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[TMP16]], <4 x i32> [[X1CAST]], <4 x float> [[TMP18]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x float> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP32]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB22:.*]], label %[[BB23:.*]], !prof [[PROF1]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB23]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> [[X0]], <4 x i32> [[X1CAST]], <4 x float> [[X2]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[X1CAST]] to <4 x float>
 ; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP17:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> zeroinitializer, <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP9]], <4 x i32> [[TMP14]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i32> [[TMP7]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <4 x i32> [[TMP21]], [[TMP14]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP22]], <4 x i32> [[TMP17]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[EXTRACT]], <4 x float> [[TMP1]], <4 x float> [[TMP2]]
@@ -2437,24 +2718,45 @@ define <8 x float>@test_int_x86_avx512_vpermi2var_ps_256(<8 x float> %x0, <8 x i
 ; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_vpermi2var_ps_256(
 ; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x float> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP9]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP3]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP4]] to i256
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP22:%.*]] = and i32 [[TMP6]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP6]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP24:%.*]] = and i32 [[TMP23]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP25]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP3]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP3]], i64 5
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP3]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP3]], i64 7
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i32> [[TMP9]] to <8 x float>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[TMP8]], <8 x i32> [[X1]], <8 x float> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x float> [[TMP10]] to <8 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB32:.*]], label %[[BB33:.*]], !prof [[PROF1]]
+; CHECK:       [[BB32]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB33]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[X0]], <8 x i32> [[X1]], <8 x float> [[X2]])
-; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i32> [[TMP7]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %1 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2)
@@ -2466,32 +2768,53 @@ define <8 x float>@test_int_x86_avx512_mask_vpermi2var_ps_256(<8 x float> %x0, <
 ; CHECK-LABEL: define <8 x float> @test_int_x86_avx512_mask_vpermi2var_ps_256(
 ; CHECK-SAME: <8 x float> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x float> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP11]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP13]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <8 x i32> [[TMP8]] to i256
-; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP15]], 0
-; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP13]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP36:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP37:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP13]], i64 2
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 7
+; CHECK-NEXT:    [[TMP38:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP13]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 7
+; CHECK-NEXT:    [[TMP39:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <8 x i32> [[TMP13]], i64 4
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 7
+; CHECK-NEXT:    [[TMP40:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP13]], i64 5
+; CHECK-NEXT:    [[TMP32:%.*]] = and i32 [[TMP15]], 7
+; CHECK-NEXT:    [[TMP41:%.*]] = or i32 [[TMP15]], [[TMP32]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP13]], i64 6
+; CHECK-NEXT:    [[TMP34:%.*]] = and i32 [[TMP33]], 7
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP13]], i64 7
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP19]], [[TMP35]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <8 x i32> [[TMP11]] to <8 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
+; CHECK-NEXT:    [[TMP17:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[TMP9]], <8 x i32> [[X1]], <8 x float> [[TMP12]])
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x float> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP43]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB33:.*]], label %[[BB34:.*]], !prof [[PROF1]]
+; CHECK:       [[BB33]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB34]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> [[X0]], <8 x i32> [[X1]], <8 x float> [[X2]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i32> [[X1]] to <8 x float>
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8 [[X3]] to <8 x i1>
-; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> zeroinitializer, <8 x i32> [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP18]], <8 x i32> [[TMP13]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x float> [[TMP1]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[TMP2]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i32> [[TMP7]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i32> [[TMP20]], [[TMP13]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP14]], <8 x i32> [[TMP21]], <8 x i32> [[TMP16]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x float> [[TMP1]], <8 x float> [[TMP2]]
@@ -2511,11 +2834,22 @@ define <2 x i64>@test_int_x86_avx512_vpermi2var_q_128(<2 x i64> %x0, <2 x i64> %
 ; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_vpermi2var_q_128(
 ; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = or i64 [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[X1]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]])
 ; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
@@ -2529,12 +2863,23 @@ define <2 x i64>@test_int_x86_avx512_mask_vpermi2var_q_128(<2 x i64> %x0, <2 x i
 ; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_mask_vpermi2var_q_128(
 ; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[_MSPROP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP9]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP8]], <2 x i64> [[X1]], <2 x i64> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[X1]], <2 x i64> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2560,11 +2905,22 @@ define <2 x i64>@test_int_x86_avx512_vpermt2var_q_128(<2 x i64> %x0, <2 x i64> %
 ; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_vpermt2var_q_128(
 ; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = or i64 [[TMP10]], [[TMP7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[X0]], <2 x i64> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB11:.*]], label %[[BB12:.*]], !prof [[PROF1]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB12]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X1]], <2 x i64> [[X0]], <2 x i64> [[X2]])
 ; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
@@ -2578,12 +2934,23 @@ define <2 x i64>@test_int_x86_avx512_mask_vpermt2var_q_128(<2 x i64> %x0, <2 x i
 ; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_mask_vpermt2var_q_128(
 ; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i64> [[_MSPROP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP9]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP8]], <2 x i64> [[X0]], <2 x i64> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X1]], <2 x i64> [[X0]], <2 x i64> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2610,12 +2977,23 @@ define <2 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_128(<2 x i64> %x0, <2 x
 ; CHECK-LABEL: define <2 x i64> @test_int_x86_avx512_maskz_vpermt2var_q_128(
 ; CHECK-SAME: <2 x i64> [[X0:%.*]], <2 x i64> [[X1:%.*]], <2 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i64> [[_MSPROP1]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP8]], <2 x i64> [[X0]], <2 x i64> [[TMP9]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP19]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X1]], <2 x i64> [[X0]], <2 x i64> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2643,11 +3021,28 @@ define <4 x i64>@test_int_x86_avx512_vpermi2var_q_256(<4 x i64> %x0, <4 x i64> %
 ; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_vpermi2var_q_256(
 ; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP14]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP6]], <4 x i64> [[X1]], <4 x i64> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]])
 ; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
@@ -2661,12 +3056,29 @@ define <4 x i64>@test_int_x86_avx512_mask_vpermi2var_q_256(<4 x i64> %x0, <4 x i
 ; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_mask_vpermi2var_q_256(
 ; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[_MSPROP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP9]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP8]], <4 x i64> [[X1]], <4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> [[X1]], <4 x i64> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2692,11 +3104,28 @@ define <4 x i64>@test_int_x86_avx512_vpermt2var_q_256(<4 x i64> %x0, <4 x i64> %
 ; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_vpermt2var_q_256(
 ; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP14]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP6]], <4 x i64> [[X0]], <4 x i64> [[TMP5]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X1]], <4 x i64> [[X0]], <4 x i64> [[X2]])
 ; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
@@ -2710,12 +3139,29 @@ define <4 x i64>@test_int_x86_avx512_mask_vpermt2var_q_256(<4 x i64> %x0, <4 x i
 ; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_mask_vpermt2var_q_256(
 ; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i64> [[_MSPROP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 3
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP9]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP8]], <4 x i64> [[X0]], <4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X1]], <4 x i64> [[X0]], <4 x i64> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -2742,12 +3188,29 @@ define <4 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_256(<4 x i64> %x0, <4 x
 ; CHECK-LABEL: define <4 x i64> @test_int_x86_avx512_maskz_vpermt2var_q_256(
 ; CHECK-SAME: <4 x i64> [[X0:%.*]], <4 x i64> [[X1:%.*]], <4 x i64> [[X2:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[TMP8]], [[TMP3]]
-; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i64> [[_MSPROP1]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 3
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP8]], <4 x i64> [[X0]], <4 x i64> [[TMP9]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP25]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB18:.*]], label %[[BB19:.*]], !prof [[PROF1]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB19]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X1]], <4 x i64> [[X0]], <4 x i64> [[X2]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP11]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[X3]] to <8 x i1>
@@ -12267,8 +12730,7 @@ define <8 x i32> @combine_vpermi2d_vpermps(<16 x i32> noundef %a) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> splat (i32 -1), <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[_MSPROP]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = or <8 x i32> [[_MSPROP2]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[_MSPROP]], <8 x i32> <i32 14, i32 13, i32 6, i32 3, i32 5, i32 15, i32 0, i32 1>, <8 x i32> [[_MSPROP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP1]], <8 x i32> <i32 14, i32 13, i32 6, i32 3, i32 5, i32 15, i32 0, i32 1>, <8 x i32> [[TMP2]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll
index 2350d75b29b44..95c1bbf251a6a 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll
@@ -16,8 +16,7 @@ define <2 x i64> @shuffle_vpermv3_v2i64(<2 x i64> %x0, <2 x i64> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP1]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[X1]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
@@ -31,8 +30,7 @@ define <2 x i64> @shuffle_vpermv3_v2i64_unary(<2 x i64> %x0) #0 {
 ; CHECK-SAME: <2 x i64> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP1]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> <i64 2, i64 0>, <2 x i64> [[X0]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
@@ -55,8 +53,19 @@ define <2 x i64> @shuffle_vpermv3_v2i64_demandedbits(<2 x i64> %x0, <2 x i64> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i64> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <2 x i64> [[M]], <i64 0, i64 4>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i64> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[T]], <2 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[T]], <2 x i64> [[X1]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
@@ -80,8 +89,19 @@ define <2 x i64> @shuffle_vpermv3_v2i64_demandedbits_negative(<2 x i64> %x0, <2
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i64> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i64> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <2 x i64> [[M]], <i64 0, i64 2>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <2 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i64> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[TMP6]], <2 x i64> [[T]], <2 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB17:.*]], label %[[BB18:.*]], !prof [[PROF1]]
+; CHECK:       [[BB17]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB18]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.x86.avx512.vpermi2var.q.128(<2 x i64> [[X0]], <2 x i64> [[T]], <2 x i64> [[X1]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
@@ -97,8 +117,7 @@ define <4 x i64> @shuffle_vpermv3_v4i64(<4 x i64> %x0, <4 x i64> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP1]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[X1]])
 ; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[R]]
@@ -112,8 +131,7 @@ define <4 x i64> @shuffle_vpermv3_v4i64_unary(<4 x i64> %x0) #0 {
 ; CHECK-SAME: <4 x i64> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP1]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> <i64 7, i64 2, i64 6, i64 0>, <4 x i64> [[X0]])
 ; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[R]]
@@ -136,8 +154,25 @@ define <4 x i64> @shuffle_vpermv3_v4i64_demandedbits(<4 x i64> %x0, <4 x i64> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i64> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i64> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <4 x i64> [[M]], <i64 0, i64 8, i64 16, i64 32>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i64> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 3
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[TMP6]], <4 x i64> [[T]], <4 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP21]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB23:.*]], label %[[BB24:.*]], !prof [[PROF1]]
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB24]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <4 x i64> @llvm.x86.avx512.vpermi2var.q.256(<4 x i64> [[X0]], <4 x i64> [[T]], <4 x i64> [[X1]])
 ; CHECK-NEXT:    store <4 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i64> [[R]]
@@ -153,8 +188,7 @@ define <8 x i64> @shuffle_vpermv3_v8i64(<8 x i64> %x0, <8 x i64> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[X1]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[R]]
@@ -168,8 +202,7 @@ define <8 x i64> @shuffle_vpermv3_v8i64_unary(<8 x i64> %x0) #0 {
 ; CHECK-SAME: <8 x i64> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP1]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0]], <8 x i64> <i64 8, i64 6, i64 10, i64 4, i64 12, i64 2, i64 14, i64 0>, <8 x i64> [[X0]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[R]]
@@ -192,8 +225,37 @@ define <8 x i64> @shuffle_vpermv3_v8i64_demandedbits(<8 x i64> %x0, <8 x i64> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i64> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i64> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <8 x i64> [[M]], <i64 0, i64 16, i64 32, i64 64, i64 256, i64 512, i64 1024, i64 -16>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i64> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i64> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i64> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i64 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i64> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i64> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i64 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[TMP6]], <8 x i64> [[T]], <8 x i64> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP33]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB35:.*]], label %[[BB36:.*]], !prof [[PROF1]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB36]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> [[X0]], <8 x i64> [[T]], <8 x i64> [[X1]])
 ; CHECK-NEXT:    store <8 x i64> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i64> [[R]]
@@ -213,8 +275,7 @@ define <4 x i32> @shuffle_vpermv3_v4i32(<4 x i32> %x0, <4 x i32> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP1]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[X1]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -228,8 +289,7 @@ define <4 x i32> @shuffle_vpermv3_v4i32_unary(<4 x i32> %x0) #0 {
 ; CHECK-SAME: <4 x i32> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP1]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> <i32 7, i32 2, i32 6, i32 0>, <4 x i32> [[X0]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -252,8 +312,25 @@ define <4 x i32> @shuffle_vpermv3_v4i32_demandedbits(<4 x i32> %x0, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <4 x i32> [[M]], <i32 0, i32 8, i32 16, i32 32>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 3
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[TMP6]], <4 x i32> [[T]], <4 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP21]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB23:.*]], label %[[BB24:.*]], !prof [[PROF1]]
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB24]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <4 x i32> @llvm.x86.avx512.vpermi2var.d.128(<4 x i32> [[X0]], <4 x i32> [[T]], <4 x i32> [[X1]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -269,8 +346,7 @@ define <8 x i32> @shuffle_vpermv3_v8i32(<8 x i32> %x0, <8 x i32> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP1]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[X1]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
@@ -284,8 +360,7 @@ define <8 x i32> @shuffle_vpermv3_v8i32_unary(<8 x i32> %x0) #0 {
 ; CHECK-SAME: <8 x i32> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP1]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> <i32 8, i32 6, i32 10, i32 4, i32 12, i32 2, i32 14, i32 0>, <8 x i32> [[X0]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
@@ -308,8 +383,37 @@ define <8 x i32> @shuffle_vpermv3_v8i32_demandedbits(<8 x i32> %x0, <8 x i32> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <8 x i32> [[M]], <i32 0, i32 16, i32 32, i32 64, i32 256, i32 512, i32 -16, i32 -32>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[TMP6]], <8 x i32> [[T]], <8 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP33]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB35:.*]], label %[[BB36:.*]], !prof [[PROF1]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB36]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.x86.avx512.vpermi2var.d.256(<8 x i32> [[X0]], <8 x i32> [[T]], <8 x i32> [[X1]])
 ; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
@@ -325,8 +429,7 @@ define <16 x i32> @shuffle_vpermv3_v16i32(<16 x i32> %x0, <16 x i32> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[X1]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
@@ -340,8 +443,7 @@ define <16 x i32> @shuffle_vpermv3_v16i32_unary(<16 x i32> %x0) #0 {
 ; CHECK-SAME: <16 x i32> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP1]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0]], <16 x i32> <i32 16, i32 14, i32 18, i32 12, i32 20, i32 10, i32 22, i32 8, i32 24, i32 6, i32 26, i32 4, i32 28, i32 2, i32 30, i32 0>, <16 x i32> [[X0]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
@@ -364,8 +466,61 @@ define <16 x i32> @shuffle_vpermv3_v16i32_demandedbits(<16 x i32> %x0, <16 x i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <16 x i32> [[M]], <i32 0, i32 32, i32 64, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 -32, i32 -64, i32 -128, i32 -256, i32 -512, i32 -1024, i32 -2048>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i32> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i32> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i32> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i32 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i32> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i32> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i32> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i32> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i32> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i32 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i32 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i32> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i32 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i32> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i32 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i32 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i32> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i32 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i32 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i32 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i32 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i32> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i32 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i32> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i32 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i32> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i32 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i32 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i32> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i32 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[TMP6]], <16 x i32> [[T]], <16 x i32> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB59:.*]], label %[[BB60:.*]], !prof [[PROF1]]
+; CHECK:       [[BB59]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB60]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> [[X0]], <16 x i32> [[T]], <16 x i32> [[X1]])
 ; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
@@ -385,8 +540,7 @@ define <8 x i16> @shuffle_vpermv3_v8i16(<8 x i16> %x0, <8 x i16> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[TMP1]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[X0]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[X1]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
@@ -400,8 +554,7 @@ define <8 x i16> @shuffle_vpermv3_v8i16_unary(<8 x i16> %x0) #0 {
 ; CHECK-SAME: <8 x i16> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[TMP1]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[X0]], <8 x i16> <i16 8, i16 6, i16 10, i16 4, i16 12, i16 2, i16 14, i16 0>, <8 x i16> [[X0]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
@@ -424,8 +577,37 @@ define <8 x i16> @shuffle_vpermv3_v8i16_demandedbits(<8 x i16> %x0, <8 x i16> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i16> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <8 x i16> [[M]], <i16 0, i16 16, i16 32, i16 64, i16 256, i16 512, i16 -16, i16 -32>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP26:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i16> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i16 [[TMP12]], 7
+; CHECK-NEXT:    [[TMP28:%.*]] = or i16 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i16> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i16> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i16> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i16 [[TMP18]], 7
+; CHECK-NEXT:    [[TMP32:%.*]] = or i16 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i16> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i16> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP30:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i16> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i16 [[TMP24]], 7
+; CHECK-NEXT:    [[TMP33:%.*]] = or i16 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[TMP6]], <8 x i16> [[T]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP33]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB35:.*]], label %[[BB36:.*]], !prof [[PROF1]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB36]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.x86.avx512.vpermi2var.hi.128(<8 x i16> [[X0]], <8 x i16> [[T]], <8 x i16> [[X1]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
@@ -441,8 +623,7 @@ define <16 x i16> @shuffle_vpermv3_v16i16(<16 x i16> %x0, <16 x i16> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[TMP1]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[X0]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[X1]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
@@ -456,8 +637,7 @@ define <16 x i16> @shuffle_vpermv3_v16i16_unary(<16 x i16> %x0) #0 {
 ; CHECK-SAME: <16 x i16> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[TMP1]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[X0]], <16 x i16> <i16 16, i16 14, i16 18, i16 12, i16 20, i16 10, i16 22, i16 8, i16 24, i16 6, i16 26, i16 4, i16 28, i16 2, i16 30, i16 0>, <16 x i16> [[X0]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
@@ -480,8 +660,61 @@ define <16 x i16> @shuffle_vpermv3_v16i16_demandedbits(<16 x i16> %x0, <16 x i16
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i16> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <16 x i16> [[M]], <i16 0, i16 32, i16 64, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 -32, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i16> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i16 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i16 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i16 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i16 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i16> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i16> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i16> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i16 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i16 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i16> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i16> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i16 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i16 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i16> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i16 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i16 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i16> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i16> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i16 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i16 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i16> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i16 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i16 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i16> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i16> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i16 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i16 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[TMP6]], <16 x i16> [[T]], <16 x i16> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB59:.*]], label %[[BB60:.*]], !prof [[PROF1]]
+; CHECK:       [[BB59]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB60]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i16> @llvm.x86.avx512.vpermi2var.hi.256(<16 x i16> [[X0]], <16 x i16> [[T]], <16 x i16> [[X1]])
 ; CHECK-NEXT:    store <16 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
@@ -497,8 +730,7 @@ define <32 x i16> @shuffle_vpermv3_v32i16(<32 x i16> %x0, <32 x i16> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[X1]])
 ; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i16> [[R]]
@@ -512,8 +744,7 @@ define <32 x i16> @shuffle_vpermv3_v32i16_unary(<32 x i16> %x0) #0 {
 ; CHECK-SAME: <32 x i16> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP1]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0]], <32 x i16> <i16 33, i16 17, i16 35, i16 19, i16 37, i16 21, i16 39, i16 23, i16 41, i16 25, i16 43, i16 27, i16 45, i16 29, i16 47, i16 31, i16 49, i16 14, i16 51, i16 12, i16 53, i16 10, i16 55, i16 8, i16 57, i16 6, i16 59, i16 4, i16 61, i16 2, i16 63, i16 0>, <32 x i16> [[X0]])
 ; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i16> [[R]]
@@ -536,8 +767,109 @@ define <32 x i16> @shuffle_vpermv3_v32i16_demandedbits(<32 x i16> %x0, <32 x i16
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i16> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <32 x i16> [[M]], <i16 0, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 0, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096, i16 0, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 0, i16 -64, i16 -128, i16 -256, i16 -512, i16 -1024, i16 -2048, i16 -4096>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i16> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i16> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <32 x i16> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i16 [[TMP10]], 31
+; CHECK-NEXT:    [[TMP74:%.*]] = or i16 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <32 x i16> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i16 [[TMP12]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i16 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i16> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i16 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP77:%.*]] = or i16 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <32 x i16> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i16 [[TMP16]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i16 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <32 x i16> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i16 [[TMP18]], 31
+; CHECK-NEXT:    [[TMP80:%.*]] = or i16 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i16> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i16 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i16 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <32 x i16> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i16 [[TMP22]], 31
+; CHECK-NEXT:    [[TMP83:%.*]] = or i16 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <32 x i16> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i16 [[TMP24]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i16 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i16> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i16 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP86:%.*]] = or i16 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <32 x i16> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i16 [[TMP28]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i16 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <32 x i16> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i16 [[TMP30]], 31
+; CHECK-NEXT:    [[TMP89:%.*]] = or i16 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i16> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i16 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i16 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <32 x i16> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i16 [[TMP34]], 31
+; CHECK-NEXT:    [[TMP92:%.*]] = or i16 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <32 x i16> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i16 [[TMP36]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i16 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i16> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i16 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP95:%.*]] = or i16 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <32 x i16> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i16 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i16 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <32 x i16> [[TMP9]], i64 16
+; CHECK-NEXT:    [[TMP43:%.*]] = and i16 [[TMP42]], 31
+; CHECK-NEXT:    [[TMP98:%.*]] = or i16 [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i16> [[TMP9]], i64 17
+; CHECK-NEXT:    [[TMP45:%.*]] = and i16 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i16 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <32 x i16> [[TMP9]], i64 18
+; CHECK-NEXT:    [[TMP47:%.*]] = and i16 [[TMP46]], 31
+; CHECK-NEXT:    [[TMP101:%.*]] = or i16 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <32 x i16> [[TMP9]], i64 19
+; CHECK-NEXT:    [[TMP49:%.*]] = and i16 [[TMP48]], 31
+; CHECK-NEXT:    [[TMP103:%.*]] = or i16 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i16> [[TMP9]], i64 20
+; CHECK-NEXT:    [[TMP51:%.*]] = and i16 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP105:%.*]] = or i16 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i16> [[TMP9]], i64 21
+; CHECK-NEXT:    [[TMP53:%.*]] = and i16 [[TMP52]], 31
+; CHECK-NEXT:    [[TMP75:%.*]] = or i16 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <32 x i16> [[TMP9]], i64 22
+; CHECK-NEXT:    [[TMP55:%.*]] = and i16 [[TMP54]], 31
+; CHECK-NEXT:    [[TMP78:%.*]] = or i16 [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i16> [[TMP9]], i64 23
+; CHECK-NEXT:    [[TMP57:%.*]] = and i16 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = or i16 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i16> [[TMP9]], i64 24
+; CHECK-NEXT:    [[TMP59:%.*]] = and i16 [[TMP58]], 31
+; CHECK-NEXT:    [[TMP84:%.*]] = or i16 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <32 x i16> [[TMP9]], i64 25
+; CHECK-NEXT:    [[TMP61:%.*]] = and i16 [[TMP60]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = or i16 [[TMP60]], [[TMP61]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i16> [[TMP9]], i64 26
+; CHECK-NEXT:    [[TMP63:%.*]] = and i16 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP90:%.*]] = or i16 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i16> [[TMP9]], i64 27
+; CHECK-NEXT:    [[TMP65:%.*]] = and i16 [[TMP64]], 31
+; CHECK-NEXT:    [[TMP93:%.*]] = or i16 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <32 x i16> [[TMP9]], i64 28
+; CHECK-NEXT:    [[TMP67:%.*]] = and i16 [[TMP66]], 31
+; CHECK-NEXT:    [[TMP96:%.*]] = or i16 [[TMP66]], [[TMP67]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i16> [[TMP9]], i64 29
+; CHECK-NEXT:    [[TMP69:%.*]] = and i16 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP99:%.*]] = or i16 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <32 x i16> [[TMP9]], i64 30
+; CHECK-NEXT:    [[TMP71:%.*]] = and i16 [[TMP70]], 31
+; CHECK-NEXT:    [[TMP102:%.*]] = or i16 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <32 x i16> [[TMP9]], i64 31
+; CHECK-NEXT:    [[TMP104:%.*]] = and i16 [[TMP72]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i16 [[TMP72]], [[TMP104]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[TMP6]], <32 x i16> [[T]], <32 x i16> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP60:%.*]] = icmp ne i16 [[TMP73]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP60]], label %[[BB107:.*]], label %[[BB108:.*]], !prof [[PROF1]]
+; CHECK:       [[BB107]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB108]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> [[X0]], <32 x i16> [[T]], <32 x i16> [[X1]])
 ; CHECK-NEXT:    store <32 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i16> [[R]]
@@ -557,8 +889,7 @@ define <16 x i8> @shuffle_vpermv3_v16i8(<16 x i8> %x0, <16 x i8> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i8> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[TMP1]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[X0]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[X1]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[R]]
@@ -572,8 +903,7 @@ define <16 x i8> @shuffle_vpermv3_v16i8_unary(<16 x i8> %x0) #0 {
 ; CHECK-SAME: <16 x i8> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i8> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[TMP1]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[X0]], <16 x i8> <i8 16, i8 14, i8 18, i8 12, i8 20, i8 10, i8 22, i8 8, i8 24, i8 6, i8 26, i8 4, i8 28, i8 2, i8 30, i8 0>, <16 x i8> [[X0]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[R]]
@@ -596,8 +926,61 @@ define <16 x i8> @shuffle_vpermv3_v16i8_demandedbits(<16 x i8> %x0, <16 x i8> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i8> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i8> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <16 x i8> [[M]], <i8 0, i8 32, i8 64, i8 -128, i8 0, i8 -32, i8 -64, i8 -128, i8 0, i8 32, i8 64, i8 -128, i8 0, i8 -32, i8 -64, i8 -128>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i8> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i8> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i8 [[TMP10]], 15
+; CHECK-NEXT:    [[TMP43:%.*]] = or i8 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i8> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i8 [[TMP12]], 15
+; CHECK-NEXT:    [[TMP44:%.*]] = or i8 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i8> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i8 [[TMP14]], 15
+; CHECK-NEXT:    [[TMP46:%.*]] = or i8 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i8> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i8 [[TMP16]], 15
+; CHECK-NEXT:    [[TMP47:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i8> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i8 [[TMP18]], 15
+; CHECK-NEXT:    [[TMP49:%.*]] = or i8 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i8> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i8 [[TMP20]], 15
+; CHECK-NEXT:    [[TMP50:%.*]] = or i8 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i8> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i8 [[TMP22]], 15
+; CHECK-NEXT:    [[TMP52:%.*]] = or i8 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i8> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i8 [[TMP24]], 15
+; CHECK-NEXT:    [[TMP53:%.*]] = or i8 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i8> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i8 [[TMP26]], 15
+; CHECK-NEXT:    [[TMP55:%.*]] = or i8 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i8> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i8 [[TMP28]], 15
+; CHECK-NEXT:    [[TMP56:%.*]] = or i8 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i8> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i8 [[TMP30]], 15
+; CHECK-NEXT:    [[TMP42:%.*]] = or i8 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i8> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i8 [[TMP32]], 15
+; CHECK-NEXT:    [[TMP45:%.*]] = or i8 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i8> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i8 [[TMP34]], 15
+; CHECK-NEXT:    [[TMP48:%.*]] = or i8 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i8> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i8 [[TMP36]], 15
+; CHECK-NEXT:    [[TMP51:%.*]] = or i8 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i8> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i8 [[TMP38]], 15
+; CHECK-NEXT:    [[TMP54:%.*]] = or i8 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i8> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i8 [[TMP40]], 15
+; CHECK-NEXT:    [[TMP57:%.*]] = or i8 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[TMP6]], <16 x i8> [[T]], <16 x i8> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i8 [[TMP57]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB59:.*]], label %[[BB60:.*]], !prof [[PROF1]]
+; CHECK:       [[BB59]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB60]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> [[X0]], <16 x i8> [[T]], <16 x i8> [[X1]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[R]]
@@ -613,8 +996,7 @@ define <32 x i8> @shuffle_vpermv3_v32i8(<32 x i8> %x0, <32 x i8> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i8> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[TMP1]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[X0]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[X1]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[R]]
@@ -628,8 +1010,7 @@ define <32 x i8> @shuffle_vpermv3_v32i8_unary(<32 x i8> %x0) #0 {
 ; CHECK-SAME: <32 x i8> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i8> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[TMP1]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[X0]], <32 x i8> <i8 33, i8 17, i8 35, i8 19, i8 37, i8 21, i8 39, i8 23, i8 41, i8 25, i8 43, i8 27, i8 45, i8 29, i8 47, i8 31, i8 49, i8 14, i8 51, i8 12, i8 53, i8 10, i8 55, i8 8, i8 57, i8 6, i8 59, i8 4, i8 61, i8 2, i8 63, i8 0>, <32 x i8> [[X0]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[R]]
@@ -652,8 +1033,109 @@ define <32 x i8> @shuffle_vpermv3_v32i8_demandedbits(<32 x i8> %x0, <32 x i8> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <32 x i8> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <32 x i8> [[M]], <i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128, i8 0, i8 0, i8 64, i8 -128, i8 0, i8 0, i8 -64, i8 -128>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <32 x i8> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <32 x i8> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i8 [[TMP10]], 31
+; CHECK-NEXT:    [[TMP74:%.*]] = or i8 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <32 x i8> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i8 [[TMP12]], 31
+; CHECK-NEXT:    [[TMP76:%.*]] = or i8 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <32 x i8> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i8 [[TMP14]], 31
+; CHECK-NEXT:    [[TMP77:%.*]] = or i8 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <32 x i8> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i8 [[TMP16]], 31
+; CHECK-NEXT:    [[TMP79:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <32 x i8> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i8 [[TMP18]], 31
+; CHECK-NEXT:    [[TMP80:%.*]] = or i8 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <32 x i8> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i8 [[TMP20]], 31
+; CHECK-NEXT:    [[TMP82:%.*]] = or i8 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <32 x i8> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i8 [[TMP22]], 31
+; CHECK-NEXT:    [[TMP83:%.*]] = or i8 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <32 x i8> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i8 [[TMP24]], 31
+; CHECK-NEXT:    [[TMP85:%.*]] = or i8 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <32 x i8> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i8 [[TMP26]], 31
+; CHECK-NEXT:    [[TMP86:%.*]] = or i8 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <32 x i8> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i8 [[TMP28]], 31
+; CHECK-NEXT:    [[TMP88:%.*]] = or i8 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <32 x i8> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i8 [[TMP30]], 31
+; CHECK-NEXT:    [[TMP89:%.*]] = or i8 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <32 x i8> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i8 [[TMP32]], 31
+; CHECK-NEXT:    [[TMP91:%.*]] = or i8 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <32 x i8> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i8 [[TMP34]], 31
+; CHECK-NEXT:    [[TMP92:%.*]] = or i8 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <32 x i8> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i8 [[TMP36]], 31
+; CHECK-NEXT:    [[TMP94:%.*]] = or i8 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <32 x i8> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i8 [[TMP38]], 31
+; CHECK-NEXT:    [[TMP95:%.*]] = or i8 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <32 x i8> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i8 [[TMP40]], 31
+; CHECK-NEXT:    [[TMP97:%.*]] = or i8 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <32 x i8> [[TMP9]], i64 16
+; CHECK-NEXT:    [[TMP43:%.*]] = and i8 [[TMP42]], 31
+; CHECK-NEXT:    [[TMP98:%.*]] = or i8 [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <32 x i8> [[TMP9]], i64 17
+; CHECK-NEXT:    [[TMP45:%.*]] = and i8 [[TMP44]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = or i8 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <32 x i8> [[TMP9]], i64 18
+; CHECK-NEXT:    [[TMP47:%.*]] = and i8 [[TMP46]], 31
+; CHECK-NEXT:    [[TMP101:%.*]] = or i8 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <32 x i8> [[TMP9]], i64 19
+; CHECK-NEXT:    [[TMP49:%.*]] = and i8 [[TMP48]], 31
+; CHECK-NEXT:    [[TMP103:%.*]] = or i8 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <32 x i8> [[TMP9]], i64 20
+; CHECK-NEXT:    [[TMP51:%.*]] = and i8 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP105:%.*]] = or i8 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <32 x i8> [[TMP9]], i64 21
+; CHECK-NEXT:    [[TMP53:%.*]] = and i8 [[TMP52]], 31
+; CHECK-NEXT:    [[TMP75:%.*]] = or i8 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <32 x i8> [[TMP9]], i64 22
+; CHECK-NEXT:    [[TMP55:%.*]] = and i8 [[TMP54]], 31
+; CHECK-NEXT:    [[TMP78:%.*]] = or i8 [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <32 x i8> [[TMP9]], i64 23
+; CHECK-NEXT:    [[TMP57:%.*]] = and i8 [[TMP56]], 31
+; CHECK-NEXT:    [[TMP81:%.*]] = or i8 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <32 x i8> [[TMP9]], i64 24
+; CHECK-NEXT:    [[TMP59:%.*]] = and i8 [[TMP58]], 31
+; CHECK-NEXT:    [[TMP84:%.*]] = or i8 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <32 x i8> [[TMP9]], i64 25
+; CHECK-NEXT:    [[TMP61:%.*]] = and i8 [[TMP60]], 31
+; CHECK-NEXT:    [[TMP87:%.*]] = or i8 [[TMP60]], [[TMP61]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <32 x i8> [[TMP9]], i64 26
+; CHECK-NEXT:    [[TMP63:%.*]] = and i8 [[TMP62]], 31
+; CHECK-NEXT:    [[TMP90:%.*]] = or i8 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i8> [[TMP9]], i64 27
+; CHECK-NEXT:    [[TMP65:%.*]] = and i8 [[TMP64]], 31
+; CHECK-NEXT:    [[TMP93:%.*]] = or i8 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <32 x i8> [[TMP9]], i64 28
+; CHECK-NEXT:    [[TMP67:%.*]] = and i8 [[TMP66]], 31
+; CHECK-NEXT:    [[TMP96:%.*]] = or i8 [[TMP66]], [[TMP67]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <32 x i8> [[TMP9]], i64 29
+; CHECK-NEXT:    [[TMP69:%.*]] = and i8 [[TMP68]], 31
+; CHECK-NEXT:    [[TMP99:%.*]] = or i8 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <32 x i8> [[TMP9]], i64 30
+; CHECK-NEXT:    [[TMP71:%.*]] = and i8 [[TMP70]], 31
+; CHECK-NEXT:    [[TMP102:%.*]] = or i8 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <32 x i8> [[TMP9]], i64 31
+; CHECK-NEXT:    [[TMP104:%.*]] = and i8 [[TMP72]], 31
+; CHECK-NEXT:    [[TMP73:%.*]] = or i8 [[TMP72]], [[TMP104]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[TMP6]], <32 x i8> [[T]], <32 x i8> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP60:%.*]] = icmp ne i8 [[TMP73]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP60]], label %[[BB107:.*]], label %[[BB108:.*]], !prof [[PROF1]]
+; CHECK:       [[BB107]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB108]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> [[X0]], <32 x i8> [[T]], <32 x i8> [[X1]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[R]]
@@ -669,8 +1151,7 @@ define <64 x i8> @shuffle_vpermv3_v64i8(<64 x i8> %x0, <64 x i8> %x1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <64 x i8> [[_MSPROP]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[TMP1]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[TMP2]])
 ; CHECK-NEXT:    [[R:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[X0]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[X1]])
 ; CHECK-NEXT:    store <64 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <64 x i8> [[R]]
@@ -684,8 +1165,7 @@ define <64 x i8> @shuffle_vpermv3_v64i8_unary(<64 x i8> %x0) #0 {
 ; CHECK-SAME: <64 x i8> [[X0:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <64 x i8> [[_MSPROP]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[TMP1]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[TMP1]])
 ; CHECK-NEXT:    [[R:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[X0]], <64 x i8> <i8 -128, i8 127, i8 126, i8 125, i8 124, i8 123, i8 122, i8 121, i8 120, i8 119, i8 118, i8 115, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <64 x i8> [[X0]])
 ; CHECK-NEXT:    store <64 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <64 x i8> [[R]]
@@ -708,8 +1188,205 @@ define <64 x i8> @shuffle_vpermv3_v64i8_demandedbits(<64 x i8> %x0, <64 x i8> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <64 x i8> [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or <64 x i8> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[T:%.*]] = or <64 x i8> [[M]], <i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128, i8 0, i8 -128>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP6]], [[TMP9]]
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <64 x i8> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <64 x i8> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i8 [[TMP10]], 63
+; CHECK-NEXT:    [[TMP139:%.*]] = or i8 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <64 x i8> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i8 [[TMP12]], 63
+; CHECK-NEXT:    [[TMP140:%.*]] = or i8 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <64 x i8> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = and i8 [[TMP14]], 63
+; CHECK-NEXT:    [[TMP142:%.*]] = or i8 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <64 x i8> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = and i8 [[TMP16]], 63
+; CHECK-NEXT:    [[TMP143:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <64 x i8> [[TMP9]], i64 4
+; CHECK-NEXT:    [[TMP19:%.*]] = and i8 [[TMP18]], 63
+; CHECK-NEXT:    [[TMP145:%.*]] = or i8 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <64 x i8> [[TMP9]], i64 5
+; CHECK-NEXT:    [[TMP21:%.*]] = and i8 [[TMP20]], 63
+; CHECK-NEXT:    [[TMP146:%.*]] = or i8 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <64 x i8> [[TMP9]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i8 [[TMP22]], 63
+; CHECK-NEXT:    [[TMP148:%.*]] = or i8 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <64 x i8> [[TMP9]], i64 7
+; CHECK-NEXT:    [[TMP25:%.*]] = and i8 [[TMP24]], 63
+; CHECK-NEXT:    [[TMP149:%.*]] = or i8 [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <64 x i8> [[TMP9]], i64 8
+; CHECK-NEXT:    [[TMP27:%.*]] = and i8 [[TMP26]], 63
+; CHECK-NEXT:    [[TMP151:%.*]] = or i8 [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <64 x i8> [[TMP9]], i64 9
+; CHECK-NEXT:    [[TMP29:%.*]] = and i8 [[TMP28]], 63
+; CHECK-NEXT:    [[TMP152:%.*]] = or i8 [[TMP28]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <64 x i8> [[TMP9]], i64 10
+; CHECK-NEXT:    [[TMP31:%.*]] = and i8 [[TMP30]], 63
+; CHECK-NEXT:    [[TMP154:%.*]] = or i8 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <64 x i8> [[TMP9]], i64 11
+; CHECK-NEXT:    [[TMP33:%.*]] = and i8 [[TMP32]], 63
+; CHECK-NEXT:    [[TMP155:%.*]] = or i8 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <64 x i8> [[TMP9]], i64 12
+; CHECK-NEXT:    [[TMP35:%.*]] = and i8 [[TMP34]], 63
+; CHECK-NEXT:    [[TMP157:%.*]] = or i8 [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <64 x i8> [[TMP9]], i64 13
+; CHECK-NEXT:    [[TMP37:%.*]] = and i8 [[TMP36]], 63
+; CHECK-NEXT:    [[TMP158:%.*]] = or i8 [[TMP36]], [[TMP37]]
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <64 x i8> [[TMP9]], i64 14
+; CHECK-NEXT:    [[TMP39:%.*]] = and i8 [[TMP38]], 63
+; CHECK-NEXT:    [[TMP160:%.*]] = or i8 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <64 x i8> [[TMP9]], i64 15
+; CHECK-NEXT:    [[TMP41:%.*]] = and i8 [[TMP40]], 63
+; CHECK-NEXT:    [[TMP161:%.*]] = or i8 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <64 x i8> [[TMP9]], i64 16
+; CHECK-NEXT:    [[TMP43:%.*]] = and i8 [[TMP42]], 63
+; CHECK-NEXT:    [[TMP163:%.*]] = or i8 [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <64 x i8> [[TMP9]], i64 17
+; CHECK-NEXT:    [[TMP45:%.*]] = and i8 [[TMP44]], 63
+; CHECK-NEXT:    [[TMP164:%.*]] = or i8 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <64 x i8> [[TMP9]], i64 18
+; CHECK-NEXT:    [[TMP47:%.*]] = and i8 [[TMP46]], 63
+; CHECK-NEXT:    [[TMP166:%.*]] = or i8 [[TMP46]], [[TMP47]]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <64 x i8> [[TMP9]], i64 19
+; CHECK-NEXT:    [[TMP49:%.*]] = and i8 [[TMP48]], 63
+; CHECK-NEXT:    [[TMP167:%.*]] = or i8 [[TMP48]], [[TMP49]]
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <64 x i8> [[TMP9]], i64 20
+; CHECK-NEXT:    [[TMP51:%.*]] = and i8 [[TMP50]], 63
+; CHECK-NEXT:    [[TMP169:%.*]] = or i8 [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <64 x i8> [[TMP9]], i64 21
+; CHECK-NEXT:    [[TMP53:%.*]] = and i8 [[TMP52]], 63
+; CHECK-NEXT:    [[TMP170:%.*]] = or i8 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <64 x i8> [[TMP9]], i64 22
+; CHECK-NEXT:    [[TMP55:%.*]] = and i8 [[TMP54]], 63
+; CHECK-NEXT:    [[TMP172:%.*]] = or i8 [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <64 x i8> [[TMP9]], i64 23
+; CHECK-NEXT:    [[TMP57:%.*]] = and i8 [[TMP56]], 63
+; CHECK-NEXT:    [[TMP173:%.*]] = or i8 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <64 x i8> [[TMP9]], i64 24
+; CHECK-NEXT:    [[TMP59:%.*]] = and i8 [[TMP58]], 63
+; CHECK-NEXT:    [[TMP175:%.*]] = or i8 [[TMP58]], [[TMP59]]
+; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <64 x i8> [[TMP9]], i64 25
+; CHECK-NEXT:    [[TMP61:%.*]] = and i8 [[TMP60]], 63
+; CHECK-NEXT:    [[TMP176:%.*]] = or i8 [[TMP60]], [[TMP61]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <64 x i8> [[TMP9]], i64 26
+; CHECK-NEXT:    [[TMP63:%.*]] = and i8 [[TMP62]], 63
+; CHECK-NEXT:    [[TMP178:%.*]] = or i8 [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <64 x i8> [[TMP9]], i64 27
+; CHECK-NEXT:    [[TMP65:%.*]] = and i8 [[TMP64]], 63
+; CHECK-NEXT:    [[TMP179:%.*]] = or i8 [[TMP64]], [[TMP65]]
+; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <64 x i8> [[TMP9]], i64 28
+; CHECK-NEXT:    [[TMP67:%.*]] = and i8 [[TMP66]], 63
+; CHECK-NEXT:    [[TMP181:%.*]] = or i8 [[TMP66]], [[TMP67]]
+; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <64 x i8> [[TMP9]], i64 29
+; CHECK-NEXT:    [[TMP69:%.*]] = and i8 [[TMP68]], 63
+; CHECK-NEXT:    [[TMP182:%.*]] = or i8 [[TMP68]], [[TMP69]]
+; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <64 x i8> [[TMP9]], i64 30
+; CHECK-NEXT:    [[TMP71:%.*]] = and i8 [[TMP70]], 63
+; CHECK-NEXT:    [[TMP184:%.*]] = or i8 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <64 x i8> [[TMP9]], i64 31
+; CHECK-NEXT:    [[TMP73:%.*]] = and i8 [[TMP72]], 63
+; CHECK-NEXT:    [[TMP185:%.*]] = or i8 [[TMP72]], [[TMP73]]
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <64 x i8> [[TMP9]], i64 32
+; CHECK-NEXT:    [[TMP75:%.*]] = and i8 [[TMP74]], 63
+; CHECK-NEXT:    [[TMP187:%.*]] = or i8 [[TMP74]], [[TMP75]]
+; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <64 x i8> [[TMP9]], i64 33
+; CHECK-NEXT:    [[TMP77:%.*]] = and i8 [[TMP76]], 63
+; CHECK-NEXT:    [[TMP188:%.*]] = or i8 [[TMP76]], [[TMP77]]
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <64 x i8> [[TMP9]], i64 34
+; CHECK-NEXT:    [[TMP79:%.*]] = and i8 [[TMP78]], 63
+; CHECK-NEXT:    [[TMP190:%.*]] = or i8 [[TMP78]], [[TMP79]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <64 x i8> [[TMP9]], i64 35
+; CHECK-NEXT:    [[TMP81:%.*]] = and i8 [[TMP80]], 63
+; CHECK-NEXT:    [[TMP191:%.*]] = or i8 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <64 x i8> [[TMP9]], i64 36
+; CHECK-NEXT:    [[TMP83:%.*]] = and i8 [[TMP82]], 63
+; CHECK-NEXT:    [[TMP193:%.*]] = or i8 [[TMP82]], [[TMP83]]
+; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <64 x i8> [[TMP9]], i64 37
+; CHECK-NEXT:    [[TMP85:%.*]] = and i8 [[TMP84]], 63
+; CHECK-NEXT:    [[TMP194:%.*]] = or i8 [[TMP84]], [[TMP85]]
+; CHECK-NEXT:    [[TMP86:%.*]] = extractelement <64 x i8> [[TMP9]], i64 38
+; CHECK-NEXT:    [[TMP87:%.*]] = and i8 [[TMP86]], 63
+; CHECK-NEXT:    [[TMP196:%.*]] = or i8 [[TMP86]], [[TMP87]]
+; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <64 x i8> [[TMP9]], i64 39
+; CHECK-NEXT:    [[TMP89:%.*]] = and i8 [[TMP88]], 63
+; CHECK-NEXT:    [[TMP197:%.*]] = or i8 [[TMP88]], [[TMP89]]
+; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <64 x i8> [[TMP9]], i64 40
+; CHECK-NEXT:    [[TMP91:%.*]] = and i8 [[TMP90]], 63
+; CHECK-NEXT:    [[TMP199:%.*]] = or i8 [[TMP90]], [[TMP91]]
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <64 x i8> [[TMP9]], i64 41
+; CHECK-NEXT:    [[TMP93:%.*]] = and i8 [[TMP92]], 63
+; CHECK-NEXT:    [[TMP201:%.*]] = or i8 [[TMP92]], [[TMP93]]
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <64 x i8> [[TMP9]], i64 42
+; CHECK-NEXT:    [[TMP95:%.*]] = and i8 [[TMP94]], 63
+; CHECK-NEXT:    [[TMP138:%.*]] = or i8 [[TMP94]], [[TMP95]]
+; CHECK-NEXT:    [[TMP96:%.*]] = extractelement <64 x i8> [[TMP9]], i64 43
+; CHECK-NEXT:    [[TMP97:%.*]] = and i8 [[TMP96]], 63
+; CHECK-NEXT:    [[TMP141:%.*]] = or i8 [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <64 x i8> [[TMP9]], i64 44
+; CHECK-NEXT:    [[TMP99:%.*]] = and i8 [[TMP98]], 63
+; CHECK-NEXT:    [[TMP144:%.*]] = or i8 [[TMP98]], [[TMP99]]
+; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <64 x i8> [[TMP9]], i64 45
+; CHECK-NEXT:    [[TMP101:%.*]] = and i8 [[TMP100]], 63
+; CHECK-NEXT:    [[TMP147:%.*]] = or i8 [[TMP100]], [[TMP101]]
+; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <64 x i8> [[TMP9]], i64 46
+; CHECK-NEXT:    [[TMP103:%.*]] = and i8 [[TMP102]], 63
+; CHECK-NEXT:    [[TMP150:%.*]] = or i8 [[TMP102]], [[TMP103]]
+; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <64 x i8> [[TMP9]], i64 47
+; CHECK-NEXT:    [[TMP105:%.*]] = and i8 [[TMP104]], 63
+; CHECK-NEXT:    [[TMP153:%.*]] = or i8 [[TMP104]], [[TMP105]]
+; CHECK-NEXT:    [[TMP106:%.*]] = extractelement <64 x i8> [[TMP9]], i64 48
+; CHECK-NEXT:    [[TMP107:%.*]] = and i8 [[TMP106]], 63
+; CHECK-NEXT:    [[TMP156:%.*]] = or i8 [[TMP106]], [[TMP107]]
+; CHECK-NEXT:    [[TMP108:%.*]] = extractelement <64 x i8> [[TMP9]], i64 49
+; CHECK-NEXT:    [[TMP109:%.*]] = and i8 [[TMP108]], 63
+; CHECK-NEXT:    [[TMP159:%.*]] = or i8 [[TMP108]], [[TMP109]]
+; CHECK-NEXT:    [[TMP110:%.*]] = extractelement <64 x i8> [[TMP9]], i64 50
+; CHECK-NEXT:    [[TMP111:%.*]] = and i8 [[TMP110]], 63
+; CHECK-NEXT:    [[TMP162:%.*]] = or i8 [[TMP110]], [[TMP111]]
+; CHECK-NEXT:    [[TMP112:%.*]] = extractelement <64 x i8> [[TMP9]], i64 51
+; CHECK-NEXT:    [[TMP113:%.*]] = and i8 [[TMP112]], 63
+; CHECK-NEXT:    [[TMP165:%.*]] = or i8 [[TMP112]], [[TMP113]]
+; CHECK-NEXT:    [[TMP114:%.*]] = extractelement <64 x i8> [[TMP9]], i64 52
+; CHECK-NEXT:    [[TMP115:%.*]] = and i8 [[TMP114]], 63
+; CHECK-NEXT:    [[TMP168:%.*]] = or i8 [[TMP114]], [[TMP115]]
+; CHECK-NEXT:    [[TMP116:%.*]] = extractelement <64 x i8> [[TMP9]], i64 53
+; CHECK-NEXT:    [[TMP117:%.*]] = and i8 [[TMP116]], 63
+; CHECK-NEXT:    [[TMP171:%.*]] = or i8 [[TMP116]], [[TMP117]]
+; CHECK-NEXT:    [[TMP118:%.*]] = extractelement <64 x i8> [[TMP9]], i64 54
+; CHECK-NEXT:    [[TMP119:%.*]] = and i8 [[TMP118]], 63
+; CHECK-NEXT:    [[TMP174:%.*]] = or i8 [[TMP118]], [[TMP119]]
+; CHECK-NEXT:    [[TMP120:%.*]] = extractelement <64 x i8> [[TMP9]], i64 55
+; CHECK-NEXT:    [[TMP121:%.*]] = and i8 [[TMP120]], 63
+; CHECK-NEXT:    [[TMP177:%.*]] = or i8 [[TMP120]], [[TMP121]]
+; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <64 x i8> [[TMP9]], i64 56
+; CHECK-NEXT:    [[TMP123:%.*]] = and i8 [[TMP122]], 63
+; CHECK-NEXT:    [[TMP180:%.*]] = or i8 [[TMP122]], [[TMP123]]
+; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <64 x i8> [[TMP9]], i64 57
+; CHECK-NEXT:    [[TMP125:%.*]] = and i8 [[TMP124]], 63
+; CHECK-NEXT:    [[TMP183:%.*]] = or i8 [[TMP124]], [[TMP125]]
+; CHECK-NEXT:    [[TMP126:%.*]] = extractelement <64 x i8> [[TMP9]], i64 58
+; CHECK-NEXT:    [[TMP127:%.*]] = and i8 [[TMP126]], 63
+; CHECK-NEXT:    [[TMP186:%.*]] = or i8 [[TMP126]], [[TMP127]]
+; CHECK-NEXT:    [[TMP128:%.*]] = extractelement <64 x i8> [[TMP9]], i64 59
+; CHECK-NEXT:    [[TMP129:%.*]] = and i8 [[TMP128]], 63
+; CHECK-NEXT:    [[TMP189:%.*]] = or i8 [[TMP128]], [[TMP129]]
+; CHECK-NEXT:    [[TMP130:%.*]] = extractelement <64 x i8> [[TMP9]], i64 60
+; CHECK-NEXT:    [[TMP131:%.*]] = and i8 [[TMP130]], 63
+; CHECK-NEXT:    [[TMP192:%.*]] = or i8 [[TMP130]], [[TMP131]]
+; CHECK-NEXT:    [[TMP132:%.*]] = extractelement <64 x i8> [[TMP9]], i64 61
+; CHECK-NEXT:    [[TMP133:%.*]] = and i8 [[TMP132]], 63
+; CHECK-NEXT:    [[TMP195:%.*]] = or i8 [[TMP132]], [[TMP133]]
+; CHECK-NEXT:    [[TMP134:%.*]] = extractelement <64 x i8> [[TMP9]], i64 62
+; CHECK-NEXT:    [[TMP135:%.*]] = and i8 [[TMP134]], 63
+; CHECK-NEXT:    [[TMP198:%.*]] = or i8 [[TMP134]], [[TMP135]]
+; CHECK-NEXT:    [[TMP136:%.*]] = extractelement <64 x i8> [[TMP9]], i64 63
+; CHECK-NEXT:    [[TMP200:%.*]] = and i8 [[TMP136]], 63
+; CHECK-NEXT:    [[TMP137:%.*]] = or i8 [[TMP136]], [[TMP200]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[TMP6]], <64 x i8> [[T]], <64 x i8> [[TMP3]])
+; CHECK-NEXT:    [[_MSCMP124:%.*]] = icmp ne i8 [[TMP137]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP124]], label %[[BB203:.*]], label %[[BB204:.*]], !prof [[PROF1]]
+; CHECK:       [[BB203]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB204]]:
 ; CHECK-NEXT:    [[R:%.*]] = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> [[X0]], <64 x i8> [[T]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    store <64 x i8> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <64 x i8> [[R]]
@@ -720,3 +1397,6 @@ define <64 x i8> @shuffle_vpermv3_v64i8_demandedbits(<64 x i8> %x0, <64 x i8> %x
 }
 
 attributes #0 = { sanitize_memory }
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
index 613146ff8227e..294f2cae7f7ab 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
@@ -990,16 +990,21 @@ define <2 x double> @test_x86_avx_vpermilvar_pd(<2 x double> %a0, <2 x i64> %a1)
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP4]], [[TMP10]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <2 x i64> [[TMP1]] to <2 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A0]], <2 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <2 x double> [[RES]] to <2 x i64>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <2 x i64> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       13:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       14:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> [[A2:%.*]], <2 x i64> [[A1]])
 ; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[RES1]]
@@ -1016,16 +1021,27 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64>
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i64 [[TMP4]], [[TMP16]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = or i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i64> [[TMP1]] to <4 x double>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A0]], <4 x i64> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double> [[RES]] to <4 x i64>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i64> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       20:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> [[A2:%.*]], <4 x i64> [[A1]])
 ; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x double> [[RES1]]
@@ -1057,16 +1073,27 @@ define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) #
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP4]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP4]], [[TMP16]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = or i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = or i32 [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP1]] to <4 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP2]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP19:%.*]], label [[TMP20:%.*]], !prof [[PROF1]]
+; CHECK:       19:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       20:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A2:%.*]], <4 x i32> [[A1]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES1]]
@@ -1091,16 +1118,27 @@ define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, ptr %a1) #0
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = or i32 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 2
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = or i32 [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i32> [[_MSLD]], i64 3
+; CHECK-NEXT:    [[TMP19:%.*]] = and i32 [[TMP18]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = or i32 [[TMP18]], [[TMP19]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A0]], <4 x i32> [[A2]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x float> [[RES]] to <4 x i32>
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[_MSLD]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP12]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
-; CHECK:       13:
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i32 [[TMP20]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP1]], label [[TMP24:%.*]], label [[TMP25:%.*]], !prof [[PROF1]]
+; CHECK:       24:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       14:
+; CHECK:       25:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> [[A3:%.*]], <4 x i32> [[A2]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP11]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[RES1]]
@@ -1118,16 +1156,39 @@ define <8 x float> @test_x86_avx_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP29:%.*]] = or i32 [[TMP4]], [[TMP28]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP9:%.*]] = or i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP2]], i64 2
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP12:%.*]] = or i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = and i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP15:%.*]] = or i32 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP2]], i64 4
+; CHECK-NEXT:    [[TMP17:%.*]] = and i32 [[TMP16]], 7
+; CHECK-NEXT:    [[TMP18:%.*]] = or i32 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP2]], i64 5
+; CHECK-NEXT:    [[TMP20:%.*]] = and i32 [[TMP19]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = or i32 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP2]], i64 6
+; CHECK-NEXT:    [[TMP23:%.*]] = and i32 [[TMP22]], 7
+; CHECK-NEXT:    [[TMP24:%.*]] = or i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP2]], i64 7
+; CHECK-NEXT:    [[TMP26:%.*]] = and i32 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i32 [[TMP25]], [[TMP26]]
 ; CHECK-NEXT:    [[A0:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
 ; CHECK-NEXT:    [[RES:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A0]], <8 x i32> [[A1:%.*]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x float> [[RES]] to <8 x i32>
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP2]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i32 [[TMP27]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP31:%.*]], label [[TMP32:%.*]], !prof [[PROF1]]
+; CHECK:       31:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn()
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       32:
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> [[A2:%.*]], <8 x i32> [[A1]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x float> [[RES1]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/or.ll b/llvm/test/Instrumentation/MemorySanitizer/or.ll
index 2d51de13a8ebb..27a1800aa495b 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/or.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/or.ll
@@ -45,7 +45,7 @@ define i8 @test_disjoint_or(i8 %a, i8 %b) sanitize_memory {
 ; CHECK-IMPRECISE:      [[C:%.*]] = or disjoint i8 [[A]], [[B]]
 ; CHECK-IMPRECISE-NEXT: store i8 [[TMP11]], ptr @__msan_retval_tls, align 8
 ;
-; CHECK-PRECISE:         [[TMP10:%.*]] = and i8 [[TMP3]], [[TMP4]]
+; CHECK-PRECISE:         [[TMP10:%.*]] = and i8 [[A]], [[B]]
 ; CHECK-PRECISE-NEXT:    [[TMP12:%.*]] = or i8 [[TMP11]], [[TMP10]]
 ; CHECK-PRECISE-NEXT:    [[C:%.*]] = or disjoint i8 [[A]], [[B]]
 ; CHECK-PRECISE-NEXT:    store i8 [[TMP12]], ptr @__msan_retval_tls, align 8
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s b/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
index c3e19a7ad4d03..b20fed789a8ed 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
@@ -20,7 +20,7 @@ aesd z23.b, z23.b, z13.b
 .arch armv9-a+sve2-sm4
 .arch armv9-a+nosve2-sm4
 sm4e z0.s, z0.s, z0.s
-// CHECK: error: instruction requires: sve2-sm4
+// CHECK: error: instruction requires: sve-sm4
 // CHECK-NEXT: sm4e z0.s, z0.s, z0.s
 
 .arch armv9-a+sve2-sha3
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
index c2ec716e45d5a..73114e57fc18a 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
@@ -22,7 +22,7 @@ aesd z23.b, z23.b, z13.b
 .arch_extension sve2-sm4
 .arch_extension nosve2-sm4
 sm4e z0.s, z0.s, z0.s
-// CHECK: error: instruction requires: sve2-sm4
+// CHECK: error: instruction requires: sve-sm4
 // CHECK-NEXT: sm4e z0.s, z0.s, z0.s
 
 .arch_extension sve2-sha3
diff --git a/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s b/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
index 956dcb1a82a6a..e5b2a4c2b57aa 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
@@ -20,7 +20,7 @@ aesd z23.b, z23.b, z13.b
 .cpu generic+sve2-sm4
 .cpu generic+nosve2-sm4
 sm4e z0.s, z0.s, z0.s
-// CHECK: error: instruction requires: sve2-sm4
+// CHECK: error: instruction requires: sve-sm4
 // CHECK-NEXT: sm4e z0.s, z0.s, z0.s
 
 .cpu generic+sve2-sha3
diff --git a/llvm/test/MC/AArch64/SVE2/sm4e.s b/llvm/test/MC/AArch64/SVE2/sm4e.s
index 329484bc4b4a5..5c9745cd90a43 100644
--- a/llvm/test/MC/AArch64/SVE2/sm4e.s
+++ b/llvm/test/MC/AArch64/SVE2/sm4e.s
@@ -13,5 +13,5 @@
 sm4e z0.s, z0.s, z31.s
 // CHECK-INST: sm4e z0.s, z0.s, z31.s
 // CHECK-ENCODING: [0xe0,0xe3,0x23,0x45]
-// CHECK-ERROR: instruction requires: sve2-sm4
+// CHECK-ERROR: instruction requires: sve-sm4
 // CHECK-UNKNOWN: 4523e3e0 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/sm4ekey.s b/llvm/test/MC/AArch64/SVE2/sm4ekey.s
index c81e39e19167a..1aee0e509c7e6 100644
--- a/llvm/test/MC/AArch64/SVE2/sm4ekey.s
+++ b/llvm/test/MC/AArch64/SVE2/sm4ekey.s
@@ -13,5 +13,5 @@
 sm4ekey z0.s, z1.s, z31.s
 // CHECK-INST: sm4ekey z0.s, z1.s, z31.s
 // CHECK-ENCODING: [0x20,0xf0,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2-sm4
+// CHECK-ERROR: instruction requires: sve-sm4
 // CHECK-UNKNOWN: 453ff020 <unknown>
diff --git a/llvm/test/MC/AArch64/coff-relocations-branch26.s b/llvm/test/MC/AArch64/coff-relocations-branch26.s
index 4cd47032309ca..51bbb4932f231 100644
--- a/llvm/test/MC/AArch64/coff-relocations-branch26.s
+++ b/llvm/test/MC/AArch64/coff-relocations-branch26.s
@@ -61,7 +61,7 @@ main:
 err:
     nop
     b .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:19: error: cannot perform a PC-relative fixup with a non-zero symbol offset
 
     .def .Lerr_target
     .scl 3
diff --git a/llvm/test/MC/AArch64/coff-relocations-diags.s b/llvm/test/MC/AArch64/coff-relocations-diags.s
index e03ba377dc18b..41fdd03df8f3f 100644
--- a/llvm/test/MC/AArch64/coff-relocations-diags.s
+++ b/llvm/test/MC/AArch64/coff-relocations-diags.s
@@ -1,17 +1,17 @@
 // RUN: not llvm-mc -triple aarch64-win32 -filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
 
   adrp x0, :got:symbol
-  // CHECK: [[#@LINE-1]]:3: error: relocation specifier :got: unsupported on COFF targets
+  // CHECK: [[#@LINE-1]]:12: error: relocation specifier :got: unsupported on COFF targets
   // CHECK-NEXT: adrp x0, :got:symbol
   // CHECK-NEXT: ^
 
   ldr x0, [x0, :got_lo12:symbol]
-  // CHECK: [[#@LINE-1]]:3: error: relocation specifier :got_lo12: unsupported on COFF targets
+  // CHECK: [[#@LINE-1]]:16: error: relocation specifier :got_lo12: unsupported on COFF targets
   // CHECK-NEXT: ldr x0, [x0, :got_lo12:symbol]
   // CHECK-NEXT: ^
 
   adrp x0, :tlsdesc:symbol
-  // CHECK: [[#@LINE-1]]:3: error: relocation specifier :tlsdesc: unsupported on COFF targets
+  // CHECK: [[#@LINE-1]]:12: error: relocation specifier :tlsdesc: unsupported on COFF targets
   // CHECK-NEXT: adrp x0, :tlsdesc:symbol
   // CHECK-NEXT: ^
   add x0, x0, :tlsdesc_lo12:symbol
diff --git a/llvm/test/MC/AArch64/ilp32-diagnostics.s b/llvm/test/MC/AArch64/ilp32-diagnostics.s
index 1529035903357..aff01ed4c76aa 100644
--- a/llvm/test/MC/AArch64/ilp32-diagnostics.s
+++ b/llvm/test/MC/AArch64/ilp32-diagnostics.s
@@ -3,102 +3,102 @@
 // RUN: FileCheck --check-prefix=ERROR %s --implicit-check-not=error: < %t2
 
 .xword sym-.
-// ERROR: [[#@LINE-1]]:8: error: 8 byte PC relative data relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:11: error: 8 byte PC relative data relocation is not supported in ILP32
 
 .xword sym+16
-// ERROR: [[#@LINE-1]]:8: error: 8 byte absolute data relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 8 byte absolute data relocation is not supported in ILP32
 
 .xword sym@AUTH(da,42)
 // ERROR: [[#@LINE-1]]:8: error: 8 byte absolute data relocation is not supported in ILP32
 
 .xword sym@AUTH(da,42,addr)
-// ERROR: [[#@LINE-1]]:8: error: 8 byte absolute data relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 8 byte absolute data relocation is not supported in ILP32
 
 movz x7, #:abs_g3:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:11: error: absolute MOV relocation is not supported in ILP32
 // ERROR:        movz x7, #:abs_g3:some_label
 
 movz x3, #:abs_g2:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x3, #:abs_g2:some_label
 
 movz x19, #:abs_g2_s:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x19, #:abs_g2_s:some_label
 
 movk x5, #:abs_g2_nc:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x5, #:abs_g2_nc:some_label
 
 movz x19, #:abs_g1_s:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x19, #:abs_g1_s:some_label
 
 movk x5, #:abs_g1_nc:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x5, #:abs_g1_nc:some_label
 
 movz x3, #:dtprel_g2:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x3, #:dtprel_g2:var
 
 movk x9, #:dtprel_g1_nc:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x9, #:dtprel_g1_nc:var
 
 movz x3, #:tprel_g2:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x3, #:tprel_g2:var
 
 movk x9, #:tprel_g1_nc:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x9, #:tprel_g1_nc:var
 
 movz x15, #:gottprel_g1:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x15, #:gottprel_g1:var
 
 movk x13, #:gottprel_g0_nc:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x13, #:gottprel_g0_nc:var
 
 ldr x10, [x0, #:gottprel_lo12:var]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 // ERROR: ldr x10, [x0, #:gottprel_lo12:var]
 
 ldr x24, [x23, #:got_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x24, [x23, #:got_auth_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 add x24, x23, #:got_auth_lo12:sym
-// ERROR: [[#@LINE-1]]:1: error: ADD AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADD AUTH relocation is not supported in ILP32
 
 ldr x24, [x23, :gottprel_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x10, [x0, #:gottprel_lo12:var]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 // ERROR: ldr x10, [x0, #:gottprel_lo12:var]
 
 ldr x24, [x23, #:got_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x24, [x23, :gottprel_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x24, :got_auth:sym
-// ERROR: [[#@LINE-1]]:1: error: LDR AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:10: error: LDR AUTH relocation is not supported in ILP32
 
 adr x24, :got_auth:sym
-// ERROR: [[#@LINE-1]]:1: error: ADR AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADR AUTH relocation is not supported in ILP32
 
 adrp x24, :tlsdesc_auth:sym
-// ERROR: [[#@LINE-1]]:1: error: ADRP AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADRP AUTH relocation is not supported in ILP32
 
 ldr x24, [x23, :tlsdesc_auth_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store AUTH relocation is not supported in ILP32
 
 add x24, x23, :tlsdesc_auth_lo12:sym
-// ERROR: [[#@LINE-1]]:1: error: ADD AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADD AUTH relocation is not supported in ILP32
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s b/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
new file mode 100644
index 0000000000000..98436c9d6aa9c
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
@@ -0,0 +1,26 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR %s
+
+ds_atomic_async_barrier_arrive_b64 v1 offset:65407
+// GFX1250: ds_atomic_async_barrier_arrive_b64 v1 offset:65407 ; encoding: [0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_async_barrier_arrive_b64 v255 offset:1040
+// GFX1250: ds_atomic_async_barrier_arrive_b64 v255 offset:1040 ; encoding: [0x10,0x04,0x58,0xd9,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_async_barrier_arrive_b64 v5
+// GFX1250: ds_atomic_async_barrier_arrive_b64 v5   ; encoding: [0x00,0x00,0x58,0xd9,0x05,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5]
+// GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] ; encoding: [0x00,0x00,0xd4,0xd9,0x02,0x04,0x00,0x02]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] offset:513
+// GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] offset:513 ; encoding: [0x01,0x02,0xd4,0xd9,0x02,0x04,0x00,0x02]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_barrier_arrive_rtn_b64 v[254:255], v2, v[4:5] offset:65535
+// GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[254:255], v2, v[4:5] offset:65535 ; encoding: [0xff,0xff,0xd4,0xd9,0x02,0x04,0x00,0xfe]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
index 48ec44b410c2c..6ebc17468eed6 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
@@ -16,3 +16,15 @@ s_wait_xcnt 0xf
 s_setprio_inc_wg 100
 // GFX1250: [0x64,0x00,0xbe,0xbf]
 // GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_monitor_sleep 1
+// GFX1250: s_monitor_sleep 1                       ; encoding: [0x01,0x00,0x84,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_monitor_sleep 32768
+// GFX1250: s_monitor_sleep 0x8000                  ; encoding: [0x00,0x80,0x84,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_monitor_sleep 0
+// GFX1250: s_monitor_sleep 0                       ; encoding: [0x00,0x00,0x84,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s
new file mode 100644
index 0000000000000..5b6bb477bdc91
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s
@@ -0,0 +1,7 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+
+ds_load_tr_b64 v[2:3], v0
+// GFX1250: ds_load_tr8_b64 v[2:3], v0              ; encoding: [0x00,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02]
+
+ds_load_tr_b128 v[2:5], v0
+// GFX1250: ds_load_tr16_b128 v[2:5], v0            ; encoding: [0x00,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s
new file mode 100644
index 0000000000000..07b4055f0ab9c
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s
@@ -0,0 +1,22 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+tensor_save s[0:1]
+// GFX1250: tensor_save s[0:1] ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_save s[0:1] th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_save s[0:1] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_save s[0:1] offset:32
+// GFX1250: tensor_save s[0:1] offset:32 ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_stop
+// GFX1250: tensor_stop ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_stop th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_stop th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s
new file mode 100644
index 0000000000000..6b2dd67b073e3
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s
@@ -0,0 +1,37 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+global_load_tr_b64 v[2:3], v0, s[0:1]
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1]  ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v0, s[0:1] offset:64
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:-64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_tr_b64 v[2:3], v[4:5], off
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v[4:5], off offset:64
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v[4:5], off offset:-64
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:-64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff]
+
+global_load_tr_b128 v[2:5], v0, s[0:1]
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v0, s[0:1] offset:64
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:-64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_tr_b128 v[2:5], v[6:7], off
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v[6:7], off offset:64
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v[6:7], off offset:-64
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:-64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s
new file mode 100644
index 0000000000000..fec8ba19f93fe
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s
@@ -0,0 +1,34 @@
+; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+; RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+tensor_load_to_lds s[0:3], s[4:11]
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19]
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11]
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19]
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
new file mode 100644
index 0000000000000..3b2e03f1d4b9e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
@@ -0,0 +1,25 @@
+; RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+tensor_load_to_lds s[0:3], s[4:11] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_load_to_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for load instructions
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for load instructions
+
+tensor_store_from_lds s[0:3], s[4:11] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_store_from_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for store instructions
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for store instructions
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
index e62eb6fbb723c..9d437c7e7909f 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
@@ -46,6 +46,30 @@ v_cvt_f32_bf16 v5, src_scc
 v_cvt_f32_bf16 v127, 0x8000
 // GFX1250: v_cvt_f32_bf16_e32 v127, 0x8000         ; encoding: [0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00]
 
+v_cvt_f16_bf8 v1, v2
+// GFX1250: v_cvt_f16_bf8_e32 v1, v2                ; encoding: [0x02,0xf1,0x02,0x7e]
+
+v_cvt_f16_bf8 v1, s2
+// GFX1250: v_cvt_f16_bf8_e32 v1, s2                ; encoding: [0x02,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1, 2
+// GFX1250: v_cvt_f16_bf8_e32 v1, 2                 ; encoding: [0x82,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1, 0x1234
+// GFX1250: v_cvt_f16_bf8_e32 v1, 0x1234            ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2
+// GFX1250: v_cvt_f16_fp8_e32 v1, v2                ; encoding: [0x02,0xef,0x02,0x7e]
+
+v_cvt_f16_fp8 v1, s2
+// GFX1250: v_cvt_f16_fp8_e32 v1, s2                ; encoding: [0x02,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1, 2
+// GFX1250: v_cvt_f16_fp8_e32 v1, 2                 ; encoding: [0x82,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1, 0x1234
+// GFX1250: v_cvt_f16_fp8_e32 v1, 0x1234            ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+
 v_cvt_pk_f16_bf8 v1, v2
 // GFX1250: v_cvt_pk_f16_bf8 v1, v2                 ; encoding: [0x02,0xed,0x02,0x7e]
 
@@ -63,3 +87,12 @@ v_cvt_pk_f16_fp8 v1, s2
 
 v_cvt_pk_f16_fp8 v1, 100
 // GFX1250: v_cvt_pk_f16_fp8 v1, 0x64               ; encoding: [0xff,0xea,0x02,0x7e,0x64,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e32 v1, s3
+// GFX1250: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
+
+v_cvt_f32_fp8_e32 v1, 3
+// GFX1250: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
+
+v_cvt_f32_fp8_e32 v1, v3
+// GFX1250: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
index 37f39546ae13d..ff16e80c2d93c 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
@@ -49,6 +49,36 @@ v_cvt_f32_bf16 v127, 0x8000
 v_cvt_f32_bf16 v5, v1.h
 // GFX1250: v_cvt_f32_bf16_e32 v5, v1.h             ; encoding: [0x81,0xe5,0x0a,0x7e]
 
+v_cvt_f16_bf8 v1.l, v2
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, v2              ; encoding: [0x02,0xf1,0x02,0x7e]
+
+v_cvt_f16_bf8 v1.l, s2
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, s2              ; encoding: [0x02,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1.l, 2
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, 2               ; encoding: [0x82,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1.l, 0x1234
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, 0x1234          ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_bf8 v1.h, v2
+// GFX1250: v_cvt_f16_bf8_e32 v1.h, v2              ; encoding: [0x02,0xf1,0x02,0x7f]
+
+v_cvt_f16_fp8 v1.l, v2
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, v2              ; encoding: [0x02,0xef,0x02,0x7e]
+
+v_cvt_f16_fp8 v1.l, s2
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, s2              ; encoding: [0x02,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1.l, 2
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, 2               ; encoding: [0x82,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1.l, 0x1234
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, 0x1234          ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1.h, v2
+// GFX1250: v_cvt_f16_fp8_e32 v1.h, v2              ; encoding: [0x02,0xef,0x02,0x7f]
+
 v_cvt_pk_f16_bf8 v1, v2
 // GFX1250: v_cvt_pk_f16_bf8 v1, v2                 ; encoding: [0x02,0xed,0x02,0x7e]
 
@@ -66,3 +96,12 @@ v_cvt_pk_f16_fp8 v1, s2
 
 v_cvt_pk_f16_fp8 v1, 100
 // GFX1250: v_cvt_pk_f16_fp8 v1, 0x64               ; encoding: [0xff,0xea,0x02,0x7e,0x64,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e32 v1, s3
+// GFX1250: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
+
+v_cvt_f32_fp8_e32 v1, 3
+// GFX1250: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
+
+v_cvt_f32_fp8_e32 v1, v3
+// GFX1250: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
index 1ec54d137b335..e53812bb3fd04 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
@@ -58,10 +58,26 @@ v_cvt_f32_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:
 // GFX1250: v_cvt_f32_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
-v_cvt_pk_f16_fp8 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
-// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff]
+v_cvt_f16_bf8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
 v_cvt_pk_f16_bf8 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
 // GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
index d674a9ea06843..bd767d14fab5f 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
@@ -62,6 +62,30 @@ v_cvt_f32_bf16 v5, v1.h quad_perm:[3,2,1,0]
 // GFX1250: v_cvt_f32_bf16_dpp v5, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x81,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_cvt_f16_bf8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_bf8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7f,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_fp8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7f,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_cvt_pk_f16_bf8 v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
 // GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
index 9ab3a8adfa511..cbc0ebd3edda0 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
@@ -14,10 +14,26 @@ v_cvt_f32_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
 // GFX1250: v_cvt_f32_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
-v_cvt_pk_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
 v_cvt_pk_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
 // GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
index 6904624471801..8b9980a31daf3 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
@@ -18,12 +18,28 @@ v_cvt_f32_bf16 v5, v1.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_cvt_f32_bf16_dpp v5, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x81,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
-v_cvt_pk_f16_fp8 v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
-v_cvt_pk_f16_fp8 v1, v2.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xea,0x02,0x7e,0x82,0x77,0x39,0x05]
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7f,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7f,0x02,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
 v_cvt_pk_f16_bf8 v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
@@ -33,3 +49,11 @@ v_cvt_pk_f16_bf8 v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
 v_cvt_pk_f16_bf8 v1, v2.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xec,0x02,0x7e,0x82,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xea,0x02,0x7e,0x82,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
index f6c7cf8006508..5874bb76b36d1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
@@ -1,6 +1,51 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
 
+v_cvt_f32_fp8_e64 v1, s3
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3                ; encoding: [0x01,0x00,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, s3 byte_sel:1
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:1     ; encoding: [0x01,0x10,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, s3 byte_sel:2
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:2     ; encoding: [0x01,0x08,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, s3 byte_sel:3
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:3     ; encoding: [0x01,0x18,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3                 ; encoding: [0x01,0x00,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3 byte_sel:1
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:1      ; encoding: [0x01,0x10,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3 byte_sel:2
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:2      ; encoding: [0x01,0x08,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3 byte_sel:3
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:3      ; encoding: [0x01,0x18,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3                ; encoding: [0x01,0x00,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 byte_sel:1
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:1     ; encoding: [0x01,0x10,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 byte_sel:2
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2     ; encoding: [0x01,0x08,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 byte_sel:3
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:3     ; encoding: [0x01,0x18,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 clamp          ; encoding: [0x01,0x80,0xec,0xd5,0x03,0x01,0x00,0x00]
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 clamp          ; encoding: [0x01,0x80,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:1 clamp ; encoding: [0x01,0x90,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
+
 v_cvt_f32_bf16_e64 v5, v1
 // GFX1250: v_cvt_f32_bf16_e64 v5, v1               ; encoding: [0x05,0x00,0xf2,0xd5,0x01,0x01,0x00,0x00]
 
@@ -76,6 +121,60 @@ v_cvt_f32_bf16_e64 v5, -1 op_sel:[1]
 v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1]
 // GFX1250: v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xfd,0x00,0x00,0x00]
 
+v_cvt_f16_bf8 v150, v2
+// GFX1250: v_cvt_f16_bf8_e64 v150, v2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v150, s2
+// GFX1250: v_cvt_f16_bf8_e64 v150, s2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150, 2
+// GFX1250: v_cvt_f16_bf8_e64 v150, 2               ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150, 0x1234
+// GFX1250: v_cvt_f16_bf8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 byte_sel:1
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 byte_sel:3
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v128, v2 op_sel:[0,1]
+// GFX1250: v_cvt_f16_bf8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 op_sel:[0,1] byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150, v2
+// GFX1250: v_cvt_f16_fp8_e64 v150, v2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150, s2
+// GFX1250: v_cvt_f16_fp8_e64 v150, s2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150, 2
+// GFX1250: v_cvt_f16_fp8_e64 v150, 2               ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150, 0x1234
+// GFX1250: v_cvt_f16_fp8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 byte_sel:1
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 byte_sel:3
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v128, v2 op_sel:[0,1]
+// GFX1250: v_cvt_f16_fp8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 op_sel:[0,1] byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
 v_cvt_pk_f16_bf8 v1, v150
 // GFX1250: v_cvt_pk_f16_bf8 v1, v150               ; encoding: [0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
index 531d734a0683d..16bfc10fb16a7 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
@@ -1,6 +1,51 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
 
+v_cvt_f32_fp8_e64 v1, s3
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3                ; encoding: [0x01,0x00,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, s3 byte_sel:1
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:1     ; encoding: [0x01,0x10,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, s3 byte_sel:2
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:2     ; encoding: [0x01,0x08,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, s3 byte_sel:3
+// GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:3     ; encoding: [0x01,0x18,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3                 ; encoding: [0x01,0x00,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3 byte_sel:1
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:1      ; encoding: [0x01,0x10,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3 byte_sel:2
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:2      ; encoding: [0x01,0x08,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, 3 byte_sel:3
+// GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:3      ; encoding: [0x01,0x18,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3                ; encoding: [0x01,0x00,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 byte_sel:1
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:1     ; encoding: [0x01,0x10,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 byte_sel:2
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2     ; encoding: [0x01,0x08,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 byte_sel:3
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:3     ; encoding: [0x01,0x18,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8_e64 v1, v3 clamp          ; encoding: [0x01,0x80,0xec,0xd5,0x03,0x01,0x00,0x00]
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 clamp          ; encoding: [0x01,0x80,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:1 clamp ; encoding: [0x01,0x90,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp
+// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
+
 v_cvt_f32_bf16_e64 v5, v1
 // GFX1250: v_cvt_f32_bf16_e64 v5, v1               ; encoding: [0x05,0x00,0xf2,0xd5,0x01,0x01,0x00,0x00]
 
@@ -79,6 +124,60 @@ v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1]
 v_cvt_f32_bf16_e64 v5, v128.h
 // GFX1250: v_cvt_f32_bf16_e64 v5, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x80,0x01,0x00,0x00]
 
+v_cvt_f16_bf8 v150.l, v2
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, s2
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, 2
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, 0x1234
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:1
+// GFX1250: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:3
+// GFX1250: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v128.h, v2
+// GFX1250: v_cvt_f16_bf8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1.h, v2 byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, v2
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, s2
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, 2
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, 0x1234
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:1
+// GFX1250: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:3
+// GFX1250: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v128.h, v2
+// GFX1250: v_cvt_f16_fp8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1.h, v2 byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
 v_cvt_pk_f16_bf8 v1, v150
 // GFX1250: v_cvt_pk_f16_bf8 v1, v150               ; encoding: [0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
index 844b4259229ed..ea22a8cbdda03 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
@@ -46,6 +46,58 @@ v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
 // GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_cvt_f16_bf8 v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_fp8 v1, v3 clamp quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x80,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:1 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x90,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
+v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:2 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x88,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
 v_cvt_pk_f16_bf8 v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
 // GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
index 32c2e54cf0e71..868bbe7cdde3a 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
@@ -50,6 +50,66 @@ v_cvt_f32_bf16_e64_dpp v5, v128.h quad_perm:[3,2,1,0]
 // GFX1250: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_cvt_f16_bf8 v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 byte_sel:3 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v128.l, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 byte_sel:3 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v128.l, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_fp8 v1, v3 clamp quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x80,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:1 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x90,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
+v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:2 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x88,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
 v_cvt_pk_f16_bf8 v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
 // GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
index 75692c7422f64..e2c9bc4387138 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
@@ -2,6 +2,78 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
 
+v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_fp8 v1, v3 clamp dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x80,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:1 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x90,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
+v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:2 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x88,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
 v_cvt_pk_f16_bf8 v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1
 // GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
@@ -10,10 +82,6 @@ v_cvt_pk_f16_bf8 v1, v2 op_sel:[1] dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v2 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
-v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
-// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
-// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
-
 v_cvt_pk_f16_fp8 v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1
 // GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
index 2c1eb47164e59..2ea30aeb38fc1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
@@ -2,14 +2,6 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
 
-v_cvt_pk_f16_bf8 v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
-// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
-
-v_cvt_pk_f16_bf8 v1, v128.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
-// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
-
 v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
@@ -18,6 +10,90 @@ v_cvt_f32_bf16_e64_dpp v5, v128.h dpp8:[7,6,5,4,3,2,1,0]
 // GFX1250: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
 
+v_cvt_f16_bf8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v128.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v128.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_fp8 v1, v3 clamp dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x80,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:22: error: invalid operand for instruction
+
+v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:1 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x90,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
+v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:2 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x88,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:39: error: not a valid operand.
+
+v_cvt_pk_f16_bf8 v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
 v_cvt_pk_f16_fp8 v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1
 // GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 // GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx950_asm_features.s b/llvm/test/MC/AMDGPU/gfx950_asm_features.s
index 7bc47914f40b7..57fc573b37ba9 100644
--- a/llvm/test/MC/AMDGPU/gfx950_asm_features.s
+++ b/llvm/test/MC/AMDGPU/gfx950_asm_features.s
@@ -40,14 +40,26 @@ global_load_lds_dwordx4 v2, s[4:5] offset:4
 // GFX950: v_permlane16_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb3,0x02,0x7e]
 v_permlane16_swap_b32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane16_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb3,0xb4,0x7f]
+v_permlane16_swap_b32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane16_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb3,0x02,0x7e]
 v_permlane16_swap_b32_e32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane16_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb3,0xb4,0x7f]
+v_permlane16_swap_b32_e32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane16_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x99,0xd1,0x02,0x01,0x00,0x00]
 v_permlane16_swap_b32_e64 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane16_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00]
+v_permlane16_swap_b32_e64 v218, v219
+
 // FIXME: Parsed as bound_ctrl:1?
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane16_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x99,0xd1,0x02,0x01,0x00,0x00]
@@ -81,14 +93,26 @@ v_permlane16_swap_b32_e64 v1, v2 bound_ctrl:1 fi:1
 // GFX950: v_permlane32_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb5,0x02,0x7e]
 v_permlane32_swap_b32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane32_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb5,0xb4,0x7f]
+v_permlane32_swap_b32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane32_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb5,0x02,0x7e]
 v_permlane32_swap_b32_e32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane32_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb5,0xb4,0x7f]
+v_permlane32_swap_b32_e32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane32_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00]
 v_permlane32_swap_b32_e64 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane32_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00]
+v_permlane32_swap_b32_e64 v218, v219
+
 // FIXME: Parsed as bound_ctrl:1?
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane32_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x9a,0xd1,0x02,0x01,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/max-branch-distance.s b/llvm/test/MC/AMDGPU/max-branch-distance.s
index 0e3337a7e04b4..46aa524f7ab4f 100644
--- a/llvm/test/MC/AMDGPU/max-branch-distance.s
+++ b/llvm/test/MC/AMDGPU/max-branch-distance.s
@@ -1,5 +1,5 @@
 //  RUN: not llvm-mc -triple=amdgcn -filetype=obj -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERROR %s
-//  ERROR: max-branch-distance.s:7:3: error: branch size exceeds simm16
+//  ERROR: max-branch-distance.s:7:12: error: branch size exceeds simm16
 
 // fill v_nop
 LBB0_0:
diff --git a/llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s b/llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s
new file mode 100644
index 0000000000000..cef1e83729004
--- /dev/null
+++ b/llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s
@@ -0,0 +1,71 @@
+@ RUN: not llvm-mc %s -triple thumbv7-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv8-m.baseline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv8-m.mainline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv6m-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv5-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck -check-prefix=CHECKSHORT %s
+
+// Thumb BL has range +- 4 Megabytes if CPU does not support Thumb2 or does not
+// have v8-M baseline ops, it is +- 16 Megabytes otherwise.
+
+.code 16
+bl      shortend
+.space 0x3fffff
+shortend:
+// CHECKSHORT-NOT: error:
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl      shortend2
+.space 0x400000
+shortend2:
+
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl      end
+.space 0xffffff
+end:
+bl      end2
+.space 0xffffff
+.global end2
+end2:
+
+bl      end3
+.space 0x1000000
+.global end3
+end3:
+
+// CHECK-NOT: error:
+// CHECKSHORT-NOT: error:
+// CHECKSHORT: [[@LINE+2]]:[[#]]: error: Relocation out of range
+// CHECK: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl      end4
+.space 0x1000000
+end4:
+
+shortstart1:
+.space 0x3ffffc
+bl shortstart1
+
+shortstart2:
+.space 0x400000
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl shortstart2
+
+start1:
+.space 0xfffffc
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl start1
+
+.global start2
+start2:
+.space 0xfffffc
+bl start2
+
+.global start3
+start3:
+.space 0xfffffd
+bl start3
+
+// CHECK-NOT: error:
+start4:
+.space 0xfffffd
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl start4
diff --git a/llvm/test/MC/ARM/Relocations/thumb-branch.s b/llvm/test/MC/ARM/Relocations/thumb-branch.s
new file mode 100644
index 0000000000000..e0a487b719d8a
--- /dev/null
+++ b/llvm/test/MC/ARM/Relocations/thumb-branch.s
@@ -0,0 +1,21 @@
+// RUN: llvm-mc -filetype=obj -triple=thumbv7a-linux-gnueabi %s -o %t
+// RUN: llvm-objdump -dr %t --triple=thumbv7a | FileCheck %s
+
+// CHECK:      f7ff fffe     bl      {{.*}}                  @ imm = #-0x4
+// CHECK-NEXT:               00000000:  R_ARM_THM_CALL       foo
+// CHECK-NEXT: f7ff effe     blx     {{.*}}                  @ imm = #-0x4
+// CHECK-NEXT:               00000004:  R_ARM_THM_CALL       callee_thumb_low
+// CHECK-NEXT: bf00          nop
+// CHECK-NEXT: f7ff effe     blx     {{.*}}                  @ imm = #-0x4
+// CHECK-NEXT:               0000000a:  R_ARM_THM_CALL       callee_thumb_high
+// CHECK-NEXT: 4770          bx      lr
+
+ .syntax unified
+ .section .text, "ax",%progbits
+ .thumb
+ bl foo
+
+ blx   callee_thumb_low
+ nop
+ blx   callee_thumb_high
+ bx lr
diff --git a/llvm/test/MC/ARM/Windows/branch-reloc-offset.s b/llvm/test/MC/ARM/Windows/branch-reloc-offset.s
index e7d59cdcf6d40..f5b2ea7040333 100644
--- a/llvm/test/MC/ARM/Windows/branch-reloc-offset.s
+++ b/llvm/test/MC/ARM/Windows/branch-reloc-offset.s
@@ -65,11 +65,11 @@ err:
 // Test errors, if referencing a symbol with an offset
 
     b .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:19: error: cannot perform a PC-relative fixup with a non-zero symbol offset
     bl .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:20: error: cannot perform a PC-relative fixup with a non-zero symbol offset
     blx .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:21: error: cannot perform a PC-relative fixup with a non-zero symbol offset
 
 // Test errors, if referencing a private label which lacks .def/.scl/.type/.endef, in another
 // section, without an offset. Such symbols are omitted from the output symbol table, so the
@@ -77,7 +77,7 @@ err:
 // section plus an offset, but such an offset is not supported with this relocation.
 
     b .Lerr_target2
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:7: error: cannot perform a PC-relative fixup with a non-zero symbol offset
 
     .def .Lerr_target
     .scl 3
diff --git a/llvm/test/MC/ARM/Windows/invalid-relocation.s b/llvm/test/MC/ARM/Windows/invalid-relocation.s
index 821de8dd55604..01528614c11db 100644
--- a/llvm/test/MC/ARM/Windows/invalid-relocation.s
+++ b/llvm/test/MC/ARM/Windows/invalid-relocation.s
@@ -9,4 +9,4 @@
 	.thumb_func
 	adr r0, invalid_relocation+1
 
-# CHECK: 10:2: error: unsupported relocation type
+# CHECK: 10:28: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/arm-memory-instructions-immediate.s b/llvm/test/MC/ARM/arm-memory-instructions-immediate.s
index d72028edc9e21..23cf547226cc1 100644
--- a/llvm/test/MC/ARM/arm-memory-instructions-immediate.s
+++ b/llvm/test/MC/ARM/arm-memory-instructions-immediate.s
@@ -21,5 +21,5 @@ foo:
 // CHECK-NEXT: strb r0, [r1, #1024]
 .ifdef ERR
     str r0, [r1, 1b]
-// ERR:[[#@LINE-1]]:5: error: unsupported relocation type
+// ERR:[[#@LINE-1]]:18: error: unsupported relocation type
 .endif
diff --git a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
index fdc49bcde4edc..d52ced6e20a10 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
@@ -12,7 +12,6 @@
 f:
         push    {r7, lr}
         mov     r7, sp
-        bl      g
         pop     {r7, pc}
 
 	.section	.data.rel.local,"aw",%progbits
@@ -20,16 +19,10 @@ ptr:
 	.long	f
 
 
-@@ make sure an R_ARM_THM_CALL relocation is generated for the call to g
-@CHECK:      Relocations [
-@CHECK-NEXT:   Section {{.*}} .rel.text {
-@CHECK-NEXT:     0x4 R_ARM_THM_CALL g
-@CHECK-NEXT:   }
-
 
 @@ make sure the relocation is with f. That is one way to make sure it includes
 @@ the thumb bit.
-@CHECK-NEXT:   Section ({{.*}}) .rel.data.rel.local {
+@CHECK:        Section ({{.*}}) .rel.data.rel.local {
 @CHECK-NEXT:     0x0 R_ARM_ABS32 f
 @CHECK-NEXT:   }
 @CHECK-NEXT: ]
diff --git a/llvm/test/MC/ARM/lower-upper-errors-2.s b/llvm/test/MC/ARM/lower-upper-errors-2.s
index 58c1f11792300..7b21b5a266cb2 100644
--- a/llvm/test/MC/ARM/lower-upper-errors-2.s
+++ b/llvm/test/MC/ARM/lower-upper-errors-2.s
@@ -8,11 +8,11 @@
 // For errors that are reported earlier, when initially reading the
 // instructions, see lower-upper-errors.s.
 
-// CHECK: [[@LINE+1]]:1: error: unsupported relocation
+// CHECK: [[@LINE+1]]:15: error: unsupported relocation
 adds r0, r0, #foo
 
-// CHECK: [[@LINE+1]]:1: error: unsupported relocation
+// CHECK: [[@LINE+1]]:[[#]]: error: unsupported relocation
 add r9, r0, #foo
 
-// CHECK: [[@LINE+1]]:1: error: expected relocatable expression
+// CHECK: [[@LINE+1]]:11: error: expected relocatable expression
 movs r11, :upper8_15:#foo
diff --git a/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s b/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s
index 0b745a97687d5..b235bde45e99c 100644
--- a/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s
+++ b/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s
@@ -16,8 +16,8 @@ bar:
     ldrd r0, r1, foo1    @ arm_pcrel_10_unscaled
     ldrd r0, r1, foo2-8  @ arm_pcrel_10_unscaled
 .ifdef ERR
-  @ ERR:[[#@LINE-3]]:5: error: unsupported relocation type
-  @ ERR:[[#@LINE-3]]:5: error: unsupported relocation type
+  @ ERR:[[#@LINE-3]]:18: error: unsupported relocation type
+  @ ERR:[[#@LINE-3]]:22: error: unsupported relocation type
 .endif
     bx lr
 
diff --git a/llvm/test/MC/ARM/pcrel-vldr-diff-section.s b/llvm/test/MC/ARM/pcrel-vldr-diff-section.s
index 44b1032a8f63e..ce058fccdcba2 100644
--- a/llvm/test/MC/ARM/pcrel-vldr-diff-section.s
+++ b/llvm/test/MC/ARM/pcrel-vldr-diff-section.s
@@ -9,6 +9,6 @@ vldr s0, foo     @ arm_pcrel_10 / t2_pcrel_10
 vldr d0, foo     @ arm_pcrel_10 / t2_pcrel_10
 vldr.16 s0,foo   @ arm_pcrel_9 / t2_pcrel_9
 
-@ CHECK: :[[#@LINE-4]]:1: error: unsupported relocation type
-@ CHECK: :[[#@LINE-4]]:1: error: unsupported relocation type
-@ CHECK: :[[#@LINE-4]]:1: error: unsupported relocation type
+@ CHECK: :[[#@LINE-4]]:10: error: unsupported relocation type
+@ CHECK: :[[#@LINE-4]]:10: error: unsupported relocation type
+@ CHECK: :[[#@LINE-4]]:12: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/quad-relocation.s b/llvm/test/MC/ARM/quad-relocation.s
index 50d8d82be5843..89c1653b957ac 100644
--- a/llvm/test/MC/ARM/quad-relocation.s
+++ b/llvm/test/MC/ARM/quad-relocation.s
@@ -3,7 +3,7 @@
 
 	.align 3
 symbol:
-@ CHECK:      :[[#@LINE+1]]:6: error: unsupported relocation type
+@ CHECK:      :[[#@LINE+1]]:7: error: unsupported relocation type
 .quad(symbol)
 @ CHECK:      :[[#@LINE+1]]:8: error: unsupported relocation type
 .8byte symbol
diff --git a/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s b/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s
index 642484080b1c0..12b5eae77f75f 100644
--- a/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s
+++ b/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s
@@ -10,4 +10,4 @@
     .equ sym0, 0x01abcdef
 .L2:
     mov r0, .L2
-@ CHECK: :[[#@LINE-1]]:5: error: unsupported relocation type
+@ CHECK: :[[#@LINE-1]]:13: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/thumb-branches.s b/llvm/test/MC/ARM/thumb-branches.s
deleted file mode 100644
index c9e1660e1cdb6..0000000000000
--- a/llvm/test/MC/ARM/thumb-branches.s
+++ /dev/null
@@ -1,71 +0,0 @@
-@ RUN: not llvm-mc %s -triple thumbv7-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv8-m.baseline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv8-m.mainline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv6m-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv5-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck -check-prefix=CHECKSHORT %s
-
-// Thumb BL has range +- 4 Megabytes if CPU does not support Thumb2 or does not
-// have v8-M baseline ops, it is +- 16 Megabytes otherwise.
-
-        .code 16
-        bl      shortend
-        .space 0x3fffff
-shortend:
-// CHECKSHORT-NOT: error:
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl      shortend2
-        .space 0x400000
-shortend2:
-
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl      end
-        .space 0xffffff
-end:
-        bl      end2
-        .space 0xffffff
-        .global end2
-end2:
-
-        bl      end3
-        .space 0x1000000
-        .global end3
-end3:
-
-// CHECK-NOT: error:
-// CHECKSHORT-NOT: error:
-// CHECKSHORT: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
-// CHECK: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl      end4
-        .space 0x1000000
-end4:
-
-shortstart1:
-        .space 0x3ffffc
-        bl shortstart1
-
-shortstart2:
-        .space 0x400000
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl shortstart2
-
-start1:
-        .space 0xfffffc
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl start1
-
-        .global start2
-start2:
-        .space 0xfffffc
-        bl start2
-
-        .global start3
-start3:
-        .space 0xfffffd
-        bl start3
-
-// CHECK-NOT: error:
-start4:
-        .space 0xfffffd
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl start4
diff --git a/llvm/test/MC/ARM/thumb1-relax-adr.s b/llvm/test/MC/ARM/thumb1-relax-adr.s
index 97b566f4833e6..f5c78612395ff 100644
--- a/llvm/test/MC/ARM/thumb1-relax-adr.s
+++ b/llvm/test/MC/ARM/thumb1-relax-adr.s
@@ -4,4 +4,4 @@
         .global func1
 _func1:
         adr r0, _func2
-@ CHECK-ERROR: :[[#@LINE-1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE-1]]:17: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/thumb1-relax-bcc.s b/llvm/test/MC/ARM/thumb1-relax-bcc.s
index 78f3477c19710..7839334bc25da 100644
--- a/llvm/test/MC/ARM/thumb1-relax-bcc.s
+++ b/llvm/test/MC/ARM/thumb1-relax-bcc.s
@@ -5,7 +5,7 @@
 
         .global func1
 _func1:
-@ CHECK-ERROR: :[[#@LINE+1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE+1]]:13: error: unsupported relocation type
         bne _func2
 
 @ CHECK-ELF: f47f affe          bne.w {{.+}} @ imm = #-4
diff --git a/llvm/test/MC/ARM/thumb1-relax-br.s b/llvm/test/MC/ARM/thumb1-relax-br.s
index b58dd6c0b413c..618c4b631ff31 100644
--- a/llvm/test/MC/ARM/thumb1-relax-br.s
+++ b/llvm/test/MC/ARM/thumb1-relax-br.s
@@ -9,7 +9,7 @@ _func1:
         @ There is no MachO relocation for Thumb1's unconditional branch, so
         @ this is unrepresentable. FIXME: I think ELF could represent this.
         b _func2
-@ CHECK-ERROR: :[[#@LINE-1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE-1]]:11: error: unsupported relocation type
 
 @ CHECK-MACHO: f7ff bffe          b.w {{.+}} @ imm = #-4
 @ CHECK-MACHO-NEXT: ARM_THUMB_RELOC_BR22
diff --git a/llvm/test/MC/ARM/thumb1-relax-ldrlit.s b/llvm/test/MC/ARM/thumb1-relax-ldrlit.s
index 8f455c89d41eb..261527ef00baa 100644
--- a/llvm/test/MC/ARM/thumb1-relax-ldrlit.s
+++ b/llvm/test/MC/ARM/thumb1-relax-ldrlit.s
@@ -3,5 +3,5 @@
 
         .global func1
 _func1:
-@ CHECK-ERROR: :[[#@LINE+1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE+1]]:17: error: unsupported relocation type
         ldr r0, _func2
diff --git a/llvm/test/MC/ARM/thumb2-branch-ranges.s b/llvm/test/MC/ARM/thumb2-branch-ranges.s
index fc8044acde8a3..495ceb87f6c18 100644
--- a/llvm/test/MC/ARM/thumb2-branch-ranges.s
+++ b/llvm/test/MC/ARM/thumb2-branch-ranges.s
@@ -26,7 +26,7 @@ end3:   bx lr
 
 // branch to thumb function is resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: b.w end4
         b.w end4
         .space 0x1000000
@@ -47,7 +47,7 @@ end6:   bx lr
 
 // conditional branch to thumb function resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: beq.w end7
         beq.w end7
         .space 0x100000
@@ -70,7 +70,7 @@ start3:
         .space 0x1000000
 // branch to thumb function resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: b.w start3
         b.w start3
 
@@ -91,12 +91,12 @@ start6:
         .space 0x100000
 // branch to thumb function resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: beq.w start6
         beq.w start6
 
 start7:
 // branch to thumb function resolved at assembly time
-// CHECK: [[#@LINE+1]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[#@LINE+1]]:[[#]]: error: Relocation out of range
         b.w start8 - start7 + 0x1000000
 start8:
diff --git a/llvm/test/MC/AsmParser/preserve-comments-aarch64-linux.s b/llvm/test/MC/AsmParser/preserve-comments-aarch64-linux.s
new file mode 100644
index 0000000000000..4a83cd955e3c7
--- /dev/null
+++ b/llvm/test/MC/AsmParser/preserve-comments-aarch64-linux.s
@@ -0,0 +1,11 @@
+	// REQUIRES: aarch64-registered-target
+	// llvm-mc prints leading tabs, formatting of this test follows preserve-comments.s
+	// RUN: llvm-mc -preserve-comments -n -triple aarch64-unknown-linux-gnu %s -o %t
+	// RUN: diff -b %s %t
+
+	.text
+
+foo:
+	// comment here
+	nop
+	// comment here too
diff --git a/llvm/test/MC/BPF/bad-tied.s b/llvm/test/MC/BPF/bad-tied.s
new file mode 100644
index 0000000000000..9cd47896c3dd2
--- /dev/null
+++ b/llvm/test/MC/BPF/bad-tied.s
@@ -0,0 +1,12 @@
+# RUN: not llvm-mc -mcpu=v4 -triple bpfel < %s 2>&1 \
+# RUN:   | grep 'error: operand is not the same as the dst register' \
+# RUN:   | count 9
+        r0 = bswap16 r1
+        r0 = bswap32 r1
+        r0 = bswap64 r1
+        r0 = atomic_fetch_add((u64*)(r2 + 0), r1)
+        r0 = atomic_fetch_and((u64*)(r2 + 0), r1)
+        r0 = atomic_fetch_or((u64*)(r2 + 0), r1)
+        r0 = atomic_fetch_xor((u64*)(r2 + 0), r1)
+        w0 = xchg32_32(r2 + 0, w1)
+        r0 = xchg_64(r2 + 0, r1)
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
new file mode 100644
index 0000000000000..e03c4327d9814
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
@@ -0,0 +1,19 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX1250 %s
+
+# GFX1250: ds_atomic_async_barrier_arrive_b64 v1 offset:65407 ; encoding: [0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00]
+0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00
+
+# GFX1250: ds_atomic_async_barrier_arrive_b64 v255 offset:1040 ; encoding: [0x10,0x04,0x58,0xd9,0xff,0x00,0x00,0x00]
+0x10,0x04,0x58,0xd9,0xff,0x00,0x00,0x00
+
+# GFX1250: ds_atomic_async_barrier_arrive_b64 v5   ; encoding: [0x00,0x00,0x58,0xd9,0x05,0x00,0x00,0x00]
+0x00,0x00,0x58,0xd9,0x05,0x00,0x00,0x00
+
+# GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[254:255], v2, v[4:5] offset:65535 ; encoding: [0xff,0xff,0xd4,0xd9,0x02,0x04,0x00,0xfe]
+0xff,0xff,0xd4,0xd9,0x02,0x04,0x00,0xfe
+
+# GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] ; encoding: [0x00,0x00,0xd4,0xd9,0x02,0x04,0x00,0x02]
+0x00,0x00,0xd4,0xd9,0x02,0x04,0x00,0x02
+
+# GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] offset:513 ; encoding: [0x01,0x02,0xd4,0xd9,0x02,0x04,0x00,0x02]
+0x01,0x02,0xd4,0xd9,0x02,0x04,0x00,0x02
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
index 55f74d3a31bf7..220f9e5084f0e 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
@@ -11,3 +11,12 @@
 
 # GFX1250: s_setprio_inc_wg 0x64 ; encoding: [0x64,0x00,0xbe,0xbf]
 0x64,0x00,0xbe,0xbf
+
+# GFX1250: s_monitor_sleep 0                       ; encoding: [0x00,0x00,0x84,0xbf]
+0x00,0x00,0x84,0xbf
+
+# GFX1250: s_monitor_sleep 0x8000                  ; encoding: [0x00,0x80,0x84,0xbf]
+0x00,0x80,0x84,0xbf
+
+# GFX1250: s_monitor_sleep 1                       ; encoding: [0x01,0x00,0x84,0xbf]
+0x01,0x00,0x84,0xbf
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt
new file mode 100644
index 0000000000000..6421c6f30e177
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt
@@ -0,0 +1,16 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+
+# GFX1250: tensor_save s[0:1] ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: tensor_save s[0:1] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+0x00,0x80,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: tensor_save s[0:1] offset:32 ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00]
+0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00
+
+# GFX1250: tensor_stop ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x7c,0xc0,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: tensor_stop th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+0x7c,0xc0,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt
new file mode 100644
index 0000000000000..9afaa075ea838
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt
@@ -0,0 +1,25 @@
+# RUN: llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
index 47eebb9d44a95..dd472d58893f5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
 0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00
 # GFX1250: v_cvt_f32_bf16_e32 v127, 0x8000         ; encoding: [0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00]
@@ -50,6 +50,44 @@
 0x81,0xe5,0x0a,0x7e
 # GFX1250: v_cvt_f32_bf16_e32 v5, v1.h             ; encoding: [0x81,0xe5,0x0a,0x7e]
 
+0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, 0x1234          ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, 0x1234            ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+0x82,0xf0,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, 2               ; encoding: [0x82,0xf0,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, 2                 ; encoding: [0x82,0xf0,0x02,0x7e]
+
+0x02,0xf0,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, s2              ; encoding: [0x02,0xf0,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, s2                ; encoding: [0x02,0xf0,0x02,0x7e]
+
+0x02,0xf1,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, v2              ; encoding: [0x02,0xf1,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, v2                ; encoding: [0x02,0xf1,0x02,0x7e]
+
+0x02,0xf1,0x02,0x7f
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.h, v2              ; encoding: [0x02,0xf1,0x02,0x7f]
+
+0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, 0x1234          ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, 0x1234            ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+0x82,0xee,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, 2               ; encoding: [0x82,0xee,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, 2                 ; encoding: [0x82,0xee,0x02,0x7e]
+
+0x02,0xee,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, s2              ; encoding: [0x02,0xee,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, s2                ; encoding: [0x02,0xee,0x02,0x7e]
+
+0x02,0xef,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, v2              ; encoding: [0x02,0xef,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, v2                ; encoding: [0x02,0xef,0x02,0x7e]
+
+0x02,0xef,0x02,0x7f
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.h, v2              ; encoding: [0x02,0xef,0x02,0x7f]
+
 0xff,0xec,0x02,0x7e,0x64,0x00,0x00,0x00
 # GFX1250: v_cvt_pk_f16_bf8 v1, 0x64               ; encoding: [0xff,0xec,0x02,0x7e,0x64,0x00,0x00,0x00]
 
@@ -69,3 +107,12 @@
 0x02,0xeb,0x02,0x7e
 # GFX1250-REAL16: v_cvt_pk_f16_fp8 v1, v2.l               ; encoding: [0x02,0xeb,0x02,0x7e]
 # GFX1250-FAKE16: v_cvt_pk_f16_fp8 v1, v2                 ; encoding: [0x02,0xeb,0x02,0x7e]
+
+0x03,0xd8,0x02,0x7e
+# GFX1250: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
+
+0x83,0xd8,0x02,0x7e
+# GFX1250: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
+
+0x03,0xd9,0x02,0x7e
+# GFX1250: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
index 25e982b7fd688..9656dcdeddae3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
 0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30
 # GFX1250: v_cvt_f32_bf16_dpp v127, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
@@ -47,6 +47,34 @@
 0xfa,0xe4,0x0a,0x7e,0x81,0x1b,0x00,0xff
 # GFX1250: v_cvt_f32_bf16_dpp v5, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x81,0x1b,0x00,0xff]
 
+0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+
+0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+
+0xfa,0xf0,0x02,0x7f,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7f,0x02,0xe4,0x00,0xff]
+
+0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+
+0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+
+0xfa,0xee,0x02,0x7f,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7f,0x02,0xe4,0x00,0xff]
+
+0xfa,0xd8,0x02,0x7e,0x03,0xe4,0x00,0xac
+# GFX1250: v_cvt_f32_fp8_dpp v1, v3 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xc ; encoding: [0xfa,0xd8,0x02,0x7e,0x03,0xe4,0x00,0xac]
+
+0xfa,0xd8,0x02,0x7e,0x03,0x1b,0x00,0x2e
+# GFX1250: v_cvt_f32_fp8_dpp v1, v3 quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xe ; encoding: [0xfa,0xd8,0x02,0x7e,0x03,0x1b,0x00,0x2e]
+
 0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff
 # GFX1250-REAL16: v_cvt_pk_f16_bf8_dpp v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
 # GFX1250-FAKE16: v_cvt_pk_f16_bf8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
index bd524af907ee0..274b58769911a 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
 0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00
 # GFX1250: v_cvt_f32_bf16_dpp v127, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00]
@@ -14,6 +14,28 @@
 0xe9,0xe4,0x0a,0x7e,0x81,0x77,0x39,0x05
 # GFX1250: v_cvt_f32_bf16_dpp v5, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x81,0x77,0x39,0x05]
 
+0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xf0,0x02,0x7f,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7f,0x02,0x77,0x39,0x05]
+
+0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xee,0x02,0x7f,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7f,0x02,0x77,0x39,0x05]
+
 0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05
 # GFX1250-REAL16: v_cvt_pk_f16_bf8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_cvt_pk_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
index 70abf4289ac11..ae98cf2e0ff58 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
@@ -2,6 +2,51 @@
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
 
+0x01,0x00,0xec,0xd5,0x83,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, 3                 ; encoding: [0x01,0x00,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+0x01,0x10,0xec,0xd5,0x83,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:1      ; encoding: [0x01,0x10,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+0x01,0x08,0xec,0xd5,0x83,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:2      ; encoding: [0x01,0x08,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+0x01,0x18,0xec,0xd5,0x83,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, 3 byte_sel:3      ; encoding: [0x01,0x18,0xec,0xd5,0x83,0x00,0x00,0x00]
+
+0x01,0x00,0xec,0xd5,0x03,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, s3                ; encoding: [0x01,0x00,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+0x01,0x10,0xec,0xd5,0x03,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:1     ; encoding: [0x01,0x10,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+0x01,0x08,0xec,0xd5,0x03,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:2     ; encoding: [0x01,0x08,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+0x01,0x18,0xec,0xd5,0x03,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, s3 byte_sel:3     ; encoding: [0x01,0x18,0xec,0xd5,0x03,0x00,0x00,0x00]
+
+0x01,0x00,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3                ; encoding: [0x01,0x00,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+0x01,0x10,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:1     ; encoding: [0x01,0x10,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+0x01,0x08,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2     ; encoding: [0x01,0x08,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+0x01,0x18,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:3     ; encoding: [0x01,0x18,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+0x01,0x80,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3 clamp          ; encoding: [0x01,0x80,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+0x01,0x90,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:1 clamp ; encoding: [0x01,0x90,0xec,0xd5,0x03,0x01,0x00,0x00]
+
+0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00
+# GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
+
 0x05,0x00,0xf2,0xd5,0xc1,0x00,0x00,0x00
 # GFX1250: v_cvt_f32_bf16_e64 v5, -1               ; encoding: [0x05,0x00,0xf2,0xd5,0xc1,0x00,0x00,0x00]
 
@@ -48,6 +93,115 @@
 # GFX1250-REAL16: v_cvt_f32_bf16_e64 v5, v255.h op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xff,0x01,0x00,0x00]
 # GFX1250-FAKE16: v_cvt_f32_bf16_e64 v5, v255 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xff,0x01,0x00,0x00]
 
+0x05,0x08,0xf2,0xd5,0x01,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, s1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x01,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x69,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, s105 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x69,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x6a,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, vcc_lo op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x6a,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x6b,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, vcc_hi op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x6b,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x7b,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, ttmp15 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7b,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x7d,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, m0 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x7d,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x7e,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, exec_lo op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7e,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x7f,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, exec_hi op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7f,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x7c,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, null op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7c,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0xc1,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, -1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0xc1,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0xfd,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xfd,0x00,0x00,0x00]
+
+0x05,0x08,0xf2,0xd5,0x80,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f32_bf16_e64 v5, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x80,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64 v5, v128 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x80,0x01,0x00,0x00]
+
+0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, 2               ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, s2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, v2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, 2               ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, s2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, v2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
 0x01,0x08,0xf6,0xd5,0x02,0x00,0x00,0x00
 # GFX1250: v_cvt_pk_f16_bf8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x00,0x00,0x00]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
index d53d532eef804..eadd315ea411b 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
@@ -50,6 +50,63 @@
 # GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 # GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v128 op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
 
+0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v128, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v128, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x01,0x80,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff
+# GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x80,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+
+0x01,0x90,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff
+# GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:1 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x90,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+
+0x01,0x88,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff
+# GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:2 clamp quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x88,0xec,0xd5,0xfa,0x00,0x00,0x00,0x03,0x1b,0x00,0xff]
+
 0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff
 # GFX1250-REAL16: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
 # GFX1250-FAKE16: v_cvt_pk_f16_bf8_e64_dpp v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
index 8df21f3f5e4df..3b2fbe10c61a5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
@@ -10,6 +10,75 @@
 # GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v128 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 
+0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x80,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05
+# GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x80,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+
+0x01,0x90,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05
+# GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:1 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x90,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+
+0x01,0x88,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05
+# GFX1250: v_cvt_f32_fp8_e64_dpp v1, v3 byte_sel:2 clamp dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x88,0xec,0xd5,0xe9,0x00,0x00,0x00,0x03,0x77,0x39,0x05]
+
 0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05
 # GFX1250-REAL16: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
 # GFX1250-FAKE16: v_cvt_pk_f16_bf8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt
index 9fc9c58387b90..01821593b0707 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt
@@ -47,9 +47,27 @@
 # GFX950:   v_permlane16_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb3,0x02,0x7e]
 0x02,0xb3,0x02,0x7e
 
+# GFX950: v_permlane16_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb3,0xb4,0x7f]
+0xdb,0xb3,0xb4,0x7f
+
+# GFX950: v_permlane16_swap_b32_e32 v218, v2      ; encoding: [0x02,0xb3,0xb4,0x7f]
+0x02,0xb3,0xb4,0x7f
+
+# GFX950: v_permlane16_swap_b32_e32 v2, v219      ; encoding: [0xdb,0xb3,0x04,0x7e]
+0xdb,0xb3,0x04,0x7e
+
 # GFX950:   v_permlane16_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x99,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x00,0x99,0xd1,0x02,0x01,0x00,0x00
 
+# GFX950: v_permlane16_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00]
+0xda,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00
+
+# GFX950: v_permlane16_swap_b32_e64 v218, v2      ; encoding: [0xda,0x00,0x99,0xd1,0x02,0x01,0x00,0x00]
+0xda,0x00,0x99,0xd1,0x02,0x01,0x00,0x00
+
+# GFX950: v_permlane16_swap_b32_e64 v2, v219      ; encoding: [0x02,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00]
+0x02,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00
+
 # GFX950:   v_permlane16_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x99,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x10,0x99,0xd1,0x02,0x01,0x00,0x00
 
@@ -63,9 +81,27 @@
 # GFX950:   v_permlane32_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb5,0x02,0x7e]
 0x02,0xb5,0x02,0x7e
 
+# GFX950: v_permlane32_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb5,0xb4,0x7f]
+0xdb,0xb5,0xb4,0x7f
+
+# GFX950: v_permlane32_swap_b32_e32 v218, v2      ; encoding: [0x02,0xb5,0xb4,0x7f]
+0x02,0xb5,0xb4,0x7f
+
+# GFX950: v_permlane32_swap_b32_e32 v2, v219      ; encoding: [0xdb,0xb5,0x04,0x7e]
+0xdb,0xb5,0x04,0x7e
+
 # GFX950:   v_permlane32_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00
 
+# GFX950: v_permlane32_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00]
+0xda,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00
+
+# GFX950: v_permlane32_swap_b32_e64 v218, v2      ; encoding: [0xda,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00]
+0xda,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00
+
+# GFX950: v_permlane32_swap_b32_e64 v2, v219      ; encoding: [0x02,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00]
+0x02,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00
+
 # GFX950:   v_permlane32_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x9a,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x10,0x9a,0xd1,0x02,0x01,0x00,0x00
 
diff --git a/llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt b/llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt
new file mode 100644
index 0000000000000..e2f06c601fd5f
--- /dev/null
+++ b/llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt
@@ -0,0 +1,19 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=xtensa -mattr=+dfpaccel -disassemble %s | FileCheck -check-prefixes=CHECK-DFPACCEL %s
+# RUN: not llvm-mc -triple=xtensa -disassemble %s 2>&1 | FileCheck --implicit-check-not=warning: -check-prefixes=CHECK-CORE %s
+
+## Verify that binary code is correctly disassembled with
+## DFPACCEL option enabled. Also verify that dissasembling without
+## DFPACCEL option generates warnings.
+
+[0xa0,0x3e,0xe3]
+# CHECK-DFPACCEL: rur	a3, f64r_lo
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xb0,0x3e,0xe3]
+# CHECK-DFPACCEL: rur	a3, f64r_hi
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xc0,0x3e,0xe3]
+# CHECK-DFPACCEL: rur	a3, f64s
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/Xtensa/fp.txt b/llvm/test/MC/Disassembler/Xtensa/fp.txt
index 993b9987c1bca..77ced9971fc11 100644
--- a/llvm/test/MC/Disassembler/Xtensa/fp.txt
+++ b/llvm/test/MC/Disassembler/Xtensa/fp.txt
@@ -7,209 +7,209 @@
 ## fp option generates warnings.
 
 [0x10,0x23,0xfa]
-# CHECK-FLOAT: abs.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: abs.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x0a]
-# CHECK-FLOAT: add.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: add.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xe0,0x23,0xfa]
-# CHECK-FLOAT: addexp.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: addexp.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xf0,0x23,0xfa]
-# CHECK-FLOAT: addexpm.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: addexpm.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xba]
-# CHECK-FLOAT: ceil.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ceil.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x35,0xfa]
-# CHECK-FLOAT: const.s f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: const.s	f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x70,0x23,0xfa]
-# CHECK-FLOAT: div0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: div0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x7a]
-# CHECK-FLOAT: divn.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: divn.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xca]
-# CHECK-FLOAT: float.s f2, a3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: float.s	f2, a3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xaa]
-# CHECK-FLOAT: floor.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: floor.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0x03,0x02]
-# CHECK-FLOAT: lsi f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsi	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0x83,0x02]
-# CHECK-FLOAT: lsip f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsip	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x08]
-# CHECK-FLOAT: lsx f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsx	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x18]
-# CHECK-FLOAT: lsxp f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsxp	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x4a]
-# CHECK-FLOAT: madd.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: madd.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x6a]
-# CHECK-FLOAT: maddn.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: maddn.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xd0,0x23,0xfa]
-# CHECK-FLOAT: mkdadj.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mkdadj.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xc0,0x23,0xfa]
-# CHECK-FLOAT: mksadj.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mksadj.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x00,0x23,0xfa]
-# CHECK-FLOAT: mov.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mov.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x8b]
-# CHECK-FLOAT: moveqz.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: moveqz.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x00,0x23,0xcb]
-# CHECK-FLOAT: movf.s f2, f3, b0
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movf.s	f2, f3, b0
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0xbb]
-# CHECK-FLOAT: movgez.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movgez.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0xab]
-# CHECK-FLOAT: movltz.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movltz.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x9b]
-# CHECK-FLOAT: movnez.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movnez.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x00,0x23,0xdb]
-# CHECK-FLOAT: movt.s f2, f3, b0
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movt.s	f2, f3, b0
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x5a]
-# CHECK-FLOAT: msub.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: msub.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x2a]
-# CHECK-FLOAT: mul.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mul.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x60,0x23,0xfa]
-# CHECK-FLOAT: neg.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: neg.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xb0,0x23,0xfa]
-# CHECK-FLOAT: nexp01.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: nexp01.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x2b]
-# CHECK-FLOAT: oeq.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: oeq.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x6b]
-# CHECK-FLOAT: ole.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ole.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x4b]
-# CHECK-FLOAT: olt.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: olt.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x80,0x23,0xfa]
-# CHECK-FLOAT: recip0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: recip0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0xfa]
-# CHECK-FLOAT: rfr a2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rfr	a2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0x8a]
-# CHECK-FLOAT: round.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: round.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xa0,0x23,0xfa]
-# CHECK-FLOAT: rsqrt0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rsqrt0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x90,0x23,0xfa]
-# CHECK-FLOAT: sqrt0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: sqrt0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0x43,0x02]
-# CHECK-FLOAT: ssi f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssi	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0xc3,0x02]
-# CHECK-FLOAT: ssip f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssip	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x48]
-# CHECK-FLOAT: ssx f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssx	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x58]
-# CHECK-FLOAT: ssxp f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssxp	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x1a]
-# CHECK-FLOAT: sub.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: sub.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0x9a]
-# CHECK-FLOAT: trunc.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: trunc.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x3b]
-# CHECK-FLOAT: ueq.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ueq.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xda]
-# CHECK-FLOAT: ufloat.s f2, a3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ufloat.s	f2, a3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x7b]
-# CHECK-FLOAT: ule.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ule.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x5b]
-# CHECK-FLOAT: ult.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ult.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x1b]
-# CHECK-FLOAT: un.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: un.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xea]
-# CHECK-FLOAT: utrunc.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: utrunc.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xfa]
-# CHECK-FLOAT: wfr f2, a3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: wfr	f2, a3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x80,0x3e,0xe3]
-# CHECK-FLOAT: rur a3, fcr
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rur	a3, fcr
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x90,0x3e,0xe3]
-# CHECK-FLOAT: rur a3, fsr
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rur	a3, fsr
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/Xtensa/threadptr.txt b/llvm/test/MC/Disassembler/Xtensa/threadptr.txt
new file mode 100644
index 0000000000000..1e8d75fa4003b
--- /dev/null
+++ b/llvm/test/MC/Disassembler/Xtensa/threadptr.txt
@@ -0,0 +1,11 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=xtensa -mattr=+threadptr -disassemble %s | FileCheck -check-prefixes=CHECK-THREADPTR %s
+# RUN: not llvm-mc -triple=xtensa -disassemble %s 2>&1 | FileCheck --implicit-check-not=warning: -check-prefixes=CHECK-CORE %s
+
+## Verify that binary code is correctly disassembled with
+## THREADPTR option enabled. Also verify that dissasembling without
+## THREADPTR option generates warnings.
+
+[0x70,0x3e,0xe3]
+# CHECK-THREADPTR: rur	a3, threadptr
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/ELF/bad-expr.s b/llvm/test/MC/ELF/bad-expr.s
index 2346f5493b98a..e7b5f19521019 100644
--- a/llvm/test/MC/ELF/bad-expr.s
+++ b/llvm/test/MC/ELF/bad-expr.s
@@ -7,6 +7,6 @@ x:
 // CHECK: [[@LINE+1]]:{{[0-9]+}}: error: symbol '__executable_start' can not be undefined in a subtraction expression
         .quad   x-__executable_start
 
-// CHECK: [[@LINE+1]]:7: error: expected relocatable expression
+// CHECK: [[@LINE+1]]:11: error: expected relocatable expression
 .long bar - foo@got
 foo:
diff --git a/llvm/test/MC/ELF/comdat.s b/llvm/test/MC/ELF/comdat.s
deleted file mode 100644
index e5862fab9521c..0000000000000
--- a/llvm/test/MC/ELF/comdat.s
+++ /dev/null
@@ -1,100 +0,0 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd - | FileCheck %s
-
-// Test that we produce the group sections and that they are before the members
-
-// CHECK:        Section {
-// CHECK:          Index: 3
-// CHECK-NEXT:     Name: .group
-// CHECK-NEXT:     Type: SHT_GROUP
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 12
-// CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 1
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 4
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000:    01000000 04000000 05000000
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-// CHECK:        Section {
-// CHECK:          Index: 6
-// CHECK-NEXT:     Name: .group
-// CHECK-NEXT:     Type: SHT_GROUP
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 8
-// CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 2
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 4
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000:    01000000 07000000
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-// CHECK:        Section {
-// CHECK:          Index: 8
-// CHECK-NEXT:     Name: .group
-// CHECK-NEXT:     Type: SHT_GROUP
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 12
-// CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 3
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 4
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000:    01000000 09000000 0A000000
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-
-// Test that g1 and g2 are local, but g3 is an undefined global.
-
-// CHECK:        Symbol {
-// CHECK:          Name: g1
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .foo
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: g2
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .group
-// CHECK-NEXT:   }
-
-// CHECK:        Symbol {
-// CHECK:          Name: g3
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined (0x0)
-// CHECK-NEXT:   }
-
-
-	.section	.foo,"axG",@progbits,g1,comdat
-g1:
-        nop
-
-        .section	.bar,"ax?",@progbits
-        nop
-
-        .section	.zed,"axG",@progbits,g2,comdat
-        nop
-
-        .section	.baz,"axG",@progbits,g3,comdat
-        .long g3
diff --git a/llvm/test/MC/ELF/group.s b/llvm/test/MC/ELF/group.s
new file mode 100644
index 0000000000000..8f6097e8a66f8
--- /dev/null
+++ b/llvm/test/MC/ELF/group.s
@@ -0,0 +1,59 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o %t
+# RUN: llvm-readelf -SsX %t | FileCheck %s
+# RUN: llvm-readelf -g %t | FileCheck %s --check-prefix=GROUP
+
+## Test that we produce the group sections and that they are before the members
+
+# CHECK:       [ 3] .group            GROUP           0000000000000000 {{.*}} 00000c 04     14   1  4
+# CHECK-NEXT:  [ 4] .foo              PROGBITS        0000000000000000 {{.*}} 000001 00 AXG  0   0  1
+# CHECK-NEXT:  [ 5] .bar              PROGBITS        0000000000000000 {{.*}} 000001 00 AXG  0   0  1
+# CHECK-NEXT:  [ 6] .group            GROUP           0000000000000000 {{.*}} 000008 04     14   2  4
+# CHECK-NEXT:  [ 7] .zed              PROGBITS        0000000000000000 {{.*}} 000001 00 AXG  0   0  1
+# CHECK-NEXT:  [ 8] .group            GROUP           0000000000000000 {{.*}} 00000c 04     14   4  4
+# CHECK-NEXT:  [ 9] .baz              PROGBITS        0000000000000000 {{.*}} 000004 00 AXG  0   0  1
+# CHECK-NEXT:  [10] .rela.baz         RELA            0000000000000000 {{.*}} 000018 18   G 14   9  8
+# CHECK-NEXT:  [11] sec               PROGBITS        0000000000000000 {{.*}} 000000 00      0   0  1
+# CHECK-NEXT:  [12] .group            GROUP           0000000000000000 {{.*}} 000008 04     14   3  4
+# CHECK-NEXT:  [13] .qux              PROGBITS        0000000000000000 {{.*}} 000000 00 AXG  0   0  1
+# CHECK-NEXT:  [14] .symtab           SYMTAB
+
+## Test that g1 and g2 are local, but g3 is an undefined global.
+
+# CHECK:      0000000000000000     0 NOTYPE  LOCAL  DEFAULT    4 (.foo)    g1
+# CHECK-NEXT: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT    6 (.group)  g2
+# CHECK-NEXT: 0000000000000000     0 SECTION LOCAL  DEFAULT   11 (sec)     sec
+# CHECK-NEXT: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT  UND           g3
+
+# GROUP:       COMDAT group section [    3] `.group' [g1] contains 2 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [    4]   .foo
+# GROUP-NEXT:     [    5]   .bar
+# GROUP-EMPTY:
+# GROUP-NEXT:  COMDAT group section [    6] `.group' [g2] contains 1 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [    7]   .zed
+# GROUP-EMPTY:
+# GROUP-NEXT:  COMDAT group section [    8] `.group' [g3] contains 2 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [    9]   .baz
+# GROUP-NEXT:     [   10]   .rela.baz
+# GROUP-EMPTY:
+# GROUP-NEXT:  group section [   12] `.group' [] contains 1 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [   13]   .qux
+
+	.section	.foo,"axG",@progbits,g1,comdat
+g1:
+        nop
+
+        .section	.bar,"ax?",@progbits
+        nop
+
+        .section	.zed,"axG",@progbits,g2,comdat
+        nop
+
+        .section	.baz,"axG",@progbits,g3,comdat
+        .long g3
+
+.section sec
+.section .qux,"axG",@progbits,sec
diff --git a/llvm/test/MC/ELF/mc-dump.s b/llvm/test/MC/ELF/mc-dump.s
new file mode 100644
index 0000000000000..3788eb093eef2
--- /dev/null
+++ b/llvm/test/MC/ELF/mc-dump.s
@@ -0,0 +1,53 @@
+# REQUIRES: asserts
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t -debug-only=mc-dump-pre,mc-dump 2>&1 | FileCheck %s --match-full-lines --strict-whitespace
+
+#CHECK-LABEL:assembler backend - pre-layout
+#      CHECK:MCSection Name:.text
+#CHECK-LABEL:assembler backend - final-layout
+#      CHECK:Sections:[
+# CHECK-NEXT:MCSection Name:.text
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 .text
+# CHECK-NEXT:0 Align Align:4 Value:0 ValueSize:1 MaxBytesToEmit:4 Nops
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 _start
+# CHECK-NEXT:0 Org Offset:3 Value:0
+# CHECK-NEXT:3 Relaxable Size:2 <MCInst #1996 <MCOperand Expr:.Ltmp0>>
+# CHECK-NEXT:  Fixup @1 Value:.Ltmp0 Kind:4001
+# CHECK-NEXT:5 Data Size:16 [48,8b,04,25,00,00,00,00,48,8b,04,25,00,00,00,00]
+# CHECK-NEXT:  Fixup @4 Value:f0@<variant 11> Kind:4017
+# CHECK-NEXT:  Fixup @12 Value:_start@<variant 11> Kind:4017
+# CHECK-NEXT:  Symbol @16 .Ltmp0 Temporary
+# CHECK-NEXT:MCSection Name:.data
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 .data
+# CHECK-NEXT:0 Align Align:4 Value:0 ValueSize:1 MaxBytesToEmit:4
+# CHECK-NEXT:0 Data Size:4 [01,00,00,00]
+# CHECK-NEXT:4 Fill Value:0 ValueSize:1 NumValues:1
+# CHECK-NEXT:5 LEB Value:.Ltmp0-_start Signed:0
+# CHECK-NEXT:]
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t -debug-only=mc-dump -save-temp-labels -g 2>&1 | FileCheck %s --check-prefix=CHECK2
+
+#      CHECK2:5 Data Size:16 [48,8b,04,25,00,00,00,00,48,8b,04,25,00,00,00,00]
+# CHECK2-NEXT:  Fixup @4 Value:f0@<variant 11> Kind:4017
+# CHECK2-NEXT:  Fixup @12 Value:_start@<variant 11> Kind:4017
+# CHECK2-NEXT:  Symbol @16 .Ltmp1
+# CHECK2-NEXT:  Symbol @0 .Ltmp3 Temporary
+# CHECK2-NEXT:  Symbol @8 .Ltmp4 Temporary
+# CHECK2-NEXT:  Symbol @16 .Ltmp5 Temporary
+# CHECK2-NEXT:  Symbol @16 .Lsec_end0 Temporary
+
+_start:
+var = _start
+.org 3
+jmp 1f
+movq f0@GOTPCREL, %rax
+movq _start@GOTPCREL, %rax
+1:
+
+.data
+.p2align 2
+.long 1
+.space 1
+.uleb128 1b-_start
diff --git a/llvm/test/MC/ELF/relocation-alias.s b/llvm/test/MC/ELF/relocation-alias.s
index 9720a55556ed0..154ba460ba685 100644
--- a/llvm/test/MC/ELF/relocation-alias.s
+++ b/llvm/test/MC/ELF/relocation-alias.s
@@ -2,8 +2,8 @@
 # RUN: llvm-objdump --no-print-imm-hex -dr %t | FileCheck %s
 # RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYM
 
-# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR0=1 %s 2>&1 | FileCheck %s --check-prefix=ERR0 --implicit-check-not=error:
-# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR=1 %s 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR0=1 %s -o %t 2>&1 | FileCheck %s --check-prefix=ERR0 --implicit-check-not=error:
+# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR=1 %s -o %t 2>&1 | FileCheck %s --check-prefix=ERR
 
 ## If a fixup symbol is equated to an undefined symbol, convert the fixup
 ## to be against the target symbol, even if there is a variant (@PLT).
@@ -51,7 +51,7 @@ memcpy_spec1 = __GI_memcpy@PLT+1
 .ifdef ERR
 .text
 ## Note, GNU as emits a relocation for this erroneous fixup.
-# ERR: {{.*}}.s:[[#@LINE+2]]:1: error: expected relocatable expression
+# ERR: {{.*}}.s:[[#@LINE+2]]:6: error: expected relocatable expression
 memcpy_plus_1 = __GI_memcpy+1
 call memcpy_plus_1@PLT
 .endif
diff --git a/llvm/test/MC/ELF/weakref.s b/llvm/test/MC/ELF/weakref.s
index a861625269538..49508463f8c53 100644
--- a/llvm/test/MC/ELF/weakref.s
+++ b/llvm/test/MC/ELF/weakref.s
@@ -113,6 +113,6 @@ alias:
 .weakref cycle1, cycle0
 call cycle0
 # ERR2: <unknown>:0: error: cyclic dependency detected for symbol 'cycle0'
-# ERR2: [[#@LINE-2]]:1: error: expected relocatable expression
+# ERR2: [[#@LINE-2]]:6: error: expected relocatable expression
 
 .endif
diff --git a/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s b/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s
index c72eef7cd9916..c668729e44c91 100644
--- a/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s
+++ b/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s
@@ -1,20 +1,20 @@
 # RUN: not llvm-mc --triple=loongarch64 --filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
 
-  beq $a0, $a1, unaligned # CHECK: :[[#@LINE]]:3: error: fixup value must be 4-byte aligned
-  beqz $a0, unaligned # CHECK: :[[#@LINE]]:3: error: fixup value must be 4-byte aligned
-  b unaligned # CHECK: :[[#@LINE]]:3: error: fixup value must be 4-byte aligned
+  beq $a0, $a1, unaligned # CHECK: :[[#@LINE]]:17: error: fixup value must be 4-byte aligned
+  beqz $a0, unaligned # CHECK: :[[#@LINE]]:13: error: fixup value must be 4-byte aligned
+  b unaligned # CHECK: :[[#@LINE]]:5: error: fixup value must be 4-byte aligned
   .byte 0
 unaligned:
   .byte 0
   .byte 0
   .byte 0
 
-  beq $a0, $a1, out_of_range_b18 # CHECK: :[[#@LINE]]:3: error: fixup value out of range [-131072, 131071]
+  beq $a0, $a1, out_of_range_b18 # CHECK: :[[#@LINE]]:17: error: fixup value out of range [-131072, 131071]
   .space 1<<18
 out_of_range_b18:
-  beqz $a0, out_of_range_b23 # CHECK: :[[#@LINE]]:3: error: fixup value out of range [-4194304, 4194303]
+  beqz $a0, out_of_range_b23 # CHECK: :[[#@LINE]]:13: error: fixup value out of range [-4194304, 4194303]
   .space 1<<23
 out_of_range_b23:
-  b out_of_range_b28 # CHECK: :[[#@LINE]]:3: error: fixup value out of range [-134217728, 134217727]
+  b out_of_range_b28 # CHECK: :[[#@LINE]]:5: error: fixup value out of range [-134217728, 134217727]
   .space 1<<28
 out_of_range_b28:
diff --git a/llvm/test/MC/M68k/Control/branch-pc-rel.s b/llvm/test/MC/M68k/Control/branch-pc-rel.s
index 912959ba94fac..bfeed3addbf2b 100644
--- a/llvm/test/MC/M68k/Control/branch-pc-rel.s
+++ b/llvm/test/MC/M68k/Control/branch-pc-rel.s
@@ -2,11 +2,9 @@
 
 	; CHECK:      beq  .LBB0_1
 	; CHECK-SAME: encoding: [0x67,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
 	beq	.LBB0_1
 	; CHECK:      bra  .LBB0_2
 	; CHECK-SAME: encoding: [0x60,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 .LBB0_1:
 	; CHECK:      add.l  #0, %d0
diff --git a/llvm/test/MC/M68k/Control/bsr.s b/llvm/test/MC/M68k/Control/bsr.s
index 6b7a7d5c6ecd9..71ceb56ea66a4 100644
--- a/llvm/test/MC/M68k/Control/bsr.s
+++ b/llvm/test/MC/M68k/Control/bsr.s
@@ -2,11 +2,9 @@
 
 	; CHECK:      bsr.b   .LBB0_1
 	; CHECK-SAME: encoding: [0x61,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
 	bsr.b .LBB0_1
 	; CHECK:      bsr.w   .LBB0_2
 	; CHECK-SAME: encoding: [0x61,0x00,A,A]
-        ; CHECK:      fixup A - offset: 2, value: .LBB0_2, kind: FK_PCRel_2
 	bsr.w	.LBB0_2
 .LBB0_1:
 	; CHECK:      add.l  #0, %d0
diff --git a/llvm/test/MC/M68k/Control/bsr32.s b/llvm/test/MC/M68k/Control/bsr32.s
index 58ed4a29a3531..5151d6281ef3f 100644
--- a/llvm/test/MC/M68k/Control/bsr32.s
+++ b/llvm/test/MC/M68k/Control/bsr32.s
@@ -2,15 +2,12 @@
 
 	; CHECK:      bsr.b   .LBB0_1
 	; CHECK-SAME: encoding: [0x61,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
 	bsr.b .LBB0_1
 	; CHECK:      bsr.w   .LBB0_2
 	; CHECK-SAME: encoding: [0x61,0x00,A,A]
-        ; CHECK:      fixup A - offset: 2, value: .LBB0_2, kind: FK_PCRel_2
 	bsr.w	.LBB0_2
   ; CHECK:     bsr.l   .LBB0_3
   ; CHECK-SAME: encoding: [0x61,0xff,A,A,A,A] 
-        ; CHECK:      fixup A - offset: 2, value: .LBB0_3, kind: FK_PCRel_4
   bsr.l .LBB0_3  
 .LBB0_1:
 	; CHECK:      add.l  #0, %d0
diff --git a/llvm/test/MC/M68k/Control/call-pc-rel.s b/llvm/test/MC/M68k/Control/call-pc-rel.s
index ee0147ef52c93..26d30530ca919 100644
--- a/llvm/test/MC/M68k/Control/call-pc-rel.s
+++ b/llvm/test/MC/M68k/Control/call-pc-rel.s
@@ -16,12 +16,10 @@ PCI:
 	; CHECK:     jsr  (BACKWARD,%pc,%d0)
         ; CHECK-OBJ: jsr  (250,%pc,%d0)
 	; CHECK-SAME: encoding: [0x4e,0xbb,0x08,A]
-        ; CHECK:      fixup A - offset: 3, value: BACKWARD+1, kind: FK_PCRel_1
 	jsr	(BACKWARD,%pc,%d0)
 	; CHECK:     jsr  (FORWARD,%pc,%d0)
         ; CHECK-OBJ: jsr  (10,%pc,%d0)
 	; CHECK-SAME: encoding: [0x4e,0xbb,0x08,A]
-        ; CHECK:      fixup A - offset: 3, value: FORWARD+1, kind: FK_PCRel_1
 	jsr	(FORWARD,%pc,%d0)
 
 ; CHECK-LABEL: PCD:
@@ -29,12 +27,10 @@ PCD:
 	; CHECK:     jsr  (BACKWARD,%pc)
 	; CHECK-OBJ: jsr  (65522,%pc)
 	; CHECK-SAME: encoding: [0x4e,0xba,A,A]
-        ; CHECK:      fixup A - offset: 2, value: BACKWARD, kind: FK_PCRel_2
 	jsr	(BACKWARD,%pc)
 	; CHECK:     jsr  (FORWARD,%pc)
 	; CHECK-OBJ: jsr  (2,%pc)
 	; CHECK-SAME: encoding: [0x4e,0xba,A,A]
-        ; CHECK:      fixup A - offset: 2, value: FORWARD, kind: FK_PCRel_2
 	jsr	(FORWARD,%pc)
 
 ; CHECK-LABEL: FORWARD:
diff --git a/llvm/test/MC/M68k/Relaxations/PIC/branch.s b/llvm/test/MC/M68k/Relaxations/PIC/branch.s
index 5035ca3ce50a7..e3384e57eae05 100644
--- a/llvm/test/MC/M68k/Relaxations/PIC/branch.s
+++ b/llvm/test/MC/M68k/Relaxations/PIC/branch.s
@@ -1,13 +1,12 @@
 ; RUN: llvm-mc -triple=m68k -motorola-integers -filetype=obj --position-independent < %s \
 ; RUN:     | llvm-objdump -d - | FileCheck %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; CHECK-LABEL: <TIGHT>:
 TIGHT:
 	; CHECK: bra  $7f
 	; INSTR: bra .LBB0_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 	.space 0x7F  ; i8::MAX
 .LBB0_2:
@@ -18,7 +17,6 @@ TIGHT:
 RELAXED:
 	; CHECK: bra  $82
 	; INSTR: bra .LBB1_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 	bra	.LBB1_2
 	.space 0x80  ; Greater than i8::MAX
 .LBB1_2:
@@ -29,7 +27,6 @@ RELAXED:
 ZERO:
 	; CHECK: bra  $2
 	; INSTR: bra .LBB3_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
 	bra	.LBB3_1
 .LBB3_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/M68k/Relaxations/PIC/branch32.s b/llvm/test/MC/M68k/Relaxations/PIC/branch32.s
index 7e40f055aff13..1d5d4f09267c6 100644
--- a/llvm/test/MC/M68k/Relaxations/PIC/branch32.s
+++ b/llvm/test/MC/M68k/Relaxations/PIC/branch32.s
@@ -1,7 +1,7 @@
 ; RUN: llvm-mc -triple=m68k --mcpu=M68020 -motorola-integers -filetype=obj --position-independent < %s \
 ; RUN:     | llvm-objdump -d - | FileCheck %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; Branch relocations are relaxed as part of object layout, so -show-encoding still 
 ; shows them as 1-byte relocactions 
@@ -10,7 +10,6 @@
 TIGHT:
 	; CHECK: bra  $7f
 	; INSTR: bra .LBB0_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 	.space 0x7F  ; i8::MAX
 .LBB0_2:
@@ -21,7 +20,6 @@ TIGHT:
 RELAXED:
 	; CHECK: bra  $82
 	; INSTR: bra .LBB1_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 	bra	.LBB1_2
 	.space 0x80  ; Greater than i8::MAX
 .LBB1_2:
@@ -34,7 +32,6 @@ RELAXED_32:
 	; CHECK-NEXT: 00 00
 	; CHECK-NEXT: 80 04
 	; INSTR: bra .LBB2_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB2_1-1, kind: FK_PCRel_1
 	bra	.LBB2_1
 	.space 0x8000  ; Greater than i16::MAX.
 .LBB2_1:
@@ -45,7 +42,6 @@ RELAXED_32:
 ZERO:
 	; CHECK: bra  $2
 	; INSTR: bra .LBB3_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
 	bra	.LBB3_1
 .LBB3_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/M68k/Relaxations/branch32.s b/llvm/test/MC/M68k/Relaxations/branch32.s
index ba5c371dada8d..766de3c1da19a 100644
--- a/llvm/test/MC/M68k/Relaxations/branch32.s
+++ b/llvm/test/MC/M68k/Relaxations/branch32.s
@@ -1,13 +1,12 @@
 ; RUN: llvm-mc -triple=m68k --mcpu=M68020 -motorola-integers -filetype=obj < %s \
 ; RUN:     | llvm-objdump -d - | FileCheck %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; CHECK-LABEL: <TIGHT>:
 TIGHT:
 	; CHECK: bra  $7f
 	; INSTR: bra .LBB0_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 	.space 0x7F  ; i8::MAX
 .LBB0_2:
@@ -18,7 +17,6 @@ TIGHT:
 RELAXED:
 	; CHECK: bra  $82
 	; INSTR: bra .LBB1_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 	bra	.LBB1_2
 	.space 0x80  ; Greater than i8::MAX
 .LBB1_2:
@@ -31,7 +29,6 @@ RELAXED_32:
 	; CHECK-NEXT: 00 00
 	; CHECK-NEXT: 80 04
 	; INSTR: bra .LBB2_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB2_1-1, kind: FK_PCRel_1
 	bra	.LBB2_1
 	.space 0x8000  ; Greater than i16::MAX.
 .LBB2_1:
@@ -42,7 +39,6 @@ RELAXED_32:
 ZERO:
 	; CHECK: bra  $2
 	; INSTR: bra .LBB3_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
 	bra	.LBB3_1
 .LBB3_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-abs.s b/llvm/test/MC/M68k/Relocations/PIC/data-abs.s
index 0df09c8cb5ccb..8a77c40117a96 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-abs.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-abs.s
@@ -1,26 +1,22 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_32 dst 0x0
 ; INSTR: move.l dst, %d0
-; FIXUP: fixup A - offset: 2, value: dst, kind: FK_Data_4
 move.l	dst, %d0
 
 ; Relocating immediate values
 
 ; RELOC: R_68K_8 str8 0x0
 ; INSTR: move.b  #str8,  (4,%sp)
-; FIXUP: fixup A - offset: 3, value: str8, kind: FK_Data_1
 move.b  #str8,  (4,%sp)
 
 ; RELOC: R_68K_16 str16 0x0
 ; INSTR: move.w  #str16,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str16, kind: FK_Data_2
 move.w  #str16, (4,%sp)
 
 ; RELOC: R_68K_32 str32 0x0
 ; INSTR: move.l  #str32,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str32, kind: FK_Data_4
 move.l  #str32, (4,%sp)
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s b/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s
index ca1ce52de1ca7..ef6c25926f02c 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s
@@ -1,19 +1,16 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTOFF8 dst1 0x0
 ; INSTR: move.l  (dst1@GOTOFF,%a5,%d0), %d0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTOFF, kind: FK_Data_1
 move.l	(dst1@GOTOFF,%a5,%d0), %d0
 
 ; RELOC: R_68K_GOTOFF16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTOFF,%a5), %d0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTOFF, kind: FK_Data_2
 move.l	(dst2@GOTOFF,%a5), %d0
 
 ; RELOC: R_68K_GOTPCREL16 dst3 0x0
 ; INSTR: lea     (dst3@GOTPCREL,%pc), %a5
-; FIXUP: fixup A - offset: 2, value: dst3@GOTPCREL, kind: FK_PCRel_2
 lea	(dst3@GOTPCREL,%pc), %a5
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s b/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s
index 9f79dd94f8198..be84c090bc7db 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTPCREL8 dst1 0x1
 ; INSTR: move.l  (dst1@GOTPCREL,%pc,%d0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTPCREL+1, kind: FK_PCRel_1
 move.l	(dst1@GOTPCREL,%pc,%d0), %a0
 
 ; RELOC: R_68K_GOTPCREL16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTPCREL,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTPCREL, kind: FK_PCRel_2
 move.l	(dst2@GOTPCREL,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s b/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s
index ef57ca589e64f..8eb1861216494 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s
@@ -1,20 +1,17 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PC8 dst1 0x1
 ; INSTR: move.l  (dst1,%pc,%a0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1+1, kind: FK_PCRel_1
 move.l	(dst1,%pc,%a0), %a0
 
 ; RELOC: R_68K_PC16 dst2 0x0
 ; INSTR: move.l  (dst2,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2, kind: FK_PCRel_2
 move.l	(dst2,%pc), %a0
 
 ; Shouldn't have any relocation
 ; RELOC-NOT: R_68K_PC
 ; INSTR: move.l  (0,%pc), %a0
-; FIXUP-NOT: fixup
 move.l	(0,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/PIC/text-plt.s b/llvm/test/MC/M68k/Relocations/PIC/text-plt.s
index 0ccd1758004a9..d5a103079c61a 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/text-plt.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/text-plt.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 --position-independent -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 --position-independent -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PLT16 target 0x0
 ; INSTR: jsr     (target@PLT,%pc)
-; FIXUP: fixup A - offset: 2, value: target@PLT, kind: FK_PCRel_2
 jsr	(target@PLT,%pc)
 
 ; RELOC: R_68K_PLT32  __tls_get_addr 0x0
 ; INSTR: bsr.l   __tls_get_addr@PLT
-; FIXUP: fixup A - offset: 2, value: __tls_get_addr@PLT, kind: FK_PCRel_4
 bsr.l __tls_get_addr@PLT
diff --git a/llvm/test/MC/M68k/Relocations/data-abs.s b/llvm/test/MC/M68k/Relocations/data-abs.s
index b15007d8da360..4ddbd326d171e 100644
--- a/llvm/test/MC/M68k/Relocations/data-abs.s
+++ b/llvm/test/MC/M68k/Relocations/data-abs.s
@@ -1,26 +1,22 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_32 dst 0x0
 ; INSTR: move.l dst, %d0
-; FIXUP: fixup A - offset: 2, value: dst, kind: FK_Data_4
 move.l	dst, %d0
 
 ; Relocating immediate values
 
 ; RELOC: R_68K_8 str8 0x0
 ; INSTR: move.b  #str8,  (4,%sp)
-; FIXUP: fixup A - offset: 3, value: str8, kind: FK_Data_1
 move.b  #str8,  (4,%sp)
 
 ; RELOC: R_68K_16 str16 0x0
 ; INSTR: move.w  #str16,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str16, kind: FK_Data_2
 move.w  #str16, (4,%sp)
 
 ; RELOC: R_68K_32 str32 0x0
 ; INSTR: move.l  #str32,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str32, kind: FK_Data_4
 move.l  #str32, (4,%sp)
diff --git a/llvm/test/MC/M68k/Relocations/data-gotoff.s b/llvm/test/MC/M68k/Relocations/data-gotoff.s
index e0c3affc6d1d4..dc56e691937f1 100644
--- a/llvm/test/MC/M68k/Relocations/data-gotoff.s
+++ b/llvm/test/MC/M68k/Relocations/data-gotoff.s
@@ -1,19 +1,16 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTOFF8 dst1 0x0
 ; INSTR: move.l  (dst1@GOTOFF,%a5,%d0), %d0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTOFF, kind: FK_Data_1
 move.l	(dst1@GOTOFF,%a5,%d0), %d0
 
 ; RELOC: R_68K_GOTOFF16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTOFF,%a5), %d0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTOFF, kind: FK_Data_2
 move.l	(dst2@GOTOFF,%a5), %d0
 
 ; RELOC: R_68K_GOTPCREL16 dst3 0x0
 ; INSTR: lea     (dst3@GOTPCREL,%pc), %a5
-; FIXUP: fixup A - offset: 2, value: dst3@GOTPCREL, kind: FK_PCRel_2
 lea	(dst3@GOTPCREL,%pc), %a5
diff --git a/llvm/test/MC/M68k/Relocations/data-gotpcrel.s b/llvm/test/MC/M68k/Relocations/data-gotpcrel.s
index ac3ba96d1f09f..ceffc77dd6e35 100644
--- a/llvm/test/MC/M68k/Relocations/data-gotpcrel.s
+++ b/llvm/test/MC/M68k/Relocations/data-gotpcrel.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTPCREL8 dst1 0x1
 ; INSTR: move.l  (dst1@GOTPCREL,%pc,%d0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTPCREL+1, kind: FK_PCRel_1
 move.l	(dst1@GOTPCREL,%pc,%d0), %a0
 
 ; RELOC: R_68K_GOTPCREL16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTPCREL,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTPCREL, kind: FK_PCRel_2
 move.l	(dst2@GOTPCREL,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/data-pc-rel.s b/llvm/test/MC/M68k/Relocations/data-pc-rel.s
index 909681da5a950..23e0bcc732696 100644
--- a/llvm/test/MC/M68k/Relocations/data-pc-rel.s
+++ b/llvm/test/MC/M68k/Relocations/data-pc-rel.s
@@ -1,20 +1,17 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PC8 dst1 0x1
 ; INSTR: move.l  (dst1,%pc,%a0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1+1, kind: FK_PCRel_1
 move.l	(dst1,%pc,%a0), %a0
 
 ; RELOC: R_68K_PC16 dst2 0x0
 ; INSTR: move.l  (dst2,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2, kind: FK_PCRel_2
 move.l	(dst2,%pc), %a0
 
 ; Shouldn't have any relocation
 ; RELOC-NOT: R_68K_PC
 ; INSTR: move.l  (0,%pc), %a0
-; FIXUP-NOT: fixup
 move.l	(0,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/text-plt.s b/llvm/test/MC/M68k/Relocations/text-plt.s
index 7de04b8b2182a..131e669474442 100644
--- a/llvm/test/MC/M68k/Relocations/text-plt.s
+++ b/llvm/test/MC/M68k/Relocations/text-plt.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PLT16 target 0x0
 ; INSTR: jsr     (target@PLT,%pc)
-; FIXUP: fixup A - offset: 2, value: target@PLT, kind: FK_PCRel_2
 jsr	(target@PLT,%pc)
 
 ; RELOC: R_68K_PLT32  __tls_get_addr 0x0
 ; INSTR: bsr.l   __tls_get_addr@PLT
-; FIXUP: fixup A - offset: 2, value: __tls_get_addr@PLT, kind: FK_PCRel_4
 bsr.l __tls_get_addr@PLT
diff --git a/llvm/test/MC/M68k/Relocations/tls.s b/llvm/test/MC/M68k/Relocations/tls.s
index ec7e11f17fe70..50f67eb5f158c 100644
--- a/llvm/test/MC/M68k/Relocations/tls.s
+++ b/llvm/test/MC/M68k/Relocations/tls.s
@@ -1,17 +1,12 @@
-; RUN: llvm-mc -triple m68k -show-encoding %s | FileCheck -check-prefix=INSTR %s
+; RUN: llvm-mc -triple m68k %s | FileCheck -check-prefix=INSTR %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -filetype=obj %s -o %t
 ; RUN: llvm-readelf -rs %t | FileCheck --check-prefix=READELF %s --implicit-check-not=TLS
 
 ; INSTR:      move.l  (le@TPOFF,%a0), %d0
-; INSTR-NEXT:                fixup A - offset: 2, value: le@TPOFF, kind: FK_Data_2
 ; INSTR-NEXT: move.l  (ie@GOTTPOFF,%a0), %a2
-; INSTR-NEXT:                fixup A - offset: 2, value: ie@GOTTPOFF, kind: FK_Data_2
 ; INSTR-NEXT: lea     (gd@TLSGD,%a0), %a0
-; INSTR-NEXT:                fixup A - offset: 2, value: gd@TLSGD, kind: FK_Data_2
 ; INSTR-NEXT: move.l  (ld@TLSLD,%a0), %d0
-; INSTR-NEXT:                fixup A - offset: 2, value: ld@TLSLD, kind: FK_Data_2
 ; INSTR-NEXT: lea     (ld@TLSLDM,%a2), %a1
-; INSTR-NEXT:                fixup A - offset: 2, value: ld@TLSLDM, kind: FK_Data_2
 
 ; READELF:      R_68K_TLS_LE16         00000000   le + 0
 ; READELF-NEXT: R_68K_TLS_IE16         00000000   ie + 0
diff --git a/llvm/test/MC/M68k/pc-rel.s b/llvm/test/MC/M68k/pc-rel.s
index a4d76d341bb05..f5d98ec855e6c 100644
--- a/llvm/test/MC/M68k/pc-rel.s
+++ b/llvm/test/MC/M68k/pc-rel.s
@@ -3,7 +3,6 @@
         ; A fixup whose size is multiple of a word.
 	; CHECK:      cmpi.l  #87, (.LBB0_1,%pc)
 	; CHECK-SAME: encoding: [0x0c,0xba,0x00,0x00,0x00,0x57,A,A]
-        ; CHECK:      fixup A - offset: 6, value: .LBB0_1, kind: FK_PCRel_2
 	cmpi.l  #87, (.LBB0_1,%pc)
 
         ; A fixup that is smaller than a word.
@@ -11,7 +10,6 @@
         ; tested by `Control/branch-pc-rel.s`.
 	; CHECK:      cmpi.l  #94, (.LBB0_2,%pc,%a0)
 	; CHECK-SAME: encoding: [0x0c,0xbb,0x00,0x00,0x00,0x5e,0x88,A]
-        ; CHECK:      fixup A - offset: 7, value: .LBB0_2+1, kind: FK_PCRel_1
 	cmpi.l  #94, (.LBB0_2,%pc,%a0)
 .LBB0_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/Mips/reloc-directive-bad-obj.s b/llvm/test/MC/Mips/reloc-directive-bad-obj.s
index 94317d0830df7..86d6d0cc66c57 100644
--- a/llvm/test/MC/Mips/reloc-directive-bad-obj.s
+++ b/llvm/test/MC/Mips/reloc-directive-bad-obj.s
@@ -1,9 +1,9 @@
-# RUN: not llvm-mc -triple mips-unknown-linux %s -show-encoding \
-# RUN:     -target-abi=o32 -filetype=obj 2>&1 | FileCheck %s
-	.text
-	nop
-	.reloc foo, R_MIPS_32, .text  # CHECK: :[[@LINE]]:2: error: unresolved relocation offset
-	nop
-	nop
-	.reloc bar, R_MIPS_32, .text  # CHECK: :[[@LINE]]:2: error: unresolved relocation offset
-	nop
+# RUN: not llvm-mc -triple mips-unknown-linux %s \
+# RUN:     -target-abi=o32 -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+.text
+nop
+.reloc foo, R_MIPS_32, .text  # CHECK: :[[@LINE]]:24: error: unresolved relocation offset
+nop
+nop
+.reloc bar, R_MIPS_32, .text  # CHECK: :[[@LINE]]:24: error: unresolved relocation offset
+nop
diff --git a/llvm/test/MC/Mips/unsupported-relocation.s b/llvm/test/MC/Mips/unsupported-relocation.s
index 311c0c7d60f20..4cbcff2f168b7 100644
--- a/llvm/test/MC/Mips/unsupported-relocation.s
+++ b/llvm/test/MC/Mips/unsupported-relocation.s
@@ -10,4 +10,4 @@ foo:
         .byte   x
 # CHECK: :[[@LINE-1]]:17: error: MIPS does not support one byte relocations
         .byte   x+1
-# CHECK: :[[@LINE-1]]:17: error: MIPS does not support one byte relocations
+# CHECK: :[[@LINE-1]]:18: error: MIPS does not support one byte relocations
diff --git a/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s b/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s
index 24fbccdb8abdb..2b97dca147bfe 100644
--- a/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s
+++ b/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s
@@ -28,21 +28,21 @@ data1:
 .word %gotpcrel(extern+4), %gotpcrel(extern-5)
 
 .ifdef ERR
-# ERR: [[#@LINE+1]]:7: error: %pltpcrel can only be used in a .word directive
+# ERR: [[#@LINE+1]]:8: error: %pltpcrel can only be used in a .word directive
 .quad %pltpcrel(g)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %pltpcrel(g-.)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %pltpcrel(extern - und)
 
-# ERR: [[#@LINE+1]]:7: error: %gotpcrel can only be used in a .word directive
+# ERR: [[#@LINE+1]]:8: error: %gotpcrel can only be used in a .word directive
 .quad %gotpcrel(g)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %gotpcrel(extern - .)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %gotpcrel(extern - und)
 .endif
diff --git a/llvm/test/MC/RISCV/Relocations/expr-err.s b/llvm/test/MC/RISCV/Relocations/expr-err.s
index 897004b646368..481a379321a20 100644
--- a/llvm/test/MC/RISCV/Relocations/expr-err.s
+++ b/llvm/test/MC/RISCV/Relocations/expr-err.s
@@ -3,26 +3,26 @@
 .Ltmp1:
 	.quad	tls
 
-	lui a0, %hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	lw a0, %lo(tls+0-.Ltmp1)(t0)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	lui a0, %tprel_hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	add a0, a0, tp, %tprel_add(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	addi a0, a0, %tprel_lo(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %tls_ie_pcrel_hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %tls_gd_pcrel_hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %pcrel_hi(tls-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %got_pcrel_hi(tls-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	addi a0, a0, %pcrel_lo(tls-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
+lui a0, %hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:10: error: expected relocatable expression
+lw a0, %lo(tls+0-.Ltmp1)(t0)
+# CHECK: :[[#@LINE-1]]:9: error: expected relocatable expression
+lui a0, %tprel_hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:10: error: expected relocatable expression
+add a0, a0, tp, %tprel_add(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:18: error: expected relocatable expression
+addi a0, a0, %tprel_lo(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:15: error: expected relocatable expression
+auipc a0, %tls_ie_pcrel_hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+auipc a0, %tls_gd_pcrel_hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+auipc a0, %pcrel_hi(tls-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+auipc a0, %got_pcrel_hi(tls-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+addi a0, a0, %pcrel_lo(tls-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:15: error: expected relocatable expression
 
 #	tail tls+32
 #	tail tls-tls
diff --git a/llvm/test/MC/RISCV/Relocations/mc-dump.s b/llvm/test/MC/RISCV/Relocations/mc-dump.s
new file mode 100644
index 0000000000000..98205925da1cd
--- /dev/null
+++ b/llvm/test/MC/RISCV/Relocations/mc-dump.s
@@ -0,0 +1,20 @@
+# REQUIRES: asserts
+# RUN: llvm-mc -filetype=obj --triple=riscv64 --mattr=+relax %s -debug-only=mc-dump -o /dev/null 2>&1 | FileCheck %s
+
+#      CHECK:Sections:[
+# CHECK-NEXT:MCSection Name:.text
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 .text
+# CHECK-NEXT:0 Align Align:4 Value:0 ValueSize:1 MaxBytesToEmit:4 Nops
+# CHECK-NEXT:0 Data LinkerRelaxable Size:8 [97,00,00,00,e7,80,00,00]
+# CHECK-NEXT:  Fixup @0 Value:specifier(19,ext) Kind:4022
+# CHECK-NEXT:  Symbol @0 $x
+# CHECK-NEXT:8 Align Align:8 Value:0 ValueSize:1 MaxBytesToEmit:8 Nops
+# CHECK-NEXT:12 Data Size:4 [13,05,30,00]
+# CHECK-NEXT:16 Align Align:8 Value:0 ValueSize:1 MaxBytesToEmit:8 Nops
+# CHECK-NEXT:]
+
+call ext
+.p2align 3
+li x10, 3
+.p2align 3
diff --git a/llvm/test/MC/RISCV/fixups-diagnostics.s b/llvm/test/MC/RISCV/fixups-diagnostics.s
index 3c99673b81fc2..d19f4050d97e4 100644
--- a/llvm/test/MC/RISCV/fixups-diagnostics.s
+++ b/llvm/test/MC/RISCV/fixups-diagnostics.s
@@ -1,9 +1,9 @@
 # RUN: not llvm-mc -triple riscv32 -filetype obj < %s -o /dev/null 2>&1 | FileCheck %s
 
-  jal a0, far_distant # CHECK: :[[@LINE]]:3: error: fixup value out of range
-  jal a0, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+  jal a0, far_distant # CHECK: :[[@LINE]]:11: error: fixup value out of range
+  jal a0, unaligned # CHECK: :[[@LINE]]:11: error: fixup value must be 2-byte aligned
 
-  blt t0, t1, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+  blt t0, t1, unaligned # CHECK: :[[@LINE]]:15: error: fixup value must be 2-byte aligned
 
   .byte 0
 unaligned:
diff --git a/llvm/test/MC/RISCV/option-exact-long-jump-disable.s b/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
index 6e4b5fdd31e17..79ca919f8b9d5 100644
--- a/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
+++ b/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
@@ -40,12 +40,12 @@ bar:
   .text
 ## Branches to defined out-of-range symbols should report an error
 test_defined_out_of_range:
-  bne a0, a1, 1f # CHECK: :[[#@LINE]]:3: error: fixup value out of range
+  bne a0, a1, 1f # CHECK: :[[#@LINE]]:15: error: fixup value out of range
   .skip (1 << 12)
 1:
-  c.beqz a0, 1f # CHECK: :[[#@LINE]]:3: error: fixup value out of range
+  c.beqz a0, 1f # CHECK: :[[#@LINE]]:14: error: fixup value out of range
   .skip (1 << 8)
 1:
-  c.j 1f # CHECK: :[[#@LINE]]:3: error: fixup value out of range
+  c.j 1f # CHECK: :[[#@LINE]]:7: error: fixup value out of range
   .skip (1 << 11)
 1:
diff --git a/llvm/test/MC/RISCV/pcrel-lo12-invalid.s b/llvm/test/MC/RISCV/pcrel-lo12-invalid.s
index 74a1f2fac9233..e6dce0d6c6981 100644
--- a/llvm/test/MC/RISCV/pcrel-lo12-invalid.s
+++ b/llvm/test/MC/RISCV/pcrel-lo12-invalid.s
@@ -2,6 +2,6 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=+relax -filetype obj < %s -o /dev/null 2>&1 | FileCheck %s
 
 1:
-  addi a0, a0, %pcrel_lo(1b) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
-  addi a0, a0, %pcrel_lo(0x123456) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
-  addi a0, a0, %pcrel_lo(foo) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
+  addi a0, a0, %pcrel_lo(1b) # CHECK: :[[@LINE]]:17: error: could not find corresponding %pcrel_hi
+  addi a0, a0, %pcrel_lo(0x123456) # CHECK: :[[@LINE]]:17: error: could not find corresponding %pcrel_hi
+  addi a0, a0, %pcrel_lo(foo) # CHECK: :[[@LINE]]:17: error: could not find corresponding %pcrel_hi
diff --git a/llvm/test/MC/RISCV/rv32-relaxation-xqci.s b/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
index 03c9914bb37d2..b38aa373c90f0 100644
--- a/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
+++ b/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
@@ -160,9 +160,9 @@ EXT_JUMP_NEGATIVE:
 .space 0x1000
 
   jal t1, EXT_JUMP
-# ERROR: [[@LINE-1]]:3: error: fixup value out of range
+# ERROR: [[@LINE-1]]:11: error: fixup value out of range
   jal t1, EXT_JUMP_NEGATIVE
-# ERROR: [[@LINE-1]]:3: error: fixup value out of range
+# ERROR: [[@LINE-1]]:11: error: fixup value out of range
 
 .space 0x1000
 .space 0x100000
diff --git a/llvm/test/MC/RISCV/rv32p-invalid.s b/llvm/test/MC/RISCV/rv32p-invalid.s
new file mode 100644
index 0000000000000..da3c67b73d0af
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv32p-invalid.s
@@ -0,0 +1,19 @@
+# RUN: not llvm-mc -triple=riscv32 --mattr=+experimental-p %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+# Imm overflow
+pli.h a0, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
+plui.h a1, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 1023]
+pli.b a0, 0x200
+# CHECK-ERROR: immediate must be an integer in the range [0, 255]
+
+pslli.b a6, a7, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 7]
+pslli.h ra, sp, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+psslai.h t0, t1, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+sslai a4, a5, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
diff --git a/llvm/test/MC/RISCV/rv32p-valid.s b/llvm/test/MC/RISCV/rv32p-valid.s
index e2e366593f9b9..c755acddc712e 100644
--- a/llvm/test/MC/RISCV/rv32p-valid.s
+++ b/llvm/test/MC/RISCV/rv32p-valid.s
@@ -1,36 +1,79 @@
 # RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-p -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-p < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-p -M no-aliases -d -r - \
-# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN:     | llvm-objdump --mattr=+experimental-p -M no-aliases -d -r --no-print-imm-hex - \
+# RUN:     | FileCheck --check-prefixes=CHECK-ASM-AND-OBJ,CHECK-OBJ %s
 
-# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
-# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
-sh1add a0, a1, a2
 # CHECK-ASM-AND-OBJ: clz a0, a1
 # CHECK-ASM: encoding: [0x13,0x95,0x05,0x60]
 clz a0, a1
+# CHECK-ASM-AND-OBJ: cls a1, a2
+# CHECK-ASM: encoding: [0x93,0x15,0x36,0x60]
+cls a1, a2
 # CHECK-ASM-AND-OBJ: sext.b a2, a3
 # CHECK-ASM: encoding: [0x13,0x96,0x46,0x60]
 sext.b a2, a3
 # CHECK-ASM-AND-OBJ: sext.h t0, t1
 # CHECK-ASM: encoding: [0x93,0x12,0x53,0x60]
 sext.h t0, t1
+# CHECK-ASM-AND-OBJ: abs a4, a5
+# CHECK-ASM: encoding: [0x13,0x97,0x77,0x60]
+abs a4, a5
+# CHECK-ASM-AND-OBJ: rev8 s0, s1
+# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x69]
+rev8 s0, s1
+# CHECK-ASM-AND-OBJ: rev s2, s3
+# CHECK-ASM: encoding: [0x13,0xd9,0xf9,0x69]
+rev s2, s3
+# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
+# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
+sh1add a0, a1, a2
+# CHECK-ASM-AND-OBJ: pack s0, s1, s2
+# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
+pack s0, s1, s2
 # CHECK-ASM-AND-OBJ: min t0, t1, t2
 # CHECK-ASM: encoding: [0xb3,0x42,0x73,0x0a]
 min t0, t1, t2
-# CHECK-ASM-AND-OBJ: minu t0, t1, t2
-# CHECK-ASM: encoding: [0xb3,0x52,0x73,0x0a]
-minu t0, t1, t2
+# CHECK-ASM-AND-OBJ: minu ra, sp, gp
+# CHECK-ASM: encoding: [0xb3,0x50,0x31,0x0a]
+minu ra, sp, gp
 # CHECK-ASM-AND-OBJ: max t3, t4, t5
 # CHECK-ASM: encoding: [0x33,0xee,0xee,0x0b]
 max t3, t4, t5
 # CHECK-ASM-AND-OBJ: maxu a4, a5, a6
 # CHECK-ASM: encoding: [0x33,0xf7,0x07,0x0b]
 maxu a4, a5, a6
-# CHECK-ASM-AND-OBJ: pack s0, s1, s2
-# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
-pack s0, s1, s2
-# CHECK-ASM-AND-OBJ: rev8 s0, s1
-# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x69]
-rev8 s0, s1
+# CHECK-ASM-AND-OBJ: pslli.b a6, a7, 0
+# CHECK-ASM: encoding: [0x1b,0xa8,0x88,0x80]
+pslli.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: pslli.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x20,0x11,0x81]
+pslli.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: psslai.h t0, t1, 2
+# CHECK-ASM: encoding: [0x9b,0x22,0x23,0xd1]
+psslai.h t0, t1, 2
+# CHECK-ASM-AND-OBJ: sslai a4, a5, 3
+# CHECK-ASM: encoding: [0x1b,0xa7,0x37,0xd2]
+sslai a4, a5, 3
+# CHECK-ASM-AND-OBJ: pli.h a5, 16
+# CHECK-ASM: encoding: [0x9b,0x27,0x10,0xb0]
+pli.h a5, 16
+# CHECK-ASM-AND-OBJ: pli.b a6, 16
+# CHECK-ASM: encoding: [0x1b,0x28,0x10,0xb4]
+pli.b a6, 16
+# CHECK-ASM-AND-OBJ: psext.h.b a7, a0
+# CHECK-ASM: encoding: [0x9b,0x28,0x45,0xe0]
+psext.h.b a7, a0
+# CHECK-ASM-AND-OBJ: psabs.h a1, a2
+# CHECK-ASM: encoding: [0x9b,0x25,0x76,0xe0]
+psabs.h a1, a2
+# CHECK-ASM-AND-OBJ: psabs.b t0, t1
+# CHECK-ASM: encoding: [0x9b,0x22,0x73,0xe4]
+psabs.b t0, t1
+# CHECK-ASM-AND-OBJ: plui.h gp, 32
+# CHECK-ASM: encoding: [0x9b,0x21,0x20,0xf0]
+plui.h gp, 32
+# CHECK-OBJ: plui.h gp, -412
+# CHECK-ASM: plui.h gp, 612
+# CHECK-ASM: encoding: [0x9b,0xa1,0x64,0xf0]
+plui.h gp, 612
diff --git a/llvm/test/MC/RISCV/rv64p-invalid.s b/llvm/test/MC/RISCV/rv64p-invalid.s
new file mode 100644
index 0000000000000..572a099fba4d4
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv64p-invalid.s
@@ -0,0 +1,21 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+experimental-p %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+# Imm overflow
+pli.h a0, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
+plui.h a1, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 1023]
+pli.w a1, -0x201
+# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
+
+pslli.b a6, a7, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 7]
+pslli.h ra, sp, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+pslli.w ra, sp, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+psslai.h t0, t1, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+psslai.w a4, a5, -1
+# CHECK-ERROR: error: immediate must be an integer in the range [0, 31]
diff --git a/llvm/test/MC/RISCV/rv64p-valid.s b/llvm/test/MC/RISCV/rv64p-valid.s
index 1ec5ee312e100..6c48ad3469b47 100644
--- a/llvm/test/MC/RISCV/rv64p-valid.s
+++ b/llvm/test/MC/RISCV/rv64p-valid.s
@@ -1,39 +1,110 @@
 # RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-p -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-p < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-p -M no-aliases -d -r - \
-# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj --triple=riscv64 -mattr=+experimental-p < %s \
+# RUN:     | llvm-objdump --triple=riscv64 --mattr=+experimental-p -M no-aliases --no-print-imm-hex -d -r - \
+# RUN:     | FileCheck --check-prefixes=CHECK-ASM-AND-OBJ,CHECK-OBJ %s
 
-# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
-# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
-sh1add a0, a1, a2
 # CHECK-ASM-AND-OBJ: clz a0, a1
 # CHECK-ASM: encoding: [0x13,0x95,0x05,0x60]
 clz a0, a1
-# CHECK-ASM-AND-OBJ: clzw s0, s1
-# CHECK-ASM: encoding: [0x1b,0x94,0x04,0x60]
-clzw s0, s1
+# CHECK-ASM-AND-OBJ: cls a1, a2
+# CHECK-ASM: encoding: [0x93,0x15,0x36,0x60]
+cls a1, a2
 # CHECK-ASM-AND-OBJ: sext.b a2, a3
 # CHECK-ASM: encoding: [0x13,0x96,0x46,0x60]
 sext.b a2, a3
 # CHECK-ASM-AND-OBJ: sext.h t0, t1
 # CHECK-ASM: encoding: [0x93,0x12,0x53,0x60]
 sext.h t0, t1
+# CHECK-ASM-AND-OBJ: abs a4, a5
+# CHECK-ASM: encoding: [0x13,0x97,0x77,0x60]
+abs a4, a5
+# CHECK-ASM-AND-OBJ: rev16 s0, s1
+# CHECK-ASM: encoding: [0x13,0xd4,0x04,0x6b]
+rev16 s0, s1
+# CHECK-ASM-AND-OBJ: rev8 s0, s1
+# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x6b]
+rev8 s0, s1
+# CHECK-ASM-AND-OBJ: rev s2, s3
+# CHECK-ASM: encoding: [0x13,0xd9,0xf9,0x6b]
+rev s2, s3
+# CHECK-ASM-AND-OBJ: clzw s0, s1
+# CHECK-ASM: encoding: [0x1b,0x94,0x04,0x60]
+clzw s0, s1
+# CHECK-ASM-AND-OBJ: clsw s2, s3
+# CHECK-ASM: encoding: [0x1b,0x99,0x39,0x60]
+clsw s2, s3
+# CHECK-ASM-AND-OBJ: absw s2, s3
+# CHECK-ASM: encoding: [0x1b,0x99,0x79,0x60]
+absw s2, s3
+# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
+# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
+sh1add a0, a1, a2
+# CHECK-ASM-AND-OBJ: pack s0, s1, s2
+# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
+pack s0, s1, s2
 # CHECK-ASM-AND-OBJ: min t0, t1, t2
 # CHECK-ASM: encoding: [0xb3,0x42,0x73,0x0a]
 min t0, t1, t2
-# CHECK-ASM-AND-OBJ: minu t0, t1, t2
-# CHECK-ASM: encoding: [0xb3,0x52,0x73,0x0a]
-minu t0, t1, t2
+# CHECK-ASM-AND-OBJ: minu ra, sp, gp
+# CHECK-ASM: encoding: [0xb3,0x50,0x31,0x0a]
+minu ra, sp, gp
 # CHECK-ASM-AND-OBJ: max t3, t4, t5
 # CHECK-ASM: encoding: [0x33,0xee,0xee,0x0b]
 max t3, t4, t5
 # CHECK-ASM-AND-OBJ: maxu a4, a5, a6
 # CHECK-ASM: encoding: [0x33,0xf7,0x07,0x0b]
 maxu a4, a5, a6
-# CHECK-ASM-AND-OBJ: pack s0, s1, s2
-# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
-pack s0, s1, s2
-# CHECK-ASM-AND-OBJ: rev8 s0, s1
-# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x6b]
-rev8 s0, s1
+# CHECK-ASM-AND-OBJ: pslli.b a6, a7
+# CHECK-ASM: encoding: [0x1b,0xa8,0x88,0x80]
+pslli.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: pslli.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x20,0x11,0x81]
+pslli.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: pslli.w ra, sp, 2
+# CHECK-ASM: encoding: [0x9b,0x20,0x21,0x82]
+pslli.w ra, sp, 2
+# CHECK-ASM-AND-OBJ: psslai.h t0, t1, 3
+# CHECK-ASM: encoding: [0x9b,0x22,0x33,0xd1]
+psslai.h t0, t1, 3
+# CHECK-ASM-AND-OBJ: psslai.w a4, a5, 4
+# CHECK-ASM: encoding: [0x1b,0xa7,0x47,0xd2]
+psslai.w a4, a5, 4
+# CHECK-ASM-AND-OBJ: pli.h a5, 5
+# CHECK-ASM: encoding: [0x9b,0x27,0x05,0xb0]
+pli.h a5, 5
+# CHECK-ASM-AND-OBJ: pli.w a5, 5
+# CHECK-ASM: encoding: [0x9b,0x27,0x05,0xb2]
+pli.w a5, 5
+# CHECK-ASM-AND-OBJ: pli.b a6, 6
+# CHECK-ASM: encoding: [0x1b,0x28,0x06,0xb4]
+pli.b a6, 6
+# CHECK-ASM-AND-OBJ: psext.h.b t3, a2
+# CHECK-ASM: encoding: [0x1b,0x2e,0x46,0xe0]
+psext.h.b t3, a2
+# CHECK-ASM-AND-OBJ: psext.w.b a2, s0
+# CHECK-ASM: encoding: [0x1b,0x26,0x44,0xe2]
+psext.w.b a2, s0
+# CHECK-ASM-AND-OBJ: psext.w.h t1, t3
+# CHECK-ASM: encoding: [0x1b,0x23,0x5e,0xe2]
+psext.w.h t1, t3
+# CHECK-ASM-AND-OBJ: psabs.h t1, t5
+# CHECK-ASM: encoding: [0x1b,0x23,0x7f,0xe0]
+psabs.h t1, t5
+# CHECK-ASM-AND-OBJ: psabs.b a0, s2
+# CHECK-ASM: encoding: [0x1b,0x25,0x79,0xe4]
+psabs.b a0, s2
+# CHECK-ASM-AND-OBJ: plui.h s2, 4
+# CHECK-ASM: encoding: [0x1b,0x29,0x04,0xf0]
+plui.h s2, 4
+# CHECK-OBJ: plui.h gp, -412
+# CHECK-ASM: plui.h gp, 612
+# CHECK-ASM: encoding: [0x9b,0xa1,0x64,0xf0]
+plui.h gp, 612
+# CHECK-ASM-AND-OBJ: plui.w a2, 1
+# CHECK-ASM: encoding: [0x1b,0x26,0x01,0xf2]
+plui.w a2, 1
+# CHECK-OBJ: plui.w a2, -1
+# CHECK-ASM: plui.w a2, 1023
+# CHECK-ASM: encoding: [0x1b,0xa6,0xff,0xf3]
+plui.w a2, 1023
diff --git a/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s b/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
index e52f8129129d7..2f799977f764f 100644
--- a/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
+++ b/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
@@ -1,7 +1,7 @@
 # RUN: not llvm-mc -triple riscv32 -filetype obj -mattr=+xandesperf < %s -o /dev/null 2>&1 | FileCheck %s
 
-  nds.bbc t0, 7, far_distant # CHECK: :[[@LINE]]:3: error: fixup value out of range
-  nds.bbc t0, 7, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+  nds.bbc t0, 7, far_distant # CHECK: :[[@LINE]]:18: error: fixup value out of range
+  nds.bbc t0, 7, unaligned # CHECK: :[[@LINE]]:18: error: fixup value must be 2-byte aligned
 
   .byte 0
 unaligned:
diff --git a/llvm/test/MC/RISCV/xandesvsintload-valid.s b/llvm/test/MC/RISCV/xandesvsintload-valid.s
new file mode 100644
index 0000000000000..19efba86a6104
--- /dev/null
+++ b/llvm/test/MC/RISCV/xandesvsintload-valid.s
@@ -0,0 +1,39 @@
+# XAndesVSIntLoad - Andes Vector INT4 Load Extension
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+xandesvsintload -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+xandesvsintload < %s \
+# RUN:     | llvm-objdump --mattr=+xandesvsintload -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ %s
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+xandesvsintload -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+xandesvsintload < %s \
+# RUN:     | llvm-objdump --mattr=+xandesvsintload -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ %s
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+# CHECK-OBJ: nds.vln8.v v8, (a0)
+# CHECK-ASM: nds.vln8.v v8, (a0)
+# CHECK-ASM: encoding: [0x5b,0x44,0x25,0x06]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vln8.v v8, (a0)
+
+# CHECK-OBJ: nds.vln8.v v8, (a0), v0.t
+# CHECK-ASM: nds.vln8.v v8, (a0), v0.t
+# CHECK-ASM: encoding: [0x5b,0x44,0x25,0x04]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vln8.v v8, (a0), v0.t
+
+# CHECK-OBJ: nds.vlnu8.v v8, (a0)
+# CHECK-ASM: nds.vlnu8.v v8, (a0)
+# CHECK-ASM: encoding: [0x5b,0x44,0x35,0x06]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vlnu8.v v8, (a0)
+
+# CHECK-OBJ: nds.vlnu8.v v8, (a0), v0.t
+# CHECK-ASM: nds.vlnu8.v v8, (a0), v0.t
+# CHECK-ASM: encoding: [0x5b,0x44,0x35,0x04]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vlnu8.v v8, (a0), v0.t
diff --git a/llvm/test/MC/RISCV/xmips-invalid.s b/llvm/test/MC/RISCV/xmips-invalid.s
index b3834e7b3407f..a17d2e347f8d9 100644
--- a/llvm/test/MC/RISCV/xmips-invalid.s
+++ b/llvm/test/MC/RISCV/xmips-invalid.s
@@ -1,5 +1,14 @@
 # RUN: not llvm-mc -triple=riscv64 < %s 2>&1 | FileCheck %s -check-prefixes=CHECK-FEATURE
-# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmov < %s 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmov,+Xmipscbop < %s 2>&1 | FileCheck %s
+
+mips.pref   8, 512(a0)
+# CHECK: error: immediate offset must be in the range [0, 511]
+
+mips.pref	8
+# CHECK: error: too few operands for instruction
+
+mips.pref	8, 511(a0)
+# CHECK-FEATURE: error: instruction requires the following: 'Xmipscbop' (MIPS hardware prefetch)
 
 mips.ccmov x0, x1, 0x10
 # CHECK: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/xmips-valid.s b/llvm/test/MC/RISCV/xmips-valid.s
index 9f31e4fa2038c..c5755ee81b499 100644
--- a/llvm/test/MC/RISCV/xmips-valid.s
+++ b/llvm/test/MC/RISCV/xmips-valid.s
@@ -1,9 +1,21 @@
-# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmov -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmov,+xmipscbop -M no-aliases -show-encoding \
 # RUN:   | FileCheck -check-prefixes=CHECK-INST,CHECK-ENC %s
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmov < %s \
-# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmov -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmov,+xmipscbop < %s \
+# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmov,+xmipscbop -M no-aliases -d - \
 # RUN:   | FileCheck -check-prefix=CHECK-DIS %s
 
+# CHECK-INST: mips.pref 8, 511(a0)
+# CHECK-ENC:  encoding: [0x0b,0x04,0xf5,0x1f]
+mips.pref	8, 511(a0)
+
+# CHECK-DIS: mips.pref  0x8, 0x1ff(a0)
+
+# CHECK-INST: mips.pref 9, 0(a0)
+# CHECK-ENC:  encoding: [0x8b,0x04,0x05,0x00]
+mips.pref	9, 0(a0)
+
+# CHECK-DIS: mips.pref  0x9, 0x0(a0)
+
 # CHECK-INST: mips.ccmov	s0, s1, s2, s3
 # CHECK-ENC:  encoding: [0x0b,0x34,0x99,0x9e]
 mips.ccmov s0, s1, s2, s3
diff --git a/llvm/test/MC/RISCV/xqcilia-valid.s b/llvm/test/MC/RISCV/xqcilia-valid.s
index 169edc42da697..872ffbe8126aa 100644
--- a/llvm/test/MC/RISCV/xqcilia-valid.s
+++ b/llvm/test/MC/RISCV/xqcilia-valid.s
@@ -10,10 +10,13 @@
 # RUN:     | llvm-objdump --mattr=+experimental-xqcilia --no-print-imm-hex -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 
+.option exact
+
 # CHECK-INST: qc.e.addai      s1, -1
 # CHECK-ENC: encoding: [0x9f,0x24,0xff,0xff,0xff,0xff]
 qc.e.addai x9, 4294967295
 
+
 # CHECK-INST: qc.e.addai      s1, -2147483648
 # CHECK-ENC: encoding: [0x9f,0x24,0x00,0x00,0x00,0x80]
 qc.e.addai x9, -2147483648
@@ -32,6 +35,7 @@ qc.e.addi x10, x9, 33554431
 # CHECK-ENC: encoding: [0x9f,0xa4,0xff,0xff,0xff,0xff]
 qc.e.andai x9, 4294967295
 
+
 # CHECK-INST: qc.e.andai      s1, -2147483648
 # CHECK-ENC: encoding: [0x9f,0xa4,0x00,0x00,0x00,0x80]
 qc.e.andai x9, -2147483648
@@ -83,6 +87,8 @@ qc.e.xori x10, x9, 33554431
 
 # Check that compress patterns work as expected
 
+.option noexact
+
 # CHECK-ALIAS: addi t0, t0, 20
 # CHECK-NOALIAS: c.addi t0, 20
 # CHECK-ENC: encoding: [0xd1,0x02]
@@ -102,3 +108,18 @@ qc.e.addi x5, x6, 0
 # CHECK-NOALIAS: c.addi16sp sp, 48
 # CHECK-ENC: encoding: [0x45,0x61]
 qc.e.addi x2, x2, 48
+
+# CHECK-ALIAS: addi s1, s1, -1
+# CHECK-NOALIAS: c.addi s1, -1
+# CHECK-ENC: encoding: [0xfd,0x14]
+qc.e.addai x9, 4294967295
+
+# CHECK-ALIAS: addi sp, sp, 48
+# CHECK-NOALIAS: c.addi16sp sp, 48
+# CHECK-ENC: encoding: [0x45,0x61]
+qc.e.addai x2, 48
+
+# CHECK-ALIAS: andi s1, s1, -1
+# CHECK-NOALIAS: c.andi s1, -1
+# CHECK-ENC: encoding: [0xfd,0x98]
+qc.e.andai x9, 4294967295
diff --git a/llvm/test/MC/VE/data-reloc-error.s b/llvm/test/MC/VE/data-reloc-error.s
index e312ac4e02d1a..8c03a796e312a 100644
--- a/llvm/test/MC/VE/data-reloc-error.s
+++ b/llvm/test/MC/VE/data-reloc-error.s
@@ -15,12 +15,12 @@ a:
 
 # CHECK:      data-reloc-error.s:10:7: error: 1-byte data relocation is not supported
 # CHECK-NEXT: .byte _GLOBAL_OFFSET_TABLE_
-# CHECK:      data-reloc-error.s:11:7: error: 1-byte pc-relative data relocation is not supported
+# CHECK:      data-reloc-error.s:11:29: error: 1-byte pc-relative data relocation is not supported
 # CHECK-NEXT: .byte _GLOBAL_OFFSET_TABLE_ - .
 # CHECK:      data-reloc-error.s:12:8: error: 2-byte data relocation is not supported
 # CHECK-NEXT: .2byte _GLOBAL_OFFSET_TABLE_
-# CHECK:      data-reloc-error.s:13:8: error: 2-byte pc-relative data relocation is not supported
+# CHECK:      data-reloc-error.s:13:30: error: 2-byte pc-relative data relocation is not supported
 # CHECK-NEXT: .2byte _GLOBAL_OFFSET_TABLE_ - .
-# CHECK:      data-reloc-error.s:14:8: error: 8-byte pc-relative data relocation is not supported
+# CHECK:      data-reloc-error.s:14:30: error: 8-byte pc-relative data relocation is not supported
 # CHECK-NEXT: .8byte _GLOBAL_OFFSET_TABLE_ - .
 
diff --git a/llvm/test/MC/WebAssembly/reference-types.s b/llvm/test/MC/WebAssembly/reference-types.s
index cfadede8295ef..08aafb23969eb 100644
--- a/llvm/test/MC/WebAssembly/reference-types.s
+++ b/llvm/test/MC/WebAssembly/reference-types.s
@@ -27,6 +27,21 @@ ref_null_test:
   drop
   end_function
 
+# CHECK-LABEL: ref_test_test:
+# CHECK: ref.null_func   # encoding: [0xd0,0x70]
+# CHECK: ref.test () -> () # encoding: [0xfb,0x14,0x80'A',0x80'A',0x80'A',0x80'A',A]
+# CHECK: # fixup A - offset: 2, value: .Ltypeindex0@TYPEINDEX, kind: fixup_uleb128_i32
+# CHECK: ref.null_func   # encoding: [0xd0,0x70]
+# CHECK: ref.test () -> (i32) # encoding: [0xfb,0x14,0x80'A',0x80'A',0x80'A',0x80'A',A]
+# CHECK: # fixup A - offset: 2, value: .Ltypeindex1@TYPEINDEX, kind: fixup_uleb128_i32
+ref_test_test:
+  .functype ref_test_test () -> (i32, i32)
+  ref.null_func
+  ref.test () -> ()
+  ref.null_func
+  ref.test () -> (i32)
+  end_function
+
 # CHECK-LABEL: ref_sig_test_funcref:
 # CHECK-NEXT: .functype ref_sig_test_funcref (funcref) -> (funcref)
 ref_sig_test_funcref:
diff --git a/llvm/test/MC/X86/Relocations/x86-16.s b/llvm/test/MC/X86/Relocations/x86-16.s
new file mode 100644
index 0000000000000..f8aaa6a9bf62f
--- /dev/null
+++ b/llvm/test/MC/X86/Relocations/x86-16.s
@@ -0,0 +1,13 @@
+# RUN: llvm-mc -filetype=obj -triple=i386-unknown-code16 %s -o %t
+# RUN: llvm-objdump -dr --triple=i386-unknown-code16 %t | FileCheck %s
+
+# CHECK:       e9 03 00                      jmp     {{.*}} <foo>
+# CHECK-NEXT:  e9 00 00                      jmp     {{.*}} <foo>
+# CHECK-LABEL: <foo>:
+# CHECK-NEXT:  0f 84 fc ff                   je      {{.*}} <foo>
+# CHECK-NEXT:  0f 84 f8 ff                   je      {{.*}} <foo>
+{disp32} jmp foo
+jmp.d32 foo
+foo:
+{disp32} je foo
+je.d32 foo
diff --git a/llvm/test/MC/X86/Relocations/x86-32.s b/llvm/test/MC/X86/Relocations/x86-32.s
new file mode 100644
index 0000000000000..f2b256517cca1
--- /dev/null
+++ b/llvm/test/MC/X86/Relocations/x86-32.s
@@ -0,0 +1,17 @@
+# RUN: llvm-mc -filetype=obj -triple=i686 %s -o %t
+# RUN: llvm-objdump -dr --no-addresses %t | sed 's/#.*//;/^ *$/d' | FileCheck %s
+
+# CHECK:       e9 05 00 00 00                jmp     {{.*}} <foo>
+# CHECK-NEXT:  e9 00 00 00 00                jmp     {{.*}} <foo>
+# CHECK-LABEL: <foo>:
+# CHECK-NEXT:  0f 84 fa ff ff ff             je      {{.*}} <foo>
+# CHECK-NEXT:  0f 84 f4 ff ff ff             je      {{.*}} <foo>
+{disp32} jmp foo
+jmp.d32 foo
+foo:
+{disp32} je foo
+je.d32 foo
+
+# CHECK-NEXT: c1 0b 0a                               rorl    $0xa, (%ebx)
+# CHECK-NEXT:                R_386_8 .text
+                rorl    $foo, (%ebx)
diff --git a/llvm/test/MC/X86/Relocations/x86-64.s b/llvm/test/MC/X86/Relocations/x86-64.s
new file mode 100644
index 0000000000000..402e6ef9356a9
--- /dev/null
+++ b/llvm/test/MC/X86/Relocations/x86-64.s
@@ -0,0 +1,102 @@
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -dr --no-addresses %t | sed 's/#.*//;/^ *$/d' | FileCheck %s
+
+# CHECK:      8b 05 00 00 00 00                      movl    (%rip), %eax
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x4
+# CHECK-NEXT: c6 05 00 00 00 00 0c                   movb    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: 66 c7 05 00 00 00 00 0c 00             movw    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x6
+# CHECK-NEXT: c7 05 00 00 00 00 0c 00 00 00          movl    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+# CHECK-NEXT: 48 c7 05 00 00 00 00 0c 00 00 00       movq    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+# CHECK-NEXT: 67 8b 05 00 00 00 00                   movl    (%eip), %eax
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x4
+# CHECK-NEXT: 67 c6 05 00 00 00 00 0c                movb    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: 67 66 c7 05 00 00 00 00 0c 00          movw    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x6
+# CHECK-NEXT: 67 c7 05 00 00 00 00 0c 00 00 00       movl    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+# CHECK-NEXT: 67 48 c7 05 00 00 00 00 0c 00 00 00    movq    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+movl	foo(%rip), %eax
+movb	$12, foo(%rip)
+movw	$12, foo(%rip)
+movl	$12, foo(%rip)
+movq	$12, foo(%rip)
+
+movl	foo(%eip), %eax
+movb	$12, foo(%eip)
+movw	$12, foo(%eip)
+movl	$12, foo(%eip)
+movq	$12, foo(%eip)
+
+# CHECK-NEXT: 48 8b 05 00 00 00 00                   movq    (%rip), %rax
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+# CHECK-NEXT: 4c 8b 35 00 00 00 00                   movq    (%rip), %r14
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+# CHECK-NEXT: 67 48 8b 05 00 00 00 00                movq    (%eip), %rax
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+# CHECK-NEXT: 67 4c 8b 35 00 00 00 00                movq    (%eip), %r14
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+movq foo@GOTPCREL(%rip), %rax
+movq foo@GOTPCREL(%rip), %r14
+movq foo@GOTPCREL(%eip), %rax
+movq foo@GOTPCREL(%eip), %r14
+
+# CHECK-NEXT: 66 0f 38 00 0d 00 00 00 00             pshufb  (%rip), %xmm1
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x4
+pshufb foo(%rip), %xmm1
+
+## PR15040
+# CHECK-NEXT: c4 e3 f9 6a 05 00 00 00 00 10          vfmaddss        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 6a 05 00 00 00 00 10          vfmaddss        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 f9 6b 05 00 00 00 00 10          vfmaddsd        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 6b 05 00 00 00 00 10          vfmaddsd        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 f9 68 05 00 00 00 00 10          vfmaddps        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 68 05 00 00 00 00 10          vfmaddps        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 f9 69 05 00 00 00 00 10          vfmaddpd        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 69 05 00 00 00 00 10          vfmaddpd        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 fd 68 05 00 00 00 00 10          vfmaddps        (%rip), %ymm1, %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 7d 68 05 00 00 00 00 10          vfmaddps        %ymm1, (%rip), %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 fd 69 05 00 00 00 00 10          vfmaddpd        (%rip), %ymm1, %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 7d 69 05 00 00 00 00 10          vfmaddpd        %ymm1, (%rip), %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+vfmaddss  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddss   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddsd  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddsd   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddps  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddps   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddpd  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddpd   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddps  foo(%rip), %ymm1, %ymm0, %ymm0
+vfmaddps   %ymm1, foo(%rip),%ymm0, %ymm0
+vfmaddpd  foo(%rip), %ymm1, %ymm0, %ymm0
+vfmaddpd   %ymm1, foo(%rip),%ymm0, %ymm0
+
+# CHECK-NEXT:<l1>:
+# CHECK-NEXT: 90                                     nop
+# CHECK-NEXT: c4 e2 79 00 05 f6 ff ff ff             vpshufb -0xa(%rip), %xmm0, %xmm0
+# CHECK-NEXT:<l2>:
+# CHECK-NEXT: 90                                     nop
+# CHECK-NEXT: c4 e3 7d 4a 05 f5 ff ff ff 10          vblendvps       %ymm1, -0xb(%rip), %ymm0, %ymm0
+l1:
+  nop
+  vpshufb l1(%rip), %xmm0, %xmm0
+l2:
+  nop
+  vblendvps %ymm1, l2(%rip), %ymm0, %ymm0
diff --git a/llvm/test/MC/X86/avx-64-att.s b/llvm/test/MC/X86/avx-64-att.s
index 39ee048c3736d..f29142549bf52 100644
--- a/llvm/test/MC/X86/avx-64-att.s
+++ b/llvm/test/MC/X86/avx-64-att.s
@@ -4116,19 +4116,6 @@ vdivpd  -4(%rcx,%rbx,8), %xmm10, %xmm11
 // CHECK: encoding: [0xc4,0xa1,0x78,0x29,0x1c,0x18]
           vmovaps %xmm3, (%rax,%r11)
 
-// CHECK: vpshufb _foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe2,0x79,0x00,0x05,A,A,A,A]
-// CHECK: kind: reloc_riprel_4byte
-_foo:
-  nop
-  vpshufb _foo(%rip), %xmm0, %xmm0
-
-// CHECK: vblendvps %ymm1, _foo2(%rip), %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0x7d,0x4a,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: _foo2-5
-_foo2:
-  nop
-  vblendvps %ymm1, _foo2(%rip), %ymm0, %ymm0
 
 // CHECK: vgatherdpd %xmm0, (%rdi,%xmm1,2), %xmm2
 // CHECK: encoding: [0xc4,0xe2,0xf9,0x92,0x14,0x4f]
diff --git a/llvm/test/MC/X86/fma4-att.s b/llvm/test/MC/X86/fma4-att.s
index c9bd954e90496..bdd45ad1717da 100644
--- a/llvm/test/MC/X86/fma4-att.s
+++ b/llvm/test/MC/X86/fma4-att.s
@@ -77,66 +77,6 @@
 // CHECK: encoding: [0xc4,0xe3,0xfd,0x69,0xc2,0x10]
           vfmaddpd   %ymm2, %ymm1, %ymm0, %ymm0
 
-// PR15040
-// CHECK: vfmaddss  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x6a,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddss  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddss   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x6a,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddss   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddsd  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x6b,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddsd  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddsd   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x6b,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddsd   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddps  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddps   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddpd  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddpd   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddps  foo(%rip), %ymm1, %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0xfd,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps  foo(%rip), %ymm1, %ymm0, %ymm0
-
-// CHECK: vfmaddps   %ymm1, foo(%rip), %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0x7d,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps   %ymm1, foo(%rip),%ymm0, %ymm0
-
-// CHECK: vfmaddpd  foo(%rip), %ymm1, %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0xfd,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd  foo(%rip), %ymm1, %ymm0, %ymm0
-
-// CHECK: vfmaddpd   %ymm1, foo(%rip), %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0x7d,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd   %ymm1, foo(%rip),%ymm0, %ymm0
 
 // vfmsub
 // CHECK: vfmsubss  (%rcx), %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/MC/X86/macho-reloc-errors-x86.s b/llvm/test/MC/X86/macho-reloc-errors-x86.s
index 4af202220073b..16c656dc472da 100644
--- a/llvm/test/MC/X86/macho-reloc-errors-x86.s
+++ b/llvm/test/MC/X86/macho-reloc-errors-x86.s
@@ -10,6 +10,6 @@ defined:
         .section __DATA,__tim2
 later:
 
-// CHECK-ERROR: 3:9: error: symbol 'thing' can not be undefined in a subtraction expression
-// CHECK-ERROR: 4:9: error: symbol 'thing2' can not be undefined in a subtraction expression
-// CHECK-ERROR: 5:9: error: Section too large, can't encode r_address (0x100000b) into 24 bits of scattered relocation entry.
+// CHECK-ERROR: 3:24: error: symbol 'thing' can not be undefined in a subtraction expression
+// CHECK-ERROR: 4:26: error: symbol 'thing2' can not be undefined in a subtraction expression
+// CHECK-ERROR: 5:24: error: Section too large, can't encode r_address (0x100000b) into 24 bits of scattered relocation entry.
diff --git a/llvm/test/MC/X86/macho-reloc-errors-x86_64.s b/llvm/test/MC/X86/macho-reloc-errors-x86_64.s
index daf84a7dbdc4f..040b66656dd25 100644
--- a/llvm/test/MC/X86/macho-reloc-errors-x86_64.s
+++ b/llvm/test/MC/X86/macho-reloc-errors-x86_64.s
@@ -9,11 +9,11 @@
         jmp thing@PLT
         mov %rax, thing@TLVP
 
-// CHECK-ERROR: 3:9: error: 32-bit absolute addressing is not supported in 64-bit mode
-// CHECK-ERROR: 4:9: error: expected relocatable expression
-// CHECK-ERROR: 5:9: error: unsupported pc-relative relocation of difference
-// CHECK-ERROR: 6:9: error: unsupported relocation with identical base
-// CHECK-ERROR: 7:9: error: unsupported relocation with subtraction expression, symbol 'thing' can not be undefined in a subtraction expression
-// CHECK-ERROR: 8:9: error: unsupported symbol modifier in relocation
-// CHECK-ERROR: 9:9: error: unsupported symbol modifier in branch relocation
-// CHECK-ERROR: 10:9: error: TLVP symbol modifier should have been rip-rel
+// CHECK-ERROR: 3:19: error: 32-bit absolute addressing is not supported in 64-bit mode
+// CHECK-ERROR: 4:28: error: expected relocatable expression
+// CHECK-ERROR: 5:25: error: unsupported pc-relative relocation of difference
+// CHECK-ERROR: 6:24: error: unsupported relocation with identical base
+// CHECK-ERROR: 7:24: error: unsupported relocation with subtraction expression, symbol 'thing' can not be undefined in a subtraction expression
+// CHECK-ERROR: 8:19: error: unsupported symbol modifier in relocation
+// CHECK-ERROR: 9:13: error: unsupported symbol modifier in branch relocation
+// CHECK-ERROR: 10:19: error: TLVP symbol modifier should have been rip-rel
diff --git a/llvm/test/MC/X86/pltoff.s b/llvm/test/MC/X86/pltoff.s
index a30ee6ccb0983..2f3551b6a8379 100644
--- a/llvm/test/MC/X86/pltoff.s
+++ b/llvm/test/MC/X86/pltoff.s
@@ -14,6 +14,6 @@ movabsq $puts@PLTOFF, %rax
 movabsq $.text@PLTOFF, %rax
 
 .ifdef ERR
-# ERR: {{.*}}.s:[[#@LINE+1]]:1: error: 64 bit reloc applied to a field with a different size
+# ERR: {{.*}}.s:[[#@LINE+1]]:7: error: 64 bit reloc applied to a field with a different size
 movl $puts@PLTOFF, %eax
 .endif
diff --git a/llvm/test/MC/X86/register-assignment.s b/llvm/test/MC/X86/register-assignment.s
index 4c7b07df050b8..043f010efda2e 100644
--- a/llvm/test/MC/X86/register-assignment.s
+++ b/llvm/test/MC/X86/register-assignment.s
@@ -1,12 +1,14 @@
 // RUN: llvm-mc -triple x86_64-unknown-unknown %s -o -      | FileCheck %s
 // RUN: not llvm-mc -triple x86_64 --defsym ERR=1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
 	
-// CHECK-NOT: .set var_xdata
+// CHECK: var_xdata = %rcx
 var_xdata = %rcx
 
 // CHECK: xorq %rcx, %rcx
 xorq var_xdata, var_xdata
 
+unused_reg = %xmm0
+
 // CHECK: .data
 // CHECK-NEXT: .byte 1	
 .data 
diff --git a/llvm/test/MC/X86/unused_reg_var_assign.s b/llvm/test/MC/X86/unused_reg_var_assign.s
deleted file mode 100644
index 53e5cd6a2747e..0000000000000
--- a/llvm/test/MC/X86/unused_reg_var_assign.s
+++ /dev/null
@@ -1,3 +0,0 @@
-# RUN: llvm-mc -triple x86_64-unknown-linux-gnu -filetype obj %s -o %t
-	
-reg_i = %xmm0
diff --git a/llvm/test/MC/X86/x86-16.s b/llvm/test/MC/X86/x86-16.s
index be0563e5b1d63..b0a4bda56fcbf 100644
--- a/llvm/test/MC/X86/x86-16.s
+++ b/llvm/test/MC/X86/x86-16.s
@@ -1056,17 +1056,6 @@ xsusldtrk
 // CHECK: encoding: [0xf2,0x0f,0x01,0xe9]
 xresldtrk
 
-// CHECK: jmp foo
-// CHECK:  encoding: [0xe9,A,A]
-// CHECK:  fixup A - offset: 1, value: foo-2, kind: FK_PCRel_2
-{disp32} jmp foo
-foo:
-
-// CHECK: je foo
-// CHECK:  encoding: [0x0f,0x84,A,A]
-// CHECK:  fixup A - offset: 2, value: foo-2, kind: FK_PCRel_2
-{disp32} je foo
-
 // CHECK: movl nearer, %ebx
 // CHECK:  encoding: [0x66,0x8b,0x1e,A,A]
 // CHECK:  fixup A - offset: 3, value: nearer, kind: FK_Data_2
diff --git a/llvm/test/MC/X86/x86-32-coverage.s b/llvm/test/MC/X86/x86-32-coverage.s
index 5475946a9d216..d585b4bc0e953 100644
--- a/llvm/test/MC/X86/x86-32-coverage.s
+++ b/llvm/test/MC/X86/x86-32-coverage.s
@@ -1421,12 +1421,6 @@
 // CHECK:  encoding: [0xd1,0x0d,0x78,0x56,0x34,0x12]
         	rorl	0x12345678
 
-// CHECK: rorl  $foo, (%ebx)
-// INTEL: ror dword ptr [ebx], foo
-// CHECK:  encoding: [0xc1,0x0b,A]
-// CHECK:    fixup A - offset: 2, value: foo, kind: FK_Data_1
-                rorl    $foo, (%ebx)
-
 // CHECK: shll	$0, 3735928559(%ebx,%ecx,8)
 // CHECK:  encoding: [0xc1,0xa4,0xcb,0xef,0xbe,0xad,0xde,0x00]
         	sall	$0,0xdeadbeef(%ebx,%ecx,8)
diff --git a/llvm/test/MC/X86/x86-32.s b/llvm/test/MC/X86/x86-32.s
index e3d473a05e59d..8bfcf82c0cf75 100644
--- a/llvm/test/MC/X86/x86-32.s
+++ b/llvm/test/MC/X86/x86-32.s
@@ -1,3 +1,4 @@
+/// See Relocations/x86-32.s for fixup/relocation tests.
 // RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
 
 	pause
@@ -1118,25 +1119,6 @@ ptwritel 0xdeadbeef(%ebx,%ecx,8)
 // CHECK:  encoding: [0xf3,0x0f,0xae,0xe0]
 ptwritel %eax
 
-// CHECK: jmp foo
-// CHECK:  encoding: [0xe9,A,A,A,A]
-// CHECK:  fixup A - offset: 1, value: foo-4, kind: FK_PCRel_4
-// CHECK: jmp foo
-// CHECK:  encoding: [0xe9,A,A,A,A]
-// CHECK:  fixup A - offset: 1, value: foo-4, kind: FK_PCRel_4
-{disp32} jmp foo
-jmp.d32 foo
-foo:
-
-// CHECK: je foo
-// CHECK:  encoding: [0x0f,0x84,A,A,A,A]
-// CHECK:  fixup A - offset: 2, value: foo-4, kind: FK_PCRel_4
-// CHECK: je foo
-// CHECK:  encoding: [0x0f,0x84,A,A,A,A]
-// CHECK:  fixup A - offset: 2, value: foo-4, kind: FK_PCRel_4
-{disp32} je foo
-je.d32 foo
-
 // CHECK: ljmpl *%cs:305419896
 // CHECK:  encoding: [0x2e,0xff,0x2d,0x78,0x56,0x34,0x12]
 ljmp %cs:*0x12345678
diff --git a/llvm/test/MC/X86/x86-64.s b/llvm/test/MC/X86/x86-64.s
index 108d1220107e3..911f4674bd2cc 100644
--- a/llvm/test/MC/X86/x86-64.s
+++ b/llvm/test/MC/X86/x86-64.s
@@ -1,3 +1,4 @@
+/// See Relocations/x86-64.s for fixup/relocation tests.
 // RUN: llvm-mc -triple x86_64-unknown-unknown -show-encoding %s > %t 2> %t.err
 // RUN: FileCheck < %t %s
 // RUN: FileCheck --check-prefix=CHECK-STDERR < %t.err %s
@@ -597,86 +598,15 @@ fdivp %st, %st(1) // CHECK: encoding: [0xde,0xf1]
 fdivp %st(1), %st // CHECK: encoding: [0xde,0xf1]
 
 
-movl	foo(%rip), %eax
-// CHECK: movl	foo(%rip), %eax
-// CHECK: encoding: [0x8b,0x05,A,A,A,A]
-// CHECK: fixup A - offset: 2, value: foo-4, kind: reloc_riprel_4byte
-
-movb	$12, foo(%rip)
-// CHECK: movb	$12, foo(%rip)
-// CHECK: encoding: [0xc6,0x05,A,A,A,A,0x0c]
-// CHECK:    fixup A - offset: 2, value: foo-5, kind: reloc_riprel_4byte
-
-movw	$12, foo(%rip)
-// CHECK: movw	$12, foo(%rip)
-// CHECK: encoding: [0x66,0xc7,0x05,A,A,A,A,0x0c,0x00]
-// CHECK:    fixup A - offset: 3, value: foo-6, kind: reloc_riprel_4byte
-
-movl	$12, foo(%rip)
-// CHECK: movl	$12, foo(%rip)
-// CHECK: encoding: [0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 2, value: foo-8, kind: reloc_riprel_4byte
-
 // rdar://37247000
 movl	$12, 1024(%rip)
 // CHECK: movl	$12, 1024(%rip)
 // CHECK: encoding: [0xc7,0x05,0x00,0x04,0x00,0x00,0x0c,0x00,0x00,0x00]
 
-movq	$12, foo(%rip)
-// CHECK:  movq	$12, foo(%rip)
-// CHECK: encoding: [0x48,0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 3, value: foo-8, kind: reloc_riprel_4byte
-
-movl	foo(%eip), %eax
-// CHECK: movl	foo(%eip), %eax
-// CHECK: encoding: [0x67,0x8b,0x05,A,A,A,A]
-// CHECK: fixup A - offset: 3, value: foo-4, kind: reloc_riprel_4byte
-
-movb	$12, foo(%eip)
-// CHECK: movb	$12, foo(%eip)
-// CHECK: encoding: [0x67,0xc6,0x05,A,A,A,A,0x0c]
-// CHECK:    fixup A - offset: 3, value: foo-5, kind: reloc_riprel_4byte
-
-movw	$12, foo(%eip)
-// CHECK: movw	$12, foo(%eip)
-// CHECK: encoding: [0x67,0x66,0xc7,0x05,A,A,A,A,0x0c,0x00]
-// CHECK:    fixup A - offset: 4, value: foo-6, kind: reloc_riprel_4byte
-
-movl	$12, foo(%eip)
-// CHECK: movl	$12, foo(%eip)
-// CHECK: encoding: [0x67,0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 3, value: foo-8, kind: reloc_riprel_4byte
-
-movq	$12, foo(%eip)
-// CHECK:  movq	$12, foo(%eip)
-// CHECK: encoding: [0x67,0x48,0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 4, value: foo-8, kind: reloc_riprel_4byte
-
 // CHECK: addq	$-424, %rax
 // CHECK: encoding: [0x48,0x05,0x58,0xfe,0xff,0xff]
 addq $-424, %rax
 
-
-// CHECK: movq	_foo@GOTPCREL(%rip), %rax
-// CHECK:  encoding: [0x48,0x8b,0x05,A,A,A,A]
-// CHECK:  fixup A - offset: 3, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%rip), %rax
-
-// CHECK: movq	_foo@GOTPCREL(%rip), %r14
-// CHECK:  encoding: [0x4c,0x8b,0x35,A,A,A,A]
-// CHECK:  fixup A - offset: 3, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%rip), %r14
-
-// CHECK: movq	_foo@GOTPCREL(%eip), %rax
-// CHECK:  encoding: [0x67,0x48,0x8b,0x05,A,A,A,A]
-// CHECK:  fixup A - offset: 4, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%eip), %rax
-
-// CHECK: movq	_foo@GOTPCREL(%eip), %r14
-// CHECK:  encoding: [0x67,0x4c,0x8b,0x35,A,A,A,A]
-// CHECK:  fixup A - offset: 4, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%eip), %r14
-
 // CHECK: movq	(%r13,%rax,8), %r13
 // CHECK:  encoding: [0x4d,0x8b,0x6c,0xc5,0x00]
 movq 0x00(%r13,%rax,8),%r13
diff --git a/llvm/test/MC/X86/x86_64-encoding.s b/llvm/test/MC/X86/x86_64-encoding.s
index 247b1dc656bc1..557687e9f6f8f 100644
--- a/llvm/test/MC/X86/x86_64-encoding.s
+++ b/llvm/test/MC/X86/x86_64-encoding.s
@@ -133,12 +133,6 @@ movq %mm1, %r8
 // CHECK:  encoding: [0x48,0x0f,0x7e,0xca]
 movq %mm1, %rdx
 
-// rdar://7840289
-// CHECK: pshufb	CPI1_0(%rip), %xmm1
-// CHECK:  encoding: [0x66,0x0f,0x38,0x00,0x0d,A,A,A,A]
-// CHECK:  fixup A - offset: 5, value: CPI1_0-4
-pshufb	CPI1_0(%rip), %xmm1
-
 // CHECK: sha1rnds4 $1, %xmm1, %xmm2
 // CHECK:   encoding: [0x0f,0x3a,0xcc,0xd1,0x01]
 sha1rnds4 $1, %xmm1, %xmm2
diff --git a/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s b/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s
index 9a81ade9fe284..0904e992a8a32 100644
--- a/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s
+++ b/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s
@@ -2,13 +2,13 @@
 
   .align 4
 
-  beq a0, a1, LBL1 # CHECK: :[[@LINE]]:3: error: fixup value out of range
+  beq a0, a1, LBL1 # CHECK: :[[@LINE]]:15: error: fixup value out of range
 LBL0:
-  beqz a0, LBL2 # CHECK: :[[@LINE]]:3: error: fixup value out of range
+  beqz a0, LBL2 # CHECK: :[[@LINE]]:12: error: fixup value out of range
 
-  call0 LBL0 # CHECK: :[[@LINE]]:3: error: fixup value must be 4-byte aligned
+  call0 LBL0 # CHECK: :[[@LINE]]:9: error: fixup value must be 4-byte aligned
 
-  loop a3, LBL0 # CHECK: :[[@LINE]]:3: error: loop fixup value out of range
+  loop a3, LBL0 # CHECK: :[[@LINE]]:12: error: loop fixup value out of range
 
   .space 1<<8
 LBL1:
diff --git a/llvm/test/MC/Xtensa/dfpaccel.s b/llvm/test/MC/Xtensa/dfpaccel.s
new file mode 100644
index 0000000000000..13e8c2865f403
--- /dev/null
+++ b/llvm/test/MC/Xtensa/dfpaccel.s
@@ -0,0 +1,53 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+dfpaccel \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+LBL0:
+
+# CHECK-INST: rur a3, f64r_lo
+# CHECK: encoding: [0xa0,0x3e,0xe3]
+rur a3, f64r_lo
+
+# CHECK-INST: rur a3, f64r_lo
+# CHECK: encoding: [0xa0,0x3e,0xe3]
+rur a3, 234
+
+# CHECK-INST: rur a3, f64r_lo
+# CHECK: encoding: [0xa0,0x3e,0xe3]
+rur.f64r_lo a3
+
+# CHECK-INST: wur a3, f64r_lo
+# CHECK: encoding: [0x30,0xea,0xf3]
+wur a3, f64r_lo
+
+# CHECK-INST: rur a3, f64r_hi
+# CHECK: encoding: [0xb0,0x3e,0xe3]
+rur a3, f64r_hi
+
+# CHECK-INST: rur a3, f64r_hi
+# CHECK: encoding: [0xb0,0x3e,0xe3]
+rur a3, 235
+
+# CHECK-INST: rur a3, f64r_hi
+# CHECK: encoding: [0xb0,0x3e,0xe3]
+rur.f64r_hi a3
+
+# CHECK-INST: wur a3, f64r_hi
+# CHECK: encoding: [0x30,0xeb,0xf3]
+wur a3, f64r_hi
+
+# CHECK-INST: rur a3, f64s
+# CHECK: encoding: [0xc0,0x3e,0xe3]
+rur a3, f64s
+
+# CHECK-INST: rur a3, f64s
+# CHECK: encoding: [0xc0,0x3e,0xe3]
+rur a3, 236
+
+# CHECK-INST: rur a3, f64s
+# CHECK: encoding: [0xc0,0x3e,0xe3]
+rur.f64s a3
+
+# CHECK-INST: wur a3, f64s
+# CHECK: encoding: [0x30,0xec,0xf3]
+wur a3, f64s
diff --git a/llvm/test/MC/Xtensa/threadptr.s b/llvm/test/MC/Xtensa/threadptr.s
new file mode 100644
index 0000000000000..c29d1ea91ed80
--- /dev/null
+++ b/llvm/test/MC/Xtensa/threadptr.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+threadptr \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+LBL0:
+
+# CHECK-INST: rur a3, threadptr
+# CHECK: encoding: [0x70,0x3e,0xe3]
+rur a3, threadptr
+
+# CHECK-INST: rur a3, threadptr
+# CHECK: encoding: [0x70,0x3e,0xe3]
+rur a3, 231
+
+# CHECK-INST: rur a3, threadptr
+# CHECK: encoding: [0x70,0x3e,0xe3]
+rur.threadptr a3
+
+# CHECK-INST: wur a3, threadptr
+# CHECK: encoding: [0x30,0xe7,0xf3]
+wur a3, threadptr
diff --git a/llvm/test/Other/fatlto.ll b/llvm/test/Other/fatlto.ll
new file mode 100644
index 0000000000000..2f7ce4bf817a8
--- /dev/null
+++ b/llvm/test/Other/fatlto.ll
@@ -0,0 +1,7 @@
+; RUN: opt -debug-pass-manager -passes='fatlto-pre-link<O2>' -verify-analysis-invalidation=0 -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -debug-pass-manager -passes='fatlto-pre-link<O2;thinlto>' -verify-analysis-invalidation=0 -disable-output %s 2>&1 | FileCheck %s --check-prefixes=CHECK,THINLTO
+
+; CHECK: Running pass: EmbedBitcodePass on [module]
+; THINLTO: Running analysis: ModuleSummaryIndexAnalysis on [module]
+; CHECK: Running pass: FatLtoCleanup on [module]
+; CHECK-NEXT: Running pass: LowerTypeTestsPass on [module]
diff --git a/llvm/test/Other/pipeline-alias-errors.ll b/llvm/test/Other/pipeline-alias-errors.ll
index 4d744144ed2ec..f27dd76568a56 100644
--- a/llvm/test/Other/pipeline-alias-errors.ll
+++ b/llvm/test/Other/pipeline-alias-errors.ll
@@ -8,6 +8,11 @@
 ; RUN: not opt -passes="lto-pre-link<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
 ; RUN: not opt -passes="lto" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
 ; RUN: not opt -passes="lto<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="fatlto-pre-link" < %s 2>&1 | FileCheck %s --check-prefix=FATLTO-MISSING-OPT-LEVEL
+; RUN: not opt -passes="fatlto-pre-link<foo>" < %s 2>&1 | FileCheck %s --check-prefix=FATLTO-INVALID-PARAMS
 
 ; MISSING-OPT-LEVEL: invalid optimization level ''
 ; INVALID-OPT-LEVEL: invalid optimization level 'foo'
+
+; FATLTO-MISSING-OPT-LEVEL: missing optimization level for fatlto-pre-link pipeline
+; FATLTO-INVALID-PARAMS: invalid fatlto-pre-link pass parameter 'foo'
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td b/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
index a8290a57e3210..7399ef726d0e2 100644
--- a/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../../include %s -o - 2>%t
+// RUN: not llvm-tblgen -gen-disassembler -I %p/../../../include %s -o - 2>%t
 // RUN: FileCheck %s < %t
 
 include "llvm/Target/Target.td"
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-bad-system-library-entry-error.td b/llvm/test/TableGen/RuntimeLibcallEmitter-bad-system-library-entry-error.td
new file mode 100644
index 0000000000000..965b3acd50cdc
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-bad-system-library-entry-error.td
@@ -0,0 +1,17 @@
+// RUN: not llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s 2>&1 | FileCheck %s
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+def SOME_FUNC : RuntimeLibcall;
+def func_a : RuntimeLibcallImpl<SOME_FUNC>;
+
+def isTargetArchA : RuntimeLibcallPredicate<[{isTargetArchA()}]>;
+
+// CHECK: [[@LINE+4]]:5: error: entry for SystemLibrary is not a RuntimeLibcallImpl
+// CHECK-NEXT: def TheSystemLibraryA : SystemRuntimeLibrary<isTargetArchA,
+// CHECK: note: invalid entry `SOME_FUNC`
+// CHECK-NEXT: def SOME_FUNC : RuntimeLibcall;
+def TheSystemLibraryA : SystemRuntimeLibrary<isTargetArchA,
+  (add SOME_FUNC)
+>;
+
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td b/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td
new file mode 100644
index 0000000000000..f825ece6cf4bb
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td
@@ -0,0 +1,128 @@
+// RUN: llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s | FileCheck %s
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+def SDIVREM_I8 : RuntimeLibcall;
+def UDIVREM_I16 : RuntimeLibcall;
+def MALLOC : RuntimeLibcall;
+def TARGET_OVERRIDE_CC : RuntimeLibcall;
+
+def __divmodqi4 : RuntimeLibcallImpl<SDIVREM_I8>;
+def __udivmodhi4 : RuntimeLibcallImpl<UDIVREM_I16>;
+
+// Test a case where a target wants to set a different calling
+// convention on a generic builtin
+def __target_override_cc : RuntimeLibcallImpl<TARGET_OVERRIDE_CC>;
+
+def malloc : RuntimeLibcallImpl<MALLOC>;
+
+def isAVR : RuntimeLibcallPredicate<[{TT.getArch() == Triple::avr}]>;
+
+def isAVRHurd : RuntimeLibcallPredicate<
+  [{TT.getArch() == Triple::avr && TT.isOSHurd()}]>;
+
+def AVRLibrary : SystemRuntimeLibrary<isAVR,
+  (add malloc, LibcallsWithCC<(add __divmodqi4, __udivmodhi4), AVR_BUILTIN>)
+>;
+
+// Test with default calling convention
+def AVRHurdLibrary : SystemRuntimeLibrary<isAVRHurd,
+  (add malloc, LibcallsWithCC<(add __divmodqi4, __udivmodhi4), AVR_BUILTIN>)> {
+  let DefaultLibcallCallingConv
+    = LibcallCallingConv<[{isFoo() ? CallingConv::Fast : CallingConv::GHC}]>;
+}
+
+def isMSP430 : RuntimeLibcallPredicate<[{TT.getArch() == Triple::msp430}]>;
+
+def MSP430LibraryWithCondCC : SystemRuntimeLibrary<isMSP430,
+  (add malloc,
+       LibcallsWithCC<(add __divmodqi4), AVR_BUILTIN, RuntimeLibcallPredicate<[{ isFoo() }]>>,
+       LibcallsWithCC<(add __udivmodhi4), MSP430_BUILTIN, RuntimeLibcallPredicate<[{ isBar() }]>>)
+>;
+
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(const llvm::Triple &TT) {
+// CHECK: if (TT.getArch() == Triple::avr && TT.isOSHurd()) {
+// CHECK-NEXT:   const CallingConv::ID DefaultCC = isFoo() ? CallingConv::Fast : CallingConv::GHC;
+// CHECK-NEXT:   for (CallingConv::ID &Entry : LibcallImplCallingConvs) {
+// CHECK-NEXT:     Entry = DefaultCC;
+// CHECK-NEXT:   }
+// CHECK-EMPTY:
+// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:        {RTLIB::MALLOC, RTLIB::malloc}, // malloc
+// CHECK-NEXT:    };
+// CHECK-EMPTY:
+// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    static const LibcallImplPair LibraryCalls_AlwaysAvailable[] = {
+// CHECK-NEXT:        {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
+// CHECK-NEXT:        {RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4}, // __udivmodhi4
+// CHECK-NEXT:    };
+// CHECK-EMPTY:
+// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls_AlwaysAvailable) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:      setLibcallImplCallingConv(Impl, CallingConv::AVR_BUILTIN);
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    return;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT: if (TT.getArch() == Triple::avr) {
+// CHECK-NEXT:   static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:       {RTLIB::MALLOC, RTLIB::malloc}, // malloc
+// CHECK-NEXT:   };
+// CHECK-EMPTY:
+// CHECK-NEXT:   for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:     setLibcallImpl(Func, Impl);
+// CHECK-NEXT:   }
+// CHECK-EMPTY:
+// CHECK-NEXT:   static const LibcallImplPair LibraryCalls_AlwaysAvailable[] = {
+// CHECK-NEXT:       {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
+// CHECK-NEXT:       {RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4}, // __udivmodhi4
+// CHECK-NEXT:   };
+// CHECK-EMPTY:
+// CHECK-NEXT:   for (const auto [Func, Impl] : LibraryCalls_AlwaysAvailable) {
+// CHECK-NEXT:     setLibcallImpl(Func, Impl);
+// CHECK-NEXT:     setLibcallImplCallingConv(Impl, CallingConv::AVR_BUILTIN);
+// CHECK-NEXT:   }
+// CHECK-EMPTY:
+// CHECK-NEXT:   return;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT:  if (TT.getArch() == Triple::msp430) {
+// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:        {RTLIB::MALLOC, RTLIB::malloc}, // malloc
+// CHECK-NEXT:    };
+// CHECK-EMPTY:
+// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    if ( isFoo() ) {
+// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_anonymous_3[] = {
+// CHECK-NEXT:          {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
+// CHECK-NEXT:      };
+// CHECK-EMPTY:
+// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_anonymous_3) {
+// CHECK-NEXT:        setLibcallImpl(Func, Impl);
+// CHECK-NEXT:        setLibcallImplCallingConv(Impl, CallingConv::AVR_BUILTIN);
+// CHECK-NEXT:      }
+// CHECK-EMPTY:
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    if ( isBar() ) {
+// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_anonymous_5[] = {
+// CHECK-NEXT:          {RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4}, // __udivmodhi4
+// CHECK-NEXT:      };
+// CHECK-EMPTY:
+// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_anonymous_5) {
+// CHECK-NEXT:        setLibcallImpl(Func, Impl);
+// CHECK-NEXT:        setLibcallImplCallingConv(Impl, CallingConv::MSP430_BUILTIN);
+// CHECK-NEXT:      }
+// CHECK-EMPTY:
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    return;
+// CHECK-NEXT:  }
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td b/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td
new file mode 100644
index 0000000000000..00c9d53846326
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td
@@ -0,0 +1,60 @@
+// RUN: llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s 2> %t.err | FileCheck %s
+// RUN: FileCheck -check-prefix=ERR %s < %t.err
+
+// Check behavior of libcall emission when multiple RuntimeLibcallImpl
+// implementations provide the same RuntimeLibcall
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+def SOME_FUNC : RuntimeLibcall;
+def OTHER_FUNC : RuntimeLibcall;
+def ANOTHER_DUP : RuntimeLibcall;
+
+def isTargetArchA : RuntimeLibcallPredicate<[{isTargetArchA()}]>;
+def isTargetArchB : RuntimeLibcallPredicate<[{isTargetArchB()}]>;
+def isTargetArchC : RuntimeLibcallPredicate<[{isTargetArchC()}]>;
+
+def func_a : RuntimeLibcallImpl<SOME_FUNC>;
+def func_b : RuntimeLibcallImpl<SOME_FUNC>;
+def func_c : RuntimeLibcallImpl<SOME_FUNC>;
+def other_func : RuntimeLibcallImpl<OTHER_FUNC>;
+
+def dup0 : RuntimeLibcallImpl<ANOTHER_DUP>;
+def dup1 : RuntimeLibcallImpl<ANOTHER_DUP>;
+
+// func_a and func_b both provide SOME_FUNC.
+
+// CHECK: if (isTargetArchA()) {
+// CHECK-NEXT: static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:   {RTLIB::SOME_FUNC, RTLIB::func_b}, // func_b
+// CHECK-NEXT: };
+
+// ERR: :[[@LINE+1]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_b, func_a
+def TheSystemLibraryA : SystemRuntimeLibrary<isTargetArchA,
+  (add func_b, func_a)
+>;
+
+// CHECK: if (isTargetArchB()) {
+// CHECK-NEXT: static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:   {RTLIB::OTHER_FUNC, RTLIB::other_func}, // other_func
+// CHECK-NEXT:  {RTLIB::SOME_FUNC, RTLIB::func_a}, // func_a
+// CHECK-NEXT: };
+
+// ERR: :[[@LINE+1]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_a, func_b
+def TheSystemLibraryB : SystemRuntimeLibrary<isTargetArchB,
+  (add func_a, other_func, func_b)
+>;
+
+// CHECK: if (isTargetArchC()) {
+// CHECK-NEXT: static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:   {RTLIB::ANOTHER_DUP, RTLIB::dup1}, // dup1
+// CHECK-NEXT:   {RTLIB::OTHER_FUNC, RTLIB::other_func}, // other_func
+// CHECK-NEXT:   {RTLIB::SOME_FUNC, RTLIB::func_a}, // func_a
+// CHECK-NEXT: };
+
+// ERR: :[[@LINE+3]]:5: warning: conflicting implementations for libcall ANOTHER_DUP: dup1, dup0
+// ERR: :[[@LINE+2]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_a, func_b
+// ERR: :[[@LINE+1]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_a, func_c
+def TheSystemLibraryC : SystemRuntimeLibrary<isTargetArchC,
+  (add func_a, dup1, other_func, func_b, func_c, dup0)
+>;
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-nested-predicates-error.td b/llvm/test/TableGen/RuntimeLibcallEmitter-nested-predicates-error.td
new file mode 100644
index 0000000000000..b8ee9104f65f9
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-nested-predicates-error.td
@@ -0,0 +1,18 @@
+// RUN: not llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s 2>&1 | FileCheck -check-prefix=ERR %s
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+def FUNC0 : RuntimeLibcall;
+def FUNC1 : RuntimeLibcall;
+
+def isFoo : RuntimeLibcallPredicate<[{isFoo()}]>;
+def isBar : RuntimeLibcallPredicate<[{isBar()}]>;
+def isTargetArch : RuntimeLibcallPredicate<[{isTargetArch()}]>;
+
+def func0 : RuntimeLibcallImpl<FUNC0>;
+def func1 : RuntimeLibcallImpl<FUNC1>;
+
+// ERR: :[[@LINE+2]]:8: error: combining nested libcall set predicates currently unhandled
+def TheSystemLibrary : SystemRuntimeLibrary<isTargetArch,
+  (add LibcallImpls<(add func0, LibcallImpls<(add func1), isBar>), isFoo>)
+>;
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter.td b/llvm/test/TableGen/RuntimeLibcallEmitter.td
index d4030dd0a7e95..0d7a69165d6f9 100644
--- a/llvm/test/TableGen/RuntimeLibcallEmitter.td
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter.td
@@ -9,6 +9,9 @@ def SRL_I64 : RuntimeLibcall;
 def SQRT_F128 : RuntimeLibcall;
 def SQRT_F80 : RuntimeLibcall;
 def BZERO : RuntimeLibcall;
+def MEMCPY : RuntimeLibcall;
+def MEMSET : RuntimeLibcall;
+def CALLOC : RuntimeLibcall;
 
 // Test default names.
 let IsDefault = true in {
@@ -22,27 +25,79 @@ let IsDefault = true in {
 // Ignore non-default in initDefaultLibCallNames.
 def bzero : RuntimeLibcallImpl<BZERO>;
 
+def ___memset : RuntimeLibcallImpl<MEMSET, "___memset">;
+def ___memcpy : RuntimeLibcallImpl<MEMCPY, "___memcpy">;
+
+def calloc  : RuntimeLibcallImpl<CALLOC, "calloc">;
+
+def CompilerRTLibcalls : LibcallImpls<(add __ashlsi3, __lshrdi3)>;
+def LibmLibcalls : LibcallImpls<(add sqrtl_f80)>;
+
+def isSimpleArch : RuntimeLibcallPredicate<[{TT.getArch() == Triple::simple}]>;
+def isFooArch : RuntimeLibcallPredicate<[{TT.getArch() == Triple::foo}]>;
+
+
+def isZOS : RuntimeLibcallPredicate<[{TT.getOS() == Triple::zos}]>;
+def isPPC : RuntimeLibcallPredicate<[{TT.getArch().isPPC()}]>;
+def isPPC64 : RuntimeLibcallPredicate<[{TT.getArch().isPPC64()}]>;
+
+
+def isFoo : RuntimeLibcallPredicate<[{isFOO()}]>;
+def isBarOS : RuntimeLibcallPredicate<[{TT.getOS() == Triple::bar}]>;
+def isBuzzArch : RuntimeLibcallPredicate<[{TT.getArch() == Triple::buzz}]>;
+def isBlahArch : RuntimeLibcallPredicate<[{TT.getArch() == Triple::blah}]>;
+
+def hasCompilerRT : RuntimeLibcallPredicate<[{TT.hasCompilerRT()}]>;
+
+
+def SimpleLibrary : SystemRuntimeLibrary<isSimpleArch,
+  (add LibmLibcalls, calloc, CompilerRTLibcalls)>;
+
+
+def LibraryWithConditionalFunc : LibcallImpls<(add sqrtl_f128, bzero, AvailableIf<___memset, isBarOS>)>;
+
+def FooLibrary : SystemRuntimeLibrary<isFooArch, (add LibraryWithConditionalFunc)>;
+
+def BuzzLibrary : SystemRuntimeLibrary<isBuzzArch, (add sqrtl_f80, CompilerRTLibcalls)>;
+
+
+
+def LibraryWithConditionalSet : LibcallImpls<(add sqrtl_f128, bzero,
+  LibcallImpls<(add CompilerRTLibcalls), hasCompilerRT>)>;
+
+
+
+def BlahLibrary : SystemRuntimeLibrary<isBlahArch, (add calloc, LibraryWithConditionalSet, AvailableIf<___memset, isBarOS>)>;
+
+
+
 // All entries should be emitted in Libcall enum.
 // CHECK: #ifdef GET_RUNTIME_LIBCALL_ENUM
 // CHECK-NEXT: namespace llvm {
 // CHECK-NEXT: namespace RTLIB {
 // CHECK-NEXT: enum Libcall : unsigned short {
 // CHECK-NEXT: BZERO = 0,
-// CHECK-NEXT: SHL_I32 = 1,
-// CHECK-NEXT: SQRT_F80 = 2,
-// CHECK-NEXT: SQRT_F128 = 3,
-// CHECK-NEXT: SRL_I64 = 4,
-// CHECK-NEXT: UNKNOWN_LIBCALL = 5
+// CHECK-NEXT: CALLOC = 1,
+// CHECK-NEXT: MEMCPY = 2,
+// CHECK-NEXT: MEMSET = 3,
+// CHECK-NEXT: SHL_I32 = 4,
+// CHECK-NEXT: SQRT_F80 = 5,
+// CHECK-NEXT: SQRT_F128 = 6,
+// CHECK-NEXT: SRL_I64 = 7,
+// CHECK-NEXT: UNKNOWN_LIBCALL = 8
 // CHECK-NEXT: };
 // CHECK-EMPTY:
 // CHECK-NEXT:enum LibcallImpl : unsigned short {
 // CHECK-NEXT:  Unsupported = 0,
-// CHECK-NEXT:  __ashlsi3 = 1, // __ashlsi3
-// CHECK-NEXT:  __lshrdi3 = 2, // __lshrdi3
-// CHECK-NEXT:  bzero = 3, // bzero
-// CHECK-NEXT:  sqrtl_f80 = 4, // sqrtl
-// CHECK-NEXT:  sqrtl_f128 = 5, // sqrtl
-// CHECK-NEXT:  NumLibcallImpls = 6
+// CHECK-NEXT:  ___memcpy = 1, // ___memcpy
+// CHECK-NEXT:  ___memset = 2, // ___memset
+// CHECK-NEXT:  __ashlsi3 = 3, // __ashlsi3
+// CHECK-NEXT:  __lshrdi3 = 4, // __lshrdi3
+// CHECK-NEXT:  bzero = 5, // bzero
+// CHECK-NEXT:  calloc = 6, // calloc
+// CHECK-NEXT:  sqrtl_f80 = 7, // sqrtl
+// CHECK-NEXT:  sqrtl_f128 = 8, // sqrtl
+// CHECK-NEXT:  NumLibcallImpls = 9
 // CHECK-NEXT: };
 // CHECK-NEXT: } // End namespace RTLIB
 // CHECK-NEXT: } // End namespace llvm
@@ -51,6 +106,9 @@ def bzero : RuntimeLibcallImpl<BZERO>;
 // CHECK: #ifdef GET_INIT_RUNTIME_LIBCALL_NAMES
 // CHECK-NEXT: const RTLIB::LibcallImpl llvm::RTLIB::RuntimeLibcallsInfo::DefaultLibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {
 // CHECK-NEXT:   RTLIB::Unsupported, // RTLIB::BZERO
+// CHECK-NEXT:   RTLIB::Unsupported, // RTLIB::CALLOC
+// CHECK-NEXT:   RTLIB::Unsupported, // RTLIB::MEMCPY
+// CHECK-NEXT:   RTLIB::Unsupported, // RTLIB::MEMSET
 // CHECK-NEXT:   RTLIB::__ashlsi3, // RTLIB::SHL_I32
 // CHECK-NEXT:   RTLIB::sqrtl_f80, // RTLIB::SQRT_F80
 // CHECK-NEXT:   RTLIB::sqrtl_f128, // RTLIB::SQRT_F128
@@ -60,18 +118,125 @@ def bzero : RuntimeLibcallImpl<BZERO>;
 // CHECK-EMPTY:
 // CHECK-NEXT: const char *const llvm::RTLIB::RuntimeLibcallsInfo::LibCallImplNames[RTLIB::NumLibcallImpls] = {
 // CHECK-NEXT: nullptr, // RTLIB::Unsupported
+// CHECK-NEXT: "___memcpy", // RTLIB::___memcpy
+// CHECK-NEXT: "___memset", // RTLIB::___memset
 // CHECK-NEXT: "__ashlsi3", // RTLIB::__ashlsi3
 // CHECK-NEXT: "__lshrdi3", // RTLIB::__lshrdi3
 // CHECK-NEXT: "bzero", // RTLIB::bzero
+// CHECK-NEXT: "calloc", // RTLIB::calloc
 // CHECK-NEXT: "sqrtl", // RTLIB::sqrtl_f80
 // CHECK-NEXT: "sqrtl", // RTLIB::sqrtl_f128
 // CHECK-NEXT: };
 
 // CHECK: const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::ImplToLibcall[RTLIB::NumLibcallImpls] = {
 // CHECK-NEXT: RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported
+// CHECK-NEXT: RTLIB::MEMCPY, // RTLIB::___memcpy
+// CHECK-NEXT: RTLIB::MEMSET, // RTLIB::___memset
 // CHECK-NEXT: RTLIB::SHL_I32, // RTLIB::__ashlsi3
 // CHECK-NEXT: RTLIB::SRL_I64, // RTLIB::__lshrdi3
 // CHECK-NEXT: RTLIB::BZERO, // RTLIB::bzero
+// CHECK-NEXT: RTLIB::CALLOC, // RTLIB::calloc
 // CHECK-NEXT: RTLIB::SQRT_F80, // RTLIB::sqrtl_f80
 // CHECK-NEXT: RTLIB::SQRT_F128, // RTLIB::sqrtl_f128
 // CHECK-NEXT: };
+
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(const llvm::Triple &TT) {
+// CHECK-NEXT:  struct LibcallImplPair {
+// CHECK-NEXT:    RTLIB::Libcall Func;
+// CHECK-NEXT:    RTLIB::LibcallImpl Impl;
+// CHECK-NEXT:  };
+// CHECK-EMPTY:
+// CHECK-NEXT: if (TT.getArch() == Triple::blah) {
+// CHECK-NEXT:     static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:         {RTLIB::BZERO, RTLIB::bzero}, // bzero
+// CHECK-NEXT:         {RTLIB::CALLOC, RTLIB::calloc}, // calloc
+// CHECK-NEXT:         {RTLIB::SQRT_F128, RTLIB::sqrtl_f128}, // sqrtl
+// CHECK-NEXT:     };
+// CHECK-EMPTY:
+// CHECK-NEXT:     for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:       setLibcallImpl(Func, Impl);
+// CHECK-NEXT:     }
+// CHECK-EMPTY:
+// CHECK-NEXT:    if (TT.hasCompilerRT()) {
+// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_hasCompilerRT[] = {
+// CHECK-NEXT:          {RTLIB::SHL_I32, RTLIB::__ashlsi3}, // __ashlsi3
+// CHECK-NEXT:          {RTLIB::SRL_I64, RTLIB::__lshrdi3}, // __lshrdi3
+// CHECK-NEXT:      };
+// CHECK-EMPTY:
+// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_hasCompilerRT) {
+// CHECK-NEXT:        setLibcallImpl(Func, Impl);
+// CHECK-NEXT:      }
+// CHECK-EMPTY:
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    if (TT.getOS() == Triple::bar) {
+// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_isBarOS[] = {
+// CHECK-NEXT:          {RTLIB::MEMSET, RTLIB::___memset}, // ___memset
+// CHECK-NEXT:      };
+// CHECK-EMPTY:
+// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_isBarOS) {
+// CHECK-NEXT:        setLibcallImpl(Func, Impl);
+// CHECK-NEXT:      }
+// CHECK-EMPTY:
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:   return;
+// CHECK-NEXT: }
+// CHECK-EMPTY:
+// CHECK-NEXT: if (TT.getArch() == Triple::buzz) {
+// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:        {RTLIB::SHL_I32, RTLIB::__ashlsi3}, // __ashlsi3
+// CHECK-NEXT:        {RTLIB::SQRT_F80, RTLIB::sqrtl_f80}, // sqrtl
+// CHECK-NEXT:        {RTLIB::SRL_I64, RTLIB::__lshrdi3}, // __lshrdi3
+// CHECK-NEXT:    };
+// CHECK-EMPTY:
+// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:   return;
+// CHECK-NEXT: }
+// CHECK-EMPTY:
+// CHECK-NEXT: if (TT.getArch() == Triple::foo) {
+// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:        {RTLIB::BZERO, RTLIB::bzero}, // bzero
+// CHECK-NEXT:        {RTLIB::SQRT_F128, RTLIB::sqrtl_f128}, // sqrtl
+// CHECK-NEXT:    };
+// CHECK-EMPTY:
+// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    if (TT.getOS() == Triple::bar) {
+// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_isBarOS[] = {
+// CHECK-NEXT:          {RTLIB::MEMSET, RTLIB::___memset}, // ___memset
+// CHECK-NEXT:      };
+// CHECK-EMPTY:
+// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_isBarOS) {
+// CHECK-NEXT:        setLibcallImpl(Func, Impl);
+// CHECK-NEXT:      }
+// CHECK-EMPTY:
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:    return;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT: if (TT.getArch() == Triple::simple) {
+// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:        {RTLIB::CALLOC, RTLIB::calloc}, // calloc
+// CHECK-NEXT:        {RTLIB::SHL_I32, RTLIB::__ashlsi3}, // __ashlsi3
+// CHECK-NEXT:        {RTLIB::SQRT_F80, RTLIB::sqrtl_f80}, // sqrtl
+// CHECK-NEXT:        {RTLIB::SRL_I64, RTLIB::__lshrdi3}, // __lshrdi3
+// CHECK-NEXT:    };
+// CHECK-EMPTY:
+// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:    }
+// CHECK-EMPTY:
+// CHECK-NEXT:   return;
+// CHECK-NEXT: }
+// CHECK-NEXT:  initDefaultLibCallImpls();
+// CHECK-NEXT: }
+// CHECK-EMPTY:
+// CHECK: #endif
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
index 1c2ae608711cc..668e7ba9b1f0f 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
@@ -787,419 +787,1863 @@ define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_no_fine_grained_memo
 }
 
 define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX90A-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP5]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
 define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(ptr addrspace(1) %ptr, double %value) #0 {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(ptr addrspace(1) %ptr, double %value) #1 {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-;---------------------------------------------------------------------
+define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX90A-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP5]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(ptr addrspace(1) %ptr, double %value) #0 {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(ptr addrspace(1) %ptr, double %value) #1 {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+;---------------------------------------------------------------------
 ; atomicrmw fsub
 ;---------------------------------------------------------------------
 
-define double @test_atomicrmw_fsub_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+define double @test_atomicrmw_fsub_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+;---------------------------------------------------------------------
+; atomicrmw fmax
+;---------------------------------------------------------------------
+
+define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX90A-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP6]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX10-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP6]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX90A-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX10-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX90A-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP6]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX10-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP6]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX90A-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP6]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX10-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP6]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
 ;---------------------------------------------------------------------
-; atomicrmw fmax
+; atomicrmw fmin
 ;---------------------------------------------------------------------
 
-define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1210,13 +2654,13 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1227,13 +2671,13 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1244,13 +2688,13 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX90A:       atomicrmw.start:
 ; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1261,18 +2705,18 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX90A:       atomicrmw.end:
 ; GFX90A-NEXT:    ret double [[TMP6]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX10:       atomicrmw.start:
 ; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1283,13 +2727,13 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX10:       atomicrmw.end:
 ; GFX10-NEXT:    ret double [[TMP6]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1300,13 +2744,13 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8
@@ -1317,18 +2761,18 @@ define double @test_atomicrmw_fmax_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
   ret double %res
 }
 
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1339,13 +2783,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1356,13 +2800,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1373,28 +2817,28 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1405,13 +2849,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1422,18 +2866,18 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1444,13 +2888,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1461,13 +2905,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1478,13 +2922,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX90A:       atomicrmw.start:
 ; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1495,18 +2939,18 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX90A:       atomicrmw.end:
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX10:       atomicrmw.start:
 ; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1517,13 +2961,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX10:       atomicrmw.end:
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1534,13 +2978,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1551,18 +2995,18 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1573,13 +3017,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1590,13 +3034,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1607,28 +3051,28 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1639,13 +3083,13 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1656,140 +3100,12 @@ define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
   ret double %res
 }
 
-;---------------------------------------------------------------------
-; atomicrmw fmin
-;---------------------------------------------------------------------
-
-define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1806,7 +3122,7 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1823,7 +3139,7 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1840,7 +3156,7 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1857,12 +3173,12 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX90A:       atomicrmw.end:
 ; GFX90A-NEXT:    ret double [[TMP6]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1879,7 +3195,7 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX10:       atomicrmw.end:
 ; GFX10-NEXT:    ret double [[TMP6]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1896,7 +3212,7 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1913,12 +3229,12 @@ define double @test_atomicrmw_fmin_f64_global_agent(ptr addrspace(1) %ptr, doubl
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1935,7 +3251,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1952,7 +3268,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1969,22 +3285,22 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2001,7 +3317,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2018,12 +3334,12 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2040,7 +3356,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2057,7 +3373,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2074,46 +3390,46 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX90A:       atomicrmw.start:
-; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
 ; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
 ; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; GFX90A-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
 ; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX90A:       atomicrmw.end:
-; GFX90A-NEXT:    ret double [[RES]]
+; GFX90A-NEXT:    ret double [[TMP6]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX10:       atomicrmw.start:
-; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
 ; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
 ; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; GFX10-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
 ; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX10:       atomicrmw.end:
-; GFX10-NEXT:    ret double [[RES]]
+; GFX10-NEXT:    ret double [[TMP6]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2130,7 +3446,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2147,12 +3463,12 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_remote_memory(ptr
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2169,7 +3485,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2186,7 +3502,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2203,22 +3519,22 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2235,7 +3551,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -2252,131 +3568,7 @@ define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_no_fine_grained_memo
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_agent__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
index 72ecbf7708d86..f9c123f5b9427 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
@@ -682,417 +682,1540 @@ define double @test_atomicrmw_fadd_f64_global_system__amdgpu_no_fine_grained_mem
 define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; COMMON:       atomicrmw.start:
 ; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
 ; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP5:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP5]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED1:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[NEWLOADED]] = bitcast i64 [[NEWLOADED1]] to double
 ; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret double [[NEWLOADED]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX90A-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP5]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(ptr addrspace(1) %ptr, double %value) #0 {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(ptr addrspace(1) %ptr, double %value) #1 {
+; GFX803-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP5]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP5]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP5]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP5]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP5]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
+; GFX12-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX12-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+;---------------------------------------------------------------------
+; atomicrmw fsub
+;---------------------------------------------------------------------
+
+define double @test_atomicrmw_fsub_f64_global_system(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[RES]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP5]]
+;
+  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret double %res
+}
+
+;---------------------------------------------------------------------
+; atomicrmw fmax
+;---------------------------------------------------------------------
+
+define double @test_atomicrmw_fmax_f64_global_system(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system(
+; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; COMMON-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX90A-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX10-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
+  ret double %res
+}
+
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
   ret double %res
 }
 
-define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
+; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; COMMON-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; COMMON-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; COMMON-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
 ; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; COMMON-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(ptr addrspace(1) %ptr, double %value) #0 {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_daz(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR1:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(ptr addrspace(1) %ptr, double %value) #1 {
-; COMMON-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory__denormal_mode_dynamic(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR2:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP3:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP5:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP1]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP5]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[NEWLOADED]]
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
+;
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
+;
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
+;
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX90A-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret double [[TMP6]]
+;
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
+;
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX10-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret double [[TMP6]]
+;
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
+;
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-;---------------------------------------------------------------------
-; atomicrmw fsub
-;---------------------------------------------------------------------
-
-define double @test_atomicrmw_fsub_f64_global_system(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX803-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX803-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX906-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX906-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP2:%.*]] = bitcast double [[NEW]] to i64
-; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
-; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
-; COMMON-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[RES]]
+; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX90A-NEXT:    ret double [[RES]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX942-NEXT:    ret double [[RES]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
+; GFX10-NEXT:    ret double [[RES]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX11-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX11-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fsub_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP2:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    store double [[NEW]], ptr addrspace(5) [[TMP2]], align 8
-; COMMON-NEXT:    [[TMP9:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP4]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP2]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP2]])
-; COMMON-NEXT:    [[TMP6:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP7:%.*]] = insertvalue { double, i1 } poison, double [[TMP6]], 0
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } [[TMP7]], i1 [[TMP9]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP8]], 1
-; COMMON-NEXT:    [[TMP5]] = extractvalue { double, i1 } [[TMP8]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP5]]
+; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
+; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX12:       atomicrmw.start:
+; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
+; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX12-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX12-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX12-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX12-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX12:       atomicrmw.end:
+; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
 ;---------------------------------------------------------------------
-; atomicrmw fmax
+; atomicrmw fmin
 ;---------------------------------------------------------------------
 
-define double @test_atomicrmw_fmax_f64_global_system(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system(
+define double @test_atomicrmw_fmin_f64_global_system(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; COMMON:       atomicrmw.start:
 ; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; COMMON-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; COMMON-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; COMMON-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
@@ -1103,18 +2226,18 @@ define double @test_atomicrmw_fmax_f64_global_system(ptr addrspace(1) %ptr, doub
 ; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst
   ret double %res
 }
 
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1125,13 +2248,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1142,13 +2265,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1159,28 +2282,28 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1191,13 +2314,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
@@ -1208,18 +2331,18 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1230,13 +2353,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1247,13 +2370,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1264,13 +2387,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX90A:       atomicrmw.start:
 ; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1281,18 +2404,18 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX90A:       atomicrmw.end:
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX10:       atomicrmw.start:
 ; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
-; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1303,13 +2426,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX10:       atomicrmw.end:
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1320,13 +2443,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
@@ -1337,18 +2460,18 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX803:       atomicrmw.start:
 ; GFX803-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX803-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX803-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX803-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX803-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1359,13 +2482,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX906:       atomicrmw.start:
 ; GFX906-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX906-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX906-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX906-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX906-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1376,13 +2499,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX908:       atomicrmw.start:
 ; GFX908-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX908-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX908-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX908-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX908-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1393,28 +2516,28 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX11:       atomicrmw.start:
 ; GFX11-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX11-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX11-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX11-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX11-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1425,13 +2548,13 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX12:       atomicrmw.start:
 ; GFX12-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
+; GFX12-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX12-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX12-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX12-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
@@ -1442,140 +2565,12 @@ define double @test_atomicrmw_fmax_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmax_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmax ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
   ret double %res
 }
 
-;---------------------------------------------------------------------
-; atomicrmw fmin
-;---------------------------------------------------------------------
-
-define double @test_atomicrmw_fmin_f64_global_system(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
+; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1592,12 +2587,12 @@ define double @test_atomicrmw_fmin_f64_global_system(ptr addrspace(1) %ptr, doub
 ; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1614,7 +2609,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1631,7 +2626,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1648,22 +2643,22 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1680,7 +2675,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1697,12 +2692,12 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1719,7 +2714,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1736,7 +2731,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1753,46 +2748,46 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX90A-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX90A:       atomicrmw.start:
-; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
 ; GFX90A-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX90A-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX90A-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX90A-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
 ; GFX90A-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; GFX90A-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX90A-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
 ; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX90A:       atomicrmw.end:
-; GFX90A-NEXT:    ret double [[RES]]
+; GFX90A-NEXT:    ret double [[TMP6]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX10-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; GFX10:       atomicrmw.start:
-; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
 ; GFX10-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
 ; GFX10-NEXT:    [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
 ; GFX10-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
 ; GFX10-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
 ; GFX10-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
-; GFX10-NEXT:    [[RES]] = bitcast i64 [[NEWLOADED]] to double
+; GFX10-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
 ; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; GFX10:       atomicrmw.end:
-; GFX10-NEXT:    ret double [[RES]]
+; GFX10-NEXT:    ret double [[TMP6]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1809,7 +2804,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1826,12 +2821,12 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_remote_memory(pt
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
+; GFX803-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX803-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1848,7 +2843,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX803:       atomicrmw.end:
 ; GFX803-NEXT:    ret double [[TMP6]]
 ;
-; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX906-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX906-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1865,7 +2860,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX906:       atomicrmw.end:
 ; GFX906-NEXT:    ret double [[TMP6]]
 ;
-; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX908-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX908-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1882,22 +2877,22 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX908:       atomicrmw.end:
 ; GFX908-NEXT:    ret double [[TMP6]]
 ;
-; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX90A-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX90A-NEXT:    ret double [[RES]]
 ;
-; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX942-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX942-NEXT:    ret double [[RES]]
 ;
-; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX10-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]], !amdgpu.ignore.denormal.mode [[META0]]
 ; GFX10-NEXT:    ret double [[RES]]
 ;
-; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX11-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX11-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1914,7 +2909,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX11:       atomicrmw.end:
 ; GFX11-NEXT:    ret double [[TMP6]]
 ;
-; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GFX12-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
 ; GFX12-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 8
 ; GFX12-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1931,131 +2926,7 @@ define double @test_atomicrmw_fmin_f64_global_system__amdgpu_no_fine_grained_mem
 ; GFX12:       atomicrmw.end:
 ; GFX12-NEXT:    ret double [[TMP6]]
 ;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
-  ret double %res
-}
-
-define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
-; COMMON-LABEL: define double @test_atomicrmw_fmin_f64_global_system__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP3:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP4:%.*]] = alloca double, align 8, addrspace(5)
-; COMMON-NEXT:    [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])
-; COMMON-NEXT:    [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    store double [[LOADED]], ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    store double [[TMP2]], ptr addrspace(5) [[TMP4]], align 8
-; COMMON-NEXT:    [[TMP10:%.*]] = call zeroext i1 @__atomic_compare_exchange(i64 8, ptr [[TMP5]], ptr addrspace(5) [[TMP3]], ptr addrspace(5) [[TMP4]], i32 5, i32 5)
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP4]])
-; COMMON-NEXT:    [[TMP7:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 8
-; COMMON-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[TMP3]])
-; COMMON-NEXT:    [[TMP8:%.*]] = insertvalue { double, i1 } poison, double [[TMP7]], 0
-; COMMON-NEXT:    [[TMP9:%.*]] = insertvalue { double, i1 } [[TMP8]], i1 [[TMP10]], 1
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { double, i1 } [[TMP9]], 1
-; COMMON-NEXT:    [[TMP6]] = extractvalue { double, i1 } [[TMP9]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
-; COMMON-NEXT:    ret double [[TMP6]]
-;
-  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 4, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %res = atomicrmw fmin ptr addrspace(1) %ptr, double %value seq_cst, align 8, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
   ret double %res
 }
 
diff --git a/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll b/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
new file mode 100644
index 0000000000000..d5aa6b10b5add
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
@@ -0,0 +1,431 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=attributor %s -S | FileCheck %s --check-prefix=AMDGCN
+
+@G = addrspace(1) global i32 zeroinitializer, align 4
+declare void @clobber(i32) #0
+declare void @clobber.p5(ptr addrspace(5)) #0
+declare ptr addrspace(1) @get_ptr() #0
+declare noalias ptr addrspace(1) @get_noalias_ptr() #0
+declare noalias ptr addrspace(1) @get_untouched_ptr() #1
+
+define void @test_nonkernel(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define void @test_nonkernel(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7:[0-9]+]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be !invariant.load, as the caller may modify %ptr
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_plain(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_plain(
+; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be !invariant.load, as %ptr may alias a pointer in @clobber
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_noalias_ptr(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_noalias_ptr(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0:![0-9]+]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_gep(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_gep(
+; AMDGCN-SAME: ptr addrspace(1) nofree readonly align 4 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i32 4
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
+  %val = load i32, ptr addrspace(1) %gep, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_noalias_gep(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_noalias_gep(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly align 4 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i32 4
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
+  %val = load i32, ptr addrspace(1) %gep, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_swap(ptr addrspace(1) noalias %ptr, i32 inreg %swap) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_swap(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]], i32 inreg [[SWAP:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    store i32 [[SWAP]], ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; cannot be !invariant.load due to the write to %ptr
+  store i32 %swap, ptr addrspace(1) %ptr, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_volatile(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_volatile(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef align 4 [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load volatile i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load volatile i32, ptr addrspace(1) %ptr, align 4
+  ;; volatiles loads cannot be !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_unordered(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_unordered(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load atomic i32, ptr addrspace(1) %ptr unordered, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_monotonic(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_monotonic(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] monotonic, align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load atomic i32, ptr addrspace(1) %ptr monotonic, align 4
+  ;; atomic loads with ordering guarantees may have side effects
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_global() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_global(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) @G, align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) @G, align 4
+  ;; is not an !invariant.load as global variables may change
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_alloca(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_alloca(
+; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; AMDGCN-NEXT:    store i32 [[VAL]], ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    call void @clobber.p5(ptr addrspace(5) noundef align 4 [[ALLOCA]]) #[[ATTR7]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[LOAD]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  %alloca = alloca i32, addrspace(5)
+  store i32 %val, ptr addrspace(5) %alloca
+  %load = load i32, ptr addrspace(5) %alloca
+  call void @clobber.p5(ptr addrspace(5) %alloca)
+  call void @clobber(i32 %load)
+  ret void
+}
+
+define internal void @copy.i32(ptr addrspace(5) %alloca, i32 %qty) {
+; AMDGCN-LABEL: define internal void @copy.i32(
+; AMDGCN-SAME: ptr addrspace(5) noalias nofree noundef writeonly align 4 captures(none) dereferenceable_or_null(4) [[ALLOCA:%.*]], i32 [[QTY:%.*]]) #[[ATTR4:[0-9]+]] {
+; AMDGCN-NEXT:    store i32 [[QTY]], ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    ret void
+;
+  store i32 %qty, ptr addrspace(5) %alloca
+  ret void
+}
+
+define amdgpu_kernel void @test_internal_alloca(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_internal_alloca(
+; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; AMDGCN-NEXT:    call void @copy.i32(ptr addrspace(5) noalias nofree noundef writeonly align 4 captures(none) dereferenceable_or_null(4) [[ALLOCA]], i32 [[VAL]]) #[[ATTR8:[0-9]+]]
+; AMDGCN-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[LOAD]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  %alloca = alloca i32, addrspace(5)
+  call void @copy.i32(ptr addrspace(5) %alloca, i32 %val)
+  %load = load i32, ptr addrspace(5) %alloca
+  call void @clobber(i32 %load)
+  ret void
+}
+
+define internal i32 @test_internal_noalias_load(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define internal i32 @test_internal_noalias_load(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    ret i32 [[VAL]]
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; is an !invariant.load due to its only caller @test_call_internal_noalias
+  ret i32 %val
+}
+
+define amdgpu_kernel void @test_call_internal_noalias(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal_noalias(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_noalias_load(ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR9:[0-9]+]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = call i32 @test_internal_noalias_load(ptr addrspace(1) %ptr)
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define internal i32 @test_internal_load(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define internal i32 @test_internal_load(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR5]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    ret i32 [[VAL]]
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be an !invariant.load since the pointer in @test_call_internal may alias
+  ret i32 %val
+}
+
+define amdgpu_kernel void @test_call_internal(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal(
+; AMDGCN-SAME: ptr addrspace(1) nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_load(ptr addrspace(1) nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR9]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = call i32 @test_internal_load(ptr addrspace(1) %ptr)
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define internal i32 @test_internal_written(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define internal i32 @test_internal_written(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR5]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    ret i32 [[VAL]]
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; cannot be an !invariant.load because of the write in caller @test_call_internal_written
+  ret i32 %val
+}
+
+define amdgpu_kernel void @test_call_internal_written(ptr addrspace(1) noalias %ptr, i32 inreg %x) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal_written(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree captures(none) [[PTR:%.*]], i32 inreg [[X:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_written(ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR9]]
+; AMDGCN-NEXT:    store i32 [[X]], ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = call i32 @test_internal_written(ptr addrspace(1) %ptr)
+  store i32 %x, ptr addrspace(1) %ptr
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_ptr() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_ptr(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = call align 4 ptr addrspace(1) @get_ptr() #[[ATTR7]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = call ptr addrspace(1) @get_ptr()
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be an !invariant.load since %ptr may alias
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_noalias_ptr() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_noalias_ptr(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = call align 4 ptr addrspace(1) @get_noalias_ptr() #[[ATTR7]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = call ptr addrspace(1) @get_noalias_ptr()
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be an !invariant.load since %ptr may have been written to before returning
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_untouched_ptr() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_untouched_ptr(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = call noalias align 4 ptr addrspace(1) @get_untouched_ptr() #[[ATTR10:[0-9]+]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = call ptr addrspace(1) @get_untouched_ptr()
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_make_buffer(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_make_buffer(
+; AMDGCN-SAME: ptr addrspace(1) nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[RSRC:%.*]] = call align 4 ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) [[PTR]], i16 noundef 0, i32 noundef 0, i32 noundef 0) #[[ATTR11:[0-9]+]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(7) [[RSRC]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %rsrc = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) %ptr, i16 0, i32 0, i32 0)
+  %val = load i32, ptr addrspace(7) %rsrc, align 4
+  ;; original %ptr may alias
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_make_buffer_noalias(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_make_buffer_noalias(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[RSRC:%.*]] = call align 4 ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) [[PTR]], i16 noundef 0, i32 noundef 0, i32 noundef 0) #[[ATTR11]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(7) [[RSRC]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %rsrc = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) %ptr, i16 0, i32 0, i32 0)
+  %val = load i32, ptr addrspace(7) %rsrc, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_selected_load(i1 inreg %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) noalias %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_selected_load(
+; AMDGCN-SAME: i1 inreg [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = select i1 [[COND]], ptr addrspace(1) [[PTR_TRUE]], ptr addrspace(1) [[PTR_FALSE]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = select i1 %cond, ptr addrspace(1) %ptr.true, ptr addrspace(1) %ptr.false
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; either pointer yields an !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_selected_load_partial_noalias(i1 inreg %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_selected_load_partial_noalias(
+; AMDGCN-SAME: i1 inreg [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = select i1 [[COND]], ptr addrspace(1) [[PTR_TRUE]], ptr addrspace(1) [[PTR_FALSE]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = select i1 %cond, ptr addrspace(1) %ptr.true, ptr addrspace(1) %ptr.false
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; %ptr.false may alias, so no !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_branch_load(i1 %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) noalias %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_branch_load(
+; AMDGCN-SAME: i1 noundef [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:  [[ENTRY:.*:]]
+; AMDGCN-NEXT:    br i1 [[COND]], label %[[TRUE:.*]], label %[[FALSE:.*]]
+; AMDGCN:       [[TRUE]]:
+; AMDGCN-NEXT:    call void @clobber(i32 noundef 1) #[[ATTR7]]
+; AMDGCN-NEXT:    br label %[[FINISH:.*]]
+; AMDGCN:       [[FALSE]]:
+; AMDGCN-NEXT:    br label %[[FINISH]]
+; AMDGCN:       [[FINISH]]:
+; AMDGCN-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[PTR_TRUE]], %[[TRUE]] ], [ [[PTR_FALSE]], %[[FALSE]] ]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %true, label %false
+true:
+  call void @clobber(i32 1)
+  br label %finish
+false:
+  br label %finish
+finish:
+  %ptr = phi ptr addrspace(1) [ %ptr.true, %true ], [ %ptr.false, %false ]
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; either pointer yields an !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_branch_load_partial_noalias(i1 %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_branch_load_partial_noalias(
+; AMDGCN-SAME: i1 noundef [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:  [[ENTRY:.*:]]
+; AMDGCN-NEXT:    br i1 [[COND]], label %[[TRUE:.*]], label %[[FALSE:.*]]
+; AMDGCN:       [[TRUE]]:
+; AMDGCN-NEXT:    call void @clobber(i32 noundef 1) #[[ATTR7]]
+; AMDGCN-NEXT:    br label %[[FINISH:.*]]
+; AMDGCN:       [[FALSE]]:
+; AMDGCN-NEXT:    br label %[[FINISH]]
+; AMDGCN:       [[FINISH]]:
+; AMDGCN-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[PTR_TRUE]], %[[TRUE]] ], [ [[PTR_FALSE]], %[[FALSE]] ]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %true, label %false
+true:
+  call void @clobber(i32 1)
+  br label %finish
+false:
+  br label %finish
+finish:
+  %ptr = phi ptr addrspace(1) [ %ptr.true, %true ], [ %ptr.false, %false ]
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; ptr.false may alias, so no !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+attributes #0 = { nofree norecurse nosync nounwind willreturn }
+attributes #1 = { nofree norecurse nosync nounwind willreturn readonly }
+;.
+; AMDGCN: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
index b0446479dac4b..6641fdb9b4ffe 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
@@ -68,7 +68,7 @@ define i64 @fn2c() {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CONV:%.*]] = sext i32 undef to i64
 ; CGSCC-NEXT:    [[ADD:%.*]] = add i64 42, [[CONV]]
-; CGSCC-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 [[ADD]]) #[[ATTR2]], !range [[RNG0:![0-9]+]]
+; CGSCC-NEXT:    [[CALL2:%.*]] = call range(i64 -2147483606, 2147483690) i64 @fn1(i64 [[ADD]]) #[[ATTR2]]
 ; CGSCC-NEXT:    ret i64 [[CALL2]]
 ;
 entry:
@@ -91,13 +91,11 @@ entry:
   ret i64 %cond
 }
 ;.
+; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+;.
 ; CGSCC: attributes #[[ATTR0]] = { mustprogress nofree nosync nounwind willreturn memory(none) }
 ; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
 ; CGSCC: attributes #[[ATTR2]] = { nofree nosync willreturn }
 ;.
-; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
-;.
-; CGSCC: [[RNG0]] = !{i64 -2147483606, i64 2147483690}
-;.
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index 07e2d5ea15752..5bff2a2e6b208 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -207,7 +207,6 @@ define void @f7_1(ptr %ptr, i1 %cnd) {
 ; CHECK-LABEL: define {{[^@]+}}@f7_1
 ; CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], i1 noundef [[CND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    [[A:%.*]] = tail call i32 @unkown_f(ptr noundef nonnull align 4 dereferenceable(4) [[PTR]]) #[[ATTR1]]
-; CHECK-NEXT:    [[PTR_0:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[B:%.*]] = tail call i32 @unkown_f(ptr noundef nonnull align 4 dereferenceable(4) [[PTR]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[CND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 1b27fc5f16f56..38f8a829cf419 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -19,7 +19,7 @@ define i32 @test0-range-check(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test0-range-check
 ; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[A:%.*]] = tail call i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3:[0-9]+]], !range [[RNG0]]
+; TUNIT-NEXT:    [[A:%.*]] = tail call range(i32 0, 10) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[A]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
@@ -32,6 +32,40 @@ define i32 @test0-range-check(ptr %p) {
   ret i32 %a
 }
 
+define i32 @test0-range-check-smaller-current-range-attr(ptr %p) {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
+; TUNIT-LABEL: define {{[^@]+}}@test0-range-check-smaller-current-range-attr
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[A:%.*]] = tail call range(i32 2, 5) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
+; TUNIT-NEXT:    ret i32 [[A]]
+;
+; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
+; CGSCC-LABEL: define {{[^@]+}}@test0-range-check-smaller-current-range-attr
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-NEXT:    [[A:%.*]] = tail call range(i32 2, 5) i32 @test0(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-NEXT:    ret i32 [[A]]
+;
+  %a = tail call range(i32 2, 5)  i32 @test0(ptr %p)
+  ret i32 %a
+}
+
+define i32 @test0-range-check-larger-current-range-attr(ptr %p) {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
+; TUNIT-LABEL: define {{[^@]+}}@test0-range-check-larger-current-range-attr
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[A:%.*]] = tail call range(i32 0, 10) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
+; TUNIT-NEXT:    ret i32 [[A]]
+;
+; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
+; CGSCC-LABEL: define {{[^@]+}}@test0-range-check-larger-current-range-attr
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-NEXT:    [[A:%.*]] = tail call range(i32 0, 100) i32 @test0(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-NEXT:    ret i32 [[A]]
+;
+  %a = tail call range(i32 0, 100) i32 @test0(ptr %p)
+  ret i32 %a
+}
+
 declare void @use3-dummy(i1, i1, i1)
 define void @use3(i1, i1, i1) {
 ; CHECK-LABEL: define {{[^@]+}}@use3
@@ -48,7 +82,7 @@ define void @test0-icmp-check(ptr %p){
   ; ret = [0, 10)
 ; TUNIT-LABEL: define {{[^@]+}}@test0-icmp-check
 ; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) {
-; TUNIT-NEXT:    [[RET:%.*]] = tail call i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]], !range [[RNG0]]
+; TUNIT-NEXT:    [[RET:%.*]] = tail call range(i32 0, 10) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
 ; TUNIT-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; TUNIT-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
@@ -284,7 +318,7 @@ define i1 @test1-check(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test1-check
 ; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[RES:%.*]] = tail call i32 @test1(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]], !range [[RNG2:![0-9]+]]
+; TUNIT-NEXT:    [[RES:%.*]] = tail call range(i32 200, 1091) i32 @test1(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; TUNIT-NEXT:    ret i1 [[CMP]]
 ;
@@ -624,7 +658,7 @@ define dso_local i32 @test4-g2(i32 %u) {
 ; TUNIT-LABEL: define {{[^@]+}}@test4-g2
 ; TUNIT-SAME: (i32 [[U:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]]) #[[ATTR4]], !range [[RNG3:![0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = tail call range(i32 1, -2147483648) i32 @test4-f2(i32 [[U]]) #[[ATTR4]]
 ; TUNIT-NEXT:    ret i32 [[CALL]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -1760,6 +1794,7 @@ declare void @barney(i32 signext, i32 signext)
 
 !0 = !{i32 0, i32 10}
 !1 = !{i32 10, i32 100}
+!2 = !{i32 2, i32 5}
 ;.
 ; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) }
 ; TUNIT: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
@@ -1778,8 +1813,6 @@ declare void @barney(i32 signext, i32 signext)
 ;.
 ; TUNIT: [[RNG0]] = !{i32 0, i32 10}
 ; TUNIT: [[RNG1]] = !{i32 10, i32 100}
-; TUNIT: [[RNG2]] = !{i32 200, i32 1091}
-; TUNIT: [[RNG3]] = !{i32 1, i32 -2147483648}
 ;.
 ; CGSCC: [[RNG0]] = !{i32 0, i32 10}
 ; CGSCC: [[RNG1]] = !{i32 10, i32 100}
diff --git a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
index 374d5ba7ff52b..75ab54059f302 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
@@ -342,14 +342,6 @@ define %S.2 @t3.helper() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[S_2:%.*]], align 8
 ; CHECK-NEXT:    call void @ext1(ptr noundef nonnull align 8 dereferenceable(24) [[RETVAL]])
-; CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load ptr, ptr [[RETVAL]], align 8
-; CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [[S_2]] poison, ptr [[DOTFCA_0_LOAD]], 0
-; CHECK-NEXT:    [[DOTFCA_1_GEP:%.*]] = getelementptr inbounds [[S_2]], ptr [[RETVAL]], i32 0, i32 1
-; CHECK-NEXT:    [[DOTFCA_1_LOAD:%.*]] = load i64, ptr [[DOTFCA_1_GEP]], align 8
-; CHECK-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [[S_2]] [[DOTFCA_0_INSERT]], i64 [[DOTFCA_1_LOAD]], 1
-; CHECK-NEXT:    [[DOTFCA_2_GEP:%.*]] = getelementptr inbounds [[S_2]], ptr [[RETVAL]], i32 0, i32 2
-; CHECK-NEXT:    [[DOTFCA_2_LOAD:%.*]] = load i64, ptr [[DOTFCA_2_GEP]], align 8
-; CHECK-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [[S_2]] [[DOTFCA_1_INSERT]], i64 [[DOTFCA_2_LOAD]], 2
 ; CHECK-NEXT:    ret [[S_2]] zeroinitializer
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index 87ad0e5be0231..beceab7ce9ed7 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -1114,7 +1114,7 @@ define i32 @test(i1 %c) {
 ; TUNIT-LABEL: define {{[^@]+}}@test
 ; TUNIT-SAME: (i1 [[C:%.*]]) {
 ; TUNIT-NEXT:    [[R1:%.*]] = call i32 @ctx_test1(i1 noundef [[C]])
-; TUNIT-NEXT:    [[R2:%.*]] = call i32 @ctx_test2(i1 noundef [[C]]), !range [[RNG0:![0-9]+]]
+; TUNIT-NEXT:    [[R2:%.*]] = call range(i32 0, -2147483648) i32 @ctx_test2(i1 noundef [[C]])
 ; TUNIT-NEXT:    [[ADD:%.*]] = add i32 [[R1]], [[R2]]
 ; TUNIT-NEXT:    ret i32 [[ADD]]
 ;
@@ -1689,8 +1689,6 @@ define i32 @readExtInitZeroInit() {
 ; TUNIT: attributes #[[ATTR15]] = { nosync nounwind memory(read) }
 ; TUNIT: attributes #[[ATTR16]] = { nounwind memory(write) }
 ;.
-; TUNIT: [[RNG0]] = !{i32 0, i32 -2147483648}
-;.
 ; CGSCC: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn }
 ; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
 ; CGSCC: attributes #[[ATTR2]] = { memory(readwrite, argmem: none) }
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll
new file mode 100644
index 0000000000000..d0d87b38e0589
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -p 'require<profile-summary>,codegenprepare' -cgpp-huge-func=0 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+@globalptr = external global ptr
+declare ptr @get_ptr(i64)
+
+; Can't recreate invoke instruction
+
+define void @addr_from_invoke() personality ptr null {
+; CHECK-LABEL: define void @addr_from_invoke() personality ptr null {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[PTR:%.*]] = invoke ptr @get_ptr(i64 0)
+; CHECK-NEXT:            to label %[[BODY_1:.*]] unwind label %[[EHCLEANUP:.*]]
+; CHECK:       [[EHCLEANUP]]:
+; CHECK-NEXT:    [[PAD:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    cleanupret from [[PAD]] unwind to caller
+; CHECK:       [[BODY_1]]:
+; CHECK-NEXT:    [[GEP1:%.*]] = bitcast ptr [[PTR]] to ptr
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast ptr [[PTR]] to ptr
+; CHECK-NEXT:    [[UNUSED:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP0]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr = invoke ptr @get_ptr(i64 0) to label %body.1 unwind label %ehcleanup
+
+body.2:
+  %unused = load <4 x i32>, ptr %gep, align 4
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  ret void
+
+ehcleanup:
+  %pad = cleanuppad within none []
+  cleanupret from %pad unwind to caller
+
+body.1:
+  %gep = getelementptr { i32 }, ptr %ptr, i64 0, i32 0
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  br label %body.2
+}
+
+define void @addr_from_arg(ptr %ptr, i1 %p) {
+; CHECK-LABEL: define void @addr_from_arg(
+; CHECK-SAME: ptr [[PTR:%.*]], i1 [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[P]], label %[[BODY_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BODY_1]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast ptr [[PTR]] to ptr
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[UNUSED:%.*]] = load <4 x i32>, ptr [[PTR]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %p, label %body.1, label %exit
+
+body.2:
+  %unused = load <4 x i32>, ptr %gep, align 4
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  ret void
+
+exit:
+  ret void
+
+body.1:
+  %gep = getelementptr { i32 }, ptr %ptr, i64 0, i32 0
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  br label %body.2
+}
+
+define void @addr_from_global(i1 %p) {
+; CHECK-LABEL: define void @addr_from_global(
+; CHECK-SAME: i1 [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[P]], label %[[BODY_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BODY_1]]:
+; CHECK-NEXT:    [[GEP1:%.*]] = bitcast ptr @globalptr to ptr
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[UNUSED:%.*]] = load <4 x i32>, ptr @globalptr, align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @globalptr, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %p, label %body.1, label %exit
+
+body.2:
+  %unused = load <4 x i32>, ptr %gep, align 4
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  ret void
+
+exit:
+  ret void
+
+body.1:
+  %gep = getelementptr { i32 }, ptr @globalptr, i64 0, i32 0
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  br label %body.2
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
index 61107a9b70163..63fd1845594c6 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true -disable-cgp-delete-phis  %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO
 target datalayout =
@@ -6,7 +7,31 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; Can we sink for different base if there is no phi for base?
 define i32 @test1(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test1
+; CHECK-YES-LABEL: define i32 @test1(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test1(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -16,10 +41,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
   ret i32 %v
@@ -27,7 +48,18 @@ fallthrough:
 
 ; Can we sink for different base if there is phi for base?
 define i32 @test2(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test2
+; CHECK-LABEL: define i32 @test2(
+; CHECK-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[B:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -37,7 +69,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: getelementptr inbounds i8, {{.+}} 40
   %b = phi ptr [%b1, %entry], [%b2, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -46,7 +77,33 @@ fallthrough:
 
 ; Can we sink for different base if there is phi for base but not valid one?
 define i32 @test3(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test3
+; CHECK-YES-LABEL: define i32 @test3(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test3(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -56,11 +113,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %b = phi ptr [%b2, %entry], [%b1, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -69,7 +121,31 @@ fallthrough:
 
 ; Can we sink for different base if both addresses are in the same block?
 define i32 @test4(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test4
+; CHECK-YES-LABEL: define i32 @test4(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test4(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   %a2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -79,10 +155,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
   ret i32 %v
@@ -91,7 +163,18 @@ fallthrough:
 ; Can we sink for different base if there is phi for base?
 ; Both addresses are in the same block.
 define i32 @test5(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test5
+; CHECK-LABEL: define i32 @test5(
+; CHECK-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[B:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   %a2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -101,7 +184,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: getelementptr inbounds i8, {{.+}} 40
   %b = phi ptr [%b1, %entry], [%b2, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -111,7 +193,33 @@ fallthrough:
 ; Can we sink for different base if there is phi for base but not valid one?
 ; Both addresses are in the same block.
 define i32 @test6(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test6
+; CHECK-YES-LABEL: define i32 @test6(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test6(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   %a2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -121,11 +229,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %b = phi ptr [%b2, %entry], [%b1, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -134,14 +237,56 @@ fallthrough:
 
 ; case with a loop. No phi node.
 define i32 @test7(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test7
+; CHECK-YES-LABEL: define i32 @test7(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI1:%.*]] = phi ptr [ [[SUNK_PHI:%.*]], %[[FALLTHROUGH]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[SUNK_PHI1]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test7(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-NO-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
 
 loop:
-; CHECK-LABEL: loop:
-; CHECK-YES: sunk_phi
   %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
   %c3 = phi ptr [%a1, %entry], [%c, %fallthrough]
   br i1 %cond, label %if.then, label %fallthrough
@@ -151,10 +296,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %v = load volatile i32, ptr %c, align 4
   %iv.inc = add i32 %iv, 1
@@ -167,7 +308,30 @@ exit:
 
 ; case with a loop. There is phi node.
 define i32 @test8(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test8
+; CHECK-LABEL: define i32 @test8(
+; CHECK-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NEXT:    [[B3:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[B]] = phi ptr [ [[B3]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
@@ -183,7 +347,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: getelementptr inbounds i8, {{.+}} 40
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %b = phi ptr [%b3, %loop], [%b2, %if.then]
   %v = load volatile i32, ptr %c, align 4
@@ -197,14 +360,60 @@ exit:
 
 ; case with a loop. There is phi node but it does not fit.
 define i32 @test9(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test9
+; CHECK-YES-LABEL: define i32 @test9(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI1:%.*]] = phi ptr [ [[SUNK_PHI:%.*]], %[[FALLTHROUGH]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-YES-NEXT:    [[B3:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[FALLTHROUGH]] ]
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[SUNK_PHI1]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[B:%.*]] = phi ptr [ [[B3]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test9(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-NO-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NO-NEXT:    [[B3:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[FALLTHROUGH]] ]
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[B:%.*]] = phi ptr [ [[B3]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
 
 loop:
-; CHECK-LABEL: loop:
-; CHECK-YES: sunk_phi
   %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
   %c3 = phi ptr [%a1, %entry], [%c, %fallthrough]
   %b3 = phi ptr [%b1, %entry], [%b2, %fallthrough]
@@ -215,11 +424,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %b = phi ptr [%b3, %loop], [%b2, %if.then]
   %v = load volatile i32, ptr %c, align 4
@@ -233,7 +437,45 @@ exit:
 
 ; Case through a loop. No phi node.
 define i32 @test10(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test10
+; CHECK-YES-LABEL: define i32 @test10(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[FALLTHROUGH]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test10(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[FALLTHROUGH]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -243,10 +485,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: br
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   br label %loop
 
@@ -257,14 +495,31 @@ loop:
   br i1 %cmp, label %loop, label %exit
 
 exit:
-; CHECK-YES: sunkaddr
   %v = load volatile i32, ptr %c, align 4
   ret i32 %v
 }
 
 ; Case through a loop. There is a phi.
 define i32 @test11(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test11
+; CHECK-LABEL: define i32 @test11(
+; CHECK-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[B:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[FALLTHROUGH]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -274,9 +529,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: phi
-; CHECK: phi
-; CHECK: br
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %b = phi ptr [%b1, %entry], [%b2, %if.then]
   br label %loop
@@ -288,25 +540,80 @@ loop:
   br i1 %cmp, label %loop, label %exit
 
 exit:
-; CHECK: sunkaddr
   %v = load volatile i32, ptr %c, align 4
   ret i32 %v
 }
 
 ; Complex case with address value from previous iteration.
 define i32 @test12(i32 %N, i1 %cond, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test12
+; CHECK-YES-LABEL: define i32 @test12(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI1:%.*]] = phi ptr [ [[SUNK_PHI:%.*]], %[[BACKEDGE]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[BACKEDGE]] ]
+; CHECK-YES-NEXT:    [[B4:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B5:%.*]], %[[BACKEDGE]] ]
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[SUNK_PHI1]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[B6:%.*]] = phi ptr [ [[B4]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[A4:%.*]] = getelementptr inbounds i64, ptr [[B4]], i64 5
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 20
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[BACKEDGE]], label %[[IF_THEN_2:.*]]
+; CHECK-YES:       [[IF_THEN_2]]:
+; CHECK-YES-NEXT:    br label %[[BACKEDGE]]
+; CHECK-YES:       [[BACKEDGE]]:
+; CHECK-YES-NEXT:    [[B5]] = phi ptr [ [[B4]], %[[FALLTHROUGH]] ], [ [[B6]], %[[IF_THEN_2]] ]
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test12(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NO-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[BACKEDGE]] ]
+; CHECK-NO-NEXT:    [[B4:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B5:%.*]], %[[BACKEDGE]] ]
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[B6:%.*]] = phi ptr [ [[B4]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[A4:%.*]] = getelementptr inbounds i64, ptr [[B4]], i64 5
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 20
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[BACKEDGE]], label %[[IF_THEN_2:.*]]
+; CHECK-NO:       [[IF_THEN_2]]:
+; CHECK-NO-NEXT:    br label %[[BACKEDGE]]
+; CHECK-NO:       [[BACKEDGE]]:
+; CHECK-NO-NEXT:    [[B5]] = phi ptr [ [[B4]], %[[FALLTHROUGH]] ], [ [[B6]], %[[IF_THEN_2]] ]
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
 
 loop:
-; CHECK-LABEL: loop:
-; CHECK-YES: sunk_phi
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: br
   %iv = phi i32 [0, %entry], [%iv.inc, %backedge]
   %c3 = phi ptr [%a1, %entry], [%c, %backedge]
   %b4 = phi ptr [%b1, %entry], [%b5, %backedge]
@@ -317,11 +624,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %b6 = phi ptr [%b4, %loop], [%b2, %if.then]
   %v = load volatile i32, ptr %c, align 4
@@ -345,7 +647,35 @@ exit:
 %struct.S = type {i32, i32}
 ; Case with index
 define i32 @test13(i1 %cond, ptr %b1, ptr %b2, i64 %Index) {
-; CHECK-LABEL: @test13
+; CHECK-YES-LABEL: define i32 @test13(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], i64 [[INDEX:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[I2:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-YES-NEXT:    [[SUNKADDR1:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 [[SUNKADDR]]
+; CHECK-YES-NEXT:    [[SUNKADDR2:%.*]] = getelementptr inbounds i8, ptr [[SUNKADDR1]], i64 4
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR2]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test13(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], i64 [[INDEX:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[B1]], i64 [[INDEX]], i32 1
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[I2:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[B2]], i64 [[INDEX]], i32 1
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[A:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds %struct.S, ptr %b1, i64 %Index, i32 1
   br i1 %cond, label %if.then, label %fallthrough
@@ -356,10 +686,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %a = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %a, align 4
   ret i32 %v
@@ -367,22 +693,55 @@ fallthrough:
 
 ; Select of Select case.
 define i64 @test14(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test14
+; CHECK-LABEL: define i64 @test14(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[S12:%.*]] = select i1 [[C1]], ptr [[B1]], ptr [[B2]]
+; CHECK-NEXT:    [[S21:%.*]] = select i1 [[C2]], ptr [[S12]], ptr [[B3]]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[S21]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-NEXT:    ret i64 [[V]]
+;
 entry:
-; CHECK-LABEL: entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
   %g3 = getelementptr inbounds i64, ptr %b3, i64 5
   %s1 = select i1 %c1, ptr %g1, ptr %g2
   %s2 = select i1 %c2, ptr %s1, ptr %g3
-; CHECK: sunkaddr
   %v = load i64 , ptr %s2, align 8
   ret i64 %v
 }
 
 ; Select of Phi case.
 define i64 @test15(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test15
+; CHECK-YES-LABEL: define i64 @test15(
+; CHECK-YES-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[S11:%.*]] = select i1 [[C2]], ptr [[SUNK_PHI]], ptr [[B3]]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[S11]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-YES-NEXT:    ret i64 [[V]]
+;
+; CHECK-NO-LABEL: define i64 @test15(
+; CHECK-NO-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[G2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    [[G3:%.*]] = getelementptr inbounds i64, ptr [[B3]], i64 5
+; CHECK-NO-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[P1:%.*]] = phi ptr [ [[G1]], %[[ENTRY]] ], [ [[G2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[S1:%.*]] = select i1 [[C2]], ptr [[P1]], ptr [[G3]]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i64, ptr [[S1]], align 8
+; CHECK-NO-NEXT:    ret i64 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -393,20 +752,27 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
   %p1 = phi ptr [%g1, %entry], [%g2, %if.then]
   %s1 = select i1 %c2, ptr %p1, ptr %g3
-; CHECK-YES: sunkaddr
-; CHECK-NO: phi
-; CHECK-NO-NEXT: select
-; CHECK-NO-NEXT: load
   %v = load i64 , ptr %s1, align 8
   ret i64 %v
 }
 
 ; Select of Phi case. Phi exists
 define i64 @test16(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test16
+; CHECK-LABEL: define i64 @test16(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[S11:%.*]] = select i1 [[C2]], ptr [[P]], ptr [[B3]]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[S11]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-NEXT:    ret i64 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -417,18 +783,43 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
   %p = phi ptr [%b1, %entry], [%b2, %if.then]
   %p1 = phi ptr [%g1, %entry], [%g2, %if.then]
   %s1 = select i1 %c2, ptr %p1, ptr %g3
-; CHECK: sunkaddr
   %v = load i64 , ptr %s1, align 8
   ret i64 %v
 }
 
 ; Phi of Select case.
 define i64 @test17(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test17
+; CHECK-YES-LABEL: define i64 @test17(
+; CHECK-YES-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[S11:%.*]] = select i1 [[C2]], ptr [[B1]], ptr [[B2]]
+; CHECK-YES-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B3]], %[[IF_THEN]] ], [ [[S11]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-YES-NEXT:    ret i64 [[V]]
+;
+; CHECK-NO-LABEL: define i64 @test17(
+; CHECK-NO-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[G2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    [[G3:%.*]] = getelementptr inbounds i64, ptr [[B3]], i64 5
+; CHECK-NO-NEXT:    [[S1:%.*]] = select i1 [[C2]], ptr [[G1]], ptr [[G2]]
+; CHECK-NO-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[P1:%.*]] = phi ptr [ [[S1]], %[[ENTRY]] ], [ [[G3]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i64, ptr [[P1]], align 8
+; CHECK-NO-NEXT:    ret i64 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -440,18 +831,48 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
   %p1 = phi ptr [%s1, %entry], [%g3, %if.then]
-; CHECK-YES: sunkaddr
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %v = load i64 , ptr %p1, align 8
   ret i64 %v
 }
 
 ; The same two addr modes by different paths
 define i32 @test18(i1 %cond1, i1 %cond2, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test18
+; CHECK-YES-LABEL: define i32 @test18(
+; CHECK-YES-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*:]]
+; CHECK-YES-NEXT:    br i1 [[COND1]], label %[[IF_THEN1:.*]], label %[[IF_THEN2:.*]]
+; CHECK-YES:       [[IF_THEN1]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN2]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN2]] ], [ [[B1]], %[[IF_THEN1]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V1:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[G1_1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    [[V2:%.*]] = load i32, ptr [[G1_1]], align 4
+; CHECK-YES-NEXT:    [[V:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test18(
+; CHECK-NO-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*:]]
+; CHECK-NO-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND1]], label %[[IF_THEN1:.*]], label %[[IF_THEN2:.*]]
+; CHECK-NO:       [[IF_THEN1]]:
+; CHECK-NO-NEXT:    [[G2:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN2]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[G2]], %[[IF_THEN1]] ], [ [[G1]], %[[IF_THEN2]] ]
+; CHECK-NO-NEXT:    [[V1:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[G1_1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    [[V2:%.*]] = load i32, ptr [[G1_1]], align 4
+; CHECK-NO-NEXT:    [[V:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b2, i64 5
   br i1 %cond1, label %if.then1, label %if.then2
@@ -467,10 +888,6 @@ if.then3:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%g2, %if.then1], [%g1, %if.then2], [%g1, %if.then3]
   %v1 = load i32, ptr %c, align 4
   %g1_1 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -481,7 +898,31 @@ fallthrough:
 
 ; Different types but null is the first?
 define i32 @test19(i1 %cond1, i1 %cond2, ptr %b2, ptr %b1) {
-; CHECK-LABEL: @test19
+; CHECK-LABEL: define i32 @test19(
+; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], ptr [[B2:%.*]], ptr [[B1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NEXT:    br i1 [[COND1]], label %[[IF_THEN1:.*]], label %[[IF_THEN2:.*]]
+; CHECK:       [[IF_THEN1]]:
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i8, ptr [[B1]], i64 40
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast ptr [[G2]] to ptr
+; CHECK-NEXT:    br label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN2]]:
+; CHECK-NEXT:    [[BC1_1:%.*]] = bitcast ptr [[G1]] to ptr
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FALLTHROUGH]], label %[[IF_THEN3:.*]]
+; CHECK:       [[IF_THEN3]]:
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i64, ptr null, i64 5
+; CHECK-NEXT:    [[BC1_2:%.*]] = bitcast ptr [[G3]] to ptr
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[C:%.*]] = phi ptr [ [[BC2]], %[[IF_THEN1]] ], [ [[BC1_1]], %[[IF_THEN2]] ], [ [[BC1_2]], %[[IF_THEN3]] ]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NEXT:    [[G1_1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NEXT:    [[BC1_1_1:%.*]] = bitcast ptr [[G1_1]] to ptr
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[BC1_1_1]], align 4
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b2, i64 5
   %bc1 = bitcast ptr %g1 to ptr
@@ -502,7 +943,6 @@ if.then3:
   br label %fallthrough
 
 fallthrough:
-; CHECK-NOT: sunk_phi
   %c = phi ptr [%bc2, %if.then1], [%bc1_1, %if.then2], [%bc1_2, %if.then3]
   %v1 = load i32, ptr %c, align 4
   %g1_1 = getelementptr inbounds i64, ptr %b2, i64 5
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
index 6a6b029b67b51..2945a007bf578 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
@@ -1,9 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -disable-complex-addr-modes=false -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr null, i64 undef
+; CHECK-NEXT:    br label %[[START:.*]]
+; CHECK:       [[START]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = phi ptr [ [[TMP0]], %[[ENTRY]] ], [ [[VAL1]], %[[START]] ]
+; CHECK-NEXT:    [[VAL2:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[VAL2]], %[[START]] ]
+; CHECK-NEXT:    [[SUNKADDR2:%.*]] = bitcast ptr [[VAL2]] to ptr
+; CHECK-NEXT:    [[LOADX:%.*]] = load i64, ptr [[SUNKADDR2]], align 8
+; CHECK-NEXT:    br label %[[START]]
+;
 entry:
   %0 = getelementptr inbounds i64, ptr null, i64 undef
   br label %start
@@ -18,7 +30,6 @@ slowpath:
   br label %exit
 
 exit:
-; CHECK: sunkaddr
   %val3 = phi ptr [ undef, %slowpath ], [ %val2, %start ]
   %val4 = phi ptr [ %elem1, %slowpath ], [ %val1, %start ]
   %val5 = phi ptr [ undef, %slowpath ], [ %val2, %start ]
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
index 5f7701c357ec3..0934393a667ee 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
@@ -1,4 +1,4 @@
-; Tests whether resume function would remain dbg.value infomation.
+; Tests whether resume function retains dbg.value information.
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 ;
 ; This file is based on coro-debug-frame-variable.ll.
diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll
index a220073248ba3..5f8e9c9c1d16d 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -19,7 +19,6 @@ entry:
   %3 = call i8 @llvm.coro.suspend(token none, i1 false), !dbg !17
   %conv = sext i8 %3 to i32, !dbg !17
   %late_local = alloca i32, align 4
-  call void @coro.devirt.trigger(ptr null)
   switch i32 %conv, label %sw.default [
     i32 0, label %sw.bb
     i32 1, label %sw.bb1
@@ -113,12 +112,6 @@ declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 ; Function Attrs: nounwind
 declare i1 @llvm.coro.end(ptr, i1, token) #5
 
-; Function Attrs: alwaysinline
-define private void @coro.devirt.trigger(ptr) #6 {
-entry:
-  ret void
-}
-
 ; Function Attrs: argmemonly nounwind readonly
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #2
 
@@ -170,15 +163,20 @@ attributes #7 = { noduplicate }
 !31 = !DILocalVariable(name: "allocated", scope: !6, file: !7, line: 55, type: !11)
 !32 = !DILocalVariable(name: "inline_asm", scope: !6, file: !7, line: 55, type: !11)
 
+; Check that the original function is visible and capture its debug info id.
 ; CHECK: define ptr @flink(i32 %x) #0 personality i32 0 !dbg ![[ORIG:[0-9]+]]
+
+; Check that the resume function is present and capture its debug info id.
+; Also check that it contains `#dbg_declare` and `#dbg_value` debug instructions
+; making the debug variables available to the debugger.
+;
 ; CHECK: define internal fastcc void @flink.resume(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[RESUME:[0-9]+]]
 ; CHECK: entry.resume:
 ; CHECK: %[[DBG_PTR:.*]] = alloca ptr
-; CHECK: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_COROHDL:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst,
-; CHECK: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_X:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst, [[EXPR_TAIL:.*]])
-; CHECK: store ptr {{.*}}, ptr %[[DBG_PTR]]
+; CHECK-NEXT: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_COROHDL:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst,
+; CHECK-NEXT: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_X:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst, [[EXPR_TAIL:.*]])
+; CHECK-NEXT: store ptr {{.*}}, ptr %[[DBG_PTR]]
 ; CHECK-NOT: alloca ptr
-; CHECK: call void @coro.devirt.trigger(ptr null)
 ; CHECK: #dbg_value(i8 0, ![[RESUME_CONST:[0-9]+]], !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed),
 ; CHECK: #dbg_value(ptr %[[DBG_PTR]], ![[RESUME_DIRECT:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst, {{[0-9]+}}, DW_OP_deref),
 ; Note that keeping the undef value here could be acceptable, too.
@@ -194,18 +192,26 @@ attributes #7 = { noduplicate }
 ; CHECK: [[DEFAULT_DEST]]:
 ; CHECK-NOT: {{.*}}:
 ; CHECK: #dbg_value(i32 %[[CALLBR_RES]]
+
+; Check that the destroy and cleanup functions are present and capture their debug info id.
+;
 ; CHECK: define internal fastcc void @flink.destroy(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[DESTROY:[0-9]+]]
 ; CHECK: define internal fastcc void @flink.cleanup(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[CLEANUP:[0-9]+]]
 
+; Check that the linkage name of the original function is set correctly.
+;
 ; CHECK: ![[ORIG]] = distinct !DISubprogram(name: "f", linkageName: "flink"
-
 ; CHECK: ![[RESUME]] = distinct !DISubprogram(name: "f", linkageName: "flink.resume"
+
+; Check that metadata for local variables in the resume function is set correctly.
+;
 ; CHECK: ![[RESUME_COROHDL]] = !DILocalVariable(name: "coro_hdl", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_X]] = !DILocalVariable(name: "x", arg: 1, scope: ![[RESUME]]
 ; CHECK: ![[RESUME_CONST]] = !DILocalVariable(name: "direct_const", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_DIRECT]] = !DILocalVariable(name: "direct_mem", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_DIRECT_VALUE]] = !DILocalVariable(name: "direct_value", scope: ![[RESUME]]
 
+; Check that the linkage names are set correctly for the destroy and cleanup functions.
+;
 ; CHECK: ![[DESTROY]] = distinct !DISubprogram(name: "f", linkageName: "flink.destroy"
-
 ; CHECK: ![[CLEANUP]] = distinct !DISubprogram(name: "f", linkageName: "flink.cleanup"
diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll
new file mode 100644
index 0000000000000..63f59a19e1c93
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll
@@ -0,0 +1,143 @@
+; Tests that we add DILabels for the suspend points.
+;
+; We check both the generated LLVM:
+; RUN: opt < %s -passes='cgscc(coro-split)' -S | FileCheck %s
+;
+; And the debug info:
+; REQUIRES: object-emission
+; RUN: opt < %s -passes='cgscc(coro-split),coro-cleanup' \
+; RUN:   | %llc_dwarf -O0 -filetype=obj -o - \
+; RUN:   | llvm-dwarfdump - \
+; RUN:   | FileCheck %s -check-prefix=DWARF
+
+source_filename = "coro.c"
+
+declare void @bar(...) local_unnamed_addr #0
+declare void @baz(...) local_unnamed_addr #0
+
+; Roughly equivalent to:
+;
+; task f() {
+;   while (true) {
+;     bar();
+;     co_await std::suspend_always{};
+;     baz();
+;     co_await std::suspend_always{};
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define ptr @f() #2 !dbg !16 {
+entry:
+  %0 = tail call token @llvm.coro.id(i32 0, ptr null, ptr @f, ptr null), !dbg !26
+  %1 = tail call i64 @llvm.coro.size.i64(), !dbg !26
+  %frame = tail call ptr @malloc(i64 %1), !dbg !26
+  %2 = tail call ptr @llvm.coro.begin(token %0, ptr %frame) #3, !dbg !26
+  br label %loop1, !dbg !27
+
+loop1:                                         ; preds = %for.cond, %entry
+  tail call void (...) @bar() #0, !dbg !33
+  %3 = tail call token @llvm.coro.save(ptr null), !dbg !34
+  %4 = tail call i8 @llvm.coro.suspend(token %3, i1 false), !dbg !34
+  switch i8 %4, label %coro_Suspend [
+    i8 0, label %loop2
+    i8 1, label %coro_Cleanup
+  ], !dbg !34
+
+loop2:                                         ; preds = %for.cond, %entry
+  tail call void (...) @baz() #0, !dbg !35
+  %5 = tail call token @llvm.coro.save(ptr null), !dbg !36
+  %6 = tail call i8 @llvm.coro.suspend(token %5, i1 false), !dbg !36
+  switch i8 %6, label %coro_Suspend [
+    i8 0, label %loop1
+    i8 1, label %coro_Cleanup
+  ], !dbg !36
+
+coro_Cleanup:                                     ; preds = %for.cond
+  %7 = tail call ptr @llvm.coro.free(token %0, ptr %2), !dbg !37
+  tail call void @free(ptr nonnull %7), !dbg !37
+  br label %coro_Suspend, !dbg !37
+
+coro_Suspend:                                     ; preds = %for.cond, %if.then, %coro_Cleanup
+  tail call i1 @llvm.coro.end(ptr null, i1 false, token none) #3, !dbg !40
+  ret ptr %2, !dbg !41
+}
+
+; Check that the resume function contains the `#dbg_label` instructions.
+; CHECK-LABEL:   define ptr @f() #1 !dbg !6 {
+; CHECK:         resume.0:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[RESUME_0:[0-9]+]], !{{[0-9]+}})
+; CHECK:         resume.1:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[RESUME_1:[0-9]+]], !{{[0-9]+}})
+
+; Check that the destroy function contains the `#dbg_label` instructions.
+; CHECK-LABEL:   define internal fastcc void @f.destroy({{.*}}) #1 !dbg !38 {
+; CHECK:         resume.0:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[DESTROY_0:[0-9]+]], !{{[0-9]+}})
+; CHECK:         resume.1:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[DESTROY_1:[0-9]+]], !{{[0-9]+}})
+
+; Check that the DILabels are correct.
+; CHECK: ![[RESUME_0]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_0", file: !{{[0-9]*}}, line: 12, column: 6, isArtificial: true, coroSuspendIdx: 0)
+; CHECK: ![[RESUME_1]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_1", file: !{{[0-9]*}}, line: 14, column: 6, isArtificial: true, coroSuspendIdx: 1)
+; CHECK: ![[DESTROY_0]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_0", file: !{{[0-9]*}}, line: 12, column: 6, isArtificial: true, coroSuspendIdx: 0)
+; CHECK: ![[DESTROY_1]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_1", file: !{{[0-9]*}}, line: 14, column: 6, isArtificial: true, coroSuspendIdx: 1)
+
+; DWARF:        {{.*}}DW_TAG_label
+; DWARF-NEXT:    DW_AT_name ("__coro_resume_0")
+; DWARF-NEXT:    DW_AT_decl_file
+; DWARF-NEXT:    DW_AT_decl_line    (12)
+; DWARF-NEXT:    DW_AT_decl_column  (6)
+; DWARF-NEXT:    DW_AT_artificial   (true)
+; DWARF-NEXT:    DW_AT_LLVM_coro_suspend_idx (0x00)
+; DWARF-NEXT:    DW_AT_low_pc
+
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #4
+
+; Function Attrs: nounwind
+declare noalias ptr @malloc(i64) local_unnamed_addr #0
+declare i64 @llvm.coro.size.i64() #1
+declare ptr @llvm.coro.begin(token, ptr writeonly) #0
+declare token @llvm.coro.save(ptr) #0
+declare i8 @llvm.coro.suspend(token, i1) #0
+declare ptr @llvm.coro.free(token, ptr nocapture readonly) #4
+declare void @free(ptr nocapture) local_unnamed_addr #0
+declare i1 @llvm.coro.end(ptr, i1, token) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind uwtable presplitcoroutine }
+attributes #3 = { noduplicate }
+attributes #4 = { argmemonly nounwind readonly }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 - manually edited", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "coro.c", directory: "/home/gor/build/bin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 4.0.0 - manually edited"}
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!16 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 8, type: !17, isLocal: false, isDefinition: true, scopeLine: 8, isOptimized: true, unit: !0, retainedNodes: !20)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!19}
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64, align: 64)
+!20 = !{!21, !22, !24}
+!21 = !DILocalVariable(name: "coro_hdl", scope: !16, file: !1, line: 9, type: !19)
+!22 = !DILocalVariable(name: "i", scope: !23, file: !1, line: 11, type: !9)
+!23 = distinct !DILexicalBlock(scope: !16, file: !1, line: 11, column: 3)
+!24 = !DILocalVariable(name: "coro_mem", scope: !16, file: !1, line: 16, type: !19)
+!26 = !DILocation(line: 9, column: 3, scope: !16)
+!27 = !DILocation(line: 10, column: 8, scope: !23)
+!33 = !DILocation(line: 11, column: 6, scope: !23)
+!34 = !DILocation(line: 12, column: 6, scope: !23)
+!35 = !DILocation(line: 13, column: 6, scope: !23)
+!36 = !DILocation(line: 14, column: 6, scope: !23)
+!37 = !DILocation(line: 16, column: 3, scope: !16)
+!40 = !DILocation(line: 16, column: 3, scope: !16)
+!41 = !DILocation(line: 17, column: 1, scope: !16)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-no-lieftime.ll b/llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
similarity index 100%
rename from llvm/test/Transforms/Coroutines/coro-split-no-lieftime.ll
rename to llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index f8e594b5626a0..6c04e15edc374 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -425,7 +425,7 @@ define ptr @test25(ptr %p) nounwind {
 ; CHECK-NEXT:    [[P_4:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP:%.*]] = load i8, ptr [[P_4]], align 1
 ; CHECK-NEXT:    store i8 0, ptr [[P_4]], align 1
-; CHECK-NEXT:    [[Q:%.*]] = call ptr @strdup(ptr [[P]]) #[[ATTR13:[0-9]+]]
+; CHECK-NEXT:    [[Q:%.*]] = call ptr @strdup(ptr [[P]]) #[[ATTR14:[0-9]+]]
 ; CHECK-NEXT:    store i8 [[TMP]], ptr [[P_4]], align 1
 ; CHECK-NEXT:    ret ptr [[Q]]
 ;
@@ -855,3 +855,36 @@ bb:
   store ptr null, ptr null, align 8
   ret void
 }
+
+define void @test_dead_on_return(ptr dead_on_return %p) {
+; CHECK-LABEL: @test_dead_on_return(
+; CHECK-NEXT:    ret void
+;
+  store i8 0, ptr %p
+  ret void
+}
+
+define void @test_dead_on_return_maythrow(ptr dead_on_return %p) {
+; CHECK-LABEL: @test_dead_on_return_maythrow(
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    ret void
+;
+  store i8 0, ptr %p
+  call void @maythrow()
+  ret void
+}
+
+define ptr @test_dead_on_return_ptr_returned(ptr dead_on_return %p) {
+; CHECK-LABEL: @test_dead_on_return_ptr_returned(
+; CHECK-NEXT:    [[LOCAL_VAR:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    call void @opaque(ptr [[LOCAL_VAR]])
+; CHECK-NEXT:    ret ptr [[P:%.*]]
+;
+  %local.var = alloca ptr
+  call void @opaque(ptr %local.var)
+  store ptr %local.var, ptr %p
+  ret ptr %p
+}
+
+declare void @opaque(ptr)
+declare void @maythrow() memory(none)
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
index 94b17510bb95d..826da89290691 100644
--- a/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -passes=early-cse -earlycse-debug-hash | FileCheck %s
-; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -aa-pipeline=basic-aa -passes='early-cse<memssa>' | FileCheck %s
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -aa-pipeline=basic-aa -passes='early-cse<memssa>' -verify-analysis-invalidation | FileCheck %s
 
 define <4 x i32> @test_cse(ptr %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
 ; CHECK-LABEL: define <4 x i32> @test_cse(
@@ -324,6 +324,22 @@ for.end:                                          ; preds = %for.cond
   ret <4 x i32> %res.0
 }
 
+define void @test_ld4_st4_no_cse(ptr %p, <16 x i8> %A, <16 x i8> %B) {
+; CHECK-LABEL: define void @test_ld4_st4_no_cse(
+; CHECK-SAME: ptr [[P:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[LD:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4.v16i8.p0(ptr [[P]])
+; CHECK-NEXT:    [[EXT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[LD]], 0
+; CHECK-NEXT:    tail call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[EXT]], <16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> zeroinitializer, ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ld = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4.v16i8.p0(ptr %p)
+  %ext = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %ld, 0
+  tail call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> %ext, <16 x i8> %A, <16 x i8> %B, <16 x i8> zeroinitializer, ptr %p)
+  ret void
+}
+
 ; Function Attrs: nounwind
 declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr nocapture)
 
diff --git a/llvm/test/Transforms/EarlyCSE/writeonly.ll b/llvm/test/Transforms/EarlyCSE/writeonly.ll
index 0bfffa3c825a3..c09b913f9ff2b 100644
--- a/llvm/test/Transforms/EarlyCSE/writeonly.ll
+++ b/llvm/test/Transforms/EarlyCSE/writeonly.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=early-cse -earlycse-debug-hash < %s | FileCheck %s
+; RUN: opt -S -passes=early-cse -earlycse-debug-hash < %s | FileCheck %s --check-prefixes=CHECK,NO-MSSA
+; RUN: opt -S -passes='early-cse<memssa>' < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 @var = global i32 undef
 declare void @foo() nounwind
@@ -15,3 +16,148 @@ define void @test() {
   store i32 2, ptr @var
   ret void
 }
+
+declare void @writeonly_void() memory(write)
+
+; Can CSE writeonly calls, including non-nounwind/willreturn.
+define void @writeonly_cse() {
+; CHECK-LABEL: @writeonly_cse(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    ret void
+;
+  call void @writeonly_void()
+  call void @writeonly_void()
+  ret void
+}
+
+; Can CSE, loads do not matter.
+define i32 @writeonly_cse_intervening_load(ptr %p) {
+; CHECK-LABEL: @writeonly_cse_intervening_load(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  call void @writeonly_void()
+  %v = load i32, ptr %p
+  call void @writeonly_void()
+  ret i32 %v
+}
+
+; Cannot CSE, the store may be to the same memory.
+define void @writeonly_cse_intervening_store(ptr %p) {
+; CHECK-LABEL: @writeonly_cse_intervening_store(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    ret void
+;
+  call void @writeonly_void()
+  store i32 0, ptr %p
+  call void @writeonly_void()
+  ret void
+}
+
+; Can CSE, the store does not alias the writeonly call.
+define void @writeonly_cse_intervening_noalias_store(ptr noalias %p) {
+; NO-MSSA-LABEL: @writeonly_cse_intervening_noalias_store(
+; NO-MSSA-NEXT:    call void @writeonly_void()
+; NO-MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; NO-MSSA-NEXT:    call void @writeonly_void()
+; NO-MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @writeonly_cse_intervening_noalias_store(
+; MSSA-NEXT:    call void @writeonly_void()
+; MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; MSSA-NEXT:    ret void
+;
+  call void @writeonly_void()
+  store i32 0, ptr %p
+  call void @writeonly_void()
+  ret void
+}
+
+; Cannot CSE loads across writeonly call.
+define i32 @load_cse_across_writeonly(ptr %p) {
+; CHECK-LABEL: @load_cse_across_writeonly(
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %v1 = load i32, ptr %p
+  call void @writeonly_void()
+  %v2 = load i32, ptr %p
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+; Can CSE loads across eliminated writeonly call.
+define i32 @load_cse_across_csed_writeonly(ptr %p) {
+; CHECK-LABEL: @load_cse_across_csed_writeonly(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  call void @writeonly_void()
+  %v1 = load i32, ptr %p
+  call void @writeonly_void()
+  %v2 = load i32, ptr %p
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+declare i32 @writeonly(ptr %p) memory(write)
+
+; Can CSE writeonly calls with arg and return.
+define i32 @writeonly_ret_cse(ptr %p) {
+; CHECK-LABEL: @writeonly_ret_cse(
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @writeonly(ptr [[P:%.*]])
+; CHECK-NEXT:    ret i32 0
+;
+  %v1 = call i32 @writeonly(ptr %p)
+  %v2 = call i32 @writeonly(ptr %p)
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+; Cannot CSE writeonly calls with different arguments.
+define i32 @writeonly_different_args(ptr %p1, ptr %p2) {
+; CHECK-LABEL: @writeonly_different_args(
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @writeonly(ptr [[P1:%.*]])
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @writeonly(ptr [[P2:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %v1 = call i32 @writeonly(ptr %p1)
+  %v2 = call i32 @writeonly(ptr %p2)
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+declare void @callee()
+
+; These are weird cases where the same call is both readonly and writeonly
+; based on call-site attributes. I believe this implies that both calls are
+; actually readnone and safe to CSE, but leave them alone to be conservative.
+define void @readonly_and_writeonly() {
+; CHECK-LABEL: @readonly_and_writeonly(
+; CHECK-NEXT:    call void @callee() #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    call void @callee() #[[ATTR1]]
+; CHECK-NEXT:    ret void
+;
+  call void @callee() memory(read)
+  call void @callee() memory(write)
+  ret void
+}
+
+define void @writeonly_and_readonly() {
+; CHECK-LABEL: @writeonly_and_readonly(
+; CHECK-NEXT:    call void @callee() #[[ATTR1]]
+; CHECK-NEXT:    call void @callee() #[[ATTR2]]
+; CHECK-NEXT:    ret void
+;
+  call void @callee() memory(write)
+  call void @callee() memory(read)
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/initializes.ll b/llvm/test/Transforms/FunctionAttrs/initializes.ll
index 5800bc1ca7864..193d0fd589884 100644
--- a/llvm/test/Transforms/FunctionAttrs/initializes.ll
+++ b/llvm/test/Transforms/FunctionAttrs/initializes.ll
@@ -649,3 +649,17 @@ define void @range_overflows_signed_64_bit_int(ptr %arg) {
   store i32 0, ptr %getelementptr
   ret void
 }
+
+; We should bail if the memset range overflows a signed 64-bit int.
+define void @memset_large_offset_nonzero_size(ptr %dst) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
+; CHECK-LABEL: define void @memset_large_offset_nonzero_size(
+; CHECK-SAME: ptr writeonly captures(none) [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 9223372036854775805
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[OFFSET]], i8 0, i64 3, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %offset = getelementptr inbounds i8, ptr %dst, i64 9223372036854775805
+  call void @llvm.memset.p0.i64(ptr %offset, i8 0, i64 3, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
deleted file mode 100644
index ab04e0a5bc697..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
+++ /dev/null
@@ -1,39 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
-
-define  void @memcpy_struct()  {
-; CHECK-LABEL: define void @memcpy_struct() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca { i8, i8, i8 }, align 1
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca {i8, i8, i8}, align 1
-  %call1 = call i32 @bar(ptr nonnull %something)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  ret void
-}
-
-
-@.i8_multi = private unnamed_addr constant [2 x [3 x i8]] [[3 x i8] [i8 1, i8 2, i8 3], [3 x i8] [i8 4, i8 5, i8 6]] , align 1
-
-define  void @memcpy_array_multidimensional()  {
-; CHECK-LABEL: define void @memcpy_array_multidimensional() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [2 x [3 x i8]], align 1
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8_multi, i32 3, i1 false)
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [2 x [3 x i8]], align 1
-  %call1 = call i32 @bar(ptr nonnull %something)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8_multi, i32 3, i1 false)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
deleted file mode 100644
index f435ffdeed2c8..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-; CHECK: [3 x i8]
-@other = private unnamed_addr global [3 x i8] [i8 1, i8 2, i8 3] , align 1
-; CHECK: [4 x i8]
-@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
-
-define  void @memcpy_multiple()  {
-; CHECK-LABEL: define void @memcpy_multiple() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @bar(ptr nonnull @other)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) @other, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [3 x i8], align 1
-  %call1 = call i32 @bar(ptr nonnull %something)
-  %call2 = call i32 @bar(ptr nonnull @other)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) @other, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
deleted file mode 100644
index c7ca7271fd3d2..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.i16 = private unnamed_addr constant [5 x i16] [i16 1, i16 2, i16 3, i16 4, i16 5] , align 1
-
-define  void @memcpy_i16_array()  {
-; CHECK-LABEL: define void @memcpy_i16_array() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING1:%.*]] = alloca [6 x i16], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) [[SOMETHING1]], ptr noundef nonnull align 1 dereferenceable(10) @.i16, i32 12, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING1]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [5 x i16], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) %something, ptr noundef nonnull align 1 dereferenceable(10) @.i16, i32 10, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
deleted file mode 100644
index 3d9c42fe1f3dd..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.str = unnamed_addr global [3 x i8] c"12\00", align 1
-
-define  void @foo()  {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [3 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.str, i32 3, i1 false)
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [3 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.str, i32 3, i1 false)
-  %call1 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
deleted file mode 100644
index e37925a78d2c3..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
+++ /dev/null
@@ -1,33 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
-
-define  void @memcpy_multiple()  {
-; CHECK-LABEL: define void @memcpy_multiple() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING2:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    [[SOMETHING1:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    [[SOMETHING3:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING2]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING1]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING3]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING2]])
-; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING1]])
-; CHECK-NEXT:    [[CALL4:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING3]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [3 x i8], align 1
-  %something1 = alloca [3 x i8], align 1
-  %something2 = alloca [3 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something1, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something2, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  %call3 = call i32 @bar(ptr nonnull %something1)
-  %call4 = call i32 @bar(ptr nonnull %something2)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
deleted file mode 100644
index 8ea9e2804370e..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.str = private unnamed_addr constant [10 x i8] c"123456789\00", align 1
-
-define  void @foo()  {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [12 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(10) @.str, i32 12, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [10 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) %something, ptr noundef nonnull align 1 dereferenceable(10) @.str, i32 10, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
deleted file mode 100644
index ad3620b14ea23..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.str = private unnamed_addr constant [62 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
-
-define  void @foo()  {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [64 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(62) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(62) @.str, i32 64, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [62 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(62) %something, ptr noundef nonnull align 1 dereferenceable(62) @.str, i32 62, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
deleted file mode 100644
index b8e02c3f996da..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-; CHECK: [17 x i8]
-@.str = private unnamed_addr constant [17 x i8] c"aaaaaaaaaaaaaaaa\00", align 1
-
-; Function Attrs: nounwind
-define  void @foo()   {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [20 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[SOMETHING]], ptr align 1 @.str, i32 17, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [20 x i8], align 1
-  call void @llvm.lifetime.start(i64 20, ptr nonnull %something) #3
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %something, ptr align 1 @.str, i32 17, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something) #3
-  call void @llvm.lifetime.end(i64 20, ptr nonnull %something) #3
-  ret void
-}
-
-declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
deleted file mode 100644
index 4ac31aa2f976d..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-; CHECK: [65 x i8]
-; CHECK-NOT: [68 x i8]
-@.str = private unnamed_addr constant [65 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaazzz\00", align 1
-
-; Function Attrs: nounwind
-define  void @foo()   {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [65 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 65, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[SOMETHING]], ptr align 1 @.str, i32 65, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 65, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [65 x i8], align 1
-  call void @llvm.lifetime.start(i64 65, ptr nonnull %something) #3
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %something, ptr align 1 @.str, i32 65, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something) #3
-  call void @llvm.lifetime.end(i64 65, ptr nonnull %something) #3
-  ret void
-}
-
-declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
deleted file mode 100644
index 64f57884cd39e..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
+++ /dev/null
@@ -1,54 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@f.string1 = private unnamed_addr constant [45 x i8] c"The quick brown dog jumps over the lazy fox.\00", align 1
-
-; Function Attrs: nounwind
-define  i32 @f() {
-; CHECK-LABEL: define i32 @f() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[STRING1:%.*]] = alloca [48 x i8], align 1
-; CHECK-NEXT:    [[POS:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[TOKEN:%.*]] = alloca ptr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 45, ptr [[STRING1]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[STRING1]], ptr align 1 @f.string1, i32 48, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[POS]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TOKEN]])
-; CHECK-NEXT:    [[CALL:%.*]] = call ptr @strchr(ptr [[STRING1]], i32 101)
-; CHECK-NEXT:    store ptr [[CALL]], ptr [[TOKEN]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TOKEN]], align 4
-; CHECK-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP1]] to i32
-; CHECK-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[STRING1]] to i32
-; CHECK-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i32 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUB_PTR_SUB]], 1
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[POS]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[POS]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TOKEN]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[POS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 45, ptr [[STRING1]])
-; CHECK-NEXT:    ret i32 [[TMP2]]
-;
-entry:
-  %string1 = alloca [45 x i8], align 1
-  %pos = alloca i32, align 4
-  %token = alloca ptr, align 4
-  call void @llvm.lifetime.start.p0i8(i64 45, ptr %string1)
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 %string1, ptr align 1 @f.string1, i32 45, i1 false)
-  call void @llvm.lifetime.start.p0i8(i64 4, ptr %pos)
-  call void @llvm.lifetime.start.p0i8(i64 4, ptr %token)
-  %call = call ptr @strchr(ptr %string1, i32 101)
-  store ptr %call, ptr %token, align 4
-  %0 = load ptr, ptr %token, align 4
-  %sub.ptr.lhs.cast = ptrtoint ptr %0 to i32
-  %sub.ptr.rhs.cast = ptrtoint ptr %string1 to i32
-  %sub.ptr.sub = sub i32 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
-  %add = add nsw i32 %sub.ptr.sub, 1
-  store i32 %add, ptr %pos, align 4
-  %1 = load i32, ptr %pos, align 4
-  call void @llvm.lifetime.end.p0i8(i64 4, ptr %token)
-  call void @llvm.lifetime.end.p0i8(i64 4, ptr %pos)
-  call void @llvm.lifetime.end.p0i8(i64 45, ptr %string1)
-  ret i32 %1
-}
-
-declare ptr @strchr(ptr, i32)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
deleted file mode 100644
index 5367572704b14..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-%struct.P = type { i32, [13 x i8] }
-
-; CHECK-NOT: [16 x i8]
-@.str = private unnamed_addr constant [13 x i8] c"hello world\0A\00", align 1
-
-; Function Attrs: nounwind
-define  i32 @main()   {
-; CHECK-LABEL: define i32 @main() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[P:%.*]] = alloca [[STRUCT_P:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[P]])
-; CHECK-NEXT:    store i32 10, ptr [[P]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [[STRUCT_P]], ptr [[P]], i32 0, i32 1, i32 0
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[ARRAYDECAY]], ptr align 1 @.str, i32 13, i1 false)
-; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(ptr [[ARRAYDECAY]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[P]])
-; CHECK-NEXT:    ret i32 0
-;
-entry:
-  %p = alloca %struct.P, align 4
-  call void @llvm.lifetime.start(i64 20, ptr nonnull %p) #2
-  store i32 10, ptr %p, align 4, !tbaa !1
-  %arraydecay = getelementptr inbounds %struct.P, ptr %p, i32 0, i32 1, i32 0
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 %arraydecay, ptr align 1 @.str, i32 13, i1 false)
-  %puts = call i32 @puts(ptr %arraydecay)
-  call void @llvm.lifetime.end(i64 20, ptr nonnull %p) #2
-  ret i32 0
-}
-
-declare i32 @puts(ptr nocapture readonly) #2
-
-!1 = !{!2, !3, i64 0}
-!2 = !{!"P", !3, i64 0, !4, i64 4}
-!3 = !{!"int", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C/C++ TBAA"}
-;.
-; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META2:![0-9]+]], i64 0}
-; CHECK: [[META1]] = !{!"P", [[META2]], i64 0, [[META3:![0-9]+]], i64 4}
-; CHECK: [[META2]] = !{!"int", [[META3]], i64 0}
-; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
-; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
-;.
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
deleted file mode 100644
index b735a77887423..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-; CHECK-NOT: [64 x i8]
-@.str = private unnamed_addr constant [62 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
-
-; Function Attrs: nounwind
-define  void @foo()   {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [62 x i8], align 1
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [62 x i8], ptr [[SOMETHING]], i32 0, i32 0
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 62, ptr nonnull [[TMP0]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[TMP0]], ptr align 1 @.str, i32 62, i1 true)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[TMP0]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 62, ptr nonnull [[TMP0]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [62 x i8], align 1
-  %0 = getelementptr inbounds [62 x i8], ptr %something, i32 0, i32 0
-  call void @llvm.lifetime.start(i64 62, ptr nonnull %0) #3
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %0, ptr align 1 @.str, i32 62, i1 true)
-  %call2 = call i32 @bar(ptr nonnull %0) #3
-  call void @llvm.lifetime.end(i64 62, ptr nonnull %0) #3
-  ret void
-}
-
-declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg b/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
deleted file mode 100644
index 7c23e4f980130..0000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not "ARM" in config.root.targets:
-    config.unsupported = True
diff --git a/llvm/test/Transforms/InferFunctionAttrs/enable-builtin.ll b/llvm/test/Transforms/InferFunctionAttrs/enable-builtin.ll
new file mode 100644
index 0000000000000..5c1d6ab377497
--- /dev/null
+++ b/llvm/test/Transforms/InferFunctionAttrs/enable-builtin.ll
@@ -0,0 +1,13 @@
+; RUN: opt -S -mtriple=amdgcn-- -passes=inferattrs %s | FileCheck -check-prefix=NOBUILTIN %s
+; RUN: opt -S -enable-builtin=malloc -mtriple=amdgcn-- -passes=inferattrs %s | FileCheck -check-prefix=WITHBUILTIN %s
+
+; Test that the -enable-builtin flag works and forces recognition of
+; malloc despite the target's default TargetLibraryInfo.
+
+; NOBUILTIN: declare ptr @malloc(i64)
+
+; WITHBUILTIN: declare noalias noundef ptr @malloc(i64 noundef) #0
+; WITHBUILTIN: attributes #0 = { mustprogress nofree nounwind willreturn allockind("alloc,uninitialized") allocsize(0) memory(inaccessiblemem: readwrite) "alloc-family"="malloc" }
+
+declare ptr @malloc(i64)
+
diff --git a/llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg b/llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg
new file mode 100644
index 0000000000000..f9dd98a21cc3e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "SystemZ" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll b/llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll
new file mode 100644
index 0000000000000..9148f9c6a6c40
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s --passes=instcombine -S | FileCheck %s
+;
+; Check that string replacements inserted by the instcombiner are properly aligned.
+; The specific case checked replaces `printf("foo\n")` with `puts("foo")`
+
+target datalayout = "i8:8:16"
+
+@msg1 = constant [17 x i8] c"Alignment Check\0A\00", align 2
+; CHECK: c"Alignment Check\00", align 2
+
+; Function Attrs: noinline nounwind
+define dso_local void @foo() #0 {
+  %call = call signext i32 (ptr, ...) @printf(ptr noundef @msg1)
+  ret void
+}
+
+declare signext i32 @printf(ptr noundef, ...) #1
diff --git a/llvm/test/Transforms/InstCombine/add4.ll b/llvm/test/Transforms/InstCombine/add4.ll
index 0e97deb4d98ad..15feb8a093e99 100644
--- a/llvm/test/Transforms/InstCombine/add4.ll
+++ b/llvm/test/Transforms/InstCombine/add4.ll
@@ -289,3 +289,126 @@ entry:
   %add = add i32 %shl, %rem
   ret i32 %add
 }
+
+define i32 @fold_add_udiv_urem_no_mul(i32 noundef %val) {
+; CHECK-LABEL: @fold_add_udiv_urem_no_mul(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[VAL:%.*]], 10
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[DIV]], -9
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP1]], [[VAL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %div = udiv i32 %val, 10
+  %rem = urem i32 %val, 10
+  %add = add i32 %div, %rem
+  ret i32 %add
+}
+
+define i32 @fold_add_udiv_urem_rem_mul(i32 noundef %val) {
+; CHECK-LABEL: @fold_add_udiv_urem_rem_mul(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[VAL:%.*]], 10
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[VAL]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[DIV]], -29
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %div = udiv i32 %val, 10
+  %rem = urem i32 %val, 10
+  %mul = mul i32 %rem, 3
+  %add = add i32 %div, %mul
+  ret i32 %add
+}
+
+define i32 @fold_add_udiv_urem_pow2_no_mul(i32 noundef %arg) {
+; CHECK-LABEL: @fold_add_udiv_urem_pow2_no_mul(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[ARG:%.*]], 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ARG]], 15
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[LSHR]], [[AND]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %lshr = lshr i32 %arg, 4
+  %and = and i32 %arg, 15
+  %add = add i32 %lshr, %and
+  ret i32 %add
+}
+
+define i32 @fold_add_udiv_urem_pow2_div_mul(i32 noundef %arg) {
+; CHECK-LABEL: @fold_add_udiv_urem_pow2_div_mul(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[ARG:%.*]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[LSHR]], -13
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP1]], [[ARG]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %lshr = lshr i32 %arg, 4
+  %mul = mul i32 %lshr, 3
+  %and = and i32 %arg, 15
+  %add = add i32 %mul, %and
+  ret i32 %add
+}
+
+define i32 @fold_add_sdiv_srem_no_mul(i32 noundef %val) {
+; CHECK-LABEL: @fold_add_sdiv_srem_no_mul(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[VAL:%.*]], 10
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[DIV]], -9
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP1]], [[VAL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %div = sdiv i32 %val, 10
+  %rem = srem i32 %val, 10
+  %add = add i32 %div, %rem
+  ret i32 %add
+}
+
+define i32 @fold_add_udiv_urem_pow2_rem_mul(i32 noundef %arg) {
+; CHECK-LABEL: @fold_add_udiv_urem_pow2_rem_mul(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[ARG:%.*]], 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ARG]], 15
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[LSHR]], [[MUL]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %lshr = lshr i32 %arg, 4
+  %and = and i32 %arg, 15
+  %mul = mul i32 %and, 3
+  %add = add i32 %lshr, %mul
+  ret i32 %add
+}
+
+define i32 @fold_add_udiv_urem_pow2_both_mul(i32 noundef %arg) {
+; CHECK-LABEL: @fold_add_udiv_urem_pow2_both_mul(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[ARG:%.*]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[ARG]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[LSHR]], -41
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %lshr = lshr i32 %arg, 4
+  %mul1 = mul i32 %lshr, 7
+  %and = and i32 %arg, 15
+  %mul2 = mul i32 %and, 3
+  %add = add i32 %mul1, %mul2
+  ret i32 %add
+}
+
+define i32 @fold_add_udiv_urem_by_two_no_mul(i32 noundef %arg) {
+; CHECK-LABEL: @fold_add_udiv_urem_by_two_no_mul(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = sub i32 [[ARG]], [[LSHR]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %lshr = lshr i32 %arg, 1
+  %and = and i32 %arg, 1
+  %add = add i32 %lshr, %and
+  ret i32 %add
+}
+
+define i32 @fold_add_sdiv_srem_by_two_no_mul(i32 noundef %arg) {
+; CHECK-LABEL: @fold_add_sdiv_srem_by_two_no_mul(
+; CHECK-NEXT:    [[DIV_NEG:%.*]] = sdiv i32 [[ARG:%.*]], -2
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[DIV_NEG]], [[ARG]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %div = sdiv i32 %arg, 2
+  %rem = srem i32 %arg, 2
+  %add = add i32 %div, %rem
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/InstCombine/binop-recurrence.ll b/llvm/test/Transforms/InstCombine/binop-recurrence.ll
new file mode 100644
index 0000000000000..b5eef4e3f516d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/binop-recurrence.ll
@@ -0,0 +1,1408 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+; Reassociate add.
+define i8 @add_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @add_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, and maintain nuw if all ops have it.
+define i8 @add_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add nuw i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw i8 %pn, 2
+  %op2 = add nuw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nuw if op1 doesn't have it.
+define i8 @add_op1_no_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_op1_no_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add nuw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nuw if op2 doesn't have it.
+define i8 @add_op2_no_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_op2_no_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nuw if rdx doesn't have it.
+define i8 @add_rdx_no_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_rdx_no_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw i8 %pn, 2
+  %op2 = add nuw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nsw even if all ops have it.
+define i8 @add_no_nsw(i32 %n) {
+; CHECK-LABEL: define i8 @add_no_nsw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nsw i8 %pn, 2
+  %op2 = add nsw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nsw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, keep nuw/nsw if all ops have them.
+define i8 @add_nuw_nsw(i32 %n) {
+; CHECK-LABEL: define i8 @add_nuw_nsw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add nuw nsw i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw nsw i8 %pn, 2
+  %op2 = add nuw nsw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw nsw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate fixed-length vector operands.
+define <16 x i8> @add_v16i8(i32 %n) {
+; CHECK-LABEL: define <16 x i8> @add_v16i8(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi <16 x i8> [ splat (i8 1), %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add <16 x i8> [[REDUCED_PHI]], splat (i8 5)
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret <16 x i8> [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi <16 x i8> [ splat (i8 0), %entry ], [ %op1, %body ]
+  %pn2 = phi <16 x i8> [ splat (i8 1), %entry ], [ %op2, %body ]
+  %op1 = add <16 x i8> %pn, splat (i8 2)
+  %op2 = add <16 x i8> %pn2, splat (i8 3)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add <16 x i8> %op2, %op1
+  ret <16 x i8> %rdx
+}
+
+; Reassociate scalable vector operands.
+define <vscale x 16 x i8> @add_nxv16i8(i32 %n) {
+; CHECK-LABEL: define <vscale x 16 x i8> @add_nxv16i8(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi <vscale x 16 x i8> [ splat (i8 1), %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add <vscale x 16 x i8> [[REDUCED_PHI]], splat (i8 5)
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi <vscale x 16 x i8> [ splat (i8 0), %entry ], [ %op1, %body ]
+  %pn2 = phi <vscale x 16 x i8> [ splat (i8 1), %entry ], [ %op2, %body ]
+  %op1 = add <vscale x 16 x i8> %pn, splat (i8 2)
+  %op2 = add <vscale x 16 x i8> %pn2, splat (i8 3)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add <vscale x 16 x i8> %op2, %op1
+  ret <vscale x 16 x i8> %rdx
+}
+
+; Check other opcodes.
+
+; Reassociate mul.
+define i8 @mul_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @mul_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 2, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = mul i8 [[REDUCED_PHI]], 15
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 2, %entry ], [ %op2, %body ]
+  %op1 = mul i8 %pn, 3
+  %op2 = mul i8 %pn2, 5
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = mul i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate mul, don't expect any flags to be propagated.
+define i8 @mul_reassoc_no_nuw_no_nsw(i32 %n) {
+; CHECK-LABEL: define i8 @mul_reassoc_no_nuw_no_nsw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 2, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = mul i8 [[REDUCED_PHI]], 15
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 2, %entry ], [ %op2, %body ]
+  %op1 = mul nuw nsw i8 %pn, 3
+  %op2 = mul nuw nsw i8 %pn2, 5
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = mul nuw nsw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate and, although it should already be optimized separately.
+define i8 @and_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @and_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 3
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 31, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 63, %entry ], [ %op2, %body ]
+  %op1 = and i8 %pn, 3
+  %op2 = and i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = and i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate or, although it should already be optimized separately.
+define i8 @or_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @or_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 7
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = or i8 %pn, 3
+  %op2 = or i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = or i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate or and propagate disjoint if all ops have it.
+; Note: This can't currently be seen as the results get optimized away.
+define i8 @or_disjoint(i32 %n) {
+; CHECK-LABEL: define i8 @or_disjoint(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 7
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = or disjoint i8 %pn, 3
+  %op2 = or disjoint i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = or disjoint i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate xor.
+define i8 @xor_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @xor_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = xor i8 [[REDUCED_PHI]], 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = xor i8 %pn, 3
+  %op2 = xor i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = xor i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate fadd if reassoc and nsz are present on all instructions.
+define float @fadd_reassoc_nsz(i32 %n) {
+; CHECK-LABEL: define float @fadd_reassoc_nsz(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi float [ 1.000000e+00, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = fadd reassoc nsz float [[REDUCED_PHI]], 5.000000e+00
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 0.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 1.0, %entry ], [ %op2, %body ]
+  %op1 = fadd reassoc nsz float %pn, 2.0
+  %op2 = fadd reassoc nsz float %pn2, 3.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fadd reassoc nsz float %op2, %op1
+  ret float %rdx
+}
+
+; Reassociate fmul if reassoc and nsz are present on all instructions.
+define float @fmul_reassoc_nsz(i32 %n) {
+; CHECK-LABEL: define float @fmul_reassoc_nsz(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi float [ 2.000000e+00, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = fmul reassoc nsz float [[REDUCED_PHI]], 1.200000e+01
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 1.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 2.0, %entry ], [ %op2, %body ]
+  %op1 = fmul reassoc nsz float %pn, 3.0
+  %op2 = fmul reassoc nsz float %pn2, 4.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fmul reassoc nsz float %op2, %op1
+  ret float %rdx
+}
+
+; Don't reassociate without `reassoc'.
+define float @fadd_no_reassoc(i32 %n) {
+; CHECK-LABEL: define float @fadd_no_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi float [ 1.000000e+00, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = fadd nsz float [[PN]], 2.000000e+00
+; CHECK-NEXT:    [[OP2]] = fadd nsz float [[PN2]], 3.000000e+00
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = fadd nsz float [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 0.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 1.0, %entry ], [ %op2, %body ]
+  %op1 = fadd nsz float %pn, 2.0
+  %op2 = fadd nsz float %pn2, 3.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fadd nsz float %op2, %op1
+  ret float %rdx
+}
+
+; Don't reassociate without `nsz'.
+define float @fadd_no_nsz(i32 %n) {
+; CHECK-LABEL: define float @fadd_no_nsz(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi float [ 1.000000e+00, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = fadd reassoc float [[PN]], 2.000000e+00
+; CHECK-NEXT:    [[OP2]] = fadd reassoc float [[PN2]], 3.000000e+00
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = fadd reassoc float [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 0.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 1.0, %entry ], [ %op2, %body ]
+  %op1 = fadd reassoc float %pn, 2.0
+  %op2 = fadd reassoc float %pn2, 3.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fadd reassoc float %op2, %op1
+  ret float %rdx
+}
+
+; Check commuted operands.
+
+; Reassociate, even if op1 has commuted operands.
+define i8 @add_op1_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_op1_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 2, %pn
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate, even if op2 has commuted operands.
+define i8 @add_op2_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_op2_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 3, %pn2
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate, even if rdx has commuted operands.
+define i8 @add_rdx_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_rdx_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op1, %op2
+  ret i8 %rdx
+}
+
+; Reassociate, even if pn has commuted incoming values.
+define i8 @add_pn_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_pn_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ %op1, %body ], [ 0, %entry ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate, even if pn2 has commuted incoming values.
+define i8 @add_pn2_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_pn2_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ %op2, %body ], [ 1, %entry ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op1, %op2
+  ret i8 %rdx
+}
+
+; Check the instructions have the same opcodes.
+
+; Don't reassociate if the first op doesn't match the rest.
+define i8 @no_mul_add_add(i32 %n) {
+; CHECK-LABEL: define i8 @no_mul_add_add(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = shl i8 [[PN]], 1
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = mul i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the second op doesn't match the rest.
+define i8 @no_add_mul_add(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_mul_add(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = mul i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = mul i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the third op doesn't match the rest.
+define i8 @no_add_add_mul(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_add_mul(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = mul i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = mul i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Check the number of uses of pn, pn2, op1 and op2.
+
+; Don't reassociate if pn has more uses.
+define i8 @no_add_pn_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_pn_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[PN]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %pn)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if pn2 has more uses.
+define i8 @no_add_pn2_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_pn2_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[PN2]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %pn2)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if op1 has more uses.
+define i8 @no_add_op1_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op1_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[OP1]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %op1)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if op2 has more uses.
+define i8 @no_add_op2_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op2_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[OP2]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %op2)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Check that init1, init2, c1 and c2 are constants.
+; Note: It should be possible to support non-constant operands, we just don't
+; do so yet.
+
+; Don't reassociate if the initial value of pn isn't constant.
+define i8 @no_add_init1(i8 %init1, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_init1(
+; CHECK-SAME: i8 [[INIT1:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ [[INIT1]], %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ %init1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the initial value of pn2 isn't constant.
+define i8 @no_add_init2(i8 %init2, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_init2(
+; CHECK-SAME: i8 [[INIT2:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ [[INIT2]], %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ %init2, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the recurrence operand of op1 isn't constant.
+define i8 @no_add_c1(i8 %c1, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_c1(
+; CHECK-SAME: i8 [[C1:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], [[C1]]
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, %c1
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the recurrence operand of op2 isn't constant.
+define i8 @no_add_c2(i8 %c2, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_c2(
+; CHECK-SAME: i8 [[C2:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], [[C2]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, %c2
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Check the incoming values of pn and pn2.
+
+; Don't reassociate if op1 doesn't recurse to pn.
+define i8 @no_add_op1_to_pn(i1 %c, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op1_to_pn(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[OP1:%.*]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op2, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if op2 doesn't recurse to pn2.
+define i8 @no_add_op2_to_pn2(i1 %c, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op2_to_pn2(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP1]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[OP2:%.*]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the phis have more than two incoming values.
+define i8 @no_add_phis_more_incoming_values(i1 %c, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_phis_more_incoming_values(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[BODY:.*]], label %[[OTHER:.*]]
+; CHECK:       [[OTHER]]:
+; CHECK-NEXT:    br label %[[BODY]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ 1, %[[OTHER]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 0, %[[OTHER]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ 1, %[[OTHER]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br i1 %c, label %body, label %other
+
+other:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ 1, %other ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ 0, %other ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ 1, %other ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the ops span different blocks.
+define i8 @no_add_op_multi_block(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op_multi_block(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OTHER:.*]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[OTHER]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[OTHER]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    br label %[[OTHER]]
+; CHECK:       [[OTHER]]:
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %other ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %other ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %other ]
+  %op1 = add i8 %pn, 2
+  br label %other
+
+other:
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+declare void @use(i8)
diff --git a/llvm/test/Transforms/InstCombine/compare-3way.ll b/llvm/test/Transforms/InstCombine/compare-3way.ll
index 5d443cd45238c..f5bbe0c9efd28 100644
--- a/llvm/test/Transforms/InstCombine/compare-3way.ll
+++ b/llvm/test/Transforms/InstCombine/compare-3way.ll
@@ -3,7 +3,7 @@
 
 declare void @use(i32)
 
-; These 18 exercise all combinations of signed comparison
+; These exercise all combinations of signed comparison
 ; for each of the three values produced by your typical
 ; 3way compare function (-1, 0, 1)
 
@@ -81,8 +81,8 @@ unreached:
 define void @test_low_sle(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_low_sle
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -105,8 +105,8 @@ unreached:
 define void @test_low_ne(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_low_ne
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -130,8 +130,8 @@ unreached:
 define void @test_low_eq(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_low_eq
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -154,8 +154,8 @@ unreached:
 define void @test_mid_sgt(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_sgt
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -178,8 +178,8 @@ unreached:
 define void @test_mid_slt(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_slt
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -252,8 +252,8 @@ unreached:
 define void @test_mid_ne(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_ne
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[EQ:%.*]] = icmp eq i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[EQ]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -277,8 +277,8 @@ unreached:
 define void @test_mid_eq(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_eq
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[EQ:%.*]] = icmp eq i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[EQ]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -348,8 +348,8 @@ unreached:
 define void @test_high_sge(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_high_sge
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -396,8 +396,8 @@ unreached:
 define void @test_high_ne(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_high_ne
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -421,8 +421,8 @@ unreached:
 define void @test_high_eq(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_high_eq
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -560,3 +560,75 @@ unreached:
   call void @use(i32 %result)
   ret void
 }
+
+define i32 @smax_smin_to_scmp(i32 %x) {
+; CHECK-LABEL: define i32 @smax_smin_to_scmp
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 1)
+  ret i32 %cond5
+}
+
+define i16 @smax_smin_to_scmp_i16(i16 %x) {
+; CHECK-LABEL: define i16 @smax_smin_to_scmp_i16
+; CHECK-SAME: (i16 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND5:%.*]] = call i16 @llvm.scmp.i16.i16(i16 [[X]], i16 0)
+; CHECK-NEXT:    ret i16 [[COND5]]
+;
+  %cond = call i16 @llvm.smax.i16(i16 %x, i16 -1)
+  %cond5 = call i16 @llvm.smin.i16(i16 %cond, i16 1)
+  ret i16 %cond5
+}
+
+; Test the reversed pattern: smax(smin(X, 1), -1) -> scmp(X, 0)
+define i32 @smin_smax_to_scmp(i32 %x) {
+; CHECK-LABEL: define i32 @smin_smax_to_scmp
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smin.i32(i32 %x, i32 1)
+  %cond5 = call i32 @llvm.smax.i32(i32 %cond, i32 -1)
+  ret i32 %cond5
+}
+
+define i32 @test_max_min_neg(i32 %x) {
+; CHECK-LABEL: define i32 @test_max_min_neg
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 -2)
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.smin.i32(i32 [[COND]], i32 1)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -2)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 1)
+  ret i32 %cond5
+}
+
+define i32 @test_max_min_neg_2(i32 %x) {
+; CHECK-LABEL: define i32 @test_max_min_neg_2
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 -1)
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.smin.i32(i32 [[COND]], i32 2)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 2)
+  ret i32 %cond5
+}
+
+define i32 @test_multiple_uses(i32 %x) {
+; CHECK-LABEL: define i32 @test_multiple_uses
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 -1)
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.smin.i32(i32 [[COND]], i32 1)
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[COND]], [[COND5]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 1)
+  %sum = add i32 %cond, %cond5
+  ret i32 %sum
+}
diff --git a/llvm/test/Transforms/InstCombine/ctpop-cttz.ll b/llvm/test/Transforms/InstCombine/ctpop-cttz.ll
index 4c5d7a7dc011b..a1f0f01e2c470 100644
--- a/llvm/test/Transforms/InstCombine/ctpop-cttz.ll
+++ b/llvm/test/Transforms/InstCombine/ctpop-cttz.ll
@@ -127,3 +127,24 @@ define <2 x i32> @ctpop3v_poison(<2 x i32> %0) {
   %5 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %4)
   ret <2 x i32> %5
 }
+
+define i32 @ctpop_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @ctpop_xor(
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[FALSE:%.*]], label [[TRUE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[RET:%.*]] = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 [[XOR]])
+; CHECK-NEXT:    ret i32 [[RET]]
+; CHECK:       false:
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp ne i32 %x, %y
+  br i1 %cmp, label %true, label %false
+true:
+  %xor = xor i32 %x, %y
+  %ret = call i32 @llvm.ctpop.i32(i32 %xor)
+  ret i32 %ret
+false:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll b/llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll
new file mode 100644
index 0000000000000..dddd63d101e61
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p instcombine -S %s | FileCheck %s
+
+define <4 x i16> @ext_identity_mask_first_vector_first_half_4xi16(<8 x i8> %x) {
+; CHECK-LABEL: define <4 x i16> @ext_identity_mask_first_vector_first_half_4xi16(
+; CHECK-SAME: <8 x i8> [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[X]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i16> %shuffle
+}
+
+define <3 x i32> @ext_identity_mask_first_vector_first_half_3xi32(<4 x i16> %x) {
+; CHECK-LABEL: define <3 x i32> @ext_identity_mask_first_vector_first_half_3xi32(
+; CHECK-SAME: <4 x i16> [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[E_1]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <3 x i32> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <4 x i16> %x to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %e.1, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x i32> %shuffle
+}
+
+define <4 x i16> @ext_no_identity_mask1(<8 x i8> %in) {
+; CHECK-LABEL: define <4 x i16> @ext_no_identity_mask1(
+; CHECK-SAME: <8 x i8> [[IN:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[IN]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %in to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x i16> %shuffle
+}
+
+define <4 x i16> @ext_no_identity_mask2(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: define <4 x i16> @ext_no_identity_mask2(
+; CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[X]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> %y, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i16> %shuffle
+}
+
+define <5 x i32> @ext_identity_mask_first_vector_first_half_5xi32(<4 x i16> %x) {
+; CHECK-LABEL: define <5 x i32> @ext_identity_mask_first_vector_first_half_5xi32(
+; CHECK-SAME: <4 x i16> [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[E_1]], <4 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <5 x i32> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <4 x i16> %x to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %e.1, <4 x i32> %e.1, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
+  ret <5 x i32> %shuffle
+}
+
+define <4 x i16> @ext_no_identity_mask_first_vector_second_half(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: define <4 x i16> @ext_no_identity_mask_first_vector_second_half(
+; CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[X]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> [[Y]], <4 x i32> <i32 0, i32 9, i32 1, i32 10>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> %y, <4 x i32> <i32 0, i32 9, i32 1, i32 10>
+  ret <4 x i16> %shuffle
+}
+
+define <4 x i16> @select_second_op(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: define <4 x i16> @select_second_op(
+; CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[Y]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> %y, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  ret <4 x i16> %shuffle
+}
+
+define <4 x i32> @load_i32_zext_to_v4i32(ptr %di) {
+; CHECK-LABEL: define <4 x i32> @load_i32_zext_to_v4i32(
+; CHECK-SAME: ptr [[DI:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[DI]], align 4
+; CHECK-NEXT:    [[VEC_INS:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[L]], i64 0
+; CHECK-NEXT:    [[VEC_BC:%.*]] = bitcast <2 x i32> [[VEC_INS]] to <8 x i8>
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[VEC_BC]] to <8 x i16>
+; CHECK-NEXT:    [[VEC_SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXT_2:%.*]] = zext nneg <4 x i16> [[VEC_SHUFFLE]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[EXT_2]]
+;
+entry:
+  %l = load i32, ptr %di
+  %vec.ins = insertelement <2 x i32> <i32 poison, i32 0>, i32 %l, i64 0
+  %vec.bc = bitcast <2 x i32> %vec.ins to <8 x i8>
+  %e.1 = zext <8 x i8> %vec.bc to <8 x i16>
+  %vec.shuffle = shufflevector <8 x i16> %e.1, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ext.2 = zext nneg <4 x i16> %vec.shuffle to <4 x i32>
+  ret <4 x i32> %ext.2
+}
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index b7d263f5eb320..752ff0cae5b78 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -688,9 +688,9 @@ define i32 @test28() nounwind  {
 ; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
 ; CHECK-NEXT:    [[T12_REC:%.*]] = xor i32 [[INDVAR]], -1
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[T12_REC]] to i64
-; CHECK-NEXT:    [[T12:%.*]] = getelementptr inbounds [[STRUCT_X:%.*]], ptr [[T45]], i64 [[TMP0]]
+; CHECK-NEXT:    [[T12:%.*]] = getelementptr inbounds i8, ptr [[T45]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[T16:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str1, ptr nonnull [[T12]]) #[[ATTR0]]
-; CHECK-NEXT:    [[T84:%.*]] = icmp eq ptr [[T12]], [[ORIENTATIONS]]
+; CHECK-NEXT:    [[T84:%.*]] = icmp eq i32 [[INDVAR]], 0
 ; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
 ; CHECK-NEXT:    br i1 [[T84]], label [[BB17:%.*]], label [[BB10]]
 ; CHECK:       bb17:
diff --git a/llvm/test/Transforms/InstCombine/icmp-gep.ll b/llvm/test/Transforms/InstCombine/icmp-gep.ll
index 260462896c39d..3f104056fb1f2 100644
--- a/llvm/test/Transforms/InstCombine/icmp-gep.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-gep.ll
@@ -221,6 +221,34 @@ define i1 @eq_base_inbounds_commute_use(i64 %y) {
   ret i1 %r
 }
 
+define i1 @ne_base_inbounds_use_scaled(ptr %x, i64 %y) {
+; CHECK-LABEL: @ne_base_inbounds_use_scaled(
+; CHECK-NEXT:    [[G_IDX:%.*]] = shl nsw i64 [[Y:%.*]], 3
+; CHECK-NEXT:    [[G:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i64 [[G_IDX]]
+; CHECK-NEXT:    call void @use(ptr [[G]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[Y]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr inbounds i64, ptr %x, i64 %y
+  call void @use(ptr %g)
+  %r = icmp ne ptr %g, %x
+  ret i1 %r
+}
+
+define i1 @ne_base_use_scaled(ptr %x, i64 %y) {
+; CHECK-LABEL: @ne_base_use_scaled(
+; CHECK-NEXT:    [[G_IDX_MASK:%.*]] = shl i64 [[Y:%.*]], 3
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[X:%.*]], i64 [[G_IDX_MASK]]
+; CHECK-NEXT:    call void @use(ptr [[G]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[G_IDX_MASK]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr i64, ptr %x, i64 %y
+  call void @use(ptr %g)
+  %r = icmp ne ptr %g, %x
+  ret i1 %r
+}
+
 define i1 @eq_bitcast_base(ptr %p, i64 %x) {
 ; CHECK-LABEL: @eq_bitcast_base(
 ; CHECK-NEXT:    [[GEP_IDX_MASK:%.*]] = and i64 [[X:%.*]], 9223372036854775807
@@ -757,3 +785,67 @@ define i1 @gep_diff_base_same_indices_nuw_nusw(ptr %x, ptr %y, i64 %z) {
   %cmp = icmp ult ptr %gep1, %gep2
   ret i1 %cmp
 }
+
+define i1 @gep_multiple_eq(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_multiple_eq(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[DOTMASK:%.*]] = and i64 [[GEP1_IDX1]], 4611686018427387903
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[DOTMASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr i32, ptr %base, i64 %idx
+  %gep2 = getelementptr i32, ptr %gep1, i64 %idx2
+  %cmp = icmp eq ptr %gep2, %base
+  ret i1 %cmp
+}
+
+define i1 @gep_multiple_eq_commuted(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_multiple_eq_commuted(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[DOTMASK:%.*]] = and i64 [[GEP1_IDX1]], 4611686018427387903
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[DOTMASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr i32, ptr %base, i64 %idx
+  %gep2 = getelementptr i32, ptr %gep1, i64 %idx2
+  %cmp = icmp eq ptr %base, %gep2
+  ret i1 %cmp
+}
+
+define i1 @gep_mugtiple_ugt_nuw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_mugtiple_ugt_nuw(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = sub i64 0, [[IDX2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IDX:%.*]], [[GEP1_IDX1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %base, i64 %idx
+  %gep2 = getelementptr nuw i32, ptr %gep1, i64 %idx2
+  %cmp = icmp ugt ptr %gep2, %base
+  ret i1 %cmp
+}
+
+define i1 @gep_mugtiple_ugt_not_all_nuw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_mugtiple_ugt_not_all_nuw(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr nuw i32, ptr [[BASE:%.*]], i64 [[IDX:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i32, ptr [[GEP1]], i64 [[IDX2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt ptr [[GEP2]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %base, i64 %idx
+  %gep2 = getelementptr i32, ptr %gep1, i64 %idx2
+  %cmp = icmp ugt ptr %gep2, %base
+  ret i1 %cmp
+}
+
+define i1 @gep_mugtiple_ugt_inbounds_nusw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_mugtiple_ugt_inbounds_nusw(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[GEP1_IDX1]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr inbounds i32, ptr %base, i64 %idx
+  %gep2 = getelementptr nusw i32, ptr %gep1, i64 %idx2
+  %cmp = icmp ugt ptr %gep2, %base
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 365c17b35a468..a090f9c4d2614 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -1900,6 +1900,108 @@ define i1 @icmp_add1_sle(i32 %x, i32 %y) {
   ret i1 %cmp
 }
 
+define i1 @icmp_slt_offset_with_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_slt_offset_with_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, -16
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @icmp_slt_offset_with_smaller_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_slt_offset_with_smaller_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, -8
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle_offset_with_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_sle_offset_with_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, 16
+  %cmp = icmp sle i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @icmp_ule_offset_with_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_ule_offset_with_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nuw i64 %shlx, 16
+  %cmp = icmp ule i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+; TODO: Handle non-power-of-2 divisors
+define i1 @icmp_ule_offset_with_common_non_pow2_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_ule_offset_with_common_non_pow2_divisor(
+; CHECK-NEXT:    [[MULX:%.*]] = mul nuw i64 [[X:%.*]], 7
+; CHECK-NEXT:    [[MULY:%.*]] = mul nuw i64 [[Y:%.*]], 7
+; CHECK-NEXT:    [[MULX_OFFSET:%.*]] = add nuw i64 [[MULX]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MULX_OFFSET]], [[MULY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mulx = mul nuw i64 %x, 7
+  %muly = mul nuw i64 %y, 7
+  %mulx_offset = add nuw i64 %mulx, 7
+  %cmp = icmp ule i64 %mulx_offset, %muly
+  ret i1 %cmp
+}
+
+define i1 @neg_icmp_slt_offset_without_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @neg_icmp_slt_offset_without_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[SHLX_OFFSET:%.*]] = add nsw i64 [[SHLX]], -32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX_OFFSET]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, -32
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @neg_icmp_slt_offset_with_wrong_sign(i64 %x, i64 %y) {
+; CHECK-LABEL: @neg_icmp_slt_offset_with_wrong_sign(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[SHLX_OFFSET:%.*]] = add nsw i64 [[SHLX]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX_OFFSET]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, 16
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
 define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add20_sge_add57(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[Y:%.*]], 37
diff --git a/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll b/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll
index 331de12ddd339..c2663d8638932 100644
--- a/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll
+++ b/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll
@@ -205,9 +205,10 @@ define <4 x double> @matrix_multiply_two_operands_negated_with_same_size(<2 x do
 
 define <2 x double> @matrix_multiply_two_operands_with_multiple_uses(<6 x double> %a, <3 x double> %b) {
 ; CHECK-LABEL: @matrix_multiply_two_operands_with_multiple_uses(
-; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x double> @llvm.matrix.multiply.v2f64.v6f64.v3f64(<6 x double> [[A:%.*]], <3 x double> [[B:%.*]], i32 2, i32 3, i32 1)
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x double> [[A]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[RES_3:%.*]] = fsub <2 x double> [[RES]], [[TMP1]]
+; CHECK-NEXT:    [[A_NEG:%.*]] = fneg <6 x double> [[A:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x double> @llvm.matrix.multiply.v2f64.v6f64.v3f64(<6 x double> [[A]], <3 x double> [[B:%.*]], i32 2, i32 3, i32 1)
+; CHECK-NEXT:    [[RES_2:%.*]] = shufflevector <6 x double> [[A_NEG]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RES_3:%.*]] = fadd <2 x double> [[RES_2]], [[RES]]
 ; CHECK-NEXT:    ret <2 x double> [[RES_3]]
 ;
   %a.neg = fneg <6 x double> %a
diff --git a/llvm/test/Transforms/InstCombine/or-bitmask.ll b/llvm/test/Transforms/InstCombine/or-bitmask.ll
index 3c992dfea569a..2ee34cdc034ad 100644
--- a/llvm/test/Transforms/InstCombine/or-bitmask.ll
+++ b/llvm/test/Transforms/InstCombine/or-bitmask.ll
@@ -451,6 +451,448 @@ define i32 @and_mul_non_disjoint(i32 %in) {
   ret i32 %out
 }
 
+define i32 @unrelated_ops(i32 %in, i32 %in2) {
+; CHECK-LABEL: @unrelated_ops(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp, %temp3
+  ret i32 %out
+}
+
+define i32 @unrelated_ops1(i32 %in, i32 %in2) {
+; CHECK-LABEL: @unrelated_ops1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  ret i32 %out
+}
+
+define i32 @unrelated_ops2(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[TMP2]], [[IN4:%.*]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[IN2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN3:%.*]], [[IN2]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %temp, %in3
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp4 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops3(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops3(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND0]], 0
+; CHECK-NEXT:    [[TEMP:%.*]] = select i1 [[CMP]], i32 0, i32 144
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[TEMP]], [[IN3:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[TMP2]], 72
+; CHECK-NEXT:    [[TEMP4:%.*]] = or disjoint i32 [[IN2:%.*]], [[TEMP2]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TEMP4]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 2
+  %cmp = icmp eq i32 %and0, 0
+  %temp = select i1 %cmp, i32 0, i32 144
+  %temp3 = or disjoint i32 %temp, %in3
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp4 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops4(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops4(
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[IN:%.*]], 12
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw nsw i32 [[TMP2]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[IN4:%.*]], [[TMP3]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND1]], 0
+; CHECK-NEXT:    [[IN3:%.*]] = select i1 [[CMP]], i32 0, i32 144
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[IN3]], [[IN2:%.*]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TMP4]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 12
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %and1 = and i32 %in, 2
+  %cmp = icmp eq i32 %and1, 0
+  %temp2 = select i1 %cmp, i32 0, i32 144
+  %temp4 = or disjoint i32 %temp2, %in3
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops5(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops5(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND0]], 0
+; CHECK-NEXT:    [[IN3:%.*]] = select i1 [[CMP]], i32 0, i32 144
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[IN3]], [[IN2:%.*]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 4
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[AND1]], 0
+; CHECK-NEXT:    [[TEMP2:%.*]] = select i1 [[CMP2]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN4:%.*]], [[TEMP2]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 2
+  %cmp = icmp eq i32 %and0, 0
+  %temp = select i1 %cmp, i32 0, i32 144
+  %temp3 = or disjoint i32 %temp, %in3
+  %and1 = and i32 %in, 4
+  %cmp2 = icmp eq i32 %and1, 0
+  %temp2 = select i1 %cmp2, i32 0, i32 288
+  %temp4 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops6(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops6(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[IN4:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[IN2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN3:%.*]], [[IN2]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %in3, %temp
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp4 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops7(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops7(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[IN4:%.*]], [[TMP2]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[IN3:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN3]], [[IN2:%.*]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %in3, %temp
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp4 = or disjoint i32 %temp2, %in2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops8(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[TMP2]], [[IN4:%.*]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[IN3:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN3]], [[IN2:%.*]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %temp, %in3
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp4 = or disjoint i32 %temp2, %in2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops_nocombine(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops_nocombine(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[TEMP]], [[IN3:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[IN]], 7
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[TMP2]], 72
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN2:%.*]], [[TEMP2]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %temp, %in3
+  %and1 = and i32 %in, 7
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp4 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_ops_nocombine1(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @unrelated_ops_nocombine1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[TEMP]], [[IN3:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[TMP2]], 36
+; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[IN2:%.*]], [[TEMP2]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %temp, %in3
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 36
+  %temp4 = or disjoint i32 %in2, %temp2
+  %out = or disjoint i32 %temp3, %temp4
+  ret i32 %out
+}
+
+define i32 @unrelated_nondisjoint(i32 %in, i32 %in2) {
+; CHECK-LABEL: @unrelated_nondisjoint(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[AND0]], 72
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[IN2:%.*]], [[TEMP]]
+; CHECK-NEXT:    [[OUT:%.*]] = or i32 [[TEMP3]], [[TEMP2]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or i32 %temp3, %temp2
+  ret i32 %out
+}
+
+define i32 @unrelated_nondisjoint1(i32 %in, i32 %in2) {
+; CHECK-LABEL: @unrelated_nondisjoint1(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[AND0]], 72
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or i32 [[IN2:%.*]], [[TEMP]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TEMP2]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  ret i32 %out
+}
+
+declare void @use(i32)
+
+define i32 @multi_use(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[OUT]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %out)
+  ret i32 %out
+}
+
+define i32 @multi_use1(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use1(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[AND0]], 72
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[IN2:%.*]], [[TEMP]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TEMP2]]
+; CHECK-NEXT:    call void @use(i32 [[TEMP3]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %temp3)
+  ret i32 %out
+}
+
+define i32 @multi_use2(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use2(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[AND0]], 72
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[IN2:%.*]], [[TEMP]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[TEMP2]]
+; CHECK-NEXT:    call void @use(i32 [[TEMP3]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %temp3)
+  ret i32 %out
+}
+
+define i32 @multi_use3(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use3(
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN:%.*]], 12
+; CHECK-NEXT:    [[TEMP2:%.*]] = mul nuw nsw i32 [[AND1]], 72
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[TEMP2]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %temp2)
+  ret i32 %out
+}
+
+define i32 @multi_use4(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use4(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[AND0]], 72
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[TEMP]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %temp)
+  ret i32 %out
+}
+
+define i32 @multi_use5(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use5(
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[IN:%.*]], 12
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %and1)
+  ret i32 %out
+}
+
+define i32 @multi_use6(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use6(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[AND0]], 72
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[TEMP]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %temp)
+  ret i32 %out
+}
+
+define i32 @multi_use7(i32 %in, i32 %in2) {
+; CHECK-LABEL: @multi_use7(
+; CHECK-NEXT:    [[AND0:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TMP2]], [[IN2:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[AND0]])
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %and1 = and i32 %in, 12
+  %temp2 = mul nuw nsw i32 %and1, 72
+  %temp3 = or disjoint i32 %in2, %temp
+  %out = or disjoint i32 %temp3, %temp2
+  call void @use(i32 %and0)
+  ret i32 %out
+}
+
+define i32 @no_chain(i32 %in, i32 %in2, i32 %in3) {
+; CHECK-LABEL: @no_chain(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[TEMP:%.*]] = mul nuw nsw i32 [[TMP1]], 72
+; CHECK-NEXT:    [[TEMP3:%.*]] = or disjoint i32 [[TEMP]], [[IN3:%.*]]
+; CHECK-NEXT:    [[OUT:%.*]] = or disjoint i32 [[TEMP3]], [[IN2:%.*]]
+; CHECK-NEXT:    ret i32 [[OUT]]
+;
+  %and0 = and i32 %in, 3
+  %temp = mul nuw nsw i32 %and0, 72
+  %temp3 = or disjoint i32 %temp, %in3
+  %out = or disjoint i32 %temp3, %in2
+  ret i32 %out
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CONSTSPLAT: {{.*}}
 ; CONSTVEC: {{.*}}
diff --git a/llvm/test/Transforms/InstCombine/pr39908.ll b/llvm/test/Transforms/InstCombine/pr39908.ll
index ca143f417fb27..5d13a331c6d2e 100644
--- a/llvm/test/Transforms/InstCombine/pr39908.ll
+++ b/llvm/test/Transforms/InstCombine/pr39908.ll
@@ -7,9 +7,7 @@ target datalayout = "p:32:32"
 
 define i1 @test(ptr %p, i32 %n) {
 ; CHECK-LABEL: @test(
-; CHECK-NEXT:    [[END:%.*]] = getelementptr inbounds [0 x %S], ptr [[P:%.*]], i32 0, i32 [[N:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[LAST:%.*]] = getelementptr inbounds i8, ptr [[END]], i32 -8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LAST]], [[P]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[N:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %end = getelementptr inbounds [0 x %S], ptr %p, i32 0, i32 %n, i32 0, i32 0
@@ -22,9 +20,7 @@ define i1 @test(ptr %p, i32 %n) {
 define i1 @test64(ptr %p, i64 %n) {
 ; CHECK-LABEL: @test64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N:%.*]] to i32
-; CHECK-NEXT:    [[END:%.*]] = getelementptr inbounds [0 x %S], ptr [[P:%.*]], i32 0, i32 [[TMP1]], i32 0, i32 0
-; CHECK-NEXT:    [[LAST:%.*]] = getelementptr inbounds i8, ptr [[END]], i32 -8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LAST]], [[P]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %end = getelementptr inbounds [0 x %S], ptr %p, i64 0, i64 %n, i32 0, i64 0
@@ -37,9 +33,7 @@ define i1 @test64(ptr %p, i64 %n) {
 define i1 @test64_overflow(ptr %p, i64 %n) {
 ; CHECK-LABEL: @test64_overflow(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N:%.*]] to i32
-; CHECK-NEXT:    [[END:%.*]] = getelementptr inbounds [0 x %S], ptr [[P:%.*]], i32 0, i32 [[TMP1]], i32 0, i32 0
-; CHECK-NEXT:    [[LAST:%.*]] = getelementptr inbounds i8, ptr [[END]], i32 -8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LAST]], [[P]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %end = getelementptr inbounds [0 x %S], ptr %p, i64 0, i64 %n, i32 0, i64 8589934592
diff --git a/llvm/test/Transforms/InstCombine/sub-gep.ll b/llvm/test/Transforms/InstCombine/sub-gep.ll
index 45a30350aafe1..11af6b4a0197f 100644
--- a/llvm/test/Transforms/InstCombine/sub-gep.ll
+++ b/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -943,6 +943,70 @@ define i64 @multiple_geps_two_chains_gep_base(ptr %base, i64 %base.idx, i64 %idx
   ret i64 %d
 }
 
+define i64 @multiple_geps_two_chains_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4) {
+; CHECK-LABEL: @multiple_geps_two_chains_multi_use(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
+; CHECK-NEXT:    [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
+; CHECK-NEXT:    [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P3_IDX]]
+; CHECK-NEXT:    [[P4_IDX1:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[P4_IDX1]]
+; CHECK-NEXT:    call void @use(ptr [[P5]])
+; CHECK-NEXT:    call void @use(ptr [[P4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[P3_IDX]], [[P4_IDX1]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
+  %p2 = getelementptr inbounds i32, ptr %p1, i64 %idx2
+  %p3 = getelementptr inbounds i32, ptr %base, i64 %idx3
+  %p4 = getelementptr inbounds i32, ptr %p3, i64 %idx4
+  call void @use(ptr %p2)
+  call void @use(ptr %p4)
+  %i1 = ptrtoint ptr %p4 to i64
+  %i2 = ptrtoint ptr %p2 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_two_chains_partial_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4, i64 %idx5, i64 %idx6) {
+; CHECK-LABEL: @multiple_geps_two_chains_partial_multi_use(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
+; CHECK-NEXT:    [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
+; CHECK-NEXT:    [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
+; CHECK-NEXT:    [[P4_IDX1:%.*]] = shl nsw i64 [[IDX7:%.*]], 2
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P4_IDX1]]
+; CHECK-NEXT:    [[P5_IDX:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr [[P5]], i64 [[P5_IDX]]
+; CHECK-NEXT:    [[P6_IDX:%.*]] = shl nsw i64 [[IDX6:%.*]], 2
+; CHECK-NEXT:    call void @use(ptr [[P3]])
+; CHECK-NEXT:    call void @use(ptr [[P4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[TMP1]], [[P3_IDX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[P4_IDX1]], [[P5_IDX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], [[P6_IDX]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub nsw i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
+  %p2 = getelementptr inbounds i32, ptr %p1, i64 %idx2
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx3
+  %p4 = getelementptr inbounds i32, ptr %base, i64 %idx4
+  %p5 = getelementptr inbounds i32, ptr %p4, i64 %idx5
+  %p6 = getelementptr inbounds i32, ptr %p5, i64 %idx6
+  call void @use(ptr %p2)
+  call void @use(ptr %p5)
+  %i1 = ptrtoint ptr %p6 to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
 define i64 @multiple_geps_inbounds(ptr %base, i64 %idx, i64 %idx2) {
 ; CHECK-LABEL: @multiple_geps_inbounds(
 ; CHECK-NEXT:    [[D:%.*]] = add nsw i64 [[IDX:%.*]], [[IDX2:%.*]]
@@ -1025,3 +1089,72 @@ define <2 x i64> @splat_geps_multiple(ptr %base, i64 %idx0, <2 x i64> %idx1, <2
   %d = sub <2 x i64> %gep2.int, %gep1.int
   ret <2 x i64> %d
 }
+
+define i64 @nuw_ptrdiff_shl_nsw(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_shl_nsw(
+; CHECK-NEXT:    [[OFFSET:%.*]] = shl nuw nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = shl nsw i64 %idx, 3
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+define i64 @nuw_ptrdiff_shl_nonsw(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_shl_nonsw(
+; CHECK-NEXT:    [[OFFSET:%.*]] = shl i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = shl i64 %idx, 3
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+define i64 @nuw_ptrdiff_mul_nsw_nneg_scale(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_mul_nsw_nneg_scale(
+; CHECK-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = mul nsw i64 %idx, 3
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+define i64 @nuw_ptrdiff_mul_nsw_unknown_scale(ptr %base, i64 %idx, i64 %scale) {
+; CHECK-LABEL: @nuw_ptrdiff_mul_nsw_unknown_scale(
+; CHECK-NEXT:    [[OFFSET:%.*]] = mul nsw i64 [[IDX:%.*]], [[SCALE:%.*]]
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = mul nsw i64 %idx, %scale
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+declare void @usei64(i64)
+
+define i64 @nuw_ptrdiff_mul_nsw_nneg_scale_multiuse(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_mul_nsw_nneg_scale_multiuse(
+; CHECK-NEXT:    [[OFFSET:%.*]] = mul nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    call void @usei64(i64 [[OFFSET]])
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = mul nsw i64 %idx, 3
+  call void @usei64(i64 %offset)
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll b/llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll
new file mode 100644
index 0000000000000..1ec3dc3e4b40a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll
@@ -0,0 +1,52 @@
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+define float @extract_from_zero_init_shuffle(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_zero_init_shuffle(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 0
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> zeroinitializer
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+
+define float @extract_from_general_splat(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_general_splat(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 1
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+define float @extract_from_general_scalable_splat(<vscale x 2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_general_scalable_splat(
+; CHECK-NEXT:    %extract = extractelement <vscale x 2 x float> %vec, i64 0
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <vscale x 2 x float> %vec, <vscale x 2 x float> poison, <vscale x 4 x i32> zeroinitializer
+  %extract = extractelement <vscale x 4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+define float @extract_from_splat_with_poison_0(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_splat_with_poison_0(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 1
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 1>
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+define float @extract_from_splat_with_poison_1(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_splat_with_poison_1(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 1
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 1, i32 poison, i32 1, i32 1>
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
index 9aa050e8cd500..86fc5bbf72e7b 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
@@ -61,8 +61,7 @@ define float @test6(<4 x float> %X) {
 
 define float @testvscale6(<vscale x 4 x float> %X) {
 ; CHECK-LABEL: @testvscale6(
-; CHECK-NEXT:    [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[X:%.*]], i64 0
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
@@ -1278,9 +1277,9 @@ define <2 x float> @fsub_splat_constant1(<2 x float> %x) {
 
 define <2 x float> @fneg(<2 x float> %x) {
 ; CHECK-LABEL: @fneg(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1]]
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %splat = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> zeroinitializer
   %r = fsub <2 x float> <float -0.0, float -0.0>, %splat
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index 83919e743d384..39f76f18b13ca 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -67,8 +67,7 @@ define float @test6(<4 x float> %X) {
 
 define float @testvscale6(<vscale x 4 x float> %X) {
 ; CHECK-LABEL: @testvscale6(
-; CHECK-NEXT:    [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[X:%.*]], i64 0
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
@@ -1382,9 +1381,9 @@ define <2 x float> @fsub_splat_constant1(<2 x float> %x) {
 
 define <2 x float> @fneg(<2 x float> %x) {
 ; CHECK-LABEL: @fneg(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1]]
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
   %r = fsub <2 x float> <float -0.0, float -0.0>, %splat
@@ -1906,9 +1905,9 @@ define <4 x i32> @PR46872(<4 x i32> %x) {
 
 define <2 x float> @fabs_unary_shuf(<2 x float> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = call nnan nsz <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP1]])
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = call nnan nsz <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP1:%.*]])
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %nx = call nsz nnan <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
   %r = shufflevector <2 x float> %nx, <2 x float> poison, <2 x i32> <i32 1, i32 0>
@@ -1917,9 +1916,9 @@ define <2 x float> @fabs_unary_shuf(<2 x float> %x) {
 
 define <4 x half> @fabs_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf_widen(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
-; CHECK-NEXT:    [[R:%.*]] = call ninf <4 x half> @llvm.fabs.v4f16(<4 x half> [[TMP1]])
-; CHECK-NEXT:    ret <4 x half> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = call ninf <2 x half> @llvm.fabs.v2f16(<2 x half> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
+; CHECK-NEXT:    ret <4 x half> [[TMP1]]
 ;
   %nx = call ninf <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
   %r = shufflevector <2 x half> %nx, <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
@@ -1928,9 +1927,9 @@ define <4 x half> @fabs_unary_shuf_widen(<2 x half> %x) {
 
 define <2 x double> @fabs_unary_shuf_narrow(<4 x double> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf_narrow(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = call nsz <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
-; CHECK-NEXT:    ret <2 x double> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = call nsz <4 x double> @llvm.fabs.v4f64(<4 x double> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %nx = call nsz <4 x double> @llvm.fabs.v4f64(<4 x double> %x)
   %r = shufflevector <4 x double> %nx, <4 x double> poison, <2 x i32> <i32 3, i32 0>
@@ -2021,9 +2020,9 @@ define <2 x float> @fabs_shuf_use3(<2 x float> %x, <2 x float> %y) {
 
 define <2 x float> @fneg_unary_shuf(<2 x float> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = fneg nnan nsz <2 x float> [[TMP1]]
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = fneg nnan nsz <2 x float> [[TMP1:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %nx = fneg nsz nnan <2 x float> %x
   %r = shufflevector <2 x float> %nx, <2 x float> poison, <2 x i32> <i32 1, i32 0>
@@ -2032,9 +2031,9 @@ define <2 x float> @fneg_unary_shuf(<2 x float> %x) {
 
 define <4 x half> @fneg_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf_widen(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
-; CHECK-NEXT:    [[R:%.*]] = fneg ninf <4 x half> [[TMP1]]
-; CHECK-NEXT:    ret <4 x half> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = fneg ninf <2 x half> [[X1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
+; CHECK-NEXT:    ret <4 x half> [[TMP1]]
 ;
   %nx = fneg ninf <2 x half> %x
   %r = shufflevector <2 x half> %nx, <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
@@ -2043,9 +2042,9 @@ define <4 x half> @fneg_unary_shuf_widen(<2 x half> %x) {
 
 define <2 x double> @fneg_unary_shuf_narrow(<4 x double> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf_narrow(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = fneg nsz <2 x double> [[TMP1]]
-; CHECK-NEXT:    ret <2 x double> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = fneg nsz <4 x double> [[X1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %nx = fneg nsz <4 x double> %x
   %r = shufflevector <4 x double> %nx, <4 x double> poison, <2 x i32> <i32 3, i32 0>
diff --git a/llvm/test/Transforms/InstCombine/vector-reverse.ll b/llvm/test/Transforms/InstCombine/vector-reverse.ll
index c9c68d2241b34..ee60d31d1ddbe 100644
--- a/llvm/test/Transforms/InstCombine/vector-reverse.ll
+++ b/llvm/test/Transforms/InstCombine/vector-reverse.ll
@@ -17,6 +17,18 @@ define <vscale x 4 x i32> @binop_reverse(<vscale x 4 x i32> %a, <vscale x 4 x i3
   ret <vscale x 4 x i32> %add
 }
 
+define <vscale x 4 x i32> @binop_intrinsic_reverse(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @binop_intrinsic_reverse(
+; CHECK-NEXT:    [[ADD:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[A_REV:%.*]], <vscale x 4 x i32> [[B_REV:%.*]])
+; CHECK-NEXT:    [[SMAX:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[SMAX]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %b)
+  %smax = call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a.rev, <vscale x 4 x i32> %b.rev)
+  ret <vscale x 4 x i32> %smax
+}
+
 ; %a.rev has multiple uses
 define <vscale x 4 x i32> @binop_reverse_1(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: @binop_reverse_1(
@@ -33,6 +45,22 @@ define <vscale x 4 x i32> @binop_reverse_1(<vscale x 4 x i32> %a, <vscale x 4 x
   ret <vscale x 4 x i32> %add
 }
 
+; %a.rev has multiple uses
+define <vscale x 4 x i32> @binop_intrinsic_reverse_1(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @binop_intrinsic_reverse_1(
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]])
+; CHECK-NEXT:    call void @use_nxv4i32(<vscale x 4 x i32> [[B_REV]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[B]], <vscale x 4 x i32> [[B1:%.*]])
+; CHECK-NEXT:    [[SMAX:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[SMAX]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %b)
+  call void @use_nxv4i32(<vscale x 4 x i32>  %a.rev)
+  %smax = call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a.rev, <vscale x 4 x i32> %b.rev)
+  ret <vscale x 4 x i32> %smax
+}
+
 ; %b.rev has multiple uses
 define <vscale x 4 x i32> @binop_reverse_2(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: @binop_reverse_2(
@@ -67,6 +95,24 @@ define <vscale x 4 x i32> @binop_reverse_3(<vscale x 4 x i32> %a, <vscale x 4 x
   ret <vscale x 4 x i32> %add
 }
 
+; %a.rev and %b.rev have multiple uses
+define <vscale x 4 x i32> @binop_intrinsic_reverse_3(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @binop_intrinsic_reverse_3(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]])
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]])
+; CHECK-NEXT:    call void @use_nxv4i32(<vscale x 4 x i32> [[A_REV]])
+; CHECK-NEXT:    call void @use_nxv4i32(<vscale x 4 x i32> [[B_REV]])
+; CHECK-NEXT:    [[SMAX:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[A_REV]], <vscale x 4 x i32> [[B_REV]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[SMAX]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %b)
+  call void @use_nxv4i32(<vscale x 4 x i32> %a.rev)
+  call void @use_nxv4i32(<vscale x 4 x i32> %b.rev)
+  %smax = call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a.rev, <vscale x 4 x i32> %b.rev)
+  ret <vscale x 4 x i32> %smax
+}
+
 ; %a.rev used as both operands
 define <vscale x 4 x i32> @binop_reverse_4(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: @binop_reverse_4(
@@ -161,9 +207,9 @@ define <vscale x 4 x i32> @binop_reverse_splat_LHS_1(<vscale x 4 x i32> %a, i32
 
 define <vscale x 4 x float> @unop_reverse(<vscale x 4 x float> %a) {
 ; CHECK-LABEL: @unop_reverse(
-; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[A:%.*]])
-; CHECK-NEXT:    [[NEG:%.*]] = fneg fast <vscale x 4 x float> [[A_REV]]
-; CHECK-NEXT:    ret <vscale x 4 x float> [[NEG]]
+; CHECK-NEXT:    [[NEG:%.*]] = fneg fast <vscale x 4 x float> [[A_REV:%.*]]
+; CHECK-NEXT:    [[NEG1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[NEG]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[NEG1]]
 ;
   %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
   %neg = fneg fast <vscale x 4 x float> %a.rev
@@ -184,6 +230,17 @@ define <vscale x 4 x float> @unop_reverse_1(<vscale x 4 x float> %a) {
   ret <vscale x 4 x float> %neg
 }
 
+define <vscale x 4 x float> @unop_intrinsic_reverse(<vscale x 4 x float> %a) {
+; CHECK-LABEL: @unop_intrinsic_reverse(
+; CHECK-NEXT:    [[NEG:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[NEG]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[ABS]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %abs = call <vscale x 4 x float> @llvm.fabs(<vscale x 4 x float> %a.rev)
+  ret <vscale x 4 x float> %abs
+}
+
 define <vscale x 4 x i1> @icmp_reverse(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: @icmp_reverse(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq <vscale x 4 x i32> [[A:%.*]], [[B:%.*]]
@@ -629,6 +686,18 @@ define <vscale x 4 x float> @reverse_binop_reverse(<vscale x 4 x float> %a, <vsc
   ret <vscale x 4 x float> %add.rev
 }
 
+define <vscale x 4 x float> @reverse_binop_intrinsic_reverse(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @reverse_binop_intrinsic_reverse(
+; CHECK-NEXT:    [[ADD:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]], <vscale x 4 x float> [[B_REV:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[ADD]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %b.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %b)
+  %maxnum = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a.rev, <vscale x 4 x float> %b.rev)
+  %maxnum.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %maxnum)
+  ret <vscale x 4 x float> %maxnum.rev
+}
+
 define <vscale x 4 x float> @reverse_binop_reverse_splat_RHS(<vscale x 4 x float> %a, float %b) {
 ; CHECK-LABEL: @reverse_binop_reverse_splat_RHS(
 ; CHECK-NEXT:    [[B_INSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[B:%.*]], i64 0
@@ -659,6 +728,63 @@ define <vscale x 4 x float> @reverse_binop_reverse_splat_LHS(<vscale x 4 x float
   ret <vscale x 4 x float> %div.rev
 }
 
+define <vscale x 4 x float> @reverse_binop_reverse_intrinsic_splat_RHS(<vscale x 4 x float> %a, float %b) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_splat_RHS(
+; CHECK-NEXT:    [[B_INSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[B:%.*]], i64 0
+; CHECK-NEXT:    [[B_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[B_INSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]], <vscale x 4 x float> [[B_SPLAT]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[MAXNUM]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %b.insert = insertelement <vscale x 4 x float> poison, float %b, i32 0
+  %b.splat = shufflevector <vscale x 4 x float> %b.insert, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+  %maxnum = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a.rev, <vscale x 4 x float> %b.splat)
+  %maxnum.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %maxnum)
+  ret <vscale x 4 x float> %maxnum.rev
+}
+
+define <vscale x 4 x float> @reverse_binop_reverse_intrinsic_splat_LHS(<vscale x 4 x float> %a, float %b) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_splat_LHS(
+; CHECK-NEXT:    [[B_INSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[B:%.*]], i64 0
+; CHECK-NEXT:    [[B_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[B_INSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[B_SPLAT]], <vscale x 4 x float> [[A_REV:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[MAXNUM]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %b.insert = insertelement <vscale x 4 x float> poison, float %b, i32 0
+  %b.splat = shufflevector <vscale x 4 x float> %b.insert, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+  %maxnum = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %b.splat, <vscale x 4 x float> %a.rev)
+  %maxnum.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %maxnum)
+  ret <vscale x 4 x float> %maxnum.rev
+}
+
+; Negative test: Make sure that splats with poison aren't considered splats
+define <4 x float> @reverse_binop_reverse_intrinsic_splat_with_poison(<4 x float> %a) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_splat_with_poison(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[A:%.*]], <4 x float> <float 1.000000e+00, float poison, float 1.000000e+00, float 1.000000e+00>)
+; CHECK-NEXT:    [[MAXNUM:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[MAXNUM_REV:%.*]] = tail call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> [[MAXNUM]])
+; CHECK-NEXT:    ret <4 x float> [[MAXNUM_REV]]
+;
+  %a.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %a)
+  %maxnum = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 1.0, float 1.0, float poison, float 1.0>, <4 x float> %a.rev)
+  %maxnum.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %maxnum)
+  ret <4 x float> %maxnum.rev
+}
+
+define <4 x float> @reverse_binop_reverse_intrinsic_constant_RHS(<4 x float> %a) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_constant_RHS(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[A:%.*]], <4 x float> <float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 0.000000e+00>)
+; CHECK-NEXT:    [[MAXNUM:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[MAXNUM_REV:%.*]] = tail call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> [[MAXNUM]])
+; CHECK-NEXT:    ret <4 x float> [[MAXNUM_REV]]
+;
+  %a.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %a)
+  %maxnum = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 0.0, float 1.0, float 2.0, float 3.0>, <4 x float> %a.rev)
+  %maxnum.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %maxnum)
+  ret <4 x float> %maxnum.rev
+}
+
 define <vscale x 4 x i1> @reverse_fcmp_reverse(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: @reverse_fcmp_reverse(
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp fast olt <vscale x 4 x float> [[A:%.*]], [[B:%.*]]
@@ -695,6 +821,27 @@ define <vscale x 4 x float> @reverse_unop_reverse(<vscale x 4 x float> %a) {
   ret <vscale x 4 x float> %neg.rev
 }
 
+define <vscale x 4 x float> @reverse_unop_intrinsic_reverse(<vscale x 4 x float> %a) {
+; CHECK-LABEL: @reverse_unop_intrinsic_reverse(
+; CHECK-NEXT:    [[ABS:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[ABS]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %abs = call <vscale x 4 x float> @llvm.fabs(<vscale x 4 x float> %a.rev)
+  %abs.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %abs)
+  ret <vscale x 4 x float> %abs.rev
+}
+
+define <vscale x 4 x float> @reverse_unop_intrinsic_reverse_scalar_arg(<vscale x 4 x float> %a, i32 %power) {
+; CHECK-LABEL: @reverse_unop_intrinsic_reverse_scalar_arg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> [[A:%.*]], i32 [[POWER:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32(<vscale x 4 x float> %a.rev, i32 %power)
+  %powi.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %powi)
+  ret <vscale x 4 x float> %powi.rev
+}
 
 declare void @use_nxv4i1(<vscale x 4 x i1>)
 declare void @use_nxv4i32(<vscale x 4 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
index 2655c20354607..36ed39a3d3242 100644
--- a/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
@@ -89,12 +89,12 @@ define i8 @extractelement_bitcast_insert_extra_use_bitcast(<vscale x 2 x i32> %a
   ret i8 %r
 }
 
+; while it may be that the extract is out-of-bounds, any valid index
+; is going to yield %v (because the mask is all-zeros).
+
 define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -104,10 +104,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 
 define i32 @extractelement_shuffle_invalid_index(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_invalid_index(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4294967295
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstCombine/vscale_extractelement.ll b/llvm/test/Transforms/InstCombine/vscale_extractelement.ll
index 07090e9099ae1..9ac8a92abb689 100644
--- a/llvm/test/Transforms/InstCombine/vscale_extractelement.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_extractelement.ll
@@ -53,12 +53,12 @@ define i8 @extractelement_bitcast_useless_insert(<vscale x 2 x i32> %a, i32 %x)
   ret i8 %r
 }
 
+; while in these tests it may be that the extract is out-of-bounds,
+; any valid index is going to yield %v (because the mask is all-zeros).
+
 define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> undef, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> undef, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -68,10 +68,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 
 define i32 @extractelement_shuffle_invalid_index(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_invalid_index(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> undef, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4294967295
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> undef, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 910750adf9d42..de71af414c25d 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -36,7 +36,7 @@ define {i8, i1} @test_uadd3(i8 %v) {
 
 define {i8, i1} @test_uadd3_poison(i8 %v) {
 ; CHECK-LABEL: @test_uadd3_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %v, i8 poison)
   ret {i8, i1} %result
@@ -52,7 +52,7 @@ define {i8, i1} @test_uadd4(i8 %v) {
 
 define {i8, i1} @test_uadd4_poison(i8 %v) {
 ; CHECK-LABEL: @test_uadd4_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 poison, i8 %v)
   ret {i8, i1} %result
@@ -86,7 +86,7 @@ define {i8, i1} @test_sadd3(i8 %v) {
 
 define {i8, i1} @test_sadd3_poison(i8 %v) {
 ; CHECK-LABEL: @test_sadd3_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %v, i8 poison)
   ret {i8, i1} %result
@@ -102,7 +102,7 @@ define {i8, i1} @test_sadd4(i8 %v) {
 
 define {i8, i1} @test_sadd4_poison(i8 %v) {
 ; CHECK-LABEL: @test_sadd4_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 poison, i8 %v)
   ret {i8, i1} %result
@@ -126,7 +126,7 @@ define {i8, i1} @test_usub2(i8 %V) {
 
 define {i8, i1} @test_usub2_poison(i8 %V) {
 ; CHECK-LABEL: @test_usub2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -142,7 +142,7 @@ define {i8, i1} @test_usub3(i8 %V) {
 
 define {i8, i1} @test_usub3_poison(i8 %V) {
 ; CHECK-LABEL: @test_usub3_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
@@ -166,7 +166,7 @@ define {i8, i1} @test_ssub2(i8 %V) {
 
 define {i8, i1} @test_ssub2_poison(i8 %V) {
 ; CHECK-LABEL: @test_ssub2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -182,7 +182,7 @@ define {i8, i1} @test_ssub3(i8 %V) {
 
 define {i8, i1} @test_ssub3_poison(i8 %V) {
 ; CHECK-LABEL: @test_ssub3_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
@@ -206,7 +206,7 @@ define {i8, i1} @test_umul2(i8 %V) {
 
 define {i8, i1} @test_umul2_poison(i8 %V) {
 ; CHECK-LABEL: @test_umul2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -230,7 +230,7 @@ define {i8, i1} @test_umul4(i8 %V) {
 
 define {i8, i1} @test_umul4_poison(i8 %V) {
 ; CHECK-LABEL: @test_umul4_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
@@ -254,7 +254,7 @@ define {i8, i1} @test_smul2(i8 %V) {
 
 define {i8, i1} @test_smul2_poison(i8 %V) {
 ; CHECK-LABEL: @test_smul2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -278,7 +278,7 @@ define {i8, i1} @test_smul4(i8 %V) {
 
 define {i8, i1} @test_smul4_poison(i8 %V) {
 ; CHECK-LABEL: @test_smul4_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
diff --git a/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll b/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
index e45aa3fd09ce0..8578aa9fa84b3 100644
--- a/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
@@ -33,7 +33,7 @@ define void @powi(double %V, ptr%P) {
 define void @powi_i16(float %V, ptr%P) {
 ; CHECK-LABEL: @powi_i16(
 ; CHECK-NEXT:    store volatile float 1.000000e+00, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    store volatile float [[V:%.*]], ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float [[D:%.*]], ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %B = tail call float @llvm.powi.f32.i16(float %V, i16 0) nounwind
@@ -44,3 +44,46 @@ define void @powi_i16(float %V, ptr%P) {
 
   ret void
 }
+
+define i32 @test_ctpop_poison(i32 %a) {
+; CHECK-LABEL: @test_ctpop_poison(
+; CHECK-NEXT:    ret i32 poison
+;
+  %res = tail call i32 @llvm.ctpop.i32(i32 poison)
+  ret i32 %res
+}
+
+define void @pow_poison(i16 %arg_int,float %arg_flt, ptr %P) {
+; CHECK-LABEL: @pow_poison(
+; CHECK-NEXT:    store volatile float poison, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile <2 x float> poison, ptr [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %2 = tail call float @llvm.powi(float poison, i16 %arg_int) nounwind
+  store volatile float %2, ptr %P
+
+  %3 = tail call float @llvm.pow(float poison, float %arg_flt) nounwind
+  store volatile float %3, ptr %P
+
+  %4 = tail call float @llvm.powi(float %arg_flt, i16 poison) nounwind
+  store volatile float %4, ptr %P
+
+  %5 = tail call float @llvm.pow(float %arg_flt, float poison) nounwind
+  store volatile float %5, ptr %P
+
+  %6 = tail call float @llvm.powi(float poison, i16 poison) nounwind
+  store volatile float %6, ptr %P
+
+  %7 = tail call float @llvm.pow(float poison, float poison) nounwind
+  store volatile float %7, ptr %P
+
+  %8 = tail call <2 x float> @llvm.pow(<2 x float> poison, <2 x float> poison) nounwind
+  store volatile <2 x float> %8, ptr %P
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index 25a35062e60b8..ead4c01535163 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -90,7 +90,7 @@ define i8 @uadd_scalar_undef(i8 %a) {
 
 define i8 @uadd_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @uadd_scalar_poison(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %x5 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 poison)
   ret i8 %x5
@@ -106,7 +106,7 @@ define <2 x i8> @uadd_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @uadd_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 poison, i8 poison>)
   ret <2 x i8> %x5v
@@ -122,7 +122,7 @@ define i8 @uadd_scalar_undef_commute(i8 %a) {
 
 define i8 @uadd_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @uadd_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %x6 = call i8 @llvm.uadd.sat.i8(i8 poison, i8 %a)
   ret i8 %x6
@@ -138,7 +138,7 @@ define <2 x i8> @uadd_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @uadd_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> %a)
   ret <2 x i8> %x5v
@@ -222,7 +222,7 @@ define i8 @sadd_scalar_undef(i8 %a) {
 
 define i8 @sadd_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @sadd_scalar_poison(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %y5 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 poison)
   ret i8 %y5
@@ -238,7 +238,7 @@ define <2 x i8> @sadd_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @sadd_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @sadd_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y5v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> poison)
   ret <2 x i8> %y5v
@@ -254,7 +254,7 @@ define i8 @sadd_scalar_undef_commute(i8 %a) {
 
 define i8 @sadd_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @sadd_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %y6 = call i8 @llvm.sadd.sat.i8(i8 poison, i8 %a)
   ret i8 %y6
@@ -270,7 +270,7 @@ define <2 x i8> @sadd_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @sadd_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @sadd_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y6v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> %a)
   ret <2 x i8> %y6v
@@ -334,7 +334,7 @@ define i8 @usub_scalar_undef(i8 %a) {
 
 define i8 @usub_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_poison(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %x4 = call i8 @llvm.usub.sat.i8(i8 %a, i8 poison)
   ret i8 %x4
@@ -350,7 +350,7 @@ define <2 x i8> @usub_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @usub_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @usub_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x4v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 poison, i8 poison>)
   ret <2 x i8> %x4v
@@ -366,7 +366,7 @@ define i8 @usub_scalar_undef_commute(i8 %a) {
 
 define i8 @usub_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %x5 = call i8 @llvm.usub.sat.i8(i8 poison, i8 %a)
   ret i8 %x5
@@ -382,7 +382,7 @@ define <2 x i8> @usub_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @usub_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @usub_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x5v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 poison, i8 poison>, <2 x i8> %a)
   ret <2 x i8> %x5v
@@ -466,7 +466,7 @@ define i8 @ssub_scalar_undef(i8 %a) {
 
 define i8 @ssub_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @ssub_scalar_poison(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %y4 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 poison)
   ret i8 %y4
@@ -482,7 +482,7 @@ define <2 x i8> @ssub_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @ssub_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @ssub_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y4v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> poison)
   ret <2 x i8> %y4v
@@ -498,7 +498,7 @@ define i8 @ssub_scalar_undef_commute(i8 %a) {
 
 define i8 @ssub_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @ssub_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %y5 = call i8 @llvm.ssub.sat.i8(i8 poison, i8 %a)
   ret i8 %y5
@@ -514,7 +514,7 @@ define <2 x i8> @ssub_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @ssub_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @ssub_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y5v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 poison, i8 poison>, <2 x i8> %a)
   ret <2 x i8> %y5v
diff --git a/llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll b/llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll
new file mode 100644
index 0000000000000..806bd820009b1
--- /dev/null
+++ b/llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -passes=licm %s | FileCheck %s
+
+; Check that hoistMinMax() in LICM salvages the debug values for the hoisted
+; cmp instructions.
+
+define i32 @test_ult(i32 %start, i32 %inv_1, i32 %inv_2) !dbg !5 {
+; CHECK-LABEL: define i32 @test_ult(
+; CHECK-SAME: i32 [[START:%.*]], i32 [[INV_1:%.*]], i32 [[INV_2:%.*]])
+; CHECK:       [[LOOP:.*]]:
+; CHECK:           #dbg_value(!DIArgList(i32 [[IV:%.*]], i32 [[INV_1]]), [[META9:![0-9]+]], !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_lt, DW_OP_stack_value), [[META14:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(!DIArgList(i32 [[IV]], i32 [[INV_2]]), [[META11:![0-9]+]], !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_lt, DW_OP_stack_value), [[META15:![0-9]+]])
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV]], [[INVARIANT_UMIN:%.*]], !dbg [[DBG16:![0-9]+]]
+entry:
+  br label %loop, !dbg !16
+
+loop:                                             ; preds = %loop, %entry
+  %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ], !dbg !17
+  %cmp_1 = icmp ult i32 %iv, %inv_1, !dbg !18
+    #dbg_value(i1 %cmp_1, !11, !DIExpression(), !18)
+  %cmp_2 = icmp ult i32 %iv, %inv_2, !dbg !19
+    #dbg_value(i1 %cmp_2, !13, !DIExpression(), !19)
+  %loop_cond = and i1 %cmp_1, %cmp_2, !dbg !20
+  %iv.next = add i32 %iv, 1, !dbg !21
+  br i1 %loop_cond, label %loop, label %exit, !dbg !22
+
+exit:                                             ; preds = %loop
+  ret i32 %iv, !dbg !23
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "temp.ll", directory: "/")
+!2 = !{i32 8}
+!3 = !{i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "test_ult", linkageName: "test_ult", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!11, !13}
+!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 3, type: !12)
+!12 = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_unsigned)
+!13 = !DILocalVariable(name: "3", scope: !5, file: !1, line: 4, type: !12)
+!16 = !DILocation(line: 1, column: 1, scope: !5)
+!17 = !DILocation(line: 2, column: 1, scope: !5)
+!18 = !DILocation(line: 3, column: 1, scope: !5)
+!19 = !DILocation(line: 4, column: 1, scope: !5)
+!20 = !DILocation(line: 5, column: 1, scope: !5)
+!21 = !DILocation(line: 6, column: 1, scope: !5)
+!22 = !DILocation(line: 7, column: 1, scope: !5)
+!23 = !DILocation(line: 8, column: 1, scope: !5)
+;.
+; CHECK: [[META9]] = !DILocalVariable(name: "2",
+; CHECK: [[META11]] = !DILocalVariable(name: "3",
+; CHECK: [[META14]] = !DILocation(line: 3,
+; CHECK: [[META15]] = !DILocation(line: 4,
+;.
diff --git a/llvm/test/Transforms/LoopFusion/four_loops.ll b/llvm/test/Transforms/LoopFusion/four_loops.ll
index 7f7f0f19b59ba..1728686ebc15c 100644
--- a/llvm/test/Transforms/LoopFusion/four_loops.ll
+++ b/llvm/test/Transforms/LoopFusion/four_loops.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 
 @A = common global [1024 x i32] zeroinitializer, align 16
@@ -5,21 +6,77 @@
 @C = common global [1024 x i32] zeroinitializer, align 16
 @D = common global [1024 x i32] zeroinitializer, align 16
 
-; CHECK: void @dep_free
-; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
-; CHECK: [[LOOP1HEADER]]
-; CHECK: br label %[[LOOP2BODY:bb[0-9]+]]
-; CHECK: [[LOOP2BODY]]
-; CHECK: br label %[[LOOP3BODY:bb[0-9]+]]
-; CHECK: [[LOOP3BODY]]
-; CHECK: br label %[[LOOP4BODY:bb[0-9]+]]
-; CHECK: [[LOOP4BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]+]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[LOOPEXIT:bb[0-9]+]]
-; CHECK: ret void
 define void @dep_free() {
+; CHECK-LABEL: define void @dep_free() {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB15:.*]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[DOT08:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP23:%.*]], %[[BB61:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV107:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT11:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[DOT016:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP36:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[INDVARS_IV75:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT8:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[DOT024:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP49:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[INDVARS_IV43:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT5:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[DOT032:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP62:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT08]], -3
+; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw i64 [[INDVARS_IV107]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nsw i32 [[TMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[INDVARS_IV107]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = srem i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 [[INDVARS_IV107]]
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP21]], align 4
+; CHECK-NEXT:    br label %[[BB22:.*]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP28:%.*]] = add nsw i32 [[DOT016]], -3
+; CHECK-NEXT:    [[TMP29:%.*]] = add nuw nsw i64 [[INDVARS_IV75]], 3
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = mul nsw i32 [[TMP28]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = trunc i64 [[INDVARS_IV75]] to i32
+; CHECK-NEXT:    [[TMP33:%.*]] = srem i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[INDVARS_IV75]]
+; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP34]], align 4
+; CHECK-NEXT:    br label %[[BB35:.*]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    [[TMP41:%.*]] = add nsw i32 [[DOT024]], -3
+; CHECK-NEXT:    [[TMP42:%.*]] = add nuw nsw i64 [[INDVARS_IV43]], 3
+; CHECK-NEXT:    [[TMP43:%.*]] = trunc i64 [[TMP42]] to i32
+; CHECK-NEXT:    [[TMP44:%.*]] = mul nsw i32 [[TMP41]], [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = trunc i64 [[INDVARS_IV43]] to i32
+; CHECK-NEXT:    [[TMP46:%.*]] = srem i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [1024 x i32], ptr @C, i64 0, i64 [[INDVARS_IV43]]
+; CHECK-NEXT:    store i32 [[TMP46]], ptr [[TMP47]], align 4
+; CHECK-NEXT:    br label %[[BB48:.*]]
+; CHECK:       [[BB48]]:
+; CHECK-NEXT:    [[TMP54:%.*]] = add nsw i32 [[DOT032]], -3
+; CHECK-NEXT:    [[TMP55:%.*]] = add nuw nsw i64 [[INDVARS_IV1]], 3
+; CHECK-NEXT:    [[TMP56:%.*]] = trunc i64 [[TMP55]] to i32
+; CHECK-NEXT:    [[TMP57:%.*]] = mul nsw i32 [[TMP54]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = trunc i64 [[INDVARS_IV1]] to i32
+; CHECK-NEXT:    [[TMP59:%.*]] = srem i32 [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [1024 x i32], ptr @D, i64 0, i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP59]], ptr [[TMP60]], align 4
+; CHECK-NEXT:    br label %[[BB61]]
+; CHECK:       [[BB52:.*]]:
+; CHECK-NEXT:    br label %[[BB63:.*]]
+; CHECK:       [[BB61]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT11]] = add nuw nsw i64 [[INDVARS_IV107]], 1
+; CHECK-NEXT:    [[TMP23]] = add nuw nsw i32 [[DOT08]], 1
+; CHECK-NEXT:    [[EXITCOND12:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT11]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT8]] = add nuw nsw i64 [[INDVARS_IV75]], 1
+; CHECK-NEXT:    [[TMP36]] = add nuw nsw i32 [[DOT016]], 1
+; CHECK-NEXT:    [[EXITCOND9:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT8]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT5]] = add nuw nsw i64 [[INDVARS_IV43]], 1
+; CHECK-NEXT:    [[TMP49]] = add nuw nsw i32 [[DOT024]], 1
+; CHECK-NEXT:    [[EXITCOND6:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT5]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[TMP62]] = add nuw nsw i32 [[DOT032]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[BB15]], label %[[BB52]]
+; CHECK:       [[BB63]]:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb15
 
diff --git a/llvm/test/Transforms/LoopFusion/guarded_peel.ll b/llvm/test/Transforms/LoopFusion/guarded_peel.ll
index c45772b0b0293..c26935908d4d6 100644
--- a/llvm/test/Transforms/LoopFusion/guarded_peel.ll
+++ b/llvm/test/Transforms/LoopFusion/guarded_peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests if we are able to fuse two guarded loops which have constant but
@@ -6,33 +7,75 @@
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
-; CHECK-LABEL: void @main(ptr noalias %A)
-; CHECK-NEXT:  entry:
-; CHECK:         br i1 %cmp4, label %for.first.entry, label %for.end
-; CHECK:       for.first.entry
-; CHECK-NEXT:    br label %for.first.peel.begin
-; CHECK:       for.first.peel.begin:
-; CHECK-NEXT:    br label %for.first.peel
-; CHECK:       for.first.peel:
-; CHECK:         br label %for.first.peel.next
-; CHECK:       for.first.peel.next:
-; CHECK-NEXT:    br label %for.first.peel2
-; CHECK:       for.first.peel2:
-; CHECK:         br label %for.first.peel.next1
-; CHECK:       for.first.peel.next1:
-; CHECK-NEXT:    br label %for.first.peel.next11
-; CHECK:       for.first.peel.next11:
-; CHECK-NEXT:    br label %for.first.entry.peel.newph
-; CHECK:       for.first.entry.peel.newph:
-; CHECK:         br label %for.first
-; CHECK:       for.first:
-; CHECK:         br i1 %cmp3, label %for.first, label %for.second.exit
-; CHECK:       for.second.exit:
-; CHECK:         br label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 define void @main(ptr noalias %A) {
+; CHECK-LABEL: define void @main(
+; CHECK-SAME: ptr noalias [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i64 0, 45
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp slt i64 2, 45
+; CHECK-NEXT:    br i1 [[CMP4]], label %[[FOR_FIRST_ENTRY:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_FIRST_ENTRY]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_BEGIN:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_BEGIN]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL:.*]]
+; CHECK:       [[FOR_FIRST_PEEL]]:
+; CHECK-NEXT:    [[SUB_PEEL:%.*]] = sub nsw i64 0, 3
+; CHECK-NEXT:    [[ADD_PEEL:%.*]] = add nsw i64 0, 3
+; CHECK-NEXT:    [[MUL_PEEL:%.*]] = mul nsw i64 [[SUB_PEEL]], [[ADD_PEEL]]
+; CHECK-NEXT:    [[REM_PEEL:%.*]] = srem i64 [[MUL_PEEL]], 0
+; CHECK-NEXT:    [[CONV_PEEL:%.*]] = trunc i64 [[REM_PEEL]] to i32
+; CHECK-NEXT:    [[ARRAYIDX_PEEL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 0
+; CHECK-NEXT:    store i32 [[CONV_PEEL]], ptr [[ARRAYIDX_PEEL]], align 4
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i64 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i64 [[INC_PEEL]], 45
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL2:.*]]
+; CHECK:       [[FOR_FIRST_PEEL2]]:
+; CHECK-NEXT:    [[SUB_PEEL3:%.*]] = sub nsw i64 [[INC_PEEL]], 3
+; CHECK-NEXT:    [[ADD_PEEL4:%.*]] = add nsw i64 [[INC_PEEL]], 3
+; CHECK-NEXT:    [[MUL_PEEL5:%.*]] = mul nsw i64 [[SUB_PEEL3]], [[ADD_PEEL4]]
+; CHECK-NEXT:    [[REM_PEEL6:%.*]] = srem i64 [[MUL_PEEL5]], [[INC_PEEL]]
+; CHECK-NEXT:    [[CONV_PEEL7:%.*]] = trunc i64 [[REM_PEEL6]] to i32
+; CHECK-NEXT:    [[ARRAYIDX_PEEL8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INC_PEEL]]
+; CHECK-NEXT:    store i32 [[CONV_PEEL7]], ptr [[ARRAYIDX_PEEL8]], align 4
+; CHECK-NEXT:    [[INC_PEEL9:%.*]] = add nsw i64 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL10:%.*]] = icmp slt i64 [[INC_PEEL9]], 45
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT1:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT1]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT11:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT11]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_ENTRY_PEEL_NEWPH:.*]]
+; CHECK:       [[FOR_FIRST_ENTRY_PEEL_NEWPH]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_FIRST]] ], [ [[INC_PEEL9]], %[[FOR_FIRST_ENTRY_PEEL_NEWPH]] ]
+; CHECK-NEXT:    [[I1_02:%.*]] = phi i64 [ [[INC14:%.*]], %[[FOR_FIRST]] ], [ 2, %[[FOR_FIRST_ENTRY_PEEL_NEWPH]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[I_05]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[I_05]], 3
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[SUB]], [[ADD]]
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[MUL]], [[I_05]]
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[REM]] to i32
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_05]]
+; CHECK-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INC]], 45
+; CHECK-NEXT:    [[SUB7:%.*]] = sub nsw i64 [[I1_02]], 3
+; CHECK-NEXT:    [[ADD8:%.*]] = add nsw i64 [[I1_02]], 3
+; CHECK-NEXT:    [[MUL9:%.*]] = mul nsw i64 [[SUB7]], [[ADD8]]
+; CHECK-NEXT:    [[REM10:%.*]] = srem i64 [[MUL9]], [[I1_02]]
+; CHECK-NEXT:    [[CONV11:%.*]] = trunc i64 [[REM10]] to i32
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[I1_02]]
+; CHECK-NEXT:    store i32 [[CONV11]], ptr [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[INC14]] = add nsw i64 [[I1_02]], 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i64 [[INC14]], 45
+; CHECK-NEXT:    br i1 [[CMP3]], label %[[FOR_FIRST]], label %[[FOR_SECOND_EXIT:.*]]
+; CHECK:       [[FOR_SECOND_EXIT]]:
+; CHECK-NEXT:    br label %[[FOR_END]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp4 = icmp slt i64 0, 45
   br i1 %cmp4, label %for.first.entry, label %for.second.guard
diff --git a/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll b/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
index 1ab9e538a3dcb..add32738bc5a7 100644
--- a/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
+++ b/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests that we do not fuse two guarded loops together.
@@ -6,28 +7,42 @@
 ; loops unsafe to fuse together.
 ; The expected output of this test is the function as below.
 
-; CHECK-LABEL: void @unsafe_exitblock(ptr noalias %A, ptr noalias %B)
-; CHECK:       for.first.guard
-; CHECK:         br i1 %cmp3, label %for.first.preheader, label %for.second.guard
-; CHECK:       for.first.preheader:
-; CHECK-NEXT:    br label %for.first
-; CHECK:       for.first:
-; CHECK:         br i1 %cmp, label %for.first, label %for.first.exit
-; CHECK:       for.first.exit:
-; CHECK-NEXT:    call void @bar()
-; CHECK-NEXT:    br label %for.second.guard
-; CHECK:       for.second.guard:
-; CHECK:         br i1 %cmp21, label %for.second.preheader, label %for.end
-; CHECK:       for.second.preheader:
-; CHECK-NEXT:    br label %for.second
-; CHECK:       for.second:
-; CHECK:         br i1 %cmp2, label %for.second, label %for.second.exit
-; CHECK:       for.second.exit:
-; CHECK-NEXT:    br label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 define void @unsafe_exitblock(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @unsafe_exitblock(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  [[FOR_FIRST_GUARD:.*:]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i64 0, 45
+; CHECK-NEXT:    br i1 [[CMP3]], label %[[FOR_FIRST_PREHEADER:.*]], label %[[FOR_SECOND_GUARD:.*]]
+; CHECK:       [[FOR_FIRST_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_FIRST]] ], [ 0, %[[FOR_FIRST_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_04]]
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_04]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INC]], 45
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_FIRST]], label %[[FOR_FIRST_EXIT:.*]]
+; CHECK:       [[FOR_FIRST_EXIT]]:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label %[[FOR_SECOND_GUARD]]
+; CHECK:       [[FOR_SECOND_GUARD]]:
+; CHECK-NEXT:    [[CMP21:%.*]] = icmp slt i64 2, 45
+; CHECK-NEXT:    br i1 [[CMP21]], label %[[FOR_SECOND_PREHEADER:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_SECOND_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_SECOND:.*]]
+; CHECK:       [[FOR_SECOND]]:
+; CHECK-NEXT:    [[J_02:%.*]] = phi i64 [ [[INC6:%.*]], %[[FOR_SECOND]] ], [ 2, %[[FOR_SECOND_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[J_02]]
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[INC6]] = add nsw i64 [[J_02]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i64 [[INC6]], 45
+; CHECK-NEXT:    br i1 [[CMP2]], label %[[FOR_SECOND]], label %[[FOR_SECOND_EXIT:.*]]
+; CHECK:       [[FOR_SECOND_EXIT]]:
+; CHECK-NEXT:    br label %[[FOR_END]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 for.first.guard:
   %cmp3 = icmp slt i64 0, 45
   br i1 %cmp3, label %for.first.preheader, label %for.second.guard
diff --git a/llvm/test/Transforms/LoopFusion/hoist_preheader.ll b/llvm/test/Transforms/LoopFusion/hoist_preheader.ll
index 433090fbd994d..89698f5cc8060 100644
--- a/llvm/test/Transforms/LoopFusion/hoist_preheader.ll
+++ b/llvm/test/Transforms/LoopFusion/hoist_preheader.ll
@@ -1,16 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 
 define void @hoist_preheader(i32 %N) {
-
-; CHECK:pre1:
-; CHECK-NEXT:  %hoistme = add i32 1, %N
-; CHECK-NEXT:  %hoistme2 = add i32 1, %hoistme
-; CHECK-NEXT:  br label %body1
+; CHECK-LABEL: define void @hoist_preheader(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[PRE1:.*]]:
+; CHECK-NEXT:    [[HOISTME:%.*]] = add i32 1, [[N]]
+; CHECK-NEXT:    [[HOISTME2:%.*]] = add i32 1, [[HOISTME]]
+; CHECK-NEXT:    br label %[[BODY1:.*]]
+; CHECK:       [[BODY1]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
+; CHECK-NEXT:    [[I2:%.*]] = phi i32 [ [[I_NEXT2:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 1, [[I]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[I]], [[N]]
+; CHECK-NEXT:    [[I_NEXT2]] = add i32 1, [[I2]]
+; CHECK-NEXT:    [[COND2:%.*]] = icmp ne i32 [[I2]], [[N]]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BODY1]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 pre1:
   br label %body1
 
-; CHECK: body1:
-; CHECK-NOT:  %hoistme
 body1:  ; preds = %pre1, %body1
   %i = phi i32 [%i_next, %body1], [0, %pre1]
   %i_next = add i32 1, %i
diff --git a/llvm/test/Transforms/LoopFusion/inner_loops.ll b/llvm/test/Transforms/LoopFusion/inner_loops.ll
index 3c90905339fa7..88ac6621baff2 100644
--- a/llvm/test/Transforms/LoopFusion/inner_loops.ll
+++ b/llvm/test/Transforms/LoopFusion/inner_loops.ll
@@ -1,26 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s 2>&1 | FileCheck %s
 
 @A = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 @B = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 
-; CHECK: void @dep_free
-; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
-; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
-; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
-; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
-; CHECK: ret void
 
 define void @dep_free() {
+; CHECK-LABEL: define void @dep_free() {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB9:.*]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[INDVARS_IV6:%.*]] = phi i64 [ [[INDVARS_IV_NEXT7:%.*]], %[[BB35:.*]] ], [ 0, %[[BB]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP36:%.*]], %[[BB35]] ]
+; CHECK-NEXT:    [[EXITCOND8:%.*]] = icmp ne i64 [[INDVARS_IV6]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND8]], label %[[BB11:.*]], label %[[BB10:.*]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    br label %[[BB37:.*]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    br label %[[BB12:.*]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[BB21:.*]] ], [ 0, %[[BB11]] ]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[BB14:.*]], label %[[BB23_PREHEADER:.*]]
+; CHECK:       [[BB23_PREHEADER]]:
+; CHECK-NEXT:    br label %[[BB23:.*]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT0]], -3
+; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[INDVARS_IV6]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = mul nsw i32 [[TMP]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[INDVARS_IV6]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = srem i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @A, i64 0, i64 [[INDVARS_IV6]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP20]], align 4
+; CHECK-NEXT:    br label %[[BB21]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br label %[[BB12]]
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[BB33:.*]] ], [ 0, %[[BB23_PREHEADER]] ]
+; CHECK-NEXT:    [[EXITCOND5:%.*]] = icmp ne i64 [[INDVARS_IV3]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND5]], label %[[BB25:.*]], label %[[BB35]]
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[TMP26:%.*]] = add nsw i32 [[DOT0]], -3
+; CHECK-NEXT:    [[TMP27:%.*]] = add nuw nsw i64 [[INDVARS_IV6]], 3
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP29:%.*]] = mul nsw i32 [[TMP26]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[INDVARS_IV6]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = srem i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @B, i64 0, i64 [[INDVARS_IV6]], i64 [[INDVARS_IV3]]
+; CHECK-NEXT:    store i32 [[TMP31]], ptr [[TMP32]], align 4
+; CHECK-NEXT:    br label %[[BB33]]
+; CHECK:       [[BB33]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nuw nsw i64 [[INDVARS_IV3]], 1
+; CHECK-NEXT:    br label %[[BB23]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT7]] = add nuw nsw i64 [[INDVARS_IV6]], 1
+; CHECK-NEXT:    [[TMP36]] = add nuw nsw i32 [[DOT0]], 1
+; CHECK-NEXT:    br label %[[BB9]]
+; CHECK:       [[BB37]]:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb9
 
diff --git a/llvm/test/Transforms/LoopFusion/loop_nest.ll b/llvm/test/Transforms/LoopFusion/loop_nest.ll
index 260e7c86eeed0..39dcd19fb05ab 100644
--- a/llvm/test/Transforms/LoopFusion/loop_nest.ll
+++ b/llvm/test/Transforms/LoopFusion/loop_nest.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 ;
 ;    int A[1024][1024];
@@ -19,28 +20,61 @@
 @A = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 @B = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 
-; CHECK: void @dep_free
-; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
-; CHECK: [[LOOP1HEADER]]
-; CHECK: br label %[[LOOP3HEADER:bb[0-9]+]]
-; CHECK: [[LOOP3HEADER]]
-; CHECK: br label %[[LOOP2HEADER:bb[0-9]+]]
-; CHECK: [[LOOP2HEADER]]
-; CHECK: br label %[[LOOP4HEADER:bb[0-9]+]]
-; CHECK: [[LOOP4HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP3HEADER]], label %[[LOOP1LATCH:bb[0-9]+]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK-NEXT: %inc.outer.fc0 = add nuw nsw i64 %indvars.iv105, 1
-; CHECK-NEXT: %add.outer.fc0 = add nuw nsw i32 %.06, 1
-; CHECK-NEXT: %cmp.outer.fc0 = icmp ne i64 %inc.outer.fc0, 100
-; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[LOOP1EXIT:bb[0-9]*]]
-; CHECK: ret void
 
 ; TODO: The current version of loop fusion does not allow the inner loops to be
 ; fused because they are not control flow equivalent and adjacent. These are
 ; limitations that can be addressed in future improvements to fusion.
 define void @dep_free() {
+; CHECK-LABEL: define void @dep_free() {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB16:.*]]
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[DOT06:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[ADD_OUTER_FC0:%.*]], %[[BB45:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV105:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INC_OUTER_FC0:%.*]], %[[BB45]] ]
+; CHECK-NEXT:    [[DOT023:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP46:%.*]], %[[BB45]] ]
+; CHECK-NEXT:    [[INDVARS_IV42:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT5:%.*]], %[[BB45]] ]
+; CHECK-NEXT:    br label %[[BB18:.*]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[INDVARS_IV74:%.*]] = phi i64 [ 0, %[[BB16]] ], [ [[INDVARS_IV_NEXT8:%.*]], %[[BB43:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, %[[BB16]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[BB43]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT06]], -3
+; CHECK-NEXT:    [[TMP19:%.*]] = add nuw nsw i64 [[INDVARS_IV105]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nsw i32 [[TMP]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[INDVARS_IV105]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = srem i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @A, i64 0, i64 [[INDVARS_IV105]], i64 [[INDVARS_IV74]]
+; CHECK-NEXT:    store i32 [[TMP23]], ptr [[TMP24]], align 4
+; CHECK-NEXT:    br label %[[BB25:.*]]
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[TMP36:%.*]] = add nsw i32 [[DOT023]], -3
+; CHECK-NEXT:    [[TMP37:%.*]] = add nuw nsw i64 [[INDVARS_IV42]], 3
+; CHECK-NEXT:    [[TMP38:%.*]] = trunc i64 [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = mul nsw i32 [[TMP36]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = trunc i64 [[INDVARS_IV42]] to i32
+; CHECK-NEXT:    [[TMP41:%.*]] = srem i32 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @B, i64 0, i64 [[INDVARS_IV42]], i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP41]], ptr [[TMP42]], align 4
+; CHECK-NEXT:    br label %[[BB43]]
+; CHECK:       [[BB31:.*]]:
+; CHECK-NEXT:    br label %[[BB47:.*]]
+; CHECK:       [[BB43]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT8]] = add nuw nsw i64 [[INDVARS_IV74]], 1
+; CHECK-NEXT:    [[EXITCOND9:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT8]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[BB18]], label %[[BB45]]
+; CHECK:       [[BB45]]:
+; CHECK-NEXT:    [[INC_OUTER_FC0]] = add nuw nsw i64 [[INDVARS_IV105]], 1
+; CHECK-NEXT:    [[ADD_OUTER_FC0]] = add nuw nsw i32 [[DOT06]], 1
+; CHECK-NEXT:    [[CMP_OUTER_FC0:%.*]] = icmp ne i64 [[INC_OUTER_FC0]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT5]] = add nuw nsw i64 [[INDVARS_IV42]], 1
+; CHECK-NEXT:    [[TMP46]] = add nuw nsw i32 [[DOT023]], 1
+; CHECK-NEXT:    [[EXITCOND6:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT5]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND6]], label %[[BB16]], label %[[BB31]]
+; CHECK:       [[BB47]]:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb16
 
diff --git a/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll b/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
index 1bc70ef4f3aaf..c9e57b671a5a9 100644
--- a/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
+++ b/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests that we do not fuse these two loops together. These loops do not have
@@ -6,29 +7,57 @@
 ; peeling).
 ; The expected output of this test is the function below.
 
-; CHECK-LABEL: void @function(ptr noalias %arg)
-; CHECK-NEXT:  for.first.preheader:
-; CHECK-NEXT:    br label %for.first
-; CHECK:       for.first:
-; CHECK:         br label %for.first.latch
-; CHECK:       for.first.latch:
-; CHECK:         br i1 %exitcond4, label %for.first, label %for.first.exit
-; CHECK:       for.first.exit:
-; CHECK-NEXT:    br label %for.next
-; CHECK:       for.next:
-; CHECK-NEXT:    br label %for.second.preheader
-; CHECK:       for.second.preheader:
-; CHECK:         br label %for.second
-; CHECK:       for.second:
-; CHECK:         br label %for.second.latch
-; CHECK:       for.second.latch:
-; CHECK:         br i1 %exitcond, label %for.second, label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
 define void @function(ptr noalias %arg) {
+; CHECK-LABEL: define void @function(
+; CHECK-SAME: ptr noalias [[ARG:%.*]]) {
+; CHECK-NEXT:  [[FOR_FIRST_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[DOT014:%.*]] = phi i32 [ 0, %[[FOR_FIRST_PREHEADER]] ], [ [[TMP15:%.*]], %[[FOR_FIRST_LATCH:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ 0, %[[FOR_FIRST_PREHEADER]] ], [ [[INDVARS_IV_NEXT3:%.*]], %[[FOR_FIRST_LATCH]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT014]], -3
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[INDVARS_IV23]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i32 [[TMP]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[INDVARS_IV23]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = srem i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    store i32 [[TMP12]], ptr [[TMP13]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH]]
+; CHECK:       [[FOR_FIRST_LATCH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3]] = add nuw nsw i64 [[INDVARS_IV23]], 1
+; CHECK-NEXT:    [[TMP15]] = add nuw nsw i32 [[DOT014]], 1
+; CHECK-NEXT:    [[EXITCOND4:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND4]], label %[[FOR_FIRST]], label %[[FOR_FIRST_EXIT:.*]]
+; CHECK:       [[FOR_FIRST_EXIT]]:
+; CHECK-NEXT:    br label %[[FOR_NEXT:.*]]
+; CHECK:       [[FOR_NEXT]]:
+; CHECK-NEXT:    br label %[[FOR_SECOND_PREHEADER:.*]]
+; CHECK:       [[FOR_SECOND_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_SECOND:.*]]
+; CHECK:       [[FOR_SECOND]]:
+; CHECK-NEXT:    [[DOT02:%.*]] = phi i32 [ 0, %[[FOR_SECOND_PREHEADER]] ], [ [[TMP28:%.*]], %[[FOR_SECOND_LATCH:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 3, %[[FOR_SECOND_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = add nsw i32 [[DOT02]], -3
+; CHECK-NEXT:    [[TMP21:%.*]] = add nuw nsw i64 [[INDVARS_IV1]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nsw i32 [[TMP20]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[INDVARS_IV1]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = srem i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    br label %[[FOR_SECOND_LATCH]]
+; CHECK:       [[FOR_SECOND_LATCH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[TMP28]] = add nuw nsw i32 [[DOT02]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_SECOND]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 for.first.preheader:
   br label %for.first
 
diff --git a/llvm/test/Transforms/LoopFusion/peel.ll b/llvm/test/Transforms/LoopFusion/peel.ll
index 0ee3a3b068037..e8a4817f4c064 100644
--- a/llvm/test/Transforms/LoopFusion/peel.ll
+++ b/llvm/test/Transforms/LoopFusion/peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests whether we can fuse two loops together if they have constant but a
@@ -17,45 +18,106 @@
 ;     B[i] = ((i-6)*(i+3)) % i;
 ; }
 
-; CHECK-LABEL: void @function(ptr noalias %arg)
-; CHECK-NEXT:  for.first.preheader:
-; CHECK-NEXT:    br label %for.first.peel.begin
-; CHECK:       for.first.peel.begin:
-; CHECK-NEXT:    br label %for.first.peel
-; CHECK:       for.first.peel:
-; CHECK:         br label %for.first.latch.peel
-; CHECK:       for.first.latch.peel:
-; CHECK:         br label %for.first.peel.next
-; CHECK:       for.first.peel.next:
-; CHECK-NEXT:    br label %for.first.peel2
-; CHECK:       for.first.peel2:
-; CHECK:         br label %for.first.latch.peel10
-; CHECK:       for.first.latch.peel10:
-; CHECK:         br label %for.first.peel.next1
-; CHECK:       for.first.peel.next1:
-; CHECK-NEXT:    br label %for.first.peel15
-; CHECK:       for.first.peel15:
-; CHECK:         br label %for.first.latch.peel23
-; CHECK:       for.first.latch.peel23:
-; CHECK:         br label %for.first.peel.next14
-; CHECK:       for.first.peel.next14:
-; CHECK-NEXT:    br label %for.first.peel.next27
-; CHECK:       for.first.peel.next27:
-; CHECK-NEXT:    br label %for.first.preheader.peel.newph
-; CHECK:       for.first.preheader.peel.newph:
-; CHECK-NEXT:    br label %for.first
-; CHECK:       for.first:
-; CHECK:         br label %for.first.latch
-; CHECK:       for.first.latch:
-; CHECK:         br label %for.second.latch
-; CHECK:       for.second.latch:
-; CHECK:         br i1 %exitcond, label %for.first, label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
 define void @function(ptr noalias %arg) {
+; CHECK-LABEL: define void @function(
+; CHECK-SAME: ptr noalias [[ARG:%.*]]) {
+; CHECK-NEXT:  [[FOR_FIRST_PREHEADER:.*:]]
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_BEGIN:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_BEGIN]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL:.*]]
+; CHECK:       [[FOR_FIRST_PEEL]]:
+; CHECK-NEXT:    [[TMP_PEEL:%.*]] = add nsw i32 0, -3
+; CHECK-NEXT:    [[TMP8_PEEL:%.*]] = add nuw nsw i64 0, 3
+; CHECK-NEXT:    [[TMP9_PEEL:%.*]] = trunc i64 [[TMP8_PEEL]] to i32
+; CHECK-NEXT:    [[TMP10_PEEL:%.*]] = mul nsw i32 [[TMP_PEEL]], [[TMP9_PEEL]]
+; CHECK-NEXT:    [[TMP11_PEEL:%.*]] = trunc i64 0 to i32
+; CHECK-NEXT:    [[TMP12_PEEL:%.*]] = srem i32 [[TMP10_PEEL]], [[TMP11_PEEL]]
+; CHECK-NEXT:    [[TMP13_PEEL:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 0
+; CHECK-NEXT:    store i32 [[TMP12_PEEL]], ptr [[TMP13_PEEL]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH_PEEL:.*]]
+; CHECK:       [[FOR_FIRST_LATCH_PEEL]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3_PEEL:%.*]] = add nuw nsw i64 0, 1
+; CHECK-NEXT:    [[TMP15_PEEL:%.*]] = add nuw nsw i32 0, 1
+; CHECK-NEXT:    [[EXITCOND4_PEEL:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3_PEEL]], 100
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL2:.*]]
+; CHECK:       [[FOR_FIRST_PEEL2]]:
+; CHECK-NEXT:    [[TMP_PEEL3:%.*]] = add nsw i32 [[TMP15_PEEL]], -3
+; CHECK-NEXT:    [[TMP8_PEEL4:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL]], 3
+; CHECK-NEXT:    [[TMP9_PEEL5:%.*]] = trunc i64 [[TMP8_PEEL4]] to i32
+; CHECK-NEXT:    [[TMP10_PEEL6:%.*]] = mul nsw i32 [[TMP_PEEL3]], [[TMP9_PEEL5]]
+; CHECK-NEXT:    [[TMP11_PEEL7:%.*]] = trunc i64 [[INDVARS_IV_NEXT3_PEEL]] to i32
+; CHECK-NEXT:    [[TMP12_PEEL8:%.*]] = srem i32 [[TMP10_PEEL6]], [[TMP11_PEEL7]]
+; CHECK-NEXT:    [[TMP13_PEEL9:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV_NEXT3_PEEL]]
+; CHECK-NEXT:    store i32 [[TMP12_PEEL8]], ptr [[TMP13_PEEL9]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH_PEEL10:.*]]
+; CHECK:       [[FOR_FIRST_LATCH_PEEL10]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3_PEEL11:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL]], 1
+; CHECK-NEXT:    [[TMP15_PEEL12:%.*]] = add nuw nsw i32 [[TMP15_PEEL]], 1
+; CHECK-NEXT:    [[EXITCOND4_PEEL13:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3_PEEL11]], 100
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT1:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT1]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL15:.*]]
+; CHECK:       [[FOR_FIRST_PEEL15]]:
+; CHECK-NEXT:    [[TMP_PEEL16:%.*]] = add nsw i32 [[TMP15_PEEL12]], -3
+; CHECK-NEXT:    [[TMP8_PEEL17:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL11]], 3
+; CHECK-NEXT:    [[TMP9_PEEL18:%.*]] = trunc i64 [[TMP8_PEEL17]] to i32
+; CHECK-NEXT:    [[TMP10_PEEL19:%.*]] = mul nsw i32 [[TMP_PEEL16]], [[TMP9_PEEL18]]
+; CHECK-NEXT:    [[TMP11_PEEL20:%.*]] = trunc i64 [[INDVARS_IV_NEXT3_PEEL11]] to i32
+; CHECK-NEXT:    [[TMP12_PEEL21:%.*]] = srem i32 [[TMP10_PEEL19]], [[TMP11_PEEL20]]
+; CHECK-NEXT:    [[TMP13_PEEL22:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV_NEXT3_PEEL11]]
+; CHECK-NEXT:    store i32 [[TMP12_PEEL21]], ptr [[TMP13_PEEL22]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH_PEEL23:.*]]
+; CHECK:       [[FOR_FIRST_LATCH_PEEL23]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3_PEEL24:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL11]], 1
+; CHECK-NEXT:    [[TMP15_PEEL25:%.*]] = add nuw nsw i32 [[TMP15_PEEL12]], 1
+; CHECK-NEXT:    [[EXITCOND4_PEEL26:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3_PEEL24]], 100
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT14:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT14]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT27:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT27]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PREHEADER_PEEL_NEWPH:.*]]
+; CHECK:       [[FOR_FIRST_PREHEADER_PEEL_NEWPH]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[DOT014:%.*]] = phi i32 [ [[TMP15_PEEL25]], %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[TMP15:%.*]], %[[FOR_SECOND_LATCH:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ [[INDVARS_IV_NEXT3_PEEL24]], %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[INDVARS_IV_NEXT3:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[DOT02:%.*]] = phi i32 [ 0, %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[TMP28:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 3, %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT014]], -3
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[INDVARS_IV23]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i32 [[TMP]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[INDVARS_IV23]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = srem i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    store i32 [[TMP12]], ptr [[TMP13]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH:.*]]
+; CHECK:       [[FOR_FIRST_LATCH]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = add nsw i32 [[DOT02]], -3
+; CHECK-NEXT:    [[TMP21:%.*]] = add nuw nsw i64 [[INDVARS_IV1]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nsw i32 [[TMP20]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[INDVARS_IV1]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = srem i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    br label %[[FOR_SECOND_LATCH]]
+; CHECK:       [[FOR_SECOND_LATCH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3]] = add nuw nsw i64 [[INDVARS_IV23]], 1
+; CHECK-NEXT:    [[TMP15]] = add nuw nsw i32 [[DOT014]], 1
+; CHECK-NEXT:    [[EXITCOND4:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[TMP28]] = add nuw nsw i32 [[DOT02]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_FIRST]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 for.first.preheader:
   br label %for.first
 
diff --git a/llvm/test/Transforms/LoopFusion/sink_preheader.ll b/llvm/test/Transforms/LoopFusion/sink_preheader.ll
index 7e4559cfc85e3..be416db30da6e 100644
--- a/llvm/test/Transforms/LoopFusion/sink_preheader.ll
+++ b/llvm/test/Transforms/LoopFusion/sink_preheader.ll
@@ -1,19 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 
 define void @sink_preheader(i32 %N) {
-; CHECK-LABEL: @sink_preheader(
-; CHECK-NEXT:  pre1:
-; CHECK-NEXT:    br label [[BODY1:%.*]]
-; CHECK:       body1:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[BODY1]] ], [ 0, [[PRE1:%.*]] ]
-; CHECK-NEXT:    [[I2:%.*]] = phi i32 [ [[I_NEXT2:%.*]], [[BODY1]] ], [ 0, [[PRE1]] ]
+; CHECK-LABEL: define void @sink_preheader(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[PRE1:.*]]:
+; CHECK-NEXT:    br label %[[BODY1:.*]]
+; CHECK:       [[BODY1]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
+; CHECK-NEXT:    [[I2:%.*]] = phi i32 [ [[I_NEXT2:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 1, [[I]]
-; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[I]], [[N:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[I]], [[N]]
 ; CHECK-NEXT:    [[BARRIER:%.*]] = add i32 1, [[N]]
 ; CHECK-NEXT:    [[I_NEXT2]] = add i32 1, [[I2]]
 ; CHECK-NEXT:    [[COND2:%.*]] = icmp ne i32 [[I2]], [[N]]
-; CHECK-NEXT:    br i1 [[COND2]], label [[BODY1]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BODY1]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[SINKME:%.*]] = add i32 1, [[BARRIER]]
 ; CHECK-NEXT:    [[SINKME2:%.*]] = add i32 1, [[BARRIER]]
 ; CHECK-NEXT:    [[SINKME3:%.*]] = add i32 1, [[SINKME2]]
diff --git a/llvm/test/Transforms/LoopInterchange/delay-cachecost-calculation.ll b/llvm/test/Transforms/LoopInterchange/delay-cachecost-calculation.ll
new file mode 100644
index 0000000000000..d80b4420c034c
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/delay-cachecost-calculation.ll
@@ -0,0 +1,77 @@
+; REQUIRES: asserts
+
+; RUN: opt -passes=loop-interchange -debug -disable-output %s 2>&1 | FileCheck %s
+
+@A = global [16 x [16 x i32]] zeroinitializer
+
+; Check that the CacheCost is calculated only when required. In this case, it
+; is computed after passing the legality check.
+;
+; for (i = 0; i < 16; i++)
+;   for (j = 0; j < 16; j++)
+;     A[j][i] += 1;
+
+; CHECK: Loops are legal to interchange
+; CHECK: Compute CacheCost
+define void @legal_to_interchange() {
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.i.latch ]
+  br label %for.j
+
+for.j:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.next, %for.j ]
+  %idx = getelementptr inbounds [16 x [16 x i32]], ptr @A, i32 0, i32 %j, i32 %i
+  %val = load i32, ptr %idx
+  %inc = add i32 %val, 1
+  store i32 %inc, ptr %idx
+  %j.next = add i32 %j, 1
+  %j.exit = icmp eq i32 %j.next, 16
+  br i1 %j.exit, label %for.i.latch, label %for.j
+
+for.i.latch:
+  %i.next = add i32 %i, 1
+  %i.exit = icmp eq i32 %i.next, 16
+  br i1 %i.exit, label %exit, label %for.i.header
+
+exit:
+  ret void
+}
+
+; Check that the CacheCost is not calculated when not required. In this case,
+; the legality check always fails so that we do not need to compute the
+; CacheCost.
+;
+; for (i = 0; i < 16; i++)
+;   for (j = 0; j < 16; j++)
+;     A[j][i] = A[i][j];
+
+; CHECK-NOT: Compute CacheCost
+define void @illegal_to_interchange() {
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.i.latch ]
+  br label %for.j
+
+for.j:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.next, %for.j ]
+  %idx.load = getelementptr inbounds [16 x [16 x i32]], ptr @A, i32 0, i32 %i, i32 %j
+  %idx.store = getelementptr inbounds [16 x [16 x i32]], ptr @A, i32 0, i32 %j, i32 %i
+  %val = load i32, ptr %idx.load
+  store i32 %val, ptr %idx.store
+  %j.next = add i32 %j, 1
+  %j.exit = icmp eq i32 %j.next, 16
+  br i1 %j.exit, label %for.i.latch, label %for.j
+
+for.i.latch:
+  %i.next = add i32 %i, 1
+  %i.exit = icmp eq i32 %i.next, 16
+  br i1 %i.exit, label %exit, label %for.i.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll b/llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll
new file mode 100644
index 0000000000000..d0b6447d92fe7
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll
@@ -0,0 +1,48 @@
+; RUN: opt -passes='default<O3>' -enable-loopinterchange -disable-output \
+; RUN:     -disable-verify -verify-analysis-invalidation=0 \
+; RUN:     -debug-pass-manager=quiet %s 2>&1 | FileCheck %s
+
+; Test the position of LoopInterchange in the pass pipeline.
+
+; CHECK-NOT:  Running pass: LoopInterchangePass
+; CHECK:      Running pass: ControlHeightReductionPass
+; CHECK-NEXT: Running pass: LoopSimplifyPass
+; CHECK-NEXT: Running pass: LCSSAPass
+; CHECK-NEXT: Running pass: LoopRotatePass
+; CHECK-NEXT: Running pass: LoopDeletionPass
+; CHECK-NEXT: Running pass: LoopRotatePass
+; CHECK-NEXT: Running pass: LoopDeletionPass
+; CHECK-NEXT: Running pass: LoopInterchangePass
+; CHECK-NEXT: Running pass: LoopDistributePass
+; CHECK-NEXT: Running pass: InjectTLIMappings
+; CHECK-NEXT: Running pass: LoopVectorizePass
+
+
+define void @foo(ptr %a, i32 %n) {
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.i.latch ]
+  br label %for.j
+
+for.j:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.next, %for.j ]
+  %tmp = mul i32 %i, %n
+  %offset = add i32 %tmp, %j
+  %idx = getelementptr inbounds i32, ptr %a, i32 %offset
+  %load = load i32, ptr %idx, align 4
+  %inc = add i32 %load, 1
+  store i32 %inc, ptr %idx, align 4
+  %j.next = add i32 %j, 1
+  %j.exit = icmp eq i32 %j.next, %n
+  br i1 %j.exit, label %for.i.latch, label %for.j
+
+for.i.latch:
+  %i.next = add i32 %i, 1
+  %i.exit = icmp eq i32 %i.next, %n
+  br i1 %i.exit, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
index 9435c544fc812..1f619898ea788 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
@@ -92,7 +92,7 @@ for.cond.cleanup:                                 ; preds = %for.body
 ; CHECK-V1-IC1: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK-V1-IC1: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK-V1-IC1: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-V1-IC1: [[PROF4]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-V1-IC1: [[PROF4]] = !{!"branch_weights", i32 1, i32 7}
 ; CHECK-V1-IC1: [[PROF5]] = !{!"branch_weights", i32 0, i32 0}
 ; CHECK-V1-IC1: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META2]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll
new file mode 100644
index 0000000000000..e6d5c541560c6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @licm_replicate_call(double %x, ptr %dst) {
+; CHECK-LABEL: define void @licm_replicate_call(
+; CHECK-SAME: double [[X:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call double @llvm.pow.f64(double [[X]], double 3.000000e+00)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call double @llvm.pow.f64(double [[X]], double 3.000000e+00)
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP1]], i32 1
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT:    [[TMP4:%.*]] = uitofp <2 x i32> [[VEC_IND]] to <2 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = uitofp <2 x i32> [[STEP_ADD]] to <2 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i32 2
+; CHECK-NEXT:    store <2 x double> [[TMP6]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2)
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[IV_AS_FP:%.*]] = uitofp i32 [[IV_TRUNC]] to double
+; CHECK-NEXT:    [[P:%.*]] = tail call double @llvm.pow.f64(double [[X]], double 3.000000e+00)
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[P]], [[IV_AS_FP]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store double [[MUL]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 128
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.trunc = trunc i64 %iv to i32
+  %iv.as.fp = uitofp i32 %iv.trunc to double
+  %p = tail call double @llvm.pow.f64(double %x, double 3.000000e+00)
+  %mul = fmul double %p, %iv.as.fp
+  %gep.dst = getelementptr inbounds double, ptr %dst, i64 %iv
+  store double %mul, ptr %gep.dst, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, 128
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare double @llvm.pow.f64(double, double)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
index 295fe1b891e04..b02b314ecbd67 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -1030,6 +1030,472 @@ for.body:                                         ; preds = %for.body.preheader,
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
 }
 
+
+define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_madd_extadd(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP10]])
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP9]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-NEON:       scalar.ph:
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_madd_extadd(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP17]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP18]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE:       scalar.ph:
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_madd_extadd(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP16]])
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP15]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE-MAXBW:       scalar.ph:
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %add = add nsw i32 %res, %mul.ab
+  %add2 = add i32 %add, %c.ext
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+
+define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_extadd_extadd(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP5]])
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE2]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP6]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-NEON:       scalar.ph:
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_extadd_extadd(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP9]], align 1
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP11]]
+; CHECK-SVE-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[TMP12]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE:       scalar.ph:
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_extadd_extadd(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP11]])
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE2]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP12]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE-MAXBW:       scalar.ph:
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %add = add nsw i32 %res, %a.ext
+  %add2 = add i32 %add, %b.ext
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+
+
+define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_extadd_madd(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP9]])
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP10]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-NEON:       scalar.ph:
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_extadd_madd(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP16]], [[TMP17]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP18]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE:       scalar.ph:
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_extadd_madd(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP15]])
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP16]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE-MAXBW:       scalar.ph:
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %add = add nsw i32 %res, %c.ext
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %add2 = add i32 %add, %mul.ab
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+
 attributes #0 = { vscale_range(1,16) }
 
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
index d2a82269e1b8f..1d3741482212a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
@@ -72,8 +72,8 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
@@ -94,14 +94,14 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP19:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
 ; CHECK-NOI8MM-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP18]], [[TMP11]]
 ; CHECK-NOI8MM-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
-; CHECK-NOI8MM-NEXT:    [[TMP22]] = add <vscale x 8 x i32> [[TMP20]], [[VEC_PHI]]
-; CHECK-NOI8MM-NEXT:    [[TMP23]] = add <vscale x 8 x i32> [[TMP21]], [[VEC_PHI1]]
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
 ; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NOI8MM-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-NOI8MM:       middle.block:
-; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i32> [[TMP23]], [[TMP22]]
-; CHECK-NOI8MM-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[BIN_RDX]])
+; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-NOI8MM-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[BIN_RDX]])
 ; CHECK-NOI8MM-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-NOI8MM:       scalar.ph:
@@ -195,8 +195,8 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
@@ -217,14 +217,14 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP19:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
 ; CHECK-NOI8MM-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP18]], [[TMP11]]
 ; CHECK-NOI8MM-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
-; CHECK-NOI8MM-NEXT:    [[TMP22]] = add <vscale x 8 x i32> [[TMP20]], [[VEC_PHI]]
-; CHECK-NOI8MM-NEXT:    [[TMP23]] = add <vscale x 8 x i32> [[TMP21]], [[VEC_PHI1]]
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
 ; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NOI8MM-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-NOI8MM:       middle.block:
-; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i32> [[TMP23]], [[TMP22]]
-; CHECK-NOI8MM-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[BIN_RDX]])
+; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-NOI8MM-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[BIN_RDX]])
 ; CHECK-NOI8MM-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-NOI8MM:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index a44b9bfb3b446..887f95ce31f47 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -2590,514 +2590,8 @@ exit:                                 ; preds = %for.cond.cleanup.loopexit, %ent
 }
 
 
-; == Partial reductions with add of an extend
-
-define i32 @zext_add_reduc_i8_i32(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i32> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = zext i8 %load.a to i32
-  %add = add i32 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i32 %add
-}
-
-
-define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i64> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = zext i8 %load.a to i64
-  %add = add i64 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i64 %add
-}
-
-
-define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i16_i64(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <8 x i64> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i16_i64(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr [[TMP1]], i32 8
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[WIDE_LOAD2]] to <8 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <8 x i64> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i16_i64(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 2
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 4 x i64> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i16, ptr %a, i64 %iv
-  %load.a = load i16, ptr %gep.a, align 2
-  %ext.a = zext i16 %load.a to i64
-  %add = add i64 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i64 %add
-}
-
-
-define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(ptr %a) #1 {
-; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP10]] = add <vscale x 8 x i64> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP10]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = zext i8 %load.a to i64
-  %add = add i64 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i64 %add
-}
-
-
-define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @sext_add_reduc_i8_i32(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @sext_add_reduc_i8_i32(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i32 @sext_add_reduc_i8_i32(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i32> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = sext i8 %load.a to i32
-  %add = add i32 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i32 %add
-}
-
-define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #1 {
-; CHECK-INTERLEAVE1-LABEL: define dso_local void @dotp_high_register_pressure(
+define dso_local void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #1 {
+; CHECK-INTERLEAVE1-LABEL: define dso_local void @not_dotp_high_register_pressure(
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[SUM:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP100:%.*]] = icmp sgt i32 [[N]], 0
@@ -3184,7 +2678,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP36]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP36]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP33]])
@@ -3198,7 +2692,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
 ;
-; CHECK-INTERLEAVED-LABEL: define dso_local void @dotp_high_register_pressure(
+; CHECK-INTERLEAVED-LABEL: define dso_local void @not_dotp_high_register_pressure(
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[SUM:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[CMP100:%.*]] = icmp sgt i32 [[N]], 0
@@ -3285,7 +2779,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE21]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE21]])
 ; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE20]])
@@ -3299,7 +2793,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVED:       scalar.ph:
 ;
-; CHECK-MAXBW-LABEL: define dso_local void @dotp_high_register_pressure(
+; CHECK-MAXBW-LABEL: define dso_local void @not_dotp_high_register_pressure(
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[SUM:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[CMP100:%.*]] = icmp sgt i32 [[N]], 0
@@ -3337,14 +2831,14 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1
@@ -3362,40 +2856,40 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-MAXBW-NEXT:    [[STRIDED_VEC14:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127>
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = sext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = mul nsw <16 x i32> [[TMP13]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP15]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP14]])
-; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = sext <16 x i8> [[STRIDED_VEC8]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP29]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP18]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP16]])
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP14]])
+; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = sext <16 x i8> [[STRIDED_VEC8]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE15]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP16]])
 ; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = sext <16 x i8> [[STRIDED_VEC9]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP31:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP21]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP31]])
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE16]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP18]])
 ; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = sext <16 x i8> [[STRIDED_VEC10]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = mul nsw <16 x i32> [[TMP19]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP24]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP20]])
-; CHECK-MAXBW-NEXT:    [[TMP32:%.*]] = sext <16 x i8> [[STRIDED_VEC11]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP32]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP27]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP22]])
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE17]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP20]])
+; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = sext <16 x i8> [[STRIDED_VEC11]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP21]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE18]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP22]])
 ; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = sext <16 x i8> [[STRIDED_VEC12]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP34:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP30]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP34]])
+; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE19]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP24]])
 ; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = sext <16 x i8> [[STRIDED_VEC13]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP26:%.*]] = mul nsw <16 x i32> [[TMP25]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP33]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP26]])
-; CHECK-MAXBW-NEXT:    [[TMP35:%.*]] = sext <16 x i8> [[STRIDED_VEC14]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP35]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP36]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE20]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP26]])
+; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = sext <16 x i8> [[STRIDED_VEC14]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP27]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE21]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-MAXBW-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP36]])
-; CHECK-MAXBW-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP33]])
-; CHECK-MAXBW-NEXT:    [[TMP40:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP30]])
-; CHECK-MAXBW-NEXT:    [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP27]])
-; CHECK-MAXBW-NEXT:    [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]])
-; CHECK-MAXBW-NEXT:    [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP21]])
-; CHECK-MAXBW-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
-; CHECK-MAXBW-NEXT:    [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-MAXBW-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE21]])
+; CHECK-MAXBW-NEXT:    [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE20]])
+; CHECK-MAXBW-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE19]])
+; CHECK-MAXBW-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE18]])
+; CHECK-MAXBW-NEXT:    [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE17]])
+; CHECK-MAXBW-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-MAXBW-NEXT:    [[TMP36:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE15]])
+; CHECK-MAXBW-NEXT:    [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
new file mode 100644
index 0000000000000..5d5ee570da0ff
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
@@ -0,0 +1,918 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
+; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+; == Partial reductions with add of an extend
+
+define i32 @zext_add_reduc_i8_i32_sve(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_sve(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3]] = add <16 x i32> [[TMP2]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_sve(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_sve(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+define i32 @zext_add_reduc_i8_i32_neon(ptr %a) #2 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_neon(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP2]])
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_neon(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP3]])
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE3]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_neon(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP2]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-MAXBW-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i64>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> [[VEC_PHI]], <vscale x 16 x i64> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <8 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr [[TMP1]], i32 8
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[WIDE_LOAD2]] to <8 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <8 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP8]], align 2
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> [[VEC_PHI]], <vscale x 8 x i64> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i16, ptr %a, i64 %iv
+  %load.a = load i16, ptr %gep.a, align 2
+  %ext.a = zext i16 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP3]])
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE3]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_predicated(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP7]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_predicated(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP7]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_predicated(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-MAXBW-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1025)
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP7]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[TMP10]], i32 0
+; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @zext_add_reduc_i8_i32_mul_use(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_mul_use(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM2:%.*]] = phi i32 [ 13, [[ENTRY]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD2]] = add i32 [[EXT_A]], [[ACCUM2]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_mul_use(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM2:%.*]] = phi i32 [ 13, [[ENTRY]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[ADD2]] = add i32 [[EXT_A]], [[ACCUM2]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_mul_use(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM2:%.*]] = phi i32 [ 13, [[ENTRY]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[ADD2]] = add i32 [[EXT_A]], [[ACCUM2]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %accum2 = phi i32 [ 13, %entry ], [ %add2, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %add2 = add i32 %ext.a, %accum2
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = sub <16 x i32> [[VEC_PHI]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = sub <16 x i32> [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = sub <16 x i32> [[VEC_PHI1]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP10]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP10]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = sub i32 %accum, %ext.a
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = sext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+attributes #1 = { vscale_range(1,16) "target-features"="+neon,+dotprod,+sve" "cpu"="neoverse-v2" }
+attributes #2 = { "target-features"="+neon,+dotprod" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 8c2958769a615..b349c55d3e09a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -367,10 +367,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i32 [[TMP5]], 3
-; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw i32 2, [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw i64 2, [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
@@ -381,10 +379,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE1]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_ST2]], ptr [[B:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw nsw i32 [[TMP15]], 3
-; CHECK-NEXT:    [[TMP17:%.*]] = sub nsw i32 2, [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw i64 2, [[TMP15]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    [[REVERSE2:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP12]])
 ; CHECK-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP13]])
@@ -1577,10 +1573,8 @@ define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A,
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i32 [[TMP6]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw i32 4, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = sub nsw i64 4, [[TMP6]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
@@ -1597,10 +1591,8 @@ define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A,
 ; CHECK-NEXT:    [[TMP19:%.*]] = mul nsw <vscale x 4 x i32> [[REVERSE4]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = shl nuw nsw <vscale x 4 x i32> [[REVERSE5]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 4
-; CHECK-NEXT:    [[TMP24:%.*]] = sub nsw i32 4, [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = sext i32 [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = sub nsw i64 4, [[TMP22]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP25]]
 ; CHECK-NEXT:    [[REVERSE6:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP17]])
 ; CHECK-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP18]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index 2171277650541..173766cc0a656 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -141,3 +141,310 @@ loop:
 exit:
   ret void
 }
+
+define void @test_interleave_store_one_constant(ptr noalias %src, ptr noalias %dst, i64 %n) #0 {
+; CHECK-LABEL: define void @test_interleave_store_one_constant(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 0, [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
+; CHECK-NEXT:    [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 0, [[MUL_RESULT3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK5]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr double, ptr [[TMP13]], i32 2
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[TMP13]], i32 4
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[TMP13]], i32 6
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <2 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <2 x double>, ptr [[TMP17]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <2 x double> [[WIDE_LOAD6]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul <2 x double> [[WIDE_LOAD7]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul <2 x double> [[WIDE_LOAD8]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <2 x double> [[TMP18]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP26]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC9:%.*]] = shufflevector <4 x double> [[TMP27]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC9]], ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC10:%.*]] = shufflevector <4 x double> [[TMP28]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC10]], ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = shufflevector <4 x double> [[TMP29]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC11]], ptr [[TMP25]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF12:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NEXT:    [[N_VEC13:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF12]]
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX14:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX14]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[TMP31]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <2 x double>, ptr [[TMP32]], align 8
+; CHECK-NEXT:    [[TMP33:%.*]] = fmul <2 x double> [[WIDE_LOAD15]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[INDEX14]]
+; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP35]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT17]] = add nuw i64 [[INDEX14]], 2
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC13]]
+; CHECK-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N18:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC13]]
+; CHECK-NEXT:    br i1 [[CMP_N18]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[L]], 5.000000e+00
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store double [[MUL]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[GEP_DST_1:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[IV]], i32 1
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_DST_1]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr double, ptr %src, i64 %iv
+  %l = load double, ptr %gep.src, align 8
+  %mul = fmul double %l, 5.000000e+00
+  %gep.dst = getelementptr [2 x double], ptr %dst, i64 %iv
+  store double %mul, ptr %gep.dst, align 8
+  %gep.dst.1 = getelementptr [2 x double], ptr %dst, i64 %iv, i32 1
+  store double 0.000000e+00, ptr %gep.dst.1, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr noalias %C, i64 %n) #0 {
+; CHECK-LABEL: define void @single_fmul_used_by_each_member(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 8
+; CHECK-NEXT:    [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 0, [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[SCEVGEP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 0, [[MUL_RESULT2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[B]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 8
+; CHECK-NEXT:    [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 0, [[MUL_RESULT6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW7]]
+; CHECK-NEXT:    [[MUL8:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT9:%.*]] = extractvalue { i64, i1 } [[MUL8]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW10:%.*]] = extractvalue { i64, i1 } [[MUL8]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i64 0, [[MUL_RESULT9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[C]], i64 [[MUL_RESULT9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[C]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK11:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK11]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP23]], i32 0
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr double, ptr [[TMP23]], i32 2
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[TMP23]], i32 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP23]], i32 6
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <2 x double>, ptr [[TMP25]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <2 x double>, ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <2 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul <2 x double> [[WIDE_LOAD12]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP30:%.*]] = fmul <2 x double> [[WIDE_LOAD13]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul <2 x double> [[WIDE_LOAD14]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> [[TMP28]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP36]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP32]], align 8
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> [[TMP29]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC15:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP33]], align 8
+; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> [[TMP30]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP38]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8
+; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> [[TMP31]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC17:%.*]] = shufflevector <4 x double> [[TMP39]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP35]], align 8
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP22]]
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP40]], align 8
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP41]], align 8
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP42]], align 8
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP43]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF22:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NEXT:    [[N_VEC23:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF22]]
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX24:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX24]]
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr double, ptr [[TMP45]], i32 0
+; CHECK-NEXT:    [[TMP47:%.*]] = load double, ptr [[TMP46]], align 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP47]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP48:%.*]] = fmul <2 x double> [[BROADCAST_SPLAT]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX24]]
+; CHECK-NEXT:    store <2 x double> [[TMP48]], ptr [[TMP49]], align 8
+; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX24]]
+; CHECK-NEXT:    store <2 x double> [[TMP48]], ptr [[TMP50]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT25]] = add nuw i64 [[INDEX24]], 1
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC23]]
+; CHECK-NEXT:    br i1 [[TMP51]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N26:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC23]]
+; CHECK-NEXT:    br i1 [[CMP_N26]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr double, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[GEP_A]], align 8
+; CHECK-NEXT:    [[DIV:%.*]] = fmul double [[L]], 5.000000e+00
+; CHECK-NEXT:    [[GEP_B_1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[IV]], i32 1
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_B_1]], align 8
+; CHECK-NEXT:    [[GEP_C_1:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[IV]], i32 1
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_C_1]], align 8
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_B]], align 8
+; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_C]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr double, ptr %A, i64 %iv
+  %l = load double, ptr %gep.A, align 8
+  %div = fmul double %l, 5.000000e+00
+  %gep.B.1 = getelementptr {double, double}, ptr %B, i64 %iv, i32 1
+  store double %div, ptr %gep.B.1, align 8
+  %gep.C.1 = getelementptr {double, double}, ptr %C, i64 %iv, i32 1
+  store double %div, ptr %gep.C.1, align 8
+  %gep.B = getelementptr {double, double}, ptr %B, i64 %iv
+  store double %div, ptr %gep.B, align 8
+  %gep.C = getelementptr {double, double}, ptr %C, i64 %iv
+  store double %div, ptr %gep.C, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-cpu"="neoverse-v2" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
new file mode 100644
index 0000000000000..5efd821ba990f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
@@ -0,0 +1,303 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF2 %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF4 %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx15.0.0"
+
+define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
+; VF2-LABEL: define void @derived_int_ivs(
+; VF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[END:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[END]], -32
+; VF2-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; VF2-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
+; VF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
+; VF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VF2-NEXT:    [[TMP3:%.*]] = mul i64 [[N_VEC]], 16
+; VF2-NEXT:    [[TMP4:%.*]] = add i64 16, [[TMP3]]
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP5:%.*]] = mul i64 [[INDEX]], 16
+; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 16, [[TMP5]]
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[STRIDED_VEC]], <2 x double> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @derived_int_ivs(
+; VF4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[END:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[END]], -32
+; VF4-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; VF4-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VF4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; VF4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; VF4-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VF4-NEXT:    [[TMP3:%.*]] = mul i64 [[N_VEC]], 16
+; VF4-NEXT:    [[TMP4:%.*]] = add i64 16, [[TMP3]]
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP5:%.*]] = mul i64 [[INDEX]], 16
+; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 16, [[TMP5]]
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[TMP8:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP8]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 16, %entry ], [ %iv.next, %loop ]
+  %gep.a.0 = getelementptr inbounds i8, ptr %a, i64 %iv
+  %l.0 = load double, ptr %gep.a.0, align 8
+  %gep.b.0 = getelementptr inbounds i8, ptr %b, i64 %iv
+  store double %l.0, ptr %gep.b.0, align 8
+  %gep.a.1 = getelementptr inbounds i8, ptr %gep.a.0, i64 8
+  %l.1 = load double, ptr %gep.a.1, align 8
+  %gep.b.1 = getelementptr inbounds i8, ptr %gep.b.0, i64 8
+  store double %l.1, ptr %gep.b.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 16
+  %ec = icmp eq i64 %iv.next, %end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
+; VF2-LABEL: define void @derived_pointer_ivs(
+; VF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr [[END:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    [[A5:%.*]] = ptrtoint ptr [[A]] to i64
+; VF2-NEXT:    [[END4:%.*]] = ptrtoint ptr [[END]] to i64
+; VF2-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; VF2-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[END4]], -16
+; VF2-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[A5]]
+; VF2-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 4
+; VF2-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; VF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; VF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; VF2:       [[VECTOR_MEMCHECK]]:
+; VF2-NEXT:    [[TMP4:%.*]] = add i64 [[END1]], -16
+; VF2-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], [[A2]]
+; VF2-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 4
+; VF2-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 4
+; VF2-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 16
+; VF2-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; VF2-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; VF2-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP3]]
+; VF2-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; VF2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VF2-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; VF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; VF2-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 16
+; VF2-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
+; VF2-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], 16
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
+; VF2-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16
+; VF2-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX6]]
+; VF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[NEXT_GEP]], align 8
+; VF2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[STRIDED_VEC]], <2 x double> [[STRIDED_VEC8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP7]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @derived_pointer_ivs(
+; VF4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr [[END:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    [[A5:%.*]] = ptrtoint ptr [[A]] to i64
+; VF4-NEXT:    [[END4:%.*]] = ptrtoint ptr [[END]] to i64
+; VF4-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; VF4-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[END4]], -16
+; VF4-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[A5]]
+; VF4-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 4
+; VF4-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; VF4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; VF4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; VF4:       [[VECTOR_MEMCHECK]]:
+; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[END1]], -16
+; VF4-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], [[A2]]
+; VF4-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 4
+; VF4-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 4
+; VF4-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 16
+; VF4-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; VF4-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; VF4-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP3]]
+; VF4-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; VF4-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VF4-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; VF4-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; VF4-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 16
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], 16
+; VF4-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
+; VF4-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16
+; VF4-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX6]]
+; VF4-NEXT:    [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[NEXT_GEP]], align 8
+; VF4-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x double> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP7]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %ptr.iv.1 = phi ptr [ %a, %entry ], [ %ptr.iv.1.next, %loop ]
+  %ptr.iv.2 = phi ptr [ %b, %entry ], [ %ptr.iv.2.next, %loop ]
+  %l.0 = load double, ptr %ptr.iv.1, align 8
+  store double %l.0, ptr %ptr.iv.2, align 8
+  %gep.a.1 = getelementptr inbounds i8, ptr %ptr.iv.1, i64 8
+  %l.1 = load double, ptr %gep.a.1, align 8
+  %gep.b.1 = getelementptr inbounds i8, ptr %ptr.iv.2, i64 8
+  store double %l.1, ptr %gep.b.1, align 8
+  %ptr.iv.1.next = getelementptr inbounds i8, ptr %ptr.iv.1, i64 16
+  %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 16
+  %ec = icmp eq ptr %ptr.iv.1.next, %end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
+; VF2-LABEL: define void @narrow_with_uniform_add_and_gep(
+; VF2-SAME: ptr noalias [[P:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP0:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 0
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP0]]
+; VF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
+; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 1)
+; VF2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
+; VF2-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @narrow_with_uniform_add_and_gep(
+; VF4-SAME: ptr noalias [[P:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; VF4-NEXT:    [[TMP0:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 0
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP0]]
+; VF4-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 1)
+; VF4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
+; VF4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+
+  %iv0 = add nuw nsw i64 %iv, 0
+  %gep0 = getelementptr i64, ptr %p, i64 %iv0
+  %load0 = load i64, ptr %gep0
+  %add0 = add i64 %load0, 1
+  store i64 %add0, ptr %gep0
+
+  %iv1 = add nuw nsw i64 %iv, 1
+  %gep1 = getelementptr i64, ptr %p, i64 %iv1
+  %load1 = load i64, ptr %gep1
+  %add1 = add i64 %load1, 1
+  store i64 %add1, ptr %gep1
+
+  %iv.next = add nuw nsw i64 %iv, 2
+  %done = icmp eq i64 %iv.next, 1024
+  br i1 %done, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 00054a8ccdea7..8cd850b3ea986 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -204,3 +204,388 @@ loop:
 exit:
   ret void
 }
+
+define void @same_constant_store_interleave_group(i64 %x, ptr noalias %dst) {
+; VF2-LABEL: define void @same_constant_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    store <2 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @same_constant_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    store <8 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 0, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @different_constants_store_interleave_group(i64 %x, i64 %y, ptr noalias %dst) {
+; VF2-LABEL: define void @different_constants_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    store <4 x i64> <i64 0, i64 1, i64 0, i64 1>, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @different_constants_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    store <8 x i64> <i64 0, i64 1, i64 0, i64 1, i64 0, i64 1, i64 0, i64 1>, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 1, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @same_live_in_store_interleave_group(i64 %x, ptr noalias %dst) {
+; VF2-LABEL: define void @same_live_in_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @same_live_in_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %x, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 %x, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @different_live_ins_store_interleave_group(i64 %x, i64 %y, ptr noalias %dst) {
+; VF2-LABEL: define void @different_live_ins_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[Y]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @different_live_ins_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[Y]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %x, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 %y, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @single_uniform_load_store_interleave_group(ptr noalias %src, ptr noalias %dst) {
+; VF2-LABEL: define void @single_uniform_load_store_interleave_group(
+; VF2-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP1]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @single_uniform_load_store_interleave_group(
+; VF4-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %l.0 = load i64, ptr %src, align 8
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %l.0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 %l.0, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @multiple_uniform_load_store_interleave_group(ptr noalias %src.0, ptr noalias %src.1, ptr noalias %dst) {
+; VF2-LABEL: define void @multiple_uniform_load_store_interleave_group(
+; VF2-SAME: ptr noalias [[SRC_0:%.*]], ptr noalias [[SRC_1:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC_0]], align 8
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP1]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP3:%.*]] = load i64, ptr [[SRC_1]], align 8
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @multiple_uniform_load_store_interleave_group(
+; VF4-SAME: ptr noalias [[SRC_0:%.*]], ptr noalias [[SRC_1:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC_0]], align 8
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP3:%.*]] = load i64, ptr [[SRC_1]], align 8
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %l.0 = load i64, ptr %src.0, align 8
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %l.0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  %l.1 = load i64, ptr %src.1, align 8
+  store i64 %l.1, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
index f179a3ae04d23..ddd334d2982f8 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
@@ -1525,8 +1525,8 @@ define i64 @mla_and_add_together_16_64(ptr nocapture noundef readonly %x, i32 no
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP4]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP2]])
 ; CHECK-NEXT:    [[TMP5]] = add i64 [[TMP3]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
index e40f51fd7bd70..25f52b2a99ddc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
@@ -8,14 +8,51 @@ define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
 ; CHECK-LABEL: define void @test_wide_integer_induction(
 ; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VEC_IND]], ptr align 8 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[TMP8]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[IV]], ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT:    store i64 [[IV1]], ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
@@ -68,3 +105,10 @@ for.body:
 for.cond.cleanup:
   ret void
 }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index c063e6ff3f482..d485a7432423a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -367,7 +367,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index c0548dd8ca545..45357dd6bf0d6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -14,42 +14,42 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_EPILOGUE:       vector.ph:
-; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP4]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_EPILOGUE:       vector.body:
 ; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
-; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP9]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
-; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP12]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP15]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP14]], <vscale x 16 x ptr> [[TMP16]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP14]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP18]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x ptr> [[TMP19]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP8]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP9]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP17]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP16]], <vscale x 16 x ptr> [[TMP18]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALAR_EPILOGUE:       middle.block:
 ; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -107,36 +107,53 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_EVL-LABEL: define void @masked_strided_factor2
 ; PREDICATED_EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
 ; PREDICATED_EVL-NEXT:  entry:
+; PREDICATED_EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_EVL:       vector.ph:
 ; PREDICATED_EVL-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
-; PREDICATED_EVL-NEXT:    br label [[FOR_BODY:%.*]]
-; PREDICATED_EVL:       for.body:
-; PREDICATED_EVL-NEXT:    [[IX_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
-; PREDICATED_EVL-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
-; PREDICATED_EVL-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; PREDICATED_EVL:       if.then:
-; PREDICATED_EVL-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IX_024]], 1
-; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[MUL]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
-; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; PREDICATED_EVL-NEXT:    [[ADD:%.*]] = or disjoint i32 [[MUL]], 1
-; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[ADD]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
-; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 [[TMP3]])
-; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = zext nneg i32 [[MUL]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP4]]
-; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I]], ptr [[ARRAYIDX6]], align 1
-; PREDICATED_EVL-NEXT:    [[SUB:%.*]] = sub i8 0, [[SPEC_SELECT_I]]
-; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[ADD]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP5]]
-; PREDICATED_EVL-NEXT:    store i8 [[SUB]], ptr [[ARRAYIDX11]], align 1
-; PREDICATED_EVL-NEXT:    br label [[FOR_INC]]
-; PREDICATED_EVL:       for.inc:
-; PREDICATED_EVL-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
-; PREDICATED_EVL-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
-; PREDICATED_EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; PREDICATED_EVL:       for.end:
-; PREDICATED_EVL-NEXT:    ret void
+; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
+; PREDICATED_EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
+; PREDICATED_EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
+; PREDICATED_EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_EVL:       vector.body:
+; PREDICATED_EVL-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[AVL:%.*]] = sub i32 1024, [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP8]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP9]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP12]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; PREDICATED_EVL-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x ptr> align 1 [[TMP15]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
+; PREDICATED_EVL-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP17]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP16]], <vscale x 16 x ptr> align 1 [[TMP18]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_EVL-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PREDICATED_EVL:       middle.block:
+; PREDICATED_EVL-NEXT:    br label [[FOR_END:%.*]]
+; PREDICATED_EVL:       scalar.ph:
 ;
 entry:
   %conv = zext i8 %guard to i32
@@ -182,58 +199,58 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_EPILOGUE:       vector.ph:
-; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP4]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_EPILOGUE:       vector.body:
 ; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
-; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
-; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 2)
-; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 3)
-; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP12]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP14]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP16]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP17]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP18]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP19]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP21:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP20]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP22:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER4]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER5]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP23:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP22]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP24:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP24]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> [[TMP25]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP26:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP26]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> [[TMP27]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP28:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP28]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> [[TMP29]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP30:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP30]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP23]], <vscale x 16 x ptr> [[TMP31]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 2)
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 3)
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP13]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP14]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP15]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP16]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP17]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP18]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER4]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER5]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP22:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP21]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP23:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP23]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP19]], <vscale x 16 x ptr> [[TMP24]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP25:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP25]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> [[TMP26]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP27:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP27]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> [[TMP28]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP29:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP29]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> [[TMP30]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALAR_EPILOGUE:       middle.block:
 ; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -307,52 +324,69 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_EVL-LABEL: define void @masked_strided_factor4
 ; PREDICATED_EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
 ; PREDICATED_EVL-NEXT:  entry:
+; PREDICATED_EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_EVL:       vector.ph:
 ; PREDICATED_EVL-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
-; PREDICATED_EVL-NEXT:    br label [[FOR_BODY:%.*]]
-; PREDICATED_EVL:       for.body:
-; PREDICATED_EVL-NEXT:    [[IX_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
-; PREDICATED_EVL-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
-; PREDICATED_EVL-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; PREDICATED_EVL:       if.then:
-; PREDICATED_EVL-NEXT:    [[IDX0:%.*]] = shl nuw nsw i32 [[IX_024]], 2
-; PREDICATED_EVL-NEXT:    [[IDX1:%.*]] = or disjoint i32 [[IDX0]], 1
-; PREDICATED_EVL-NEXT:    [[IDX2:%.*]] = or disjoint i32 [[IDX0]], 2
-; PREDICATED_EVL-NEXT:    [[IDX3:%.*]] = or disjoint i32 [[IDX0]], 3
-; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[IDX0]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
-; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAY1IDX0]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[IDX1]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
-; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAY1IDX1]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = zext nneg i32 [[IDX2]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP4]]
-; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAY1IDX2]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = zext nneg i32 [[IDX3]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP6]]
-; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAY1IDX3]], align 1
-; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I1:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 [[TMP3]])
-; PREDICATED_EVL-NEXT:    [[SUB1:%.*]] = sub i8 0, [[SPEC_SELECT_I1]]
-; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I2:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP5]], i8 [[TMP7]])
-; PREDICATED_EVL-NEXT:    [[SUB2:%.*]] = sub i8 0, [[SPEC_SELECT_I2]]
-; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = zext nneg i32 [[IDX0]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP8]]
-; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I1]], ptr [[ARRAY3IDX0]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = zext nneg i32 [[IDX1]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP9]]
-; PREDICATED_EVL-NEXT:    store i8 [[SUB1]], ptr [[ARRAY3IDX1]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = zext nneg i32 [[IDX2]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP10]]
-; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I2]], ptr [[ARRAY3IDX2]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg i32 [[IDX3]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP11]]
-; PREDICATED_EVL-NEXT:    store i8 [[SUB2]], ptr [[ARRAY3IDX3]], align 1
-; PREDICATED_EVL-NEXT:    br label [[FOR_INC]]
-; PREDICATED_EVL:       for.inc:
-; PREDICATED_EVL-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
-; PREDICATED_EVL-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
-; PREDICATED_EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; PREDICATED_EVL:       for.end:
-; PREDICATED_EVL-NEXT:    ret void
+; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
+; PREDICATED_EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
+; PREDICATED_EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
+; PREDICATED_EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_EVL:       vector.body:
+; PREDICATED_EVL-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[AVL:%.*]] = sub i32 1024, [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
+; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 2)
+; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 3)
+; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP12]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP13:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP13]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP14]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP15]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP16]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP17]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP18]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; PREDICATED_EVL-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; PREDICATED_EVL-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER4]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER5]])
+; PREDICATED_EVL-NEXT:    [[TMP22:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP21]]
+; PREDICATED_EVL-NEXT:    [[TMP23:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP23]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP19]], <vscale x 16 x ptr> align 1 [[TMP24]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP25:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP25]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> align 1 [[TMP26]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP27:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP27]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> align 1 [[TMP28]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP29:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP29]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> align 1 [[TMP30]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_EVL-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_EVL-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; PREDICATED_EVL:       middle.block:
+; PREDICATED_EVL-NEXT:    br label [[FOR_END:%.*]]
+; PREDICATED_EVL:       scalar.ph:
 ;
 entry:
   %conv = zext i8 %guard to i32
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
index f404b89492886..4f3b8d84310bd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
@@ -4,11 +4,11 @@
 
 ; REQUIRES: asserts
 
-; Make sure we do not vectorize a loop with a widened int induction.
-define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
+; Make sure we do not vectorize a loop with a widened pointer induction.
+define void @test_wide_pointer_induction(ptr noalias %a, i64 %N) {
 ; CHECK-NOT: LV: Vector loop of width {{.+}} costs:
 ;
-; CHECK: define void @test_wide_integer_induction(
+; CHECK: define void @test_wide_pointer_induction(
 ; CHECK-NOT: vector.body
 ;
 entry:
@@ -16,9 +16,11 @@ entry:
 
 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.ptr = phi ptr [ %a, %entry ], [ %iv.ptr.next, %loop ]
   %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
-  store i64 %iv, ptr %arrayidx, align 8
+  store ptr %iv.ptr, ptr %arrayidx, align 8
   %iv.next = add nuw nsw i64 %iv, 1
+  %iv.ptr.next = getelementptr i64, ptr %iv.ptr, i32 1
   %exitcond.not = icmp eq i64 %iv.next, %N
   br i1 %exitcond.not, label %exit, label %loop
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll b/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
index 93bd44f5c6220..e4892db592d6b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
@@ -9,7 +9,7 @@
 define void @vp_select(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; DEBUGLOC-LABEL: define void @vp_select(
 ; DEBUGLOC: vector.body:
-; DEBUGLOC:   = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> %{{.+}}, <vscale x 4 x i32> %{{.+}}, <vscale x 4 x i32> %{{.+}}, i32 %{{.+}}), !dbg ![[SELLOC:[0-9]+]]
+; DEBUGLOC:   = select <vscale x 4 x i1> %{{.+}}, <vscale x 4 x i32> %{{.+}}, <vscale x 4 x i32> %{{.+}}, !dbg ![[SELLOC:[0-9]+]]
 ; DEBUGLOC: loop:
 ; DEBUGLOC:   = select i1 %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, !dbg ![[SELLOC]]
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
index 09b274de30214..29b27cdb7556d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
@@ -40,7 +40,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP8:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP22:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP22]]
 ; RV64-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 [[TMP10]]
 ; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 [[TMP11]]
 ; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP13]], align 4
@@ -48,7 +49,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i32> [[REVERSE]], splat (i32 1)
 ; RV64-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP17:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP17:%.*]] = mul i64 -1, [[TMP23]]
 ; RV64-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP16]]
 ; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 [[TMP17]]
 ; RV64-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]])
@@ -98,7 +100,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP11:%.*]] = mul i32 0, [[TMP10]]
-; RV32-NEXT:    [[TMP12:%.*]] = sub i32 1, [[TMP10]]
+; RV32-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP10]], 1
+; RV32-NEXT:    [[TMP12:%.*]] = mul i32 -1, [[TMP24]]
 ; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 [[TMP11]]
 ; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 [[TMP12]]
 ; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
@@ -107,7 +110,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP18:%.*]] = mul i32 0, [[TMP17]]
-; RV32-NEXT:    [[TMP19:%.*]] = sub i32 1, [[TMP17]]
+; RV32-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP17]], 1
+; RV32-NEXT:    [[TMP19:%.*]] = mul i32 -1, [[TMP25]]
 ; RV32-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 [[TMP18]]
 ; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP20]], i32 [[TMP19]]
 ; RV32-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]])
@@ -157,11 +161,13 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP9:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP11:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP12:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP12:%.*]] = mul i64 -1, [[TMP32]]
 ; RV64-UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP11]]
 ; RV64-UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP12]]
 ; RV64-UF2-NEXT:    [[TMP15:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP16:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP33:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP16:%.*]] = mul i64 -1, [[TMP33]]
 ; RV64-UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP15]]
 ; RV64-UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i64 [[TMP16]]
 ; RV64-UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
@@ -172,11 +178,13 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP20:%.*]] = add <vscale x 4 x i32> [[REVERSE2]], splat (i32 1)
 ; RV64-UF2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP34:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP23:%.*]] = mul i64 -1, [[TMP34]]
 ; RV64-UF2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP22]]
 ; RV64-UF2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP24]], i64 [[TMP23]]
 ; RV64-UF2-NEXT:    [[TMP26:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP27:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP35:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP27:%.*]] = mul i64 -1, [[TMP35]]
 ; RV64-UF2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP26]]
 ; RV64-UF2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP28]], i64 [[TMP27]]
 ; RV64-UF2-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP19]])
@@ -246,7 +254,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP8:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP22:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP22]]
 ; RV64-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[TMP10]]
 ; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP11]]
 ; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
@@ -254,7 +263,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP14:%.*]] = fadd <vscale x 4 x float> [[REVERSE]], splat (float 1.000000e+00)
 ; RV64-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP17:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP17:%.*]] = mul i64 -1, [[TMP23]]
 ; RV64-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[TMP16]]
 ; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP17]]
 ; RV64-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP14]])
@@ -304,7 +314,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP11:%.*]] = mul i32 0, [[TMP10]]
-; RV32-NEXT:    [[TMP12:%.*]] = sub i32 1, [[TMP10]]
+; RV32-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP10]], 1
+; RV32-NEXT:    [[TMP12:%.*]] = mul i32 -1, [[TMP24]]
 ; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i32 [[TMP11]]
 ; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i32 [[TMP12]]
 ; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
@@ -313,7 +324,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP18:%.*]] = mul i32 0, [[TMP17]]
-; RV32-NEXT:    [[TMP19:%.*]] = sub i32 1, [[TMP17]]
+; RV32-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP17]], 1
+; RV32-NEXT:    [[TMP19:%.*]] = mul i32 -1, [[TMP25]]
 ; RV32-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 [[TMP18]]
 ; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i32 [[TMP19]]
 ; RV32-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP15]])
@@ -363,11 +375,13 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP9:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP11:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP12:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP12:%.*]] = mul i64 -1, [[TMP32]]
 ; RV64-UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP11]]
 ; RV64-UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP12]]
 ; RV64-UF2-NEXT:    [[TMP15:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP16:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP33:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP16:%.*]] = mul i64 -1, [[TMP33]]
 ; RV64-UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP15]]
 ; RV64-UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP16]]
 ; RV64-UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
@@ -378,11 +392,13 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP20:%.*]] = fadd <vscale x 4 x float> [[REVERSE2]], splat (float 1.000000e+00)
 ; RV64-UF2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP34:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP23:%.*]] = mul i64 -1, [[TMP34]]
 ; RV64-UF2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[TMP22]]
 ; RV64-UF2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[TMP23]]
 ; RV64-UF2-NEXT:    [[TMP26:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP27:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP35:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP27:%.*]] = mul i64 -1, [[TMP35]]
 ; RV64-UF2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[TMP26]]
 ; RV64-UF2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP27]]
 ; RV64-UF2-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP19]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index dd8b7d6ea7e42..b4e49a60e0887 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -334,22 +334,24 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %22 = zext i32 %21 to i64
 ; CHECK-NEXT:    %23 = getelementptr inbounds i32, ptr %B, i64 %22
 ; CHECK-NEXT:    %24 = mul i64 0, %18
-; CHECK-NEXT:    %25 = sub i64 1, %18
-; CHECK-NEXT:    %26 = getelementptr inbounds i32, ptr %23, i64 %24
-; CHECK-NEXT:    %27 = getelementptr inbounds i32, ptr %26, i64 %25
-; CHECK-NEXT:    %wide.load = load <vscale x 4 x i32>, ptr %27, align 4
+; CHECK-NEXT:    %25 = sub i64 %18, 1
+; CHECK-NEXT:    %26 = mul i64 -1, %25
+; CHECK-NEXT:    %27 = getelementptr inbounds i32, ptr %23, i64 %24
+; CHECK-NEXT:    %28 = getelementptr inbounds i32, ptr %27, i64 %26
+; CHECK-NEXT:    %wide.load = load <vscale x 4 x i32>, ptr %28, align 4
 ; CHECK-NEXT:    %reverse = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %wide.load)
-; CHECK-NEXT:    %28 = add <vscale x 4 x i32> %reverse, splat (i32 1)
-; CHECK-NEXT:    %29 = getelementptr inbounds i32, ptr %A, i64 %22
-; CHECK-NEXT:    %30 = mul i64 0, %18
-; CHECK-NEXT:    %31 = sub i64 1, %18
-; CHECK-NEXT:    %32 = getelementptr inbounds i32, ptr %29, i64 %30
-; CHECK-NEXT:    %33 = getelementptr inbounds i32, ptr %32, i64 %31
-; CHECK-NEXT:    %reverse4 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %28)
-; CHECK-NEXT:    store <vscale x 4 x i32> %reverse4, ptr %33, align 4
+; CHECK-NEXT:    %29 = add <vscale x 4 x i32> %reverse, splat (i32 1)
+; CHECK-NEXT:    %30 = getelementptr inbounds i32, ptr %A, i64 %22
+; CHECK-NEXT:    %31 = mul i64 0, %18
+; CHECK-NEXT:    %32 = sub i64 %18, 1
+; CHECK-NEXT:    %33 = mul i64 -1, %32
+; CHECK-NEXT:    %34 = getelementptr inbounds i32, ptr %30, i64 %31
+; CHECK-NEXT:    %35 = getelementptr inbounds i32, ptr %34, i64 %33
+; CHECK-NEXT:    %reverse4 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %29)
+; CHECK-NEXT:    store <vscale x 4 x i32> %reverse4, ptr %35, align 4
 ; CHECK-NEXT:    %index.next = add nuw i64 %index, %18
-; CHECK-NEXT:    %34 = icmp eq i64 %index.next, %n.vec
-; CHECK-NEXT:    br i1 %34, <null operand!>, label %vector.body
+; CHECK-NEXT:    %36 = icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1 %36, <null operand!>, label %vector.body
 ; CHECK-NEXT:  LV: created middle.block
 ; CHECK-NEXT:  LV: draw edge from vector.body
 ; CHECK-NEXT:  LV: vectorizing VPBB: middle.block in BB: middle.block
@@ -380,8 +382,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %i.0 = add nsw i32 %i.0.in8, -1
 ; CHECK-NEXT:    %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
-; CHECK-NEXT:    %35 = load i32, ptr %arrayidx, align 4
-; CHECK-NEXT:    %add9 = add i32 %35, 1
+; CHECK-NEXT:    %37 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    %add9 = add i32 %37, 1
 ; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
 ; CHECK-NEXT:    store i32 %add9, ptr %arrayidx3, align 4
 ; CHECK-NEXT:    %cmp = icmp ugt i64 %indvars.iv, 1
@@ -743,22 +745,24 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %22 = zext i32 %21 to i64
 ; CHECK-NEXT:    %23 = getelementptr inbounds float, ptr %B, i64 %22
 ; CHECK-NEXT:    %24 = mul i64 0, %18
-; CHECK-NEXT:    %25 = sub i64 1, %18
-; CHECK-NEXT:    %26 = getelementptr inbounds float, ptr %23, i64 %24
-; CHECK-NEXT:    %27 = getelementptr inbounds float, ptr %26, i64 %25
-; CHECK-NEXT:    %wide.load = load <vscale x 4 x float>, ptr %27, align 4
+; CHECK-NEXT:    %25 = sub i64 %18, 1
+; CHECK-NEXT:    %26 = mul i64 -1, %25
+; CHECK-NEXT:    %27 = getelementptr inbounds float, ptr %23, i64 %24
+; CHECK-NEXT:    %28 = getelementptr inbounds float, ptr %27, i64 %26
+; CHECK-NEXT:    %wide.load = load <vscale x 4 x float>, ptr %28, align 4
 ; CHECK-NEXT:    %reverse = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %wide.load)
-; CHECK-NEXT:    %28 = fadd <vscale x 4 x float> %reverse, splat (float 1.000000e+00)
-; CHECK-NEXT:    %29 = getelementptr inbounds float, ptr %A, i64 %22
-; CHECK-NEXT:    %30 = mul i64 0, %18
-; CHECK-NEXT:    %31 = sub i64 1, %18
-; CHECK-NEXT:    %32 = getelementptr inbounds float, ptr %29, i64 %30
-; CHECK-NEXT:    %33 = getelementptr inbounds float, ptr %32, i64 %31
-; CHECK-NEXT:    %reverse4 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %28)
-; CHECK-NEXT:    store <vscale x 4 x float> %reverse4, ptr %33, align 4
+; CHECK-NEXT:    %29 = fadd <vscale x 4 x float> %reverse, splat (float 1.000000e+00)
+; CHECK-NEXT:    %30 = getelementptr inbounds float, ptr %A, i64 %22
+; CHECK-NEXT:    %31 = mul i64 0, %18
+; CHECK-NEXT:    %32 = sub i64 %18, 1
+; CHECK-NEXT:    %33 = mul i64 -1, %32
+; CHECK-NEXT:    %34 = getelementptr inbounds float, ptr %30, i64 %31
+; CHECK-NEXT:    %35 = getelementptr inbounds float, ptr %34, i64 %33
+; CHECK-NEXT:    %reverse4 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %29)
+; CHECK-NEXT:    store <vscale x 4 x float> %reverse4, ptr %35, align 4
 ; CHECK-NEXT:    %index.next = add nuw i64 %index, %18
-; CHECK-NEXT:    %34 = icmp eq i64 %index.next, %n.vec
-; CHECK-NEXT:    br i1 %34, <null operand!>, label %vector.body
+; CHECK-NEXT:    %36 = icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1 %36, <null operand!>, label %vector.body
 ; CHECK-NEXT:  LV: created middle.block
 ; CHECK-NEXT:  LV: draw edge from vector.body
 ; CHECK-NEXT:  LV: vectorizing VPBB: middle.block in BB: middle.block
@@ -789,8 +793,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %i.0 = add nsw i32 %i.0.in8, -1
 ; CHECK-NEXT:    %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
-; CHECK-NEXT:    %35 = load float, ptr %arrayidx, align 4
-; CHECK-NEXT:    %conv1 = fadd float %35, 1.000000e+00
+; CHECK-NEXT:    %37 = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:    %conv1 = fadd float %37, 1.000000e+00
 ; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
 ; CHECK-NEXT:    store float %conv1, ptr %arrayidx3, align 4
 ; CHECK-NEXT:    %cmp = icmp ugt i64 %indvars.iv, 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index f059730a245be..1e1ed49f9f2ed 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -44,7 +44,7 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK-NEXT:    [[TMP23:%.*]] = ashr <vscale x 8 x i32> [[TMP15]], zeroinitializer
 ; CHECK-NEXT:    [[VP_OP3:%.*]] = or <vscale x 8 x i32> [[TMP23]], zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ult <vscale x 8 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x i32> @llvm.vp.select.nxv8i32(<vscale x 8 x i1> [[TMP16]], <vscale x 8 x i32> [[VP_OP3]], <vscale x 8 x i32> zeroinitializer, i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP17:%.*]] = select <vscale x 8 x i1> [[TMP16]], <vscale x 8 x i32> [[VP_OP3]], <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc <vscale x 8 x i32> [[TMP17]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP24]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc <vscale x 8 x i32> [[VP_OP]] to <vscale x 8 x i16>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
index 38ed3f24ab230..091eb8720260b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
@@ -953,16 +953,55 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_ptrtoint(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], <vscale x 2 x i64> [[VEC_IND]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = ptrtoint <vscale x 2 x ptr> [[TMP14]] to <vscale x 2 x i64>
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br label %[[EXIT:.*]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64
 ; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    store i64 [[TMP0]], ptr [[GEP2]], align 8
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP49:![0-9]+]]
 ; IF-EVL:       [[EXIT]]:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -1047,4 +1086,6 @@ exit:
 ; IF-EVL: [[LOOP45]] = distinct !{[[LOOP45]], [[META6]]}
 ; IF-EVL: [[LOOP46]] = distinct !{[[LOOP46]], [[META6]], [[META7]], [[META8]]}
 ; IF-EVL: [[LOOP47]] = distinct !{[[LOOP47]], [[META6]]}
+; IF-EVL: [[LOOP48]] = distinct !{[[LOOP48]], [[META6]], [[META7]], [[META8]]}
+; IF-EVL: [[LOOP49]] = distinct !{[[LOOP49]], [[META8]], [[META6]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
index e5364136135d7..2926011857ae9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
@@ -47,7 +47,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 3)
-; IF-EVL-OUTLOOP-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP18]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP11]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP19:%.*]] = select <vscale x 4 x i1> [[TMP18]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP:%.*]] = add <vscale x 4 x i32> [[TMP19]], [[VEC_PHI]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP20]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[VP_OP]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP11]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP11]] to i64
@@ -101,7 +101,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0
 ; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-INLOOP-NEXT:    [[TMP19:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 3)
-; IF-EVL-INLOOP-NEXT:    [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP12]])
+; IF-EVL-INLOOP-NEXT:    [[TMP20:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-INLOOP-NEXT:    [[TMP22]] = add i32 [[TMP21]], [[VEC_PHI]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP12]] to i64
@@ -514,41 +514,127 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-LABEL: define i32 @step_cond_add(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP:       vector.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP10]], splat (i32 1)
+; IF-EVL-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP11]]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL-OUTLOOP:       vector.body:
+; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT1:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = mul i32 1, [[TMP12]]
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP17:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP19]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP12]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT1]] = add i64 [[TMP20]], [[EVL_BASED_IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[EVL_BASED_IV]], [[TMP8]]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N_VEC]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-OUTLOOP:       middle.block:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-OUTLOOP:       scalar.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[VECTOR_PH]] ], [ [[ADD:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP37:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[EVL_BASED_IV]] to i32
+; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP37]], [[IV_TRUNC]]
 ; IF-EVL-OUTLOOP-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[TMP37]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[ADD]] = add nsw i32 [[SELECT]], [[RDX]]
-; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add nuw nsw i64 [[EVL_BASED_IV]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL-OUTLOOP:       for.end:
-; IF-EVL-OUTLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-OUTLOOP-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; IF-EVL-INLOOP-LABEL: define i32 @step_cond_add(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP:       vector.ph:
+; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 1)
+; IF-EVL-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP10]]
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
-; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP:       vector.body:
+; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT1:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[VECTOR_PH]] ], [ [[ADD:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV1]]
+; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i32 1, [[TMP11]]
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-INLOOP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
+; IF-EVL-INLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
+; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[TMP15:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
+; IF-EVL-INLOOP-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
+; IF-EVL-INLOOP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[ADD]] = add i32 [[TMP17]], [[RDX]]
+; IF-EVL-INLOOP-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT1]] = add i64 [[TMP19]], [[EVL_BASED_IV1]]
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[EVL_BASED_IV]], [[TMP8]]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-INLOOP-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N_VEC]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-INLOOP:       middle.block:
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-INLOOP:       scalar.ph:
+; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL-INLOOP:       for.body:
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX1:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[EVL_BASED_IV]] to i32
+; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; IF-EVL-INLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP28]], [[IV_TRUNC]]
 ; IF-EVL-INLOOP-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[TMP28]], i32 0
-; IF-EVL-INLOOP-NEXT:    [[ADD]] = add nsw i32 [[SELECT]], [[RDX]]
-; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add nuw nsw i64 [[EVL_BASED_IV]], 1
-; IF-EVL-INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-INLOOP-NEXT:    [[ADD1]] = add nsw i32 [[SELECT]], [[RDX1]]
+; IF-EVL-INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL-INLOOP:       for.end:
-; IF-EVL-INLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD1]], [[FOR_BODY]] ], [ [[ADD]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-INLOOP-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; NO-VP-OUTLOOP-LABEL: define i32 @step_cond_add(
@@ -696,49 +782,148 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-LABEL: define i32 @step_cond_add_pred(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP:       vector.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-OUTLOOP-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i64> [[TMP10]], splat (i64 1)
+; IF-EVL-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP11]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
+; IF-EVL-OUTLOOP-NEXT:    [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP13]]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
-; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ], [ [[TMP37:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP:       vector.body:
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT7:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT:    [[TMP15:%.*]] = mul i32 1, [[TMP14]]
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP15]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-OUTLOOP-NEXT:    [[TMP17:%.*]] = mul i64 1, [[TMP16]]
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP38:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; IF-EVL-OUTLOOP-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP21:%.*]] = icmp sle <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND2]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP23:%.*]] = select <vscale x 4 x i1> [[TMP18]], <vscale x 4 x i1> [[TMP21]], <vscale x 4 x i1> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP22]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP24]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP25]], [[IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND_NEXT7]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT4]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK1:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-OUTLOOP:       middle.block:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP24]])
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-OUTLOOP:       scalar.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL-OUTLOOP:       for.body:
+; IF-EVL-OUTLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP38:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP38]], [[IV_TRUNC]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[MIDDLE_BLOCK]]
 ; IF-EVL-OUTLOOP:       if.then:
 ; IF-EVL-OUTLOOP-NEXT:    [[ADD_PRED:%.*]] = add nsw i32 [[BC_MERGE_RDX]], [[TMP38]]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[MIDDLE_BLOCK]]
 ; IF-EVL-OUTLOOP:       for.inc:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP37]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[RDX_ADD]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL-OUTLOOP:       for.end:
-; IF-EVL-OUTLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[TMP37]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[RDX_ADD]], [[MIDDLE_BLOCK]] ], [ [[TMP27]], [[MIDDLE_BLOCK1]] ]
 ; IF-EVL-OUTLOOP-NEXT:    ret i32 [[RDX_ADD_LCSSA]]
 ;
 ; IF-EVL-INLOOP-LABEL: define i32 @step_cond_add_pred(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP:       vector.ph:
+; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 1)
+; IF-EVL-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP10]]
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
-; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ], [ [[TMP32:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP:       vector.body:
+; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[IV]]
+; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i32 1, [[TMP11]]
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; IF-EVL-INLOOP-NEXT:    [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; IF-EVL-INLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i32 0
+; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[TMP15:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
+; IF-EVL-INLOOP-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> [[TMP15]], i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[TMP17]] = add i32 [[TMP16]], [[VEC_PHI]]
+; IF-EVL-INLOOP-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[IV]]
+; IF-EVL-INLOOP-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-INLOOP-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK1:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-INLOOP:       middle.block:
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-INLOOP:       scalar.ph:
+; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL-INLOOP:       for.body:
+; IF-EVL-INLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; IF-EVL-INLOOP-NEXT:    [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
 ; IF-EVL-INLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP35]], [[IV_TRUNC]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[MIDDLE_BLOCK]]
 ; IF-EVL-INLOOP:       if.then:
 ; IF-EVL-INLOOP-NEXT:    [[ADD_PRED:%.*]] = add nsw i32 [[BC_MERGE_RDX]], [[TMP35]]
 ; IF-EVL-INLOOP-NEXT:    br label [[MIDDLE_BLOCK]]
 ; IF-EVL-INLOOP:       for.inc:
-; IF-EVL-INLOOP-NEXT:    [[TMP32]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-INLOOP-NEXT:    [[RDX_ADD]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL-INLOOP:       for.end:
-; IF-EVL-INLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[TMP32]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[RDX_ADD]], [[MIDDLE_BLOCK]] ], [ [[TMP17]], [[MIDDLE_BLOCK1]] ]
 ; IF-EVL-INLOOP-NEXT:    ret i32 [[RDX_ADD_LCSSA]]
 ;
 ; NO-VP-OUTLOOP-LABEL: define i32 @step_cond_add_pred(
@@ -906,8 +1091,10 @@ for.end:
 ; IF-EVL-OUTLOOP: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
 ; IF-EVL-OUTLOOP: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
 ; IF-EVL-OUTLOOP: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
-; IF-EVL-OUTLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; IF-EVL-OUTLOOP: [[META8]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; IF-EVL-OUTLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-OUTLOOP: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
+; IF-EVL-OUTLOOP: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-OUTLOOP: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META1]]}
 ;.
 ; IF-EVL-INLOOP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; IF-EVL-INLOOP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -916,8 +1103,10 @@ for.end:
 ; IF-EVL-INLOOP: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
 ; IF-EVL-INLOOP: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
 ; IF-EVL-INLOOP: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
-; IF-EVL-INLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; IF-EVL-INLOOP: [[META8]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; IF-EVL-INLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-INLOOP: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
+; IF-EVL-INLOOP: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-INLOOP: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META1]]}
 ;.
 ; NO-VP-OUTLOOP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; NO-VP-OUTLOOP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
index 172335d0e4473..0490d63f67d4e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
@@ -592,6 +592,151 @@ for.end:
   ret i32 %for1
 }
 
+define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
+; IF-EVL-LABEL: define void @first_order_recurrence_indvar(
+; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP2]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP18]], 2
+; IF-EVL-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+; IF-EVL-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT:    [[TMP12:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP12]]
+; IF-EVL-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; IF-EVL-NEXT:    [[TMP19:%.*]] = mul nuw i32 [[TMP13]], 2
+; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i32 [[TMP19]], 1
+; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 33, i32 [[TMP10]]
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[PREV_EVL:%.*]] = phi i32 [ [[TMP5]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[TC]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP11]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP7]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP20]] = add <vscale x 2 x i64> [[VEC_IND]], splat (i64 42)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP20]], i32 -1, <vscale x 2 x i1> splat (i1 true), i32 [[PREV_EVL]], i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP9]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br label %[[FOR_END:.*]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 33, %[[ENTRY]] ]
+; IF-EVL-NEXT:    br label %[[FOR_BODY:.*]]
+; IF-EVL:       [[FOR_BODY]]:
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV1_NEXT:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP14:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP14]] = add i64 [[IV1]], 42
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[IV1]]
+; IF-EVL-NEXT:    store i64 [[FOR1]], ptr [[ARRAYIDX]], align 8
+; IF-EVL-NEXT:    [[IV1_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV1_NEXT]], [[TC]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; IF-EVL:       [[FOR_END]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @first_order_recurrence_indvar(
+; NO-VP-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NO-VP:       [[VECTOR_PH]]:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP4]], 2
+; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
+; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
+; NO-VP-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP9]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-VP-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP16:%.*]] = mul nuw i32 [[TMP14]], 2
+; NO-VP-NEXT:    [[TMP20:%.*]] = sub i32 [[TMP16]], 1
+; NO-VP-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 33, i32 [[TMP20]]
+; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NO-VP:       [[VECTOR_BODY]]:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP12]] = add <vscale x 2 x i64> [[VEC_IND]], splat (i64 42)
+; NO-VP-NEXT:    [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP12]], i32 -1)
+; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[INDEX]]
+; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP11]], i32 0
+; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP13]], ptr [[TMP15]], align 8
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-VP-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; NO-VP:       [[MIDDLE_BLOCK]]:
+; NO-VP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP21:%.*]] = mul nuw i32 [[TMP17]], 2
+; NO-VP-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP21]], 1
+; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 2 x i64> [[TMP12]], i32 [[TMP19]]
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NO-VP:       [[SCALAR_PH]]:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; NO-VP-NEXT:    br label %[[FOR_BODY:.*]]
+; NO-VP:       [[FOR_BODY]]:
+; NO-VP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV1_NEXT:%.*]], %[[FOR_BODY]] ]
+; NO-VP-NEXT:    [[FOR1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP18]] = add i64 [[IV1]], 42
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[IV1]]
+; NO-VP-NEXT:    store i64 [[FOR1]], ptr [[ARRAYIDX]], align 8
+; NO-VP-NEXT:    [[IV1_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV1_NEXT]], [[TC]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; NO-VP:       [[FOR_END]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars = phi i64 [ 0, %entry ], [ %indvars.next, %for.body ]
+  %for1 = phi i64 [ 33, %entry ], [ %x, %for.body ]
+
+  %x = add i64 %indvars, 42
+
+  %arrayidx = getelementptr inbounds nuw i64, ptr %A, i64 %indvars
+  store i64 %for1, ptr %arrayidx
+
+  %indvars.next = add nuw nsw i64 %indvars, 1
+  %exitcond.not = icmp eq i64 %indvars.next, %TC
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
 ;.
@@ -606,6 +751,8 @@ for.end:
 ; IF-EVL: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
 ; IF-EVL: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]]}
 ; IF-EVL: [[META10]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; IF-EVL: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]], [[META1]]}
 ;.
 ; NO-VP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; NO-VP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -617,4 +764,6 @@ for.end:
 ; NO-VP: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
 ; NO-VP: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
 ; NO-VP: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; NO-VP: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; NO-VP: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll
index a52da79ee3963..2b7a9fbdfd801 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll
@@ -12,18 +12,58 @@
 define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %index, i64 %n) {
 ; IF-EVL-LABEL: @gather_scatter(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], <vscale x 2 x i64> [[VEC_IND]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x float> @llvm.vp.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <vscale x 2 x i64> [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> [[WIDE_MASKED_GATHER2]], <vscale x 2 x ptr> align 4 [[TMP16]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], [[TMP8]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV]]
+; IF-EVL-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX]], i64 [[INDVARS_IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX3]], align 8
-; IF-EVL-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], i64 [[TMP0]]
+; IF-EVL-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN]], i64 [[TMP0]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-; IF-EVL-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], i64 [[TMP0]]
+; IF-EVL-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[TMP0]]
 ; IF-EVL-NEXT:    store float [[TMP1]], ptr [[ARRAYIDX7]], align 4
-; IF-EVL-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; IF-EVL-NEXT:    [[INDVARS_IV_NEXT1]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index 83deebe42d38f..8b1441450dd94 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -602,7 +602,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -723,7 +723,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -844,7 +844,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -965,7 +965,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp ugt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -1311,7 +1311,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = fcmp fast olt <vscale x 4 x float> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x float> @llvm.vp.select.nxv4f32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -1432,7 +1432,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = fcmp fast ogt <vscale x 4 x float> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x float> @llvm.vp.select.nxv4f32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
index 96db5bf4e9acc..91d94e52d0990 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
@@ -33,7 +33,8 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP7]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP18]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i64 1, [[TMP18]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP18]], 1
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 -1, [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[TMP8]], i64 [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP16]], i64 [[TMP10]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -41,7 +42,8 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP7]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = mul i64 0, [[TMP19]]
-; IF-EVL-NEXT:    [[TMP15:%.*]] = sub i64 1, [[TMP19]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP19]], 1
+; IF-EVL-NEXT:    [[TMP15:%.*]] = mul i64 -1, [[TMP23]]
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP13]], i64 [[TMP14]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[TMP22]], i64 [[TMP15]]
 ; IF-EVL-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -136,7 +138,8 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[PTR1:%.*]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP26]]
-; IF-EVL-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP26]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = sub i64 [[TMP26]], 1
+; IF-EVL-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP15]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[TMP16]], i64 [[TMP17]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[TMP19]], i64 [[TMP18]]
 ; IF-EVL-NEXT:    [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -145,7 +148,8 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP27]]
-; IF-EVL-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP27]]
+; IF-EVL-NEXT:    [[TMP30:%.*]] = sub i64 [[TMP27]], 1
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 -1, [[TMP30]]
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[TMP21]], i64 [[TMP22]]
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 [[TMP23]]
 ; IF-EVL-NEXT:    [[VP_REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -261,7 +265,8 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP6]] to i64
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP9]]
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP9]]
+; IF-EVL-NEXT:    [[TMP29:%.*]] = sub i64 [[TMP9]], 1
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP29]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
@@ -271,7 +276,8 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[OFFSET_IDX]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP6]] to i64
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP16]]
-; IF-EVL-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP16]]
+; IF-EVL-NEXT:    [[TMP30:%.*]] = sub i64 [[TMP16]], 1
+; IF-EVL-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP30]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP15]], i64 [[TMP17]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP18]]
 ; IF-EVL-NEXT:    [[VP_REVERSE1:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
@@ -279,7 +285,8 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[D:%.*]], i64 [[OFFSET_IDX]]
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP6]] to i64
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 0, [[TMP22]]
-; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 1, [[TMP22]]
+; IF-EVL-NEXT:    [[TMP31:%.*]] = sub i64 [[TMP22]], 1
+; IF-EVL-NEXT:    [[TMP24:%.*]] = mul i64 -1, [[TMP31]]
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP21]], i64 [[TMP23]]
 ; IF-EVL-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[TMP24]]
 ; IF-EVL-NEXT:    [[VP_REVERSE2:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
index 5c94ce180578f..984b64c55ce16 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -34,7 +34,8 @@ define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP11]] to i64
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = sub i64 1, [[TMP15]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 -1, [[TMP23]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[ARRAYIDX13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[TMP18]], i64 [[TMP17]]
 ; CHECK-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
index 83ae1f2d704e2..bb61f431d064d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
@@ -10,7 +10,6 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -24,7 +23,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -67,7 +66,6 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -81,7 +79,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -124,7 +122,6 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -138,7 +135,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -181,7 +178,6 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -195,7 +191,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -238,7 +234,6 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -252,7 +247,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -290,7 +285,6 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -304,7 +298,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -342,7 +336,6 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -356,7 +349,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -398,7 +391,6 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -412,7 +404,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -454,7 +446,6 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -468,7 +459,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
index 5c5492eec743c..2e1bcaaf3fc29 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
@@ -9,7 +9,6 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -23,7 +22,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -63,7 +62,6 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -77,7 +75,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -115,7 +113,6 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -129,7 +126,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -167,7 +164,6 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -181,7 +177,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -219,7 +215,6 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -233,7 +228,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -271,7 +266,6 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -285,7 +279,7 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -323,7 +317,6 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -337,7 +330,7 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -375,7 +368,6 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -389,7 +381,7 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -427,7 +419,6 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -441,7 +432,7 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -479,7 +470,6 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -493,7 +483,7 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -527,8 +517,37 @@ exit:
 }
 
 define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
-; IF-EVL: LV: Vectorization is possible but not beneficial.
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%.+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<[[N:%.+]]> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
 
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[INDEX:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[INDEX_NEXT:%.+]]>
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[INDEX_EVL:%.+]]> = phi ir<0>, vp<[[INDEX_EVL_NEXT:%.+]]>
+; IF-EVL-NEXT:     ir<[[IV:%.+]]> = WIDEN-INDUCTION  ir<0>, ir<1>, vp<[[EVL]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<[[N]]>, vp<[[INDEX_EVL]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[INDEX_EVL]]>, ir<1>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-GEP Inv[Var] ir<[[GEP:%.+]]> = getelementptr inbounds ir<%b>, ir<[[IV]]>
+; IF-EVL-NEXT:     WIDEN-CAST ir<[[PTRTOINT:%.+]]> = ptrtoint ir<[[GEP]]> to i64
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[SCALAR_STEPS]]>
+; IF-EVL-NEXT:     vp<[[VECTOR_PTR:%.+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[VECTOR_PTR]]>, ir<[[PTRTOINT]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     EMIT-SCALAR vp<[[ZEXT:%.+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[INDEX_EVL_NEXT]]> = add vp<[[ZEXT]]>, vp<[[INDEX_EVL]]>
+; IF-EVL-NEXT:     EMIT vp<[[INDEX_NEXT]]> = add vp<[[INDEX]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[INDEX_NEXT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+; IF-EVL-NEXT: Successor(s): middle.block
 entry:
   br label %loop
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index bf07255b30cd3..aa15a20a21d7a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -28,7 +28,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 
 ; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count
@@ -47,7 +46,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-OUTLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-OUTLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-OUTLOOP-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; IF-EVL-OUTLOOP-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-OUTLOOP-NEXT:    WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -71,7 +70,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ;
 
 ; IF-EVL-INLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-INLOOP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-INLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-INLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-INLOOP-NEXT: Live-in ir<%n> = original trip-count
@@ -87,7 +85,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-INLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-INLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-INLOOP-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; IF-EVL-INLOOP-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-INLOOP-NEXT:    WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index fc81bba4e9c26..563e515726c5d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -15,7 +15,6 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -29,7 +28,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NEXT:    EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:    WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -97,7 +96,7 @@ for.cond.cleanup:
 
 define void @safe_dep(ptr %p) {
 ; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; CHECK-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; CHECK-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in ir<512> = original trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
deleted file mode 100644
index 8c0e4da51f720..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ /dev/null
@@ -1,77 +0,0 @@
-; REQUIRES: asserts
-
- ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
- ; RUN: -force-tail-folding-style=data-with-evl \
- ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
- ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
-
- define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
- ; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
- ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
- ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}'
- ; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
- ;
- ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
- ; IF-EVL-NEXT: Live-in ir<[[VFUF:%.+]]> = VF * UF
- ; IF-EVL-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
- ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
-
- ; IF-EVL:      ir-bb<entry>:
- ; IF-EVL-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
-
- ; IF-EVL:      ir-bb<vector.ph>:
- ; IF-EVL-NEXT:   IR   %0 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %1 = mul nuw i64 %0, 4
- ; IF-EVL-NEXT:   IR   %2 = sub i64 %1, 1
- ; IF-EVL-NEXT:   IR   %n.rnd.up = add i64 %N, %2
- ; IF-EVL-NEXT:   IR   %n.mod.vf = urem i64 %n.rnd.up, %1
- ; IF-EVL-NEXT:   IR   %n.vec = sub i64 %n.rnd.up, %n.mod.vf
- ; IF-EVL-NEXT:   IR   %3 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %4 = mul nuw i64 %3, 4
- ; IF-EVL-NEXT: Successor(s): vector.body
- ; IF-EVL-EMPTY:
- ; IF-EVL-NEXT: vector.body:
- ; IF-EVL-NEXT:   EMIT-SCALAR vp<[[IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[IV_NEXT_EXIT:%.+]]>, vector.body ]
- ; IF-EVL-NEXT:   EMIT-SCALAR vp<[[EVL_PHI:%.+]]>  = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[IV_NEX:%.+]]>, vector.body ]
- ; IF-EVL-NEXT:   EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
- ; IF-EVL-NEXT:   CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:   CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[SUB:%.+]]> = sub ir<0>, ir<[[LD2]]>
- ; IF-EVL-NEXT:   WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<[[LD2]]>, ir<[[SUB]]>, vp<[[EVL]]>)
- ; IF-EVL-NEXT:   WIDEN ir<[[ADD:%.+]]> = add vp<[[SELECT]]>, ir<[[LD1]]>
- ; IF-EVL-NEXT:   CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
- ; IF-EVL-NEXT:   WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:   EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
- ; IF-EVL-NEXT:   EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<[[VFUF]]>
- ; IF-EVL-NEXT:   EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  ir<[[VTC]]>
-
- entry:
-   br label %for.body
-
- for.body:
-   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
-   %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv
-   %0 = load i32, ptr %arrayidx, align 4
-   %arrayidx3 = getelementptr inbounds i32, ptr %c, i64 %indvars.iv
-   %1 = load i32, ptr %arrayidx3, align 4
-   %cmp4 = icmp sgt i32 %0, %1
-   %2 = sub i32 0, %1
-   %cond.p = select i1 %cmp4, i32 %1, i32 %2
-   %cond = add i32 %cond.p, %0
-   %arrayidx15 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
-   store i32 %cond, ptr %arrayidx15, align 4
-   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-   %exitcond.not = icmp eq i64 %indvars.iv.next, %N
-   br i1 %exitcond.not, label %exit, label %for.body
-
- exit:
-   ret void
- }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 2c6fe4f5c808e..147e949808b54 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
-; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
+; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
index ff9cf682b6e9b..49d33d3087ed2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
@@ -338,6 +338,79 @@ loop.exit:
   ret void
 }
 
+; Same as @drop_vector_nuw_nsw, except built with avx1; in this case,
+; we make scalar clones of the 'sub' operation. These clones also need
+; cleared flags.
+define void @drop_nonvector_nuw_nsw_avx1(ptr noalias nocapture readonly %input, ptr %output, ptr noalias %ptrs) local_unnamed_addr #1 {
+; CHECK-LABEL: define void @drop_nonvector_nuw_nsw_avx1(
+; CHECK-SAME: ptr noalias readonly captures(none) [[INPUT:%.*]], ptr [[OUTPUT:%.*]], ptr noalias [[PTRS:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[PTRS]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x ptr> [[TMP14]], ptr [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x ptr> [[TMP15]], ptr [[TMP12]], i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x ptr> [[TMP16]], ptr [[TMP13]], i32 3
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds ptr, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <4 x ptr> [[TMP17]], ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr float, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP20]], i32 4, <4 x i1> [[TMP19]], <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i32 0
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP22]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.inc, %if.end ]
+  %i23 = icmp eq i64 %iv, 0
+  %gep = getelementptr inbounds ptr, ptr %ptrs, i64 %iv
+  %i27 = sub nuw nsw i64 %iv, 1
+  %i29 = getelementptr inbounds float, ptr %input, i64 %i27
+  store ptr %i29, ptr %gep
+  br i1 %i23, label %if.end, label %if.then
+
+if.then:
+  %i30 = load float, ptr %i29, align 4, !invariant.load !0
+  br label %if.end
+
+if.end:
+  %i34 = phi float [ 0.000000e+00, %loop.header ], [ %i30, %if.then ]
+  %i35 = getelementptr inbounds float, ptr %output, i64 %iv
+  store float %i34, ptr %i35, align 4
+  %iv.inc = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.inc, 4
+  br i1 %exitcond, label %loop.exit, label %loop.header
+
+loop.exit:
+  ret void
+}
+
 ; Preserve poison-generating flags from 'sub', which is not contributing to any address computation
 ; of any masked load/store/gather/scatter.
 define void @preserve_nuw_nsw_no_addr(ptr %output) local_unnamed_addr #0 {
@@ -358,7 +431,7 @@ define void @preserve_nuw_nsw_no_addr(ptr %output) local_unnamed_addr #0 {
 ; CHECK-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -411,7 +484,7 @@ define void @drop_scalar_exact(ptr noalias nocapture readonly %input, ptr %outpu
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -465,7 +538,7 @@ define void @drop_zext_nneg(ptr noalias %p, ptr noalias %p1) #0 {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -520,7 +593,7 @@ define void @preserve_vector_exact_no_addr(ptr noalias nocapture readonly %input
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -572,7 +645,7 @@ define void @preserve_exact_no_addr(ptr %output) local_unnamed_addr #0 {
 ; CHECK-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -720,7 +793,7 @@ define void @pr70590_recipe_without_underlying_instr(i64 %n, ptr noalias %dst) {
 ; CHECK-NEXT:    store <4 x i8> [[PREDPHI]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 
@@ -820,7 +893,7 @@ define void @recipe_without_underlying_instr_lanes_used(i64 %n, ptr noalias %dst
 ; CHECK-NEXT:    store <4 x i8> [[PREDPHI]], ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 
@@ -879,7 +952,7 @@ define void @Bgep_inbounds_unconditionally_due_to_store(ptr noalias %B, ptr read
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 
@@ -911,5 +984,6 @@ exit:
 }
 
 attributes #0 = { noinline nounwind uwtable "target-features"="+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl" }
+attributes #1 = { "target-features"="+avx" }
 
 !0 = !{}
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 0a2bb8d5682f2..c101d6a19aa2e 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
-; CHECK-NEXT:    [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK:       vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
-; CHECK-NEXT:    [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
-; CHECK-NEXT:    [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
+; CHECK-NEXT:    [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
 ; CHECK-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index cf10d32e063ab..d224da795997d 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -1,26 +1,185 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck --check-prefixes=CHECK,IC1VF4 %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck --check-prefixes=CHECK,IC4VF4 %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck --check-prefixes=CHECK,IC4VF1 %s
 
 define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
-; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
-; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
-; CHECK-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ]
-; CHECK-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+; IC1VF4-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; IC1VF4-SAME: ptr [[A:%.*]]) {
+; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC1VF4:       [[VECTOR_PH]]:
+; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC1VF4:       [[VECTOR_BODY]]:
+; IC1VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 19999, i64 19998, i64 19997, i64 19996>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
+; IC1VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; IC1VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 0
+; IC1VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 -3
+; IC1VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; IC1VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC1VF4-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i64> [[REVERSE]], splat (i64 3)
+; IC1VF4-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; IC1VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; IC1VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC1VF4-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20000
+; IC1VF4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC1VF4:       [[MIDDLE_BLOCK]]:
+; IC1VF4-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP4]])
+; IC1VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP6]], 9223372036854775807
+; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP6]], i64 331
+; IC1VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC1VF4:       [[SCALAR_PH]]:
+; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 19999, %[[ENTRY]] ]
+; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; IC1VF4-NEXT:    br label %[[LOOP:.*]]
+; IC1VF4:       [[LOOP]]:
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IC1VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; IC1VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; IC1VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; IC1VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IC1VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; IC1VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC1VF4:       [[EXIT]]:
+; IC1VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC1VF4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF4-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; IC4VF4-SAME: ptr [[A:%.*]]) {
+; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF4:       [[VECTOR_PH]]:
+; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF4:       [[VECTOR_BODY]]:
+; IC4VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 19999, i64 19998, i64 19997, i64 19996>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
+; IC4VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
+; IC4VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; IC4VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 0
+; IC4VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 -3
+; IC4VF4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 -4
+; IC4VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 -3
+; IC4VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 -8
+; IC4VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 -3
+; IC4VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 -12
+; IC4VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 -3
+; IC4VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; IC4VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
+; IC4VF4-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD4]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i64>, ptr [[TMP6]], align 8
+; IC4VF4-NEXT:    [[REVERSE7:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD6]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; IC4VF4-NEXT:    [[REVERSE9:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD8]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[TMP9:%.*]] = icmp sgt <4 x i64> [[REVERSE]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP10:%.*]] = icmp sgt <4 x i64> [[REVERSE5]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP11:%.*]] = icmp sgt <4 x i64> [[REVERSE7]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP12:%.*]] = icmp sgt <4 x i64> [[REVERSE9]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP13]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP14]] = select <4 x i1> [[TMP10]], <4 x i64> [[STEP_ADD]], <4 x i64> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP15]] = select <4 x i1> [[TMP11]], <4 x i64> [[STEP_ADD_2]], <4 x i64> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP16]] = select <4 x i1> [[TMP12]], <4 x i64> [[STEP_ADD_3]], <4 x i64> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; IC4VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 -4)
+; IC4VF4-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20000
+; IC4VF4-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC4VF4:       [[MIDDLE_BLOCK]]:
+; IC4VF4-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[TMP13]], <4 x i64> [[TMP14]])
+; IC4VF4-NEXT:    [[RDX_MINMAX10:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[RDX_MINMAX]], <4 x i64> [[TMP15]])
+; IC4VF4-NEXT:    [[RDX_MINMAX11:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[RDX_MINMAX10]], <4 x i64> [[TMP16]])
+; IC4VF4-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[RDX_MINMAX11]])
+; IC4VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP18]], 9223372036854775807
+; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP18]], i64 331
+; IC4VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF4:       [[SCALAR_PH]]:
+; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 19999, %[[ENTRY]] ]
+; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; IC4VF4-NEXT:    br label %[[LOOP:.*]]
+; IC4VF4:       [[LOOP]]:
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IC4VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; IC4VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; IC4VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; IC4VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IC4VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; IC4VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC4VF4:       [[EXIT]]:
+; IC4VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF1-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; IC4VF1-SAME: ptr [[A:%.*]]) {
+; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF1:       [[VECTOR_PH]]:
+; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF1:       [[VECTOR_BODY]]:
+; IC4VF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI1:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI2:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI3:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
+; IC4VF1-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], -2
+; IC4VF1-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], -3
+; IC4VF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; IC4VF1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; IC4VF1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; IC4VF1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; IC4VF1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP3]], align 8
+; IC4VF1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; IC4VF1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; IC4VF1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; IC4VF1-NEXT:    [[TMP11:%.*]] = icmp sgt i64 [[TMP7]], 3
+; IC4VF1-NEXT:    [[TMP12:%.*]] = icmp sgt i64 [[TMP8]], 3
+; IC4VF1-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP9]], 3
+; IC4VF1-NEXT:    [[TMP14:%.*]] = icmp sgt i64 [[TMP10]], 3
+; IC4VF1-NEXT:    [[TMP15]] = select i1 [[TMP11]], i64 [[OFFSET_IDX]], i64 [[VEC_PHI]]
+; IC4VF1-NEXT:    [[TMP16]] = select i1 [[TMP12]], i64 [[TMP0]], i64 [[VEC_PHI1]]
+; IC4VF1-NEXT:    [[TMP17]] = select i1 [[TMP13]], i64 [[TMP1]], i64 [[VEC_PHI2]]
+; IC4VF1-NEXT:    [[TMP18]] = select i1 [[TMP14]], i64 [[TMP2]], i64 [[VEC_PHI3]]
+; IC4VF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; IC4VF1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20000
+; IC4VF1-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC4VF1:       [[MIDDLE_BLOCK]]:
+; IC4VF1-NEXT:    [[RDX_MINMAX:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP15]], i64 [[TMP16]])
+; IC4VF1-NEXT:    [[RDX_MINMAX4:%.*]] = call i64 @llvm.smin.i64(i64 [[RDX_MINMAX]], i64 [[TMP17]])
+; IC4VF1-NEXT:    [[RDX_MINMAX5:%.*]] = call i64 @llvm.smin.i64(i64 [[RDX_MINMAX4]], i64 [[TMP18]])
+; IC4VF1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[RDX_MINMAX5]], 9223372036854775807
+; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[RDX_MINMAX5]], i64 331
+; IC4VF1-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF1:       [[SCALAR_PH]]:
+; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 19999, %[[ENTRY]] ]
+; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; IC4VF1-NEXT:    br label %[[LOOP:.*]]
+; IC4VF1:       [[LOOP]]:
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IC4VF1-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; IC4VF1-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; IC4VF1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; IC4VF1-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IC4VF1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; IC4VF1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC4VF1:       [[EXIT]]:
+; IC4VF1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF1-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
 ;
 entry:
   br label %loop
@@ -43,23 +202,354 @@ exit:                                             ; preds = %loop
 @table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1
 
 define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
-; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
-; CHECK-SAME: i16 noundef [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
-; CHECK-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
-; CHECK-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
-; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+; IC1VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; IC1VF4-SAME: i16 noundef [[VAL:%.*]]) {
+; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC1VF4:       [[VECTOR_PH]]:
+; IC1VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[VAL]], i64 0
+; IC1VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC1VF4:       [[VECTOR_BODY]]:
+; IC1VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC1VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC1VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC1VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; IC1VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 -3
+; IC1VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 1
+; IC1VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC1VF4-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i16> [[REVERSE]], [[BROADCAST_SPLAT]]
+; IC1VF4-NEXT:    [[TMP4:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC1VF4-NEXT:    [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP4]], <4 x i16> [[VEC_PHI]]
+; IC1VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC1VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC1VF4-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC1VF4-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC1VF4:       [[MIDDLE_BLOCK]]:
+; IC1VF4-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[TMP5]])
+; IC1VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP7]], 32767
+; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP7]], i16 0
+; IC1VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC1VF4:       [[SCALAR_PH]]:
+; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC1VF4-NEXT:    br label %[[LOOP:.*]]
+; IC1VF4:       [[LOOP]]:
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC1VF4-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; IC1VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; IC1VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC1VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC1VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC1VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC1VF4:       [[EXIT]]:
+; IC1VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC1VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; IC4VF4-SAME: i16 noundef [[VAL:%.*]]) {
+; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF4:       [[VECTOR_PH]]:
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[VAL]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF4:       [[VECTOR_BODY]]:
+; IC4VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44:.*]] ]
+; IC4VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP108:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP109:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP110:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP111:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 -4)
+; IC4VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 0, i32 1, i32 2, i32 3>
+; IC4VF4-NEXT:    [[VEC_IV8:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 4, i32 5, i32 6, i32 7>
+; IC4VF4-NEXT:    [[VEC_IV11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 8, i32 9, i32 10, i32 11>
+; IC4VF4-NEXT:    [[VEC_IV14:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 12, i32 13, i32 14, i32 15>
+; IC4VF4-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV8]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV11]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV14]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; IC4VF4:       [[PRED_LOAD_IF]]:
+; IC4VF4-NEXT:    [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 0
+; IC4VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP5]]
+; IC4VF4-NEXT:    [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 1
+; IC4VF4-NEXT:    [[TMP8:%.*]] = insertelement <4 x i16> poison, i16 [[TMP7]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE]]:
+; IC4VF4-NEXT:    [[TMP9:%.*]] = phi <4 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; IC4VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; IC4VF4:       [[PRED_LOAD_IF15]]:
+; IC4VF4-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP11]]
+; IC4VF4-NEXT:    [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 1
+; IC4VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x i16> [[TMP9]], i16 [[TMP13]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE16]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE16]]:
+; IC4VF4-NEXT:    [[TMP15:%.*]] = phi <4 x i16> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF15]] ]
+; IC4VF4-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP16]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; IC4VF4:       [[PRED_LOAD_IF17]]:
+; IC4VF4-NEXT:    [[TMP17:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP17]]
+; IC4VF4-NEXT:    [[TMP19:%.*]] = load i16, ptr [[TMP18]], align 1
+; IC4VF4-NEXT:    [[TMP20:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP19]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE18]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE18]]:
+; IC4VF4-NEXT:    [[TMP21:%.*]] = phi <4 x i16> [ [[TMP15]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP20]], %[[PRED_LOAD_IF17]] ]
+; IC4VF4-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP22]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; IC4VF4:       [[PRED_LOAD_IF19]]:
+; IC4VF4-NEXT:    [[TMP23:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF4-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP23]]
+; IC4VF4-NEXT:    [[TMP25:%.*]] = load i16, ptr [[TMP24]], align 1
+; IC4VF4-NEXT:    [[TMP26:%.*]] = insertelement <4 x i16> [[TMP21]], i16 [[TMP25]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE20]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE20]]:
+; IC4VF4-NEXT:    [[TMP27:%.*]] = phi <4 x i16> [ [[TMP21]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP26]], %[[PRED_LOAD_IF19]] ]
+; IC4VF4-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP28]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; IC4VF4:       [[PRED_LOAD_IF21]]:
+; IC4VF4-NEXT:    [[TMP29:%.*]] = add i16 [[OFFSET_IDX]], -4
+; IC4VF4-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP29]]
+; IC4VF4-NEXT:    [[TMP31:%.*]] = load i16, ptr [[TMP30]], align 1
+; IC4VF4-NEXT:    [[TMP32:%.*]] = insertelement <4 x i16> poison, i16 [[TMP31]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE22]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE22]]:
+; IC4VF4-NEXT:    [[TMP33:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP32]], %[[PRED_LOAD_IF21]] ]
+; IC4VF4-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP34]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; IC4VF4:       [[PRED_LOAD_IF23]]:
+; IC4VF4-NEXT:    [[TMP35:%.*]] = add i16 [[OFFSET_IDX]], -5
+; IC4VF4-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP35]]
+; IC4VF4-NEXT:    [[TMP37:%.*]] = load i16, ptr [[TMP36]], align 1
+; IC4VF4-NEXT:    [[TMP38:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP37]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE24]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE24]]:
+; IC4VF4-NEXT:    [[TMP39:%.*]] = phi <4 x i16> [ [[TMP33]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP38]], %[[PRED_LOAD_IF23]] ]
+; IC4VF4-NEXT:    [[TMP40:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP40]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; IC4VF4:       [[PRED_LOAD_IF25]]:
+; IC4VF4-NEXT:    [[TMP41:%.*]] = add i16 [[OFFSET_IDX]], -6
+; IC4VF4-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP41]]
+; IC4VF4-NEXT:    [[TMP43:%.*]] = load i16, ptr [[TMP42]], align 1
+; IC4VF4-NEXT:    [[TMP44:%.*]] = insertelement <4 x i16> [[TMP39]], i16 [[TMP43]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE26]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE26]]:
+; IC4VF4-NEXT:    [[TMP45:%.*]] = phi <4 x i16> [ [[TMP39]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP44]], %[[PRED_LOAD_IF25]] ]
+; IC4VF4-NEXT:    [[TMP46:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP46]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; IC4VF4:       [[PRED_LOAD_IF27]]:
+; IC4VF4-NEXT:    [[TMP47:%.*]] = add i16 [[OFFSET_IDX]], -7
+; IC4VF4-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP47]]
+; IC4VF4-NEXT:    [[TMP49:%.*]] = load i16, ptr [[TMP48]], align 1
+; IC4VF4-NEXT:    [[TMP50:%.*]] = insertelement <4 x i16> [[TMP45]], i16 [[TMP49]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE28]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE28]]:
+; IC4VF4-NEXT:    [[TMP51:%.*]] = phi <4 x i16> [ [[TMP45]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP50]], %[[PRED_LOAD_IF27]] ]
+; IC4VF4-NEXT:    [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP52]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; IC4VF4:       [[PRED_LOAD_IF29]]:
+; IC4VF4-NEXT:    [[TMP53:%.*]] = add i16 [[OFFSET_IDX]], -8
+; IC4VF4-NEXT:    [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP53]]
+; IC4VF4-NEXT:    [[TMP55:%.*]] = load i16, ptr [[TMP54]], align 1
+; IC4VF4-NEXT:    [[TMP56:%.*]] = insertelement <4 x i16> poison, i16 [[TMP55]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE30]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE30]]:
+; IC4VF4-NEXT:    [[TMP57:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP56]], %[[PRED_LOAD_IF29]] ]
+; IC4VF4-NEXT:    [[TMP58:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP58]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; IC4VF4:       [[PRED_LOAD_IF31]]:
+; IC4VF4-NEXT:    [[TMP59:%.*]] = add i16 [[OFFSET_IDX]], -9
+; IC4VF4-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP59]]
+; IC4VF4-NEXT:    [[TMP61:%.*]] = load i16, ptr [[TMP60]], align 1
+; IC4VF4-NEXT:    [[TMP62:%.*]] = insertelement <4 x i16> [[TMP57]], i16 [[TMP61]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE32]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE32]]:
+; IC4VF4-NEXT:    [[TMP63:%.*]] = phi <4 x i16> [ [[TMP57]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP62]], %[[PRED_LOAD_IF31]] ]
+; IC4VF4-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP64]], label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
+; IC4VF4:       [[PRED_LOAD_IF33]]:
+; IC4VF4-NEXT:    [[TMP65:%.*]] = add i16 [[OFFSET_IDX]], -10
+; IC4VF4-NEXT:    [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP65]]
+; IC4VF4-NEXT:    [[TMP67:%.*]] = load i16, ptr [[TMP66]], align 1
+; IC4VF4-NEXT:    [[TMP68:%.*]] = insertelement <4 x i16> [[TMP63]], i16 [[TMP67]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE34]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE34]]:
+; IC4VF4-NEXT:    [[TMP69:%.*]] = phi <4 x i16> [ [[TMP63]], %[[PRED_LOAD_CONTINUE32]] ], [ [[TMP68]], %[[PRED_LOAD_IF33]] ]
+; IC4VF4-NEXT:    [[TMP70:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP70]], label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
+; IC4VF4:       [[PRED_LOAD_IF35]]:
+; IC4VF4-NEXT:    [[TMP71:%.*]] = add i16 [[OFFSET_IDX]], -11
+; IC4VF4-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP71]]
+; IC4VF4-NEXT:    [[TMP73:%.*]] = load i16, ptr [[TMP72]], align 1
+; IC4VF4-NEXT:    [[TMP74:%.*]] = insertelement <4 x i16> [[TMP69]], i16 [[TMP73]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE36]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE36]]:
+; IC4VF4-NEXT:    [[TMP75:%.*]] = phi <4 x i16> [ [[TMP69]], %[[PRED_LOAD_CONTINUE34]] ], [ [[TMP74]], %[[PRED_LOAD_IF35]] ]
+; IC4VF4-NEXT:    [[TMP76:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP76]], label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
+; IC4VF4:       [[PRED_LOAD_IF37]]:
+; IC4VF4-NEXT:    [[TMP77:%.*]] = add i16 [[OFFSET_IDX]], -12
+; IC4VF4-NEXT:    [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP77]]
+; IC4VF4-NEXT:    [[TMP79:%.*]] = load i16, ptr [[TMP78]], align 1
+; IC4VF4-NEXT:    [[TMP80:%.*]] = insertelement <4 x i16> poison, i16 [[TMP79]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE38]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE38]]:
+; IC4VF4-NEXT:    [[TMP81:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE36]] ], [ [[TMP80]], %[[PRED_LOAD_IF37]] ]
+; IC4VF4-NEXT:    [[TMP82:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP82]], label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
+; IC4VF4:       [[PRED_LOAD_IF39]]:
+; IC4VF4-NEXT:    [[TMP83:%.*]] = add i16 [[OFFSET_IDX]], -13
+; IC4VF4-NEXT:    [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP83]]
+; IC4VF4-NEXT:    [[TMP85:%.*]] = load i16, ptr [[TMP84]], align 1
+; IC4VF4-NEXT:    [[TMP86:%.*]] = insertelement <4 x i16> [[TMP81]], i16 [[TMP85]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE40]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE40]]:
+; IC4VF4-NEXT:    [[TMP87:%.*]] = phi <4 x i16> [ [[TMP81]], %[[PRED_LOAD_CONTINUE38]] ], [ [[TMP86]], %[[PRED_LOAD_IF39]] ]
+; IC4VF4-NEXT:    [[TMP88:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP88]], label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
+; IC4VF4:       [[PRED_LOAD_IF41]]:
+; IC4VF4-NEXT:    [[TMP89:%.*]] = add i16 [[OFFSET_IDX]], -14
+; IC4VF4-NEXT:    [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP89]]
+; IC4VF4-NEXT:    [[TMP91:%.*]] = load i16, ptr [[TMP90]], align 1
+; IC4VF4-NEXT:    [[TMP92:%.*]] = insertelement <4 x i16> [[TMP87]], i16 [[TMP91]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE42]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE42]]:
+; IC4VF4-NEXT:    [[TMP93:%.*]] = phi <4 x i16> [ [[TMP87]], %[[PRED_LOAD_CONTINUE40]] ], [ [[TMP92]], %[[PRED_LOAD_IF41]] ]
+; IC4VF4-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP94]], label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_IF43]]:
+; IC4VF4-NEXT:    [[TMP95:%.*]] = add i16 [[OFFSET_IDX]], -15
+; IC4VF4-NEXT:    [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP95]]
+; IC4VF4-NEXT:    [[TMP97:%.*]] = load i16, ptr [[TMP96]], align 1
+; IC4VF4-NEXT:    [[TMP98:%.*]] = insertelement <4 x i16> [[TMP93]], i16 [[TMP97]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE44]]:
+; IC4VF4-NEXT:    [[TMP99:%.*]] = phi <4 x i16> [ [[TMP93]], %[[PRED_LOAD_CONTINUE42]] ], [ [[TMP98]], %[[PRED_LOAD_IF43]] ]
+; IC4VF4-NEXT:    [[TMP100:%.*]] = icmp ugt <4 x i16> [[TMP27]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP101:%.*]] = icmp ugt <4 x i16> [[TMP51]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP102:%.*]] = icmp ugt <4 x i16> [[TMP75]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP103:%.*]] = icmp ugt <4 x i16> [[TMP99]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP104:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP105:%.*]] = add nsw <4 x i16> [[STEP_ADD]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP106:%.*]] = add nsw <4 x i16> [[STEP_ADD_2]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP107:%.*]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP108]] = select <4 x i1> [[TMP100]], <4 x i16> [[TMP104]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP109]] = select <4 x i1> [[TMP101]], <4 x i16> [[TMP105]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP110]] = select <4 x i1> [[TMP102]], <4 x i16> [[TMP106]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP111]] = select <4 x i1> [[TMP103]], <4 x i16> [[TMP107]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[TMP112:%.*]] = select <4 x i1> [[TMP0]], <4 x i16> [[TMP108]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP113:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP109]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP114:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP110]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP115:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP111]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; IC4VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD_3]], splat (i16 -4)
+; IC4VF4-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC4VF4:       [[MIDDLE_BLOCK]]:
+; IC4VF4-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP112]], <4 x i16> [[TMP113]])
+; IC4VF4-NEXT:    [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP114]])
+; IC4VF4-NEXT:    [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP115]])
+; IC4VF4-NEXT:    [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
+; IC4VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
+; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
+; IC4VF4-NEXT:    br label %[[EXIT:.*]]
+; IC4VF4:       [[SCALAR_PH]]:
+; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
+; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
+; IC4VF4-NEXT:    br label %[[LOOP:.*]]
+; IC4VF4:       [[LOOP]]:
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF4-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; IC4VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC4VF4:       [[EXIT]]:
+; IC4VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF1-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; IC4VF1-SAME: i16 noundef [[VAL:%.*]]) {
+; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF1:       [[VECTOR_PH]]:
+; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF1:       [[VECTOR_BODY]]:
+; IC4VF1-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI1:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI2:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI3:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF1-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF1-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF1-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC4VF1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP0]]
+; IC4VF1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP1]]
+; IC4VF1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP2]]
+; IC4VF1-NEXT:    [[TMP7:%.*]] = load i16, ptr [[TMP3]], align 1
+; IC4VF1-NEXT:    [[TMP8:%.*]] = load i16, ptr [[TMP4]], align 1
+; IC4VF1-NEXT:    [[TMP9:%.*]] = load i16, ptr [[TMP5]], align 1
+; IC4VF1-NEXT:    [[TMP10:%.*]] = load i16, ptr [[TMP6]], align 1
+; IC4VF1-NEXT:    [[TMP11:%.*]] = icmp ugt i16 [[TMP7]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP12:%.*]] = icmp ugt i16 [[TMP8]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP13:%.*]] = icmp ugt i16 [[TMP9]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP14:%.*]] = icmp ugt i16 [[TMP10]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP15:%.*]] = add nsw i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP16:%.*]] = add nsw i16 [[TMP0]], -1
+; IC4VF1-NEXT:    [[TMP17:%.*]] = add nsw i16 [[TMP1]], -1
+; IC4VF1-NEXT:    [[TMP18:%.*]] = add nsw i16 [[TMP2]], -1
+; IC4VF1-NEXT:    [[TMP19]] = select i1 [[TMP11]], i16 [[TMP15]], i16 [[VEC_PHI]]
+; IC4VF1-NEXT:    [[TMP20]] = select i1 [[TMP12]], i16 [[TMP16]], i16 [[VEC_PHI1]]
+; IC4VF1-NEXT:    [[TMP21]] = select i1 [[TMP13]], i16 [[TMP17]], i16 [[VEC_PHI2]]
+; IC4VF1-NEXT:    [[TMP22]] = select i1 [[TMP14]], i16 [[TMP18]], i16 [[VEC_PHI3]]
+; IC4VF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC4VF1-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC4VF1-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC4VF1:       [[MIDDLE_BLOCK]]:
+; IC4VF1-NEXT:    [[RDX_MINMAX:%.*]] = call i16 @llvm.smin.i16(i16 [[TMP19]], i16 [[TMP20]])
+; IC4VF1-NEXT:    [[RDX_MINMAX4:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX]], i16 [[TMP21]])
+; IC4VF1-NEXT:    [[RDX_MINMAX5:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX4]], i16 [[TMP22]])
+; IC4VF1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[RDX_MINMAX5]], 32767
+; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[RDX_MINMAX5]], i16 0
+; IC4VF1-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF1:       [[SCALAR_PH]]:
+; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4VF1-NEXT:    br label %[[LOOP:.*]]
+; IC4VF1:       [[LOOP]]:
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF1-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF1-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; IC4VF1-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC4VF1:       [[EXIT]]:
+; IC4VF1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF1-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
 ;
 entry:
   br label %loop
@@ -83,23 +573,354 @@ exit:                                             ; preds = %loop
 @tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1
 
 define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
-; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
-; CHECK-SAME: half noundef [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
-; CHECK-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
-; CHECK-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
-; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+; IC1VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; IC1VF4-SAME: half noundef [[VAL:%.*]]) {
+; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC1VF4:       [[VECTOR_PH]]:
+; IC1VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x half> poison, half [[VAL]], i64 0
+; IC1VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x half> [[BROADCAST_SPLATINSERT]], <4 x half> poison, <4 x i32> zeroinitializer
+; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC1VF4:       [[VECTOR_BODY]]:
+; IC1VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC1VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC1VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC1VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds half, ptr [[TMP0]], i32 0
+; IC1VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds half, ptr [[TMP1]], i32 -3
+; IC1VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP2]], align 1
+; IC1VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x half> [[WIDE_LOAD]], <4 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC1VF4-NEXT:    [[TMP3:%.*]] = fcmp ugt <4 x half> [[REVERSE]], [[BROADCAST_SPLAT]]
+; IC1VF4-NEXT:    [[TMP4:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC1VF4-NEXT:    [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP4]], <4 x i16> [[VEC_PHI]]
+; IC1VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC1VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC1VF4-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC1VF4-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC1VF4:       [[MIDDLE_BLOCK]]:
+; IC1VF4-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[TMP5]])
+; IC1VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP7]], 32767
+; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP7]], i16 0
+; IC1VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC1VF4:       [[SCALAR_PH]]:
+; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC1VF4-NEXT:    br label %[[LOOP:.*]]
+; IC1VF4:       [[LOOP]]:
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC1VF4-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; IC1VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; IC1VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC1VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC1VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC1VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC1VF4:       [[EXIT]]:
+; IC1VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC1VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; IC4VF4-SAME: half noundef [[VAL:%.*]]) {
+; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF4:       [[VECTOR_PH]]:
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x half> poison, half [[VAL]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x half> [[BROADCAST_SPLATINSERT]], <4 x half> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF4:       [[VECTOR_BODY]]:
+; IC4VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44:.*]] ]
+; IC4VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP108:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP109:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP110:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP111:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 -4)
+; IC4VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 0, i32 1, i32 2, i32 3>
+; IC4VF4-NEXT:    [[VEC_IV8:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 4, i32 5, i32 6, i32 7>
+; IC4VF4-NEXT:    [[VEC_IV11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 8, i32 9, i32 10, i32 11>
+; IC4VF4-NEXT:    [[VEC_IV14:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 12, i32 13, i32 14, i32 15>
+; IC4VF4-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV8]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV11]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV14]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; IC4VF4:       [[PRED_LOAD_IF]]:
+; IC4VF4-NEXT:    [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 0
+; IC4VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP5]]
+; IC4VF4-NEXT:    [[TMP7:%.*]] = load half, ptr [[TMP6]], align 1
+; IC4VF4-NEXT:    [[TMP8:%.*]] = insertelement <4 x half> poison, half [[TMP7]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE]]:
+; IC4VF4-NEXT:    [[TMP9:%.*]] = phi <4 x half> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; IC4VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; IC4VF4:       [[PRED_LOAD_IF15]]:
+; IC4VF4-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP11]]
+; IC4VF4-NEXT:    [[TMP13:%.*]] = load half, ptr [[TMP12]], align 1
+; IC4VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x half> [[TMP9]], half [[TMP13]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE16]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE16]]:
+; IC4VF4-NEXT:    [[TMP15:%.*]] = phi <4 x half> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF15]] ]
+; IC4VF4-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP16]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; IC4VF4:       [[PRED_LOAD_IF17]]:
+; IC4VF4-NEXT:    [[TMP17:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP17]]
+; IC4VF4-NEXT:    [[TMP19:%.*]] = load half, ptr [[TMP18]], align 1
+; IC4VF4-NEXT:    [[TMP20:%.*]] = insertelement <4 x half> [[TMP15]], half [[TMP19]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE18]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE18]]:
+; IC4VF4-NEXT:    [[TMP21:%.*]] = phi <4 x half> [ [[TMP15]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP20]], %[[PRED_LOAD_IF17]] ]
+; IC4VF4-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP22]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; IC4VF4:       [[PRED_LOAD_IF19]]:
+; IC4VF4-NEXT:    [[TMP23:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF4-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP23]]
+; IC4VF4-NEXT:    [[TMP25:%.*]] = load half, ptr [[TMP24]], align 1
+; IC4VF4-NEXT:    [[TMP26:%.*]] = insertelement <4 x half> [[TMP21]], half [[TMP25]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE20]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE20]]:
+; IC4VF4-NEXT:    [[TMP27:%.*]] = phi <4 x half> [ [[TMP21]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP26]], %[[PRED_LOAD_IF19]] ]
+; IC4VF4-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP28]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; IC4VF4:       [[PRED_LOAD_IF21]]:
+; IC4VF4-NEXT:    [[TMP29:%.*]] = add i16 [[OFFSET_IDX]], -4
+; IC4VF4-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP29]]
+; IC4VF4-NEXT:    [[TMP31:%.*]] = load half, ptr [[TMP30]], align 1
+; IC4VF4-NEXT:    [[TMP32:%.*]] = insertelement <4 x half> poison, half [[TMP31]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE22]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE22]]:
+; IC4VF4-NEXT:    [[TMP33:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP32]], %[[PRED_LOAD_IF21]] ]
+; IC4VF4-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP34]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; IC4VF4:       [[PRED_LOAD_IF23]]:
+; IC4VF4-NEXT:    [[TMP35:%.*]] = add i16 [[OFFSET_IDX]], -5
+; IC4VF4-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP35]]
+; IC4VF4-NEXT:    [[TMP37:%.*]] = load half, ptr [[TMP36]], align 1
+; IC4VF4-NEXT:    [[TMP38:%.*]] = insertelement <4 x half> [[TMP33]], half [[TMP37]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE24]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE24]]:
+; IC4VF4-NEXT:    [[TMP39:%.*]] = phi <4 x half> [ [[TMP33]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP38]], %[[PRED_LOAD_IF23]] ]
+; IC4VF4-NEXT:    [[TMP40:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP40]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; IC4VF4:       [[PRED_LOAD_IF25]]:
+; IC4VF4-NEXT:    [[TMP41:%.*]] = add i16 [[OFFSET_IDX]], -6
+; IC4VF4-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP41]]
+; IC4VF4-NEXT:    [[TMP43:%.*]] = load half, ptr [[TMP42]], align 1
+; IC4VF4-NEXT:    [[TMP44:%.*]] = insertelement <4 x half> [[TMP39]], half [[TMP43]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE26]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE26]]:
+; IC4VF4-NEXT:    [[TMP45:%.*]] = phi <4 x half> [ [[TMP39]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP44]], %[[PRED_LOAD_IF25]] ]
+; IC4VF4-NEXT:    [[TMP46:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP46]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; IC4VF4:       [[PRED_LOAD_IF27]]:
+; IC4VF4-NEXT:    [[TMP47:%.*]] = add i16 [[OFFSET_IDX]], -7
+; IC4VF4-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP47]]
+; IC4VF4-NEXT:    [[TMP49:%.*]] = load half, ptr [[TMP48]], align 1
+; IC4VF4-NEXT:    [[TMP50:%.*]] = insertelement <4 x half> [[TMP45]], half [[TMP49]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE28]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE28]]:
+; IC4VF4-NEXT:    [[TMP51:%.*]] = phi <4 x half> [ [[TMP45]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP50]], %[[PRED_LOAD_IF27]] ]
+; IC4VF4-NEXT:    [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP52]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; IC4VF4:       [[PRED_LOAD_IF29]]:
+; IC4VF4-NEXT:    [[TMP53:%.*]] = add i16 [[OFFSET_IDX]], -8
+; IC4VF4-NEXT:    [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP53]]
+; IC4VF4-NEXT:    [[TMP55:%.*]] = load half, ptr [[TMP54]], align 1
+; IC4VF4-NEXT:    [[TMP56:%.*]] = insertelement <4 x half> poison, half [[TMP55]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE30]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE30]]:
+; IC4VF4-NEXT:    [[TMP57:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP56]], %[[PRED_LOAD_IF29]] ]
+; IC4VF4-NEXT:    [[TMP58:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP58]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; IC4VF4:       [[PRED_LOAD_IF31]]:
+; IC4VF4-NEXT:    [[TMP59:%.*]] = add i16 [[OFFSET_IDX]], -9
+; IC4VF4-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP59]]
+; IC4VF4-NEXT:    [[TMP61:%.*]] = load half, ptr [[TMP60]], align 1
+; IC4VF4-NEXT:    [[TMP62:%.*]] = insertelement <4 x half> [[TMP57]], half [[TMP61]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE32]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE32]]:
+; IC4VF4-NEXT:    [[TMP63:%.*]] = phi <4 x half> [ [[TMP57]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP62]], %[[PRED_LOAD_IF31]] ]
+; IC4VF4-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP64]], label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
+; IC4VF4:       [[PRED_LOAD_IF33]]:
+; IC4VF4-NEXT:    [[TMP65:%.*]] = add i16 [[OFFSET_IDX]], -10
+; IC4VF4-NEXT:    [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP65]]
+; IC4VF4-NEXT:    [[TMP67:%.*]] = load half, ptr [[TMP66]], align 1
+; IC4VF4-NEXT:    [[TMP68:%.*]] = insertelement <4 x half> [[TMP63]], half [[TMP67]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE34]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE34]]:
+; IC4VF4-NEXT:    [[TMP69:%.*]] = phi <4 x half> [ [[TMP63]], %[[PRED_LOAD_CONTINUE32]] ], [ [[TMP68]], %[[PRED_LOAD_IF33]] ]
+; IC4VF4-NEXT:    [[TMP70:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP70]], label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
+; IC4VF4:       [[PRED_LOAD_IF35]]:
+; IC4VF4-NEXT:    [[TMP71:%.*]] = add i16 [[OFFSET_IDX]], -11
+; IC4VF4-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP71]]
+; IC4VF4-NEXT:    [[TMP73:%.*]] = load half, ptr [[TMP72]], align 1
+; IC4VF4-NEXT:    [[TMP74:%.*]] = insertelement <4 x half> [[TMP69]], half [[TMP73]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE36]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE36]]:
+; IC4VF4-NEXT:    [[TMP75:%.*]] = phi <4 x half> [ [[TMP69]], %[[PRED_LOAD_CONTINUE34]] ], [ [[TMP74]], %[[PRED_LOAD_IF35]] ]
+; IC4VF4-NEXT:    [[TMP76:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP76]], label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
+; IC4VF4:       [[PRED_LOAD_IF37]]:
+; IC4VF4-NEXT:    [[TMP77:%.*]] = add i16 [[OFFSET_IDX]], -12
+; IC4VF4-NEXT:    [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP77]]
+; IC4VF4-NEXT:    [[TMP79:%.*]] = load half, ptr [[TMP78]], align 1
+; IC4VF4-NEXT:    [[TMP80:%.*]] = insertelement <4 x half> poison, half [[TMP79]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE38]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE38]]:
+; IC4VF4-NEXT:    [[TMP81:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE36]] ], [ [[TMP80]], %[[PRED_LOAD_IF37]] ]
+; IC4VF4-NEXT:    [[TMP82:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP82]], label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
+; IC4VF4:       [[PRED_LOAD_IF39]]:
+; IC4VF4-NEXT:    [[TMP83:%.*]] = add i16 [[OFFSET_IDX]], -13
+; IC4VF4-NEXT:    [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP83]]
+; IC4VF4-NEXT:    [[TMP85:%.*]] = load half, ptr [[TMP84]], align 1
+; IC4VF4-NEXT:    [[TMP86:%.*]] = insertelement <4 x half> [[TMP81]], half [[TMP85]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE40]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE40]]:
+; IC4VF4-NEXT:    [[TMP87:%.*]] = phi <4 x half> [ [[TMP81]], %[[PRED_LOAD_CONTINUE38]] ], [ [[TMP86]], %[[PRED_LOAD_IF39]] ]
+; IC4VF4-NEXT:    [[TMP88:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP88]], label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
+; IC4VF4:       [[PRED_LOAD_IF41]]:
+; IC4VF4-NEXT:    [[TMP89:%.*]] = add i16 [[OFFSET_IDX]], -14
+; IC4VF4-NEXT:    [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP89]]
+; IC4VF4-NEXT:    [[TMP91:%.*]] = load half, ptr [[TMP90]], align 1
+; IC4VF4-NEXT:    [[TMP92:%.*]] = insertelement <4 x half> [[TMP87]], half [[TMP91]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE42]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE42]]:
+; IC4VF4-NEXT:    [[TMP93:%.*]] = phi <4 x half> [ [[TMP87]], %[[PRED_LOAD_CONTINUE40]] ], [ [[TMP92]], %[[PRED_LOAD_IF41]] ]
+; IC4VF4-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP94]], label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_IF43]]:
+; IC4VF4-NEXT:    [[TMP95:%.*]] = add i16 [[OFFSET_IDX]], -15
+; IC4VF4-NEXT:    [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP95]]
+; IC4VF4-NEXT:    [[TMP97:%.*]] = load half, ptr [[TMP96]], align 1
+; IC4VF4-NEXT:    [[TMP98:%.*]] = insertelement <4 x half> [[TMP93]], half [[TMP97]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE44]]:
+; IC4VF4-NEXT:    [[TMP99:%.*]] = phi <4 x half> [ [[TMP93]], %[[PRED_LOAD_CONTINUE42]] ], [ [[TMP98]], %[[PRED_LOAD_IF43]] ]
+; IC4VF4-NEXT:    [[TMP100:%.*]] = fcmp ugt <4 x half> [[TMP27]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP101:%.*]] = fcmp ugt <4 x half> [[TMP51]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP102:%.*]] = fcmp ugt <4 x half> [[TMP75]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP103:%.*]] = fcmp ugt <4 x half> [[TMP99]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP104:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP105:%.*]] = add nsw <4 x i16> [[STEP_ADD]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP106:%.*]] = add nsw <4 x i16> [[STEP_ADD_2]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP107:%.*]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP108]] = select <4 x i1> [[TMP100]], <4 x i16> [[TMP104]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP109]] = select <4 x i1> [[TMP101]], <4 x i16> [[TMP105]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP110]] = select <4 x i1> [[TMP102]], <4 x i16> [[TMP106]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP111]] = select <4 x i1> [[TMP103]], <4 x i16> [[TMP107]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[TMP112:%.*]] = select <4 x i1> [[TMP0]], <4 x i16> [[TMP108]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP113:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP109]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP114:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP110]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP115:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP111]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; IC4VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD_3]], splat (i16 -4)
+; IC4VF4-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC4VF4:       [[MIDDLE_BLOCK]]:
+; IC4VF4-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP112]], <4 x i16> [[TMP113]])
+; IC4VF4-NEXT:    [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP114]])
+; IC4VF4-NEXT:    [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP115]])
+; IC4VF4-NEXT:    [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
+; IC4VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
+; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
+; IC4VF4-NEXT:    br label %[[EXIT:.*]]
+; IC4VF4:       [[SCALAR_PH]]:
+; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
+; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
+; IC4VF4-NEXT:    br label %[[LOOP:.*]]
+; IC4VF4:       [[LOOP]]:
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF4-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; IC4VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC4VF4:       [[EXIT]]:
+; IC4VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF1-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; IC4VF1-SAME: half noundef [[VAL:%.*]]) {
+; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF1:       [[VECTOR_PH]]:
+; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF1:       [[VECTOR_BODY]]:
+; IC4VF1-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI1:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI2:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI3:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF1-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF1-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF1-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC4VF1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP0]]
+; IC4VF1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP1]]
+; IC4VF1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP2]]
+; IC4VF1-NEXT:    [[TMP7:%.*]] = load half, ptr [[TMP3]], align 1
+; IC4VF1-NEXT:    [[TMP8:%.*]] = load half, ptr [[TMP4]], align 1
+; IC4VF1-NEXT:    [[TMP9:%.*]] = load half, ptr [[TMP5]], align 1
+; IC4VF1-NEXT:    [[TMP10:%.*]] = load half, ptr [[TMP6]], align 1
+; IC4VF1-NEXT:    [[TMP11:%.*]] = fcmp ugt half [[TMP7]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP12:%.*]] = fcmp ugt half [[TMP8]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP13:%.*]] = fcmp ugt half [[TMP9]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP14:%.*]] = fcmp ugt half [[TMP10]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP15:%.*]] = add nsw i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP16:%.*]] = add nsw i16 [[TMP0]], -1
+; IC4VF1-NEXT:    [[TMP17:%.*]] = add nsw i16 [[TMP1]], -1
+; IC4VF1-NEXT:    [[TMP18:%.*]] = add nsw i16 [[TMP2]], -1
+; IC4VF1-NEXT:    [[TMP19]] = select i1 [[TMP11]], i16 [[TMP15]], i16 [[VEC_PHI]]
+; IC4VF1-NEXT:    [[TMP20]] = select i1 [[TMP12]], i16 [[TMP16]], i16 [[VEC_PHI1]]
+; IC4VF1-NEXT:    [[TMP21]] = select i1 [[TMP13]], i16 [[TMP17]], i16 [[VEC_PHI2]]
+; IC4VF1-NEXT:    [[TMP22]] = select i1 [[TMP14]], i16 [[TMP18]], i16 [[VEC_PHI3]]
+; IC4VF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC4VF1-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC4VF1-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC4VF1:       [[MIDDLE_BLOCK]]:
+; IC4VF1-NEXT:    [[RDX_MINMAX:%.*]] = call i16 @llvm.smin.i16(i16 [[TMP19]], i16 [[TMP20]])
+; IC4VF1-NEXT:    [[RDX_MINMAX4:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX]], i16 [[TMP21]])
+; IC4VF1-NEXT:    [[RDX_MINMAX5:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX4]], i16 [[TMP22]])
+; IC4VF1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[RDX_MINMAX5]], 32767
+; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[RDX_MINMAX5]], i16 0
+; IC4VF1-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF1:       [[SCALAR_PH]]:
+; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4VF1-NEXT:    br label %[[LOOP:.*]]
+; IC4VF1:       [[LOOP]]:
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF1-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF1-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; IC4VF1-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC4VF1:       [[EXIT]]:
+; IC4VF1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF1-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
 ;
 entry:
   br label %loop
@@ -199,4 +1020,3 @@ loop:                                             ; preds = %entry, %loop
 exit:                                             ; preds = %loop
   ret i64 %spec.select
 }
-
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 796c1d116aa19..13cc1b657d231 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
-; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
-; CHECK-NEXT:    [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
-; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
new file mode 100644
index 0000000000000..2747895f06a7b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=2 -S %s | FileCheck %s
+
+declare noalias noundef ptr @calloc(i64 noundef, i64 noundef)
+
+define void @reuse_lcssa_phi_for_add_rec1(ptr %head) {
+; CHECK-LABEL: define void @reuse_lcssa_phi_for_add_rec1(
+; CHECK-SAME: ptr [[HEAD:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi ptr [ [[HEAD]], %[[ENTRY]] ], [ [[L_1:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[L_1]] = load ptr, ptr [[FOR]], align 8
+; CHECK-NEXT:    [[IV_2_NEXT]] = add nuw nsw i32 [[IV_2]], 1
+; CHECK-NEXT:    [[EC_1:%.*]] = icmp eq ptr [[L_1]], null
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT:    br i1 [[EC_1]], label %[[PH:.*]], label %[[LOOP_1]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_2_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[SRC_2:%.*]] = tail call noalias noundef dereferenceable_or_null(8) ptr @calloc(i64 1, i64 8)
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[IV_2_LCSSA]], 1
+; CHECK-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[IV_LCSSA]], [[N_VEC]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[IV_LCSSA]], [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[SRC_2]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr ptr, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr ptr, ptr [[TMP6]], i32 -1
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x ptr> [[WIDE_LOAD]], <2 x ptr> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x ptr> [[REVERSE]], i32 0
+; CHECK-NEXT:    store ptr null, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x ptr> [[REVERSE]], i32 1
+; CHECK-NEXT:    store ptr null, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[IV_LCSSA]], %[[PH]] ]
+; CHECK-NEXT:    br label %[[LOOP_2:.*]]
+; CHECK:       [[LOOP_2]]:
+; CHECK-NEXT:    [[IV_3:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_3:%.*]], %[[LOOP_2]] ]
+; CHECK-NEXT:    [[IV_NEXT_3]] = add nsw i64 [[IV_3]], -1
+; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr ptr, ptr [[SRC_2]], i64 [[IV_3]]
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[GEP_SRC_2]], align 8
+; CHECK-NEXT:    store ptr null, ptr [[L]], align 8
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i64 [[IV_3]] to i32
+; CHECK-NEXT:    [[EC_2:%.*]] = icmp sgt i32 [[T]], 1
+; CHECK-NEXT:    br i1 [[EC_2]], label %[[LOOP_2]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop.1 ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop.1 ]
+  %for = phi ptr [ %head, %entry ], [ %l.1, %loop.1 ]
+  %l.1 = load ptr, ptr %for, align 8
+  %iv.2.next = add nuw nsw i32 %iv.2, 1
+  %ec.1 = icmp eq ptr %l.1, null
+  %iv.next = add nuw i64 %iv, 1
+  br i1 %ec.1, label %ph, label %loop.1
+
+ph:
+  %iv.lcssa = phi i64 [ %iv, %loop.1 ]
+  %iv.2.next.lcssa = phi i32 [ %iv.2.next, %loop.1 ]
+  %src.2 = tail call noalias noundef dereferenceable_or_null(8) ptr @calloc(i64 1, i64 8)
+  br label %loop.2
+
+loop.2:
+  %iv.3 = phi i64 [ %iv.lcssa, %ph ], [ %iv.next.3, %loop.2 ]
+  %iv.next.3 = add nsw i64 %iv.3, -1
+  %gep.src.2 = getelementptr ptr, ptr %src.2, i64 %iv.3
+  %l = load ptr, ptr %gep.src.2, align 8
+  store ptr null, ptr %l, align 8
+  %t = trunc nuw i64 %iv.3 to i32
+  %ec.2 = icmp sgt i32 %t, 1
+  br i1 %ec.2, label %loop.2, label %for.end
+
+for.end:
+  ret void
+}
+
+declare i32 @val()
+
+define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) {
+; CHECK-LABEL: define void @runtime_checks_ptr_inductions(
+; CHECK-SAME: ptr [[DST_1:%.*]], ptr [[DST_2:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[DST_11:%.*]] = ptrtoint ptr [[DST_1]] to i64
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[PTR_IV_1:%.*]] = phi ptr [ [[DST_1]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @val()
+; CHECK-NEXT:    [[SEL_DST:%.*]] = select i1 [[C]], ptr [[DST_1]], ptr [[DST_2]]
+; CHECK-NEXT:    [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 1
+; CHECK-NEXT:    [[EC_1:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[EC_1]], label %[[LOOP_2_HEADER_PREHEADER:.*]], label %[[LOOP_1]]
+; CHECK:       [[LOOP_2_HEADER_PREHEADER]]:
+; CHECK-NEXT:    [[SEL_DST_LCSSA2:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[PTR_IV_1_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[SEL_DST_LCSSA:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[SEL_DST_LCSSA23:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA2]] to i64
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDVAR_LCSSA]], [[DST_11]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[SEL_DST_LCSSA23]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 2
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR_IV_1_LCSSA]], i64 1022
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[SEL_DST_LCSSA]], i64 1022
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_IV_1_LCSSA]], i64 [[INDEX]]
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[SEL_DST_LCSSA]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-NEXT:    store <2 x i8> [[WIDE_LOAD]], ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1022
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1023, %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_2_HEADER_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP_2_HEADER:.*]]
+; CHECK:       [[LOOP_2_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[DEC7:%.*]], %[[LOOP_2_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[EC_2:%.*]] = icmp eq i32 [[IV]], 1024
+; CHECK-NEXT:    br i1 [[EC_2]], label %[[EXIT:.*]], label %[[LOOP_2_LATCH]]
+; CHECK:       [[LOOP_2_LATCH]]:
+; CHECK-NEXT:    [[DEC7]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[PTR_IV_3_NEXT]] = getelementptr i8, ptr [[PTR_IV_3]], i64 1
+; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV_3]], align 1
+; CHECK-NEXT:    [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 1
+; CHECK-NEXT:    store i8 [[L]], ptr [[PTR_IV_2]], align 1
+; CHECK-NEXT:    br label %[[LOOP_2_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %ptr.iv.1 = phi ptr [ %dst.1, %entry ], [ %ptr.iv.1.next, %loop.1 ]
+  %call = call i32 @val()
+  %sel.dst = select i1 %c, ptr %dst.1, ptr %dst.2
+  %ptr.iv.1.next = getelementptr i8, ptr %ptr.iv.1, i64 1
+  %ec.1 = icmp eq i32 %call, 0
+  br i1 %ec.1, label %loop.2.header, label %loop.1
+
+loop.2.header:
+  %iv = phi i32 [ 1, %loop.1 ], [ %dec7, %loop.2.latch ]
+  %ptr.iv.2 = phi ptr [ %ptr.iv.1, %loop.1 ], [ %ptr.iv.2.next, %loop.2.latch ]
+  %ptr.iv.3 = phi ptr [ %sel.dst, %loop.1 ], [ %ptr.iv.3.next, %loop.2.latch ]
+  %ec.2 = icmp eq i32 %iv, 1024
+  br i1 %ec.2, label %exit, label %loop.2.latch
+
+loop.2.latch:
+  %dec7 = add i32 %iv, 1
+  %ptr.iv.3.next = getelementptr i8, ptr %ptr.iv.3, i64 1
+  %l = load i8, ptr %ptr.iv.3, align 1
+  %ptr.iv.2.next = getelementptr i8, ptr %ptr.iv.2, i64 1
+  store i8 %l, ptr %ptr.iv.2, align 1
+  br label %loop.2.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
index 079f6b73e8861..e901d9801a143 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
-; CHECK-NEXT:    [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
 ; CHECK-NEXT:    [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll b/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll
new file mode 100644
index 0000000000000..def239eed33bc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s
+
+define void @neg_cond(ptr noalias %p, ptr noalias %q) {
+; CHECK-LABEL: define void @neg_cond(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> splat (i32 42), <4 x i32> splat (i32 43)
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[P_GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P_GEP]], align 4
+; CHECK-NEXT:    [[Q_GEP:%.*]] = getelementptr i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT:    [[Y:%.*]] = load i32, ptr [[Q_GEP]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 42
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[CMP]], true
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[NOT]], i32 42, i32 43
+; CHECK-NEXT:    store i32 [[SEL]], ptr [[P_GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %p.gep = getelementptr i32, ptr %p, i32 %iv
+  %x = load i32, ptr %p.gep
+  %q.gep = getelementptr i32, ptr %q, i32 %iv
+  %y = load i32, ptr %q.gep
+  %cmp = icmp eq i32 %x, 42
+  %not = xor i1 %cmp, 1
+  %sel = select i1 %not, i32 42, i32 43
+  store i32 %sel, ptr %p.gep
+  %iv.next = add i32 %iv, 1
+  %done = icmp eq i32 %iv.next, 1024
+  br i1 %done, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
new file mode 100644
index 0000000000000..0fe893abec86c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 | FileCheck %s
+
+define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size(ptr noalias %p1, ptr noalias %p2) nofree nosync {
+; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size(
+; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK:       loop.inc:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK:       loop.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ -1, [[LOOP_INC]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
+;
+entry:
+  call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 1024) ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 1024) ]
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 1024
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ -1, %loop.inc ]
+  ret i64 %retval
+}
+
+define i64 @early_exit_alignment_and_deref_via_assumption_with_constant_size_too_small(ptr noalias %p1, ptr noalias %p2) nofree nosync {
+; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_via_assumption_with_constant_size_too_small(
+; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK:       loop.inc:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1025
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK:       loop.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ -1, [[LOOP_INC]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
+;
+entry:
+  call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 1024) ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 1024) ]
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 1025
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ -1, %loop.inc ]
+  ret i64 %retval
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
index 9428737814146..4af3fa9202c77 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
@@ -238,7 +238,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
+; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
 ; CHECK-NEXT:   EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -284,12 +284,12 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR:%.+]]> = vector-pointer ir<%arrayidx>
 ; CHECK-NEXT:     WIDEN ir<[[LOAD:%.+]]> = load vp<[[ADDR]]>
-; CHECK-NEXT:     EXTENDED-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (ir<[[LOAD]]> zext to i64)
+; CHECK-NEXT:     EXPRESSION vp<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (ir<[[LOAD]]> zext to i64)
 ; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
@@ -329,7 +329,7 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX0:%.+]]> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR0:%.+]]> = vector-pointer ir<[[ARRAYIDX0]]>
@@ -337,7 +337,7 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX1:%.+]]> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR1:%.+]]> = vector-pointer ir<[[ARRAYIDX1]]>
 ; CHECK-NEXT:     WIDEN ir<[[LOAD1:%.+]]> = load vp<[[ADDR1]]>
-; CHECK-NEXT:     MULACC-REDUCE ir<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (mul nsw ir<[[LOAD0]]>, ir<[[LOAD1]]>)
+; CHECK-NEXT:     EXPRESSION vp<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (mul nsw ir<[[LOAD0]]>, ir<[[LOAD1]]>)
 ; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
@@ -379,7 +379,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX0:%.+]]> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR0:%.+]]> = vector-pointer ir<[[ARRAYIDX0]]>
@@ -387,7 +387,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX1:%.+]]> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR1:%.+]]> = vector-pointer ir<[[ARRAYIDX1]]>
 ; CHECK-NEXT:     WIDEN ir<[[LOAD1:%.+]]> = load vp<[[ADDR1]]>
-; CHECK-NEXT:     MULACC-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (mul nsw (ir<[[LOAD0]]> sext to i64), (ir<[[LOAD1]]> sext to i64))
+; CHECK-NEXT:     EXPRESSION vp<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (mul nsw (ir<[[LOAD0]]> sext to i64), (ir<[[LOAD1]]> sext to i64))
 ; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll b/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
index daf4a7b799dd4..bbf893f6127b0 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
@@ -12,20 +12,10 @@
 ;
 
 define <4 x double> @x86_pblendvb_v4f64_v2f64(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d) {
-; SSE-LABEL: @x86_pblendvb_v4f64_v2f64(
-; SSE-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; SSE-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; SSE-NEXT:    ret <4 x double> [[DOTV]]
-;
-; AVX2-LABEL: @x86_pblendvb_v4f64_v2f64(
-; AVX2-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; AVX2-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; AVX2-NEXT:    ret <4 x double> [[DOTV]]
-;
-; AVX512-LABEL: @x86_pblendvb_v4f64_v2f64(
-; AVX512-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; AVX512-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[CMP]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; AVX512-NEXT:    ret <4 x double> [[DOTV]]
+; CHECK-LABEL: @x86_pblendvb_v4f64_v2f64(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[CMP]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[DOTV]]
 ;
   %a.bc = bitcast <4 x double> %a to <32 x i8>
   %b.bc = bitcast <4 x double> %b to <32 x i8>
@@ -46,20 +36,10 @@ define <4 x double> @x86_pblendvb_v4f64_v2f64(<4 x double> %a, <4 x double> %b,
 }
 
 define <8 x float> @x86_pblendvb_v8f32_v4f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
-; SSE-LABEL: @x86_pblendvb_v8f32_v4f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; SSE-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; SSE-NEXT:    ret <8 x float> [[DOTV]]
-;
-; AVX2-LABEL: @x86_pblendvb_v8f32_v4f32(
-; AVX2-NEXT:    [[TMP1:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; AVX2-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; AVX2-NEXT:    ret <8 x float> [[DOTV]]
-;
-; AVX512-LABEL: @x86_pblendvb_v8f32_v4f32(
-; AVX512-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; AVX512-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; AVX512-NEXT:    ret <8 x float> [[DOTV]]
+; CHECK-LABEL: @x86_pblendvb_v8f32_v4f32(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[DOTV]]
 ;
   %a.bc = bitcast <8 x float> %a to <32 x i8>
   %b.bc = bitcast <8 x float> %b to <32 x i8>
@@ -80,20 +60,10 @@ define <8 x float> @x86_pblendvb_v8f32_v4f32(<8 x float> %a, <8 x float> %b, <8
 }
 
 define <4 x i64> @x86_pblendvb_v4i64_v2i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; SSE-LABEL: @x86_pblendvb_v4i64_v2i64(
-; SSE-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; SSE-NEXT:    ret <4 x i64> [[TMP2]]
-;
-; AVX2-LABEL: @x86_pblendvb_v4i64_v2i64(
-; AVX2-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; AVX2-NEXT:    ret <4 x i64> [[TMP2]]
-;
-; AVX512-LABEL: @x86_pblendvb_v4i64_v2i64(
-; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; AVX512-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[CMP]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; AVX512-NEXT:    ret <4 x i64> [[TMP1]]
+; CHECK-LABEL: @x86_pblendvb_v4i64_v2i64(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[CMP]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -216,35 +186,15 @@ define <4 x i64> @x86_pblendvb_v16i16_v8i16(<4 x i64> %a, <4 x i64> %b, <4 x i64
 }
 
 define <4 x i64> @x86_pblendvb_v32i8_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; SSE-LABEL: @x86_pblendvb_v32i8_v16i8(
-; SSE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; SSE-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; SSE-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; SSE-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; SSE-NEXT:    [[TMP3:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; SSE-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[TMP3]], <32 x i8> [[TMP2]], <32 x i8> [[TMP1]]
-; SSE-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; SSE-NEXT:    ret <4 x i64> [[RES]]
-;
-; AVX2-LABEL: @x86_pblendvb_v32i8_v16i8(
-; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[TMP3:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; AVX2-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[TMP3]], <32 x i8> [[TMP2]], <32 x i8> [[TMP1]]
-; AVX2-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; AVX2-NEXT:    ret <4 x i64> [[RES]]
-;
-; AVX512-LABEL: @x86_pblendvb_v32i8_v16i8(
-; AVX512-NEXT:    [[A_BC:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[B_BC:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; AVX512-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[CMP]], <32 x i8> [[B_BC]], <32 x i8> [[A_BC]]
-; AVX512-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; AVX512-NEXT:    ret <4 x i64> [[RES]]
+; CHECK-LABEL: @x86_pblendvb_v32i8_v16i8(
+; CHECK-NEXT:    [[A_BC:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[B_BC:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
+; CHECK-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[CMP]], <32 x i8> [[B_BC]], <32 x i8> [[A_BC]]
+; CHECK-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -424,35 +374,15 @@ define <8 x i64> @x86_pblendvb_v32i16_v16i16(<8 x i64> %a, <8 x i64> %b, <8 x i6
 }
 
 define <8 x i64> @x86_pblendvb_v64i8_v32i8(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
-; SSE-LABEL: @x86_pblendvb_v64i8_v32i8(
-; SSE-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; SSE-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; SSE-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; SSE-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; SSE-NEXT:    [[TMP3:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; SSE-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[TMP3]], <64 x i8> [[TMP2]], <64 x i8> [[TMP1]]
-; SSE-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; SSE-NEXT:    ret <8 x i64> [[RES]]
-;
-; AVX2-LABEL: @x86_pblendvb_v64i8_v32i8(
-; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[TMP3:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; AVX2-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[TMP3]], <64 x i8> [[TMP2]], <64 x i8> [[TMP1]]
-; AVX2-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; AVX2-NEXT:    ret <8 x i64> [[RES]]
-;
-; AVX512-LABEL: @x86_pblendvb_v64i8_v32i8(
-; AVX512-NEXT:    [[A_BC:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[B_BC:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; AVX512-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[CMP]], <64 x i8> [[B_BC]], <64 x i8> [[A_BC]]
-; AVX512-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; AVX512-NEXT:    ret <8 x i64> [[RES]]
+; CHECK-LABEL: @x86_pblendvb_v64i8_v32i8(
+; CHECK-NEXT:    [[A_BC:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[B_BC:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
+; CHECK-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[CMP]], <64 x i8> [[B_BC]], <64 x i8> [[A_BC]]
+; CHECK-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
 ;
   %a.bc = bitcast <8 x i64> %a to <64 x i8>
   %b.bc = bitcast <8 x i64> %b to <64 x i8>
diff --git a/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll b/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
new file mode 100644
index 0000000000000..63279b04bda28
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt --Os -S %s -o - | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64e-apple-ios19.0.0"
+
+; This test checks if we generate a phi for the <vscale 16 x float> value or we manage
+; to promote it during inlining.
+
+; Function Attrs: mustprogress optsize ssp uwtable(sync)
+define void @pluto() #0 {
+; CHECK-LABEL: define void @pluto(
+; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT:    br label %[[SNORK_EXIT:.*]]
+; CHECK:       [[SNORK_EXIT]]:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi <vscale x 16 x float> [ undef, [[TMP0:%.*]] ], [ [[SPEC_SELECT:%.*]], %[[SNORK_EXIT]] ]
+; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[TMP2]], <vscale x 16 x float> [[TMP3]], <vscale x 16 x float> [[DOT0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[SPEC_SELECT]], i64 0)
+; CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> [[TMP4]])
+; CHECK-NEXT:    br label %[[SNORK_EXIT]]
+;
+  br label %1
+
+1:                                                ; preds = %1, %0
+  call void @ham() #5
+  br label %1
+}
+
+; Function Attrs: alwaysinline mustprogress optsize ssp uwtable(sync)
+define void @ham() #1 {
+; CHECK-LABEL: define void @ham(
+; CHECK-SAME: ) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[SNORK_EXIT:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP1]], <vscale x 16 x float> [[TMP2]], <vscale x 16 x float> undef
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[SPEC_SELECT]], i64 0)
+; CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> [[TMP3]])
+; CHECK-NEXT:    ret void
+;
+  %1 = alloca <vscale x 16 x float>, align 16
+  %2 = load i64, ptr inttoptr (i64 48 to ptr), align 8
+  %3 = call i64 @snork(i64 noundef %2, ptr noundef nonnull align 16 %1) #5
+  %4 = load <vscale x 16 x float>, ptr %1, align 16
+  %5 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> %4, i64 0)
+  call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> %5)
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float>, <vscale x 4 x float>, i64 immarg) #2
+
+; Function Attrs: alwaysinline mustprogress nounwind optsize ssp uwtable(sync)
+define i64 @snork(i64 noundef %0, ptr noundef nonnull align 16 %1) #3 {
+; CHECK-LABEL: define noundef i64 @snork(
+; CHECK-SAME: i64 noundef [[TMP0:%.*]], ptr noundef nonnull writeonly align 16 captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i64 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[BB4:.*]], label %[[BB6:.*]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT:    store <vscale x 16 x float> [[TMP5]], ptr [[TMP1]], align 16
+; CHECK-NEXT:    br label %[[BB6]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    ret i64 0
+;
+  %3 = icmp sgt i64 %0, 0
+  br i1 %3, label %4, label %6
+
+4:                                                ; preds = %2
+  %5 = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+  store <vscale x 16 x float> %5, ptr %1, align 16
+  br label %6
+
+6:                                                ; preds = %4, %2
+  ret i64 0
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float>, i64 immarg) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
+declare void @llvm.aarch64.sme.mopa.nxv4f32(i32 immarg, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>) #4
+
+attributes #0 = { mustprogress optsize ssp uwtable(sync) "aarch64_new_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "no-builtin-calloc" "no-builtin-stpcpy" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m4" }
+attributes #1 = { alwaysinline mustprogress optsize ssp uwtable(sync) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "no-builtin-calloc" "no-builtin-stpcpy" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m4" }
+attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { alwaysinline mustprogress nounwind optsize ssp uwtable(sync) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "no-builtin-calloc" "no-builtin-stpcpy" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m4" }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
+attributes #5 = { optsize "aarch64_inout_za" "aarch64_pstate_sm_enabled" "no-builtin-calloc" "no-builtin-stpcpy" }
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
index e53b8687af915..45f18dd567396 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
@@ -24,7 +24,8 @@ define void @test_fill_with_foreach([2 x i64] %elems.coerce) {
 ; CHECK-NEXT:    [[ELEMS_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x i64] [[ELEMS_COERCE]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = inttoptr i64 [[ELEMS_COERCE_FCA_0_EXTRACT]] to ptr
 ; CHECK-NEXT:    [[ELEMS_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x i64] [[ELEMS_COERCE]], 1
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[ELEMS_COERCE_FCA_1_EXTRACT]]
+; CHECK-NEXT:    [[ADD_PTR_I_IDX:%.*]] = shl nsw i64 [[ELEMS_COERCE_FCA_1_EXTRACT]], 2
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 [[ADD_PTR_I_IDX]]
 ; CHECK-NEXT:    [[CMP_NOT_I_I_I_I:%.*]] = icmp slt i64 [[ELEMS_COERCE_FCA_1_EXTRACT]], 0
 ; CHECK-NEXT:    br i1 [[CMP_NOT_I_I_I_I]], label [[ERROR:%.*]], label [[FOR_COND_PREHEADER_SPLIT:%.*]]
 ; CHECK:       for.cond.preheader.split:
diff --git a/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll b/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll
index 78b8a4aa126c9..322c38d090fe1 100644
--- a/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll
+++ b/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll
@@ -20,6 +20,14 @@
 @y3 = internal unnamed_addr constant ptr @x0
 @load_relative_2.table = private unnamed_addr constant [4 x ptr] [ptr @y3, ptr @y2, ptr @y1, ptr @y0]
 
+@b0 = private unnamed_addr constant [8 x i8] c"00000000"
+@b1 = private unnamed_addr constant [8 x i8] c"11111111"
+@b2 = private unnamed_addr constant [8 x i8] c"22222222"
+@load_relative_3.table = private unnamed_addr constant [3 x ptr] [
+  ptr getelementptr inbounds (i8, ptr @b0, i64 8),
+  ptr getelementptr inbounds (i8, ptr @b1, i64 8),
+  ptr getelementptr inbounds (i8, ptr @b2, i64 8)]
+
 ;.
 ; x86_64-apple-darwin: @a0 = private constant i32 0
 ; x86_64-apple-darwin: @a1 = private constant i32 1
@@ -34,6 +42,10 @@
 ; x86_64-apple-darwin: @y2 = internal constant ptr @x1
 ; x86_64-apple-darwin: @y3 = internal constant ptr @x0
 ; x86_64-apple-darwin: @load_relative_2.table.rel = private unnamed_addr constant [4 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @y3 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y2 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y1 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y0 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32)], align 4
+; x86_64-apple-darwin: @b0 = private constant [8 x i8] c"00000000"
+; x86_64-apple-darwin: @b1 = private constant [8 x i8] c"11111111"
+; x86_64-apple-darwin: @b2 = private constant [8 x i8] c"22222222"
+; x86_64-apple-darwin: @load_relative_3.table.rel = private unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b0, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b1, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b2, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32)], align 4
 ;.
 ; aarch64: @a0 = private constant i32 0
 ; aarch64: @a1 = private constant i32 1
@@ -48,6 +60,10 @@
 ; aarch64: @y2 = internal constant ptr @x1
 ; aarch64: @y3 = internal constant ptr @x0
 ; aarch64: @load_relative_2.table.rel = private unnamed_addr constant [4 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @y3 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y2 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y1 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y0 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32)], align 4
+; aarch64: @b0 = private constant [8 x i8] c"00000000"
+; aarch64: @b1 = private constant [8 x i8] c"11111111"
+; aarch64: @b2 = private constant [8 x i8] c"22222222"
+; aarch64: @load_relative_3.table.rel = private unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b0, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b1, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b2, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32)], align 4
 ;.
 ; x86_64: @a0 = private unnamed_addr constant i32 0
 ; x86_64: @a1 = private unnamed_addr constant i32 1
@@ -62,6 +78,10 @@
 ; x86_64: @y2 = internal unnamed_addr constant ptr @x1
 ; x86_64: @y3 = internal unnamed_addr constant ptr @x0
 ; x86_64: @load_relative_2.table.rel = private unnamed_addr constant [4 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @y3 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y2 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y1 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y0 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32)], align 4
+; x86_64: @b0 = private unnamed_addr constant [8 x i8] c"00000000"
+; x86_64: @b1 = private unnamed_addr constant [8 x i8] c"11111111"
+; x86_64: @b2 = private unnamed_addr constant [8 x i8] c"22222222"
+; x86_64: @load_relative_3.table.rel = private unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b0, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b1, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b2, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32)], align 4
 ;.
 define ptr @load_relative_1(i64 %offset) {
 ; x86_64-apple-darwin-LABEL: define ptr @load_relative_1(
@@ -110,6 +130,31 @@ define ptr @load_relative_2(i64 %offset) {
   %load = load ptr, ptr %gep
   ret ptr %load
 }
+
+define ptr @load_relative_3(i64 %offset) {
+; x86_64-apple-darwin-LABEL: define ptr @load_relative_3(
+; x86_64-apple-darwin-SAME: i64 [[OFFSET:%.*]]) {
+; x86_64-apple-darwin-NEXT:    [[RELTABLE_SHIFT:%.*]] = shl i64 [[OFFSET]], 2
+; x86_64-apple-darwin-NEXT:    [[RELTABLE_INTRINSIC:%.*]] = call ptr @llvm.load.relative.i64(ptr @load_relative_3.table.rel, i64 [[RELTABLE_SHIFT]])
+; x86_64-apple-darwin-NEXT:    ret ptr [[RELTABLE_INTRINSIC]]
+;
+; aarch64-LABEL: define ptr @load_relative_3(
+; aarch64-SAME: i64 [[OFFSET:%.*]]) {
+; aarch64-NEXT:    [[RELTABLE_SHIFT:%.*]] = shl i64 [[OFFSET]], 2
+; aarch64-NEXT:    [[RELTABLE_INTRINSIC:%.*]] = call ptr @llvm.load.relative.i64(ptr @load_relative_3.table.rel, i64 [[RELTABLE_SHIFT]])
+; aarch64-NEXT:    ret ptr [[RELTABLE_INTRINSIC]]
+;
+; x86_64-LABEL: define ptr @load_relative_3(
+; x86_64-SAME: i64 [[OFFSET:%.*]]) {
+; x86_64-NEXT:    [[RELTABLE_SHIFT:%.*]] = shl i64 [[OFFSET]], 2
+; x86_64-NEXT:    [[RELTABLE_INTRINSIC:%.*]] = call ptr @llvm.load.relative.i64(ptr @load_relative_3.table.rel, i64 [[RELTABLE_SHIFT]])
+; x86_64-NEXT:    ret ptr [[RELTABLE_INTRINSIC]]
+;
+  %gep = getelementptr inbounds [3 x ptr], ptr @load_relative_3.table, i64 0, i64 %offset
+  %load = load ptr, ptr %gep
+  ret ptr %load
+}
+
 ;.
 ; x86_64-apple-darwin: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
 ;.
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
index e460f558f4723..8dc88f7b96716 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes=slp-vectorizer,instcombine -pass-remarks-output=%t | FileCheck %s
+; RUN: opt < %s -S -passes=slp-vectorizer -pass-remarks-output=%t | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=REMARK %s
-; RUN: opt < %s -S -aa-pipeline=basic-aa -passes='slp-vectorizer,instcombine' -pass-remarks-output=%t | FileCheck %s
+; RUN: opt < %s -S -aa-pipeline=basic-aa -passes='slp-vectorizer' -pass-remarks-output=%t | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=REMARK %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -16,10 +16,10 @@ target triple = "aarch64--linux-gnu"
 
 define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @gather_multiple_use(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i64 1
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i64 2
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], splat (i32 15)
 ; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], splat (i32 65537)
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw <4 x i32> [[TMP6]], splat (i32 65535)
@@ -57,22 +57,26 @@ define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
 @data = global [6 x [258 x i8]] zeroinitializer, align 1
 define void @gather_load(ptr noalias %ptr) {
 ; CHECK-LABEL: @gather_load(
-; CHECK-NEXT:    [[ARRAYIDX182:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX183:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX185:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 8
-; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 258), align 1
+; CHECK-NEXT:    [[ARRAYIDX182:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX183:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX185:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX149:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 1, i64 0
+; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr [[ARRAYIDX149]], align 1
 ; CHECK-NEXT:    [[CONV150:%.*]] = zext i8 [[L0]] to i16
-; CHECK-NEXT:    [[ADD152:%.*]] = add nuw nsw i16 [[CONV150]], 10
-; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 517), align 1
+; CHECK-NEXT:    [[ADD152:%.*]] = add i16 10, [[CONV150]]
+; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 2, i64 1
+; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr [[ARRAYIDX155]], align 1
 ; CHECK-NEXT:    [[CONV156:%.*]] = zext i8 [[L1]] to i16
-; CHECK-NEXT:    [[ADD158:%.*]] = add nuw nsw i16 [[CONV156]], 20
-; CHECK-NEXT:    [[L2:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 776), align 1
+; CHECK-NEXT:    [[ADD158:%.*]] = add i16 20, [[CONV156]]
+; CHECK-NEXT:    [[ARRAYIDX161:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 3, i64 2
+; CHECK-NEXT:    [[L2:%.*]] = load i8, ptr [[ARRAYIDX161]], align 1
 ; CHECK-NEXT:    [[CONV162:%.*]] = zext i8 [[L2]] to i16
-; CHECK-NEXT:    [[ADD164:%.*]] = add nuw nsw i16 [[CONV162]], 30
-; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 1035), align 1
+; CHECK-NEXT:    [[ADD164:%.*]] = add i16 30, [[CONV162]]
+; CHECK-NEXT:    [[ARRAYIDX167:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 4, i64 3
+; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr [[ARRAYIDX167]], align 1
 ; CHECK-NEXT:    [[CONV168:%.*]] = zext i8 [[L3]] to i16
-; CHECK-NEXT:    [[ADD170:%.*]] = add nuw nsw i16 [[CONV168]], 40
+; CHECK-NEXT:    [[ADD170:%.*]] = add i16 40, [[CONV168]]
 ; CHECK-NEXT:    store i16 [[ADD152]], ptr [[ARRAYIDX182]], align 2
 ; CHECK-NEXT:    store i16 [[ADD158]], ptr [[ARRAYIDX183]], align 2
 ; CHECK-NEXT:    store i16 [[ADD164]], ptr [[ARRAYIDX184]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
index 900d5f293b5b8..f83628ed17b65 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=slp-vectorizer,dce,instcombine < %s | FileCheck %s --check-prefix=GENERIC
-; RUN: opt -S -mcpu=kryo -passes=slp-vectorizer,dce,instcombine < %s | FileCheck %s --check-prefix=KRYO
+; RUN: opt -S -passes=slp-vectorizer < %s | FileCheck %s --check-prefix=GENERIC
+; RUN: opt -S -mcpu=kryo -passes=slp-vectorizer < %s | FileCheck %s --check-prefix=KRYO
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -36,57 +36,49 @@ define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture read
 ; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; GENERIC-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; GENERIC-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; GENERIC-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
-; GENERIC-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
+; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
+; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i32 [[TMP5]]
 ; GENERIC-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; GENERIC-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
-; GENERIC-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
+; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
+; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP8]]
 ; GENERIC-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; GENERIC-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; GENERIC-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
-; GENERIC-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
+; GENERIC-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
+; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP9]]
 ; GENERIC-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; GENERIC-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
-; GENERIC-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
+; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
+; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP11]]
 ; GENERIC-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; GENERIC-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; GENERIC-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
-; GENERIC-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
+; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
+; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP14]]
 ; GENERIC-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; GENERIC-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; GENERIC-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
-; GENERIC-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
+; GENERIC-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
+; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP15]]
 ; GENERIC-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; GENERIC-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; GENERIC-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; GENERIC-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
-; GENERIC-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
+; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
+; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP17]]
 ; GENERIC-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; GENERIC-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; GENERIC-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; GENERIC-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
-; GENERIC-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
+; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
+; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP20]]
 ; GENERIC-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
 ; GENERIC-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP28]] to i32
 ; GENERIC-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
@@ -109,57 +101,49 @@ define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture read
 ; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; KRYO-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; KRYO-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; KRYO-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
-; KRYO-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
-; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
+; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
+; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i32 [[TMP5]]
 ; KRYO-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; KRYO-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
-; KRYO-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
+; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
+; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP8]]
 ; KRYO-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; KRYO-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; KRYO-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
-; KRYO-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
+; KRYO-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
+; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP9]]
 ; KRYO-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; KRYO-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; KRYO-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
-; KRYO-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
-; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
+; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
+; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP11]]
 ; KRYO-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; KRYO-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; KRYO-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
-; KRYO-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
-; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
+; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
+; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP14]]
 ; KRYO-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; KRYO-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; KRYO-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
-; KRYO-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
-; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
+; KRYO-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
+; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP15]]
 ; KRYO-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; KRYO-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; KRYO-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; KRYO-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
-; KRYO-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
-; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
+; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
+; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP17]]
 ; KRYO-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; KRYO-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; KRYO-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; KRYO-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
-; KRYO-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
-; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
+; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
+; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP20]]
 ; KRYO-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
 ; KRYO-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP28]] to i32
 ; KRYO-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
@@ -293,55 +277,55 @@ define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture read
 ; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; GENERIC-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; GENERIC-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; GENERIC-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
-; GENERIC-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
+; GENERIC-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]
+; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
 ; GENERIC-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
 ; GENERIC-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; GENERIC-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
+; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
 ; GENERIC-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
 ; GENERIC-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; GENERIC-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; GENERIC-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
+; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
 ; GENERIC-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
 ; GENERIC-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; GENERIC-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
+; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
 ; GENERIC-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
 ; GENERIC-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; GENERIC-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; GENERIC-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
+; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
 ; GENERIC-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
 ; GENERIC-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; GENERIC-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; GENERIC-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
+; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
 ; GENERIC-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
 ; GENERIC-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; GENERIC-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; GENERIC-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; GENERIC-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
+; GENERIC-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
 ; GENERIC-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
 ; GENERIC-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; GENERIC-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; GENERIC-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; GENERIC-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
+; GENERIC-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
 ; GENERIC-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
 ; GENERIC-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
@@ -366,55 +350,55 @@ define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture read
 ; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; KRYO-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; KRYO-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; KRYO-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
-; KRYO-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
+; KRYO-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]
+; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
 ; KRYO-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
 ; KRYO-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; KRYO-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
+; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
 ; KRYO-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
 ; KRYO-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; KRYO-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; KRYO-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
+; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
 ; KRYO-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
 ; KRYO-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; KRYO-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; KRYO-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
+; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
 ; KRYO-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
 ; KRYO-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; KRYO-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; KRYO-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
+; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
 ; KRYO-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
 ; KRYO-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; KRYO-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; KRYO-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
+; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
 ; KRYO-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
 ; KRYO-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; KRYO-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; KRYO-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; KRYO-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
+; KRYO-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
 ; KRYO-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
 ; KRYO-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; KRYO-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; KRYO-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; KRYO-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
+; KRYO-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
 ; KRYO-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
 ; KRYO-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index c43b79e138a30..89e133bb1c6a1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 ; These tests check that we remove from consideration pairs of seed
@@ -46,9 +46,9 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Z:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Z:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -58,29 +58,25 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[ADD16]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw nsw i32 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i64 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = zext nneg i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[G:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP6]]
 ; CHECK-NEXT:    [[T6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[SUM_032]]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
-; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP7]]
 ; CHECK-NEXT:    [[T8:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i32> [[TMP4]], [[TMP2]]
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
-; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP10]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP9]]
 ; CHECK-NEXT:    [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
-; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP11]]
 ; CHECK-NEXT:    [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
 ; CHECK-NEXT:    [[ADD16]] = add nsw i32 [[ADD11]], [[T12]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1
@@ -143,7 +139,7 @@ define i32 @getelementptr_2x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[Y:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[Y:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -153,26 +149,22 @@ define i32 @getelementptr_2x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[OP_RDX]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw nsw i32 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i64 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[TMP4]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[G:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
-; CHECK-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP9]]
 ; CHECK-NEXT:    [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    [[T11:%.*]] = add nsw i32 [[T4]], [[Z:%.*]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[T11]] to i64
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[T11]]
 ; CHECK-NEXT:    [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[T10]], i64 2
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[T12]], i64 3
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[T10]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[T12]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP8]], <2 x i32> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
 ; CHECK-NEXT:    [[OP_RDX]] = add i32 [[TMP14]], [[SUM_032]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 5808d64d925c6..ac476c521a591 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 
@@ -25,42 +25,42 @@ define void @test_i16_extend(ptr %p.1, ptr %p.2, i32 %idx.i32) {
 ; CHECK-LABEL: @test_i16_extend(
 ; CHECK-NEXT:    [[P_0:%.*]] = load ptr, ptr @global, align 8
 ; CHECK-NEXT:    [[IDX_0:%.*]] = zext i32 [[IDX_I32:%.*]] to i64
-; CHECK-NEXT:    [[T53:%.*]] = getelementptr inbounds nuw i16, ptr [[P_1:%.*]], i64 [[IDX_0]]
-; CHECK-NEXT:    [[T56:%.*]] = getelementptr inbounds nuw i16, ptr [[P_2:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[T53:%.*]] = getelementptr inbounds i16, ptr [[P_1:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[T56:%.*]] = getelementptr inbounds i16, ptr [[P_2:%.*]], i64 [[IDX_0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[T53]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr [[T56]], align 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[TMP3]] to <8 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <8 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[TMP6]] to i64
 ; CHECK-NEXT:    [[T60:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[L_1:%.*]] = load i32, ptr [[T60]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP5]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; CHECK-NEXT:    [[T71:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[L_2:%.*]] = load i32, ptr [[T71]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i32 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
 ; CHECK-NEXT:    [[T82:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[L_3:%.*]] = load i32, ptr [[T82]], align 4
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP5]], i64 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP5]], i32 3
 ; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
 ; CHECK-NEXT:    [[T93:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[L_4:%.*]] = load i32, ptr [[T93]], align 4
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP5]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP5]], i32 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
 ; CHECK-NEXT:    [[T104:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[L_5:%.*]] = load i32, ptr [[T104]], align 4
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP5]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP5]], i32 5
 ; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
 ; CHECK-NEXT:    [[T115:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP17]]
 ; CHECK-NEXT:    [[L_6:%.*]] = load i32, ptr [[T115]], align 4
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP5]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP5]], i32 6
 ; CHECK-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP18]] to i64
 ; CHECK-NEXT:    [[T126:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[L_7:%.*]] = load i32, ptr [[T126]], align 4
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP5]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP5]], i32 7
 ; CHECK-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
 ; CHECK-NEXT:    [[T137:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP21]]
 ; CHECK-NEXT:    [[L_8:%.*]] = load i32, ptr [[T137]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
index f138065101c4b..6c5220d13b7a2 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=slp-vectorizer,instcombine -mtriple=aarch64--linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64--linux-gnu < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64"
@@ -133,7 +133,7 @@ define i16 @reduce_blockstrided2(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-LABEL: @reduce_blockstrided2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[X:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM]]
@@ -165,7 +165,7 @@ define i16 @reduce_blockstrided2(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM15]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX32]], align 2
-; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX33]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX36]], align 2
@@ -254,9 +254,9 @@ define i16 @reduce_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-LABEL: @reduce_blockstrided3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L0:%.*]] = load i16, ptr [[X:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 1
 ; CHECK-NEXT:    [[L1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[L2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM]]
@@ -269,10 +269,12 @@ define i16 @reduce_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[ADD8]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM9]]
 ; CHECK-NEXT:    [[L6:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[STRIDE]], 3
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
 ; CHECK-NEXT:    [[L8:%.*]] = load i16, ptr [[Y:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 1
 ; CHECK-NEXT:    [[L9:%.*]] = load i16, ptr [[ARRAYIDX15]], align 2
-; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 2
 ; CHECK-NEXT:    [[L10:%.*]] = load i16, ptr [[ARRAYIDX16]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[L12:%.*]] = load i16, ptr [[ARRAYIDX20]], align 2
@@ -347,9 +349,9 @@ define i16 @reduce_blockstrided4(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX4]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[Y]], align 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr [[ARRAYIDX20]], align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i16> [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i16> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP4]], <4 x i16> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <8 x i16> [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP6]])
 ; CHECK-NEXT:    ret i16 [[TMP7]]
 ;
@@ -414,33 +416,128 @@ define i32 @reduce_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[OFF1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[OFF2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
+; TODO: Dead code must be removed below.
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[CONV_2:%.*]] = zext i8 [[TMP32]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
+; CHECK-NEXT:    [[CONV2_2:%.*]] = zext i8 [[TMP33]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
+; CHECK-NEXT:    [[CONV4_2:%.*]] = zext i8 [[TMP34]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
+; CHECK-NEXT:    [[CONV6_2:%.*]] = zext i8 [[TMP35]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 1
+; CHECK-NEXT:    [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; CHECK-NEXT:    [[CONV9_2:%.*]] = zext i8 [[TMP36]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 1
+; CHECK-NEXT:    [[TMP37:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
+; CHECK-NEXT:    [[CONV11_2:%.*]] = zext i8 [[TMP37]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 5
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
+; CHECK-NEXT:    [[CONV14_2:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 5
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
+; CHECK-NEXT:    [[CONV16_2:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 2
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
+; CHECK-NEXT:    [[CONV21_2:%.*]] = zext i8 [[TMP8]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
+; CHECK-NEXT:    [[CONV23_2:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 6
+; CHECK-NEXT:    [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
+; CHECK-NEXT:    [[CONV26_2:%.*]] = zext i8 [[TMP41]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 6
+; CHECK-NEXT:    [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
+; CHECK-NEXT:    [[CONV28_2:%.*]] = zext i8 [[TMP42]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 3
+; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
+; CHECK-NEXT:    [[CONV33_2:%.*]] = zext i8 [[TMP43]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 3
+; CHECK-NEXT:    [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
+; CHECK-NEXT:    [[CONV35_2:%.*]] = zext i8 [[TMP44]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 7
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
+; CHECK-NEXT:    [[CONV38_2:%.*]] = zext i8 [[TMP14]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 7
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
+; CHECK-NEXT:    [[CONV40_2:%.*]] = zext i8 [[TMP15]] to i32
+; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr [[ADD_PTR_2]], align 1
+; CHECK-NEXT:    [[CONV_3:%.*]] = zext i8 [[TMP16]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i8, ptr [[ADD_PTR64_2]], align 1
+; CHECK-NEXT:    [[CONV2_3:%.*]] = zext i8 [[TMP17]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[TMP48:%.*]] = load i8, ptr [[ARRAYIDX3_3]], align 1
+; CHECK-NEXT:    [[CONV4_3:%.*]] = zext i8 [[TMP48]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[TMP49:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
+; CHECK-NEXT:    [[CONV6_3:%.*]] = zext i8 [[TMP49]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 1
+; CHECK-NEXT:    [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
+; CHECK-NEXT:    [[CONV9_3:%.*]] = zext i8 [[TMP50]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 1
+; CHECK-NEXT:    [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
+; CHECK-NEXT:    [[CONV11_3:%.*]] = zext i8 [[TMP51]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 5
+; CHECK-NEXT:    [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX13_3]], align 1
+; CHECK-NEXT:    [[CONV14_3:%.*]] = zext i8 [[TMP22]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 5
+; CHECK-NEXT:    [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
+; CHECK-NEXT:    [[CONV16_3:%.*]] = zext i8 [[TMP23]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 2
+; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
+; CHECK-NEXT:    [[CONV21_3:%.*]] = zext i8 [[TMP24]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
+; CHECK-NEXT:    [[CONV23_3:%.*]] = zext i8 [[TMP25]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 6
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX25_3]], align 1
+; CHECK-NEXT:    [[CONV26_3:%.*]] = zext i8 [[TMP26]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 6
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
+; CHECK-NEXT:    [[CONV28_3:%.*]] = zext i8 [[TMP27]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
+; CHECK-NEXT:    [[CONV33_3:%.*]] = zext i8 [[TMP28]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
+; CHECK-NEXT:    [[CONV35_3:%.*]] = zext i8 [[TMP29]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 7
+; CHECK-NEXT:    [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX37_3]], align 1
+; CHECK-NEXT:    [[CONV38_3:%.*]] = zext i8 [[TMP30]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 7
+; CHECK-NEXT:    [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
+; CHECK-NEXT:    [[CONV40_3:%.*]] = zext i8 [[TMP31]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i8>, ptr [[ADD_PTR64]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP8]], <16 x i8> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP38:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP39:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP38]], <4 x i8> [[TMP4]], i64 4)
+; CHECK-NEXT:    [[TMP40:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP39]], <4 x i8> [[TMP1]], i64 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP40]], <4 x i8> [[TMP5]], i64 12)
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext <16 x i8> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP12]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP45:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP46:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP45]], <4 x i8> [[TMP12]], i64 4)
+; CHECK-NEXT:    [[TMP47:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP46]], <4 x i8> [[TMP3]], i64 8)
+; CHECK-NEXT:    [[TMP18:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP47]], <4 x i8> [[TMP13]], i64 12)
 ; CHECK-NEXT:    [[TMP19:%.*]] = zext <16 x i8> [[TMP18]] to <16 x i32>
-; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw nsw <16 x i32> [[TMP11]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <16 x i32> [[TMP11]], [[TMP19]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP20]])
 ; CHECK-NEXT:    ret i32 [[TMP21]]
 ;
@@ -677,7 +774,7 @@ entry:
 define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocapture noundef readonly %y, ptr nocapture noundef writeonly %z, i32 noundef %stride) {
 ; CHECK-LABEL: @store_blockstrided3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[X:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ADD4:%.*]] = add nsw i32 [[STRIDE:%.*]], 1
 ; CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[ADD4]] to i64
@@ -696,7 +793,7 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ADD26:%.*]] = add nsw i32 [[MUL21]], 1
 ; CHECK-NEXT:    [[IDXPROM27:%.*]] = sext i32 [[ADD26]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM27]]
-; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds nuw i8, ptr [[Y:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i32, ptr [[Y:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX35]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM5]]
 ; CHECK-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM11]]
@@ -705,20 +802,21 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ARRAYIDX60:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM23]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX60]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX65:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM27]]
-; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds nuw i8, ptr [[Z:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 1
 ; CHECK-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP3]], [[TMP0]]
-; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 6
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[X]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i32>, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i32>, ptr [[Y]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i32>, ptr [[ARRAYIDX41]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw <2 x i32> [[TMP8]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw <2 x i32> [[TMP9]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> [[TMP11]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-NEXT:    [[ARRAYIDX84:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 28
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nsw <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[ARRAYIDX84:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 7
 ; CHECK-NEXT:    [[MUL81:%.*]] = mul nsw i32 [[TMP4]], [[TMP1]]
 ; CHECK-NEXT:    [[MUL87:%.*]] = mul nsw i32 [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 44
+; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 11
 ; CHECK-NEXT:    [[TMP17:%.*]] = load <2 x i32>, ptr [[ARRAYIDX28]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = load <2 x i32>, ptr [[ARRAYIDX64]], align 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = load <2 x i32>, ptr [[ARRAYIDX49]], align 4
@@ -727,9 +825,10 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    store <4 x i32> [[TMP12]], ptr [[ARRAYIDX72]], align 4
 ; CHECK-NEXT:    store i32 [[MUL81]], ptr [[ARRAYIDX76]], align 4
 ; CHECK-NEXT:    store i32 [[MUL87]], ptr [[ARRAYIDX88]], align 4
-; CHECK-NEXT:    [[TMP20:%.*]] = mul nsw <2 x i32> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nsw <2 x i32> [[TMP16]], [[TMP18]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> [[TMP21]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i32> [[TMP15]], <2 x i32> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i32> [[TMP17]], <2 x i32> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP20:%.*]] = mul nsw <4 x i32> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    store <4 x i32> [[TMP19]], ptr [[ARRAYIDX84]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -837,9 +936,10 @@ define void @store_blockstrided4(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX4]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[Y]], align 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr [[ARRAYIDX20]], align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i16> [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i16> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP4]], <4 x i16> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <8 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 ; CHECK-NEXT:    store <8 x i16> [[TMP6]], ptr [[DST0:%.*]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -912,35 +1012,132 @@ define void @store_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[OFF1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[OFF2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
-; CHECK-NEXT:    [[DST4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0:%.*]], i64 16
-; CHECK-NEXT:    [[DST8:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0]], i64 32
-; CHECK-NEXT:    [[DST12:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0]], i64 48
+; TODO: Dead code must be removed below.
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[CONV_2:%.*]] = zext i8 [[TMP32]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
+; CHECK-NEXT:    [[CONV2_2:%.*]] = zext i8 [[TMP33]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
+; CHECK-NEXT:    [[CONV4_2:%.*]] = zext i8 [[TMP34]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
+; CHECK-NEXT:    [[CONV6_2:%.*]] = zext i8 [[TMP35]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 1
+; CHECK-NEXT:    [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; CHECK-NEXT:    [[CONV9_2:%.*]] = zext i8 [[TMP36]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 1
+; CHECK-NEXT:    [[TMP37:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
+; CHECK-NEXT:    [[CONV11_2:%.*]] = zext i8 [[TMP37]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 5
+; CHECK-NEXT:    [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
+; CHECK-NEXT:    [[CONV14_2:%.*]] = zext i8 [[TMP38]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 5
+; CHECK-NEXT:    [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
+; CHECK-NEXT:    [[CONV16_2:%.*]] = zext i8 [[TMP39]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 2
+; CHECK-NEXT:    [[TMP40:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
+; CHECK-NEXT:    [[CONV21_2:%.*]] = zext i8 [[TMP40]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 2
+; CHECK-NEXT:    [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
+; CHECK-NEXT:    [[CONV23_2:%.*]] = zext i8 [[TMP41]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 6
+; CHECK-NEXT:    [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
+; CHECK-NEXT:    [[CONV26_2:%.*]] = zext i8 [[TMP42]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 6
+; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
+; CHECK-NEXT:    [[CONV28_2:%.*]] = zext i8 [[TMP43]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 3
+; CHECK-NEXT:    [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
+; CHECK-NEXT:    [[CONV33_2:%.*]] = zext i8 [[TMP44]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
+; CHECK-NEXT:    [[CONV35_2:%.*]] = zext i8 [[TMP45]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 7
+; CHECK-NEXT:    [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
+; CHECK-NEXT:    [[CONV38_2:%.*]] = zext i8 [[TMP46]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 7
+; CHECK-NEXT:    [[TMP47:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
+; CHECK-NEXT:    [[CONV40_2:%.*]] = zext i8 [[TMP47]] to i32
+; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP48:%.*]] = load i8, ptr [[ADD_PTR_2]], align 1
+; CHECK-NEXT:    [[CONV_3:%.*]] = zext i8 [[TMP48]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i8, ptr [[ADD_PTR64_2]], align 1
+; CHECK-NEXT:    [[CONV2_3:%.*]] = zext i8 [[TMP49]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX3_3]], align 1
+; CHECK-NEXT:    [[CONV4_3:%.*]] = zext i8 [[TMP50]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
+; CHECK-NEXT:    [[CONV6_3:%.*]] = zext i8 [[TMP51]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
+; CHECK-NEXT:    [[CONV9_3:%.*]] = zext i8 [[TMP20]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 1
+; CHECK-NEXT:    [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
+; CHECK-NEXT:    [[CONV11_3:%.*]] = zext i8 [[TMP21]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 5
+; CHECK-NEXT:    [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX13_3]], align 1
+; CHECK-NEXT:    [[CONV14_3:%.*]] = zext i8 [[TMP22]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 5
+; CHECK-NEXT:    [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
+; CHECK-NEXT:    [[CONV16_3:%.*]] = zext i8 [[TMP23]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 2
+; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
+; CHECK-NEXT:    [[CONV21_3:%.*]] = zext i8 [[TMP24]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
+; CHECK-NEXT:    [[CONV23_3:%.*]] = zext i8 [[TMP25]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 6
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX25_3]], align 1
+; CHECK-NEXT:    [[CONV26_3:%.*]] = zext i8 [[TMP26]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 6
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
+; CHECK-NEXT:    [[CONV28_3:%.*]] = zext i8 [[TMP27]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
+; CHECK-NEXT:    [[CONV33_3:%.*]] = zext i8 [[TMP28]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
+; CHECK-NEXT:    [[CONV35_3:%.*]] = zext i8 [[TMP29]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 7
+; CHECK-NEXT:    [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX37_3]], align 1
+; CHECK-NEXT:    [[CONV38_3:%.*]] = zext i8 [[TMP30]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 7
+; CHECK-NEXT:    [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
+; CHECK-NEXT:    [[CONV40_3:%.*]] = zext i8 [[TMP31]] to i32
+; CHECK-NEXT:    [[DST4:%.*]] = getelementptr inbounds i32, ptr [[DST0:%.*]], i64 4
+; CHECK-NEXT:    [[DST8:%.*]] = getelementptr inbounds i32, ptr [[DST0]], i64 8
+; CHECK-NEXT:    [[DST12:%.*]] = getelementptr inbounds i32, ptr [[DST0]], i64 12
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i8> [[TMP7]] to <4 x i32>
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <4 x i32> [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i8>, ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext <4 x i8> [[TMP10]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32>
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw nsw <4 x i32> [[TMP11]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul <4 x i32> [[TMP11]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load <4 x i8>, ptr [[ADD_PTR64]], align 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = zext <4 x i8> [[TMP15]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP17:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
 ; CHECK-NEXT:    [[TMP18:%.*]] = zext <4 x i8> [[TMP17]] to <4 x i32>
-; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw nsw <4 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul <4 x i32> [[TMP16]], [[TMP18]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[DST0]], align 4
 ; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[DST4]], align 4
 ; CHECK-NEXT:    store <4 x i32> [[TMP14]], ptr [[DST8]], align 4
@@ -1198,20 +1395,20 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[ST1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[ST2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_1]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_2]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
@@ -1225,33 +1422,29 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ADD_PTR_2]], align 1
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP15]], <16 x i8> [[TMP16]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP13]], <4 x i8> [[TMP4]], i64 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP14]], <4 x i8> [[TMP8]], i64 8)
+; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP15]], <4 x i8> [[TMP12]], i64 12)
 ; CHECK-NEXT:    [[TMP18:%.*]] = zext <16 x i8> [[TMP17]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP19:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_2]], align 1
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <16 x i8> [[TMP20]], <16 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP19]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <16 x i8> [[TMP22]], <16 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP22:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP20:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP22]], <4 x i8> [[TMP5]], i64 4)
+; CHECK-NEXT:    [[TMP21:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP20]], <4 x i8> [[TMP9]], i64 8)
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP21]], <4 x i8> [[TMP19]], i64 12)
 ; CHECK-NEXT:    [[TMP25:%.*]] = zext <16 x i8> [[TMP24]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP26:%.*]] = sub nsw <16 x i32> [[TMP18]], [[TMP25]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_3]], align 1
-; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i8> [[TMP10]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <16 x i8> [[TMP28]], <16 x i8> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <4 x i8> [[TMP27]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <16 x i8> [[TMP30]], <16 x i8> [[TMP31]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP29:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP30:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP29]], <4 x i8> [[TMP6]], i64 4)
+; CHECK-NEXT:    [[TMP28:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP30]], <4 x i8> [[TMP10]], i64 8)
+; CHECK-NEXT:    [[TMP32:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP28]], <4 x i8> [[TMP27]], i64 12)
 ; CHECK-NEXT:    [[TMP33:%.*]] = zext <16 x i8> [[TMP32]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP34:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
-; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <16 x i8> [[TMP35]], <16 x i8> [[TMP36]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <4 x i8> [[TMP34]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <16 x i8> [[TMP37]], <16 x i8> [[TMP38]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP35:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP3]], i64 0)
+; CHECK-NEXT:    [[TMP36:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP35]], <4 x i8> [[TMP7]], i64 4)
+; CHECK-NEXT:    [[TMP37:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP36]], <4 x i8> [[TMP11]], i64 8)
+; CHECK-NEXT:    [[TMP39:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP37]], <4 x i8> [[TMP34]], i64 12)
 ; CHECK-NEXT:    [[TMP40:%.*]] = zext <16 x i8> [[TMP39]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP41:%.*]] = sub nsw <16 x i32> [[TMP33]], [[TMP40]]
 ; CHECK-NEXT:    [[TMP42:%.*]] = shl nsw <16 x i32> [[TMP41]], splat (i32 16)
@@ -1259,11 +1452,11 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <16 x i32> [[TMP43]], <16 x i32> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 ; CHECK-NEXT:    [[TMP45:%.*]] = add nsw <16 x i32> [[TMP43]], [[TMP44]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = sub nsw <16 x i32> [[TMP43]], [[TMP44]]
-; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i32> [[TMP46]], <16 x i32> [[TMP45]], <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i32> [[TMP45]], <16 x i32> [[TMP46]], <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <16 x i32> [[TMP47]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
 ; CHECK-NEXT:    [[TMP49:%.*]] = add nsw <16 x i32> [[TMP47]], [[TMP48]]
 ; CHECK-NEXT:    [[TMP50:%.*]] = sub nsw <16 x i32> [[TMP47]], [[TMP48]]
-; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <16 x i32> [[TMP50]], <16 x i32> [[TMP49]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 4, i32 5, i32 22, i32 23, i32 8, i32 9, i32 26, i32 27, i32 12, i32 13, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <16 x i32> [[TMP49]], <16 x i32> [[TMP50]], <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 20, i32 21, i32 6, i32 7, i32 24, i32 25, i32 10, i32 11, i32 28, i32 29, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP53:%.*]] = sub nsw <16 x i32> [[TMP51]], [[TMP52]]
 ; CHECK-NEXT:    [[TMP54:%.*]] = add nsw <16 x i32> [[TMP51]], [[TMP52]]
@@ -1271,7 +1464,7 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <16 x i32> [[TMP55]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP57:%.*]] = add nsw <16 x i32> [[TMP55]], [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = sub nsw <16 x i32> [[TMP55]], [[TMP56]]
-; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP57]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP57]], <16 x i32> [[TMP58]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP60:%.*]] = lshr <16 x i32> [[TMP59]], splat (i32 15)
 ; CHECK-NEXT:    [[TMP61:%.*]] = and <16 x i32> [[TMP60]], splat (i32 65537)
 ; CHECK-NEXT:    [[TMP62:%.*]] = mul nuw <16 x i32> [[TMP61]], splat (i32 65535)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
index c84c333391350..0f47c6b3ac902 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -167,7 +167,9 @@ define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i32> [[V0]], [[V1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP3_31]]
 ;
   %v0.0 = extractelement <2 x i32> %v0, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
index e4fcb1ed08be9..370ed1f258aca 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -167,7 +167,9 @@ define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i32> [[V0]], [[V1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP3_31]]
 ;
   %v0.0 = extractelement <2 x i32> %v0, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
index 783a1e83c6724..a8d1c94d59be3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-LABEL: @uadd_sat_v2i16(
@@ -264,8 +264,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:  bb:
 ; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX8-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -273,8 +276,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -317,19 +323,27 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX8-LABEL: @uadd_sat_v4i16(
 ; GFX8-NEXT:  bb:
-; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX8-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX8-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX9-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
index 7e31ec9a0b39a..b09022e8289a1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-LABEL: @uadd_sat_v2i16(
@@ -12,7 +12,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -47,7 +47,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -82,7 +82,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -117,7 +117,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -152,7 +152,7 @@ define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -177,7 +177,7 @@ define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -202,7 +202,7 @@ define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -227,7 +227,7 @@ define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -255,7 +255,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    ret <3 x i16> [[INS_2]]
@@ -264,8 +264,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:  bb:
 ; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX8-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -273,8 +276,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -309,7 +315,7 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
 ; GFX7-NEXT:    [[ADD_3:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
@@ -317,19 +323,27 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX8-LABEL: @uadd_sat_v4i16(
 ; GFX8-NEXT:  bb:
-; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX8-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX8-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX9-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
index 46c6c10125b95..57ca3db075689 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-LABEL: @uadd_sat_v2i16(
@@ -12,7 +12,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -24,7 +24,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -54,7 +54,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -66,7 +66,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -96,7 +96,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -108,7 +108,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -138,7 +138,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -150,7 +150,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -180,7 +180,7 @@ define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.umin.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.umin.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -205,7 +205,7 @@ define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.umax.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.umax.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -230,7 +230,7 @@ define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -255,7 +255,7 @@ define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.smax.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.smax.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -283,7 +283,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    ret <3 x i16> [[INS_2]]
@@ -299,7 +299,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
@@ -308,8 +308,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP0:%.*]] = call <3 x i16> @llvm.umin.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP0]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -344,7 +347,7 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
 ; GFX7-NEXT:    [[ADD_3:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
@@ -358,9 +361,10 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG1]])
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP1:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP3]])
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[INS_1]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -368,10 +372,14 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG1:%.*]])
-; GFX9-NEXT:    [[TMP1:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP8:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP7]])
 ; GFX9-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP2]], <2 x i16> [[TMP4]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP8]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
index 7faf11c34d76e..88a03bb72c276 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
 
 ; Regression test for a bug in the SLP vectorizer that was causing
 ; these rotates to be incorrectly combined into a vector rotate.
@@ -9,16 +9,16 @@ target triple = "wasm32-unknown-unknown"
 
 define void @foo(<2 x i64> %x, <4 x i32> %y, ptr %out) #0 {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[A:%.*]] = extractelement <2 x i64> [[X:%.*]], i64 0
-; CHECK-NEXT:    [[B:%.*]] = extractelement <4 x i32> [[Y:%.*]], i64 2
+; CHECK-NEXT:    [[A:%.*]] = extractelement <2 x i64> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[B:%.*]] = extractelement <4 x i32> [[Y:%.*]], i32 2
 ; CHECK-NEXT:    [[CONV6:%.*]] = zext i32 [[B]] to i64
 ; CHECK-NEXT:    [[C:%.*]] = tail call i64 @llvm.fshl.i64(i64 [[A]], i64 [[A]], i64 [[CONV6]])
 ; CHECK-NEXT:    store i64 [[C]], ptr [[OUT:%.*]], align 8
-; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i64> [[X]], i64 1
-; CHECK-NEXT:    [[E:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i64> [[X]], i32 1
+; CHECK-NEXT:    [[E:%.*]] = extractelement <4 x i32> [[Y]], i32 3
 ; CHECK-NEXT:    [[CONV17:%.*]] = zext i32 [[E]] to i64
 ; CHECK-NEXT:    [[F:%.*]] = tail call i64 @llvm.fshl.i64(i64 [[D]], i64 [[D]], i64 [[CONV17]])
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[OUT]], i32 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[OUT]], i32 1
 ; CHECK-NEXT:    store i64 [[F]], ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
index 77d36f0107665..92ffbad73d5f3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -1,21 +1,21 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
 
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-LABEL: @ceil_floor(
-; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
-; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i64 4
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
 ; SSE-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
 ; SSE-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
 ; SSE-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
@@ -24,31 +24,31 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
 ; SSE-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
 ; SSE-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
-; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1
-; SSE-NEXT:    [[R23:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i64 4
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i64 6
-; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
 ; SLM-LABEL: @ceil_floor(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; SLM-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SLM-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; SLM-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SLM-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -56,20 +56,20 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SLM-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX-LABEL: @ceil_floor(
-; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX-NEXT:    [[TMP8:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP9]])
 ; AVX-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP10]])
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i64 3
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -77,20 +77,20 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX2-LABEL: @ceil_floor(
-; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX2-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; AVX2-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX2-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX2-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index 18d79752b0b44..f504821b5fa67 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -1,21 +1,21 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
 
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-LABEL: @ceil_floor(
-; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
-; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i64 4
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
 ; SSE-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
 ; SSE-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
 ; SSE-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
@@ -24,75 +24,75 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
 ; SSE-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
 ; SSE-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
-; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1
-; SSE-NEXT:    [[R23:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i64 4
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i64 6
-; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
 ; SLM-LABEL: @ceil_floor(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; SLM-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SLM-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; SLM-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SLM-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SLM-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX-LABEL: @ceil_floor(
-; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX-NEXT:    [[TMP8:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP9]])
 ; AVX-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i64 3
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP10]])
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX2-LABEL: @ceil_floor(
-; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX2-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; AVX2-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX2-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX2-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x float> [[R71]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
index 3e2c305dbed65..65e5458b25d2f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-LABEL: @sitofp_uitofp(
@@ -12,7 +12,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @sitofp_uitofp(
@@ -20,7 +21,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @sitofp_uitofp(
@@ -74,7 +76,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @fptosi_fptoui(
@@ -82,7 +85,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @fptosi_fptoui(
@@ -133,36 +137,50 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 define <8 x float> @fneg_fabs(<8 x float> %a) {
 ; SSE2-LABEL: @fneg_fabs(
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SSE2-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SSE2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; SLM-LABEL: @fneg_fabs(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SLM-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SLM-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SLM-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SLM-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SLM-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SLM-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX-LABEL: @fneg_fabs(
-; AVX-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX2-LABEL: @fneg_fabs(
-; AVX2-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX512-LABEL: @fneg_fabs(
-; AVX512-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX512-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
   %a0 = extractelement <8 x float> %a, i32 0
@@ -214,7 +232,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @sext_zext(
@@ -222,7 +241,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @sext_zext(
@@ -275,7 +295,9 @@ define <8 x float> @sitofp_4i32_8i16(<4 x i32> %a, <8 x i16> %b) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
-; CHECK-NEXT:    [[R71:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
 ;
   %a0 = extractelement <4 x i32> %a, i32 0
@@ -321,7 +343,7 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
index 880523d6474ac..fad46870ec475 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-LABEL: @sitofp_uitofp(
@@ -12,7 +12,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @sitofp_uitofp(
@@ -20,7 +21,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @sitofp_uitofp(
@@ -74,7 +76,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @fptosi_fptoui(
@@ -82,7 +85,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @fptosi_fptoui(
@@ -133,36 +137,50 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 define <8 x float> @fneg_fabs(<8 x float> %a) {
 ; SSE2-LABEL: @fneg_fabs(
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SSE2-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SSE2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; SLM-LABEL: @fneg_fabs(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SLM-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SLM-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SLM-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SLM-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SLM-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SLM-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX-LABEL: @fneg_fabs(
-; AVX-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX2-LABEL: @fneg_fabs(
-; AVX2-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX512-LABEL: @fneg_fabs(
-; AVX512-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX512-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
   %a0 = extractelement <8 x float> %a, i32 0
@@ -214,7 +232,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @sext_zext(
@@ -222,7 +241,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @sext_zext(
@@ -275,7 +295,9 @@ define <8 x float> @sitofp_4i32_8i16(<4 x i32> %a, <8 x i16> %b) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
-; CHECK-NEXT:    [[R71:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
 ;
   %a0 = extractelement <4 x i32> %a, i32 0
@@ -321,7 +343,7 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
index 5cee6984df04f..99b13bdc05082 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
@@ -1,34 +1,46 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
 
 define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fadd_fsub_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fadd_fsub_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fadd_fsub_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fadd_fsub_v8f32(
@@ -80,35 +92,51 @@ define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fmul_fdiv_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v8f32(
-; AVX2-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX2-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX2-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v8f32(
@@ -154,30 +182,39 @@ define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SSE-LABEL: @fmul_fdiv_v4f32_const(
-; SSE-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; SSE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; SSE-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v4f32_const(
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
-; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
+; SLM-NEXT:    [[AB2:%.*]] = fdiv float [[A2]], 1.000000e+00
+; SLM-NEXT:    [[AB3:%.*]] = fdiv float [[A3]], 5.000000e-01
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[A2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i64 3
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v4f32_const(
-; AVX-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v4f32_const(
-; AVX2-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX2-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX2-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v4f32_const(
-; AVX512-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX512-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX512-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
index 9a2f959ac63bc..7f9475917b566 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
@@ -1,34 +1,46 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
 
 define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fadd_fsub_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fadd_fsub_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fadd_fsub_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fadd_fsub_v8f32(
@@ -80,35 +92,51 @@ define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fmul_fdiv_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v8f32(
-; AVX2-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX2-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX2-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v8f32(
@@ -154,30 +182,39 @@ define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SSE-LABEL: @fmul_fdiv_v4f32_const(
-; SSE-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; SSE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; SSE-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v4f32_const(
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
-; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
+; SLM-NEXT:    [[AB2:%.*]] = fdiv float [[A2]], 1.000000e+00
+; SLM-NEXT:    [[AB3:%.*]] = fdiv float [[A3]], 5.000000e-01
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[A2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i64 3
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v4f32_const(
-; AVX-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v4f32_const(
-; AVX2-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX2-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX2-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v4f32_const(
-; AVX512-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX512-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX512-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
index f8c5df9944538..11ab7770a5383 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
@@ -1,26 +1,32 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @add_sub_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @add_sub_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32(
@@ -130,19 +136,25 @@ define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_shl_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32(
@@ -204,7 +216,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SSE-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32_const(
@@ -212,7 +225,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32_const(
@@ -262,24 +276,25 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 
 define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_lshr_shl_v8i32(
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
-; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i64 6
-; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i64 7
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SSE-NEXT:    [[TMP6:%.*]] = lshr <8 x i32> [[A]], [[B]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP7:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP10]]
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
 ; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @ashr_lshr_shl_v8i32(
@@ -293,7 +308,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SLM-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SLM-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SLM-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX1-LABEL: @ashr_lshr_shl_v8i32(
@@ -307,7 +324,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX1-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX2-LABEL: @ashr_lshr_shl_v8i32(
@@ -321,7 +340,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @ashr_lshr_shl_v8i32(
@@ -335,7 +356,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX512-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -407,100 +430,130 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
 
 define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; SSE-LABEL: @sdiv_v8i32_undefs(
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SSE-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SSE-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SSE-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SSE-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SSE-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SSE-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SSE-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @sdiv_v8i32_undefs(
-; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SLM-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SLM-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SLM-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SLM-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SLM-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SLM-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SLM-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SLM-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX1-LABEL: @sdiv_v8i32_undefs(
-; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; AVX1-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; AVX1-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX1-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX1-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; AVX1-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX1-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX1-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX1-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; AVX1-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; AVX1-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; AVX1-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX2-LABEL: @sdiv_v8i32_undefs(
-; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX2-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX2-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX2-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX2-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX2-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX2-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @sdiv_v8i32_undefs(
-; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX512-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX512-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX512-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX512-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX512-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX512-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX512-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -533,26 +586,28 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; SSE-LABEL: @add_sub_v8i32_splat(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SSE-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @add_sub_v8i32_splat(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SLM-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32_splat(
-; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX1-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX1-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX1-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -560,7 +615,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX1-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX2-LABEL: @add_sub_v8i32_splat(
-; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX2-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX2-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -568,7 +623,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX512-LABEL: @add_sub_v8i32_splat(
-; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX512-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX512-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index b84ef027f67c5..9589ec24d49d4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -1,26 +1,32 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @add_sub_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @add_sub_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32(
@@ -130,19 +136,25 @@ define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_shl_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32(
@@ -204,7 +216,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SSE-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32_const(
@@ -212,7 +225,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32_const(
@@ -262,24 +276,25 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 
 define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_lshr_shl_v8i32(
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
-; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i64 6
-; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i64 7
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SSE-NEXT:    [[TMP6:%.*]] = lshr <8 x i32> [[A]], [[B]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP7:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP10]]
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
 ; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @ashr_lshr_shl_v8i32(
@@ -293,7 +308,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SLM-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SLM-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SLM-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX1-LABEL: @ashr_lshr_shl_v8i32(
@@ -307,7 +324,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX1-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX2-LABEL: @ashr_lshr_shl_v8i32(
@@ -321,7 +340,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @ashr_lshr_shl_v8i32(
@@ -335,7 +356,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX512-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -407,100 +430,130 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
 
 define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; SSE-LABEL: @sdiv_v8i32_undefs(
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SSE-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SSE-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SSE-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SSE-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SSE-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SSE-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SSE-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
-; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @sdiv_v8i32_undefs(
-; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SLM-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SLM-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SLM-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SLM-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SLM-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SLM-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SLM-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SLM-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX1-LABEL: @sdiv_v8i32_undefs(
-; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; AVX1-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; AVX1-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX1-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX1-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; AVX1-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX1-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX1-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX1-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; AVX1-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
-; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; AVX1-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; AVX1-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX2-LABEL: @sdiv_v8i32_undefs(
-; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX2-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX2-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX2-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX2-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX2-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX2-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @sdiv_v8i32_undefs(
-; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX512-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX512-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX512-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX512-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX512-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX512-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX512-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -533,26 +586,28 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; SSE-LABEL: @add_sub_v8i32_splat(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SSE-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @add_sub_v8i32_splat(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SLM-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32_splat(
-; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX1-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX1-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX1-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -560,7 +615,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX1-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX2-LABEL: @add_sub_v8i32_splat(
-; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX2-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX2-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -568,7 +623,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX512-LABEL: @add_sub_v8i32_splat(
-; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX512-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX512-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
index 3ee6c55e429bd..c2bd9e648e4fd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
 
 define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @g(
@@ -86,8 +86,8 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[Y]], <2 x i32> <i32 3, i32 6>
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i8> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i32 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sdiv i8 [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    ret i8 [[TMP8]]
 ;
@@ -107,14 +107,14 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 
 define i8 @k(<4 x i8> %x) {
 ; CHECK-LABEL: @k(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
@@ -136,14 +136,14 @@ define i8 @k_bb(<4 x i8> %x) {
 ; CHECK-LABEL: @k_bb(
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
index 60e008d12e6e1..e4e45eb99fbdf 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
 
 define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @g(
@@ -39,8 +39,14 @@ define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
 
 define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @h_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> <i8 undef, i8 poison, i8 poison, i8 poison>, <4 x i32> <i32 4, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 5, i32 6>
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> undef, i32 0
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 3
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i8> poison, i8 [[X0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i8> [[TMP1]], i8 [[X3]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i8> [[TMP4]], i8 [[Y1]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i8> [[TMP5]], i8 [[Y2]], i32 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[TMP2]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x i8> [[TMP3]]
 ;
@@ -87,8 +93,8 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[Y]], <2 x i32> <i32 3, i32 6>
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i8> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i32 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sdiv i8 [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    ret i8 [[TMP8]]
 ;
@@ -108,14 +114,14 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 
 define i8 @k(<4 x i8> %x) {
 ; CHECK-LABEL: @k(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
@@ -137,14 +143,14 @@ define i8 @k_bb(<4 x i8> %x) {
 ; CHECK-LABEL: @k_bb(
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
index fd5f09bf2adc0..7d5ea22436c7c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
 
 ;
 ; Check that we can commute operands based on the predicate.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
index 35619d6d3ad1d..25aa24b4cc81d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
 
 ;
 ; Check that we can commute operands based on the predicate.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
index b85ec5bce8192..815ad044ca077 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
 
 ;
 ; 128-bit vectors
@@ -169,21 +169,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -214,14 +214,14 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; PR50392
 define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: @test_v4f64_partial_swizzle(
-; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
+; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <2 x i32> <i32 0, i32 4>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; CHECK-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3
+; CHECK-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i32 3
 ; CHECK-NEXT:    ret <4 x double> [[R03]]
 ;
   %a0 = extractelement <4 x double> %a, i64 0
@@ -243,21 +243,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -305,21 +305,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -351,21 +351,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -413,21 +413,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
index b83d35541bbae..fb037b87da546 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
 
 ;
 ; 128-bit vectors
@@ -169,21 +169,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -214,15 +214,15 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; PR50392
 define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64_partial_swizzle(
-; SSE-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
+; SSE-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; SSE-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; SSE-NEXT:    [[R0212:%.*]] = insertelement <4 x double> [[TMP4]], double 0.000000e+00, i64 1
-; SSE-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R0212]], double [[R3]], i64 3
+; SSE-NEXT:    [[R021:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+; SSE-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R021]], double [[R3]], i32 3
 ; SSE-NEXT:    ret <4 x double> [[R03]]
 ;
 ; SLM-LABEL: @test_v4f64_partial_swizzle(
@@ -232,21 +232,21 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 1, i32 2>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <2 x i32> <i32 0, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
-; SLM-NEXT:    [[R00:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, double [[R0]], i64 0
+; SLM-NEXT:    [[R00:%.*]] = insertelement <4 x double> zeroinitializer, double [[R0]], i32 0
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R031:%.*]] = shufflevector <4 x double> [[R00]], <4 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SLM-NEXT:    ret <4 x double> [[R031]]
 ;
 ; AVX-LABEL: @test_v4f64_partial_swizzle(
-; AVX-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; AVX-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
+; AVX-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; AVX-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
 ; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <2 x i32> <i32 0, i32 4>
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; AVX-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; AVX-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; AVX-NEXT:    [[R0212:%.*]] = insertelement <4 x double> [[TMP4]], double 0.000000e+00, i64 1
-; AVX-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R0212]], double [[R3]], i64 3
+; AVX-NEXT:    [[R021:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+; AVX-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R021]], double [[R3]], i32 3
 ; AVX-NEXT:    ret <4 x double> [[R03]]
 ;
   %a0 = extractelement <4 x double> %a, i64 0
@@ -268,21 +268,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -330,21 +330,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -376,21 +376,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -438,21 +438,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
index 40b6a8c32f5d0..6f655b3e9ae0b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
 
 ;
 ; 128-bit vectors
@@ -148,21 +148,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -194,21 +194,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -256,21 +256,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -302,21 +302,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -364,21 +364,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
index 09113323d3ab7..2ab81d9b8d308 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
 
 ;
 ; 128-bit vectors
@@ -148,21 +148,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -194,21 +194,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -256,21 +256,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -302,21 +302,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -364,21 +364,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index 9c9a89ac76d8b..58fb5f772207d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+sse2     -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx      -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx2     -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx512f  -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx512vl -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+sse2     -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx      -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx2     -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx512f  -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx512vl -S | FileCheck %s --check-prefix=AVX
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -17,15 +17,15 @@ target triple = "x86_64-unknown-linux-gnu"
 define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_zext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[TMP6]]
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -33,15 +33,15 @@ define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ;
 ; AVX-LABEL: @PR31243_zext(
 ; AVX-NEXT:  entry:
-; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; AVX-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; AVX-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[TMP6]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -76,15 +76,15 @@ entry:
 define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_sext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; SSE-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; SSE-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i32
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; SSE-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i32
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -92,15 +92,15 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ;
 ; AVX-LABEL: @PR31243_sext(
 ; AVX-NEXT:  entry:
-; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; AVX-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; AVX-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i32
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i32
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
index 61938d01e57ac..de72521345435 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: opt < %s -passes=slp-vectorizer,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 
@@ -10,16 +10,16 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 define void @shuffle_operands1(ptr noalias %from, ptr noalias %to, double %v1, double %v2) {
 ; CHECK-LABEL: @shuffle_operands1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; SSE2-LABEL: @shuffle_operands1(
 ; SSE2-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i64 0
-; SSE2-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i32 0
+; SSE2-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i32 1
 ; SSE2-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; SSE2-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    ret void
@@ -43,7 +43,7 @@ define void @vecload_vs_broadcast(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -57,7 +57,7 @@ define void @vecload_vs_broadcast(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP2]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -91,7 +91,7 @@ define void @vecload_vs_broadcast2(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -105,7 +105,7 @@ define void @vecload_vs_broadcast2(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -139,7 +139,7 @@ define void @vecload_vs_broadcast3(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -153,7 +153,7 @@ define void @vecload_vs_broadcast3(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -185,12 +185,12 @@ define void @shuffle_nodes_match1(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK-NEXT:    br label [[LP:%.*]]
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr i8, ptr [[FROM:%.*]], i32 8
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, ptr [[FROM:%.*]], i64 1
 ; CHECK-NEXT:    [[V0_1:%.*]] = load double, ptr [[FROM]], align 4
 ; CHECK-NEXT:    [[V0_2:%.*]] = load double, ptr [[FROM_1]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
@@ -205,7 +205,7 @@ define void @shuffle_nodes_match1(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP1]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -239,7 +239,7 @@ define void @vecload_vs_broadcast4(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -253,7 +253,7 @@ define void @vecload_vs_broadcast4(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP1]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -286,13 +286,13 @@ define void @shuffle_nodes_match2(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK-NEXT:    br label [[LP:%.*]]
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr i8, ptr [[FROM:%.*]], i32 8
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, ptr [[FROM:%.*]], i64 1
 ; CHECK-NEXT:    [[V0_1:%.*]] = load double, ptr [[FROM]], align 4
 ; CHECK-NEXT:    [[V0_2:%.*]] = load double, ptr [[FROM_1]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i32 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -306,7 +306,7 @@ define void @shuffle_nodes_match2(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -347,23 +347,20 @@ define void @good_load_order() {
 ; CHECK:       for.body3:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi float [ [[TMP0]], [[FOR_COND1_PREHEADER]] ], [ [[TMP13:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
-; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP12]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
 ; CHECK-NEXT:    [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
 ; CHECK-NEXT:    store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
@@ -382,23 +379,20 @@ define void @good_load_order() {
 ; SSE2:       for.body3:
 ; SSE2-NEXT:    [[TMP1:%.*]] = phi float [ [[TMP0]], [[FOR_COND1_PREHEADER]] ], [ [[TMP13:%.*]], [[FOR_BODY3]] ]
 ; SSE2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
-; SSE2-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
-; SSE2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
-; SSE2-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
-; SSE2-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
-; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
+; SSE2-NEXT:    [[TMP2:%.*]] = add nsw i64 [[INDVARS_IV]], 1
+; SSE2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP2]]
+; SSE2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; SSE2-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], 4
+; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP3]]
 ; SSE2-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
 ; SSE2-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
 ; SSE2-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
-; SSE2-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP1]], i64 0
+; SSE2-NEXT:    [[TMP12:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; SSE2-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP12]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; SSE2-NEXT:    [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
 ; SSE2-NEXT:    store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
 ; SSE2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; SSE2-NEXT:    [[TMP12:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; SSE2-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
+; SSE2-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
 ; SSE2-NEXT:    [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
 ; SSE2-NEXT:    [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
 ; SSE2-NEXT:    store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll
index 2588012847d09..6fd2de8ad8ab5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-- -mcpu=corei7 < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=corei7 < %s | FileCheck %s
 
 define void @test1(float %a, float %b, float %c, float %d, ptr nocapture %p) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptosi <4 x float> [[TMP3]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -30,10 +30,10 @@ entry:
 define void @test1_vec(float %a, float %b, float %c, float %d, ptr nocapture %p) {
 ; CHECK-LABEL: @test1_vec(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptosi <4 x float> [[TMP3]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[P:%.*]], align 16, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
@@ -54,10 +54,10 @@ entry:
 define void @test2(i32 %a, i32 %b, i32 %c, i32 %d, ptr nocapture %p) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C:%.*]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], splat (i32 1)
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[P:%.*]], align 4, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
@@ -79,10 +79,10 @@ entry:
 
 define void @test2_vec(i32 %0, i32 %1, i32 %2, i32 %3, ptr nocapture %4) {
 ; CHECK-LABEL: @test2_vec(
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0:%.*]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP1:%.*]], i64 1
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP2:%.*]], i64 2
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP3:%.*]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP2:%.*]], i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP3:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[TMP9]], splat (i32 1)
 ; CHECK-NEXT:    store <4 x i32> [[TMP10]], ptr [[TMP4:%.*]], align 16, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
index bae127527bbbe..9e4f10ec7b349 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
@@ -1,19 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2   -slp-threshold=-1 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx    | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2   | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2   -slp-threshold=-1 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx    | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2   | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s
 
 define void @store_i32(ptr nocapture %0, i32 %1, i32 %2) {
 ; CHECK-LABEL: @store_i32(
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <4 x i32> [[TMP4]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP7]], splat (i32 15)
-; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[TMP8]], <4 x i32> splat (i32 255))
-; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ult <4 x i32> [[TMP8]], splat (i32 255)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP8]], <4 x i32> splat (i32 255)
+; CHECK-NEXT:    store <4 x i32> [[TMP10]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i32, ptr %0, align 4, !tbaa !2
@@ -50,13 +51,14 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) {
 ; CHECK-LABEL: @store_i8(
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[TMP0:%.*]], align 1, !tbaa [[TBAA4:![0-9]+]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = lshr <4 x i32> [[TMP8]], splat (i32 15)
-; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[TMP9]], <4 x i32> splat (i32 255))
-; CHECK-NEXT:    [[TMP11:%.*]] = trunc nuw <4 x i32> [[TMP10]] to <4 x i8>
-; CHECK-NEXT:    store <4 x i8> [[TMP11]], ptr [[TMP0]], align 1, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult <4 x i32> [[TMP9]], splat (i32 255)
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP9]], <4 x i32> splat (i32 255)
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc <4 x i32> [[TMP11]] to <4 x i8>
+; CHECK-NEXT:    store <4 x i8> [[TMP12]], ptr [[TMP0]], align 1, !tbaa [[TBAA4]]
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i8, ptr %0, align 1, !tbaa !6
@@ -101,7 +103,7 @@ define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) {
 ; CHECK-LABEL: @store_i64(
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]]
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
index a9c0eb3f9f2b9..1b11c3dcc081c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
 
 
 @b = global [8 x i32] zeroinitializer, align 16
@@ -13,14 +13,14 @@ define void @foo() {
 ; SSE-LABEL: @foo(
 ; SSE-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 16
 ; SSE-NEXT:    store i32 [[TMP1]], ptr @a, align 16
-; SSE-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @b, i64 8), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 4), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 8), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 12), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 16), align 16
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 20), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 24), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 28), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @b, i64 0, i64 2), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 2), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 4), align 16
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 6), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 7), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @foo(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
index d2617a1986764..db38a62017391 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
@@ -1,89 +1,104 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
 
 define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; SSE-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; SSE-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; SSE-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; SSE-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; SSE-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; SSE-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; SSE-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX2-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX2-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512F-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512F-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512F-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512F-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512F-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX512F-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512F-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512F-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512VL-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512VL-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512VL-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512VL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512VL-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX512VL-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512VL-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512VL-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    ret void
 ;
   %3 = getelementptr inbounds i32, ptr %1, i64 1
@@ -107,64 +122,64 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 
 define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_2(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP5]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP9]], ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP13]], ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
 ; SSE-NEXT:    store i32 [[TMP17]], ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_2(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_2(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX2-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_2(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512F-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -172,8 +187,8 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load_2(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -207,39 +222,39 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; SSE-LABEL: @gather_load_3(
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 1
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP4]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], 2
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP8]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; SSE-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 3
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP12]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; SSE-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 4
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
 ; SSE-NEXT:    store i32 [[TMP16]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; SSE-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP20:%.*]] = add i32 [[TMP19]], 1
-; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 20
+; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 5
 ; SSE-NEXT:    store i32 [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; SSE-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP24:%.*]] = add i32 [[TMP23]], 2
-; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
+; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 6
 ; SSE-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; SSE-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP28:%.*]] = add i32 [[TMP27]], 3
-; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 28
+; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 7
 ; SSE-NEXT:    store i32 [[TMP28]], ptr [[TMP25]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; SSE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], 4
 ; SSE-NEXT:    store i32 [[TMP32]], ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
@@ -247,28 +262,28 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX-LABEL: @gather_load_3(
 ; AVX-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; AVX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; AVX-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; AVX-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; AVX-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; AVX-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
-; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
-; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i64 2
-; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i64 3
-; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i64 4
-; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i64 5
-; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i64 6
-; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i64 7
+; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i32 0
+; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i32 2
+; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i32 3
+; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i32 4
+; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i32 5
+; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i32 6
+; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i32 7
 ; AVX-NEXT:    [[TMP26:%.*]] = add <8 x i32> [[TMP25]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP26]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
@@ -340,20 +355,20 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 
 define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture readonly %t1) {
 ; SSE-LABEL: @gather_load_4(
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[T0:%.*]], i64 4
-; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 8
-; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 12
-; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 16
-; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 20
-; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 24
-; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 28
-; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i32, ptr [[T0:%.*]], i64 1
+; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 2
+; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 3
+; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 4
+; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 5
+; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 6
+; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 7
+; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; SSE-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -381,13 +396,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_4(
-; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; AVX-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -396,14 +411,14 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; AVX-NEXT:    [[T23:%.*]] = load i32, ptr [[T22]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T27:%.*]] = load i32, ptr [[T26]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T31:%.*]] = load i32, ptr [[T30]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i64 0
-; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i64 1
-; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i64 2
-; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i64 3
-; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i64 4
-; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i64 5
-; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i64 6
-; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i32 1
+; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i32 2
+; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i32 3
+; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i32 4
+; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i32 5
+; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i32 6
+; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i32 7
 ; AVX-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP9]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
@@ -481,153 +496,154 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_div(
 ; SSE-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 16
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
 ; SSE-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP15:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP17:%.*]] = shufflevector <4 x float> [[TMP15]], <4 x float> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
-; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP19:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; SSE-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> [[TMP12]], <4 x i32> <i32 1, i32 2, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP21:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i64 3
+; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i32 3
 ; SSE-NEXT:    [[TMP23:%.*]] = fdiv <4 x float> [[TMP19]], [[TMP22]]
 ; SSE-NEXT:    store <4 x float> [[TMP23]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; SSE-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; SSE-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; SSE-NEXT:    [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; SSE-NEXT:    [[TMP31:%.*]] = load float, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; SSE-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; SSE-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i64 0
-; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i64 1
-; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i64 2
-; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i64 3
-; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i64 0
-; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i64 1
-; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i64 2
-; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i64 3
+; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i32 0
+; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i32 1
+; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i32 2
+; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i32 3
+; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i32 0
+; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i32 1
+; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i32 2
+; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i32 3
 ; SSE-NEXT:    [[TMP48:%.*]] = fdiv <4 x float> [[TMP43]], [[TMP47]]
 ; SSE-NEXT:    store <4 x float> [[TMP48]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_div(
 ; AVX-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_div(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX2-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX2-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX2-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX2-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX2-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX2-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX2-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX2-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX2-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX2-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_div(
 ; AVX512F-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512F-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512F-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
@@ -637,6 +653,7 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX512VL-LABEL: @gather_load_div(
 ; AVX512VL-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512VL-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
index 8f31200a3683d..bfa3610804967 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
@@ -1,89 +1,104 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
 
 define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; SSE-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; SSE-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; SSE-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; SSE-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; SSE-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; SSE-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; SSE-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX2-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX2-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512F-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512F-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512F-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512F-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512F-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX512F-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512F-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512F-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512VL-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512VL-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512VL-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512VL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512VL-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX512VL-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512VL-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512VL-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    ret void
 ;
   %3 = getelementptr inbounds i32, ptr %1, i64 1
@@ -107,64 +122,64 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 
 define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_2(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP5]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP9]], ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP13]], ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
 ; SSE-NEXT:    store i32 [[TMP17]], ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_2(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_2(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX2-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_2(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512F-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -172,8 +187,8 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load_2(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -207,39 +222,39 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; SSE-LABEL: @gather_load_3(
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 1
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP4]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], 2
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP8]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; SSE-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 3
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP12]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; SSE-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 4
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
 ; SSE-NEXT:    store i32 [[TMP16]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; SSE-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP20:%.*]] = add i32 [[TMP19]], 1
-; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 20
+; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 5
 ; SSE-NEXT:    store i32 [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; SSE-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP24:%.*]] = add i32 [[TMP23]], 2
-; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
+; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 6
 ; SSE-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; SSE-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP28:%.*]] = add i32 [[TMP27]], 3
-; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 28
+; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 7
 ; SSE-NEXT:    store i32 [[TMP28]], ptr [[TMP25]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; SSE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], 4
 ; SSE-NEXT:    store i32 [[TMP32]], ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
@@ -247,28 +262,28 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX-LABEL: @gather_load_3(
 ; AVX-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; AVX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; AVX-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; AVX-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; AVX-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; AVX-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
-; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
-; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i64 2
-; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i64 3
-; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i64 4
-; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i64 5
-; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i64 6
-; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i64 7
+; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i32 0
+; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i32 2
+; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i32 3
+; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i32 4
+; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i32 5
+; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i32 6
+; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i32 7
 ; AVX-NEXT:    [[TMP26:%.*]] = add <8 x i32> [[TMP25]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP26]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
@@ -340,20 +355,20 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 
 define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture readonly %t1) {
 ; SSE-LABEL: @gather_load_4(
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[T0:%.*]], i64 4
-; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 8
-; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 12
-; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 16
-; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 20
-; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 24
-; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 28
-; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i32, ptr [[T0:%.*]], i64 1
+; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 2
+; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 3
+; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 4
+; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 5
+; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 6
+; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 7
+; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; SSE-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -381,13 +396,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_4(
-; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; AVX-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -396,14 +411,14 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; AVX-NEXT:    [[T23:%.*]] = load i32, ptr [[T22]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T27:%.*]] = load i32, ptr [[T26]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T31:%.*]] = load i32, ptr [[T30]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i64 0
-; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i64 1
-; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i64 2
-; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i64 3
-; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i64 4
-; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i64 5
-; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i64 6
-; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i32 1
+; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i32 2
+; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i32 3
+; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i32 4
+; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i32 5
+; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i32 6
+; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i32 7
 ; AVX-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP9]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
@@ -481,153 +496,154 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_div(
 ; SSE-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 16
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
 ; SSE-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP15:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP17:%.*]] = shufflevector <4 x float> [[TMP15]], <4 x float> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
-; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP19:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; SSE-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> [[TMP12]], <4 x i32> <i32 1, i32 2, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP21:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i64 3
+; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i32 3
 ; SSE-NEXT:    [[TMP23:%.*]] = fdiv <4 x float> [[TMP19]], [[TMP22]]
 ; SSE-NEXT:    store <4 x float> [[TMP23]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; SSE-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; SSE-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; SSE-NEXT:    [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; SSE-NEXT:    [[TMP31:%.*]] = load float, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; SSE-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; SSE-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i64 0
-; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i64 1
-; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i64 2
-; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i64 3
-; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i64 0
-; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i64 1
-; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i64 2
-; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i64 3
+; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i32 0
+; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i32 1
+; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i32 2
+; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i32 3
+; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i32 0
+; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i32 1
+; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i32 2
+; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i32 3
 ; SSE-NEXT:    [[TMP48:%.*]] = fdiv <4 x float> [[TMP43]], [[TMP47]]
 ; SSE-NEXT:    store <4 x float> [[TMP48]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_div(
 ; AVX-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_div(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX2-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX2-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX2-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX2-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX2-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX2-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX2-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX2-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX2-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX2-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_div(
 ; AVX512F-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512F-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512F-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
@@ -637,6 +653,7 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX512VL-LABEL: @gather_load_div(
 ; AVX512VL-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512VL-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
index 42a50384787c8..a4949bc67b0f1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer,instcombine -S < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S < %s | FileCheck %s
 ; These code should be fully vectorized by D57059 patch
 
 target triple = "x86_64-unknown-linux-gnu"
 
 define <4 x i32> @foo(<4 x i32> %x, i32 %f) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i32> poison, i32 [[F:%.*]], i64 0
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i32> undef, i32 [[F:%.*]], i32 0
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[F]], 1
-; CHECK-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[ADD]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i64 0
+; CHECK-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[ADD]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
@@ -28,7 +28,7 @@ define <4 x i32> @foo(<4 x i32> %x, i32 %f) {
 
 define <4 x i32> @bar(<4 x i32> %x, i32 %f) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F:%.*]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
index e7239f906c59d..bada001ebbc6c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
@@ -1,12 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple=x86_64-- -passes=slp-vectorizer,instcombine -S < %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -passes=slp-vectorizer -S < %s | FileCheck %s
 
 ; These conversions should be vectorized by reviews.llvm.org/D57059
 
 define dso_local <4 x float> @foo(<4 x i32> %0) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP0:%.*]] to <4 x float>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[TMP1:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP0]] to <4 x float>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 0, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[TMP3]]
 ;
   %2 = extractelement <4 x i32> %0, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll b/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
index 54120f5402f57..70ab6d09a9236 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s
 
 define float @dotf(<4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @dotf(
diff --git a/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll b/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll
index 94c361caf90fa..0688a235f80e4 100644
--- a/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86_64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu -o - < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu -o - < %s | FileCheck %s %}
 ; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -o - < %s | FileCheck %s %}
 
 define i32 @crash() {
diff --git a/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll b/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
index b85c78ec8d2d0..cec99c694391b 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s
-; RUN: opt -passes=slp-vectorizer,instcombine -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s --check-prefix COMBINE
+; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s --check-prefix COMBINE
 
 define void @test1(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test1(
@@ -19,8 +19,14 @@ define void @test1(ptr %in, ptr %out) {
 ; COMBINE-LABEL: @test1(
 ; COMBINE-NEXT:  entry:
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[IN:%.*]], align 1
-; COMBINE-NEXT:    [[TMP1:%.*]] = zext <8 x i32> [[TMP0]] to <8 x i64>
-; COMBINE-NEXT:    store <8 x i64> [[TMP1]], ptr [[OUT:%.*]], align 8
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i64, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP0]], i64 0)
+; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP5:%.*]] = zext <8 x i32> [[TMP4]] to <8 x i64>
+; COMBINE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    store <8 x i64> [[TMP1]], ptr [[OUT]], align 8
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -61,9 +67,14 @@ define void @test2(ptr %in, ptr %out) {
 ; COMBINE-LABEL: @test2(
 ; COMBINE-NEXT:  entry:
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[IN:%.*]], align 1
-; COMBINE-NEXT:    [[TMP1:%.*]] = zext <8 x i32> [[TMP0]] to <8 x i64>
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i64, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP0]], i64 0)
+; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP1:%.*]] = zext <8 x i32> [[TMP4]] to <8 x i64>
+; COMBINE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
 ; COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
-; COMBINE-NEXT:    store <8 x i64> [[TMP2]], ptr [[OUT:%.*]], align 8
+; COMBINE-NEXT:    store <8 x i64> [[TMP2]], ptr [[OUT]], align 8
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -99,8 +110,11 @@ define void @test3(<16 x i32> %0, ptr %out) {
 ;
 ; COMBINE-LABEL: @test3(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0:%.*]], <16 x i32> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; COMBINE-NEXT:    [[TMP3:%.*]] = call <64 x i32> @llvm.vector.insert.v64i32.v16i32(<64 x i32> poison, <16 x i32> [[TMP0:%.*]], i64 0)
+; COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i32> [[TMP3]], <64 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i32> [[TMP3]], <64 x i32> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i32, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT]], align 4
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -134,8 +148,12 @@ define void @test4(ptr %in, ptr %out) {
 ; COMBINE-LABEL: @test4(
 ; COMBINE-NEXT:  entry:
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[IN:%.*]], align 4
-; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i32, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP0]], i64 0)
+; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT]], align 4
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -165,7 +183,11 @@ define void @test5(ptr %out) {
 ;
 ; COMBINE-LABEL: @test5(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    store <8 x i32> zeroinitializer, ptr [[OUT:%.*]], align 4
+; COMBINE-NEXT:    [[TMP0:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> zeroinitializer, i64 0)
+; COMBINE-NEXT:    [[TMP1:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> [[TMP0]], <8 x i32> zeroinitializer, i64 8)
+; COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT:%.*]], i64 0
+; COMBINE-NEXT:    store <8 x i32> [[TMP2]], ptr [[TMP3]], align 4
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -219,28 +241,37 @@ define void @test6(ptr %in0, ptr %in1, ptr %in2) {
 ;
 ; COMBINE-LABEL: @test6(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[IN0:%.*]], i64 32
+; COMBINE-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[IN0:%.*]], i64 32
 ; COMBINE-NEXT:    [[LOAD2:%.*]] = load <4 x float>, ptr [[GEP1]], align 16
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x float>, ptr [[IN0]], align 16
 ; COMBINE-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr [[IN1:%.*]], align 1
-; COMBINE-NEXT:    [[TMP2:%.*]] = uitofp <32 x i8> [[TMP1]] to <32 x float>
+; COMBINE-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP4:%.*]] = zext <32 x i8> [[TMP11]] to <32 x i16>
+; COMBINE-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i16> [[TMP4]], <32 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP19:%.*]] = shufflevector <32 x i16> [[TMP4]], <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP2:%.*]] = uitofp <32 x i16> [[TMP19]] to <32 x float>
 ; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[LOAD2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[TMP3]], <16 x float> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; COMBINE-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP3]], <4 x float> [[LOAD2]], i64 8)
 ; COMBINE-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[TMP5]], <16 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; COMBINE-NEXT:    [[TMP7:%.*]] = fmul <32 x float> [[TMP6]], [[TMP2]]
-; COMBINE-NEXT:    [[GEP10:%.*]] = getelementptr inbounds nuw i8, ptr [[IN1]], i64 32
-; COMBINE-NEXT:    [[GEP11:%.*]] = getelementptr inbounds nuw i8, ptr [[IN2:%.*]], i64 128
+; COMBINE-NEXT:    [[GEP10:%.*]] = getelementptr inbounds i8, ptr [[IN1]], i64 32
+; COMBINE-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i8, ptr [[IN2:%.*]], i64 128
 ; COMBINE-NEXT:    [[TMP8:%.*]] = load <8 x float>, ptr [[IN0]], align 16
 ; COMBINE-NEXT:    store <32 x float> [[TMP7]], ptr [[IN2]], align 16
 ; COMBINE-NEXT:    [[LOAD5:%.*]] = load <16 x i8>, ptr [[GEP10]], align 1
-; COMBINE-NEXT:    [[TMP9:%.*]] = uitofp <16 x i8> [[LOAD5]] to <16 x float>
-; COMBINE-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[LOAD2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP10]], <16 x float> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP13:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; COMBINE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP15:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; COMBINE-NEXT:    [[TMP13:%.*]] = call <32 x i8> @llvm.vector.insert.v32i8.v16i8(<32 x i8> poison, <16 x i8> [[LOAD5]], i64 0)
+; COMBINE-NEXT:    [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP24:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP25:%.*]] = zext <16 x i8> [[TMP24]] to <16 x i16>
+; COMBINE-NEXT:    [[TMP26:%.*]] = shufflevector <16 x i16> [[TMP25]], <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i16> [[TMP25]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP9:%.*]] = uitofp <16 x i16> [[TMP18]] to <16 x float>
+; COMBINE-NEXT:    [[TMP20:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> poison, <4 x float> [[LOAD2]], i64 0)
+; COMBINE-NEXT:    [[TMP21:%.*]] = call <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float> [[TMP8]], i64 0)
+; COMBINE-NEXT:    [[TMP22:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP20]], <4 x float> [[TMP21]], i64 4)
+; COMBINE-NEXT:    [[TMP23:%.*]] = call <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float> [[TMP8]], i64 4)
+; COMBINE-NEXT:    [[TMP15:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP22]], <4 x float> [[TMP23]], i64 8)
 ; COMBINE-NEXT:    [[TMP16:%.*]] = shufflevector <16 x float> [[TMP15]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3>
 ; COMBINE-NEXT:    [[TMP17:%.*]] = fmul <16 x float> [[TMP16]], [[TMP9]]
 ; COMBINE-NEXT:    store <16 x float> [[TMP17]], ptr [[GEP11]], align 16
@@ -353,7 +384,21 @@ define i32 @test7() {
 ;
 ; COMBINE-LABEL: @test7(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    store <16 x float> poison, ptr null, align 16
+; COMBINE-NEXT:    [[TMP0:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> zeroinitializer, i64 0)
+; COMBINE-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP0]], <8 x float> zeroinitializer, i64 8)
+; COMBINE-NEXT:    [[TMP2:%.*]] = fsub <16 x float> [[TMP1]], [[TMP1]]
+; COMBINE-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[TMP1]], [[TMP1]]
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> [[TMP3]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> poison, <4 x float> zeroinitializer, i64 0)
+; COMBINE-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP6]], <4 x float> zeroinitializer, i64 4)
+; COMBINE-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP7]], <4 x float> zeroinitializer, i64 8)
+; COMBINE-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP8]], <4 x float> zeroinitializer, i64 12)
+; COMBINE-NEXT:    [[TMP10:%.*]] = fadd <16 x float> [[TMP9]], [[TMP5]]
+; COMBINE-NEXT:    [[TMP11:%.*]] = fsub <16 x float> [[TMP9]], [[TMP5]]
+; COMBINE-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP10]], <16 x float> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[TMP9]], [[TMP12]]
+; COMBINE-NEXT:    store <16 x float> [[TMP13]], ptr null, align 16
 ; COMBINE-NEXT:    ret i32 0
 ;
 entry:
diff --git a/llvm/test/Transforms/SROA/select-gep.ll b/llvm/test/Transforms/SROA/select-gep.ll
index 1342a2ca4ea2b..b48b0f77aa991 100644
--- a/llvm/test/Transforms/SROA/select-gep.ll
+++ b/llvm/test/Transforms/SROA/select-gep.ll
@@ -201,6 +201,47 @@ define i32 @test_select_idx_mem2reg(i1 %c) {
   ret i32 %res
 }
 
+; Test gep with a select-like zext index unfolding on an alloca that is
+; splittable and promotable.
+define i64 @test_select_like_zext_idx_mem2reg(i1 %c) {
+; CHECK-LABEL: @test_select_like_zext_idx_mem2reg(
+; CHECK-NEXT:    [[IDX:%.*]] = zext i1 [[C:%.*]] to i64
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C]], i64 2, i64 1
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %alloca = alloca [2 x i64], align 8
+  store i64 1, ptr %alloca
+  %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1
+  store i64 2, ptr %gep1
+  %idx = zext i1 %c to i64
+  %gep2 = getelementptr inbounds i64, ptr %alloca, i64 %idx
+  %res = load i64, ptr %gep2
+  ret i64 %res
+}
+
+; Test gep with a zext index that is not equivalent to a select. No unfolding
+; or promotion should take place.
+define i64 @test_zext_unlike_select_idx_mem2reg(i8 %c) {
+; CHECK-LABEL: @test_zext_unlike_select_idx_mem2reg(
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x i64], align 8
+; CHECK-NEXT:    store i64 1, ptr [[ALLOCA]], align 4
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 1
+; CHECK-NEXT:    store i64 2, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[IDX:%.*]] = zext i8 [[C:%.*]] to i64
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 [[IDX]]
+; CHECK-NEXT:    [[RES:%.*]] = load i64, ptr [[GEP2]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %alloca = alloca [2 x i64], align 8
+  store i64 1, ptr %alloca
+  %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1
+  store i64 2, ptr %gep1
+  %idx = zext i8 %c to i64
+  %gep2 = getelementptr inbounds i64, ptr %alloca, i64 %idx
+  %res = load i64, ptr %gep2
+  ret i64 %res
+}
+
 ; Test gep of index select unfolding on an alloca that escaped, and as such
 ; is not splittable or promotable.
 ; FIXME: Ideally, no transform would take place in this case.
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
index 2ac94afd95910..4c2aa18211684 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
@@ -34,7 +34,8 @@ define i32 @switch_of_powers(i32 %x) {
 ; RV64ZBB-LABEL: @switch_of_powers(
 ; RV64ZBB-NEXT:  entry:
 ; RV64ZBB-NEXT:    [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
-; RV64ZBB-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers, i32 0, i32 [[TMP0]]
+; RV64ZBB-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; RV64ZBB-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers, i64 0, i64 [[TMP1]]
 ; RV64ZBB-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; RV64ZBB-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
index 6c9450bb8998e..11fca66cbf714 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
@@ -27,7 +27,8 @@ define i32 @f(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 7
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -67,7 +68,8 @@ define i8 @char(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 9
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -114,7 +116,8 @@ define void @h(i32 %x) {
 ; CHECK-NEXT:    [[SWITCH_SHIFTAMT:%.*]] = mul nuw nsw i32 [[X]], 8
 ; CHECK-NEXT:    [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 89655594, [[SWITCH_SHIFTAMT]]
 ; CHECK-NEXT:    [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load float, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
@@ -159,7 +162,8 @@ define ptr @foostring(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
index 98a2b33d87e34..5bc4b38cf4c11 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
@@ -50,7 +50,8 @@ define i32 @bar(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.bar, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.bar, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
index cf244e6b63457..3abde486c4e35 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
@@ -10,8 +10,8 @@ define i64 @test(i3 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i3 [[ARG:%.*]], -4
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i4
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i64], ptr @switch.table.test, i32 0, i4 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i64], ptr @switch.table.test, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    [[V3:%.*]] = add i64 [[SWITCH_LOAD]], 0
 ; CHECK-NEXT:    ret i64 [[V3]]
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll
index 529826758c138..d307ca4632912 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll
@@ -8,7 +8,8 @@ define i32 @switch_of_powers_two(i32 %arg) {
 ; CHECK-SAME: i32 [[ARG:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[ARG]], i1 true)
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers_two, i32 0, i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers_two, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
index 37001f4fba2aa..63c16a2f7b84c 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
@@ -10,8 +10,8 @@ define i64 @_TFO6reduce1E5toRawfS0_FT_Si(i2) {
 ; CHECK-LABEL: @_TFO6reduce1E5toRawfS0_FT_Si(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TMP0:%.*]], -2
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table._TFO6reduce1E5toRawfS0_FT_Si, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table._TFO6reduce1E5toRawfS0_FT_Si, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    ret i64 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll
index ec42efb12aad1..b5548de8195c7 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll
@@ -9,7 +9,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define { ptr, i64 } @switch_to_lookup_bitcast(i8 %0) unnamed_addr {
 ; CHECK-LABEL: @switch_to_lookup_bitcast(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_bitcast, i32 0, i8 [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP0:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_bitcast, i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { ptr, i64 } undef, ptr [[SWITCH_LOAD]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { ptr, i64 } [[TMP1]], i64 1, 1
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll
index c7d1c2a237eff..4463e8931700d 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll
@@ -9,7 +9,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define { ptr, i64 } @switch_to_lookup_gep(i8 %0) unnamed_addr {
 ; CHECK-LABEL: @switch_to_lookup_gep(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_gep, i32 0, i8 [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP0:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_gep, i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { ptr, i64 } undef, ptr [[SWITCH_LOAD]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { ptr, i64 } [[TMP1]], i64 1, 1
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
index 7c97827d108d0..4ee7a2710afbe 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
@@ -13,7 +13,8 @@ define i1 @zot(i32 %arg) {
 ; CHECK-NEXT:    %0 = icmp ult i32 %arg, 3
 ; CHECK-NEXT:    br i1 %0, label %switch.lookup, label %bb6
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    %switch.gep = getelementptr inbounds [3 x ptr], ptr @switch.table.zot, i32 0, i32 %arg
+; CHECK-NEXT:    %1 = zext i32 %arg to i64
+; CHECK-NEXT:    %switch.gep = getelementptr inbounds [3 x ptr], ptr @switch.table.zot, i64 0, i64 %1
 ; CHECK-NEXT:    %switch.load = load ptr, ptr %switch.gep, align 8
 ; CHECK-NEXT:    br label %bb6
 ; CHECK:       bb6:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
index e188cd8057df2..169faff7e228b 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
@@ -17,7 +17,7 @@ define i8 @switch_to_lookup_i64(i64 %x){
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i8 [ [[SWITCH_LOAD:%.*]], [[SWITCH_LOOKUP]] ], [ 10, [[START:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[COMMON_RET_OP]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i64, i32 0, i64 [[X]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i64, i64 0, i64 [[X]]
 ; CHECK-NEXT:    [[SWITCH_LOAD]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
@@ -61,7 +61,8 @@ define i8 @switch_to_lookup_i128(i128 %x){
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i8 [ [[SWITCH_LOAD:%.*]], [[SWITCH_LOOKUP]] ], [ 10, [[START:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[COMMON_RET_OP]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i128, i32 0, i128 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i128 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i128, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index 95bf296af9b0c..9124b356a46bf 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -50,7 +50,8 @@ define i32 @f(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 7
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -90,7 +91,8 @@ define i8 @char(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 9
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -137,7 +139,8 @@ define void @h(i32 %x) {
 ; CHECK-NEXT:    [[SWITCH_SHIFTAMT:%.*]] = mul nuw nsw i32 [[X]], 8
 ; CHECK-NEXT:    [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 89655594, [[SWITCH_SHIFTAMT]]
 ; CHECK-NEXT:    [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load float, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
@@ -182,7 +185,8 @@ define ptr @foostring(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -221,9 +225,11 @@ define i32 @earlyreturncrash(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[SW_EPILOG:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
-; CHECK-NEXT:    [[SWITCH_GEP1:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash.1, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP1:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash.1, i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[SWITCH_LOAD2:%.*]] = load i32, ptr [[SWITCH_GEP1]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
@@ -404,7 +410,8 @@ define i32 @large(i32 %x) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 199
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [199 x i32], ptr @switch.table.large, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [199 x i32], ptr @switch.table.large, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -835,7 +842,8 @@ define i32 @cprop(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 7
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.cprop, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.cprop, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -885,7 +893,8 @@ define i32 @unreachable_case(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 9
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i32], ptr @switch.table.unreachable_case, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i32], ptr @switch.table.unreachable_case, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -920,7 +929,8 @@ return:
 define i32 @unreachable_default(i32 %x)  {
 ; CHECK-LABEL: @unreachable_default(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.unreachable_default, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.unreachable_default, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1000,7 +1010,8 @@ define i32 @nodefaultnoholes(i32 %c) {
 ; CHECK-NEXT:    call void @exit(i32 1)
 ; CHECK-NEXT:    unreachable
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.nodefaultnoholes, i32 0, i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.nodefaultnoholes, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1037,7 +1048,8 @@ define i32 @nodefaultwithholes(i32 %c) {
 ; CHECK-NEXT:    call void @exit(i32 1)
 ; CHECK-NEXT:    unreachable
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.nodefaultwithholes, i32 0, i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.nodefaultwithholes, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1102,7 +1114,8 @@ define i32 @threecases(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[C:%.*]], 3
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.threecases, i32 0, i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.threecases, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -1561,8 +1574,8 @@ define i32 @covered_switch_with_bit_tests(i3) {
 ; CHECK-NEXT:    [[SWITCH_LOBIT:%.*]] = trunc i8 [[SWITCH_SHIFTED]] to i1
 ; CHECK-NEXT:    br i1 [[SWITCH_LOBIT]], label [[SWITCH_LOOKUP:%.*]], label [[L6:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i4
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.covered_switch_with_bit_tests, i32 0, i4 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.covered_switch_with_bit_tests, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[L6]]
 ; CHECK:       l6:
@@ -1714,8 +1727,8 @@ define i32 @signed_overflow1(i8 %n) {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TRUNC]], -2
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow1, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow1, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1750,8 +1763,8 @@ define i32 @signed_overflow2(i8 %n) {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TRUNC]], -2
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow2, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow2, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -2215,7 +2228,8 @@ return:
 define i32 @constant_hole_unreachable_default_firstundef(i32 %x) {
 ; CHECK-LABEL: @constant_hole_unreachable_default_firstundef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_firstundef, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_firstundef, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -2241,7 +2255,8 @@ return:
 define i32 @constant_hole_unreachable_default_lastundef(i32 %x) {
 ; CHECK-LABEL: @constant_hole_unreachable_default_lastundef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_lastundef, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_lastundef, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -2359,7 +2374,8 @@ return:
 define i32 @linearmap_hole_unreachable_default(i32 %x) {
 ; CHECK-LABEL: @linearmap_hole_unreachable_default(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.linearmap_hole_unreachable_default, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.linearmap_hole_unreachable_default, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
index 467ede9b75c33..c2afba965702a 100644
--- a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
+++ b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
@@ -11,7 +11,8 @@ define i32 @test1(i32 %a) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 4
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test1, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test1, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -80,7 +81,8 @@ define i32 @test3(i32 %a) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 3
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.test3, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.test3, i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -185,7 +187,8 @@ define i32 @test6(i32 %a) optsize {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 4
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test6, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test6, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -248,7 +251,8 @@ define i32 @test8(i32 %a) optsize {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 5
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.test8, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.test8, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -280,7 +284,8 @@ define i32 @test9(i32 %a) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 8
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.test9, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.test9, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll
index bead0dc4c567a..72eea9a064ebd 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll
@@ -8,7 +8,7 @@ define i64 @test_1(i64 %0) {
 ; CHECK-SAME: i64 [[TMP0:%.*]]) {
 ; CHECK-NEXT:  switch.lookup:
 ; CHECK-NEXT:    [[TMP1:%.*]] = urem i64 [[TMP0]], 4
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table.test_1, i32 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table.test_1, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    ret i64 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
index 50998e447b71d..fe2e897125eb8 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
@@ -309,3 +309,444 @@ end:
   %t0 = phi i8 [ 42, %case1 ], [ 42, %case2 ], [ 44, %case3 ], [ 44, %case4 ]
   ret i8 %t0
 }
+
+define i1 @no_range(i8 %f) {
+; CHECK-LABEL: @no_range(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 60
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 60
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 60, label %bb2
+  i8 61, label %bb2
+  i8 62, label %bb2
+  i8 63, label %bb2
+  i8 124, label %bb2
+  i8 188, label %bb2
+  i8 252, label %bb2
+  i8 189, label %bb2
+  i8 190, label %bb2
+  i8 191, label %bb2
+  i8 125, label %bb2
+  i8 126, label %bb2
+  i8 127, label %bb2
+  i8 253, label %bb2
+  i8 254, label %bb2
+  i8 255, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_no_range(i8 %f) {
+; CHECK-LABEL: @negative_no_range(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 52, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 61, label [[BB2]]
+; CHECK-NEXT:      i8 62, label [[BB2]]
+; CHECK-NEXT:      i8 63, label [[BB2]]
+; CHECK-NEXT:      i8 124, label [[BB2]]
+; CHECK-NEXT:      i8 -68, label [[BB2]]
+; CHECK-NEXT:      i8 -4, label [[BB2]]
+; CHECK-NEXT:      i8 -67, label [[BB2]]
+; CHECK-NEXT:      i8 -66, label [[BB2]]
+; CHECK-NEXT:      i8 -65, label [[BB2]]
+; CHECK-NEXT:      i8 125, label [[BB2]]
+; CHECK-NEXT:      i8 126, label [[BB2]]
+; CHECK-NEXT:      i8 127, label [[BB2]]
+; CHECK-NEXT:      i8 -3, label [[BB2]]
+; CHECK-NEXT:      i8 -2, label [[BB2]]
+; CHECK-NEXT:      i8 -1, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[_0_SROA_0_0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 52, label %bb2
+  i8 61, label %bb2
+  i8 62, label %bb2
+  i8 63, label %bb2
+  i8 124, label %bb2
+  i8 188, label %bb2
+  i8 252, label %bb2
+  i8 189, label %bb2
+  i8 190, label %bb2
+  i8 191, label %bb2
+  i8 125, label %bb2
+  i8 126, label %bb2
+  i8 127, label %bb2
+  i8 253, label %bb2
+  i8 254, label %bb2
+  i8 255, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+; Using ranges.
+
+define i1 @range0to4odd(i8 range(i8 0, 4) %f) {
+; CHECK-LABEL: @range0to4odd(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 1, label %bb2
+  i8 3, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range1to4odd(i8 range(i8 1, 4) %f) {
+; CHECK-LABEL: @range1to4odd(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 1, label %bb2
+  i8 3, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to8odd(i8 range(i8 0, 8) %f) {
+; CHECK-LABEL: @range0to8odd(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 1, label %bb2
+  i8 3, label %bb2
+  i8 5, label %bb2
+  i8 7, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to8most_significant_bit(i8 range(i8 0, 8) %f) {
+; CHECK-LABEL: @range0to8most_significant_bit(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 4, label %bb2
+  i8 5, label %bb2
+  i8 6, label %bb2
+  i8 7, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to15_middle_two_bits(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_middle_two_bits(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 7, label [[BB2]]
+; CHECK-NEXT:      i8 14, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[_0_SROA_0_0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_pow_2(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_pow_2(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[SWITCH_AND:%.*]] = and i8 [[TMP0]], -2
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to5even(i8 range(i8 0, 5) %f) {
+; CHECK-LABEL: @negative_range0to5even(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i8 [[F:%.*]], 2
+; CHECK-NEXT:    [[SWITCH_AND:%.*]] = and i8 [[TMP0]], -3
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  switch i8 %f, label %bb1 [
+  i8 2, label %bb2
+  i8 4, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to15_corner_case(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_corner_case(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[F:%.*]], 15
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[COND]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[DOT]]
+;
+  switch i8 %f, label %bb1 [
+  i8 15, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_corner_case(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_corner_case(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[SWITCH_SELECTCMP_CASE1:%.*]] = icmp eq i8 [[F:%.*]], 15
+; CHECK-NEXT:    [[SWITCH_SELECTCMP_CASE2:%.*]] = icmp eq i8 [[F]], 8
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = or i1 [[SWITCH_SELECTCMP_CASE1]], [[SWITCH_SELECTCMP_CASE2]]
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 15, label %bb2
+  i8 8,  label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+; Out of range scenarios. Check if the cases, that have a value out of range
+; are eliminated and the optimization is performed.
+
+define i1 @range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_out_of_range_non_prime(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  i8 22, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to15_out_of_range_non_prime_more(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_out_of_range_non_prime_more(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  i8 22, label %bb2
+  i8 23, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_out_of_range_non_prime(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 14, label [[BB2]]
+; CHECK-NEXT:      i8 15, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  i8 23, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_out_of_range(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_out_of_range(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 7, label [[BB2]]
+; CHECK-NEXT:      i8 14, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[_0_SROA_0_0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 150, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_all_out_of_range(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_all_out_of_range(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    ret i1 false
+;
+  switch i8 %f, label %bb1 [
+  i8 22, label %bb2
+  i8 23, label %bb2
+  i8 30, label %bb2
+  i8 31, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_mask.ll b/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
index 51c15774d4762..bbd1505b25974 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
@@ -186,7 +186,8 @@ define void @switch_lookup_with_nonconst_range(i32 %x, i1 %cond) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[ADD]], 6
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[LOR_END:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.switch_lookup_with_nonconst_range, i32 0, i32 [[ADD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[ADD]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.switch_lookup_with_nonconst_range, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[LOR_END]]
 ; CHECK:       lor.end:
diff --git a/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/basic.ll b/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/basic.ll
new file mode 100644
index 0000000000000..ee3a0539bf300
--- /dev/null
+++ b/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/basic.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -passes=declare-runtime-libcalls -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; Check an already declared function
+; CHECK: declare float @logf(float)
+declare float @logf(float)
+
+; Check an already defined function
+; CHECK: define float @sinf(float %x) {
+define float @sinf(float %x) {
+  ret float %x
+}
+
+; CHECK: declare void @acosf(...)
+; CHECK: declare void @__umodti3(...)
+
diff --git a/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll b/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll
index b438b5e24b3a8..38752ac4ff571 100644
--- a/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll
+++ b/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll
@@ -23,7 +23,7 @@ bb:
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
 attributes #0 = { argmemonly nofree nosync nounwind willreturn }
-attributes #1 = { "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" }
+attributes #1 = { "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" }
 attributes #2 = { nofree nosync nounwind readnone speculatable willreturn }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll b/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
index 09c03991ad7c3..60700412686ea 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
@@ -9,15 +9,23 @@ define void @zext_v4i8_all_lanes_used(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_all_lanes_used(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 24
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 255
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP1]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP9]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -68,13 +76,20 @@ define void @zext_v4i8_3_lanes_used_1(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_3_lanes_used_1(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 24
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -93,13 +108,19 @@ define void @zext_v4i8_3_lanes_used_2(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_3_lanes_used_2(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 24
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP1]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -118,11 +139,17 @@ define void @zext_v4i8_2_lanes_used_1(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_2_lanes_used_1(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 255
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP3]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -139,11 +166,16 @@ define void @zext_v4i8_2_lanes_used_2(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_2_lanes_used_2(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 255
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP1]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP3]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -160,15 +192,22 @@ define void @zext_v4i8_all_lanes_used_noundef(<4 x i8> noundef %src) {
 ; CHECK-LABEL: define void @zext_v4i8_all_lanes_used_noundef(
 ; CHECK-SAME: <4 x i8> noundef [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i8> [[SRC]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP0]], 24
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 255
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 255
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP0]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP8]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP6]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP4]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP1]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -221,15 +260,23 @@ define void @zext_v4i16_all_lanes_used(<4 x i16> %src) {
 ; CHECK-LABEL: define void @zext_v4i16_all_lanes_used(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i16> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 48
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 65535
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 65535
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP1]], 65535
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i16> [[SRC]] to <4 x i64>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i64> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i64> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i64> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i64> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP9]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP7]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP5]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -250,11 +297,15 @@ define void @zext_v2i32_all_lanes_used(<2 x i32> %src) {
 ; CHECK-LABEL: define void @zext_v2i32_all_lanes_used(
 ; CHECK-SAME: <2 x i32> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <2 x i32> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <2 x i32> [[SRC]] to <2 x i64>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <2 x i64> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <2 x i64> [[EXT9]], i64 1
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP5]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -266,3 +317,32 @@ entry:
   call void @use.i64(i64 %ext.1)
   ret void
 }
+
+define void @zext_nxv4i8_all_lanes_used(<vscale x 4 x i8> %src) {
+; CHECK-LABEL: define void @zext_nxv4i8_all_lanes_used(
+; CHECK-SAME: <vscale x 4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <vscale x 4 x i8> [[SRC]] to <vscale x 4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <vscale x 4 x i8> %src to <vscale x 4 x i32>
+  %ext.0 = extractelement <vscale x 4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <vscale x 4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <vscale x 4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <vscale x 4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll
index 605c0772f6e38..391215b3e4159 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll
@@ -6,13 +6,12 @@ target triple = "arm64-apple-darwin"
 define void @load_extract_insert_store_const_idx(ptr %A) {
 ; CHECK-LABEL: @load_extract_insert_store_const_idx(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <225 x double>, ptr [[A:%.*]], i32 0, i64 0
-; CHECK-NEXT:    [[EXT_0:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[EXT_0:%.*]] = load double, ptr [[TMP0:%.*]], align 8
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul double 2.000000e+01, [[EXT_0]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i32 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <225 x double>, ptr [[TMP0]], i32 0, i64 1
 ; CHECK-NEXT:    [[EXT_1:%.*]] = load double, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[EXT_1]], [[MUL]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <225 x double>, ptr [[TMP0]], i64 0, i64 1
 ; CHECK-NEXT:    store double [[SUB]], ptr [[TMP2]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
index 551d6d1cabd41..e6e5f5196d3da 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
@@ -15,8 +15,7 @@ define i32 @load_extract_idx_0(ptr %x) {
 
 define i32 @vscale_load_extract_idx_0(ptr %x) {
 ; CHECK-LABEL: @vscale_load_extract_idx_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <vscale x 4 x i32>, ptr [[X:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 16
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1:%.*]], align 16
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %lv = load <vscale x 4 x i32>, ptr %x
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index 1c128c8f56a03..a40d514a520ca 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -997,10 +997,8 @@ define <4 x i64> @bitcast_smax_v8i32_v4i32(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-NEXT:    [[A_BC0:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[B_BC0:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[A_BC0]], [[B_BC0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[B]], <4 x i64> [[B]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[B_BC1:%.*]] = bitcast <4 x i64> [[TMP2]] to <8 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[A]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[A_BC1:%.*]] = bitcast <4 x i64> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[B_BC1:%.*]] = bitcast <4 x i64> [[B]] to <8 x i32>
+; CHECK-NEXT:    [[A_BC1:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
 ; CHECK-NEXT:    [[CONCAT:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[B_BC1]], <8 x i32> [[A_BC1]]
 ; CHECK-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[CONCAT]] to <4 x i64>
 ; CHECK-NEXT:    ret <4 x i64> [[RES]]
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
index 46a622148c871..59ece31631d88 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
@@ -16,8 +16,7 @@ define i32 @reducebase_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define i32 @reduceshuffle_onein_v4i32(<4 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_v4i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %x = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -27,8 +26,7 @@ define i32 @reduceshuffle_onein_v4i32(<4 x i32> %a) {
 
 define i32 @reduceshuffle_onein_const_v4i32(<4 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_const_v4i32(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[X:%.*]] = xor <4 x i32> [[S]], splat (i32 -1)
+; CHECK-NEXT:    [[X:%.*]] = xor <4 x i32> [[S:%.*]], splat (i32 -1)
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -222,8 +220,7 @@ define i32 @reducebase_v16i32(<16 x i32> %a, <16 x i32> %b) {
 
 define i32 @reduceshuffle_onein_v16i32(<16 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_v16i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <16 x i32> [[A:%.*]], <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %x = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -233,8 +230,7 @@ define i32 @reduceshuffle_onein_v16i32(<16 x i32> %a) {
 
 define i32 @reduceshuffle_onein_ext_v16i32(<16 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_ext_v16i32(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <16 x i32> [[A:%.*]], <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[X:%.*]] = xor <16 x i32> [[S]], splat (i32 -1)
+; CHECK-NEXT:    [[X:%.*]] = xor <16 x i32> [[S:%.*]], splat (i32 -1)
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -353,8 +349,7 @@ define i16 @reducebase_v16i16(<16 x i16> %a, <16 x i16> %b) {
 
 define i16 @reduceshuffle_onein_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_v16i16(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[X:%.*]])
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -364,8 +359,7 @@ define i16 @reduceshuffle_onein_v16i16(<16 x i16> %a) {
 
 define i16 @reduceshuffle_onein_ext_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_ext_v16i16(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[X:%.*]] = xor <16 x i16> [[S]], splat (i16 -1)
+; CHECK-NEXT:    [[X:%.*]] = xor <16 x i16> [[S:%.*]], splat (i16 -1)
 ; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[X]])
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
index c123458669088..f4aee898ec838 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
@@ -486,10 +486,7 @@ define <vscale x 1 x i64> @urem_nxv1i64_unspeculatable(i64 %x, i64 %y, i32 zeroe
 
 define <vscale x 1 x i64> @sdiv_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = sdiv i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> splat (i64 1)
 ;
 ; NO-VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -530,10 +527,7 @@ define <vscale x 1 x i64> @sdiv_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale
 
 define <vscale x 1 x i64> @udiv_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = udiv i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> splat (i64 1)
 ;
 ; NO-VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -574,10 +568,7 @@ define <vscale x 1 x i64> @udiv_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale
 
 define <vscale x 1 x i64> @srem_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = srem i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> zeroinitializer
 ;
 ; NO-VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -618,10 +609,7 @@ define <vscale x 1 x i64> @srem_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale
 
 define <vscale x 1 x i64> @urem_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = urem i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> zeroinitializer
 ;
 ; NO-VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -1572,8 +1560,7 @@ define <1 x i64> @add_v1i64_allonesmask(<1 x i64> %x, i64 %y, i32 zeroext %evl)
 ; VEC-COMBINE-64-NEXT:    [[MASK:%.*]] = shufflevector <1 x i1> [[SPLAT]], <1 x i1> poison, <1 x i32> zeroinitializer
 ; VEC-COMBINE-64-NEXT:    [[TMP1:%.*]] = add i64 [[Y:%.*]], 42
 ; VEC-COMBINE-64-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-64-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i64> [[DOTSPLATINSERT]], <1 x i64> poison, <1 x i32> zeroinitializer
-; VEC-COMBINE-64-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.vp.mul.v1i64(<1 x i64> [[X:%.*]], <1 x i64> [[TMP2]], <1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-64-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.vp.mul.v1i64(<1 x i64> [[X:%.*]], <1 x i64> [[DOTSPLATINSERT]], <1 x i1> [[MASK]], i32 [[EVL:%.*]])
 ; VEC-COMBINE-64-NEXT:    ret <1 x i64> [[TMP3]]
 ;
 ; NO-VEC-COMBINE-LABEL: @add_v1i64_allonesmask(
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index 7cbe1c6cec906..d11fb1426c94e 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -567,3 +567,22 @@ define float @constant_fold_crash_commute(<4 x float> %x) {
   %c = fadd float %b, %a
   ret float %c
 }
+
+; Test case for https://github.com/llvm/llvm-project/issues/147218.
+define i64 @instsimplify_folder_crash(<4 x i64> %in) {
+; CHECK-LABEL: @instsimplify_folder_crash(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE_1:%.*]] = shufflevector <4 x i64> [[IN:%.*]], <4 x i64> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    [[E_0:%.*]] = extractelement <4 x i64> zeroinitializer, i64 0
+; CHECK-NEXT:    [[E_1:%.*]] = extractelement <4 x i64> [[SHUFFLE_1]], i64 1
+; CHECK-NEXT:    [[OR:%.*]] = or i64 [[E_1]], [[E_0]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+entry:
+  %shuffle.1 = shufflevector <4 x i64> %in, <4 x i64> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %e.0 = extractelement <4 x i64> zeroinitializer, i64 0
+  %e.1 = extractelement <4 x i64> %shuffle.1, i64 1
+  %shift = shufflevector <4 x i64> %shuffle.1, <4 x i64> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+  %or = or i64 %e.1, %e.0
+  ret i64 %or
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll b/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll
index cd2bc757eb9d2..5358e0419e7a7 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll
@@ -48,13 +48,13 @@ define <4 x float> @ext2_v4f32(<4 x float> %x, <4 x float> %y) {
 define <4 x float> @ext2_v2f32v4f32(<2 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @ext2_v2f32v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[X:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
-  %e = extractelement <2 x float> %x, i32 2
+  %e = extractelement <2 x float> %x, i32 1
   %n = fneg float %e
-  %r = insertelement <4 x float> %y, float %n, i32 2
+  %r = insertelement <4 x float> %y, float %n, i32 1
   ret <4 x float> %r
 }
 
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
index d46c8c0de4037..b26e5ec2698a5 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
@@ -6,8 +6,7 @@
 define void @multiple_extract(ptr %p) {
 ; CHECK-LABEL: @multiple_extract(
 ; CHECK-NEXT:    [[VP:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i32>, ptr [[VP]], i32 0, i64 0
-; CHECK-NEXT:    [[E0:%.*]] = load i32, ptr [[TMP1]], align 16
+; CHECK-NEXT:    [[E0:%.*]] = load i32, ptr [[VP]], align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <2 x i32>, ptr [[VP]], i32 0, i64 1
 ; CHECK-NEXT:    [[E1:%.*]] = load i32, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    store i32 [[E0]], ptr [[P]], align 4
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
index 40437ca345224..977da754ec5a7 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
@@ -657,11 +657,10 @@ define <2 x float> @load_f32_insert_v2f32_msan(ptr align 16 dereferenceable(16)
 ; PR30986 - split vector loads for scalarized operations
 define <2 x i64> @PR30986(ptr %0) {
 ; CHECK-LABEL: @PR30986(
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2:%.*]], align 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP2]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[TMP8]], i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
index 84c223be88621..388b655641b7d 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -553,8 +553,7 @@ define <4 x float> @load_v2f32_extract_insert_v4f32(ptr align 16 dereferenceable
 
 define <4 x float> @load_v8f32_extract_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; SSE2-LABEL: @load_v8f32_extract_insert_v4f32(
-; SSE2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <8 x float>, ptr [[P:%.*]], i32 0, i32 0
-; SSE2-NEXT:    [[S:%.*]] = load float, ptr [[TMP1]], align 4
+; SSE2-NEXT:    [[S:%.*]] = load float, ptr [[TMP1:%.*]], align 4
 ; SSE2-NEXT:    [[R:%.*]] = insertelement <4 x float> undef, float [[S]], i32 0
 ; SSE2-NEXT:    ret <4 x float> [[R]]
 ;
@@ -590,8 +589,7 @@ define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(ptr align 16 derefer
 define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <2 x i16>, ptr [[P:%.*]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i16>, ptr [[GEP]], i32 0, i32 0
-; CHECK-NEXT:    [[S:%.*]] = load i16, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[S:%.*]] = load i16, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <8 x i16> undef, i16 [[S]], i64 0
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
@@ -605,11 +603,10 @@ define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceab
 ; PR30986 - split vector loads for scalarized operations
 define <2 x i64> @PR30986(ptr %0) {
 ; CHECK-LABEL: @PR30986(
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2:%.*]], align 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP2]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[TMP8]], i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/pr114901.ll b/llvm/test/Transforms/VectorCombine/X86/pr114901.ll
index d6917e1007cf4..982944b8b7bd2 100644
--- a/llvm/test/Transforms/VectorCombine/X86/pr114901.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/pr114901.ll
@@ -17,8 +17,7 @@ define i1 @PR114901(<4 x i32> %a) {
 ; AVX-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; AVX-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A]], <i32 poison, i32 -8, i32 poison, i32 42>
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP2:%.*]] = ashr <4 x i1> [[SHIFT]], [[TMP1]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[SHIFT]], i64 1
 ; AVX-NEXT:    ret i1 [[R]]
 ;
   %e1 = extractelement <4 x i32> %a, i32 1
@@ -43,8 +42,7 @@ define i1 @PR114901_flip(<4 x i32> %a) {
 ; AVX-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
 ; AVX-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A]], <i32 poison, i32 -8, i32 poison, i32 42>
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP2:%.*]] = ashr <4 x i1> [[TMP1]], [[SHIFT]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
 ; AVX-NEXT:    ret i1 [[R]]
 ;
   %e1 = extractelement <4 x i32> %a, i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
index 2588f9116f322..0c995bde9f25a 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
@@ -34,18 +34,12 @@ define <8 x i16> @src_v4tov8_i16(<4 x i1> %a, <4 x i1> %b, <4 x i16> %x, <4 x i1
 define <8 x i16> @src_v8tov8_i16(<8 x i1> %a, <8 x i1> %b, <8 x i16> %x, <8 x i16> %y, <8 x i16> %z) {
 ; SSE-LABEL: define <8 x i16> @src_v8tov8_i16(
 ; SSE-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i1> [[A]], <8 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X]], <8 x i16> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[Z]], <8 x i16> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[TMP2]], <8 x i16> [[TMP3]]
+; SSE-NEXT:    [[RES:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
 ; SSE-NEXT:    ret <8 x i16> [[RES]]
 ;
 ; AVX2-LABEL: define <8 x i16> @src_v8tov8_i16(
 ; AVX2-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i1> [[A]], <8 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X]], <8 x i16> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[Z]], <8 x i16> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[TMP2]], <8 x i16> [[TMP3]]
+; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
 ; AVX2-NEXT:    ret <8 x i16> [[RES]]
 ;
 ; AVX512-LABEL: define <8 x i16> @src_v8tov8_i16(
diff --git a/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll b/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
index 58b7f8de004d0..9e43a28bf1e59 100644
--- a/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
+++ b/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
@@ -152,12 +152,12 @@ define <vscale x 4 x float> @fma_scalable(float %x, float %y, float %z) {
   ret <vscale x 4 x float> %v
 }
 
-; TODO: We should be able to scalarize this if we preserve the scalar argument.
 define <4 x float> @scalar_argument(float %x) {
 ; CHECK-LABEL: define <4 x float> @scalar_argument(
 ; CHECK-SAME: float [[X:%.*]]) {
-; CHECK-NEXT:    [[X_INSERT:%.*]] = insertelement <4 x float> poison, float [[X]], i32 0
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[X_INSERT]], i32 42)
+; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.powi.f32.i32(float [[X]], i32 42)
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> poison, i32 42)
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[V]]
 ;
   %x.insert = insertelement <4 x float> poison, float %x, i32 0
diff --git a/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll b/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll
index 190f4b240537e..3030cc02f8689 100644
--- a/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll
+++ b/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll
@@ -4,12 +4,21 @@
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-percentile-cutoff-hot=0 -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-percentile-cutoff-hot=1000000 -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=REMOVE
 
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[7]=0;runtime_check=1000000>)' -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=MIXED
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[7]=1000000;runtime_check=0>)' -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=MIXED2
+
 ; CHECK: remark: <unknown>:0:0: Allowed check: Kind=test_check F=test_runtime BB=entry1
 ; CHECK: remark: <unknown>:0:0: Allowed check: Kind=7 F=test_ubsan BB=entry2
 
 ; REMOVE: remark: <unknown>:0:0: Removed check: Kind=test_check F=test_runtime BB=entry1
 ; REMOVE: remark: <unknown>:0:0: Removed check: Kind=7 F=test_ubsan BB=entry2
 
+; MIXED: remark: <unknown>:0:0: Removed check: Kind=test_check F=test_runtime BB=entry1
+; MIXED: remark: <unknown>:0:0: Allowed check: Kind=7 F=test_ubsan BB=entry2
+
+; MIXED2: remark: <unknown>:0:0: Allowed check: Kind=test_check F=test_runtime BB=entry1
+; MIXED2: remark: <unknown>:0:0: Removed check: Kind=7 F=test_ubsan BB=entry2
+
 target triple = "x86_64-pc-linux-gnu"
 
 define i1 @test_runtime() local_unnamed_addr {
diff --git a/llvm/test/Verifier/branch-weight.ll b/llvm/test/Verifier/branch-weight.ll
index e3b0f340e31bc..4c87a98359019 100644
--- a/llvm/test/Verifier/branch-weight.ll
+++ b/llvm/test/Verifier/branch-weight.ll
@@ -1,21 +1,65 @@
 ; Test MD_prof validation
 
 ; RUN: split-file %s %t
+
 ; RUN: opt -passes=verify %t/valid.ll --disable-output
-; RUN: not opt -passes=verify %t/invalid1.ll --disable-output 2>&1 | FileCheck %s
-; RUN: not opt -passes=verify %t/invalid2.ll --disable-output 2>&1 | FileCheck %s
+
+; RUN: not opt -passes=verify %t/wrong-count.ll --disable-output 2>&1 | FileCheck %s --check-prefix=WRONG-COUNT
+; RUN: not opt -passes=verify %t/invalid-name1.ll --disable-output 2>&1 | FileCheck %s
+; RUN: not opt -passes=verify %t/invalid-name2.ll --disable-output 2>&1 | FileCheck %s
+
+; RUN: opt -passes=verify %t/unknown-correct.ll --disable-output
+
+; RUN: not opt -passes=verify %t/unknown-invalid.ll --disable-output 2>&1 | FileCheck %s --check-prefix=EXTRA-ARGS
+; RUN: not opt -passes=verify %t/unknown-on-function1.ll --disable-output 2>&1 | FileCheck %s --check-prefix=ON-FUNCTION1
+; RUN: not opt -passes=verify %t/unknown-on-function2.ll --disable-output 2>&1 | FileCheck %s --check-prefix=ON-FUNCTION2
+; RUN: not opt -passes=verify %t/invalid-unknown-placement.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-UNKNOWN-PLACEMENT
 
 ;--- valid.ll
-define void @test(i1 %0) {
-  br i1 %0, label %2, label %3, !prof !0
-2:
+declare void @to_invoke()
+declare i32 @__gxx_personality_v0(...)
+
+define void @invoker() personality ptr @__gxx_personality_v0 {
+  invoke void @to_invoke() to label %exit unwind label %lpad, !prof !0
+lpad:
+  %ll = landingpad {ptr, i32}
+  cleanup
   ret void
-3:
+exit:
   ret void
 }
+
+define i32 @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %exit, !prof !0
+yes:
+  switch i32 %a, label %exit [ i32 1, label %case_b
+                               i32 2, label %case_c], !prof !1
+case_b:
+  br label %exit
+case_c:
+  br label %exit
+exit:
+  %r = select i1 %c, i32 1, i32 2, !prof !0
+  ret i32 %r
+}
 !0 = !{!"branch_weights", i32 1, i32 2}
+!1 = !{!"branch_weights", i32 1, i32 2, i32 3}
+
+;--- wrong-count.ll
+define void @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %no, !prof !0
+yes:
+  ret void
+no:
+  ret void
+}
+!0 = !{!"branch_weights", i32 1, i32 2, i32 3}
 
-;--- invalid1.ll
+; WRONG-COUNT: Wrong number of operands
+
+;--- invalid-name1.ll
 define void @test(i1 %0) {
   br i1 %0, label %2, label %3, !prof !0
 2:
@@ -25,7 +69,7 @@ define void @test(i1 %0) {
 }
 !0 = !{!"invalid", i32 1, i32 2}
 
-;--- invalid2.ll
+;--- invalid-name2.ll
 define void @test(i1 %0) {
   br i1 %0, label %2, label %3, !prof !0
 2:
@@ -37,3 +81,71 @@ define void @test(i1 %0) {
 !0 = !{!"function_entry_count", i32 1}
 
 ; CHECK: expected either branch_weights or VP profile name
+
+;--- unknown-correct.ll
+declare void @to_invoke()
+declare i32 @__gxx_personality_v0(...)
+
+define void @invoker() personality ptr @__gxx_personality_v0 {
+  invoke void @to_invoke() to label %exit unwind label %lpad, !prof !0
+lpad:
+  %ll = landingpad {ptr, i32}
+  cleanup
+  ret void
+exit:
+  ret void
+}
+
+define i32 @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %exit, !prof !0
+yes:
+  switch i32 %a, label %exit [ i32 1, label %case_b
+                               i32 2, label %case_c], !prof !0
+case_b:
+  br label %exit
+case_c:
+  br label %exit
+exit:
+  %r = select i1 %c, i32 1, i32 2, !prof !0
+  ret i32 %r
+}
+
+!0 = !{!"unknown"}
+
+;--- unknown-invalid.ll
+define void @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %no, !prof !0
+yes:
+  ret void
+no:
+  ret void
+}
+
+!0 = !{!"unknown", i32 12, i32 67}
+; EXTRA-ARGS: 'unknown' !prof should have no additional operands
+
+;--- unknown-on-function1.ll
+define void @test() !prof !0 {
+  ret void
+}
+
+!0 = !{!"unknown"}
+; ON-FUNCTION1: 'unknown' !prof metadata should appear only on instructions supporting the 'branch_weights' metadata
+
+;--- unknown-on-function2.ll
+define void @test() !prof !0 {
+  ret void
+}
+
+!0 = !{!"unknown", i64 123}
+; ON-FUNCTION2: first operand should be 'function_entry_count' or 'synthetic_function_entry_count'
+
+;--- invalid-unknown-placement.ll
+define i32 @test() {
+  %r = add i32 1, 2, !prof !0
+  ret i32 %r
+}
+!0 = !{!"unknown"}
+; INVALID-UNKNOWN-PLACEMENT: 'unknown' !prof should only appear on instructions on which 'branch_weights' would
diff --git a/llvm/test/Verifier/dead-on-return.ll b/llvm/test/Verifier/dead-on-return.ll
new file mode 100644
index 0000000000000..eb1d67fa319c2
--- /dev/null
+++ b/llvm/test/Verifier/dead-on-return.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: Attribute 'dead_on_return' applied to incompatible type!
+; CHECK-NEXT: ptr @arg_not_pointer
+define void @arg_not_pointer(i32 dead_on_return %arg) {
+  ret void
+}
diff --git a/llvm/test/Verifier/invalid-vp-intrinsics.ll b/llvm/test/Verifier/invalid-vp-intrinsics.ll
index 08639352c3ea9..9923867757e7a 100644
--- a/llvm/test/Verifier/invalid-vp-intrinsics.ll
+++ b/llvm/test/Verifier/invalid-vp-intrinsics.ll
@@ -33,3 +33,36 @@ define void @test_vp_icmp(<4 x i32> %a, <4 x i32> %b, <4 x i1> %m, i32 %n) {
   %r1 = call <4 x i1> @llvm.vp.icmp.v4i32(<4 x i32> %a, <4 x i32> %b, metadata !"oeq", <4 x i1> %m, i32 %n)
   ret void
 }
+
+; CHECK: The splice index exceeds the range [-VL, VL-1] where VL is the known minimum number of elements in the vector
+define <2 x double> @splice_v2f64_idx_neg3(<2 x double> %a, <2 x double> %b, i32 %evl1, i32 %evl2) #0 {
+  %res = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %a, <2 x double> %b, i32 -3, <2 x i1> splat (i1 1), i32 %evl1, i32 %evl2)
+  ret <2 x double> %res
+}
+
+; CHECK: The splice index exceeds the range [-VL, VL-1] where VL is the known minimum number of elements in the vector
+define <vscale x 2 x double> @splice_nxv2f64_idx_neg3_vscale_min1(<vscale x 2 x double> %a, <vscale x 2 x double> %b, i32 %evl1, i32 %evl2) #0 {
+  %res = call <vscale x 2 x double> @llvm.experimental.vp.splice.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, i32 -3, <vscale x 2 x i1> splat (i1 1), i32 %evl1, i32 %evl2)
+  ret <vscale x 2 x double> %res
+}
+
+; CHECK: The splice index exceeds the range [-VL, VL-1] where VL is the known minimum number of elements in the vector
+define <vscale x 2 x double> @splice_nxv2f64_idx_neg5_vscale_min2(<vscale x 2 x double> %a, <vscale x 2 x double> %b, i32 %evl1, i32 %evl2) #1 {
+  %res = call <vscale x 2 x double> @llvm.experimental.vp.splice.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evl1, i32 %evl2)
+  ret <vscale x 2 x double> %res
+}
+
+; CHECK: The splice index exceeds the range [-VL, VL-1] where VL is the known minimum number of elements in the vector
+define <2 x double> @splice_v2f64_idx2(<2 x double> %a, <2 x double> %b, i32 %evl1, i32 %evl2) #0 {
+  %res = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %a, <2 x double> %b, i32 2, <2 x i1> splat (i1 1), i32 %evl1, i32 %evl2)
+  ret <2 x double> %res
+}
+
+; CHECK: The splice index exceeds the range [-VL, VL-1] where VL is the known minimum number of elements in the vector
+define <2 x double> @splice_v2f64_idx3(<2 x double> %a, <2 x double> %b, i32 %evl1, i32 %evl2) #1 {
+  %res = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %a, <2 x double> %b, i32 4, <2 x i1> splat (i1 1), i32 %evl1, i32 %evl2)
+  ret <2 x double> %res
+}
+
+attributes #0 = { vscale_range(1,16) }
+attributes #1 = { vscale_range(2,16) }
diff --git a/llvm/test/Verifier/value-profile.ll b/llvm/test/Verifier/value-profile.ll
new file mode 100644
index 0000000000000..f38bc7728a703
--- /dev/null
+++ b/llvm/test/Verifier/value-profile.ll
@@ -0,0 +1,38 @@
+; Test MD_prof "VP" validation
+
+; RUN: split-file %s %t
+; RUN: opt -passes=verify %t/valid.ll --disable-output
+; RUN: not opt -passes=verify %t/invalid-kind.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-KIND
+; RUN: not opt -passes=verify %t/invalid-count.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-COUNT
+; RUN: not opt -passes=verify %t/invalid-place.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-PLACE
+
+;--- valid.ll
+define void @test(ptr %0) {
+  call void %0(), !prof !0
+  ret void
+}
+!0 = !{!"VP", i32 0, i32 20, i64 1234, i64 10, i64 5678, i64 5}
+
+;--- invalid-kind.ll
+define void @test(ptr %0) {
+  call void %0(), !prof !0
+  ret void
+}
+!0 = !{!"VP", i32 3, i32 20, i64 1234, i64 10, i64 5678, i64 5}
+; INVALID-KIND: Invalid VP !prof kind
+
+;--- invalid-count.ll
+define void @test(ptr %0) {
+  call void %0(), !prof !0
+  ret void
+}
+!0 = !{!"VP", i32 1, i64 1234, i64 10, i64 5678, i64 5}
+; INVALID-COUNT: VP !prof should have an even number of arguments after 'VP'
+
+;--- invalid-place.ll
+define i32 @test(i32 %0) {
+  %r = add i32 %0, 1, !prof !0
+  ret i32 %r
+}
+!0 = !{!"VP", i32 1, i32 20, i64 1234, i64 10, i64 5678, i64 5}
+; INVALID-PLACE: VP !prof indirect call or memop size expected to be applied to CallBase instructions only
diff --git a/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c b/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c
index d1ec7e70d443c..c66f760bf4cab 100644
--- a/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c
+++ b/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c
@@ -1,4 +1,4 @@
-// REQUIRES : system-darwin
+// REQUIRES: system-darwin
 // RUN: dsymutil -oso-prepend-path %p/.. -dump-debug-map \
 // RUN: %p/../Inputs/global_downgraded_to_static.x86_64 2>&1 | FileCheck %s
 //
diff --git a/llvm/test/tools/llc/new-pm/pipeline.mir b/llvm/test/tools/llc/new-pm/pipeline.mir
index 761a3a424ee67..1113f2a26c449 100644
--- a/llvm/test/tools/llc/new-pm/pipeline.mir
+++ b/llvm/test/tools/llc/new-pm/pipeline.mir
@@ -1,10 +1,18 @@
 # RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir -passes=no-op-machine-function --print-pipeline-passes -filetype=null < %s | FileCheck %s --match-full-lines
 # RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir -passes='require<machine-dom-tree>,print<machine-dom-tree>' -print-pipeline-passes < %s | FileCheck --check-prefix=ANALYSIS %s
 
+# Check same nested pass managers
+# RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir \
+# RUN:     -passes='function(machine-function(machine-function(machine-cp)))' \
+# RUN:     -print-pipeline-passes < %s | FileCheck %s --check-prefix=NESTED
+
+# RUN: not llc -mtriple=x86_64-pc-linux-gnu -x mir -passes='machine-function(whoops(verify))' -print-pipeline-passes < %s 2>&1
+
 # CHECK: function(machine-function(no-op-machine-function)),PrintMIRPreparePass,function(machine-function(verify,print))
 
 # ANALYSIS: require<machine-dom-tree>,print<machine-dom-tree>
 
+# NESTED: function(machine-function(machine-cp))
 ---
 name: f
 body: |
diff --git a/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir b/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir
index f1a30bc7964ca..7a50bcaf212c7 100644
--- a/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir
+++ b/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir
@@ -1,4 +1,4 @@
-# REQUIRES x86_64-registered-target
+# REQUIRES: x86-registered-target
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=fast -print-pipeline-passes %s -o - 2>&1 | FileCheck %s
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=greedy -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-GREEDY
 
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
index 3887b43c6eda4..5f9122d01da9a 100644
--- a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
+++ b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
@@ -33,4 +33,4 @@ CHECK-SAME:     "mcdc":{"count":0,"covered":0,"notcovered":0,"percent":0},
 CHECK-SAME:     "regions":{"count":1,"covered":1,"notcovered":0,"percent":100}}}
 CHECK-SAME: ],
 CHECK-SAME: "type":"llvm.coverage.json.export"
-CHECK-SAME: "version":"3.0.0"
+CHECK-SAME: "version":"3.0.1"
diff --git a/llvm/test/tools/llvm-cov/mcdc-export-json.test b/llvm/test/tools/llvm-cov/mcdc-export-json.test
index 3f9201d338cf1..e6dbd17bee5b2 100644
--- a/llvm/test/tools/llvm-cov/mcdc-export-json.test
+++ b/llvm/test/tools/llvm-cov/mcdc-export-json.test
@@ -1,10 +1,10 @@
 // RUN: llvm-profdata merge %S/Inputs/mcdc-general.proftext -o %t.profdata
 // RUN: llvm-cov export --format=text %S/Inputs/mcdc-general.o -instr-profile %t.profdata | FileCheck %s
 
-// CHECK: 12,7,12,27,2,4,0,5,[true,true,true,true]
-// CHECK: 15,7,15,13,1,2,0,5,[true,true]
-// CHECK: 15,19,15,25,1,1,0,5,[true,false]
-// CHECK: 18,7,19,15,1,3,0,5,[true,true,false,true]
+// CHECK: 12,7,12,27,2,4,0,0,5,[true,true,true,true]
+// CHECK: 15,7,15,13,1,2,0,0,5,[true,true]
+// CHECK: 15,19,15,25,1,1,0,0,5,[true,false]
+// CHECK: 18,7,19,15,1,3,0,0,5,[true,true,false,true]
 // CHECK: "mcdc":{"count":12,"covered":10,"notcovered":2,"percent":83.333333333333343}
 
 Instructions for regenerating the test:
diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll b/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll
index ec16137cee1d4..9a2d0a52d9757 100644
--- a/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll
+++ b/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll
@@ -31,7 +31,7 @@ entry:
   ret i32 0, !dbg !31
 }
 
-attributes #0 = { mustprogress noinline norecurse nounwind optnone ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress noinline norecurse nounwind optnone ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.module.flags = !{!18, !19, !20, !21, !22, !23, !24}
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test
index 64c18669264ef..6023b99702e20 100644
--- a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test
+++ b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test
@@ -167,9 +167,9 @@ entry:
   ret i32 0, !dbg !60
 }
 
-attributes #0 = { minsize mustprogress nofree noinline norecurse nounwind optsize ssp memory(readwrite, argmem: none) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
-attributes #1 = { minsize mustprogress noinline nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
-attributes #2 = { minsize mustprogress norecurse nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { minsize mustprogress nofree noinline norecurse nounwind optsize ssp memory(readwrite, argmem: none) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+attributes #1 = { minsize mustprogress noinline nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+attributes #2 = { minsize mustprogress norecurse nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 attributes #3 = { minsize optsize }
 attributes #4 = { minsize nounwind optsize }
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s
index 99e39567b1ad6..ef9d4463ebe52 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s
@@ -5066,19 +5066,19 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  2      2     1.00                        movs	p0.b, p0/z, p0.b
 # CHECK-NEXT:  2      2     1.00                        movs	p15.b, p15.b
 # CHECK-NEXT:  2      2     1.00                        movs	p15.b, p15/z, p15.b
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ID_AA64ZFR0_EL1
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL1
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL12
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL2
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL3
 # CHECK-NEXT:  1      4     1.00                        msb	z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  2      5     2.00                        msb	z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        msb	z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        msb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL1, x3
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL12, x3
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL2, x3
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL3, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL1, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL12, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL2, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL3, x3
 # CHECK-NEXT:  1      4     1.00                        mul	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  1      4     1.00                        mul	z0.b, z1.b, z2.b
 # CHECK-NEXT:  2      5     2.00                        mul	z0.d, p7/m, z0.d, z31.d
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
index 5ffaf9138d482..dee46a304582b 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
@@ -1185,10 +1185,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1197,13 +1197,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1215,14 +1215,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [1] Code Region - G02
 
@@ -1231,10 +1231,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1243,13 +1243,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1261,14 +1261,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [2] Code Region - G03
 
@@ -1277,10 +1277,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1289,13 +1289,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1307,14 +1307,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [3] Code Region - G04
 
@@ -1323,10 +1323,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1900
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.74
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1334,14 +1334,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.16b }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1352,15 +1352,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.0    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [4] Code Region - G05
 
@@ -1369,10 +1369,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1380,14 +1380,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4s, v2.4s }, [x27], #32
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1398,15 +1398,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [5] Code Region - G06
 
@@ -1415,10 +1415,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1426,14 +1426,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2d, v2.2d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1444,15 +1444,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [6] Code Region - G07
 
@@ -1461,10 +1461,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.53
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 4.3
+# CHECK-NEXT: Block RThroughput: 4.6
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1472,14 +1472,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8h, v2.8h }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1490,15 +1490,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [7] Code Region - G08
 
@@ -1507,7 +1507,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.92
 # CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -1518,14 +1518,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1536,15 +1536,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [8] Code Region - G09
 
@@ -1553,7 +1553,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.92
 # CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -1564,14 +1564,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1582,26 +1582,26 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [9] Code Region - G10
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      608
+# CHECK-NEXT: Total Cycles:      708
 # CHECK-NEXT: Total uOps:        2700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.44
-# CHECK-NEXT: IPC:               1.64
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.81
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 5.7
 
 # CHECK:      Timeline view:
@@ -1610,14 +1610,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER.  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeER.   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,7]     . D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,9]     . D===eE-----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER   .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,7]     .   DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1628,42 +1628,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.1    2.2       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.2    2.2       <total>
 
 # CHECK:      [10] Code Region - G11
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      675
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.44
-# CHECK-NEXT: IPC:               1.48
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeER.   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D==eE-----R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    . .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER . .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeE-----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .   DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1673,43 +1673,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    2.5       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.5       <total>
 
 # CHECK:      [11] Code Region - G12
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      675
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.44
-# CHECK-NEXT: IPC:               1.48
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eE-----R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    . .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER . .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1719,25 +1719,25 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    2.5       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.5       <total>
 
 # CHECK:      [12] Code Region - G13
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1210
+# CHECK-NEXT: Total Cycles:      1212
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.31
 # CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 5.7
@@ -1746,16 +1746,16 @@ add x0, x27, 1
 # CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789          01
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    ..   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE-----R.    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER    .    ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE-----R    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeeER   .    ..   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eE-----R   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeER .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE-----R .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D========eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,9]     .  D=========eE------R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-----R.    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .    .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER .    .  .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-----R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER    .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-----R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . D======eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,9]     .    . D=======eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1765,16 +1765,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 9.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.3    0.2    2.6       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 9.     1     8.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.4    2.6       <total>
 
 # CHECK:      [13] Code Region - G14
 
@@ -1783,10 +1783,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.50
 # CHECK-NEXT: IPC:               0.25
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
@@ -1794,14 +1794,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1812,15 +1812,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [14] Code Region - G15
 
@@ -1829,10 +1829,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.50
 # CHECK-NEXT: IPC:               0.25
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
@@ -1840,14 +1840,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1858,15 +1858,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [15] Code Region - G16
 
@@ -1875,10 +1875,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.66
 # CHECK-NEXT: IPC:               0.83
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1886,14 +1886,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1	{ v1.d }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1904,15 +1904,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [16] Code Region - G17
 
@@ -1921,10 +1921,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.92
 # CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1932,14 +1932,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.4s }, [x27], #4
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1950,15 +1950,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [17] Code Region - G18
 
@@ -1967,10 +1967,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.92
 # CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1978,14 +1978,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.2d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1996,15 +1996,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [18] Code Region - G19
 
@@ -2013,10 +2013,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.71
 # CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -2024,14 +2024,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.8h }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .  DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .   DeE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2042,42 +2042,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.1    3.0       <total>
 
 # CHECK:      [19] Code Region - G20
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      909
 # CHECK-NEXT: Total uOps:        2900
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.69
-# CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.19
+# CHECK-NEXT: IPC:               1.10
+# CHECK-NEXT: Block RThroughput: 5.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     01234567
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=eeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D==eE------R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    . .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE------R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   . .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     . DeE------R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeeeER . .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .   DeE------R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeeeeeER .   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .  DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2087,43 +2087,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.2    0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.4    3.0       <total>
 
 # CHECK:      [20] Code Region - G21
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      709
 # CHECK-NEXT: Total uOps:        2700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.29
-# CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.81
+# CHECK-NEXT: IPC:               1.41
+# CHECK-NEXT: Block RThroughput: 5.4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.2s, v2.2s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER  .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER .   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .   DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2134,15 +2134,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    3.0       <total>
 
 # CHECK:      [21] Code Region - G22
 
@@ -2151,25 +2151,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      3310
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.79
 # CHECK-NEXT: IPC:               0.30
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     . D=======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     . D=======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     .  D==============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .   D=====================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    D============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2179,16 +2179,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 3.     1     8.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 5.     1     15.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     22.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     29.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.0   0.1    3.0       <total>
 
 # CHECK:      [22] Code Region - G23
 
@@ -2197,7 +2197,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.62
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2208,14 +2208,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2226,15 +2226,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [23] Code Region - G24
 
@@ -2243,7 +2243,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      2603
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.96
 # CHECK-NEXT: IPC:               0.38
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2254,14 +2254,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     D=========eE------R .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D================eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D================eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .D=================eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D================eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D=================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     .D========eE------R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==============eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D===============eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==============eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D===============eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2272,15 +2272,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 7.     1     18.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     18.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.5   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 7.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     15.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     11.3   0.1    3.0       <total>
 
 # CHECK:      [24] Code Region - G25
 
@@ -2289,7 +2289,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.90
 # CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2300,14 +2300,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.2s, v2.2s }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2318,15 +2318,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [25] Code Region - G26
 
@@ -2335,7 +2335,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.90
 # CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2346,14 +2346,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.16b, v2.16b }, [x27], #2
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2364,15 +2364,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [26] Code Region - G27
 
@@ -2381,10 +2381,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      609
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.60
 # CHECK-NEXT: IPC:               1.64
-# CHECK-NEXT: Block RThroughput: 5.5
+# CHECK-NEXT: Block RThroughput: 5.6
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -2392,14 +2392,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     .    DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2410,42 +2410,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.1    3.0       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.87
-# CHECK-NEXT: IPC:               1.32
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2455,43 +2455,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.01
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.77
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2501,43 +2501,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2010
+# CHECK-NEXT: Total Cycles:      2011
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.84
 # CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .   .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .    .    .   .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .    .    .   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE------R .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D========eeeeeeeeER   .   .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .  D=========eE------R   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     .   D================eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789 
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .    .    .    .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.    .    .    .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D======eeeeeeeeER  .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .    . D======eE------R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     .    .   D============eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2547,16 +2547,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     9.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 7.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 9.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.3    0.2    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 7.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     13.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 9.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.6    0.3    3.0       <total>
 
 # CHECK:      [30] Code Region - G31
 
@@ -2565,7 +2565,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.87
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 7.5
@@ -2575,15 +2575,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2593,16 +2593,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [31] Code Region - G32
 
@@ -2611,7 +2611,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.87
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 7.5
@@ -2621,15 +2621,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2639,43 +2639,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.87
-# CHECK-NEXT: IPC:               1.32
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2685,43 +2685,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.01
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.77
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2731,43 +2731,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.87
-# CHECK-NEXT: IPC:               1.32
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2777,43 +2777,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      959
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        4600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.80
-# CHECK-NEXT: IPC:               1.04
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.55
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .D=eE-------R  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .  .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     . D==eE------R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===eeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D====eE-------R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeE-------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     .    DeE------R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .   DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2823,43 +2823,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 9.     1     5.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.4    3.2       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.2       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1008
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.76
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789         
 
 # CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .D=eE-------R  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeeER  .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     . D===eE-------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeeER .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===eE-------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeE-------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeER  .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeE-------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeER.   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2869,43 +2869,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 5.     1     4.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.3       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.3       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.75
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789          
 
 # CHECK:      [0,0]     DeeeeeeeeER    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE-------R  .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE------R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeeER  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===eE-------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====eE-------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeER .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2915,16 +2915,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.3       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.3       <total>
 
 # CHECK:      [38] Code Region - G39
 
@@ -2933,7 +2933,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.12
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -2943,15 +2943,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2961,16 +2961,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [39] Code Region - G40
 
@@ -2979,7 +2979,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.12
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -2989,15 +2989,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3007,16 +3007,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [40] Code Region - G41
 
@@ -3025,7 +3025,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        4600
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.19
 # CHECK-NEXT: IPC:               0.48
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -3035,15 +3035,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     D=eE------R    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D========eE------R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=======eeeeeeeeER.  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     . D========eE------R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=========eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==========eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=========eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D==========eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=====eeeeeeeeER.  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    D=====eE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D======eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D======eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=====eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .   D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3053,43 +3053,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 5.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     10.0   2.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 7.     1     11.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: 9.     1     11.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.9    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     6.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 5.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     7.0    2.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 7.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     6.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     5.4    0.3    3.0       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1008
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.76
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789         
 
 # CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=eE------R   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,5]     . D===eE------R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3099,43 +3099,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1008
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4700
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.66
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789         
 
 # CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE------R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3145,43 +3145,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      708
+# CHECK-NEXT: Total Cycles:      808
 # CHECK-NEXT: Total uOps:        3900
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.51
-# CHECK-NEXT: IPC:               1.41
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.83
+# CHECK-NEXT: IPC:               1.24
+# CHECK-NEXT: Block RThroughput: 7.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE------R   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeE-R   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .  D===eE-----R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===eeeeeeER   ldp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     .  D====eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789      
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER.   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .    .D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeeER   ldp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .    . D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3191,43 +3191,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    1.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.9    0.3    2.8       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.4    2.6       <total>
 
 # CHECK:      [44] Code Region - G45
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.52
-# CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: Block RThroughput: 5.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER ..   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D=eE----R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER..   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .D=eE----R..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER.   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeeeER   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     . D==eE----R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeE-R   ldp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .  D==eE---R   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeeeeeER . .   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     .DeE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeER.   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .    DeE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeE-R   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .    .DeE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3237,16 +3237,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ldp	s1, s2, [x27, #248]!
 # CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    1.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     3.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.2    0.1    2.0       <total>
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    1.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    2.0       <total>
 
 # CHECK:      [45] Code Region - G46
 
@@ -3255,7 +3255,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.99
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -3266,14 +3266,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeER   .    .   ldp	x1, x2, [x27], #496
 # CHECK-NEXT: [0,1]     D=eE--R   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .    .   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eE--R  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .    .   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eE--R .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeER    .   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,7]     .D===eE---R    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeeeeER   ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: [0,9]     . D=======eE---R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .    .   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eE--R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .    .   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D=eE--R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER    .   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,7]     .  D=eE---R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,9]     .   D=====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3284,15 +3284,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    4.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: 9.     1     8.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.5    0.5    1.2       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    4.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: 9.     1     6.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.5    1.2       <total>
 
 # CHECK:      [46] Code Region - G47
 
@@ -3301,25 +3301,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789   
 
 # CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ldr	s1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ldr	d1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ldr	q1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ldr	h1, [x27], #254
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ldr	s1, [x27], #254
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ldr	d1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ldr	q1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3330,15 +3330,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	s1, [x27], #254
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldr	q1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldr	h1, [x27], #254
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldr	s1, [x27], #254
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldr	d1, [x27], #254
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldr	q1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [47] Code Region - G48
 
@@ -3347,25 +3347,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789   
 
 # CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ldr	q1, [x27, #254]!
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3376,15 +3376,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -3393,25 +3393,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldr	w1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER  .   ldr	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldr	w1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   ldr	x1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeER   ldrb	w1, [x27], #254
-# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   ldr	w1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   ldr	x1, [x27, #254]!
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   ldrb	w1, [x27], #254
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3423,14 +3423,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	w1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	x1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	w1, [x27, #254]!
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	x1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldrb	w1, [x27], #254
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	w1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldr	x1, [x27, #254]!
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldrb	w1, [x27], #254
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [49] Code Region - G50
 
@@ -3439,25 +3439,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldrb	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER  .   ldrh	w1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrh	w1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   ldrsb	w1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeER   ldrsb	x1, [x27], #254
-# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   ldrh	w1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   ldrsb	w1, [x27], #254
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   ldrsb	x1, [x27], #254
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3469,14 +3469,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrb	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrh	w1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrh	w1, [x27, #254]!
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldrsb	w1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldrsb	x1, [x27], #254
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldrh	w1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldrsb	w1, [x27], #254
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldrsb	x1, [x27], #254
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [50] Code Region - G51
 
@@ -3485,25 +3485,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldrsb	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER  .   ldrsb	x1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrsh	w1, [x27], #254
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   ldrsh	x1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeER   ldrsh	w1, [x27, #254]!
-# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   ldrsh	w1, [x27], #254
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   ldrsh	x1, [x27], #254
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   ldrsh	w1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3515,14 +3515,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsb	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrsb	x1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsh	w1, [x27], #254
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldrsh	x1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldrsh	w1, [x27, #254]!
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldrsh	w1, [x27], #254
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldrsh	x1, [x27], #254
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldrsh	w1, [x27, #254]!
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [51] Code Region - G52
 
@@ -3531,10 +3531,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.37
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
@@ -3542,13 +3542,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE--R .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER.   ldrsw	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeE-R   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===eE-R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeE-R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,7]     . D==eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .  D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3560,14 +3560,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsh	x1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrsw	x1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    1.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    0.8       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.2    0.1    0.8       <total>
 
 # CHECK:      [52] Code Region - G53
 
@@ -3576,24 +3576,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.2s }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3604,15 +3604,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
@@ -3621,24 +3621,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.16b }, [x27], #16
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3649,15 +3649,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
@@ -3666,24 +3666,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.4s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3694,15 +3694,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
@@ -3711,24 +3711,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.76
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.5
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,7]     .   DeER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3738,42 +3738,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.1    0.0       <total>
 
 # CHECK:      [56] Code Region - G57
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      504
+# CHECK-NEXT: Total Cycles:      604
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.16
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.30
+# CHECK-NEXT: IPC:               1.66
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.  .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeER.   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,5]     .  D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    DeER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3783,42 +3783,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    0.0       <total>
 
 # CHECK:      [57] Code Region - G58
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      504
+# CHECK-NEXT: Total Cycles:      604
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.16
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.30
+# CHECK-NEXT: IPC:               1.66
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     . DeER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER .   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .   DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3829,41 +3829,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    0.0       <total>
 
 # CHECK:      [58] Code Region - G59
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.84
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.39
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3873,42 +3874,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.12
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.59
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3918,42 +3920,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [60] Code Region - G61
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.84
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.39
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  DeeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3963,43 +3966,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.7    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [61] Code Region - G62
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.11
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.59
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.    .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER    .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeER  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,5]     . D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4009,43 +4012,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      804
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        4200
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.22
-# CHECK-NEXT: IPC:               1.24
-# CHECK-NEXT: Block RThroughput: 8.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.19
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 8.4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.    ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     D=eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER    ..   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeER  ..   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,5]     . D==eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER ..   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D==eER ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4055,42 +4058,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.41
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 7.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.79
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4100,16 +4104,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.1    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [64] Code Region - G65
 
@@ -4118,25 +4122,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.53
 # CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 5.5
+# CHECK-NEXT: Block RThroughput: 6.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789   
 
 # CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER    . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeER .   st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,5]     . D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D===eeeeER.   st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,7]     . D====eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===eeeeER   st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .  D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeER .   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,5]     .   D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeER.   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,7]     .    D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .DeeeeER   st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4146,16 +4150,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.3    0.6       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.3    0.6       <total>
 
 # CHECK:      [65] Code Region - G66
 
@@ -4164,25 +4168,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st1	{ v1.b }[8], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4193,15 +4197,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.0       <total>
 
 # CHECK:      [66] Code Region - G67
 
@@ -4210,25 +4214,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.35
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st1	{ v1.s }[0], [x27], #4
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,9]     .    DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4239,15 +4243,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.1    1.0       <total>
 
 # CHECK:      [67] Code Region - G68
 
@@ -4256,25 +4260,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.74
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.5
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.2s, v2.2s }, [x27], #16
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,5]     .  DeE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,9]     .    DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4285,42 +4289,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.1    1.0       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total Cycles:      705
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.14
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.69
+# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789  
+
+# CHECK:      [0,0]     DeeeeER   ..   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeER ..   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE--R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeER..   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .  D=eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .   D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4330,43 +4334,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    1.0       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total Cycles:      606
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.74
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.96
+# CHECK-NEXT: IPC:               1.65
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789  
+
+# CHECK:      [0,0]     DeeeeER   ..   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  ..   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     . DeE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeER..   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .   DeE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .   D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,9]     .    D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4377,15 +4381,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.2    1.0       <total>
 
 # CHECK:      [70] Code Region - G71
 
@@ -4394,25 +4398,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4423,15 +4427,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.0       <total>
 
 # CHECK:      [71] Code Region - G72
 
@@ -4440,25 +4444,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4469,42 +4473,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.0       <total>
 
 # CHECK:      [72] Code Region - G73
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      607
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.61
-# CHECK-NEXT: IPC:               1.65
-# CHECK-NEXT: Block RThroughput: 4.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.96
+# CHECK-NEXT: IPC:               1.41
+# CHECK-NEXT: Block RThroughput: 5.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeER .    . .   st2g	x26, [x27], #4064
-# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.    . .   st2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,3]     D==eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,7]     . D==eE---R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D==eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,9]     .  D===eE---R   add	x0, x27, #1
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789    
+
+# CHECK:      [0,0]     DeER .    .  .   st2g	x26, [x27], #4064
+# CHECK-NEXT: [0,1]     D=eER.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.    .  .   st2g	x26, [x27, #4064]!
+# CHECK-NEXT: [0,3]     .D=eER    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER  .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,5]     .  DeE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,7]     .    DeE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .DeeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,9]     .    . DeE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4515,42 +4519,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2g	x26, [x27], #4064
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2g	x26, [x27, #4064]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.2    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2g	x26, [x27, #4064]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 9.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.3    1.0       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      708
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.37
-# CHECK-NEXT: IPC:               1.41
-# CHECK-NEXT: Block RThroughput: 7.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.77
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeER .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE---R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeER.    ..   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE---R.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4560,43 +4564,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.9       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      707
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.81
-# CHECK-NEXT: IPC:               1.41
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.38
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeER  .  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeER .  .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE---R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeER  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  DeeeeeER  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=eE---R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeER  .    ..   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE---R  .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeER.    ..   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE---R.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeER ..   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4606,43 +4610,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.7    0.2    1.7       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.7       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.28
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4652,43 +4656,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.0       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.28
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4698,43 +4702,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.0       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      807
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4200
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.20
-# CHECK-NEXT: IPC:               1.24
-# CHECK-NEXT: Block RThroughput: 8.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.17
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeER  .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D==eE---R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeER ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,7]     .    . DeE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4744,43 +4748,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.9       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1205
+# CHECK-NEXT: Total Cycles:      1307
 # CHECK-NEXT: Total uOps:        5800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.81
-# CHECK-NEXT: IPC:               0.83
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.44
+# CHECK-NEXT: IPC:               0.77
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE----R .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER    ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     . DeE-----R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  DeeeeeeER   ..   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,5]     .  D=eE----R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D==eeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,7]     .    D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D=eeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,9]     .    . D=eE-----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          
+
+# CHECK:      [0,0]     DeeeeeeER .    .   .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE----R .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .   .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     .   DeE----R   .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeER .   .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .DeE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D=eeeeeeeER .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,7]     .    .   DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .    . DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4790,43 +4794,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    1.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.8    0.4    2.3       <total>
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.1    0.8    2.1       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1006
+# CHECK-NEXT: Total Cycles:      1107
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.77
-# CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 9.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.34
+# CHECK-NEXT: IPC:               0.90
+# CHECK-NEXT: Block RThroughput: 9.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeER  .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE---R  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.    .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE----R.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeER  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeER.   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .   D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    D==eeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789        
+
+# CHECK:      [0,0]     DeeeeeER  .    . .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE---R  .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    . .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  . .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .  DeE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   DeeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4836,43 +4840,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.4    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.6    1.9       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1058
+# CHECK-NEXT: Total Cycles:      1207
 # CHECK-NEXT: Total uOps:        5200
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.91
-# CHECK-NEXT: IPC:               0.95
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.31
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 10.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeER.    . .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeeeeER. .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .   D===eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D===eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,7]     .    D====eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D===eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeER.    .  .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeE----R.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeeeeeeER  .  .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DeE----R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeeeeER.  .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .    . DeE----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D=eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,7]     .    .   D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4882,43 +4886,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    1.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.9    0.6    2.2       <total>
+# CHECK-NEXT: 3.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    2.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.7    2.0       <total>
 
 # CHECK:      [81] Code Region - G82
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.28
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4928,43 +4932,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.0       <total>
 
 # CHECK:      [82] Code Region - G83
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      904
 # CHECK-NEXT: Total uOps:        3600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.11
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.98
+# CHECK-NEXT: IPC:               1.11
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     D=eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,5]     . D==eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE--R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eE-R   stg	x26, [x27], #4064
-# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeeeeeER . .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     .DeE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeER .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,5]     .    DeE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeE-R   stg	x26, [x27], #4064
+# CHECK-NEXT: [0,9]     .    .  D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4974,16 +4978,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    1.0       stg	x26, [x27], #4064
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    1.3       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    1.0       stg	x26, [x27], #4064
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.1    0.5    1.3       <total>
 
 # CHECK:      [83] Code Region - G84
 
@@ -4992,24 +4996,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.37
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   stg	x26, [x27, #4064]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   stgp	x1, x2, [x27], #992
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   stgp	x1, x2, [x27, #992]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   stp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   stp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stgp	x1, x2, [x27], #992
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stgp	x1, x2, [x27, #992]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   stp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   stp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5020,15 +5024,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stg	x26, [x27, #4064]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stgp	x1, x2, [x27], #992
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stgp	x1, x2, [x27, #992]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       stp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       stp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stgp	x1, x2, [x27], #992
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stgp	x1, x2, [x27, #992]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       stp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       stp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [84] Code Region - G85
 
@@ -5037,25 +5041,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.55
 # CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeER.    .   stp	q1, q2, [x27], #992
 # CHECK-NEXT: [0,1]     D==eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .   stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D===eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D==eeER  .   stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D===eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===eeER .   stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     . D====eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D====eER.   stp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     . D=====eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=eeER   .   stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D==eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeER  .   stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER .   stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .   D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eER.   stp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5066,15 +5070,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	q1, q2, [x27], #992
 # CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       stp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.8    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       stp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    0.0       <total>
 
 # CHECK:      [85] Code Region - G86
 
@@ -5083,24 +5087,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.37
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   stp	x1, x2, [x27], #496
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   str	b1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   str	h1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   str	b1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   str	h1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5111,15 +5115,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       str	b1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	h1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       str	b1, [x27], #254
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       str	h1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [86] Code Region - G87
 
@@ -5128,24 +5132,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   str	s1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   str	q1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   str	b1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   str	h1, [x27, #254]!
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   str	d1, [x27], #254
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   str	q1, [x27], #254
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   str	b1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   str	h1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5156,15 +5160,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	q1, [x27], #254
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       str	b1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	h1, [x27, #254]!
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	d1, [x27], #254
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       str	q1, [x27], #254
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       str	b1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       str	h1, [x27, #254]!
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [87] Code Region - G88
 
@@ -5173,24 +5177,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.56
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 4.6
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   str	s1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   str	q1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eER .   str	w1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eER.   str	x1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   str	d1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   str	q1, [x27, #254]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeER .   str	w1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeER.   str	x1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5201,15 +5205,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	q1, [x27, #254]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       str	w1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	x1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	d1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       str	q1, [x27, #254]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       str	w1, [x27], #254
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       str	x1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [88] Code Region - G89
 
@@ -5218,24 +5222,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   str	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   str	x1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   strb	w1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eER .   strb	w1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eER.   strh	w1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   str	x1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   strb	w1, [x27], #254
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeER .   strb	w1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeER.   strh	w1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5246,15 +5250,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	x1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       strb	w1, [x27], #254
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       strb	w1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       strh	w1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	x1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       strb	w1, [x27], #254
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       strb	w1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       strh	w1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [89] Code Region - G90
 
@@ -5263,24 +5267,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   strh	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   stz2g	x26, [x27], #4064
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   stz2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eER .   stzg	x26, [x27], #4064
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eER.   stzg	x26, [x27, #4064]!
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stz2g	x26, [x27], #4064
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stz2g	x26, [x27, #4064]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeER .   stzg	x26, [x27], #4064
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeER.   stzg	x26, [x27, #4064]!
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5291,36 +5295,36 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       strh	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stz2g	x26, [x27], #4064
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stz2g	x26, [x27, #4064]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       stzg	x26, [x27], #4064
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       stzg	x26, [x27, #4064]!
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stz2g	x26, [x27], #4064
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stz2g	x26, [x27, #4064]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       stzg	x26, [x27], #4064
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       stzg	x26, [x27, #4064]!
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [90] Code Region - G91
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      400
-# CHECK-NEXT: Total Cycles:      110
+# CHECK-NEXT: Total Cycles:      143
 # CHECK-NEXT: Total uOps:        600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.45
-# CHECK-NEXT: IPC:               3.64
-# CHECK-NEXT: Block RThroughput: 1.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.20
+# CHECK-NEXT: IPC:               2.80
+# CHECK-NEXT: Block RThroughput: 1.2
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldr	x1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D====eeeeER   ldr	x2, [x1], #254
-# CHECK-NEXT: [0,3]     D=eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5332,5 +5336,5 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	x1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     5.0    0.0    0.0       ldr	x2, [x1], #254
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.3    2.0       <total>
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/llvm-mca-markers.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/llvm-mca-markers.s
new file mode 100644
index 0000000000000..d5a291b1d7102
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/llvm-mca-markers.s
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+// RUN: llvm-mca -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 -iterations=1 -resource-pressure=false < %s | FileCheck %s
+
+.text
+// LLVM-MCA-BEGIN Empty
+// Empty sequence
+// LLVM-MCA-END
+
+  mul x1, x1, x1
+// LLVM-MCA-BEGIN NotEmpty
+  add x0, x0, x1
+// LLVM-MCA-END
+  mul x2, x2, x2
+
+# CHECK:      [0] Code Region - NotEmpty
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      1
+# CHECK-NEXT: Total Cycles:      4
+# CHECK-NEXT: Total uOps:        1
+
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    0.25
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 0.2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.17                        add	x0, x0, x1
diff --git a/llvm/test/tools/llvm-mca/X86/show-encoding.s b/llvm/test/tools/llvm-mca/X86/show-encoding.s
index 9d2015664271c..b20b7fc31cde8 100644
--- a/llvm/test/tools/llvm-mca/X86/show-encoding.s
+++ b/llvm/test/tools/llvm-mca/X86/show-encoding.s
@@ -74,4 +74,4 @@
 # WITHENCODINGS-NEXT:  1      1     0.50                         6     c4 c1 30 c6 c1 ff             vshufps	$255, %xmm9, %xmm9, %xmm0
 # WITHENCODINGS-NEXT:  1      1     1.00           *             6     c4 01 78 11 0c 8b             vmovups	%xmm9, (%r11,%r9,4)
 # WITHENCODINGS-NEXT:  1      1     0.50                         3     44 39 c6                      cmpl	%r8d, %esi
-# WITHENCODINGS-NEXT:  1      1     0.50                         6     0f 8c 00 00 00 00             jl	-90
+# WITHENCODINGS-NEXT:  1      1     0.50                         2     7c 00                         jl	-90
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
index cc7faea67bed2..737e902f4023c 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
+++ b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
@@ -33,10 +33,10 @@
 # ATT-NEXT: <BB0>:
 # ATT-NEXT:   pushq  %rax
 # ATT-NEXT:   movl   %edx, %eax
-# ATT-NEXT:   je     <BB2>
-# ATT-NEXT: <BB1>:
-# ATT-NEXT:   xorl   %esi, %esi
+# ATT-NEXT:   je     <BB4>
 # ATT-NEXT: <BB2>:
+# ATT-NEXT:   xorl   %esi, %esi
+# ATT-NEXT: <BB4>:
 # ATT-NEXT:   callq  <bar>
 # ATT-NEXT:   retq
 # ATT:      <foo.cold>:
@@ -60,10 +60,10 @@
 # INTEL-NEXT: <BB0>:
 # INTEL-NEXT:   push  rax
 # INTEL-NEXT:   mov   eax, edx
-# INTEL-NEXT:   je    <BB2>
-# INTEL-NEXT: <BB1>:
-# INTEL-NEXT:   xor   esi, esi
+# INTEL-NEXT:   je    <BB4>
 # INTEL-NEXT: <BB2>:
+# INTEL-NEXT:   xor   esi, esi
+# INTEL-NEXT: <BB4>:
 # INTEL-NEXT:   call  <bar>
 # INTEL-NEXT:   ret
 # INTEL:      <foo.cold>:
@@ -139,17 +139,20 @@ Sections:
     Type:   SHT_LLVM_BB_ADDR_MAP
     Link:   .text.bar
     Entries:
-      - Version: 1
+      - Version: 2
         BBRanges:
          - BaseAddress: 0x5000
            BBEntries:
-            - AddressOffset: 0x0
+            - ID:            0
+              AddressOffset: 0x0
               Size:          0x1
               Metadata:      0x1
-            - AddressOffset: 0x4
+            - ID:            2
+              AddressOffset: 0x4
               Size:          0x2
               Metadata:      0x0
-            - AddressOffset: 0x0
+            - ID:            4
+              AddressOffset: 0x0
               Size:          0x6
               Metadata:      0x0
 
@@ -230,17 +233,20 @@ Sections:
                AddressOffset: 0x0
                Size:          0x1
                Metadata:      0x2
-      - Version: 1
+      - Version: 2
         BBRanges:
          - BaseAddress: 0x5000
            BBEntries:
-            - AddressOffset: 0x0
+            - ID:            0
+              AddressOffset: 0x0
               Size:          0x1
               Metadata:      0x1
-            - AddressOffset: 0x4
+            - ID:            2
+              AddressOffset: 0x4
               Size:          0x2
               Metadata:      0x0
-            - AddressOffset: 0x0
+            - ID:            4
+              AddressOffset: 0x0
               Size:          0x6
               Metadata:      0x0
 
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
new file mode 100644
index 0000000000000..ed08c850cb4ad
--- /dev/null
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
@@ -0,0 +1,124 @@
+ <html>
+  <head>
+  <style>
+  table, th, td {
+    border: 1px solid black;
+  }
+  table.center {
+    margin-left: auto;
+    margin-right: auto;
+  }
+  </style>
+  </head>
+  <body>
+  <table>
+  <caption><b>Location Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>LLVM IR Instruction</th>
+    <th>Function Name</th>
+    <th>Basic Block Name</th>
+    <th>Action</th>
+    <th>Origin</th>
+  </tr>
+  </tr>
+    <tr>
+    <td>test.ll</td>
+    <td>LoopVectorizePass</td>
+    <td>add</td>
+    <td>fn</td>
+    <td>no-name</td>
+    <td>not-generate</td>
+    <td><details><summary>View Origin StackTrace</summary><pre>Stack Trace 0:
+ #0 0x00005895d035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9
+ #1 0x00005895d03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11
+ #2 0x00005895d03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5
+ #3 0x00005895d03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14
+ #4 0x00005895d06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9
+ #5 0x00005895d06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9
+ #6 0x00005895d06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9
+ #7 0x00005895d0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22
+ #8 0x00005895d05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22
+ #9 0x00005895d05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31
+#10 0x00005895d05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8
+#11 0x00005895d107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#12 0x00005895d04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8
+#13 0x00005895ce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#14 0x00005895cfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#15 0x00005895ce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#16 0x00005895cfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12
+#17 0x00005895d1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#18 0x00005895cfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5</pre></details></td>
+    </tr>
+  <tr>
+</table>
+<br>
+<table>
+  <caption><b>Summary of Location Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+    <tr>
+    <td>LoopVectorizePass</td>
+    <td>1</td>
+    </tr>
+  <tr>
+</table>
+<br>
+<br>
+<table>
+  <caption><b>SP Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>Function Name</th>
+    <th>Action</th>
+  </tr>
+<tr>
+        <td colspan='4'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<table>
+  <caption><b>Summary of SP Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+  <tr>
+<tr>
+        <td colspan='2'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<br>
+<table>
+  <caption><b>Variable Location Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>Variable</th>
+    <th>Function</th>
+    <th>Action</th>
+  </tr>
+<tr>
+        <td colspan='4'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<table>
+  <caption><b>Summary of Variable Location Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+  <tr>
+<tr>
+        <td colspan='2'> No bugs found </td>
+      </tr>
+    </table>
+</body>
+  </html>
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html
index f70ebbf86b0a2..d8941cbdedbef 100644
--- a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html
@@ -128,6 +128,15 @@
     <td>drop</td>
     </tr>
   <tr>
+  </tr>
+    <tr>
+    <td>wrstabs.c</td>
+    <td>Simplify the CFG</td>
+    <td>tindex</td>
+    <td>stab_bool_type</td>
+    <td>drop</td>
+    </tr>
+  <tr>
   </tr>
     <tr>
     <td>prdbg.c</td>
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json b/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
new file mode 100644
index 0000000000000..ab59bc295add1
--- /dev/null
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
@@ -0,0 +1 @@
+{"file":"test.ll", "pass":"LoopVectorizePass", "bugs": [[{"action":"not-generate","bb-name":"no-name","fn-name":"fn","instr":"add","metadata":"DILocation", "origin": "Stack Trace 0:\n #0 0x00005895d035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9\n #1 0x00005895d03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11\n #2 0x00005895d03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5\n #3 0x00005895d03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14\n #4 0x00005895d06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9\n #5 0x00005895d06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9\n #6 0x00005895d06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9\n #7 0x00005895d0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22\n #8 0x00005895d05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22\n #9 0x00005895d05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31\n#10 0x00005895d05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8\n#11 0x00005895d107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#12 0x00005895d04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8\n#13 0x00005895ce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#14 0x00005895cfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#15 0x00005895ce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#16 0x00005895cfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12\n#17 0x00005895d1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#18 0x00005895cfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5"}]]}
diff --git a/llvm/test/tools/llvm-original-di-preservation/basic.test b/llvm/test/tools/llvm-original-di-preservation/basic.test
index 81f987aa221bb..5ef670b42c667 100644
--- a/llvm/test/tools/llvm-original-di-preservation/basic.test
+++ b/llvm/test/tools/llvm-original-di-preservation/basic.test
@@ -1,5 +1,6 @@
 RUN: %llvm-original-di-preservation %p/Inputs/sample.json %t.html | FileCheck %s
 RUN: diff -w %p/Inputs/expected-sample.html %t.html
+CHECK: The {{.+}}.html generated.
 CHECK-NOT: Skipped lines:
 
 RUN: %llvm-original-di-preservation %p/Inputs/corrupted.json %t2.html | FileCheck %s -check-prefix=CORRUPTED
@@ -9,5 +10,8 @@ CORRUPTED: Skipped bugs: 1
 
 RUN: %llvm-original-di-preservation -compress %p/Inputs/sample.json %t3.html | FileCheck %s -check-prefix=COMPRESSED
 RUN: diff -w %p/Inputs/expected-compressed.html %t3.html
+COMPRESSED: The {{.+}}.html generated.
 COMPRESSED-NOT: Skipped lines:
 
+RUN: %llvm-original-di-preservation %p/Inputs/origin.json %t4.html | FileCheck %s
+RUN: diff -w %p/Inputs/expected-origin.html %t4.html
diff --git a/llvm/tools/dsymutil/CMakeLists.txt b/llvm/tools/dsymutil/CMakeLists.txt
index 366b63b2a8695..d3cdbe466de08 100644
--- a/llvm/tools/dsymutil/CMakeLists.txt
+++ b/llvm/tools/dsymutil/CMakeLists.txt
@@ -15,6 +15,7 @@ set(LLVM_LINK_COMPONENTS
   DWARFLinkerClassic
   DWARFLinkerParallel
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   MC
   Object
   Option
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
index e88e076dca996..138c5d0a513ed 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
@@ -39,10 +39,10 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
 #include "llvm/DebugInfo/DWARF/DWARFSection.h"
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCCodeEmitter.h"
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index ba628079170d5..c322b4f6c9828 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -878,7 +878,8 @@ static void exitOnLazyCallThroughFailure() { exit(1); }
 Expected<orc::ThreadSafeModule>
 loadModule(StringRef Path, orc::ThreadSafeContext TSCtx) {
   SMDiagnostic Err;
-  auto M = parseIRFile(Path, Err, *TSCtx.getContext());
+  auto M = TSCtx.withContextDo(
+      [&](LLVMContext *Ctx) { return parseIRFile(Path, Err, *Ctx); });
   if (!M) {
     std::string ErrMsg;
     {
diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
index 024693a24cc23..06de33dc070e0 100644
--- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
@@ -62,7 +62,7 @@
 #include <utility>
 
 /// The semantic version combined as a string.
-#define LLVM_COVERAGE_EXPORT_JSON_STR "3.0.0"
+#define LLVM_COVERAGE_EXPORT_JSON_STR "3.0.1"
 
 /// Unique type identifier for JSON coverage export.
 #define LLVM_COVERAGE_EXPORT_JSON_TYPE_STR "llvm.coverage.json.export"
@@ -113,7 +113,7 @@ json::Array renderMCDCRecord(const coverage::MCDCRecord &Record) {
   const auto [TrueDecisions, FalseDecisions] = Record.getDecisions();
   return json::Array({CMR.LineStart, CMR.ColumnStart, CMR.LineEnd,
                       CMR.ColumnEnd, TrueDecisions, FalseDecisions,
-                      CMR.ExpandedFileID, int64_t(CMR.Kind),
+                      CMR.FileID, CMR.ExpandedFileID, int64_t(CMR.Kind),
                       gatherConditions(Record)});
 }
 
diff --git a/llvm/tools/llvm-dwarfdump/CMakeLists.txt b/llvm/tools/llvm-dwarfdump/CMakeLists.txt
index 7c68aaac05d98..aeb1b8f14d830 100644
--- a/llvm/tools/llvm-dwarfdump/CMakeLists.txt
+++ b/llvm/tools/llvm-dwarfdump/CMakeLists.txt
@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   AllTargetsDescs
   AllTargetsInfos
   MC
diff --git a/llvm/tools/llvm-dwarfdump/Statistics.cpp b/llvm/tools/llvm-dwarfdump/Statistics.cpp
index af9a93ad35b9c..f92b2b55fa8dc 100644
--- a/llvm/tools/llvm-dwarfdump/Statistics.cpp
+++ b/llvm/tools/llvm-dwarfdump/Statistics.cpp
@@ -12,7 +12,7 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/JSON.h"
 
diff --git a/llvm/tools/llvm-dwarfutil/CMakeLists.txt b/llvm/tools/llvm-dwarfutil/CMakeLists.txt
index 23a526782dd7b..88335928b68dd 100644
--- a/llvm/tools/llvm-dwarfutil/CMakeLists.txt
+++ b/llvm/tools/llvm-dwarfutil/CMakeLists.txt
@@ -11,6 +11,7 @@ set(LLVM_LINK_COMPONENTS
   DWARFLinkerClassic
   DWARFLinkerParallel
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   MC
   ObjCopy
   Object
diff --git a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
index 285dcf75ecd9d..ccc1324bc66d7 100644
--- a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
+++ b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
@@ -13,7 +13,7 @@
 #include "llvm/DWARFLinker/Classic/DWARFStreamer.h"
 #include "llvm/DWARFLinker/Parallel/DWARFLinker.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Object/ObjectFile.h"
 #include <memory>
 #include <vector>
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 7823b02298e27..e7d8083da19d2 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -41,8 +41,7 @@ PressureTracker::PressureTracker(const MCSchedModel &Model)
   }
 
   ResourceUsers.resize(NextResourceUsersIdx);
-  std::fill(ResourceUsers.begin(), ResourceUsers.end(),
-            std::make_pair<unsigned, unsigned>(~0U, 0U));
+  llvm::fill(ResourceUsers, std::make_pair<unsigned, unsigned>(~0U, 0U));
 }
 
 void PressureTracker::getResourceUsers(uint64_t ResourceMask,
diff --git a/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp b/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
index 4ef8053bff410..54566dc860c57 100644
--- a/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
+++ b/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
@@ -38,10 +38,10 @@ RegisterFileStatistics::RegisterFileStatistics(const MCSubtargetInfo &sti)
   unsigned NumRegFiles = std::max(PI.NumRegisterFiles, 1U);
 
   PRFUsage.resize(NumRegFiles);
-  std::fill(PRFUsage.begin(), PRFUsage.end(), RFUEmpty);
+  llvm::fill(PRFUsage, RFUEmpty);
 
   MoveElimInfo.resize(NumRegFiles);
-  std::fill(MoveElimInfo.begin(), MoveElimInfo.end(), MEIEmpty);
+  llvm::fill(MoveElimInfo, MEIEmpty);
 }
 
 void RegisterFileStatistics::updateRegisterFileUsage(
diff --git a/llvm/tools/llvm-mca/Views/TimelineView.cpp b/llvm/tools/llvm-mca/Views/TimelineView.cpp
index 2eca48aadfd70..193bceca190df 100644
--- a/llvm/tools/llvm-mca/Views/TimelineView.cpp
+++ b/llvm/tools/llvm-mca/Views/TimelineView.cpp
@@ -28,14 +28,14 @@ TimelineView::TimelineView(const MCSubtargetInfo &sti, MCInstPrinter &Printer,
   NumInstructions *= Iterations;
   Timeline.resize(NumInstructions);
   TimelineViewEntry InvalidTVEntry = {-1, 0, 0, 0, 0};
-  std::fill(Timeline.begin(), Timeline.end(), InvalidTVEntry);
+  llvm::fill(Timeline, InvalidTVEntry);
 
   WaitTimeEntry NullWTEntry = {0, 0, 0};
-  std::fill(WaitTime.begin(), WaitTime.end(), NullWTEntry);
+  llvm::fill(WaitTime, NullWTEntry);
 
   std::pair<unsigned, int> NullUsedBufferEntry = {/* Invalid resource ID*/ 0,
                                                   /* unknown buffer size */ -1};
-  std::fill(UsedBuffer.begin(), UsedBuffer.end(), NullUsedBufferEntry);
+  llvm::fill(UsedBuffer, NullUsedBufferEntry);
 }
 
 void TimelineView::onReservedBuffers(const InstRef &IR,
diff --git a/llvm/tools/llvm-objdump/CMakeLists.txt b/llvm/tools/llvm-objdump/CMakeLists.txt
index 0306736388979..7e3197f0a0bd3 100644
--- a/llvm/tools/llvm-objdump/CMakeLists.txt
+++ b/llvm/tools/llvm-objdump/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   BinaryFormat
   DebugInfoBTF
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   Demangle
   MC
   MCDisassembler
diff --git a/llvm/tools/llvm-objdump/SourcePrinter.cpp b/llvm/tools/llvm-objdump/SourcePrinter.cpp
index bf3363795a4d9..3630502172977 100644
--- a/llvm/tools/llvm-objdump/SourcePrinter.cpp
+++ b/llvm/tools/llvm-objdump/SourcePrinter.cpp
@@ -15,7 +15,8 @@
 #include "SourcePrinter.h"
 #include "llvm-objdump.h"
 #include "llvm/ADT/SmallSet.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "objdump"
@@ -45,7 +46,7 @@ void LiveVariable::print(raw_ostream &OS, const MCRegisterInfo &MRI) const {
     return {};
   };
 
-  DWARFExpressionPrinter::printCompact(&Expression, OS, GetRegName);
+  printDwarfExpressionCompact(&Expression, OS, GetRegName);
 }
 
 void LiveVariablePrinter::addVariable(DWARFDie FuncDie, DWARFDie VarDie) {
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index a7cbd4bfc8387..45eac90aef935 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -1705,7 +1705,10 @@ static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) {
   if (WeightStr.getAsInteger(10, Weight) || Weight < 1)
     exitWithError("input weight must be a positive integer");
 
-  return {std::string(FileName), Weight};
+  llvm::SmallString<128> ResolvedFileName;
+  llvm::sys::fs::expand_tilde(FileName, ResolvedFileName);
+
+  return {std::string(ResolvedFileName), Weight};
 }
 
 static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) {
diff --git a/llvm/tools/obj2yaml/CMakeLists.txt b/llvm/tools/obj2yaml/CMakeLists.txt
index ac8ff8c85dd9c..fef0ecc220c87 100644
--- a/llvm/tools/obj2yaml/CMakeLists.txt
+++ b/llvm/tools/obj2yaml/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   DebugInfoDWARF
   Object
   ObjectYAML
+  ProfileData
   Support
   )
 
diff --git a/llvm/tools/opt/optdriver.cpp b/llvm/tools/opt/optdriver.cpp
index de46efa13025d..892b63b5be251 100644
--- a/llvm/tools/opt/optdriver.cpp
+++ b/llvm/tools/opt/optdriver.cpp
@@ -203,6 +203,10 @@ static cl::list<std::string> DisableBuiltins(
     "disable-builtin",
     cl::desc("Disable specific target library builtin function"));
 
+static cl::list<std::string> EnableBuiltins(
+    "enable-builtin",
+    cl::desc("Enable specific target library builtin functions"));
+
 static cl::opt<bool> EnableDebugify(
     "enable-debugify",
     cl::desc(
@@ -670,7 +674,7 @@ extern "C" int optMain(
   else {
     // Disable individual builtin functions in TargetLibraryInfo.
     LibFunc F;
-    for (auto &FuncName : DisableBuiltins)
+    for (auto &FuncName : DisableBuiltins) {
       if (TLII.getLibFunc(FuncName, F))
         TLII.setUnavailable(F);
       else {
@@ -678,6 +682,17 @@ extern "C" int optMain(
                << FuncName << '\n';
         return 1;
       }
+    }
+
+    for (auto &FuncName : EnableBuiltins) {
+      if (TLII.getLibFunc(FuncName, F))
+        TLII.setAvailable(F);
+      else {
+        errs() << argv[0] << ": cannot enable nonexistent builtin function "
+               << FuncName << '\n';
+        return 1;
+      }
+    }
   }
 
   if (UseNPM) {
diff --git a/llvm/tools/yaml2obj/CMakeLists.txt b/llvm/tools/yaml2obj/CMakeLists.txt
index 4a10ec851587d..ad413405255c6 100644
--- a/llvm/tools/yaml2obj/CMakeLists.txt
+++ b/llvm/tools/yaml2obj/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   Object
   ObjectYAML
+  ProfileData
   Support
   )
 
diff --git a/llvm/unittests/ADT/ArrayRefTest.cpp b/llvm/unittests/ADT/ArrayRefTest.cpp
index 985db1625454c..736c8fbb26b38 100644
--- a/llvm/unittests/ADT/ArrayRefTest.cpp
+++ b/llvm/unittests/ADT/ArrayRefTest.cpp
@@ -104,6 +104,48 @@ TEST(ArrayRefTest, DropFront) {
   EXPECT_TRUE(AR1.drop_front(2).equals(AR2));
 }
 
+TEST(ArrayRefTest, ConsumeFront) {
+  static const int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  ArrayRef<int> AR1(TheNumbers);
+  ArrayRef<int> AR2(&TheNumbers[2], AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[0]);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[1]);
+  EXPECT_TRUE(AR1.equals(AR2));
+}
+
+TEST(ArrayRefTest, ConsumeBack) {
+  static const int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  ArrayRef<int> AR1(TheNumbers);
+  ArrayRef<int> AR2(TheNumbers, AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[5]);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[4]);
+  EXPECT_TRUE(AR1.equals(AR2));
+}
+
+TEST(ArrayRefTest, MutableArryaRefConsumeFront) {
+  int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  MutableArrayRef<int> AR1(TheNumbers);
+  MutableArrayRef<int> AR2(&TheNumbers[2], AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[0]);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[1]);
+  EXPECT_TRUE(AR1.equals(AR2));
+
+  AR1.consume_front() = 33;
+  EXPECT_EQ(TheNumbers[2], 33);
+}
+
+TEST(ArrayRefTest, MutableArryaRefConsumeBack) {
+  int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  MutableArrayRef<int> AR1(TheNumbers);
+  MutableArrayRef<int> AR2(TheNumbers, AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[5]);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[4]);
+  EXPECT_TRUE(AR1.equals(AR2));
+
+  AR1.consume_back() = 33;
+  EXPECT_EQ(TheNumbers[3], 33);
+}
+
 TEST(ArrayRefTest, DropWhile) {
   static const int TheNumbers[] = {1, 3, 5, 8, 10, 11};
   ArrayRef<int> AR1(TheNumbers);
@@ -189,6 +231,27 @@ TEST(ArrayRefTest, EmptyEquals) {
   EXPECT_TRUE(ArrayRef<unsigned>() == ArrayRef<unsigned>());
 }
 
+TEST(ArrayRefTest, Compare) {
+  ArrayRef<char> Ban("Ban");
+  ArrayRef<char> Banana("Banana");
+  ArrayRef<char> Band("Band");
+
+  EXPECT_TRUE(Ban < Banana);
+  EXPECT_TRUE(Ban <= Banana);
+  EXPECT_FALSE(Ban > Banana);
+  EXPECT_FALSE(Ban >= Banana);
+
+  EXPECT_FALSE(Banana < Banana);
+  EXPECT_TRUE(Banana <= Banana);
+  EXPECT_FALSE(Banana > Banana);
+  EXPECT_TRUE(Banana >= Banana);
+
+  EXPECT_TRUE(Banana < Band);
+  EXPECT_TRUE(Banana <= Band);
+  EXPECT_FALSE(Banana > Band);
+  EXPECT_FALSE(Banana >= Band);
+}
+
 TEST(ArrayRefTest, ConstConvert) {
   int buf[4];
   for (int i = 0; i < 4; ++i)
diff --git a/llvm/unittests/ADT/DynamicAPIntTest.cpp b/llvm/unittests/ADT/DynamicAPIntTest.cpp
index 932b750608b3e..fafd8fda08687 100644
--- a/llvm/unittests/ADT/DynamicAPIntTest.cpp
+++ b/llvm/unittests/ADT/DynamicAPIntTest.cpp
@@ -31,6 +31,16 @@ class TypeNames {
 };
 TYPED_TEST_SUITE(IntTest, TypeList, TypeNames);
 
+TYPED_TEST(IntTest, ValueInit) {
+  APInt Large(65, 0, true);
+  Large.setBit(64);
+  TypeParam DynLarge(1ll << 63);
+  EXPECT_EQ(TypeParam(Large), DynLarge + DynLarge);
+  APInt Small(64, -1, true);
+  TypeParam DynSmall(Small.getSExtValue());
+  EXPECT_EQ(TypeParam(Small), DynSmall);
+}
+
 TYPED_TEST(IntTest, ops) {
   TypeParam Two(2), Five(5), Seven(7), Ten(10);
   EXPECT_EQ(Five + Five, Ten);
diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index 286cfa745fd14..9fda0d912a2f5 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -1591,6 +1591,17 @@ TEST(STLExtrasTest, Includes) {
   }
 }
 
+TEST(STLExtrasTest, Fill) {
+  std::vector<int> V1 = {1, 2, 3};
+  std::vector<int> V2;
+  int Val = 4;
+  fill(V1, Val);
+  EXPECT_THAT(V1, ElementsAre(Val, Val, Val));
+  V2.resize(4);
+  fill(V2, Val);
+  EXPECT_THAT(V2, ElementsAre(Val, Val, Val, Val));
+}
+
 struct Foo;
 struct Bar {};
 
diff --git a/llvm/unittests/ADT/STLForwardCompatTest.cpp b/llvm/unittests/ADT/STLForwardCompatTest.cpp
index b856386aa3e45..e3d500aa7b55a 100644
--- a/llvm/unittests/ADT/STLForwardCompatTest.cpp
+++ b/llvm/unittests/ADT/STLForwardCompatTest.cpp
@@ -45,6 +45,25 @@ TYPED_TEST(STLForwardCompatRemoveCVRefTest, RemoveCVRefT) {
                             llvm::remove_cvref_t<From>>::value));
 }
 
+template <typename T> class TypeIdentityTest : public ::testing::Test {
+public:
+  using TypeIdentity = llvm::type_identity<T>;
+};
+
+struct A {
+  struct B {};
+};
+using TypeIdentityTestTypes =
+    ::testing::Types<int, volatile int, A, const A::B>;
+
+TYPED_TEST_SUITE(TypeIdentityTest, TypeIdentityTestTypes, /*NameGenerator*/);
+
+TYPED_TEST(TypeIdentityTest, Identity) {
+  // TestFixture is the instantiated TypeIdentityTest.
+  EXPECT_TRUE(
+      (std::is_same_v<TypeParam, typename TestFixture::TypeIdentity::type>));
+}
+
 TEST(TransformTest, TransformStd) {
   std::optional<int> A;
 
diff --git a/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
index abc16e1aff414..7b00bd03cace4 100644
--- a/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/IR2Vec.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Dominators.h"
@@ -20,15 +21,20 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Transforms/Utils/Cloning.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <cstring>
 
 using namespace llvm;
+using namespace testing;
 
 namespace llvm {
 LLVM_ABI extern cl::opt<bool> EnableDetailedFunctionProperties;
 LLVM_ABI extern cl::opt<bool> BigBasicBlockInstructionThreshold;
 LLVM_ABI extern cl::opt<bool> MediumBasicBlockInstrutionThreshold;
+LLVM_ABI extern cl::opt<float> ir2vec::OpcWeight;
+LLVM_ABI extern cl::opt<float> ir2vec::TypeWeight;
+LLVM_ABI extern cl::opt<float> ir2vec::ArgWeight;
 } // namespace llvm
 
 namespace {
@@ -36,17 +42,64 @@ namespace {
 class FunctionPropertiesAnalysisTest : public testing::Test {
 public:
   FunctionPropertiesAnalysisTest() {
+    createTestVocabulary(1);
+    MAM.registerPass([&] { return IR2VecVocabAnalysis(Vocabulary); });
+    MAM.registerPass([&] { return PassInstrumentationAnalysis(); });
+    FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
     FAM.registerPass([&] { return DominatorTreeAnalysis(); });
     FAM.registerPass([&] { return LoopAnalysis(); });
     FAM.registerPass([&] { return PassInstrumentationAnalysis(); });
+
+    ir2vec::OpcWeight = 1.0;
+    ir2vec::TypeWeight = 1.0;
+    ir2vec::ArgWeight = 1.0;
+  }
+
+private:
+  float OriginalOpcWeight = ir2vec::OpcWeight;
+  float OriginalTypeWeight = ir2vec::TypeWeight;
+  float OriginalArgWeight = ir2vec::ArgWeight;
+
+  void createTestVocabulary(unsigned Dim) {
+    llvm::SmallVector<std::string, 32> SampleEntities = {
+        "add",        "sub",      "mul",        "icmp",          "br",
+        "ret",        "store",    "load",       "alloca",        "phi",
+        "call",       "voidTy",   "floatTy",    "integerTy",     "functionTy",
+        "structTy",   "arrayTy",  "pointerTy",  "vectorTy",      "emptyTy",
+        "labelTy",    "tokenTy",  "metadataTy", "unknownTy",     "function",
+        "pointer",    "constant", "variable",   "getelementptr", "invoke",
+        "landingpad", "resume",   "catch",      "cleanup"};
+    float EmbVal = 0.1f;
+
+    // Helper lambda to add entries to the vocabulary
+    auto addEntry = [&](std::string key) {
+      Vocabulary[key] = ir2vec::Embedding(Dim, EmbVal);
+      EmbVal += 0.1f;
+    };
+
+    for (auto &Name : SampleEntities)
+      addEntry(Name);
   }
 
 protected:
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<LoopInfo> LI;
   FunctionAnalysisManager FAM;
+  ModuleAnalysisManager MAM;
+  ir2vec::Vocab Vocabulary;
+
+  void TearDown() override {
+    // Restore original IR2Vec weights
+    ir2vec::OpcWeight = OriginalOpcWeight;
+    ir2vec::TypeWeight = OriginalTypeWeight;
+    ir2vec::ArgWeight = OriginalArgWeight;
+  }
 
   FunctionPropertiesInfo buildFPI(Function &F) {
+    // FunctionPropertiesInfo assumes IR2VecVocabAnalysis has been run to
+    // use IR2Vec.
+    auto VocabResult = MAM.getResult<IR2VecVocabAnalysis>(*F.getParent());
+    (void)VocabResult;
     return FunctionPropertiesInfo::getFunctionPropertiesInfo(F, FAM);
   }
 
@@ -71,6 +124,12 @@ class FunctionPropertiesAnalysisTest : public testing::Test {
             return CB;
     return nullptr;
   }
+
+  std::unique_ptr<ir2vec::Embedder> createEmbedder(const Function &F) {
+    auto Emb = ir2vec::Embedder::create(IR2VecKind::Symbolic, F, Vocabulary);
+    EXPECT_TRUE(static_cast<bool>(Emb));
+    return Emb;
+  }
 };
 
 TEST_F(FunctionPropertiesAnalysisTest, BasicTest) {
@@ -113,6 +172,8 @@ define internal i32 @top() {
   EXPECT_EQ(BranchesFeatures.StoreInstCount, 0);
   EXPECT_EQ(BranchesFeatures.MaxLoopDepth, 0);
   EXPECT_EQ(BranchesFeatures.TopLevelLoopCount, 0);
+  EXPECT_TRUE(BranchesFeatures.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*BranchesFunction)->getFunctionVector()));
 
   Function *TopFunction = M->getFunction("top");
   FunctionPropertiesInfo TopFeatures = buildFPI(*TopFunction);
@@ -120,6 +181,8 @@ define internal i32 @top() {
   EXPECT_EQ(TopFeatures.BlocksReachedFromConditionalInstruction, 0);
   EXPECT_EQ(TopFeatures.Uses, 0);
   EXPECT_EQ(TopFeatures.DirectCallsToDefinedFunctions, 1);
+  EXPECT_TRUE(TopFeatures.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*TopFunction)->getFunctionVector()));
   EXPECT_EQ(BranchesFeatures.LoadInstCount, 0);
   EXPECT_EQ(BranchesFeatures.StoreInstCount, 0);
   EXPECT_EQ(BranchesFeatures.MaxLoopDepth, 0);
@@ -159,6 +222,9 @@ define internal i32 @top() {
   EXPECT_EQ(DetailedBranchesFeatures.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedBranchesFeatures.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedBranchesFeatures.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(
+      DetailedBranchesFeatures.getFunctionEmbedding().approximatelyEquals(
+          createEmbedder(*BranchesFunction)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -210,6 +276,8 @@ define i64 @f1() {
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -240,6 +308,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.TotalInstructionCount = 3;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.DirectCallsToDefinedFunctions = 0;
@@ -252,6 +322,9 @@ define i32 @f2(i32 %a) {
   auto IR = llvm::InlineFunction(*CB, IFI);
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
+
   EXPECT_TRUE(FPU.finishAndTest(FAM));
   EXPECT_EQ(FPI, ExpectedFinal);
 }
@@ -294,6 +367,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.TotalInstructionCount = 9;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.DirectCallsToDefinedFunctions = 0;
@@ -307,6 +382,9 @@ define i32 @f2(i32 %a) {
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -356,6 +434,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.TotalInstructionCount = 9;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal;
   ExpectedFinal.BasicBlockCount = 6;
@@ -374,6 +454,9 @@ define i32 @f2(i32 %a) {
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -422,6 +505,8 @@ declare i32 @__gxx_personality_v0(...)
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size());
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount());
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
 }
 
 TEST_F(FunctionPropertiesAnalysisTest, InvokeUnreachableHandler) {
@@ -475,6 +560,8 @@ declare i32 @__gxx_personality_v0(...)
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size() - 1);
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount() - 2);
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EXPECT_EQ(FPI, FunctionPropertiesInfo::getFunctionPropertiesInfo(*F1, FAM));
 }
 
@@ -581,6 +668,8 @@ define void @outer() personality i8* null {
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size() - 1);
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount() - 2);
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EXPECT_EQ(FPI, FunctionPropertiesInfo::getFunctionPropertiesInfo(*F1, FAM));
 }
 
@@ -637,6 +726,8 @@ if.then:
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size() - 1);
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount() - 2);
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EXPECT_EQ(FPI, FunctionPropertiesInfo::getFunctionPropertiesInfo(*F1, FAM));
 }
 
@@ -689,6 +780,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
   ExpectedInitial.MaxLoopDepth = 1;
   ExpectedInitial.TopLevelLoopCount = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.BasicBlockCount = 6;
@@ -705,6 +798,9 @@ define i32 @f2(i32 %a) {
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -733,7 +829,7 @@ cond.false:                                       ; preds = %entry
 extra2:
   br label %cond.end
 
-cond.end:                                         ; preds = %cond.false, %cond.true
+cond.end:                                         ; preds = %extra2, %cond.true
   %cond = phi i64 [ %conv2, %cond.true ], [ %call3, %extra ]
   ret i64 %cond
 }
@@ -757,6 +853,8 @@ declare void @llvm.trap()
   ExpectedInitial.BlocksReachedFromConditionalInstruction = 2;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.BasicBlockCount = 4;
@@ -772,6 +870,9 @@ declare void @llvm.trap()
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -817,6 +918,8 @@ declare void @f3()
   ExpectedInitial.BlocksReachedFromConditionalInstruction = 0;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.BasicBlockCount = 6;
@@ -832,6 +935,9 @@ declare void @f3()
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -885,6 +991,8 @@ define i64 @f1(i64 %e) {
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -910,6 +1018,8 @@ declare float @llvm.cos.f32(float)
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -943,6 +1053,8 @@ declare float @f5()
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 1);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -972,6 +1084,8 @@ define i64 @f1(i64 %a) {
   EnableDetailedFunctionProperties.setValue(true);
   FunctionPropertiesInfo DetailedF1Properties = buildFPI(*F1);
   EXPECT_EQ(DetailedF1Properties.CriticalEdgeCount, 1);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -997,6 +1111,8 @@ declare <4 x ptr> @f4()
   EXPECT_EQ(DetailedF1Properties.CallReturnsVectorIntCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallReturnsVectorFloatCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallReturnsVectorPointerCount, 1);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
diff --git a/llvm/unittests/Analysis/IR2VecTest.cpp b/llvm/unittests/Analysis/IR2VecTest.cpp
index 90d07d080443b..70d4808dc6d54 100644
--- a/llvm/unittests/Analysis/IR2VecTest.cpp
+++ b/llvm/unittests/Analysis/IR2VecTest.cpp
@@ -109,6 +109,18 @@ TEST(EmbeddingTest, ConstructorsAndAccessors) {
   }
 }
 
+TEST(EmbeddingTest, AddVectorsOutOfPlace) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = {0.5, 1.5, -1.0};
+
+  Embedding E3 = E1 + E2;
+  EXPECT_THAT(E3, ElementsAre(1.5, 3.5, 2.0));
+
+  // Check that E1 and E2 are unchanged
+  EXPECT_THAT(E1, ElementsAre(1.0, 2.0, 3.0));
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
+}
+
 TEST(EmbeddingTest, AddVectors) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {0.5, 1.5, -1.0};
@@ -120,6 +132,18 @@ TEST(EmbeddingTest, AddVectors) {
   EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
 }
 
+TEST(EmbeddingTest, SubtractVectorsOutOfPlace) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = {0.5, 1.5, -1.0};
+
+  Embedding E3 = E1 - E2;
+  EXPECT_THAT(E3, ElementsAre(0.5, 0.5, 4.0));
+
+  // Check that E1 and E2 are unchanged
+  EXPECT_THAT(E1, ElementsAre(1.0, 2.0, 3.0));
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
+}
+
 TEST(EmbeddingTest, SubtractVectors) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {0.5, 1.5, -1.0};
@@ -131,6 +155,21 @@ TEST(EmbeddingTest, SubtractVectors) {
   EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
 }
 
+TEST(EmbeddingTest, ScaleVector) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  E1 *= 0.5f;
+  EXPECT_THAT(E1, ElementsAre(0.5, 1.0, 1.5));
+}
+
+TEST(EmbeddingTest, ScaleVectorOutOfPlace) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = E1 * 0.5f;
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.0, 1.5));
+
+  // Check that E1 is unchanged
+  EXPECT_THAT(E1, ElementsAre(1.0, 2.0, 3.0));
+}
+
 TEST(EmbeddingTest, AddScaledVector) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {2.0, 0.5, -1.0};
@@ -148,14 +187,14 @@ TEST(EmbeddingTest, ApproximatelyEqual) {
   EXPECT_TRUE(E1.approximatelyEquals(E2)); // Diff = 1e-7
 
   Embedding E3 = {1.00002, 2.00002, 3.00002}; // Diff = 2e-5
-  EXPECT_FALSE(E1.approximatelyEquals(E3));
+  EXPECT_FALSE(E1.approximatelyEquals(E3, 1e-6));
   EXPECT_TRUE(E1.approximatelyEquals(E3, 3e-5));
 
   Embedding E_clearly_within = {1.0000005, 2.0000005, 3.0000005}; // Diff = 5e-7
   EXPECT_TRUE(E1.approximatelyEquals(E_clearly_within));
 
   Embedding E_clearly_outside = {1.00001, 2.00001, 3.00001}; // Diff = 1e-5
-  EXPECT_FALSE(E1.approximatelyEquals(E_clearly_outside));
+  EXPECT_FALSE(E1.approximatelyEquals(E_clearly_outside, 1e-6));
 
   Embedding E4 = {1.0, 2.0, 3.5}; // Large diff
   EXPECT_FALSE(E1.approximatelyEquals(E4, 0.01));
@@ -174,6 +213,12 @@ TEST(EmbeddingTest, AccessOutOfBounds) {
   EXPECT_DEATH(E[4] = 4.0, "Index out of bounds");
 }
 
+TEST(EmbeddingTest, MismatchedDimensionsAddVectorsOutOfPlace) {
+  Embedding E1 = {1.0, 2.0};
+  Embedding E2 = {1.0};
+  EXPECT_DEATH(E1 + E2, "Vectors must have the same dimension");
+}
+
 TEST(EmbeddingTest, MismatchedDimensionsAddVectors) {
   Embedding E1 = {1.0, 2.0};
   Embedding E2 = {1.0};
@@ -210,10 +255,7 @@ TEST(IR2VecTest, CreateSymbolicEmbedder) {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
   Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
 
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  EXPECT_TRUE(static_cast<bool>(Result));
-
-  auto *Emb = Result->get();
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
   EXPECT_NE(Emb, nullptr);
 }
 
@@ -228,12 +270,6 @@ TEST(IR2VecTest, CreateInvalidMode) {
   // static_cast an invalid int to IR2VecKind
   auto Result = Embedder::create(static_cast<IR2VecKind>(-1), *F, V);
   EXPECT_FALSE(static_cast<bool>(Result));
-
-  std::string ErrMsg;
-  llvm::handleAllErrors(
-      Result.takeError(),
-      [&](const llvm::ErrorInfoBase &EIB) { ErrMsg = EIB.message(); });
-  EXPECT_NE(ErrMsg.find("Unknown IR2VecKind"), std::string::npos);
 }
 
 TEST(IR2VecTest, LookupVocab) {
@@ -281,7 +317,7 @@ TEST(IR2VecTest, IR2VecVocabResultValidity) {
   EXPECT_EQ(validResult.getDimension(), 2u);
 }
 
-// Fixture for IR2Vec tests requiring IR setup and weight management.
+// Fixture for IR2Vec tests requiring IR setup.
 class IR2VecTestFixture : public ::testing::Test {
 protected:
   Vocab V;
@@ -292,14 +328,10 @@ class IR2VecTestFixture : public ::testing::Test {
   Instruction *AddInst = nullptr;
   Instruction *RetInst = nullptr;
 
-  float OriginalOpcWeight = ::OpcWeight;
-  float OriginalTypeWeight = ::TypeWeight;
-  float OriginalArgWeight = ::ArgWeight;
-
   void SetUp() override {
     V = {{"add", {1.0, 2.0}},
-         {"integerTy", {0.5, 0.5}},
-         {"constant", {0.2, 0.3}},
+         {"integerTy", {0.25, 0.25}},
+         {"constant", {0.04, 0.06}},
          {"variable", {0.0, 0.0}},
          {"unknownTy", {0.0, 0.0}}};
 
@@ -316,25 +348,11 @@ class IR2VecTestFixture : public ::testing::Test {
     AddInst = BinaryOperator::CreateAdd(Arg, Const, "add", BB);
     RetInst = ReturnInst::Create(Ctx, AddInst, BB);
   }
-
-  void setWeights(float OpcWeight, float TypeWeight, float ArgWeight) {
-    ::OpcWeight = OpcWeight;
-    ::TypeWeight = TypeWeight;
-    ::ArgWeight = ArgWeight;
-  }
-
-  void TearDown() override {
-    // Restore original global weights
-    ::OpcWeight = OriginalOpcWeight;
-    ::TypeWeight = OriginalTypeWeight;
-    ::ArgWeight = OriginalArgWeight;
-  }
 };
 
 TEST_F(IR2VecTestFixture, GetInstVecMap) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &InstMap = Emb->getInstVecMap();
 
@@ -355,9 +373,8 @@ TEST_F(IR2VecTestFixture, GetInstVecMap) {
 }
 
 TEST_F(IR2VecTestFixture, GetBBVecMap) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &BBMap = Emb->getBBVecMap();
 
@@ -372,9 +389,8 @@ TEST_F(IR2VecTestFixture, GetBBVecMap) {
 }
 
 TEST_F(IR2VecTestFixture, GetBBVector) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &BBVec = Emb->getBBVector(*BB);
 
@@ -384,9 +400,8 @@ TEST_F(IR2VecTestFixture, GetBBVector) {
 }
 
 TEST_F(IR2VecTestFixture, GetFunctionVector) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &FuncVec = Emb->getFunctionVector();
 
@@ -397,23 +412,6 @@ TEST_F(IR2VecTestFixture, GetFunctionVector) {
               ElementsAre(DoubleNear(1.29, 1e-6), DoubleNear(2.31, 1e-6)));
 }
 
-TEST_F(IR2VecTestFixture, GetFunctionVectorWithCustomWeights) {
-  setWeights(1.0, 1.0, 1.0);
-
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
-
-  const auto &FuncVec = Emb->getFunctionVector();
-
-  EXPECT_EQ(FuncVec.size(), 2u);
-
-  // Expected: 1*([1.0 2.0] + [0.0 0.0]) + 1*([0.5 0.5] + [0.0 0.0]) + 1*([0.2
-  // 0.3] + [0.0 0.0])
-  EXPECT_THAT(FuncVec,
-              ElementsAre(DoubleNear(1.7, 1e-6), DoubleNear(2.8, 1e-6)));
-}
-
 TEST(IR2VecTest, IR2VecVocabAnalysisWithPrepopulatedVocab) {
   Vocab InitialVocab = {{"key1", {1.1, 2.2}}, {"key2", {3.3, 4.4}}};
   Vocab ExpectedVocab = InitialVocab;
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index dbe72286c3a0b..7a48105a1dc99 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -16,6 +16,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -909,11 +910,11 @@ TEST(ValueTracking, propagatesPoison) {
       {false, "call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt)", 0},
       {false, "call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt)", 1},
       {false, "call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt)", 2},
-      {false, "call float @llvm.sqrt.f32(float %fx)", 0},
-      {false, "call float @llvm.powi.f32.i32(float %fx, i32 %x)", 0},
+      {true, "call float @llvm.sqrt.f32(float %fx)", 0},
+      {true, "call float @llvm.powi.f32.i32(float %fx, i32 %x)", 0},
       {false, "call float @llvm.sin.f32(float %fx)", 0},
       {false, "call float @llvm.cos.f32(float %fx)", 0},
-      {false, "call float @llvm.pow.f32(float %fx, float %fy)", 0},
+      {true, "call float @llvm.pow.f32(float %fx, float %fy)", 0},
       {false, "call float @llvm.exp.f32(float %fx)", 0},
       {false, "call float @llvm.exp2.f32(float %fx)", 0},
       {false, "call float @llvm.log.f32(float %fx)", 0},
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index 360fc9c8af842..0058daf680816 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -9,6 +9,7 @@
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -78,8 +79,12 @@ class SelectionDAGAddressAnalysisTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
+    FunctionAnalysisManager FAM;
+    FAM.registerPass([&] { return TM->getTargetIRAnalysis(); });
+
+    TargetTransformInfo TTI = TM->getTargetIRAnalysis().run(*F, FAM);
     DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr);
+              nullptr, TTI.hasBranchDivergence(F));
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index a60b87c80a02c..30a1406da7734 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/SDPatternMatch.h"
@@ -76,8 +77,12 @@ class SelectionDAGPatternMatchTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
+    FunctionAnalysisManager FAM;
+    FAM.registerPass([&] { return TM->getTargetIRAnalysis(); });
+
+    TargetTransformInfo TTI = TM->getTargetIRAnalysis().run(*F, FAM);
     DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr);
+              nullptr, TTI.hasBranchDivergence(F));
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
@@ -110,6 +115,14 @@ TEST_F(SelectionDAGPatternMatchTest, matchValueType) {
 
   using namespace SDPatternMatch;
   EXPECT_TRUE(sd_match(Op0, m_SpecificVT(Int32VT)));
+  EXPECT_TRUE(sd_match(Op0, m_SpecificScalarVT(Int32VT)));
+  EXPECT_TRUE(sd_match(Op0, m_SpecificScalarVT(Int32VT, m_Value())));
+  EXPECT_FALSE(sd_match(Op0, m_SpecificScalarVT(Float32VT)));
+  EXPECT_FALSE(sd_match(Op0, m_SpecificScalarVT(Float32VT, m_Value())));
+  EXPECT_TRUE(sd_match(Op2, m_SpecificVectorElementVT(Int32VT)));
+  EXPECT_TRUE(sd_match(Op2, m_SpecificVectorElementVT(Int32VT, m_Value())));
+  EXPECT_FALSE(sd_match(Op2, m_SpecificVectorElementVT(Float32VT)));
+  EXPECT_FALSE(sd_match(Op2, m_SpecificVectorElementVT(Float32VT, m_Value())));
   EVT BindVT;
   EXPECT_TRUE(sd_match(Op1, m_VT(BindVT)));
   EXPECT_EQ(BindVT, Float32VT);
@@ -299,6 +312,23 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue UMinLikeULT = DAG->getSelect(DL, MVT::i32, ICMP_ULT, Op0, Op1);
   SDValue UMinLikeULE = DAG->getSelect(DL, MVT::i32, ICMP_ULE, Op0, Op1);
 
+  SDValue CCSMaxLikeGT = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETGT);
+  SDValue CCSMaxLikeGE = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETGE);
+  SDValue CCSMaxLikeLT = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETLT);
+  SDValue CCSMaxLikeLE = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETLE);
+  SDValue CCUMaxLikeUGT = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETUGT);
+  SDValue CCUMaxLikeUGE = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETUGE);
+  SDValue CCUMaxLikeULT = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETULT);
+  SDValue CCUMaxLikeULE = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETULE);
+  SDValue CCSMinLikeLT = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETLT);
+  SDValue CCSMinLikeGT = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETGT);
+  SDValue CCSMinLikeLE = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETLE);
+  SDValue CCSMinLikeGE = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETGE);
+  SDValue CCUMinLikeULT = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETULT);
+  SDValue CCUMinLikeUGT = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETUGT);
+  SDValue CCUMinLikeULE = DAG->getSelectCC(DL, Op0, Op1, Op0, Op1, ISD::SETULE);
+  SDValue CCUMinLikeUGE = DAG->getSelectCC(DL, Op0, Op1, Op1, Op0, ISD::SETUGE);
+
   SDValue SFAdd = DAG->getNode(ISD::STRICT_FADD, DL, {Float32VT, MVT::Other},
                                {DAG->getEntryNode(), Op2, Op2});
 
@@ -349,21 +379,37 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   EXPECT_TRUE(sd_match(SMax, m_SMaxLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMaxLikeGT, m_SMaxLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMaxLikeGE, m_SMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMaxLikeGT, m_SMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMaxLikeGE, m_SMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMaxLikeLT, m_SMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMaxLikeLE, m_SMaxLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMin, m_c_BinOp(ISD::SMIN, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMin, m_SMin(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMin, m_SMinLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMinLikeLT, m_SMinLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMinLikeLE, m_SMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMinLikeGT, m_SMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMinLikeGE, m_SMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMinLikeLT, m_SMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCSMinLikeLE, m_SMinLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMax, m_c_BinOp(ISD::UMAX, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMax, m_UMax(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMax, m_UMaxLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMaxLikeUGT, m_UMaxLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMaxLikeUGE, m_UMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMaxLikeUGT, m_UMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMaxLikeUGE, m_UMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMaxLikeULT, m_UMaxLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMaxLikeULE, m_UMaxLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMin, m_c_BinOp(ISD::UMIN, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMin, m_UMin(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMin, m_UMinLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMinLikeULT, m_UMinLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMinLikeULE, m_UMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMinLikeUGT, m_UMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMinLikeUGE, m_UMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMinLikeULT, m_UMinLike(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(CCUMinLikeULE, m_UMinLike(m_Value(), m_Value())));
 
   // By default, it matches any of the results.
   EXPECT_TRUE(
@@ -817,3 +863,89 @@ TEST_F(SelectionDAGPatternMatchTest, matchReassociatableOp) {
   EXPECT_FALSE(sd_match(
       ORS0123, m_ReassociatableOr(m_Value(), m_Value(), m_Value(), m_Value())));
 }
+
+TEST_F(SelectionDAGPatternMatchTest, MatchZeroOneAllOnes) {
+  using namespace SDPatternMatch;
+
+  SDLoc DL;
+  EVT VT = EVT::getIntegerVT(Context, 32);
+
+  // Scalar constant 0
+  SDValue Zero = DAG->getConstant(0, DL, VT);
+  EXPECT_TRUE(sd_match(Zero, DAG.get(), m_Zero()));
+  EXPECT_FALSE(sd_match(Zero, DAG.get(), m_One()));
+  EXPECT_FALSE(sd_match(Zero, DAG.get(), m_AllOnes()));
+
+  // Scalar constant 1
+  SDValue One = DAG->getConstant(1, DL, VT);
+  EXPECT_FALSE(sd_match(One, DAG.get(), m_Zero()));
+  EXPECT_TRUE(sd_match(One, DAG.get(), m_One()));
+  EXPECT_FALSE(sd_match(One, DAG.get(), m_AllOnes()));
+
+  // Scalar constant -1
+  SDValue AllOnes =
+      DAG->getConstant(APInt::getAllOnes(VT.getSizeInBits()), DL, VT);
+  EXPECT_FALSE(sd_match(AllOnes, DAG.get(), m_Zero()));
+  EXPECT_FALSE(sd_match(AllOnes, DAG.get(), m_One()));
+  EXPECT_TRUE(sd_match(AllOnes, DAG.get(), m_AllOnes()));
+
+  EVT VecF32 = EVT::getVectorVT(Context, MVT::f32, 4);
+  EVT VecVT = EVT::getVectorVT(Context, MVT::i32, 4);
+
+  // m_Zero: splat vector of 0 → bitcast
+  {
+    SDValue SplatVal = DAG->getConstant(0, DL, MVT::i32);
+    SDValue VecSplat = DAG->getSplatBuildVector(VecVT, DL, SplatVal);
+    SDValue Bitcasted = DAG->getNode(ISD::BITCAST, DL, VecF32, VecSplat);
+    EXPECT_TRUE(sd_match(Bitcasted, DAG.get(), m_Zero()));
+  }
+
+  // m_One: splat vector of 1 → bitcast
+  {
+    SDValue SplatVal = DAG->getConstant(1, DL, MVT::i32);
+    SDValue VecSplat = DAG->getSplatBuildVector(VecVT, DL, SplatVal);
+    SDValue Bitcasted = DAG->getNode(ISD::BITCAST, DL, VecF32, VecSplat);
+    EXPECT_FALSE(sd_match(Bitcasted, DAG.get(), m_One()));
+  }
+
+  // m_AllOnes: splat vector of -1 → bitcast
+  {
+    SDValue SplatVal = DAG->getConstant(APInt::getAllOnes(32), DL, MVT::i32);
+    SDValue VecSplat = DAG->getSplatBuildVector(VecVT, DL, SplatVal);
+    SDValue Bitcasted = DAG->getNode(ISD::BITCAST, DL, VecF32, VecSplat);
+    EXPECT_TRUE(sd_match(Bitcasted, DAG.get(), m_AllOnes()));
+  }
+
+  // splat vector with one undef → default should NOT match
+  SDValue Undef = DAG->getUNDEF(MVT::i32);
+
+  {
+    // m_Zero: Undef + constant 0
+    SDValue Zero = DAG->getConstant(0, DL, MVT::i32);
+    SmallVector<SDValue, 4> Ops(4, Zero);
+    Ops[2] = Undef;
+    SDValue Vec = DAG->getBuildVector(VecVT, DL, Ops);
+    EXPECT_FALSE(sd_match(Vec, DAG.get(), m_Zero()));
+    EXPECT_TRUE(sd_match(Vec, DAG.get(), m_Zero(true)));
+  }
+
+  {
+    // m_One: Undef + constant 1
+    SDValue One = DAG->getConstant(1, DL, MVT::i32);
+    SmallVector<SDValue, 4> Ops(4, One);
+    Ops[1] = Undef;
+    SDValue Vec = DAG->getBuildVector(VecVT, DL, Ops);
+    EXPECT_FALSE(sd_match(Vec, DAG.get(), m_One()));
+    EXPECT_TRUE(sd_match(Vec, DAG.get(), m_One(true)));
+  }
+
+  {
+    // m_AllOnes: Undef + constant -1
+    SDValue AllOnes = DAG->getConstant(APInt::getAllOnes(32), DL, MVT::i32);
+    SmallVector<SDValue, 4> Ops(4, AllOnes);
+    Ops[0] = Undef;
+    SDValue Vec = DAG->getBuildVector(VecVT, DL, Ops);
+    EXPECT_FALSE(sd_match(Vec, DAG.get(), m_AllOnes()));
+    EXPECT_TRUE(sd_match(Vec, DAG.get(), m_AllOnes(true)));
+  }
+}
diff --git a/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt b/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt
index 0c5b3f28ca3d5..dc8240f904b4c 100644
--- a/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt
+++ b/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   BinaryFormat
   CodeGenTypes
   DebugInfoDWARF
+  DebugInfoDWARFLowLevel
   MC
   Object
   ObjectYAML
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
index 2f91b1f86f6dd..d70e13dec6a62 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp
@@ -13,6 +13,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/DebugInfo/DWARF/DWARFUnwindTablePrinter.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
 
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
index 9225ab0125f2f..3e0cc8fed84c0 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
@@ -10,7 +10,8 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/DWARFExpressionPrinter.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/TargetRegistry.h"
@@ -70,7 +71,7 @@ void DWARFExpressionCompactPrinterTest::TestExprPrinter(
     return {};
   };
 
-  DWARFExpressionPrinter::printCompact(&Expr, OS, GetRegName);
+  printDwarfExpressionCompact(&Expr, OS, GetRegName);
   EXPECT_EQ(OS.str(), Expected);
 }
 
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
index ec9c0dddcbc0c..78d383ed0b22e 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCopyBytesTest.cpp
@@ -11,7 +11,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
-#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
+#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCCodeEmitter.h"
diff --git a/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
index a0ae9308d33ab..61b99ddf0995d 100644
--- a/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
@@ -128,29 +128,25 @@ TEST(RTDyldObjectLinkingLayerTest, TestOverrideObjectFlags) {
   };
 
   // Create a module with two void() functions: foo and bar.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
   ThreadSafeModule M;
   {
-    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    auto Ctx = std::make_unique<LLVMContext>();
+    ModuleBuilder MB(*Ctx, TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
     Function *FooImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "foo");
-    BasicBlock *FooEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "foo");
+    BasicBlock *FooEntry = BasicBlock::Create(*Ctx, "entry", FooImpl);
     IRBuilder<> B1(FooEntry);
     B1.CreateRetVoid();
 
     Function *BarImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "bar");
-    BasicBlock *BarEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", BarImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "bar");
+    BasicBlock *BarEntry = BasicBlock::Create(*Ctx, "entry", BarImpl);
     IRBuilder<> B2(BarEntry);
     B2.CreateRetVoid();
 
-    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+    M = ThreadSafeModule(MB.takeModule(), std::move(Ctx));
   }
 
   // Create a simple stack and set the override flags option.
@@ -207,21 +203,19 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) {
   };
 
   // Create a module with two void() functions: foo and bar.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
   ThreadSafeModule M;
   {
-    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    auto Ctx = std::make_unique<LLVMContext>();
+    ModuleBuilder MB(*Ctx, TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
     Function *FooImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "foo");
-    BasicBlock *FooEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "foo");
+    BasicBlock *FooEntry = BasicBlock::Create(*Ctx, "entry", FooImpl);
     IRBuilder<> B(FooEntry);
     B.CreateRetVoid();
 
-    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+    M = ThreadSafeModule(MB.takeModule(), std::move(Ctx));
   }
 
   // Create a simple stack and set the override flags option.
@@ -258,21 +252,19 @@ TEST(RTDyldObjectLinkingLayerTest, TestMemoryBufferNamePropagation) {
     GTEST_SKIP();
 
   // Create a module with two void() functions: foo and bar.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
   ThreadSafeModule M;
   {
-    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    auto Ctx = std::make_unique<LLVMContext>();
+    ModuleBuilder MB(*Ctx, TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
     Function *FooImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "foo");
-    BasicBlock *FooEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "foo");
+    BasicBlock *FooEntry = BasicBlock::Create(*Ctx, "entry", FooImpl);
     IRBuilder<> B1(FooEntry);
     B1.CreateRetVoid();
 
-    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+    M = ThreadSafeModule(MB.takeModule(), std::move(Ctx));
   }
 
   ExecutionSession ES{std::make_unique<UnsupportedExecutorProcessControl>()};
diff --git a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
index 962277fb4a0cb..cd10ffe53d0f0 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
@@ -172,8 +172,8 @@ TEST_F(ReOptimizeLayerTest, BasicReOptimization) {
       });
   EXPECT_THAT_ERROR(ROLayer->reigsterRuntimeFunctions(*JD), Succeeded());
 
-  ThreadSafeContext Ctx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("<main>", *Ctx.getContext());
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("<main>", *Ctx);
   M->setTargetTriple(Triple(sys::getProcessTriple()));
 
   (void)createRetFunction(M.get(), "main", 42);
diff --git a/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
index 5b094e2edd58c..bbb9e8d3d6a75 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
@@ -7,6 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
 #include "gtest/gtest.h"
 
 #include <atomic>
@@ -18,23 +25,42 @@ using namespace llvm::orc;
 
 namespace {
 
+const llvm::StringRef FooSrc = R"(
+  define void @foo() {
+    ret void
+  }
+)";
+
+static ThreadSafeModule parseModule(llvm::StringRef Source,
+                                    llvm::StringRef Name) {
+  auto Ctx = std::make_unique<LLVMContext>();
+  SMDiagnostic Err;
+  auto M = parseIR(MemoryBufferRef(Source, Name), Err, *Ctx);
+  if (!M) {
+    Err.print("Testcase source failed to parse: ", errs());
+    exit(1);
+  }
+  return ThreadSafeModule(std::move(M), std::move(Ctx));
+}
+
 TEST(ThreadSafeModuleTest, ContextWhollyOwnedByOneModule) {
   // Test that ownership of a context can be transferred to a single
   // ThreadSafeModule.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("M", *TSCtx.getContext());
-  ThreadSafeModule TSM(std::move(M), std::move(TSCtx));
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  ThreadSafeModule TSM(std::move(M), std::move(Ctx));
 }
 
 TEST(ThreadSafeModuleTest, ContextOwnershipSharedByTwoModules) {
   // Test that ownership of a context can be shared between more than one
   // ThreadSafeModule.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
+  auto Ctx = std::make_unique<LLVMContext>();
 
-  auto M1 = std::make_unique<Module>("M1", *TSCtx.getContext());
-  ThreadSafeModule TSM1(std::move(M1), TSCtx);
+  auto M1 = std::make_unique<Module>("M1", *Ctx);
+  auto M2 = std::make_unique<Module>("M2", *Ctx);
 
-  auto M2 = std::make_unique<Module>("M2", *TSCtx.getContext());
+  ThreadSafeContext TSCtx(std::move(Ctx));
+  ThreadSafeModule TSM1(std::move(M1), TSCtx);
   ThreadSafeModule TSM2(std::move(M2), std::move(TSCtx));
 }
 
@@ -45,12 +71,14 @@ TEST(ThreadSafeModuleTest, ContextOwnershipSharedWithClient) {
 
   {
     // Create and destroy a module.
-    auto M1 = std::make_unique<Module>("M1", *TSCtx.getContext());
+    auto M1 = TSCtx.withContextDo(
+        [](LLVMContext *Ctx) { return std::make_unique<Module>("M1", *Ctx); });
     ThreadSafeModule TSM1(std::move(M1), TSCtx);
   }
 
   // Verify that the context is still available for re-use.
-  auto M2 = std::make_unique<Module>("M2", *TSCtx.getContext());
+  auto M2 = TSCtx.withContextDo(
+      [](LLVMContext *Ctx) { return std::make_unique<Module>("M2", *Ctx); });
   ThreadSafeModule TSM2(std::move(M2), std::move(TSCtx));
 }
 
@@ -59,60 +87,69 @@ TEST(ThreadSafeModuleTest, ThreadSafeModuleMoveAssignment) {
   // to the field order) to ensure that overwriting with an empty
   // ThreadSafeModule does not destroy the context early.
   ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("M", *TSCtx.getContext());
+  auto M = TSCtx.withContextDo(
+      [](LLVMContext *Ctx) { return std::make_unique<Module>("M", *Ctx); });
   ThreadSafeModule TSM(std::move(M), std::move(TSCtx));
   TSM = ThreadSafeModule();
 }
 
-TEST(ThreadSafeModuleTest, BasicContextLockAPI) {
-  // Test that basic lock API calls work.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("M", *TSCtx.getContext());
-  ThreadSafeModule TSM(std::move(M), TSCtx);
-
-  { auto L = TSCtx.getLock(); }
+TEST(ThreadSafeModuleTest, WithContextDoPreservesContext) {
+  // Test that withContextDo passes through the LLVMContext that was used
+  // to create the ThreadSafeContext.
 
-  { auto L = TSM.getContext().getLock(); }
-}
-
-TEST(ThreadSafeModuleTest, ContextLockPreservesContext) {
-  // Test that the existence of a context lock preserves the attached
-  // context.
-  // The trick to verify this is a bit of a hack: We attach a Module
-  // (without the ThreadSafeModule wrapper) to the context, then verify
-  // that this Module destructs safely (which it will not if its context
-  // has been destroyed) even though all references to the context have
-  // been thrown away (apart from the lock).
-
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto L = TSCtx.getLock();
-  auto &Ctx = *TSCtx.getContext();
-  auto M = std::make_unique<Module>("M", Ctx);
-  TSCtx = ThreadSafeContext();
+  auto Ctx = std::make_unique<LLVMContext>();
+  LLVMContext *OriginalCtx = Ctx.get();
+  ThreadSafeContext TSCtx(std::move(Ctx));
+  TSCtx.withContextDo(
+      [&](LLVMContext *ClosureCtx) { EXPECT_EQ(ClosureCtx, OriginalCtx); });
 }
 
 TEST(ThreadSafeModuleTest, WithModuleDo) {
   // Test non-const version of withModuleDo.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  ThreadSafeModule TSM(std::make_unique<Module>("M", *TSCtx.getContext()),
-                       TSCtx);
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  ThreadSafeModule TSM(std::move(M), std::move(Ctx));
   TSM.withModuleDo([](Module &M) {});
 }
 
 TEST(ThreadSafeModuleTest, WithModuleDoConst) {
   // Test const version of withModuleDo.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  const ThreadSafeModule TSM(std::make_unique<Module>("M", *TSCtx.getContext()),
-                             TSCtx);
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  const ThreadSafeModule TSM(std::move(M), std::move(Ctx));
   TSM.withModuleDo([](const Module &M) {});
 }
 
 TEST(ThreadSafeModuleTest, ConsumingModuleDo) {
   // Test consumingModuleDo.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  ThreadSafeModule TSM(std::make_unique<Module>("M", *TSCtx.getContext()),
-                       TSCtx);
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  ThreadSafeModule TSM(std::move(M), std::move(Ctx));
   TSM.consumingModuleDo([](std::unique_ptr<Module> M) {});
 }
 
+TEST(ThreadSafeModuleTest, CloneToNewContext) {
+  auto TSM1 = parseModule(FooSrc, "foo.ll");
+  auto TSM2 = cloneToNewContext(TSM1);
+  TSM2.withModuleDo([&](Module &NewM) {
+    EXPECT_FALSE(verifyModule(NewM, &errs()));
+    TSM1.withModuleDo([&](Module &OrigM) {
+      EXPECT_NE(&NewM.getContext(), &OrigM.getContext());
+    });
+  });
+}
+
+TEST(ObjectFormatsTest, CloneToContext) {
+  auto TSM1 = parseModule(FooSrc, "foo.ll");
+
+  auto TSCtx = ThreadSafeContext(std::make_unique<LLVMContext>());
+  auto TSM2 = cloneToContext(TSM1, TSCtx);
+
+  TSM2.withModuleDo([&](Module &M) {
+    EXPECT_FALSE(verifyModule(M, &errs()));
+    TSCtx.withContextDo(
+        [&](LLVMContext *Ctx) { EXPECT_EQ(&M.getContext(), Ctx); });
+  });
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/FileCheck/FileCheckTest.cpp b/llvm/unittests/FileCheck/FileCheckTest.cpp
index 91fce69b9a6c4..2c4130330e970 100644
--- a/llvm/unittests/FileCheck/FileCheckTest.cpp
+++ b/llvm/unittests/FileCheck/FileCheckTest.cpp
@@ -1439,7 +1439,7 @@ TEST_F(FileCheckTest, Substitution) {
   // Substitution of an undefined string variable fails and error holds that
   // variable's name.
   StringSubstitution StringSubstitution(&Context, "VAR404", 42);
-  Expected<std::string> SubstValue = StringSubstitution.getResult();
+  Expected<std::string> SubstValue = StringSubstitution.getResultRegex();
   expectUndefErrors({"VAR404"}, SubstValue.takeError());
 
   // Numeric substitution blocks constituted of defined numeric variables are
@@ -1452,20 +1452,20 @@ TEST_F(FileCheckTest, Substitution) {
       std::move(NVarUse), ExpressionFormat(ExpressionFormat::Kind::HexUpper));
   NumericSubstitution SubstitutionN(&Context, "N", std::move(ExpressionN),
                                     /*InsertIdx=*/30);
-  SubstValue = SubstitutionN.getResult();
+  SubstValue = SubstitutionN.getResultRegex();
   ASSERT_THAT_EXPECTED(SubstValue, Succeeded());
   EXPECT_EQ("A", *SubstValue);
 
   // Substitution of an undefined numeric variable fails, error holds name of
   // undefined variable.
   NVar.clearValue();
-  SubstValue = SubstitutionN.getResult();
+  SubstValue = SubstitutionN.getResultRegex();
   expectUndefErrors({"N"}, SubstValue.takeError());
 
   // Substitution of a defined string variable returns the right value.
   Pattern P(Check::CheckPlain, &Context, 1);
   StringSubstitution = llvm::StringSubstitution(&Context, "FOO", 42);
-  SubstValue = StringSubstitution.getResult();
+  SubstValue = StringSubstitution.getResultRegex();
   ASSERT_THAT_EXPECTED(SubstValue, Succeeded());
   EXPECT_EQ("BAR", *SubstValue);
 }
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index 2412cc9d26c7a..281d509227a46 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_unittest(LLVMFrontendTests
   OpenMPCompositionTest.cpp
   OpenMPDecompositionTest.cpp
   OpenMPDirectiveNameTest.cpp
+  OpenMPDirectiveNameParserTest.cpp
 
   DEPENDS
   acc_gen
diff --git a/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp b/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
index dde82ac34ea28..98b33fdfb8c12 100644
--- a/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
+++ b/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 #include "gtest/gtest.h"
 
 using namespace llvm::hlsl::rootsig;
@@ -17,7 +17,7 @@ TEST(HLSLRootSignatureTest, DescriptorCBVClauseDump) {
   DescriptorTableClause Clause;
   Clause.Type = ClauseType::CBuffer;
   Clause.Reg = {RegisterType::BReg, 0};
-  Clause.setDefaultFlags();
+  Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_1);
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -37,7 +37,7 @@ TEST(HLSLRootSignatureTest, DescriptorSRVClauseDump) {
   Clause.NumDescriptors = NumDescriptorsUnbounded;
   Clause.Space = 42;
   Clause.Offset = 3;
-  Clause.Flags = DescriptorRangeFlags::None;
+  Clause.Flags = llvm::dxbc::DescriptorRangeFlags::None;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -50,13 +50,20 @@ TEST(HLSLRootSignatureTest, DescriptorSRVClauseDump) {
 }
 
 TEST(HLSLRootSignatureTest, DescriptorUAVClauseDump) {
+  using llvm::dxbc::DescriptorRangeFlags;
   DescriptorTableClause Clause;
   Clause.Type = ClauseType::UAV;
   Clause.Reg = {RegisterType::UReg, 92374};
   Clause.NumDescriptors = 3298;
   Clause.Space = 932847;
   Clause.Offset = 1;
-  Clause.Flags = DescriptorRangeFlags::ValidFlags;
+  auto ValidDescriptorRangeFlags =
+      DescriptorRangeFlags::DescriptorsVolatile |
+      DescriptorRangeFlags::DataVolatile |
+      DescriptorRangeFlags::DataStaticWhileSetAtExecute |
+      DescriptorRangeFlags::DataStatic |
+      DescriptorRangeFlags::DescriptorsStaticKeepingBufferBoundsChecks;
+  Clause.Flags = ValidDescriptorRangeFlags;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -80,7 +87,7 @@ TEST(HLSLRootSignatureTest, DescriptorSamplerClauseDump) {
   Clause.NumDescriptors = 2;
   Clause.Space = 42;
   Clause.Offset = DescriptorTableOffsetAppend;
-  Clause.Flags = DescriptorRangeFlags::ValidSamplerFlags;
+  Clause.Flags = llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -93,10 +100,44 @@ TEST(HLSLRootSignatureTest, DescriptorSamplerClauseDump) {
   EXPECT_EQ(Out, Expected);
 }
 
+TEST(HLSLRootSignatureTest, DescriptorCBVV10ClauseDump) {
+  DescriptorTableClause Clause;
+  Clause.Type = ClauseType::CBuffer;
+  Clause.Reg = {RegisterType::BReg, 0};
+  Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Clause;
+  OS.flush();
+
+  std::string Expected = "CBV(b0, numDescriptors = 1, space = 0, "
+                         "offset = DescriptorTableOffsetAppend, "
+                         "flags = DescriptorsVolatile | DataVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
+TEST(HLSLRootSignatureTest, DescriptorSamplerV10ClauseDump) {
+  DescriptorTableClause Clause;
+  Clause.Type = ClauseType::Sampler;
+  Clause.Reg = {RegisterType::SReg, 0};
+  Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Clause;
+  OS.flush();
+
+  std::string Expected = "Sampler(s0, numDescriptors = 1, space = 0, offset = "
+                         "DescriptorTableOffsetAppend, "
+                         "flags = DescriptorsVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
 TEST(HLSLRootSignatureTest, DescriptorTableDump) {
   DescriptorTable Table;
   Table.NumClauses = 4;
-  Table.Visibility = ShaderVisibility::Geometry;
+  Table.Visibility = llvm::dxbc::ShaderVisibility::Geometry;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -112,7 +153,7 @@ TEST(HLSLRootSignatureTest, RootCBVDump) {
   RootDescriptor Descriptor;
   Descriptor.Type = DescriptorType::CBuffer;
   Descriptor.Reg = {RegisterType::BReg, 0};
-  Descriptor.setDefaultFlags();
+  Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_1);
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -125,13 +166,47 @@ TEST(HLSLRootSignatureTest, RootCBVDump) {
   EXPECT_EQ(Out, Expected);
 }
 
+TEST(HLSLRootSignatureTest, RootSRV10Dump) {
+  RootDescriptor Descriptor;
+  Descriptor.Type = DescriptorType::SRV;
+  Descriptor.Reg = {RegisterType::TReg, 0};
+  Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Descriptor;
+  OS.flush();
+
+  std::string Expected = "RootSRV(t0, space = 0, "
+                         "visibility = All, "
+                         "flags = DataVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
+TEST(HLSLRootSignatureTest, RootUAVV10Dump) {
+  RootDescriptor Descriptor;
+  Descriptor.Type = DescriptorType::UAV;
+  Descriptor.Reg = {RegisterType::UReg, 0};
+  Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Descriptor;
+  OS.flush();
+
+  std::string Expected = "RootUAV(u0, space = 0, "
+                         "visibility = All, "
+                         "flags = DataVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
 TEST(HLSLRootSignatureTest, RootSRVDump) {
   RootDescriptor Descriptor;
   Descriptor.Type = DescriptorType::SRV;
   Descriptor.Reg = {RegisterType::TReg, 0};
   Descriptor.Space = 42;
-  Descriptor.Visibility = ShaderVisibility::Geometry;
-  Descriptor.Flags = RootDescriptorFlags::None;
+  Descriptor.Visibility = llvm::dxbc::ShaderVisibility::Geometry;
+  Descriptor.Flags = llvm::dxbc::RootDescriptorFlags::None;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -144,12 +219,17 @@ TEST(HLSLRootSignatureTest, RootSRVDump) {
 }
 
 TEST(HLSLRootSignatureTest, RootUAVDump) {
+  using llvm::dxbc::RootDescriptorFlags;
   RootDescriptor Descriptor;
   Descriptor.Type = DescriptorType::UAV;
   Descriptor.Reg = {RegisterType::UReg, 92374};
   Descriptor.Space = 932847;
-  Descriptor.Visibility = ShaderVisibility::Hull;
-  Descriptor.Flags = RootDescriptorFlags::ValidFlags;
+  Descriptor.Visibility = llvm::dxbc::ShaderVisibility::Hull;
+  auto ValidRootDescriptorFlags =
+      RootDescriptorFlags::DataVolatile |
+      RootDescriptorFlags::DataStaticWhileSetAtExecute |
+      RootDescriptorFlags::DataStatic;
+  Descriptor.Flags = ValidRootDescriptorFlags;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -194,18 +274,18 @@ TEST(HLSLRootSignatureTest, DefinedStaticSamplerDump) {
   StaticSampler Sampler;
   Sampler.Reg = {RegisterType::SReg, 0};
 
-  Sampler.Filter = SamplerFilter::ComparisonMinMagLinearMipPoint;
-  Sampler.AddressU = TextureAddressMode::Mirror;
-  Sampler.AddressV = TextureAddressMode::Border;
-  Sampler.AddressW = TextureAddressMode::Clamp;
+  Sampler.Filter = llvm::dxbc::SamplerFilter::ComparisonMinMagLinearMipPoint;
+  Sampler.AddressU = llvm::dxbc::TextureAddressMode::Mirror;
+  Sampler.AddressV = llvm::dxbc::TextureAddressMode::Border;
+  Sampler.AddressW = llvm::dxbc::TextureAddressMode::Clamp;
   Sampler.MipLODBias = 4.8f;
   Sampler.MaxAnisotropy = 32;
-  Sampler.CompFunc = ComparisonFunc::NotEqual;
-  Sampler.BorderColor = StaticBorderColor::OpaqueBlack;
+  Sampler.CompFunc = llvm::dxbc::ComparisonFunc::NotEqual;
+  Sampler.BorderColor = llvm::dxbc::StaticBorderColor::OpaqueBlack;
   Sampler.MinLOD = 1.0f;
   Sampler.MaxLOD = 32.0f;
   Sampler.Space = 7;
-  Sampler.Visibility = ShaderVisibility::Domain;
+  Sampler.Visibility = llvm::dxbc::ShaderVisibility::Domain;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -249,7 +329,7 @@ TEST(HLSLRootSignatureTest, SetRootConstantsDump) {
   Constants.Num32BitConstants = 983;
   Constants.Reg = {RegisterType::BReg, 34593};
   Constants.Space = 7;
-  Constants.Visibility = ShaderVisibility::Pixel;
+  Constants.Visibility = llvm::dxbc::ShaderVisibility::Pixel;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -262,7 +342,7 @@ TEST(HLSLRootSignatureTest, SetRootConstantsDump) {
 }
 
 TEST(HLSLRootSignatureTest, NoneRootFlagsDump) {
-  RootFlags Flags = RootFlags::None;
+  llvm::dxbc::RootFlags Flags = llvm::dxbc::RootFlags::None;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -274,11 +354,23 @@ TEST(HLSLRootSignatureTest, NoneRootFlagsDump) {
 }
 
 TEST(HLSLRootSignatureTest, AllRootFlagsDump) {
-  RootFlags Flags = RootFlags::ValidFlags;
+  using llvm::dxbc::RootFlags;
+  auto ValidRootFlags = RootFlags::AllowInputAssemblerInputLayout |
+                        RootFlags::DenyVertexShaderRootAccess |
+                        RootFlags::DenyHullShaderRootAccess |
+                        RootFlags::DenyDomainShaderRootAccess |
+                        RootFlags::DenyGeometryShaderRootAccess |
+                        RootFlags::DenyPixelShaderRootAccess |
+                        RootFlags::AllowStreamOutput |
+                        RootFlags::LocalRootSignature |
+                        RootFlags::DenyAmplificationShaderRootAccess |
+                        RootFlags::DenyMeshShaderRootAccess |
+                        RootFlags::CBVSRVUAVHeapDirectlyIndexed |
+                        RootFlags::SamplerHeapDirectlyIndexed;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
-  OS << Flags;
+  OS << ValidRootFlags;
   OS.flush();
 
   std::string Expected = "RootFlags("
diff --git a/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp b/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp
index 0ef6fe84f0ec9..be3f51e0e83d5 100644
--- a/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp
+++ b/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
 #include "gtest/gtest.h"
 
 using namespace llvm::hlsl::rootsig;
diff --git a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
new file mode 100644
index 0000000000000..0363a08cc0f03
--- /dev/null
+++ b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
@@ -0,0 +1,171 @@
+//===- llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "gtest/gtest.h"
+
+#include <cctype>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+using namespace llvm;
+
+static const omp::DirectiveNameParser &getParser() {
+  static omp::DirectiveNameParser Parser(omp::SourceLanguage::C |
+                                         omp::SourceLanguage::Fortran);
+  return Parser;
+}
+
+static std::vector<std::string> tokenize(StringRef S) {
+  std::vector<std::string> Tokens;
+
+  using TokenIterator = std::istream_iterator<std::string>;
+  std::string Copy = S.str();
+  std::istringstream Stream(Copy);
+
+  for (auto I = TokenIterator(Stream), E = TokenIterator(); I != E; ++I)
+    Tokens.push_back(*I);
+  return Tokens;
+}
+
+static std::string &prepareParamName(std::string &Name) {
+  for (size_t I = 0, E = Name.size(); I != E; ++I) {
+    // The parameter name must only have alphanumeric characters.
+    if (!isalnum(Name[I]))
+      Name[I] = 'X';
+  }
+  return Name;
+}
+
+namespace llvm {
+template <> struct enum_iteration_traits<omp::Directive> {
+  static constexpr bool is_iterable = true;
+};
+} // namespace llvm
+
+// Test tokenizing.
+
+class Tokenize : public testing::TestWithParam<omp::Directive> {};
+
+static bool isEqual(const SmallVector<StringRef> &A,
+                    const std::vector<std::string> &B) {
+  if (A.size() != B.size())
+    return false;
+
+  for (size_t I = 0, E = A.size(); I != E; ++I) {
+    if (A[I] != StringRef(B[I]))
+      return false;
+  }
+  return true;
+}
+
+TEST_P(Tokenize, T) {
+  omp::Directive DirId = GetParam();
+  StringRef Name = omp::getOpenMPDirectiveName(DirId, omp::FallbackVersion);
+
+  SmallVector<StringRef> tokens1 = omp::DirectiveNameParser::tokenize(Name);
+  std::vector<std::string> tokens2 = tokenize(Name);
+  ASSERT_TRUE(isEqual(tokens1, tokens2));
+}
+
+static std::string
+getParamName1(const testing::TestParamInfo<Tokenize::ParamType> &Info) {
+  omp::Directive DirId = Info.param;
+  std::string Name =
+      omp::getOpenMPDirectiveName(DirId, omp::FallbackVersion).str();
+  return prepareParamName(Name);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    DirectiveNameParserTest, Tokenize,
+    testing::ValuesIn(
+        llvm::enum_seq(static_cast<omp::Directive>(0),
+                       static_cast<omp::Directive>(omp::Directive_enumSize))),
+    getParamName1);
+
+// Test parsing of valid names.
+
+using ValueType = std::tuple<omp::Directive, unsigned>;
+
+class ParseValid : public testing::TestWithParam<ValueType> {};
+
+TEST_P(ParseValid, T) {
+  auto [DirId, Version] = GetParam();
+  if (DirId == omp::Directive::OMPD_unknown)
+    return;
+
+  std::string Name = omp::getOpenMPDirectiveName(DirId, Version).str();
+
+  // Tokenize and parse
+  auto &Parser = getParser();
+  auto *State = Parser.initial();
+  ASSERT_TRUE(State != nullptr);
+
+  std::vector<std::string> Tokens = tokenize(Name);
+  for (auto &Tok : Tokens) {
+    State = Parser.consume(State, Tok);
+    ASSERT_TRUE(State != nullptr);
+  }
+
+  ASSERT_EQ(State->Value, DirId);
+}
+
+static std::string
+getParamName2(const testing::TestParamInfo<ParseValid::ParamType> &Info) {
+  auto [DirId, Version] = Info.param;
+  std::string Name = omp::getOpenMPDirectiveName(DirId, Version).str() + "v" +
+                     std::to_string(Version);
+  return prepareParamName(Name);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    DirectiveNameParserTest, ParseValid,
+    testing::Combine(testing::ValuesIn(llvm::enum_seq(
+                         static_cast<omp::Directive>(0),
+                         static_cast<omp::Directive>(omp::Directive_enumSize))),
+                     testing::ValuesIn(omp::getOpenMPVersions())),
+    getParamName2);
+
+// Test parsing of invalid names
+
+class ParseInvalid : public testing::TestWithParam<std::string> {};
+
+TEST_P(ParseInvalid, T) {
+  std::string Name = GetParam();
+
+  auto &Parser = getParser();
+  auto *State = Parser.initial();
+  ASSERT_TRUE(State != nullptr);
+
+  std::vector<std::string> Tokens = tokenize(Name);
+  for (auto &Tok : Tokens)
+    State = Parser.consume(State, Tok);
+
+  ASSERT_TRUE(State == nullptr || State->Value == omp::Directive::OMPD_unknown);
+}
+
+namespace {
+using namespace std;
+
+INSTANTIATE_TEST_SUITE_P(DirectiveNameParserTest, ParseInvalid,
+                         testing::Values(
+                             // Names that contain invalid tokens
+                             "bad"s, "target teams invalid"s,
+                             "target sections parallel"s,
+                             "target teams distribute parallel for wrong"s,
+                             // Valid beginning, but not a complete name
+                             "begin declare"s,
+                             // Complete name with extra tokens
+                             "distribute simd target"s));
+} // namespace
diff --git a/llvm/unittests/FuzzMutate/StrategiesTest.cpp b/llvm/unittests/FuzzMutate/StrategiesTest.cpp
index 28d950269b49d..a7d8b6c336089 100644
--- a/llvm/unittests/FuzzMutate/StrategiesTest.cpp
+++ b/llvm/unittests/FuzzMutate/StrategiesTest.cpp
@@ -89,9 +89,12 @@ void IterateOnSource(StringRef Source, IRMutator &Mutator) {
   }
 }
 
-static void mutateAndVerifyModule(StringRef Source,
-                                  std::unique_ptr<IRMutator> &Mutator,
-                                  int repeat = 100) {
+using ModuleVerifier = std::function<void(Module &)>;
+
+static void
+mutateAndVerifyModule(StringRef Source, std::unique_ptr<IRMutator> &Mutator,
+                      int repeat = 100,
+                      ArrayRef<ModuleVerifier> ExtraModuleVerifiers = {}) {
   LLVMContext Ctx;
   auto M = parseAssembly(Source.data(), Ctx);
   std::mt19937 mt(Seed);
@@ -99,13 +102,19 @@ static void mutateAndVerifyModule(StringRef Source,
   for (int i = 0; i < repeat; i++) {
     Mutator->mutateModule(*M, RandInt(mt), IRMutator::getModuleSize(*M) + 1024);
     ASSERT_FALSE(verifyModule(*M, &errs()));
+    for (auto &ModuleVerifier : ExtraModuleVerifiers) {
+      ModuleVerifier(*M);
+    }
   }
 }
+
 template <class Strategy>
-static void mutateAndVerifyModule(StringRef Source, int repeat = 100) {
+static void
+mutateAndVerifyModule(StringRef Source, int repeat = 100,
+                      ArrayRef<ModuleVerifier> ExtraModuleVerifiers = {}) {
   auto Mutator = createMutator<Strategy>();
   ASSERT_TRUE(Mutator);
-  mutateAndVerifyModule(Source, Mutator, repeat);
+  mutateAndVerifyModule(Source, Mutator, repeat, ExtraModuleVerifiers);
 }
 
 TEST(InjectorIRStrategyTest, EmptyModule) {
@@ -763,4 +772,35 @@ TEST(AllStrategies, SpecialTerminator) {
   mutateAndVerifyModule<SinkInstructionStrategy>(Source);
 }
 
+TEST(AllStrategies, AMDGCNLegalAddrspace) {
+  StringRef Source = "\n\
+    target triple = \"amdgcn-amd-amdhsa\"\n\
+    ; minimum values required by the fuzzer (e.g., default addrspace for allocas and globals)\n\
+    target datalayout = \"A5-G1\"\n\
+    define amdgpu_gfx void @strict_wwm_amdgpu_cs_main(<4 x i32> inreg %desc, i32 %index) {\n\
+    %desc.int = bitcast <4 x i32> %desc to i128\n\
+    %desc.ptr = inttoptr i128 %desc.int to ptr addrspace(8)\n\
+    ret void\n\
+  }\n\
+  ";
+
+  ModuleVerifier AddrSpaceVerifier = [](Module &M) {
+    Function *F = M.getFunction("strict_wwm_amdgpu_cs_main");
+    EXPECT_TRUE(F != nullptr);
+    for (BasicBlock &BB : *F) {
+      for (Instruction &I : BB) {
+        if (StoreInst *S = dyn_cast<StoreInst>(&I)) {
+          EXPECT_TRUE(S->getPointerAddressSpace() != 8);
+        } else if (LoadInst *L = dyn_cast<LoadInst>(&I)) {
+          EXPECT_TRUE(L->getPointerAddressSpace() != 8);
+        }
+      }
+    }
+  };
+
+  int Repeat = 100;
+  mutateAndVerifyModule<SinkInstructionStrategy>(Source, Repeat,
+                                                 {AddrSpaceVerifier});
+}
+
 } // namespace
diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
index cb1a56a26f063..aac0f96845f2f 100644
--- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
+++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
@@ -164,7 +164,7 @@ TEST(BasicBlockDbgInfoTest, DropSourceAtomOnSplit) {
     !1 = !DIFile(filename: "dummy", directory: "dummy")
     !2 = !{i32 7, !"Dwarf Version", i32 5}
     !3 = !{i32 2, !"Debug Info Version", i32 3}
-    !10 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13)
+    !10 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13, keyInstructions: true)
     !11 = !DISubroutineType(types: !12)
     !12 = !{null}
     !13 = !{}
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index 520735dfc3268..4f2ede3321080 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -920,9 +920,11 @@ TEST_F(IRBuilderTest, DIBuilder) {
     // --------------------------
     DILocation *LabelLoc = DILocation::get(Ctx, 1, 0, BarScope);
     DILabel *AlwaysPreserveLabel = DIB.createLabel(
-        BarScope, "meles_meles", File, 1, /*AlwaysPreserve*/ true);
-    DILabel *Label =
-        DIB.createLabel(BarScope, "badger", File, 1, /*AlwaysPreserve*/ false);
+        BarScope, "meles_meles", File, 1, /*Column*/ 0, /*IsArtificial*/ false,
+        /*CoroSuspendIdx*/ std::nullopt, /*AlwaysPreserve*/ true);
+    DILabel *Label = DIB.createLabel(
+        BarScope, "badger", File, 1, /*Column*/ 0, /*IsArtificial*/ false,
+        /*CoroSuspendIdx*/ std::nullopt, /*AlwaysPreserve*/ false);
 
     { /* dbg.label | DbgLabelRecord */
       // Insert before I and check order.
diff --git a/llvm/unittests/IR/LegacyPassManagerTest.cpp b/llvm/unittests/IR/LegacyPassManagerTest.cpp
index 71241e44831e0..00ceebaff4160 100644
--- a/llvm/unittests/IR/LegacyPassManagerTest.cpp
+++ b/llvm/unittests/IR/LegacyPassManagerTest.cpp
@@ -359,7 +359,8 @@ namespace llvm {
     struct CustomOptPassGate : public OptPassGate {
       bool Skip;
       CustomOptPassGate(bool Skip) : Skip(Skip) { }
-      bool shouldRunPass(const StringRef PassName, StringRef IRDescription) override {
+      bool shouldRunPass(StringRef PassName,
+                         StringRef IRDescription) const override {
         return !Skip;
       }
       bool isEnabled() const override { return true; }
diff --git a/llvm/unittests/IR/MetadataTest.cpp b/llvm/unittests/IR/MetadataTest.cpp
index df415602cef59..ba8367f4a396e 100644
--- a/llvm/unittests/IR/MetadataTest.cpp
+++ b/llvm/unittests/IR/MetadataTest.cpp
@@ -3023,12 +3023,13 @@ TEST_F(DISubprogramTest, get) {
   assert(!IsLocalToUnit && IsDefinition && !IsOptimized &&
          "bools and SPFlags have to match");
   SPFlags |= DISubprogram::SPFlagDefinition;
+  bool KeyInstructions = false;
 
   auto *N = DISubprogram::get(
       Context, Scope, Name, LinkageName, File, Line, Type, ScopeLine,
       ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit,
       TemplateParams, Declaration, RetainedNodes, ThrownTypes, Annotations,
-      TargetFuncName);
+      TargetFuncName, KeyInstructions);
 
   EXPECT_EQ(dwarf::DW_TAG_subprogram, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
@@ -3053,125 +3054,138 @@ TEST_F(DISubprogramTest, get) {
   EXPECT_EQ(ThrownTypes, N->getThrownTypes().get());
   EXPECT_EQ(Annotations, N->getAnnotations().get());
   EXPECT_EQ(TargetFuncName, N->getTargetFuncName());
+  EXPECT_EQ(KeyInstructions, N->getKeyInstructionsEnabled());
   EXPECT_EQ(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
 
   EXPECT_NE(N, DISubprogram::get(Context, getCompositeType(), Name, LinkageName,
                                  File, Line, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, "other", LinkageName, File,
                                  Line, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
-  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, "other", File, Line,
-                                 Type, ScopeLine, ContainingType, VirtualIndex,
-                                 ThisAdjustment, Flags, SPFlags, Unit,
-                                 TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
+  EXPECT_NE(N, DISubprogram::get(
+                   Context, Scope, Name, "other", File, Line, Type, ScopeLine,
+                   ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags,
+                   Unit, TemplateParams, Declaration, RetainedNodes,
+                   ThrownTypes, Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, getFile(),
                                  Line, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File,
                                  Line + 1, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  getSubroutineType(), ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagLocalToUnit, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagDefinition, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine + 1, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, getCompositeType(),
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagVirtual, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType,
                                  VirtualIndex + 1, ThisAdjustment, Flags,
                                  SPFlags, Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagOptimized, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, nullptr,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
-  EXPECT_NE(N,
-            DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
-                              Type, ScopeLine, ContainingType, VirtualIndex,
-                              ThisAdjustment, Flags, SPFlags, Unit, getTuple(),
-                              Declaration, RetainedNodes, ThrownTypes,
-                              Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
+  EXPECT_NE(N, DISubprogram::get(
+                   Context, Scope, Name, LinkageName, File, Line, Type,
+                   ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
+                   Flags, SPFlags, Unit, getTuple(), Declaration, RetainedNodes,
+                   ThrownTypes, Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, getSubprogram(), RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, getTuple(),
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 getTuple(), Annotations, TargetFuncName));
+                                 getTuple(), Annotations, TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, getTuple(), TargetFuncName));
+                                 ThrownTypes, getTuple(), TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, "other"));
+                                 ThrownTypes, Annotations, "other",
+                                 KeyInstructions));
+  EXPECT_NE(N,
+            DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+                              Type, ScopeLine, ContainingType, VirtualIndex,
+                              ThisAdjustment, Flags, SPFlags, Unit,
+                              TemplateParams, Declaration, RetainedNodes,
+                              ThrownTypes, Annotations, TargetFuncName, true));
 
   TempDISubprogram Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp
index 423f92ea07b39..21606fff1f32b 100644
--- a/llvm/unittests/Object/ELFObjectFileTest.cpp
+++ b/llvm/unittests/Object/ELFObjectFileTest.cpp
@@ -737,7 +737,7 @@ TEST(ELFObjectFileTest, ReadBBAddrMap) {
     Type: SHT_LLVM_BB_ADDR_MAP
     Link: 2
     Entries:
-      - Version: 1
+      - Version: 2
         BBRanges:
           - BaseAddress: 0x33333
             BBEntries:
@@ -904,7 +904,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
   SmallString<128> UnsupportedLowVersionYamlString(CommonYamlString);
   UnsupportedLowVersionYamlString += R"(
       - Version: 1
-        Feature: 0x4
+        Feature: 0x0
         BBRanges:
           - BBEntries:
               - AddressOffset: 0x0
@@ -915,8 +915,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
   {
     SCOPED_TRACE("unsupported version");
     DoCheck(UnsupportedLowVersionYamlString,
-            "version should be >= 2 for SHT_LLVM_BB_ADDR_MAP when PGO features "
-            "are enabled: version = 1 feature = 4");
+            "unsupported SHT_LLVM_BB_ADDR_MAP version: 1");
   }
 
   // Check that we fail when function entry count is enabled but not provided.
diff --git a/llvm/unittests/Support/DataExtractorTest.cpp b/llvm/unittests/Support/DataExtractorTest.cpp
index e019cf6fc9256..a6e2c94c2f15b 100644
--- a/llvm/unittests/Support/DataExtractorTest.cpp
+++ b/llvm/unittests/Support/DataExtractorTest.cpp
@@ -78,15 +78,30 @@ TEST(DataExtractorTest, SignedNumbers) {
 
   EXPECT_EQ(-128, DE.getSigned(&offset, 1));
   EXPECT_EQ(1U, offset);
+  offset = 0;
+  EXPECT_EQ(-128, DE.getS8(&offset));
+  EXPECT_EQ(1U, offset);
+
   offset = 0;
   EXPECT_EQ(-32624, DE.getSigned(&offset, 2));
   EXPECT_EQ(2U, offset);
+  offset = 0;
+  EXPECT_EQ(-32624, DE.getS16(&offset));
+  EXPECT_EQ(2U, offset);
+
   offset = 0;
   EXPECT_EQ(-2137980929, DE.getSigned(&offset, 4));
   EXPECT_EQ(4U, offset);
+  offset = 0;
+  EXPECT_EQ(-2137980929, DE.getS32(&offset));
+  EXPECT_EQ(4U, offset);
+
   offset = 0;
   EXPECT_EQ(-9182558167379214336LL, DE.getSigned(&offset, 8));
   EXPECT_EQ(8U, offset);
+  offset = 0;
+  EXPECT_EQ(-9182558167379214336LL, DE.getS64(&offset));
+  EXPECT_EQ(8U, offset);
 }
 
 TEST(DataExtractorTest, Strings) {
diff --git a/llvm/unittests/Support/ParallelTest.cpp b/llvm/unittests/Support/ParallelTest.cpp
index ccf5f2bb91324..041067d068883 100644
--- a/llvm/unittests/Support/ParallelTest.cpp
+++ b/llvm/unittests/Support/ParallelTest.cpp
@@ -67,17 +67,17 @@ TEST(Parallel, TransformReduce) {
 
   // Check that we handle non-divisible task sizes as above.
   uint32_t range[2050];
-  std::fill(std::begin(range), std::end(range), 1);
+  llvm::fill(range, 1);
   sum = parallelTransformReduce(range, 0U, std::plus<uint32_t>(), identity);
   EXPECT_EQ(sum, 2050U);
 
-  std::fill(std::begin(range), std::end(range), 2);
+  llvm::fill(range, 2);
   sum = parallelTransformReduce(range, 0U, std::plus<uint32_t>(), identity);
   EXPECT_EQ(sum, 4100U);
 
   // Avoid one large task.
   uint32_t range2[3060];
-  std::fill(std::begin(range2), std::end(range2), 1);
+  llvm::fill(range2, 1);
   sum = parallelTransformReduce(range2, 0U, std::plus<uint32_t>(), identity);
   EXPECT_EQ(sum, 3060U);
 }
diff --git a/llvm/unittests/Support/RISCVAttributeParserTest.cpp b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
index 777dc4d0f4d42..7e862822fdd74 100644
--- a/llvm/unittests/Support/RISCVAttributeParserTest.cpp
+++ b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
@@ -45,7 +45,7 @@ static bool testAttribute(unsigned Tag, unsigned Value, unsigned ExpectedTag,
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<unsigned> Attr = Parser.getAttributeValue("", ExpectedTag);
-  return Attr && *Attr == ExpectedValue;
+  return Attr == ExpectedValue;
 }
 
 static bool testTagString(unsigned Tag, const char *name) {
diff --git a/llvm/unittests/Support/TrailingObjectsTest.cpp b/llvm/unittests/Support/TrailingObjectsTest.cpp
index 2590f375b6598..9184a4dd0cc23 100644
--- a/llvm/unittests/Support/TrailingObjectsTest.cpp
+++ b/llvm/unittests/Support/TrailingObjectsTest.cpp
@@ -45,9 +45,10 @@ class Class1 final : private TrailingObjects<Class1, short> {
   template <typename... Ty>
   using FixedSizeStorage = TrailingObjects::FixedSizeStorage<Ty...>;
 
-  using TrailingObjects::totalSizeToAlloc;
   using TrailingObjects::additionalSizeToAlloc;
   using TrailingObjects::getTrailingObjects;
+  using TrailingObjects::getTrailingObjectsNonStrict;
+  using TrailingObjects::totalSizeToAlloc;
 };
 
 // Here, there are two singular optional object types appended. Note
@@ -123,11 +124,11 @@ TEST(TrailingObjects, OneArg) {
   EXPECT_EQ(Class1::totalSizeToAlloc<short>(3),
             sizeof(Class1) + sizeof(short) * 3);
 
-  EXPECT_EQ(C->getTrailingObjects<short>(), reinterpret_cast<short *>(C + 1));
+  EXPECT_EQ(C->getTrailingObjects(), reinterpret_cast<short *>(C + 1));
   EXPECT_EQ(C->get(0), 1);
   EXPECT_EQ(C->get(2), 3);
 
-  EXPECT_EQ(C->getTrailingObjects(), C->getTrailingObjects<short>());
+  EXPECT_EQ(C->getTrailingObjects(), C->getTrailingObjectsNonStrict<short>());
 
   delete C;
 }
diff --git a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
index f06f03bb35a5d..28d69f30bfd1a 100644
--- a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
@@ -8,6 +8,7 @@
 #include "AArch64SelectionDAGInfo.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -62,8 +63,12 @@ class AArch64SelectionDAGTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
+    FunctionAnalysisManager FAM;
+    FAM.registerPass([&] { return TM->getTargetIRAnalysis(); });
+
+    TargetTransformInfo TTI = TM->getTargetIRAnalysis().run(*F, FAM);
     DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr);
+              nullptr, TTI.hasBranchDivergence(F));
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
index 0e6c1172ee9b5..2744dbc0907bf 100644
--- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
+++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
@@ -374,9 +374,9 @@ TEST_P(RISCVInstrInfoTest, GetDestEEW) {
   EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VMSEQ_VV), 4), 0u);
   EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VMAND_MM), 0), 0u);
   EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VIOTA_M), 3), 3u);
-  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VQMACCU_2x8x2), 3), 5u);
-  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VFWMACC_4x4x4), 4), 5u);
-  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::THVdotVMAQA_VV), 5), 5u);
+  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::SF_VQMACCU_2x8x2), 3), 5u);
+  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::SF_VFWMACC_4x4x4), 4), 5u);
+  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::TH_VMAQA_VV), 5), 5u);
 }
 
 } // namespace
diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp
index c13d33e32c6d9..0a9ac9bb0596d 100644
--- a/llvm/unittests/TargetParser/Host.cpp
+++ b/llvm/unittests/TargetParser/Host.cpp
@@ -122,6 +122,14 @@ TEST(getLinuxHostCPUName, AArch64) {
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
                                               "CPU part        : 0xd48"),
             "cortex-x2");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
+                                              "CPU part        : 0xd85\n"
+                                              "CPU part        : 0xd87"),
+            "cortex-x925");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
+                                              "CPU part        : 0xd87\n"
+                                              "CPU part        : 0xd85"),
+            "cortex-x925");
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n"
                                               "CPU part        : 0xc00"),
             "falkor");
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 66e335a33a3f7..9780eba5dc7a7 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1133,6 +1133,7 @@ R"(All available -march extensions for RISC-V
     xandesvbfhcvt        5.0
     xandesvdot           5.0
     xandesvpackfph       5.0
+    xandesvsintload      5.0
     xcvalu               1.0
     xcvbi                1.0
     xcvbitmanip          1.0
@@ -1140,6 +1141,7 @@ R"(All available -march extensions for RISC-V
     xcvmac               1.0
     xcvmem               1.0
     xcvsimd              1.0
+    xmipscbop            1.0
     xmipscmov            1.0
     xmipslsp             1.0
     xsfcease             1.0
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 97ee8dd1cb67b..aeeea0a391ba9 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1164,6 +1164,7 @@ INSTANTIATE_TEST_SUITE_P(
                       AArch64CPUTestParams("a64fx", "armv8.2-a"),
                       AArch64CPUTestParams("fujitsu-monaka", "armv9.3-a"),
                       AArch64CPUTestParams("carmel", "armv8.2-a"),
+                      AArch64CPUTestParams("gb10", "armv9.2-a"),
                       AArch64CPUTestParams("grace", "armv9-a"),
                       AArch64CPUTestParams("olympus", "armv9.2-a"),
                       AArch64CPUTestParams("saphira", "armv8.4-a"),
@@ -1260,7 +1261,7 @@ INSTANTIATE_TEST_SUITE_P(
     AArch64CPUAliasTestParams::PrintToStringParamName);
 
 // Note: number of CPUs includes aliases.
-static constexpr unsigned NumAArch64CPUArchs = 90;
+static constexpr unsigned NumAArch64CPUArchs = 91;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector<StringRef, NumAArch64CPUArchs> List;
@@ -1441,6 +1442,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_SVEAES,       AArch64::AEK_SME_MOP4,
       AArch64::AEK_SME_TMOP,     AArch64::AEK_SVEBITPERM,
       AArch64::AEK_SSVE_BITPERM, AArch64::AEK_SVESHA3,
+      AArch64::AEK_SVESM4,
   };
 
   std::vector<StringRef> Features;
@@ -1478,6 +1480,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve-aes"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-aes"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve-sm4"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sm4"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve-sha3"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sha3"));
@@ -1652,6 +1655,7 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
       {"sve-f16f32mm", "nosve-f16f32mm", "+sve-f16f32mm", "-sve-f16f32mm"},
       {"sve2", "nosve2", "+sve2", "-sve2"},
       {"sve-aes", "nosve-aes", "+sve-aes", "-sve-aes"},
+      {"sve-sm4", "nosve-sm4", "+sve-sm4", "-sve-sm4"},
       {"sve-sha3", "nosve-sha3", "+sve-sha3", "-sve-sha3"},
       {"sve2-aes", "nosve2-aes", "+sve2-aes", "-sve2-aes"},
       {"sve2-sm4", "nosve2-sm4", "+sve2-sm4", "-sve2-sm4"},
@@ -2161,6 +2165,12 @@ AArch64ExtensionDependenciesBaseArchTestParams
          {"sve2"},
          {"sve2-aes", "sve-aes"}},
 
+        // -sve2-sm4 should disable sve-sm4 (only)
+        {AArch64::ARMV9_6A,
+         {"sve2", "sve-sm4", "nosve2-sm4"},
+         {"sve2"},
+         {"sve2-sm4", "sve-sm4"}},
+
         // -sve2-sha3 should disable sve-sha3 (only)
         {AArch64::ARMV9_6A,
          {"sve2", "sve-sha3", "nosve2-sha3"},
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index 4d547011c1568..ca48e77cf7caa 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TargetParser/Triple.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/VersionTuple.h"
 #include "gtest/gtest.h"
@@ -2757,6 +2758,159 @@ TEST(TripleTest, FileFormat) {
   EXPECT_EQ(Triple::DXContainer, Triple("dxil-apple-macosx").getObjectFormat());
 }
 
+TEST(TripleTest, DefaultExceptionHandling) {
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-unknown-linux-gnu").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-unknown-freebsd").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("wasm32-unknown-wasi-wasm").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("wasm64-unknown-wasi-wasm").getDefaultExceptionHandling());
+
+  EXPECT_EQ(
+      ExceptionHandling::ARM,
+      Triple("arm-unknown-linux-android16").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("armv6-unknown-netbsd-eabi").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::ARM,
+            Triple("armv7-suse-linux-gnueabihf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::ARM,
+            Triple("thumbv7-linux-gnueabihf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("thumbv7-none-netbsd").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::ZOS,
+            Triple("s390x-ibm-zos").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("systemz-ibm-linux").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("sparcel-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("amdgcn--").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("nvptx64--").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("spirv32--").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("spirv64--").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-pc-windows-msvc-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("i686-pc-windows-msvc").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i386-apple-darwin9").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("x86_64-unknown-uefi").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI, // ???
+            Triple("x86_64-unknown-msvc").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("riscv32-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("riscv64-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("xcore-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("xtensa-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("lanai-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("arc-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(
+      ExceptionHandling::DwarfCFI,
+      Triple("loongarch32-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(
+      ExceptionHandling::DwarfCFI,
+      Triple("loongarch64-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("msp430-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("m68k-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("csky-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::AIX,
+            Triple("powerpc-ibm-aix").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::AIX,
+            Triple("powerpc---xcoff").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("powerpc-apple-macosx").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-pc-cygwin-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-pc-cygwin-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("x86_64-pc-win32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("x86_64-pc-windows-coreclr").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("ve-unknown-linux").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("ve-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("armv7k-apple-watchos").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::SjLj,
+            Triple("armv7-apple-ios").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mips64-mti-linux-gnu").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mips").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mips-pc-windows-msvc").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("mips-pc-windows-msvc-coff").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("mipsel-windows--coff").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mipsel-windows-msvc").getDefaultExceptionHandling());
+
+  EXPECT_EQ(
+      ExceptionHandling::WinEH,
+      Triple("aarch64-unknown-windows-msvc").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("aarch64-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("arm64_32-apple-ios").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("arm64-apple-macosx").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("xcore-xmos-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("i386-pc-windows-msvc").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i386-mingw32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("i386-pc-win32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i386-pc-mingw32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-pc-cygwin").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-pc-mingw64").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-win32-gnu").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-scei-ps4").getDefaultExceptionHandling());
+}
+
 TEST(TripleTest, NormalizeWindows) {
   EXPECT_EQ("i686-pc-windows-msvc", Triple::normalize("i686-pc-win32"));
   EXPECT_EQ("i686-unknown-windows-msvc", Triple::normalize("i686-win32"));
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 09b32bf09df0e..b8b035751637a 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -1192,8 +1192,8 @@ TEST_F(CloneInstruction, cloneKeyInstructions) {
     !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
     !1 = !DIFile(filename: "test.cpp",  directory: "")
     !2 = !{i32 1, !"Debug Info Version", i32 3}
-    !3 = distinct !DISubprogram(name: "test", scope: !0, unit: !0)
-    !4 = distinct !DISubprogram(name: "inlined", scope: !0, unit: !0, retainedNodes: !{!5})
+    !3 = distinct !DISubprogram(name: "test", scope: !0, unit: !0, keyInstructions: true)
+    !4 = distinct !DISubprogram(name: "inlined", scope: !0, unit: !0, retainedNodes: !{!5}, keyInstructions: true)
     !5 = !DILocalVariable(name: "awaitables", scope: !4)
     !6 = !DILocation(line: 1, scope: !4, inlinedAt: !8, atomGroup: 1, atomRank: 1)
     !7 = !DILocation(line: 2, scope: !3, atomGroup: 1, atomRank: 1)
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
index 3c66f985fe6e1..7f9a59bd428a0 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
@@ -315,7 +315,8 @@ define void @foo(ptr noalias %ptr, float %val) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 4);
@@ -359,7 +360,8 @@ define void @foo(ptr noalias %ptr, float %val) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 4);
@@ -419,7 +421,8 @@ define void @foo(ptr noalias %ptr, <2 x float> %val0, i64 %val1) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 3);
@@ -460,7 +463,8 @@ define void @foo(ptr noalias %ptr, float %v, <2 x float> %val) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 3);
@@ -503,7 +507,8 @@ define void @foo(ptr noalias %ptr, <2 x float> %val0) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/false,
+                              /*CollectLoads=*/true);
 
   // Find the loads
   auto It = std::next(BB->begin(), 2);
diff --git a/llvm/unittests/tools/llvm-mca/CMakeLists.txt b/llvm/unittests/tools/llvm-mca/CMakeLists.txt
index 9c69509b4b54d..26b00f8f1fd73 100644
--- a/llvm/unittests/tools/llvm-mca/CMakeLists.txt
+++ b/llvm/unittests/tools/llvm-mca/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   MC
   MCA
+  Passes
   Support
   TargetParser
   )
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index f916998fc3c10..6dec4110decbe 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -1541,7 +1541,7 @@ void AsmMatcherInfo::buildInfo() {
     Variant.Name = AsmVariant->getValueAsString("Name");
     Variant.AsmVariantNo = AsmVariant->getValueAsInt("Variant");
 
-    for (const CodeGenInstruction *CGI : Target.getInstructionsByEnumValue()) {
+    for (const CodeGenInstruction *CGI : Target.getInstructions()) {
 
       // If the tblgen -match-prefix option is specified (for tblgen hackers),
       // filter the set of instructions we consider.
@@ -3724,7 +3724,8 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
         "<< ActualIdx);\n";
   OS << "      if (ActualIdx < Operands.size())\n";
   OS << "        DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \" (\";\n";
-  OS << "                        Operands[ActualIdx]->print(dbgs()); dbgs() << "
+  OS << "                        Operands[ActualIdx]->print(dbgs(), "
+        "*getContext().getAsmInfo()); dbgs() << "
         "\"): \");\n";
   OS << "      else\n";
   OS << "        DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \": \");\n";
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index a3ea0f5e32e90..cbe645778fa3f 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -1314,7 +1314,7 @@ AsmWriterEmitter::AsmWriterEmitter(const RecordKeeper &R)
   unsigned Variant = AsmWriter->getValueAsInt("Variant");
 
   // Get the instruction numbering.
-  NumberedInstructions = Target.getInstructionsByEnumValue();
+  NumberedInstructions = Target.getInstructions();
 
   for (const auto &[Idx, I] : enumerate(NumberedInstructions)) {
     if (!I->AsmString.empty() && I->TheDef->getName() != "PHI")
diff --git a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
index 9d77631862ee5..61eedc077941e 100644
--- a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
@@ -11,17 +11,112 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/SetTheory.h"
 #include "llvm/TableGen/TableGenBackend.h"
+
 using namespace llvm;
 
 namespace {
+// Pair of a RuntimeLibcallPredicate and LibcallCallingConv to use as a map key.
+struct PredicateWithCC {
+  const Record *Predicate = nullptr;
+  const Record *CallingConv = nullptr;
+
+  PredicateWithCC() = default;
+  PredicateWithCC(std::pair<const Record *, const Record *> P)
+      : Predicate(P.first), CallingConv(P.second) {}
+
+  PredicateWithCC(const Record *P, const Record *C)
+      : Predicate(P), CallingConv(C) {}
+};
+
+inline bool operator==(PredicateWithCC LHS, PredicateWithCC RHS) {
+  return LHS.Predicate == RHS.Predicate && LHS.CallingConv == RHS.CallingConv;
+}
+} // namespace
+
+namespace llvm {
+template <> struct DenseMapInfo<PredicateWithCC, void> {
+  static inline PredicateWithCC getEmptyKey() {
+    return DenseMapInfo<
+        std::pair<const Record *, const Record *>>::getEmptyKey();
+  }
+
+  static inline PredicateWithCC getTombstoneKey() {
+    return DenseMapInfo<
+        std::pair<const Record *, const Record *>>::getTombstoneKey();
+  }
+
+  static unsigned getHashValue(const PredicateWithCC Val) {
+    auto Pair = std::make_pair(Val.Predicate, Val.CallingConv);
+    return DenseMapInfo<
+        std::pair<const Record *, const Record *>>::getHashValue(Pair);
+  }
+
+  static bool isEqual(PredicateWithCC LHS, PredicateWithCC RHS) {
+    return LHS == RHS;
+  }
+};
+} // namespace llvm
+
+namespace {
+
+class AvailabilityPredicate {
+  const Record *TheDef;
+  StringRef PredicateString;
+
+public:
+  AvailabilityPredicate(const Record *Def) : TheDef(Def) {
+    if (TheDef)
+      PredicateString = TheDef->getValueAsString("Cond");
+  }
+
+  const Record *getDef() const { return TheDef; }
+
+  bool isAlwaysAvailable() const { return PredicateString.empty(); }
+
+  void emitIf(raw_ostream &OS) const {
+    OS << "if (" << PredicateString << ") {\n";
+  }
+
+  void emitEndIf(raw_ostream &OS) const { OS << "}\n"; }
+
+  void emitTableVariableNameSuffix(raw_ostream &OS) const {
+    if (TheDef)
+      OS << '_' << TheDef->getName();
+  }
+};
+
+class RuntimeLibcallEmitter;
+class RuntimeLibcallImpl;
+
+/// Used to apply predicates to nested sets of libcalls.
+struct LibcallPredicateExpander : SetTheory::Expander {
+  const RuntimeLibcallEmitter &LibcallEmitter;
+  DenseMap<const RuntimeLibcallImpl *,
+           std::pair<std::vector<const Record *>, const Record *>> &Func2Preds;
+
+  LibcallPredicateExpander(
+      const RuntimeLibcallEmitter &LibcallEmitter,
+      DenseMap<const RuntimeLibcallImpl *,
+               std::pair<std::vector<const Record *>, const Record *>>
+          &Func2Preds)
+      : LibcallEmitter(LibcallEmitter), Func2Preds(Func2Preds) {}
+
+  void expand(SetTheory &ST, const Record *Def,
+              SetTheory::RecSet &Elts) override;
+};
 
 class RuntimeLibcall {
   const Record *TheDef = nullptr;
+  const size_t EnumVal;
 
 public:
   RuntimeLibcall() = delete;
-  RuntimeLibcall(const Record *Def) : TheDef(Def) { assert(Def); }
+  RuntimeLibcall(const Record *Def, size_t EnumVal)
+      : TheDef(Def), EnumVal(EnumVal) {
+    assert(Def);
+  }
 
   ~RuntimeLibcall() { assert(TheDef); }
 
@@ -29,6 +124,8 @@ class RuntimeLibcall {
 
   StringRef getName() const { return TheDef->getName(); }
 
+  size_t getEnumVal() const { return EnumVal; }
+
   void emitEnumEntry(raw_ostream &OS) const {
     OS << "RTLIB::" << TheDef->getValueAsString("Name");
   }
@@ -37,12 +134,14 @@ class RuntimeLibcall {
 class RuntimeLibcallImpl {
   const Record *TheDef;
   const RuntimeLibcall *Provides = nullptr;
+  const size_t EnumVal;
 
 public:
   RuntimeLibcallImpl(
       const Record *Def,
-      const DenseMap<const Record *, const RuntimeLibcall *> &ProvideMap)
-      : TheDef(Def) {
+      const DenseMap<const Record *, const RuntimeLibcall *> &ProvideMap,
+      size_t EnumVal)
+      : TheDef(Def), EnumVal(EnumVal) {
     if (const Record *ProvidesDef = Def->getValueAsDef("Provides"))
       Provides = ProvideMap.lookup(ProvidesDef);
   }
@@ -53,12 +152,18 @@ class RuntimeLibcallImpl {
 
   StringRef getName() const { return TheDef->getName(); }
 
+  size_t getEnumVal() const { return EnumVal; }
+
   const RuntimeLibcall *getProvides() const { return Provides; }
 
   StringRef getLibcallFuncName() const {
     return TheDef->getValueAsString("LibCallFuncName");
   }
 
+  const Record *getCallingConv() const {
+    return TheDef->getValueAsOptionalDef("CallingConv");
+  }
+
   void emitQuotedLibcallFuncName(raw_ostream &OS) const {
     OS << '\"' << getLibcallFuncName() << '\"';
   }
@@ -68,17 +173,36 @@ class RuntimeLibcallImpl {
   void emitEnumEntry(raw_ostream &OS) const {
     OS << "RTLIB::" << TheDef->getName();
   }
+
+  void emitSetImplCall(raw_ostream &OS) const {
+    OS << "setLibcallImpl(";
+    Provides->emitEnumEntry(OS);
+    OS << ", ";
+    emitEnumEntry(OS);
+    OS << "); // " << getLibcallFuncName() << '\n';
+  }
+
+  void emitTableEntry(raw_ostream &OS) const {
+    OS << '{';
+    Provides->emitEnumEntry(OS);
+    OS << ", ";
+    emitEnumEntry(OS);
+    OS << "}, // " << getLibcallFuncName() << '\n';
+  }
+
+  void emitSetCallingConv(raw_ostream &OS) const {}
+};
+
+struct LibcallsWithCC {
+  std::vector<const RuntimeLibcallImpl *> LibcallImpls;
+  const Record *CallingConv = nullptr;
 };
 
 class RuntimeLibcallEmitter {
 private:
   const RecordKeeper &Records;
-
   DenseMap<const Record *, const RuntimeLibcall *> Def2RuntimeLibcall;
-
-  const RuntimeLibcall *getRuntimeLibcall(const Record *Def) const {
-    return Def2RuntimeLibcall.lookup(Def);
-  }
+  DenseMap<const Record *, const RuntimeLibcallImpl *> Def2RuntimeLibcallImpl;
 
   std::vector<RuntimeLibcall> RuntimeLibcallDefList;
   std::vector<RuntimeLibcallImpl> RuntimeLibcallImplDefList;
@@ -86,16 +210,12 @@ class RuntimeLibcallEmitter {
   DenseMap<const RuntimeLibcall *, const RuntimeLibcallImpl *>
       LibCallToDefaultImpl;
 
-  void
-  emitTargetOverrideFunc(raw_ostream &OS, StringRef FuncName,
-                         ArrayRef<RuntimeLibcallImpl> LibCallImplList) const;
-
+private:
   void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
 
-  void emitWindowsArm64LibCallNameOverrides(raw_ostream &OS) const;
-
   void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
-  void emitGetInitRuntimeLibcallUtils(raw_ostream &OS) const;
+
+  void emitSystemRuntimeLibrarySetCalls(raw_ostream &OS) const;
 
 public:
   RuntimeLibcallEmitter(const RecordKeeper &R) : Records(R) {
@@ -105,8 +225,9 @@ class RuntimeLibcallEmitter {
 
     RuntimeLibcallDefList.reserve(AllRuntimeLibcalls.size());
 
+    size_t CallTypeEnumVal = 0;
     for (const Record *RuntimeLibcallDef : AllRuntimeLibcalls) {
-      RuntimeLibcallDefList.emplace_back(RuntimeLibcallDef);
+      RuntimeLibcallDefList.emplace_back(RuntimeLibcallDef, CallTypeEnumVal++);
       Def2RuntimeLibcall[RuntimeLibcallDef] = &RuntimeLibcallDefList.back();
     }
 
@@ -117,12 +238,15 @@ class RuntimeLibcallEmitter {
         Records.getAllDerivedDefinitions("RuntimeLibcallImpl");
     RuntimeLibcallImplDefList.reserve(AllRuntimeLibcallImpls.size());
 
+    size_t LibCallImplEnumVal = 1;
     for (const Record *LibCallImplDef : AllRuntimeLibcallImpls) {
-      RuntimeLibcallImplDefList.emplace_back(LibCallImplDef,
-                                             Def2RuntimeLibcall);
+      RuntimeLibcallImplDefList.emplace_back(LibCallImplDef, Def2RuntimeLibcall,
+                                             LibCallImplEnumVal++);
 
       RuntimeLibcallImpl &LibCallImpl = RuntimeLibcallImplDefList.back();
 
+      Def2RuntimeLibcallImpl[LibCallImplDef] = &LibCallImpl;
+
       // const RuntimeLibcallImpl &LibCallImpl =
       // RuntimeLibcallImplDefList.back();
       if (LibCallImpl.isDefault()) {
@@ -135,16 +259,12 @@ class RuntimeLibcallEmitter {
     }
   }
 
-  std::vector<RuntimeLibcallImpl>
-  getRuntimeLibcallImplSet(StringRef Name) const {
-    std::vector<RuntimeLibcallImpl> Result;
-    ArrayRef<const Record *> ImplSet =
-        Records.getAllDerivedDefinitionsIfDefined(Name);
-    Result.reserve(ImplSet.size());
+  const RuntimeLibcall *getRuntimeLibcall(const Record *Def) const {
+    return Def2RuntimeLibcall.lookup(Def);
+  }
 
-    for (const Record *LibCallImplDef : ImplSet)
-      Result.emplace_back(LibCallImplDef, Def2RuntimeLibcall);
-    return Result;
+  const RuntimeLibcallImpl *getRuntimeLibcallImpl(const Record *Def) const {
+    return Def2RuntimeLibcallImpl.lookup(Def);
   }
 
   void run(raw_ostream &OS);
@@ -152,102 +272,42 @@ class RuntimeLibcallEmitter {
 
 } // End anonymous namespace.
 
-/// Emit a method \p FuncName of RTLIB::RuntimeLibcallsInfo to override the
-/// libcall names in \p LibCallImplList.
-void RuntimeLibcallEmitter::emitTargetOverrideFunc(
-    raw_ostream &OS, StringRef FuncName,
-    ArrayRef<RuntimeLibcallImpl> LibCallImplList) const {
-  OS << "void llvm::RTLIB::RuntimeLibcallsInfo::" << FuncName << "() {\n";
-
-  if (LibCallImplList.empty()) {
-    OS << "  llvm_unreachable(\"override set not defined\");\n";
-  } else {
-    // for (const Record *LibCallImpl : LibCallImplList) {
-    for (const RuntimeLibcallImpl &LibCallImpl : LibCallImplList) {
-      const RuntimeLibcall *Provides = LibCallImpl.getProvides();
-      OS << "  LibcallImpls[";
-      Provides->emitEnumEntry(OS);
-      OS << "] = ";
-      LibCallImpl.emitEnumEntry(OS);
-      OS << ";\n";
-    }
-  }
-
-  OS << "}\n\n";
-}
-
 void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
   OS << "#ifdef GET_RUNTIME_LIBCALL_ENUM\n"
         "namespace llvm {\n"
         "namespace RTLIB {\n"
         "enum Libcall : unsigned short {\n";
 
-  size_t CallTypeEnumVal = 0;
   for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
     StringRef Name = LibCall.getName();
-    OS << "  " << Name << " = " << CallTypeEnumVal++ << ",\n";
+    OS << "  " << Name << " = " << LibCall.getEnumVal() << ",\n";
   }
 
   // TODO: Emit libcall names as string offset table.
 
-  OS << "  UNKNOWN_LIBCALL = " << CallTypeEnumVal
+  OS << "  UNKNOWN_LIBCALL = " << RuntimeLibcallDefList.size()
      << "\n};\n\n"
         "enum LibcallImpl : unsigned short {\n"
         "  Unsupported = 0,\n";
 
   // FIXME: Emit this in a different namespace. And maybe use enum class.
-  size_t LibCallImplEnumVal = 1;
   for (const RuntimeLibcallImpl &LibCall : RuntimeLibcallImplDefList) {
-    OS << "  " << LibCall.getName() << " = " << LibCallImplEnumVal++ << ", // "
+    OS << "  " << LibCall.getName() << " = " << LibCall.getEnumVal() << ", // "
        << LibCall.getLibcallFuncName() << '\n';
   }
 
-  OS << "  NumLibcallImpls = " << LibCallImplEnumVal
+  OS << "  NumLibcallImpls = " << RuntimeLibcallImplDefList.size() + 1
      << "\n};\n"
         "} // End namespace RTLIB\n"
         "} // End namespace llvm\n"
         "#endif\n\n";
 }
 
-void RuntimeLibcallEmitter::emitWindowsArm64LibCallNameOverrides(
-    raw_ostream &OS) const {
-  // FIXME: Stop treating this as a special case
-  OS << "void "
-        "llvm::RTLIB::RuntimeLibcallsInfo::setWindowsArm64LibCallNameOverrides("
-        ") {\n"
-        "  static const RTLIB::LibcallImpl "
-        "WindowsArm64RoutineImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {\n";
-  for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
-    auto I = LibCallToDefaultImpl.find(&LibCall);
-    if (I == LibCallToDefaultImpl.end())
-      OS << "    RTLIB::Unsupported,";
-    else {
-      const RuntimeLibcallImpl *LibCallImpl = I->second;
-      assert(LibCallImpl);
-      OS << "    RTLIB::arm64ec_" << LibCallImpl->getName() << ',';
-    }
-
-    OS << " // ";
-    LibCall.emitEnumEntry(OS);
-    OS << '\n';
-  }
-
-  OS << "    RTLIB::Unsupported // RTLIB::UNKNOWN_LIBCALL\n"
-        "  };\n\n"
-        "  std::memcpy(LibcallImpls, WindowsArm64RoutineImpls,\n"
-        "              sizeof(LibcallImpls));\n"
-        "  static_assert(sizeof(LibcallImpls) == "
-        "sizeof(WindowsArm64RoutineImpls),\n"
-        "                \"libcall array size should match\");\n"
-        "}\n#endif\n\n";
-}
-
 void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
     raw_ostream &OS) const {
   // TODO: Emit libcall names as string offset table.
 
-  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_NAMES\n"
-        "const RTLIB::LibcallImpl "
+  OS << "const RTLIB::LibcallImpl "
         "llvm::RTLIB::RuntimeLibcallsInfo::"
         "DefaultLibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {\n";
 
@@ -259,7 +319,7 @@ void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
       const RuntimeLibcallImpl *LibCallImpl = I->second;
       OS << "  ";
       LibCallImpl->emitEnumEntry(OS);
-      OS << ",";
+      OS << ',';
     }
 
     OS << " // ";
@@ -297,48 +357,229 @@ void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
     OS << '\n';
   }
   OS << "};\n\n";
+}
 
-  std::vector<RuntimeLibcallImpl> ZOSRuntimeLibcallImplList =
-      getRuntimeLibcallImplSet("ZOSRuntimeLibcallImpl");
-  emitTargetOverrideFunc(OS, "setZOSLibCallNameOverrides",
-                         ZOSRuntimeLibcallImplList);
+void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
+    raw_ostream &OS) const {
+  OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets("
+        "const llvm::Triple &TT) {\n"
+        "  struct LibcallImplPair {\n"
+        "    RTLIB::Libcall Func;\n"
+        "    RTLIB::LibcallImpl Impl;\n"
+        "  };\n";
+  ArrayRef<const Record *> AllLibs =
+      Records.getAllDerivedDefinitions("SystemRuntimeLibrary");
+
+  for (const Record *R : AllLibs) {
+    OS << '\n';
 
-  std::vector<RuntimeLibcallImpl> PPCRuntimeLibcallImplList =
-      getRuntimeLibcallImplSet("PPCRuntimeLibcallImpl");
-  emitTargetOverrideFunc(OS, "setPPCLibCallNameOverrides",
-                         PPCRuntimeLibcallImplList);
+    AvailabilityPredicate TopLevelPredicate(R->getValueAsDef("TriplePred"));
 
-  emitWindowsArm64LibCallNameOverrides(OS);
-}
+    OS << indent(2);
+    TopLevelPredicate.emitIf(OS);
 
-void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallUtils(
-    raw_ostream &OS) const {
-  // FIXME: Hack we shouldn't really need
-  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_UTILS\n"
-        "static inline bool isAtomicLibCall(llvm::RTLIB::Libcall LC) {\n"
-        "  switch (LC) {\n";
-  for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
-    StringRef Name = LibCall.getName();
-    if (Name.contains("ATOMIC")) {
-      OS << "  case ";
-      LibCall.emitEnumEntry(OS);
-      OS << ":\n";
+    if (const Record *DefaultCCClass =
+            R->getValueAsDef("DefaultLibcallCallingConv")) {
+      StringRef DefaultCC =
+          DefaultCCClass->getValueAsString("CallingConv").trim();
+
+      if (!DefaultCC.empty()) {
+        OS << "    const CallingConv::ID DefaultCC = " << DefaultCC << ";\n"
+           << "    for (CallingConv::ID &Entry : LibcallImplCallingConvs) {\n"
+              "      Entry = DefaultCC;\n"
+              "    }\n\n";
+      }
+    }
+
+    SetTheory Sets;
+
+    DenseMap<const RuntimeLibcallImpl *,
+             std::pair<std::vector<const Record *>, const Record *>>
+        Func2Preds;
+    Sets.addExpander("LibcallImpls", std::make_unique<LibcallPredicateExpander>(
+                                         *this, Func2Preds));
+
+    const SetTheory::RecVec *Elements =
+        Sets.expand(R->getValueAsDef("MemberList"));
+
+    // Sort to get deterministic output
+    SetVector<PredicateWithCC> PredicateSorter;
+    PredicateSorter.insert(
+        PredicateWithCC()); // No predicate or CC override first.
+
+    DenseMap<PredicateWithCC, LibcallsWithCC> Pred2Funcs;
+    for (const Record *Elt : *Elements) {
+      const RuntimeLibcallImpl *LibCallImpl = getRuntimeLibcallImpl(Elt);
+      if (!LibCallImpl) {
+        PrintError(R, "entry for SystemLibrary is not a RuntimeLibcallImpl");
+        PrintNote(Elt->getLoc(), "invalid entry `" + Elt->getName() + "`");
+        continue;
+      }
+
+      auto It = Func2Preds.find(LibCallImpl);
+      if (It == Func2Preds.end()) {
+        Pred2Funcs[PredicateWithCC()].LibcallImpls.push_back(LibCallImpl);
+        continue;
+      }
+
+      for (const Record *Pred : It->second.first) {
+        const Record *CC = It->second.second;
+        PredicateWithCC Key(Pred, CC);
+
+        auto &Entry = Pred2Funcs[Key];
+        Entry.LibcallImpls.push_back(LibCallImpl);
+        Entry.CallingConv = It->second.second;
+        PredicateSorter.insert(Key);
+      }
     }
+
+    SmallVector<PredicateWithCC, 0> SortedPredicates =
+        PredicateSorter.takeVector();
+
+    llvm::sort(SortedPredicates, [](PredicateWithCC A, PredicateWithCC B) {
+      StringRef AName = A.Predicate ? A.Predicate->getName() : "";
+      StringRef BName = B.Predicate ? B.Predicate->getName() : "";
+      return AName < BName;
+    });
+
+    for (PredicateWithCC Entry : SortedPredicates) {
+      AvailabilityPredicate SubsetPredicate(Entry.Predicate);
+      unsigned IndentDepth = 2;
+
+      auto It = Pred2Funcs.find(Entry);
+      if (It == Pred2Funcs.end())
+        continue;
+
+      if (!SubsetPredicate.isAlwaysAvailable()) {
+        IndentDepth = 4;
+
+        OS << indent(IndentDepth);
+        SubsetPredicate.emitIf(OS);
+      }
+
+      LibcallsWithCC &FuncsWithCC = It->second;
+
+      std::vector<const RuntimeLibcallImpl *> &Funcs = FuncsWithCC.LibcallImpls;
+
+      // Ensure we only emit a unique implementation per libcall in the
+      // selection table.
+      //
+      // FIXME: We need to generate separate functions for
+      // is-libcall-available and should-libcall-be-used to avoid this.
+      //
+      // This also makes it annoying to make use of the default set, since the
+      // entries from the default set may win over the replacements unless
+      // they are explicitly removed.
+      stable_sort(Funcs, [](const RuntimeLibcallImpl *A,
+                            const RuntimeLibcallImpl *B) {
+        return A->getProvides()->getEnumVal() < B->getProvides()->getEnumVal();
+      });
+
+      auto UniqueI = llvm::unique(
+          Funcs, [&](const RuntimeLibcallImpl *A, const RuntimeLibcallImpl *B) {
+            if (A->getProvides() == B->getProvides()) {
+              PrintWarning(R->getLoc(),
+                           Twine("conflicting implementations for libcall " +
+                                 A->getProvides()->getName() + ": " +
+                                 A->getLibcallFuncName() + ", " +
+                                 B->getLibcallFuncName()));
+              return true;
+            }
+
+            return false;
+          });
+
+      Funcs.erase(UniqueI, Funcs.end());
+
+      OS << indent(IndentDepth + 2)
+         << "static const LibcallImplPair LibraryCalls";
+      SubsetPredicate.emitTableVariableNameSuffix(OS);
+      OS << "[] = {\n";
+      for (const RuntimeLibcallImpl *LibCallImpl : Funcs) {
+        OS << indent(IndentDepth + 6);
+        LibCallImpl->emitTableEntry(OS);
+      }
+
+      OS << indent(IndentDepth + 2) << "};\n\n"
+         << indent(IndentDepth + 2)
+         << "for (const auto [Func, Impl] : LibraryCalls";
+      SubsetPredicate.emitTableVariableNameSuffix(OS);
+      OS << ") {\n"
+         << indent(IndentDepth + 4) << "setLibcallImpl(Func, Impl);\n";
+
+      if (FuncsWithCC.CallingConv) {
+        StringRef CCEnum =
+            FuncsWithCC.CallingConv->getValueAsString("CallingConv");
+        OS << indent(IndentDepth + 4) << "setLibcallImplCallingConv(Impl, "
+           << CCEnum << ");\n";
+      }
+
+      OS << indent(IndentDepth + 2) << "}\n";
+      OS << '\n';
+
+      if (!SubsetPredicate.isAlwaysAvailable()) {
+        OS << indent(IndentDepth);
+        SubsetPredicate.emitEndIf(OS);
+        OS << '\n';
+      }
+    }
+
+    OS << indent(4) << "return;\n" << indent(2);
+    TopLevelPredicate.emitEndIf(OS);
   }
 
-  OS << "    return true;\n"
-        "  default:\n"
-        "    return false;\n"
-        "  }\n\n"
-        "  llvm_unreachable(\"covered switch over libcalls\");\n"
-        "}\n#endif\n\n";
+  // Fallback to the old default set for manual table entries.
+  //
+  // TODO: Remove this when targets have switched to using generated tables by
+  // default.
+  OS << "  initDefaultLibCallImpls();\n";
+
+  OS << "}\n\n";
 }
 
 void RuntimeLibcallEmitter::run(raw_ostream &OS) {
   emitSourceFileHeader("Runtime LibCalls Source Fragment", OS, Records);
   emitGetRuntimeLibcallEnum(OS);
+
+  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_NAMES\n";
   emitGetInitRuntimeLibcallNames(OS);
-  emitGetInitRuntimeLibcallUtils(OS);
+  OS << "#endif\n\n";
+
+  OS << "#ifdef GET_SET_TARGET_RUNTIME_LIBCALL_SETS\n";
+  emitSystemRuntimeLibrarySetCalls(OS);
+  OS << "#endif\n\n";
+}
+
+void LibcallPredicateExpander::expand(SetTheory &ST, const Record *Def,
+                                      SetTheory::RecSet &Elts) {
+  assert(Def->isSubClassOf("LibcallImpls"));
+
+  SetTheory::RecSet TmpElts;
+
+  ST.evaluate(Def->getValueInit("MemberList"), TmpElts, Def->getLoc());
+
+  Elts.insert(TmpElts.begin(), TmpElts.end());
+
+  AvailabilityPredicate AP(Def->getValueAsDef("AvailabilityPredicate"));
+  const Record *CCClass = Def->getValueAsOptionalDef("CallingConv");
+
+  // This is assuming we aren't conditionally applying a calling convention to
+  // some subsets, and not another, but this doesn't appear to be used.
+
+  for (const Record *LibcallImplDef : TmpElts) {
+    const RuntimeLibcallImpl *LibcallImpl =
+        LibcallEmitter.getRuntimeLibcallImpl(LibcallImplDef);
+    if (!AP.isAlwaysAvailable() || CCClass) {
+      auto [It, Inserted] = Func2Preds.insert({LibcallImpl, {{}, CCClass}});
+      if (!Inserted) {
+        PrintError(
+            Def, "combining nested libcall set predicates currently unhandled");
+      }
+
+      It->second.first.push_back(AP.getDef());
+      It->second.second = CCClass;
+    }
+  }
 }
 
 static TableGen::Emitter::OptClass<RuntimeLibcallEmitter>
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index 14dffb438fcba..3ab6f40a6deec 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -475,7 +475,7 @@ void CodeEmitterGen::run(raw_ostream &O) {
   Target.reverseBitsForLittleEndianEncoding();
 
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
 
   if (Target.hasVariableLengthEncodings()) {
     emitVarLenCodeEmitter(Records, O);
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index eccd1aff422d0..90f0a2ac8c268 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -358,7 +358,7 @@ const Record *MapTableEmitter::getInstrForColumn(const Record *KeyInstr,
 
 unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) {
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
   StringRef Namespace = Target.getInstNamespace();
   ArrayRef<const ListInit *> ValueCols = InstrMapDesc.getValueCols();
   unsigned NumCol = ValueCols.size();
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 86be0d12d7b8a..c3b406a545072 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -2103,7 +2103,7 @@ TreePatternNodePtr TreePatternNode::clone() const {
 /// RemoveAllTypes - Recursively strip all the types of this tree.
 void TreePatternNode::RemoveAllTypes() {
   // Reset to unknown type.
-  std::fill(Types.begin(), Types.end(), TypeSetByHwMode());
+  llvm::fill(Types, TypeSetByHwMode());
   if (isLeaf())
     return;
   for (TreePatternNode &Child : children())
@@ -4122,8 +4122,7 @@ void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern,
 }
 
 void CodeGenDAGPatterns::InferInstructionFlags() {
-  ArrayRef<const CodeGenInstruction *> Instructions =
-      Target.getInstructionsByEnumValue();
+  ArrayRef<const CodeGenInstruction *> Instructions = Target.getInstructions();
 
   unsigned Errors = 0;
 
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index 6260eeee331e1..c43cc9afe1e3c 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -200,62 +200,6 @@ void CodeGenRegister::buildObjectGraph(CodeGenRegBank &RegBank) {
   }
 }
 
-namespace {
-
-// Iterate over all register units in a set of registers.
-class RegUnitIterator {
-  CodeGenRegister::Vec::const_iterator RegI, RegE;
-  CodeGenRegister::RegUnitList::iterator UnitI, UnitE;
-  static CodeGenRegister::RegUnitList Sentinel;
-
-public:
-  RegUnitIterator(const CodeGenRegister::Vec &Regs)
-      : RegI(Regs.begin()), RegE(Regs.end()) {
-
-    if (RegI == RegE) {
-      UnitI = Sentinel.end();
-      UnitE = Sentinel.end();
-    } else {
-      UnitI = (*RegI)->getRegUnits().begin();
-      UnitE = (*RegI)->getRegUnits().end();
-      advance();
-    }
-  }
-
-  bool isValid() const { return UnitI != UnitE; }
-
-  unsigned operator*() const {
-    assert(isValid());
-    return *UnitI;
-  }
-
-  const CodeGenRegister *getReg() const {
-    assert(isValid());
-    return *RegI;
-  }
-
-  /// Preincrement.  Move to the next unit.
-  void operator++() {
-    assert(isValid() && "Cannot advance beyond the last operand");
-    ++UnitI;
-    advance();
-  }
-
-protected:
-  void advance() {
-    while (UnitI == UnitE) {
-      if (++RegI == RegE)
-        break;
-      UnitI = (*RegI)->getRegUnits().begin();
-      UnitE = (*RegI)->getRegUnits().end();
-    }
-  }
-};
-
-CodeGenRegister::RegUnitList RegUnitIterator::Sentinel;
-
-} // end anonymous namespace
-
 // Inherit register units from subregisters.
 // Return true if the RegUnits changed.
 bool CodeGenRegister::inheritRegUnits(CodeGenRegBank &RegBank) {
@@ -1131,10 +1075,12 @@ void CodeGenRegisterClass::getSuperRegClasses(const CodeGenSubRegIndex *SubIdx,
 void CodeGenRegisterClass::buildRegUnitSet(
     const CodeGenRegBank &RegBank, std::vector<unsigned> &RegUnits) const {
   std::vector<unsigned> TmpUnits;
-  for (RegUnitIterator UnitI(Members); UnitI.isValid(); ++UnitI) {
-    const RegUnit &RU = RegBank.getRegUnit(*UnitI);
-    if (!RU.Artificial)
-      TmpUnits.push_back(*UnitI);
+  for (const CodeGenRegister *Reg : Members) {
+    for (unsigned UnitI : Reg->getRegUnits()) {
+      const RegUnit &RU = RegBank.getRegUnit(UnitI);
+      if (!RU.Artificial)
+        TmpUnits.push_back(UnitI);
+    }
   }
   llvm::sort(TmpUnits);
   std::unique_copy(TmpUnits.begin(), TmpUnits.end(),
diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
index a93b5be01754f..50346c29e8862 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -77,14 +77,12 @@ struct InstRegexOp : public SetTheory::Operator {
 
   void apply(SetTheory &ST, const DagInit *Expr, SetTheory::RecSet &Elts,
              ArrayRef<SMLoc> Loc) override {
-    ArrayRef<const CodeGenInstruction *> Instructions =
-        Target.getInstructionsByEnumValue();
-
-    unsigned NumGeneric = Target.getNumFixedInstructions();
-    unsigned NumPseudos = Target.getNumPseudoInstructions();
-    auto Generics = Instructions.slice(0, NumGeneric);
-    auto Pseudos = Instructions.slice(NumGeneric, NumPseudos);
-    auto NonPseudos = Instructions.slice(NumGeneric + NumPseudos);
+    ArrayRef<const CodeGenInstruction *> Generics =
+        Target.getGenericInstructions();
+    ArrayRef<const CodeGenInstruction *> Pseudos =
+        Target.getTargetPseudoInstructions();
+    ArrayRef<const CodeGenInstruction *> NonPseudos =
+        Target.getTargetNonPseudoInstructions();
 
     for (const Init *Arg : Expr->getArgs()) {
       const StringInit *SI = dyn_cast<StringInit>(Arg);
@@ -145,7 +143,7 @@ struct InstRegexOp : public SetTheory::Operator {
       }
 
       // Target instructions are split into two ranges: pseudo instructions
-      // first, than non-pseudos. Each range is in lexicographical order
+      // first, then non-pseudos. Each range is in lexicographical order
       // sorted by name. Find the sub-ranges that start with our prefix.
       struct Comp {
         bool operator()(const CodeGenInstruction *LHS, StringRef RHS) {
@@ -589,7 +587,7 @@ void CodeGenSchedModels::collectSchedRW() {
 
   // Find all SchedReadWrites referenced by instruction defs.
   ConstRecVec SWDefs, SRDefs;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     const Record *SchedDef = Inst->TheDef;
     if (SchedDef->isValueUnset("SchedRW"))
       continue;
@@ -838,7 +836,7 @@ void CodeGenSchedModels::collectSchedClasses() {
 
   // Create a SchedClass for each unique combination of itinerary class and
   // SchedRW list.
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     const Record *ItinDef = Inst->TheDef->getValueAsDef("Itinerary");
     IdxVec Writes, Reads;
     if (!Inst->TheDef->isValueUnset("SchedRW"))
@@ -863,7 +861,7 @@ void CodeGenSchedModels::collectSchedClasses() {
   LLVM_DEBUG(
       dbgs()
       << "\n+++ ITINERARIES and/or MACHINE MODELS (collectSchedClasses) +++\n");
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     StringRef InstName = Inst->TheDef->getName();
     unsigned SCIdx = getSchedClassIdx(*Inst);
     if (!SCIdx) {
@@ -1924,7 +1922,7 @@ void CodeGenSchedModels::checkCompleteness() {
     const bool HasItineraries = ProcModel.hasItineraries();
     if (!ProcModel.ModelDef->getValueAsBit("CompleteModel"))
       continue;
-    for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+    for (const CodeGenInstruction *Inst : Target.getInstructions()) {
       if (Inst->hasNoSchedulingInfo)
         continue;
       if (ProcModel.isUnsupported(*Inst))
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
index f519582387db9..a9002c5860fa3 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
@@ -88,7 +88,7 @@ StringRef CodeGenTarget::getName() const { return TargetRec->getName(); }
 /// namespace. The namespace is cached because it is requested multiple times.
 StringRef CodeGenTarget::getInstNamespace() const {
   if (InstNamespace.empty()) {
-    for (const CodeGenInstruction *Inst : getInstructionsByEnumValue()) {
+    for (const CodeGenInstruction *Inst : getInstructions()) {
       // We are not interested in the "TargetOpcode" namespace.
       if (Inst->Namespace != "TargetOpcode") {
         InstNamespace = Inst->Namespace;
@@ -211,10 +211,9 @@ void CodeGenTarget::ReadInstructions() const {
 
   // Parse the instructions defined in the .td file.
   for (const Record *R : Insts) {
-    auto &Inst = Instructions[R];
-    Inst = std::make_unique<CodeGenInstruction>(R);
-    if (Inst->isVariableLengthEncoding())
-      HasVariableLengthEncodings = true;
+    auto [II, _] =
+        InstructionMap.try_emplace(R, std::make_unique<CodeGenInstruction>(R));
+    HasVariableLengthEncodings |= II->second->isVariableLengthEncoding();
   }
 }
 
@@ -242,9 +241,9 @@ unsigned CodeGenTarget::getNumFixedInstructions() {
 /// Return all of the instructions defined by the target, ordered by
 /// their enum value.
 void CodeGenTarget::ComputeInstrsByEnum() const {
-  const auto &Insts = getInstructions();
+  const auto &InstMap = getInstructionMap();
   for (const char *Name : FixedInstrs) {
-    const CodeGenInstruction *Instr = GetInstByName(Name, Insts, Records);
+    const CodeGenInstruction *Instr = GetInstByName(Name, InstMap, Records);
     assert(Instr && "Missing target independent instruction");
     assert(Instr->Namespace == "TargetOpcode" && "Bad namespace");
     InstrsByEnum.push_back(Instr);
@@ -253,23 +252,24 @@ void CodeGenTarget::ComputeInstrsByEnum() const {
   assert(EndOfPredefines == getNumFixedInstructions() &&
          "Missing generic opcode");
 
-  for (const auto &I : Insts) {
-    const CodeGenInstruction *CGI = I.second.get();
+  for (const auto &[_, CGIUp] : InstMap) {
+    const CodeGenInstruction *CGI = CGIUp.get();
     if (CGI->Namespace != "TargetOpcode") {
       InstrsByEnum.push_back(CGI);
-      if (CGI->TheDef->getValueAsBit("isPseudo"))
-        ++NumPseudoInstructions;
+      NumPseudoInstructions += CGI->TheDef->getValueAsBit("isPseudo");
     }
   }
 
-  assert(InstrsByEnum.size() == Insts.size() && "Missing predefined instr");
+  assert(InstrsByEnum.size() == InstMap.size() && "Missing predefined instr");
 
   // All of the instructions are now in random order based on the map iteration.
   llvm::sort(
       InstrsByEnum.begin() + EndOfPredefines, InstrsByEnum.end(),
       [](const CodeGenInstruction *Rec1, const CodeGenInstruction *Rec2) {
-        const auto &D1 = *Rec1->TheDef;
-        const auto &D2 = *Rec2->TheDef;
+        const Record &D1 = *Rec1->TheDef;
+        const Record &D2 = *Rec2->TheDef;
+        // Sort all pseudo instructions before non-pseudo ones, and sort by name
+        // within.
         return std::tuple(!D1.getValueAsBit("isPseudo"), D1.getName()) <
                std::tuple(!D2.getValueAsBit("isPseudo"), D2.getName());
       });
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h
index 52871f33a301a..ac138344c83c6 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.h
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.h
@@ -59,7 +59,7 @@ class CodeGenTarget {
   const Record *TargetRec;
 
   mutable DenseMap<const Record *, std::unique_ptr<CodeGenInstruction>>
-      Instructions;
+      InstructionMap;
   mutable std::unique_ptr<CodeGenRegBank> RegBank;
   mutable ArrayRef<const Record *> RegAltNameIndices;
   mutable SmallVector<ValueTypeByHwMode, 8> LegalValueTypes;
@@ -154,43 +154,52 @@ class CodeGenTarget {
 
 private:
   DenseMap<const Record *, std::unique_ptr<CodeGenInstruction>> &
-  getInstructions() const {
-    if (Instructions.empty())
+  getInstructionMap() const {
+    if (InstructionMap.empty())
       ReadInstructions();
-    return Instructions;
+    return InstructionMap;
   }
 
 public:
   CodeGenInstruction &getInstruction(const Record *InstRec) const {
-    if (Instructions.empty())
-      ReadInstructions();
-    auto I = Instructions.find(InstRec);
-    assert(I != Instructions.end() && "Not an instruction");
+    auto I = getInstructionMap().find(InstRec);
+    assert(I != InstructionMap.end() && "Not an instruction");
     return *I->second;
   }
 
   /// Returns the number of predefined instructions.
   static unsigned getNumFixedInstructions();
 
-  /// Returns the number of pseudo instructions.
-  unsigned getNumPseudoInstructions() const {
-    if (InstrsByEnum.empty())
-      ComputeInstrsByEnum();
-    return NumPseudoInstructions;
-  }
-
   /// Return all of the instructions defined by the target, ordered by their
   /// enum value.
   /// The following order of instructions is also guaranteed:
   /// - fixed / generic instructions as declared in TargetOpcodes.def, in order;
   /// - pseudo instructions in lexicographical order sorted by name;
   /// - other instructions in lexicographical order sorted by name.
-  ArrayRef<const CodeGenInstruction *> getInstructionsByEnumValue() const {
+  ArrayRef<const CodeGenInstruction *> getInstructions() const {
     if (InstrsByEnum.empty())
       ComputeInstrsByEnum();
     return InstrsByEnum;
   }
 
+  // Functions that return various slices of `getInstructions`, ordered by
+  // their enum values.
+  ArrayRef<const CodeGenInstruction *> getGenericInstructions() const {
+    return getInstructions().take_front(getNumFixedInstructions());
+  }
+
+  ArrayRef<const CodeGenInstruction *> getTargetInstructions() const {
+    return getInstructions().drop_front(getNumFixedInstructions());
+  }
+
+  ArrayRef<const CodeGenInstruction *> getTargetPseudoInstructions() const {
+    return getTargetInstructions().take_front(NumPseudoInstructions);
+  }
+
+  ArrayRef<const CodeGenInstruction *> getTargetNonPseudoInstructions() const {
+    return getTargetInstructions().drop_front(NumPseudoInstructions);
+  }
+
   /// Return the integer enum value corresponding to this instruction record.
   unsigned getInstrIntValue(const Record *R) const {
     if (InstrsByEnum.empty())
@@ -198,12 +207,6 @@ class CodeGenTarget {
     return getInstruction(R).EnumVal;
   }
 
-  typedef ArrayRef<const CodeGenInstruction *>::const_iterator inst_iterator;
-  inst_iterator inst_begin() const {
-    return getInstructionsByEnumValue().begin();
-  }
-  inst_iterator inst_end() const { return getInstructionsByEnumValue().end(); }
-
   /// Return whether instructions have variable length encodings on this target.
   bool hasVariableLengthEncodings() const { return HasVariableLengthEncodings; }
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 327ac5f87777b..70141ba738bdb 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -1448,7 +1448,7 @@ void InstructionOpcodeMatcher::initOpcodeValuesMap(
     const CodeGenTarget &Target) {
   OpcodeValues.clear();
 
-  for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue())
+  for (const CodeGenInstruction *I : Target.getInstructions())
     OpcodeValues[I] = Target.getInstrIntValue(I->TheDef);
 }
 
diff --git a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
index e5e03b8bd8bd2..77a2ae60ad6f9 100644
--- a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
@@ -226,7 +226,7 @@ void VarLenInst::buildRec(const DagInit *DI) {
 void VarLenCodeEmitterGen::run(raw_ostream &OS) {
   CodeGenTarget Target(Records);
 
-  auto NumberedInstructions = Target.getInstructionsByEnumValue();
+  auto NumberedInstructions = Target.getInstructions();
 
   for (const CodeGenInstruction *CGI : NumberedInstructions) {
     const Record *R = CGI->TheDef;
diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp
index 4e307c6504c86..9a84343b98d95 100644
--- a/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -98,6 +98,10 @@ class CompressInstEmitter {
     // Tied operand index within the instruction.
     int TiedOpIdx = -1;
   };
+  struct ArgData {
+    unsigned DAGOpNo;
+    unsigned MIOpNo;
+  };
   struct CompressPat {
     // The source instruction definition.
     CodeGenInstruction Source;
@@ -114,8 +118,9 @@ class CompressInstEmitter {
 
     bool IsCompressOnly;
     CompressPat(const CodeGenInstruction &S, const CodeGenInstruction &D,
-                std::vector<const Record *> RF, IndexedMap<OpData> &SourceMap,
-                IndexedMap<OpData> &DestMap, bool IsCompressOnly)
+                std::vector<const Record *> RF,
+                const IndexedMap<OpData> &SourceMap,
+                const IndexedMap<OpData> &DestMap, bool IsCompressOnly)
         : Source(S), Dest(D), PatReqFeatures(std::move(RF)),
           SourceOperandMap(SourceMap), DestOperandMap(DestMap),
           IsCompressOnly(IsCompressOnly) {}
@@ -126,27 +131,23 @@ class CompressInstEmitter {
   std::vector<CompressPat> CompressPatterns;
   void addDagOperandMapping(const Record *Rec, const DagInit *Dag,
                             const CodeGenInstruction &Inst,
-                            IndexedMap<OpData> &OperandMap, bool IsSourceInst,
-                            unsigned *SourceLastTiedOpPtr);
+                            IndexedMap<OpData> &OperandMap,
+                            StringMap<ArgData> &Operands, bool IsSourceInst);
   void evaluateCompressPat(const Record *Compress);
   void emitCompressInstEmitter(raw_ostream &OS, EmitterType EType);
   bool validateTypes(const Record *DagOpType, const Record *InstOpType,
                      bool IsSourceInst);
   bool validateRegister(const Record *Reg, const Record *RegClass);
-  void createDagOperandMapping(const Record *Rec,
-                               StringMap<unsigned> &SourceOperands,
-                               StringMap<unsigned> &DestOperands,
-                               const DagInit *SourceDag, const DagInit *DestDag,
-                               IndexedMap<OpData> &SourceOperandMap,
-                               bool HasSourceTiedOp);
+  void checkDagOperandMapping(const Record *Rec,
+                              const StringMap<ArgData> &DestOperands,
+                              const DagInit *SourceDag, const DagInit *DestDag);
 
   void createInstOperandMapping(const Record *Rec, const DagInit *SourceDag,
                                 const DagInit *DestDag,
                                 IndexedMap<OpData> &SourceOperandMap,
                                 IndexedMap<OpData> &DestOperandMap,
-                                StringMap<unsigned> &SourceOperands,
-                                const CodeGenInstruction &DestInst,
-                                unsigned SourceLastTiedOp);
+                                StringMap<ArgData> &SourceOperands,
+                                const CodeGenInstruction &DestInst);
 
 public:
   CompressInstEmitter(const RecordKeeper &R) : Records(R), Target(R) {}
@@ -198,6 +199,10 @@ bool CompressInstEmitter::validateTypes(const Record *DagOpType,
   return true;
 }
 
+static bool validateArgsTypes(const Init *Arg1, const Init *Arg2) {
+  return cast<DefInit>(Arg1)->getDef() == cast<DefInit>(Arg2)->getDef();
+}
+
 /// The patterns in the Dag contain different types of operands:
 /// Register operands, e.g.: GPRC:$rs1; Fixed registers, e.g: X1; Immediate
 /// operands, e.g.: simm6:$imm; Fixed immediate operands, e.g.: 0. This function
@@ -209,11 +214,12 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
                                                const DagInit *Dag,
                                                const CodeGenInstruction &Inst,
                                                IndexedMap<OpData> &OperandMap,
-                                               bool IsSourceInst,
-                                               unsigned *SourceLastTiedOpPtr) {
+                                               StringMap<ArgData> &Operands,
+                                               bool IsSourceInst) {
   unsigned NumMIOperands = 0;
-  for (const auto &Op : Inst.Operands)
-    NumMIOperands += Op.MINumOperands;
+  if (!Inst.Operands.empty())
+    NumMIOperands =
+        Inst.Operands.back().MIOperandNo + Inst.Operands.back().MINumOperands;
   OperandMap.grow(NumMIOperands);
 
   // TiedCount keeps track of the number of operands skipped in Inst
@@ -223,16 +229,12 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
   // are represented.
   unsigned TiedCount = 0;
   unsigned OpNo = 0;
-  if (IsSourceInst)
-    *SourceLastTiedOpPtr = std::numeric_limits<unsigned int>::max();
   for (const auto &Opnd : Inst.Operands) {
     int TiedOpIdx = Opnd.getTiedRegister();
     if (-1 != TiedOpIdx) {
+      assert((unsigned)TiedOpIdx < OpNo);
       // Set the entry in OperandMap for the tied operand we're skipping.
-      OperandMap[OpNo].Kind = OperandMap[TiedOpIdx].Kind;
-      OperandMap[OpNo].Data = OperandMap[TiedOpIdx].Data;
-      if (IsSourceInst)
-        *SourceLastTiedOpPtr = OpNo;
+      OperandMap[OpNo] = OperandMap[TiedOpIdx];
       ++OpNo;
       ++TiedCount;
       continue;
@@ -289,6 +291,30 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
       } else {
         llvm_unreachable("Unhandled CompressPat argument type!");
       }
+
+      // Create a mapping between the operand name in the Dag (e.g. $rs1) and
+      // its index in the list of Dag operands and check that operands with the
+      // same name have the same type. For example in 'C_ADD $rs1, $rs2' we
+      // generate the mapping $rs1 --> 0, $rs2 ---> 1. If the operand appears
+      // twice in the same Dag (tied in the compressed instruction), we note
+      // the previous index in the TiedOpIdx field.
+      StringRef ArgName = Dag->getArgNameStr(DAGOpNo);
+      if (ArgName.empty())
+        continue;
+
+      if (IsSourceInst) {
+        auto It = Operands.find(ArgName);
+        if (It != Operands.end()) {
+          OperandMap[OpNo].TiedOpIdx = It->getValue().MIOpNo;
+          if (!validateArgsTypes(Dag->getArg(It->getValue().DAGOpNo),
+                                 Dag->getArg(DAGOpNo)))
+            PrintFatalError(Rec->getLoc(),
+                            "Input Operand '" + ArgName +
+                                "' has a mismatched tied operand!");
+        }
+      }
+
+      Operands[ArgName] = {DAGOpNo, OpNo};
     }
   }
 }
@@ -330,60 +356,29 @@ static bool verifyDagOpCount(const CodeGenInstruction &Inst, const DagInit *Dag,
   return true;
 }
 
-static bool validateArgsTypes(const Init *Arg1, const Init *Arg2) {
-  return cast<DefInit>(Arg1)->getDef() == cast<DefInit>(Arg2)->getDef();
-}
-
-// Creates a mapping between the operand name in the Dag (e.g. $rs1) and
-// its index in the list of Dag operands and checks that operands with the same
-// name have the same types. For example in 'C_ADD $rs1, $rs2' we generate the
-// mapping $rs1 --> 0, $rs2 ---> 1. If the operand appears twice in the (tied)
-// same Dag we use the last occurrence for indexing.
-void CompressInstEmitter::createDagOperandMapping(
-    const Record *Rec, StringMap<unsigned> &SourceOperands,
-    StringMap<unsigned> &DestOperands, const DagInit *SourceDag,
-    const DagInit *DestDag, IndexedMap<OpData> &SourceOperandMap,
-    bool HasSourceTiedOp) {
-  for (unsigned I = 0; I < DestDag->getNumArgs(); ++I) {
-    // Skip fixed immediates and registers, they were handled in
-    // addDagOperandMapping.
-    if ("" == DestDag->getArgNameStr(I))
-      continue;
-    DestOperands[DestDag->getArgNameStr(I)] = I;
-  }
+// Check that all names in the source DAG appear in the destionation DAG.
+void CompressInstEmitter::checkDagOperandMapping(
+    const Record *Rec, const StringMap<ArgData> &DestOperands,
+    const DagInit *SourceDag, const DagInit *DestDag) {
 
   for (unsigned I = 0; I < SourceDag->getNumArgs(); ++I) {
     // Skip fixed immediates and registers, they were handled in
     // addDagOperandMapping.
-    if ("" == SourceDag->getArgNameStr(I))
+    StringRef ArgName = SourceDag->getArgNameStr(I);
+    if (ArgName.empty())
       continue;
 
-    StringMap<unsigned>::iterator It =
-        SourceOperands.find(SourceDag->getArgNameStr(I));
-    if (It != SourceOperands.end()) {
-      // Operand sharing the same name in the Dag should be mapped as tied.
-      if (HasSourceTiedOp)
-        SourceOperandMap[I + 1].TiedOpIdx = It->getValue() + 1;
-      else
-        SourceOperandMap[I].TiedOpIdx = It->getValue();
-      if (!validateArgsTypes(SourceDag->getArg(It->getValue()),
-                             SourceDag->getArg(I)))
-        PrintFatalError(Rec->getLoc(),
-                        "Input Operand '" + SourceDag->getArgNameStr(I) +
-                            "' has a mismatched tied operand!\n");
-    }
-    It = DestOperands.find(SourceDag->getArgNameStr(I));
+    auto It = DestOperands.find(ArgName);
     if (It == DestOperands.end())
-      PrintFatalError(Rec->getLoc(), "Operand " + SourceDag->getArgNameStr(I) +
+      PrintFatalError(Rec->getLoc(), "Operand " + ArgName +
                                          " defined in Input Dag but not used in"
-                                         " Output Dag!\n");
+                                         " Output Dag!");
     // Input Dag operand types must match output Dag operand type.
-    if (!validateArgsTypes(DestDag->getArg(It->getValue()),
+    if (!validateArgsTypes(DestDag->getArg(It->getValue().DAGOpNo),
                            SourceDag->getArg(I)))
       PrintFatalError(Rec->getLoc(), "Type mismatch between Input and "
                                      "Output Dag operand '" +
-                                         SourceDag->getArgNameStr(I) + "'!");
-    SourceOperands[SourceDag->getArgNameStr(I)] = I;
+                                         ArgName + "'!");
   }
 }
 
@@ -393,8 +388,7 @@ void CompressInstEmitter::createDagOperandMapping(
 void CompressInstEmitter::createInstOperandMapping(
     const Record *Rec, const DagInit *SourceDag, const DagInit *DestDag,
     IndexedMap<OpData> &SourceOperandMap, IndexedMap<OpData> &DestOperandMap,
-    StringMap<unsigned> &SourceOperands, const CodeGenInstruction &DestInst,
-    unsigned SourceLastTiedOp) {
+    StringMap<ArgData> &SourceOperands, const CodeGenInstruction &DestInst) {
   // TiedCount keeps track of the number of operands skipped in Inst
   // operands list to get to the corresponding Dag operand.
   unsigned TiedCount = 0;
@@ -404,8 +398,8 @@ void CompressInstEmitter::createInstOperandMapping(
     int TiedInstOpIdx = Operand.getTiedRegister();
     if (TiedInstOpIdx != -1) {
       ++TiedCount;
-      DestOperandMap[OpNo].Data = DestOperandMap[TiedInstOpIdx].Data;
-      DestOperandMap[OpNo].Kind = DestOperandMap[TiedInstOpIdx].Kind;
+      assert((unsigned)TiedInstOpIdx < OpNo);
+      DestOperandMap[OpNo] = DestOperandMap[TiedInstOpIdx];
       if (DestOperandMap[OpNo].Kind == OpData::Operand)
         // No need to fill the SourceOperandMap here since it was mapped to
         // destination operand 'TiedInstOpIdx' in a previous iteration.
@@ -424,29 +418,21 @@ void CompressInstEmitter::createInstOperandMapping(
         continue;
 
       unsigned DagArgIdx = OpNo - TiedCount;
-      StringMap<unsigned>::iterator SourceOp =
-          SourceOperands.find(DestDag->getArgNameStr(DagArgIdx));
+      StringRef ArgName = DestDag->getArgNameStr(DagArgIdx);
+      auto SourceOp = SourceOperands.find(ArgName);
       if (SourceOp == SourceOperands.end())
         PrintFatalError(Rec->getLoc(),
-                        "Output Dag operand '" +
-                            DestDag->getArgNameStr(DagArgIdx) +
+                        "Output Dag operand '" + ArgName +
                             "' has no matching input Dag operand.");
 
-      assert(DestDag->getArgNameStr(DagArgIdx) ==
-                 SourceDag->getArgNameStr(SourceOp->getValue()) &&
+      assert(ArgName ==
+                 SourceDag->getArgNameStr(SourceOp->getValue().DAGOpNo) &&
              "Incorrect operand mapping detected!\n");
 
-      // Following four lines ensure the correct handling of a single tied
-      // operand in the Source Inst. SourceDagOp points to the position of
-      // appropriate Dag argument which is not correct in presence of tied
-      // operand in the Source Inst and must be incremented by 1 to reflect
-      // correct position of the operand in Source Inst
-      unsigned SourceDagOp = SourceOp->getValue();
-      if (SourceDagOp >= SourceLastTiedOp)
-        SourceDagOp++;
-      DestOperandMap[OpNo].Data.Operand = SourceDagOp;
-      SourceOperandMap[SourceDagOp].Data.Operand = OpNo;
-      LLVM_DEBUG(dbgs() << "    " << SourceDagOp << " ====> " << OpNo << "\n");
+      unsigned SourceOpNo = SourceOp->getValue().MIOpNo;
+      DestOperandMap[OpNo].Data.Operand = SourceOpNo;
+      SourceOperandMap[SourceOpNo].Data.Operand = OpNo;
+      LLVM_DEBUG(dbgs() << "    " << SourceOpNo << " ====> " << OpNo << "\n");
     }
   }
 }
@@ -505,27 +491,24 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
   // Fill the mapping from the source to destination instructions.
 
   IndexedMap<OpData> SourceOperandMap;
-  unsigned SourceLastTiedOp; // postion of the last tied operand in Source Inst
+  // Map from arg name to DAG operand number and MI operand number.
+  StringMap<ArgData> SourceOperands;
   // Create a mapping between source Dag operands and source Inst operands.
   addDagOperandMapping(Rec, SourceDag, SourceInst, SourceOperandMap,
-                       /*IsSourceInst*/ true, &SourceLastTiedOp);
+                       SourceOperands, /*IsSourceInst*/ true);
 
   IndexedMap<OpData> DestOperandMap;
+  // Map from arg name to DAG operand number and MI operand number.
+  StringMap<ArgData> DestOperands;
   // Create a mapping between destination Dag operands and destination Inst
   // operands.
-  addDagOperandMapping(Rec, DestDag, DestInst, DestOperandMap,
-                       /*IsSourceInst*/ false, nullptr);
-
-  StringMap<unsigned> SourceOperands;
-  StringMap<unsigned> DestOperands;
-  bool HasSourceTiedOp =
-      SourceLastTiedOp != std::numeric_limits<unsigned int>::max();
-  createDagOperandMapping(Rec, SourceOperands, DestOperands, SourceDag, DestDag,
-                          SourceOperandMap, HasSourceTiedOp);
+  addDagOperandMapping(Rec, DestDag, DestInst, DestOperandMap, DestOperands,
+                       /*IsSourceInst*/ false);
+
+  checkDagOperandMapping(Rec, DestOperands, SourceDag, DestDag);
   // Create operand mapping between the source and destination instructions.
   createInstOperandMapping(Rec, SourceDag, DestDag, SourceOperandMap,
-                           DestOperandMap, SourceOperands, DestInst,
-                           SourceLastTiedOp);
+                           DestOperandMap, SourceOperands, DestInst);
 
   // Get the target features for the CompressPat.
   std::vector<const Record *> PatReqFeatures;
@@ -534,9 +517,9 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
     return R->getValueAsBit("AssemblerMatcherPredicate");
   });
 
-  CompressPatterns.push_back(CompressPat(
-      SourceInst, DestInst, std::move(PatReqFeatures), SourceOperandMap,
-      DestOperandMap, Rec->getValueAsBit("isCompressOnly")));
+  CompressPatterns.emplace_back(SourceInst, DestInst, std::move(PatReqFeatures),
+                                SourceOperandMap, DestOperandMap,
+                                Rec->getValueAsBit("isCompressOnly"));
 }
 
 static void
@@ -623,7 +606,7 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &OS,
   if (!AsmWriter->getValueAsInt("PassSubtarget"))
     PrintFatalError(AsmWriter->getLoc(),
                     "'PassSubtarget' is false. SubTargetInfo object is needed "
-                    "for target features.\n");
+                    "for target features.");
 
   StringRef TargetName = Target.getName();
 
@@ -779,7 +762,7 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &OS,
               << ").getReg() ==  MI.getOperand("
               << SourceOperandMap[OpNo].TiedOpIdx << ").getReg()) &&\n";
         else
-          PrintFatalError("Unexpected tied operand types!\n");
+          PrintFatalError("Unexpected tied operand types!");
       }
       for (unsigned SubOp = 0; SubOp != SourceOperand.MINumOperands; ++SubOp) {
         // Check for fixed immediates\registers in the source instruction.
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index af25975f7c7ec..a50fd06435a10 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
@@ -83,7 +84,7 @@ static cl::opt<bool> LargeTable(
              "in the table instead of the default 16 bits."),
     cl::init(false), cl::cat(DisassemblerEmitterCat));
 
-static cl::opt<bool> UseFnTableInDecodetoMCInst(
+static cl::opt<bool> UseFnTableInDecodeToMCInst(
     "use-fn-table-in-decode-to-mcinst",
     cl::desc(
         "Use a table of function pointers instead of a switch case in the\n"
@@ -215,13 +216,6 @@ struct EncodingIDAndOpcode {
 using EncodingIDsVec = std::vector<EncodingIDAndOpcode>;
 using NamespacesHwModesMap = std::map<std::string, std::set<StringRef>>;
 
-raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) {
-  if (Value.EncodingDef != Value.Inst->TheDef)
-    OS << Value.EncodingDef->getName() << ":";
-  OS << Value.Inst->TheDef->getName();
-  return OS;
-}
-
 class DecoderEmitter {
   const RecordKeeper &RK;
   std::vector<EncodingAndInst> NumberedEncodings;
@@ -252,84 +246,96 @@ class DecoderEmitter {
   StringRef PredicateNamespace;
 };
 
-} // end anonymous namespace
-
 // The set (BIT_TRUE, BIT_FALSE, BIT_UNSET) represents a ternary logic system
 // for a bit value.
 //
 // BIT_UNFILTERED is used as the init value for a filter position.  It is used
 // only for filter processings.
-typedef enum : uint8_t {
-  BIT_FALSE,     // '0'
-  BIT_TRUE,      // '1'
-  BIT_UNSET,     // '?'
-  BIT_UNFILTERED // unfiltered
-} bit_value_t;
-
-static bool ValueSet(bit_value_t V) {
-  return (V == BIT_TRUE || V == BIT_FALSE);
-}
-
-static bool ValueNotSet(bit_value_t V) { return (V == BIT_UNSET); }
-
-static int Value(bit_value_t V) {
-  return ValueNotSet(V) ? -1 : (V == BIT_FALSE ? 0 : 1);
-}
+struct BitValue {
+  enum bit_value_t : uint8_t {
+    BIT_FALSE,     // '0'
+    BIT_TRUE,      // '1'
+    BIT_UNSET,     // '?', printed as '_'
+    BIT_UNFILTERED // unfiltered, printed as '.'
+  };
 
-static bit_value_t bitFromBits(const BitsInit &bits, unsigned index) {
-  if (const BitInit *bit = dyn_cast<BitInit>(bits.getBit(index)))
-    return bit->getValue() ? BIT_TRUE : BIT_FALSE;
+  BitValue(bit_value_t V) : V(V) {}
+  explicit BitValue(const Init *Init) {
+    if (const auto *Bit = dyn_cast<BitInit>(Init))
+      V = Bit->getValue() ? BIT_TRUE : BIT_FALSE;
+    else
+      V = BIT_UNSET;
+  }
+  BitValue(const BitsInit &Bits, unsigned Idx) : BitValue(Bits.getBit(Idx)) {}
 
-  // The bit is uninitialized.
-  return BIT_UNSET;
-}
+  bool isSet() const { return V == BIT_TRUE || V == BIT_FALSE; }
+  bool isUnset() const { return V == BIT_UNSET; }
+  std::optional<uint64_t> getValue() const {
+    if (isSet())
+      return static_cast<uint64_t>(V);
+    return std::nullopt;
+  }
 
-// Prints the bit value for each position.
-static void dumpBits(raw_ostream &OS, const BitsInit &bits) {
-  for (unsigned index = bits.getNumBits(); index > 0; --index) {
-    switch (bitFromBits(bits, index - 1)) {
-    case BIT_TRUE:
-      OS << "1";
-      break;
+  // For printing a bit value.
+  operator StringRef() const {
+    switch (V) {
     case BIT_FALSE:
-      OS << "0";
-      break;
+      return "0";
+    case BIT_TRUE:
+      return "1";
     case BIT_UNSET:
-      OS << "_";
-      break;
-    default:
-      llvm_unreachable("unexpected return value from bitFromBits");
+      return "_";
+    case BIT_UNFILTERED:
+      return ".";
     }
+    llvm_unreachable("Unknow bit value");
   }
+
+  bool operator==(bit_value_t Other) const { return Other == V; }
+  bool operator!=(bit_value_t Other) const { return Other != V; }
+
+private:
+  bit_value_t V;
+};
+
+} // end anonymous namespace
+
+static raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) {
+  if (Value.EncodingDef != Value.Inst->TheDef)
+    OS << Value.EncodingDef->getName() << ":";
+  OS << Value.Inst->TheDef->getName();
+  return OS;
 }
 
-static const BitsInit &getBitsField(const Record &def, StringRef str) {
-  const RecordVal *RV = def.getValue(str);
+// Prints the bit value for each position.
+static void dumpBits(raw_ostream &OS, const BitsInit &Bits) {
+  for (const Init *Bit : reverse(Bits.getBits()))
+    OS << BitValue(Bit);
+}
+
+static const BitsInit &getBitsField(const Record &Def, StringRef FieldName) {
+  const RecordVal *RV = Def.getValue(FieldName);
   if (const BitsInit *Bits = dyn_cast<BitsInit>(RV->getValue()))
     return *Bits;
 
-  // variable length instruction
-  VarLenInst VLI = VarLenInst(cast<DagInit>(RV->getValue()), RV);
+  // Handle variable length instructions.
+  VarLenInst VLI(cast<DagInit>(RV->getValue()), RV);
   SmallVector<const Init *, 16> Bits;
 
   for (const auto &SI : VLI) {
-    if (const BitsInit *BI = dyn_cast<BitsInit>(SI.Value)) {
-      for (unsigned Idx = 0U; Idx < BI->getNumBits(); ++Idx) {
-        Bits.push_back(BI->getBit(Idx));
-      }
-    } else if (const BitInit *BI = dyn_cast<BitInit>(SI.Value)) {
+    if (const BitsInit *BI = dyn_cast<BitsInit>(SI.Value))
+      llvm::append_range(Bits, BI->getBits());
+    else if (const BitInit *BI = dyn_cast<BitInit>(SI.Value))
       Bits.push_back(BI);
-    } else {
-      for (unsigned Idx = 0U; Idx < SI.BitWidth; ++Idx)
-        Bits.push_back(UnsetInit::get(def.getRecords()));
-    }
+    else
+      Bits.append(SI.BitWidth, UnsetInit::get(Def.getRecords()));
   }
 
-  return *BitsInit::get(def.getRecords(), Bits);
+  return *BitsInit::get(Def.getRecords(), Bits);
 }
 
 // Representation of the instruction to work on.
-typedef std::vector<bit_value_t> insn_t;
+typedef std::vector<BitValue> insn_t;
 
 namespace {
 
@@ -480,7 +486,7 @@ class FilterChooser {
 
   // Array of bit values passed down from our parent.
   // Set to all BIT_UNFILTERED's for Parent == NULL.
-  std::vector<bit_value_t> FilterBitValues;
+  std::vector<BitValue> FilterBitValues;
 
   // Links to the FilterChooser above us in the decoding tree.
   const FilterChooser *Parent;
@@ -506,15 +512,15 @@ class FilterChooser {
                 const std::map<unsigned, std::vector<OperandInfo>> &Ops,
                 unsigned BW, const DecoderEmitter *E)
       : AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
-        FilterBitValues(BW, BIT_UNFILTERED), Parent(nullptr), BestIndex(-1),
-        BitWidth(BW), Emitter(E) {
+        FilterBitValues(BW, BitValue::BIT_UNFILTERED), Parent(nullptr),
+        BestIndex(-1), BitWidth(BW), Emitter(E) {
     doFilter();
   }
 
   FilterChooser(ArrayRef<EncodingAndInst> Insts,
                 ArrayRef<EncodingIDAndOpcode> IDs,
                 const std::map<unsigned, std::vector<OperandInfo>> &Ops,
-                const std::vector<bit_value_t> &ParentFilterBitValues,
+                const std::vector<BitValue> &ParentFilterBitValues,
                 const FilterChooser &parent)
       : AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
         FilterBitValues(ParentFilterBitValues), Parent(&parent), BestIndex(-1),
@@ -532,7 +538,7 @@ class FilterChooser {
   void insnWithID(insn_t &Insn, unsigned Opcode) const {
     const Record *EncodingDef = AllInstructions[Opcode].EncodingDef;
     const BitsInit &Bits = getBitsField(*EncodingDef, "Inst");
-    Insn.resize(std::max(BitWidth, Bits.getNumBits()), BIT_UNSET);
+    Insn.resize(std::max(BitWidth, Bits.getNumBits()), BitValue::BIT_UNSET);
     // We may have a SoftFail bitmask, which specifies a mask where an encoding
     // may differ from the value in "Inst" and yet still be valid, but the
     // disassembler should return SoftFail instead of Success.
@@ -541,22 +547,13 @@ class FilterChooser {
     const RecordVal *RV = EncodingDef->getValue("SoftFail");
     const BitsInit *SFBits = RV ? dyn_cast<BitsInit>(RV->getValue()) : nullptr;
     for (unsigned i = 0; i < Bits.getNumBits(); ++i) {
-      if (SFBits && bitFromBits(*SFBits, i) == BIT_TRUE)
-        Insn[i] = BIT_UNSET;
+      if (SFBits && BitValue(*SFBits, i) == BitValue::BIT_TRUE)
+        Insn[i] = BitValue::BIT_UNSET;
       else
-        Insn[i] = bitFromBits(Bits, i);
+        Insn[i] = BitValue(Bits, i);
     }
   }
 
-  // Emit the name of the encoding/instruction pair.
-  void emitNameWithID(raw_ostream &OS, unsigned Opcode) const {
-    const Record *EncodingDef = AllInstructions[Opcode].EncodingDef;
-    const Record *InstDef = AllInstructions[Opcode].Inst->TheDef;
-    if (EncodingDef != InstDef)
-      OS << EncodingDef->getName() << ":";
-    OS << InstDef->getName();
-  }
-
   // Populates the field of the insn given the start position and the number of
   // consecutive bits to scan for.
   //
@@ -568,7 +565,7 @@ class FilterChooser {
 
   /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
   /// filter array as a series of chars.
-  void dumpFilterArray(raw_ostream &OS, ArrayRef<bit_value_t> Filter) const;
+  void dumpFilterArray(raw_ostream &OS, ArrayRef<BitValue> Filter) const;
 
   /// dumpStack - dumpStack traverses the filter chooser chain and calls
   /// dumpFilterArray on each filter chooser up to the top level one.
@@ -579,8 +576,8 @@ class FilterChooser {
     return Filters[BestIndex];
   }
 
-  bool PositionFiltered(unsigned i) const {
-    return ValueSet(FilterBitValues[i]);
+  bool PositionFiltered(unsigned Idx) const {
+    return FilterBitValues[Idx].isSet();
   }
 
   // Calculates the island(s) needed to decode the instruction.
@@ -697,12 +694,12 @@ Filter::Filter(const FilterChooser &owner, unsigned startBit, unsigned numBits,
 // match the remaining undecoded encoding bits against the singleton.
 void Filter::recurse() {
   // Starts by inheriting our parent filter chooser's filter bit values.
-  std::vector<bit_value_t> BitValueArray(Owner.FilterBitValues);
+  std::vector<BitValue> BitValueArray(Owner.FilterBitValues);
 
   if (!VariableInstructions.empty()) {
     // Conservatively marks each segment position as BIT_UNSET.
     for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex)
-      BitValueArray[StartBit + bitIndex] = BIT_UNSET;
+      BitValueArray[StartBit + bitIndex] = BitValue::BIT_UNSET;
 
     // Delegates to an inferior filter chooser for further processing on this
     // group of instructions whose segment values are variable.
@@ -723,12 +720,10 @@ void Filter::recurse() {
   // Otherwise, create sub choosers.
   for (const auto &Inst : FilteredInstructions) {
     // Marks all the segment positions with either BIT_TRUE or BIT_FALSE.
-    for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex) {
-      if (Inst.first & (1ULL << bitIndex))
-        BitValueArray[StartBit + bitIndex] = BIT_TRUE;
-      else
-        BitValueArray[StartBit + bitIndex] = BIT_FALSE;
-    }
+    for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex)
+      BitValueArray[StartBit + bitIndex] = Inst.first & (1ULL << bitIndex)
+                                               ? BitValue::BIT_TRUE
+                                               : BitValue::BIT_FALSE;
 
     // Delegates to an inferior filter chooser for further processing on this
     // category of instructions.
@@ -1087,7 +1082,7 @@ void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS,
       "DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const "
       "MCDisassembler *Decoder, bool &DecodeComplete";
 
-  if (UseFnTableInDecodetoMCInst) {
+  if (UseFnTableInDecodeToMCInst) {
     // Emit a function for each case first.
     for (const auto &[Index, Decoder] : enumerate(Decoders)) {
       OS << Indent << "template <typename InsnType>\n";
@@ -1110,7 +1105,7 @@ void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS,
   Indent += 2;
   OS << Indent << "DecodeComplete = true;\n";
 
-  if (UseFnTableInDecodetoMCInst) {
+  if (UseFnTableInDecodeToMCInst) {
     // Build a table of function pointers.
     OS << Indent << "using DecodeFnTy = DecodeStatus (*)(" << DecodeParams
        << ");\n";
@@ -1151,10 +1146,10 @@ std::pair<bool, uint64_t> FilterChooser::fieldFromInsn(const insn_t &Insn,
   uint64_t Field = 0;
 
   for (unsigned i = 0; i < NumBits; ++i) {
-    if (Insn[StartBit + i] == BIT_UNSET)
+    if (Insn[StartBit + i] == BitValue::BIT_UNSET)
       return {false, Field};
 
-    if (Insn[StartBit + i] == BIT_TRUE)
+    if (Insn[StartBit + i] == BitValue::BIT_TRUE)
       Field = Field | (1ULL << i);
   }
 
@@ -1164,23 +1159,9 @@ std::pair<bool, uint64_t> FilterChooser::fieldFromInsn(const insn_t &Insn,
 /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
 /// filter array as a series of chars.
 void FilterChooser::dumpFilterArray(raw_ostream &OS,
-                                    ArrayRef<bit_value_t> Filter) const {
-  for (unsigned bitIndex = BitWidth; bitIndex > 0; bitIndex--) {
-    switch (Filter[bitIndex - 1]) {
-    case BIT_UNFILTERED:
-      OS << ".";
-      break;
-    case BIT_UNSET:
-      OS << "_";
-      break;
-    case BIT_TRUE:
-      OS << "1";
-      break;
-    case BIT_FALSE:
-      OS << "0";
-      break;
-    }
-  }
+                                    ArrayRef<BitValue> Filter) const {
+  for (unsigned bitIndex = BitWidth; bitIndex > 0; bitIndex--)
+    OS << Filter[bitIndex - 1];
 }
 
 /// dumpStack - dumpStack traverses the filter chooser chain and calls
@@ -1211,28 +1192,28 @@ unsigned FilterChooser::getIslands(std::vector<Island> &Islands,
   unsigned State = 0;
 
   for (unsigned i = 0; i < BitWidth; ++i) {
-    int64_t Val = Value(Insn[i]);
+    std::optional<uint64_t> Val = Insn[i].getValue();
     bool Filtered = PositionFiltered(i);
     switch (State) {
     default:
       llvm_unreachable("Unreachable code!");
     case 0:
     case 1:
-      if (Filtered || Val == -1) {
+      if (Filtered || !Val) {
         State = 1; // Still in Water
       } else {
         State = 2; // Into the Island
         StartBit = i;
-        FieldVal = Val;
+        FieldVal = *Val;
       }
       break;
     case 2:
-      if (Filtered || Val == -1) {
+      if (Filtered || !Val) {
         State = 1; // Into the Water
         Islands.push_back({StartBit, i - StartBit, FieldVal});
       } else {
         State = 2; // Still in Island
-        FieldVal |= Val << (i - StartBit);
+        FieldVal |= *Val << (i - StartBit);
       }
       break;
     }
@@ -1311,7 +1292,7 @@ std::pair<unsigned, bool> FilterChooser::getDecoderIndex(DecoderSet &Decoders,
   // FIXME: emitDecoder() function can take a buffer directly rather than
   // a stream.
   raw_svector_ostream S(Decoder);
-  indent Indent(UseFnTableInDecodetoMCInst ? 2 : 4);
+  indent Indent(UseFnTableInDecodeToMCInst ? 2 : 4);
   bool HasCompleteDecoder = emitDecoder(S, Indent, Opc);
 
   // Using the full decoder string as the key value here is a bit
@@ -1453,22 +1434,19 @@ void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
   APInt PositiveMask(BitWidth, 0ULL);
   APInt NegativeMask(BitWidth, 0ULL);
   for (unsigned i = 0; i < BitWidth; ++i) {
-    bit_value_t B = bitFromBits(*SFBits, i);
-    bit_value_t IB = bitFromBits(*InstBits, i);
+    BitValue B(*SFBits, i);
+    BitValue IB(*InstBits, i);
 
-    if (B != BIT_TRUE)
+    if (B != BitValue::BIT_TRUE)
       continue;
 
-    switch (IB) {
-    case BIT_FALSE:
+    if (IB == BitValue::BIT_FALSE) {
       // The bit is meant to be false, so emit a check to see if it is true.
       PositiveMask.setBit(i);
-      break;
-    case BIT_TRUE:
+    } else if (IB == BitValue::BIT_TRUE) {
       // The bit is meant to be true, so emit a check to see if it is false.
       NegativeMask.setBit(i);
-      break;
-    default:
+    } else {
       // The bit is not set; this must be an error!
       errs() << "SoftFail Conflict: bit SoftFail{" << i << "} in "
              << AllInstructions[Opc] << " is set but Inst{" << i
@@ -1651,8 +1629,7 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   // FILTERED bit positions provide no entropy and are not worthy of pursuing.
   // Filter::recurse() set either BIT_TRUE or BIT_FALSE for each position.
   for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex)
-    if (FilterBitValues[BitIndex] == BIT_TRUE ||
-        FilterBitValues[BitIndex] == BIT_FALSE)
+    if (FilterBitValues[BitIndex].isSet())
       bitAttrs[BitIndex] = ATTR_FILTERED;
 
   for (const auto &OpcPair : Opcodes) {
@@ -1663,17 +1640,17 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
     for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) {
       switch (bitAttrs[BitIndex]) {
       case ATTR_NONE:
-        if (insn[BitIndex] == BIT_UNSET)
+        if (insn[BitIndex] == BitValue::BIT_UNSET)
           bitAttrs[BitIndex] = ATTR_ALL_UNSET;
         else
           bitAttrs[BitIndex] = ATTR_ALL_SET;
         break;
       case ATTR_ALL_SET:
-        if (insn[BitIndex] == BIT_UNSET)
+        if (insn[BitIndex] == BitValue::BIT_UNSET)
           bitAttrs[BitIndex] = ATTR_MIXED;
         break;
       case ATTR_ALL_UNSET:
-        if (insn[BitIndex] != BIT_UNSET)
+        if (insn[BitIndex] != BitValue::BIT_UNSET)
           bitAttrs[BitIndex] = ATTR_MIXED;
         break;
       case ATTR_MIXED:
@@ -1868,7 +1845,7 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
     return;
   }
 
-  // We don't know how to decode these instructions!  Dump the
+  // We don't know how to decode these instructions! Dump the
   // conflict set and bail.
 
   // Print out useful conflict information for postmortem analysis.
@@ -1877,14 +1854,12 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
   dumpStack(errs(), "\t\t");
 
   for (auto Opcode : Opcodes) {
-    errs() << '\t';
-    emitNameWithID(errs(), Opcode.EncodingID);
-    errs() << " ";
-    dumpBits(
-        errs(),
-        getBitsField(*AllInstructions[Opcode.EncodingID].EncodingDef, "Inst"));
+    const EncodingAndInst &Enc = AllInstructions[Opcode.EncodingID];
+    errs() << '\t' << Enc << ' ';
+    dumpBits(errs(), getBitsField(*Enc.EncodingDef, "Inst"));
     errs() << '\n';
   }
+  PrintFatalError("Decoding conflict encountered");
 }
 
 static std::string findOperandDecoderMethod(const Record *Record) {
@@ -2597,7 +2572,7 @@ namespace {
   if (HwModeNames.empty())
     HwModeNames.push_back("");
 
-  const auto &NumberedInstructions = Target.getInstructionsByEnumValue();
+  const auto &NumberedInstructions = Target.getInstructions();
   NumberedEncodings.reserve(NumberedInstructions.size());
   for (const auto &NumberedInstruction : NumberedInstructions) {
     const Record *InstDef = NumberedInstruction->TheDef;
diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp
index 70d835e699ffe..15a11a3a99406 100644
--- a/llvm/utils/TableGen/DisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp
@@ -103,8 +103,7 @@ static void emitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
   if (Target.getName() == "X86") {
     DisassemblerTables Tables;
 
-    for (const auto &[Idx, NumberedInst] :
-         enumerate(Target.getInstructionsByEnumValue()))
+    for (const auto &[Idx, NumberedInst] : enumerate(Target.getInstructions()))
       RecognizableInstr::processInstr(Tables, *NumberedInst, Idx);
 
     if (Tables.hasConflicts()) {
@@ -120,7 +119,7 @@ static void emitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
   // below (which depends on a Size table-gen Record), and also uses a custom
   // disassembler.
   if (Target.getName() == "WebAssembly") {
-    emitWebAssemblyDisassemblerTables(OS, Target.getInstructionsByEnumValue());
+    emitWebAssemblyDisassemblerTables(OS, Target.getInstructions());
     return;
   }
 
diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp
index a8a234675dec1..7ca34ad05a72d 100644
--- a/llvm/utils/TableGen/InstrDocsEmitter.cpp
+++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp
@@ -72,7 +72,7 @@ static void EmitInstrDocs(const RecordKeeper &RK, raw_ostream &OS) {
   writeTitle(Title, OS);
   OS << "\n";
 
-  for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *II : Target.getInstructions()) {
     const Record *Inst = II->TheDef;
 
     // Don't print the target-independent instructions.
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index e72055b0b5037..f028fcd082642 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -91,10 +91,10 @@ class InstrInfoEmitter {
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
   void emitOperandNameMappings(
       raw_ostream &OS, const CodeGenTarget &Target,
-      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
+      ArrayRef<const CodeGenInstruction *> TargetInstructions);
   void emitLogicalOperandSizeMappings(
       raw_ostream &OS, StringRef Namespace,
-      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
+      ArrayRef<const CodeGenInstruction *> TargetInstructions);
 
   // Operand information.
   unsigned CollectOperandInfo(OperandInfoListTy &OperandInfoList,
@@ -201,7 +201,7 @@ InstrInfoEmitter::CollectOperandInfo(OperandInfoListTy &OperandInfoList,
                                      OperandInfoMapTy &OperandInfoMap) {
   const CodeGenTarget &Target = CDP.getTargetInfo();
   unsigned Offset = 0;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     OperandInfoTy OperandInfo = GetOperandInfo(*Inst);
     if (OperandInfoMap.try_emplace(OperandInfo, Offset).second) {
       OperandInfoList.push_back(OperandInfo);
@@ -234,9 +234,12 @@ void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS,
 /// - A function called getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
 ///   for looking up the operand index for an instruction, given a value from
 ///   OpName enum
+///
+/// Fixed/Predefined instructions do not have UseNamedOperandTable enabled, so
+/// we can just skip them. Hence accept just the TargetInstructions.
 void InstrInfoEmitter::emitOperandNameMappings(
     raw_ostream &OS, const CodeGenTarget &Target,
-    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
+    ArrayRef<const CodeGenInstruction *> TargetInstructions) {
   StringRef Namespace = Target.getInstNamespace();
 
   /// To facilitate assigning OpName enum values in the sorted alphabetical
@@ -260,9 +263,7 @@ void InstrInfoEmitter::emitOperandNameMappings(
 
   // Fixed/Predefined instructions do not have UseNamedOperandTable enabled, so
   // we can just skip them.
-  const unsigned NumFixedInsts = Target.getNumFixedInstructions();
-  for (const CodeGenInstruction *Inst :
-       NumberedInstructions.drop_front(NumFixedInsts)) {
+  for (const CodeGenInstruction *Inst : TargetInstructions) {
     if (!Inst->TheDef->getValueAsBit("UseNamedOperandTable"))
       continue;
     std::map<unsigned, unsigned> OpList;
@@ -476,19 +477,18 @@ void InstrInfoEmitter::emitOperandTypeMappings(
   OS << "#endif // GET_INSTRINFO_MEM_OPERAND_SIZE\n\n";
 }
 
+// Fixed/Predefined instructions do not have UseLogicalOperandMappings
+// enabled, so we can just skip them. Hence accept TargetInstructions.
 void InstrInfoEmitter::emitLogicalOperandSizeMappings(
     raw_ostream &OS, StringRef Namespace,
-    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
+    ArrayRef<const CodeGenInstruction *> TargetInstructions) {
   std::map<std::vector<unsigned>, unsigned> LogicalOpSizeMap;
   std::map<unsigned, std::vector<std::string>> InstMap;
 
   size_t LogicalOpListSize = 0U;
   std::vector<unsigned> LogicalOpList;
 
-  // Fixed/Predefined instructions do not have UseLogicalOperandMappings
-  // enabled, so we can just skip them.
-  const unsigned NumFixedInsts = CDP.getTargetInfo().getNumFixedInstructions();
-  for (const auto *Inst : NumberedInstructions.drop_front(NumFixedInsts)) {
+  for (const auto *Inst : TargetInstructions) {
     if (!Inst->TheDef->getValueAsBit("UseLogicalOperandMappings"))
       continue;
 
@@ -645,7 +645,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
       Target.getName(), "", "computeAvailableFeatures", SubtargetFeatures, OS);
 
   std::vector<std::vector<const Record *>> FeatureBitsets;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     FeatureBitsets.emplace_back();
     for (const Record *Predicate :
          Inst->TheDef->getValueAsListOfDefs("Predicates")) {
@@ -696,7 +696,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
      << "  static constexpr " << getMinimalTypeForRange(FeatureBitsets.size())
      << " RequiredFeaturesRefs[] = {\n";
   unsigned InstIdx = 0;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     OS << "    CEFBS";
     unsigned NumPredicates = 0;
     for (const Record *Predicate :
@@ -818,7 +818,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   const CodeGenTarget &Target = CDP.getTargetInfo();
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
 
   emitEnums(OS, NumberedInstructions);
 
@@ -1057,9 +1057,12 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n";
 
+  ArrayRef<const CodeGenInstruction *> TargetInstructions =
+      Target.getTargetInstructions();
+
   if (HasUseNamedOperandTable) {
     Timer.startTimer("Emit operand name mappings");
-    emitOperandNameMappings(OS, Target, NumberedInstructions);
+    emitOperandNameMappings(OS, Target, TargetInstructions);
   }
 
   Timer.startTimer("Emit operand type mappings");
@@ -1067,7 +1070,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   if (HasUseLogicalOperandMappings) {
     Timer.startTimer("Emit logical operand size mappings");
-    emitLogicalOperandSizeMappings(OS, TargetName, NumberedInstructions);
+    emitLogicalOperandSizeMappings(OS, TargetName, TargetInstructions);
   }
 
   Timer.startTimer("Emit helper methods");
diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp
index 3c422a32dcaf7..ed7a4fedbf730 100644
--- a/llvm/utils/TableGen/X86DisassemblerTables.cpp
+++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp
@@ -603,8 +603,7 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_W_OPSIZE_KZ_B_U:
     return false;
   default:
-    errs() << "Unknown instruction class: "
-           << stringForContext((InstructionContext)parent) << "\n";
+    errs() << "Unknown instruction class: " << stringForContext(parent) << "\n";
     llvm_unreachable("Unknown instruction class");
   }
 }
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index bcc5712b9154c..d63570a88a4cb 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -625,7 +625,7 @@ void X86FoldTablesEmitter::run(raw_ostream &OS) {
   std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts;
 
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
 
   for (const CodeGenInstruction *Inst : NumberedInstructions) {
     const Record *Rec = Inst->TheDef;
diff --git a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
index 5a10aba8f357d..2e8351c951980 100644
--- a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
+++ b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
@@ -373,8 +373,7 @@ void X86InstrMappingEmitter::emitSSE2AVXTable(
 void X86InstrMappingEmitter::run(raw_ostream &OS) {
   emitSourceFileHeader("X86 instruction mapping", OS);
 
-  ArrayRef<const CodeGenInstruction *> Insts =
-      Target.getInstructionsByEnumValue();
+  ArrayRef<const CodeGenInstruction *> Insts = Target.getInstructions();
   printClassDef(OS);
   emitCompressEVEXTable(Insts, OS);
   emitNFTransformTable(Insts, OS);
diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp
index ddbfb2af9869f..f469fcabf7c3a 100644
--- a/llvm/utils/TableGen/X86MnemonicTables.cpp
+++ b/llvm/utils/TableGen/X86MnemonicTables.cpp
@@ -40,7 +40,7 @@ void X86MnemonicTablesEmitter::run(raw_ostream &OS) {
   // Hold all instructions grouped by mnemonic
   StringMap<SmallVector<const CodeGenInstruction *, 0>> MnemonicToCGInstrMap;
 
-  for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *I : Target.getInstructions()) {
     const Record *Def = I->TheDef;
     // Filter non-X86 instructions.
     if (!Def->isSubClassOf("X86Inst"))
diff --git a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
index f4a679345d712..d4ec80b3d5be8 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
@@ -6,6 +6,7 @@ static_library("Core") {
     "//llvm/lib/BinaryFormat",
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/Demangle",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
diff --git a/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
index 5100478114d91..b856d1c2f7fb0 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
@@ -14,6 +14,7 @@ static_library("Rewrite") {
     "//llvm/lib/DWARFLinker/Classic",
     "//llvm/lib/DWP",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/ExecutionEngine/JITLink",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
index 21e348da90b90..18b13d5c382c9 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
@@ -51,6 +51,7 @@ static_library("modernize") {
     "UseNullptrCheck.cpp",
     "UseOverrideCheck.cpp",
     "UseRangesCheck.cpp",
+    "UseScopedLockCheck.cpp",
     "UseStartsEndsWithCheck.cpp",
     "UseStdFormatCheck.cpp",
     "UseStdNumbersCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
index 98c78dc15b324..7c05c77d9753a 100644
--- a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
@@ -94,7 +94,6 @@ static_library("Driver") {
     "ToolChains/PPCFreeBSD.cpp",
     "ToolChains/PPCLinux.cpp",
     "ToolChains/PS4CPU.cpp",
-    "ToolChains/RISCVToolchain.cpp",
     "ToolChains/SPIRV.cpp",
     "ToolChains/SPIRVOpenMP.cpp",
     "ToolChains/SYCL.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn
index 4b79e989545b6..21266ed7f9051 100644
--- a/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn
@@ -31,6 +31,7 @@ static_library("Interpreter") {
     "Interpreter.cpp",
     "InterpreterUtils.cpp",
     "InterpreterValuePrinter.cpp",
+    "RemoteJITUtils.cpp",
     "Value.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 1c62221e4d84d..ed68387321d53 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -48,7 +48,7 @@ write_lit_config("lit_site_cfg") {
     "LLVM_LINK_LLVM_DYLIB=0",
     "CLANG_LINK_CLANG_DYLIB=0",
     "LLVM_WITH_Z3=0",
-    "Z3_INCLUDE_DIR=",
+    "TEST_WITH_Z3_MOCK=0",
     "CLANG_BUILD_EXAMPLES=0",
     "CLANG_BUILT_STANDALONE=0",
     "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
@@ -58,15 +58,6 @@ write_lit_config("lit_site_cfg") {
     "CLANG_PLUGIN_SUPPORT=0",  # FIXME: Analysis/plugins need global -fPIC
     "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin", dir),
     "CLANG_VENDOR_UTI=org.llvm.clang",
-
-    # This is only used if LLVM_USE_SANITIZER includes lsan and the host
-    # OS is macOS. Since the GN build currently never uses LLVM_USE_SANITIZER,
-    # this is never read.  If it's ever needed,
-    # utils/gn/build/toolchain/BUILD.gn should get the compiler from a variable
-    # that's also read here -- but that should happen after multi-toolchain
-    # builds exist, to make sure it's a toolchain var.
-    "CMAKE_CXX_COMPILER=c++",
-    "CMAKE_C_COMPILER=cc",
     "ENABLE_BACKTRACES=1",
     "ENABLE_SHARED=0",
     "LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS=0",
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 7b9966d9bfc58..6051674a790e8 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -1155,6 +1155,7 @@ if (current_toolchain == default_toolchain) {
       "__iterator/ostreambuf_iterator.h",
       "__iterator/permutable.h",
       "__iterator/prev.h",
+      "__iterator/product_iterator.h",
       "__iterator/projected.h",
       "__iterator/ranges_iterator_traits.h",
       "__iterator/readable_traits.h",
@@ -1167,13 +1168,19 @@ if (current_toolchain == default_toolchain) {
       "__iterator/unreachable_sentinel.h",
       "__iterator/wrap_iter.h",
       "__locale",
+      "__locale_dir/check_grouping.h",
+      "__locale_dir/get_c_locale.h",
       "__locale_dir/locale_base_api.h",
       "__locale_dir/locale_base_api/android.h",
       "__locale_dir/locale_base_api/bsd_locale_fallbacks.h",
       "__locale_dir/locale_base_api/ibm.h",
       "__locale_dir/locale_base_api/musl.h",
       "__locale_dir/locale_base_api/openbsd.h",
+      "__locale_dir/messages.h",
+      "__locale_dir/money.h",
+      "__locale_dir/num.h",
       "__locale_dir/pad_and_output.h",
+      "__locale_dir/scan_keyword.h",
       "__locale_dir/support/apple.h",
       "__locale_dir/support/bsd_like.h",
       "__locale_dir/support/freebsd.h",
@@ -1182,6 +1189,9 @@ if (current_toolchain == default_toolchain) {
       "__locale_dir/support/no_locale/characters.h",
       "__locale_dir/support/no_locale/strtonum.h",
       "__locale_dir/support/windows.h",
+      "__locale_dir/time.h",
+      "__locale_dir/wbuffer_convert.h",
+      "__locale_dir/wstring_convert.h",
       "__math/abs.h",
       "__math/copysign.h",
       "__math/error_functions.h",
@@ -1434,9 +1444,8 @@ if (current_toolchain == default_toolchain) {
       "__tuple/tuple_size.h",
       "__tuple/tuple_types.h",
       "__type_traits/add_cv_quals.h",
-      "__type_traits/add_lvalue_reference.h",
       "__type_traits/add_pointer.h",
-      "__type_traits/add_rvalue_reference.h",
+      "__type_traits/add_reference.h",
       "__type_traits/aligned_storage.h",
       "__type_traits/aligned_union.h",
       "__type_traits/alignment_of.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
index e6ace69d5cbdf..4ad599820ac57 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
@@ -39,6 +39,7 @@ static_library("CPlusPlus") {
     "CPlusPlusNameParser.cpp",
     "Coroutines.cpp",
     "CxxStringTypes.cpp",
+    "Generic.cpp",
     "GenericBitset.cpp",
     "GenericOptional.cpp",
     "LibCxx.cpp",
@@ -60,5 +61,6 @@ static_library("CPlusPlus") {
     "LibStdcppTuple.cpp",
     "LibStdcppUniquePointer.cpp",
     "MSVCUndecoratedNameParser.cpp",
+    "MsvcStl.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
index 1b4ab1df0a353..672770ef8ed51 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
@@ -38,7 +38,6 @@ static_library("Utility") {
     "OpenBSDSignals.cpp",
     "RegisterContextDarwin_arm.cpp",
     "RegisterContextDarwin_arm64.cpp",
-    "RegisterContextDarwin_i386.cpp",
     "RegisterContextDarwin_riscv32.cpp",
     "RegisterContextDarwin_x86_64.cpp",
     "RegisterContextDummy.cpp",
@@ -51,7 +50,6 @@ static_library("Utility") {
     "RegisterContextLinux_s390x.cpp",
     "RegisterContextLinux_x86_64.cpp",
     "RegisterContextMach_arm.cpp",
-    "RegisterContextMach_i386.cpp",
     "RegisterContextMach_x86_64.cpp",
     "RegisterContextMemory.cpp",
     "RegisterContextNetBSD_i386.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn
index fa8f8b49721c3..8963e65f4a425 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn
@@ -18,6 +18,7 @@ static_library("mach-core") {
   include_dirs = [ "//lldb/source" ]
   sources = [
     "ProcessMachCore.cpp",
+    "RegisterContextUnifiedCore.cpp",
     "ThreadMachCore.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
index 21b06af6f09c1..fe8ae24f315ff 100644
--- a/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/ValueObject/BUILD.gn
@@ -39,7 +39,7 @@ static_library("ValueObject") {
     "ValueObjectList.cpp",
     "ValueObjectMemory.cpp",
     "ValueObjectRegister.cpp",
-    "ValueObjectSyntheticFilter.cpp",
+    "ValueObjectSynthetic.cpp",
     "ValueObjectUpdater.cpp",
     "ValueObjectVTable.cpp",
     "ValueObjectVariable.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index e9f91057cd32b..b8f3b4f8451db 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -298,6 +298,7 @@ write_cmake_config("llvm-config") {
   values = [
     "LLVM_BUILD_LLVM_DYLIB=",
     "LLVM_BUILD_SHARED_LIBS=",
+    "LLVM_ENABLE_LLVM_C_EXPORT_ANNOTATIONS=",
     "LLVM_ENABLE_TELEMETRY=",
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
     "LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE=",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
index 87e58608617ae..7cf5a8148f929 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
@@ -6,6 +6,12 @@ tablegen("Attributes") {
   tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
+tablegen("RuntimeLibcalls") {
+  visibility = [ ":public_tablegen" ]
+  args = [ "-gen-runtime-libcalls" ]
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
+}
+
 tablegen("IntrinsicImpl") {
   visibility = [ "//llvm/lib/IR" ]
   args = [ "-gen-intrinsic-impl" ]
@@ -117,6 +123,9 @@ group("public_tablegen") {
     # IR's public headers include IntrinsicEnums.inc.
     ":IntrinsicEnums",
 
+    # IR's public headers include RuntimeLibcalls.inc.
+    ":RuntimeLibcalls",
+
     # FIXME: These are only included from .cpp files at first glance.
     # Try removing them from the public_tablegen target.
     ":IntrinsicsAArch64",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
index 87368557c42c8..8b38f1a9f5623 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
@@ -6,6 +6,7 @@ static_library("AsmPrinter") {
     "//llvm/lib/CodeGen",
     "//llvm/lib/DebugInfo/CodeView",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/MSF",
     "//llvm/lib/IR",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn
index b3a8e0331e682..3258f519cf5a7 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn
@@ -6,6 +6,7 @@ static_library("Classic") {
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DWARFLinker",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn
index 117508a5cc924..9844d3ea39b16 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn
@@ -6,6 +6,7 @@ static_library("Parallel") {
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DWARFLinker",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
index cb46f7cf55fe0..1f857e360f889 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
@@ -2,6 +2,7 @@ static_library("DWARF") {
   output_name = "LLVMDebugInfoDWARF"
   deps = [
     "//llvm/lib/BinaryFormat",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
@@ -12,7 +13,6 @@ static_library("DWARF") {
     "DWARFAcceleratorTable.cpp",
     "DWARFAddressRange.cpp",
     "DWARFCFIPrinter.cpp",
-    "DWARFCFIProgram.cpp",
     "DWARFCompileUnit.cpp",
     "DWARFContext.cpp",
     "DWARFDebugAbbrev.cpp",
@@ -28,7 +28,7 @@ static_library("DWARF") {
     "DWARFDebugRangeList.cpp",
     "DWARFDebugRnglists.cpp",
     "DWARFDie.cpp",
-    "DWARFExpression.cpp",
+    "DWARFExpressionPrinter.cpp",
     "DWARFFormValue.cpp",
     "DWARFGdbIndex.cpp",
     "DWARFListTable.cpp",
@@ -36,6 +36,7 @@ static_library("DWARF") {
     "DWARFTypeUnit.cpp",
     "DWARFUnit.cpp",
     "DWARFUnitIndex.cpp",
+    "DWARFUnwindTablePrinter.cpp",
     "DWARFVerifier.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn
new file mode 100644
index 0000000000000..be0c3bd25e101
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn
@@ -0,0 +1,13 @@
+static_library("LowLevel") {
+  output_name = "LLVMDebugInfoDWARFLowLevel"
+  deps = [
+    "//llvm/lib/BinaryFormat",
+    "//llvm/lib/Support",
+    "//llvm/lib/TargetParser",
+  ]
+  sources = [
+    "DWARFCFIProgram.cpp",
+    "DWARFExpression.cpp",
+    "DWARFUnwindTable.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
index cca2cb7ee5016..ecccdf5ea63ec 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
@@ -4,6 +4,7 @@ static_library("LogicalView") {
     "//llvm/lib/BinaryFormat",
     "//llvm/lib/DebugInfo/CodeView",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/PDB",
     "//llvm/lib/Demangle",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
index 7346de64f6c9e..4c1c613529f76 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
@@ -7,6 +7,8 @@ static_library("HLSL") {
   sources = [
     "CBuffer.cpp",
     "HLSLResource.cpp",
-    "HLSLRootSignatureUtils.cpp",
+    "HLSLRootSignature.cpp",
+    "RootSignatureMetadata.cpp",
+    "RootSignatureValidations.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
index affa26d20bc9f..227a67f4da156 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
@@ -13,6 +13,7 @@ static_library("OpenMP") {
   ]
   public_deps = [ "//llvm/include/llvm/Frontend/OpenMP:public_tablegen" ]
   sources = [
+    "DirectiveNameParser.cpp",
     "OMP.cpp",
     "OMPContext.cpp",
     "OMPIRBuilder.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
index 90f997c58fa9c..28063dc794a90 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
@@ -2,6 +2,7 @@ static_library("ProfileData") {
   output_name = "LLVMProfileData"
   deps = [
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/Symbolize",
     "//llvm/lib/IR",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index b16fe19bddfd1..d327e810c536d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -29,6 +29,7 @@ static_library("Utils") {
     "CtorUtils.cpp",
     "DXILUpgrade.cpp",
     "Debugify.cpp",
+    "DeclareRuntimeLibcalls.cpp",
     "DemoteRegToStack.cpp",
     "EntryExitInstrumenter.cpp",
     "EscapeEnumerator.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
index 8b5c7a667fa10..acf97582ac5d3 100644
--- a/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
@@ -14,6 +14,7 @@ driver_executable("dsymutil") {
     "//llvm/lib/DWARFLinker/Classic",
     "//llvm/lib/DWARFLinker/Parallel",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Option",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
index ab044d165a79d..877f9fdce1f75 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
@@ -1,6 +1,7 @@
 executable("llvm-dwarfdump") {
   deps = [
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
index 3654acef9fdfd..8c8088656ee13 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
@@ -13,6 +13,7 @@ executable("llvm-dwarfutil") {
     "//llvm/lib/DWARFLinker/Classic",
     "//llvm/lib/DWARFLinker/Parallel",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/ObjCopy",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
index e70c7603c4a4b..4e9d162b28c0d 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
@@ -45,6 +45,7 @@ driver_executable("llvm-objdump") {
     "//llvm/include/llvm/Config:config",
     "//llvm/lib/CodeGen",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/PDB",
     "//llvm/lib/DebugInfo/Symbolize",
     "//llvm/lib/Debuginfod",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
index d8b6697ee0738..d0f91a5cdeb0b 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
@@ -5,6 +5,7 @@ unittest("DebugInfoDWARFTests") {
     "//llvm/lib/CodeGen",
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/ObjectYAML",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
index de55e6317f334..c29277c808d2b 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
@@ -19,6 +19,7 @@ unittest("LLVMFrontendTests") {
     "OpenMPCompositionTest.cpp",
     "OpenMPContextTest.cpp",
     "OpenMPDecompositionTest.cpp",
+    "OpenMPDirectiveNameParserTest.cpp",
     "OpenMPDirectiveNameTest.cpp",
     "OpenMPIRBuilderTest.cpp",
     "OpenMPParsingTest.cpp",
diff --git a/llvm/utils/llvm-original-di-preservation.py b/llvm/utils/llvm-original-di-preservation.py
index dc1fa518ca8e6..03793b1136f8d 100755
--- a/llvm/utils/llvm-original-di-preservation.py
+++ b/llvm/utils/llvm-original-di-preservation.py
@@ -13,7 +13,8 @@
 
 
 class DILocBug:
-    def __init__(self, action, bb_name, fn_name, instr):
+    def __init__(self, origin, action, bb_name, fn_name, instr):
+        self.origin = origin
         self.action = action
         self.bb_name = bb_name
         self.fn_name = fn_name
@@ -79,6 +80,15 @@ def generate_html_report(
         table_title_di_loc
     )
 
+    # If any DILocation bug has an origin stack trace, we emit an extra column in the table, which we must therefore
+    # determine up-front.
+    has_origin_col = any(
+        x.origin is not None
+        for per_file_bugs in di_location_bugs.values()
+        for per_pass_bugs in per_file_bugs.values()
+        for x in per_pass_bugs
+    )
+
     header_di_loc = [
         "File",
         "LLVM Pass Name",
@@ -87,6 +97,8 @@ def generate_html_report(
         "Basic Block Name",
         "Action",
     ]
+    if has_origin_col:
+        header_di_loc.append("Origin")
 
     for column in header_di_loc:
         table_di_loc += "    <th>{0}</th>\n".format(column.strip())
@@ -112,6 +124,13 @@ def generate_html_report(
                 row.append(x.fn_name)
                 row.append(x.bb_name)
                 row.append(x.action)
+                if has_origin_col:
+                    if x.origin is not None:
+                        row.append(
+                            f"<details><summary>View Origin StackTrace</summary><pre>{x.origin}</pre></details>"
+                        )
+                    else:
+                        row.append("")
                 row.append("    </tr>\n")
             # Dump the bugs info into the table.
             for column in row:
@@ -428,9 +447,9 @@ def Main():
         sys.exit(1)
 
     # Use the defaultdict in order to make multidim dicts.
-    di_location_bugs = defaultdict(lambda: defaultdict(dict))
-    di_subprogram_bugs = defaultdict(lambda: defaultdict(dict))
-    di_variable_bugs = defaultdict(lambda: defaultdict(dict))
+    di_location_bugs = defaultdict(lambda: defaultdict(list))
+    di_subprogram_bugs = defaultdict(lambda: defaultdict(list))
+    di_variable_bugs = defaultdict(lambda: defaultdict(list))
 
     # Use the ordered dict to make a summary.
     di_location_bugs_summary = OrderedDict()
@@ -470,9 +489,9 @@ def Main():
                 skipped_lines += 1
                 continue
 
-            di_loc_bugs = []
-            di_sp_bugs = []
-            di_var_bugs = []
+            di_loc_bugs = di_location_bugs[bugs_file][bugs_pass]
+            di_sp_bugs = di_subprogram_bugs[bugs_file][bugs_pass]
+            di_var_bugs = di_variable_bugs[bugs_file][bugs_pass]
 
             # Omit duplicated bugs.
             di_loc_set = set()
@@ -487,6 +506,7 @@ def Main():
 
                 if bugs_metadata == "DILocation":
                     try:
+                        origin = bug.get("origin")
                         action = bug["action"]
                         bb_name = bug["bb-name"]
                         fn_name = bug["fn-name"]
@@ -494,7 +514,7 @@ def Main():
                     except:
                         skipped_bugs += 1
                         continue
-                    di_loc_bug = DILocBug(action, bb_name, fn_name, instr)
+                    di_loc_bug = DILocBug(origin, action, bb_name, fn_name, instr)
                     if not str(di_loc_bug) in di_loc_set:
                         di_loc_set.add(str(di_loc_bug))
                         if opts.compress:
diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py
index 8343dee937f78..e9591b00e2b5a 100755
--- a/llvm/utils/release/github-upload-release.py
+++ b/llvm/utils/release/github-upload-release.py
@@ -42,18 +42,45 @@ def create_release(repo, release, tag=None, name=None, message=None):
         name = "LLVM {}".format(release)
 
     if not message:
+        # Note that these lines are not length limited because if we do so, GitHub
+        # assumes that should be how it is laid out on the page. We want GitHub to
+        # do the reflowing for us instead.
         message = dedent(
             """\
-            LLVM {} Release
+LLVM {release} Release
 
-            # A note on binaries
+## Package Types
 
-            Volunteers make binaries for the LLVM project, which will be uploaded
-            when they have had time to test and build these binaries. They might
-            not be available directly or not at all for each release. We suggest
-            you use the binaries from your distribution or build your own if you
-            rely on a specific platform or configuration."""
-        ).format(release)
+Each platform has one binary release package. The file name starts with either `LLVM-` or `clang+llvm-` and ends with the platform's name. For example, `LLVM-{release}-Linux-ARM64.tar.xz` contains LLVM binaries for Arm64 Linux.
+
+Except for Windows. Where `LLVM-*.exe` is an installer intended for using LLVM as a toolchain and `clang+llvm-` contains the contents of the installer, plus libraries and tools not normally used in a toolchain. You most likely want the `LLVM-` installer, unless you are developing software which itself uses LLVM, in which case choose `clang+llvm-`.
+
+If you do not find a release package for your platform, you may be able to find a community built package on the LLVM Discourse forum thread for this release. Remember that these are built by volunteers and may not always be available.
+
+If you rely on a platform or configuration that is not one of the defaults, we suggest you use the binaries that your platform provides, or build your own release packages.
+
+In addition, source archives are available:
+* `<sub-project>-{release}.src.tar.xz` are archives of the sources of specific sub-projects of `llvm-project` (except for `test-suite` which is an archive of the [LLVM Test Suite](https://github.com/llvm/llvm-test-suite)).
+* To get all the `llvm-project` source code for this release, choose `llvm-project-{release}.src.tar.xz`.
+
+## Verifying Packages
+
+All packages come with a matching `.sig` or `.jsonl` file. You should use these to verify the integrity of the packages.
+
+If it has a `.sig` file, it should have been signed by the release managers using GPG. Download the keys from the [LLVM website](https://releases.llvm.org/release-keys.asc), import them into your keyring and use them to verify the file:
+```
+$ gpg --import release-keys.asc
+$ gpg --verify <package file name>.sig <package file name>
+```
+
+If it has a `.jsonl` file, use [gh](https://cli.github.com/manual/gh_attestation_verify) to verify the package:
+```
+gh attestation verify --repo llvm/llvm-project <package file name>
+(if you are able to connect to GitHub)
+gh attestation verify --repo llvm/llvm-project <package file name> --bundle <package file name>.jsonl
+(using attestation file on disk)
+```"""
+        ).format(release=release)
 
     prerelease = True if "rc" in release else False
 
diff --git a/mlir/cmake/modules/IRDLToCpp.cmake b/mlir/cmake/modules/IRDLToCpp.cmake
index 8470ccdf55166..3fa1601c694d2 100644
--- a/mlir/cmake/modules/IRDLToCpp.cmake
+++ b/mlir/cmake/modules/IRDLToCpp.cmake
@@ -5,7 +5,7 @@ function(add_irdl_to_cpp_target target irdl_file)
 
     # The command output depends on the executable to ensure IRDL sources are properly rebuilt
     # if the tool changes.
-    DEPENDS ${MLIR_IRDL_TO_CPP_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/${irdl_file}
+    DEPENDS ${MLIR_IRDL_TO_CPP_TARGET} ${MLIR_IRDL_TO_CPP_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/${irdl_file}
     COMMENT "Building ${irdl_file}..."
   )
   add_custom_target(${target} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${irdl_file}.cpp.inc)
diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md
index b8bd0f507a510..bef9e7f54948d 100644
--- a/mlir/docs/Bindings/Python.md
+++ b/mlir/docs/Bindings/Python.md
@@ -51,7 +51,10 @@ python -m pip install --upgrade pip
 # packages will be installed there.
 python -m pip install -r mlir/python/requirements.txt
 
-# Now run `cmake`, `ninja`, et al.
+# Now run your build command with `cmake`, `ninja`, et al.
+
+# Run mlir tests. For example, to run python bindings tests only using ninja:
+ninja check-mlir-python
 ```
 
 For interactive use, it is sufficient to add the
@@ -859,7 +862,7 @@ mutually exclusive with a more complete mapping of the backing constructs.
 
 ## Testing
 
-Tests should be added in the `test/Bindings/Python` directory and should
+Tests should be added in the `mlir/test/python` directory and should
 typically be `.py` files that have a lit run line.
 
 We use `lit` and `FileCheck` based tests:
diff --git a/mlir/docs/Tutorials/transform/Ch1.md b/mlir/docs/Tutorials/transform/Ch1.md
index b0fdf085854c7..7463a9afbe97e 100644
--- a/mlir/docs/Tutorials/transform/Ch1.md
+++ b/mlir/docs/Tutorials/transform/Ch1.md
@@ -19,13 +19,13 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
   %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
     ins(%biased, %c0f : tensor<512x512xf32>, f32)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
@@ -41,7 +41,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">):
+      %arg2: !transform.op<"linalg.elementwise">):
     transform.yield
   }
 }
@@ -72,11 +72,11 @@ To check or debug a transform sequence, it is possible to print various entities
 transform.sequence failures(propagate) {
 ^bb0(%arg0: !transform.any_op,
      %arg1: !transform.op<"linalg.matmul">,
-     %arg2: !transform.op<"linalg.elemwise_binary">):
+     %arg2: !transform.op<"linalg.elementwise">):
   transform.debug.emit_remark_at %arg1, "matmul"
       : !transform.op<"linalg.matmul">
   transform.debug.emit_remark_at %arg2, "elemwise_binaries"
-      : !transform.op<"linalg.elemwise_binary">
+      : !transform.op<"linalg.elementwise">
   transform.yield
 }
 ```
@@ -89,10 +89,10 @@ Since we don’t want to recompile the compiler every time we change a transform
 ```sh
 $ mlir-opt sequence.mlir --pass-pipeline="
     builtin.module(transform-interpreter{
-        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary})"
+        debug-bind-trailing-args=linalg.matmul,linalg.elementwise})"
 ```
 
-The `sequence.mlir` file contains _both_ the payload IR function _and_ the transform IR sequence nested in the same module. The transform interpreter pass will apply the `@__transform_main` named sequence to the anchor operation of the pass. In our case, we also asked the interpreter pass to associate the two extra arguments of the top-level sequence with all `linalg.matmul` and `linalg.elemwise_binary` payload operations through the respective pass options. Running this pass results in the expected remarks:
+The `sequence.mlir` file contains _both_ the payload IR function _and_ the transform IR sequence nested in the same module. The transform interpreter pass will apply the `@__transform_main` named sequence to the anchor operation of the pass. In our case, we also asked the interpreter pass to associate the two extra arguments of the top-level sequence with all `linalg.matmul` and `linalg.elementwise` payload operations through the respective pass options. Running this pass results in the expected remarks:
 
 ```sh
 sequence.mlir:7:13: remark: matmul
@@ -100,13 +100,13 @@ sequence.mlir:7:13: remark: matmul
             ^
 sequence.mlir:7:13: note: see current operation: %0 = linalg.matmul ins(%arg0, %arg1 : tensor<512x512xf32>, tensor<512x512xf32>) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
 sequence.mlir:10:13: remark: elemwise_binaries
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
             ^
-sequence.mlir:10:13: note: see current operation: %1 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>} ins(%0, %arg2 : tensor<512x512xf32>, tensor<512x512xf32>) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
+sequence.mlir:10:13: note: see current operation: %1 = linalg.elementwise kind=#linalg.elementwise_kind<add>> ins(%0, %arg2 : tensor<512x512xf32>, tensor<512x512xf32>) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
 sequence.mlir:14:13: remark: elemwise_binaries
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
             ^
-sequence.mlir:14:13: note: see current operation: %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>} ins(%1, %cst : tensor<512x512xf32>, f32) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
+sequence.mlir:14:13: note: see current operation: %2 = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>> ins(%1, %cst : tensor<512x512xf32>, f32) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
 ```
 
 Note that `%arg2` is associated with both elementwise payload operations. Any handle is associated with a list of entities. Individual transformations may or may not care about the order of elements in that list.
@@ -121,7 +121,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // The actual tiling transformation takes tile sizes as attributes.
     %loop, %tiled = transform.structured.tile_using_forall %arg1
                     tile_sizes [4, 32]
@@ -163,10 +163,10 @@ func.func @fc_relu(%arg0: tensor<512x512xf32>,
           : tensor<4x32xf32> into tensor<512x512xf32>
     }
   }
-  %1 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+  %1 = linalg.elementwise kind=#linalg.elementwise_kind<add>>
     ins(%0, %arg2 : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
-  %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+  %2 = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>>
     ins(%1, %cst : tensor<512x512xf32>, f32)
     outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
   return %2 : tensor<512x512xf32>
@@ -185,7 +185,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // The actual tiling transformation takes tile sizes as attributes.
     %loop, %tiled = transform.structured.tile_using_forall %arg1 tile_sizes [4, 32]
         : (!transform.op<"linalg.matmul">) -> (!transform.any_op, !transform.any_op)
@@ -219,7 +219,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // We can cast one type to another as long as operations are compatible
     // with both types. This creates "aliasing" handles.
     %casted = transform.cast %arg1 : !transform.op<"linalg.matmul">
@@ -248,7 +248,7 @@ sequence.mlir:28:3: error: op uses a handle invalidated by a previously executed
   transform.debug.emit_remark_at %matmul, "elemwise_binaries" : !transform.op<"linalg.matmul">
   ^
 sequence.mlir:21:29: note: handle to invalidated ops
-^bb0(%root: !transform.any_op, %matmul: !transform.op<"linalg.matmul">, %elemwise: !transform.op<"linalg.elemwise_binary">):
+^bb0(%root: !transform.any_op, %matmul: !transform.op<"linalg.matmul">, %elemwise: !transform.op<"linalg.elementwise">):
                             ^
 sequence.mlir:27:19: note: invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them
   %loop, %tiled = transform.structured.tile_using_forall %mm tile_sizes [4, 32]
@@ -263,12 +263,12 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
     %add, %max = transform.split_handle %arg2
-        : (!transform.op<"linalg.elemwise_binary">)
+        : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It
@@ -308,12 +308,12 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
     %add, %max = transform.split_handle %arg2
-        : (!transform.op<"linalg.elemwise_binary">)
+        : (!transform.op<"linalg.elementwise">)
           -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It
@@ -384,7 +384,7 @@ test/Examples/transform/Ch1/invalidation-2.mlir:106:18: note: invalidated by thi
   %func, %call = transform.loop.outline %outline_target {func_name = "outlined"}
                  ^
 test/Examples/transform/Ch1/invalidation-2.mlir:24:13: note: ancestor payload op
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
             ^
 test/Examples/transform/Ch1/invalidation-2.mlir:24:13: note: nested payload op
   %matmul = linalg.matmul ins(%lhs, %rhs: tensor<512x512xf32>, tensor<512x512xf32>)
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index 6a6cefd8785aa..0f45f5607bab9 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -290,12 +290,12 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
     %add, %max = transform.split_handle %arg2
-        : (!transform.op<"linalg.elemwise_binary">)
+        : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It
diff --git a/mlir/docs/Tutorials/transform/Ch4.md b/mlir/docs/Tutorials/transform/Ch4.md
index ac60fcb914685..f9a0dff5c90e3 100644
--- a/mlir/docs/Tutorials/transform/Ch4.md
+++ b/mlir/docs/Tutorials/transform/Ch4.md
@@ -42,13 +42,13 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
             outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
   %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
     ins(%biased, %c0f : tensor<512x512xf32>, f32)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
@@ -59,7 +59,7 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 
 In Chapter 1, we were calling the test transform interpreter pass with
 additional arguments, `bind-first-extra-to-ops=linalg.matmul
-bind-second-extra-to-ops=linalg.elemwise_binary`, to provide initial
+bind-second-extra-to-ops=linalg.elementwise`, to provide initial
 associations for operation handles. Instead, we can use match operations to
 discover relevant operations in the payload IR. Match operations can be combined
 with “regular” transform operations using, e.g., the
@@ -97,7 +97,7 @@ module @transforms attributes { transform.with_named_sequence } {
   // rewriter sequence on success.
   transform.named_sequence @match_elemwise(
       %entry: !transform.any_op {transform.readonly}) -> !transform.any_op {
-    transform.match.operation_name %entry ["linalg.elemwise_binary"]
+    transform.match.operation_name %entry ["linalg.elementwise"]
       : !transform.any_op
     transform.yield %entry : !transform.any_op
   }
@@ -127,7 +127,7 @@ module @transforms attributes { transform.with_named_sequence } {
 This script can be executed using the non-test interpreter pass running on the
 root operation of the translation unit without additional flags: `mlir-opt
 --transform-interpreter`. It will emit corresponding remarks at
-`linalg.elemwise_binary` and `linalg.matmul` operations. In debug builds, the
+`linalg.elementwise` and `linalg.matmul` operations. In debug builds, the
 infrastructure provides a convenient method to understand the matching process
 by passing `-debug-only=transform-matcher` to `mlir-opt` or a derived tool. It
 will print the silenceable failure messages produced by the match operations
@@ -169,7 +169,7 @@ transform.named_sequence @match_matmul_elemwise(
     %last: !transform.any_op {transform.readonly})
     -> (!transform.any_op, !transform.any_op, !transform.any_op) {
   // The last operation must be an elementwise binary.
-  transform.match.operation_name %last ["linalg.elemwise_binary"]
+  transform.match.operation_name %last ["linalg.elementwise"]
     : !transform.any_op
   // Its first operand must be defined by another operation, to which we
   // will get a handle here. We are guaranteed that the first operand exists
@@ -179,7 +179,7 @@ transform.named_sequence @match_matmul_elemwise(
   %middle = transform.get_producer_of_operand %last[0]
     : (!transform.any_op) -> !transform.any_op
   // The defining operation must itself be an elementwise binary.
-  transform.match.operation_name %middle ["linalg.elemwise_binary"]
+  transform.match.operation_name %middle ["linalg.elementwise"]
     : !transform.any_op
   // And the first operand of that operation must be defined by yet another
   // operation.
@@ -399,7 +399,7 @@ transform.named_sequence @match_matmul_elemwise(
     -> (!transform.any_op, !transform.any_op, !transform.any_op,
         !transform.param<i32>) {
   // The last operation must be an elementwise binary.
-  transform.match.operation_name %last ["linalg.elemwise_binary"]
+  transform.match.operation_name %last ["linalg.elementwise"]
     : !transform.any_op
 
   // One of its operands must be defined by another operation, to which we
@@ -413,7 +413,7 @@ transform.named_sequence @match_matmul_elemwise(
     %def = transform.get_defining_op %operand
       : (!transform.any_value) -> !transform.any_op
     // The defining operation must itself be an elementwise binary.
-    transform.match.operation_name %def ["linalg.elemwise_binary"]
+    transform.match.operation_name %def ["linalg.elementwise"]
       : !transform.any_op
     transform.yield %def : !transform.any_op
   }
diff --git a/mlir/docs/Tutorials/transform/ChH.md b/mlir/docs/Tutorials/transform/ChH.md
index aadfddb89393f..ce24c5add631c 100644
--- a/mlir/docs/Tutorials/transform/ChH.md
+++ b/mlir/docs/Tutorials/transform/ChH.md
@@ -290,7 +290,7 @@ scf.forall (%co) in (2) {
   scf.forall (%n, %y, %xo) in (5, 80, 20) {
     tensor.extract_slice
     // Implicit dimensions [ni=0:1, y=0:1, xi=0:5, ci=0:64]
-    %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> } // ...
+    %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed> // ...
     scf.forall.in_parallel {
       tensor.parallel_insert_slice // ...
     }
diff --git a/mlir/examples/toy/Ch2/include/toy/Ops.td b/mlir/examples/toy/Ch2/include/toy/Ops.td
index 1a1b136af71fb..ef65c9c8d682b 100644
--- a/mlir/examples/toy/Ch2/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch2/include/toy/Ops.td
@@ -297,7 +297,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
index bf4c099fe05d2..96925bebf07b7 100644
--- a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, {});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch3/include/toy/Ops.td b/mlir/examples/toy/Ch3/include/toy/Ops.td
index 021802b5c4fcb..485980420a20b 100644
--- a/mlir/examples/toy/Ch3/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch3/include/toy/Ops.td
@@ -298,7 +298,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
index bf4c099fe05d2..c8cba82cb63a0 100644
--- a/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch4/include/toy/Ops.td b/mlir/examples/toy/Ch4/include/toy/Ops.td
index 4441e48ca53c0..0b32b1b0c7726 100644
--- a/mlir/examples/toy/Ch4/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch4/include/toy/Ops.td
@@ -335,7 +335,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
index b56e2f7c90086..9371815577b1b 100644
--- a/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch5/include/toy/Ops.td b/mlir/examples/toy/Ch5/include/toy/Ops.td
index 5b7c966de6f08..d11d18dc93cf1 100644
--- a/mlir/examples/toy/Ch5/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch5/include/toy/Ops.td
@@ -335,7 +335,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
index b56e2f7c90086..9371815577b1b 100644
--- a/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch6/include/toy/Ops.td b/mlir/examples/toy/Ch6/include/toy/Ops.td
index fdbc239a171df..63950f467455c 100644
--- a/mlir/examples/toy/Ch6/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch6/include/toy/Ops.td
@@ -335,7 +335,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
index de1eafc4d8678..22f75e042fe08 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
@@ -92,7 +92,7 @@ class PrintOpLowering : public ConversionPattern {
       auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
       auto loop =
           rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      for (Operation &nested : *loop.getBody())
+      for (Operation &nested : make_early_inc_range(*loop.getBody()))
         rewriter.eraseOp(&nested);
       loopIvs.push_back(loop.getInductionVar());
 
diff --git a/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
index b56e2f7c90086..9371815577b1b 100644
--- a/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch7/include/toy/Ops.td b/mlir/examples/toy/Ch7/include/toy/Ops.td
index 71ab7b0aeebb9..bdf8ad0bc21a1 100644
--- a/mlir/examples/toy/Ch7/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch7/include/toy/Ops.td
@@ -360,7 +360,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
index a20a2c888a175..54eeb275ae85e 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
@@ -92,7 +92,7 @@ class PrintOpLowering : public ConversionPattern {
       auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
       auto loop =
           rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      for (Operation &nested : *loop.getBody())
+      for (Operation &nested : make_early_inc_range(*loop.getBody()))
         rewriter.eraseOp(&nested);
       loopIvs.push_back(loop.getInductionVar());
 
diff --git a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
index e554e375209f1..2490f17163b80 100644
--- a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
@@ -182,7 +182,7 @@ class MLIRGenImpl {
         return nullptr;
       argTypes.push_back(type);
     }
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
@@ -441,10 +441,10 @@ class MLIRGenImpl {
     for (auto &var : lit.getValues()) {
       if (auto *number = llvm::dyn_cast<NumberExprAST>(var.get())) {
         attrElements.push_back(getConstantAttr(*number));
-        typeElements.push_back(getType(std::nullopt));
+        typeElements.push_back(getType(/*shape=*/{}));
       } else if (auto *lit = llvm::dyn_cast<LiteralExprAST>(var.get())) {
         attrElements.push_back(getConstantAttr(*lit));
-        typeElements.push_back(getType(std::nullopt));
+        typeElements.push_back(getType(/*shape=*/{}));
       } else {
         auto *structLit = llvm::cast<StructLiteralExprAST>(var.get());
         auto attrTypePair = getConstantAttr(*structLit);
diff --git a/mlir/include/mlir-c/BuiltinTypes.h b/mlir/include/mlir-c/BuiltinTypes.h
index 6875fab7bf796..c981bfd0967c5 100644
--- a/mlir/include/mlir-c/BuiltinTypes.h
+++ b/mlir/include/mlir-c/BuiltinTypes.h
@@ -289,9 +289,12 @@ MLIR_CAPI_EXPORTED int64_t mlirShapedTypeGetRank(MlirType type);
 /// Checks whether the given shaped type has a static shape.
 MLIR_CAPI_EXPORTED bool mlirShapedTypeHasStaticShape(MlirType type);
 
-/// Checks wither the dim-th dimension of the given shaped type is dynamic.
+/// Checks whether the dim-th dimension of the given shaped type is dynamic.
 MLIR_CAPI_EXPORTED bool mlirShapedTypeIsDynamicDim(MlirType type, intptr_t dim);
 
+/// Checks whether the dim-th dimension of the given shaped type is static.
+MLIR_CAPI_EXPORTED bool mlirShapedTypeIsStaticDim(MlirType type, intptr_t dim);
+
 /// Returns the dim-th dimension of the given ranked shaped type.
 MLIR_CAPI_EXPORTED int64_t mlirShapedTypeGetDimSize(MlirType type,
                                                     intptr_t dim);
@@ -300,17 +303,25 @@ MLIR_CAPI_EXPORTED int64_t mlirShapedTypeGetDimSize(MlirType type,
 /// in shaped types.
 MLIR_CAPI_EXPORTED bool mlirShapedTypeIsDynamicSize(int64_t size);
 
+/// Checks whether the given shaped type dimension value is statically-sized.
+MLIR_CAPI_EXPORTED bool mlirShapedTypeIsStaticSize(int64_t size);
+
 /// Returns the value indicating a dynamic size in a shaped type. Prefer
-/// mlirShapedTypeIsDynamicSize to direct comparisons with this value.
+/// mlirShapedTypeIsDynamicSize and mlirShapedTypeIsStaticSize to direct
+/// comparisons with this value.
 MLIR_CAPI_EXPORTED int64_t mlirShapedTypeGetDynamicSize(void);
 
 /// Checks whether the given value is used as a placeholder for dynamic strides
 /// and offsets in shaped types.
 MLIR_CAPI_EXPORTED bool mlirShapedTypeIsDynamicStrideOrOffset(int64_t val);
 
+/// Checks whether the given dimension value of a stride or an offset is
+/// statically-sized.
+MLIR_CAPI_EXPORTED bool mlirShapedTypeIsStaticStrideOrOffset(int64_t val);
+
 /// Returns the value indicating a dynamic stride or offset in a shaped type.
-/// Prefer mlirShapedTypeGetDynamicStrideOrOffset to direct comparisons with
-/// this value.
+/// Prefer mlirShapedTypeIsDynamicStrideOrOffset and
+/// mlirShapedTypeIsStaticStrideOrOffset to direct comparisons with this value.
 MLIR_CAPI_EXPORTED int64_t mlirShapedTypeGetDynamicStrideOrOffset(void);
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
index a52ee2ee89caf..cc2f543e79f69 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
@@ -23,7 +23,7 @@ namespace amdgpu {
 
 #define GEN_PASS_DECL_AMDGPUEMULATEATOMICSPASS
 #define GEN_PASS_DECL_AMDGPURESOLVESTRIDEDMETADATAPASS
-#define GEN_PASS_DECL_AMDGPUTRANSFERREADTOLOADPASS
+#define GEN_PASS_DECL_AMDGPUMASKEDLOADTOLOADPASS
 #define GEN_PASS_REGISTRATION
 #include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
 
@@ -35,8 +35,8 @@ void populateAmdgpuEmulateAtomicsPatterns(ConversionTarget &target,
 void populateAmdgpuResolveStridedMetadataPatterns(RewritePatternSet &patterns,
                                                   PatternBenefit benefit = 1);
 
-void populateAmdgpuTransferReadToLoadPatterns(RewritePatternSet &patterns,
-                                              PatternBenefit benefit = 1);
+void populateAmdgpuMaskedloadToLoadPatterns(RewritePatternSet &patterns,
+                                            PatternBenefit benefit = 1);
 
 } // namespace amdgpu
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
index 0e858108acf35..8d0e6829ab0cc 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
@@ -51,8 +51,8 @@ def AmdgpuResolveStridedMetadataPass : Pass<"amdgpu-resolve-strided-metadata"> {
   ];
 }
 
-def AmdgpuTransferReadToLoadPass : Pass<"amdgpu-transfer-read-to-load"> {
-  let summary = "Lower the operations from the vector transfer_read to vector load";
+def AmdgpuMaskedloadToLoadPass : Pass<"amdgpu-maskedload-to-load"> {
+  let summary = "Lower the operations from the vector maskedload to vector load";
   let description = [{
     This pass creates a transfer read op lowering optimization. The lowering
     will produce a conditional check at runtime. If within bounds, a vector
diff --git a/mlir/include/mlir/Dialect/Affine/Utils.h b/mlir/include/mlir/Dialect/Affine/Utils.h
index ae5a68a6be157..ac11f5a7c24c7 100644
--- a/mlir/include/mlir/Dialect/Affine/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Utils.h
@@ -198,10 +198,9 @@ AffineExpr substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
 /// of its input list. `indexRemap`'s dimensional inputs are expected to
 /// correspond to memref's indices, and its symbolic inputs if any should be
 /// provided in `symbolOperands`.
-///
-/// `domOpFilter`, if non-null, restricts the replacement to only those
-/// operations that are dominated by the former; similarly, `postDomOpFilter`
-/// restricts replacement to only those operations that are postdominated by it.
+//
+/// If `userFilterFn` is specified, restrict replacement to only those users
+/// that pass the specified filter (i.e., the filter returns true).
 ///
 /// 'allowNonDereferencingOps', if set, allows replacement of non-dereferencing
 /// uses of a memref without any requirement for access index rewrites as long
@@ -224,13 +223,14 @@ AffineExpr substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
 //  d1, d2) -> (d0 - d1, d2), and %ii will be the extra operand. Without any
 //  extra operands, note that 'indexRemap' would just be applied to existing
 //  indices (%i, %j).
+//
 //  TODO: allow extraIndices to be added at any position.
 LogicalResult replaceAllMemRefUsesWith(
     Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices = {},
     AffineMap indexRemap = AffineMap(), ArrayRef<Value> extraOperands = {},
-    ArrayRef<Value> symbolOperands = {}, Operation *domOpFilter = nullptr,
-    Operation *postDomOpFilter = nullptr, bool allowNonDereferencingOps = false,
-    bool replaceInDeallocOp = false);
+    ArrayRef<Value> symbolOperands = {},
+    llvm::function_ref<bool(Operation *)> userFilterFn = nullptr,
+    bool allowNonDereferencingOps = false, bool replaceInDeallocOp = false);
 
 /// Performs the same replacement as the other version above but only for the
 /// dereferencing uses of `oldMemRef` in `op`, except in cases where
diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 0bee876ac9bfa..7c50c2036ffdc 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -65,6 +65,10 @@ class ConstantIntOp : public arith::ConstantOp {
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     int64_t value);
 
+  /// Build a constant int op that produces an integer from an APInt
+  static void build(OpBuilder &builder, OperationState &result, Type type,
+                    const APInt &value);
+
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
   }
@@ -154,6 +158,11 @@ Value getReductionOp(AtomicRMWKind op, OpBuilder &builder, Location loc,
                      Value lhs, Value rhs);
 
 arith::CmpIPredicate invertPredicate(arith::CmpIPredicate pred);
+
+/// Creates an `arith.constant` operation with a zero value of type `type`. This
+/// method asserts if `type` is invalid for representing zero with
+/// `arith.constant`.
+Value getZeroConstant(OpBuilder &builder, Location loc, Type type);
 } // namespace arith
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
index c1529a36465ac..6245f88db3d19 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
@@ -712,7 +712,7 @@ AliasingOpOperandList defaultGetAliasingOpOperands(Value value,
 /// This is the default implementation of
 /// BufferizableOpInterface::getBufferType. Should not be called from other
 /// places.
-FailureOr<BaseMemRefType>
+FailureOr<BufferLikeType>
 defaultGetBufferType(Value value, const BufferizationOptions &options,
                      const BufferizationState &state,
                      SmallVector<Value> &invocationStack);
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td
index cafe05fe5f189..246ae77f327cf 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td
@@ -525,7 +525,7 @@ def BufferizableOpInterface : OpInterface<"BufferizableOpInterface"> {
           Note: This interface method should never be called directly from user
           code. Always use `bufferization::getBufferType`.
         }],
-        /*retType=*/"::mlir::FailureOr<::mlir::BaseMemRefType>",
+        /*retType=*/"::mlir::FailureOr<::mlir::bufferization::BufferLikeType>",
         /*methodName=*/"getBufferType",
         /*args=*/(ins "::mlir::Value":$value,
                       "const ::mlir::bufferization::BufferizationOptions &":$options,
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 32c53ea9c494a..f175b15c8770f 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -111,7 +111,7 @@ def Bufferization_AllocTensorOp : Bufferization_Op<"alloc_tensor",
     AliasingValueList getAliasingValues(
         OpOperand &opOperand, const AnalysisState &state);
 
-    FailureOr<BaseMemRefType> getBufferType(
+    FailureOr<BufferLikeType> getBufferType(
         Value value, const BufferizationOptions &options,
         const BufferizationState &state,
         SmallVector<Value> &invocationStack);
@@ -478,10 +478,10 @@ def Bufferization_ToTensorOp : Bufferization_Op<"to_tensor", [
 
     bool isWritable(Value value, const AnalysisState &state);
 
-    FailureOr<BaseMemRefType> getBufferType(
+    FailureOr<BufferLikeType> getBufferType(
         Value value, const BufferizationOptions &options,
         const BufferizationState &state, SmallVector<Value> &invocationStack) {
-      return ::llvm::cast<BaseMemRefType>(getBuffer().getType());
+      return getBuffer().getType();
     }
   }];
 
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
index cbb6054fcf886..a2bfcb7ed2b75 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
@@ -18,7 +18,6 @@
 
 namespace mlir::bufferization {
 struct BufferizationOptions;
-class BufferizationState;
 class BufferLikeType;
 } // namespace mlir::bufferization
 
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h b/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
index f56c10555f02c..e8a81c74bd77a 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
@@ -32,7 +32,7 @@ template <typename ConcreteModel, typename ConcreteOp>
 struct OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel
     : public BufferizableOpInterface::ExternalModel<ConcreteModel, ConcreteOp> {
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -110,7 +110,7 @@ struct OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel
     if (!bufferType)
       return op->emitOpError("could not infer buffer type of block argument");
 
-    return bufferType;
+    return cast<BufferLikeType>(bufferType);
   }
 
 protected:
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index 91ee89919e58e..7fe2da8f7e044 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -1618,10 +1618,20 @@ def EmitC_ClassOp
         return
       }
     }
+    // Class with a final speciferAdd commentMore actions
+    emitc.class final @modelClass {
+      emitc.field @fieldName0 : !emitc.array<1xf32> = {emitc.opaque = "input_tensor"}
+      emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @fieldName0 : !emitc.array<1xf32>
+        %2 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+      }
+    }
     ```
   }];
 
-  let arguments = (ins SymbolNameAttr:$sym_name);
+  let arguments = (ins SymbolNameAttr:$sym_name, UnitAttr:$final_specifier);
 
   let regions = (region AnyRegion:$body);
 
@@ -1632,7 +1642,8 @@ def EmitC_ClassOp
 
   let hasCustomAssemblyFormat = 1;
 
-  let assemblyFormat = [{ $sym_name attr-dict-with-keyword $body }];
+  let assemblyFormat =
+      [{ (`final` $final_specifier^)? $sym_name attr-dict-with-keyword $body }];
 }
 
 def EmitC_FieldOp : EmitC_Op<"field", [Symbol]> {
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td b/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td
index 6e0f6f1d78eda..e745ef02d7034 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td
@@ -228,6 +228,41 @@ def GPUThreadMappingAttr
   }];
 }
 
+def GPULaneMappingAttr
+    : GPU_Attr<"GPULaneMapping", "lane", [
+      DeclareAttrInterfaceMethods<DeviceMappingAttrInterface> ]> {
+  let parameters = (ins
+    EnumParameter<MappingIdEnum>:$lane
+  );
+  let assemblyFormat = "`<` params `>`";
+  let description = [{
+    An attribute that allows defining lane parallelism for GPU devices.
+
+    It can be consumed by lowering to generate GPU.
+
+    #### 3D mapping mode
+
+    Unsupported
+
+    #### Linear mapping mode
+
+    The linear lane id is obtained by linearizing the index of the lane.
+    If required, predication occurs on the linear id. This allows specifying
+    predication on a 1D subset of the (linearized) lanes.
+  }];
+}
+
+def GPUMappingMaskAttr : GPU_Attr<"GPUMappingMask", "mask", [
+  DeclareAttrInterfaceMethods<DeviceMaskingAttrInterface> ] >  {
+  let parameters = (ins "uint64_t":$mask);
+  let assemblyFormat = "`<` params `>`";
+  let description = [{
+    Attribute describing how to filter the processing units that a region is
+    mapped to. The masking is a bitfield that specifies for each processing
+    unit whether it is active or not.
+  }];
+}
+
 def GPUMemorySpaceMappingAttr : GPU_Attr<"GPUMemorySpaceMapping", "memory_space", [
   DeclareAttrInterfaceMethods<DeviceMappingAttrInterface> ] >  {
   let parameters = (ins
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index a81b2e83ddefe..1dbaf5db7b618 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1304,8 +1304,8 @@ def GPU_ShuffleOp : GPU_Op<
     Results<(outs AnyIntegerOrFloatOr1DVector:$shuffleResult, I1:$valid)> {
   let summary = "Shuffles values within a subgroup.";
   let description = [{
-    The "shuffle" op moves values to a across lanes (a.k.a., invocations,
-    work items) within the same subgroup. The `width` argument specifies the
+    The "shuffle" op moves values across lanes in a subgroup (a.k.a., local
+    invocation) within the same subgroup. The `width` argument specifies the
     number of lanes that participate in the shuffle, and must be uniform
     across all lanes. Further, the first `width` lanes of the subgroup must
     be active.
@@ -1366,6 +1366,54 @@ def GPU_ShuffleOp : GPU_Op<
   ];
 }
 
+def GPU_RotateOp : GPU_Op<
+    "rotate", [Pure, AllTypesMatch<["value", "rotateResult"]>]>,
+    Arguments<(ins AnyIntegerOrFloatOr1DVector:$value, I32:$offset, I32:$width)>,
+    Results<(outs AnyIntegerOrFloatOr1DVector:$rotateResult, I1:$valid)> {
+  let summary = "Rotate values within a subgroup.";
+  let description = [{
+    The "rotate" op moves values across lanes in a subgroup (a.k.a., local
+    invocations) within the same subgroup. The `width` argument specifies the
+    number of lanes that participate in the rotation, and must be uniform across
+    all participating lanes. Further, the first `width` lanes of the subgroup
+    must be active.
+
+    `width` must be a power of two, and `offset` must be in the range
+    `[0, width)`.
+
+    Return the `rotateResult` of the invocation whose id within the group is
+    calculated as follows:
+
+    ```mlir
+    Invocation ID = ((LaneId + offset) & (width - 1)) + (LaneId & ~(width - 1))
+    ```
+
+    Returns the `rotateResult` and `true` if the current lane id is smaller than
+    `width`, and poison value and `false` otherwise.
+
+    example:
+
+    ```mlir
+    %offset = arith.constant 1 : i32
+    %width = arith.constant 16 : i32
+    %1, %2 = gpu.rotate %0, %offset, %width : f32
+    ```
+
+    For lane `k`, returns the value from lane `(k + cst1) % width`.
+  }];
+
+  let assemblyFormat = [{
+    $value `,` $offset `,` $width attr-dict `:` type($value)
+  }];
+
+  let builders = [
+    // Helper function that creates a rotate with constant offset/width.
+    OpBuilder<(ins "Value":$value, "int32_t":$offset, "int32_t":$width)>
+  ];
+
+  let hasVerifier = 1;
+}
+
 def GPU_BarrierOp : GPU_Op<"barrier"> {
   let summary = "Synchronizes all work items of a workgroup.";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h
index 4b5f684525046..804f4e01dbb4f 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h
@@ -11,7 +11,6 @@
 
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
-#include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 
 namespace mlir {
@@ -57,7 +56,7 @@ mapForallToBlocksImpl(RewriterBase &rewriter, TransformOpInterface transformOp,
 DiagnosedSilenceableFailure
 mapOneForallToThreadsImpl(RewriterBase &rewriter,
                           std::optional<TransformOpInterface> transformOp,
-                          scf::ForallOp forallOp, ArrayRef<int64_t> blockDims,
+                          scf::ForallOp forallOp, ArrayRef<int64_t> blockSizes,
                           int64_t warpSize, bool syncAfterDistribute);
 
 /// Search `scf.forall` ops nested under `target` and map each such op to an
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
index 36b579485fc04..87423c639945f 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
@@ -54,6 +54,20 @@ def ApplyGPUSubgroupReduceToNVVMConversionPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyGPUToROCDLConversionPatternsOp : Op<Transform_Dialect,
+    "apply_conversion_patterns.gpu.gpu_to_rocdl",
+    [DeclareOpInterfaceMethods<ConversionPatternDescriptorOpInterface,
+                               ["verifyTypeConverter"]>]> {
+  let description = [{
+    Collects patterns that convert GPU dialect ops to ROCDL dialect ops. These
+    patterns require an "LLVMTypeConverter".
+  }];
+  let arguments = (ins StrAttr:$chipset);
+  let assemblyFormat = [{
+    `chipset` `=` $chipset attr-dict
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // Apply...PatternsOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h b/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h
index 52fc6f4d5c71b..9cbdfd2fa218c 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h
@@ -28,27 +28,24 @@ namespace transform {
 namespace gpu {
 
 /// Helper type for functions that generate ids for the mapping of a scf.forall.
-/// Operates on both 1) an "original" basis that represents the individual
-/// thread and block ids and 2) a "scaled" basis that represents grouped ids
-/// (e.g. block clusters, warpgroups and warps).
-/// The mapping of ids is done in the "scaled" basis (i.e. when mapping to warps
-/// a division by 32 occurs).
-/// The predication is in the "original" basis using the "active" quantities
-/// (`activeMappingSizes`, `availableMappingSizes` and `activeIdOps`).
 struct IdBuilderResult {
-  // Ops used to replace the forall induction variables.
+  /// Error message, if not empty then building the ids failed.
+  std::string errorMsg;
+  /// Values used to replace the forall induction variables.
   SmallVector<Value> mappingIdOps;
-  // Available mapping sizes used to predicate the forall body when they are
-  // larger than the predicate mapping sizes.
-  SmallVector<int64_t> availableMappingSizes;
-  // Actual mapping sizes used to predicate the forall body when they are
-  // smaller than the available mapping sizes.
-  SmallVector<int64_t> activeMappingSizes;
-  // Ops used to predicate the forall body when activeMappingSizes is smaller
-  // than the available mapping sizes.
-  SmallVector<Value> activeIdOps;
+  /// Values used to predicate the forall body when activeMappingSizes is
+  /// smaller than the available mapping sizes.
+  SmallVector<Value> predicateOps;
 };
 
+inline raw_ostream &operator<<(raw_ostream &os, const IdBuilderResult &res) {
+  llvm::interleaveComma(res.mappingIdOps, os << "----mappingIdOps: ");
+  os << "\n";
+  llvm::interleaveComma(res.predicateOps, os << "----predicateOps: ");
+  os << "\n";
+  return os;
+}
+
 /// Common gpu id builder type, allows the configuration of lowering for various
 /// mapping schemes. Takes:
 ///   - A rewriter with insertion point set before the forall op to rewrite.
@@ -80,8 +77,10 @@ struct GpuIdBuilder {
 /// used for indexing rewrites as well as 3D sizes for predicate generation.
 /// If `useLinearMapping` is true, the `idBuilder` method returns nD values
 /// used for indexing rewrites as well as 1D sizes for predicate generation.
+/// If `mask` is provided, it will be used to filter the active blocks.
 struct GpuBlockIdBuilder : public GpuIdBuilder {
-  GpuBlockIdBuilder(MLIRContext *ctx, bool useLinearMapping = false);
+  GpuBlockIdBuilder(MLIRContext *ctx, bool useLinearMapping = false,
+                    DeviceMaskingAttrInterface mask = nullptr);
 };
 
 /// Builder for warpgroup ids used to map scf.forall to reindexed warpgroups.
@@ -89,9 +88,11 @@ struct GpuBlockIdBuilder : public GpuIdBuilder {
 /// used for indexing rewrites as well as 3D sizes for predicate generation.
 /// If `useLinearMapping` is true, the `idBuilder` method returns nD values
 /// used for indexing rewrites as well as 1D sizes for predicate generation.
+/// If `mask` is provided, it will be used to filter the active warpgroups.
 struct GpuWarpgroupIdBuilder : public GpuIdBuilder {
   GpuWarpgroupIdBuilder(MLIRContext *ctx, int64_t warpSize,
-                        bool useLinearMapping = false);
+                        bool useLinearMapping = false,
+                        DeviceMaskingAttrInterface mask = nullptr);
   int64_t warpSize = 32;
   /// In the future this may be configured by the transformation.
   static constexpr int64_t kNumWarpsPerGroup = 4;
@@ -102,9 +103,11 @@ struct GpuWarpgroupIdBuilder : public GpuIdBuilder {
 /// used for indexing rewrites as well as 3D sizes for predicate generation.
 /// If `useLinearMapping` is true, the `idBuilder` method returns nD values
 /// used for indexing rewrites as well as 1D sizes for predicate generation.
+/// If `mask` is provided, it will be used to filter the active warps.
 struct GpuWarpIdBuilder : public GpuIdBuilder {
   GpuWarpIdBuilder(MLIRContext *ctx, int64_t warpSize,
-                   bool useLinearMapping = false);
+                   bool useLinearMapping = false,
+                   DeviceMaskingAttrInterface mask = nullptr);
   int64_t warpSize = 32;
 };
 
@@ -113,8 +116,21 @@ struct GpuWarpIdBuilder : public GpuIdBuilder {
 /// used for indexing rewrites as well as 3D sizes for predicate generation.
 /// If `useLinearMapping` is true, the `idBuilder` method returns nD values
 /// used for indexing rewrites as well as 1D sizes for predicate generation.
+/// If `mask` is provided, it will be used to filter the active threads.
 struct GpuThreadIdBuilder : public GpuIdBuilder {
-  GpuThreadIdBuilder(MLIRContext *ctx, bool useLinearMapping = false);
+  GpuThreadIdBuilder(MLIRContext *ctx, bool useLinearMapping = false,
+                     DeviceMaskingAttrInterface mask = nullptr);
+};
+
+/// Builder for lane id.
+/// The `idBuilder` method returns nD values used for indexing rewrites as well
+/// as 1D sizes for predicate generation.
+/// This `useLinearMapping` case is the only supported case.
+/// If `mask` is provided, it will be used to filter the active lanes.
+struct GpuLaneIdBuilder : public GpuIdBuilder {
+  GpuLaneIdBuilder(MLIRContext *ctx, int64_t warpSize, bool unused,
+                   DeviceMaskingAttrInterface mask = nullptr);
+  int64_t warpSize = 32;
 };
 
 /// Determine if the size of the kernel configuration is supported by the
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index 9c5bbae1022f7..cfad07e57021f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -87,3 +87,13 @@ mlir_tablegen(VCIXConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(VCIXOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=vcix)
 mlir_tablegen(VCIXOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=vcix)
 add_public_tablegen_target(MLIRVCIXConversionsIncGen)
+
+add_mlir_dialect(XeVMOps xevm)
+add_mlir_doc(XeVMOps XeVMDialect Dialects/ -gen-dialect-doc -dialect=xevm)
+set(LLVM_TARGET_DEFINITIONS XeVMOps.td)
+mlir_tablegen(XeVMConversions.inc -gen-llvmir-conversions)
+mlir_tablegen(XeVMOpsEnums.h.inc -gen-enum-decls)
+mlir_tablegen(XeVMOpsEnums.cpp.inc -gen-enum-defs)
+mlir_tablegen(XeVMOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=xevm)
+mlir_tablegen(XeVMOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=xevm)
+add_public_tablegen_target(MLIRXeVMConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index b1799a9bccc17..e7b44fd94b26d 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -604,7 +604,7 @@ def FCmpPredicate : LLVM_EnumAttr<
 def ICmpPredicate : LLVM_EnumAttr<
     "ICmpPredicate",
     "::llvm::CmpInst::Predicate",
-    "lvm.icmp comparison predicate",
+    "llvm.icmp comparison predicate",
     [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE,
      ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE,
      ICmpPredicateUGT, ICmpPredicateUGE],
diff --git a/mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h
new file mode 100644
index 0000000000000..85bc19ad59794
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h
@@ -0,0 +1,28 @@
+//===-- XeVMDialect.h - MLIR XeVM target definitions ------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_
+#define MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_
+
+#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+
+#include "mlir/Dialect/LLVMIR/XeVMOpsEnums.h.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOpsAttributes.h.inc"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOps.h.inc"
+
+#include "mlir/Dialect/LLVMIR/XeVMOpsDialect.h.inc"
+
+#endif /* MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_ */
diff --git a/mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td
new file mode 100644
index 0000000000000..b5e81d595d74c
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td
@@ -0,0 +1,561 @@
+//===-- XeVMOps.td - XeVM dialect definition ---------------*- tablegen -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef XEVMIR_OPS
+#define XEVMIR_OPS
+
+include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td"
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+include "mlir/IR/OpBase.td"
+include "mlir/IR/EnumAttr.td"
+
+def XeVM_Dialect : Dialect {
+  let name = "xevm";
+  let cppNamespace = "::mlir::xevm";
+  let summary = "The XeVM dialect that extends LLVM dialect and models Intel "
+                "GPU's hardware features.";
+  let description = [{
+    The XeVM dialect is extension to the LLVM dialect that models hardware
+    features of Intel GPUs. The dialect is designed to work with the Xe
+    architecture for Intel GPUs, supporting advanced operations like 2D block
+    loads, stores, prefetch and matrix multiply-add (MMA) operations.
+  }];
+  let dependentDialects = ["LLVM::LLVMDialect"];
+
+  let extraClassDeclaration = [{
+    /// Get the name for the attribute used to specify cache control
+    /// decorations.
+    static constexpr ::llvm::StringRef getCacheControlsAttrName() {
+      return ::llvm::StringLiteral("xevm.DecorationCacheControl");
+    }
+  }];
+
+  let useDefaultAttributePrinterParser = 1;
+}
+
+class XeVM_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
+    : AttrDef<XeVM_Dialect, attrName, traits> {
+  let mnemonic = attrMnemonic;
+}
+
+class XeVM_Op<string mnemonic, list<Trait> traits = []>
+    : LLVM_OpBase<XeVM_Dialect, mnemonic, traits> {
+
+  code extraBaseClassDeclaration = [{
+    void printProperties(::mlir::MLIRContext *ctx,
+            ::mlir::OpAsmPrinter &p, const Properties &prop,
+            ::mlir::ArrayRef<::llvm::StringRef> elidedProps) {
+      Attribute propAttr = getPropertiesAsAttr(ctx, prop);
+      if (propAttr)
+        p << "<" << propAttr << ">";
+    }
+
+    static ::mlir::ParseResult parseProperties(::mlir::OpAsmParser &parser,
+                                     ::mlir::OperationState &result) {
+      if (mlir::succeeded(parser.parseOptionalLess())) {
+        if (parser.parseAttribute(result.propertiesAttr) || parser.parseGreater())
+          return failure();
+      }
+      return success();
+    }
+
+  }];
+}
+
+def XeVM_ElemType : AnyTypeOf<[AnyI8, AnyI16, AnyI32, F32, TF32, F16, BF16]>;
+
+//===----------------------------------------------------------------------===//
+// XeVM Load Cache Control
+//   L1, L2, L3 - cache levels
+//   uc - uncached
+//   c - cached
+//   s - streaming
+//   ir - invalidated after read
+//===----------------------------------------------------------------------===//
+
+def LoadCacheControl_L1uc_L2uc_L3uc
+    : I32EnumAttrCase<"L1UC_L2UC_L3UC", 1, "L1uc_L2uc_L3uc">;
+def LoadCacheControl_L1uc_L2uc_L3c
+    : I32EnumAttrCase<"L1UC_L2UC_L3C", 2, "L1uc_L2uc_L3c">;
+def LoadCacheControl_L1uc_L2c_L3uc
+    : I32EnumAttrCase<"L1UC_L2C_L3UC", 3, "L1uc_L2c_L3uc">;
+def LoadCacheControl_L1uc_L2c_L3c
+    : I32EnumAttrCase<"L1UC_L2C_L3C", 4, "L1uc_L2c_L3c">;
+def LoadCacheControl_L1c_L2uc_L3uc
+    : I32EnumAttrCase<"L1C_L2UC_L3UC", 5, "L1c_L2uc_L3uc">;
+def LoadCacheControl_L1c_L2uc_L3c
+    : I32EnumAttrCase<"L1C_L2UC_L3C", 6, "L1c_L2uc_L3c">;
+def LoadCacheControl_L1c_L2c_L3uc
+    : I32EnumAttrCase<"L1C_L2C_L3UC", 7, "L1c_L2c_L3uc">;
+def LoadCacheControl_L1c_L2c_L3c
+    : I32EnumAttrCase<"L1C_L2C_L3C", 8, "L1c_L2c_L3c">;
+def LoadCacheControl_L1s_L2uc_L3uc
+    : I32EnumAttrCase<"L1S_L2UC_L3UC", 9, "L1s_L2uc_L3uc">;
+def LoadCacheControl_L1s_L2uc_L3c
+    : I32EnumAttrCase<"L1S_L2UC_L3C", 10, "L1s_L2uc_L3c">;
+def LoadCacheControl_L1s_L2c_L3uc
+    : I32EnumAttrCase<"L1S_L2C_L3UC", 11, "L1s_L2c_L3uc">;
+def LoadCacheControl_L1s_L2c_L3c
+    : I32EnumAttrCase<"L1S_L2C_L3C", 12, "L1s_L2c_L3c">;
+def LoadCacheControlInvalidateRead
+    : I32EnumAttrCase<"INVALIDATE_READ", 13, "ir">;
+
+def XeVM_LoadCacheControl
+    : I32EnumAttr<
+          "LoadCacheControl", "XeVM load ops cache control",
+          [LoadCacheControl_L1uc_L2uc_L3uc, LoadCacheControl_L1uc_L2uc_L3c,
+           LoadCacheControl_L1uc_L2c_L3uc, LoadCacheControl_L1uc_L2c_L3c,
+           LoadCacheControl_L1c_L2uc_L3uc, LoadCacheControl_L1c_L2uc_L3c,
+           LoadCacheControl_L1c_L2c_L3uc, LoadCacheControl_L1c_L2c_L3c,
+           LoadCacheControl_L1s_L2uc_L3uc, LoadCacheControl_L1s_L2uc_L3c,
+           LoadCacheControl_L1s_L2c_L3uc, LoadCacheControl_L1s_L2c_L3c,
+           LoadCacheControlInvalidateRead]> {
+  let cppNamespace = "::mlir::xevm";
+  let genSpecializedAttr = 0;
+}
+
+def XeVM_LoadCacheControlAttr
+    : EnumAttr<XeVM_Dialect, XeVM_LoadCacheControl, "load_cache_control"> {
+  let summary = [{Describe the cache settings for load operators}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+//===----------------------------------------------------------------------===//
+// XeVM Store Cache Control
+//   L1, L2, L3 - cache levels
+//   uc - uncached
+//   wb - write-back
+//   wt - write-through
+//   s - streaming
+//===----------------------------------------------------------------------===//
+
+def StoreCacheControl_L1uc_L2uc_L3uc
+    : I32EnumAttrCase<"L1UC_L2UC_L3UC", 1, "L1uc_L2uc_L3uc">;
+def StoreCacheControl_L1uc_L2uc_L3wb
+    : I32EnumAttrCase<"L1UC_L2UC_L3WB", 2, "L1uc_L2uc_L3wb">;
+def StoreCacheControl_L1uc_L2wb_L3uc
+    : I32EnumAttrCase<"L1UC_L2WB_L3UC", 3, "L1uc_L2wb_L3uc">;
+def StoreCacheControl_L1uc_L2wb_L3wb
+    : I32EnumAttrCase<"L1UC_L2WB_L3WB", 4, "L1uc_L2wb_L3wb">;
+def StoreCacheControl_L1wt_L2uc_L3uc
+    : I32EnumAttrCase<"L1WT_L2UC_L3UC", 5, "L1wt_L2uc_L3uc">;
+def StoreCacheControl_L1wt_L2uc_L3wb
+    : I32EnumAttrCase<"L1WT_L2UC_L3WB", 6, "L1wt_L2uc_L3wb">;
+def StoreCacheControl_L1wt_L2wb_L3uc
+    : I32EnumAttrCase<"L1WT_L2WB_L3UC", 7, "L1wt_L2wb_L3uc">;
+def StoreCacheControl_L1wt_L2wb_L3wb
+    : I32EnumAttrCase<"L1WT_L2WB_L3WB", 8, "L1wt_L2wb_L3wb">;
+def StoreCacheControl_L1s_L2uc_L3uc
+    : I32EnumAttrCase<"L1S_L2UC_L3UC", 9, "L1s_L2uc_L3uc">;
+def StoreCacheControl_L1s_L2uc_L3wb
+    : I32EnumAttrCase<"L1S_L2UC_L3WB", 10, "L1s_L2uc_L3wb">;
+def StoreCacheControl_L1s_L2wb_L3uc
+    : I32EnumAttrCase<"L1S_L2WB_L3UC", 11, "L1s_L2wb_L3uc">;
+def StoreCacheControl_L1s_L2wb_L3wb
+    : I32EnumAttrCase<"L1S_L2WB_L3WB", 12, "L1s_L2wb_L3wb">;
+def StoreCacheControl_L1wb_L2uc_L3uc
+    : I32EnumAttrCase<"L1WB_L2UC_L3UC", 13, "L1wb_L2uc_L3uc">;
+def StoreCacheControl_L1wb_L2wb_L3uc
+    : I32EnumAttrCase<"L1WB_L2WB_L3UC", 14, "L1wb_L2wb_L3uc">;
+def StoreCacheControl_L1wb_L2uc_L3wb
+    : I32EnumAttrCase<"L1WB_L2UC_L3WB", 15, "L1wb_L2uc_L3wb">;
+
+def XeVM_StoreCacheControl
+    : I32EnumAttr<
+          "StoreCacheControl", "XeVM store ops cache control",
+          [StoreCacheControl_L1uc_L2uc_L3uc, StoreCacheControl_L1uc_L2uc_L3wb,
+           StoreCacheControl_L1uc_L2wb_L3uc, StoreCacheControl_L1uc_L2wb_L3wb,
+           StoreCacheControl_L1wt_L2uc_L3uc, StoreCacheControl_L1wt_L2uc_L3wb,
+           StoreCacheControl_L1wt_L2wb_L3uc, StoreCacheControl_L1wt_L2wb_L3wb,
+           StoreCacheControl_L1s_L2uc_L3uc, StoreCacheControl_L1s_L2uc_L3wb,
+           StoreCacheControl_L1s_L2wb_L3uc, StoreCacheControl_L1s_L2wb_L3wb,
+           StoreCacheControl_L1wb_L2uc_L3uc, StoreCacheControl_L1wb_L2wb_L3uc,
+           StoreCacheControl_L1wb_L2uc_L3wb]> {
+  let cppNamespace = "::mlir::xevm";
+  let genSpecializedAttr = 0;
+}
+
+def XeVM_StoreCacheControlAttr
+    : EnumAttr<XeVM_Dialect, XeVM_StoreCacheControl, "store_cache_control"> {
+  let summary = [{Describe the cache settings for store operators}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def XeVM_BlockLoad2dOp
+    : XeVM_Op<"blockload2d">,
+      Results<(outs FixedVectorOfRankAndType<[1], [XeVM_ElemType]>:$res)>,
+      Arguments<(ins Arg<LLVM_AnyPointer, "", [MemRead]>:$ptr, I32:$base_width,
+          I32:$base_height, I32:$base_pitch, I32:$x, I32:$y,
+          I32Attr:$elem_size_in_bits, I32Attr:$tile_width, I32Attr:$tile_height,
+          I32Attr:$v_blocks, I1Attr:$transpose, I1Attr:$pack_register,
+          OptionalAttr<XeVM_LoadCacheControlAttr>:$cache_control)> {
+
+  let summary = "2D block load";
+
+  let description = [{
+    The `xevm.blockload2d` operation loads a two dimensional matrix tile
+    from a base matrix residing in global memory. The parameters are:
+      $ptr - the base address of the base matrix containing the tile to load
+      $base_width - the width of the base matrix in number of bytes.
+      $base_height - the number of rows in the base matrix
+      $base_pitch - the physical stride between the first columns of the current
+         row and the subsequent row in number of bytes.
+      $x, $y, $tile_width, $tile_height - the starting offsets and shape of
+      the tile to load in number of elements.
+      $elem_size_in_bits - the size in bits of the matrix element type
+        - 32 for f32, tf32
+        - 16 for f16, int16, bf16
+        - 8 for int8
+      $v_blocks - number of consecutive tiles in innermost dimension direction to load
+      $transpose - transpose the tile in registers (useful for 32 bit element type)
+      $pack_register - pack element types narrower than register bit width.
+        [M, N] => [M/factor, N, factor] where factor is register_size_in_bits / elem_size_in_bits
+      $cache_control - an enumerator that sets the cache behaviour
+
+    Notes:
+      - the $transpose and $pack_register parameters are mutual exclusive
+      - transposing the tile loaded is used for A matrix in backward path or used for the B matrix operand
+        (D = C + A * B), where A has row-major layout and B should have column-major layout in memory.
+      - if the tile loaded contains out of bound elements of the matrix, they are filled with 0.
+
+    Example:
+    ```mlir
+      %base_width_a = arith.constant 32 : i32
+      %base_height_a = arith.constant 8 : i32
+      %base_pitch_a = arith.constant 32 : i32
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      %loaded_a = xevm.blockload2d %src, %base_width_a, %base_height_a, %base_pitch_a, %x, %y
+                    <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32,
+                      v_blocks=1 : i32, transpose=false : i32, pack_register=false,
+                      cache_control=#xevm.load_cache_control<Default>}>
+                    : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+    ```
+  }];
+
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeVM_BlockStore2dOp
+    : XeVM_Op<"blockstore2d">,
+      Arguments<(ins Arg<LLVM_AnyPointer, "", [MemWrite]>:$ptr, I32:$base_width,
+          I32:$base_height, I32:$base_pitch, I32:$x, I32:$y,
+          I32Attr:$elem_size_in_bits, I32Attr:$tile_width, I32Attr:$tile_height,
+          FixedVectorOfRankAndType<[1], [XeVM_ElemType]>:$stored_val,
+          OptionalAttr<XeVM_StoreCacheControlAttr>:$cache_control)> {
+
+  let summary = "2D block store";
+
+  let description = [{
+    The `xevm.blockstore2d` operation stores a two dimensional tile into a
+    larger matrix residing in global memory. The parameters are:
+      $ptr - the base address of the target matrix where to store the tile
+      $base_width - the width of the base matrix in number of bytes.
+      $base_height - the number of rows in the base matrix
+      $base_pitch - the physical stride between the first columns of the current
+         row and the subsequent row in number of bytes.
+      $x, $y, $tile_width, $tile_height - the starting offsets and shape of the tile to store
+      in number of elements.
+      $elem_size_in_bits - the size in bits of the matrix element
+        - 32 for f32, tf32
+        - 16 for f16, int16, bf16
+        - 8 for int8
+      $cache_control - an enumerator that sets the cache behaviour
+      $stored_val - the tile to store
+
+    Example:
+    ```mlir
+      %base_width_c = arith.constant 64 : i32
+      %base_height_c = arith.constant 8 : i32
+      %base_pitch_c = arith.constant 64 : i32
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      xevm.blockstore2d %dst, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %src
+        <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32,
+          cache_control=#xevm.load_cache_control<Default>}>
+        : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+    ```
+  }];
+
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` `(` type(operands) `)`
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+    /// Default value for v_blocks is 1.
+    constexpr uint32_t getVBlocks() {
+      return 1;
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+def MemScopeLane : I32EnumAttrCase<"LANE", 0, "lane">;
+def MemScopeSubgroup : I32EnumAttrCase<"SUBGROUP", 1, "subgroup">;
+def MemScopeWorkgroup : I32EnumAttrCase<"WORKGROUP", 2, "workgroup">;
+def MemScopeCluster : I32EnumAttrCase<"CLUSTER", 3, "cluster">;
+def MemScopeDevice : I32EnumAttrCase<"DEVICE", 4, "device">;
+def MemScopeSystem : I32EnumAttrCase<"SYSTEM", 5, "system">;
+
+def XeVM_MemScope
+    : I32EnumAttr<"MemScope", "XeVM memory scope",
+                  [MemScopeLane, MemScopeSubgroup, MemScopeWorkgroup,
+                   MemScopeCluster, MemScopeDevice, MemScopeSystem]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::xevm";
+}
+def XeVM_MemScopeAttr : EnumAttr<XeVM_Dialect, XeVM_MemScope, "mem_scope"> {
+  let summary = [{Describe memory scopes}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def AddrSpacePrivate : I32EnumAttrCase<"PRIVATE", 0, "private">;
+def AddrSpaceGlobal : I32EnumAttrCase<"GLOBAL", 1, "global">;
+def AddrSpaceConstant : I32EnumAttrCase<"CONSTANT", 2, "constant">;
+def AddrSpaceShared : I32EnumAttrCase<"SHARED", 3, "shared">;
+def AddrSpaceGeneric : I32EnumAttrCase<"GENERIC", 4, "generic">;
+
+def XeVM_AddrSpace
+    : I32EnumAttr<"AddrSpace", "Address spaces",
+                  [AddrSpacePrivate, AddrSpaceGlobal, AddrSpaceConstant,
+                   AddrSpaceShared, AddrSpaceGeneric]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "mlir::xevm";
+}
+def XeVM_AddrSpaceAttr : EnumAttr<XeVM_Dialect, XeVM_AddrSpace, "addr_space"> {
+  let summary = [{Describe address spaces}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def XeVM_MemfenceOp
+    : XeVM_Op<"memfence">,
+      Arguments<(ins XeVM_MemScopeAttr:$scope,
+          DefaultValuedAttr<XeVM_AddrSpaceAttr,
+                            "mlir::xevm::AddrSpace::GENERIC">:$addrspace)> {
+  let summary = "Work-item's memory fence.";
+  let description = [{
+    This operation ensures that all prior memory accesses of this
+    work-item to `addrspace` are visible to all other work-items in `scope`.
+    Parameters description:
+    $scope - specify the memory scope at which all other work-items should observe
+                memory operations prior to the fence.
+    $addrspace - specify the address space of work-item's memory accesses
+                to be affected by the fence.
+  }];
+  let assemblyFormat = [{prop-dict  attr-dict}];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+}
+
+def XeVM_PrefetchOp
+    : XeVM_Op<"prefetch">,
+      Arguments<(ins Arg<AnyTypeOf<[LLVM_PointerInAddressSpace<1>,
+                                    LLVM_PointerInAddressSpace<4>]>>:$ptr,
+          OptionalAttr<XeVM_LoadCacheControlAttr>:$cache_control)> {
+  let summary = "Prefetch data into a cache subsystem.";
+  let description = [{
+    Work-item issues a prefetch from global memory to cache:
+    $ptr - LLVM pointer with address space. Address space must be 1 (global)
+      or 4 (generic)
+    $cache_control - specify caching options
+  }];
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` `(` type(operands) `)`
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+}
+
+def XeVM_BlockPrefetch2dOp
+    : XeVM_Op<"blockprefetch2d">,
+      Arguments<(ins Arg<LLVM_AnyPointer, "", [MemRead]>:$ptr, I32:$base_width,
+          I32:$base_height, I32:$base_pitch, I32:$x, I32:$y,
+          I32Attr:$elem_size_in_bits, I32Attr:$tile_width, I32Attr:$tile_height,
+          I32Attr:$v_blocks,
+          OptionalAttr<XeVM_LoadCacheControlAttr>:$cache_control)> {
+
+  let summary = "2D block prefetch";
+
+  let description = [{
+    The `xevm.blockprefetch2d` operation prefetches a two dimensional tile
+    from a larger base matrix residing in global memory. The parameters are:
+      $ptr - the base address of the base matrix containing the tile to prefetch
+      $base_width - the width of the base matrix in number of bytes.
+      $base_height - the number of rows in the base matrix
+      $base_pitch - the physical stride between the first columns of the current
+         row and the subsequent row in number of bytes.
+      $x, $y, $tile_width, $tile_height - the starting offsets and shape of tile
+      to prefetch in number of elements.
+      $elem_size_in_bits - the size in bits of the matrix element
+      - 32 for f32, bf32
+      - 16 for f16, int16, bf16
+      - 8 for int8, int4, int2
+      $v_blocks - number of tiles in innermost dimension direction to prefetch
+      $cache_control - an enumerator that sets the cache behaviour
+
+    Example:
+    ```mlir
+      xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y
+        <{elem_size_in_bits=8 : i32, tile_width=32 : i32, tile_height=8 : i32,
+          v_blocks=1 : i32, cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}>
+        : (!llvm.ptr<1>, i32, i32, i32, i32, i32)
+    ```
+  }];
+
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` `(` type(operands) `)`
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeVM_MatrixElemType
+    : AnyTypeOf<[AnyI8, AnyI16, AnyI32, F32, TF32, F16, BF16]>;
+
+/// Enum attribute of the different element types.
+def XeVM_ET_BF16 : I32EnumAttrCase<"BF16", 8, "bf16">;
+def XeVM_ET_F16 : I32EnumAttrCase<"F16", 9, "f16">;
+def XeVM_ET_S8 : I32EnumAttrCase<"S8", 10, "s8">;
+def XeVM_ET_U8 : I32EnumAttrCase<"U8", 11, "u8">;
+def XeVM_ET_S4 : I32EnumAttrCase<"S4", 12, "s4">;
+def XeVM_ET_U4 : I32EnumAttrCase<"U4", 13, "u4">;
+def XeVM_ET_TF32 : I32EnumAttrCase<"TF32", 14, "tf32">;
+def XeVM_ET_F32 : I32EnumAttrCase<"F32", 15, "f32">;
+def XeVM_ET_S32 : I32EnumAttrCase<"S32", 16, "s32">;
+
+def XeVM_ElemTypeAttr : I32EnumAttr<"ElemType", "XeVM element type",
+                                    [XeVM_ET_BF16, XeVM_ET_F16, XeVM_ET_S8,
+                                     XeVM_ET_U8, XeVM_ET_S4, XeVM_ET_U4,
+                                     XeVM_ET_TF32, XeVM_ET_F32, XeVM_ET_S32]> {
+  let cppNamespace = "::mlir::xevm";
+}
+
+def XeVM_MMAShapeAttr : XeVM_Attr<"MMAShape", "mma_shape"> {
+  let description = [{
+    MMA operation is represented as D=AxB+C, where
+      A has the shape MxK.
+      B has the shape KxN.
+      D and C have the shape MxN.
+    This attribute encodes the shape of all matrices that participate in MMA.
+  }];
+  let parameters = (ins "int":$m, "int":$n, "int":$k);
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def XeVM_MMATypesAttr : XeVM_Attr<"MMATypes", "mma_types"> {
+  let parameters = (ins "xevm::ElemType":$d, "xevm::ElemType":$a,
+      "xevm::ElemType":$b, OptionalParameter<"xevm::ElemType">:$c);
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def XeVM_MMAOp
+    : XeVM_Op<"mma">,
+      Results<(outs FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$d)>,
+      Arguments<(ins FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$a,
+          FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$b,
+          Optional<FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>>:$c,
+          XeVM_MMAShapeAttr:$shape, XeVM_MMATypesAttr:$types)> {
+
+  let summary = "Subgroup matrix multiply-add";
+
+  let description = [{
+    The `xevm.mma` is a cooperative operation where all threads/lanes in
+    a subgroup participates and carries out matrix multiplication plus accumulation:
+
+      D = C + A x B
+
+      where the A, B, C input matrices and the result D have shapes:
+        D : MxN
+        C : MxN
+        A : MxK
+        B : KxN
+
+    Parameters:
+      `a` - vector of matrix A elements.
+      `b` - vector of matrix B elements.
+      `c` - (optional) vector of matrix C elements.
+      `shape` - the shape of the matrices, specified as `M`, `N`, and `K` values.
+      `types` - the data types of the matrices, specified as `D`, `A`, `B`, and optionally `C`.
+
+    Example:
+    ```mlir
+      %d = xevm.mma %a, %b, %c { shape=<m=8, n=16, k=16>, types=<d=f32, a=f16, b=f16, c=f32> }
+             : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+    ```
+  }];
+
+  let assemblyFormat = [{
+    $a `,` $b (`,` $c^)? ` `
+    `{`
+      `shape` `=` $shape `,`
+      `types` `=` $types
+    `}` attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// XeVM target attribute.
+//===----------------------------------------------------------------------===//
+
+def XeVM_TargetAttr : XeVM_Attr<"XeVMTarget", "target"> {
+  let description = [{
+    GPU target attribute for controlling compilation of Intel GPU targets. All
+    parameters decay into default values if not present.
+
+    Examples:
+
+    1. Target with default values.
+    ```
+      gpu.module @mymodule [#xevm.target] attributes {...} {
+        ...
+      }
+    ```
+  }];
+  let parameters =
+      (ins DefaultValuedParameter<"int", "2",
+                                  "Optimization level to apply.">:$O,
+          StringRefParameter<"Target triple.",
+                             "\"spirv64-unknown-unknown\"">:$triple,
+          StringRefParameter<"Target chip.", "\"bmg\"">:$chip,
+          OptionalParameter<"::mlir::DictionaryAttr",
+                            "Target specific flags.">:$flags,
+          OptionalParameter<"::mlir::ArrayAttr",
+                            "Files to link to the LLVM module.">:$linkFiles);
+  let assemblyFormat = [{
+    (`<` struct($O, $triple, $chip, $flags, $linkFiles)^ `>`)?
+  }];
+  let builders = [AttrBuilder<
+      (ins CArg<"int", "2">:$optLevel,
+          CArg<"::llvm::StringRef", "\"spirv64-unknown-unknown\"">:$triple,
+          CArg<"::llvm::StringRef", "\"bmg\"">:$chip,
+          CArg<"::mlir::DictionaryAttr", "nullptr">:$targetFlags,
+          CArg<"::mlir::ArrayAttr", "nullptr">:$linkFiles),
+      [{
+      return Base::get($_ctxt, optLevel, triple, chip, targetFlags, linkFiles);
+    }]>];
+  let skipDefaultBuilders = 1;
+  let genVerifyDecl = 1;
+}
+
+#endif // XEVMIR_OPS
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
index 6344861c53ac5..3637147c5a90d 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
@@ -44,56 +44,6 @@ structured_op: !LinalgStructuredOpConfig
         - !ScalarExpression
           scalar_arg: I
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: elemwise_unary
-  cpp_class_name: ElemwiseUnaryOp
-  doc: |-
-    Applies the unary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: I
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: O
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: fun
-    kind: unary_fn_attr
-    default_fn: exp
-  - !LinalgOperandDefConfig
-    name: cast
-    kind: type_fn_attr
-    default_fn: cast_signed
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<() -> ()>
-    - affine_map<() -> ()>
-  iterator_types: []
-  assignments:
-  - !ScalarAssign
-    arg: O
-    value: !ScalarExpression
-      scalar_fn:
-        kind: unary
-        attr_name: fun
-        operands:
-        - !ScalarExpression
-          scalar_fn:
-            kind: type
-            attr_name: cast
-            type_var: U
-            operands:
-            - !ScalarExpression
-              scalar_arg: I
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: exp
   cpp_class_name: ExpOp
@@ -549,70 +499,6 @@ structured_op: !LinalgStructuredOpConfig
         - !ScalarExpression
           scalar_arg: I
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: elemwise_binary
-  cpp_class_name: ElemwiseBinaryOp
-  doc: |-
-    Applies the binary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: lhs
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: rhs
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: O
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: fun
-    kind: binary_fn_attr
-    default_fn: add
-  - !LinalgOperandDefConfig
-    name: cast
-    kind: type_fn_attr
-    default_fn: cast_signed
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<() -> ()>
-    - affine_map<() -> ()>
-    - affine_map<() -> ()>
-  iterator_types: []
-  assignments:
-  - !ScalarAssign
-    arg: O
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        attr_name: fun
-        operands:
-        - !ScalarExpression
-          scalar_fn:
-            kind: type
-            attr_name: cast
-            type_var: U
-            operands:
-            - !ScalarExpression
-              scalar_arg: lhs
-        - !ScalarExpression
-          scalar_fn:
-            kind: type
-            attr_name: cast
-            type_var: U
-            operands:
-            - !ScalarExpression
-              scalar_arg: rhs
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: add
   cpp_class_name: AddOp
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td
index 1e48a5e3a20ee..c384e8b638382 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td
@@ -93,17 +93,21 @@ def Linalg_PackOp : Linalg_RelayoutOp<"pack", [
     tensor of rank `n + k` with a tiled and packed layout (maybe with padding)
     and optionally transposes the tiled source tensor dimensions.
 
-    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions that are
-    being tiled, where `0 < k <= n`. The order of the dimensions matters:
-     - The tiled dimensions (of size `inner_tiles`) are added to the end of the result
-    tensor in the order in which they appear in `inner_dims_pos`.
-     - `inner_dims_pos[i]` specifies the source tensor dimension tiled by
-    `inner_tiles[i]`.
-
     `inner_tiles` (mandatory) specifies `k` tile sizes. These tile sizes
     correspond to the least significant ("inner") result tensor dimension sizes,
     in the same order. Tile sizes can be static or dynamic.
 
+    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions that are
+    being tiled, where `0 <= k <= n`.
+     - `inner_dims_pos[i]` specifies the source tensor dimension tiled by
+    `inner_tiles[i]` where `0 <= i < k`. All the values in `inner_dims_pos` are
+    within [0, n).
+     - The tiled dimensions (of size `inner_tiles`) are added to the end of the
+     result tensor in the order in which they appear, i.e.
+     `shape(result)[rank(result) + i] = inner_tiles[i]` for `0 <= i < k`.
+     - The following relationship for the tiled dimensions holds:
+     `shape(result)[inner_dims_pos[i]] = shape(source)[inner_dims_pos[i]] / inner_tiles[i]`.
+
     Example: If `inner_tiles = [16, 32]`, the result tensor has a shape of
     `...x16x32`. If `inner_dims_pos = [0, 1]`, the 0th source dimension is tiled
     by 16 and the 1st source dimension is tiled by 32. Other source dimensions
@@ -116,7 +120,19 @@ def Linalg_PackOp : Linalg_RelayoutOp<"pack", [
     %0 = linalg.pack %source inner_dims_pos = [0, 1] inner_tiles = [8, 32]
         into %dest : tensor<128x256xf32> -> tensor<16x8 x 8x32 xf32>
     //                                             \  /   \  /
-    //                                       outer dims  inner dims
+    //                                 Outer Dims: 16x8   Inner Dims: 8x32
+
+    // CHW to CHWhw
+    %0 = linalg.pack %source inner_dims_pos = [2, 1] inner_tiles = [4, 2]
+        into %dest : tensor<3x20x24xf32> -> tensor<3x10x6 x 4x2 xf32>
+    //                                              \  /    \ /
+    //                                 Outer Dims: 3x10x6  Inner Dims: 4x2
+
+    // HCW to HCWhw
+    %0 = linalg.pack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2]
+        into %dest : tensor<18x3x32xf32> -> tensor<9x3x8 x 4x2 xf32>
+    //                                              \  /   \ /
+    //                                 Outer Dims: 9x3x8  Inner Dims: 4x2
     ```
 
     `outer_dims_perm` (optional) specifies a permutation for the outer
@@ -246,13 +262,6 @@ def Linalg_UnPackOp : Linalg_RelayoutOp<"unpack"> {
     The "unpack" operation converts a source tensor of rank `n` with a tiled and
     packed layout to a result tensor of rank `n - k`.
 
-    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions with
-    which the last `k` source tensor dimensions are combined, where
-    `0 < k <= n/2`. Each `inner_dims_pos` element must be `>= 0` and `< n - k`.
-    The order of the dimensions in `inner_dims_pos` matters: dimension
-    `inner_dims_pos[i]` is combined with dimension `n - k + i` (assuming that
-    `outer_dims_perm` is not specified).
-
     `inner_tiles` (mandatory) specifies `k` tile sizes. These tile sizes
     correspond to the least significant ("inner") source tensor dimension sizes.
     The behavior of this op is undefined if:
@@ -262,21 +271,50 @@ def Linalg_UnPackOp : Linalg_RelayoutOp<"unpack"> {
       `inner_dims_pos[i]` (assuming that `outer_dims_perm` is not specified)
       evenly.
 
+    `inner_dims_pos` (mandatory) specifies `k` result tensor (i.e. unpacked
+    tensor) dimensions that were tiled with the `inner_tiles` to create the
+    packed source tensor. The source tensor (i.e. packed tensor) dimensions can
+    be unpacked given `inner_dims_pos` as follows.
+    - For `0 <= i < k` the following relationship holds:
+    `shape(result)[inner_dims_pos[i]] <= shape(source)[n-k+i] * shape(source)[inner_dims_pos[i]]`.
+    - For `0 <= j < n-k` and `j` not in `inner_dims_pos` the following relationship holds:
+    `shape(result)[j] = shape(source)[j]`.
+
     `outer_dims_perm` (optional) specifies a permutation for the outer
     dimensions. If specified, it must have `n - k` elements. If specified, this
     permutation is applied before combining any dimensions.
 
-    Example:
+    Note, the unpack operation may drop any padding introduced by the pack
+    operation and hence the following holds
+    `NumElementsOf(source) >= NumElementsOf(result)`.
+
+    Examples:
 
     ```mlir
     // NCnc to NC:
     %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [8, 32]
-        into %dest : tensor<16x8x8x32xf32> -> tensor<128x256xf32>
+        into %dest : tensor<16x8 x 8x32 xf32> -> tensor<128x256xf32>
+    //                      \  /   \  /
+    //          Outer Dims: 16x8  Inner Dims: 8x32
 
     // CK to KCck:
     %0 = linalg.unpack %source outer_dims_perm = [1, 0] inner_dims_pos = [0, 1]
-        inner_tiles = [8, 32] into %dest
-        : tensor<8x16x8x32xf32> -> tensor<128x256xf32>
+        inner_tiles = [8, 32]
+        into %dest : tensor<8x16 x 8x32 xf32> -> tensor<128x256xf32>
+    //                      \  /   \  /
+    //          Outer Dims: 8x16  Inner Dims: 8x32
+
+    // CHW to CHWhw:
+    %0 = linalg.unpack %source inner_dims_pos = [2, 1] inner_tiles = [4, 2]
+        into %dest : tensor<3x10x6 x 4x2 xf32> -> tensor<3x20x24xf32>
+    //                       \  /    \ /
+    //          Outer Dims: 3x10x6  Inner Dims: 4x2
+
+    // HCW to HCWhw
+    %0 = linalg.unpack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2]
+        into %dest : tensor<9x3x8 x 4x2 xf32> -> tensor<18x3x32xf32>
+    //                       \  /   \ /
+    //          Outer Dims: 9x3x8   Inner Dims: 4x2
     ```
   }];
   let arguments = (ins AnyRankedTensor:$source,
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index d64f94a49f781..4360055e78691 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -1402,6 +1402,7 @@ def PromoteOp : Op<Transform_Dialect, "structured.promote",
                        DefaultValuedAttr<I64ArrayAttr, "{}">:$operands_to_promote,
                        DefaultValuedAttr<BoolArrayAttr, "{}">:$use_full_tile_buffers,
                        UnitAttr:$use_full_tiles_by_default,
+                       UnitAttr:$use_original_subview_size,
                        UnitAttr:$use_alloca,
                        OptionalAttr<AnyAttr>:$memory_space,
                        OptionalAttr<DeviceMappingArrayAttr>:$mapping,
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 2b4855f49695c..74280fdd82f4e 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -422,6 +422,13 @@ struct LinalgPromotionOptions {
     useFullTileBuffersDefault = use;
     return *this;
   }
+  /// If true, buffers will be allocated with the original subview size. This
+  /// may result in more dynamic allocations, in case of dynamic sizes.
+  bool useOriginalSubviewSize = false;
+  LinalgPromotionOptions &setUseOriginalSubviewSize(bool originalSize) {
+    useOriginalSubviewSize = originalSize;
+    return *this;
+  }
   /// Alignment of promoted buffer. If `std::nullopt` do not specify alignment.
   std::optional<unsigned> alignment;
   LinalgPromotionOptions &setAlignment(unsigned align) {
@@ -796,7 +803,8 @@ FailureOr<LinalgOp> specializeGenericOp(RewriterBase &rewriter,
                                         GenericOp genericOp);
 
 /// Create a new buffer using the `allocationFn` provided. The size of this
-/// buffer is the smallest constant bounding size along each dimension that
+/// buffer is either the original subview size when 'useOriginalSubviewSize' is
+/// set to true or the smallest constant bounding size along each dimension that
 /// can be computed for the size of the result of `subView`. Returns the
 /// allocated buffer as `fullLocalView` and the view that matches the size of
 /// the result of subview operation as `partialLocalView`.
@@ -806,6 +814,7 @@ struct PromotionInfo {
 };
 FailureOr<PromotionInfo>
 promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView,
+                          bool useOriginalSubviewSize,
                           const AllocBufferCallbackFn &allocationFn,
                           DataLayout &layout);
 
@@ -1984,10 +1993,15 @@ void populateDecomposeWinogradOpsPatterns(RewritePatternSet &patterns);
 /// convert to a `linalg.dot`.
 void populateContractionOpRankReducingPatterns(RewritePatternSet &patterns);
 
+/// Function type which is used to control folding operations like `tensor.pad`
+/// and `tensor.extract_slice` into linalg.pack/unpack ops.
+using ControlFoldIntoPackUnpackFn = std::function<bool(OpOperand *opOperand)>;
 /// Populates `patterns` with patterns that fold operations like `tensor.pad`
 /// and `tensor.extract_slice` into `tensor.pack` and `tensor.unpack` operations
 /// respectively.
-void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns);
+void populateFoldIntoPackAndUnpackPatterns(
+    RewritePatternSet &patterns,
+    const ControlFoldIntoPackUnpackFn &controlFn = nullptr);
 
 /// Populates `patterns` with patterns that fold operations like `linalg.pack`
 /// and `linalg.unpack` into `tensor.empty`.
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index b0fb5b0785142..09bb3932ef293 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -2333,6 +2333,7 @@ def MemRef_ViewOp : MemRef_Op<"view", [
 
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td b/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
index a8d135caa74f0..f3e40aaa29075 100644
--- a/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
@@ -182,6 +182,68 @@ def ResolveShapedTypeResultDimsPass : Pass<"resolve-shaped-type-result-dims"> {
   ];
 }
 
+def ReifyResultShapesPass : Pass<"reify-result-shapes"> {
+  let summary ="Reifies the results of `tensor::PadOp` and `tensor::ConcatOp`.";
+  let description = [{
+    This pass reifies the shapes of a subset of `ReifyRankedShapedTypeOpInterface`
+    ops with `tensor` results.
+    
+    The pass currently only supports result shape type reification for:
+      - tensor::PadOp
+      - tensor::ConcatOp
+    It addresses a representation gap where implicit op semantics are needed to
+    infer static result types from dynamic operands.
+    But it does so by using `ReifyRankedShapedTypeOpInterface` as the source of
+    truth rather than the op itself. As a consequence, this cannot generalize
+    today.
+
+    TODO: in the future, we should consider coupling this information with op
+    "transfer functions" (e.g. `IndexingMapOpInterface`) to provide a source of
+    truth that can work across result shape inference, canonicalization and op
+    verifiers.
+
+    The pass replaces the operations with their reified versions, when more
+    static information can be derived, and inserts casts when results shapes
+    are updated.
+
+    Example:
+    ```mlir
+    #map = affine_map<(d0) -> (-d0 + 256)>
+    func.func @func(%arg0: f32, %arg1: index, %arg2: tensor<64x?x64xf32>)
+        -> tensor<1x?x64xf32>
+    {
+      %0 = affine.apply #map(%arg1)
+      %extracted_slice = tensor.extract_slice %arg2[0, 0, 0] [1, %arg1, 64] [1, 1, 1]
+        : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+      %padded = tensor.pad %extracted_slice low[0, 0, 0] high[0, %0, 0] {
+      ^bb0(%arg3: index, %arg4: index, %arg5: index):
+        tensor.yield %arg0 : f32
+      } : tensor<1x?x64xf32> to tensor<1x?x64xf32>
+      return %padded : tensor<1x?x64xf32>
+    }
+
+    // mlir-opt --reify-result-shapes
+    #map = affine_map<()[s0] -> (-s0 + 256)>
+    func.func @func(%arg0: f32, %arg1: index, %arg2: tensor<64x?x64xf32>)
+        -> tensor<1x?x64xf32>
+    {
+      %0 = affine.apply #map()[%arg1]
+      %extracted_slice = tensor.extract_slice %arg2[0, 0, 0] [1, %arg1, 64] [1, 1, 1]
+        : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+      %padded = tensor.pad %extracted_slice low[0, 0, 0] high[0, %0, 0] {
+      ^bb0(%arg3: index, %arg4: index, %arg5: index):
+        tensor.yield %arg0 : f32
+      } : tensor<1x?x64xf32> to tensor<1x256x64xf32>
+      %cast = tensor.cast %padded : tensor<1x256x64xf32> to tensor<1x?x64xf32>
+      return %cast : tensor<1x?x64xf32>
+    }
+    ```
+  }];
+  let dependentDialects = [
+    "affine::AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
+  ];
+}
+
 def ExpandStridedMetadataPass : Pass<"expand-strided-metadata"> {
   let summary = "Expand memref operations into easier to analyze constructs";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h b/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
index c2b8cb05be922..33e3d94f02b1c 100644
--- a/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
@@ -23,6 +23,7 @@ class RewritePatternSet;
 class RewriterBase;
 class Value;
 class ValueRange;
+class ReifyRankedShapedTypeOpInterface;
 
 namespace arith {
 class WideIntEmulationConverter;
@@ -208,7 +209,6 @@ FailureOr<Value> replaceWithIndependentOp(RewriterBase &rewriter,
 memref::AllocaOp allocToAlloca(
     RewriterBase &rewriter, memref::AllocOp alloc,
     function_ref<bool(memref::AllocOp, memref::DeallocOp)> filter = nullptr);
-
 } // namespace memref
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 8cbdf710cfa6e..66378f116784e 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -2010,6 +2010,25 @@ def OpenACC_EnterDataOp : OpenACC_Op<"enter_data",
 
     /// The i-th data operand passed.
     Value getDataOperand(unsigned i);
+
+    /// Add an entry to the 'async-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addAsyncOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add a value to the 'async'. DeviceType array is supplied even though it
+    /// should always be empty, so this can mirror other versions of this
+    /// function.
+    void addAsyncOperand(MLIRContext *, mlir::Value,
+                         llvm::ArrayRef<DeviceType>);
+    /// Add an entry to the 'wait-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addWaitOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add an array-like entry  to the 'wait'. DeviceType array is supplied
+    /// even though it should always be empty, so this can mirror other versions
+    /// of this function.
+    void addWaitOperands(MLIRContext *, bool hasDevnum, mlir::ValueRange,
+                         llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
@@ -2064,6 +2083,26 @@ def OpenACC_ExitDataOp : OpenACC_Op<"exit_data",
 
     /// The i-th data operand passed.
     Value getDataOperand(unsigned i);
+
+    /// Add an entry to the 'async-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addAsyncOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add a value to the 'async'. DeviceType array is supplied even though it
+    /// should always be empty, so this can mirror other versions of this
+    /// function.
+    void addAsyncOperand(MLIRContext *, mlir::Value,
+                         llvm::ArrayRef<DeviceType>);
+
+    /// Add an entry to the 'wait-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addWaitOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add an array-like entry  to the 'wait'. DeviceType array is supplied
+    /// even though it should always be empty, so this can mirror other versions
+    /// of this function.
+    void addWaitOperands(MLIRContext *, bool hasDevnum, mlir::ValueRange,
+                         llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
@@ -2732,18 +2771,15 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     must be modified as well.
   }];
 
-  let arguments = (ins SymbolNameAttr:$sym_name,
-                       SymbolNameAttr:$func_name,
-                       OptionalAttr<StrArrayAttr>:$bindName,
-                       OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
-                       OptionalAttr<DeviceTypeArrayAttr>:$worker,
-                       OptionalAttr<DeviceTypeArrayAttr>:$vector,
-                       OptionalAttr<DeviceTypeArrayAttr>:$seq,
-                       UnitAttr:$nohost,
-                       UnitAttr:$implicit,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gang,
-                       OptionalAttr<I64ArrayAttr>:$gangDim,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
+  let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
+      OptionalAttr<StrArrayAttr>:$bindName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$worker,
+      OptionalAttr<DeviceTypeArrayAttr>:$vector,
+      OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
+      UnitAttr:$implicit, OptionalAttr<DeviceTypeArrayAttr>:$gang,
+      OptionalAttr<I64ArrayAttr>:$gangDim,
+      OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
 
   let extraClassDeclaration = [{
     static StringRef getGangDimKeyword() { return "dim"; }
@@ -2989,6 +3025,21 @@ def OpenACC_UpdateOp : OpenACC_Op<"update",
     /// Return the wait devnum value clause for the given device_type if
     /// present.
     mlir::Value getWaitDevnum(mlir::acc::DeviceType deviceType);
+    /// Add an entry to the 'async-only' attribute (clause spelled without
+    /// arguments)for each of the additional device types (or a none if it is
+    /// empty).
+    void addAsyncOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add a value to the 'async' with the current list of device types.
+    void addAsyncOperand(MLIRContext *, mlir::Value,
+                         llvm::ArrayRef<DeviceType>);
+    /// Add an entry to the 'wait-only' attribute (clause spelled without
+    /// arguments)for each of the additional device types (or a none if it is
+    /// empty).
+    void addWaitOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add an array-like entry  to the 'wait' with the current list of device
+    /// types.
+    void addWaitOperands(MLIRContext *, bool hasDevnum, mlir::ValueRange,
+                         llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
index f9a85626a3f14..faf820dcfdb29 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
@@ -15,14 +15,10 @@
 #ifndef MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 #define MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "llvm/ADT/SmallVector.h"
 
-#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h.inc"
-
-#define GET_ATTRDEF_CLASSES
-#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h.inc"
-
 #include "mlir/Dialect/OpenMP/OpenMPClauseOps.h.inc"
 
 namespace mlir {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
index 248ac2eb72c61..ab11a6094e3e7 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/OpenACCMPOpsInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -33,8 +34,6 @@
 
 #include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.h.inc"
 
-#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
-
 #define GET_OP_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOps.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
index 989ab1710c211..5ebf7c8090c13 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
@@ -13,7 +13,8 @@
 #ifndef MLIR_DIALECT_OPENMP_OPENMPINTERFACES_H_
 #define MLIR_DIALECT_OPENMP_OPENMPINTERFACES_H_
 
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
new file mode 100644
index 0000000000000..6d196decfad03
--- /dev/null
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
@@ -0,0 +1,18 @@
+//===- OpenMPOpsAttributes.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
+#define MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
+
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h"
+#include "mlir/IR/BuiltinAttributes.h"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h.inc"
+
+#endif // MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
new file mode 100644
index 0000000000000..edf316ea9ca8b
--- /dev/null
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
@@ -0,0 +1,19 @@
+//===- OpenMPOpsEnums.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
+#define MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
+
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h.inc"
+
+#endif // MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
diff --git a/mlir/include/mlir/Dialect/SCF/IR/DeviceMappingInterface.td b/mlir/include/mlir/Dialect/SCF/IR/DeviceMappingInterface.td
index 96db2a40cf58e..24e3d5ceb45b5 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/DeviceMappingInterface.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/DeviceMappingInterface.td
@@ -60,8 +60,68 @@ def DeviceMappingAttrInterface : AttrInterface<"DeviceMappingAttrInterface"> {
   ];
 }
 
+def DeviceMaskingAttrInterface : AttrInterface<"DeviceMaskingAttrInterface"> {
+  let cppNamespace = "::mlir";
+  let description = [{
+    Attribute interface describing how to filter the processing units that a
+    region is mapped to.
+
+    For instance, consider the following example mask which specifies processing
+    units 2, 4 and 5 are active:
+    ```
+              8       4       0
+      mask  : 0 0 0 1 1 0 1 0 0
+    ```
+    The logical ID for an active processing unit is defined as its position
+    relative to the other active processing units. In this example, we have:
+    ```
+      Processing Unit    LogicalID
+              0             N/A
+              1             N/A
+              2              0
+              3             N/A
+              4              1
+              5              2
+              6             N/A
+              7             N/A
+    ```
+  }];
+
+ let methods = [
+    InterfaceMethod<
+      /*desc=*/[{
+        Create the logical active id for a given physical id.
+        Expects a physicalLinearMappingId of I64Type.
+      }],
+      /*retTy=*/"Value",
+      /*methodName=*/"createLogicalLinearMappingId",
+      /*args=*/(ins "OpBuilder&":$builder, "Value":$physicalLinearMappingId)
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return the dynamic condition determining whether a given physical id is
+        active under the mask.
+        Expects a physicalLinearMappingId of I64Type.
+      }],
+      /*retTy=*/"Value",
+      /*methodName=*/"createIsActiveIdPredicate",
+      /*args=*/(ins "OpBuilder&":$builder, "Value":$physicalLinearMappingId)
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return the maximal number of pysical ids supported.
+        This is to account for temporary implementation limitations (e.g. i64)
+        and fail gracefully with actionnable error messages.
+      }],
+      /*retTy=*/"int64_t",
+      /*methodName=*/"getMaxNumPhysicalIds",
+      /*args=*/(ins)
+    >,
+  ];
+}
+
 def DeviceMappingArrayAttr :
-  TypedArrayAttrBase<DeviceMappingAttrInterface,
+  TypedArrayAttrBase<AnyAttrOf<[DeviceMappingAttrInterface, DeviceMaskingAttrInterface]>,
   "Device Mapping array attribute"> { }
 
 #endif // MLIR_DEVICEMAPPINGINTERFACE
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index 8b14cef7437d4..2d15544e871b3 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -611,6 +611,18 @@ def ForallOp : SCF_Op<"forall", [
     /// Returns operations within scf.forall.in_parallel whose destination
     /// operand is the block argument `bbArg`.
     SmallVector<Operation*> getCombiningOps(BlockArgument bbArg);
+
+    /// Returns the subset of DeviceMappingArrayAttrs of type
+    /// DeviceMappingAttrInterface.
+    SmallVector<DeviceMappingAttrInterface> getDeviceMappingAttrs();
+
+    /// Returns the at most one DeviceMaskingAttrInterface in the mapping.
+    /// If more than one DeviceMaskingAttrInterface is specified, returns
+    /// failure. If no mapping is present, returns nullptr.
+    FailureOr<DeviceMaskingAttrInterface> getDeviceMaskingAttr();
+
+    /// Returns true if the mapping specified for this forall op is linear.
+    bool usesLinearMapping();
   }];
 }
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td
index 309079e549846..46a705eefc262 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td
@@ -24,8 +24,25 @@ class SPIRV_ArithmeticBinaryOp<string mnemonic, Type type,
       SPIRV_BinaryOp<mnemonic, type, type,
                    !listconcat(traits,
                                [Pure, AllTypesMatch<["operand1", "operand2", "result"]>])> {
-  // In addition to normal types arithmetic instructions can support cooperative
-  // matrix.
+  let arguments = (ins
+    SPIRV_ScalarOrVectorOf<type>:$operand1,
+    SPIRV_ScalarOrVectorOf<type>:$operand2
+  );
+
+  let results = (outs
+    SPIRV_ScalarOrVectorOf<type>:$result
+  );
+  let assemblyFormat = "operands attr-dict `:` type($result)";
+}
+
+class SPIRV_ArithmeticBinaryOpWithCoopMatrix<string mnemonic, Type type,
+                                             list<Trait> traits = []> :
+      // Operands type same as result type.
+      SPIRV_BinaryOp<mnemonic, type, type,
+                   !listconcat(traits,
+                               [Pure, AllTypesMatch<["operand1", "operand2", "result"]>])> {
+  // In addition to normal types these arithmetic instructions can support
+  // cooperative matrix.
   let arguments = (ins
     SPIRV_ScalarOrVectorOrCoopMatrixOf<type>:$operand1,
     SPIRV_ScalarOrVectorOrCoopMatrixOf<type>:$operand2
@@ -82,7 +99,7 @@ class SPIRV_ArithmeticExtendedBinaryOp<string mnemonic,
 
 // -----
 
-def SPIRV_FAddOp : SPIRV_ArithmeticBinaryOp<"FAdd", SPIRV_Float, [Commutative]> {
+def SPIRV_FAddOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"FAdd", SPIRV_Float, [Commutative]> {
   let summary = "Floating-point addition of Operand 1 and Operand 2.";
 
   let description = [{
@@ -104,7 +121,7 @@ def SPIRV_FAddOp : SPIRV_ArithmeticBinaryOp<"FAdd", SPIRV_Float, [Commutative]>
 
 // -----
 
-def SPIRV_FDivOp : SPIRV_ArithmeticBinaryOp<"FDiv", SPIRV_Float, []> {
+def SPIRV_FDivOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"FDiv", SPIRV_Float, []> {
   let summary = "Floating-point division of Operand 1 divided by Operand 2.";
 
   let description = [{
@@ -154,7 +171,7 @@ def SPIRV_FModOp : SPIRV_ArithmeticBinaryOp<"FMod", SPIRV_Float, []> {
 
 // -----
 
-def SPIRV_FMulOp : SPIRV_ArithmeticBinaryOp<"FMul", SPIRV_Float, [Commutative]> {
+def SPIRV_FMulOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"FMul", SPIRV_Float, [Commutative]> {
   let summary = "Floating-point multiplication of Operand 1 and Operand 2.";
 
   let description = [{
@@ -229,7 +246,7 @@ def SPIRV_FRemOp : SPIRV_ArithmeticBinaryOp<"FRem", SPIRV_Float, []> {
 
 // -----
 
-def SPIRV_FSubOp : SPIRV_ArithmeticBinaryOp<"FSub", SPIRV_Float, []> {
+def SPIRV_FSubOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"FSub", SPIRV_Float, []> {
   let summary = "Floating-point subtraction of Operand 2 from Operand 1.";
 
   let description = [{
@@ -251,9 +268,9 @@ def SPIRV_FSubOp : SPIRV_ArithmeticBinaryOp<"FSub", SPIRV_Float, []> {
 
 // -----
 
-def SPIRV_IAddOp : SPIRV_ArithmeticBinaryOp<"IAdd",
-                                        SPIRV_Integer,
-                                        [Commutative, UsableInSpecConstantOp]> {
+def SPIRV_IAddOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"IAdd",
+                                                          SPIRV_Integer,
+                                                          [Commutative, UsableInSpecConstantOp]> {
   let summary = "Integer addition of Operand 1 and Operand 2.";
 
   let description = [{
@@ -322,9 +339,9 @@ def SPIRV_IAddCarryOp : SPIRV_ArithmeticExtendedBinaryOp<"IAddCarry",
 
 // -----
 
-def SPIRV_IMulOp : SPIRV_ArithmeticBinaryOp<"IMul",
-                                        SPIRV_Integer,
-                                        [Commutative, UsableInSpecConstantOp]> {
+def SPIRV_IMulOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"IMul",
+                                                          SPIRV_Integer,
+                                                          [Commutative, UsableInSpecConstantOp]> {
   let summary = "Integer multiplication of Operand 1 and Operand 2.";
 
   let description = [{
@@ -354,9 +371,9 @@ def SPIRV_IMulOp : SPIRV_ArithmeticBinaryOp<"IMul",
 
 // -----
 
-def SPIRV_ISubOp : SPIRV_ArithmeticBinaryOp<"ISub",
-                                        SPIRV_Integer,
-                                        [UsableInSpecConstantOp]> {
+def SPIRV_ISubOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"ISub",
+                                                          SPIRV_Integer,
+                                                          [UsableInSpecConstantOp]> {
   let summary = "Integer subtraction of Operand 2 from Operand 1.";
 
   let description = [{
@@ -460,9 +477,9 @@ def SPIRV_DotOp : SPIRV_Op<"Dot",
 
 // -----
 
-def SPIRV_SDivOp : SPIRV_ArithmeticBinaryOp<"SDiv",
-                                        SPIRV_Integer,
-                                        [UsableInSpecConstantOp]> {
+def SPIRV_SDivOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"SDiv",
+                                                          SPIRV_Integer,
+                                                          [UsableInSpecConstantOp]> {
   let summary = "Signed-integer division of Operand 1 divided by Operand 2.";
 
   let description = [{
@@ -622,9 +639,9 @@ def SPIRV_SRemOp : SPIRV_ArithmeticBinaryOp<"SRem",
 
 // -----
 
-def SPIRV_UDivOp : SPIRV_ArithmeticBinaryOp<"UDiv",
-                                        SPIRV_Integer,
-                                        [UnsignedOp, UsableInSpecConstantOp]> {
+def SPIRV_UDivOp : SPIRV_ArithmeticBinaryOpWithCoopMatrix<"UDiv",
+                                                          SPIRV_Integer,
+                                                          [UnsignedOp, UsableInSpecConstantOp]> {
   let summary = "Unsigned-integer division of Operand 1 divided by Operand 2.";
 
   let description = [{
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
index d2ba76cdad904..d874817e6888d 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
@@ -422,6 +422,8 @@ def SPV_NV_ray_tracing_motion_blur       : I32EnumAttrCase<"SPV_NV_ray_tracing_m
 
 def SPV_NVX_multiview_per_view_attributes : I32EnumAttrCase<"SPV_NVX_multiview_per_view_attributes", 5015>;
 
+def SPV_ARM_tensors                      : I32EnumAttrCase<"SPV_ARM_tensors", 6000>;
+
 def SPIRV_ExtensionAttr :
     SPIRV_I32EnumAttr<"Extension", "supported SPIR-V extensions", "ext", [
       SPV_KHR_16bit_storage, SPV_KHR_8bit_storage, SPV_KHR_device_group,
@@ -445,6 +447,7 @@ def SPIRV_ExtensionAttr :
       SPV_EXT_shader_atomic_float_add, SPV_EXT_shader_atomic_float_min_max,
       SPV_EXT_shader_image_int64, SPV_EXT_shader_atomic_float16_add,
       SPV_EXT_mesh_shader,
+      SPV_ARM_tensors,
       SPV_AMD_gpu_shader_half_float_fetch, SPV_AMD_shader_ballot,
       SPV_AMD_shader_explicit_vertex_parameter, SPV_AMD_shader_fragment_mask,
       SPV_AMD_shader_image_load_store_lod, SPV_AMD_texture_gather_bias_lod,
@@ -1311,6 +1314,24 @@ def SPIRV_C_GeometryStreams                             : I32EnumAttrCase<"Geome
 def SPIRV_C_MultiViewport                               : I32EnumAttrCase<"MultiViewport", 57> {
   list<I32EnumAttrCase> implies = [SPIRV_C_Geometry];
 }
+def SPIRV_C_TensorsARM                                  : I32EnumAttrCase<"TensorsARM", 4174> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_Int8];
+  list<Availability> availability = [
+    Extension<[SPV_ARM_tensors]>
+  ];
+}
+def SPIRV_C_StorageTensorArrayDynamicIndexingEXT        : I32EnumAttrCase<"StorageTensorArrayDynamicIndexingEXT", 4175> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_TensorsARM, SPIRV_C_Shader];
+  list<Availability> availability = [
+    Extension<[SPV_ARM_tensors]>
+  ];
+}
+def SPIRV_C_StorageTensorArrayNonUniformIndexingEXT     : I32EnumAttrCase<"StorageTensorArrayNonUniformIndexingEXT", 4176> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_TensorsARM, SPIRV_C_ShaderNonUniform];
+  list<Availability> availability = [
+    Extension<[SPV_ARM_tensors]>
+  ];
+}
 def SPIRV_C_WorkgroupMemoryExplicitLayout8BitAccessKHR  : I32EnumAttrCase<"WorkgroupMemoryExplicitLayout8BitAccessKHR", 4429> {
   list<I32EnumAttrCase> implies = [SPIRV_C_WorkgroupMemoryExplicitLayoutKHR];
   list<Availability> availability = [
@@ -1523,6 +1544,8 @@ def SPIRV_CapabilityAttr :
       SPIRV_C_IntegerFunctions2INTEL, SPIRV_C_TessellationPointSize,
       SPIRV_C_GeometryPointSize, SPIRV_C_ImageCubeArray, SPIRV_C_ImageRect,
       SPIRV_C_GeometryStreams, SPIRV_C_MultiViewport,
+      SPIRV_C_TensorsARM, SPIRV_C_StorageTensorArrayDynamicIndexingEXT,
+      SPIRV_C_StorageTensorArrayNonUniformIndexingEXT,
       SPIRV_C_WorkgroupMemoryExplicitLayout8BitAccessKHR, SPIRV_C_VariablePointers,
       SPIRV_C_RayTraversalPrimitiveCullingKHR, SPIRV_C_SampleMaskOverrideCoverageNV,
       SPIRV_C_GeometryShaderPassthroughNV, SPIRV_C_PerViewAttributesNV,
@@ -4179,7 +4202,7 @@ def SPIRV_IsPtrType : CPred<"::llvm::isa<::mlir::spirv::PointerType>($_self)">;
 def SPIRV_IsRTArrayType : CPred<"::llvm::isa<::mlir::spirv::RuntimeArrayType>($_self)">;
 def SPIRV_IsSampledImageType : CPred<"::llvm::isa<::mlir::spirv::SampledImageType>($_self)">;
 def SPIRV_IsStructType : CPred<"::llvm::isa<::mlir::spirv::StructType>($_self)">;
-
+def SPIRV_IsTensorArmType : CPred<"::llvm::isa<::mlir::spirv::TensorArmType>($_self)">;
 
 // See https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_types
 // for the definition of the following types and type categories.
@@ -4217,6 +4240,8 @@ def SPIRV_AnyStruct : DialectType<SPIRV_Dialect, SPIRV_IsStructType,
                                 "any SPIR-V struct type">;
 def SPIRV_AnySampledImage : DialectType<SPIRV_Dialect, SPIRV_IsSampledImageType,
                                 "any SPIR-V sampled image type">;
+def SPIRV_AnyTensorArm : DialectType<SPIRV_Dialect, SPIRV_IsTensorArmType,
+                                 "any SPIR-V tensorArm type">;
 
 def SPIRV_Numerical : AnyTypeOf<[SPIRV_Integer, SPIRV_AnyFloat]>;
 def SPIRV_Scalar : AnyTypeOf<[SPIRV_Numerical, SPIRV_Bool]>;
@@ -4228,7 +4253,7 @@ def SPIRV_Type : AnyTypeOf<[
     SPIRV_Void, SPIRV_Bool, SPIRV_Integer, SPIRV_AnyFloat, SPIRV_Vector,
     SPIRV_AnyPtr, SPIRV_AnyArray, SPIRV_AnyRTArray, SPIRV_AnyStruct,
     SPIRV_AnyCooperativeMatrix, SPIRV_AnyMatrix, SPIRV_AnySampledImage,
-    SPIRV_AnyImage
+    SPIRV_AnyImage, SPIRV_AnyTensorArm
   ]>;
 
 def SPIRV_SignedInt : SignedIntOfWidths<[8, 16, 32, 64]>;
@@ -4525,6 +4550,7 @@ def SPIRV_OC_OpGroupNonUniformBitwiseXor      : I32EnumAttrCase<"OpGroupNonUnifo
 def SPIRV_OC_OpGroupNonUniformLogicalAnd      : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
 def SPIRV_OC_OpGroupNonUniformLogicalOr       : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
 def SPIRV_OC_OpGroupNonUniformLogicalXor      : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
+def SPIRV_OC_OpTypeTensorARM                  : I32EnumAttrCase<"OpTypeTensorARM", 4163>;
 def SPIRV_OC_OpSubgroupBallotKHR              : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
 def SPIRV_OC_OpGroupNonUniformRotateKHR       : I32EnumAttrCase<"OpGroupNonUniformRotateKHR", 4431>;
 def SPIRV_OC_OpSDot                           : I32EnumAttrCase<"OpSDot", 4450>;
@@ -4638,7 +4664,9 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpGroupNonUniformFMax, SPIRV_OC_OpGroupNonUniformBitwiseAnd,
       SPIRV_OC_OpGroupNonUniformBitwiseOr, SPIRV_OC_OpGroupNonUniformBitwiseXor,
       SPIRV_OC_OpGroupNonUniformLogicalAnd, SPIRV_OC_OpGroupNonUniformLogicalOr,
-      SPIRV_OC_OpGroupNonUniformLogicalXor, SPIRV_OC_OpSubgroupBallotKHR,
+      SPIRV_OC_OpGroupNonUniformLogicalXor,
+      SPIRV_OC_OpTypeTensorARM,
+      SPIRV_OC_OpSubgroupBallotKHR,
       SPIRV_OC_OpGroupNonUniformRotateKHR, SPIRV_OC_OpSDot, SPIRV_OC_OpUDot,
       SPIRV_OC_OpSUDot, SPIRV_OC_OpSDotAccSat, SPIRV_OC_OpUDotAccSat,
       SPIRV_OC_OpSUDotAccSat, SPIRV_OC_OpTypeCooperativeMatrixKHR,
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td
index 46732ba19afed..fd75532ae3d70 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td
@@ -112,7 +112,7 @@ def SPIRV_KHRCooperativeMatrixLoadOp : SPIRV_KhrVendorOp<"CooperativeMatrixLoad"
   }];
 
   let assemblyFormat = [{
-    $pointer `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? attr-dict `:`
+    $pointer `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? ( `,` $alignment^ )? attr-dict `:`
       type(operands) `->` type($result)
   }];
 
@@ -123,11 +123,13 @@ def SPIRV_KHRCooperativeMatrixLoadOp : SPIRV_KhrVendorOp<"CooperativeMatrixLoad"
     Capability<[SPIRV_C_CooperativeMatrixKHR]>
   ];
 
+  // TODO: Add scope operand for MakePointer*. See #145485.
   let arguments = (ins
     SPIRV_AnyPtr:$pointer,
     SPIRV_KHR_CooperativeMatrixLayoutAttr:$matrix_layout,
     SPIRV_Integer:$stride,
-    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand
+    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand,
+    OptionalAttr<I32Attr>:$alignment
   );
 
   let results = (outs
@@ -139,7 +141,7 @@ def SPIRV_KHRCooperativeMatrixLoadOp : SPIRV_KhrVendorOp<"CooperativeMatrixLoad"
                    "spirv::ConstantOp":$stride,
                    "spirv::CooperativeMatrixLayoutKHR":$layout), [{
       build($_builder, $_state, result, pointer, layout, stride,
-            spirv::MemoryAccessAttr{});
+            spirv::MemoryAccessAttr{}, IntegerAttr{});
     }]>
   ];
 }
@@ -194,7 +196,7 @@ def SPIRV_KHRCooperativeMatrixStoreOp : SPIRV_KhrVendorOp<"CooperativeMatrixStor
   }];
 
   let assemblyFormat = [{
-    $pointer `,` $object `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? attr-dict `:`
+    $pointer `,` $object `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? ( `,` $alignment^ )? attr-dict `:`
       type(operands)
   }];
 
@@ -205,12 +207,14 @@ def SPIRV_KHRCooperativeMatrixStoreOp : SPIRV_KhrVendorOp<"CooperativeMatrixStor
     Capability<[SPIRV_C_CooperativeMatrixKHR]>
   ];
 
+  // TODO: Add scope operand for MakePointer*. See #145485.
   let arguments = (ins
     SPIRV_AnyPtr:$pointer,
     SPIRV_AnyCooperativeMatrix:$object,
     SPIRV_KHR_CooperativeMatrixLayoutAttr:$matrix_layout,
     SPIRV_Integer:$stride,
-    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand
+    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand,
+    OptionalAttr<I32Attr>:$alignment
   );
 
   let results = (outs);
@@ -220,7 +224,7 @@ def SPIRV_KHRCooperativeMatrixStoreOp : SPIRV_KhrVendorOp<"CooperativeMatrixStor
                    "spirv::ConstantOp":$stride,
                    "spirv::CooperativeMatrixLayoutKHR":$layout), [{
       build($_builder, $_state, pointer, object, layout, stride,
-            spirv::MemoryAccessAttr{});
+            spirv::MemoryAccessAttr{}, IntegerAttr{});
     }]>
   ];
 }
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
index 787535d0a6bd2..212cba61d396c 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
@@ -29,6 +29,7 @@ namespace spirv {
 namespace detail {
 struct ArrayTypeStorage;
 struct CooperativeMatrixTypeStorage;
+struct TensorArmTypeStorage;
 struct ImageTypeStorage;
 struct MatrixTypeStorage;
 struct PointerTypeStorage;
@@ -96,7 +97,8 @@ class ScalarType : public SPIRVType {
   std::optional<int64_t> getSizeInBytes();
 };
 
-// SPIR-V composite type: VectorType, SPIR-V ArrayType, or SPIR-V StructType.
+// SPIR-V composite type: VectorType, SPIR-V ArrayType, SPIR-V
+// StructType, or SPIR-V TensorArmType.
 class CompositeType : public SPIRVType {
 public:
   using SPIRVType::SPIRVType;
@@ -477,6 +479,46 @@ class MatrixType : public Type::TypeBase<MatrixType, CompositeType,
                        std::optional<StorageClass> storage = std::nullopt);
 };
 
+/// SPIR-V TensorARM Type
+class TensorArmType
+    : public Type::TypeBase<TensorArmType, CompositeType,
+                            detail::TensorArmTypeStorage, ShapedType::Trait> {
+public:
+  using Base::Base;
+
+  using ShapedTypeTraits = ShapedType::Trait<TensorArmType>;
+  using ShapedTypeTraits::getDimSize;
+  using ShapedTypeTraits::getDynamicDimIndex;
+  using ShapedTypeTraits::getElementTypeBitWidth;
+  using ShapedTypeTraits::getNumDynamicDims;
+  using ShapedTypeTraits::getNumElements;
+  using ShapedTypeTraits::getRank;
+  using ShapedTypeTraits::hasStaticShape;
+  using ShapedTypeTraits::isDynamicDim;
+
+  static constexpr StringLiteral name = "spirv.arm.tensor";
+
+  // TensorArm supports minimum rank of 1, hence an empty shape here means
+  // unranked.
+  static TensorArmType get(ArrayRef<int64_t> shape, Type elementType);
+  TensorArmType cloneWith(std::optional<ArrayRef<int64_t>> shape,
+                          Type elementType) const;
+
+  static LogicalResult
+  verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
+                   ArrayRef<int64_t> shape, Type elementType);
+
+  Type getElementType() const;
+  ArrayRef<int64_t> getShape() const;
+  bool hasRank() const { return !getShape().empty(); }
+  operator ShapedType() const { return llvm::cast<ShapedType>(*this); }
+
+  void getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
+                     std::optional<StorageClass> storage = std::nullopt);
+  void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
+                       std::optional<StorageClass> storage = std::nullopt);
+};
+
 } // namespace spirv
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index 47962f75558ea..7d396e5c64c28 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -1771,8 +1771,7 @@ def Tensor_SplatOp : Tensor_Op<"splat", [
   ]> {
   let summary = "tensor splat or broadcast operation";
   let description = [{
-    Broadcast the operand to all elements of the result tensor. The operand is
-    required to be of integer/index/float type.
+    Broadcast the operand to all elements of the result tensor.
 
     An additional argument of type `index` must be provided for each dynamic
     dimension present in the result type.
@@ -1795,8 +1794,7 @@ def Tensor_SplatOp : Tensor_Op<"splat", [
     ```
   }];
 
-  let arguments = (ins AnyTypeOf<[AnySignlessInteger, Index, AnyFloat],
-                                 "integer/index/float type">:$input,
+  let arguments = (ins AnyType:$input,
                        Variadic<Index>:$dynamicSizes);
   let results = (outs AnyRankedTensor:$aggregate);
 
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h b/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h
index cdbebd581d9f5..9ee5079559d2b 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h
@@ -43,32 +43,22 @@ class TargetEnv {
   bool allows(Profile prof) const { return enabledProfiles.count(prof) != 0; }
 
   bool allowsAnyOf(ArrayRef<Profile> profs) const {
-    const auto *chosen = llvm::find_if(
-        profs, [this](tosa::Profile prof) { return allows(prof); });
-    return chosen != profs.end() ? true : false;
+    return llvm::any_of(profs, [&](Profile prof) { return allows(prof); });
   }
 
   bool allowsAllOf(ArrayRef<Profile> profs) const {
-    bool is_allowed = true;
-    llvm::for_each(profs,
-                   [&](tosa::Profile prof) { is_allowed &= allows(prof); });
-    return is_allowed;
+    return llvm::all_of(profs, [&](Profile prof) { return allows(prof); });
   }
 
   // Returns true if the given extension is allowed.
   bool allows(Extension ext) const { return enabledExtensions.count(ext) != 0; }
 
   bool allowsAnyOf(ArrayRef<Extension> exts) const {
-    const auto *chosen = llvm::find_if(
-        exts, [this](tosa::Extension ext) { return allows(ext); });
-    return chosen != exts.end() ? true : false;
+    return llvm::any_of(exts, [&](Extension ext) { return allows(ext); });
   }
 
   bool allowsAllOf(ArrayRef<Extension> exts) const {
-    bool is_allowed = true;
-    llvm::for_each(exts,
-                   [&](tosa::Extension ext) { is_allowed &= allows(ext); });
-    return is_allowed;
+    return llvm::all_of(exts, [&](Extension ext) { return allows(ext); });
   }
 
 private:
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index f93dd901535c3..20889558be314 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1490,9 +1490,9 @@ def Tosa_SelectOp : Tosa_ElementwiseOp<"select"> {
   }];
 
   let arguments = (ins
-    Tosa_I1Tensor:$input1,
-    Tosa_Tensor:$input2,
-    Tosa_Tensor:$input3
+    Tosa_I1Tensor:$input1,  // pred
+    Tosa_Tensor:$input2,    // on true
+    Tosa_Tensor:$input3     // on false
   );
 
   let results = (outs
@@ -1512,6 +1512,13 @@ def Tosa_SelectOp : Tosa_ElementwiseOp<"select"> {
     operands attr-dict `:` `(` type($input1) `,` type($input2) `,` type($input3)
     `)` `->` type($output)
   }];
+
+  let extraClassDeclaration = [{
+    // Custom getters for readability
+    ::mlir::TypedValue<::mlir::TensorType> getPred() { return getInput1(); }
+    ::mlir::TypedValue<::mlir::TensorType> getOnTrue() { return getInput2(); }
+    ::mlir::TypedValue<::mlir::TensorType> getOnFalse() { return getInput3(); }
+  }];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
index d005a4cc6859c..b96682843538c 100644
--- a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
@@ -127,4 +127,18 @@ def TosaReduceTransposes : Pass<"tosa-reduce-transposes", "func::FuncOp"> {
   }];
 }
 
+def TosaConvertIntegerTypeToSignless : Pass<"tosa-convert-integer-type-to-signless", "func::FuncOp"> {
+  let summary = "Convert integer types to signless";
+  let description = [{
+    This pass converts signed or unsigned integer types to signless. It
+    currently does this greedily for all operators and can also change the
+    signature of the function. Should the signature of the entrypoint
+    function change, it will be the responsibility of the user to carry
+    signedness information of the inputs and outputs independently.
+
+    This can be a useful transformation for conversion to other formats
+    that require strict adherence to the TOSA specification.
+  }];
+}
+
 #endif // MLIR_DIALECT_TOSA_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/Transform/CMakeLists.txt b/mlir/include/mlir/Dialect/Transform/CMakeLists.txt
index b6155b5f573f1..e70479b2a39f2 100644
--- a/mlir/include/mlir/Dialect/Transform/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Transform/CMakeLists.txt
@@ -5,3 +5,4 @@ add_subdirectory(IRDLExtension)
 add_subdirectory(LoopExtension)
 add_subdirectory(PDLExtension)
 add_subdirectory(Transforms)
+add_subdirectory(TuneExtension)
diff --git a/mlir/include/mlir/Dialect/Transform/TuneExtension/CMakeLists.txt b/mlir/include/mlir/Dialect/Transform/TuneExtension/CMakeLists.txt
new file mode 100644
index 0000000000000..9afca813afda6
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Transform/TuneExtension/CMakeLists.txt
@@ -0,0 +1,6 @@
+set(LLVM_TARGET_DEFINITIONS TuneExtensionOps.td)
+mlir_tablegen(TuneExtensionOps.h.inc -gen-op-decls)
+mlir_tablegen(TuneExtensionOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRTransformDialectTuneExtensionOpsIncGen)
+
+add_mlir_doc(TuneExtensionOps TuneExtensionOps Dialects/ -gen-op-doc)
diff --git a/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtension.h b/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtension.h
new file mode 100644
index 0000000000000..1453d1754297f
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtension.h
@@ -0,0 +1,21 @@
+//===- TuneExtension.h - Tune extension for Transform dialect ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSION_H
+#define MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSION_H
+
+namespace mlir {
+class DialectRegistry;
+
+namespace transform {
+/// Registers the tune extension of the Transform dialect in the given registry.
+void registerTuneExtension(DialectRegistry &dialectRegistry);
+} // namespace transform
+} // namespace mlir
+
+#endif // MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSION_H
diff --git a/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.h b/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.h
new file mode 100644
index 0000000000000..74e1d28ffac82
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.h
@@ -0,0 +1,19 @@
+//===- TuneExtensionOps.h - Tune ext. for Transform dialect -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSIONOPS_H
+#define MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSIONOPS_H
+
+#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/OpDefinition.h"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.h.inc"
+
+#endif // MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSIONOPS_H
diff --git a/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.td b/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.td
new file mode 100644
index 0000000000000..d68d451afac40
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.td
@@ -0,0 +1,55 @@
+//===- TuneExtensionOps.td - Transform dialect operations --*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSIONOPS
+#define MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSIONOPS
+
+include "mlir/Dialect/Transform/IR/TransformDialect.td"
+include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/IR/BuiltinAttributes.td"
+include "mlir/IR/CommonAttrConstraints.td"
+
+def KnobOp : Op<Transform_Dialect, "tune.knob", [
+  DeclareOpInterfaceMethods<TransformOpInterface>,
+  DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+]> {
+  let summary = "Represents a tunable parameter with a set of options";
+
+  let description = [{
+    Provides a representation for "tunables" within schedules.
+
+    Each op represents a single tunable, which has a `name` and a set
+    of valid `options` described by an attribute. Without a specified
+    `selected` option, this op represents a non-deterministic choice
+    that has yet to be resolved -- as such, the interpreter runtime
+    semantics is to raise a failure.
+
+    The non-deterministic choice is resolved through providing a
+    `selected` attribute. When provided, the interpreter runtime
+    semantics are to return the `selected` attribute as a param through
+    the op's result.
+
+    -----
+
+    In case the `options` attribute is an `ArrayAttr`, the verifier
+    checks that the provided `selected` attribute occurs in `options`.
+  }];
+  let cppNamespace = [{ mlir::transform::tune }];
+  let hasVerifier = 1;
+
+  let arguments = (ins Builtin_StringAttr:$name,
+                       AnyAttr:$options,
+                       OptionalAttr<AnyAttr>:$selected);
+  let results = (outs TransformParamTypeInterface:$result);
+
+  let assemblyFormat =
+      "`<` $name `>` (`=` $selected^ `from`)? `options` `=` $options attr-dict `->` type(results)";
+}
+
+#endif // MLIR_DIALECT_TRANSFORM_TUNEEXTENSION_TUNEEXTENSIONOPS
diff --git a/mlir/include/mlir/Dialect/Vector/IR/Vector.td b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
index 1922cc63ef353..5125ae7c13717 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/Vector.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
@@ -21,7 +21,10 @@ def Vector_Dialect : Dialect {
 
   let useDefaultAttributePrinterParser = 1;
   let hasConstantMaterializer = 1;
-  let dependentDialects = ["arith::ArithDialect"];
+  let dependentDialects = [
+    "arith::ArithDialect",
+    "ub::UBDialect"
+  ];
 }
 
 // Base class for Vector dialect ops.
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index e6b85de5a522a..ec2c87ca1cf44 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1543,30 +1543,29 @@ def Vector_TransferReadOp :
   }];
 
   let builders = [
-    /// 1. Builder that sets padding to zero and an empty mask (variant with attrs).
+    /// 1. Builder that sets padding to `padding` or poison if not provided and
+    /// an empty mask (variant with attrs).
     OpBuilder<(ins "VectorType":$vectorType,
                    "Value":$source,
                    "ValueRange":$indices,
+                   "std::optional<Value>":$padding,
                    "AffineMapAttr":$permutationMapAttr,
                    "ArrayAttr":$inBoundsAttr)>,
-    /// 2. Builder that sets padding to zero and an empty mask (variant without attrs).
+    /// 2. Builder that sets padding to `padding` or poison if not provided and
+    /// an empty mask (variant without attrs).
     OpBuilder<(ins "VectorType":$vectorType,
                    "Value":$source,
                    "ValueRange":$indices,
+                   "std::optional<Value>":$padding,
                    "AffineMap":$permutationMap,
                    CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>,
-    /// 3. Builder that sets permutation map to 'getMinorIdentityMap'.
+    /// 3. Builder that sets padding to `padding` or poison if not provided and
+    /// permutation map to 'getMinorIdentityMap'.
     OpBuilder<(ins "VectorType":$vectorType,
                    "Value":$source,
                    "ValueRange":$indices,
-                   "Value":$padding,
-                   CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>,
-    /// 4. Builder that sets padding to zero and permutation map to
-    /// 'getMinorIdentityMap'.
-    OpBuilder<(ins "VectorType":$vectorType,
-                   "Value":$source,
-                   "ValueRange":$indices,
-                   CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>,
+                   "std::optional<Value>":$padding,
+                   CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>
   ];
 
   let extraClassDeclaration = [{
@@ -2493,6 +2492,7 @@ def Vector_ConstantMaskOp :
 
   let assemblyFormat = "$mask_dim_sizes attr-dict `:` type(results)";
   let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 def Vector_CreateMaskOp :
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index 34a94e6ea7051..3dc7d38440ca5 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -137,12 +137,14 @@ void populateVectorReductionToContractPatterns(RewritePatternSet &patterns,
 void populateVectorTransferFullPartialPatterns(
     RewritePatternSet &patterns, const VectorTransformsOptions &options);
 
-/// Collect a set of patterns to reduce the rank of the operands of vector
-/// transfer ops to operate on the largest contigious vector.
-/// These patterns are useful when lowering to dialects with 1d vector type
-/// such as llvm and it will result fewer memory reads.
-void populateVectorTransferCollapseInnerMostContiguousDimsPatterns(
-    RewritePatternSet &patterns, PatternBenefit benefit = 1);
+/// Collect a set of patterns to collapse the most inner unit dims in xfer Ops
+///
+/// These patters reduce the rank of the operands of vector transfer ops to
+/// operate on vectors without trailing unit dims. This helps reduce the rank of
+/// the operands, which can be helpful when lowering to dialects that only
+/// support 1D vector type such as LLVM.
+void populateDropInnerMostUnitDimsXferOpPatterns(RewritePatternSet &patterns,
+                                                 PatternBenefit benefit = 1);
 
 /// Patterns that remove redundant Vector Ops by re-ordering them with
 /// e.g. elementwise Ops:
diff --git a/mlir/include/mlir/ExecutionEngine/MemRefUtils.h b/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
index f355dfb8648ec..6e72f7c23bdcf 100644
--- a/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
+++ b/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
@@ -68,8 +68,8 @@ makeStridedMemRefDescriptor(T *ptr, T *alignedPtr, ArrayRef<int64_t> shape,
   assert(shape.size() == N);
   assert(shapeAlloc.size() == N);
   StridedMemRefType<T, N> descriptor;
-  descriptor.basePtr = static_cast<T *>(ptr);
-  descriptor.data = static_cast<T *>(alignedPtr);
+  descriptor.basePtr = ptr;
+  descriptor.data = alignedPtr;
   descriptor.offset = 0;
   std::copy(shape.begin(), shape.end(), descriptor.sizes);
   auto strides = makeStrides<N>(shapeAlloc);
@@ -89,8 +89,8 @@ makeStridedMemRefDescriptor(T *ptr, T *alignedPtr, ArrayRef<int64_t> shape = {},
   assert(shape.size() == N);
   assert(shapeAlloc.size() == N);
   StridedMemRefType<T, 0> descriptor;
-  descriptor.basePtr = static_cast<T *>(ptr);
-  descriptor.data = static_cast<T *>(alignedPtr);
+  descriptor.basePtr = ptr;
+  descriptor.data = alignedPtr;
   descriptor.offset = 0;
   return descriptor;
 }
diff --git a/mlir/include/mlir/IR/BuiltinTypeInterfaces.td b/mlir/include/mlir/IR/BuiltinTypeInterfaces.td
index 367aeb6ac512b..c879e5efd77fe 100644
--- a/mlir/include/mlir/IR/BuiltinTypeInterfaces.td
+++ b/mlir/include/mlir/IR/BuiltinTypeInterfaces.td
@@ -36,7 +36,7 @@ def VectorElementTypeInterface : TypeInterface<"VectorElementTypeInterface"> {
     This may change in the future, for example, to require types to provide
     their size or alignment given a data layout. Please post an RFC before
     adding this interface to additional types. Implementing this interface on
-    downstream types is discourged, until we specified the exact properties of
+    downstream types is discouraged, until we specified the exact properties of
     a vector element type in more detail.
   }];
 }
@@ -221,7 +221,17 @@ def ShapedTypeInterface : TypeInterface<"ShapedType"> {
 
     /// Whether the given shape has any size that indicates a dynamic dimension.
     static bool isDynamicShape(ArrayRef<int64_t> dSizes) {
-      return any_of(dSizes, [](int64_t dSize) { return isDynamic(dSize); });
+      return llvm::any_of(dSizes, isDynamic);
+    }
+
+    /// Whether the given dimension size indicates a statically-sized dimension.
+    static constexpr bool isStatic(int64_t dValue) {
+      return dValue != kDynamic;
+    }
+
+    /// Whether the given shape has static dimensions only.
+    static bool isStaticShape(ArrayRef<int64_t> dSizes) {
+      return llvm::all_of(dSizes, isStatic);
     }
 
     /// Return the number of elements present in the given shape.
@@ -273,11 +283,18 @@ def ShapedTypeInterface : TypeInterface<"ShapedType"> {
       return ::mlir::ShapedType::isDynamic($_type.getShape()[idx]);
     }
 
+    /// Returns true if this dimension has a static size (for ranked types);
+    /// aborts for unranked types.
+    bool isStaticDim(unsigned idx) const {
+      assert(idx < getRank() && "invalid index for shaped type");
+      return ::mlir::ShapedType::isStatic($_type.getShape()[idx]);
+    }
+
     /// Returns if this type has a static shape, i.e. if the type is ranked and
     /// all dimensions have known size (>= 0).
     bool hasStaticShape() const {
       return $_type.hasRank() &&
-             !::mlir::ShapedType::isDynamicShape($_type.getShape());
+             ::mlir::ShapedType::isStaticShape($_type.getShape());
     }
 
     /// Returns if this type has a static shape and the shape is equal to
diff --git a/mlir/include/mlir/IR/ODSSupport.h b/mlir/include/mlir/IR/ODSSupport.h
index b24a2470470ff..036ce4e5b725f 100644
--- a/mlir/include/mlir/IR/ODSSupport.h
+++ b/mlir/include/mlir/IR/ODSSupport.h
@@ -118,6 +118,9 @@ convertFromAttribute(SmallVectorImpl<int32_t> &storage, Attribute attr,
 /// Convert the provided ArrayRef<int64_t> to a DenseI64ArrayAttr attribute.
 Attribute convertToAttribute(MLIRContext *ctx, ArrayRef<int64_t> storage);
 
+/// Convert the provided ArrayRef<int32_t> to a DenseI32ArrayAttr attribute.
+Attribute convertToAttribute(MLIRContext *ctx, ArrayRef<int32_t> storage);
+
 } // namespace mlir
 
 #endif // MLIR_IR_ODSSUPPORT_H
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 8710b970e8d75..d70aa346eaa1f 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -207,6 +207,9 @@ class AsmPrinter {
   /// provide a valid type for the attribute.
   virtual void printAttributeWithoutType(Attribute attr);
 
+  /// Print the given named attribute.
+  virtual void printNamedAttribute(NamedAttribute attr);
+
   /// Print the alias for the given attribute, return failure if no alias could
   /// be printed.
   virtual LogicalResult printAlias(Attribute attr);
diff --git a/mlir/include/mlir/IR/Unit.h b/mlir/include/mlir/IR/Unit.h
index 63117a7664a7d..0e99f8e25f326 100644
--- a/mlir/include/mlir/IR/Unit.h
+++ b/mlir/include/mlir/IR/Unit.h
@@ -22,7 +22,7 @@ class Region;
 class Block;
 class Value;
 
-/// IRUnit is a union of the different types of IR objects that consistute the
+/// IRUnit is a union of the different types of IR objects that constitute the
 /// IR structure (other than Type and Attribute), that is Operation, Region, and
 /// Block.
 class IRUnit : public PointerUnion<Operation *, Region *, Block *, Value> {
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index 261b0e00bdf86..c6fcf1a0d510b 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -46,6 +46,7 @@
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.h"
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/AllInterfaces.h"
 #include "mlir/Dialect/Linalg/Transforms/RuntimeOpVerification.h"
@@ -152,7 +153,8 @@ inline void registerAllDialects(DialectRegistry &registry) {
                   ub::UBDialect,
                   vector::VectorDialect,
                   x86vector::X86VectorDialect,
-                  xegpu::XeGPUDialect>();
+                  xegpu::XeGPUDialect,
+                  xevm::XeVMDialect>();
   // clang-format on
 
   // Register all external models.
diff --git a/mlir/include/mlir/InitAllExtensions.h b/mlir/include/mlir/InitAllExtensions.h
index f356b91b1b6c0..0f2d0e45008cc 100644
--- a/mlir/include/mlir/InitAllExtensions.h
+++ b/mlir/include/mlir/InitAllExtensions.h
@@ -52,6 +52,7 @@
 #include "mlir/Dialect/Transform/IRDLExtension/IRDLExtension.h"
 #include "mlir/Dialect/Transform/LoopExtension/LoopExtension.h"
 #include "mlir/Dialect/Transform/PDLExtension/PDLExtension.h"
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtension.h"
 #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.h"
 #include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h"
@@ -107,6 +108,7 @@ inline void registerAllExtensions(DialectRegistry &registry) {
   transform::registerIRDLExtension(registry);
   transform::registerLoopExtension(registry);
   transform::registerPDLExtension(registry);
+  transform::registerTuneExtension(registry);
   vector::registerTransformDialectExtension(registry);
   arm_neon::registerTransformDialectExtension(registry);
   arm_sve::registerTransformDialectExtension(registry);
diff --git a/mlir/include/mlir/Query/Matcher/Marshallers.h b/mlir/include/mlir/Query/Matcher/Marshallers.h
index 012bf7b9ec4a9..5fe6965f32efb 100644
--- a/mlir/include/mlir/Query/Matcher/Marshallers.h
+++ b/mlir/include/mlir/Query/Matcher/Marshallers.h
@@ -108,6 +108,9 @@ class MatcherDescriptor {
                                 const llvm::ArrayRef<ParserValue> args,
                                 Diagnostics *error) const = 0;
 
+  // If the matcher is variadic, it can take any number of arguments.
+  virtual bool isVariadic() const = 0;
+
   // Returns the number of arguments accepted by the matcher.
   virtual unsigned getNumArgs() const = 0;
 
@@ -140,6 +143,8 @@ class FixedArgCountMatcherDescriptor : public MatcherDescriptor {
     return marshaller(matcherFunc, matcherName, nameRange, args, error);
   }
 
+  bool isVariadic() const override { return false; }
+
   unsigned getNumArgs() const override { return argKinds.size(); }
 
   void getArgKinds(unsigned argNo, std::vector<ArgKind> &kinds) const override {
@@ -153,6 +158,54 @@ class FixedArgCountMatcherDescriptor : public MatcherDescriptor {
   const std::vector<ArgKind> argKinds;
 };
 
+class VariadicOperatorMatcherDescriptor : public MatcherDescriptor {
+public:
+  using VarOp = DynMatcher::VariadicOperator;
+  VariadicOperatorMatcherDescriptor(unsigned minCount, unsigned maxCount,
+                                    VarOp varOp, StringRef matcherName)
+      : minCount(minCount), maxCount(maxCount), varOp(varOp),
+        matcherName(matcherName) {}
+
+  VariantMatcher create(SourceRange nameRange, ArrayRef<ParserValue> args,
+                        Diagnostics *error) const override {
+    if (args.size() < minCount || maxCount < args.size()) {
+      addError(error, nameRange, ErrorType::RegistryWrongArgCount,
+               {llvm::Twine("requires between "), llvm::Twine(minCount),
+                llvm::Twine(" and "), llvm::Twine(maxCount),
+                llvm::Twine(" args, got "), llvm::Twine(args.size())});
+      return VariantMatcher();
+    }
+
+    std::vector<VariantMatcher> innerArgs;
+    for (int64_t i = 0, e = args.size(); i != e; ++i) {
+      const ParserValue &arg = args[i];
+      const VariantValue &value = arg.value;
+      if (!value.isMatcher()) {
+        addError(error, arg.range, ErrorType::RegistryWrongArgType,
+                 {llvm::Twine(i + 1), llvm::Twine("matcher: "),
+                  llvm::Twine(value.getTypeAsString())});
+        return VariantMatcher();
+      }
+      innerArgs.push_back(value.getMatcher());
+    }
+    return VariantMatcher::VariadicOperatorMatcher(varOp, std::move(innerArgs));
+  }
+
+  bool isVariadic() const override { return true; }
+
+  unsigned getNumArgs() const override { return 0; }
+
+  void getArgKinds(unsigned argNo, std::vector<ArgKind> &kinds) const override {
+    kinds.push_back(ArgKind(ArgKind::Matcher));
+  }
+
+private:
+  const unsigned minCount;
+  const unsigned maxCount;
+  const VarOp varOp;
+  const StringRef matcherName;
+};
+
 // Helper function to check if argument count matches expected count
 inline bool checkArgCount(SourceRange nameRange, size_t expectedArgCount,
                           llvm::ArrayRef<ParserValue> args,
@@ -224,6 +277,14 @@ makeMatcherAutoMarshall(ReturnType (*matcherFunc)(ArgTypes...),
       reinterpret_cast<void (*)()>(matcherFunc), matcherName, argKinds);
 }
 
+// Variadic operator overload.
+template <unsigned MinCount, unsigned MaxCount>
+std::unique_ptr<MatcherDescriptor>
+makeMatcherAutoMarshall(VariadicOperatorMatcherFunc<MinCount, MaxCount> func,
+                        StringRef matcherName) {
+  return std::make_unique<VariadicOperatorMatcherDescriptor>(
+      MinCount, MaxCount, func.varOp, matcherName);
+}
 } // namespace mlir::query::matcher::internal
 
 #endif // MLIR_TOOLS_MLIRQUERY_MATCHER_MARSHALLERS_H
diff --git a/mlir/include/mlir/Query/Matcher/MatchFinder.h b/mlir/include/mlir/Query/Matcher/MatchFinder.h
index f8abf20ef60bb..6d06ca13d1344 100644
--- a/mlir/include/mlir/Query/Matcher/MatchFinder.h
+++ b/mlir/include/mlir/Query/Matcher/MatchFinder.h
@@ -21,7 +21,9 @@
 
 namespace mlir::query::matcher {
 
-/// A class that provides utilities to find operations in the IR.
+/// Finds and collects matches from the IR. After construction
+/// `collectMatches` can be used to traverse the IR and apply
+/// matchers.
 class MatchFinder {
 
 public:
diff --git a/mlir/include/mlir/Query/Matcher/MatchersInternal.h b/mlir/include/mlir/Query/Matcher/MatchersInternal.h
index 183b2514e109f..88109430b6feb 100644
--- a/mlir/include/mlir/Query/Matcher/MatchersInternal.h
+++ b/mlir/include/mlir/Query/Matcher/MatchersInternal.h
@@ -8,11 +8,11 @@
 //
 // Implements the base layer of the matcher framework.
 //
-// Matchers are methods that return a Matcher which provides a method one of the
-// following methods: match(Operation *op), match(Operation *op,
-// SetVector<Operation *> &matchedOps)
+// Matchers are methods that return a Matcher which provides a
+// `match(...)` method whose parameters define the context of the match.
+// Support includes simple (unary) matchers as well as matcher combinators
+// (anyOf, allOf, etc.)
 //
-// The matcher functions are defined in include/mlir/IR/Matchers.h.
 // This file contains the wrapper classes needed to construct matchers for
 // mlir-query.
 //
@@ -25,6 +25,15 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 
 namespace mlir::query::matcher {
+class DynMatcher;
+namespace internal {
+
+bool allOfVariadicOperator(Operation *op, SetVector<Operation *> *matchedOps,
+                           ArrayRef<DynMatcher> innerMatchers);
+bool anyOfVariadicOperator(Operation *op, SetVector<Operation *> *matchedOps,
+                           ArrayRef<DynMatcher> innerMatchers);
+
+} // namespace internal
 
 // Defaults to false if T has no match() method with the signature:
 // match(Operation* op).
@@ -84,6 +93,27 @@ class MatcherFnImpl : public MatcherInterface {
   MatcherFn matcherFn;
 };
 
+// VariadicMatcher takes a vector of Matchers and returns true if any Matchers
+// match the given operation.
+using VariadicOperatorFunction = bool (*)(Operation *op,
+                                          SetVector<Operation *> *matchedOps,
+                                          ArrayRef<DynMatcher> innerMatchers);
+
+template <VariadicOperatorFunction Func>
+class VariadicMatcher : public MatcherInterface {
+public:
+  VariadicMatcher(std::vector<DynMatcher> matchers)
+      : matchers(std::move(matchers)) {}
+
+  bool match(Operation *op) override { return Func(op, nullptr, matchers); }
+  bool match(Operation *op, SetVector<Operation *> &matchedOps) override {
+    return Func(op, &matchedOps, matchers);
+  }
+
+private:
+  std::vector<DynMatcher> matchers;
+};
+
 // Matcher wraps a MatcherInterface implementation and provides match()
 // methods that redirect calls to the underlying implementation.
 class DynMatcher {
@@ -92,6 +122,31 @@ class DynMatcher {
   DynMatcher(MatcherInterface *implementation)
       : implementation(implementation) {}
 
+  // Construct from a variadic function.
+  enum VariadicOperator {
+    // Matches operations for which all provided matchers match.
+    AllOf,
+    // Matches operations for which at least one of the provided matchers
+    // matches.
+    AnyOf
+  };
+
+  static std::unique_ptr<DynMatcher>
+  constructVariadic(VariadicOperator Op,
+                    std::vector<DynMatcher> innerMatchers) {
+    switch (Op) {
+    case AllOf:
+      return std::make_unique<DynMatcher>(
+          new VariadicMatcher<internal::allOfVariadicOperator>(
+              std::move(innerMatchers)));
+    case AnyOf:
+      return std::make_unique<DynMatcher>(
+          new VariadicMatcher<internal::anyOfVariadicOperator>(
+              std::move(innerMatchers)));
+    }
+    llvm_unreachable("Invalid Op value.");
+  }
+
   template <typename MatcherFn>
   static std::unique_ptr<DynMatcher>
   constructDynMatcherFromMatcherFn(MatcherFn &matcherFn) {
@@ -113,6 +168,59 @@ class DynMatcher {
   std::string functionName;
 };
 
+// VariadicOperatorMatcher related types.
+template <typename... Ps>
+class VariadicOperatorMatcher {
+public:
+  VariadicOperatorMatcher(DynMatcher::VariadicOperator varOp, Ps &&...params)
+      : varOp(varOp), params(std::forward<Ps>(params)...) {}
+
+  operator std::unique_ptr<DynMatcher>() const & {
+    return DynMatcher::constructVariadic(
+        varOp, getMatchers(std::index_sequence_for<Ps...>()));
+  }
+
+  operator std::unique_ptr<DynMatcher>() && {
+    return DynMatcher::constructVariadic(
+        varOp, std::move(*this).getMatchers(std::index_sequence_for<Ps...>()));
+  }
+
+private:
+  // Helper method to unpack the tuple into a vector.
+  template <std::size_t... Is>
+  std::vector<DynMatcher> getMatchers(std::index_sequence<Is...>) const & {
+    return {DynMatcher(std::get<Is>(params))...};
+  }
+
+  template <std::size_t... Is>
+  std::vector<DynMatcher> getMatchers(std::index_sequence<Is...>) && {
+    return {DynMatcher(std::get<Is>(std::move(params)))...};
+  }
+
+  const DynMatcher::VariadicOperator varOp;
+  std::tuple<Ps...> params;
+};
+
+// Overloaded function object to generate VariadicOperatorMatcher objects from
+// arbitrary matchers.
+template <unsigned MinCount, unsigned MaxCount>
+struct VariadicOperatorMatcherFunc {
+  DynMatcher::VariadicOperator varOp;
+
+  template <typename... Ms>
+  VariadicOperatorMatcher<Ms...> operator()(Ms &&...Ps) const {
+    static_assert(MinCount <= sizeof...(Ms) && sizeof...(Ms) <= MaxCount,
+                  "invalid number of parameters for variadic matcher");
+    return VariadicOperatorMatcher<Ms...>(varOp, std::forward<Ms>(Ps)...);
+  }
+};
+
+namespace internal {
+const VariadicOperatorMatcherFunc<1, std::numeric_limits<unsigned>::max()>
+    anyOf = {DynMatcher::AnyOf};
+const VariadicOperatorMatcherFunc<1, std::numeric_limits<unsigned>::max()>
+    allOf = {DynMatcher::AllOf};
+} // namespace internal
 } // namespace mlir::query::matcher
 
 #endif // MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERSINTERNAL_H
diff --git a/mlir/include/mlir/Query/Matcher/SliceMatchers.h b/mlir/include/mlir/Query/Matcher/SliceMatchers.h
index 441205b3a9615..7181648f06f89 100644
--- a/mlir/include/mlir/Query/Matcher/SliceMatchers.h
+++ b/mlir/include/mlir/Query/Matcher/SliceMatchers.h
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file provides matchers for MLIRQuery that peform slicing analysis
+// This file defines slicing-analysis matchers that extend and abstract the
+// core implementations from `SliceAnalysis.h`.
 //
 //===----------------------------------------------------------------------===//
 
@@ -16,9 +17,9 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/IR/Operation.h"
 
-/// A matcher encapsulating `getBackwardSlice` method from SliceAnalysis.h.
-/// Additionally, it limits the slice computation to a certain depth level using
-/// a custom filter.
+/// Computes the backward-slice of all transitive defs reachable from `rootOp`,
+/// if `innerMatcher` matches. The traversal stops once the desired depth level
+/// is reached.
 ///
 /// Example: starting from node 9, assuming the matcher
 /// computes the slice for the first two depth levels:
@@ -119,6 +120,77 @@ bool BackwardSliceMatcher<Matcher>::matches(
                            : backwardSlice.size() >= 1;
 }
 
+/// Computes the backward-slice of all transitive defs reachable from `rootOp`,
+/// if `innerMatcher` matches. Traversal stops where `filterMatcher` matches.
+template <typename BaseMatcher, typename Filter>
+class PredicateBackwardSliceMatcher {
+public:
+  PredicateBackwardSliceMatcher(BaseMatcher innerMatcher, Filter filterMatcher,
+                                bool inclusive, bool omitBlockArguments,
+                                bool omitUsesFromAbove)
+      : innerMatcher(std::move(innerMatcher)),
+        filterMatcher(std::move(filterMatcher)), inclusive(inclusive),
+        omitBlockArguments(omitBlockArguments),
+        omitUsesFromAbove(omitUsesFromAbove) {}
+
+  bool match(Operation *rootOp, SetVector<Operation *> &backwardSlice) {
+    backwardSlice.clear();
+    BackwardSliceOptions options;
+    options.inclusive = inclusive;
+    options.omitUsesFromAbove = omitUsesFromAbove;
+    options.omitBlockArguments = omitBlockArguments;
+    if (innerMatcher.match(rootOp)) {
+      options.filter = [&](Operation *subOp) {
+        return !filterMatcher.match(subOp);
+      };
+      LogicalResult result = getBackwardSlice(rootOp, &backwardSlice, options);
+      assert(result.succeeded() && "expected backward slice to succeed");
+      (void)result;
+      return options.inclusive ? backwardSlice.size() > 1
+                               : backwardSlice.size() >= 1;
+    }
+    return false;
+  }
+
+private:
+  BaseMatcher innerMatcher;
+  Filter filterMatcher;
+  bool inclusive;
+  bool omitBlockArguments;
+  bool omitUsesFromAbove;
+};
+
+/// Computes the forward-slice of all users reachable from `rootOp`,
+/// if `innerMatcher` matches. Traversal stops where `filterMatcher` matches.
+template <typename BaseMatcher, typename Filter>
+class PredicateForwardSliceMatcher {
+public:
+  PredicateForwardSliceMatcher(BaseMatcher innerMatcher, Filter filterMatcher,
+                               bool inclusive)
+      : innerMatcher(std::move(innerMatcher)),
+        filterMatcher(std::move(filterMatcher)), inclusive(inclusive) {}
+
+  bool match(Operation *rootOp, SetVector<Operation *> &forwardSlice) {
+    forwardSlice.clear();
+    ForwardSliceOptions options;
+    options.inclusive = inclusive;
+    if (innerMatcher.match(rootOp)) {
+      options.filter = [&](Operation *subOp) {
+        return !filterMatcher.match(subOp);
+      };
+      getForwardSlice(rootOp, &forwardSlice, options);
+      return options.inclusive ? forwardSlice.size() > 1
+                               : forwardSlice.size() >= 1;
+    }
+    return false;
+  }
+
+private:
+  BaseMatcher innerMatcher;
+  Filter filterMatcher;
+  bool inclusive;
+};
+
 /// Matches transitive defs of a top-level operation up to N levels.
 template <typename Matcher>
 inline BackwardSliceMatcher<Matcher>
@@ -130,7 +202,7 @@ m_GetDefinitions(Matcher innerMatcher, int64_t maxDepth, bool inclusive,
                                        omitUsesFromAbove);
 }
 
-/// Matches all transitive defs of a top-level operation up to N levels
+/// Matches all transitive defs of a top-level operation up to N levels.
 template <typename Matcher>
 inline BackwardSliceMatcher<Matcher> m_GetAllDefinitions(Matcher innerMatcher,
                                                          int64_t maxDepth) {
@@ -139,6 +211,28 @@ inline BackwardSliceMatcher<Matcher> m_GetAllDefinitions(Matcher innerMatcher,
                                        false, false);
 }
 
+/// Matches all transitive defs of a top-level operation and stops where
+/// `filterMatcher` rejects.
+template <typename BaseMatcher, typename Filter>
+inline PredicateBackwardSliceMatcher<BaseMatcher, Filter>
+m_GetDefinitionsByPredicate(BaseMatcher innerMatcher, Filter filterMatcher,
+                            bool inclusive, bool omitBlockArguments,
+                            bool omitUsesFromAbove) {
+  return PredicateBackwardSliceMatcher<BaseMatcher, Filter>(
+      std::move(innerMatcher), std::move(filterMatcher), inclusive,
+      omitBlockArguments, omitUsesFromAbove);
+}
+
+/// Matches all users of a top-level operation and stops where
+/// `filterMatcher` rejects.
+template <typename BaseMatcher, typename Filter>
+inline PredicateForwardSliceMatcher<BaseMatcher, Filter>
+m_GetUsersByPredicate(BaseMatcher innerMatcher, Filter filterMatcher,
+                      bool inclusive) {
+  return PredicateForwardSliceMatcher<BaseMatcher, Filter>(
+      std::move(innerMatcher), std::move(filterMatcher), inclusive);
+}
+
 } // namespace mlir::query::matcher
 
 #endif // MLIR_TOOLS_MLIRQUERY_MATCHERS_SLICEMATCHERS_H
diff --git a/mlir/include/mlir/Query/Matcher/VariantValue.h b/mlir/include/mlir/Query/Matcher/VariantValue.h
index 98c0a18e25101..1a47576de1841 100644
--- a/mlir/include/mlir/Query/Matcher/VariantValue.h
+++ b/mlir/include/mlir/Query/Matcher/VariantValue.h
@@ -26,7 +26,12 @@ enum class ArgKind { Boolean, Matcher, Signed, String };
 // A variant matcher object to abstract simple and complex matchers into a
 // single object type.
 class VariantMatcher {
-  class MatcherOps;
+  class MatcherOps {
+  public:
+    std::optional<DynMatcher>
+    constructVariadicOperator(DynMatcher::VariadicOperator varOp,
+                              ArrayRef<VariantMatcher> innerMatchers) const;
+  };
 
   // Payload interface to be specialized by each matcher type. It follows a
   // similar interface as VariantMatcher itself.
@@ -43,6 +48,9 @@ class VariantMatcher {
 
   // Clones the provided matcher.
   static VariantMatcher SingleMatcher(DynMatcher matcher);
+  static VariantMatcher
+  VariadicOperatorMatcher(DynMatcher::VariadicOperator varOp,
+                          ArrayRef<VariantMatcher> args);
 
   // Makes the matcher the "null" matcher.
   void reset();
@@ -61,6 +69,7 @@ class VariantMatcher {
       : value(std::move(value)) {}
 
   class SinglePayload;
+  class VariadicOpPayload;
 
   std::shared_ptr<const Payload> value;
 };
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 81858812d2623..f6437657c9a93 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -221,12 +221,13 @@ class TypeConverter {
   ///
   /// The conversion functions take a non-null Type or subclass of Type and a
   /// non-null Attribute (or subclass of Attribute), and returns a
-  /// `AttributeConversionResult`. This result can either contan an `Attribute`,
-  /// which may be `nullptr`, representing the conversion's success,
-  /// `AttributeConversionResult::na()` (the default empty value), indicating
-  /// that the conversion function did not apply and that further conversion
-  /// functions should be checked, or `AttributeConversionResult::abort()`
-  /// indicating that the conversion process should be aborted.
+  /// `AttributeConversionResult`. This result can either contain an
+  /// `Attribute`, which may be `nullptr`, representing the conversion's
+  /// success, `AttributeConversionResult::na()` (the default empty value),
+  /// indicating that the conversion function did not apply and that further
+  /// conversion functions should be checked, or
+  /// `AttributeConversionResult::abort()` indicating that the conversion
+  /// process should be aborted.
   ///
   /// Registered conversion functions are callled in the reverse of the order in
   /// which they were registered.
diff --git a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
index 953d65e467b7a..d05374f667a51 100644
--- a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Casting.h"
 #include <cassert>
 #include <optional>
 
diff --git a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
index 016e59dcb744e..e625f626d12fd 100644
--- a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Casting.h"
 #include <cassert>
 #include <optional>
 
diff --git a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
index 6cc5ec639f448..f4b02b496a5c5 100644
--- a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
+++ b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Analysis//FlatLinearValueConstraints.h"
 
-#include "mlir/Analysis/Presburger/LinearTransform.h"
 #include "mlir/Analysis/Presburger/PresburgerSpace.h"
 #include "mlir/Analysis/Presburger/Simplex.h"
 #include "mlir/Analysis/Presburger/Utils.h"
@@ -17,7 +16,6 @@
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/InterleavedRange.h"
diff --git a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
index 2181d9484f223..17e48e0d069b7 100644
--- a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
+++ b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
@@ -25,16 +25,12 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
 #include <functional>
 #include <memory>
-#include <numeric>
 #include <optional>
-#include <sstream>
-#include <string>
 #include <utility>
 #include <vector>
 
@@ -2218,7 +2214,7 @@ IntegerRelation::unionBoundingBox(const IntegerRelation &otherCst) {
       auto constOtherLb = otherCst.getConstantBound(BoundType::LB, d);
       if (!constLb.has_value() || !constOtherLb.has_value())
         return failure();
-      std::fill(minLb.begin(), minLb.end(), 0);
+      llvm::fill(minLb, 0);
       minLb.back() = std::min(*constLb, *constOtherLb);
     }
 
@@ -2234,12 +2230,12 @@ IntegerRelation::unionBoundingBox(const IntegerRelation &otherCst) {
       auto constOtherUb = otherCst.getConstantBound(BoundType::UB, d);
       if (!constUb.has_value() || !constOtherUb.has_value())
         return failure();
-      std::fill(maxUb.begin(), maxUb.end(), 0);
+      llvm::fill(maxUb, 0);
       maxUb.back() = std::max(*constUb, *constOtherUb);
     }
 
-    std::fill(newLb.begin(), newLb.end(), 0);
-    std::fill(newUb.begin(), newUb.end(), 0);
+    llvm::fill(newLb, 0);
+    llvm::fill(newUb, 0);
 
     // The divisor for lb, ub, otherLb, otherUb at this point is lbDivisor,
     // and so it's the divisor for newLb and newUb as well.
diff --git a/mlir/lib/Analysis/Presburger/PWMAFunction.cpp b/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
index 3ed77998c92ad..55d17ad47e991 100644
--- a/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
+++ b/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
@@ -143,7 +143,7 @@ void MultiAffineFunction::mergeDivs(MultiAffineFunction &other) {
   SmallVector<DynamicAPInt, 8> div(other.divs.getNumVars() + 1);
   for (unsigned i = 0; i < nDivs; ++i) {
     // Zero fill.
-    std::fill(div.begin(), div.end(), 0);
+    llvm::fill(div, 0);
     // Fill div with dividend from `divs`. Do not fill the constant.
     std::copy(divs.getDividend(i).begin(), divs.getDividend(i).end() - 1,
               div.begin());
diff --git a/mlir/lib/Analysis/Presburger/Simplex.cpp b/mlir/lib/Analysis/Presburger/Simplex.cpp
index 4eb6e6da2950c..08290db55f2c7 100644
--- a/mlir/lib/Analysis/Presburger/Simplex.cpp
+++ b/mlir/lib/Analysis/Presburger/Simplex.cpp
@@ -18,7 +18,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <functional>
diff --git a/mlir/lib/Analysis/Presburger/Utils.cpp b/mlir/lib/Analysis/Presburger/Utils.cpp
index f12bd5239245d..2aaa4c033d0ce 100644
--- a/mlir/lib/Analysis/Presburger/Utils.cpp
+++ b/mlir/lib/Analysis/Presburger/Utils.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Analysis/Presburger/PresburgerSpace.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <cstdint>
diff --git a/mlir/lib/Analysis/SliceAnalysis.cpp b/mlir/lib/Analysis/SliceAnalysis.cpp
index 4cdf47a405c01..36a9812bd7972 100644
--- a/mlir/lib/Analysis/SliceAnalysis.cpp
+++ b/mlir/lib/Analysis/SliceAnalysis.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 
 ///
 /// Implements Analysis functions specific to slicing in Function.
diff --git a/mlir/lib/Bindings/Python/IRAttributes.cpp b/mlir/lib/Bindings/Python/IRAttributes.cpp
index 12725a0ed0939..8f79caf08a6d0 100644
--- a/mlir/lib/Bindings/Python/IRAttributes.cpp
+++ b/mlir/lib/Bindings/Python/IRAttributes.cpp
@@ -1670,7 +1670,7 @@ class PyStridedLayoutAttribute
         [](int64_t rank, DefaultingPyMlirContext ctx) {
           auto dynamic = mlirShapedTypeGetDynamicStrideOrOffset();
           std::vector<int64_t> strides(rank);
-          std::fill(strides.begin(), strides.end(), dynamic);
+          llvm::fill(strides, dynamic);
           MlirAttribute attr = mlirStridedLayoutAttrGet(
               ctx->get(), dynamic, strides.size(), strides.data());
           return PyStridedLayoutAttribute(ctx->getRef(), attr);
diff --git a/mlir/lib/Bindings/Python/IRTypes.cpp b/mlir/lib/Bindings/Python/IRTypes.cpp
index 0f2719c10a027..b11e3f75b8463 100644
--- a/mlir/lib/Bindings/Python/IRTypes.cpp
+++ b/mlir/lib/Bindings/Python/IRTypes.cpp
@@ -544,6 +544,15 @@ void mlir::PyShapedType::bindDerived(ClassTy &c) {
       nb::arg("dim"),
       "Returns whether the dim-th dimension of the given shaped type is "
       "dynamic.");
+  c.def(
+      "is_static_dim",
+      [](PyShapedType &self, intptr_t dim) -> bool {
+        self.requireHasRank();
+        return mlirShapedTypeIsStaticDim(self, dim);
+      },
+      nb::arg("dim"),
+      "Returns whether the dim-th dimension of the given shaped type is "
+      "static.");
   c.def(
       "get_dim_size",
       [](PyShapedType &self, intptr_t dim) {
@@ -558,6 +567,12 @@ void mlir::PyShapedType::bindDerived(ClassTy &c) {
       nb::arg("dim_size"),
       "Returns whether the given dimension size indicates a dynamic "
       "dimension.");
+  c.def_static(
+      "is_static_size",
+      [](int64_t size) -> bool { return mlirShapedTypeIsStaticSize(size); },
+      nb::arg("dim_size"),
+      "Returns whether the given dimension size indicates a static "
+      "dimension.");
   c.def(
       "is_dynamic_stride_or_offset",
       [](PyShapedType &self, int64_t val) -> bool {
@@ -567,6 +582,15 @@ void mlir::PyShapedType::bindDerived(ClassTy &c) {
       nb::arg("dim_size"),
       "Returns whether the given value is used as a placeholder for dynamic "
       "strides and offsets in shaped types.");
+  c.def(
+      "is_static_stride_or_offset",
+      [](PyShapedType &self, int64_t val) -> bool {
+        self.requireHasRank();
+        return mlirShapedTypeIsStaticStrideOrOffset(val);
+      },
+      nb::arg("dim_size"),
+      "Returns whether the given shaped type stride or offset value is "
+      "statically-sized.");
   c.def_prop_ro(
       "shape",
       [](PyShapedType &self) {
diff --git a/mlir/lib/CAPI/IR/BuiltinTypes.cpp b/mlir/lib/CAPI/IR/BuiltinTypes.cpp
index a080adf0f8103..9d8554aabff8a 100644
--- a/mlir/lib/CAPI/IR/BuiltinTypes.cpp
+++ b/mlir/lib/CAPI/IR/BuiltinTypes.cpp
@@ -332,6 +332,11 @@ bool mlirShapedTypeIsDynamicDim(MlirType type, intptr_t dim) {
       .isDynamicDim(static_cast<unsigned>(dim));
 }
 
+bool mlirShapedTypeIsStaticDim(MlirType type, intptr_t dim) {
+  return llvm::cast<ShapedType>(unwrap(type))
+      .isStaticDim(static_cast<unsigned>(dim));
+}
+
 int64_t mlirShapedTypeGetDimSize(MlirType type, intptr_t dim) {
   return llvm::cast<ShapedType>(unwrap(type))
       .getDimSize(static_cast<unsigned>(dim));
@@ -343,10 +348,18 @@ bool mlirShapedTypeIsDynamicSize(int64_t size) {
   return ShapedType::isDynamic(size);
 }
 
+bool mlirShapedTypeIsStaticSize(int64_t size) {
+  return ShapedType::isStatic(size);
+}
+
 bool mlirShapedTypeIsDynamicStrideOrOffset(int64_t val) {
   return ShapedType::isDynamic(val);
 }
 
+bool mlirShapedTypeIsStaticStrideOrOffset(int64_t val) {
+  return ShapedType::isStatic(val);
+}
+
 int64_t mlirShapedTypeGetDynamicStrideOrOffset() {
   return ShapedType::kDynamic;
 }
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 4fbe6a03f6bad..3b143ca1ef9ce 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index 3596b3235a631..cf9bb3a000050 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -14,7 +14,10 @@
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
+#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -32,6 +35,7 @@ using namespace mlir::amdgpu;
 namespace {
 // Define commonly used chipsets versions for convenience.
 constexpr Chipset kGfx942 = Chipset(9, 4, 2);
+constexpr Chipset kGfx950 = Chipset(9, 5, 0);
 
 struct ArithToAMDGPUConversionPass final
     : impl::ArithToAMDGPUConversionPassBase<ArithToAMDGPUConversionPass> {
@@ -73,6 +77,28 @@ struct TruncfToFloat16RewritePattern final
                                 PatternRewriter &rewriter) const override;
 };
 
+struct ScalingExtFRewritePattern final
+    : OpRewritePattern<arith::ScalingExtFOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  ScalingExtFRewritePattern(MLIRContext *ctx)
+      : OpRewritePattern::OpRewritePattern(ctx) {}
+
+  LogicalResult matchAndRewrite(arith::ScalingExtFOp op,
+                                PatternRewriter &rewriter) const override;
+};
+
+struct ScalingTruncFRewritePattern final
+    : OpRewritePattern<arith::ScalingTruncFOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  ScalingTruncFRewritePattern(MLIRContext *ctx)
+      : OpRewritePattern::OpRewritePattern(ctx) {}
+
+  LogicalResult matchAndRewrite(arith::ScalingTruncFOp op,
+                                PatternRewriter &rewriter) const override;
+};
+
 } // end namespace
 
 static bool isSupportedF8(Type elementType, Chipset chipset) {
@@ -395,6 +421,247 @@ LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
   return success();
 }
 
+/// Get the broadcasted / splatted value for a chain of ops.
+static Value getOriginalVectorValue(Value value) {
+  Value current = value;
+  while (Operation *definingOp = current.getDefiningOp()) {
+    bool skipOp = llvm::TypeSwitch<Operation *, bool>(definingOp)
+                      .Case<vector::ShapeCastOp>([&current](auto op) {
+                        current = op.getSource();
+                        return true;
+                      })
+                      .Case<vector::BroadcastOp>([&current](auto op) {
+                        current = op.getSource();
+                        return false;
+                      })
+                      .Case<vector::SplatOp>([&current](auto op) {
+                        current = op.getInput();
+                        return false;
+                      })
+                      .Default([](Operation *) { return false; });
+
+    if (!skipOp) {
+      break;
+    }
+  }
+  return current;
+}
+
+LogicalResult
+ScalingExtFRewritePattern::matchAndRewrite(arith::ScalingExtFOp op,
+                                           PatternRewriter &rewriter) const {
+  Location loc = op.getLoc();
+  constexpr int64_t opWidth = 2;
+
+  Value in = op.getIn();
+  Value scale = op.getScale();
+  Value out = op.getOut();
+
+  Type f32 = rewriter.getF32Type();
+  Type inType = getElementTypeOrSelf(in);
+  Type scaleType = getElementTypeOrSelf(scale);
+  Type outType = getElementTypeOrSelf(out);
+
+  VectorType outVecType = dyn_cast<VectorType>(out.getType());
+  VectorType scaleVecType = dyn_cast<VectorType>(scale.getType());
+
+  if (outVecType && outVecType.isScalable())
+    return failure();
+
+  Type scaleF32Type =
+      scaleVecType ? VectorType::get(scaleVecType.getShape(), f32) : f32;
+  if (scaleType.getIntOrFloatBitWidth() < 32)
+    scale = rewriter.create<arith::ExtFOp>(loc, scaleF32Type, scale);
+  else if (scaleType.getIntOrFloatBitWidth() > 32)
+    scale = rewriter.create<arith::TruncFOp>(loc, scaleF32Type, scale);
+
+  VectorType extScaleResultType = VectorType::get(opWidth, outType);
+
+  if (!outVecType) {
+    Value inCast =
+        rewriter.create<vector::SplatOp>(loc, VectorType::get(1, inType), in);
+    // TODO: replace this with non-packed ScaledExtOp
+    Value scaleExt = rewriter.create<amdgpu::ScaledExtPackedOp>(
+        loc, extScaleResultType, inCast, scale, 0);
+    scaleExt = rewriter.replaceOpWithNewOp<vector::ExtractOp>(op, scaleExt, 0);
+    return success();
+  }
+
+  VectorType inVecType = cast<VectorType>(in.getType());
+  Value origScale = getOriginalVectorValue(op.getScale());
+
+  ArrayRef<int64_t> inShape = inVecType.getShape();
+  SmallVector<int64_t> originalScaleShape;
+  if (auto origScaleVecType = dyn_cast<VectorType>(origScale.getType()))
+    llvm::append_range(originalScaleShape, origScaleVecType.getShape());
+
+  originalScaleShape.insert(originalScaleShape.end(),
+                            inShape.size() - originalScaleShape.size(), 1);
+
+  auto maybeRatio = computeShapeRatio(inShape, originalScaleShape);
+  assert(maybeRatio &&
+         "failed to derive block size from broadcast or splat operation");
+
+  SmallVector<int64_t> ratio =
+      maybeRatio.value_or(SmallVector<int64_t>(inShape.size(), 1));
+
+  int64_t blockSize = computeProduct(ratio);
+
+  Value zero = rewriter.create<arith::ConstantOp>(
+      loc, outType, rewriter.getFloatAttr(outType, 0.0));
+  Value result = rewriter.createOrFold<vector::SplatOp>(loc, outVecType, zero);
+
+  for (SmallVector<int64_t> offsets : StaticTileOffsetRange(inShape, ratio)) {
+    SmallVector<int64_t> strides(offsets.size(), 1);
+    Value block = rewriter.create<vector::ExtractStridedSliceOp>(
+        loc, in, offsets, ratio, strides);
+    VectorType block1DType = VectorType::get(blockSize, inType);
+    Value block1D =
+        rewriter.create<vector::ShapeCastOp>(loc, block1DType, block);
+    Value uniformScale =
+        rewriter.create<vector::ExtractOp>(loc, scale, offsets);
+
+    VectorType blockResultType = VectorType::get(blockSize, outType);
+    Value blockResult =
+        rewriter.createOrFold<vector::SplatOp>(loc, blockResultType, zero);
+
+    for (int64_t i = 0, sliceWidth = std::min(opWidth, blockSize - i);
+         i < blockSize;
+         i += sliceWidth, sliceWidth = std::min(opWidth, blockSize - i)) {
+      Value slice = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, block1D, i, sliceWidth, 1);
+      // TODO: replace this with non-packed ScaledExtOp for sliceWidth == 1
+      Value scaleExt = rewriter.create<amdgpu::ScaledExtPackedOp>(
+          loc, extScaleResultType, slice, uniformScale, 0);
+      if (sliceWidth != opWidth)
+        scaleExt = rewriter.create<vector::ExtractStridedSliceOp>(
+            loc, scaleExt, 0, sliceWidth, 1);
+      blockResult = rewriter.create<vector::InsertStridedSliceOp>(
+          loc, scaleExt, blockResult, i, 1);
+    }
+
+    VectorType resultType = VectorType::get(ratio, outType);
+    Value cast =
+        rewriter.create<vector::ShapeCastOp>(loc, resultType, blockResult);
+    result = rewriter.create<vector::InsertStridedSliceOp>(loc, cast, result,
+                                                           offsets, strides);
+  }
+
+  rewriter.replaceOp(op, result);
+
+  return success();
+}
+
+LogicalResult
+ScalingTruncFRewritePattern::matchAndRewrite(arith::ScalingTruncFOp op,
+                                             PatternRewriter &rewriter) const {
+  Location loc = op.getLoc();
+  constexpr int64_t opWidth = 2;
+
+  Value in = op.getIn();
+  Value scale = op.getScale();
+  Value out = op.getOut();
+
+  Type f32 = rewriter.getF32Type();
+  Type inType = getElementTypeOrSelf(in);
+  Type scaleType = getElementTypeOrSelf(scale);
+  Type outType = getElementTypeOrSelf(out);
+
+  VectorType outVecType = dyn_cast<VectorType>(out.getType());
+  VectorType scaleVecType = dyn_cast<VectorType>(scale.getType());
+
+  if (outVecType && outVecType.isScalable())
+    return failure();
+
+  Type scaleF32Type =
+      scaleVecType ? VectorType::get(scaleVecType.getShape(), f32) : f32;
+  if (scaleType.getIntOrFloatBitWidth() < 32)
+    scale = rewriter.create<arith::ExtFOp>(loc, scaleF32Type, scale);
+  else if (scaleType.getIntOrFloatBitWidth() > 32)
+    scale = rewriter.create<arith::TruncFOp>(loc, scaleF32Type, scale);
+
+  Value zero = rewriter.create<arith::ConstantOp>(
+      loc, outType, rewriter.getFloatAttr(outType, 0.0));
+  unsigned numPackedElem = 32 / outType.getIntOrFloatBitWidth();
+  VectorType truncScaleResultType = VectorType::get(numPackedElem, outType);
+
+  if (!outVecType) {
+    Type inVecType = VectorType::get(1, inType);
+    Value inCast = rewriter.create<vector::SplatOp>(loc, inVecType, in);
+    // TODO: replace this with non-packed ScaledTruncOp
+    Value scaleTrunc = rewriter.create<amdgpu::PackedScaledTruncOp>(
+        loc, truncScaleResultType, inCast, scale, 0, /*existing=*/nullptr);
+    scaleTrunc =
+        rewriter.replaceOpWithNewOp<vector::ExtractOp>(op, scaleTrunc, 0);
+    return success();
+  }
+
+  VectorType inVecType = cast<VectorType>(in.getType());
+  Value origScale = getOriginalVectorValue(op.getScale());
+
+  ArrayRef<int64_t> inShape = inVecType.getShape();
+  SmallVector<int64_t> originalScaleShape;
+  if (auto origScaleVecType = dyn_cast<VectorType>(origScale.getType()))
+    llvm::append_range(originalScaleShape, origScaleVecType.getShape());
+
+  originalScaleShape.insert(originalScaleShape.end(),
+                            inShape.size() - originalScaleShape.size(), 1);
+
+  auto maybeRatio = computeShapeRatio(inShape, originalScaleShape);
+  assert(maybeRatio &&
+         "failed to derive block size from broadcast or splat operation");
+
+  SmallVector<int64_t> ratio =
+      maybeRatio.value_or(SmallVector<int64_t>(inShape.size(), 1));
+
+  int64_t blockSize = computeProduct(ratio);
+
+  Value result = rewriter.createOrFold<vector::SplatOp>(loc, outVecType, zero);
+
+  for (SmallVector<int64_t> offsets : StaticTileOffsetRange(inShape, ratio)) {
+    SmallVector<int64_t> strides(offsets.size(), 1);
+    Value block = rewriter.create<vector::ExtractStridedSliceOp>(
+        loc, in, offsets, ratio, strides);
+    VectorType block1DType = VectorType::get(blockSize, inType);
+    Value block1D =
+        rewriter.create<vector::ShapeCastOp>(loc, block1DType, block);
+    Value uniformScale =
+        rewriter.create<vector::ExtractOp>(loc, scale, offsets);
+
+    VectorType blockResultType = VectorType::get(blockSize, outType);
+    Value blockResult =
+        rewriter.createOrFold<vector::SplatOp>(loc, blockResultType, zero);
+
+    for (int64_t i = 0, sliceWidth = std::min(opWidth, blockSize - i);
+         i < blockSize;
+         i += sliceWidth, sliceWidth = std::min(opWidth, blockSize - i)) {
+      Value slice = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, block1D, i, sliceWidth, 1);
+      // TODO: replace this with non-packed ScaledTruncOp for sliceWidth == 1
+      Value scaleTrunc = rewriter.create<amdgpu::PackedScaledTruncOp>(
+          loc, truncScaleResultType, slice, uniformScale, 0,
+          /*existing=*/nullptr);
+      int64_t packedWidth =
+          cast<VectorType>(scaleTrunc.getType()).getNumElements();
+      if (packedWidth != opWidth)
+        scaleTrunc = rewriter.create<vector::ExtractStridedSliceOp>(
+            loc, scaleTrunc, 0, sliceWidth, 1);
+      blockResult = rewriter.create<vector::InsertStridedSliceOp>(
+          loc, scaleTrunc, blockResult, i, 1);
+    }
+
+    VectorType resultType = VectorType::get(ratio, outType);
+    Value cast =
+        rewriter.create<vector::ShapeCastOp>(loc, resultType, blockResult);
+    result = rewriter.create<vector::InsertStridedSliceOp>(loc, cast, result,
+                                                           offsets, strides);
+  }
+
+  rewriter.replaceOp(op, result);
+
+  return success();
+}
+
 void mlir::arith::populateArithToAMDGPUConversionPatterns(
     RewritePatternSet &patterns, bool convertFP8Arithmetic,
     bool saturateFP8Truncf, bool allowPackedF16Rtz, Chipset chipset) {
@@ -406,6 +673,11 @@ void mlir::arith::populateArithToAMDGPUConversionPatterns(
   }
   if (allowPackedF16Rtz)
     patterns.add<TruncfToFloat16RewritePattern>(patterns.getContext());
+
+  if (chipset >= kGfx950) {
+    patterns.add<ScalingExtFRewritePattern>(patterns.getContext());
+    patterns.add<ScalingTruncFRewritePattern>(patterns.getContext());
+  }
 }
 
 void ArithToAMDGPUConversionPass::runOnOperation() {
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index b8e5aa87244fa..f7bf581adc9e3 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Pass/Pass.h"
 #include <type_traits>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 0c3f942b5cbd9..21ea444e31821 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
 #include "mlir/Dialect/ArmSME/Utils/Utils.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index c95e375ce9afe..94f7caa315cf7 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -10,7 +10,6 @@
 
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index 0b5a5d8e71063..b9991f36cdaaf 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
@@ -111,8 +110,8 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
             rewriter.create<memref::CastOp>(op->getLoc(), memrefType, alloc);
     }
 
-    rewriter.replaceOp(op, alloc);
     rewriter.create<memref::CopyOp>(loc, op.getInput(), alloc);
+    rewriter.replaceOp(op, alloc);
     return success();
   }
 };
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index 6956c81f2a2d3..e5e862315941d 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCOMPLEXTOLLVMPASS
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 050bedf67b542..56269d189873a 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include <optional>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp b/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp
index 5e7d2d8491533..4225f815ab69d 100644
--- a/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp
+++ b/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp
@@ -12,11 +12,9 @@
 
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRV.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "complex-to-spirv-pattern"
 
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 5089179435f1e..0c832c452718b 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -14,9 +14,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include <memory>
 #include <type_traits>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index cf416e140c85f..88a8b7fb185c5 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/PrintCallHelper.h"
-#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -25,8 +24,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/ADT/StringRef.h"
-#include <functional>
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCONTROLFLOWTOLLVMPASS
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 1c592d665f3e4..9831dcaaaccc8 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Conversion/ControlFlowToSCF/ControlFlowToSCF.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
diff --git a/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp b/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
index d8f3e99510953..caaffc5a33155 100644
--- a/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
@@ -11,16 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.h"
-#include "../SPIRVCommon/Pattern.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "cf-to-spirv-pattern"
diff --git a/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp b/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
index 2cf44051ce5f5..ed5d6d4a7fe40 100644
--- a/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
+++ b/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
@@ -9,11 +9,9 @@
 #include "mlir/Analysis/DataLayoutAnalysis.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include <memory>
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 72977e6cc0e47..36235636d6ba2 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -20,33 +20,22 @@
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
-#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
-#include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Type.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormatVariadic.h"
-#include <algorithm>
-#include <functional>
 #include <optional>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp b/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
index a5c2b92ca191d..7cc9680236b01 100644
--- a/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
+++ b/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
@@ -11,14 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h"
-#include "../SPIRVCommon/Pattern.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
 #include "mlir/IR/AffineMap.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "func-to-spirv-pattern"
 
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 512820bab4097..167cabbc57db9 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -21,9 +21,7 @@
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/GPUCommon/GPUToLLVM.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
@@ -40,8 +38,6 @@
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "gpu-to-llvm"
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index cef250232daf5..1ef6edea93c58 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index e543ca98d8d4e..456bfaba980ca 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
-#include "mlir/Transforms/Passes.h"
 
 #include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
@@ -34,14 +33,12 @@
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
-#include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/FormatVariadic.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
 #include "../GPUCommon/IndexIntrinsicsOpLowering.h"
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index b96c3a4e6dfb6..b99ed261ecfa3 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -13,13 +13,11 @@
 #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include <optional>
@@ -122,6 +120,16 @@ class GPUShuffleConversion final : public OpConversionPattern<gpu::ShuffleOp> {
                   ConversionPatternRewriter &rewriter) const override;
 };
 
+/// Pattern to convert a gpu.rotate op into a spirv.GroupNonUniformRotateKHROp.
+class GPURotateConversion final : public OpConversionPattern<gpu::RotateOp> {
+public:
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(gpu::RotateOp rotateOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
 class GPUPrintfConversion final : public OpConversionPattern<gpu::PrintfOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
@@ -488,6 +496,41 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Rotate
+//===----------------------------------------------------------------------===//
+
+LogicalResult GPURotateConversion::matchAndRewrite(
+    gpu::RotateOp rotateOp, OpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  const spirv::TargetEnv &targetEnv =
+      getTypeConverter<SPIRVTypeConverter>()->getTargetEnv();
+  unsigned subgroupSize =
+      targetEnv.getAttr().getResourceLimits().getSubgroupSize();
+  IntegerAttr widthAttr;
+  if (!matchPattern(rotateOp.getWidth(), m_Constant(&widthAttr)) ||
+      widthAttr.getValue().getZExtValue() > subgroupSize)
+    return rewriter.notifyMatchFailure(
+        rotateOp,
+        "rotate width is not a constant or larger than target subgroup size");
+
+  Location loc = rotateOp.getLoc();
+  auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
+  Value rotateResult = rewriter.create<spirv::GroupNonUniformRotateKHROp>(
+      loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
+  Value validVal;
+  if (widthAttr.getValue().getZExtValue() == subgroupSize) {
+    validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
+  } else {
+    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+                                              laneId, adaptor.getWidth());
+  }
+
+  rewriter.replaceOp(rotateOp, {rotateResult, validVal});
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Group ops
 //===----------------------------------------------------------------------===//
@@ -776,7 +819,7 @@ void mlir::populateGPUToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
                                       RewritePatternSet &patterns) {
   patterns.add<
       GPUBarrierConversion, GPUFuncOpConversion, GPUModuleConversion,
-      GPUReturnOpConversion, GPUShuffleConversion,
+      GPUReturnOpConversion, GPUShuffleConversion, GPURotateConversion,
       LaunchConfigConversion<gpu::BlockIdOp, spirv::BuiltIn::WorkgroupId>,
       LaunchConfigConversion<gpu::GridDimOp, spirv::BuiltIn::NumWorkgroups>,
       LaunchConfigConversion<gpu::BlockDimOp, spirv::BuiltIn::WorkgroupSize>,
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 509b6343057b9..0b2c06a08db2d 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -21,7 +21,6 @@
 #include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index d2f5e35853550..7bb86b5ce1ddd 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -12,10 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
-#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 7c441830e1e3b..4821962f989e6 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 using namespace index;
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index 86d6643820376..e34d5f74d232f 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -53,7 +53,7 @@ MemRefDescriptor MemRefDescriptor::fromStaticShape(
 
   // Extract all strides and offsets and verify they are static.
   auto [strides, offset] = type.getStridesAndOffset();
-  assert(!ShapedType::isDynamic(offset) && "expected static offset");
+  assert(ShapedType::isStatic(offset) && "expected static offset");
   assert(!llvm::any_of(strides, ShapedType::isDynamic) &&
          "expected static strides");
 
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 57c8f4402cf4b..7312594c761f7 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-#include "MemRefDescriptor.h"
 #include "mlir/Conversion/LLVMCommon/MemRefBuilder.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
@@ -610,7 +609,7 @@ bool LLVMTypeConverter::canConvertToBarePtr(BaseMemRefType type) {
     if (ShapedType::isDynamic(stride))
       return false;
 
-  return !ShapedType::isDynamic(offset);
+  return ShapedType::isStatic(offset);
 }
 
 /// Convert a memref type to a bare pointer to the memref element type.
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 9504bcb0eff09..c3f213147b7a7 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTLINALGTOSTANDARDPASS
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index c4792884eb34a..a0ce7d3b75fc2 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp b/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
index e065c804507ac..93d8b49b93bd1 100644
--- a/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
+++ b/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/PatternMatch.h"
@@ -21,7 +20,6 @@
 #include "mlir/Transforms/DialectConversion.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
-#include "../GPUCommon/IndexIntrinsicsOpLowering.h"
 #include "../GPUCommon/OpToFuncCallLowering.h"
 #include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 501bfa223fb18..59db14ed816be 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -12,14 +12,12 @@
 
 #include "../SPIRVCommon/Pattern.h"
 #include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "math-to-spirv-pattern"
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 7484e4b07390e..00bbdcb12e326 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -24,7 +24,6 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/Pass/Pass.h"
-#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/Support/MathExtras.h"
 #include <optional>
 
@@ -44,7 +43,7 @@ static constexpr LLVM::GEPNoWrapFlags kNoWrapFlags =
 namespace {
 
 static bool isStaticStrideOrOffset(int64_t strideOrOffset) {
-  return !ShapedType::isDynamic(strideOrOffset);
+  return ShapedType::isStatic(strideOrOffset);
 }
 
 static FailureOr<LLVM::LLVMFuncOp>
@@ -1469,7 +1468,7 @@ struct MemRefReshapeOpLowering
       Value stride = nullptr;
       int64_t targetRank = targetMemRefType.getRank();
       for (auto i : llvm::reverse(llvm::seq<int64_t>(0, targetRank))) {
-        if (!ShapedType::isDynamic(strides[i])) {
+        if (ShapedType::isStatic(strides[i])) {
           // If the stride for this dimension is dynamic, then use the product
           // of the sizes of the inner dimensions.
           stride =
@@ -1723,7 +1722,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
                 ArrayRef<int64_t> shape, ValueRange dynamicSizes, unsigned idx,
                 Type indexType) const {
     assert(idx < shape.size());
-    if (!ShapedType::isDynamic(shape[idx]))
+    if (ShapedType::isStatic(shape[idx]))
       return createIndexAttrConstant(rewriter, loc, indexType, shape[idx]);
     // Count the number of dynamic dims in range [0, idx]
     unsigned nDynamic =
@@ -1739,7 +1738,7 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<memref::ViewOp> {
                   ArrayRef<int64_t> strides, Value nextSize,
                   Value runningStride, unsigned idx, Type indexType) const {
     assert(idx < strides.size());
-    if (!ShapedType::isDynamic(strides[idx]))
+    if (ShapedType::isStatic(strides[idx]))
       return createIndexAttrConstant(rewriter, loc, indexType, strides[idx]);
     if (nextSize)
       return runningStride
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index f8867c65fe7d9..b866afbce98b0 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -21,7 +21,6 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Visitors.h"
-#include "llvm/Support/Debug.h"
 #include <cassert>
 #include <optional>
 
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index 8950985d3b67b..479725aae8afd 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -8,11 +8,9 @@
 
 #include "mlir/Conversion/OpenACCToSCF/ConvertOpenACCToSCF.h"
 
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7a0a7f86bc1e9..77a2708653576 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -12,7 +12,6 @@
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 945d38e929e08..7d20109b3db59 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -9,14 +9,12 @@
 #include "mlir/Conversion/PDLToPDLInterp/PDLToPDLInterp.h"
 
 #include "PredicateTree.h"
-#include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
index ac8ed4fdff7c3..3e434ea7e14c4 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
@@ -9,11 +9,8 @@
 #include "PredicateTree.h"
 #include "RootOrdering.h"
 
-#include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
-#include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/IR/BuiltinOps.h"
-#include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 114d634629d77..ac6dcf2513e99 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -19,8 +19,6 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Transforms/Transforms.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 345e8494194eb..dcb48529a74e6 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -13,13 +13,10 @@
 #include "mlir/Conversion/SCFToEmitC/SCFToEmitC.h"
 
 #include "mlir/Conversion/ConvertToEmitC/ToEmitCInterface.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/EmitC/IR/EmitC.h"
 #include "mlir/Dialect/EmitC/Transforms/TypeConversions.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 84959d5ff8aba..844e66e927c4d 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -29,7 +29,6 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
 #include "mlir/Transforms/RegionUtils.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/Support/Debug.h"
 #include <optional>
 
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
index 99631705851fd..c816356cf3f96 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
@@ -10,14 +10,8 @@
 
 #include "mlir/Conversion/SCFToGPU/SCFToGPU.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/Support/CommandLine.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTAFFINEFORTOGPUPASS
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 71786e856c6db..584ac2f11b670 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index a59001de299fd..78d13278fef53 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -12,10 +12,8 @@
 
 #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/Support/FormatVariadic.h"
 
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
index 3ef1d84ee2647..d974f184c08fc 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
 #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
 #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index 0e9eb9799c3e0..d7ae9f0e94fe8 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -11,8 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h"
-
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
@@ -269,8 +267,9 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       copyInfo.push_back(info);
     }
     // Create a call to the kernel and copy the data back.
-    rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, kernelFunc,
-                                              ArrayRef<Value>());
+    Operation *callOp = rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+        op, kernelFunc, ArrayRef<Value>());
+    rewriter.setInsertionPointAfter(callOp);
     for (CopyInfo info : copyInfo)
       copy(loc, info.src, info.dst, info.size, rewriter);
     return success();
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 2e4561e7cc3ea..cea9d1fdec809 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -11,11 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h"
-#include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Conversion/SPIRVCommon/AttrToLLVMConverter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
@@ -23,7 +21,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "spirv-to-llvm-pattern"
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index 1f55fb7456f66..da9ad3dd67328 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -11,10 +11,8 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassRegistry.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp b/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp
index b0ca0ca13d062..55eb11c87d230 100644
--- a/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp
+++ b/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp
@@ -12,9 +12,6 @@
 
 #include "mlir/Conversion/TensorToLinalg/TensorToLinalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/IR/AffineMap.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "tensor-to-linalg-pattern"
 
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index ef3f4bc22fb89..2c4d27502a521 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -11,14 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/TensorToSPIRV/TensorToSPIRV.h"
-#include "../SPIRVCommon/Pattern.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/AffineMap.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "tensor-to-spirv-pattern"
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 9dea12355a519..40ad63610e23f 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace tosa;
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp
index ede3c9e0040fd..46bdf2272ef01 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp
@@ -14,11 +14,8 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOARITHPASS
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index c2be08ef40f21..2f608bbd637b4 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -27,11 +27,9 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 
-#include <numeric>
 #include <type_traits>
 
 using namespace mlir;
@@ -759,7 +757,7 @@ computeTargetSize(PatternRewriter &rewriter, Location loc, IndexPool &indexPool,
   // dimension greater than 1 with a different value is undefined behavior.
   for (auto operand : operands) {
     auto size = cast<RankedTensorType>(operand.getType()).getDimSize(dim);
-    if (!ShapedType::isDynamic(size) && size > 1)
+    if (ShapedType::isStatic(size) && size > 1)
       return {rewriter.getIndexAttr(size), operand};
   }
 
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index c460a8bb2f4b2..b89fde4fbc17e 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -14,21 +14,13 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
-#include "mlir/Interfaces/InferTypeOpInterface.h"
-
-#include <numeric>
 #include <type_traits>
 
 using namespace mlir;
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
index 80df3908991dd..3733d043f9ebc 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
@@ -20,11 +20,9 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
-#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOLINALGNAMED
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
index 01a7cd7ac94db..c6a3ba9f1439f 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
@@ -21,11 +21,9 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
-#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/Passes.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp
index 8c39f5e8a6363..dba6f79d840a1 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Conversion/TosaToMLProgram/TosaToMLProgram.h"
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index ef144fc7f0d54..03f9d20ad69de 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -14,9 +14,7 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace tosa;
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
index 3fd72223d3139..14dfd98f5d47b 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
@@ -16,11 +16,9 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOSCFPASS
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 615c7ca1cfd15..9ceb5c0c7f2fe 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -14,7 +14,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
 #include "mlir/IR/PatternMatch.h"
@@ -84,7 +83,7 @@ TensorType inferReshapeExpandedType(TensorType inputType,
         return totalSize / totalSizeNoPlaceholder;
       });
 
-  bool resultIsStatic = !ShapedType::isDynamicShape(resultShape);
+  bool resultIsStatic = ShapedType::isStaticShape(resultShape);
 
   // A syntactic restriction in 'tensor.expand_shape' forbids a dynamically
   // shaped input from being reshaped into a statically shaped result. We may
@@ -306,7 +305,7 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
       int64_t size = i.value();
       size_t index = i.index();
       sizes.push_back(size == -1 ? ShapedType::kDynamic : size);
-      if (!ShapedType::isDynamic(sizes.back()))
+      if (ShapedType::isStatic(sizes.back()))
         continue;
 
       auto dim = rewriter.create<tensor::DimOp>(loc, input, index);
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp
index e1dbe7db30480..a7f415c574843 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp
@@ -17,9 +17,7 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOTENSORPASS
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 9017bcb5283c9..18adaa793787c 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -12,8 +12,6 @@
 
 #include "mlir/Conversion/VectorToGPU/VectorToGPU.h"
 
-#include <type_traits>
-
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -28,11 +26,9 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Region.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index 67c0eca15638a..549d0210af7ad 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 002dfebd2b602..9f5b8fcca6c26 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -12,7 +12,6 @@
 
 #include <numeric>
 #include <optional>
-#include <type_traits>
 
 #include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
 
@@ -20,7 +19,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
@@ -29,7 +27,6 @@
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTVECTORTOSCF
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index de2af69eba9ec..21d8e1d9f1156 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -1022,6 +1022,51 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
   }
 };
 
+struct VectorToElementOpConvert final
+    : OpConversionPattern<vector::ToElementsOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(vector::ToElementsOp toElementsOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    SmallVector<Value> results(toElementsOp->getNumResults());
+    Location loc = toElementsOp.getLoc();
+
+    // Input vectors of size 1 are converted to scalars by the type converter.
+    // We cannot use `spirv::CompositeExtractOp` directly in this case.
+    // For a scalar source, the result is just the scalar itself.
+    if (isa<spirv::ScalarType>(adaptor.getSource().getType())) {
+      results[0] = adaptor.getSource();
+      rewriter.replaceOp(toElementsOp, results);
+      return success();
+    }
+
+    Type srcElementType = toElementsOp.getElements().getType().front();
+    Type elementType = getTypeConverter()->convertType(srcElementType);
+    if (!elementType)
+      return rewriter.notifyMatchFailure(
+          toElementsOp,
+          llvm::formatv("failed to convert element type '{0}' to SPIR-V",
+                        srcElementType));
+
+    for (auto [idx, element] : llvm::enumerate(toElementsOp.getElements())) {
+      // Create an CompositeExtract operation only for results that are not
+      // dead.
+      if (element.use_empty())
+        continue;
+
+      Value result = rewriter.create<spirv::CompositeExtractOp>(
+          loc, elementType, adaptor.getSource(),
+          rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
+      results[idx] = result;
+    }
+
+    rewriter.replaceOp(toElementsOp, results);
+    return success();
+  }
+};
+
 } // namespace
 #define CL_INT_MAX_MIN_OPS                                                     \
   spirv::CLUMaxOp, spirv::CLUMinOp, spirv::CLSMaxOp, spirv::CLSMinOp
@@ -1039,8 +1084,8 @@ void mlir::populateVectorToSPIRVPatterns(
       VectorExtractElementOpConvert, VectorExtractOpConvert,
       VectorExtractStridedSliceOpConvert, VectorFmaOpConvert<spirv::GLFmaOp>,
       VectorFmaOpConvert<spirv::CLFmaOp>, VectorFromElementsOpConvert,
-      VectorInsertElementOpConvert, VectorInsertOpConvert,
-      VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
+      VectorToElementOpConvert, VectorInsertElementOpConvert,
+      VectorInsertOpConvert, VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
       VectorReductionPattern<CL_INT_MAX_MIN_OPS>,
       VectorReductionFloatMinMax<CL_FLOAT_MAX_MIN_OPS>,
       VectorReductionFloatMinMax<GL_FLOAT_MAX_MIN_OPS>, VectorShapeCast,
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
index 710c74a70f9e7..b3ef23085c186 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index adcee1933e8a7..0ec7129a40a66 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include <algorithm>
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
index 8709a27e0168e..17bbe54ea6c0c 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_mlir_dialect_library(MLIRAMDGPUTransforms
   EmulateAtomics.cpp
   ResolveStridedMetadata.cpp
-  TransferReadToLoad.cpp
+  MaskedloadToLoad.cpp
 
   ADDITIONAL_HEADER_DIRS
   {$MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AMDGPU/Transforms
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
new file mode 100644
index 0000000000000..9a368f372c296
--- /dev/null
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -0,0 +1,167 @@
+//===- MaskedloadToLoad.cpp - Lowers maskedload to load -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
+
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/Support/MathExtras.h"
+
+namespace mlir::amdgpu {
+#define GEN_PASS_DEF_AMDGPUMASKEDLOADTOLOADPASS
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
+} // namespace mlir::amdgpu
+
+using namespace mlir;
+using namespace mlir::amdgpu;
+
+/// This pattern supports lowering of: `vector.maskedload` to `vector.load`
+/// and `arith.select` if the memref is in buffer address space.
+static LogicalResult baseInBufferAddrSpace(PatternRewriter &rewriter,
+                                           vector::MaskedLoadOp maskedOp) {
+  auto memRefType = dyn_cast<MemRefType>(maskedOp.getBase().getType());
+  if (!memRefType)
+    return rewriter.notifyMatchFailure(maskedOp, "not a memref source");
+
+  Attribute addrSpace = memRefType.getMemorySpace();
+  if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(addrSpace))
+    return rewriter.notifyMatchFailure(maskedOp, "no address space");
+
+  if (dyn_cast<amdgpu::AddressSpaceAttr>(addrSpace).getValue() !=
+      amdgpu::AddressSpace::FatRawBuffer)
+    return rewriter.notifyMatchFailure(maskedOp, "not in buffer address space");
+
+  return success();
+}
+
+static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
+                                           vector::MaskedLoadOp maskedOp) {
+  VectorType vectorType = maskedOp.getVectorType();
+  Value load = builder.create<vector::LoadOp>(
+      loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
+  Value res = builder.create<arith::SelectOp>(
+      loc, vectorType, maskedOp.getMask(), load, maskedOp.getPassThru());
+  return res;
+}
+
+static constexpr char kMaskedloadNeedsMask[] =
+    "amdgpu.buffer_maskedload_needs_mask";
+
+namespace {
+
+struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::MaskedLoadOp maskedOp,
+                                PatternRewriter &rewriter) const override {
+    if (maskedOp->hasAttr(kMaskedloadNeedsMask))
+      return failure();
+
+    if (failed(baseInBufferAddrSpace(rewriter, maskedOp))) {
+      return failure();
+    }
+
+    Location loc = maskedOp.getLoc();
+    Value src = maskedOp.getBase();
+
+    VectorType vectorType = maskedOp.getVectorType();
+    int64_t vectorSize = vectorType.getNumElements();
+    int64_t elementBitWidth = vectorType.getElementTypeBitWidth();
+    SmallVector<OpFoldResult> indices = maskedOp.getIndices();
+
+    auto stridedMetadata =
+        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+    SmallVector<OpFoldResult> strides =
+        stridedMetadata.getConstifiedMixedStrides();
+    SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
+    OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset();
+    memref::LinearizedMemRefInfo linearizedInfo;
+    OpFoldResult linearizedIndices;
+    std::tie(linearizedInfo, linearizedIndices) =
+        memref::getLinearizedMemRefOffsetAndSize(rewriter, loc, elementBitWidth,
+                                                 elementBitWidth, offset, sizes,
+                                                 strides, indices);
+
+    // delta = bufferSize - linearizedOffset
+    Value vectorSizeOffset =
+        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+    Value linearIndex =
+        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
+    Value totalSize = getValueOrCreateConstantIndexOp(
+        rewriter, loc, linearizedInfo.linearizedSize);
+    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+
+    // 1) check if delta < vectorSize
+    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
+
+    // 2) check if (detla % elements_per_word != 0)
+    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+        loc, llvm::divideCeil(32, elementBitWidth));
+    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::ne,
+        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
+        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+
+    // We take the fallback of maskedload default lowering only it is both
+    // out-of-bounds and not word aligned. The fallback ensures correct results
+    // when loading at the boundary of the buffer since buffer load returns
+    // inconsistent zeros for the whole word when boundary is crossed.
+    Value ifCondition =
+        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+
+    auto thenBuilder = [&](OpBuilder &builder, Location loc) {
+      Operation *read = builder.clone(*maskedOp.getOperation());
+      read->setAttr(kMaskedloadNeedsMask, builder.getUnitAttr());
+      Value readResult = read->getResult(0);
+      builder.create<scf::YieldOp>(loc, readResult);
+    };
+
+    auto elseBuilder = [&](OpBuilder &builder, Location loc) {
+      Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp);
+      rewriter.create<scf::YieldOp>(loc, res);
+    };
+
+    auto ifOp =
+        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+
+    rewriter.replaceOp(maskedOp, ifOp);
+
+    return success();
+  }
+};
+
+} // namespace
+
+void mlir::amdgpu::populateAmdgpuMaskedloadToLoadPatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<MaskedLoadLowering>(patterns.getContext(), benefit);
+}
+
+struct AmdgpuMaskedloadToLoadPass final
+    : amdgpu::impl::AmdgpuMaskedloadToLoadPassBase<AmdgpuMaskedloadToLoadPass> {
+  void runOnOperation() override {
+    RewritePatternSet patterns(&getContext());
+    populateAmdgpuMaskedloadToLoadPatterns(patterns);
+    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) {
+      return signalPassFailure();
+    }
+  }
+};
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
deleted file mode 100644
index f5b12a9524cc9..0000000000000
--- a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
+++ /dev/null
@@ -1,239 +0,0 @@
-//===- TransferReadToLoad.cpp - Lowers masked transfer read to load -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
-
-#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/TypeUtilities.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Support/LogicalResult.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/MathExtras.h"
-
-namespace mlir::amdgpu {
-#define GEN_PASS_DEF_AMDGPUTRANSFERREADTOLOADPASS
-#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
-} // namespace mlir::amdgpu
-
-using namespace mlir;
-using namespace mlir::amdgpu;
-
-/// This pattern supports lowering of:
-/// `vector.transfer_read` to a combination of `vector.load`, `arith.select` and
-/// `vector.broadcast` if all of the following hold:
-/// - The transfer op is masked.
-/// - The memref is in buffer address space.
-/// - Stride of most minor memref dimension must be 1.
-/// - Out-of-bounds masking is not required.
-/// - If the memref's element type is a vector type then it coincides with the
-///   result type.
-/// - The permutation map doesn't perform permutation (broadcasting is allowed).
-/// Note: those conditions mostly come from TransferReadToVectorLoadLowering
-/// pass.
-static LogicalResult transferPreconditions(
-    PatternRewriter &rewriter, VectorTransferOpInterface xferOp,
-    bool &requiresBroadcasting, VectorType &unbroadcastedVectorType) {
-  if (!xferOp.getMask())
-    return rewriter.notifyMatchFailure(xferOp, "Only support masked transfer");
-
-  // Permutations are handled by VectorToSCF or
-  // populateVectorTransferPermutationMapLoweringPatterns.
-  // We let the 0-d corner case pass-through as it is supported.
-  SmallVector<unsigned> broadcastedDims;
-  if (!xferOp.getPermutationMap().isMinorIdentityWithBroadcasting(
-          &broadcastedDims))
-    return rewriter.notifyMatchFailure(xferOp, "not minor identity + bcast");
-
-  auto memRefType = dyn_cast<MemRefType>(xferOp.getShapedType());
-  if (!memRefType)
-    return rewriter.notifyMatchFailure(xferOp, "not a memref source");
-
-  Attribute addrSpace = memRefType.getMemorySpace();
-  if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(addrSpace))
-    return rewriter.notifyMatchFailure(xferOp, "no address space");
-
-  if (dyn_cast<amdgpu::AddressSpaceAttr>(addrSpace).getValue() !=
-      amdgpu::AddressSpace::FatRawBuffer)
-    return rewriter.notifyMatchFailure(xferOp, "not in buffer address space");
-
-  // Non-unit strides are handled by VectorToSCF.
-  if (!memRefType.isLastDimUnitStride())
-    return rewriter.notifyMatchFailure(xferOp, "!= 1 stride needs VectorToSCF");
-
-  if (memRefType.getElementTypeBitWidth() < 8)
-    return rewriter.notifyMatchFailure(xferOp, "unsupported sub-byte type");
-
-  // If there is broadcasting involved then we first load the unbroadcasted
-  // vector, and then broadcast it with `vector.broadcast`.
-  ArrayRef<int64_t> vectorShape = xferOp.getVectorType().getShape();
-  SmallVector<int64_t> unbroadcastedVectorShape(vectorShape);
-  for (unsigned i : broadcastedDims)
-    unbroadcastedVectorShape[i] = 1;
-  unbroadcastedVectorType = xferOp.getVectorType().cloneWith(
-      unbroadcastedVectorShape, xferOp.getVectorType().getElementType());
-  requiresBroadcasting = !broadcastedDims.empty();
-
-  // `vector.load` supports vector types as memref's elements only when the
-  // resulting vector type is the same as the element type.
-  auto memrefElTy = memRefType.getElementType();
-  if (isa<VectorType>(memrefElTy) && memrefElTy != unbroadcastedVectorType)
-    return rewriter.notifyMatchFailure(xferOp, "incompatible element type");
-
-  // Otherwise, element types of the memref and the vector must match.
-  if (!isa<VectorType>(memrefElTy) &&
-      memrefElTy != xferOp.getVectorType().getElementType())
-    return rewriter.notifyMatchFailure(xferOp, "non-matching element type");
-
-  // Out-of-bounds dims are handled by MaterializeTransferMask.
-  if (xferOp.hasOutOfBoundsDim())
-    return rewriter.notifyMatchFailure(xferOp, "out-of-bounds needs mask");
-
-  if (xferOp.getVectorType().getRank() != 1)
-    // vector.maskedload operates on 1-D vectors.
-    return rewriter.notifyMatchFailure(
-        xferOp, "vector type is not rank 1, can't create masked load, needs "
-                "VectorToSCF");
-
-  return success();
-}
-
-static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
-                                           vector::TransferReadOp readOp,
-                                           bool requiresBroadcasting,
-                                           VectorType unbroadcastedVectorType) {
-  Value fill = builder.create<vector::SplatOp>(loc, unbroadcastedVectorType,
-                                               readOp.getPadding());
-  Value load = builder.create<vector::LoadOp>(
-      loc, unbroadcastedVectorType, readOp.getBase(), readOp.getIndices());
-  Value res = builder.create<arith::SelectOp>(loc, unbroadcastedVectorType,
-                                              readOp.getMask(), load, fill);
-  // Insert a broadcasting op if required.
-  if (requiresBroadcasting) {
-    res = builder.create<vector::BroadcastOp>(loc, readOp.getVectorType(), res);
-  }
-  return res;
-}
-
-static constexpr char kTransferReadNeedsMask[] =
-    "amdgpu.buffer_transfer_read_needs_mask";
-
-namespace {
-
-struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
-                                PatternRewriter &rewriter) const override {
-    if (readOp->hasAttr(kTransferReadNeedsMask))
-      return failure();
-
-    bool requiresBroadcasting = false;
-    VectorType unbroadcastedVectorType;
-    if (failed(transferPreconditions(rewriter, readOp, requiresBroadcasting,
-                                     unbroadcastedVectorType))) {
-      return failure();
-    }
-
-    Location loc = readOp.getLoc();
-    Value src = readOp.getBase();
-
-    VectorType vectorType = readOp.getVectorType();
-    int64_t vectorSize = vectorType.getNumElements();
-    int64_t elementBitWidth = vectorType.getElementTypeBitWidth();
-    SmallVector<OpFoldResult> indices = readOp.getIndices();
-
-    auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
-    SmallVector<OpFoldResult> strides =
-        stridedMetadata.getConstifiedMixedStrides();
-    SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
-    OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset();
-    memref::LinearizedMemRefInfo linearizedInfo;
-    OpFoldResult linearizedIndices;
-    std::tie(linearizedInfo, linearizedIndices) =
-        memref::getLinearizedMemRefOffsetAndSize(rewriter, loc, elementBitWidth,
-                                                 elementBitWidth, offset, sizes,
-                                                 strides, indices);
-
-    // delta = bufferSize - linearizedOffset
-    Value vectorSizeOffset =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
-    Value linearIndex =
-        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
-    Value totalSize = getValueOrCreateConstantIndexOp(
-        rewriter, loc, linearizedInfo.linearizedSize);
-    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
-
-    // 1) check if delta < vectorSize
-    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
-        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
-
-    // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
-        loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
-        loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
-
-    // We take the fallback of transfer_read default lowering only it is both
-    // out-of-bounds and not word aligned. The fallback ensures correct results
-    // when loading at the boundary of the buffer since buffer load returns
-    // inconsistent zeros for the whole word when boundary is crossed.
-    Value ifCondition =
-        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
-
-    auto thenBuilder = [&](OpBuilder &builder, Location loc) {
-      Operation *read = builder.clone(*readOp.getOperation());
-      read->setAttr(kTransferReadNeedsMask, builder.getUnitAttr());
-      Value readResult = read->getResult(0);
-      builder.create<scf::YieldOp>(loc, readResult);
-    };
-
-    auto elseBuilder = [&](OpBuilder &builder, Location loc) {
-      Value res = createVectorLoadForMaskedLoad(
-          builder, loc, readOp, requiresBroadcasting, unbroadcastedVectorType);
-      rewriter.create<scf::YieldOp>(loc, res);
-    };
-
-    auto ifOp =
-        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
-
-    rewriter.replaceOp(readOp, ifOp);
-
-    return success();
-  }
-};
-
-} // namespace
-
-void mlir::amdgpu::populateAmdgpuTransferReadToLoadPatterns(
-    RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<TransferReadLowering>(patterns.getContext(), benefit);
-}
-
-struct AmdgpuTransferReadToLoadPass final
-    : amdgpu::impl::AmdgpuTransferReadToLoadPassBase<
-          AmdgpuTransferReadToLoadPass> {
-  void runOnOperation() override {
-    RewritePatternSet patterns(&getContext());
-    populateAmdgpuTransferReadToLoadPatterns(patterns);
-    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) {
-      return signalPassFailure();
-    }
-  }
-};
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
index 6f79665c2bb60..a8067349344bf 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
@@ -391,7 +391,7 @@ static void addOrderingConstraints(const FlatAffineValueConstraints &srcDomain,
   unsigned numCommonLoops = getNumCommonLoops(srcDomain, dstDomain);
   unsigned numCommonLoopConstraints = std::min(numCommonLoops, loopDepth);
   for (unsigned i = 0; i < numCommonLoopConstraints; ++i) {
-    std::fill(eq.begin(), eq.end(), 0);
+    llvm::fill(eq, 0);
     eq[i] = -1;
     eq[i + numSrcDims] = 1;
     if (i == loopDepth - 1) {
@@ -433,7 +433,7 @@ static void computeDirectionVector(
   // Constraint variables format:
   // [num-common-loops][num-src-dim-ids][num-dst-dim-ids][num-symbols][constant]
   for (unsigned j = 0; j < numCommonLoops; ++j) {
-    std::fill(eq.begin(), eq.end(), 0);
+    llvm::fill(eq, 0);
     eq[j] = 1;
     eq[j + numCommonLoops] = 1;
     eq[j + numCommonLoops + numSrcDims] = -1;
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
index f344d59234e6a..86edc2bcc2761 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
@@ -12,17 +12,12 @@
 
 #include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
 #include "mlir/Analysis/Presburger/IntegerRelation.h"
-#include "mlir/Analysis/Presburger/LinearTransform.h"
-#include "mlir/Analysis/Presburger/Simplex.h"
 #include "mlir/Analysis/Presburger/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
-#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/Support/LLVM.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -526,7 +521,7 @@ LogicalResult mlir::affine::getRelationFromMap(AffineMap &map,
   SmallVector<int64_t, 8> eq(localVarCst.getNumCols());
   for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) {
     // Zero fill.
-    std::fill(eq.begin(), eq.end(), 0);
+    llvm::fill(eq, 0);
     // Fill equality.
     for (unsigned j = 0, f = oldDimNum; j < f; ++j)
       eq[j] = flatExprs[i][j];
diff --git a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
index 01cc500148385..2f85e0b5da043 100644
--- a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
@@ -20,13 +20,9 @@
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "llvm/Support/MathExtras.h"
 
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/Debug.h"
 #include <numeric>
 #include <optional>
-#include <type_traits>
 
 #define DEBUG_TYPE "affine-loop-analysis"
 
diff --git a/mlir/lib/Dialect/Affine/Analysis/NestedMatcher.cpp b/mlir/lib/Dialect/Affine/Analysis/NestedMatcher.cpp
index c158b98381a82..c0f9348693535 100644
--- a/mlir/lib/Dialect/Affine/Analysis/NestedMatcher.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/NestedMatcher.cpp
@@ -11,10 +11,8 @@
 #include "mlir/Dialect/Affine/Analysis/NestedMatcher.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Allocator.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 using namespace mlir::affine;
diff --git a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
index 3144ca723df90..8bdb4c3593335 100644
--- a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
@@ -18,12 +18,9 @@
 #include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/IntegerSet.h"
-#include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
@@ -1432,7 +1429,7 @@ LogicalResult mlir::affine::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
 
     // Check for a negative index.
     FlatAffineValueConstraints lcst(*region.getConstraints());
-    std::fill(ineq.begin(), ineq.end(), 0);
+    llvm::fill(ineq, 0);
     // d_i <= -1;
     lcst.addBound(BoundType::UB, r, -1);
     outOfBounds = !lcst.isEmpty();
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index bd4b2e56808b6..e8e8f624d806e 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -23,14 +23,12 @@
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/MathExtras.h"
-#include <limits>
 #include <numeric>
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index c11f1bca5d49d..f18cec5a14fae 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -28,9 +28,7 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
 #include <algorithm>
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
index c0ef28c648ac5..5430bdc4ff858 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineExpandIndexOps.cpp
@@ -15,8 +15,6 @@
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
-#include "mlir/Dialect/Affine/Utils.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
index e3f316443161f..27cb0cae18337 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
@@ -15,8 +15,6 @@
 #include "mlir/Dialect/Affine/Analysis/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace affine {
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
index 16c0d3d8be3dc..a0dcc20a4b763 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/Affine/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/Dominance.h"
-#include <algorithm>
 
 namespace mlir {
 namespace affine {
diff --git a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
index f28fb3acb7db7..4fd0cf9b3cd25 100644
--- a/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/DecomposeAffineOps.cpp
@@ -14,7 +14,6 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/Support/Debug.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp
index 05c77070a70c1..c3575cb7d5896 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp
@@ -10,13 +10,9 @@
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Utils/Utils.h"
-#include "mlir/Transforms/Passes.h"
-#include "mlir/Transforms/RegionUtils.h"
-#include "llvm/Support/Debug.h"
 
 namespace mlir {
 namespace affine {
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 4b4eb9ce37b4c..95848d0b67547 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -22,9 +22,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -445,10 +443,15 @@ static Value createPrivateMemRef(AffineForOp forOp,
   // Replace all users of 'oldMemRef' with 'newMemRef'.
   Operation *domFilter =
       getDominanceFilterForPrivateMemRefRepl(sliceInsertionBlock, storeOps);
+  auto userFilterFn = [&](Operation *user) {
+    auto domInfo = std::make_unique<DominanceInfo>(
+        domFilter->getParentOfType<FunctionOpInterface>());
+    return domInfo->dominates(domFilter, user);
+  };
   LogicalResult res = replaceAllMemRefUsesWith(
       oldMemRef, newMemRef, /*extraIndices=*/{}, indexRemap,
       /*extraOperands=*/outerIVs,
-      /*symbolOperands=*/{}, domFilter);
+      /*symbolOperands=*/{}, userFilterFn);
   assert(succeeded(res) &&
          "replaceAllMemrefUsesWith should always succeed here");
   (void)res;
@@ -997,6 +1000,8 @@ struct GreedyFusion {
           if (producerConsumerMemrefs.count(
                   cast<AffineWriteOpInterface>(op).getMemRef()))
             dstMemrefOps.push_back(op);
+        if (dstMemrefOps.empty())
+          continue;
         unsigned dstLoopDepthTest =
             getInnermostCommonLoopDepth(dstMemrefOps) - numSurroundingLoops;
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index 45bda68bb8639..8a989ff985c6f 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -17,12 +17,9 @@
 #include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
 #include "mlir/Dialect/Affine/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/IRMapping.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include <optional>
@@ -121,7 +118,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   // If the cache size is zero, set the minimum valid tile size. No good reason
   // to pick another specific size over this.
   if (cacheSizeInKiB == 0) {
-    std::fill(tileSizes->begin(), tileSizes->end(), 1);
+    llvm::fill(*tileSizes, 1);
     return;
   }
 
@@ -136,8 +133,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   std::optional<int64_t> fp = getMemoryFootprintBytes(band[0], 0);
   if (!fp) {
     // Fill with default tile sizes if footprint is unknown.
-    std::fill(tileSizes->begin(), tileSizes->end(),
-              LoopTiling::kDefaultTileSize);
+    llvm::fill(*tileSizes, LoopTiling::kDefaultTileSize);
     if (avoidMaxMinBounds)
       adjustToDivisorsOfTripCounts(band, tileSizes);
     LLVM_DEBUG(
@@ -151,7 +147,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   uint64_t excessFactor = llvm::divideCeil(*fp, cacheSizeBytes);
   if (excessFactor <= 1) {
     // No need of any tiling - set tile size to 1.
-    std::fill(tileSizes->begin(), tileSizes->end(), 1);
+    llvm::fill(*tileSizes, 1);
     return;
   }
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
index 13640f085951e..6157e8131f890 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
@@ -39,12 +39,6 @@
 #include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/IRMapping.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/CommandLine.h"
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 4be99aa197380..05a352f39a93c 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -18,11 +18,9 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
 #include "mlir/Dialect/Affine/Utils.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/Debug.h"
 
@@ -115,13 +113,16 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
   // non-dereferencing uses of the memref (dealloc's are fine though).
-  if (failed(replaceAllMemRefUsesWith(
-          oldMemRef, newMemRef,
-          /*extraIndices=*/{ivModTwoOp},
-          /*indexRemap=*/AffineMap(),
-          /*extraOperands=*/{},
-          /*symbolOperands=*/{},
-          /*domOpFilter=*/&*forOp.getBody()->begin()))) {
+  auto userFilterFn = [&](Operation *user) {
+    auto domInfo = std::make_unique<DominanceInfo>(
+        forOp->getParentOfType<FunctionOpInterface>());
+    return domInfo->dominates(&*forOp.getBody()->begin(), user);
+  };
+  if (failed(replaceAllMemRefUsesWith(oldMemRef, newMemRef,
+                                      /*extraIndices=*/{ivModTwoOp},
+                                      /*indexRemap=*/AffineMap(),
+                                      /*extraOperands=*/{},
+                                      /*symbolOperands=*/{}, userFilterFn))) {
     LLVM_DEBUG(
         forOp.emitError("memref replacement for double buffering failed"));
     ivModTwoOp.erase();
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp
index 2cb78d2b94c63..8493b604ae743 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp
@@ -17,7 +17,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/IntEqClasses.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index f6f192a6d964a..7fae260767e0a 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -1257,7 +1257,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
   auto transfer = state.builder.create<vector::TransferReadOp>(
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices, permutationMap);
+      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
+      /*padding=*/std::nullopt, permutationMap);
 
   // Register replacement for future uses in the scope.
   state.registerOpVectorReplacement(loadOp, transfer);
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 4aa1fe318efa8..0501616ad912c 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -1967,6 +1967,12 @@ static LogicalResult generateCopy(
   if (begin == end)
     return success();
 
+  // Record the last op in the block for which we are performing copy
+  // generation. We later do the memref replacement only in [begin, lastCopyOp]
+  // so that the original memref's used in the data movement code themselves
+  // don't get replaced.
+  Operation *lastCopyOp = end->getPrevNode();
+
   // Is the copy out point at the end of the block where we are doing
   // explicit copying.
   bool isCopyOutAtEndOfBlock = (end == copyOutPlacementStart);
@@ -2143,12 +2149,6 @@ static LogicalResult generateCopy(
     }
   }
 
-  // Record the last operation where we want the memref replacement to end. We
-  // later do the memref replacement only in [begin, postDomFilter] so
-  // that the original memref's used in the data movement code themselves don't
-  // get replaced.
-  auto postDomFilter = std::prev(end);
-
   // Create fully composed affine maps for each memref.
   auto memAffineMap = b.getMultiDimIdentityMap(memIndices.size());
   fullyComposeAffineMapAndOperands(&memAffineMap, &memIndices);
@@ -2244,13 +2244,17 @@ static LogicalResult generateCopy(
   if (!isBeginAtStartOfBlock)
     prevOfBegin = std::prev(begin);
 
+  auto userFilterFn = [&](Operation *user) {
+    auto *ancestorUser = block->findAncestorOpInBlock(*user);
+    return ancestorUser && !ancestorUser->isBeforeInBlock(&*begin) &&
+           !lastCopyOp->isBeforeInBlock(ancestorUser);
+  };
+
   // *Only* those uses within the range [begin, end) of 'block' are replaced.
   (void)replaceAllMemRefUsesWith(memref, fastMemRef,
                                  /*extraIndices=*/{}, indexRemap,
                                  /*extraOperands=*/regionSymbols,
-                                 /*symbolOperands=*/{},
-                                 /*domOpFilter=*/&*begin,
-                                 /*postDomOpFilter=*/&*postDomFilter);
+                                 /*symbolOperands=*/{}, userFilterFn);
 
   *nBegin = isBeginAtStartOfBlock ? block->begin() : std::next(prevOfBegin);
 
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index cde8223107859..83816a1543708 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -27,7 +27,6 @@
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/LogicalResult.h"
 #include <optional>
 
 #define DEBUG_TYPE "affine-utils"
@@ -1305,9 +1304,9 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
 LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices,
     AffineMap indexRemap, ArrayRef<Value> extraOperands,
-    ArrayRef<Value> symbolOperands, Operation *domOpFilter,
-    Operation *postDomOpFilter, bool allowNonDereferencingOps,
-    bool replaceInDeallocOp) {
+    ArrayRef<Value> symbolOperands,
+    llvm::function_ref<bool(Operation *)> userFilterFn,
+    bool allowNonDereferencingOps, bool replaceInDeallocOp) {
   unsigned newMemRefRank = cast<MemRefType>(newMemRef.getType()).getRank();
   (void)newMemRefRank; // unused in opt mode
   unsigned oldMemRefRank = cast<MemRefType>(oldMemRef.getType()).getRank();
@@ -1328,61 +1327,52 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
 
   std::unique_ptr<DominanceInfo> domInfo;
   std::unique_ptr<PostDominanceInfo> postDomInfo;
-  if (domOpFilter)
-    domInfo = std::make_unique<DominanceInfo>(
-        domOpFilter->getParentOfType<FunctionOpInterface>());
-
-  if (postDomOpFilter)
-    postDomInfo = std::make_unique<PostDominanceInfo>(
-        postDomOpFilter->getParentOfType<FunctionOpInterface>());
 
   // Walk all uses of old memref; collect ops to perform replacement. We use a
   // DenseSet since an operation could potentially have multiple uses of a
   // memref (although rare), and the replacement later is going to erase ops.
   DenseSet<Operation *> opsToReplace;
-  for (auto *op : oldMemRef.getUsers()) {
-    // Skip this use if it's not dominated by domOpFilter.
-    if (domOpFilter && !domInfo->dominates(domOpFilter, op))
-      continue;
-
-    // Skip this use if it's not post-dominated by postDomOpFilter.
-    if (postDomOpFilter && !postDomInfo->postDominates(postDomOpFilter, op))
+  for (auto *user : oldMemRef.getUsers()) {
+    // Check if this user doesn't pass the filter.
+    if (userFilterFn && !userFilterFn(user))
       continue;
 
     // Skip dealloc's - no replacement is necessary, and a memref replacement
     // at other uses doesn't hurt these dealloc's.
-    if (hasSingleEffect<MemoryEffects::Free>(op, oldMemRef) &&
+    if (hasSingleEffect<MemoryEffects::Free>(user, oldMemRef) &&
         !replaceInDeallocOp)
       continue;
 
     // Check if the memref was used in a non-dereferencing context. It is fine
     // for the memref to be used in a non-dereferencing way outside of the
     // region where this replacement is happening.
-    if (!isa<AffineMapAccessInterface>(*op)) {
+    if (!isa<AffineMapAccessInterface>(*user)) {
       if (!allowNonDereferencingOps) {
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Memref replacement failed: non-deferencing memref op: \n"
-                   << *op << '\n');
+        LLVM_DEBUG(
+            llvm::dbgs()
+            << "Memref replacement failed: non-deferencing memref user: \n"
+            << *user << '\n');
         return failure();
       }
       // Non-dereferencing ops with the MemRefsNormalizable trait are
       // supported for replacement.
-      if (!op->hasTrait<OpTrait::MemRefsNormalizable>()) {
+      if (!user->hasTrait<OpTrait::MemRefsNormalizable>()) {
         LLVM_DEBUG(llvm::dbgs() << "Memref replacement failed: use without a "
                                    "memrefs normalizable trait: \n"
-                                << *op << '\n');
+                                << *user << '\n');
         return failure();
       }
     }
 
-    // We'll first collect and then replace --- since replacement erases the op
-    // that has the use, and that op could be postDomFilter or domFilter itself!
-    opsToReplace.insert(op);
+    // We'll first collect and then replace --- since replacement erases the
+    // user that has the use, and that user could be postDomFilter or domFilter
+    // itself!
+    opsToReplace.insert(user);
   }
 
-  for (auto *op : opsToReplace) {
+  for (auto *user : opsToReplace) {
     if (failed(replaceAllMemRefUsesWith(
-            oldMemRef, newMemRef, op, extraIndices, indexRemap, extraOperands,
+            oldMemRef, newMemRef, user, extraIndices, indexRemap, extraOperands,
             symbolOperands, allowNonDereferencingOps)))
       llvm_unreachable("memref replacement guaranteed to succeed here");
   }
@@ -1763,8 +1753,7 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
                                       /*indexRemap=*/layoutMap,
                                       /*extraOperands=*/{},
                                       /*symbolOperands=*/symbolOperands,
-                                      /*domOpFilter=*/nullptr,
-                                      /*postDomOpFilter=*/nullptr,
+                                      /*userFilterFn=*/nullptr,
                                       /*allowNonDereferencingOps=*/true))) {
     // If it failed (due to escapes for example), bail out.
     newAlloc.erase();
@@ -1854,8 +1843,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
                                       /*indexRemap=*/oldLayoutMap,
                                       /*extraOperands=*/{},
                                       /*symbolOperands=*/oldStrides,
-                                      /*domOpFilter=*/nullptr,
-                                      /*postDomOpFilter=*/nullptr,
+                                      /*userFilterFn=*/nullptr,
                                       /*allowNonDereferencingOps=*/true))) {
     // If it failed (due to escapes for example), bail out.
     newReinterpretCast.erase();
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 5194f2b58669a..ae2a00cedf6f5 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -262,6 +262,12 @@ void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIntegerAttr(type, value));
 }
 
+void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
+                                 Type type, const APInt &value) {
+  arith::ConstantOp::build(builder, result, type,
+                           builder.getIntegerAttr(type, value));
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -292,6 +298,16 @@ bool arith::ConstantIndexOp::classof(Operation *op) {
   return false;
 }
 
+Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
+                                   Type type) {
+  // TODO: Incorporate this check to `FloatAttr::get*`.
+  assert(!isa<Float8E8M0FNUType>(getElementTypeOrSelf(type)) &&
+         "type doesn't have a zero representation");
+  TypedAttr zeroAttr = builder.getZeroAttr(type);
+  assert(zeroAttr && "unsupported type for zero attribute");
+  return builder.create<arith::ConstantOp>(loc, zeroAttr);
+}
+
 //===----------------------------------------------------------------------===//
 // AddIOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index 85d1b5ac73bf4..afee162053bea 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -181,7 +181,7 @@ struct SelectOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -196,17 +196,17 @@ struct SelectOpInterface
     if (failed(trueType) || failed(falseType))
       return failure();
     if (*trueType == *falseType)
-      return *trueType;
+      return cast<BufferLikeType>(*trueType);
     if (trueType->getMemorySpace() != falseType->getMemorySpace())
       return op->emitError("inconsistent memory space on true/false operands");
 
     // If the buffers have different types, they differ only in their layout
     // map.
     auto memrefType = llvm::cast<MemRefType>(*trueType);
-    return getMemRefTypeWithFullyDynamicLayout(
+    return cast<BufferLikeType>(getMemRefTypeWithFullyDynamicLayout(
         RankedTensorType::get(memrefType.getShape(),
                               memrefType.getElementType()),
-        memrefType.getMemorySpace());
+        memrefType.getMemorySpace()));
   }
 };
 
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index d52ff4d4257c7..3dbb93b8a0669 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -426,6 +426,7 @@ struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
     // Create the new `transfer_read`.
     auto newReadOp = rewriter.create<vector::TransferReadOp>(
         readOp.getLoc(), collapsedVT, collapsedMem, indices,
+        readOp.getPadding(),
         ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
 
     // Cast back to the original vector type.
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index c289faaf9983c..8601bb5aaada9 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -723,8 +723,8 @@ class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
       // Switch the coroutine completion token to available state.
       rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
 
-    rewriter.eraseOp(op);
     rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    rewriter.eraseOp(op);
 
     return success();
   }
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index ca914df8b7890..2bf326a07783e 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -10,7 +10,6 @@
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/AsmState.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp
index ea726a4bfc3fb..aeedeb3ecd2a5 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h"
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 
 namespace mlir {
 namespace bufferization {
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 5e69fa386ecbc..8f17a82fabe03 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -947,7 +947,7 @@ AliasingOpOperandList bufferization::detail::defaultGetAliasingOpOperands(
   return AliasingOpOperandList(std::move(result));
 }
 
-FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
+FailureOr<BufferLikeType> bufferization::detail::defaultGetBufferType(
     Value value, const BufferizationOptions &options,
     const BufferizationState &bufferizationState,
     SmallVector<Value> &invocationStack) {
@@ -955,8 +955,10 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
   auto tensorType = cast<TensorType>(value.getType());
 
   // No further analysis is possible for a block argument.
-  if (llvm::isa<BlockArgument>(value))
-    return bufferization::getMemRefType(tensorType, options);
+  if (llvm::isa<BlockArgument>(value)) {
+    return cast<BufferLikeType>(
+        bufferization::getMemRefType(tensorType, options));
+  }
 
   // Value is an OpResult.
   Operation *op = getOwnerOfValue(value);
@@ -968,8 +970,8 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
     // If the OpResult has an equivalent OpOperand, both OpResult and
     // OpOperand bufferize to the exact same buffer type.
     Value equivalentOperand = aliases.getAliases().front().opOperand->get();
-    return asMemRefType(getBufferType(equivalentOperand, options,
-                                      bufferizationState, invocationStack));
+    return getBufferType(equivalentOperand, options, bufferizationState,
+                         invocationStack);
   }
 
   // If we do not know the memory space and there is no default memory space,
@@ -979,7 +981,8 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
   if (!memSpace.has_value())
     return op->emitError("could not infer memory space");
 
-  return getMemRefType(tensorType, options, /*layout=*/{}, *memSpace);
+  return cast<BufferLikeType>(
+      getMemRefType(tensorType, options, /*layout=*/{}, *memSpace));
 }
 
 bool bufferization::detail::defaultIsRepetitiveRegion(
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 9bd87d66c7d36..875a06546c9f0 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/Matchers.h"
 #include <optional>
@@ -44,7 +43,7 @@ FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
         failed(target.getStridesAndOffset(targetStrides, targetOffset)))
       return false;
     auto dynamicToStatic = [](int64_t a, int64_t b) {
-      return ShapedType::isDynamic(a) && !ShapedType::isDynamic(b);
+      return ShapedType::isDynamic(a) && ShapedType::isStatic(b);
     };
     if (dynamicToStatic(sourceOffset, targetOffset))
       return false;
@@ -222,7 +221,7 @@ AliasingValueList AllocTensorOp::getAliasingValues(OpOperand &opOperand,
   return {};
 }
 
-FailureOr<BaseMemRefType>
+FailureOr<BufferLikeType>
 AllocTensorOp::getBufferType(Value value, const BufferizationOptions &options,
                              const BufferizationState &state,
                              SmallVector<Value> &invocationStack) {
@@ -245,7 +244,8 @@ AllocTensorOp::getBufferType(Value value, const BufferizationOptions &options,
     return getOperation()->emitError("could not infer memory space");
   }
 
-  return getMemRefTypeWithStaticIdentityLayout(getType(), memorySpace);
+  return cast<BufferLikeType>(
+      getMemRefTypeWithStaticIdentityLayout(getType(), memorySpace));
 }
 
 LogicalResult AllocTensorOp::verify() {
diff --git a/mlir/lib/Dialect/Bufferization/Pipelines/BufferizationPipelines.cpp b/mlir/lib/Dialect/Bufferization/Pipelines/BufferizationPipelines.cpp
index d82e3a46dbff2..51feec79e2174 100644
--- a/mlir/lib/Dialect/Bufferization/Pipelines/BufferizationPipelines.cpp
+++ b/mlir/lib/Dialect/Bufferization/Pipelines/BufferizationPipelines.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Dialect/Bufferization/Pipelines/Passes.h"
 
 #include "mlir/Dialect/Bufferization/Transforms/Passes.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/Passes.h"
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index acf5f7767d12a..6924e88c50e1b 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -12,7 +12,6 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 namespace bufferization {
@@ -33,7 +32,7 @@ static bool hasFullyDynamicLayoutMap(MemRefType type) {
     return false;
   if (!llvm::all_of(strides, ShapedType::isDynamic))
     return false;
-  if (!ShapedType::isDynamic(offset))
+  if (ShapedType::isStatic(offset))
     return false;
   return true;
 }
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
index ff2c83d228dbb..a66be7d3b2f52 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferUtils.cpp
@@ -17,11 +17,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/Interfaces/ControlFlowInterfaces.h"
-#include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Pass/Pass.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallString.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
index 72f47b8b468ea..8f983ab1eae36 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
@@ -14,7 +14,6 @@
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 #include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SetVector.h"
 
 using namespace mlir;
 using namespace mlir::bufferization;
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 6472ef3eff2ac..246555dc8c699 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -14,14 +14,12 @@
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 #include "mlir/Dialect/Bufferization/Transforms/OneShotModuleBufferize.h"
 #include "mlir/Dialect/Bufferization/Transforms/Transforms.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Pass/PassManager.h"
-#include "mlir/Transforms/Passes.h"
 #include <optional>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
index ae011904cb972..c10d290f5d0ed 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/DropEquivalentBufferResults.cpp
@@ -31,8 +31,6 @@
 
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/IR/Operation.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 namespace bufferization {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
index 6f27563a45548..b7db2e847a335 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorElimination.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 namespace bufferization {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorToAllocTensor.cpp b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorToAllocTensor.cpp
index 22215836986c1..baf55e1fdcddb 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorToAllocTensor.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/EmptyTensorToAllocTensor.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/Transforms.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 453ed43bcadd2..2a98203da9d7d 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Operation.h"
 #include <optional>
@@ -211,7 +210,7 @@ struct CallOpInterface
     return result;
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -229,12 +228,13 @@ struct CallOpInterface
     Type resultType =
         funcType.getResult(cast<OpResult>(value).getResultNumber());
     if (auto bufferizedType = dyn_cast<BaseMemRefType>(resultType))
-      return bufferizedType;
+      return cast<BufferLikeType>(bufferizedType);
 
     // Otherwise, call the type converter to compute the bufferized type.
     auto tensorType = cast<TensorType>(resultType);
-    return options.functionArgTypeConverterFn(
-        tensorType, *options.defaultMemorySpaceFn(tensorType), funcOp, options);
+    return cast<BufferLikeType>(options.functionArgTypeConverterFn(
+        tensorType, *options.defaultMemorySpaceFn(tensorType), funcOp,
+        options));
   }
 
   /// All function arguments are writable. It is the responsibility of the
@@ -396,7 +396,7 @@ struct FuncOpInterface
     return getAliasingBranchOpOperands(op, cast<BlockArgument>(value), state);
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -405,8 +405,8 @@ struct FuncOpInterface
 
     // Function arguments are special.
     if (bbArg.getOwner() == &funcOp.getBody().front())
-      return getBufferizedFunctionArgType(funcOp, bbArg.getArgNumber(),
-                                          options);
+      return cast<BufferLikeType>(
+          getBufferizedFunctionArgType(funcOp, bbArg.getArgNumber(), options));
 
     return OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel::
         getBufferType(op, value, options, state, invocationStack);
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
index 2a17ae4f6a249..a611126e320ad 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/LowerDeallocations.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
index 33a922d59224b..a8e8353ca6191 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
@@ -40,14 +40,12 @@
 
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 
-#include <optional>
 #include <random>
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
 #include "mlir/Dialect/Bufferization/Transforms/Transforms.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/AsmState.h"
 #include "mlir/IR/Dominance.h"
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OptimizeAllocationLiveness.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OptimizeAllocationLiveness.cpp
index e17b39cd7e371..605a48709fe84 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OptimizeAllocationLiveness.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OptimizeAllocationLiveness.cpp
@@ -15,12 +15,10 @@
 #include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h"
 #include "mlir/Dialect/Bufferization/Transforms/Passes.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
 
 #define DEBUG_TYPE "optimize-allocation-liveness"
 #define DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index c8c53374d676b..4862d1f722785 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -20,6 +20,7 @@ add_mlir_dialect_library(MLIRGPUDialect
   MLIRFunctionInterfaces
   MLIRInferIntRangeInterface
   MLIRIR
+  MLIRMathDialect
   MLIRMemRefDialect
   MLIRSideEffectInterfaces
   MLIRSupport
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 1d77d3d5735e1..30b5ac9809139 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -14,6 +14,7 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h"
+#include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
@@ -106,6 +107,64 @@ int64_t GPUThreadMappingAttr::getRelativeIndex() const {
              : getMappingId();
 }
 
+int64_t GPULaneMappingAttr::getMappingId() const {
+  return static_cast<int64_t>(getLane());
+}
+
+bool GPULaneMappingAttr::isLinearMapping() const {
+  return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
+}
+
+int64_t GPULaneMappingAttr::getRelativeIndex() const {
+  return isLinearMapping()
+             ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
+             : getMappingId();
+}
+
+int64_t GPUMappingMaskAttr::getMaxNumPhysicalIds() const { return 64; }
+
+///                 8       4       0
+/// Example mask  : 0 0 0 1 1 0 1 0 0
+///
+/// Active physical (resp. logical) is  2 (0), 4 (1) and 5 (2).
+/// Logical id for e.g. 5 (2) constructs filter (1 << 5 - 1).
+///
+/// Example mask  : 0 0 0 1 1 0 1 0 0
+/// Example filter: 0 0 0 0 1 1 1 1 1
+/// Intersection  : 0 0 0 0 1 0 1 0 0
+/// PopCnt        : 2
+Value GPUMappingMaskAttr::createLogicalLinearMappingId(
+    OpBuilder &b, Value physicalLinearMappingId) const {
+  Location loc = physicalLinearMappingId.getLoc();
+  Value mask = b.create<arith::ConstantOp>(loc, b.getI64IntegerAttr(getMask()));
+  Value one = b.create<arith::ConstantOp>(loc, b.getI64IntegerAttr(1));
+  Value filter = b.create<arith::ShLIOp>(loc, one, physicalLinearMappingId);
+  filter = b.create<arith::SubIOp>(loc, filter, one);
+  Value filteredId = b.create<arith::AndIOp>(loc, mask, filter);
+  return b.create<math::CtPopOp>(loc, filteredId);
+}
+
+///                 8       4       0
+/// Example mask  : 0 0 0 1 1 0 1 0 0
+///
+/// Active physical (resp. logical) is  2 (0), 4 (1) and 5 (2).
+/// Logical id for e.g. 5 (2) constructs filter (1 << 5).
+///
+/// Example mask  : 0 0 0 1 1 0 1 0 0
+/// Example filter: 0 0 0 1 0 0 0 0 0
+/// Intersection  : 0 0 0 1 0 0 0 0 0
+/// Cmp           : 1
+Value GPUMappingMaskAttr::createIsActiveIdPredicate(
+    OpBuilder &b, Value physicalLinearMappingId) const {
+  Location loc = physicalLinearMappingId.getLoc();
+  Value mask = b.create<arith::ConstantOp>(loc, b.getI64IntegerAttr(getMask()));
+  Value one = b.create<arith::ConstantOp>(loc, b.getI64IntegerAttr(1));
+  Value filter = b.create<arith::ShLIOp>(loc, one, physicalLinearMappingId);
+  Value filtered = b.create<arith::AndIOp>(loc, mask, filter);
+  Value zero = b.create<arith::ConstantOp>(loc, b.getI64IntegerAttr(0));
+  return b.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne, filtered, zero);
+}
+
 int64_t GPUMemorySpaceMappingAttr::getMappingId() const {
   return static_cast<int64_t>(getAddressSpace());
 }
@@ -1147,8 +1206,7 @@ void LaunchFuncOp::build(OpBuilder &builder, OperationState &result,
   prop.kernel = kernelSymbol;
   size_t segmentSizesLen = std::size(prop.operandSegmentSizes);
   // Initialize the segment sizes to 1.
-  for (auto &sz : prop.operandSegmentSizes)
-    sz = 1;
+  llvm::fill(prop.operandSegmentSizes, 1);
   prop.operandSegmentSizes[0] = asyncDependencies.size();
   if (!clusterSize.has_value()) {
     prop.operandSegmentSizes[segmentSizesLen - 4] = 0;
@@ -1196,8 +1254,7 @@ void LaunchFuncOp::build(OpBuilder &builder, OperationState &result,
   prop.kernel = kernel;
   size_t segmentSizesLen = std::size(prop.operandSegmentSizes);
   // Initialize the segment sizes to 1.
-  for (auto &sz : prop.operandSegmentSizes)
-    sz = 1;
+  llvm::fill(prop.operandSegmentSizes, 1);
   prop.operandSegmentSizes[0] = 0;
   if (!clusterSize.has_value()) {
     prop.operandSegmentSizes[segmentSizesLen - 4] = 0;
@@ -1331,6 +1388,49 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
         mode);
 }
 
+//===----------------------------------------------------------------------===//
+// RotateOp
+//===----------------------------------------------------------------------===//
+
+void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
+                     int32_t offset, int32_t width) {
+  build(builder, result, value,
+        builder.create<arith::ConstantOp>(result.location,
+                                          builder.getI32IntegerAttr(offset)),
+        builder.create<arith::ConstantOp>(result.location,
+                                          builder.getI32IntegerAttr(width)));
+}
+
+LogicalResult RotateOp::verify() {
+  auto offsetConstOp = getOffset().getDefiningOp<arith::ConstantOp>();
+  if (!offsetConstOp)
+    return emitOpError() << "offset is not a constant value";
+
+  auto offsetIntAttr =
+      llvm::dyn_cast<mlir::IntegerAttr>(offsetConstOp.getValue());
+
+  auto widthConstOp = getWidth().getDefiningOp<arith::ConstantOp>();
+  if (!widthConstOp)
+    return emitOpError() << "width is not a constant value";
+
+  auto widthIntAttr =
+      llvm::dyn_cast<mlir::IntegerAttr>(widthConstOp.getValue());
+
+  llvm::APInt offsetValue = offsetIntAttr.getValue();
+  llvm::APInt widthValue = widthIntAttr.getValue();
+
+  if (!widthValue.isPowerOf2())
+    return emitOpError() << "width must be a power of two";
+
+  if (offsetValue.sge(widthValue) || offsetValue.slt(0)) {
+    int64_t widthValueInt = widthValue.getSExtValue();
+    return emitOpError() << "offset must be in the range [0, " << widthValueInt
+                         << ")";
+  }
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // BarrierOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/GPU/IR/InferIntRangeInterfaceImpls.cpp b/mlir/lib/Dialect/GPU/IR/InferIntRangeInterfaceImpls.cpp
index f5e30a278f06b..bee3f392c91b5 100644
--- a/mlir/lib/Dialect/GPU/IR/InferIntRangeInterfaceImpls.cpp
+++ b/mlir/lib/Dialect/GPU/IR/InferIntRangeInterfaceImpls.cpp
@@ -10,9 +10,7 @@
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/InferIntRangeInterface.h"
-#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp b/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp
index 78ff31a75ca4c..2c3e4661d266a 100644
--- a/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp
+++ b/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp
@@ -18,19 +18,14 @@
 #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
 #include "mlir/Conversion/IndexToLLVM/IndexToLLVM.h"
 #include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
-#include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
 #include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h"
 #include "mlir/Conversion/NVVMToLLVM/NVVMToLLVM.h"
 #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
 #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
-#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h"
 #include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Pipelines/Passes.h"
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Pass/PassOptions.h"
diff --git a/mlir/lib/Dialect/GPU/TransformOps/CMakeLists.txt b/mlir/lib/Dialect/GPU/TransformOps/CMakeLists.txt
index b26788f675ce5..e5cc0254f1ffe 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/TransformOps/CMakeLists.txt
@@ -24,4 +24,5 @@ add_mlir_dialect_library(MLIRGPUTransformOps
   # ConversionPatterns
   MLIRNVGPUToNVVM
   MLIRGPUToNVVMTransforms
+  MLIRGPUToROCDLTransforms
   )  
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 6446235c06fb2..c9e91535df946 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -10,15 +10,15 @@
 
 #include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
+#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/TransformOps/Utils.h"
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
+#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
@@ -42,6 +42,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/InterleavedRange.h"
+#include "llvm/Support/LogicalResult.h"
 #include <type_traits>
 
 using namespace mlir;
@@ -129,6 +130,41 @@ LogicalResult transform::ApplyGPUSubgroupReduceToNVVMConversionPatternsOp::
   return success();
 }
 
+void transform::ApplyGPUToROCDLConversionPatternsOp::populatePatterns(
+    TypeConverter &typeConverter, RewritePatternSet &patterns) {
+  auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter);
+  populateGpuMemorySpaceAttributeConversions(
+      llvmTypeConverter, [](AddressSpace space) {
+        switch (space) {
+        case AddressSpace::Global:
+          return ROCDL::ROCDLDialect::kGlobalMemoryAddressSpace;
+        case AddressSpace::Workgroup:
+          return ROCDL::ROCDLDialect::kSharedMemoryAddressSpace;
+        case AddressSpace::Private:
+          return ROCDL::ROCDLDialect::kPrivateMemoryAddressSpace;
+        }
+        llvm_unreachable("unknown address space enum value");
+      });
+  FailureOr<amdgpu::Chipset> maybeChipset =
+      amdgpu::Chipset::parse(getChipset());
+  assert(llvm::succeeded(maybeChipset) && "expected valid chipset");
+  populateGpuToROCDLConversionPatterns(
+      llvmTypeConverter, patterns, mlir::gpu::amd::Runtime::HIP, *maybeChipset);
+}
+
+LogicalResult
+transform::ApplyGPUToROCDLConversionPatternsOp::verifyTypeConverter(
+    transform::TypeConverterBuilderOpInterface builder) {
+  FailureOr<amdgpu::Chipset> maybeChipset =
+      amdgpu::Chipset::parse(getChipset());
+  if (failed(maybeChipset)) {
+    return emitOpError("Invalid chipset name: " + getChipset());
+  }
+  if (builder.getTypeConverterType() != "LLVMTypeConverter")
+    return emitOpError("expected LLVMTypeConverter");
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Apply...PatternsOp
 //===----------------------------------------------------------------------===//s
@@ -313,11 +349,14 @@ checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp,
                                      llvm::IsaPred<GPUWarpMappingAttr>);
   bool hasThreadMapping = llvm::any_of(forallOp.getMapping().value(),
                                        llvm::IsaPred<GPUThreadMappingAttr>);
+  bool hasLaneMapping = llvm::any_of(forallOp.getMapping().value(),
+                                     llvm::IsaPred<GPULaneMappingAttr>);
   int64_t countMappingTypes = 0;
   countMappingTypes += hasBlockMapping ? 1 : 0;
   countMappingTypes += hasWarpgroupMapping ? 1 : 0;
   countMappingTypes += hasWarpMapping ? 1 : 0;
   countMappingTypes += hasThreadMapping ? 1 : 0;
+  countMappingTypes += hasLaneMapping ? 1 : 0;
   if (countMappingTypes > 1) {
     return definiteFailureHelper(
         transformOp, forallOp,
@@ -330,7 +369,8 @@ checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp,
         "scf.forall op requires a mapping attribute of kind 'block'");
   }
   if (std::is_same<MappingKindType, ThreadMappingKind>::value &&
-      !hasThreadMapping && !hasWarpMapping && !hasWarpgroupMapping) {
+      !hasLaneMapping && !hasThreadMapping && !hasWarpMapping &&
+      !hasWarpgroupMapping) {
     return definiteFailureHelper(transformOp, forallOp,
                                  "scf.forall op requires a mapping attribute "
                                  "of kind 'thread' or 'warp'");
@@ -347,16 +387,25 @@ checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp,
     seen.insert(map);
   }
 
-  auto isLinear = [](Attribute a) {
-    return cast<DeviceMappingAttrInterface>(a).isLinearMapping();
+  auto isLinear = [](DeviceMappingAttrInterface attr) {
+    return attr.isLinearMapping();
   };
-  if (llvm::any_of(forallOp.getMapping()->getValue(), isLinear) &&
-      !llvm::all_of(forallOp.getMapping()->getValue(), isLinear)) {
+  if (llvm::any_of(forallOp.getDeviceMappingAttrs(), isLinear) &&
+      !llvm::all_of(forallOp.getDeviceMappingAttrs(), isLinear)) {
     return definiteFailureHelper(
         transformOp, forallOp,
         "cannot mix linear and non-linear mapping modes");
   }
 
+  FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr =
+      forallOp.getDeviceMaskingAttr();
+  if (succeeded(maybeMaskingAttr) && *maybeMaskingAttr &&
+      !forallOp.usesLinearMapping()) {
+    return definiteFailureHelper(
+        transformOp, forallOp,
+        "device masking is only available in linear mapping mode");
+  }
+
   return DiagnosedSilenceableFailure::success();
 }
 
@@ -377,9 +426,7 @@ verifyGpuMapping(std::optional<TransformOpInterface> transformOp,
   if (forallOp.getNumResults() > 0)
     return definiteFailureHelper(transformOp, forallOp,
                                  "only bufferized scf.forall can be mapped");
-  bool useLinearMapping = cast<DeviceMappingAttrInterface>(
-                              forallOp.getMapping()->getValue().front())
-                              .isLinearMapping();
+  bool useLinearMapping = forallOp.usesLinearMapping();
   // TODO: This would be more natural with support for Optional<EnumParameter>
   // in GPUDeviceMappingAttr.
   int64_t maxNumMappingsSupported =
@@ -432,8 +479,10 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   assert(forallOp.isNormalized() && numParallelIterations.has_value() &&
          "requires statically sized, normalized forall op");
   SmallVector<int64_t> tmpMappingSizes = numParallelIterations.value();
+  SmallVector<DeviceMappingAttrInterface> forallMappingAttrsVec =
+      forallOp.getDeviceMappingAttrs();
   SetVector<Attribute> forallMappingAttrs;
-  forallMappingAttrs.insert_range(forallOp.getMapping()->getValue());
+  forallMappingAttrs.insert_range(forallMappingAttrsVec);
   auto comparator = [](Attribute a, Attribute b) -> bool {
     return cast<DeviceMappingAttrInterface>(a).getMappingId() <
            cast<DeviceMappingAttrInterface>(b).getMappingId();
@@ -473,13 +522,24 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   SmallVector<int64_t> originalBasis(availableMappingSizes);
   bool originalBasisWasProvided = !originalBasis.empty();
   if (!originalBasisWasProvided) {
+    LDBG("----originalBasis was not provided, deriving it and there will be no "
+         "predication");
     originalBasis = forallMappingSizes;
     while (originalBasis.size() < 3)
       originalBasis.push_back(1);
+  } else {
+    LDBG("----originalBasis was provided, using it, there will be predication");
   }
+  LLVM_DEBUG(
+      llvm::interleaveComma(originalBasis, DBGS() << "------originalBasis: ");
+      llvm::dbgs() << "\n");
 
   IdBuilderResult builderResult =
       gpuIdBuilder.idBuilder(rewriter, loc, forallMappingSizes, originalBasis);
+  if (!builderResult.errorMsg.empty())
+    return definiteFailureHelper(transformOp, forallOp, builderResult.errorMsg);
+
+  LLVM_DEBUG(DBGS() << builderResult);
 
   // Step 4. Map the induction variables to the mappingIdOps, this may involve
   // a permutation.
@@ -490,6 +550,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
            forallMappingAttrs.getArrayRef().take_front(forallOp.getRank()))) {
     auto mappingAttr = cast<DeviceMappingAttrInterface>(dim);
     Value peIdOp = mappingIdOps[mappingAttr.getRelativeIndex()];
+    LDBG("----map: " << iv << " to " << peIdOp);
     bvm.map(iv, peIdOp);
   }
 
@@ -498,32 +559,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   // originalBasis and no predication occurs.
   Value predicate;
   if (originalBasisWasProvided) {
-    SmallVector<int64_t> activeMappingSizes = builderResult.activeMappingSizes;
-    SmallVector<int64_t> availableMappingSizes =
-        builderResult.availableMappingSizes;
-    SmallVector<Value> activeIdOps = builderResult.activeIdOps;
-    LDBG("----activeMappingSizes: " << llvm::interleaved(activeMappingSizes));
-    LDBG("----availableMappingSizes: "
-         << llvm::interleaved(availableMappingSizes));
-    LDBG("----activeIdOps: " << llvm::interleaved(activeIdOps));
-    for (auto [activeId, activeMappingSize, availableMappingSize] :
-         llvm::zip_equal(activeIdOps, activeMappingSizes,
-                         availableMappingSizes)) {
-      if (activeMappingSize > availableMappingSize) {
-        return definiteFailureHelper(
-            transformOp, forallOp,
-            "Trying to map to fewer GPU threads than loop iterations but "
-            "overprovisioning is not yet supported. "
-            "Try additional tiling of the before mapping or map to more "
-            "threads.");
-      }
-      if (activeMappingSize == availableMappingSize)
-        continue;
-      Value idx =
-          rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
-      Value tmpPredicate = rewriter.create<arith::CmpIOp>(
-          loc, arith::CmpIPredicate::ult, activeId, idx);
-      LDBG("----predicate: " << tmpPredicate);
+    for (Value tmpPredicate : builderResult.predicateOps) {
       predicate = predicate ? rewriter.create<arith::AndIOp>(loc, predicate,
                                                              tmpPredicate)
                             : tmpPredicate;
@@ -691,12 +727,17 @@ DiagnosedSilenceableFailure transform::MapForallToBlocks::applyToOne(
 
   // The BlockIdBuilder adapts to whatever is thrown at it.
   bool useLinearMapping = false;
-  if (topLevelForallOp.getMapping()) {
-    auto mappingAttr = cast<DeviceMappingAttrInterface>(
-        topLevelForallOp.getMapping()->getValue().front());
-    useLinearMapping = mappingAttr.isLinearMapping();
-  }
-  GpuBlockIdBuilder gpuBlockIdBuilder(getContext(), useLinearMapping);
+  if (topLevelForallOp.getMapping())
+    useLinearMapping = topLevelForallOp.usesLinearMapping();
+
+  FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr =
+      topLevelForallOp.getDeviceMaskingAttr();
+  assert(succeeded(maybeMaskingAttr) && "unexpected failed maybeMaskingAttr");
+  assert((!*maybeMaskingAttr || useLinearMapping) &&
+         "masking requires linear mapping");
+
+  GpuBlockIdBuilder gpuBlockIdBuilder(getContext(), useLinearMapping,
+                                      *maybeMaskingAttr);
 
   diag = mlir::transform::gpu::mapForallToBlocksImpl(
       rewriter, transformOp, topLevelForallOp, gridDims, gpuBlockIdBuilder);
@@ -753,8 +794,8 @@ static DiagnosedSilenceableFailure
 getThreadIdBuilder(std::optional<TransformOpInterface> transformOp,
                    scf::ForallOp forallOp, ArrayRef<int64_t> blockSizes,
                    int64_t warpSize, GpuIdBuilder &gpuIdBuilder) {
-  auto mappingAttr = cast<DeviceMappingAttrInterface>(
-      forallOp.getMapping()->getValue().front());
+  DeviceMappingAttrInterface mappingAttr =
+      forallOp.getDeviceMappingAttrs().front();
   bool useLinearMapping = mappingAttr.isLinearMapping();
 
   // Sanity checks that may result in runtime verification errors.
@@ -777,18 +818,30 @@ getThreadIdBuilder(std::optional<TransformOpInterface> transformOp,
   if (!diag.succeeded())
     return diag;
 
+  FailureOr<DeviceMaskingAttrInterface> maybeMaskingAttr =
+      forallOp.getDeviceMaskingAttr();
+  assert(succeeded(maybeMaskingAttr) && "unexpected failed maybeMaskingAttr");
+  assert((!*maybeMaskingAttr || useLinearMapping) &&
+         "masking requires linear mapping");
+
   // Start mapping.
   MLIRContext *ctx = forallOp.getContext();
   gpuIdBuilder =
       TypeSwitch<DeviceMappingAttrInterface, GpuIdBuilder>(mappingAttr)
           .Case([&](GPUWarpgroupMappingAttr) {
-            return GpuWarpgroupIdBuilder(ctx, warpSize, useLinearMapping);
+            return GpuWarpgroupIdBuilder(ctx, warpSize, useLinearMapping,
+                                         *maybeMaskingAttr);
           })
           .Case([&](GPUWarpMappingAttr) {
-            return GpuWarpIdBuilder(ctx, warpSize, useLinearMapping);
+            return GpuWarpIdBuilder(ctx, warpSize, useLinearMapping,
+                                    *maybeMaskingAttr);
           })
           .Case([&](GPUThreadMappingAttr) {
-            return GpuThreadIdBuilder(ctx, useLinearMapping);
+            return GpuThreadIdBuilder(ctx, useLinearMapping, *maybeMaskingAttr);
+          })
+          .Case([&](GPULaneMappingAttr) {
+            return GpuLaneIdBuilder(ctx, warpSize, useLinearMapping,
+                                    *maybeMaskingAttr);
           })
           .Default([&](DeviceMappingAttrInterface) -> GpuIdBuilder {
             llvm_unreachable("unknown mapping attribute");
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index 9853e80828390..f6bdbe384c08f 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -10,21 +10,16 @@
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/TransformOps/GPUTransformOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
-#include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/Value.h"
@@ -32,7 +27,6 @@
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/InterleavedRange.h"
 
@@ -44,15 +38,60 @@ using namespace mlir::transform::gpu;
 #define DEBUG_TYPE "gpu-transforms"
 
 #define DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")
-#define LDBG(X) LLVM_DEBUG(DBGS() << (X) << "\n")
+#define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
 #define DBGS_ALIAS() (llvm::dbgs() << '[' << DEBUG_TYPE_ALIAS << "] ")
 
+/// Build predicates to filter execution by only the activeIds. Along each
+/// dimension, 3 cases appear:
+///   1. activeMappingSize > availableMappingSize: this is an unsupported case
+///      as this requires additional looping. An error message is produced to
+///      advise the user to tile more or to use more threads.
+///   2. activeMappingSize == availableMappingSize: no predication is needed.
+///   3. activeMappingSize < availableMappingSize: only a subset of threads
+///      should be active and we produce the boolean `id < activeMappingSize`
+///      for further use in building predicated execution.
+static FailureOr<SmallVector<Value>>
+buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
+                ArrayRef<int64_t> activeMappingSizes,
+                ArrayRef<int64_t> availableMappingSizes,
+                std::string &errorMsg) {
+  // clang-format off
+  LLVM_DEBUG(
+    llvm::interleaveComma(
+      activeMappingSizes, DBGS() << "----activeMappingSizes: ");
+    DBGS() << "\n";
+    llvm::interleaveComma(
+      availableMappingSizes, DBGS() << "----availableMappingSizes: ");
+    DBGS() << "\n";);
+  // clang-format on
+
+  SmallVector<Value> predicateOps;
+  for (auto [activeId, activeMappingSize, availableMappingSize] :
+       llvm::zip_equal(activeIds, activeMappingSizes, availableMappingSizes)) {
+    if (activeMappingSize > availableMappingSize) {
+      errorMsg = "Trying to map to fewer GPU threads than loop iterations but "
+                 "overprovisioning is not yet supported. Try additional tiling "
+                 "before mapping or map to more threads.";
+      return failure();
+    }
+    if (activeMappingSize == availableMappingSize)
+      continue;
+    Value idx = rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
+    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+                                                activeId, idx);
+    predicateOps.push_back(pred);
+  }
+  return predicateOps;
+}
+
 /// Return a flattened thread id for the workgroup with given sizes.
 template <typename ThreadOrBlockIdOp>
 static Value buildLinearId(RewriterBase &rewriter, Location loc,
                            ArrayRef<OpFoldResult> originalBasisOfr) {
-  LLVM_DEBUG(DBGS() << "----buildLinearId with originalBasisOfr:  "
-                    << llvm::interleaved(originalBasisOfr) << "\n");
+  LLVM_DEBUG(llvm::interleaveComma(
+                 originalBasisOfr,
+                 DBGS() << "----buildLinearId with originalBasisOfr:  ");
+             llvm::dbgs() << "\n");
   assert(originalBasisOfr.size() == 3 && "expected 3 sizes");
   IndexType indexType = rewriter.getIndexType();
   AffineExpr tx, ty, tz, bdx, bdy;
@@ -75,48 +114,86 @@ static Value buildLinearId(RewriterBase &rewriter, Location loc,
 /// it in the basis of `forallMappingSizes`. The linear id builder returns an
 /// n-D vector of ids for indexing and 1-D size + id for predicate generation.
 template <typename ThreadOrBlockIdOp>
-static GpuIdBuilderFnType commonLinearIdBuilderFn(int64_t multiplicity = 1) {
-  auto res = [multiplicity](RewriterBase &rewriter, Location loc,
-                            ArrayRef<int64_t> forallMappingSizes,
-                            ArrayRef<int64_t> originalBasis) {
+static GpuIdBuilderFnType
+commonLinearIdBuilderFn(int64_t multiplicity = 1,
+                        DeviceMaskingAttrInterface mask = nullptr) {
+  auto res = [multiplicity, mask](RewriterBase &rewriter, Location loc,
+                                  ArrayRef<int64_t> forallMappingSizes,
+                                  ArrayRef<int64_t> originalBasis) {
+    // 0. Early-exit mask case.
+    if (mask) {
+      if (computeProduct(originalBasis) >
+          mask.getMaxNumPhysicalIds() * multiplicity) {
+        return IdBuilderResult{
+            /*errorMsg=*/std::string(
+                "mask representation too short to capture all physical ids: ") +
+                std::to_string(mask.getMaxNumPhysicalIds()),
+            /*mappingIdOps=*/{},
+            /*predicateOps=*/{}};
+      }
+    }
+
+    // 1. Compute linearId.
     SmallVector<OpFoldResult> originalBasisOfr =
         getAsIndexOpFoldResult(rewriter.getContext(), originalBasis);
-    OpFoldResult linearId =
+    Value physicalLinearId =
         buildLinearId<ThreadOrBlockIdOp>(rewriter, loc, originalBasisOfr);
+
+    // 2. Compute scaledLinearId.
+    AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
+    OpFoldResult scaledLinearIdOfr = affine::makeComposedFoldedAffineApply(
+        rewriter, loc, d0.floorDiv(multiplicity), {physicalLinearId});
+
+    // 2.b. Adjust with mask if needed.
+    Value scaledLinearIdI64;
+    Value scaledLinearId =
+        getValueOrCreateConstantIndexOp(rewriter, loc, scaledLinearIdOfr);
+    if (mask) {
+      scaledLinearId =
+          getValueOrCreateConstantIndexOp(rewriter, loc, scaledLinearIdOfr);
+      scaledLinearIdI64 = rewriter.create<arith::IndexCastUIOp>(
+          loc, rewriter.getI64Type(), scaledLinearId);
+      Value logicalLinearIdI64 =
+          mask.createLogicalLinearMappingId(rewriter, scaledLinearIdI64);
+      scaledLinearId = rewriter.create<arith::IndexCastUIOp>(
+          loc, rewriter.getIndexType(), logicalLinearIdI64);
+      LDBG("------adjusting linearId with mask: " << scaledLinearId);
+    }
+
+    // 3. Compute remapped indices.
+    SmallVector<Value> ids;
     // Sizes in [0 .. n] -> [n .. 0] order to properly compute strides in
     // "row-major" order.
     SmallVector<int64_t> reverseBasisSizes(llvm::reverse(forallMappingSizes));
     SmallVector<int64_t> strides = computeStrides(reverseBasisSizes);
-    AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
-    OpFoldResult scaledLinearId = affine::makeComposedFoldedAffineApply(
-        rewriter, loc, d0.floorDiv(multiplicity), {linearId});
     SmallVector<AffineExpr> delinearizingExprs = delinearize(d0, strides);
-    SmallVector<Value> ids;
     // Reverse back to be in [0 .. n] order.
     for (AffineExpr e : llvm::reverse(delinearizingExprs)) {
       ids.push_back(
           affine::makeComposedAffineApply(rewriter, loc, e, {scaledLinearId}));
     }
 
-    LLVM_DEBUG(DBGS() << "--delinearization basis: "
-                      << llvm::interleaved(reverseBasisSizes) << "\n";
-               DBGS() << "--delinearization strides: "
-                      << llvm::interleaved(strides) << "\n";
-               DBGS() << "--delinearization exprs: "
-                      << llvm::interleaved(delinearizingExprs) << "\n";
-               DBGS() << "--ids: " << llvm::interleaved(ids) << "\n");
-
-    // Return n-D ids for indexing and 1-D size + id for predicate generation.
-    return IdBuilderResult{
-        /*mappingIdOps=*/ids,
-        /*availableMappingSizes=*/
-        SmallVector<int64_t>{computeProduct(originalBasis)},
-        // `forallMappingSizes` iterate in the scaled basis, they need to be
-        // scaled back into the original basis to provide tight
-        // activeMappingSizes quantities for predication.
-        /*activeMappingSizes=*/
-        SmallVector<int64_t>{computeProduct(forallMappingSizes) * multiplicity},
-        /*activeIdOps=*/SmallVector<Value>{cast<Value>(linearId)}};
+    std::string errorMsg;
+    SmallVector<Value> predicateOps;
+    // 4. If mask present, it takes precedence to determine predication.
+    if (mask) {
+      Value isActiveIdPredicate =
+          mask.createIsActiveIdPredicate(rewriter, scaledLinearIdI64);
+      LDBG("------adjusting predicate with mask: " << isActiveIdPredicate);
+      predicateOps.push_back(isActiveIdPredicate);
+    } else {
+      // 4.b. Otherwise, handle predicates using physicalLinearId.
+      FailureOr<SmallVector<Value>> maybePredicateOps =
+          buildPredicates(rewriter, loc, physicalLinearId,
+                          computeProduct(forallMappingSizes) * multiplicity,
+                          computeProduct(originalBasis), errorMsg);
+      if (succeeded(maybePredicateOps))
+        predicateOps = *maybePredicateOps;
+    }
+
+    return IdBuilderResult{/*errorMsg=*/errorMsg,
+                           /*mappingIdOps=*/ids,
+                           /*predicateOps=*/predicateOps};
   };
 
   return res;
@@ -143,16 +220,67 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
     // In the 3-D mapping case, unscale the first dimension by the multiplicity.
     SmallVector<int64_t> forallMappingSizeInOriginalBasis(forallMappingSizes);
     forallMappingSizeInOriginalBasis[0] *= multiplicity;
-    return IdBuilderResult{
-        /*mappingIdOps=*/scaledIds,
-        /*availableMappingSizes=*/SmallVector<int64_t>{originalBasis},
-        // `forallMappingSizes` iterate in the scaled basis, they need to be
-        // scaled back into the original basis to provide tight
-        // activeMappingSizes quantities for predication.
-        /*activeMappingSizes=*/
-        SmallVector<int64_t>{forallMappingSizeInOriginalBasis},
-        /*activeIdOps=*/ids};
+
+    std::string errorMsg;
+    SmallVector<Value> predicateOps;
+    FailureOr<SmallVector<Value>> maybePredicateOps =
+        buildPredicates(rewriter, loc, ids, forallMappingSizeInOriginalBasis,
+                        originalBasis, errorMsg);
+    if (succeeded(maybePredicateOps))
+      predicateOps = *maybePredicateOps;
+
+    return IdBuilderResult{/*errorMsg=*/errorMsg,
+                           /*mappingIdOps=*/scaledIds,
+                           /*predicateOps=*/predicateOps};
+  };
+  return res;
+}
+
+/// Create a lane id builder that takes the `originalBasis` and decompose
+/// it in the basis of `forallMappingSizes`. The linear id builder returns an
+/// n-D vector of ids for indexing and 1-D size + id for predicate generation.
+static GpuIdBuilderFnType laneIdBuilderFn(int64_t warpSize) {
+  auto res = [warpSize](RewriterBase &rewriter, Location loc,
+                        ArrayRef<int64_t> forallMappingSizes,
+                        ArrayRef<int64_t> originalBasis) {
+    // 1. Compute linearId.
+    SmallVector<OpFoldResult> originalBasisOfr =
+        getAsIndexOpFoldResult(rewriter.getContext(), originalBasis);
+    Value physicalLinearId =
+        buildLinearId<ThreadIdOp>(rewriter, loc, originalBasisOfr);
+
+    // 2. Compute laneId.
+    AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
+    OpFoldResult laneId = affine::makeComposedFoldedAffineApply(
+        rewriter, loc, d0 % warpSize, {physicalLinearId});
+
+    // 3. Compute remapped indices.
+    SmallVector<Value> ids;
+    // Sizes in [0 .. n] -> [n .. 0] order to properly compute strides in
+    // "row-major" order.
+    SmallVector<int64_t> reverseBasisSizes(llvm::reverse(forallMappingSizes));
+    SmallVector<int64_t> strides = computeStrides(reverseBasisSizes);
+    SmallVector<AffineExpr> delinearizingExprs = delinearize(d0, strides);
+    // Reverse back to be in [0 .. n] order.
+    for (AffineExpr e : llvm::reverse(delinearizingExprs)) {
+      ids.push_back(
+          affine::makeComposedAffineApply(rewriter, loc, e, {laneId}));
+    }
+
+    // 4. Handle predicates using laneId.
+    std::string errorMsg;
+    SmallVector<Value> predicateOps;
+    FailureOr<SmallVector<Value>> maybePredicateOps = buildPredicates(
+        rewriter, loc, cast<Value>(laneId), computeProduct(forallMappingSizes),
+        computeProduct(originalBasis), errorMsg);
+    if (succeeded(maybePredicateOps))
+      predicateOps = *maybePredicateOps;
+
+    return IdBuilderResult{/*errorMsg=*/errorMsg,
+                           /*mappingIdOps=*/ids,
+                           /*predicateOps=*/predicateOps};
   };
+
   return res;
 }
 
@@ -176,49 +304,68 @@ GpuIdBuilder::GpuIdBuilder(MLIRContext *ctx, bool useLinearMapping,
   }
 }
 
-GpuBlockIdBuilder::GpuBlockIdBuilder(MLIRContext *ctx, bool useLinearMapping)
+GpuBlockIdBuilder::GpuBlockIdBuilder(MLIRContext *ctx, bool useLinearMapping,
+                                     DeviceMaskingAttrInterface mask)
     : GpuIdBuilder(ctx, useLinearMapping, [](MLIRContext *ctx, MappingId id) {
         return GPUBlockMappingAttr::get(ctx, id);
       }) {
+  assert((!mask || useLinearMapping) && "mask requires linear mapping");
   idBuilder = useLinearMapping
-                  ? commonLinearIdBuilderFn<BlockIdOp>(/*multiplicity=*/1)
+                  ? commonLinearIdBuilderFn<BlockIdOp>(/*multiplicity=*/1, mask)
                   : common3DIdBuilderFn<BlockIdOp>(/*multiplicity=*/1);
 }
 
 GpuWarpgroupIdBuilder::GpuWarpgroupIdBuilder(MLIRContext *ctx, int64_t warpSize,
-                                             bool useLinearMapping)
+                                             bool useLinearMapping,
+                                             DeviceMaskingAttrInterface mask)
     : GpuIdBuilder(ctx, useLinearMapping,
                    [](MLIRContext *ctx, MappingId id) {
                      return GPUWarpgroupMappingAttr::get(ctx, id);
                    }),
       warpSize(warpSize) {
+  assert((!mask || useLinearMapping) && "mask requires linear mapping");
   idBuilder = useLinearMapping
                   ? commonLinearIdBuilderFn<ThreadIdOp>(
-                        /*multiplicity=*/kNumWarpsPerGroup * warpSize)
+                        /*multiplicity=*/kNumWarpsPerGroup * warpSize, mask)
                   : common3DIdBuilderFn<ThreadIdOp>(
                         /*multiplicity=*/kNumWarpsPerGroup * warpSize);
 }
 
 GpuWarpIdBuilder::GpuWarpIdBuilder(MLIRContext *ctx, int64_t warpSize,
-                                   bool useLinearMapping)
+                                   bool useLinearMapping,
+                                   DeviceMaskingAttrInterface mask)
     : GpuIdBuilder(ctx, useLinearMapping,
                    [](MLIRContext *ctx, MappingId id) {
                      return GPUWarpMappingAttr::get(ctx, id);
                    }),
       warpSize(warpSize) {
-  idBuilder =
-      useLinearMapping
-          ? commonLinearIdBuilderFn<ThreadIdOp>(/*multiplicity=*/warpSize)
-          : common3DIdBuilderFn<ThreadIdOp>(/*multiplicity=*/warpSize);
+  assert((!mask || useLinearMapping) && "mask requires linear mapping");
+  idBuilder = useLinearMapping
+                  ? commonLinearIdBuilderFn<ThreadIdOp>(
+                        /*multiplicity=*/warpSize, mask)
+                  : common3DIdBuilderFn<ThreadIdOp>(/*multiplicity=*/warpSize);
 }
 
-GpuThreadIdBuilder::GpuThreadIdBuilder(MLIRContext *ctx, bool useLinearMapping)
+GpuThreadIdBuilder::GpuThreadIdBuilder(MLIRContext *ctx, bool useLinearMapping,
+                                       DeviceMaskingAttrInterface mask)
     : GpuIdBuilder(ctx, useLinearMapping, [](MLIRContext *ctx, MappingId id) {
         return GPUThreadMappingAttr::get(ctx, id);
       }) {
-  idBuilder = useLinearMapping
-                  ? commonLinearIdBuilderFn<ThreadIdOp>(/*multiplicity=*/1)
-                  : common3DIdBuilderFn<ThreadIdOp>(/*multiplicity=*/1);
+  idBuilder =
+      useLinearMapping
+          ? commonLinearIdBuilderFn<ThreadIdOp>(/*multiplicity=*/1, mask)
+          : common3DIdBuilderFn<ThreadIdOp>(/*multiplicity=*/1);
+}
+
+GpuLaneIdBuilder::GpuLaneIdBuilder(MLIRContext *ctx, int64_t warpSize,
+                                   bool unused, DeviceMaskingAttrInterface mask)
+    : GpuIdBuilder(ctx, /*useLinearMapping=*/true,
+                   [](MLIRContext *ctx, MappingId id) {
+                     return GPULaneMappingAttr::get(ctx, id);
+                   }),
+      warpSize(warpSize) {
+  assert(!mask && "mask NYI for lanes, unclear it should be at all");
+  idBuilder = laneIdBuilderFn(/*periodicity=*/warpSize);
 }
 
 DiagnosedSilenceableFailure checkGpuLimits(TransformOpInterface transformOp,
diff --git a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
index d35f72e5a9e26..98dc8ad3aa416 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
@@ -20,8 +20,6 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
-#include "llvm/Support/ErrorHandling.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index 99a91ecd5642c..c39ba4a41898d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -19,11 +19,8 @@
 #include "mlir/Dialect/GPU/Utils/GPUUtils.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/IRMapping.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/GPU/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/GPU/Transforms/BufferDeallocationOpInterfaceImpl.cpp
index 6ccc0a26426c1..8f0877be5825a 100644
--- a/mlir/lib/Dialect/GPU/Transforms/BufferDeallocationOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/BufferDeallocationOpInterfaceImpl.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Dialect/GPU/Transforms/BufferDeallocationOpInterfaceImpl.h"
 #include "mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h"
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index f63af8da28087..65b9407a7efba 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -19,7 +18,6 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/GPU/Transforms/EliminateBarriers.cpp b/mlir/lib/Dialect/GPU/Transforms/EliminateBarriers.cpp
index 912d4dc99a885..57b529daa3c1e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/EliminateBarriers.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/EliminateBarriers.cpp
@@ -22,7 +22,6 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
index c40ddd9b15afc..153ceb23a6ecd 100644
--- a/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/GlobalIdRewriter.cpp
@@ -14,9 +14,7 @@
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/Dialect/Index/IR/IndexOps.h"
-#include "mlir/IR/Builders.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index a64ec8d52daf0..34ea9fcab4188 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Utils/GPUUtils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
index f8a548af6b3e8..9a69e6dde4274 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp
@@ -13,14 +13,7 @@
 
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
-#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
index 9f2900214e8b1..2d6df0ff6d02d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ShuffleRewriter.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
index d80578235f3c3..05631ad87dd71 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupIdRewriter.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/Index/IR/IndexOps.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
index af9be4cccecfc..1b3d13623c548 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SubgroupReduceLowering.cpp
@@ -22,7 +22,6 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index d83fd3800eb91..d987b72e98354 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -110,3 +110,26 @@ add_mlir_dialect_library(MLIRVCIXDialect
   MLIRLLVMDialect
   MLIRSideEffectInterfaces
   )
+
+add_mlir_dialect_library(MLIRXeVMDialect
+  IR/XeVMDialect.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
+
+  DEPENDS
+  MLIRGPUCompilationAttrInterfacesIncGen
+  MLIRXeVMOpsIncGen
+  MLIRXeVMConversionsIncGen
+  intrinsics_gen
+
+  LINK_COMPONENTS
+  AsmParser
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRDialectUtils
+  MLIRIR
+  MLIRLLVMDialect
+  MLIRSideEffectInterfaces
+)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 7276464389559..6dcd94e6eea17 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -128,7 +128,6 @@ REGISTER_ENUM_TYPE(Visibility);
 /// second template argument.
 template <typename EnumTy, typename RetTy = EnumTy>
 static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser,
-                                      OperationState &result,
                                       EnumTy defaultValue) {
   SmallVector<StringRef, 10> names;
   for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
@@ -1396,14 +1395,14 @@ ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
       getCConvAttrName(result.name),
-      CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
-                                              parser, result, LLVM::CConv::C)));
+      CConvAttr::get(parser.getContext(),
+                     parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
 
   result.addAttribute(
       getTailCallKindAttrName(result.name),
       TailCallKindAttr::get(parser.getContext(),
                             parseOptionalLLVMKeyword<TailCallKind>(
-                                parser, result, LLVM::TailCallKind::None)));
+                                parser, LLVM::TailCallKind::None)));
 
   // Parse a function pointer for indirect calls.
   if (parseOptionalCallFuncPtr(parser, operands))
@@ -1631,8 +1630,8 @@ ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
       getCConvAttrName(result.name),
-      CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
-                                              parser, result, LLVM::CConv::C)));
+      CConvAttr::get(parser.getContext(),
+                     parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
 
   // Parse a function pointer for indirect calls.
   if (parseOptionalCallFuncPtr(parser, operands))
@@ -2245,8 +2244,7 @@ void GlobalOp::print(OpAsmPrinter &p) {
                            getGlobalTypeAttrName(), getConstantAttrName(),
                            getValueAttrName(), getLinkageAttrName(),
                            getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
-                           getVisibility_AttrName(), getComdatAttrName(),
-                           getUnnamedAddrAttrName()});
+                           getVisibility_AttrName(), getComdatAttrName()});
 
   // Print the trailing type unless it's a string global.
   if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
@@ -2298,16 +2296,16 @@ static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
                                              OperationState &result) {
   MLIRContext *ctx = parser.getContext();
   // Parse optional linkage, default to External.
-  result.addAttribute(OpType::getLinkageAttrName(result.name),
-                      LLVM::LinkageAttr::get(
-                          ctx, parseOptionalLLVMKeyword<Linkage>(
-                                   parser, result, LLVM::Linkage::External)));
+  result.addAttribute(
+      OpType::getLinkageAttrName(result.name),
+      LLVM::LinkageAttr::get(ctx, parseOptionalLLVMKeyword<Linkage>(
+                                      parser, LLVM::Linkage::External)));
 
   // Parse optional visibility, default to Default.
   result.addAttribute(OpType::getVisibility_AttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
-                              parser, result, LLVM::Visibility::Default)));
+                              parser, LLVM::Visibility::Default)));
 
   if (succeeded(parser.parseOptionalKeyword("thread_local")))
     result.addAttribute(OpType::getThreadLocal_AttrName(result.name),
@@ -2317,7 +2315,7 @@ static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
   result.addAttribute(OpType::getUnnamedAddrAttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
-                              parser, result, LLVM::UnnamedAddr::None)));
+                              parser, LLVM::UnnamedAddr::None)));
 
   return success();
 }
@@ -2588,7 +2586,7 @@ void AliasOp::print(OpAsmPrinter &p) {
                           {SymbolTable::getSymbolAttrName(),
                            getAliasTypeAttrName(), getLinkageAttrName(),
                            getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
-                           getVisibility_AttrName(), getUnnamedAddrAttrName()});
+                           getVisibility_AttrName()});
 
   // Print the trailing type.
   p << " : " << getType() << ' ';
@@ -2825,29 +2823,28 @@ buildLLVMFunctionType(OpAsmParser &parser, SMLoc loc, ArrayRef<Type> inputs,
 //
 ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
   // Default to external linkage if no keyword is provided.
-  result.addAttribute(
-      getLinkageAttrName(result.name),
-      LinkageAttr::get(parser.getContext(),
-                       parseOptionalLLVMKeyword<Linkage>(
-                           parser, result, LLVM::Linkage::External)));
+  result.addAttribute(getLinkageAttrName(result.name),
+                      LinkageAttr::get(parser.getContext(),
+                                       parseOptionalLLVMKeyword<Linkage>(
+                                           parser, LLVM::Linkage::External)));
 
   // Parse optional visibility, default to Default.
   result.addAttribute(getVisibility_AttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
-                              parser, result, LLVM::Visibility::Default)));
+                              parser, LLVM::Visibility::Default)));
 
   // Parse optional UnnamedAddr, default to None.
   result.addAttribute(getUnnamedAddrAttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
-                              parser, result, LLVM::UnnamedAddr::None)));
+                              parser, LLVM::UnnamedAddr::None)));
 
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
       getCConvAttrName(result.name),
-      CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
-                                              parser, result, LLVM::CConv::C)));
+      CConvAttr::get(parser.getContext(),
+                     parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
 
   StringAttr nameAttr;
   SmallVector<OpAsmParser::Argument> entryArgs;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
new file mode 100644
index 0000000000000..9e497829ba723
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -0,0 +1,366 @@
+//===-- XeVMDialect.cpp - XeVM dialect registration -------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
+#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace mlir;
+using namespace mlir::xevm;
+
+#include "mlir/Dialect/LLVMIR/XeVMOpsDialect.cpp.inc"
+#include "mlir/Dialect/LLVMIR/XeVMOpsEnums.cpp.inc"
+
+namespace {
+static constexpr uint32_t subgroupSize = 16;
+
+template <typename Op>
+LogicalResult verifyMatrixInput(Op op) {
+  static_assert(llvm::is_one_of<Op, BlockLoad2dOp, BlockStore2dOp,
+                                BlockPrefetch2dOp>::value,
+                "Unexpected template parameter");
+
+  std::optional<int64_t> width = getConstantIntValue(op.getBaseWidth());
+  std::optional<int64_t> pitch = getConstantIntValue(op.getBasePitch());
+  if (pitch && width && *pitch < *width)
+    return op->emitOpError(
+        "4th operand (base pitch) should be >= 2nd operand (base width)");
+
+  uint32_t elemSize = op.getElemSizeInBits();
+  if (elemSize < 8 || !llvm::isPowerOf2_32(elemSize) || elemSize > 32)
+    return op->emitOpError("expecting 'elem_size_in_bits' to be 8, 16, or 32");
+
+  uint32_t tileHeight = op.getTileHeight();
+  if (tileHeight > 32 || !llvm::isPowerOf2_32(tileHeight))
+    return op->emitOpError("expecting tile_height to be 1, 2, 4, 8, 16, or 32");
+
+  uint32_t vBlocks = op.getVBlocks();
+  if (vBlocks > 8 || !llvm::isPowerOf2_32(vBlocks))
+    return op->emitOpError("expecting v_blocks to be 1, 2, 4, or 8");
+
+  return success();
+}
+
+LogicalResult verify2DBlockLoadRestriction(BlockLoad2dOp op) {
+  VectorType resTy = op.getRes().getType();
+  if (!resTy.getElementType().isIntOrFloat())
+    return op.emitOpError()
+           << "expecting result element type to be int or float";
+  unsigned resElemTySize = resTy.getElementType().getIntOrFloatBitWidth();
+  unsigned resSize = resTy.getNumElements() * resElemTySize;
+  unsigned expectedSize = op.getElemSizeInBits() * op.getTileHeight() *
+                          op.getTileWidth() * op.getVBlocks() / subgroupSize;
+  if (resSize != expectedSize)
+    return op.emitOpError() << "result size of " << resSize
+                            << " bits does not match the expected size of "
+                            << expectedSize << " bits";
+
+  if (op.getTranspose() && op.getPackRegister())
+    return op.emitOpError("transpose and pack_register are mutually exclusive");
+
+  if (!op.getTranspose() && !op.getPackRegister()) {
+    uint32_t tileHeight = op.getTileHeight();
+    if (tileHeight < 1 || tileHeight > 32)
+      return op.emitOpError("expecting tile_height to be between 1 and 32");
+
+    uint32_t tileWidth = op.getTileWidth();
+    uint32_t vBlocks = op.getVBlocks();
+    switch (op.getElemSizeInBits()) {
+    case 8:
+      if (tileWidth < 4 || tileWidth > 64)
+        return op.emitOpError("expecting tile_width to be between 4 and 64");
+      if (vBlocks != 1 && vBlocks != 2 && vBlocks != 4)
+        return op.emitOpError("expecting v_blocks to be 1, 2, or 4");
+      if (tileWidth * vBlocks > 64)
+        return op.emitOpError(
+            "tile_width * v_blocks should be less than or equal "
+            "to 64 for 8 bit elements");
+      break;
+    case 16:
+      if (tileWidth < 2 || tileWidth > 32)
+        return op.emitOpError("expecting tile_width to be between 2 and 32");
+      if (vBlocks != 1 && vBlocks != 2 && vBlocks != 4)
+        return op.emitOpError("expecting v_blocks to be 1, 2, or 4");
+      if (tileWidth * vBlocks > 32)
+        return op.emitOpError(
+            "tile_width * v_blocks should be less than or equal "
+            "to 32 for 16 bit elements");
+      break;
+    case 32:
+      if (tileWidth < 1 || tileWidth > 16)
+        return op.emitOpError("expecting tile_width to be between 1 and 16");
+      if (vBlocks != 1 && vBlocks != 2)
+        return op.emitOpError("expecting v_blocks to be 1 or 2");
+      if (tileWidth * vBlocks > 16)
+        return op.emitOpError(
+            "tile_width * v_blocks should be less than or equal "
+            "to 16 for 32 bit elements");
+      break;
+    case 64:
+      if (tileWidth < 1 || tileWidth > 8)
+        return op.emitOpError("expecting tile_width to be between 1 and 8");
+      if (vBlocks != 1)
+        return op.emitOpError("expecting v_blocks to be 1");
+      break;
+    default:
+      return op.emitOpError(
+          "expecting elem_size_in_bits to be 8, 16, 32, or 64");
+    }
+
+    return success();
+  }
+
+  if (op.getTranspose()) {
+    assert(!op.getPackRegister() && "Expecting pack_register should be false");
+
+    uint32_t vBlocks = op.getVBlocks();
+    if (vBlocks != 1)
+      return op.emitOpError("expecting v_blocks to be 1");
+
+    uint32_t tileHeight = op.getTileHeight();
+    uint32_t tileWidth = op.getTileWidth();
+    switch (op.getElemSizeInBits()) {
+    case 32:
+      if (tileHeight < 1 || tileHeight > 32)
+        return op.emitOpError("expecting tile_height to be between 1 and 32");
+      if (tileWidth < 1 || tileWidth > 8)
+        return op.emitOpError("expecting tile_width to be between 1 and 8");
+      break;
+    case 64:
+      if (tileHeight != 8)
+        return op.emitOpError(
+            "expecting tile_height to be 8 for 64 bit elements");
+      if (tileWidth != 1 && tileWidth != 2 && tileWidth != 4)
+        return op.emitOpError("expecting tile_width to be 1, 2, or 4");
+      break;
+    default:
+      return op.emitOpError("transpose is only supported for 32 and 64 bit "
+                            "elements");
+    }
+
+    return success();
+  }
+
+  assert(op.getPackRegister() && !op.getTranspose() &&
+         "Expecting pack_register should be true and transpose should be "
+         "false");
+
+  uint32_t vBlocks = op.getVBlocks();
+  if (vBlocks != 1 && vBlocks != 2 && vBlocks != 4)
+    return op.emitOpError("expecting v_blocks to be 1, 2, or 4");
+
+  uint32_t tileHeight = op.getTileHeight();
+  uint32_t tileWidth = op.getTileWidth();
+  switch (op.getElemSizeInBits()) {
+  case 8:
+    if (tileHeight < 4 || tileHeight > 32)
+      return op.emitOpError("expecting tile_height to be between 4 and 32");
+    if (tileWidth < 4 || tileWidth > 16)
+      return op.emitOpError("expecting tile_width to be between 4 and 16");
+    break;
+  case 16:
+    if (tileHeight < 2 || tileHeight > 32)
+      return op.emitOpError("expecting tile_height to be between 2 and 32");
+    if (tileWidth < 2 || tileWidth > 16)
+      return op.emitOpError("expecting tile_width to be between 2 and 16");
+    if (tileWidth * vBlocks > 32)
+      return op.emitOpError(
+          "tile_width * v_blocks should be less than or equal "
+          "to 32 for 16 bit elements");
+    break;
+  default:
+    return op.emitOpError("pack_register is only supported for 8 and 16 bit "
+                          "elements");
+  }
+
+  return success();
+}
+
+static LogicalResult verify2DBlockStoreRestriction(BlockStore2dOp op) {
+  uint32_t tileHeight = op.getTileHeight();
+  if (tileHeight < 1 || tileHeight > 8)
+    return op.emitOpError("expecting tile_height to be between 1 and 8");
+
+  uint32_t tileWidth = op.getTileWidth();
+  switch (op.getElemSizeInBits()) {
+  case 8:
+    if (tileWidth < 4 || tileWidth > 64)
+      return op.emitOpError("expecting tile_width to be between 4 and 64");
+    break;
+  case 16:
+    if (tileWidth < 2 || tileWidth > 32)
+      return op.emitOpError("expecting tile_width to be between 2 and 32");
+    break;
+  case 32:
+    if (tileWidth < 1 || tileWidth > 16)
+      return op.emitOpError("expecting tile_width to be between 1 and 16");
+    break;
+  case 64:
+    if (tileWidth < 1 || tileWidth > 8)
+      return op.emitOpError("expecting tile_width to be between 1 and 8");
+    break;
+  default:
+    return op.emitOpError("expecting elem_size_in_bits to be 8, 16, 32, or 64");
+  }
+
+  uint32_t vBlocks = op.getVBlocks();
+  if (vBlocks != 1)
+    return op.emitOpError("expecting v_blocks to be 1");
+  return success();
+}
+
+} // namespace
+
+LogicalResult BlockLoad2dOp::verify() {
+  if (verify2DBlockLoadRestriction(*this).failed())
+    return failure();
+
+  if (verifyMatrixInput(*this).failed())
+    return failure();
+
+  VectorType resTy = getRes().getType();
+  if (!resTy.getElementType().isIntOrFloat())
+    return emitOpError() << "expecting result element type to be int of float";
+  unsigned resElemTySize = resTy.getElementType().getIntOrFloatBitWidth();
+  if (getElemSizeInBits() == 32 || getPackRegister()) {
+    if (resElemTySize != 32)
+      return emitOpError() << "expecting result element type to be 32 bits";
+  }
+
+  uint32_t tileWidth = getTileWidth();
+  if (getPackRegister()) {
+    if (tileWidth != 16)
+      return emitOpError(
+          "tile_width when pack_register is true should be equal "
+          "to subgroup size (16 elements)");
+    return success();
+  }
+
+  return success();
+}
+
+LogicalResult BlockStore2dOp::verify() {
+  if (verify2DBlockStoreRestriction(*this).failed())
+    return failure();
+
+  if (verifyMatrixInput(*this).failed())
+    return failure();
+
+  uint32_t tileWidth = getTileWidth();
+  switch (getElemSizeInBits()) {
+  case 8:
+    if (tileWidth != 16 && tileWidth != 32)
+      return emitOpError("tile_width for 8 bit elements should be equal to "
+                         "16 or 32");
+    break;
+  case 16:
+    if (tileWidth != 16)
+      return emitOpError("tile_width for 16 bit elements should be equal "
+                         "to 16");
+    break;
+  case 32:
+    if (tileWidth != 16)
+      return emitOpError("tile_width for 32 bit elements should be equal "
+                         "to 16");
+    break;
+  default:
+    llvm_unreachable("unexpected element size");
+  }
+
+  return success();
+}
+
+LogicalResult BlockPrefetch2dOp::verify() {
+  if (verifyMatrixInput(*this).failed())
+    return failure();
+
+  uint32_t tileWidth = getTileWidth();
+  switch (getElemSizeInBits()) {
+  case 8:
+    if (tileWidth != 16 && tileWidth != 32)
+      return emitOpError("tile_width for 8 bit elements should be equal to "
+                         "16 or 32");
+    break;
+  case 16:
+    if (tileWidth != 16)
+      return emitOpError("tile_width for 16 bit elements should be equal "
+                         "to 16");
+    break;
+  case 32:
+    if (tileWidth != 8 && tileWidth != 16)
+      return emitOpError(
+          "tile_width for 32 bit elements should be equal to 8 or 16");
+    break;
+  default:
+    llvm_unreachable("unexpected element size");
+  }
+
+  return success();
+}
+
+LogicalResult MMAOp::verify() {
+  if (getC()) {
+    if (getResult().getType() != getC().getType())
+      return emitOpError("type of C operand must match result type");
+  }
+  return success();
+}
+
+LogicalResult
+XeVMTargetAttr::verify(function_ref<InFlightDiagnostic()> emitError, int O,
+                       StringRef triple, StringRef chip, DictionaryAttr flags,
+                       ArrayAttr linkFiles) {
+  if (O < 0 || O > 3) {
+    return emitError()
+           << "The optimization level must be a number between 0 and 3.";
+  }
+  if (triple.empty()) {
+    return emitError() << "The target triple cannot be empty.";
+  }
+  if (chip.empty()) {
+    return emitError() << "The target chip cannot be empty.";
+  }
+  if (linkFiles) {
+    for (Attribute fileAttr : linkFiles) {
+      if (auto fileStrAttr = llvm::dyn_cast<StringAttr>(fileAttr)) {
+        StringRef filePath = fileStrAttr.getValue();
+        if (filePath.empty()) {
+          return emitError() << "File paths in linkFiles cannot be empty.";
+        }
+        if (!llvm::sys::fs::exists(filePath)) {
+          return emitError() << "File '" << filePath << "' does not exist.";
+        }
+      }
+    }
+  }
+  return success();
+}
+
+void XeVMDialect::initialize() {
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/LLVMIR/XeVMOps.cpp.inc"
+      >();
+
+  addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/LLVMIR/XeVMOpsAttributes.cpp.inc"
+      >();
+  declarePromisedInterface<mlir::gpu::TargetAttrInterface,
+                           mlir::xevm::XeVMTargetAttr>();
+}
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOps.cpp.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOpsAttributes.cpp.inc"
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
index c256b18dd2b17..b6e168e95ee86 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
@@ -11,27 +11,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/Interfaces/DestinationStyleOpInterface.h"
-#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
-#include "mlir/Parser/Parser.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/InliningUtils.h"
 
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
index ca7f31dd6b518..94f2002fc51fa 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
@@ -25,8 +25,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <numeric>
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 8075df730ccc6..3aa6ac3ea0918 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -19,20 +19,16 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
-#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypeInterfaces.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
@@ -42,7 +38,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -4569,7 +4564,7 @@ static SmallVector<OpFoldResult> getMixedTilesImpl(OpTy op) {
   SmallVector<OpFoldResult> mixedInnerTiles;
   unsigned dynamicValIndex = 0;
   for (int64_t staticTile : op.getStaticInnerTiles()) {
-    if (!ShapedType::isDynamic(staticTile))
+    if (ShapedType::isStatic(staticTile))
       mixedInnerTiles.push_back(builder.getI64IntegerAttr(staticTile));
     else
       mixedInnerTiles.push_back(op.getInnerTiles()[dynamicValIndex++]);
@@ -4834,7 +4829,7 @@ bool PackOp::requirePaddingValue(ArrayRef<int64_t> inputShape,
     std::optional<int64_t> constantTile = getConstantIntValue(tileSize);
 
     if (!constantTile) {
-      if (!ShapedType::isDynamic(outputTileSizes[pos]) &&
+      if (ShapedType::isStatic(outputTileSizes[pos]) &&
           (inputShape[pos] % outputTileSizes[pos] != 0))
         return true;
     } else if (inputShape[pos] % (*constantTile) != 0) {
@@ -4940,7 +4935,7 @@ SmallVector<OpFoldResult> PackOp::getResultShape(
   // use dispatchIndexOpFoldResults on the result, and rely on exact number of
   // dynamic dims returned by that.
   for (unsigned i = 0; i < resultDims.size(); ++i) {
-    if (!ShapedType::isDynamic(resultTypeShape[i]))
+    if (ShapedType::isStatic(resultTypeShape[i]))
       continue;
     resultDims[i] =
         getValueOrCreateConstantIndexOp(builder, loc, resultDims[i]);
diff --git a/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp
index 8d6d9dc690b55..f56ef485069f8 100644
--- a/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.h"
 
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Interfaces/IndexingMapOpInterface.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp b/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
index 4591802ce74ac..a1a91ca060b1d 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
@@ -14,11 +14,9 @@
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h"
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
-#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
-#include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
index 67bb1de59dd43..2fe72a326651d 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/MatchInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/InterleavedRange.h"
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 8571d641e26d1..5d5f9de465561 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -24,9 +24,6 @@
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/SCF/Transforms/TileUsingInterface.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
-#include "mlir/Dialect/Transform/IR/TransformDialect.h"
-#include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
@@ -39,7 +36,6 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Support/TypeID.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
@@ -2065,7 +2061,7 @@ transform::PadOp::apply(transform::TransformRewriter &rewriter,
       rewriter.setInsertionPoint(linalgTarget);
       for (OpOperand &operand : linalgTarget->getOpOperands()) {
         for (auto [i, dim] : llvm::enumerate(linalgTarget.getShape(&operand))) {
-          if (!ShapedType::isDynamic(dim))
+          if (ShapedType::isStatic(dim))
             continue;
           options.setSizeToPadTo(operand.getOperandNumber(), i,
                                  tensor::getMixedSize(rewriter,
@@ -2408,6 +2404,9 @@ transform::PromoteOp::applyToOne(transform::TransformRewriter &rewriter,
   if (getUseFullTilesByDefault())
     promotionOptions = promotionOptions.setUseFullTileBuffersByDefault(
         getUseFullTilesByDefault());
+  if (getUseOriginalSubviewSize())
+    promotionOptions =
+        promotionOptions.setUseOriginalSubviewSize(getUseOriginalSubviewSize());
   if (getUseAlloca())
     promotionOptions = promotionOptions.setUseAlloca(getUseAlloca());
   if (!getUseFullTileBuffers().empty())
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
index 81842e4bea631..3908d73f5e0e1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
@@ -14,7 +14,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
 
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp b/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp
index 428422e6e875a..9a16318342d10 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp
@@ -13,12 +13,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 9044d89c80bd6..1f6d96ca0f81f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -8,11 +8,9 @@
 
 #include "mlir/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp
index b53180b5cf7c3..edcb39d6e9a94 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp
@@ -10,13 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index 81d44ba04fa1d..a7732b939e70d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -19,7 +19,6 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include <utility>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 573420f6a9aa9..39e2aac27e213 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -14,7 +14,6 @@
 //===----------------------------------------------------------------------===//
 //
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
@@ -24,7 +23,6 @@
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Debug.h"
 
 using namespace mlir;
 using namespace mlir::tensor;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 7188987e5e938..70574903f7111 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -6,17 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Linalg/Passes.h"
-
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/Dominance.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -358,6 +353,12 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
   return newGenericOp;
 }
 
+static bool isGenericOutsNotUsed(linalg::GenericOp genericOp) {
+  return llvm::all_of(genericOp.getDpsInitsMutable(), [&](OpOperand &operand) {
+    return genericOp.getMatchingBlockArgument(&operand).use_empty();
+  });
+}
+
 /// Bubbles up linalg.pack op through a producer generic op. This
 /// swap pack(generic) to generic(pack). The new generic op works on packed
 /// domain; pack ops are created for input and output operands. E.g.,
@@ -470,12 +471,15 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
       getOrCreatePackedViewOfOperand(rewriter, genericOp.getLoc(), *packInfo,
                                      genericOp, opOperand);
 
-  // If the dps init operand of the generic is a tensor.empty forward the pack
-  // op destination.
+  // Forward the new tensor.empty as a destination if it is one of the following
+  // situations:
+  // 1) The dps init operand is a tensor.empty.
+  // 2) The dps init is a write-only operand, i.e., it is not used in the
+  // genericOp
   Value dest = packedOutOperand;
-  if (auto initTensor = genericOp.getDpsInitOperand(0)
-                            ->get()
-                            .getDefiningOp<tensor::EmptyOp>()) {
+  auto initTensor =
+      genericOp.getDpsInitOperand(0)->get().getDefiningOp<tensor::EmptyOp>();
+  if (initTensor || isGenericOutsNotUsed(genericOp)) {
     dest = packOpDest;
   }
   // pack(unpack) isn't naively foldable because the unpack op can be from
@@ -1101,12 +1105,15 @@ pushDownUnPackOpThroughGenericOp(RewriterBase &rewriter, GenericOp genericOp,
                                      genericOp, genericOp.getDpsInitOperand(0));
   auto destPack = packedOutOperand.getDefiningOp<linalg::PackOp>();
 
-  // If the dps init operand of the generic is a tensor.empty, do not pack it
-  // and forward the new tensor.empty as a destination.
+  // Forward the new tensor.empty as a destination if it is one of the following
+  // situations:
+  // 1) The dps init operand is a tensor.empty.
+  // 2) The dps init is a write-only operand, i.e., it is not used in the
+  // genericOp
   Value dest = packedOutOperand;
-  if (auto initTensor = genericOp.getDpsInitOperand(0)
-                            ->get()
-                            .getDefiningOp<tensor::EmptyOp>()) {
+  auto initTensor =
+      genericOp.getDpsInitOperand(0)->get().getDefiningOp<tensor::EmptyOp>();
+  if (initTensor || isGenericOutsNotUsed(genericOp)) {
     if (destPack)
       dest = destPack.getDest();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index daeae2c9d947d..692bf595267d4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 //
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include <map>
-#include <optional>
 #include <utility>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index 20ab5ecbe3c4a..ef24eb881d68b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -9,15 +9,12 @@
 #include "mlir/Dialect/Linalg/Passes.h"
 
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Func/Transforms/FuncConversions.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include <iterator>
-#include <memory>
 #include <utility>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index c5d9a729a4136..e0062d15e61ca 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -22,15 +22,12 @@
 #include "mlir/Dialect/MemRef/Transforms/Transforms.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tensor/Transforms/Transforms.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Transforms/FoldUtils.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index 28d6752fc2d38..c4af09ca01421 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Dialect/Linalg/Passes.h"
 
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp b/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp
index b710bde87f9f3..b6c4b98301e48 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 #include "mlir/Dialect/Bufferization/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp b/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
index fcf049e9ce722..b1c0c3b161b20 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
@@ -17,7 +17,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_LINALGFOLDINTOELEMENTWISEPASS
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index 703db8373c31d..44469bc404a7c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 4fc8a17554435..9bc7be2623849 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -10,10 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -23,17 +20,9 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/RegionUtils.h"
-#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
-#include <optional>
-#include <set>
-
 #define DEBUG_TYPE "linalg-fusion"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 78cee47c497ed..05f2157b77aeb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -13,17 +13,12 @@
 
 #include "mlir/Dialect/Linalg/Passes.h"
 
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/Debug.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_LINALGGENERALIZENAMEDOPSPASS
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index bfeeb7756860d..94ed46442180c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/Utils/Utils.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/AsmState.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/Matchers.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index d833e04d60264..f2e51c29f3241 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -15,23 +15,14 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Affine/Utils.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Utils/Utils.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 
 using llvm::dbgs;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index 785330736b39a..a92a0c83e0316 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -18,14 +18,9 @@
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include <type_traits>
 
 #define DEBUG_TYPE "linalg-interchange"
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 5b2ec4a6ff528..488041a43a2ef 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -10,7 +10,6 @@
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index 92cfba2549a3f..ee1957aaa6a53 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -29,15 +29,12 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
-#include "mlir/Interfaces/TilingInterface.h"
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include <iterator>
 #include <numeric>
 #include <optional>
-#include <utility>
 
 namespace mlir::linalg {
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 0984b6988b93b..2afa2f9b71c2a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -197,7 +197,9 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
 /// Fold a `pad` -> `pack` into `pack` if they have the same padding values and
 /// the pad op has zero low paddings, or if `pack` has no padding values.
 struct FoldPadWithPackOp : public OpRewritePattern<PackOp> {
-  using OpRewritePattern<PackOp>::OpRewritePattern;
+public:
+  FoldPadWithPackOp(MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<PackOp>(context), controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(PackOp packOp,
                                 PatternRewriter &rewriter) const override {
@@ -206,6 +208,10 @@ struct FoldPadWithPackOp : public OpRewritePattern<PackOp> {
     if (!padOp || padOp.getNofold() || !padOp.hasZeroLowPad())
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&packOp.getSourceMutable()))
+      return failure();
+
     Value constantPaddingValue = padOp.getConstantPaddingValue();
     if (!constantPaddingValue)
       return failure();
@@ -220,13 +226,20 @@ struct FoldPadWithPackOp : public OpRewritePattern<PackOp> {
         packOp.getOuterDimsPerm());
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold a `unpack` -> `extract_slice` into the `unpack` since it already
 /// has extract_slice semantics.
 struct FoldUnpackWithExtractSliceOp
     : public OpRewritePattern<tensor::ExtractSliceOp> {
-  using OpRewritePattern<tensor::ExtractSliceOp>::OpRewritePattern;
+public:
+  FoldUnpackWithExtractSliceOp(MLIRContext *context,
+                               ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<tensor::ExtractSliceOp>(context),
+        controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp,
                                 PatternRewriter &rewriter) const override {
@@ -234,6 +247,10 @@ struct FoldUnpackWithExtractSliceOp
     if (!unpackOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&sliceOp.getSourceMutable()))
+      return failure();
+
     if (sliceOp.getResultType().getRank() != unpackOp.getDestType().getRank()) {
       return rewriter.notifyMatchFailure(
           sliceOp, "rank-reduced folding is not supported");
@@ -255,6 +272,9 @@ struct FoldUnpackWithExtractSliceOp
         unpackOp.getMixedTiles(), unpackOp.getOuterDimsPerm());
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 // Applies 'permutation' on 'inVec' and stores the result in resVec.
@@ -284,7 +304,12 @@ static bool checkAndPermute(ArrayRef<int64_t> permutation,
 /// semantics.
 struct FoldProducerPackWithConsumerLinalgTransposeOp
     : public OpInterfaceRewritePattern<linalg::LinalgOp> {
-  using OpInterfaceRewritePattern<linalg::LinalgOp>::OpInterfaceRewritePattern;
+
+public:
+  FoldProducerPackWithConsumerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpInterfaceRewritePattern<linalg::LinalgOp>(context),
+        controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp,
                                 PatternRewriter &rewriter) const override {
@@ -293,6 +318,10 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp
     if (!packOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&linalgOp->getOpOperand(0)))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -331,13 +360,20 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold 'transpose' -> 'pack' into 'pack' since 'pack' already has transpose
 /// semantics.
 struct FoldConsumerPackWithProducerLinalgTransposeOp
     : public OpRewritePattern<PackOp> {
-  using OpRewritePattern<PackOp>::OpRewritePattern;
+
+public:
+  FoldConsumerPackWithProducerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<PackOp>(context), controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(PackOp packOp,
                                 PatternRewriter &rewriter) const override {
@@ -345,6 +381,10 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
     if (!linalgOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&packOp.getSourceMutable()))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -375,13 +415,21 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold 'unpack' -> 'transpose' into 'unpack' since 'unpack' already has
 /// transpose semantics.
 struct FoldProducerUnPackWithConsumerLinalgTransposeOp
     : public OpInterfaceRewritePattern<linalg::LinalgOp> {
-  using OpInterfaceRewritePattern<linalg::LinalgOp>::OpInterfaceRewritePattern;
+
+public:
+  FoldProducerUnPackWithConsumerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpInterfaceRewritePattern<linalg::LinalgOp>(context),
+        controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp,
                                 PatternRewriter &rewriter) const override {
@@ -390,6 +438,10 @@ struct FoldProducerUnPackWithConsumerLinalgTransposeOp
     if (!unPackOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&linalgOp->getOpOperand(0)))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -416,6 +468,9 @@ struct FoldProducerUnPackWithConsumerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold 'transpose' -> 'unpack' into 'unpack' since 'unpack' already has
@@ -424,12 +479,21 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
     : public OpRewritePattern<UnPackOp> {
   using OpRewritePattern<UnPackOp>::OpRewritePattern;
 
+public:
+  FoldConsumerUnPackWithProducerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<UnPackOp>(context), controlFn(std::move(controlFn)) {}
+
   LogicalResult matchAndRewrite(UnPackOp unPackOp,
                                 PatternRewriter &rewriter) const override {
     auto linalgOp = unPackOp.getSource().getDefiningOp<linalg::LinalgOp>();
     if (!linalgOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&unPackOp.getSourceMutable()))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -474,6 +538,9 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// tensor.empty does not define any tensor contents, so an unpadded pack
@@ -521,13 +588,14 @@ struct FoldEmptyTensorWithUnPackOp : public OpRewritePattern<UnPackOp> {
 
 } // namespace
 
-void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns) {
+void populateFoldIntoPackAndUnpackPatterns(
+    RewritePatternSet &patterns, const ControlFoldIntoPackUnpackFn &controlFn) {
   patterns.insert<FoldUnpackWithExtractSliceOp, FoldPadWithPackOp,
                   FoldProducerPackWithConsumerLinalgTransposeOp,
                   FoldConsumerPackWithProducerLinalgTransposeOp,
                   FoldConsumerUnPackWithProducerLinalgTransposeOp,
                   FoldProducerUnPackWithConsumerLinalgTransposeOp>(
-      patterns.getContext());
+      patterns.getContext(), controlFn);
 }
 
 void populateSimplifyPackAndUnpackPatterns(RewritePatternSet &patterns) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index eda3373b4d639..5eb3761f7aca1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -261,9 +261,8 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     // 2.a. Skip scalar-like operands.
     Type operandType = operand.getType();
     if (!isa<RankedTensorType>(operandType)) {
-      assert(!isa<ShapedType>(operandType) ||
-             isa<VectorType>(operandType) &&
-                 "Unexpected non-vector ShapedType");
+      assert((!isa<ShapedType>(operandType) || isa<VectorType>(operandType)) &&
+             "Unexpected non-vector ShapedType");
       newOperands.push_back(operand);
       continue;
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index d599ddd220dde..b274502e16903 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -18,9 +18,6 @@
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
@@ -30,7 +27,6 @@
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 using namespace mlir;
@@ -148,6 +144,9 @@ struct LinalgOpInstancePromotionOptions {
   llvm::SmallSet<int64_t, 4> operandsNumbersToCopyIn;
   /// True if the full view should be used for the promoted buffer.
   DenseMap<Value, bool> useFullTileBuffers;
+  /// True if the original subview size should be used. This means the full tile
+  /// buffer is the same size as the partial view.
+  bool useOriginalSubviewSize;
 
   /// Callback functions for allocation and deallocation of promoted buffers, as
   /// well as to copy the data into and out of these buffers.
@@ -170,6 +169,7 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
       options.useFullTileBuffers.value_or(llvm::SmallBitVector());
   vUseFullTileBuffers.resize(linalgOp->getNumOperands(),
                              options.useFullTileBuffersDefault);
+  useOriginalSubviewSize = options.useOriginalSubviewSize;
 
   for (OpOperand &opOperand : linalgOp->getOpOperands()) {
     int64_t operandNumber = opOperand.getOperandNumber();
@@ -237,7 +237,8 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
 // by a partial `copy` op.
 FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
     OpBuilder &b, Location loc, memref::SubViewOp subView,
-    const AllocBufferCallbackFn &allocationFn, DataLayout &layout) {
+    bool useOriginalSubviewSize, const AllocBufferCallbackFn &allocationFn,
+    DataLayout &layout) {
   auto viewType = subView.getType();
   auto rank = viewType.getRank();
   SmallVector<Value, 4> fullSizes;
@@ -254,7 +255,8 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
     // to look for the bound.
     LLVM_DEBUG(llvm::dbgs() << "Extract tightest: " << rangeValue.size << "\n");
     Value size;
-    if (auto attr = llvm::dyn_cast_if_present<Attribute>(rangeValue.size)) {
+    if (llvm::isa_and_present<Attribute>(rangeValue.size) ||
+        useOriginalSubviewSize) {
       size = getValueOrCreateConstantIndexOp(b, loc, rangeValue.size);
     } else {
       FailureOr<int64_t> upperBound =
@@ -295,7 +297,8 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     memref::SubViewOp subView =
         cast<memref::SubViewOp>(v.second.getDefiningOp());
     auto promotionInfo = promoteSubviewAsNewBuffer(
-        b, b.getLoc(), subView, options.allocationFn, layout);
+        b, b.getLoc(), subView, options.useOriginalSubviewSize,
+        options.allocationFn, layout);
     if (failed(promotionInfo))
       return failure();
     promotionInfoMap[v.first] = *promotionInfo;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
index 933f5966cc6ba..455e1a6d146d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
@@ -18,9 +18,7 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Support/TypeID.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/Debug.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_LINALGSPECIALIZEGENERICOPSPASS
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 8a5a2e54cdda2..4741afe8a417d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -10,13 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Linalg/Passes.h"
-
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -26,12 +23,9 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/ValueRange.h"
 #include "mlir/Transforms/FoldUtils.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/CommandLine.h"
 #include <utility>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index a775699f99343..eab74dab4eb75 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -26,16 +26,11 @@
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/Matchers.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
-#include <type_traits>
 #include <utility>
 
 #define DEBUG_TYPE "linalg-transforms"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index 436d485ab9368..092aecceef6b3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -12,14 +12,8 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/ValueRange.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/RWMutex.h"
-#include <memory>
-#include <numeric>
 
 namespace mlir {
 namespace linalg {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index e624f589917d1..934781d1cab75 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -11,7 +11,6 @@
 
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #define DEBUG_TYPE "linalg-transpose-matmul"
 
@@ -38,7 +37,7 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
         matmulOp, "only matmul ops with non-extended semantics are supported");
   }
 
-  if (!bufferization::hasTensorSemantics(matmulOp))
+  if (!matmulOp.hasPureTensorSemantics())
     return rewriter.notifyMatchFailure(
         matmulOp, "only matmul ops with tensors are supported");
 
@@ -93,7 +92,7 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
         batchMatmulOp, "ops with user-defined maps are not supported");
   }
 
-  if (!bufferization::hasTensorSemantics(batchMatmulOp))
+  if (!batchMatmulOp.hasPureTensorSemantics())
     return rewriter.notifyMatchFailure(
         batchMatmulOp, "only matmul ops with tensors are supported");
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 830ae5414c6bd..5a8c5eab3f444 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
@@ -38,12 +37,10 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
-#include <type_traits>
 
 using namespace mlir;
 using namespace mlir::linalg;
@@ -338,7 +335,7 @@ VectorizationState::precomputeIterSpaceValueSizes(RewriterBase &rewriter,
                                                   LinalgOp linalgOp) {
   // TODO: Support 0-d vectors.
   for (int vecDim = 0, end = canonicalVecShape.size(); vecDim < end; ++vecDim) {
-    if (!ShapedType::isDynamic(iterSpaceStaticSizes[vecDim])) {
+    if (ShapedType::isStatic(iterSpaceStaticSizes[vecDim])) {
       // Create constant index op for static dimensions.
       iterSpaceValueSizes.push_back(rewriter.create<arith::ConstantIndexOp>(
           linalgOp.getLoc(), iterSpaceStaticSizes[vecDim]));
@@ -1191,7 +1188,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
     auto transferReadOp = rewriter.create<vector::TransferReadOp>(
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
-        permutationMap, inBounds);
+        /*padding=*/std::nullopt, permutationMap, inBounds);
 
     // Mask this broadcasting xfer_read here rather than relying on the generic
     // path (the generic path assumes identity masking map, which wouldn't be
@@ -1227,8 +1224,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
   }
 
   auto transferReadOp = rewriter.create<vector::TransferReadOp>(
-      loc, resultType, extractOp.getTensor(), transferReadIdxs, permutationMap,
-      inBounds);
+      loc, resultType, extractOp.getTensor(), transferReadIdxs,
+      /*padding=*/std::nullopt, permutationMap, inBounds);
 
   LDBG("Vectorised as contiguous load: " << extractOp);
   return VectorizationHookResult{VectorizationHookStatus::NewOp,
@@ -1384,7 +1381,7 @@ vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
 /// performed to the maximal common vector size implied by the `linalgOp`
 /// iteration space. This eager broadcasting is introduced in the
 /// permutation_map of the vector.transfer_read operations. The eager
-/// broadcasting makes it trivial to detrmine where broadcast, transposes and
+/// broadcasting makes it trivial to determine where broadcast, transposes and
 /// reductions should occur, without any bookkeeping. The tradeoff is that, in
 /// the absence of good canonicalizations, the amount of work increases.
 /// This is not deemed a problem as we expect canonicalizations and foldings to
@@ -1439,7 +1436,8 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
     Operation *read = rewriter.create<vector::TransferReadOp>(
-        loc, readType, opOperand->get(), indices, readMap);
+        loc, readType, opOperand->get(), indices,
+        /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
     Value readValue = read->getResult(0);
 
@@ -1654,13 +1652,13 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
     for (unsigned i = 0; i < vecToStoreRank; i++)
       inBoundsVal[i] =
           (destShape[destRank - vecToStoreRank + i] >= vecToStoreShape[i]) &&
-          !ShapedType::isDynamic(destShape[destRank - vecToStoreRank + i]);
+          ShapedType::isStatic(destShape[destRank - vecToStoreRank + i]);
   }
 
   // If missing, initialize the write indices to 0.
-  assert(writeIndices.empty() ||
-         writeIndices.size() == static_cast<size_t>(destRank) &&
-             "Invalid number of write indices!");
+  assert((writeIndices.empty() ||
+          writeIndices.size() == static_cast<size_t>(destRank)) &&
+         "Invalid number of write indices!");
   if (writeIndices.empty()) {
     auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
     writeIndices.assign(destRank, zero);
@@ -2641,6 +2639,7 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
 
   Value readValue = rewriter.create<vector::TransferReadOp>(
       loc, readType, copyOp.getSource(), indices,
+      /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
     readValue =
@@ -3487,15 +3486,18 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = rewriter.create<vector::TransferReadOp>(loc, lhsType, lhsShaped,
-                                                        lhsPadding);
+    Value lhs = rewriter.create<vector::TransferReadOp>(
+        loc, lhsType, lhsShaped, lhsPadding,
+        /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = rewriter.create<vector::TransferReadOp>(loc, rhsType, rhsShaped,
-                                                    rhsPadding);
-    Value res = rewriter.create<vector::TransferReadOp>(loc, resType, resShaped,
-                                                        resPadding);
+      rhs = rewriter.create<vector::TransferReadOp>(
+          loc, rhsType, rhsShaped, rhsPadding,
+          /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
+    Value res = rewriter.create<vector::TransferReadOp>(
+        loc, resType, resShaped, resPadding,
+        /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
     // The base vectorization case for channeled convolution is input:
     // {n,w,c}, weight: {kw,c,f}, output: {n,w,f}. To reuse the base pattern
@@ -3742,19 +3744,22 @@ struct Conv1DGenerator
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
     Value lhs = rewriter.create<vector::TransferReadOp>(
-        loc, lhsType, lhsShaped, ValueRange{zero, zero, zero});
+        loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
+        /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = rewriter.create<vector::TransferReadOp>(loc, rhsType, rhsShaped,
-                                                        ValueRange{zero, zero});
+    Value rhs = rewriter.create<vector::TransferReadOp>(
+        loc, rhsType, rhsShaped, ValueRange{zero, zero},
+        /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
     Value res = rewriter.create<vector::TransferReadOp>(
-        loc, resType, resShaped, ValueRange{zero, zero, zero});
+        loc, resType, resShaped, ValueRange{zero, zero, zero},
+        /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
         resType.getShape(), resType.getScalableDims(), res.getDefiningOp());
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index 4e90defebcf5c..9fd084487e3fd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/Support/MathExtras.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 209309ddb413a..e89e80b07d20d 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
@@ -31,8 +30,6 @@
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/OpImplementation.h"
-#include "mlir/Pass/Pass.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include <optional>
@@ -697,7 +694,7 @@ computeSliceParameters(OpBuilder &builder, Location loc, Value valueToTile,
     int64_t shapeSize = shape[r];
     std::optional<int64_t> sizeCst = getConstantIntValue(size);
     auto hasTileSizeOne = sizeCst == 1;
-    auto dividesEvenly = sizeCst && !ShapedType::isDynamic(shapeSize) &&
+    auto dividesEvenly = sizeCst && ShapedType::isStatic(shapeSize) &&
                          ((shapeSize % *sizeCst) == 0);
     if (!hasTileSizeOne && !dividesEvenly) {
       LLVM_DEBUG(llvm::dbgs() << "makeTiledShape: shapeSize=" << shapeSize
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 3c4d2562e6999..d1a9920aa66c5 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -99,7 +99,7 @@ static void constifyIndexValues(SmallVectorImpl<OpFoldResult> &values,
          "incorrect number of const values");
   for (auto [i, cstVal] : llvm::enumerate(constValues)) {
     Builder builder(values[i].getContext());
-    if (!ShapedType::isDynamic(cstVal)) {
+    if (ShapedType::isStatic(cstVal)) {
       // Constant value is known, use it directly.
       values[i] = builder.getIndexAttr(cstVal);
       continue;
@@ -189,7 +189,7 @@ struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
     for (unsigned dim = 0, e = memrefType.getRank(); dim < e; ++dim) {
       int64_t dimSize = memrefType.getDimSize(dim);
       // If this is already static dimension, keep it.
-      if (!ShapedType::isDynamic(dimSize)) {
+      if (ShapedType::isStatic(dimSize)) {
         newShapeConstants.push_back(dimSize);
         continue;
       }
@@ -615,21 +615,21 @@ bool CastOp::canFoldIntoConsumerOp(CastOp castOp) {
   for (auto it : llvm::zip(sourceType.getShape(), resultType.getShape())) {
     auto ss = std::get<0>(it), st = std::get<1>(it);
     if (ss != st)
-      if (ShapedType::isDynamic(ss) && !ShapedType::isDynamic(st))
+      if (ShapedType::isDynamic(ss) && ShapedType::isStatic(st))
         return false;
   }
 
   // If cast is towards more static offset along any dimension, don't fold.
   if (sourceOffset != resultOffset)
     if (ShapedType::isDynamic(sourceOffset) &&
-        !ShapedType::isDynamic(resultOffset))
+        ShapedType::isStatic(resultOffset))
       return false;
 
   // If cast is towards more static strides along any dimension, don't fold.
   for (auto it : llvm::zip(sourceStrides, resultStrides)) {
     auto ss = std::get<0>(it), st = std::get<1>(it);
     if (ss != st)
-      if (ShapedType::isDynamic(ss) && !ShapedType::isDynamic(st))
+      if (ShapedType::isDynamic(ss) && ShapedType::isStatic(st))
         return false;
   }
 
@@ -679,7 +679,7 @@ bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
 
     for (unsigned i = 0, e = aT.getRank(); i != e; ++i) {
       int64_t aDim = aT.getDimSize(i), bDim = bT.getDimSize(i);
-      if (!ShapedType::isDynamic(aDim) && !ShapedType::isDynamic(bDim) &&
+      if (ShapedType::isStatic(aDim) && ShapedType::isStatic(bDim) &&
           aDim != bDim)
         return false;
     }
@@ -1862,7 +1862,7 @@ LogicalResult ReinterpretCastOp::verify() {
   // Match sizes in result memref type and in static_sizes attribute.
   for (auto [idx, resultSize, expectedSize] :
        llvm::enumerate(resultType.getShape(), getStaticSizes())) {
-    if (!ShapedType::isDynamic(resultSize) && resultSize != expectedSize)
+    if (ShapedType::isStatic(resultSize) && resultSize != expectedSize)
       return emitError("expected result type with size = ")
              << (ShapedType::isDynamic(expectedSize)
                      ? std::string("dynamic")
@@ -1881,7 +1881,7 @@ LogicalResult ReinterpretCastOp::verify() {
 
   // Match offset in result memref type and in static_offsets attribute.
   int64_t expectedOffset = getStaticOffsets().front();
-  if (!ShapedType::isDynamic(resultOffset) && resultOffset != expectedOffset)
+  if (ShapedType::isStatic(resultOffset) && resultOffset != expectedOffset)
     return emitError("expected result type with offset = ")
            << (ShapedType::isDynamic(expectedOffset)
                    ? std::string("dynamic")
@@ -1891,7 +1891,7 @@ LogicalResult ReinterpretCastOp::verify() {
   // Match strides in result memref type and in static_strides attribute.
   for (auto [idx, resultStride, expectedStride] :
        llvm::enumerate(resultStrides, getStaticStrides())) {
-    if (!ShapedType::isDynamic(resultStride) && resultStride != expectedStride)
+    if (ShapedType::isStatic(resultStride) && resultStride != expectedStride)
       return emitError("expected result type with stride = ")
              << (ShapedType::isDynamic(expectedStride)
                      ? std::string("dynamic")
@@ -1928,7 +1928,7 @@ OpFoldResult ReinterpretCastOp::fold(FoldAdaptor /*operands*/) {
   }
 
   // reinterpret_cast(x) w/o offset/shape/stride changes -> x
-  if (!ShapedType::isDynamicShape(getType().getShape()) &&
+  if (ShapedType::isStaticShape(getType().getShape()) &&
       src.getType() == getType() && getStaticOffsets().front() == 0) {
     return src;
   }
@@ -2379,7 +2379,7 @@ LogicalResult ExpandShapeOp::verify() {
   DenseI64ArrayAttr staticOutputShapes = getStaticOutputShapeAttr();
   ArrayRef<int64_t> resShape = getResult().getType().getShape();
   for (auto [pos, shape] : llvm::enumerate(resShape)) {
-    if (!ShapedType::isDynamic(shape) && shape != staticOutputShapes[pos]) {
+    if (ShapedType::isStatic(shape) && shape != staticOutputShapes[pos]) {
       return emitOpError("invalid output shape provided at pos ") << pos;
     }
   }
@@ -2422,7 +2422,7 @@ computeCollapsedLayoutMap(MemRefType srcType,
     ArrayRef<int64_t> ref = llvm::ArrayRef(reassoc);
     while (srcShape[ref.back()] == 1 && ref.size() > 1)
       ref = ref.drop_back();
-    if (!ShapedType::isDynamic(srcShape[ref.back()]) || ref.size() == 1) {
+    if (ShapedType::isStatic(srcShape[ref.back()]) || ref.size() == 1) {
       resultStrides.push_back(srcStrides[ref.back()]);
     } else {
       // Dynamically-sized dims may turn out to be dims of size 1 at runtime, so
@@ -3463,6 +3463,16 @@ LogicalResult ViewOp::verify() {
 
 Value ViewOp::getViewSource() { return getSource(); }
 
+OpFoldResult ViewOp::fold(FoldAdaptor adaptor) {
+  MemRefType sourceMemrefType = getSource().getType();
+  MemRefType resultMemrefType = getResult().getType();
+
+  if (resultMemrefType == sourceMemrefType && resultMemrefType.hasStaticShape())
+    return getViewSource();
+
+  return {};
+}
+
 namespace {
 
 struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
@@ -3499,7 +3509,7 @@ struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
     for (unsigned dim = 0, e = rank; dim < e; ++dim) {
       int64_t dimSize = memrefType.getDimSize(dim);
       // If this is already static dimension, keep it.
-      if (!ShapedType::isDynamic(dimSize)) {
+      if (ShapedType::isStatic(dimSize)) {
         newShapeConstants.push_back(dimSize);
         continue;
       }
diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
index 637f5ec1c9f9b..9049faccadef3 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
@@ -13,6 +13,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms
   IndependenceTransforms.cpp
   MultiBuffer.cpp
   NormalizeMemRefs.cpp
+  ReifyResultShapes.cpp
   ResolveShapedTypeResultDims.cpp
   RuntimeOpVerification.cpp
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index cfd529c46a41d..2ba798f48ac7c 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -118,7 +118,7 @@ resolveSubviewStridedMetadata(RewriterBase &rewriter,
   // Assert that the computed offset matches the offset of the result type of
   // the subview op (if both are static).
   std::optional<int64_t> computedOffset = getConstantIntValue(finalOffset);
-  if (computedOffset && !ShapedType::isDynamic(resultOffset))
+  if (computedOffset && ShapedType::isStatic(resultOffset))
     assert(*computedOffset == resultOffset &&
            "mismatch between computed offset and result type offset");
 #endif // NDEBUG
@@ -158,7 +158,7 @@ resolveSubviewStridedMetadata(RewriterBase &rewriter,
     // Assert that the computed stride matches the stride of the result type of
     // the subview op (if both are static).
     std::optional<int64_t> computedStride = getConstantIntValue(strides[i]);
-    if (computedStride && !ShapedType::isDynamic(resultStrides[j]))
+    if (computedStride && ShapedType::isStatic(resultStrides[j]))
       assert(*computedStride == resultStrides[j] &&
              "mismatch between computed stride and result type stride");
     ++j;
@@ -458,7 +458,7 @@ getCollapsedSize(memref::CollapseShapeOp collapseShape, OpBuilder &builder,
   MemRefType collapseShapeType = collapseShape.getResultType();
 
   uint64_t size = collapseShapeType.getDimSize(groupId);
-  if (!ShapedType::isDynamic(size)) {
+  if (ShapedType::isStatic(size)) {
     collapsedSize.push_back(builder.getIndexAttr(size));
     return collapsedSize;
   }
@@ -1091,7 +1091,7 @@ class ExtractStridedMetadataOpCastFolder
 
     auto getConstantOrValue = [&rewriter](int64_t constant,
                                           OpFoldResult ofr) -> OpFoldResult {
-      return !ShapedType::isDynamic(constant)
+      return ShapedType::isStatic(constant)
                  ? OpFoldResult(rewriter.getIndexAttr(constant))
                  : ofr;
     };
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index b408962690810..d6fcb8d9f0501 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -297,8 +297,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
                                           /*indexRemap=*/layoutMap,
                                           /*extraOperands=*/{},
                                           /*symbolOperands=*/{},
-                                          /*domOpFilter=*/nullptr,
-                                          /*postDomOpFilter=*/nullptr,
+                                          /*userFilterFn=*/nullptr,
                                           /*allowNonDereferencingOps=*/true,
                                           /*replaceInDeallocOp=*/true))) {
         // If it failed (due to escapes for example), bail out.
@@ -407,8 +406,7 @@ void NormalizeMemRefs::normalizeFuncOpMemRefs(func::FuncOp funcOp,
                                         /*indexRemap=*/layoutMap,
                                         /*extraOperands=*/{},
                                         /*symbolOperands=*/{},
-                                        /*domOpFilter=*/nullptr,
-                                        /*postDomOpFilter=*/nullptr,
+                                        /*userFilterFn=*/nullptr,
                                         /*allowNonDereferencingOps=*/true,
                                         /*replaceInDeallocOp=*/true))) {
       // If it failed (due to escapes for example), bail out. Removing the
@@ -457,8 +455,7 @@ void NormalizeMemRefs::normalizeFuncOpMemRefs(func::FuncOp funcOp,
                                               /*indexRemap=*/layoutMap,
                                               /*extraOperands=*/{},
                                               /*symbolOperands=*/{},
-                                              /*domOpFilter=*/nullptr,
-                                              /*postDomOpFilter=*/nullptr,
+                                              /*userFilterFn=*/nullptr,
                                               /*allowNonDereferencingOps=*/true,
                                               /*replaceInDeallocOp=*/true))) {
             newOp->erase();
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
new file mode 100644
index 0000000000000..e6b9e2f7e8213
--- /dev/null
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -0,0 +1,159 @@
+//===- ReifyResultShapes.cpp - Reify result shapes ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This transform reifies result shapes of `ReifyRankedShapedTypeOpInterface`
+// operations with ranked `memref` and `tensor` results.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/MemRef/Transforms/Passes.h"
+
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Interfaces/DestinationStyleOpInterface.h"
+#include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "llvm/Support/InterleavedRange.h"
+
+#define DEBUG_TYPE "reify-result-shapes"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
+
+namespace mlir {
+namespace memref {
+#define GEN_PASS_DEF_REIFYRESULTSHAPESPASS
+#include "mlir/Dialect/MemRef/Transforms/Passes.h.inc"
+} // namespace memref
+} // namespace mlir
+
+using namespace mlir;
+
+/// Reifies the results of `op`, potentially replacing `op` with a reified
+/// version. Returns `failure` if `mlir::reifyResultShapes` returned failure,
+/// otherwise it always succeeds. Users of this transform should always expect
+/// it to modify the IR, even when it fails. If any of the result types changes,
+/// the transform will insert cast operations to the old type to keep the IR
+/// consistent.
+static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
+                                         ReifyRankedShapedTypeOpInterface op) {
+  LLVM_DEBUG({ DBGS() << " reifying op: " << op << "\n"; });
+  // Get the reified out shapes.
+  ReifiedRankedShapedTypeDims reifiedResultShapes;
+  if (failed(mlir::reifyResultShapes(rewriter, op, reifiedResultShapes)) ||
+      reifiedResultShapes.empty()) {
+    return op->emitWarning() << "failed to get the reified shapes";
+  }
+
+  bool modified = false;
+  // Compute the new output types.
+  SmallVector<Type> outTypes;
+  for (const auto &[oldTy, reifiedShape] :
+       llvm::zip(op->getResultTypes(), reifiedResultShapes)) {
+    // Skip if it's not a memref or tensor type.
+    if (!isa<RankedTensorType, MemRefType>(oldTy)) {
+      outTypes.push_back(oldTy);
+      continue;
+    }
+
+    ShapedType shapedTy = dyn_cast<ShapedType>(oldTy);
+
+    SmallVector<int64_t> shape = llvm::to_vector(shapedTy.getShape());
+    for (auto &&[dim, ofr] : llvm::zip_equal(shape, reifiedShape)) {
+      std::optional<int64_t> maybeCst = getConstantIntValue(ofr);
+      // If the reified dim is dynamic set it appropriately.
+      if (!maybeCst.has_value()) {
+        dim = ShapedType::kDynamic;
+        continue;
+      }
+      // Set the static dim.
+      dim = *maybeCst;
+    }
+
+    // If the shape didn't change continue.
+    if (shape == shapedTy.getShape()) {
+      outTypes.push_back(oldTy);
+      continue;
+    }
+    modified = true;
+    outTypes.push_back(shapedTy.cloneWith(shape, shapedTy.getElementType()));
+  }
+
+  // Return if we don't need to update.
+  if (!modified) {
+    LLVM_DEBUG({ DBGS() << "- op doesn't require update\n"; });
+    return success();
+  }
+
+  LLVM_DEBUG({
+    DBGS() << "- oldTypes: " << llvm::interleaved_array(op->getResultTypes())
+           << " \n";
+    DBGS() << "- outTypes: " << llvm::interleaved_array(outTypes) << " \n";
+  });
+
+  // We now have outTypes that need to be turned to cast ops.
+  Location loc = op->getLoc();
+  SmallVector<Value> newResults;
+  // TODO: `mlir::reifyResultShapes` and op verifiers may not agree atm.
+  // This is a confluence problem that will need to be addressed.
+  // For now, we know PadOp and ConcatOp are fine.
+  assert((isa<tensor::PadOp, tensor::ConcatOp>(op.getOperation())) &&
+         "incorrect op");
+  Operation *newOp = rewriter.clone(*op);
+  for (auto [reifiedTy, oldRes] : llvm::zip(outTypes, op->getResults())) {
+    OpResult newRes = newOp->getResult(oldRes.getResultNumber());
+    Type oldTy = oldRes.getType();
+    // Continue if the type remained invariant or is not shaped.
+    if (oldTy == reifiedTy || !isa<MemRefType, RankedTensorType>(oldTy)) {
+      newResults.push_back(newRes);
+      continue;
+    }
+
+    // Update the type.
+    newRes.setType(reifiedTy);
+    if (isa<RankedTensorType>(reifiedTy)) {
+      newResults.push_back(rewriter.create<tensor::CastOp>(loc, oldTy, newRes));
+    } else {
+      assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
+      newResults.push_back(rewriter.create<memref::CastOp>(loc, oldTy, newRes));
+    }
+  }
+
+  LLVM_DEBUG({
+    DBGS() << "- reified results " << llvm::interleaved_array(newResults)
+           << "\n";
+  });
+  rewriter.replaceOp(op, newResults);
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// Pass registration
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct ReifyResultShapesPass final
+    : public memref::impl::ReifyResultShapesPassBase<ReifyResultShapesPass> {
+  void runOnOperation() override;
+};
+} // namespace
+
+void ReifyResultShapesPass::runOnOperation() {
+  SmallVector<ReifyRankedShapedTypeOpInterface> ops;
+  getOperation()->walk([&](ReifyRankedShapedTypeOpInterface op) {
+    // Handle ops that are not DPS and that do not carry an tied operand shapes.
+    // For now, limit to tensor::PadOp and tensor::ConcatOp.
+    if (!isa<tensor::PadOp, tensor::ConcatOp>(op.getOperation()))
+      return;
+    ops.push_back(op);
+  });
+  IRRewriter rewriter(&getContext());
+  for (ReifyRankedShapedTypeOpInterface op : ops) {
+    rewriter.setInsertionPoint(op);
+    (void)reifyOpResultShapes(rewriter, op);
+  }
+}
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index b8cc91da722f0..5a36984c9013c 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -264,7 +264,7 @@ static void shardShape(const InShape &inShape, const MeshShape &meshShape,
       // add halo sizes if requested
       int haloAxis = 0;
       for (auto [tensorAxis, innerSplitAxes] : llvm::enumerate(splitAxes)) {
-        if (!ShapedType::isDynamic(outShape[tensorAxis]) &&
+        if (ShapedType::isStatic(outShape[tensorAxis]) &&
             !innerSplitAxes.empty()) {
           if (haloSizes[haloAxis * 2] >= 0 &&
               haloSizes[haloAxis * 2 + 1] >= 0) {
@@ -415,7 +415,7 @@ LogicalResult MeshOp::verify() {
     return emitOpError("rank of mesh is expected to be a positive integer");
 
   for (int64_t dimSize : getShape()) {
-    if (dimSize < 0 && !ShapedType::isDynamic(dimSize))
+    if (dimSize < 0 && ShapedType::isStatic(dimSize))
       return emitOpError("dimension size of a mesh is expected to be "
                          "non-negative or dynamic");
   }
@@ -609,7 +609,7 @@ LogicalResult ShardingOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   auto shardedDimsOffsets = getStaticShardedDimsOffsets();
   if (!shardedDimsOffsets.empty()) {
     auto meshShape = mesh.value().getShape();
-    assert(!ShapedType::isDynamicShape(meshShape));
+    assert(ShapedType::isStaticShape(meshShape));
     uint64_t pos = 0;
     for (auto [tensorAxis, innerSplitAxes] : llvm::enumerate(getSplitAxes())) {
       if (!innerSplitAxes.empty()) {
@@ -621,7 +621,7 @@ LogicalResult ShardingOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
           if (shardedDimsOffsets.size() <= pos + i) {
             return emitError() << "sharded dims offsets has wrong size.";
           }
-          if (!ShapedType::isDynamic(shardedDimsOffsets[pos + i])) {
+          if (ShapedType::isStatic(shardedDimsOffsets[pos + i])) {
             if (shardedDimsOffsets[pos + i] < off) {
               return emitError()
                      << "sharded dims offsets must be non-decreasing.";
@@ -1036,8 +1036,8 @@ static LogicalResult verifyInGroupDevice(Location loc, StringRef deviceName,
   }
 
   for (size_t i = 0; i < device.size(); ++i) {
-    if (!ShapedType::isDynamic(device[i]) &&
-        !ShapedType::isDynamic(meshShape[meshAxes[i]]) &&
+    if (ShapedType::isStatic(device[i]) &&
+        ShapedType::isStatic(meshShape[meshAxes[i]]) &&
         meshShape[meshAxes[i]] <= device[i]) {
       return emitError(loc)
              << "Out of bounds coordinate " << i << " for in-group device \""
@@ -1065,8 +1065,7 @@ static LogicalResult verifyDimensionCompatibility(Location loc,
                                                   int64_t expectedDimSize,
                                                   int64_t resultDimSize,
                                                   int64_t resultAxis) {
-  if (!ShapedType::isDynamic(resultDimSize) &&
-      expectedDimSize != resultDimSize) {
+  if (ShapedType::isStatic(resultDimSize) && expectedDimSize != resultDimSize) {
     return emitError(loc) << "Dimension size mismatch for result axis "
                           << resultAxis << ". Expected "
                           << (ShapedType::isDynamic(expectedDimSize)
diff --git a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
index 6751fafaf1776..b823837bf826f 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
@@ -56,10 +56,10 @@ static llvm::raw_ostream &operator<<(llvm::raw_ostream &stream,
 template <typename Stream, typename Range>
 static Stream &printRange(Stream &stream, Range &&range) {
   stream << "[";
-  llvm::for_each(range, [&stream](auto &v) {
+  for (auto &v : range) {
     stream << v;
     stream << ", ";
-  });
+  }
   return stream << "]";
 }
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index 69a80b1a05bf4..a0284b093da94 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -453,8 +453,8 @@ tryUpdateHaloInResharding(ImplicitLocOpBuilder &builder, MeshOp mesh,
   auto srcHaloSizes = sourceSharding.getStaticHaloSizes();
   auto tgtHaloSizes = targetSharding.getStaticHaloSizes();
   assert(srcHaloSizes.empty() || srcHaloSizes.size() == tgtHaloSizes.size());
-  assert(((srcHaloSizes.empty() || !ShapedType::isDynamicShape(srcHaloSizes)) &&
-          !ShapedType::isDynamicShape(tgtHaloSizes) &&
+  assert(((srcHaloSizes.empty() || ShapedType::isStaticShape(srcHaloSizes)) &&
+          ShapedType::isStaticShape(tgtHaloSizes) &&
           sourceShard.getType().hasStaticShape()) &&
          "dynamic shapes/halos are not supported yet for mesh-spmdization");
   auto rank = sourceShard.getType().getRank();
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 37acb6acbfa91..80c807e774a7e 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -3169,6 +3169,53 @@ void ExitDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
   results.add<RemoveConstantIfCondition<ExitDataOp>>(context);
 }
 
+void ExitDataOp::addAsyncOnly(MLIRContext *context,
+                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  setAsyncAttr(mlir::UnitAttr::get(context));
+}
+
+void ExitDataOp::addAsyncOperand(
+    MLIRContext *context, mlir::Value newValue,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  getAsyncOperandMutable().append(newValue);
+}
+
+void ExitDataOp::addWaitOnly(MLIRContext *context,
+                             llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  setWaitAttr(mlir::UnitAttr::get(context));
+}
+
+void ExitDataOp::addWaitOperands(
+    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  // if hasDevnum, the first value is the devnum. The 'rest' go into the
+  // operands list.
+  if (hasDevnum) {
+    getWaitDevnumMutable().append(newValues.front());
+    newValues = newValues.drop_front();
+  }
+
+  getWaitOperandsMutable().append(newValues);
+}
+
 //===----------------------------------------------------------------------===//
 // EnterDataOp
 //===----------------------------------------------------------------------===//
@@ -3218,6 +3265,53 @@ void EnterDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
   results.add<RemoveConstantIfCondition<EnterDataOp>>(context);
 }
 
+void EnterDataOp::addAsyncOnly(
+    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  setAsyncAttr(mlir::UnitAttr::get(context));
+}
+
+void EnterDataOp::addAsyncOperand(
+    MLIRContext *context, mlir::Value newValue,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  getAsyncOperandMutable().append(newValue);
+}
+
+void EnterDataOp::addWaitOnly(MLIRContext *context,
+                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  setWaitAttr(mlir::UnitAttr::get(context));
+}
+
+void EnterDataOp::addWaitOperands(
+    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  // if hasDevnum, the first value is the devnum. The 'rest' go into the
+  // operands list.
+  if (hasDevnum) {
+    getWaitDevnumMutable().append(newValues.front());
+    newValues = newValues.drop_front();
+  }
+
+  getWaitOperandsMutable().append(newValues);
+}
+
 //===----------------------------------------------------------------------===//
 // AtomicReadOp
 //===----------------------------------------------------------------------===//
@@ -3760,6 +3854,49 @@ mlir::Value UpdateOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
                             deviceType);
 }
 
+void UpdateOp::addAsyncOnly(MLIRContext *context,
+                            llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
+      context, getAsyncOnlyAttr(), effectiveDeviceTypes));
+}
+
+void UpdateOp::addAsyncOperand(
+    MLIRContext *context, mlir::Value newValue,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
+      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,
+      getAsyncOperandsMutable()));
+}
+
+void UpdateOp::addWaitOnly(MLIRContext *context,
+                           llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),
+                                                     effectiveDeviceTypes));
+}
+
+void UpdateOp::addWaitOperands(
+    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+
+  llvm::SmallVector<int32_t> segments;
+  if (getWaitOperandsSegments())
+    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));
+
+  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
+      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,
+      getWaitOperandsMutable(), segments));
+  setWaitOperandsSegments(segments);
+
+  llvm::SmallVector<mlir::Attribute> hasDevnums;
+  if (getHasWaitDevnumAttr())
+    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));
+  hasDevnums.insert(
+      hasDevnums.end(),
+      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),
+      mlir::BoolAttr::get(context, hasDevnum));
+  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));
+}
+
 //===----------------------------------------------------------------------===//
 // WaitOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e94d570b57122..ffc84781f77ff 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -15,11 +15,13 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
+#include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/FoldInterfaces.h"
 
 #include "llvm/ADT/ArrayRef.h"
@@ -2640,6 +2642,23 @@ LogicalResult SimdOp::verify() {
     return emitError()
            << "'omp.composite' attribute present in non-composite wrapper";
 
+  // Firstprivate is not allowed for SIMD in the standard. Check that none of
+  // the private decls are for firstprivate.
+  std::optional<ArrayAttr> privateSyms = getPrivateSyms();
+  if (privateSyms) {
+    for (const Attribute &sym : *privateSyms) {
+      auto symRef = cast<SymbolRefAttr>(sym);
+      omp::PrivateClauseOp privatizer =
+          SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
+              getOperation(), symRef);
+      if (!privatizer)
+        return emitError() << "Cannot find privatizer '" << symRef << "'";
+      if (privatizer.getDataSharingType() ==
+          DataSharingClauseType::FirstPrivate)
+        return emitError() << "FIRSTPRIVATE cannot be used with SIMD";
+    }
+  }
+
   return success();
 }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 79012dbd32f80..5a3bd984530db 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -1175,13 +1175,11 @@ LogicalResult ForallOp::verify() {
       return emitOpError("type mismatch between ")
              << i << "-th output and corresponding block argument";
   if (getMapping().has_value() && !getMapping()->empty()) {
-    if (static_cast<int64_t>(getMapping()->size()) != numLoops)
+    if (getDeviceMappingAttrs().size() != numLoops)
       return emitOpError() << "mapping attribute size must match op rank";
-    for (auto map : getMapping()->getValue()) {
-      if (!isa<DeviceMappingAttrInterface>(map))
-        return emitOpError()
-               << getMappingAttrName() << " is not device mapping attribute";
-    }
+    if (failed(getDeviceMaskingAttr()))
+      return emitOpError() << getMappingAttrName()
+                           << " supports at most one device masking attribute";
   }
 
   // Verify mixed static/dynamic control variables.
@@ -1435,6 +1433,39 @@ SmallVector<Operation *> ForallOp::getCombiningOps(BlockArgument bbArg) {
   return storeOps;
 }
 
+SmallVector<DeviceMappingAttrInterface> ForallOp::getDeviceMappingAttrs() {
+  SmallVector<DeviceMappingAttrInterface> res;
+  if (!getMapping())
+    return res;
+  for (auto attr : getMapping()->getValue()) {
+    auto m = dyn_cast<DeviceMappingAttrInterface>(attr);
+    if (m)
+      res.push_back(m);
+  }
+  return res;
+}
+
+FailureOr<DeviceMaskingAttrInterface> ForallOp::getDeviceMaskingAttr() {
+  DeviceMaskingAttrInterface res;
+  if (!getMapping())
+    return res;
+  for (auto attr : getMapping()->getValue()) {
+    auto m = dyn_cast<DeviceMaskingAttrInterface>(attr);
+    if (m && res)
+      return failure();
+    if (m)
+      res = m;
+  }
+  return res;
+}
+
+bool ForallOp::usesLinearMapping() {
+  SmallVector<DeviceMappingAttrInterface> ifaces = getDeviceMappingAttrs();
+  if (ifaces.empty())
+    return false;
+  return ifaces.front().isLinearMapping();
+}
+
 std::optional<SmallVector<Value>> ForallOp::getLoopInductionVars() {
   return SmallVector<Value>{getBody()->getArguments().take_front(getRank())};
 }
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 58562536be61f..d36d91249ed36 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -274,7 +274,7 @@ struct IfOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -313,15 +313,15 @@ struct IfOpInterface
 
     // Best case: Both branches have the exact same buffer type.
     if (thenBufferType == elseBufferType)
-      return thenBufferType;
+      return cast<BufferLikeType>(thenBufferType);
 
     // Memory space mismatch.
     if (thenBufferType.getMemorySpace() != elseBufferType.getMemorySpace())
       return op->emitError("inconsistent memory space on then/else branches");
 
     // Layout maps are different: Promote to fully dynamic layout map.
-    return getMemRefTypeWithFullyDynamicLayout(
-        cast<TensorType>(opResult.getType()), thenBufferType.getMemorySpace());
+    return cast<BufferLikeType>(getMemRefTypeWithFullyDynamicLayout(
+        cast<TensorType>(opResult.getType()), thenBufferType.getMemorySpace()));
   }
 };
 
@@ -392,7 +392,7 @@ struct IndexSwitchOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -436,7 +436,7 @@ struct IndexSwitchOpInterface
           cast<TensorType>(value.getType()), bufferType.getMemorySpace());
     }
 
-    return bufferType;
+    return cast<BufferLikeType>(bufferType);
   }
 };
 
@@ -522,13 +522,13 @@ getBbArgReplacements(RewriterBase &rewriter, Block::BlockArgListType bbArgs,
 /// If both buffer types are equal, no casts are needed the computed buffer type
 /// can be used directly. Otherwise, the buffer types can only differ in their
 /// layout map and a cast must be inserted.
-static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
+static FailureOr<BufferLikeType> computeLoopRegionIterArgBufferType(
     Operation *loopOp, BlockArgument iterArg, Value initArg, Value yieldedValue,
     const BufferizationOptions &options, const BufferizationState &state,
     SmallVector<Value> &invocationStack) {
   // Determine the buffer type of the init_arg.
-  auto initArgBufferType = bufferization::detail::asMemRefType(
-      bufferization::getBufferType(initArg, options, state, invocationStack));
+  auto initArgBufferType =
+      bufferization::getBufferType(initArg, options, state, invocationStack);
   if (failed(initArgBufferType))
     return failure();
 
@@ -547,16 +547,15 @@ static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
   }
 
   // Compute the buffer type of the yielded value.
-  BaseMemRefType yieldedValueBufferType;
+  BufferLikeType yieldedValueBufferType;
   if (isa<BaseMemRefType>(yieldedValue.getType())) {
     // scf.yield was already bufferized.
-    yieldedValueBufferType = cast<BaseMemRefType>(yieldedValue.getType());
+    yieldedValueBufferType = cast<BufferLikeType>(yieldedValue.getType());
   } else {
     // Note: This typically triggers a recursive call for the buffer type of
     // the iter_arg.
-    auto maybeBufferType =
-        bufferization::detail::asMemRefType(bufferization::getBufferType(
-            yieldedValue, options, state, invocationStack));
+    auto maybeBufferType = bufferization::getBufferType(yieldedValue, options,
+                                                        state, invocationStack);
     if (failed(maybeBufferType))
       return failure();
     yieldedValueBufferType = *maybeBufferType;
@@ -584,8 +583,8 @@ static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
         "expected same shape");
   }
 #endif // NDEBUG
-  return getMemRefTypeWithFullyDynamicLayout(
-      iterTensorType, yieldedBufferType.getMemorySpace());
+  return cast<BufferLikeType>(getMemRefTypeWithFullyDynamicLayout(
+      iterTensorType, yieldedBufferType.getMemorySpace()));
 }
 
 /// Return `true` if the given loop may have 0 iterations.
@@ -708,7 +707,7 @@ struct ForOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -719,12 +718,8 @@ struct ForOpInterface
     if (auto opResult = dyn_cast<OpResult>(value)) {
       // The type of an OpResult must match the corresponding iter_arg type.
       BlockArgument bbArg = forOp.getTiedLoopRegionIterArg(opResult);
-      auto bufferType =
-          bufferization::getBufferType(bbArg, options, state, invocationStack);
-      if (failed(bufferType))
-        return failure();
-      assert(isa<BaseMemRefType>(*bufferType) && "expected memref type");
-      return cast<BaseMemRefType>(*bufferType);
+      return bufferization::getBufferType(bbArg, options, state,
+                                          invocationStack);
     }
 
     // Compute result/argument number.
@@ -1047,7 +1042,7 @@ struct WhileOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -1081,10 +1076,10 @@ struct WhileOpInterface
     Value conditionYieldedVal = whileOp.getConditionOp().getArgs()[resultNum];
     if (!isa<TensorType>(conditionYieldedVal.getType())) {
       // scf.condition was already bufferized.
-      return cast<BaseMemRefType>(conditionYieldedVal.getType());
+      return cast<BufferLikeType>(conditionYieldedVal.getType());
     }
-    return bufferization::detail::asMemRefType(bufferization::getBufferType(
-        conditionYieldedVal, options, state, invocationStack));
+    return bufferization::getBufferType(conditionYieldedVal, options, state,
+                                        invocationStack);
   }
 
   /// Assert that yielded values of an scf.while op are equivalent to their
@@ -1303,7 +1298,7 @@ struct ForallOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -1312,15 +1307,15 @@ struct ForallOpInterface
     if (auto bbArg = dyn_cast<BlockArgument>(value))
       // A tensor block argument has the same bufferized type as the
       // corresponding output operand.
-      return bufferization::detail::asMemRefType(
-          bufferization::getBufferType(forallOp.getTiedOpOperand(bbArg)->get(),
-                                       options, state, invocationStack));
+      return bufferization::getBufferType(
+          forallOp.getTiedOpOperand(bbArg)->get(), options, state,
+          invocationStack);
 
     // The bufferized result type is the same as the bufferized type of the
     // corresponding output operand.
-    return bufferization::detail::asMemRefType(bufferization::getBufferType(
+    return bufferization::getBufferType(
         forallOp.getOutputs()[cast<OpResult>(value).getResultNumber()], options,
-        state, invocationStack));
+        state, invocationStack);
   }
 
   bool isRepetitiveRegion(Operation *op, unsigned index) const {
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index 4aacbe739ca5d..bcecef5e6e0a9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -106,6 +106,20 @@ bool LoopPipelinerInternal::initializeLoopInfo(
   lb = forOp.getLowerBound();
   step = forOp.getStep();
 
+  std::vector<std::pair<Operation *, unsigned>> schedule;
+  options.getScheduleFn(forOp, schedule);
+  if (schedule.empty()) {
+    LDBG("--empty schedule -> BAIL");
+    return false;
+  }
+
+  opOrder.reserve(schedule.size());
+  for (auto &opSchedule : schedule) {
+    maxStage = std::max(maxStage, opSchedule.second);
+    stages[opSchedule.first] = opSchedule.second;
+    opOrder.push_back(opSchedule.first);
+  }
+
   dynamicLoop = true;
   auto upperBoundCst = getConstantIntValue(ub);
   auto lowerBoundCst = getConstantIntValue(lb);
@@ -124,7 +138,7 @@ bool LoopPipelinerInternal::initializeLoopInfo(
       return false;
     }
     int64_t numIteration = llvm::divideCeilSigned(ubImm - lbImm, stepImm);
-    if (numIteration > maxStage) {
+    if (numIteration >= maxStage) {
       dynamicLoop = false;
     } else if (!options.supportDynamicLoops) {
       LDBG("--fewer loop iterations than pipeline stages -> BAIL");
@@ -137,19 +151,6 @@ bool LoopPipelinerInternal::initializeLoopInfo(
     LDBG("--no epilogue or predicate set -> BAIL");
     return false;
   }
-  std::vector<std::pair<Operation *, unsigned>> schedule;
-  options.getScheduleFn(forOp, schedule);
-  if (schedule.empty()) {
-    LDBG("--empty schedule -> BAIL");
-    return false;
-  }
-
-  opOrder.reserve(schedule.size());
-  for (auto &opSchedule : schedule) {
-    maxStage = std::max(maxStage, opSchedule.second);
-    stages[opSchedule.first] = opSchedule.second;
-    opOrder.push_back(opSchedule.first);
-  }
 
   // All operations need to have a stage.
   for (Operation &op : forOp.getBody()->without_terminator()) {
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 8ab5bdc0c5dc5..f4047be68ccf2 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -301,19 +301,8 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
 /// constants, or optional otherwise. Trip count is computed as
 /// ceilDiv(highBound - lowBound, step).
 static std::optional<int64_t> getConstantTripCount(scf::ForOp forOp) {
-  std::optional<int64_t> lbCstOp = getConstantIntValue(forOp.getLowerBound());
-  std::optional<int64_t> ubCstOp = getConstantIntValue(forOp.getUpperBound());
-  std::optional<int64_t> stepCstOp = getConstantIntValue(forOp.getStep());
-  if (!lbCstOp.has_value() || !ubCstOp.has_value() || !stepCstOp.has_value())
-    return {};
-
-  // Constant loop bounds computation.
-  int64_t lbCst = lbCstOp.value();
-  int64_t ubCst = ubCstOp.value();
-  int64_t stepCst = stepCstOp.value();
-  assert(lbCst >= 0 && ubCst >= 0 && stepCst > 0 &&
-         "expected positive loop bounds and step");
-  return llvm::divideCeilSigned(ubCst - lbCst, stepCst);
+  return constantTripCount(forOp.getLowerBound(), forOp.getUpperBound(),
+                           forOp.getStep());
 }
 
 /// Generates unrolled copies of scf::ForOp 'loopBodyBlock', with
diff --git a/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp b/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp
index 2ff3efdc96a7f..fa20cc179f892 100644
--- a/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp
@@ -23,7 +23,8 @@ namespace mlir::spirv {
 
 static LogicalResult
 verifyCoopMatrixAccess(Operation *op, Type pointer, Type coopMatrix,
-                       spirv::MemoryAccessAttr memoryOperand) {
+                       spirv::MemoryAccessAttr memoryOperand,
+                       IntegerAttr alignment) {
   auto pointerType = cast<PointerType>(pointer);
   Type pointeeType = pointerType.getPointeeType();
   if (!isa<ScalarType, VectorType>(pointeeType)) {
@@ -49,13 +50,18 @@ verifyCoopMatrixAccess(Operation *op, Type pointer, Type coopMatrix,
           "not compatible with memory operand 'MakePointerVisible'");
     }
 
-    // The 'Aligned' memory operand requires an alignment literal to follow,
-    // which needs to be implemented on the level of op parsing and
-    // (de-)serialization.
-    // TODO: Consider adding support for this attribute value.
-    if (spirv::bitEnumContainsAll(memoryOperand.getValue(),
-                                  spirv::MemoryAccess::Aligned)) {
-      return op->emitOpError("has unhandled memory operand 'Aligned'");
+    // TODO: Need to check that NonPrivatePointer is set for MakePointer*. See
+    // #145485.
+
+    if (spirv::bitEnumContainsAll(operandSet, spirv::MemoryAccess::Aligned) &&
+        !alignment) {
+      return op->emitOpError("missing value for the 'Aligned' memory operand");
+    }
+
+    if (!spirv::bitEnumContainsAll(operandSet, spirv::MemoryAccess::Aligned) &&
+        alignment) {
+      return op->emitOpError(
+          "found alignment attribute for non-'Aligned' memory operand");
     }
   }
 
@@ -72,7 +78,8 @@ verifyCoopMatrixAccess(Operation *op, Type pointer, Type coopMatrix,
 
 LogicalResult KHRCooperativeMatrixLoadOp::verify() {
   return verifyCoopMatrixAccess(*this, getPointer().getType(),
-                                getResult().getType(), getMemoryOperandAttr());
+                                getResult().getType(), getMemoryOperandAttr(),
+                                getAlignmentAttr());
 }
 
 //===----------------------------------------------------------------------===//
@@ -81,7 +88,8 @@ LogicalResult KHRCooperativeMatrixLoadOp::verify() {
 
 LogicalResult KHRCooperativeMatrixStoreOp::verify() {
   return verifyCoopMatrixAccess(*this, getPointer().getType(),
-                                getObject().getType(), getMemoryOperandAttr());
+                                getObject().getType(), getMemoryOperandAttr(),
+                                getAlignmentAttr());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index a21acef1c4b43..88c7adf3dfcb3 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -194,6 +194,13 @@ static Type parseAndVerifyType(SPIRVDialect const &dialect,
           << t.getNumElements();
       return Type();
     }
+  } else if (auto t = dyn_cast<TensorArmType>(type)) {
+    if (!isa<ScalarType>(t.getElementType())) {
+      parser.emitError(
+          typeLoc, "only scalar element type allowed in tensor type but found ")
+          << t.getElementType();
+      return Type();
+    }
   } else {
     parser.emitError(typeLoc, "cannot use ")
         << type << " to compose SPIR-V types";
@@ -363,6 +370,52 @@ static Type parseCooperativeMatrixType(SPIRVDialect const &dialect,
   return CooperativeMatrixType::get(elementTy, dims[0], dims[1], scope, use);
 }
 
+// tensor-arm-type ::=
+//   `!spirv.arm.tensor` `<` dim0 `x` dim1 `x` ... `x` dimN `x` element-type`>`
+static Type parseTensorArmType(SPIRVDialect const &dialect,
+                               DialectAsmParser &parser) {
+  if (parser.parseLess())
+    return {};
+
+  bool unranked = false;
+  SmallVector<int64_t, 4> dims;
+  SMLoc countLoc = parser.getCurrentLocation();
+
+  if (parser.parseOptionalStar().succeeded()) {
+    unranked = true;
+    if (parser.parseXInDimensionList())
+      return {};
+  } else if (parser.parseDimensionList(dims, /*allowDynamic=*/true)) {
+    return {};
+  }
+
+  if (!unranked && dims.empty()) {
+    parser.emitError(countLoc, "arm.tensors do not support rank zero");
+    return {};
+  }
+
+  if (llvm::is_contained(dims, 0)) {
+    parser.emitError(countLoc, "arm.tensors do not support zero dimensions");
+    return {};
+  }
+
+  if (llvm::any_of(dims, [](int64_t dim) { return dim < 0; }) &&
+      llvm::any_of(dims, [](int64_t dim) { return dim > 0; })) {
+    parser.emitError(countLoc, "arm.tensor shape dimensions must be either "
+                               "fully dynamic or completed shaped");
+    return {};
+  }
+
+  auto elementTy = parseAndVerifyType(dialect, parser);
+  if (!elementTy)
+    return {};
+
+  if (parser.parseGreater())
+    return {};
+
+  return TensorArmType::get(dims, elementTy);
+}
+
 // TODO: Reorder methods to be utilities first and parse*Type
 // methods in alphabetical order
 //
@@ -759,6 +812,8 @@ Type SPIRVDialect::parseType(DialectAsmParser &parser) const {
     return parseStructType(*this, parser);
   if (keyword == "matrix")
     return parseMatrixType(*this, parser);
+  if (keyword == "arm.tensor")
+    return parseTensorArmType(*this, parser);
   parser.emitError(parser.getNameLoc(), "unknown SPIR-V type: ") << keyword;
   return Type();
 }
@@ -855,10 +910,28 @@ static void print(MatrixType type, DialectAsmPrinter &os) {
   os << ">";
 }
 
+static void print(TensorArmType type, DialectAsmPrinter &os) {
+  os << "arm.tensor<";
+
+  llvm::interleave(
+      type.getShape(), os,
+      [&](int64_t dim) {
+        if (ShapedType::isDynamic(dim))
+          os << '?';
+        else
+          os << dim;
+      },
+      "x");
+  if (!type.hasRank()) {
+    os << "*";
+  }
+  os << "x" << type.getElementType() << ">";
+}
+
 void SPIRVDialect::printType(Type type, DialectAsmPrinter &os) const {
   TypeSwitch<Type>(type)
       .Case<ArrayType, CooperativeMatrixType, PointerType, RuntimeArrayType,
-            ImageType, SampledImageType, StructType, MatrixType>(
+            ImageType, SampledImageType, StructType, MatrixType, TensorArmType>(
           [&](auto type) { print(type, os); })
       .Default([](Type) { llvm_unreachable("unhandled SPIR-V type"); });
 }
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 7148027dae78d..eb2974d62fdd1 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -547,6 +547,12 @@ ParseResult spirv::ConstantOp::parse(OpAsmParser &parser,
       return failure();
   }
 
+  if (llvm::isa<TensorArmType>(type)) {
+    if (parser.parseOptionalColon().succeeded())
+      if (parser.parseType(type))
+        return failure();
+  }
+
   return parser.addTypeToList(type, result.types);
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
index 93e0c9b33c546..2b90df42af5cc 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
@@ -18,8 +18,9 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 
+#include <algorithm>
 #include <cstdint>
-#include <iterator>
+#include <numeric>
 
 using namespace mlir;
 using namespace mlir::spirv;
@@ -96,7 +97,7 @@ bool CompositeType::classof(Type type) {
     return isValid(vectorType);
   return llvm::isa<spirv::ArrayType, spirv::CooperativeMatrixType,
                    spirv::MatrixType, spirv::RuntimeArrayType,
-                   spirv::StructType>(type);
+                   spirv::StructType, spirv::TensorArmType>(type);
 }
 
 bool CompositeType::isValid(VectorType type) {
@@ -107,8 +108,8 @@ bool CompositeType::isValid(VectorType type) {
 
 Type CompositeType::getElementType(unsigned index) const {
   return TypeSwitch<Type, Type>(*this)
-      .Case<ArrayType, CooperativeMatrixType, RuntimeArrayType, VectorType>(
-          [](auto type) { return type.getElementType(); })
+      .Case<ArrayType, CooperativeMatrixType, RuntimeArrayType, VectorType,
+            TensorArmType>([](auto type) { return type.getElementType(); })
       .Case<MatrixType>([](MatrixType type) { return type.getColumnType(); })
       .Case<StructType>(
           [index](StructType type) { return type.getElementType(index); })
@@ -125,6 +126,8 @@ unsigned CompositeType::getNumElements() const {
     return structType.getNumElements();
   if (auto vectorType = llvm::dyn_cast<VectorType>(*this))
     return vectorType.getNumElements();
+  if (auto tensorArmType = dyn_cast<TensorArmType>(*this))
+    return tensorArmType.getNumElements();
   if (llvm::isa<CooperativeMatrixType>(*this)) {
     llvm_unreachable(
         "invalid to query number of elements of spirv Cooperative Matrix type");
@@ -151,6 +154,13 @@ void CompositeType::getExtensions(
         return llvm::cast<ScalarType>(type.getElementType())
             .getExtensions(extensions, storage);
       })
+      .Case<TensorArmType>([&](TensorArmType type) {
+        static constexpr Extension ext{Extension::SPV_ARM_tensors};
+        extensions.push_back(ext);
+        return llvm::cast<ScalarType>(type.getElementType())
+            .getExtensions(extensions, storage);
+      })
+
       .Default([](Type) { llvm_unreachable("invalid composite type"); });
 }
 
@@ -171,6 +181,12 @@ void CompositeType::getCapabilities(
         return llvm::cast<ScalarType>(type.getElementType())
             .getCapabilities(capabilities, storage);
       })
+      .Case<TensorArmType>([&](TensorArmType type) {
+        static constexpr Capability cap{Capability::TensorsARM};
+        capabilities.push_back(cap);
+        return llvm::cast<ScalarType>(type.getElementType())
+            .getCapabilities(capabilities, storage);
+      })
       .Default([](Type) { llvm_unreachable("invalid composite type"); });
 }
 
@@ -186,6 +202,13 @@ std::optional<int64_t> CompositeType::getSizeInBytes() {
       return std::nullopt;
     return *elementSize * vectorType.getNumElements();
   }
+  if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(*this)) {
+    std::optional<int64_t> elementSize =
+        llvm::cast<ScalarType>(tensorArmType.getElementType()).getSizeInBytes();
+    if (!elementSize)
+      return std::nullopt;
+    return *elementSize * tensorArmType.getNumElements();
+  }
   return std::nullopt;
 }
 
@@ -691,6 +714,8 @@ bool SPIRVType::classof(Type type) {
     return true;
   if (auto vectorType = llvm::dyn_cast<VectorType>(type))
     return CompositeType::isValid(vectorType);
+  if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(type))
+    return llvm::isa<ScalarType>(tensorArmType.getElementType());
   return false;
 }
 
@@ -712,6 +737,8 @@ void SPIRVType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
     matrixType.getExtensions(extensions, storage);
   } else if (auto ptrType = llvm::dyn_cast<PointerType>(*this)) {
     ptrType.getExtensions(extensions, storage);
+  } else if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(*this)) {
+    tensorArmType.getExtensions(extensions, storage);
   } else {
     llvm_unreachable("invalid SPIR-V Type to getExtensions");
   }
@@ -732,6 +759,8 @@ void SPIRVType::getCapabilities(
     matrixType.getCapabilities(capabilities, storage);
   } else if (auto ptrType = llvm::dyn_cast<PointerType>(*this)) {
     ptrType.getCapabilities(capabilities, storage);
+  } else if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(*this)) {
+    tensorArmType.getCapabilities(capabilities, storage);
   } else {
     llvm_unreachable("invalid SPIR-V Type to getCapabilities");
   }
@@ -1203,11 +1232,85 @@ void MatrixType::getCapabilities(
   llvm::cast<SPIRVType>(getColumnType()).getCapabilities(capabilities, storage);
 }
 
+//===----------------------------------------------------------------------===//
+// TensorArmType
+//===----------------------------------------------------------------------===//
+
+struct spirv::detail::TensorArmTypeStorage final : TypeStorage {
+  using KeyTy = std::tuple<ArrayRef<int64_t>, Type>;
+
+  static TensorArmTypeStorage *construct(TypeStorageAllocator &allocator,
+                                         const KeyTy &key) {
+    auto [shape, elementType] = key;
+    shape = allocator.copyInto(shape);
+    return new (allocator.allocate<TensorArmTypeStorage>())
+        TensorArmTypeStorage(shape, elementType);
+  }
+
+  static llvm::hash_code hashKey(const KeyTy &key) {
+    auto [shape, elementType] = key;
+    return llvm::hash_combine(shape, elementType);
+  }
+
+  bool operator==(const KeyTy &key) const {
+    return key == KeyTy(shape, elementType);
+  }
+
+  TensorArmTypeStorage(ArrayRef<int64_t> shape, Type elementType)
+      : shape(std::move(shape)), elementType(std::move(elementType)) {}
+
+  ArrayRef<int64_t> shape;
+  Type elementType;
+};
+
+TensorArmType TensorArmType::get(ArrayRef<int64_t> shape, Type elementType) {
+  return Base::get(elementType.getContext(), shape, elementType);
+}
+
+TensorArmType TensorArmType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
+                                       Type elementType) const {
+  return TensorArmType::get(shape.value_or(getShape()), elementType);
+}
+
+Type TensorArmType::getElementType() const { return getImpl()->elementType; }
+ArrayRef<int64_t> TensorArmType::getShape() const { return getImpl()->shape; }
+
+void TensorArmType::getExtensions(
+    SPIRVType::ExtensionArrayRefVector &extensions,
+    std::optional<StorageClass> storage) {
+
+  llvm::cast<SPIRVType>(getElementType()).getExtensions(extensions, storage);
+  static constexpr Extension ext{Extension::SPV_ARM_tensors};
+  extensions.push_back(ext);
+}
+
+void TensorArmType::getCapabilities(
+    SPIRVType::CapabilityArrayRefVector &capabilities,
+    std::optional<StorageClass> storage) {
+  llvm::cast<SPIRVType>(getElementType())
+      .getCapabilities(capabilities, storage);
+  static constexpr Capability cap{Capability::TensorsARM};
+  capabilities.push_back(cap);
+}
+
+LogicalResult
+TensorArmType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
+                                ArrayRef<int64_t> shape, Type elementType) {
+  if (llvm::is_contained(shape, 0))
+    return emitError() << "arm.tensor do not support dimensions = 0";
+  if (llvm::any_of(shape, [](int64_t dim) { return dim < 0; }) &&
+      llvm::any_of(shape, [](int64_t dim) { return dim > 0; }))
+    return emitError()
+           << "arm.tensor shape dimensions must be either fully dynamic or "
+              "completed shaped";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // SPIR-V Dialect
 //===----------------------------------------------------------------------===//
 
 void SPIRVDialect::registerTypes() {
   addTypes<ArrayType, CooperativeMatrixType, ImageType, MatrixType, PointerType,
-           RuntimeArrayType, SampledImageType, StructType>();
+           RuntimeArrayType, SampledImageType, StructType, TensorArmType>();
 }
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 34ae83b25c397..ea7918fa14f23 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -518,7 +518,7 @@ SparseTensorEncodingAttr::translateShape(ArrayRef<int64_t> srcShape,
   SmallVector<AffineExpr> dimRep;
   dimRep.reserve(srcShape.size());
   for (int64_t sz : srcShape) {
-    if (!ShapedType::isDynamic(sz)) {
+    if (ShapedType::isStatic(sz)) {
       // Push back the max coordinate for the given dimension/level size.
       dimRep.push_back(getAffineConstantExpr(sz - 1, getContext()));
     } else {
@@ -1531,7 +1531,7 @@ OpFoldResult LvlOp::fold(FoldAdaptor adaptor) {
   };
 
   SmallVector<Size> lvlShape = stt.getLvlShape();
-  if (!ShapedType::isDynamic(lvlShape[lvl]))
+  if (ShapedType::isStatic(lvlShape[lvl]))
     return getIndexAttr(lvlShape[lvl]);
 
   return {};
@@ -1876,7 +1876,7 @@ LogicalResult ConcatenateOp::verify() {
   for (Dimension d = 0; d < dimRank; d++) {
     const Size dstSh = dstTp.getDimShape()[d];
     if (d == concatDim) {
-      if (!ShapedType::isDynamic(dstSh)) {
+      if (ShapedType::isStatic(dstSh)) {
         // If we reach here, then all inputs have static shapes.  So we
         // can use `getDimShape()[d]` instead of `*getDynamicDimSize(d)`
         // to avoid redundant assertions in the loop.
@@ -1894,7 +1894,7 @@ LogicalResult ConcatenateOp::verify() {
       Size prev = dstSh;
       for (const auto src : getInputs()) {
         const auto sh = getSparseTensorType(src).getDimShape()[d];
-        if (!ShapedType::isDynamic(prev) && sh != prev)
+        if (ShapedType::isStatic(prev) && sh != prev)
           return emitError("All dimensions (expect for the concatenating one) "
                            "should be equal.");
         prev = sh;
@@ -2058,7 +2058,7 @@ LogicalResult SortOp::verify() {
   const auto checkDim = [&](Value v, Size minSize,
                             const char *message) -> LogicalResult {
     const Size sh = getMemRefType(v).getShape()[0];
-    if (!ShapedType::isDynamic(sh) && sh < minSize)
+    if (ShapedType::isStatic(sh) && sh < minSize)
       return emitError(
           llvm::formatv("{0} got {1} < {2}", message, sh, minSize));
     return success();
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index ce7f6b2865375..0473e058646b9 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -259,7 +259,7 @@ translateMap(linalg::GenericOp op, PatternRewriter &rewriter) {
   // translation.
   auto populateCstMapping = [ctx](DenseMap<AffineExpr, AffineExpr> &cstMapping,
                                   unsigned pos, int64_t lvlSz) {
-    if (!ShapedType::isDynamic(lvlSz)) {
+    if (ShapedType::isStatic(lvlSz)) {
       auto c0 = getAffineConstantExpr(0, ctx);
       auto lvlExp = getAffineDimExpr(pos, ctx);
       auto szExp = getAffineConstantExpr(lvlSz, ctx);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 14ced56b8365f..001ea62b07360 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -1348,7 +1348,7 @@ struct SparseAssembleOpConverter : public OpConversionPattern<AssembleOp> {
     Level trailCOORank = stt.getLvlRank() - trailCOOStart;
     // Sets up SparseTensorSpecifier.
     for (Level lvl = 0, lvlRank = stt.getLvlRank(); lvl < lvlRank; lvl++) {
-      assert(!ShapedType::isDynamic(stt.getDimShape()[lvl]));
+      assert(ShapedType::isStatic(stt.getDimShape()[lvl]));
 
       // Sets up the level size.
       auto lvlSize = constantIndex(rewriter, loc, stt.getLvlShape()[lvl]);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 7f0b657687442..50ccb43d432b6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -86,7 +86,7 @@ static Value createOrFoldLvlCall(OpBuilder &builder, Location loc,
   const Dimension dim =
       stt.isIdentity() ? lvl : stt.getDimToLvl().getDimPosition(lvl);
   const Size sz = stt.getDynamicDimSize(dim);
-  if (!ShapedType::isDynamic(sz))
+  if (ShapedType::isStatic(sz))
     return constantIndex(builder, loc, sz);
   // If we cannot statically compute the size from the shape, then we
   // must dynamically query it.  (In principle we could also dynamically
@@ -103,7 +103,7 @@ static Value createOrFoldDimCall(OpBuilder &builder, Location loc,
                                  SparseTensorType stt, Value tensor,
                                  Dimension dim) {
   const Size sz = stt.getDynamicDimSize(dim);
-  if (!ShapedType::isDynamic(sz))
+  if (ShapedType::isStatic(sz))
     return constantIndex(builder, loc, sz);
   if (stt.hasEncoding())
     return genDimSizeCall(builder, loc, tensor, dim);
@@ -729,9 +729,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
     createFuncCall(rewriter, loc, name, {},
                    {tensor, lvlCoords, values, filled, added, count},
                    EmitCInterface::On);
+    Operation *parent = getTop(op);
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
-    Operation *parent = getTop(op);
     rewriter.setInsertionPointAfter(parent);
     rewriter.create<memref::DeallocOp>(loc, values);
     rewriter.create<memref::DeallocOp>(loc, filled);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index 093a35a3d4c94..d4a02bf7a70b6 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -1245,7 +1245,7 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
       // by concatenate op verifier, which saves us from computing the offset
       // dynamically.
       const Size sz = getSparseTensorType(input).getDynamicDimSize(conDim);
-      assert(!ShapedType::isDynamic(sz));
+      assert(ShapedType::isStatic(sz));
       offset = rewriter.create<arith::AddIOp>(loc, offset,
                                               constantIndex(rewriter, loc, sz));
       iterArg = foreachOp.getResult(0);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
index 8981de58306da..347363f20d746 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
@@ -167,10 +167,10 @@ IterationGraphSorter::IterationGraphSorter(
 AffineMap IterationGraphSorter::sort(SortMask mask, Value ignored) {
   // Reset the adjacency matrix that represents the iteration graph.
   for (auto &row : itGraph)
-    std::fill(row.begin(), row.end(), false);
+    llvm::fill(row, false);
 
   // Reset in-degree.
-  std::fill(inDegree.begin(), inDegree.end(), 0);
+  llvm::fill(inDegree, 0);
 
   // Add the constraints for the loop to level map.
   for (auto [in, map] : llvm::zip(ins, loop2InsLvl)) {
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
index f6fea08e2e717..3e3422162a8da 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.cpp
@@ -23,7 +23,7 @@ using namespace mlir::tensor;
 static OpFoldResult getCollapsedOutputDimFromInputShape(
     OpBuilder &builder, Location loc, int64_t dimIndex, Value src,
     ArrayRef<int64_t> dstStaticShape, ArrayRef<AffineMap> reassociationMap) {
-  if (!ShapedType::isDynamic(dstStaticShape[dimIndex])) {
+  if (ShapedType::isStatic(dstStaticShape[dimIndex])) {
     // Static dimension: return Attribute.
     return builder.getIndexAttr(dstStaticShape[dimIndex]);
   }
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 22a25fd1a5af8..a3e863254405c 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -292,7 +292,7 @@ bool mlir::tensor::preservesStaticInformation(Type source, Type target) {
 
   // If cast is towards more static sizes along any dimension, don't fold.
   for (auto t : llvm::zip(sourceType.getShape(), targetType.getShape())) {
-    if (!ShapedType::isDynamic(std::get<0>(t)) &&
+    if (ShapedType::isStatic(std::get<0>(t)) &&
         ShapedType::isDynamic(std::get<1>(t)))
       return false;
   }
@@ -1235,7 +1235,7 @@ struct FoldEmptyTensorWithCastOp : public OpRewritePattern<CastOp> {
 
       // Case 2 : The tensor cast shape is static, but empty tensor result
       // shape is dynamic.
-      if (!ShapedType::isDynamic(newDim)) {
+      if (ShapedType::isStatic(newDim)) {
         newMixedSizes.push_back(rewriter.getIndexAttr(newDim));
         continue;
       }
@@ -1872,9 +1872,9 @@ OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
   if (!sourceTy || !resultTy || sourceTy != resultTy)
     return {};
 
-  // If the source and result are both 1D tensors and have the same type, the
-  // reshape has no effect, even if the tensor is dynamically shaped.
-  if (sourceTy.getRank() == 1)
+  // If the source and result are both 0D or 1D tensors and have the same type,
+  // the reshape has no effect, even if the tensor is dynamically shaped.
+  if (sourceTy.getRank() <= 1)
     return source;
 
   if (auto fromElements = getShape().getDefiningOp<tensor::FromElementsOp>()) {
@@ -2197,7 +2197,7 @@ struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
 
     for (const auto &[inputDim, innerReassoc] : llvm::enumerate(reassoc)) {
       for (uint64_t outDim : innerReassoc) {
-        if (!ShapedType::isDynamic(newOutputShape[outDim]))
+        if (ShapedType::isStatic(newOutputShape[outDim]))
           continue;
 
         // If the cast's src type is dynamic, don't infer any of the
@@ -3579,7 +3579,7 @@ struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
         continue;
       OpFoldResult sliceSize = innerSliceOp.getMixedSizes()[en.index()];
       int64_t sourceSize = innerSliceOp.getSourceType().getShape()[en.index()];
-      assert(!ShapedType::isDynamic(sourceSize) &&
+      assert(ShapedType::isStatic(sourceSize) &&
              "expected padded dimension to have a static size");
       if (getConstantIntValue(sliceSize) != sourceSize) {
         return rewriter.notifyMatchFailure(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 729c048db4560..829b2ab92ac24 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -49,7 +49,7 @@ struct CastOpInterface
     return {{op->getResult(0), BufferRelation::Equivalent}};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -68,20 +68,22 @@ struct CastOpInterface
     if (isa<UnrankedTensorType>(castOp.getSource().getType())) {
       // When casting to a ranked tensor, we cannot infer any static offset or
       // strides from the source. Assume fully dynamic.
-      return getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace);
+      return cast<BufferLikeType>(
+          getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace));
     }
 
     // Case 2: Casting to an unranked tensor type
     if (isa<UnrankedTensorType>(castOp.getType())) {
-      return getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace);
+      return cast<BufferLikeType>(
+          getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace));
     }
 
     // Case 3: Ranked tensor -> ranked tensor. The offsets and strides do not
     // change.
     auto rankedResultType = cast<RankedTensorType>(castOp.getType());
-    return MemRefType::get(
+    return cast<BufferLikeType>(MemRefType::get(
         rankedResultType.getShape(), rankedResultType.getElementType(),
-        llvm::cast<MemRefType>(*maybeSrcBufferType).getLayout(), memorySpace);
+        llvm::cast<MemRefType>(*maybeSrcBufferType).getLayout(), memorySpace));
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -141,7 +143,7 @@ struct CollapseShapeOpInterface
     return {{op->getOpResult(0), BufferRelation::Equivalent}};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -157,12 +159,13 @@ struct CollapseShapeOpInterface
     if (!canBeCollapsed) {
       // If dims cannot be collapsed, this op bufferizes to a new allocation.
       RankedTensorType tensorResultType = collapseShapeOp.getResultType();
-      return bufferization::getMemRefTypeWithStaticIdentityLayout(
-          tensorResultType, srcBufferType.getMemorySpace());
+      return cast<BufferLikeType>(
+          bufferization::getMemRefTypeWithStaticIdentityLayout(
+              tensorResultType, srcBufferType.getMemorySpace()));
     }
 
-    return memref::CollapseShapeOp::computeCollapsedType(
-        srcBufferType, collapseShapeOp.getReassociationIndices());
+    return cast<BufferLikeType>(memref::CollapseShapeOp::computeCollapsedType(
+        srcBufferType, collapseShapeOp.getReassociationIndices()));
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -319,7 +322,7 @@ struct ExpandShapeOpInterface
     return {{op->getOpResult(0), BufferRelation::Equivalent}};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -334,7 +337,7 @@ struct ExpandShapeOpInterface
         expandShapeOp.getReassociationIndices());
     if (failed(maybeResultType))
       return failure();
-    return *maybeResultType;
+    return cast<BufferLikeType>(*maybeResultType);
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -404,7 +407,7 @@ struct ExtractSliceOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -417,10 +420,10 @@ struct ExtractSliceOpInterface
     SmallVector<OpFoldResult> mixedOffsets = extractSliceOp.getMixedOffsets();
     SmallVector<OpFoldResult> mixedSizes = extractSliceOp.getMixedSizes();
     SmallVector<OpFoldResult> mixedStrides = extractSliceOp.getMixedStrides();
-    return memref::SubViewOp::inferRankReducedResultType(
+    return cast<BufferLikeType>(memref::SubViewOp::inferRankReducedResultType(
         extractSliceOp.getType().getShape(),
         llvm::cast<MemRefType>(*srcMemrefType), mixedOffsets, mixedSizes,
-        mixedStrides);
+        mixedStrides));
   }
 };
 
@@ -501,8 +504,8 @@ struct FromElementsOpInterface
         /*copy=*/false);
     if (failed(tensorAlloc))
       return failure();
-    FailureOr<BaseMemRefType> memrefType = bufferization::detail::asMemRefType(
-        bufferization::getBufferType(*tensorAlloc, options, state));
+    FailureOr<BufferLikeType> memrefType =
+        bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
     Value buffer = rewriter.create<bufferization::ToBufferOp>(
@@ -753,7 +756,7 @@ struct PadOpInterface
     return {};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -765,9 +768,10 @@ struct PadOpInterface
     if (failed(maybeSrcBufferType))
       return failure();
     MemRefLayoutAttrInterface layout;
-    return MemRefType::get(padOp.getResultType().getShape(),
-                           padOp.getResultType().getElementType(), layout,
-                           maybeSrcBufferType->getMemorySpace());
+    return cast<BufferLikeType>(
+        MemRefType::get(padOp.getResultType().getShape(),
+                        padOp.getResultType().getElementType(), layout,
+                        maybeSrcBufferType->getMemorySpace()));
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -927,7 +931,7 @@ struct ReshapeOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -937,9 +941,9 @@ struct ReshapeOpInterface
         reshapeOp.getSource(), options, state, invocationStack);
     if (failed(maybeSourceBufferType))
       return failure();
-    return getMemRefTypeWithStaticIdentityLayout(
+    return cast<BufferLikeType>(getMemRefTypeWithStaticIdentityLayout(
         reshapeOp.getResult().getType(),
-        cast<BaseMemRefType>(maybeSourceBufferType.value()).getMemorySpace());
+        cast<BaseMemRefType>(maybeSourceBufferType.value()).getMemorySpace()));
   }
 };
 
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index c19d3733769b7..1d21096e8920b 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -344,7 +344,7 @@ LogicalResult SelectOp::canonicalize(SelectOp op, PatternRewriter &rewriter) {
     return failure();
   rewriter.modifyOpInPlace(op, [&]() {
     op.getOperation()->setOperands(
-        {notOp.getInput1(), op.getInput3(), op.getInput2()});
+        {notOp.getInput1(), op.getOnFalse(), op.getOnTrue()});
   });
   return success();
 }
@@ -1510,8 +1510,8 @@ OpFoldResult SliceOp::fold(FoldAdaptor adaptor) {
 }
 
 OpFoldResult tosa::SelectOp::fold(FoldAdaptor adaptor) {
-  if (getInput2() == getInput3())
-    return getInput2();
+  if (getOnTrue() == getOnFalse())
+    return getOnTrue();
 
   auto predicate =
       llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getInput1());
@@ -1520,8 +1520,8 @@ OpFoldResult tosa::SelectOp::fold(FoldAdaptor adaptor) {
 
   if (!predicate.isSplat())
     return {};
-  return predicate.getSplatValue<APInt>().getBoolValue() ? getInput2()
-                                                         : getInput3();
+  return predicate.getSplatValue<APInt>().getBoolValue() ? getOnTrue()
+                                                         : getOnFalse();
 }
 
 OpFoldResult TileOp::fold(FoldAdaptor adaptor) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 3135fbd49bfbe..5170a11523845 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -849,7 +849,7 @@ static LogicalResult verifyPoolingOp(T op) {
              << kernelSize << ") / " << strideSize;
 
     const int64_t calculatedOutSize = calculatedOutSizeMinusOne.value() + 1;
-    if (!ShapedType::isDynamic(outputSize) && calculatedOutSize != outputSize)
+    if (ShapedType::isStatic(outputSize) && calculatedOutSize != outputSize)
       return op.emitOpError("calculated output ")
              << dimName << " did not match expected: "
              << "calculated=" << calculatedOutSize
@@ -1301,12 +1301,12 @@ LogicalResult tosa::RFFT2dOp::verify() {
     return success();
 
   const int64_t height = inputType.getDimSize(1);
-  if (!ShapedType::isDynamic(height) &&
+  if (ShapedType::isStatic(height) &&
       failed(verifyDimIsPowerOfTwo(*this, height, "height")))
     return failure();
 
   const int64_t width = inputType.getDimSize(2);
-  if (!ShapedType::isDynamic(width) &&
+  if (ShapedType::isStatic(width) &&
       failed(verifyDimIsPowerOfTwo(*this, width, "width")))
     return failure();
 
@@ -1323,7 +1323,7 @@ LogicalResult tosa::RFFT2dOp::verify() {
 
   // Output width dimension expected to be input_width / 2 + 1
   const int64_t outputWidth = outputType.getDimSize(2);
-  if (!ShapedType::isDynamic(width) && !ShapedType::isDynamic(outputWidth) &&
+  if (ShapedType::isStatic(width) && ShapedType::isStatic(outputWidth) &&
       (outputWidth != (width / 2) + 1))
     return emitOpError(
                "expected output width to be equal to input_width / 2 + 1, got ")
@@ -1357,13 +1357,13 @@ LogicalResult tosa::FFT2dOp::verify() {
 
   const int64_t height = trySelectStaticDim(inputRealType.getDimSize(1),
                                             inputImagType.getDimSize(1));
-  if (!ShapedType::isDynamic(height) &&
+  if (ShapedType::isStatic(height) &&
       failed(verifyDimIsPowerOfTwo(*this, height, "height")))
     return failure();
 
   const int64_t width = trySelectStaticDim(inputRealType.getDimSize(2),
                                            inputImagType.getDimSize(2));
-  if (!ShapedType::isDynamic(width) &&
+  if (ShapedType::isStatic(width) &&
       failed(verifyDimIsPowerOfTwo(*this, width, "width")))
     return failure();
 
@@ -1965,7 +1965,7 @@ LogicalResult tosa::TableOp::verify() {
   for (auto it : llvm::enumerate(llvm::zip(inputDims, outputDims))) {
     int64_t dim = it.index();
     auto [inputDim, outputDim] = it.value();
-    if (!ShapedType::isDynamic(outputDim) && outputDim != inputDim) {
+    if (ShapedType::isStatic(outputDim) && outputDim != inputDim) {
       return emitOpError() << "dim(result, " << dim << ") = " << outputDim
                            << " doesn't match dim(input, " << dim
                            << ") = " << inputDim;
@@ -2100,7 +2100,7 @@ LogicalResult tosa::ReshapeOp::inferReturnTypeComponents(
   int64_t numElements = inputShape.getNumElements();
   int64_t staticMul = 1;
   for (auto val : newShapeValue) {
-    if (!ShapedType::isDynamic(val)) {
+    if (ShapedType::isStatic(val)) {
       staticMul *= val;
     }
   }
@@ -2988,12 +2988,12 @@ static LogicalResult poolingInferReturnTypes(
   int64_t height = inputShape.getDimSize(1);
   int64_t width = inputShape.getDimSize(2);
 
-  if (!ShapedType::isDynamic(height)) {
+  if (ShapedType::isStatic(height)) {
     int64_t padded = height + pad[0] + pad[1] - kernel[0];
     outputShape[1] = padded / stride[0] + 1;
   }
 
-  if (!ShapedType::isDynamic(width)) {
+  if (ShapedType::isStatic(width)) {
     int64_t padded = width + pad[2] + pad[3] - kernel[1];
     outputShape[2] = padded / stride[1] + 1;
   }
@@ -3042,16 +3042,14 @@ LogicalResult Conv2DOp::inferReturnTypeComponents(
   llvm::ArrayRef<int64_t> stride = adaptor.getStride();
   llvm::ArrayRef<int64_t> padding = adaptor.getPad();
 
-  if (!ShapedType::isDynamic(inputHeight) &&
-      !ShapedType::isDynamic(weightHeight)) {
+  if (ShapedType::isStatic(inputHeight) && ShapedType::isStatic(weightHeight)) {
     int64_t inputSize = inputHeight + padding[0] + padding[1];
     int64_t filterSize = (weightHeight - 1) * dilation[0] + 1;
     int64_t unstridedResult = inputSize - filterSize + 1;
     outputShape[1] = (unstridedResult - 1) / stride[0] + 1;
   }
 
-  if (!ShapedType::isDynamic(inputWidth) &&
-      !ShapedType::isDynamic(weightWidth)) {
+  if (ShapedType::isStatic(inputWidth) && ShapedType::isStatic(weightWidth)) {
     int64_t inputSize = inputWidth + padding[2] + padding[3];
     int64_t filterSize = (weightWidth - 1) * dilation[1] + 1;
     int64_t unstridedResult = inputSize - filterSize + 1;
@@ -3111,24 +3109,21 @@ LogicalResult Conv3DOp::inferReturnTypeComponents(
   llvm::ArrayRef<int64_t> stride = adaptor.getStride();
   llvm::ArrayRef<int64_t> pad = adaptor.getPad();
 
-  if (!ShapedType::isDynamic(inputDepth) &&
-      !ShapedType::isDynamic(weightDepth)) {
+  if (ShapedType::isStatic(inputDepth) && ShapedType::isStatic(weightDepth)) {
     int32_t inputSize = inputDepth + pad[0] + pad[1];
     int32_t filterSize = (weightDepth - 1) * dilation[0] + 1;
     int32_t unstridedResult = inputSize - filterSize + 1;
     outputShape[1] = (unstridedResult - 1) / stride[0] + 1;
   }
 
-  if (!ShapedType::isDynamic(inputHeight) &&
-      !ShapedType::isDynamic(weightHeight)) {
+  if (ShapedType::isStatic(inputHeight) && ShapedType::isStatic(weightHeight)) {
     int32_t inputSize = inputHeight + pad[2] + pad[3];
     int32_t filterSize = (weightHeight - 1) * dilation[1] + 1;
     int32_t unstridedResult = inputSize - filterSize + 1;
     outputShape[2] = (unstridedResult - 1) / stride[1] + 1;
   }
 
-  if (!ShapedType::isDynamic(inputWidth) &&
-      !ShapedType::isDynamic(weightWidth)) {
+  if (ShapedType::isStatic(inputWidth) && ShapedType::isStatic(weightWidth)) {
     int32_t inputSize = inputWidth + pad[4] + pad[5];
     int32_t filterSize = (weightWidth - 1) * dilation[2] + 1;
     int32_t unstridedResult = inputSize - filterSize + 1;
@@ -3213,8 +3208,8 @@ LogicalResult DepthwiseConv2DOp::inferReturnTypeComponents(
 
   // If both inputChannels and depthChannels are available we can determine
   // the output channels.
-  if (!ShapedType::isDynamic(inputChannels) &&
-      !ShapedType::isDynamic(depthChannels)) {
+  if (ShapedType::isStatic(inputChannels) &&
+      ShapedType::isStatic(depthChannels)) {
     outputShape[3] = inputChannels * depthChannels;
   }
 
@@ -3230,16 +3225,14 @@ LogicalResult DepthwiseConv2DOp::inferReturnTypeComponents(
   llvm::ArrayRef<int64_t> padding = adaptor.getPad();
   llvm::ArrayRef<int64_t> stride = adaptor.getStride();
 
-  if (!ShapedType::isDynamic(inputHeight) &&
-      !ShapedType::isDynamic(weightHeight)) {
+  if (ShapedType::isStatic(inputHeight) && ShapedType::isStatic(weightHeight)) {
     int64_t inputSize = inputHeight + padding[0] + padding[1];
     int64_t filterSize = (weightHeight - 1) * dilation[0] + 1;
     int64_t unstridedResult = inputSize - filterSize + 1;
     outputShape[1] = (unstridedResult - 1) / stride[0] + 1;
   }
 
-  if (!ShapedType::isDynamic(inputWidth) &&
-      !ShapedType::isDynamic(weightWidth)) {
+  if (ShapedType::isStatic(inputWidth) && ShapedType::isStatic(weightWidth)) {
     int64_t inputSize = inputWidth + padding[2] + padding[3];
     int64_t filterSize = (weightWidth - 1) * dilation[1] + 1;
     int64_t unstridedResult = inputSize - filterSize + 1;
@@ -3299,16 +3292,14 @@ LogicalResult TransposeConv2DOp::inferReturnTypeComponents(
   llvm::ArrayRef<int64_t> padding = adaptor.getOutPad();
   llvm::ArrayRef<int64_t> stride = adaptor.getStride();
 
-  if (!ShapedType::isDynamic(inputHeight) &&
-      !ShapedType::isDynamic(weightHeight)) {
+  if (ShapedType::isStatic(inputHeight) && ShapedType::isStatic(weightHeight)) {
     int64_t calculateSize =
         (inputHeight - 1) * stride[0] + padding[0] + padding[1] + weightHeight;
     outputShape[1] =
         ShapedType::isDynamic(outputShape[1]) ? calculateSize : outputShape[1];
   }
 
-  if (!ShapedType::isDynamic(inputWidth) &&
-      !ShapedType::isDynamic(weightWidth)) {
+  if (ShapedType::isStatic(inputWidth) && ShapedType::isStatic(weightWidth)) {
     int64_t calculateSize =
         (inputWidth - 1) * stride[1] + padding[2] + padding[3] + weightWidth;
     outputShape[2] =
@@ -3354,7 +3345,7 @@ LogicalResult TransposeConv2DOp::verify() {
 
   if (weightType) {
     const int64_t kernelHeight = weightType.getDimSize(1);
-    if (!ShapedType::isDynamic(kernelHeight)) {
+    if (ShapedType::isStatic(kernelHeight)) {
       if (failed(checkPadAgainstKernelDim(outPadTop, kernelHeight,
                                           "out_pad_top", "KH")))
         return failure();
@@ -3365,7 +3356,7 @@ LogicalResult TransposeConv2DOp::verify() {
     }
 
     const int64_t kernelWidth = weightType.getDimSize(2);
-    if (!ShapedType::isDynamic(kernelWidth)) {
+    if (ShapedType::isStatic(kernelWidth)) {
       if (failed(checkPadAgainstKernelDim(outPadLeft, kernelWidth,
                                           "out_pad_left", "KW")))
         return failure();
@@ -3388,8 +3379,8 @@ LogicalResult TransposeConv2DOp::verify() {
     const int64_t kernelHeight = weightType.getDimSize(1);
     const int64_t outputHeight = outputType.getDimSize(1);
 
-    if (!ShapedType::isDynamic(inputHeight) &&
-        !ShapedType::isDynamic(outputHeight)) {
+    if (ShapedType::isStatic(inputHeight) &&
+        ShapedType::isStatic(outputHeight)) {
       if (outputHeight !=
           (inputHeight - 1) * strideY + outPadTop + outPadBottom + kernelHeight)
         return emitOpError(
@@ -3404,8 +3395,7 @@ LogicalResult TransposeConv2DOp::verify() {
     const int64_t kernelWidth = weightType.getDimSize(2);
     const int64_t outputWidth = outputType.getDimSize(2);
 
-    if (!ShapedType::isDynamic(inputWidth) &&
-        !ShapedType::isDynamic(outputWidth)) {
+    if (ShapedType::isStatic(inputWidth) && ShapedType::isStatic(outputWidth)) {
       if (outputWidth !=
           (inputWidth - 1) * strideX + outPadLeft + outPadRight + kernelWidth)
         return emitOpError(
@@ -3829,16 +3819,16 @@ LogicalResult ReverseOp::verify() {
 
 LogicalResult tosa::SelectOp::verify() {
   // verify input2 and input3 have same element type as output
-  if (verifySameElementTypes(*this, /* inType = */ getInput2().getType(),
+  if (verifySameElementTypes(*this, /* inType = */ getOnTrue().getType(),
                              /* outType = */ getOutput().getType())
           .failed() ||
-      verifySameElementTypes(*this, /* inType = */ getInput3().getType(),
+      verifySameElementTypes(*this, /* inType = */ getOnFalse().getType(),
                              /* outType = */ getOutput().getType())
           .failed()) {
     return failure();
   }
   // verify input1 has element type of bool
-  auto predicateType = llvm::dyn_cast<ShapedType>(getInput1().getType());
+  auto predicateType = llvm::dyn_cast<ShapedType>(getPred().getType());
   if (!predicateType) {
     return emitOpError("expect shaped tensor for input1, got ")
            << getInput1().getType();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt
index bbf079faea3d0..803993bb1008d 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_mlir_dialect_library(MLIRTosaTransforms
+  TosaConvertIntegerTypeToSignless.cpp
   TosaDecomposeTransposeConv.cpp
   TosaDecomposeDepthwise.cpp
   TosaFolders.cpp
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp
new file mode 100644
index 0000000000000..7c51592dabe23
--- /dev/null
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp
@@ -0,0 +1,139 @@
+//===- TosaConvertIntegerTypeToSignless.cpp
+//-------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-------------------------------------------------------------------------------===//
+
+// -----------
+// Motivation:
+// -----------
+
+// The TOSA specification uses a signless type system, which means that
+// information about signedness must be encapsulated by the operations
+// themselves. For example, tosa.rescale provides the attributes
+// `input_unsigned` and `output_unsigned` to indicate whether the input/output
+// should be interpreted as unsigned or signed.
+
+// The TOSA dialect, on the other hand, allows the use of signed or unsigned
+// types in addition to signless. As such, when converting from TOSA dialect to
+// other formats, we need to ensure that we conform to the TOSA specification.
+
+// ---------
+// Overview:
+// ---------
+
+// This pass converts signed or unsigned integer types to signless. It currently
+// does this greedily for all operators and can also change the signature of the
+// function. Should the signature of the entrypoint function change, it will be
+// the responsibility of the user to carry signedness information of the inputs
+// and outputs independently.
+
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
+#include "mlir/Dialect/Tosa/IR/TosaOps.h"
+#include "mlir/Dialect/Tosa/Transforms/Passes.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+namespace tosa {
+
+#define GEN_PASS_DEF_TOSACONVERTINTEGERTYPETOSIGNLESS
+#include "mlir/Dialect/Tosa/Transforms/Passes.h.inc"
+
+namespace {
+class ToSignlessTensorTypeConverter : public TypeConverter {
+  static Type convertType(Type type) {
+    const auto tensorType = dyn_cast<TensorType>(type);
+    if (!tensorType)
+      return type;
+
+    const auto intType = dyn_cast<IntegerType>(tensorType.getElementType());
+    if (!intType ||
+        intType.getSignedness() == IntegerType::SignednessSemantics::Signless)
+      return type;
+
+    const auto signlessType = IntegerType::get(
+        intType.getContext(), intType.getWidth(), IntegerType::Signless);
+    return tensorType.cloneWith(std::nullopt, signlessType);
+  }
+
+public:
+  explicit ToSignlessTensorTypeConverter() { addConversion(convertType); }
+};
+
+class ConvertGenericOpWithIntegerTensorType : public ConversionPattern {
+public:
+  ConvertGenericOpWithIntegerTensorType(TypeConverter &typeConverter,
+                                        MLIRContext *context)
+      : ConversionPattern(typeConverter, MatchAnyOpTypeTag{}, 0, context) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    // Typically TOSA operators have a single result, but some have an
+    // arbitrary number. 4 seems like a good balance as an optimization
+    // hint for storing result types.
+    constexpr unsigned int numResults = 4;
+
+    // Convert integer types to signless
+    SmallVector<Type, numResults> resultTypes;
+    if (failed(typeConverter->convertTypes(op->getResultTypes(), resultTypes)))
+      return failure();
+
+    // Create new op with replaced operands and results
+    auto *newOp = Operation::create(
+        op->getLoc(), op->getName(), resultTypes, operands, op->getAttrs(),
+        op->getPropertiesStorage(), op->getSuccessors(), op->getNumRegions());
+
+    // Handle regions in e.g. tosa.cond_if and tosa.while_loop
+    for (auto regions : llvm::zip(op->getRegions(), newOp->getRegions())) {
+      Region &before = std::get<0>(regions);
+      Region &parent = std::get<1>(regions);
+      rewriter.inlineRegionBefore(before, parent, parent.end());
+      if (failed(rewriter.convertRegionTypes(&parent, *typeConverter)))
+        return failure();
+    }
+
+    // Replace with rewritten op
+    rewriter.insert(newOp);
+    rewriter.replaceOp(op, newOp->getResults());
+    return success();
+  }
+};
+
+class TosaConvertIntegerTypeToSignless
+    : public impl::TosaConvertIntegerTypeToSignlessBase<
+          TosaConvertIntegerTypeToSignless> {
+public:
+  void runOnOperation() override {
+    MLIRContext *context = &getContext();
+    ConversionTarget target(*context);
+    ToSignlessTensorTypeConverter typeConverter;
+
+    target.addDynamicallyLegalOp<func::FuncOp>([&](func::FuncOp op) {
+      return typeConverter.isSignatureLegal(op.getFunctionType()) &&
+             typeConverter.isLegal(&op.getBody());
+    });
+    target.markUnknownOpDynamicallyLegal([&](Operation *op) {
+      return typeConverter.isLegal(op->getOperandTypes()) &&
+             typeConverter.isLegal(op->getResultTypes());
+    });
+
+    RewritePatternSet patterns(context);
+    populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(
+        patterns, typeConverter);
+    patterns.add<ConvertGenericOpWithIntegerTensorType>(typeConverter, context);
+
+    if (failed(
+            applyFullConversion(getOperation(), target, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+
+} // namespace
+
+} // namespace tosa
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
index 02a3ad83bdefa..7997753469527 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
@@ -169,9 +169,9 @@ struct ConvertTosaOp<tosa::SelectOp> : public OpRewritePattern<tosa::SelectOp> {
   LogicalResult matchAndRewrite(tosa::SelectOp tosaOp,
                                 PatternRewriter &rewriter) const override {
 
-    Value input1 = tosaOp.getInput1();
-    Value input2 = tosaOp.getInput2();
-    Value input3 = tosaOp.getInput3();
+    Value input1 = tosaOp.getPred();
+    Value input2 = tosaOp.getOnTrue();
+    Value input3 = tosaOp.getOnFalse();
     Value output = tosaOp.getResult();
 
     auto outputType = dyn_cast<RankedTensorType>(output.getType());
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
index de08e7e9a4394..a4edccfd4c9c7 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
@@ -188,8 +188,8 @@ LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::RFFT2dOp op) {
 
 template <>
 LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::SelectOp op) {
-  addValue(op.getInput2());
-  addValue(op.getInput3());
+  addValue(op.getOnTrue());
+  addValue(op.getOnFalse());
   addValue(op.getOutput());
   return success();
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 5a732ec78158d..7f85cd52f6bde 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -602,8 +602,7 @@ void TosaReduceTransposes::runOnOperation() {
         !llvm::isa<RankedTensorType>(output.getType()))
       return;
 
-    llvm::for_each(transposeOp.getPerms(),
-                   [&perms](const auto i) { perms.emplace_back(i); });
+    llvm::append_range(perms, transposeOp.getPerms());
 
     // We let --canonicalize deal with identity transpose.
     if (llvm::equal(llvm::seq<int32_t>(0, perms.size()), perms))
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 229f42d3178b5..3f27849b8c90c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -1320,13 +1320,14 @@ void TosaValidation::runOnOperation() {
 
     // validate operator element types:
     // - rescale operator is allowed to have ui8/ui16/ui32
-    //   operands/results
+    //   operands/results when strictOpSpecAlignment is false
     // - perform valid element type check at the beginning to
     //   protect rest of code against quantized element types
-    const bool opIsRescale = isa<tosa::RescaleOp>(op);
+    const bool allowUnsigned =
+        !strictOpSpecAlignment && isa<tosa::RescaleOp>(op);
     for (Value operand : op->getOperands()) {
       auto elementTy = getElementTypeOrSelf(operand);
-      if (!isValidElementType(elementTy, opIsRescale)) {
+      if (!isValidElementType(elementTy, allowUnsigned)) {
         op->emitOpError() << "is not profile-aligned: element type "
                           << elementTy << " is not legal";
         return signalPassFailure();
@@ -1334,7 +1335,7 @@ void TosaValidation::runOnOperation() {
     }
     for (Type resultTy : op->getResultTypes()) {
       auto elementTy = getElementTypeOrSelf(resultTy);
-      if (!isValidElementType(elementTy, opIsRescale)) {
+      if (!isValidElementType(elementTy, allowUnsigned)) {
         op->emitOpError() << "is not profile-aligned: element type "
                           << elementTy << " is not legal";
         return signalPassFailure();
diff --git a/mlir/lib/Dialect/Transform/CMakeLists.txt b/mlir/lib/Dialect/Transform/CMakeLists.txt
index 0c0d5ebe0c212..6e628353258d6 100644
--- a/mlir/lib/Dialect/Transform/CMakeLists.txt
+++ b/mlir/lib/Dialect/Transform/CMakeLists.txt
@@ -5,4 +5,5 @@ add_subdirectory(IRDLExtension)
 add_subdirectory(LoopExtension)
 add_subdirectory(PDLExtension)
 add_subdirectory(Transforms)
+add_subdirectory(TuneExtension)
 add_subdirectory(Utils)
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index bb9bdd70625e4..0db0317461c03 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -799,8 +799,8 @@ transform::ApplyRegisteredPassOp::apply(transform::TransformRewriter &rewriter,
     if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
       // The corresponding value attribute(s) is/are passed in via a param.
       // Obtain the param-operand via its specified index.
-      size_t dynamicOptionIdx = paramOperand.getIndex().getInt();
-      assert(dynamicOptionIdx < dynamicOptions.size() &&
+      int64_t dynamicOptionIdx = paramOperand.getIndex().getInt();
+      assert(dynamicOptionIdx < static_cast<int64_t>(dynamicOptions.size()) &&
              "the number of ParamOperandAttrs in the options DictionaryAttr"
              "should be the same as the number of options passed as params");
       ArrayRef<Attribute> attrsAssociatedToParam =
@@ -1015,8 +1015,9 @@ LogicalResult transform::ApplyRegisteredPassOp::verify() {
   std::function<LogicalResult(Attribute)> checkOptionValue =
       [&](Attribute valueAttr) -> LogicalResult {
     if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
-      size_t dynamicOptionIdx = paramOperand.getIndex().getInt();
-      if (dynamicOptionIdx < 0 || dynamicOptionIdx >= dynamicOptions.size())
+      int64_t dynamicOptionIdx = paramOperand.getIndex().getInt();
+      if (dynamicOptionIdx < 0 ||
+          dynamicOptionIdx >= static_cast<int64_t>(dynamicOptions.size()))
         return emitOpError()
                << "dynamic option index " << dynamicOptionIdx
                << " is out of bounds for the number of dynamic options: "
diff --git a/mlir/lib/Dialect/Transform/TuneExtension/CMakeLists.txt b/mlir/lib/Dialect/Transform/TuneExtension/CMakeLists.txt
new file mode 100644
index 0000000000000..56b90a9a04edf
--- /dev/null
+++ b/mlir/lib/Dialect/Transform/TuneExtension/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_mlir_dialect_library(MLIRTransformTuneExtension
+  TuneExtension.cpp
+  TuneExtensionOps.cpp
+
+  DEPENDS
+  MLIRTransformDialectTuneExtensionOpsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRTransformDialect
+)
diff --git a/mlir/lib/Dialect/Transform/TuneExtension/TuneExtension.cpp b/mlir/lib/Dialect/Transform/TuneExtension/TuneExtension.cpp
new file mode 100644
index 0000000000000..e18f1e2748540
--- /dev/null
+++ b/mlir/lib/Dialect/Transform/TuneExtension/TuneExtension.cpp
@@ -0,0 +1,32 @@
+//===- TuneExtension.cpp - Tune extension for the Transform dialect -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtension.h"
+
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.h"
+#include "mlir/IR/DialectRegistry.h"
+
+using namespace mlir;
+
+class TuneExtension
+    : public transform::TransformDialectExtension<TuneExtension> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TuneExtension)
+
+  void init() {
+    registerTransformOps<
+#define GET_OP_LIST
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.cpp.inc"
+        >();
+  }
+};
+
+void mlir::transform::registerTuneExtension(DialectRegistry &dialectRegistry) {
+  dialectRegistry.addExtensions<TuneExtension>();
+}
diff --git a/mlir/lib/Dialect/Transform/TuneExtension/TuneExtensionOps.cpp b/mlir/lib/Dialect/Transform/TuneExtension/TuneExtensionOps.cpp
new file mode 100644
index 0000000000000..75c1cc53e2606
--- /dev/null
+++ b/mlir/lib/Dialect/Transform/TuneExtension/TuneExtensionOps.cpp
@@ -0,0 +1,62 @@
+//===- TuneExtensionOps.cpp - Tune extension for the Transform dialect ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Transform/IR/TransformOps.h"
+#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/PatternMatch.h"
+#include "llvm/Support/Debug.h"
+
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.h"
+
+using namespace mlir;
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.cpp.inc"
+
+#define DEBUG_TYPE "transform-tune"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "] ")
+
+//===----------------------------------------------------------------------===//
+// KnobOp
+//===----------------------------------------------------------------------===//
+
+void transform::tune::KnobOp::getEffects(
+    SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
+  producesHandle(getOperation()->getOpResults(), effects);
+  onlyReadsPayload(effects);
+}
+
+DiagnosedSilenceableFailure
+transform::tune::KnobOp::apply(transform::TransformRewriter &rewriter,
+                               transform::TransformResults &results,
+                               transform::TransformState &state) {
+  if (getSelected()) {
+    results.setParams(llvm::cast<OpResult>(getResult()), *getSelected());
+    return DiagnosedSilenceableFailure::success();
+  }
+
+  return emitDefiniteFailure()
+         << "non-deterministic choice " << getName()
+         << " is only resolved through providing a `selected` attr";
+}
+
+LogicalResult transform::tune::KnobOp::verify() {
+  if (auto selected = getSelected()) {
+    if (auto optionsArray = dyn_cast<ArrayAttr>(getOptions())) {
+      if (!llvm::is_contained(optionsArray, selected))
+        return emitOpError("provided `selected` attribute is not an element of "
+                           "`options` array of attributes");
+    } else
+      LLVM_DEBUG(DBGS() << "cannot verify `selected` attribute " << selected
+                        << " is an element of `options` attribute "
+                        << getOptions());
+  }
+
+  return success();
+}
diff --git a/mlir/lib/Dialect/Utils/CMakeLists.txt b/mlir/lib/Dialect/Utils/CMakeLists.txt
index a0096e5f299d5..f5bb687ae071a 100644
--- a/mlir/lib/Dialect/Utils/CMakeLists.txt
+++ b/mlir/lib/Dialect/Utils/CMakeLists.txt
@@ -6,6 +6,7 @@ add_mlir_library(MLIRDialectUtils
 
   DEPENDS
   MLIRDialectUtilsIncGen
+  MLIRArithDialect
 
   LINK_LIBS PUBLIC
   MLIRIR
diff --git a/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp b/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
index aa566c0086a2f..a32732e8b7007 100644
--- a/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
+++ b/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
@@ -505,7 +505,7 @@ LogicalResult mlir::reshapeLikeShapesAreCompatible(
         linearizedStaticShape *= dim.value();
     }
     if (foundDynamicShape) {
-      if (!ShapedType::isDynamic(collapsedShape[map.index()])) {
+      if (ShapedType::isStatic(collapsedShape[map.index()])) {
         return emitError(
             "expected dimension " + Twine(map.index()) +
             " of collapsed type to be dynamic since one or more of the "
diff --git a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
index be01ff2fa3781..1cded38c4419e 100644
--- a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
+++ b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
@@ -280,13 +280,13 @@ std::optional<int64_t> constantTripCount(OpFoldResult lb, OpFoldResult ub,
 
 bool hasValidSizesOffsets(SmallVector<int64_t> sizesOrOffsets) {
   return llvm::none_of(sizesOrOffsets, [](int64_t value) {
-    return !ShapedType::isDynamic(value) && value < 0;
+    return ShapedType::isStatic(value) && value < 0;
   });
 }
 
 bool hasValidStrides(SmallVector<int64_t> strides) {
   return llvm::none_of(strides, [](int64_t value) {
-    return !ShapedType::isDynamic(value) && value == 0;
+    return ShapedType::isStatic(value) && value == 0;
   });
 }
 
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index a11dbe2589205..214d2ba7e1b8e 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -398,6 +398,18 @@ std::optional<int64_t> vector::getConstantVscaleMultiplier(Value value) {
   return {};
 }
 
+/// Converts an IntegerAttr to have the specified type if needed.
+/// This handles cases where constant attributes have a different type than the
+/// target element type. If the input attribute is not an IntegerAttr or already
+/// has the correct type, returns it unchanged.
+static Attribute convertIntegerAttr(Attribute attr, Type expectedType) {
+  if (auto intAttr = mlir::dyn_cast<IntegerAttr>(attr)) {
+    if (intAttr.getType() != expectedType)
+      return IntegerAttr::get(expectedType, intAttr.getInt());
+  }
+  return attr;
+}
+
 //===----------------------------------------------------------------------===//
 // CombiningKindAttr
 //===----------------------------------------------------------------------===//
@@ -1095,7 +1107,7 @@ Type ContractionOp::getExpectedMaskType() {
         rhsType.getScalableDims()[dimIdx];
   }
 
-  assert(!ShapedType::isDynamicShape(maskShape) &&
+  assert(ShapedType::isStaticShape(maskShape) &&
          "Mask shape couldn't be computed");
 
   return VectorType::get(maskShape,
@@ -2049,7 +2061,7 @@ static Value extractInsertFoldConstantOp(OpType op, AdaptorType adaptor,
   // `opChange` is a flag. If it is true, it means to update `op` in place.
   bool opChange = false;
   for (unsigned i = 0, e = staticPosition.size(); i < e; ++i) {
-    if (!ShapedType::isDynamic(staticPosition[i]))
+    if (ShapedType::isStatic(staticPosition[i]))
       continue;
     Attribute positionAttr = dynamicPositionAttr[index];
     Value position = dynamicPosition[index++];
@@ -2464,8 +2476,37 @@ static OpFoldResult foldFromElementsToElements(FromElementsOp fromElementsOp) {
   return {};
 }
 
+/// Fold vector.from_elements to a constant when all operands are constants.
+/// Example:
+///   %c1 = arith.constant 1 : i32
+///   %c2 = arith.constant 2 : i32
+///   %v = vector.from_elements %c1, %c2 : vector<2xi32>
+/// =>
+///   %v = arith.constant dense<[1, 2]> : vector<2xi32>
+///
+static OpFoldResult foldFromElementsToConstant(FromElementsOp fromElementsOp,
+                                               ArrayRef<Attribute> elements) {
+  if (llvm::any_of(elements, [](Attribute attr) { return !attr; }))
+    return {};
+
+  auto destVecType = fromElementsOp.getDest().getType();
+  auto destEltType = destVecType.getElementType();
+  // Constant attributes might have a different type than the return type.
+  // Convert them before creating the dense elements attribute.
+  auto convertedElements = llvm::map_to_vector(elements, [&](Attribute attr) {
+    return convertIntegerAttr(attr, destEltType);
+  });
+
+  return DenseElementsAttr::get(destVecType, convertedElements);
+}
+
 OpFoldResult FromElementsOp::fold(FoldAdaptor adaptor) {
-  return foldFromElementsToElements(*this);
+  if (auto res = foldFromElementsToElements(*this))
+    return res;
+  if (auto res = foldFromElementsToConstant(*this, adaptor.getElements()))
+    return res;
+
+  return {};
 }
 
 /// Rewrite a vector.from_elements into a vector.splat if all elements are the
@@ -3293,7 +3334,6 @@ class InsertSplatToSplat final : public OpRewritePattern<InsertOp> {
     return success();
   }
 };
-
 } // namespace
 
 static Attribute
@@ -3332,17 +3372,6 @@ foldDenseElementsAttrDestInsertOp(InsertOp insertOp, Attribute srcAttr,
 
   /// Converts the expected type to an IntegerAttr if there's
   /// a mismatch.
-  auto convertIntegerAttr = [](Attribute attr, Type expectedType) -> Attribute {
-    if (auto intAttr = mlir::dyn_cast<IntegerAttr>(attr)) {
-      if (intAttr.getType() != expectedType)
-        return IntegerAttr::get(expectedType, intAttr.getInt());
-    }
-    return attr;
-  };
-
-  // The `convertIntegerAttr` method specifically handles the case
-  // for `llvm.mlir.constant` which can hold an attribute with a
-  // different type than the return type.
   if (auto denseSource = llvm::dyn_cast<DenseElementsAttr>(srcAttr)) {
     for (auto value : denseSource.getValues<Attribute>())
       insertedValues.push_back(convertIntegerAttr(value, destEltType));
@@ -3357,12 +3386,26 @@ foldDenseElementsAttrDestInsertOp(InsertOp insertOp, Attribute srcAttr,
   return newAttr;
 }
 
+/// Folder to replace the `dest` operand of the insert op with the root dest of
+/// the insert op use chain.
+static Value foldInsertUseChain(InsertOp insertOp) {
+  auto destInsert = insertOp.getDest().getDefiningOp<InsertOp>();
+  if (!destInsert)
+    return {};
+
+  if (insertOp.getMixedPosition() != destInsert.getMixedPosition())
+    return {};
+
+  insertOp.setOperand(1, destInsert.getDest());
+  return insertOp.getResult();
+}
+
 void InsertOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
   results.add<InsertToBroadcast, BroadcastFolder, InsertSplatToSplat>(context);
 }
 
-OpFoldResult vector::InsertOp::fold(FoldAdaptor adaptor) {
+OpFoldResult InsertOp::fold(FoldAdaptor adaptor) {
   // Do not create constants with more than `vectorSizeFoldThreashold` elements,
   // unless the source vector constant has a single use.
   constexpr int64_t vectorSizeFoldThreshold = 256;
@@ -3377,6 +3420,8 @@ OpFoldResult vector::InsertOp::fold(FoldAdaptor adaptor) {
   SmallVector<Value> operands = {getValueToStore(), getDest()};
   auto inplaceFolded = extractInsertFoldConstantOp(*this, adaptor, operands);
 
+  if (auto res = foldInsertUseChain(*this))
+    return res;
   if (auto res = foldPoisonIndexInsertExtractOp(
           getContext(), adaptor.getStaticPosition(), kPoisonIndex))
     return res;
@@ -4261,33 +4306,39 @@ void ExtractStridedSliceOp::getCanonicalizationPatterns(
 /// 1. Builder that sets padding to zero and an empty mask (variant with attrs).
 void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                            VectorType vectorType, Value source,
-                           ValueRange indices, AffineMapAttr permutationMapAttr,
+                           ValueRange indices, std::optional<Value> padding,
+                           AffineMapAttr permutationMapAttr,
                            /*optional*/ ArrayAttr inBoundsAttr) {
+
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  Value padding = builder.create<arith::ConstantOp>(
-      result.location, elemType, builder.getZeroAttr(elemType));
+  if (!padding)
+    padding = builder.create<ub::PoisonOp>(result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
-        padding, /*mask=*/Value(), inBoundsAttr);
+        *padding, /*mask=*/Value(), inBoundsAttr);
 }
 
 /// 2. Builder that sets padding to zero an empty mask (variant without attrs).
 void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                            VectorType vectorType, Value source,
-                           ValueRange indices, AffineMap permutationMap,
+                           ValueRange indices, std::optional<Value> padding,
+                           AffineMap permutationMap,
                            std::optional<ArrayRef<bool>> inBounds) {
   auto permutationMapAttr = AffineMapAttr::get(permutationMap);
   auto inBoundsAttr = (inBounds && !inBounds.value().empty())
                           ? builder.getBoolArrayAttr(inBounds.value())
                           : builder.getBoolArrayAttr(
                                 SmallVector<bool>(vectorType.getRank(), false));
-  build(builder, result, vectorType, source, indices, permutationMapAttr,
-        inBoundsAttr);
+  Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
+  if (!padding)
+    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+  build(builder, result, vectorType, source, indices, *padding,
+        permutationMapAttr, inBoundsAttr);
 }
 
 /// 3. Builder that sets permutation map to 'getMinorIdentityMap'.
 void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                            VectorType vectorType, Value source,
-                           ValueRange indices, Value padding,
+                           ValueRange indices, std::optional<Value> padding,
                            std::optional<ArrayRef<bool>> inBounds) {
   AffineMap permutationMap = getTransferMinorIdentityMap(
       llvm::cast<ShapedType>(source.getType()), vectorType);
@@ -4296,23 +4347,14 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                           ? builder.getBoolArrayAttr(inBounds.value())
                           : builder.getBoolArrayAttr(
                                 SmallVector<bool>(vectorType.getRank(), false));
+  Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
+  if (!padding)
+    padding = builder.create<ub::PoisonOp>(result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
-        padding,
+        *padding,
         /*mask=*/Value(), inBoundsAttr);
 }
 
-/// 4. Builder that sets padding to zero and permutation map to
-/// 'getMinorIdentityMap'.
-void TransferReadOp::build(OpBuilder &builder, OperationState &result,
-                           VectorType vectorType, Value source,
-                           ValueRange indices,
-                           std::optional<ArrayRef<bool>> inBounds) {
-  Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  Value padding = builder.create<arith::ConstantOp>(
-      result.location, elemType, builder.getZeroAttr(elemType));
-  build(builder, result, vectorType, source, indices, padding, inBounds);
-}
-
 template <typename EmitFun>
 static LogicalResult verifyPermutationMap(AffineMap permutationMap,
                                           EmitFun emitOpError) {
@@ -6567,6 +6609,28 @@ bool ConstantMaskOp::isAllOnesMask() {
   return true;
 }
 
+OpFoldResult ConstantMaskOp::fold(FoldAdaptor adaptor) {
+  ArrayRef<int64_t> bounds = getMaskDimSizes();
+  ArrayRef<int64_t> vectorSizes = getVectorType().getShape();
+
+  auto createBoolSplat = [&](bool x) {
+    return SplatElementsAttr::get(getVectorType(),
+                                  BoolAttr::get(getContext(), x));
+  };
+
+  // Check the corner case of 0-D vectors first.
+  if (vectorSizes.empty()) {
+    assert(bounds.size() == 1 && "invalid sizes for zero rank mask");
+    return createBoolSplat(bounds[0] == 1);
+  }
+  // Fold vector.constant_mask to splat if possible.
+  if (bounds == vectorSizes)
+    return createBoolSplat(true);
+  if (llvm::all_of(bounds, [](int64_t x) { return x == 0; }))
+    return createBoolSplat(false);
+  return OpFoldResult();
+}
+
 //===----------------------------------------------------------------------===//
 // CreateMaskOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index af90ed8f5deaf..fb99e22c77ea0 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -173,7 +173,7 @@ struct DistributedLoadStoreHelper {
     }
     SmallVector<bool> inBounds(indices.size(), true);
     return b.create<vector::TransferReadOp>(
-        loc, cast<VectorType>(type), buffer, indices,
+        loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 678a88627ca82..7cac1cbafdd64 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -445,16 +445,36 @@ struct LinearizeVectorExtract final
   }
 };
 
-/// This pattern converts the InsertOp to a ShuffleOp that works on a
-/// linearized vector.
-/// Following,
-///   vector.insert %source %destination [ position ]
-/// is converted to :
-///   %source_1d = vector.shape_cast %source
-///   %destination_1d = vector.shape_cast %destination
-///   %out_1d = vector.shuffle %destination_1d, %source_1d [ shuffle_indices_1d
-///   ] %out_nd = vector.shape_cast %out_1d
-/// `shuffle_indices_1d` is computed using the position of the original insert.
+/// This pattern linearizes `vector.insert` operations. It generates a 1-D
+/// version of the `vector.insert` operation when inserting a scalar into a
+/// vector. It generates a 1-D `vector.shuffle` operation when inserting a
+/// vector into another vector.
+///
+/// Example #1:
+///
+///     %0 = vector.insert %source, %destination[0] :
+///       vector<2x4xf32> into vector<2x2x4xf32>
+///
+///   is converted to:
+///
+///     %0 = vector.shape_cast %source : vector<2x4xf32> to vector<8xf32>
+///     %1 = vector.shape_cast %destination :
+///            vector<2x2x4xf32> to vector<16xf32>
+///     %2 = vector.shuffle %1, %0 [16, 17, 18, 19, 20, 21, 22, 23
+///                                  8, 9, 10, 11, 12, 13, 14, 15] :
+///            vector<16xf32>, vector<8xf32>
+///     %3 = vector.shape_cast %2 : vector<16xf32> to vector<2x2x4xf32>
+///
+/// Example #2:
+///
+///     %0 = vector.insert %source, %destination[1, 2]: f32 into vector<2x4xf32>
+///
+///   is converted to:
+///
+///     %0 = vector.shape_cast %destination : vector<2x4xf32> to vector<8xf32>
+///     %1 = vector.insert %source, %0[6]: f32 into vector<8xf32>
+///     %2 = vector.shape_cast %1 : vector<8xf32> to vector<2x4xf32>
+///
 struct LinearizeVectorInsert final
     : public OpConversionPattern<vector::InsertOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -468,25 +488,19 @@ struct LinearizeVectorInsert final
         insertOp.getDestVectorType());
     assert(dstTy && "vector type destination expected.");
 
-    // dynamic position is not supported
+    // Dynamic position is not supported.
     if (insertOp.hasDynamicPosition())
       return rewriter.notifyMatchFailure(insertOp,
                                          "dynamic position is not supported.");
     auto srcTy = insertOp.getValueToStoreType();
     auto srcAsVec = dyn_cast<VectorType>(srcTy);
-    uint64_t srcSize = 0;
-    if (srcAsVec) {
-      srcSize = srcAsVec.getNumElements();
-    } else {
-      return rewriter.notifyMatchFailure(insertOp,
-                                         "scalars are not supported.");
-    }
+    uint64_t srcSize = srcAsVec ? srcAsVec.getNumElements() : 1;
 
     auto dstShape = insertOp.getDestVectorType().getShape();
     const auto dstSize = insertOp.getDestVectorType().getNumElements();
     auto dstSizeForOffsets = dstSize;
 
-    // compute linearized offset
+    // Compute linearized offset.
     int64_t linearizedOffset = 0;
     auto offsetsNd = insertOp.getStaticPosition();
     for (auto [dim, offset] : llvm::enumerate(offsetsNd)) {
@@ -494,22 +508,35 @@ struct LinearizeVectorInsert final
       linearizedOffset += offset * dstSizeForOffsets;
     }
 
+    Location loc = insertOp.getLoc();
+    Value valueToStore = adaptor.getValueToStore();
+
+    if (!isa<VectorType>(valueToStore.getType())) {
+      // Scalar case: generate a 1-D insert.
+      Value result = rewriter.createOrFold<vector::InsertOp>(
+          loc, valueToStore, adaptor.getDest(), linearizedOffset);
+      rewriter.replaceOp(insertOp, result);
+      return success();
+    }
+
+    // Vector case: generate a shuffle.
     llvm::SmallVector<int64_t, 2> indices(dstSize);
     auto *origValsUntil = indices.begin();
     std::advance(origValsUntil, linearizedOffset);
-    std::iota(indices.begin(), origValsUntil,
-              0); // original values that remain [0, offset)
+
+    // Original values that remain [0, offset).
+    std::iota(indices.begin(), origValsUntil, 0);
     auto *newValsUntil = origValsUntil;
     std::advance(newValsUntil, srcSize);
-    std::iota(origValsUntil, newValsUntil,
-              dstSize); // new values [offset, offset+srcNumElements)
-    std::iota(newValsUntil, indices.end(),
-              linearizedOffset + srcSize); // the rest of original values
-                                           // [offset+srcNumElements, end)
+    // New values [offset, offset+srcNumElements).
+    std::iota(origValsUntil, newValsUntil, dstSize);
+    // The rest of original values [offset+srcNumElements, end);
+    std::iota(newValsUntil, indices.end(), linearizedOffset + srcSize);
 
-    rewriter.replaceOpWithNewOp<vector::ShuffleOp>(
-        insertOp, dstTy, adaptor.getDest(), adaptor.getValueToStore(), indices);
+    Value result = rewriter.createOrFold<vector::ShuffleOp>(
+        loc, dstTy, adaptor.getDest(), valueToStore, indices);
 
+    rewriter.replaceOp(insertOp, result);
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 785a8aaf3f0a9..efdae93e730bd 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -660,7 +660,8 @@ class FlattenContiguousRowMajorTransferReadPattern
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     vector::TransferReadOp flatRead = rewriter.create<vector::TransferReadOp>(
-        loc, flatVectorType, collapsedSource, collapsedIndices, collapsedMap);
+        loc, flatVectorType, collapsedSource, collapsedIndices,
+        transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
     // 4. Replace the old transfer_read with the new one reading from the
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index 36fc55f3f311d..bcaea1c79471f 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -2266,11 +2266,6 @@ void mlir::vector::populateVectorMaskMaterializationPatterns(
 
 void mlir::vector::populateDropUnitDimWithShapeCastPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
-  // TODO: Consider either:
-  //  * including DropInnerMostUnitDimsTransferRead and
-  //    DropInnerMostUnitDimsTransferWrite, or
-  //  * better naming to distinguish this and
-  //    populateVectorTransferCollapseInnerMostContiguousDimsPatterns.
   patterns.add<DropUnitDimFromElementwiseOps, DropUnitDimsFromScfForOp,
                DropUnitDimsFromTransposeOp>(patterns.getContext(), benefit);
 }
@@ -2305,9 +2300,8 @@ void mlir::vector::populateVectorReductionToContractPatterns(
       patterns.getContext(), benefit);
 }
 
-void mlir::vector::
-    populateVectorTransferCollapseInnerMostContiguousDimsPatterns(
-        RewritePatternSet &patterns, PatternBenefit benefit) {
+void mlir::vector::populateDropInnerMostUnitDimsXferOpPatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<DropInnerMostUnitDimsTransferRead,
                DropInnerMostUnitDimsTransferWrite>(patterns.getContext(),
                                                    benefit);
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 674a93d7c520e..7e4984582b373 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -339,7 +339,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
     // FIXME: This computation is too weak - it ignores the read indices.
     for (unsigned i = 0; i < readRank; i++)
       inBoundsVal[i] = (sourceShape[i] == inputVectorSizes[i]) &&
-                       !ShapedType::isDynamic(sourceShape[i]);
+                       ShapedType::isStatic(sourceShape[i]);
   }
   auto transferReadOp = builder.create<vector::TransferReadOp>(
       loc,
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 2793c7a35bc97..3bea6cb50ff7b 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -135,8 +135,8 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
          shape.size() == strides.size() && shape.size() == offsets.size());
 
   Type srcTy = source.getType();
-  assert(isa<IntegerType>(srcTy) ||
-         isa<MemRefType>(srcTy) && "Source has to be either int or memref.");
+  assert((isa<IntegerType, MemRefType>(srcTy)) &&
+         "Source has to be either int or memref.");
 
   llvm::SmallVector<Value> dynamicOffsets;
   llvm::SmallVector<Value> dynamicShape;
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index e3563d10bc6f1..be7b860dd1729 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -376,10 +376,12 @@ struct WgToSgElementwiseOp : public ConversionPattern {
       // Copy all attributes, but update "layout_result_0" to drop
       // sgLayout/sgData
       for (auto attr : op->getAttrs()) {
-        if (auto layout = dyn_cast<xegpu::LayoutAttr>(attr.getValue()))
-          state.addAttribute(attr.getName(), layout.dropSgLayoutAndData());
-        else
+        if (auto layout = dyn_cast<xegpu::LayoutAttr>(attr.getValue())) {
+          if (auto newLayout = layout.dropSgLayoutAndData())
+            state.addAttribute(attr.getName(), newLayout);
+        } else {
           state.addAttribute(attr.getName(), attr.getValue());
+        }
       }
       Operation *newOp = rewriter.create(state);
       newResults.push_back(newOp->getResult(0));
@@ -629,8 +631,10 @@ void XeGPUWgToSgDistributePass::runOnOperation() {
       std::string name = xegpu::getLayoutName(result);
       if (auto layout = op->getAttrOfType<xegpu::LayoutAttr>(name)) {
         op->removeAttr(name);
-        if (!isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::ConditionOp>(op))
-          op->setAttr(name, layout.dropSgLayoutAndData());
+        if (!isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::ConditionOp>(op)) {
+          if (auto newLayout = layout.dropSgLayoutAndData())
+            op->setAttr(name, newLayout);
+        }
       }
     }
   });
diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp
index cc81f9d19aca7..0652202fe331a 100644
--- a/mlir/lib/IR/AffineExpr.cpp
+++ b/mlir/lib/IR/AffineExpr.cpp
@@ -781,9 +781,45 @@ static AffineExpr simplifyAdd(AffineExpr lhs, AffineExpr rhs) {
   if (isPositiveRhs && lhs == llrhs && rlrhs == -rrhs) {
     return lhs % rlrhs;
   }
+
+  // Try simplify lhs's last operand with rhs. e.g:
+  // (s0 * 64 + s1) + (s1 // c * -c) --->
+  // s0 * 64 + (s1 + s1 // c * -c) -->
+  // s0 * 64 + s1 % c
+  if (lBinOpExpr && lBinOpExpr.getKind() == AffineExprKind::Add) {
+    if (auto simplified = simplifyAdd(lBinOpExpr.getRHS(), rhs))
+      return lBinOpExpr.getLHS() + simplified;
+  }
   return nullptr;
 }
 
+/// Get the canonical order of two commutative exprs arguments.
+static std::pair<AffineExpr, AffineExpr>
+orderCommutativeArgs(AffineExpr expr1, AffineExpr expr2) {
+  auto sym1 = dyn_cast<AffineSymbolExpr>(expr1);
+  auto sym2 = dyn_cast<AffineSymbolExpr>(expr2);
+  // Try to order by symbol/dim position first.
+  if (sym1 && sym2)
+    return sym1.getPosition() < sym2.getPosition() ? std::pair{expr1, expr2}
+                                                   : std::pair{expr2, expr1};
+
+  auto dim1 = dyn_cast<AffineDimExpr>(expr1);
+  auto dim2 = dyn_cast<AffineDimExpr>(expr2);
+  if (dim1 && dim2)
+    return dim1.getPosition() < dim2.getPosition() ? std::pair{expr1, expr2}
+                                                   : std::pair{expr2, expr1};
+
+  // Put dims before symbols.
+  if (dim1 && sym2)
+    return {dim1, sym2};
+
+  if (sym1 && dim2)
+    return {dim2, sym1};
+
+  // Otherwise, keep original order.
+  return {expr1, expr2};
+}
+
 AffineExpr AffineExpr::operator+(int64_t v) const {
   return *this + getAffineConstantExpr(v, getContext());
 }
@@ -791,9 +827,11 @@ AffineExpr AffineExpr::operator+(AffineExpr other) const {
   if (auto simplified = simplifyAdd(*this, other))
     return simplified;
 
+  auto [lhs, rhs] = orderCommutativeArgs(*this, other);
+
   StorageUniquer &uniquer = getContext()->getAffineUniquer();
   return uniquer.get<AffineBinaryOpExprStorage>(
-      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Add), *this, other);
+      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Add), lhs, rhs);
 }
 
 /// Simplify a multiply expression. Return nullptr if it can't be simplified.
@@ -856,9 +894,11 @@ AffineExpr AffineExpr::operator*(AffineExpr other) const {
   if (auto simplified = simplifyMul(*this, other))
     return simplified;
 
+  auto [lhs, rhs] = orderCommutativeArgs(*this, other);
+
   StorageUniquer &uniquer = getContext()->getAffineUniquer();
   return uniquer.get<AffineBinaryOpExprStorage>(
-      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Mul), *this, other);
+      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Mul), lhs, rhs);
 }
 
 // Unary minus, delegate to operator*.
@@ -1362,7 +1402,7 @@ LogicalResult SimpleAffineExprFlattener::visitModExpr(AffineBinaryOpExpr expr) {
       break;
   // If yes, modulo expression here simplifies to zero.
   if (i == lhs.size()) {
-    std::fill(lhs.begin(), lhs.end(), 0);
+    llvm::fill(lhs, 0);
     return success();
   }
 
@@ -1442,7 +1482,7 @@ LogicalResult SimpleAffineExprFlattener::addLocalVariableSemiAffine(
     if (failed(addLocalIdSemiAffine(lhs, rhs, localExpr)))
       return failure();
   }
-  std::fill(result.begin(), result.end(), 0);
+  llvm::fill(result, 0);
   if (loc == -1)
     result[getLocalVarStartIndex() + numLocals - 1] = 1;
   else
@@ -1529,7 +1569,7 @@ LogicalResult SimpleAffineExprFlattener::visitDivExpr(AffineBinaryOpExpr expr,
   }
   // Set the expression on stack to the local var introduced to capture the
   // result of the division (floor or ceil).
-  std::fill(lhs.begin(), lhs.end(), 0);
+  llvm::fill(lhs, 0);
   if (loc == -1)
     lhs[getLocalVarStartIndex() + numLocals - 1] = 1;
   else
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 2838843e2eba6..7235b442aa567 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -14,12 +14,8 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MathExtras.h"
-#include "llvm/Support/raw_ostream.h"
-#include <iterator>
 #include <numeric>
 #include <optional>
 #include <type_traits>
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index c7cc6a02ad208..f95ad290a1981 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -36,7 +36,6 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -442,6 +441,7 @@ class AsmPrinter::Impl {
   /// Print the given attribute without considering an alias.
   void printAttributeImpl(Attribute attr,
                           AttrTypeElision typeElision = AttrTypeElision::Never);
+  void printNamedAttribute(NamedAttribute attr);
 
   /// Print the alias for the given attribute, return failure if no alias could
   /// be printed.
@@ -481,7 +481,6 @@ class AsmPrinter::Impl {
   void printOptionalAttrDict(ArrayRef<NamedAttribute> attrs,
                              ArrayRef<StringRef> elidedAttrs = {},
                              bool withKeyword = false);
-  void printNamedAttribute(NamedAttribute attr);
   void printTrailingLocation(Location loc, bool allowAlias = true);
   void printLocationInternal(LocationAttr loc, bool pretty = false,
                              bool isTopLevel = false);
@@ -801,6 +800,10 @@ class DummyAliasOperationPrinter : private OpAsmPrinter {
   void printAttributeWithoutType(Attribute attr) override {
     printAttribute(attr);
   }
+  void printNamedAttribute(NamedAttribute attr) override {
+    printAttribute(attr.getValue());
+  }
+
   LogicalResult printAlias(Attribute attr) override {
     initializer.visit(attr);
     return success();
@@ -975,6 +978,10 @@ class DummyAliasDialectAsmPrinter : public DialectAsmPrinter {
     recordAliasResult(
         initializer.visit(attr, canBeDeferred, /*elideType=*/true));
   }
+  void printNamedAttribute(NamedAttribute attr) override {
+    printAttribute(attr.getValue());
+  }
+
   LogicalResult printAlias(Attribute attr) override {
     printAttribute(attr);
     return success();
@@ -2381,7 +2388,6 @@ void AsmPrinter::Impl::printAttribute(Attribute attr,
     return;
   return printAttributeImpl(attr, typeElision);
 }
-
 void AsmPrinter::Impl::printAttributeImpl(Attribute attr,
                                           AttrTypeElision typeElision) {
   if (!isa<BuiltinDialect>(attr.getDialect())) {
@@ -2973,6 +2979,11 @@ void AsmPrinter::printAttributeWithoutType(Attribute attr) {
   impl->printAttribute(attr, Impl::AttrTypeElision::Must);
 }
 
+void AsmPrinter::printNamedAttribute(NamedAttribute attr) {
+  assert(impl && "expected AsmPrinter::printNamedAttribute to be overriden");
+  impl->printNamedAttribute(attr);
+}
+
 void AsmPrinter::printKeywordOrString(StringRef keyword) {
   assert(impl && "expected AsmPrinter::printKeywordOrString to be overriden");
   ::printKeywordOrString(keyword, impl->getStream());
diff --git a/mlir/lib/IR/Block.cpp b/mlir/lib/IR/Block.cpp
index 959a60c044502..57825d9b42178 100644
--- a/mlir/lib/IR/Block.cpp
+++ b/mlir/lib/IR/Block.cpp
@@ -10,8 +10,6 @@
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Operation.h"
-#include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index 5f7bc50afc418..f657db142eeb9 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -12,11 +12,8 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/IRMapping.h"
-#include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/SymbolTable.h"
 #include "llvm/ADT/SmallVectorExtras.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index 617dcc222cd6e..fd898b7493c7f 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -18,8 +18,6 @@
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APSInt.h"
-#include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include <optional>
@@ -232,8 +230,8 @@ void StridedLayoutAttr::print(llvm::raw_ostream &os) const {
 /// Returns true if this layout is static, i.e. the strides and offset all have
 /// a known value > 0.
 bool StridedLayoutAttr::hasStaticLayout() const {
-  return !ShapedType::isDynamic(getOffset()) &&
-         !ShapedType::isDynamicShape(getStrides());
+  return ShapedType::isStatic(getOffset()) &&
+         ShapedType::isStaticShape(getStrides());
 }
 
 /// Returns the strided layout as an affine map.
@@ -1820,7 +1818,7 @@ AffineMap mlir::makeStridedLinearLayoutMap(ArrayRef<int64_t> strides,
 
   // AffineExpr for offset.
   // Static case.
-  if (!ShapedType::isDynamic(offset)) {
+  if (ShapedType::isStatic(offset)) {
     auto cst = getAffineConstantExpr(offset, context);
     expr = cst;
   } else {
@@ -1836,7 +1834,7 @@ AffineMap mlir::makeStridedLinearLayoutMap(ArrayRef<int64_t> strides,
     auto d = getAffineDimExpr(dim, context);
     AffineExpr mult;
     // Static case.
-    if (!ShapedType::isDynamic(stride))
+    if (ShapedType::isStatic(stride))
       mult = getAffineConstantExpr(stride, context);
     else
       // Dynamic case, new symbol for each new stride.
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 2867d9b4ef68a..6d7e2aa0ece7d 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -17,9 +17,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeRange.h"
 
 using namespace mlir;
diff --git a/mlir/lib/IR/BuiltinTypeInterfaces.cpp b/mlir/lib/IR/BuiltinTypeInterfaces.cpp
index c663f6c909460..ce076a3b4ac5e 100644
--- a/mlir/lib/IR/BuiltinTypeInterfaces.cpp
+++ b/mlir/lib/IR/BuiltinTypeInterfaces.cpp
@@ -7,9 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/Diagnostics.h"
 #include "llvm/ADT/APFloat.h"
-#include "llvm/ADT/Sequence.h"
 
 using namespace mlir;
 using namespace mlir::detail;
diff --git a/mlir/lib/IR/BuiltinTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
index 6661efa8907b7..1604ebba190a1 100644
--- a/mlir/lib/IR/BuiltinTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -17,9 +17,7 @@
 #include "mlir/IR/TensorEncoding.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/ADT/APFloat.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
@@ -323,7 +321,7 @@ RankedTensorType::verify(function_ref<InFlightDiagnostic()> emitError,
                          ArrayRef<int64_t> shape, Type elementType,
                          Attribute encoding) {
   for (int64_t s : shape)
-    if (s < 0 && !ShapedType::isDynamic(s))
+    if (s < 0 && ShapedType::isStatic(s))
       return emitError() << "invalid tensor dimension size";
   if (auto v = llvm::dyn_cast_or_null<VerifiableTensorEncoding>(encoding))
     if (failed(v.verifyEncoding(shape, elementType, emitError)))
@@ -646,7 +644,7 @@ LogicalResult MemRefType::verify(function_ref<InFlightDiagnostic()> emitError,
 
   // Negative sizes are not allowed except for `kDynamic`.
   for (int64_t s : shape)
-    if (s < 0 && !ShapedType::isDynamic(s))
+    if (s < 0 && ShapedType::isStatic(s))
       return emitError() << "invalid memref size";
 
   assert(layout && "missing layout specification");
diff --git a/mlir/lib/IR/Diagnostics.cpp b/mlir/lib/IR/Diagnostics.cpp
index 3a0f81d240336..3e337951bcd3f 100644
--- a/mlir/lib/IR/Diagnostics.cpp
+++ b/mlir/lib/IR/Diagnostics.cpp
@@ -727,7 +727,7 @@ SourceMgrDiagnosticVerifierHandlerImpl::computeExpectedDiags(
     raw_ostream &os, llvm::SourceMgr &mgr, const llvm::MemoryBuffer *buf) {
   // If the buffer is invalid, return an empty list.
   if (!buf)
-    return std::nullopt;
+    return {};
   auto &expectedDiags = expectedDiagsPerFile[buf->getBufferIdentifier()];
 
   // The number of the last line that did not correlate to a designator.
diff --git a/mlir/lib/IR/Dialect.cpp b/mlir/lib/IR/Dialect.cpp
index 47568594947cf..f84fe89157074 100644
--- a/mlir/lib/IR/Dialect.cpp
+++ b/mlir/lib/IR/Dialect.cpp
@@ -17,12 +17,9 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/TypeID.h"
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Regex.h"
 #include <memory>
 
diff --git a/mlir/lib/IR/Dominance.cpp b/mlir/lib/IR/Dominance.cpp
index 1c54e09d29b9b..0e53b431b5d31 100644
--- a/mlir/lib/IR/Dominance.cpp
+++ b/mlir/lib/IR/Dominance.cpp
@@ -14,7 +14,6 @@
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/RegionKindInterface.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/GenericDomTreeConstruction.h"
 
 using namespace mlir;
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index ce5e63a32dc72..716d9c85a377d 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -25,11 +25,7 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
-#include "mlir/IR/Types.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/mlir/lib/IR/ODSSupport.cpp b/mlir/lib/IR/ODSSupport.cpp
index 5b0a3e22139e1..69b4a563e6b75 100644
--- a/mlir/lib/IR/ODSSupport.cpp
+++ b/mlir/lib/IR/ODSSupport.cpp
@@ -173,3 +173,8 @@ Attribute mlir::convertToAttribute(MLIRContext *ctx,
                                    ArrayRef<int64_t> storage) {
   return DenseI64ArrayAttr::get(ctx, storage);
 }
+
+Attribute mlir::convertToAttribute(MLIRContext *ctx,
+                                   ArrayRef<int32_t> storage) {
+  return DenseI32ArrayAttr::get(ctx, storage);
+}
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index a479ff97f3f16..8bcfa465e4a22 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -19,7 +19,6 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/FoldInterfaces.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <numeric>
 #include <optional>
@@ -810,13 +809,16 @@ void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties,
     ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
     llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
                                                   elidedProps.end());
-    bool atLeastOneAttr = llvm::any_of(attrs, [&](NamedAttribute attr) {
-      return !elidedAttrsSet.contains(attr.getName().strref());
-    });
-    if (atLeastOneAttr) {
-      p << "<";
-      p.printOptionalAttrDict(dictAttr.getValue(), elidedProps);
-      p << ">";
+    auto filteredAttrs =
+        llvm::make_filter_range(attrs, [&](NamedAttribute attr) {
+          return !elidedAttrsSet.contains(attr.getName().strref());
+        });
+    if (!filteredAttrs.empty()) {
+      p << "<{";
+      interleaveComma(filteredAttrs, p, [&](NamedAttribute attr) {
+        p.printNamedAttribute(attr);
+      });
+      p << "}>";
     }
   } else {
     p << "<" << properties << ">";
diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp
index d3e9aebc4dc24..394ac7765ed61 100644
--- a/mlir/lib/IR/OperationSupport.cpp
+++ b/mlir/lib/IR/OperationSupport.cpp
@@ -15,7 +15,6 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/Support/SHA1.h"
 #include <numeric>
 #include <optional>
diff --git a/mlir/lib/IR/TypeRange.cpp b/mlir/lib/IR/TypeRange.cpp
index 26ffc0d5485de..88e788aa1b2b8 100644
--- a/mlir/lib/IR/TypeRange.cpp
+++ b/mlir/lib/IR/TypeRange.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/TypeRange.h"
-#include "mlir/IR/Operation.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/IR/TypeUtilities.cpp b/mlir/lib/IR/TypeUtilities.cpp
index ec646cad841ae..d2d115ea39277 100644
--- a/mlir/lib/IR/TypeUtilities.cpp
+++ b/mlir/lib/IR/TypeUtilities.cpp
@@ -62,7 +62,7 @@ LogicalResult mlir::verifyCompatibleShape(ArrayRef<int64_t> shape1,
   for (auto dims : llvm::zip(shape1, shape2)) {
     int64_t dim1 = std::get<0>(dims);
     int64_t dim2 = std::get<1>(dims);
-    if (!ShapedType::isDynamic(dim1) && !ShapedType::isDynamic(dim2) &&
+    if (ShapedType::isStatic(dim1) && ShapedType::isStatic(dim2) &&
         dim1 != dim2)
       return failure();
   }
diff --git a/mlir/lib/IR/Unit.cpp b/mlir/lib/IR/Unit.cpp
index c109d387e6b74..58c1d27db87f0 100644
--- a/mlir/lib/IR/Unit.cpp
+++ b/mlir/lib/IR/Unit.cpp
@@ -12,7 +12,6 @@
 #include "mlir/IR/Region.h"
 #include "llvm/Support/raw_ostream.h"
 #include <iterator>
-#include <sstream>
 
 using namespace mlir;
 
diff --git a/mlir/lib/Interfaces/ViewLikeInterface.cpp b/mlir/lib/Interfaces/ViewLikeInterface.cpp
index d70cfebbe53e6..3112da9ef182a 100644
--- a/mlir/lib/Interfaces/ViewLikeInterface.cpp
+++ b/mlir/lib/Interfaces/ViewLikeInterface.cpp
@@ -124,12 +124,12 @@ mlir::detail::verifyOffsetSizeAndStrideOp(OffsetSizeAndStrideOpInterface op) {
     return failure();
 
   for (int64_t offset : op.getStaticOffsets()) {
-    if (offset < 0 && !ShapedType::isDynamic(offset))
+    if (offset < 0 && ShapedType::isStatic(offset))
       return op->emitError("expected offsets to be non-negative, but got ")
              << offset;
   }
   for (int64_t size : op.getStaticSizes()) {
-    if (size < 0 && !ShapedType::isDynamic(size))
+    if (size < 0 && ShapedType::isStatic(size))
       return op->emitError("expected sizes to be non-negative, but got ")
              << size;
   }
diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index d1371d2b508f4..dafec39fd5eb0 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -829,7 +829,7 @@ void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
 
   // An atomic failure variable for the async executors.
   std::vector<std::atomic<bool>> activePMs(asyncExecutors.size());
-  std::fill(activePMs.begin(), activePMs.end(), false);
+  llvm::fill(activePMs, false);
   std::atomic<bool> hasFailure = false;
   parallelForEach(context, opInfos, [&](OpPMInfo &opInfo) {
     // Find an executor for this operation.
diff --git a/mlir/lib/Query/Matcher/CMakeLists.txt b/mlir/lib/Query/Matcher/CMakeLists.txt
index 629479bf7adc1..ba202762fdfbb 100644
--- a/mlir/lib/Query/Matcher/CMakeLists.txt
+++ b/mlir/lib/Query/Matcher/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_mlir_library(MLIRQueryMatcher
   MatchFinder.cpp
+  MatchersInternal.cpp
   Parser.cpp
   RegistryManager.cpp
   VariantValue.cpp
diff --git a/mlir/lib/Query/Matcher/MatchersInternal.cpp b/mlir/lib/Query/Matcher/MatchersInternal.cpp
new file mode 100644
index 0000000000000..01f412ade846b
--- /dev/null
+++ b/mlir/lib/Query/Matcher/MatchersInternal.cpp
@@ -0,0 +1,33 @@
+//===--- MatchersInternal.cpp----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Query/Matcher/MatchersInternal.h"
+#include "llvm/ADT/SetVector.h"
+
+namespace mlir::query::matcher {
+
+namespace internal {
+
+bool allOfVariadicOperator(Operation *op, SetVector<Operation *> *matchedOps,
+                           ArrayRef<DynMatcher> innerMatchers) {
+  return llvm::all_of(innerMatchers, [&](const DynMatcher &matcher) {
+    if (matchedOps)
+      return matcher.match(op, *matchedOps);
+    return matcher.match(op);
+  });
+}
+bool anyOfVariadicOperator(Operation *op, SetVector<Operation *> *matchedOps,
+                           ArrayRef<DynMatcher> innerMatchers) {
+  return llvm::any_of(innerMatchers, [&](const DynMatcher &matcher) {
+    if (matchedOps)
+      return matcher.match(op, *matchedOps);
+    return matcher.match(op);
+  });
+}
+} // namespace internal
+} // namespace mlir::query::matcher
diff --git a/mlir/lib/Query/Matcher/RegistryManager.cpp b/mlir/lib/Query/Matcher/RegistryManager.cpp
index 4b511c5f009e7..08b610453b11a 100644
--- a/mlir/lib/Query/Matcher/RegistryManager.cpp
+++ b/mlir/lib/Query/Matcher/RegistryManager.cpp
@@ -64,7 +64,7 @@ std::vector<ArgKind> RegistryManager::getAcceptedCompletionTypes(
     unsigned argNumber = ctxEntry.second;
     std::vector<ArgKind> nextTypeSet;
 
-    if (argNumber < ctor->getNumArgs())
+    if (ctor->isVariadic() || argNumber < ctor->getNumArgs())
       ctor->getArgKinds(argNumber, nextTypeSet);
 
     typeSet.insert(nextTypeSet.begin(), nextTypeSet.end());
@@ -83,7 +83,7 @@ RegistryManager::getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes,
     const internal::MatcherDescriptor &matcher = *m.getValue();
     llvm::StringRef name = m.getKey();
 
-    unsigned numArgs = matcher.getNumArgs();
+    unsigned numArgs = matcher.isVariadic() ? 1 : matcher.getNumArgs();
     std::vector<std::vector<ArgKind>> argKinds(numArgs);
 
     for (const ArgKind &kind : acceptedTypes) {
@@ -115,6 +115,9 @@ RegistryManager::getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes,
       }
     }
 
+    if (matcher.isVariadic())
+      os << ",...";
+
     os << ")";
     typedText += "(";
 
diff --git a/mlir/lib/Query/Matcher/VariantValue.cpp b/mlir/lib/Query/Matcher/VariantValue.cpp
index 1cb2d48f9d56f..98ed4cc5df311 100644
--- a/mlir/lib/Query/Matcher/VariantValue.cpp
+++ b/mlir/lib/Query/Matcher/VariantValue.cpp
@@ -27,12 +27,64 @@ class VariantMatcher::SinglePayload : public VariantMatcher::Payload {
   DynMatcher matcher;
 };
 
+class VariantMatcher::VariadicOpPayload : public VariantMatcher::Payload {
+public:
+  VariadicOpPayload(DynMatcher::VariadicOperator varOp,
+                    std::vector<VariantMatcher> args)
+      : varOp(varOp), args(std::move(args)) {}
+
+  std::optional<DynMatcher> getDynMatcher() const override {
+    std::vector<DynMatcher> dynMatchers;
+    for (auto variantMatcher : args) {
+      std::optional<DynMatcher> dynMatcher = variantMatcher.getDynMatcher();
+      if (dynMatcher)
+        dynMatchers.push_back(dynMatcher.value());
+    }
+    auto result = DynMatcher::constructVariadic(varOp, dynMatchers);
+    return *result;
+  }
+
+  std::string getTypeAsString() const override {
+    std::string inner;
+    llvm::interleave(
+        args, [&](auto const &arg) { inner += arg.getTypeAsString(); },
+        [&] { inner += " & "; });
+    return inner;
+  }
+
+private:
+  const DynMatcher::VariadicOperator varOp;
+  const std::vector<VariantMatcher> args;
+};
+
 VariantMatcher::VariantMatcher() = default;
 
 VariantMatcher VariantMatcher::SingleMatcher(DynMatcher matcher) {
   return VariantMatcher(std::make_shared<SinglePayload>(std::move(matcher)));
 }
 
+VariantMatcher
+VariantMatcher::VariadicOperatorMatcher(DynMatcher::VariadicOperator varOp,
+                                        ArrayRef<VariantMatcher> args) {
+  return VariantMatcher(
+      std::make_shared<VariadicOpPayload>(varOp, std::move(args)));
+}
+
+std::optional<DynMatcher> VariantMatcher::MatcherOps::constructVariadicOperator(
+    DynMatcher::VariadicOperator varOp,
+    ArrayRef<VariantMatcher> innerMatchers) const {
+  std::vector<DynMatcher> dynMatchers;
+  for (const auto &innerMatcher : innerMatchers) {
+    if (!innerMatcher.value)
+      return std::nullopt;
+    std::optional<DynMatcher> inner = innerMatcher.value->getDynMatcher();
+    if (!inner)
+      return std::nullopt;
+    dynMatchers.push_back(*inner);
+  }
+  return *DynMatcher::constructVariadic(varOp, dynMatchers);
+}
+
 std::optional<DynMatcher> VariantMatcher::getDynMatcher() const {
   return value ? value->getDynMatcher() : std::nullopt;
 }
@@ -120,11 +172,11 @@ void VariantValue::setSigned(int64_t newValue) {
 // Boolean
 bool VariantValue::isBoolean() const { return type == ValueType::Boolean; }
 
-bool VariantValue::getBoolean() const { return value.Signed; }
+bool VariantValue::getBoolean() const { return value.Boolean; }
 
 void VariantValue::setBoolean(bool newValue) {
   type = ValueType::Boolean;
-  value.Signed = newValue;
+  value.Boolean = newValue;
 }
 
 bool VariantValue::isString() const { return type == ValueType::String; }
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
index 803284d6df86a..637e1f3cdef87 100644
--- a/mlir/lib/Query/Query.cpp
+++ b/mlir/lib/Query/Query.cpp
@@ -10,6 +10,7 @@
 #include "QueryParser.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/IRMapping.h"
+#include "mlir/IR/Verifier.h"
 #include "mlir/Query/Matcher/MatchFinder.h"
 #include "mlir/Query/QuerySession.h"
 #include "llvm/ADT/SetVector.h"
@@ -68,6 +69,8 @@ static Operation *extractFunction(std::vector<Operation *> &ops,
   // Clone operations and build function body
   std::vector<Operation *> clonedOps;
   std::vector<Value> clonedVals;
+  // TODO: Handle extraction of operations with compute payloads defined via
+  // regions.
   for (Operation *slicedOp : slice) {
     Operation *clonedOp =
         clonedOps.emplace_back(builder.clone(*slicedOp, mapper));
@@ -129,6 +132,8 @@ LogicalResult MatchQuery::run(llvm::raw_ostream &os, QuerySession &qs) const {
         finder.flattenMatchedOps(matches);
     Operation *function =
         extractFunction(flattenedMatches, rootOp->getContext(), functionName);
+    if (failed(verify(function)))
+      return mlir::failure();
     os << "\n" << *function << "\n\n";
     function->erase();
     return mlir::success();
diff --git a/mlir/lib/TableGen/Argument.cpp b/mlir/lib/TableGen/Argument.cpp
index c847760f8a467..8dec21d9a354c 100644
--- a/mlir/lib/TableGen/Argument.cpp
+++ b/mlir/lib/TableGen/Argument.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Argument.h"
-#include "llvm/TableGen/Record.h"
 
 using namespace mlir;
 using namespace mlir::tblgen;
diff --git a/mlir/lib/TableGen/Attribute.cpp b/mlir/lib/TableGen/Attribute.cpp
index 142d194260942..d53ccea576b8b 100644
--- a/mlir/lib/TableGen/Attribute.cpp
+++ b/mlir/lib/TableGen/Attribute.cpp
@@ -11,7 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/Operator.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/TableGen/Class.cpp b/mlir/lib/TableGen/Class.cpp
index fedf64fd96b0d..c65f67d50a47d 100644
--- a/mlir/lib/TableGen/Class.cpp
+++ b/mlir/lib/TableGen/Class.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Class.h"
-#include "mlir/TableGen/Format.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
 
diff --git a/mlir/lib/TableGen/Interfaces.cpp b/mlir/lib/TableGen/Interfaces.cpp
index 29d8497aacff4..ec7adf3b02c21 100644
--- a/mlir/lib/TableGen/Interfaces.cpp
+++ b/mlir/lib/TableGen/Interfaces.cpp
@@ -8,9 +8,7 @@
 
 #include "mlir/TableGen/Interfaces.h"
 #include "llvm/ADT/FunctionExtras.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 07520a2f94d77..da86b00ebc300 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -15,7 +15,6 @@
 #include "mlir/TableGen/Predicate.h"
 #include "mlir/TableGen/Trait.h"
 #include "mlir/TableGen/Type.h"
-#include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -26,7 +25,6 @@
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
-#include <list>
 
 #define DEBUG_TYPE "mlir-tblgen-operator"
 
diff --git a/mlir/lib/TableGen/Predicate.cpp b/mlir/lib/TableGen/Predicate.cpp
index 11afa27caf709..a48c7468251ef 100644
--- a/mlir/lib/TableGen/Predicate.cpp
+++ b/mlir/lib/TableGen/Predicate.cpp
@@ -14,7 +14,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/TableGen/Property.cpp b/mlir/lib/TableGen/Property.cpp
index 47f43267cd197..b003d74c7bdee 100644
--- a/mlir/lib/TableGen/Property.cpp
+++ b/mlir/lib/TableGen/Property.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Property.h"
-#include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/Operator.h"
 #include "mlir/TableGen/Predicate.h"
 #include "llvm/TableGen/Record.h"
diff --git a/mlir/lib/TableGen/Trait.cpp b/mlir/lib/TableGen/Trait.cpp
index 6246ba959b00a..8fb8c3472f43d 100644
--- a/mlir/lib/TableGen/Trait.cpp
+++ b/mlir/lib/TableGen/Trait.cpp
@@ -13,8 +13,6 @@
 #include "mlir/TableGen/Trait.h"
 #include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/Predicate.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index c04548688bcf6..06977e07fda5b 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -186,6 +186,10 @@ struct CppEmitter {
   /// Return the existing or a new name for a Value.
   StringRef getOrCreateName(Value val);
 
+  /// Return the existing or a new name for a loop induction variable of an
+  /// emitc::ForOp.
+  StringRef getOrCreateInductionVarName(Value val);
+
   // Returns the textual representation of a subscript operation.
   std::string getSubscriptName(emitc::SubscriptOp op);
 
@@ -201,23 +205,39 @@ struct CppEmitter {
   /// Whether to map an mlir integer to a unsigned integer in C++.
   bool shouldMapToUnsigned(IntegerType::SignednessSemantics val);
 
-  /// RAII helper function to manage entering/exiting C++ scopes.
+  /// Abstract RAII helper function to manage entering/exiting C++ scopes.
   struct Scope {
+    ~Scope() { emitter.labelInScopeCount.pop(); }
+
+  private:
+    llvm::ScopedHashTableScope<Value, std::string> valueMapperScope;
+    llvm::ScopedHashTableScope<Block *, std::string> blockMapperScope;
+
+  protected:
     Scope(CppEmitter &emitter)
         : valueMapperScope(emitter.valueMapper),
           blockMapperScope(emitter.blockMapper), emitter(emitter) {
-      emitter.valueInScopeCount.push(emitter.valueInScopeCount.top());
       emitter.labelInScopeCount.push(emitter.labelInScopeCount.top());
     }
-    ~Scope() {
-      emitter.valueInScopeCount.pop();
-      emitter.labelInScopeCount.pop();
+    CppEmitter &emitter;
+  };
+
+  /// RAII helper function to manage entering/exiting functions, while re-using
+  /// value names.
+  struct FunctionScope : Scope {
+    FunctionScope(CppEmitter &emitter) : Scope(emitter) {
+      // Re-use value names.
+      emitter.resetValueCounter();
     }
+  };
 
-  private:
-    llvm::ScopedHashTableScope<Value, std::string> valueMapperScope;
-    llvm::ScopedHashTableScope<Block *, std::string> blockMapperScope;
-    CppEmitter &emitter;
+  /// RAII helper function to manage entering/exiting emitc::forOp loops and
+  /// handle induction variable naming.
+  struct LoopScope : Scope {
+    LoopScope(CppEmitter &emitter) : Scope(emitter) {
+      emitter.increaseLoopNestingLevel();
+    }
+    ~LoopScope() { emitter.decreaseLoopNestingLevel(); }
   };
 
   /// Returns wether the Value is assigned to a C++ variable in the scope.
@@ -253,6 +273,15 @@ struct CppEmitter {
     return operandExpression == emittedExpression;
   };
 
+  // Resets the value counter to 0.
+  void resetValueCounter();
+
+  // Increases the loop nesting level by 1.
+  void increaseLoopNestingLevel();
+
+  // Decreases the loop nesting level by 1.
+  void decreaseLoopNestingLevel();
+
 private:
   using ValueMapper = llvm::ScopedHashTable<Value, std::string>;
   using BlockMapper = llvm::ScopedHashTable<Block *, std::string>;
@@ -274,11 +303,19 @@ struct CppEmitter {
   /// Map from block to name of C++ label.
   BlockMapper blockMapper;
 
-  /// The number of values in the current scope. This is used to declare the
-  /// names of values in a scope.
-  std::stack<int64_t> valueInScopeCount;
+  /// Default values representing outermost scope.
+  llvm::ScopedHashTableScope<Value, std::string> defaultValueMapperScope;
+  llvm::ScopedHashTableScope<Block *, std::string> defaultBlockMapperScope;
+
   std::stack<int64_t> labelInScopeCount;
 
+  /// Keeps track of the amount of nested loops the emitter currently operates
+  /// in.
+  uint64_t loopNestingLevel{0};
+
+  /// Emitter-level count of created values to enable unique identifiers.
+  unsigned int valueCount{0};
+
   /// State of the current expression being emitted.
   ExpressionOp emittedExpression;
   SmallVector<int> emittedExpressionPrecedence;
@@ -860,7 +897,6 @@ static LogicalResult printOperation(CppEmitter &emitter,
 }
 
 static LogicalResult printOperation(CppEmitter &emitter, emitc::ForOp forOp) {
-
   raw_indented_ostream &os = emitter.ostream();
 
   // Utility function to determine whether a value is an expression that will be
@@ -879,12 +915,12 @@ static LogicalResult printOperation(CppEmitter &emitter, emitc::ForOp forOp) {
           emitter.emitType(forOp.getLoc(), forOp.getInductionVar().getType())))
     return failure();
   os << " ";
-  os << emitter.getOrCreateName(forOp.getInductionVar());
+  os << emitter.getOrCreateInductionVarName(forOp.getInductionVar());
   os << " = ";
   if (failed(emitter.emitOperand(forOp.getLowerBound())))
     return failure();
   os << "; ";
-  os << emitter.getOrCreateName(forOp.getInductionVar());
+  os << emitter.getOrCreateInductionVarName(forOp.getInductionVar());
   os << " < ";
   Value upperBound = forOp.getUpperBound();
   bool upperBoundRequiresParentheses = requiresParentheses(upperBound);
@@ -895,13 +931,15 @@ static LogicalResult printOperation(CppEmitter &emitter, emitc::ForOp forOp) {
   if (upperBoundRequiresParentheses)
     os << ")";
   os << "; ";
-  os << emitter.getOrCreateName(forOp.getInductionVar());
+  os << emitter.getOrCreateInductionVarName(forOp.getInductionVar());
   os << " += ";
   if (failed(emitter.emitOperand(forOp.getStep())))
     return failure();
   os << ") {\n";
   os.indent();
 
+  CppEmitter::LoopScope lScope(emitter);
+
   Region &forRegion = forOp.getRegion();
   auto regionOps = forRegion.getOps();
 
@@ -988,8 +1026,6 @@ static LogicalResult printOperation(CppEmitter &emitter,
 }
 
 static LogicalResult printOperation(CppEmitter &emitter, ModuleOp moduleOp) {
-  CppEmitter::Scope scope(emitter);
-
   for (Operation &op : moduleOp) {
     if (failed(emitter.emitOperation(op, /*trailingSemicolon=*/false)))
       return failure();
@@ -998,10 +1034,11 @@ static LogicalResult printOperation(CppEmitter &emitter, ModuleOp moduleOp) {
 }
 
 static LogicalResult printOperation(CppEmitter &emitter, ClassOp classOp) {
-  CppEmitter::Scope classScope(emitter);
   raw_indented_ostream &os = emitter.ostream();
-  os << "class " << classOp.getSymName() << " {\n";
-  os << "public:\n";
+  os << "class " << classOp.getSymName();
+  if (classOp.getFinalSpecifier())
+    os << " final";
+  os << " {\n public:\n";
   os.indent();
 
   for (Operation &op : classOp) {
@@ -1042,8 +1079,6 @@ static LogicalResult printOperation(CppEmitter &emitter, FileOp file) {
   if (!emitter.shouldEmitFile(file))
     return success();
 
-  CppEmitter::Scope scope(emitter);
-
   for (Operation &op : file) {
     if (failed(emitter.emitOperation(op, /*trailingSemicolon=*/false)))
       return failure();
@@ -1159,7 +1194,7 @@ static LogicalResult printOperation(CppEmitter &emitter,
     return functionOp.emitOpError() << "cannot emit array type as result type";
   }
 
-  CppEmitter::Scope scope(emitter);
+  CppEmitter::FunctionScope scope(emitter);
   raw_indented_ostream &os = emitter.ostream();
   if (failed(emitter.emitTypes(functionOp.getLoc(),
                                functionOp.getFunctionType().getResults())))
@@ -1187,7 +1222,7 @@ static LogicalResult printOperation(CppEmitter &emitter,
         "with multiple blocks needs variables declared at top");
   }
 
-  CppEmitter::Scope scope(emitter);
+  CppEmitter::FunctionScope scope(emitter);
   raw_indented_ostream &os = emitter.ostream();
   if (functionOp.getSpecifiers()) {
     for (Attribute specifier : functionOp.getSpecifiersAttr()) {
@@ -1221,7 +1256,6 @@ static LogicalResult printOperation(CppEmitter &emitter,
 
 static LogicalResult printOperation(CppEmitter &emitter,
                                     DeclareFuncOp declareFuncOp) {
-  CppEmitter::Scope scope(emitter);
   raw_indented_ostream &os = emitter.ostream();
 
   auto functionOp = SymbolTable::lookupNearestSymbolFrom<emitc::FuncOp>(
@@ -1253,8 +1287,8 @@ static LogicalResult printOperation(CppEmitter &emitter,
 CppEmitter::CppEmitter(raw_ostream &os, bool declareVariablesAtTop,
                        StringRef fileId)
     : os(os), declareVariablesAtTop(declareVariablesAtTop),
-      fileId(fileId.str()) {
-  valueInScopeCount.push(0);
+      fileId(fileId.str()), defaultValueMapperScope(valueMapper),
+      defaultBlockMapperScope(blockMapper) {
   labelInScopeCount.push(0);
 }
 
@@ -1295,7 +1329,26 @@ StringRef CppEmitter::getOrCreateName(Value val) {
     assert(!hasDeferredEmission(val.getDefiningOp()) &&
            "cacheDeferredOpResult should have been called on this value, "
            "update the emitOperation function.");
-    valueMapper.insert(val, formatv("v{0}", ++valueInScopeCount.top()));
+
+    valueMapper.insert(val, formatv("v{0}", ++valueCount));
+  }
+  return *valueMapper.begin(val);
+}
+
+/// Return the existing or a new name for a loop induction variable Value.
+/// Loop induction variables follow natural naming: i, j, k, ..., t, uX.
+StringRef CppEmitter::getOrCreateInductionVarName(Value val) {
+  if (!valueMapper.count(val)) {
+
+    int64_t identifier = 'i' + loopNestingLevel;
+
+    if (identifier >= 'i' && identifier <= 't') {
+      valueMapper.insert(val,
+                         formatv("{0}{1}", (char)identifier, ++valueCount));
+    } else {
+      // If running out of letters, continue with uX.
+      valueMapper.insert(val, formatv("u{0}", ++valueCount));
+    }
   }
   return *valueMapper.begin(val);
 }
@@ -1836,6 +1889,12 @@ LogicalResult CppEmitter::emitTupleType(Location loc, ArrayRef<Type> types) {
   return success();
 }
 
+void CppEmitter::resetValueCounter() { valueCount = 0; }
+
+void CppEmitter::increaseLoopNestingLevel() { loopNestingLevel++; }
+
+void CppEmitter::decreaseLoopNestingLevel() { loopNestingLevel--; }
+
 LogicalResult emitc::translateToCpp(Operation *op, raw_ostream &os,
                                     bool declareVariablesAtTop,
                                     StringRef fileId) {
diff --git a/mlir/lib/Target/LLVM/ModuleToObject.cpp b/mlir/lib/Target/LLVM/ModuleToObject.cpp
index 04e870acfc5fa..5055cd9938c72 100644
--- a/mlir/lib/Target/LLVM/ModuleToObject.cpp
+++ b/mlir/lib/Target/LLVM/ModuleToObject.cpp
@@ -16,8 +16,6 @@
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
@@ -28,7 +26,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index 565f153b50c4f..b4d53d403cb27 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -16,10 +16,7 @@
 #include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
-#include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/DialectResourceBlobManager.h"
 #include "mlir/Target/LLVM/NVVM/Utils.h"
 #include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
diff --git a/mlir/lib/Target/LLVM/ROCDL/Utils.cpp b/mlir/lib/Target/LLVM/ROCDL/Utils.cpp
index 04b1b22279e5d..abcc3ddcda463 100644
--- a/mlir/lib/Target/LLVM/ROCDL/Utils.cpp
+++ b/mlir/lib/Target/LLVM/ROCDL/Utils.cpp
@@ -12,7 +12,6 @@
 
 #include "mlir/Target/LLVM/ROCDL/Utils.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Frontend/Offloading/Utility.h"
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp b/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
index 5f5ca2af277c3..fbad5c2fb78d9 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
@@ -8,7 +8,6 @@
 
 #include "DataLayoutImporter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index 779c1d12720c1..69ae29f23ba20 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -12,15 +12,11 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Location.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/ErrorHandling.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index 2af3ae0bd7f4f..a55445deddc2d 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -221,7 +221,9 @@ llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) {
 llvm::DILabel *DebugTranslation::translateImpl(DILabelAttr attr) {
   return llvm::DILabel::get(llvmCtx, translate(attr.getScope()),
                             getMDStringOrNull(attr.getName()),
-                            translate(attr.getFile()), attr.getLine());
+                            translate(attr.getFile()), attr.getLine(),
+                            /*Column=*/0, /*IsArtificial=*/false,
+                            /*CoroSuspendIdx=*/std::nullopt);
 }
 
 llvm::DILexicalBlock *DebugTranslation::translateImpl(DILexicalBlockAttr attr) {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
index e67aa892afe09..8d5d7f9b649f2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
@@ -16,15 +16,11 @@
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/ModuleImport.h"
 
-#include "llvm/ADT/PostOrderIterator.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
-#include "llvm/Support/ModRef.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index ba87d1ad0dd84..70029d7e15a90 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -22,7 +22,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/MatrixBuilder.h"
-#include "llvm/IR/Operator.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp
index 2c0b665ad0d83..ad01a64c6d3da 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Target/LLVMIR/ModuleImport.h"
 
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/IntrinsicsNVPTX.h"
 
 using namespace mlir;
 using namespace mlir::NVVM;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 32b9144233f02..eecca64c4bf81 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -13,7 +13,6 @@
 
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
-#include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
index d9cf85e4aecab..d162afda31b0e 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h"
@@ -23,7 +22,6 @@
 
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
-#include "llvm/Support/FormatVariadic.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3806db3ceab25..336f71ccd06a3 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -370,7 +370,7 @@ static LogicalResult checkImplementationStatus(Operation &op) {
     }
   };
   auto checkReduction = [&todo](auto op, LogicalResult &result) {
-    if (isa<omp::TeamsOp>(op) || isa<omp::SimdOp>(op))
+    if (isa<omp::TeamsOp>(op))
       if (!op.getReductionVars().empty() || op.getReductionByref() ||
           op.getReductionSyms())
         result = todo("reduction");
@@ -1291,6 +1291,11 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
     mapInitializationArgs(op, moduleTranslation, reductionDecls,
                           reductionVariableMap, i);
 
+    // TODO In some cases (specially on the GPU), the init regions may
+    // contains stack alloctaions. If the region is inlined in a loop, this is
+    // problematic. Instead of just inlining the region, handle allocations by
+    // hoisting fixed length allocations to the function entry and using
+    // stacksave and restore for variable length ones.
     if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
                                        "omp.reduction.neutral", builder,
                                        moduleTranslation, &phis)))
@@ -2847,11 +2852,10 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   auto simdOp = cast<omp::SimdOp>(opInst);
 
-  // TODO: Replace this with proper composite translation support.
-  // Currently, simd information on composite constructs is ignored, so e.g.
-  // 'do/for simd' will be treated the same as a standalone 'do/for'. This is
-  // allowed by the spec, since it's equivalent to using a SIMD length of 1.
-  if (simdOp.isComposite()) {
+  // Ignore simd in composite constructs with unsupported clauses
+  // TODO: Replace this once simd + clause combinations are properly supported
+  if (simdOp.isComposite() &&
+      (simdOp.getReductionByref().has_value() || simdOp.getIfExpr())) {
     if (failed(convertIgnoredWrapper(simdOp, moduleTranslation)))
       return failure();
 
@@ -2864,6 +2868,17 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
 
   PrivateVarsInfo privateVarsInfo(simdOp);
 
+  MutableArrayRef<BlockArgument> reductionArgs =
+      cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();
+  DenseMap<Value, llvm::Value *> reductionVariableMap;
+  SmallVector<llvm::Value *> privateReductionVariables(
+      simdOp.getNumReductionVars());
+  SmallVector<DeferredStore> deferredStores;
+  SmallVector<omp::DeclareReductionOp> reductionDecls;
+  collectReductionDecls(simdOp, reductionDecls);
+  llvm::ArrayRef<bool> isByRef = getIsByRef(simdOp.getReductionByref());
+  assert(isByRef.size() == simdOp.getNumReductionVars());
+
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
       findAllocaInsertPoint(builder, moduleTranslation);
 
@@ -2872,11 +2887,28 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
   if (handleError(afterAllocas, opInst).failed())
     return failure();
 
+  if (failed(allocReductionVars(simdOp, reductionArgs, builder,
+                                moduleTranslation, allocaIP, reductionDecls,
+                                privateReductionVariables, reductionVariableMap,
+                                deferredStores, isByRef)))
+    return failure();
+
   if (handleError(initPrivateVars(builder, moduleTranslation, privateVarsInfo),
                   opInst)
           .failed())
     return failure();
 
+  // No call to copyFirstPrivateVars because FIRSTPRIVATE is not allowed for
+  // SIMD.
+
+  assert(afterAllocas.get()->getSinglePredecessor());
+  if (failed(initReductionVars(simdOp, reductionArgs, builder,
+                               moduleTranslation,
+                               afterAllocas.get()->getSinglePredecessor(),
+                               reductionDecls, privateReductionVariables,
+                               reductionVariableMap, isByRef, deferredStores)))
+    return failure();
+
   llvm::ConstantInt *simdlen = nullptr;
   if (std::optional<uint64_t> simdlenVar = simdOp.getSimdlen())
     simdlen = builder.getInt64(simdlenVar.value());
@@ -2921,6 +2953,50 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
                             : nullptr,
                         order, simdlen, safelen);
 
+  // We now need to reduce the per-simd-lane reduction variable into the
+  // original variable. This works a bit differently to other reductions (e.g.
+  // wsloop) because we don't need to call into the OpenMP runtime to handle
+  // threads: everything happened in this one thread.
+  for (auto [i, tuple] : llvm::enumerate(
+           llvm::zip(reductionDecls, isByRef, simdOp.getReductionVars(),
+                     privateReductionVariables))) {
+    auto [decl, byRef, reductionVar, privateReductionVar] = tuple;
+
+    OwningReductionGen gen = makeReductionGen(decl, builder, moduleTranslation);
+    llvm::Value *originalVariable = moduleTranslation.lookupValue(reductionVar);
+    llvm::Type *reductionType = moduleTranslation.convertType(decl.getType());
+
+    // We have one less load for by-ref case because that load is now inside of
+    // the reduction region.
+    llvm::Value *redValue = originalVariable;
+    if (!byRef)
+      redValue =
+          builder.CreateLoad(reductionType, redValue, "red.value." + Twine(i));
+    llvm::Value *privateRedValue = builder.CreateLoad(
+        reductionType, privateReductionVar, "red.private.value." + Twine(i));
+    llvm::Value *reduced;
+
+    auto res = gen(builder.saveIP(), redValue, privateRedValue, reduced);
+    if (failed(handleError(res, opInst)))
+      return failure();
+    builder.restoreIP(res.get());
+
+    // For by-ref case, the store is inside of the reduction region.
+    if (!byRef)
+      builder.CreateStore(reduced, originalVariable);
+  }
+
+  // After the construct, deallocate private reduction variables.
+  SmallVector<Region *> reductionRegions;
+  llvm::transform(reductionDecls, std::back_inserter(reductionRegions),
+                  [](omp::DeclareReductionOp reductionDecl) {
+                    return &reductionDecl.getCleanupRegion();
+                  });
+  if (failed(inlineOmpRegionCleanup(reductionRegions, privateReductionVariables,
+                                    moduleTranslation, builder,
+                                    "omp.reduction.cleanup")))
+    return failure();
+
   return cleanupPrivateVars(builder, moduleTranslation, simdOp.getLoc(),
                             privateVarsInfo.llvmVars,
                             privateVarsInfo.privatizers);
@@ -3459,11 +3535,13 @@ static llvm::Value *
 getRefPtrIfDeclareTarget(mlir::Value value,
                          LLVM::ModuleTranslation &moduleTranslation) {
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
+  Operation *op = value.getDefiningOp();
+  if (auto addrCast = llvm::dyn_cast_if_present<LLVM::AddrSpaceCastOp>(op))
+    op = addrCast->getOperand(0).getDefiningOp();
 
   // An easier way to do this may just be to keep track of any pointer
   // references and their mapping to their respective operation
-  if (auto addressOfOp =
-          llvm::dyn_cast_if_present<LLVM::AddressOfOp>(value.getDefiningOp())) {
+  if (auto addressOfOp = llvm::dyn_cast_if_present<LLVM::AddressOfOp>(op)) {
     if (auto gOp = llvm::dyn_cast_or_null<LLVM::GlobalOp>(
             addressOfOp->getParentOfType<mlir::ModuleOp>().lookupSymbol(
                 addressOfOp.getGlobalName()))) {
@@ -5324,9 +5402,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5420,6 +5516,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp
index 638edca5efde8..3cacbc4b36461 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp
@@ -13,8 +13,6 @@
 
 #include "mlir/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
-#include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/Operation.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp
index b78b002d32292..4bf787faef8c2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp
@@ -19,7 +19,6 @@
 
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsRISCV.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index e353d11c0d0ad..4e7f1d3185129 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Target/LLVMIR/LLVMImportInterface.h"
-#include "mlir/Target/LLVMIR/Import.h"
 #include "mlir/Target/LLVMIR/ModuleImport.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 803ecb39f4daf..bfda223fe0f5f 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -30,7 +30,6 @@
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/IR/Constants.h"
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 8dcffc66f7f7b..8908703cc1368 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -22,21 +22,16 @@
 #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
 #include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h"
 #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
-#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
-#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/AttrTypeSubElements.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
-#include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 
-#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Analysis/TargetFolder.h"
@@ -47,7 +42,6 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
diff --git a/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp b/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
index af78dcd16f792..e4ba478f1d3b5 100644
--- a/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
+++ b/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/MLIRContext.h"
 
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp b/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp
index 47235c4a5a52e..e6d8e34ea7372 100644
--- a/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp
+++ b/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp
@@ -21,7 +21,6 @@
 #include "mlir/Tools/mlir-translate/Translation.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index b30da773d4896..55d6a380d0bff 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -164,6 +164,7 @@ LogicalResult spirv::Deserializer::processInstruction(
   case spirv::Opcode::OpTypeRuntimeArray:
   case spirv::Opcode::OpTypeStruct:
   case spirv::Opcode::OpTypePointer:
+  case spirv::Opcode::OpTypeTensorARM:
   case spirv::Opcode::OpTypeCooperativeMatrixKHR:
     return processType(opcode, operands);
   case spirv::Opcode::OpTypeForwardPointer:
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index b9d9a9015eb61..b1abd8b3dffe9 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -935,6 +935,8 @@ LogicalResult spirv::Deserializer::processType(spirv::Opcode opcode,
     return processStructType(operands);
   case spirv::Opcode::OpTypeMatrix:
     return processMatrixType(operands);
+  case spirv::Opcode::OpTypeTensorARM:
+    return processTensorARMType(operands);
   default:
     return emitError(unknownLoc, "unhandled type instruction");
   }
@@ -1238,6 +1240,55 @@ spirv::Deserializer::processMatrixType(ArrayRef<uint32_t> operands) {
   return success();
 }
 
+LogicalResult
+spirv::Deserializer::processTensorARMType(ArrayRef<uint32_t> operands) {
+  unsigned size = operands.size();
+  if (size < 2 || size > 4)
+    return emitError(unknownLoc, "OpTypeTensorARM must have 2-4 operands "
+                                 "(result_id, element_type, (rank), (shape)) ")
+           << size;
+
+  Type elementTy = getType(operands[1]);
+  if (!elementTy)
+    return emitError(unknownLoc,
+                     "OpTypeTensorARM references undefined element type ")
+           << operands[1];
+
+  if (size == 2) {
+    typeMap[operands[0]] = TensorArmType::get({}, elementTy);
+    return success();
+  }
+
+  IntegerAttr rankAttr = getConstantInt(operands[2]);
+  if (!rankAttr)
+    return emitError(unknownLoc, "OpTypeTensorARM rank must come from a "
+                                 "scalar integer constant instruction");
+  unsigned rank = rankAttr.getValue().getZExtValue();
+  if (size == 3) {
+    SmallVector<int64_t, 4> shape(rank, ShapedType::kDynamic);
+    typeMap[operands[0]] = TensorArmType::get(shape, elementTy);
+    return success();
+  }
+
+  std::optional<std::pair<Attribute, Type>> shapeInfo =
+      getConstant(operands[3]);
+  if (!shapeInfo)
+    return emitError(unknownLoc, "OpTypeTensorARM shape must come from a "
+                                 "constant instruction of type OpTypeArray");
+
+  ArrayAttr shapeArrayAttr = llvm::dyn_cast<ArrayAttr>(shapeInfo->first);
+  SmallVector<int64_t, 1> shape;
+  for (auto dimAttr : shapeArrayAttr.getValue()) {
+    auto dimIntAttr = llvm::dyn_cast<IntegerAttr>(dimAttr);
+    if (!dimIntAttr)
+      return emitError(unknownLoc, "OpTypeTensorARM shape has an invalid "
+                                   "dimension size");
+    shape.push_back(dimIntAttr.getValue().getSExtValue());
+  }
+  typeMap[operands[0]] = TensorArmType::get(shape, elementTy);
+  return success();
+}
+
 LogicalResult
 spirv::Deserializer::processTypeForwardPointer(ArrayRef<uint32_t> operands) {
   if (operands.size() != 2)
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
index e4556e7652b17..1bc9e4a3c75d8 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
@@ -291,6 +291,8 @@ class Deserializer {
 
   LogicalResult processMatrixType(ArrayRef<uint32_t> operands);
 
+  LogicalResult processTensorARMType(ArrayRef<uint32_t> operands);
+
   LogicalResult processTypeForwardPointer(ArrayRef<uint32_t> operands);
 
   //===--------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
index 0ec468d4c1665..cbd1d4255e99a 100644
--- a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
+++ b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Target/SPIRV/SPIRVBinaryUtils.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_VERSION_MAJOR
 #include "llvm/Support/Debug.h"
 
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp b/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
index 80d1bb85611db..f8527908b83e0 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
@@ -15,7 +15,6 @@
 
 #include "mlir/Target/SPIRV/Serialization.h"
 
-#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "spirv-serialization"
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index d258bfd852961..ebebd2d283afa 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -729,6 +729,54 @@ LogicalResult Serializer::prepareBasicType(
     return success();
   }
 
+  if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(type)) {
+    uint32_t elementTypeID = 0;
+    uint32_t rank = 0;
+    uint32_t shapeID = 0;
+    uint32_t rankID = 0;
+    if (failed(processTypeImpl(loc, tensorArmType.getElementType(),
+                               elementTypeID, serializationCtx))) {
+      return failure();
+    }
+    if (tensorArmType.hasRank()) {
+      ArrayRef<int64_t> dims = tensorArmType.getShape();
+      rank = dims.size();
+      rankID = prepareConstantInt(loc, mlirBuilder.getI32IntegerAttr(rank));
+      if (rankID == 0) {
+        return failure();
+      }
+
+      bool shaped = llvm::all_of(dims, [](const auto &dim) { return dim > 0; });
+      if (rank > 0 && shaped) {
+        auto I32Type = IntegerType::get(type.getContext(), 32);
+        auto shapeType = ArrayType::get(I32Type, rank);
+        if (rank == 1) {
+          SmallVector<uint64_t, 1> index(rank);
+          shapeID = prepareDenseElementsConstant(
+              loc, shapeType,
+              mlirBuilder.getI32TensorAttr(SmallVector<int32_t>(dims)), 0,
+              index);
+        } else {
+          shapeID = prepareArrayConstant(
+              loc, shapeType,
+              mlirBuilder.getI32ArrayAttr(SmallVector<int32_t>(dims)));
+        }
+        if (shapeID == 0) {
+          return failure();
+        }
+      }
+    }
+    typeEnum = spirv::Opcode::OpTypeTensorARM;
+    operands.push_back(elementTypeID);
+    if (rankID == 0)
+      return success();
+    operands.push_back(rankID);
+    if (shapeID == 0)
+      return success();
+    operands.push_back(shapeID);
+    return success();
+  }
+
   // TODO: Handle other types.
   return emitError(loc, "unhandled type in serialization: ") << type;
 }
diff --git a/mlir/lib/Target/SPIRV/TranslateRegistration.cpp b/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
index 682fff2784775..ac338d555e320 100644
--- a/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
+++ b/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
@@ -14,8 +14,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/Parser/Parser.h"
 #include "mlir/Support/FileUtilities.h"
@@ -24,9 +22,7 @@
 #include "mlir/Tools/mlir-translate/Translation.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/ToolOutputFile.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
index 02c3ea1ce9b65..da29fab979641 100644
--- a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
@@ -14,7 +14,6 @@
 
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 
 namespace mlir {
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 0a02a01977cac..608bdcb948176 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -33,11 +33,9 @@
 
 #include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
 #include "mlir/Analysis/DataFlow/LivenessAnalysis.h"
-#include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Transforms/StripDebugInfo.cpp b/mlir/lib/Transforms/StripDebugInfo.cpp
index 87f5f76d3d432..de6cc03c6b54c 100644
--- a/mlir/lib/Transforms/StripDebugInfo.cpp
+++ b/mlir/lib/Transforms/StripDebugInfo.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Transforms/Passes.h"
 
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Pass/Pass.h"
 
diff --git a/mlir/lib/Transforms/Utils/CFGToSCF.cpp b/mlir/lib/Transforms/Utils/CFGToSCF.cpp
index 7c1781044d2a2..dbde75e17b3b8 100644
--- a/mlir/lib/Transforms/Utils/CFGToSCF.cpp
+++ b/mlir/lib/Transforms/Utils/CFGToSCF.cpp
@@ -115,14 +115,12 @@
 
 #include "mlir/Transforms/CFGToSCF.h"
 
-#include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 955a106c21941..d118fe422f2f2 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -16,8 +16,6 @@
 #include "mlir/IR/Iterators.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Rewrite/PatternApplicator.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -2713,8 +2711,7 @@ legalizeUnresolvedMaterialization(RewriterBase &rewriter,
 }
 
 LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
-  if (ops.empty())
-    return success();
+  assert(!ops.empty() && "expected at least one operation");
   const ConversionTarget &target = opLegalizer.getTarget();
 
   // Compute the set of operations and blocks to convert.
@@ -3417,6 +3414,38 @@ void mlir::registerConversionPDLFunctions(RewritePatternSet &patterns) {
 // Op Conversion Entry Points
 //===----------------------------------------------------------------------===//
 
+/// This is the type of Action that is dispatched when a conversion is applied.
+class ApplyConversionAction
+    : public tracing::ActionImpl<ApplyConversionAction> {
+public:
+  using Base = tracing::ActionImpl<ApplyConversionAction>;
+  ApplyConversionAction(ArrayRef<IRUnit> irUnits) : Base(irUnits) {}
+  static constexpr StringLiteral tag = "apply-conversion";
+  static constexpr StringLiteral desc =
+      "Encapsulate the application of a dialect conversion";
+
+  void print(raw_ostream &os) const override { os << tag; }
+};
+
+static LogicalResult applyConversion(ArrayRef<Operation *> ops,
+                                     const ConversionTarget &target,
+                                     const FrozenRewritePatternSet &patterns,
+                                     ConversionConfig config,
+                                     OpConversionMode mode) {
+  if (ops.empty())
+    return success();
+  MLIRContext *ctx = ops.front()->getContext();
+  LogicalResult status = success();
+  SmallVector<IRUnit> irUnits(ops.begin(), ops.end());
+  ctx->executeAction<ApplyConversionAction>(
+      [&] {
+        OperationConverter opConverter(target, patterns, config, mode);
+        status = opConverter.convertOperations(ops);
+      },
+      irUnits);
+  return status;
+}
+
 //===----------------------------------------------------------------------===//
 // Partial Conversion
 //===----------------------------------------------------------------------===//
@@ -3424,9 +3453,8 @@ void mlir::registerConversionPDLFunctions(RewritePatternSet &patterns) {
 LogicalResult mlir::applyPartialConversion(
     ArrayRef<Operation *> ops, const ConversionTarget &target,
     const FrozenRewritePatternSet &patterns, ConversionConfig config) {
-  OperationConverter opConverter(target, patterns, config,
-                                 OpConversionMode::Partial);
-  return opConverter.convertOperations(ops);
+  return applyConversion(ops, target, patterns, config,
+                         OpConversionMode::Partial);
 }
 LogicalResult
 mlir::applyPartialConversion(Operation *op, const ConversionTarget &target,
@@ -3443,9 +3471,7 @@ LogicalResult mlir::applyFullConversion(ArrayRef<Operation *> ops,
                                         const ConversionTarget &target,
                                         const FrozenRewritePatternSet &patterns,
                                         ConversionConfig config) {
-  OperationConverter opConverter(target, patterns, config,
-                                 OpConversionMode::Full);
-  return opConverter.convertOperations(ops);
+  return applyConversion(ops, target, patterns, config, OpConversionMode::Full);
 }
 LogicalResult mlir::applyFullConversion(Operation *op,
                                         const ConversionTarget &target,
@@ -3512,9 +3538,8 @@ LogicalResult mlir::applyAnalysisConversion(
   // Convert the cloned operations. The original IR will remain unchanged.
   SmallVector<Operation *> opsToConvert = llvm::map_to_vector(
       ops, [&](Operation *op) { return mapping.lookup(op); });
-  OperationConverter opConverter(target, patterns, config,
-                                 OpConversionMode::Analysis);
-  LogicalResult status = opConverter.convertOperations(opsToConvert);
+  LogicalResult status = applyConversion(opsToConvert, target, patterns, config,
+                                         OpConversionMode::Analysis);
 
   // Remap `legalizableOps`, so that they point to the original ops and not the
   // cloned ops.
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 5a719200e0026..b82d850413946 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -22,8 +22,6 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/mlir/lib/Transforms/Utils/Inliner.cpp b/mlir/lib/Transforms/Utils/Inliner.cpp
index ae34c4ad49e24..b639e87f52744 100644
--- a/mlir/lib/Transforms/Utils/Inliner.cpp
+++ b/mlir/lib/Transforms/Utils/Inliner.cpp
@@ -17,12 +17,10 @@
 #include "mlir/IR/Threading.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Support/DebugStringHelper.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "inlining"
@@ -541,7 +539,7 @@ Inliner::Impl::optimizeSCCAsync(MutableArrayRef<CallGraphNode *> nodesToVisit,
 
   // An atomic failure variable for the async executors.
   std::vector<std::atomic<bool>> activePMs(pipelines.size());
-  std::fill(activePMs.begin(), activePMs.end(), false);
+  llvm::fill(activePMs, false);
   return failableParallelForEach(ctx, nodesToVisit, [&](CallGraphNode *node) {
     // Find a pass manager for this operation.
     auto it = llvm::find_if(activePMs, [](std::atomic<bool> &isActive) {
diff --git a/mlir/lib/Transforms/Utils/InliningUtils.cpp b/mlir/lib/Transforms/Utils/InliningUtils.cpp
index 8594136570a76..eeb40529cc2fe 100644
--- a/mlir/lib/Transforms/Utils/InliningUtils.cpp
+++ b/mlir/lib/Transforms/Utils/InliningUtils.cpp
@@ -16,7 +16,6 @@
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/CallInterfaces.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index 70a97f26b01f6..353f8a58b3ace 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -11,12 +11,10 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/IR/Block.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
@@ -25,7 +23,6 @@
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 
 #include <deque>
 #include <iterator>
diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp
index 28fe2a7501c06..08cac1fe3695c 100644
--- a/mlir/lib/Transforms/ViewOpGraph.cpp
+++ b/mlir/lib/Transforms/ViewOpGraph.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Transforms/ViewOpGraph.h"
 
-#include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/IR/Block.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Operation.h"
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index b2daabb2a5957..7a0c95ebb8200 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -180,6 +180,15 @@ ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   DIALECT_NAME transform
   EXTENSION_NAME transform_debug_extension)
 
+declare_mlir_dialect_extension_python_bindings(
+ADD_TO_PARENT MLIRPythonSources.Dialects
+ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
+  TD_FILE dialects/TransformTuneExtensionOps.td
+  SOURCES
+    dialects/transform/tune.py
+  DIALECT_NAME transform
+  EXTENSION_NAME transform_tune_extension)
+
 declare_mlir_dialect_python_bindings(
   ADD_TO_PARENT MLIRPythonSources.Dialects
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
diff --git a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
index 70bca3c75d842..ed476da28d6be 100644
--- a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
@@ -2497,6 +2497,11 @@ class ShapedType(Type):
         Returns whether the given dimension size indicates a dynamic dimension.
         """
     @staticmethod
+    def is_static_size(dim_size: int) -> bool:
+        """
+        Returns whether the given dimension size indicates a static dimension.
+        """
+    @staticmethod
     def isinstance(other: Type) -> bool: ...
     def __init__(self, cast_from_type: Type) -> None: ...
     def get_dim_size(self, dim: int) -> int:
@@ -2507,10 +2512,18 @@ class ShapedType(Type):
         """
         Returns whether the dim-th dimension of the given shaped type is dynamic.
         """
+    def is_static_dim(self, dim: int) -> bool:
+        """
+        Returns whether the dim-th dimension of the given shaped type is static.
+        """
     def is_dynamic_stride_or_offset(self, dim_size: int) -> bool:
         """
         Returns whether the given value is used as a placeholder for dynamic strides and offsets in shaped types.
         """
+    def is_static_stride_or_offset(self, dim_size: int) -> bool:
+        """
+        Returns whether the given shaped type stride or offset value is statically-sized.
+        """
     @property
     def element_type(self) -> Type:
         """
diff --git a/mlir/python/mlir/dialects/TransformTuneExtensionOps.td b/mlir/python/mlir/dialects/TransformTuneExtensionOps.td
new file mode 100644
index 0000000000000..ff3047592ab12
--- /dev/null
+++ b/mlir/python/mlir/dialects/TransformTuneExtensionOps.td
@@ -0,0 +1,19 @@
+//===-- TransformTuneExtensionOps.td - Binding entry point -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Entry point of the generated Python bindings for the Tune extension of the
+// Transform dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PYTHON_BINDINGS_TRANSFORM_DEBUG_EXTENSION_OPS
+#define PYTHON_BINDINGS_TRANSFORM_DEBUG_EXTENSION_OPS
+
+include "mlir/Dialect/Transform/TuneExtension/TuneExtensionOps.td"
+
+#endif // PYTHON_BINDINGS_TRANSFORM_DEBUG_EXTENSION_OPS
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index 48e724d80c926..1b359da40a291 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -21,21 +21,6 @@ def copy(
     O[None] = cast(U, I[None])
 
 
-@linalg_structured_op
-def elemwise_unary(
-    I=TensorDef(T1),
-    O=TensorDef(U, output=True),
-    fun=UnaryFnAttrDef(default=UnaryFn.exp),
-    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
-):
-    """Applies the unary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-    """
-    O[None] = fun(cast(U, I[None]))
-
-
 @linalg_structured_op
 def exp(
     I=TensorDef(T1),
@@ -192,22 +177,6 @@ def erf(
     O[None] = UnaryFn.erf(I[None])
 
 
-@linalg_structured_op
-def elemwise_binary(
-    lhs=TensorDef(T1),
-    rhs=TensorDef(T2),
-    O=TensorDef(U, output=True),
-    fun=BinaryFnAttrDef(default=BinaryFn.add),
-    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
-):
-    """Applies the binary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-    """
-    O[None] = fun(cast(U, lhs[None]), cast(U, rhs[None]))
-
-
 @linalg_structured_op
 def add(
     lhs=TensorDef(T1),
diff --git a/mlir/python/mlir/dialects/transform/tune.py b/mlir/python/mlir/dialects/transform/tune.py
new file mode 100644
index 0000000000000..f63f88a382422
--- /dev/null
+++ b/mlir/python/mlir/dialects/transform/tune.py
@@ -0,0 +1,82 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from typing import Optional, Sequence
+
+from ...ir import (
+    Type,
+    Attribute,
+    ArrayAttr,
+    StringAttr,
+    F64Type,
+    IntegerType,
+    IntegerAttr,
+    FloatAttr,
+    BoolAttr,
+)
+from .._transform_tune_extension_ops_gen import *
+from .._transform_tune_extension_ops_gen import _Dialect
+
+try:
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class KnobOp(KnobOp):
+    def __init__(
+        self,
+        result: Type,  # !transform.any_param or !transform.param<Type>
+        name: Union[StringAttr, str],
+        options: Union[
+            ArrayAttr, Sequence[Union[Attribute, bool, int, float, str]], Attribute
+        ],
+        *,
+        selected: Optional[Attribute] = None,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(name, str):
+            name = StringAttr.get(name)
+
+        def map_to_attr(value):
+            if isinstance(value, bool):
+                return BoolAttr.get(value)
+            if isinstance(value, int):
+                return IntegerAttr.get(IntegerType.get_signless(64), value)
+            if isinstance(value, float):
+                return FloatAttr.get(F64Type.get(), value)
+            if isinstance(value, str):
+                return StringAttr.get(value)
+            assert isinstance(value, Attribute)
+            return value
+
+        if isinstance(options, Sequence) and not isinstance(options, ArrayAttr):
+            options = ArrayAttr.get([map_to_attr(opt) for opt in options])
+
+        super().__init__(
+            result,
+            name,
+            options,
+            selected=selected and map_to_attr(selected),
+            loc=loc,
+            ip=ip,
+        )
+
+
+def knob(
+    result: Type,  # !transform.any_param or !transform.param<Type>
+    name: Union[StringAttr, str],
+    options: Union[
+        ArrayAttr, Sequence[Union[Attribute, bool, int, float, str]], Attribute
+    ],
+    *,
+    selected: Optional[Attribute] = None,
+    loc=None,
+    ip=None,
+):
+    return KnobOp(result, name, options, selected=selected, loc=loc, ip=ip)
diff --git a/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir b/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir
new file mode 100644
index 0000000000000..095f3e575eca8
--- /dev/null
+++ b/mlir/test/Conversion/ArithToAMDGPU/scaling-extf.mlir
@@ -0,0 +1,262 @@
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx950" | FileCheck %s
+
+// CHECK-LABEL: @conversion_f8_f32_fallback
+// CHECK:         %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
+// CHECK-NEXT:    %[[SCALE_EXT:.+]] = arith.extf %arg1 : vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_00:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_00:.+]] = vector.shape_cast %[[IN_SLICE_00]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_00:.+]] = vector.extract %[[SCALE_EXT]][0, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_00:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_00]][0], %[[SCALE_SCALAR_00]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_00:.+]] = vector.extract_strided_slice %[[PACKED_00]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_00:.+]] = vector.shape_cast %[[OUT_SLICE_00]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_00]], %[[CST]] {offsets = [0, 0], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_01:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_01:.+]] = vector.shape_cast %[[IN_SLICE_01]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_01:.+]] = vector.extract %[[SCALE_EXT]][0, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_01:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_01]][0], %[[SCALE_SCALAR_01]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_01:.+]] = vector.extract_strided_slice %[[PACKED_01]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_01:.+]] = vector.shape_cast %[[OUT_SLICE_01]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_01]], %[[ACC_A]] {offsets = [0, 1], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_10:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_10:.+]] = vector.shape_cast %[[IN_SLICE_10]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_10:.+]] = vector.extract %[[SCALE_EXT]][1, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_10:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_10]][0], %[[SCALE_SCALAR_10]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_10:.+]] = vector.extract_strided_slice %[[PACKED_10]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_10:.+]] = vector.shape_cast %[[OUT_SLICE_10]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_10]], %[[ACC_B]] {offsets = [1, 0], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_11:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_11:.+]] = vector.shape_cast %[[IN_SLICE_11]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_11:.+]] = vector.extract %[[SCALE_EXT]][1, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_11:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_11]][0], %[[SCALE_SCALAR_11]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_11:.+]] = vector.extract_strided_slice %[[PACKED_11]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_11:.+]] = vector.shape_cast %[[OUT_SLICE_11]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_11]], %[[ACC_A]] {offsets = [1, 1], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    return %[[ACC_B]] : vector<2x2xf32>
+func.func @conversion_f8_f32_fallback(%in: vector<2x2xf8E5M2>, %scale: vector<2x2xf8E8M0FNU>) -> vector<2x2xf32> {
+    %ext = arith.scaling_extf %in, %scale : vector<2x2xf8E5M2>, vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+    return %ext : vector<2x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_f4_f32_fallback
+// CHECK:         %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
+// CHECK-NEXT:    %[[SCALE_EXT:.+]] = arith.extf %arg1 : vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_00:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_00:.+]] = vector.shape_cast %[[IN_SLICE_00]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_00:.+]] = vector.extract %[[SCALE_EXT]][0, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_00:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_00]][0], %[[SCALE_SCALAR_00]] : vector<1xf4E2M1FN> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_00:.+]] = vector.extract_strided_slice %[[PACKED_00]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_00:.+]] = vector.shape_cast %[[OUT_SLICE_00]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_00]], %[[CST]] {offsets = [0, 0], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_01:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_01:.+]] = vector.shape_cast %[[IN_SLICE_01]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_01:.+]] = vector.extract %[[SCALE_EXT]][0, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_01:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_01]][0], %[[SCALE_SCALAR_01]] : vector<1xf4E2M1FN> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_01:.+]] = vector.extract_strided_slice %[[PACKED_01]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_01:.+]] = vector.shape_cast %[[OUT_SLICE_01]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_01]], %[[ACC_A]] {offsets = [0, 1], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_10:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_10:.+]] = vector.shape_cast %[[IN_SLICE_10]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_10:.+]] = vector.extract %[[SCALE_EXT]][1, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_10:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_10]][0], %[[SCALE_SCALAR_10]] : vector<1xf4E2M1FN> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_10:.+]] = vector.extract_strided_slice %[[PACKED_10]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_10:.+]] = vector.shape_cast %[[OUT_SLICE_10]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_10]], %[[ACC_B]] {offsets = [1, 0], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_11:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_11:.+]] = vector.shape_cast %[[IN_SLICE_11]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_11:.+]] = vector.extract %[[SCALE_EXT]][1, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_11:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_11]][0], %[[SCALE_SCALAR_11]] : vector<1xf4E2M1FN> to vector<2xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_11:.+]] = vector.extract_strided_slice %[[PACKED_11]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_VEC_11:.+]] = vector.shape_cast %[[OUT_SLICE_11]] : vector<1xf32> to vector<1x1xf32>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_11]], %[[ACC_A]] {offsets = [1, 1], strides = [1, 1]} : vector<1x1xf32> into vector<2x2xf32>
+// CHECK-NEXT:    return %[[ACC_B]] : vector<2x2xf32>
+func.func @conversion_f4_f32_fallback(%in: vector<2x2xf4E2M1FN>, %scale: vector<2x2xf8E8M0FNU>) -> vector<2x2xf32> {
+    %ext = arith.scaling_extf %in, %scale : vector<2x2xf4E2M1FN>, vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+    return %ext : vector<2x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_f8_f16_fallback
+// CHECK:         %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf16>
+// CHECK-NEXT:    %[[SCALE_EXT:.+]] = arith.extf %arg1 : vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_00:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_00:.+]] = vector.shape_cast %[[IN_SLICE_00]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_00:.+]] = vector.extract %[[SCALE_EXT]][0, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_00:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_00]][0], %[[SCALE_SCALAR_00]] : vector<1xf8E5M2> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_00:.+]] = vector.extract_strided_slice %[[PACKED_00]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_00:.+]] = vector.shape_cast %[[OUT_SLICE_00]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_00]], %[[CST]] {offsets = [0, 0], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    %[[IN_SLICE_01:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_01:.+]] = vector.shape_cast %[[IN_SLICE_01]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_01:.+]] = vector.extract %[[SCALE_EXT]][0, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_01:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_01]][0], %[[SCALE_SCALAR_01]] : vector<1xf8E5M2> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_01:.+]] = vector.extract_strided_slice %[[PACKED_01]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_01:.+]] = vector.shape_cast %[[OUT_SLICE_01]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_01]], %[[ACC_A]] {offsets = [0, 1], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    %[[IN_SLICE_10:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_10:.+]] = vector.shape_cast %[[IN_SLICE_10]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_10:.+]] = vector.extract %[[SCALE_EXT]][1, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_10:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_10]][0], %[[SCALE_SCALAR_10]] : vector<1xf8E5M2> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_10:.+]] = vector.extract_strided_slice %[[PACKED_10]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_10:.+]] = vector.shape_cast %[[OUT_SLICE_10]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_10]], %[[ACC_B]] {offsets = [1, 0], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    %[[IN_SLICE_11:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf8E5M2> to vector<1x1xf8E5M2>
+// CHECK-NEXT:    %[[IN_VEC_11:.+]] = vector.shape_cast %[[IN_SLICE_11]] : vector<1x1xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_11:.+]] = vector.extract %[[SCALE_EXT]][1, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_11:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_11]][0], %[[SCALE_SCALAR_11]] : vector<1xf8E5M2> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_11:.+]] = vector.extract_strided_slice %[[PACKED_11]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_11:.+]] = vector.shape_cast %[[OUT_SLICE_11]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_11]], %[[ACC_A]] {offsets = [1, 1], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    return %[[ACC_B]] : vector<2x2xf16>
+func.func @conversion_f8_f16_fallback(%in: vector<2x2xf8E5M2>, %scale: vector<2x2xf8E8M0FNU>) -> vector<2x2xf16> {
+    %ext = arith.scaling_extf %in, %scale : vector<2x2xf8E5M2>, vector<2x2xf8E8M0FNU> to vector<2x2xf16>
+    return %ext : vector<2x2xf16>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_f4_f16_fallback
+// CHECK:         %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf16>
+// CHECK-NEXT:    %[[SCALE_EXT:.+]] = arith.extf %arg1 : vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+// CHECK-NEXT:    %[[IN_SLICE_00:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_00:.+]] = vector.shape_cast %[[IN_SLICE_00]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_00:.+]] = vector.extract %[[SCALE_EXT]][0, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_00:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_00]][0], %[[SCALE_SCALAR_00]] : vector<1xf4E2M1FN> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_00:.+]] = vector.extract_strided_slice %[[PACKED_00]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_00:.+]] = vector.shape_cast %[[OUT_SLICE_00]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_00]], %[[CST]] {offsets = [0, 0], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    %[[IN_SLICE_01:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_01:.+]] = vector.shape_cast %[[IN_SLICE_01]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_01:.+]] = vector.extract %[[SCALE_EXT]][0, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_01:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_01]][0], %[[SCALE_SCALAR_01]] : vector<1xf4E2M1FN> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_01:.+]] = vector.extract_strided_slice %[[PACKED_01]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_01:.+]] = vector.shape_cast %[[OUT_SLICE_01]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_01]], %[[ACC_A]] {offsets = [0, 1], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    %[[IN_SLICE_10:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 0], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_10:.+]] = vector.shape_cast %[[IN_SLICE_10]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_10:.+]] = vector.extract %[[SCALE_EXT]][1, 0] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_10:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_10]][0], %[[SCALE_SCALAR_10]] : vector<1xf4E2M1FN> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_10:.+]] = vector.extract_strided_slice %[[PACKED_10]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_10:.+]] = vector.shape_cast %[[OUT_SLICE_10]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_VEC_10]], %[[ACC_B]] {offsets = [1, 0], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    %[[IN_SLICE_11:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 1], strides = [1, 1]} : vector<2x2xf4E2M1FN> to vector<1x1xf4E2M1FN>
+// CHECK-NEXT:    %[[IN_VEC_11:.+]] = vector.shape_cast %[[IN_SLICE_11]] : vector<1x1xf4E2M1FN> to vector<1xf4E2M1FN>
+// CHECK-NEXT:    %[[SCALE_SCALAR_11:.+]] = vector.extract %[[SCALE_EXT]][1, 1] : f32 from vector<2x2xf32>
+// CHECK-NEXT:    %[[PACKED_11:.+]] = amdgpu.scaled_ext_packed %[[IN_VEC_11]][0], %[[SCALE_SCALAR_11]] : vector<1xf4E2M1FN> to vector<2xf16>
+// CHECK-NEXT:    %[[OUT_SLICE_11:.+]] = vector.extract_strided_slice %[[PACKED_11]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf16> to vector<1xf16>
+// CHECK-NEXT:    %[[OUT_VEC_11:.+]] = vector.shape_cast %[[OUT_SLICE_11]] : vector<1xf16> to vector<1x1xf16>
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_VEC_11]], %[[ACC_A]] {offsets = [1, 1], strides = [1, 1]} : vector<1x1xf16> into vector<2x2xf16>
+// CHECK-NEXT:    return %[[ACC_B]] : vector<2x2xf16>
+func.func @conversion_f4_f16_fallback(%in: vector<2x2xf4E2M1FN>, %scale: vector<2x2xf8E8M0FNU>) -> vector<2x2xf16> {
+    %ext = arith.scaling_extf %in, %scale : vector<2x2xf4E2M1FN>, vector<2x2xf8E8M0FNU> to vector<2x2xf16>
+    return %ext : vector<2x2xf16>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_broadcast
+// CHECK-DAG:     %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<8x2x4xf32>
+// CHECK-DAG:     %[[BCAST:.+]] = vector.broadcast %arg1
+// CHECK-DAG:     %[[IN_CAST:.+]] = vector.shape_cast %arg0
+// CHECK-DAG:     %[[SCALE_CAST:.+]] = vector.shape_cast %[[BCAST]]
+// CHECK-DAG:     %[[SCALE_EXT:.+]] = arith.extf %[[SCALE_CAST]]
+// CHECK-DAG:     vector.extract_strided_slice %[[IN_CAST]] {offsets = [0, 0, 0], sizes = [1, 1, 4], strides = [1, 1, 1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.extract %[[SCALE_EXT]][0, 0, 0]
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.scaled_ext_packed
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [0], strides = [1]}
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [2], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.scaled_ext_packed
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [2], strides = [1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}} {offsets = [0, 0, 0], strides = [1, 1, 1]}
+// CHECK-NEXT:    vector.extract_strided_slice %[[IN_CAST]] {offsets = [0, 1, 0], sizes = [1, 1, 4], strides = [1, 1, 1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.extract %[[SCALE_EXT]][0, 1, 0]
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.scaled_ext_packed
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [0], strides = [1]}
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [2], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.scaled_ext_packed
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [2], strides = [1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [0, 1, 0], strides = [1, 1, 1]} 
+func.func @conversion_broadcast(%in: vector<8x8xf8E5M2>, %scale: vector<8x2xf8E8M0FNU>) -> vector<8x8xf32> {
+    %bc = vector.broadcast %scale : vector<8x2xf8E8M0FNU> to vector<4x8x2xf8E8M0FNU>
+    %cast1 = vector.shape_cast %in : vector<8x8xf8E5M2> to vector<8x2x4xf8E5M2>
+    %cast2 = vector.shape_cast %bc : vector<4x8x2xf8E8M0FNU> to vector<8x2x4xf8E8M0FNU>
+    %ext = arith.scaling_extf %cast1, %cast2 : vector<8x2x4xf8E5M2>, vector<8x2x4xf8E8M0FNU> to vector<8x2x4xf32>
+    %cast3 = vector.shape_cast %ext : vector<8x2x4xf32> to vector<8x8xf32>
+    return %cast3 : vector<8x8xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_broadcast_odd
+// CHECK-NEXT:    %[[CST_PARTIAL:.+]] = arith.constant dense<0.000000e+00> : vector<3xf32>
+// CHECK-NEXT:    %[[CST_FINAL:.+]] = arith.constant dense<0.000000e+00> : vector<6xf32>
+// CHECK-NEXT:    %[[SCALE_BC:.+]] = vector.broadcast %arg1 : vector<2xf8E8M0FNU> to vector<3x2xf8E8M0FNU>
+// CHECK-NEXT:    %[[SCALE_FLAT:.+]] = vector.shape_cast %[[SCALE_BC]] : vector<3x2xf8E8M0FNU> to vector<6xf8E8M0FNU>
+// CHECK-NEXT:    %[[SCALE_EXT:.+]] = arith.extf %[[SCALE_FLAT]] : vector<6xf8E8M0FNU> to vector<6xf32>
+// CHECK-NEXT:    %[[IN_SLICE_0:.+]] = vector.extract_strided_slice %arg0 {offsets = [0], sizes = [3], strides = [1]} : vector<6xf8E5M2> to vector<3xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_0:.+]] = vector.extract %[[SCALE_EXT]][0] : f32 from vector<6xf32>
+// CHECK-NEXT:    %[[IN_CHUNK_0A:.+]] = vector.extract_strided_slice %[[IN_SLICE_0]] {offsets = [0], sizes = [2], strides = [1]} : vector<3xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[PACKED_0A:.+]] = amdgpu.scaled_ext_packed %[[IN_CHUNK_0A]][0], %[[SCALE_SCALAR_0]] : vector<2xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[PARTIAL_ACC_0:.+]] = vector.insert_strided_slice %[[PACKED_0A]], %[[CST_PARTIAL]] {offsets = [0], strides = [1]} : vector<2xf32> into vector<3xf32>
+// CHECK-NEXT:    %[[IN_CHUNK_0B:.+]] = vector.extract_strided_slice %[[IN_SLICE_0]] {offsets = [2], sizes = [1], strides = [1]} : vector<3xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[PACKED_0B_RAW:.+]] = amdgpu.scaled_ext_packed %[[IN_CHUNK_0B]][0], %[[SCALE_SCALAR_0]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[PACKED_0B:.+]] = vector.extract_strided_slice %[[PACKED_0B_RAW]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_0:.+]] = vector.insert_strided_slice %[[PACKED_0B]], %[[PARTIAL_ACC_0]] {offsets = [2], strides = [1]} : vector<1xf32> into vector<3xf32>
+// CHECK-NEXT:    %[[FINAL_ACC_A:.+]] = vector.insert_strided_slice %[[OUT_SLICE_0]], %[[CST_FINAL]] {offsets = [0], strides = [1]} : vector<3xf32> into vector<6xf32>
+// CHECK-NEXT:    %[[IN_SLICE_1:.+]] = vector.extract_strided_slice %arg0 {offsets = [3], sizes = [3], strides = [1]} : vector<6xf8E5M2> to vector<3xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_SCALAR_1:.+]] = vector.extract %[[SCALE_EXT]][3] : f32 from vector<6xf32>
+// CHECK-NEXT:    %[[IN_CHUNK_1A:.+]] = vector.extract_strided_slice %[[IN_SLICE_1]] {offsets = [0], sizes = [2], strides = [1]} : vector<3xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[PACKED_1A:.+]] = amdgpu.scaled_ext_packed %[[IN_CHUNK_1A]][0], %[[SCALE_SCALAR_1]] : vector<2xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[PARTIAL_ACC_1:.+]] = vector.insert_strided_slice %[[PACKED_1A]], %[[CST_PARTIAL]] {offsets = [0], strides = [1]} : vector<2xf32> into vector<3xf32>
+// CHECK-NEXT:    %[[IN_CHUNK_1B:.+]] = vector.extract_strided_slice %[[IN_SLICE_1]] {offsets = [2], sizes = [1], strides = [1]} : vector<3xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[PACKED_1B_RAW:.+]] = amdgpu.scaled_ext_packed %[[IN_CHUNK_1B]][0], %[[SCALE_SCALAR_1]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[PACKED_1B:.+]] = vector.extract_strided_slice %[[PACKED_1B_RAW]] {offsets = [0], sizes = [1], strides = [1]} : vector<2xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[OUT_SLICE_1:.+]] = vector.insert_strided_slice %[[PACKED_1B]], %[[PARTIAL_ACC_1]] {offsets = [2], strides = [1]} : vector<1xf32> into vector<3xf32>
+// CHECK-NEXT:    %[[RESULT:.+]] = vector.insert_strided_slice %[[OUT_SLICE_1]], %[[FINAL_ACC_A]] {offsets = [3], strides = [1]} : vector<3xf32> into vector<6xf32>
+// CHECK-NEXT:    return %[[RESULT]] : vector<6xf32>
+func.func @conversion_broadcast_odd(%in: vector<6xf8E5M2>, %scale: vector<2xf8E8M0FNU>) -> vector<6xf32> {
+    %bc = vector.broadcast %scale : vector<2xf8E8M0FNU> to vector<3x2xf8E8M0FNU>
+    %cast = vector.shape_cast %bc : vector<3x2xf8E8M0FNU> to vector<6xf8E8M0FNU>
+    %ext = arith.scaling_extf %in, %cast : vector<6xf8E5M2>, vector<6xf8E8M0FNU> to vector<6xf32>
+    return %ext : vector<6xf32>
+}
+
+// -----
+// CHECK-LABEL: @conversion_splat
+// CHECK-DAG:     %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<4xf32>
+// CHECK-DAG:     %[[SCALE_SPLAT:.+]] = vector.splat %arg1 : vector<4xf8E8M0FNU>
+// CHECK-DAG:     %[[SCALE_EXTF:.+]] = arith.extf %[[SCALE_SPLAT]] : vector<4xf8E8M0FNU> to vector<4xf32>
+// CHECK-DAG:     %[[SCALE_SCALAR:.+]] = vector.extract %[[SCALE_EXTF]][0] : f32 from vector<4xf32>
+// CHECK:         %[[IN_CHUNK0:.+]] = vector.extract_strided_slice %arg0 {offsets = [0], sizes = [2], strides = [1]} : vector<4xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK0:.+]] = amdgpu.scaled_ext_packed %[[IN_CHUNK0]][0], %[[SCALE_SCALAR]] : vector<2xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[ACCUM_A:.+]] = vector.insert_strided_slice %[[OUT_CHUNK0]], %[[CST]] {offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
+// CHECK-NEXT:    %[[IN_CHUNK1:.+]] = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK1:.+]] = amdgpu.scaled_ext_packed %[[IN_CHUNK1]][0], %[[SCALE_SCALAR]] : vector<2xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[FINAL_RESULT:.+]] = vector.insert_strided_slice %[[OUT_CHUNK1]], %[[ACCUM_A]] {offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
+// CHECK-NEXT:    return %[[FINAL_RESULT]] : vector<4xf32>
+func.func @conversion_splat(%in: vector<4xf8E5M2>, %scale: f8E8M0FNU) -> vector<4xf32> {
+    %splat = vector.splat %scale : vector<4xf8E8M0FNU>
+    %ext = arith.scaling_extf %in, %splat : vector<4xf8E5M2>, vector<4xf8E8M0FNU> to vector<4xf32>
+    return %ext : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_scalar
+// CHECK:         %[[SCALE_F32:.+]] = arith.extf %arg1 : f8E8M0FNU to f32
+// CHECK-NEXT:    %[[SPLAT_IN:.+]] = vector.splat %arg0 : vector<1xf8E5M2>
+// CHECK-NEXT:    %[[PACKED_EXT:.+]] = amdgpu.scaled_ext_packed %[[SPLAT_IN]][0], %[[SCALE_F32]] : vector<1xf8E5M2> to vector<2xf32>
+// CHECK-NEXT:    %[[RESULT:.+]] = vector.extract %[[PACKED_EXT]][0] : f32 from vector<2xf32>
+// CHECK-NEXT:    return %[[RESULT]] : f32
+func.func @conversion_scalar(%in: f8E5M2, %scale: f8E8M0FNU) -> f32 {
+    %ext = arith.scaling_extf %in, %scale : f8E5M2, f8E8M0FNU to f32
+    return %ext : f32
+}
diff --git a/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir b/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir
new file mode 100644
index 0000000000000..0519050c5ecc4
--- /dev/null
+++ b/mlir/test/Conversion/ArithToAMDGPU/scaling-truncf.mlir
@@ -0,0 +1,193 @@
+// RUN: mlir-opt --split-input-file %s -convert-arith-to-amdgpu="chipset=gfx950" | FileCheck %s
+
+// CHECK-LABEL: @conversion_f8_fallback
+// CHECK-DAG:     %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf8E5M2>
+// CHECK-DAG:     %[[SCALE_EXT:.+]] = arith.extf %arg1 : vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+// CHECK:         %[[IN_SLICE_00:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_00:.+]] = vector.shape_cast %[[IN_SLICE_00]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_00:.+]] = vector.extract %[[SCALE_EXT]][0, 0]
+// CHECK-NEXT:    %[[PACKED_00:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_00]] into undef[0], %[[SCALE_SCALAR_00]]
+// CHECK-NEXT:    %[[OUT_SLICE_00:.+]] = vector.extract_strided_slice %[[PACKED_00]]
+// CHECK-NEXT:    %[[OUT_SCALAR_00:.+]] = vector.shape_cast %[[OUT_SLICE_00]]
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_00]], %[[CST]]
+// CHECK-NEXT:    %[[IN_SLICE_01:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 1], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_01:.+]] = vector.shape_cast %[[IN_SLICE_01]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_01:.+]] = vector.extract %[[SCALE_EXT]][0, 1]
+// CHECK-NEXT:    %[[PACKED_01:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_01]] into undef[0], %[[SCALE_SCALAR_01]]
+// CHECK-NEXT:    %[[OUT_SLICE_01:.+]] = vector.extract_strided_slice %[[PACKED_01]]
+// CHECK-NEXT:    %[[OUT_SCALAR_01:.+]] = vector.shape_cast %[[OUT_SLICE_01]]
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_01]], %[[ACC_A]]
+// CHECK-NEXT:    %[[IN_SLICE_10:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 0], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_10:.+]] = vector.shape_cast %[[IN_SLICE_10]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_10:.+]] = vector.extract %[[SCALE_EXT]][1, 0]
+// CHECK-NEXT:    %[[PACKED_10:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_10]] into undef[0], %[[SCALE_SCALAR_10]]
+// CHECK-NEXT:    %[[OUT_SLICE_10:.+]] = vector.extract_strided_slice %[[PACKED_10]]
+// CHECK-NEXT:    %[[OUT_SCALAR_10:.+]] = vector.shape_cast %[[OUT_SLICE_10]]
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_10]], %[[ACC_B]]
+// CHECK-NEXT:    %[[IN_SLICE_11:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_11:.+]] = vector.shape_cast %[[IN_SLICE_11]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_11:.+]] = vector.extract %[[SCALE_EXT]][1, 1]
+// CHECK-NEXT:    %[[PACKED_11:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_11]] into undef[0], %[[SCALE_SCALAR_11]]
+// CHECK-NEXT:    %[[OUT_SLICE_11:.+]] = vector.extract_strided_slice %[[PACKED_11]]
+// CHECK-NEXT:    %[[OUT_SCALAR_11:.+]] = vector.shape_cast %[[OUT_SLICE_11]]
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_11]], %[[ACC_A]]
+// CHECK-NEXT:    return %[[ACC_B]] : vector<2x2xf8E5M2>
+func.func @conversion_f8_fallback(%in: vector<2x2xf32>, %scale: vector<2x2xf8E8M0FNU>) -> vector<2x2xf8E5M2> {
+    %ext = arith.scaling_truncf %in, %scale : vector<2x2xf32>, vector<2x2xf8E8M0FNU> to vector<2x2xf8E5M2>
+    return %ext : vector<2x2xf8E5M2>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_f4_fallback
+// CHECK-DAG:     %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<2x2xf4E2M1FN>
+// CHECK-DAG:     %[[SCALE_EXT:.+]] = arith.extf %arg1 : vector<2x2xf8E8M0FNU> to vector<2x2xf32>
+// CHECK:         %[[IN_SLICE_00:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_00:.+]] = vector.shape_cast %[[IN_SLICE_00]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_00:.+]] = vector.extract %[[SCALE_EXT]][0, 0]
+// CHECK-NEXT:    %[[PACKED_00:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_00]] into undef[0], %[[SCALE_SCALAR_00]]
+// CHECK-NEXT:    %[[OUT_SLICE_00:.+]] = vector.extract_strided_slice %[[PACKED_00]]
+// CHECK-NEXT:    %[[OUT_SCALAR_00:.+]] = vector.shape_cast %[[OUT_SLICE_00]]
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_00]], %[[CST]]
+// CHECK-NEXT:    %[[IN_SLICE_01:.+]] = vector.extract_strided_slice %arg0 {offsets = [0, 1], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_01:.+]] = vector.shape_cast %[[IN_SLICE_01]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_01:.+]] = vector.extract %[[SCALE_EXT]][0, 1]
+// CHECK-NEXT:    %[[PACKED_01:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_01]] into undef[0], %[[SCALE_SCALAR_01]]
+// CHECK-NEXT:    %[[OUT_SLICE_01:.+]] = vector.extract_strided_slice %[[PACKED_01]]
+// CHECK-NEXT:    %[[OUT_SCALAR_01:.+]] = vector.shape_cast %[[OUT_SLICE_01]]
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_01]], %[[ACC_A]]
+// CHECK-NEXT:    %[[IN_SLICE_10:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 0], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_10:.+]] = vector.shape_cast %[[IN_SLICE_10]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_10:.+]] = vector.extract %[[SCALE_EXT]][1, 0]
+// CHECK-NEXT:    %[[PACKED_10:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_10]] into undef[0], %[[SCALE_SCALAR_10]]
+// CHECK-NEXT:    %[[OUT_SLICE_10:.+]] = vector.extract_strided_slice %[[PACKED_10]]
+// CHECK-NEXT:    %[[OUT_SCALAR_10:.+]] = vector.shape_cast %[[OUT_SLICE_10]]
+// CHECK-NEXT:    %[[ACC_A:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_10]], %[[ACC_B]]
+// CHECK-NEXT:    %[[IN_SLICE_11:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 1], strides = [1, 1]}
+// CHECK-NEXT:    %[[IN_SCALAR_11:.+]] = vector.shape_cast %[[IN_SLICE_11]]
+// CHECK-NEXT:    %[[SCALE_SCALAR_11:.+]] = vector.extract %[[SCALE_EXT]][1, 1]
+// CHECK-NEXT:    %[[PACKED_11:.+]] = amdgpu.packed_scaled_trunc %[[IN_SCALAR_11]] into undef[0], %[[SCALE_SCALAR_11]]
+// CHECK-NEXT:    %[[OUT_SLICE_11:.+]] = vector.extract_strided_slice %[[PACKED_11]]
+// CHECK-NEXT:    %[[OUT_SCALAR_11:.+]] = vector.shape_cast %[[OUT_SLICE_11]]
+// CHECK-NEXT:    %[[ACC_B:.+]] = vector.insert_strided_slice %[[OUT_SCALAR_11]], %[[ACC_A]]
+// CHECK-NEXT:    return %[[ACC_B]] : vector<2x2xf4E2M1FN>
+func.func @conversion_f4_fallback(%in: vector<2x2xf32>, %scale: vector<2x2xf8E8M0FNU>) -> vector<2x2xf4E2M1FN> {
+    %ext = arith.scaling_truncf %in, %scale : vector<2x2xf32>, vector<2x2xf8E8M0FNU> to vector<2x2xf4E2M1FN>
+    return %ext : vector<2x2xf4E2M1FN>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_broadcast
+// CHECK-DAG:     %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<8x2x4xf8E5M2>
+// CHECK-DAG:     %[[BCAST:.+]] = vector.broadcast %arg1
+// CHECK-DAG:     %[[IN_CAST:.+]] = vector.shape_cast %arg0
+// CHECK-DAG:     %[[SCALE_CAST:.+]] = vector.shape_cast %[[BCAST]]
+// CHECK-DAG:     %[[SCALE_EXT:.+]] = arith.extf %[[SCALE_CAST]]
+// CHECK-DAG:     vector.extract_strided_slice %[[IN_CAST]] {offsets = [0, 0, 0], sizes = [1, 1, 4], strides = [1, 1, 1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.extract %[[SCALE_EXT]][0, 0, 0]
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.packed_scaled_trunc
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [0], strides = [1]}
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [2], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.packed_scaled_trunc
+// CHECK-NEXT:    vector.extract_strided_slice
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [2], strides = [1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}} {offsets = [0, 0, 0], strides = [1, 1, 1]}
+// CHECK-NEXT:    vector.extract_strided_slice %[[IN_CAST]] {offsets = [0, 1, 0], sizes = [1, 1, 4], strides = [1, 1, 1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.extract %[[SCALE_EXT]][0, 1, 0]
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.packed_scaled_trunc
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [0], strides = [1]}
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [2], sizes = [2], strides = [1]}
+// CHECK-NEXT:    amdgpu.packed_scaled_trunc
+// CHECK-NEXT:    vector.extract_strided_slice %{{.+}} {offsets = [0], sizes = [2], strides = [1]}
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [2], strides = [1]}
+// CHECK-NEXT:    vector.shape_cast
+// CHECK-NEXT:    vector.insert_strided_slice %{{.+}}, %{{.+}} {offsets = [0, 1, 0], strides = [1, 1, 1]} 
+func.func @conversion_broadcast(%in: vector<8x8xf32>, %scale: vector<8x2xf8E8M0FNU>) -> vector<8x8xf8E5M2> {
+    %bc = vector.broadcast %scale : vector<8x2xf8E8M0FNU> to vector<4x8x2xf8E8M0FNU>
+    %cast1 = vector.shape_cast %in : vector<8x8xf32> to vector<8x2x4xf32>
+    %cast2 = vector.shape_cast %bc : vector<4x8x2xf8E8M0FNU> to vector<8x2x4xf8E8M0FNU>
+    %ext = arith.scaling_truncf %cast1, %cast2 : vector<8x2x4xf32>, vector<8x2x4xf8E8M0FNU> to vector<8x2x4xf8E5M2>
+    %cast3 = vector.shape_cast %ext : vector<8x2x4xf8E5M2> to vector<8x8xf8E5M2>
+    return %cast3 : vector<8x8xf8E5M2>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_broadcast_odd
+// CHECK-NEXT:    %[[CST3:.+]] = arith.constant dense<0.000000e+00> : vector<3xf8E5M2>
+// CHECK-NEXT:    %[[CST6:.+]] = arith.constant dense<0.000000e+00> : vector<6xf8E5M2>
+// CHECK-NEXT:    %[[SCALE_BCAST:.+]] = vector.broadcast %arg1 : vector<2xf8E8M0FNU> to vector<3x2xf8E8M0FNU>
+// CHECK-NEXT:    %[[SCALE_FLAT:.+]] = vector.shape_cast %[[SCALE_BCAST]] : vector<3x2xf8E8M0FNU> to vector<6xf8E8M0FNU>
+// CHECK-NEXT:    %[[SCALE_EXTF:.+]] = arith.extf %[[SCALE_FLAT]] : vector<6xf8E8M0FNU> to vector<6xf32>
+// CHECK-NEXT:    %[[IN_CHUNK0:.+]] = vector.extract_strided_slice %arg0 {offsets = [0], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
+// CHECK-NEXT:    %[[SCALE0:.+]] = vector.extract %[[SCALE_EXTF]][0] : f32 from vector<6xf32>
+// CHECK-NEXT:    %[[IN_CHUNK0_PART0:.+]] = vector.extract_strided_slice %[[IN_CHUNK0]] {offsets = [0], sizes = [2], strides = [1]} : vector<3xf32> to vector<2xf32>
+// CHECK-NEXT:    %[[PACKED0_PART0:.+]] = amdgpu.packed_scaled_trunc %[[IN_CHUNK0_PART0]] into undef[0], %[[SCALE0]] : vector<2xf32> to vector<4xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK0_PART0:.+]] = vector.extract_strided_slice %[[PACKED0_PART0]] {offsets = [0], sizes = [2], strides = [1]} : vector<4xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[ACCUM0_PART0:.+]] = vector.insert_strided_slice %[[OUT_CHUNK0_PART0]], %[[CST3]] {offsets = [0], strides = [1]} : vector<2xf8E5M2> into vector<3xf8E5M2>
+// CHECK-NEXT:    %[[IN_CHUNK0_PART1:.+]] = vector.extract_strided_slice %[[IN_CHUNK0]] {offsets = [2], sizes = [1], strides = [1]} : vector<3xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[PACKED0_PART1:.+]] = amdgpu.packed_scaled_trunc %[[IN_CHUNK0_PART1]] into undef[0], %[[SCALE0]] : vector<1xf32> to vector<4xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK0_PART1:.+]] = vector.extract_strided_slice %[[PACKED0_PART1]] {offsets = [0], sizes = [1], strides = [1]} : vector<4xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[CHUNK0_RES:.+]] = vector.insert_strided_slice %[[OUT_CHUNK0_PART1]], %[[ACCUM0_PART0]] {offsets = [2], strides = [1]} : vector<1xf8E5M2> into vector<3xf8E5M2>
+// CHECK-NEXT:    %[[FINAL_ACCUM_A:.+]] = vector.insert_strided_slice %[[CHUNK0_RES]], %[[CST6]] {offsets = [0], strides = [1]} : vector<3xf8E5M2> into vector<6xf8E5M2>
+// CHECK-NEXT:    %[[IN_CHUNK1:.+]] = vector.extract_strided_slice %arg0 {offsets = [3], sizes = [3], strides = [1]} : vector<6xf32> to vector<3xf32>
+// CHECK-NEXT:    %[[SCALE1:.+]] = vector.extract %[[SCALE_EXTF]][3] : f32 from vector<6xf32>
+// CHECK-NEXT:    %[[IN_CHUNK1_PART0:.+]] = vector.extract_strided_slice %[[IN_CHUNK1]] {offsets = [0], sizes = [2], strides = [1]} : vector<3xf32> to vector<2xf32>
+// CHECK-NEXT:    %[[PACKED1_PART0:.+]] = amdgpu.packed_scaled_trunc %[[IN_CHUNK1_PART0]] into undef[0], %[[SCALE1]] : vector<2xf32> to vector<4xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK1_PART0:.+]] = vector.extract_strided_slice %[[PACKED1_PART0]] {offsets = [0], sizes = [2], strides = [1]} : vector<4xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[ACCUM1_PART0:.+]] = vector.insert_strided_slice %[[OUT_CHUNK1_PART0]], %[[CST3]] {offsets = [0], strides = [1]} : vector<2xf8E5M2> into vector<3xf8E5M2>
+// CHECK-NEXT:    %[[IN_CHUNK1_PART1:.+]] = vector.extract_strided_slice %[[IN_CHUNK1]] {offsets = [2], sizes = [1], strides = [1]} : vector<3xf32> to vector<1xf32>
+// CHECK-NEXT:    %[[PACKED1_PART1:.+]] = amdgpu.packed_scaled_trunc %[[IN_CHUNK1_PART1]] into undef[0], %[[SCALE1]] : vector<1xf32> to vector<4xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK1_PART1:.+]] = vector.extract_strided_slice %[[PACKED1_PART1]] {offsets = [0], sizes = [1], strides = [1]} : vector<4xf8E5M2> to vector<1xf8E5M2>
+// CHECK-NEXT:    %[[CHUNK1_RES:.+]] = vector.insert_strided_slice %[[OUT_CHUNK1_PART1]], %[[ACCUM1_PART0]] {offsets = [2], strides = [1]} : vector<1xf8E5M2> into vector<3xf8E5M2>
+// CHECK-NEXT:    %[[FINAL_RESULT:.+]] = vector.insert_strided_slice %[[CHUNK1_RES]], %[[FINAL_ACCUM_A]] {offsets = [3], strides = [1]} : vector<3xf8E5M2> into vector<6xf8E5M2>
+// CHECK-NEXT:    return %[[FINAL_RESULT]] : vector<6xf8E5M2>
+func.func @conversion_broadcast_odd(%in: vector<6xf32>, %scale: vector<2xf8E8M0FNU>) -> vector<6xf8E5M2> {
+    %bc = vector.broadcast %scale : vector<2xf8E8M0FNU> to vector<3x2xf8E8M0FNU>
+    %cast = vector.shape_cast %bc : vector<3x2xf8E8M0FNU> to vector<6xf8E8M0FNU>
+    %ext = arith.scaling_truncf %in, %cast : vector<6xf32>, vector<6xf8E8M0FNU> to vector<6xf8E5M2>
+    return %ext : vector<6xf8E5M2>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_splat
+// CHECK-DAG:     %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<4xf8E5M2>
+// CHECK-DAG:     %[[SCALE_SPLAT:.+]] = vector.splat %arg1 : vector<4xf8E8M0FNU>
+// CHECK-DAG:     %[[SCALE_EXTF:.+]] = arith.extf %[[SCALE_SPLAT]] : vector<4xf8E8M0FNU> to vector<4xf32>
+// CHECK-DAG:     %[[SCALE_SCALAR:.+]] = vector.extract %[[SCALE_EXTF]][0] : f32 from vector<4xf32>
+// CHECK:         %[[IN_CHUNK0:.+]] = vector.extract_strided_slice %arg0 {offsets = [0], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+// CHECK-NEXT:    %[[PACKED0:.+]] = amdgpu.packed_scaled_trunc %[[IN_CHUNK0]] into undef[0], %[[SCALE_SCALAR]] : vector<2xf32> to vector<4xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK0:.+]] = vector.extract_strided_slice %[[PACKED0]] {offsets = [0], sizes = [2], strides = [1]} : vector<4xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[ACCUM_A:.+]] = vector.insert_strided_slice %[[OUT_CHUNK0]], %[[CST]] {offsets = [0], strides = [1]} : vector<2xf8E5M2> into vector<4xf8E5M2>
+// CHECK-NEXT:    %[[IN_CHUNK1:.+]] = vector.extract_strided_slice %arg0 {offsets = [2], sizes = [2], strides = [1]} : vector<4xf32> to vector<2xf32>
+// CHECK-NEXT:    %[[PACKED1:.+]] = amdgpu.packed_scaled_trunc %[[IN_CHUNK1]] into undef[0], %[[SCALE_SCALAR]] : vector<2xf32> to vector<4xf8E5M2>
+// CHECK-NEXT:    %[[OUT_CHUNK1:.+]] = vector.extract_strided_slice %[[PACKED1]] {offsets = [0], sizes = [2], strides = [1]} : vector<4xf8E5M2> to vector<2xf8E5M2>
+// CHECK-NEXT:    %[[FINAL_RESULT:.+]] = vector.insert_strided_slice %[[OUT_CHUNK1]], %[[ACCUM_A]] {offsets = [2], strides = [1]} : vector<2xf8E5M2> into vector<4xf8E5M2>
+// CHECK-NEXT:    return %[[FINAL_RESULT]] : vector<4xf8E5M2>
+func.func @conversion_splat(%in: vector<4xf32>, %scale: f8E8M0FNU) -> vector<4xf8E5M2> {
+    %splat = vector.splat %scale : vector<4xf8E8M0FNU>
+    %ext = arith.scaling_truncf %in, %splat : vector<4xf32>, vector<4xf8E8M0FNU> to vector<4xf8E5M2>
+    return %ext : vector<4xf8E5M2>
+}
+
+// -----
+
+// CHECK-LABEL: @conversion_scalar
+// CHECK:         %[[SCALE_F32:.+]] = arith.extf %arg1 : f8E8M0FNU to f32
+// CHECK-NEXT:    %[[SPLAT_IN:.+]] = vector.splat %arg0 : vector<1xf32>
+// CHECK-NEXT:    %[[PACKED_TRUNC:.+]] = amdgpu.packed_scaled_trunc %[[SPLAT_IN]] into undef[0], %[[SCALE_F32]]
+// CHECK-NEXT:    %[[RESULT:.+]] = vector.extract %[[PACKED_TRUNC]][0]
+// CHECK-NEXT:    return %[[RESULT]] : f8E5M2
+func.func @conversion_scalar(%in: f32, %scale: f8E8M0FNU) -> f8E5M2 {
+    %ext = arith.scaling_truncf %in, %scale : f32, f8E8M0FNU to f8E5M2
+    return %ext : f8E5M2
+}
diff --git a/mlir/test/Conversion/GPUToSPIRV/rotate.mlir b/mlir/test/Conversion/GPUToSPIRV/rotate.mlir
new file mode 100644
index 0000000000000..b96dd37219b46
--- /dev/null
+++ b/mlir/test/Conversion/GPUToSPIRV/rotate.mlir
@@ -0,0 +1,102 @@
+// RUN: mlir-opt -split-input-file -convert-gpu-to-spirv -verify-diagnostics %s -o - | FileCheck %s
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @rotate()
+  gpu.func @rotate() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 16 : i32
+    %val = arith.constant 42.0 : f32
+
+    // CHECK: %[[OFFSET:.+]] = spirv.Constant 4 : i32
+    // CHECK: %[[WIDTH:.+]] = spirv.Constant 16 : i32
+    // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
+    // CHECK: %{{.+}} = spirv.GroupNonUniformRotateKHR <Subgroup> %[[VAL]], %[[OFFSET]], cluster_size(%[[WIDTH]]) : f32, i32, i32 -> f32
+    // CHECK: %{{.+}} = spirv.Constant true
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @rotate_width_less_than_subgroup_size()
+  gpu.func @rotate_width_less_than_subgroup_size() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 8 : i32
+    %val = arith.constant 42.0 : f32
+
+    // CHECK: %[[OFFSET:.+]] = spirv.Constant 4 : i32
+    // CHECK: %[[WIDTH:.+]] = spirv.Constant 8 : i32
+    // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
+    // CHECK: %{{.+}} = spirv.GroupNonUniformRotateKHR <Subgroup> %[[VAL]], %[[OFFSET]], cluster_size(%[[WIDTH]]) : f32, i32, i32 -> f32
+    // CHECK: %[[INVOCATION_ID_ADDR:.+]] = spirv.mlir.addressof @__builtin__SubgroupLocalInvocationId__
+    // CHECK: %[[INVOCATION_ID:.+]] = spirv.Load "Input" %[[INVOCATION_ID_ADDR]]
+    // CHECK: %{{.+}} = spirv.ULessThan %[[INVOCATION_ID]], %[[WIDTH]]
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  gpu.func @rotate_with_bigger_than_subgroup_size() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 32 : i32
+    %val = arith.constant 42.0 : f32
+
+    // expected-error @+1 {{failed to legalize operation 'gpu.rotate'}}
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  gpu.func @rotate_non_const_width(%width: i32) kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %val = arith.constant 42.0 : f32
+
+    // expected-error @+1 {{'gpu.rotate' op width is not a constant value}}
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
diff --git a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
index 4701ac5d96009..99ab0e1dc4eef 100644
--- a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
+++ b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
@@ -246,6 +246,41 @@ func.func @extract_dynamic_cst(%arg0 : vector<4xf32>) -> f32 {
 
 // -----
 
+// CHECK-LABEL: func.func @to_elements_one_element 
+// CHECK-SAME:     %[[A:.*]]: vector<1xf32>)
+//      CHECK:   %[[ELEM0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1xf32> to f32
+//      CHECK:   return %[[ELEM0]] : f32
+func.func @to_elements_one_element(%a: vector<1xf32>) -> (f32) {
+  %0:1 = vector.to_elements %a : vector<1xf32>
+  return %0#0 : f32
+}
+
+// CHECK-LABEL: func.func @to_elements_no_dead_elements
+// CHECK-SAME:     %[[A:.*]]: vector<4xf32>)
+//      CHECK:   %[[ELEM0:.*]] = spirv.CompositeExtract %[[A]][0 : i32] : vector<4xf32>
+//      CHECK:   %[[ELEM1:.*]] = spirv.CompositeExtract %[[A]][1 : i32] : vector<4xf32>
+//      CHECK:   %[[ELEM2:.*]] = spirv.CompositeExtract %[[A]][2 : i32] : vector<4xf32>
+//      CHECK:   %[[ELEM3:.*]] = spirv.CompositeExtract %[[A]][3 : i32] : vector<4xf32>
+//      CHECK:   return %[[ELEM0]], %[[ELEM1]], %[[ELEM2]], %[[ELEM3]] : f32, f32, f32, f32
+func.func @to_elements_no_dead_elements(%a: vector<4xf32>) -> (f32, f32, f32, f32) {
+  %0:4 = vector.to_elements %a : vector<4xf32>
+  return %0#0, %0#1, %0#2, %0#3 : f32, f32, f32, f32
+}
+
+// CHECK-LABEL: func.func @to_elements_dead_elements
+// CHECK-SAME:     %[[A:.*]]: vector<4xf32>)
+//  CHECK-NOT:   spirv.CompositeExtract %[[A]][0 : i32]
+//      CHECK:   %[[ELEM1:.*]] = spirv.CompositeExtract %[[A]][1 : i32] : vector<4xf32>
+//  CHECK-NOT:   spirv.CompositeExtract %[[A]][2 : i32]
+//      CHECK:   %[[ELEM3:.*]] = spirv.CompositeExtract %[[A]][3 : i32] : vector<4xf32>
+//      CHECK:   return %[[ELEM1]], %[[ELEM3]] : f32, f32
+func.func @to_elements_dead_elements(%a: vector<4xf32>) -> (f32, f32) {
+  %0:4 = vector.to_elements %a : vector<4xf32>
+  return %0#1, %0#3 : f32, f32
+}
+
+// -----
+
 // CHECK-LABEL: @from_elements_0d
 //  CHECK-SAME: %[[ARG0:.+]]: f32
 //       CHECK:   %[[RETVAL:.+]] = builtin.unrealized_conversion_cast %[[ARG0]]
diff --git a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir b/mlir/test/Dialect/AMDGPU/maskedload-to-load.mlir
similarity index 56%
rename from mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
rename to mlir/test/Dialect/AMDGPU/maskedload-to-load.mlir
index 20999af10553e..febe46bf7a759 100644
--- a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
+++ b/mlir/test/Dialect/AMDGPU/maskedload-to-load.mlir
@@ -1,17 +1,17 @@
-// RUN: mlir-opt %s --amdgpu-transfer-read-to-load --split-input-file | FileCheck %s
+// RUN: mlir-opt %s --amdgpu-maskedload-to-load --split-input-file | FileCheck %s
 
 // CHECK-LABEL: func @transfer_to_maskedload_fatrawbuffer(
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<4xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xf32>
+func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<4xi1>, %passthru : vector<4xf32>) -> vector<4xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi1>, vector<4xf32> into vector<4xf32>
   return %res : vector<4xf32>
 }
 
 // CHECK: %[[IF:.*]] = scf.if
-// CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]]
+// CHECK: vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG1]]]
 
 // CHECK: } else {
 // CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
@@ -25,10 +25,10 @@ func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.ad
 // CHECK-LABEL: func @transfer_to_maskedload_fatrawbuffer_f16(
 // CHECK-SAME: %[[ARG0:.+]]: memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>,
 // CHECK-SAME: %[[ARG1:.+]]: index, %[[ARG2:.+]]: index,
-// CHECK-SAME: %[[ARG3:.+]]: vector<4xi1>)
-func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>) -> vector<4xf16> {
-  %cf0 = arith.constant 0.0 : f16
-  %res = vector.transfer_read %mem[%idx0, %idx1], %cf0, %mask {in_bounds = [true]} : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf16>
+// CHECK-SAME: %[[ARG3:.+]]: vector<4xi1>
+// CHECK-SAME: %[[ARG4:.+]]: vector<4xf16>
+func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>, %passthru : vector<4xf16>) -> vector<4xf16> {
+  %res = vector.maskedload %mem[%idx0, %idx1], %mask, %passthru : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi1>, vector<4xf16> into vector<4xf16>
   return %res : vector<4xf16>
 }
 // CHECK-DAG: %[[C0:.*]] = arith.constant 0
@@ -45,7 +45,7 @@ func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgp
 
 // CHECK: %[[COND:.*]] = arith.andi %[[COND1]], %[[COND2]]
 // CHECK: %[[IF:.*]] = scf.if %[[COND]] -> (vector<4xf16>) {
-// CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG2]]]
+// CHECK: vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG2]]]
 // CHECK: } else {
 // CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG2]]]
 // CHECK: return %[[IF]] : vector<4xf16>
@@ -58,13 +58,11 @@ func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgp
 // CHECK-SAME: %[[ARG0:.*]]: memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>
 // CHECK-SAME: %[[ARG1:.*]]: index, %[[ARG2:.*]]: index
 // CHECK-SAME: %[[ARG3:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>) -> vector<4xi8> {
-  %cf0 = arith.constant 0 : i8
-  %res = vector.transfer_read %mem[%idx0, %idx1], %cf0, %mask {in_bounds = [true]} : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi8>
+// CHECK-SAME: %[[ARG4:.*]]: vector<4xi8>
+func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>, %passthru : vector<4xi8>) -> vector<4xi8> {
+  %res = vector.maskedload %mem[%idx0, %idx1], %mask, %passthru : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi1>, vector<4xi8> into vector<4xi8>
   return %res : vector<4xi8>
 }
-
-// CHECK:     %[[CST:.*]] = arith.constant dense<0> : vector<4xi8>
 // CHECK:     %[[C0:.*]] = arith.constant 0 : index
 // CHECK:     %[[C4:.*]] = arith.constant 4 : index
 // CHECK:     %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:2, %[[STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG0]]
@@ -79,13 +77,12 @@ func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8,
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index, %mask : vector<4xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32>, vector<4xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xf32>
+func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index, %mask : vector<4xi1>, %passthru : vector<4xf32>) -> vector<4xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru : memref<8x8xf32>, vector<4xi1>, vector<4xf32> into vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
-// CHECK: %[[RES:.*]] = vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[CST]], %[[ARG2]]
+// CHECK: %[[RES:.*]] = vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[ARG2]], %[[ARG3]]
 // CHECK: return %[[RES]] : vector<4xf32>
 
 // -----
@@ -94,49 +91,26 @@ func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index,
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #gpu.address_space<workgroup>>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_addrspace(%mem : memref<8x8xf32, #gpu.address_space<workgroup>>, %idx : index, %mask : vector<4xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32, #gpu.address_space<workgroup>>, vector<4xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xf32>
+func.func @transfer_to_maskedload_addrspace(%mem : memref<8x8xf32, #gpu.address_space<workgroup>>, %idx : index, %mask : vector<4xi1>, %passthru : vector<4xf32>) -> vector<4xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru : memref<8x8xf32, #gpu.address_space<workgroup>>, vector<4xi1>, vector<4xf32> into vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
-// CHECK: %[[RES:.*]] = vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[CST]], %[[ARG2]]
+// CHECK: %[[RES:.*]] = vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[ARG2]], %[[ARG3]]
 // CHECK: return %[[RES]] : vector<4xf32>
 
 // -----
 
-// CHECK-LABEL: func @transfer_broadcasting(
-// CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>
-// CHECK-SAME: %[[ARG1:.*]]: index
-// CHECK-SAME: %[[ARG2:.*]]: vector<1xi1>
-#broadcast_1d = affine_map<(d0, d1) -> (0)>
-func.func @transfer_broadcasting(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<1xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask
-    {in_bounds = [true], permutation_map = #broadcast_1d}
-      : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf32>
-  return %res : vector<4xf32>
-}
-// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
-// CHECK: %[[IF:.*]] = scf.if
-// CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
-// CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[SELECT]] : vector<1xf32> to vector<4xf32>
-
-// -----
-
 // CHECK-LABEL: func @transfer_scalar(
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<1xi1>
-func.func @transfer_scalar(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<1xi1>) -> vector<1xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask
-    {in_bounds = [true]}
-      : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<1xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<1xf32>
+func.func @transfer_scalar(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<1xi1>, %passthru : vector<1xf32>) -> vector<1xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru
+      : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<1xi1>, vector<1xf32> into vector<1xf32>
   return %res : vector<1xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
 // CHECK: %[[IF:.*]] = scf.if
 // CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG1]]]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
+// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[ARG3]]
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
index 81b04ccceaf27..72ced5b53879b 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
@@ -21,7 +21,7 @@ func.func @vec1d_1(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK: for {{.*}} step 128
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
-// CHECK-NEXT: %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT: %{{.*}} = ub.poison : f32
 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i0 = 0 to %M { // vectorized due to scalar -> vector
      %a0 = affine.load %A[%c0, %c0] : memref<?x?xf32>
@@ -47,7 +47,7 @@ func.func @vec1d_2(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
    %P = memref.dim %B, %c2 : memref<?x?x?xf32>
 
 // CHECK:for [[IV3:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128
-// CHECK-NEXT: %[[CST:.*]] = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT: %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %[[CST]] : memref<?x?xf32>, vector<128xf32>
    affine.for %i3 = 0 to %M { // vectorized
      %a3 = affine.load %A[%c0, %i3] : memref<?x?xf32>
@@ -76,7 +76,7 @@ func.func @vec1d_3(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK-NEXT:   for [[IV9:%[0-9a-zA-Z_]*]] = 0 to [[ARG_N]] {
 // CHECK-NEXT:   %[[APP9_0:[0-9a-zA-Z_]+]] = affine.apply {{.*}}([[IV9]], [[IV8]])
 // CHECK-NEXT:   %[[APP9_1:[0-9a-zA-Z_]+]] = affine.apply {{.*}}([[IV9]], [[IV8]])
-// CHECK-NEXT:   %[[CST:.*]] = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT:   %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:   {{.*}} = vector.transfer_read %{{.*}}[%[[APP9_0]], %[[APP9_1]]], %[[CST]] : memref<?x?xf32>, vector<128xf32>
    affine.for %i8 = 0 to %M { // vectorized
      affine.for %i9 = 0 to %N {
@@ -280,7 +280,7 @@ func.func @vec_rejected_3(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 
 // CHECK:for [[IV4:%[0-9a-zA-Z_]+]] = 0 to [[ARG_M]] step 128 {
 // CHECK-NEXT:   for [[IV5:%[0-9a-zA-Z_]*]] = 0 to [[ARG_N]] {
-// CHECK-NEXT:     %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT:     %{{.*}} = ub.poison : f32
 // CHECK-NEXT:     {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{[a-zA-Z0-9_]*}} : memref<?x?xf32>, vector<128xf32>
    affine.for %i4 = 0 to %M { // vectorized
      affine.for %i5 = 0 to %N { // not vectorized, would vectorize with --test-fastest-varying=1
@@ -424,7 +424,7 @@ func.func @vec_rejected_8(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK:   for [[IV18:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128
 // CHECK:     %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK:     %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
-// CHECK:     %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK:     %{{.*}} = ub.poison : f32
 // CHECK:     {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %{{.*}} in DFS post-order prevents vectorizing %{{.*}}
      affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
@@ -458,7 +458,7 @@ func.func @vec_rejected_9(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK:   for [[IV18:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128
 // CHECK:      %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
-// CHECK-NEXT: %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT: %{{.*}} = ub.poison : f32
 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %i18 in DFS post-order prevents vectorizing %{{.*}}
      affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
index 15a7133cf0f65..7d4d111c09799 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
@@ -11,7 +11,7 @@ func.func @vec_affine_apply(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24x48xf3
 // CHECK-NEXT:      affine.for %[[ARG4:.*]] = 0 to 48 step 8 {
 // CHECK-NEXT:        %[[S0:.*]] = affine.apply #[[$MAP_ID0]](%[[ARG3]])
 // CHECK-NEXT:        %[[S1:.*]] = affine.apply #[[$MAP_ID1]](%[[ARG4]])
-// CHECK-NEXT:        %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:        %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:        %[[S2:.*]] = vector.transfer_read %[[ARG0]][%[[ARG2]], %[[S0]], %[[S1]]], %[[CST]] : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:        vector.transfer_write %[[S2]], %[[ARG1]][%[[ARG2]], %[[ARG3]], %[[ARG4]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:      }
@@ -42,7 +42,7 @@ func.func @vec_affine_apply_2(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24x48x
 // CHECK-NEXT:   affine.for %[[ARG3:.*]] = 0 to 12 {
 // CHECK-NEXT:     affine.for %[[ARG4:.*]] = 0 to 48 step 8 {
 // CHECK-NEXT:       %[[S0:.*]] = affine.apply #[[$MAP_ID2]](%[[ARG4]])
-// CHECK-NEXT:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:       %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:       %[[S1:.*]] = vector.transfer_read %[[ARG0]][%[[ARG2]], %[[ARG3]], %[[S0]]], %[[CST]] : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:       vector.transfer_write %[[S1]], %[[ARG1]][%[[ARG2]], %[[ARG3]], %[[ARG4]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:     }
@@ -140,7 +140,7 @@ func.func @affine_map_with_expr_2(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24
 // CHECK-NEXT:       %[[S0:.*]] = affine.apply #[[$MAP_ID3]](%[[ARG3]], %[[ARG4]], %[[I0]])
 // CHECK-NEXT:       %[[S1:.*]] = affine.apply #[[$MAP_ID4]](%[[ARG3]], %[[ARG4]], %[[I0]])
 // CHECK-NEXT:       %[[S2:.*]] = affine.apply #[[$MAP_ID5]](%[[ARG3]], %[[ARG4]], %[[I0]])
-// CHECK-NEXT:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:       %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:       %[[S3:.*]] = vector.transfer_read %[[ARG0]][%[[S0]], %[[S1]], %[[S2]]], %[[CST]] {permutation_map = #[[$MAP_ID6]]} : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:       vector.transfer_write %[[S3]], %[[ARG1]][%[[ARG3]], %[[ARG4]], %[[ARG5]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:     }
diff --git a/mlir/test/Dialect/Affine/affine-data-copy.mlir b/mlir/test/Dialect/Affine/affine-data-copy.mlir
index a1f0d952e7c63..a745271eb9ca8 100644
--- a/mlir/test/Dialect/Affine/affine-data-copy.mlir
+++ b/mlir/test/Dialect/Affine/affine-data-copy.mlir
@@ -447,3 +447,51 @@ func.func @memref_def_inside(%arg0: index) {
   // LIMITED-MEM-NEXT:   memref.dealloc %{{.*}} : memref<1xf32>
   return
 }
+
+// Test with uses across multiple blocks.
+
+memref.global "private" constant @__constant_1x2x1xi32_1 : memref<1x2x1xi32> = dense<0> {alignment = 64 : i64}
+
+// CHECK-LABEL: func @multiple_blocks
+func.func @multiple_blocks(%arg0: index) -> memref<1x2x1xi32> {
+  %c1_i32 = arith.constant 1 : i32
+  %c3_i32 = arith.constant 3 : i32
+  %0 = memref.get_global @__constant_1x2x1xi32_1 : memref<1x2x1xi32>
+  %alloc = memref.alloc() {alignment = 64 : i64} : memref<1x2x1xi32>
+  memref.copy %0, %alloc : memref<1x2x1xi32> to memref<1x2x1xi32>
+  cf.br ^bb1(%alloc : memref<1x2x1xi32>)
+^bb1(%1: memref<1x2x1xi32>):  // 2 preds: ^bb0, ^bb2
+// CHECK: ^bb1(%[[MEM:.*]]: memref<1x2x1xi32>):
+  %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<1x2x1xi1>
+  // CHECK: %[[BUF:.*]] = memref.alloc() : memref<1x2x1xi32>
+  affine.for %arg1 = 0 to 1 {
+    affine.for %arg2 = 0 to 2 {
+      affine.for %arg3 = 0 to 1 {
+        // CHECK: affine.load %[[BUF]]
+        %3 = affine.load %1[%arg1, %arg2, %arg3] : memref<1x2x1xi32>
+        %4 = arith.cmpi slt, %3, %c3_i32 : i32
+        affine.store %4, %alloc_0[%arg1, %arg2, %arg3] : memref<1x2x1xi1>
+      }
+    }
+  }
+  // CHECK: memref.dealloc %[[BUF]]
+  %2 = memref.load %alloc_0[%arg0, %arg0, %arg0] : memref<1x2x1xi1>
+  cf.cond_br %2, ^bb2, ^bb3
+^bb2:  // pred: ^bb1
+// CHECK: ^bb2
+  %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<1x2x1xi32>
+  affine.for %arg1 = 0 to 1 {
+    affine.for %arg2 = 0 to 2 {
+      affine.for %arg3 = 0 to 1 {
+        // Ensure that this reference isn't replaced.
+        %3 = affine.load %1[%arg1, %arg2, %arg3] : memref<1x2x1xi32>
+        // CHECK: affine.load %[[MEM]]
+        %4 = arith.addi %3, %c1_i32 : i32
+        affine.store %4, %alloc_1[%arg1, %arg2, %arg3] : memref<1x2x1xi32>
+      }
+    }
+  }
+  cf.br ^bb1(%alloc_1 : memref<1x2x1xi32>)
+^bb3:  // pred: ^bb1
+  return %1 : memref<1x2x1xi32>
+}
diff --git a/mlir/test/Dialect/Affine/decompose-affine-ops.mlir b/mlir/test/Dialect/Affine/decompose-affine-ops.mlir
index 6acdc436fe677..e5db5cd9181d1 100644
--- a/mlir/test/Dialect/Affine/decompose-affine-ops.mlir
+++ b/mlir/test/Dialect/Affine/decompose-affine-ops.mlir
@@ -27,8 +27,7 @@ func.func @simple_test_1(%0: index, %1: index, %2: index, %lb: index, %ub: index
 // CHECK-DAG: #[[$c42:.*]] = affine_map<() -> (42)>
 // CHECK-DAG: #[[$id:.*]] = affine_map<()[s0] -> (s0)>
 // CHECK-DAG: #[[$add:.*]] = affine_map<()[s0, s1] -> (s0 + s1)>
-// CHECK-DAG: #[[$div32div4timesm4:.*]] = affine_map<()[s0] -> (((s0 floordiv 32) floordiv 4) * -4)>
-// CHECK-DAG: #[[$div32:.*]] = affine_map<()[s0] -> (s0 floordiv 32)>
+// CHECK-DAG: #[[$div32mod4:.*]] = affine_map<()[s0] -> ((s0 floordiv 32) mod 4)>
 
 // CHECK-LABEL:  func.func @simple_test_2
 //  CHECK-SAME:  %[[I0:[0-9a-zA-Z]+]]: index,
@@ -45,10 +44,8 @@ func.func @simple_test_2(%0: index, %1: index, %2: index, %lb: index, %ub: index
     // CHECK:   %[[R2:.*]] = affine.apply #[[$add]]()[%[[c42]], %[[R1]]]
     // CHECK:   scf.for %[[j:.*]] =
     scf.for %j = %lb to %ub step %step {
-      // CHECK:     %[[R3:.*]] = affine.apply #[[$div32div4timesm4]]()[%[[j]]]
-      // CHECK:     %[[R4:.*]] = affine.apply #[[$add]]()[%[[R2]], %[[R3]]]
-      // CHECK:     %[[R5:.*]] = affine.apply #[[$div32]]()[%[[j]]]
-      // CHECK:      %[[a:.*]] = affine.apply #[[$add]]()[%[[R4]], %[[R5]]]
+      // CHECK:     %[[R3:.*]] = affine.apply #[[$div32mod4]]()[%[[j]]]
+      // CHECK:     %[[a:.*]] = affine.apply #[[$add]]()[%[[R2]], %[[R3]]]
       %a = affine.apply affine_map<(d0)[s0] -> ((d0 floordiv 32) mod 4 + s0 + 42)>(%j)[%i]
 
       // CHECK:     "some_side_effecting_consumer"(%[[a]]) : (index) -> ()
@@ -67,8 +64,7 @@ func.func @simple_test_2(%0: index, %1: index, %2: index, %lb: index, %ub: index
 // CHECK-DAG: #[[$div4timesm32:.*]] = affine_map<()[s0] -> ((s0 floordiv 4) * -32)>
 // CHECK-DAG: #[[$times8:.*]] = affine_map<()[s0] -> (s0 * 8)>
 // CHECK-DAG: #[[$id:.*]] = affine_map<()[s0] -> (s0)>
-// CHECK-DAG: #[[$div32div4timesm4:.*]] = affine_map<()[s0] -> (((s0 floordiv 32) floordiv 4) * -4)>
-// CHECK-DAG: #[[$div32:.*]] = affine_map<()[s0] -> (s0 floordiv 32)>
+// CHECK-DAG: #[[$div32mod4:.*]] = affine_map<()[s0] -> ((s0 floordiv 32) mod 4)>
 
 // CHECK-LABEL:  func.func @larger_test
 //  CHECK-SAME:  %[[I0:[0-9a-zA-Z]+]]: index,
@@ -126,10 +122,8 @@ func.func @larger_test(%0: index, %1: index, %2: index, %lb: index, %ub: index,
           // CHECK-NEXT: %[[e:.*]] = affine.apply #[[$add]]()[%[[c]], %[[idk]]]
           %e = affine.apply affine_map<()[s0, s1] -> (s0 + s1 * 8 - (s1 floordiv 4) * 32)>()[%k, %0]
 
-          // CHECK-NEXT: %[[R15:.*]] = affine.apply #[[$div32div4timesm4]]()[%[[k]]]
-          // CHECK-NEXT: %[[R16:.*]] = affine.apply #[[$add]]()[%[[idj]], %[[R15]]]
-          // CHECK-NEXT: %[[R17:.*]] = affine.apply #[[$div32]]()[%[[k]]]
-          // CHECK-NEXT: %[[f:.*]] = affine.apply #[[$add]]()[%[[R16]], %[[R17]]]
+          // CHECK-NEXT: %[[R15:.*]] = affine.apply #[[$div32mod4]]()[%[[k]]]
+          // CHECK-NEXT: %[[f:.*]] = affine.apply #[[$add]]()[%[[idj]], %[[R15]]]
           %f = affine.apply affine_map<(d0)[s0] -> ((d0 floordiv 32) mod 4 + s0)>(%k)[%j]
 
           // CHECK-NEXT: %[[g:.*]] = affine.apply #[[$add]]()[%[[b]], %[[idk]]]
diff --git a/mlir/test/Dialect/Affine/loop-fusion-4.mlir b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
index ca8099b9bb51f..b059b5a98405d 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-4.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
@@ -666,3 +666,80 @@ func.func @unrolled(%arg0: memref<2x4xf32>, %arg1: memref<1x2x4xf32>) {
   // PRODUCER-CONSUMER-MAXIMAL:          affine.load %{{.*}}[0, %{{.*}}, %{{.*}}]
   return
 }
+
+// -----
+
+// Exercises fix for crash reported at https://github.com/llvm/llvm-project/issues/139231
+
+#map = affine_map<(d0, d1) -> (d0 + d1)>
+#map1 = affine_map<(d0, d1) -> (d0 * 2 + d1 * 2)>
+module {
+  func.func @zero_candidates() {
+    %cst = arith.constant 2.221140e+03 : f32
+    %cst_0 = arith.constant 2.606200e+03 : f32
+    %cst_1 = arith.constant 3.224000e+03 : f32
+    %cst_2 = arith.constant 0.000000e+00 : f32
+    %alloc = memref.alloc() {alignment = 64 : i64} : memref<3x7x5x6xf32>
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 7 {
+        affine.for %arg2 = 0 to 5 {
+          affine.for %arg3 = 0 to 6 {
+            affine.store %cst_1, %alloc[%arg0, %arg1, %arg2, %arg3] : memref<3x7x5x6xf32>
+          }
+        }
+      }
+    }
+    %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<3x10x7x6xf32>
+    %subview = memref.subview %alloc_3[0, 2, 1, 0] [3, 7, 5, 6] [1, 1, 1, 1] : memref<3x10x7x6xf32> to memref<3x7x5x6xf32, strided<[420, 42, 6, 1], offset: 90>>
+    memref.copy %alloc, %subview : memref<3x7x5x6xf32> to memref<3x7x5x6xf32, strided<[420, 42, 6, 1], offset: 90>>
+    %alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<3x10x3x6x1xf32>
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 10 {
+        affine.for %arg2 = 0 to 3 {
+          affine.for %arg3 = 0 to 6 {
+            affine.for %arg4 = 0 to 1 {
+              affine.store %cst_2, %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+            }
+          }
+        }
+      }
+    }
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 10 {
+        affine.for %arg2 = 0 to 3 {
+          affine.for %arg3 = 0 to 6 {
+            affine.for %arg4 = 0 to 1 {
+              affine.for %arg5 = 0 to 1 {
+                affine.for %arg6 = 0 to 2 {
+                  %0 = affine.apply #map(%arg1, %arg5)
+                  %1 = affine.apply #map1(%arg2, %arg6)
+                  %2 = affine.load %alloc_3[%arg0, %0, %1, %arg3] : memref<3x10x7x6xf32>
+                  %3 = affine.load %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+                  %4 = arith.mulf %2, %cst_0 : f32
+                  %5 = arith.addf %3, %4 : f32
+                  affine.store %5, %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    %alloc_5 = memref.alloc() {alignment = 64 : i64} : memref<3x10x3x6xf32>
+    %expand_shape = memref.expand_shape %alloc_5 [[0], [1], [2], [3, 4]] output_shape [3, 10, 3, 6, 1] : memref<3x10x3x6xf32> into memref<3x10x3x6x1xf32>
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 10 {
+        affine.for %arg2 = 0 to 3 {
+          affine.for %arg3 = 0 to 6 {
+            affine.for %arg4 = 0 to 1 {
+              %0 = affine.load %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+              %1 = arith.addf %0, %cst : f32
+              affine.store %1, %expand_shape[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+            }
+          }
+        }
+      }
+    }
+    return
+  }
+}
diff --git a/mlir/test/Dialect/Affine/simplify-structures.mlir b/mlir/test/Dialect/Affine/simplify-structures.mlir
index 6f2737a982752..653c2cb521637 100644
--- a/mlir/test/Dialect/Affine/simplify-structures.mlir
+++ b/mlir/test/Dialect/Affine/simplify-structures.mlir
@@ -508,7 +508,7 @@ func.func @test_not_trivially_true_or_false_returning_three_results() -> (index,
 // -----
 
 // Test simplification of mod expressions.
-// CHECK-DAG:   #[[$MOD:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s3 + s4 * s1 + (s0 - s1) mod s2)>
+// CHECK-DAG:   #[[$MOD:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s1 * s4 + s3 + (s0 - s1) mod s2)>
 // CHECK-DAG:   #[[$SIMPLIFIED_MOD_RHS:.*]] = affine_map<()[s0, s1, s2, s3] -> (s3 mod (s2 - s0 * s1))>
 // CHECK-DAG:   #[[$MODULO_AND_PRODUCT:.*]] = affine_map<()[s0, s1, s2, s3] -> (s0 * s1 + s3 - (-s0 + s3) mod s2)>
 // CHECK-LABEL: func @semiaffine_simplification_mod
@@ -547,7 +547,7 @@ func.func @semiaffine_simplification_floordiv_and_ceildiv(%arg0: index, %arg1: i
 
 // Test simplification of product expressions.
 // CHECK-DAG:   #[[$PRODUCT:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s3 + s4 + (s0 - s1) * s2)>
-// CHECK-DAG:   #[[$SUM_OF_PRODUCTS:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s2 + s2 * s0 + s3 + s3 * s0 + s3 * s1 + s4 + s4 * s1)>
+// CHECK-DAG:   #[[$SUM_OF_PRODUCTS:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s0 * s2 + s0 * s3 + s1 * s3 + s1 * s4 + s2 + s3 + s4)>
 // CHECK-LABEL: func @semiaffine_simplification_product
 // CHECK-SAME:  (%[[ARG0:.*]]: index, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index, %[[ARG5:.*]]: index)
 func.func @semiaffine_simplification_product(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index) -> (index, index) {
diff --git a/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir b/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir
index 5f923cdafb956..49bd2eddbdedd 100644
--- a/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir
+++ b/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir
@@ -11,8 +11,8 @@
 
 // CHECK-LABEL:       @base_case
 // CHECK-SAME:          %[[I:.+]]: index, %[[J:.+]]: index, %[[M:.+]]:
-// CHECK:               %[[PAD:.+]] = arith.constant 0 : i8
-// CHECK:               %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:           %[[PAD:.+]] = arith.constant 123 : i8
+// CHECK-DAG:           %[[C0:.+]] = arith.constant 0 : index
 // CHECK:               %[[COLLAPSE:.+]] = memref.collapse_shape %[[M]]
 // CHECK-SAME{LITERAL}:   [[0], [1], [2, 3]]
 // CHECK-SAME:            : memref<?x?x?x8xi8> into memref<?x?x?xi8>
@@ -36,8 +36,8 @@ func.func @base_case(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<
 
 // CHECK-LABEL:       @with_3d_vector
 // CHECK-SAME:          %[[I:.+]]: index, %[[J:.+]]: index, %[[M:.+]]:
-// CHECK:               %[[PAD:.+]] = arith.constant 0 : i8
-// CHECK:               %[[COLLAPSED:.+]] = memref.collapse_shape %[[M]]
+// CHECK-DAG:           %[[PAD:.+]] = arith.constant 123 : i8
+// CHECK-DAG:           %[[COLLAPSED:.+]] = memref.collapse_shape %[[M]]
 // CHECK-SAME{LITERAL}:   [[0], [1, 2, 3]]
 // CHECK-SAME:            : memref<?x?x2x8xi8> into memref<?x?xi8>
 // CHECK-NEXT:          %[[T0:.+]] = vector.transfer_read %[[COLLAPSED]][%[[I]], %[[J]]], %[[PAD]] {in_bounds = [true]}
diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
index da3c26ce36ba5..8031732011839 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
@@ -272,10 +272,10 @@ func.func @materialize_in_dest_raw(%f: f32, %f2: f32, %idx: index) -> (tensor<5x
 
 // -----
 
-// CHECK-LABEL: func.func @test_dialect_op(
+// CHECK:       func.func @custom_op(
 // CHECK-SAME:    %[[ARG:.*]]: !test.test_tensor<[32, 64], f64>
 // CHECK-SAME:  ) -> !test.test_tensor<[32, 128], f64> {
-func.func @test_dialect_op(%arg: !test.test_tensor<[32, 64], f64>)
+func.func @custom_op(%arg: !test.test_tensor<[32, 64], f64>)
     -> !test.test_tensor<[32, 128], f64> {
   // CHECK: %[[MEMREF:.*]] = bufferization.to_buffer %[[ARG]]
   // CHECK: %[[DUMMY:.*]] = "test.dummy_memref_op"(%[[MEMREF]])
@@ -288,3 +288,22 @@ func.func @test_dialect_op(%arg: !test.test_tensor<[32, 64], f64>)
   // CHECK: return %[[OUT]]
   return %out : !test.test_tensor<[32, 128], f64>
 }
+
+// -----
+
+// CHECK:       func.func @custom_origin_op()
+// CHECK-SAME:  -> !test.test_tensor<[42], f64> {
+func.func @custom_origin_op() -> !test.test_tensor<[42], f64> {
+  // CHECK: %[[MEMREF:.*]] = "test.create_memref_op"() : ()
+  // CHECK-SAME: -> !test.test_memref<[21], f64>
+  // CHECK: %[[DUMMY:.*]] = "test.dummy_memref_op"(%[[MEMREF]])
+  // CHECK-SAME: : (!test.test_memref<[21], f64>)
+  // CHECK-SAME: -> !test.test_memref<[42], f64>
+  %in = "test.create_tensor_op"() : () -> !test.test_tensor<[21], f64>
+  %out = "test.dummy_tensor_op"(%in) : (!test.test_tensor<[21], f64>)
+    -> !test.test_tensor<[42], f64>
+
+  // CHECK: %[[OUT:.*]] = bufferization.to_tensor %[[DUMMY]]
+  // CHECK: return %[[OUT]]
+  return %out : !test.test_tensor<[42], f64>
+}
diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir
index ce1be7b5618fe..162ff0662e91e 100644
--- a/mlir/test/Dialect/GPU/invalid.mlir
+++ b/mlir/test/Dialect/GPU/invalid.mlir
@@ -478,6 +478,84 @@ func.func @shuffle_unsupported_type_vec(%arg0 : vector<[4]xf32>, %arg1 : i32, %a
 
 // -----
 
+func.func @rotate_mismatching_type(%arg0 : f32) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op failed to verify that all of {value, rotateResult} have same type}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (i32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_type(%arg0 : index) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got 'index'}}
+  %rotate, %valid = gpu.rotate %arg0, %offset, %width : index
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_type_vec(%arg0 : vector<[4]xf32>) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got 'vector<[4]xf32>'}}
+  %rotate, %valid = gpu.rotate %arg0, %offset, %width : vector<[4]xf32>
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_width(%arg0 : f32) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 15 : i32
+  // expected-error@+1 {{op width must be a power of two}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_offset(%arg0 : f32) {
+  %offset = arith.constant 16 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op offset must be in the range [0, 16)}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_offset_minus(%arg0 : f32) {
+  %offset = arith.constant -1 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op offset must be in the range [0, 16)}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_offset_non_constant(%arg0 : f32, %offset : i32) {
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op offset is not a constant value}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_width_non_constant(%arg0 : f32, %width : i32) {
+  %offset = arith.constant 0 : i32
+  // expected-error@+1 {{op width is not a constant value}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
 module {
   gpu.module @gpu_funcs {
     // expected-error @+1 {{custom op 'gpu.func' gpu.func requires named arguments}}
diff --git a/mlir/test/Dialect/GPU/ops.mlir b/mlir/test/Dialect/GPU/ops.mlir
index 9dbe16774f517..2aef80f73feb3 100644
--- a/mlir/test/Dialect/GPU/ops.mlir
+++ b/mlir/test/Dialect/GPU/ops.mlir
@@ -140,6 +140,10 @@ module attributes {gpu.container_module} {
       // CHECK: gpu.shuffle idx %{{.*}}, %{{.*}}, %{{.*}} : f32
       %shfl3, %pred3 = gpu.shuffle idx %arg0, %offset, %width : f32
 
+      // CHECK: gpu.rotate %{{.*}}, %{{.*}}, %{{.*}} : f32
+      %rotate_width = arith.constant 16 : i32
+      %rotate, %pred4 = gpu.rotate %arg0, %offset, %rotate_width : f32
+
       "gpu.barrier"() : () -> ()
 
       "some_op"(%bIdX, %tIdX) : (index, index) -> ()
diff --git a/mlir/test/Dialect/GPU/transform-gpu-failing.mlir b/mlir/test/Dialect/GPU/transform-gpu-failing.mlir
index 8d7a1aa2a55fc..bc052a0230a8e 100644
--- a/mlir/test/Dialect/GPU/transform-gpu-failing.mlir
+++ b/mlir/test/Dialect/GPU/transform-gpu-failing.mlir
@@ -405,6 +405,67 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+func.func @masking_mapping_attribute_requires_linear_mapping(
+    %x: memref<32xf32>, %y: memref<32xf32>, %t: memref<32 x f32>, %alpha : f32, %stream : !gpu.async.token) -> memref<32xf32> {
+  %one = arith.constant 1 : index
+  %c9 = arith.constant 9 : index
+  %c7 = arith.constant 7 : index
+  %name = gpu.launch async[%stream] blocks(%arg3, %arg4, %arg5) in (%arg9 = %one, %arg10 = %one, %arg11 = %one)
+            threads(%arg6, %arg7, %arg8) in (%arg12 = %one, %arg13 = %one, %arg14 = %one)
+  {
+    scf.forall (%i) in (%c7) {
+        %4 = memref.load %x[%i] : memref<32xf32>
+        %5 = memref.load %y[%i] : memref<32xf32>
+        %6 = math.fma %alpha, %4, %5 : f32
+        memref.store %6, %y[%i] : memref<32xf32>
+     }  { mapping = [#gpu.warp<x>, #gpu.mask<0x33>] }
+    gpu.terminator
+  }
+
+  return %y : memref<32xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    // expected-error @below {{device masking is only available in linear mapping mode}}
+    transform.gpu.map_nested_forall_to_threads %funcop block_dims = [1, 1, 1] : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @masking_mapping_attribute_requires_linear_mapping(
+    %x: memref<32xf32>, %y: memref<32xf32>, %t: memref<32 x f32>, %alpha : f32, %stream : !gpu.async.token) -> memref<32xf32> {
+  %one = arith.constant 1 : index
+  %c99 = arith.constant 99 : index
+  %name = gpu.launch async[%stream] blocks(%arg3, %arg4, %arg5) in (%arg9 = %one, %arg10 = %one, %arg11 = %one)
+            threads(%arg6, %arg7, %arg8) in (%arg12 = %one, %arg13 = %one, %arg14 = %one)
+  {
+    scf.forall (%i) in (%c99) {
+        %4 = memref.load %x[%i] : memref<32xf32>
+        %5 = memref.load %y[%i] : memref<32xf32>
+        %6 = math.fma %alpha, %4, %5 : f32
+        memref.store %6, %y[%i] : memref<32xf32>
+     }  { mapping = [#gpu.thread<linear_dim_0>, #gpu.mask<0xff>] }
+    gpu.terminator
+  }
+
+  return %y : memref<32xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %funcop = transform.structured.match ops{["gpu.launch"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    // expected-error @below {{mask representation too short to capture all physical ids: 64}}
+    transform.gpu.map_nested_forall_to_threads %funcop block_dims = [128, 1, 1] : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
 func.func public @not_a_block_mapping_attribute(%arg0: memref<32x32xf32>, %arg1: memref<32x32xf32>, %arg2: memref<32x32xf32>) {
   scf.forall (%arg3, %arg4) in (1, 1) {
     linalg.matmul ins(%arg0, %arg1 : memref<32x32xf32>, memref<32x32xf32>) outs(%arg2 : memref<32x32xf32>)
diff --git a/mlir/test/Dialect/GPU/transform-gpu.mlir b/mlir/test/Dialect/GPU/transform-gpu.mlir
index 09ae0f4af686f..465e8fdd66422 100644
--- a/mlir/test/Dialect/GPU/transform-gpu.mlir
+++ b/mlir/test/Dialect/GPU/transform-gpu.mlir
@@ -691,3 +691,148 @@ module attributes {transform.with_named_sequence} {
       transform.yield
   }
 }
+
+// -----
+
+#map = affine_map<(d0) -> (d0 * 128)>
+#map1 = affine_map<(d0) -> (d0 * 32)>
+
+// CHECK-DAG: #[[$MAPB:.*]] = affine_map<()[s0] -> (s0 * 128)>
+// CHECK-DAG: #[[$MAPLANE:.*]] = affine_map<()[s0, s1] -> ((s0 + s1 * 73) mod 32)>
+// CHECK-DAG: #[[$MAPI:.*]] = affine_map<()[s0, s1] -> (s0 * 32 + s1 * 2336 - ((s0 + s1 * 73) floordiv 2) * 64)>
+// CHECK-DAG: #[[$MAPJ:.*]] = affine_map<()[s0, s1] -> ((((s0 + s1 * 73) mod 32) floordiv 2) * 32)>
+
+// CHECK-LABEL: func.func @simple_fill(
+func.func @simple_fill(%arg0: memref<128x256xf32>) -> memref<128x256xf32> {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0.000000e+00> : vector<16x32xf32>
+    //   CHECK:   %[[C6:.*]] = arith.constant 6 : index
+    //   CHECK:   gpu.launch
+  scf.forall (%arg1) in (1) {
+    //   CHECK:     %[[BIDX:.*]] = gpu.block_id  x
+    //   CHECK:     %[[BLX:.*]] = affine.apply #[[$MAPB]]()[%[[BIDX]]]
+    %0 = affine.apply #map(%arg1)
+    %subview = memref.subview %arg0[%0, 0] [128, 256] [1, 1]
+      : memref<128x256xf32> to memref<128x256xf32, strided<[256, 1], offset: ?>>
+
+    // %arg2 and %arg3 map to lanes [0, 6) and are turned into epxressions
+    // involving threadIdx.x/y by the map_nested_forall_to_threads
+    // transformation. This results in a if (linear_thread_id < 6) conditional.
+    scf.forall (%arg2, %arg3) in (2, 3) {
+      //       CHECK:     %[[TIDX:.*]] = gpu.thread_id  x
+      //       CHECK:     %[[TIDY:.*]] = gpu.thread_id  y
+      //       CHECK:     %[[LID:.*]] = affine.apply #[[$MAPLANE]]()[%[[TIDX]], %[[TIDY]]]
+      //       CHECK:     %[[COND:.*]] = arith.cmpi ult, %[[LID]], %[[C6]]
+      //       CHECK:     scf.if %[[COND]]
+      //       CHECK:       %[[I:.*]] = affine.apply #[[$MAPI]]()[%[[TIDX]], %[[TIDY]]]
+      //       CHECK:       %[[J:.*]] = affine.apply #[[$MAPJ]]()[%[[TIDX]], %[[TIDY]]]
+      //       CHECK:       memref.subview %{{.*}}[%[[I]], %[[J]]]
+      %1 = affine.apply #map1(%arg2)
+      %2 = affine.apply #map1(%arg3)
+      %subview_0 = memref.subview %subview[%1, %2] [16, 32] [1, 1] 
+        : memref<128x256xf32, strided<[256, 1], offset: ?>> to memref<16x32xf32, strided<[256, 1], offset: ?>>
+      vector.transfer_write %cst, %subview_0[%c0, %c0] {in_bounds = [true, true]} 
+        : vector<16x32xf32>, memref<16x32xf32, strided<[256, 1], offset: ?>>
+
+    // This could be obtained e.g. if a previous transformation mapped this loop
+    // to lanes. This can aslo be written by hand as valid IR.
+    } {mapping = [#gpu.lane<linear_dim_0>, #gpu.lane<linear_dim_1>]}
+  } {mapping = [#gpu.block<x>]}
+  return %arg0 : memref<128x256xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module_op
+      : (!transform.any_op) -> !transform.any_op
+    %gpu_launch = transform.gpu.map_forall_to_blocks %func generate_gpu_launch
+      : (!transform.any_op) -> !transform.any_op
+
+    // This transformation maps scf.forall ivs to a particular mapping of thread
+    // ids (laneid, threadid, warpid or warpgroupid).
+    transform.gpu.map_nested_forall_to_threads %gpu_launch block_dims = [73, 5, 1]
+      : (!transform.any_op) -> !transform.any_op
+      transform.yield
+  }
+}
+
+// -----
+
+#map = affine_map<(d0) -> (d0 *  128)>
+#map1 = affine_map<(d0) -> (d0 * 32)>
+
+// CHECK-DAG: #[[$MAPB:.*]] = affine_map<()[s0] -> (s0 * 128)>
+// CHECK-DAG: #[[$MAP_LIN_W:.*]] = affine_map<()[s0, s1] -> ((s0 + s1 * 73) floordiv 32)>
+// CHECK-DAG: #[[$MAP_W0:.*]] = affine_map<()[s0] -> (s0 * 32 - (s0 floordiv 2) * 64)>
+// CHECK-DAG: #[[$MAP_W1:.*]] = affine_map<()[s0] -> ((s0 floordiv 2) * 32)>
+
+// CHECK-LABEL: func.func @simple_fill(
+func.func @simple_fill(%arg0: memref<128xf32>) -> memref<128xf32> {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0.000000e+00> : vector<32xf32>
+//   CHECK-DAG:   %[[C0_i64:.*]] = arith.constant 0 : i64
+//   CHECK-DAG:   %[[C1_i64:.*]] = arith.constant 1 : i64
+/// 0x2f1 is 753
+//   CHECK-DAG:   %[[C753_i64:.*]] = arith.constant 753 : i64
+
+//       CHECK:   gpu.launch
+  scf.forall (%arg1) in (1) {
+//       CHECK:     %[[BIDX:.*]] = gpu.block_id  x
+//       CHECK:     %[[BLX:.*]] = affine.apply #[[$MAPB]]()[%[[BIDX]]]
+    %0 = affine.apply #map(%arg1)
+    %subview = memref.subview %arg0[%0] [128] [1] : memref<128xf32> to memref<128xf32, strided<[1], offset: ?>>
+
+    // %arg2 and %arg3 map to lanes [0, 6) and are turned into epxressions
+    // involving threadIdx.x/y by the map_nested_forall_to_threads
+    // transformation. This results in a if (linear_thread_id < 6) conditional.
+    scf.forall (%arg2, %arg3) in (2, 3) {
+      //       CHECK:     %[[TIDX:.*]] = gpu.thread_id  x
+      //       CHECK:     %[[TIDY:.*]] = gpu.thread_id  y
+
+      //       CHECK:     %[[LIN_W:.*]] = affine.apply #[[$MAP_LIN_W]]()[%[[TIDX]], %[[TIDY]]]
+      //
+      // Compute the active warps below using the mask + popcnt
+      //       CHECK:     %[[LIN_W_i64:.*]] = arith.index_castui %[[LIN_W]] : index to i64
+      //       CHECK:     %[[TWO_POW_W:.*]] = arith.shli %[[C1_i64]], %[[LIN_W_i64]] : i64
+      //       CHECK:     %[[FILTER_TILL_W:.*]] = arith.subi %[[TWO_POW_W]], %[[C1_i64]] : i64
+      //       CHECK:     %[[ACTIVE_TILL_W:.*]] = arith.andi %[[FILTER_TILL_W]], %[[C753_i64]] : i64
+      //       CHECK:     %[[LOGICAL_ID_W_i64:.*]] = math.ctpop %[[ACTIVE_TILL_W]] : i64
+      //       CHECK:     %[[LOGICAL_ID_W:.*]] = arith.index_castui %[[LOGICAL_ID_W_i64]] : i64 to index
+      //
+      // Dynamically compute whether this warp is active below using the mask + popcnt
+      //       CHECK:     %[[IS_ACTIVE_W_MASK:.*]] = arith.andi %[[TWO_POW_W]], %[[C753_i64]] : i64
+      //       CHECK:     %[[IS_ACTIVE_W:.*]] = arith.cmpi ne, %[[IS_ACTIVE_W_MASK]], %[[C0_i64]] : i64
+      //       CHECK:     scf.if %[[IS_ACTIVE_W]] {
+
+      //       CHECK:       %[[W0:.*]] = affine.apply #[[$MAP_W0]]()[%[[LOGICAL_ID_W]]]
+      //       CHECK:       %[[W1:.*]] = affine.apply #[[$MAP_W1]]()[%[[LOGICAL_ID_W]]]
+      //       CHECK:       memref.subview %{{.*}}[%[[W0]]] [%[[W1]]]
+      %1 = affine.apply #map1(%arg2)
+      %2 = affine.apply #map1(%arg3)
+      %subview_0 = memref.subview %subview[%1] [%2] [1] : memref<128xf32, strided<[1], offset: ?>> to memref<?xf32, strided<[1], offset: ?>>
+      vector.transfer_write %cst, %subview_0[%c0] {in_bounds = [true]} : vector<32xf32>, memref<?xf32, strided<[1], offset: ?>>
+
+    // This could be obtained e.g. if a previous transformation mapped this loop
+    // to lanes. This can aslo be written by hand as valid IR.
+    // This additionally uses the hex mask: 0x 10 1111 0001
+    } {mapping = [#gpu.warp<linear_dim_0>, #gpu.warp<linear_dim_1>, #gpu.mask<0x2f1>]}
+
+    memref.copy %subview, %subview : memref<128xf32, strided<[1], offset: ?>> to memref<128xf32, strided<[1], offset: ?>>
+  } {mapping = [#gpu.block<x>]}
+  return %arg0 : memref<128xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module_op
+      : (!transform.any_op) -> !transform.any_op
+    %gpu_launch = transform.gpu.map_forall_to_blocks %func generate_gpu_launch
+      : (!transform.any_op) -> !transform.any_op
+
+    // This transformation maps scf.forall ivs to a particular mapping of thread
+    // ids (laneid, threadid, warpid or warpgroupid).
+    transform.gpu.map_nested_forall_to_threads %gpu_launch block_dims = [73, 5, 1]
+      : (!transform.any_op) -> !transform.any_op
+      transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 251ca716c7a7a..bd1106e304c60 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -1875,7 +1875,7 @@ llvm.mlir.global @bad_struct_array_init_elements() : !llvm.array<1x!llvm.struct<
   llvm.return %0 : !llvm.array<1x!llvm.struct<(i32, f32)>>
 }
 
-// ----
+// -----
 
 llvm.mlir.global internal constant @bad_array_attr_simple_type() : !llvm.array<2 x f64> {
   // expected-error@below {{'llvm.mlir.constant' op for array with an array attribute must have a struct element type}}
@@ -1883,10 +1883,51 @@ llvm.mlir.global internal constant @bad_array_attr_simple_type() : !llvm.array<2
   llvm.return %0 : !llvm.array<2 x f64>
 }
 
-// ----
+// -----
 
 llvm.func @inlineAsmMustTail(%arg0: i32, %arg1 : !llvm.ptr) {
   // expected-error@+1 {{op tail call kind 'musttail' is not supported}}
   %8 = llvm.inline_asm tail_call_kind = <musttail> "foo", "=r,=r,r" %arg0 : (i32) -> !llvm.struct<(i8, i8)>
   llvm.return
 }
+
+// -----
+
+llvm.func @invalid_xevm_prefetch(%arg0: !llvm.ptr) {
+  // expected-error@+1 {{op operand #0 must be LLVM pointer in address space 1 or LLVM pointer in address space 4}}
+  xevm.prefetch %arg0 <{cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr)
+  llvm.return
+}
+
+// -----
+
+llvm.func @invalid_xevm_mma(%loaded_c_casted: vector<4xf32>, %loaded_a: vector<8xi16>, %loaded_b_casted: vector<8xi32>) -> vector<8xf32> {
+  // expected-error@+1 {{op type of C operand must match result type}}
+  %c_result = xevm.mma %loaded_a, %loaded_b_casted, %loaded_c_casted {shape = <m = 8, n = 16, k = 16>, types = <d = f32, a = f16, b = f16, c = f32>} : (vector<8xi16>, vector<8xi32>, vector<4xf32>) -> vector<8xf32>
+  llvm.return %c_result : vector<8xf32>
+}
+
+// -----
+
+llvm.func @invalid_xevm_matrix_1(%c: !llvm.ptr<1>, %base_width_c: i32, %base_height_c: i32, %base_pitch_c: i32, %x: i32, %y: i32, %c_result_casted: vector<8xi32>) {
+  // expected-error@+1 {{op expecting tile_width to be between 1 and 8}}
+  xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted <{elem_size_in_bits=64 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  llvm.return
+}
+
+// -----
+
+llvm.func @invalid_xevm_matrix_2(%c: !llvm.ptr<1>, %base_width_c: i32, %base_height_c: i32, %base_pitch_c: i32, %x: i32, %y: i32, %c_result_casted: vector<8xi32>) {
+  // expected-error@+1 {{op expecting elem_size_in_bits to be 8, 16, 32, or 64}}
+  xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted <{elem_size_in_bits=18 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  llvm.return
+}
+
+// -----
+
+llvm.func @invalid_xevm_matrix_3(%a: !llvm.ptr<1>, %base_width_a: i32, %base_height_a: i32, %base_pitch_a: i32, %x: i32, %y: i32) -> vector<8xi16> {
+  // expected-error@+1 {{op result size of 128 bits does not match the expected size of 208 bits}}
+  %loaded_a = xevm.blockload2d %a, %base_width_a, %base_height_a, %base_pitch_a, %x, %y <{elem_size_in_bits=16 : i32, tile_width=26 : i32, tile_height=8 : i32, v_blocks=1 : i32, transpose=false, pack_register=false, cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+  llvm.return %loaded_a : vector<8xi16>
+}
+
diff --git a/mlir/test/Dialect/LLVMIR/xevm.mlir b/mlir/test/Dialect/LLVMIR/xevm.mlir
new file mode 100644
index 0000000000000..3dd5f872f898c
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/xevm.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics | FileCheck %s
+
+// CHECK-LABEL: func.func @blockload2d(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>,
+// CHECK-SAME: %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32, %[[ARG5:.*]]: i32)
+func.func @blockload2d(%a: !llvm.ptr<1>, %base_width_a: i32, %base_height_a: i32,
+  %base_pitch_a: i32, %x: i32, %y: i32) -> vector<8xi16> {
+  // CHECK: %[[VAR0:.*]] = xevm.blockload2d %[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]]
+  // CHECK-DAG: elem_size_in_bits = 16 : i32
+  // CHECK-DAG: tile_width = 16 : i32
+  // CHECK-DAG: tile_height = 8 : i32
+  // CHECK-DAG: v_blocks = 1 : i32
+  // CHECK-DAG: transpose = false
+  // CHECK-DAG: pack_register = false
+  // CHECK-DAG: cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>
+  // CHECK: (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+  %loaded_a = xevm.blockload2d %a, %base_width_a, %base_height_a, %base_pitch_a, %x, %y
+    <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+    transpose=false, pack_register=false, cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}>
+    : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+  return %loaded_a : vector<8xi16>
+}
+
+// -----
+// CHECK-LABEL: func.func @blockstore2d(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>,
+// CHECK-SAME: %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32, %[[ARG5:.*]]: i32,
+// CHECK-SAME: %[[ARG6:.*]]: vector<8xi32>)
+func.func @blockstore2d(%c: !llvm.ptr<1>, %base_width_c: i32, %base_height_c: i32,
+  %base_pitch_c: i32, %x: i32, %y: i32, %c_result_casted: vector<8xi32>) {
+  // CHECK: xevm.blockstore2d %[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]], %[[ARG6]]
+  // CHECK-DAG: elem_size_in_bits = 32 : i32
+  // CHECK-DAG: tile_width = 16 : i32
+  // CHECK-DAG: tile_height = 8 : i32
+  // CHECK: (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted
+    <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32}>
+    : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @blockprefetch2d(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>,
+// CHECK-SAME: %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32, %[[ARG5:.*]]: i32)
+func.func @blockprefetch2d(%ptr: !llvm.ptr<1>, %base_width: i32, %base_height: i32,
+  %base_pitch: i32, %x: i32, %y: i32) {
+  // CHECK: xevm.blockprefetch2d %[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]]
+  // CHECK-DAG: elem_size_in_bits = 8 : i32
+  // CHECK-DAG: tile_width = 32 : i32
+  // CHECK-DAG: tile_height = 8 : i32
+  // CHECK-DAG: v_blocks = 1 : i32
+  // CHECK-DAG: cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>
+  // CHECK:  (!llvm.ptr<1>, i32, i32, i32, i32, i32)
+  xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y <{elem_size_in_bits=8 : i32,
+    tile_width=32 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+    cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32)
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @mma(
+// CHECK-SAME: %[[ARG0:.*]]: vector<8xf32>, %[[ARG1:.*]]: vector<8xi16>, %[[ARG2:.*]]: vector<8xi32>)
+func.func @mma(%loaded_c_casted: vector<8xf32>, %loaded_a: vector<8xi16>, %loaded_b_casted: vector<8xi32>) -> vector<8xf32> {
+  // CHECK: %0 = xevm.mma %[[ARG1]], %[[ARG2]], %[[ARG0]] {shape = <m = 8, n = 16, k = 16>,
+  // CHECK-SAME: types = <d = f32, a = f16, b = f16, c = f32>} : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+  %c_result = xevm.mma %loaded_a, %loaded_b_casted, %loaded_c_casted { shape=<m=8, n=16, k=16>,
+    types=<d=f32, a=f16, b=f16, c=f32> } : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+  return %c_result : vector<8xf32>
+}
+
+// -----
+// CHECK-LABEL: func.func @memfence()
+func.func @memfence() {
+  // CHECK: xevm.memfence
+  // CHECK-DAG: addrspace = #xevm.addr_space<global>
+  // CHECK-DAG: scope = #xevm.mem_scope<workgroup>
+  xevm.memfence <{addrspace=#xevm.addr_space<global>, scope=#xevm.mem_scope<workgroup>}>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @prefetch(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>)
+func.func @prefetch(%ptr: !llvm.ptr<1>) {
+  // CHECK: xevm.prefetch %[[ARG0]]
+  // CHECK-SAME: <{cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>)
+  xevm.prefetch %ptr <{cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>)
+  return
+}
+
+// -----
+// CHECK-LABEL: @xevm_module [#xevm.target<O = 3, chip = "pvc">] {
+gpu.module @xevm_module [#xevm.target<O = 3, chip = "pvc">]{
+}
diff --git a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
index 31c9e9ed3c501..6fc8d9f152f4e 100644
--- a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
+++ b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
@@ -435,6 +435,40 @@ func.func @elem_pack_transpose_outer_dims(%arg0: tensor<128x256xi32>, %init: ten
 
 // -----
 
+#map0 = affine_map<(d0, d1) -> (d0, d1)>
+func.func @elem_pack_transpose_outer_dims_unused_init(%arg0: tensor<128x256xi32>, %init: tensor<128x256xi32>) -> tensor<16x4x32x16xi32>{
+  %elem = linalg.generic {indexing_maps = [#map0, #map0], iterator_types = ["parallel", "parallel"]}
+      ins(%arg0 : tensor<128x256xi32>)
+      outs(%init : tensor<128x256xi32>) {
+    ^bb0(%arg3: i32, %arg4: i32):
+      %4 = arith.addi %arg3, %arg3 : i32
+      linalg.yield %4 : i32
+  } -> tensor<128x256xi32>
+  %empty = tensor.empty() : tensor<16x4x32x16xi32>
+  %pack = linalg.pack %elem
+    outer_dims_perm = [1, 0]
+    inner_dims_pos = [0, 1]
+    inner_tiles = [32, 16]
+    into %empty : tensor<128x256xi32> -> tensor<16x4x32x16xi32>
+  return %pack : tensor<16x4x32x16xi32>
+}
+
+// CHECK: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+// CHECK-LABEL: func.func @elem_pack_transpose_outer_dims
+// CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]
+// CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]
+// CHECK:         %[[ARG1_EMPTY:.+]] = tensor.empty() : tensor<16x4x32x16xi32>
+// CHECK:         %[[ARG0_EMPTY:.+]] = tensor.empty() : tensor<16x4x32x16xi32>
+// CHECK:         %[[PACKED_ARG0:.+]] = linalg.pack %[[ARG0]]
+// CHECK-SAME:      outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:      into %[[ARG0_EMPTY]]
+// CHECK:         %[[RES:.+]] = linalg.generic
+// CHECK-SAME:      indexing_maps = [#[[$MAP]], #[[$MAP]]]
+// CHECK-SAME:      ins(%[[PACKED_ARG0]]
+// CHECK-SAME:      outs(%[[ARG1_EMPTY]]
+
+// -----
+
 #map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
 
 func.func @unpack_on_output(%arg0: tensor<12x2x56x56x32xf32>) -> tensor<12x56x56x64xf32> {
@@ -497,7 +531,7 @@ func.func @unpack_on_input(%arg0: tensor<12x2x56x56x32xf32>, %init: tensor<12x56
 
 #map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
 
-func.func @unpack_element_type_change(%arg0: tensor<12x2x56x56x32xf32>, %init: tensor<12x56x56x64xf16>) -> tensor<12x56x56x64xf16> {
+func.func @unpack_element_type_change_no_use(%arg0: tensor<12x2x56x56x32xf32>, %init: tensor<12x56x56x64xf16>) -> tensor<12x56x56x64xf16> {
   %0 = tensor.empty() : tensor<12x56x56x64xf32>
   %1 = linalg.unpack %arg0 outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32] into %0 : tensor<12x2x56x56x32xf32> -> tensor<12x56x56x64xf32>
   %2 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%1: tensor<12x56x56x64xf32>) outs(%init : tensor<12x56x56x64xf16>) {
@@ -509,17 +543,14 @@ func.func @unpack_element_type_change(%arg0: tensor<12x2x56x56x32xf32>, %init: t
 }
 
 // CHECK: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
-// CHECK-LABEL: func.func @unpack_element_type_change
+// CHECK-LABEL: func.func @unpack_element_type_change_no_use
 // CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]
 // CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]
-// CHECK:         %[[ARG1_PACK_EMPTY:.+]] = tensor.empty() : tensor<12x2x56x56x32xf16>
-// CHECK:         %[[ARG1_PACK:.+]] = linalg.pack %[[ARG1]]
-// CHECK-SAME:      outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32]
-// CHECK-SAME:      into %[[ARG1_PACK_EMPTY]]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<12x2x56x56x32xf16>
 // CHECK:         %[[RES:.+]] = linalg.generic
 // CHECK-SAME:      indexing_maps = [#[[$MAP]], #[[$MAP]]]
 // CHECK-SAME:      ins(%[[ARG0]]
-// CHECK-SAME:      outs(%[[ARG1_PACK]]
+// CHECK-SAME:      outs(%[[EMPTY]]
 // CHECK:         %[[UNPACK:.+]] = linalg.unpack %[[RES]]
 // CHECK-SAME:      outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32]
 // CHECK-SAME:      into %[[ARG1]]
@@ -1402,13 +1433,10 @@ func.func @push_unpack_in_padded_domain_foldable(%arg0: tensor<8x8x4x8xf32>, %de
 // CHECK-SAME:    %[[ARG0:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[ARG1:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[ARG2:[a-zA-Z0-9]+]]
-// CHECK:         %[[ARG2_PACK_EMPTY:.+]] = tensor.empty
-// CHECK:         %[[ARG2_PACK:.+]] = linalg.pack %[[ARG2]]
-// CHECK-SAME:      inner_dims_pos = [0, 1] inner_tiles = [4, 8]
-// CHECK-SAME:      into %[[ARG2_PACK_EMPTY]]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty
 // CHECK:         %[[GENERIC:.+]] = linalg.generic
 // CHECK-SAME:    ins(%[[ARG0]] : tensor<8x8x4x8xf32>)
-// CHECK-SAME:    outs(%[[ARG2_PACK]] : tensor<?x8x4x8xbf16>)
+// CHECK-SAME:    outs(%[[EMPTY]] : tensor<?x8x4x8xbf16>)
 // CHECK:         %[[UNPACK:.+]] = linalg.unpack %[[GENERIC]]
 // CHECK-SAME:    into %[[ARG2]]
 // CHECK:         return %[[UNPACK]] : tensor<?x64xbf16>
diff --git a/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir b/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
index bbd6e0fc8e2cc..290c6c7c36f76 100644
--- a/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
@@ -434,117 +434,6 @@ func.func @generalize_const(%min: f64, %max: f64, %seed: i32, %O: tensor<16x32xf
 
 // -----
 
-// Verifies the default value of the fun attribute is an exp op.
-func.func @generalize_elemwise_exp(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_exp
-// CHECK:        = math.exp
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_log(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<log>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_log
-// CHECK:        = math.log
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_abs(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<abs>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_abs
-// CHECK:        = math.absf
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_ceil(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<ceil>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_ceil
-// CHECK:        = math.ceil
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_floor(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<floor>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_floor
-// CHECK:        = math.floor
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_negf(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<negf>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_negf
-// CHECK:        = arith.negf
-
-// -----
-
-// Verifies the default value of the fun attribute is an add op.
-func.func @generalize_elemwise_add(%lhs : tensor<4x8xf32>, %rhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_binary ins(%lhs, %rhs: tensor<4x8xf32>, tensor<4x8xf32>)
-                              outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_add
-// CHECK:        = arith.addf
-
-// -----
-
-// Verifies the fun attribute controls the binary function used.
-func.func @generalize_elemwise_mul(%lhs : tensor<4x8xf32>, %rhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<mul>}
-                              ins(%lhs, %rhs: tensor<4x8xf32>, tensor<4x8xf32>)
-                              outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_mul
-// CHECK:        = arith.mulf
-
-// -----
-
-// Verifies pointwise ops support rank zero input tensors
-func.func @generalize_elemwise_rank_zero(%lhs : tensor<f32>, %rhs : tensor<f32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<sub>}
-                              ins(%lhs, %rhs: tensor<f32>, tensor<f32>)
-                              outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_rank_zero
-// CHECK:       linalg.generic
-// CHECK-SAME:  iterator_types = ["parallel", "parallel"]
-// CHECK:        = arith.subf
-
-// -----
-
 // Verifies the fun attribute controls the binary function used.
 func.func @generalize_copy(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
   %0 = linalg.copy ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index 9c2961231fc0a..da1dfc7b6a624 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -1824,6 +1824,16 @@ func.func @unpack_invalid_outer_dims_perm(%source: tensor<128x256xf32>, %dest: t
 
 // -----
 
+// The outer dims in the output tensor are incorrectly/unexpectedly transposed.
+// This could be fixed by adding `outer_dims_perm = [1, 0]` (the default value assumes no transpose).
+func.func @pack_invalid_result_shape(%input: tensor<256x128xf32>, %output: tensor<4x16x32x16xf32>) -> tensor<4x16x32x16xf32> {
+  // expected-error@+1 {{the shape of output is not large enough to hold the packed data. Expected at least 'tensor<16x4x32x16xf32>', got 'tensor<4x16x32x16xf32>'}}
+  %0 = linalg.pack %input inner_dims_pos = [1, 0] inner_tiles = [32, 16] into %output : tensor<256x128xf32> -> tensor<4x16x32x16xf32>
+  return %0 : tensor<4x16x32x16xf32>
+}
+
+// -----
+
 func.func @pack_invalid(%input: tensor<256x128xf32>, %output: tensor<8x8x32x16xf32>) -> tensor<8x8x32x16xf32> {
   // expected-error@+1 {{the shape of output is not large enough to hold the packed data. Expected at least 'tensor<8x8x16x32xf32>', got 'tensor<8x8x32x16xf32>'}}
   %0 = linalg.pack %input inner_dims_pos = [1, 0] inner_tiles = [16, 32] into %output : tensor<256x128xf32> -> tensor<8x8x32x16xf32>
@@ -1899,14 +1909,6 @@ module {
 
 // -----
 
-func.func @elemwise_unary_invalid_mixed_types(%arg0 : tensor<?xi32>) -> tensor<?xi32> {
-  // expected-error @below {{unsupported non numeric type}}
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?xi32>) outs(%arg0 : tensor<?xi32>) -> tensor<?xi32>
-  return %0 : tensor<?xi32>
-}
-
-// -----
-
 func.func @matmul_invalid_mixed_types(%t: tensor<?xf16>, %f: vector<4xf16>)
   -> (tensor<?xf16>, vector<4xf16>)
 {
diff --git a/mlir/test/Dialect/Linalg/library-calls.mlir b/mlir/test/Dialect/Linalg/library-calls.mlir
index 1fa675d8b4b68..77c9d4a911447 100644
--- a/mlir/test/Dialect/Linalg/library-calls.mlir
+++ b/mlir/test/Dialect/Linalg/library-calls.mlir
@@ -59,43 +59,3 @@ module {
     return
   }
 }
-
-
-// -----
-
-// CHECK: func.func private @linalg_elemwise_unary_negf_view16x8xf32_view16x8xf32(memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>) attributes {llvm.emit_c_interface}
-// CHECK: func.func private @linalg_elemwise_unary_negf_view16xf32_view16xf32(memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>) attributes {llvm.emit_c_interface}
-
-func.func @test_neg(%A : memref<16x8xf32>, %B: memref<16x8xf32>, %C: memref<16xf32>, %D: memref<16xf32>) {
-  linalg.elemwise_unary {fun = #linalg.unary_fn<negf>}
-                              ins(%A: memref<16x8xf32>) outs(%B: memref<16x8xf32>)
-  linalg.elemwise_unary {fun = #linalg.unary_fn<negf>}
-                              ins(%C: memref<16xf32>) outs(%D: memref<16xf32>)
-  return
-}
-
-// -----
-
-// CHECK: func.func private @linalg_elemwise_unary_exp_view16x8xf32_view16x8xf32(memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>) attributes {llvm.emit_c_interface}
-// CHECK: func.func private @linalg_elemwise_unary_exp_view16xf32_view16xf32(memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>) attributes {llvm.emit_c_interface}
-
-func.func @test_exp(%A : memref<16x8xf32>, %B: memref<16x8xf32>, %C: memref<16xf32>, %D: memref<16xf32>) {
-  linalg.elemwise_unary {fun = #linalg.unary_fn<exp>}
-                              ins(%A: memref<16x8xf32>) outs(%B: memref<16x8xf32>)
-  linalg.elemwise_unary {fun = #linalg.unary_fn<exp>}
-                              ins(%C: memref<16xf32>) outs(%D: memref<16xf32>)
-  return
-}
-
-// -----
-
-// CHECK: func.func private @linalg_elemwise_binary_add_view16x8xf32_view16x8xf32_view16x8xf32(memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>) attributes {llvm.emit_c_interface}
-// CHECK: func.func private @linalg_elemwise_binary_add_view16xf32_view16xf32_view16xf32(memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>) attributes {llvm.emit_c_interface}
-
-func.func @test_add(%A : memref<16x8xf32>, %B: memref<16x8xf32>, %C: memref<16x8xf32>, %D: memref<16xf32>, %E: memref<16xf32>, %F: memref<16xf32>) {
-  linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-                              ins(%A, %B: memref<16x8xf32>, memref<16x8xf32>) outs(%C: memref<16x8xf32>)
-  linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-                              ins(%D, %E: memref<16xf32>, memref<16xf32>) outs(%F: memref<16xf32>)
-  return
-}
diff --git a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
index bfe7a07cb38a5..618ba3402ff52 100644
--- a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
+++ b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
@@ -842,15 +842,15 @@ module attributes { transform.with_named_sequence } {
     // expected-remark @below {{op result}}
     // expected-note @below {{value handle points to an op result #0}}
     // expected-remark @below {{single user}}
-    linalg.elemwise_unary {fun = #linalg.unary_fn<negf>} ins(%2 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
+    linalg.negf ins(%2 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
     // expected-remark @below {{matched result value}}
     // expected-remark @below {{op result}}
     // expected-note @below {{value handle points to an op result #0}}
-    linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
+    linalg.exp ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
     // expected-remark @below {{matched result value}}
     // expected-remark @below {{op result}}
     // expected-note @below {{value handle points to an op result #0}}
-    linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
+    linalg.exp ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
     return
   }
 }
diff --git a/mlir/test/Dialect/Linalg/named-ops.mlir b/mlir/test/Dialect/Linalg/named-ops.mlir
index 470bc1c78640c..412f40d501154 100644
--- a/mlir/test/Dialect/Linalg/named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops.mlir
@@ -2771,6 +2771,101 @@ func.func @pad_and_pack_partially_dynamic(%source: tensor<?x?xf32>, %dest: tenso
 
 // -----
 
+func.func @pack_transposed_inner_dims_with_padding(%source: tensor<1x5x7xf32>, %dest: tensor<1x3x2x4x2xf32>, %pad: f32) -> tensor<1x3x2x4x2xf32> {
+  %0 = linalg.pack %source padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [4, 2] into %dest : tensor<1x5x7xf32> -> tensor<1x3x2x4x2xf32>
+  return %0 : tensor<1x3x2x4x2xf32>
+}
+
+// CHECK-LABEL: func.func @pack_transposed_inner_dims_with_padding(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<1x5x7xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<1x3x2x4x2xf32>,
+// CHECK-SAME:  %[[PAD:.*]]: f32)
+// CHECK:       %{{.*}} = linalg.pack
+// CHECK-SAME:      inner_dims_pos = [2, 1]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<1x5x7xf32> -> tensor<1x3x2x4x2xf32>
+
+// -----
+
+// The function suffix "with_padding" refers to the padding that was introduced by the pack operation. But here
+// we are dropping the padding. Creating a tensor with less elements than what we started with.
+func.func @unpack_descending_inner_dims_with_padding(%source: tensor<1x3x2x4x2xf32>, %dest: tensor<1x5x7xf32>) -> tensor<1x5x7xf32> {
+  %0 = linalg.unpack %source inner_dims_pos = [2, 1] inner_tiles = [4, 2] into %dest : tensor<1x3x2x4x2xf32> -> tensor<1x5x7xf32>
+  return %0 : tensor<1x5x7xf32>
+}
+
+// CHECK-LABEL: func.func @unpack_descending_inner_dims_with_padding(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<1x3x2x4x2xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<1x5x7xf32>)
+// CHECK:       %{{.*}} = linalg.unpack
+// CHECK-SAME:      inner_dims_pos = [2, 1]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<1x3x2x4x2xf32> -> tensor<1x5x7xf32>
+
+// -----
+
+func.func @pack_non_adjacent_inner_dims(%source: tensor<20x1x12xf32>, %dest: tensor<10x1x3x4x2xf32>) -> tensor<10x1x3x4x2xf32> {
+  %0 = linalg.pack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2] into %dest : tensor<20x1x12xf32> -> tensor<10x1x3x4x2xf32>
+  return %0 : tensor<10x1x3x4x2xf32>
+}
+
+// CHECK-LABEL: func.func @pack_non_adjacent_inner_dims(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<20x1x12xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<10x1x3x4x2xf32>)
+// CHECK:       %{{.*}} = linalg.pack
+// CHECK-SAME:      inner_dims_pos = [2, 0]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<20x1x12xf32> -> tensor<10x1x3x4x2xf32>
+
+// -----
+
+func.func @unpack_non_adjacent_inner_dims(%source: tensor<10x1x3x4x2xf32>, %dest: tensor<20x1x12xf32>) -> tensor<20x1x12xf32> {
+  %0 = linalg.unpack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2] into %dest : tensor<10x1x3x4x2xf32> -> tensor<20x1x12xf32>
+  return %0 : tensor<20x1x12xf32>
+}
+
+// CHECK-LABEL: func.func @unpack_non_adjacent_inner_dims(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<10x1x3x4x2xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<20x1x12xf32>)
+// CHECK:       %{{.*}} = linalg.unpack
+// CHECK-SAME:      inner_dims_pos = [2, 0]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<10x1x3x4x2xf32> -> tensor<20x1x12xf32>
+
+// -----
+
+func.func @pack_implementing_transpose(%source: tensor<3x5x7xf32>, %dest: tensor<3x7x5xf32>) -> tensor<3x7x5xf32> {
+  %0 = linalg.pack %source outer_dims_perm = [0, 2, 1] inner_dims_pos = [] inner_tiles = [] into %dest : tensor<3x5x7xf32> -> tensor<3x7x5xf32>
+  return %0 : tensor<3x7x5xf32>
+}
+
+// CHECK-LABEL: func.func @pack_implementing_transpose(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<3x5x7xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<3x7x5xf32>)
+// CHECK:       %{{.*}} = linalg.pack
+// CHECK-SAME:      outer_dims_perm = [0, 2, 1]
+// CHECK-SAME:      inner_dims_pos = []
+// CHECK-SAME:      inner_tiles = []
+// CHECK-SAME:      into %[[DEST]] : tensor<3x5x7xf32> -> tensor<3x7x5xf32>
+
+// -----
+
+func.func @unpack_implementing_transpose(%source: tensor<3x7x5xf32>, %dest: tensor<3x5x7xf32>) -> tensor<3x5x7xf32> {
+  %0 = linalg.unpack %source outer_dims_perm = [0, 2, 1] inner_dims_pos = [] inner_tiles = [] into %dest : tensor<3x7x5xf32> -> tensor<3x5x7xf32>
+  return %0 : tensor<3x5x7xf32>
+}
+
+// CHECK-LABEL: func.func @unpack_implementing_transpose(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<3x7x5xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<3x5x7xf32>)
+// CHECK:       %{{.*}} = linalg.unpack
+// CHECK-SAME:      outer_dims_perm = [0, 2, 1]
+// CHECK-SAME:      inner_dims_pos = []
+// CHECK-SAME:      inner_tiles = []
+// CHECK-SAME:      into %[[DEST]] : tensor<3x7x5xf32> -> tensor<3x5x7xf32>
+
+// -----
+
 func.func @unpack_fully_dynamic(%source: tensor<?x?x?x?xf32>, %dest: tensor<?x?xf32>, %tile_n : index, %tile_m : index) -> tensor<?x?xf32> {
   %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [%tile_n, %tile_m] into %dest : tensor<?x?x?x?xf32> -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
diff --git a/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir b/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir
index 5b7c2baf9d84f..a0922bdfcfbe4 100644
--- a/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir
+++ b/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir
@@ -1,43 +1,5 @@
 // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only" -split-input-file | FileCheck %s
 
-// CHECK-LABEL: @elementwise_no_conflict
-func.func @elementwise_no_conflict(%a: tensor<5xf32>,
-                                   %b: tensor<5xf32>) -> tensor<5xf32> {
-  // CHECK: linalg.elemwise_binary
-  // CHECK-SAME: {__inplace_operands_attr__ = ["true", "true", "true"], fun = #linalg.binary_fn<add>}
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-      ins(%a, %b : tensor<5xf32>, tensor<5xf32>)
-      outs(%a : tensor<5xf32>) -> tensor<5xf32>
-  return %0 : tensor<5xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @elementwise_no_conflict_2
-func.func @elementwise_no_conflict_2(%a: tensor<5xf32>) -> tensor<5xf32> {
-  // CHECK: linalg.elemwise_binary
-  // CHECK-SAME: {__inplace_operands_attr__ = ["true", "true", "true"], fun = #linalg.binary_fn<add>}
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-      ins(%a, %a : tensor<5xf32>, tensor<5xf32>)
-      outs(%a : tensor<5xf32>) -> tensor<5xf32>
-  return %0 : tensor<5xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @elementwise_no_conflict_3
-func.func @elementwise_no_conflict_3(%a: tensor<5xf32>) -> tensor<5xf32> {
-  %c0f = arith.constant 1.0 : f32
-  // CHECK: linalg.elemwise_binary
-  // CHECK-SAME: {__inplace_operands_attr__ = ["true", "none", "true"], fun = #linalg.binary_fn<add>}
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-      ins(%a, %c0f : tensor<5xf32>, f32)
-      outs(%a : tensor<5xf32>) -> tensor<5xf32>
-  return %0 : tensor<5xf32>
-}
-
-// -----
-
 func.func @not_elementwise(%a: tensor<5x6xf32>) -> tensor<5x6xf32> {
   %cst = arith.constant 5.0 : f32
   // CHECK: tensor.extract_slice
diff --git a/mlir/test/Dialect/Linalg/promotion_options.mlir b/mlir/test/Dialect/Linalg/promotion_options.mlir
index caa72ba24316f..dbc073c2665f9 100644
--- a/mlir/test/Dialect/Linalg/promotion_options.mlir
+++ b/mlir/test/Dialect/Linalg/promotion_options.mlir
@@ -42,3 +42,59 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+func.func @matmul_f32(%A: memref<512x256xf32>, %B: memref<256x512xf32>, %C: memref<256x256xf32>, %s0: index, %s1: index, %s2: index) {
+  %c0 = arith.constant 0 : index
+  %c256 = arith.constant 256 : index
+  %c512 = arith.constant 512 : index
+  scf.for %arg4 = %c0 to %c512 step %s0 {
+    scf.for %arg5 = %c0 to %c512 step %s1 {
+      scf.for %arg6 = %c0 to %c256 step %s2 {
+        %i0 = affine.min affine_map<(d0)[s0] -> (-d0 + 512, s0)>(%arg4)[%s0]
+        %i1 = affine.min affine_map<(d0)[s0] -> (-d0 + 512, s0)>(%arg5)[%s1]
+        %i2 = affine.min affine_map<(d0)[s0] -> (-d0 + 256, s0)>(%arg6)[%s2]
+        %0 = memref.subview %A[%arg4, %arg6][%i0, %i2][1, 1] : memref<512x256xf32> to memref<?x?xf32, strided<[256, 1], offset: ?>>
+        %1 = memref.subview %B[%arg6, %arg5][%i2, %i1][1, 1] : memref<256x512xf32> to memref<?x?xf32, strided<[512, 1], offset: ?>>
+        %2 = memref.subview %C[%arg4, %arg5][%i0, %i1][1, 1] : memref<256x256xf32> to memref<?x?xf32, strided<[256, 1], offset: ?>>
+        linalg.matmul
+          ins(%0, %1: memref<?x?xf32, strided<[256, 1], offset: ?>>,
+                      memref<?x?xf32, strided<[512, 1], offset: ?>>)
+          outs(%2: memref<?x?xf32, strided<[256, 1], offset: ?>>)
+      }
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @matmul_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: memref<512x256xf32>
+// CHECK-SAME:      %[[ARG1:.*]]: memref<256x512xf32>
+// CHECK-SAME:      %[[ARG2:.*]]: memref<256x256xf32>
+// CHECK-SAME:      %[[ARG3:.*]]: index, %[[ARG4:.*]]: index, %[[ARG5:.*]]: index
+// CHECK:           %[[C4:.*]] = arith.constant 4 : index
+
+// CHECK:                 %[[i0:.*]] = affine.min
+// CHECK:                 %[[i1:.*]] = affine.min
+// CHECK:                 %[[i2:.*]] = affine.min
+
+// CHECK:                 %[[VAL_13:.*]] = arith.muli %[[i0]], %[[i2]] : index
+// CHECK:                 %[[VAL_14:.*]] = arith.muli %[[VAL_13]], %[[C4]] : index
+// CHECK:                 %[[VAL_15:.*]] = memref.alloc(%[[VAL_14]]) : memref<?xi8>
+
+// CHECK:                 %[[VAL_18:.*]] = arith.muli %[[i2]], %[[i1]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.muli %[[VAL_18]], %[[C4]] : index
+// CHECK:                 %[[VAL_20:.*]] = memref.alloc(%[[VAL_19]]) : memref<?xi8>
+
+// CHECK:                 %[[VAL_23:.*]] = arith.muli %[[i0]], %[[i1]] : index
+// CHECK:                 %[[VAL_24:.*]] = arith.muli %[[VAL_23]], %[[C4]] : index
+// CHECK:                 %[[VAL_25:.*]] = memref.alloc(%[[VAL_24]]) : memref<?xi8>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.structured.promote %0 { use_original_subview_size } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir b/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir
index 5bdb5073ee865..e5216089692b4 100644
--- a/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir
@@ -26,8 +26,8 @@ module {
       // CHECK: %[[T1:.*]] = linalg.fill {{.*}} outs(%[[T0]]
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary ins(%[[T1]]
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.elementwise kind=#linalg.elementwise_kind<exp> ins(%[[T1]]
+      %7 = linalg.elementwise kind=#linalg.elementwise_kind<exp> ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -76,8 +76,8 @@ module {
       %4 = affine.min #map2(%arg3)[%arg0]
       %5 = tensor.extract_slice %o[%3] [%4] [1] : tensor<64xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary ins(%[[INIT_TENSOR]]
-      %7 = linalg.elemwise_unary ins(%0 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.exp ins(%[[INIT_TENSOR]]
+      %7 = linalg.exp ins(%0 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<64xf32>
       }
@@ -177,8 +177,8 @@ module {
       // CHECK: %[[T1:.*]] = linalg.fill {{.*}} outs(%[[T0]]
       %6 = tensor.extract_slice %arg1[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary {{.*}} outs(%[[T1]]
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.exp {{.*}} outs(%[[T1]]
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -228,8 +228,8 @@ module {
       // CHECK: %[[T2:.*]] = linalg.fill {{.*}} outs(%[[T1]]
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T3:.*]] = linalg.elemwise_unary ins(%[[T2]] : tensor<?xf32>) outs(%[[T0]] : tensor<?xf32>)
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T3:.*]] = linalg.exp ins(%[[T2]] : tensor<?xf32>) outs(%[[T0]] : tensor<?xf32>)
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -261,7 +261,7 @@ module {
     %c2 = arith.constant 2 : index
     %c1 = arith.constant 1 : index
     %c0 = arith.constant 0 : index
-    %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<abs>} ins(%arg0 : tensor<?x?x?xf32>) outs(%arg1 : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+    %0 = linalg.elementwise kind=#linalg.elementwise_kind<abs> ins(%arg0 : tensor<?x?x?xf32>) outs(%arg1 : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
     %dim = tensor.dim %arg1, %c0 : tensor<?x?x?xf32>
     %dim_0 = tensor.dim %arg1, %c1 : tensor<?x?x?xf32>
     %dim_1 = tensor.dim %arg1, %c2 : tensor<?x?x?xf32>
@@ -272,12 +272,12 @@ module {
       %2 = scf.for %arg4 = %c0 to %dim_0 step %c1 iter_args(%arg5 = %arg3) -> (tensor<?x?x?xf32>) {
         %3 = scf.for %arg6 = %c0 to %dim_1 step %c1 iter_args(%arg7 = %arg5) -> (tensor<?x?x?xf32>) {
           // CHECK:  %[[EX1:.*]] = tensor.extract_slice %[[BBARG2]]{{.*}}: tensor<?x?x?xf32> to tensor<1x1x1xf32>
-          // CHECK:  linalg.elemwise_unary {fun = #linalg.unary_fn<abs>} ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX1]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
+          // CHECK:  linalg.elementwise kind=#linalg.elementwise_kind<abs> ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX1]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
           // CHECK:  %[[EX2:.*]] = tensor.extract_slice %[[BBARG2]]{{.*}} : tensor<?x?x?xf32> to tensor<1x1x1xf32>
-          // CHECK:  linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX2]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
+          // CHECK:  linalg.elementwise kind=#linalg.elementwise_kind<exp> ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX2]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
           %extracted_slice = tensor.extract_slice %0[%arg2, %arg4, %arg6] [1, 1, 1] [1, 1, 1] : tensor<?x?x?xf32> to tensor<1x1x1xf32>
           %extracted_slice_2 = tensor.extract_slice %arg7[%arg2, %arg4, %arg6] [1, 1, 1] [1, 1, 1] : tensor<?x?x?xf32> to tensor<1x1x1xf32>
-          %4 = linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins(%extracted_slice : tensor<1x1x1xf32>) outs(%extracted_slice_2 : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
+          %4 = linalg.elementwise kind=#linalg.elementwise_kind<exp> ins(%extracted_slice : tensor<1x1x1xf32>) outs(%extracted_slice_2 : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
           %inserted_slice = tensor.insert_slice %4 into %arg7[%arg2, %arg4, %arg6] [1, 1, 1] [1, 1, 1] : tensor<1x1x1xf32> into tensor<?x?x?xf32>
           scf.yield %inserted_slice : tensor<?x?x?xf32>
         }
@@ -290,7 +290,7 @@ module {
 
   module attributes {transform.with_named_sequence} {
     transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-      %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+      %0 = transform.structured.match ops{["linalg.elementwise"]} in %arg0 : (!transform.any_op) -> !transform.any_op
       %1 = transform.structured.match ops{["scf.for"]} in %arg0 : (!transform.any_op) -> !transform.any_op
       %2:2 = transform.split_handle %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       %3:3 = transform.split_handle %1 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
@@ -340,8 +340,8 @@ module {
       // CHECK: %[[T1:.*]]:2 = linalg.generic {{.*}} ins(%[[T0]]
       %6 = tensor.extract_slice %0#0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary ins(%[[T1]]#0
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.exp ins(%[[T1]]#0
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -376,8 +376,8 @@ module {
       %2 = tensor.extract_slice %0[%i][1][1] : tensor<2xf32> to tensor<1xf32>
       %3 = tensor.extract_slice %arg1[%i][1][1] : tensor<2xf32> to tensor<1xf32>
       // CHECK: %[[FUSED:.+]] = linalg.fill
-      // CHECK: elemwise_unary ins(%[[FUSED]]
-      %4 = linalg.elemwise_unary ins(%2 : tensor<1xf32>) outs(%3 : tensor<1xf32>) -> tensor<1xf32>
+      // CHECK: exp ins(%[[FUSED]]
+      %4 = linalg.exp ins(%2 : tensor<1xf32>) outs(%3 : tensor<1xf32>) -> tensor<1xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %4 into %arg1[%i][1][1] : tensor<1xf32> into tensor<2xf32>
       }
@@ -446,7 +446,7 @@ module {
       // CHECK: %[[T1:.*]]:2 = linalg.generic {{.*}}
       %6 = tensor.extract_slice %0#0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T1]]#0 into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -515,7 +515,7 @@ module {
       // CHECK: %[[T1:.*]] = linalg.generic {{.*}}
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T1]] into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -582,7 +582,7 @@ module {
       // CHECK: %[[T1:.*]] = linalg.generic {{.*}}
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T1]] into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -658,7 +658,7 @@ module {
       // CHECK: %[[T2:.*]] = linalg.generic {{.*}}
       %7 = tensor.extract_slice %1[%4] [%5] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %8 = linalg.elemwise_unary ins(%7 : tensor<?xf32>) outs(%6 : tensor<?xf32>) -> tensor<?xf32>
+      %8 = linalg.exp ins(%7 : tensor<?xf32>) outs(%6 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T2]] into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %8 into %o[%2] [%5] [1] : tensor<?xf32> into tensor<?xf32>
diff --git a/mlir/test/Dialect/Linalg/transform-op-fuse.mlir b/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
index 962858076db93..9a44f95afb586 100644
--- a/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
@@ -5,19 +5,19 @@ func.func @fuse_unary(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<
 
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:    scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
-  %1 = linalg.elemwise_binary ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %1 = linalg.add ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %1 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -31,23 +31,23 @@ func.func @fuse_unary(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<
 
   //     CHECK: %[[PARTIAL_RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: %[[RES:.*]] = scf.for {{.*}}%[[PARTIAL_RES]]
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
-  %1 = linalg.elemwise_binary ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %1 = linalg.add ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %1 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.loop.peel %loops#0 : (!transform.op<"scf.for">) -> (!transform.any_op, !transform.any_op)
@@ -107,20 +107,20 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         %[[RES:.*]] = scf.for
 // CHECK:           scf.for
 // CHECK:             linalg.unpack
-// CHECK:             linalg.elemwise_unary
+// CHECK:             linalg.exp
 // CHECK:         return %[[RES]]
 func.func @unpack_elemwise(%arg0: tensor<16x48x8x8xf32>, %arg1: tensor<128x384xf32>) -> tensor<128x384xf32> {
   %0 = tensor.empty() : tensor<128x384xf32>
   %1 = linalg.unpack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %0
       : tensor<16x48x8x8xf32> -> tensor<128x384xf32>
-  %2 = linalg.elemwise_unary ins(%1: tensor<128x384xf32>)
+  %2 = linalg.exp ins(%1: tensor<128x384xf32>)
                              outs(%arg1: tensor<128x384xf32>) -> tensor<128x384xf32>
   return %2 : tensor<128x384xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.exp"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [16, 32], tile_interchange = [0, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -133,20 +133,20 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         %[[RES:.*]] = scf.for
 // CHECK:           scf.for
 // CHECK:             linalg.pack
-// CHECK:             linalg.elemwise_unary
+// CHECK:             linalg.exp
 // CHECK:         return %[[RES]]
 func.func @pack_elemwise(%arg0: tensor<128x384xf32>, %arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32> {
   %0 = tensor.empty() : tensor<16x48x8x8xf32>
   %1 = linalg.pack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %0
       : tensor<128x384xf32> -> tensor<16x48x8x8xf32>
-  %2 = linalg.elemwise_unary ins(%1: tensor<16x48x8x8xf32>)
+  %2 = linalg.exp ins(%1: tensor<16x48x8x8xf32>)
                              outs(%arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32>
   return %2 : tensor<16x48x8x8xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.exp"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [3, 5, 0, 0]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -159,20 +159,20 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         linalg.pack
 // CHECK:         %[[RES:.*]] = scf.for
 // CHECK:           scf.for
-// CHECK:             linalg.elemwise_unary
+// CHECK:             linalg.exp
 // CHECK:         return %[[RES]]
 func.func @nofuse_pack_elemwise(%arg0: tensor<128x384xf32>, %arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32> {
   %0 = tensor.empty() : tensor<16x48x8x8xf32>
   %1 = linalg.pack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %0
       : tensor<128x384xf32> -> tensor<16x48x8x8xf32>
-  %2 = linalg.elemwise_unary ins(%1: tensor<16x48x8x8xf32>)
+  %2 = linalg.exp ins(%1: tensor<16x48x8x8xf32>)
                              outs(%arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32>
   return %2 : tensor<16x48x8x8xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.exp"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:3 = transform.structured.fuse %0 {tile_sizes = [3, 5, 2, 0]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -186,24 +186,24 @@ func.func @fuse_through_slice(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) ->
 
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg0: tensor<?x?xf32>) -> tensor<?x?xf32>
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
   %dim0 = tensor.dim %arg1, %c0 : tensor<?x?xf32>
   %dim1 = tensor.dim %arg1, %c1 : tensor<?x?xf32>
   %1 = tensor.extract_slice %0 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
-  %2 = linalg.elemwise_binary ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %2 = linalg.add ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %2 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1], apply_cleanup = true}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.yield
@@ -217,10 +217,10 @@ func.func @fuse_through_slice_and_cast_chain(%arg0: tensor<100x100xf32>, %arg1:
 
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<100x100xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<100x100xf32>)
                              outs(%arg0: tensor<100x100xf32>) -> tensor<100x100xf32>
   %1 = tensor.cast %0 : tensor<100x100xf32> to tensor<100x?xf32>
   %2 = tensor.extract_slice %1 [1, 1] [98, 98] [1, 1] : tensor<100x?xf32> to tensor<98x98xf32>
@@ -230,14 +230,14 @@ func.func @fuse_through_slice_and_cast_chain(%arg0: tensor<100x100xf32>, %arg1:
   %dim0 = tensor.dim %arg1, %c0 : tensor<?x?xf32>
   %dim1 = tensor.dim %arg1, %c1 : tensor<?x?xf32>
   %4 = tensor.extract_slice %3 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
-  %5 = linalg.elemwise_binary ins(%4, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %5 = linalg.add ins(%4, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %5 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1], apply_cleanup = true}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.yield
@@ -253,8 +253,8 @@ func.func @fuse_unrelated_slices(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>)
   //     CHECK: %[[SLICE2:.+]] = tensor.extract_slice %[[SLICE1]]
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]], %[[SLICE2]]
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -262,17 +262,17 @@ func.func @fuse_unrelated_slices(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>)
   %dim1 = tensor.dim %arg1, %c1 : tensor<?x?xf32>
   %slice1 = tensor.extract_slice %arg0 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
   %slice2 = tensor.extract_slice %slice1 [1, 1] [10, 10] [1, 1] : tensor<?x?xf32> to tensor<10x10xf32>
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg0: tensor<?x?xf32>) -> tensor<?x?xf32>
   %1 = tensor.extract_slice %0 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
-  %2 = linalg.elemwise_binary ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %2 = linalg.add ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %2, %slice2 : tensor<?x?xf32>, tensor<10x10xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1], apply_cleanup = true}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.yield
diff --git a/mlir/test/Dialect/Linalg/transform-op-generalize.mlir b/mlir/test/Dialect/Linalg/transform-op-generalize.mlir
index a0aa33c072dd4..331c9c0fbbfd5 100644
--- a/mlir/test/Dialect/Linalg/transform-op-generalize.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-generalize.mlir
@@ -3,9 +3,9 @@
 // CHECK-LABEL: func.func @generalize_unary
 func.func @generalize_unary(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
 
-  // CHECK-NOT:   linalg.elemwise_unary
+  // CHECK-NOT:   linalg.exp
   //     CHECK:   linalg.generic
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
 }
diff --git a/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir
index f62e257f80016..c3ee8929dc3f3 100644
--- a/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir
@@ -31,9 +31,9 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic(
 // CHECK-SAME: %[[ARG1:.*]]: tensor<1x1x3xf32>
 
 // CHECK-DAG:  %[[C0:.+]] = arith.constant 0 : index
-// CHECK-DAG:  %[[CST:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:  %[[PV:.+]] = ub.poison : f32
 
-// CHECK:   %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true, true]} : tensor<3x3x3xf32>, vector<1x1x3xf32>
+// CHECK:   %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true]} : tensor<3x3x3xf32>, vector<1x1x3xf32>
 // CHECK:   vector.transfer_write %[[READ]], %[[ARG1]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32>
 
  // -----
@@ -64,12 +64,12 @@ func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16
 // CHECK-SAME:      %[[ARG5:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
 
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[C79:.*]] = arith.constant 79 : index
 // CHECK:           %[[ADD1:.*]] = arith.addi %[[ARG1]], %[[ARG2]] : index
 // CHECK:           %[[ADD2:.*]] = arith.addi %[[ARG3]], %[[ARG4]] : index
 
-// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[ADD1]], %[[C79]], %[[ADD2]]], %[[CST]] {in_bounds = [true, true]} : tensor<45x80x16xf32>, vector<1x4xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[ADD1]], %[[C79]], %[[ADD2]]], %[[PV]] {in_bounds = [true, true]} : tensor<45x80x16xf32>, vector<1x4xf32>
 // CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG5]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
 // CHECK:           return %[[WRITE]] : tensor<1x4xf32>
 // CHECK:         }
@@ -97,11 +97,11 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<8
 // CHECK-SAME:      %[[ARG1:.*]]: index,
 // CHECK-SAME:      %[[ARG2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
 
-// CHECK-DAG:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C79:.*]] = arith.constant 79 : index
 
-// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[C79]], %[[ARG1]]], %[[CST]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[C79]], %[[ARG1]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
 // CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG2]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
 // CHECK:           return %[[WRITE]] : tensor<1x4xf32>
 // CHECK:         }
@@ -164,9 +164,9 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16
 // CHECK-SAME:                                                                 %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C16:.*]] = arith.constant 16 : index
-// CHECK-DAG:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
 
-// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[C16]], %[[C0]]], %[[CST]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[C16]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
 // CHECK:           %[[VAL_9:.*]] = vector.transfer_write %[[VAL_8]], %[[VAL_1]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
 // CHECK:           return %[[VAL_9]] : tensor<1x4xf32>
 // CHECK:         }
@@ -229,12 +229,12 @@ func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>,
 // CHECK-SAME: %[[ARG3:.*]]: tensor<4x7x2xf32>
 // CHECK-SAME: %[[ARG4:.*]]: tensor<4x7x3x2xf32>
 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG: %[[C0_i32:.*]] = arith.constant 0 : i32
+// CHECK-DAG: %[[PV:.*]] = ub.poison : i32
 // CHECK-DAG: %[[CST:.*]] = arith.constant dense<3> : vector<7x2x4x3xindex>
 // CHECK-DAG: %[[CST_1:.*]] = arith.constant dense<true> : vector<4x7x3x2xi1>
 // CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32>
-// CHECK:    %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[C0_i32]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
-// CHECK:    %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[C0_i32]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
+// CHECK:    %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
+// CHECK:    %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
 // CHECK:    %[[CAST:.*]] = arith.index_cast %[[V0]] : vector<4x3xi32> to vector<4x3xindex>
 // CHECK:    %[[B1:.*]] = vector.broadcast %[[CAST]] : vector<4x3xindex> to vector<7x2x4x3xindex>
 // CHECK:    %[[CAST_1:.*]] = arith.index_cast %[[V1]] : vector<4x3xi32> to vector<4x3xindex>
@@ -382,7 +382,7 @@ func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32
 // CHECK-SAME:                    %[[VAL_0:.*]]: tensor<6xf32>
 // CHECK-SAME:                    %[[VAL_1:.*]]: tensor<5xi32>
 // CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[VAL_3:.*]] = ub.poison : i32
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant dense<0> : vector<5xindex>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant dense<5> : vector<5xindex>
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant dense<true> : vector<5xi1>
@@ -480,13 +480,14 @@ func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1:
 // CHECK-LABEL:   func.func @vectorize_nd_tensor_extract_block_arg(
 // CHECK-SAME:                                                     %[[VAL_0:.*]]: tensor<5x6xf32>,
 // CHECK-SAME:                                                     %[[VAL_1:.*]]: tensor<5xindex>) -> tensor<5xf32> {
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : index
 // CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4]> : vector<5xindex>
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant dense<true> : vector<5xi1>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32>
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant dense<6> : vector<5xindex>
 // CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<5xf32>
-// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[VAL_2]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[PAD]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex>
 // CHECK:           %[[VAL_9:.*]] = arith.muli %[[VAL_8]], %[[VAL_6]] : vector<5xindex>
 // CHECK:           %[[VAL_10:.*]] = arith.addi %[[VAL_9]], %[[VAL_3]] : vector<5xindex>
 // CHECK:           %[[VAL_11:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]]] {{\[}}%[[VAL_10]]], %[[VAL_4]], %[[VAL_5]] : tensor<5x6xf32>, vector<5xindex>, vector<5xi1>, vector<5xf32> into vector<5xf32>
@@ -559,7 +560,7 @@ func.func @vectorize_nd_tensor_extract_scalar_broadcast(%src: tensor<3x3xf32>, %
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[C2:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C1]], %[[C2]]], %[[PAD]] : tensor<3x3xf32>, vector<f32>
 // CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1x1x3xf32>
 // CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32>
@@ -583,7 +584,7 @@ func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>)
 // CHECK-LABEL:   func.func @extract_scalar_from_0d_into_0d(
 // CHECK-SAME:      %[[SRC:.*]]: tensor<f32>,
 // CHECK-SAME:      %[[INIT:.*]]: tensor<f32>) -> tensor<f32> {
-// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
 // CHECK:           vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32>
 
@@ -606,7 +607,7 @@ func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32
 // CHECK-SAME:      %[[SRC:.*]]: tensor<f32>,
 // CHECK-SAME:      %[[INIT:.*]]: tensor<1xf32>) -> tensor<1xf32> {
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
 // CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32>
 // CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32>
@@ -654,7 +655,7 @@ func.func @scalar_read_with_broadcast_from_column_tensor(%init: tensor<1x1x4xi32
 
 // CHECK-LABEL:   func.func @scalar_read_with_broadcast_from_column_tensor
 // CHECK-SAME:      %[[INIT:.*]]: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
-// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : i32
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[SRC:.*]] = arith.constant dense<{{\[\[}}0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[C0]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32>
@@ -688,7 +689,7 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic_column(
 // CHECK-SAME:      %[[SRC:.*]]: tensor<3x3x3xf32>,
 // CHECK-SAME:      %[[INIT:.*]]: tensor<3x1x1xf32>)
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C0]], %[[C0]], %[[C0]]], %[[CST_0]] : tensor<3x3x3xf32>, vector<f32>
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] : tensor<3x3x3xf32>, vector<f32>
 // CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<3x1x1xf32>
 // CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<3x1x1xf32>, tensor<3x1x1xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization/extract.mlir b/mlir/test/Dialect/Linalg/vectorization/extract.mlir
index d0d3b58a05704..76ac4b8398069 100644
--- a/mlir/test/Dialect/Linalg/vectorization/extract.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/extract.mlir
@@ -299,7 +299,7 @@ func.func @masked_dynamic_vectorize_nd_tensor_extract_with_affine_apply_gather(%
 // CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_7:.*]] = tensor.dim %[[VAL_2]], %[[VAL_6]] : tensor<?x?xf32>
 // CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_7]] : vector<1x4xi1>
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_10]] { vector.transfer_read %[[VAL_2]]{{\[}}%[[VAL_8]], %[[VAL_8]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<1x4xf32> } : vector<1x4xi1> -> vector<1x4xf32>
 // CHECK:           %[[VAL_12:.*]] = vector.step : vector<4xindex>
@@ -356,7 +356,7 @@ func.func @extract_masked_vectorize(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf3
 // CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_7:.*]] = tensor.dim %[[VAL_1]], %[[VAL_6]] : tensor<?x?xf32>
 // CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_7]] : vector<3x3xi1>
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_10]] { vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_8]], %[[VAL_8]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<3x3xf32> } : vector<3x3xi1> -> vector<3x3xf32>
 // CHECK:           %[[VAL_12:.*]] = arith.constant dense<true> : vector<3x3xi1>
@@ -458,7 +458,7 @@ func.func @scalar_broadcast(%init : tensor<1x1x3xi32>, %src: tensor<1x3x2x4xi32>
 // CHECK:           %[[MASK_RES:.*]] = vector.create_mask %[[C1]], %[[C1_2]], %[[C3]] : vector<1x1x4xi1>
 
 /// Read and broadcast the scalar
-// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[PAD:.*]] = ub.poison : i32
 // CHECK:           %[[MASK_READ:.*]] = vector.constant_mask [1] : vector<1xi1>
 // CHECK:           %[[READ:.*]] = vector.mask %[[MASK_READ]] {
 // CHECK-SAME:          vector.transfer_read %[[SRC]]{{\[}}%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]],  %[[PAD]]
diff --git a/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
index b282c57e3e4cb..4eeae4c064519 100644
--- a/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
@@ -161,7 +161,7 @@ module attributes {transform.with_named_sequence} {
 // CHECK-SAME:     %[[ARG_0:.*]]: tensor<f32>, %[[ARG_1:.*]]: tensor<f32>, %[[ARG_2:.*]]: tensor<f32>)
 func.func @generic_0d(%arg0: tensor<f32>, %arg1: tensor<f32>,
                       %arg2: tensor<f32>) -> tensor<f32> {
-// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ_0:.*]] = vector.transfer_read %[[ARG_0]][], %[[PAD]] : tensor<f32>, vector<f32>
 // CHECK:           %[[ARG_0_AS_SCALAR:.*]] = vector.extract %[[READ_0]][] : f32 from vector<f32>
 // CHECK:           %[[READ_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[PAD]] : tensor<f32>, vector<f32>
@@ -770,11 +770,11 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG: #[[$MAP3:.*]] = affine_map<(d0, d1) -> (d1, 0, d0, 0)>
 //     CHECK: func @generic_vectorize_broadcast_transpose
 // CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:   %[[CF:.*]] = arith.constant 0.000000e+00 : f32
-//     CHECK:   %[[V0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP0]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
-//     CHECK:   %[[V1:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP1]]} : memref<4xf32>, vector<4x4x4x4xf32>
-//     CHECK:   %[[V2:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP2]]} : memref<4xf32>, vector<4x4x4x4xf32>
-//     CHECK:   %[[V3:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP3]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
+// CHECK-DAG:   %[[PV:.*]] = ub.poison : f32
+//     CHECK:   %[[V0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP0]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
+//     CHECK:   %[[V1:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP1]]} : memref<4xf32>, vector<4x4x4x4xf32>
+//     CHECK:   %[[V2:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP2]]} : memref<4xf32>, vector<4x4x4x4xf32>
+//     CHECK:   %[[V3:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP3]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
 //     CHECK:   %[[SUB:.*]] = arith.subf %[[V0]], %[[V1]] : vector<4x4x4x4xf32>
 //     CHECK:   %[[ADD0:.*]] = arith.addf %[[V2]], %[[SUB]] : vector<4x4x4x4xf32>
 //     CHECK:   %[[ADD1:.*]] = arith.addf %[[V3]], %[[ADD0]] : vector<4x4x4x4xf32>
@@ -1702,7 +1702,7 @@ module attributes {transform.with_named_sequence} {
 
 // CHECK-LABEL:   func.func @generic_with_reduction_and_broadcast(
 // CHECK-SAME:                                                    %[[VAL_0:.*]]: tensor<1x12x197x197xf32>) -> tensor<1x12x197x1xf32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_3:.*]] = tensor.empty() : tensor<1x12x197x1xf32>
 // CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]]], %[[VAL_1]] {in_bounds = [true, true, true, true]} : tensor<1x12x197x197xf32>, vector<1x12x197x197xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir b/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
index 6722de817f6bf..9e501affdd2a5 100644
--- a/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
@@ -211,10 +211,10 @@ func.func @dynamic_generic_with_reduction_and_broadcast(%arg0: tensor<?x?xf32>,
 // CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf32>
 // CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_7:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]] : vector<4x4xi1>
 // CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_7]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x4xf32> } : vector<4x4xi1> -> vector<4x4xf32>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_10:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_11:.*]] = vector.create_mask %[[VAL_3]] : vector<4xi1>
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_11]] { vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_10]] {in_bounds = [true], permutation_map = #[[$MAP]]} : tensor<?x?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.multi_reduction <add>, %[[VAL_9]], %[[VAL_12]] [1] : vector<4x4xf32> to vector<4xf32> } : vector<4x4xi1> -> vector<4xf32>
@@ -257,7 +257,7 @@ func.func @vectorize_dynamic_2d_transpose(%arg0: tensor<?x?xf32>,
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : tensor<?x?xf32>, vector<4x8xf32> } : vector<8x4xi1> -> vector<4x8xf32>
 // CHECK:           %[[VAL_12:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]] : vector<4x8xi1>
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
-// CHECK:           %[[VAL_14:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_14:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_15:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
 // CHECK:           %[[VAL_16:.*]] = arith.addf %[[VAL_10]], %[[VAL_13]] : vector<4x8xf32>
 // CHECK:           %[[VAL_17:.*]] = vector.mask %[[VAL_12]] { vector.transfer_write %[[VAL_16]], %{{.*}} {in_bounds = [true, true]} : vector<4x8xf32>, tensor<?x?xf32> } : vector<4x8xi1> -> tensor<?x?xf32>
@@ -372,12 +372,12 @@ func.func @vectorize_dynamic_reduction_2d_scalable(%arg0: tensor<?x?xf32>,
 // CHECK:    %[[C1_IDX:.*]] = arith.constant 1 : index
 // CHECK:    %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_IDX]] : tensor<?x?xf32>
 // CHECK:    %[[C0_IDX:.*]] = arith.constant 0 : index
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_2D:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<4x[8]xi1>
-// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2D]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]], %[[C0_IDX]]], %[[C0_f32]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[8]xf32> } : vector<4x[8]xi1> -> vector<4x[8]xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2D]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]], %[[C0_IDX]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[8]xf32> } : vector<4x[8]xi1> -> vector<4x[8]xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_1D:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<4xi1>
-// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_1D]] { vector.transfer_read %[[ARG_1]][%[[C0_IDX]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_1D]] { vector.transfer_read %[[ARG_1]][%[[C0_IDX]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:    %[[REDUCE:.*]] = vector.mask %[[MASK_2D]] { vector.multi_reduction <add>, %[[VEC_RD_0]], %[[VEC_RD_1]] [1] : vector<4x[8]xf32> to vector<4xf32> } : vector<4x[8]xi1> -> vector<4xf32>
 // CHECK:    %[[C0_IDX:.*]] = arith.constant 0 : index
 // CHECK:    %{{.*}} = vector.mask %[[MASK_1D]] { vector.transfer_write %[[REDUCE]], %[[ARG_1]][%[[C0_IDX]]] {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
@@ -408,11 +408,11 @@ func.func @vectorize_dynamic_reduction_scalable_1d(%arg0: tensor<?xf32>,
 // CHECK:          %[[C0_IDX:.*]] = arith.constant 0 : index
 // CHECK:          %[[DIM_A0_0:.*]] = tensor.dim %[[ARG_0]], %[[C0_IDX]] : tensor<?xf32>
 // CHECK:          %[[C0_IDX:.*]] = arith.constant 0 : index
-// CHECK:          %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:          %[[PV:.*]] = ub.poison : f32
 // CHECK:          %[[MASK:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<[4]xi1>
-// CHECK:          %[[VEC_RD_0:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
-// CHECK:          %[[C0_F32:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:          %[[VEC_RD_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[C0_F32]] : tensor<f32>, vector<f32>
+// CHECK:          %[[VEC_RD_0:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
+// CHECK:          %[[PV:.*]] = ub.poison : f32
+// CHECK:          %[[VEC_RD_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[PV]] : tensor<f32>, vector<f32>
 // CHECK:          %[[ACC_f32:.*]] = vector.extract %[[VEC_RD_1]][] : f32 from vector<f32>
 // CHECK:          %[[REDUCE:.*]] = vector.mask %[[MASK]] { vector.multi_reduction <add>, %[[VEC_RD_0]], %[[ACC_f32]] [0] : vector<[4]xf32> to f32 } : vector<[4]xi1> -> f32
 // CHECK:          %[[VEC_f32:.*]] = vector.broadcast %[[REDUCE]] : f32 to vector<f32>
@@ -532,13 +532,13 @@ func.func @vectorize_partial_dynamic_identity(%arg0: tensor<8x?xf32>,
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_6:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 8 : index
 // CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x32xi1>
 // CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_10:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_12:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_12:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x32xf32>
 // CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
@@ -576,13 +576,13 @@ func.func @vectorize_partial_dynamic_identity_scalable(%arg0: tensor<8x?xf32>,
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_6:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 8 : index
 // CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x[32]xi1>
 // CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_10:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_12:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_12:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x[32]xf32>
 // CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
@@ -647,14 +647,14 @@ func.func @vectorize_static_shape_with_mask(%arg0: tensor<8x30xf32>,
 // CHECK-LABEL:   func.func @vectorize_static_shape_with_mask(
 // CHECK-SAME:      %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> {
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_4:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 8 : index
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 30 : index
 // CHECK:           %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x32xi1>
 // CHECK:           %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_11:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_11:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
 // CHECK:           %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x32xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.constant 0 : index
@@ -689,14 +689,14 @@ func.func @vectorize_static_shape_with_mask_scalable(%arg0: tensor<8x30xf32>,
 // CHECK-LABEL:   func.func @vectorize_static_shape_with_mask_scalable(
 // CHECK-SAME:      %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> {
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_4:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 8 : index
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 30 : index
 // CHECK:           %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x[32]xi1>
 // CHECK:           %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_11:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_11:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
 // CHECK:           %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x[32]xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.constant 0 : index
@@ -733,15 +733,15 @@ func.func @vectorize_dynamic_matvec_trailing_reduction_dim(%arg0: tensor<?x?xf32
 // CHECK:    %[[C1_idx:.*]] = arith.constant 1 : index
 // CHECK:    %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_idx]] : tensor<?x?xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_2d:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<4x[4]xi1>
-// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[4]xf32> } : vector<4x[4]xi1> -> vector<4x[4]xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[4]xf32> } : vector<4x[4]xi1> -> vector<4x[4]xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d1:.*]] = vector.create_mask %[[DIM_A0_1]] : vector<[4]xi1>
-// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<4x[4]xf32> } : vector<[4]xi1> -> vector<4x[4]xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[PV]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<4x[4]xf32> } : vector<[4]xi1> -> vector<4x[4]xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d2:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<4xi1>
-// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:    %[[MUL:.*]] = arith.mulf %[[VEC_RD_0:.*]], %[[VEC_RD_1:.*]] : vector<4x[4]xf32>
 // CHECK:    %[[REDUCE:.*]] = vector.mask %[[MASK_2d]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_RD_2]] [1] : vector<4x[4]xf32> to vector<4xf32> } : vector<4x[4]xi1> -> vector<4xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
@@ -776,15 +776,15 @@ func.func @vectorize_dynamic_matvec_trailing_reduction_dim(%arg0: tensor<?x?xf32
 // CHECK:    %[[C1_idx:.*]] = arith.constant 1 : index
 // CHECK:    %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_idx]] : tensor<?x?xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_2d:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<[4]x4xi1>
-// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<[4]x4xf32> } : vector<[4]x4xi1> -> vector<[4]x4xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<[4]x4xf32> } : vector<[4]x4xi1> -> vector<[4]x4xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d1:.*]] = vector.create_mask %[[DIM_A0_1]] : vector<4xi1>
-// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<[4]x4xf32> } : vector<4xi1> -> vector<[4]x4xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[PV]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<[4]x4xf32> } : vector<4xi1> -> vector<[4]x4xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d2:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<[4]xi1>
-// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
+// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
 // CHECK:    %[[MUL:.*]] = arith.mulf %[[VEC_RD_0:.*]], %[[VEC_RD_1:.*]] : vector<[4]x4xf32>
 // CHECK:    %[[REDUCE:.*]] = vector.mask %[[MASK_2d]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_RD_2]] [1] : vector<[4]x4xf32> to vector<[4]xf32> } : vector<[4]x4xi1> -> vector<[4]xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
@@ -822,10 +822,10 @@ func.func @vectorize_linalg_index_scalable(%dest: tensor<?xindex>) -> tensor<?xi
 // CHECK:           %[[C0:.*]] = arith.constant 0 : index
 // CHECK:           %[[D0:.*]] = tensor.dim %[[DEST]], %[[C0]] : tensor<?xindex>
 // CHECK:           %[[C0_1:.*]] = arith.constant 0 : index
-// CHECK:           %[[C0_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[PV:.*]] = ub.poison : index
 // CHECK:           %[[MASK:.*]] = vector.create_mask %[[D0]] : vector<[4]xi1>
 // TODO: This xfer_read is not used - avoid creating it.
-// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{\[}}%[[C0_1]]], %[[C0_2]] {in_bounds = [true]} : tensor<?xindex>, vector<[4]xindex> } : vector<[4]xi1> -> vector<[4]xindex>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{\[}}%[[C0_1]]], %[[PV]] {in_bounds = [true]} : tensor<?xindex>, vector<[4]xindex> } : vector<[4]xi1> -> vector<[4]xindex>
 // CHECK:           %[[STEP:.*]] = vector.step : vector<[4]xindex>
 // CHECK:           %[[C0_3:.*]] = arith.constant 0 : index
 // CHECK:           %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[STEP]], %[[DEST]]{{\[}}%[[C0_3]]] {in_bounds = [true]} : vector<[4]xindex>, tensor<?xindex> } : vector<[4]xi1> -> tensor<?xindex>
diff --git a/mlir/test/Dialect/MemRef/canonicalize.mlir b/mlir/test/Dialect/MemRef/canonicalize.mlir
index a91e54a126100..16b7a5c8bcb08 100644
--- a/mlir/test/Dialect/MemRef/canonicalize.mlir
+++ b/mlir/test/Dialect/MemRef/canonicalize.mlir
@@ -1208,3 +1208,24 @@ func.func @fold_assume_alignment_chain(%0: memref<128xf32>) -> memref<128xf32> {
   // CHECK: return %[[ALIGN]]
   return %2 : memref<128xf32>
 }
+
+// -----
+
+// CHECK-LABEL: func @fold_view_same_source_result_types
+func.func @fold_view_same_source_result_types(%0: memref<128xi8>) -> memref<128xi8> {
+  %c0 = arith.constant 0: index
+  // CHECK-NOT: memref.view
+  %res = memref.view %0[%c0][] : memref<128xi8> to memref<128xi8>
+  return %res : memref<128xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @non_fold_view_same_source_res_types
+//  CHECK-SAME:   %[[ARG0:[a-zA-Z0-9]+]]
+func.func @non_fold_view_same_source_res_types(%0: memref<?xi8>, %arg0 : index) -> memref<?xi8> {
+  %c0 = arith.constant 0: index
+  // CHECK: memref.view
+  %res = memref.view %0[%c0][%arg0] : memref<?xi8> to memref<?xi8>
+  return %res : memref<?xi8>
+}
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 060b3cd2455a0..7608ad57c7967 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -480,6 +480,39 @@ func.func @omp_simd_pretty_simdlen_safelen(%lb : index, %ub : index, %step : ind
 
 // -----
 
+func.func @omp_simd_bad_privatizer(%lb : index, %ub : index, %step : index) {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+  // expected-error @below {{Cannot find privatizer '@not_defined'}}
+  omp.simd private(@not_defined %1 -> %arg0 : !llvm.ptr) {
+    omp.loop_nest (%arg2) : index = (%lb) to (%ub) inclusive step (%step) {
+      omp.yield
+    }
+  }
+}
+
+// -----
+
+omp.private {type = firstprivate} @_QFEp_firstprivate_i32 : i32 copy {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  %0 = llvm.load %arg0 : !llvm.ptr -> i32
+  llvm.store %0, %arg1 : i32, !llvm.ptr
+  omp.yield(%arg1 : !llvm.ptr)
+}
+func.func @omp_simd_firstprivate(%lb : index, %ub : index, %step : index) {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+  // expected-error @below {{FIRSTPRIVATE cannot be used with SIMD}}
+  omp.simd private(@_QFEp_firstprivate_i32 %1 -> %arg0 : !llvm.ptr) {
+    omp.loop_nest (%arg2) : index = (%lb) to (%ub) inclusive step (%step) {
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 // expected-error @below {{op expects alloc region to yield a value of the reduction type}}
 omp.declare_reduction @add_f32 : f32
 alloc {
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
index 5e32a3a78c032..8ba8013d008a0 100644
--- a/mlir/test/Dialect/SCF/canonicalize.mlir
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -1811,7 +1811,7 @@ module {
       %4 = affine.min #map2(%arg3)[%dim, %arg0]
       %extracted_slice0 = tensor.extract_slice %arg4[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
       %extracted_slice1 = tensor.extract_slice %arg5[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
-      %5 = linalg.elemwise_unary ins(%extracted_slice0 : tensor<?xf32>) outs(%extracted_slice1 : tensor<?xf32>) -> tensor<?xf32>
+      %5 = linalg.exp ins(%extracted_slice0 : tensor<?xf32>) outs(%extracted_slice1 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %5 into %arg5[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -1825,7 +1825,7 @@ module {
 //  CHECK-SAME:                       shared_outs(%[[ITER_ARG_5:.*]] = %[[ARG2]]) -> (tensor<?xf32>) {
 //       CHECK:      %[[OPERAND0:.*]] = tensor.extract_slice %[[ARG1]]
 //       CHECK:      %[[OPERAND1:.*]] = tensor.extract_slice %[[ITER_ARG_5]]
-//       CHECK:      %[[ELEM:.*]] = linalg.elemwise_unary ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
+//       CHECK:      %[[ELEM:.*]] = linalg.exp ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
 //       CHECK:      scf.forall.in_parallel {
 //  CHECK-NEXT:         tensor.parallel_insert_slice %[[ELEM]] into %[[ITER_ARG_5]]
 //  CHECK-NEXT:      }
@@ -1851,7 +1851,7 @@ module {
       %extracted_slice_0 = tensor.extract_slice %arg6[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
       %extracted_slice_1 = tensor.extract_slice %arg7[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
       %extracted_slice_2 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
-      %5 = linalg.elemwise_unary ins(%extracted_slice : tensor<?xf32>) outs(%extracted_slice_1 : tensor<?xf32>) -> tensor<?xf32>
+      %5 = linalg.exp ins(%extracted_slice : tensor<?xf32>) outs(%extracted_slice_1 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %5 into %arg6[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %extracted_slice into %arg5[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -1868,7 +1868,7 @@ module {
 //  CHECK-SAME:                       shared_outs(%[[ITER_ARG_6:.*]] = %[[ARG2]]) -> (tensor<?xf32>) {
 //       CHECK:      %[[OPERAND0:.*]] = tensor.extract_slice %[[ARG1]]
 //       CHECK:      %[[OPERAND1:.*]] = tensor.extract_slice %[[ARG3]]
-//       CHECK:      %[[ELEM:.*]] = linalg.elemwise_unary ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
+//       CHECK:      %[[ELEM:.*]] = linalg.exp ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
 //       CHECK:      scf.forall.in_parallel {
 //  CHECK-NEXT:         tensor.parallel_insert_slice %[[ELEM]] into %[[ITER_ARG_6]]
 //  CHECK-NEXT:      }
diff --git a/mlir/test/Dialect/SCF/invalid.mlir b/mlir/test/Dialect/SCF/invalid.mlir
index b944852ceba3f..bb7958083e55c 100644
--- a/mlir/test/Dialect/SCF/invalid.mlir
+++ b/mlir/test/Dialect/SCF/invalid.mlir
@@ -684,6 +684,24 @@ func.func @forall_wrong_terminator_op() -> () {
 
 // -----
 
+func.func @at_most_one_masking_attribute(%in: tensor<100xf32>, %out: tensor<100xf32>) {
+  %c1 = arith.constant 1 : index
+  %num_threads = arith.constant 100 : index
+
+  // expected-error @below {{"mapping" supports at most one device masking attribute}}
+  %result = scf.forall (%thread_idx) in (%num_threads) shared_outs(%o = %out) -> (tensor<100xf32>) {
+      %1 = tensor.extract_slice %in[%thread_idx][1][1] : tensor<100xf32> to tensor<1xf32>
+      scf.forall.in_parallel {
+        tensor.parallel_insert_slice %1 into %o[%thread_idx][1][1] :
+          tensor<1xf32> into tensor<100xf32>
+      }
+  }  { mapping = [#gpu.thread<x>, #gpu.mask<0x1>, #gpu.mask<0x2>] }
+
+  return
+}
+
+// -----
+
 func.func @switch_wrong_case_count(%arg0: index) {
   // expected-error @below {{'scf.index_switch' op has 0 case regions but 1 case values}}
   "scf.index_switch"(%arg0) ({
diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir
index 25e5703cea5c6..86af637fc05d7 100644
--- a/mlir/test/Dialect/SCF/loop-pipelining.mlir
+++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir
@@ -34,6 +34,42 @@ func.func @simple_pipeline(%A: memref<?xf32>, %result: memref<?xf32>) {
   return
 }
 
+// -----
+
+// A static loop does not satisfy `numIteration >= maxStage`
+
+// CHECK-LABEL: func.func @iteration_lt_stage(
+//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+//   CHECK-DAG:   %[[C_NEG1:.*]] = arith.constant -1 : index
+//   CHECK-DAG:   %[[TRUE:.*]] = arith.constant true
+//   CHECK-DAG:   %[[FALSE:.*]] = arith.constant false
+// Prologue:
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[FALSE]]
+// Kernel:
+//       CHECK:   scf.for %[[IV:.*]] = %[[C0:.*]] to %[[C_NEG1:.*]] step %[[C1:.*]]
+// Epilogue:
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[FALSE]]
+//       CHECK:   scf.if %[[FALSE]]
+func.func @iteration_lt_stage(%A: memref<?xf32>, %result: memref<?xf32>) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  %cf = arith.constant 1.0 : f32
+  scf.for %i0 = %c0 to %c2 step %c1 {
+    %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>
+    %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 0 } : f32
+    memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 1 } : memref<?xf32>
+  }  { __test_pipelining_loop__ }
+  return
+}
+
 
 // -----
 
diff --git a/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir b/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
index 8733ff93768ab..61a35b7c991ba 100644
--- a/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
@@ -58,6 +58,15 @@ spirv.func @cooperative_matrix_load_stride_i16(%ptr : !spirv.ptr<i32, StorageBuf
   spirv.Return
 }
 
+// CHECK-LABEL: @cooperative_matrix_load_aligned
+spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
+  // CHECK:      {{%.*}} = spirv.KHR.CooperativeMatrixLoad {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16 :
+  // CHECK-SAME:   !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+  %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <RowMajor>, <Aligned>, 16 :
+    !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+  spirv.Return
+}
+
 // CHECK-LABEL: @cooperative_matrix_store
 spirv.func @cooperative_matrix_store(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
                                      %m : !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>) "None" {
@@ -90,6 +99,16 @@ spirv.func @cooperative_matrix_store_stride_i16(%ptr : !spirv.ptr<i32, StorageBu
   spirv.Return
 }
 
+// CHECK-LABEL: @cooperative_matrix_store_aligned
+spirv.func @cooperative_matrix_store_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
+                                     %m : !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>) "None" {
+  // CHECK:      spirv.KHR.CooperativeMatrixStore {{%.*}}, {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16 :
+  // CHECK-SAME:   !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>, i32
+  spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Aligned>, 16 :
+    !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>, i32
+  spirv.Return
+}
+
 // -----
 
 spirv.func @cooperative_matrix_load_bad_ptr(%ptr : !spirv.ptr<!spirv.struct<(f32 [0])>, StorageBuffer>, %stride : i32) "None" {
@@ -120,7 +139,7 @@ spirv.func @cooperative_matrix_load_bad_operad(%ptr : !spirv.ptr<i32, StorageBuf
 // -----
 
 spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
-  // expected-error @+1 {{op has unhandled memory operand 'Aligned'}}
+  // expected-error @+1 {{missing value for the 'Aligned' memory operand}}
   %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <ColumnMajor>, <Aligned> :
     !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<8x16xi32, Subgroup, MatrixA>
   spirv.Return
@@ -129,7 +148,7 @@ spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer
 // -----
 
 spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
-  // expected-error @+1 {{op has unhandled memory operand 'Aligned'}}
+  // expected-error @+1 {{missing value for the 'Aligned' memory operand}}
   %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <ColumnMajor>, <Volatile|Aligned> :
     !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<8x16xi32, Subgroup, MatrixA>
   spirv.Return
@@ -179,7 +198,7 @@ spirv.func @cooperative_matrix_store_bad_operand(%ptr : !spirv.ptr<i32, StorageB
 
 spirv.func @cooperative_matrix_store(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
                                      %m : !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>) "None" {
-  // expected-error @+1 {{op has unhandled memory operand 'Aligned'}}
+  // expected-error @+1 {{missing value for the 'Aligned' memory operand}}
   spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Aligned> :
     !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>, i32
   spirv.Return
@@ -187,6 +206,15 @@ spirv.func @cooperative_matrix_store(%ptr : !spirv.ptr<i32, StorageBuffer>, %str
 
 // -----
 
+spirv.func @cooperative_matrix_store_bad_operand_arg(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
+  // expected-error @+1 {{found alignment attribute for non-'Aligned' memory operand}}
+  %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <RowMajor>, <MakePointerVisible>, 16 :
+    !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
+
 spirv.func @cooperative_matrix_muladd(%a : !spirv.coopmatrix<8x16xi32, Subgroup, MatrixA>,
                                       %b : !spirv.coopmatrix<16x4xi32, Subgroup, MatrixB>,
                                       %c : !spirv.coopmatrix<8x4xi32, Subgroup, MatrixAcc>) "None" {
@@ -549,3 +577,46 @@ spirv.func @matrix_times_scalar(%a: !spirv.coopmatrix<2x2xf32, Workgroup, Matrix
   %p = spirv.MatrixTimesScalar %a, %b : !spirv.coopmatrix<2x2xf32, Workgroup, MatrixA>, f16
   spirv.Return
 }
+
+// -----
+
+// These binary arithmetic instructions do not support coop matrix operands.
+
+spirv.func @fmod(%a: !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>, %b: !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>) "None" {
+  // expected-error @+1 {{op operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values of length 2/3/4/8/16}}
+  %p = spirv.FMod %a, %b : !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
+
+spirv.func @frem(%a: !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>, %b: !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>) "None" {
+  // expected-error @+1 {{op operand #0 must be 16/32/64-bit float or vector of 16/32/64-bit float values of length 2/3/4/8/16}}
+  %p = spirv.FRem %a, %b : !spirv.coopmatrix<2x2xf32, Subgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
+spirv.func @smod(%a: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>, %b: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>) "None" {
+  // expected-error @+1 {{operand #0 must be 8/16/32/64-bit integer or vector of 8/16/32/64-bit integer values of length 2/3/4/8/16}}
+  %p = spirv.SMod %a, %b : !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
+
+spirv.func @srem(%a: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>, %b: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>) "None" {
+  // expected-error @+1 {{operand #0 must be 8/16/32/64-bit integer or vector of 8/16/32/64-bit integer values of length 2/3/4/8/16}}
+  %p = spirv.SRem %a, %b : !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
+
+spirv.func @umod(%a: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>, %b: !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>) "None" {
+  // expected-error @+1 {{operand #0 must be 8/16/32/64-bit integer or vector of 8/16/32/64-bit integer values of length 2/3/4/8/16}}
+  %p = spirv.UMod %a, %b : !spirv.coopmatrix<2x2xi32, Subgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
diff --git a/mlir/test/Dialect/SPIRV/IR/types.mlir b/mlir/test/Dialect/SPIRV/IR/types.mlir
index c23894c62826b..7d45b5ea82643 100644
--- a/mlir/test/Dialect/SPIRV/IR/types.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/types.mlir
@@ -564,3 +564,54 @@ func.func private @matrix_size_type(!spirv.matrix< x vector<3xi32>>) -> ()
 func.func private @matrix_size_type(!spirv.matrix<2.0 x vector<3xi32>>) -> ()
 
 // -----
+
+//===----------------------------------------------------------------------===//
+// TensorArm
+//===----------------------------------------------------------------------===//
+
+// CHECK: func private @arm_tensor_type_single_dim_i32(!spirv.arm.tensor<1xi32>)
+func.func private @arm_tensor_type_single_dim_i32(!spirv.arm.tensor<1xi32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_multi_dim_i32(!spirv.arm.tensor<1x2x3xi32>)
+func.func private @arm_tensor_type_multi_dim_i32(!spirv.arm.tensor<1x2x3xi32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_single_dim_f16(!spirv.arm.tensor<1xf16>)
+func.func private @arm_tensor_type_single_dim_f16(!spirv.arm.tensor<1xf16>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_multi_dim_f16(!spirv.arm.tensor<1x2x3xf16>)
+func.func private @arm_tensor_type_multi_dim_f16(!spirv.arm.tensor<1x2x3xf16>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_dynamic_dim(!spirv.arm.tensor<?xi32>)
+func.func private @arm_tensor_type_dynamic_dim(!spirv.arm.tensor<?xi32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_dynamic_dim_2(!spirv.arm.tensor<?x?xi32>)
+func.func private @arm_tensor_type_dynamic_dim_2(!spirv.arm.tensor<?x?xi32>) -> ()
+// -----
+
+// expected-error @+1 {{arm.tensor shape dimensions must be either fully dynamic or completed shaped}}
+func.func private @arm_tensor_type_dynamic_dim(!spirv.arm.tensor<1x?xi32>) -> ()
+
+// -----
+
+// expected-error @+1 {{arm.tensors do not support rank zero}}
+func.func private @arm_tensor_rank_zero(!spirv.arm.tensor<i32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_unranked(!spirv.arm.tensor<*xi32>)
+func.func private @arm_tensor_type_unranked(!spirv.arm.tensor<*xi32>) -> ()
+
+// -----
+
+// expected-error @+1 {{arm.tensors do not support zero dimensions}}
+func.func private @arm_tensor_type_zero_dim(!spirv.arm.tensor<0xi32>) -> ()
diff --git a/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir b/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir
index 99d6a3dc390e0..35fd7c33e4cfe 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir
@@ -23,6 +23,7 @@
 // CHECK-DAG:   %[[c0:.*]] = arith.constant 0 : index
 // CHECK-DAG:   %[[c1:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[c16:.*]] = arith.constant 16 : index
+// CHECK-DAG:   %[[mask:.*]] = arith.constant dense<true> : vector<16xi1>
 // CHECK:       %[[p:.*]] = memref.load %{{.*}}[%[[c0]]] : memref<?xi32>
 // CHECK:       %[[a:.*]] = arith.extui %[[p]] : i32 to i64
 // CHECK:       %[[q:.*]] = arith.index_cast %[[a]] : i64 to index
@@ -31,7 +32,6 @@
 // CHECK:       %[[s:.*]] = arith.index_cast %[[b]] : i64 to index
 // CHECK:       %[[boundary:.*]] = affine.apply #[[$map0]]()[%[[q]], %[[s]]]
 // CHECK:       scf.for %[[i:.*]] = %[[q]] to %[[boundary]] step %[[c16]] {
-// CHECK:         %[[mask:.*]] = vector.constant_mask [16] : vector<16xi1>
 // CHECK:         %[[li:.*]] = vector.load %{{.*}}[%[[i]]] : memref<?xi32>, vector<16xi32>
 // CHECK:         %[[zi:.*]] = arith.extui %[[li]] : vector<16xi32> to vector<16xi64>
 // CHECK:         %[[la:.*]] = vector.load %{{.*}}[%[[i]]] : memref<?xf32>, vector<16xf32>
diff --git a/mlir/test/Dialect/Tensor/bufferize.mlir b/mlir/test/Dialect/Tensor/bufferize.mlir
index c0adc8a49bf70..296ca02564e35 100644
--- a/mlir/test/Dialect/Tensor/bufferize.mlir
+++ b/mlir/test/Dialect/Tensor/bufferize.mlir
@@ -615,6 +615,21 @@ func.func @tensor.splat(%f: f32) -> tensor<10x2x4xf32> {
 
 // -----
 
+// CHECK-LABEL:   func @tensor.splat_other(
+// CHECK-SAME:        %[[F:.*]]: !test.memref_element)
+// CHECK-DAG:       %[[ALLOC:.*]] = memref.alloc() {{.*}} : memref<10x2x4x!test.memref_element>
+// CHECK:           %[[ALLOC_T:.*]] = bufferization.to_tensor %[[ALLOC]]
+// CHECK:           %[[MAPPED:.*]] = linalg.map
+// CHECK:                 outs(%[[ALLOC_T]] : tensor<10x2x4x!test.memref_element>)
+// CHECK:             linalg.yield %[[F]]
+// CHECK:           return %[[MAPPED]] : tensor<10x2x4x!test.memref_element>
+func.func @tensor.splat_other(%f: !test.memref_element) -> tensor<10x2x4x!test.memref_element> {
+  %t = tensor.splat %f : tensor<10x2x4x!test.memref_element>
+  return %t : tensor<10x2x4x!test.memref_element>
+}
+
+// -----
+
 // CHECK-LABEL:   func @tensor.concat(
 // CHECK-SAME:        %[[F:.*]]: tensor<8xf32>)
 // CHECK:           %[[F_MEMREF:.*]] = bufferization.to_buffer %[[F]]
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index 3f9236095138b..95c5b8c91edf5 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -971,6 +971,17 @@ func.func @fold_reshape_1d(%input: tensor<?xf32>, %shape: tensor<1xindex>) -> te
 
 // -----
 
+// CHECK-LABEL: func @fold_reshape_0d
+//  CHECK-SAME: %[[INPUT:[a-zA-Z0-9_]+]]: tensor<f32>
+//  CHECK-SAME: %[[SHAPE:[a-zA-Z0-9_]+]]: tensor<0xindex>
+//       CHECK: return %[[INPUT]]
+func.func @fold_reshape_0d(%input: tensor<f32>, %shape: tensor<0xindex>) -> tensor<f32> {
+  %0 = tensor.reshape %input(%shape) : (tensor<f32>, tensor<0xindex>) -> tensor<f32>
+  return %0 : tensor<f32>
+}
+
+// -----
+
 // CHECK-LABEL: func @fold_extract_constant_splat
 //   CHECK-NOT: tensor.extract_slice
 //       CHECK: arith.constant dense<42> : tensor<4x4xi32>
diff --git a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
index 84eb60248b8be..16efa73f87a2a 100644
--- a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
+++ b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt -split-input-file -test-linalg-transform-patterns=test-fold-into-pack-and-unpack  %s | FileCheck %s
+// RUN: mlir-opt -split-input-file -test-linalg-transform-patterns=test-fold-into-pack-and-unpack-control  %s | FileCheck %s --check-prefix=CONTROL
 
 func.func @fold_unpack_slice(%arg0 : tensor<?x?x8x4xf32>, %arg1 : tensor<?x?xf32>,
     %arg2 : index, %arg3 : index) -> tensor<?x?xf32> {
@@ -373,6 +374,36 @@ func.func @linalg_transpose_linalg.pack_fold(%arg0: tensor<56x57x1x64xf32>) -> t
 
 // -----
 
+func.func @linalg_transpose_linalg.pack_fold_multi_result(%arg0: tensor<56x57x1x64xf32>) -> (tensor<1x56x57x64xf32>, tensor<1x57x56x2x32xf32>) {
+  %0 = tensor.empty() : tensor<1x56x57x64xf32>
+  %transposed = linalg.transpose
+    ins(%arg0 : tensor<56x57x1x64xf32>)
+    outs(%0 : tensor<1x56x57x64xf32>)
+    permutation = [2, 0, 1, 3]
+
+  %1 = tensor.empty() : tensor<1x57x56x2x32xf32>
+  %pack = linalg.pack %transposed
+    outer_dims_perm = [0, 2, 1, 3]
+    inner_dims_pos = [3]
+    inner_tiles = [32]
+    into %1 : tensor<1x56x57x64xf32> -> tensor<1x57x56x2x32xf32>
+  return %transposed, %pack : tensor<1x56x57x64xf32>, tensor<1x57x56x2x32xf32>
+}
+// CHECK-LABEL: func @linalg_transpose_linalg.pack_fold_multi_result(
+//  CHECK-SAME:     %[[ARG0:.+]]: tensor<56x57x1x64xf32>)
+//       CHECK:   %[[TRANSPOSE:.+]] = linalg.transpose
+//       CHECK:   %[[PACK:.+]] = linalg.pack %[[ARG0]]
+//  CHECK-SAME:      outer_dims_perm = [2, 1, 0, 3]
+//       CHECK:   return %[[TRANSPOSE]], %[[PACK]]
+
+// CONTROL-LABEL: func @linalg_transpose_linalg.pack_fold_multi_result(
+//       CONTROL:   %[[TRANSPOSE:.+]] = linalg.transpose
+//       CONTROL:   %[[PACK:.+]] = linalg.pack %[[TRANSPOSE]]
+//  CONTROL-SAME:      outer_dims_perm = [0, 2, 1, 3]
+//       CONTROL:   return %[[TRANSPOSE]], %[[PACK]]
+
+// -----
+
 func.func @linalg_transpose_linalg.pack_fold_with_padding(%arg0: tensor<56x57x1x55xf32>, %padding: f32) -> tensor<1x57x56x2x32xf32> {
   %0 = tensor.empty() : tensor<1x56x57x55xf32>
   %transpose = linalg.transpose
@@ -550,6 +581,36 @@ func.func @linalg_transpose_linalg.unpack_fold(%arg0: tensor<1x1x4x16xi32>) -> t
 
 // -----
 
+func.func @linalg_transpose_linalg.unpack_fold_multi_result(%arg0: tensor<1x1x4x16xi32>) -> (tensor<1x1x16x4xi32>, tensor<16x4xi32>) {
+  %0 = tensor.empty() : tensor<1x1x16x4xi32>
+  %transposed = linalg.transpose ins(%arg0 : tensor<1x1x4x16xi32>)
+                outs(%0 : tensor<1x1x16x4xi32>)
+                permutation = [1, 0, 3, 2]
+  %1 = tensor.empty() : tensor<16x4xi32>
+  %unpack = linalg.unpack %transposed
+            outer_dims_perm = [0, 1]
+            inner_dims_pos = [0, 1]
+            inner_tiles = [16, 4] into
+            %1 : tensor<1x1x16x4xi32> -> tensor<16x4xi32>
+  return %transposed, %unpack : tensor<1x1x16x4xi32>, tensor<16x4xi32>
+}
+//CHECK-LABEL:  func.func @linalg_transpose_linalg.unpack_fold_multi_result(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<1x1x4x16xi32>) 
+//      CHECK:     %[[TRANSPOSE:.+]] = linalg.transpose
+//      CHECK:     %[[UNPACK:.+]] = linalg.unpack %[[ARG0]]
+// CHECK-SAME:        outer_dims_perm = [1, 0]
+//      CHECK:     return %[[TRANSPOSE]], %[[UNPACK]]
+//      CHECK:   }
+
+//CONTROL-LABEL:  func.func @linalg_transpose_linalg.unpack_fold_multi_result(
+//      CONTROL:     %[[TRANSPOSE:.+]] = linalg.transpose
+//      CONTROL:     %[[UNPACK:.+]] = linalg.unpack %[[TRANSPOSE]]
+// CONTROL-SAME:        outer_dims_perm = [0, 1]
+//      CONTROL:     return %[[TRANSPOSE]], %[[UNPACK]]
+//      CONTROL:   }
+
+// -----
+
 func.func @linalg_transpose_linalg.unpack_fold_partial_tile(%arg0: tensor<1x1x4x16xi32>) -> tensor<15x3xi32> {
   %0 = tensor.empty() : tensor<1x1x16x4xi32>
   %transposed = linalg.transpose ins(%arg0 : tensor<1x1x4x16xi32>)
diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir
index f35d52e700084..665657a67dc61 100644
--- a/mlir/test/Dialect/Tensor/invalid.mlir
+++ b/mlir/test/Dialect/Tensor/invalid.mlir
@@ -466,9 +466,10 @@ func.func @invalid_splat(%v : f32) {
 
 // -----
 
-func.func @invalid_splat(%v : vector<8xf32>) {
-  // expected-error@+1 {{must be integer/index/float type}}
-  %w = tensor.splat %v : tensor<8xvector<8xf32>>
+// expected-note@+1 {{prior use here}}
+func.func @invalid_splat(%v : f32) {
+  // expected-error@+1 {{expects different type than prior uses: 'i32' vs 'f32'}}
+  %w = tensor.splat %v : tensor<1xi32>
   return
 }
 
diff --git a/mlir/test/Dialect/Tensor/ops.mlir b/mlir/test/Dialect/Tensor/ops.mlir
index 930986211cb6d..681a934ba0698 100644
--- a/mlir/test/Dialect/Tensor/ops.mlir
+++ b/mlir/test/Dialect/Tensor/ops.mlir
@@ -313,13 +313,17 @@ func.func @pad_to_static_size(%arg0: tensor<?x?xf32>, %ub0: index, %ub1: index,
 // -----
 
 // CHECK-LABEL: func @test_splat_op
-// CHECK-SAME: [[S:%arg[0-9]+]]: f32
-func.func @test_splat_op(%s : f32) {
-  // CHECK: tensor.splat [[S]] : tensor<8xf32>
+// CHECK-SAME: %[[S:.*]]: f32
+// CHECK-SAME: %[[P:.*]]: !llvm.ptr
+func.func @test_splat_op(%s : f32, %p : !llvm.ptr) {
+  // CHECK: tensor.splat %[[S]] : tensor<8xf32>
   %v = tensor.splat %s : tensor<8xf32>
 
-  // CHECK: tensor.splat [[S]] : tensor<4xf32>
+  // CHECK: tensor.splat %[[S]] : tensor<4xf32>
   %u = "tensor.splat"(%s) : (f32) -> tensor<4xf32>
+
+  // CHECK: tensor.splat %[[P]] : tensor<8x!llvm.ptr>
+  %w = tensor.splat %p : tensor<8x!llvm.ptr>
   return
 }
 
diff --git a/mlir/test/Dialect/Tensor/reify-shapes.mlir b/mlir/test/Dialect/Tensor/reify-shapes.mlir
new file mode 100644
index 0000000000000..5569d90f8b731
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/reify-shapes.mlir
@@ -0,0 +1,31 @@
+// RUN: mlir-opt -reify-result-shapes  %s | FileCheck %s
+
+// The test below checks concat op reification. In the first case, no cast is inserted while on the second a cast gets inserted.
+// CHECK-LABEL:  func.func @concat_reification
+func.func @concat_reification(%arg0: tensor<4x7x3xf32>, %arg1 : tensor<4x4x3xf32>, %arg2: tensor<?x?x?xf32>)
+  -> (tensor<4x11x3xf32>, tensor<?x?x?xf32>) {
+  // CHECK: %[[RES0:.*]] = tensor.concat dim(1) %{{.*}} : (tensor<4x7x3xf32>, tensor<4x4x3xf32>) -> tensor<4x11x3xf32>
+  %1 = tensor.concat dim(1) %arg0, %arg1 : (tensor<4x7x3xf32>, tensor<4x4x3xf32>) -> tensor<4x11x3xf32>
+  // CHECK: %[[V0:.*]] = tensor.concat dim(2) %{{.*}} : (tensor<4x7x3xf32>, tensor<?x?x?xf32>) -> tensor<4x7x?xf32>
+  // CHECK: %[[RES1:.*]] = tensor.cast %[[V0]] : tensor<4x7x?xf32> to tensor<?x?x?xf32>
+  %2 = tensor.concat dim(2) %arg0, %arg2 : (tensor<4x7x3xf32>, tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+  // CHECK: return %[[RES0]], %[[RES1]] : tensor<4x11x3xf32>, tensor<?x?x?xf32>
+  return %1, %2 : tensor<4x11x3xf32>, tensor<?x?x?xf32>
+}
+
+// CHECK-LABEL:  func.func @pad_reification
+func.func @pad_reification(%cst : f32, %idx : index, %t: tensor<64x?x64xf32>) -> tensor<1x?x64xf32> {
+  %pad_amt = affine.apply affine_map<(d0) -> (-d0 + 256)>(%idx)
+  %es = tensor.extract_slice %t[0, 0, 0] [1, %idx, 64] [1, 1, 1] 
+    : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+
+  // CHECK: tensor.pad
+  // CHECK: : tensor<1x?x64xf32> to tensor<1x256x64xf32>
+  // CHECK: tensor.cast %{{.*}} : tensor<1x256x64xf32> to tensor<1x?x64xf32>
+  %padded = tensor.pad %es low[0, 0, 0] high[0, %pad_amt, 0] {
+    ^bb0(%a: index, %b: index, %c: index):
+    tensor.yield %cst : f32
+  } : tensor<1x?x64xf32> to tensor<1x?x64xf32>
+
+  return %padded : tensor<1x?x64xf32>
+}
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 6fa6756822acb..5a424c41775c9 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -2000,6 +2000,7 @@ func.func @test_rescale_input_unsigned(%arg0: tensor<1x1xui8>) -> (tensor<1x1xi8
   %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
   %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
   %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.rescale' op is not profile-aligned: element type 'ui8' is not legal}}
   %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xui8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xi8>
   return %r : tensor<1x1xi8>
 }
@@ -2012,6 +2013,7 @@ func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui
   %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
   %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
   %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.rescale' op is not profile-aligned: element type 'ui8' is not legal}}
   %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
   return %r : tensor<1x1xui8>
 }
diff --git a/mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir b/mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir
new file mode 100644
index 0000000000000..38ac8d8fb66d9
--- /dev/null
+++ b/mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-opt --split-input-file --tosa-convert-integer-type-to-signless %s | FileCheck %s
+
+// -----
+
+// CHECK-LABEL: test_rescale_output_unsigned
+// CHECK: %arg0: tensor<1x1xi8>
+func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: %[[RESCALE:.*]] = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
+  // CHECK: return %[[RESCALE]] : tensor<1x1xi8>
+  return %r : tensor<1x1xui8>
+}
+
+// -----
+
+// CHECK-LABEL: test_rescale_input_unsigned
+// CHECK: %arg0: tensor<1x1xi16>
+func.func @test_rescale_input_unsigned(%arg0: tensor<1x1xui16>) -> (tensor<1x1xi8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<32768> : tensor<1xi16>}> : () -> tensor<1xi16>
+  // CHECK: %[[RESCALE:.*]] = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi8>) -> tensor<1x1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xui16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi8>) -> tensor<1x1xi8>
+  // CHECK: return %[[RESCALE]] : tensor<1x1xi8>
+  return %r : tensor<1x1xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_unsigned_function_signature
+// CHECK: %arg0: tensor<1xi8>, %arg1: tensor<1xi8>
+func.func @test_unsigned_function_signature(%arg0: tensor<1xui8>, %arg1: tensor<1xui8>) -> (tensor<1xui8>, tensor<1xui8>) {
+  // CHECK: return %arg0, %arg1 : tensor<1xi8>, tensor<1xi8>
+  return %arg0, %arg1 : tensor<1xui8>, tensor<1xui8>
+}
+
+// -----
+
+// CHECK-LABEL: test_no_change
+// CHECK: %arg0: tensor<13x21x3xi8>
+func.func @test_no_change(%arg0: tensor<13x21x3xi8>) -> tensor<13x21x3xi8> {
+  %0 = tosa.reverse %arg0 {axis = 0 : i32} : (tensor<13x21x3xi8>) -> tensor<13x21x3xi8>
+  // CHECK: return %0 : tensor<13x21x3xi8>
+  return %0 : tensor<13x21x3xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_regions
+// CHECK: %arg0: tensor<i8>, %arg1: tensor<i8>
+func.func @test_regions(%arg0: tensor<ui8>, %arg1: tensor<ui8>, %arg2: tensor<i1>) -> tensor<ui8> {
+  // CHECK: tosa.cond_if %arg2 -> (tensor<i8>)
+  %0 = "tosa.cond_if"(%arg2, %arg0, %arg1) ({
+  ^bb0(%arg3: tensor<ui8>, %arg4: tensor<ui8>):
+    // CHECK: %1 = tosa.add %arg0, %arg1 : (tensor<i8>, tensor<i8>) -> tensor<i8>
+    %1 = tosa.add %arg0, %arg1 : (tensor<ui8>, tensor<ui8>) -> tensor<ui8>
+    // CHECK: tosa.yield %1 : tensor<i8>
+    tosa.yield %1 : tensor<ui8>
+  },  {
+  ^bb0(%arg3: tensor<ui8>, %arg4: tensor<ui8>):
+    // CHECK: %1 = tosa.sub %arg0, %arg1 : (tensor<i8>, tensor<i8>) -> tensor<i8>
+    %1 = tosa.sub %arg0, %arg1 : (tensor<ui8>, tensor<ui8>) -> tensor<ui8>
+    // CHECK: tosa.yield %1 : tensor<i8>
+    tosa.yield %1 : tensor<ui8>
+  }) : (tensor<i1>, tensor<ui8>, tensor<ui8>) -> tensor<ui8>
+  // CHECK: return %0 : tensor<i8>
+  return %0 : tensor<ui8>
+}
diff --git a/mlir/test/Dialect/Tosa/tosa-validation-valid.mlir b/mlir/test/Dialect/Tosa/tosa-validation-valid.mlir
new file mode 100644
index 0000000000000..cab14201dc0ce
--- /dev/null
+++ b/mlir/test/Dialect/Tosa/tosa-validation-valid.mlir
@@ -0,0 +1,31 @@
+//--------------------------------------------------------------------------------------------------
+// Test valid IR in terms of the shape and type of tensor, and the argument type of
+// operation. Excludes the profile compilance checking since it is performed earlier in the
+// validation flow.
+//--------------------------------------------------------------------------------------------------
+
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics --tosa-validate="profile=pro_int,pro_fp extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow,doubleround,inexactround" | FileCheck %s
+
+// -----
+
+// CHECK-LABEL: test_rescale_input_unsigned
+func.func @test_rescale_input_unsigned(%arg0: tensor<1x1xui8>) -> (tensor<1x1xi8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xui8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xi8>
+  return %r : tensor<1x1xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_rescale_output_unsigned
+func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
+  return %r : tensor<1x1xui8>
+}
diff --git a/mlir/test/Dialect/Transform/test-tune-extension-invalid.mlir b/mlir/test/Dialect/Transform/test-tune-extension-invalid.mlir
new file mode 100644
index 0000000000000..2e5f433abeb71
--- /dev/null
+++ b/mlir/test/Dialect/Transform/test-tune-extension-invalid.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s --transform-interpreter --split-input-file --verify-diagnostics
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    // expected-error@below {{provided `selected` attribute is not an element of `options` array of attributes}}
+    %heads_or_tails = transform.tune.knob<"coin"> = 1 from options = [true, false] -> !transform.any_param
+    transform.yield
+  }
+}
+
+// -----
+
+func.func private @f()
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    // expected-error@below {{non-deterministic choice "coin" is only resolved through providing a `selected` attr}}
+    %heads_or_tails = transform.tune.knob<"coin"> options = [true, false] -> !transform.any_param
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Transform/test-tune-extension.mlir b/mlir/test/Dialect/Transform/test-tune-extension.mlir
new file mode 100644
index 0000000000000..0a253c6d5f837
--- /dev/null
+++ b/mlir/test/Dialect/Transform/test-tune-extension.mlir
@@ -0,0 +1,61 @@
+// RUN: mlir-opt %s --transform-interpreter --split-input-file \
+// RUN:     --verify-diagnostics | FileCheck %s
+
+// CHECK-LABEL: @schedule_with_nondet_knobs
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @schedule_with_nondet_knobs(%arg0: !transform.any_op {transform.readonly}) {
+    // CHECK: %[[HEADS_OR_TAILS:.*]] = transform.tune.knob<"coin"> options = [true, false] -> !transform.any_param
+    %heads_or_tails = transform.tune.knob<"coin"> options = [true, false] -> !transform.any_param
+    // CHECK: transform.tune.knob<"animal"> options = ["cat", "dog", unit] -> !transform.any_param
+    %chosen_category = transform.tune.knob<"animal"> options = ["cat", "dog", unit] -> !transform.any_param
+    // CHECK: transform.tune.knob<"tile_size"> options = [2, 4, 8, 16, 24, 32] -> !transform.any_param
+    %chosen_tile_size = transform.tune.knob<"tile_size"> options = [2, 4, 8, 16, 24, 32] -> !transform.any_param
+    // CHECK: transform.tune.knob<"magic_value"> options = [2.000000e+00 : f32, 2.250000e+00 : f32, 2.500000e+00 : f32, 2.750000e+00 : f32, 3.000000e+00 : f32] -> !transform.any_param
+    %chosen_constant = transform.tune.knob<"magic_value"> options = [2.0 : f32, 2.25 : f32, 2.5 : f32, 2.75 : f32, 3.0 : f32] -> !transform.any_param
+    // CHECK: transform.debug.emit_param_as_remark %[[HEADS_OR_TAILS]]
+    transform.debug.emit_param_as_remark %heads_or_tails : !transform.any_param
+    transform.yield
+  }
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    // Dummy sequence to appease -transform-interpreter invocation
+    transform.yield
+  }
+}
+
+// -----
+
+// Schedule where non-determinism on knobs has been resolved by selecting a valid option.
+
+// CHECK-LABEL: payload_for_schedule_with_selected_knobs
+func.func private @payload_for_schedule_with_selected_knobs()
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    // CHECK: %[[HEADS_OR_TAILS:.*]] = transform.tune.knob<"coin"> = true from options = [true, false] -> !transform.any_param
+    %heads_or_tails = transform.tune.knob<"coin"> = true from options = [true, false] -> !transform.any_param
+    // expected-remark@below {{true}}
+    transform.debug.emit_param_as_remark %heads_or_tails : !transform.any_param
+
+    // CHECK: transform.tune.knob<"animal"> = "dog" from options = ["cat", "dog", unit] -> !transform.any_param
+    %chosen_category = transform.tune.knob<"animal"> = "dog" from options = ["cat", "dog", unit] -> !transform.any_param
+    // CHECK: transform.tune.knob<"tile_size"> = 8 : i64 from options = [2, 4, 8, 16, 24, 32] -> !transform.any_param
+    %chosen_tile_size = transform.tune.knob<"tile_size"> = 8 from options = [2, 4, 8, 16, 24, 32] -> !transform.any_param
+    // CHECK: transform.tune.knob<"magic_value"> = 2.500000e+00 : f32 from options = [2.000000e+00 : f32, 2.250000e+00 : f32, 2.500000e+00 : f32, 2.750000e+00 : f32, 3.000000e+00 : f32] -> !transform.any_param
+    %chosen_constant = transform.tune.knob<"magic_value"> = 2.5 : f32  from options = [2.0 : f32, 2.25 : f32, 2.5 : f32, 2.75 : f32, 3.0 : f32] -> !transform.any_param
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK: #[[AFFINE_SET:.*]] = affine_set<(d0) : (d0 - 2 >= 0)>
+// CHECK: payload_for_schedule_where_selected_knob_being_a_member_of_options_is_unverified
+func.func private @payload_for_schedule_where_selected_knob_being_a_member_of_options_is_unverified()
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    // CHECK: transform.tune.knob<"bounded"> = 4242 : i64 from options = #[[AFFINE_SET]] -> !transform.any_param
+    %value_in_half_range = transform.tune.knob<"bounded"> = 4242 from options = affine_set<(d0) : (d0 - 2 >= 0)>  -> !transform.any_param
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 65b73375831da..8a9e27378df61 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -14,7 +14,7 @@ func.func @create_vector_mask_to_constant_mask() -> (vector<4x3xi1>) {
 // CHECK-LABEL: create_scalable_vector_mask_to_constant_mask
 func.func @create_scalable_vector_mask_to_constant_mask() -> (vector<[8]xi1>) {
   %c-1 = arith.constant -1 : index
-  // CHECK: vector.constant_mask [0] : vector<[8]xi1>
+  // CHECK: arith.constant dense<false> : vector<[8]xi1>
   %0 = vector.create_mask %c-1 : vector<[8]xi1>
   return %0 : vector<[8]xi1>
 }
@@ -36,7 +36,7 @@ func.func @create_vector_mask_to_constant_mask_truncation() -> (vector<4x3xi1>)
 func.func @create_vector_mask_to_constant_mask_truncation_neg() -> (vector<4x3xi1>) {
   %cneg2 = arith.constant -2 : index
   %c5 = arith.constant 5 : index
-  // CHECK: vector.constant_mask [0, 0] : vector<4x3xi1>
+  // CHECK: arith.constant dense<false> : vector<4x3xi1>
   %0 = vector.create_mask %c5, %cneg2 : vector<4x3xi1>
   return %0 : vector<4x3xi1>
 }
@@ -47,7 +47,7 @@ func.func @create_vector_mask_to_constant_mask_truncation_neg() -> (vector<4x3xi
 func.func @create_vector_mask_to_constant_mask_truncation_zero() -> (vector<4x3xi1>) {
   %c2 = arith.constant 2 : index
   %c0 = arith.constant 0 : index
-  // CHECK: vector.constant_mask [0, 0] : vector<4x3xi1>
+  // CHECK: arith.constant dense<false> : vector<4x3xi1>
   %0 = vector.create_mask %c0, %c2 : vector<4x3xi1>
   return %0 : vector<4x3xi1>
 }
@@ -60,7 +60,7 @@ func.func @create_vector_mask_to_constant_mask_scalable_all_true() -> (vector<8x
   %c16 = arith.constant 16 : index
   %0 = vector.vscale
   %1 = arith.muli %0, %c16 : index
-  // CHECK: vector.constant_mask [8, 16] : vector<8x[16]xi1>
+  // CHECK: arith.constant dense<true> : vector<8x[16]xi1>
   %10 = vector.create_mask %c8, %1 : vector<8x[16]xi1>
   return %10 : vector<8x[16]xi1>
 }
@@ -272,6 +272,30 @@ func.func @extract_from_non_constant_create_mask(%dim0: index) -> vector<[2]xi1>
 
 // -----
 
+// CHECK-LABEL: constant_mask_to_true_splat
+func.func @constant_mask_to_true_splat() -> vector<2x4xi1> {
+  // CHECK: arith.constant dense<true>
+  // CHECK-NOT: vector.constant_mask
+  %0 = vector.constant_mask [2, 4] : vector<2x4xi1>
+  return %0 : vector<2x4xi1>
+}
+
+// CHECK-LABEL: constant_mask_to_false_splat
+func.func @constant_mask_to_false_splat() -> vector<2x4xi1> {
+  // CHECK: arith.constant dense<false>
+  // CHECK-NOT: vector.constant_mask
+  %0 = vector.constant_mask [0, 0] : vector<2x4xi1>
+  return %0 : vector<2x4xi1>
+}
+
+// CHECK-LABEL: constant_mask_to_true_splat_0d
+func.func @constant_mask_to_true_splat_0d() -> vector<i1> {
+  // CHECK: arith.constant dense<true>
+  // CHECK-NOT: vector.constant_mask
+  %0 = vector.constant_mask [1] : vector<i1>
+  return %0 : vector<i1>
+}
+
 // CHECK-LABEL: constant_mask_transpose_to_transposed_constant_mask
 func.func @constant_mask_transpose_to_transposed_constant_mask() -> (vector<2x3x4xi1>, vector<4x2x3xi1>) {
   //     CHECK: vector.constant_mask [1, 2, 3] : vector<2x3x4xi1>
@@ -289,7 +313,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x2xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x2xi1>
-  // CHECK: vector.constant_mask [2, 2] : vector<2x2xi1>
+  // CHECK: arith.constant dense<true> : vector<2x2xi1>
   return %1 : vector<2x2xi1>
 }
 
@@ -322,7 +346,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x2xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x2xi1>
-  // CHECK: vector.constant_mask [0, 0] : vector<2x2xi1>
+  // CHECK: arith.constant dense<false> : vector<2x2xi1>
   return %1 : vector<2x2xi1>
 }
 
@@ -333,7 +357,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x1xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [0, 2], sizes = [2, 1], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x1xi1>
-  // CHECK: vector.constant_mask [0, 0] : vector<2x1xi1>
+  // CHECK: arith.constant dense<false> : vector<2x1xi1>
   return %1 : vector<2x1xi1>
 }
 
@@ -344,7 +368,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x1xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [0, 1], sizes = [2, 1], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x1xi1>
-  // CHECK: vector.constant_mask [2, 1] : vector<2x1xi1>
+  // CHECK: arith.constant dense<true> : vector<2x1xi1>
   return %1 : vector<2x1xi1>
 }
 
@@ -3075,6 +3099,33 @@ func.func @from_elements_to_elements_shuffle(%a: vector<4x2xf32>) -> vector<4x2x
 
 // -----
 
+// CHECK-LABEL: func @from_elements_all_elements_constant(
+func.func @from_elements_all_elements_constant() -> vector<2x2xi32> {
+  %c0_i32 = arith.constant 0 : i32
+  %c1_i32 = arith.constant 1 : i32
+  %c2_i32 = arith.constant 2 : i32
+  %c3_i32 = arith.constant 3 : i32
+  // CHECK: %[[RES:.*]] = arith.constant dense<{{\[\[0, 1\], \[2, 3\]\]}}> : vector<2x2xi32>
+  %res = vector.from_elements %c0_i32, %c1_i32, %c2_i32, %c3_i32 : vector<2x2xi32>
+  // CHECK: return %[[RES]]
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @from_elements_partial_elements_constant(
+// CHECK-SAME:     %[[A:.*]]: f32
+func.func @from_elements_partial_elements_constant(%arg0: f32) -> vector<2xf32> {
+  // CHECK: %[[C:.*]] = arith.constant 1.000000e+00 : f32
+  %c = arith.constant 1.0 : f32
+  // CHECK: %[[RES:.*]] = vector.from_elements %[[A]], %[[C]] : vector<2xf32>
+  %res = vector.from_elements %arg0, %c : vector<2xf32>
+  // CHECK: return %[[RES]]
+  return %res : vector<2xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @vector_insert_const_regression(
 //       CHECK:   llvm.mlir.undef
 //       CHECK:   vector.insert
@@ -3419,3 +3470,32 @@ func.func @fold_insert_constant_indices(%arg : vector<4x1xi32>) -> vector<4x1xi3
   %res = vector.insert %1, %arg[%0, %0] : i32 into vector<4x1xi32>
   return %res : vector<4x1xi32>
 }
+
+// -----
+
+// CHECK-LABEL: @fold_insert_use_chain(
+//  CHECK-SAME:   %[[ARG:.*]]: vector<4x4xf32>,
+//  CHECK-SAME:   %[[VAL:.*]]: f32,
+//  CHECK-SAME:   %[[POS:.*]]: index) -> vector<4x4xf32> {
+//  CHECK-NEXT:   %[[RES:.*]] = vector.insert %[[VAL]], %[[ARG]] {{\[}}%[[POS]], 0] : f32 into vector<4x4xf32>
+//  CHECK-NEXT:   return %[[RES]] : vector<4x4xf32>
+func.func @fold_insert_use_chain(%arg : vector<4x4xf32>, %val : f32, %pos: index) -> vector<4x4xf32> {
+  %v_0 = vector.insert %val, %arg[%pos, 0] : f32 into vector<4x4xf32>
+  %v_1 = vector.insert %val, %v_0[%pos, 0] : f32 into vector<4x4xf32>
+  %v_2 = vector.insert %val, %v_1[%pos, 0] : f32 into vector<4x4xf32>
+  return %v_2 : vector<4x4xf32>  
+}
+
+// -----
+
+// CHECK-LABEL: @no_fold_insert_use_chain_mismatch_static_position(
+//  CHECK-SAME:   %[[ARG:.*]]: vector<4xf32>,
+//  CHECK-SAME:   %[[VAL:.*]]: f32) -> vector<4xf32> {
+//       CHECK:   %[[V_0:.*]] = vector.insert %[[VAL]], %[[ARG]] [0] : f32 into vector<4xf32>
+//       CHECK:   %[[V_1:.*]] = vector.insert %[[VAL]], %[[V_0]] [1] : f32 into vector<4xf32>
+//       CHECK:   return %[[V_1]] : vector<4xf32>
+func.func @no_fold_insert_use_chain_mismatch_static_position(%arg : vector<4xf32>, %val : f32) -> vector<4xf32> {
+  %v_0 = vector.insert %val, %arg[0] : f32 into vector<4xf32>
+  %v_1 = vector.insert %val, %v_0[1] : f32 into vector<4xf32>
+  return %v_1 : vector<4xf32>  
+}
diff --git a/mlir/test/Dialect/Vector/linearize.mlir b/mlir/test/Dialect/Vector/linearize.mlir
index 9cbf319ffddb2..894171500d9d6 100644
--- a/mlir/test/Dialect/Vector/linearize.mlir
+++ b/mlir/test/Dialect/Vector/linearize.mlir
@@ -294,6 +294,20 @@ func.func @test_vector_extract_scalable(%arg0: vector<2x8x[2]xf32>) -> vector<8x
 
 // -----
 
+// CHECK-LABEL: test_vector_insert_scalar
+// CHECK-SAME: (%[[DEST:.*]]: vector<2x4xf32>, %[[SRC:.*]]: f32) -> vector<2x4xf32> {
+func.func @test_vector_insert_scalar(%arg0: vector<2x4xf32>, %arg1: f32) -> vector<2x4xf32> {
+
+  // CHECK: %[[DEST_1D:.*]] = vector.shape_cast %[[DEST]] : vector<2x4xf32> to vector<8xf32>
+  // CHECK: %[[INSERT_1D:.*]] = vector.insert %[[SRC]], %[[DEST_1D]] [6] : f32 into vector<8xf32>
+  // CHECK: %[[RES:.*]] = vector.shape_cast %[[INSERT_1D]] : vector<8xf32> to vector<2x4xf32>
+  // CHECK: return %[[RES]] : vector<2x4xf32>
+  %0 = vector.insert %arg1, %arg0[1, 2]: f32 into vector<2x4xf32>
+  return %0 : vector<2x4xf32>
+}
+
+// -----
+
 // CHECK-LABEL: test_vector_insert
 // CHECK-SAME: (%[[DEST:.*]]: vector<2x8x4xf32>, %[[SRC:.*]]: vector<8x4xf32>) -> vector<2x8x4xf32> {
 func.func @test_vector_insert(%arg0: vector<2x8x4xf32>, %arg1: vector<8x4xf32>) -> vector<2x8x4xf32> {
@@ -444,7 +458,7 @@ func.func @linearize_scalable_vector_splat(%arg0: i32) -> vector<4x[2]xi32> {
 // CHECK-LABEL: linearize_create_mask
 // CHECK-SAME: (%[[ARG0:.*]]: index, %[[ARG1:.*]]: index) -> vector<1x16xi1>
 func.func @linearize_create_mask(%arg0 : index, %arg1 : index) -> vector<1x16xi1> {
-  
+
   // CHECK: %[[C0:.*]] = arith.constant 0 : index
   // CHECK: %[[CMP:.*]] = arith.cmpi sgt, %[[ARG0]], %[[C0]] : index
   // CHECK: %[[INDEXCAST:.*]] = arith.index_cast %[[CMP]] : i1 to index
diff --git a/mlir/test/Dialect/Vector/vector-mem-transforms.mlir b/mlir/test/Dialect/Vector/vector-mem-transforms.mlir
index 8cb25c7578495..e6593320f1bde 100644
--- a/mlir/test/Dialect/Vector/vector-mem-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-mem-transforms.mlir
@@ -83,7 +83,7 @@ func.func @maskedstore2(%base: memref<16xf32>, %value: vector<16xf32>)  {
 // CHECK-SAME:                  %[[A1:.*]]: vector<16xi32>,
 // CHECK-SAME:                  %[[A2:.*]]: vector<16xf32>) -> vector<16xf32> {
 // CHECK-NEXT:      %[[C:.*]] = arith.constant 0 : index
-// CHECK-NEXT:      %[[M:.*]] = vector.constant_mask [16] : vector<16xi1>
+// CHECK-NEXT:      %[[M:.*]] = arith.constant dense<true> : vector<16xi1>
 // CHECK-NEXT:      %[[G:.*]] = vector.gather %[[A0]][%[[C]]] [%[[A1]]], %[[M]], %[[A2]] : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
 // CHECK-NEXT:      return %[[G]] : vector<16xf32>
 func.func @gather1(%base: memref<16xf32>, %indices: vector<16xi32>, %pass_thru: vector<16xf32>) -> vector<16xf32> {
@@ -112,7 +112,7 @@ func.func @gather2(%base: memref<16xf32>, %indices: vector<16xi32>, %pass_thru:
 // CHECK-SAME:                   %[[A1:.*]]: vector<16xi32>,
 // CHECK-SAME:                   %[[A2:.*]]: vector<16xf32>) {
 // CHECK-NEXT:      %[[C:.*]] = arith.constant 0 : index
-// CHECK-NEXT:      %[[M:.*]] = vector.constant_mask [16] : vector<16xi1>
+// CHECK-NEXT:      %[[M:.*]] = arith.constant dense<true> : vector<16xi1>
 // CHECK-NEXT:      vector.scatter %[[A0]][%[[C]]] [%[[A1]]], %[[M]], %[[A2]] : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>
 // CHECK-NEXT:      return
 func.func @scatter1(%base: memref<16xf32>, %indices: vector<16xi32>, %value: vector<16xf32>) {
diff --git a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
index 0f04d3b79b535..d18edd0ac5563 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
@@ -85,8 +85,8 @@ func.func @transfer_read_dims_mismatch_contiguous(
 
 // CHECK-LABEL: func.func @transfer_read_dims_mismatch_contiguous(
 // CHECK-SAME:    %[[MEM:.+]]: memref<5x4x3x2xi8, {{.+}}>) -> vector<2x3x2xi8> {
-// CHECK:         %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:         %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:     %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK-DAG:     %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[COLLAPSED_MEM:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1, 2, 3]]
 // CHECK-SAME:          : memref<5x4x3x2xi8, {{.+}}> into memref<5x24xi8, {{.+}}>
@@ -116,8 +116,8 @@ func.func @transfer_read_dims_mismatch_contiguous_unit_dims(
 // CHECK-LABEL: func.func @transfer_read_dims_mismatch_contiguous_unit_dims(
 // CHECK-SAME:    %[[MEM:.+]]: memref<6x5x4x3x2xi8, strided<[120, 24, 6, 2, 1], offset: ?>>)
 // CHECK-SAME:    -> vector<1x1x4x3x2xi8>
-// CHECK:       %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:       %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
 // CHECK:       %[[COLLAPSED:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3, 4]]
 // CHECK-SAME:    : memref<6x5x4x3x2xi8, strided<[120, 24, 6, 2, 1], offset: ?>>
@@ -149,8 +149,8 @@ func.func @transfer_read_non_contiguous_unit_dims(
 
 // CHECK-LABEL:   func.func @transfer_read_non_contiguous_unit_dims(
 // CHECK-SAME:      %[[MEM:.*]]: memref<5x4x3x2xi8, strided<[48, 6, 2, 1], offset: ?>>) -> vector<1x1x3x2xi8> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i8
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0 : i8
+// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_3:.*]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3]]
 // CHECK-SAME:        : memref<5x4x3x2xi8, strided<[48, 6, 2, 1], offset: ?>> into memref<5x4x6xi8, strided<[48, 6, 1], offset: ?>>
@@ -182,8 +182,8 @@ func.func @transfer_read_dims_mismatch_non_zero_indices(
 // CHECK-LABEL:   func.func @transfer_read_dims_mismatch_non_zero_indices(
 // CHECK-SAME:      %[[IDX_1:.+]]: index, %[[IDX_2:.+]]: index,
 // CHECK-SAME:      %[[MEM:.+]]: memref<1x43x4x6xi32>
-// CHECK:           %[[C0_I32:.+]] = arith.constant 0 : i32
-// CHECK:           %[[C_0:.+]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C0_I32:.+]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C_0:.+]] = arith.constant 0 : index
 // CHECK:           %[[COLLAPSED_IN:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3]]
 // CHECK-SAME:         : memref<1x43x4x6xi32> into memref<1x43x24xi32>
@@ -241,8 +241,8 @@ func.func @transfer_read_leading_dynamic_dims(
 
 // CHECK-LABEL: func @transfer_read_leading_dynamic_dims
 // CHECK-SAME:    %[[MEM:.+]]: memref<?x?x8x4xi8, {{.+}}>, %[[IDX_1:.+]]: index, %[[IDX_2:.+]]: index
-// CHECK:         %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:         %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:     %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK-DAG:     %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[COLLAPSED:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3]]
 // CHECK-SAME:      : memref<?x?x8x4xi8, {{.+}}> into memref<?x?x32xi8, {{.+}}>
@@ -304,8 +304,8 @@ func.func @transfer_read_dynamic_dim_to_flatten(
 // CHECK-SAME:    %[[IDX_1:arg0]]
 // CHECK-SAME:    %[[IDX_2:arg1]]
 // CHECK-SAME:    %[[MEM:arg2]]
-// CHECK:              %[[C0_I32:.+]] = arith.constant 0 : i32
-// CHECK:              %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:          %[[C0_I32:.+]] = arith.constant 0 : i32
+// CHECK-DAG:          %[[C0:.+]] = arith.constant 0 : index
 // CHECK:              %[[COLLAPSED:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}:  [[0], [1], [2, 3]]
 // CHECK-SAME:           memref<1x?x4x6xi32> into memref<1x?x24xi32>
diff --git a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
index 7cfbcdf101d11..1161dbd4b2166 100644
--- a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
+++ b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
@@ -1132,8 +1132,8 @@ func.func @warp_execute_nd_distribute(%laneid: index, %v0: vector<1x64x1xf32>, %
 
   //     CHECK-SCF-IF: gpu.barrier
   //     CHECK-SCF-IF: %[[WID:.*]] = affine.apply #[[$TIMES2]]()[%[[LANEID]]]
-  // CHECK-SCF-IF-DAG: %[[R0:.*]] = vector.transfer_read %{{.*}}[%[[LANEID]], %[[C0]], %[[C0]]], %cst {in_bounds = [true, true, true]} : memref<32x64x1xf32, 3>, vector<1x64x1xf32>
-  // CHECK-SCF-IF-DAG: %[[R1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[WID]], %[[C0]]], %cst {in_bounds = [true, true, true]} : memref<1x64x128xf32, 3>, vector<1x2x128xf32>
+  // CHECK-SCF-IF-DAG: %[[R0:.*]] = vector.transfer_read %{{.*}}[%[[LANEID]], %[[C0]], %[[C0]]], %{{.*}} {in_bounds = [true, true, true]} : memref<32x64x1xf32, 3>, vector<1x64x1xf32>
+  // CHECK-SCF-IF-DAG: %[[R1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[WID]], %[[C0]]], %{{.*}} {in_bounds = [true, true, true]} : memref<1x64x128xf32, 3>, vector<1x2x128xf32>
   //     CHECK-SCF-IF: return %[[R0]], %[[R1]] : vector<1x64x1xf32>, vector<1x2x128xf32>
   return %r#0, %r#1 : vector<1x64x1xf32>, vector<1x2x128xf32>
 }
diff --git a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir
index 64f01d61d6e80..09df1e4da43e2 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir
@@ -1,6 +1,25 @@
 // RUN: mlir-opt --xegpu-wg-to-sg-distribute -split-input-file %s | FileCheck %s
 
 gpu.module @test_elementwise_ops {
+
+  // CHECK-LABEL: unary_ops_sg_layout_only
+  gpu.func @unary_ops_sg_layout_only(%a: memref<24x32xf32>) {
+    %tdesc_a = xegpu.create_nd_tdesc %a[0, 0] : memref<24x32xf32>
+      -> !xegpu.tensor_desc<24x32xf32, #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>>
+    %load_a = xegpu.load_nd %tdesc_a
+      : !xegpu.tensor_desc<24x32xf32, #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>>
+      -> vector<24x32xf32>
+    // CHECK: math.exp {{.*}} : vector<12x8xf32>
+    %exp = math.exp %load_a
+      {layout_result_0 = #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>}
+      : vector<24x32xf32>
+    // CHECK: arith.negf {{.*}} : vector<12x8xf32>
+    %negf = arith.negf %load_a
+      {layout_result_0 = #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>}
+      : vector<24x32xf32>
+    gpu.return
+  }
+
   // CHECK-LABEL: unary_ops
   gpu.func @unary_ops(%a: memref<24x32xf32>) {
     %tdesc_a = xegpu.create_nd_tdesc %a[0, 0] : memref<24x32xf32>
diff --git a/mlir/test/Examples/transform/Ch1/invalidation-1.mlir b/mlir/test/Examples/transform/Ch1/invalidation-1.mlir
index 2264ade7f9b77..d8b8039631de9 100644
--- a/mlir/test/Examples/transform/Ch1/invalidation-1.mlir
+++ b/mlir/test/Examples/transform/Ch1/invalidation-1.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" \
 // RUN:   --split-input-file --verify-diagnostics
 
@@ -16,7 +16,7 @@ module attributes {transform.with_named_sequence} {
       %arg0: !transform.any_op,
       // expected-note @below {{handle to invalidated ops}}
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // The actual tiling transformation takes tile sizes as attributes.
     // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}}
     %tiled, %loop = transform.structured.tile_using_forall %arg1 tile_sizes [4, 32]
@@ -39,14 +39,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -57,7 +57,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // We can cast one type to another as long as operations are compatible
     // with both types. This creates "aliasing" handles.
     // expected-note @below {{handle to invalidated ops}}
@@ -88,14 +88,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
diff --git a/mlir/test/Examples/transform/Ch1/invalidation-2.mlir b/mlir/test/Examples/transform/Ch1/invalidation-2.mlir
index 0a84a5cb6c68a..e09aca31a90da 100644
--- a/mlir/test/Examples/transform/Ch1/invalidation-2.mlir
+++ b/mlir/test/Examples/transform/Ch1/invalidation-2.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" \
 // RUN:   --split-input-file --verify-diagnostics
 // ****************************** IMPORTANT NOTE ******************************
@@ -23,14 +23,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
   // Elementwise addition.
 
   // expected-note @below {{ancestor payload op}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -47,11 +47,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
   // Since the %arg2 handle is associated with both elementwise operations,
   // we need to split it into two handles so we can target only the second
   // elementwise operation.
-  %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+  %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
       -> (!transform.any_op, !transform.any_op)
 
   // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch1/sequence.mlir b/mlir/test/Examples/transform/Ch1/sequence.mlir
index 3107adccf78fd..f8dfd2de01d97 100644
--- a/mlir/test/Examples/transform/Ch1/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch1/sequence.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" |\
 // RUN: FileCheck %s
 
@@ -20,21 +20,21 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
 
 // CHECK: func @outlined
 // CHECK:   linalg.matmul
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<add>
 
 // CHECK-LABEL: func @fc_relu
 // CHECK: scf.forall
@@ -46,9 +46,9 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 // CHECK:     %[[SLICE8:.+]] = tensor.extract_slice
 // CHECK:     func.call @outlined(%[[SLICE4]], %[[SLICE5]], %[[SLICE6]], %[[SLICE7]], %[[SLICE8]])
 // CHECK-NOT: linalg.matmul
-// CHECK-NOT: linalg.elemwise_binary
+// CHECK-NOT: linalg.elementwise
 // CHECK:     scf.forall.in_parallel
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
 // CHECK:   scf.forall.in_parallel
 
 // Declaration of the "microkernel" function that we will be targeting.
@@ -63,11 +63,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
-    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
   
     // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch2/sequence.mlir b/mlir/test/Examples/transform/Ch2/sequence.mlir
index 976df1d55503a..8d6e129c25f7d 100644
--- a/mlir/test/Examples/transform/Ch2/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch2/sequence.mlir
@@ -1,6 +1,6 @@
 // RUN: transform-opt-ch2 %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" |\
 // RUN: FileCheck %s
 
@@ -20,14 +20,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -42,9 +42,9 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 // CHECK:     %[[SLICE8:.+]] = tensor.extract_slice
 // CHECK:     func.call @microkernel(%[[SLICE4]], %[[SLICE5]], %[[SLICE6]], %[[SLICE7]], %[[SLICE8]])
 // CHECK-NOT: linalg.matmul
-// CHECK-NOT: linalg.elemwise_binary
+// CHECK-NOT: linalg.elementwise
 // CHECK:     scf.forall.in_parallel
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
 // CHECK:   scf.forall.in_parallel
 
 // Declaration of the "microkernel" function that we will be targeting.
@@ -59,11 +59,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
-    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch3/sequence.mlir b/mlir/test/Examples/transform/Ch3/sequence.mlir
index 8dc33c3560c26..4d28518ca3d9e 100644
--- a/mlir/test/Examples/transform/Ch3/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch3/sequence.mlir
@@ -1,6 +1,6 @@
 // RUN: transform-opt-ch3 %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" |\
 // RUN: FileCheck %s
 
@@ -20,14 +20,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -42,9 +42,9 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 // CHECK:     %[[SLICE8:.+]] = tensor.extract_slice
 // CHECK:     func.call @microkernel(%[[SLICE4]], %[[SLICE5]], %[[SLICE6]], %[[SLICE7]], %[[SLICE8]])
 // CHECK-NOT: linalg.matmul
-// CHECK-NOT: linalg.elemwise_binary
+// CHECK-NOT: linalg.elementwise
 // CHECK:     scf.forall.in_parallel
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
 // CHECK:   scf.forall.in_parallel
 
 // Declaration of the "microkernel" function that we will be targeting.
@@ -59,11 +59,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
-    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
   
     // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch4/multiple.mlir b/mlir/test/Examples/transform/Ch4/multiple.mlir
index 2c2c705905521..e6ab5ca65235d 100644
--- a/mlir/test/Examples/transform/Ch4/multiple.mlir
+++ b/mlir/test/Examples/transform/Ch4/multiple.mlir
@@ -12,15 +12,15 @@ func.func @fc_relu_operands_00(
 
   // Elementwise addition.
   // expected-remark @below {{add # 0}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
   // expected-remark @below {{max # 0}}
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -37,15 +37,15 @@ func.func @fc_relu_operands_01(
 
   // Elementwise addition.
   // expected-remark @below {{add # 1}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
   // expected-remark @below {{max # 1}}
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%c0f, %biased : f32, tensor<512x512xf32>)
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%c0f, %biased : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -97,7 +97,7 @@ module @transforms attributes { transform.with_named_sequence } {
       -> (!transform.any_op, !transform.any_op, !transform.any_op,
           !transform.param<i32>) {
     // The last operation must be an elementwise binary.
-    transform.match.operation_name %last ["linalg.elemwise_binary"]
+    transform.match.operation_name %last ["linalg.elementwise"]
       : !transform.any_op
 
     // One of its operands must be defined by another operation, to which we
@@ -111,7 +111,7 @@ module @transforms attributes { transform.with_named_sequence } {
       %def = transform.get_defining_op %operand 
         : (!transform.any_value) -> !transform.any_op
       // The defining operation must itself be an elementwise binary.
-      transform.match.operation_name %def ["linalg.elemwise_binary"]
+      transform.match.operation_name %def ["linalg.elementwise"]
         : !transform.any_op
       transform.yield %def : !transform.any_op
     }
diff --git a/mlir/test/Examples/transform/Ch4/sequence.mlir b/mlir/test/Examples/transform/Ch4/sequence.mlir
index 304533f706413..7dd725dfe6884 100644
--- a/mlir/test/Examples/transform/Ch4/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch4/sequence.mlir
@@ -22,15 +22,15 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 
   // Elementwise addition.
   // expected-remark @below {{elementwise binary}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
   // expected-remark @below {{elementwise binary}}
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -82,7 +82,7 @@ module @transforms attributes { transform.with_named_sequence } {
   // rewriter sequence on success.
   transform.named_sequence @match_elemwise(
       %entry: !transform.any_op {transform.readonly}) -> !transform.any_op {
-    transform.match.operation_name %entry ["linalg.elemwise_binary"] 
+    transform.match.operation_name %entry ["linalg.elementwise"] 
       : !transform.any_op
     transform.yield %entry : !transform.any_op
   }
@@ -113,7 +113,7 @@ module @transforms attributes { transform.with_named_sequence } {
       %last: !transform.any_op {transform.readonly}) 
       -> (!transform.any_op, !transform.any_op, !transform.any_op) {
     // The last operation must be an elementwise binary.
-    transform.match.operation_name %last ["linalg.elemwise_binary"]
+    transform.match.operation_name %last ["linalg.elementwise"]
       : !transform.any_op
     // Its first operand must be defined by another operation, to which we
     // will get a handle here. We are guaranteed that the first operand exists
@@ -123,7 +123,7 @@ module @transforms attributes { transform.with_named_sequence } {
     %middle = transform.get_producer_of_operand %last[0]
       : (!transform.any_op) -> !transform.any_op
     // The defining operation must itself be an elementwise binary.
-    transform.match.operation_name %middle ["linalg.elemwise_binary"]
+    transform.match.operation_name %middle ["linalg.elementwise"]
       : !transform.any_op
     // And the first operand of that operation must be defined by yet another
     // operation.
diff --git a/mlir/test/IR/affine-map.mlir b/mlir/test/IR/affine-map.mlir
index 977aec2536b1e..6277b28561f36 100644
--- a/mlir/test/IR/affine-map.mlir
+++ b/mlir/test/IR/affine-map.mlir
@@ -139,7 +139,7 @@
 #map44 = affine_map<(i, j) -> (i - 2*j, j * 6 floordiv 4)>
 
 // Simplifications
-// CHECK: #map{{[0-9]*}} = affine_map<(d0, d1, d2)[s0] -> (d0 + d1 + d2 + 1, d2 + d1, (d0 * s0) * 8)>
+// CHECK: #map{{[0-9]*}} = affine_map<(d0, d1, d2)[s0] -> (d0 + d1 + d2 + 1, d1 + d2, (d0 * s0) * 8)>
 #map45 = affine_map<(i, j, k) [N] -> (1 + i + 3 + j - 3 + k, k + 5 + j - 5, 2*i*4*N)>
 
 // CHECK: #map{{[0-9]*}} = affine_map<(d0, d1, d2) -> (0, d1, d0 * 2, 0)>
diff --git a/mlir/test/IR/properties.mlir b/mlir/test/IR/properties.mlir
index dde9100cde142..b541447e85c7b 100644
--- a/mlir/test/IR/properties.mlir
+++ b/mlir/test/IR/properties.mlir
@@ -2,10 +2,10 @@
 // # RUN: mlir-opt %s -mlir-print-op-generic -split-input-file  | mlir-opt -mlir-print-op-generic | FileCheck %s --check-prefix=GENERIC
 
 // CHECK:   test.with_properties
-// CHECK-SAME: a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4]{{$}}
+// CHECK-SAME: a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4], array32 = [5, 6]{{$}}
 // GENERIC:   "test.with_properties"()
-// GENERIC-SAME: <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, b = "foo", c = "bar", flag = true}> : () -> ()
-test.with_properties a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4]
+// GENERIC-SAME: <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, array32 = array<i32: 5, 6>, b = "foo", c = "bar", flag = true}> : () -> ()
+test.with_properties a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4], array32 = [5, 6]
 
 // CHECK:   test.with_nice_properties
 // CHECK-SAME:    "foo bar" is -3{{$}}
diff --git a/mlir/test/IR/region.mlir b/mlir/test/IR/region.mlir
index 0b959915d6bbb..e2088817c5204 100644
--- a/mlir/test/IR/region.mlir
+++ b/mlir/test/IR/region.mlir
@@ -106,3 +106,10 @@ func.func @named_region_has_wrong_number_of_blocks() {
 test.single_no_terminator_custom_asm_op {
   "important_dont_drop"() : () -> ()
 }
+
+// -----
+
+// CHECK: test.dummy_op_with_region_ref
+test.dummy_op_with_region_ref {
+  ^bb0:
+}
diff --git a/mlir/test/Integration/Dialect/Transform/match_matmul.mlir b/mlir/test/Integration/Dialect/Transform/match_matmul.mlir
index 7b695cb027252..a374d9a611258 100644
--- a/mlir/test/Integration/Dialect/Transform/match_matmul.mlir
+++ b/mlir/test/Integration/Dialect/Transform/match_matmul.mlir
@@ -69,7 +69,7 @@ func.func @matmul_with_extra_ops_in_func(%lhs: tensor<10x20xf32>, %rhs: tensor<2
   // expected-remark @below {{fill}}
   %fill = linalg.fill ins(%cst : f64) outs(%empty : tensor<10x15xf32>) -> tensor<10x15xf32>
 
-  %real_lhs = linalg.elemwise_binary { fun = #linalg.binary_fn<mul> } 
+  %real_lhs = linalg.mul
     ins(%lhs, %lhs : tensor<10x20xf32>, tensor<10x20xf32>) outs(%lhs : tensor<10x20xf32>) -> tensor<10x20xf32>
 
   // expected-remark @below {{matmul}}
diff --git a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
index 0f69875d596f1..d09373bdb3f14 100644
--- a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
+++ b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
@@ -19,7 +19,7 @@ module {
     }
     %in_operand_2 = tensor.empty() : tensor<64xf32>
     %out_operand_3 = tensor.empty() : tensor<64xf32>
-    %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>} ins(%1#1, %in_operand_2 : tensor<64xf32>, tensor<64xf32>) outs(%out_operand_3 : tensor<64xf32>) -> tensor<64xf32>
+    %2 = linalg.add ins(%1#1, %in_operand_2 : tensor<64xf32>, tensor<64xf32>) outs(%out_operand_3 : tensor<64xf32>) -> tensor<64xf32>
     return %2 : tensor<64xf32>
   }
 }
@@ -50,7 +50,7 @@ module attributes {transform.with_named_sequence} {
 //      CHECK:      %[[INSERT_MAT:.*]] = tensor.insert_slice %[[MAT_OUT]] into %[[FIRST_OUT_ARG]][%[[IV]]] [32] [1]
 //      CHECK:      %[[SLICE_OPERAND2:.*]] = tensor.extract_slice %0[%[[IV]]] [32] [1]
 //      CHECK:      %[[SLICE_OUT:.*]] = tensor.extract_slice %[[ELEM_OUT_ARG]][%[[IV]]] [32] [1]
-//      CHECK:      %[[ELEM_OUT:.*]] = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+//      CHECK:      %[[ELEM_OUT:.*]] = linalg.add
 // CHECK-SAME:              ins(%[[MAT_OUT]], %[[SLICE_OPERAND2]] :
 // CHECK-SAME:              outs(%[[SLICE_OUT]] :
 //      CHECK:      %[[INSERT_ELEM:.*]] = tensor.insert_slice %[[ELEM_OUT]] into %[[ELEM_OUT_ARG]][%[[IV]]] [32] [1]
@@ -76,7 +76,7 @@ module {
     }
     %in_operand_2 = tensor.empty() : tensor<64x64xf32>
     %out_operand_3 = tensor.empty() : tensor<64x64xf32>
-    %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>} ins(%1#1, %in_operand_2 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%out_operand_3 : tensor<64x64xf32>) -> tensor<64x64xf32>
+    %2 = linalg.add ins(%1#1, %in_operand_2 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%out_operand_3 : tensor<64x64xf32>) -> tensor<64x64xf32>
     return %2 : tensor<64x64xf32>
   }
 }
@@ -109,7 +109,7 @@ module attributes {transform.with_named_sequence} {
 // CHECK-SAME:              outs(%[[MAT_OUT_SLICE]] :
 //      CHECK:      %[[SLICE_OPERAND2:.*]] = tensor.extract_slice %[[OUT_INIT]][%[[IV1]], %[[IV2]]] [32, 32] [1, 1]
 //      CHECK:      %[[SLICE_OUT:.*]] = tensor.extract_slice %[[ELEM_OUT_ARG]][%[[IV1]], %[[IV2]]] [32, 32] [1, 1]
-//      CHECK:      %[[ELEM_OUT:.*]] = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+//      CHECK:      %[[ELEM_OUT:.*]] = linalg.add
 // CHECK-SAME:              ins(%[[MAT_OUT]], %[[SLICE_OPERAND2]] :
 // CHECK-SAME:              outs(%[[SLICE_OUT]] :
 //      CHECK:      scf.forall.in_parallel {
diff --git a/mlir/test/Target/Cpp/for_loop_induction_vars.mlir b/mlir/test/Target/Cpp/for_loop_induction_vars.mlir
new file mode 100644
index 0000000000000..aa85612aef36a
--- /dev/null
+++ b/mlir/test/Target/Cpp/for_loop_induction_vars.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s
+
+// CHECK-LABEL: test_for_siblings
+func.func @test_for_siblings() {
+  %start = emitc.literal "0" : index
+  %stop = emitc.literal "10" : index
+  %step = emitc.literal "1" : index
+
+  %var1 = "emitc.variable"() <{value = 0 : index}> : () -> !emitc.lvalue<index>
+  %var2 = "emitc.variable"() <{value = 0 : index}> : () -> !emitc.lvalue<index>
+
+  // CHECK: for (size_t [[ITER0:i[0-9]*]] = {{.*}}; [[ITER0]] < {{.*}}; [[ITER0]] += {{.*}}) {
+  emitc.for %i0 = %start to %stop step %step {
+    // CHECK: for (size_t [[ITER1:j[0-9]*]] = {{.*}}; [[ITER1]] < {{.*}}; [[ITER1]] += {{.*}}) {
+    emitc.for %i1 = %start to %stop step %step {
+      // CHECK: {{.*}} = [[ITER0]];
+      //"emitc.assign"(%var1,%i0) : (!emitc.lvalue<!emitc.size_t>, !emitc.size_t) -> ()
+      emitc.assign %i0 : index to %var1 : !emitc.lvalue<index>
+      // CHECK: {{.*}} = [[ITER1]];
+      //"emitc.assign"(%var2,%i1) : (!emitc.lvalue<!emitc.size_t>, !emitc.size_t) -> ()
+      emitc.assign %i1 : index to %var2 : !emitc.lvalue<index>
+    }
+  }
+  // CHECK: for (size_t [[ITER2:i[0-9]*]] = {{.*}}; [[ITER2]] < {{.*}}; [[ITER2]] += {{.*}})
+  emitc.for %ki2 = %start to %stop step %step {
+    // CHECK: for (size_t [[ITER3:j[0-9]*]] = {{.*}}; [[ITER3]] < {{.*}}; [[ITER3]] += {{.*}})
+    emitc.for %i3 = %start to %stop step %step {
+      %1 = emitc.call_opaque "f"() : () -> i32
+    }
+  }
+  return
+}
+
+// CHECK-LABEL: test_for_nesting
+func.func @test_for_nesting() {
+  %start = emitc.literal "0" : index
+  %stop = emitc.literal "10" : index
+  %step = emitc.literal "1" : index
+
+  // CHECK-COUNT-12: for (size_t [[ITER:[i-t][0-9]*]] = {{.*}}; [[ITER]] < {{.*}}; [[ITER]] += {{.*}}) {
+  emitc.for %i0 = %start to %stop step %step {
+    emitc.for %i1 = %start to %stop step %step {
+      emitc.for %i2 = %start to %stop step %step {
+        emitc.for %i3 = %start to %stop step %step {
+          emitc.for %i4 = %start to %stop step %step {
+            emitc.for %i5 = %start to %stop step %step {
+              emitc.for %i6 = %start to %stop step %step {
+                emitc.for %i7 = %start to %stop step %step {
+                  emitc.for %i8 = %start to %stop step %step {
+                    emitc.for %i9 = %start to %stop step %step {
+                      emitc.for %i10 = %start to %stop step %step {
+                        emitc.for %i11 = %start to %stop step %step {
+                          // CHECK: for (size_t [[ITERu0:u13]] = {{.*}}; [[ITERu0]] < {{.*}}; [[ITERu0]] += {{.*}}) {
+                          emitc.for %i14 = %start to %stop step %step {
+                            // CHECK: for (size_t [[ITERu1:u14]] = {{.*}}; [[ITERu1]] < {{.*}}; [[ITERu1]] += {{.*}}) {
+                            emitc.for %i15 = %start to %stop step %step {
+                              %0 = emitc.call_opaque "f"() : () -> i32
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  return
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 0000000000000..45e5d2612e2c2
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @test() {
+    omp.target {
+      omp.terminator
+    }  loc(#loc4)
+    llvm.return
+  }  loc(#loc3)
+}
+#file = #llvm.di_file<"target.f90" in "">
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+ emissionKind = Full>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
+#sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
+ name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
+#loc1 = loc("target.f90":1:1)
+#loc2 = loc("target.f90":46:3)
+#loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc2])
+
+// CHECK: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
index a755cef98d7c4..c3320382f8d45 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
@@ -41,7 +41,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
         omp.terminator loc(#loc9)
       } loc(#loc9)
       omp.terminator loc(#loc9)
-    } loc(#loc9)
+    } loc(#loc15)
     llvm.return loc(#loc9)
   } loc(#loc14)
   llvm.mlir.global internal @_QFEarray() {addr_space = 0 : i32} : !llvm.array<16384 x i32> {
@@ -59,7 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = true, emissionKind = LineTablesOnly>
 #di_subroutine_type = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #di_subprogram = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #di_compile_unit, scope = #di_file, name = "main", file = #di_file, subprogramFlags = "Definition|Optimized|MainSubprogram", type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit, name = "target", file = #di_file, subprogramFlags = "Definition", type = #di_subroutine_type>
 #loc14 = loc(fused<#di_subprogram>[#loc3])
+#loc15 = loc(fused<#di_subprogram1>[#loc9])
 
 
 // CHECK: call void @__kmpc_distribute_static{{.*}}!dbg
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
index 4495e1559a889..492610251769c 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
@@ -16,7 +16,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %16 = llvm.load %arg1 : !llvm.ptr -> i32 loc(#loc5)
       llvm.store %16, %arg0 : i32, !llvm.ptr loc(#loc5)
       omp.terminator loc(#loc5)
-    } loc(#loc5)
+    } loc(#loc16)
     llvm.return loc(#loc6)
   } loc(#loc15)
 }
@@ -31,9 +31,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
   compileUnit = #di_compile_unit, scope = #di_file, name = "main",
   file = #di_file, subprogramFlags = "Definition|MainSubprogram",
   type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit,
+  name = "target", file = #di_file, subprogramFlags = "Definition",
+  type = #di_subroutine_type>
 #loc1 = loc("test.f90":3:18)
 #loc2 = loc("test.f90":7:7)
 #loc3 = loc("test.f90":9:18)
 #loc5 = loc("test.f90":11:7)
 #loc6 = loc("test.f90":12:7)
 #loc15 = loc(fused<#di_subprogram>[#loc2])
+#loc16 = loc(fused<#di_subprogram1>[#loc5])
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
index eaa88d9dd6053..3bd724f42e8ce 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
@@ -20,7 +20,7 @@ module attributes {omp.is_target_device = false} {
     ^bb3:  // pred: ^bb1
       llvm.store %13, %arg1 : i32, !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc3)
     llvm.return
   } loc(#loc2)
 }
@@ -34,7 +34,10 @@ module attributes {omp.is_target_device = false} {
  types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, name = "main", file=#file,
  subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, name = "target", file=#file,
+ subprogramFlags = "Definition", type = #sp_ty>
 
 #loc1 = loc("test.f90":6:7)
 #loc2 = loc(fused<#sp>[#loc1])
+#loc3 = loc(fused<#sp1>[#loc1])
 
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
index ea92589bbd031..8f42995af23a8 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui32>>, llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_target_device = true} {
@@ -47,7 +49,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -57,8 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "dyn_ptr", arg: 1, scope: ![[SP]]{{.*}}flags: DIFlagArtificial)
 // CHECK: !DILocalVariable(name: "x", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
index 22db86fd85e2c..11a07dfd9a180 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {omp.is_target_device = false} {
@@ -45,7 +47,7 @@ module attributes {omp.is_target_device = false} {
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    }  loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -55,8 +57,9 @@ module attributes {omp.is_target_device = false} {
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "i", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
index 9c8344d69dc74..ab687f198b9b4 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
@@ -10,7 +10,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -21,9 +21,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
index 78dc6e18a40a7..6cf75af38f916 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
@@ -11,7 +11,7 @@ module attributes {omp.is_target_device = false} {
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    }  loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -22,9 +22,13 @@ module attributes {omp.is_target_device = false} {
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
index 3c45f1f1c76fb..b18338ea35cc3 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
@@ -6,8 +6,10 @@
 #cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = false, emissionKind = Full>
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "test", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "kernel", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
 #int_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
 #var_x = #llvm.di_local_variable<scope = #sp, name = "x", file = #di_file, type = #int_ty>
+#var_x1 = #llvm.di_local_variable<scope = #sp1, name = "x", file = #di_file, type = #int_ty>
 module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, f128 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, f80 = dense<128> : vector<2xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr = dense<64> : vector<4xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, i1 = dense<8> : vector<2xi64>, i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, "dlti.endianness" = "little", "dlti.stack_alignment" = 128 : i64, "dlti.mangling_mode" = "e">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", fir.target_cpu = "x86-64", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 21.0.0 (/home/haqadeer/work/src/aomp-llvm-project/flang 793f9220ab32f92fc3b253efec2e332c18090e53)", llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_gpu = false, omp.is_target_device = false, omp.requires = #omp<clause_requires none>, omp.target_triples = ["amdgcn-amd-amdhsa"], omp.version = #omp.version<version = 52>} {
   llvm.func @_QQmain() attributes {fir.bindc_name = "test", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
     %0 = llvm.mlir.constant(1 : i64) : i64
@@ -16,7 +18,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
     %5 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "x"}
     omp.target map_entries(%5 -> %arg0 : !llvm.ptr) {
       %6 = llvm.mlir.constant(1 : i32) : i32
-      llvm.intr.dbg.declare #var_x = %arg0 : !llvm.ptr loc(#loc2)
+      llvm.intr.dbg.declare #var_x1 = %arg0 : !llvm.ptr loc(#loc3)
       omp.parallel {
         %7 = llvm.load %arg0 : !llvm.ptr -> i32
         %8 = llvm.add %7, %6 : i32
@@ -24,12 +26,14 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
         omp.terminator
       }
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc10)
 }
 #loc1 = loc("target.f90":1:7)
 #loc2 = loc("target.f90":3:18)
+#loc3 = loc("target.f90":6:18)
+#loc4 = loc(fused<#sp1>[#loc3])
 #loc10 = loc(fused<#sp>[#loc1])
 
 
diff --git a/mlir/test/Target/LLVMIR/openmp-parallel-do-simd.mlir b/mlir/test/Target/LLVMIR/openmp-parallel-do-simd.mlir
new file mode 100644
index 0000000000000..bfa57b85245ca
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-parallel-do-simd.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Check that omp.simd as a leaf of a composite construct still generates
+// the appropriate loop vectorization attribute.
+
+// CHECK-LABEL: define internal void @test_parallel_do_simd..omp_par
+// CHECK: omp.par.entry:
+// CHECK: omp.par.region:
+// CHECK: omp_loop.header:
+// CHECK: omp_loop.inc:
+// CHECK-NEXT:   %omp_loop.next = add nuw i32 %omp_loop.iv, 1
+// CHECK-NEXT:   br label %omp_loop.header, !llvm.loop ![[LOOP_ATTR:.*]]
+// CHECK: ![[LOOP_ATTR]] = distinct !{![[LOOP_ATTR]], ![[LPAR:.*]], ![[LVEC:.*]]}
+// CHECK: ![[LPAR]] = !{!"llvm.loop.parallel_accesses", ![[PAR_ACC:.*]]}
+// CHECK: ![[LVEC]] = !{!"llvm.loop.vectorize.enable", i1 true}
+
+llvm.func @test_parallel_do_simd() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %2 = llvm.mlir.constant(1000 : i32) : i32
+  %3 = llvm.mlir.constant(1 : i32) : i32
+  %4 = llvm.mlir.constant(1 : i64) : i64
+  omp.parallel {
+    %5 = llvm.mlir.constant(1 : i64) : i64
+    %6 = llvm.alloca %5 x i32 {bindc_name = "i", pinned} : (i64) -> !llvm.ptr
+    %7 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop {
+      omp.simd {
+        omp.loop_nest (%arg0) : i32 = (%3) to (%2) inclusive step (%3) {
+          llvm.store %arg0, %6 : i32, !llvm.ptr
+          omp.yield
+        }
+      } {omp.composite}
+    } {omp.composite}
+    omp.terminator
+  }
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-reduction-byref.mlir b/mlir/test/Target/LLVMIR/openmp-simd-reduction-byref.mlir
new file mode 100644
index 0000000000000..3e73fd22e69ca
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-simd-reduction-byref.mlir
@@ -0,0 +1,98 @@
+// RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
+
+llvm.func @init(%arg0: !llvm.ptr {llvm.nocapture}, %arg1: !llvm.ptr {llvm.nocapture}) {
+  llvm.return
+}
+llvm.func @combine(%arg0: !llvm.ptr {llvm.nocapture}, %arg1: !llvm.ptr {llvm.nocapture}) {
+  llvm.return
+}
+llvm.func @cleanup(%arg0: !llvm.ptr {llvm.nocapture}) {
+  llvm.return
+}
+omp.private {type = private} @_QFsimd_reductionEi_private_i32 : i32
+omp.declare_reduction @add_reduction_byref_box_2xf32 : !llvm.ptr alloc {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> : (i64) -> !llvm.ptr
+  omp.yield(%1 : !llvm.ptr)
+} init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  llvm.call @init(%arg0, %arg1) : (!llvm.ptr, !llvm.ptr) -> ()
+  omp.yield(%arg1 : !llvm.ptr)
+} combiner {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  llvm.call @combine(%arg0, %arg1) : (!llvm.ptr, !llvm.ptr) -> ()
+  omp.yield(%arg0 : !llvm.ptr)
+} cleanup {
+^bb0(%arg0: !llvm.ptr):
+  llvm.call @cleanup(%arg0) : (!llvm.ptr) -> ()
+  omp.yield
+}
+llvm.func @_QPsimd_reduction(%arg0: !llvm.ptr {fir.bindc_name = "a", llvm.nocapture}, %arg1: !llvm.ptr {fir.bindc_name = "sum", llvm.nocapture}) {
+  %0 = llvm.mlir.constant(1024 : i32) : i32
+  %1 = llvm.mlir.constant(1 : i32) : i32
+  %2 = llvm.mlir.constant(1 : i64) : i64
+  %3 = llvm.alloca %2 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> : (i64) -> !llvm.ptr
+  %4 = llvm.alloca %2 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  omp.simd private(@_QFsimd_reductionEi_private_i32 %4 -> %arg2 : !llvm.ptr) reduction(byref @add_reduction_byref_box_2xf32 %3 -> %arg3 : !llvm.ptr) {
+    omp.loop_nest (%arg4) : i32 = (%1) to (%0) inclusive step (%1) {
+      llvm.store %arg4, %arg2 : i32, !llvm.ptr
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// CHECK-LABEL: define void @_QPsimd_reduction
+// CHECK:         %[[MOLD:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
+// CHECK:         %[[ORIG_I:.*]] = alloca i32, i64 1, align 4
+// CHECK:         %[[PRIV_I:.*]] = alloca i32, align 4
+// CHECK:         %[[RED_VAR:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
+// CHECK:         %[[PTR_RED_VAR:.*]] = alloca ptr, align 8
+// CHECK:         br label %[[VAL_5:.*]]
+// CHECK:       omp.region.after_alloca:                          ; preds = %[[VAL_6:.*]]
+// CHECK:         br label %[[VAL_7:.*]]
+// CHECK:       entry:                                            ; preds = %[[VAL_5]]
+// CHECK:         br label %[[VAL_8:.*]]
+// CHECK:       omp.private.init:                                 ; preds = %[[VAL_7]]
+// CHECK:         br label %[[VAL_9:.*]]
+// CHECK:       omp.reduction.init:                               ; preds = %[[VAL_8]]
+// CHECK:         store ptr %[[RED_VAR]], ptr %[[PTR_RED_VAR]], align 8
+// CHECK:         call void @init(ptr %[[MOLD]], ptr %[[RED_VAR]])
+// CHECK:         br label %[[VAL_10:.*]]
+// CHECK:       omp.simd.region:                                  ; preds = %[[VAL_9]]
+// CHECK:         br label %[[VAL_11:.*]]
+// CHECK:       omp_loop.preheader:                               ; preds = %[[VAL_10]]
+// CHECK:         br label %[[VAL_12:.*]]
+// CHECK:       omp_loop.header:                                  ; preds = %[[VAL_13:.*]], %[[VAL_11]]
+// CHECK:         %[[VAL_14:.*]] = phi i32 [ 0, %[[VAL_11]] ], [ %[[VAL_15:.*]], %[[VAL_13]] ]
+// CHECK:         br label %[[VAL_16:.*]]
+// CHECK:       omp_loop.cond:                                    ; preds = %[[VAL_12]]
+// CHECK:         %[[VAL_17:.*]] = icmp ult i32 %[[VAL_14]], 1024
+// CHECK:         br i1 %[[VAL_17]], label %[[VAL_18:.*]], label %[[VAL_19:.*]]
+// CHECK:       omp_loop.body:                                    ; preds = %[[VAL_16]]
+// CHECK:         %[[VAL_20:.*]] = mul i32 %[[VAL_14]], 1
+// CHECK:         %[[VAL_21:.*]] = add i32 %[[VAL_20]], 1
+// CHECK:         br label %[[VAL_22:.*]]
+// CHECK:       omp.loop_nest.region:                             ; preds = %[[VAL_18]]
+// CHECK:         store i32 %[[VAL_21]], ptr %[[PRIV_I]], align 4, !llvm.access.group ![[ACCESS_GROUP:.*]]
+// CHECK:         br label %[[VAL_23:.*]]
+// CHECK:       omp.region.cont1:                                 ; preds = %[[VAL_22]]
+// CHECK:         br label %[[VAL_13]]
+// CHECK:       omp_loop.inc:                                     ; preds = %[[VAL_23]]
+// CHECK:         %[[VAL_15]] = add nuw i32 %[[VAL_14]], 1
+// CHECK:         br label %[[VAL_12]], !llvm.loop ![[LOOP:.*]]
+// CHECK:       omp_loop.exit:                                    ; preds = %[[VAL_16]]
+// CHECK:         br label %[[VAL_24:.*]]
+// CHECK:       omp_loop.after:                                   ; preds = %[[VAL_19]]
+// CHECK:         br label %[[VAL_25:.*]]
+// CHECK:       omp.region.cont:                                  ; preds = %[[VAL_24]]
+// CHECK:         %[[RED_VAR2:.*]] = load ptr, ptr %[[PTR_RED_VAR]], align 8
+// CHECK:         call void @combine(ptr %[[MOLD]], ptr %[[RED_VAR2]])
+// CHECK:         %[[RED_VAR3:.*]] = load ptr, ptr %[[PTR_RED_VAR]], align 8
+// CHECK:         call void @cleanup(ptr %[[RED_VAR3]])
+// CHECK:         ret void
+
+// CHECK: ![[ACCESS_GROUP]] = distinct !{}
+// CHECK: ![[LOOP]] = distinct !{![[LOOP]], ![[PARALLEL_ACCESS:.*]], ![[VECTORIZE:.*]]}
+// CHECK: ![[PARALLEL_ACCESS]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP]]}
+// CHECK: ![[VECTORIZE]] = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-reduction-simple.mlir b/mlir/test/Target/LLVMIR/openmp-simd-reduction-simple.mlir
new file mode 100644
index 0000000000000..ab3c87cca56ea
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-simd-reduction-simple.mlir
@@ -0,0 +1,96 @@
+// RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
+
+omp.private {type = private} @_QFsimd_reductionEi_private_i32 : i32
+omp.declare_reduction @add_reduction_f32 : f32 init {
+^bb0(%arg0: f32):
+  %0 = llvm.mlir.constant(0.000000e+00 : f32) : f32
+  omp.yield(%0 : f32)
+} combiner {
+^bb0(%arg0: f32, %arg1: f32):
+  %0 = llvm.fadd %arg0, %arg1 {fastmathFlags = #llvm.fastmath<contract>} : f32
+  omp.yield(%0 : f32)
+}
+llvm.func @_QPsimd_reduction(%arg0: !llvm.ptr {fir.bindc_name = "a", llvm.nocapture}, %arg1: !llvm.ptr {fir.bindc_name = "sum", llvm.nocapture}) {
+  %0 = llvm.mlir.constant(0.000000e+00 : f32) : f32
+  %1 = llvm.mlir.constant(1 : i32) : i32
+  %2 = llvm.mlir.constant(1024 : i32) : i32
+  %3 = llvm.mlir.constant(1 : i64) : i64
+  %4 = llvm.alloca %3 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  llvm.store %0, %arg1 : f32, !llvm.ptr
+  omp.simd private(@_QFsimd_reductionEi_private_i32 %4 -> %arg2 : !llvm.ptr) reduction(@add_reduction_f32 %arg1 -> %arg3 : !llvm.ptr) {
+    omp.loop_nest (%arg4) : i32 = (%1) to (%2) inclusive step (%1) {
+      llvm.store %arg4, %arg2 : i32, !llvm.ptr
+      %5 = llvm.load %arg3 : !llvm.ptr -> f32
+      %6 = llvm.load %arg2 : !llvm.ptr -> i32
+      %7 = llvm.sext %6 : i32 to i64
+      %8 = llvm.sub %7, %3 overflow<nsw> : i64
+      %9 = llvm.getelementptr %arg0[%8] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+      %10 = llvm.load %9 : !llvm.ptr -> f32
+      %11 = llvm.fadd %5, %10 {fastmathFlags = #llvm.fastmath<contract>} : f32
+      llvm.store %11, %arg3 : f32, !llvm.ptr
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// CHECK-LABEL: define void @_QPsimd_reduction(
+// CHECK:         %[[ORIG_I:.*]] = alloca i32, i64 1, align 4
+// CHECK:         store float 0.000000e+00, ptr %[[ORIG_SUM:.*]], align 4
+// CHECK:         %[[PRIV_I:.*]] = alloca i32, align 4
+// CHECK:         %[[RED_VAR:.*]] = alloca float, align 4
+// CHECK:         br label %[[VAL_4:.*]]
+// CHECK:       omp.region.after_alloca:                          ; preds = %[[VAL_5:.*]]
+// CHECK:         br label %[[VAL_6:.*]]
+// CHECK:       entry:                                            ; preds = %[[VAL_4]]
+// CHECK:         br label %[[VAL_7:.*]]
+// CHECK:       omp.private.init:                                 ; preds = %[[VAL_6]]
+// CHECK:         br label %[[VAL_8:.*]]
+// CHECK:       omp.reduction.init:                               ; preds = %[[VAL_7]]
+// CHECK:         store float 0.000000e+00, ptr %[[RED_VAR]], align 4
+// CHECK:         br label %[[VAL_9:.*]]
+// CHECK:       omp.simd.region:                                  ; preds = %[[VAL_8]]
+// CHECK:         br label %[[VAL_10:.*]]
+// CHECK:       omp_loop.preheader:                               ; preds = %[[VAL_9]]
+// CHECK:         br label %[[VAL_11:.*]]
+// CHECK:       omp_loop.header:                                  ; preds = %[[VAL_12:.*]], %[[VAL_10]]
+// CHECK:         %[[VAL_13:.*]] = phi i32 [ 0, %[[VAL_10]] ], [ %[[VAL_14:.*]], %[[VAL_12]] ]
+// CHECK:         br label %[[VAL_15:.*]]
+// CHECK:       omp_loop.cond:                                    ; preds = %[[VAL_11]]
+// CHECK:         %[[VAL_16:.*]] = icmp ult i32 %[[VAL_13]], 1024
+// CHECK:         br i1 %[[VAL_16]], label %[[VAL_17:.*]], label %[[VAL_18:.*]]
+// CHECK:       omp_loop.body:                                    ; preds = %[[VAL_15]]
+// CHECK:         %[[VAL_19:.*]] = mul i32 %[[VAL_13]], 1
+// CHECK:         %[[VAL_20:.*]] = add i32 %[[VAL_19]], 1
+// CHECK:         br label %[[VAL_21:.*]]
+// CHECK:       omp.loop_nest.region:                             ; preds = %[[VAL_17]]
+// CHECK:         store i32 %[[VAL_20]], ptr %[[PRIV_I]], align 4, !llvm.access.group ![[ACCESS_GROUP:.*]]
+// CHECK:         %[[RED_VAL:.*]] = load float, ptr %[[RED_VAR]], align 4, !llvm.access.group ![[ACCESS_GROUP]]
+// CHECK:         %[[VAL_23:.*]] = load i32, ptr %[[PRIV_I]], align 4, !llvm.access.group ![[ACCESS_GROUP]]
+// CHECK:         %[[VAL_24:.*]] = sext i32 %[[VAL_23]] to i64
+// CHECK:         %[[VAL_25:.*]] = sub nsw i64 %[[VAL_24]], 1
+// CHECK:         %[[VAL_26:.*]] = getelementptr float, ptr %[[VAL_27:.*]], i64 %[[VAL_25]]
+// CHECK:         %[[VAL_28:.*]] = load float, ptr %[[VAL_26]], align 4, !llvm.access.group ![[ACCESS_GROUP]]
+// CHECK:         %[[VAL_29:.*]] = fadd contract float %[[RED_VAL]], %[[VAL_28]]
+// CHECK:         store float %[[VAL_29]], ptr %[[RED_VAR]], align 4, !llvm.access.group ![[ACCESS_GROUP]]
+// CHECK:         br label %[[VAL_30:.*]]
+// CHECK:       omp.region.cont1:                                 ; preds = %[[VAL_21]]
+// CHECK:         br label %[[VAL_12]]
+// CHECK:       omp_loop.inc:                                     ; preds = %[[VAL_30]]
+// CHECK:         %[[VAL_14]] = add nuw i32 %[[VAL_13]], 1
+// CHECK:         br label %[[VAL_11]], !llvm.loop ![[LOOP:.*]]
+// CHECK:       omp_loop.exit:                                    ; preds = %[[VAL_15]]
+// CHECK:         br label %[[VAL_31:.*]]
+// CHECK:       omp_loop.after:                                   ; preds = %[[VAL_18]]
+// CHECK:         br label %[[VAL_32:.*]]
+// CHECK:       omp.region.cont:                                  ; preds = %[[VAL_31]]
+// CHECK:         %[[SUM_VAL:.*]] = load float, ptr %[[ORIG_SUM]], align 4
+// CHECK:         %[[RED_VAL:.*]] = load float, ptr %[[RED_VAR]], align 4
+// CHECK:         %[[COMBINED_VAL:.*]] = fadd contract float %[[SUM_VAL]], %[[RED_VAL]]
+// CHECK:         store float %[[COMBINED_VAL]], ptr %[[ORIG_SUM]], align 4
+// CHECK:         ret void
+
+// CHECK: ![[ACCESS_GROUP]] = distinct !{}
+// CHECK: ![[LOOP]] = distinct !{![[LOOP]], ![[PARALLEL_ACCESS:.*]], ![[VECTORIZE:.*]]}
+// CHECK: ![[PARALLEL_ACCESS]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP]]}
+// CHECK: ![[VECTORIZE]] = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/mlir/test/Target/LLVMIR/openmp-teams-distribute-parallel-do-simd.mlir b/mlir/test/Target/LLVMIR/openmp-teams-distribute-parallel-do-simd.mlir
new file mode 100644
index 0000000000000..4d766cc1ac4f4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-teams-distribute-parallel-do-simd.mlir
@@ -0,0 +1,51 @@
+// RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
+
+// Check that omp.simd as a leaf of a composite construct still generates
+// the appropriate loop vectorization attribute.
+
+// CHECK-LABEL: define internal void @test_teams_distribute_parallel_do_simd..omp_par.2
+// CHECK: teams.body:
+// CHECK: omp.teams.region:
+
+// CHECK-LABEL: define internal void @test_teams_distribute_parallel_do_simd..omp_par.1
+// CHECK: omp.par.entry:
+// CHECK: omp.par.region:
+// CHECK: distribute.exit:
+
+// CHECK-LABEL: define internal void @test_teams_distribute_parallel_do_simd..omp_par
+// CHECK: distribute.body:
+// CHECK: omp.distribute.region:
+// CHECK: omp_loop.header:
+// CHECK: omp_loop.inc:
+// CHECK-NEXT:   %omp_loop.next = add nuw i32 %omp_loop.iv, 1
+// CHECK-NEXT:   br label %omp_loop.header, !llvm.loop ![[LOOP_ATTR:.*]]
+
+// CHECK: ![[LOOP_ATTR]] = distinct !{![[LOOP_ATTR]], ![[LPAR:.*]], ![[LVEC:.*]]}
+// CHECK: ![[LPAR]] = !{!"llvm.loop.parallel_accesses", ![[PAR_ACC:.*]]}
+// CHECK: ![[LVEC]] = !{!"llvm.loop.vectorize.enable", i1 true}
+
+omp.private {type = private} @_QFEi_private_i32 : i32
+llvm.func @test_teams_distribute_parallel_do_simd() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+  %2 = llvm.mlir.constant(1000 : i32) : i32
+  %3 = llvm.mlir.constant(1 : i32) : i32
+  %4 = llvm.mlir.constant(1 : i64) : i64
+  omp.teams {
+    omp.parallel {
+      omp.distribute {
+        omp.wsloop {
+          omp.simd private(@_QFEi_private_i32 %1 -> %arg0 : !llvm.ptr) {
+            omp.loop_nest (%arg1) : i32 = (%3) to (%2) inclusive step (%3) {
+              llvm.store %arg1, %arg0 : i32, !llvm.ptr
+              omp.yield
+            }
+          } {omp.composite}
+        } {omp.composite}
+      } {omp.composite}
+      omp.terminator
+    } {omp.composite}
+    omp.terminator
+  }
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 97608ca3b4df1..e2c32b254c200 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -26,20 +26,6 @@ llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
 
 // -----
 
-llvm.func @do_simd(%lb : i32, %ub : i32, %step : i32) {
-  omp.wsloop {
-    // expected-warning@below {{simd information on composite construct discarded}}
-    omp.simd {
-      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-        omp.yield
-      }
-    } {omp.composite}
-  } {omp.composite}
-  llvm.return
-}
-
-// -----
-
 llvm.func @distribute_allocate(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
   // expected-error@below {{not yet implemented: Unhandled clause allocate in omp.distribute operation}}
   // expected-error@below {{LLVM Translation failed for operation: omp.distribute}}
@@ -143,36 +129,6 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
 
 // -----
 
-omp.declare_reduction @add_f32 : f32
-init {
-^bb0(%arg: f32):
-  %0 = llvm.mlir.constant(0.0 : f32) : f32
-  omp.yield (%0 : f32)
-}
-combiner {
-^bb1(%arg0: f32, %arg1: f32):
-  %1 = llvm.fadd %arg0, %arg1 : f32
-  omp.yield (%1 : f32)
-}
-atomic {
-^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
-  %2 = llvm.load %arg3 : !llvm.ptr -> f32
-  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
-  omp.yield
-}
-llvm.func @simd_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error@below {{not yet implemented: Unhandled clause reduction in omp.simd operation}}
-  // expected-error@below {{LLVM Translation failed for operation: omp.simd}}
-  omp.simd reduction(@add_f32 %x -> %prv : !llvm.ptr) {
-    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-      omp.yield
-    }
-  }
-  llvm.return
-}
-
-// -----
-
 omp.declare_reduction @add_f32 : f32
 init {
 ^bb0(%arg: f32):
@@ -533,3 +489,43 @@ llvm.func @wsloop_order(%lb : i32, %ub : i32, %step : i32) {
   }
   llvm.return
 }
+
+// -----
+
+llvm.func @do_simd_if(%1 : !llvm.ptr, %5 : i32, %4 : i32, %6 : i1) {
+  omp.wsloop {
+    // expected-warning@below {{simd information on composite construct discarded}}
+    omp.simd if(%6) {
+      omp.loop_nest (%arg0) : i32 = (%5) to (%4) inclusive step (%5) {
+        llvm.store %arg0, %1 : i32, !llvm.ptr
+        omp.yield
+      }
+    } {omp.composite}
+  } {omp.composite}
+  llvm.return
+}
+
+// -----
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+llvm.func @do_simd_reduction(%1 : !llvm.ptr, %3 : !llvm.ptr, %6 : i32, %7 : i32) {
+  omp.wsloop reduction(@add_reduction_i32 %3 -> %arg0 : !llvm.ptr) {
+    // expected-warning@below {{simd information on composite construct discarded}}
+    omp.simd reduction(@add_reduction_i32 %arg0 -> %arg1 : !llvm.ptr) {
+      omp.loop_nest (%arg2) : i32 = (%7) to (%6) inclusive step (%7) {
+        llvm.store %arg2, %1 : i32, !llvm.ptr
+        %12 = llvm.load %arg1 : !llvm.ptr -> i32
+        omp.yield
+      }
+    } {omp.composite}
+  } {omp.composite}
+  llvm.return
+}
diff --git a/mlir/test/Target/SPIRV/image-ops.mlir b/mlir/test/Target/SPIRV/image-ops.mlir
index b8d19f0f9a7d1..1ea66af6404dd 100644
--- a/mlir/test/Target/SPIRV/image-ops.mlir
+++ b/mlir/test/Target/SPIRV/image-ops.mlir
@@ -42,7 +42,7 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, ImageQuery, Link
 
 // -----
 
-spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, StorageImageWriteWithoutFormat, Linkage], []> {
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, StorageImageWriteWithoutFormat, StorageImageReadWithoutFormat, Linkage], []> {
   spirv.func @image_read_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>) "None" {
     // CHECK: spirv.ImageRead {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32> -> vector<4xf32>
     %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32> -> vector<4xf32>
diff --git a/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir b/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir
index 153ff47937972..77949908e8883 100644
--- a/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir
+++ b/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir
@@ -30,6 +30,15 @@ spirv.module Logical GLSL450 requires
     spirv.Return
   }
 
+  // CHECK-LABEL: @cooperative_matrix_load_3
+  spirv.func @cooperative_matrix_load_3(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
+    // CHECK:      {{%.+}} = spirv.KHR.CooperativeMatrixLoad {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16
+    // CHECK-SAME:   : !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+    %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <RowMajor>, <Aligned>, 16 :
+      !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+    spirv.Return
+  }
+
   // CHECK-LABEL: @cooperative_matrix_store_1
   spirv.func @cooperative_matrix_store_1(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
                                          %m : !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>) "None" {
@@ -38,6 +47,11 @@ spirv.module Logical GLSL450 requires
     spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor> :
       !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
 
+    // CHECK:      spirv.KHR.CooperativeMatrixStore {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16
+    // CHECK-SAME:   : !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
+    spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Aligned>, 16 :
+      !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
+
     // CHECK-NEXT:  spirv.KHR.CooperativeMatrixStore {{%.*}}, {{%.*}}, <RowMajor>, <Volatile|Nontemporal>
     spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Volatile|Nontemporal> :
       !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
diff --git a/mlir/test/Target/SPIRV/tensorARM.mlir b/mlir/test/Target/SPIRV/tensorARM.mlir
new file mode 100644
index 0000000000000..75b648ebfd008
--- /dev/null
+++ b/mlir/test/Target/SPIRV/tensorARM.mlir
@@ -0,0 +1,66 @@
+// RUN: mlir-translate -no-implicit-module -test-spirv-roundtrip %s | FileCheck %s
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, TensorsARM], [SPV_ARM_tensors]> {
+  // CHECK: spirv.func @shaped_int_arm_tensor(%arg0: !spirv.arm.tensor<2xi32>) "None" {
+  spirv.func @shaped_int_arm_tensor(%arg0 : !spirv.arm.tensor<2xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @shaped_rank2_int_arm_tensor(%arg0: !spirv.arm.tensor<2x3xi32>) "None" {
+  spirv.func @shaped_rank2_int_arm_tensor(%arg0 : !spirv.arm.tensor<2x3xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @ui64_arm_tensor_const() -> !spirv.arm.tensor<3xi64> "None" {
+  spirv.func @ui64_arm_tensor_const() -> !spirv.arm.tensor<3xui64> "None" {
+    // CHECK: spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xi64>
+    %0 = spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xui64>
+
+    spirv.ReturnValue %0: !spirv.arm.tensor<3xui64>
+  }
+
+// -----
+
+  // CHECK: spirv.func @si32_arm_tensor_const() -> !spirv.arm.tensor<3xsi32> "None" {
+  spirv.func @si32_arm_tensor_const() -> !spirv.arm.tensor<3xsi32> "None" {
+    // CHECK: spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xsi32>
+    %0 = spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xsi32>
+
+    spirv.ReturnValue %0 : !spirv.arm.tensor<3xsi32>
+  }
+
+// -----
+
+  // CHECK: spirv.func @float_arm_tensor_const() -> !spirv.arm.tensor<3xf32> "None" {
+  spirv.func @float_arm_tensor_const() -> !spirv.arm.tensor<3xf32> "None" {
+    // CHECK: spirv.Constant dense<[3.000000e+00, 4.000000e+00, 5.000000e+00]> : !spirv.arm.tensor<3xf32>
+    %0 = spirv.Constant dense<[3., 4., 5.]> : !spirv.arm.tensor<3xf32>
+
+    spirv.ReturnValue %0 : !spirv.arm.tensor<3xf32>
+  }
+
+// -----
+
+  // CHECK: spirv.func @unranked_int_arm_tensor(%arg0: !spirv.arm.tensor<*xi32>) "None" {
+  spirv.func @unranked_int_arm_tensor(%arg0 : !spirv.arm.tensor<*xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @unshaped_int_arm_tensor(%arg0: !spirv.arm.tensor<?xi32>) "None" {
+  spirv.func @unshaped_int_arm_tensor(%arg0 : !spirv.arm.tensor<?xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @unshaped_int_arm_tensor_2(%arg0: !spirv.arm.tensor<?x?xi32>) "None" {
+  spirv.func @unshaped_int_arm_tensor_2(%arg0 : !spirv.arm.tensor<?x?xi32>) "None" {
+    spirv.Return
+  }
+}
diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir
index 79518b04e7158..68c863cff69bf 100644
--- a/mlir/test/Transforms/test-legalizer.mlir
+++ b/mlir/test/Transforms/test-legalizer.mlir
@@ -1,10 +1,18 @@
-// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics %s | FileCheck %s
-
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics -profile-actions-to=- %s | FileCheck %s
+
+//      CHECK: "name": "pass-execution", "cat": "PERF", "ph": "B"
+//      CHECK: "name": "apply-conversion", "cat": "PERF", "ph": "B"
+//      CHECK: "name": "apply-pattern", "cat": "PERF", "ph": "B"
+//      CHECK: "name": "apply-pattern", "cat": "PERF", "ph": "E"
+// Note: Listener notifications appear after the pattern application because
+// the conversion driver sends all notifications at the end of the conversion
+// in bulk.
 //      CHECK: notifyOperationInserted: test.legal_op_a, was unlinked
 // CHECK-NEXT: notifyOperationReplaced: test.illegal_op_a
 // CHECK-NEXT: notifyOperationModified: func.return
 // CHECK-NEXT: notifyOperationErased: test.illegal_op_a
-
+//      CHECK: "name": "apply-conversion", "cat": "PERF", "ph": "E"
+//      CHECK: "name": "pass-execution", "cat": "PERF", "ph": "E"
 // CHECK-LABEL: verifyDirectPattern
 func.func @verifyDirectPattern() -> i32 {
   // CHECK-NEXT:  "test.legal_op_a"() <{status = "Success"}
@@ -434,7 +442,7 @@ func.func @test_properties_rollback() {
   // CHECK: test.with_properties a = 32,
   // expected-remark @below{{op 'test.with_properties' is not legalizable}}
   test.with_properties
-      a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4]
+      a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4], array32 = [5, 6]
       {modify_inplace}
   "test.return"() : () -> ()
 }
diff --git a/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp b/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
index 684d491b532f4..727c84cc9c07a 100644
--- a/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
@@ -130,6 +130,11 @@ struct TestLinalgTransforms
       *this, "test-fold-into-pack-and-unpack",
       llvm::cl::desc("Test folding ops into linalg.pack and linalg.unpack"),
       llvm::cl::init(false)};
+  Option<bool> testFoldIntoPackAndUnpackWithControlFn{
+      *this, "test-fold-into-pack-and-unpack-control",
+      llvm::cl::desc(
+          "Test controlling folding ops into linalg.pack and linalg.unpack"),
+      llvm::cl::init(false)};
   Option<bool> testSimplifyPackUnpackPatterns{
       *this, "test-simplify-pack-unpack-patterns",
       llvm::cl::desc("Test patterns to simplify linalg.pack and linalg.unpack"),
@@ -222,9 +227,11 @@ static void applyDecomposeWinogradOps(func::FuncOp funcOp) {
   (void)applyPatternsGreedily(funcOp, std::move(patterns));
 }
 
-static void applyFoldIntoPackAndUnpackPatterns(Operation *rootOp) {
+static void applyFoldIntoPackAndUnpackPatterns(
+    Operation *rootOp,
+    linalg::ControlFoldIntoPackUnpackFn controlFn = nullptr) {
   RewritePatternSet patterns(rootOp->getContext());
-  linalg::populateFoldIntoPackAndUnpackPatterns(patterns);
+  linalg::populateFoldIntoPackAndUnpackPatterns(patterns, controlFn);
   (void)applyPatternsGreedily(rootOp, std::move(patterns));
 }
 
@@ -263,6 +270,19 @@ void TestLinalgTransforms::runOnOperation() {
   Operation *rootOp = getOperation();
   if (testFoldIntoPackAndUnpack)
     applyFoldIntoPackAndUnpackPatterns(rootOp);
+  if (testFoldIntoPackAndUnpackWithControlFn) {
+    linalg::ControlFoldIntoPackUnpackFn controlFn = [](OpOperand *opOperand) {
+      Operation *producer = opOperand->get().getDefiningOp();
+      Operation *consumer = opOperand->getOwner();
+      // If we have a pack/unpack consumer and a producer that has multiple
+      // uses, do not apply the folding patterns.
+      if (isa<linalg::PackOp, linalg::UnPackOp>(consumer) &&
+          isa<TilingInterface>(producer) && !producer->hasOneUse())
+        return false;
+      return true;
+    };
+    applyFoldIntoPackAndUnpackPatterns(rootOp, controlFn);
+  }
   if (testSimplifyPackUnpackPatterns)
     applySimplifyPackUnpackPatterns(rootOp);
 }
diff --git a/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp b/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp
index 9ed1b3a47be36..70bab21b83256 100644
--- a/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp
+++ b/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp
@@ -381,3 +381,26 @@ void test::printAttrElideType(AsmPrinter &printer, Operation *op, TypeAttr type,
                               Attribute attr) {
   printer.printAttributeWithoutType(attr);
 }
+
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummyRegionRef
+//===----------------------------------------------------------------------===//
+
+ParseResult test::parseDummyRegionRef(OpAsmParser &parser, Region &region) {
+  return success();
+}
+
+void test::printDummyRegionRef(OpAsmPrinter &printer, Operation *op,
+                               Region &region) { /* do nothing */ }
+
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummySuccessorRef
+//===----------------------------------------------------------------------===//
+
+ParseResult test::parseDummySuccessorRef(OpAsmParser &parser,
+                                         Block *successor) {
+  return success();
+}
+
+void test::printDummySuccessorRef(OpAsmPrinter &printer, Operation *op,
+                                  Block *successor) { /* do nothing */ }
diff --git a/mlir/test/lib/Dialect/Test/TestFormatUtils.h b/mlir/test/lib/Dialect/Test/TestFormatUtils.h
index 6d4df7d82ffa5..e914f9a27b79b 100644
--- a/mlir/test/lib/Dialect/Test/TestFormatUtils.h
+++ b/mlir/test/lib/Dialect/Test/TestFormatUtils.h
@@ -207,6 +207,24 @@ mlir::ParseResult parseAttrElideType(mlir::AsmParser &parser,
 void printAttrElideType(mlir::AsmPrinter &printer, mlir::Operation *op,
                         mlir::TypeAttr type, mlir::Attribute attr);
 
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummyRegionRef
+//===----------------------------------------------------------------------===//
+
+mlir::ParseResult parseDummyRegionRef(mlir::OpAsmParser &parser,
+                                      mlir::Region &region);
+void printDummyRegionRef(mlir::OpAsmPrinter &printer, mlir::Operation *op,
+                         mlir::Region &region);
+
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummySuccessorRef
+//===----------------------------------------------------------------------===//
+
+mlir::ParseResult parseDummySuccessorRef(mlir::OpAsmParser &parser,
+                                         mlir::Block *successor);
+void printDummySuccessorRef(mlir::OpAsmPrinter &printer, mlir::Operation *op,
+                            mlir::Block *successor);
+
 } // end namespace test
 
 #endif // MLIR_TESTFORMATUTILS_H
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index 6c1a5d3441530..3ab4ef2680978 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -1420,3 +1420,37 @@ ::mlir::LogicalResult test::TestDummyTensorOp::bufferize(
 
   return mlir::success();
 }
+
+::mlir::LogicalResult test::TestCreateTensorOp::bufferize(
+    ::mlir::RewriterBase &rewriter,
+    const ::mlir::bufferization::BufferizationOptions &options,
+    ::mlir::bufferization::BufferizationState &state) {
+  // Note: mlir::bufferization::getBufferType() would internally call
+  // TestCreateTensorOp::getBufferType()
+  const auto bufferizedOutType =
+      mlir::bufferization::getBufferType(getOutput(), options, state);
+  if (mlir::failed(bufferizedOutType))
+    return failure();
+
+  // replace op with memref analogy
+  auto createMemrefOp =
+      rewriter.create<test::TestCreateMemrefOp>(getLoc(), *bufferizedOutType);
+
+  mlir::bufferization::replaceOpWithBufferizedValues(
+      rewriter, getOperation(), createMemrefOp.getResult());
+
+  return mlir::success();
+}
+
+mlir::FailureOr<mlir::bufferization::BufferLikeType>
+test::TestCreateTensorOp::getBufferType(
+    mlir::Value value, const mlir::bufferization::BufferizationOptions &,
+    const mlir::bufferization::BufferizationState &,
+    llvm::SmallVector<::mlir::Value> &) {
+  const auto type = dyn_cast<test::TestTensorType>(value.getType());
+  if (type == nullptr)
+    return failure();
+
+  return cast<mlir::bufferization::BufferLikeType>(test::TestMemrefType::get(
+      getContext(), type.getShape(), type.getElementType(), nullptr));
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 1c961d272f192..535f5e9b4a15d 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -3179,13 +3179,15 @@ def TestOpWithProperties : TEST_Op<"with_properties"> {
     `b` `=` $b `,`
     `c` `=` $c `,`
     `flag` `=` $flag `,`
-    `array` `=` $array attr-dict}];
+    `array` `=` $array `,`
+    `array32` `=` $array32 attr-dict}];
   let arguments = (ins
     I64Prop:$a,
     StrAttr:$b, // Attributes can directly be used here.
     StringProp:$c,
     BoolProp:$flag,
-    IntArrayProp<I64Prop>:$array // example of an array
+    IntArrayProp<I64Prop>:$array, // Example of an array.
+    IntArrayProp<I32Prop>:$array32 // Example of an array.
   );
 }
 
@@ -3615,29 +3617,16 @@ def TestAllocWithMultipleResults : TEST_Op<"alloc_with_multiple_results"> {
 // Test Ops bufferization
 //===----------------------------------------------------------------------===//
 
-def TestDummyTensorOp : TEST_Op<"dummy_tensor_op", [BufferizableOpInterface]> {
+def TestDummyTensorOp : TEST_Op<"dummy_tensor_op",
+    [DeclareOpInterfaceMethods<BufferizableOpInterface,
+        ["bufferize", "bufferizesToMemoryRead",
+         "bufferizesToMemoryWrite", "getAliasingValues"]>]> {
   let arguments = (ins
     Arg<TestTensorType>:$input
   );
   let results = (outs
     Arg<TestTensorType>:$output
   );
-  let extraClassDeclaration = [{
-    // BufferizableOpInterface
-    bool bufferizesToMemoryRead(mlir::OpOperand&,
-      const mlir::bufferization::AnalysisState&);
-
-    bool bufferizesToMemoryWrite(mlir::OpOperand&,
-      const mlir::bufferization::AnalysisState&);
-
-    mlir::bufferization::AliasingValueList getAliasingValues(mlir::OpOperand&,
-      const mlir::bufferization::AnalysisState&);
-
-    mlir::LogicalResult bufferize(
-      mlir::RewriterBase& rewriter,
-      const mlir::bufferization::BufferizationOptions& options,
-      mlir::bufferization::BufferizationState &state);
-  }];
 
   let extraClassDefinition = [{
     bool test::TestDummyTensorOp::bufferizesToMemoryRead(::mlir::OpOperand&,
@@ -3665,4 +3654,55 @@ def TestDummyMemrefOp : TEST_Op<"dummy_memref_op", []> {
   );
 }
 
+def TestCreateTensorOp : TEST_Op<"create_tensor_op",
+    [DeclareOpInterfaceMethods<BufferizableOpInterface,
+        ["bufferize", "getBufferType", "bufferizesToMemoryRead",
+         "bufferizesToMemoryWrite", "getAliasingValues",
+         "bufferizesToAllocation"]>]> {
+  let arguments = (ins);
+  let results = (outs Arg<TestTensorType>:$output);
+  let extraClassDefinition = [{
+    bool test::TestCreateTensorOp::bufferizesToMemoryRead(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return true;
+    }
+    bool test::TestCreateTensorOp::bufferizesToMemoryWrite(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return true;
+    }
+    bool test::TestCreateTensorOp::bufferizesToAllocation(mlir::Value value) {
+      return false;
+    }
+
+    ::mlir::bufferization::AliasingValueList
+    test::TestCreateTensorOp::getAliasingValues(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return {};
+    }
+  }];
+}
+
+def TestCreateMemrefOp : TEST_Op<"create_memref_op"> {
+  let arguments = (ins);
+  let results = (outs Arg<TestMemrefType>:$output);
+}
+
+//===----------------------------------------------------------------------===//
+// Test assembly format references
+//===----------------------------------------------------------------------===//
+
+def TestOpWithRegionRef : TEST_Op<"dummy_op_with_region_ref", [NoTerminator]> {
+  let regions = (region AnyRegion:$body);
+  let assemblyFormat = [{
+    $body attr-dict custom<DummyRegionRef>(ref($body))
+  }];
+}
+
+def TestOpWithSuccessorRef : TEST_Op<"dummy_op_with_successor_ref"> {
+  let successors = (successor AnySuccessor:$successor);
+  let assemblyFormat = [{
+    $successor attr-dict custom<DummySuccessorRef>(ref($successor))
+  }];
+}
+
 #endif // TEST_OPS
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index 71000b98fb8f9..a7285ab8cb15a 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -369,7 +369,7 @@ struct TestVectorTransferCollapseInnerMostContiguousDims
 
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
-    populateVectorTransferCollapseInnerMostContiguousDimsPatterns(patterns);
+    populateDropInnerMostUnitDimsXferOpPatterns(patterns);
     (void)applyPatternsGreedily(getOperation(), std::move(patterns));
   }
 };
diff --git a/mlir/test/mlir-query/complex-test.mlir b/mlir/test/mlir-query/backward-slice-union.mlir
similarity index 71%
rename from mlir/test/mlir-query/complex-test.mlir
rename to mlir/test/mlir-query/backward-slice-union.mlir
index ad96f03747a43..f8f88c2043749 100644
--- a/mlir/test/mlir-query/complex-test.mlir
+++ b/mlir/test/mlir-query/backward-slice-union.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-query %s -c "m getAllDefinitions(hasOpName(\"arith.addf\"),2)" | FileCheck %s
+// RUN: mlir-query %s -c "m anyOf(getAllDefinitions(hasOpName(\"arith.addf\"),2),getAllDefinitions(hasOpName(\"tensor.extract\"),1))" | FileCheck %s
 
 #map = affine_map<(d0, d1) -> (d0, d1)>
 func.func @slice_use_from_above(%arg0: tensor<5x5xf32>, %arg1: tensor<5x5xf32>) {
@@ -19,14 +19,23 @@ func.func @slice_use_from_above(%arg0: tensor<5x5xf32>, %arg1: tensor<5x5xf32>)
 }
 
 // CHECK: Match #1:
-
 // CHECK: %[[LINALG:.*]] = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel"]} 
 // CHECK-SAME: ins(%arg0 : tensor<5x5xf32>) outs(%arg1 : tensor<5x5xf32>)
+
+// CHECK: {{.*}}.mlir:7:10: note: "root" binds here
 // CHECK: %[[ADDF1:.*]] = arith.addf %in, %in : f32
 
 // CHECK: Match #2:
+// CHECK: %[[COLLAPSED:.*]] = tensor.collapse_shape %[[LINALG]] {{\[\[.*\]\]}} : tensor<5x5xf32> into tensor<25xf32>
+// CHECK: %[[C2:.*]] = arith.constant {{.*}} : index
 
+// CHECK: {{.*}}.mlir:14:18: note: "root" binds here
+// CHECK: %[[EXTRACTED:.*]] = tensor.extract %[[COLLAPSED]][%[[C2]]] : tensor<25xf32>
+
+// CHECK: Match #3:
 // CHECK: %[[COLLAPSED:.*]] = tensor.collapse_shape %[[LINALG]] {{\[\[.*\]\]}} : tensor<5x5xf32> into tensor<25xf32>
 // CHECK: %[[C2:.*]] = arith.constant {{.*}} : index
 // CHECK: %[[EXTRACTED:.*]] = tensor.extract %[[COLLAPSED]][%[[C2]]] : tensor<25xf32>
+
+// CHECK: {{.*}}.mlir:15:10: note: "root" binds here
 // CHECK: %[[ADDF2:.*]] = arith.addf %[[EXTRACTED]], %[[EXTRACTED]] : f32  
diff --git a/mlir/test/mlir-query/forward-slice-by-predicate.mlir b/mlir/test/mlir-query/forward-slice-by-predicate.mlir
new file mode 100644
index 0000000000000..e11378da89d9f
--- /dev/null
+++ b/mlir/test/mlir-query/forward-slice-by-predicate.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-query %s -c "m getUsersByPredicate(anyOf(hasOpName(\"memref.alloc\"),isConstantOp()),anyOf(hasOpName(\"affine.load\"), hasOpName(\"memref.dealloc\")),true)" | FileCheck %s
+
+func.func @slice_depth1_loop_nest_with_offsets() {
+  %0 = memref.alloc() : memref<100xf32>
+  %cst = arith.constant 7.000000e+00 : f32
+  affine.for %i0 = 0 to 16 {
+    %a0 = affine.apply affine_map<(d0) -> (d0 + 2)>(%i0)
+    affine.store %cst, %0[%a0] : memref<100xf32>
+  }
+  affine.for %i1 = 4 to 8 {
+    %a1 = affine.apply affine_map<(d0) -> (d0 - 1)>(%i1)
+    %1 = affine.load %0[%a1] : memref<100xf32>
+  }
+  return
+}
+
+// CHECK: Match #1:
+// CHECK: {{.*}}.mlir:4:8: note: "root" binds here
+// CHECK: %[[ALLOC:.*]] = memref.alloc() : memref<100xf32>
+
+// CHECK: affine.store %cst, %0[%a0] : memref<100xf32>
+
+// CHECK: Match #2:
+// CHECK: {{.*}}.mlir:5:10: note: "root" binds here
+// CHECK: %[[CST:.*]] = arith.constant 7.000000e+00 : f32
+
+// CHECK: affine.store %[[CST]], %0[%a0] : memref<100xf32>
diff --git a/mlir/test/mlir-query/logical-operator-test.mlir b/mlir/test/mlir-query/logical-operator-test.mlir
new file mode 100644
index 0000000000000..ac05428287abd
--- /dev/null
+++ b/mlir/test/mlir-query/logical-operator-test.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-query %s -c "m allOf(hasOpName(\"memref.alloca\"), hasOpAttrName(\"alignment\"))" | FileCheck %s
+
+func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref<?x?xf32> {
+  %0 = memref.alloca(%arg0, %arg1) : memref<?x?xf32>
+  memref.alloca(%arg0, %arg1) {alignment = 32} : memref<?x?xf32>
+  return %0 : memref<?x?xf32>
+}
+
+// CHECK: Match #1:
+// CHECK: {{.*}}.mlir:5:3: note: "root" binds here
+// CHECK: memref.alloca(%arg0, %arg1) {alignment = 32} : memref<?x?xf32>
diff --git a/mlir/test/mlir-query/slice-function-extraction.mlir b/mlir/test/mlir-query/slice-function-extraction.mlir
new file mode 100644
index 0000000000000..e55d5e77c5736
--- /dev/null
+++ b/mlir/test/mlir-query/slice-function-extraction.mlir
@@ -0,0 +1,29 @@
+// RUN: mlir-query %s -c "m getDefinitionsByPredicate(hasOpName(\"memref.store\"),hasOpName(\"memref.alloc\"),true,false,false).extract(\"backward_slice\")" | FileCheck %s
+
+// CHECK:       func.func @backward_slice(%{{.*}}: memref<10xf32>) -> (f32, index, index, f32, index, index, f32) {
+// CHECK:         %[[CST0:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:    %[[C0:.*]] = arith.constant 0 : index
+// CHECK-NEXT:    %[[I0:.*]] = affine.apply affine_map<()[s0] -> (s0)>()[%[[C0]]]
+// CHECK-NEXT:    memref.store %[[CST0]], %{{.*}}[%[[I0]]] : memref<10xf32>
+// CHECK-NEXT:    %[[CST2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:    %[[I1:.*]] = affine.apply affine_map<() -> (0)>()
+// CHECK-NEXT:    memref.store %[[CST2]], %{{.*}}[%[[I1]]] : memref<10xf32>
+// CHECK-NEXT:    %[[C1:.*]] = arith.constant 0 : index
+// CHECK-NEXT:    %[[LOAD:.*]] = memref.load %{{.*}}[%[[C1]]] : memref<10xf32>
+// CHECK-NEXT:    memref.store %[[LOAD]], %{{.*}}[%[[C1]]] : memref<10xf32>
+// CHECK-NEXT:    return %[[CST0]], %[[C0]], %[[I0]], %[[CST2]], %[[I1]], %[[C1]], %[[LOAD]] : f32, index, index, f32, index, index, f32
+
+func.func @slicing_memref_store_trivial() {
+  %0 = memref.alloc() : memref<10xf32>
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant 0.000000e+00 : f32
+  affine.for %i1 = 0 to 10 {
+    %1 = affine.apply affine_map<()[s0] -> (s0)>()[%c0]
+    memref.store %cst, %0[%1] : memref<10xf32>
+    %2 = memref.load %0[%c0] : memref<10xf32>
+    %3 = affine.apply affine_map<()[] -> (0)>()[]
+    memref.store %cst, %0[%3] : memref<10xf32>
+    memref.store %2, %0[%c0] : memref<10xf32>
+  }
+  return
+}
diff --git a/mlir/test/mlir-tblgen/op-decl-and-defs.td b/mlir/test/mlir-tblgen/op-decl-and-defs.td
index a3dce9b31f8d2..3ccefd4d82366 100644
--- a/mlir/test/mlir-tblgen/op-decl-and-defs.td
+++ b/mlir/test/mlir-tblgen/op-decl-and-defs.td
@@ -125,6 +125,7 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 // CHECK:   ::llvm::LogicalResult verifyInvariants();
 // CHECK:   static void getCanonicalizationPatterns(::mlir::RewritePatternSet &results, ::mlir::MLIRContext *context);
 // CHECK:   ::llvm::LogicalResult fold(FoldAdaptor adaptor, ::llvm::SmallVectorImpl<::mlir::OpFoldResult> &results);
+// CHECK:   static ::llvm::LogicalResult setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError);
 // CHECK:   // Display a graph for debugging purposes.
 // CHECK:   void displayGraph();
 // CHECK: };
diff --git a/mlir/test/mlir-tblgen/op-format-spec.td b/mlir/test/mlir-tblgen/op-format-spec.td
index 02bf65609b21a..1541cd09f53e0 100644
--- a/mlir/test/mlir-tblgen/op-format-spec.td
+++ b/mlir/test/mlir-tblgen/op-format-spec.td
@@ -49,6 +49,19 @@ def DirectiveCustomValidD : TestFormat_Op<[{
 def DirectiveCustomValidE : TestFormat_Op<[{
   custom<MyDirective>(prop-dict) attr-dict
 }]>, Arguments<(ins UnitAttr:$flag)>;
+def DirectiveCustomValidF : TestFormat_Op<[{
+  $operand custom<MyDirective>(ref($operand)) attr-dict
+}]>, Arguments<(ins Optional<I64>:$operand)>;
+def DirectiveCustomValidG : TestFormat_Op<[{
+  $body custom<MyDirective>(ref($body)) attr-dict
+}]> {
+  let regions = (region AnyRegion:$body);
+}
+def DirectiveCustomValidH : TestFormat_Op<[{
+  $successor custom<MyDirective>(ref($successor)) attr-dict
+}]> {
+  let successors = (successor AnySuccessor:$successor);
+}
 
 //===----------------------------------------------------------------------===//
 // functional-type
@@ -82,7 +95,7 @@ def DirectiveRegionsValid : TestFormat_Op<[{
 //===----------------------------------------------------------------------===//
 
 // CHECK-NOT: error:
-def DirectiveResultsInvalidA : TestFormat_Op<[{
+def DirectiveResultsValid : TestFormat_Op<[{
   type(results) attr-dict
 }]>;
 
@@ -91,7 +104,7 @@ def DirectiveResultsInvalidA : TestFormat_Op<[{
 //===----------------------------------------------------------------------===//
 
 // CHECK-NOT: error:
-def DirectiveSuccessorsInvalidA : TestFormat_Op<[{
+def DirectiveSuccessorsValid : TestFormat_Op<[{
   successors attr-dict
 }]>;
 
@@ -154,7 +167,7 @@ def OptionalValidA : TestFormat_Op<[{
 //===----------------------------------------------------------------------===//
 
 // CHECK-NOT: error
-def StringInvalidA : TestFormat_Op<[{ custom<Foo>("foo") attr-dict }]>;
+def StringValid : TestFormat_Op<[{ custom<Foo>("foo") attr-dict }]>;
 
 //===----------------------------------------------------------------------===//
 // Variables
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 981fb5aff2aa2..fa929c2df9d10 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -290,10 +290,10 @@ test.format_optional_prop_dict <{a = [], b = 1 : i32}>
 // CHECK: test.format_optional_prop_dict {{$}}
 test.format_optional_prop_dict <{}>
 
-// CHECK: test.format_optional_prop_dict < {a = ["foo"]}>
+// CHECK: test.format_optional_prop_dict <{a = ["foo"]}>
 test.format_optional_prop_dict <{a = ["foo"]}>
 
-// CHECK: test.format_optional_prop_dict < {b = 2 : i32}>
+// CHECK: test.format_optional_prop_dict <{b = 2 : i32}>
 test.format_optional_prop_dict <{b = 2 : i32}>
 
 // CHECK: test.format_optional_prop_dict <{a = ["foo"], b = 2 : i32}>
@@ -513,15 +513,15 @@ test.format_infer_variadic_type_from_non_variadic %i64, %i64 : i64
 // CHECK: test.format_infer_type_variadic_operands(%[[I32]], %[[I32]] : i32, i32) (%[[I64]], %[[I64]] : i64, i64)
 %ignored_res13:4 = test.format_infer_type_variadic_operands(%i32, %i32 : i32, i32) (%i64, %i64 : i64, i64)
 
-// CHECK: test.with_properties_and_attr 16 < {rhs = 16 : i64}>
+// CHECK: test.with_properties_and_attr 16 <{rhs = 16 : i64}>
 test.with_properties_and_attr 16 <{rhs = 16 : i64}>
 
-// CHECK: test.with_properties_and_inferred_type 16 < {packed, rhs = 16 : i64}>
+// CHECK: test.with_properties_and_inferred_type 16 <{packed, rhs = 16 : i64}>
 %should_be_i32 = test.with_properties_and_inferred_type 16 <{packed, rhs = 16 : i64}>
 // Assert through the verifier that its inferred as i32.
 test.format_all_types_match_var %should_be_i32, %i32 : i32
 
-// CHECK: test.using_property_in_custom_and_other [1, 4, 20] < {other = 16 : i64}>
+// CHECK: test.using_property_in_custom_and_other [1, 4, 20] <{other = 16 : i64}>
 test.using_property_in_custom_and_other [1, 4, 20] <{other = 16 : i64}>
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/mlir-tblgen/op-format.td b/mlir/test/mlir-tblgen/op-format.td
index 09e068b91a40b..1790737a3a349 100644
--- a/mlir/test/mlir-tblgen/op-format.td
+++ b/mlir/test/mlir-tblgen/op-format.td
@@ -109,3 +109,23 @@ def OptionalGroupC : TestFormat_Op<[{
 def OptionalGroupD : TestFormat_Op<[{
   (custom<Custom>($a, $b)^)? attr-dict
 }], [AttrSizedOperandSegments]>, Arguments<(ins Optional<I64>:$a, Optional<I64>:$b)>;
+
+// CHECK-LABEL: RegionRef::parse
+// CHECK:   auto odsResult = parseCustom(parser, *bodyRegion);
+// CHECK-LABEL: RegionRef::print
+// CHECK:   printCustom(_odsPrinter, *this, getBody());
+def RegionRef : TestFormat_Op<[{
+  $body custom<Custom>(ref($body)) attr-dict
+}]> {
+  let regions = (region AnyRegion:$body);
+}
+
+// CHECK-LABEL: SuccessorRef::parse
+// CHECK:   auto odsResult = parseCustom(parser, successorSuccessor);
+// CHECK-LABEL: SuccessorRef::print
+// CHECK:   printCustom(_odsPrinter, *this, getSuccessor());
+def SuccessorRef : TestFormat_Op<[{
+  $successor custom<Custom>(ref($successor)) attr-dict
+}]> {
+  let successors = (successor AnySuccessor:$successor);
+}
diff --git a/mlir/test/mlir-tblgen/op-operand.td b/mlir/test/mlir-tblgen/op-operand.td
index a2fa1f7046a97..ab8d721ed5427 100644
--- a/mlir/test/mlir-tblgen/op-operand.td
+++ b/mlir/test/mlir-tblgen/op-operand.td
@@ -13,6 +13,9 @@ def OpA : NS_Op<"one_normal_operand_op", []> {
   let arguments = (ins I32:$input);
 }
 
+// DECL-LABEL: class OpA : {{.*}} {
+// DECL: static constexpr int odsIndex_input = 0;
+
 // CHECK-LABEL: OpA definitions
 
 // CHECK:      void OpA::build
@@ -28,6 +31,9 @@ def OpB : NS_Op<"one_variadic_operand_op", []> {
   let arguments = (ins Variadic<I32>:$input);
 }
 
+// DECL-LABEL: class OpB : {{.*}} {
+// DECL: static constexpr int odsIndex_input = 0;
+
 // CHECK-LABEL: OpB::build
 // CHECK:         ::mlir::ValueRange input
 // CHECK-NOT:     assert
@@ -37,6 +43,11 @@ def OpD : NS_Op<"mix_variadic_and_normal_inputs_op", [SameVariadicOperandSize]>
   let arguments = (ins Variadic<AnyTensor>:$input1, AnyTensor:$input2, Variadic<AnyTensor>:$input3);
 }
 
+// DECL-LABEL: class OpD : {{.*}} {
+// DECL: static constexpr int odsIndex_input1 = 0;
+// DECL: static constexpr int odsIndex_input2 = 1;
+// DECL: static constexpr int odsIndex_input3 = 2;
+
 // DECL-LABEL: ::mlir::Operation::operand_range getInput1
 // DECL-NEXT: return getODSOperands(0);
 
diff --git a/mlir/test/mlir-translate/emitc_classops.mlir b/mlir/test/mlir-translate/emitc_classops.mlir
index e42844412860e..4b7ddf4630d55 100644
--- a/mlir/test/mlir-translate/emitc_classops.mlir
+++ b/mlir/test/mlir-translate/emitc_classops.mlir
@@ -12,7 +12,32 @@ emitc.class @modelClass {
     }
 }
 
-// CHECK: class modelClass {
+// CHECK-LABEL: class modelClass {
+// CHECK-NEXT: public:
+// CHECK-NEXT:  float[1] fieldName0;
+// CHECK-NEXT:  float[1] fieldName1;
+// CHECK-NEXT:  void execute() {
+// CHECK-NEXT:    size_t v1 = 0;
+// CHECK-NEXT:    float[1] v2 = fieldName0;
+// CHECK-NEXT:    float[1] v3 = fieldName1;
+// CHECK-NEXT:    return;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT: };
+
+emitc.class final @finalClass {
+    emitc.field @fieldName0 : !emitc.array<1xf32> 
+    emitc.field @fieldName1 : !emitc.array<1xf32> 
+    emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @fieldName0 : !emitc.array<1xf32>
+        %2 = get_field @fieldName1 : !emitc.array<1xf32>
+        %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+    }
+}
+
+// CHECK-LABEL: class finalClass final {
 // CHECK-NEXT: public:
 // CHECK-NEXT:  float[1] fieldName0;
 // CHECK-NEXT:  float[1] fieldName1;
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index f1e2afa0f2408..709a1d2424f35 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -49,42 +49,6 @@ def fill_buffer(out):
     print(module)
 
 
-# CHECK-LABEL: TEST: testNamedStructuredOpCustomForm
-@run
-def testNamedStructuredOpCustomForm():
-    with Context() as ctx, Location.unknown():
-        module = Module.create()
-        f32 = F32Type.get()
-        with InsertionPoint(module.body):
-
-            @func.FuncOp.from_py_func(
-                RankedTensorType.get((4, 8), f32), RankedTensorType.get((4, 8), f32)
-            )
-            def named_form(lhs, rhs):
-                init_result = tensor.EmptyOp([4, 8], f32)
-                # Check for the named form with custom format
-                #      CHECK: linalg.elemwise_unary
-                # CHECK-SAME:    cast = #linalg.type_fn<cast_signed>
-                # CHECK-SAME:    fun = #linalg.unary_fn<exp>
-                # CHECK-SAME:    ins(%{{.*}} : tensor<4x8xf32>) outs(%{{.*}} : tensor<4x8xf32>)
-                unary_result = linalg.elemwise_unary(lhs, outs=[init_result.result])
-                #      CHECK: linalg.elemwise_binary
-                # CHECK-SAME:    cast = #linalg.type_fn<cast_unsigned>
-                # CHECK-SAME:    fun = #linalg.binary_fn<mul>
-                # CHECK-SAME:    ins(%{{.*}}, %{{.*}} : tensor<4x8xf32>, tensor<4x8xf32>) outs(%{{.*}} : tensor<4x8xf32>)
-                #      CHECK: return
-                binary_result = linalg.elemwise_binary(
-                    lhs,
-                    rhs,
-                    outs=[init_result.result],
-                    fun=BinaryFn.mul,
-                    cast=TypeFn.cast_unsigned,
-                )
-                return unary_result, binary_result
-
-    print(module)
-
-
 # CHECK-LABEL: TEST: testIdentityRegionOps
 @run
 def testIdentityRegionOps():
diff --git a/mlir/test/python/dialects/transform_tune_ext.py b/mlir/test/python/dialects/transform_tune_ext.py
new file mode 100644
index 0000000000000..dfb93594bca52
--- /dev/null
+++ b/mlir/test/python/dialects/transform_tune_ext.py
@@ -0,0 +1,72 @@
+# RUN: %PYTHON %s | FileCheck %s
+
+from mlir.ir import *
+from mlir.dialects import transform
+from mlir.dialects.transform import tune, debug
+
+
+def run(f):
+    print("\nTEST:", f.__name__)
+    with Context(), Location.unknown():
+        module = Module.create()
+        with InsertionPoint(module.body):
+            sequence = transform.SequenceOp(
+                transform.FailurePropagationMode.Propagate,
+                [],
+                transform.AnyOpType.get(),
+            )
+            with InsertionPoint(sequence.body):
+                f(sequence.bodyTarget)
+                transform.YieldOp()
+        print(module)
+    return f
+
+
+# CHECK-LABEL: TEST: testKnobOp
+@run
+def testKnobOp(target):
+    any_param = transform.AnyParamType.get()
+
+    # CHECK: %[[HEADS_OR_TAILS:.*]] = transform.tune.knob<"coin"> options = [true, false] -> !transform.any_param
+    heads_or_tails = tune.KnobOp(
+        result=any_param, name=StringAttr.get("coin"), options=[True, False]
+    )
+    # CHECK: transform.tune.knob<"animal"> options = ["cat", "dog", unit] -> !transform.any_param
+    tune.KnobOp(any_param, name="animal", options=["cat", "dog", UnitAttr.get()])
+    # CHECK: transform.tune.knob<"tile_size"> options = [2, 4, 8, 16, 24, 32] -> !transform.any_param
+    tune.KnobOp(any_param, "tile_size", [2, 4, 8, 16, 24, 32])
+    # CHECK: transform.tune.knob<"magic_value"> options = [2.000000e+00, 2.250000e+00, 2.500000e+00, 2.750000e+00, 3.000000e+00] -> !transform.any_param
+    tune.knob(any_param, "magic_value", [2.0, 2.25, 2.5, 2.75, 3.0])
+
+    # CHECK: transform.debug.emit_param_as_remark %[[HEADS_OR_TAILS]]
+    debug.emit_param_as_remark(heads_or_tails)
+
+    # CHECK: %[[HEADS:.*]] = transform.tune.knob<"coin"> = true from options = [true, false] -> !transform.any_param
+    heads = tune.KnobOp(any_param, "coin", options=[True, False], selected=True)
+    # CHECK: transform.tune.knob<"animal"> = "dog" from options = ["cat", "dog", unit] -> !transform.any_param
+    tune.KnobOp(
+        any_param, name="animal", options=["cat", "dog", UnitAttr.get()], selected="dog"
+    )
+    # CHECK: transform.tune.knob<"tile_size"> = 8 : i64 from options = [2, 4, 8, 16, 24, 32] -> !transform.any_param
+    tune.KnobOp(any_param, "tile_size", [2, 4, 8, 16, 24, 32], selected=8)
+    # CHECK: transform.tune.knob<"magic_value"> = 2.500000e+00 : f64 from options = [2.000000e+00, 2.250000e+00, 2.500000e+00, 2.750000e+00, 3.000000e+00] -> !transform.any_param
+    tune.knob(any_param, "magic_value", [2.0, 2.25, 2.5, 2.75, 3.0], selected=2.5)
+
+    # CHECK: transform.debug.emit_param_as_remark %[[HEADS]]
+    debug.emit_param_as_remark(heads)
+
+    # CHECK: transform.tune.knob<"range_as_a_dict"> = 4 : i64 from options = {start = 2 : i64, step = 2 : i64, stop = 16 : i64} -> !transform.any_param
+    # NB: Membership of `selected` in non-ArrayAttr `options` is _not_ verified.
+    i64 = IntegerType.get_signless(64)
+    tune.knob(
+        any_param,
+        "range_as_a_dict",
+        DictAttr.get(
+            {
+                "start": IntegerAttr.get(i64, 2),
+                "stop": IntegerAttr.get(i64, 16),
+                "step": IntegerAttr.get(i64, 2),
+            }
+        ),
+        selected=4,
+    )
diff --git a/mlir/test/python/integration/dialects/linalg/opsrun.py b/mlir/test/python/integration/dialects/linalg/opsrun.py
index 69f97e15e139d..8f202318146ee 100644
--- a/mlir/test/python/integration/dialects/linalg/opsrun.py
+++ b/mlir/test/python/integration/dialects/linalg/opsrun.py
@@ -19,37 +19,6 @@ def log(*args):
     sys.stderr.flush()
 
 
-elemwise_boiler = """
-func.func @main() -> f32 attributes {llvm.emit_c_interface} {
-  %v0 = arith.constant 0.0 : f32
-  %v1 = arith.constant 1.0 : f32
-  %v2 = arith.constant 2.0 : f32
-
-  %lhs = memref.alloc() : memref<f32>
-  %rhs = memref.alloc() : memref<4x8xf32>
-  %O0 = memref.alloc() : memref<4x8xf32>
-  %O1 = memref.alloc() : memref<4x8xf32>
-  linalg.fill ins(%v1 : f32) outs(%lhs : memref<f32>)
-  linalg.fill ins(%v2 : f32) outs(%rhs : memref<4x8xf32>)
-  linalg.fill ins(%v0 : f32) outs(%O0 : memref<4x8xf32>)
-  linalg.fill ins(%v0 : f32) outs(%O1 : memref<4x8xf32>)
-
-  call @elemwise_exp_add_on_buffers(%lhs, %rhs, %O0) :
-    (memref<f32>, memref<4x8xf32>, memref<4x8xf32>) -> ()
-  call @elemwise_log_mul_on_buffers(%lhs, %rhs, %O1) :
-    (memref<f32>, memref<4x8xf32>, memref<4x8xf32>) -> ()
-
-  %c0 = arith.constant 0 : index
-  %res0 = memref.load %O0[%c0, %c0] : memref<4x8xf32>
-  %res1 = memref.load %O1[%c0, %c0] : memref<4x8xf32>
-
-  %0 = arith.addf %res0, %res1 : f32
-
-  // TODO: FFI-based solution to allow testing and printing with python code.
-  return %0 : f32
-}
-"""
-
 fill_boiler = """
 func.func @main() -> i32 attributes {llvm.emit_c_interface} {
   %O0 = memref.alloc() : memref<i32>
@@ -177,96 +146,6 @@ def transform(module, boilerplate):
     return mod
 
 
-def test_elemwise_builtin():
-    with Context() as ctx, Location.unknown():
-        module = Module.create()
-        f32 = F32Type.get()
-        i8 = IntegerType.get_signless(8)
-        with InsertionPoint(module.body):
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_exp_add_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(lhs, outs=[out])
-                linalg.elemwise_binary(out, rhs, outs=[out])
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_log_mul_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(lhs, outs=[out], fun=UnaryFn.log)
-                linalg.elemwise_binary(out, rhs, outs=[out], fun=BinaryFn.mul)
-
-        execution_engine = ExecutionEngine(transform(module, elemwise_boiler))
-
-        # TODO: FFI-based solution to allow testing and printing with python code.
-        # Prepare arguments: one result f32.
-        # Arguments must be passed as pointers.
-        c_float_p = ctypes.c_float * 1
-        res = c_float_p(-1.0)
-        execution_engine.invoke("main", res)
-
-        log("RESULT: ", res[0])
-        # elemwise_exp_add_on_buffers: exp(1.0) + 2.0 = 4.71828182846
-        # elemwise_log_mul_on_buffers: log(1.0) * 2.0 = 0.0
-        # CHECK: RESULT: 4.71828
-
-
-test_elemwise_builtin()
-
-
-def test_elemwise_generic():
-    with Context() as ctx, Location.unknown():
-        module = Module.create()
-        f32 = F32Type.get()
-        i8 = IntegerType.get_signless(8)
-        with InsertionPoint(module.body):
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_exp_add_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(lhs, outs=[out], emit_generic=True)
-                linalg.elemwise_binary(out, rhs, outs=[out], emit_generic=True)
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_log_mul_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(
-                    lhs, outs=[out], fun=UnaryFn.log, emit_generic=True
-                )
-                linalg.elemwise_binary(
-                    out, rhs, outs=[out], fun=BinaryFn.mul, emit_generic=True
-                )
-
-        execution_engine = ExecutionEngine(transform(module, elemwise_boiler))
-
-        # TODO: FFI-based solution to allow testing and printing with python code.
-        # Prepare arguments: one result f32.
-        # Arguments must be passed as pointers.
-        c_float_p = ctypes.c_float * 1
-        res = c_float_p(-1.0)
-        execution_engine.invoke("main", res)
-
-        log("RESULT: ", res[0])
-        # elemwise_exp_add_on_buffers: exp(1.0) + 2.0 = 4.71828182846
-        # elemwise_log_mul_on_buffers: log(1.0) * 2.0 = 0.0
-        # CHECK: RESULT: 4.71828
-
-
-test_elemwise_generic()
-
-
 def test_fill_builtin():
     with Context() as ctx, Location.unknown():
         module = Module.create()
diff --git a/mlir/test/python/ir/builtin_types.py b/mlir/test/python/ir/builtin_types.py
index 75502c285cd14..b42bfd9bc6587 100644
--- a/mlir/test/python/ir/builtin_types.py
+++ b/mlir/test/python/ir/builtin_types.py
@@ -330,8 +330,29 @@ def testConcreteShapedType():
         print("dim size:", vector.get_dim_size(1))
         # CHECK: is_dynamic_size: False
         print("is_dynamic_size:", vector.is_dynamic_size(3))
+        # CHECK: is_static_size: True
+        print("is_static_size:", vector.is_static_size(3))
         # CHECK: is_dynamic_stride_or_offset: False
         print("is_dynamic_stride_or_offset:", vector.is_dynamic_stride_or_offset(1))
+        # CHECK: is_static_stride_or_offset: True
+        print("is_static_stride_or_offset:", vector.is_static_stride_or_offset(1))
+
+        dynamic_size_val = vector.get_dynamic_size()
+        dynamic_stride_val = vector.get_dynamic_stride_or_offset()
+        # CHECK: is_dynamic_size_with_dynamic: True
+        print("is_dynamic_size_with_dynamic:", vector.is_dynamic_size(dynamic_size_val))
+        # CHECK: is_static_size_with_dynamic: False
+        print("is_static_size_with_dynamic:", vector.is_static_size(dynamic_size_val))
+        # CHECK: is_dynamic_stride_or_offset_with_dynamic: True
+        print(
+            "is_dynamic_stride_or_offset_with_dynamic:",
+            vector.is_dynamic_stride_or_offset(dynamic_stride_val),
+        )
+        # CHECK: is_static_stride_or_offset_with_dynamic: False
+        print(
+            "is_static_stride_or_offset_with_dynamic:",
+            vector.is_static_stride_or_offset(dynamic_stride_val),
+        )
         # CHECK: isinstance(ShapedType): True
         print("isinstance(ShapedType):", isinstance(vector, ShapedType))
 
diff --git a/mlir/tools/mlir-query/mlir-query.cpp b/mlir/tools/mlir-query/mlir-query.cpp
index 78c0ec97c0cdf..8a17a33c61838 100644
--- a/mlir/tools/mlir-query/mlir-query.cpp
+++ b/mlir/tools/mlir-query/mlir-query.cpp
@@ -40,12 +40,22 @@ int main(int argc, char **argv) {
   query::matcher::Registry matcherRegistry;
 
   // Matchers registered in alphabetical order for consistency:
+  matcherRegistry.registerMatcher("allOf", query::matcher::internal::allOf);
+  matcherRegistry.registerMatcher("anyOf", query::matcher::internal::anyOf);
+  matcherRegistry.registerMatcher(
+      "getAllDefinitions",
+      query::matcher::m_GetAllDefinitions<query::matcher::DynMatcher>);
   matcherRegistry.registerMatcher(
       "getDefinitions",
       query::matcher::m_GetDefinitions<query::matcher::DynMatcher>);
   matcherRegistry.registerMatcher(
-      "getAllDefinitions",
-      query::matcher::m_GetAllDefinitions<query::matcher::DynMatcher>);
+      "getDefinitionsByPredicate",
+      query::matcher::m_GetDefinitionsByPredicate<query::matcher::DynMatcher,
+                                                  query::matcher::DynMatcher>);
+  matcherRegistry.registerMatcher(
+      "getUsersByPredicate",
+      query::matcher::m_GetUsersByPredicate<query::matcher::DynMatcher,
+                                            query::matcher::DynMatcher>);
   matcherRegistry.registerMatcher("hasOpAttrName",
                                   static_cast<HasOpAttrName *>(m_Attr));
   matcherRegistry.registerMatcher("hasOpName", static_cast<HasOpName *>(m_Op));
diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index 728a9819ca0d1..d9aa901ee2b28 100644
--- a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -245,9 +245,8 @@ void DefGen::createParentWithTraits() {
                    ? cast<NativeTrait>(&trait)->getFullyQualifiedTraitName()
                    : cast<InterfaceTrait>(&trait)->getFullyQualifiedTraitName();
       }));
-  llvm::for_each(traitNames, [&](auto &traitName) {
+  for (auto &traitName : traitNames)
     defParent.addTemplateParam(traitName);
-  });
 
   // Add OpAsmInterface::Trait if we automatically generate mnemonic alias
   // method.
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 6008ed4673d1b..cbb4030f3adb4 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -2223,6 +2223,17 @@ generateNamedOperandGetters(const Operator &op, Class &opClass,
                     "'SameVariadicOperandSize' traits");
   }
 
+  // Print the ods names so they don't need to be hardcoded in the source.
+  for (int i = 0; i != numOperands; ++i) {
+    const auto &operand = op.getOperand(i);
+    if (operand.name.empty())
+      continue;
+
+    opClass.declare<Field>("static constexpr int", Twine("odsIndex_") +
+                                                       operand.name + " = " +
+                                                       Twine(i));
+  }
+
   // First emit a few "sink" getter methods upon which we layer all nicer named
   // getter methods.
   // If generating for an adaptor, the method is put into the non-templated
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 11edf2523f1aa..67fc7636af60f 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -1302,8 +1302,8 @@ if (!dict) {
 }
 // keep track of used keys in the input dictionary to be able to error out
 // if there are some unknown ones.
-DenseSet<StringAttr> usedKeys;
-MLIRContext *ctx = dict.getContext();
+::mlir::DenseSet<::mlir::StringAttr> usedKeys;
+::mlir::MLIRContext *ctx = dict.getContext();
 (void)ctx;
 )decl";
 
@@ -1315,7 +1315,7 @@ auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
          ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {{
   {0};
 };
-auto {1}AttrName = StringAttr::get(ctx, "{1}");
+auto {1}AttrName = ::mlir::StringAttr::get(ctx, "{1}");
 usedKeys.insert({1}AttrName);
 auto attr = dict.get({1}AttrName);
 if (!attr && {2}) {{
@@ -1363,7 +1363,7 @@ if (attr && ::mlir::failed(setFromAttr(prop.{1}, attr, emitError)))
     bool isRequired = !attr.isOptional() && !attr.hasDefaultValue();
     body << formatv(R"decl(
 auto &propStorage = prop.{0};
-auto {0}AttrName = StringAttr::get(ctx, "{0}");
+auto {0}AttrName = ::mlir::StringAttr::get(ctx, "{0}");
 auto attr = dict.get({0}AttrName);
 usedKeys.insert({0}AttrName);
 if (attr || /*isRequired=*/{1}) {{
@@ -1384,7 +1384,7 @@ if (attr || /*isRequired=*/{1}) {{
                     namedAttr.name, isRequired);
   }
   body << R"decl(
-for (NamedAttribute attr : dict) {
+for (::mlir::NamedAttribute attr : dict) {
   if (!usedKeys.contains(attr.getName()))
     return emitError() << "unknown key '" << attr.getName() <<
         "' when parsing properties dictionary";
@@ -3376,11 +3376,13 @@ OpFormatParser::parseVariableImpl(SMLoc loc, StringRef name, Context ctx) {
     if (ctx == TopLevelContext || ctx == CustomDirectiveContext) {
       if (hasAllRegions || !seenRegions.insert(region).second)
         return emitError(loc, "region '" + name + "' is already bound");
-    } else if (ctx == RefDirectiveContext && !seenRegions.count(region)) {
-      return emitError(loc, "region '" + name +
-                                "' must be bound before it is referenced");
+    } else if (ctx == RefDirectiveContext) {
+      if (!seenRegions.count(region))
+        return emitError(loc, "region '" + name +
+                                  "' must be bound before it is referenced");
     } else {
-      return emitError(loc, "regions can only be used at the top level");
+      return emitError(loc, "regions can only be used at the top level "
+                            "or in a ref directive");
     }
     return create<RegionVariable>(region);
   }
@@ -3396,11 +3398,13 @@ OpFormatParser::parseVariableImpl(SMLoc loc, StringRef name, Context ctx) {
     if (ctx == TopLevelContext || ctx == CustomDirectiveContext) {
       if (hasAllSuccessors || !seenSuccessors.insert(successor).second)
         return emitError(loc, "successor '" + name + "' is already bound");
-    } else if (ctx == RefDirectiveContext && !seenSuccessors.count(successor)) {
-      return emitError(loc, "successor '" + name +
-                                "' must be bound before it is referenced");
+    } else if (ctx == RefDirectiveContext) {
+      if (!seenSuccessors.count(successor))
+        return emitError(loc, "successor '" + name +
+                                  "' must be bound before it is referenced");
     } else {
-      return emitError(loc, "successors can only be used at the top level");
+      return emitError(loc, "successors can only be used at the top level "
+                            "or in a ref directive");
     }
 
     return create<SuccessorVariable>(successor);
@@ -3775,7 +3779,8 @@ LogicalResult OpFormatParser::verifyOptionalGroupElement(SMLoc loc,
         Property prop = propEle->getVar()->prop;
         if (isAnchor && !(prop.hasDefaultValue() && prop.hasOptionalParser()))
           return emitError(loc, "only properties with default values "
-                                "that can be optionally parsed "
+                                "that can be optionally parsed (have the `let "
+                                "optionalParser = ...` field defined) "
                                 "can be used to anchor an optional group");
         return success();
       })
@@ -3848,8 +3853,14 @@ void mlir::tblgen::generateOpFormat(const Operator &constOp, OpClass &opClass,
   // TODO: Operator doesn't expose all necessary functionality via
   // the const interface.
   Operator &op = const_cast<Operator &>(constOp);
-  if (!op.hasAssemblyFormat())
+  if (!op.hasAssemblyFormat()) {
+    // We still need to generate the parsed attribute properties setter for
+    // allowing it to be reused in custom assembly implementations.
+    OperationFormat format(op, hasProperties);
+    format.hasPropDict = true;
+    genParsedAttrPropertiesSetter(format, op, opClass);
     return;
+  }
 
   // Parse the format description.
   llvm::SourceMgr mgr;
diff --git a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
index 13c83c00d1523..ffe815b9be5e2 100644
--- a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
+++ b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
@@ -188,11 +188,11 @@ TEST_F(DeserializationTest, OpMemberNameSuccess) {
   std::swap(typeDecl, binary);
 
   SmallVector<uint32_t, 5> operands1 = {structType, 0};
-  (void)spirv::encodeStringLiteralInto(operands1, "i1");
+  spirv::encodeStringLiteralInto(operands1, "i1");
   addInstruction(spirv::Opcode::OpMemberName, operands1);
 
   SmallVector<uint32_t, 5> operands2 = {structType, 1};
-  (void)spirv::encodeStringLiteralInto(operands2, "i2");
+  spirv::encodeStringLiteralInto(operands2, "i2");
   addInstruction(spirv::Opcode::OpMemberName, operands2);
 
   binary.append(typeDecl.begin(), typeDecl.end());
@@ -227,7 +227,7 @@ TEST_F(DeserializationTest, OpMemberNameExcessOperands) {
   std::swap(typeDecl, binary);
 
   SmallVector<uint32_t, 5> operands = {structType, 0};
-  (void)spirv::encodeStringLiteralInto(operands, "int32");
+  spirv::encodeStringLiteralInto(operands, "int32");
   operands.push_back(42);
   addInstruction(spirv::Opcode::OpMemberName, operands);
 
diff --git a/mlir/unittests/IR/AffineExprTest.cpp b/mlir/unittests/IR/AffineExprTest.cpp
index 8a2d697540d5c..f8494ecb971c2 100644
--- a/mlir/unittests/IR/AffineExprTest.cpp
+++ b/mlir/unittests/IR/AffineExprTest.cpp
@@ -84,6 +84,20 @@ TEST(AffineExprTest, constantFolding) {
   ASSERT_EQ(cminfloordivcn1.getKind(), AffineExprKind::FloorDiv);
 }
 
+TEST(AffineExprTest, commutative) {
+  MLIRContext ctx;
+  OpBuilder b(&ctx);
+  auto c2 = b.getAffineConstantExpr(1);
+  auto d0 = b.getAffineDimExpr(0);
+  auto d1 = b.getAffineDimExpr(1);
+  auto s0 = b.getAffineSymbolExpr(0);
+  auto s1 = b.getAffineSymbolExpr(1);
+
+  ASSERT_EQ(d0 * d1, d1 * d0);
+  ASSERT_EQ(s0 + s1, s1 + s0);
+  ASSERT_EQ(s0 * c2, c2 * s0);
+}
+
 TEST(AffineExprTest, divisionSimplification) {
   MLIRContext ctx;
   OpBuilder b(&ctx);
@@ -147,3 +161,12 @@ TEST(AffineExprTest, simpleAffineExprFlattenerRegression) {
   ASSERT_TRUE(isa<AffineConstantExpr>(result));
   ASSERT_EQ(cast<AffineConstantExpr>(result).getValue(), 7);
 }
+
+TEST(AffineExprTest, simplifyCommutative) {
+  MLIRContext ctx;
+  OpBuilder b(&ctx);
+  auto s0 = b.getAffineSymbolExpr(0);
+  auto s1 = b.getAffineSymbolExpr(1);
+
+  ASSERT_EQ(s0 * s1 - s1 * s0 + 1, 1);
+}
diff --git a/offload/cmake/caches/AMDGPULibcBot.cmake b/offload/cmake/caches/AMDGPULibcBot.cmake
index 728dfe3f0a3f1..a772043c79669 100644
--- a/offload/cmake/caches/AMDGPULibcBot.cmake
+++ b/offload/cmake/caches/AMDGPULibcBot.cmake
@@ -18,3 +18,4 @@ set(CLANG_DEFAULT_RTLIB "compiler-rt" STRING "")
 
 set(LLVM_RUNTIME_TARGETS default;amdgcn-amd-amdhsa CACHE STRING "")
 set(RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES "compiler-rt;libc" CACHE STRING "")
+set(RUNTIMES_amdgcn-amd-amdhsa_LIBC_GPU_TEST_JOBS 4 CACHE STRING "")
diff --git a/offload/liboffload/API/Common.td b/offload/liboffload/API/Common.td
index 79c3bd46f1984..850a01d06759e 100644
--- a/offload/liboffload/API/Common.td
+++ b/offload/liboffload/API/Common.td
@@ -1,3 +1,15 @@
+//===-- Common.td - Common definitions for Offload ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains shared Offload API definitions
+//
+//===----------------------------------------------------------------------===//
+
 def : Macro {
   let name = "OL_VERSION_MAJOR";
   let desc = "Major version of the Offload API";
@@ -176,7 +188,7 @@ def : Function {
   let desc = "Release the resources in use by Offload";
   let details = [
     "This decrements an internal reference count. When this reaches 0, all resources will be released",
-    "Subsequent API calls made after this are not valid"
+    "Subsequent API calls to methods other than `olInit` made after resources are released will return OL_ERRC_UNINITIALIZED"
   ];
   let params = [];
   let returns = [];
diff --git a/offload/liboffload/API/Device.td b/offload/liboffload/API/Device.td
index 4abc24f3ba27f..94bd6cbf0e5be 100644
--- a/offload/liboffload/API/Device.td
+++ b/offload/liboffload/API/Device.td
@@ -31,7 +31,8 @@ def : Enum {
     TaggedEtor<"PLATFORM", "ol_platform_handle_t", "the platform associated with the device">,
     TaggedEtor<"NAME", "char[]", "Device name">,
     TaggedEtor<"VENDOR", "char[]", "Device vendor">,
-    TaggedEtor<"DRIVER_VERSION", "char[]", "Driver version">
+    TaggedEtor<"DRIVER_VERSION", "char[]", "Driver version">,
+    TaggedEtor<"MAX_WORK_GROUP_SIZE", "ol_dimensions_t", "Maximum work group size in each dimension">,
   ];
 }
 
diff --git a/offload/liboffload/API/README.md b/offload/liboffload/API/README.md
index fda1ad39fa937..dad88df6b8768 100644
--- a/offload/liboffload/API/README.md
+++ b/offload/liboffload/API/README.md
@@ -1,8 +1,5 @@
 # Offload API definitions
 
-**Note**: This is a work-in-progress. It is loosely based on equivalent
-tooling in Unified Runtime.
-
 The Tablegen files in this directory are used to define the Offload API. They
 are used with the `offload-tblgen` tool to generate API headers, print headers,
 and other implementation details.
@@ -10,17 +7,48 @@ and other implementation details.
 The root file is `OffloadAPI.td` - additional `.td` files can be included in
 this file to add them to the API.
 
+## Modifying the API
+
+API modifications, including additions, can be made by modifying the existing
+`.td` files. It is also possible to add a new tablegen file to the API by adding
+it to the includes in `OffloadAPI.td`. When Offload is rebuilt the new
+definition will be included in the generated files.
+
+Most API changes and additions do not require any additional work beyond this,
+other than new functions which are described below.
+
+### Adding a new function to the API
+
+When a new function is added (e.g. `offloadDeviceFoo`), the actual entry
+point is automatically generated, which contains validation and tracing code.
+It expects an implementation function (`offloadDeviceFoo_impl`) to be defined,
+which it will call into. The definition of this implementation function should
+be added to `liboffload/src/OffloadImpl.cpp`
+
+In short, the steps to add a new function are:
+* Add the new function definition to the `.td` files.
+* Build the `LLVMOffload` target. The relevant files will be regenerated, but
+  the library will fail to link because it is missing the implementation
+  function.
+* Add the new implementation function to `liboffload/src/OffloadImpl.cpp`. You
+  can copy the new function declaration from the generated
+  `OffloadImplFuncDecls.inc` file.
+* Rebuild `LLVMOffload`
+
 ## API Objects
+
 The API consists of a number of objects, which always have a *name* field and
 *description* field, and are one of the following types:
 
 ### Function
+
 Represents an API entry point function. Has a list of returns and parameters.
-Also has fields for details (representing a bullet-point list of
-information about the function that would otherwise be too detailed for the
-description), and analogues (equivalent functions in other APIs).
+Also has fields for details (representing a bullet-point list of information
+about the function that would otherwise be too detailed for the description),
+and analogues (equivalent functions in other APIs).
 
 #### Parameter
+
 Represents a parameter to a function, has *type*, *name*, and *desc* fields.
 Also has a *flags* field containing flags representing whether the parameter is
 in, out, or optional.
@@ -30,17 +58,24 @@ A *handle* type is a pointer to an opaque struct, used to abstract over
 plugin-specific implementation details.
 
 There are two special variants of a *parameter*:
-* **RangedParameter** - Represents a parameter that has a range described by other parameters. Generally these are pointers to an arbitrary number of objects. The range is used for generating validation and printing code. E.g, a range might be between `(0, NumDevices)`
-* **TypeTaggedParameter** - Represents a parameter (usually of `void*` type) that has the type and size of its pointee data described by other function parameters. The type is usually described by a type-tagged enum. This allows functions (e.g. `olGetDeviceInfo`) to return data of an arbitrary type.
+* **RangedParameter** - Represents a parameter that has a range described by
+  other parameters. Generally these are pointers to an arbitrary number of
+  objects. The range is used for generating validation and printing code. E.g,
+  a range might be between `(0, NumDevices)`
+* **TypeTaggedParameter** - Represents a parameter (usually of `void*` type)
+  that has the type and size of its pointee data described by other function
+  parameters. The type is usually described by a type-tagged enum. This allows
+  functions (e.g. `olGetDeviceInfo`) to return data of an arbitrary type.
 
 #### Return
+
 A return represents a possible return code from the function, and optionally a
 list of conditions in which this value may be returned. The conditions list is
-not expected to be exhaustive. A condition is considered free-form text, but
-if it is wrapped in \`backticks\` then it is treated as literal code
-representing an error condition (e.g. `someParam < 1`). These conditions are
-used to automatically create validation checks by the `offload-tblgen`
-validation generator.
+not expected to be exhaustive. A condition is considered free-form text, but if
+it is wrapped in \`backticks\` then it is treated as literal code representing
+an error condition (e.g. `someParam < 1`). These conditions are used to
+automatically create validation checks by the `offload-tblgen` validation
+generator.
 
 Returns are automatically generated for functions with pointer or handle
 parameters, so API authors do not need to exhaustively add null checks for
@@ -49,6 +84,7 @@ values automatically.
 
 
 ### Struct
+
 Represents a struct. Contains a list of members, which each have a *type*,
 *name*, and *desc*.
 
@@ -59,24 +95,28 @@ actual C++ inheritance, but instead explicitly has those members copied in,
 which preserves ABI compatibility with C.
 
 ### Enum
+
 Represents a C-style enum. Contains a list of `etor` values, which have a name
 and description.
 
-A `TaggedEtor` record type also exists which additionally takes a type. This type
-is used when the enum is used as a parameter to a function with a type-tagged
-function parameter (e.g. `olGetDeviceInfo`).
+A `TaggedEtor` record type also exists which additionally takes a type. This
+type is used when the enum is used as a parameter to a function with a
+type-tagged function parameter (e.g. `olGetDeviceInfo`).
 
 All enums automatically get a `<enum_name>_FORCE_UINT32 = 0x7fffffff` value,
 which forces the underlying type to be uint32.
 
 ### Handle
+
 Represents a pointer to an opaque struct, as described in the Parameter section.
 It does not take any extra fields.
 
 ### Typedef
+
 Represents a typedef, contains only a *value* field.
 
 ### Macro
+
 Represents a C preprocessor `#define`. Contains a *value* field. Optionally
 takes a *condition* field, which allows the macro to be conditionally defined,
 and an *alt_value* field, which represents the value if the condition is false.
@@ -90,15 +130,20 @@ files, rather than requiring a mix of C source and tablegen.
 ## Generation
 
 ### API header
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-api
 ```
 The comments in the generated header are in Doxygen format, although
 generating documentation from them hasn't been implemented yet.
 
-The entirety of this header is generated by Tablegen, rather than having a predefined header file that includes one or more `.inc` files. This is because this header is expected to be part of the installation and distributed to end-users, so should be self-contained.
+The entirety of this header is generated by Tablegen, rather than having a
+predefined header file that includes one or more `.inc` files. This is because
+this header is expected to be part of the installation and distributed to
+end-users, so should be self-contained.
 
 ### Entry Points
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-entry-points
 ```
@@ -111,6 +156,7 @@ function calls with arguments and results. The tracing can be enabled with the
 `OFFLOAD_TRACE` environment variable.
 
 ### Implementation function declarations
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-impl-func-decls
 ```
@@ -118,11 +164,12 @@ Generates declarations of the implementation of functions of every entry point
 in the API, e.g. `offloadDeviceFoo_impl` for `offloadDeviceFoo`.
 
 ### Print header
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-print-header
 ```
-This header contains `std::ostream &operator<<(std::ostream&)` definitions for
-various API objects, including function parameters.
+This header contains `llvm::raw_ostream &operator<<(llvm::raw_ostream &)`
+definitions for various API objects, including function parameters.
 
 As with the API header, it is expected that this header is part of the installed
 package, so it is entirely generated by Tablegen.
@@ -130,21 +177,7 @@ package, so it is entirely generated by Tablegen.
 For ease of implementation, and since it is not strictly part of the API, this
 is a C++ header file. If a C version is desirable it could be added.
 
-### Future Tablegen backends
-`RecordTypes.hpp` contains wrappers for all of the API object types, which will
-allow more backends to be easily added in future.
-
-## Adding to the API
-
-A new object can be added to the API by adding to one of the existing `.td`
-files. It is also possible to add a new tablegen file to the API by adding it
-to the includes in `OffloadAPI.td`. When the `OffloadGenerate` target is
-rebuilt, the new definition will be included in the generated files.
-
-### Adding a new entry point
+### Additional Tablegen backends
 
-When a new entry point is added (e.g. `offloadDeviceFoo`), the actual entry
-point is automatically generated, which contains validation and tracing code.
-It expects an implementation function (`offloadDeviceFoo_impl`) to be defined,
-which it will call into. The definition of this implementation function should
-be added to `src/OffloadImpl.cpp`
+`RecordTypes.hpp` contains wrappers for all of the API object types, which
+allows new backends to be easily added if needed.
diff --git a/offload/liboffload/README.md b/offload/liboffload/README.md
index 95c9bf54d7bad..267d27172828c 100644
--- a/offload/liboffload/README.md
+++ b/offload/liboffload/README.md
@@ -1,8 +1,34 @@
-# Offload New API
+# Liboffload
 
-This directory contains the implementation of the experimental work-in-progress
-new API for Offload. It builds on top of the existing plugin implementations but
-provides a single level of abstraction suitable for runtimes for languages other
-than OpenMP to be built on top of.
+This directory contains the implementation of the work-in-progress new API for
+Offload. It builds on top of the existing plugin implementations but provides a
+single level of abstraction suitable for implementation of many offloading
+language runtimes, rather than just OpenMP.
 
-See the [API definition readme](API/README.md) for implementation details.
\ No newline at end of file
+## Testing liboffload
+
+The main test suite for liboffload can be run with the `check-offload-unit`
+target, which runs the `offload.unittests` executable. The test suite will
+automatically run on every available device, but can be restricted to a single
+platform (CUDA, AMDGPU) with a command line argument:
+
+```sh
+$ ./offload.unittests --platform=CUDA
+```
+
+Tracing of Offload API calls can be enabled by setting the `OFFLOAD_TRACE`
+environment variable. This works with any program that uses liboffload.
+
+```sh
+$ OFFLOAD_TRACE=1 ./offload.unittests
+---> olInit()-> OL_SUCCESS
+# etc
+```
+
+The host plugin is not currently supported.
+
+## Modifying liboffload
+
+The main header (`OffloadAPI.h`) and some implementation details are
+autogenerated with tablegen. See the [API definition README](API/README.md)
+for implementation details.
diff --git a/offload/liboffload/src/Helpers.hpp b/offload/liboffload/src/Helpers.hpp
index 425934b6760d8..8b85945508b98 100644
--- a/offload/liboffload/src/Helpers.hpp
+++ b/offload/liboffload/src/Helpers.hpp
@@ -61,39 +61,41 @@ llvm::Error getInfoArray(size_t array_length, size_t ParamValueSize,
                      array_length * sizeof(T), memcpy);
 }
 
-template <>
-inline llvm::Error
-getInfo<const char *>(size_t ParamValueSize, void *ParamValue,
-                      size_t *ParamValueSizeRet, const char *Value) {
-  return getInfoArray(strlen(Value) + 1, ParamValueSize, ParamValue,
-                      ParamValueSizeRet, Value);
+llvm::Error getInfoString(size_t ParamValueSize, void *ParamValue,
+                          size_t *ParamValueSizeRet, llvm::StringRef Value) {
+  return getInfoArray(Value.size() + 1, ParamValueSize, ParamValue,
+                      ParamValueSizeRet, Value.data());
 }
 
-class ReturnHelper {
+class InfoWriter {
 public:
-  ReturnHelper(size_t ParamValueSize, void *ParamValue,
-               size_t *ParamValueSizeRet)
-      : ParamValueSize(ParamValueSize), ParamValue(ParamValue),
-        ParamValueSizeRet(ParamValueSizeRet) {}
+  InfoWriter(size_t Size, void *Target, size_t *SizeRet)
+      : Size(Size), Target(Target), SizeRet(SizeRet) {};
+  InfoWriter() = delete;
+  InfoWriter(InfoWriter &) = delete;
+  ~InfoWriter() = default;
 
-  // A version where in/out info size is represented by a single pointer
-  // to a value which is updated on return
-  ReturnHelper(size_t *ParamValueSize, void *ParamValue)
-      : ParamValueSize(*ParamValueSize), ParamValue(ParamValue),
-        ParamValueSizeRet(ParamValueSize) {}
+  template <typename T> llvm::Error write(llvm::Expected<T> &&Val) {
+    if (Val)
+      return getInfo(Size, Target, SizeRet, *Val);
+    return Val.takeError();
+  }
 
-  // Scalar return Value
-  template <class T> llvm::Error operator()(const T &t) {
-    return getInfo(ParamValueSize, ParamValue, ParamValueSizeRet, t);
+  template <typename T>
+  llvm::Error writeArray(llvm::Expected<T> &&Val, size_t Elems) {
+    if (Val)
+      return getInfoArray(Elems, Size, Target, SizeRet, *Val);
+    return Val.takeError();
   }
 
-  // Array return Value
-  template <class T> llvm::Error operator()(const T *t, size_t s) {
-    return getInfoArray(s, ParamValueSize, ParamValue, ParamValueSizeRet, t);
+  llvm::Error writeString(llvm::Expected<llvm::StringRef> &&Val) {
+    if (Val)
+      return getInfoString(Size, Target, SizeRet, *Val);
+    return Val.takeError();
   }
 
-protected:
-  size_t ParamValueSize;
-  void *ParamValue;
-  size_t *ParamValueSizeRet;
+private:
+  size_t Size;
+  void *Target;
+  size_t *SizeRet;
 };
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 6adebb25a2db0..f9da638436705 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -13,6 +13,7 @@
 
 #include "OffloadImpl.hpp"
 #include "Helpers.hpp"
+#include "OffloadPrint.hpp"
 #include "PluginManager.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <OffloadAPI.h>
@@ -96,7 +97,10 @@ struct AllocInfo {
 
 // Global shared state for liboffload
 struct OffloadContext;
-static OffloadContext *OffloadContextVal;
+// This pointer is non-null if and only if the context is valid and fully
+// initialized
+static std::atomic<OffloadContext *> OffloadContextVal;
+std::mutex OffloadContextValMutex;
 struct OffloadContext {
   OffloadContext(OffloadContext &) = delete;
   OffloadContext(OffloadContext &&) = delete;
@@ -107,6 +111,7 @@ struct OffloadContext {
   bool ValidationEnabled = true;
   DenseMap<void *, AllocInfo> AllocInfoMap{};
   SmallVector<ol_platform_impl_t, 4> Platforms{};
+  size_t RefCount;
 
   ol_device_handle_t HostDevice() {
     // The host platform is always inserted last
@@ -145,20 +150,18 @@ constexpr ol_platform_backend_t pluginNameToBackend(StringRef Name) {
 #define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();
 #include "Shared/Targets.def"
 
-Error initPlugins() {
-  auto *Context = new OffloadContext{};
-
+Error initPlugins(OffloadContext &Context) {
   // Attempt to create an instance of each supported plugin.
 #define PLUGIN_TARGET(Name)                                                    \
   do {                                                                         \
-    Context->Platforms.emplace_back(ol_platform_impl_t{                        \
+    Context.Platforms.emplace_back(ol_platform_impl_t{                         \
         std::unique_ptr<GenericPluginTy>(createPlugin_##Name()),               \
         pluginNameToBackend(#Name)});                                          \
   } while (false);
 #include "Shared/Targets.def"
 
   // Preemptively initialize all devices in the plugin
-  for (auto &Platform : Context->Platforms) {
+  for (auto &Platform : Context.Platforms) {
     // Do not use the host plugin - it isn't supported.
     if (Platform.BackendType == OL_PLATFORM_BACKEND_UNKNOWN)
       continue;
@@ -178,52 +181,76 @@ Error initPlugins() {
   }
 
   // Add the special host device
-  auto &HostPlatform = Context->Platforms.emplace_back(
+  auto &HostPlatform = Context.Platforms.emplace_back(
       ol_platform_impl_t{nullptr, OL_PLATFORM_BACKEND_HOST});
   HostPlatform.Devices.emplace_back(-1, nullptr, nullptr, InfoTreeNode{});
-  Context->HostDevice()->Platform = &HostPlatform;
-
-  Context->TracingEnabled = std::getenv("OFFLOAD_TRACE");
-  Context->ValidationEnabled = !std::getenv("OFFLOAD_DISABLE_VALIDATION");
+  Context.HostDevice()->Platform = &HostPlatform;
 
-  OffloadContextVal = Context;
+  Context.TracingEnabled = std::getenv("OFFLOAD_TRACE");
+  Context.ValidationEnabled = !std::getenv("OFFLOAD_DISABLE_VALIDATION");
 
   return Plugin::success();
 }
 
-// TODO: We can properly reference count here and manage the resources in a more
-// clever way
 Error olInit_impl() {
-  static std::once_flag InitFlag;
-  std::optional<Error> InitResult{};
-  std::call_once(InitFlag, [&] { InitResult = initPlugins(); });
+  std::lock_guard<std::mutex> Lock{OffloadContextValMutex};
 
-  if (InitResult)
-    return std::move(*InitResult);
-  return Error::success();
+  if (isOffloadInitialized()) {
+    OffloadContext::get().RefCount++;
+    return Plugin::success();
+  }
+
+  // Use a temporary to ensure that entry points querying OffloadContextVal do
+  // not get a partially initialized context
+  auto *NewContext = new OffloadContext{};
+  Error InitResult = initPlugins(*NewContext);
+  OffloadContextVal.store(NewContext);
+  OffloadContext::get().RefCount++;
+
+  return InitResult;
+}
+
+Error olShutDown_impl() {
+  std::lock_guard<std::mutex> Lock{OffloadContextValMutex};
+
+  if (--OffloadContext::get().RefCount != 0)
+    return Error::success();
+
+  llvm::Error Result = Error::success();
+  auto *OldContext = OffloadContextVal.exchange(nullptr);
+
+  for (auto &P : OldContext->Platforms) {
+    // Host plugin is nullptr and has no deinit
+    if (!P.Plugin)
+      continue;
+
+    if (auto Res = P.Plugin->deinit())
+      Result = llvm::joinErrors(std::move(Result), std::move(Res));
+  }
+
+  delete OldContext;
+  return Result;
 }
-Error olShutDown_impl() { return Error::success(); }
 
 Error olGetPlatformInfoImplDetail(ol_platform_handle_t Platform,
                                   ol_platform_info_t PropName, size_t PropSize,
                                   void *PropValue, size_t *PropSizeRet) {
-  ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
+  InfoWriter Info(PropSize, PropValue, PropSizeRet);
   bool IsHost = Platform->BackendType == OL_PLATFORM_BACKEND_HOST;
 
   switch (PropName) {
   case OL_PLATFORM_INFO_NAME:
-    return ReturnValue(IsHost ? "Host" : Platform->Plugin->getName());
+    return Info.writeString(IsHost ? "Host" : Platform->Plugin->getName());
   case OL_PLATFORM_INFO_VENDOR_NAME:
     // TODO: Implement this
-    return ReturnValue("Unknown platform vendor");
+    return Info.writeString("Unknown platform vendor");
   case OL_PLATFORM_INFO_VERSION: {
-    return ReturnValue(formatv("v{0}.{1}.{2}", OL_VERSION_MAJOR,
-                               OL_VERSION_MINOR, OL_VERSION_PATCH)
-                           .str()
-                           .c_str());
+    return Info.writeString(formatv("v{0}.{1}.{2}", OL_VERSION_MAJOR,
+                                    OL_VERSION_MINOR, OL_VERSION_PATCH)
+                                .str());
   }
   case OL_PLATFORM_INFO_BACKEND: {
-    return ReturnValue(Platform->BackendType);
+    return Info.write<ol_platform_backend_t>(Platform->BackendType);
   }
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
@@ -250,36 +277,108 @@ Error olGetPlatformInfoSize_impl(ol_platform_handle_t Platform,
 Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
                                 ol_device_info_t PropName, size_t PropSize,
                                 void *PropValue, size_t *PropSizeRet) {
+  assert(Device != OffloadContext::get().HostDevice());
+  InfoWriter Info(PropSize, PropValue, PropSizeRet);
 
-  ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
+  auto makeError = [&](ErrorCode Code, StringRef Err) {
+    std::string ErrBuffer;
+    llvm::raw_string_ostream(ErrBuffer) << PropName << ": " << Err;
+    return Plugin::error(ErrorCode::UNIMPLEMENTED, ErrBuffer.c_str());
+  };
 
   // Find the info if it exists under any of the given names
-  auto GetInfoString = [&](std::vector<std::string> Names) {
-    if (Device == OffloadContext::get().HostDevice())
-      return "Host";
-
-    for (auto Name : Names) {
-      if (auto Entry = Device->Info.get(Name))
+  auto getInfoString =
+      [&](std::vector<std::string> Names) -> llvm::Expected<const char *> {
+    for (auto &Name : Names) {
+      if (auto Entry = Device->Info.get(Name)) {
+        if (!std::holds_alternative<std::string>((*Entry)->Value))
+          return makeError(ErrorCode::BACKEND_FAILURE,
+                           "plugin returned incorrect type");
         return std::get<std::string>((*Entry)->Value).c_str();
+      }
+    }
+
+    return makeError(ErrorCode::UNIMPLEMENTED,
+                     "plugin did not provide a response for this information");
+  };
+
+  auto getInfoXyz =
+      [&](std::vector<std::string> Names) -> llvm::Expected<ol_dimensions_t> {
+    for (auto &Name : Names) {
+      if (auto Entry = Device->Info.get(Name)) {
+        auto Node = *Entry;
+        ol_dimensions_t Out{0, 0, 0};
+
+        auto getField = [&](StringRef Name, uint32_t &Dest) {
+          if (auto F = Node->get(Name)) {
+            if (!std::holds_alternative<size_t>((*F)->Value))
+              return makeError(
+                  ErrorCode::BACKEND_FAILURE,
+                  "plugin returned incorrect type for dimensions element");
+            Dest = std::get<size_t>((*F)->Value);
+          } else
+            return makeError(ErrorCode::BACKEND_FAILURE,
+                             "plugin didn't provide all values for dimensions");
+          return Plugin::success();
+        };
+
+        if (auto Res = getField("x", Out.x))
+          return Res;
+        if (auto Res = getField("y", Out.y))
+          return Res;
+        if (auto Res = getField("z", Out.z))
+          return Res;
+
+        return Out;
+      }
     }
 
-    return "";
+    return makeError(ErrorCode::UNIMPLEMENTED,
+                     "plugin did not provide a response for this information");
   };
 
   switch (PropName) {
   case OL_DEVICE_INFO_PLATFORM:
-    return ReturnValue(Device->Platform);
+    return Info.write<void *>(Device->Platform);
+  case OL_DEVICE_INFO_TYPE:
+    return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_GPU);
+  case OL_DEVICE_INFO_NAME:
+    return Info.writeString(getInfoString({"Device Name"}));
+  case OL_DEVICE_INFO_VENDOR:
+    return Info.writeString(getInfoString({"Vendor Name"}));
+  case OL_DEVICE_INFO_DRIVER_VERSION:
+    return Info.writeString(
+        getInfoString({"CUDA Driver Version", "HSA Runtime Version"}));
+  case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
+    return Info.write(getInfoXyz({"Workgroup Max Size per Dimension" /*AMD*/,
+                                  "Maximum Block Dimensions" /*CUDA*/}));
+  default:
+    return createOffloadError(ErrorCode::INVALID_ENUMERATION,
+                              "getDeviceInfo enum '%i' is invalid", PropName);
+  }
+
+  return Error::success();
+}
+
+Error olGetDeviceInfoImplDetailHost(ol_device_handle_t Device,
+                                    ol_device_info_t PropName, size_t PropSize,
+                                    void *PropValue, size_t *PropSizeRet) {
+  assert(Device == OffloadContext::get().HostDevice());
+  InfoWriter Info(PropSize, PropValue, PropSizeRet);
+
+  switch (PropName) {
+  case OL_DEVICE_INFO_PLATFORM:
+    return Info.write<void *>(Device->Platform);
   case OL_DEVICE_INFO_TYPE:
-    return Device == OffloadContext::get().HostDevice()
-               ? ReturnValue(OL_DEVICE_TYPE_HOST)
-               : ReturnValue(OL_DEVICE_TYPE_GPU);
+    return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_HOST);
   case OL_DEVICE_INFO_NAME:
-    return ReturnValue(GetInfoString({"Device Name"}));
+    return Info.writeString("Virtual Host Device");
   case OL_DEVICE_INFO_VENDOR:
-    return ReturnValue(GetInfoString({"Vendor Name"}));
+    return Info.writeString("Liboffload");
   case OL_DEVICE_INFO_DRIVER_VERSION:
-    return ReturnValue(
-        GetInfoString({"CUDA Driver Version", "HSA Runtime Version"}));
+    return Info.writeString(LLVM_VERSION_STRING);
+  case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
+    return Info.write<ol_dimensions_t>(ol_dimensions_t{1, 1, 1});
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "getDeviceInfo enum '%i' is invalid", PropName);
@@ -290,12 +389,18 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
 
 Error olGetDeviceInfo_impl(ol_device_handle_t Device, ol_device_info_t PropName,
                            size_t PropSize, void *PropValue) {
+  if (Device == OffloadContext::get().HostDevice())
+    return olGetDeviceInfoImplDetailHost(Device, PropName, PropSize, PropValue,
+                                         nullptr);
   return olGetDeviceInfoImplDetail(Device, PropName, PropSize, PropValue,
                                    nullptr);
 }
 
 Error olGetDeviceInfoSize_impl(ol_device_handle_t Device,
                                ol_device_info_t PropName, size_t *PropSizeRet) {
+  if (Device == OffloadContext::get().HostDevice())
+    return olGetDeviceInfoImplDetailHost(Device, PropName, 0, nullptr,
+                                         PropSizeRet);
   return olGetDeviceInfoImplDetail(Device, PropName, 0, nullptr, PropSizeRet);
 }
 
diff --git a/offload/libomptarget/OpenMP/Mapping.cpp b/offload/libomptarget/OpenMP/Mapping.cpp
index 14f5e7dc9d19f..9b3533895f2a6 100644
--- a/offload/libomptarget/OpenMP/Mapping.cpp
+++ b/offload/libomptarget/OpenMP/Mapping.cpp
@@ -326,6 +326,28 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
   // data transfer.
   if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
       (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
+
+    // If we have something like:
+    //   #pragma omp target map(to: s.myarr[0:10]) map(to: s.myarr[0:10])
+    // then we see two "new" mappings of the struct member s.myarr here --
+    // and both have the "IsNewEntry" flag set, so trigger the copy to device
+    // below.  But, the shadow pointer is only initialised on the target for
+    // the first copy, and the second copy clobbers it.  So, this condition
+    // avoids the (second) copy here if we have already set shadow pointer info.
+    auto FailOnPtrFound = [HstPtrBegin, Size](ShadowPtrInfoTy &SP) {
+      if (SP.HstPtrAddr >= HstPtrBegin &&
+          SP.HstPtrAddr < (void *)((char *)HstPtrBegin + Size))
+        return OFFLOAD_FAIL;
+      return OFFLOAD_SUCCESS;
+    };
+    if (LR.TPR.getEntry()->foreachShadowPointerInfo(FailOnPtrFound) ==
+        OFFLOAD_FAIL) {
+      DP("Multiple new mappings of %" PRId64 " bytes detected (hst:" DPxMOD
+         ") -> (tgt:" DPxMOD ")\n",
+         Size, DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
+      return std::move(LR.TPR);
+    }
+
     DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
        DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
 
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 35cb297f65188..832c31c43b5d2 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3504,6 +3504,9 @@ static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   case HSA_STATUS_ERROR_INVALID_SYMBOL_NAME:
     OffloadErrCode = ErrorCode::NOT_FOUND;
     break;
+  case HSA_STATUS_ERROR_INVALID_CODE_OBJECT:
+    OffloadErrCode = ErrorCode::INVALID_BINARY;
+    break;
   default:
     OffloadErrCode = ErrorCode::UNKNOWN;
   }
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index fbc798faec24b..162b149ab483e 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -317,7 +317,7 @@ struct GenericKernelTy {
                            AsyncInfoWrapperTy &AsyncInfoWrapper) const = 0;
 
   /// Get the kernel name.
-  const char *getName() const { return Name; }
+  const char *getName() const { return Name.c_str(); }
 
   /// Get the kernel image.
   DeviceImageTy &getImage() const {
@@ -413,7 +413,7 @@ struct GenericKernelTy {
   }
 
   /// The kernel name.
-  const char *Name;
+  std::string Name;
 
   /// The image that contains this kernel.
   DeviceImageTy *ImagePtr = nullptr;
@@ -1198,6 +1198,8 @@ struct GenericPluginTy {
     return reinterpret_cast<Ty *>(Allocator.Allocate(sizeof(Ty), alignof(Ty)));
   }
 
+  template <typename Ty> void free(Ty *Mem) { Allocator.Deallocate(Mem); }
+
   /// Get the reference to the global handler of this plugin.
   GenericGlobalHandlerTy &getGlobalHandler() {
     assert(GlobalHandler && "Global handler not initialized");
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index ac7031b6e881c..81b9d423e13d8 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -456,7 +456,7 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
     KernelEnvironment = KernelEnvironmentTy{};
     DP("Failed to read kernel environment for '%s' Using default Bare (0) "
        "execution mode\n",
-       Name);
+       getName());
   }
 
   // Max = Config.Max > 0 ? min(Config.Max, Device.Max) : Device.Max;
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
index e5332686fcffb..361a781e8f9b6 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
@@ -31,6 +31,7 @@ DLWRAP(cuDeviceGet, 2)
 DLWRAP(cuDeviceGetAttribute, 3)
 DLWRAP(cuDeviceGetCount, 1)
 DLWRAP(cuFuncGetAttribute, 3)
+DLWRAP(cuFuncSetAttribute, 3)
 
 // Device info
 DLWRAP(cuDeviceGetName, 3)
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
index 1c5b421768894..b6c022c8e7e8b 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
@@ -258,6 +258,7 @@ typedef enum CUdevice_attribute_enum {
 
 typedef enum CUfunction_attribute_enum {
   CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK = 0,
+  CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES = 8,
 } CUfunction_attribute;
 
 typedef enum CUctx_flags_enum {
@@ -295,6 +296,7 @@ CUresult cuDeviceGet(CUdevice *, int);
 CUresult cuDeviceGetAttribute(int *, CUdevice_attribute, CUdevice);
 CUresult cuDeviceGetCount(int *);
 CUresult cuFuncGetAttribute(int *, CUfunction_attribute, CUfunction);
+CUresult cuFuncSetAttribute(CUfunction, CUfunction_attribute, int);
 
 // Device info
 CUresult cuDeviceGetName(char *, int, CUdevice);
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index f1164074f9ea9..53089df2d0f0d 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -160,6 +160,9 @@ struct CUDAKernelTy : public GenericKernelTy {
 private:
   /// The CUDA kernel function to execute.
   CUfunction Func;
+  /// The maximum amount of dynamic shared memory per thread group. By default,
+  /// this is set to 48 KB.
+  mutable uint32_t MaxDynCGroupMemLimit = 49152;
 };
 
 /// Class wrapping a CUDA stream reference. These are the objects handled by the
@@ -940,6 +943,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (Res == CUDA_SUCCESS)
       Info.add("Device Name", TmpChar);
 
+    Info.add("Vendor Name", "NVIDIA");
+
     Res = cuDeviceTotalMem(&TmpSt, Device);
     if (Res == CUDA_SUCCESS)
       Info.add("Global Memory Size", TmpSt, "bytes");
@@ -1302,6 +1307,16 @@ Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   if (GenericDevice.getRPCServer())
     GenericDevice.Plugin.getRPCServer().Thread->notify();
 
+  // In case we require more memory than the current limit.
+  if (MaxDynCGroupMem >= MaxDynCGroupMemLimit) {
+    CUresult AttrResult = cuFuncSetAttribute(
+        Func, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, MaxDynCGroupMem);
+    Plugin::check(
+        AttrResult,
+        "Error in cuLaunchKernel while setting the memory limits: %s");
+    MaxDynCGroupMemLimit = MaxDynCGroupMem;
+  }
+
   CUresult Res = cuLaunchKernel(Func, NumBlocks[0], NumBlocks[1], NumBlocks[2],
                                 NumThreads[0], NumThreads[1], NumThreads[2],
                                 MaxDynCGroupMem, Stream, nullptr, Config);
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index a35910aece986..d950572265b4c 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -151,7 +151,12 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
   ///
   /// TODO: This currently does nothing, and should be implemented as part of
   /// broader memory handling logic for this plugin
-  Error unloadBinaryImpl(DeviceImageTy *) override { return Plugin::success(); }
+  Error unloadBinaryImpl(DeviceImageTy *Image) override {
+    auto Elf = reinterpret_cast<GenELF64DeviceImageTy *>(Image);
+    DynamicLibrary::closeLibrary(Elf->getDynamicLibrary());
+    Plugin.free(Elf);
+    return Plugin::success();
+  }
 
   /// Deinitialize the device, which is a no-op
   Error deinitImpl() override { return Plugin::success(); }
@@ -212,8 +217,7 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
 
     // Load the temporary file as a dynamic library.
     std::string ErrMsg;
-    DynamicLibrary DynLib =
-        DynamicLibrary::getPermanentLibrary(TmpFileName, &ErrMsg);
+    DynamicLibrary DynLib = DynamicLibrary::getLibrary(TmpFileName, &ErrMsg);
 
     // Check if the loaded library is valid.
     if (!DynLib.isValid())
diff --git a/offload/test/mapping/duplicate_mappings_1.cpp b/offload/test/mapping/duplicate_mappings_1.cpp
new file mode 100644
index 0000000000000..5bed3f161b4fe
--- /dev/null
+++ b/offload/test/mapping/duplicate_mappings_1.cpp
@@ -0,0 +1,27 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic -Wno-openmp-mapping && %libomptarget-run-generic
+
+// clang-format on
+
+#include <assert.h>
+
+struct Inner {
+  int *data;
+  Inner(int size) { data = new int[size](); }
+  ~Inner() { delete[] data; }
+};
+
+struct Outer {
+  Inner i;
+  Outer() : i(10) {}
+};
+
+int main() {
+  Outer o;
+#pragma omp target map(tofrom : o.i.data[0 : 10]) map(tofrom : o.i.data[0 : 10])
+  {
+    o.i.data[0] = 42;
+  }
+  assert(o.i.data[0] == 42);
+  return 0;
+}
diff --git a/offload/test/mapping/duplicate_mappings_2.cpp b/offload/test/mapping/duplicate_mappings_2.cpp
new file mode 100644
index 0000000000000..ca8112ee72544
--- /dev/null
+++ b/offload/test/mapping/duplicate_mappings_2.cpp
@@ -0,0 +1,29 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic -Wno-openmp-mapping && %libomptarget-run-generic
+
+#include <assert.h>
+
+// clang-format on
+
+struct Inner {
+  int *data;
+  Inner(int size) { data = new int[size](); }
+  ~Inner() { delete[] data; }
+};
+#pragma omp declare mapper(Inner i) map(i, i.data[0 : 10])
+
+struct Outer {
+  Inner i;
+  Outer() : i(10) {}
+};
+#pragma omp declare mapper(Outer o) map(o, o.i)
+
+int main() {
+  Outer o;
+#pragma omp target map(tofrom : o)
+  {
+    o.i.data[0] = 42;
+  }
+  assert(o.i.data[0] == 42);
+  return 0;
+}
diff --git a/offload/test/mapping/map_ptr_and_star_global.c b/offload/test/mapping/map_ptr_and_star_global.c
new file mode 100644
index 0000000000000..c3b0dd2f49e6b
--- /dev/null
+++ b/offload/test/mapping/map_ptr_and_star_global.c
@@ -0,0 +1,83 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+int x[10];
+int *p;
+
+void f1() {
+  p = &x[0];
+  p[0] = 111;
+  p[1] = 222;
+  p[2] = 333;
+  p[3] = 444;
+
+#pragma omp target enter data map(to : p)
+#pragma omp target enter data map(to : p[0 : 5])
+
+  int **p_mappedptr = (int **)omp_get_mapped_ptr(&p, omp_get_default_device());
+  int *x0_mappedptr =
+      (int *)omp_get_mapped_ptr(&x[0], omp_get_default_device());
+  int *x0_hostaddr = &x[0];
+
+  printf("p_mappedptr %s null\n", p_mappedptr == (int **)NULL ? "==" : "!=");
+  printf("x0_mappedptr %s null\n", x0_mappedptr == (int *)NULL ? "==" : "!=");
+
+// CHECK: p_mappedptr != null
+// CHECK: x0_mappedptr != null
+
+// p is predetermined firstprivate, so its address will be different from
+// the mapped address for this construct. So, any changes to p within the
+// region will not be visible after the construct.
+#pragma omp target map(*p) map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+// For the remaining constructs, p is not firstprivate, so its address will
+// be the same as the mapped address, and changes to p will be visible to any
+// subsequent regions.
+#pragma omp target map(to : *p, p)                                             \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // EXPECTED: 111 1 1 0
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+#pragma omp target map(to : p, *p)                                             \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-1],
+           x0_hostaddr == &p[-1]);
+    // EXPECTED: 222 1 1 0
+    // CHECK:    {{[0-9]+}} 0 0 0
+    p++;
+  }
+
+#pragma omp target map(present, alloc : p)                                     \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-2],
+           x0_hostaddr == &p[-2]);
+    // EXPECTED: 333 1 1 0
+    // CHECK:    111 1 0 0
+  }
+
+  // The following map(from:p) should not bring back p, because p is an
+  // attached pointer. So, it should still point to the same original
+  // location, &x[0], on host.
+#pragma omp target exit data map(always, from : p)
+  printf("%d %d\n", p[0], p == &x[0]);
+  // CHECK: 111 1
+
+#pragma omp target exit data map(delete : p[0 : 5], p)
+}
+
+int main() { f1(); }
diff --git a/offload/test/mapping/map_ptr_and_star_local.c b/offload/test/mapping/map_ptr_and_star_local.c
new file mode 100644
index 0000000000000..f0ca84d1cc4dd
--- /dev/null
+++ b/offload/test/mapping/map_ptr_and_star_local.c
@@ -0,0 +1,83 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+int x[10];
+
+void f1() {
+  int *p;
+  p = &x[0];
+  p[0] = 111;
+  p[1] = 222;
+  p[2] = 333;
+  p[3] = 444;
+
+#pragma omp target enter data map(to : p)
+#pragma omp target enter data map(to : p[0 : 5])
+
+  int **p_mappedptr = (int **)omp_get_mapped_ptr(&p, omp_get_default_device());
+  int *x0_mappedptr =
+      (int *)omp_get_mapped_ptr(&x[0], omp_get_default_device());
+  int *x0_hostaddr = &x[0];
+
+  printf("p_mappedptr %s null\n", p_mappedptr == (int **)NULL ? "==" : "!=");
+  printf("x0_mappedptr %s null\n", x0_mappedptr == (int *)NULL ? "==" : "!=");
+
+// CHECK: p_mappedptr != null
+// CHECK: x0_mappedptr != null
+
+// p is predetermined firstprivate, so its address will be different from
+// the mapped address for this construct. So, any changes to p within the
+// region will not be visible after the construct.
+#pragma omp target map(*p) map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+// For the remaining constructs, p is not firstprivate, so its address will
+// be the same as the mapped address, and changes to p will be visible to any
+// subsequent regions.
+#pragma omp target map(to : *p, p)                                             \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // EXPECTED: 111 1 1 0
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+#pragma omp target map(to : p, *p)                                             \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-1],
+           x0_hostaddr == &p[-1]);
+    // EXPECTED: 222 1 1 0
+    // CHECK:    {{[0-9]+}} 0 0 0
+    p++;
+  }
+
+#pragma omp target map(present, alloc : p)                                     \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-2],
+           x0_hostaddr == &p[-2]);
+    // EXPECTED: 333 1 1 0
+    // CHECK:    111 1 0 0
+  }
+
+  // The following map(from:p) should not bring back p, because p is an
+  // attached pointer. So, it should still point to the same original
+  // location, &x[0], on host.
+#pragma omp target exit data map(always, from : p)
+  printf("%d %d\n", p[0], p == &x[0]);
+  // CHECK: 111 1
+
+#pragma omp target exit data map(delete : p[0 : 5], p)
+}
+
+int main() { f1(); }
diff --git a/offload/test/mapping/map_ptr_and_subscript_global.c b/offload/test/mapping/map_ptr_and_subscript_global.c
new file mode 100644
index 0000000000000..a3a10b6c9b212
--- /dev/null
+++ b/offload/test/mapping/map_ptr_and_subscript_global.c
@@ -0,0 +1,83 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+int x[10];
+int *p;
+
+void f1() {
+  p = &x[0];
+  p[0] = 111;
+  p[1] = 222;
+  p[2] = 333;
+  p[3] = 444;
+
+#pragma omp target enter data map(to : p)
+#pragma omp target enter data map(to : p[0 : 5])
+
+  int **p_mappedptr = (int **)omp_get_mapped_ptr(&p, omp_get_default_device());
+  int *x0_mappedptr =
+      (int *)omp_get_mapped_ptr(&x[0], omp_get_default_device());
+  int *x0_hostaddr = &x[0];
+
+  printf("p_mappedptr %s null\n", p_mappedptr == (int **)NULL ? "==" : "!=");
+  printf("x0_mappedptr %s null\n", x0_mappedptr == (int *)NULL ? "==" : "!=");
+
+// CHECK: p_mappedptr != null
+// CHECK: x0_mappedptr != null
+
+// p is predetermined firstprivate, so its address will be different from
+// the mapped address for this construct. So, any changes to p within the
+// region will not be visible after the construct.
+#pragma omp target map(p[0]) map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+// For the remaining constructs, p is not firstprivate, so its address will
+// be the same as the mapped address, and changes to p will be visible to any
+// subsequent regions.
+#pragma omp target map(to : p[0], p)                                           \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // EXPECTED: 111 1 1 0
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+#pragma omp target map(to : p, p[0])                                           \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-1],
+           x0_hostaddr == &p[-1]);
+    // EXPECTED: 222 1 1 0
+    // CHECK:    111 0 0 0
+    p++;
+  }
+
+#pragma omp target map(present, alloc : p)                                     \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-2],
+           x0_hostaddr == &p[-2]);
+    // EXPECTED: 333 1 1 0
+    // CHECK:    111 1 0 0
+  }
+
+  // The following map(from:p) should not bring back p, because p is an
+  // attached pointer. So, it should still point to the same original
+  // location, &x[0], on host.
+#pragma omp target exit data map(always, from : p)
+  printf("%d %d\n", p[0], p == &x[0]);
+  // CHECK: 111 1
+
+#pragma omp target exit data map(delete : p[0 : 5], p)
+}
+
+int main() { f1(); }
diff --git a/offload/test/mapping/map_ptr_and_subscript_local.c b/offload/test/mapping/map_ptr_and_subscript_local.c
new file mode 100644
index 0000000000000..bb44999541a7b
--- /dev/null
+++ b/offload/test/mapping/map_ptr_and_subscript_local.c
@@ -0,0 +1,83 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+int x[10];
+
+void f1() {
+  int *p;
+  p = &x[0];
+  p[0] = 111;
+  p[1] = 222;
+  p[2] = 333;
+  p[3] = 444;
+
+#pragma omp target enter data map(to : p)
+#pragma omp target enter data map(to : p[0 : 5])
+
+  int **p_mappedptr = (int **)omp_get_mapped_ptr(&p, omp_get_default_device());
+  int *x0_mappedptr =
+      (int *)omp_get_mapped_ptr(&x[0], omp_get_default_device());
+  int *x0_hostaddr = &x[0];
+
+  printf("p_mappedptr %s null\n", p_mappedptr == (int **)NULL ? "==" : "!=");
+  printf("x0_mappedptr %s null\n", x0_mappedptr == (int *)NULL ? "==" : "!=");
+
+// CHECK: p_mappedptr != null
+// CHECK: x0_mappedptr != null
+
+// p is predetermined firstprivate, so its address will be different from
+// the mapped address for this construct. So, any changes to p within the
+// region will not be visible after the construct.
+#pragma omp target map(p[0]) map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+// For the remaining constructs, p is not firstprivate, so its address will
+// be the same as the mapped address, and changes to p will be visible to any
+// subsequent regions.
+#pragma omp target map(to : p[0], p)                                           \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[0],
+           x0_hostaddr == &p[0]);
+    // EXPECTED: 111 1 1 0
+    // CHECK:    111 0 1 0
+    p++;
+  }
+
+#pragma omp target map(to : p, p[0])                                           \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-1],
+           x0_hostaddr == &p[-1]);
+    // EXPECTED: 222 1 1 0
+    // CHECK:    111 0 0 0
+    p++;
+  }
+
+#pragma omp target map(present, alloc : p)                                     \
+    map(to : p_mappedptr, x0_mappedptr, x0_hostaddr)
+  {
+    printf("%d %d %d %d\n", p[0], p_mappedptr == &p, x0_mappedptr == &p[-2],
+           x0_hostaddr == &p[-2]);
+    // EXPECTED: 333 1 1 0
+    // CHECK:    111 1 0 0
+  }
+
+  // The following map(from:p) should not bring back p, because p is an
+  // attached pointer. So, it should still point to the same original
+  // location, &x[0], on host.
+#pragma omp target exit data map(always, from : p)
+  printf("%d %d\n", p[0], p == &x[0]);
+  // CHECK: 111 1
+
+#pragma omp target exit data map(delete : p[0 : 5], p)
+}
+
+int main() { f1(); }
diff --git a/offload/test/mapping/map_structptr_and_member_global.c b/offload/test/mapping/map_structptr_and_member_global.c
new file mode 100644
index 0000000000000..10e72e070dbc5
--- /dev/null
+++ b/offload/test/mapping/map_structptr_and_member_global.c
@@ -0,0 +1,88 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+typedef struct {
+  short x;
+  int *p;
+  long y;
+} S;
+
+S s[10], *ps;
+
+void f1() {
+  ps = &s[0];
+  s[0].x = 111;
+  s[1].x = 222;
+  s[2].x = 333;
+  s[3].x = 444;
+
+#pragma omp target enter data map(to : s)
+#pragma omp target enter data map(to : ps, ps->x)
+
+  S **ps_mappedptr = (S **)omp_get_mapped_ptr(&ps, omp_get_default_device());
+  short *s0_mappedptr =
+      (short *)omp_get_mapped_ptr(&s[0].x, omp_get_default_device());
+  short *s0_hostaddr = &s[0].x;
+
+  printf("ps_mappedptr %s null\n", ps_mappedptr == (S **)NULL ? "==" : "!=");
+  printf("s0_mappedptr %s null\n", s0_mappedptr == (short *)NULL ? "==" : "!=");
+
+// CHECK: ps_mappedptr != null
+// CHECK: s0_mappedptr != null
+
+// ps is predetermined firstprivate, so its address will be different from
+// the mapped address for this construct. So, any changes to p within the
+// region will not be visible after the construct.
+#pragma omp target map(ps->x) map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps, s0_mappedptr == &ps->x,
+           s0_hostaddr == &ps->x);
+    // CHECK: 111 0 1 0
+    ps++;
+  }
+
+// For the remaining constructs, ps is not firstprivate, so its address will
+// be the same as the mapped address, and changes to ps will be visible to any
+// subsequent regions.
+#pragma omp target map(to : ps->x, ps)                                         \
+    map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps, s0_mappedptr == &ps->x,
+           s0_hostaddr == &ps->x);
+    // EXPECTED: 111 1 1 0
+    // CHECK:    111 0 1 0
+    ps++;
+  }
+
+#pragma omp target map(to : ps, ps->x)                                         \
+    map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps,
+           s0_mappedptr == &ps[-1].x, s0_hostaddr == &ps[-1].x);
+    // EXPECTED: 222 1 1 0
+    // CHECK:    111 0 0 0
+    ps++;
+  }
+
+#pragma omp target map(present, alloc : ps)                                    \
+    map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps,
+           s0_mappedptr == &ps[-2].x, s0_hostaddr == &ps[-2].x);
+    // EXPECTED: 333 1 1 0
+    // CHECK:    111 1 0 0
+  }
+
+  // The following map(from:ps) should not bring back ps, because ps is an
+  // attached pointer. So, it should still point to the same original
+  // location, &s[0], on host.
+#pragma omp target exit data map(always, from : ps)
+  printf("%d %d\n", ps->x, ps == &s[0]);
+  // CHECK: 111 1
+
+#pragma omp target exit data map(delete : ps, s)
+}
+
+int main() { f1(); }
diff --git a/offload/test/mapping/map_structptr_and_member_local.c b/offload/test/mapping/map_structptr_and_member_local.c
new file mode 100644
index 0000000000000..9e59551ad3d6c
--- /dev/null
+++ b/offload/test/mapping/map_structptr_and_member_local.c
@@ -0,0 +1,87 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+typedef struct {
+  short x;
+  int *p;
+  long y;
+} S;
+
+void f1() {
+  S s[10], *ps;
+  ps = &s[0];
+  s[0].x = 111;
+  s[1].x = 222;
+  s[2].x = 333;
+  s[3].x = 444;
+
+#pragma omp target enter data map(to : s)
+#pragma omp target enter data map(to : ps, ps->x)
+
+  S **ps_mappedptr = (S **)omp_get_mapped_ptr(&ps, omp_get_default_device());
+  short *s0_mappedptr =
+      (short *)omp_get_mapped_ptr(&s[0].x, omp_get_default_device());
+  short *s0_hostaddr = &s[0].x;
+
+  printf("ps_mappedptr %s null\n", ps_mappedptr == (S **)NULL ? "==" : "!=");
+  printf("s0_mappedptr %s null\n", s0_mappedptr == (short *)NULL ? "==" : "!=");
+
+// CHECK: ps_mappedptr != null
+// CHECK: s0_mappedptr != null
+
+// ps is predetermined firstprivate, so its address will be different from
+// the mapped address for this construct. So, any changes to p within the
+// region will not be visible after the construct.
+#pragma omp target map(ps->x) map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps, s0_mappedptr == &ps->x,
+           s0_hostaddr == &ps->x);
+    // CHECK: 111 0 1 0
+    ps++;
+  }
+
+// For the remaining constructs, ps is not firstprivate, so its address will
+// be the same as the mapped address, and changes to ps will be visible to any
+// subsequent regions.
+#pragma omp target map(to : ps->x, ps)                                         \
+    map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps, s0_mappedptr == &ps->x,
+           s0_hostaddr == &ps->x);
+    // EXPECTED: 111 1 1 0
+    // CHECK:    111 0 1 0
+    ps++;
+  }
+
+#pragma omp target map(to : ps, ps->x)                                         \
+    map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps,
+           s0_mappedptr == &ps[-1].x, s0_hostaddr == &ps[-1].x);
+    // EXPECTED: 222 1 1 0
+    // CHECK:    111 0 0 0
+    ps++;
+  }
+
+#pragma omp target map(present, alloc : ps)                                    \
+    map(to : ps_mappedptr, s0_mappedptr, s0_hostaddr)
+  {
+    printf("%d %d %d %d\n", ps->x, ps_mappedptr == &ps,
+           s0_mappedptr == &ps[-2].x, s0_hostaddr == &ps[-2].x);
+    // EXPECTED: 333 1 1 0
+    // CHECK:    111 1 0 0
+  }
+
+  // The following map(from:ps) should not bring back ps, because ps is an
+  // attached pointer. So, it should still point to the same original
+  // location, &s[0], on host.
+#pragma omp target exit data map(always, from : ps)
+  printf("%d %d\n", ps->x, ps == &s[0]);
+  // CHECK: 111 1
+
+#pragma omp target exit data map(delete : ps, s)
+}
+
+int main() { f1(); }
diff --git a/offload/test/offloading/disable_default_device.c b/offload/test/offloading/disable_default_device.c
new file mode 100644
index 0000000000000..77ea9d4669f19
--- /dev/null
+++ b/offload/test/offloading/disable_default_device.c
@@ -0,0 +1,22 @@
+// RUN: %libomptarget-compile-generic
+// RUN:   env OMP_TARGET_OFFLOAD=disabled %libomptarget-run-generic 2>&1 |
+// %fcheck-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+// Sanity checks to make sure that this works and is thread safe.
+int main() {
+  // CHECK: number of devices 0
+  printf("number of devices %d\n", omp_get_num_devices());
+  // CHECK:initial device 0
+  printf("initial device %d\n", omp_get_initial_device());
+  // CHECK:default device 0
+  printf("default device %d\n", omp_get_default_device());
+  // CHECK: PASS
+  if (omp_get_initial_device() == omp_get_default_device()) {
+    printf("PASS\n");
+    return 0;
+  }
+  return 1;
+}
diff --git a/offload/tools/offload-tblgen/APIGen.cpp b/offload/tools/offload-tblgen/APIGen.cpp
index 800c9cadfe38b..35b080e3c8011 100644
--- a/offload/tools/offload-tblgen/APIGen.cpp
+++ b/offload/tools/offload-tblgen/APIGen.cpp
@@ -34,7 +34,8 @@ static std::string MakeComment(StringRef in) {
     }
     out += std::string("/// ") +
            in.substr(LineStart, LineBreak - LineStart).str() + "\n";
-    LineStart = LineBreak + 1;
+    if (LineBreak != std::string::npos)
+      LineStart = LineBreak + 1;
   }
 
   return out;
diff --git a/offload/tools/offload-tblgen/MiscGen.cpp b/offload/tools/offload-tblgen/MiscGen.cpp
index bc3d204f78d89..14787729503e9 100644
--- a/offload/tools/offload-tblgen/MiscGen.cpp
+++ b/offload/tools/offload-tblgen/MiscGen.cpp
@@ -82,7 +82,7 @@ void EmitOffloadErrcodes(const RecordKeeper &Records, raw_ostream &OS) {
 #endif
 
 // Error codes are shared between PluginInterface and liboffload.
-// To add new error codes, add them to offload/liboffload/API/Common.td and run the GenerateOffload target.
+// To add new error codes, add them to offload/liboffload/API/Common.td.
 
 )";
 
diff --git a/offload/tools/offload-tblgen/PrintGen.cpp b/offload/tools/offload-tblgen/PrintGen.cpp
index a964ff09d0f6e..d1189688a90a3 100644
--- a/offload/tools/offload-tblgen/PrintGen.cpp
+++ b/offload/tools/offload-tblgen/PrintGen.cpp
@@ -213,6 +213,11 @@ template <typename T> inline void printTagged(llvm::raw_ostream &os, const void
                   "enum {0} value);\n",
                   EnumRec{R}.getName());
   }
+  for (auto *R : Records.getAllDerivedDefinitions("Struct")) {
+    OS << formatv("inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os, "
+                  "const struct {0} param);\n",
+                  StructRec{R}.getName());
+  }
   OS << "\n";
 
   // Create definitions
diff --git a/offload/tools/offload-tblgen/RecordTypes.hpp b/offload/tools/offload-tblgen/RecordTypes.hpp
index bb7c06c643da5..65c0a4ce4a2c7 100644
--- a/offload/tools/offload-tblgen/RecordTypes.hpp
+++ b/offload/tools/offload-tblgen/RecordTypes.hpp
@@ -157,9 +157,15 @@ class ParamRec {
   bool isHandleType() const { return getType().ends_with("_handle_t"); }
   bool isFptrType() const { return getType().ends_with("_cb_t"); }
   StringRef getDesc() const { return rec->getValueAsString("desc"); }
-  bool isIn() const { return dyn_cast<BitInit>(flags->getBit(0))->getValue(); }
-  bool isOut() const { return dyn_cast<BitInit>(flags->getBit(1))->getValue(); }
-  bool isOpt() const { return dyn_cast<BitInit>(flags->getBit(2))->getValue(); }
+  bool getFlagBit(unsigned int Bit) const {
+    if (auto *BitValue = dyn_cast<BitInit>(flags->getBit(Bit)))
+      return BitValue->getValue();
+    assert(false && "Parameter flags has no default or set value");
+    return false;
+  }
+  bool isIn() const { return getFlagBit(0); }
+  bool isOut() const { return getFlagBit(1); }
+  bool isOpt() const { return getFlagBit(2); }
 
   const Record *getRec() const { return rec; }
   std::optional<std::pair<StringRef, StringRef>> getRange() const {
diff --git a/offload/unittests/CMakeLists.txt b/offload/unittests/CMakeLists.txt
index 7cd41e1dcdafd..1eae785b8765f 100644
--- a/offload/unittests/CMakeLists.txt
+++ b/offload/unittests/CMakeLists.txt
@@ -81,9 +81,34 @@ function(add_offload_unittest test_dirname)
   target_include_directories(${target_name} PRIVATE ${PLUGINS_TEST_INCLUDE})
 endfunction()
 
+function(add_conformance_test test_name)
+  set(target_name "${test_name}.conformance")
+
+  list(TRANSFORM ARGN PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" OUTPUT_VARIABLE files)
+
+  if(NOT TARGET libc)
+    message(WARNING "Cannot run conformance tests without the LLVM C library")
+    return()
+  endif()
+
+  add_executable(${target_name} ${files})
+  add_dependencies(${target_name} ${PLUGINS_TEST_COMMON} ${test_name}.bin)
+  target_compile_definitions(${target_name} PRIVATE DEVICE_CODE_PATH="${CONFORMANCE_TEST_DEVICE_CODE_PATH}")
+  target_link_libraries(${target_name} PRIVATE ${PLUGINS_TEST_COMMON} libc)
+  target_include_directories(${target_name} PRIVATE ${PLUGINS_TEST_INCLUDE})
+  set_target_properties(${target_name} PROPERTIES EXCLUDE_FROM_ALL TRUE)
+
+  add_custom_target(offload.conformance.${test_name}
+    COMMAND $<TARGET_FILE:${target_name}>
+    DEPENDS ${target_name}
+    COMMENT "Running conformance test ${test_name}")
+  add_dependencies(offload.conformance offload.conformance.${test_name})
+endfunction()
+
 set(OFFLOAD_TESTS_FORCE_NVPTX_ARCH "" CACHE STRING
   "Force building of NVPTX device code for Offload unit tests with the given arch, e.g. sm_61")
 set(OFFLOAD_TESTS_FORCE_AMDGPU_ARCH "" CACHE STRING
   "Force building of AMDGPU device code for Offload unit tests with the given arch, e.g. gfx1030")
 
 add_subdirectory(OffloadAPI)
+add_subdirectory(Conformance)
diff --git a/offload/unittests/Conformance/CMakeLists.txt b/offload/unittests/Conformance/CMakeLists.txt
new file mode 100644
index 0000000000000..bc3141757372a
--- /dev/null
+++ b/offload/unittests/Conformance/CMakeLists.txt
@@ -0,0 +1,8 @@
+add_custom_target(offload.conformance)
+
+set(PLUGINS_TEST_COMMON LLVMOffload LLVMSupport)
+set(PLUGINS_TEST_INCLUDE ${LIBOMPTARGET_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/common)
+
+add_subdirectory(device_code)
+
+add_conformance_test(sin sin.cpp)
diff --git a/offload/unittests/Conformance/device_code/CMakeLists.txt b/offload/unittests/Conformance/device_code/CMakeLists.txt
new file mode 100644
index 0000000000000..223f04ccfb698
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/CMakeLists.txt
@@ -0,0 +1,4 @@
+# FIXME: Currently missing dependencies to build GPU portion automatically.
+add_offload_test_device_code(sin.c sin)
+
+set(OFFLOAD_TEST_DEVICE_CODE_PATH ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
diff --git a/offload/unittests/Conformance/device_code/sin.c b/offload/unittests/Conformance/device_code/sin.c
new file mode 100644
index 0000000000000..e969e60f352a2
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/sin.c
@@ -0,0 +1,4 @@
+#include <gpuintrin.h>
+#include <math.h>
+
+__gpu_kernel void kernel(double *out) { *out = sin(*out); }
diff --git a/offload/unittests/Conformance/sin.cpp b/offload/unittests/Conformance/sin.cpp
new file mode 100644
index 0000000000000..9e15690a9e9d7
--- /dev/null
+++ b/offload/unittests/Conformance/sin.cpp
@@ -0,0 +1,8 @@
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include <OffloadAPI.h>
+#include <math.h>
+
+llvm::StringRef DeviceBinsDirectory = DEVICE_CODE_PATH;
+
+int main() { llvm::errs() << sin(0.0) << "\n"; }
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
index 0247744911eaa..c534c45205993 100644
--- a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
@@ -77,6 +77,15 @@ TEST_P(olGetDeviceInfoTest, SuccessDriverVersion) {
   ASSERT_EQ(std::strlen(DriverVersion.data()), Size - 1);
 }
 
+TEST_P(olGetDeviceInfoTest, SuccessMaxWorkGroupSize) {
+  ol_dimensions_t Value{0, 0, 0};
+  ASSERT_SUCCESS(olGetDeviceInfo(Device, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE,
+                                 sizeof(Value), &Value));
+  ASSERT_GT(Value.x, 0u);
+  ASSERT_GT(Value.y, 0u);
+  ASSERT_GT(Value.z, 0u);
+}
+
 TEST_P(olGetDeviceInfoTest, InvalidNullHandleDevice) {
   ol_device_type_t DeviceType;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
index edd2704a722dd..a908078a25211 100644
--- a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
@@ -44,6 +44,14 @@ TEST_P(olGetDeviceInfoSizeTest, SuccessDriverVersion) {
   ASSERT_NE(Size, 0ul);
 }
 
+TEST_P(olGetDeviceInfoSizeTest, SuccessMaxWorkGroupSize) {
+  size_t Size = 0;
+  ASSERT_SUCCESS(
+      olGetDeviceInfoSize(Device, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE, &Size));
+  ASSERT_EQ(Size, sizeof(ol_dimensions_t));
+  ASSERT_EQ(Size, sizeof(uint32_t) * 3);
+}
+
 TEST_P(olGetDeviceInfoSizeTest, InvalidNullHandle) {
   size_t Size = 0;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/device_code/CMakeLists.txt b/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
index 132c7a7c51fb8..d867e2aae1316 100644
--- a/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
+++ b/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
@@ -2,6 +2,16 @@ add_offload_test_device_code(foo.c foo)
 add_offload_test_device_code(bar.c bar)
 # Compile with optimizations to eliminate AMDGPU implicit arguments.
 add_offload_test_device_code(noargs.c noargs -O3)
+add_offload_test_device_code(localmem.c localmem)
+add_offload_test_device_code(localmem_reduction.c localmem_reduction)
+add_offload_test_device_code(localmem_static.c localmem_static)
 
-add_custom_target(offload_device_binaries DEPENDS foo.bin bar.bin noargs.bin)
+add_custom_target(offload_device_binaries DEPENDS
+    foo.bin
+    bar.bin
+    noargs.bin
+    localmem.bin
+    localmem_reduction.bin
+    localmem_static.bin
+)
 set(OFFLOAD_TEST_DEVICE_CODE_PATH ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
diff --git a/offload/unittests/OffloadAPI/device_code/localmem.c b/offload/unittests/OffloadAPI/device_code/localmem.c
new file mode 100644
index 0000000000000..d70847900bc43
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device_code/localmem.c
@@ -0,0 +1,11 @@
+#include <gpuintrin.h>
+#include <stdint.h>
+
+extern __gpu_local uint32_t shared_mem[];
+
+__gpu_kernel void localmem(uint32_t *out) {
+  shared_mem[__gpu_thread_id(0)] = __gpu_thread_id(0);
+  shared_mem[__gpu_thread_id(0)] *= 2;
+  out[__gpu_thread_id(0) + (__gpu_num_threads(0) * __gpu_block_id(0))] =
+      shared_mem[__gpu_thread_id(0)];
+}
diff --git a/offload/unittests/OffloadAPI/device_code/localmem_reduction.c b/offload/unittests/OffloadAPI/device_code/localmem_reduction.c
new file mode 100644
index 0000000000000..8a9a46cfb6a11
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device_code/localmem_reduction.c
@@ -0,0 +1,16 @@
+#include <gpuintrin.h>
+#include <stdint.h>
+
+extern __gpu_local uint32_t shared_mem[];
+
+__gpu_kernel void localmem_reduction(uint32_t *out) {
+  shared_mem[__gpu_thread_id(0)] = 2;
+
+  __gpu_sync_threads();
+
+  if (__gpu_thread_id(0) == 0) {
+    out[__gpu_block_id(0)] = 0;
+    for (uint32_t i = 0; i < __gpu_num_threads(0); i++)
+      out[__gpu_block_id(0)] += shared_mem[i];
+  }
+}
diff --git a/offload/unittests/OffloadAPI/device_code/localmem_static.c b/offload/unittests/OffloadAPI/device_code/localmem_static.c
new file mode 100644
index 0000000000000..928b48422a0d6
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device_code/localmem_static.c
@@ -0,0 +1,17 @@
+#include <gpuintrin.h>
+#include <stdint.h>
+
+[[clang::loader_uninitialized]]
+__gpu_local uint32_t shared_mem[64];
+
+__gpu_kernel void localmem_static(uint32_t *out) {
+  shared_mem[__gpu_thread_id(0)] = 2;
+
+  __gpu_sync_threads();
+
+  if (__gpu_thread_id(0) == 0) {
+    out[__gpu_block_id(0)] = 0;
+    for (uint32_t i = 0; i < __gpu_num_threads(0); i++)
+      out[__gpu_block_id(0)] += shared_mem[i];
+  }
+}
diff --git a/offload/unittests/OffloadAPI/init/olInit.cpp b/offload/unittests/OffloadAPI/init/olInit.cpp
index 8e27e77cd0fb5..508615152b4f1 100644
--- a/offload/unittests/OffloadAPI/init/olInit.cpp
+++ b/offload/unittests/OffloadAPI/init/olInit.cpp
@@ -15,8 +15,20 @@
 
 struct olInitTest : ::testing::Test {};
 
+TEST_F(olInitTest, Success) {
+  ASSERT_SUCCESS(olInit());
+  ASSERT_SUCCESS(olShutDown());
+}
+
 TEST_F(olInitTest, Uninitialized) {
   ASSERT_ERROR(OL_ERRC_UNINITIALIZED,
                olIterateDevices(
                    [](ol_device_handle_t, void *) { return false; }, nullptr));
 }
+
+TEST_F(olInitTest, RepeatedInit) {
+  for (size_t I = 0; I < 10; I++) {
+    ASSERT_SUCCESS(olInit());
+    ASSERT_SUCCESS(olShutDown());
+  }
+}
diff --git a/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp b/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
index a3da334afccac..b444aed9a6bea 100644
--- a/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
+++ b/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
@@ -38,21 +38,19 @@ struct LaunchKernelTestBase : OffloadQueueTest {
   ol_kernel_launch_size_args_t LaunchArgs{};
 };
 
-struct olLaunchKernelTest : LaunchKernelTestBase {
-  void SetUp() override {
-    RETURN_ON_FATAL_FAILURE(LaunchKernelTestBase::SetUpKernel("foo"));
-  }
-};
-OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernelTest);
-
-struct olLaunchKernelNoArgsTest : LaunchKernelTestBase {
-  void SetUp() override {
-    RETURN_ON_FATAL_FAILURE(LaunchKernelTestBase::SetUpKernel("noargs"));
-  }
-};
-OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernelNoArgsTest);
-
-TEST_P(olLaunchKernelTest, Success) {
+#define KERNEL_TEST(NAME, KERNEL)                                              \
+  struct olLaunchKernel##NAME##Test : LaunchKernelTestBase {                   \
+    void SetUp() override { LaunchKernelTestBase::SetUpKernel(#KERNEL); }      \
+  };                                                                           \
+  OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernel##NAME##Test);
+
+KERNEL_TEST(Foo, foo)
+KERNEL_TEST(NoArgs, noargs)
+KERNEL_TEST(LocalMem, localmem)
+KERNEL_TEST(LocalMemReduction, localmem_reduction)
+KERNEL_TEST(LocalMemStatic, localmem_static)
+
+TEST_P(olLaunchKernelFooTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
@@ -80,7 +78,7 @@ TEST_P(olLaunchKernelNoArgsTest, Success) {
   ASSERT_SUCCESS(olWaitQueue(Queue));
 }
 
-TEST_P(olLaunchKernelTest, SuccessSynchronous) {
+TEST_P(olLaunchKernelFooTest, SuccessSynchronous) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
@@ -99,3 +97,74 @@ TEST_P(olLaunchKernelTest, SuccessSynchronous) {
 
   ASSERT_SUCCESS(olMemFree(Mem));
 }
+
+TEST_P(olLaunchKernelLocalMemTest, Success) {
+  LaunchArgs.NumGroups.x = 4;
+  LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x *
+                                sizeof(uint32_t),
+                            &Mem));
+  struct {
+    void *Mem;
+  } Args{Mem};
+
+  ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x; i++)
+    ASSERT_EQ(Data[i], (i % 64) * 2);
+
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
+
+TEST_P(olLaunchKernelLocalMemReductionTest, Success) {
+  LaunchArgs.NumGroups.x = 4;
+  LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));
+  struct {
+    void *Mem;
+  } Args{Mem};
+
+  ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
+    ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);
+
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
+
+TEST_P(olLaunchKernelLocalMemStaticTest, Success) {
+  LaunchArgs.NumGroups.x = 4;
+  LaunchArgs.DynSharedMemory = 0;
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));
+  struct {
+    void *Mem;
+  } Args{Mem};
+
+  ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
+    ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);
+
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index a2cacc8792b15..f62cabee6ea84 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2725,11 +2725,12 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
 #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
   /* Same fields as in the #else branch, but in reverse order */
 #if OMPX_TASKGRAPH
-  unsigned reserved31 : 5;
+  unsigned reserved31 : 4;
   unsigned onced : 1;
 #else
-  unsigned reserved31 : 6;
+  unsigned reserved31 : 5;
 #endif
+  unsigned hidden_helper : 1;
   unsigned target : 1;
   unsigned native : 1;
   unsigned freed : 1;
@@ -2741,7 +2742,7 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
   unsigned task_serial : 1;
   unsigned tasktype : 1;
   unsigned reserved : 8;
-  unsigned hidden_helper : 1;
+  unsigned free_agent_eligible : 1;
   unsigned detachable : 1;
   unsigned priority_specified : 1;
   unsigned proxy : 1;
@@ -2762,7 +2763,8 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
   unsigned priority_specified : 1; /* set if the compiler provides priority
                                       setting for the task */
   unsigned detachable : 1; /* 1 == can detach */
-  unsigned hidden_helper : 1; /* 1 == hidden helper task */
+  unsigned free_agent_eligible : 1; /* set if task can be executed by a
+                                       free-agent thread */
   unsigned reserved : 8; /* reserved for compiler use */
 
   /* Library flags */ /* Total library flags must be 16 bits */
@@ -2780,11 +2782,12 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
   unsigned freed : 1; /* 1==freed, 0==allocated        */
   unsigned native : 1; /* 1==gcc-compiled task, 0==intel */
   unsigned target : 1;
+  unsigned hidden_helper : 1; /* 1 == hidden helper task */
 #if OMPX_TASKGRAPH
   unsigned onced : 1; /* 1==ran once already, 0==never ran, record & replay purposes */
-  unsigned reserved31 : 5; /* reserved for library use */
+  unsigned reserved31 : 4; /* reserved for library use */
 #else
-  unsigned reserved31 : 6; /* reserved for library use */
+  unsigned reserved31 : 5; /* reserved for library use */
 #endif
 #endif
 } kmp_tasking_flags_t;
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 417eceb8ebecc..6afea9b994de4 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -1214,6 +1214,12 @@ void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
   // Reset for next parallel region
   this_thr->th.th_set_proc_bind = proc_bind_default;
 
+  // OpenMP 6.0 12.1.2 requires the num_threads 'strict' modifier to also have
+  // effect when parallel execution is disabled by a corresponding if clause
+  // attached to the parallel directive.
+  if (this_thr->th.th_nt_strict && this_thr->th.th_set_nproc > 1)
+    __kmpc_error(this_thr->th.th_nt_loc, this_thr->th.th_nt_sev,
+                 this_thr->th.th_nt_msg);
   // Reset num_threads for next parallel region
   this_thr->th.th_set_nproc = 0;
 
diff --git a/openmp/runtime/tools/message-converter.py b/openmp/runtime/tools/message-converter.py
index a493d64c1692d..5e2aeba6c99d6 100644
--- a/openmp/runtime/tools/message-converter.py
+++ b/openmp/runtime/tools/message-converter.py
@@ -11,7 +11,6 @@
 #
 
 import argparse
-import datetime
 import os
 import platform
 import re
@@ -188,11 +187,10 @@ def insert_header(f, data, commentChar="//"):
     f.write(
         "{0} Do not edit this file! {0}\n"
         "{0} The file was generated from"
-        " {1} by {2} on {3}. {0}\n\n".format(
+        " {1} by {2}. {0}\n\n".format(
             commentChar,
             os.path.basename(data.filename),
             os.path.basename(__file__),
-            datetime.datetime.now().ctime(),
         )
     )
 
diff --git a/third-party/unittest/googletest/src/gtest.cc b/third-party/unittest/googletest/src/gtest.cc
index 30a5cc3f83a7e..37d380a789831 100644
--- a/third-party/unittest/googletest/src/gtest.cc
+++ b/third-party/unittest/googletest/src/gtest.cc
@@ -43,6 +43,7 @@
 #include <algorithm>
 #include <chrono>  // NOLINT
 #include <cmath>
+#include <csignal>
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
diff --git a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
index b5cd6dbcbd4f9..97c25bcdece7b 100644
--- a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
@@ -106,6 +106,7 @@ cc_library(
         "//llvm:DWARFLinker",
         "//llvm:DWP",
         "//llvm:DebugInfoDWARF",
+        "//llvm:DebugInfoDWARFLowLevel",
         "//llvm:Demangle",
         "//llvm:JITLink",
         "//llvm:MC",
@@ -296,6 +297,7 @@ cc_library(
         "//llvm:BinaryFormat",
         "//llvm:CodeGen",
         "//llvm:DebugInfoDWARF",
+        "//llvm:DebugInfoDWARFLowLevel",
         "//llvm:Demangle",
         "//llvm:ExecutionEngine",
         "//llvm:MC",
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 6f65cfca32943..ac4e3813fbba8 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1829,6 +1829,7 @@ cc_library(
         "//llvm:Target",
         "//llvm:TargetParser",
         "//llvm:TransformUtils",
+        "//llvm:config",
     ],
 )
 
@@ -2344,6 +2345,7 @@ cc_binary(
     stamp = 0,
     deps = [
         ":basic",
+        ":config",
         ":frontend",
         ":interpreter",
         ":lex",
@@ -2352,8 +2354,10 @@ cc_binary(
         "//llvm:Core",
         "//llvm:LineEditor",
         "//llvm:Option",
+        "//llvm:OrcDebugging",
         "//llvm:OrcJIT",
         "//llvm:Support",
+        "//llvm:TargetParser",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index fe6c1257e33b8..d3fc6912cd4e4 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -268,6 +268,19 @@ libc_support_library(
     hdrs = ["hdr/unistd_overlay.h"],
 )
 
+libc_support_library(
+    name = "hdr_wchar_macros",
+    hdrs = ["hdr/wchar_macros.h"],
+    deps = [
+        ":hdr_wchar_overlay",
+    ],
+)
+
+libc_support_library(
+    name = "hdr_wchar_overlay",
+    hdrs = ["hdr/wchar_overlay.h"],
+)
+
 ############################ Type Proxy Header Files ###########################
 
 libc_support_library(
@@ -432,6 +445,22 @@ libc_support_library(
     hdrs = ["hdr/types/struct_msghdr.h"],
 )
 
+libc_support_library(
+    name = "types_wchar_t",
+    hdrs = ["hdr/types/wchar_t.h"],
+    deps = [
+        ":hdr_wchar_overlay",
+    ],
+)
+
+libc_support_library(
+    name = "types_wint_t",
+    hdrs = ["hdr/types/wint_t.h"],
+    deps = [
+        ":hdr_wchar_overlay",
+    ],
+)
+
 ############################### Support libraries ##############################
 
 libc_support_library(
@@ -723,6 +752,7 @@ libc_support_library(
     name = "__support_blockstore",
     hdrs = ["src/__support/blockstore.h"],
     deps = [
+        ":__support_cpp_array",
         ":__support_cpp_new",
         ":__support_libc_assert",
     ],
@@ -1645,6 +1675,17 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_wctype_utils",
+    hdrs = ["src/__support/wctype_utils.h"],
+    deps = [
+        ":__support_cpp_optional",
+        ":__support_macros_attributes",
+        ":__support_macros_config",
+        ":types_wint_t",
+    ],
+)
+
 ########################## externally shared targets ###########################
 
 libc_header_library(
@@ -5685,3 +5726,288 @@ libc_function(
         ":types_struct_timespec",
     ],
 )
+
+############################## wchar targets ###############################
+
+libc_function(
+    name = "btowc",
+    srcs = ["src/wchar/btowc.cpp"],
+    hdrs = ["src/wchar/btowc.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_wctype_utils",
+        ":hdr_wchar_macros",
+        ":types_wint_t",
+    ],
+)
+
+libc_function(
+    name = "wcpcpy",
+    srcs = ["src/wchar/wcpcpy.cpp"],
+    hdrs = ["src/wchar/wcpcpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcpncpy",
+    srcs = ["src/wchar/wcpncpy.cpp"],
+    hdrs = ["src/wchar/wcpncpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscat",
+    srcs = ["src/wchar/wcscat.cpp"],
+    hdrs = ["src/wchar/wcscat.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcschr",
+    srcs = ["src/wchar/wcschr.cpp"],
+    hdrs = ["src/wchar/wcschr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscmp",
+    srcs = ["src/wchar/wcscmp.cpp"],
+    hdrs = ["src/wchar/wcscmp.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscpy",
+    srcs = ["src/wchar/wcscpy.cpp"],
+    hdrs = ["src/wchar/wcscpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscspn",
+    srcs = ["src/wchar/wcscspn.cpp"],
+    hdrs = ["src/wchar/wcscspn.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcslen",
+    srcs = ["src/wchar/wcslen.cpp"],
+    hdrs = ["src/wchar/wcslen.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsncat",
+    srcs = ["src/wchar/wcsncat.cpp"],
+    hdrs = ["src/wchar/wcsncat.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsncmp",
+    srcs = ["src/wchar/wcsncmp.cpp"],
+    hdrs = ["src/wchar/wcsncmp.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsncpy",
+    srcs = ["src/wchar/wcsncpy.cpp"],
+    hdrs = ["src/wchar/wcsncpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcspbrk",
+    srcs = ["src/wchar/wcspbrk.cpp"],
+    hdrs = ["src/wchar/wcspbrk.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsrchr",
+    srcs = ["src/wchar/wcsrchr.cpp"],
+    hdrs = ["src/wchar/wcsrchr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsspn",
+    srcs = ["src/wchar/wcsspn.cpp"],
+    hdrs = ["src/wchar/wcsspn.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsstr",
+    srcs = ["src/wchar/wcsstr.cpp"],
+    hdrs = ["src/wchar/wcsstr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wctob",
+    srcs = ["src/wchar/wctob.cpp"],
+    hdrs = ["src/wchar/wctob.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":__support_wctype_utils",
+        ":hdr_stdio_macros",
+        ":types_wint_t",
+    ],
+)
+
+libc_function(
+    name = "wmemchr",
+    srcs = ["src/wchar/wmemchr.cpp"],
+    hdrs = ["src/wchar/wmemchr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemcmp",
+    srcs = ["src/wchar/wmemcmp.cpp"],
+    hdrs = ["src/wchar/wmemcmp.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemcpy",
+    srcs = ["src/wchar/wmemcpy.cpp"],
+    hdrs = ["src/wchar/wmemcpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemmove",
+    srcs = ["src/wchar/wmemmove.cpp"],
+    hdrs = ["src/wchar/wmemmove.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmempcpy",
+    srcs = ["src/wchar/wmempcpy.cpp"],
+    hdrs = ["src/wchar/wmempcpy.h"],
+    deps = [
+        ":__support_common",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemset",
+    srcs = ["src/wchar/wmemset.cpp"],
+    hdrs = ["src/wchar/wmemset.h"],
+    deps = [
+        ":__support_common",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel
new file mode 100644
index 0000000000000..f94065237ef4f
--- /dev/null
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel
@@ -0,0 +1,224 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Tests for LLVM libc wchar.h functions.
+
+load("//libc/test:libc_test_rules.bzl", "libc_test")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+libc_test(
+    name = "btowc_test",
+    srcs = ["btowc_test.cpp"],
+    deps = [
+        "//libc:btowc",
+        "//libc:hdr_wchar_macros",
+    ],
+)
+
+libc_test(
+    name = "wcpcpy_test",
+    srcs = ["wcpcpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcpcpy",
+    ],
+)
+
+libc_test(
+    name = "wcpncpy_test",
+    srcs = ["wcpncpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcpncpy",
+    ],
+)
+
+libc_test(
+    name = "wcscat_test",
+    srcs = ["wcscat_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcscat",
+    ],
+)
+
+libc_test(
+    name = "wcschr_test",
+    srcs = ["wcschr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcschr",
+    ],
+)
+
+libc_test(
+    name = "wcscmp_test",
+    srcs = ["wcscmp_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcscmp",
+    ],
+)
+
+libc_test(
+    name = "wcscpy_test",
+    srcs = ["wcscpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcscpy",
+    ],
+)
+
+libc_test(
+    name = "wcscspn_test",
+    srcs = ["wcscspn_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wcscspn",
+    ],
+)
+
+libc_test(
+    name = "wcslen_test",
+    srcs = ["wcslen_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wcslen",
+    ],
+)
+
+libc_test(
+    name = "wcsncat_test",
+    srcs = ["wcsncat_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsncat",
+    ],
+)
+
+libc_test(
+    name = "wcsncmp_test",
+    srcs = ["wcsncmp_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsncmp",
+    ],
+)
+
+libc_test(
+    name = "wcsncpy_test",
+    srcs = ["wcsncpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsncpy",
+    ],
+)
+
+libc_test(
+    name = "wcsrchr_test",
+    srcs = ["wcsrchr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsrchr",
+    ],
+)
+
+libc_test(
+    name = "wcspbrk_test",
+    srcs = ["wcspbrk_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcspbrk",
+    ],
+)
+
+libc_test(
+    name = "wcsspn_test",
+    srcs = ["wcsspn_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wcsspn",
+    ],
+)
+
+libc_test(
+    name = "wcsstr_test",
+    srcs = ["wcsstr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsstr",
+    ],
+)
+
+libc_test(
+    name = "wctob_test",
+    srcs = ["wctob_test.cpp"],
+    deps = [
+        "//libc:hdr_stdio_macros",
+        "//libc:wctob",
+    ],
+)
+
+libc_test(
+    name = "wmemchr_test",
+    srcs = ["wmemchr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wmemchr",
+    ],
+)
+
+libc_test(
+    name = "wmemcmp_test",
+    srcs = ["wmemcmp_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemcmp",
+    ],
+)
+
+libc_test(
+    name = "wmemcpy_test",
+    srcs = ["wmemcpy_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemcpy",
+    ],
+)
+
+libc_test(
+    name = "wmemmove_test",
+    srcs = ["wmemmove_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemmove",
+    ],
+)
+
+libc_test(
+    name = "wmempcpy_test",
+    srcs = ["wmempcpy_test.cpp"],
+    deps = [
+        "//libc:wmempcpy",
+    ],
+)
+
+libc_test(
+    name = "wmemset_test",
+    srcs = ["wmemset_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemset",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 6ec7cb5f23776..202f224b07982 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -307,6 +307,7 @@ cc_library(
         "//llvm:BinaryFormat",
         "//llvm:Core",
         "//llvm:DebugInfoDWARF",
+        "//llvm:DebugInfoDWARFLowLevel",
         "//llvm:ExecutionEngine",
         "//llvm:Support",
     ],
@@ -454,6 +455,7 @@ cc_library(
         ":UtilityPrivateHeaders",
         "//llvm:DebugInfo",
         "//llvm:DebugInfoDWARF",
+        "//llvm:DebugInfoDWARFLowLevel",
         "//llvm:Support",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index a2108e24d908e..caf609fd172ea 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -2127,6 +2127,7 @@ cc_library(
         ":PluginProcessUtility",
         "//lldb:Breakpoint",
         "//lldb:Core",
+        "//lldb:Headers",
         "//lldb:Host",
         "//lldb:Symbol",
         "//lldb:SymbolHeaders",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index c8ace44527555..bea07cd44b143 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -269,6 +269,7 @@ cc_library(
         "include/llvm-c/ExternC.h",
         "include/llvm-c/Support.h",
         "include/llvm-c/Types.h",
+        "include/llvm-c/Visibility.h",
         "include/llvm-c/blake3.h",
         "include/llvm/ExecutionEngine/JITSymbol.h",
         "include/llvm/Support/Extension.def",
@@ -493,6 +494,7 @@ cc_library(
         ":DebugInfo",
         ":DebugInfoCodeView",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":DebugInfoPDB",
         ":Demangle",
         ":MC",
@@ -572,6 +574,7 @@ cc_library(
     deps = [
         ":BinaryFormat",
         ":DebugInfo",
+        ":DebugInfoDWARFLowLevel",
         ":MC",
         ":Object",
         ":Support",
@@ -579,6 +582,20 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "DebugInfoDWARFLowLevel",
+    srcs = glob([
+        "lib/DebugInfo/DWARF/LowLevel/*.cpp",
+    ]),
+    hdrs = glob(["include/llvm/DebugInfo/DWARF/LowLevel/*.h"]),
+    copts = llvm_copts,
+    deps = [
+        ":BinaryFormat",
+        ":Support",
+        ":TargetParser",
+    ],
+)
+
 cc_library(
     name = "DebugInfoGSYM",
     srcs = glob([
@@ -764,7 +781,10 @@ gentbl_cc_library(
     tbl_outs = {"include/llvm/IR/RuntimeLibcalls.inc": ["-gen-runtime-libcalls"]},
     tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/RuntimeLibcalls.td",
-    deps = [":IRTdFiles"],
+    deps = [
+        ":IRTdFiles",
+        ":TableGenTdFiles",
+    ],
 )
 
 gentbl_cc_library(
@@ -969,6 +989,8 @@ cc_library(
         "include/llvm/Analysis/SimplifyQuery.h",
         "include/llvm/Analysis/ValueTracking.h",
         "include/llvm/Analysis/WithCache.h",
+        "include/llvm/ProfileData/InstrProf.h",
+        "include/llvm/ProfileData/InstrProfData.inc",
     ] + [":llvm_intrinsics_headers"],
     copts = llvm_copts,
     textual_hdrs = glob([
@@ -1163,6 +1185,7 @@ cc_library(
         ":Core",
         ":DebugInfo",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":Demangle",
         ":Object",
         ":Support",
@@ -1325,6 +1348,12 @@ td_library(
     includes = ["include"],
 )
 
+td_library(
+    name = "TableGenTdFiles",
+    srcs = glob(["include/llvm/TableGen/*.td"]),
+    includes = ["include"],
+)
+
 td_library(
     name = "CommonTargetTdFiles",
     srcs = [":common_target_td_sources"],
@@ -1691,6 +1720,7 @@ cc_library(
     ]),
     copts = llvm_copts,
     deps = [
+        ":BinaryFormat",
         ":Core",
         ":Support",
     ],
@@ -2090,6 +2120,7 @@ cc_library(
         ":Core",
         ":DebugInfoCodeView",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":IPO",
         ":IRPrinter",
         ":Instrumentation",
@@ -3231,6 +3262,7 @@ cc_library(
         ":CodeGenTypes",
         ":DWARFLinkerBase",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":MC",
         ":Support",
         ":Target",
@@ -3249,6 +3281,7 @@ cc_library(
         ":BinaryFormat",
         ":CodeGen",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":Support",
         ":Target",
     ],
@@ -3549,6 +3582,7 @@ cc_library(
         ":DWARFLinkerParallel",
         ":DebugInfo",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":DsymutilTableGen",
         ":MC",
         ":Object",
@@ -3966,6 +4000,7 @@ cc_binary(
         ":BinaryFormat",
         ":DebugInfo",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":MC",
         ":Object",
         ":Support",
@@ -3997,6 +4032,7 @@ cc_binary(
         ":DWARFLinker",
         ":DWARFLinkerParallel",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":DwarfutilOptionsTableGen",
         ":MC",
         ":ObjCopy",
@@ -4614,6 +4650,7 @@ cc_library(
         ":DebugInfo",
         ":DebugInfoBTF",
         ":DebugInfoDWARF",
+        ":DebugInfoDWARFLowLevel",
         ":Debuginfod",
         ":Demangle",
         ":MC",
@@ -5136,6 +5173,11 @@ cc_binary(
     ],
 )
 
+filegroup(
+    name = "gdb-scripts",
+    srcs = glob(["utils/gdb-scripts/*"]),
+)
+
 ################################################################################
 # Begin testonly libraries
 
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 5320dc32163ef..93bbe736c59c2 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -30,6 +30,11 @@ exports_files([
     "utils/textmate/mlir.json",
 ])
 
+filegroup(
+    name = "gdb-scripts",
+    srcs = glob(["utils/gdb-scripts/*"]),
+)
+
 bool_flag(
     name = "enable_cuda",
     build_setting_default = False,
@@ -5071,6 +5076,8 @@ cc_library(
             "lib/Dialect/LLVMIR/IR/VCIX*.h",
             "lib/Dialect/LLVMIR/IR/*X86Vector*.cpp",
             "lib/Dialect/LLVMIR/IR/*X86Vector*.h",
+            "lib/Dialect/LLVMIR/IR/XeVM*.cpp",
+            "lib/Dialect/LLVMIR/IR/XeVM*.h",
         ],
     ),
     hdrs = glob(
@@ -5083,6 +5090,7 @@ cc_library(
             "include/mlir/Dialect/LLVMIR/ROCDL*.h",
             "include/mlir/Dialect/LLVMIR/VCIX*.h",
             "include/mlir/Dialect/LLVMIR/*X86Vector*.h",
+            "include/mlir/Dialect/LLVMIR/XeVM*.h",
         ],
     ),
     includes = ["include"],
@@ -5313,6 +5321,7 @@ cc_library(
         ":InferIntRangeInterface",
         ":InferTypeOpInterface",
         ":InliningUtils",
+        ":MathDialect",
         ":MemRefDialect",
         ":SCFDialect",
         ":SideEffectInterfaces",
@@ -5497,6 +5506,7 @@ cc_library(
         ":GPUDialect",
         ":GPUToGPURuntimeTransforms",
         ":GPUToNVVMTransforms",
+        ":GPUToROCDLTransforms",
         ":GPUTransformOpsIncGen",
         ":GPUTransforms",
         ":IR",
@@ -5504,6 +5514,7 @@ cc_library(
         ":MemRefDialect",
         ":NVGPUDialect",
         ":NVVMDialect",
+        ":ROCDLDialect",
         ":SCFDialect",
         ":Support",
         ":TransformDialect",
@@ -9065,6 +9076,7 @@ cc_library(
         ":X86VectorTransforms",
         ":XeGPUDialect",
         ":XeGPUTransforms",
+        ":XeVMDialect",
     ],
 )
 
@@ -11171,6 +11183,7 @@ cc_library(
         ":Dialect",
         ":DialectUtils",
         ":FuncDialect",
+        ":FuncTransforms",
         ":IR",
         ":InferTypeOpInterface",
         ":InliningUtils",
@@ -12361,6 +12374,7 @@ cc_library(
         ":BufferizationDialect",
         ":BufferizationInterfaces",
         ":ControlFlowDialect",
+        ":DestinationStyleOpInterface",
         ":DialectUtils",
         ":FuncDialect",
         ":GPUDialect",
@@ -13529,3 +13543,60 @@ cc_library(
         "//llvm:Support",
     ],
 )
+
+cc_library(
+    name = "XeVMDialect",
+    srcs = ["lib/Dialect/LLVMIR/IR/XeVMDialect.cpp"],
+    hdrs = ["include/mlir/Dialect/LLVMIR/XeVMDialect.h"],
+    includes = ["include"],
+    deps = [
+        ":BytecodeOpInterface",
+        ":DialectUtils",
+        ":GPUDialect",
+        ":IR",
+        ":LLVMDialect",
+        ":ToLLVMIRTranslation",
+        ":XeVMOpsIncGen",
+        "//llvm:Support",
+    ],
+)
+
+td_library(
+    name = "XeVMTdFiles",
+    srcs = ["include/mlir/Dialect/LLVMIR/XeVMOps.td"],
+    includes = ["include"],
+    deps = [
+        ":GPUOpsTdFiles",
+        ":LLVMOpsTdFiles",
+        ":OpBaseTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "XeVMOpsIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/LLVMIR/XeVMOps.h.inc": ["-gen-op-decls"],
+        "include/mlir/Dialect/LLVMIR/XeVMOps.cpp.inc": ["-gen-op-defs"],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsDialect.h.inc": [
+            "-gen-dialect-decls",
+            "-dialect=xevm",
+        ],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsDialect.cpp.inc": [
+            "-gen-dialect-defs",
+            "-dialect=xevm",
+        ],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsEnums.h.inc": ["-gen-enum-decls"],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsEnums.cpp.inc": ["-gen-enum-defs"],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsAttributes.h.inc": [
+            "-gen-attrdef-decls",
+            "-attrdefs-dialect=xevm",
+        ],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsAttributes.cpp.inc": [
+            "-gen-attrdef-defs",
+            "-attrdefs-dialect=xevm",
+        ],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/LLVMIR/XeVMOps.td",
+    deps = [":XeVMTdFiles"],
+)